LLVM  mainline
ScalarEvolution.cpp
Go to the documentation of this file.
00001 //===- ScalarEvolution.cpp - Scalar Evolution Analysis --------------------===//
00002 //
00003 //                     The LLVM Compiler Infrastructure
00004 //
00005 // This file is distributed under the University of Illinois Open Source
00006 // License. See LICENSE.TXT for details.
00007 //
00008 //===----------------------------------------------------------------------===//
00009 //
00010 // This file contains the implementation of the scalar evolution analysis
00011 // engine, which is used primarily to analyze expressions involving induction
00012 // variables in loops.
00013 //
00014 // There are several aspects to this library.  First is the representation of
00015 // scalar expressions, which are represented as subclasses of the SCEV class.
00016 // These classes are used to represent certain types of subexpressions that we
00017 // can handle. We only create one SCEV of a particular shape, so
00018 // pointer-comparisons for equality are legal.
00019 //
00020 // One important aspect of the SCEV objects is that they are never cyclic, even
00021 // if there is a cycle in the dataflow for an expression (ie, a PHI node).  If
00022 // the PHI node is one of the idioms that we can represent (e.g., a polynomial
00023 // recurrence) then we represent it directly as a recurrence node, otherwise we
00024 // represent it as a SCEVUnknown node.
00025 //
00026 // In addition to being able to represent expressions of various types, we also
00027 // have folders that are used to build the *canonical* representation for a
00028 // particular expression.  These folders are capable of using a variety of
00029 // rewrite rules to simplify the expressions.
00030 //
00031 // Once the folders are defined, we can implement the more interesting
00032 // higher-level code, such as the code that recognizes PHI nodes of various
00033 // types, computes the execution count of a loop, etc.
00034 //
00035 // TODO: We should use these routines and value representations to implement
00036 // dependence analysis!
00037 //
00038 //===----------------------------------------------------------------------===//
00039 //
00040 // There are several good references for the techniques used in this analysis.
00041 //
00042 //  Chains of recurrences -- a method to expedite the evaluation
00043 //  of closed-form functions
00044 //  Olaf Bachmann, Paul S. Wang, Eugene V. Zima
00045 //
00046 //  On computational properties of chains of recurrences
00047 //  Eugene V. Zima
00048 //
00049 //  Symbolic Evaluation of Chains of Recurrences for Loop Optimization
00050 //  Robert A. van Engelen
00051 //
00052 //  Efficient Symbolic Analysis for Optimizing Compilers
00053 //  Robert A. van Engelen
00054 //
00055 //  Using the chains of recurrences algebra for data dependence testing and
00056 //  induction variable substitution
00057 //  MS Thesis, Johnie Birch
00058 //
00059 //===----------------------------------------------------------------------===//
00060 
00061 #include "llvm/Analysis/ScalarEvolution.h"
00062 #include "llvm/ADT/Optional.h"
00063 #include "llvm/ADT/STLExtras.h"
00064 #include "llvm/ADT/SmallPtrSet.h"
00065 #include "llvm/ADT/Statistic.h"
00066 #include "llvm/Analysis/AssumptionCache.h"
00067 #include "llvm/Analysis/ConstantFolding.h"
00068 #include "llvm/Analysis/InstructionSimplify.h"
00069 #include "llvm/Analysis/LoopInfo.h"
00070 #include "llvm/Analysis/ScalarEvolutionExpressions.h"
00071 #include "llvm/Analysis/TargetLibraryInfo.h"
00072 #include "llvm/Analysis/ValueTracking.h"
00073 #include "llvm/IR/ConstantRange.h"
00074 #include "llvm/IR/Constants.h"
00075 #include "llvm/IR/DataLayout.h"
00076 #include "llvm/IR/DerivedTypes.h"
00077 #include "llvm/IR/Dominators.h"
00078 #include "llvm/IR/GetElementPtrTypeIterator.h"
00079 #include "llvm/IR/GlobalAlias.h"
00080 #include "llvm/IR/GlobalVariable.h"
00081 #include "llvm/IR/InstIterator.h"
00082 #include "llvm/IR/Instructions.h"
00083 #include "llvm/IR/LLVMContext.h"
00084 #include "llvm/IR/Metadata.h"
00085 #include "llvm/IR/Operator.h"
00086 #include "llvm/Support/CommandLine.h"
00087 #include "llvm/Support/Debug.h"
00088 #include "llvm/Support/ErrorHandling.h"
00089 #include "llvm/Support/MathExtras.h"
00090 #include "llvm/Support/raw_ostream.h"
00091 #include <algorithm>
00092 using namespace llvm;
00093 
00094 #define DEBUG_TYPE "scalar-evolution"
00095 
00096 STATISTIC(NumArrayLenItCounts,
00097           "Number of trip counts computed with array length");
00098 STATISTIC(NumTripCountsComputed,
00099           "Number of loops with predictable loop counts");
00100 STATISTIC(NumTripCountsNotComputed,
00101           "Number of loops without predictable loop counts");
00102 STATISTIC(NumBruteForceTripCountsComputed,
00103           "Number of loops with trip counts computed by force");
00104 
00105 static cl::opt<unsigned>
00106 MaxBruteForceIterations("scalar-evolution-max-iterations", cl::ReallyHidden,
00107                         cl::desc("Maximum number of iterations SCEV will "
00108                                  "symbolically execute a constant "
00109                                  "derived loop"),
00110                         cl::init(100));
00111 
00112 // FIXME: Enable this with XDEBUG when the test suite is clean.
00113 static cl::opt<bool>
00114 VerifySCEV("verify-scev",
00115            cl::desc("Verify ScalarEvolution's backedge taken counts (slow)"));
00116 
00117 INITIALIZE_PASS_BEGIN(ScalarEvolution, "scalar-evolution",
00118                 "Scalar Evolution Analysis", false, true)
00119 INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)
00120 INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass)
00121 INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
00122 INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
00123 INITIALIZE_PASS_END(ScalarEvolution, "scalar-evolution",
00124                 "Scalar Evolution Analysis", false, true)
00125 char ScalarEvolution::ID = 0;
00126 
00127 //===----------------------------------------------------------------------===//
00128 //                           SCEV class definitions
00129 //===----------------------------------------------------------------------===//
00130 
00131 //===----------------------------------------------------------------------===//
00132 // Implementation of the SCEV class.
00133 //
00134 
00135 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
00136 void SCEV::dump() const {
00137   print(dbgs());
00138   dbgs() << '\n';
00139 }
00140 #endif
00141 
00142 void SCEV::print(raw_ostream &OS) const {
00143   switch (static_cast<SCEVTypes>(getSCEVType())) {
00144   case scConstant:
00145     cast<SCEVConstant>(this)->getValue()->printAsOperand(OS, false);
00146     return;
00147   case scTruncate: {
00148     const SCEVTruncateExpr *Trunc = cast<SCEVTruncateExpr>(this);
00149     const SCEV *Op = Trunc->getOperand();
00150     OS << "(trunc " << *Op->getType() << " " << *Op << " to "
00151        << *Trunc->getType() << ")";
00152     return;
00153   }
00154   case scZeroExtend: {
00155     const SCEVZeroExtendExpr *ZExt = cast<SCEVZeroExtendExpr>(this);
00156     const SCEV *Op = ZExt->getOperand();
00157     OS << "(zext " << *Op->getType() << " " << *Op << " to "
00158        << *ZExt->getType() << ")";
00159     return;
00160   }
00161   case scSignExtend: {
00162     const SCEVSignExtendExpr *SExt = cast<SCEVSignExtendExpr>(this);
00163     const SCEV *Op = SExt->getOperand();
00164     OS << "(sext " << *Op->getType() << " " << *Op << " to "
00165        << *SExt->getType() << ")";
00166     return;
00167   }
00168   case scAddRecExpr: {
00169     const SCEVAddRecExpr *AR = cast<SCEVAddRecExpr>(this);
00170     OS << "{" << *AR->getOperand(0);
00171     for (unsigned i = 1, e = AR->getNumOperands(); i != e; ++i)
00172       OS << ",+," << *AR->getOperand(i);
00173     OS << "}<";
00174     if (AR->getNoWrapFlags(FlagNUW))
00175       OS << "nuw><";
00176     if (AR->getNoWrapFlags(FlagNSW))
00177       OS << "nsw><";
00178     if (AR->getNoWrapFlags(FlagNW) &&
00179         !AR->getNoWrapFlags((NoWrapFlags)(FlagNUW | FlagNSW)))
00180       OS << "nw><";
00181     AR->getLoop()->getHeader()->printAsOperand(OS, /*PrintType=*/false);
00182     OS << ">";
00183     return;
00184   }
00185   case scAddExpr:
00186   case scMulExpr:
00187   case scUMaxExpr:
00188   case scSMaxExpr: {
00189     const SCEVNAryExpr *NAry = cast<SCEVNAryExpr>(this);
00190     const char *OpStr = nullptr;
00191     switch (NAry->getSCEVType()) {
00192     case scAddExpr: OpStr = " + "; break;
00193     case scMulExpr: OpStr = " * "; break;
00194     case scUMaxExpr: OpStr = " umax "; break;
00195     case scSMaxExpr: OpStr = " smax "; break;
00196     }
00197     OS << "(";
00198     for (SCEVNAryExpr::op_iterator I = NAry->op_begin(), E = NAry->op_end();
00199          I != E; ++I) {
00200       OS << **I;
00201       if (std::next(I) != E)
00202         OS << OpStr;
00203     }
00204     OS << ")";
00205     switch (NAry->getSCEVType()) {
00206     case scAddExpr:
00207     case scMulExpr:
00208       if (NAry->getNoWrapFlags(FlagNUW))
00209         OS << "<nuw>";
00210       if (NAry->getNoWrapFlags(FlagNSW))
00211         OS << "<nsw>";
00212     }
00213     return;
00214   }
00215   case scUDivExpr: {
00216     const SCEVUDivExpr *UDiv = cast<SCEVUDivExpr>(this);
00217     OS << "(" << *UDiv->getLHS() << " /u " << *UDiv->getRHS() << ")";
00218     return;
00219   }
00220   case scUnknown: {
00221     const SCEVUnknown *U = cast<SCEVUnknown>(this);
00222     Type *AllocTy;
00223     if (U->isSizeOf(AllocTy)) {
00224       OS << "sizeof(" << *AllocTy << ")";
00225       return;
00226     }
00227     if (U->isAlignOf(AllocTy)) {
00228       OS << "alignof(" << *AllocTy << ")";
00229       return;
00230     }
00231 
00232     Type *CTy;
00233     Constant *FieldNo;
00234     if (U->isOffsetOf(CTy, FieldNo)) {
00235       OS << "offsetof(" << *CTy << ", ";
00236       FieldNo->printAsOperand(OS, false);
00237       OS << ")";
00238       return;
00239     }
00240 
00241     // Otherwise just print it normally.
00242     U->getValue()->printAsOperand(OS, false);
00243     return;
00244   }
00245   case scCouldNotCompute:
00246     OS << "***COULDNOTCOMPUTE***";
00247     return;
00248   }
00249   llvm_unreachable("Unknown SCEV kind!");
00250 }
00251 
00252 Type *SCEV::getType() const {
00253   switch (static_cast<SCEVTypes>(getSCEVType())) {
00254   case scConstant:
00255     return cast<SCEVConstant>(this)->getType();
00256   case scTruncate:
00257   case scZeroExtend:
00258   case scSignExtend:
00259     return cast<SCEVCastExpr>(this)->getType();
00260   case scAddRecExpr:
00261   case scMulExpr:
00262   case scUMaxExpr:
00263   case scSMaxExpr:
00264     return cast<SCEVNAryExpr>(this)->getType();
00265   case scAddExpr:
00266     return cast<SCEVAddExpr>(this)->getType();
00267   case scUDivExpr:
00268     return cast<SCEVUDivExpr>(this)->getType();
00269   case scUnknown:
00270     return cast<SCEVUnknown>(this)->getType();
00271   case scCouldNotCompute:
00272     llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
00273   }
00274   llvm_unreachable("Unknown SCEV kind!");
00275 }
00276 
00277 bool SCEV::isZero() const {
00278   if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(this))
00279     return SC->getValue()->isZero();
00280   return false;
00281 }
00282 
00283 bool SCEV::isOne() const {
00284   if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(this))
00285     return SC->getValue()->isOne();
00286   return false;
00287 }
00288 
00289 bool SCEV::isAllOnesValue() const {
00290   if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(this))
00291     return SC->getValue()->isAllOnesValue();
00292   return false;
00293 }
00294 
00295 /// isNonConstantNegative - Return true if the specified scev is negated, but
00296 /// not a constant.
00297 bool SCEV::isNonConstantNegative() const {
00298   const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(this);
00299   if (!Mul) return false;
00300 
00301   // If there is a constant factor, it will be first.
00302   const SCEVConstant *SC = dyn_cast<SCEVConstant>(Mul->getOperand(0));
00303   if (!SC) return false;
00304 
00305   // Return true if the value is negative, this matches things like (-42 * V).
00306   return SC->getValue()->getValue().isNegative();
00307 }
00308 
00309 SCEVCouldNotCompute::SCEVCouldNotCompute() :
00310   SCEV(FoldingSetNodeIDRef(), scCouldNotCompute) {}
00311 
00312 bool SCEVCouldNotCompute::classof(const SCEV *S) {
00313   return S->getSCEVType() == scCouldNotCompute;
00314 }
00315 
00316 const SCEV *ScalarEvolution::getConstant(ConstantInt *V) {
00317   FoldingSetNodeID ID;
00318   ID.AddInteger(scConstant);
00319   ID.AddPointer(V);
00320   void *IP = nullptr;
00321   if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S;
00322   SCEV *S = new (SCEVAllocator) SCEVConstant(ID.Intern(SCEVAllocator), V);
00323   UniqueSCEVs.InsertNode(S, IP);
00324   return S;
00325 }
00326 
00327 const SCEV *ScalarEvolution::getConstant(const APInt &Val) {
00328   return getConstant(ConstantInt::get(getContext(), Val));
00329 }
00330 
00331 const SCEV *
00332 ScalarEvolution::getConstant(Type *Ty, uint64_t V, bool isSigned) {
00333   IntegerType *ITy = cast<IntegerType>(getEffectiveSCEVType(Ty));
00334   return getConstant(ConstantInt::get(ITy, V, isSigned));
00335 }
00336 
00337 SCEVCastExpr::SCEVCastExpr(const FoldingSetNodeIDRef ID,
00338                            unsigned SCEVTy, const SCEV *op, Type *ty)
00339   : SCEV(ID, SCEVTy), Op(op), Ty(ty) {}
00340 
00341 SCEVTruncateExpr::SCEVTruncateExpr(const FoldingSetNodeIDRef ID,
00342                                    const SCEV *op, Type *ty)
00343   : SCEVCastExpr(ID, scTruncate, op, ty) {
00344   assert((Op->getType()->isIntegerTy() || Op->getType()->isPointerTy()) &&
00345          (Ty->isIntegerTy() || Ty->isPointerTy()) &&
00346          "Cannot truncate non-integer value!");
00347 }
00348 
00349 SCEVZeroExtendExpr::SCEVZeroExtendExpr(const FoldingSetNodeIDRef ID,
00350                                        const SCEV *op, Type *ty)
00351   : SCEVCastExpr(ID, scZeroExtend, op, ty) {
00352   assert((Op->getType()->isIntegerTy() || Op->getType()->isPointerTy()) &&
00353          (Ty->isIntegerTy() || Ty->isPointerTy()) &&
00354          "Cannot zero extend non-integer value!");
00355 }
00356 
00357 SCEVSignExtendExpr::SCEVSignExtendExpr(const FoldingSetNodeIDRef ID,
00358                                        const SCEV *op, Type *ty)
00359   : SCEVCastExpr(ID, scSignExtend, op, ty) {
00360   assert((Op->getType()->isIntegerTy() || Op->getType()->isPointerTy()) &&
00361          (Ty->isIntegerTy() || Ty->isPointerTy()) &&
00362          "Cannot sign extend non-integer value!");
00363 }
00364 
00365 void SCEVUnknown::deleted() {
00366   // Clear this SCEVUnknown from various maps.
00367   SE->forgetMemoizedResults(this);
00368 
00369   // Remove this SCEVUnknown from the uniquing map.
00370   SE->UniqueSCEVs.RemoveNode(this);
00371 
00372   // Release the value.
00373   setValPtr(nullptr);
00374 }
00375 
00376 void SCEVUnknown::allUsesReplacedWith(Value *New) {
00377   // Clear this SCEVUnknown from various maps.
00378   SE->forgetMemoizedResults(this);
00379 
00380   // Remove this SCEVUnknown from the uniquing map.
00381   SE->UniqueSCEVs.RemoveNode(this);
00382 
00383   // Update this SCEVUnknown to point to the new value. This is needed
00384   // because there may still be outstanding SCEVs which still point to
00385   // this SCEVUnknown.
00386   setValPtr(New);
00387 }
00388 
00389 bool SCEVUnknown::isSizeOf(Type *&AllocTy) const {
00390   if (ConstantExpr *VCE = dyn_cast<ConstantExpr>(getValue()))
00391     if (VCE->getOpcode() == Instruction::PtrToInt)
00392       if (ConstantExpr *CE = dyn_cast<ConstantExpr>(VCE->getOperand(0)))
00393         if (CE->getOpcode() == Instruction::GetElementPtr &&
00394             CE->getOperand(0)->isNullValue() &&
00395             CE->getNumOperands() == 2)
00396           if (ConstantInt *CI = dyn_cast<ConstantInt>(CE->getOperand(1)))
00397             if (CI->isOne()) {
00398               AllocTy = cast<PointerType>(CE->getOperand(0)->getType())
00399                                  ->getElementType();
00400               return true;
00401             }
00402 
00403   return false;
00404 }
00405 
00406 bool SCEVUnknown::isAlignOf(Type *&AllocTy) const {
00407   if (ConstantExpr *VCE = dyn_cast<ConstantExpr>(getValue()))
00408     if (VCE->getOpcode() == Instruction::PtrToInt)
00409       if (ConstantExpr *CE = dyn_cast<ConstantExpr>(VCE->getOperand(0)))
00410         if (CE->getOpcode() == Instruction::GetElementPtr &&
00411             CE->getOperand(0)->isNullValue()) {
00412           Type *Ty =
00413             cast<PointerType>(CE->getOperand(0)->getType())->getElementType();
00414           if (StructType *STy = dyn_cast<StructType>(Ty))
00415             if (!STy->isPacked() &&
00416                 CE->getNumOperands() == 3 &&
00417                 CE->getOperand(1)->isNullValue()) {
00418               if (ConstantInt *CI = dyn_cast<ConstantInt>(CE->getOperand(2)))
00419                 if (CI->isOne() &&
00420                     STy->getNumElements() == 2 &&
00421                     STy->getElementType(0)->isIntegerTy(1)) {
00422                   AllocTy = STy->getElementType(1);
00423                   return true;
00424                 }
00425             }
00426         }
00427 
00428   return false;
00429 }
00430 
00431 bool SCEVUnknown::isOffsetOf(Type *&CTy, Constant *&FieldNo) const {
00432   if (ConstantExpr *VCE = dyn_cast<ConstantExpr>(getValue()))
00433     if (VCE->getOpcode() == Instruction::PtrToInt)
00434       if (ConstantExpr *CE = dyn_cast<ConstantExpr>(VCE->getOperand(0)))
00435         if (CE->getOpcode() == Instruction::GetElementPtr &&
00436             CE->getNumOperands() == 3 &&
00437             CE->getOperand(0)->isNullValue() &&
00438             CE->getOperand(1)->isNullValue()) {
00439           Type *Ty =
00440             cast<PointerType>(CE->getOperand(0)->getType())->getElementType();
00441           // Ignore vector types here so that ScalarEvolutionExpander doesn't
00442           // emit getelementptrs that index into vectors.
00443           if (Ty->isStructTy() || Ty->isArrayTy()) {
00444             CTy = Ty;
00445             FieldNo = CE->getOperand(2);
00446             return true;
00447           }
00448         }
00449 
00450   return false;
00451 }
00452 
00453 //===----------------------------------------------------------------------===//
00454 //                               SCEV Utilities
00455 //===----------------------------------------------------------------------===//
00456 
00457 namespace {
00458   /// SCEVComplexityCompare - Return true if the complexity of the LHS is less
00459   /// than the complexity of the RHS.  This comparator is used to canonicalize
00460   /// expressions.
00461   class SCEVComplexityCompare {
00462     const LoopInfo *const LI;
00463   public:
00464     explicit SCEVComplexityCompare(const LoopInfo *li) : LI(li) {}
00465 
00466     // Return true or false if LHS is less than, or at least RHS, respectively.
00467     bool operator()(const SCEV *LHS, const SCEV *RHS) const {
00468       return compare(LHS, RHS) < 0;
00469     }
00470 
00471     // Return negative, zero, or positive, if LHS is less than, equal to, or
00472     // greater than RHS, respectively. A three-way result allows recursive
00473     // comparisons to be more efficient.
00474     int compare(const SCEV *LHS, const SCEV *RHS) const {
00475       // Fast-path: SCEVs are uniqued so we can do a quick equality check.
00476       if (LHS == RHS)
00477         return 0;
00478 
00479       // Primarily, sort the SCEVs by their getSCEVType().
00480       unsigned LType = LHS->getSCEVType(), RType = RHS->getSCEVType();
00481       if (LType != RType)
00482         return (int)LType - (int)RType;
00483 
00484       // Aside from the getSCEVType() ordering, the particular ordering
00485       // isn't very important except that it's beneficial to be consistent,
00486       // so that (a + b) and (b + a) don't end up as different expressions.
00487       switch (static_cast<SCEVTypes>(LType)) {
00488       case scUnknown: {
00489         const SCEVUnknown *LU = cast<SCEVUnknown>(LHS);
00490         const SCEVUnknown *RU = cast<SCEVUnknown>(RHS);
00491 
00492         // Sort SCEVUnknown values with some loose heuristics. TODO: This is
00493         // not as complete as it could be.
00494         const Value *LV = LU->getValue(), *RV = RU->getValue();
00495 
00496         // Order pointer values after integer values. This helps SCEVExpander
00497         // form GEPs.
00498         bool LIsPointer = LV->getType()->isPointerTy(),
00499              RIsPointer = RV->getType()->isPointerTy();
00500         if (LIsPointer != RIsPointer)
00501           return (int)LIsPointer - (int)RIsPointer;
00502 
00503         // Compare getValueID values.
00504         unsigned LID = LV->getValueID(),
00505                  RID = RV->getValueID();
00506         if (LID != RID)
00507           return (int)LID - (int)RID;
00508 
00509         // Sort arguments by their position.
00510         if (const Argument *LA = dyn_cast<Argument>(LV)) {
00511           const Argument *RA = cast<Argument>(RV);
00512           unsigned LArgNo = LA->getArgNo(), RArgNo = RA->getArgNo();
00513           return (int)LArgNo - (int)RArgNo;
00514         }
00515 
00516         // For instructions, compare their loop depth, and their operand
00517         // count.  This is pretty loose.
00518         if (const Instruction *LInst = dyn_cast<Instruction>(LV)) {
00519           const Instruction *RInst = cast<Instruction>(RV);
00520 
00521           // Compare loop depths.
00522           const BasicBlock *LParent = LInst->getParent(),
00523                            *RParent = RInst->getParent();
00524           if (LParent != RParent) {
00525             unsigned LDepth = LI->getLoopDepth(LParent),
00526                      RDepth = LI->getLoopDepth(RParent);
00527             if (LDepth != RDepth)
00528               return (int)LDepth - (int)RDepth;
00529           }
00530 
00531           // Compare the number of operands.
00532           unsigned LNumOps = LInst->getNumOperands(),
00533                    RNumOps = RInst->getNumOperands();
00534           return (int)LNumOps - (int)RNumOps;
00535         }
00536 
00537         return 0;
00538       }
00539 
00540       case scConstant: {
00541         const SCEVConstant *LC = cast<SCEVConstant>(LHS);
00542         const SCEVConstant *RC = cast<SCEVConstant>(RHS);
00543 
00544         // Compare constant values.
00545         const APInt &LA = LC->getValue()->getValue();
00546         const APInt &RA = RC->getValue()->getValue();
00547         unsigned LBitWidth = LA.getBitWidth(), RBitWidth = RA.getBitWidth();
00548         if (LBitWidth != RBitWidth)
00549           return (int)LBitWidth - (int)RBitWidth;
00550         return LA.ult(RA) ? -1 : 1;
00551       }
00552 
00553       case scAddRecExpr: {
00554         const SCEVAddRecExpr *LA = cast<SCEVAddRecExpr>(LHS);
00555         const SCEVAddRecExpr *RA = cast<SCEVAddRecExpr>(RHS);
00556 
00557         // Compare addrec loop depths.
00558         const Loop *LLoop = LA->getLoop(), *RLoop = RA->getLoop();
00559         if (LLoop != RLoop) {
00560           unsigned LDepth = LLoop->getLoopDepth(),
00561                    RDepth = RLoop->getLoopDepth();
00562           if (LDepth != RDepth)
00563             return (int)LDepth - (int)RDepth;
00564         }
00565 
00566         // Addrec complexity grows with operand count.
00567         unsigned LNumOps = LA->getNumOperands(), RNumOps = RA->getNumOperands();
00568         if (LNumOps != RNumOps)
00569           return (int)LNumOps - (int)RNumOps;
00570 
00571         // Lexicographically compare.
00572         for (unsigned i = 0; i != LNumOps; ++i) {
00573           long X = compare(LA->getOperand(i), RA->getOperand(i));
00574           if (X != 0)
00575             return X;
00576         }
00577 
00578         return 0;
00579       }
00580 
00581       case scAddExpr:
00582       case scMulExpr:
00583       case scSMaxExpr:
00584       case scUMaxExpr: {
00585         const SCEVNAryExpr *LC = cast<SCEVNAryExpr>(LHS);
00586         const SCEVNAryExpr *RC = cast<SCEVNAryExpr>(RHS);
00587 
00588         // Lexicographically compare n-ary expressions.
00589         unsigned LNumOps = LC->getNumOperands(), RNumOps = RC->getNumOperands();
00590         if (LNumOps != RNumOps)
00591           return (int)LNumOps - (int)RNumOps;
00592 
00593         for (unsigned i = 0; i != LNumOps; ++i) {
00594           if (i >= RNumOps)
00595             return 1;
00596           long X = compare(LC->getOperand(i), RC->getOperand(i));
00597           if (X != 0)
00598             return X;
00599         }
00600         return (int)LNumOps - (int)RNumOps;
00601       }
00602 
00603       case scUDivExpr: {
00604         const SCEVUDivExpr *LC = cast<SCEVUDivExpr>(LHS);
00605         const SCEVUDivExpr *RC = cast<SCEVUDivExpr>(RHS);
00606 
00607         // Lexicographically compare udiv expressions.
00608         long X = compare(LC->getLHS(), RC->getLHS());
00609         if (X != 0)
00610           return X;
00611         return compare(LC->getRHS(), RC->getRHS());
00612       }
00613 
00614       case scTruncate:
00615       case scZeroExtend:
00616       case scSignExtend: {
00617         const SCEVCastExpr *LC = cast<SCEVCastExpr>(LHS);
00618         const SCEVCastExpr *RC = cast<SCEVCastExpr>(RHS);
00619 
00620         // Compare cast expressions by operand.
00621         return compare(LC->getOperand(), RC->getOperand());
00622       }
00623 
00624       case scCouldNotCompute:
00625         llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
00626       }
00627       llvm_unreachable("Unknown SCEV kind!");
00628     }
00629   };
00630 }
00631 
00632 /// GroupByComplexity - Given a list of SCEV objects, order them by their
00633 /// complexity, and group objects of the same complexity together by value.
00634 /// When this routine is finished, we know that any duplicates in the vector are
00635 /// consecutive and that complexity is monotonically increasing.
00636 ///
00637 /// Note that we go take special precautions to ensure that we get deterministic
00638 /// results from this routine.  In other words, we don't want the results of
00639 /// this to depend on where the addresses of various SCEV objects happened to
00640 /// land in memory.
00641 ///
00642 static void GroupByComplexity(SmallVectorImpl<const SCEV *> &Ops,
00643                               LoopInfo *LI) {
00644   if (Ops.size() < 2) return;  // Noop
00645   if (Ops.size() == 2) {
00646     // This is the common case, which also happens to be trivially simple.
00647     // Special case it.
00648     const SCEV *&LHS = Ops[0], *&RHS = Ops[1];
00649     if (SCEVComplexityCompare(LI)(RHS, LHS))
00650       std::swap(LHS, RHS);
00651     return;
00652   }
00653 
00654   // Do the rough sort by complexity.
00655   std::stable_sort(Ops.begin(), Ops.end(), SCEVComplexityCompare(LI));
00656 
00657   // Now that we are sorted by complexity, group elements of the same
00658   // complexity.  Note that this is, at worst, N^2, but the vector is likely to
00659   // be extremely short in practice.  Note that we take this approach because we
00660   // do not want to depend on the addresses of the objects we are grouping.
00661   for (unsigned i = 0, e = Ops.size(); i != e-2; ++i) {
00662     const SCEV *S = Ops[i];
00663     unsigned Complexity = S->getSCEVType();
00664 
00665     // If there are any objects of the same complexity and same value as this
00666     // one, group them.
00667     for (unsigned j = i+1; j != e && Ops[j]->getSCEVType() == Complexity; ++j) {
00668       if (Ops[j] == S) { // Found a duplicate.
00669         // Move it to immediately after i'th element.
00670         std::swap(Ops[i+1], Ops[j]);
00671         ++i;   // no need to rescan it.
00672         if (i == e-2) return;  // Done!
00673       }
00674     }
00675   }
00676 }
00677 
00678 namespace {
00679 struct FindSCEVSize {
00680   int Size;
00681   FindSCEVSize() : Size(0) {}
00682 
00683   bool follow(const SCEV *S) {
00684     ++Size;
00685     // Keep looking at all operands of S.
00686     return true;
00687   }
00688   bool isDone() const {
00689     return false;
00690   }
00691 };
00692 }
00693 
00694 // Returns the size of the SCEV S.
00695 static inline int sizeOfSCEV(const SCEV *S) {
00696   FindSCEVSize F;
00697   SCEVTraversal<FindSCEVSize> ST(F);
00698   ST.visitAll(S);
00699   return F.Size;
00700 }
00701 
00702 namespace {
00703 
00704 struct SCEVDivision : public SCEVVisitor<SCEVDivision, void> {
00705 public:
00706   // Computes the Quotient and Remainder of the division of Numerator by
00707   // Denominator.
00708   static void divide(ScalarEvolution &SE, const SCEV *Numerator,
00709                      const SCEV *Denominator, const SCEV **Quotient,
00710                      const SCEV **Remainder) {
00711     assert(Numerator && Denominator && "Uninitialized SCEV");
00712 
00713     SCEVDivision D(SE, Numerator, Denominator);
00714 
00715     // Check for the trivial case here to avoid having to check for it in the
00716     // rest of the code.
00717     if (Numerator == Denominator) {
00718       *Quotient = D.One;
00719       *Remainder = D.Zero;
00720       return;
00721     }
00722 
00723     if (Numerator->isZero()) {
00724       *Quotient = D.Zero;
00725       *Remainder = D.Zero;
00726       return;
00727     }
00728 
00729     // A simple case when N/1. The quotient is N.
00730     if (Denominator->isOne()) {
00731       *Quotient = Numerator;
00732       *Remainder = D.Zero;
00733       return;
00734     }
00735 
00736     // Split the Denominator when it is a product.
00737     if (const SCEVMulExpr *T = dyn_cast<const SCEVMulExpr>(Denominator)) {
00738       const SCEV *Q, *R;
00739       *Quotient = Numerator;
00740       for (const SCEV *Op : T->operands()) {
00741         divide(SE, *Quotient, Op, &Q, &R);
00742         *Quotient = Q;
00743 
00744         // Bail out when the Numerator is not divisible by one of the terms of
00745         // the Denominator.
00746         if (!R->isZero()) {
00747           *Quotient = D.Zero;
00748           *Remainder = Numerator;
00749           return;
00750         }
00751       }
00752       *Remainder = D.Zero;
00753       return;
00754     }
00755 
00756     D.visit(Numerator);
00757     *Quotient = D.Quotient;
00758     *Remainder = D.Remainder;
00759   }
00760 
00761   // Except in the trivial case described above, we do not know how to divide
00762   // Expr by Denominator for the following functions with empty implementation.
00763   void visitTruncateExpr(const SCEVTruncateExpr *Numerator) {}
00764   void visitZeroExtendExpr(const SCEVZeroExtendExpr *Numerator) {}
00765   void visitSignExtendExpr(const SCEVSignExtendExpr *Numerator) {}
00766   void visitUDivExpr(const SCEVUDivExpr *Numerator) {}
00767   void visitSMaxExpr(const SCEVSMaxExpr *Numerator) {}
00768   void visitUMaxExpr(const SCEVUMaxExpr *Numerator) {}
00769   void visitUnknown(const SCEVUnknown *Numerator) {}
00770   void visitCouldNotCompute(const SCEVCouldNotCompute *Numerator) {}
00771 
00772   void visitConstant(const SCEVConstant *Numerator) {
00773     if (const SCEVConstant *D = dyn_cast<SCEVConstant>(Denominator)) {
00774       APInt NumeratorVal = Numerator->getValue()->getValue();
00775       APInt DenominatorVal = D->getValue()->getValue();
00776       uint32_t NumeratorBW = NumeratorVal.getBitWidth();
00777       uint32_t DenominatorBW = DenominatorVal.getBitWidth();
00778 
00779       if (NumeratorBW > DenominatorBW)
00780         DenominatorVal = DenominatorVal.sext(NumeratorBW);
00781       else if (NumeratorBW < DenominatorBW)
00782         NumeratorVal = NumeratorVal.sext(DenominatorBW);
00783 
00784       APInt QuotientVal(NumeratorVal.getBitWidth(), 0);
00785       APInt RemainderVal(NumeratorVal.getBitWidth(), 0);
00786       APInt::sdivrem(NumeratorVal, DenominatorVal, QuotientVal, RemainderVal);
00787       Quotient = SE.getConstant(QuotientVal);
00788       Remainder = SE.getConstant(RemainderVal);
00789       return;
00790     }
00791   }
00792 
00793   void visitAddRecExpr(const SCEVAddRecExpr *Numerator) {
00794     const SCEV *StartQ, *StartR, *StepQ, *StepR;
00795     assert(Numerator->isAffine() && "Numerator should be affine");
00796     divide(SE, Numerator->getStart(), Denominator, &StartQ, &StartR);
00797     divide(SE, Numerator->getStepRecurrence(SE), Denominator, &StepQ, &StepR);
00798     // Bail out if the types do not match.
00799     Type *Ty = Denominator->getType();
00800     if (Ty != StartQ->getType() || Ty != StartR->getType() ||
00801         Ty != StepQ->getType() || Ty != StepR->getType()) {
00802       Quotient = Zero;
00803       Remainder = Numerator;
00804       return;
00805     }
00806     Quotient = SE.getAddRecExpr(StartQ, StepQ, Numerator->getLoop(),
00807                                 Numerator->getNoWrapFlags());
00808     Remainder = SE.getAddRecExpr(StartR, StepR, Numerator->getLoop(),
00809                                  Numerator->getNoWrapFlags());
00810   }
00811 
00812   void visitAddExpr(const SCEVAddExpr *Numerator) {
00813     SmallVector<const SCEV *, 2> Qs, Rs;
00814     Type *Ty = Denominator->getType();
00815 
00816     for (const SCEV *Op : Numerator->operands()) {
00817       const SCEV *Q, *R;
00818       divide(SE, Op, Denominator, &Q, &R);
00819 
00820       // Bail out if types do not match.
00821       if (Ty != Q->getType() || Ty != R->getType()) {
00822         Quotient = Zero;
00823         Remainder = Numerator;
00824         return;
00825       }
00826 
00827       Qs.push_back(Q);
00828       Rs.push_back(R);
00829     }
00830 
00831     if (Qs.size() == 1) {
00832       Quotient = Qs[0];
00833       Remainder = Rs[0];
00834       return;
00835     }
00836 
00837     Quotient = SE.getAddExpr(Qs);
00838     Remainder = SE.getAddExpr(Rs);
00839   }
00840 
00841   void visitMulExpr(const SCEVMulExpr *Numerator) {
00842     SmallVector<const SCEV *, 2> Qs;
00843     Type *Ty = Denominator->getType();
00844 
00845     bool FoundDenominatorTerm = false;
00846     for (const SCEV *Op : Numerator->operands()) {
00847       // Bail out if types do not match.
00848       if (Ty != Op->getType()) {
00849         Quotient = Zero;
00850         Remainder = Numerator;
00851         return;
00852       }
00853 
00854       if (FoundDenominatorTerm) {
00855         Qs.push_back(Op);
00856         continue;
00857       }
00858 
00859       // Check whether Denominator divides one of the product operands.
00860       const SCEV *Q, *R;
00861       divide(SE, Op, Denominator, &Q, &R);
00862       if (!R->isZero()) {
00863         Qs.push_back(Op);
00864         continue;
00865       }
00866 
00867       // Bail out if types do not match.
00868       if (Ty != Q->getType()) {
00869         Quotient = Zero;
00870         Remainder = Numerator;
00871         return;
00872       }
00873 
00874       FoundDenominatorTerm = true;
00875       Qs.push_back(Q);
00876     }
00877 
00878     if (FoundDenominatorTerm) {
00879       Remainder = Zero;
00880       if (Qs.size() == 1)
00881         Quotient = Qs[0];
00882       else
00883         Quotient = SE.getMulExpr(Qs);
00884       return;
00885     }
00886 
00887     if (!isa<SCEVUnknown>(Denominator)) {
00888       Quotient = Zero;
00889       Remainder = Numerator;
00890       return;
00891     }
00892 
00893     // The Remainder is obtained by replacing Denominator by 0 in Numerator.
00894     ValueToValueMap RewriteMap;
00895     RewriteMap[cast<SCEVUnknown>(Denominator)->getValue()] =
00896         cast<SCEVConstant>(Zero)->getValue();
00897     Remainder = SCEVParameterRewriter::rewrite(Numerator, SE, RewriteMap, true);
00898 
00899     if (Remainder->isZero()) {
00900       // The Quotient is obtained by replacing Denominator by 1 in Numerator.
00901       RewriteMap[cast<SCEVUnknown>(Denominator)->getValue()] =
00902           cast<SCEVConstant>(One)->getValue();
00903       Quotient =
00904           SCEVParameterRewriter::rewrite(Numerator, SE, RewriteMap, true);
00905       return;
00906     }
00907 
00908     // Quotient is (Numerator - Remainder) divided by Denominator.
00909     const SCEV *Q, *R;
00910     const SCEV *Diff = SE.getMinusSCEV(Numerator, Remainder);
00911     if (sizeOfSCEV(Diff) > sizeOfSCEV(Numerator)) {
00912       // This SCEV does not seem to simplify: fail the division here.
00913       Quotient = Zero;
00914       Remainder = Numerator;
00915       return;
00916     }
00917     divide(SE, Diff, Denominator, &Q, &R);
00918     assert(R == Zero &&
00919            "(Numerator - Remainder) should evenly divide Denominator");
00920     Quotient = Q;
00921   }
00922 
00923 private:
00924   SCEVDivision(ScalarEvolution &S, const SCEV *Numerator,
00925                const SCEV *Denominator)
00926       : SE(S), Denominator(Denominator) {
00927     Zero = SE.getConstant(Denominator->getType(), 0);
00928     One = SE.getConstant(Denominator->getType(), 1);
00929 
00930     // By default, we don't know how to divide Expr by Denominator.
00931     // Providing the default here simplifies the rest of the code.
00932     Quotient = Zero;
00933     Remainder = Numerator;
00934   }
00935 
00936   ScalarEvolution &SE;
00937   const SCEV *Denominator, *Quotient, *Remainder, *Zero, *One;
00938 };
00939 
00940 }
00941 
00942 //===----------------------------------------------------------------------===//
00943 //                      Simple SCEV method implementations
00944 //===----------------------------------------------------------------------===//
00945 
00946 /// BinomialCoefficient - Compute BC(It, K).  The result has width W.
00947 /// Assume, K > 0.
00948 static const SCEV *BinomialCoefficient(const SCEV *It, unsigned K,
00949                                        ScalarEvolution &SE,
00950                                        Type *ResultTy) {
00951   // Handle the simplest case efficiently.
00952   if (K == 1)
00953     return SE.getTruncateOrZeroExtend(It, ResultTy);
00954 
00955   // We are using the following formula for BC(It, K):
00956   //
00957   //   BC(It, K) = (It * (It - 1) * ... * (It - K + 1)) / K!
00958   //
00959   // Suppose, W is the bitwidth of the return value.  We must be prepared for
00960   // overflow.  Hence, we must assure that the result of our computation is
00961   // equal to the accurate one modulo 2^W.  Unfortunately, division isn't
00962   // safe in modular arithmetic.
00963   //
00964   // However, this code doesn't use exactly that formula; the formula it uses
00965   // is something like the following, where T is the number of factors of 2 in
00966   // K! (i.e. trailing zeros in the binary representation of K!), and ^ is
00967   // exponentiation:
00968   //
00969   //   BC(It, K) = (It * (It - 1) * ... * (It - K + 1)) / 2^T / (K! / 2^T)
00970   //
00971   // This formula is trivially equivalent to the previous formula.  However,
00972   // this formula can be implemented much more efficiently.  The trick is that
00973   // K! / 2^T is odd, and exact division by an odd number *is* safe in modular
00974   // arithmetic.  To do exact division in modular arithmetic, all we have
00975   // to do is multiply by the inverse.  Therefore, this step can be done at
00976   // width W.
00977   //
00978   // The next issue is how to safely do the division by 2^T.  The way this
00979   // is done is by doing the multiplication step at a width of at least W + T
00980   // bits.  This way, the bottom W+T bits of the product are accurate. Then,
00981   // when we perform the division by 2^T (which is equivalent to a right shift
00982   // by T), the bottom W bits are accurate.  Extra bits are okay; they'll get
00983   // truncated out after the division by 2^T.
00984   //
00985   // In comparison to just directly using the first formula, this technique
00986   // is much more efficient; using the first formula requires W * K bits,
00987   // but this formula less than W + K bits. Also, the first formula requires
00988   // a division step, whereas this formula only requires multiplies and shifts.
00989   //
00990   // It doesn't matter whether the subtraction step is done in the calculation
00991   // width or the input iteration count's width; if the subtraction overflows,
00992   // the result must be zero anyway.  We prefer here to do it in the width of
00993   // the induction variable because it helps a lot for certain cases; CodeGen
00994   // isn't smart enough to ignore the overflow, which leads to much less
00995   // efficient code if the width of the subtraction is wider than the native
00996   // register width.
00997   //
00998   // (It's possible to not widen at all by pulling out factors of 2 before
00999   // the multiplication; for example, K=2 can be calculated as
01000   // It/2*(It+(It*INT_MIN/INT_MIN)+-1). However, it requires
01001   // extra arithmetic, so it's not an obvious win, and it gets
01002   // much more complicated for K > 3.)
01003 
01004   // Protection from insane SCEVs; this bound is conservative,
01005   // but it probably doesn't matter.
01006   if (K > 1000)
01007     return SE.getCouldNotCompute();
01008 
01009   unsigned W = SE.getTypeSizeInBits(ResultTy);
01010 
01011   // Calculate K! / 2^T and T; we divide out the factors of two before
01012   // multiplying for calculating K! / 2^T to avoid overflow.
01013   // Other overflow doesn't matter because we only care about the bottom
01014   // W bits of the result.
01015   APInt OddFactorial(W, 1);
01016   unsigned T = 1;
01017   for (unsigned i = 3; i <= K; ++i) {
01018     APInt Mult(W, i);
01019     unsigned TwoFactors = Mult.countTrailingZeros();
01020     T += TwoFactors;
01021     Mult = Mult.lshr(TwoFactors);
01022     OddFactorial *= Mult;
01023   }
01024 
01025   // We need at least W + T bits for the multiplication step
01026   unsigned CalculationBits = W + T;
01027 
01028   // Calculate 2^T, at width T+W.
01029   APInt DivFactor = APInt::getOneBitSet(CalculationBits, T);
01030 
01031   // Calculate the multiplicative inverse of K! / 2^T;
01032   // this multiplication factor will perform the exact division by
01033   // K! / 2^T.
01034   APInt Mod = APInt::getSignedMinValue(W+1);
01035   APInt MultiplyFactor = OddFactorial.zext(W+1);
01036   MultiplyFactor = MultiplyFactor.multiplicativeInverse(Mod);
01037   MultiplyFactor = MultiplyFactor.trunc(W);
01038 
01039   // Calculate the product, at width T+W
01040   IntegerType *CalculationTy = IntegerType::get(SE.getContext(),
01041                                                       CalculationBits);
01042   const SCEV *Dividend = SE.getTruncateOrZeroExtend(It, CalculationTy);
01043   for (unsigned i = 1; i != K; ++i) {
01044     const SCEV *S = SE.getMinusSCEV(It, SE.getConstant(It->getType(), i));
01045     Dividend = SE.getMulExpr(Dividend,
01046                              SE.getTruncateOrZeroExtend(S, CalculationTy));
01047   }
01048 
01049   // Divide by 2^T
01050   const SCEV *DivResult = SE.getUDivExpr(Dividend, SE.getConstant(DivFactor));
01051 
01052   // Truncate the result, and divide by K! / 2^T.
01053 
01054   return SE.getMulExpr(SE.getConstant(MultiplyFactor),
01055                        SE.getTruncateOrZeroExtend(DivResult, ResultTy));
01056 }
01057 
01058 /// evaluateAtIteration - Return the value of this chain of recurrences at
01059 /// the specified iteration number.  We can evaluate this recurrence by
01060 /// multiplying each element in the chain by the binomial coefficient
01061 /// corresponding to it.  In other words, we can evaluate {A,+,B,+,C,+,D} as:
01062 ///
01063 ///   A*BC(It, 0) + B*BC(It, 1) + C*BC(It, 2) + D*BC(It, 3)
01064 ///
01065 /// where BC(It, k) stands for binomial coefficient.
01066 ///
01067 const SCEV *SCEVAddRecExpr::evaluateAtIteration(const SCEV *It,
01068                                                 ScalarEvolution &SE) const {
01069   const SCEV *Result = getStart();
01070   for (unsigned i = 1, e = getNumOperands(); i != e; ++i) {
01071     // The computation is correct in the face of overflow provided that the
01072     // multiplication is performed _after_ the evaluation of the binomial
01073     // coefficient.
01074     const SCEV *Coeff = BinomialCoefficient(It, i, SE, getType());
01075     if (isa<SCEVCouldNotCompute>(Coeff))
01076       return Coeff;
01077 
01078     Result = SE.getAddExpr(Result, SE.getMulExpr(getOperand(i), Coeff));
01079   }
01080   return Result;
01081 }
01082 
01083 //===----------------------------------------------------------------------===//
01084 //                    SCEV Expression folder implementations
01085 //===----------------------------------------------------------------------===//
01086 
01087 const SCEV *ScalarEvolution::getTruncateExpr(const SCEV *Op,
01088                                              Type *Ty) {
01089   assert(getTypeSizeInBits(Op->getType()) > getTypeSizeInBits(Ty) &&
01090          "This is not a truncating conversion!");
01091   assert(isSCEVable(Ty) &&
01092          "This is not a conversion to a SCEVable type!");
01093   Ty = getEffectiveSCEVType(Ty);
01094 
01095   FoldingSetNodeID ID;
01096   ID.AddInteger(scTruncate);
01097   ID.AddPointer(Op);
01098   ID.AddPointer(Ty);
01099   void *IP = nullptr;
01100   if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S;
01101 
01102   // Fold if the operand is constant.
01103   if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(Op))
01104     return getConstant(
01105       cast<ConstantInt>(ConstantExpr::getTrunc(SC->getValue(), Ty)));
01106 
01107   // trunc(trunc(x)) --> trunc(x)
01108   if (const SCEVTruncateExpr *ST = dyn_cast<SCEVTruncateExpr>(Op))
01109     return getTruncateExpr(ST->getOperand(), Ty);
01110 
01111   // trunc(sext(x)) --> sext(x) if widening or trunc(x) if narrowing
01112   if (const SCEVSignExtendExpr *SS = dyn_cast<SCEVSignExtendExpr>(Op))
01113     return getTruncateOrSignExtend(SS->getOperand(), Ty);
01114 
01115   // trunc(zext(x)) --> zext(x) if widening or trunc(x) if narrowing
01116   if (const SCEVZeroExtendExpr *SZ = dyn_cast<SCEVZeroExtendExpr>(Op))
01117     return getTruncateOrZeroExtend(SZ->getOperand(), Ty);
01118 
01119   // trunc(x1+x2+...+xN) --> trunc(x1)+trunc(x2)+...+trunc(xN) if we can
01120   // eliminate all the truncates, or we replace other casts with truncates.
01121   if (const SCEVAddExpr *SA = dyn_cast<SCEVAddExpr>(Op)) {
01122     SmallVector<const SCEV *, 4> Operands;
01123     bool hasTrunc = false;
01124     for (unsigned i = 0, e = SA->getNumOperands(); i != e && !hasTrunc; ++i) {
01125       const SCEV *S = getTruncateExpr(SA->getOperand(i), Ty);
01126       if (!isa<SCEVCastExpr>(SA->getOperand(i)))
01127         hasTrunc = isa<SCEVTruncateExpr>(S);
01128       Operands.push_back(S);
01129     }
01130     if (!hasTrunc)
01131       return getAddExpr(Operands);
01132     UniqueSCEVs.FindNodeOrInsertPos(ID, IP);  // Mutates IP, returns NULL.
01133   }
01134 
01135   // trunc(x1*x2*...*xN) --> trunc(x1)*trunc(x2)*...*trunc(xN) if we can
01136   // eliminate all the truncates, or we replace other casts with truncates.
01137   if (const SCEVMulExpr *SM = dyn_cast<SCEVMulExpr>(Op)) {
01138     SmallVector<const SCEV *, 4> Operands;
01139     bool hasTrunc = false;
01140     for (unsigned i = 0, e = SM->getNumOperands(); i != e && !hasTrunc; ++i) {
01141       const SCEV *S = getTruncateExpr(SM->getOperand(i), Ty);
01142       if (!isa<SCEVCastExpr>(SM->getOperand(i)))
01143         hasTrunc = isa<SCEVTruncateExpr>(S);
01144       Operands.push_back(S);
01145     }
01146     if (!hasTrunc)
01147       return getMulExpr(Operands);
01148     UniqueSCEVs.FindNodeOrInsertPos(ID, IP);  // Mutates IP, returns NULL.
01149   }
01150 
01151   // If the input value is a chrec scev, truncate the chrec's operands.
01152   if (const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(Op)) {
01153     SmallVector<const SCEV *, 4> Operands;
01154     for (unsigned i = 0, e = AddRec->getNumOperands(); i != e; ++i)
01155       Operands.push_back(getTruncateExpr(AddRec->getOperand(i), Ty));
01156     return getAddRecExpr(Operands, AddRec->getLoop(), SCEV::FlagAnyWrap);
01157   }
01158 
01159   // The cast wasn't folded; create an explicit cast node. We can reuse
01160   // the existing insert position since if we get here, we won't have
01161   // made any changes which would invalidate it.
01162   SCEV *S = new (SCEVAllocator) SCEVTruncateExpr(ID.Intern(SCEVAllocator),
01163                                                  Op, Ty);
01164   UniqueSCEVs.InsertNode(S, IP);
01165   return S;
01166 }
01167 
01168 // Get the limit of a recurrence such that incrementing by Step cannot cause
01169 // signed overflow as long as the value of the recurrence within the
01170 // loop does not exceed this limit before incrementing.
01171 static const SCEV *getSignedOverflowLimitForStep(const SCEV *Step,
01172                                                  ICmpInst::Predicate *Pred,
01173                                                  ScalarEvolution *SE) {
01174   unsigned BitWidth = SE->getTypeSizeInBits(Step->getType());
01175   if (SE->isKnownPositive(Step)) {
01176     *Pred = ICmpInst::ICMP_SLT;
01177     return SE->getConstant(APInt::getSignedMinValue(BitWidth) -
01178                            SE->getSignedRange(Step).getSignedMax());
01179   }
01180   if (SE->isKnownNegative(Step)) {
01181     *Pred = ICmpInst::ICMP_SGT;
01182     return SE->getConstant(APInt::getSignedMaxValue(BitWidth) -
01183                            SE->getSignedRange(Step).getSignedMin());
01184   }
01185   return nullptr;
01186 }
01187 
01188 // Get the limit of a recurrence such that incrementing by Step cannot cause
01189 // unsigned overflow as long as the value of the recurrence within the loop does
01190 // not exceed this limit before incrementing.
01191 static const SCEV *getUnsignedOverflowLimitForStep(const SCEV *Step,
01192                                                    ICmpInst::Predicate *Pred,
01193                                                    ScalarEvolution *SE) {
01194   unsigned BitWidth = SE->getTypeSizeInBits(Step->getType());
01195   *Pred = ICmpInst::ICMP_ULT;
01196 
01197   return SE->getConstant(APInt::getMinValue(BitWidth) -
01198                          SE->getUnsignedRange(Step).getUnsignedMax());
01199 }
01200 
01201 namespace {
01202 
01203 struct ExtendOpTraitsBase {
01204   typedef const SCEV *(ScalarEvolution::*GetExtendExprTy)(const SCEV *, Type *);
01205 };
01206 
01207 // Used to make code generic over signed and unsigned overflow.
01208 template <typename ExtendOp> struct ExtendOpTraits {
01209   // Members present:
01210   //
01211   // static const SCEV::NoWrapFlags WrapType;
01212   //
01213   // static const ExtendOpTraitsBase::GetExtendExprTy GetExtendExpr;
01214   //
01215   // static const SCEV *getOverflowLimitForStep(const SCEV *Step,
01216   //                                           ICmpInst::Predicate *Pred,
01217   //                                           ScalarEvolution *SE);
01218 };
01219 
01220 template <>
01221 struct ExtendOpTraits<SCEVSignExtendExpr> : public ExtendOpTraitsBase {
01222   static const SCEV::NoWrapFlags WrapType = SCEV::FlagNSW;
01223 
01224   static const GetExtendExprTy GetExtendExpr;
01225 
01226   static const SCEV *getOverflowLimitForStep(const SCEV *Step,
01227                                              ICmpInst::Predicate *Pred,
01228                                              ScalarEvolution *SE) {
01229     return getSignedOverflowLimitForStep(Step, Pred, SE);
01230   }
01231 };
01232 
01233 const ExtendOpTraitsBase::GetExtendExprTy ExtendOpTraits<
01234     SCEVSignExtendExpr>::GetExtendExpr = &ScalarEvolution::getSignExtendExpr;
01235 
01236 template <>
01237 struct ExtendOpTraits<SCEVZeroExtendExpr> : public ExtendOpTraitsBase {
01238   static const SCEV::NoWrapFlags WrapType = SCEV::FlagNUW;
01239 
01240   static const GetExtendExprTy GetExtendExpr;
01241 
01242   static const SCEV *getOverflowLimitForStep(const SCEV *Step,
01243                                              ICmpInst::Predicate *Pred,
01244                                              ScalarEvolution *SE) {
01245     return getUnsignedOverflowLimitForStep(Step, Pred, SE);
01246   }
01247 };
01248 
01249 const ExtendOpTraitsBase::GetExtendExprTy ExtendOpTraits<
01250     SCEVZeroExtendExpr>::GetExtendExpr = &ScalarEvolution::getZeroExtendExpr;
01251 }
01252 
01253 // The recurrence AR has been shown to have no signed/unsigned wrap or something
01254 // close to it. Typically, if we can prove NSW/NUW for AR, then we can just as
01255 // easily prove NSW/NUW for its preincrement or postincrement sibling. This
01256 // allows normalizing a sign/zero extended AddRec as such: {sext/zext(Step +
01257 // Start),+,Step} => {(Step + sext/zext(Start),+,Step} As a result, the
01258 // expression "Step + sext/zext(PreIncAR)" is congruent with
01259 // "sext/zext(PostIncAR)"
01260 template <typename ExtendOpTy>
01261 static const SCEV *getPreStartForExtend(const SCEVAddRecExpr *AR, Type *Ty,
01262                                         ScalarEvolution *SE) {
01263   auto WrapType = ExtendOpTraits<ExtendOpTy>::WrapType;
01264   auto GetExtendExpr = ExtendOpTraits<ExtendOpTy>::GetExtendExpr;
01265 
01266   const Loop *L = AR->getLoop();
01267   const SCEV *Start = AR->getStart();
01268   const SCEV *Step = AR->getStepRecurrence(*SE);
01269 
01270   // Check for a simple looking step prior to loop entry.
01271   const SCEVAddExpr *SA = dyn_cast<SCEVAddExpr>(Start);
01272   if (!SA)
01273     return nullptr;
01274 
01275   // Create an AddExpr for "PreStart" after subtracting Step. Full SCEV
01276   // subtraction is expensive. For this purpose, perform a quick and dirty
01277   // difference, by checking for Step in the operand list.
01278   SmallVector<const SCEV *, 4> DiffOps;
01279   for (const SCEV *Op : SA->operands())
01280     if (Op != Step)
01281       DiffOps.push_back(Op);
01282 
01283   if (DiffOps.size() == SA->getNumOperands())
01284     return nullptr;
01285 
01286   // Try to prove `WrapType` (SCEV::FlagNSW or SCEV::FlagNUW) on `PreStart` +
01287   // `Step`:
01288 
01289   // 1. NSW/NUW flags on the step increment.
01290   const SCEV *PreStart = SE->getAddExpr(DiffOps, SA->getNoWrapFlags());
01291   const SCEVAddRecExpr *PreAR = dyn_cast<SCEVAddRecExpr>(
01292       SE->getAddRecExpr(PreStart, Step, L, SCEV::FlagAnyWrap));
01293 
01294   // "{S,+,X} is <nsw>/<nuw>" and "the backedge is taken at least once" implies
01295   // "S+X does not sign/unsign-overflow".
01296   //
01297 
01298   const SCEV *BECount = SE->getBackedgeTakenCount(L);
01299   if (PreAR && PreAR->getNoWrapFlags(WrapType) &&
01300       !isa<SCEVCouldNotCompute>(BECount) && SE->isKnownPositive(BECount))
01301     return PreStart;
01302 
01303   // 2. Direct overflow check on the step operation's expression.
01304   unsigned BitWidth = SE->getTypeSizeInBits(AR->getType());
01305   Type *WideTy = IntegerType::get(SE->getContext(), BitWidth * 2);
01306   const SCEV *OperandExtendedStart =
01307       SE->getAddExpr((SE->*GetExtendExpr)(PreStart, WideTy),
01308                      (SE->*GetExtendExpr)(Step, WideTy));
01309   if ((SE->*GetExtendExpr)(Start, WideTy) == OperandExtendedStart) {
01310     if (PreAR && AR->getNoWrapFlags(WrapType)) {
01311       // If we know `AR` == {`PreStart`+`Step`,+,`Step`} is `WrapType` (FlagNSW
01312       // or FlagNUW) and that `PreStart` + `Step` is `WrapType` too, then
01313       // `PreAR` == {`PreStart`,+,`Step`} is also `WrapType`.  Cache this fact.
01314       const_cast<SCEVAddRecExpr *>(PreAR)->setNoWrapFlags(WrapType);
01315     }
01316     return PreStart;
01317   }
01318 
01319   // 3. Loop precondition.
01320   ICmpInst::Predicate Pred;
01321   const SCEV *OverflowLimit =
01322       ExtendOpTraits<ExtendOpTy>::getOverflowLimitForStep(Step, &Pred, SE);
01323 
01324   if (OverflowLimit &&
01325       SE->isLoopEntryGuardedByCond(L, Pred, PreStart, OverflowLimit)) {
01326     return PreStart;
01327   }
01328   return nullptr;
01329 }
01330 
01331 // Get the normalized zero or sign extended expression for this AddRec's Start.
01332 template <typename ExtendOpTy>
01333 static const SCEV *getExtendAddRecStart(const SCEVAddRecExpr *AR, Type *Ty,
01334                                         ScalarEvolution *SE) {
01335   auto GetExtendExpr = ExtendOpTraits<ExtendOpTy>::GetExtendExpr;
01336 
01337   const SCEV *PreStart = getPreStartForExtend<ExtendOpTy>(AR, Ty, SE);
01338   if (!PreStart)
01339     return (SE->*GetExtendExpr)(AR->getStart(), Ty);
01340 
01341   return SE->getAddExpr((SE->*GetExtendExpr)(AR->getStepRecurrence(*SE), Ty),
01342                         (SE->*GetExtendExpr)(PreStart, Ty));
01343 }
01344 
01345 // Try to prove away overflow by looking at "nearby" add recurrences.  A
01346 // motivating example for this rule: if we know `{0,+,4}` is `ult` `-1` and it
01347 // does not itself wrap then we can conclude that `{1,+,4}` is `nuw`.
01348 //
01349 // Formally:
01350 //
01351 //     {S,+,X} == {S-T,+,X} + T
01352 //  => Ext({S,+,X}) == Ext({S-T,+,X} + T)
01353 //
01354 // If ({S-T,+,X} + T) does not overflow  ... (1)
01355 //
01356 //  RHS == Ext({S-T,+,X} + T) == Ext({S-T,+,X}) + Ext(T)
01357 //
01358 // If {S-T,+,X} does not overflow  ... (2)
01359 //
01360 //  RHS == Ext({S-T,+,X}) + Ext(T) == {Ext(S-T),+,Ext(X)} + Ext(T)
01361 //      == {Ext(S-T)+Ext(T),+,Ext(X)}
01362 //
01363 // If (S-T)+T does not overflow  ... (3)
01364 //
01365 //  RHS == {Ext(S-T)+Ext(T),+,Ext(X)} == {Ext(S-T+T),+,Ext(X)}
01366 //      == {Ext(S),+,Ext(X)} == LHS
01367 //
01368 // Thus, if (1), (2) and (3) are true for some T, then
01369 //   Ext({S,+,X}) == {Ext(S),+,Ext(X)}
01370 //
01371 // (3) is implied by (1) -- "(S-T)+T does not overflow" is simply "({S-T,+,X}+T)
01372 // does not overflow" restricted to the 0th iteration.  Therefore we only need
01373 // to check for (1) and (2).
01374 //
01375 // In the current context, S is `Start`, X is `Step`, Ext is `ExtendOpTy` and T
01376 // is `Delta` (defined below).
01377 //
01378 template <typename ExtendOpTy>
01379 bool ScalarEvolution::proveNoWrapByVaryingStart(const SCEV *Start,
01380                                                 const SCEV *Step,
01381                                                 const Loop *L) {
01382   auto WrapType = ExtendOpTraits<ExtendOpTy>::WrapType;
01383 
01384   // We restrict `Start` to a constant to prevent SCEV from spending too much
01385   // time here.  It is correct (but more expensive) to continue with a
01386   // non-constant `Start` and do a general SCEV subtraction to compute
01387   // `PreStart` below.
01388   //
01389   const SCEVConstant *StartC = dyn_cast<SCEVConstant>(Start);
01390   if (!StartC)
01391     return false;
01392 
01393   APInt StartAI = StartC->getValue()->getValue();
01394 
01395   for (unsigned Delta : {-2, -1, 1, 2}) {
01396     const SCEV *PreStart = getConstant(StartAI - Delta);
01397 
01398     // Give up if we don't already have the add recurrence we need because
01399     // actually constructing an add recurrence is relatively expensive.
01400     const SCEVAddRecExpr *PreAR = [&]() {
01401       FoldingSetNodeID ID;
01402       ID.AddInteger(scAddRecExpr);
01403       ID.AddPointer(PreStart);
01404       ID.AddPointer(Step);
01405       ID.AddPointer(L);
01406       void *IP = nullptr;
01407       return static_cast<SCEVAddRecExpr *>(
01408           this->UniqueSCEVs.FindNodeOrInsertPos(ID, IP));
01409     }();
01410 
01411     if (PreAR && PreAR->getNoWrapFlags(WrapType)) {  // proves (2)
01412       const SCEV *DeltaS = getConstant(StartC->getType(), Delta);
01413       ICmpInst::Predicate Pred = ICmpInst::BAD_ICMP_PREDICATE;
01414       const SCEV *Limit = ExtendOpTraits<ExtendOpTy>::getOverflowLimitForStep(
01415           DeltaS, &Pred, this);
01416       if (Limit && isKnownPredicate(Pred, PreAR, Limit))  // proves (1)
01417         return true;
01418     }
01419   }
01420 
01421   return false;
01422 }
01423 
01424 const SCEV *ScalarEvolution::getZeroExtendExpr(const SCEV *Op,
01425                                                Type *Ty) {
01426   assert(getTypeSizeInBits(Op->getType()) < getTypeSizeInBits(Ty) &&
01427          "This is not an extending conversion!");
01428   assert(isSCEVable(Ty) &&
01429          "This is not a conversion to a SCEVable type!");
01430   Ty = getEffectiveSCEVType(Ty);
01431 
01432   // Fold if the operand is constant.
01433   if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(Op))
01434     return getConstant(
01435       cast<ConstantInt>(ConstantExpr::getZExt(SC->getValue(), Ty)));
01436 
01437   // zext(zext(x)) --> zext(x)
01438   if (const SCEVZeroExtendExpr *SZ = dyn_cast<SCEVZeroExtendExpr>(Op))
01439     return getZeroExtendExpr(SZ->getOperand(), Ty);
01440 
01441   // Before doing any expensive analysis, check to see if we've already
01442   // computed a SCEV for this Op and Ty.
01443   FoldingSetNodeID ID;
01444   ID.AddInteger(scZeroExtend);
01445   ID.AddPointer(Op);
01446   ID.AddPointer(Ty);
01447   void *IP = nullptr;
01448   if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S;
01449 
01450   // zext(trunc(x)) --> zext(x) or x or trunc(x)
01451   if (const SCEVTruncateExpr *ST = dyn_cast<SCEVTruncateExpr>(Op)) {
01452     // It's possible the bits taken off by the truncate were all zero bits. If
01453     // so, we should be able to simplify this further.
01454     const SCEV *X = ST->getOperand();
01455     ConstantRange CR = getUnsignedRange(X);
01456     unsigned TruncBits = getTypeSizeInBits(ST->getType());
01457     unsigned NewBits = getTypeSizeInBits(Ty);
01458     if (CR.truncate(TruncBits).zeroExtend(NewBits).contains(
01459             CR.zextOrTrunc(NewBits)))
01460       return getTruncateOrZeroExtend(X, Ty);
01461   }
01462 
01463   // If the input value is a chrec scev, and we can prove that the value
01464   // did not overflow the old, smaller, value, we can zero extend all of the
01465   // operands (often constants).  This allows analysis of something like
01466   // this:  for (unsigned char X = 0; X < 100; ++X) { int Y = X; }
01467   if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Op))
01468     if (AR->isAffine()) {
01469       const SCEV *Start = AR->getStart();
01470       const SCEV *Step = AR->getStepRecurrence(*this);
01471       unsigned BitWidth = getTypeSizeInBits(AR->getType());
01472       const Loop *L = AR->getLoop();
01473 
01474       // If we have special knowledge that this addrec won't overflow,
01475       // we don't need to do any further analysis.
01476       if (AR->getNoWrapFlags(SCEV::FlagNUW))
01477         return getAddRecExpr(
01478             getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty, this),
01479             getZeroExtendExpr(Step, Ty), L, AR->getNoWrapFlags());
01480 
01481       // Check whether the backedge-taken count is SCEVCouldNotCompute.
01482       // Note that this serves two purposes: It filters out loops that are
01483       // simply not analyzable, and it covers the case where this code is
01484       // being called from within backedge-taken count analysis, such that
01485       // attempting to ask for the backedge-taken count would likely result
01486       // in infinite recursion. In the later case, the analysis code will
01487       // cope with a conservative value, and it will take care to purge
01488       // that value once it has finished.
01489       const SCEV *MaxBECount = getMaxBackedgeTakenCount(L);
01490       if (!isa<SCEVCouldNotCompute>(MaxBECount)) {
01491         // Manually compute the final value for AR, checking for
01492         // overflow.
01493 
01494         // Check whether the backedge-taken count can be losslessly casted to
01495         // the addrec's type. The count is always unsigned.
01496         const SCEV *CastedMaxBECount =
01497           getTruncateOrZeroExtend(MaxBECount, Start->getType());
01498         const SCEV *RecastedMaxBECount =
01499           getTruncateOrZeroExtend(CastedMaxBECount, MaxBECount->getType());
01500         if (MaxBECount == RecastedMaxBECount) {
01501           Type *WideTy = IntegerType::get(getContext(), BitWidth * 2);
01502           // Check whether Start+Step*MaxBECount has no unsigned overflow.
01503           const SCEV *ZMul = getMulExpr(CastedMaxBECount, Step);
01504           const SCEV *ZAdd = getZeroExtendExpr(getAddExpr(Start, ZMul), WideTy);
01505           const SCEV *WideStart = getZeroExtendExpr(Start, WideTy);
01506           const SCEV *WideMaxBECount =
01507             getZeroExtendExpr(CastedMaxBECount, WideTy);
01508           const SCEV *OperandExtendedAdd =
01509             getAddExpr(WideStart,
01510                        getMulExpr(WideMaxBECount,
01511                                   getZeroExtendExpr(Step, WideTy)));
01512           if (ZAdd == OperandExtendedAdd) {
01513             // Cache knowledge of AR NUW, which is propagated to this AddRec.
01514             const_cast<SCEVAddRecExpr *>(AR)->setNoWrapFlags(SCEV::FlagNUW);
01515             // Return the expression with the addrec on the outside.
01516             return getAddRecExpr(
01517                 getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty, this),
01518                 getZeroExtendExpr(Step, Ty), L, AR->getNoWrapFlags());
01519           }
01520           // Similar to above, only this time treat the step value as signed.
01521           // This covers loops that count down.
01522           OperandExtendedAdd =
01523             getAddExpr(WideStart,
01524                        getMulExpr(WideMaxBECount,
01525                                   getSignExtendExpr(Step, WideTy)));
01526           if (ZAdd == OperandExtendedAdd) {
01527             // Cache knowledge of AR NW, which is propagated to this AddRec.
01528             // Negative step causes unsigned wrap, but it still can't self-wrap.
01529             const_cast<SCEVAddRecExpr *>(AR)->setNoWrapFlags(SCEV::FlagNW);
01530             // Return the expression with the addrec on the outside.
01531             return getAddRecExpr(
01532                 getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty, this),
01533                 getSignExtendExpr(Step, Ty), L, AR->getNoWrapFlags());
01534           }
01535         }
01536 
01537         // If the backedge is guarded by a comparison with the pre-inc value
01538         // the addrec is safe. Also, if the entry is guarded by a comparison
01539         // with the start value and the backedge is guarded by a comparison
01540         // with the post-inc value, the addrec is safe.
01541         if (isKnownPositive(Step)) {
01542           const SCEV *N = getConstant(APInt::getMinValue(BitWidth) -
01543                                       getUnsignedRange(Step).getUnsignedMax());
01544           if (isLoopBackedgeGuardedByCond(L, ICmpInst::ICMP_ULT, AR, N) ||
01545               (isLoopEntryGuardedByCond(L, ICmpInst::ICMP_ULT, Start, N) &&
01546                isLoopBackedgeGuardedByCond(L, ICmpInst::ICMP_ULT,
01547                                            AR->getPostIncExpr(*this), N))) {
01548             // Cache knowledge of AR NUW, which is propagated to this AddRec.
01549             const_cast<SCEVAddRecExpr *>(AR)->setNoWrapFlags(SCEV::FlagNUW);
01550             // Return the expression with the addrec on the outside.
01551             return getAddRecExpr(
01552                 getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty, this),
01553                 getZeroExtendExpr(Step, Ty), L, AR->getNoWrapFlags());
01554           }
01555         } else if (isKnownNegative(Step)) {
01556           const SCEV *N = getConstant(APInt::getMaxValue(BitWidth) -
01557                                       getSignedRange(Step).getSignedMin());
01558           if (isLoopBackedgeGuardedByCond(L, ICmpInst::ICMP_UGT, AR, N) ||
01559               (isLoopEntryGuardedByCond(L, ICmpInst::ICMP_UGT, Start, N) &&
01560                isLoopBackedgeGuardedByCond(L, ICmpInst::ICMP_UGT,
01561                                            AR->getPostIncExpr(*this), N))) {
01562             // Cache knowledge of AR NW, which is propagated to this AddRec.
01563             // Negative step causes unsigned wrap, but it still can't self-wrap.
01564             const_cast<SCEVAddRecExpr *>(AR)->setNoWrapFlags(SCEV::FlagNW);
01565             // Return the expression with the addrec on the outside.
01566             return getAddRecExpr(
01567                 getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty, this),
01568                 getSignExtendExpr(Step, Ty), L, AR->getNoWrapFlags());
01569           }
01570         }
01571       }
01572 
01573       if (proveNoWrapByVaryingStart<SCEVZeroExtendExpr>(Start, Step, L)) {
01574         const_cast<SCEVAddRecExpr *>(AR)->setNoWrapFlags(SCEV::FlagNUW);
01575         return getAddRecExpr(
01576             getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty, this),
01577             getZeroExtendExpr(Step, Ty), L, AR->getNoWrapFlags());
01578       }
01579     }
01580 
01581   // The cast wasn't folded; create an explicit cast node.
01582   // Recompute the insert position, as it may have been invalidated.
01583   if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S;
01584   SCEV *S = new (SCEVAllocator) SCEVZeroExtendExpr(ID.Intern(SCEVAllocator),
01585                                                    Op, Ty);
01586   UniqueSCEVs.InsertNode(S, IP);
01587   return S;
01588 }
01589 
01590 const SCEV *ScalarEvolution::getSignExtendExpr(const SCEV *Op,
01591                                                Type *Ty) {
01592   assert(getTypeSizeInBits(Op->getType()) < getTypeSizeInBits(Ty) &&
01593          "This is not an extending conversion!");
01594   assert(isSCEVable(Ty) &&
01595          "This is not a conversion to a SCEVable type!");
01596   Ty = getEffectiveSCEVType(Ty);
01597 
01598   // Fold if the operand is constant.
01599   if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(Op))
01600     return getConstant(
01601       cast<ConstantInt>(ConstantExpr::getSExt(SC->getValue(), Ty)));
01602 
01603   // sext(sext(x)) --> sext(x)
01604   if (const SCEVSignExtendExpr *SS = dyn_cast<SCEVSignExtendExpr>(Op))
01605     return getSignExtendExpr(SS->getOperand(), Ty);
01606 
01607   // sext(zext(x)) --> zext(x)
01608   if (const SCEVZeroExtendExpr *SZ = dyn_cast<SCEVZeroExtendExpr>(Op))
01609     return getZeroExtendExpr(SZ->getOperand(), Ty);
01610 
01611   // Before doing any expensive analysis, check to see if we've already
01612   // computed a SCEV for this Op and Ty.
01613   FoldingSetNodeID ID;
01614   ID.AddInteger(scSignExtend);
01615   ID.AddPointer(Op);
01616   ID.AddPointer(Ty);
01617   void *IP = nullptr;
01618   if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S;
01619 
01620   // If the input value is provably positive, build a zext instead.
01621   if (isKnownNonNegative(Op))
01622     return getZeroExtendExpr(Op, Ty);
01623 
01624   // sext(trunc(x)) --> sext(x) or x or trunc(x)
01625   if (const SCEVTruncateExpr *ST = dyn_cast<SCEVTruncateExpr>(Op)) {
01626     // It's possible the bits taken off by the truncate were all sign bits. If
01627     // so, we should be able to simplify this further.
01628     const SCEV *X = ST->getOperand();
01629     ConstantRange CR = getSignedRange(X);
01630     unsigned TruncBits = getTypeSizeInBits(ST->getType());
01631     unsigned NewBits = getTypeSizeInBits(Ty);
01632     if (CR.truncate(TruncBits).signExtend(NewBits).contains(
01633             CR.sextOrTrunc(NewBits)))
01634       return getTruncateOrSignExtend(X, Ty);
01635   }
01636 
01637   // sext(C1 + (C2 * x)) --> C1 + sext(C2 * x) if C1 < C2
01638   if (auto SA = dyn_cast<SCEVAddExpr>(Op)) {
01639     if (SA->getNumOperands() == 2) {
01640       auto SC1 = dyn_cast<SCEVConstant>(SA->getOperand(0));
01641       auto SMul = dyn_cast<SCEVMulExpr>(SA->getOperand(1));
01642       if (SMul && SC1) {
01643         if (auto SC2 = dyn_cast<SCEVConstant>(SMul->getOperand(0))) {
01644           const APInt &C1 = SC1->getValue()->getValue();
01645           const APInt &C2 = SC2->getValue()->getValue();
01646           if (C1.isStrictlyPositive() && C2.isStrictlyPositive() &&
01647               C2.ugt(C1) && C2.isPowerOf2())
01648             return getAddExpr(getSignExtendExpr(SC1, Ty),
01649                               getSignExtendExpr(SMul, Ty));
01650         }
01651       }
01652     }
01653   }
01654   // If the input value is a chrec scev, and we can prove that the value
01655   // did not overflow the old, smaller, value, we can sign extend all of the
01656   // operands (often constants).  This allows analysis of something like
01657   // this:  for (signed char X = 0; X < 100; ++X) { int Y = X; }
01658   if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Op))
01659     if (AR->isAffine()) {
01660       const SCEV *Start = AR->getStart();
01661       const SCEV *Step = AR->getStepRecurrence(*this);
01662       unsigned BitWidth = getTypeSizeInBits(AR->getType());
01663       const Loop *L = AR->getLoop();
01664 
01665       // If we have special knowledge that this addrec won't overflow,
01666       // we don't need to do any further analysis.
01667       if (AR->getNoWrapFlags(SCEV::FlagNSW))
01668         return getAddRecExpr(
01669             getExtendAddRecStart<SCEVSignExtendExpr>(AR, Ty, this),
01670             getSignExtendExpr(Step, Ty), L, SCEV::FlagNSW);
01671 
01672       // Check whether the backedge-taken count is SCEVCouldNotCompute.
01673       // Note that this serves two purposes: It filters out loops that are
01674       // simply not analyzable, and it covers the case where this code is
01675       // being called from within backedge-taken count analysis, such that
01676       // attempting to ask for the backedge-taken count would likely result
01677       // in infinite recursion. In the later case, the analysis code will
01678       // cope with a conservative value, and it will take care to purge
01679       // that value once it has finished.
01680       const SCEV *MaxBECount = getMaxBackedgeTakenCount(L);
01681       if (!isa<SCEVCouldNotCompute>(MaxBECount)) {
01682         // Manually compute the final value for AR, checking for
01683         // overflow.
01684 
01685         // Check whether the backedge-taken count can be losslessly casted to
01686         // the addrec's type. The count is always unsigned.
01687         const SCEV *CastedMaxBECount =
01688           getTruncateOrZeroExtend(MaxBECount, Start->getType());
01689         const SCEV *RecastedMaxBECount =
01690           getTruncateOrZeroExtend(CastedMaxBECount, MaxBECount->getType());
01691         if (MaxBECount == RecastedMaxBECount) {
01692           Type *WideTy = IntegerType::get(getContext(), BitWidth * 2);
01693           // Check whether Start+Step*MaxBECount has no signed overflow.
01694           const SCEV *SMul = getMulExpr(CastedMaxBECount, Step);
01695           const SCEV *SAdd = getSignExtendExpr(getAddExpr(Start, SMul), WideTy);
01696           const SCEV *WideStart = getSignExtendExpr(Start, WideTy);
01697           const SCEV *WideMaxBECount =
01698             getZeroExtendExpr(CastedMaxBECount, WideTy);
01699           const SCEV *OperandExtendedAdd =
01700             getAddExpr(WideStart,
01701                        getMulExpr(WideMaxBECount,
01702                                   getSignExtendExpr(Step, WideTy)));
01703           if (SAdd == OperandExtendedAdd) {
01704             // Cache knowledge of AR NSW, which is propagated to this AddRec.
01705             const_cast<SCEVAddRecExpr *>(AR)->setNoWrapFlags(SCEV::FlagNSW);
01706             // Return the expression with the addrec on the outside.
01707             return getAddRecExpr(
01708                 getExtendAddRecStart<SCEVSignExtendExpr>(AR, Ty, this),
01709                 getSignExtendExpr(Step, Ty), L, AR->getNoWrapFlags());
01710           }
01711           // Similar to above, only this time treat the step value as unsigned.
01712           // This covers loops that count up with an unsigned step.
01713           OperandExtendedAdd =
01714             getAddExpr(WideStart,
01715                        getMulExpr(WideMaxBECount,
01716                                   getZeroExtendExpr(Step, WideTy)));
01717           if (SAdd == OperandExtendedAdd) {
01718             // If AR wraps around then
01719             //
01720             //    abs(Step) * MaxBECount > unsigned-max(AR->getType())
01721             // => SAdd != OperandExtendedAdd
01722             //
01723             // Thus (AR is not NW => SAdd != OperandExtendedAdd) <=>
01724             // (SAdd == OperandExtendedAdd => AR is NW)
01725 
01726             const_cast<SCEVAddRecExpr *>(AR)->setNoWrapFlags(SCEV::FlagNW);
01727 
01728             // Return the expression with the addrec on the outside.
01729             return getAddRecExpr(
01730                 getExtendAddRecStart<SCEVSignExtendExpr>(AR, Ty, this),
01731                 getZeroExtendExpr(Step, Ty), L, AR->getNoWrapFlags());
01732           }
01733         }
01734 
01735         // If the backedge is guarded by a comparison with the pre-inc value
01736         // the addrec is safe. Also, if the entry is guarded by a comparison
01737         // with the start value and the backedge is guarded by a comparison
01738         // with the post-inc value, the addrec is safe.
01739         ICmpInst::Predicate Pred;
01740         const SCEV *OverflowLimit =
01741             getSignedOverflowLimitForStep(Step, &Pred, this);
01742         if (OverflowLimit &&
01743             (isLoopBackedgeGuardedByCond(L, Pred, AR, OverflowLimit) ||
01744              (isLoopEntryGuardedByCond(L, Pred, Start, OverflowLimit) &&
01745               isLoopBackedgeGuardedByCond(L, Pred, AR->getPostIncExpr(*this),
01746                                           OverflowLimit)))) {
01747           // Cache knowledge of AR NSW, then propagate NSW to the wide AddRec.
01748           const_cast<SCEVAddRecExpr *>(AR)->setNoWrapFlags(SCEV::FlagNSW);
01749           return getAddRecExpr(
01750               getExtendAddRecStart<SCEVSignExtendExpr>(AR, Ty, this),
01751               getSignExtendExpr(Step, Ty), L, AR->getNoWrapFlags());
01752         }
01753       }
01754       // If Start and Step are constants, check if we can apply this
01755       // transformation:
01756       // sext{C1,+,C2} --> C1 + sext{0,+,C2} if C1 < C2
01757       auto SC1 = dyn_cast<SCEVConstant>(Start);
01758       auto SC2 = dyn_cast<SCEVConstant>(Step);
01759       if (SC1 && SC2) {
01760         const APInt &C1 = SC1->getValue()->getValue();
01761         const APInt &C2 = SC2->getValue()->getValue();
01762         if (C1.isStrictlyPositive() && C2.isStrictlyPositive() && C2.ugt(C1) &&
01763             C2.isPowerOf2()) {
01764           Start = getSignExtendExpr(Start, Ty);
01765           const SCEV *NewAR = getAddRecExpr(getConstant(AR->getType(), 0), Step,
01766                                             L, AR->getNoWrapFlags());
01767           return getAddExpr(Start, getSignExtendExpr(NewAR, Ty));
01768         }
01769       }
01770 
01771       if (proveNoWrapByVaryingStart<SCEVSignExtendExpr>(Start, Step, L)) {
01772         const_cast<SCEVAddRecExpr *>(AR)->setNoWrapFlags(SCEV::FlagNSW);
01773         return getAddRecExpr(
01774             getExtendAddRecStart<SCEVSignExtendExpr>(AR, Ty, this),
01775             getSignExtendExpr(Step, Ty), L, AR->getNoWrapFlags());
01776       }
01777     }
01778 
01779   // The cast wasn't folded; create an explicit cast node.
01780   // Recompute the insert position, as it may have been invalidated.
01781   if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S;
01782   SCEV *S = new (SCEVAllocator) SCEVSignExtendExpr(ID.Intern(SCEVAllocator),
01783                                                    Op, Ty);
01784   UniqueSCEVs.InsertNode(S, IP);
01785   return S;
01786 }
01787 
01788 /// getAnyExtendExpr - Return a SCEV for the given operand extended with
01789 /// unspecified bits out to the given type.
01790 ///
01791 const SCEV *ScalarEvolution::getAnyExtendExpr(const SCEV *Op,
01792                                               Type *Ty) {
01793   assert(getTypeSizeInBits(Op->getType()) < getTypeSizeInBits(Ty) &&
01794          "This is not an extending conversion!");
01795   assert(isSCEVable(Ty) &&
01796          "This is not a conversion to a SCEVable type!");
01797   Ty = getEffectiveSCEVType(Ty);
01798 
01799   // Sign-extend negative constants.
01800   if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(Op))
01801     if (SC->getValue()->getValue().isNegative())
01802       return getSignExtendExpr(Op, Ty);
01803 
01804   // Peel off a truncate cast.
01805   if (const SCEVTruncateExpr *T = dyn_cast<SCEVTruncateExpr>(Op)) {
01806     const SCEV *NewOp = T->getOperand();
01807     if (getTypeSizeInBits(NewOp->getType()) < getTypeSizeInBits(Ty))
01808       return getAnyExtendExpr(NewOp, Ty);
01809     return getTruncateOrNoop(NewOp, Ty);
01810   }
01811 
01812   // Next try a zext cast. If the cast is folded, use it.
01813   const SCEV *ZExt = getZeroExtendExpr(Op, Ty);
01814   if (!isa<SCEVZeroExtendExpr>(ZExt))
01815     return ZExt;
01816 
01817   // Next try a sext cast. If the cast is folded, use it.
01818   const SCEV *SExt = getSignExtendExpr(Op, Ty);
01819   if (!isa<SCEVSignExtendExpr>(SExt))
01820     return SExt;
01821 
01822   // Force the cast to be folded into the operands of an addrec.
01823   if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Op)) {
01824     SmallVector<const SCEV *, 4> Ops;
01825     for (const SCEV *Op : AR->operands())
01826       Ops.push_back(getAnyExtendExpr(Op, Ty));
01827     return getAddRecExpr(Ops, AR->getLoop(), SCEV::FlagNW);
01828   }
01829 
01830   // If the expression is obviously signed, use the sext cast value.
01831   if (isa<SCEVSMaxExpr>(Op))
01832     return SExt;
01833 
01834   // Absent any other information, use the zext cast value.
01835   return ZExt;
01836 }
01837 
01838 /// CollectAddOperandsWithScales - Process the given Ops list, which is
01839 /// a list of operands to be added under the given scale, update the given
01840 /// map. This is a helper function for getAddRecExpr. As an example of
01841 /// what it does, given a sequence of operands that would form an add
01842 /// expression like this:
01843 ///
01844 ///    m + n + 13 + (A * (o + p + (B * (q + m + 29)))) + r + (-1 * r)
01845 ///
01846 /// where A and B are constants, update the map with these values:
01847 ///
01848 ///    (m, 1+A*B), (n, 1), (o, A), (p, A), (q, A*B), (r, 0)
01849 ///
01850 /// and add 13 + A*B*29 to AccumulatedConstant.
01851 /// This will allow getAddRecExpr to produce this:
01852 ///
01853 ///    13+A*B*29 + n + (m * (1+A*B)) + ((o + p) * A) + (q * A*B)
01854 ///
01855 /// This form often exposes folding opportunities that are hidden in
01856 /// the original operand list.
01857 ///
01858 /// Return true iff it appears that any interesting folding opportunities
01859 /// may be exposed. This helps getAddRecExpr short-circuit extra work in
01860 /// the common case where no interesting opportunities are present, and
01861 /// is also used as a check to avoid infinite recursion.
01862 ///
01863 static bool
01864 CollectAddOperandsWithScales(DenseMap<const SCEV *, APInt> &M,
01865                              SmallVectorImpl<const SCEV *> &NewOps,
01866                              APInt &AccumulatedConstant,
01867                              const SCEV *const *Ops, size_t NumOperands,
01868                              const APInt &Scale,
01869                              ScalarEvolution &SE) {
01870   bool Interesting = false;
01871 
01872   // Iterate over the add operands. They are sorted, with constants first.
01873   unsigned i = 0;
01874   while (const SCEVConstant *C = dyn_cast<SCEVConstant>(Ops[i])) {
01875     ++i;
01876     // Pull a buried constant out to the outside.
01877     if (Scale != 1 || AccumulatedConstant != 0 || C->getValue()->isZero())
01878       Interesting = true;
01879     AccumulatedConstant += Scale * C->getValue()->getValue();
01880   }
01881 
01882   // Next comes everything else. We're especially interested in multiplies
01883   // here, but they're in the middle, so just visit the rest with one loop.
01884   for (; i != NumOperands; ++i) {
01885     const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(Ops[i]);
01886     if (Mul && isa<SCEVConstant>(Mul->getOperand(0))) {
01887       APInt NewScale =
01888         Scale * cast<SCEVConstant>(Mul->getOperand(0))->getValue()->getValue();
01889       if (Mul->getNumOperands() == 2 && isa<SCEVAddExpr>(Mul->getOperand(1))) {
01890         // A multiplication of a constant with another add; recurse.
01891         const SCEVAddExpr *Add = cast<SCEVAddExpr>(Mul->getOperand(1));
01892         Interesting |=
01893           CollectAddOperandsWithScales(M, NewOps, AccumulatedConstant,
01894                                        Add->op_begin(), Add->getNumOperands(),
01895                                        NewScale, SE);
01896       } else {
01897         // A multiplication of a constant with some other value. Update
01898         // the map.
01899         SmallVector<const SCEV *, 4> MulOps(Mul->op_begin()+1, Mul->op_end());
01900         const SCEV *Key = SE.getMulExpr(MulOps);
01901         std::pair<DenseMap<const SCEV *, APInt>::iterator, bool> Pair =
01902           M.insert(std::make_pair(Key, NewScale));
01903         if (Pair.second) {
01904           NewOps.push_back(Pair.first->first);
01905         } else {
01906           Pair.first->second += NewScale;
01907           // The map already had an entry for this value, which may indicate
01908           // a folding opportunity.
01909           Interesting = true;
01910         }
01911       }
01912     } else {
01913       // An ordinary operand. Update the map.
01914       std::pair<DenseMap<const SCEV *, APInt>::iterator, bool> Pair =
01915         M.insert(std::make_pair(Ops[i], Scale));
01916       if (Pair.second) {
01917         NewOps.push_back(Pair.first->first);
01918       } else {
01919         Pair.first->second += Scale;
01920         // The map already had an entry for this value, which may indicate
01921         // a folding opportunity.
01922         Interesting = true;
01923       }
01924     }
01925   }
01926 
01927   return Interesting;
01928 }
01929 
01930 namespace {
01931   struct APIntCompare {
01932     bool operator()(const APInt &LHS, const APInt &RHS) const {
01933       return LHS.ult(RHS);
01934     }
01935   };
01936 }
01937 
01938 // We're trying to construct a SCEV of type `Type' with `Ops' as operands and
01939 // `OldFlags' as can't-wrap behavior.  Infer a more aggressive set of
01940 // can't-overflow flags for the operation if possible.
01941 static SCEV::NoWrapFlags
01942 StrengthenNoWrapFlags(ScalarEvolution *SE, SCEVTypes Type,
01943                       const SmallVectorImpl<const SCEV *> &Ops,
01944                       SCEV::NoWrapFlags OldFlags) {
01945   using namespace std::placeholders;
01946 
01947   bool CanAnalyze =
01948       Type == scAddExpr || Type == scAddRecExpr || Type == scMulExpr;
01949   (void)CanAnalyze;
01950   assert(CanAnalyze && "don't call from other places!");
01951 
01952   int SignOrUnsignMask = SCEV::FlagNUW | SCEV::FlagNSW;
01953   SCEV::NoWrapFlags SignOrUnsignWrap =
01954       ScalarEvolution::maskFlags(OldFlags, SignOrUnsignMask);
01955 
01956   // If FlagNSW is true and all the operands are non-negative, infer FlagNUW.
01957   auto IsKnownNonNegative =
01958     std::bind(std::mem_fn(&ScalarEvolution::isKnownNonNegative), SE, _1);
01959 
01960   if (SignOrUnsignWrap == SCEV::FlagNSW &&
01961       std::all_of(Ops.begin(), Ops.end(), IsKnownNonNegative))
01962     return ScalarEvolution::setFlags(OldFlags,
01963                                      (SCEV::NoWrapFlags)SignOrUnsignMask);
01964 
01965   return OldFlags;
01966 }
01967 
01968 /// getAddExpr - Get a canonical add expression, or something simpler if
01969 /// possible.
01970 const SCEV *ScalarEvolution::getAddExpr(SmallVectorImpl<const SCEV *> &Ops,
01971                                         SCEV::NoWrapFlags Flags) {
01972   assert(!(Flags & ~(SCEV::FlagNUW | SCEV::FlagNSW)) &&
01973          "only nuw or nsw allowed");
01974   assert(!Ops.empty() && "Cannot get empty add!");
01975   if (Ops.size() == 1) return Ops[0];
01976 #ifndef NDEBUG
01977   Type *ETy = getEffectiveSCEVType(Ops[0]->getType());
01978   for (unsigned i = 1, e = Ops.size(); i != e; ++i)
01979     assert(getEffectiveSCEVType(Ops[i]->getType()) == ETy &&
01980            "SCEVAddExpr operand types don't match!");
01981 #endif
01982 
01983   Flags = StrengthenNoWrapFlags(this, scAddExpr, Ops, Flags);
01984 
01985   // Sort by complexity, this groups all similar expression types together.
01986   GroupByComplexity(Ops, LI);
01987 
01988   // If there are any constants, fold them together.
01989   unsigned Idx = 0;
01990   if (const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(Ops[0])) {
01991     ++Idx;
01992     assert(Idx < Ops.size());
01993     while (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(Ops[Idx])) {
01994       // We found two constants, fold them together!
01995       Ops[0] = getConstant(LHSC->getValue()->getValue() +
01996                            RHSC->getValue()->getValue());
01997       if (Ops.size() == 2) return Ops[0];
01998       Ops.erase(Ops.begin()+1);  // Erase the folded element
01999       LHSC = cast<SCEVConstant>(Ops[0]);
02000     }
02001 
02002     // If we are left with a constant zero being added, strip it off.
02003     if (LHSC->getValue()->isZero()) {
02004       Ops.erase(Ops.begin());
02005       --Idx;
02006     }
02007 
02008     if (Ops.size() == 1) return Ops[0];
02009   }
02010 
02011   // Okay, check to see if the same value occurs in the operand list more than
02012   // once.  If so, merge them together into an multiply expression.  Since we
02013   // sorted the list, these values are required to be adjacent.
02014   Type *Ty = Ops[0]->getType();
02015   bool FoundMatch = false;
02016   for (unsigned i = 0, e = Ops.size(); i != e-1; ++i)
02017     if (Ops[i] == Ops[i+1]) {      //  X + Y + Y  -->  X + Y*2
02018       // Scan ahead to count how many equal operands there are.
02019       unsigned Count = 2;
02020       while (i+Count != e && Ops[i+Count] == Ops[i])
02021         ++Count;
02022       // Merge the values into a multiply.
02023       const SCEV *Scale = getConstant(Ty, Count);
02024       const SCEV *Mul = getMulExpr(Scale, Ops[i]);
02025       if (Ops.size() == Count)
02026         return Mul;
02027       Ops[i] = Mul;
02028       Ops.erase(Ops.begin()+i+1, Ops.begin()+i+Count);
02029       --i; e -= Count - 1;
02030       FoundMatch = true;
02031     }
02032   if (FoundMatch)
02033     return getAddExpr(Ops, Flags);
02034 
02035   // Check for truncates. If all the operands are truncated from the same
02036   // type, see if factoring out the truncate would permit the result to be
02037   // folded. eg., trunc(x) + m*trunc(n) --> trunc(x + trunc(m)*n)
02038   // if the contents of the resulting outer trunc fold to something simple.
02039   for (; Idx < Ops.size() && isa<SCEVTruncateExpr>(Ops[Idx]); ++Idx) {
02040     const SCEVTruncateExpr *Trunc = cast<SCEVTruncateExpr>(Ops[Idx]);
02041     Type *DstType = Trunc->getType();
02042     Type *SrcType = Trunc->getOperand()->getType();
02043     SmallVector<const SCEV *, 8> LargeOps;
02044     bool Ok = true;
02045     // Check all the operands to see if they can be represented in the
02046     // source type of the truncate.
02047     for (unsigned i = 0, e = Ops.size(); i != e; ++i) {
02048       if (const SCEVTruncateExpr *T = dyn_cast<SCEVTruncateExpr>(Ops[i])) {
02049         if (T->getOperand()->getType() != SrcType) {
02050           Ok = false;
02051           break;
02052         }
02053         LargeOps.push_back(T->getOperand());
02054       } else if (const SCEVConstant *C = dyn_cast<SCEVConstant>(Ops[i])) {
02055         LargeOps.push_back(getAnyExtendExpr(C, SrcType));
02056       } else if (const SCEVMulExpr *M = dyn_cast<SCEVMulExpr>(Ops[i])) {
02057         SmallVector<const SCEV *, 8> LargeMulOps;
02058         for (unsigned j = 0, f = M->getNumOperands(); j != f && Ok; ++j) {
02059           if (const SCEVTruncateExpr *T =
02060                 dyn_cast<SCEVTruncateExpr>(M->getOperand(j))) {
02061             if (T->getOperand()->getType() != SrcType) {
02062               Ok = false;
02063               break;
02064             }
02065             LargeMulOps.push_back(T->getOperand());
02066           } else if (const SCEVConstant *C =
02067                        dyn_cast<SCEVConstant>(M->getOperand(j))) {
02068             LargeMulOps.push_back(getAnyExtendExpr(C, SrcType));
02069           } else {
02070             Ok = false;
02071             break;
02072           }
02073         }
02074         if (Ok)
02075           LargeOps.push_back(getMulExpr(LargeMulOps));
02076       } else {
02077         Ok = false;
02078         break;
02079       }
02080     }
02081     if (Ok) {
02082       // Evaluate the expression in the larger type.
02083       const SCEV *Fold = getAddExpr(LargeOps, Flags);
02084       // If it folds to something simple, use it. Otherwise, don't.
02085       if (isa<SCEVConstant>(Fold) || isa<SCEVUnknown>(Fold))
02086         return getTruncateExpr(Fold, DstType);
02087     }
02088   }
02089 
02090   // Skip past any other cast SCEVs.
02091   while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scAddExpr)
02092     ++Idx;
02093 
02094   // If there are add operands they would be next.
02095   if (Idx < Ops.size()) {
02096     bool DeletedAdd = false;
02097     while (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(Ops[Idx])) {
02098       // If we have an add, expand the add operands onto the end of the operands
02099       // list.
02100       Ops.erase(Ops.begin()+Idx);
02101       Ops.append(Add->op_begin(), Add->op_end());
02102       DeletedAdd = true;
02103     }
02104 
02105     // If we deleted at least one add, we added operands to the end of the list,
02106     // and they are not necessarily sorted.  Recurse to resort and resimplify
02107     // any operands we just acquired.
02108     if (DeletedAdd)
02109       return getAddExpr(Ops);
02110   }
02111 
02112   // Skip over the add expression until we get to a multiply.
02113   while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scMulExpr)
02114     ++Idx;
02115 
02116   // Check to see if there are any folding opportunities present with
02117   // operands multiplied by constant values.
02118   if (Idx < Ops.size() && isa<SCEVMulExpr>(Ops[Idx])) {
02119     uint64_t BitWidth = getTypeSizeInBits(Ty);
02120     DenseMap<const SCEV *, APInt> M;
02121     SmallVector<const SCEV *, 8> NewOps;
02122     APInt AccumulatedConstant(BitWidth, 0);
02123     if (CollectAddOperandsWithScales(M, NewOps, AccumulatedConstant,
02124                                      Ops.data(), Ops.size(),
02125                                      APInt(BitWidth, 1), *this)) {
02126       // Some interesting folding opportunity is present, so its worthwhile to
02127       // re-generate the operands list. Group the operands by constant scale,
02128       // to avoid multiplying by the same constant scale multiple times.
02129       std::map<APInt, SmallVector<const SCEV *, 4>, APIntCompare> MulOpLists;
02130       for (SmallVectorImpl<const SCEV *>::const_iterator I = NewOps.begin(),
02131            E = NewOps.end(); I != E; ++I)
02132         MulOpLists[M.find(*I)->second].push_back(*I);
02133       // Re-generate the operands list.
02134       Ops.clear();
02135       if (AccumulatedConstant != 0)
02136         Ops.push_back(getConstant(AccumulatedConstant));
02137       for (std::map<APInt, SmallVector<const SCEV *, 4>, APIntCompare>::iterator
02138            I = MulOpLists.begin(), E = MulOpLists.end(); I != E; ++I)
02139         if (I->first != 0)
02140           Ops.push_back(getMulExpr(getConstant(I->first),
02141                                    getAddExpr(I->second)));
02142       if (Ops.empty())
02143         return getConstant(Ty, 0);
02144       if (Ops.size() == 1)
02145         return Ops[0];
02146       return getAddExpr(Ops);
02147     }
02148   }
02149 
02150   // If we are adding something to a multiply expression, make sure the
02151   // something is not already an operand of the multiply.  If so, merge it into
02152   // the multiply.
02153   for (; Idx < Ops.size() && isa<SCEVMulExpr>(Ops[Idx]); ++Idx) {
02154     const SCEVMulExpr *Mul = cast<SCEVMulExpr>(Ops[Idx]);
02155     for (unsigned MulOp = 0, e = Mul->getNumOperands(); MulOp != e; ++MulOp) {
02156       const SCEV *MulOpSCEV = Mul->getOperand(MulOp);
02157       if (isa<SCEVConstant>(MulOpSCEV))
02158         continue;
02159       for (unsigned AddOp = 0, e = Ops.size(); AddOp != e; ++AddOp)
02160         if (MulOpSCEV == Ops[AddOp]) {
02161           // Fold W + X + (X * Y * Z)  -->  W + (X * ((Y*Z)+1))
02162           const SCEV *InnerMul = Mul->getOperand(MulOp == 0);
02163           if (Mul->getNumOperands() != 2) {
02164             // If the multiply has more than two operands, we must get the
02165             // Y*Z term.
02166             SmallVector<const SCEV *, 4> MulOps(Mul->op_begin(),
02167                                                 Mul->op_begin()+MulOp);
02168             MulOps.append(Mul->op_begin()+MulOp+1, Mul->op_end());
02169             InnerMul = getMulExpr(MulOps);
02170           }
02171           const SCEV *One = getConstant(Ty, 1);
02172           const SCEV *AddOne = getAddExpr(One, InnerMul);
02173           const SCEV *OuterMul = getMulExpr(AddOne, MulOpSCEV);
02174           if (Ops.size() == 2) return OuterMul;
02175           if (AddOp < Idx) {
02176             Ops.erase(Ops.begin()+AddOp);
02177             Ops.erase(Ops.begin()+Idx-1);
02178           } else {
02179             Ops.erase(Ops.begin()+Idx);
02180             Ops.erase(Ops.begin()+AddOp-1);
02181           }
02182           Ops.push_back(OuterMul);
02183           return getAddExpr(Ops);
02184         }
02185 
02186       // Check this multiply against other multiplies being added together.
02187       for (unsigned OtherMulIdx = Idx+1;
02188            OtherMulIdx < Ops.size() && isa<SCEVMulExpr>(Ops[OtherMulIdx]);
02189            ++OtherMulIdx) {
02190         const SCEVMulExpr *OtherMul = cast<SCEVMulExpr>(Ops[OtherMulIdx]);
02191         // If MulOp occurs in OtherMul, we can fold the two multiplies
02192         // together.
02193         for (unsigned OMulOp = 0, e = OtherMul->getNumOperands();
02194              OMulOp != e; ++OMulOp)
02195           if (OtherMul->getOperand(OMulOp) == MulOpSCEV) {
02196             // Fold X + (A*B*C) + (A*D*E) --> X + (A*(B*C+D*E))
02197             const SCEV *InnerMul1 = Mul->getOperand(MulOp == 0);
02198             if (Mul->getNumOperands() != 2) {
02199               SmallVector<const SCEV *, 4> MulOps(Mul->op_begin(),
02200                                                   Mul->op_begin()+MulOp);
02201               MulOps.append(Mul->op_begin()+MulOp+1, Mul->op_end());
02202               InnerMul1 = getMulExpr(MulOps);
02203             }
02204             const SCEV *InnerMul2 = OtherMul->getOperand(OMulOp == 0);
02205             if (OtherMul->getNumOperands() != 2) {
02206               SmallVector<const SCEV *, 4> MulOps(OtherMul->op_begin(),
02207                                                   OtherMul->op_begin()+OMulOp);
02208               MulOps.append(OtherMul->op_begin()+OMulOp+1, OtherMul->op_end());
02209               InnerMul2 = getMulExpr(MulOps);
02210             }
02211             const SCEV *InnerMulSum = getAddExpr(InnerMul1,InnerMul2);
02212             const SCEV *OuterMul = getMulExpr(MulOpSCEV, InnerMulSum);
02213             if (Ops.size() == 2) return OuterMul;
02214             Ops.erase(Ops.begin()+Idx);
02215             Ops.erase(Ops.begin()+OtherMulIdx-1);
02216             Ops.push_back(OuterMul);
02217             return getAddExpr(Ops);
02218           }
02219       }
02220     }
02221   }
02222 
02223   // If there are any add recurrences in the operands list, see if any other
02224   // added values are loop invariant.  If so, we can fold them into the
02225   // recurrence.
02226   while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scAddRecExpr)
02227     ++Idx;
02228 
02229   // Scan over all recurrences, trying to fold loop invariants into them.
02230   for (; Idx < Ops.size() && isa<SCEVAddRecExpr>(Ops[Idx]); ++Idx) {
02231     // Scan all of the other operands to this add and add them to the vector if
02232     // they are loop invariant w.r.t. the recurrence.
02233     SmallVector<const SCEV *, 8> LIOps;
02234     const SCEVAddRecExpr *AddRec = cast<SCEVAddRecExpr>(Ops[Idx]);
02235     const Loop *AddRecLoop = AddRec->getLoop();
02236     for (unsigned i = 0, e = Ops.size(); i != e; ++i)
02237       if (isLoopInvariant(Ops[i], AddRecLoop)) {
02238         LIOps.push_back(Ops[i]);
02239         Ops.erase(Ops.begin()+i);
02240         --i; --e;
02241       }
02242 
02243     // If we found some loop invariants, fold them into the recurrence.
02244     if (!LIOps.empty()) {
02245       //  NLI + LI + {Start,+,Step}  -->  NLI + {LI+Start,+,Step}
02246       LIOps.push_back(AddRec->getStart());
02247 
02248       SmallVector<const SCEV *, 4> AddRecOps(AddRec->op_begin(),
02249                                              AddRec->op_end());
02250       AddRecOps[0] = getAddExpr(LIOps);
02251 
02252       // Build the new addrec. Propagate the NUW and NSW flags if both the
02253       // outer add and the inner addrec are guaranteed to have no overflow.
02254       // Always propagate NW.
02255       Flags = AddRec->getNoWrapFlags(setFlags(Flags, SCEV::FlagNW));
02256       const SCEV *NewRec = getAddRecExpr(AddRecOps, AddRecLoop, Flags);
02257 
02258       // If all of the other operands were loop invariant, we are done.
02259       if (Ops.size() == 1) return NewRec;
02260 
02261       // Otherwise, add the folded AddRec by the non-invariant parts.
02262       for (unsigned i = 0;; ++i)
02263         if (Ops[i] == AddRec) {
02264           Ops[i] = NewRec;
02265           break;
02266         }
02267       return getAddExpr(Ops);
02268     }
02269 
02270     // Okay, if there weren't any loop invariants to be folded, check to see if
02271     // there are multiple AddRec's with the same loop induction variable being
02272     // added together.  If so, we can fold them.
02273     for (unsigned OtherIdx = Idx+1;
02274          OtherIdx < Ops.size() && isa<SCEVAddRecExpr>(Ops[OtherIdx]);
02275          ++OtherIdx)
02276       if (AddRecLoop == cast<SCEVAddRecExpr>(Ops[OtherIdx])->getLoop()) {
02277         // Other + {A,+,B}<L> + {C,+,D}<L>  -->  Other + {A+C,+,B+D}<L>
02278         SmallVector<const SCEV *, 4> AddRecOps(AddRec->op_begin(),
02279                                                AddRec->op_end());
02280         for (; OtherIdx != Ops.size() && isa<SCEVAddRecExpr>(Ops[OtherIdx]);
02281              ++OtherIdx)
02282           if (const SCEVAddRecExpr *OtherAddRec =
02283                 dyn_cast<SCEVAddRecExpr>(Ops[OtherIdx]))
02284             if (OtherAddRec->getLoop() == AddRecLoop) {
02285               for (unsigned i = 0, e = OtherAddRec->getNumOperands();
02286                    i != e; ++i) {
02287                 if (i >= AddRecOps.size()) {
02288                   AddRecOps.append(OtherAddRec->op_begin()+i,
02289                                    OtherAddRec->op_end());
02290                   break;
02291                 }
02292                 AddRecOps[i] = getAddExpr(AddRecOps[i],
02293                                           OtherAddRec->getOperand(i));
02294               }
02295               Ops.erase(Ops.begin() + OtherIdx); --OtherIdx;
02296             }
02297         // Step size has changed, so we cannot guarantee no self-wraparound.
02298         Ops[Idx] = getAddRecExpr(AddRecOps, AddRecLoop, SCEV::FlagAnyWrap);
02299         return getAddExpr(Ops);
02300       }
02301 
02302     // Otherwise couldn't fold anything into this recurrence.  Move onto the
02303     // next one.
02304   }
02305 
02306   // Okay, it looks like we really DO need an add expr.  Check to see if we
02307   // already have one, otherwise create a new one.
02308   FoldingSetNodeID ID;
02309   ID.AddInteger(scAddExpr);
02310   for (unsigned i = 0, e = Ops.size(); i != e; ++i)
02311     ID.AddPointer(Ops[i]);
02312   void *IP = nullptr;
02313   SCEVAddExpr *S =
02314     static_cast<SCEVAddExpr *>(UniqueSCEVs.FindNodeOrInsertPos(ID, IP));
02315   if (!S) {
02316     const SCEV **O = SCEVAllocator.Allocate<const SCEV *>(Ops.size());
02317     std::uninitialized_copy(Ops.begin(), Ops.end(), O);
02318     S = new (SCEVAllocator) SCEVAddExpr(ID.Intern(SCEVAllocator),
02319                                         O, Ops.size());
02320     UniqueSCEVs.InsertNode(S, IP);
02321   }
02322   S->setNoWrapFlags(Flags);
02323   return S;
02324 }
02325 
02326 static uint64_t umul_ov(uint64_t i, uint64_t j, bool &Overflow) {
02327   uint64_t k = i*j;
02328   if (j > 1 && k / j != i) Overflow = true;
02329   return k;
02330 }
02331 
02332 /// Compute the result of "n choose k", the binomial coefficient.  If an
02333 /// intermediate computation overflows, Overflow will be set and the return will
02334 /// be garbage. Overflow is not cleared on absence of overflow.
02335 static uint64_t Choose(uint64_t n, uint64_t k, bool &Overflow) {
02336   // We use the multiplicative formula:
02337   //     n(n-1)(n-2)...(n-(k-1)) / k(k-1)(k-2)...1 .
02338   // At each iteration, we take the n-th term of the numeral and divide by the
02339   // (k-n)th term of the denominator.  This division will always produce an
02340   // integral result, and helps reduce the chance of overflow in the
02341   // intermediate computations. However, we can still overflow even when the
02342   // final result would fit.
02343 
02344   if (n == 0 || n == k) return 1;
02345   if (k > n) return 0;
02346 
02347   if (k > n/2)
02348     k = n-k;
02349 
02350   uint64_t r = 1;
02351   for (uint64_t i = 1; i <= k; ++i) {
02352     r = umul_ov(r, n-(i-1), Overflow);
02353     r /= i;
02354   }
02355   return r;
02356 }
02357 
02358 /// Determine if any of the operands in this SCEV are a constant or if
02359 /// any of the add or multiply expressions in this SCEV contain a constant.
02360 static bool containsConstantSomewhere(const SCEV *StartExpr) {
02361   SmallVector<const SCEV *, 4> Ops;
02362   Ops.push_back(StartExpr);
02363   while (!Ops.empty()) {
02364     const SCEV *CurrentExpr = Ops.pop_back_val();
02365     if (isa<SCEVConstant>(*CurrentExpr))
02366       return true;
02367 
02368     if (isa<SCEVAddExpr>(*CurrentExpr) || isa<SCEVMulExpr>(*CurrentExpr)) {
02369       const auto *CurrentNAry = cast<SCEVNAryExpr>(CurrentExpr);
02370       Ops.append(CurrentNAry->op_begin(), CurrentNAry->op_end());
02371     }
02372   }
02373   return false;
02374 }
02375 
02376 /// getMulExpr - Get a canonical multiply expression, or something simpler if
02377 /// possible.
02378 const SCEV *ScalarEvolution::getMulExpr(SmallVectorImpl<const SCEV *> &Ops,
02379                                         SCEV::NoWrapFlags Flags) {
02380   assert(Flags == maskFlags(Flags, SCEV::FlagNUW | SCEV::FlagNSW) &&
02381          "only nuw or nsw allowed");
02382   assert(!Ops.empty() && "Cannot get empty mul!");
02383   if (Ops.size() == 1) return Ops[0];
02384 #ifndef NDEBUG
02385   Type *ETy = getEffectiveSCEVType(Ops[0]->getType());
02386   for (unsigned i = 1, e = Ops.size(); i != e; ++i)
02387     assert(getEffectiveSCEVType(Ops[i]->getType()) == ETy &&
02388            "SCEVMulExpr operand types don't match!");
02389 #endif
02390 
02391   Flags = StrengthenNoWrapFlags(this, scMulExpr, Ops, Flags);
02392 
02393   // Sort by complexity, this groups all similar expression types together.
02394   GroupByComplexity(Ops, LI);
02395 
02396   // If there are any constants, fold them together.
02397   unsigned Idx = 0;
02398   if (const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(Ops[0])) {
02399 
02400     // C1*(C2+V) -> C1*C2 + C1*V
02401     if (Ops.size() == 2)
02402         if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(Ops[1]))
02403           // If any of Add's ops are Adds or Muls with a constant,
02404           // apply this transformation as well.
02405           if (Add->getNumOperands() == 2)
02406             if (containsConstantSomewhere(Add))
02407               return getAddExpr(getMulExpr(LHSC, Add->getOperand(0)),
02408                                 getMulExpr(LHSC, Add->getOperand(1)));
02409 
02410     ++Idx;
02411     while (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(Ops[Idx])) {
02412       // We found two constants, fold them together!
02413       ConstantInt *Fold = ConstantInt::get(getContext(),
02414                                            LHSC->getValue()->getValue() *
02415                                            RHSC->getValue()->getValue());
02416       Ops[0] = getConstant(Fold);
02417       Ops.erase(Ops.begin()+1);  // Erase the folded element
02418       if (Ops.size() == 1) return Ops[0];
02419       LHSC = cast<SCEVConstant>(Ops[0]);
02420     }
02421 
02422     // If we are left with a constant one being multiplied, strip it off.
02423     if (cast<SCEVConstant>(Ops[0])->getValue()->equalsInt(1)) {
02424       Ops.erase(Ops.begin());
02425       --Idx;
02426     } else if (cast<SCEVConstant>(Ops[0])->getValue()->isZero()) {
02427       // If we have a multiply of zero, it will always be zero.
02428       return Ops[0];
02429     } else if (Ops[0]->isAllOnesValue()) {
02430       // If we have a mul by -1 of an add, try distributing the -1 among the
02431       // add operands.
02432       if (Ops.size() == 2) {
02433         if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(Ops[1])) {
02434           SmallVector<const SCEV *, 4> NewOps;
02435           bool AnyFolded = false;
02436           for (SCEVAddRecExpr::op_iterator I = Add->op_begin(),
02437                  E = Add->op_end(); I != E; ++I) {
02438             const SCEV *Mul = getMulExpr(Ops[0], *I);
02439             if (!isa<SCEVMulExpr>(Mul)) AnyFolded = true;
02440             NewOps.push_back(Mul);
02441           }
02442           if (AnyFolded)
02443             return getAddExpr(NewOps);
02444         }
02445         else if (const SCEVAddRecExpr *
02446                  AddRec = dyn_cast<SCEVAddRecExpr>(Ops[1])) {
02447           // Negation preserves a recurrence's no self-wrap property.
02448           SmallVector<const SCEV *, 4> Operands;
02449           for (SCEVAddRecExpr::op_iterator I = AddRec->op_begin(),
02450                  E = AddRec->op_end(); I != E; ++I) {
02451             Operands.push_back(getMulExpr(Ops[0], *I));
02452           }
02453           return getAddRecExpr(Operands, AddRec->getLoop(),
02454                                AddRec->getNoWrapFlags(SCEV::FlagNW));
02455         }
02456       }
02457     }
02458 
02459     if (Ops.size() == 1)
02460       return Ops[0];
02461   }
02462 
02463   // Skip over the add expression until we get to a multiply.
02464   while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scMulExpr)
02465     ++Idx;
02466 
02467   // If there are mul operands inline them all into this expression.
02468   if (Idx < Ops.size()) {
02469     bool DeletedMul = false;
02470     while (const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(Ops[Idx])) {
02471       // If we have an mul, expand the mul operands onto the end of the operands
02472       // list.
02473       Ops.erase(Ops.begin()+Idx);
02474       Ops.append(Mul->op_begin(), Mul->op_end());
02475       DeletedMul = true;
02476     }
02477 
02478     // If we deleted at least one mul, we added operands to the end of the list,
02479     // and they are not necessarily sorted.  Recurse to resort and resimplify
02480     // any operands we just acquired.
02481     if (DeletedMul)
02482       return getMulExpr(Ops);
02483   }
02484 
02485   // If there are any add recurrences in the operands list, see if any other
02486   // added values are loop invariant.  If so, we can fold them into the
02487   // recurrence.
02488   while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scAddRecExpr)
02489     ++Idx;
02490 
02491   // Scan over all recurrences, trying to fold loop invariants into them.
02492   for (; Idx < Ops.size() && isa<SCEVAddRecExpr>(Ops[Idx]); ++Idx) {
02493     // Scan all of the other operands to this mul and add them to the vector if
02494     // they are loop invariant w.r.t. the recurrence.
02495     SmallVector<const SCEV *, 8> LIOps;
02496     const SCEVAddRecExpr *AddRec = cast<SCEVAddRecExpr>(Ops[Idx]);
02497     const Loop *AddRecLoop = AddRec->getLoop();
02498     for (unsigned i = 0, e = Ops.size(); i != e; ++i)
02499       if (isLoopInvariant(Ops[i], AddRecLoop)) {
02500         LIOps.push_back(Ops[i]);
02501         Ops.erase(Ops.begin()+i);
02502         --i; --e;
02503       }
02504 
02505     // If we found some loop invariants, fold them into the recurrence.
02506     if (!LIOps.empty()) {
02507       //  NLI * LI * {Start,+,Step}  -->  NLI * {LI*Start,+,LI*Step}
02508       SmallVector<const SCEV *, 4> NewOps;
02509       NewOps.reserve(AddRec->getNumOperands());
02510       const SCEV *Scale = getMulExpr(LIOps);
02511       for (unsigned i = 0, e = AddRec->getNumOperands(); i != e; ++i)
02512         NewOps.push_back(getMulExpr(Scale, AddRec->getOperand(i)));
02513 
02514       // Build the new addrec. Propagate the NUW and NSW flags if both the
02515       // outer mul and the inner addrec are guaranteed to have no overflow.
02516       //
02517       // No self-wrap cannot be guaranteed after changing the step size, but
02518       // will be inferred if either NUW or NSW is true.
02519       Flags = AddRec->getNoWrapFlags(clearFlags(Flags, SCEV::FlagNW));
02520       const SCEV *NewRec = getAddRecExpr(NewOps, AddRecLoop, Flags);
02521 
02522       // If all of the other operands were loop invariant, we are done.
02523       if (Ops.size() == 1) return NewRec;
02524 
02525       // Otherwise, multiply the folded AddRec by the non-invariant parts.
02526       for (unsigned i = 0;; ++i)
02527         if (Ops[i] == AddRec) {
02528           Ops[i] = NewRec;
02529           break;
02530         }
02531       return getMulExpr(Ops);
02532     }
02533 
02534     // Okay, if there weren't any loop invariants to be folded, check to see if
02535     // there are multiple AddRec's with the same loop induction variable being
02536     // multiplied together.  If so, we can fold them.
02537 
02538     // {A1,+,A2,+,...,+,An}<L> * {B1,+,B2,+,...,+,Bn}<L>
02539     // = {x=1 in [ sum y=x..2x [ sum z=max(y-x, y-n)..min(x,n) [
02540     //       choose(x, 2x)*choose(2x-y, x-z)*A_{y-z}*B_z
02541     //   ]]],+,...up to x=2n}.
02542     // Note that the arguments to choose() are always integers with values
02543     // known at compile time, never SCEV objects.
02544     //
02545     // The implementation avoids pointless extra computations when the two
02546     // addrec's are of different length (mathematically, it's equivalent to
02547     // an infinite stream of zeros on the right).
02548     bool OpsModified = false;
02549     for (unsigned OtherIdx = Idx+1;
02550          OtherIdx != Ops.size() && isa<SCEVAddRecExpr>(Ops[OtherIdx]);
02551          ++OtherIdx) {
02552       const SCEVAddRecExpr *OtherAddRec =
02553         dyn_cast<SCEVAddRecExpr>(Ops[OtherIdx]);
02554       if (!OtherAddRec || OtherAddRec->getLoop() != AddRecLoop)
02555         continue;
02556 
02557       bool Overflow = false;
02558       Type *Ty = AddRec->getType();
02559       bool LargerThan64Bits = getTypeSizeInBits(Ty) > 64;
02560       SmallVector<const SCEV*, 7> AddRecOps;
02561       for (int x = 0, xe = AddRec->getNumOperands() +
02562              OtherAddRec->getNumOperands() - 1; x != xe && !Overflow; ++x) {
02563         const SCEV *Term = getConstant(Ty, 0);
02564         for (int y = x, ye = 2*x+1; y != ye && !Overflow; ++y) {
02565           uint64_t Coeff1 = Choose(x, 2*x - y, Overflow);
02566           for (int z = std::max(y-x, y-(int)AddRec->getNumOperands()+1),
02567                  ze = std::min(x+1, (int)OtherAddRec->getNumOperands());
02568                z < ze && !Overflow; ++z) {
02569             uint64_t Coeff2 = Choose(2*x - y, x-z, Overflow);
02570             uint64_t Coeff;
02571             if (LargerThan64Bits)
02572               Coeff = umul_ov(Coeff1, Coeff2, Overflow);
02573             else
02574               Coeff = Coeff1*Coeff2;
02575             const SCEV *CoeffTerm = getConstant(Ty, Coeff);
02576             const SCEV *Term1 = AddRec->getOperand(y-z);
02577             const SCEV *Term2 = OtherAddRec->getOperand(z);
02578             Term = getAddExpr(Term, getMulExpr(CoeffTerm, Term1,Term2));
02579           }
02580         }
02581         AddRecOps.push_back(Term);
02582       }
02583       if (!Overflow) {
02584         const SCEV *NewAddRec = getAddRecExpr(AddRecOps, AddRec->getLoop(),
02585                                               SCEV::FlagAnyWrap);
02586         if (Ops.size() == 2) return NewAddRec;
02587         Ops[Idx] = NewAddRec;
02588         Ops.erase(Ops.begin() + OtherIdx); --OtherIdx;
02589         OpsModified = true;
02590         AddRec = dyn_cast<SCEVAddRecExpr>(NewAddRec);
02591         if (!AddRec)
02592           break;
02593       }
02594     }
02595     if (OpsModified)
02596       return getMulExpr(Ops);
02597 
02598     // Otherwise couldn't fold anything into this recurrence.  Move onto the
02599     // next one.
02600   }
02601 
02602   // Okay, it looks like we really DO need an mul expr.  Check to see if we
02603   // already have one, otherwise create a new one.
02604   FoldingSetNodeID ID;
02605   ID.AddInteger(scMulExpr);
02606   for (unsigned i = 0, e = Ops.size(); i != e; ++i)
02607     ID.AddPointer(Ops[i]);
02608   void *IP = nullptr;
02609   SCEVMulExpr *S =
02610     static_cast<SCEVMulExpr *>(UniqueSCEVs.FindNodeOrInsertPos(ID, IP));
02611   if (!S) {
02612     const SCEV **O = SCEVAllocator.Allocate<const SCEV *>(Ops.size());
02613     std::uninitialized_copy(Ops.begin(), Ops.end(), O);
02614     S = new (SCEVAllocator) SCEVMulExpr(ID.Intern(SCEVAllocator),
02615                                         O, Ops.size());
02616     UniqueSCEVs.InsertNode(S, IP);
02617   }
02618   S->setNoWrapFlags(Flags);
02619   return S;
02620 }
02621 
02622 /// getUDivExpr - Get a canonical unsigned division expression, or something
02623 /// simpler if possible.
02624 const SCEV *ScalarEvolution::getUDivExpr(const SCEV *LHS,
02625                                          const SCEV *RHS) {
02626   assert(getEffectiveSCEVType(LHS->getType()) ==
02627          getEffectiveSCEVType(RHS->getType()) &&
02628          "SCEVUDivExpr operand types don't match!");
02629 
02630   if (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(RHS)) {
02631     if (RHSC->getValue()->equalsInt(1))
02632       return LHS;                               // X udiv 1 --> x
02633     // If the denominator is zero, the result of the udiv is undefined. Don't
02634     // try to analyze it, because the resolution chosen here may differ from
02635     // the resolution chosen in other parts of the compiler.
02636     if (!RHSC->getValue()->isZero()) {
02637       // Determine if the division can be folded into the operands of
02638       // its operands.
02639       // TODO: Generalize this to non-constants by using known-bits information.
02640       Type *Ty = LHS->getType();
02641       unsigned LZ = RHSC->getValue()->getValue().countLeadingZeros();
02642       unsigned MaxShiftAmt = getTypeSizeInBits(Ty) - LZ - 1;
02643       // For non-power-of-two values, effectively round the value up to the
02644       // nearest power of two.
02645       if (!RHSC->getValue()->getValue().isPowerOf2())
02646         ++MaxShiftAmt;
02647       IntegerType *ExtTy =
02648         IntegerType::get(getContext(), getTypeSizeInBits(Ty) + MaxShiftAmt);
02649       if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(LHS))
02650         if (const SCEVConstant *Step =
02651             dyn_cast<SCEVConstant>(AR->getStepRecurrence(*this))) {
02652           // {X,+,N}/C --> {X/C,+,N/C} if safe and N/C can be folded.
02653           const APInt &StepInt = Step->getValue()->getValue();
02654           const APInt &DivInt = RHSC->getValue()->getValue();
02655           if (!StepInt.urem(DivInt) &&
02656               getZeroExtendExpr(AR, ExtTy) ==
02657               getAddRecExpr(getZeroExtendExpr(AR->getStart(), ExtTy),
02658                             getZeroExtendExpr(Step, ExtTy),
02659                             AR->getLoop(), SCEV::FlagAnyWrap)) {
02660             SmallVector<const SCEV *, 4> Operands;
02661             for (unsigned i = 0, e = AR->getNumOperands(); i != e; ++i)
02662               Operands.push_back(getUDivExpr(AR->getOperand(i), RHS));
02663             return getAddRecExpr(Operands, AR->getLoop(),
02664                                  SCEV::FlagNW);
02665           }
02666           /// Get a canonical UDivExpr for a recurrence.
02667           /// {X,+,N}/C => {Y,+,N}/C where Y=X-(X%N). Safe when C%N=0.
02668           // We can currently only fold X%N if X is constant.
02669           const SCEVConstant *StartC = dyn_cast<SCEVConstant>(AR->getStart());
02670           if (StartC && !DivInt.urem(StepInt) &&
02671               getZeroExtendExpr(AR, ExtTy) ==
02672               getAddRecExpr(getZeroExtendExpr(AR->getStart(), ExtTy),
02673                             getZeroExtendExpr(Step, ExtTy),
02674                             AR->getLoop(), SCEV::FlagAnyWrap)) {
02675             const APInt &StartInt = StartC->getValue()->getValue();
02676             const APInt &StartRem = StartInt.urem(StepInt);
02677             if (StartRem != 0)
02678               LHS = getAddRecExpr(getConstant(StartInt - StartRem), Step,
02679                                   AR->getLoop(), SCEV::FlagNW);
02680           }
02681         }
02682       // (A*B)/C --> A*(B/C) if safe and B/C can be folded.
02683       if (const SCEVMulExpr *M = dyn_cast<SCEVMulExpr>(LHS)) {
02684         SmallVector<const SCEV *, 4> Operands;
02685         for (unsigned i = 0, e = M->getNumOperands(); i != e; ++i)
02686           Operands.push_back(getZeroExtendExpr(M->getOperand(i), ExtTy));
02687         if (getZeroExtendExpr(M, ExtTy) == getMulExpr(Operands))
02688           // Find an operand that's safely divisible.
02689           for (unsigned i = 0, e = M->getNumOperands(); i != e; ++i) {
02690             const SCEV *Op = M->getOperand(i);
02691             const SCEV *Div = getUDivExpr(Op, RHSC);
02692             if (!isa<SCEVUDivExpr>(Div) && getMulExpr(Div, RHSC) == Op) {
02693               Operands = SmallVector<const SCEV *, 4>(M->op_begin(),
02694                                                       M->op_end());
02695               Operands[i] = Div;
02696               return getMulExpr(Operands);
02697             }
02698           }
02699       }
02700       // (A+B)/C --> (A/C + B/C) if safe and A/C and B/C can be folded.
02701       if (const SCEVAddExpr *A = dyn_cast<SCEVAddExpr>(LHS)) {
02702         SmallVector<const SCEV *, 4> Operands;
02703         for (unsigned i = 0, e = A->getNumOperands(); i != e; ++i)
02704           Operands.push_back(getZeroExtendExpr(A->getOperand(i), ExtTy));
02705         if (getZeroExtendExpr(A, ExtTy) == getAddExpr(Operands)) {
02706           Operands.clear();
02707           for (unsigned i = 0, e = A->getNumOperands(); i != e; ++i) {
02708             const SCEV *Op = getUDivExpr(A->getOperand(i), RHS);
02709             if (isa<SCEVUDivExpr>(Op) ||
02710                 getMulExpr(Op, RHS) != A->getOperand(i))
02711               break;
02712             Operands.push_back(Op);
02713           }
02714           if (Operands.size() == A->getNumOperands())
02715             return getAddExpr(Operands);
02716         }
02717       }
02718 
02719       // Fold if both operands are constant.
02720       if (const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(LHS)) {
02721         Constant *LHSCV = LHSC->getValue();
02722         Constant *RHSCV = RHSC->getValue();
02723         return getConstant(cast<ConstantInt>(ConstantExpr::getUDiv(LHSCV,
02724                                                                    RHSCV)));
02725       }
02726     }
02727   }
02728 
02729   FoldingSetNodeID ID;
02730   ID.AddInteger(scUDivExpr);
02731   ID.AddPointer(LHS);
02732   ID.AddPointer(RHS);
02733   void *IP = nullptr;
02734   if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S;
02735   SCEV *S = new (SCEVAllocator) SCEVUDivExpr(ID.Intern(SCEVAllocator),
02736                                              LHS, RHS);
02737   UniqueSCEVs.InsertNode(S, IP);
02738   return S;
02739 }
02740 
02741 static const APInt gcd(const SCEVConstant *C1, const SCEVConstant *C2) {
02742   APInt A = C1->getValue()->getValue().abs();
02743   APInt B = C2->getValue()->getValue().abs();
02744   uint32_t ABW = A.getBitWidth();
02745   uint32_t BBW = B.getBitWidth();
02746 
02747   if (ABW > BBW)
02748     B = B.zext(ABW);
02749   else if (ABW < BBW)
02750     A = A.zext(BBW);
02751 
02752   return APIntOps::GreatestCommonDivisor(A, B);
02753 }
02754 
02755 /// getUDivExactExpr - Get a canonical unsigned division expression, or
02756 /// something simpler if possible. There is no representation for an exact udiv
02757 /// in SCEV IR, but we can attempt to remove factors from the LHS and RHS.
02758 /// We can't do this when it's not exact because the udiv may be clearing bits.
02759 const SCEV *ScalarEvolution::getUDivExactExpr(const SCEV *LHS,
02760                                               const SCEV *RHS) {
02761   // TODO: we could try to find factors in all sorts of things, but for now we
02762   // just deal with u/exact (multiply, constant). See SCEVDivision towards the
02763   // end of this file for inspiration.
02764 
02765   const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(LHS);
02766   if (!Mul)
02767     return getUDivExpr(LHS, RHS);
02768 
02769   if (const SCEVConstant *RHSCst = dyn_cast<SCEVConstant>(RHS)) {
02770     // If the mulexpr multiplies by a constant, then that constant must be the
02771     // first element of the mulexpr.
02772     if (const SCEVConstant *LHSCst =
02773             dyn_cast<SCEVConstant>(Mul->getOperand(0))) {
02774       if (LHSCst == RHSCst) {
02775         SmallVector<const SCEV *, 2> Operands;
02776         Operands.append(Mul->op_begin() + 1, Mul->op_end());
02777         return getMulExpr(Operands);
02778       }
02779 
02780       // We can't just assume that LHSCst divides RHSCst cleanly, it could be
02781       // that there's a factor provided by one of the other terms. We need to
02782       // check.
02783       APInt Factor = gcd(LHSCst, RHSCst);
02784       if (!Factor.isIntN(1)) {
02785         LHSCst = cast<SCEVConstant>(
02786             getConstant(LHSCst->getValue()->getValue().udiv(Factor)));
02787         RHSCst = cast<SCEVConstant>(
02788             getConstant(RHSCst->getValue()->getValue().udiv(Factor)));
02789         SmallVector<const SCEV *, 2> Operands;
02790         Operands.push_back(LHSCst);
02791         Operands.append(Mul->op_begin() + 1, Mul->op_end());
02792         LHS = getMulExpr(Operands);
02793         RHS = RHSCst;
02794         Mul = dyn_cast<SCEVMulExpr>(LHS);
02795         if (!Mul)
02796           return getUDivExactExpr(LHS, RHS);
02797       }
02798     }
02799   }
02800 
02801   for (int i = 0, e = Mul->getNumOperands(); i != e; ++i) {
02802     if (Mul->getOperand(i) == RHS) {
02803       SmallVector<const SCEV *, 2> Operands;
02804       Operands.append(Mul->op_begin(), Mul->op_begin() + i);
02805       Operands.append(Mul->op_begin() + i + 1, Mul->op_end());
02806       return getMulExpr(Operands);
02807     }
02808   }
02809 
02810   return getUDivExpr(LHS, RHS);
02811 }
02812 
02813 /// getAddRecExpr - Get an add recurrence expression for the specified loop.
02814 /// Simplify the expression as much as possible.
02815 const SCEV *ScalarEvolution::getAddRecExpr(const SCEV *Start, const SCEV *Step,
02816                                            const Loop *L,
02817                                            SCEV::NoWrapFlags Flags) {
02818   SmallVector<const SCEV *, 4> Operands;
02819   Operands.push_back(Start);
02820   if (const SCEVAddRecExpr *StepChrec = dyn_cast<SCEVAddRecExpr>(Step))
02821     if (StepChrec->getLoop() == L) {
02822       Operands.append(StepChrec->op_begin(), StepChrec->op_end());
02823       return getAddRecExpr(Operands, L, maskFlags(Flags, SCEV::FlagNW));
02824     }
02825 
02826   Operands.push_back(Step);
02827   return getAddRecExpr(Operands, L, Flags);
02828 }
02829 
02830 /// getAddRecExpr - Get an add recurrence expression for the specified loop.
02831 /// Simplify the expression as much as possible.
02832 const SCEV *
02833 ScalarEvolution::getAddRecExpr(SmallVectorImpl<const SCEV *> &Operands,
02834                                const Loop *L, SCEV::NoWrapFlags Flags) {
02835   if (Operands.size() == 1) return Operands[0];
02836 #ifndef NDEBUG
02837   Type *ETy = getEffectiveSCEVType(Operands[0]->getType());
02838   for (unsigned i = 1, e = Operands.size(); i != e; ++i)
02839     assert(getEffectiveSCEVType(Operands[i]->getType()) == ETy &&
02840            "SCEVAddRecExpr operand types don't match!");
02841   for (unsigned i = 0, e = Operands.size(); i != e; ++i)
02842     assert(isLoopInvariant(Operands[i], L) &&
02843            "SCEVAddRecExpr operand is not loop-invariant!");
02844 #endif
02845 
02846   if (Operands.back()->isZero()) {
02847     Operands.pop_back();
02848     return getAddRecExpr(Operands, L, SCEV::FlagAnyWrap); // {X,+,0}  -->  X
02849   }
02850 
02851   // It's tempting to want to call getMaxBackedgeTakenCount count here and
02852   // use that information to infer NUW and NSW flags. However, computing a
02853   // BE count requires calling getAddRecExpr, so we may not yet have a
02854   // meaningful BE count at this point (and if we don't, we'd be stuck
02855   // with a SCEVCouldNotCompute as the cached BE count).
02856 
02857   Flags = StrengthenNoWrapFlags(this, scAddRecExpr, Operands, Flags);
02858 
02859   // Canonicalize nested AddRecs in by nesting them in order of loop depth.
02860   if (const SCEVAddRecExpr *NestedAR = dyn_cast<SCEVAddRecExpr>(Operands[0])) {
02861     const Loop *NestedLoop = NestedAR->getLoop();
02862     if (L->contains(NestedLoop) ?
02863         (L->getLoopDepth() < NestedLoop->getLoopDepth()) :
02864         (!NestedLoop->contains(L) &&
02865          DT->dominates(L->getHeader(), NestedLoop->getHeader()))) {
02866       SmallVector<const SCEV *, 4> NestedOperands(NestedAR->op_begin(),
02867                                                   NestedAR->op_end());
02868       Operands[0] = NestedAR->getStart();
02869       // AddRecs require their operands be loop-invariant with respect to their
02870       // loops. Don't perform this transformation if it would break this
02871       // requirement.
02872       bool AllInvariant = true;
02873       for (unsigned i = 0, e = Operands.size(); i != e; ++i)
02874         if (!isLoopInvariant(Operands[i], L)) {
02875           AllInvariant = false;
02876           break;
02877         }
02878       if (AllInvariant) {
02879         // Create a recurrence for the outer loop with the same step size.
02880         //
02881         // The outer recurrence keeps its NW flag but only keeps NUW/NSW if the
02882         // inner recurrence has the same property.
02883         SCEV::NoWrapFlags OuterFlags =
02884           maskFlags(Flags, SCEV::FlagNW | NestedAR->getNoWrapFlags());
02885 
02886         NestedOperands[0] = getAddRecExpr(Operands, L, OuterFlags);
02887         AllInvariant = true;
02888         for (unsigned i = 0, e = NestedOperands.size(); i != e; ++i)
02889           if (!isLoopInvariant(NestedOperands[i], NestedLoop)) {
02890             AllInvariant = false;
02891             break;
02892           }
02893         if (AllInvariant) {
02894           // Ok, both add recurrences are valid after the transformation.
02895           //
02896           // The inner recurrence keeps its NW flag but only keeps NUW/NSW if
02897           // the outer recurrence has the same property.
02898           SCEV::NoWrapFlags InnerFlags =
02899             maskFlags(NestedAR->getNoWrapFlags(), SCEV::FlagNW | Flags);
02900           return getAddRecExpr(NestedOperands, NestedLoop, InnerFlags);
02901         }
02902       }
02903       // Reset Operands to its original state.
02904       Operands[0] = NestedAR;
02905     }
02906   }
02907 
02908   // Okay, it looks like we really DO need an addrec expr.  Check to see if we
02909   // already have one, otherwise create a new one.
02910   FoldingSetNodeID ID;
02911   ID.AddInteger(scAddRecExpr);
02912   for (unsigned i = 0, e = Operands.size(); i != e; ++i)
02913     ID.AddPointer(Operands[i]);
02914   ID.AddPointer(L);
02915   void *IP = nullptr;
02916   SCEVAddRecExpr *S =
02917     static_cast<SCEVAddRecExpr *>(UniqueSCEVs.FindNodeOrInsertPos(ID, IP));
02918   if (!S) {
02919     const SCEV **O = SCEVAllocator.Allocate<const SCEV *>(Operands.size());
02920     std::uninitialized_copy(Operands.begin(), Operands.end(), O);
02921     S = new (SCEVAllocator) SCEVAddRecExpr(ID.Intern(SCEVAllocator),
02922                                            O, Operands.size(), L);
02923     UniqueSCEVs.InsertNode(S, IP);
02924   }
02925   S->setNoWrapFlags(Flags);
02926   return S;
02927 }
02928 
02929 const SCEV *ScalarEvolution::getSMaxExpr(const SCEV *LHS,
02930                                          const SCEV *RHS) {
02931   SmallVector<const SCEV *, 2> Ops;
02932   Ops.push_back(LHS);
02933   Ops.push_back(RHS);
02934   return getSMaxExpr(Ops);
02935 }
02936 
02937 const SCEV *
02938 ScalarEvolution::getSMaxExpr(SmallVectorImpl<const SCEV *> &Ops) {
02939   assert(!Ops.empty() && "Cannot get empty smax!");
02940   if (Ops.size() == 1) return Ops[0];
02941 #ifndef NDEBUG
02942   Type *ETy = getEffectiveSCEVType(Ops[0]->getType());
02943   for (unsigned i = 1, e = Ops.size(); i != e; ++i)
02944     assert(getEffectiveSCEVType(Ops[i]->getType()) == ETy &&
02945            "SCEVSMaxExpr operand types don't match!");
02946 #endif
02947 
02948   // Sort by complexity, this groups all similar expression types together.
02949   GroupByComplexity(Ops, LI);
02950 
02951   // If there are any constants, fold them together.
02952   unsigned Idx = 0;
02953   if (const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(Ops[0])) {
02954     ++Idx;
02955     assert(Idx < Ops.size());
02956     while (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(Ops[Idx])) {
02957       // We found two constants, fold them together!
02958       ConstantInt *Fold = ConstantInt::get(getContext(),
02959                               APIntOps::smax(LHSC->getValue()->getValue(),
02960                                              RHSC->getValue()->getValue()));
02961       Ops[0] = getConstant(Fold);
02962       Ops.erase(Ops.begin()+1);  // Erase the folded element
02963       if (Ops.size() == 1) return Ops[0];
02964       LHSC = cast<SCEVConstant>(Ops[0]);
02965     }
02966 
02967     // If we are left with a constant minimum-int, strip it off.
02968     if (cast<SCEVConstant>(Ops[0])->getValue()->isMinValue(true)) {
02969       Ops.erase(Ops.begin());
02970       --Idx;
02971     } else if (cast<SCEVConstant>(Ops[0])->getValue()->isMaxValue(true)) {
02972       // If we have an smax with a constant maximum-int, it will always be
02973       // maximum-int.
02974       return Ops[0];
02975     }
02976 
02977     if (Ops.size() == 1) return Ops[0];
02978   }
02979 
02980   // Find the first SMax
02981   while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scSMaxExpr)
02982     ++Idx;
02983 
02984   // Check to see if one of the operands is an SMax. If so, expand its operands
02985   // onto our operand list, and recurse to simplify.
02986   if (Idx < Ops.size()) {
02987     bool DeletedSMax = false;
02988     while (const SCEVSMaxExpr *SMax = dyn_cast<SCEVSMaxExpr>(Ops[Idx])) {
02989       Ops.erase(Ops.begin()+Idx);
02990       Ops.append(SMax->op_begin(), SMax->op_end());
02991       DeletedSMax = true;
02992     }
02993 
02994     if (DeletedSMax)
02995       return getSMaxExpr(Ops);
02996   }
02997 
02998   // Okay, check to see if the same value occurs in the operand list twice.  If
02999   // so, delete one.  Since we sorted the list, these values are required to
03000   // be adjacent.
03001   for (unsigned i = 0, e = Ops.size()-1; i != e; ++i)
03002     //  X smax Y smax Y  -->  X smax Y
03003     //  X smax Y         -->  X, if X is always greater than Y
03004     if (Ops[i] == Ops[i+1] ||
03005         isKnownPredicate(ICmpInst::ICMP_SGE, Ops[i], Ops[i+1])) {
03006       Ops.erase(Ops.begin()+i+1, Ops.begin()+i+2);
03007       --i; --e;
03008     } else if (isKnownPredicate(ICmpInst::ICMP_SLE, Ops[i], Ops[i+1])) {
03009       Ops.erase(Ops.begin()+i, Ops.begin()+i+1);
03010       --i; --e;
03011     }
03012 
03013   if (Ops.size() == 1) return Ops[0];
03014 
03015   assert(!Ops.empty() && "Reduced smax down to nothing!");
03016 
03017   // Okay, it looks like we really DO need an smax expr.  Check to see if we
03018   // already have one, otherwise create a new one.
03019   FoldingSetNodeID ID;
03020   ID.AddInteger(scSMaxExpr);
03021   for (unsigned i = 0, e = Ops.size(); i != e; ++i)
03022     ID.AddPointer(Ops[i]);
03023   void *IP = nullptr;
03024   if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S;
03025   const SCEV **O = SCEVAllocator.Allocate<const SCEV *>(Ops.size());
03026   std::uninitialized_copy(Ops.begin(), Ops.end(), O);
03027   SCEV *S = new (SCEVAllocator) SCEVSMaxExpr(ID.Intern(SCEVAllocator),
03028                                              O, Ops.size());
03029   UniqueSCEVs.InsertNode(S, IP);
03030   return S;
03031 }
03032 
03033 const SCEV *ScalarEvolution::getUMaxExpr(const SCEV *LHS,
03034                                          const SCEV *RHS) {
03035   SmallVector<const SCEV *, 2> Ops;
03036   Ops.push_back(LHS);
03037   Ops.push_back(RHS);
03038   return getUMaxExpr(Ops);
03039 }
03040 
03041 const SCEV *
03042 ScalarEvolution::getUMaxExpr(SmallVectorImpl<const SCEV *> &Ops) {
03043   assert(!Ops.empty() && "Cannot get empty umax!");
03044   if (Ops.size() == 1) return Ops[0];
03045 #ifndef NDEBUG
03046   Type *ETy = getEffectiveSCEVType(Ops[0]->getType());
03047   for (unsigned i = 1, e = Ops.size(); i != e; ++i)
03048     assert(getEffectiveSCEVType(Ops[i]->getType()) == ETy &&
03049            "SCEVUMaxExpr operand types don't match!");
03050 #endif
03051 
03052   // Sort by complexity, this groups all similar expression types together.
03053   GroupByComplexity(Ops, LI);
03054 
03055   // If there are any constants, fold them together.
03056   unsigned Idx = 0;
03057   if (const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(Ops[0])) {
03058     ++Idx;
03059     assert(Idx < Ops.size());
03060     while (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(Ops[Idx])) {
03061       // We found two constants, fold them together!
03062       ConstantInt *Fold = ConstantInt::get(getContext(),
03063                               APIntOps::umax(LHSC->getValue()->getValue(),
03064                                              RHSC->getValue()->getValue()));
03065       Ops[0] = getConstant(Fold);
03066       Ops.erase(Ops.begin()+1);  // Erase the folded element
03067       if (Ops.size() == 1) return Ops[0];
03068       LHSC = cast<SCEVConstant>(Ops[0]);
03069     }
03070 
03071     // If we are left with a constant minimum-int, strip it off.
03072     if (cast<SCEVConstant>(Ops[0])->getValue()->isMinValue(false)) {
03073       Ops.erase(Ops.begin());
03074       --Idx;
03075     } else if (cast<SCEVConstant>(Ops[0])->getValue()->isMaxValue(false)) {
03076       // If we have an umax with a constant maximum-int, it will always be
03077       // maximum-int.
03078       return Ops[0];
03079     }
03080 
03081     if (Ops.size() == 1) return Ops[0];
03082   }
03083 
03084   // Find the first UMax
03085   while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scUMaxExpr)
03086     ++Idx;
03087 
03088   // Check to see if one of the operands is a UMax. If so, expand its operands
03089   // onto our operand list, and recurse to simplify.
03090   if (Idx < Ops.size()) {
03091     bool DeletedUMax = false;
03092     while (const SCEVUMaxExpr *UMax = dyn_cast<SCEVUMaxExpr>(Ops[Idx])) {
03093       Ops.erase(Ops.begin()+Idx);
03094       Ops.append(UMax->op_begin(), UMax->op_end());
03095       DeletedUMax = true;
03096     }
03097 
03098     if (DeletedUMax)
03099       return getUMaxExpr(Ops);
03100   }
03101 
03102   // Okay, check to see if the same value occurs in the operand list twice.  If
03103   // so, delete one.  Since we sorted the list, these values are required to
03104   // be adjacent.
03105   for (unsigned i = 0, e = Ops.size()-1; i != e; ++i)
03106     //  X umax Y umax Y  -->  X umax Y
03107     //  X umax Y         -->  X, if X is always greater than Y
03108     if (Ops[i] == Ops[i+1] ||
03109         isKnownPredicate(ICmpInst::ICMP_UGE, Ops[i], Ops[i+1])) {
03110       Ops.erase(Ops.begin()+i+1, Ops.begin()+i+2);
03111       --i; --e;
03112     } else if (isKnownPredicate(ICmpInst::ICMP_ULE, Ops[i], Ops[i+1])) {
03113       Ops.erase(Ops.begin()+i, Ops.begin()+i+1);
03114       --i; --e;
03115     }
03116 
03117   if (Ops.size() == 1) return Ops[0];
03118 
03119   assert(!Ops.empty() && "Reduced umax down to nothing!");
03120 
03121   // Okay, it looks like we really DO need a umax expr.  Check to see if we
03122   // already have one, otherwise create a new one.
03123   FoldingSetNodeID ID;
03124   ID.AddInteger(scUMaxExpr);
03125   for (unsigned i = 0, e = Ops.size(); i != e; ++i)
03126     ID.AddPointer(Ops[i]);
03127   void *IP = nullptr;
03128   if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S;
03129   const SCEV **O = SCEVAllocator.Allocate<const SCEV *>(Ops.size());
03130   std::uninitialized_copy(Ops.begin(), Ops.end(), O);
03131   SCEV *S = new (SCEVAllocator) SCEVUMaxExpr(ID.Intern(SCEVAllocator),
03132                                              O, Ops.size());
03133   UniqueSCEVs.InsertNode(S, IP);
03134   return S;
03135 }
03136 
03137 const SCEV *ScalarEvolution::getSMinExpr(const SCEV *LHS,
03138                                          const SCEV *RHS) {
03139   // ~smax(~x, ~y) == smin(x, y).
03140   return getNotSCEV(getSMaxExpr(getNotSCEV(LHS), getNotSCEV(RHS)));
03141 }
03142 
03143 const SCEV *ScalarEvolution::getUMinExpr(const SCEV *LHS,
03144                                          const SCEV *RHS) {
03145   // ~umax(~x, ~y) == umin(x, y)
03146   return getNotSCEV(getUMaxExpr(getNotSCEV(LHS), getNotSCEV(RHS)));
03147 }
03148 
03149 const SCEV *ScalarEvolution::getSizeOfExpr(Type *IntTy, Type *AllocTy) {
03150   // We can bypass creating a target-independent
03151   // constant expression and then folding it back into a ConstantInt.
03152   // This is just a compile-time optimization.
03153   return getConstant(IntTy,
03154                      F->getParent()->getDataLayout().getTypeAllocSize(AllocTy));
03155 }
03156 
03157 const SCEV *ScalarEvolution::getOffsetOfExpr(Type *IntTy,
03158                                              StructType *STy,
03159                                              unsigned FieldNo) {
03160   // We can bypass creating a target-independent
03161   // constant expression and then folding it back into a ConstantInt.
03162   // This is just a compile-time optimization.
03163   return getConstant(
03164       IntTy,
03165       F->getParent()->getDataLayout().getStructLayout(STy)->getElementOffset(
03166           FieldNo));
03167 }
03168 
03169 const SCEV *ScalarEvolution::getUnknown(Value *V) {
03170   // Don't attempt to do anything other than create a SCEVUnknown object
03171   // here.  createSCEV only calls getUnknown after checking for all other
03172   // interesting possibilities, and any other code that calls getUnknown
03173   // is doing so in order to hide a value from SCEV canonicalization.
03174 
03175   FoldingSetNodeID ID;
03176   ID.AddInteger(scUnknown);
03177   ID.AddPointer(V);
03178   void *IP = nullptr;
03179   if (SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) {
03180     assert(cast<SCEVUnknown>(S)->getValue() == V &&
03181            "Stale SCEVUnknown in uniquing map!");
03182     return S;
03183   }
03184   SCEV *S = new (SCEVAllocator) SCEVUnknown(ID.Intern(SCEVAllocator), V, this,
03185                                             FirstUnknown);
03186   FirstUnknown = cast<SCEVUnknown>(S);
03187   UniqueSCEVs.InsertNode(S, IP);
03188   return S;
03189 }
03190 
03191 //===----------------------------------------------------------------------===//
03192 //            Basic SCEV Analysis and PHI Idiom Recognition Code
03193 //
03194 
03195 /// isSCEVable - Test if values of the given type are analyzable within
03196 /// the SCEV framework. This primarily includes integer types, and it
03197 /// can optionally include pointer types if the ScalarEvolution class
03198 /// has access to target-specific information.
03199 bool ScalarEvolution::isSCEVable(Type *Ty) const {
03200   // Integers and pointers are always SCEVable.
03201   return Ty->isIntegerTy() || Ty->isPointerTy();
03202 }
03203 
03204 /// getTypeSizeInBits - Return the size in bits of the specified type,
03205 /// for which isSCEVable must return true.
03206 uint64_t ScalarEvolution::getTypeSizeInBits(Type *Ty) const {
03207   assert(isSCEVable(Ty) && "Type is not SCEVable!");
03208   return F->getParent()->getDataLayout().getTypeSizeInBits(Ty);
03209 }
03210 
03211 /// getEffectiveSCEVType - Return a type with the same bitwidth as
03212 /// the given type and which represents how SCEV will treat the given
03213 /// type, for which isSCEVable must return true. For pointer types,
03214 /// this is the pointer-sized integer type.
03215 Type *ScalarEvolution::getEffectiveSCEVType(Type *Ty) const {
03216   assert(isSCEVable(Ty) && "Type is not SCEVable!");
03217 
03218   if (Ty->isIntegerTy()) {
03219     return Ty;
03220   }
03221 
03222   // The only other support type is pointer.
03223   assert(Ty->isPointerTy() && "Unexpected non-pointer non-integer type!");
03224   return F->getParent()->getDataLayout().getIntPtrType(Ty);
03225 }
03226 
03227 const SCEV *ScalarEvolution::getCouldNotCompute() {
03228   return &CouldNotCompute;
03229 }
03230 
03231 namespace {
03232   // Helper class working with SCEVTraversal to figure out if a SCEV contains
03233   // a SCEVUnknown with null value-pointer. FindInvalidSCEVUnknown::FindOne
03234   // is set iff if find such SCEVUnknown.
03235   //
03236   struct FindInvalidSCEVUnknown {
03237     bool FindOne;
03238     FindInvalidSCEVUnknown() { FindOne = false; }
03239     bool follow(const SCEV *S) {
03240       switch (static_cast<SCEVTypes>(S->getSCEVType())) {
03241       case scConstant:
03242         return false;
03243       case scUnknown:
03244         if (!cast<SCEVUnknown>(S)->getValue())
03245           FindOne = true;
03246         return false;
03247       default:
03248         return true;
03249       }
03250     }
03251     bool isDone() const { return FindOne; }
03252   };
03253 }
03254 
03255 bool ScalarEvolution::checkValidity(const SCEV *S) const {
03256   FindInvalidSCEVUnknown F;
03257   SCEVTraversal<FindInvalidSCEVUnknown> ST(F);
03258   ST.visitAll(S);
03259 
03260   return !F.FindOne;
03261 }
03262 
03263 /// getSCEV - Return an existing SCEV if it exists, otherwise analyze the
03264 /// expression and create a new one.
03265 const SCEV *ScalarEvolution::getSCEV(Value *V) {
03266   assert(isSCEVable(V->getType()) && "Value is not SCEVable!");
03267 
03268   ValueExprMapType::iterator I = ValueExprMap.find_as(V);
03269   if (I != ValueExprMap.end()) {
03270     const SCEV *S = I->second;
03271     if (checkValidity(S))
03272       return S;
03273     else
03274       ValueExprMap.erase(I);
03275   }
03276   const SCEV *S = createSCEV(V);
03277 
03278   // The process of creating a SCEV for V may have caused other SCEVs
03279   // to have been created, so it's necessary to insert the new entry
03280   // from scratch, rather than trying to remember the insert position
03281   // above.
03282   ValueExprMap.insert(std::make_pair(SCEVCallbackVH(V, this), S));
03283   return S;
03284 }
03285 
03286 /// getNegativeSCEV - Return a SCEV corresponding to -V = -1*V
03287 ///
03288 const SCEV *ScalarEvolution::getNegativeSCEV(const SCEV *V) {
03289   if (const SCEVConstant *VC = dyn_cast<SCEVConstant>(V))
03290     return getConstant(
03291                cast<ConstantInt>(ConstantExpr::getNeg(VC->getValue())));
03292 
03293   Type *Ty = V->getType();
03294   Ty = getEffectiveSCEVType(Ty);
03295   return getMulExpr(V,
03296                   getConstant(cast<ConstantInt>(Constant::getAllOnesValue(Ty))));
03297 }
03298 
03299 /// getNotSCEV - Return a SCEV corresponding to ~V = -1-V
03300 const SCEV *ScalarEvolution::getNotSCEV(const SCEV *V) {
03301   if (const SCEVConstant *VC = dyn_cast<SCEVConstant>(V))
03302     return getConstant(
03303                 cast<ConstantInt>(ConstantExpr::getNot(VC->getValue())));
03304 
03305   Type *Ty = V->getType();
03306   Ty = getEffectiveSCEVType(Ty);
03307   const SCEV *AllOnes =
03308                    getConstant(cast<ConstantInt>(Constant::getAllOnesValue(Ty)));
03309   return getMinusSCEV(AllOnes, V);
03310 }
03311 
03312 /// getMinusSCEV - Return LHS-RHS.  Minus is represented in SCEV as A+B*-1.
03313 const SCEV *ScalarEvolution::getMinusSCEV(const SCEV *LHS, const SCEV *RHS,
03314                                           SCEV::NoWrapFlags Flags) {
03315   assert(!maskFlags(Flags, SCEV::FlagNUW) && "subtraction does not have NUW");
03316 
03317   // Fast path: X - X --> 0.
03318   if (LHS == RHS)
03319     return getConstant(LHS->getType(), 0);
03320 
03321   // X - Y --> X + -Y.
03322   // X -(nsw || nuw) Y --> X + -Y.
03323   return getAddExpr(LHS, getNegativeSCEV(RHS));
03324 }
03325 
03326 /// getTruncateOrZeroExtend - Return a SCEV corresponding to a conversion of the
03327 /// input value to the specified type.  If the type must be extended, it is zero
03328 /// extended.
03329 const SCEV *
03330 ScalarEvolution::getTruncateOrZeroExtend(const SCEV *V, Type *Ty) {
03331   Type *SrcTy = V->getType();
03332   assert((SrcTy->isIntegerTy() || SrcTy->isPointerTy()) &&
03333          (Ty->isIntegerTy() || Ty->isPointerTy()) &&
03334          "Cannot truncate or zero extend with non-integer arguments!");
03335   if (getTypeSizeInBits(SrcTy) == getTypeSizeInBits(Ty))
03336     return V;  // No conversion
03337   if (getTypeSizeInBits(SrcTy) > getTypeSizeInBits(Ty))
03338     return getTruncateExpr(V, Ty);
03339   return getZeroExtendExpr(V, Ty);
03340 }
03341 
03342 /// getTruncateOrSignExtend - Return a SCEV corresponding to a conversion of the
03343 /// input value to the specified type.  If the type must be extended, it is sign
03344 /// extended.
03345 const SCEV *
03346 ScalarEvolution::getTruncateOrSignExtend(const SCEV *V,
03347                                          Type *Ty) {
03348   Type *SrcTy = V->getType();
03349   assert((SrcTy->isIntegerTy() || SrcTy->isPointerTy()) &&
03350          (Ty->isIntegerTy() || Ty->isPointerTy()) &&
03351          "Cannot truncate or zero extend with non-integer arguments!");
03352   if (getTypeSizeInBits(SrcTy) == getTypeSizeInBits(Ty))
03353     return V;  // No conversion
03354   if (getTypeSizeInBits(SrcTy) > getTypeSizeInBits(Ty))
03355     return getTruncateExpr(V, Ty);
03356   return getSignExtendExpr(V, Ty);
03357 }
03358 
03359 /// getNoopOrZeroExtend - Return a SCEV corresponding to a conversion of the
03360 /// input value to the specified type.  If the type must be extended, it is zero
03361 /// extended.  The conversion must not be narrowing.
03362 const SCEV *
03363 ScalarEvolution::getNoopOrZeroExtend(const SCEV *V, Type *Ty) {
03364   Type *SrcTy = V->getType();
03365   assert((SrcTy->isIntegerTy() || SrcTy->isPointerTy()) &&
03366          (Ty->isIntegerTy() || Ty->isPointerTy()) &&
03367          "Cannot noop or zero extend with non-integer arguments!");
03368   assert(getTypeSizeInBits(SrcTy) <= getTypeSizeInBits(Ty) &&
03369          "getNoopOrZeroExtend cannot truncate!");
03370   if (getTypeSizeInBits(SrcTy) == getTypeSizeInBits(Ty))
03371     return V;  // No conversion
03372   return getZeroExtendExpr(V, Ty);
03373 }
03374 
03375 /// getNoopOrSignExtend - Return a SCEV corresponding to a conversion of the
03376 /// input value to the specified type.  If the type must be extended, it is sign
03377 /// extended.  The conversion must not be narrowing.
03378 const SCEV *
03379 ScalarEvolution::getNoopOrSignExtend(const SCEV *V, Type *Ty) {
03380   Type *SrcTy = V->getType();
03381   assert((SrcTy->isIntegerTy() || SrcTy->isPointerTy()) &&
03382          (Ty->isIntegerTy() || Ty->isPointerTy()) &&
03383          "Cannot noop or sign extend with non-integer arguments!");
03384   assert(getTypeSizeInBits(SrcTy) <= getTypeSizeInBits(Ty) &&
03385          "getNoopOrSignExtend cannot truncate!");
03386   if (getTypeSizeInBits(SrcTy) == getTypeSizeInBits(Ty))
03387     return V;  // No conversion
03388   return getSignExtendExpr(V, Ty);
03389 }
03390 
03391 /// getNoopOrAnyExtend - Return a SCEV corresponding to a conversion of
03392 /// the input value to the specified type. If the type must be extended,
03393 /// it is extended with unspecified bits. The conversion must not be
03394 /// narrowing.
03395 const SCEV *
03396 ScalarEvolution::getNoopOrAnyExtend(const SCEV *V, Type *Ty) {
03397   Type *SrcTy = V->getType();
03398   assert((SrcTy->isIntegerTy() || SrcTy->isPointerTy()) &&
03399          (Ty->isIntegerTy() || Ty->isPointerTy()) &&
03400          "Cannot noop or any extend with non-integer arguments!");
03401   assert(getTypeSizeInBits(SrcTy) <= getTypeSizeInBits(Ty) &&
03402          "getNoopOrAnyExtend cannot truncate!");
03403   if (getTypeSizeInBits(SrcTy) == getTypeSizeInBits(Ty))
03404     return V;  // No conversion
03405   return getAnyExtendExpr(V, Ty);
03406 }
03407 
03408 /// getTruncateOrNoop - Return a SCEV corresponding to a conversion of the
03409 /// input value to the specified type.  The conversion must not be widening.
03410 const SCEV *
03411 ScalarEvolution::getTruncateOrNoop(const SCEV *V, Type *Ty) {
03412   Type *SrcTy = V->getType();
03413   assert((SrcTy->isIntegerTy() || SrcTy->isPointerTy()) &&
03414          (Ty->isIntegerTy() || Ty->isPointerTy()) &&
03415          "Cannot truncate or noop with non-integer arguments!");
03416   assert(getTypeSizeInBits(SrcTy) >= getTypeSizeInBits(Ty) &&
03417          "getTruncateOrNoop cannot extend!");
03418   if (getTypeSizeInBits(SrcTy) == getTypeSizeInBits(Ty))
03419     return V;  // No conversion
03420   return getTruncateExpr(V, Ty);
03421 }
03422 
03423 /// getUMaxFromMismatchedTypes - Promote the operands to the wider of
03424 /// the types using zero-extension, and then perform a umax operation
03425 /// with them.
03426 const SCEV *ScalarEvolution::getUMaxFromMismatchedTypes(const SCEV *LHS,
03427                                                         const SCEV *RHS) {
03428   const SCEV *PromotedLHS = LHS;
03429   const SCEV *PromotedRHS = RHS;
03430 
03431   if (getTypeSizeInBits(LHS->getType()) > getTypeSizeInBits(RHS->getType()))
03432     PromotedRHS = getZeroExtendExpr(RHS, LHS->getType());
03433   else
03434     PromotedLHS = getNoopOrZeroExtend(LHS, RHS->getType());
03435 
03436   return getUMaxExpr(PromotedLHS, PromotedRHS);
03437 }
03438 
03439 /// getUMinFromMismatchedTypes - Promote the operands to the wider of
03440 /// the types using zero-extension, and then perform a umin operation
03441 /// with them.
03442 const SCEV *ScalarEvolution::getUMinFromMismatchedTypes(const SCEV *LHS,
03443                                                         const SCEV *RHS) {
03444   const SCEV *PromotedLHS = LHS;
03445   const SCEV *PromotedRHS = RHS;
03446 
03447   if (getTypeSizeInBits(LHS->getType()) > getTypeSizeInBits(RHS->getType()))
03448     PromotedRHS = getZeroExtendExpr(RHS, LHS->getType());
03449   else
03450     PromotedLHS = getNoopOrZeroExtend(LHS, RHS->getType());
03451 
03452   return getUMinExpr(PromotedLHS, PromotedRHS);
03453 }
03454 
03455 /// getPointerBase - Transitively follow the chain of pointer-type operands
03456 /// until reaching a SCEV that does not have a single pointer operand. This
03457 /// returns a SCEVUnknown pointer for well-formed pointer-type expressions,
03458 /// but corner cases do exist.
03459 const SCEV *ScalarEvolution::getPointerBase(const SCEV *V) {
03460   // A pointer operand may evaluate to a nonpointer expression, such as null.
03461   if (!V->getType()->isPointerTy())
03462     return V;
03463 
03464   if (const SCEVCastExpr *Cast = dyn_cast<SCEVCastExpr>(V)) {
03465     return getPointerBase(Cast->getOperand());
03466   }
03467   else if (const SCEVNAryExpr *NAry = dyn_cast<SCEVNAryExpr>(V)) {
03468     const SCEV *PtrOp = nullptr;
03469     for (SCEVNAryExpr::op_iterator I = NAry->op_begin(), E = NAry->op_end();
03470          I != E; ++I) {
03471       if ((*I)->getType()->isPointerTy()) {
03472         // Cannot find the base of an expression with multiple pointer operands.
03473         if (PtrOp)
03474           return V;
03475         PtrOp = *I;
03476       }
03477     }
03478     if (!PtrOp)
03479       return V;
03480     return getPointerBase(PtrOp);
03481   }
03482   return V;
03483 }
03484 
03485 /// PushDefUseChildren - Push users of the given Instruction
03486 /// onto the given Worklist.
03487 static void
03488 PushDefUseChildren(Instruction *I,
03489                    SmallVectorImpl<Instruction *> &Worklist) {
03490   // Push the def-use children onto the Worklist stack.
03491   for (User *U : I->users())
03492     Worklist.push_back(cast<Instruction>(U));
03493 }
03494 
03495 /// ForgetSymbolicValue - This looks up computed SCEV values for all
03496 /// instructions that depend on the given instruction and removes them from
03497 /// the ValueExprMapType map if they reference SymName. This is used during PHI
03498 /// resolution.
03499 void
03500 ScalarEvolution::ForgetSymbolicName(Instruction *PN, const SCEV *SymName) {
03501   SmallVector<Instruction *, 16> Worklist;
03502   PushDefUseChildren(PN, Worklist);
03503 
03504   SmallPtrSet<Instruction *, 8> Visited;
03505   Visited.insert(PN);
03506   while (!Worklist.empty()) {
03507     Instruction *I = Worklist.pop_back_val();
03508     if (!Visited.insert(I).second)
03509       continue;
03510 
03511     ValueExprMapType::iterator It =
03512       ValueExprMap.find_as(static_cast<Value *>(I));
03513     if (It != ValueExprMap.end()) {
03514       const SCEV *Old = It->second;
03515 
03516       // Short-circuit the def-use traversal if the symbolic name
03517       // ceases to appear in expressions.
03518       if (Old != SymName && !hasOperand(Old, SymName))
03519         continue;
03520 
03521       // SCEVUnknown for a PHI either means that it has an unrecognized
03522       // structure, it's a PHI that's in the progress of being computed
03523       // by createNodeForPHI, or it's a single-value PHI. In the first case,
03524       // additional loop trip count information isn't going to change anything.
03525       // In the second case, createNodeForPHI will perform the necessary
03526       // updates on its own when it gets to that point. In the third, we do
03527       // want to forget the SCEVUnknown.
03528       if (!isa<PHINode>(I) ||
03529           !isa<SCEVUnknown>(Old) ||
03530           (I != PN && Old == SymName)) {
03531         forgetMemoizedResults(Old);
03532         ValueExprMap.erase(It);
03533       }
03534     }
03535 
03536     PushDefUseChildren(I, Worklist);
03537   }
03538 }
03539 
03540 /// createNodeForPHI - PHI nodes have two cases.  Either the PHI node exists in
03541 /// a loop header, making it a potential recurrence, or it doesn't.
03542 ///
03543 const SCEV *ScalarEvolution::createNodeForPHI(PHINode *PN) {
03544   if (const Loop *L = LI->getLoopFor(PN->getParent()))
03545     if (L->getHeader() == PN->getParent()) {
03546       // The loop may have multiple entrances or multiple exits; we can analyze
03547       // this phi as an addrec if it has a unique entry value and a unique
03548       // backedge value.
03549       Value *BEValueV = nullptr, *StartValueV = nullptr;
03550       for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) {
03551         Value *V = PN->getIncomingValue(i);
03552         if (L->contains(PN->getIncomingBlock(i))) {
03553           if (!BEValueV) {
03554             BEValueV = V;
03555           } else if (BEValueV != V) {
03556             BEValueV = nullptr;
03557             break;
03558           }
03559         } else if (!StartValueV) {
03560           StartValueV = V;
03561         } else if (StartValueV != V) {
03562           StartValueV = nullptr;
03563           break;
03564         }
03565       }
03566       if (BEValueV && StartValueV) {
03567         // While we are analyzing this PHI node, handle its value symbolically.
03568         const SCEV *SymbolicName = getUnknown(PN);
03569         assert(ValueExprMap.find_as(PN) == ValueExprMap.end() &&
03570                "PHI node already processed?");
03571         ValueExprMap.insert(std::make_pair(SCEVCallbackVH(PN, this), SymbolicName));
03572 
03573         // Using this symbolic name for the PHI, analyze the value coming around
03574         // the back-edge.
03575         const SCEV *BEValue = getSCEV(BEValueV);
03576 
03577         // NOTE: If BEValue is loop invariant, we know that the PHI node just
03578         // has a special value for the first iteration of the loop.
03579 
03580         // If the value coming around the backedge is an add with the symbolic
03581         // value we just inserted, then we found a simple induction variable!
03582         if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(BEValue)) {
03583           // If there is a single occurrence of the symbolic value, replace it
03584           // with a recurrence.
03585           unsigned FoundIndex = Add->getNumOperands();
03586           for (unsigned i = 0, e = Add->getNumOperands(); i != e; ++i)
03587             if (Add->getOperand(i) == SymbolicName)
03588               if (FoundIndex == e) {
03589                 FoundIndex = i;
03590                 break;
03591               }
03592 
03593           if (FoundIndex != Add->getNumOperands()) {
03594             // Create an add with everything but the specified operand.
03595             SmallVector<const SCEV *, 8> Ops;
03596             for (unsigned i = 0, e = Add->getNumOperands(); i != e; ++i)
03597               if (i != FoundIndex)
03598                 Ops.push_back(Add->getOperand(i));
03599             const SCEV *Accum = getAddExpr(Ops);
03600 
03601             // This is not a valid addrec if the step amount is varying each
03602             // loop iteration, but is not itself an addrec in this loop.
03603             if (isLoopInvariant(Accum, L) ||
03604                 (isa<SCEVAddRecExpr>(Accum) &&
03605                  cast<SCEVAddRecExpr>(Accum)->getLoop() == L)) {
03606               SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap;
03607 
03608               // If the increment doesn't overflow, then neither the addrec nor
03609               // the post-increment will overflow.
03610               if (const AddOperator *OBO = dyn_cast<AddOperator>(BEValueV)) {
03611                 if (OBO->getOperand(0) == PN) {
03612                   if (OBO->hasNoUnsignedWrap())
03613                     Flags = setFlags(Flags, SCEV::FlagNUW);
03614                   if (OBO->hasNoSignedWrap())
03615                     Flags = setFlags(Flags, SCEV::FlagNSW);
03616                 }
03617               } else if (GEPOperator *GEP = dyn_cast<GEPOperator>(BEValueV)) {
03618                 // If the increment is an inbounds GEP, then we know the address
03619                 // space cannot be wrapped around. We cannot make any guarantee
03620                 // about signed or unsigned overflow because pointers are
03621                 // unsigned but we may have a negative index from the base
03622                 // pointer. We can guarantee that no unsigned wrap occurs if the
03623                 // indices form a positive value.
03624                 if (GEP->isInBounds() && GEP->getOperand(0) == PN) {
03625                   Flags = setFlags(Flags, SCEV::FlagNW);
03626 
03627                   const SCEV *Ptr = getSCEV(GEP->getPointerOperand());
03628                   if (isKnownPositive(getMinusSCEV(getSCEV(GEP), Ptr)))
03629                     Flags = setFlags(Flags, SCEV::FlagNUW);
03630                 }
03631 
03632                 // We cannot transfer nuw and nsw flags from subtraction
03633                 // operations -- sub nuw X, Y is not the same as add nuw X, -Y
03634                 // for instance.
03635               }
03636 
03637               const SCEV *StartVal = getSCEV(StartValueV);
03638               const SCEV *PHISCEV = getAddRecExpr(StartVal, Accum, L, Flags);
03639 
03640               // Since the no-wrap flags are on the increment, they apply to the
03641               // post-incremented value as well.
03642               if (isLoopInvariant(Accum, L))
03643                 (void)getAddRecExpr(getAddExpr(StartVal, Accum),
03644                                     Accum, L, Flags);
03645 
03646               // Okay, for the entire analysis of this edge we assumed the PHI
03647               // to be symbolic.  We now need to go back and purge all of the
03648               // entries for the scalars that use the symbolic expression.
03649               ForgetSymbolicName(PN, SymbolicName);
03650               ValueExprMap[SCEVCallbackVH(PN, this)] = PHISCEV;
03651               return PHISCEV;
03652             }
03653           }
03654         } else if (const SCEVAddRecExpr *AddRec =
03655                      dyn_cast<SCEVAddRecExpr>(BEValue)) {
03656           // Otherwise, this could be a loop like this:
03657           //     i = 0;  for (j = 1; ..; ++j) { ....  i = j; }
03658           // In this case, j = {1,+,1}  and BEValue is j.
03659           // Because the other in-value of i (0) fits the evolution of BEValue
03660           // i really is an addrec evolution.
03661           if (AddRec->getLoop() == L && AddRec->isAffine()) {
03662             const SCEV *StartVal = getSCEV(StartValueV);
03663 
03664             // If StartVal = j.start - j.stride, we can use StartVal as the
03665             // initial step of the addrec evolution.
03666             if (StartVal == getMinusSCEV(AddRec->getOperand(0),
03667                                          AddRec->getOperand(1))) {
03668               // FIXME: For constant StartVal, we should be able to infer
03669               // no-wrap flags.
03670               const SCEV *PHISCEV =
03671                 getAddRecExpr(StartVal, AddRec->getOperand(1), L,
03672                               SCEV::FlagAnyWrap);
03673 
03674               // Okay, for the entire analysis of this edge we assumed the PHI
03675               // to be symbolic.  We now need to go back and purge all of the
03676               // entries for the scalars that use the symbolic expression.
03677               ForgetSymbolicName(PN, SymbolicName);
03678               ValueExprMap[SCEVCallbackVH(PN, this)] = PHISCEV;
03679               return PHISCEV;
03680             }
03681           }
03682         }
03683       }
03684     }
03685 
03686   // If the PHI has a single incoming value, follow that value, unless the
03687   // PHI's incoming blocks are in a different loop, in which case doing so
03688   // risks breaking LCSSA form. Instcombine would normally zap these, but
03689   // it doesn't have DominatorTree information, so it may miss cases.
03690   if (Value *V =
03691           SimplifyInstruction(PN, F->getParent()->getDataLayout(), TLI, DT, AC))
03692     if (LI->replacementPreservesLCSSAForm(PN, V))
03693       return getSCEV(V);
03694 
03695   // If it's not a loop phi, we can't handle it yet.
03696   return getUnknown(PN);
03697 }
03698 
03699 /// createNodeForGEP - Expand GEP instructions into add and multiply
03700 /// operations. This allows them to be analyzed by regular SCEV code.
03701 ///
03702 const SCEV *ScalarEvolution::createNodeForGEP(GEPOperator *GEP) {
03703   Type *IntPtrTy = getEffectiveSCEVType(GEP->getType());
03704   Value *Base = GEP->getOperand(0);
03705   // Don't attempt to analyze GEPs over unsized objects.
03706   if (!Base->getType()->getPointerElementType()->isSized())
03707     return getUnknown(GEP);
03708 
03709   // Don't blindly transfer the inbounds flag from the GEP instruction to the
03710   // Add expression, because the Instruction may be guarded by control flow
03711   // and the no-overflow bits may not be valid for the expression in any
03712   // context.
03713   SCEV::NoWrapFlags Wrap = GEP->isInBounds() ? SCEV::FlagNSW : SCEV::FlagAnyWrap;
03714 
03715   const SCEV *TotalOffset = getConstant(IntPtrTy, 0);
03716   gep_type_iterator GTI = gep_type_begin(GEP);
03717   for (GetElementPtrInst::op_iterator I = std::next(GEP->op_begin()),
03718                                       E = GEP->op_end();
03719        I != E; ++I) {
03720     Value *Index = *I;
03721     // Compute the (potentially symbolic) offset in bytes for this index.
03722     if (StructType *STy = dyn_cast<StructType>(*GTI++)) {
03723       // For a struct, add the member offset.
03724       unsigned FieldNo = cast<ConstantInt>(Index)->getZExtValue();
03725       const SCEV *FieldOffset = getOffsetOfExpr(IntPtrTy, STy, FieldNo);
03726 
03727       // Add the field offset to the running total offset.
03728       TotalOffset = getAddExpr(TotalOffset, FieldOffset);
03729     } else {
03730       // For an array, add the element offset, explicitly scaled.
03731       const SCEV *ElementSize = getSizeOfExpr(IntPtrTy, *GTI);
03732       const SCEV *IndexS = getSCEV(Index);
03733       // Getelementptr indices are signed.
03734       IndexS = getTruncateOrSignExtend(IndexS, IntPtrTy);
03735 
03736       // Multiply the index by the element size to compute the element offset.
03737       const SCEV *LocalOffset = getMulExpr(IndexS, ElementSize, Wrap);
03738 
03739       // Add the element offset to the running total offset.
03740       TotalOffset = getAddExpr(TotalOffset, LocalOffset);
03741     }
03742   }
03743 
03744   // Get the SCEV for the GEP base.
03745   const SCEV *BaseS = getSCEV(Base);
03746 
03747   // Add the total offset from all the GEP indices to the base.
03748   return getAddExpr(BaseS, TotalOffset, Wrap);
03749 }
03750 
03751 /// GetMinTrailingZeros - Determine the minimum number of zero bits that S is
03752 /// guaranteed to end in (at every loop iteration).  It is, at the same time,
03753 /// the minimum number of times S is divisible by 2.  For example, given {4,+,8}
03754 /// it returns 2.  If S is guaranteed to be 0, it returns the bitwidth of S.
03755 uint32_t
03756 ScalarEvolution::GetMinTrailingZeros(const SCEV *S) {
03757   if (const SCEVConstant *C = dyn_cast<SCEVConstant>(S))
03758     return C->getValue()->getValue().countTrailingZeros();
03759 
03760   if (const SCEVTruncateExpr *T = dyn_cast<SCEVTruncateExpr>(S))
03761     return std::min(GetMinTrailingZeros(T->getOperand()),
03762                     (uint32_t)getTypeSizeInBits(T->getType()));
03763 
03764   if (const SCEVZeroExtendExpr *E = dyn_cast<SCEVZeroExtendExpr>(S)) {
03765     uint32_t OpRes = GetMinTrailingZeros(E->getOperand());
03766     return OpRes == getTypeSizeInBits(E->getOperand()->getType()) ?
03767              getTypeSizeInBits(E->getType()) : OpRes;
03768   }
03769 
03770   if (const SCEVSignExtendExpr *E = dyn_cast<SCEVSignExtendExpr>(S)) {
03771     uint32_t OpRes = GetMinTrailingZeros(E->getOperand());
03772     return OpRes == getTypeSizeInBits(E->getOperand()->getType()) ?
03773              getTypeSizeInBits(E->getType()) : OpRes;
03774   }
03775 
03776   if (const SCEVAddExpr *A = dyn_cast<SCEVAddExpr>(S)) {
03777     // The result is the min of all operands results.
03778     uint32_t MinOpRes = GetMinTrailingZeros(A->getOperand(0));
03779     for (unsigned i = 1, e = A->getNumOperands(); MinOpRes && i != e; ++i)
03780       MinOpRes = std::min(MinOpRes, GetMinTrailingZeros(A->getOperand(i)));
03781     return MinOpRes;
03782   }
03783 
03784   if (const SCEVMulExpr *M = dyn_cast<SCEVMulExpr>(S)) {
03785     // The result is the sum of all operands results.
03786     uint32_t SumOpRes = GetMinTrailingZeros(M->getOperand(0));
03787     uint32_t BitWidth = getTypeSizeInBits(M->getType());
03788     for (unsigned i = 1, e = M->getNumOperands();
03789          SumOpRes != BitWidth && i != e; ++i)
03790       SumOpRes = std::min(SumOpRes + GetMinTrailingZeros(M->getOperand(i)),
03791                           BitWidth);
03792     return SumOpRes;
03793   }
03794 
03795   if (const SCEVAddRecExpr *A = dyn_cast<SCEVAddRecExpr>(S)) {
03796     // The result is the min of all operands results.
03797     uint32_t MinOpRes = GetMinTrailingZeros(A->getOperand(0));
03798     for (unsigned i = 1, e = A->getNumOperands(); MinOpRes && i != e; ++i)
03799       MinOpRes = std::min(MinOpRes, GetMinTrailingZeros(A->getOperand(i)));
03800     return MinOpRes;
03801   }
03802 
03803   if (const SCEVSMaxExpr *M = dyn_cast<SCEVSMaxExpr>(S)) {
03804     // The result is the min of all operands results.
03805     uint32_t MinOpRes = GetMinTrailingZeros(M->getOperand(0));
03806     for (unsigned i = 1, e = M->getNumOperands(); MinOpRes && i != e; ++i)
03807       MinOpRes = std::min(MinOpRes, GetMinTrailingZeros(M->getOperand(i)));
03808     return MinOpRes;
03809   }
03810 
03811   if (const SCEVUMaxExpr *M = dyn_cast<SCEVUMaxExpr>(S)) {
03812     // The result is the min of all operands results.
03813     uint32_t MinOpRes = GetMinTrailingZeros(M->getOperand(0));
03814     for (unsigned i = 1, e = M->getNumOperands(); MinOpRes && i != e; ++i)
03815       MinOpRes = std::min(MinOpRes, GetMinTrailingZeros(M->getOperand(i)));
03816     return MinOpRes;
03817   }
03818 
03819   if (const SCEVUnknown *U = dyn_cast<SCEVUnknown>(S)) {
03820     // For a SCEVUnknown, ask ValueTracking.
03821     unsigned BitWidth = getTypeSizeInBits(U->getType());
03822     APInt Zeros(BitWidth, 0), Ones(BitWidth, 0);
03823     computeKnownBits(U->getValue(), Zeros, Ones,
03824                      F->getParent()->getDataLayout(), 0, AC, nullptr, DT);
03825     return Zeros.countTrailingOnes();
03826   }
03827 
03828   // SCEVUDivExpr
03829   return 0;
03830 }
03831 
03832 /// GetRangeFromMetadata - Helper method to assign a range to V from
03833 /// metadata present in the IR.
03834 static Optional<ConstantRange> GetRangeFromMetadata(Value *V) {
03835   if (Instruction *I = dyn_cast<Instruction>(V)) {
03836     if (MDNode *MD = I->getMetadata(LLVMContext::MD_range)) {
03837       ConstantRange TotalRange(
03838           cast<IntegerType>(I->getType())->getBitWidth(), false);
03839 
03840       unsigned NumRanges = MD->getNumOperands() / 2;
03841       assert(NumRanges >= 1);
03842 
03843       for (unsigned i = 0; i < NumRanges; ++i) {
03844         ConstantInt *Lower =
03845             mdconst::extract<ConstantInt>(MD->getOperand(2 * i + 0));
03846         ConstantInt *Upper =
03847             mdconst::extract<ConstantInt>(MD->getOperand(2 * i + 1));
03848         ConstantRange Range(Lower->getValue(), Upper->getValue());
03849         TotalRange = TotalRange.unionWith(Range);
03850       }
03851 
03852       return TotalRange;
03853     }
03854   }
03855 
03856   return None;
03857 }
03858 
03859 /// getRange - Determine the range for a particular SCEV.  If SignHint is
03860 /// HINT_RANGE_UNSIGNED (resp. HINT_RANGE_SIGNED) then getRange prefers ranges
03861 /// with a "cleaner" unsigned (resp. signed) representation.
03862 ///
03863 ConstantRange
03864 ScalarEvolution::getRange(const SCEV *S,
03865                           ScalarEvolution::RangeSignHint SignHint) {
03866   DenseMap<const SCEV *, ConstantRange> &Cache =
03867       SignHint == ScalarEvolution::HINT_RANGE_UNSIGNED ? UnsignedRanges
03868                                                        : SignedRanges;
03869 
03870   // See if we've computed this range already.
03871   DenseMap<const SCEV *, ConstantRange>::iterator I = Cache.find(S);
03872   if (I != Cache.end())
03873     return I->second;
03874 
03875   if (const SCEVConstant *C = dyn_cast<SCEVConstant>(S))
03876     return setRange(C, SignHint, ConstantRange(C->getValue()->getValue()));
03877 
03878   unsigned BitWidth = getTypeSizeInBits(S->getType());
03879   ConstantRange ConservativeResult(BitWidth, /*isFullSet=*/true);
03880 
03881   // If the value has known zeros, the maximum value will have those known zeros
03882   // as well.
03883   uint32_t TZ = GetMinTrailingZeros(S);
03884   if (TZ != 0) {
03885     if (SignHint == ScalarEvolution::HINT_RANGE_UNSIGNED)
03886       ConservativeResult =
03887           ConstantRange(APInt::getMinValue(BitWidth),
03888                         APInt::getMaxValue(BitWidth).lshr(TZ).shl(TZ) + 1);
03889     else
03890       ConservativeResult = ConstantRange(
03891           APInt::getSignedMinValue(BitWidth),
03892           APInt::getSignedMaxValue(BitWidth).ashr(TZ).shl(TZ) + 1);
03893   }
03894 
03895   if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(S)) {
03896     ConstantRange X = getRange(Add->getOperand(0), SignHint);
03897     for (unsigned i = 1, e = Add->getNumOperands(); i != e; ++i)
03898       X = X.add(getRange(Add->getOperand(i), SignHint));
03899     return setRange(Add, SignHint, ConservativeResult.intersectWith(X));
03900   }
03901 
03902   if (const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(S)) {
03903     ConstantRange X = getRange(Mul->getOperand(0), SignHint);
03904     for (unsigned i = 1, e = Mul->getNumOperands(); i != e; ++i)
03905       X = X.multiply(getRange(Mul->getOperand(i), SignHint));
03906     return setRange(Mul, SignHint, ConservativeResult.intersectWith(X));
03907   }
03908 
03909   if (const SCEVSMaxExpr *SMax = dyn_cast<SCEVSMaxExpr>(S)) {
03910     ConstantRange X = getRange(SMax->getOperand(0), SignHint);
03911     for (unsigned i = 1, e = SMax->getNumOperands(); i != e; ++i)
03912       X = X.smax(getRange(SMax->getOperand(i), SignHint));
03913     return setRange(SMax, SignHint, ConservativeResult.intersectWith(X));
03914   }
03915 
03916   if (const SCEVUMaxExpr *UMax = dyn_cast<SCEVUMaxExpr>(S)) {
03917     ConstantRange X = getRange(UMax->getOperand(0), SignHint);
03918     for (unsigned i = 1, e = UMax->getNumOperands(); i != e; ++i)
03919       X = X.umax(getRange(UMax->getOperand(i), SignHint));
03920     return setRange(UMax, SignHint, ConservativeResult.intersectWith(X));
03921   }
03922 
03923   if (const SCEVUDivExpr *UDiv = dyn_cast<SCEVUDivExpr>(S)) {
03924     ConstantRange X = getRange(UDiv->getLHS(), SignHint);
03925     ConstantRange Y = getRange(UDiv->getRHS(), SignHint);
03926     return setRange(UDiv, SignHint,
03927                     ConservativeResult.intersectWith(X.udiv(Y)));
03928   }
03929 
03930   if (const SCEVZeroExtendExpr *ZExt = dyn_cast<SCEVZeroExtendExpr>(S)) {
03931     ConstantRange X = getRange(ZExt->getOperand(), SignHint);
03932     return setRange(ZExt, SignHint,
03933                     ConservativeResult.intersectWith(X.zeroExtend(BitWidth)));
03934   }
03935 
03936   if (const SCEVSignExtendExpr *SExt = dyn_cast<SCEVSignExtendExpr>(S)) {
03937     ConstantRange X = getRange(SExt->getOperand(), SignHint);
03938     return setRange(SExt, SignHint,
03939                     ConservativeResult.intersectWith(X.signExtend(BitWidth)));
03940   }
03941 
03942   if (const SCEVTruncateExpr *Trunc = dyn_cast<SCEVTruncateExpr>(S)) {
03943     ConstantRange X = getRange(Trunc->getOperand(), SignHint);
03944     return setRange(Trunc, SignHint,
03945                     ConservativeResult.intersectWith(X.truncate(BitWidth)));
03946   }
03947 
03948   if (const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(S)) {
03949     // If there's no unsigned wrap, the value will never be less than its
03950     // initial value.
03951     if (AddRec->getNoWrapFlags(SCEV::FlagNUW))
03952       if (const SCEVConstant *C = dyn_cast<SCEVConstant>(AddRec->getStart()))
03953         if (!C->getValue()->isZero())
03954           ConservativeResult =
03955             ConservativeResult.intersectWith(
03956               ConstantRange(C->getValue()->getValue(), APInt(BitWidth, 0)));
03957 
03958     // If there's no signed wrap, and all the operands have the same sign or
03959     // zero, the value won't ever change sign.
03960     if (AddRec->getNoWrapFlags(SCEV::FlagNSW)) {
03961       bool AllNonNeg = true;
03962       bool AllNonPos = true;
03963       for (unsigned i = 0, e = AddRec->getNumOperands(); i != e; ++i) {
03964         if (!isKnownNonNegative(AddRec->getOperand(i))) AllNonNeg = false;
03965         if (!isKnownNonPositive(AddRec->getOperand(i))) AllNonPos = false;
03966       }
03967       if (AllNonNeg)
03968         ConservativeResult = ConservativeResult.intersectWith(
03969           ConstantRange(APInt(BitWidth, 0),
03970                         APInt::getSignedMinValue(BitWidth)));
03971       else if (AllNonPos)
03972         ConservativeResult = ConservativeResult.intersectWith(
03973           ConstantRange(APInt::getSignedMinValue(BitWidth),
03974                         APInt(BitWidth, 1)));
03975     }
03976 
03977     // TODO: non-affine addrec
03978     if (AddRec->isAffine()) {
03979       Type *Ty = AddRec->getType();
03980       const SCEV *MaxBECount = getMaxBackedgeTakenCount(AddRec->getLoop());
03981       if (!isa<SCEVCouldNotCompute>(MaxBECount) &&
03982           getTypeSizeInBits(MaxBECount->getType()) <= BitWidth) {
03983 
03984         // Check for overflow.  This must be done with ConstantRange arithmetic
03985         // because we could be called from within the ScalarEvolution overflow
03986         // checking code.
03987 
03988         MaxBECount = getNoopOrZeroExtend(MaxBECount, Ty);
03989         ConstantRange MaxBECountRange = getUnsignedRange(MaxBECount);
03990         ConstantRange ZExtMaxBECountRange =
03991             MaxBECountRange.zextOrTrunc(BitWidth * 2 + 1);
03992 
03993         const SCEV *Start = AddRec->getStart();
03994         const SCEV *Step = AddRec->getStepRecurrence(*this);
03995         ConstantRange StepSRange = getSignedRange(Step);
03996         ConstantRange SExtStepSRange = StepSRange.sextOrTrunc(BitWidth * 2 + 1);
03997 
03998         ConstantRange StartURange = getUnsignedRange(Start);
03999         ConstantRange EndURange =
04000             StartURange.add(MaxBECountRange.multiply(StepSRange));
04001 
04002         // Check for unsigned overflow.
04003         ConstantRange ZExtStartURange =
04004             StartURange.zextOrTrunc(BitWidth * 2 + 1);
04005         ConstantRange ZExtEndURange = EndURange.zextOrTrunc(BitWidth * 2 + 1);
04006         if (ZExtStartURange.add(ZExtMaxBECountRange.multiply(SExtStepSRange)) ==
04007             ZExtEndURange) {
04008           APInt Min = APIntOps::umin(StartURange.getUnsignedMin(),
04009                                      EndURange.getUnsignedMin());
04010           APInt Max = APIntOps::umax(StartURange.getUnsignedMax(),
04011                                      EndURange.getUnsignedMax());
04012           bool IsFullRange = Min.isMinValue() && Max.isMaxValue();
04013           if (!IsFullRange)
04014             ConservativeResult =
04015                 ConservativeResult.intersectWith(ConstantRange(Min, Max + 1));
04016         }
04017 
04018         ConstantRange StartSRange = getSignedRange(Start);
04019         ConstantRange EndSRange =
04020             StartSRange.add(MaxBECountRange.multiply(StepSRange));
04021 
04022         // Check for signed overflow. This must be done with ConstantRange
04023         // arithmetic because we could be called from within the ScalarEvolution
04024         // overflow checking code.
04025         ConstantRange SExtStartSRange =
04026             StartSRange.sextOrTrunc(BitWidth * 2 + 1);
04027         ConstantRange SExtEndSRange = EndSRange.sextOrTrunc(BitWidth * 2 + 1);
04028         if (SExtStartSRange.add(ZExtMaxBECountRange.multiply(SExtStepSRange)) ==
04029             SExtEndSRange) {
04030           APInt Min = APIntOps::smin(StartSRange.getSignedMin(),
04031                                      EndSRange.getSignedMin());
04032           APInt Max = APIntOps::smax(StartSRange.getSignedMax(),
04033                                      EndSRange.getSignedMax());
04034           bool IsFullRange = Min.isMinSignedValue() && Max.isMaxSignedValue();
04035           if (!IsFullRange)
04036             ConservativeResult =
04037                 ConservativeResult.intersectWith(ConstantRange(Min, Max + 1));
04038         }
04039       }
04040     }
04041 
04042     return setRange(AddRec, SignHint, ConservativeResult);
04043   }
04044 
04045   if (const SCEVUnknown *U = dyn_cast<SCEVUnknown>(S)) {
04046     // Check if the IR explicitly contains !range metadata.
04047     Optional<ConstantRange> MDRange = GetRangeFromMetadata(U->getValue());
04048     if (MDRange.hasValue())
04049       ConservativeResult = ConservativeResult.intersectWith(MDRange.getValue());
04050 
04051     // Split here to avoid paying the compile-time cost of calling both
04052     // computeKnownBits and ComputeNumSignBits.  This restriction can be lifted
04053     // if needed.
04054     const DataLayout &DL = F->getParent()->getDataLayout();
04055     if (SignHint == ScalarEvolution::HINT_RANGE_UNSIGNED) {
04056       // For a SCEVUnknown, ask ValueTracking.
04057       APInt Zeros(BitWidth, 0), Ones(BitWidth, 0);
04058       computeKnownBits(U->getValue(), Zeros, Ones, DL, 0, AC, nullptr, DT);
04059       if (Ones != ~Zeros + 1)
04060         ConservativeResult =
04061             ConservativeResult.intersectWith(ConstantRange(Ones, ~Zeros + 1));
04062     } else {
04063       assert(SignHint == ScalarEvolution::HINT_RANGE_SIGNED &&
04064              "generalize as needed!");
04065       unsigned NS = ComputeNumSignBits(U->getValue(), DL, 0, AC, nullptr, DT);
04066       if (NS > 1)
04067         ConservativeResult = ConservativeResult.intersectWith(
04068             ConstantRange(APInt::getSignedMinValue(BitWidth).ashr(NS - 1),
04069                           APInt::getSignedMaxValue(BitWidth).ashr(NS - 1) + 1));
04070     }
04071 
04072     return setRange(U, SignHint, ConservativeResult);
04073   }
04074 
04075   return setRange(S, SignHint, ConservativeResult);
04076 }
04077 
04078 /// createSCEV - We know that there is no SCEV for the specified value.
04079 /// Analyze the expression.
04080 ///
04081 const SCEV *ScalarEvolution::createSCEV(Value *V) {
04082   if (!isSCEVable(V->getType()))
04083     return getUnknown(V);
04084 
04085   unsigned Opcode = Instruction::UserOp1;
04086   if (Instruction *I = dyn_cast<Instruction>(V)) {
04087     Opcode = I->getOpcode();
04088 
04089     // Don't attempt to analyze instructions in blocks that aren't
04090     // reachable. Such instructions don't matter, and they aren't required
04091     // to obey basic rules for definitions dominating uses which this
04092     // analysis depends on.
04093     if (!DT->isReachableFromEntry(I->getParent()))
04094       return getUnknown(V);
04095   } else if (ConstantExpr *CE = dyn_cast<ConstantExpr>(V))
04096     Opcode = CE->getOpcode();
04097   else if (ConstantInt *CI = dyn_cast<ConstantInt>(V))
04098     return getConstant(CI);
04099   else if (isa<ConstantPointerNull>(V))
04100     return getConstant(V->getType(), 0);
04101   else if (GlobalAlias *GA = dyn_cast<GlobalAlias>(V))
04102     return GA->mayBeOverridden() ? getUnknown(V) : getSCEV(GA->getAliasee());
04103   else
04104     return getUnknown(V);
04105 
04106   Operator *U = cast<Operator>(V);
04107   switch (Opcode) {
04108   case Instruction::Add: {
04109     // The simple thing to do would be to just call getSCEV on both operands
04110     // and call getAddExpr with the result. However if we're looking at a
04111     // bunch of things all added together, this can be quite inefficient,
04112     // because it leads to N-1 getAddExpr calls for N ultimate operands.
04113     // Instead, gather up all the operands and make a single getAddExpr call.
04114     // LLVM IR canonical form means we need only traverse the left operands.
04115     //
04116     // Don't apply this instruction's NSW or NUW flags to the new
04117     // expression. The instruction may be guarded by control flow that the
04118     // no-wrap behavior depends on. Non-control-equivalent instructions can be
04119     // mapped to the same SCEV expression, and it would be incorrect to transfer
04120     // NSW/NUW semantics to those operations.
04121     SmallVector<const SCEV *, 4> AddOps;
04122     AddOps.push_back(getSCEV(U->getOperand(1)));
04123     for (Value *Op = U->getOperand(0); ; Op = U->getOperand(0)) {
04124       unsigned Opcode = Op->getValueID() - Value::InstructionVal;
04125       if (Opcode != Instruction::Add && Opcode != Instruction::Sub)
04126         break;
04127       U = cast<Operator>(Op);
04128       const SCEV *Op1 = getSCEV(U->getOperand(1));
04129       if (Opcode == Instruction::Sub)
04130         AddOps.push_back(getNegativeSCEV(Op1));
04131       else
04132         AddOps.push_back(Op1);
04133     }
04134     AddOps.push_back(getSCEV(U->getOperand(0)));
04135     return getAddExpr(AddOps);
04136   }
04137   case Instruction::Mul: {
04138     // Don't transfer NSW/NUW for the same reason as AddExpr.
04139     SmallVector<const SCEV *, 4> MulOps;
04140     MulOps.push_back(getSCEV(U->getOperand(1)));
04141     for (Value *Op = U->getOperand(0);
04142          Op->getValueID() == Instruction::Mul + Value::InstructionVal;
04143          Op = U->getOperand(0)) {
04144       U = cast<Operator>(Op);
04145       MulOps.push_back(getSCEV(U->getOperand(1)));
04146     }
04147     MulOps.push_back(getSCEV(U->getOperand(0)));
04148     return getMulExpr(MulOps);
04149   }
04150   case Instruction::UDiv:
04151     return getUDivExpr(getSCEV(U->getOperand(0)),
04152                        getSCEV(U->getOperand(1)));
04153   case Instruction::Sub:
04154     return getMinusSCEV(getSCEV(U->getOperand(0)),
04155                         getSCEV(U->getOperand(1)));
04156   case Instruction::And:
04157     // For an expression like x&255 that merely masks off the high bits,
04158     // use zext(trunc(x)) as the SCEV expression.
04159     if (ConstantInt *CI = dyn_cast<ConstantInt>(U->getOperand(1))) {
04160       if (CI->isNullValue())
04161         return getSCEV(U->getOperand(1));
04162       if (CI->isAllOnesValue())
04163         return getSCEV(U->getOperand(0));
04164       const APInt &A = CI->getValue();
04165 
04166       // Instcombine's ShrinkDemandedConstant may strip bits out of
04167       // constants, obscuring what would otherwise be a low-bits mask.
04168       // Use computeKnownBits to compute what ShrinkDemandedConstant
04169       // knew about to reconstruct a low-bits mask value.
04170       unsigned LZ = A.countLeadingZeros();
04171       unsigned TZ = A.countTrailingZeros();
04172       unsigned BitWidth = A.getBitWidth();
04173       APInt KnownZero(BitWidth, 0), KnownOne(BitWidth, 0);
04174       computeKnownBits(U->getOperand(0), KnownZero, KnownOne,
04175                        F->getParent()->getDataLayout(), 0, AC, nullptr, DT);
04176 
04177       APInt EffectiveMask =
04178           APInt::getLowBitsSet(BitWidth, BitWidth - LZ - TZ).shl(TZ);
04179       if ((LZ != 0 || TZ != 0) && !((~A & ~KnownZero) & EffectiveMask)) {
04180         const SCEV *MulCount = getConstant(
04181             ConstantInt::get(getContext(), APInt::getOneBitSet(BitWidth, TZ)));
04182         return getMulExpr(
04183             getZeroExtendExpr(
04184                 getTruncateExpr(
04185                     getUDivExactExpr(getSCEV(U->getOperand(0)), MulCount),
04186                     IntegerType::get(getContext(), BitWidth - LZ - TZ)),
04187                 U->getType()),
04188             MulCount);
04189       }
04190     }
04191     break;
04192 
04193   case Instruction::Or:
04194     // If the RHS of the Or is a constant, we may have something like:
04195     // X*4+1 which got turned into X*4|1.  Handle this as an Add so loop
04196     // optimizations will transparently handle this case.
04197     //
04198     // In order for this transformation to be safe, the LHS must be of the
04199     // form X*(2^n) and the Or constant must be less than 2^n.
04200     if (ConstantInt *CI = dyn_cast<ConstantInt>(U->getOperand(1))) {
04201       const SCEV *LHS = getSCEV(U->getOperand(0));
04202       const APInt &CIVal = CI->getValue();
04203       if (GetMinTrailingZeros(LHS) >=
04204           (CIVal.getBitWidth() - CIVal.countLeadingZeros())) {
04205         // Build a plain add SCEV.
04206         const SCEV *S = getAddExpr(LHS, getSCEV(CI));
04207         // If the LHS of the add was an addrec and it has no-wrap flags,
04208         // transfer the no-wrap flags, since an or won't introduce a wrap.
04209         if (const SCEVAddRecExpr *NewAR = dyn_cast<SCEVAddRecExpr>(S)) {
04210           const SCEVAddRecExpr *OldAR = cast<SCEVAddRecExpr>(LHS);
04211           const_cast<SCEVAddRecExpr *>(NewAR)->setNoWrapFlags(
04212             OldAR->getNoWrapFlags());
04213         }
04214         return S;
04215       }
04216     }
04217     break;
04218   case Instruction::Xor:
04219     if (ConstantInt *CI = dyn_cast<ConstantInt>(U->getOperand(1))) {
04220       // If the RHS of the xor is a signbit, then this is just an add.
04221       // Instcombine turns add of signbit into xor as a strength reduction step.
04222       if (CI->getValue().isSignBit())
04223         return getAddExpr(getSCEV(U->getOperand(0)),
04224                           getSCEV(U->getOperand(1)));
04225 
04226       // If the RHS of xor is -1, then this is a not operation.
04227       if (CI->isAllOnesValue())
04228         return getNotSCEV(getSCEV(U->getOperand(0)));
04229 
04230       // Model xor(and(x, C), C) as and(~x, C), if C is a low-bits mask.
04231       // This is a variant of the check for xor with -1, and it handles
04232       // the case where instcombine has trimmed non-demanded bits out
04233       // of an xor with -1.
04234       if (BinaryOperator *BO = dyn_cast<BinaryOperator>(U->getOperand(0)))
04235         if (ConstantInt *LCI = dyn_cast<ConstantInt>(BO->getOperand(1)))
04236           if (BO->getOpcode() == Instruction::And &&
04237               LCI->getValue() == CI->getValue())
04238             if (const SCEVZeroExtendExpr *Z =
04239                   dyn_cast<SCEVZeroExtendExpr>(getSCEV(U->getOperand(0)))) {
04240               Type *UTy = U->getType();
04241               const SCEV *Z0 = Z->getOperand();
04242               Type *Z0Ty = Z0->getType();
04243               unsigned Z0TySize = getTypeSizeInBits(Z0Ty);
04244 
04245               // If C is a low-bits mask, the zero extend is serving to
04246               // mask off the high bits. Complement the operand and
04247               // re-apply the zext.
04248               if (APIntOps::isMask(Z0TySize, CI->getValue()))
04249                 return getZeroExtendExpr(getNotSCEV(Z0), UTy);
04250 
04251               // If C is a single bit, it may be in the sign-bit position
04252               // before the zero-extend. In this case, represent the xor
04253               // using an add, which is equivalent, and re-apply the zext.
04254               APInt Trunc = CI->getValue().trunc(Z0TySize);
04255               if (Trunc.zext(getTypeSizeInBits(UTy)) == CI->getValue() &&
04256                   Trunc.isSignBit())
04257                 return getZeroExtendExpr(getAddExpr(Z0, getConstant(Trunc)),
04258                                          UTy);
04259             }
04260     }
04261     break;
04262 
04263   case Instruction::Shl:
04264     // Turn shift left of a constant amount into a multiply.
04265     if (ConstantInt *SA = dyn_cast<ConstantInt>(U->getOperand(1))) {
04266       uint32_t BitWidth = cast<IntegerType>(U->getType())->getBitWidth();
04267 
04268       // If the shift count is not less than the bitwidth, the result of
04269       // the shift is undefined. Don't try to analyze it, because the
04270       // resolution chosen here may differ from the resolution chosen in
04271       // other parts of the compiler.
04272       if (SA->getValue().uge(BitWidth))
04273         break;
04274 
04275       Constant *X = ConstantInt::get(getContext(),
04276         APInt::getOneBitSet(BitWidth, SA->getZExtValue()));
04277       return getMulExpr(getSCEV(U->getOperand(0)), getSCEV(X));
04278     }
04279     break;
04280 
04281   case Instruction::LShr:
04282     // Turn logical shift right of a constant into a unsigned divide.
04283     if (ConstantInt *SA = dyn_cast<ConstantInt>(U->getOperand(1))) {
04284       uint32_t BitWidth = cast<IntegerType>(U->getType())->getBitWidth();
04285 
04286       // If the shift count is not less than the bitwidth, the result of
04287       // the shift is undefined. Don't try to analyze it, because the
04288       // resolution chosen here may differ from the resolution chosen in
04289       // other parts of the compiler.
04290       if (SA->getValue().uge(BitWidth))
04291         break;
04292 
04293       Constant *X = ConstantInt::get(getContext(),
04294         APInt::getOneBitSet(BitWidth, SA->getZExtValue()));
04295       return getUDivExpr(getSCEV(U->getOperand(0)), getSCEV(X));
04296     }
04297     break;
04298 
04299   case Instruction::AShr:
04300     // For a two-shift sext-inreg, use sext(trunc(x)) as the SCEV expression.
04301     if (ConstantInt *CI = dyn_cast<ConstantInt>(U->getOperand(1)))
04302       if (Operator *L = dyn_cast<Operator>(U->getOperand(0)))
04303         if (L->getOpcode() == Instruction::Shl &&
04304             L->getOperand(1) == U->getOperand(1)) {
04305           uint64_t BitWidth = getTypeSizeInBits(U->getType());
04306 
04307           // If the shift count is not less than the bitwidth, the result of
04308           // the shift is undefined. Don't try to analyze it, because the
04309           // resolution chosen here may differ from the resolution chosen in
04310           // other parts of the compiler.
04311           if (CI->getValue().uge(BitWidth))
04312             break;
04313 
04314           uint64_t Amt = BitWidth - CI->getZExtValue();
04315           if (Amt == BitWidth)
04316             return getSCEV(L->getOperand(0));       // shift by zero --> noop
04317           return
04318             getSignExtendExpr(getTruncateExpr(getSCEV(L->getOperand(0)),
04319                                               IntegerType::get(getContext(),
04320                                                                Amt)),
04321                               U->getType());
04322         }
04323     break;
04324 
04325   case Instruction::Trunc:
04326     return getTruncateExpr(getSCEV(U->getOperand(0)), U->getType());
04327 
04328   case Instruction::ZExt:
04329     return getZeroExtendExpr(getSCEV(U->getOperand(0)), U->getType());
04330 
04331   case Instruction::SExt:
04332     return getSignExtendExpr(getSCEV(U->getOperand(0)), U->getType());
04333 
04334   case Instruction::BitCast:
04335     // BitCasts are no-op casts so we just eliminate the cast.
04336     if (isSCEVable(U->getType()) && isSCEVable(U->getOperand(0)->getType()))
04337       return getSCEV(U->getOperand(0));
04338     break;
04339 
04340   // It's tempting to handle inttoptr and ptrtoint as no-ops, however this can
04341   // lead to pointer expressions which cannot safely be expanded to GEPs,
04342   // because ScalarEvolution doesn't respect the GEP aliasing rules when
04343   // simplifying integer expressions.
04344 
04345   case Instruction::GetElementPtr:
04346     return createNodeForGEP(cast<GEPOperator>(U));
04347 
04348   case Instruction::PHI:
04349     return createNodeForPHI(cast<PHINode>(U));
04350 
04351   case Instruction::Select:
04352     // This could be a smax or umax that was lowered earlier.
04353     // Try to recover it.
04354     if (ICmpInst *ICI = dyn_cast<ICmpInst>(U->getOperand(0))) {
04355       Value *LHS = ICI->getOperand(0);
04356       Value *RHS = ICI->getOperand(1);
04357       switch (ICI->getPredicate()) {
04358       case ICmpInst::ICMP_SLT:
04359       case ICmpInst::ICMP_SLE:
04360         std::swap(LHS, RHS);
04361         // fall through
04362       case ICmpInst::ICMP_SGT:
04363       case ICmpInst::ICMP_SGE:
04364         // a >s b ? a+x : b+x  ->  smax(a, b)+x
04365         // a >s b ? b+x : a+x  ->  smin(a, b)+x
04366         if (getTypeSizeInBits(LHS->getType()) <=
04367             getTypeSizeInBits(U->getType())) {
04368           const SCEV *LS = getNoopOrSignExtend(getSCEV(LHS), U->getType());
04369           const SCEV *RS = getNoopOrSignExtend(getSCEV(RHS), U->getType());
04370           const SCEV *LA = getSCEV(U->getOperand(1));
04371           const SCEV *RA = getSCEV(U->getOperand(2));
04372           const SCEV *LDiff = getMinusSCEV(LA, LS);
04373           const SCEV *RDiff = getMinusSCEV(RA, RS);
04374           if (LDiff == RDiff)
04375             return getAddExpr(getSMaxExpr(LS, RS), LDiff);
04376           LDiff = getMinusSCEV(LA, RS);
04377           RDiff = getMinusSCEV(RA, LS);
04378           if (LDiff == RDiff)
04379             return getAddExpr(getSMinExpr(LS, RS), LDiff);
04380         }
04381         break;
04382       case ICmpInst::ICMP_ULT:
04383       case ICmpInst::ICMP_ULE:
04384         std::swap(LHS, RHS);
04385         // fall through
04386       case ICmpInst::ICMP_UGT:
04387       case ICmpInst::ICMP_UGE:
04388         // a >u b ? a+x : b+x  ->  umax(a, b)+x
04389         // a >u b ? b+x : a+x  ->  umin(a, b)+x
04390         if (getTypeSizeInBits(LHS->getType()) <=
04391             getTypeSizeInBits(U->getType())) {
04392           const SCEV *LS = getNoopOrZeroExtend(getSCEV(LHS), U->getType());
04393           const SCEV *RS = getNoopOrZeroExtend(getSCEV(RHS), U->getType());
04394           const SCEV *LA = getSCEV(U->getOperand(1));
04395           const SCEV *RA = getSCEV(U->getOperand(2));
04396           const SCEV *LDiff = getMinusSCEV(LA, LS);
04397           const SCEV *RDiff = getMinusSCEV(RA, RS);
04398           if (LDiff == RDiff)
04399             return getAddExpr(getUMaxExpr(LS, RS), LDiff);
04400           LDiff = getMinusSCEV(LA, RS);
04401           RDiff = getMinusSCEV(RA, LS);
04402           if (LDiff == RDiff)
04403             return getAddExpr(getUMinExpr(LS, RS), LDiff);
04404         }
04405         break;
04406       case ICmpInst::ICMP_NE:
04407         // n != 0 ? n+x : 1+x  ->  umax(n, 1)+x
04408         if (getTypeSizeInBits(LHS->getType()) <=
04409                 getTypeSizeInBits(U->getType()) &&
04410             isa<ConstantInt>(RHS) && cast<ConstantInt>(RHS)->isZero()) {
04411           const SCEV *One = getConstant(U->getType(), 1);
04412           const SCEV *LS = getNoopOrZeroExtend(getSCEV(LHS), U->getType());
04413           const SCEV *LA = getSCEV(U->getOperand(1));
04414           const SCEV *RA = getSCEV(U->getOperand(2));
04415           const SCEV *LDiff = getMinusSCEV(LA, LS);
04416           const SCEV *RDiff = getMinusSCEV(RA, One);
04417           if (LDiff == RDiff)
04418             return getAddExpr(getUMaxExpr(One, LS), LDiff);
04419         }
04420         break;
04421       case ICmpInst::ICMP_EQ:
04422         // n == 0 ? 1+x : n+x  ->  umax(n, 1)+x
04423         if (getTypeSizeInBits(LHS->getType()) <=
04424                 getTypeSizeInBits(U->getType()) &&
04425             isa<ConstantInt>(RHS) && cast<ConstantInt>(RHS)->isZero()) {
04426           const SCEV *One = getConstant(U->getType(), 1);
04427           const SCEV *LS = getNoopOrZeroExtend(getSCEV(LHS), U->getType());
04428           const SCEV *LA = getSCEV(U->getOperand(1));
04429           const SCEV *RA = getSCEV(U->getOperand(2));
04430           const SCEV *LDiff = getMinusSCEV(LA, One);
04431           const SCEV *RDiff = getMinusSCEV(RA, LS);
04432           if (LDiff == RDiff)
04433             return getAddExpr(getUMaxExpr(One, LS), LDiff);
04434         }
04435         break;
04436       default:
04437         break;
04438       }
04439     }
04440 
04441   default: // We cannot analyze this expression.
04442     break;
04443   }
04444 
04445   return getUnknown(V);
04446 }
04447 
04448 
04449 
04450 //===----------------------------------------------------------------------===//
04451 //                   Iteration Count Computation Code
04452 //
04453 
04454 unsigned ScalarEvolution::getSmallConstantTripCount(Loop *L) {
04455   if (BasicBlock *ExitingBB = L->getExitingBlock())
04456     return getSmallConstantTripCount(L, ExitingBB);
04457 
04458   // No trip count information for multiple exits.
04459   return 0;
04460 }
04461 
04462 /// getSmallConstantTripCount - Returns the maximum trip count of this loop as a
04463 /// normal unsigned value. Returns 0 if the trip count is unknown or not
04464 /// constant. Will also return 0 if the maximum trip count is very large (>=
04465 /// 2^32).
04466 ///
04467 /// This "trip count" assumes that control exits via ExitingBlock. More
04468 /// precisely, it is the number of times that control may reach ExitingBlock
04469 /// before taking the branch. For loops with multiple exits, it may not be the
04470 /// number times that the loop header executes because the loop may exit
04471 /// prematurely via another branch.
04472 unsigned ScalarEvolution::getSmallConstantTripCount(Loop *L,
04473                                                     BasicBlock *ExitingBlock) {
04474   assert(ExitingBlock && "Must pass a non-null exiting block!");
04475   assert(L->isLoopExiting(ExitingBlock) &&
04476          "Exiting block must actually branch out of the loop!");
04477   const SCEVConstant *ExitCount =
04478       dyn_cast<SCEVConstant>(getExitCount(L, ExitingBlock));
04479   if (!ExitCount)
04480     return 0;
04481 
04482   ConstantInt *ExitConst = ExitCount->getValue();
04483 
04484   // Guard against huge trip counts.
04485   if (ExitConst->getValue().getActiveBits() > 32)
04486     return 0;
04487 
04488   // In case of integer overflow, this returns 0, which is correct.
04489   return ((unsigned)ExitConst->getZExtValue()) + 1;
04490 }
04491 
04492 unsigned ScalarEvolution::getSmallConstantTripMultiple(Loop *L) {
04493   if (BasicBlock *ExitingBB = L->getExitingBlock())
04494     return getSmallConstantTripMultiple(L, ExitingBB);
04495 
04496   // No trip multiple information for multiple exits.
04497   return 0;
04498 }
04499 
04500 /// getSmallConstantTripMultiple - Returns the largest constant divisor of the
04501 /// trip count of this loop as a normal unsigned value, if possible. This
04502 /// means that the actual trip count is always a multiple of the returned
04503 /// value (don't forget the trip count could very well be zero as well!).
04504 ///
04505 /// Returns 1 if the trip count is unknown or not guaranteed to be the
04506 /// multiple of a constant (which is also the case if the trip count is simply
04507 /// constant, use getSmallConstantTripCount for that case), Will also return 1
04508 /// if the trip count is very large (>= 2^32).
04509 ///
04510 /// As explained in the comments for getSmallConstantTripCount, this assumes
04511 /// that control exits the loop via ExitingBlock.
04512 unsigned
04513 ScalarEvolution::getSmallConstantTripMultiple(Loop *L,
04514                                               BasicBlock *ExitingBlock) {
04515   assert(ExitingBlock && "Must pass a non-null exiting block!");
04516   assert(L->isLoopExiting(ExitingBlock) &&
04517          "Exiting block must actually branch out of the loop!");
04518   const SCEV *ExitCount = getExitCount(L, ExitingBlock);
04519   if (ExitCount == getCouldNotCompute())
04520     return 1;
04521 
04522   // Get the trip count from the BE count by adding 1.
04523   const SCEV *TCMul = getAddExpr(ExitCount,
04524                                  getConstant(ExitCount->getType(), 1));
04525   // FIXME: SCEV distributes multiplication as V1*C1 + V2*C1. We could attempt
04526   // to factor simple cases.
04527   if (const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(TCMul))
04528     TCMul = Mul->getOperand(0);
04529 
04530   const SCEVConstant *MulC = dyn_cast<SCEVConstant>(TCMul);
04531   if (!MulC)
04532     return 1;
04533 
04534   ConstantInt *Result = MulC->getValue();
04535 
04536   // Guard against huge trip counts (this requires checking
04537   // for zero to handle the case where the trip count == -1 and the
04538   // addition wraps).
04539   if (!Result || Result->getValue().getActiveBits() > 32 ||
04540       Result->getValue().getActiveBits() == 0)
04541     return 1;
04542 
04543   return (unsigned)Result->getZExtValue();
04544 }
04545 
04546 // getExitCount - Get the expression for the number of loop iterations for which
04547 // this loop is guaranteed not to exit via ExitingBlock. Otherwise return
04548 // SCEVCouldNotCompute.
04549 const SCEV *ScalarEvolution::getExitCount(Loop *L, BasicBlock *ExitingBlock) {
04550   return getBackedgeTakenInfo(L).getExact(ExitingBlock, this);
04551 }
04552 
04553 /// getBackedgeTakenCount - If the specified loop has a predictable
04554 /// backedge-taken count, return it, otherwise return a SCEVCouldNotCompute
04555 /// object. The backedge-taken count is the number of times the loop header
04556 /// will be branched to from within the loop. This is one less than the
04557 /// trip count of the loop, since it doesn't count the first iteration,
04558 /// when the header is branched to from outside the loop.
04559 ///
04560 /// Note that it is not valid to call this method on a loop without a
04561 /// loop-invariant backedge-taken count (see
04562 /// hasLoopInvariantBackedgeTakenCount).
04563 ///
04564 const SCEV *ScalarEvolution::getBackedgeTakenCount(const Loop *L) {
04565   return getBackedgeTakenInfo(L).getExact(this);
04566 }
04567 
04568 /// getMaxBackedgeTakenCount - Similar to getBackedgeTakenCount, except
04569 /// return the least SCEV value that is known never to be less than the
04570 /// actual backedge taken count.
04571 const SCEV *ScalarEvolution::getMaxBackedgeTakenCount(const Loop *L) {
04572   return getBackedgeTakenInfo(L).getMax(this);
04573 }
04574 
04575 /// PushLoopPHIs - Push PHI nodes in the header of the given loop
04576 /// onto the given Worklist.
04577 static void
04578 PushLoopPHIs(const Loop *L, SmallVectorImpl<Instruction *> &Worklist) {
04579   BasicBlock *Header = L->getHeader();
04580 
04581   // Push all Loop-header PHIs onto the Worklist stack.
04582   for (BasicBlock::iterator I = Header->begin();
04583        PHINode *PN = dyn_cast<PHINode>(I); ++I)
04584     Worklist.push_back(PN);
04585 }
04586 
04587 const ScalarEvolution::BackedgeTakenInfo &
04588 ScalarEvolution::getBackedgeTakenInfo(const Loop *L) {
04589   // Initially insert an invalid entry for this loop. If the insertion
04590   // succeeds, proceed to actually compute a backedge-taken count and
04591   // update the value. The temporary CouldNotCompute value tells SCEV
04592   // code elsewhere that it shouldn't attempt to request a new
04593   // backedge-taken count, which could result in infinite recursion.
04594   std::pair<DenseMap<const Loop *, BackedgeTakenInfo>::iterator, bool> Pair =
04595     BackedgeTakenCounts.insert(std::make_pair(L, BackedgeTakenInfo()));
04596   if (!Pair.second)
04597     return Pair.first->second;
04598 
04599   // ComputeBackedgeTakenCount may allocate memory for its result. Inserting it
04600   // into the BackedgeTakenCounts map transfers ownership. Otherwise, the result
04601   // must be cleared in this scope.
04602   BackedgeTakenInfo Result = ComputeBackedgeTakenCount(L);
04603 
04604   if (Result.getExact(this) != getCouldNotCompute()) {
04605     assert(isLoopInvariant(Result.getExact(this), L) &&
04606            isLoopInvariant(Result.getMax(this), L) &&
04607            "Computed backedge-taken count isn't loop invariant for loop!");
04608     ++NumTripCountsComputed;
04609   }
04610   else if (Result.getMax(this) == getCouldNotCompute() &&
04611            isa<PHINode>(L->getHeader()->begin())) {
04612     // Only count loops that have phi nodes as not being computable.
04613     ++NumTripCountsNotComputed;
04614   }
04615 
04616   // Now that we know more about the trip count for this loop, forget any
04617   // existing SCEV values for PHI nodes in this loop since they are only
04618   // conservative estimates made without the benefit of trip count
04619   // information. This is similar to the code in forgetLoop, except that
04620   // it handles SCEVUnknown PHI nodes specially.
04621   if (Result.hasAnyInfo()) {
04622     SmallVector<Instruction *, 16> Worklist;
04623     PushLoopPHIs(L, Worklist);
04624 
04625     SmallPtrSet<Instruction *, 8> Visited;
04626     while (!Worklist.empty()) {
04627       Instruction *I = Worklist.pop_back_val();
04628       if (!Visited.insert(I).second)
04629         continue;
04630 
04631       ValueExprMapType::iterator It =
04632         ValueExprMap.find_as(static_cast<Value *>(I));
04633       if (It != ValueExprMap.end()) {
04634         const SCEV *Old = It->second;
04635 
04636         // SCEVUnknown for a PHI either means that it has an unrecognized
04637         // structure, or it's a PHI that's in the progress of being computed
04638         // by createNodeForPHI.  In the former case, additional loop trip
04639         // count information isn't going to change anything. In the later
04640         // case, createNodeForPHI will perform the necessary updates on its
04641         // own when it gets to that point.
04642         if (!isa<PHINode>(I) || !isa<SCEVUnknown>(Old)) {
04643           forgetMemoizedResults(Old);
04644           ValueExprMap.erase(It);
04645         }
04646         if (PHINode *PN = dyn_cast<PHINode>(I))
04647           ConstantEvolutionLoopExitValue.erase(PN);
04648       }
04649 
04650       PushDefUseChildren(I, Worklist);
04651     }
04652   }
04653 
04654   // Re-lookup the insert position, since the call to
04655   // ComputeBackedgeTakenCount above could result in a
04656   // recusive call to getBackedgeTakenInfo (on a different
04657   // loop), which would invalidate the iterator computed
04658   // earlier.
04659   return BackedgeTakenCounts.find(L)->second = Result;
04660 }
04661 
04662 /// forgetLoop - This method should be called by the client when it has
04663 /// changed a loop in a way that may effect ScalarEvolution's ability to
04664 /// compute a trip count, or if the loop is deleted.
04665 void ScalarEvolution::forgetLoop(const Loop *L) {
04666   // Drop any stored trip count value.
04667   DenseMap<const Loop*, BackedgeTakenInfo>::iterator BTCPos =
04668     BackedgeTakenCounts.find(L);
04669   if (BTCPos != BackedgeTakenCounts.end()) {
04670     BTCPos->second.clear();
04671     BackedgeTakenCounts.erase(BTCPos);
04672   }
04673 
04674   // Drop information about expressions based on loop-header PHIs.
04675   SmallVector<Instruction *, 16> Worklist;
04676   PushLoopPHIs(L, Worklist);
04677 
04678   SmallPtrSet<Instruction *, 8> Visited;
04679   while (!Worklist.empty()) {
04680     Instruction *I = Worklist.pop_back_val();
04681     if (!Visited.insert(I).second)
04682       continue;
04683 
04684     ValueExprMapType::iterator It =
04685       ValueExprMap.find_as(static_cast<Value *>(I));
04686     if (It != ValueExprMap.end()) {
04687       forgetMemoizedResults(It->second);
04688       ValueExprMap.erase(It);
04689       if (PHINode *PN = dyn_cast<PHINode>(I))
04690         ConstantEvolutionLoopExitValue.erase(PN);
04691     }
04692 
04693     PushDefUseChildren(I, Worklist);
04694   }
04695 
04696   // Forget all contained loops too, to avoid dangling entries in the
04697   // ValuesAtScopes map.
04698   for (Loop::iterator I = L->begin(), E = L->end(); I != E; ++I)
04699     forgetLoop(*I);
04700 }
04701 
04702 /// forgetValue - This method should be called by the client when it has
04703 /// changed a value in a way that may effect its value, or which may
04704 /// disconnect it from a def-use chain linking it to a loop.
04705 void ScalarEvolution::forgetValue(Value *V) {
04706   Instruction *I = dyn_cast<Instruction>(V);
04707   if (!I) return;
04708 
04709   // Drop information about expressions based on loop-header PHIs.
04710   SmallVector<Instruction *, 16> Worklist;
04711   Worklist.push_back(I);
04712 
04713   SmallPtrSet<Instruction *, 8> Visited;
04714   while (!Worklist.empty()) {
04715     I = Worklist.pop_back_val();
04716     if (!Visited.insert(I).second)
04717       continue;
04718 
04719     ValueExprMapType::iterator It =
04720       ValueExprMap.find_as(static_cast<Value *>(I));
04721     if (It != ValueExprMap.end()) {
04722       forgetMemoizedResults(It->second);
04723       ValueExprMap.erase(It);
04724       if (PHINode *PN = dyn_cast<PHINode>(I))
04725         ConstantEvolutionLoopExitValue.erase(PN);
04726     }
04727 
04728     PushDefUseChildren(I, Worklist);
04729   }
04730 }
04731 
04732 /// getExact - Get the exact loop backedge taken count considering all loop
04733 /// exits. A computable result can only be return for loops with a single exit.
04734 /// Returning the minimum taken count among all exits is incorrect because one
04735 /// of the loop's exit limit's may have been skipped. HowFarToZero assumes that
04736 /// the limit of each loop test is never skipped. This is a valid assumption as
04737 /// long as the loop exits via that test. For precise results, it is the
04738 /// caller's responsibility to specify the relevant loop exit using
04739 /// getExact(ExitingBlock, SE).
04740 const SCEV *
04741 ScalarEvolution::BackedgeTakenInfo::getExact(ScalarEvolution *SE) const {
04742   // If any exits were not computable, the loop is not computable.
04743   if (!ExitNotTaken.isCompleteList()) return SE->getCouldNotCompute();
04744 
04745   // We need exactly one computable exit.
04746   if (!ExitNotTaken.ExitingBlock) return SE->getCouldNotCompute();
04747   assert(ExitNotTaken.ExactNotTaken && "uninitialized not-taken info");
04748 
04749   const SCEV *BECount = nullptr;
04750   for (const ExitNotTakenInfo *ENT = &ExitNotTaken;
04751        ENT != nullptr; ENT = ENT->getNextExit()) {
04752 
04753     assert(ENT->ExactNotTaken != SE->getCouldNotCompute() && "bad exit SCEV");
04754 
04755     if (!BECount)
04756       BECount = ENT->ExactNotTaken;
04757     else if (BECount != ENT->ExactNotTaken)
04758       return SE->getCouldNotCompute();
04759   }
04760   assert(BECount && "Invalid not taken count for loop exit");
04761   return BECount;
04762 }
04763 
04764 /// getExact - Get the exact not taken count for this loop exit.
04765 const SCEV *
04766 ScalarEvolution::BackedgeTakenInfo::getExact(BasicBlock *ExitingBlock,
04767                                              ScalarEvolution *SE) const {
04768   for (const ExitNotTakenInfo *ENT = &ExitNotTaken;
04769        ENT != nullptr; ENT = ENT->getNextExit()) {
04770 
04771     if (ENT->ExitingBlock == ExitingBlock)
04772       return ENT->ExactNotTaken;
04773   }
04774   return SE->getCouldNotCompute();
04775 }
04776 
04777 /// getMax - Get the max backedge taken count for the loop.
04778 const SCEV *
04779 ScalarEvolution::BackedgeTakenInfo::getMax(ScalarEvolution *SE) const {
04780   return Max ? Max : SE->getCouldNotCompute();
04781 }
04782 
04783 bool ScalarEvolution::BackedgeTakenInfo::hasOperand(const SCEV *S,
04784                                                     ScalarEvolution *SE) const {
04785   if (Max && Max != SE->getCouldNotCompute() && SE->hasOperand(Max, S))
04786     return true;
04787 
04788   if (!ExitNotTaken.ExitingBlock)
04789     return false;
04790 
04791   for (const ExitNotTakenInfo *ENT = &ExitNotTaken;
04792        ENT != nullptr; ENT = ENT->getNextExit()) {
04793 
04794     if (ENT->ExactNotTaken != SE->getCouldNotCompute()
04795         && SE->hasOperand(ENT->ExactNotTaken, S)) {
04796       return true;
04797     }
04798   }
04799   return false;
04800 }
04801 
04802 /// Allocate memory for BackedgeTakenInfo and copy the not-taken count of each
04803 /// computable exit into a persistent ExitNotTakenInfo array.
04804 ScalarEvolution::BackedgeTakenInfo::BackedgeTakenInfo(
04805   SmallVectorImpl< std::pair<BasicBlock *, const SCEV *> > &ExitCounts,
04806   bool Complete, const SCEV *MaxCount) : Max(MaxCount) {
04807 
04808   if (!Complete)
04809     ExitNotTaken.setIncomplete();
04810 
04811   unsigned NumExits = ExitCounts.size();
04812   if (NumExits == 0) return;
04813 
04814   ExitNotTaken.ExitingBlock = ExitCounts[0].first;
04815   ExitNotTaken.ExactNotTaken = ExitCounts[0].second;
04816   if (NumExits == 1) return;
04817 
04818   // Handle the rare case of multiple computable exits.
04819   ExitNotTakenInfo *ENT = new ExitNotTakenInfo[NumExits-1];
04820 
04821   ExitNotTakenInfo *PrevENT = &ExitNotTaken;
04822   for (unsigned i = 1; i < NumExits; ++i, PrevENT = ENT, ++ENT) {
04823     PrevENT->setNextExit(ENT);
04824     ENT->ExitingBlock = ExitCounts[i].first;
04825     ENT->ExactNotTaken = ExitCounts[i].second;
04826   }
04827 }
04828 
04829 /// clear - Invalidate this result and free the ExitNotTakenInfo array.
04830 void ScalarEvolution::BackedgeTakenInfo::clear() {
04831   ExitNotTaken.ExitingBlock = nullptr;
04832   ExitNotTaken.ExactNotTaken = nullptr;
04833   delete[] ExitNotTaken.getNextExit();
04834 }
04835 
04836 /// ComputeBackedgeTakenCount - Compute the number of times the backedge
04837 /// of the specified loop will execute.
04838 ScalarEvolution::BackedgeTakenInfo
04839 ScalarEvolution::ComputeBackedgeTakenCount(const Loop *L) {
04840   SmallVector<BasicBlock *, 8> ExitingBlocks;
04841   L->getExitingBlocks(ExitingBlocks);
04842 
04843   SmallVector<std::pair<BasicBlock *, const SCEV *>, 4> ExitCounts;
04844   bool CouldComputeBECount = true;
04845   BasicBlock *Latch = L->getLoopLatch(); // may be NULL.
04846   const SCEV *MustExitMaxBECount = nullptr;
04847   const SCEV *MayExitMaxBECount = nullptr;
04848 
04849   // Compute the ExitLimit for each loop exit. Use this to populate ExitCounts
04850   // and compute maxBECount.
04851   for (unsigned i = 0, e = ExitingBlocks.size(); i != e; ++i) {
04852     BasicBlock *ExitBB = ExitingBlocks[i];
04853     ExitLimit EL = ComputeExitLimit(L, ExitBB);
04854 
04855     // 1. For each exit that can be computed, add an entry to ExitCounts.
04856     // CouldComputeBECount is true only if all exits can be computed.
04857     if (EL.Exact == getCouldNotCompute())
04858       // We couldn't compute an exact value for this exit, so
04859       // we won't be able to compute an exact value for the loop.
04860       CouldComputeBECount = false;
04861     else
04862       ExitCounts.push_back(std::make_pair(ExitBB, EL.Exact));
04863 
04864     // 2. Derive the loop's MaxBECount from each exit's max number of
04865     // non-exiting iterations. Partition the loop exits into two kinds:
04866     // LoopMustExits and LoopMayExits.
04867     //
04868     // If the exit dominates the loop latch, it is a LoopMustExit otherwise it
04869     // is a LoopMayExit.  If any computable LoopMustExit is found, then
04870     // MaxBECount is the minimum EL.Max of computable LoopMustExits. Otherwise,
04871     // MaxBECount is conservatively the maximum EL.Max, where CouldNotCompute is
04872     // considered greater than any computable EL.Max.
04873     if (EL.Max != getCouldNotCompute() && Latch &&
04874         DT->dominates(ExitBB, Latch)) {
04875       if (!MustExitMaxBECount)
04876         MustExitMaxBECount = EL.Max;
04877       else {
04878         MustExitMaxBECount =
04879           getUMinFromMismatchedTypes(MustExitMaxBECount, EL.Max);
04880       }
04881     } else if (MayExitMaxBECount != getCouldNotCompute()) {
04882       if (!MayExitMaxBECount || EL.Max == getCouldNotCompute())
04883         MayExitMaxBECount = EL.Max;
04884       else {
04885         MayExitMaxBECount =
04886           getUMaxFromMismatchedTypes(MayExitMaxBECount, EL.Max);
04887       }
04888     }
04889   }
04890   const SCEV *MaxBECount = MustExitMaxBECount ? MustExitMaxBECount :
04891     (MayExitMaxBECount ? MayExitMaxBECount : getCouldNotCompute());
04892   return BackedgeTakenInfo(ExitCounts, CouldComputeBECount, MaxBECount);
04893 }
04894 
04895 /// ComputeExitLimit - Compute the number of times the backedge of the specified
04896 /// loop will execute if it exits via the specified block.
04897 ScalarEvolution::ExitLimit
04898 ScalarEvolution::ComputeExitLimit(const Loop *L, BasicBlock *ExitingBlock) {
04899 
04900   // Okay, we've chosen an exiting block.  See what condition causes us to
04901   // exit at this block and remember the exit block and whether all other targets
04902   // lead to the loop header.
04903   bool MustExecuteLoopHeader = true;
04904   BasicBlock *Exit = nullptr;
04905   for (succ_iterator SI = succ_begin(ExitingBlock), SE = succ_end(ExitingBlock);
04906        SI != SE; ++SI)
04907     if (!L->contains(*SI)) {
04908       if (Exit) // Multiple exit successors.
04909         return getCouldNotCompute();
04910       Exit = *SI;
04911     } else if (*SI != L->getHeader()) {
04912       MustExecuteLoopHeader = false;
04913     }
04914 
04915   // At this point, we know we have a conditional branch that determines whether
04916   // the loop is exited.  However, we don't know if the branch is executed each
04917   // time through the loop.  If not, then the execution count of the branch will
04918   // not be equal to the trip count of the loop.
04919   //
04920   // Currently we check for this by checking to see if the Exit branch goes to
04921   // the loop header.  If so, we know it will always execute the same number of
04922   // times as the loop.  We also handle the case where the exit block *is* the
04923   // loop header.  This is common for un-rotated loops.
04924   //
04925   // If both of those tests fail, walk up the unique predecessor chain to the
04926   // header, stopping if there is an edge that doesn't exit the loop. If the
04927   // header is reached, the execution count of the branch will be equal to the
04928   // trip count of the loop.
04929   //
04930   //  More extensive analysis could be done to handle more cases here.
04931   //
04932   if (!MustExecuteLoopHeader && ExitingBlock != L->getHeader()) {
04933     // The simple checks failed, try climbing the unique predecessor chain
04934     // up to the header.
04935     bool Ok = false;
04936     for (BasicBlock *BB = ExitingBlock; BB; ) {
04937       BasicBlock *Pred = BB->getUniquePredecessor();
04938       if (!Pred)
04939         return getCouldNotCompute();
04940       TerminatorInst *PredTerm = Pred->getTerminator();
04941       for (unsigned i = 0, e = PredTerm->getNumSuccessors(); i != e; ++i) {
04942         BasicBlock *PredSucc = PredTerm->getSuccessor(i);
04943         if (PredSucc == BB)
04944           continue;
04945         // If the predecessor has a successor that isn't BB and isn't
04946         // outside the loop, assume the worst.
04947         if (L->contains(PredSucc))
04948           return getCouldNotCompute();
04949       }
04950       if (Pred == L->getHeader()) {
04951         Ok = true;
04952         break;
04953       }
04954       BB = Pred;
04955     }
04956     if (!Ok)
04957       return getCouldNotCompute();
04958   }
04959 
04960   bool IsOnlyExit = (L->getExitingBlock() != nullptr);
04961   TerminatorInst *Term = ExitingBlock->getTerminator();
04962   if (BranchInst *BI = dyn_cast<BranchInst>(Term)) {
04963     assert(BI->isConditional() && "If unconditional, it can't be in loop!");
04964     // Proceed to the next level to examine the exit condition expression.
04965     return ComputeExitLimitFromCond(L, BI->getCondition(), BI->getSuccessor(0),
04966                                     BI->getSuccessor(1),
04967                                     /*ControlsExit=*/IsOnlyExit);
04968   }
04969 
04970   if (SwitchInst *SI = dyn_cast<SwitchInst>(Term))
04971     return ComputeExitLimitFromSingleExitSwitch(L, SI, Exit,
04972                                                 /*ControlsExit=*/IsOnlyExit);
04973 
04974   return getCouldNotCompute();
04975 }
04976 
04977 /// ComputeExitLimitFromCond - Compute the number of times the
04978 /// backedge of the specified loop will execute if its exit condition
04979 /// were a conditional branch of ExitCond, TBB, and FBB.
04980 ///
04981 /// @param ControlsExit is true if ExitCond directly controls the exit
04982 /// branch. In this case, we can assume that the loop exits only if the
04983 /// condition is true and can infer that failing to meet the condition prior to
04984 /// integer wraparound results in undefined behavior.
04985 ScalarEvolution::ExitLimit
04986 ScalarEvolution::ComputeExitLimitFromCond(const Loop *L,
04987                                           Value *ExitCond,
04988                                           BasicBlock *TBB,
04989                                           BasicBlock *FBB,
04990                                           bool ControlsExit) {
04991   // Check if the controlling expression for this loop is an And or Or.
04992   if (BinaryOperator *BO = dyn_cast<BinaryOperator>(ExitCond)) {
04993     if (BO->getOpcode() == Instruction::And) {
04994       // Recurse on the operands of the and.
04995       bool EitherMayExit = L->contains(TBB);
04996       ExitLimit EL0 = ComputeExitLimitFromCond(L, BO->getOperand(0), TBB, FBB,
04997                                                ControlsExit && !EitherMayExit);
04998       ExitLimit EL1 = ComputeExitLimitFromCond(L, BO->getOperand(1), TBB, FBB,
04999                                                ControlsExit && !EitherMayExit);
05000       const SCEV *BECount = getCouldNotCompute();
05001       const SCEV *MaxBECount = getCouldNotCompute();
05002       if (EitherMayExit) {
05003         // Both conditions must be true for the loop to continue executing.
05004         // Choose the less conservative count.
05005         if (EL0.Exact == getCouldNotCompute() ||
05006             EL1.Exact == getCouldNotCompute())
05007           BECount = getCouldNotCompute();
05008         else
05009           BECount = getUMinFromMismatchedTypes(EL0.Exact, EL1.Exact);
05010         if (EL0.Max == getCouldNotCompute())
05011           MaxBECount = EL1.Max;
05012         else if (EL1.Max == getCouldNotCompute())
05013           MaxBECount = EL0.Max;
05014         else
05015           MaxBECount = getUMinFromMismatchedTypes(EL0.Max, EL1.Max);
05016       } else {
05017         // Both conditions must be true at the same time for the loop to exit.
05018         // For now, be conservative.
05019         assert(L->contains(FBB) && "Loop block has no successor in loop!");
05020         if (EL0.Max == EL1.Max)
05021           MaxBECount = EL0.Max;
05022         if (EL0.Exact == EL1.Exact)
05023           BECount = EL0.Exact;
05024       }
05025 
05026       return ExitLimit(BECount, MaxBECount);
05027     }
05028     if (BO->getOpcode() == Instruction::Or) {
05029       // Recurse on the operands of the or.
05030       bool EitherMayExit = L->contains(FBB);
05031       ExitLimit EL0 = ComputeExitLimitFromCond(L, BO->getOperand(0), TBB, FBB,
05032                                                ControlsExit && !EitherMayExit);
05033       ExitLimit EL1 = ComputeExitLimitFromCond(L, BO->getOperand(1), TBB, FBB,
05034                                                ControlsExit && !EitherMayExit);
05035       const SCEV *BECount = getCouldNotCompute();
05036       const SCEV *MaxBECount = getCouldNotCompute();
05037       if (EitherMayExit) {
05038         // Both conditions must be false for the loop to continue executing.
05039         // Choose the less conservative count.
05040         if (EL0.Exact == getCouldNotCompute() ||
05041             EL1.Exact == getCouldNotCompute())
05042           BECount = getCouldNotCompute();
05043         else
05044           BECount = getUMinFromMismatchedTypes(EL0.Exact, EL1.Exact);
05045         if (EL0.Max == getCouldNotCompute())
05046           MaxBECount = EL1.Max;
05047         else if (EL1.Max == getCouldNotCompute())
05048           MaxBECount = EL0.Max;
05049         else
05050           MaxBECount = getUMinFromMismatchedTypes(EL0.Max, EL1.Max);
05051       } else {
05052         // Both conditions must be false at the same time for the loop to exit.
05053         // For now, be conservative.
05054         assert(L->contains(TBB) && "Loop block has no successor in loop!");
05055         if (EL0.Max == EL1.Max)
05056           MaxBECount = EL0.Max;
05057         if (EL0.Exact == EL1.Exact)
05058           BECount = EL0.Exact;
05059       }
05060 
05061       return ExitLimit(BECount, MaxBECount);
05062     }
05063   }
05064 
05065   // With an icmp, it may be feasible to compute an exact backedge-taken count.
05066   // Proceed to the next level to examine the icmp.
05067   if (ICmpInst *ExitCondICmp = dyn_cast<ICmpInst>(ExitCond))
05068     return ComputeExitLimitFromICmp(L, ExitCondICmp, TBB, FBB, ControlsExit);
05069 
05070   // Check for a constant condition. These are normally stripped out by
05071   // SimplifyCFG, but ScalarEvolution may be used by a pass which wishes to
05072   // preserve the CFG and is temporarily leaving constant conditions
05073   // in place.
05074   if (ConstantInt *CI = dyn_cast<ConstantInt>(ExitCond)) {
05075     if (L->contains(FBB) == !CI->getZExtValue())
05076       // The backedge is always taken.
05077       return getCouldNotCompute();
05078     else
05079       // The backedge is never taken.
05080       return getConstant(CI->getType(), 0);
05081   }
05082 
05083   // If it's not an integer or pointer comparison then compute it the hard way.
05084   return ComputeExitCountExhaustively(L, ExitCond, !L->contains(TBB));
05085 }
05086 
05087 /// ComputeExitLimitFromICmp - Compute the number of times the
05088 /// backedge of the specified loop will execute if its exit condition
05089 /// were a conditional branch of the ICmpInst ExitCond, TBB, and FBB.
05090 ScalarEvolution::ExitLimit
05091 ScalarEvolution::ComputeExitLimitFromICmp(const Loop *L,
05092                                           ICmpInst *ExitCond,
05093                                           BasicBlock *TBB,
05094                                           BasicBlock *FBB,
05095                                           bool ControlsExit) {
05096 
05097   // If the condition was exit on true, convert the condition to exit on false
05098   ICmpInst::Predicate Cond;
05099   if (!L->contains(FBB))
05100     Cond = ExitCond->getPredicate();
05101   else
05102     Cond = ExitCond->getInversePredicate();
05103 
05104   // Handle common loops like: for (X = "string"; *X; ++X)
05105   if (LoadInst *LI = dyn_cast<LoadInst>(ExitCond->getOperand(0)))
05106     if (Constant *RHS = dyn_cast<Constant>(ExitCond->getOperand(1))) {
05107       ExitLimit ItCnt =
05108         ComputeLoadConstantCompareExitLimit(LI, RHS, L, Cond);
05109       if (ItCnt.hasAnyInfo())
05110         return ItCnt;
05111     }
05112 
05113   const SCEV *LHS = getSCEV(ExitCond->getOperand(0));
05114   const SCEV *RHS = getSCEV(ExitCond->getOperand(1));
05115 
05116   // Try to evaluate any dependencies out of the loop.
05117   LHS = getSCEVAtScope(LHS, L);
05118   RHS = getSCEVAtScope(RHS, L);
05119 
05120   // At this point, we would like to compute how many iterations of the
05121   // loop the predicate will return true for these inputs.
05122   if (isLoopInvariant(LHS, L) && !isLoopInvariant(RHS, L)) {
05123     // If there is a loop-invariant, force it into the RHS.
05124     std::swap(LHS, RHS);
05125     Cond = ICmpInst::getSwappedPredicate(Cond);
05126   }
05127 
05128   // Simplify the operands before analyzing them.
05129   (void)SimplifyICmpOperands(Cond, LHS, RHS);
05130 
05131   // If we have a comparison of a chrec against a constant, try to use value
05132   // ranges to answer this query.
05133   if (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(RHS))
05134     if (const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(LHS))
05135       if (AddRec->getLoop() == L) {
05136         // Form the constant range.
05137         ConstantRange CompRange(
05138             ICmpInst::makeConstantRange(Cond, RHSC->getValue()->getValue()));
05139 
05140         const SCEV *Ret = AddRec->getNumIterationsInRange(CompRange, *this);
05141         if (!isa<SCEVCouldNotCompute>(Ret)) return Ret;
05142       }
05143 
05144   switch (Cond) {
05145   case ICmpInst::ICMP_NE: {                     // while (X != Y)
05146     // Convert to: while (X-Y != 0)
05147     ExitLimit EL = HowFarToZero(getMinusSCEV(LHS, RHS), L, ControlsExit);
05148     if (EL.hasAnyInfo()) return EL;
05149     break;
05150   }
05151   case ICmpInst::ICMP_EQ: {                     // while (X == Y)
05152     // Convert to: while (X-Y == 0)
05153     ExitLimit EL = HowFarToNonZero(getMinusSCEV(LHS, RHS), L);
05154     if (EL.hasAnyInfo()) return EL;
05155     break;
05156   }
05157   case ICmpInst::ICMP_SLT:
05158   case ICmpInst::ICMP_ULT: {                    // while (X < Y)
05159     bool IsSigned = Cond == ICmpInst::ICMP_SLT;
05160     ExitLimit EL = HowManyLessThans(LHS, RHS, L, IsSigned, ControlsExit);
05161     if (EL.hasAnyInfo()) return EL;
05162     break;
05163   }
05164   case ICmpInst::ICMP_SGT:
05165   case ICmpInst::ICMP_UGT: {                    // while (X > Y)
05166     bool IsSigned = Cond == ICmpInst::ICMP_SGT;
05167     ExitLimit EL = HowManyGreaterThans(LHS, RHS, L, IsSigned, ControlsExit);
05168     if (EL.hasAnyInfo()) return EL;
05169     break;
05170   }
05171   default:
05172 #if 0
05173     dbgs() << "ComputeBackedgeTakenCount ";
05174     if (ExitCond->getOperand(0)->getType()->isUnsigned())
05175       dbgs() << "[unsigned] ";
05176     dbgs() << *LHS << "   "
05177          << Instruction::getOpcodeName(Instruction::ICmp)
05178          << "   " << *RHS << "\n";
05179 #endif
05180     break;
05181   }
05182   return ComputeExitCountExhaustively(L, ExitCond, !L->contains(TBB));
05183 }
05184 
05185 ScalarEvolution::ExitLimit
05186 ScalarEvolution::ComputeExitLimitFromSingleExitSwitch(const Loop *L,
05187                                                       SwitchInst *Switch,
05188                                                       BasicBlock *ExitingBlock,
05189                                                       bool ControlsExit) {
05190   assert(!L->contains(ExitingBlock) && "Not an exiting block!");
05191 
05192   // Give up if the exit is the default dest of a switch.
05193   if (Switch->getDefaultDest() == ExitingBlock)
05194     return getCouldNotCompute();
05195 
05196   assert(L->contains(Switch->getDefaultDest()) &&
05197          "Default case must not exit the loop!");
05198   const SCEV *LHS = getSCEVAtScope(Switch->getCondition(), L);
05199   const SCEV *RHS = getConstant(Switch->findCaseDest(ExitingBlock));
05200 
05201   // while (X != Y) --> while (X-Y != 0)
05202   ExitLimit EL = HowFarToZero(getMinusSCEV(LHS, RHS), L, ControlsExit);
05203   if (EL.hasAnyInfo())
05204     return EL;
05205 
05206   return getCouldNotCompute();
05207 }
05208 
05209 static ConstantInt *
05210 EvaluateConstantChrecAtConstant(const SCEVAddRecExpr *AddRec, ConstantInt *C,
05211                                 ScalarEvolution &SE) {
05212   const SCEV *InVal = SE.getConstant(C);
05213   const SCEV *Val = AddRec->evaluateAtIteration(InVal, SE);
05214   assert(isa<SCEVConstant>(Val) &&
05215          "Evaluation of SCEV at constant didn't fold correctly?");
05216   return cast<SCEVConstant>(Val)->getValue();
05217 }
05218 
05219 /// ComputeLoadConstantCompareExitLimit - Given an exit condition of
05220 /// 'icmp op load X, cst', try to see if we can compute the backedge
05221 /// execution count.
05222 ScalarEvolution::ExitLimit
05223 ScalarEvolution::ComputeLoadConstantCompareExitLimit(
05224   LoadInst *LI,
05225   Constant *RHS,
05226   const Loop *L,
05227   ICmpInst::Predicate predicate) {
05228 
05229   if (LI->isVolatile()) return getCouldNotCompute();
05230 
05231   // Check to see if the loaded pointer is a getelementptr of a global.
05232   // TODO: Use SCEV instead of manually grubbing with GEPs.
05233   GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(LI->getOperand(0));
05234   if (!GEP) return getCouldNotCompute();
05235 
05236   // Make sure that it is really a constant global we are gepping, with an
05237   // initializer, and make sure the first IDX is really 0.
05238   GlobalVariable *GV = dyn_cast<GlobalVariable>(GEP->getOperand(0));
05239   if (!GV || !GV->isConstant() || !GV->hasDefinitiveInitializer() ||
05240       GEP->getNumOperands() < 3 || !isa<Constant>(GEP->getOperand(1)) ||
05241       !cast<Constant>(GEP->getOperand(1))->isNullValue())
05242     return getCouldNotCompute();
05243 
05244   // Okay, we allow one non-constant index into the GEP instruction.
05245   Value *VarIdx = nullptr;
05246   std::vector<Constant*> Indexes;
05247   unsigned VarIdxNum = 0;
05248   for (unsigned i = 2, e = GEP->getNumOperands(); i != e; ++i)
05249     if (ConstantInt *CI = dyn_cast<ConstantInt>(GEP->getOperand(i))) {
05250       Indexes.push_back(CI);
05251     } else if (!isa<ConstantInt>(GEP->getOperand(i))) {
05252       if (VarIdx) return getCouldNotCompute();  // Multiple non-constant idx's.
05253       VarIdx = GEP->getOperand(i);
05254       VarIdxNum = i-2;
05255       Indexes.push_back(nullptr);
05256     }
05257 
05258   // Loop-invariant loads may be a byproduct of loop optimization. Skip them.
05259   if (!VarIdx)
05260     return getCouldNotCompute();
05261 
05262   // Okay, we know we have a (load (gep GV, 0, X)) comparison with a constant.
05263   // Check to see if X is a loop variant variable value now.
05264   const SCEV *Idx = getSCEV(VarIdx);
05265   Idx = getSCEVAtScope(Idx, L);
05266 
05267   // We can only recognize very limited forms of loop index expressions, in
05268   // particular, only affine AddRec's like {C1,+,C2}.
05269   const SCEVAddRecExpr *IdxExpr = dyn_cast<SCEVAddRecExpr>(Idx);
05270   if (!IdxExpr || !IdxExpr->isAffine() || isLoopInvariant(IdxExpr, L) ||
05271       !isa<SCEVConstant>(IdxExpr->getOperand(0)) ||
05272       !isa<SCEVConstant>(IdxExpr->getOperand(1)))
05273     return getCouldNotCompute();
05274 
05275   unsigned MaxSteps = MaxBruteForceIterations;
05276   for (unsigned IterationNum = 0; IterationNum != MaxSteps; ++IterationNum) {
05277     ConstantInt *ItCst = ConstantInt::get(
05278                            cast<IntegerType>(IdxExpr->getType()), IterationNum);
05279     ConstantInt *Val = EvaluateConstantChrecAtConstant(IdxExpr, ItCst, *this);
05280 
05281     // Form the GEP offset.
05282     Indexes[VarIdxNum] = Val;
05283 
05284     Constant *Result = ConstantFoldLoadThroughGEPIndices(GV->getInitializer(),
05285                                                          Indexes);
05286     if (!Result) break;  // Cannot compute!
05287 
05288     // Evaluate the condition for this iteration.
05289     Result = ConstantExpr::getICmp(predicate, Result, RHS);
05290     if (!isa<ConstantInt>(Result)) break;  // Couldn't decide for sure
05291     if (cast<ConstantInt>(Result)->getValue().isMinValue()) {
05292 #if 0
05293       dbgs() << "\n***\n*** Computed loop count " << *ItCst
05294              << "\n*** From global " << *GV << "*** BB: " << *L->getHeader()
05295              << "***\n";
05296 #endif
05297       ++NumArrayLenItCounts;
05298       return getConstant(ItCst);   // Found terminating iteration!
05299     }
05300   }
05301   return getCouldNotCompute();
05302 }
05303 
05304 
05305 /// CanConstantFold - Return true if we can constant fold an instruction of the
05306 /// specified type, assuming that all operands were constants.
05307 static bool CanConstantFold(const Instruction *I) {
05308   if (isa<BinaryOperator>(I) || isa<CmpInst>(I) ||
05309       isa<SelectInst>(I) || isa<CastInst>(I) || isa<GetElementPtrInst>(I) ||
05310       isa<LoadInst>(I))
05311     return true;
05312 
05313   if (const CallInst *CI = dyn_cast<CallInst>(I))
05314     if (const Function *F = CI->getCalledFunction())
05315       return canConstantFoldCallTo(F);
05316   return false;
05317 }
05318 
05319 /// Determine whether this instruction can constant evolve within this loop
05320 /// assuming its operands can all constant evolve.
05321 static bool canConstantEvolve(Instruction *I, const Loop *L) {
05322   // An instruction outside of the loop can't be derived from a loop PHI.
05323   if (!L->contains(I)) return false;
05324 
05325   if (isa<PHINode>(I)) {
05326     // We don't currently keep track of the control flow needed to evaluate
05327     // PHIs, so we cannot handle PHIs inside of loops.
05328     return L->getHeader() == I->getParent();
05329   }
05330 
05331   // If we won't be able to constant fold this expression even if the operands
05332   // are constants, bail early.
05333   return CanConstantFold(I);
05334 }
05335 
05336 /// getConstantEvolvingPHIOperands - Implement getConstantEvolvingPHI by
05337 /// recursing through each instruction operand until reaching a loop header phi.
05338 static PHINode *
05339 getConstantEvolvingPHIOperands(Instruction *UseInst, const Loop *L,
05340                                DenseMap<Instruction *, PHINode *> &PHIMap) {
05341 
05342   // Otherwise, we can evaluate this instruction if all of its operands are
05343   // constant or derived from a PHI node themselves.
05344   PHINode *PHI = nullptr;
05345   for (Instruction::op_iterator OpI = UseInst->op_begin(),
05346          OpE = UseInst->op_end(); OpI != OpE; ++OpI) {
05347 
05348     if (isa<Constant>(*OpI)) continue;
05349 
05350     Instruction *OpInst = dyn_cast<Instruction>(*OpI);
05351     if (!OpInst || !canConstantEvolve(OpInst, L)) return nullptr;
05352 
05353     PHINode *P = dyn_cast<PHINode>(OpInst);
05354     if (!P)
05355       // If this operand is already visited, reuse the prior result.
05356       // We may have P != PHI if this is the deepest point at which the
05357       // inconsistent paths meet.
05358       P = PHIMap.lookup(OpInst);
05359     if (!P) {
05360       // Recurse and memoize the results, whether a phi is found or not.
05361       // This recursive call invalidates pointers into PHIMap.
05362       P = getConstantEvolvingPHIOperands(OpInst, L, PHIMap);
05363       PHIMap[OpInst] = P;
05364     }
05365     if (!P)
05366       return nullptr;  // Not evolving from PHI
05367     if (PHI && PHI != P)
05368       return nullptr;  // Evolving from multiple different PHIs.
05369     PHI = P;
05370   }
05371   // This is a expression evolving from a constant PHI!
05372   return PHI;
05373 }
05374 
05375 /// getConstantEvolvingPHI - Given an LLVM value and a loop, return a PHI node
05376 /// in the loop that V is derived from.  We allow arbitrary operations along the
05377 /// way, but the operands of an operation must either be constants or a value
05378 /// derived from a constant PHI.  If this expression does not fit with these
05379 /// constraints, return null.
05380 static PHINode *getConstantEvolvingPHI(Value *V, const Loop *L) {
05381   Instruction *I = dyn_cast<Instruction>(V);
05382   if (!I || !canConstantEvolve(I, L)) return nullptr;
05383 
05384   if (PHINode *PN = dyn_cast<PHINode>(I)) {
05385     return PN;
05386   }
05387 
05388   // Record non-constant instructions contained by the loop.
05389   DenseMap<Instruction *, PHINode *> PHIMap;
05390   return getConstantEvolvingPHIOperands(I, L, PHIMap);
05391 }
05392 
05393 /// EvaluateExpression - Given an expression that passes the
05394 /// getConstantEvolvingPHI predicate, evaluate its value assuming the PHI node
05395 /// in the loop has the value PHIVal.  If we can't fold this expression for some
05396 /// reason, return null.
05397 static Constant *EvaluateExpression(Value *V, const Loop *L,
05398                                     DenseMap<Instruction *, Constant *> &Vals,
05399                                     const DataLayout &DL,
05400                                     const TargetLibraryInfo *TLI) {
05401   // Convenient constant check, but redundant for recursive calls.
05402   if (Constant *C = dyn_cast<Constant>(V)) return C;
05403   Instruction *I = dyn_cast<Instruction>(V);
05404   if (!I) return nullptr;
05405 
05406   if (Constant *C = Vals.lookup(I)) return C;
05407 
05408   // An instruction inside the loop depends on a value outside the loop that we
05409   // weren't given a mapping for, or a value such as a call inside the loop.
05410   if (!canConstantEvolve(I, L)) return nullptr;
05411 
05412   // An unmapped PHI can be due to a branch or another loop inside this loop,
05413   // or due to this not being the initial iteration through a loop where we
05414   // couldn't compute the evolution of this particular PHI last time.
05415   if (isa<PHINode>(I)) return nullptr;
05416 
05417   std::vector<Constant*> Operands(I->getNumOperands());
05418 
05419   for (unsigned i = 0, e = I->getNumOperands(); i != e; ++i) {
05420     Instruction *Operand = dyn_cast<Instruction>(I->getOperand(i));
05421     if (!Operand) {
05422       Operands[i] = dyn_cast<Constant>(I->getOperand(i));
05423       if (!Operands[i]) return nullptr;
05424       continue;
05425     }
05426     Constant *C = EvaluateExpression(Operand, L, Vals, DL, TLI);
05427     Vals[Operand] = C;
05428     if (!C) return nullptr;
05429     Operands[i] = C;
05430   }
05431 
05432   if (CmpInst *CI = dyn_cast<CmpInst>(I))
05433     return ConstantFoldCompareInstOperands(CI->getPredicate(), Operands[0],
05434                                            Operands[1], DL, TLI);
05435   if (LoadInst *LI = dyn_cast<LoadInst>(I)) {
05436     if (!LI->isVolatile())
05437       return ConstantFoldLoadFromConstPtr(Operands[0], DL);
05438   }
05439   return ConstantFoldInstOperands(I->getOpcode(), I->getType(), Operands, DL,
05440                                   TLI);
05441 }
05442 
05443 /// getConstantEvolutionLoopExitValue - If we know that the specified Phi is
05444 /// in the header of its containing loop, we know the loop executes a
05445 /// constant number of times, and the PHI node is just a recurrence
05446 /// involving constants, fold it.
05447 Constant *
05448 ScalarEvolution::getConstantEvolutionLoopExitValue(PHINode *PN,
05449                                                    const APInt &BEs,
05450                                                    const Loop *L) {
05451   DenseMap<PHINode*, Constant*>::const_iterator I =
05452     ConstantEvolutionLoopExitValue.find(PN);
05453   if (I != ConstantEvolutionLoopExitValue.end())
05454     return I->second;
05455 
05456   if (BEs.ugt(MaxBruteForceIterations))
05457     return ConstantEvolutionLoopExitValue[PN] = nullptr;  // Not going to evaluate it.
05458 
05459   Constant *&RetVal = ConstantEvolutionLoopExitValue[PN];
05460 
05461   DenseMap<Instruction *, Constant *> CurrentIterVals;
05462   BasicBlock *Header = L->getHeader();
05463   assert(PN->getParent() == Header && "Can't evaluate PHI not in loop header!");
05464 
05465   // Since the loop is canonicalized, the PHI node must have two entries.  One
05466   // entry must be a constant (coming in from outside of the loop), and the
05467   // second must be derived from the same PHI.
05468   bool SecondIsBackedge = L->contains(PN->getIncomingBlock(1));
05469   PHINode *PHI = nullptr;
05470   for (BasicBlock::iterator I = Header->begin();
05471        (PHI = dyn_cast<PHINode>(I)); ++I) {
05472     Constant *StartCST =
05473       dyn_cast<Constant>(PHI->getIncomingValue(!SecondIsBackedge));
05474     if (!StartCST) continue;
05475     CurrentIterVals[PHI] = StartCST;
05476   }
05477   if (!CurrentIterVals.count(PN))
05478     return RetVal = nullptr;
05479 
05480   Value *BEValue = PN->getIncomingValue(SecondIsBackedge);
05481 
05482   // Execute the loop symbolically to determine the exit value.
05483   if (BEs.getActiveBits() >= 32)
05484     return RetVal = nullptr; // More than 2^32-1 iterations?? Not doing it!
05485 
05486   unsigned NumIterations = BEs.getZExtValue(); // must be in range
05487   unsigned IterationNum = 0;
05488   const DataLayout &DL = F->getParent()->getDataLayout();
05489   for (; ; ++IterationNum) {
05490     if (IterationNum == NumIterations)
05491       return RetVal = CurrentIterVals[PN];  // Got exit value!
05492 
05493     // Compute the value of the PHIs for the next iteration.
05494     // EvaluateExpression adds non-phi values to the CurrentIterVals map.
05495     DenseMap<Instruction *, Constant *> NextIterVals;
05496     Constant *NextPHI =
05497         EvaluateExpression(BEValue, L, CurrentIterVals, DL, TLI);
05498     if (!NextPHI)
05499       return nullptr;        // Couldn't evaluate!
05500     NextIterVals[PN] = NextPHI;
05501 
05502     bool StoppedEvolving = NextPHI == CurrentIterVals[PN];
05503 
05504     // Also evaluate the other PHI nodes.  However, we don't get to stop if we
05505     // cease to be able to evaluate one of them or if they stop evolving,
05506     // because that doesn't necessarily prevent us from computing PN.
05507     SmallVector<std::pair<PHINode *, Constant *>, 8> PHIsToCompute;
05508     for (DenseMap<Instruction *, Constant *>::const_iterator
05509            I = CurrentIterVals.begin(), E = CurrentIterVals.end(); I != E; ++I){
05510       PHINode *PHI = dyn_cast<PHINode>(I->first);
05511       if (!PHI || PHI == PN || PHI->getParent() != Header) continue;
05512       PHIsToCompute.push_back(std::make_pair(PHI, I->second));
05513     }
05514     // We use two distinct loops because EvaluateExpression may invalidate any
05515     // iterators into CurrentIterVals.
05516     for (SmallVectorImpl<std::pair<PHINode *, Constant*> >::const_iterator
05517              I = PHIsToCompute.begin(), E = PHIsToCompute.end(); I != E; ++I) {
05518       PHINode *PHI = I->first;
05519       Constant *&NextPHI = NextIterVals[PHI];
05520       if (!NextPHI) {   // Not already computed.
05521         Value *BEValue = PHI->getIncomingValue(SecondIsBackedge);
05522         NextPHI = EvaluateExpression(BEValue, L, CurrentIterVals, DL, TLI);
05523       }
05524       if (NextPHI != I->second)
05525         StoppedEvolving = false;
05526     }
05527 
05528     // If all entries in CurrentIterVals == NextIterVals then we can stop
05529     // iterating, the loop can't continue to change.
05530     if (StoppedEvolving)
05531       return RetVal = CurrentIterVals[PN];
05532 
05533     CurrentIterVals.swap(NextIterVals);
05534   }
05535 }
05536 
05537 /// ComputeExitCountExhaustively - If the loop is known to execute a
05538 /// constant number of times (the condition evolves only from constants),
05539 /// try to evaluate a few iterations of the loop until we get the exit
05540 /// condition gets a value of ExitWhen (true or false).  If we cannot
05541 /// evaluate the trip count of the loop, return getCouldNotCompute().
05542 const SCEV *ScalarEvolution::ComputeExitCountExhaustively(const Loop *L,
05543                                                           Value *Cond,
05544                                                           bool ExitWhen) {
05545   PHINode *PN = getConstantEvolvingPHI(Cond, L);
05546   if (!PN) return getCouldNotCompute();
05547 
05548   // If the loop is canonicalized, the PHI will have exactly two entries.
05549   // That's the only form we support here.
05550   if (PN->getNumIncomingValues() != 2) return getCouldNotCompute();
05551 
05552   DenseMap<Instruction *, Constant *> CurrentIterVals;
05553   BasicBlock *Header = L->getHeader();
05554   assert(PN->getParent() == Header && "Can't evaluate PHI not in loop header!");
05555 
05556   // One entry must be a constant (coming in from outside of the loop), and the
05557   // second must be derived from the same PHI.
05558   bool SecondIsBackedge = L->contains(PN->getIncomingBlock(1));
05559   PHINode *PHI = nullptr;
05560   for (BasicBlock::iterator I = Header->begin();
05561        (PHI = dyn_cast<PHINode>(I)); ++I) {
05562     Constant *StartCST =
05563       dyn_cast<Constant>(PHI->getIncomingValue(!SecondIsBackedge));
05564     if (!StartCST) continue;
05565     CurrentIterVals[PHI] = StartCST;
05566   }
05567   if (!CurrentIterVals.count(PN))
05568     return getCouldNotCompute();
05569 
05570   // Okay, we find a PHI node that defines the trip count of this loop.  Execute
05571   // the loop symbolically to determine when the condition gets a value of
05572   // "ExitWhen".
05573   unsigned MaxIterations = MaxBruteForceIterations;   // Limit analysis.
05574   const DataLayout &DL = F->getParent()->getDataLayout();
05575   for (unsigned IterationNum = 0; IterationNum != MaxIterations;++IterationNum){
05576     ConstantInt *CondVal = dyn_cast_or_null<ConstantInt>(
05577         EvaluateExpression(Cond, L, CurrentIterVals, DL, TLI));
05578 
05579     // Couldn't symbolically evaluate.
05580     if (!CondVal) return getCouldNotCompute();
05581 
05582     if (CondVal->getValue() == uint64_t(ExitWhen)) {
05583       ++NumBruteForceTripCountsComputed;
05584       return getConstant(Type::getInt32Ty(getContext()), IterationNum);
05585     }
05586 
05587     // Update all the PHI nodes for the next iteration.
05588     DenseMap<Instruction *, Constant *> NextIterVals;
05589 
05590     // Create a list of which PHIs we need to compute. We want to do this before
05591     // calling EvaluateExpression on them because that may invalidate iterators
05592     // into CurrentIterVals.
05593     SmallVector<PHINode *, 8> PHIsToCompute;
05594     for (DenseMap<Instruction *, Constant *>::const_iterator
05595            I = CurrentIterVals.begin(), E = CurrentIterVals.end(); I != E; ++I){
05596       PHINode *PHI = dyn_cast<PHINode>(I->first);
05597       if (!PHI || PHI->getParent() != Header) continue;
05598       PHIsToCompute.push_back(PHI);
05599     }
05600     for (SmallVectorImpl<PHINode *>::const_iterator I = PHIsToCompute.begin(),
05601              E = PHIsToCompute.end(); I != E; ++I) {
05602       PHINode *PHI = *I;
05603       Constant *&NextPHI = NextIterVals[PHI];
05604       if (NextPHI) continue;    // Already computed!
05605 
05606       Value *BEValue = PHI->getIncomingValue(SecondIsBackedge);
05607       NextPHI = EvaluateExpression(BEValue, L, CurrentIterVals, DL, TLI);
05608     }
05609     CurrentIterVals.swap(NextIterVals);
05610   }
05611 
05612   // Too many iterations were needed to evaluate.
05613   return getCouldNotCompute();
05614 }
05615 
05616 /// getSCEVAtScope - Return a SCEV expression for the specified value
05617 /// at the specified scope in the program.  The L value specifies a loop
05618 /// nest to evaluate the expression at, where null is the top-level or a
05619 /// specified loop is immediately inside of the loop.
05620 ///
05621 /// This method can be used to compute the exit value for a variable defined
05622 /// in a loop by querying what the value will hold in the parent loop.
05623 ///
05624 /// In the case that a relevant loop exit value cannot be computed, the
05625 /// original value V is returned.
05626 const SCEV *ScalarEvolution::getSCEVAtScope(const SCEV *V, const Loop *L) {
05627   // Check to see if we've folded this expression at this loop before.
05628   SmallVector<std::pair<const Loop *, const SCEV *>, 2> &Values = ValuesAtScopes[V];
05629   for (unsigned u = 0; u < Values.size(); u++) {
05630     if (Values[u].first == L)
05631       return Values[u].second ? Values[u].second : V;
05632   }
05633   Values.push_back(std::make_pair(L, static_cast<const SCEV *>(nullptr)));
05634   // Otherwise compute it.
05635   const SCEV *C = computeSCEVAtScope(V, L);
05636   SmallVector<std::pair<const Loop *, const SCEV *>, 2> &Values2 = ValuesAtScopes[V];
05637   for (unsigned u = Values2.size(); u > 0; u--) {
05638     if (Values2[u - 1].first == L) {
05639       Values2[u - 1].second = C;
05640       break;
05641     }
05642   }
05643   return C;
05644 }
05645 
05646 /// This builds up a Constant using the ConstantExpr interface.  That way, we
05647 /// will return Constants for objects which aren't represented by a
05648 /// SCEVConstant, because SCEVConstant is restricted to ConstantInt.
05649 /// Returns NULL if the SCEV isn't representable as a Constant.
05650 static Constant *BuildConstantFromSCEV(const SCEV *V) {
05651   switch (static_cast<SCEVTypes>(V->getSCEVType())) {
05652     case scCouldNotCompute:
05653     case scAddRecExpr:
05654       break;
05655     case scConstant:
05656       return cast<SCEVConstant>(V)->getValue();
05657     case scUnknown:
05658       return dyn_cast<Constant>(cast<SCEVUnknown>(V)->getValue());
05659     case scSignExtend: {
05660       const SCEVSignExtendExpr *SS = cast<SCEVSignExtendExpr>(V);
05661       if (Constant *CastOp = BuildConstantFromSCEV(SS->getOperand()))
05662         return ConstantExpr::getSExt(CastOp, SS->getType());
05663       break;
05664     }
05665     case scZeroExtend: {
05666       const SCEVZeroExtendExpr *SZ = cast<SCEVZeroExtendExpr>(V);
05667       if (Constant *CastOp = BuildConstantFromSCEV(SZ->getOperand()))
05668         return ConstantExpr::getZExt(CastOp, SZ->getType());
05669       break;
05670     }
05671     case scTruncate: {
05672       const SCEVTruncateExpr *ST = cast<SCEVTruncateExpr>(V);
05673       if (Constant *CastOp = BuildConstantFromSCEV(ST->getOperand()))
05674         return ConstantExpr::getTrunc(CastOp, ST->getType());
05675       break;
05676     }
05677     case scAddExpr: {
05678       const SCEVAddExpr *SA = cast<SCEVAddExpr>(V);
05679       if (Constant *C = BuildConstantFromSCEV(SA->getOperand(0))) {
05680         if (PointerType *PTy = dyn_cast<PointerType>(C->getType())) {
05681           unsigned AS = PTy->getAddressSpace();
05682           Type *DestPtrTy = Type::getInt8PtrTy(C->getContext(), AS);
05683           C = ConstantExpr::getBitCast(C, DestPtrTy);
05684         }
05685         for (unsigned i = 1, e = SA->getNumOperands(); i != e; ++i) {
05686           Constant *C2 = BuildConstantFromSCEV(SA->getOperand(i));
05687           if (!C2) return nullptr;
05688 
05689           // First pointer!
05690           if (!C->getType()->isPointerTy() && C2->getType()->isPointerTy()) {
05691             unsigned AS = C2->getType()->getPointerAddressSpace();
05692             std::swap(C, C2);
05693             Type *DestPtrTy = Type::getInt8PtrTy(C->getContext(), AS);
05694             // The offsets have been converted to bytes.  We can add bytes to an
05695             // i8* by GEP with the byte count in the first index.
05696             C = ConstantExpr::getBitCast(C, DestPtrTy);
05697           }
05698 
05699           // Don't bother trying to sum two pointers. We probably can't
05700           // statically compute a load that results from it anyway.
05701           if (C2->getType()->isPointerTy())
05702             return nullptr;
05703 
05704           if (PointerType *PTy = dyn_cast<PointerType>(C->getType())) {
05705             if (PTy->getElementType()->isStructTy())
05706               C2 = ConstantExpr::getIntegerCast(
05707                   C2, Type::getInt32Ty(C->getContext()), true);
05708             C = ConstantExpr::getGetElementPtr(PTy->getElementType(), C, C2);
05709           } else
05710             C = ConstantExpr::getAdd(C, C2);
05711         }
05712         return C;
05713       }
05714       break;
05715     }
05716     case scMulExpr: {
05717       const SCEVMulExpr *SM = cast<SCEVMulExpr>(V);
05718       if (Constant *C = BuildConstantFromSCEV(SM->getOperand(0))) {
05719         // Don't bother with pointers at all.
05720         if (C->getType()->isPointerTy()) return nullptr;
05721         for (unsigned i = 1, e = SM->getNumOperands(); i != e; ++i) {
05722           Constant *C2 = BuildConstantFromSCEV(SM->getOperand(i));
05723           if (!C2 || C2->getType()->isPointerTy()) return nullptr;
05724           C = ConstantExpr::getMul(C, C2);
05725         }
05726         return C;
05727       }
05728       break;
05729     }
05730     case scUDivExpr: {
05731       const SCEVUDivExpr *SU = cast<SCEVUDivExpr>(V);
05732       if (Constant *LHS = BuildConstantFromSCEV(SU->getLHS()))
05733         if (Constant *RHS = BuildConstantFromSCEV(SU->getRHS()))
05734           if (LHS->getType() == RHS->getType())
05735             return ConstantExpr::getUDiv(LHS, RHS);
05736       break;
05737     }
05738     case scSMaxExpr:
05739     case scUMaxExpr:
05740       break; // TODO: smax, umax.
05741   }
05742   return nullptr;
05743 }
05744 
05745 const SCEV *ScalarEvolution::computeSCEVAtScope(const SCEV *V, const Loop *L) {
05746   if (isa<SCEVConstant>(V)) return V;
05747 
05748   // If this instruction is evolved from a constant-evolving PHI, compute the
05749   // exit value from the loop without using SCEVs.
05750   if (const SCEVUnknown *SU = dyn_cast<SCEVUnknown>(V)) {
05751     if (Instruction *I = dyn_cast<Instruction>(SU->getValue())) {
05752       const Loop *LI = (*this->LI)[I->getParent()];
05753       if (LI && LI->getParentLoop() == L)  // Looking for loop exit value.
05754         if (PHINode *PN = dyn_cast<PHINode>(I))
05755           if (PN->getParent() == LI->getHeader()) {
05756             // Okay, there is no closed form solution for the PHI node.  Check
05757             // to see if the loop that contains it has a known backedge-taken
05758             // count.  If so, we may be able to force computation of the exit
05759             // value.
05760             const SCEV *BackedgeTakenCount = getBackedgeTakenCount(LI);
05761             if (const SCEVConstant *BTCC =
05762                   dyn_cast<SCEVConstant>(BackedgeTakenCount)) {
05763               // Okay, we know how many times the containing loop executes.  If
05764               // this is a constant evolving PHI node, get the final value at
05765               // the specified iteration number.
05766               Constant *RV = getConstantEvolutionLoopExitValue(PN,
05767                                                    BTCC->getValue()->getValue(),
05768                                                                LI);
05769               if (RV) return getSCEV(RV);
05770             }
05771           }
05772 
05773       // Okay, this is an expression that we cannot symbolically evaluate
05774       // into a SCEV.  Check to see if it's possible to symbolically evaluate
05775       // the arguments into constants, and if so, try to constant propagate the
05776       // result.  This is particularly useful for computing loop exit values.
05777       if (CanConstantFold(I)) {
05778         SmallVector<Constant *, 4> Operands;
05779         bool MadeImprovement = false;
05780         for (unsigned i = 0, e = I->getNumOperands(); i != e; ++i) {
05781           Value *Op = I->getOperand(i);
05782           if (Constant *C = dyn_cast<Constant>(Op)) {
05783             Operands.push_back(C);
05784             continue;
05785           }
05786 
05787           // If any of the operands is non-constant and if they are
05788           // non-integer and non-pointer, don't even try to analyze them
05789           // with scev techniques.
05790           if (!isSCEVable(Op->getType()))
05791             return V;
05792 
05793           const SCEV *OrigV = getSCEV(Op);
05794           const SCEV *OpV = getSCEVAtScope(OrigV, L);
05795           MadeImprovement |= OrigV != OpV;
05796 
05797           Constant *C = BuildConstantFromSCEV(OpV);
05798           if (!C) return V;
05799           if (C->getType() != Op->getType())
05800             C = ConstantExpr::getCast(CastInst::getCastOpcode(C, false,
05801                                                               Op->getType(),
05802                                                               false),
05803                                       C, Op->getType());
05804           Operands.push_back(C);
05805         }
05806 
05807         // Check to see if getSCEVAtScope actually made an improvement.
05808         if (MadeImprovement) {
05809           Constant *C = nullptr;
05810           const DataLayout &DL = F->getParent()->getDataLayout();
05811           if (const CmpInst *CI = dyn_cast<CmpInst>(I))
05812             C = ConstantFoldCompareInstOperands(CI->getPredicate(), Operands[0],
05813                                                 Operands[1], DL, TLI);
05814           else if (const LoadInst *LI = dyn_cast<LoadInst>(I)) {
05815             if (!LI->isVolatile())
05816               C = ConstantFoldLoadFromConstPtr(Operands[0], DL);
05817           } else
05818             C = ConstantFoldInstOperands(I->getOpcode(), I->getType(), Operands,
05819                                          DL, TLI);
05820           if (!C) return V;
05821           return getSCEV(C);
05822         }
05823       }
05824     }
05825 
05826     // This is some other type of SCEVUnknown, just return it.
05827     return V;
05828   }
05829 
05830   if (const SCEVCommutativeExpr *Comm = dyn_cast<SCEVCommutativeExpr>(V)) {
05831     // Avoid performing the look-up in the common case where the specified
05832     // expression has no loop-variant portions.
05833     for (unsigned i = 0, e = Comm->getNumOperands(); i != e; ++i) {
05834       const SCEV *OpAtScope = getSCEVAtScope(Comm->getOperand(i), L);
05835       if (OpAtScope != Comm->getOperand(i)) {
05836         // Okay, at least one of these operands is loop variant but might be
05837         // foldable.  Build a new instance of the folded commutative expression.
05838         SmallVector<const SCEV *, 8> NewOps(Comm->op_begin(),
05839                                             Comm->op_begin()+i);
05840         NewOps.push_back(OpAtScope);
05841 
05842         for (++i; i != e; ++i) {
05843           OpAtScope = getSCEVAtScope(Comm->getOperand(i), L);
05844           NewOps.push_back(OpAtScope);
05845         }
05846         if (isa<SCEVAddExpr>(Comm))
05847           return getAddExpr(NewOps);
05848         if (isa<SCEVMulExpr>(Comm))
05849           return getMulExpr(NewOps);
05850         if (isa<SCEVSMaxExpr>(Comm))
05851           return getSMaxExpr(NewOps);
05852         if (isa<SCEVUMaxExpr>(Comm))
05853           return getUMaxExpr(NewOps);
05854         llvm_unreachable("Unknown commutative SCEV type!");
05855       }
05856     }
05857     // If we got here, all operands are loop invariant.
05858     return Comm;
05859   }
05860 
05861   if (const SCEVUDivExpr *Div = dyn_cast<SCEVUDivExpr>(V)) {
05862     const SCEV *LHS = getSCEVAtScope(Div->getLHS(), L);
05863     const SCEV *RHS = getSCEVAtScope(Div->getRHS(), L);
05864     if (LHS == Div->getLHS() && RHS == Div->getRHS())
05865       return Div;   // must be loop invariant
05866     return getUDivExpr(LHS, RHS);
05867   }
05868 
05869   // If this is a loop recurrence for a loop that does not contain L, then we
05870   // are dealing with the final value computed by the loop.
05871   if (const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(V)) {
05872     // First, attempt to evaluate each operand.
05873     // Avoid performing the look-up in the common case where the specified
05874     // expression has no loop-variant portions.
05875     for (unsigned i = 0, e = AddRec->getNumOperands(); i != e; ++i) {
05876       const SCEV *OpAtScope = getSCEVAtScope(AddRec->getOperand(i), L);
05877       if (OpAtScope == AddRec->getOperand(i))
05878         continue;
05879 
05880       // Okay, at least one of these operands is loop variant but might be
05881       // foldable.  Build a new instance of the folded commutative expression.
05882       SmallVector<const SCEV *, 8> NewOps(AddRec->op_begin(),
05883                                           AddRec->op_begin()+i);
05884       NewOps.push_back(OpAtScope);
05885       for (++i; i != e; ++i)
05886         NewOps.push_back(getSCEVAtScope(AddRec->getOperand(i), L));
05887 
05888       const SCEV *FoldedRec =
05889         getAddRecExpr(NewOps, AddRec->getLoop(),
05890                       AddRec->getNoWrapFlags(SCEV::FlagNW));
05891       AddRec = dyn_cast<SCEVAddRecExpr>(FoldedRec);
05892       // The addrec may be folded to a nonrecurrence, for example, if the
05893       // induction variable is multiplied by zero after constant folding. Go
05894       // ahead and return the folded value.
05895       if (!AddRec)
05896         return FoldedRec;
05897       break;
05898     }
05899 
05900     // If the scope is outside the addrec's loop, evaluate it by using the
05901     // loop exit value of the addrec.
05902     if (!AddRec->getLoop()->contains(L)) {
05903       // To evaluate this recurrence, we need to know how many times the AddRec
05904       // loop iterates.  Compute this now.
05905       const SCEV *BackedgeTakenCount = getBackedgeTakenCount(AddRec->getLoop());
05906       if (BackedgeTakenCount == getCouldNotCompute()) return AddRec;
05907 
05908       // Then, evaluate the AddRec.
05909       return AddRec->evaluateAtIteration(BackedgeTakenCount, *this);
05910     }
05911 
05912     return AddRec;
05913   }
05914 
05915   if (const SCEVZeroExtendExpr *Cast = dyn_cast<SCEVZeroExtendExpr>(V)) {
05916     const SCEV *Op = getSCEVAtScope(Cast->getOperand(), L);
05917     if (Op == Cast->getOperand())
05918       return Cast;  // must be loop invariant
05919     return getZeroExtendExpr(Op, Cast->getType());
05920   }
05921 
05922   if (const SCEVSignExtendExpr *Cast = dyn_cast<SCEVSignExtendExpr>(V)) {
05923     const SCEV *Op = getSCEVAtScope(Cast->getOperand(), L);
05924     if (Op == Cast->getOperand())
05925       return Cast;  // must be loop invariant
05926     return getSignExtendExpr(Op, Cast->getType());
05927   }
05928 
05929   if (const SCEVTruncateExpr *Cast = dyn_cast<SCEVTruncateExpr>(V)) {
05930     const SCEV *Op = getSCEVAtScope(Cast->getOperand(), L);
05931     if (Op == Cast->getOperand())
05932       return Cast;  // must be loop invariant
05933     return getTruncateExpr(Op, Cast->getType());
05934   }
05935 
05936   llvm_unreachable("Unknown SCEV type!");
05937 }
05938 
05939 /// getSCEVAtScope - This is a convenience function which does
05940 /// getSCEVAtScope(getSCEV(V), L).
05941 const SCEV *ScalarEvolution::getSCEVAtScope(Value *V, const Loop *L) {
05942   return getSCEVAtScope(getSCEV(V), L);
05943 }
05944 
05945 /// SolveLinEquationWithOverflow - Finds the minimum unsigned root of the
05946 /// following equation:
05947 ///
05948 ///     A * X = B (mod N)
05949 ///
05950 /// where N = 2^BW and BW is the common bit width of A and B. The signedness of
05951 /// A and B isn't important.
05952 ///
05953 /// If the equation does not have a solution, SCEVCouldNotCompute is returned.
05954 static const SCEV *SolveLinEquationWithOverflow(const APInt &A, const APInt &B,
05955                                                ScalarEvolution &SE) {
05956   uint32_t BW = A.getBitWidth();
05957   assert(BW == B.getBitWidth() && "Bit widths must be the same.");
05958   assert(A != 0 && "A must be non-zero.");
05959 
05960   // 1. D = gcd(A, N)
05961   //
05962   // The gcd of A and N may have only one prime factor: 2. The number of
05963   // trailing zeros in A is its multiplicity
05964   uint32_t Mult2 = A.countTrailingZeros();
05965   // D = 2^Mult2
05966 
05967   // 2. Check if B is divisible by D.
05968   //
05969   // B is divisible by D if and only if the multiplicity of prime factor 2 for B
05970   // is not less than multiplicity of this prime factor for D.
05971   if (B.countTrailingZeros() < Mult2)
05972     return SE.getCouldNotCompute();
05973 
05974   // 3. Compute I: the multiplicative inverse of (A / D) in arithmetic
05975   // modulo (N / D).
05976   //
05977   // (N / D) may need BW+1 bits in its representation.  Hence, we'll use this
05978   // bit width during computations.
05979   APInt AD = A.lshr(Mult2).zext(BW + 1);  // AD = A / D
05980   APInt Mod(BW + 1, 0);
05981   Mod.setBit(BW - Mult2);  // Mod = N / D
05982   APInt I = AD.multiplicativeInverse(Mod);
05983 
05984   // 4. Compute the minimum unsigned root of the equation:
05985   // I * (B / D) mod (N / D)
05986   APInt Result = (I * B.lshr(Mult2).zext(BW + 1)).urem(Mod);
05987 
05988   // The result is guaranteed to be less than 2^BW so we may truncate it to BW
05989   // bits.
05990   return SE.getConstant(Result.trunc(BW));
05991 }
05992 
05993 /// SolveQuadraticEquation - Find the roots of the quadratic equation for the
05994 /// given quadratic chrec {L,+,M,+,N}.  This returns either the two roots (which
05995 /// might be the same) or two SCEVCouldNotCompute objects.
05996 ///
05997 static std::pair<const SCEV *,const SCEV *>
05998 SolveQuadraticEquation(const SCEVAddRecExpr *AddRec, ScalarEvolution &SE) {
05999   assert(AddRec->getNumOperands() == 3 && "This is not a quadratic chrec!");
06000   const SCEVConstant *LC = dyn_cast<SCEVConstant>(AddRec->getOperand(0));
06001   const SCEVConstant *MC = dyn_cast<SCEVConstant>(AddRec->getOperand(1));
06002   const SCEVConstant *NC = dyn_cast<SCEVConstant>(AddRec->getOperand(2));
06003 
06004   // We currently can only solve this if the coefficients are constants.
06005   if (!LC || !MC || !NC) {
06006     const SCEV *CNC = SE.getCouldNotCompute();
06007     return std::make_pair(CNC, CNC);
06008   }
06009 
06010   uint32_t BitWidth = LC->getValue()->getValue().getBitWidth();
06011   const APInt &L = LC->getValue()->getValue();
06012   const APInt &M = MC->getValue()->getValue();
06013   const APInt &N = NC->getValue()->getValue();
06014   APInt Two(BitWidth, 2);
06015   APInt Four(BitWidth, 4);
06016 
06017   {
06018     using namespace APIntOps;
06019     const APInt& C = L;
06020     // Convert from chrec coefficients to polynomial coefficients AX^2+BX+C
06021     // The B coefficient is M-N/2
06022     APInt B(M);
06023     B -= sdiv(N,Two);
06024 
06025     // The A coefficient is N/2
06026     APInt A(N.sdiv(Two));
06027 
06028     // Compute the B^2-4ac term.
06029     APInt SqrtTerm(B);
06030     SqrtTerm *= B;
06031     SqrtTerm -= Four * (A * C);
06032 
06033     if (SqrtTerm.isNegative()) {
06034       // The loop is provably infinite.
06035       const SCEV *CNC = SE.getCouldNotCompute();
06036       return std::make_pair(CNC, CNC);
06037     }
06038 
06039     // Compute sqrt(B^2-4ac). This is guaranteed to be the nearest
06040     // integer value or else APInt::sqrt() will assert.
06041     APInt SqrtVal(SqrtTerm.sqrt());
06042 
06043     // Compute the two solutions for the quadratic formula.
06044     // The divisions must be performed as signed divisions.
06045     APInt NegB(-B);
06046     APInt TwoA(A << 1);
06047     if (TwoA.isMinValue()) {
06048       const SCEV *CNC = SE.getCouldNotCompute();
06049       return std::make_pair(CNC, CNC);
06050     }
06051 
06052     LLVMContext &Context = SE.getContext();
06053 
06054     ConstantInt *Solution1 =
06055       ConstantInt::get(Context, (NegB + SqrtVal).sdiv(TwoA));
06056     ConstantInt *Solution2 =
06057       ConstantInt::get(Context, (NegB - SqrtVal).sdiv(TwoA));
06058 
06059     return std::make_pair(SE.getConstant(Solution1),
06060                           SE.getConstant(Solution2));
06061   } // end APIntOps namespace
06062 }
06063 
06064 /// HowFarToZero - Return the number of times a backedge comparing the specified
06065 /// value to zero will execute.  If not computable, return CouldNotCompute.
06066 ///
06067 /// This is only used for loops with a "x != y" exit test. The exit condition is
06068 /// now expressed as a single expression, V = x-y. So the exit test is
06069 /// effectively V != 0.  We know and take advantage of the fact that this
06070 /// expression only being used in a comparison by zero context.
06071 ScalarEvolution::ExitLimit
06072 ScalarEvolution::HowFarToZero(const SCEV *V, const Loop *L, bool ControlsExit) {
06073   // If the value is a constant
06074   if (const SCEVConstant *C = dyn_cast<SCEVConstant>(V)) {
06075     // If the value is already zero, the branch will execute zero times.
06076     if (C->getValue()->isZero()) return C;
06077     return getCouldNotCompute();  // Otherwise it will loop infinitely.
06078   }
06079 
06080   const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(V);
06081   if (!AddRec || AddRec->getLoop() != L)
06082     return getCouldNotCompute();
06083 
06084   // If this is a quadratic (3-term) AddRec {L,+,M,+,N}, find the roots of
06085   // the quadratic equation to solve it.
06086   if (AddRec->isQuadratic() && AddRec->getType()->isIntegerTy()) {
06087     std::pair<const SCEV *,const SCEV *> Roots =
06088       SolveQuadraticEquation(AddRec, *this);
06089     const SCEVConstant *R1 = dyn_cast<SCEVConstant>(Roots.first);
06090     const SCEVConstant *R2 = dyn_cast<SCEVConstant>(Roots.second);
06091     if (R1 && R2) {
06092 #if 0
06093       dbgs() << "HFTZ: " << *V << " - sol#1: " << *R1
06094              << "  sol#2: " << *R2 << "\n";
06095 #endif
06096       // Pick the smallest positive root value.
06097       if (ConstantInt *CB =
06098           dyn_cast<ConstantInt>(ConstantExpr::getICmp(CmpInst::ICMP_ULT,
06099                                                       R1->getValue(),
06100                                                       R2->getValue()))) {
06101         if (!CB->getZExtValue())
06102           std::swap(R1, R2);   // R1 is the minimum root now.
06103 
06104         // We can only use this value if the chrec ends up with an exact zero
06105         // value at this index.  When solving for "X*X != 5", for example, we
06106         // should not accept a root of 2.
06107         const SCEV *Val = AddRec->evaluateAtIteration(R1, *this);
06108         if (Val->isZero())
06109           return R1;  // We found a quadratic root!
06110       }
06111     }
06112     return getCouldNotCompute();
06113   }
06114 
06115   // Otherwise we can only handle this if it is affine.
06116   if (!AddRec->isAffine())
06117     return getCouldNotCompute();
06118 
06119   // If this is an affine expression, the execution count of this branch is
06120   // the minimum unsigned root of the following equation:
06121   //
06122   //     Start + Step*N = 0 (mod 2^BW)
06123   //
06124   // equivalent to:
06125   //
06126   //             Step*N = -Start (mod 2^BW)
06127   //
06128   // where BW is the common bit width of Start and Step.
06129 
06130   // Get the initial value for the loop.
06131   const SCEV *Start = getSCEVAtScope(AddRec->getStart(), L->getParentLoop());
06132   const SCEV *Step = getSCEVAtScope(AddRec->getOperand(1), L->getParentLoop());
06133 
06134   // For now we handle only constant steps.
06135   //
06136   // TODO: Handle a nonconstant Step given AddRec<NUW>. If the
06137   // AddRec is NUW, then (in an unsigned sense) it cannot be counting up to wrap
06138   // to 0, it must be counting down to equal 0. Consequently, N = Start / -Step.
06139   // We have not yet seen any such cases.
06140   const SCEVConstant *StepC = dyn_cast<SCEVConstant>(Step);
06141   if (!StepC || StepC->getValue()->equalsInt(0))
06142     return getCouldNotCompute();
06143 
06144   // For positive steps (counting up until unsigned overflow):
06145   //   N = -Start/Step (as unsigned)
06146   // For negative steps (counting down to zero):
06147   //   N = Start/-Step
06148   // First compute the unsigned distance from zero in the direction of Step.
06149   bool CountDown = StepC->getValue()->getValue().isNegative();
06150   const SCEV *Distance = CountDown ? Start : getNegativeSCEV(Start);
06151 
06152   // Handle unitary steps, which cannot wraparound.
06153   // 1*N = -Start; -1*N = Start (mod 2^BW), so:
06154   //   N = Distance (as unsigned)
06155   if (StepC->getValue()->equalsInt(1) || StepC->getValue()->isAllOnesValue()) {
06156     ConstantRange CR = getUnsignedRange(Start);
06157     const SCEV *MaxBECount;
06158     if (!CountDown && CR.getUnsignedMin().isMinValue())
06159       // When counting up, the worst starting value is 1, not 0.
06160       MaxBECount = CR.getUnsignedMax().isMinValue()
06161         ? getConstant(APInt::getMinValue(CR.getBitWidth()))
06162         : getConstant(APInt::getMaxValue(CR.getBitWidth()));
06163     else
06164       MaxBECount = getConstant(CountDown ? CR.getUnsignedMax()
06165                                          : -CR.getUnsignedMin());
06166     return ExitLimit(Distance, MaxBECount);
06167   }
06168 
06169   // As a special case, handle the instance where Step is a positive power of
06170   // two. In this case, determining whether Step divides Distance evenly can be
06171   // done by counting and comparing the number of trailing zeros of Step and
06172   // Distance.
06173   if (!CountDown) {
06174     const APInt &StepV = StepC->getValue()->getValue();
06175     // StepV.isPowerOf2() returns true if StepV is an positive power of two.  It
06176     // also returns true if StepV is maximally negative (eg, INT_MIN), but that
06177     // case is not handled as this code is guarded by !CountDown.
06178     if (StepV.isPowerOf2() &&
06179         GetMinTrailingZeros(Distance) >= StepV.countTrailingZeros())
06180       return getUDivExactExpr(Distance, Step);
06181   }
06182 
06183   // If the condition controls loop exit (the loop exits only if the expression
06184   // is true) and the addition is no-wrap we can use unsigned divide to
06185   // compute the backedge count.  In this case, the step may not divide the
06186   // distance, but we don't care because if the condition is "missed" the loop
06187   // will have undefined behavior due to wrapping.
06188   if (ControlsExit && AddRec->getNoWrapFlags(SCEV::FlagNW)) {
06189     const SCEV *Exact =
06190         getUDivExpr(Distance, CountDown ? getNegativeSCEV(Step) : Step);
06191     return ExitLimit(Exact, Exact);
06192   }
06193 
06194   // Then, try to solve the above equation provided that Start is constant.
06195   if (const SCEVConstant *StartC = dyn_cast<SCEVConstant>(Start))
06196     return SolveLinEquationWithOverflow(StepC->getValue()->getValue(),
06197                                         -StartC->getValue()->getValue(),
06198                                         *this);
06199   return getCouldNotCompute();
06200 }
06201 
06202 /// HowFarToNonZero - Return the number of times a backedge checking the
06203 /// specified value for nonzero will execute.  If not computable, return
06204 /// CouldNotCompute
06205 ScalarEvolution::ExitLimit
06206 ScalarEvolution::HowFarToNonZero(const SCEV *V, const Loop *L) {
06207   // Loops that look like: while (X == 0) are very strange indeed.  We don't
06208   // handle them yet except for the trivial case.  This could be expanded in the
06209   // future as needed.
06210 
06211   // If the value is a constant, check to see if it is known to be non-zero
06212   // already.  If so, the backedge will execute zero times.
06213   if (const SCEVConstant *C = dyn_cast<SCEVConstant>(V)) {
06214     if (!C->getValue()->isNullValue())
06215       return getConstant(C->getType(), 0);
06216     return getCouldNotCompute();  // Otherwise it will loop infinitely.
06217   }
06218 
06219   // We could implement others, but I really doubt anyone writes loops like
06220   // this, and if they did, they would already be constant folded.
06221   return getCouldNotCompute();
06222 }
06223 
06224 /// getPredecessorWithUniqueSuccessorForBB - Return a predecessor of BB
06225 /// (which may not be an immediate predecessor) which has exactly one
06226 /// successor from which BB is reachable, or null if no such block is
06227 /// found.
06228 ///
06229 std::pair<BasicBlock *, BasicBlock *>
06230 ScalarEvolution::getPredecessorWithUniqueSuccessorForBB(BasicBlock *BB) {
06231   // If the block has a unique predecessor, then there is no path from the
06232   // predecessor to the block that does not go through the direct edge
06233   // from the predecessor to the block.
06234   if (BasicBlock *Pred = BB->getSinglePredecessor())
06235     return std::make_pair(Pred, BB);
06236 
06237   // A loop's header is defined to be a block that dominates the loop.
06238   // If the header has a unique predecessor outside the loop, it must be
06239   // a block that has exactly one successor that can reach the loop.
06240   if (Loop *L = LI->getLoopFor(BB))
06241     return std::make_pair(L->getLoopPredecessor(), L->getHeader());
06242 
06243   return std::pair<BasicBlock *, BasicBlock *>();
06244 }
06245 
06246 /// HasSameValue - SCEV structural equivalence is usually sufficient for
06247 /// testing whether two expressions are equal, however for the purposes of
06248 /// looking for a condition guarding a loop, it can be useful to be a little
06249 /// more general, since a front-end may have replicated the controlling
06250 /// expression.
06251 ///
06252 static bool HasSameValue(const SCEV *A, const SCEV *B) {
06253   // Quick check to see if they are the same SCEV.
06254   if (A == B) return true;
06255 
06256   // Otherwise, if they're both SCEVUnknown, it's possible that they hold
06257   // two different instructions with the same value. Check for this case.
06258   if (const SCEVUnknown *AU = dyn_cast<SCEVUnknown>(A))
06259     if (const SCEVUnknown *BU = dyn_cast<SCEVUnknown>(B))
06260       if (const Instruction *AI = dyn_cast<Instruction>(AU->getValue()))
06261         if (const Instruction *BI = dyn_cast<Instruction>(BU->getValue()))
06262           if (AI->isIdenticalTo(BI) && !AI->mayReadFromMemory())
06263             return true;
06264 
06265   // Otherwise assume they may have a different value.
06266   return false;
06267 }
06268 
06269 /// SimplifyICmpOperands - Simplify LHS and RHS in a comparison with
06270 /// predicate Pred. Return true iff any changes were made.
06271 ///
06272 bool ScalarEvolution::SimplifyICmpOperands(ICmpInst::Predicate &Pred,
06273                                            const SCEV *&LHS, const SCEV *&RHS,
06274                                            unsigned Depth) {
06275   bool Changed = false;
06276 
06277   // If we hit the max recursion limit bail out.
06278   if (Depth >= 3)
06279     return false;
06280 
06281   // Canonicalize a constant to the right side.
06282   if (const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(LHS)) {
06283     // Check for both operands constant.
06284     if (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(RHS)) {
06285       if (ConstantExpr::getICmp(Pred,
06286                                 LHSC->getValue(),
06287                                 RHSC->getValue())->isNullValue())
06288         goto trivially_false;
06289       else
06290         goto trivially_true;
06291     }
06292     // Otherwise swap the operands to put the constant on the right.
06293     std::swap(LHS, RHS);
06294     Pred = ICmpInst::getSwappedPredicate(Pred);
06295     Changed = true;
06296   }
06297 
06298   // If we're comparing an addrec with a value which is loop-invariant in the
06299   // addrec's loop, put the addrec on the left. Also make a dominance check,
06300   // as both operands could be addrecs loop-invariant in each other's loop.
06301   if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(RHS)) {
06302     const Loop *L = AR->getLoop();
06303     if (isLoopInvariant(LHS, L) && properlyDominates(LHS, L->getHeader())) {
06304       std::swap(LHS, RHS);
06305       Pred = ICmpInst::getSwappedPredicate(Pred);
06306       Changed = true;
06307     }
06308   }
06309 
06310   // If there's a constant operand, canonicalize comparisons with boundary
06311   // cases, and canonicalize *-or-equal comparisons to regular comparisons.
06312   if (const SCEVConstant *RC = dyn_cast<SCEVConstant>(RHS)) {
06313     const APInt &RA = RC->getValue()->getValue();
06314     switch (Pred) {
06315     default: llvm_unreachable("Unexpected ICmpInst::Predicate value!");
06316     case ICmpInst::ICMP_EQ:
06317     case ICmpInst::ICMP_NE:
06318       // Fold ((-1) * %a) + %b == 0 (equivalent to %b-%a == 0) into %a == %b.
06319       if (!RA)
06320         if (const SCEVAddExpr *AE = dyn_cast<SCEVAddExpr>(LHS))
06321           if (const SCEVMulExpr *ME = dyn_cast<SCEVMulExpr>(AE->getOperand(0)))
06322             if (AE->getNumOperands() == 2 && ME->getNumOperands() == 2 &&
06323                 ME->getOperand(0)->isAllOnesValue()) {
06324               RHS = AE->getOperand(1);
06325               LHS = ME->getOperand(1);
06326               Changed = true;
06327             }
06328       break;
06329     case ICmpInst::ICMP_UGE:
06330       if ((RA - 1).isMinValue()) {
06331         Pred = ICmpInst::ICMP_NE;
06332         RHS = getConstant(RA - 1);
06333         Changed = true;
06334         break;
06335       }
06336       if (RA.isMaxValue()) {
06337         Pred = ICmpInst::ICMP_EQ;
06338         Changed = true;
06339         break;
06340       }
06341       if (RA.isMinValue()) goto trivially_true;
06342 
06343       Pred = ICmpInst::ICMP_UGT;
06344       RHS = getConstant(RA - 1);
06345       Changed = true;
06346       break;
06347     case ICmpInst::ICMP_ULE:
06348       if ((RA + 1).isMaxValue()) {
06349         Pred = ICmpInst::ICMP_NE;
06350         RHS = getConstant(RA + 1);
06351         Changed = true;
06352         break;
06353       }
06354       if (RA.isMinValue()) {
06355         Pred = ICmpInst::ICMP_EQ;
06356         Changed = true;
06357         break;
06358       }
06359       if (RA.isMaxValue()) goto trivially_true;
06360 
06361       Pred = ICmpInst::ICMP_ULT;
06362       RHS = getConstant(RA + 1);
06363       Changed = true;
06364       break;
06365     case ICmpInst::ICMP_SGE:
06366       if ((RA - 1).isMinSignedValue()) {
06367         Pred = ICmpInst::ICMP_NE;
06368         RHS = getConstant(RA - 1);
06369         Changed = true;
06370         break;
06371       }
06372       if (RA.isMaxSignedValue()) {
06373         Pred = ICmpInst::ICMP_EQ;
06374         Changed = true;
06375         break;
06376       }
06377       if (RA.isMinSignedValue()) goto trivially_true;
06378 
06379       Pred = ICmpInst::ICMP_SGT;
06380       RHS = getConstant(RA - 1);
06381       Changed = true;
06382       break;
06383     case ICmpInst::ICMP_SLE:
06384       if ((RA + 1).isMaxSignedValue()) {
06385         Pred = ICmpInst::ICMP_NE;
06386         RHS = getConstant(RA + 1);
06387         Changed = true;
06388         break;
06389       }
06390       if (RA.isMinSignedValue()) {
06391         Pred = ICmpInst::ICMP_EQ;
06392         Changed = true;
06393         break;
06394       }
06395       if (RA.isMaxSignedValue()) goto trivially_true;
06396 
06397       Pred = ICmpInst::ICMP_SLT;
06398       RHS = getConstant(RA + 1);
06399       Changed = true;
06400       break;
06401     case ICmpInst::ICMP_UGT:
06402       if (RA.isMinValue()) {
06403         Pred = ICmpInst::ICMP_NE;
06404         Changed = true;
06405         break;
06406       }
06407       if ((RA + 1).isMaxValue()) {
06408         Pred = ICmpInst::ICMP_EQ;
06409         RHS = getConstant(RA + 1);
06410         Changed = true;
06411         break;
06412       }
06413       if (RA.isMaxValue()) goto trivially_false;
06414       break;
06415     case ICmpInst::ICMP_ULT:
06416       if (RA.isMaxValue()) {
06417         Pred = ICmpInst::ICMP_NE;
06418         Changed = true;
06419         break;
06420       }
06421       if ((RA - 1).isMinValue()) {
06422         Pred = ICmpInst::ICMP_EQ;
06423         RHS = getConstant(RA - 1);
06424         Changed = true;
06425         break;
06426       }
06427       if (RA.isMinValue()) goto trivially_false;
06428       break;
06429     case ICmpInst::ICMP_SGT:
06430       if (RA.isMinSignedValue()) {
06431         Pred = ICmpInst::ICMP_NE;
06432         Changed = true;
06433         break;
06434       }
06435       if ((RA + 1).isMaxSignedValue()) {
06436         Pred = ICmpInst::ICMP_EQ;
06437         RHS = getConstant(RA + 1);
06438         Changed = true;
06439         break;
06440       }
06441       if (RA.isMaxSignedValue()) goto trivially_false;
06442       break;
06443     case ICmpInst::ICMP_SLT:
06444       if (RA.isMaxSignedValue()) {
06445         Pred = ICmpInst::ICMP_NE;
06446         Changed = true;
06447         break;
06448       }
06449       if ((RA - 1).isMinSignedValue()) {
06450        Pred = ICmpInst::ICMP_EQ;
06451        RHS = getConstant(RA - 1);
06452         Changed = true;
06453        break;
06454       }
06455       if (RA.isMinSignedValue()) goto trivially_false;
06456       break;
06457     }
06458   }
06459 
06460   // Check for obvious equality.
06461   if (HasSameValue(LHS, RHS)) {
06462     if (ICmpInst::isTrueWhenEqual(Pred))
06463       goto trivially_true;
06464     if (ICmpInst::isFalseWhenEqual(Pred))
06465       goto trivially_false;
06466   }
06467 
06468   // If possible, canonicalize GE/LE comparisons to GT/LT comparisons, by
06469   // adding or subtracting 1 from one of the operands.
06470   switch (Pred) {
06471   case ICmpInst::ICMP_SLE:
06472     if (!getSignedRange(RHS).getSignedMax().isMaxSignedValue()) {
06473       RHS = getAddExpr(getConstant(RHS->getType(), 1, true), RHS,
06474                        SCEV::FlagNSW);
06475       Pred = ICmpInst::ICMP_SLT;
06476       Changed = true;
06477     } else if (!getSignedRange(LHS).getSignedMin().isMinSignedValue()) {
06478       LHS = getAddExpr(getConstant(RHS->getType(), (uint64_t)-1, true), LHS,
06479                        SCEV::FlagNSW);
06480       Pred = ICmpInst::ICMP_SLT;
06481       Changed = true;
06482     }
06483     break;
06484   case ICmpInst::ICMP_SGE:
06485     if (!getSignedRange(RHS).getSignedMin().isMinSignedValue()) {
06486       RHS = getAddExpr(getConstant(RHS->getType(), (uint64_t)-1, true), RHS,
06487                        SCEV::FlagNSW);
06488       Pred = ICmpInst::ICMP_SGT;
06489       Changed = true;
06490     } else if (!getSignedRange(LHS).getSignedMax().isMaxSignedValue()) {
06491       LHS = getAddExpr(getConstant(RHS->getType(), 1, true), LHS,
06492                        SCEV::FlagNSW);
06493       Pred = ICmpInst::ICMP_SGT;
06494       Changed = true;
06495     }
06496     break;
06497   case ICmpInst::ICMP_ULE:
06498     if (!getUnsignedRange(RHS).getUnsignedMax().isMaxValue()) {
06499       RHS = getAddExpr(getConstant(RHS->getType(), 1, true), RHS,
06500                        SCEV::FlagNUW);
06501       Pred = ICmpInst::ICMP_ULT;
06502       Changed = true;
06503     } else if (!getUnsignedRange(LHS).getUnsignedMin().isMinValue()) {
06504       LHS = getAddExpr(getConstant(RHS->getType(), (uint64_t)-1, true), LHS,
06505                        SCEV::FlagNUW);
06506       Pred = ICmpInst::ICMP_ULT;
06507       Changed = true;
06508     }
06509     break;
06510   case ICmpInst::ICMP_UGE:
06511     if (!getUnsignedRange(RHS).getUnsignedMin().isMinValue()) {
06512       RHS = getAddExpr(getConstant(RHS->getType(), (uint64_t)-1, true), RHS,
06513                        SCEV::FlagNUW);
06514       Pred = ICmpInst::ICMP_UGT;
06515       Changed = true;
06516     } else if (!getUnsignedRange(LHS).getUnsignedMax().isMaxValue()) {
06517       LHS = getAddExpr(getConstant(RHS->getType(), 1, true), LHS,
06518                        SCEV::FlagNUW);
06519       Pred = ICmpInst::ICMP_UGT;
06520       Changed = true;
06521     }
06522     break;
06523   default:
06524     break;
06525   }
06526 
06527   // TODO: More simplifications are possible here.
06528 
06529   // Recursively simplify until we either hit a recursion limit or nothing
06530   // changes.
06531   if (Changed)
06532     return SimplifyICmpOperands(Pred, LHS, RHS, Depth+1);
06533 
06534   return Changed;
06535 
06536 trivially_true:
06537   // Return 0 == 0.
06538   LHS = RHS = getConstant(ConstantInt::getFalse(getContext()));
06539   Pred = ICmpInst::ICMP_EQ;
06540   return true;
06541 
06542 trivially_false:
06543   // Return 0 != 0.
06544   LHS = RHS = getConstant(ConstantInt::getFalse(getContext()));
06545   Pred = ICmpInst::ICMP_NE;
06546   return true;
06547 }
06548 
06549 bool ScalarEvolution::isKnownNegative(const SCEV *S) {
06550   return getSignedRange(S).getSignedMax().isNegative();
06551 }
06552 
06553 bool ScalarEvolution::isKnownPositive(const SCEV *S) {
06554   return getSignedRange(S).getSignedMin().isStrictlyPositive();
06555 }
06556 
06557 bool ScalarEvolution::isKnownNonNegative(const SCEV *S) {
06558   return !getSignedRange(S).getSignedMin().isNegative();
06559 }
06560 
06561 bool ScalarEvolution::isKnownNonPositive(const SCEV *S) {
06562   return !getSignedRange(S).getSignedMax().isStrictlyPositive();
06563 }
06564 
06565 bool ScalarEvolution::isKnownNonZero(const SCEV *S) {
06566   return isKnownNegative(S) || isKnownPositive(S);
06567 }
06568 
06569 bool ScalarEvolution::isKnownPredicate(ICmpInst::Predicate Pred,
06570                                        const SCEV *LHS, const SCEV *RHS) {
06571   // Canonicalize the inputs first.
06572   (void)SimplifyICmpOperands(Pred, LHS, RHS);
06573 
06574   // If LHS or RHS is an addrec, check to see if the condition is true in
06575   // every iteration of the loop.
06576   // If LHS and RHS are both addrec, both conditions must be true in
06577   // every iteration of the loop.
06578   const SCEVAddRecExpr *LAR = dyn_cast<SCEVAddRecExpr>(LHS);
06579   const SCEVAddRecExpr *RAR = dyn_cast<SCEVAddRecExpr>(RHS);
06580   bool LeftGuarded = false;
06581   bool RightGuarded = false;
06582   if (LAR) {
06583     const Loop *L = LAR->getLoop();
06584     if (isLoopEntryGuardedByCond(L, Pred, LAR->getStart(), RHS) &&
06585         isLoopBackedgeGuardedByCond(L, Pred, LAR->getPostIncExpr(*this), RHS)) {
06586       if (!RAR) return true;
06587       LeftGuarded = true;
06588     }
06589   }
06590   if (RAR) {
06591     const Loop *L = RAR->getLoop();
06592     if (isLoopEntryGuardedByCond(L, Pred, LHS, RAR->getStart()) &&
06593         isLoopBackedgeGuardedByCond(L, Pred, LHS, RAR->getPostIncExpr(*this))) {
06594       if (!LAR) return true;
06595       RightGuarded = true;
06596     }
06597   }
06598   if (LeftGuarded && RightGuarded)
06599     return true;
06600 
06601   // Otherwise see what can be done with known constant ranges.
06602   return isKnownPredicateWithRanges(Pred, LHS, RHS);
06603 }
06604 
06605 bool
06606 ScalarEvolution::isKnownPredicateWithRanges(ICmpInst::Predicate Pred,
06607                                             const SCEV *LHS, const SCEV *RHS) {
06608   if (HasSameValue(LHS, RHS))
06609     return ICmpInst::isTrueWhenEqual(Pred);
06610 
06611   // This code is split out from isKnownPredicate because it is called from
06612   // within isLoopEntryGuardedByCond.
06613   switch (Pred) {
06614   default:
06615     llvm_unreachable("Unexpected ICmpInst::Predicate value!");
06616   case ICmpInst::ICMP_SGT:
06617     std::swap(LHS, RHS);
06618   case ICmpInst::ICMP_SLT: {
06619     ConstantRange LHSRange = getSignedRange(LHS);
06620     ConstantRange RHSRange = getSignedRange(RHS);
06621     if (LHSRange.getSignedMax().slt(RHSRange.getSignedMin()))
06622       return true;
06623     if (LHSRange.getSignedMin().sge(RHSRange.getSignedMax()))
06624       return false;
06625     break;
06626   }
06627   case ICmpInst::ICMP_SGE:
06628     std::swap(LHS, RHS);
06629   case ICmpInst::ICMP_SLE: {
06630     ConstantRange LHSRange = getSignedRange(LHS);
06631     ConstantRange RHSRange = getSignedRange(RHS);
06632     if (LHSRange.getSignedMax().sle(RHSRange.getSignedMin()))
06633       return true;
06634     if (LHSRange.getSignedMin().sgt(RHSRange.getSignedMax()))
06635       return false;
06636     break;
06637   }
06638   case ICmpInst::ICMP_UGT:
06639     std::swap(LHS, RHS);
06640   case ICmpInst::ICMP_ULT: {
06641     ConstantRange LHSRange = getUnsignedRange(LHS);
06642     ConstantRange RHSRange = getUnsignedRange(RHS);
06643     if (LHSRange.getUnsignedMax().ult(RHSRange.getUnsignedMin()))
06644       return true;
06645     if (LHSRange.getUnsignedMin().uge(RHSRange.getUnsignedMax()))
06646       return false;
06647     break;
06648   }
06649   case ICmpInst::ICMP_UGE:
06650     std::swap(LHS, RHS);
06651   case ICmpInst::ICMP_ULE: {
06652     ConstantRange LHSRange = getUnsignedRange(LHS);
06653     ConstantRange RHSRange = getUnsignedRange(RHS);
06654     if (LHSRange.getUnsignedMax().ule(RHSRange.getUnsignedMin()))
06655       return true;
06656     if (LHSRange.getUnsignedMin().ugt(RHSRange.getUnsignedMax()))
06657       return false;
06658     break;
06659   }
06660   case ICmpInst::ICMP_NE: {
06661     if (getUnsignedRange(LHS).intersectWith(getUnsignedRange(RHS)).isEmptySet())
06662       return true;
06663     if (getSignedRange(LHS).intersectWith(getSignedRange(RHS)).isEmptySet())
06664       return true;
06665 
06666     const SCEV *Diff = getMinusSCEV(LHS, RHS);
06667     if (isKnownNonZero(Diff))
06668       return true;
06669     break;
06670   }
06671   case ICmpInst::ICMP_EQ:
06672     // The check at the top of the function catches the case where
06673     // the values are known to be equal.
06674     break;
06675   }
06676   return false;
06677 }
06678 
06679 /// isLoopBackedgeGuardedByCond - Test whether the backedge of the loop is
06680 /// protected by a conditional between LHS and RHS.  This is used to
06681 /// to eliminate casts.
06682 bool
06683 ScalarEvolution::isLoopBackedgeGuardedByCond(const Loop *L,
06684                                              ICmpInst::Predicate Pred,
06685                                              const SCEV *LHS, const SCEV *RHS) {
06686   // Interpret a null as meaning no loop, where there is obviously no guard
06687   // (interprocedural conditions notwithstanding).
06688   if (!L) return true;
06689 
06690   if (isKnownPredicateWithRanges(Pred, LHS, RHS)) return true;
06691 
06692   BasicBlock *Latch = L->getLoopLatch();
06693   if (!Latch)
06694     return false;
06695 
06696   BranchInst *LoopContinuePredicate =
06697     dyn_cast<BranchInst>(Latch->getTerminator());
06698   if (LoopContinuePredicate && LoopContinuePredicate->isConditional() &&
06699       isImpliedCond(Pred, LHS, RHS,
06700                     LoopContinuePredicate->getCondition(),
06701                     LoopContinuePredicate->getSuccessor(0) != L->getHeader()))
06702     return true;
06703 
06704   // Check conditions due to any @llvm.assume intrinsics.
06705   for (auto &AssumeVH : AC->assumptions()) {
06706     if (!AssumeVH)
06707       continue;
06708     auto *CI = cast<CallInst>(AssumeVH);
06709     if (!DT->dominates(CI, Latch->getTerminator()))
06710       continue;
06711 
06712     if (isImpliedCond(Pred, LHS, RHS, CI->getArgOperand(0), false))
06713       return true;
06714   }
06715 
06716   struct ClearWalkingBEDominatingCondsOnExit {
06717     ScalarEvolution &SE;
06718 
06719     explicit ClearWalkingBEDominatingCondsOnExit(ScalarEvolution &SE)
06720         : SE(SE){};
06721 
06722     ~ClearWalkingBEDominatingCondsOnExit() {
06723       SE.WalkingBEDominatingConds = false;
06724     }
06725   };
06726 
06727   // We don't want more than one activation of the following loop on the stack
06728   // -- that can lead to O(n!) time complexity.
06729   if (WalkingBEDominatingConds)
06730     return false;
06731 
06732   WalkingBEDominatingConds = true;
06733   ClearWalkingBEDominatingCondsOnExit ClearOnExit(*this);
06734 
06735   // If the loop is not reachable from the entry block, we risk running into an
06736   // infinite loop as we walk up into the dom tree.  These loops do not matter
06737   // anyway, so we just return a conservative answer when we see them.
06738   if (!DT->isReachableFromEntry(L->getHeader()))
06739     return false;
06740 
06741   for (DomTreeNode *DTN = (*DT)[Latch], *HeaderDTN = (*DT)[L->getHeader()];
06742        DTN != HeaderDTN;
06743        DTN = DTN->getIDom()) {
06744 
06745     assert(DTN && "should reach the loop header before reaching the root!");
06746 
06747     BasicBlock *BB = DTN->getBlock();
06748     BasicBlock *PBB = BB->getSinglePredecessor();
06749     if (!PBB)
06750       continue;
06751 
06752     BranchInst *ContinuePredicate = dyn_cast<BranchInst>(PBB->getTerminator());
06753     if (!ContinuePredicate || !ContinuePredicate->isConditional())
06754       continue;
06755 
06756     Value *Condition = ContinuePredicate->getCondition();
06757 
06758     // If we have an edge `E` within the loop body that dominates the only
06759     // latch, the condition guarding `E` also guards the backedge.  This
06760     // reasoning works only for loops with a single latch.
06761 
06762     BasicBlockEdge DominatingEdge(PBB, BB);
06763     if (DominatingEdge.isSingleEdge()) {
06764       // We're constructively (and conservatively) enumerating edges within the
06765       // loop body that dominate the latch.  The dominator tree better agree
06766       // with us on this:
06767       assert(DT->dominates(DominatingEdge, Latch) && "should be!");
06768 
06769       if (isImpliedCond(Pred, LHS, RHS, Condition,
06770                         BB != ContinuePredicate->getSuccessor(0)))
06771         return true;
06772     }
06773   }
06774 
06775   return false;
06776 }
06777 
06778 /// isLoopEntryGuardedByCond - Test whether entry to the loop is protected
06779 /// by a conditional between LHS and RHS.  This is used to help avoid max
06780 /// expressions in loop trip counts, and to eliminate casts.
06781 bool
06782 ScalarEvolution::isLoopEntryGuardedByCond(const Loop *L,
06783                                           ICmpInst::Predicate Pred,
06784                                           const SCEV *LHS, const SCEV *RHS) {
06785   // Interpret a null as meaning no loop, where there is obviously no guard
06786   // (interprocedural conditions notwithstanding).
06787   if (!L) return false;
06788 
06789   if (isKnownPredicateWithRanges(Pred, LHS, RHS)) return true;
06790 
06791   // Starting at the loop predecessor, climb up the predecessor chain, as long
06792   // as there are predecessors that can be found that have unique successors
06793   // leading to the original header.
06794   for (std::pair<BasicBlock *, BasicBlock *>
06795          Pair(L->getLoopPredecessor(), L->getHeader());
06796        Pair.first;
06797        Pair = getPredecessorWithUniqueSuccessorForBB(Pair.first)) {
06798 
06799     BranchInst *LoopEntryPredicate =
06800       dyn_cast<BranchInst>(Pair.first->getTerminator());
06801     if (!LoopEntryPredicate ||
06802         LoopEntryPredicate->isUnconditional())
06803       continue;
06804 
06805     if (isImpliedCond(Pred, LHS, RHS,
06806                       LoopEntryPredicate->getCondition(),
06807                       LoopEntryPredicate->getSuccessor(0) != Pair.second))
06808       return true;
06809   }
06810 
06811   // Check conditions due to any @llvm.assume intrinsics.
06812   for (auto &AssumeVH : AC->assumptions()) {
06813     if (!AssumeVH)
06814       continue;
06815     auto *CI = cast<CallInst>(AssumeVH);
06816     if (!DT->dominates(CI, L->getHeader()))
06817       continue;
06818 
06819     if (isImpliedCond(Pred, LHS, RHS, CI->getArgOperand(0), false))
06820       return true;
06821   }
06822 
06823   return false;
06824 }
06825 
06826 /// RAII wrapper to prevent recursive application of isImpliedCond.
06827 /// ScalarEvolution's PendingLoopPredicates set must be empty unless we are
06828 /// currently evaluating isImpliedCond.
06829 struct MarkPendingLoopPredicate {
06830   Value *Cond;
06831   DenseSet<Value*> &LoopPreds;
06832   bool Pending;
06833 
06834   MarkPendingLoopPredicate(Value *C, DenseSet<Value*> &LP)
06835     : Cond(C), LoopPreds(LP) {
06836     Pending = !LoopPreds.insert(Cond).second;
06837   }
06838   ~MarkPendingLoopPredicate() {
06839     if (!Pending)
06840       LoopPreds.erase(Cond);
06841   }
06842 };
06843 
06844 /// isImpliedCond - Test whether the condition described by Pred, LHS,
06845 /// and RHS is true whenever the given Cond value evaluates to true.
06846 bool ScalarEvolution::isImpliedCond(ICmpInst::Predicate Pred,
06847                                     const SCEV *LHS, const SCEV *RHS,
06848                                     Value *FoundCondValue,
06849                                     bool Inverse) {
06850   MarkPendingLoopPredicate Mark(FoundCondValue, PendingLoopPredicates);
06851   if (Mark.Pending)
06852     return false;
06853 
06854   // Recursively handle And and Or conditions.
06855   if (BinaryOperator *BO = dyn_cast<BinaryOperator>(FoundCondValue)) {
06856     if (BO->getOpcode() == Instruction::And) {
06857       if (!Inverse)
06858         return isImpliedCond(Pred, LHS, RHS, BO->getOperand(0), Inverse) ||
06859                isImpliedCond(Pred, LHS, RHS, BO->getOperand(1), Inverse);
06860     } else if (BO->getOpcode() == Instruction::Or) {
06861       if (Inverse)
06862         return isImpliedCond(Pred, LHS, RHS, BO->getOperand(0), Inverse) ||
06863                isImpliedCond(Pred, LHS, RHS, BO->getOperand(1), Inverse);
06864     }
06865   }
06866 
06867   ICmpInst *ICI = dyn_cast<ICmpInst>(FoundCondValue);
06868   if (!ICI) return false;
06869 
06870   // Now that we found a conditional branch that dominates the loop or controls
06871   // the loop latch. Check to see if it is the comparison we are looking for.
06872   ICmpInst::Predicate FoundPred;
06873   if (Inverse)
06874     FoundPred = ICI->getInversePredicate();
06875   else
06876     FoundPred = ICI->getPredicate();
06877 
06878   const SCEV *FoundLHS = getSCEV(ICI->getOperand(0));
06879   const SCEV *FoundRHS = getSCEV(ICI->getOperand(1));
06880 
06881   // Balance the types.
06882   if (getTypeSizeInBits(LHS->getType()) <
06883       getTypeSizeInBits(FoundLHS->getType())) {
06884     if (CmpInst::isSigned(Pred)) {
06885       LHS = getSignExtendExpr(LHS, FoundLHS->getType());
06886       RHS = getSignExtendExpr(RHS, FoundLHS->getType());
06887     } else {
06888       LHS = getZeroExtendExpr(LHS, FoundLHS->getType());
06889       RHS = getZeroExtendExpr(RHS, FoundLHS->getType());
06890     }
06891   } else if (getTypeSizeInBits(LHS->getType()) >
06892       getTypeSizeInBits(FoundLHS->getType())) {
06893     if (CmpInst::isSigned(FoundPred)) {
06894       FoundLHS = getSignExtendExpr(FoundLHS, LHS->getType());
06895       FoundRHS = getSignExtendExpr(FoundRHS, LHS->getType());
06896     } else {
06897       FoundLHS = getZeroExtendExpr(FoundLHS, LHS->getType());
06898       FoundRHS = getZeroExtendExpr(FoundRHS, LHS->getType());
06899     }
06900   }
06901 
06902   // Canonicalize the query to match the way instcombine will have
06903   // canonicalized the comparison.
06904   if (SimplifyICmpOperands(Pred, LHS, RHS))
06905     if (LHS == RHS)
06906       return CmpInst::isTrueWhenEqual(Pred);
06907   if (SimplifyICmpOperands(FoundPred, FoundLHS, FoundRHS))
06908     if (FoundLHS == FoundRHS)
06909       return CmpInst::isFalseWhenEqual(FoundPred);
06910 
06911   // Check to see if we can make the LHS or RHS match.
06912   if (LHS == FoundRHS || RHS == FoundLHS) {
06913     if (isa<SCEVConstant>(RHS)) {
06914       std::swap(FoundLHS, FoundRHS);
06915       FoundPred = ICmpInst::getSwappedPredicate(FoundPred);
06916     } else {
06917       std::swap(LHS, RHS);
06918       Pred = ICmpInst::getSwappedPredicate(Pred);
06919     }
06920   }
06921 
06922   // Check whether the found predicate is the same as the desired predicate.
06923   if (FoundPred == Pred)
06924     return isImpliedCondOperands(Pred, LHS, RHS, FoundLHS, FoundRHS);
06925 
06926   // Check whether swapping the found predicate makes it the same as the
06927   // desired predicate.
06928   if (ICmpInst::getSwappedPredicate(FoundPred) == Pred) {
06929     if (isa<SCEVConstant>(RHS))
06930       return isImpliedCondOperands(Pred, LHS, RHS, FoundRHS, FoundLHS);
06931     else
06932       return isImpliedCondOperands(ICmpInst::getSwappedPredicate(Pred),
06933                                    RHS, LHS, FoundLHS, FoundRHS);
06934   }
06935 
06936   // Check if we can make progress by sharpening ranges.
06937   if (FoundPred == ICmpInst::ICMP_NE &&
06938       (isa<SCEVConstant>(FoundLHS) || isa<SCEVConstant>(FoundRHS))) {
06939 
06940     const SCEVConstant *C = nullptr;
06941     const SCEV *V = nullptr;
06942 
06943     if (isa<SCEVConstant>(FoundLHS)) {
06944       C = cast<SCEVConstant>(FoundLHS);
06945       V = FoundRHS;
06946     } else {
06947       C = cast<SCEVConstant>(FoundRHS);
06948       V = FoundLHS;
06949     }
06950 
06951     // The guarding predicate tells us that C != V. If the known range
06952     // of V is [C, t), we can sharpen the range to [C + 1, t).  The
06953     // range we consider has to correspond to same signedness as the
06954     // predicate we're interested in folding.
06955 
06956     APInt Min = ICmpInst::isSigned(Pred) ?
06957         getSignedRange(V).getSignedMin() : getUnsignedRange(V).getUnsignedMin();
06958 
06959     if (Min == C->getValue()->getValue()) {
06960       // Given (V >= Min && V != Min) we conclude V >= (Min + 1).
06961       // This is true even if (Min + 1) wraps around -- in case of
06962       // wraparound, (Min + 1) < Min, so (V >= Min => V >= (Min + 1)).
06963 
06964       APInt SharperMin = Min + 1;
06965 
06966       switch (Pred) {
06967         case ICmpInst::ICMP_SGE:
06968         case ICmpInst::ICMP_UGE:
06969           // We know V `Pred` SharperMin.  If this implies LHS `Pred`
06970           // RHS, we're done.
06971           if (isImpliedCondOperands(Pred, LHS, RHS, V,
06972                                     getConstant(SharperMin)))
06973             return true;
06974 
06975         case ICmpInst::ICMP_SGT:
06976         case ICmpInst::ICMP_UGT:
06977           // We know from the range information that (V `Pred` Min ||
06978           // V == Min).  We know from the guarding condition that !(V
06979           // == Min).  This gives us
06980           //
06981           //       V `Pred` Min || V == Min && !(V == Min)
06982           //   =>  V `Pred` Min
06983           //
06984           // If V `Pred` Min implies LHS `Pred` RHS, we're done.
06985 
06986           if (isImpliedCondOperands(Pred, LHS, RHS, V, getConstant(Min)))
06987             return true;
06988 
06989         default:
06990           // No change
06991           break;
06992       }
06993     }
06994   }
06995 
06996   // Check whether the actual condition is beyond sufficient.
06997   if (FoundPred == ICmpInst::ICMP_EQ)
06998     if (ICmpInst::isTrueWhenEqual(Pred))
06999       if (isImpliedCondOperands(Pred, LHS, RHS, FoundLHS, FoundRHS))
07000         return true;
07001   if (Pred == ICmpInst::ICMP_NE)
07002     if (!ICmpInst::isTrueWhenEqual(FoundPred))
07003       if (isImpliedCondOperands(FoundPred, LHS, RHS, FoundLHS, FoundRHS))
07004         return true;
07005 
07006   // Otherwise assume the worst.
07007   return false;
07008 }
07009 
07010 /// isImpliedCondOperands - Test whether the condition described by Pred,
07011 /// LHS, and RHS is true whenever the condition described by Pred, FoundLHS,
07012 /// and FoundRHS is true.
07013 bool ScalarEvolution::isImpliedCondOperands(ICmpInst::Predicate Pred,
07014                                             const SCEV *LHS, const SCEV *RHS,
07015                                             const SCEV *FoundLHS,
07016                                             const SCEV *FoundRHS) {
07017   if (isImpliedCondOperandsViaRanges(Pred, LHS, RHS, FoundLHS, FoundRHS))
07018     return true;
07019 
07020   return isImpliedCondOperandsHelper(Pred, LHS, RHS,
07021                                      FoundLHS, FoundRHS) ||
07022          // ~x < ~y --> x > y
07023          isImpliedCondOperandsHelper(Pred, LHS, RHS,
07024                                      getNotSCEV(FoundRHS),
07025                                      getNotSCEV(FoundLHS));
07026 }
07027 
07028 
07029 /// If Expr computes ~A, return A else return nullptr
07030 static const SCEV *MatchNotExpr(const SCEV *Expr) {
07031   const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(Expr);
07032   if (!Add || Add->getNumOperands() != 2) return nullptr;
07033 
07034   const SCEVConstant *AddLHS = dyn_cast<SCEVConstant>(Add->getOperand(0));
07035   if (!(AddLHS && AddLHS->getValue()->getValue().isAllOnesValue()))
07036     return nullptr;
07037 
07038   const SCEVMulExpr *AddRHS = dyn_cast<SCEVMulExpr>(Add->getOperand(1));
07039   if (!AddRHS || AddRHS->getNumOperands() != 2) return nullptr;
07040 
07041   const SCEVConstant *MulLHS = dyn_cast<SCEVConstant>(AddRHS->getOperand(0));
07042   if (!(MulLHS && MulLHS->getValue()->getValue().isAllOnesValue()))
07043     return nullptr;
07044 
07045   return AddRHS->getOperand(1);
07046 }
07047 
07048 
07049 /// Is MaybeMaxExpr an SMax or UMax of Candidate and some other values?
07050 template<typename MaxExprType>
07051 static bool IsMaxConsistingOf(const SCEV *MaybeMaxExpr,
07052                               const SCEV *Candidate) {
07053   const MaxExprType *MaxExpr = dyn_cast<MaxExprType>(MaybeMaxExpr);
07054   if (!MaxExpr) return false;
07055 
07056   auto It = std::find(MaxExpr->op_begin(), MaxExpr->op_end(), Candidate);
07057   return It != MaxExpr->op_end();
07058 }
07059 
07060 
07061 /// Is MaybeMinExpr an SMin or UMin of Candidate and some other values?
07062 template<typename MaxExprType>
07063 static bool IsMinConsistingOf(ScalarEvolution &SE,
07064                               const SCEV *MaybeMinExpr,
07065                               const SCEV *Candidate) {
07066   const SCEV *MaybeMaxExpr = MatchNotExpr(MaybeMinExpr);
07067   if (!MaybeMaxExpr)
07068     return false;
07069 
07070   return IsMaxConsistingOf<MaxExprType>(MaybeMaxExpr, SE.getNotSCEV(Candidate));
07071 }
07072 
07073 
07074 /// Is LHS `Pred` RHS true on the virtue of LHS or RHS being a Min or Max
07075 /// expression?
07076 static bool IsKnownPredicateViaMinOrMax(ScalarEvolution &SE,
07077                                         ICmpInst::Predicate Pred,
07078                                         const SCEV *LHS, const