LLVM  mainline
LoopVectorize.cpp
Go to the documentation of this file.
00001 //===- LoopVectorize.cpp - A Loop Vectorizer ------------------------------===//
00002 //
00003 //                     The LLVM Compiler Infrastructure
00004 //
00005 // This file is distributed under the University of Illinois Open Source
00006 // License. See LICENSE.TXT for details.
00007 //
00008 //===----------------------------------------------------------------------===//
00009 //
00010 // This is the LLVM loop vectorizer. This pass modifies 'vectorizable' loops
00011 // and generates target-independent LLVM-IR.
00012 // The vectorizer uses the TargetTransformInfo analysis to estimate the costs
00013 // of instructions in order to estimate the profitability of vectorization.
00014 //
00015 // The loop vectorizer combines consecutive loop iterations into a single
00016 // 'wide' iteration. After this transformation the index is incremented
00017 // by the SIMD vector width, and not by one.
00018 //
00019 // This pass has three parts:
00020 // 1. The main loop pass that drives the different parts.
00021 // 2. LoopVectorizationLegality - A unit that checks for the legality
00022 //    of the vectorization.
00023 // 3. InnerLoopVectorizer - A unit that performs the actual
00024 //    widening of instructions.
00025 // 4. LoopVectorizationCostModel - A unit that checks for the profitability
00026 //    of vectorization. It decides on the optimal vector width, which
00027 //    can be one, if vectorization is not profitable.
00028 //
00029 //===----------------------------------------------------------------------===//
00030 //
00031 // The reduction-variable vectorization is based on the paper:
00032 //  D. Nuzman and R. Henderson. Multi-platform Auto-vectorization.
00033 //
00034 // Variable uniformity checks are inspired by:
00035 //  Karrenberg, R. and Hack, S. Whole Function Vectorization.
00036 //
00037 // The interleaved access vectorization is based on the paper:
00038 //  Dorit Nuzman, Ira Rosen and Ayal Zaks.  Auto-Vectorization of Interleaved
00039 //  Data for SIMD
00040 //
00041 // Other ideas/concepts are from:
00042 //  A. Zaks and D. Nuzman. Autovectorization in GCC-two years later.
00043 //
00044 //  S. Maleki, Y. Gao, M. Garzaran, T. Wong and D. Padua.  An Evaluation of
00045 //  Vectorizing Compilers.
00046 //
00047 //===----------------------------------------------------------------------===//
00048 
00049 #include "llvm/Transforms/Vectorize.h"
00050 #include "llvm/ADT/DenseMap.h"
00051 #include "llvm/ADT/Hashing.h"
00052 #include "llvm/ADT/MapVector.h"
00053 #include "llvm/ADT/SetVector.h"
00054 #include "llvm/ADT/SmallPtrSet.h"
00055 #include "llvm/ADT/SmallSet.h"
00056 #include "llvm/ADT/SmallVector.h"
00057 #include "llvm/ADT/Statistic.h"
00058 #include "llvm/ADT/StringExtras.h"
00059 #include "llvm/Analysis/AliasAnalysis.h"
00060 #include "llvm/Analysis/BasicAliasAnalysis.h"
00061 #include "llvm/Analysis/AliasSetTracker.h"
00062 #include "llvm/Analysis/AssumptionCache.h"
00063 #include "llvm/Analysis/BlockFrequencyInfo.h"
00064 #include "llvm/Analysis/CodeMetrics.h"
00065 #include "llvm/Analysis/DemandedBits.h"
00066 #include "llvm/Analysis/GlobalsModRef.h"
00067 #include "llvm/Analysis/LoopAccessAnalysis.h"
00068 #include "llvm/Analysis/LoopInfo.h"
00069 #include "llvm/Analysis/LoopIterator.h"
00070 #include "llvm/Analysis/LoopPass.h"
00071 #include "llvm/Analysis/ScalarEvolution.h"
00072 #include "llvm/Analysis/ScalarEvolutionExpander.h"
00073 #include "llvm/Analysis/ScalarEvolutionExpressions.h"
00074 #include "llvm/Analysis/TargetTransformInfo.h"
00075 #include "llvm/Analysis/ValueTracking.h"
00076 #include "llvm/IR/Constants.h"
00077 #include "llvm/IR/DataLayout.h"
00078 #include "llvm/IR/DebugInfo.h"
00079 #include "llvm/IR/DerivedTypes.h"
00080 #include "llvm/IR/DiagnosticInfo.h"
00081 #include "llvm/IR/Dominators.h"
00082 #include "llvm/IR/Function.h"
00083 #include "llvm/IR/IRBuilder.h"
00084 #include "llvm/IR/Instructions.h"
00085 #include "llvm/IR/IntrinsicInst.h"
00086 #include "llvm/IR/LLVMContext.h"
00087 #include "llvm/IR/Module.h"
00088 #include "llvm/IR/PatternMatch.h"
00089 #include "llvm/IR/Type.h"
00090 #include "llvm/IR/Value.h"
00091 #include "llvm/IR/ValueHandle.h"
00092 #include "llvm/IR/Verifier.h"
00093 #include "llvm/Pass.h"
00094 #include "llvm/Support/BranchProbability.h"
00095 #include "llvm/Support/CommandLine.h"
00096 #include "llvm/Support/Debug.h"
00097 #include "llvm/Support/raw_ostream.h"
00098 #include "llvm/Transforms/Scalar.h"
00099 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
00100 #include "llvm/Transforms/Utils/Local.h"
00101 #include "llvm/Analysis/VectorUtils.h"
00102 #include "llvm/Transforms/Utils/LoopUtils.h"
00103 #include <algorithm>
00104 #include <functional>
00105 #include <map>
00106 #include <tuple>
00107 
00108 using namespace llvm;
00109 using namespace llvm::PatternMatch;
00110 
00111 #define LV_NAME "loop-vectorize"
00112 #define DEBUG_TYPE LV_NAME
00113 
00114 STATISTIC(LoopsVectorized, "Number of loops vectorized");
00115 STATISTIC(LoopsAnalyzed, "Number of loops analyzed for vectorization");
00116 
00117 static cl::opt<bool>
00118 EnableIfConversion("enable-if-conversion", cl::init(true), cl::Hidden,
00119                    cl::desc("Enable if-conversion during vectorization."));
00120 
00121 /// We don't vectorize loops with a known constant trip count below this number.
00122 static cl::opt<unsigned>
00123 TinyTripCountVectorThreshold("vectorizer-min-trip-count", cl::init(16),
00124                              cl::Hidden,
00125                              cl::desc("Don't vectorize loops with a constant "
00126                                       "trip count that is smaller than this "
00127                                       "value."));
00128 
00129 static cl::opt<bool> MaximizeBandwidth(
00130     "vectorizer-maximize-bandwidth", cl::init(false), cl::Hidden,
00131     cl::desc("Maximize bandwidth when selecting vectorization factor which "
00132              "will be determined by the smallest type in loop."));
00133 
00134 /// This enables versioning on the strides of symbolically striding memory
00135 /// accesses in code like the following.
00136 ///   for (i = 0; i < N; ++i)
00137 ///     A[i * Stride1] += B[i * Stride2] ...
00138 ///
00139 /// Will be roughly translated to
00140 ///    if (Stride1 == 1 && Stride2 == 1) {
00141 ///      for (i = 0; i < N; i+=4)
00142 ///       A[i:i+3] += ...
00143 ///    } else
00144 ///      ...
00145 static cl::opt<bool> EnableMemAccessVersioning(
00146     "enable-mem-access-versioning", cl::init(true), cl::Hidden,
00147     cl::desc("Enable symbolic stride memory access versioning"));
00148 
00149 static cl::opt<bool> EnableInterleavedMemAccesses(
00150     "enable-interleaved-mem-accesses", cl::init(false), cl::Hidden,
00151     cl::desc("Enable vectorization on interleaved memory accesses in a loop"));
00152 
00153 /// Maximum factor for an interleaved memory access.
00154 static cl::opt<unsigned> MaxInterleaveGroupFactor(
00155     "max-interleave-group-factor", cl::Hidden,
00156     cl::desc("Maximum factor for an interleaved access group (default = 8)"),
00157     cl::init(8));
00158 
00159 /// We don't interleave loops with a known constant trip count below this
00160 /// number.
00161 static const unsigned TinyTripCountInterleaveThreshold = 128;
00162 
00163 static cl::opt<unsigned> ForceTargetNumScalarRegs(
00164     "force-target-num-scalar-regs", cl::init(0), cl::Hidden,
00165     cl::desc("A flag that overrides the target's number of scalar registers."));
00166 
00167 static cl::opt<unsigned> ForceTargetNumVectorRegs(
00168     "force-target-num-vector-regs", cl::init(0), cl::Hidden,
00169     cl::desc("A flag that overrides the target's number of vector registers."));
00170 
00171 /// Maximum vectorization interleave count.
00172 static const unsigned MaxInterleaveFactor = 16;
00173 
00174 static cl::opt<unsigned> ForceTargetMaxScalarInterleaveFactor(
00175     "force-target-max-scalar-interleave", cl::init(0), cl::Hidden,
00176     cl::desc("A flag that overrides the target's max interleave factor for "
00177              "scalar loops."));
00178 
00179 static cl::opt<unsigned> ForceTargetMaxVectorInterleaveFactor(
00180     "force-target-max-vector-interleave", cl::init(0), cl::Hidden,
00181     cl::desc("A flag that overrides the target's max interleave factor for "
00182              "vectorized loops."));
00183 
00184 static cl::opt<unsigned> ForceTargetInstructionCost(
00185     "force-target-instruction-cost", cl::init(0), cl::Hidden,
00186     cl::desc("A flag that overrides the target's expected cost for "
00187              "an instruction to a single constant value. Mostly "
00188              "useful for getting consistent testing."));
00189 
00190 static cl::opt<unsigned> SmallLoopCost(
00191     "small-loop-cost", cl::init(20), cl::Hidden,
00192     cl::desc(
00193         "The cost of a loop that is considered 'small' by the interleaver."));
00194 
00195 static cl::opt<bool> LoopVectorizeWithBlockFrequency(
00196     "loop-vectorize-with-block-frequency", cl::init(false), cl::Hidden,
00197     cl::desc("Enable the use of the block frequency analysis to access PGO "
00198              "heuristics minimizing code growth in cold regions and being more "
00199              "aggressive in hot regions."));
00200 
00201 // Runtime interleave loops for load/store throughput.
00202 static cl::opt<bool> EnableLoadStoreRuntimeInterleave(
00203     "enable-loadstore-runtime-interleave", cl::init(true), cl::Hidden,
00204     cl::desc(
00205         "Enable runtime interleaving until load/store ports are saturated"));
00206 
00207 /// The number of stores in a loop that are allowed to need predication.
00208 static cl::opt<unsigned> NumberOfStoresToPredicate(
00209     "vectorize-num-stores-pred", cl::init(1), cl::Hidden,
00210     cl::desc("Max number of stores to be predicated behind an if."));
00211 
00212 static cl::opt<bool> EnableIndVarRegisterHeur(
00213     "enable-ind-var-reg-heur", cl::init(true), cl::Hidden,
00214     cl::desc("Count the induction variable only once when interleaving"));
00215 
00216 static cl::opt<bool> EnableCondStoresVectorization(
00217     "enable-cond-stores-vec", cl::init(false), cl::Hidden,
00218     cl::desc("Enable if predication of stores during vectorization."));
00219 
00220 static cl::opt<unsigned> MaxNestedScalarReductionIC(
00221     "max-nested-scalar-reduction-interleave", cl::init(2), cl::Hidden,
00222     cl::desc("The maximum interleave count to use when interleaving a scalar "
00223              "reduction in a nested loop."));
00224 
00225 static cl::opt<unsigned> PragmaVectorizeMemoryCheckThreshold(
00226     "pragma-vectorize-memory-check-threshold", cl::init(128), cl::Hidden,
00227     cl::desc("The maximum allowed number of runtime memory checks with a "
00228              "vectorize(enable) pragma."));
00229 
00230 static cl::opt<unsigned> VectorizeSCEVCheckThreshold(
00231     "vectorize-scev-check-threshold", cl::init(16), cl::Hidden,
00232     cl::desc("The maximum number of SCEV checks allowed."));
00233 
00234 static cl::opt<unsigned> PragmaVectorizeSCEVCheckThreshold(
00235     "pragma-vectorize-scev-check-threshold", cl::init(128), cl::Hidden,
00236     cl::desc("The maximum number of SCEV checks allowed with a "
00237              "vectorize(enable) pragma"));
00238 
00239 namespace {
00240 
00241 // Forward declarations.
00242 class LoopVectorizeHints;
00243 class LoopVectorizationLegality;
00244 class LoopVectorizationCostModel;
00245 class LoopVectorizationRequirements;
00246 
00247 /// \brief This modifies LoopAccessReport to initialize message with
00248 /// loop-vectorizer-specific part.
00249 class VectorizationReport : public LoopAccessReport {
00250 public:
00251   VectorizationReport(Instruction *I = nullptr)
00252       : LoopAccessReport("loop not vectorized: ", I) {}
00253 
00254   /// \brief This allows promotion of the loop-access analysis report into the
00255   /// loop-vectorizer report.  It modifies the message to add the
00256   /// loop-vectorizer-specific part of the message.
00257   explicit VectorizationReport(const LoopAccessReport &R)
00258       : LoopAccessReport(Twine("loop not vectorized: ") + R.str(),
00259                          R.getInstr()) {}
00260 };
00261 
00262 /// A helper function for converting Scalar types to vector types.
00263 /// If the incoming type is void, we return void. If the VF is 1, we return
00264 /// the scalar type.
00265 static Type* ToVectorTy(Type *Scalar, unsigned VF) {
00266   if (Scalar->isVoidTy() || VF == 1)
00267     return Scalar;
00268   return VectorType::get(Scalar, VF);
00269 }
00270 
00271 /// A helper function that returns GEP instruction and knows to skip a
00272 /// 'bitcast'. The 'bitcast' may be skipped if the source and the destination
00273 /// pointee types of the 'bitcast' have the same size.
00274 /// For example:
00275 ///   bitcast double** %var to i64* - can be skipped
00276 ///   bitcast double** %var to i8*  - can not
00277 static GetElementPtrInst *getGEPInstruction(Value *Ptr) {
00278 
00279   if (isa<GetElementPtrInst>(Ptr))
00280     return cast<GetElementPtrInst>(Ptr);
00281 
00282   if (isa<BitCastInst>(Ptr) &&
00283       isa<GetElementPtrInst>(cast<BitCastInst>(Ptr)->getOperand(0))) {
00284     Type *BitcastTy = Ptr->getType();
00285     Type *GEPTy = cast<BitCastInst>(Ptr)->getSrcTy();
00286     if (!isa<PointerType>(BitcastTy) || !isa<PointerType>(GEPTy))
00287       return nullptr;
00288     Type *Pointee1Ty = cast<PointerType>(BitcastTy)->getPointerElementType();
00289     Type *Pointee2Ty = cast<PointerType>(GEPTy)->getPointerElementType();
00290     const DataLayout &DL = cast<BitCastInst>(Ptr)->getModule()->getDataLayout();
00291     if (DL.getTypeSizeInBits(Pointee1Ty) == DL.getTypeSizeInBits(Pointee2Ty))
00292       return cast<GetElementPtrInst>(cast<BitCastInst>(Ptr)->getOperand(0));
00293   }
00294   return nullptr;
00295 }
00296 
00297 /// InnerLoopVectorizer vectorizes loops which contain only one basic
00298 /// block to a specified vectorization factor (VF).
00299 /// This class performs the widening of scalars into vectors, or multiple
00300 /// scalars. This class also implements the following features:
00301 /// * It inserts an epilogue loop for handling loops that don't have iteration
00302 ///   counts that are known to be a multiple of the vectorization factor.
00303 /// * It handles the code generation for reduction variables.
00304 /// * Scalarization (implementation using scalars) of un-vectorizable
00305 ///   instructions.
00306 /// InnerLoopVectorizer does not perform any vectorization-legality
00307 /// checks, and relies on the caller to check for the different legality
00308 /// aspects. The InnerLoopVectorizer relies on the
00309 /// LoopVectorizationLegality class to provide information about the induction
00310 /// and reduction variables that were found to a given vectorization factor.
00311 class InnerLoopVectorizer {
00312 public:
00313   InnerLoopVectorizer(Loop *OrigLoop, PredicatedScalarEvolution &PSE,
00314                       LoopInfo *LI, DominatorTree *DT,
00315                       const TargetLibraryInfo *TLI,
00316                       const TargetTransformInfo *TTI, unsigned VecWidth,
00317                       unsigned UnrollFactor)
00318       : OrigLoop(OrigLoop), PSE(PSE), LI(LI), DT(DT), TLI(TLI), TTI(TTI),
00319         VF(VecWidth), UF(UnrollFactor), Builder(PSE.getSE()->getContext()),
00320         Induction(nullptr), OldInduction(nullptr), WidenMap(UnrollFactor),
00321         TripCount(nullptr), VectorTripCount(nullptr), Legal(nullptr),
00322         AddedSafetyChecks(false) {}
00323 
00324   // Perform the actual loop widening (vectorization).
00325   // MinimumBitWidths maps scalar integer values to the smallest bitwidth they
00326   // can be validly truncated to. The cost model has assumed this truncation
00327   // will happen when vectorizing.
00328   void vectorize(LoopVectorizationLegality *L,
00329                  MapVector<Instruction*,uint64_t> MinimumBitWidths) {
00330     MinBWs = MinimumBitWidths;
00331     Legal = L;
00332     // Create a new empty loop. Unlink the old loop and connect the new one.
00333     createEmptyLoop();
00334     // Widen each instruction in the old loop to a new one in the new loop.
00335     // Use the Legality module to find the induction and reduction variables.
00336     vectorizeLoop();
00337   }
00338 
00339   // Return true if any runtime check is added.
00340   bool IsSafetyChecksAdded() {
00341     return AddedSafetyChecks;
00342   }
00343 
00344   virtual ~InnerLoopVectorizer() {}
00345 
00346 protected:
00347   /// A small list of PHINodes.
00348   typedef SmallVector<PHINode*, 4> PhiVector;
00349   /// When we unroll loops we have multiple vector values for each scalar.
00350   /// This data structure holds the unrolled and vectorized values that
00351   /// originated from one scalar instruction.
00352   typedef SmallVector<Value*, 2> VectorParts;
00353 
00354   // When we if-convert we need to create edge masks. We have to cache values
00355   // so that we don't end up with exponential recursion/IR.
00356   typedef DenseMap<std::pair<BasicBlock*, BasicBlock*>,
00357                    VectorParts> EdgeMaskCache;
00358 
00359   /// Create an empty loop, based on the loop ranges of the old loop.
00360   void createEmptyLoop();
00361   /// Create a new induction variable inside L.
00362   PHINode *createInductionVariable(Loop *L, Value *Start, Value *End,
00363                                    Value *Step, Instruction *DL);
00364   /// Copy and widen the instructions from the old loop.
00365   virtual void vectorizeLoop();
00366 
00367   /// \brief The Loop exit block may have single value PHI nodes where the
00368   /// incoming value is 'Undef'. While vectorizing we only handled real values
00369   /// that were defined inside the loop. Here we fix the 'undef case'.
00370   /// See PR14725.
00371   void fixLCSSAPHIs();
00372 
00373   /// Shrinks vector element sizes based on information in "MinBWs".
00374   void truncateToMinimalBitwidths();
00375   
00376   /// A helper function that computes the predicate of the block BB, assuming
00377   /// that the header block of the loop is set to True. It returns the *entry*
00378   /// mask for the block BB.
00379   VectorParts createBlockInMask(BasicBlock *BB);
00380   /// A helper function that computes the predicate of the edge between SRC
00381   /// and DST.
00382   VectorParts createEdgeMask(BasicBlock *Src, BasicBlock *Dst);
00383 
00384   /// A helper function to vectorize a single BB within the innermost loop.
00385   void vectorizeBlockInLoop(BasicBlock *BB, PhiVector *PV);
00386   
00387   /// Vectorize a single PHINode in a block. This method handles the induction
00388   /// variable canonicalization. It supports both VF = 1 for unrolled loops and
00389   /// arbitrary length vectors.
00390   void widenPHIInstruction(Instruction *PN, VectorParts &Entry,
00391                            unsigned UF, unsigned VF, PhiVector *PV);
00392 
00393   /// Insert the new loop to the loop hierarchy and pass manager
00394   /// and update the analysis passes.
00395   void updateAnalysis();
00396 
00397   /// This instruction is un-vectorizable. Implement it as a sequence
00398   /// of scalars. If \p IfPredicateStore is true we need to 'hide' each
00399   /// scalarized instruction behind an if block predicated on the control
00400   /// dependence of the instruction.
00401   virtual void scalarizeInstruction(Instruction *Instr,
00402                                     bool IfPredicateStore=false);
00403 
00404   /// Vectorize Load and Store instructions,
00405   virtual void vectorizeMemoryInstruction(Instruction *Instr);
00406 
00407   /// Create a broadcast instruction. This method generates a broadcast
00408   /// instruction (shuffle) for loop invariant values and for the induction
00409   /// value. If this is the induction variable then we extend it to N, N+1, ...
00410   /// this is needed because each iteration in the loop corresponds to a SIMD
00411   /// element.
00412   virtual Value *getBroadcastInstrs(Value *V);
00413 
00414   /// This function adds (StartIdx, StartIdx + Step, StartIdx + 2*Step, ...)
00415   /// to each vector element of Val. The sequence starts at StartIndex.
00416   virtual Value *getStepVector(Value *Val, int StartIdx, Value *Step);
00417 
00418   /// When we go over instructions in the basic block we rely on previous
00419   /// values within the current basic block or on loop invariant values.
00420   /// When we widen (vectorize) values we place them in the map. If the values
00421   /// are not within the map, they have to be loop invariant, so we simply
00422   /// broadcast them into a vector.
00423   VectorParts &getVectorValue(Value *V);
00424 
00425   /// Try to vectorize the interleaved access group that \p Instr belongs to.
00426   void vectorizeInterleaveGroup(Instruction *Instr);
00427 
00428   /// Generate a shuffle sequence that will reverse the vector Vec.
00429   virtual Value *reverseVector(Value *Vec);
00430 
00431   /// Returns (and creates if needed) the original loop trip count.
00432   Value *getOrCreateTripCount(Loop *NewLoop);
00433 
00434   /// Returns (and creates if needed) the trip count of the widened loop.
00435   Value *getOrCreateVectorTripCount(Loop *NewLoop);
00436 
00437   /// Emit a bypass check to see if the trip count would overflow, or we
00438   /// wouldn't have enough iterations to execute one vector loop.
00439   void emitMinimumIterationCountCheck(Loop *L, BasicBlock *Bypass);
00440   /// Emit a bypass check to see if the vector trip count is nonzero.
00441   void emitVectorLoopEnteredCheck(Loop *L, BasicBlock *Bypass);
00442   /// Emit a bypass check to see if all of the SCEV assumptions we've
00443   /// had to make are correct.
00444   void emitSCEVChecks(Loop *L, BasicBlock *Bypass);
00445   /// Emit bypass checks to check any memory assumptions we may have made.
00446   void emitMemRuntimeChecks(Loop *L, BasicBlock *Bypass);
00447 
00448   /// This is a helper class that holds the vectorizer state. It maps scalar
00449   /// instructions to vector instructions. When the code is 'unrolled' then
00450   /// then a single scalar value is mapped to multiple vector parts. The parts
00451   /// are stored in the VectorPart type.
00452   struct ValueMap {
00453     /// C'tor.  UnrollFactor controls the number of vectors ('parts') that
00454     /// are mapped.
00455     ValueMap(unsigned UnrollFactor) : UF(UnrollFactor) {}
00456 
00457     /// \return True if 'Key' is saved in the Value Map.
00458     bool has(Value *Key) const { return MapStorage.count(Key); }
00459 
00460     /// Initializes a new entry in the map. Sets all of the vector parts to the
00461     /// save value in 'Val'.
00462     /// \return A reference to a vector with splat values.
00463     VectorParts &splat(Value *Key, Value *Val) {
00464       VectorParts &Entry = MapStorage[Key];
00465       Entry.assign(UF, Val);
00466       return Entry;
00467     }
00468 
00469     ///\return A reference to the value that is stored at 'Key'.
00470     VectorParts &get(Value *Key) {
00471       VectorParts &Entry = MapStorage[Key];
00472       if (Entry.empty())
00473         Entry.resize(UF);
00474       assert(Entry.size() == UF);
00475       return Entry;
00476     }
00477 
00478   private:
00479     /// The unroll factor. Each entry in the map stores this number of vector
00480     /// elements.
00481     unsigned UF;
00482 
00483     /// Map storage. We use std::map and not DenseMap because insertions to a
00484     /// dense map invalidates its iterators.
00485     std::map<Value *, VectorParts> MapStorage;
00486   };
00487 
00488   /// The original loop.
00489   Loop *OrigLoop;
00490   /// A wrapper around ScalarEvolution used to add runtime SCEV checks. Applies
00491   /// dynamic knowledge to simplify SCEV expressions and converts them to a
00492   /// more usable form.
00493   PredicatedScalarEvolution &PSE;
00494   /// Loop Info.
00495   LoopInfo *LI;
00496   /// Dominator Tree.
00497   DominatorTree *DT;
00498   /// Alias Analysis.
00499   AliasAnalysis *AA;
00500   /// Target Library Info.
00501   const TargetLibraryInfo *TLI;
00502   /// Target Transform Info.
00503   const TargetTransformInfo *TTI;
00504 
00505   /// The vectorization SIMD factor to use. Each vector will have this many
00506   /// vector elements.
00507   unsigned VF;
00508 
00509 protected:
00510   /// The vectorization unroll factor to use. Each scalar is vectorized to this
00511   /// many different vector instructions.
00512   unsigned UF;
00513 
00514   /// The builder that we use
00515   IRBuilder<> Builder;
00516 
00517   // --- Vectorization state ---
00518 
00519   /// The vector-loop preheader.
00520   BasicBlock *LoopVectorPreHeader;
00521   /// The scalar-loop preheader.
00522   BasicBlock *LoopScalarPreHeader;
00523   /// Middle Block between the vector and the scalar.
00524   BasicBlock *LoopMiddleBlock;
00525   ///The ExitBlock of the scalar loop.
00526   BasicBlock *LoopExitBlock;
00527   ///The vector loop body.
00528   SmallVector<BasicBlock *, 4> LoopVectorBody;
00529   ///The scalar loop body.
00530   BasicBlock *LoopScalarBody;
00531   /// A list of all bypass blocks. The first block is the entry of the loop.
00532   SmallVector<BasicBlock *, 4> LoopBypassBlocks;
00533 
00534   /// The new Induction variable which was added to the new block.
00535   PHINode *Induction;
00536   /// The induction variable of the old basic block.
00537   PHINode *OldInduction;
00538   /// Maps scalars to widened vectors.
00539   ValueMap WidenMap;
00540   /// Store instructions that should be predicated, as a pair
00541   ///   <StoreInst, Predicate>
00542   SmallVector<std::pair<StoreInst*,Value*>, 4> PredicatedStores;
00543   EdgeMaskCache MaskCache;
00544   /// Trip count of the original loop.
00545   Value *TripCount;
00546   /// Trip count of the widened loop (TripCount - TripCount % (VF*UF))
00547   Value *VectorTripCount;
00548 
00549   /// Map of scalar integer values to the smallest bitwidth they can be legally
00550   /// represented as. The vector equivalents of these values should be truncated
00551   /// to this type.
00552   MapVector<Instruction*,uint64_t> MinBWs;
00553   LoopVectorizationLegality *Legal;
00554 
00555   // Record whether runtime check is added.
00556   bool AddedSafetyChecks;
00557 };
00558 
00559 class InnerLoopUnroller : public InnerLoopVectorizer {
00560 public:
00561   InnerLoopUnroller(Loop *OrigLoop, PredicatedScalarEvolution &PSE,
00562                     LoopInfo *LI, DominatorTree *DT,
00563                     const TargetLibraryInfo *TLI,
00564                     const TargetTransformInfo *TTI, unsigned UnrollFactor)
00565       : InnerLoopVectorizer(OrigLoop, PSE, LI, DT, TLI, TTI, 1, UnrollFactor) {}
00566 
00567 private:
00568   void scalarizeInstruction(Instruction *Instr,
00569                             bool IfPredicateStore = false) override;
00570   void vectorizeMemoryInstruction(Instruction *Instr) override;
00571   Value *getBroadcastInstrs(Value *V) override;
00572   Value *getStepVector(Value *Val, int StartIdx, Value *Step) override;
00573   Value *reverseVector(Value *Vec) override;
00574 };
00575 
00576 /// \brief Look for a meaningful debug location on the instruction or it's
00577 /// operands.
00578 static Instruction *getDebugLocFromInstOrOperands(Instruction *I) {
00579   if (!I)
00580     return I;
00581 
00582   DebugLoc Empty;
00583   if (I->getDebugLoc() != Empty)
00584     return I;
00585 
00586   for (User::op_iterator OI = I->op_begin(), OE = I->op_end(); OI != OE; ++OI) {
00587     if (Instruction *OpInst = dyn_cast<Instruction>(*OI))
00588       if (OpInst->getDebugLoc() != Empty)
00589         return OpInst;
00590   }
00591 
00592   return I;
00593 }
00594 
00595 /// \brief Set the debug location in the builder using the debug location in the
00596 /// instruction.
00597 static void setDebugLocFromInst(IRBuilder<> &B, const Value *Ptr) {
00598   if (const Instruction *Inst = dyn_cast_or_null<Instruction>(Ptr))
00599     B.SetCurrentDebugLocation(Inst->getDebugLoc());
00600   else
00601     B.SetCurrentDebugLocation(DebugLoc());
00602 }
00603 
00604 #ifndef NDEBUG
00605 /// \return string containing a file name and a line # for the given loop.
00606 static std::string getDebugLocString(const Loop *L) {
00607   std::string Result;
00608   if (L) {
00609     raw_string_ostream OS(Result);
00610     if (const DebugLoc LoopDbgLoc = L->getStartLoc())
00611       LoopDbgLoc.print(OS);
00612     else
00613       // Just print the module name.
00614       OS << L->getHeader()->getParent()->getParent()->getModuleIdentifier();
00615     OS.flush();
00616   }
00617   return Result;
00618 }
00619 #endif
00620 
00621 /// \brief Propagate known metadata from one instruction to another.
00622 static void propagateMetadata(Instruction *To, const Instruction *From) {
00623   SmallVector<std::pair<unsigned, MDNode *>, 4> Metadata;
00624   From->getAllMetadataOtherThanDebugLoc(Metadata);
00625 
00626   for (auto M : Metadata) {
00627     unsigned Kind = M.first;
00628 
00629     // These are safe to transfer (this is safe for TBAA, even when we
00630     // if-convert, because should that metadata have had a control dependency
00631     // on the condition, and thus actually aliased with some other
00632     // non-speculated memory access when the condition was false, this would be
00633     // caught by the runtime overlap checks).
00634     if (Kind != LLVMContext::MD_tbaa &&
00635         Kind != LLVMContext::MD_alias_scope &&
00636         Kind != LLVMContext::MD_noalias &&
00637         Kind != LLVMContext::MD_fpmath &&
00638         Kind != LLVMContext::MD_nontemporal)
00639       continue;
00640 
00641     To->setMetadata(Kind, M.second);
00642   }
00643 }
00644 
00645 /// \brief Propagate known metadata from one instruction to a vector of others.
00646 static void propagateMetadata(SmallVectorImpl<Value *> &To,
00647                               const Instruction *From) {
00648   for (Value *V : To)
00649     if (Instruction *I = dyn_cast<Instruction>(V))
00650       propagateMetadata(I, From);
00651 }
00652 
00653 /// \brief The group of interleaved loads/stores sharing the same stride and
00654 /// close to each other.
00655 ///
00656 /// Each member in this group has an index starting from 0, and the largest
00657 /// index should be less than interleaved factor, which is equal to the absolute
00658 /// value of the access's stride.
00659 ///
00660 /// E.g. An interleaved load group of factor 4:
00661 ///        for (unsigned i = 0; i < 1024; i+=4) {
00662 ///          a = A[i];                           // Member of index 0
00663 ///          b = A[i+1];                         // Member of index 1
00664 ///          d = A[i+3];                         // Member of index 3
00665 ///          ...
00666 ///        }
00667 ///
00668 ///      An interleaved store group of factor 4:
00669 ///        for (unsigned i = 0; i < 1024; i+=4) {
00670 ///          ...
00671 ///          A[i]   = a;                         // Member of index 0
00672 ///          A[i+1] = b;                         // Member of index 1
00673 ///          A[i+2] = c;                         // Member of index 2
00674 ///          A[i+3] = d;                         // Member of index 3
00675 ///        }
00676 ///
00677 /// Note: the interleaved load group could have gaps (missing members), but
00678 /// the interleaved store group doesn't allow gaps.
00679 class InterleaveGroup {
00680 public:
00681   InterleaveGroup(Instruction *Instr, int Stride, unsigned Align)
00682       : Align(Align), SmallestKey(0), LargestKey(0), InsertPos(Instr) {
00683     assert(Align && "The alignment should be non-zero");
00684 
00685     Factor = std::abs(Stride);
00686     assert(Factor > 1 && "Invalid interleave factor");
00687 
00688     Reverse = Stride < 0;
00689     Members[0] = Instr;
00690   }
00691 
00692   bool isReverse() const { return Reverse; }
00693   unsigned getFactor() const { return Factor; }
00694   unsigned getAlignment() const { return Align; }
00695   unsigned getNumMembers() const { return Members.size(); }
00696 
00697   /// \brief Try to insert a new member \p Instr with index \p Index and
00698   /// alignment \p NewAlign. The index is related to the leader and it could be
00699   /// negative if it is the new leader.
00700   ///
00701   /// \returns false if the instruction doesn't belong to the group.
00702   bool insertMember(Instruction *Instr, int Index, unsigned NewAlign) {
00703     assert(NewAlign && "The new member's alignment should be non-zero");
00704 
00705     int Key = Index + SmallestKey;
00706 
00707     // Skip if there is already a member with the same index.
00708     if (Members.count(Key))
00709       return false;
00710 
00711     if (Key > LargestKey) {
00712       // The largest index is always less than the interleave factor.
00713       if (Index >= static_cast<int>(Factor))
00714         return false;
00715 
00716       LargestKey = Key;
00717     } else if (Key < SmallestKey) {
00718       // The largest index is always less than the interleave factor.
00719       if (LargestKey - Key >= static_cast<int>(Factor))
00720         return false;
00721 
00722       SmallestKey = Key;
00723     }
00724 
00725     // It's always safe to select the minimum alignment.
00726     Align = std::min(Align, NewAlign);
00727     Members[Key] = Instr;
00728     return true;
00729   }
00730 
00731   /// \brief Get the member with the given index \p Index
00732   ///
00733   /// \returns nullptr if contains no such member.
00734   Instruction *getMember(unsigned Index) const {
00735     int Key = SmallestKey + Index;
00736     if (!Members.count(Key))
00737       return nullptr;
00738 
00739     return Members.find(Key)->second;
00740   }
00741 
00742   /// \brief Get the index for the given member. Unlike the key in the member
00743   /// map, the index starts from 0.
00744   unsigned getIndex(Instruction *Instr) const {
00745     for (auto I : Members)
00746       if (I.second == Instr)
00747         return I.first - SmallestKey;
00748 
00749     llvm_unreachable("InterleaveGroup contains no such member");
00750   }
00751 
00752   Instruction *getInsertPos() const { return InsertPos; }
00753   void setInsertPos(Instruction *Inst) { InsertPos = Inst; }
00754 
00755 private:
00756   unsigned Factor; // Interleave Factor.
00757   bool Reverse;
00758   unsigned Align;
00759   DenseMap<int, Instruction *> Members;
00760   int SmallestKey;
00761   int LargestKey;
00762 
00763   // To avoid breaking dependences, vectorized instructions of an interleave
00764   // group should be inserted at either the first load or the last store in
00765   // program order.
00766   //
00767   // E.g. %even = load i32             // Insert Position
00768   //      %add = add i32 %even         // Use of %even
00769   //      %odd = load i32
00770   //
00771   //      store i32 %even
00772   //      %odd = add i32               // Def of %odd
00773   //      store i32 %odd               // Insert Position
00774   Instruction *InsertPos;
00775 };
00776 
00777 /// \brief Drive the analysis of interleaved memory accesses in the loop.
00778 ///
00779 /// Use this class to analyze interleaved accesses only when we can vectorize
00780 /// a loop. Otherwise it's meaningless to do analysis as the vectorization
00781 /// on interleaved accesses is unsafe.
00782 ///
00783 /// The analysis collects interleave groups and records the relationships
00784 /// between the member and the group in a map.
00785 class InterleavedAccessInfo {
00786 public:
00787   InterleavedAccessInfo(PredicatedScalarEvolution &PSE, Loop *L,
00788                         DominatorTree *DT)
00789       : PSE(PSE), TheLoop(L), DT(DT) {}
00790 
00791   ~InterleavedAccessInfo() {
00792     SmallSet<InterleaveGroup *, 4> DelSet;
00793     // Avoid releasing a pointer twice.
00794     for (auto &I : InterleaveGroupMap)
00795       DelSet.insert(I.second);
00796     for (auto *Ptr : DelSet)
00797       delete Ptr;
00798   }
00799 
00800   /// \brief Analyze the interleaved accesses and collect them in interleave
00801   /// groups. Substitute symbolic strides using \p Strides.
00802   void analyzeInterleaving(const ValueToValueMap &Strides);
00803 
00804   /// \brief Check if \p Instr belongs to any interleave group.
00805   bool isInterleaved(Instruction *Instr) const {
00806     return InterleaveGroupMap.count(Instr);
00807   }
00808 
00809   /// \brief Get the interleave group that \p Instr belongs to.
00810   ///
00811   /// \returns nullptr if doesn't have such group.
00812   InterleaveGroup *getInterleaveGroup(Instruction *Instr) const {
00813     if (InterleaveGroupMap.count(Instr))
00814       return InterleaveGroupMap.find(Instr)->second;
00815     return nullptr;
00816   }
00817 
00818 private:
00819   /// A wrapper around ScalarEvolution, used to add runtime SCEV checks.
00820   /// Simplifies SCEV expressions in the context of existing SCEV assumptions.
00821   /// The interleaved access analysis can also add new predicates (for example
00822   /// by versioning strides of pointers).
00823   PredicatedScalarEvolution &PSE;
00824   Loop *TheLoop;
00825   DominatorTree *DT;
00826 
00827   /// Holds the relationships between the members and the interleave group.
00828   DenseMap<Instruction *, InterleaveGroup *> InterleaveGroupMap;
00829 
00830   /// \brief The descriptor for a strided memory access.
00831   struct StrideDescriptor {
00832     StrideDescriptor(int Stride, const SCEV *Scev, unsigned Size,
00833                      unsigned Align)
00834         : Stride(Stride), Scev(Scev), Size(Size), Align(Align) {}
00835 
00836     StrideDescriptor() : Stride(0), Scev(nullptr), Size(0), Align(0) {}
00837 
00838     int Stride; // The access's stride. It is negative for a reverse access.
00839     const SCEV *Scev; // The scalar expression of this access
00840     unsigned Size;    // The size of the memory object.
00841     unsigned Align;   // The alignment of this access.
00842   };
00843 
00844   /// \brief Create a new interleave group with the given instruction \p Instr,
00845   /// stride \p Stride and alignment \p Align.
00846   ///
00847   /// \returns the newly created interleave group.
00848   InterleaveGroup *createInterleaveGroup(Instruction *Instr, int Stride,
00849                                          unsigned Align) {
00850     assert(!InterleaveGroupMap.count(Instr) &&
00851            "Already in an interleaved access group");
00852     InterleaveGroupMap[Instr] = new InterleaveGroup(Instr, Stride, Align);
00853     return InterleaveGroupMap[Instr];
00854   }
00855 
00856   /// \brief Release the group and remove all the relationships.
00857   void releaseGroup(InterleaveGroup *Group) {
00858     for (unsigned i = 0; i < Group->getFactor(); i++)
00859       if (Instruction *Member = Group->getMember(i))
00860         InterleaveGroupMap.erase(Member);
00861 
00862     delete Group;
00863   }
00864 
00865   /// \brief Collect all the accesses with a constant stride in program order.
00866   void collectConstStridedAccesses(
00867       MapVector<Instruction *, StrideDescriptor> &StrideAccesses,
00868       const ValueToValueMap &Strides);
00869 };
00870 
00871 /// Utility class for getting and setting loop vectorizer hints in the form
00872 /// of loop metadata.
00873 /// This class keeps a number of loop annotations locally (as member variables)
00874 /// and can, upon request, write them back as metadata on the loop. It will
00875 /// initially scan the loop for existing metadata, and will update the local
00876 /// values based on information in the loop.
00877 /// We cannot write all values to metadata, as the mere presence of some info,
00878 /// for example 'force', means a decision has been made. So, we need to be
00879 /// careful NOT to add them if the user hasn't specifically asked so.
00880 class LoopVectorizeHints {
00881   enum HintKind {
00882     HK_WIDTH,
00883     HK_UNROLL,
00884     HK_FORCE
00885   };
00886 
00887   /// Hint - associates name and validation with the hint value.
00888   struct Hint {
00889     const char * Name;
00890     unsigned Value; // This may have to change for non-numeric values.
00891     HintKind Kind;
00892 
00893     Hint(const char * Name, unsigned Value, HintKind Kind)
00894       : Name(Name), Value(Value), Kind(Kind) { }
00895 
00896     bool validate(unsigned Val) {
00897       switch (Kind) {
00898       case HK_WIDTH:
00899         return isPowerOf2_32(Val) && Val <= VectorizerParams::MaxVectorWidth;
00900       case HK_UNROLL:
00901         return isPowerOf2_32(Val) && Val <= MaxInterleaveFactor;
00902       case HK_FORCE:
00903         return (Val <= 1);
00904       }
00905       return false;
00906     }
00907   };
00908 
00909   /// Vectorization width.
00910   Hint Width;
00911   /// Vectorization interleave factor.
00912   Hint Interleave;
00913   /// Vectorization forced
00914   Hint Force;
00915 
00916   /// Return the loop metadata prefix.
00917   static StringRef Prefix() { return "llvm.loop."; }
00918 
00919 public:
00920   enum ForceKind {
00921     FK_Undefined = -1, ///< Not selected.
00922     FK_Disabled = 0,   ///< Forcing disabled.
00923     FK_Enabled = 1,    ///< Forcing enabled.
00924   };
00925 
00926   LoopVectorizeHints(const Loop *L, bool DisableInterleaving)
00927       : Width("vectorize.width", VectorizerParams::VectorizationFactor,
00928               HK_WIDTH),
00929         Interleave("interleave.count", DisableInterleaving, HK_UNROLL),
00930         Force("vectorize.enable", FK_Undefined, HK_FORCE),
00931         TheLoop(L) {
00932     // Populate values with existing loop metadata.
00933     getHintsFromMetadata();
00934 
00935     // force-vector-interleave overrides DisableInterleaving.
00936     if (VectorizerParams::isInterleaveForced())
00937       Interleave.Value = VectorizerParams::VectorizationInterleave;
00938 
00939     DEBUG(if (DisableInterleaving && Interleave.Value == 1) dbgs()
00940           << "LV: Interleaving disabled by the pass manager\n");
00941   }
00942 
00943   /// Mark the loop L as already vectorized by setting the width to 1.
00944   void setAlreadyVectorized() {
00945     Width.Value = Interleave.Value = 1;
00946     Hint Hints[] = {Width, Interleave};
00947     writeHintsToMetadata(Hints);
00948   }
00949 
00950   bool allowVectorization(Function *F, Loop *L, bool AlwaysVectorize) const {
00951     if (getForce() == LoopVectorizeHints::FK_Disabled) {
00952       DEBUG(dbgs() << "LV: Not vectorizing: #pragma vectorize disable.\n");
00953       emitOptimizationRemarkAnalysis(F->getContext(),
00954                                      vectorizeAnalysisPassName(), *F,
00955                                      L->getStartLoc(), emitRemark());
00956       return false;
00957     }
00958 
00959     if (!AlwaysVectorize && getForce() != LoopVectorizeHints::FK_Enabled) {
00960       DEBUG(dbgs() << "LV: Not vectorizing: No #pragma vectorize enable.\n");
00961       emitOptimizationRemarkAnalysis(F->getContext(),
00962                                      vectorizeAnalysisPassName(), *F,
00963                                      L->getStartLoc(), emitRemark());
00964       return false;
00965     }
00966 
00967     if (getWidth() == 1 && getInterleave() == 1) {
00968       // FIXME: Add a separate metadata to indicate when the loop has already
00969       // been vectorized instead of setting width and count to 1.
00970       DEBUG(dbgs() << "LV: Not vectorizing: Disabled/already vectorized.\n");
00971       // FIXME: Add interleave.disable metadata. This will allow
00972       // vectorize.disable to be used without disabling the pass and errors
00973       // to differentiate between disabled vectorization and a width of 1.
00974       emitOptimizationRemarkAnalysis(
00975           F->getContext(), vectorizeAnalysisPassName(), *F, L->getStartLoc(),
00976           "loop not vectorized: vectorization and interleaving are explicitly "
00977           "disabled, or vectorize width and interleave count are both set to "
00978           "1");
00979       return false;
00980     }
00981 
00982     return true;
00983   }
00984 
00985   /// Dumps all the hint information.
00986   std::string emitRemark() const {
00987     VectorizationReport R;
00988     if (Force.Value == LoopVectorizeHints::FK_Disabled)
00989       R << "vectorization is explicitly disabled";
00990     else {
00991       R << "use -Rpass-analysis=loop-vectorize for more info";
00992       if (Force.Value == LoopVectorizeHints::FK_Enabled) {
00993         R << " (Force=true";
00994         if (Width.Value != 0)
00995           R << ", Vector Width=" << Width.Value;
00996         if (Interleave.Value != 0)
00997           R << ", Interleave Count=" << Interleave.Value;
00998         R << ")";
00999       }
01000     }
01001 
01002     return R.str();
01003   }
01004 
01005   unsigned getWidth() const { return Width.Value; }
01006   unsigned getInterleave() const { return Interleave.Value; }
01007   enum ForceKind getForce() const { return (ForceKind)Force.Value; }
01008   const char *vectorizeAnalysisPassName() const {
01009     // If hints are provided that don't disable vectorization use the
01010     // AlwaysPrint pass name to force the frontend to print the diagnostic.
01011     if (getWidth() == 1)
01012       return LV_NAME;
01013     if (getForce() == LoopVectorizeHints::FK_Disabled)
01014       return LV_NAME;
01015     if (getForce() == LoopVectorizeHints::FK_Undefined && getWidth() == 0)
01016       return LV_NAME;
01017     return DiagnosticInfo::AlwaysPrint;
01018   }
01019 
01020   bool allowReordering() const {
01021     // When enabling loop hints are provided we allow the vectorizer to change
01022     // the order of operations that is given by the scalar loop. This is not
01023     // enabled by default because can be unsafe or inefficient. For example,
01024     // reordering floating-point operations will change the way round-off
01025     // error accumulates in the loop.
01026     return getForce() == LoopVectorizeHints::FK_Enabled || getWidth() > 1;
01027   }
01028 
01029 private:
01030   /// Find hints specified in the loop metadata and update local values.
01031   void getHintsFromMetadata() {
01032     MDNode *LoopID = TheLoop->getLoopID();
01033     if (!LoopID)
01034       return;
01035 
01036     // First operand should refer to the loop id itself.
01037     assert(LoopID->getNumOperands() > 0 && "requires at least one operand");
01038     assert(LoopID->getOperand(0) == LoopID && "invalid loop id");
01039 
01040     for (unsigned i = 1, ie = LoopID->getNumOperands(); i < ie; ++i) {
01041       const MDString *S = nullptr;
01042       SmallVector<Metadata *, 4> Args;
01043 
01044       // The expected hint is either a MDString or a MDNode with the first
01045       // operand a MDString.
01046       if (const MDNode *MD = dyn_cast<MDNode>(LoopID->getOperand(i))) {
01047         if (!MD || MD->getNumOperands() == 0)
01048           continue;
01049         S = dyn_cast<MDString>(MD->getOperand(0));
01050         for (unsigned i = 1, ie = MD->getNumOperands(); i < ie; ++i)
01051           Args.push_back(MD->getOperand(i));
01052       } else {
01053         S = dyn_cast<MDString>(LoopID->getOperand(i));
01054         assert(Args.size() == 0 && "too many arguments for MDString");
01055       }
01056 
01057       if (!S)
01058         continue;
01059 
01060       // Check if the hint starts with the loop metadata prefix.
01061       StringRef Name = S->getString();
01062       if (Args.size() == 1)
01063         setHint(Name, Args[0]);
01064     }
01065   }
01066 
01067   /// Checks string hint with one operand and set value if valid.
01068   void setHint(StringRef Name, Metadata *Arg) {
01069     if (!Name.startswith(Prefix()))
01070       return;
01071     Name = Name.substr(Prefix().size(), StringRef::npos);
01072 
01073     const ConstantInt *C = mdconst::dyn_extract<ConstantInt>(Arg);
01074     if (!C) return;
01075     unsigned Val = C->getZExtValue();
01076 
01077     Hint *Hints[] = {&Width, &Interleave, &Force};
01078     for (auto H : Hints) {
01079       if (Name == H->Name) {
01080         if (H->validate(Val))
01081           H->Value = Val;
01082         else
01083           DEBUG(dbgs() << "LV: ignoring invalid hint '" << Name << "'\n");
01084         break;
01085       }
01086     }
01087   }
01088 
01089   /// Create a new hint from name / value pair.
01090   MDNode *createHintMetadata(StringRef Name, unsigned V) const {
01091     LLVMContext &Context = TheLoop->getHeader()->getContext();
01092     Metadata *MDs[] = {MDString::get(Context, Name),
01093                        ConstantAsMetadata::get(
01094                            ConstantInt::get(Type::getInt32Ty(Context), V))};
01095     return MDNode::get(Context, MDs);
01096   }
01097 
01098   /// Matches metadata with hint name.
01099   bool matchesHintMetadataName(MDNode *Node, ArrayRef<Hint> HintTypes) {
01100     MDString* Name = dyn_cast<MDString>(Node->getOperand(0));
01101     if (!Name)
01102       return false;
01103 
01104     for (auto H : HintTypes)
01105       if (Name->getString().endswith(H.Name))
01106         return true;
01107     return false;
01108   }
01109 
01110   /// Sets current hints into loop metadata, keeping other values intact.
01111   void writeHintsToMetadata(ArrayRef<Hint> HintTypes) {
01112     if (HintTypes.size() == 0)
01113       return;
01114 
01115     // Reserve the first element to LoopID (see below).
01116     SmallVector<Metadata *, 4> MDs(1);
01117     // If the loop already has metadata, then ignore the existing operands.
01118     MDNode *LoopID = TheLoop->getLoopID();
01119     if (LoopID) {
01120       for (unsigned i = 1, ie = LoopID->getNumOperands(); i < ie; ++i) {
01121         MDNode *Node = cast<MDNode>(LoopID->getOperand(i));
01122         // If node in update list, ignore old value.
01123         if (!matchesHintMetadataName(Node, HintTypes))
01124           MDs.push_back(Node);
01125       }
01126     }
01127 
01128     // Now, add the missing hints.
01129     for (auto H : HintTypes)
01130       MDs.push_back(createHintMetadata(Twine(Prefix(), H.Name).str(), H.Value));
01131 
01132     // Replace current metadata node with new one.
01133     LLVMContext &Context = TheLoop->getHeader()->getContext();
01134     MDNode *NewLoopID = MDNode::get(Context, MDs);
01135     // Set operand 0 to refer to the loop id itself.
01136     NewLoopID->replaceOperandWith(0, NewLoopID);
01137 
01138     TheLoop->setLoopID(NewLoopID);
01139   }
01140 
01141   /// The loop these hints belong to.
01142   const Loop *TheLoop;
01143 };
01144 
01145 static void emitAnalysisDiag(const Function *TheFunction, const Loop *TheLoop,
01146                              const LoopVectorizeHints &Hints,
01147                              const LoopAccessReport &Message) {
01148   const char *Name = Hints.vectorizeAnalysisPassName();
01149   LoopAccessReport::emitAnalysis(Message, TheFunction, TheLoop, Name);
01150 }
01151 
01152 static void emitMissedWarning(Function *F, Loop *L,
01153                               const LoopVectorizeHints &LH) {
01154   emitOptimizationRemarkMissed(F->getContext(), LV_NAME, *F, L->getStartLoc(),
01155                                LH.emitRemark());
01156 
01157   if (LH.getForce() == LoopVectorizeHints::FK_Enabled) {
01158     if (LH.getWidth() != 1)
01159       emitLoopVectorizeWarning(
01160           F->getContext(), *F, L->getStartLoc(),
01161           "failed explicitly specified loop vectorization");
01162     else if (LH.getInterleave() != 1)
01163       emitLoopInterleaveWarning(
01164           F->getContext(), *F, L->getStartLoc(),
01165           "failed explicitly specified loop interleaving");
01166   }
01167 }
01168 
01169 /// LoopVectorizationLegality checks if it is legal to vectorize a loop, and
01170 /// to what vectorization factor.
01171 /// This class does not look at the profitability of vectorization, only the
01172 /// legality. This class has two main kinds of checks:
01173 /// * Memory checks - The code in canVectorizeMemory checks if vectorization
01174 ///   will change the order of memory accesses in a way that will change the
01175 ///   correctness of the program.
01176 /// * Scalars checks - The code in canVectorizeInstrs and canVectorizeMemory
01177 /// checks for a number of different conditions, such as the availability of a
01178 /// single induction variable, that all types are supported and vectorize-able,
01179 /// etc. This code reflects the capabilities of InnerLoopVectorizer.
01180 /// This class is also used by InnerLoopVectorizer for identifying
01181 /// induction variable and the different reduction variables.
01182 class LoopVectorizationLegality {
01183 public:
01184   LoopVectorizationLegality(Loop *L, PredicatedScalarEvolution &PSE,
01185                             DominatorTree *DT, TargetLibraryInfo *TLI,
01186                             AliasAnalysis *AA, Function *F,
01187                             const TargetTransformInfo *TTI,
01188                             LoopAccessAnalysis *LAA,
01189                             LoopVectorizationRequirements *R,
01190                             const LoopVectorizeHints *H)
01191       : NumPredStores(0), TheLoop(L), PSE(PSE), TLI(TLI), TheFunction(F),
01192         TTI(TTI), DT(DT), LAA(LAA), LAI(nullptr), InterleaveInfo(PSE, L, DT),
01193         Induction(nullptr), WidestIndTy(nullptr), HasFunNoNaNAttr(false),
01194         Requirements(R), Hints(H) {}
01195 
01196   /// ReductionList contains the reduction descriptors for all
01197   /// of the reductions that were found in the loop.
01198   typedef DenseMap<PHINode *, RecurrenceDescriptor> ReductionList;
01199 
01200   /// InductionList saves induction variables and maps them to the
01201   /// induction descriptor.
01202   typedef MapVector<PHINode*, InductionDescriptor> InductionList;
01203 
01204   /// Returns true if it is legal to vectorize this loop.
01205   /// This does not mean that it is profitable to vectorize this
01206   /// loop, only that it is legal to do so.
01207   bool canVectorize();
01208 
01209   /// Returns the Induction variable.
01210   PHINode *getInduction() { return Induction; }
01211 
01212   /// Returns the reduction variables found in the loop.
01213   ReductionList *getReductionVars() { return &Reductions; }
01214 
01215   /// Returns the induction variables found in the loop.
01216   InductionList *getInductionVars() { return &Inductions; }
01217 
01218   /// Returns the widest induction type.
01219   Type *getWidestInductionType() { return WidestIndTy; }
01220 
01221   /// Returns True if V is an induction variable in this loop.
01222   bool isInductionVariable(const Value *V);
01223 
01224   /// Returns True if PN is a reduction variable in this loop.
01225   bool isReductionVariable(PHINode *PN) { return Reductions.count(PN); }
01226 
01227   /// Return true if the block BB needs to be predicated in order for the loop
01228   /// to be vectorized.
01229   bool blockNeedsPredication(BasicBlock *BB);
01230 
01231   /// Check if this  pointer is consecutive when vectorizing. This happens
01232   /// when the last index of the GEP is the induction variable, or that the
01233   /// pointer itself is an induction variable.
01234   /// This check allows us to vectorize A[idx] into a wide load/store.
01235   /// Returns:
01236   /// 0 - Stride is unknown or non-consecutive.
01237   /// 1 - Address is consecutive.
01238   /// -1 - Address is consecutive, and decreasing.
01239   int isConsecutivePtr(Value *Ptr);
01240 
01241   /// Returns true if the value V is uniform within the loop.
01242   bool isUniform(Value *V);
01243 
01244   /// Returns true if this instruction will remain scalar after vectorization.
01245   bool isUniformAfterVectorization(Instruction* I) { return Uniforms.count(I); }
01246 
01247   /// Returns the information that we collected about runtime memory check.
01248   const RuntimePointerChecking *getRuntimePointerChecking() const {
01249     return LAI->getRuntimePointerChecking();
01250   }
01251 
01252   const LoopAccessInfo *getLAI() const {
01253     return LAI;
01254   }
01255 
01256   /// \brief Check if \p Instr belongs to any interleaved access group.
01257   bool isAccessInterleaved(Instruction *Instr) {
01258     return InterleaveInfo.isInterleaved(Instr);
01259   }
01260 
01261   /// \brief Get the interleaved access group that \p Instr belongs to.
01262   const InterleaveGroup *getInterleavedAccessGroup(Instruction *Instr) {
01263     return InterleaveInfo.getInterleaveGroup(Instr);
01264   }
01265 
01266   unsigned getMaxSafeDepDistBytes() { return LAI->getMaxSafeDepDistBytes(); }
01267 
01268   bool hasStride(Value *V) { return StrideSet.count(V); }
01269   bool mustCheckStrides() { return !StrideSet.empty(); }
01270   SmallPtrSet<Value *, 8>::iterator strides_begin() {
01271     return StrideSet.begin();
01272   }
01273   SmallPtrSet<Value *, 8>::iterator strides_end() { return StrideSet.end(); }
01274 
01275   /// Returns true if the target machine supports masked store operation
01276   /// for the given \p DataType and kind of access to \p Ptr.
01277   bool isLegalMaskedStore(Type *DataType, Value *Ptr) {
01278     return isConsecutivePtr(Ptr) && TTI->isLegalMaskedStore(DataType);
01279   }
01280   /// Returns true if the target machine supports masked load operation
01281   /// for the given \p DataType and kind of access to \p Ptr.
01282   bool isLegalMaskedLoad(Type *DataType, Value *Ptr) {
01283     return isConsecutivePtr(Ptr) && TTI->isLegalMaskedLoad(DataType);
01284   }
01285   /// Returns true if vector representation of the instruction \p I
01286   /// requires mask.
01287   bool isMaskRequired(const Instruction* I) {
01288     return (MaskedOp.count(I) != 0);
01289   }
01290   unsigned getNumStores() const {
01291     return LAI->getNumStores();
01292   }
01293   unsigned getNumLoads() const {
01294     return LAI->getNumLoads();
01295   }
01296   unsigned getNumPredStores() const {
01297     return NumPredStores;
01298   }
01299 private:
01300   /// Check if a single basic block loop is vectorizable.
01301   /// At this point we know that this is a loop with a constant trip count
01302   /// and we only need to check individual instructions.
01303   bool canVectorizeInstrs();
01304 
01305   /// When we vectorize loops we may change the order in which
01306   /// we read and write from memory. This method checks if it is
01307   /// legal to vectorize the code, considering only memory constrains.
01308   /// Returns true if the loop is vectorizable
01309   bool canVectorizeMemory();
01310 
01311   /// Return true if we can vectorize this loop using the IF-conversion
01312   /// transformation.
01313   bool canVectorizeWithIfConvert();
01314 
01315   /// Collect the variables that need to stay uniform after vectorization.
01316   void collectLoopUniforms();
01317 
01318   /// Return true if all of the instructions in the block can be speculatively
01319   /// executed. \p SafePtrs is a list of addresses that are known to be legal
01320   /// and we know that we can read from them without segfault.
01321   bool blockCanBePredicated(BasicBlock *BB, SmallPtrSetImpl<Value *> &SafePtrs);
01322 
01323   /// \brief Collect memory access with loop invariant strides.
01324   ///
01325   /// Looks for accesses like "a[i * StrideA]" where "StrideA" is loop
01326   /// invariant.
01327   void collectStridedAccess(Value *LoadOrStoreInst);
01328 
01329   /// Report an analysis message to assist the user in diagnosing loops that are
01330   /// not vectorized.  These are handled as LoopAccessReport rather than
01331   /// VectorizationReport because the << operator of VectorizationReport returns
01332   /// LoopAccessReport.
01333   void emitAnalysis(const LoopAccessReport &Message) const {
01334     emitAnalysisDiag(TheFunction, TheLoop, *Hints, Message);
01335   }
01336 
01337   unsigned NumPredStores;
01338 
01339   /// The loop that we evaluate.
01340   Loop *TheLoop;
01341   /// A wrapper around ScalarEvolution used to add runtime SCEV checks.
01342   /// Applies dynamic knowledge to simplify SCEV expressions in the context
01343   /// of existing SCEV assumptions. The analysis will also add a minimal set
01344   /// of new predicates if this is required to enable vectorization and
01345   /// unrolling.
01346   PredicatedScalarEvolution &PSE;
01347   /// Target Library Info.
01348   TargetLibraryInfo *TLI;
01349   /// Parent function
01350   Function *TheFunction;
01351   /// Target Transform Info
01352   const TargetTransformInfo *TTI;
01353   /// Dominator Tree.
01354   DominatorTree *DT;
01355   // LoopAccess analysis.
01356   LoopAccessAnalysis *LAA;
01357   // And the loop-accesses info corresponding to this loop.  This pointer is
01358   // null until canVectorizeMemory sets it up.
01359   const LoopAccessInfo *LAI;
01360 
01361   /// The interleave access information contains groups of interleaved accesses
01362   /// with the same stride and close to each other.
01363   InterleavedAccessInfo InterleaveInfo;
01364 
01365   //  ---  vectorization state --- //
01366 
01367   /// Holds the integer induction variable. This is the counter of the
01368   /// loop.
01369   PHINode *Induction;
01370   /// Holds the reduction variables.
01371   ReductionList Reductions;
01372   /// Holds all of the induction variables that we found in the loop.
01373   /// Notice that inductions don't need to start at zero and that induction
01374   /// variables can be pointers.
01375   InductionList Inductions;
01376   /// Holds the widest induction type encountered.
01377   Type *WidestIndTy;
01378 
01379   /// Allowed outside users. This holds the reduction
01380   /// vars which can be accessed from outside the loop.
01381   SmallPtrSet<Value*, 4> AllowedExit;
01382   /// This set holds the variables which are known to be uniform after
01383   /// vectorization.
01384   SmallPtrSet<Instruction*, 4> Uniforms;
01385 
01386   /// Can we assume the absence of NaNs.
01387   bool HasFunNoNaNAttr;
01388 
01389   /// Vectorization requirements that will go through late-evaluation.
01390   LoopVectorizationRequirements *Requirements;
01391 
01392   /// Used to emit an analysis of any legality issues.
01393   const LoopVectorizeHints *Hints;
01394 
01395   ValueToValueMap Strides;
01396   SmallPtrSet<Value *, 8> StrideSet;
01397 
01398   /// While vectorizing these instructions we have to generate a
01399   /// call to the appropriate masked intrinsic
01400   SmallPtrSet<const Instruction *, 8> MaskedOp;
01401 };
01402 
01403 /// LoopVectorizationCostModel - estimates the expected speedups due to
01404 /// vectorization.
01405 /// In many cases vectorization is not profitable. This can happen because of
01406 /// a number of reasons. In this class we mainly attempt to predict the
01407 /// expected speedup/slowdowns due to the supported instruction set. We use the
01408 /// TargetTransformInfo to query the different backends for the cost of
01409 /// different operations.
01410 class LoopVectorizationCostModel {
01411 public:
01412   LoopVectorizationCostModel(Loop *L, PredicatedScalarEvolution &PSE,
01413                              LoopInfo *LI, LoopVectorizationLegality *Legal,
01414                              const TargetTransformInfo &TTI,
01415                              const TargetLibraryInfo *TLI, DemandedBits *DB,
01416                              AssumptionCache *AC, const Function *F,
01417                              const LoopVectorizeHints *Hints)
01418       : TheLoop(L), PSE(PSE), LI(LI), Legal(Legal), TTI(TTI), TLI(TLI), DB(DB),
01419         AC(AC), TheFunction(F), Hints(Hints) {}
01420 
01421   /// Information about vectorization costs
01422   struct VectorizationFactor {
01423     unsigned Width; // Vector width with best cost
01424     unsigned Cost; // Cost of the loop with that width
01425   };
01426   /// \return The most profitable vectorization factor and the cost of that VF.
01427   /// This method checks every power of two up to VF. If UserVF is not ZERO
01428   /// then this vectorization factor will be selected if vectorization is
01429   /// possible.
01430   VectorizationFactor selectVectorizationFactor(bool OptForSize);
01431 
01432   /// \return The size (in bits) of the smallest and widest types in the code
01433   /// that needs to be vectorized. We ignore values that remain scalar such as
01434   /// 64 bit loop indices.
01435   std::pair<unsigned, unsigned> getSmallestAndWidestTypes();
01436 
01437   /// \return The desired interleave count.
01438   /// If interleave count has been specified by metadata it will be returned.
01439   /// Otherwise, the interleave count is computed and returned. VF and LoopCost
01440   /// are the selected vectorization factor and the cost of the selected VF.
01441   unsigned selectInterleaveCount(bool OptForSize, unsigned VF,
01442                                  unsigned LoopCost);
01443 
01444   /// \return The most profitable unroll factor.
01445   /// This method finds the best unroll-factor based on register pressure and
01446   /// other parameters. VF and LoopCost are the selected vectorization factor
01447   /// and the cost of the selected VF.
01448   unsigned computeInterleaveCount(bool OptForSize, unsigned VF,
01449                                   unsigned LoopCost);
01450 
01451   /// \brief A struct that represents some properties of the register usage
01452   /// of a loop.
01453   struct RegisterUsage {
01454     /// Holds the number of loop invariant values that are used in the loop.
01455     unsigned LoopInvariantRegs;
01456     /// Holds the maximum number of concurrent live intervals in the loop.
01457     unsigned MaxLocalUsers;
01458     /// Holds the number of instructions in the loop.
01459     unsigned NumInstructions;
01460   };
01461 
01462   /// \return Returns information about the register usages of the loop for the
01463   /// given vectorization factors.
01464   SmallVector<RegisterUsage, 8>
01465   calculateRegisterUsage(const SmallVector<unsigned, 8> &VFs);
01466 
01467   /// Collect values we want to ignore in the cost model.
01468   void collectValuesToIgnore();
01469 
01470 private:
01471   /// Returns the expected execution cost. The unit of the cost does
01472   /// not matter because we use the 'cost' units to compare different
01473   /// vector widths. The cost that is returned is *not* normalized by
01474   /// the factor width.
01475   unsigned expectedCost(unsigned VF);
01476 
01477   /// Returns the execution time cost of an instruction for a given vector
01478   /// width. Vector width of one means scalar.
01479   unsigned getInstructionCost(Instruction *I, unsigned VF);
01480 
01481   /// Returns whether the instruction is a load or store and will be a emitted
01482   /// as a vector operation.
01483   bool isConsecutiveLoadOrStore(Instruction *I);
01484 
01485   /// Report an analysis message to assist the user in diagnosing loops that are
01486   /// not vectorized.  These are handled as LoopAccessReport rather than
01487   /// VectorizationReport because the << operator of VectorizationReport returns
01488   /// LoopAccessReport.
01489   void emitAnalysis(const LoopAccessReport &Message) const {
01490     emitAnalysisDiag(TheFunction, TheLoop, *Hints, Message);
01491   }
01492 
01493 public:
01494   /// Map of scalar integer values to the smallest bitwidth they can be legally
01495   /// represented as. The vector equivalents of these values should be truncated
01496   /// to this type.
01497   MapVector<Instruction*,uint64_t> MinBWs;
01498 
01499   /// The loop that we evaluate.
01500   Loop *TheLoop;
01501   /// Predicated scalar evolution analysis.
01502   PredicatedScalarEvolution &PSE;
01503   /// Loop Info analysis.
01504   LoopInfo *LI;
01505   /// Vectorization legality.
01506   LoopVectorizationLegality *Legal;
01507   /// Vector target information.
01508   const TargetTransformInfo &TTI;
01509   /// Target Library Info.
01510   const TargetLibraryInfo *TLI;
01511   /// Demanded bits analysis.
01512   DemandedBits *DB;
01513   /// Assumption cache.
01514   AssumptionCache *AC;
01515   const Function *TheFunction;
01516   /// Loop Vectorize Hint.
01517   const LoopVectorizeHints *Hints;
01518   /// Values to ignore in the cost model.
01519   SmallPtrSet<const Value *, 16> ValuesToIgnore;
01520   /// Values to ignore in the cost model when VF > 1.
01521   SmallPtrSet<const Value *, 16> VecValuesToIgnore;
01522 };
01523 
01524 /// \brief This holds vectorization requirements that must be verified late in
01525 /// the process. The requirements are set by legalize and costmodel. Once
01526 /// vectorization has been determined to be possible and profitable the
01527 /// requirements can be verified by looking for metadata or compiler options.
01528 /// For example, some loops require FP commutativity which is only allowed if
01529 /// vectorization is explicitly specified or if the fast-math compiler option
01530 /// has been provided.
01531 /// Late evaluation of these requirements allows helpful diagnostics to be
01532 /// composed that tells the user what need to be done to vectorize the loop. For
01533 /// example, by specifying #pragma clang loop vectorize or -ffast-math. Late
01534 /// evaluation should be used only when diagnostics can generated that can be
01535 /// followed by a non-expert user.
01536 class LoopVectorizationRequirements {
01537 public:
01538   LoopVectorizationRequirements()
01539       : NumRuntimePointerChecks(0), UnsafeAlgebraInst(nullptr) {}
01540 
01541   void addUnsafeAlgebraInst(Instruction *I) {
01542     // First unsafe algebra instruction.
01543     if (!UnsafeAlgebraInst)
01544       UnsafeAlgebraInst = I;
01545   }
01546 
01547   void addRuntimePointerChecks(unsigned Num) { NumRuntimePointerChecks = Num; }
01548 
01549   bool doesNotMeet(Function *F, Loop *L, const LoopVectorizeHints &Hints) {
01550     const char *Name = Hints.vectorizeAnalysisPassName();
01551     bool Failed = false;
01552     if (UnsafeAlgebraInst && !Hints.allowReordering()) {
01553       emitOptimizationRemarkAnalysisFPCommute(
01554           F->getContext(), Name, *F, UnsafeAlgebraInst->getDebugLoc(),
01555           VectorizationReport() << "cannot prove it is safe to reorder "
01556                                    "floating-point operations");
01557       Failed = true;
01558     }
01559 
01560     // Test if runtime memcheck thresholds are exceeded.
01561     bool PragmaThresholdReached =
01562         NumRuntimePointerChecks > PragmaVectorizeMemoryCheckThreshold;
01563     bool ThresholdReached =
01564         NumRuntimePointerChecks > VectorizerParams::RuntimeMemoryCheckThreshold;
01565     if ((ThresholdReached && !Hints.allowReordering()) ||
01566         PragmaThresholdReached) {
01567       emitOptimizationRemarkAnalysisAliasing(
01568           F->getContext(), Name, *F, L->getStartLoc(),
01569           VectorizationReport()
01570               << "cannot prove it is safe to reorder memory operations");
01571       DEBUG(dbgs() << "LV: Too many memory checks needed.\n");
01572       Failed = true;
01573     }
01574 
01575     return Failed;
01576   }
01577 
01578 private:
01579   unsigned NumRuntimePointerChecks;
01580   Instruction *UnsafeAlgebraInst;
01581 };
01582 
01583 static void addInnerLoop(Loop &L, SmallVectorImpl<Loop *> &V) {
01584   if (L.empty())
01585     return V.push_back(&L);
01586 
01587   for (Loop *InnerL : L)
01588     addInnerLoop(*InnerL, V);
01589 }
01590 
01591 /// The LoopVectorize Pass.
01592 struct LoopVectorize : public FunctionPass {
01593   /// Pass identification, replacement for typeid
01594   static char ID;
01595 
01596   explicit LoopVectorize(bool NoUnrolling = false, bool AlwaysVectorize = true)
01597     : FunctionPass(ID),
01598       DisableUnrolling(NoUnrolling),
01599       AlwaysVectorize(AlwaysVectorize) {
01600     initializeLoopVectorizePass(*PassRegistry::getPassRegistry());
01601   }
01602 
01603   ScalarEvolution *SE;
01604   LoopInfo *LI;
01605   TargetTransformInfo *TTI;
01606   DominatorTree *DT;
01607   BlockFrequencyInfo *BFI;
01608   TargetLibraryInfo *TLI;
01609   DemandedBits *DB;
01610   AliasAnalysis *AA;
01611   AssumptionCache *AC;
01612   LoopAccessAnalysis *LAA;
01613   bool DisableUnrolling;
01614   bool AlwaysVectorize;
01615 
01616   BlockFrequency ColdEntryFreq;
01617 
01618   bool runOnFunction(Function &F) override {
01619     SE = &getAnalysis<ScalarEvolutionWrapperPass>().getSE();
01620     LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
01621     TTI = &getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
01622     DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree();
01623     BFI = &getAnalysis<BlockFrequencyInfoWrapperPass>().getBFI();
01624     auto *TLIP = getAnalysisIfAvailable<TargetLibraryInfoWrapperPass>();
01625     TLI = TLIP ? &TLIP->getTLI() : nullptr;
01626     AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
01627     AC = &getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F);
01628     LAA = &getAnalysis<LoopAccessAnalysis>();
01629     DB = &getAnalysis<DemandedBits>();
01630 
01631     // Compute some weights outside of the loop over the loops. Compute this
01632     // using a BranchProbability to re-use its scaling math.
01633     const BranchProbability ColdProb(1, 5); // 20%
01634     ColdEntryFreq = BlockFrequency(BFI->getEntryFreq()) * ColdProb;
01635 
01636     // Don't attempt if
01637     // 1. the target claims to have no vector registers, and
01638     // 2. interleaving won't help ILP.
01639     //
01640     // The second condition is necessary because, even if the target has no
01641     // vector registers, loop vectorization may still enable scalar
01642     // interleaving.
01643     if (!TTI->getNumberOfRegisters(true) && TTI->getMaxInterleaveFactor(1) < 2)
01644       return false;
01645 
01646     // Build up a worklist of inner-loops to vectorize. This is necessary as
01647     // the act of vectorizing or partially unrolling a loop creates new loops
01648     // and can invalidate iterators across the loops.
01649     SmallVector<Loop *, 8> Worklist;
01650 
01651     for (Loop *L : *LI)
01652       addInnerLoop(*L, Worklist);
01653 
01654     LoopsAnalyzed += Worklist.size();
01655 
01656     // Now walk the identified inner loops.
01657     bool Changed = false;
01658     while (!Worklist.empty())
01659       Changed |= processLoop(Worklist.pop_back_val());
01660 
01661     // Process each loop nest in the function.
01662     return Changed;
01663   }
01664 
01665   static void AddRuntimeUnrollDisableMetaData(Loop *L) {
01666     SmallVector<Metadata *, 4> MDs;
01667     // Reserve first location for self reference to the LoopID metadata node.
01668     MDs.push_back(nullptr);
01669     bool IsUnrollMetadata = false;
01670     MDNode *LoopID = L->getLoopID();
01671     if (LoopID) {
01672       // First find existing loop unrolling disable metadata.
01673       for (unsigned i = 1, ie = LoopID->getNumOperands(); i < ie; ++i) {
01674         MDNode *MD = dyn_cast<MDNode>(LoopID->getOperand(i));
01675         if (MD) {
01676           const MDString *S = dyn_cast<MDString>(MD->getOperand(0));
01677           IsUnrollMetadata =
01678               S && S->getString().startswith("llvm.loop.unroll.disable");
01679         }
01680         MDs.push_back(LoopID->getOperand(i));
01681       }
01682     }
01683 
01684     if (!IsUnrollMetadata) {
01685       // Add runtime unroll disable metadata.
01686       LLVMContext &Context = L->getHeader()->getContext();
01687       SmallVector<Metadata *, 1> DisableOperands;
01688       DisableOperands.push_back(
01689           MDString::get(Context, "llvm.loop.unroll.runtime.disable"));
01690       MDNode *DisableNode = MDNode::get(Context, DisableOperands);
01691       MDs.push_back(DisableNode);
01692       MDNode *NewLoopID = MDNode::get(Context, MDs);
01693       // Set operand 0 to refer to the loop id itself.
01694       NewLoopID->replaceOperandWith(0, NewLoopID);
01695       L->setLoopID(NewLoopID);
01696     }
01697   }
01698 
01699   bool processLoop(Loop *L) {
01700     assert(L->empty() && "Only process inner loops.");
01701 
01702 #ifndef NDEBUG
01703     const std::string DebugLocStr = getDebugLocString(L);
01704 #endif /* NDEBUG */
01705 
01706     DEBUG(dbgs() << "\nLV: Checking a loop in \""
01707                  << L->getHeader()->getParent()->getName() << "\" from "
01708                  << DebugLocStr << "\n");
01709 
01710     LoopVectorizeHints Hints(L, DisableUnrolling);
01711 
01712     DEBUG(dbgs() << "LV: Loop hints:"
01713                  << " force="
01714                  << (Hints.getForce() == LoopVectorizeHints::FK_Disabled
01715                          ? "disabled"
01716                          : (Hints.getForce() == LoopVectorizeHints::FK_Enabled
01717                                 ? "enabled"
01718                                 : "?")) << " width=" << Hints.getWidth()
01719                  << " unroll=" << Hints.getInterleave() << "\n");
01720 
01721     // Function containing loop
01722     Function *F = L->getHeader()->getParent();
01723 
01724     // Looking at the diagnostic output is the only way to determine if a loop
01725     // was vectorized (other than looking at the IR or machine code), so it
01726     // is important to generate an optimization remark for each loop. Most of
01727     // these messages are generated by emitOptimizationRemarkAnalysis. Remarks
01728     // generated by emitOptimizationRemark and emitOptimizationRemarkMissed are
01729     // less verbose reporting vectorized loops and unvectorized loops that may
01730     // benefit from vectorization, respectively.
01731 
01732     if (!Hints.allowVectorization(F, L, AlwaysVectorize)) {
01733       DEBUG(dbgs() << "LV: Loop hints prevent vectorization.\n");
01734       return false;
01735     }
01736 
01737     // Check the loop for a trip count threshold:
01738     // do not vectorize loops with a tiny trip count.
01739     const unsigned TC = SE->getSmallConstantTripCount(L);
01740     if (TC > 0u && TC < TinyTripCountVectorThreshold) {
01741       DEBUG(dbgs() << "LV: Found a loop with a very small trip count. "
01742                    << "This loop is not worth vectorizing.");
01743       if (Hints.getForce() == LoopVectorizeHints::FK_Enabled)
01744         DEBUG(dbgs() << " But vectorizing was explicitly forced.\n");
01745       else {
01746         DEBUG(dbgs() << "\n");
01747         emitAnalysisDiag(F, L, Hints, VectorizationReport()
01748                                           << "vectorization is not beneficial "
01749                                              "and is not explicitly forced");
01750         return false;
01751       }
01752     }
01753 
01754     PredicatedScalarEvolution PSE(*SE);
01755 
01756     // Check if it is legal to vectorize the loop.
01757     LoopVectorizationRequirements Requirements;
01758     LoopVectorizationLegality LVL(L, PSE, DT, TLI, AA, F, TTI, LAA,
01759                                   &Requirements, &Hints);
01760     if (!LVL.canVectorize()) {
01761       DEBUG(dbgs() << "LV: Not vectorizing: Cannot prove legality.\n");
01762       emitMissedWarning(F, L, Hints);
01763       return false;
01764     }
01765 
01766     // Use the cost model.
01767     LoopVectorizationCostModel CM(L, PSE, LI, &LVL, *TTI, TLI, DB, AC, F,
01768                                   &Hints);
01769     CM.collectValuesToIgnore();
01770 
01771     // Check the function attributes to find out if this function should be
01772     // optimized for size.
01773     bool OptForSize = Hints.getForce() != LoopVectorizeHints::FK_Enabled &&
01774                       F->optForSize();
01775 
01776     // Compute the weighted frequency of this loop being executed and see if it
01777     // is less than 20% of the function entry baseline frequency. Note that we
01778     // always have a canonical loop here because we think we *can* vectorize.
01779     // FIXME: This is hidden behind a flag due to pervasive problems with
01780     // exactly what block frequency models.
01781     if (LoopVectorizeWithBlockFrequency) {
01782       BlockFrequency LoopEntryFreq = BFI->getBlockFreq(L->getLoopPreheader());
01783       if (Hints.getForce() != LoopVectorizeHints::FK_Enabled &&
01784           LoopEntryFreq < ColdEntryFreq)
01785         OptForSize = true;
01786     }
01787 
01788     // Check the function attributes to see if implicit floats are allowed.
01789     // FIXME: This check doesn't seem possibly correct -- what if the loop is
01790     // an integer loop and the vector instructions selected are purely integer
01791     // vector instructions?
01792     if (F->hasFnAttribute(Attribute::NoImplicitFloat)) {
01793       DEBUG(dbgs() << "LV: Can't vectorize when the NoImplicitFloat"
01794             "attribute is used.\n");
01795       emitAnalysisDiag(
01796           F, L, Hints,
01797           VectorizationReport()
01798               << "loop not vectorized due to NoImplicitFloat attribute");
01799       emitMissedWarning(F, L, Hints);
01800       return false;
01801     }
01802 
01803     // Select the optimal vectorization factor.
01804     const LoopVectorizationCostModel::VectorizationFactor VF =
01805         CM.selectVectorizationFactor(OptForSize);
01806 
01807     // Select the interleave count.
01808     unsigned IC = CM.selectInterleaveCount(OptForSize, VF.Width, VF.Cost);
01809 
01810     // Get user interleave count.
01811     unsigned UserIC = Hints.getInterleave();
01812 
01813     // Identify the diagnostic messages that should be produced.
01814     std::string VecDiagMsg, IntDiagMsg;
01815     bool VectorizeLoop = true, InterleaveLoop = true;
01816 
01817     if (Requirements.doesNotMeet(F, L, Hints)) {
01818       DEBUG(dbgs() << "LV: Not vectorizing: loop did not meet vectorization "
01819                       "requirements.\n");
01820       emitMissedWarning(F, L, Hints);
01821       return false;
01822     }
01823 
01824     if (VF.Width == 1) {
01825       DEBUG(dbgs() << "LV: Vectorization is possible but not beneficial.\n");
01826       VecDiagMsg =
01827           "the cost-model indicates that vectorization is not beneficial";
01828       VectorizeLoop = false;
01829     }
01830 
01831     if (IC == 1 && UserIC <= 1) {
01832       // Tell the user interleaving is not beneficial.
01833       DEBUG(dbgs() << "LV: Interleaving is not beneficial.\n");
01834       IntDiagMsg =
01835           "the cost-model indicates that interleaving is not beneficial";
01836       InterleaveLoop = false;
01837       if (UserIC == 1)
01838         IntDiagMsg +=
01839             " and is explicitly disabled or interleave count is set to 1";
01840     } else if (IC > 1 && UserIC == 1) {
01841       // Tell the user interleaving is beneficial, but it explicitly disabled.
01842       DEBUG(dbgs()
01843             << "LV: Interleaving is beneficial but is explicitly disabled.");
01844       IntDiagMsg = "the cost-model indicates that interleaving is beneficial "
01845                    "but is explicitly disabled or interleave count is set to 1";
01846       InterleaveLoop = false;
01847     }
01848 
01849     // Override IC if user provided an interleave count.
01850     IC = UserIC > 0 ? UserIC : IC;
01851 
01852     // Emit diagnostic messages, if any.
01853     const char *VAPassName = Hints.vectorizeAnalysisPassName();
01854     if (!VectorizeLoop && !InterleaveLoop) {
01855       // Do not vectorize or interleaving the loop.
01856       emitOptimizationRemarkAnalysis(F->getContext(), VAPassName, *F,
01857                                      L->getStartLoc(), VecDiagMsg);
01858       emitOptimizationRemarkAnalysis(F->getContext(), LV_NAME, *F,
01859                                      L->getStartLoc(), IntDiagMsg);
01860       return false;
01861     } else if (!VectorizeLoop && InterleaveLoop) {
01862       DEBUG(dbgs() << "LV: Interleave Count is " << IC << '\n');
01863       emitOptimizationRemarkAnalysis(F->getContext(), VAPassName, *F,
01864                                      L->getStartLoc(), VecDiagMsg);
01865     } else if (VectorizeLoop && !InterleaveLoop) {
01866       DEBUG(dbgs() << "LV: Found a vectorizable loop (" << VF.Width << ") in "
01867                    << DebugLocStr << '\n');
01868       emitOptimizationRemarkAnalysis(F->getContext(), LV_NAME, *F,
01869                                      L->getStartLoc(), IntDiagMsg);
01870     } else if (VectorizeLoop && InterleaveLoop) {
01871       DEBUG(dbgs() << "LV: Found a vectorizable loop (" << VF.Width << ") in "
01872                    << DebugLocStr << '\n');
01873       DEBUG(dbgs() << "LV: Interleave Count is " << IC << '\n');
01874     }
01875 
01876     if (!VectorizeLoop) {
01877       assert(IC > 1 && "interleave count should not be 1 or 0");
01878       // If we decided that it is not legal to vectorize the loop then
01879       // interleave it.
01880       InnerLoopUnroller Unroller(L, PSE, LI, DT, TLI, TTI, IC);
01881       Unroller.vectorize(&LVL, CM.MinBWs);
01882 
01883       emitOptimizationRemark(F->getContext(), LV_NAME, *F, L->getStartLoc(),
01884                              Twine("interleaved loop (interleaved count: ") +
01885                                  Twine(IC) + ")");
01886     } else {
01887       // If we decided that it is *legal* to vectorize the loop then do it.
01888       InnerLoopVectorizer LB(L, PSE, LI, DT, TLI, TTI, VF.Width, IC);
01889       LB.vectorize(&LVL, CM.MinBWs);
01890       ++LoopsVectorized;
01891 
01892       // Add metadata to disable runtime unrolling scalar loop when there's no
01893       // runtime check about strides and memory. Because at this situation,
01894       // scalar loop is rarely used not worthy to be unrolled.
01895       if (!LB.IsSafetyChecksAdded())
01896         AddRuntimeUnrollDisableMetaData(L);
01897 
01898       // Report the vectorization decision.
01899       emitOptimizationRemark(F->getContext(), LV_NAME, *F, L->getStartLoc(),
01900                              Twine("vectorized loop (vectorization width: ") +
01901                                  Twine(VF.Width) + ", interleaved count: " +
01902                                  Twine(IC) + ")");
01903     }
01904 
01905     // Mark the loop as already vectorized to avoid vectorizing again.
01906     Hints.setAlreadyVectorized();
01907 
01908     DEBUG(verifyFunction(*L->getHeader()->getParent()));
01909     return true;
01910   }
01911 
01912   void getAnalysisUsage(AnalysisUsage &AU) const override {
01913     AU.addRequired<AssumptionCacheTracker>();
01914     AU.addRequiredID(LoopSimplifyID);
01915     AU.addRequiredID(LCSSAID);
01916     AU.addRequired<BlockFrequencyInfoWrapperPass>();
01917     AU.addRequired<DominatorTreeWrapperPass>();
01918     AU.addRequired<LoopInfoWrapperPass>();
01919     AU.addRequired<ScalarEvolutionWrapperPass>();
01920     AU.addRequired<TargetTransformInfoWrapperPass>();
01921     AU.addRequired<AAResultsWrapperPass>();
01922     AU.addRequired<LoopAccessAnalysis>();
01923     AU.addRequired<DemandedBits>();
01924     AU.addPreserved<LoopInfoWrapperPass>();
01925     AU.addPreserved<DominatorTreeWrapperPass>();
01926     AU.addPreserved<BasicAAWrapperPass>();
01927     AU.addPreserved<AAResultsWrapperPass>();
01928     AU.addPreserved<GlobalsAAWrapperPass>();
01929   }
01930 
01931 };
01932 
01933 } // end anonymous namespace
01934 
01935 //===----------------------------------------------------------------------===//
01936 // Implementation of LoopVectorizationLegality, InnerLoopVectorizer and
01937 // LoopVectorizationCostModel.
01938 //===----------------------------------------------------------------------===//
01939 
01940 Value *InnerLoopVectorizer::getBroadcastInstrs(Value *V) {
01941   // We need to place the broadcast of invariant variables outside the loop.
01942   Instruction *Instr = dyn_cast<Instruction>(V);
01943   bool NewInstr =
01944       (Instr && std::find(LoopVectorBody.begin(), LoopVectorBody.end(),
01945                           Instr->getParent()) != LoopVectorBody.end());
01946   bool Invariant = OrigLoop->isLoopInvariant(V) && !NewInstr;
01947 
01948   // Place the code for broadcasting invariant variables in the new preheader.
01949   IRBuilder<>::InsertPointGuard Guard(Builder);
01950   if (Invariant)
01951     Builder.SetInsertPoint(LoopVectorPreHeader->getTerminator());
01952 
01953   // Broadcast the scalar into all locations in the vector.
01954   Value *Shuf = Builder.CreateVectorSplat(VF, V, "broadcast");
01955 
01956   return Shuf;
01957 }
01958 
01959 Value *InnerLoopVectorizer::getStepVector(Value *Val, int StartIdx,
01960                                           Value *Step) {
01961   assert(Val->getType()->isVectorTy() && "Must be a vector");
01962   assert(Val->getType()->getScalarType()->isIntegerTy() &&
01963          "Elem must be an integer");
01964   assert(Step->getType() == Val->getType()->getScalarType() &&
01965          "Step has wrong type");
01966   // Create the types.
01967   Type *ITy = Val->getType()->getScalarType();
01968   VectorType *Ty = cast<VectorType>(Val->getType());
01969   int VLen = Ty->getNumElements();
01970   SmallVector<Constant*, 8> Indices;
01971 
01972   // Create a vector of consecutive numbers from zero to VF.
01973   for (int i = 0; i < VLen; ++i)
01974     Indices.push_back(ConstantInt::get(ITy, StartIdx + i));
01975 
01976   // Add the consecutive indices to the vector value.
01977   Constant *Cv = ConstantVector::get(Indices);
01978   assert(Cv->getType() == Val->getType() && "Invalid consecutive vec");
01979   Step = Builder.CreateVectorSplat(VLen, Step);
01980   assert(Step->getType() == Val->getType() && "Invalid step vec");
01981   // FIXME: The newly created binary instructions should contain nsw/nuw flags,
01982   // which can be found from the original scalar operations.
01983   Step = Builder.CreateMul(Cv, Step);
01984   return Builder.CreateAdd(Val, Step, "induction");
01985 }
01986 
01987 int LoopVectorizationLegality::isConsecutivePtr(Value *Ptr) {
01988   assert(Ptr->getType()->isPointerTy() && "Unexpected non-ptr");
01989   auto *SE = PSE.getSE();
01990   // Make sure that the pointer does not point to structs.
01991   if (Ptr->getType()->getPointerElementType()->isAggregateType())
01992     return 0;
01993 
01994   // If this value is a pointer induction variable we know it is consecutive.
01995   PHINode *Phi = dyn_cast_or_null<PHINode>(Ptr);
01996   if (Phi && Inductions.count(Phi)) {
01997     InductionDescriptor II = Inductions[Phi];
01998     return II.getConsecutiveDirection();
01999   }
02000 
02001   GetElementPtrInst *Gep = getGEPInstruction(Ptr);
02002   if (!Gep)
02003     return 0;
02004 
02005   unsigned NumOperands = Gep->getNumOperands();
02006   Value *GpPtr = Gep->getPointerOperand();
02007   // If this GEP value is a consecutive pointer induction variable and all of
02008   // the indices are constant then we know it is consecutive. We can
02009   Phi = dyn_cast<PHINode>(GpPtr);
02010   if (Phi && Inductions.count(Phi)) {
02011 
02012     // Make sure that the pointer does not point to structs.
02013     PointerType *GepPtrType = cast<PointerType>(GpPtr->getType());
02014     if (GepPtrType->getElementType()->isAggregateType())
02015       return 0;
02016 
02017     // Make sure that all of the index operands are loop invariant.
02018     for (unsigned i = 1; i < NumOperands; ++i)
02019       if (!SE->isLoopInvariant(PSE.getSCEV(Gep->getOperand(i)), TheLoop))
02020         return 0;
02021 
02022     InductionDescriptor II = Inductions[Phi];
02023     return II.getConsecutiveDirection();
02024   }
02025 
02026   unsigned InductionOperand = getGEPInductionOperand(Gep);
02027 
02028   // Check that all of the gep indices are uniform except for our induction
02029   // operand.
02030   for (unsigned i = 0; i != NumOperands; ++i)
02031     if (i != InductionOperand &&
02032         !SE->isLoopInvariant(PSE.getSCEV(Gep->getOperand(i)), TheLoop))
02033       return 0;
02034 
02035   // We can emit wide load/stores only if the last non-zero index is the
02036   // induction variable.
02037   const SCEV *Last = nullptr;
02038   if (!Strides.count(Gep))
02039     Last = PSE.getSCEV(Gep->getOperand(InductionOperand));
02040   else {
02041     // Because of the multiplication by a stride we can have a s/zext cast.
02042     // We are going to replace this stride by 1 so the cast is safe to ignore.
02043     //
02044     //  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
02045     //  %0 = trunc i64 %indvars.iv to i32
02046     //  %mul = mul i32 %0, %Stride1
02047     //  %idxprom = zext i32 %mul to i64  << Safe cast.
02048     //  %arrayidx = getelementptr inbounds i32* %B, i64 %idxprom
02049     //
02050     Last = replaceSymbolicStrideSCEV(PSE, Strides,
02051                                      Gep->getOperand(InductionOperand), Gep);
02052     if (const SCEVCastExpr *C = dyn_cast<SCEVCastExpr>(Last))
02053       Last =
02054           (C->getSCEVType() == scSignExtend || C->getSCEVType() == scZeroExtend)
02055               ? C->getOperand()
02056               : Last;
02057   }
02058   if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Last)) {
02059     const SCEV *Step = AR->getStepRecurrence(*SE);
02060 
02061     // The memory is consecutive because the last index is consecutive
02062     // and all other indices are loop invariant.
02063     if (Step->isOne())
02064       return 1;
02065     if (Step->isAllOnesValue())
02066       return -1;
02067   }
02068 
02069   return 0;
02070 }
02071 
02072 bool LoopVectorizationLegality::isUniform(Value *V) {
02073   return LAI->isUniform(V);
02074 }
02075 
02076 InnerLoopVectorizer::VectorParts&
02077 InnerLoopVectorizer::getVectorValue(Value *V) {
02078   assert(V != Induction && "The new induction variable should not be used.");
02079   assert(!V->getType()->isVectorTy() && "Can't widen a vector");
02080 
02081   // If we have a stride that is replaced by one, do it here.
02082   if (Legal->hasStride(V))
02083     V = ConstantInt::get(V->getType(), 1);
02084 
02085   // If we have this scalar in the map, return it.
02086   if (WidenMap.has(V))
02087     return WidenMap.get(V);
02088 
02089   // If this scalar is unknown, assume that it is a constant or that it is
02090   // loop invariant. Broadcast V and save the value for future uses.
02091   Value *B = getBroadcastInstrs(V);
02092   return WidenMap.splat(V, B);
02093 }
02094 
02095 Value *InnerLoopVectorizer::reverseVector(Value *Vec) {
02096   assert(Vec->getType()->isVectorTy() && "Invalid type");
02097   SmallVector<Constant*, 8> ShuffleMask;
02098   for (unsigned i = 0; i < VF; ++i)
02099     ShuffleMask.push_back(Builder.getInt32(VF - i - 1));
02100 
02101   return Builder.CreateShuffleVector(Vec, UndefValue::get(Vec->getType()),
02102                                      ConstantVector::get(ShuffleMask),
02103                                      "reverse");
02104 }
02105 
02106 // Get a mask to interleave \p NumVec vectors into a wide vector.
02107 // I.e.  <0, VF, VF*2, ..., VF*(NumVec-1), 1, VF+1, VF*2+1, ...>
02108 // E.g. For 2 interleaved vectors, if VF is 4, the mask is:
02109 //      <0, 4, 1, 5, 2, 6, 3, 7>
02110 static Constant *getInterleavedMask(IRBuilder<> &Builder, unsigned VF,
02111                                     unsigned NumVec) {
02112   SmallVector<Constant *, 16> Mask;
02113   for (unsigned i = 0; i < VF; i++)
02114     for (unsigned j = 0; j < NumVec; j++)
02115       Mask.push_back(Builder.getInt32(j * VF + i));
02116 
02117   return ConstantVector::get(Mask);
02118 }
02119 
02120 // Get the strided mask starting from index \p Start.
02121 // I.e.  <Start, Start + Stride, ..., Start + Stride*(VF-1)>
02122 static Constant *getStridedMask(IRBuilder<> &Builder, unsigned Start,
02123                                 unsigned Stride, unsigned VF) {
02124   SmallVector<Constant *, 16> Mask;
02125   for (unsigned i = 0; i < VF; i++)
02126     Mask.push_back(Builder.getInt32(Start + i * Stride));
02127 
02128   return ConstantVector::get(Mask);
02129 }
02130 
02131 // Get a mask of two parts: The first part consists of sequential integers
02132 // starting from 0, The second part consists of UNDEFs.
02133 // I.e. <0, 1, 2, ..., NumInt - 1, undef, ..., undef>
02134 static Constant *getSequentialMask(IRBuilder<> &Builder, unsigned NumInt,
02135                                    unsigned NumUndef) {
02136   SmallVector<Constant *, 16> Mask;
02137   for (unsigned i = 0; i < NumInt; i++)
02138     Mask.push_back(Builder.getInt32(i));
02139 
02140   Constant *Undef = UndefValue::get(Builder.getInt32Ty());
02141   for (unsigned i = 0; i < NumUndef; i++)
02142     Mask.push_back(Undef);
02143 
02144   return ConstantVector::get(Mask);
02145 }
02146 
02147 // Concatenate two vectors with the same element type. The 2nd vector should
02148 // not have more elements than the 1st vector. If the 2nd vector has less
02149 // elements, extend it with UNDEFs.
02150 static Value *ConcatenateTwoVectors(IRBuilder<> &Builder, Value *V1,
02151                                     Value *V2) {
02152   VectorType *VecTy1 = dyn_cast<VectorType>(V1->getType());
02153   VectorType *VecTy2 = dyn_cast<VectorType>(V2->getType());
02154   assert(VecTy1 && VecTy2 &&
02155          VecTy1->getScalarType() == VecTy2->getScalarType() &&
02156          "Expect two vectors with the same element type");
02157 
02158   unsigned NumElts1 = VecTy1->getNumElements();
02159   unsigned NumElts2 = VecTy2->getNumElements();
02160   assert(NumElts1 >= NumElts2 && "Unexpect the first vector has less elements");
02161 
02162   if (NumElts1 > NumElts2) {
02163     // Extend with UNDEFs.
02164     Constant *ExtMask =
02165         getSequentialMask(Builder, NumElts2, NumElts1 - NumElts2);
02166     V2 = Builder.CreateShuffleVector(V2, UndefValue::get(VecTy2), ExtMask);
02167   }
02168 
02169   Constant *Mask = getSequentialMask(Builder, NumElts1 + NumElts2, 0);
02170   return Builder.CreateShuffleVector(V1, V2, Mask);
02171 }
02172 
02173 // Concatenate vectors in the given list. All vectors have the same type.
02174 static Value *ConcatenateVectors(IRBuilder<> &Builder,
02175                                  ArrayRef<Value *> InputList) {
02176   unsigned NumVec = InputList.size();
02177   assert(NumVec > 1 && "Should be at least two vectors");
02178 
02179   SmallVector<Value *, 8> ResList;
02180   ResList.append(InputList.begin(), InputList.end());
02181   do {
02182     SmallVector<Value *, 8> TmpList;
02183     for (unsigned i = 0; i < NumVec - 1; i += 2) {
02184       Value *V0 = ResList[i], *V1 = ResList[i + 1];
02185       assert((V0->getType() == V1->getType() || i == NumVec - 2) &&
02186              "Only the last vector may have a different type");
02187 
02188       TmpList.push_back(ConcatenateTwoVectors(Builder, V0, V1));
02189     }
02190 
02191     // Push the last vector if the total number of vectors is odd.
02192     if (NumVec % 2 != 0)
02193       TmpList.push_back(ResList[NumVec - 1]);
02194 
02195     ResList = TmpList;
02196     NumVec = ResList.size();
02197   } while (NumVec > 1);
02198 
02199   return ResList[0];
02200 }
02201 
02202 // Try to vectorize the interleave group that \p Instr belongs to.
02203 //
02204 // E.g. Translate following interleaved load group (factor = 3):
02205 //   for (i = 0; i < N; i+=3) {
02206 //     R = Pic[i];             // Member of index 0
02207 //     G = Pic[i+1];           // Member of index 1
02208 //     B = Pic[i+2];           // Member of index 2
02209 //     ... // do something to R, G, B
02210 //   }
02211 // To:
02212 //   %wide.vec = load <12 x i32>                       ; Read 4 tuples of R,G,B
02213 //   %R.vec = shuffle %wide.vec, undef, <0, 3, 6, 9>   ; R elements
02214 //   %G.vec = shuffle %wide.vec, undef, <1, 4, 7, 10>  ; G elements
02215 //   %B.vec = shuffle %wide.vec, undef, <2, 5, 8, 11>  ; B elements
02216 //
02217 // Or translate following interleaved store group (factor = 3):
02218 //   for (i = 0; i < N; i+=3) {
02219 //     ... do something to R, G, B
02220 //     Pic[i]   = R;           // Member of index 0
02221 //     Pic[i+1] = G;           // Member of index 1
02222 //     Pic[i+2] = B;           // Member of index 2
02223 //   }
02224 // To:
02225 //   %R_G.vec = shuffle %R.vec, %G.vec, <0, 1, 2, ..., 7>
02226 //   %B_U.vec = shuffle %B.vec, undef, <0, 1, 2, 3, u, u, u, u>
02227 //   %interleaved.vec = shuffle %R_G.vec, %B_U.vec,
02228 //        <0, 4, 8, 1, 5, 9, 2, 6, 10, 3, 7, 11>    ; Interleave R,G,B elements
02229 //   store <12 x i32> %interleaved.vec              ; Write 4 tuples of R,G,B
02230 void InnerLoopVectorizer::vectorizeInterleaveGroup(Instruction *Instr) {
02231   const InterleaveGroup *Group = Legal->getInterleavedAccessGroup(Instr);
02232   assert(Group && "Fail to get an interleaved access group.");
02233 
02234   // Skip if current instruction is not the insert position.
02235   if (Instr != Group->getInsertPos())
02236     return;
02237 
02238   LoadInst *LI = dyn_cast<LoadInst>(Instr);
02239   StoreInst *SI = dyn_cast<StoreInst>(Instr);
02240   Value *Ptr = LI ? LI->getPointerOperand() : SI->getPointerOperand();
02241 
02242   // Prepare for the vector type of the interleaved load/store.
02243   Type *ScalarTy = LI ? LI->getType() : SI->getValueOperand()->getType();
02244   unsigned InterleaveFactor = Group->getFactor();
02245   Type *VecTy = VectorType::get(ScalarTy, InterleaveFactor * VF);
02246   Type *PtrTy = VecTy->getPointerTo(Ptr->getType()->getPointerAddressSpace());
02247 
02248   // Prepare for the new pointers.
02249   setDebugLocFromInst(Builder, Ptr);
02250   VectorParts &PtrParts = getVectorValue(Ptr);
02251   SmallVector<Value *, 2> NewPtrs;
02252   unsigned Index = Group->getIndex(Instr);
02253   for (unsigned Part = 0; Part < UF; Part++) {
02254     // Extract the pointer for current instruction from the pointer vector. A
02255     // reverse access uses the pointer in the last lane.
02256     Value *NewPtr = Builder.CreateExtractElement(
02257         PtrParts[Part],
02258         Group->isReverse() ? Builder.getInt32(VF - 1) : Builder.getInt32(0));
02259 
02260     // Notice current instruction could be any index. Need to adjust the address
02261     // to the member of index 0.
02262     //
02263     // E.g.  a = A[i+1];     // Member of index 1 (Current instruction)
02264     //       b = A[i];       // Member of index 0
02265     // Current pointer is pointed to A[i+1], adjust it to A[i].
02266     //
02267     // E.g.  A[i+1] = a;     // Member of index 1
02268     //       A[i]   = b;     // Member of index 0
02269     //       A[i+2] = c;     // Member of index 2 (Current instruction)
02270     // Current pointer is pointed to A[i+2], adjust it to A[i].
02271     NewPtr = Builder.CreateGEP(NewPtr, Builder.getInt32(-Index));
02272 
02273     // Cast to the vector pointer type.
02274     NewPtrs.push_back(Builder.CreateBitCast(NewPtr, PtrTy));
02275   }
02276 
02277   setDebugLocFromInst(Builder, Instr);
02278   Value *UndefVec = UndefValue::get(VecTy);
02279 
02280   // Vectorize the interleaved load group.
02281   if (LI) {
02282     for (unsigned Part = 0; Part < UF; Part++) {
02283       Instruction *NewLoadInstr = Builder.CreateAlignedLoad(
02284           NewPtrs[Part], Group->getAlignment(), "wide.vec");
02285 
02286       for (unsigned i = 0; i < InterleaveFactor; i++) {
02287         Instruction *Member = Group->getMember(i);
02288 
02289         // Skip the gaps in the group.
02290         if (!Member)
02291           continue;
02292 
02293         Constant *StrideMask = getStridedMask(Builder, i, InterleaveFactor, VF);
02294         Value *StridedVec = Builder.CreateShuffleVector(
02295             NewLoadInstr, UndefVec, StrideMask, "strided.vec");
02296 
02297         // If this member has different type, cast the result type.
02298         if (Member->getType() != ScalarTy) {
02299           VectorType *OtherVTy = VectorType::get(Member->getType(), VF);
02300           StridedVec = Builder.CreateBitOrPointerCast(StridedVec, OtherVTy);
02301         }
02302 
02303         VectorParts &Entry = WidenMap.get(Member);
02304         Entry[Part] =
02305             Group->isReverse() ? reverseVector(StridedVec) : StridedVec;
02306       }
02307 
02308       propagateMetadata(NewLoadInstr, Instr);
02309     }
02310     return;
02311   }
02312 
02313   // The sub vector type for current instruction.
02314   VectorType *SubVT = VectorType::get(ScalarTy, VF);
02315 
02316   // Vectorize the interleaved store group.
02317   for (unsigned Part = 0; Part < UF; Part++) {
02318     // Collect the stored vector from each member.
02319     SmallVector<Value *, 4> StoredVecs;
02320     for (unsigned i = 0; i < InterleaveFactor; i++) {
02321       // Interleaved store group doesn't allow a gap, so each index has a member
02322       Instruction *Member = Group->getMember(i);
02323       assert(Member && "Fail to get a member from an interleaved store group");
02324 
02325       Value *StoredVec =
02326           getVectorValue(dyn_cast<StoreInst>(Member)->getValueOperand())[Part];
02327       if (Group->isReverse())
02328         StoredVec = reverseVector(StoredVec);
02329 
02330       // If this member has different type, cast it to an unified type.
02331       if (StoredVec->getType() != SubVT)
02332         StoredVec = Builder.CreateBitOrPointerCast(StoredVec, SubVT);
02333 
02334       StoredVecs.push_back(StoredVec);
02335     }
02336 
02337     // Concatenate all vectors into a wide vector.
02338     Value *WideVec = ConcatenateVectors(Builder, StoredVecs);
02339 
02340     // Interleave the elements in the wide vector.
02341     Constant *IMask = getInterleavedMask(Builder, VF, InterleaveFactor);
02342     Value *IVec = Builder.CreateShuffleVector(WideVec, UndefVec, IMask,
02343                                               "interleaved.vec");
02344 
02345     Instruction *NewStoreInstr =
02346         Builder.CreateAlignedStore(IVec, NewPtrs[Part], Group->getAlignment());
02347     propagateMetadata(NewStoreInstr, Instr);
02348   }
02349 }
02350 
02351 void InnerLoopVectorizer::vectorizeMemoryInstruction(Instruction *Instr) {
02352   // Attempt to issue a wide load.
02353   LoadInst *LI = dyn_cast<LoadInst>(Instr);
02354   StoreInst *SI = dyn_cast<StoreInst>(Instr);
02355 
02356   assert((LI || SI) && "Invalid Load/Store instruction");
02357 
02358   // Try to vectorize the interleave group if this access is interleaved.
02359   if (Legal->isAccessInterleaved(Instr))
02360     return vectorizeInterleaveGroup(Instr);
02361 
02362   Type *ScalarDataTy = LI ? LI->getType() : SI->getValueOperand()->getType();
02363   Type *DataTy = VectorType::get(ScalarDataTy, VF);
02364   Value *Ptr = LI ? LI->getPointerOperand() : SI->getPointerOperand();
02365   unsigned Alignment = LI ? LI->getAlignment() : SI->getAlignment();
02366   // An alignment of 0 means target abi alignment. We need to use the scalar's
02367   // target abi alignment in such a case.
02368   const DataLayout &DL = Instr->getModule()->getDataLayout();
02369   if (!Alignment)
02370     Alignment = DL.getABITypeAlignment(ScalarDataTy);
02371   unsigned AddressSpace = Ptr->getType()->getPointerAddressSpace();
02372   unsigned ScalarAllocatedSize = DL.getTypeAllocSize(ScalarDataTy);
02373   unsigned VectorElementSize = DL.getTypeStoreSize(DataTy) / VF;
02374 
02375   if (SI && Legal->blockNeedsPredication(SI->getParent()) &&
02376       !Legal->isMaskRequired(SI))
02377     return scalarizeInstruction(Instr, true);
02378 
02379   if (ScalarAllocatedSize != VectorElementSize)
02380     return scalarizeInstruction(Instr);
02381 
02382   // If the pointer is loop invariant or if it is non-consecutive,
02383   // scalarize the load.
02384   int ConsecutiveStride = Legal->isConsecutivePtr(Ptr);
02385   bool Reverse = ConsecutiveStride < 0;
02386   bool UniformLoad = LI && Legal->isUniform(Ptr);
02387   if (!ConsecutiveStride || UniformLoad)
02388     return scalarizeInstruction(Instr);
02389 
02390   Constant *Zero = Builder.getInt32(0);
02391   VectorParts &Entry = WidenMap.get(Instr);
02392 
02393   // Handle consecutive loads/stores.
02394   GetElementPtrInst *Gep = getGEPInstruction(Ptr);
02395   if (Gep && Legal->isInductionVariable(Gep->getPointerOperand())) {
02396     setDebugLocFromInst(Builder, Gep);
02397     Value *PtrOperand = Gep->getPointerOperand();
02398     Value *FirstBasePtr = getVectorValue(PtrOperand)[0];
02399     FirstBasePtr = Builder.CreateExtractElement(FirstBasePtr, Zero);
02400 
02401     // Create the new GEP with the new induction variable.
02402     GetElementPtrInst *Gep2 = cast<GetElementPtrInst>(Gep->clone());
02403     Gep2->setOperand(0, FirstBasePtr);
02404     Gep2->setName("gep.indvar.base");
02405     Ptr = Builder.Insert(Gep2);
02406   } else if (Gep) {
02407     setDebugLocFromInst(Builder, Gep);
02408     assert(PSE.getSE()->isLoopInvariant(PSE.getSCEV(Gep->getPointerOperand()),
02409                                         OrigLoop) &&
02410            "Base ptr must be invariant");
02411 
02412     // The last index does not have to be the induction. It can be
02413     // consecutive and be a function of the index. For example A[I+1];
02414     unsigned NumOperands = Gep->getNumOperands();
02415     unsigned InductionOperand = getGEPInductionOperand(Gep);
02416     // Create the new GEP with the new induction variable.
02417     GetElementPtrInst *Gep2 = cast<GetElementPtrInst>(Gep->clone());
02418 
02419     for (unsigned i = 0; i < NumOperands; ++i) {
02420       Value *GepOperand = Gep->getOperand(i);
02421       Instruction *GepOperandInst = dyn_cast<Instruction>(GepOperand);
02422 
02423       // Update last index or loop invariant instruction anchored in loop.
02424       if (i == InductionOperand ||
02425           (GepOperandInst && OrigLoop->contains(GepOperandInst))) {
02426         assert((i == InductionOperand ||
02427                 PSE.getSE()->isLoopInvariant(PSE.getSCEV(GepOperandInst),
02428                                              OrigLoop)) &&
02429                "Must be last index or loop invariant");
02430 
02431         VectorParts &GEPParts = getVectorValue(GepOperand);
02432         Value *Index = GEPParts[0];
02433         Index = Builder.CreateExtractElement(Index, Zero);
02434         Gep2->setOperand(i, Index);
02435         Gep2->setName("gep.indvar.idx");
02436       }
02437     }
02438     Ptr = Builder.Insert(Gep2);
02439   } else {
02440     // Use the induction element ptr.
02441     assert(isa<PHINode>(Ptr) && "Invalid induction ptr");
02442     setDebugLocFromInst(Builder, Ptr);
02443     VectorParts &PtrVal = getVectorValue(Ptr);
02444     Ptr = Builder.CreateExtractElement(PtrVal[0], Zero);
02445   }
02446 
02447   VectorParts Mask = createBlockInMask(Instr->getParent());
02448   // Handle Stores:
02449   if (SI) {
02450     assert(!Legal->isUniform(SI->getPointerOperand()) &&
02451            "We do not allow storing to uniform addresses");
02452     setDebugLocFromInst(Builder, SI);
02453     // We don't want to update the value in the map as it might be used in
02454     // another expression. So don't use a reference type for "StoredVal".
02455     VectorParts StoredVal = getVectorValue(SI->getValueOperand());
02456 
02457     for (unsigned Part = 0; Part < UF; ++Part) {
02458       // Calculate the pointer for the specific unroll-part.
02459       Value *PartPtr =
02460           Builder.CreateGEP(nullptr, Ptr, Builder.getInt32(Part * VF));
02461 
02462       if (Reverse) {
02463         // If we store to reverse consecutive memory locations, then we need
02464         // to reverse the order of elements in the stored value.
02465         StoredVal[Part] = reverseVector(StoredVal[Part]);
02466         // If the address is consecutive but reversed, then the
02467         // wide store needs to start at the last vector element.
02468         PartPtr = Builder.CreateGEP(nullptr, Ptr, Builder.getInt32(-Part * VF));
02469         PartPtr = Builder.CreateGEP(nullptr, PartPtr, Builder.getInt32(1 - VF));
02470         Mask[Part] = reverseVector(Mask[Part]);
02471       }
02472 
02473       Value *VecPtr = Builder.CreateBitCast(PartPtr,
02474                                             DataTy->getPointerTo(AddressSpace));
02475 
02476       Instruction *NewSI;
02477       if (Legal->isMaskRequired(SI))
02478         NewSI = Builder.CreateMaskedStore(StoredVal[Part], VecPtr, Alignment,
02479                                           Mask[Part]);
02480       else 
02481         NewSI = Builder.CreateAlignedStore(StoredVal[Part], VecPtr, Alignment);
02482       propagateMetadata(NewSI, SI);
02483     }
02484     return;
02485   }
02486 
02487   // Handle loads.
02488   assert(LI && "Must have a load instruction");
02489   setDebugLocFromInst(Builder, LI);
02490   for (unsigned Part = 0; Part < UF; ++Part) {
02491     // Calculate the pointer for the specific unroll-part.
02492     Value *PartPtr =
02493         Builder.CreateGEP(nullptr, Ptr, Builder.getInt32(Part * VF));
02494 
02495     if (Reverse) {
02496       // If the address is consecutive but reversed, then the
02497       // wide load needs to start at the last vector element.
02498       PartPtr = Builder.CreateGEP(nullptr, Ptr, Builder.getInt32(-Part * VF));
02499       PartPtr = Builder.CreateGEP(nullptr, PartPtr, Builder.getInt32(1 - VF));
02500       Mask[Part] = reverseVector(Mask[Part]);
02501     }
02502 
02503     Instruction* NewLI;
02504     Value *VecPtr = Builder.CreateBitCast(PartPtr,
02505                                           DataTy->getPointerTo(AddressSpace));
02506     if (Legal->isMaskRequired(LI))
02507       NewLI = Builder.CreateMaskedLoad(VecPtr, Alignment, Mask[Part],
02508                                        UndefValue::get(DataTy),
02509                                        "wide.masked.load");
02510     else
02511       NewLI = Builder.CreateAlignedLoad(VecPtr, Alignment, "wide.load");
02512     propagateMetadata(NewLI, LI);
02513     Entry[Part] = Reverse ? reverseVector(NewLI) :  NewLI;
02514   }
02515 }
02516 
02517 void InnerLoopVectorizer::scalarizeInstruction(Instruction *Instr,
02518                                                bool IfPredicateStore) {
02519   assert(!Instr->getType()->isAggregateType() && "Can't handle vectors");
02520   // Holds vector parameters or scalars, in case of uniform vals.
02521   SmallVector<VectorParts, 4> Params;
02522 
02523   setDebugLocFromInst(Builder, Instr);
02524 
02525   // Find all of the vectorized parameters.
02526   for (unsigned op = 0, e = Instr->getNumOperands(); op != e; ++op) {
02527     Value *SrcOp = Instr->getOperand(op);
02528 
02529     // If we are accessing the old induction variable, use the new one.
02530     if (SrcOp == OldInduction) {
02531       Params.push_back(getVectorValue(SrcOp));
02532       continue;
02533     }
02534 
02535     // Try using previously calculated values.
02536     Instruction *SrcInst = dyn_cast<Instruction>(SrcOp);
02537 
02538     // If the src is an instruction that appeared earlier in the basic block,
02539     // then it should already be vectorized.
02540     if (SrcInst && OrigLoop->contains(SrcInst)) {
02541       assert(WidenMap.has(SrcInst) && "Source operand is unavailable");
02542       // The parameter is a vector value from earlier.
02543       Params.push_back(WidenMap.get(SrcInst));
02544     } else {
02545       // The parameter is a scalar from outside the loop. Maybe even a constant.
02546       VectorParts Scalars;
02547       Scalars.append(UF, SrcOp);
02548       Params.push_back(Scalars);
02549     }
02550   }
02551 
02552   assert(Params.size() == Instr->getNumOperands() &&
02553          "Invalid number of operands");
02554 
02555   // Does this instruction return a value ?
02556   bool IsVoidRetTy = Instr->getType()->isVoidTy();
02557 
02558   Value *UndefVec = IsVoidRetTy ? nullptr :
02559     UndefValue::get(VectorType::get(Instr->getType(), VF));
02560   // Create a new entry in the WidenMap and initialize it to Undef or Null.
02561   VectorParts &VecResults = WidenMap.splat(Instr, UndefVec);
02562 
02563   VectorParts Cond;
02564   if (IfPredicateStore) {
02565     assert(Instr->getParent()->getSinglePredecessor() &&
02566            "Only support single predecessor blocks");
02567     Cond = createEdgeMask(Instr->getParent()->getSinglePredecessor(),
02568                           Instr->getParent());
02569   }
02570 
02571   // For each vector unroll 'part':
02572   for (unsigned Part = 0; Part < UF; ++Part) {
02573     // For each scalar that we create:
02574     for (unsigned Width = 0; Width < VF; ++Width) {
02575 
02576       // Start if-block.
02577       Value *Cmp = nullptr;
02578       if (IfPredicateStore) {
02579         Cmp = Builder.CreateExtractElement(Cond[Part], Builder.getInt32(Width));
02580         Cmp = Builder.CreateICmp(ICmpInst::ICMP_EQ, Cmp,
02581                                  ConstantInt::get(Cmp->getType(), 1));
02582       }
02583 
02584       Instruction *Cloned = Instr->clone();
02585       if (!IsVoidRetTy)
02586         Cloned->setName(Instr->getName() + ".cloned");
02587       // Replace the operands of the cloned instructions with extracted scalars.
02588       for (unsigned op = 0, e = Instr->getNumOperands(); op != e; ++op) {
02589         Value *Op = Params[op][Part];
02590         // Param is a vector. Need to extract the right lane.
02591         if (Op->getType()->isVectorTy())
02592           Op = Builder.CreateExtractElement(Op, Builder.getInt32(Width));
02593         Cloned->setOperand(op, Op);
02594       }
02595 
02596       // Place the cloned scalar in the new loop.
02597       Builder.Insert(Cloned);
02598 
02599       // If the original scalar returns a value we need to place it in a vector
02600       // so that future users will be able to use it.
02601       if (!IsVoidRetTy)
02602         VecResults[Part] = Builder.CreateInsertElement(VecResults[Part], Cloned,
02603                                                        Builder.getInt32(Width));
02604       // End if-block.
02605       if (IfPredicateStore)
02606         PredicatedStores.push_back(std::make_pair(cast<StoreInst>(Cloned),
02607                                                   Cmp));
02608     }
02609   }
02610 }
02611 
02612 PHINode *InnerLoopVectorizer::createInductionVariable(Loop *L, Value *Start,
02613                                                       Value *End, Value *Step,
02614                                                       Instruction *DL) {
02615   BasicBlock *Header = L->getHeader();
02616   BasicBlock *Latch = L->getLoopLatch();
02617   // As we're just creating this loop, it's possible no latch exists
02618   // yet. If so, use the header as this will be a single block loop.
02619   if (!Latch)
02620     Latch = Header;
02621 
02622   IRBuilder<> Builder(&*Header->getFirstInsertionPt());
02623   setDebugLocFromInst(Builder, getDebugLocFromInstOrOperands(OldInduction));
02624   auto *Induction = Builder.CreatePHI(Start->getType(), 2, "index");
02625 
02626   Builder.SetInsertPoint(Latch->getTerminator());
02627   
02628   // Create i+1 and fill the PHINode.
02629   Value *Next = Builder.CreateAdd(Induction, Step, "index.next");
02630   Induction->addIncoming(Start, L->getLoopPreheader());
02631   Induction->addIncoming(Next, Latch);
02632   // Create the compare.
02633   Value *ICmp = Builder.CreateICmpEQ(Next, End);
02634   Builder.CreateCondBr(ICmp, L->getExitBlock(), Header);
02635   
02636   // Now we have two terminators. Remove the old one from the block.
02637   Latch->getTerminator()->eraseFromParent();
02638 
02639   return Induction;
02640 }
02641 
02642 Value *InnerLoopVectorizer::getOrCreateTripCount(Loop *L) {
02643   if (TripCount)
02644     return TripCount;
02645 
02646   IRBuilder<> Builder(L->getLoopPreheader()->getTerminator());
02647   // Find the loop boundaries.
02648   ScalarEvolution *SE = PSE.getSE();
02649   const SCEV *BackedgeTakenCount = SE->getBackedgeTakenCount(OrigLoop);
02650   assert(BackedgeTakenCount != SE->getCouldNotCompute() &&
02651          "Invalid loop count");
02652 
02653   Type *IdxTy = Legal->getWidestInductionType();
02654   
02655   // The exit count might have the type of i64 while the phi is i32. This can
02656   // happen if we have an induction variable that is sign extended before the
02657   // compare. The only way that we get a backedge taken count is that the
02658   // induction variable was signed and as such will not overflow. In such a case
02659   // truncation is legal.
02660   if (BackedgeTakenCount->getType()->getPrimitiveSizeInBits() >
02661       IdxTy->getPrimitiveSizeInBits())
02662     BackedgeTakenCount = SE->getTruncateOrNoop(BackedgeTakenCount, IdxTy);
02663   BackedgeTakenCount = SE->getNoopOrZeroExtend(BackedgeTakenCount, IdxTy);
02664   
02665   // Get the total trip count from the count by adding 1.
02666   const SCEV *ExitCount = SE->getAddExpr(
02667       BackedgeTakenCount, SE->getOne(BackedgeTakenCount->getType()));
02668 
02669   const DataLayout &DL = L->getHeader()->getModule()->getDataLayout();
02670 
02671   // Expand the trip count and place the new instructions in the preheader.
02672   // Notice that the pre-header does not change, only the loop body.
02673   SCEVExpander Exp(*SE, DL, "induction");
02674 
02675   // Count holds the overall loop count (N).
02676   TripCount = Exp.expandCodeFor(ExitCount, ExitCount->getType(),
02677                                 L->getLoopPreheader()->getTerminator());
02678 
02679   if (TripCount->getType()->isPointerTy())
02680     TripCount =
02681       CastInst::CreatePointerCast(TripCount, IdxTy,
02682                                   "exitcount.ptrcnt.to.int",
02683                                   L->getLoopPreheader()->getTerminator());
02684 
02685   return TripCount;
02686 }
02687 
02688 Value *InnerLoopVectorizer::getOrCreateVectorTripCount(Loop *L) {
02689   if (VectorTripCount)
02690     return VectorTripCount;
02691   
02692   Value *TC = getOrCreateTripCount(L);
02693   IRBuilder<> Builder(L->getLoopPreheader()->getTerminator());
02694   
02695   // Now we need to generate the expression for N - (N % VF), which is
02696   // the part that the vectorized body will execute.
02697   // The loop step is equal to the vectorization factor (num of SIMD elements)
02698   // times the unroll factor (num of SIMD instructions).
02699   Constant *Step = ConstantInt::get(TC->getType(), VF * UF);
02700   Value *R = Builder.CreateURem(TC, Step, "n.mod.vf");
02701   VectorTripCount = Builder.CreateSub(TC, R, "n.vec");
02702 
02703   return VectorTripCount;
02704 }
02705 
02706 void InnerLoopVectorizer::emitMinimumIterationCountCheck(Loop *L,
02707                                                          BasicBlock *Bypass) {
02708   Value *Count = getOrCreateTripCount(L);
02709   BasicBlock *BB = L->getLoopPreheader();
02710   IRBuilder<> Builder(BB->getTerminator());
02711 
02712   // Generate code to check that the loop's trip count that we computed by
02713   // adding one to the backedge-taken count will not overflow.
02714   Value *CheckMinIters =
02715     Builder.CreateICmpULT(Count,
02716                           ConstantInt::get(Count->getType(), VF * UF),
02717                           "min.iters.check");
02718   
02719   BasicBlock *NewBB = BB->splitBasicBlock(BB->getTerminator(),
02720                                           "min.iters.checked");
02721   if (L->getParentLoop())
02722     L->getParentLoop()->addBasicBlockToLoop(NewBB, *LI);
02723   ReplaceInstWithInst(BB->getTerminator(),
02724                       BranchInst::Create(Bypass, NewBB, CheckMinIters));
02725   LoopBypassBlocks.push_back(BB);
02726 }
02727 
02728 void InnerLoopVectorizer::emitVectorLoopEnteredCheck(Loop *L,
02729                                                      BasicBlock *Bypass) {
02730   Value *TC = getOrCreateVectorTripCount(L);
02731   BasicBlock *BB = L->getLoopPreheader();
02732   IRBuilder<> Builder(BB->getTerminator());
02733   
02734   // Now, compare the new count to zero. If it is zero skip the vector loop and
02735   // jump to the scalar loop.
02736   Value *Cmp = Builder.CreateICmpEQ(TC, Constant::getNullValue(TC->getType()),
02737                                     "cmp.zero");
02738 
02739   // Generate code to check that the loop's trip count that we computed by
02740   // adding one to the backedge-taken count will not overflow.
02741   BasicBlock *NewBB = BB->splitBasicBlock(BB->getTerminator(),
02742                                           "vector.ph");
02743   if (L->getParentLoop())
02744     L->getParentLoop()->addBasicBlockToLoop(NewBB, *LI);
02745   ReplaceInstWithInst(BB->getTerminator(),
02746                       BranchInst::Create(Bypass, NewBB, Cmp));
02747   LoopBypassBlocks.push_back(BB);
02748 }
02749 
02750 void InnerLoopVectorizer::emitSCEVChecks(Loop *L, BasicBlock *Bypass) {
02751   BasicBlock *BB = L->getLoopPreheader();
02752 
02753   // Generate the code to check that the SCEV assumptions that we made.
02754   // We want the new basic block to start at the first instruction in a
02755   // sequence of instructions that form a check.
02756   SCEVExpander Exp(*PSE.getSE(), Bypass->getModule()->getDataLayout(),
02757                    "scev.check");
02758   Value *SCEVCheck =
02759       Exp.expandCodeForPredicate(&PSE.getUnionPredicate(), BB->getTerminator());
02760 
02761   if (auto *C = dyn_cast<ConstantInt>(SCEVCheck))
02762     if (C->isZero())
02763       return;
02764 
02765   // Create a new block containing the stride check.
02766   BB->setName("vector.scevcheck");
02767   auto *NewBB = BB->splitBasicBlock(BB->getTerminator(), "vector.ph");
02768   if (L->getParentLoop())
02769     L->getParentLoop()->addBasicBlockToLoop(NewBB, *LI);
02770   ReplaceInstWithInst(BB->getTerminator(),
02771                       BranchInst::Create(Bypass, NewBB, SCEVCheck));
02772   LoopBypassBlocks.push_back(BB);
02773   AddedSafetyChecks = true;
02774 }
02775 
02776 void InnerLoopVectorizer::emitMemRuntimeChecks(Loop *L,
02777                                                BasicBlock *Bypass) {
02778   BasicBlock *BB = L->getLoopPreheader();
02779 
02780   // Generate the code that checks in runtime if arrays overlap. We put the
02781   // checks into a separate block to make the more common case of few elements
02782   // faster.
02783   Instruction *FirstCheckInst;
02784   Instruction *MemRuntimeCheck;
02785   std::tie(FirstCheckInst, MemRuntimeCheck) =
02786       Legal->getLAI()->addRuntimeChecks(BB->getTerminator());
02787   if (!MemRuntimeCheck)
02788     return;
02789 
02790   // Create a new block containing the memory check.
02791   BB->setName("vector.memcheck");
02792   auto *NewBB = BB->splitBasicBlock(BB->getTerminator(), "vector.ph");
02793   if (L->getParentLoop())
02794     L->getParentLoop()->addBasicBlockToLoop(NewBB, *LI);
02795   ReplaceInstWithInst(BB->getTerminator(),
02796                       BranchInst::Create(Bypass, NewBB, MemRuntimeCheck));
02797   LoopBypassBlocks.push_back(BB);
02798   AddedSafetyChecks = true;
02799 }
02800 
02801 
02802 void InnerLoopVectorizer::createEmptyLoop() {
02803   /*
02804    In this function we generate a new loop. The new loop will contain
02805    the vectorized instructions while the old loop will continue to run the
02806    scalar remainder.
02807 
02808        [ ] <-- loop iteration number check.
02809     /   |
02810    /    v
02811   |    [ ] <-- vector loop bypass (may consist of multiple blocks).
02812   |  /  |
02813   | /   v
02814   ||   [ ]     <-- vector pre header.
02815   |/    |
02816   |     v
02817   |    [  ] \
02818   |    [  ]_|   <-- vector loop.
02819   |     |
02820   |     v
02821   |   -[ ]   <--- middle-block.
02822   |  /  |
02823   | /   v
02824   -|- >[ ]     <--- new preheader.
02825    |    |
02826    |    v
02827    |   [ ] \
02828    |   [ ]_|   <-- old scalar loop to handle remainder.
02829     \   |
02830      \  v
02831       >[ ]     <-- exit block.
02832    ...
02833    */
02834 
02835   BasicBlock *OldBasicBlock = OrigLoop->getHeader();
02836   BasicBlock *VectorPH = OrigLoop->getLoopPreheader();
02837   BasicBlock *ExitBlock = OrigLoop->getExitBlock();
02838   assert(VectorPH && "Invalid loop structure");
02839   assert(ExitBlock && "Must have an exit block");
02840 
02841   // Some loops have a single integer induction variable, while other loops
02842   // don't. One example is c++ iterators that often have multiple pointer
02843   // induction variables. In the code below we also support a case where we
02844   // don't have a single induction variable.
02845   //
02846   // We try to obtain an induction variable from the original loop as hard
02847   // as possible. However if we don't find one that:
02848   //   - is an integer
02849   //   - counts from zero, stepping by one
02850   //   - is the size of the widest induction variable type
02851   // then we create a new one.
02852   OldInduction = Legal->getInduction();
02853   Type *IdxTy = Legal->getWidestInductionType();
02854 
02855   // Split the single block loop into the two loop structure described above.
02856   BasicBlock *VecBody =
02857       VectorPH->splitBasicBlock(VectorPH->getTerminator(), "vector.body");
02858   BasicBlock *MiddleBlock =
02859   VecBody->splitBasicBlock(VecBody->getTerminator(), "middle.block");
02860   BasicBlock *ScalarPH =
02861   MiddleBlock->splitBasicBlock(MiddleBlock->getTerminator(), "scalar.ph");
02862 
02863   // Create and register the new vector loop.
02864   Loop* Lp = new Loop();
02865   Loop *ParentLoop = OrigLoop->getParentLoop();
02866 
02867   // Insert the new loop into the loop nest and register the new basic blocks
02868   // before calling any utilities such as SCEV that require valid LoopInfo.
02869   if (ParentLoop) {
02870     ParentLoop->addChildLoop(Lp);
02871     ParentLoop->addBasicBlockToLoop(ScalarPH, *LI);
02872     ParentLoop->addBasicBlockToLoop(MiddleBlock, *LI);
02873   } else {
02874     LI->addTopLevelLoop(Lp);
02875   }
02876   Lp->addBasicBlockToLoop(VecBody, *LI);
02877 
02878   // Find the loop boundaries.
02879   Value *Count = getOrCreateTripCount(Lp);
02880 
02881   Value *StartIdx = ConstantInt::get(IdxTy, 0);
02882 
02883   // We need to test whether the backedge-taken count is uint##_max. Adding one
02884   // to it will cause overflow and an incorrect loop trip count in the vector
02885   // body. In case of overflow we want to directly jump to the scalar remainder
02886   // loop.
02887   emitMinimumIterationCountCheck(Lp, ScalarPH);
02888   // Now, compare the new count to zero. If it is zero skip the vector loop and
02889   // jump to the scalar loop.
02890   emitVectorLoopEnteredCheck(Lp, ScalarPH);
02891   // Generate the code to check any assumptions that we've made for SCEV
02892   // expressions.
02893   emitSCEVChecks(Lp, ScalarPH);
02894 
02895   // Generate the code that checks in runtime if arrays overlap. We put the
02896   // checks into a separate block to make the more common case of few elements
02897   // faster.
02898   emitMemRuntimeChecks(Lp, ScalarPH);
02899   
02900   // Generate the induction variable.
02901   // The loop step is equal to the vectorization factor (num of SIMD elements)
02902   // times the unroll factor (num of SIMD instructions).
02903   Value *CountRoundDown = getOrCreateVectorTripCount(Lp);
02904   Constant *Step = ConstantInt::get(IdxTy, VF * UF);
02905   Induction =
02906     createInductionVariable(Lp, StartIdx, CountRoundDown, Step,
02907                             getDebugLocFromInstOrOperands(OldInduction));
02908 
02909   // We are going to resume the execution of the scalar loop.
02910   // Go over all of the induction variables that we found and fix the
02911   // PHIs that are left in the scalar version of the loop.
02912   // The starting values of PHI nodes depend on the counter of the last
02913   // iteration in the vectorized loop.
02914   // If we come from a bypass edge then we need to start from the original
02915   // start value.
02916 
02917   // This variable saves the new starting index for the scalar loop. It is used
02918   // to test if there are any tail iterations left once the vector loop has
02919   // completed.
02920   LoopVectorizationLegality::InductionList::iterator I, E;
02921   LoopVectorizationLegality::InductionList *List = Legal->getInductionVars();
02922   for (I = List->begin(), E = List->end(); I != E; ++I) {
02923     PHINode *OrigPhi = I->first;
02924     InductionDescriptor II = I->second;
02925 
02926     // Create phi nodes to merge from the  backedge-taken check block.
02927     PHINode *BCResumeVal = PHINode::Create(OrigPhi->getType(), 3,
02928                                            "bc.resume.val",
02929                                            ScalarPH->getTerminator());
02930     Value *EndValue;
02931     if (OrigPhi == OldInduction) {
02932       // We know what the end value is.
02933       EndValue = CountRoundDown;
02934     } else {
02935       IRBuilder<> B(LoopBypassBlocks.back()->getTerminator());
02936       Value *CRD = B.CreateSExtOrTrunc(CountRoundDown,
02937                                        II.getStepValue()->getType(),
02938                                        "cast.crd");
02939       EndValue = II.transform(B, CRD);
02940       EndValue->setName("ind.end");
02941     }
02942 
02943     // The new PHI merges the original incoming value, in case of a bypass,
02944     // or the value at the end of the vectorized loop.
02945     BCResumeVal->addIncoming(EndValue, MiddleBlock);
02946 
02947     // Fix the scalar body counter (PHI node).
02948     unsigned BlockIdx = OrigPhi->getBasicBlockIndex(ScalarPH);
02949 
02950     // The old induction's phi node in the scalar body needs the truncated
02951     // value.
02952     for (unsigned I = 0, E = LoopBypassBlocks.size(); I != E; ++I)
02953       BCResumeVal->addIncoming(II.getStartValue(), LoopBypassBlocks[I]);
02954     OrigPhi->setIncomingValue(BlockIdx, BCResumeVal);
02955   }
02956 
02957   // Add a check in the middle block to see if we have completed
02958   // all of the iterations in the first vector loop.
02959   // If (N - N%VF) == N, then we *don't* need to run the remainder.
02960   Value *CmpN = CmpInst::Create(Instruction::ICmp, CmpInst::ICMP_EQ, Count,
02961                                 CountRoundDown, "cmp.n",
02962                                 MiddleBlock->getTerminator());
02963   ReplaceInstWithInst(MiddleBlock->getTerminator(),
02964                       BranchInst::Create(ExitBlock, ScalarPH, CmpN));
02965 
02966   // Get ready to start creating new instructions into the vectorized body.
02967   Builder.SetInsertPoint(&*VecBody->getFirstInsertionPt());
02968 
02969   // Save the state.
02970   LoopVectorPreHeader = Lp->getLoopPreheader();
02971   LoopScalarPreHeader = ScalarPH;
02972   LoopMiddleBlock = MiddleBlock;
02973   LoopExitBlock = ExitBlock;
02974   LoopVectorBody.push_back(VecBody);
02975   LoopScalarBody = OldBasicBlock;
02976 
02977   LoopVectorizeHints Hints(Lp, true);
02978   Hints.setAlreadyVectorized();
02979 }
02980 
02981 namespace {
02982 struct CSEDenseMapInfo {
02983   static bool canHandle(Instruction *I) {
02984     return isa<InsertElementInst>(I) || isa<ExtractElementInst>(I) ||
02985            isa<ShuffleVectorInst>(I) || isa<GetElementPtrInst>(I);
02986   }
02987   static inline Instruction *getEmptyKey() {
02988     return DenseMapInfo<Instruction *>::getEmptyKey();
02989   }
02990   static inline Instruction *getTombstoneKey() {
02991     return DenseMapInfo<Instruction *>::getTombstoneKey();
02992   }
02993   static unsigned getHashValue(Instruction *I) {
02994     assert(canHandle(I) && "Unknown instruction!");
02995     return hash_combine(I->getOpcode(), hash_combine_range(I->value_op_begin(),
02996                                                            I->value_op_end()));
02997   }
02998   static bool isEqual(Instruction *LHS, Instruction *RHS) {
02999     if (LHS == getEmptyKey() || RHS == getEmptyKey() ||
03000         LHS == getTombstoneKey() || RHS == getTombstoneKey())
03001       return LHS == RHS;
03002     return LHS->isIdenticalTo(RHS);
03003   }
03004 };
03005 }
03006 
03007 /// \brief Check whether this block is a predicated block.
03008 /// Due to if predication of stores we might create a sequence of "if(pred) a[i]
03009 /// = ...;  " blocks. We start with one vectorized basic block. For every
03010 /// conditional block we split this vectorized block. Therefore, every second
03011 /// block will be a predicated one.
03012 static bool isPredicatedBlock(unsigned BlockNum) {
03013   return BlockNum % 2;
03014 }
03015 
03016 ///\brief Perform cse of induction variable instructions.
03017 static void cse(SmallVector<BasicBlock *, 4> &BBs) {
03018   // Perform simple cse.
03019   SmallDenseMap<Instruction *, Instruction *, 4, CSEDenseMapInfo> CSEMap;
03020   for (unsigned i = 0, e = BBs.size(); i != e; ++i) {
03021     BasicBlock *BB = BBs[i];
03022     for (BasicBlock::iterator I = BB->begin(), E = BB->end(); I != E;) {
03023       Instruction *In = &*I++;
03024 
03025       if (!CSEDenseMapInfo::canHandle(In))
03026         continue;
03027 
03028       // Check if we can replace this instruction with any of the
03029       // visited instructions.
03030       if (Instruction *V = CSEMap.lookup(In)) {
03031         In->replaceAllUsesWith(V);
03032         In->eraseFromParent();
03033         continue;
03034       }
03035       // Ignore instructions in conditional blocks. We create "if (pred) a[i] =
03036       // ...;" blocks for predicated stores. Every second block is a predicated
03037       // block.
03038       if (isPredicatedBlock(i))
03039         continue;
03040 
03041       CSEMap[In] = In;
03042     }
03043   }
03044 }
03045 
03046 /// \brief Adds a 'fast' flag to floating point operations.
03047 static Value *addFastMathFlag(Value *V) {
03048   if (isa<FPMathOperator>(V)){
03049     FastMathFlags Flags;
03050     Flags.setUnsafeAlgebra();
03051     cast<Instruction>(V)->setFastMathFlags(Flags);
03052   }
03053   return V;
03054 }
03055 
03056 /// Estimate the overhead of scalarizing a value. Insert and Extract are set if
03057 /// the result needs to be inserted and/or extracted from vectors.
03058 static unsigned getScalarizationOverhead(Type *Ty, bool Insert, bool Extract,
03059                                          const TargetTransformInfo &TTI) {
03060   if (Ty->isVoidTy())
03061     return 0;
03062 
03063   assert(Ty->isVectorTy() && "Can only scalarize vectors");
03064   unsigned Cost = 0;
03065 
03066   for (int i = 0, e = Ty->getVectorNumElements(); i < e; ++i) {
03067     if (Insert)
03068       Cost += TTI.getVectorInstrCost(Instruction::InsertElement, Ty, i);
03069     if (Extract)
03070       Cost += TTI.getVectorInstrCost(Instruction::ExtractElement, Ty, i);
03071   }
03072 
03073   return Cost;
03074 }
03075 
03076 // Estimate cost of a call instruction CI if it were vectorized with factor VF.
03077 // Return the cost of the instruction, including scalarization overhead if it's
03078 // needed. The flag NeedToScalarize shows if the call needs to be scalarized -
03079 // i.e. either vector version isn't available, or is too expensive.
03080 static unsigned getVectorCallCost(CallInst *CI, unsigned VF,
03081                                   const TargetTransformInfo &TTI,
03082                                   const TargetLibraryInfo *TLI,
03083                                   bool &NeedToScalarize) {
03084   Function *F = CI->getCalledFunction();
03085   StringRef FnName = CI->getCalledFunction()->getName();
03086   Type *ScalarRetTy = CI->getType();
03087   SmallVector<Type *, 4> Tys, ScalarTys;
03088   for (auto &ArgOp : CI->arg_operands())
03089     ScalarTys.push_back(ArgOp->getType());
03090 
03091   // Estimate cost of scalarized vector call. The source operands are assumed
03092   // to be vectors, so we need to extract individual elements from there,
03093   // execute VF scalar calls, and then gather the result into the vector return
03094   // value.
03095   unsigned ScalarCallCost = TTI.getCallInstrCost(F, ScalarRetTy, ScalarTys);
03096   if (VF == 1)
03097     return ScalarCallCost;
03098 
03099   // Compute corresponding vector type for return value and arguments.
03100   Type *RetTy = ToVectorTy(ScalarRetTy, VF);
03101   for (unsigned i = 0, ie = ScalarTys.size(); i != ie; ++i)
03102     Tys.push_back(ToVectorTy(ScalarTys[i], VF));
03103 
03104   // Compute costs of unpacking argument values for the scalar calls and
03105   // packing the return values to a vector.
03106   unsigned ScalarizationCost =
03107       getScalarizationOverhead(RetTy, true, false, TTI);
03108   for (unsigned i = 0, ie = Tys.size(); i != ie; ++i)
03109     ScalarizationCost += getScalarizationOverhead(Tys[i], false, true, TTI);
03110 
03111   unsigned Cost = ScalarCallCost * VF + ScalarizationCost;
03112 
03113   // If we can't emit a vector call for this function, then the currently found
03114   // cost is the cost we need to return.
03115   NeedToScalarize = true;
03116   if (!TLI || !TLI->isFunctionVectorizable(FnName, VF) || CI->isNoBuiltin())
03117     return Cost;
03118 
03119   // If the corresponding vector cost is cheaper, return its cost.
03120   unsigned VectorCallCost = TTI.getCallInstrCost(nullptr, RetTy, Tys);
03121   if (VectorCallCost < Cost) {
03122     NeedToScalarize = false;
03123     return VectorCallCost;
03124   }
03125   return Cost;
03126 }
03127 
03128 // Estimate cost of an intrinsic call instruction CI if it were vectorized with
03129 // factor VF.  Return the cost of the instruction, including scalarization
03130 // overhead if it's needed.
03131 static unsigned getVectorIntrinsicCost(CallInst *CI, unsigned VF,
03132                                        const TargetTransformInfo &TTI,
03133                                        const TargetLibraryInfo *TLI) {
03134   Intrinsic::ID ID = getIntrinsicIDForCall(CI, TLI);
03135   assert(ID && "Expected intrinsic call!");
03136 
03137   Type *RetTy = ToVectorTy(CI->getType(), VF);
03138   SmallVector<Type *, 4> Tys;
03139   for (unsigned i = 0, ie = CI->getNumArgOperands(); i != ie; ++i)
03140     Tys.push_back(ToVectorTy(CI->getArgOperand(i)->getType(), VF));
03141 
03142   return TTI.getIntrinsicInstrCost(ID, RetTy, Tys);
03143 }
03144 
03145 static Type *smallestIntegerVectorType(Type *T1, Type *T2) {
03146   IntegerType *I1 = cast<IntegerType>(T1->getVectorElementType());
03147   IntegerType *I2 = cast<IntegerType>(T2->getVectorElementType());
03148   return I1->getBitWidth() < I2->getBitWidth() ? T1 : T2;
03149 }
03150 static Type *largestIntegerVectorType(Type *T1, Type *T2) {
03151   IntegerType *I1 = cast<IntegerType>(T1->getVectorElementType());
03152   IntegerType *I2 = cast<IntegerType>(T2->getVectorElementType());
03153   return I1->getBitWidth() > I2->getBitWidth() ? T1 : T2;
03154 }
03155 
03156 void InnerLoopVectorizer::truncateToMinimalBitwidths() {
03157   // For every instruction `I` in MinBWs, truncate the operands, create a
03158   // truncated version of `I` and reextend its result. InstCombine runs
03159   // later and will remove any ext/trunc pairs.
03160   //
03161   for (auto &KV : MinBWs) {
03162     VectorParts &Parts = WidenMap.get(KV.first);
03163     for (Value *&I : Parts) {
03164       if (I->use_empty())
03165         continue;
03166       Type *OriginalTy = I->getType();
03167       Type *ScalarTruncatedTy = IntegerType::get(OriginalTy->getContext(),
03168                                                  KV.second);
03169       Type *TruncatedTy = VectorType::get(ScalarTruncatedTy,
03170                                           OriginalTy->getVectorNumElements());
03171       if (TruncatedTy == OriginalTy)
03172         continue;
03173 
03174       IRBuilder<> B(cast<Instruction>(I));
03175       auto ShrinkOperand = [&](Value *V) -> Value* {
03176         if (auto *ZI = dyn_cast<ZExtInst>(V))
03177           if (ZI->getSrcTy() == TruncatedTy)
03178             return ZI->getOperand(0);
03179         return B.CreateZExtOrTrunc(V, TruncatedTy);
03180       };
03181 
03182       // The actual instruction modification depends on the instruction type,
03183       // unfortunately.
03184       Value *NewI = nullptr;
03185       if (BinaryOperator *BO = dyn_cast<BinaryOperator>(I)) {
03186         NewI = B.CreateBinOp(BO->getOpcode(),
03187                              ShrinkOperand(BO->getOperand(0)),
03188                              ShrinkOperand(BO->getOperand(1)));
03189         cast<BinaryOperator>(NewI)->copyIRFlags(I);
03190       } else if (ICmpInst *CI = dyn_cast<ICmpInst>(I)) {
03191         NewI = B.CreateICmp(CI->getPredicate(),
03192                             ShrinkOperand(CI->getOperand(0)),
03193                             ShrinkOperand(CI->getOperand(1)));
03194       } else if (SelectInst *SI = dyn_cast<SelectInst>(I)) {
03195         NewI = B.CreateSelect(SI->getCondition(),
03196                               ShrinkOperand(SI->getTrueValue()),
03197                               ShrinkOperand(SI->getFalseValue()));
03198       } else if (CastInst *CI = dyn_cast<CastInst>(I)) {
03199         switch (CI->getOpcode()) {
03200         default: llvm_unreachable("Unhandled cast!");
03201         case Instruction::Trunc:
03202           NewI = ShrinkOperand(CI->getOperand(0));
03203           break;
03204         case Instruction::SExt:
03205           NewI = B.CreateSExtOrTrunc(CI->getOperand(0),
03206                                      smallestIntegerVectorType(OriginalTy,
03207                                                                TruncatedTy));
03208           break;
03209         case Instruction::ZExt:
03210           NewI = B.CreateZExtOrTrunc(CI->getOperand(0),
03211                                      smallestIntegerVectorType(OriginalTy,
03212                                                                TruncatedTy));
03213           break;
03214         }
03215       } else if (ShuffleVectorInst *SI = dyn_cast<ShuffleVectorInst>(I)) {
03216         auto Elements0 = SI->getOperand(0)->getType()->getVectorNumElements();
03217         auto *O0 =
03218           B.CreateZExtOrTrunc(SI->getOperand(0),
03219                               VectorType::get(ScalarTruncatedTy, Elements0));
03220         auto Elements1 = SI->getOperand(1)->getType()->getVectorNumElements();
03221         auto *O1 =
03222           B.CreateZExtOrTrunc(SI->getOperand(1),
03223                               VectorType::get(ScalarTruncatedTy, Elements1));
03224 
03225         NewI = B.CreateShuffleVector(O0, O1, SI->getMask());
03226       } else if (isa<LoadInst>(I)) {
03227         // Don't do anything with the operands, just extend the result.
03228         continue;
03229       } else {
03230         llvm_unreachable("Unhandled instruction type!");
03231       }
03232 
03233       // Lastly, extend the result.
03234       NewI->takeName(cast<Instruction>(I));
03235       Value *Res = B.CreateZExtOrTrunc(NewI, OriginalTy);
03236       I->replaceAllUsesWith(Res);
03237       cast<Instruction>(I)->eraseFromParent();
03238       I = Res;
03239     }
03240   }
03241 
03242   // We'll have created a bunch of ZExts that are now parentless. Clean up.
03243   for (auto &KV : MinBWs) {
03244     VectorParts &Parts = WidenMap.get(KV.first);
03245     for (Value *&I : Parts) {
03246       ZExtInst *Inst = dyn_cast<ZExtInst>(I);
03247       if (Inst && Inst->use_empty()) {
03248         Value *NewI = Inst->getOperand(0);
03249         Inst->eraseFromParent();
03250         I = NewI;
03251       }
03252     }
03253   }
03254 }
03255 
03256 void InnerLoopVectorizer::vectorizeLoop() {
03257   //===------------------------------------------------===//
03258   //
03259   // Notice: any optimization or new instruction that go
03260   // into the code below should be also be implemented in
03261   // the cost-model.
03262   //
03263   //===------------------------------------------------===//
03264   Constant *Zero = Builder.getInt32(0);
03265 
03266   // In order to support reduction variables we need to be able to vectorize
03267   // Phi nodes. Phi nodes have cycles, so we need to vectorize them in two
03268   // stages. First, we create a new vector PHI node with no incoming edges.
03269   // We use this value when we vectorize all of the instructions that use the
03270   // PHI. Next, after all of the instructions in the block are complete we
03271   // add the new incoming edges to the PHI. At this point all of the
03272   // instructions in the basic block are vectorized, so we can use them to
03273   // construct the PHI.
03274   PhiVector RdxPHIsToFix;
03275 
03276   // Scan the loop in a topological order to ensure that defs are vectorized
03277   // before users.
03278   LoopBlocksDFS DFS(OrigLoop);
03279   DFS.perform(LI);
03280 
03281   // Vectorize all of the blocks in the original loop.
03282   for (LoopBlocksDFS::RPOIterator bb = DFS.beginRPO(),
03283        be = DFS.endRPO(); bb != be; ++bb)
03284     vectorizeBlockInLoop(*bb, &RdxPHIsToFix);
03285 
03286   // Insert truncates and extends for any truncated instructions as hints to
03287   // InstCombine.
03288   if (VF > 1)
03289     truncateToMinimalBitwidths();
03290   
03291   // At this point every instruction in the original loop is widened to
03292   // a vector form. We are almost done. Now, we need to fix the PHI nodes
03293   // that we vectorized. The PHI nodes are currently empty because we did
03294   // not want to introduce cycles. Notice that the remaining PHI nodes
03295   // that we need to fix are reduction variables.
03296 
03297   // Create the 'reduced' values for each of the induction vars.
03298   // The reduced values are the vector values that we scalarize and combine
03299   // after the loop is finished.
03300   for (PhiVector::iterator it = RdxPHIsToFix.begin(), e = RdxPHIsToFix.end();
03301        it != e; ++it) {
03302     PHINode *RdxPhi = *it;
03303     assert(RdxPhi && "Unable to recover vectorized PHI");
03304 
03305     // Find the reduction variable descriptor.
03306     assert(Legal->isReductionVariable(RdxPhi) &&
03307            "Unable to find the reduction variable");
03308     RecurrenceDescriptor RdxDesc = (*Legal->getReductionVars())[RdxPhi];
03309 
03310     RecurrenceDescriptor::RecurrenceKind RK = RdxDesc.getRecurrenceKind();
03311     TrackingVH<Value> ReductionStartValue = RdxDesc.getRecurrenceStartValue();
03312     Instruction *LoopExitInst = RdxDesc.getLoopExitInstr();
03313     RecurrenceDescriptor::MinMaxRecurrenceKind MinMaxKind =
03314         RdxDesc.getMinMaxRecurrenceKind();
03315     setDebugLocFromInst(Builder, ReductionStartValue);
03316 
03317     // We need to generate a reduction vector from the incoming scalar.
03318     // To do so, we need to generate the 'identity' vector and override
03319     // one of the elements with the incoming scalar reduction. We need
03320     // to do it in the vector-loop preheader.
03321     Builder.SetInsertPoint(LoopBypassBlocks[1]->getTerminator());
03322 
03323     // This is the vector-clone of the value that leaves the loop.
03324     VectorParts &VectorExit = getVectorValue(LoopExitInst);
03325     Type *VecTy = VectorExit[0]->getType();
03326 
03327     // Find the reduction identity variable. Zero for addition, or, xor,
03328     // one for multiplication, -1 for And.
03329     Value *Identity;
03330     Value *VectorStart;
03331     if (RK == RecurrenceDescriptor::RK_IntegerMinMax ||
03332         RK == RecurrenceDescriptor::RK_FloatMinMax) {
03333       // MinMax reduction have the start value as their identify.
03334       if (VF == 1) {
03335         VectorStart = Identity = ReductionStartValue;
03336       } else {
03337         VectorStart = Identity =
03338             Builder.CreateVectorSplat(VF, ReductionStartValue, "minmax.ident");
03339       }
03340     } else {
03341       // Handle other reduction kinds:
03342       Constant *Iden = RecurrenceDescriptor::getRecurrenceIdentity(
03343           RK, VecTy->getScalarType());
03344       if (VF == 1) {
03345         Identity = Iden;
03346         // This vector is the Identity vector where the first element is the
03347         // incoming scalar reduction.
03348         VectorStart = ReductionStartValue;
03349       } else {
03350         Identity = ConstantVector::getSplat(VF, Iden);
03351 
03352         // This vector is the Identity vector where the first element is the
03353         // incoming scalar reduction.
03354         VectorStart =
03355             Builder.CreateInsertElement(Identity, ReductionStartValue, Zero);
03356       }
03357     }
03358 
03359     // Fix the vector-loop phi.
03360 
03361     // Reductions do not have to start at zero. They can start with
03362     // any loop invariant values.
03363     VectorParts &VecRdxPhi = WidenMap.get(RdxPhi);
03364     BasicBlock *Latch = OrigLoop->getLoopLatch();
03365     Value *LoopVal = RdxPhi->getIncomingValueForBlock(Latch);
03366     VectorParts &Val = getVectorValue(LoopVal);
03367     for (unsigned part = 0; part < UF; ++part) {
03368       // Make sure to add the reduction stat value only to the
03369       // first unroll part.
03370       Value *StartVal = (part == 0) ? VectorStart : Identity;
03371       cast<PHINode>(VecRdxPhi[part])->addIncoming(StartVal,
03372                                                   LoopVectorPreHeader);
03373       cast<PHINode>(VecRdxPhi[part])->addIncoming(Val[part],
03374                                                   LoopVectorBody.back());
03375     }
03376 
03377     // Before each round, move the insertion point right between
03378     // the PHIs and the values we are going to write.
03379     // This allows us to write both PHINodes and the extractelement
03380     // instructions.
03381     Builder.SetInsertPoint(&*LoopMiddleBlock->getFirstInsertionPt());
03382 
03383     VectorParts RdxParts = getVectorValue(LoopExitInst);
03384     setDebugLocFromInst(Builder, LoopExitInst);
03385 
03386     // If the vector reduction can be performed in a smaller type, we truncate
03387     // then extend the loop exit value to enable InstCombine to evaluate the
03388     // entire expression in the smaller type.
03389     if (VF > 1 && RdxPhi->getType() != RdxDesc.getRecurrenceType()) {
03390       Type *RdxVecTy = VectorType::get(RdxDesc.getRecurrenceType(), VF);
03391       Builder.SetInsertPoint(LoopVectorBody.back()->getTerminator());
03392       for (unsigned part = 0; part < UF; ++part) {
03393         Value *Trunc = Builder.CreateTrunc(RdxParts[part], RdxVecTy);
03394         Value *Extnd = RdxDesc.isSigned() ? Builder.CreateSExt(Trunc, VecTy)
03395                                           : Builder.CreateZExt(Trunc, VecTy);
03396         for (Value::user_iterator UI = RdxParts[part]->user_begin();
03397              UI != RdxParts[part]->user_end();)
03398           if (*UI != Trunc) {
03399             (*UI++)->replaceUsesOfWith(RdxParts[part], Extnd);
03400             RdxParts[part] = Extnd;
03401           } else {
03402             ++UI;
03403           }
03404       }
03405       Builder.SetInsertPoint(&*LoopMiddleBlock->getFirstInsertionPt());
03406       for (unsigned part = 0; part < UF; ++part)
03407         RdxParts[part] = Builder.CreateTrunc(RdxParts[part], RdxVecTy);
03408     }
03409 
03410     // Reduce all of the unrolled parts into a single vector.
03411     Value *ReducedPartRdx = RdxParts[0];
03412     unsigned Op = RecurrenceDescriptor::getRecurrenceBinOp(RK);
03413     setDebugLocFromInst(Builder, ReducedPartRdx);
03414     for (unsigned part = 1; part < UF; ++part) {
03415       if (Op != Instruction::ICmp && Op != Instruction::FCmp)
03416         // Floating point operations had to be 'fast' to enable the reduction.
03417         ReducedPartRdx = addFastMathFlag(
03418             Builder.CreateBinOp((Instruction::BinaryOps)Op, RdxParts[part],
03419                                 ReducedPartRdx, "bin.rdx"));
03420       else
03421         ReducedPartRdx = RecurrenceDescriptor::createMinMaxOp(
03422             Builder, MinMaxKind, ReducedPartRdx, RdxParts[part]);
03423     }
03424 
03425     if (VF > 1) {
03426       // VF is a power of 2 so we can emit the reduction using log2(VF) shuffles
03427       // and vector ops, reducing the set of values being computed by half each
03428       // round.
03429       assert(isPowerOf2_32(VF) &&
03430              "Reduction emission only supported for pow2 vectors!");
03431       Value *TmpVec = ReducedPartRdx;
03432       SmallVector<Constant*, 32> ShuffleMask(VF, nullptr);
03433       for (unsigned i = VF; i != 1; i >>= 1) {
03434         // Move the upper half of the vector to the lower half.
03435         for (unsigned j = 0; j != i/2; ++j)
03436           ShuffleMask[j] = Builder.getInt32(i/2 + j);
03437 
03438         // Fill the rest of the mask with undef.
03439         std::fill(&ShuffleMask[i/2], ShuffleMask.end(),
03440                   UndefValue::get(Builder.getInt32Ty()));
03441 
03442         Value *Shuf =
03443         Builder.CreateShuffleVector(TmpVec,
03444                                     UndefValue::get(TmpVec->getType()),
03445                                     ConstantVector::get(ShuffleMask),
03446                                     "rdx.shuf");
03447 
03448         if (Op != Instruction::ICmp && Op != Instruction::FCmp)
03449           // Floating point operations had to be 'fast' to enable the reduction.
03450           TmpVec = addFastMathFlag(Builder.CreateBinOp(
03451               (Instruction::BinaryOps)Op, TmpVec, Shuf, "bin.rdx"));
03452         else
03453           TmpVec = RecurrenceDescriptor::createMinMaxOp(Builder, MinMaxKind,
03454                                                         TmpVec, Shuf);
03455       }
03456 
03457       // The result is in the first element of the vector.
03458       ReducedPartRdx = Builder.CreateExtractElement(TmpVec,
03459                                                     Builder.getInt32(0));
03460 
03461       // If the reduction can be performed in a smaller type, we need to extend
03462       // the reduction to the wider type before we branch to the original loop.
03463       if (RdxPhi->getType() != RdxDesc.getRecurrenceType())
03464         ReducedPartRdx =
03465             RdxDesc.isSigned()
03466                 ? Builder.CreateSExt(ReducedPartRdx, RdxPhi->getType())
03467                 : Builder.CreateZExt(ReducedPartRdx, RdxPhi->getType());
03468     }
03469 
03470     // Create a phi node that merges control-flow from the backedge-taken check
03471     // block and the middle block.
03472     PHINode *BCBlockPhi = PHINode::Create(RdxPhi->getType(), 2, "bc.merge.rdx",
03473                                           LoopScalarPreHeader->getTerminator());
03474     for (unsigned I = 0, E = LoopBypassBlocks.size(); I != E; ++I)
03475       BCBlockPhi->addIncoming(ReductionStartValue, LoopBypassBlocks[I]);
03476     BCBlockPhi->addIncoming(ReducedPartRdx, LoopMiddleBlock);
03477 
03478     // Now, we need to fix the users of the reduction variable
03479     // inside and outside of the scalar remainder loop.
03480     // We know that the loop is in LCSSA form. We need to update the
03481     // PHI nodes in the exit blocks.
03482     for (BasicBlock::iterator LEI = LoopExitBlock->begin(),
03483          LEE = LoopExitBlock->end(); LEI != LEE; ++LEI) {
03484       PHINode *LCSSAPhi = dyn_cast<PHINode>(LEI);
03485       if (!LCSSAPhi) break;
03486 
03487       // All PHINodes need to have a single entry edge, or two if
03488       // we already fixed them.
03489       assert(LCSSAPhi->getNumIncomingValues() < 3 && "Invalid LCSSA PHI");
03490 
03491       // We found our reduction value exit-PHI. Update it with the
03492       // incoming bypass edge.
03493       if (LCSSAPhi->getIncomingValue(0) == LoopExitInst) {
03494         // Add an edge coming from the bypass.
03495         LCSSAPhi->addIncoming(ReducedPartRdx, LoopMiddleBlock);
03496         break;
03497       }
03498     }// end of the LCSSA phi scan.
03499 
03500     // Fix the scalar loop reduction variable with the incoming reduction sum
03501     // from the vector body and from the backedge value.
03502     int IncomingEdgeBlockIdx =
03503     (RdxPhi)->getBasicBlockIndex(OrigLoop->getLoopLatch());
03504     assert(IncomingEdgeBlockIdx >= 0 && "Invalid block index");
03505     // Pick the other block.
03506     int SelfEdgeBlockIdx = (IncomingEdgeBlockIdx ? 0 : 1);
03507     (RdxPhi)->setIncomingValue(SelfEdgeBlockIdx, BCBlockPhi);
03508     (RdxPhi)->setIncomingValue(IncomingEdgeBlockIdx, LoopExitInst);
03509   }// end of for each redux variable.
03510 
03511   fixLCSSAPHIs();
03512 
03513   // Make sure DomTree is updated.
03514   updateAnalysis();
03515   
03516   // Predicate any stores.
03517   for (auto KV : PredicatedStores) {
03518     BasicBlock::iterator I(KV.first);
03519     auto *BB = SplitBlock(I->getParent(), &*std::next(I), DT, LI);
03520     auto *T = SplitBlockAndInsertIfThen(KV.second, &*I, /*Unreachable=*/false,
03521                                         /*BranchWeights=*/nullptr, DT);
03522     I->moveBefore(T);
03523     I->getParent()->setName("pred.store.if");
03524     BB->setName("pred.store.continue");
03525   }
03526   DEBUG(DT->verifyDomTree());
03527   // Remove redundant induction instructions.
03528   cse(LoopVectorBody);
03529 }
03530 
03531 void InnerLoopVectorizer::fixLCSSAPHIs() {
03532   for (BasicBlock::iterator LEI = LoopExitBlock->begin(),
03533        LEE = LoopExitBlock->end(); LEI != LEE; ++LEI) {
03534     PHINode *LCSSAPhi = dyn_cast<PHINode>(LEI);
03535     if (!LCSSAPhi) break;
03536     if (LCSSAPhi->getNumIncomingValues() == 1)
03537       LCSSAPhi->addIncoming(UndefValue::get(LCSSAPhi->getType()),
03538                             LoopMiddleBlock);
03539   }
03540 }
03541 
03542 InnerLoopVectorizer::VectorParts
03543 InnerLoopVectorizer::createEdgeMask(BasicBlock *Src, BasicBlock *Dst) {
03544   assert(std::find(pred_begin(Dst), pred_end(Dst), Src) != pred_end(Dst) &&
03545          "Invalid edge");
03546 
03547   // Look for cached value.
03548   std::pair<BasicBlock*, BasicBlock*> Edge(Src, Dst);
03549   EdgeMaskCache::iterator ECEntryIt = MaskCache.find(Edge);
03550   if (ECEntryIt != MaskCache.end())
03551     return ECEntryIt->second;
03552 
03553   VectorParts SrcMask = createBlockInMask(Src);
03554 
03555   // The terminator has to be a branch inst!
03556   BranchInst *BI = dyn_cast<BranchInst>(Src->getTerminator());
03557   assert(BI && "Unexpected terminator found");
03558 
03559   if (BI->isConditional()) {
03560     VectorParts EdgeMask = getVectorValue(BI->getCondition());
03561 
03562     if (BI->getSuccessor(0) != Dst)
03563       for (unsigned part = 0; part < UF; ++part)
03564         EdgeMask[part] = Builder.CreateNot(EdgeMask[part]);
03565 
03566     for (unsigned part = 0; part < UF; ++part)
03567       EdgeMask[part] = Builder.CreateAnd(EdgeMask[part], SrcMask[part]);
03568 
03569     MaskCache[Edge] = EdgeMask;
03570     return EdgeMask;
03571   }
03572 
03573   MaskCache[Edge] = SrcMask;
03574   return SrcMask;
03575 }
03576 
03577 InnerLoopVectorizer::VectorParts
03578 InnerLoopVectorizer::createBlockInMask(BasicBlock *BB) {
03579   assert(OrigLoop->contains(BB) && "Block is not a part of a loop");
03580 
03581   // Loop incoming mask is all-one.
03582   if (OrigLoop->getHeader() == BB) {
03583     Value *C = ConstantInt::get(IntegerType::getInt1Ty(BB->getContext()), 1);
03584     return getVectorValue(C);
03585   }
03586 
03587   // This is the block mask. We OR all incoming edges, and with zero.
03588   Value *Zero = ConstantInt::get(IntegerType::getInt1Ty(BB->getContext()), 0);
03589   VectorParts BlockMask = getVectorValue(Zero);
03590 
03591   // For each pred:
03592   for (pred_iterator it = pred_begin(BB), e = pred_end(BB); it != e; ++it) {
03593     VectorParts EM = createEdgeMask(*it, BB);
03594     for (unsigned part = 0; part < UF; ++part)
03595       BlockMask[part] = Builder.CreateOr(BlockMask[part], EM[part]);
03596   }
03597 
03598   return BlockMask;
03599 }
03600 
03601 void InnerLoopVectorizer::widenPHIInstruction(
03602     Instruction *PN, InnerLoopVectorizer::VectorParts &Entry, unsigned UF,
03603     unsigned VF, PhiVector *PV) {
03604   PHINode* P = cast<PHINode>(PN);
03605   // Handle reduction variables:
03606   if (Legal->isReductionVariable(P)) {
03607     for (unsigned part = 0; part < UF; ++part) {
03608       // This is phase one of vectorizing PHIs.
03609       Type *VecTy = (VF == 1) ? PN->getType() :
03610       VectorType::get(PN->getType(), VF);
03611       Entry[part] = PHINode::Create(
03612           VecTy, 2, "vec.phi", &*LoopVectorBody.back()->getFirstInsertionPt());
03613     }
03614     PV->push_back(P);
03615     return;
03616   }
03617 
03618   setDebugLocFromInst(Builder, P);
03619   // Check for PHI nodes that are lowered to vector selects.
03620   if (P->getParent() != OrigLoop->getHeader()) {
03621     // We know that all PHIs in non-header blocks are converted into
03622     // selects, so we don't have to worry about the insertion order and we
03623     // can just use the builder.
03624     // At this point we generate the predication tree. There may be
03625     // duplications since this is a simple recursive scan, but future
03626     // optimizations will clean it up.
03627 
03628     unsigned NumIncoming = P->getNumIncomingValues();
03629 
03630     // Generate a sequence of selects of the form:
03631     // SELECT(Mask3, In3,
03632     //      SELECT(Mask2, In2,
03633     //                   ( ...)))
03634     for (unsigned In = 0; In < NumIncoming; In++) {
03635       VectorParts Cond = createEdgeMask(P->getIncomingBlock(In),
03636                                         P->getParent());
03637       VectorParts &In0 = getVectorValue(P->getIncomingValue(In));
03638 
03639       for (unsigned part = 0; part < UF; ++part) {
03640         // We might have single edge PHIs (blocks) - use an identity
03641         // 'select' for the first PHI operand.
03642         if (In == 0)
03643           Entry[part] = Builder.CreateSelect(Cond[part], In0[part],
03644                                              In0[part]);
03645         else
03646           // Select between the current value and the previous incoming edge
03647           // based on the incoming mask.
03648           Entry[part] = Builder.CreateSelect(Cond[part], In0[part],
03649                                              Entry[part], "predphi");
03650       }
03651     }
03652     return;
03653   }
03654 
03655   // This PHINode must be an induction variable.
03656   // Make sure that we know about it.
03657   assert(Legal->getInductionVars()->count(P) &&
03658          "Not an induction variable");
03659 
03660   InductionDescriptor II = Legal->getInductionVars()->lookup(P);
03661 
03662   // FIXME: The newly created binary instructions should contain nsw/nuw flags,
03663   // which can be found from the original scalar operations.
03664   switch (II.getKind()) {
03665     case InductionDescriptor::IK_NoInduction:
03666       llvm_unreachable("Unknown induction");
03667     case InductionDescriptor::IK_IntInduction: {
03668       assert(P->getType() == II.getStartValue()->getType() &&
03669              "Types must match");
03670       // Handle other induction variables that are now based on the
03671       // canonical one.
03672       Value *V = Induction;
03673       if (P != OldInduction) {
03674         V = Builder.CreateSExtOrTrunc(Induction, P->getType());
03675         V = II.transform(Builder, V);
03676         V->setName("offset.idx");
03677       }
03678       Value *Broadcasted = getBroadcastInstrs(V);
03679       // After broadcasting the induction variable we need to make the vector
03680       // consecutive by adding 0, 1, 2, etc.
03681       for (unsigned part = 0; part < UF; ++part)
03682         Entry[part] = getStepVector(Broadcasted, VF * part, II.getStepValue());
03683       return;
03684     }
03685     case InductionDescriptor::IK_PtrInduction:
03686       // Handle the pointer induction variable case.
03687       assert(P->getType()->isPointerTy() && "Unexpected type.");
03688       // This is the normalized GEP that starts counting at zero.
03689       Value *PtrInd = Induction;
03690       PtrInd = Builder.CreateSExtOrTrunc(PtrInd, II.getStepValue()->getType());
03691       // This is the vector of results. Notice that we don't generate
03692       // vector geps because scalar geps result in better code.
03693       for (unsigned part = 0; part < UF; ++part) {
03694         if (VF == 1) {
03695           int EltIndex = part;
03696           Constant *Idx = ConstantInt::get(PtrInd->getType(), EltIndex);
03697           Value *GlobalIdx = Builder.CreateAdd(PtrInd, Idx);
03698           Value *SclrGep = II.transform(Builder, GlobalIdx);
03699           SclrGep->setName("next.gep");
03700           Entry[part] = SclrGep;
03701           continue;
03702         }
03703 
03704         Value *VecVal = UndefValue::get(VectorType::get(P->getType(), VF));
03705         for (unsigned int i = 0; i < VF; ++i) {
03706           int EltIndex = i + part * VF;
03707           Constant *Idx = ConstantInt::get(PtrInd->getType(), EltIndex);
03708           Value *GlobalIdx = Builder.CreateAdd(PtrInd, Idx);
03709           Value *SclrGep = II.transform(Builder, GlobalIdx);
03710           SclrGep->setName("next.gep");
03711           VecVal = Builder.CreateInsertElement(VecVal, SclrGep,
03712                                                Builder.getInt32(i),
03713                                                "insert.gep");
03714         }
03715         Entry[part] = VecVal;
03716       }
03717       return;
03718   }
03719 }
03720 
03721 void InnerLoopVectorizer::vectorizeBlockInLoop(BasicBlock *BB, PhiVector *PV) {
03722   // For each instruction in the old loop.
03723   for (BasicBlock::iterator it = BB->begin(), e = BB->end(); it != e; ++it) {
03724     VectorParts &Entry = WidenMap.get(&*it);
03725 
03726     switch (it->getOpcode()) {
03727     case Instruction::Br:
03728       // Nothing to do for PHIs and BR, since we already took care of the
03729       // loop control flow instructions.
03730       continue;
03731     case Instruction::PHI: {
03732       // Vectorize PHINodes.
03733       widenPHIInstruction(&*it, Entry, UF, VF, PV);
03734       continue;
03735     }// End of PHI.
03736 
03737     case Instruction::Add:
03738     case Instruction::FAdd:
03739     case Instruction::Sub:
03740     case Instruction::FSub:
03741     case Instruction::Mul:
03742     case Instruction::FMul:
03743     case Instruction::UDiv:
03744     case Instruction::SDiv:
03745     case Instruction::FDiv:
03746     case Instruction::URem:
03747     case Instruction::SRem:
03748     case Instruction::FRem:
03749     case Instruction::Shl:
03750     case Instruction::LShr:
03751     case Instruction::AShr:
03752     case Instruction::And:
03753     case Instruction::Or:
03754     case Instruction::Xor: {
03755       // Just widen binops.
03756       BinaryOperator *BinOp = dyn_cast<BinaryOperator>(it);
03757       setDebugLocFromInst(Builder, BinOp);
03758       VectorParts &A = getVectorValue(it->getOperand(0));
03759       VectorParts &B = getVectorValue(it->getOperand(1));
03760 
03761       // Use this vector value for all users of the original instruction.
03762       for (unsigned Part = 0; Part < UF; ++Part) {
03763         Value *V = Builder.CreateBinOp(BinOp->getOpcode(), A[Part], B[Part]);
03764 
03765         if (BinaryOperator *VecOp = dyn_cast<BinaryOperator>(V))
03766           VecOp->copyIRFlags(BinOp);
03767 
03768         Entry[Part] = V;
03769       }
03770 
03771       propagateMetadata(Entry, &*it);
03772       break;
03773     }
03774     case Instruction::Select: {
03775       // Widen selects.
03776       // If the selector is loop invariant we can create a select
03777       // instruction with a scalar condition. Otherwise, use vector-select.
03778       auto *SE = PSE.getSE();
03779       bool InvariantCond =
03780           SE->isLoopInvariant(PSE.getSCEV(it->getOperand(0)), OrigLoop);
03781       setDebugLocFromInst(Builder, &*it);
03782 
03783       // The condition can be loop invariant  but still defined inside the
03784       // loop. This means that we can't just use the original 'cond' value.
03785       // We have to take the 'vectorized' value and pick the first lane.
03786       // Instcombine will make this a no-op.
03787       VectorParts &Cond = getVectorValue(it->getOperand(0));
03788       VectorParts &Op0  = getVectorValue(it->getOperand(1));
03789       VectorParts &Op1  = getVectorValue(it->getOperand(2));
03790       
03791       Value *ScalarCond = (VF == 1) ? Cond[0] :
03792         Builder.CreateExtractElement(Cond[0], Builder.getInt32(0));
03793 
03794       for (unsigned Part = 0; Part < UF; ++Part) {
03795         Entry[Part] = Builder.CreateSelect(
03796           InvariantCond ? ScalarCond : Cond[Part],
03797           Op0[Part],
03798           Op1[Part]);
03799       }
03800 
03801       propagateMetadata(Entry, &*it);
03802       break;
03803     }
03804 
03805     case Instruction::ICmp:
03806     case Instruction::FCmp: {
03807       // Widen compares. Generate vector compares.
03808       bool FCmp = (it->getOpcode() == Instruction::FCmp);
03809       CmpInst *Cmp = dyn_cast<CmpInst>(it);
03810       setDebugLocFromInst(Builder, &*it);
03811       VectorParts &A = getVectorValue(it->getOperand(0));
03812       VectorParts &B = getVectorValue(it->getOperand(1));
03813       for (unsigned Part = 0; Part < UF; ++Part) {
03814         Value *C = nullptr;
03815         if (FCmp) {
03816           C = Builder.CreateFCmp(Cmp->getPredicate(), A[Part], B[Part]);
03817           cast<FCmpInst>(C)->copyFastMathFlags(&*it);
03818         } else {
03819           C = Builder.CreateICmp(Cmp->getPredicate(), A[Part], B[Part]);
03820         }
03821         Entry[Part] = C;
03822       }
03823 
03824       propagateMetadata(Entry, &*it);
03825       break;
03826     }
03827 
03828     case Instruction::Store:
03829     case Instruction::Load:
03830       vectorizeMemoryInstruction(&*it);
03831         break;
03832     case Instruction::ZExt:
03833     case Instruction::SExt:
03834     case Instruction::FPToUI:
03835     case Instruction::FPToSI:
03836     case Instruction::FPExt:
03837     case Instruction::PtrToInt:
03838     case Instruction::IntToPtr:
03839     case Instruction::SIToFP:
03840     case Instruction::UIToFP:
03841     case Instruction::Trunc:
03842     case Instruction::FPTrunc:
03843     case Instruction::BitCast: {
03844       CastInst *CI = dyn_cast<CastInst>(it);
03845       setDebugLocFromInst(Builder, &*it);
03846       /// Optimize the special case where the source is the induction
03847       /// variable. Notice that we can only optimize the 'trunc' case
03848       /// because: a. FP conversions lose precision, b. sext/zext may wrap,
03849       /// c. other casts depend on pointer size.
03850       if (CI->getOperand(0) == OldInduction &&
03851           it->getOpcode() == Instruction::Trunc) {
03852         Value *ScalarCast = Builder.CreateCast(CI->getOpcode(), Induction,
03853                                                CI->getType());
03854         Value *Broadcasted = getBroadcastInstrs(ScalarCast);
03855         InductionDescriptor II =
03856             Legal->getInductionVars()->lookup(OldInduction);
03857         Constant *Step = ConstantInt::getSigned(
03858             CI->getType(), II.getStepValue()->getSExtValue());
03859         for (unsigned Part = 0; Part < UF; ++Part)
03860           Entry[Part] = getStepVector(Broadcasted, VF * Part, Step);
03861         propagateMetadata(Entry, &*it);
03862         break;
03863       }
03864       /// Vectorize casts.
03865       Type *DestTy = (VF == 1) ? CI->getType() :
03866                                  VectorType::get(CI->getType(), VF);
03867 
03868       VectorParts &A = getVectorValue(it->getOperand(0));
03869       for (unsigned Part = 0; Part < UF; ++Part)
03870         Entry[Part] = Builder.CreateCast(CI->getOpcode(), A[Part], DestTy);
03871       propagateMetadata(Entry, &*it);
03872       break;
03873     }
03874 
03875     case Instruction::Call: {
03876       // Ignore dbg intrinsics.
03877       if (isa<DbgInfoIntrinsic>(it))
03878         break;
03879       setDebugLocFromInst(Builder, &*it);
03880 
03881       Module *M = BB->getParent()->getParent();
03882       CallInst *CI = cast<CallInst>(it);
03883 
03884       StringRef FnName = CI->getCalledFunction()->getName();
03885       Function *F = CI->getCalledFunction();
03886       Type *RetTy = ToVectorTy(CI->getType(), VF);
03887       SmallVector<Type *, 4> Tys;
03888       for (unsigned i = 0, ie = CI->getNumArgOperands(); i != ie; ++i)
03889         Tys.push_back(ToVectorTy(CI->getArgOperand(i)->getType(), VF));
03890 
03891       Intrinsic::ID ID = getIntrinsicIDForCall(CI, TLI);
03892       if (ID &&
03893           (ID == Intrinsic::assume || ID == Intrinsic::lifetime_end ||
03894            ID == Intrinsic::lifetime_start)) {
03895         scalarizeInstruction(&*it);
03896         break;
03897       }
03898       // The flag shows whether we use Intrinsic or a usual Call for vectorized
03899       // version of the instruction.
03900       // Is it beneficial to perform intrinsic call compared to lib call?
03901       bool NeedToScalarize;
03902       unsigned CallCost = getVectorCallCost(CI, VF, *TTI, TLI, NeedToScalarize);
03903       bool UseVectorIntrinsic =
03904           ID && getVectorIntrinsicCost(CI, VF, *TTI, TLI) <= CallCost;
03905       if (!UseVectorIntrinsic && NeedToScalarize) {
03906         scalarizeInstruction(&*it);
03907         break;
03908       }
03909 
03910       for (unsigned Part = 0; Part < UF; ++Part) {
03911         SmallVector<Value *, 4> Args;
03912         for (unsigned i = 0, ie = CI->getNumArgOperands(); i != ie; ++i) {
03913           Value *Arg = CI->getArgOperand(i);
03914           // Some intrinsics have a scalar argument - don't replace it with a
03915           // vector.
03916           if (!UseVectorIntrinsic || !hasVectorInstrinsicScalarOpd(ID, i)) {
03917             VectorParts &VectorArg = getVectorValue(CI->getArgOperand(i));
03918             Arg = VectorArg[Part];
03919           }
03920           Args.push_back(Arg);
03921         }
03922 
03923         Function *VectorF;
03924         if (UseVectorIntrinsic) {
03925           // Use vector version of the intrinsic.
03926           Type *TysForDecl[] = {CI->getType()};
03927           if (VF > 1)
03928             TysForDecl[0] = VectorType::get(CI->getType()->getScalarType(), VF);
03929           VectorF = Intrinsic::getDeclaration(M, ID, TysForDecl);
03930         } else {
03931           // Use vector version of the library call.
03932           StringRef VFnName = TLI->getVectorizedFunction(FnName, VF);
03933           assert(!VFnName.empty() && "Vector function name is empty.");
03934           VectorF = M->getFunction(VFnName);
03935           if (!VectorF) {
03936             // Generate a declaration
03937             FunctionType *FTy = FunctionType::get(RetTy, Tys, false);
03938             VectorF =
03939                 Function::Create(FTy, Function::ExternalLinkage, VFnName, M);
03940             VectorF->copyAttributesFrom(F);
03941           }
03942         }
03943         assert(VectorF && "Can't create vector function.");
03944         Entry[Part] = Builder.CreateCall(VectorF, Args);
03945       }
03946 
03947       propagateMetadata(Entry, &*it);
03948       break;
03949     }
03950 
03951     default:
03952       // All other instructions are unsupported. Scalarize them.
03953       scalarizeInstruction(&*it);
03954       break;
03955     }// end of switch.
03956   }// end of for_each instr.
03957 }
03958 
03959 void InnerLoopVectorizer::updateAnalysis() {
03960   // Forget the original basic block.
03961   PSE.getSE()->forgetLoop(OrigLoop);
03962 
03963   // Update the dominator tree information.
03964   assert(DT->properlyDominates(LoopBypassBlocks.front(), LoopExitBlock) &&
03965          "Entry does not dominate exit.");
03966 
03967   for (unsigned I = 1, E = LoopBypassBlocks.size(); I != E; ++I)
03968     DT->addNewBlock(LoopBypassBlocks[I], LoopBypassBlocks[I-1]);
03969   DT->addNewBlock(LoopVectorPreHeader, LoopBypassBlocks.back());
03970 
03971   // We don't predicate stores by this point, so the vector body should be a
03972   // single loop.
03973   assert(LoopVectorBody.size() == 1 && "Expected single block loop!");
03974   DT->addNewBlock(LoopVectorBody[0], LoopVectorPreHeader);
03975 
03976   DT->addNewBlock(LoopMiddleBlock, LoopVectorBody.back());
03977   DT->addNewBlock(LoopScalarPreHeader, LoopBypassBlocks[0]);
03978   DT->changeImmediateDominator(LoopScalarBody, LoopScalarPreHeader);
03979   DT->changeImmediateDominator(LoopExitBlock, LoopBypassBlocks[0]);
03980 
03981   DEBUG(DT->verifyDomTree());
03982 }
03983 
03984 /// \brief Check whether it is safe to if-convert this phi node.
03985 ///
03986 /// Phi nodes with constant expressions that can trap are not safe to if
03987 /// convert.
03988 static bool canIfConvertPHINodes(BasicBlock *BB) {
03989   for (BasicBlock::iterator I = BB->begin(), E = BB->end(); I != E; ++I) {
03990     PHINode *Phi = dyn_cast<PHINode>(I);
03991     if (!Phi)
03992       return true;
03993     for (unsigned p = 0, e = Phi->getNumIncomingValues(); p != e; ++p)
03994       if (Constant *C = dyn_cast<Constant>(Phi->getIncomingValue(p)))
03995         if (C->canTrap())
03996           return false;
03997   }
03998   return true;
03999 }
04000 
04001 bool LoopVectorizationLegality::canVectorizeWithIfConvert() {
04002   if (!EnableIfConversion) {
04003     emitAnalysis(VectorizationReport() << "if-conversion is disabled");
04004     return false;
04005   }
04006 
04007   assert(TheLoop->getNumBlocks() > 1 && "Single block loops are vectorizable");
04008 
04009   // A list of pointers that we can safely read and write to.
04010   SmallPtrSet<Value *, 8> SafePointes;
04011 
04012   // Collect safe addresses.
04013   for (Loop::block_iterator BI = TheLoop->block_begin(),
04014          BE = TheLoop->block_end(); BI != BE; ++BI) {
04015     BasicBlock *BB = *BI;
04016 
04017     if (blockNeedsPredication(BB))
04018       continue;
04019 
04020     for (BasicBlock::iterator I = BB->begin(), E = BB->end(); I != E; ++I) {
04021       if (LoadInst *LI = dyn_cast<LoadInst>(I))
04022         SafePointes.insert(LI->getPointerOperand());
04023       else if (StoreInst *SI = dyn_cast<StoreInst>(I))
04024         SafePointes.insert(SI->getPointerOperand());
04025     }
04026   }
04027 
04028   // Collect the blocks that need predication.
04029   BasicBlock *Header = TheLoop->getHeader();
04030   for (Loop::block_iterator BI = TheLoop->block_begin(),
04031          BE = TheLoop->block_end(); BI != BE; ++BI) {
04032     BasicBlock *BB = *BI;
04033 
04034     // We don't support switch statements inside loops.
04035     if (!isa<BranchInst>(BB->getTerminator())) {
04036       emitAnalysis(VectorizationReport(BB->getTerminator())
04037                    << "loop contains a switch statement");
04038       return false;
04039     }
04040 
04041     // We must be able to predicate all blocks that need to be predicated.
04042     if (blockNeedsPredication(BB)) {
04043       if (!blockCanBePredicated(BB, SafePointes)) {
04044         emitAnalysis(VectorizationReport(BB->getTerminator())
04045                      << "control flow cannot be substituted for a select");
04046         return false;
04047       }
04048     } else if (BB != Header && !canIfConvertPHINodes(BB)) {
04049       emitAnalysis(VectorizationReport(BB->getTerminator())
04050                    << "control flow cannot be substituted for a select");
04051       return false;
04052     }
04053   }
04054 
04055   // We can if-convert this loop.
04056   return true;
04057 }
04058 
04059 bool LoopVectorizationLegality::canVectorize() {
04060   // We must have a loop in canonical form. Loops with indirectbr in them cannot
04061   // be canonicalized.
04062   if (!TheLoop->getLoopPreheader()) {
04063     emitAnalysis(
04064         VectorizationReport() <<
04065         "loop control flow is not understood by vectorizer");
04066     return false;
04067   }
04068 
04069   // We can only vectorize innermost loops.
04070   if (!TheLoop->empty()) {
04071     emitAnalysis(VectorizationReport() << "loop is not the innermost loop");
04072     return false;
04073   }
04074 
04075   // We must have a single backedge.
04076   if (TheLoop->getNumBackEdges() != 1) {
04077     emitAnalysis(
04078         VectorizationReport() <<
04079         "loop control flow is not understood by vectorizer");
04080     return false;
04081   }
04082 
04083   // We must have a single exiting block.
04084   if (!TheLoop->getExitingBlock()) {
04085     emitAnalysis(
04086         VectorizationReport() <<
04087         "loop control flow is not understood by vectorizer");
04088     return false;
04089   }
04090 
04091   // We only handle bottom-tested loops, i.e. loop in which the condition is
04092   // checked at the end of each iteration. With that we can assume that all
04093   // instructions in the loop are executed the same number of times.
04094   if (TheLoop->getExitingBlock() != TheLoop->getLoopLatch()) {
04095     emitAnalysis(
04096         VectorizationReport() <<
04097         "loop control flow is not understood by vectorizer");
04098     return false;
04099   }
04100 
04101   // We need to have a loop header.
04102   DEBUG(dbgs() << "LV: Found a loop: " <<
04103         TheLoop->getHeader()->getName() << '\n');
04104 
04105   // Check if we can if-convert non-single-bb loops.
04106   unsigned NumBlocks = TheLoop->getNumBlocks();
04107   if (NumBlocks != 1 && !canVectorizeWithIfConvert()) {
04108     DEBUG(dbgs() << "LV: Can't if-convert the loop.\n");
04109     return false;
04110   }
04111 
04112   // ScalarEvolution needs to be able to find the exit count.
04113   const SCEV *ExitCount = PSE.getSE()->getBackedgeTakenCount(TheLoop);
04114   if (ExitCount == PSE.getSE()->getCouldNotCompute()) {
04115     emitAnalysis(VectorizationReport()
04116                  << "could not determine number of loop iterations");
04117     DEBUG(dbgs() << "LV: SCEV could not compute the loop exit count.\n");
04118     return false;
04119   }
04120 
04121   // Check if we can vectorize the instructions and CFG in this loop.
04122   if (!canVectorizeInstrs()) {
04123     DEBUG(dbgs() << "LV: Can't vectorize the instructions or CFG\n");
04124     return false;
04125   }
04126 
04127   // Go over each instruction and look at memory deps.
04128   if (!canVectorizeMemory()) {
04129     DEBUG(dbgs() << "LV: Can't vectorize due to memory conflicts\n");
04130     return false;
04131   }
04132 
04133   // Collect all of the variables that remain uniform after vectorization.
04134   collectLoopUniforms();
04135 
04136   DEBUG(dbgs() << "LV: We can vectorize this loop"
04137                << (LAI->getRuntimePointerChecking()->Need
04138                        ? " (with a runtime bound check)"
04139                        : "")
04140                << "!\n");
04141 
04142   bool UseInterleaved = TTI->enableInterleavedAccessVectorization();
04143 
04144   // If an override option has been passed in for interleaved accesses, use it.
04145   if (EnableInterleavedMemAccesses.getNumOccurrences() > 0)
04146     UseInterleaved = EnableInterleavedMemAccesses;
04147 
04148   // Analyze interleaved memory accesses.
04149   if (UseInterleaved)
04150     InterleaveInfo.analyzeInterleaving(Strides);
04151 
04152   unsigned SCEVThreshold = VectorizeSCEVCheckThreshold;
04153   if (Hints->getForce() == LoopVectorizeHints::FK_Enabled)
04154     SCEVThreshold = PragmaVectorizeSCEVCheckThreshold;
04155 
04156   if (PSE.getUnionPredicate().getComplexity() > SCEVThreshold) {
04157     emitAnalysis(VectorizationReport()
04158                  << "Too many SCEV assumptions need to be made and checked "
04159                  << "at runtime");
04160     DEBUG(dbgs() << "LV: Too many SCEV checks needed.\n");
04161     return false;
04162   }
04163 
04164   // Okay! We can vectorize. At this point we don't have any other mem analysis
04165   // which may limit our maximum vectorization factor, so just return true with
04166   // no restrictions.
04167   return true;
04168 }
04169 
04170 static Type *convertPointerToIntegerType(const DataLayout &DL, Type *Ty) {
04171   if (Ty->isPointerTy())
04172     return DL.getIntPtrType(Ty);
04173 
04174   // It is possible that char's or short's overflow when we ask for the loop's
04175   // trip count, work around this by changing the type size.
04176   if (Ty->getScalarSizeInBits() < 32)
04177     return Type::getInt32Ty(Ty->getContext());
04178 
04179   return Ty;
04180 }
04181 
04182 static Type* getWiderType(const DataLayout &DL, Type *Ty0, Type *Ty1) {
04183   Ty0 = convertPointerToIntegerType(DL, Ty0);
04184   Ty1 = convertPointerToIntegerType(DL, Ty1);
04185   if (Ty0->getScalarSizeInBits() > Ty1->getScalarSizeInBits())
04186     return Ty0;
04187   return Ty1;
04188 }
04189 
04190 /// \brief Check that the instruction has outside loop users and is not an
04191 /// identified reduction variable.
04192 static bool hasOutsideLoopUser(const Loop *TheLoop, Instruction *Inst,
04193                                SmallPtrSetImpl<Value *> &Reductions) {
04194   // Reduction instructions are allowed to have exit users. All other
04195   // instructions must not have external users.
04196   if (!Reductions.count(Inst))
04197     //Check that all of the users of the loop are inside the BB.
04198     for (User *U : Inst->users()) {
04199       Instruction *UI = cast<Instruction>(U);
04200       // This user may be a reduction exit value.
04201       if (!TheLoop->contains(UI)) {
04202         DEBUG(dbgs() << "LV: Found an outside user for : " << *UI << '\n');
04203         return true;
04204       }
04205     }
04206   return false;
04207 }
04208 
04209 bool LoopVectorizationLegality::canVectorizeInstrs() {
04210   BasicBlock *Header = TheLoop->getHeader();
04211 
04212   // Look for the attribute signaling the absence of NaNs.
04213   Function &F = *Header->getParent();
04214   const DataLayout &DL = F.getParent()->getDataLayout();
04215   if (F.hasFnAttribute("no-nans-fp-math"))
04216     HasFunNoNaNAttr =
04217         F.getFnAttribute("no-nans-fp-math").getValueAsString() == "true";
04218 
04219   // For each block in the loop.
04220   for (Loop::block_iterator bb = TheLoop->block_begin(),
04221        be = TheLoop->block_end(); bb != be; ++bb) {
04222 
04223     // Scan the instructions in the block and look for hazards.
04224     for (BasicBlock::iterator it = (*bb)->begin(), e = (*bb)->end(); it != e;
04225          ++it) {
04226 
04227       if (PHINode *Phi = dyn_cast<PHINode>(it)) {
04228         Type *PhiTy = Phi->getType();
04229         // Check that this PHI type is allowed.
04230         if (!PhiTy->isIntegerTy() &&
04231             !PhiTy->isFloatingPointTy() &&
04232             !PhiTy->isPointerTy()) {
04233           emitAnalysis(VectorizationReport(&*it)
04234                        << "loop control flow is not understood by vectorizer");
04235           DEBUG(dbgs() << "LV: Found an non-int non-pointer PHI.\n");
04236           return false;
04237         }
04238 
04239         // If this PHINode is not in the header block, then we know that we
04240         // can convert it to select during if-conversion. No need to check if
04241         // the PHIs in this block are induction or reduction variables.
04242         if (*bb != Header) {
04243           // Check that this instruction has no outside users or is an
04244           // identified reduction value with an outside user.
04245           if (!hasOutsideLoopUser(TheLoop, &*it, AllowedExit))
04246             continue;
04247           emitAnalysis(VectorizationReport(&*it) <<
04248                        "value could not be identified as "
04249                        "an induction or reduction variable");
04250           return false;
04251         }
04252 
04253         // We only allow if-converted PHIs with exactly two incoming values.
04254         if (Phi->getNumIncomingValues() != 2) {
04255           emitAnalysis(VectorizationReport(&*it)
04256                        << "control flow not understood by vectorizer");
04257           DEBUG(dbgs() << "LV: Found an invalid PHI.\n");
04258           return false;
04259         }
04260 
04261         InductionDescriptor ID;
04262         if (InductionDescriptor::isInductionPHI(Phi, PSE.getSE(), ID)) {
04263           Inductions[Phi] = ID;
04264           // Get the widest type.
04265           if (!WidestIndTy)
04266             WidestIndTy = convertPointerToIntegerType(DL, PhiTy);
04267           else
04268             WidestIndTy = getWiderType(DL, PhiTy, WidestIndTy);
04269 
04270           // Int inductions are special because we only allow one IV.
04271           if (ID.getKind() == InductionDescriptor::IK_IntInduction &&
04272               ID.getStepValue()->isOne() &&
04273               isa<Constant>(ID.getStartValue()) &&
04274                 cast<Constant>(ID.getStartValue())->isNullValue()) {
04275             // Use the phi node with the widest type as induction. Use the last
04276             // one if there are multiple (no good reason for doing this other
04277             // than it is expedient). We've checked that it begins at zero and
04278             // steps by one, so this is a canonical induction variable.
04279             if (!Induction || PhiTy == WidestIndTy)
04280               Induction = Phi;
04281           }
04282 
04283           DEBUG(dbgs() << "LV: Found an induction variable.\n");
04284 
04285           // Until we explicitly handle the case of an induction variable with
04286           // an outside loop user we have to give up vectorizing this loop.
04287           if (hasOutsideLoopUser(TheLoop, &*it, AllowedExit)) {
04288             emitAnalysis(VectorizationReport(&*it) <<
04289                          "use of induction value outside of the "
04290                          "loop is not handled by vectorizer");
04291             return false;
04292           }
04293 
04294           continue;
04295         }
04296 
04297         RecurrenceDescriptor RedDes;
04298         if (RecurrenceDescriptor::isReductionPHI(Phi, TheLoop, RedDes)) {
04299           if (RedDes.hasUnsafeAlgebra())
04300             Requirements->addUnsafeAlgebraInst(RedDes.getUnsafeAlgebraInst());
04301           AllowedExit.insert(RedDes.getLoopExitInstr());
04302           Reductions[Phi] = RedDes;
04303           continue;
04304         }
04305 
04306         emitAnalysis(VectorizationReport(&*it) <<
04307                      "value that could not be identified as "
04308                      "reduction is used outside the loop");
04309         DEBUG(dbgs() << "LV: Found an unidentified PHI."<< *Phi <<"\n");
04310         return false;
04311       }// end of PHI handling
04312 
04313       // We handle calls that:
04314       //   * Are debug info intrinsics.
04315       //   * Have a mapping to an IR intrinsic.
04316       //   * Have a vector version available.
04317       CallInst *CI = dyn_cast<CallInst>(it);
04318       if (CI && !getIntrinsicIDForCall(CI, TLI) && !isa<DbgInfoIntrinsic>(CI) &&
04319           !(CI->getCalledFunction() && TLI &&
04320             TLI->isFunctionVectorizable(CI->getCalledFunction()->getName()))) {
04321         emitAnalysis(VectorizationReport(&*it)
04322                      << "call instruction cannot be vectorized");
04323         DEBUG(dbgs() << "LV: Found a non-intrinsic, non-libfunc callsite.\n");
04324         return false;
04325       }
04326 
04327       // Intrinsics such as powi,cttz and ctlz are legal to vectorize if the
04328       // second argument is the same (i.e. loop invariant)
04329       if (CI &&
04330           hasVectorInstrinsicScalarOpd(getIntrinsicIDForCall(CI, TLI), 1)) {
04331         auto *SE = PSE.getSE();
04332         if (!SE->isLoopInvariant(PSE.getSCEV(CI->getOperand(1)), TheLoop)) {
04333           emitAnalysis(VectorizationReport(&*it)
04334                        << "intrinsic instruction cannot be vectorized");
04335           DEBUG(dbgs() << "LV: Found unvectorizable intrinsic " << *CI << "\n");
04336           return false;
04337         }
04338       }
04339 
04340       // Check that the instruction return type is vectorizable.
04341       // Also, we can't vectorize extractelement instructions.
04342       if ((!VectorType::isValidElementType(it->getType()) &&
04343            !it->getType()->isVoidTy()) || isa<ExtractElementInst>(it)) {
04344         emitAnalysis(VectorizationReport(&*it)
04345                      << "instruction return type cannot be vectorized");
04346         DEBUG(dbgs() << "LV: Found unvectorizable type.\n");
04347         return false;
04348       }
04349 
04350       // Check that the stored type is vectorizable.
04351       if (StoreInst *ST = dyn_cast<StoreInst>(it)) {
04352         Type *T = ST->getValueOperand()->getType();
04353         if (!VectorType::isValidElementType(T)) {
04354           emitAnalysis(VectorizationReport(ST) <<
04355                        "store instruction cannot be vectorized");
04356           return false;
04357         }
04358         if (EnableMemAccessVersioning)
04359           collectStridedAccess(ST);
04360       }
04361 
04362       if (EnableMemAccessVersioning)
04363         if (LoadInst *LI = dyn_cast<LoadInst>(it))
04364           collectStridedAccess(LI);
04365 
04366       // Reduction instructions are allowed to have exit users.
04367       // All other instructions must not have external users.
04368       if (hasOutsideLoopUser(TheLoop, &*it, AllowedExit)) {
04369         emitAnalysis(VectorizationReport(&*it) <<
04370                      "value cannot be used outside the loop");
04371         return false;
04372       }
04373 
04374     } // next instr.
04375 
04376   }
04377 
04378   if (!Induction) {
04379     DEBUG(dbgs() << "LV: Did not find one integer induction var.\n");
04380     if (Inductions.empty()) {
04381       emitAnalysis(VectorizationReport()
04382                    << "loop induction variable could not be identified");
04383       return false;
04384     }
04385   }
04386 
04387   // Now we know the widest induction type, check if our found induction
04388   // is the same size. If it's not, unset it here and InnerLoopVectorizer
04389   // will create another.
04390   if (Induction && WidestIndTy != Induction->getType())
04391     Induction = nullptr;
04392 
04393   return true;
04394 }
04395 
04396 void LoopVectorizationLegality::collectStridedAccess(Value *MemAccess) {
04397   Value *Ptr = nullptr;
04398   if (LoadInst *LI = dyn_cast<LoadInst>(MemAccess))
04399     Ptr = LI->getPointerOperand();
04400   else if (StoreInst *SI = dyn_cast<StoreInst>(MemAccess))
04401     Ptr = SI->getPointerOperand();
04402   else
04403     return;
04404 
04405   Value *Stride = getStrideFromPointer(Ptr, PSE.getSE(), TheLoop);
04406   if (!Stride)
04407     return;
04408 
04409   DEBUG(dbgs() << "LV: Found a strided access that we can version");
04410   DEBUG(dbgs() << "  Ptr: " << *Ptr << " Stride: " << *Stride << "\n");
04411   Strides[Ptr] = Stride;
04412   StrideSet.insert(Stride);
04413 }
04414 
04415 void LoopVectorizationLegality::collectLoopUniforms() {
04416   // We now know that the loop is vectorizable!
04417   // Collect variables that will remain uniform after vectorization.
04418   std::vector<Value*> Worklist;
04419   BasicBlock *Latch = TheLoop->getLoopLatch();
04420 
04421   // Start with the conditional branch and walk up the block.
04422   Worklist.push_back(Latch->getTerminator()->getOperand(0));
04423 
04424   // Also add all consecutive pointer values; these values will be uniform
04425   // after vectorization (and subsequent cleanup) and, until revectorization is
04426   // supported, all dependencies must also be uniform.
04427   for (Loop::block_iterator B = TheLoop->block_begin(),
04428        BE = TheLoop->block_end(); B != BE; ++B)
04429     for (BasicBlock::iterator I = (*B)->begin(), IE = (*B)->end();
04430          I != IE; ++I)
04431       if (I->getType()->isPointerTy() && isConsecutivePtr(&*I))
04432         Worklist.insert(Worklist.end(), I->op_begin(), I->op_end());
04433 
04434   while (!Worklist.empty()) {
04435     Instruction *I = dyn_cast<Instruction>(Worklist.back());
04436     Worklist.pop_back();
04437 
04438     // Look at instructions inside this loop.
04439     // Stop when reaching PHI nodes.
04440     // TODO: we need to follow values all over the loop, not only in this block.
04441     if (!I || !TheLoop->contains(I) || isa<PHINode>(I))
04442       continue;
04443 
04444     // This is a known uniform.
04445     Uniforms.insert(I);
04446 
04447     // Insert all operands.
04448     Worklist.insert(Worklist.end(), I->op_begin(), I->op_end());
04449   }
04450 }
04451 
04452 bool LoopVectorizationLegality::canVectorizeMemory() {
04453   LAI = &LAA->getInfo(TheLoop, Strides);
04454   auto &OptionalReport = LAI->getReport();
04455   if (OptionalReport)
04456     emitAnalysis(VectorizationReport(*OptionalReport));
04457   if (!LAI->canVectorizeMemory())
04458     return false;
04459 
04460   if (LAI->hasStoreToLoopInvariantAddress()) {
04461     emitAnalysis(
04462         VectorizationReport()
04463         << "write to a loop invariant address could not be vectorized");
04464     DEBUG(dbgs() << "LV: We don't allow storing to uniform addresses\n");
04465     return false;
04466   }
04467 
04468   Requirements->addRuntimePointerChecks(LAI->getNumRuntimePointerChecks());
04469   PSE.addPredicate(LAI->PSE.getUnionPredicate());
04470 
04471   return true;
04472 }
04473 
04474 bool LoopVectorizationLegality::isInductionVariable(const Value *V) {
04475   Value *In0 = const_cast<Value*>(V);
04476   PHINode *PN = dyn_cast_or_null<PHINode>(In0);
04477   if (!PN)
04478     return false;
04479 
04480   return Inductions.count(PN);
04481 }
04482 
04483 bool LoopVectorizationLegality::blockNeedsPredication(BasicBlock *BB)  {
04484   return LoopAccessInfo::blockNeedsPredication(BB, TheLoop, DT);
04485 }
04486 
04487 bool LoopVectorizationLegality::blockCanBePredicated(BasicBlock *BB,
04488                                            SmallPtrSetImpl<Value *> &SafePtrs) {
04489   
04490   for (BasicBlock::iterator it = BB->begin(), e = BB->end(); it != e; ++it) {
04491     // Check that we don't have a constant expression that can trap as operand.
04492     for (Instruction::op_iterator OI = it->op_begin(), OE = it->op_end();
04493          OI != OE; ++OI) {
04494       if (Constant *C = dyn_cast<Constant>(*OI))
04495         if (C->canTrap())
04496           return false;
04497     }
04498     // We might be able to hoist the load.
04499     if (it->mayReadFromMemory()) {
04500       LoadInst *LI = dyn_cast<LoadInst>(it);
04501       if (!LI)
04502         return false;
04503       if (!SafePtrs.count(LI->getPointerOperand())) {
04504         if (isLegalMaskedLoad(LI->getType(), LI->getPointerOperand())) {
04505           MaskedOp.insert(LI);
04506           continue;
04507         }
04508         return false;
04509       }
04510     }
04511 
04512     // We don't predicate stores at the moment.
04513     if (it->mayWriteToMemory()) {
04514       StoreInst *SI = dyn_cast<StoreInst>(it);
04515       // We only support predication of stores in basic blocks with one
04516       // predecessor.
04517       if (!SI)
04518         return false;
04519 
04520       bool isSafePtr = (SafePtrs.count(SI->getPointerOperand()) != 0);
04521       bool isSinglePredecessor = SI->getParent()->getSinglePredecessor();
04522       
04523       if (++NumPredStores > NumberOfStoresToPredicate || !isSafePtr ||
04524           !isSinglePredecessor) {
04525         // Build a masked store if it is legal for the target, otherwise
04526         // scalarize the block.
04527         bool isLegalMaskedOp =
04528           isLegalMaskedStore(SI->getValueOperand()->getType(),
04529                              SI->getPointerOperand());
04530         if (isLegalMaskedOp) {
04531           --NumPredStores;
04532           MaskedOp.insert(SI);
04533           continue;
04534         }
04535         return false;
04536       }
04537     }
04538     if (it->mayThrow())
04539       return false;
04540 
04541     // The instructions below can trap.
04542     switch (it->getOpcode()) {
04543     default: continue;
04544     case Instruction::UDiv:
04545     case Instruction::SDiv:
04546     case Instruction::URem:
04547     case Instruction::SRem:
04548       return false;
04549     }
04550   }
04551 
04552   return true;
04553 }
04554 
04555 void InterleavedAccessInfo::collectConstStridedAccesses(
04556     MapVector<Instruction *, StrideDescriptor> &StrideAccesses,
04557     const ValueToValueMap &Strides) {
04558   // Holds load/store instructions in program order.
04559   SmallVector<Instruction *, 16> AccessList;
04560 
04561   for (auto *BB : TheLoop->getBlocks()) {
04562     bool IsPred = LoopAccessInfo::blockNeedsPredication(BB, TheLoop, DT);
04563 
04564     for (auto &I : *BB) {
04565       if (!isa<LoadInst>(&I) && !isa<StoreInst>(&I))
04566         continue;
04567       // FIXME: Currently we can't handle mixed accesses and predicated accesses
04568       if (IsPred)
04569         return;
04570 
04571       AccessList.push_back(&I);
04572     }
04573   }
04574 
04575   if (AccessList.empty())
04576     return;
04577 
04578   auto &DL = TheLoop->getHeader()->getModule()->getDataLayout();
04579   for (auto I : AccessList) {
04580     LoadInst *LI = dyn_cast<LoadInst>(I);
04581     StoreInst *SI = dyn_cast<StoreInst>(I);
04582 
04583     Value *Ptr = LI ? LI->getPointerOperand() : SI->getPointerOperand();
04584     int Stride = isStridedPtr(PSE, Ptr, TheLoop, Strides);
04585 
04586     // The factor of the corresponding interleave group.
04587     unsigned Factor = std::abs(Stride);
04588 
04589     // Ignore the access if the factor is too small or too large.
04590     if (Factor < 2 || Factor > MaxInterleaveGroupFactor)
04591       continue;
04592 
04593     const SCEV *Scev = replaceSymbolicStrideSCEV(PSE, Strides, Ptr);
04594     PointerType *PtrTy = dyn_cast<PointerType>(Ptr->getType());
04595     unsigned Size = DL.getTypeAllocSize(PtrTy->getElementType());
04596 
04597     // An alignment of 0 means target ABI alignment.
04598     unsigned Align = LI ? LI->getAlignment() : SI->getAlignment();
04599     if (!Align)
04600       Align = DL.getABITypeAlignment(PtrTy->getElementType());
04601 
04602     StrideAccesses[I] = StrideDescriptor(Stride, Scev, Size, Align);
04603   }
04604 }
04605 
04606 // Analyze interleaved accesses and collect them into interleave groups.
04607 //
04608 // Notice that the vectorization on interleaved groups will change instruction
04609 // orders and may break dependences. But the memory dependence check guarantees
04610 // that there is no overlap between two pointers of different strides, element
04611 // sizes or underlying bases.
04612 //
04613 // For pointers sharing the same stride, element size and underlying base, no
04614 // need to worry about Read-After-Write dependences and Write-After-Read
04615 // dependences.
04616 //
04617 // E.g. The RAW dependence:  A[i] = a;
04618 //                           b = A[i];
04619 // This won't exist as it is a store-load forwarding conflict, which has
04620 // already been checked and forbidden in the dependence check.
04621 //
04622 // E.g. The WAR dependence:  a = A[i];  // (1)
04623 //                           A[i] = b;  // (2)
04624 // The store group of (2) is always inserted at or below (2), and the load group
04625 // of (1) is always inserted at or above (1). The dependence is safe.
04626 void InterleavedAccessInfo::analyzeInterleaving(
04627     const ValueToValueMap &Strides) {
04628   DEBUG(dbgs() << "LV: Analyzing interleaved accesses...\n");
04629 
04630   // Holds all the stride accesses.
04631   MapVector<Instruction *, StrideDescriptor> StrideAccesses;
04632   collectConstStridedAccesses(StrideAccesses, Strides);
04633 
04634   if (StrideAccesses.empty())
04635     return;
04636 
04637   // Holds all interleaved store groups temporarily.
04638   SmallSetVector<InterleaveGroup *, 4> StoreGroups;
04639 
04640   // Search the load-load/write-write pair B-A in bottom-up order and try to
04641   // insert B into the interleave group of A according to 3 rules:
04642   //   1. A and B have the same stride.
04643   //   2. A and B have the same memory object size.
04644   //   3. B belongs to the group according to the distance.
04645   //
04646   // The bottom-up order can avoid breaking the Write-After-Write dependences
04647   // between two pointers of the same base.
04648   // E.g.  A[i]   = a;   (1)
04649   //       A[i]   = b;   (2)
04650   //       A[i+1] = c    (3)
04651   // We form the group (2)+(3) in front, so (1) has to form groups with accesses
04652   // above (1), which guarantees that (1) is always above (2).
04653   for (auto I = StrideAccesses.rbegin(), E = StrideAccesses.rend(); I != E;
04654        ++I) {
04655     Instruction *A = I->first;
04656     StrideDescriptor DesA = I->second;
04657 
04658     InterleaveGroup *Group = getInterleaveGroup(A);
04659     if (!Group) {
04660       DEBUG(dbgs() << "LV: Creating an interleave group with:" << *A << '\n');
04661       Group = createInterleaveGroup(A, DesA.Stride, DesA.Align);
04662     }
04663 
04664     if (A->mayWriteToMemory())
04665       StoreGroups.insert(Group);
04666 
04667     for (auto II = std::next(I); II != E; ++II) {
04668       Instruction *B = II->first;
04669       StrideDescriptor DesB = II->second;
04670 
04671       // Ignore if B is already in a group or B is a different memory operation.
04672       if (isInterleaved(B) || A->mayReadFromMemory() != B->mayReadFromMemory())
04673         continue;
04674 
04675       // Check the rule 1 and 2.
04676       if (DesB.Stride != DesA.Stride || DesB.Size != DesA.Size)
04677         continue;
04678 
04679       // Calculate the distance and prepare for the rule 3.
04680       const SCEVConstant *DistToA = dyn_cast<SCEVConstant>(
04681           PSE.getSE()->getMinusSCEV(DesB.Scev, DesA.Scev));
04682       if (!DistToA)
04683         continue;
04684 
04685       int DistanceToA = DistToA->getAPInt().getSExtValue();
04686 
04687       // Skip if the distance is not multiple of size as they are not in the
04688       // same group.
04689       if (DistanceToA % static_cast<int>(DesA.Size))
04690         continue;
04691 
04692       // The index of B is the index of A plus the related index to A.
04693       int IndexB =
04694           Group->getIndex(A) + DistanceToA / static_cast<int>(DesA.Size);
04695 
04696       // Try to insert B into the group.
04697       if (Group->insertMember(B, IndexB, DesB.Align)) {
04698         DEBUG(dbgs() << "LV: Inserted:" << *B << '\n'
04699                      << "    into the interleave group with" << *A << '\n');
04700         InterleaveGroupMap[B] = Group;
04701 
04702         // Set the first load in program order as the insert position.
04703         if (B->mayReadFromMemory())
04704           Group->setInsertPos(B);
04705       }
04706     } // Iteration on instruction B
04707   }   // Iteration on instruction A
04708 
04709   // Remove interleaved store groups with gaps.
04710   for (InterleaveGroup *Group : StoreGroups)
04711     if (Group->getNumMembers() != Group->getFactor())
04712       releaseGroup(Group);
04713 }
04714 
04715 LoopVectorizationCostModel::VectorizationFactor
04716 LoopVectorizationCostModel::selectVectorizationFactor(bool OptForSize) {
04717   // Width 1 means no vectorize
04718   VectorizationFactor Factor = { 1U, 0U };
04719   if (OptForSize && Legal->getRuntimePointerChecking()->Need) {
04720     emitAnalysis(VectorizationReport() <<
04721                  "runtime pointer checks needed. Enable vectorization of this "
04722                  "loop with '#pragma clang loop vectorize(enable)' when "
04723                  "compiling with -Os/-Oz");
04724     DEBUG(dbgs() <<
04725           "LV: Aborting. Runtime ptr check is required with -Os/-Oz.\n");
04726     return Factor;
04727   }
04728 
04729   if (!EnableCondStoresVectorization && Legal->getNumPredStores()) {
04730     emitAnalysis(VectorizationReport() <<
04731                  "store that is conditionally executed prevents vectorization");
04732     DEBUG(dbgs() << "LV: No vectorization. There are conditional stores.\n");
04733     return Factor;
04734   }
04735 
04736   // Find the trip count.
04737   unsigned TC = PSE.getSE()->getSmallConstantTripCount(TheLoop);
04738   DEBUG(dbgs() << "LV: Found trip count: " << TC << '\n');
04739 
04740   MinBWs = computeMinimumValueSizes(TheLoop->getBlocks(), *DB, &TTI);
04741   unsigned SmallestType, WidestType;
04742   std::tie(SmallestType, WidestType) = getSmallestAndWidestTypes();
04743   unsigned WidestRegister = TTI.getRegisterBitWidth(true);
04744   unsigned MaxSafeDepDist = -1U;
04745   if (Legal->getMaxSafeDepDistBytes() != -1U)
04746     MaxSafeDepDist = Legal->getMaxSafeDepDistBytes() * 8;
04747   WidestRegister = ((WidestRegister < MaxSafeDepDist) ?
04748                     WidestRegister : MaxSafeDepDist);
04749   unsigned MaxVectorSize = WidestRegister / WidestType;
04750 
04751   DEBUG(dbgs() << "LV: The Smallest and Widest types: " << SmallestType << " / "
04752                << WidestType << " bits.\n");
04753   DEBUG(dbgs() << "LV: The Widest register is: "
04754           << WidestRegister << " bits.\n");
04755 
04756   if (MaxVectorSize == 0) {
04757     DEBUG(dbgs() << "LV: The target has no vector registers.\n");
04758     MaxVectorSize = 1;
04759   }
04760 
04761   assert(MaxVectorSize <= 64 && "Did not expect to pack so many elements"
04762          " into one vector!");
04763 
04764   unsigned VF = MaxVectorSize;
04765   if (MaximizeBandwidth && !OptForSize) {
04766     // Collect all viable vectorization factors.
04767     SmallVector<unsigned, 8> VFs;
04768     unsigned NewMaxVectorSize = WidestRegister / SmallestType;
04769     for (unsigned VS = MaxVectorSize; VS <= NewMaxVectorSize; VS *= 2)
04770       VFs.push_back(VS);
04771 
04772     // For each VF calculate its register usage.
04773     auto RUs = calculateRegisterUsage(VFs);
04774 
04775     // Select the largest VF which doesn't require more registers than existing
04776     // ones.
04777     unsigned TargetNumRegisters = TTI.getNumberOfRegisters(true);
04778     for (int i = RUs.size() - 1; i >= 0; --i) {
04779       if (RUs[i].MaxLocalUsers <= TargetNumRegisters) {
04780         VF = VFs[i];
04781         break;
04782       }
04783     }
04784   }
04785 
04786   // If we optimize the program for size, avoid creating the tail loop.
04787   if (OptForSize) {
04788     // If we are unable to calculate the trip count then don't try to vectorize.
04789     if (TC < 2) {
04790       emitAnalysis
04791         (VectorizationReport() <<
04792          "unable to calculate the loop count due to complex control flow");
04793       DEBUG(dbgs() << "LV: Aborting. A tail loop is required with -Os/-Oz.\n");
04794       return Factor;
04795     }
04796 
04797     // Find the maximum SIMD width that can fit within the trip count.
04798     VF = TC % MaxVectorSize;
04799 
04800     if (VF == 0)
04801       VF = MaxVectorSize;
04802     else {
04803       // If the trip count that we found modulo the vectorization factor is not
04804       // zero then we require a tail.
04805       emitAnalysis(VectorizationReport() <<
04806                    "cannot optimize for size and vectorize at the "
04807                    "same time. Enable vectorization of this loop "
04808                    "with '#pragma clang loop vectorize(enable)' "
04809                    "when compiling with -Os/-Oz");
04810       DEBUG(dbgs() << "LV: Aborting. A tail loop is required with -Os/-Oz.\n");
04811       return Factor;
04812     }
04813   }
04814 
04815   int UserVF = Hints->getWidth();
04816   if (UserVF != 0) {
04817     assert(isPowerOf2_32(UserVF) && "VF needs to be a power of two");
04818     DEBUG(dbgs() << "LV: Using user VF " << UserVF << ".\n");
04819 
04820     Factor.Width = UserVF;
04821     return Factor;
04822   }
04823 
04824   float Cost = expectedCost(1);
04825 #ifndef NDEBUG
04826   const float ScalarCost = Cost;
04827 #endif /* NDEBUG */
04828   unsigned Width = 1;
04829   DEBUG(dbgs() << "LV: Scalar loop costs: " << (int)ScalarCost << ".\n");
04830 
04831   bool ForceVectorization = Hints->getForce() == LoopVectorizeHints::FK_Enabled;
04832   // Ignore scalar width, because the user explicitly wants vectorization.
04833   if (ForceVectorization && VF > 1) {
04834     Width = 2;
04835     Cost = expectedCost(Width) / (float)Width;
04836   }
04837 
04838   for (unsigned i=2; i <= VF; i*=2) {
04839     // Notice that the vector loop needs to be executed less times, so
04840     // we need to divide the cost of the vector loops by the width of
04841     // the vector elements.
04842     float VectorCost = expectedCost(i) / (float)i;
04843     DEBUG(dbgs() << "LV: Vector loop of width " << i << " costs: " <<
04844           (int)VectorCost << ".\n");
04845     if (VectorCost < Cost) {
04846       Cost = VectorCost;
04847       Width = i;
04848     }
04849   }
04850 
04851   DEBUG(if (ForceVectorization && Width > 1 && Cost >= ScalarCost) dbgs()
04852         << "LV: Vectorization seems to be not beneficial, "
04853         << "but was forced by a user.\n");
04854   DEBUG(dbgs() << "LV: Selecting VF: "<< Width << ".\n");
04855   Factor.Width = Width;
04856   Factor.Cost = Width * Cost;
04857   return Factor;
04858 }
04859 
04860 std::pair<unsigned, unsigned>
04861 LoopVectorizationCostModel::getSmallestAndWidestTypes() {
04862   unsigned MinWidth = -1U;
04863   unsigned MaxWidth = 8;
04864   const DataLayout &DL = TheFunction->getParent()->getDataLayout();
04865 
04866   // For each block.
04867   for (Loop::block_iterator bb = TheLoop->block_begin(),
04868        be = TheLoop->block_end(); bb != be; ++bb) {
04869     BasicBlock *BB = *bb;
04870 
04871     // For each instruction in the loop.
04872     for (BasicBlock::iterator it = BB->begin(), e = BB->end(); it != e; ++it) {
04873       Type *T = it->getType();
04874 
04875       // Skip ignored values.
04876       if (ValuesToIgnore.count(&*it))
04877         continue;
04878 
04879       // Only examine Loads, Stores and PHINodes.
04880       if (!isa<LoadInst>(it) && !isa<StoreInst>(it) && !isa<PHINode>(it))
04881         continue;
04882 
04883       // Examine PHI nodes that are reduction variables. Update the type to
04884       // account for the recurrence type.
04885       if (PHINode *PN = dyn_cast<PHINode>(it)) {
04886         if (!Legal->isReductionVariable(PN))
04887           continue;
04888         RecurrenceDescriptor RdxDesc = (*Legal->getReductionVars())[PN];
04889         T = RdxDesc.getRecurrenceType();
04890       }
04891 
04892       // Examine the stored values.
04893       if (StoreInst *ST = dyn_cast<StoreInst>(it))
04894         T = ST->getValueOperand()->getType();
04895 
04896       // Ignore loaded pointer types and stored pointer types that are not
04897       // consecutive. However, we do want to take consecutive stores/loads of
04898       // pointer vectors into account.
04899       if (T->isPointerTy() && !isConsecutiveLoadOrStore(&*it))
04900         continue;
04901 
04902       MinWidth = std::min(MinWidth,
04903                           (unsigned)DL.getTypeSizeInBits(T->getScalarType()));
04904       MaxWidth = std::max(MaxWidth,
04905                           (unsigned)DL.getTypeSizeInBits(T->getScalarType()));
04906     }
04907   }
04908 
04909   return {MinWidth, MaxWidth};
04910 }
04911 
04912 unsigned LoopVectorizationCostModel::selectInterleaveCount(bool OptForSize,
04913                                                            unsigned VF,
04914                                                            unsigned LoopCost) {
04915 
04916   // -- The interleave heuristics --
04917   // We interleave the loop in order to expose ILP and reduce the loop overhead.
04918   // There are many micro-architectural considerations that we can't predict
04919   // at this level. For example, frontend pressure (on decode or fetch) due to
04920   // code size, or the number and capabilities of the execution ports.
04921   //
04922   // We use the following heuristics to select the interleave count:
04923   // 1. If the code has reductions, then we interleave to break the cross
04924   // iteration dependency.
04925   // 2. If the loop is really small, then we interleave to reduce the loop
04926   // overhead.
04927   // 3. We don't interleave if we think that we will spill registers to memory
04928   // due to the increased register pressure.
04929 
04930   // When we optimize for size, we don't interleave.
04931   if (OptForSize)
04932     return 1;
04933 
04934   // We used the distance for the interleave count.
04935   if (Legal->getMaxSafeDepDistBytes() != -1U)
04936     return 1;
04937 
04938   // Do not interleave loops with a relatively small trip count.
04939   unsigned TC = PSE.getSE()->getSmallConstantTripCount(TheLoop);
04940   if (TC > 1 && TC < TinyTripCountInterleaveThreshold)
04941     return 1;
04942 
04943   unsigned TargetNumRegisters = TTI.getNumberOfRegisters(VF > 1);
04944   DEBUG(dbgs() << "LV: The target has " << TargetNumRegisters <<
04945         " registers\n");
04946 
04947   if (VF == 1) {
04948     if (ForceTargetNumScalarRegs.getNumOccurrences() > 0)
04949       TargetNumRegisters = ForceTargetNumScalarRegs;
04950   } else {
04951     if (ForceTargetNumVectorRegs.getNumOccurrences() > 0)
04952       TargetNumRegisters = ForceTargetNumVectorRegs;
04953   }
04954 
04955   RegisterUsage R = calculateRegisterUsage({VF})[0];
04956   // We divide by these constants so assume that we have at least one
04957   // instruction that uses at least one register.
04958   R.MaxLocalUsers = std::max(R.MaxLocalUsers, 1U);
04959   R.NumInstructions = std::max(R.NumInstructions, 1U);
04960 
04961   // We calculate the interleave count using the following formula.
04962   // Subtract the number of loop invariants from the number of available
04963   // registers. These registers are used by all of the interleaved instances.
04964   // Next, divide the remaining registers by the number of registers that is
04965   // required by the loop, in order to estimate how many parallel instances
04966   // fit without causing spills. All of this is rounded down if necessary to be
04967   // a power of two. We want power of two interleave count to simplify any
04968   // addressing operations or alignment considerations.
04969   unsigned IC = PowerOf2Floor((TargetNumRegisters - R.LoopInvariantRegs) /
04970                               R.MaxLocalUsers);
04971 
04972   // Don't count the induction variable as interleaved.
04973   if (EnableIndVarRegisterHeur)
04974     IC = PowerOf2Floor((TargetNumRegisters - R.LoopInvariantRegs - 1) /
04975                        std::max(1U, (R.MaxLocalUsers - 1)));
04976 
04977   // Clamp the interleave ranges to reasonable counts.
04978   unsigned MaxInterleaveCount = TTI.getMaxInterleaveFactor(VF);
04979 
04980   // Check if the user has overridden the max.
04981   if (VF == 1) {
04982     if (ForceTargetMaxScalarInterleaveFactor.getNumOccurrences() > 0)
04983       MaxInterleaveCount = ForceTargetMaxScalarInterleaveFactor;
04984   } else {
04985     if (ForceTargetMaxVectorInterleaveFactor.getNumOccurrences() > 0)
04986       MaxInterleaveCount = ForceTargetMaxVectorInterleaveFactor;
04987   }
04988 
04989   // If we did not calculate the cost for VF (because the user selected the VF)
04990   // then we calculate the cost of VF here.
04991   if (LoopCost == 0)
04992     LoopCost = expectedCost(VF);
04993 
04994   // Clamp the calculated IC to be between the 1 and the max interleave count
04995   // that the target allows.
04996   if (IC > MaxInterleaveCount)
04997     IC = MaxInterleaveCount;
04998   else if (IC < 1)
04999     IC = 1;
05000 
05001   // Interleave if we vectorized this loop and there is a reduction that could
05002   // benefit from interleaving.
05003   if (VF > 1 && Legal->getReductionVars()->size()) {
05004     DEBUG(dbgs() << "LV: Interleaving because of reductions.\n");
05005     return IC;
05006   }
05007 
05008   // Note that if we've already vectorized the loop we will have done the
05009   // runtime check and so interleaving won't require further checks.
05010   bool InterleavingRequiresRuntimePointerCheck =
05011       (VF == 1 && Legal->getRuntimePointerChecking()->Need);
05012 
05013   // We want to interleave small loops in order to reduce the loop overhead and
05014   // potentially expose ILP opportunities.
05015   DEBUG(dbgs() << "LV: Loop cost is " << LoopCost << '\n');
05016   if (!InterleavingRequiresRuntimePointerCheck && LoopCost < SmallLoopCost) {
05017     // We assume that the cost overhead is 1 and we use the cost model
05018     // to estimate the cost of the loop and interleave until the cost of the
05019     // loop overhead is about 5% of the cost of the loop.
05020     unsigned SmallIC =
05021         std::min(IC, (unsigned)PowerOf2Floor(SmallLoopCost / LoopCost));
05022 
05023     // Interleave until store/load ports (estimated by max interleave count) are
05024     // saturated.
05025     unsigned NumStores = Legal->getNumStores();
05026     unsigned NumLoads = Legal->getNumLoads();
05027     unsigned StoresIC = IC / (NumStores ? NumStores : 1);
05028     unsigned LoadsIC = IC / (NumLoads ? NumLoads : 1);
05029 
05030     // If we have a scalar reduction (vector reductions are already dealt with
05031     // by this point), we can increase the critical path length if the loop
05032     // we're interleaving is inside another loop. Limit, by default to 2, so the
05033     // critical path only gets increased by one reduction operation.
05034     if (Legal->getReductionVars()->size() &&
05035         TheLoop->getLoopDepth() > 1) {
05036       unsigned F = static_cast<unsigned>(MaxNestedScalarReductionIC);
05037       SmallIC = std::min(SmallIC, F);
05038       StoresIC = std::min(StoresIC, F);
05039       LoadsIC = std::min(LoadsIC, F);
05040     }
05041 
05042     if (EnableLoadStoreRuntimeInterleave &&
05043         std::max(StoresIC, LoadsIC) > SmallIC) {
05044       DEBUG(dbgs() << "LV: Interleaving to saturate store or load ports.\n");
05045       return std::max(StoresIC, LoadsIC);
05046     }
05047 
05048     DEBUG(dbgs() << "LV: Interleaving to reduce branch cost.\n");
05049     return SmallIC;
05050   }
05051 
05052   // Interleave if this is a large loop (small loops are already dealt with by
05053   // this point) that could benefit from interleaving.
05054   bool HasReductions = (Legal->getReductionVars()->size() > 0);
05055   if (TTI.enableAggressiveInterleaving(HasReductions)) {
05056     DEBUG(dbgs() << "LV: Interleaving to expose ILP.\n");
05057     return IC;
05058   }
05059 
05060   DEBUG(dbgs() << "LV: Not Interleaving.\n");
05061   return 1;
05062 }
05063 
05064 SmallVector<LoopVectorizationCostModel::RegisterUsage, 8>
05065 LoopVectorizationCostModel::calculateRegisterUsage(
05066     const SmallVector<unsigned, 8> &VFs) {
05067   // This function calculates the register usage by measuring the highest number
05068   // of values that are alive at a single location. Obviously, this is a very
05069   // rough estimation. We scan the loop in a topological order in order and
05070   // assign a number to each instruction. We use RPO to ensure that defs are
05071   // met before their users. We assume that each instruction that has in-loop
05072   // users starts an interval. We record every time that an in-loop value is
05073   // used, so we have a list of the first and last occurrences of each
05074   // instruction. Next, we transpose this data structure into a multi map that
05075   // holds the list of intervals that *end* at a specific location. This multi
05076   // map allows us to perform a linear search. We scan the instructions linearly
05077   // and record each time that a new interval starts, by placing it in a set.
05078   // If we find this value in the multi-map then we remove it from the set.
05079   // The max register usage is the maximum size of the set.
05080   // We also search for instructions that are defined outside the loop, but are
05081   // used inside the loop. We need this number separately from the max-interval
05082   // usage number because when we unroll, loop-invariant values do not take
05083   // more register.
05084   LoopBlocksDFS DFS(TheLoop);
05085   DFS.perform(LI);
05086 
05087   RegisterUsage RU;
05088   RU.NumInstructions = 0;
05089 
05090   // Each 'key' in the map opens a new interval. The values
05091   // of the map are the index of the 'last seen' usage of the
05092   // instruction that is the key.
05093   typedef DenseMap<Instruction*, unsigned> IntervalMap;
05094   // Maps instruction to its index.
05095   DenseMap<unsigned, Instruction*> IdxToInstr;
05096   // Marks the end of each interval.
05097   IntervalMap EndPoint;
05098   // Saves the list of instruction indices that are used in the loop.
05099   SmallSet<Instruction*, 8> Ends;
05100   // Saves the list of values that are used in the loop but are
05101   // defined outside the loop, such as arguments and constants.
05102   SmallPtrSet<Value*, 8> LoopInvariants;
05103 
05104   unsigned Index = 0;
05105   for (LoopBlocksDFS::RPOIterator bb = DFS.beginRPO(),
05106        be = DFS.endRPO(); bb != be; ++bb) {
05107     RU.NumInstructions += (*bb)->size();
05108     for (Instruction &I : **bb) {
05109       IdxToInstr[Index++] = &I;
05110 
05111       // Save the end location of each USE.
05112       for (unsigned i = 0; i < I.getNumOperands(); ++i) {
05113         Value *U = I.getOperand(i);
05114         Instruction *Instr = dyn_cast<Instruction>(U);
05115 
05116         // Ignore non-instruction values such as arguments, constants, etc.
05117         if (!Instr) continue;
05118 
05119         // If this instruction is outside the loop then record it and continue.
05120         if (!TheLoop->contains(Instr)) {
05121           LoopInvariants.insert(Instr);
05122           continue;
05123         }
05124 
05125         // Overwrite previous end points.
05126         EndPoint[Instr] = Index;
05127         Ends.insert(Instr);
05128       }
05129     }
05130   }
05131 
05132   // Saves the list of intervals that end with the index in 'key'.
05133   typedef SmallVector<Instruction*, 2> InstrList;
05134   DenseMap<unsigned, InstrList> TransposeEnds;
05135 
05136   // Transpose the EndPoints to a list of values that end at each index.
05137   for (IntervalMap::iterator it = EndPoint.begin(), e = EndPoint.end();
05138        it != e; ++it)
05139     TransposeEnds[it->second].push_back(it->first);
05140 
05141   SmallSet<Instruction*, 8> OpenIntervals;
05142 
05143   // Get the size of the widest register.
05144   unsigned MaxSafeDepDist = -1U;
05145   if (Legal->getMaxSafeDepDistBytes() != -1U)
05146     MaxSafeDepDist = Legal->getMaxSafeDepDistBytes() * 8;
05147   unsigned WidestRegister =
05148       std::min(TTI.getRegisterBitWidth(true), MaxSafeDepDist);
05149   const DataLayout &DL = TheFunction->getParent()->getDataLayout();
05150 
05151   SmallVector<RegisterUsage, 8> RUs(VFs.size());
05152   SmallVector<unsigned, 8> MaxUsages(VFs.size(), 0);
05153 
05154   DEBUG(dbgs() << "LV(REG): Calculating max register usage:\n");
05155 
05156   // A lambda that gets the register usage for the given type and VF.
05157   auto GetRegUsage = [&DL, WidestRegister](Type *Ty, unsigned VF) {
05158     unsigned TypeSize = DL.getTypeSizeInBits(Ty->getScalarType());
05159     return std::max<unsigned>(1, VF * TypeSize / WidestRegister);
05160   };
05161 
05162   for (unsigned int i = 0; i < Index; ++i) {
05163     Instruction *I = IdxToInstr[i];
05164     // Ignore instructions that are never used within the loop.
05165     if (!Ends.count(I)) continue;
05166 
05167     // Remove all of the instructions that end at this location.
05168     InstrList &List = TransposeEnds[i];
05169     for (unsigned int j = 0, e = List.size(); j < e; ++j)
05170       OpenIntervals.erase(List[j]);
05171 
05172     // Skip ignored values.
05173     if (ValuesToIgnore.count(I))
05174       continue;
05175 
05176     // For each VF find the maximum usage of registers.
05177     for (unsigned j = 0, e = VFs.size(); j < e; ++j) {
05178       if (VFs[j] == 1) {
05179         MaxUsages[j] = std::max(MaxUsages[j], OpenIntervals.size());
05180         continue;
05181       }
05182 
05183       // Count the number of live intervals.
05184       unsigned RegUsage = 0;
05185       for (auto Inst : OpenIntervals) {
05186         // Skip ignored values for VF > 1.
05187         if (VecValuesToIgnore.count(Inst))
05188           continue;
05189         RegUsage += GetRegUsage(Inst->getType(), VFs[j]);
05190       }
05191       MaxUsages[j] = std::max(MaxUsages[j], RegUsage);
05192     }
05193 
05194     DEBUG(dbgs() << "LV(REG): At #" << i << " Interval # "
05195                  << OpenIntervals.size() << '\n');
05196 
05197     // Add the current instruction to the list of open intervals.
05198     OpenIntervals.insert(I);
05199   }
05200 
05201   for (unsigned i = 0, e = VFs.size(); i < e; ++i) {
05202     unsigned Invariant = 0;
05203     if (VFs[i] == 1)
05204       Invariant = LoopInvariants.size();
05205     else {
05206       for (auto Inst : LoopInvariants)
05207         Invariant += GetRegUsage(Inst->getType(), VFs[i]);
05208     }
05209 
05210     DEBUG(dbgs() << "LV(REG): VF = " << VFs[i] <<  '\n');
05211     DEBUG(dbgs() << "LV(REG): Found max usage: " << MaxUsages[i] << '\n');
05212     DEBUG(dbgs() << "LV(REG): Found invariant usage: " << Invariant << '\n');
05213     DEBUG(dbgs() << "LV(REG): LoopSize: " << RU.NumInstructions << '\n');
05214 
05215     RU.LoopInvariantRegs = Invariant;
05216     RU.MaxLocalUsers = MaxUsages[i];
05217     RUs[i] = RU;
05218   }
05219 
05220   return RUs;
05221 }
05222 
05223 unsigned LoopVectorizationCostModel::expectedCost(unsigned VF) {
05224   unsigned Cost = 0;
05225 
05226   // For each block.
05227   for (Loop::block_iterator bb = TheLoop->block_begin(),
05228        be = TheLoop->block_end(); bb != be; ++bb) {
05229     unsigned BlockCost = 0;
05230     BasicBlock *BB = *bb;
05231 
05232     // For each instruction in the old loop.
05233     for (BasicBlock::iterator it = BB->begin(), e = BB->end(); it != e; ++it) {
05234       // Skip dbg intrinsics.
05235       if (isa<DbgInfoIntrinsic>(it))
05236         continue;
05237 
05238       // Skip ignored values.
05239       if (ValuesToIgnore.count(&*it))
05240         continue;
05241 
05242       unsigned C = getInstructionCost(&*it, VF);
05243 
05244       // Check if we should override the cost.
05245       if (ForceTargetInstructionCost.getNumOccurrences() > 0)
05246         C = ForceTargetInstructionCost;
05247 
05248       BlockCost += C;
05249       DEBUG(dbgs() << "LV: Found an estimated cost of " << C << " for VF " <<
05250             VF << " For instruction: " << *it << '\n');
05251     }
05252 
05253     // We assume that if-converted blocks have a 50% chance of being executed.
05254     // When the code is scalar then some of the blocks are avoided due to CF.
05255     // When the code is vectorized we execute all code paths.
05256     if (VF == 1 && Legal->blockNeedsPredication(*bb))
05257       BlockCost /= 2;
05258 
05259     Cost += BlockCost;
05260   }
05261 
05262   return Cost;
05263 }
05264 
05265 /// \brief Check whether the address computation for a non-consecutive memory
05266 /// access looks like an unlikely candidate for being merged into the indexing
05267 /// mode.
05268 ///
05269 /// We look for a GEP which has one index that is an induction variable and all
05270 /// other indices are loop invariant. If the stride of this access is also
05271 /// within a small bound we decide that this address computation can likely be
05272 /// merged into the addressing mode.
05273 /// In all other cases, we identify the address computation as complex.
05274 static bool isLikelyComplexAddressComputation(Value *Ptr,
05275                                               LoopVectorizationLegality *Legal,
05276                                               ScalarEvolution *SE,
05277                                               const Loop *TheLoop) {
05278   GetElementPtrInst *Gep = dyn_cast<GetElementPtrInst>(Ptr);
05279   if (!Gep)
05280     return true;
05281 
05282   // We are looking for a gep with all loop invariant indices except for one
05283   // which should be an induction variable.
05284   unsigned NumOperands = Gep->getNumOperands();
05285   for (unsigned i = 1; i < NumOperands; ++i) {
05286     Value *Opd = Gep->getOperand(i);
05287     if (!SE->isLoopInvariant(SE->getSCEV(Opd), TheLoop) &&
05288         !Legal->isInductionVariable(Opd))
05289       return true;
05290   }
05291 
05292   // Now we know we have a GEP ptr, %inv, %ind, %inv. Make sure that the step
05293   // can likely be merged into the address computation.
05294   unsigned MaxMergeDistance = 64;
05295 
05296   const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(SE->getSCEV(Ptr));
05297   if (!AddRec)
05298     return true;
05299 
05300   // Check the step is constant.
05301   const SCEV *Step = AddRec->getStepRecurrence(*SE);
05302   // Calculate the pointer stride and check if it is consecutive.
05303   const SCEVConstant *C = dyn_cast<SCEVConstant>(Step);
05304   if (!C)
05305     return true;
05306 
05307   const APInt &APStepVal = C->getAPInt();
05308 
05309   // Huge step value - give up.
05310   if (APStepVal.getBitWidth() > 64)
05311     return true;
05312 
05313   int64_t StepVal = APStepVal.getSExtValue();
05314 
05315   return StepVal > MaxMergeDistance;
05316 }
05317 
05318 static bool isStrideMul(Instruction *I, LoopVectorizationLegality *Legal) {
05319   return Legal->hasStride(I->getOperand(0)) ||
05320          Legal->hasStride(I->getOperand(1));
05321 }
05322 
05323 unsigned
05324 LoopVectorizationCostModel::getInstructionCost(Instruction *I, unsigned VF) {
05325   // If we know that this instruction will remain uniform, check the cost of
05326   // the scalar version.
05327   if (Legal->isUniformAfterVectorization(I))
05328     VF = 1;
05329 
05330   Type *RetTy = I->getType();
05331   if (VF > 1 && MinBWs.count(I))
05332     RetTy = IntegerType::get(RetTy->getContext(), MinBWs[I]);
05333   Type *VectorTy = ToVectorTy(RetTy, VF);
05334   auto SE = PSE.getSE();
05335 
05336   // TODO: We need to estimate the cost of intrinsic calls.
05337   switch (I->getOpcode()) {
05338   case Instruction::GetElementPtr:
05339     // We mark this instruction as zero-cost because the cost of GEPs in
05340     // vectorized code depends on whether the corresponding memory instruction
05341     // is scalarized or not. Therefore, we handle GEPs with the memory
05342     // instruction cost.
05343     return 0;
05344   case Instruction::Br: {
05345     return TTI.getCFInstrCost(I->getOpcode());
05346   }
05347   case Instruction::PHI:
05348     //TODO: IF-converted IFs become selects.
05349     return 0;
05350   case Instruction::Add:
05351   case Instruction::FAdd:
05352   case Instruction::Sub:
05353   case Instruction::FSub:
05354   case Instruction::Mul:
05355   case Instruction::FMul:
05356   case Instruction::UDiv:
05357   case Instruction::SDiv:
05358   case Instruction::FDiv:
05359   case Instruction::URem:
05360   case Instruction::SRem:
05361   case Instruction::FRem:
05362   case Instruction::Shl:
05363   case Instruction::LShr:
05364   case Instruction::AShr:
05365   case Instruction::And:
05366   case Instruction::Or:
05367   case Instruction::Xor: {
05368     // Since we will replace the stride by 1 the multiplication should go away.
05369     if (I->getOpcode() == Instruction::Mul && isStrideMul(I, Legal))
05370       return 0;
05371     // Certain instructions can be cheaper to vectorize if they have a constant
05372     // second vector operand. One example of this are shifts on x86.
05373     TargetTransformInfo::OperandValueKind Op1VK =
05374       TargetTransformInfo::OK_AnyValue;
05375     TargetTransformInfo::OperandValueKind Op2VK =
05376       TargetTransformInfo::OK_AnyValue;
05377     TargetTransformInfo::OperandValueProperties Op1VP =
05378         TargetTransformInfo::OP_None;
05379     TargetTransformInfo::OperandValueProperties Op2VP =
05380         TargetTransformInfo::OP_None;
05381     Value *Op2 = I->getOperand(1);
05382 
05383     // Check for a splat of a constant or for a non uniform vector of constants.
05384     if (isa<ConstantInt>(Op2)) {
05385       ConstantInt *CInt = cast<ConstantInt>(Op2);
05386       if (CInt && CInt->getValue().isPowerOf2())
05387         Op2VP = TargetTransformInfo::OP_PowerOf2;
05388       Op2VK = TargetTransformInfo::OK_UniformConstantValue;
05389     } else if (isa<ConstantVector>(Op2) || isa<ConstantDataVector>(Op2)) {
05390       Op2VK = TargetTransformInfo::OK_NonUniformConstantValue;
05391       Constant *SplatValue = cast<Constant>(Op2)->getSplatValue();
05392       if (SplatValue) {
05393         ConstantInt *CInt = dyn_cast<ConstantInt>(SplatValue);
05394         if (CInt && CInt->getValue().isPowerOf2())
05395           Op2VP = TargetTransformInfo::OP_PowerOf2;
05396         Op2VK = TargetTransformInfo::OK_UniformConstantValue;
05397       }
05398     }
05399 
05400     return TTI.getArithmeticInstrCost(I->getOpcode(), VectorTy, Op1VK, Op2VK,
05401                                       Op1VP, Op2VP);
05402   }
05403   case Instruction::Select: {
05404     SelectInst *SI = cast<SelectInst>(I);
05405     const SCEV *CondSCEV = SE->getSCEV(SI->getCondition());
05406     bool ScalarCond = (SE->isLoopInvariant(CondSCEV, TheLoop));
05407     Type *CondTy = SI->getCondition()->getType();
05408     if (!ScalarCond)
05409       CondTy = VectorType::get(CondTy, VF);
05410 
05411     return TTI.getCmpSelInstrCost(I->getOpcode(), VectorTy, CondTy);
05412   }
05413   case Instruction::ICmp:
05414   case Instruction::FCmp: {
05415     Type *ValTy = I->getOperand(0)->getType();
05416     Instruction *Op0AsInstruction = dyn_cast<Instruction>(I->getOperand(0));
05417     auto It = MinBWs.find(Op0AsInstruction);
05418     if (VF > 1 && It != MinBWs.end())
05419       ValTy = IntegerType::get(ValTy->getContext(), It->second);
05420     VectorTy = ToVectorTy(ValTy, VF);
05421     return TTI.getCmpSelInstrCost(I->getOpcode(), VectorTy);
05422   }
05423   case Instruction::Store:
05424   case Instruction::Load: {
05425     StoreInst *SI = dyn_cast<StoreInst>(I);
05426     LoadInst *LI = dyn_cast<LoadInst>(I);
05427     Type *ValTy = (SI ? SI->getValueOperand()->getType() :
05428                    LI->getType());
05429     VectorTy = ToVectorTy(ValTy, VF);
05430 
05431     unsigned Alignment = SI ? SI->getAlignment() : LI->getAlignment();
05432     unsigned AS = SI ? SI->getPointerAddressSpace() :
05433       LI->getPointerAddressSpace();
05434     Value *Ptr = SI ? SI->getPointerOperand() : LI->getPointerOperand();
05435     // We add the cost of address computation here instead of with the gep
05436     // instruction because only here we know whether the operation is
05437     // scalarized.
05438     if (VF == 1)
05439       return TTI.getAddressComputationCost(VectorTy) +
05440         TTI.getMemoryOpCost(I->getOpcode(), VectorTy, Alignment, AS);
05441 
05442     // For an interleaved access, calculate the total cost of the whole
05443     // interleave group.
05444     if (Legal->isAccessInterleaved(I)) {
05445       auto Group = Legal->getInterleavedAccessGroup(I);
05446       assert(Group && "Fail to get an interleaved access group.");
05447 
05448       // Only calculate the cost once at the insert position.
05449       if (Group->getInsertPos() != I)
05450         return 0;
05451 
05452       unsigned InterleaveFactor = Group->getFactor();
05453       Type *WideVecTy =
05454           VectorType::get(VectorTy->getVectorElementType(),
05455                           VectorTy->getVectorNumElements() * InterleaveFactor);
05456 
05457       // Holds the indices of existing members in an interleaved load group.
05458       // An interleaved store group doesn't need this as it dones't allow gaps.
05459       SmallVector<unsigned, 4> Indices;
05460       if (LI) {
05461         for (unsigned i = 0; i < InterleaveFactor; i++)
05462           if (Group->getMember(i))
05463             Indices.push_back(i);
05464       }
05465 
05466       // Calculate the cost of the whole interleaved group.
05467       unsigned Cost = TTI.getInterleavedMemoryOpCost(
05468           I->getOpcode(), WideVecTy, Group->getFactor(), Indices,
05469           Group->getAlignment(), AS);
05470 
05471       if (Group->isReverse())
05472         Cost +=
05473             Group->getNumMembers() *
05474             TTI.getShuffleCost(TargetTransformInfo::SK_Reverse, VectorTy, 0);
05475 
05476       // FIXME: The interleaved load group with a huge gap could be even more
05477       // expensive than scalar operations. Then we could ignore such group and
05478       // use scalar operations instead.
05479       return Cost;
05480     }
05481 
05482     // Scalarized loads/stores.
05483     int ConsecutiveStride = Legal->isConsecutivePtr(Ptr);
05484     bool Reverse = ConsecutiveStride < 0;
05485     const DataLayout &DL = I->getModule()->getDataLayout();
05486     unsigned ScalarAllocatedSize = DL.getTypeAllocSize(ValTy);
05487     unsigned VectorElementSize = DL.getTypeStoreSize(VectorTy) / VF;
05488     if (!ConsecutiveStride || ScalarAllocatedSize != VectorElementSize) {
05489       bool IsComplexComputation =
05490         isLikelyComplexAddressComputation(Ptr, Legal, SE, TheLoop);
05491       unsigned Cost = 0;
05492       // The cost of extracting from the value vector and pointer vector.
05493       Type *PtrTy = ToVectorTy(Ptr->getType(), VF);
05494       for (unsigned i = 0; i < VF; ++i) {
05495         //  The cost of extracting the pointer operand.
05496         Cost += TTI.getVectorInstrCost(Instruction::ExtractElement, PtrTy, i);
05497         // In case of STORE, the cost of ExtractElement from the vector.
05498         // In case of LOAD, the cost of InsertElement into the returned
05499         // vector.
05500         Cost += TTI.getVectorInstrCost(SI ? Instruction::ExtractElement :
05501                                             Instruction::InsertElement,
05502                                             VectorTy, i);
05503       }
05504 
05505       // The cost of the scalar loads/stores.
05506       Cost += VF * TTI.getAddressComputationCost(PtrTy, IsComplexComputation);
05507       Cost += VF * TTI.getMemoryOpCost(I->getOpcode(), ValTy->getScalarType(),
05508                                        Alignment, AS);
05509       return Cost;
05510     }
05511 
05512     // Wide load/stores.
05513     unsigned Cost = TTI.getAddressComputationCost(VectorTy);
05514     if (Legal->isMaskRequired(I))
05515       Cost += TTI.getMaskedMemoryOpCost(I->getOpcode(), VectorTy, Alignment,
05516                                         AS);
05517     else
05518       Cost += TTI.getMemoryOpCost(I->getOpcode(), VectorTy, Alignment, AS);
05519 
05520     if (Reverse)
05521       Cost += TTI.getShuffleCost(TargetTransformInfo::SK_Reverse,
05522                                   VectorTy, 0);
05523     return Cost;
05524   }
05525   case Instruction::ZExt:
05526   case Instruction::SExt:
05527   case Instruction::FPToUI:
05528   case Instruction::FPToSI:
05529   case Instruction::FPExt:
05530   case Instruction::PtrToInt:
05531   case Instruction::IntToPtr:
05532   case Instruction::SIToFP:
05533   case Instruction::UIToFP:
05534   case Instruction::Trunc:
05535   case Instruction::FPTrunc:
05536   case Instruction::BitCast: {
05537     // We optimize the truncation of induction variable.
05538     // The cost of these is the same as the scalar operation.
05539     if (I->getOpcode() == Instruction::Trunc &&
05540         Legal->isInductionVariable(I->getOperand(0)))
05541       return TTI.getCastInstrCost(I->getOpcode(), I->getType(),
05542                                   I->getOperand(0)->getType());
05543     
05544     Type *SrcScalarTy = I->getOperand(0)->getType();
05545     Type *SrcVecTy = ToVectorTy(SrcScalarTy, VF);
05546     if (VF > 1 && MinBWs.count(I)) {
05547       // This cast is going to be shrunk. This may remove the cast or it might
05548       // turn it into slightly different cast. For example, if MinBW == 16,
05549       // "zext i8 %1 to i32" becomes "zext i8 %1 to i16".
05550       //
05551       // Calculate the modified src and dest types.
05552       Type *MinVecTy = VectorTy;
05553       if (I->getOpcode() == Instruction::Trunc) {
05554         SrcVecTy = smallestIntegerVectorType(SrcVecTy, MinVecTy);
05555         VectorTy = largestIntegerVectorType(ToVectorTy(I->getType(), VF),
05556                                             MinVecTy);
05557       } else if (I->getOpcode() == Instruction::ZExt ||
05558                  I->getOpcode() == Instruction::SExt) {
05559         SrcVecTy = largestIntegerVectorType(SrcVecTy, MinVecTy);
05560         VectorTy = smallestIntegerVectorType(ToVectorTy(I->getType(), VF),
05561                                              MinVecTy);
05562       }
05563     }
05564     
05565     return TTI.getCastInstrCost(I->getOpcode(), VectorTy, SrcVecTy);
05566   }
05567   case Instruction::Call: {
05568     bool NeedToScalarize;
05569     CallInst *CI = cast<CallInst>(I);
05570     unsigned CallCost = getVectorCallCost(CI, VF, TTI, TLI, NeedToScalarize);
05571     if (getIntrinsicIDForCall(CI, TLI))
05572       return std::min(CallCost, getVectorIntrinsicCost(CI, VF, TTI, TLI));
05573     return CallCost;
05574   }
05575   default: {
05576     // We are scalarizing the instruction. Return the cost of the scalar
05577     // instruction, plus the cost of insert and extract into vector
05578     // elements, times the vector width.
05579     unsigned Cost = 0;
05580 
05581     if (!RetTy->isVoidTy() && VF != 1) {
05582       unsigned InsCost = TTI.getVectorInstrCost(Instruction::InsertElement,
05583                                                 VectorTy);
05584       unsigned ExtCost = TTI.getVectorInstrCost(Instruction::ExtractElement,
05585                                                 VectorTy);
05586 
05587       // The cost of inserting the results plus extracting each one of the
05588       // operands.
05589       Cost += VF * (InsCost + ExtCost * I->getNumOperands());
05590     }
05591 
05592     // The cost of executing VF copies of the scalar instruction. This opcode
05593     // is unknown. Assume that it is the same as 'mul'.
05594     Cost += VF * TTI.getArithmeticInstrCost(Instruction::Mul, VectorTy);
05595     return Cost;
05596   }
05597   }// end of switch.
05598 }
05599 
05600 char LoopVectorize::ID = 0;
05601 static const char lv_name[] = "Loop Vectorization";
05602 INITIALIZE_PASS_BEGIN(LoopVectorize, LV_NAME, lv_name, false, false)
05603 INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)
05604 INITIALIZE_PASS_DEPENDENCY(BasicAAWrapperPass)
05605 INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass)
05606 INITIALIZE_PASS_DEPENDENCY(GlobalsAAWrapperPass)
05607 INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)
05608 INITIALIZE_PASS_DEPENDENCY(BlockFrequencyInfoWrapperPass)
05609 INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
05610 INITIALIZE_PASS_DEPENDENCY(ScalarEvolutionWrapperPass)
05611 INITIALIZE_PASS_DEPENDENCY(LCSSA)
05612 INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass)
05613 INITIALIZE_PASS_DEPENDENCY(LoopSimplify)
05614 INITIALIZE_PASS_DEPENDENCY(LoopAccessAnalysis)
05615 INITIALIZE_PASS_DEPENDENCY(DemandedBits)
05616 INITIALIZE_PASS_END(LoopVectorize, LV_NAME, lv_name, false, false)
05617 
05618 namespace llvm {
05619   Pass *createLoopVectorizePass(bool NoUnrolling, bool AlwaysVectorize) {
05620     return new LoopVectorize(NoUnrolling, AlwaysVectorize);
05621   }
05622 }
05623 
05624 bool LoopVectorizationCostModel::isConsecutiveLoadOrStore(Instruction *Inst) {
05625   // Check for a store.
05626   if (StoreInst *ST = dyn_cast<StoreInst>(Inst))
05627     return Legal->isConsecutivePtr(ST->getPointerOperand()) != 0;
05628 
05629   // Check for a load.
05630   if (LoadInst *LI = dyn_cast<LoadInst>(Inst))
05631     return Legal->isConsecutivePtr(LI->getPointerOperand()) != 0;
05632 
05633   return false;
05634 }
05635 
05636 void LoopVectorizationCostModel::collectValuesToIgnore() {
05637   // Ignore ephemeral values.
05638   CodeMetrics::collectEphemeralValues(TheLoop, AC, ValuesToIgnore);
05639 
05640   // Ignore type-promoting instructions we identified during reduction
05641   // detection.
05642   for (auto &Reduction : *Legal->getReductionVars()) {
05643     RecurrenceDescriptor &RedDes = Reduction.second;
05644     SmallPtrSetImpl<Instruction *> &Casts = RedDes.getCastInsts();
05645     VecValuesToIgnore.insert(Casts.begin(), Casts.end());
05646   }
05647 
05648   // Ignore induction phis that are only used in either GetElementPtr or ICmp
05649   // instruction to exit loop. Induction variables usually have large types and
05650   // can have big impact when estimating register usage.
05651   // This is for when VF > 1.
05652   for (auto &Induction : *Legal->getInductionVars()) {
05653     auto *PN = Induction.first;
05654     auto *UpdateV = PN->getIncomingValueForBlock(TheLoop->getLoopLatch());
05655 
05656     // Check that the PHI is only used by the induction increment (UpdateV) or
05657     // by GEPs. Then check that UpdateV is only used by a compare instruction or
05658     // the loop header PHI.
05659     // FIXME: Need precise def-use analysis to determine if this instruction
05660     // variable will be vectorized.
05661     if (std::all_of(PN->user_begin(), PN->user_end(),
05662                     [&](const User *U) -> bool {
05663                       return U == UpdateV || isa<GetElementPtrInst>(U);
05664                     }) &&
05665         std::all_of(UpdateV->user_begin(), UpdateV->user_end(),
05666                     [&](const User *U) -> bool {
05667                       return U == PN || isa<ICmpInst>(U);
05668                     })) {
05669       VecValuesToIgnore.insert(PN);
05670       VecValuesToIgnore.insert(UpdateV);
05671     }
05672   }
05673 
05674   // Ignore instructions that will not be vectorized.
05675   // This is for when VF > 1.
05676   for (auto bb = TheLoop->block_begin(), be = TheLoop->block_end(); bb != be;
05677        ++bb) {
05678     for (auto &Inst : **bb) {
05679       switch (Inst.getOpcode()) {
05680       case Instruction::GetElementPtr: {
05681         // Ignore GEP if its last operand is an induction variable so that it is
05682         // a consecutive load/store and won't be vectorized as scatter/gather
05683         // pattern.
05684 
05685         GetElementPtrInst *Gep = cast<GetElementPtrInst>(&Inst);
05686         unsigned NumOperands = Gep->getNumOperands();
05687         unsigned InductionOperand = getGEPInductionOperand(Gep);
05688         bool GepToIgnore = true;
05689 
05690         // Check that all of the gep indices are uniform except for the
05691         // induction operand.
05692         for (unsigned i = 0; i != NumOperands; ++i) {
05693           if (i != InductionOperand &&
05694               !PSE.getSE()->isLoopInvariant(PSE.getSCEV(Gep->getOperand(i)),
05695                                             TheLoop)) {
05696             GepToIgnore = false;
05697             break;
05698           }
05699         }
05700 
05701         if (GepToIgnore)
05702           VecValuesToIgnore.insert(&Inst);
05703         break;
05704       }
05705       }
05706     }
05707   }
05708 }
05709 
05710 void InnerLoopUnroller::scalarizeInstruction(Instruction *Instr,
05711                                              bool IfPredicateStore) {
05712   assert(!Instr->getType()->isAggregateType() && "Can't handle vectors");
05713   // Holds vector parameters or scalars, in case of uniform vals.
05714   SmallVector<VectorParts, 4> Params;
05715 
05716   setDebugLocFromInst(Builder, Instr);
05717 
05718   // Find all of the vectorized parameters.
05719   for (unsigned op = 0, e = Instr->getNumOperands(); op != e; ++op) {
05720     Value *SrcOp = Instr->getOperand(op);
05721 
05722     // If we are accessing the old induction variable, use the new one.
05723     if (SrcOp == OldInduction) {
05724       Params.push_back(getVectorValue(SrcOp));
05725       continue;
05726     }
05727 
05728     // Try using previously calculated values.
05729     Instruction *SrcInst = dyn_cast<Instruction>(SrcOp);
05730 
05731     // If the src is an instruction that appeared earlier in the basic block
05732     // then it should already be vectorized.
05733     if (SrcInst && OrigLoop->contains(SrcInst)) {
05734       assert(WidenMap.has(SrcInst) && "Source operand is unavailable");
05735       // The parameter is a vector value from earlier.
05736       Params.push_back(WidenMap.get(SrcInst));
05737     } else {
05738       // The parameter is a scalar from outside the loop. Maybe even a constant.
05739       VectorParts Scalars;
05740       Scalars.append(UF, SrcOp);
05741       Params.push_back(Scalars);
05742     }
05743   }
05744 
05745   assert(Params.size() == Instr->getNumOperands() &&
05746          "Invalid number of operands");
05747 
05748   // Does this instruction return a value ?
05749   bool IsVoidRetTy = Instr->getType()->isVoidTy();
05750 
05751   Value *UndefVec = IsVoidRetTy ? nullptr :
05752   UndefValue::get(Instr->getType());
05753   // Create a new entry in the WidenMap and initialize it to Undef or Null.
05754   VectorParts &VecResults = WidenMap.splat(Instr, UndefVec);
05755 
05756   VectorParts Cond;
05757   if (IfPredicateStore) {
05758     assert(Instr->getParent()->getSinglePredecessor() &&
05759            "Only support single predecessor blocks");
05760     Cond = createEdgeMask(Instr->getParent()->getSinglePredecessor(),
05761                           Instr->getParent());
05762   }
05763 
05764   // For each vector unroll 'part':
05765   for (unsigned Part = 0; Part < UF; ++Part) {
05766     // For each scalar that we create:
05767 
05768     // Start an "if (pred) a[i] = ..." block.
05769     Value *Cmp = nullptr;
05770     if (IfPredicateStore) {
05771       if (Cond[Part]->getType()->isVectorTy())
05772         Cond[Part] =
05773             Builder.CreateExtractElement(Cond[Part], Builder.getInt32(0));
05774       Cmp = Builder.CreateICmp(ICmpInst::ICMP_EQ, Cond[Part],
05775                                ConstantInt::get(Cond[Part]->getType(), 1));
05776     }
05777 
05778     Instruction *Cloned = Instr->clone();
05779       if (!IsVoidRetTy)
05780         Cloned->setName(Instr->getName() + ".cloned");
05781       // Replace the operands of the cloned instructions with extracted scalars.
05782       for (unsigned op = 0, e = Instr->getNumOperands(); op != e; ++op) {
05783         Value *Op = Params[op][Part];
05784         Cloned->setOperand(op, Op);
05785       }
05786 
05787       // Place the cloned scalar in the new loop.
05788       Builder.Insert(Cloned);
05789 
05790       // If the original scalar returns a value we need to place it in a vector
05791       // so that future users will be able to use it.
05792       if (!IsVoidRetTy)
05793         VecResults[Part] = Cloned;
05794 
05795       // End if-block.
05796       if (IfPredicateStore)
05797         PredicatedStores.push_back(std::make_pair(cast<StoreInst>(Cloned),
05798                                                   Cmp));
05799   }
05800 }
05801 
05802 void InnerLoopUnroller::vectorizeMemoryInstruction(Instruction *Instr) {
05803   StoreInst *SI = dyn_cast<StoreInst>(Instr);
05804   bool IfPredicateStore = (SI && Legal->blockNeedsPredication(SI->getParent()));
05805 
05806   return scalarizeInstruction(Instr, IfPredicateStore);
05807 }
05808 
05809 Value *InnerLoopUnroller::reverseVector(Value *Vec) {
05810   return Vec;
05811 }
05812 
05813 Value *InnerLoopUnroller::getBroadcastInstrs(Value *V) {
05814   return V;
05815 }
05816 
05817 Value *InnerLoopUnroller::getStepVector(Value *Val, int StartIdx, Value *Step) {
05818   // When unrolling and the VF is 1, we only need to add a simple scalar.
05819   Type *ITy = Val->getType();
05820   assert(!ITy->isVectorTy() && "Val must be a scalar");
05821   Constant *C = ConstantInt::get(ITy, StartIdx);
05822   return Builder.CreateAdd(Val, Builder.CreateMul(C, Step), "induction");
05823 }