/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

Bug Summary

File:	llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
Warning:	line 8682, column 5 Called C++ object pointer is null
Annotated Source Code

Press '?' to see keyboard shortcuts
Show analyzer invocation
clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name LoopVectorize.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -fhalf-no-semantic-interposition -mframe-pointer=none -fmath-errno -fno-rounding-math -mconstructor-aliases -munwind-tables -target-cpu x86-64 -tune-cpu generic -debugger-tuning=gdb -ffunction-sections -fdata-sections -fcoverage-compilation-dir=/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/build-llvm/lib/Transforms/Vectorize -resource-dir /usr/lib/llvm-13/lib/clang/13.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/build-llvm/lib/Transforms/Vectorize -I /build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize -I /build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/build-llvm/include -I /build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/include -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0/backward -internal-isystem /usr/lib/llvm-13/lib/clang/13.0.0/include -internal-isystem /usr/local/include -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../x86_64-linux-gnu/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-comment -std=c++14 -fdeprecated-macro -fdebug-compilation-dir=/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/build-llvm/lib/Transforms/Vectorize -fdebug-prefix-map=/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f=. -ferror-limit 19 -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -D__GCC_HAVE_DWARF2_CFI_ASM=1 -o /tmp/scan-build-2021-04-14-063029-18377-1 -x c++ /build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
1//===- LoopVectorize.cpp - A Loop Vectorizer ------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This is the LLVM loop vectorizer. This pass modifies 'vectorizable' loops
10// and generates target-independent LLVM-IR.
11// The vectorizer uses the TargetTransformInfo analysis to estimate the costs
12// of instructions in order to estimate the profitability of vectorization.
13//
14// The loop vectorizer combines consecutive loop iterations into a single
15// 'wide' iteration. After this transformation the index is incremented
16// by the SIMD vector width, and not by one.
17//
18// This pass has three parts:
19// 1. The main loop pass that drives the different parts.
20// 2. LoopVectorizationLegality - A unit that checks for the legality
21//    of the vectorization.
22// 3. InnerLoopVectorizer - A unit that performs the actual
23//    widening of instructions.
24// 4. LoopVectorizationCostModel - A unit that checks for the profitability
25//    of vectorization. It decides on the optimal vector width, which
26//    can be one, if vectorization is not profitable.
27//
28// There is a development effort going on to migrate loop vectorizer to the
29// VPlan infrastructure and to introduce outer loop vectorization support (see
30// docs/Proposal/VectorizationPlan.rst and
31// http://lists.llvm.org/pipermail/llvm-dev/2017-December/119523.html). For this
32// purpose, we temporarily introduced the VPlan-native vectorization path: an
33// alternative vectorization path that is natively implemented on top of the
34// VPlan infrastructure. See EnableVPlanNativePath for enabling.
35//
36//===----------------------------------------------------------------------===//
37//
38// The reduction-variable vectorization is based on the paper:
39//  D. Nuzman and R. Henderson. Multi-platform Auto-vectorization.
40//
41// Variable uniformity checks are inspired by:
42//  Karrenberg, R. and Hack, S. Whole Function Vectorization.
43//
44// The interleaved access vectorization is based on the paper:
45//  Dorit Nuzman, Ira Rosen and Ayal Zaks.  Auto-Vectorization of Interleaved
46//  Data for SIMD
47//
48// Other ideas/concepts are from:
49//  A. Zaks and D. Nuzman. Autovectorization in GCC-two years later.
50//
51//  S. Maleki, Y. Gao, M. Garzaran, T. Wong and D. Padua.  An Evaluation of
52//  Vectorizing Compilers.
53//
54//===----------------------------------------------------------------------===//

56#include "llvm/Transforms/Vectorize/LoopVectorize.h"
57#include "LoopVectorizationPlanner.h"
58#include "VPRecipeBuilder.h"
59#include "VPlan.h"
60#include "VPlanHCFGBuilder.h"
61#include "VPlanPredicator.h"
62#include "VPlanTransforms.h"
63#include "llvm/ADT/APInt.h"
64#include "llvm/ADT/ArrayRef.h"
65#include "llvm/ADT/DenseMap.h"
66#include "llvm/ADT/DenseMapInfo.h"
67#include "llvm/ADT/Hashing.h"
68#include "llvm/ADT/MapVector.h"
69#include "llvm/ADT/None.h"
70#include "llvm/ADT/Optional.h"
71#include "llvm/ADT/STLExtras.h"
72#include "llvm/ADT/SmallPtrSet.h"
73#include "llvm/ADT/SmallVector.h"
74#include "llvm/ADT/Statistic.h"
75#include "llvm/ADT/StringRef.h"
76#include "llvm/ADT/Twine.h"
77#include "llvm/ADT/iterator_range.h"
78#include "llvm/Analysis/AssumptionCache.h"
79#include "llvm/Analysis/BasicAliasAnalysis.h"
80#include "llvm/Analysis/BlockFrequencyInfo.h"
81#include "llvm/Analysis/CFG.h"
82#include "llvm/Analysis/CodeMetrics.h"
83#include "llvm/Analysis/DemandedBits.h"
84#include "llvm/Analysis/GlobalsModRef.h"
85#include "llvm/Analysis/LoopAccessAnalysis.h"
86#include "llvm/Analysis/LoopAnalysisManager.h"
87#include "llvm/Analysis/LoopInfo.h"
88#include "llvm/Analysis/LoopIterator.h"
89#include "llvm/Analysis/MemorySSA.h"
90#include "llvm/Analysis/OptimizationRemarkEmitter.h"
91#include "llvm/Analysis/ProfileSummaryInfo.h"
92#include "llvm/Analysis/ScalarEvolution.h"
93#include "llvm/Analysis/ScalarEvolutionExpressions.h"
94#include "llvm/Analysis/TargetLibraryInfo.h"
95#include "llvm/Analysis/TargetTransformInfo.h"
96#include "llvm/Analysis/VectorUtils.h"
97#include "llvm/IR/Attributes.h"
98#include "llvm/IR/BasicBlock.h"
99#include "llvm/IR/CFG.h"
100#include "llvm/IR/Constant.h"
101#include "llvm/IR/Constants.h"
102#include "llvm/IR/DataLayout.h"
103#include "llvm/IR/DebugInfoMetadata.h"
104#include "llvm/IR/DebugLoc.h"
105#include "llvm/IR/DerivedTypes.h"
106#include "llvm/IR/DiagnosticInfo.h"
107#include "llvm/IR/Dominators.h"
108#include "llvm/IR/Function.h"
109#include "llvm/IR/IRBuilder.h"
110#include "llvm/IR/InstrTypes.h"
111#include "llvm/IR/Instruction.h"
112#include "llvm/IR/Instructions.h"
113#include "llvm/IR/IntrinsicInst.h"
114#include "llvm/IR/Intrinsics.h"
115#include "llvm/IR/LLVMContext.h"
116#include "llvm/IR/Metadata.h"
117#include "llvm/IR/Module.h"
118#include "llvm/IR/Operator.h"
119#include "llvm/IR/PatternMatch.h"
120#include "llvm/IR/Type.h"
121#include "llvm/IR/Use.h"
122#include "llvm/IR/User.h"
123#include "llvm/IR/Value.h"
124#include "llvm/IR/ValueHandle.h"
125#include "llvm/IR/Verifier.h"
126#include "llvm/InitializePasses.h"
127#include "llvm/Pass.h"
128#include "llvm/Support/Casting.h"
129#include "llvm/Support/CommandLine.h"
130#include "llvm/Support/Compiler.h"
131#include "llvm/Support/Debug.h"
132#include "llvm/Support/ErrorHandling.h"
133#include "llvm/Support/InstructionCost.h"
134#include "llvm/Support/MathExtras.h"
135#include "llvm/Support/raw_ostream.h"
136#include "llvm/Transforms/Utils/BasicBlockUtils.h"
137#include "llvm/Transforms/Utils/InjectTLIMappings.h"
138#include "llvm/Transforms/Utils/LoopSimplify.h"
139#include "llvm/Transforms/Utils/LoopUtils.h"
140#include "llvm/Transforms/Utils/LoopVersioning.h"
141#include "llvm/Transforms/Utils/ScalarEvolutionExpander.h"
142#include "llvm/Transforms/Utils/SizeOpts.h"
143#include "llvm/Transforms/Vectorize/LoopVectorizationLegality.h"
144#include <algorithm>
145#include <cassert>
146#include <cstdint>
147#include <cstdlib>
148#include <functional>
149#include <iterator>
150#include <limits>
151#include <memory>
152#include <string>
153#include <tuple>
154#include <utility>

156using namespace llvm;

158#define LV_NAME"loop-vectorize" "loop-vectorize"
159#define DEBUG_TYPE"loop-vectorize" LV_NAME"loop-vectorize"

161#ifndef NDEBUG
162const char VerboseDebug[] = DEBUG_TYPE"loop-vectorize" "-verbose";
163#endif

165/// @{
166/// Metadata attribute names
167const char LLVMLoopVectorizeFollowupAll[] = "llvm.loop.vectorize.followup_all";
168const char LLVMLoopVectorizeFollowupVectorized[] =
  "llvm.loop.vectorize.followup_vectorized";
170const char LLVMLoopVectorizeFollowupEpilogue[] =
  "llvm.loop.vectorize.followup_epilogue";
172/// @}

174STATISTIC(LoopsVectorized, "Number of loops vectorized")static llvm::Statistic LoopsVectorized = {"loop-vectorize", "LoopsVectorized"
, "Number of loops vectorized"};
175STATISTIC(LoopsAnalyzed, "Number of loops analyzed for vectorization")static llvm::Statistic LoopsAnalyzed = {"loop-vectorize", "LoopsAnalyzed"
, "Number of loops analyzed for vectorization"};
176STATISTIC(LoopsEpilogueVectorized, "Number of epilogues vectorized")static llvm::Statistic LoopsEpilogueVectorized = {"loop-vectorize"
, "LoopsEpilogueVectorized", "Number of epilogues vectorized"
};

178static cl::opt<bool> EnableEpilogueVectorization(
  "enable-epilogue-vectorization", cl::init(true), cl::Hidden,
  cl::desc("Enable vectorization of epilogue loops."));

182static cl::opt<unsigned> EpilogueVectorizationForceVF(
  "epilogue-vectorization-force-VF", cl::init(1), cl::Hidden,
  cl::desc("When epilogue vectorization is enabled, and a value greater than "
           "1 is specified, forces the given VF for all applicable epilogue "
           "loops."));

188static cl::opt<unsigned> EpilogueVectorizationMinVF(
  "epilogue-vectorization-minimum-VF", cl::init(16), cl::Hidden,
  cl::desc("Only loops with vectorization factor equal to or larger than "
           "the specified value are considered for epilogue vectorization."));

193/// Loops with a known constant trip count below this number are vectorized only
194/// if no scalar iteration overheads are incurred.
195static cl::opt<unsigned> TinyTripCountVectorThreshold(
  "vectorizer-min-trip-count", cl::init(16), cl::Hidden,
  cl::desc("Loops with a constant trip count that is smaller than this "
           "value are vectorized only if no scalar iteration overheads "
           "are incurred."));

201static cl::opt<unsigned> PragmaVectorizeMemoryCheckThreshold(
  "pragma-vectorize-memory-check-threshold", cl::init(128), cl::Hidden,
  cl::desc("The maximum allowed number of runtime memory checks with a "
           "vectorize(enable) pragma."));

206// Option prefer-predicate-over-epilogue indicates that an epilogue is undesired,
207// that predication is preferred, and this lists all options. I.e., the
208// vectorizer will try to fold the tail-loop (epilogue) into the vector body
209// and predicate the instructions accordingly. If tail-folding fails, there are
210// different fallback strategies depending on these values:
211namespace PreferPredicateTy {
enum Option {
  ScalarEpilogue = 0,
  PredicateElseScalarEpilogue,
  PredicateOrDontVectorize
};
217} // namespace PreferPredicateTy

219static cl::opt<PreferPredicateTy::Option> PreferPredicateOverEpilogue(
  "prefer-predicate-over-epilogue",
  cl::init(PreferPredicateTy::ScalarEpilogue),
  cl::Hidden,
  cl::desc("Tail-folding and predication preferences over creating a scalar "
           "epilogue loop."),
  cl::values(clEnumValN(PreferPredicateTy::ScalarEpilogue,llvm::cl::OptionEnumValue { "scalar-epilogue", int(PreferPredicateTy
::ScalarEpilogue), "Don't tail-predicate loops, create scalar epilogue"
 }
                       "scalar-epilogue",llvm::cl::OptionEnumValue { "scalar-epilogue", int(PreferPredicateTy
::ScalarEpilogue), "Don't tail-predicate loops, create scalar epilogue"
 }
                       "Don't tail-predicate loops, create scalar epilogue")llvm::cl::OptionEnumValue { "scalar-epilogue", int(PreferPredicateTy
::ScalarEpilogue), "Don't tail-predicate loops, create scalar epilogue"
 },
            clEnumValN(PreferPredicateTy::PredicateElseScalarEpilogue,llvm::cl::OptionEnumValue { "predicate-else-scalar-epilogue",
 int(PreferPredicateTy::PredicateElseScalarEpilogue), "prefer tail-folding, create scalar epilogue if tail "
 "folding fails." }
                       "predicate-else-scalar-epilogue",llvm::cl::OptionEnumValue { "predicate-else-scalar-epilogue",
 int(PreferPredicateTy::PredicateElseScalarEpilogue), "prefer tail-folding, create scalar epilogue if tail "
 "folding fails." }
                       "prefer tail-folding, create scalar epilogue if tail "llvm::cl::OptionEnumValue { "predicate-else-scalar-epilogue",
 int(PreferPredicateTy::PredicateElseScalarEpilogue), "prefer tail-folding, create scalar epilogue if tail "
 "folding fails." }
                       "folding fails.")llvm::cl::OptionEnumValue { "predicate-else-scalar-epilogue",
 int(PreferPredicateTy::PredicateElseScalarEpilogue), "prefer tail-folding, create scalar epilogue if tail "
 "folding fails." },
            clEnumValN(PreferPredicateTy::PredicateOrDontVectorize,llvm::cl::OptionEnumValue { "predicate-dont-vectorize", int(PreferPredicateTy
::PredicateOrDontVectorize), "prefers tail-folding, don't attempt vectorization if "
 "tail-folding fails." }
                       "predicate-dont-vectorize",llvm::cl::OptionEnumValue { "predicate-dont-vectorize", int(PreferPredicateTy
::PredicateOrDontVectorize), "prefers tail-folding, don't attempt vectorization if "
 "tail-folding fails." }
                       "prefers tail-folding, don't attempt vectorization if "llvm::cl::OptionEnumValue { "predicate-dont-vectorize", int(PreferPredicateTy
::PredicateOrDontVectorize), "prefers tail-folding, don't attempt vectorization if "
 "tail-folding fails." }
                       "tail-folding fails.")llvm::cl::OptionEnumValue { "predicate-dont-vectorize", int(PreferPredicateTy
::PredicateOrDontVectorize), "prefers tail-folding, don't attempt vectorization if "
 "tail-folding fails." }));

237static cl::opt<bool> MaximizeBandwidth(
  "vectorizer-maximize-bandwidth", cl::init(false), cl::Hidden,
  cl::desc("Maximize bandwidth when selecting vectorization factor which "
           "will be determined by the smallest type in loop."));

242static cl::opt<bool> EnableInterleavedMemAccesses(
  "enable-interleaved-mem-accesses", cl::init(false), cl::Hidden,
  cl::desc("Enable vectorization on interleaved memory accesses in a loop"));

246/// An interleave-group may need masking if it resides in a block that needs
247/// predication, or in order to mask away gaps.
248static cl::opt<bool> EnableMaskedInterleavedMemAccesses(
  "enable-masked-interleaved-mem-accesses", cl::init(false), cl::Hidden,
  cl::desc("Enable vectorization on masked interleaved memory accesses in a loop"));

252static cl::opt<unsigned> TinyTripCountInterleaveThreshold(
  "tiny-trip-count-interleave-threshold", cl::init(128), cl::Hidden,
  cl::desc("We don't interleave loops with a estimated constant trip count "
           "below this number"));

257static cl::opt<unsigned> ForceTargetNumScalarRegs(
  "force-target-num-scalar-regs", cl::init(0), cl::Hidden,
  cl::desc("A flag that overrides the target's number of scalar registers."));

261static cl::opt<unsigned> ForceTargetNumVectorRegs(
  "force-target-num-vector-regs", cl::init(0), cl::Hidden,
  cl::desc("A flag that overrides the target's number of vector registers."));

265static cl::opt<unsigned> ForceTargetMaxScalarInterleaveFactor(
  "force-target-max-scalar-interleave", cl::init(0), cl::Hidden,
  cl::desc("A flag that overrides the target's max interleave factor for "
           "scalar loops."));

270static cl::opt<unsigned> ForceTargetMaxVectorInterleaveFactor(
  "force-target-max-vector-interleave", cl::init(0), cl::Hidden,
  cl::desc("A flag that overrides the target's max interleave factor for "
           "vectorized loops."));

275static cl::opt<unsigned> ForceTargetInstructionCost(
  "force-target-instruction-cost", cl::init(0), cl::Hidden,
  cl::desc("A flag that overrides the target's expected cost for "
           "an instruction to a single constant value. Mostly "
           "useful for getting consistent testing."));

281static cl::opt<bool> ForceTargetSupportsScalableVectors(
  "force-target-supports-scalable-vectors", cl::init(false), cl::Hidden,
  cl::desc(
      "Pretend that scalable vectors are supported, even if the target does "
      "not support them. This flag should only be used for testing."));

287static cl::opt<unsigned> SmallLoopCost(
  "small-loop-cost", cl::init(20), cl::Hidden,
  cl::desc(
      "The cost of a loop that is considered 'small' by the interleaver."));

292static cl::opt<bool> LoopVectorizeWithBlockFrequency(
  "loop-vectorize-with-block-frequency", cl::init(true), cl::Hidden,
  cl::desc("Enable the use of the block frequency analysis to access PGO "
           "heuristics minimizing code growth in cold regions and being more "
           "aggressive in hot regions."));

298// Runtime interleave loops for load/store throughput.
299static cl::opt<bool> EnableLoadStoreRuntimeInterleave(
  "enable-loadstore-runtime-interleave", cl::init(true), cl::Hidden,
  cl::desc(
      "Enable runtime interleaving until load/store ports are saturated"));

304/// Interleave small loops with scalar reductions.
305static cl::opt<bool> InterleaveSmallLoopScalarReduction(
  "interleave-small-loop-scalar-reduction", cl::init(false), cl::Hidden,
  cl::desc("Enable interleaving for loops with small iteration counts that "
           "contain scalar reductions to expose ILP."));

310/// The number of stores in a loop that are allowed to need predication.
311static cl::opt<unsigned> NumberOfStoresToPredicate(
  "vectorize-num-stores-pred", cl::init(1), cl::Hidden,
  cl::desc("Max number of stores to be predicated behind an if."));

315static cl::opt<bool> EnableIndVarRegisterHeur(
  "enable-ind-var-reg-heur", cl::init(true), cl::Hidden,
  cl::desc("Count the induction variable only once when interleaving"));

319static cl::opt<bool> EnableCondStoresVectorization(
  "enable-cond-stores-vec", cl::init(true), cl::Hidden,
  cl::desc("Enable if predication of stores during vectorization."));

323static cl::opt<unsigned> MaxNestedScalarReductionIC(
  "max-nested-scalar-reduction-interleave", cl::init(2), cl::Hidden,
  cl::desc("The maximum interleave count to use when interleaving a scalar "
           "reduction in a nested loop."));

328static cl::opt<bool>
  PreferInLoopReductions("prefer-inloop-reductions", cl::init(false),
                         cl::Hidden,
                         cl::desc("Prefer in-loop vector reductions, "
                                  "overriding the targets preference."));

334cl::opt<bool> EnableStrictReductions(
  "enable-strict-reductions", cl::init(false), cl::Hidden,
  cl::desc("Enable the vectorisation of loops with in-order (strict) "
           "FP reductions"));

339static cl::opt<bool> PreferPredicatedReductionSelect(
  "prefer-predicated-reduction-select", cl::init(false), cl::Hidden,
  cl::desc(
      "Prefer predicating a reduction operation over an after loop select."));

344cl::opt<bool> EnableVPlanNativePath(
  "enable-vplan-native-path", cl::init(false), cl::Hidden,
  cl::desc("Enable VPlan-native vectorization path with "
           "support for outer loop vectorization."));

349// FIXME: Remove this switch once we have divergence analysis. Currently we
350// assume divergent non-backedge branches when this switch is true.
351cl::opt<bool> EnableVPlanPredication(
  "enable-vplan-predication", cl::init(false), cl::Hidden,
  cl::desc("Enable VPlan-native vectorization path predicator with "
           "support for outer loop vectorization."));

356// This flag enables the stress testing of the VPlan H-CFG construction in the
357// VPlan-native vectorization path. It must be used in conjuction with
358// -enable-vplan-native-path. -vplan-verify-hcfg can also be used to enable the
359// verification of the H-CFGs built.
360static cl::opt<bool> VPlanBuildStressTest(
  "vplan-build-stress-test", cl::init(false), cl::Hidden,
  cl::desc(
      "Build VPlan for every supported loop nest in the function and bail "
      "out right after the build (stress test the VPlan H-CFG construction "
      "in the VPlan-native vectorization path)."));

367cl::opt<bool> llvm::EnableLoopInterleaving(
  "interleave-loops", cl::init(true), cl::Hidden,
  cl::desc("Enable loop interleaving in Loop vectorization passes"));
370cl::opt<bool> llvm::EnableLoopVectorization(
  "vectorize-loops", cl::init(true), cl::Hidden,
  cl::desc("Run the Loop vectorization passes"));

374cl::opt<bool> PrintVPlansInDotFormat(
  "vplan-print-in-dot-format", cl::init(false), cl::Hidden,
  cl::desc("Use dot format instead of plain text when dumping VPlans"));

378/// A helper function that returns the type of loaded or stored value.
379static Type *getMemInstValueType(Value *I) {
assert((isa<LoadInst>(I) || isa<StoreInst>(I)) &&(((isa<LoadInst>(I) || isa<StoreInst>(I)) &&
 "Expected Load or Store instruction") ? static_cast<void>
 (0) : __assert_fail ("(isa<LoadInst>(I) || isa<StoreInst>(I)) && \"Expected Load or Store instruction\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 381, __PRETTY_FUNCTION__))
       "Expected Load or Store instruction")(((isa<LoadInst>(I) || isa<StoreInst>(I)) &&
 "Expected Load or Store instruction") ? static_cast<void>
 (0) : __assert_fail ("(isa<LoadInst>(I) || isa<StoreInst>(I)) && \"Expected Load or Store instruction\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 381, __PRETTY_FUNCTION__));
if (auto *LI = dyn_cast<LoadInst>(I))
  return LI->getType();
return cast<StoreInst>(I)->getValueOperand()->getType();
385}

387/// A helper function that returns true if the given type is irregular. The
388/// type is irregular if its allocated size doesn't equal the store size of an
389/// element of the corresponding vector type.
390static bool hasIrregularType(Type *Ty, const DataLayout &DL) {
// Determine if an array of N elements of type Ty is "bitcast compatible"
// with a <N x Ty> vector.
// This is only true if there is no padding between the array elements.
return DL.getTypeAllocSizeInBits(Ty) != DL.getTypeSizeInBits(Ty);
395}

397/// A helper function that returns the reciprocal of the block probability of
398/// predicated blocks. If we return X, we are assuming the predicated block
399/// will execute once for every X iterations of the loop header.
400///
401/// TODO: We should use actual block probability here, if available. Currently,
402///       we always assume predicated blocks have a 50% chance of executing.
403static unsigned getReciprocalPredBlockProb() { return 2; }

405/// A helper function that returns an integer or floating-point constant with
406/// value C.
407static Constant *getSignedIntOrFpConstant(Type *Ty, int64_t C) {
return Ty->isIntegerTy() ? ConstantInt::getSigned(Ty, C)
                         : ConstantFP::get(Ty, C);
410}

412/// Returns "best known" trip count for the specified loop \p L as defined by
413/// the following procedure:
414///   1) Returns exact trip count if it is known.
415///   2) Returns expected trip count according to profile data if any.
416///   3) Returns upper bound estimate if it is known.
417///   4) Returns None if all of the above failed.
418static Optional<unsigned> getSmallBestKnownTC(ScalarEvolution &SE, Loop *L) {
// Check if exact trip count is known.
if (unsigned ExpectedTC = SE.getSmallConstantTripCount(L))
  return ExpectedTC;

// Check if there is an expected trip count available from profile data.
if (LoopVectorizeWithBlockFrequency)
  if (auto EstimatedTC = getLoopEstimatedTripCount(L))
    return EstimatedTC;

// Check if upper bound estimate is known.
if (unsigned ExpectedTC = SE.getSmallConstantMaxTripCount(L))
  return ExpectedTC;

return None;
433}

435// Forward declare GeneratedRTChecks.
436class GeneratedRTChecks;

438namespace llvm {

440/// InnerLoopVectorizer vectorizes loops which contain only one basic
441/// block to a specified vectorization factor (VF).
442/// This class performs the widening of scalars into vectors, or multiple
443/// scalars. This class also implements the following features:
444/// * It inserts an epilogue loop for handling loops that don't have iteration
445///   counts that are known to be a multiple of the vectorization factor.
446/// * It handles the code generation for reduction variables.
447/// * Scalarization (implementation using scalars) of un-vectorizable
448///   instructions.
449/// InnerLoopVectorizer does not perform any vectorization-legality
450/// checks, and relies on the caller to check for the different legality
451/// aspects. The InnerLoopVectorizer relies on the
452/// LoopVectorizationLegality class to provide information about the induction
453/// and reduction variables that were found to a given vectorization factor.
454class InnerLoopVectorizer {
455public:
InnerLoopVectorizer(Loop *OrigLoop, PredicatedScalarEvolution &PSE,
                    LoopInfo *LI, DominatorTree *DT,
                    const TargetLibraryInfo *TLI,
                    const TargetTransformInfo *TTI, AssumptionCache *AC,
                    OptimizationRemarkEmitter *ORE, ElementCount VecWidth,
                    unsigned UnrollFactor, LoopVectorizationLegality *LVL,
                    LoopVectorizationCostModel *CM, BlockFrequencyInfo *BFI,
                    ProfileSummaryInfo *PSI, GeneratedRTChecks &RTChecks)
    : OrigLoop(OrigLoop), PSE(PSE), LI(LI), DT(DT), TLI(TLI), TTI(TTI),
      AC(AC), ORE(ORE), VF(VecWidth), UF(UnrollFactor),
      Builder(PSE.getSE()->getContext()), Legal(LVL), Cost(CM), BFI(BFI),
      PSI(PSI), RTChecks(RTChecks) {
  // Query this against the original loop and save it here because the profile
  // of the original loop header may change as the transformation happens.
  OptForSizeBasedOnProfile = llvm::shouldOptimizeForSize(
      OrigLoop->getHeader(), PSI, BFI, PGSOQueryType::IRPass);
}

virtual ~InnerLoopVectorizer() = default;

/// Create a new empty loop that will contain vectorized instructions later
/// on, while the old loop will be used as the scalar remainder. Control flow
/// is generated around the vectorized (and scalar epilogue) loops consisting
/// of various checks and bypasses. Return the pre-header block of the new
/// loop.
/// In the case of epilogue vectorization, this function is overriden to
/// handle the more complex control flow around the loops.
virtual BasicBlock *createVectorizedLoopSkeleton();

/// Widen a single instruction within the innermost loop.
void widenInstruction(Instruction &I, VPValue *Def, VPUser &Operands,
                      VPTransformState &State);

/// Widen a single call instruction within the innermost loop.
void widenCallInstruction(CallInst &I, VPValue *Def, VPUser &ArgOperands,
                          VPTransformState &State);

/// Widen a single select instruction within the innermost loop.
void widenSelectInstruction(SelectInst &I, VPValue *VPDef, VPUser &Operands,
                            bool InvariantCond, VPTransformState &State);

/// Fix the vectorized code, taking care of header phi's, live-outs, and more.
void fixVectorizedLoop(VPTransformState &State);

// Return true if any runtime check is added.
bool areSafetyChecksAdded() { return AddedSafetyChecks; }

/// A type for vectorized values in the new loop. Each value from the
/// original loop, when vectorized, is represented by UF vector values in the
/// new unrolled loop, where UF is the unroll factor.
using VectorParts = SmallVector<Value *, 2>;

/// Vectorize a single GetElementPtrInst based on information gathered and
/// decisions taken during planning.
void widenGEP(GetElementPtrInst *GEP, VPValue *VPDef, VPUser &Indices,
              unsigned UF, ElementCount VF, bool IsPtrLoopInvariant,
              SmallBitVector &IsIndexLoopInvariant, VPTransformState &State);

/// Vectorize a single PHINode in a block. This method handles the induction
/// variable canonicalization. It supports both VF = 1 for unrolled loops and
/// arbitrary length vectors.
void widenPHIInstruction(Instruction *PN, RecurrenceDescriptor *RdxDesc,
                         VPWidenPHIRecipe *PhiR, VPTransformState &State);

/// A helper function to scalarize a single Instruction in the innermost loop.
/// Generates a sequence of scalar instances for each lane between \p MinLane
/// and \p MaxLane, times each part between \p MinPart and \p MaxPart,
/// inclusive. Uses the VPValue operands from \p Operands instead of \p
/// Instr's operands.
void scalarizeInstruction(Instruction *Instr, VPValue *Def, VPUser &Operands,
                          const VPIteration &Instance, bool IfPredicateInstr,
                          VPTransformState &State);

/// Widen an integer or floating-point induction variable \p IV. If \p Trunc
/// is provided, the integer induction variable will first be truncated to
/// the corresponding type.
void widenIntOrFpInduction(PHINode *IV, Value *Start, TruncInst *Trunc,
                           VPValue *Def, VPValue *CastDef,
                           VPTransformState &State);

/// Construct the vector value of a scalarized value \p V one lane at a time.
void packScalarIntoVectorValue(VPValue *Def, const VPIteration &Instance,
                               VPTransformState &State);

/// Try to vectorize interleaved access group \p Group with the base address
/// given in \p Addr, optionally masking the vector operations if \p
/// BlockInMask is non-null. Use \p State to translate given VPValues to IR
/// values in the vectorized loop.
void vectorizeInterleaveGroup(const InterleaveGroup<Instruction> *Group,
                              ArrayRef<VPValue *> VPDefs,
                              VPTransformState &State, VPValue *Addr,
                              ArrayRef<VPValue *> StoredValues,
                              VPValue *BlockInMask = nullptr);

/// Vectorize Load and Store instructions with the base address given in \p
/// Addr, optionally masking the vector operations if \p BlockInMask is
/// non-null. Use \p State to translate given VPValues to IR values in the
/// vectorized loop.
void vectorizeMemoryInstruction(Instruction *Instr, VPTransformState &State,
                                VPValue *Def, VPValue *Addr,
                                VPValue *StoredValue, VPValue *BlockInMask);

/// Set the debug location in the builder using the debug location in
/// the instruction.
void setDebugLocFromInst(IRBuilder<> &B, const Value *Ptr);

/// Fix the non-induction PHIs in the OrigPHIsToFix vector.
void fixNonInductionPHIs(VPTransformState &State);

/// Create a broadcast instruction. This method generates a broadcast
/// instruction (shuffle) for loop invariant values and for the induction
/// value. If this is the induction variable then we extend it to N, N+1, ...
/// this is needed because each iteration in the loop corresponds to a SIMD
/// element.
virtual Value *getBroadcastInstrs(Value *V);

572protected:
friend class LoopVectorizationPlanner;

/// A small list of PHINodes.
using PhiVector = SmallVector<PHINode *, 4>;

/// A type for scalarized values in the new loop. Each value from the
/// original loop, when scalarized, is represented by UF x VF scalar values
/// in the new unrolled loop, where UF is the unroll factor and VF is the
/// vectorization factor.
using ScalarParts = SmallVector<SmallVector<Value *, 4>, 2>;

/// Set up the values of the IVs correctly when exiting the vector loop.
void fixupIVUsers(PHINode *OrigPhi, const InductionDescriptor &II,
                  Value *CountRoundDown, Value *EndValue,
                  BasicBlock *MiddleBlock);

/// Create a new induction variable inside L.
PHINode *createInductionVariable(Loop *L, Value *Start, Value *End,
                                 Value *Step, Instruction *DL);

/// Handle all cross-iteration phis in the header.
void fixCrossIterationPHIs(VPTransformState &State);

/// Fix a first-order recurrence. This is the second phase of vectorizing
/// this phi node.
void fixFirstOrderRecurrence(PHINode *Phi, VPTransformState &State);

/// Fix a reduction cross-iteration phi. This is the second phase of
/// vectorizing this phi node.
void fixReduction(PHINode *Phi, VPTransformState &State);

/// Clear NSW/NUW flags from reduction instructions if necessary.
void clearReductionWrapFlags(RecurrenceDescriptor &RdxDesc,
                             VPTransformState &State);

/// Fixup the LCSSA phi nodes in the unique exit block.  This simply
/// means we need to add the appropriate incoming value from the middle
/// block as exiting edges from the scalar epilogue loop (if present) are
/// already in place, and we exit the vector loop exclusively to the middle
/// block.
void fixLCSSAPHIs(VPTransformState &State);

/// Iteratively sink the scalarized operands of a predicated instruction into
/// the block that was created for it.
void sinkScalarOperands(Instruction *PredInst);

/// Shrinks vector element sizes to the smallest bitwidth they can be legally
/// represented as.
void truncateToMinimalBitwidths(VPTransformState &State);

/// This function adds
/// (StartIdx * Step, (StartIdx + 1) * Step, (StartIdx + 2) * Step, ...)
/// to each vector element of Val. The sequence starts at StartIndex.
/// \p Opcode is relevant for FP induction variable.
virtual Value *getStepVector(Value *Val, int StartIdx, Value *Step,
                             Instruction::BinaryOps Opcode =
                             Instruction::BinaryOpsEnd);

/// Compute scalar induction steps. \p ScalarIV is the scalar induction
/// variable on which to base the steps, \p Step is the size of the step, and
/// \p EntryVal is the value from the original loop that maps to the steps.
/// Note that \p EntryVal doesn't have to be an induction variable - it
/// can also be a truncate instruction.
void buildScalarSteps(Value *ScalarIV, Value *Step, Instruction *EntryVal,
                      const InductionDescriptor &ID, VPValue *Def,
                      VPValue *CastDef, VPTransformState &State);

/// Create a vector induction phi node based on an existing scalar one. \p
/// EntryVal is the value from the original loop that maps to the vector phi
/// node, and \p Step is the loop-invariant step. If \p EntryVal is a
/// truncate instruction, instead of widening the original IV, we widen a
/// version of the IV truncated to \p EntryVal's type.
void createVectorIntOrFpInductionPHI(const InductionDescriptor &II,
                                     Value *Step, Value *Start,
                                     Instruction *EntryVal, VPValue *Def,
                                     VPValue *CastDef,
                                     VPTransformState &State);

/// Returns true if an instruction \p I should be scalarized instead of
/// vectorized for the chosen vectorization factor.
bool shouldScalarizeInstruction(Instruction *I) const;

/// Returns true if we should generate a scalar version of \p IV.
bool needsScalarInduction(Instruction *IV) const;

/// If there is a cast involved in the induction variable \p ID, which should
/// be ignored in the vectorized loop body, this function records the
/// VectorLoopValue of the respective Phi also as the VectorLoopValue of the
/// cast. We had already proved that the casted Phi is equal to the uncasted
/// Phi in the vectorized loop (under a runtime guard), and therefore
/// there is no need to vectorize the cast - the same value can be used in the
/// vector loop for both the Phi and the cast.
/// If \p VectorLoopValue is a scalarized value, \p Lane is also specified,
/// Otherwise, \p VectorLoopValue is a widened/vectorized value.
///
/// \p EntryVal is the value from the original loop that maps to the vector
/// phi node and is used to distinguish what is the IV currently being
/// processed - original one (if \p EntryVal is a phi corresponding to the
/// original IV) or the "newly-created" one based on the proof mentioned above
/// (see also buildScalarSteps() and createVectorIntOrFPInductionPHI()). In the
/// latter case \p EntryVal is a TruncInst and we must not record anything for
/// that IV, but it's error-prone to expect callers of this routine to care
/// about that, hence this explicit parameter.
void recordVectorLoopValueForInductionCast(
    const InductionDescriptor &ID, const Instruction *EntryVal,
    Value *VectorLoopValue, VPValue *CastDef, VPTransformState &State,
    unsigned Part, unsigned Lane = UINT_MAX(2147483647 *2U +1U));

/// Generate a shuffle sequence that will reverse the vector Vec.
virtual Value *reverseVector(Value *Vec);

/// Returns (and creates if needed) the original loop trip count.
Value *getOrCreateTripCount(Loop *NewLoop);

/// Returns (and creates if needed) the trip count of the widened loop.
Value *getOrCreateVectorTripCount(Loop *NewLoop);

/// Returns a bitcasted value to the requested vector type.
/// Also handles bitcasts of vector<float> <-> vector<pointer> types.
Value *createBitOrPointerCast(Value *V, VectorType *DstVTy,
                              const DataLayout &DL);

/// Emit a bypass check to see if the vector trip count is zero, including if
/// it overflows.
void emitMinimumIterationCountCheck(Loop *L, BasicBlock *Bypass);

/// Emit a bypass check to see if all of the SCEV assumptions we've
/// had to make are correct. Returns the block containing the checks or
/// nullptr if no checks have been added.
BasicBlock *emitSCEVChecks(Loop *L, BasicBlock *Bypass);

/// Emit bypass checks to check any memory assumptions we may have made.
/// Returns the block containing the checks or nullptr if no checks have been
/// added.
BasicBlock *emitMemRuntimeChecks(Loop *L, BasicBlock *Bypass);

/// Compute the transformed value of Index at offset StartValue using step
/// StepValue.
/// For integer induction, returns StartValue + Index * StepValue.
/// For pointer induction, returns StartValue[Index * StepValue].
/// FIXME: The newly created binary instructions should contain nsw/nuw
/// flags, which can be found from the original scalar operations.
Value *emitTransformedIndex(IRBuilder<> &B, Value *Index, ScalarEvolution *SE,
                            const DataLayout &DL,
                            const InductionDescriptor &ID) const;

/// Emit basic blocks (prefixed with \p Prefix) for the iteration check,
/// vector loop preheader, middle block and scalar preheader. Also
/// allocate a loop object for the new vector loop and return it.
Loop *createVectorLoopSkeleton(StringRef Prefix);

/// Create new phi nodes for the induction variables to resume iteration count
/// in the scalar epilogue, from where the vectorized loop left off (given by
/// \p VectorTripCount).
/// In cases where the loop skeleton is more complicated (eg. epilogue
/// vectorization) and the resume values can come from an additional bypass
/// block, the \p AdditionalBypass pair provides information about the bypass
/// block and the end value on the edge from bypass to this loop.
void createInductionResumeValues(
    Loop *L, Value *VectorTripCount,
    std::pair<BasicBlock *, Value *> AdditionalBypass = {nullptr, nullptr});

/// Complete the loop skeleton by adding debug MDs, creating appropriate
/// conditional branches in the middle block, preparing the builder and
/// running the verifier. Take in the vector loop \p L as argument, and return
/// the preheader of the completed vector loop.
BasicBlock *completeLoopSkeleton(Loop *L, MDNode *OrigLoopID);

/// Add additional metadata to \p To that was not present on \p Orig.
///
/// Currently this is used to add the noalias annotations based on the
/// inserted memchecks.  Use this for instructions that are *cloned* into the
/// vector loop.
void addNewMetadata(Instruction *To, const Instruction *Orig);

/// Add metadata from one instruction to another.
///
/// This includes both the original MDs from \p From and additional ones (\see
/// addNewMetadata).  Use this for *newly created* instructions in the vector
/// loop.
void addMetadata(Instruction *To, Instruction *From);

/// Similar to the previous function but it adds the metadata to a
/// vector of instructions.
void addMetadata(ArrayRef<Value *> To, Instruction *From);

/// Allow subclasses to override and print debug traces before/after vplan
/// execution, when trace information is requested.
virtual void printDebugTracesAtStart(){};
virtual void printDebugTracesAtEnd(){};

/// The original loop.
Loop *OrigLoop;

/// A wrapper around ScalarEvolution used to add runtime SCEV checks. Applies
/// dynamic knowledge to simplify SCEV expressions and converts them to a
/// more usable form.
PredicatedScalarEvolution &PSE;

/// Loop Info.
LoopInfo *LI;

/// Dominator Tree.
DominatorTree *DT;

/// Alias Analysis.
AAResults *AA;

/// Target Library Info.
const TargetLibraryInfo *TLI;

/// Target Transform Info.
const TargetTransformInfo *TTI;

/// Assumption Cache.
AssumptionCache *AC;

/// Interface to emit optimization remarks.
OptimizationRemarkEmitter *ORE;

/// LoopVersioning.  It's only set up (non-null) if memchecks were
/// used.
///
/// This is currently only used to add no-alias metadata based on the
/// memchecks.  The actually versioning is performed manually.
std::unique_ptr<LoopVersioning> LVer;

/// The vectorization SIMD factor to use. Each vector will have this many
/// vector elements.
ElementCount VF;

/// The vectorization unroll factor to use. Each scalar is vectorized to this
/// many different vector instructions.
unsigned UF;

/// The builder that we use
IRBuilder<> Builder;

// --- Vectorization state ---

/// The vector-loop preheader.
BasicBlock *LoopVectorPreHeader;

/// The scalar-loop preheader.
BasicBlock *LoopScalarPreHeader;

/// Middle Block between the vector and the scalar.
BasicBlock *LoopMiddleBlock;

/// The (unique) ExitBlock of the scalar loop.  Note that
/// there can be multiple exiting edges reaching this block.
BasicBlock *LoopExitBlock;

/// The vector loop body.
BasicBlock *LoopVectorBody;

/// The scalar loop body.
BasicBlock *LoopScalarBody;

/// A list of all bypass blocks. The first block is the entry of the loop.
SmallVector<BasicBlock *, 4> LoopBypassBlocks;

/// The new Induction variable which was added to the new block.
PHINode *Induction = nullptr;

/// The induction variable of the old basic block.
PHINode *OldInduction = nullptr;

/// Store instructions that were predicated.
SmallVector<Instruction *, 4> PredicatedInstructions;

/// Trip count of the original loop.
Value *TripCount = nullptr;

/// Trip count of the widened loop (TripCount - TripCount % (VF*UF))
Value *VectorTripCount = nullptr;

/// The legality analysis.
LoopVectorizationLegality *Legal;

/// The profitablity analysis.
LoopVectorizationCostModel *Cost;

// Record whether runtime checks are added.
bool AddedSafetyChecks = false;

// Holds the end values for each induction variable. We save the end values
// so we can later fix-up the external users of the induction variables.
DenseMap<PHINode *, Value *> IVEndValues;

// Vector of original scalar PHIs whose corresponding widened PHIs need to be
// fixed up at the end of vector code generation.
SmallVector<PHINode *, 8> OrigPHIsToFix;

/// BFI and PSI are used to check for profile guided size optimizations.
BlockFrequencyInfo *BFI;
ProfileSummaryInfo *PSI;

// Whether this loop should be optimized for size based on profile guided size
// optimizatios.
bool OptForSizeBasedOnProfile;

/// Structure to hold information about generated runtime checks, responsible
/// for cleaning the checks, if vectorization turns out unprofitable.
GeneratedRTChecks &RTChecks;
878};

880class InnerLoopUnroller : public InnerLoopVectorizer {
881public:
InnerLoopUnroller(Loop *OrigLoop, PredicatedScalarEvolution &PSE,
                  LoopInfo *LI, DominatorTree *DT,
                  const TargetLibraryInfo *TLI,
                  const TargetTransformInfo *TTI, AssumptionCache *AC,
                  OptimizationRemarkEmitter *ORE, unsigned UnrollFactor,
                  LoopVectorizationLegality *LVL,
                  LoopVectorizationCostModel *CM, BlockFrequencyInfo *BFI,
                  ProfileSummaryInfo *PSI, GeneratedRTChecks &Check)
    : InnerLoopVectorizer(OrigLoop, PSE, LI, DT, TLI, TTI, AC, ORE,
                          ElementCount::getFixed(1), UnrollFactor, LVL, CM,
                          BFI, PSI, Check) {}

894private:
Value *getBroadcastInstrs(Value *V) override;
Value *getStepVector(Value *Val, int StartIdx, Value *Step,
                     Instruction::BinaryOps Opcode =
                     Instruction::BinaryOpsEnd) override;
Value *reverseVector(Value *Vec) override;
900};

902/// Encapsulate information regarding vectorization of a loop and its epilogue.
903/// This information is meant to be updated and used across two stages of
904/// epilogue vectorization.
905struct EpilogueLoopVectorizationInfo {
ElementCount MainLoopVF = ElementCount::getFixed(0);
unsigned MainLoopUF = 0;
ElementCount EpilogueVF = ElementCount::getFixed(0);
unsigned EpilogueUF = 0;
BasicBlock *MainLoopIterationCountCheck = nullptr;
BasicBlock *EpilogueIterationCountCheck = nullptr;
BasicBlock *SCEVSafetyCheck = nullptr;
BasicBlock *MemSafetyCheck = nullptr;
Value *TripCount = nullptr;
Value *VectorTripCount = nullptr;

EpilogueLoopVectorizationInfo(unsigned MVF, unsigned MUF, unsigned EVF,
                              unsigned EUF)
    : MainLoopVF(ElementCount::getFixed(MVF)), MainLoopUF(MUF),
      EpilogueVF(ElementCount::getFixed(EVF)), EpilogueUF(EUF) {
  assert(EUF == 1 &&((EUF == 1 && "A high UF for the epilogue loop is likely not beneficial."
) ? static_cast<void> (0) : __assert_fail ("EUF == 1 && \"A high UF for the epilogue loop is likely not beneficial.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 922, __PRETTY_FUNCTION__))
         "A high UF for the epilogue loop is likely not beneficial.")((EUF == 1 && "A high UF for the epilogue loop is likely not beneficial."
) ? static_cast<void> (0) : __assert_fail ("EUF == 1 && \"A high UF for the epilogue loop is likely not beneficial.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 922, __PRETTY_FUNCTION__));
}
924};

926/// An extension of the inner loop vectorizer that creates a skeleton for a
927/// vectorized loop that has its epilogue (residual) also vectorized.
928/// The idea is to run the vplan on a given loop twice, firstly to setup the
929/// skeleton and vectorize the main loop, and secondly to complete the skeleton
930/// from the first step and vectorize the epilogue.  This is achieved by
931/// deriving two concrete strategy classes from this base class and invoking
932/// them in succession from the loop vectorizer planner.
933class InnerLoopAndEpilogueVectorizer : public InnerLoopVectorizer {
934public:
InnerLoopAndEpilogueVectorizer(
    Loop *OrigLoop, PredicatedScalarEvolution &PSE, LoopInfo *LI,
    DominatorTree *DT, const TargetLibraryInfo *TLI,
    const TargetTransformInfo *TTI, AssumptionCache *AC,
    OptimizationRemarkEmitter *ORE, EpilogueLoopVectorizationInfo &EPI,
    LoopVectorizationLegality *LVL, llvm::LoopVectorizationCostModel *CM,
    BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI,
    GeneratedRTChecks &Checks)
    : InnerLoopVectorizer(OrigLoop, PSE, LI, DT, TLI, TTI, AC, ORE,
                          EPI.MainLoopVF, EPI.MainLoopUF, LVL, CM, BFI, PSI,
                          Checks),
      EPI(EPI) {}

// Override this function to handle the more complex control flow around the
// three loops.
BasicBlock *createVectorizedLoopSkeleton() final override {
  return createEpilogueVectorizedLoopSkeleton();
}

/// The interface for creating a vectorized skeleton using one of two
/// different strategies, each corresponding to one execution of the vplan
/// as described above.
virtual BasicBlock *createEpilogueVectorizedLoopSkeleton() = 0;

/// Holds and updates state information required to vectorize the main loop
/// and its epilogue in two separate passes. This setup helps us avoid
/// regenerating and recomputing runtime safety checks. It also helps us to
/// shorten the iteration-count-check path length for the cases where the
/// iteration count of the loop is so small that the main vector loop is
/// completely skipped.
EpilogueLoopVectorizationInfo &EPI;
966};

968/// A specialized derived class of inner loop vectorizer that performs
969/// vectorization of *main* loops in the process of vectorizing loops and their
970/// epilogues.
971class EpilogueVectorizerMainLoop : public InnerLoopAndEpilogueVectorizer {
972public:
EpilogueVectorizerMainLoop(
    Loop *OrigLoop, PredicatedScalarEvolution &PSE, LoopInfo *LI,
    DominatorTree *DT, const TargetLibraryInfo *TLI,
    const TargetTransformInfo *TTI, AssumptionCache *AC,
    OptimizationRemarkEmitter *ORE, EpilogueLoopVectorizationInfo &EPI,
    LoopVectorizationLegality *LVL, llvm::LoopVectorizationCostModel *CM,
    BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI,
    GeneratedRTChecks &Check)
    : InnerLoopAndEpilogueVectorizer(OrigLoop, PSE, LI, DT, TLI, TTI, AC, ORE,
                                     EPI, LVL, CM, BFI, PSI, Check) {}
/// Implements the interface for creating a vectorized skeleton using the
/// *main loop* strategy (ie the first pass of vplan execution).
BasicBlock *createEpilogueVectorizedLoopSkeleton() final override;

987protected:
/// Emits an iteration count bypass check once for the main loop (when \p
/// ForEpilogue is false) and once for the epilogue loop (when \p
/// ForEpilogue is true).
BasicBlock *emitMinimumIterationCountCheck(Loop *L, BasicBlock *Bypass,
                                           bool ForEpilogue);
void printDebugTracesAtStart() override;
void printDebugTracesAtEnd() override;
995};

997// A specialized derived class of inner loop vectorizer that performs
998// vectorization of *epilogue* loops in the process of vectorizing loops and
999// their epilogues.
1000class EpilogueVectorizerEpilogueLoop : public InnerLoopAndEpilogueVectorizer {
1001public:
EpilogueVectorizerEpilogueLoop(
    Loop *OrigLoop, PredicatedScalarEvolution &PSE, LoopInfo *LI,
    DominatorTree *DT, const TargetLibraryInfo *TLI,
    const TargetTransformInfo *TTI, AssumptionCache *AC,
    OptimizationRemarkEmitter *ORE, EpilogueLoopVectorizationInfo &EPI,
    LoopVectorizationLegality *LVL, llvm::LoopVectorizationCostModel *CM,
    BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI,
    GeneratedRTChecks &Checks)
    : InnerLoopAndEpilogueVectorizer(OrigLoop, PSE, LI, DT, TLI, TTI, AC, ORE,
                                     EPI, LVL, CM, BFI, PSI, Checks) {}
/// Implements the interface for creating a vectorized skeleton using the
/// *epilogue loop* strategy (ie the second pass of vplan execution).
BasicBlock *createEpilogueVectorizedLoopSkeleton() final override;

1016protected:
/// Emits an iteration count bypass check after the main vector loop has
/// finished to see if there are any iterations left to execute by either
/// the vector epilogue or the scalar epilogue.
BasicBlock *emitMinimumVectorEpilogueIterCountCheck(Loop *L,
                                                    BasicBlock *Bypass,
                                                    BasicBlock *Insert);
void printDebugTracesAtStart() override;
void printDebugTracesAtEnd() override;
1025};
1026} // end namespace llvm

1028/// Look for a meaningful debug location on the instruction or it's
1029/// operands.
1030static Instruction *getDebugLocFromInstOrOperands(Instruction *I) {
if (!I)
  return I;

DebugLoc Empty;
if (I->getDebugLoc() != Empty)
  return I;

for (Use &Op : I->operands()) {
  if (Instruction *OpInst = dyn_cast<Instruction>(Op))
    if (OpInst->getDebugLoc() != Empty)
      return OpInst;
}

return I;
1045}

1047void InnerLoopVectorizer::setDebugLocFromInst(IRBuilder<> &B, const Value *Ptr) {
if (const Instruction *Inst = dyn_cast_or_null<Instruction>(Ptr)) {
  const DILocation *DIL = Inst->getDebugLoc();
  if (DIL && Inst->getFunction()->isDebugInfoForProfiling() &&
      !isa<DbgInfoIntrinsic>(Inst)) {
    assert(!VF.isScalable() && "scalable vectors not yet supported.")((!VF.isScalable() && "scalable vectors not yet supported."
) ? static_cast<void> (0) : __assert_fail ("!VF.isScalable() && \"scalable vectors not yet supported.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1052, __PRETTY_FUNCTION__));
    auto NewDIL =
        DIL->cloneByMultiplyingDuplicationFactor(UF * VF.getKnownMinValue());
    if (NewDIL)
      B.SetCurrentDebugLocation(NewDIL.getValue());
    else
      LLVM_DEBUG(dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "Failed to create new discriminator: "
 << DIL->getFilename() << " Line: " << DIL
->getLine(); } } while (false)
                 << "Failed to create new discriminator: "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "Failed to create new discriminator: "
 << DIL->getFilename() << " Line: " << DIL
->getLine(); } } while (false)
                 << DIL->getFilename() << " Line: " << DIL->getLine())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "Failed to create new discriminator: "
 << DIL->getFilename() << " Line: " << DIL
->getLine(); } } while (false);
  }
  else
    B.SetCurrentDebugLocation(DIL);
} else
  B.SetCurrentDebugLocation(DebugLoc());
1066}

1068/// Write a record \p DebugMsg about vectorization failure to the debug
1069/// output stream. If \p I is passed, it is an instruction that prevents
1070/// vectorization.
1071#ifndef NDEBUG
1072static void debugVectorizationFailure(const StringRef DebugMsg,
  Instruction *I) {
dbgs() << "LV: Not vectorizing: " << DebugMsg;
if (I != nullptr)
  dbgs() << " " << *I;
else
  dbgs() << '.';
dbgs() << '\n';
1080}
1081#endif

1083/// Create an analysis remark that explains why vectorization failed
1084///
1085/// \p PassName is the name of the pass (e.g. can be AlwaysPrint).  \p
1086/// RemarkName is the identifier for the remark.  If \p I is passed it is an
1087/// instruction that prevents vectorization.  Otherwise \p TheLoop is used for
1088/// the location of the remark.  \return the remark object that can be
1089/// streamed to.
1090static OptimizationRemarkAnalysis createLVAnalysis(const char *PassName,
  StringRef RemarkName, Loop *TheLoop, Instruction *I) {
Value *CodeRegion = TheLoop->getHeader();
DebugLoc DL = TheLoop->getStartLoc();

if (I) {
  CodeRegion = I->getParent();
  // If there is no debug location attached to the instruction, revert back to
  // using the loop's.
  if (I->getDebugLoc())
    DL = I->getDebugLoc();
}

OptimizationRemarkAnalysis R(PassName, RemarkName, DL, CodeRegion);
R << "loop not vectorized: ";
return R;
1106}

1108/// Return a value for Step multiplied by VF.
1109static Value *createStepForVF(IRBuilder<> &B, Constant *Step, ElementCount VF) {
assert(isa<ConstantInt>(Step) && "Expected an integer step")((isa<ConstantInt>(Step) && "Expected an integer step"
) ? static_cast<void> (0) : __assert_fail ("isa<ConstantInt>(Step) && \"Expected an integer step\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1110, __PRETTY_FUNCTION__));
Constant *StepVal = ConstantInt::get(
    Step->getType(),
    cast<ConstantInt>(Step)->getSExtValue() * VF.getKnownMinValue());
return VF.isScalable() ? B.CreateVScale(StepVal) : StepVal;
1115}

1117namespace llvm {

1119/// Return the runtime value for VF.
1120Value *getRuntimeVF(IRBuilder<> &B, Type *Ty, ElementCount VF) {
Constant *EC = ConstantInt::get(Ty, VF.getKnownMinValue());
return VF.isScalable() ? B.CreateVScale(EC) : EC;
1123}

1125void reportVectorizationFailure(const StringRef DebugMsg,
  const StringRef OREMsg, const StringRef ORETag,
  OptimizationRemarkEmitter *ORE, Loop *TheLoop, Instruction *I) {
LLVM_DEBUG(debugVectorizationFailure(DebugMsg, I))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { debugVectorizationFailure(DebugMsg, I);
 } } while (false);
LoopVectorizeHints Hints(TheLoop, true /* doesn't matter */, *ORE);
ORE->emit(createLVAnalysis(Hints.vectorizeAnalysisPassName(),
              ORETag, TheLoop, I) << OREMsg);
1132}

1134} // end namespace llvm

1136#ifndef NDEBUG
1137/// \return string containing a file name and a line # for the given loop.
1138static std::string getDebugLocString(const Loop *L) {
std::string Result;
if (L) {
  raw_string_ostream OS(Result);
  if (const DebugLoc LoopDbgLoc = L->getStartLoc())
    LoopDbgLoc.print(OS);
  else
    // Just print the module name.
    OS << L->getHeader()->getParent()->getParent()->getModuleIdentifier();
  OS.flush();
}
return Result;
1150}
1151#endif

1153void InnerLoopVectorizer::addNewMetadata(Instruction *To,
                                       const Instruction *Orig) {
// If the loop was versioned with memchecks, add the corresponding no-alias
// metadata.
if (LVer && (isa<LoadInst>(Orig) || isa<StoreInst>(Orig)))
  LVer->annotateInstWithNoAlias(To, Orig);
1159}

1161void InnerLoopVectorizer::addMetadata(Instruction *To,
                                    Instruction *From) {
propagateMetadata(To, From);
addNewMetadata(To, From);
1165}

1167void InnerLoopVectorizer::addMetadata(ArrayRef<Value *> To,
                                    Instruction *From) {
for (Value *V : To) {
  if (Instruction *I = dyn_cast<Instruction>(V))
    addMetadata(I, From);
}
1173}

1175namespace llvm {

1177// Loop vectorization cost-model hints how the scalar epilogue loop should be
1178// lowered.
1179enum ScalarEpilogueLowering {

// The default: allowing scalar epilogues.
CM_ScalarEpilogueAllowed,

// Vectorization with OptForSize: don't allow epilogues.
CM_ScalarEpilogueNotAllowedOptSize,

// A special case of vectorisation with OptForSize: loops with a very small
// trip count are considered for vectorization under OptForSize, thereby
// making sure the cost of their loop body is dominant, free of runtime
// guards and scalar iteration overheads.
CM_ScalarEpilogueNotAllowedLowTripLoop,

// Loop hint predicate indicating an epilogue is undesired.
CM_ScalarEpilogueNotNeededUsePredicate,

// Directive indicating we must either tail fold or not vectorize
CM_ScalarEpilogueNotAllowedUsePredicate
1198};

1200/// LoopVectorizationCostModel - estimates the expected speedups due to
1201/// vectorization.
1202/// In many cases vectorization is not profitable. This can happen because of
1203/// a number of reasons. In this class we mainly attempt to predict the
1204/// expected speedup/slowdowns due to the supported instruction set. We use the
1205/// TargetTransformInfo to query the different backends for the cost of
1206/// different operations.
1207class LoopVectorizationCostModel {
1208public:
LoopVectorizationCostModel(ScalarEpilogueLowering SEL, Loop *L,
                           PredicatedScalarEvolution &PSE, LoopInfo *LI,
                           LoopVectorizationLegality *Legal,
                           const TargetTransformInfo &TTI,
                           const TargetLibraryInfo *TLI, DemandedBits *DB,
                           AssumptionCache *AC,
                           OptimizationRemarkEmitter *ORE, const Function *F,
                           const LoopVectorizeHints *Hints,
                           InterleavedAccessInfo &IAI)
    : ScalarEpilogueStatus(SEL), TheLoop(L), PSE(PSE), LI(LI), Legal(Legal),
      TTI(TTI), TLI(TLI), DB(DB), AC(AC), ORE(ORE), TheFunction(F),
      Hints(Hints), InterleaveInfo(IAI) {}

/// \return An upper bound for the vectorization factor, or None if
/// vectorization and interleaving should be avoided up front.
Optional<ElementCount> computeMaxVF(ElementCount UserVF, unsigned UserIC);

/// \return True if runtime checks are required for vectorization, and false
/// otherwise.
bool runtimeChecksRequired();

/// \return The most profitable vectorization factor and the cost of that VF.
/// This method checks every power of two up to MaxVF. If UserVF is not ZERO
/// then this vectorization factor will be selected if vectorization is
/// possible.
VectorizationFactor selectVectorizationFactor(ElementCount MaxVF);
VectorizationFactor
selectEpilogueVectorizationFactor(const ElementCount MaxVF,
                                  const LoopVectorizationPlanner &LVP);

/// Setup cost-based decisions for user vectorization factor.
void selectUserVectorizationFactor(ElementCount UserVF) {
  collectUniformsAndScalars(UserVF);
  collectInstsToScalarize(UserVF);
}

/// \return The size (in bits) of the smallest and widest types in the code
/// that needs to be vectorized. We ignore values that remain scalar such as
/// 64 bit loop indices.
std::pair<unsigned, unsigned> getSmallestAndWidestTypes();

/// \return The desired interleave count.
/// If interleave count has been specified by metadata it will be returned.
/// Otherwise, the interleave count is computed and returned. VF and LoopCost
/// are the selected vectorization factor and the cost of the selected VF.
unsigned selectInterleaveCount(ElementCount VF, unsigned LoopCost);

/// Memory access instruction may be vectorized in more than one way.
/// Form of instruction after vectorization depends on cost.
/// This function takes cost-based decisions for Load/Store instructions
/// and collects them in a map. This decisions map is used for building
/// the lists of loop-uniform and loop-scalar instructions.
/// The calculated cost is saved with widening decision in order to
/// avoid redundant calculations.
void setCostBasedWideningDecision(ElementCount VF);

/// A struct that represents some properties of the register usage
/// of a loop.
struct RegisterUsage {
  /// Holds the number of loop invariant values that are used in the loop.
  /// The key is ClassID of target-provided register class.
  SmallMapVector<unsigned, unsigned, 4> LoopInvariantRegs;
  /// Holds the maximum number of concurrent live intervals in the loop.
  /// The key is ClassID of target-provided register class.
  SmallMapVector<unsigned, unsigned, 4> MaxLocalUsers;
};

/// \return Returns information about the register usages of the loop for the
/// given vectorization factors.
SmallVector<RegisterUsage, 8>
calculateRegisterUsage(ArrayRef<ElementCount> VFs);

/// Collect values we want to ignore in the cost model.
void collectValuesToIgnore();

/// Split reductions into those that happen in the loop, and those that happen
/// outside. In loop reductions are collected into InLoopReductionChains.
void collectInLoopReductions();

/// \returns The smallest bitwidth each instruction can be represented with.
/// The vector equivalents of these instructions should be truncated to this
/// type.
const MapVector<Instruction *, uint64_t> &getMinimalBitwidths() const {
  return MinBWs;
}

/// \returns True if it is more profitable to scalarize instruction \p I for
/// vectorization factor \p VF.
bool isProfitableToScalarize(Instruction *I, ElementCount VF) const {
  assert(VF.isVector() &&((VF.isVector() && "Profitable to scalarize relevant only for VF > 1."
) ? static_cast<void> (0) : __assert_fail ("VF.isVector() && \"Profitable to scalarize relevant only for VF > 1.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1299, __PRETTY_FUNCTION__))
         "Profitable to scalarize relevant only for VF > 1.")((VF.isVector() && "Profitable to scalarize relevant only for VF > 1."
) ? static_cast<void> (0) : __assert_fail ("VF.isVector() && \"Profitable to scalarize relevant only for VF > 1.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1299, __PRETTY_FUNCTION__));

  // Cost model is not run in the VPlan-native path - return conservative
  // result until this changes.
  if (EnableVPlanNativePath)
    return false;

  auto Scalars = InstsToScalarize.find(VF);
  assert(Scalars != InstsToScalarize.end() &&((Scalars != InstsToScalarize.end() && "VF not yet analyzed for scalarization profitability"
) ? static_cast<void> (0) : __assert_fail ("Scalars != InstsToScalarize.end() && \"VF not yet analyzed for scalarization profitability\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1308, __PRETTY_FUNCTION__))
         "VF not yet analyzed for scalarization profitability")((Scalars != InstsToScalarize.end() && "VF not yet analyzed for scalarization profitability"
) ? static_cast<void> (0) : __assert_fail ("Scalars != InstsToScalarize.end() && \"VF not yet analyzed for scalarization profitability\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1308, __PRETTY_FUNCTION__));
  return Scalars->second.find(I) != Scalars->second.end();
}

/// Returns true if \p I is known to be uniform after vectorization.
bool isUniformAfterVectorization(Instruction *I, ElementCount VF) const {
  if (VF.isScalar())
    return true;

  // Cost model is not run in the VPlan-native path - return conservative
  // result until this changes.
  if (EnableVPlanNativePath)
    return false;

  auto UniformsPerVF = Uniforms.find(VF);
  assert(UniformsPerVF != Uniforms.end() &&((UniformsPerVF != Uniforms.end() && "VF not yet analyzed for uniformity"
) ? static_cast<void> (0) : __assert_fail ("UniformsPerVF != Uniforms.end() && \"VF not yet analyzed for uniformity\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1324, __PRETTY_FUNCTION__))
         "VF not yet analyzed for uniformity")((UniformsPerVF != Uniforms.end() && "VF not yet analyzed for uniformity"
) ? static_cast<void> (0) : __assert_fail ("UniformsPerVF != Uniforms.end() && \"VF not yet analyzed for uniformity\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1324, __PRETTY_FUNCTION__));
  return UniformsPerVF->second.count(I);
}

/// Returns true if \p I is known to be scalar after vectorization.
bool isScalarAfterVectorization(Instruction *I, ElementCount VF) const {
  if (VF.isScalar())
    return true;

  // Cost model is not run in the VPlan-native path - return conservative
  // result until this changes.
  if (EnableVPlanNativePath)
    return false;

  auto ScalarsPerVF = Scalars.find(VF);
  assert(ScalarsPerVF != Scalars.end() &&((ScalarsPerVF != Scalars.end() && "Scalar values are not calculated for VF"
) ? static_cast<void> (0) : __assert_fail ("ScalarsPerVF != Scalars.end() && \"Scalar values are not calculated for VF\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1340, __PRETTY_FUNCTION__))
         "Scalar values are not calculated for VF")((ScalarsPerVF != Scalars.end() && "Scalar values are not calculated for VF"
) ? static_cast<void> (0) : __assert_fail ("ScalarsPerVF != Scalars.end() && \"Scalar values are not calculated for VF\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1340, __PRETTY_FUNCTION__));
  return ScalarsPerVF->second.count(I);
}

/// \returns True if instruction \p I can be truncated to a smaller bitwidth
/// for vectorization factor \p VF.
bool canTruncateToMinimalBitwidth(Instruction *I, ElementCount VF) const {
  return VF.isVector() && MinBWs.find(I) != MinBWs.end() &&
         !isProfitableToScalarize(I, VF) &&
         !isScalarAfterVectorization(I, VF);
}

/// Decision that was taken during cost calculation for memory instruction.
enum InstWidening {
  CM_Unknown,
  CM_Widen,         // For consecutive accesses with stride +1.
  CM_Widen_Reverse, // For consecutive accesses with stride -1.
  CM_Interleave,
  CM_GatherScatter,
  CM_Scalarize
};

/// Save vectorization decision \p W and \p Cost taken by the cost model for
/// instruction \p I and vector width \p VF.
void setWideningDecision(Instruction *I, ElementCount VF, InstWidening W,
                         InstructionCost Cost) {
  assert(VF.isVector() && "Expected VF >=2")((VF.isVector() && "Expected VF >=2") ? static_cast
<void> (0) : __assert_fail ("VF.isVector() && \"Expected VF >=2\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1366, __PRETTY_FUNCTION__));
  WideningDecisions[std::make_pair(I, VF)] = std::make_pair(W, Cost);
}

/// Save vectorization decision \p W and \p Cost taken by the cost model for
/// interleaving group \p Grp and vector width \p VF.
void setWideningDecision(const InterleaveGroup<Instruction> *Grp,
                         ElementCount VF, InstWidening W,
                         InstructionCost Cost) {
  assert(VF.isVector() && "Expected VF >=2")((VF.isVector() && "Expected VF >=2") ? static_cast
<void> (0) : __assert_fail ("VF.isVector() && \"Expected VF >=2\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1375, __PRETTY_FUNCTION__));
  /// Broadcast this decicion to all instructions inside the group.
  /// But the cost will be assigned to one instruction only.
  for (unsigned i = 0; i < Grp->getFactor(); ++i) {
    if (auto *I = Grp->getMember(i)) {
      if (Grp->getInsertPos() == I)
        WideningDecisions[std::make_pair(I, VF)] = std::make_pair(W, Cost);
      else
        WideningDecisions[std::make_pair(I, VF)] = std::make_pair(W, 0);
    }
  }
}

/// Return the cost model decision for the given instruction \p I and vector
/// width \p VF. Return CM_Unknown if this instruction did not pass
/// through the cost modeling.
InstWidening getWideningDecision(Instruction *I, ElementCount VF) const {
  assert(VF.isVector() && "Expected VF to be a vector VF")((VF.isVector() && "Expected VF to be a vector VF") ?
 static_cast<void> (0) : __assert_fail ("VF.isVector() && \"Expected VF to be a vector VF\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1392, __PRETTY_FUNCTION__));
  // Cost model is not run in the VPlan-native path - return conservative
  // result until this changes.
  if (EnableVPlanNativePath)
    return CM_GatherScatter;

  std::pair<Instruction *, ElementCount> InstOnVF = std::make_pair(I, VF);
  auto Itr = WideningDecisions.find(InstOnVF);
  if (Itr == WideningDecisions.end())
    return CM_Unknown;
  return Itr->second.first;
}

/// Return the vectorization cost for the given instruction \p I and vector
/// width \p VF.
InstructionCost getWideningCost(Instruction *I, ElementCount VF) {
  assert(VF.isVector() && "Expected VF >=2")((VF.isVector() && "Expected VF >=2") ? static_cast
<void> (0) : __assert_fail ("VF.isVector() && \"Expected VF >=2\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1408, __PRETTY_FUNCTION__));
  std::pair<Instruction *, ElementCount> InstOnVF = std::make_pair(I, VF);
  assert(WideningDecisions.find(InstOnVF) != WideningDecisions.end() &&((WideningDecisions.find(InstOnVF) != WideningDecisions.end()
 && "The cost is not calculated") ? static_cast<void
> (0) : __assert_fail ("WideningDecisions.find(InstOnVF) != WideningDecisions.end() && \"The cost is not calculated\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1411, __PRETTY_FUNCTION__))
         "The cost is not calculated")((WideningDecisions.find(InstOnVF) != WideningDecisions.end()
 && "The cost is not calculated") ? static_cast<void
> (0) : __assert_fail ("WideningDecisions.find(InstOnVF) != WideningDecisions.end() && \"The cost is not calculated\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1411, __PRETTY_FUNCTION__));
  return WideningDecisions[InstOnVF].second;
}

/// Return True if instruction \p I is an optimizable truncate whose operand
/// is an induction variable. Such a truncate will be removed by adding a new
/// induction variable with the destination type.
bool isOptimizableIVTruncate(Instruction *I, ElementCount VF) {
  // If the instruction is not a truncate, return false.
  auto *Trunc = dyn_cast<TruncInst>(I);
  if (!Trunc)
    return false;

  // Get the source and destination types of the truncate.
  Type *SrcTy = ToVectorTy(cast<CastInst>(I)->getSrcTy(), VF);
  Type *DestTy = ToVectorTy(cast<CastInst>(I)->getDestTy(), VF);

  // If the truncate is free for the given types, return false. Replacing a
  // free truncate with an induction variable would add an induction variable
  // update instruction to each iteration of the loop. We exclude from this
  // check the primary induction variable since it will need an update
  // instruction regardless.
  Value *Op = Trunc->getOperand(0);
  if (Op != Legal->getPrimaryInduction() && TTI.isTruncateFree(SrcTy, DestTy))
    return false;

  // If the truncated value is not an induction variable, return false.
  return Legal->isInductionPhi(Op);
}

/// Collects the instructions to scalarize for each predicated instruction in
/// the loop.
void collectInstsToScalarize(ElementCount VF);

/// Collect Uniform and Scalar values for the given \p VF.
/// The sets depend on CM decision for Load/Store instructions
/// that may be vectorized as interleave, gather-scatter or scalarized.
void collectUniformsAndScalars(ElementCount VF) {
  // Do the analysis once.
  if (VF.isScalar() || Uniforms.find(VF) != Uniforms.end())
    return;
  setCostBasedWideningDecision(VF);
  collectLoopUniforms(VF);
  collectLoopScalars(VF);
}

/// Returns true if the target machine supports masked store operation
/// for the given \p DataType and kind of access to \p Ptr.
bool isLegalMaskedStore(Type *DataType, Value *Ptr, Align Alignment) const {
  return Legal->isConsecutivePtr(Ptr) &&
         TTI.isLegalMaskedStore(DataType, Alignment);
}

/// Returns true if the target machine supports masked load operation
/// for the given \p DataType and kind of access to \p Ptr.
bool isLegalMaskedLoad(Type *DataType, Value *Ptr, Align Alignment) const {
  return Legal->isConsecutivePtr(Ptr) &&
         TTI.isLegalMaskedLoad(DataType, Alignment);
}

/// Returns true if the target machine supports masked scatter operation
/// for the given \p DataType.
bool isLegalMaskedScatter(Type *DataType, Align Alignment) const {
  return TTI.isLegalMaskedScatter(DataType, Alignment);
}

/// Returns true if the target machine supports masked gather operation
/// for the given \p DataType.
bool isLegalMaskedGather(Type *DataType, Align Alignment) const {
  return TTI.isLegalMaskedGather(DataType, Alignment);
}

/// Returns true if the target machine can represent \p V as a masked gather
/// or scatter operation.
bool isLegalGatherOrScatter(Value *V) {
  bool LI = isa<LoadInst>(V);
  bool SI = isa<StoreInst>(V);
  if (!LI && !SI)
    return false;
  auto *Ty = getMemInstValueType(V);
  Align Align = getLoadStoreAlignment(V);
  return (LI && isLegalMaskedGather(Ty, Align)) ||
         (SI && isLegalMaskedScatter(Ty, Align));
}

/// Returns true if the target machine supports all of the reduction
/// variables found for the given VF.
bool canVectorizeReductions(ElementCount VF) {
  return (all_of(Legal->getReductionVars(), [&](auto &Reduction) -> bool {
    RecurrenceDescriptor RdxDesc = Reduction.second;
    return TTI.isLegalToVectorizeReduction(RdxDesc, VF);
  }));
}

/// Returns true if \p I is an instruction that will be scalarized with
/// predication. Such instructions include conditional stores and
/// instructions that may divide by zero.
/// If a non-zero VF has been calculated, we check if I will be scalarized
/// predication for that VF.
bool
isScalarWithPredication(Instruction *I,
                        ElementCount VF = ElementCount::getFixed(1)) const;

// Returns true if \p I is an instruction that will be predicated either
// through scalar predication or masked load/store or masked gather/scatter.
// Superset of instructions that return true for isScalarWithPredication.
bool isPredicatedInst(Instruction *I) {
  if (!blockNeedsPredication(I->getParent()))
    return false;
  // Loads and stores that need some form of masked operation are predicated
  // instructions.
  if (isa<LoadInst>(I) || isa<StoreInst>(I))
    return Legal->isMaskRequired(I);
  return isScalarWithPredication(I);
}

/// Returns true if \p I is a memory instruction with consecutive memory
/// access that can be widened.
bool
memoryInstructionCanBeWidened(Instruction *I,
                              ElementCount VF = ElementCount::getFixed(1));

/// Returns true if \p I is a memory instruction in an interleaved-group
/// of memory accesses that can be vectorized with wide vector loads/stores
/// and shuffles.
bool
interleavedAccessCanBeWidened(Instruction *I,
                              ElementCount VF = ElementCount::getFixed(1));

/// Check if \p Instr belongs to any interleaved access group.
bool isAccessInterleaved(Instruction *Instr) {
  return InterleaveInfo.isInterleaved(Instr);
}

/// Get the interleaved access group that \p Instr belongs to.
const InterleaveGroup<Instruction> *
getInterleavedAccessGroup(Instruction *Instr) {
  return InterleaveInfo.getInterleaveGroup(Instr);
}

/// Returns true if we're required to use a scalar epilogue for at least
/// the final iteration of the original loop.
bool requiresScalarEpilogue() const {
  if (!isScalarEpilogueAllowed())
    return false;
  // If we might exit from anywhere but the latch, must run the exiting
  // iteration in scalar form.
  if (TheLoop->getExitingBlock() != TheLoop->getLoopLatch())
    return true;
  return InterleaveInfo.requiresScalarEpilogue();
}

/// Returns true if a scalar epilogue is not allowed due to optsize or a
/// loop hint annotation.
bool isScalarEpilogueAllowed() const {
  return ScalarEpilogueStatus == CM_ScalarEpilogueAllowed;
}

/// Returns true if all loop blocks should be masked to fold tail loop.
bool foldTailByMasking() const { return FoldTailByMasking; }

bool blockNeedsPredication(BasicBlock *BB) const {
  return foldTailByMasking() || Legal->blockNeedsPredication(BB);
}

/// A SmallMapVector to store the InLoop reduction op chains, mapping phi
/// nodes to the chain of instructions representing the reductions. Uses a
/// MapVector to ensure deterministic iteration order.
using ReductionChainMap =
    SmallMapVector<PHINode *, SmallVector<Instruction *, 4>, 4>;

/// Return the chain of instructions representing an inloop reduction.
const ReductionChainMap &getInLoopReductionChains() const {
  return InLoopReductionChains;
}

/// Returns true if the Phi is part of an inloop reduction.
bool isInLoopReduction(PHINode *Phi) const {
  return InLoopReductionChains.count(Phi);
}

/// Estimate cost of an intrinsic call instruction CI if it were vectorized
/// with factor VF.  Return the cost of the instruction, including
/// scalarization overhead if it's needed.
InstructionCost getVectorIntrinsicCost(CallInst *CI, ElementCount VF) const;

/// Estimate cost of a call instruction CI if it were vectorized with factor
/// VF. Return the cost of the instruction, including scalarization overhead
/// if it's needed. The flag NeedToScalarize shows if the call needs to be
/// scalarized -
/// i.e. either vector version isn't available, or is too expensive.
InstructionCost getVectorCallCost(CallInst *CI, ElementCount VF,
                                  bool &NeedToScalarize) const;

/// Invalidates decisions already taken by the cost model.
void invalidateCostModelingDecisions() {
  WideningDecisions.clear();
  Uniforms.clear();
  Scalars.clear();
}

1612private:
unsigned NumPredStores = 0;

/// \return An upper bound for the vectorization factor, a power-of-2 larger
/// than zero. One is returned if vectorization should best be avoided due
/// to cost.
ElementCount computeFeasibleMaxVF(unsigned ConstTripCount,
                                  ElementCount UserVF);

/// The vectorization cost is a combination of the cost itself and a boolean
/// indicating whether any of the contributing operations will actually
/// operate on
/// vector values after type legalization in the backend. If this latter value
/// is
/// false, then all operations will be scalarized (i.e. no vectorization has
/// actually taken place).
using VectorizationCostTy = std::pair<InstructionCost, bool>;

/// Returns the expected execution cost. The unit of the cost does
/// not matter because we use the 'cost' units to compare different
/// vector widths. The cost that is returned is *not* normalized by
/// the factor width.
VectorizationCostTy expectedCost(ElementCount VF);

/// Returns the execution time cost of an instruction for a given vector
/// width. Vector width of one means scalar.
VectorizationCostTy getInstructionCost(Instruction *I, ElementCount VF);

/// The cost-computation logic from getInstructionCost which provides
/// the vector type as an output parameter.
InstructionCost getInstructionCost(Instruction *I, ElementCount VF,
                                   Type *&VectorTy);

/// Return the cost of instructions in an inloop reduction pattern, if I is
/// part of that pattern.
InstructionCost getReductionPatternCost(Instruction *I, ElementCount VF,
                                        Type *VectorTy,
                                        TTI::TargetCostKind CostKind);

/// Calculate vectorization cost of memory instruction \p I.
InstructionCost getMemoryInstructionCost(Instruction *I, ElementCount VF);

/// The cost computation for scalarized memory instruction.
InstructionCost getMemInstScalarizationCost(Instruction *I, ElementCount VF);

/// The cost computation for interleaving group of memory instructions.
InstructionCost getInterleaveGroupCost(Instruction *I, ElementCount VF);

/// The cost computation for Gather/Scatter instruction.
InstructionCost getGatherScatterCost(Instruction *I, ElementCount VF);

/// The cost computation for widening instruction \p I with consecutive
/// memory access.
InstructionCost getConsecutiveMemOpCost(Instruction *I, ElementCount VF);

/// The cost calculation for Load/Store instruction \p I with uniform pointer -
/// Load: scalar load + broadcast.
/// Store: scalar store + (loop invariant value stored? 0 : extract of last
/// element)
InstructionCost getUniformMemOpCost(Instruction *I, ElementCount VF);

/// Estimate the overhead of scalarizing an instruction. This is a
/// convenience wrapper for the type-based getScalarizationOverhead API.
InstructionCost getScalarizationOverhead(Instruction *I,
                                         ElementCount VF) const;

/// Returns whether the instruction is a load or store and will be a emitted
/// as a vector operation.
bool isConsecutiveLoadOrStore(Instruction *I);

/// Returns true if an artificially high cost for emulated masked memrefs
/// should be used.
bool useEmulatedMaskMemRefHack(Instruction *I);

/// Map of scalar integer values to the smallest bitwidth they can be legally
/// represented as. The vector equivalents of these values should be truncated
/// to this type.
MapVector<Instruction *, uint64_t> MinBWs;

/// A type representing the costs for instructions if they were to be
/// scalarized rather than vectorized. The entries are Instruction-Cost
/// pairs.
using ScalarCostsTy = DenseMap<Instruction *, InstructionCost>;

/// A set containing all BasicBlocks that are known to present after
/// vectorization as a predicated block.
SmallPtrSet<BasicBlock *, 4> PredicatedBBsAfterVectorization;

/// Records whether it is allowed to have the original scalar loop execute at
/// least once. This may be needed as a fallback loop in case runtime
/// aliasing/dependence checks fail, or to handle the tail/remainder
/// iterations when the trip count is unknown or doesn't divide by the VF,
/// or as a peel-loop to handle gaps in interleave-groups.
/// Under optsize and when the trip count is very small we don't allow any
/// iterations to execute in the scalar loop.
ScalarEpilogueLowering ScalarEpilogueStatus = CM_ScalarEpilogueAllowed;

/// All blocks of loop are to be masked to fold tail of scalar iterations.
bool FoldTailByMasking = false;

/// A map holding scalar costs for different vectorization factors. The
/// presence of a cost for an instruction in the mapping indicates that the
/// instruction will be scalarized when vectorizing with the associated
/// vectorization factor. The entries are VF-ScalarCostTy pairs.
DenseMap<ElementCount, ScalarCostsTy> InstsToScalarize;

/// Holds the instructions known to be uniform after vectorization.
/// The data is collected per VF.
DenseMap<ElementCount, SmallPtrSet<Instruction *, 4>> Uniforms;

/// Holds the instructions known to be scalar after vectorization.
/// The data is collected per VF.
DenseMap<ElementCount, SmallPtrSet<Instruction *, 4>> Scalars;

/// Holds the instructions (address computations) that are forced to be
/// scalarized.
DenseMap<ElementCount, SmallPtrSet<Instruction *, 4>> ForcedScalars;

/// PHINodes of the reductions that should be expanded in-loop along with
/// their associated chains of reduction operations, in program order from top
/// (PHI) to bottom
ReductionChainMap InLoopReductionChains;

/// A Map of inloop reduction operations and their immediate chain operand.
/// FIXME: This can be removed once reductions can be costed correctly in
/// vplan. This was added to allow quick lookup to the inloop operations,
/// without having to loop through InLoopReductionChains.
DenseMap<Instruction *, Instruction *> InLoopReductionImmediateChains;

/// Returns the expected difference in cost from scalarizing the expression
/// feeding a predicated instruction \p PredInst. The instructions to
/// scalarize and their scalar costs are collected in \p ScalarCosts. A
/// non-negative return value implies the expression will be scalarized.
/// Currently, only single-use chains are considered for scalarization.
int computePredInstDiscount(Instruction *PredInst, ScalarCostsTy &ScalarCosts,
                            ElementCount VF);

/// Collect the instructions that are uniform after vectorization. An
/// instruction is uniform if we represent it with a single scalar value in
/// the vectorized loop corresponding to each vector iteration. Examples of
/// uniform instructions include pointer operands of consecutive or
/// interleaved memory accesses. Note that although uniformity implies an
/// instruction will be scalar, the reverse is not true. In general, a
/// scalarized instruction will be represented by VF scalar values in the
/// vectorized loop, each corresponding to an iteration of the original
/// scalar loop.
void collectLoopUniforms(ElementCount VF);

/// Collect the instructions that are scalar after vectorization. An
/// instruction is scalar if it is known to be uniform or will be scalarized
/// during vectorization. Non-uniform scalarized instructions will be
/// represented by VF values in the vectorized loop, each corresponding to an
/// iteration of the original scalar loop.
void collectLoopScalars(ElementCount VF);

/// Keeps cost model vectorization decision and cost for instructions.
/// Right now it is used for memory instructions only.
using DecisionList = DenseMap<std::pair<Instruction *, ElementCount>,
                              std::pair<InstWidening, InstructionCost>>;

DecisionList WideningDecisions;

/// Returns true if \p V is expected to be vectorized and it needs to be
/// extracted.
bool needsExtract(Value *V, ElementCount VF) const {
  Instruction *I = dyn_cast<Instruction>(V);
  if (VF.isScalar() || !I || !TheLoop->contains(I) ||
      TheLoop->isLoopInvariant(I))
    return false;

  // Assume we can vectorize V (and hence we need extraction) if the
  // scalars are not computed yet. This can happen, because it is called
  // via getScalarizationOverhead from setCostBasedWideningDecision, before
  // the scalars are collected. That should be a safe assumption in most
  // cases, because we check if the operands have vectorizable types
  // beforehand in LoopVectorizationLegality.
  return Scalars.find(VF) == Scalars.end() ||
         !isScalarAfterVectorization(I, VF);
};

/// Returns a range containing only operands needing to be extracted.
SmallVector<Value *, 4> filterExtractingOperands(Instruction::op_range Ops,
                                                 ElementCount VF) const {
  return SmallVector<Value *, 4>(make_filter_range(
      Ops, [this, VF](Value *V) { return this->needsExtract(V, VF); }));
}

/// Determines if we have the infrastructure to vectorize loop \p L and its
/// epilogue, assuming the main loop is vectorized by \p VF.
bool isCandidateForEpilogueVectorization(const Loop &L,
                                         const ElementCount VF) const;

/// Returns true if epilogue vectorization is considered profitable, and
/// false otherwise.
/// \p VF is the vectorization factor chosen for the original loop.
bool isEpilogueVectorizationProfitable(const ElementCount VF) const;

1809public:
/// The loop that we evaluate.
Loop *TheLoop;

/// Predicated scalar evolution analysis.
PredicatedScalarEvolution &PSE;

/// Loop Info analysis.
LoopInfo *LI;

/// Vectorization legality.
LoopVectorizationLegality *Legal;

/// Vector target information.
const TargetTransformInfo &TTI;

/// Target Library Info.
const TargetLibraryInfo *TLI;

/// Demanded bits analysis.
DemandedBits *DB;

/// Assumption cache.
AssumptionCache *AC;

/// Interface to emit optimization remarks.
OptimizationRemarkEmitter *ORE;

const Function *TheFunction;

/// Loop Vectorize Hint.
const LoopVectorizeHints *Hints;

/// The interleave access information contains groups of interleaved accesses
/// with the same stride and close to each other.
InterleavedAccessInfo &InterleaveInfo;

/// Values to ignore in the cost model.
SmallPtrSet<const Value *, 16> ValuesToIgnore;

/// Values to ignore in the cost model when VF > 1.
SmallPtrSet<const Value *, 16> VecValuesToIgnore;

/// Profitable vector factors.
SmallVector<VectorizationFactor, 8> ProfitableVFs;
1854};
1855} // end namespace llvm

1857/// Helper struct to manage generating runtime checks for vectorization.
1858///
1859/// The runtime checks are created up-front in temporary blocks to allow better
1860/// estimating the cost and un-linked from the existing IR. After deciding to
1861/// vectorize, the checks are moved back. If deciding not to vectorize, the
1862/// temporary blocks are completely removed.
1863class GeneratedRTChecks {
/// Basic block which contains the generated SCEV checks, if any.
BasicBlock *SCEVCheckBlock = nullptr;

/// The value representing the result of the generated SCEV checks. If it is
/// nullptr, either no SCEV checks have been generated or they have been used.
Value *SCEVCheckCond = nullptr;

/// Basic block which contains the generated memory runtime checks, if any.
BasicBlock *MemCheckBlock = nullptr;

/// The value representing the result of the generated memory runtime checks.
/// If it is nullptr, either no memory runtime checks have been generated or
/// they have been used.
Instruction *MemRuntimeCheckCond = nullptr;

DominatorTree *DT;
LoopInfo *LI;

SCEVExpander SCEVExp;
SCEVExpander MemCheckExp;

1885public:
GeneratedRTChecks(ScalarEvolution &SE, DominatorTree *DT, LoopInfo *LI,
                  const DataLayout &DL)
    : DT(DT), LI(LI), SCEVExp(SE, DL, "scev.check"),
      MemCheckExp(SE, DL, "scev.check") {}

/// Generate runtime checks in SCEVCheckBlock and MemCheckBlock, so we can
/// accurately estimate the cost of the runtime checks. The blocks are
/// un-linked from the IR and is added back during vector code generation. If
/// there is no vector code generation, the check blocks are removed
/// completely.
void Create(Loop *L, const LoopAccessInfo &LAI,
            const SCEVUnionPredicate &UnionPred) {

  BasicBlock *LoopHeader = L->getHeader();
  BasicBlock *Preheader = L->getLoopPreheader();

  // Use SplitBlock to create blocks for SCEV & memory runtime checks to
  // ensure the blocks are properly added to LoopInfo & DominatorTree. Those
  // may be used by SCEVExpander. The blocks will be un-linked from their
  // predecessors and removed from LI & DT at the end of the function.
  if (!UnionPred.isAlwaysTrue()) {
    SCEVCheckBlock = SplitBlock(Preheader, Preheader->getTerminator(), DT, LI,
                                nullptr, "vector.scevcheck");

    SCEVCheckCond = SCEVExp.expandCodeForPredicate(
        &UnionPred, SCEVCheckBlock->getTerminator());
  }

  const auto &RtPtrChecking = *LAI.getRuntimePointerChecking();
  if (RtPtrChecking.Need) {
    auto *Pred = SCEVCheckBlock ? SCEVCheckBlock : Preheader;
    MemCheckBlock = SplitBlock(Pred, Pred->getTerminator(), DT, LI, nullptr,
                               "vector.memcheck");

    std::tie(std::ignore, MemRuntimeCheckCond) =
        addRuntimeChecks(MemCheckBlock->getTerminator(), L,
                         RtPtrChecking.getChecks(), MemCheckExp);
    assert(MemRuntimeCheckCond &&((MemRuntimeCheckCond && "no RT checks generated although RtPtrChecking "
 "claimed checks are required") ? static_cast<void> (0)
 : __assert_fail ("MemRuntimeCheckCond && \"no RT checks generated although RtPtrChecking \" \"claimed checks are required\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1925, __PRETTY_FUNCTION__))
           "no RT checks generated although RtPtrChecking "((MemRuntimeCheckCond && "no RT checks generated although RtPtrChecking "
 "claimed checks are required") ? static_cast<void> (0)
 : __assert_fail ("MemRuntimeCheckCond && \"no RT checks generated although RtPtrChecking \" \"claimed checks are required\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1925, __PRETTY_FUNCTION__))
           "claimed checks are required")((MemRuntimeCheckCond && "no RT checks generated although RtPtrChecking "
 "claimed checks are required") ? static_cast<void> (0)
 : __assert_fail ("MemRuntimeCheckCond && \"no RT checks generated although RtPtrChecking \" \"claimed checks are required\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1925, __PRETTY_FUNCTION__));
  }

  if (!MemCheckBlock && !SCEVCheckBlock)
    return;

  // Unhook the temporary block with the checks, update various places
  // accordingly.
  if (SCEVCheckBlock)
    SCEVCheckBlock->replaceAllUsesWith(Preheader);
  if (MemCheckBlock)
    MemCheckBlock->replaceAllUsesWith(Preheader);

  if (SCEVCheckBlock) {
    SCEVCheckBlock->getTerminator()->moveBefore(Preheader->getTerminator());
    new UnreachableInst(Preheader->getContext(), SCEVCheckBlock);
    Preheader->getTerminator()->eraseFromParent();
  }
  if (MemCheckBlock) {
    MemCheckBlock->getTerminator()->moveBefore(Preheader->getTerminator());
    new UnreachableInst(Preheader->getContext(), MemCheckBlock);
    Preheader->getTerminator()->eraseFromParent();
  }

  DT->changeImmediateDominator(LoopHeader, Preheader);
  if (MemCheckBlock) {
    DT->eraseNode(MemCheckBlock);
    LI->removeBlock(MemCheckBlock);
  }
  if (SCEVCheckBlock) {
    DT->eraseNode(SCEVCheckBlock);
    LI->removeBlock(SCEVCheckBlock);
  }
}

/// Remove the created SCEV & memory runtime check blocks & instructions, if
/// unused.
~GeneratedRTChecks() {
  SCEVExpanderCleaner SCEVCleaner(SCEVExp, *DT);
  SCEVExpanderCleaner MemCheckCleaner(MemCheckExp, *DT);
  if (!SCEVCheckCond)
    SCEVCleaner.markResultUsed();

  if (!MemRuntimeCheckCond)
    MemCheckCleaner.markResultUsed();

  if (MemRuntimeCheckCond) {
    auto &SE = *MemCheckExp.getSE();
    // Memory runtime check generation creates compares that use expanded
    // values. Remove them before running the SCEVExpanderCleaners.
    for (auto &I : make_early_inc_range(reverse(*MemCheckBlock))) {
      if (MemCheckExp.isInsertedInstruction(&I))
        continue;
      SE.forgetValue(&I);
      SE.eraseValueFromMap(&I);
      I.eraseFromParent();
    }
  }
  MemCheckCleaner.cleanup();
  SCEVCleaner.cleanup();

  if (SCEVCheckCond)
    SCEVCheckBlock->eraseFromParent();
  if (MemRuntimeCheckCond)
    MemCheckBlock->eraseFromParent();
}

/// Adds the generated SCEVCheckBlock before \p LoopVectorPreHeader and
/// adjusts the branches to branch to the vector preheader or \p Bypass,
/// depending on the generated condition.
BasicBlock *emitSCEVChecks(Loop *L, BasicBlock *Bypass,
                           BasicBlock *LoopVectorPreHeader,
                           BasicBlock *LoopExitBlock) {
  if (!SCEVCheckCond)
    return nullptr;
  if (auto *C = dyn_cast<ConstantInt>(SCEVCheckCond))
    if (C->isZero())
      return nullptr;

  auto *Pred = LoopVectorPreHeader->getSinglePredecessor();

  BranchInst::Create(LoopVectorPreHeader, SCEVCheckBlock);
  // Create new preheader for vector loop.
  if (auto *PL = LI->getLoopFor(LoopVectorPreHeader))
    PL->addBasicBlockToLoop(SCEVCheckBlock, *LI);

  SCEVCheckBlock->getTerminator()->eraseFromParent();
  SCEVCheckBlock->moveBefore(LoopVectorPreHeader);
  Pred->getTerminator()->replaceSuccessorWith(LoopVectorPreHeader,
                                              SCEVCheckBlock);

  DT->addNewBlock(SCEVCheckBlock, Pred);
  DT->changeImmediateDominator(LoopVectorPreHeader, SCEVCheckBlock);

  ReplaceInstWithInst(
      SCEVCheckBlock->getTerminator(),
      BranchInst::Create(Bypass, LoopVectorPreHeader, SCEVCheckCond));
  // Mark the check as used, to prevent it from being removed during cleanup.
  SCEVCheckCond = nullptr;
  return SCEVCheckBlock;
}

/// Adds the generated MemCheckBlock before \p LoopVectorPreHeader and adjusts
/// the branches to branch to the vector preheader or \p Bypass, depending on
/// the generated condition.
BasicBlock *emitMemRuntimeChecks(Loop *L, BasicBlock *Bypass,
                                 BasicBlock *LoopVectorPreHeader) {
  // Check if we generated code that checks in runtime if arrays overlap.
  if (!MemRuntimeCheckCond)
    return nullptr;

  auto *Pred = LoopVectorPreHeader->getSinglePredecessor();
  Pred->getTerminator()->replaceSuccessorWith(LoopVectorPreHeader,
                                              MemCheckBlock);

  DT->addNewBlock(MemCheckBlock, Pred);
  DT->changeImmediateDominator(LoopVectorPreHeader, MemCheckBlock);
  MemCheckBlock->moveBefore(LoopVectorPreHeader);

  if (auto *PL = LI->getLoopFor(LoopVectorPreHeader))
    PL->addBasicBlockToLoop(MemCheckBlock, *LI);

  ReplaceInstWithInst(
      MemCheckBlock->getTerminator(),
      BranchInst::Create(Bypass, LoopVectorPreHeader, MemRuntimeCheckCond));
  MemCheckBlock->getTerminator()->setDebugLoc(
      Pred->getTerminator()->getDebugLoc());

  // Mark the check as used, to prevent it from being removed during cleanup.
  MemRuntimeCheckCond = nullptr;
  return MemCheckBlock;
}
2057};

2059// Return true if \p OuterLp is an outer loop annotated with hints for explicit
2060// vectorization. The loop needs to be annotated with #pragma omp simd
2061// simdlen(#) or #pragma clang vectorize(enable) vectorize_width(#). If the
2062// vector length information is not provided, vectorization is not considered
2063// explicit. Interleave hints are not allowed either. These limitations will be
2064// relaxed in the future.
2065// Please, note that we are currently forced to abuse the pragma 'clang
2066// vectorize' semantics. This pragma provides *auto-vectorization hints*
2067// (i.e., LV must check that vectorization is legal) whereas pragma 'omp simd'
2068// provides *explicit vectorization hints* (LV can bypass legal checks and
2069// assume that vectorization is legal). However, both hints are implemented
2070// using the same metadata (llvm.loop.vectorize, processed by
2071// LoopVectorizeHints). This will be fixed in the future when the native IR
2072// representation for pragma 'omp simd' is introduced.
2073static bool isExplicitVecOuterLoop(Loop *OuterLp,
                                 OptimizationRemarkEmitter *ORE) {
assert(!OuterLp->isInnermost() && "This is not an outer loop")((!OuterLp->isInnermost() && "This is not an outer loop"
) ? static_cast<void> (0) : __assert_fail ("!OuterLp->isInnermost() && \"This is not an outer loop\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2075, __PRETTY_FUNCTION__));
LoopVectorizeHints Hints(OuterLp, true /*DisableInterleaving*/, *ORE);

// Only outer loops with an explicit vectorization hint are supported.
// Unannotated outer loops are ignored.
if (Hints.getForce() == LoopVectorizeHints::FK_Undefined)
  return false;

Function *Fn = OuterLp->getHeader()->getParent();
if (!Hints.allowVectorization(Fn, OuterLp,
                              true /*VectorizeOnlyWhenForced*/)) {
  LLVM_DEBUG(dbgs() << "LV: Loop hints prevent outer loop vectorization.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Loop hints prevent outer loop vectorization.\n"
; } } while (false);
  return false;
}

if (Hints.getInterleave() > 1) {
  // TODO: Interleave support is future work.
  LLVM_DEBUG(dbgs() << "LV: Not vectorizing: Interleave is not supported for "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Not vectorizing: Interleave is not supported for "
 "outer loops.\n"; } } while (false)
                       "outer loops.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Not vectorizing: Interleave is not supported for "
 "outer loops.\n"; } } while (false);
  Hints.emitRemarkWithHints();
  return false;
}

return true;
2099}

2101static void collectSupportedLoops(Loop &L, LoopInfo *LI,
                                OptimizationRemarkEmitter *ORE,
                                SmallVectorImpl<Loop *> &V) {
// Collect inner loops and outer loops without irreducible control flow. For
// now, only collect outer loops that have explicit vectorization hints. If we
// are stress testing the VPlan H-CFG construction, we collect the outermost
// loop of every loop nest.
if (L.isInnermost() || VPlanBuildStressTest ||
    (EnableVPlanNativePath && isExplicitVecOuterLoop(&L, ORE))) {
  LoopBlocksRPO RPOT(&L);
  RPOT.perform(LI);
  if (!containsIrreducibleCFG<const BasicBlock *>(RPOT, *LI)) {
    V.push_back(&L);
    // TODO: Collect inner loops inside marked outer loops in case
    // vectorization fails for the outer loop. Do not invoke
    // 'containsIrreducibleCFG' again for inner loops when the outer loop is
    // already known to be reducible. We can use an inherited attribute for
    // that.
    return;
  }
}
for (Loop *InnerL : L)
  collectSupportedLoops(*InnerL, LI, ORE, V);
2124}

2126namespace {

2128/// The LoopVectorize Pass.
2129struct LoopVectorize : public FunctionPass {
/// Pass identification, replacement for typeid
static char ID;

LoopVectorizePass Impl;

explicit LoopVectorize(bool InterleaveOnlyWhenForced = false,
                       bool VectorizeOnlyWhenForced = false)
    : FunctionPass(ID),
      Impl({InterleaveOnlyWhenForced, VectorizeOnlyWhenForced}) {
  initializeLoopVectorizePass(*PassRegistry::getPassRegistry());
}

bool runOnFunction(Function &F) override {
  if (skipFunction(F))
    return false;

  auto *SE = &getAnalysis<ScalarEvolutionWrapperPass>().getSE();
  auto *LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
  auto *TTI = &getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
  auto *DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree();
  auto *BFI = &getAnalysis<BlockFrequencyInfoWrapperPass>().getBFI();
  auto *TLIP = getAnalysisIfAvailable<TargetLibraryInfoWrapperPass>();
  auto *TLI = TLIP ? &TLIP->getTLI(F) : nullptr;
  auto *AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
  auto *AC = &getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F);
  auto *LAA = &getAnalysis<LoopAccessLegacyAnalysis>();
  auto *DB = &getAnalysis<DemandedBitsWrapperPass>().getDemandedBits();
  auto *ORE = &getAnalysis<OptimizationRemarkEmitterWrapperPass>().getORE();
  auto *PSI = &getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI();

  std::function<const LoopAccessInfo &(Loop &)> GetLAA =
      [&](Loop &L) -> const LoopAccessInfo & { return LAA->getInfo(&L); };

  return Impl.runImpl(F, *SE, *LI, *TTI, *DT, *BFI, TLI, *DB, *AA, *AC,
                      GetLAA, *ORE, PSI).MadeAnyChange;
}

void getAnalysisUsage(AnalysisUsage &AU) const override {
  AU.addRequired<AssumptionCacheTracker>();
  AU.addRequired<BlockFrequencyInfoWrapperPass>();
  AU.addRequired<DominatorTreeWrapperPass>();
  AU.addRequired<LoopInfoWrapperPass>();
  AU.addRequired<ScalarEvolutionWrapperPass>();
  AU.addRequired<TargetTransformInfoWrapperPass>();
  AU.addRequired<AAResultsWrapperPass>();
  AU.addRequired<LoopAccessLegacyAnalysis>();
  AU.addRequired<DemandedBitsWrapperPass>();
  AU.addRequired<OptimizationRemarkEmitterWrapperPass>();
  AU.addRequired<InjectTLIMappingsLegacy>();

  // We currently do not preserve loopinfo/dominator analyses with outer loop
  // vectorization. Until this is addressed, mark these analyses as preserved
  // only for non-VPlan-native path.
  // TODO: Preserve Loop and Dominator analyses for VPlan-native path.
  if (!EnableVPlanNativePath) {
    AU.addPreserved<LoopInfoWrapperPass>();
    AU.addPreserved<DominatorTreeWrapperPass>();
  }

  AU.addPreserved<BasicAAWrapperPass>();
  AU.addPreserved<GlobalsAAWrapperPass>();
  AU.addRequired<ProfileSummaryInfoWrapperPass>();
}
2193};

2195} // end anonymous namespace

2197//===----------------------------------------------------------------------===//
2198// Implementation of LoopVectorizationLegality, InnerLoopVectorizer and
2199// LoopVectorizationCostModel and LoopVectorizationPlanner.
2200//===----------------------------------------------------------------------===//

2202Value *InnerLoopVectorizer::getBroadcastInstrs(Value *V) {
// We need to place the broadcast of invariant variables outside the loop,
// but only if it's proven safe to do so. Else, broadcast will be inside
// vector loop body.
Instruction *Instr = dyn_cast<Instruction>(V);
bool SafeToHoist = OrigLoop->isLoopInvariant(V) &&
                   (!Instr ||
                    DT->dominates(Instr->getParent(), LoopVectorPreHeader));
// Place the code for broadcasting invariant variables in the new preheader.
IRBuilder<>::InsertPointGuard Guard(Builder);
if (SafeToHoist)
  Builder.SetInsertPoint(LoopVectorPreHeader->getTerminator());

// Broadcast the scalar into all locations in the vector.
Value *Shuf = Builder.CreateVectorSplat(VF, V, "broadcast");

return Shuf;
2219}

2221void InnerLoopVectorizer::createVectorIntOrFpInductionPHI(
  const InductionDescriptor &II, Value *Step, Value *Start,
  Instruction *EntryVal, VPValue *Def, VPValue *CastDef,
  VPTransformState &State) {
assert((isa<PHINode>(EntryVal) || isa<TruncInst>(EntryVal)) &&(((isa<PHINode>(EntryVal) || isa<TruncInst>(EntryVal
)) && "Expected either an induction phi-node or a truncate of it!"
) ? static_cast<void> (0) : __assert_fail ("(isa<PHINode>(EntryVal) || isa<TruncInst>(EntryVal)) && \"Expected either an induction phi-node or a truncate of it!\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2226, __PRETTY_FUNCTION__))
       "Expected either an induction phi-node or a truncate of it!")(((isa<PHINode>(EntryVal) || isa<TruncInst>(EntryVal
)) && "Expected either an induction phi-node or a truncate of it!"
) ? static_cast<void> (0) : __assert_fail ("(isa<PHINode>(EntryVal) || isa<TruncInst>(EntryVal)) && \"Expected either an induction phi-node or a truncate of it!\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2226, __PRETTY_FUNCTION__));

// Construct the initial value of the vector IV in the vector loop preheader
auto CurrIP = Builder.saveIP();
Builder.SetInsertPoint(LoopVectorPreHeader->getTerminator());
if (isa<TruncInst>(EntryVal)) {
  assert(Start->getType()->isIntegerTy() &&((Start->getType()->isIntegerTy() && "Truncation requires an integer type"
) ? static_cast<void> (0) : __assert_fail ("Start->getType()->isIntegerTy() && \"Truncation requires an integer type\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2233, __PRETTY_FUNCTION__))
         "Truncation requires an integer type")((Start->getType()->isIntegerTy() && "Truncation requires an integer type"
) ? static_cast<void> (0) : __assert_fail ("Start->getType()->isIntegerTy() && \"Truncation requires an integer type\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2233, __PRETTY_FUNCTION__));
  auto *TruncType = cast<IntegerType>(EntryVal->getType());
  Step = Builder.CreateTrunc(Step, TruncType);
  Start = Builder.CreateCast(Instruction::Trunc, Start, TruncType);
}
Value *SplatStart = Builder.CreateVectorSplat(VF, Start);
Value *SteppedStart =
    getStepVector(SplatStart, 0, Step, II.getInductionOpcode());

// We create vector phi nodes for both integer and floating-point induction
// variables. Here, we determine the kind of arithmetic we will perform.
Instruction::BinaryOps AddOp;
Instruction::BinaryOps MulOp;
if (Step->getType()->isIntegerTy()) {
  AddOp = Instruction::Add;
  MulOp = Instruction::Mul;
} else {
  AddOp = II.getInductionOpcode();
  MulOp = Instruction::FMul;
}

// Multiply the vectorization factor by the step using integer or
// floating-point arithmetic as appropriate.
Type *StepType = Step->getType();
if (Step->getType()->isFloatingPointTy())
  StepType = IntegerType::get(StepType->getContext(),
                              StepType->getScalarSizeInBits());
Value *RuntimeVF = getRuntimeVF(Builder, StepType, VF);
if (Step->getType()->isFloatingPointTy())
  RuntimeVF = Builder.CreateSIToFP(RuntimeVF, Step->getType());
Value *Mul = Builder.CreateBinOp(MulOp, Step, RuntimeVF);

// Create a vector splat to use in the induction update.
//
// FIXME: If the step is non-constant, we create the vector splat with
//        IRBuilder. IRBuilder can constant-fold the multiply, but it doesn't
//        handle a constant vector splat.
Value *SplatVF = isa<Constant>(Mul)
                     ? ConstantVector::getSplat(VF, cast<Constant>(Mul))
                     : Builder.CreateVectorSplat(VF, Mul);
Builder.restoreIP(CurrIP);

// We may need to add the step a number of times, depending on the unroll
// factor. The last of those goes into the PHI.
PHINode *VecInd = PHINode::Create(SteppedStart->getType(), 2, "vec.ind",
                                  &*LoopVectorBody->getFirstInsertionPt());
VecInd->setDebugLoc(EntryVal->getDebugLoc());
Instruction *LastInduction = VecInd;
for (unsigned Part = 0; Part < UF; ++Part) {
  State.set(Def, LastInduction, Part);

  if (isa<TruncInst>(EntryVal))
    addMetadata(LastInduction, EntryVal);
  recordVectorLoopValueForInductionCast(II, EntryVal, LastInduction, CastDef,
                                        State, Part);

  LastInduction = cast<Instruction>(
      Builder.CreateBinOp(AddOp, LastInduction, SplatVF, "step.add"));
  LastInduction->setDebugLoc(EntryVal->getDebugLoc());
}

// Move the last step to the end of the latch block. This ensures consistent
// placement of all induction updates.
auto *LoopVectorLatch = LI->getLoopFor(LoopVectorBody)->getLoopLatch();
auto *Br = cast<BranchInst>(LoopVectorLatch->getTerminator());
auto *ICmp = cast<Instruction>(Br->getCondition());
LastInduction->moveBefore(ICmp);
LastInduction->setName("vec.ind.next");

VecInd->addIncoming(SteppedStart, LoopVectorPreHeader);
VecInd->addIncoming(LastInduction, LoopVectorLatch);
2304}

2306bool InnerLoopVectorizer::shouldScalarizeInstruction(Instruction *I) const {
return Cost->isScalarAfterVectorization(I, VF) ||
       Cost->isProfitableToScalarize(I, VF);
2309}

2311bool InnerLoopVectorizer::needsScalarInduction(Instruction *IV) const {
if (shouldScalarizeInstruction(IV))
  return true;
auto isScalarInst = [&](User *U) -> bool {
  auto *I = cast<Instruction>(U);
  return (OrigLoop->contains(I) && shouldScalarizeInstruction(I));
};
return llvm::any_of(IV->users(), isScalarInst);
2319}

2321void InnerLoopVectorizer::recordVectorLoopValueForInductionCast(
  const InductionDescriptor &ID, const Instruction *EntryVal,
  Value *VectorLoopVal, VPValue *CastDef, VPTransformState &State,
  unsigned Part, unsigned Lane) {
assert((isa<PHINode>(EntryVal) || isa<TruncInst>(EntryVal)) &&(((isa<PHINode>(EntryVal) || isa<TruncInst>(EntryVal
)) && "Expected either an induction phi-node or a truncate of it!"
) ? static_cast<void> (0) : __assert_fail ("(isa<PHINode>(EntryVal) || isa<TruncInst>(EntryVal)) && \"Expected either an induction phi-node or a truncate of it!\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2326, __PRETTY_FUNCTION__))
       "Expected either an induction phi-node or a truncate of it!")(((isa<PHINode>(EntryVal) || isa<TruncInst>(EntryVal
)) && "Expected either an induction phi-node or a truncate of it!"
) ? static_cast<void> (0) : __assert_fail ("(isa<PHINode>(EntryVal) || isa<TruncInst>(EntryVal)) && \"Expected either an induction phi-node or a truncate of it!\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2326, __PRETTY_FUNCTION__));

// This induction variable is not the phi from the original loop but the
// newly-created IV based on the proof that casted Phi is equal to the
// uncasted Phi in the vectorized loop (under a runtime guard possibly). It
// re-uses the same InductionDescriptor that original IV uses but we don't
// have to do any recording in this case - that is done when original IV is
// processed.
if (isa<TruncInst>(EntryVal))
  return;

const SmallVectorImpl<Instruction *> &Casts = ID.getCastInsts();
if (Casts.empty())
  return;
// Only the first Cast instruction in the Casts vector is of interest.
// The rest of the Casts (if exist) have no uses outside the
// induction update chain itself.
if (Lane < UINT_MAX(2147483647 *2U +1U))
  State.set(CastDef, VectorLoopVal, VPIteration(Part, Lane));
else
  State.set(CastDef, VectorLoopVal, Part);
2347}

2349void InnerLoopVectorizer::widenIntOrFpInduction(PHINode *IV, Value *Start,
                                              TruncInst *Trunc, VPValue *Def,
                                              VPValue *CastDef,
                                              VPTransformState &State) {
assert((IV->getType()->isIntegerTy() || IV != OldInduction) &&(((IV->getType()->isIntegerTy() || IV != OldInduction) &&
 "Primary induction variable must have an integer type") ? static_cast
<void> (0) : __assert_fail ("(IV->getType()->isIntegerTy() || IV != OldInduction) && \"Primary induction variable must have an integer type\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2354, __PRETTY_FUNCTION__))
       "Primary induction variable must have an integer type")(((IV->getType()->isIntegerTy() || IV != OldInduction) &&
 "Primary induction variable must have an integer type") ? static_cast
<void> (0) : __assert_fail ("(IV->getType()->isIntegerTy() || IV != OldInduction) && \"Primary induction variable must have an integer type\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2354, __PRETTY_FUNCTION__));

auto II = Legal->getInductionVars().find(IV);
assert(II != Legal->getInductionVars().end() && "IV is not an induction")((II != Legal->getInductionVars().end() && "IV is not an induction"
) ? static_cast<void> (0) : __assert_fail ("II != Legal->getInductionVars().end() && \"IV is not an induction\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2357, __PRETTY_FUNCTION__));

auto ID = II->second;
assert(IV->getType() == ID.getStartValue()->getType() && "Types must match")((IV->getType() == ID.getStartValue()->getType() &&
 "Types must match") ? static_cast<void> (0) : __assert_fail
 ("IV->getType() == ID.getStartValue()->getType() && \"Types must match\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2360, __PRETTY_FUNCTION__));

// The value from the original loop to which we are mapping the new induction
// variable.
Instruction *EntryVal = Trunc ? cast<Instruction>(Trunc) : IV;

auto &DL = OrigLoop->getHeader()->getModule()->getDataLayout();

// Generate code for the induction step. Note that induction steps are
// required to be loop-invariant
auto CreateStepValue = [&](const SCEV *Step) -> Value * {
  assert(PSE.getSE()->isLoopInvariant(Step, OrigLoop) &&((PSE.getSE()->isLoopInvariant(Step, OrigLoop) && "Induction step should be loop invariant"
) ? static_cast<void> (0) : __assert_fail ("PSE.getSE()->isLoopInvariant(Step, OrigLoop) && \"Induction step should be loop invariant\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2372, __PRETTY_FUNCTION__))
         "Induction step should be loop invariant")((PSE.getSE()->isLoopInvariant(Step, OrigLoop) && "Induction step should be loop invariant"
) ? static_cast<void> (0) : __assert_fail ("PSE.getSE()->isLoopInvariant(Step, OrigLoop) && \"Induction step should be loop invariant\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2372, __PRETTY_FUNCTION__));
  if (PSE.getSE()->isSCEVable(IV->getType())) {
    SCEVExpander Exp(*PSE.getSE(), DL, "induction");
    return Exp.expandCodeFor(Step, Step->getType(),
                             LoopVectorPreHeader->getTerminator());
  }
  return cast<SCEVUnknown>(Step)->getValue();
};

// The scalar value to broadcast. This is derived from the canonical
// induction variable. If a truncation type is given, truncate the canonical
// induction variable and step. Otherwise, derive these values from the
// induction descriptor.
auto CreateScalarIV = [&](Value *&Step) -> Value * {
  Value *ScalarIV = Induction;
  if (IV != OldInduction) {
    ScalarIV = IV->getType()->isIntegerTy()
                   ? Builder.CreateSExtOrTrunc(Induction, IV->getType())
                   : Builder.CreateCast(Instruction::SIToFP, Induction,
                                        IV->getType());
    ScalarIV = emitTransformedIndex(Builder, ScalarIV, PSE.getSE(), DL, ID);
    ScalarIV->setName("offset.idx");
  }
  if (Trunc) {
    auto *TruncType = cast<IntegerType>(Trunc->getType());
    assert(Step->getType()->isIntegerTy() &&((Step->getType()->isIntegerTy() && "Truncation requires an integer step"
) ? static_cast<void> (0) : __assert_fail ("Step->getType()->isIntegerTy() && \"Truncation requires an integer step\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2398, __PRETTY_FUNCTION__))
           "Truncation requires an integer step")((Step->getType()->isIntegerTy() && "Truncation requires an integer step"
) ? static_cast<void> (0) : __assert_fail ("Step->getType()->isIntegerTy() && \"Truncation requires an integer step\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2398, __PRETTY_FUNCTION__));
    ScalarIV = Builder.CreateTrunc(ScalarIV, TruncType);
    Step = Builder.CreateTrunc(Step, TruncType);
  }
  return ScalarIV;
};

// Create the vector values from the scalar IV, in the absence of creating a
// vector IV.
auto CreateSplatIV = [&](Value *ScalarIV, Value *Step) {
  Value *Broadcasted = getBroadcastInstrs(ScalarIV);
  for (unsigned Part = 0; Part < UF; ++Part) {
    assert(!VF.isScalable() && "scalable vectors not yet supported.")((!VF.isScalable() && "scalable vectors not yet supported."
) ? static_cast<void> (0) : __assert_fail ("!VF.isScalable() && \"scalable vectors not yet supported.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2410, __PRETTY_FUNCTION__));
    Value *EntryPart =
        getStepVector(Broadcasted, VF.getKnownMinValue() * Part, Step,
                      ID.getInductionOpcode());
    State.set(Def, EntryPart, Part);
    if (Trunc)
      addMetadata(EntryPart, Trunc);
    recordVectorLoopValueForInductionCast(ID, EntryVal, EntryPart, CastDef,
                                          State, Part);
  }
};

// Fast-math-flags propagate from the original induction instruction.
IRBuilder<>::FastMathFlagGuard FMFG(Builder);
if (ID.getInductionBinOp() && isa<FPMathOperator>(ID.getInductionBinOp()))
  Builder.setFastMathFlags(ID.getInductionBinOp()->getFastMathFlags());

// Now do the actual transformations, and start with creating the step value.
Value *Step = CreateStepValue(ID.getStep());
if (VF.isZero() || VF.isScalar()) {
  Value *ScalarIV = CreateScalarIV(Step);
  CreateSplatIV(ScalarIV, Step);
  return;
}

// Determine if we want a scalar version of the induction variable. This is
// true if the induction variable itself is not widened, or if it has at
// least one user in the loop that is not widened.
auto NeedsScalarIV = needsScalarInduction(EntryVal);
if (!NeedsScalarIV) {
  createVectorIntOrFpInductionPHI(ID, Step, Start, EntryVal, Def, CastDef,
                                  State);
  return;
}

// Try to create a new independent vector induction variable. If we can't
// create the phi node, we will splat the scalar induction variable in each
// loop iteration.
if (!shouldScalarizeInstruction(EntryVal)) {
  createVectorIntOrFpInductionPHI(ID, Step, Start, EntryVal, Def, CastDef,
                                  State);
  Value *ScalarIV = CreateScalarIV(Step);
  // Create scalar steps that can be used by instructions we will later
  // scalarize. Note that the addition of the scalar steps will not increase
  // the number of instructions in the loop in the common case prior to
  // InstCombine. We will be trading one vector extract for each scalar step.
  buildScalarSteps(ScalarIV, Step, EntryVal, ID, Def, CastDef, State);
  return;
}

// All IV users are scalar instructions, so only emit a scalar IV, not a
// vectorised IV. Except when we tail-fold, then the splat IV feeds the
// predicate used by the masked loads/stores.
Value *ScalarIV = CreateScalarIV(Step);
if (!Cost->isScalarEpilogueAllowed())
  CreateSplatIV(ScalarIV, Step);
buildScalarSteps(ScalarIV, Step, EntryVal, ID, Def, CastDef, State);
2467}

2469Value *InnerLoopVectorizer::getStepVector(Value *Val, int StartIdx, Value *Step,
                                        Instruction::BinaryOps BinOp) {
// Create and check the types.
auto *ValVTy = cast<VectorType>(Val->getType());
ElementCount VLen = ValVTy->getElementCount();

Type *STy = Val->getType()->getScalarType();
assert((STy->isIntegerTy() || STy->isFloatingPointTy()) &&(((STy->isIntegerTy() || STy->isFloatingPointTy()) &&
 "Induction Step must be an integer or FP") ? static_cast<
void> (0) : __assert_fail ("(STy->isIntegerTy() || STy->isFloatingPointTy()) && \"Induction Step must be an integer or FP\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2477, __PRETTY_FUNCTION__))
       "Induction Step must be an integer or FP")(((STy->isIntegerTy() || STy->isFloatingPointTy()) &&
 "Induction Step must be an integer or FP") ? static_cast<
void> (0) : __assert_fail ("(STy->isIntegerTy() || STy->isFloatingPointTy()) && \"Induction Step must be an integer or FP\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2477, __PRETTY_FUNCTION__));
assert(Step->getType() == STy && "Step has wrong type")((Step->getType() == STy && "Step has wrong type")
 ? static_cast<void> (0) : __assert_fail ("Step->getType() == STy && \"Step has wrong type\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2478, __PRETTY_FUNCTION__));

SmallVector<Constant *, 8> Indices;

// Create a vector of consecutive numbers from zero to VF.
VectorType *InitVecValVTy = ValVTy;
Type *InitVecValSTy = STy;
if (STy->isFloatingPointTy()) {
  InitVecValSTy =
      IntegerType::get(STy->getContext(), STy->getScalarSizeInBits());
  InitVecValVTy = VectorType::get(InitVecValSTy, VLen);
}
Value *InitVec = Builder.CreateStepVector(InitVecValVTy);

// Add on StartIdx
Value *StartIdxSplat = Builder.CreateVectorSplat(
    VLen, ConstantInt::get(InitVecValSTy, StartIdx));
InitVec = Builder.CreateAdd(InitVec, StartIdxSplat);

if (STy->isIntegerTy()) {
  Step = Builder.CreateVectorSplat(VLen, Step);
  assert(Step->getType() == Val->getType() && "Invalid step vec")((Step->getType() == Val->getType() && "Invalid step vec"
) ? static_cast<void> (0) : __assert_fail ("Step->getType() == Val->getType() && \"Invalid step vec\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2499, __PRETTY_FUNCTION__));
  // FIXME: The newly created binary instructions should contain nsw/nuw flags,
  // which can be found from the original scalar operations.
  Step = Builder.CreateMul(InitVec, Step);
  return Builder.CreateAdd(Val, Step, "induction");
}

// Floating point induction.
assert((BinOp == Instruction::FAdd || BinOp == Instruction::FSub) &&(((BinOp == Instruction::FAdd || BinOp == Instruction::FSub) &&
 "Binary Opcode should be specified for FP induction") ? static_cast
<void> (0) : __assert_fail ("(BinOp == Instruction::FAdd || BinOp == Instruction::FSub) && \"Binary Opcode should be specified for FP induction\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2508, __PRETTY_FUNCTION__))
       "Binary Opcode should be specified for FP induction")(((BinOp == Instruction::FAdd || BinOp == Instruction::FSub) &&
 "Binary Opcode should be specified for FP induction") ? static_cast
<void> (0) : __assert_fail ("(BinOp == Instruction::FAdd || BinOp == Instruction::FSub) && \"Binary Opcode should be specified for FP induction\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2508, __PRETTY_FUNCTION__));
InitVec = Builder.CreateUIToFP(InitVec, ValVTy);
Step = Builder.CreateVectorSplat(VLen, Step);
Value *MulOp = Builder.CreateFMul(InitVec, Step);
return Builder.CreateBinOp(BinOp, Val, MulOp, "induction");
2513}

2515void InnerLoopVectorizer::buildScalarSteps(Value *ScalarIV, Value *Step,
                                         Instruction *EntryVal,
                                         const InductionDescriptor &ID,
                                         VPValue *Def, VPValue *CastDef,
                                         VPTransformState &State) {
// We shouldn't have to build scalar steps if we aren't vectorizing.
assert(VF.isVector() && "VF should be greater than one")((VF.isVector() && "VF should be greater than one") ?
 static_cast<void> (0) : __assert_fail ("VF.isVector() && \"VF should be greater than one\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2521, __PRETTY_FUNCTION__));
// Get the value type and ensure it and the step have the same integer type.
Type *ScalarIVTy = ScalarIV->getType()->getScalarType();
assert(ScalarIVTy == Step->getType() &&((ScalarIVTy == Step->getType() && "Val and Step should have the same type"
) ? static_cast<void> (0) : __assert_fail ("ScalarIVTy == Step->getType() && \"Val and Step should have the same type\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2525, __PRETTY_FUNCTION__))
       "Val and Step should have the same type")((ScalarIVTy == Step->getType() && "Val and Step should have the same type"
) ? static_cast<void> (0) : __assert_fail ("ScalarIVTy == Step->getType() && \"Val and Step should have the same type\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2525, __PRETTY_FUNCTION__));

// We build scalar steps for both integer and floating-point induction
// variables. Here, we determine the kind of arithmetic we will perform.
Instruction::BinaryOps AddOp;
Instruction::BinaryOps MulOp;
if (ScalarIVTy->isIntegerTy()) {
  AddOp = Instruction::Add;
  MulOp = Instruction::Mul;
} else {
  AddOp = ID.getInductionOpcode();
  MulOp = Instruction::FMul;
}

// Determine the number of scalars we need to generate for each unroll
// iteration. If EntryVal is uniform, we only need to generate the first
// lane. Otherwise, we generate all VF values.
bool IsUniform =
    Cost->isUniformAfterVectorization(cast<Instruction>(EntryVal), VF);
unsigned Lanes = IsUniform ? 1 : VF.getKnownMinValue();
// Compute the scalar steps and save the results in State.
Type *IntStepTy = IntegerType::get(ScalarIVTy->getContext(),
                                   ScalarIVTy->getScalarSizeInBits());
Type *VecIVTy = nullptr;
Value *UnitStepVec = nullptr, *SplatStep = nullptr, *SplatIV = nullptr;
if (!IsUniform && VF.isScalable()) {
  VecIVTy = VectorType::get(ScalarIVTy, VF);
  UnitStepVec = Builder.CreateStepVector(VectorType::get(IntStepTy, VF));
  SplatStep = Builder.CreateVectorSplat(VF, Step);
  SplatIV = Builder.CreateVectorSplat(VF, ScalarIV);
}

for (unsigned Part = 0; Part < UF; ++Part) {
  Value *StartIdx0 =
      createStepForVF(Builder, ConstantInt::get(IntStepTy, Part), VF);

  if (!IsUniform && VF.isScalable()) {
    auto *SplatStartIdx = Builder.CreateVectorSplat(VF, StartIdx0);
    auto *InitVec = Builder.CreateAdd(SplatStartIdx, UnitStepVec);
    if (ScalarIVTy->isFloatingPointTy())
      InitVec = Builder.CreateSIToFP(InitVec, VecIVTy);
    auto *Mul = Builder.CreateBinOp(MulOp, InitVec, SplatStep);
    auto *Add = Builder.CreateBinOp(AddOp, SplatIV, Mul);
    State.set(Def, Add, Part);
    recordVectorLoopValueForInductionCast(ID, EntryVal, Add, CastDef, State,
                                          Part);
    // It's useful to record the lane values too for the known minimum number
    // of elements so we do those below. This improves the code quality when
    // trying to extract the first element, for example.
  }

  if (ScalarIVTy->isFloatingPointTy())
    StartIdx0 = Builder.CreateSIToFP(StartIdx0, ScalarIVTy);

  for (unsigned Lane = 0; Lane < Lanes; ++Lane) {
    Value *StartIdx = Builder.CreateBinOp(
        AddOp, StartIdx0, getSignedIntOrFpConstant(ScalarIVTy, Lane));
    // The step returned by `createStepForVF` is a runtime-evaluated value
    // when VF is scalable. Otherwise, it should be folded into a Constant.
    assert((VF.isScalable() || isa<Constant>(StartIdx)) &&(((VF.isScalable() || isa<Constant>(StartIdx)) &&
 "Expected StartIdx to be folded to a constant when VF is not "
 "scalable") ? static_cast<void> (0) : __assert_fail ("(VF.isScalable() || isa<Constant>(StartIdx)) && \"Expected StartIdx to be folded to a constant when VF is not \" \"scalable\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2586, __PRETTY_FUNCTION__))
           "Expected StartIdx to be folded to a constant when VF is not "(((VF.isScalable() || isa<Constant>(StartIdx)) &&
 "Expected StartIdx to be folded to a constant when VF is not "
 "scalable") ? static_cast<void> (0) : __assert_fail ("(VF.isScalable() || isa<Constant>(StartIdx)) && \"Expected StartIdx to be folded to a constant when VF is not \" \"scalable\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2586, __PRETTY_FUNCTION__))
           "scalable")(((VF.isScalable() || isa<Constant>(StartIdx)) &&
 "Expected StartIdx to be folded to a constant when VF is not "
 "scalable") ? static_cast<void> (0) : __assert_fail ("(VF.isScalable() || isa<Constant>(StartIdx)) && \"Expected StartIdx to be folded to a constant when VF is not \" \"scalable\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2586, __PRETTY_FUNCTION__));
    auto *Mul = Builder.CreateBinOp(MulOp, StartIdx, Step);
    auto *Add = Builder.CreateBinOp(AddOp, ScalarIV, Mul);
    State.set(Def, Add, VPIteration(Part, Lane));
    recordVectorLoopValueForInductionCast(ID, EntryVal, Add, CastDef, State,
                                          Part, Lane);
  }
}
2594}

2596void InnerLoopVectorizer::packScalarIntoVectorValue(VPValue *Def,
                                                  const VPIteration &Instance,
                                                  VPTransformState &State) {
Value *ScalarInst = State.get(Def, Instance);
Value *VectorValue = State.get(Def, Instance.Part);
VectorValue = Builder.CreateInsertElement(
    VectorValue, ScalarInst,
    Instance.Lane.getAsRuntimeExpr(State.Builder, VF));
State.set(Def, VectorValue, Instance.Part);
2605}

2607Value *InnerLoopVectorizer::reverseVector(Value *Vec) {
assert(Vec->getType()->isVectorTy() && "Invalid type")((Vec->getType()->isVectorTy() && "Invalid type"
) ? static_cast<void> (0) : __assert_fail ("Vec->getType()->isVectorTy() && \"Invalid type\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2608, __PRETTY_FUNCTION__));
return Builder.CreateVectorReverse(Vec, "reverse");
2610}

2612// Return whether we allow using masked interleave-groups (for dealing with
2613// strided loads/stores that reside in predicated blocks, or for dealing
2614// with gaps).
2615static bool useMaskedInterleavedAccesses(const TargetTransformInfo &TTI) {
// If an override option has been passed in for interleaved accesses, use it.
if (EnableMaskedInterleavedMemAccesses.getNumOccurrences() > 0)
  return EnableMaskedInterleavedMemAccesses;

return TTI.enableMaskedInterleavedAccessVectorization();
2621}

2623// Try to vectorize the interleave group that \p Instr belongs to.
2624//
2625// E.g. Translate following interleaved load group (factor = 3):
2626//   for (i = 0; i < N; i+=3) {
2627//     R = Pic[i];             // Member of index 0
2628//     G = Pic[i+1];           // Member of index 1
2629//     B = Pic[i+2];           // Member of index 2
2630//     ... // do something to R, G, B
2631//   }
2632// To:
2633//   %wide.vec = load <12 x i32>                       ; Read 4 tuples of R,G,B
2634//   %R.vec = shuffle %wide.vec, poison, <0, 3, 6, 9>   ; R elements
2635//   %G.vec = shuffle %wide.vec, poison, <1, 4, 7, 10>  ; G elements
2636//   %B.vec = shuffle %wide.vec, poison, <2, 5, 8, 11>  ; B elements
2637//
2638// Or translate following interleaved store group (factor = 3):
2639//   for (i = 0; i < N; i+=3) {
2640//     ... do something to R, G, B
2641//     Pic[i]   = R;           // Member of index 0
2642//     Pic[i+1] = G;           // Member of index 1
2643//     Pic[i+2] = B;           // Member of index 2
2644//   }
2645// To:
2646//   %R_G.vec = shuffle %R.vec, %G.vec, <0, 1, 2, ..., 7>
2647//   %B_U.vec = shuffle %B.vec, poison, <0, 1, 2, 3, u, u, u, u>
2648//   %interleaved.vec = shuffle %R_G.vec, %B_U.vec,
2649//        <0, 4, 8, 1, 5, 9, 2, 6, 10, 3, 7, 11>    ; Interleave R,G,B elements
2650//   store <12 x i32> %interleaved.vec              ; Write 4 tuples of R,G,B
2651void InnerLoopVectorizer::vectorizeInterleaveGroup(
  const InterleaveGroup<Instruction> *Group, ArrayRef<VPValue *> VPDefs,
  VPTransformState &State, VPValue *Addr, ArrayRef<VPValue *> StoredValues,
  VPValue *BlockInMask) {
Instruction *Instr = Group->getInsertPos();
const DataLayout &DL = Instr->getModule()->getDataLayout();

// Prepare for the vector type of the interleaved load/store.
Type *ScalarTy = getMemInstValueType(Instr);
unsigned InterleaveFactor = Group->getFactor();
assert(!VF.isScalable() && "scalable vectors not yet supported.")((!VF.isScalable() && "scalable vectors not yet supported."
) ? static_cast<void> (0) : __assert_fail ("!VF.isScalable() && \"scalable vectors not yet supported.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2661, __PRETTY_FUNCTION__));
auto *VecTy = VectorType::get(ScalarTy, VF * InterleaveFactor);

// Prepare for the new pointers.
SmallVector<Value *, 2> AddrParts;
unsigned Index = Group->getIndex(Instr);

// TODO: extend the masked interleaved-group support to reversed access.
assert((!BlockInMask || !Group->isReverse()) &&(((!BlockInMask || !Group->isReverse()) && "Reversed masked interleave-group not supported."
) ? static_cast<void> (0) : __assert_fail ("(!BlockInMask || !Group->isReverse()) && \"Reversed masked interleave-group not supported.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2670, __PRETTY_FUNCTION__))
       "Reversed masked interleave-group not supported.")(((!BlockInMask || !Group->isReverse()) && "Reversed masked interleave-group not supported."
) ? static_cast<void> (0) : __assert_fail ("(!BlockInMask || !Group->isReverse()) && \"Reversed masked interleave-group not supported.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2670, __PRETTY_FUNCTION__));

// If the group is reverse, adjust the index to refer to the last vector lane
// instead of the first. We adjust the index from the first vector lane,
// rather than directly getting the pointer for lane VF - 1, because the
// pointer operand of the interleaved access is supposed to be uniform. For
// uniform instructions, we're only required to generate a value for the
// first vector lane in each unroll iteration.
assert(!VF.isScalable() &&((!VF.isScalable() && "scalable vector reverse operation is not implemented"
) ? static_cast<void> (0) : __assert_fail ("!VF.isScalable() && \"scalable vector reverse operation is not implemented\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2679, __PRETTY_FUNCTION__))
       "scalable vector reverse operation is not implemented")((!VF.isScalable() && "scalable vector reverse operation is not implemented"
) ? static_cast<void> (0) : __assert_fail ("!VF.isScalable() && \"scalable vector reverse operation is not implemented\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2679, __PRETTY_FUNCTION__));
if (Group->isReverse())
  Index += (VF.getKnownMinValue() - 1) * Group->getFactor();

for (unsigned Part = 0; Part < UF; Part++) {
  Value *AddrPart = State.get(Addr, VPIteration(Part, 0));
  setDebugLocFromInst(Builder, AddrPart);

  // Notice current instruction could be any index. Need to adjust the address
  // to the member of index 0.
  //
  // E.g.  a = A[i+1];     // Member of index 1 (Current instruction)
  //       b = A[i];       // Member of index 0
  // Current pointer is pointed to A[i+1], adjust it to A[i].
  //
  // E.g.  A[i+1] = a;     // Member of index 1
  //       A[i]   = b;     // Member of index 0
  //       A[i+2] = c;     // Member of index 2 (Current instruction)
  // Current pointer is pointed to A[i+2], adjust it to A[i].

  bool InBounds = false;
  if (auto *gep = dyn_cast<GetElementPtrInst>(AddrPart->stripPointerCasts()))
    InBounds = gep->isInBounds();
  AddrPart = Builder.CreateGEP(ScalarTy, AddrPart, Builder.getInt32(-Index));
  cast<GetElementPtrInst>(AddrPart)->setIsInBounds(InBounds);

  // Cast to the vector pointer type.
  unsigned AddressSpace = AddrPart->getType()->getPointerAddressSpace();
  Type *PtrTy = VecTy->getPointerTo(AddressSpace);
  AddrParts.push_back(Builder.CreateBitCast(AddrPart, PtrTy));
}

setDebugLocFromInst(Builder, Instr);
Value *PoisonVec = PoisonValue::get(VecTy);

Value *MaskForGaps = nullptr;
if (Group->requiresScalarEpilogue() && !Cost->isScalarEpilogueAllowed()) {
  assert(!VF.isScalable() && "scalable vectors not yet supported.")((!VF.isScalable() && "scalable vectors not yet supported."
) ? static_cast<void> (0) : __assert_fail ("!VF.isScalable() && \"scalable vectors not yet supported.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2716, __PRETTY_FUNCTION__));
  MaskForGaps = createBitMaskForGaps(Builder, VF.getKnownMinValue(), *Group);
  assert(MaskForGaps && "Mask for Gaps is required but it is null")((MaskForGaps && "Mask for Gaps is required but it is null"
) ? static_cast<void> (0) : __assert_fail ("MaskForGaps && \"Mask for Gaps is required but it is null\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2718, __PRETTY_FUNCTION__));
}

// Vectorize the interleaved load group.
if (isa<LoadInst>(Instr)) {
  // For each unroll part, create a wide load for the group.
  SmallVector<Value *, 2> NewLoads;
  for (unsigned Part = 0; Part < UF; Part++) {
    Instruction *NewLoad;
    if (BlockInMask || MaskForGaps) {
      assert(useMaskedInterleavedAccesses(*TTI) &&((useMaskedInterleavedAccesses(*TTI) && "masked interleaved groups are not allowed."
) ? static_cast<void> (0) : __assert_fail ("useMaskedInterleavedAccesses(*TTI) && \"masked interleaved groups are not allowed.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2729, __PRETTY_FUNCTION__))
             "masked interleaved groups are not allowed.")((useMaskedInterleavedAccesses(*TTI) && "masked interleaved groups are not allowed."
) ? static_cast<void> (0) : __assert_fail ("useMaskedInterleavedAccesses(*TTI) && \"masked interleaved groups are not allowed.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2729, __PRETTY_FUNCTION__));
      Value *GroupMask = MaskForGaps;
      if (BlockInMask) {
        Value *BlockInMaskPart = State.get(BlockInMask, Part);
        assert(!VF.isScalable() && "scalable vectors not yet supported.")((!VF.isScalable() && "scalable vectors not yet supported."
) ? static_cast<void> (0) : __assert_fail ("!VF.isScalable() && \"scalable vectors not yet supported.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2733, __PRETTY_FUNCTION__));
        Value *ShuffledMask = Builder.CreateShuffleVector(
            BlockInMaskPart,
            createReplicatedMask(InterleaveFactor, VF.getKnownMinValue()),
            "interleaved.mask");
        GroupMask = MaskForGaps
                        ? Builder.CreateBinOp(Instruction::And, ShuffledMask,
                                              MaskForGaps)
                        : ShuffledMask;
      }
      NewLoad =
          Builder.CreateMaskedLoad(AddrParts[Part], Group->getAlign(),
                                   GroupMask, PoisonVec, "wide.masked.vec");
    }
    else
      NewLoad = Builder.CreateAlignedLoad(VecTy, AddrParts[Part],
                                          Group->getAlign(), "wide.vec");
    Group->addMetadata(NewLoad);
    NewLoads.push_back(NewLoad);
  }

  // For each member in the group, shuffle out the appropriate data from the
  // wide loads.
  unsigned J = 0;
  for (unsigned I = 0; I < InterleaveFactor; ++I) {
    Instruction *Member = Group->getMember(I);

    // Skip the gaps in the group.
    if (!Member)
      continue;

    assert(!VF.isScalable() && "scalable vectors not yet supported.")((!VF.isScalable() && "scalable vectors not yet supported."
) ? static_cast<void> (0) : __assert_fail ("!VF.isScalable() && \"scalable vectors not yet supported.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2764, __PRETTY_FUNCTION__));
    auto StrideMask =
        createStrideMask(I, InterleaveFactor, VF.getKnownMinValue());
    for (unsigned Part = 0; Part < UF; Part++) {
      Value *StridedVec = Builder.CreateShuffleVector(
          NewLoads[Part], StrideMask, "strided.vec");

      // If this member has different type, cast the result type.
      if (Member->getType() != ScalarTy) {
        assert(!VF.isScalable() && "VF is assumed to be non scalable.")((!VF.isScalable() && "VF is assumed to be non scalable."
) ? static_cast<void> (0) : __assert_fail ("!VF.isScalable() && \"VF is assumed to be non scalable.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2773, __PRETTY_FUNCTION__));
        VectorType *OtherVTy = VectorType::get(Member->getType(), VF);
        StridedVec = createBitOrPointerCast(StridedVec, OtherVTy, DL);
      }

      if (Group->isReverse())
        StridedVec = reverseVector(StridedVec);

      State.set(VPDefs[J], StridedVec, Part);
    }
    ++J;
  }
  return;
}

// The sub vector type for current instruction.
assert(!VF.isScalable() && "VF is assumed to be non scalable.")((!VF.isScalable() && "VF is assumed to be non scalable."
) ? static_cast<void> (0) : __assert_fail ("!VF.isScalable() && \"VF is assumed to be non scalable.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2789, __PRETTY_FUNCTION__));
auto *SubVT = VectorType::get(ScalarTy, VF);

// Vectorize the interleaved store group.
for (unsigned Part = 0; Part < UF; Part++) {
  // Collect the stored vector from each member.
  SmallVector<Value *, 4> StoredVecs;
  for (unsigned i = 0; i < InterleaveFactor; i++) {
    // Interleaved store group doesn't allow a gap, so each index has a member
    assert(Group->getMember(i) && "Fail to get a member from an interleaved store group")((Group->getMember(i) && "Fail to get a member from an interleaved store group"
) ? static_cast<void> (0) : __assert_fail ("Group->getMember(i) && \"Fail to get a member from an interleaved store group\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2798, __PRETTY_FUNCTION__));

    Value *StoredVec = State.get(StoredValues[i], Part);

    if (Group->isReverse())
      StoredVec = reverseVector(StoredVec);

    // If this member has different type, cast it to a unified type.

    if (StoredVec->getType() != SubVT)
      StoredVec = createBitOrPointerCast(StoredVec, SubVT, DL);

    StoredVecs.push_back(StoredVec);
  }

  // Concatenate all vectors into a wide vector.
  Value *WideVec = concatenateVectors(Builder, StoredVecs);

  // Interleave the elements in the wide vector.
  assert(!VF.isScalable() && "scalable vectors not yet supported.")((!VF.isScalable() && "scalable vectors not yet supported."
) ? static_cast<void> (0) : __assert_fail ("!VF.isScalable() && \"scalable vectors not yet supported.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2817, __PRETTY_FUNCTION__));
  Value *IVec = Builder.CreateShuffleVector(
      WideVec, createInterleaveMask(VF.getKnownMinValue(), InterleaveFactor),
      "interleaved.vec");

  Instruction *NewStoreInstr;
  if (BlockInMask) {
    Value *BlockInMaskPart = State.get(BlockInMask, Part);
    Value *ShuffledMask = Builder.CreateShuffleVector(
        BlockInMaskPart,
        createReplicatedMask(InterleaveFactor, VF.getKnownMinValue()),
        "interleaved.mask");
    NewStoreInstr = Builder.CreateMaskedStore(
        IVec, AddrParts[Part], Group->getAlign(), ShuffledMask);
  }
  else
    NewStoreInstr =
        Builder.CreateAlignedStore(IVec, AddrParts[Part], Group->getAlign());

  Group->addMetadata(NewStoreInstr);
}
2838}

2840void InnerLoopVectorizer::vectorizeMemoryInstruction(
  Instruction *Instr, VPTransformState &State, VPValue *Def, VPValue *Addr,
  VPValue *StoredValue, VPValue *BlockInMask) {
// Attempt to issue a wide load.
LoadInst *LI = dyn_cast<LoadInst>(Instr);
StoreInst *SI = dyn_cast<StoreInst>(Instr);

assert((LI || SI) && "Invalid Load/Store instruction")(((LI || SI) && "Invalid Load/Store instruction") ? static_cast
<void> (0) : __assert_fail ("(LI || SI) && \"Invalid Load/Store instruction\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2847, __PRETTY_FUNCTION__));
assert((!SI || StoredValue) && "No stored value provided for widened store")(((!SI || StoredValue) && "No stored value provided for widened store"
) ? static_cast<void> (0) : __assert_fail ("(!SI || StoredValue) && \"No stored value provided for widened store\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2848, __PRETTY_FUNCTION__));
assert((!LI || !StoredValue) && "Stored value provided for widened load")(((!LI || !StoredValue) && "Stored value provided for widened load"
) ? static_cast<void> (0) : __assert_fail ("(!LI || !StoredValue) && \"Stored value provided for widened load\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2849, __PRETTY_FUNCTION__));

LoopVectorizationCostModel::InstWidening Decision =
    Cost->getWideningDecision(Instr, VF);
assert((Decision == LoopVectorizationCostModel::CM_Widen ||(((Decision == LoopVectorizationCostModel::CM_Widen || Decision
 == LoopVectorizationCostModel::CM_Widen_Reverse || Decision ==
 LoopVectorizationCostModel::CM_GatherScatter) && "CM decision is not to widen the memory instruction"
) ? static_cast<void> (0) : __assert_fail ("(Decision == LoopVectorizationCostModel::CM_Widen || Decision == LoopVectorizationCostModel::CM_Widen_Reverse || Decision == LoopVectorizationCostModel::CM_GatherScatter) && \"CM decision is not to widen the memory instruction\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2856, __PRETTY_FUNCTION__))
        Decision == LoopVectorizationCostModel::CM_Widen_Reverse ||(((Decision == LoopVectorizationCostModel::CM_Widen || Decision
 == LoopVectorizationCostModel::CM_Widen_Reverse || Decision ==
 LoopVectorizationCostModel::CM_GatherScatter) && "CM decision is not to widen the memory instruction"
) ? static_cast<void> (0) : __assert_fail ("(Decision == LoopVectorizationCostModel::CM_Widen || Decision == LoopVectorizationCostModel::CM_Widen_Reverse || Decision == LoopVectorizationCostModel::CM_GatherScatter) && \"CM decision is not to widen the memory instruction\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2856, __PRETTY_FUNCTION__))
        Decision == LoopVectorizationCostModel::CM_GatherScatter) &&(((Decision == LoopVectorizationCostModel::CM_Widen || Decision
 == LoopVectorizationCostModel::CM_Widen_Reverse || Decision ==
 LoopVectorizationCostModel::CM_GatherScatter) && "CM decision is not to widen the memory instruction"
) ? static_cast<void> (0) : __assert_fail ("(Decision == LoopVectorizationCostModel::CM_Widen || Decision == LoopVectorizationCostModel::CM_Widen_Reverse || Decision == LoopVectorizationCostModel::CM_GatherScatter) && \"CM decision is not to widen the memory instruction\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2856, __PRETTY_FUNCTION__))
       "CM decision is not to widen the memory instruction")(((Decision == LoopVectorizationCostModel::CM_Widen || Decision
 == LoopVectorizationCostModel::CM_Widen_Reverse || Decision ==
 LoopVectorizationCostModel::CM_GatherScatter) && "CM decision is not to widen the memory instruction"
) ? static_cast<void> (0) : __assert_fail ("(Decision == LoopVectorizationCostModel::CM_Widen || Decision == LoopVectorizationCostModel::CM_Widen_Reverse || Decision == LoopVectorizationCostModel::CM_GatherScatter) && \"CM decision is not to widen the memory instruction\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2856, __PRETTY_FUNCTION__));

Type *ScalarDataTy = getMemInstValueType(Instr);

auto *DataTy = VectorType::get(ScalarDataTy, VF);
const Align Alignment = getLoadStoreAlignment(Instr);

// Determine if the pointer operand of the access is either consecutive or
// reverse consecutive.
bool Reverse = (Decision == LoopVectorizationCostModel::CM_Widen_Reverse);
bool ConsecutiveStride =
    Reverse || (Decision == LoopVectorizationCostModel::CM_Widen);
bool CreateGatherScatter =
    (Decision == LoopVectorizationCostModel::CM_GatherScatter);

// Either Ptr feeds a vector load/store, or a vector GEP should feed a vector
// gather/scatter. Otherwise Decision should have been to Scalarize.
assert((ConsecutiveStride || CreateGatherScatter) &&(((ConsecutiveStride || CreateGatherScatter) && "The instruction should be scalarized"
) ? static_cast<void> (0) : __assert_fail ("(ConsecutiveStride || CreateGatherScatter) && \"The instruction should be scalarized\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2874, __PRETTY_FUNCTION__))
       "The instruction should be scalarized")(((ConsecutiveStride || CreateGatherScatter) && "The instruction should be scalarized"
) ? static_cast<void> (0) : __assert_fail ("(ConsecutiveStride || CreateGatherScatter) && \"The instruction should be scalarized\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2874, __PRETTY_FUNCTION__));
(void)ConsecutiveStride;

VectorParts BlockInMaskParts(UF);
bool isMaskRequired = BlockInMask;
if (isMaskRequired)
  for (unsigned Part = 0; Part < UF; ++Part)
    BlockInMaskParts[Part] = State.get(BlockInMask, Part);

const auto CreateVecPtr = [&](unsigned Part, Value *Ptr) -> Value * {
  // Calculate the pointer for the specific unroll-part.
  GetElementPtrInst *PartPtr = nullptr;

  bool InBounds = false;
  if (auto *gep = dyn_cast<GetElementPtrInst>(Ptr->stripPointerCasts()))
    InBounds = gep->isInBounds();
  if (Reverse) {
    // If the address is consecutive but reversed, then the
    // wide store needs to start at the last vector element.
    // RunTimeVF =  VScale * VF.getKnownMinValue()
    // For fixed-width VScale is 1, then RunTimeVF = VF.getKnownMinValue()
    Value *RunTimeVF = getRuntimeVF(Builder, Builder.getInt32Ty(), VF);
    // NumElt = -Part * RunTimeVF
    Value *NumElt = Builder.CreateMul(Builder.getInt32(-Part), RunTimeVF);
    // LastLane = 1 - RunTimeVF
    Value *LastLane = Builder.CreateSub(Builder.getInt32(1), RunTimeVF);
    PartPtr =
        cast<GetElementPtrInst>(Builder.CreateGEP(ScalarDataTy, Ptr, NumElt));
    PartPtr->setIsInBounds(InBounds);
    PartPtr = cast<GetElementPtrInst>(
        Builder.CreateGEP(ScalarDataTy, PartPtr, LastLane));
    PartPtr->setIsInBounds(InBounds);
    if (isMaskRequired) // Reverse of a null all-one mask is a null mask.
      BlockInMaskParts[Part] = reverseVector(BlockInMaskParts[Part]);
  } else {
    Value *Increment = createStepForVF(Builder, Builder.getInt32(Part), VF);
    PartPtr = cast<GetElementPtrInst>(
        Builder.CreateGEP(ScalarDataTy, Ptr, Increment));
    PartPtr->setIsInBounds(InBounds);
  }

  unsigned AddressSpace = Ptr->getType()->getPointerAddressSpace();
  return Builder.CreateBitCast(PartPtr, DataTy->getPointerTo(AddressSpace));
};

// Handle Stores:
if (SI) {
  setDebugLocFromInst(Builder, SI);

  for (unsigned Part = 0; Part < UF; ++Part) {
    Instruction *NewSI = nullptr;
    Value *StoredVal = State.get(StoredValue, Part);
    if (CreateGatherScatter) {
      Value *MaskPart = isMaskRequired ? BlockInMaskParts[Part] : nullptr;
      Value *VectorGep = State.get(Addr, Part);
      NewSI = Builder.CreateMaskedScatter(StoredVal, VectorGep, Alignment,
                                          MaskPart);
    } else {
      if (Reverse) {
        // If we store to reverse consecutive memory locations, then we need
        // to reverse the order of elements in the stored value.
        StoredVal = reverseVector(StoredVal);
        // We don't want to update the value in the map as it might be used in
        // another expression. So don't call resetVectorValue(StoredVal).
      }
      auto *VecPtr = CreateVecPtr(Part, State.get(Addr, VPIteration(0, 0)));
      if (isMaskRequired)
        NewSI = Builder.CreateMaskedStore(StoredVal, VecPtr, Alignment,
                                          BlockInMaskParts[Part]);
      else
        NewSI = Builder.CreateAlignedStore(StoredVal, VecPtr, Alignment);
    }
    addMetadata(NewSI, SI);
  }
  return;
}

// Handle loads.
assert(LI && "Must have a load instruction")((LI && "Must have a load instruction") ? static_cast
<void> (0) : __assert_fail ("LI && \"Must have a load instruction\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2952, __PRETTY_FUNCTION__));
setDebugLocFromInst(Builder, LI);
for (unsigned Part = 0; Part < UF; ++Part) {
  Value *NewLI;
  if (CreateGatherScatter) {
    Value *MaskPart = isMaskRequired ? BlockInMaskParts[Part] : nullptr;
    Value *VectorGep = State.get(Addr, Part);
    NewLI = Builder.CreateMaskedGather(VectorGep, Alignment, MaskPart,
                                       nullptr, "wide.masked.gather");
    addMetadata(NewLI, LI);
  } else {
    auto *VecPtr = CreateVecPtr(Part, State.get(Addr, VPIteration(0, 0)));
    if (isMaskRequired)
      NewLI = Builder.CreateMaskedLoad(
          VecPtr, Alignment, BlockInMaskParts[Part], PoisonValue::get(DataTy),
          "wide.masked.load");
    else
      NewLI =
          Builder.CreateAlignedLoad(DataTy, VecPtr, Alignment, "wide.load");

    // Add metadata to the load, but setVectorValue to the reverse shuffle.
    addMetadata(NewLI, LI);
    if (Reverse)
      NewLI = reverseVector(NewLI);
  }

  State.set(Def, NewLI, Part);
}
2980}

2982void InnerLoopVectorizer::scalarizeInstruction(Instruction *Instr, VPValue *Def,
                                             VPUser &User,
                                             const VPIteration &Instance,
                                             bool IfPredicateInstr,
                                             VPTransformState &State) {
assert(!Instr->getType()->isAggregateType() && "Can't handle vectors")((!Instr->getType()->isAggregateType() && "Can't handle vectors"
) ? static_cast<void> (0) : __assert_fail ("!Instr->getType()->isAggregateType() && \"Can't handle vectors\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2987, __PRETTY_FUNCTION__));

// llvm.experimental.noalias.scope.decl intrinsics must only be duplicated for
// the first lane and part.
if (isa<NoAliasScopeDeclInst>(Instr))
  if (!Instance.isFirstIteration())
    return;

setDebugLocFromInst(Builder, Instr);

// Does this instruction return a value ?
bool IsVoidRetTy = Instr->getType()->isVoidTy();

Instruction *Cloned = Instr->clone();
if (!IsVoidRetTy)
  Cloned->setName(Instr->getName() + ".cloned");

State.Builder.SetInsertPoint(Builder.GetInsertBlock(),
                             Builder.GetInsertPoint());
// Replace the operands of the cloned instructions with their scalar
// equivalents in the new loop.
for (unsigned op = 0, e = User.getNumOperands(); op != e; ++op) {
  auto *Operand = dyn_cast<Instruction>(Instr->getOperand(op));
  auto InputInstance = Instance;
  if (!Operand || !OrigLoop->contains(Operand) ||
      (Cost->isUniformAfterVectorization(Operand, State.VF)))
    InputInstance.Lane = VPLane::getFirstLane();
  auto *NewOp = State.get(User.getOperand(op), InputInstance);
  Cloned->setOperand(op, NewOp);
}
addNewMetadata(Cloned, Instr);

// Place the cloned scalar in the new loop.
Builder.Insert(Cloned);

State.set(Def, Cloned, Instance);

// If we just cloned a new assumption, add it the assumption cache.
if (auto *II = dyn_cast<AssumeInst>(Cloned))
  AC->registerAssumption(II);

// End if-block.
if (IfPredicateInstr)
  PredicatedInstructions.push_back(Cloned);
3031}

3033PHINode *InnerLoopVectorizer::createInductionVariable(Loop *L, Value *Start,
                                                    Value *End, Value *Step,
                                                    Instruction *DL) {
BasicBlock *Header = L->getHeader();
BasicBlock *Latch = L->getLoopLatch();
// As we're just creating this loop, it's possible no latch exists
// yet. If so, use the header as this will be a single block loop.
if (!Latch)
  Latch = Header;

IRBuilder<> Builder(&*Header->getFirstInsertionPt());
Instruction *OldInst = getDebugLocFromInstOrOperands(OldInduction);
setDebugLocFromInst(Builder, OldInst);
auto *Induction = Builder.CreatePHI(Start->getType(), 2, "index");

Builder.SetInsertPoint(Latch->getTerminator());
setDebugLocFromInst(Builder, OldInst);

// Create i+1 and fill the PHINode.
Value *Next = Builder.CreateAdd(Induction, Step, "index.next");
Induction->addIncoming(Start, L->getLoopPreheader());
Induction->addIncoming(Next, Latch);
// Create the compare.
Value *ICmp = Builder.CreateICmpEQ(Next, End);
Builder.CreateCondBr(ICmp, L->getUniqueExitBlock(), Header);

// Now we have two terminators. Remove the old one from the block.
Latch->getTerminator()->eraseFromParent();

return Induction;
3063}

3065Value *InnerLoopVectorizer::getOrCreateTripCount(Loop *L) {
if (TripCount)
  return TripCount;

assert(L && "Create Trip Count for null loop.")((L && "Create Trip Count for null loop.") ? static_cast
<void> (0) : __assert_fail ("L && \"Create Trip Count for null loop.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3069, __PRETTY_FUNCTION__));
IRBuilder<> Builder(L->getLoopPreheader()->getTerminator());
// Find the loop boundaries.
ScalarEvolution *SE = PSE.getSE();
const SCEV *BackedgeTakenCount = PSE.getBackedgeTakenCount();
assert(!isa<SCEVCouldNotCompute>(BackedgeTakenCount) &&((!isa<SCEVCouldNotCompute>(BackedgeTakenCount) &&
 "Invalid loop count") ? static_cast<void> (0) : __assert_fail
 ("!isa<SCEVCouldNotCompute>(BackedgeTakenCount) && \"Invalid loop count\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3075, __PRETTY_FUNCTION__))
       "Invalid loop count")((!isa<SCEVCouldNotCompute>(BackedgeTakenCount) &&
 "Invalid loop count") ? static_cast<void> (0) : __assert_fail
 ("!isa<SCEVCouldNotCompute>(BackedgeTakenCount) && \"Invalid loop count\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3075, __PRETTY_FUNCTION__));

Type *IdxTy = Legal->getWidestInductionType();
assert(IdxTy && "No type for induction")((IdxTy && "No type for induction") ? static_cast<
void> (0) : __assert_fail ("IdxTy && \"No type for induction\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3078, __PRETTY_FUNCTION__));

// The exit count might have the type of i64 while the phi is i32. This can
// happen if we have an induction variable that is sign extended before the
// compare. The only way that we get a backedge taken count is that the
// induction variable was signed and as such will not overflow. In such a case
// truncation is legal.
if (SE->getTypeSizeInBits(BackedgeTakenCount->getType()) >
    IdxTy->getPrimitiveSizeInBits())
  BackedgeTakenCount = SE->getTruncateOrNoop(BackedgeTakenCount, IdxTy);
BackedgeTakenCount = SE->getNoopOrZeroExtend(BackedgeTakenCount, IdxTy);

// Get the total trip count from the count by adding 1.
const SCEV *ExitCount = SE->getAddExpr(
    BackedgeTakenCount, SE->getOne(BackedgeTakenCount->getType()));

const DataLayout &DL = L->getHeader()->getModule()->getDataLayout();

// Expand the trip count and place the new instructions in the preheader.
// Notice that the pre-header does not change, only the loop body.
SCEVExpander Exp(*SE, DL, "induction");

// Count holds the overall loop count (N).
TripCount = Exp.expandCodeFor(ExitCount, ExitCount->getType(),
                              L->getLoopPreheader()->getTerminator());

if (TripCount->getType()->isPointerTy())
  TripCount =
      CastInst::CreatePointerCast(TripCount, IdxTy, "exitcount.ptrcnt.to.int",
                                  L->getLoopPreheader()->getTerminator());

return TripCount;
3110}

3112Value *InnerLoopVectorizer::getOrCreateVectorTripCount(Loop *L) {
if (VectorTripCount)
  return VectorTripCount;

Value *TC = getOrCreateTripCount(L);
IRBuilder<> Builder(L->getLoopPreheader()->getTerminator());

Type *Ty = TC->getType();
// This is where we can make the step a runtime constant.
Value *Step = createStepForVF(Builder, ConstantInt::get(Ty, UF), VF);

// If the tail is to be folded by masking, round the number of iterations N
// up to a multiple of Step instead of rounding down. This is done by first
// adding Step-1 and then rounding down. Note that it's ok if this addition
// overflows: the vector induction variable will eventually wrap to zero given
// that it starts at zero and its Step is a power of two; the loop will then
// exit, with the last early-exit vector comparison also producing all-true.
if (Cost->foldTailByMasking()) {
  assert(isPowerOf2_32(VF.getKnownMinValue() * UF) &&((isPowerOf2_32(VF.getKnownMinValue() * UF) && "VF*UF must be a power of 2 when folding tail by masking"
) ? static_cast<void> (0) : __assert_fail ("isPowerOf2_32(VF.getKnownMinValue() * UF) && \"VF*UF must be a power of 2 when folding tail by masking\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3131, __PRETTY_FUNCTION__))
         "VF*UF must be a power of 2 when folding tail by masking")((isPowerOf2_32(VF.getKnownMinValue() * UF) && "VF*UF must be a power of 2 when folding tail by masking"
) ? static_cast<void> (0) : __assert_fail ("isPowerOf2_32(VF.getKnownMinValue() * UF) && \"VF*UF must be a power of 2 when folding tail by masking\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3131, __PRETTY_FUNCTION__));
  assert(!VF.isScalable() &&((!VF.isScalable() && "Tail folding not yet supported for scalable vectors"
) ? static_cast<void> (0) : __assert_fail ("!VF.isScalable() && \"Tail folding not yet supported for scalable vectors\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3133, __PRETTY_FUNCTION__))
         "Tail folding not yet supported for scalable vectors")((!VF.isScalable() && "Tail folding not yet supported for scalable vectors"
) ? static_cast<void> (0) : __assert_fail ("!VF.isScalable() && \"Tail folding not yet supported for scalable vectors\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3133, __PRETTY_FUNCTION__));
  TC = Builder.CreateAdd(
      TC, ConstantInt::get(Ty, VF.getKnownMinValue() * UF - 1), "n.rnd.up");
}

// Now we need to generate the expression for the part of the loop that the
// vectorized body will execute. This is equal to N - (N % Step) if scalar
// iterations are not required for correctness, or N - Step, otherwise. Step
// is equal to the vectorization factor (number of SIMD elements) times the
// unroll factor (number of SIMD instructions).
Value *R = Builder.CreateURem(TC, Step, "n.mod.vf");

// There are two cases where we need to ensure (at least) the last iteration
// runs in the scalar remainder loop. Thus, if the step evenly divides
// the trip count, we set the remainder to be equal to the step. If the step
// does not evenly divide the trip count, no adjustment is necessary since
// there will already be scalar iterations. Note that the minimum iterations
// check ensures that N >= Step. The cases are:
// 1) If there is a non-reversed interleaved group that may speculatively
//    access memory out-of-bounds.
// 2) If any instruction may follow a conditionally taken exit. That is, if
//    the loop contains multiple exiting blocks, or a single exiting block
//    which is not the latch.
if (VF.isVector() && Cost->requiresScalarEpilogue()) {
  auto *IsZero = Builder.CreateICmpEQ(R, ConstantInt::get(R->getType(), 0));
  R = Builder.CreateSelect(IsZero, Step, R);
}

VectorTripCount = Builder.CreateSub(TC, R, "n.vec");

return VectorTripCount;
3164}

3166Value *InnerLoopVectorizer::createBitOrPointerCast(Value *V, VectorType *DstVTy,
                                                 const DataLayout &DL) {
// Verify that V is a vector type with same number of elements as DstVTy.
auto *DstFVTy = cast<FixedVectorType>(DstVTy);
unsigned VF = DstFVTy->getNumElements();
auto *SrcVecTy = cast<FixedVectorType>(V->getType());
assert((VF == SrcVecTy->getNumElements()) && "Vector dimensions do not match")(((VF == SrcVecTy->getNumElements()) && "Vector dimensions do not match"
) ? static_cast<void> (0) : __assert_fail ("(VF == SrcVecTy->getNumElements()) && \"Vector dimensions do not match\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3172, __PRETTY_FUNCTION__));
Type *SrcElemTy = SrcVecTy->getElementType();
Type *DstElemTy = DstFVTy->getElementType();
assert((DL.getTypeSizeInBits(SrcElemTy) == DL.getTypeSizeInBits(DstElemTy)) &&(((DL.getTypeSizeInBits(SrcElemTy) == DL.getTypeSizeInBits(DstElemTy
)) && "Vector elements must have same size") ? static_cast
<void> (0) : __assert_fail ("(DL.getTypeSizeInBits(SrcElemTy) == DL.getTypeSizeInBits(DstElemTy)) && \"Vector elements must have same size\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3176, __PRETTY_FUNCTION__))
       "Vector elements must have same size")(((DL.getTypeSizeInBits(SrcElemTy) == DL.getTypeSizeInBits(DstElemTy
)) && "Vector elements must have same size") ? static_cast
<void> (0) : __assert_fail ("(DL.getTypeSizeInBits(SrcElemTy) == DL.getTypeSizeInBits(DstElemTy)) && \"Vector elements must have same size\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3176, __PRETTY_FUNCTION__));

// Do a direct cast if element types are castable.
if (CastInst::isBitOrNoopPointerCastable(SrcElemTy, DstElemTy, DL)) {
  return Builder.CreateBitOrPointerCast(V, DstFVTy);
}
// V cannot be directly casted to desired vector type.
// May happen when V is a floating point vector but DstVTy is a vector of
// pointers or vice-versa. Handle this using a two-step bitcast using an
// intermediate Integer type for the bitcast i.e. Ptr <-> Int <-> Float.
assert((DstElemTy->isPointerTy() != SrcElemTy->isPointerTy()) &&(((DstElemTy->isPointerTy() != SrcElemTy->isPointerTy()
) && "Only one type should be a pointer type") ? static_cast
<void> (0) : __assert_fail ("(DstElemTy->isPointerTy() != SrcElemTy->isPointerTy()) && \"Only one type should be a pointer type\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3187, __PRETTY_FUNCTION__))
       "Only one type should be a pointer type")(((DstElemTy->isPointerTy() != SrcElemTy->isPointerTy()
) && "Only one type should be a pointer type") ? static_cast
<void> (0) : __assert_fail ("(DstElemTy->isPointerTy() != SrcElemTy->isPointerTy()) && \"Only one type should be a pointer type\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3187, __PRETTY_FUNCTION__));
assert((DstElemTy->isFloatingPointTy() != SrcElemTy->isFloatingPointTy()) &&(((DstElemTy->isFloatingPointTy() != SrcElemTy->isFloatingPointTy
()) && "Only one type should be a floating point type"
) ? static_cast<void> (0) : __assert_fail ("(DstElemTy->isFloatingPointTy() != SrcElemTy->isFloatingPointTy()) && \"Only one type should be a floating point type\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3189, __PRETTY_FUNCTION__))
       "Only one type should be a floating point type")(((DstElemTy->isFloatingPointTy() != SrcElemTy->isFloatingPointTy
()) && "Only one type should be a floating point type"
) ? static_cast<void> (0) : __assert_fail ("(DstElemTy->isFloatingPointTy() != SrcElemTy->isFloatingPointTy()) && \"Only one type should be a floating point type\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3189, __PRETTY_FUNCTION__));
Type *IntTy =
    IntegerType::getIntNTy(V->getContext(), DL.getTypeSizeInBits(SrcElemTy));
auto *VecIntTy = FixedVectorType::get(IntTy, VF);
Value *CastVal = Builder.CreateBitOrPointerCast(V, VecIntTy);
return Builder.CreateBitOrPointerCast(CastVal, DstFVTy);
3195}

3197void InnerLoopVectorizer::emitMinimumIterationCountCheck(Loop *L,
                                                       BasicBlock *Bypass) {
Value *Count = getOrCreateTripCount(L);
// Reuse existing vector loop preheader for TC checks.
// Note that new preheader block is generated for vector loop.
BasicBlock *const TCCheckBlock = LoopVectorPreHeader;
IRBuilder<> Builder(TCCheckBlock->getTerminator());

// Generate code to check if the loop's trip count is less than VF * UF, or
// equal to it in case a scalar epilogue is required; this implies that the
// vector trip count is zero. This check also covers the case where adding one
// to the backedge-taken count overflowed leading to an incorrect trip count
// of zero. In this case we will also jump to the scalar loop.
auto P = Cost->requiresScalarEpilogue() ? ICmpInst::ICMP_ULE
                                        : ICmpInst::ICMP_ULT;

// If tail is to be folded, vector loop takes care of all iterations.
Value *CheckMinIters = Builder.getFalse();
if (!Cost->foldTailByMasking()) {
  Value *Step =
      createStepForVF(Builder, ConstantInt::get(Count->getType(), UF), VF);
  CheckMinIters = Builder.CreateICmp(P, Count, Step, "min.iters.check");
}
// Create new preheader for vector loop.
LoopVectorPreHeader =
    SplitBlock(TCCheckBlock, TCCheckBlock->getTerminator(), DT, LI, nullptr,
               "vector.ph");

assert(DT->properlyDominates(DT->getNode(TCCheckBlock),((DT->properlyDominates(DT->getNode(TCCheckBlock), DT->
getNode(Bypass)->getIDom()) && "TC check is expected to dominate Bypass"
) ? static_cast<void> (0) : __assert_fail ("DT->properlyDominates(DT->getNode(TCCheckBlock), DT->getNode(Bypass)->getIDom()) && \"TC check is expected to dominate Bypass\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3227, __PRETTY_FUNCTION__))
                             DT->getNode(Bypass)->getIDom()) &&((DT->properlyDominates(DT->getNode(TCCheckBlock), DT->
getNode(Bypass)->getIDom()) && "TC check is expected to dominate Bypass"
) ? static_cast<void> (0) : __assert_fail ("DT->properlyDominates(DT->getNode(TCCheckBlock), DT->getNode(Bypass)->getIDom()) && \"TC check is expected to dominate Bypass\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3227, __PRETTY_FUNCTION__))
       "TC check is expected to dominate Bypass")((DT->properlyDominates(DT->getNode(TCCheckBlock), DT->
getNode(Bypass)->getIDom()) && "TC check is expected to dominate Bypass"
) ? static_cast<void> (0) : __assert_fail ("DT->properlyDominates(DT->getNode(TCCheckBlock), DT->getNode(Bypass)->getIDom()) && \"TC check is expected to dominate Bypass\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3227, __PRETTY_FUNCTION__));

// Update dominator for Bypass & LoopExit.
DT->changeImmediateDominator(Bypass, TCCheckBlock);
DT->changeImmediateDominator(LoopExitBlock, TCCheckBlock);

ReplaceInstWithInst(
    TCCheckBlock->getTerminator(),
    BranchInst::Create(Bypass, LoopVectorPreHeader, CheckMinIters));
LoopBypassBlocks.push_back(TCCheckBlock);
3237}

3239BasicBlock *InnerLoopVectorizer::emitSCEVChecks(Loop *L, BasicBlock *Bypass) {

BasicBlock *const SCEVCheckBlock =
    RTChecks.emitSCEVChecks(L, Bypass, LoopVectorPreHeader, LoopExitBlock);
if (!SCEVCheckBlock)
  return nullptr;

assert(!(SCEVCheckBlock->getParent()->hasOptSize() ||((!(SCEVCheckBlock->getParent()->hasOptSize() || (OptForSizeBasedOnProfile
 && Cost->Hints->getForce() != LoopVectorizeHints
::FK_Enabled)) && "Cannot SCEV check stride or overflow when optimizing for size"
) ? static_cast<void> (0) : __assert_fail ("!(SCEVCheckBlock->getParent()->hasOptSize() || (OptForSizeBasedOnProfile && Cost->Hints->getForce() != LoopVectorizeHints::FK_Enabled)) && \"Cannot SCEV check stride or overflow when optimizing for size\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3249, __PRETTY_FUNCTION__))
         (OptForSizeBasedOnProfile &&((!(SCEVCheckBlock->getParent()->hasOptSize() || (OptForSizeBasedOnProfile
 && Cost->Hints->getForce() != LoopVectorizeHints
::FK_Enabled)) && "Cannot SCEV check stride or overflow when optimizing for size"
) ? static_cast<void> (0) : __assert_fail ("!(SCEVCheckBlock->getParent()->hasOptSize() || (OptForSizeBasedOnProfile && Cost->Hints->getForce() != LoopVectorizeHints::FK_Enabled)) && \"Cannot SCEV check stride or overflow when optimizing for size\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3249, __PRETTY_FUNCTION__))
          Cost->Hints->getForce() != LoopVectorizeHints::FK_Enabled)) &&((!(SCEVCheckBlock->getParent()->hasOptSize() || (OptForSizeBasedOnProfile
 && Cost->Hints->getForce() != LoopVectorizeHints
::FK_Enabled)) && "Cannot SCEV check stride or overflow when optimizing for size"
) ? static_cast<void> (0) : __assert_fail ("!(SCEVCheckBlock->getParent()->hasOptSize() || (OptForSizeBasedOnProfile && Cost->Hints->getForce() != LoopVectorizeHints::FK_Enabled)) && \"Cannot SCEV check stride or overflow when optimizing for size\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3249, __PRETTY_FUNCTION__))
       "Cannot SCEV check stride or overflow when optimizing for size")((!(SCEVCheckBlock->getParent()->hasOptSize() || (OptForSizeBasedOnProfile
 && Cost->Hints->getForce() != LoopVectorizeHints
::FK_Enabled)) && "Cannot SCEV check stride or overflow when optimizing for size"
) ? static_cast<void> (0) : __assert_fail ("!(SCEVCheckBlock->getParent()->hasOptSize() || (OptForSizeBasedOnProfile && Cost->Hints->getForce() != LoopVectorizeHints::FK_Enabled)) && \"Cannot SCEV check stride or overflow when optimizing for size\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3249, __PRETTY_FUNCTION__));


// Update dominator only if this is first RT check.
if (LoopBypassBlocks.empty()) {
  DT->changeImmediateDominator(Bypass, SCEVCheckBlock);
  DT->changeImmediateDominator(LoopExitBlock, SCEVCheckBlock);
}

LoopBypassBlocks.push_back(SCEVCheckBlock);
AddedSafetyChecks = true;
return SCEVCheckBlock;
3261}

3263BasicBlock *InnerLoopVectorizer::emitMemRuntimeChecks(Loop *L,
                                                    BasicBlock *Bypass) {
// VPlan-native path does not do any analysis for runtime checks currently.
if (EnableVPlanNativePath)
  return nullptr;

BasicBlock *const MemCheckBlock =
    RTChecks.emitMemRuntimeChecks(L, Bypass, LoopVectorPreHeader);

// Check if we generated code that checks in runtime if arrays overlap. We put
// the checks into a separate block to make the more common case of few
// elements faster.
if (!MemCheckBlock)
  return nullptr;

if (MemCheckBlock->getParent()->hasOptSize() || OptForSizeBasedOnProfile) {
  assert(Cost->Hints->getForce() == LoopVectorizeHints::FK_Enabled &&((Cost->Hints->getForce() == LoopVectorizeHints::FK_Enabled
 && "Cannot emit memory checks when optimizing for size, unless forced "
 "to vectorize.") ? static_cast<void> (0) : __assert_fail
 ("Cost->Hints->getForce() == LoopVectorizeHints::FK_Enabled && \"Cannot emit memory checks when optimizing for size, unless forced \" \"to vectorize.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3281, __PRETTY_FUNCTION__))
         "Cannot emit memory checks when optimizing for size, unless forced "((Cost->Hints->getForce() == LoopVectorizeHints::FK_Enabled
 && "Cannot emit memory checks when optimizing for size, unless forced "
 "to vectorize.") ? static_cast<void> (0) : __assert_fail
 ("Cost->Hints->getForce() == LoopVectorizeHints::FK_Enabled && \"Cannot emit memory checks when optimizing for size, unless forced \" \"to vectorize.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3281, __PRETTY_FUNCTION__))
         "to vectorize.")((Cost->Hints->getForce() == LoopVectorizeHints::FK_Enabled
 && "Cannot emit memory checks when optimizing for size, unless forced "
 "to vectorize.") ? static_cast<void> (0) : __assert_fail
 ("Cost->Hints->getForce() == LoopVectorizeHints::FK_Enabled && \"Cannot emit memory checks when optimizing for size, unless forced \" \"to vectorize.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3281, __PRETTY_FUNCTION__));
  ORE->emit([&]() {
    return OptimizationRemarkAnalysis(DEBUG_TYPE"loop-vectorize", "VectorizationCodeSize",
                                      L->getStartLoc(), L->getHeader())
           << "Code-size may be reduced by not forcing "
              "vectorization, or by source-code modifications "
              "eliminating the need for runtime checks "
              "(e.g., adding 'restrict').";
  });
}

LoopBypassBlocks.push_back(MemCheckBlock);

AddedSafetyChecks = true;

// We currently don't use LoopVersioning for the actual loop cloning but we
// still use it to add the noalias metadata.
LVer = std::make_unique<LoopVersioning>(
    *Legal->getLAI(),
    Legal->getLAI()->getRuntimePointerChecking()->getChecks(), OrigLoop, LI,
    DT, PSE.getSE());
LVer->prepareNoAliasMetadata();
return MemCheckBlock;
3304}

3306Value *InnerLoopVectorizer::emitTransformedIndex(
  IRBuilder<> &B, Value *Index, ScalarEvolution *SE, const DataLayout &DL,
  const InductionDescriptor &ID) const {

SCEVExpander Exp(*SE, DL, "induction");
auto Step = ID.getStep();
auto StartValue = ID.getStartValue();
assert(Index->getType() == Step->getType() &&((Index->getType() == Step->getType() && "Index type does not match StepValue type"
) ? static_cast<void> (0) : __assert_fail ("Index->getType() == Step->getType() && \"Index type does not match StepValue type\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3314, __PRETTY_FUNCTION__))
       "Index type does not match StepValue type")((Index->getType() == Step->getType() && "Index type does not match StepValue type"
) ? static_cast<void> (0) : __assert_fail ("Index->getType() == Step->getType() && \"Index type does not match StepValue type\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3314, __PRETTY_FUNCTION__));

// Note: the IR at this point is broken. We cannot use SE to create any new
// SCEV and then expand it, hoping that SCEV's simplification will give us
// a more optimal code. Unfortunately, attempt of doing so on invalid IR may
// lead to various SCEV crashes. So all we can do is to use builder and rely
// on InstCombine for future simplifications. Here we handle some trivial
// cases only.
auto CreateAdd = [&B](Value *X, Value *Y) {
  assert(X->getType() == Y->getType() && "Types don't match!")((X->getType() == Y->getType() && "Types don't match!"
) ? static_cast<void> (0) : __assert_fail ("X->getType() == Y->getType() && \"Types don't match!\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3323, __PRETTY_FUNCTION__));
  if (auto *CX = dyn_cast<ConstantInt>(X))
    if (CX->isZero())
      return Y;
  if (auto *CY = dyn_cast<ConstantInt>(Y))
    if (CY->isZero())
      return X;
  return B.CreateAdd(X, Y);
};

auto CreateMul = [&B](Value *X, Value *Y) {
  assert(X->getType() == Y->getType() && "Types don't match!")((X->getType() == Y->getType() && "Types don't match!"
) ? static_cast<void> (0) : __assert_fail ("X->getType() == Y->getType() && \"Types don't match!\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3334, __PRETTY_FUNCTION__));
  if (auto *CX = dyn_cast<ConstantInt>(X))
    if (CX->isOne())
      return Y;
  if (auto *CY = dyn_cast<ConstantInt>(Y))
    if (CY->isOne())
      return X;
  return B.CreateMul(X, Y);
};

// Get a suitable insert point for SCEV expansion. For blocks in the vector
// loop, choose the end of the vector loop header (=LoopVectorBody), because
// the DomTree is not kept up-to-date for additional blocks generated in the
// vector loop. By using the header as insertion point, we guarantee that the
// expanded instructions dominate all their uses.
auto GetInsertPoint = [this, &B]() {
  BasicBlock *InsertBB = B.GetInsertPoint()->getParent();
  if (InsertBB != LoopVectorBody &&
      LI->getLoopFor(LoopVectorBody) == LI->getLoopFor(InsertBB))
    return LoopVectorBody->getTerminator();
  return &*B.GetInsertPoint();
};

switch (ID.getKind()) {
case InductionDescriptor::IK_IntInduction: {
  assert(Index->getType() == StartValue->getType() &&((Index->getType() == StartValue->getType() && "Index type does not match StartValue type"
) ? static_cast<void> (0) : __assert_fail ("Index->getType() == StartValue->getType() && \"Index type does not match StartValue type\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3360, __PRETTY_FUNCTION__))
         "Index type does not match StartValue type")((Index->getType() == StartValue->getType() && "Index type does not match StartValue type"
) ? static_cast<void> (0) : __assert_fail ("Index->getType() == StartValue->getType() && \"Index type does not match StartValue type\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3360, __PRETTY_FUNCTION__));
  if (ID.getConstIntStepValue() && ID.getConstIntStepValue()->isMinusOne())
    return B.CreateSub(StartValue, Index);
  auto *Offset = CreateMul(
      Index, Exp.expandCodeFor(Step, Index->getType(), GetInsertPoint()));
  return CreateAdd(StartValue, Offset);
}
case InductionDescriptor::IK_PtrInduction: {
  assert(isa<SCEVConstant>(Step) &&((isa<SCEVConstant>(Step) && "Expected constant step for pointer induction"
) ? static_cast<void> (0) : __assert_fail ("isa<SCEVConstant>(Step) && \"Expected constant step for pointer induction\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3369, __PRETTY_FUNCTION__))
         "Expected constant step for pointer induction")((isa<SCEVConstant>(Step) && "Expected constant step for pointer induction"
) ? static_cast<void> (0) : __assert_fail ("isa<SCEVConstant>(Step) && \"Expected constant step for pointer induction\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3369, __PRETTY_FUNCTION__));
  return B.CreateGEP(
      StartValue->getType()->getPointerElementType(), StartValue,
      CreateMul(Index,
                Exp.expandCodeFor(Step, Index->getType(), GetInsertPoint())));
}
case InductionDescriptor::IK_FpInduction: {
  assert(Step->getType()->isFloatingPointTy() && "Expected FP Step value")((Step->getType()->isFloatingPointTy() && "Expected FP Step value"
) ? static_cast<void> (0) : __assert_fail ("Step->getType()->isFloatingPointTy() && \"Expected FP Step value\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3376, __PRETTY_FUNCTION__));
  auto InductionBinOp = ID.getInductionBinOp();
  assert(InductionBinOp &&((InductionBinOp && (InductionBinOp->getOpcode() ==
 Instruction::FAdd || InductionBinOp->getOpcode() == Instruction
::FSub) && "Original bin op should be defined for FP induction"
) ? static_cast<void> (0) : __assert_fail ("InductionBinOp && (InductionBinOp->getOpcode() == Instruction::FAdd || InductionBinOp->getOpcode() == Instruction::FSub) && \"Original bin op should be defined for FP induction\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3381, __PRETTY_FUNCTION__))
         (InductionBinOp->getOpcode() == Instruction::FAdd ||((InductionBinOp && (InductionBinOp->getOpcode() ==
 Instruction::FAdd || InductionBinOp->getOpcode() == Instruction
::FSub) && "Original bin op should be defined for FP induction"
) ? static_cast<void> (0) : __assert_fail ("InductionBinOp && (InductionBinOp->getOpcode() == Instruction::FAdd || InductionBinOp->getOpcode() == Instruction::FSub) && \"Original bin op should be defined for FP induction\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3381, __PRETTY_FUNCTION__))
          InductionBinOp->getOpcode() == Instruction::FSub) &&((InductionBinOp && (InductionBinOp->getOpcode() ==
 Instruction::FAdd || InductionBinOp->getOpcode() == Instruction
::FSub) && "Original bin op should be defined for FP induction"
) ? static_cast<void> (0) : __assert_fail ("InductionBinOp && (InductionBinOp->getOpcode() == Instruction::FAdd || InductionBinOp->getOpcode() == Instruction::FSub) && \"Original bin op should be defined for FP induction\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3381, __PRETTY_FUNCTION__))
         "Original bin op should be defined for FP induction")((InductionBinOp && (InductionBinOp->getOpcode() ==
 Instruction::FAdd || InductionBinOp->getOpcode() == Instruction
::FSub) && "Original bin op should be defined for FP induction"
) ? static_cast<void> (0) : __assert_fail ("InductionBinOp && (InductionBinOp->getOpcode() == Instruction::FAdd || InductionBinOp->getOpcode() == Instruction::FSub) && \"Original bin op should be defined for FP induction\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3381, __PRETTY_FUNCTION__));

  Value *StepValue = cast<SCEVUnknown>(Step)->getValue();
  Value *MulExp = B.CreateFMul(StepValue, Index);
  return B.CreateBinOp(InductionBinOp->getOpcode(), StartValue, MulExp,
                       "induction");
}
case InductionDescriptor::IK_NoInduction:
  return nullptr;
}
llvm_unreachable("invalid enum")::llvm::llvm_unreachable_internal("invalid enum", "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3391);
3392}

3394Loop *InnerLoopVectorizer::createVectorLoopSkeleton(StringRef Prefix) {
LoopScalarBody = OrigLoop->getHeader();
LoopVectorPreHeader = OrigLoop->getLoopPreheader();
LoopExitBlock = OrigLoop->getUniqueExitBlock();
assert(LoopExitBlock && "Must have an exit block")((LoopExitBlock && "Must have an exit block") ? static_cast
<void> (0) : __assert_fail ("LoopExitBlock && \"Must have an exit block\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3398, __PRETTY_FUNCTION__));
assert(LoopVectorPreHeader && "Invalid loop structure")((LoopVectorPreHeader && "Invalid loop structure") ? static_cast
<void> (0) : __assert_fail ("LoopVectorPreHeader && \"Invalid loop structure\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3399, __PRETTY_FUNCTION__));

LoopMiddleBlock =
    SplitBlock(LoopVectorPreHeader, LoopVectorPreHeader->getTerminator(), DT,
               LI, nullptr, Twine(Prefix) + "middle.block");
LoopScalarPreHeader =
    SplitBlock(LoopMiddleBlock, LoopMiddleBlock->getTerminator(), DT, LI,
               nullptr, Twine(Prefix) + "scalar.ph");

// Set up branch from middle block to the exit and scalar preheader blocks.
// completeLoopSkeleton will update the condition to use an iteration check,
// if required to decide whether to execute the remainder.
BranchInst *BrInst =
    BranchInst::Create(LoopExitBlock, LoopScalarPreHeader, Builder.getTrue());
auto *ScalarLatchTerm = OrigLoop->getLoopLatch()->getTerminator();
BrInst->setDebugLoc(ScalarLatchTerm->getDebugLoc());
ReplaceInstWithInst(LoopMiddleBlock->getTerminator(), BrInst);

// We intentionally don't let SplitBlock to update LoopInfo since
// LoopVectorBody should belong to another loop than LoopVectorPreHeader.
// LoopVectorBody is explicitly added to the correct place few lines later.
LoopVectorBody =
    SplitBlock(LoopVectorPreHeader, LoopVectorPreHeader->getTerminator(), DT,
               nullptr, nullptr, Twine(Prefix) + "vector.body");

// Update dominator for loop exit.
DT->changeImmediateDominator(LoopExitBlock, LoopMiddleBlock);

// Create and register the new vector loop.
Loop *Lp = LI->AllocateLoop();
Loop *ParentLoop = OrigLoop->getParentLoop();

// Insert the new loop into the loop nest and register the new basic blocks
// before calling any utilities such as SCEV that require valid LoopInfo.
if (ParentLoop) {
  ParentLoop->addChildLoop(Lp);
} else {
  LI->addTopLevelLoop(Lp);
}
Lp->addBasicBlockToLoop(LoopVectorBody, *LI);
return Lp;
3440}

3442void InnerLoopVectorizer::createInductionResumeValues(
  Loop *L, Value *VectorTripCount,
  std::pair<BasicBlock *, Value *> AdditionalBypass) {
assert(VectorTripCount && L && "Expected valid arguments")((VectorTripCount && L && "Expected valid arguments"
) ? static_cast<void> (0) : __assert_fail ("VectorTripCount && L && \"Expected valid arguments\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3445, __PRETTY_FUNCTION__));
assert(((AdditionalBypass.first && AdditionalBypass.second) ||((((AdditionalBypass.first && AdditionalBypass.second
) || (!AdditionalBypass.first && !AdditionalBypass.second
)) && "Inconsistent information about additional bypass."
) ? static_cast<void> (0) : __assert_fail ("((AdditionalBypass.first && AdditionalBypass.second) || (!AdditionalBypass.first && !AdditionalBypass.second)) && \"Inconsistent information about additional bypass.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3448, __PRETTY_FUNCTION__))
        (!AdditionalBypass.first && !AdditionalBypass.second)) &&((((AdditionalBypass.first && AdditionalBypass.second
) || (!AdditionalBypass.first && !AdditionalBypass.second
)) && "Inconsistent information about additional bypass."
) ? static_cast<void> (0) : __assert_fail ("((AdditionalBypass.first && AdditionalBypass.second) || (!AdditionalBypass.first && !AdditionalBypass.second)) && \"Inconsistent information about additional bypass.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3448, __PRETTY_FUNCTION__))
       "Inconsistent information about additional bypass.")((((AdditionalBypass.first && AdditionalBypass.second
) || (!AdditionalBypass.first && !AdditionalBypass.second
)) && "Inconsistent information about additional bypass."
) ? static_cast<void> (0) : __assert_fail ("((AdditionalBypass.first && AdditionalBypass.second) || (!AdditionalBypass.first && !AdditionalBypass.second)) && \"Inconsistent information about additional bypass.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3448, __PRETTY_FUNCTION__));
// We are going to resume the execution of the scalar loop.
// Go over all of the induction variables that we found and fix the
// PHIs that are left in the scalar version of the loop.
// The starting values of PHI nodes depend on the counter of the last
// iteration in the vectorized loop.
// If we come from a bypass edge then we need to start from the original
// start value.
for (auto &InductionEntry : Legal->getInductionVars()) {
  PHINode *OrigPhi = InductionEntry.first;
  InductionDescriptor II = InductionEntry.second;

  // Create phi nodes to merge from the  backedge-taken check block.
  PHINode *BCResumeVal =
      PHINode::Create(OrigPhi->getType(), 3, "bc.resume.val",
                      LoopScalarPreHeader->getTerminator());
  // Copy original phi DL over to the new one.
  BCResumeVal->setDebugLoc(OrigPhi->getDebugLoc());
  Value *&EndValue = IVEndValues[OrigPhi];
  Value *EndValueFromAdditionalBypass = AdditionalBypass.second;
  if (OrigPhi == OldInduction) {
    // We know what the end value is.
    EndValue = VectorTripCount;
  } else {
    IRBuilder<> B(L->getLoopPreheader()->getTerminator());

    // Fast-math-flags propagate from the original induction instruction.
    if (II.getInductionBinOp() && isa<FPMathOperator>(II.getInductionBinOp()))
      B.setFastMathFlags(II.getInductionBinOp()->getFastMathFlags());

    Type *StepType = II.getStep()->getType();
    Instruction::CastOps CastOp =
        CastInst::getCastOpcode(VectorTripCount, true, StepType, true);
    Value *CRD = B.CreateCast(CastOp, VectorTripCount, StepType, "cast.crd");
    const DataLayout &DL = LoopScalarBody->getModule()->getDataLayout();
    EndValue = emitTransformedIndex(B, CRD, PSE.getSE(), DL, II);
    EndValue->setName("ind.end");

    // Compute the end value for the additional bypass (if applicable).
    if (AdditionalBypass.first) {
      B.SetInsertPoint(&(*AdditionalBypass.first->getFirstInsertionPt()));
      CastOp = CastInst::getCastOpcode(AdditionalBypass.second, true,
                                       StepType, true);
      CRD =
          B.CreateCast(CastOp, AdditionalBypass.second, StepType, "cast.crd");
      EndValueFromAdditionalBypass =
          emitTransformedIndex(B, CRD, PSE.getSE(), DL, II);
      EndValueFromAdditionalBypass->setName("ind.end");
    }
  }
  // The new PHI merges the original incoming value, in case of a bypass,
  // or the value at the end of the vectorized loop.
  BCResumeVal->addIncoming(EndValue, LoopMiddleBlock);

  // Fix the scalar body counter (PHI node).
  // The old induction's phi node in the scalar body needs the truncated
  // value.
  for (BasicBlock *BB : LoopBypassBlocks)
    BCResumeVal->addIncoming(II.getStartValue(), BB);

  if (AdditionalBypass.first)
    BCResumeVal->setIncomingValueForBlock(AdditionalBypass.first,
                                          EndValueFromAdditionalBypass);

  OrigPhi->setIncomingValueForBlock(LoopScalarPreHeader, BCResumeVal);
}
3514}

3516BasicBlock *InnerLoopVectorizer::completeLoopSkeleton(Loop *L,
                                                    MDNode *OrigLoopID) {
assert(L && "Expected valid loop.")((L && "Expected valid loop.") ? static_cast<void>
 (0) : __assert_fail ("L && \"Expected valid loop.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3518, __PRETTY_FUNCTION__));

// The trip counts should be cached by now.
Value *Count = getOrCreateTripCount(L);
Value *VectorTripCount = getOrCreateVectorTripCount(L);

auto *ScalarLatchTerm = OrigLoop->getLoopLatch()->getTerminator();

// Add a check in the middle block to see if we have completed
// all of the iterations in the first vector loop.
// If (N - N%VF) == N, then we *don't* need to run the remainder.
// If tail is to be folded, we know we don't need to run the remainder.
if (!Cost->foldTailByMasking()) {
  Instruction *CmpN = CmpInst::Create(Instruction::ICmp, CmpInst::ICMP_EQ,
                                      Count, VectorTripCount, "cmp.n",
                                      LoopMiddleBlock->getTerminator());

  // Here we use the same DebugLoc as the scalar loop latch terminator instead
  // of the corresponding compare because they may have ended up with
  // different line numbers and we want to avoid awkward line stepping while
  // debugging. Eg. if the compare has got a line number inside the loop.
  CmpN->setDebugLoc(ScalarLatchTerm->getDebugLoc());
  cast<BranchInst>(LoopMiddleBlock->getTerminator())->setCondition(CmpN);
}

// Get ready to start creating new instructions into the vectorized body.
assert(LoopVectorPreHeader == L->getLoopPreheader() &&((LoopVectorPreHeader == L->getLoopPreheader() && "Inconsistent vector loop preheader"
) ? static_cast<void> (0) : __assert_fail ("LoopVectorPreHeader == L->getLoopPreheader() && \"Inconsistent vector loop preheader\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3545, __PRETTY_FUNCTION__))
       "Inconsistent vector loop preheader")((LoopVectorPreHeader == L->getLoopPreheader() && "Inconsistent vector loop preheader"
) ? static_cast<void> (0) : __assert_fail ("LoopVectorPreHeader == L->getLoopPreheader() && \"Inconsistent vector loop preheader\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3545, __PRETTY_FUNCTION__));
Builder.SetInsertPoint(&*LoopVectorBody->getFirstInsertionPt());

Optional<MDNode *> VectorizedLoopID =
    makeFollowupLoopID(OrigLoopID, {LLVMLoopVectorizeFollowupAll,
                                    LLVMLoopVectorizeFollowupVectorized});
if (VectorizedLoopID.hasValue()) {
  L->setLoopID(VectorizedLoopID.getValue());

  // Do not setAlreadyVectorized if loop attributes have been defined
  // explicitly.
  return LoopVectorPreHeader;
}

// Keep all loop hints from the original loop on the vector loop (we'll
// replace the vectorizer-specific hints below).
if (MDNode *LID = OrigLoop->getLoopID())
  L->setLoopID(LID);

LoopVectorizeHints Hints(L, true, *ORE);
Hints.setAlreadyVectorized();

3567#ifdef EXPENSIVE_CHECKS
assert(DT->verify(DominatorTree::VerificationLevel::Fast))((DT->verify(DominatorTree::VerificationLevel::Fast)) ? static_cast
<void> (0) : __assert_fail ("DT->verify(DominatorTree::VerificationLevel::Fast)"
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3568, __PRETTY_FUNCTION__));
LI->verify(*DT);
3570#endif

return LoopVectorPreHeader;
3573}

3575BasicBlock *InnerLoopVectorizer::createVectorizedLoopSkeleton() {
/*
 In this function we generate a new loop. The new loop will contain
 the vectorized instructions while the old loop will continue to run the
 scalar remainder.

     [ ] <-- loop iteration number check.
  /   |
 /    v
|    [ ] <-- vector loop bypass (may consist of multiple blocks).
|  /  |
| /   v
||   [ ]     <-- vector pre header.
|/    |
|     v
|    [  ] \
|    [  ]_|   <-- vector loop.
|     |
|     v
|   -[ ]   <--- middle-block.
|  /  |
| /   v
-|- >[ ]     <--- new preheader.
 |    |
 |    v
 |   [ ] \
 |   [ ]_|   <-- old scalar loop to handle remainder.
  \   |
   \  v
    >[ ]     <-- exit block.
 ...
 */

// Get the metadata of the original loop before it gets modified.
MDNode *OrigLoopID = OrigLoop->getLoopID();

// Create an empty vector loop, and prepare basic blocks for the runtime
// checks.
Loop *Lp = createVectorLoopSkeleton("");

// Now, compare the new count to zero. If it is zero skip the vector loop and
// jump to the scalar loop. This check also covers the case where the
// backedge-taken count is uint##_max: adding one to it will overflow leading
// to an incorrect trip count of zero. In this (rare) case we will also jump
// to the scalar loop.
emitMinimumIterationCountCheck(Lp, LoopScalarPreHeader);

// Generate the code to check any assumptions that we've made for SCEV
// expressions.
emitSCEVChecks(Lp, LoopScalarPreHeader);

// Generate the code that checks in runtime if arrays overlap. We put the
// checks into a separate block to make the more common case of few elements
// faster.
emitMemRuntimeChecks(Lp, LoopScalarPreHeader);

// Some loops have a single integer induction variable, while other loops
// don't. One example is c++ iterators that often have multiple pointer
// induction variables. In the code below we also support a case where we
// don't have a single induction variable.
//
// We try to obtain an induction variable from the original loop as hard
// as possible. However if we don't find one that:
//   - is an integer
//   - counts from zero, stepping by one
//   - is the size of the widest induction variable type
// then we create a new one.
OldInduction = Legal->getPrimaryInduction();
Type *IdxTy = Legal->getWidestInductionType();
Value *StartIdx = ConstantInt::get(IdxTy, 0);
// The loop step is equal to the vectorization factor (num of SIMD elements)
// times the unroll factor (num of SIMD instructions).
Builder.SetInsertPoint(&*Lp->getHeader()->getFirstInsertionPt());
Value *Step = createStepForVF(Builder, ConstantInt::get(IdxTy, UF), VF);
Value *CountRoundDown = getOrCreateVectorTripCount(Lp);
Induction =
    createInductionVariable(Lp, StartIdx, CountRoundDown, Step,
                            getDebugLocFromInstOrOperands(OldInduction));

// Emit phis for the new starting index of the scalar loop.
createInductionResumeValues(Lp, CountRoundDown);

return completeLoopSkeleton(Lp, OrigLoopID);
3658}

3660// Fix up external users of the induction variable. At this point, we are
3661// in LCSSA form, with all external PHIs that use the IV having one input value,
3662// coming from the remainder loop. We need those PHIs to also have a correct
3663// value for the IV when arriving directly from the middle block.
3664void InnerLoopVectorizer::fixupIVUsers(PHINode *OrigPhi,
                                     const InductionDescriptor &II,
                                     Value *CountRoundDown, Value *EndValue,
                                     BasicBlock *MiddleBlock) {
// There are two kinds of external IV usages - those that use the value
// computed in the last iteration (the PHI) and those that use the penultimate
// value (the value that feeds into the phi from the loop latch).
// We allow both, but they, obviously, have different values.

assert(OrigLoop->getUniqueExitBlock() && "Expected a single exit block")((OrigLoop->getUniqueExitBlock() && "Expected a single exit block"
) ? static_cast<void> (0) : __assert_fail ("OrigLoop->getUniqueExitBlock() && \"Expected a single exit block\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3673, __PRETTY_FUNCTION__));

DenseMap<Value *, Value *> MissingVals;

// An external user of the last iteration's value should see the value that
// the remainder loop uses to initialize its own IV.
Value *PostInc = OrigPhi->getIncomingValueForBlock(OrigLoop->getLoopLatch());
for (User *U : PostInc->users()) {
  Instruction *UI = cast<Instruction>(U);
  if (!OrigLoop->contains(UI)) {
    assert(isa<PHINode>(UI) && "Expected LCSSA form")((isa<PHINode>(UI) && "Expected LCSSA form") ? static_cast
<void> (0) : __assert_fail ("isa<PHINode>(UI) && \"Expected LCSSA form\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3683, __PRETTY_FUNCTION__));
    MissingVals[UI] = EndValue;
  }
}

// An external user of the penultimate value need to see EndValue - Step.
// The simplest way to get this is to recompute it from the constituent SCEVs,
// that is Start + (Step * (CRD - 1)).
for (User *U : OrigPhi->users()) {
  auto *UI = cast<Instruction>(U);
  if (!OrigLoop->contains(UI)) {
    const DataLayout &DL =
        OrigLoop->getHeader()->getModule()->getDataLayout();
    assert(isa<PHINode>(UI) && "Expected LCSSA form")((isa<PHINode>(UI) && "Expected LCSSA form") ? static_cast
<void> (0) : __assert_fail ("isa<PHINode>(UI) && \"Expected LCSSA form\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3696, __PRETTY_FUNCTION__));

    IRBuilder<> B(MiddleBlock->getTerminator());

    // Fast-math-flags propagate from the original induction instruction.
    if (II.getInductionBinOp() && isa<FPMathOperator>(II.getInductionBinOp()))
      B.setFastMathFlags(II.getInductionBinOp()->getFastMathFlags());

    Value *CountMinusOne = B.CreateSub(
        CountRoundDown, ConstantInt::get(CountRoundDown->getType(), 1));
    Value *CMO =
        !II.getStep()->getType()->isIntegerTy()
            ? B.CreateCast(Instruction::SIToFP, CountMinusOne,
                           II.getStep()->getType())
            : B.CreateSExtOrTrunc(CountMinusOne, II.getStep()->getType());
    CMO->setName("cast.cmo");
    Value *Escape = emitTransformedIndex(B, CMO, PSE.getSE(), DL, II);
    Escape->setName("ind.escape");
    MissingVals[UI] = Escape;
  }
}

for (auto &I : MissingVals) {
  PHINode *PHI = cast<PHINode>(I.first);
  // One corner case we have to handle is two IVs "chasing" each-other,
  // that is %IV2 = phi [...], [ %IV1, %latch ]
  // In this case, if IV1 has an external use, we need to avoid adding both
  // "last value of IV1" and "penultimate value of IV2". So, verify that we
  // don't already have an incoming value for the middle block.
  if (PHI->getBasicBlockIndex(MiddleBlock) == -1)
    PHI->addIncoming(I.second, MiddleBlock);
}
3728}

3730namespace {

3732struct CSEDenseMapInfo {
static bool canHandle(const Instruction *I) {
  return isa<InsertElementInst>(I) || isa<ExtractElementInst>(I) ||
         isa<ShuffleVectorInst>(I) || isa<GetElementPtrInst>(I);
}

static inline Instruction *getEmptyKey() {
  return DenseMapInfo<Instruction *>::getEmptyKey();
}

static inline Instruction *getTombstoneKey() {
  return DenseMapInfo<Instruction *>::getTombstoneKey();
}

static unsigned getHashValue(const Instruction *I) {
  assert(canHandle(I) && "Unknown instruction!")((canHandle(I) && "Unknown instruction!") ? static_cast
<void> (0) : __assert_fail ("canHandle(I) && \"Unknown instruction!\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3747, __PRETTY_FUNCTION__));
  return hash_combine(I->getOpcode(), hash_combine_range(I->value_op_begin(),
                                                         I->value_op_end()));
}

static bool isEqual(const Instruction *LHS, const Instruction *RHS) {
  if (LHS == getEmptyKey() || RHS == getEmptyKey() ||
      LHS == getTombstoneKey() || RHS == getTombstoneKey())
    return LHS == RHS;
  return LHS->isIdenticalTo(RHS);
}
3758};

3760} // end anonymous namespace

3762///Perform cse of induction variable instructions.
3763static void cse(BasicBlock *BB) {
// Perform simple cse.
SmallDenseMap<Instruction *, Instruction *, 4, CSEDenseMapInfo> CSEMap;
for (BasicBlock::iterator I = BB->begin(), E = BB->end(); I != E;) {
  Instruction *In = &*I++;

  if (!CSEDenseMapInfo::canHandle(In))
    continue;

  // Check if we can replace this instruction with any of the
  // visited instructions.
  if (Instruction *V = CSEMap.lookup(In)) {
    In->replaceAllUsesWith(V);
    In->eraseFromParent();
    continue;
  }

  CSEMap[In] = In;
}
3782}

3784InstructionCost
3785LoopVectorizationCostModel::getVectorCallCost(CallInst *CI, ElementCount VF,
                                            bool &NeedToScalarize) const {
Function *F = CI->getCalledFunction();
Type *ScalarRetTy = CI->getType();
SmallVector<Type *, 4> Tys, ScalarTys;
for (auto &ArgOp : CI->arg_operands())
  ScalarTys.push_back(ArgOp->getType());

// Estimate cost of scalarized vector call. The source operands are assumed
// to be vectors, so we need to extract individual elements from there,
// execute VF scalar calls, and then gather the result into the vector return
// value.
InstructionCost ScalarCallCost =
    TTI.getCallInstrCost(F, ScalarRetTy, ScalarTys, TTI::TCK_RecipThroughput);
if (VF.isScalar())
  return ScalarCallCost;

// Compute corresponding vector type for return value and arguments.
Type *RetTy = ToVectorTy(ScalarRetTy, VF);
for (Type *ScalarTy : ScalarTys)
  Tys.push_back(ToVectorTy(ScalarTy, VF));

// Compute costs of unpacking argument values for the scalar calls and
// packing the return values to a vector.
InstructionCost ScalarizationCost = getScalarizationOverhead(CI, VF);

InstructionCost Cost =
    ScalarCallCost * VF.getKnownMinValue() + ScalarizationCost;

// If we can't emit a vector call for this function, then the currently found
// cost is the cost we need to return.
NeedToScalarize = true;
VFShape Shape = VFShape::get(*CI, VF, false /*HasGlobalPred*/);
Function *VecFunc = VFDatabase(*CI).getVectorizedFunction(Shape);

if (!TLI || CI->isNoBuiltin() || !VecFunc)
  return Cost;

// If the corresponding vector cost is cheaper, return its cost.
InstructionCost VectorCallCost =
    TTI.getCallInstrCost(nullptr, RetTy, Tys, TTI::TCK_RecipThroughput);
if (VectorCallCost < Cost) {
  NeedToScalarize = false;
  Cost = VectorCallCost;
}
return Cost;
3831}

3833static Type *MaybeVectorizeType(Type *Elt, ElementCount VF) {
if (VF.isScalar() || (!Elt->isIntOrPtrTy() && !Elt->isFloatingPointTy()))
  return Elt;
return VectorType::get(Elt, VF);
3837}

3839InstructionCost
3840LoopVectorizationCostModel::getVectorIntrinsicCost(CallInst *CI,
                                                 ElementCount VF) const {
Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);
assert(ID && "Expected intrinsic call!")((ID && "Expected intrinsic call!") ? static_cast<
void> (0) : __assert_fail ("ID && \"Expected intrinsic call!\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3843, __PRETTY_FUNCTION__));
Type *RetTy = MaybeVectorizeType(CI->getType(), VF);
FastMathFlags FMF;
if (auto *FPMO = dyn_cast<FPMathOperator>(CI))
  FMF = FPMO->getFastMathFlags();

SmallVector<const Value *> Arguments(CI->arg_begin(), CI->arg_end());
FunctionType *FTy = CI->getCalledFunction()->getFunctionType();
SmallVector<Type *> ParamTys;
std::transform(FTy->param_begin(), FTy->param_end(),
               std::back_inserter(ParamTys),
               [&](Type *Ty) { return MaybeVectorizeType(Ty, VF); });

IntrinsicCostAttributes CostAttrs(ID, RetTy, Arguments, ParamTys, FMF,
                                  dyn_cast<IntrinsicInst>(CI));
return TTI.getIntrinsicInstrCost(CostAttrs,
                                 TargetTransformInfo::TCK_RecipThroughput);
3860}

3862static Type *smallestIntegerVectorType(Type *T1, Type *T2) {
auto *I1 = cast<IntegerType>(cast<VectorType>(T1)->getElementType());
auto *I2 = cast<IntegerType>(cast<VectorType>(T2)->getElementType());
return I1->getBitWidth() < I2->getBitWidth() ? T1 : T2;
3866}

3868static Type *largestIntegerVectorType(Type *T1, Type *T2) {
auto *I1 = cast<IntegerType>(cast<VectorType>(T1)->getElementType());
auto *I2 = cast<IntegerType>(cast<VectorType>(T2)->getElementType());
return I1->getBitWidth() > I2->getBitWidth() ? T1 : T2;
3872}

3874void InnerLoopVectorizer::truncateToMinimalBitwidths(VPTransformState &State) {
// For every instruction `I` in MinBWs, truncate the operands, create a
// truncated version of `I` and reextend its result. InstCombine runs
// later and will remove any ext/trunc pairs.
SmallPtrSet<Value *, 4> Erased;
for (const auto &KV : Cost->getMinimalBitwidths()) {
  // If the value wasn't vectorized, we must maintain the original scalar
  // type. The absence of the value from State indicates that it
  // wasn't vectorized.
  VPValue *Def = State.Plan->getVPValue(KV.first);
  if (!State.hasAnyVectorValue(Def))
    continue;
  for (unsigned Part = 0; Part < UF; ++Part) {
    Value *I = State.get(Def, Part);
    if (Erased.count(I) || I->use_empty() || !isa<Instruction>(I))
      continue;
    Type *OriginalTy = I->getType();
    Type *ScalarTruncatedTy =
        IntegerType::get(OriginalTy->getContext(), KV.second);
    auto *TruncatedTy = FixedVectorType::get(
        ScalarTruncatedTy,
        cast<FixedVectorType>(OriginalTy)->getNumElements());
    if (TruncatedTy == OriginalTy)
      continue;

    IRBuilder<> B(cast<Instruction>(I));
    auto ShrinkOperand = [&](Value *V) -> Value * {
      if (auto *ZI = dyn_cast<ZExtInst>(V))
        if (ZI->getSrcTy() == TruncatedTy)
          return ZI->getOperand(0);
      return B.CreateZExtOrTrunc(V, TruncatedTy);
    };

    // The actual instruction modification depends on the instruction type,
    // unfortunately.
    Value *NewI = nullptr;
    if (auto *BO = dyn_cast<BinaryOperator>(I)) {
      NewI = B.CreateBinOp(BO->getOpcode(), ShrinkOperand(BO->getOperand(0)),
                           ShrinkOperand(BO->getOperand(1)));

      // Any wrapping introduced by shrinking this operation shouldn't be
      // considered undefined behavior. So, we can't unconditionally copy
      // arithmetic wrapping flags to NewI.
      cast<BinaryOperator>(NewI)->copyIRFlags(I, /*IncludeWrapFlags=*/false);
    } else if (auto *CI = dyn_cast<ICmpInst>(I)) {
      NewI =
          B.CreateICmp(CI->getPredicate(), ShrinkOperand(CI->getOperand(0)),
                       ShrinkOperand(CI->getOperand(1)));
    } else if (auto *SI = dyn_cast<SelectInst>(I)) {
      NewI = B.CreateSelect(SI->getCondition(),
                            ShrinkOperand(SI->getTrueValue()),
                            ShrinkOperand(SI->getFalseValue()));
    } else if (auto *CI = dyn_cast<CastInst>(I)) {
      switch (CI->getOpcode()) {
      default:
        llvm_unreachable("Unhandled cast!")::llvm::llvm_unreachable_internal("Unhandled cast!", "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3929);
      case Instruction::Trunc:
        NewI = ShrinkOperand(CI->getOperand(0));
        break;
      case Instruction::SExt:
        NewI = B.CreateSExtOrTrunc(
            CI->getOperand(0),
            smallestIntegerVectorType(OriginalTy, TruncatedTy));
        break;
      case Instruction::ZExt:
        NewI = B.CreateZExtOrTrunc(
            CI->getOperand(0),
            smallestIntegerVectorType(OriginalTy, TruncatedTy));
        break;
      }
    } else if (auto *SI = dyn_cast<ShuffleVectorInst>(I)) {
      auto Elements0 = cast<FixedVectorType>(SI->getOperand(0)->getType())
                           ->getNumElements();
      auto *O0 = B.CreateZExtOrTrunc(
          SI->getOperand(0),
          FixedVectorType::get(ScalarTruncatedTy, Elements0));
      auto Elements1 = cast<FixedVectorType>(SI->getOperand(1)->getType())
                           ->getNumElements();
      auto *O1 = B.CreateZExtOrTrunc(
          SI->getOperand(1),
          FixedVectorType::get(ScalarTruncatedTy, Elements1));

      NewI = B.CreateShuffleVector(O0, O1, SI->getShuffleMask());
    } else if (isa<LoadInst>(I) || isa<PHINode>(I)) {
      // Don't do anything with the operands, just extend the result.
      continue;
    } else if (auto *IE = dyn_cast<InsertElementInst>(I)) {
      auto Elements = cast<FixedVectorType>(IE->getOperand(0)->getType())
                          ->getNumElements();
      auto *O0 = B.CreateZExtOrTrunc(
          IE->getOperand(0),
          FixedVectorType::get(ScalarTruncatedTy, Elements));
      auto *O1 = B.CreateZExtOrTrunc(IE->getOperand(1), ScalarTruncatedTy);
      NewI = B.CreateInsertElement(O0, O1, IE->getOperand(2));
    } else if (auto *EE = dyn_cast<ExtractElementInst>(I)) {
      auto Elements = cast<FixedVectorType>(EE->getOperand(0)->getType())
                          ->getNumElements();
      auto *O0 = B.CreateZExtOrTrunc(
          EE->getOperand(0),
          FixedVectorType::get(ScalarTruncatedTy, Elements));
      NewI = B.CreateExtractElement(O0, EE->getOperand(2));
    } else {
      // If we don't know what to do, be conservative and don't do anything.
      continue;
    }

    // Lastly, extend the result.
    NewI->takeName(cast<Instruction>(I));
    Value *Res = B.CreateZExtOrTrunc(NewI, OriginalTy);
    I->replaceAllUsesWith(Res);
    cast<Instruction>(I)->eraseFromParent();
    Erased.insert(I);
    State.reset(Def, Res, Part);
  }
}

// We'll have created a bunch of ZExts that are now parentless. Clean up.
for (const auto &KV : Cost->getMinimalBitwidths()) {
  // If the value wasn't vectorized, we must maintain the original scalar
  // type. The absence of the value from State indicates that it
  // wasn't vectorized.
  VPValue *Def = State.Plan->getVPValue(KV.first);
  if (!State.hasAnyVectorValue(Def))
    continue;
  for (unsigned Part = 0; Part < UF; ++Part) {
    Value *I = State.get(Def, Part);
    ZExtInst *Inst = dyn_cast<ZExtInst>(I);
    if (Inst && Inst->use_empty()) {
      Value *NewI = Inst->getOperand(0);
      Inst->eraseFromParent();
      State.reset(Def, NewI, Part);
    }
  }
}
4008}

4010void InnerLoopVectorizer::fixVectorizedLoop(VPTransformState &State) {
// Insert truncates and extends for any truncated instructions as hints to
// InstCombine.
if (VF.isVector())
  truncateToMinimalBitwidths(State);

// Fix widened non-induction PHIs by setting up the PHI operands.
if (OrigPHIsToFix.size()) {
  assert(EnableVPlanNativePath &&((EnableVPlanNativePath && "Unexpected non-induction PHIs for fixup in non VPlan-native path"
) ? static_cast<void> (0) : __assert_fail ("EnableVPlanNativePath && \"Unexpected non-induction PHIs for fixup in non VPlan-native path\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4019, __PRETTY_FUNCTION__))
         "Unexpected non-induction PHIs for fixup in non VPlan-native path")((EnableVPlanNativePath && "Unexpected non-induction PHIs for fixup in non VPlan-native path"
) ? static_cast<void> (0) : __assert_fail ("EnableVPlanNativePath && \"Unexpected non-induction PHIs for fixup in non VPlan-native path\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4019, __PRETTY_FUNCTION__));
  fixNonInductionPHIs(State);
}

// At this point every instruction in the original loop is widened to a
// vector form. Now we need to fix the recurrences in the loop. These PHI
// nodes are currently empty because we did not want to introduce cycles.
// This is the second stage of vectorizing recurrences.
fixCrossIterationPHIs(State);

// Forget the original basic block.
PSE.getSE()->forgetLoop(OrigLoop);

// Fix-up external users of the induction variables.
for (auto &Entry : Legal->getInductionVars())
  fixupIVUsers(Entry.first, Entry.second,
               getOrCreateVectorTripCount(LI->getLoopFor(LoopVectorBody)),
               IVEndValues[Entry.first], LoopMiddleBlock);

fixLCSSAPHIs(State);
for (Instruction *PI : PredicatedInstructions)
  sinkScalarOperands(&*PI);

// Remove redundant induction instructions.
cse(LoopVectorBody);

// Set/update profile weights for the vector and remainder loops as original
// loop iterations are now distributed among them. Note that original loop
// represented by LoopScalarBody becomes remainder loop after vectorization.
//
// For cases like foldTailByMasking() and requiresScalarEpiloque() we may
// end up getting slightly roughened result but that should be OK since
// profile is not inherently precise anyway. Note also possible bypass of
// vector code caused by legality checks is ignored, assigning all the weight
// to the vector loop, optimistically.
//
// For scalable vectorization we can't know at compile time how many iterations
// of the loop are handled in one vector iteration, so instead assume a pessimistic
// vscale of '1'.
setProfileInfoAfterUnrolling(
    LI->getLoopFor(LoopScalarBody), LI->getLoopFor(LoopVectorBody),
    LI->getLoopFor(LoopScalarBody), VF.getKnownMinValue() * UF);
4061}

4063void InnerLoopVectorizer::fixCrossIterationPHIs(VPTransformState &State) {
// In order to support recurrences we need to be able to vectorize Phi nodes.
// Phi nodes have cycles, so we need to vectorize them in two stages. This is
// stage #2: We now need to fix the recurrences by adding incoming edges to
// the currently empty PHI nodes. At this point every instruction in the
// original loop is widened to a vector form so we can use them to construct
// the incoming edges.
for (PHINode &Phi : OrigLoop->getHeader()->phis()) {
  // Handle first-order recurrences and reductions that need to be fixed.
  if (Legal->isFirstOrderRecurrence(&Phi))
    fixFirstOrderRecurrence(&Phi, State);
  else if (Legal->isReductionVariable(&Phi))
    fixReduction(&Phi, State);
}
4077}

4079void InnerLoopVectorizer::fixFirstOrderRecurrence(PHINode *Phi,
                                                VPTransformState &State) {
// This is the second phase of vectorizing first-order recurrences. An
// overview of the transformation is described below. Suppose we have the
// following loop.
//
//   for (int i = 0; i < n; ++i)
//     b[i] = a[i] - a[i - 1];
//
// There is a first-order recurrence on "a". For this loop, the shorthand
// scalar IR looks like:
//
//   scalar.ph:
//     s_init = a[-1]
//     br scalar.body
//
//   scalar.body:
//     i = phi [0, scalar.ph], [i+1, scalar.body]
//     s1 = phi [s_init, scalar.ph], [s2, scalar.body]
//     s2 = a[i]
//     b[i] = s2 - s1
//     br cond, scalar.body, ...
//
// In this example, s1 is a recurrence because it's value depends on the
// previous iteration. In the first phase of vectorization, we created a
// temporary value for s1. We now complete the vectorization and produce the
// shorthand vector IR shown below (for VF = 4, UF = 1).
//
//   vector.ph:
//     v_init = vector(..., ..., ..., a[-1])
//     br vector.body
//
//   vector.body
//     i = phi [0, vector.ph], [i+4, vector.body]
//     v1 = phi [v_init, vector.ph], [v2, vector.body]
//     v2 = a[i, i+1, i+2, i+3];
//     v3 = vector(v1(3), v2(0, 1, 2))
//     b[i, i+1, i+2, i+3] = v2 - v3
//     br cond, vector.body, middle.block
//
//   middle.block:
//     x = v2(3)
//     br scalar.ph
//
//   scalar.ph:
//     s_init = phi [x, middle.block], [a[-1], otherwise]
//     br scalar.body
//
// After execution completes the vector loop, we extract the next value of
// the recurrence (x) to use as the initial value in the scalar loop.

// Get the original loop preheader and single loop latch.
auto *Preheader = OrigLoop->getLoopPreheader();
auto *Latch = OrigLoop->getLoopLatch();

// Get the initial and previous values of the scalar recurrence.
auto *ScalarInit = Phi->getIncomingValueForBlock(Preheader);
auto *Previous = Phi->getIncomingValueForBlock(Latch);

// Create a vector from the initial value.
auto *VectorInit = ScalarInit;
if (VF.isVector()) {
  Builder.SetInsertPoint(LoopVectorPreHeader->getTerminator());
  assert(!VF.isScalable() && "VF is assumed to be non scalable.")((!VF.isScalable() && "VF is assumed to be non scalable."
) ? static_cast<void> (0) : __assert_fail ("!VF.isScalable() && \"VF is assumed to be non scalable.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4142, __PRETTY_FUNCTION__));
  VectorInit = Builder.CreateInsertElement(
      PoisonValue::get(VectorType::get(VectorInit->getType(), VF)), VectorInit,
      Builder.getInt32(VF.getKnownMinValue() - 1), "vector.recur.init");
}

VPValue *PhiDef = State.Plan->getVPValue(Phi);
VPValue *PreviousDef = State.Plan->getVPValue(Previous);
// We constructed a temporary phi node in the first phase of vectorization.
// This phi node will eventually be deleted.
Builder.SetInsertPoint(cast<Instruction>(State.get(PhiDef, 0)));

// Create a phi node for the new recurrence. The current value will either be
// the initial value inserted into a vector or loop-varying vector value.
auto *VecPhi = Builder.CreatePHI(VectorInit->getType(), 2, "vector.recur");
VecPhi->addIncoming(VectorInit, LoopVectorPreHeader);

// Get the vectorized previous value of the last part UF - 1. It appears last
// among all unrolled iterations, due to the order of their construction.
Value *PreviousLastPart = State.get(PreviousDef, UF - 1);

// Find and set the insertion point after the previous value if it is an
// instruction.
BasicBlock::iterator InsertPt;
// Note that the previous value may have been constant-folded so it is not
// guaranteed to be an instruction in the vector loop.
// FIXME: Loop invariant values do not form recurrences. We should deal with
//        them earlier.
if (LI->getLoopFor(LoopVectorBody)->isLoopInvariant(PreviousLastPart))
  InsertPt = LoopVectorBody->getFirstInsertionPt();
else {
  Instruction *PreviousInst = cast<Instruction>(PreviousLastPart);
  if (isa<PHINode>(PreviousLastPart))
    // If the previous value is a phi node, we should insert after all the phi
    // nodes in the block containing the PHI to avoid breaking basic block
    // verification. Note that the basic block may be different to
    // LoopVectorBody, in case we predicate the loop.
    InsertPt = PreviousInst->getParent()->getFirstInsertionPt();
  else
    InsertPt = ++PreviousInst->getIterator();
}
Builder.SetInsertPoint(&*InsertPt);

// We will construct a vector for the recurrence by combining the values for
// the current and previous iterations. This is the required shuffle mask.
assert(!VF.isScalable())((!VF.isScalable()) ? static_cast<void> (0) : __assert_fail
 ("!VF.isScalable()", "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4187, __PRETTY_FUNCTION__));
SmallVector<int, 8> ShuffleMask(VF.getKnownMinValue());
ShuffleMask[0] = VF.getKnownMinValue() - 1;
for (unsigned I = 1; I < VF.getKnownMinValue(); ++I)
  ShuffleMask[I] = I + VF.getKnownMinValue() - 1;

// The vector from which to take the initial value for the current iteration
// (actual or unrolled). Initially, this is the vector phi node.
Value *Incoming = VecPhi;

// Shuffle the current and previous vector and update the vector parts.
for (unsigned Part = 0; Part < UF; ++Part) {
  Value *PreviousPart = State.get(PreviousDef, Part);
  Value *PhiPart = State.get(PhiDef, Part);
  auto *Shuffle =
      VF.isVector()
          ? Builder.CreateShuffleVector(Incoming, PreviousPart, ShuffleMask)
          : Incoming;
  PhiPart->replaceAllUsesWith(Shuffle);
  cast<Instruction>(PhiPart)->eraseFromParent();
  State.reset(PhiDef, Shuffle, Part);
  Incoming = PreviousPart;
}

// Fix the latch value of the new recurrence in the vector loop.
VecPhi->addIncoming(Incoming, LI->getLoopFor(LoopVectorBody)->getLoopLatch());

// Extract the last vector element in the middle block. This will be the
// initial value for the recurrence when jumping to the scalar loop.
auto *ExtractForScalar = Incoming;
if (VF.isVector()) {
  Builder.SetInsertPoint(LoopMiddleBlock->getTerminator());
  ExtractForScalar = Builder.CreateExtractElement(
      ExtractForScalar, Builder.getInt32(VF.getKnownMinValue() - 1),
      "vector.recur.extract");
}
// Extract the second last element in the middle block if the
// Phi is used outside the loop. We need to extract the phi itself
// and not the last element (the phi update in the current iteration). This
// will be the value when jumping to the exit block from the LoopMiddleBlock,
// when the scalar loop is not run at all.
Value *ExtractForPhiUsedOutsideLoop = nullptr;
if (VF.isVector())
  ExtractForPhiUsedOutsideLoop = Builder.CreateExtractElement(
      Incoming, Builder.getInt32(VF.getKnownMinValue() - 2),
      "vector.recur.extract.for.phi");
// When loop is unrolled without vectorizing, initialize
// ExtractForPhiUsedOutsideLoop with the value just prior to unrolled value of
// `Incoming`. This is analogous to the vectorized case above: extracting the
// second last element when VF > 1.
else if (UF > 1)
  ExtractForPhiUsedOutsideLoop = State.get(PreviousDef, UF - 2);

// Fix the initial value of the original recurrence in the scalar loop.
Builder.SetInsertPoint(&*LoopScalarPreHeader->begin());
auto *Start = Builder.CreatePHI(Phi->getType(), 2, "scalar.recur.init");
for (auto *BB : predecessors(LoopScalarPreHeader)) {
  auto *Incoming = BB == LoopMiddleBlock ? ExtractForScalar : ScalarInit;
  Start->addIncoming(Incoming, BB);
}

Phi->setIncomingValueForBlock(LoopScalarPreHeader, Start);
Phi->setName("scalar.recur");

// Finally, fix users of the recurrence outside the loop. The users will need
// either the last value of the scalar recurrence or the last value of the
// vector recurrence we extracted in the middle block. Since the loop is in
// LCSSA form, we just need to find all the phi nodes for the original scalar
// recurrence in the exit block, and then add an edge for the middle block.
// Note that LCSSA does not imply single entry when the original scalar loop
// had multiple exiting edges (as we always run the last iteration in the
// scalar epilogue); in that case, the exiting path through middle will be
// dynamically dead and the value picked for the phi doesn't matter.
for (PHINode &LCSSAPhi : LoopExitBlock->phis())
  if (any_of(LCSSAPhi.incoming_values(),
             [Phi](Value *V) { return V == Phi; }))
    LCSSAPhi.addIncoming(ExtractForPhiUsedOutsideLoop, LoopMiddleBlock);
4264}

4266static bool useOrderedReductions(RecurrenceDescriptor &RdxDesc) {
return EnableStrictReductions && RdxDesc.isOrdered();
4268}

4270void InnerLoopVectorizer::fixReduction(PHINode *Phi, VPTransformState &State) {
// Get it's reduction variable descriptor.
assert(Legal->isReductionVariable(Phi) &&((Legal->isReductionVariable(Phi) && "Unable to find the reduction variable"
) ? static_cast<void> (0) : __assert_fail ("Legal->isReductionVariable(Phi) && \"Unable to find the reduction variable\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4273, __PRETTY_FUNCTION__))
       "Unable to find the reduction variable")((Legal->isReductionVariable(Phi) && "Unable to find the reduction variable"
) ? static_cast<void> (0) : __assert_fail ("Legal->isReductionVariable(Phi) && \"Unable to find the reduction variable\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4273, __PRETTY_FUNCTION__));
RecurrenceDescriptor RdxDesc = Legal->getReductionVars()[Phi];

RecurKind RK = RdxDesc.getRecurrenceKind();
TrackingVH<Value> ReductionStartValue = RdxDesc.getRecurrenceStartValue();
Instruction *LoopExitInst = RdxDesc.getLoopExitInstr();
setDebugLocFromInst(Builder, ReductionStartValue);
bool IsInLoopReductionPhi = Cost->isInLoopReduction(Phi);

VPValue *LoopExitInstDef = State.Plan->getVPValue(LoopExitInst);
// This is the vector-clone of the value that leaves the loop.
Type *VecTy = State.get(LoopExitInstDef, 0)->getType();

// Wrap flags are in general invalid after vectorization, clear them.
clearReductionWrapFlags(RdxDesc, State);

// Fix the vector-loop phi.

// Reductions do not have to start at zero. They can start with
// any loop invariant values.
BasicBlock *Latch = OrigLoop->getLoopLatch();
Value *LoopVal = Phi->getIncomingValueForBlock(Latch);

for (unsigned Part = 0; Part < UF; ++Part) {
  Value *VecRdxPhi = State.get(State.Plan->getVPValue(Phi), Part);
  Value *Val = State.get(State.Plan->getVPValue(LoopVal), Part);
  if (IsInLoopReductionPhi && useOrderedReductions(RdxDesc) &&
      State.VF.isVector())
    Val = State.get(State.Plan->getVPValue(LoopVal), UF - 1);
  cast<PHINode>(VecRdxPhi)
    ->addIncoming(Val, LI->getLoopFor(LoopVectorBody)->getLoopLatch());
}

// Before each round, move the insertion point right between
// the PHIs and the values we are going to write.
// This allows us to write both PHINodes and the extractelement
// instructions.
Builder.SetInsertPoint(&*LoopMiddleBlock->getFirstInsertionPt());

setDebugLocFromInst(Builder, LoopExitInst);

Type *PhiTy = Phi->getType();
// If tail is folded by masking, the vector value to leave the loop should be
// a Select choosing between the vectorized LoopExitInst and vectorized Phi,
// instead of the former. For an inloop reduction the reduction will already
// be predicated, and does not need to be handled here.
if (Cost->foldTailByMasking() && !IsInLoopReductionPhi) {
  for (unsigned Part = 0; Part < UF; ++Part) {
    Value *VecLoopExitInst = State.get(LoopExitInstDef, Part);
    Value *Sel = nullptr;
    for (User *U : VecLoopExitInst->users()) {
      if (isa<SelectInst>(U)) {
        assert(!Sel && "Reduction exit feeding two selects")((!Sel && "Reduction exit feeding two selects") ? static_cast
<void> (0) : __assert_fail ("!Sel && \"Reduction exit feeding two selects\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4325, __PRETTY_FUNCTION__));
        Sel = U;
      } else
        assert(isa<PHINode>(U) && "Reduction exit must feed Phi's or select")((isa<PHINode>(U) && "Reduction exit must feed Phi's or select"
) ? static_cast<void> (0) : __assert_fail ("isa<PHINode>(U) && \"Reduction exit must feed Phi's or select\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4328, __PRETTY_FUNCTION__));
    }
    assert(Sel && "Reduction exit feeds no select")((Sel && "Reduction exit feeds no select") ? static_cast
<void> (0) : __assert_fail ("Sel && \"Reduction exit feeds no select\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4330, __PRETTY_FUNCTION__));
    State.reset(LoopExitInstDef, Sel, Part);

    // If the target can create a predicated operator for the reduction at no
    // extra cost in the loop (for example a predicated vadd), it can be
    // cheaper for the select to remain in the loop than be sunk out of it,
    // and so use the select value for the phi instead of the old
    // LoopExitValue.
    if (PreferPredicatedReductionSelect ||
        TTI->preferPredicatedReductionSelect(
            RdxDesc.getOpcode(), PhiTy,
            TargetTransformInfo::ReductionFlags())) {
      auto *VecRdxPhi =
          cast<PHINode>(State.get(State.Plan->getVPValue(Phi), Part));
      VecRdxPhi->setIncomingValueForBlock(
          LI->getLoopFor(LoopVectorBody)->getLoopLatch(), Sel);
    }
  }
}

// If the vector reduction can be performed in a smaller type, we truncate
// then extend the loop exit value to enable InstCombine to evaluate the
// entire expression in the smaller type.
if (VF.isVector() && PhiTy != RdxDesc.getRecurrenceType()) {
  assert(!IsInLoopReductionPhi && "Unexpected truncated inloop reduction!")((!IsInLoopReductionPhi && "Unexpected truncated inloop reduction!"
) ? static_cast<void> (0) : __assert_fail ("!IsInLoopReductionPhi && \"Unexpected truncated inloop reduction!\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4354, __PRETTY_FUNCTION__));
  assert(!VF.isScalable() && "scalable vectors not yet supported.")((!VF.isScalable() && "scalable vectors not yet supported."
) ? static_cast<void> (0) : __assert_fail ("!VF.isScalable() && \"scalable vectors not yet supported.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4355, __PRETTY_FUNCTION__));
  Type *RdxVecTy = VectorType::get(RdxDesc.getRecurrenceType(), VF);
  Builder.SetInsertPoint(
      LI->getLoopFor(LoopVectorBody)->getLoopLatch()->getTerminator());
  VectorParts RdxParts(UF);
  for (unsigned Part = 0; Part < UF; ++Part) {
    RdxParts[Part] = State.get(LoopExitInstDef, Part);
    Value *Trunc = Builder.CreateTrunc(RdxParts[Part], RdxVecTy);
    Value *Extnd = RdxDesc.isSigned() ? Builder.CreateSExt(Trunc, VecTy)
                                      : Builder.CreateZExt(Trunc, VecTy);
    for (Value::user_iterator UI = RdxParts[Part]->user_begin();
         UI != RdxParts[Part]->user_end();)
      if (*UI != Trunc) {
        (*UI++)->replaceUsesOfWith(RdxParts[Part], Extnd);
        RdxParts[Part] = Extnd;
      } else {
        ++UI;
      }
  }
  Builder.SetInsertPoint(&*LoopMiddleBlock->getFirstInsertionPt());
  for (unsigned Part = 0; Part < UF; ++Part) {
    RdxParts[Part] = Builder.CreateTrunc(RdxParts[Part], RdxVecTy);
    State.reset(LoopExitInstDef, RdxParts[Part], Part);
  }
}

// Reduce all of the unrolled parts into a single vector.
Value *ReducedPartRdx = State.get(LoopExitInstDef, 0);
unsigned Op = RecurrenceDescriptor::getOpcode(RK);

// The middle block terminator has already been assigned a DebugLoc here (the
// OrigLoop's single latch terminator). We want the whole middle block to
// appear to execute on this line because: (a) it is all compiler generated,
// (b) these instructions are always executed after evaluating the latch
// conditional branch, and (c) other passes may add new predecessors which
// terminate on this line. This is the easiest way to ensure we don't
// accidentally cause an extra step back into the loop while debugging.
setDebugLocFromInst(Builder, LoopMiddleBlock->getTerminator());
if (IsInLoopReductionPhi && useOrderedReductions(RdxDesc))
  ReducedPartRdx = State.get(LoopExitInstDef, UF - 1);
else {
  // Floating-point operations should have some FMF to enable the reduction.
  IRBuilderBase::FastMathFlagGuard FMFG(Builder);
  Builder.setFastMathFlags(RdxDesc.getFastMathFlags());
  for (unsigned Part = 1; Part < UF; ++Part) {
    Value *RdxPart = State.get(LoopExitInstDef, Part);
    if (Op != Instruction::ICmp && Op != Instruction::FCmp) {
      ReducedPartRdx = Builder.CreateBinOp(
          (Instruction::BinaryOps)Op, RdxPart, ReducedPartRdx, "bin.rdx");
    } else {
      ReducedPartRdx = createMinMaxOp(Builder, RK, ReducedPartRdx, RdxPart);
    }
  }
}

// Create the reduction after the loop. Note that inloop reductions create the
// target reduction in the loop using a Reduction recipe.
if (VF.isVector() && !IsInLoopReductionPhi) {
  ReducedPartRdx =
      createTargetReduction(Builder, TTI, RdxDesc, ReducedPartRdx);
  // If the reduction can be performed in a smaller type, we need to extend
  // the reduction to the wider type before we branch to the original loop.
  if (PhiTy != RdxDesc.getRecurrenceType())
    ReducedPartRdx = RdxDesc.isSigned()
                         ? Builder.CreateSExt(ReducedPartRdx, PhiTy)
                         : Builder.CreateZExt(ReducedPartRdx, PhiTy);
}

// Create a phi node that merges control-flow from the backedge-taken check
// block and the middle block.
PHINode *BCBlockPhi = PHINode::Create(PhiTy, 2, "bc.merge.rdx",
                                      LoopScalarPreHeader->getTerminator());
for (unsigned I = 0, E = LoopBypassBlocks.size(); I != E; ++I)
  BCBlockPhi->addIncoming(ReductionStartValue, LoopBypassBlocks[I]);
BCBlockPhi->addIncoming(ReducedPartRdx, LoopMiddleBlock);

// Now, we need to fix the users of the reduction variable
// inside and outside of the scalar remainder loop.

// We know that the loop is in LCSSA form. We need to update the PHI nodes
// in the exit blocks.  See comment on analogous loop in
// fixFirstOrderRecurrence for a more complete explaination of the logic.
for (PHINode &LCSSAPhi : LoopExitBlock->phis())
  if (any_of(LCSSAPhi.incoming_values(),
             [LoopExitInst](Value *V) { return V == LoopExitInst; }))
    LCSSAPhi.addIncoming(ReducedPartRdx, LoopMiddleBlock);

// Fix the scalar loop reduction variable with the incoming reduction sum
// from the vector body and from the backedge value.
int IncomingEdgeBlockIdx =
  Phi->getBasicBlockIndex(OrigLoop->getLoopLatch());
assert(IncomingEdgeBlockIdx >= 0 && "Invalid block index")((IncomingEdgeBlockIdx >= 0 && "Invalid block index"
) ? static_cast<void> (0) : __assert_fail ("IncomingEdgeBlockIdx >= 0 && \"Invalid block index\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4446, __PRETTY_FUNCTION__));
// Pick the other block.
int SelfEdgeBlockIdx = (IncomingEdgeBlockIdx ? 0 : 1);
Phi->setIncomingValue(SelfEdgeBlockIdx, BCBlockPhi);
Phi->setIncomingValue(IncomingEdgeBlockIdx, LoopExitInst);
4451}

4453void InnerLoopVectorizer::clearReductionWrapFlags(RecurrenceDescriptor &RdxDesc,
                                                VPTransformState &State) {
RecurKind RK = RdxDesc.getRecurrenceKind();
if (RK != RecurKind::Add && RK != RecurKind::Mul)
  return;

Instruction *LoopExitInstr = RdxDesc.getLoopExitInstr();
assert(LoopExitInstr && "null loop exit instruction")((LoopExitInstr && "null loop exit instruction") ? static_cast
<void> (0) : __assert_fail ("LoopExitInstr && \"null loop exit instruction\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4460, __PRETTY_FUNCTION__));
SmallVector<Instruction *, 8> Worklist;
SmallPtrSet<Instruction *, 8> Visited;
Worklist.push_back(LoopExitInstr);
Visited.insert(LoopExitInstr);

while (!Worklist.empty()) {
  Instruction *Cur = Worklist.pop_back_val();
  if (isa<OverflowingBinaryOperator>(Cur))
    for (unsigned Part = 0; Part < UF; ++Part) {
      Value *V = State.get(State.Plan->getVPValue(Cur), Part);
      cast<Instruction>(V)->dropPoisonGeneratingFlags();
    }

  for (User *U : Cur->users()) {
    Instruction *UI = cast<Instruction>(U);
    if ((Cur != LoopExitInstr || OrigLoop->contains(UI->getParent())) &&
        Visited.insert(UI).second)
      Worklist.push_back(UI);
  }
}
4481}

4483void InnerLoopVectorizer::fixLCSSAPHIs(VPTransformState &State) {
for (PHINode &LCSSAPhi : LoopExitBlock->phis()) {
  if (LCSSAPhi.getBasicBlockIndex(LoopMiddleBlock) != -1)
    // Some phis were already hand updated by the reduction and recurrence
    // code above, leave them alone.
    continue;

  auto *IncomingValue = LCSSAPhi.getIncomingValue(0);
  // Non-instruction incoming values will have only one value.

  VPLane Lane = VPLane::getFirstLane();
  if (isa<Instruction>(IncomingValue) &&
      !Cost->isUniformAfterVectorization(cast<Instruction>(IncomingValue),
                                         VF))
    Lane = VPLane::getLastLaneForVF(VF);

  // Can be a loop invariant incoming value or the last scalar value to be
  // extracted from the vectorized loop.
  Builder.SetInsertPoint(LoopMiddleBlock->getTerminator());
  Value *lastIncomingValue =
      OrigLoop->isLoopInvariant(IncomingValue)
          ? IncomingValue
          : State.get(State.Plan->getVPValue(IncomingValue),
                      VPIteration(UF - 1, Lane));
  LCSSAPhi.addIncoming(lastIncomingValue, LoopMiddleBlock);
}
4509}

4511void InnerLoopVectorizer::sinkScalarOperands(Instruction *PredInst) {
// The basic block and loop containing the predicated instruction.
auto *PredBB = PredInst->getParent();
auto *VectorLoop = LI->getLoopFor(PredBB);

// Initialize a worklist with the operands of the predicated instruction.
SetVector<Value *> Worklist(PredInst->op_begin(), PredInst->op_end());

// Holds instructions that we need to analyze again. An instruction may be
// reanalyzed if we don't yet know if we can sink it or not.
SmallVector<Instruction *, 8> InstsToReanalyze;

// Returns true if a given use occurs in the predicated block. Phi nodes use
// their operands in their corresponding predecessor blocks.
auto isBlockOfUsePredicated = [&](Use &U) -> bool {
  auto *I = cast<Instruction>(U.getUser());
  BasicBlock *BB = I->getParent();
  if (auto *Phi = dyn_cast<PHINode>(I))
    BB = Phi->getIncomingBlock(
        PHINode::getIncomingValueNumForOperand(U.getOperandNo()));
  return BB == PredBB;
};

// Iteratively sink the scalarized operands of the predicated instruction
// into the block we created for it. When an instruction is sunk, it's
// operands are then added to the worklist. The algorithm ends after one pass
// through the worklist doesn't sink a single instruction.
bool Changed;
do {
  // Add the instructions that need to be reanalyzed to the worklist, and
  // reset the changed indicator.
  Worklist.insert(InstsToReanalyze.begin(), InstsToReanalyze.end());
  InstsToReanalyze.clear();
  Changed = false;

  while (!Worklist.empty()) {
    auto *I = dyn_cast<Instruction>(Worklist.pop_back_val());

    // We can't sink an instruction if it is a phi node, is already in the
    // predicated block, is not in the loop, or may have side effects.
    if (!I || isa<PHINode>(I) || I->getParent() == PredBB ||
        !VectorLoop->contains(I) || I->mayHaveSideEffects())
      continue;

    // It's legal to sink the instruction if all its uses occur in the
    // predicated block. Otherwise, there's nothing to do yet, and we may
    // need to reanalyze the instruction.
    if (!llvm::all_of(I->uses(), isBlockOfUsePredicated)) {
      InstsToReanalyze.push_back(I);
      continue;
    }

    // Move the instruction to the beginning of the predicated block, and add
    // it's operands to the worklist.
    I->moveBefore(&*PredBB->getFirstInsertionPt());
    Worklist.insert(I->op_begin(), I->op_end());

    // The sinking may have enabled other instructions to be sunk, so we will
    // need to iterate.
    Changed = true;
  }
} while (Changed);
4573}

4575void InnerLoopVectorizer::fixNonInductionPHIs(VPTransformState &State) {
for (PHINode *OrigPhi : OrigPHIsToFix) {
  VPWidenPHIRecipe *VPPhi =
      cast<VPWidenPHIRecipe>(State.Plan->getVPValue(OrigPhi));
  PHINode *NewPhi = cast<PHINode>(State.get(VPPhi, 0));
  // Make sure the builder has a valid insert point.
  Builder.SetInsertPoint(NewPhi);
  for (unsigned i = 0; i < VPPhi->getNumOperands(); ++i) {
    VPValue *Inc = VPPhi->getIncomingValue(i);
    VPBasicBlock *VPBB = VPPhi->getIncomingBlock(i);
    NewPhi->addIncoming(State.get(Inc, 0), State.CFG.VPBB2IRBB[VPBB]);
  }
}
4588}

4590void InnerLoopVectorizer::widenGEP(GetElementPtrInst *GEP, VPValue *VPDef,
                                 VPUser &Operands, unsigned UF,
                                 ElementCount VF, bool IsPtrLoopInvariant,
                                 SmallBitVector &IsIndexLoopInvariant,
                                 VPTransformState &State) {
// Construct a vector GEP by widening the operands of the scalar GEP as
// necessary. We mark the vector GEP 'inbounds' if appropriate. A GEP
// results in a vector of pointers when at least one operand of the GEP
// is vector-typed. Thus, to keep the representation compact, we only use
// vector-typed operands for loop-varying values.

if (VF.isVector() && IsPtrLoopInvariant && IsIndexLoopInvariant.all()) {
  // If we are vectorizing, but the GEP has only loop-invariant operands,
  // the GEP we build (by only using vector-typed operands for
  // loop-varying values) would be a scalar pointer. Thus, to ensure we
  // produce a vector of pointers, we need to either arbitrarily pick an
  // operand to broadcast, or broadcast a clone of the original GEP.
  // Here, we broadcast a clone of the original.
  //
  // TODO: If at some point we decide to scalarize instructions having
  //       loop-invariant operands, this special case will no longer be
  //       required. We would add the scalarization decision to
  //       collectLoopScalars() and teach getVectorValue() to broadcast
  //       the lane-zero scalar value.
  auto *Clone = Builder.Insert(GEP->clone());
  for (unsigned Part = 0; Part < UF; ++Part) {
    Value *EntryPart = Builder.CreateVectorSplat(VF, Clone);
    State.set(VPDef, EntryPart, Part);
    addMetadata(EntryPart, GEP);
  }
} else {
  // If the GEP has at least one loop-varying operand, we are sure to
  // produce a vector of pointers. But if we are only unrolling, we want
  // to produce a scalar GEP for each unroll part. Thus, the GEP we
  // produce with the code below will be scalar (if VF == 1) or vector
  // (otherwise). Note that for the unroll-only case, we still maintain
  // values in the vector mapping with initVector, as we do for other
  // instructions.
  for (unsigned Part = 0; Part < UF; ++Part) {
    // The pointer operand of the new GEP. If it's loop-invariant, we
    // won't broadcast it.
    auto *Ptr = IsPtrLoopInvariant
                    ? State.get(Operands.getOperand(0), VPIteration(0, 0))
                    : State.get(Operands.getOperand(0), Part);

    // Collect all the indices for the new GEP. If any index is
    // loop-invariant, we won't broadcast it.
    SmallVector<Value *, 4> Indices;
    for (unsigned I = 1, E = Operands.getNumOperands(); I < E; I++) {
      VPValue *Operand = Operands.getOperand(I);
      if (IsIndexLoopInvariant[I - 1])
        Indices.push_back(State.get(Operand, VPIteration(0, 0)));
      else
        Indices.push_back(State.get(Operand, Part));
    }

    // Create the new GEP. Note that this GEP may be a scalar if VF == 1,
    // but it should be a vector, otherwise.
    auto *NewGEP =
        GEP->isInBounds()
            ? Builder.CreateInBoundsGEP(GEP->getSourceElementType(), Ptr,
                                        Indices)
            : Builder.CreateGEP(GEP->getSourceElementType(), Ptr, Indices);
    assert((VF.isScalar() || NewGEP->getType()->isVectorTy()) &&(((VF.isScalar() || NewGEP->getType()->isVectorTy()) &&
 "NewGEP is not a pointer vector") ? static_cast<void> (
0) : __assert_fail ("(VF.isScalar() || NewGEP->getType()->isVectorTy()) && \"NewGEP is not a pointer vector\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4654, __PRETTY_FUNCTION__))
           "NewGEP is not a pointer vector")(((VF.isScalar() || NewGEP->getType()->isVectorTy()) &&
 "NewGEP is not a pointer vector") ? static_cast<void> (
0) : __assert_fail ("(VF.isScalar() || NewGEP->getType()->isVectorTy()) && \"NewGEP is not a pointer vector\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4654, __PRETTY_FUNCTION__));
    State.set(VPDef, NewGEP, Part);
    addMetadata(NewGEP, GEP);
  }
}
4659}

4661void InnerLoopVectorizer::widenPHIInstruction(Instruction *PN,
                                            RecurrenceDescriptor *RdxDesc,
                                            VPWidenPHIRecipe *PhiR,
                                            VPTransformState &State) {
PHINode *P = cast<PHINode>(PN);
if (EnableVPlanNativePath) {
  // Currently we enter here in the VPlan-native path for non-induction
  // PHIs where all control flow is uniform. We simply widen these PHIs.
  // Create a vector phi with no operands - the vector phi operands will be
  // set at the end of vector code generation.
  Type *VecTy = (State.VF.isScalar())
                    ? PN->getType()
                    : VectorType::get(PN->getType(), State.VF);
  Value *VecPhi = Builder.CreatePHI(VecTy, PN->getNumOperands(), "vec.phi");
  State.set(PhiR, VecPhi, 0);
  OrigPHIsToFix.push_back(P);

  return;
}

assert(PN->getParent() == OrigLoop->getHeader() &&((PN->getParent() == OrigLoop->getHeader() && "Non-header phis should have been handled elsewhere"
) ? static_cast<void> (0) : __assert_fail ("PN->getParent() == OrigLoop->getHeader() && \"Non-header phis should have been handled elsewhere\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4682, __PRETTY_FUNCTION__))
       "Non-header phis should have been handled elsewhere")((PN->getParent() == OrigLoop->getHeader() && "Non-header phis should have been handled elsewhere"
) ? static_cast<void> (0) : __assert_fail ("PN->getParent() == OrigLoop->getHeader() && \"Non-header phis should have been handled elsewhere\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4682, __PRETTY_FUNCTION__));

VPValue *StartVPV = PhiR->getStartValue();
Value *StartV = StartVPV ? StartVPV->getLiveInIRValue() : nullptr;
// In order to support recurrences we need to be able to vectorize Phi nodes.
// Phi nodes have cycles, so we need to vectorize them in two stages. This is
// stage #1: We create a new vector PHI node with no incoming edges. We'll use
// this value when we vectorize all of the instructions that use the PHI.
if (RdxDesc || Legal->isFirstOrderRecurrence(P)) {
  Value *Iden = nullptr;
  bool ScalarPHI =
      (State.VF.isScalar()) || Cost->isInLoopReduction(cast<PHINode>(PN));
  Type *VecTy =
      ScalarPHI ? PN->getType() : VectorType::get(PN->getType(), State.VF);

  if (RdxDesc) {
    assert(Legal->isReductionVariable(P) && StartV &&((Legal->isReductionVariable(P) && StartV &&
 "RdxDesc should only be set for reduction variables; in that case "
 "a StartV is also required") ? static_cast<void> (0) :
 __assert_fail ("Legal->isReductionVariable(P) && StartV && \"RdxDesc should only be set for reduction variables; in that case \" \"a StartV is also required\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4700, __PRETTY_FUNCTION__))
           "RdxDesc should only be set for reduction variables; in that case "((Legal->isReductionVariable(P) && StartV &&
 "RdxDesc should only be set for reduction variables; in that case "
 "a StartV is also required") ? static_cast<void> (0) :
 __assert_fail ("Legal->isReductionVariable(P) && StartV && \"RdxDesc should only be set for reduction variables; in that case \" \"a StartV is also required\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4700, __PRETTY_FUNCTION__))
           "a StartV is also required")((Legal->isReductionVariable(P) && StartV &&
 "RdxDesc should only be set for reduction variables; in that case "
 "a StartV is also required") ? static_cast<void> (0) :
 __assert_fail ("Legal->isReductionVariable(P) && StartV && \"RdxDesc should only be set for reduction variables; in that case \" \"a StartV is also required\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4700, __PRETTY_FUNCTION__));
    RecurKind RK = RdxDesc->getRecurrenceKind();
    if (RecurrenceDescriptor::isMinMaxRecurrenceKind(RK)) {
      // MinMax reduction have the start value as their identify.
      if (ScalarPHI) {
        Iden = StartV;
      } else {
        IRBuilderBase::InsertPointGuard IPBuilder(Builder);
        Builder.SetInsertPoint(LoopVectorPreHeader->getTerminator());
        StartV = Iden =
            Builder.CreateVectorSplat(State.VF, StartV, "minmax.ident");
      }
    } else {
      Constant *IdenC = RecurrenceDescriptor::getRecurrenceIdentity(
          RK, VecTy->getScalarType(), RdxDesc->getFastMathFlags());
      Iden = IdenC;

      if (!ScalarPHI) {
        Iden = ConstantVector::getSplat(State.VF, IdenC);
        IRBuilderBase::InsertPointGuard IPBuilder(Builder);
        Builder.SetInsertPoint(LoopVectorPreHeader->getTerminator());
        Constant *Zero = Builder.getInt32(0);
        StartV = Builder.CreateInsertElement(Iden, StartV, Zero);
      }
    }
  }

  for (unsigned Part = 0; Part < State.UF; ++Part) {
    // This is phase one of vectorizing PHIs.
    Value *EntryPart = PHINode::Create(
        VecTy, 2, "vec.phi", &*LoopVectorBody->getFirstInsertionPt());
    State.set(PhiR, EntryPart, Part);
    if (StartV) {
      // Make sure to add the reduction start value only to the
      // first unroll part.
      Value *StartVal = (Part == 0) ? StartV : Iden;
      cast<PHINode>(EntryPart)->addIncoming(StartVal, LoopVectorPreHeader);
    }
  }
  return;
}

assert(!Legal->isReductionVariable(P) &&((!Legal->isReductionVariable(P) && "reductions should be handled above"
) ? static_cast<void> (0) : __assert_fail ("!Legal->isReductionVariable(P) && \"reductions should be handled above\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4743, __PRETTY_FUNCTION__))
       "reductions should be handled above")((!Legal->isReductionVariable(P) && "reductions should be handled above"
) ? static_cast<void> (0) : __assert_fail ("!Legal->isReductionVariable(P) && \"reductions should be handled above\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4743, __PRETTY_FUNCTION__));

setDebugLocFromInst(Builder, P);

// This PHINode must be an induction variable.
// Make sure that we know about it.
assert(Legal->getInductionVars().count(P) && "Not an induction variable")((Legal->getInductionVars().count(P) && "Not an induction variable"
) ? static_cast<void> (0) : __assert_fail ("Legal->getInductionVars().count(P) && \"Not an induction variable\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4749, __PRETTY_FUNCTION__));

InductionDescriptor II = Legal->getInductionVars().lookup(P);
const DataLayout &DL = OrigLoop->getHeader()->getModule()->getDataLayout();

// FIXME: The newly created binary instructions should contain nsw/nuw flags,
// which can be found from the original scalar operations.
switch (II.getKind()) {
case InductionDescriptor::IK_NoInduction:
  llvm_unreachable("Unknown induction")::llvm::llvm_unreachable_internal("Unknown induction", "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4758);
case InductionDescriptor::IK_IntInduction:
case InductionDescriptor::IK_FpInduction:
  llvm_unreachable("Integer/fp induction is handled elsewhere.")::llvm::llvm_unreachable_internal("Integer/fp induction is handled elsewhere."
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4761);
case InductionDescriptor::IK_PtrInduction: {
  // Handle the pointer induction variable case.
  assert(P->getType()->isPointerTy() && "Unexpected type.")((P->getType()->isPointerTy() && "Unexpected type."
) ? static_cast<void> (0) : __assert_fail ("P->getType()->isPointerTy() && \"Unexpected type.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4764, __PRETTY_FUNCTION__));
  assert(!VF.isScalable() && "Currently unsupported for scalable vectors")((!VF.isScalable() && "Currently unsupported for scalable vectors"
) ? static_cast<void> (0) : __assert_fail ("!VF.isScalable() && \"Currently unsupported for scalable vectors\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4765, __PRETTY_FUNCTION__));

  if (Cost->isScalarAfterVectorization(P, State.VF)) {
    // This is the normalized GEP that starts counting at zero.
    Value *PtrInd =
        Builder.CreateSExtOrTrunc(Induction, II.getStep()->getType());
    // Determine the number of scalars we need to generate for each unroll
    // iteration. If the instruction is uniform, we only need to generate the
    // first lane. Otherwise, we generate all VF values.
    unsigned Lanes = Cost->isUniformAfterVectorization(P, State.VF)
                         ? 1
                         : State.VF.getKnownMinValue();
    for (unsigned Part = 0; Part < UF; ++Part) {
      for (unsigned Lane = 0; Lane < Lanes; ++Lane) {
        Constant *Idx = ConstantInt::get(
            PtrInd->getType(), Lane + Part * State.VF.getKnownMinValue());
        Value *GlobalIdx = Builder.CreateAdd(PtrInd, Idx);
        Value *SclrGep =
            emitTransformedIndex(Builder, GlobalIdx, PSE.getSE(), DL, II);
        SclrGep->setName("next.gep");
        State.set(PhiR, SclrGep, VPIteration(Part, Lane));
      }
    }
    return;
  }
  assert(isa<SCEVConstant>(II.getStep()) &&((isa<SCEVConstant>(II.getStep()) && "Induction step not a SCEV constant!"
) ? static_cast<void> (0) : __assert_fail ("isa<SCEVConstant>(II.getStep()) && \"Induction step not a SCEV constant!\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4791, __PRETTY_FUNCTION__))
         "Induction step not a SCEV constant!")((isa<SCEVConstant>(II.getStep()) && "Induction step not a SCEV constant!"
) ? static_cast<void> (0) : __assert_fail ("isa<SCEVConstant>(II.getStep()) && \"Induction step not a SCEV constant!\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4791, __PRETTY_FUNCTION__));
  Type *PhiType = II.getStep()->getType();

  // Build a pointer phi
  Value *ScalarStartValue = II.getStartValue();
  Type *ScStValueType = ScalarStartValue->getType();
  PHINode *NewPointerPhi =
      PHINode::Create(ScStValueType, 2, "pointer.phi", Induction);
  NewPointerPhi->addIncoming(ScalarStartValue, LoopVectorPreHeader);

  // A pointer induction, performed by using a gep
  BasicBlock *LoopLatch = LI->getLoopFor(LoopVectorBody)->getLoopLatch();
  Instruction *InductionLoc = LoopLatch->getTerminator();
  const SCEV *ScalarStep = II.getStep();
  SCEVExpander Exp(*PSE.getSE(), DL, "induction");
  Value *ScalarStepValue =
      Exp.expandCodeFor(ScalarStep, PhiType, InductionLoc);
  Value *InductionGEP = GetElementPtrInst::Create(
      ScStValueType->getPointerElementType(), NewPointerPhi,
      Builder.CreateMul(
          ScalarStepValue,
          ConstantInt::get(PhiType, State.VF.getKnownMinValue() * State.UF)),
      "ptr.ind", InductionLoc);
  NewPointerPhi->addIncoming(InductionGEP, LoopLatch);

  // Create UF many actual address geps that use the pointer
  // phi as base and a vectorized version of the step value
  // (<step*0, ..., step*N>) as offset.
  for (unsigned Part = 0; Part < State.UF; ++Part) {
    Type *VecPhiType = VectorType::get(PhiType, State.VF);
    Value *StartOffset =
        ConstantInt::get(VecPhiType, Part * State.VF.getKnownMinValue());
    // Create a vector of consecutive numbers from zero to VF.
    StartOffset =
        Builder.CreateAdd(StartOffset, Builder.CreateStepVector(VecPhiType));

    Value *GEP = Builder.CreateGEP(
        ScStValueType->getPointerElementType(), NewPointerPhi,
        Builder.CreateMul(StartOffset,
                          Builder.CreateVectorSplat(
                              State.VF.getKnownMinValue(), ScalarStepValue),
                          "vector.gep"));
    State.set(PhiR, GEP, Part);
  }
}
}
4837}

4839/// A helper function for checking whether an integer division-related
4840/// instruction may divide by zero (in which case it must be predicated if
4841/// executed conditionally in the scalar code).
4842/// TODO: It may be worthwhile to generalize and check isKnownNonZero().
4843/// Non-zero divisors that are non compile-time constants will not be
4844/// converted into multiplication, so we will still end up scalarizing
4845/// the division, but can do so w/o predication.
4846static bool mayDivideByZero(Instruction &I) {
assert((I.getOpcode() == Instruction::UDiv ||(((I.getOpcode() == Instruction::UDiv || I.getOpcode() == Instruction
::SDiv || I.getOpcode() == Instruction::URem || I.getOpcode()
 == Instruction::SRem) && "Unexpected instruction") ?
 static_cast<void> (0) : __assert_fail ("(I.getOpcode() == Instruction::UDiv || I.getOpcode() == Instruction::SDiv || I.getOpcode() == Instruction::URem || I.getOpcode() == Instruction::SRem) && \"Unexpected instruction\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4851, __PRETTY_FUNCTION__))
        I.getOpcode() == Instruction::SDiv ||(((I.getOpcode() == Instruction::UDiv || I.getOpcode() == Instruction
::SDiv || I.getOpcode() == Instruction::URem || I.getOpcode()
 == Instruction::SRem) && "Unexpected instruction") ?
 static_cast<void> (0) : __assert_fail ("(I.getOpcode() == Instruction::UDiv || I.getOpcode() == Instruction::SDiv || I.getOpcode() == Instruction::URem || I.getOpcode() == Instruction::SRem) && \"Unexpected instruction\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4851, __PRETTY_FUNCTION__))
        I.getOpcode() == Instruction::URem ||(((I.getOpcode() == Instruction::UDiv || I.getOpcode() == Instruction
::SDiv || I.getOpcode() == Instruction::URem || I.getOpcode()
 == Instruction::SRem) && "Unexpected instruction") ?
 static_cast<void> (0) : __assert_fail ("(I.getOpcode() == Instruction::UDiv || I.getOpcode() == Instruction::SDiv || I.getOpcode() == Instruction::URem || I.getOpcode() == Instruction::SRem) && \"Unexpected instruction\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4851, __PRETTY_FUNCTION__))
        I.getOpcode() == Instruction::SRem) &&(((I.getOpcode() == Instruction::UDiv || I.getOpcode() == Instruction
::SDiv || I.getOpcode() == Instruction::URem || I.getOpcode()
 == Instruction::SRem) && "Unexpected instruction") ?
 static_cast<void> (0) : __assert_fail ("(I.getOpcode() == Instruction::UDiv || I.getOpcode() == Instruction::SDiv || I.getOpcode() == Instruction::URem || I.getOpcode() == Instruction::SRem) && \"Unexpected instruction\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4851, __PRETTY_FUNCTION__))
       "Unexpected instruction")(((I.getOpcode() == Instruction::UDiv || I.getOpcode() == Instruction
::SDiv || I.getOpcode() == Instruction::URem || I.getOpcode()
 == Instruction::SRem) && "Unexpected instruction") ?
 static_cast<void> (0) : __assert_fail ("(I.getOpcode() == Instruction::UDiv || I.getOpcode() == Instruction::SDiv || I.getOpcode() == Instruction::URem || I.getOpcode() == Instruction::SRem) && \"Unexpected instruction\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4851, __PRETTY_FUNCTION__));
Value *Divisor = I.getOperand(1);
auto *CInt = dyn_cast<ConstantInt>(Divisor);
return !CInt || CInt->isZero();
4855}

4857void InnerLoopVectorizer::widenInstruction(Instruction &I, VPValue *Def,
                                         VPUser &User,
                                         VPTransformState &State) {
switch (I.getOpcode()) {
case Instruction::Call:
case Instruction::Br:
case Instruction::PHI:
case Instruction::GetElementPtr:
case Instruction::Select:
  llvm_unreachable("This instruction is handled by a different recipe.")::llvm::llvm_unreachable_internal("This instruction is handled by a different recipe."
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4866);
case Instruction::UDiv:
case Instruction::SDiv:
case Instruction::SRem:
case Instruction::URem:
case Instruction::Add:
case Instruction::FAdd:
case Instruction::Sub:
case Instruction::FSub:
case Instruction::FNeg:
case Instruction::Mul:
case Instruction::FMul:
case Instruction::FDiv:
case Instruction::FRem:
case Instruction::Shl:
case Instruction::LShr:
case Instruction::AShr:
case Instruction::And:
case Instruction::Or:
case Instruction::Xor: {
  // Just widen unops and binops.
  setDebugLocFromInst(Builder, &I);

  for (unsigned Part = 0; Part < UF; ++Part) {
    SmallVector<Value *, 2> Ops;
    for (VPValue *VPOp : User.operands())
      Ops.push_back(State.get(VPOp, Part));

    Value *V = Builder.CreateNAryOp(I.getOpcode(), Ops);

    if (auto *VecOp = dyn_cast<Instruction>(V))
      VecOp->copyIRFlags(&I);

    // Use this vector value for all users of the original instruction.
    State.set(Def, V, Part);
    addMetadata(V, &I);
  }

  break;
}
case Instruction::ICmp:
case Instruction::FCmp: {
  // Widen compares. Generate vector compares.
  bool FCmp = (I.getOpcode() == Instruction::FCmp);
  auto *Cmp = cast<CmpInst>(&I);
  setDebugLocFromInst(Builder, Cmp);
  for (unsigned Part = 0; Part < UF; ++Part) {
    Value *A = State.get(User.getOperand(0), Part);
    Value *B = State.get(User.getOperand(1), Part);
    Value *C = nullptr;
    if (FCmp) {
      // Propagate fast math flags.
      IRBuilder<>::FastMathFlagGuard FMFG(Builder);
      Builder.setFastMathFlags(Cmp->getFastMathFlags());
      C = Builder.CreateFCmp(Cmp->getPredicate(), A, B);
    } else {
      C = Builder.CreateICmp(Cmp->getPredicate(), A, B);
    }
    State.set(Def, C, Part);
    addMetadata(C, &I);
  }

  break;
}

case Instruction::ZExt:
case Instruction::SExt:
case Instruction::FPToUI:
case Instruction::FPToSI:
case Instruction::FPExt:
case Instruction::PtrToInt:
case Instruction::IntToPtr:
case Instruction::SIToFP:
case Instruction::UIToFP:
case Instruction::Trunc:
case Instruction::FPTrunc:
case Instruction::BitCast: {
  auto *CI = cast<CastInst>(&I);
  setDebugLocFromInst(Builder, CI);

  /// Vectorize casts.
  Type *DestTy =
      (VF.isScalar()) ? CI->getType() : VectorType::get(CI->getType(), VF);

  for (unsigned Part = 0; Part < UF; ++Part) {
    Value *A = State.get(User.getOperand(0), Part);
    Value *Cast = Builder.CreateCast(CI->getOpcode(), A, DestTy);
    State.set(Def, Cast, Part);
    addMetadata(Cast, &I);
  }
  break;
}
default:
  // This instruction is not vectorized by simple widening.
  LLVM_DEBUG(dbgs() << "LV: Found an unhandled instruction: " << I)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Found an unhandled instruction: "
 << I; } } while (false);
  llvm_unreachable("Unhandled instruction!")::llvm::llvm_unreachable_internal("Unhandled instruction!", "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4961);
} // end of switch.
4963}

4965void InnerLoopVectorizer::widenCallInstruction(CallInst &I, VPValue *Def,
                                             VPUser &ArgOperands,
                                             VPTransformState &State) {
assert(!isa<DbgInfoIntrinsic>(I) &&((!isa<DbgInfoIntrinsic>(I) && "DbgInfoIntrinsic should have been dropped during VPlan construction"
) ? static_cast<void> (0) : __assert_fail ("!isa<DbgInfoIntrinsic>(I) && \"DbgInfoIntrinsic should have been dropped during VPlan construction\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4969, __PRETTY_FUNCTION__))
       "DbgInfoIntrinsic should have been dropped during VPlan construction")((!isa<DbgInfoIntrinsic>(I) && "DbgInfoIntrinsic should have been dropped during VPlan construction"
) ? static_cast<void> (0) : __assert_fail ("!isa<DbgInfoIntrinsic>(I) && \"DbgInfoIntrinsic should have been dropped during VPlan construction\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4969, __PRETTY_FUNCTION__));
setDebugLocFromInst(Builder, &I);

Module *M = I.getParent()->getParent()->getParent();
auto *CI = cast<CallInst>(&I);

SmallVector<Type *, 4> Tys;
for (Value *ArgOperand : CI->arg_operands())
  Tys.push_back(ToVectorTy(ArgOperand->getType(), VF.getKnownMinValue()));

Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);

// The flag shows whether we use Intrinsic or a usual Call for vectorized
// version of the instruction.
// Is it beneficial to perform intrinsic call compared to lib call?
bool NeedToScalarize = false;
InstructionCost CallCost = Cost->getVectorCallCost(CI, VF, NeedToScalarize);
InstructionCost IntrinsicCost = ID ? Cost->getVectorIntrinsicCost(CI, VF) : 0;
bool UseVectorIntrinsic = ID && IntrinsicCost <= CallCost;
assert((UseVectorIntrinsic || !NeedToScalarize) &&(((UseVectorIntrinsic || !NeedToScalarize) && "Instruction should be scalarized elsewhere."
) ? static_cast<void> (0) : __assert_fail ("(UseVectorIntrinsic || !NeedToScalarize) && \"Instruction should be scalarized elsewhere.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4989, __PRETTY_FUNCTION__))
       "Instruction should be scalarized elsewhere.")(((UseVectorIntrinsic || !NeedToScalarize) && "Instruction should be scalarized elsewhere."
) ? static_cast<void> (0) : __assert_fail ("(UseVectorIntrinsic || !NeedToScalarize) && \"Instruction should be scalarized elsewhere.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4989, __PRETTY_FUNCTION__));
assert((IntrinsicCost.isValid() || CallCost.isValid()) &&(((IntrinsicCost.isValid() || CallCost.isValid()) && "Either the intrinsic cost or vector call cost must be valid"
) ? static_cast<void> (0) : __assert_fail ("(IntrinsicCost.isValid() || CallCost.isValid()) && \"Either the intrinsic cost or vector call cost must be valid\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4991, __PRETTY_FUNCTION__))
       "Either the intrinsic cost or vector call cost must be valid")(((IntrinsicCost.isValid() || CallCost.isValid()) && "Either the intrinsic cost or vector call cost must be valid"
) ? static_cast<void> (0) : __assert_fail ("(IntrinsicCost.isValid() || CallCost.isValid()) && \"Either the intrinsic cost or vector call cost must be valid\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4991, __PRETTY_FUNCTION__));

for (unsigned Part = 0; Part < UF; ++Part) {
  SmallVector<Value *, 4> Args;
  for (auto &I : enumerate(ArgOperands.operands())) {
    // Some intrinsics have a scalar argument - don't replace it with a
    // vector.
    Value *Arg;
    if (!UseVectorIntrinsic || !hasVectorInstrinsicScalarOpd(ID, I.index()))
      Arg = State.get(I.value(), Part);
    else
      Arg = State.get(I.value(), VPIteration(0, 0));
    Args.push_back(Arg);
  }

  Function *VectorF;
  if (UseVectorIntrinsic) {
    // Use vector version of the intrinsic.
    Type *TysForDecl[] = {CI->getType()};
    if (VF.isVector())
      TysForDecl[0] = VectorType::get(CI->getType()->getScalarType(), VF);
    VectorF = Intrinsic::getDeclaration(M, ID, TysForDecl);
    assert(VectorF && "Can't retrieve vector intrinsic.")((VectorF && "Can't retrieve vector intrinsic.") ? static_cast
<void> (0) : __assert_fail ("VectorF && \"Can't retrieve vector intrinsic.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 5013, __PRETTY_FUNCTION__));
  } else {
    // Use vector version of the function call.
    const VFShape Shape = VFShape::get(*CI, VF, false /*HasGlobalPred*/);
5017#ifndef NDEBUG
    assert(VFDatabase(*CI).getVectorizedFunction(Shape) != nullptr &&((VFDatabase(*CI).getVectorizedFunction(Shape) != nullptr &&
 "Can't create vector function.") ? static_cast<void> (
0) : __assert_fail ("VFDatabase(*CI).getVectorizedFunction(Shape) != nullptr && \"Can't create vector function.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 5019, __PRETTY_FUNCTION__))
           "Can't create vector function.")((VFDatabase(*CI).getVectorizedFunction(Shape) != nullptr &&
 "Can't create vector function.") ? static_cast<void> (
0) : __assert_fail ("VFDatabase(*CI).getVectorizedFunction(Shape) != nullptr && \"Can't create vector function.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 5019, __PRETTY_FUNCTION__));
5020#endif
      VectorF = VFDatabase(*CI).getVectorizedFunction(Shape);
  }
    SmallVector<OperandBundleDef, 1> OpBundles;
    CI->getOperandBundlesAsDefs(OpBundles);
    CallInst *V = Builder.CreateCall(VectorF, Args, OpBundles);

    if (isa<FPMathOperator>(V))
      V->copyFastMathFlags(CI);

    State.set(Def, V, Part);
    addMetadata(V, &I);
}
5033}

5035void InnerLoopVectorizer::widenSelectInstruction(SelectInst &I, VPValue *VPDef,
                                               VPUser &Operands,
                                               bool InvariantCond,
                                               VPTransformState &State) {
setDebugLocFromInst(Builder, &I);

// The condition can be loop invariant  but still defined inside the
// loop. This means that we can't just use the original 'cond' value.
// We have to take the 'vectorized' value and pick the first lane.
// Instcombine will make this a no-op.
auto *InvarCond = InvariantCond
                      ? State.get(Operands.getOperand(0), VPIteration(0, 0))
                      : nullptr;

for (unsigned Part = 0; Part < UF; ++Part) {
  Value *Cond =
      InvarCond ? InvarCond : State.get(Operands.getOperand(0), Part);
  Value *Op0 = State.get(Operands.getOperand(1), Part);
  Value *Op1 = State.get(Operands.getOperand(2), Part);
  Value *Sel = Builder.CreateSelect(Cond, Op0, Op1);
  State.set(VPDef, Sel, Part);
  addMetadata(Sel, &I);
}
5058}

5060void LoopVectorizationCostModel::collectLoopScalars(ElementCount VF) {
// We should not collect Scalars more than once per VF. Right now, this
// function is called from collectUniformsAndScalars(), which already does
// this check. Collecting Scalars for VF=1 does not make any sense.
assert(VF.isVector() && Scalars.find(VF) == Scalars.end() &&((VF.isVector() && Scalars.find(VF) == Scalars.end() &&
 "This function should not be visited twice for the same VF")
 ? static_cast<void> (0) : __assert_fail ("VF.isVector() && Scalars.find(VF) == Scalars.end() && \"This function should not be visited twice for the same VF\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 5065, __PRETTY_FUNCTION__))
       "This function should not be visited twice for the same VF")((VF.isVector() && Scalars.find(VF) == Scalars.end() &&
 "This function should not be visited twice for the same VF")
 ? static_cast<void> (0) : __assert_fail ("VF.isVector() && Scalars.find(VF) == Scalars.end() && \"This function should not be visited twice for the same VF\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 5065, __PRETTY_FUNCTION__));

SmallSetVector<Instruction *, 8> Worklist;

// These sets are used to seed the analysis with pointers used by memory
// accesses that will remain scalar.
SmallSetVector<Instruction *, 8> ScalarPtrs;
SmallPtrSet<Instruction *, 8> PossibleNonScalarPtrs;
auto *Latch = TheLoop->getLoopLatch();

// A helper that returns true if the use of Ptr by MemAccess will be scalar.
// The pointer operands of loads and stores will be scalar as long as the
// memory access is not a gather or scatter operation. The value operand of a
// store will remain scalar if the store is scalarized.
auto isScalarUse = [&](Instruction *MemAccess, Value *Ptr) {
  InstWidening WideningDecision = getWideningDecision(MemAccess, VF);
  assert(WideningDecision != CM_Unknown &&((WideningDecision != CM_Unknown && "Widening decision should be ready at this moment"
) ? static_cast<void> (0) : __assert_fail ("WideningDecision != CM_Unknown && \"Widening decision should be ready at this moment\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 5082, __PRETTY_FUNCTION__))
         "Widening decision should be ready at this moment")((WideningDecision != CM_Unknown && "Widening decision should be ready at this moment"
) ? static_cast<void> (0) : __assert_fail ("WideningDecision != CM_Unknown && \"Widening decision should be ready at this moment\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 5082, __PRETTY_FUNCTION__));
  if (auto *Store = dyn_cast<StoreInst>(MemAccess))
    if (Ptr == Store->getValueOperand())
      return WideningDecision == CM_Scalarize;
  assert(Ptr == getLoadStorePointerOperand(MemAccess) &&((Ptr == getLoadStorePointerOperand(MemAccess) && "Ptr is neither a value or pointer operand"
) ? static_cast<void> (0) : __assert_fail ("Ptr == getLoadStorePointerOperand(MemAccess) && \"Ptr is neither a value or pointer operand\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 5087, __PRETTY_FUNCTION__))
         "Ptr is neither a value or pointer operand")((Ptr == getLoadStorePointerOperand(MemAccess) && "Ptr is neither a value or pointer operand"
) ? static_cast<void> (0) : __assert_fail ("Ptr == getLoadStorePointerOperand(MemAccess) && \"Ptr is neither a value or pointer operand\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 5087, __PRETTY_FUNCTION__));
  return WideningDecision != CM_GatherScatter;
};

// A helper that returns true if the given value is a bitcast or
// getelementptr instruction contained in the loop.
auto isLoopVaryingBitCastOrGEP = [&](Value *V) {
  return ((isa<BitCastInst>(V) && V->getType()->isPointerTy()) ||
          isa<GetElementPtrInst>(V)) &&
         !TheLoop->isLoopInvariant(V);
};

auto isScalarPtrInduction = [&](Instruction *MemAccess, Value *Ptr) {
  if (!isa<PHINode>(Ptr) ||
      !Legal->getInductionVars().count(cast<PHINode>(Ptr)))
    return false;
  auto &Induction = Legal->getInductionVars()[cast<PHINode>(Ptr)];
  if (Induction.getKind() != InductionDescriptor::IK_PtrInduction)
    return false;
  return isScalarUse(MemAccess, Ptr);
};

// A helper that evaluates a memory access's use of a pointer. If the
// pointer is actually the pointer induction of a loop, it is being
// inserted into Worklist. If the use will be a scalar use, and the
// pointer is only used by memory accesses, we place the pointer in
// ScalarPtrs. Otherwise, the pointer is placed in PossibleNonScalarPtrs.
auto evaluatePtrUse = [&](Instruction *MemAccess, Value *Ptr) {
  if (isScalarPtrInduction(MemAccess, Ptr)) {
    Worklist.insert(cast<Instruction>(Ptr));
    Instruction *Update = cast<Instruction>(
        cast<PHINode>(Ptr)->getIncomingValueForBlock(Latch));
    Worklist.insert(Update);
    LLVM_DEBUG(dbgs() << "LV: Found new scalar instruction: " << *Ptrdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Found new scalar instruction: "
 << *Ptr << "\n"; } } while (false)
                      << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Found new scalar instruction: "
 << *Ptr << "\n"; } } while (false);
    LLVM_DEBUG(dbgs() << "LV: Found new scalar instruction: " << *Updatedo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Found new scalar instruction: "
 << *Update << "\n"; } } while (false)
                      << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Found new scalar instruction: "
 << *Update << "\n"; } } while (false);
    return;
  }
  // We only care about bitcast and getelementptr instructions contained in
  // the loop.
  if (!isLoopVaryingBitCastOrGEP(Ptr))
    return;

  // If the pointer has already been identified as scalar (e.g., if it was
  // also identified as uniform), there's nothing to do.
  auto *I = cast<Instruction>(Ptr);
  if (Worklist.count(I))
    return;

  // If the use of the pointer will be a scalar use, and all users of the
  // pointer are memory accesses, place the pointer in ScalarPtrs. Otherwise,
  // place the pointer in PossibleNonScalarPtrs.
  if (isScalarUse(MemAccess, Ptr) && llvm::all_of(I->users(), [&](User *U) {
        return isa<LoadInst>(U) || isa<StoreInst>(U);
      }))
    ScalarPtrs.insert(I);
  else
    PossibleNonScalarPtrs.insert(I);
};

// We seed the scalars analysis with three classes of instructions: (1)
// instructions marked uniform-after-vectorization and (2) bitcast,
// getelementptr and (pointer) phi instructions used by memory accesses
// requiring a scalar use.
//
// (1) Add to the worklist all instructions that have been identified as
// uniform-after-vectorization.
Worklist.insert(Uniforms[VF].begin(), Uniforms[VF].end());

// (2) Add to the worklist all bitcast and getelementptr instructions used by
// memory accesses requiring a scalar use. The pointer operands of loads and
// stores will be scalar as long as the memory accesses is not a gather or
// scatter operation. The value operand of a store will remain scalar if the
// store is scalarized.
for (auto *BB : TheLoop->blocks())
  for (auto &I : *BB) {
    if (auto *Load = dyn_cast<LoadInst>(&I)) {
      evaluatePtrUse(Load, Load->getPointerOperand());
    } else if (auto *Store = dyn_cast<StoreInst>(&I)) {
      evaluatePtrUse(Store, Store->getPointerOperand());
      evaluatePtrUse(Store, Store->getValueOperand());
    }
  }
for (auto *I : ScalarPtrs)
  if (!PossibleNonScalarPtrs.count(I)) {
    LLVM_DEBUG(dbgs() << "LV: Found scalar instruction: " << *I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Found scalar instruction: "
 << *I << "\n"; } } while (false);
    Worklist.insert(I);
  }

// Insert the forced scalars.
// FIXME: Currently widenPHIInstruction() often creates a dead vector
// induction variable when the PHI user is scalarized.
auto ForcedScalar = ForcedScalars.find(VF);
if (ForcedScalar != ForcedScalars.end())
  for (auto *I : ForcedScalar->second)
    Worklist.insert(I);

// Expand the worklist by looking through any bitcasts and getelementptr
// instructions we've already identified as scalar. This is similar to the
// expansion step in collectLoopUniforms(); however, here we're only
// expanding to include additional bitcasts and getelementptr instructions.
unsigned Idx = 0;
while (Idx != Worklist.size()) {
  Instruction *Dst = Worklist[Idx++];
  if (!isLoopVaryingBitCastOrGEP(Dst->getOperand(0)))
    continue;
  auto *Src = cast<Instruction>(Dst->getOperand(0));
  if (llvm::all_of(Src->users(), [&](User *U) -> bool {
        auto *J = cast<Instruction>(U);
        return !TheLoop->contains(J) || Worklist.count(J) ||
               ((isa<LoadInst>(J) || isa<StoreInst>(J)) &&
                isScalarUse(J, Src));
      })) {
    Worklist.insert(Src);
    LLVM_DEBUG(dbgs() << "LV: Found scalar instruction: " << *Src << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Found scalar instruction: "
 << *Src << "\n"; } } while (false);
  }
}

// An induction variable will remain scalar if all users of the induction
// variable and induction variable update remain scalar.
for (auto &Induction : Legal->getInductionVars()) {
  auto *Ind = Induction.first;
  auto *IndUpdate = cast<Instruction>(Ind->getIncomingValueForBlock(Latch));

  // If tail-folding is applied, the primary induction variable will be used
  // to feed a vector compare.
  if (Ind == Legal->getPrimaryInduction() && foldTailByMasking())
    continue;

  // Determine if all users of the induction variable are scalar after
  // vectorization.
  auto ScalarInd = llvm::all_of(Ind->users(), [&](User *U) -> bool {
    auto *I = cast<Instruction>(U);
    return I == IndUpdate || !TheLoop->contains(I) || Worklist.count(I);
  });
  if (!ScalarInd)
    continue;

  // Determine if all users of the induction variable update instruction are
  // scalar after vectorization.
  auto ScalarIndUpdate =
      llvm::all_of(IndUpdate->users(), [&](User *U) -> bool {
        auto *I = cast<Instruction>(U);
        return I == Ind || !TheLoop->contains(I) || Worklist.count(I);
      });
  if (!ScalarIndUpdate)
    continue;

  // The induction variable and its update instruction will remain scalar.
  Worklist.insert(Ind);
  Worklist.insert(IndUpdate);
  LLVM_DEBUG(dbgs() << "LV: Found scalar instruction: " << *Ind << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Found scalar instruction: "
 << *Ind << "\n"; } } while (false);
  LLVM_DEBUG(dbgs() << "LV: Found scalar instruction: " << *IndUpdatedo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Found scalar instruction: "
 << *IndUpdate << "\n"; } } while (false)
                    << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Found scalar instruction: "
 << *IndUpdate << "\n"; } } while (false);
}

Scalars[VF].insert(Worklist.begin(), Worklist.end());
5245}

5247bool LoopVectorizationCostModel::isScalarWithPredication(
  Instruction *I, ElementCount VF) const {
if (!blockNeedsPredication(I->getParent()))
  return false;
switch(I->getOpcode()) {
default:
  break;
case Instruction::Load:
case Instruction::Store: {
  if (!Legal->isMaskRequired(I))
    return false;
  auto *Ptr = getLoadStorePointerOperand(I);
  auto *Ty = getMemInstValueType(I);
  // We have already decided how to vectorize this instruction, get that
  // result.
  if (VF.isVector()) {
    InstWidening WideningDecision = getWideningDecision(I, VF);
    assert(WideningDecision != CM_Unknown &&((WideningDecision != CM_Unknown && "Widening decision should be ready at this moment"
) ? static_cast<void> (0) : __assert_fail ("WideningDecision != CM_Unknown && \"Widening decision should be ready at this moment\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 5265, __PRETTY_FUNCTION__))
           "Widening decision should be ready at this moment")((WideningDecision != CM_Unknown && "Widening decision should be ready at this moment"
) ? static_cast<void> (0) : __assert_fail ("WideningDecision != CM_Unknown && \"Widening decision should be ready at this moment\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 5265, __PRETTY_FUNCTION__));
    return WideningDecision == CM_Scalarize;
  }
  const Align Alignment = getLoadStoreAlignment(I);
  return isa<LoadInst>(I) ? !(isLegalMaskedLoad(Ty, Ptr, Alignment) ||
                              isLegalMaskedGather(Ty, Alignment))
                          : !(isLegalMaskedStore(Ty, Ptr, Alignment) ||
                              isLegalMaskedScatter(Ty, Alignment));
}
case Instruction::UDiv:
case Instruction::SDiv:
case Instruction::SRem:
case Instruction::URem:
  return mayDivideByZero(*I);
}
return false;
5281}

5283bool LoopVectorizationCostModel::interleavedAccessCanBeWidened(
  Instruction *I, ElementCount VF) {
assert(isAccessInterleaved(I) && "Expecting interleaved access.")((isAccessInterleaved(I) && "Expecting interleaved access."
) ? static_cast<void> (0) : __assert_fail ("isAccessInterleaved(I) && \"Expecting interleaved access.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 5285, __PRETTY_FUNCTION__));
assert(getWideningDecision(I, VF) == CM_Unknown &&((getWideningDecision(I, VF) == CM_Unknown && "Decision should not be set yet."
) ? static_cast<void> (0) : __assert_fail ("getWideningDecision(I, VF) == CM_Unknown && \"Decision should not be set yet.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 5287, __PRETTY_FUNCTION__))
       "Decision should not be set yet.")((getWideningDecision(I, VF) == CM_Unknown && "Decision should not be set yet."
) ? static_cast<void> (0) : __assert_fail ("getWideningDecision(I, VF) == CM_Unknown && \"Decision should not be set yet.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 5287, __PRETTY_FUNCTION__));
auto *Group = getInterleavedAccessGroup(I);
assert(Group && "Must have a group.")((Group && "Must have a group.") ? static_cast<void
> (0) : __assert_fail ("Group && \"Must have a group.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 5289, __PRETTY_FUNCTION__));

// If the instruction's allocated size doesn't equal it's type size, it
// requires padding and will be scalarized.
auto &DL = I->getModule()->getDataLayout();
auto *ScalarTy = getMemInstValueType(I);
if (hasIrregularType(ScalarTy, DL))
  return false;

// Check if masking is required.
// A Group may need masking for one of two reasons: it resides in a block that
// needs predication, or it was decided to use masking to deal with gaps.
bool PredicatedAccessRequiresMasking =
    Legal->blockNeedsPredication(I->getParent()) && Legal->isMaskRequired(I);
bool AccessWithGapsRequiresMasking =
    Group->requiresScalarEpilogue() && !isScalarEpilogueAllowed();
if (!PredicatedAccessRequiresMasking && !AccessWithGapsRequiresMasking)
  return true;

// If masked interleaving is required, we expect that the user/target had
// enabled it, because otherwise it either wouldn't have been created or
// it should have been invalidated by the CostModel.
assert(useMaskedInterleavedAccesses(TTI) &&((useMaskedInterleavedAccesses(TTI) && "Masked interleave-groups for predicated accesses are not enabled."
) ? static_cast<void> (0) : __assert_fail ("useMaskedInterleavedAccesses(TTI) && \"Masked interleave-groups for predicated accesses are not enabled.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 5312, __PRETTY_FUNCTION__))
       "Masked interleave-groups for predicated accesses are not enabled.")((useMaskedInterleavedAccesses(TTI) && "Masked interleave-groups for predicated accesses are not enabled."
) ? static_cast<void> (0) : __assert_fail ("useMaskedInterleavedAccesses(TTI) && \"Masked interleave-groups for predicated accesses are not enabled.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 5312, __PRETTY_FUNCTION__));

auto *Ty = getMemInstValueType(I);
const Align Alignment = getLoadStoreAlignment(I);
return isa<LoadInst>(I) ? TTI.isLegalMaskedLoad(Ty, Alignment)
                        : TTI.isLegalMaskedStore(Ty, Alignment);
5318}

5320bool LoopVectorizationCostModel::memoryInstructionCanBeWidened(
  Instruction *I, ElementCount VF) {
// Get and ensure we have a valid memory instruction.
LoadInst *LI = dyn_cast<LoadInst>(I);
StoreInst *SI = dyn_cast<StoreInst>(I);
assert((LI || SI) && "Invalid memory instruction")(((LI || SI) && "Invalid memory instruction") ? static_cast
<void> (0) : __assert_fail ("(LI || SI) && \"Invalid memory instruction\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 5325, __PRETTY_FUNCTION__));

auto *Ptr = getLoadStorePointerOperand(I);

// In order to be widened, the pointer should be consecutive, first of all.
if (!Legal->isConsecutivePtr(Ptr))
  return false;

// If the instruction is a store located in a predicated block, it will be
// scalarized.
if (isScalarWithPredication(I))
  return false;

// If the instruction's allocated size doesn't equal it's type size, it
// requires padding and will be scalarized.
auto &DL = I->getModule()->getDataLayout();
auto *ScalarTy = LI ? LI->getType() : SI->getValueOperand()->getType();
if (hasIrregularType(ScalarTy, DL))
  return false;

return true;
5346}

5348void LoopVectorizationCostModel::collectLoopUniforms(ElementCount VF) {
// We should not collect Uniforms more than once per VF. Right now,
// this function is called from collectUniformsAndScalars(), which
// already does this check. Collecting Uniforms for VF=1 does not make any
// sense.

assert(VF.isVector() && Uniforms.find(VF) == Uniforms.end() &&((VF.isVector() && Uniforms.find(VF) == Uniforms.end(
) && "This function should not be visited twice for the same VF"
) ? static_cast<void> (0) : __assert_fail ("VF.isVector() && Uniforms.find(VF) == Uniforms.end() && \"This function should not be visited twice for the same VF\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 5355, __PRETTY_FUNCTION__))
       "This function should not be visited twice for the same VF")((VF.isVector() && Uniforms.find(VF) == Uniforms.end(
) && "This function should not be visited twice for the same VF"
) ? static_cast<void> (0) : __assert_fail ("VF.isVector() && Uniforms.find(VF) == Uniforms.end() && \"This function should not be visited twice for the same VF\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 5355, __PRETTY_FUNCTION__));

// Visit the list of Uniforms. If we'll not find any uniform value, we'll
// not analyze again.  Uniforms.count(VF) will return 1.
Uniforms[VF].clear();

// We now know that the loop is vectorizable!
// Collect instructions inside the loop that will remain uniform after
// vectorization.

// Global values, params and instructions outside of current loop are out of
// scope.
auto isOutOfScope = [&](Value *V) -> bool {
  Instruction *I = dyn_cast<Instruction>(V);
  return (!I || !TheLoop->contains(I));
};

SetVector<Instruction *> Worklist;
BasicBlock *Latch = TheLoop->getLoopLatch();

// Instructions that are scalar with predication must not be considered
// uniform after vectorization, because that would create an erroneous
// replicating region where only a single instance out of VF should be formed.
// TODO: optimize such seldom cases if found important, see PR40816.
auto addToWorklistIfAllowed = [&](Instruction *I) -> void {
  if (isOutOfScope(I)) {
    LLVM_DEBUG(dbgs() << "LV: Found not uniform due to scope: "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Found not uniform due to scope: "
 << *I << "\n"; } } while (false)
                      << *I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Found not uniform due to scope: "
 << *I << "\n"; } } while (false);
    return;
  }
  if (isScalarWithPredication(I, VF)) {
    LLVM_DEBUG(dbgs() << "LV: Found not uniform being ScalarWithPredication: "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Found not uniform being ScalarWithPredication: "
 << *I << "\n"; } } while (false)
                      << *I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Found not uniform being ScalarWithPredication: "
 << *I << "\n"; } } while (false);
    return;
  }
  LLVM_DEBUG(dbgs() << "LV: Found uniform instruction: " << *I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Found uniform instruction: "
 << *I << "\n"; } } while (false);
  Worklist.insert(I);
};

// Start with the conditional branch. If the branch condition is an
// instruction contained in the loop that is only used by the branch, it is
// uniform.
auto *Cmp = dyn_cast<Instruction>(Latch->getTerminator()->getOperand(0));
if (Cmp && TheLoop->contains(Cmp) && Cmp->hasOneUse())
  addToWorklistIfAllowed(Cmp);

auto isUniformDecision = [&](Instruction *I, ElementCount VF) {
  InstWidening WideningDecision = getWideningDecision(I, VF);
  assert(WideningDecision != CM_Unknown &&((WideningDecision != CM_Unknown && "Widening decision should be ready at this moment"
) ? static_cast<void> (0) : __assert_fail ("WideningDecision != CM_Unknown && \"Widening decision should be ready at this moment\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 5404, __PRETTY_FUNCTION__))
         "Widening decision should be ready at this moment")((WideningDecision != CM_Unknown && "Widening decision should be ready at this moment"
) ? static_cast<void> (0) : __assert_fail ("WideningDecision != CM_Unknown && \"Widening decision should be ready at this moment\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 5404, __PRETTY_FUNCTION__));

  // A uniform memory op is itself uniform.  We exclude uniform stores
  // here as they demand the last lane, not the first one.
  if (isa<LoadInst>(I) && Legal->isUniformMemOp(*I)) {
    assert(WideningDecision == CM_Scalarize)((WideningDecision == CM_Scalarize) ? static_cast<void>
 (0) : __assert_fail ("WideningDecision == CM_Scalarize", "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 5409, __PRETTY_FUNCTION__));
    return true;
  }

  return (WideningDecision == CM_Widen ||
          WideningDecision == CM_Widen_Reverse ||
          WideningDecision == CM_Interleave);
};


// Returns true if Ptr is the pointer operand of a memory access instruction
// I, and I is known to not require scalarization.
auto isVectorizedMemAccessUse = [&](Instruction *I, Value *Ptr) -> bool {
  return getLoadStorePointerOperand(I) == Ptr && isUniformDecision(I, VF);
};

// Holds a list of values which are known to have at least one uniform use.
// Note that there may be other uses which aren't uniform.  A "uniform use"
// here is something which only demands lane 0 of the unrolled iterations;
// it does not imply that all lanes produce the same value (e.g. this is not
// the usual meaning of uniform)
SetVector<Value *> HasUniformUse;

// Scan the loop for instructions which are either a) known to have only
// lane 0 demanded or b) are uses which demand only lane 0 of their operand.
for (auto *BB : TheLoop->blocks())
  for (auto &I : *BB) {
    // If there's no pointer operand, there's nothing to do.
    auto *Ptr = getLoadStorePointerOperand(&I);
    if (!Ptr)
      continue;

    // A uniform memory op is itself uniform.  We exclude uniform stores
    // here as they demand the last lane, not the first one.
    if (isa<LoadInst>(I) && Legal->isUniformMemOp(I))
      addToWorklistIfAllowed(&I);

    if (isUniformDecision(&I, VF)) {
      assert(isVectorizedMemAccessUse(&I, Ptr) && "consistency check")((isVectorizedMemAccessUse(&I, Ptr) && "consistency check"
) ? static_cast<void> (0) : __assert_fail ("isVectorizedMemAccessUse(&I, Ptr) && \"consistency check\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 5447, __PRETTY_FUNCTION__));
      HasUniformUse.insert(Ptr);
    }
  }

// Add to the worklist any operands which have *only* uniform (e.g. lane 0
// demanding) users.  Since loops are assumed to be in LCSSA form, this
// disallows uses outside the loop as well.
for (auto *V : HasUniformUse) {
  if (isOutOfScope(V))
    continue;
  auto *I = cast<Instruction>(V);
  auto UsersAreMemAccesses =
    llvm::all_of(I->users(), [&](User *U) -> bool {
      return isVectorizedMemAccessUse(cast<Instruction>(U), V);
    });
  if (UsersAreMemAccesses)
    addToWorklistIfAllowed(I);
}

// Expand Worklist in topological order: whenever a new instruction
// is added , its users should be already inside Worklist.  It ensures
// a uniform instruction will only be used by uniform instructions.
unsigned idx = 0;
while (idx != Worklist.size()) {
  Instruction *I = Worklist[idx++];

  for (auto OV : I->operand_values()) {
    // isOutOfScope operands cannot be uniform instructions.
    if (isOutOfScope(OV))
      continue;
    // First order recurrence Phi's should typically be considered
    // non-uniform.
    auto *OP = dyn_cast<PHINode>(OV);
    if (OP && Legal->isFirstOrderRecurrence(OP))
      continue;
    // If all the users of the operand are uniform, then add the
    // operand into the uniform worklist.
    auto *OI = cast<Instruction>(OV);
    if (llvm::all_of(OI->users(), [&](User *U) -> bool {
          auto *J = cast<Instruction>(U);
          return Worklist.count(J) || isVectorizedMemAccessUse(J, OI);
        }))
      addToWorklistIfAllowed(OI);
  }
}

// For an instruction to be added into Worklist above, all its users inside
// the loop should also be in Worklist. However, this condition cannot be
// true for phi nodes that form a cyclic dependence. We must process phi
// nodes separately. An induction variable will remain uniform if all users
// of the induction variable and induction variable update remain uniform.
// The code below handles both pointer and non-pointer induction variables.
for (auto &Induction : Legal->getInductionVars()) {
  auto *Ind = Induction.first;
  auto *IndUpdate = cast<Instruction>(Ind->getIncomingValueForBlock(Latch));

  // Determine if all users of the induction variable are uniform after
  // vectorization.
  auto UniformInd = llvm::all_of(Ind->users(), [&](User *U) -> bool {
    auto *I = cast<Instruction>(U);
    return I == IndUpdate || !TheLoop->contains(I) || Worklist.count(I) ||
           isVectorizedMemAccessUse(I, Ind);
  });
  if (!UniformInd)
    continue;

  // Determine if all users of the induction variable update instruction are
  // uniform after vectorization.
  auto UniformIndUpdate =
      llvm::all_of(IndUpdate->users(), [&](User *U) -> bool {
        auto *I = cast<Instruction>(U);
        return I == Ind || !TheLoop->contains(I) || Worklist.count(I) ||
               isVectorizedMemAccessUse(I, IndUpdate);
      });
  if (!UniformIndUpdate)
    continue;

  // The induction variable and its update instruction will remain uniform.
  addToWorklistIfAllowed(Ind);
  addToWorklistIfAllowed(IndUpdate);
}

Uniforms[VF].insert(Worklist.begin(), Worklist.end());
5531}

5533bool LoopVectorizationCostModel::runtimeChecksRequired() {
LLVM_DEBUG(dbgs() << "LV: Performing code size checks.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Performing code size checks.\n"
; } } while (false);

if (Legal->getRuntimePointerChecking()->Need) {
  reportVectorizationFailure("Runtime ptr check is required with -Os/-Oz",
      "runtime pointer checks needed. Enable vectorization of this "
      "loop with '#pragma clang loop vectorize(enable)' when "
      "compiling with -Os/-Oz",
      "CantVersionLoopWithOptForSize", ORE, TheLoop);
  return true;
}

if (!PSE.getUnionPredicate().getPredicates().empty()) {
  reportVectorizationFailure("Runtime SCEV check is required with -Os/-Oz",
      "runtime SCEV checks needed. Enable vectorization of this "
      "loop with '#pragma clang loop vectorize(enable)' when "
      "compiling with -Os/-Oz",
      "CantVersionLoopWithOptForSize", ORE, TheLoop);
  return true;
}

// FIXME: Avoid specializing for stride==1 instead of bailing out.
if (!Legal->getLAI()->getSymbolicStrides().empty()) {
  reportVectorizationFailure("Runtime stride check for small trip count",
      "runtime stride == 1 checks needed. Enable vectorization of "
      "this loop without such check by compiling with -Os/-Oz",
      "CantVersionLoopWithOptForSize", ORE, TheLoop);
  return true;
}

return false;
5564}

5566Optional<ElementCount>
5567LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
if (Legal->getRuntimePointerChecking()->Need && TTI.hasBranchDivergence()) {
  // TODO: It may by useful to do since it's still likely to be dynamically
  // uniform if the target can skip.
  reportVectorizationFailure(
      "Not inserting runtime ptr check for divergent target",
      "runtime pointer checks needed. Not enabled for divergent target",
      "CantVersionLoopWithDivergentTarget", ORE, TheLoop);
  return None;
}

unsigned TC = PSE.getSE()->getSmallConstantTripCount(TheLoop);
LLVM_DEBUG(dbgs() << "LV: Found trip count: " << TC << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Found trip count: "
 << TC << '\n'; } } while (false);
if (TC == 1) {
  reportVectorizationFailure("Single iteration (non) loop",
      "loop trip count is one, irrelevant for vectorization",
      "SingleIterationLoop", ORE, TheLoop);
  return None;
}

switch (ScalarEpilogueStatus) {
case CM_ScalarEpilogueAllowed:
  return computeFeasibleMaxVF(TC, UserVF);
case CM_ScalarEpilogueNotAllowedUsePredicate:
  LLVM_FALLTHROUGH[[gnu::fallthrough]];
case CM_ScalarEpilogueNotNeededUsePredicate:
  LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: vector predicate hint/switch found.\n"
 << "LV: Not allowing scalar epilogue, creating predicated "
 << "vector loop.\n"; } } while (false)
      dbgs() << "LV: vector predicate hint/switch found.\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: vector predicate hint/switch found.\n"
 << "LV: Not allowing scalar epilogue, creating predicated "
 << "vector loop.\n"; } } while (false)
             << "LV: Not allowing scalar epilogue, creating predicated "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: vector predicate hint/switch found.\n"
 << "LV: Not allowing scalar epilogue, creating predicated "
 << "vector loop.\n"; } } while (false)
             << "vector loop.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: vector predicate hint/switch found.\n"
 << "LV: Not allowing scalar epilogue, creating predicated "
 << "vector loop.\n"; } } while (false);
  break;
case CM_ScalarEpilogueNotAllowedLowTripLoop:
  // fallthrough as a special case of OptForSize
case CM_ScalarEpilogueNotAllowedOptSize:
  if (ScalarEpilogueStatus == CM_ScalarEpilogueNotAllowedOptSize)
    LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Not allowing scalar epilogue due to -Os/-Oz.\n"
; } } while (false)
        dbgs() << "LV: Not allowing scalar epilogue due to -Os/-Oz.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Not allowing scalar epilogue due to -Os/-Oz.\n"
; } } while (false);
  else
    LLVM_DEBUG(dbgs() << "LV: Not allowing scalar epilogue due to low trip "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Not allowing scalar epilogue due to low trip "
 << "count.\n"; } } while (false)
                      << "count.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Not allowing scalar epilogue due to low trip "
 << "count.\n"; } } while (false);

  // Bail if runtime checks are required, which are not good when optimising
  // for size.
  if (runtimeChecksRequired())
    return None;

  break;
}

// The only loops we can vectorize without a scalar epilogue, are loops with
// a bottom-test and a single exiting block. We'd have to handle the fact
// that not every instruction executes on the last iteration.  This will
// require a lane mask which varies through the vector loop body.  (TODO)
if (TheLoop->getExitingBlock() != TheLoop->getLoopLatch()) {
  // If there was a tail-folding hint/switch, but we can't fold the tail by
  // masking, fallback to a vectorization with a scalar epilogue.
  if (ScalarEpilogueStatus == CM_ScalarEpilogueNotNeededUsePredicate) {
    LLVM_DEBUG(dbgs() << "LV: Cannot fold tail by masking: vectorize with a "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Cannot fold tail by masking: vectorize with a "
 "scalar epilogue instead.\n"; } } while (false)
                         "scalar epilogue instead.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Cannot fold tail by masking: vectorize with a "
 "scalar epilogue instead.\n"; } } while (false);
    ScalarEpilogueStatus = CM_ScalarEpilogueAllowed;
    return computeFeasibleMaxVF(TC, UserVF);
  }
  return None;
}

// Now try the tail folding

// Invalidate interleave groups that require an epilogue if we can't mask
// the interleave-group.
if (!useMaskedInterleavedAccesses(TTI)) {
  assert(WideningDecisions.empty() && Uniforms.empty() && Scalars.empty() &&((WideningDecisions.empty() && Uniforms.empty() &&
 Scalars.empty() && "No decisions should have been taken at this point"
) ? static_cast<void> (0) : __assert_fail ("WideningDecisions.empty() && Uniforms.empty() && Scalars.empty() && \"No decisions should have been taken at this point\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 5638, __PRETTY_FUNCTION__))
         "No decisions should have been taken at this point")((WideningDecisions.empty() && Uniforms.empty() &&
 Scalars.empty() && "No decisions should have been taken at this point"
) ? static_cast<void> (0) : __assert_fail ("WideningDecisions.empty() && Uniforms.empty() && Scalars.empty() && \"No decisions should have been taken at this point\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 5638, __PRETTY_FUNCTION__));
  // Note: There is no need to invalidate any cost modeling decisions here, as
  // non where taken so far.
  InterleaveInfo.invalidateGroupsRequiringScalarEpilogue();
}

ElementCount MaxVF = computeFeasibleMaxVF(TC, UserVF);
assert(!MaxVF.isScalable() &&((!MaxVF.isScalable() && "Scalable vectors do not yet support tail folding"
) ? static_cast<void> (0) : __assert_fail ("!MaxVF.isScalable() && \"Scalable vectors do not yet support tail folding\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 5646, __PRETTY_FUNCTION__))
       "Scalable vectors do not yet support tail folding")((!MaxVF.isScalable() && "Scalable vectors do not yet support tail folding"
) ? static_cast<void> (0) : __assert_fail ("!MaxVF.isScalable() && \"Scalable vectors do not yet support tail folding\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 5646, __PRETTY_FUNCTION__));
assert((UserVF.isNonZero() || isPowerOf2_32(MaxVF.getFixedValue())) &&(((UserVF.isNonZero() || isPowerOf2_32(MaxVF.getFixedValue())
) && "MaxVF must be a power of 2") ? static_cast<void
> (0) : __assert_fail ("(UserVF.isNonZero() || isPowerOf2_32(MaxVF.getFixedValue())) && \"MaxVF must be a power of 2\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 5648, __PRETTY_FUNCTION__))
       "MaxVF must be a power of 2")(((UserVF.isNonZero() || isPowerOf2_32(MaxVF.getFixedValue())
) && "MaxVF must be a power of 2") ? static_cast<void
> (0) : __assert_fail ("(UserVF.isNonZero() || isPowerOf2_32(MaxVF.getFixedValue())) && \"MaxVF must be a power of 2\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 5648, __PRETTY_FUNCTION__));
unsigned MaxVFtimesIC =
    UserIC ? MaxVF.getFixedValue() * UserIC : MaxVF.getFixedValue();
// Avoid tail folding if the trip count is known to be a multiple of any VF we
// chose.
ScalarEvolution *SE = PSE.getSE();
const SCEV *BackedgeTakenCount = PSE.getBackedgeTakenCount();
const SCEV *ExitCount = SE->getAddExpr(
    BackedgeTakenCount, SE->getOne(BackedgeTakenCount->getType()));
const SCEV *Rem = SE->getURemExpr(
    SE->applyLoopGuards(ExitCount, TheLoop),
    SE->getConstant(BackedgeTakenCount->getType(), MaxVFtimesIC));
if (Rem->isZero()) {
  // Accept MaxVF if we do not have a tail.
  LLVM_DEBUG(dbgs() << "LV: No tail will remain for any chosen VF.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: No tail will remain for any chosen VF.\n"
; } } while (false);
  return MaxVF;
}

// If we don't know the precise trip count, or if the trip count that we
// found modulo the vectorization factor is not zero, try to fold the tail
// by masking.
// FIXME: look for a smaller MaxVF that does divide TC rather than masking.
if (Legal->prepareToFoldTailByMasking()) {
  FoldTailByMasking = true;
  return MaxVF;
}

// If there was a tail-folding hint/switch, but we can't fold the tail by
// masking, fallback to a vectorization with a scalar epilogue.
if (ScalarEpilogueStatus == CM_ScalarEpilogueNotNeededUsePredicate) {
  LLVM_DEBUG(dbgs() << "LV: Cannot fold tail by masking: vectorize with a "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Cannot fold tail by masking: vectorize with a "
 "scalar epilogue instead.\n"; } } while (false)
                       "scalar epilogue instead.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Cannot fold tail by masking: vectorize with a "
 "scalar epilogue instead.\n"; } } while (false);
  ScalarEpilogueStatus = CM_ScalarEpilogueAllowed;
  return MaxVF;
}

if (ScalarEpilogueStatus == CM_ScalarEpilogueNotAllowedUsePredicate) {
  LLVM_DEBUG(dbgs() << "LV: Can't fold tail by masking: don't vectorize\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Can't fold tail by masking: don't vectorize\n"
; } } while (false);
  return None;
}

if (TC == 0) {
  reportVectorizationFailure(
      "Unable to calculate the loop count due to complex control flow",
      "unable to calculate the loop count due to complex control flow",
      "UnknownLoopCountComplexCFG", ORE, TheLoop);
  return None;
}

reportVectorizationFailure(
    "Cannot optimize for size and vectorize at the same time.",
    "cannot optimize for size and vectorize at the same time. "
    "Enable vectorization of this loop with '#pragma clang loop "
    "vectorize(enable)' when compiling with -Os/-Oz",
    "NoTailLoopWithOptForSize", ORE, TheLoop);
return None;
5704}

5706ElementCount
5707LoopVectorizationCostModel::computeFeasibleMaxVF(unsigned ConstTripCount,
                                               ElementCount UserVF) {
bool IgnoreScalableUserVF = UserVF.isScalable() &&
                            !TTI.supportsScalableVectors() &&
                            !ForceTargetSupportsScalableVectors;
if (IgnoreScalableUserVF) {
  LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Ignoring VF=" <<
 UserVF << " because target does not support scalable vectors.\n"
; } } while (false)
      dbgs() << "LV: Ignoring VF=" << UserVFdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Ignoring VF=" <<
 UserVF << " because target does not support scalable vectors.\n"
; } } while (false)
             << " because target does not support scalable vectors.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Ignoring VF=" <<
 UserVF << " because target does not support scalable vectors.\n"
; } } while (false);
  ORE->emit([&]() {
    return OptimizationRemarkAnalysis(DEBUG_TYPE"loop-vectorize", "IgnoreScalableUserVF",
                                      TheLoop->getStartLoc(),
                                      TheLoop->getHeader())
           << "Ignoring VF=" << ore::NV("UserVF", UserVF)
           << " because target does not support scalable vectors.";
  });
}

// Beyond this point two scenarios are handled. If UserVF isn't specified
// then a suitable VF is chosen. If UserVF is specified and there are
// dependencies, check if it's legal. However, if a UserVF is specified and
// there are no dependencies, then there's nothing to do.
if (UserVF.isNonZero() && !IgnoreScalableUserVF) {
  if (!canVectorizeReductions(UserVF)) {
    reportVectorizationFailure(
        "LV: Scalable vectorization not supported for the reduction "
        "operations found in this loop. Using fixed-width "
        "vectorization instead.",
        "Scalable vectorization not supported for the reduction operations "
        "found in this loop. Using fixed-width vectorization instead.",
        "ScalableVFUnfeasible", ORE, TheLoop);
    return computeFeasibleMaxVF(
        ConstTripCount, ElementCount::getFixed(UserVF.getKnownMinValue()));
  }

  if (Legal->isSafeForAnyVectorWidth())
    return UserVF;
}

MinBWs = computeMinimumValueSizes(TheLoop->getBlocks(), *DB, &TTI);
unsigned SmallestType, WidestType;
std::tie(SmallestType, WidestType) = getSmallestAndWidestTypes();
unsigned WidestRegister =
    TTI.getRegisterBitWidth(TargetTransformInfo::RGK_FixedWidthVector)
        .getFixedSize();

// Get the maximum safe dependence distance in bits computed by LAA.
// It is computed by MaxVF * sizeOf(type) * 8, where type is taken from
// the memory accesses that is most restrictive (involved in the smallest
// dependence distance).
unsigned MaxSafeVectorWidthInBits = Legal->getMaxSafeVectorWidthInBits();

// If the user vectorization factor is legally unsafe, clamp it to a safe
// value. Otherwise, return as is.
if (UserVF.isNonZero() && !IgnoreScalableUserVF) {
  unsigned MaxSafeElements =
      PowerOf2Floor(MaxSafeVectorWidthInBits / WidestType);
  ElementCount MaxSafeVF = ElementCount::getFixed(MaxSafeElements);

  if (UserVF.isScalable()) {
    Optional<unsigned> MaxVScale = TTI.getMaxVScale();

    // Scale VF by vscale before checking if it's safe.
    MaxSafeVF = ElementCount::getScalable(
        MaxVScale ? (MaxSafeElements / MaxVScale.getValue()) : 0);

    if (MaxSafeVF.isZero()) {
      // The dependence distance is too small to use scalable vectors,
      // fallback on fixed.
      LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Max legal vector width too small, scalable vectorization "
 "unfeasible. Using fixed-width vectorization instead.\n"; } }
 while (false)
          dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Max legal vector width too small, scalable vectorization "
 "unfeasible. Using fixed-width vectorization instead.\n"; } }
 while (false)
          << "LV: Max legal vector width too small, scalable vectorization "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Max legal vector width too small, scalable vectorization "
 "unfeasible. Using fixed-width vectorization instead.\n"; } }
 while (false)
             "unfeasible. Using fixed-width vectorization instead.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Max legal vector width too small, scalable vectorization "
 "unfeasible. Using fixed-width vectorization instead.\n"; } }
 while (false);
      ORE->emit([&]() {
        return OptimizationRemarkAnalysis(DEBUG_TYPE"loop-vectorize", "ScalableVFUnfeasible",
                                          TheLoop->getStartLoc(),
                                          TheLoop->getHeader())
               << "Max legal vector width too small, scalable vectorization "
               << "unfeasible. Using fixed-width vectorization instead.";
      });
      return computeFeasibleMaxVF(
          ConstTripCount, ElementCount::getFixed(UserVF.getKnownMinValue()));
    }
  }

  LLVM_DEBUG(dbgs() << "LV: The max safe VF is: " << MaxSafeVF << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: The max safe VF is: "
 << MaxSafeVF << ".\n"; } } while (false);

  if (ElementCount::isKnownLE(UserVF, MaxSafeVF))
    return UserVF;

  LLVM_DEBUG(dbgs() << "LV: User VF=" << UserVFdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: User VF=" <<
 UserVF << " is unsafe, clamping to max safe VF=" <<
 MaxSafeVF << ".\n"; } } while (false)
                    << " is unsafe, clamping to max safe VF=" << MaxSafeVFdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: User VF=" <<
 UserVF << " is unsafe, clamping to max safe VF=" <<
 MaxSafeVF << ".\n"; } } while (false)
                    << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: User VF=" <<
 UserVF << " is unsafe, clamping to max safe VF=" <<
 MaxSafeVF << ".\n"; } } while (false);
  ORE->emit([&]() {
    return OptimizationRemarkAnalysis(DEBUG_TYPE"loop-vectorize", "VectorizationFactor",
                                      TheLoop->getStartLoc(),
                                      TheLoop->getHeader())
           << "User-specified vectorization factor "
           << ore::NV("UserVectorizationFactor", UserVF)
           << " is unsafe, clamping to maximum safe vectorization factor "
           << ore::NV("VectorizationFactor", MaxSafeVF);
  });
  return MaxSafeVF;
}

WidestRegister = std::min(WidestRegister, MaxSafeVectorWidthInBits);

// Ensure MaxVF is a power of 2; the dependence distance bound may not be.
// Note that both WidestRegister and WidestType may not be a powers of 2.
auto MaxVectorSize =
    ElementCount::getFixed(PowerOf2Floor(WidestRegister / WidestType));

LLVM_DEBUG(dbgs() << "LV: The Smallest and Widest types: " << SmallestTypedo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: The Smallest and Widest types: "
 << SmallestType << " / " << WidestType <<
 " bits.\n"; } } while (false)
                  << " / " << WidestType << " bits.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: The Smallest and Widest types: "
 << SmallestType << " / " << WidestType <<
 " bits.\n"; } } while (false);
LLVM_DEBUG(dbgs() << "LV: The Widest register safe to use is: "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: The Widest register safe to use is: "
 << WidestRegister << " bits.\n"; } } while (false
)
                  << WidestRegister << " bits.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: The Widest register safe to use is: "
 << WidestRegister << " bits.\n"; } } while (false
);

assert(MaxVectorSize.getFixedValue() <= WidestRegister &&((MaxVectorSize.getFixedValue() <= WidestRegister &&
 "Did not expect to pack so many elements" " into one vector!"
) ? static_cast<void> (0) : __assert_fail ("MaxVectorSize.getFixedValue() <= WidestRegister && \"Did not expect to pack so many elements\" \" into one vector!\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 5826, __PRETTY_FUNCTION__))
       "Did not expect to pack so many elements"((MaxVectorSize.getFixedValue() <= WidestRegister &&
 "Did not expect to pack so many elements" " into one vector!"
) ? static_cast<void> (0) : __assert_fail ("MaxVectorSize.getFixedValue() <= WidestRegister && \"Did not expect to pack so many elements\" \" into one vector!\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 5826, __PRETTY_FUNCTION__))
       " into one vector!")((MaxVectorSize.getFixedValue() <= WidestRegister &&
 "Did not expect to pack so many elements" " into one vector!"
) ? static_cast<void> (0) : __assert_fail ("MaxVectorSize.getFixedValue() <= WidestRegister && \"Did not expect to pack so many elements\" \" into one vector!\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 5826, __PRETTY_FUNCTION__));
if (MaxVectorSize.getFixedValue() == 0) {
  LLVM_DEBUG(dbgs() << "LV: The target has no vector registers.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: The target has no vector registers.\n"
; } } while (false);
  return ElementCount::getFixed(1);
} else if (ConstTripCount && ConstTripCount < MaxVectorSize.getFixedValue() &&
           isPowerOf2_32(ConstTripCount)) {
  // We need to clamp the VF to be the ConstTripCount. There is no point in
  // choosing a higher viable VF as done in the loop below.
  LLVM_DEBUG(dbgs() << "LV: Clamping the MaxVF to the constant trip count: "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Clamping the MaxVF to the constant trip count: "
 << ConstTripCount << "\n"; } } while (false)
                    << ConstTripCount << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Clamping the MaxVF to the constant trip count: "
 << ConstTripCount << "\n"; } } while (false);
  return ElementCount::getFixed(ConstTripCount);
}

ElementCount MaxVF = MaxVectorSize;
if (TTI.shouldMaximizeVectorBandwidth(!isScalarEpilogueAllowed()) ||
    (MaximizeBandwidth && isScalarEpilogueAllowed())) {
  // Collect all viable vectorization factors larger than the default MaxVF
  // (i.e. MaxVectorSize).
  SmallVector<ElementCount, 8> VFs;
  auto MaxVectorSizeMaxBW =
      ElementCount::getFixed(WidestRegister / SmallestType);
  for (ElementCount VS = MaxVectorSize * 2;
       ElementCount::isKnownLE(VS, MaxVectorSizeMaxBW); VS *= 2)
    VFs.push_back(VS);

  // For each VF calculate its register usage.
  auto RUs = calculateRegisterUsage(VFs);

  // Select the largest VF which doesn't require more registers than existing
  // ones.
  for (int i = RUs.size() - 1; i >= 0; --i) {
    bool Selected = true;
    for (auto &pair : RUs[i].MaxLocalUsers) {
      unsigned TargetNumRegisters = TTI.getNumberOfRegisters(pair.first);
      if (pair.second > TargetNumRegisters)
        Selected = false;
    }
    if (Selected) {
      MaxVF = VFs[i];
      break;
    }
  }
  if (ElementCount MinVF =
          TTI.getMinimumVF(SmallestType, /*IsScalable=*/false)) {
    if (ElementCount::isKnownLT(MaxVF, MinVF)) {
      LLVM_DEBUG(dbgs() << "LV: Overriding calculated MaxVF(" << MaxVFdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Overriding calculated MaxVF("
 << MaxVF << ") with target's minimum: " <<
 MinVF << '\n'; } } while (false)
                        << ") with target's minimum: " << MinVF << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Overriding calculated MaxVF("
 << MaxVF << ") with target's minimum: " <<
 MinVF << '\n'; } } while (false);
      MaxVF = MinVF;
    }
  }
}
return MaxVF;
5878}

5880VectorizationFactor
5881LoopVectorizationCostModel::selectVectorizationFactor(ElementCount MaxVF) {
// FIXME: This can be fixed for scalable vectors later, because at this stage
// the LoopVectorizer will only consider vectorizing a loop with scalable
// vectors when the loop has a hint to enable vectorization for a given VF.
assert(!MaxVF.isScalable() && "scalable vectors not yet supported")((!MaxVF.isScalable() && "scalable vectors not yet supported"
) ? static_cast<void> (0) : __assert_fail ("!MaxVF.isScalable() && \"scalable vectors not yet supported\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 5885, __PRETTY_FUNCTION__));

InstructionCost ExpectedCost = expectedCost(ElementCount::getFixed(1)).first;
LLVM_DEBUG(dbgs() << "LV: Scalar loop costs: " << ExpectedCost << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Scalar loop costs: "
 << ExpectedCost << ".\n"; } } while (false);
assert(ExpectedCost.isValid() && "Unexpected invalid cost for scalar loop")((ExpectedCost.isValid() && "Unexpected invalid cost for scalar loop"
) ? static_cast<void> (0) : __assert_fail ("ExpectedCost.isValid() && \"Unexpected invalid cost for scalar loop\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 5889, __PRETTY_FUNCTION__));

auto Width = ElementCount::getFixed(1);
const float ScalarCost = *ExpectedCost.getValue();
float Cost = ScalarCost;

bool ForceVectorization = Hints->getForce() == LoopVectorizeHints::FK_Enabled;
if (ForceVectorization && MaxVF.isVector()) {
  // Ignore scalar width, because the user explicitly wants vectorization.
  // Initialize cost to max so that VF = 2 is, at least, chosen during cost
  // evaluation.
  Cost = std::numeric_limits<float>::max();
}

for (auto i = ElementCount::getFixed(2); ElementCount::isKnownLE(i, MaxVF);
     i *= 2) {
  // Notice that the vector loop needs to be executed less times, so
  // we need to divide the cost of the vector loops by the width of
  // the vector elements.
  VectorizationCostTy C = expectedCost(i);
  assert(C.first.isValid() && "Unexpected invalid cost for vector loop")((C.first.isValid() && "Unexpected invalid cost for vector loop"
) ? static_cast<void> (0) : __assert_fail ("C.first.isValid() && \"Unexpected invalid cost for vector loop\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 5909, __PRETTY_FUNCTION__));
  float VectorCost = *C.first.getValue() / (float)i.getFixedValue();
  LLVM_DEBUG(dbgs() << "LV: Vector loop of width " << ido { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Vector loop of width "
 << i << " costs: " << (int)VectorCost <<
 ".\n"; } } while (false)
                    << " costs: " << (int)VectorCost << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Vector loop of width "
 << i << " costs: " << (int)VectorCost <<
 ".\n"; } } while (false);
  if (!C.second && !ForceVectorization) {
    LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Not considering vector loop of width "
 << i << " because it will not generate any vector instructions.\n"
; } } while (false)
        dbgs() << "LV: Not considering vector loop of width " << ido { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Not considering vector loop of width "
 << i << " because it will not generate any vector instructions.\n"
; } } while (false)
               << " because it will not generate any vector instructions.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Not considering vector loop of width "
 << i << " because it will not generate any vector instructions.\n"
; } } while (false);
    continue;
  }

  // If profitable add it to ProfitableVF list.
  if (VectorCost < ScalarCost) {
    ProfitableVFs.push_back(VectorizationFactor(
        {i, (unsigned)VectorCost}));
  }

  if (VectorCost < Cost) {
    Cost = VectorCost;
    Width = i;
  }
}

if (!EnableCondStoresVectorization && NumPredStores) {
  reportVectorizationFailure("There are conditional stores.",
      "store that is conditionally executed prevents vectorization",
      "ConditionalStore", ORE, TheLoop);
  Width = ElementCount::getFixed(1);
  Cost = ScalarCost;
}

LLVM_DEBUG(if (ForceVectorization && !Width.isScalar() && Cost >= ScalarCost) dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { if (ForceVectorization && !Width
.isScalar() && Cost >= ScalarCost) dbgs() <<
 "LV: Vectorization seems to be not beneficial, " << "but was forced by a user.\n"
; } } while (false)
           << "LV: Vectorization seems to be not beneficial, "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { if (ForceVectorization && !Width
.isScalar() && Cost >= ScalarCost) dbgs() <<
 "LV: Vectorization seems to be not beneficial, " << "but was forced by a user.\n"
; } } while (false)
           << "but was forced by a user.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { if (ForceVectorization && !Width
.isScalar() && Cost >= ScalarCost) dbgs() <<
 "LV: Vectorization seems to be not beneficial, " << "but was forced by a user.\n"
; } } while (false);
LLVM_DEBUG(dbgs() << "LV: Selecting VF: " << Width << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Selecting VF: " <<
 Width << ".\n"; } } while (false);
VectorizationFactor Factor = {Width,
                              (unsigned)(Width.getKnownMinValue() * Cost)};
return Factor;
5947}

5949bool LoopVectorizationCostModel::isCandidateForEpilogueVectorization(
  const Loop &L, ElementCount VF) const {
// Cross iteration phis such as reductions need special handling and are
// currently unsupported.
if (any_of(L.getHeader()->phis(), [&](PHINode &Phi) {
      return Legal->isFirstOrderRecurrence(&Phi) ||
             Legal->isReductionVariable(&Phi);
    }))
  return false;

// Phis with uses outside of the loop require special handling and are
// currently unsupported.
for (auto &Entry : Legal->getInductionVars()) {
  // Look for uses of the value of the induction at the last iteration.
  Value *PostInc = Entry.first->getIncomingValueForBlock(L.getLoopLatch());
  for (User *U : PostInc->users())
    if (!L.contains(cast<Instruction>(U)))
      return false;
  // Look for uses of penultimate value of the induction.
  for (User *U : Entry.first->users())
    if (!L.contains(cast<Instruction>(U)))
      return false;
}

// Induction variables that are widened require special handling that is
// currently not supported.
if (any_of(Legal->getInductionVars(), [&](auto &Entry) {
      return !(this->isScalarAfterVectorization(Entry.first, VF) ||
               this->isProfitableToScalarize(Entry.first, VF));
    }))
  return false;

return true;
5982}

5984bool LoopVectorizationCostModel::isEpilogueVectorizationProfitable(
  const ElementCount VF) const {
// FIXME: We need a much better cost-model to take different parameters such
// as register pressure, code size increase and cost of extra branches into
// account. For now we apply a very crude heuristic and only consider loops
// with vectorization factors larger than a certain value.
// We also consider epilogue vectorization unprofitable for targets that don't
// consider interleaving beneficial (eg. MVE).
if (TTI.getMaxInterleaveFactor(VF.getKnownMinValue()) <= 1)
  return false;
if (VF.getFixedValue() >= EpilogueVectorizationMinVF)
  return true;
return false;
5997}

5999VectorizationFactor
6000LoopVectorizationCostModel::selectEpilogueVectorizationFactor(
  const ElementCount MainLoopVF, const LoopVectorizationPlanner &LVP) {
VectorizationFactor Result = VectorizationFactor::Disabled();
if (!EnableEpilogueVectorization) {
  LLVM_DEBUG(dbgs() << "LEV: Epilogue vectorization is disabled.\n";)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LEV: Epilogue vectorization is disabled.\n"
;; } } while (false);
  return Result;
}

if (!isScalarEpilogueAllowed()) {
  LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LEV: Unable to vectorize epilogue because no epilogue is "
 "allowed.\n";; } } while (false)
      dbgs() << "LEV: Unable to vectorize epilogue because no epilogue is "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LEV: Unable to vectorize epilogue because no epilogue is "
 "allowed.\n";; } } while (false)
                "allowed.\n";)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LEV: Unable to vectorize epilogue because no epilogue is "
 "allowed.\n";; } } while (false);
  return Result;
}

// FIXME: This can be fixed for scalable vectors later, because at this stage
// the LoopVectorizer will only consider vectorizing a loop with scalable
// vectors when the loop has a hint to enable vectorization for a given VF.
if (MainLoopVF.isScalable()) {
  LLVM_DEBUG(dbgs() << "LEV: Epilogue vectorization for scalable vectors not "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LEV: Epilogue vectorization for scalable vectors not "
 "yet supported.\n"; } } while (false)
                       "yet supported.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LEV: Epilogue vectorization for scalable vectors not "
 "yet supported.\n"; } } while (false);
  return Result;
}

// Not really a cost consideration, but check for unsupported cases here to
// simplify the logic.
if (!isCandidateForEpilogueVectorization(*TheLoop, MainLoopVF)) {
  LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LEV: Unable to vectorize epilogue because the loop is "
 "not a supported candidate.\n";; } } while (false)
      dbgs() << "LEV: Unable to vectorize epilogue because the loop is "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LEV: Unable to vectorize epilogue because the loop is "
 "not a supported candidate.\n";; } } while (false)
                "not a supported candidate.\n";)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LEV: Unable to vectorize epilogue because the loop is "
 "not a supported candidate.\n";; } } while (false);
  return Result;
}

if (EpilogueVectorizationForceVF > 1) {
  LLVM_DEBUG(dbgs() << "LEV: Epilogue vectorization factor is forced.\n";)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LEV: Epilogue vectorization factor is forced.\n"
;; } } while (false);
  if (LVP.hasPlanWithVFs(
          {MainLoopVF, ElementCount::getFixed(EpilogueVectorizationForceVF)}))
    return {ElementCount::getFixed(EpilogueVectorizationForceVF), 0};
  else {
    LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LEV: Epilogue vectorization forced factor is not viable.\n"
;; } } while (false)
        dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LEV: Epilogue vectorization forced factor is not viable.\n"
;; } } while (false)
            << "LEV: Epilogue vectorization forced factor is not viable.\n";)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LEV: Epilogue vectorization forced factor is not viable.\n"
;; } } while (false);
    return Result;
  }
}

if (TheLoop->getHeader()->getParent()->hasOptSize() ||
    TheLoop->getHeader()->getParent()->hasMinSize()) {
  LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LEV: Epilogue vectorization skipped due to opt for size.\n"
;; } } while (false)
      dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LEV: Epilogue vectorization skipped due to opt for size.\n"
;; } } while (false)
          << "LEV: Epilogue vectorization skipped due to opt for size.\n";)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LEV: Epilogue vectorization skipped due to opt for size.\n"
;; } } while (false);
  return Result;
}

if (!isEpilogueVectorizationProfitable(MainLoopVF))
  return Result;

for (auto &NextVF : ProfitableVFs)
  if (ElementCount::isKnownLT(NextVF.Width, MainLoopVF) &&
      (Result.Width.getFixedValue() == 1 || NextVF.Cost < Result.Cost) &&
      LVP.hasPlanWithVFs({MainLoopVF, NextVF.Width}))
    Result = NextVF;

if (Result != VectorizationFactor::Disabled())
  LLVM_DEBUG(dbgs() << "LEV: Vectorizing epilogue loop with VF = "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LEV: Vectorizing epilogue loop with VF = "
 << Result.Width.getFixedValue() << "\n";; } } while
 (false)
                    << Result.Width.getFixedValue() << "\n";)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LEV: Vectorizing epilogue loop with VF = "
 << Result.Width.getFixedValue() << "\n";; } } while
 (false);
return Result;
6067}

6069std::pair<unsigned, unsigned>
6070LoopVectorizationCostModel::getSmallestAndWidestTypes() {
unsigned MinWidth = -1U;
unsigned MaxWidth = 8;
const DataLayout &DL = TheFunction->getParent()->getDataLayout();

// For each block.
for (BasicBlock *BB : TheLoop->blocks()) {
  // For each instruction in the loop.
  for (Instruction &I : BB->instructionsWithoutDebug()) {
    Type *T = I.getType();

    // Skip ignored values.
    if (ValuesToIgnore.count(&I))
      continue;

    // Only examine Loads, Stores and PHINodes.
    if (!isa<LoadInst>(I) && !isa<StoreInst>(I) && !isa<PHINode>(I))
      continue;

    // Examine PHI nodes that are reduction variables. Update the type to
    // account for the recurrence type.
    if (auto *PN = dyn_cast<PHINode>(&I)) {
      if (!Legal->isReductionVariable(PN))
        continue;
      RecurrenceDescriptor RdxDesc = Legal->getReductionVars()[PN];
      if (PreferInLoopReductions || useOrderedReductions(RdxDesc) ||
          TTI.preferInLoopReduction(RdxDesc.getOpcode(),
                                    RdxDesc.getRecurrenceType(),
                                    TargetTransformInfo::ReductionFlags()))
        continue;
      T = RdxDesc.getRecurrenceType();
    }

    // Examine the stored values.
    if (auto *ST = dyn_cast<StoreInst>(&I))
      T = ST->getValueOperand()->getType();

    // Ignore loaded pointer types and stored pointer types that are not
    // vectorizable.
    //
    // FIXME: The check here attempts to predict whether a load or store will
    //        be vectorized. We only know this for certain after a VF has
    //        been selected. Here, we assume that if an access can be
    //        vectorized, it will be. We should also look at extending this
    //        optimization to non-pointer types.
    //
    if (T->isPointerTy() && !isConsecutiveLoadOrStore(&I) &&
        !isAccessInterleaved(&I) && !isLegalGatherOrScatter(&I))
      continue;

    MinWidth = std::min(MinWidth,
                        (unsigned)DL.getTypeSizeInBits(T->getScalarType()));
    MaxWidth = std::max(MaxWidth,
                        (unsigned)DL.getTypeSizeInBits(T->getScalarType()));
  }
}

return {MinWidth, MaxWidth};
6128}

6130unsigned LoopVectorizationCostModel::selectInterleaveCount(ElementCount VF,
                                                         unsigned LoopCost) {
// -- The interleave heuristics --
// We interleave the loop in order to expose ILP and reduce the loop overhead.
// There are many micro-architectural considerations that we can't predict
// at this level. For example, frontend pressure (on decode or fetch) due to
// code size, or the number and capabilities of the execution ports.
//
// We use the following heuristics to select the interleave count:
// 1. If the code has reductions, then we interleave to break the cross
// iteration dependency.
// 2. If the loop is really small, then we interleave to reduce the loop
// overhead.
// 3. We don't interleave if we think that we will spill registers to memory
// due to the increased register pressure.

if (!isScalarEpilogueAllowed())
  return 1;

// We used the distance for the interleave count.
if (Legal->getMaxSafeDepDistBytes() != -1U)
  return 1;

auto BestKnownTC = getSmallBestKnownTC(*PSE.getSE(), TheLoop);
const bool HasReductions = !Legal->getReductionVars().empty();
// Do not interleave loops with a relatively small known or estimated trip
// count. But we will interleave when InterleaveSmallLoopScalarReduction is
// enabled, and the code has scalar reductions(HasReductions && VF = 1),
// because with the above conditions interleaving can expose ILP and break
// cross iteration dependences for reductions.
if (BestKnownTC && (*BestKnownTC < TinyTripCountInterleaveThreshold) &&
    !(InterleaveSmallLoopScalarReduction && HasReductions && VF.isScalar()))
  return 1;

RegisterUsage R = calculateRegisterUsage({VF})[0];
// We divide by these constants so assume that we have at least one
// instruction that uses at least one register.
for (auto& pair : R.MaxLocalUsers) {
  pair.second = std::max(pair.second, 1U);
}

// We calculate the interleave count using the following formula.
// Subtract the number of loop invariants from the number of available
// registers. These registers are used by all of the interleaved instances.
// Next, divide the remaining registers by the number of registers that is
// required by the loop, in order to estimate how many parallel instances
// fit without causing spills. All of this is rounded down if necessary to be
// a power of two. We want power of two interleave count to simplify any
// addressing operations or alignment considerations.
// We also want power of two interleave counts to ensure that the induction
// variable of the vector loop wraps to zero, when tail is folded by masking;
// this currently happens when OptForSize, in which case IC is set to 1 above.
unsigned IC = UINT_MAX(2147483647 *2U +1U);

for (auto& pair : R.MaxLocalUsers) {
  unsigned TargetNumRegisters = TTI.getNumberOfRegisters(pair.first);
  LLVM_DEBUG(dbgs() << "LV: The target has " << TargetNumRegistersdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: The target has " <<
 TargetNumRegisters << " registers of " << TTI.getRegisterClassName
(pair.first) << " register class\n"; } } while (false)
                    << " registers of "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: The target has " <<
 TargetNumRegisters << " registers of " << TTI.getRegisterClassName
(pair.first) << " register class\n"; } } while (false)
                    << TTI.getRegisterClassName(pair.first) << " register class\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: The target has " <<
 TargetNumRegisters << " registers of " << TTI.getRegisterClassName
(pair.first) << " register class\n"; } } while (false);
  if (VF.isScalar()) {
    if (ForceTargetNumScalarRegs.getNumOccurrences() > 0)
      TargetNumRegisters = ForceTargetNumScalarRegs;
  } else {
    if (ForceTargetNumVectorRegs.getNumOccurrences() > 0)
      TargetNumRegisters = ForceTargetNumVectorRegs;
  }
  unsigned MaxLocalUsers = pair.second;
  unsigned LoopInvariantRegs = 0;
  if (R.LoopInvariantRegs.find(pair.first) != R.LoopInvariantRegs.end())
    LoopInvariantRegs = R.LoopInvariantRegs[pair.first];

  unsigned TmpIC = PowerOf2Floor((TargetNumRegisters - LoopInvariantRegs) / MaxLocalUsers);
  // Don't count the induction variable as interleaved.
  if (EnableIndVarRegisterHeur) {
    TmpIC =
        PowerOf2Floor((TargetNumRegisters - LoopInvariantRegs - 1) /
                      std::max(1U, (MaxLocalUsers - 1)));
  }

  IC = std::min(IC, TmpIC);
}

// Clamp the interleave ranges to reasonable counts.
unsigned MaxInterleaveCount =
    TTI.getMaxInterleaveFactor(VF.getKnownMinValue());

// Check if the user has overridden the max.
if (VF.isScalar()) {
  if (ForceTargetMaxScalarInterleaveFactor.getNumOccurrences() > 0)
    MaxInterleaveCount = ForceTargetMaxScalarInterleaveFactor;
} else {
  if (ForceTargetMaxVectorInterleaveFactor.getNumOccurrences() > 0)
    MaxInterleaveCount = ForceTargetMaxVectorInterleaveFactor;
}

// If trip count is known or estimated compile time constant, limit the
// interleave count to be less than the trip count divided by VF, provided it
// is at least 1.
//
// For scalable vectors we can't know if interleaving is beneficial. It may
// not be beneficial for small loops if none of the lanes in the second vector
// iterations is enabled. However, for larger loops, there is likely to be a
// similar benefit as for fixed-width vectors. For now, we choose to leave
// the InterleaveCount as if vscale is '1', although if some information about
// the vector is known (e.g. min vector size), we can make a better decision.
if (BestKnownTC) {
  MaxInterleaveCount =
      std::min(*BestKnownTC / VF.getKnownMinValue(), MaxInterleaveCount);
  // Make sure MaxInterleaveCount is greater than 0.
  MaxInterleaveCount = std::max(1u, MaxInterleaveCount);
}

assert(MaxInterleaveCount > 0 &&((MaxInterleaveCount > 0 && "Maximum interleave count must be greater than 0"
) ? static_cast<void> (0) : __assert_fail ("MaxInterleaveCount > 0 && \"Maximum interleave count must be greater than 0\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 6243, __PRETTY_FUNCTION__))
       "Maximum interleave count must be greater than 0")((MaxInterleaveCount > 0 && "Maximum interleave count must be greater than 0"
) ? static_cast<void> (0) : __assert_fail ("MaxInterleaveCount > 0 && \"Maximum interleave count must be greater than 0\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 6243, __PRETTY_FUNCTION__));

// Clamp the calculated IC to be between the 1 and the max interleave count
// that the target and trip count allows.
if (IC > MaxInterleaveCount)
  IC = MaxInterleaveCount;
else
  // Make sure IC is greater than 0.
  IC = std::max(1u, IC);

assert(IC > 0 && "Interleave count must be greater than 0.")((IC > 0 && "Interleave count must be greater than 0."
) ? static_cast<void> (0) : __assert_fail ("IC > 0 && \"Interleave count must be greater than 0.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 6253, __PRETTY_FUNCTION__));

// If we did not calculate the cost for VF (because the user selected the VF)
// then we calculate the cost of VF here.
if (LoopCost == 0) {
  assert(expectedCost(VF).first.isValid() && "Expected a valid cost")((expectedCost(VF).first.isValid() && "Expected a valid cost"
) ? static_cast<void> (0) : __assert_fail ("expectedCost(VF).first.isValid() && \"Expected a valid cost\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 6258, __PRETTY_FUNCTION__));
  LoopCost = *expectedCost(VF).first.getValue();
}

assert(LoopCost && "Non-zero loop cost expected")((LoopCost && "Non-zero loop cost expected") ? static_cast
<void> (0) : __assert_fail ("LoopCost && \"Non-zero loop cost expected\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 6262, __PRETTY_FUNCTION__));

// Interleave if we vectorized this loop and there is a reduction that could
// benefit from interleaving.
if (VF.isVector() && HasReductions) {
  LLVM_DEBUG(dbgs() << "LV: Interleaving because of reductions.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Interleaving because of reductions.\n"
; } } while (false);
  return IC;
}

// Note that if we've already vectorized the loop we will have done the
// runtime check and so interleaving won't require further checks.
bool InterleavingRequiresRuntimePointerCheck =
    (VF.isScalar() && Legal->getRuntimePointerChecking()->Need);

// We want to interleave small loops in order to reduce the loop overhead and
// potentially expose ILP opportunities.
LLVM_DEBUG(dbgs() << "LV: Loop cost is " << LoopCost << '\n'do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Loop cost is " <<
 LoopCost << '\n' << "LV: IC is " << IC <<
 '\n' << "LV: VF is " << VF << '\n'; } } while
 (false)
                  << "LV: IC is " << IC << '\n'do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Loop cost is " <<
 LoopCost << '\n' << "LV: IC is " << IC <<
 '\n' << "LV: VF is " << VF << '\n'; } } while
 (false)
                  << "LV: VF is " << VF << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Loop cost is " <<
 LoopCost << '\n' << "LV: IC is " << IC <<
 '\n' << "LV: VF is " << VF << '\n'; } } while
 (false);
const bool AggressivelyInterleaveReductions =
    TTI.enableAggressiveInterleaving(HasReductions);
if (!InterleavingRequiresRuntimePointerCheck && LoopCost < SmallLoopCost) {
  // We assume that the cost overhead is 1 and we use the cost model
  // to estimate the cost of the loop and interleave until the cost of the
  // loop overhead is about 5% of the cost of the loop.
  unsigned SmallIC =
      std::min(IC, (unsigned)PowerOf2Floor(SmallLoopCost / LoopCost));

  // Interleave until store/load ports (estimated by max interleave count) are
  // saturated.
  unsigned NumStores = Legal->getNumStores();
  unsigned NumLoads = Legal->getNumLoads();
  unsigned StoresIC = IC / (NumStores ? NumStores : 1);
  unsigned LoadsIC = IC / (NumLoads ? NumLoads : 1);

  // If we have a scalar reduction (vector reductions are already dealt with
  // by this point), we can increase the critical path length if the loop
  // we're interleaving is inside another loop. Limit, by default to 2, so the
  // critical path only gets increased by one reduction operation.
  if (HasReductions && TheLoop->getLoopDepth() > 1) {
    unsigned F = static_cast<unsigned>(MaxNestedScalarReductionIC);
    SmallIC = std::min(SmallIC, F);
    StoresIC = std::min(StoresIC, F);
    LoadsIC = std::min(LoadsIC, F);
  }

  if (EnableLoadStoreRuntimeInterleave &&
      std::max(StoresIC, LoadsIC) > SmallIC) {
    LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Interleaving to saturate store or load ports.\n"
; } } while (false)
        dbgs() << "LV: Interleaving to saturate store or load ports.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Interleaving to saturate store or load ports.\n"
; } } while (false);
    return std::max(StoresIC, LoadsIC);
  }

  // If there are scalar reductions and TTI has enabled aggressive
  // interleaving for reductions, we will interleave to expose ILP.
  if (InterleaveSmallLoopScalarReduction && VF.isScalar() &&
      AggressivelyInterleaveReductions) {
    LLVM_DEBUG(dbgs() << "LV: Interleaving to expose ILP.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Interleaving to expose ILP.\n"
; } } while (false);
    // Interleave no less than SmallIC but not as aggressive as the normal IC
    // to satisfy the rare situation when resources are too limited.
    return std::max(IC / 2, SmallIC);
  } else {
    LLVM_DEBUG(dbgs() << "LV: Interleaving to reduce branch cost.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Interleaving to reduce branch cost.\n"
; } } while (false);
    return SmallIC;
  }
}

// Interleave if this is a large loop (small loops are already dealt with by
// this point) that could benefit from interleaving.
if (AggressivelyInterleaveReductions) {
  LLVM_DEBUG(dbgs() << "LV: Interleaving to expose ILP.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Interleaving to expose ILP.\n"
; } } while (false);
  return IC;
}

LLVM_DEBUG(dbgs() << "LV: Not Interleaving.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Not Interleaving.\n"
; } } while (false);
return 1;
6338}

6340SmallVector<LoopVectorizationCostModel::RegisterUsage, 8>
6341LoopVectorizationCostModel::calculateRegisterUsage(ArrayRef<ElementCount> VFs) {
// This function calculates the register usage by measuring the highest number
// of values that are alive at a single location. Obviously, this is a very
// rough estimation. We scan the loop in a topological order in order and
// assign a number to each instruction. We use RPO to ensure that defs are
// met before their users. We assume that each instruction that has in-loop
// users starts an interval. We record every time that an in-loop value is
// used, so we have a list of the first and last occurrences of each
// instruction. Next, we transpose this data structure into a multi map that
// holds the list of intervals that *end* at a specific location. This multi
// map allows us to perform a linear search. We scan the instructions linearly
// and record each time that a new interval starts, by placing it in a set.
// If we find this value in the multi-map then we remove it from the set.
// The max register usage is the maximum size of the set.
// We also search for instructions that are defined outside the loop, but are
// used inside the loop. We need this number separately from the max-interval
// usage number because when we unroll, loop-invariant values do not take
// more register.
LoopBlocksDFS DFS(TheLoop);
DFS.perform(LI);

RegisterUsage RU;

// Each 'key' in the map opens a new interval. The values
// of the map are the index of the 'last seen' usage of the
// instruction that is the key.
using IntervalMap = DenseMap<Instruction *, unsigned>;

// Maps instruction to its index.
SmallVector<Instruction *, 64> IdxToInstr;
// Marks the end of each interval.
IntervalMap EndPoint;
// Saves the list of instruction indices that are used in the loop.
SmallPtrSet<Instruction *, 8> Ends;
// Saves the list of values that are used in the loop but are
// defined outside the loop, such as arguments and constants.
SmallPtrSet<Value *, 8> LoopInvariants;

for (BasicBlock *BB : make_range(DFS.beginRPO(), DFS.endRPO())) {
  for (Instruction &I : BB->instructionsWithoutDebug()) {
    IdxToInstr.push_back(&I);

    // Save the end location of each USE.
    for (Value *U : I.operands()) {
      auto *Instr = dyn_cast<Instruction>(U);

      // Ignore non-instruction values such as arguments, constants, etc.
      if (!Instr)
        continue;

      // If this instruction is outside the loop then record it and continue.
      if (!TheLoop->contains(Instr)) {
        LoopInvariants.insert(Instr);
        continue;
      }

      // Overwrite previous end points.
      EndPoint[Instr] = IdxToInstr.size();
      Ends.insert(Instr);
    }
  }
}

// Saves the list of intervals that end with the index in 'key'.
using InstrList = SmallVector<Instruction *, 2>;
DenseMap<unsigned, InstrList> TransposeEnds;

// Transpose the EndPoints to a list of values that end at each index.
for (auto &Interval : EndPoint)
  TransposeEnds[Interval.second].push_back(Interval.first);

SmallPtrSet<Instruction *, 8> OpenIntervals;
SmallVector<RegisterUsage, 8> RUs(VFs.size());
SmallVector<SmallMapVector<unsigned, unsigned, 4>, 8> MaxUsages(VFs.size());

LLVM_DEBUG(dbgs() << "LV(REG): Calculating max register usage:\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV(REG): Calculating max register usage:\n"
; } } while (false);

// A lambda that gets the register usage for the given type and VF.
const auto &TTICapture = TTI;
auto GetRegUsage = [&TTICapture](Type *Ty, ElementCount VF) {
  if (Ty->isTokenTy() || !VectorType::isValidElementType(Ty))
    return 0U;
  return TTICapture.getRegUsageForType(VectorType::get(Ty, VF));
};

for (unsigned int i = 0, s = IdxToInstr.size(); i < s; ++i) {
  Instruction *I = IdxToInstr[i];

  // Remove all of the instructions that end at this location.
  InstrList &List = TransposeEnds[i];
  for (Instruction *ToRemove : List)
    OpenIntervals.erase(ToRemove);

  // Ignore instructions that are never used within the loop.
  if (!Ends.count(I))
    continue;

  // Skip ignored values.
  if (ValuesToIgnore.count(I))
    continue;

  // For each VF find the maximum usage of registers.
  for (unsigned j = 0, e = VFs.size(); j < e; ++j) {
    // Count the number of live intervals.
    SmallMapVector<unsigned, unsigned, 4> RegUsage;

    if (VFs[j].isScalar()) {
      for (auto Inst : OpenIntervals) {
        unsigned ClassID = TTI.getRegisterClassForType(false, Inst->getType());
        if (RegUsage.find(ClassID) == RegUsage.end())
          RegUsage[ClassID] = 1;
        else
          RegUsage[ClassID] += 1;
      }
    } else {
      collectUniformsAndScalars(VFs[j]);
      for (auto Inst : OpenIntervals) {
        // Skip ignored values for VF > 1.
        if (VecValuesToIgnore.count(Inst))
          continue;
        if (isScalarAfterVectorization(Inst, VFs[j])) {
          unsigned ClassID = TTI.getRegisterClassForType(false, Inst->getType());
          if (RegUsage.find(ClassID) == RegUsage.end())
            RegUsage[ClassID] = 1;
          else
            RegUsage[ClassID] += 1;
        } else {
          unsigned ClassID = TTI.getRegisterClassForType(true, Inst->getType());
          if (RegUsage.find(ClassID) == RegUsage.end())
            RegUsage[ClassID] = GetRegUsage(Inst->getType(), VFs[j]);
          else
            RegUsage[ClassID] += GetRegUsage(Inst->getType(), VFs[j]);
        }
      }
    }

    for (auto& pair : RegUsage) {
      if (MaxUsages[j].find(pair.first) != MaxUsages[j].end())
        MaxUsages[j][pair.first] = std::max(MaxUsages[j][pair.first], pair.second);
      else
        MaxUsages[j][pair.first] = pair.second;
    }
  }

  LLVM_DEBUG(dbgs() << "LV(REG): At #" << i << " Interval # "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV(REG): At #" <<
 i << " Interval # " << OpenIntervals.size() <<
 '\n'; } } while (false)
                    << OpenIntervals.size() << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV(REG): At #" <<
 i << " Interval # " << OpenIntervals.size() <<
 '\n'; } } while (false);

  // Add the current instruction to the list of open intervals.
  OpenIntervals.insert(I);
}

for (unsigned i = 0, e = VFs.size(); i < e; ++i) {
  SmallMapVector<unsigned, unsigned, 4> Invariant;

  for (auto Inst : LoopInvariants) {
    unsigned Usage =
        VFs[i].isScalar() ? 1 : GetRegUsage(Inst->getType(), VFs[i]);
    unsigned ClassID =
        TTI.getRegisterClassForType(VFs[i].isVector(), Inst->getType());
    if (Invariant.find(ClassID) == Invariant.end())
      Invariant[ClassID] = Usage;
    else
      Invariant[ClassID] += Usage;
  }

  LLVM_DEBUG({do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { { dbgs() << "LV(REG): VF = " <<
 VFs[i] << '\n'; dbgs() << "LV(REG): Found max usage: "
 << MaxUsages[i].size() << " item\n"; for (const auto
 &pair : MaxUsages[i]) { dbgs() << "LV(REG): RegisterClass: "
 << TTI.getRegisterClassName(pair.first) << ", " <<
 pair.second << " registers\n"; } dbgs() << "LV(REG): Found invariant usage: "
 << Invariant.size() << " item\n"; for (const auto
 &pair : Invariant) { dbgs() << "LV(REG): RegisterClass: "
 << TTI.getRegisterClassName(pair.first) << ", " <<
 pair.second << " registers\n"; } }; } } while (false)
    dbgs() << "LV(REG): VF = " << VFs[i] << '\n';do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { { dbgs() << "LV(REG): VF = " <<
 VFs[i] << '\n'; dbgs() << "LV(REG): Found max usage: "
 << MaxUsages[i].size() << " item\n"; for (const auto
 &pair : MaxUsages[i]) { dbgs() << "LV(REG): RegisterClass: "
 << TTI.getRegisterClassName(pair.first) << ", " <<
 pair.second << " registers\n"; } dbgs() << "LV(REG): Found invariant usage: "
 << Invariant.size() << " item\n"; for (const auto
 &pair : Invariant) { dbgs() << "LV(REG): RegisterClass: "
 << TTI.getRegisterClassName(pair.first) << ", " <<
 pair.second << " registers\n"; } }; } } while (false)
    dbgs() << "LV(REG): Found max usage: " << MaxUsages[i].size()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { { dbgs() << "LV(REG): VF = " <<
 VFs[i] << '\n'; dbgs() << "LV(REG): Found max usage: "
 << MaxUsages[i].size() << " item\n"; for (const auto
 &pair : MaxUsages[i]) { dbgs() << "LV(REG): RegisterClass: "
 << TTI.getRegisterClassName(pair.first) << ", " <<
 pair.second << " registers\n"; } dbgs() << "LV(REG): Found invariant usage: "
 << Invariant.size() << " item\n"; for (const auto
 &pair : Invariant) { dbgs() << "LV(REG): RegisterClass: "
 << TTI.getRegisterClassName(pair.first) << ", " <<
 pair.second << " registers\n"; } }; } } while (false)
           << " item\n";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { { dbgs() << "LV(REG): VF = " <<
 VFs[i] << '\n'; dbgs() << "LV(REG): Found max usage: "
 << MaxUsages[i].size() << " item\n"; for (const auto
 &pair : MaxUsages[i]) { dbgs() << "LV(REG): RegisterClass: "
 << TTI.getRegisterClassName(pair.first) << ", " <<
 pair.second << " registers\n"; } dbgs() << "LV(REG): Found invariant usage: "
 << Invariant.size() << " item\n"; for (const auto
 &pair : Invariant) { dbgs() << "LV(REG): RegisterClass: "
 << TTI.getRegisterClassName(pair.first) << ", " <<
 pair.second << " registers\n"; } }; } } while (false)
    for (const auto &pair : MaxUsages[i]) {do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { { dbgs() << "LV(REG): VF = " <<
 VFs[i] << '\n'; dbgs() << "LV(REG): Found max usage: "
 << MaxUsages[i].size() << " item\n"; for (const auto
 &pair : MaxUsages[i]) { dbgs() << "LV(REG): RegisterClass: "
 << TTI.getRegisterClassName(pair.first) << ", " <<
 pair.second << " registers\n"; } dbgs() << "LV(REG): Found invariant usage: "
 << Invariant.size() << " item\n"; for (const auto
 &pair : Invariant) { dbgs() << "LV(REG): RegisterClass: "
 << TTI.getRegisterClassName(pair.first) << ", " <<
 pair.second << " registers\n"; } }; } } while (false)
      dbgs() << "LV(REG): RegisterClass: "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { { dbgs() << "LV(REG): VF = " <<
 VFs[i] << '\n'; dbgs() << "LV(REG): Found max usage: "
 << MaxUsages[i].size() << " item\n"; for (const auto
 &pair : MaxUsages[i]) { dbgs() << "LV(REG): RegisterClass: "
 << TTI.getRegisterClassName(pair.first) << ", " <<
 pair.second << " registers\n"; } dbgs() << "LV(REG): Found invariant usage: "
 << Invariant.size() << " item\n"; for (const auto
 &pair : Invariant) { dbgs() << "LV(REG): RegisterClass: "
 << TTI.getRegisterClassName(pair.first) << ", " <<
 pair.second << " registers\n"; } }; } } while (false)
             << TTI.getRegisterClassName(pair.first) << ", " << pair.seconddo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { { dbgs() << "LV(REG): VF = " <<
 VFs[i] << '\n'; dbgs() << "LV(REG): Found max usage: "
 << MaxUsages[i].size() << " item\n"; for (const auto
 &pair : MaxUsages[i]) { dbgs() << "LV(REG): RegisterClass: "
 << TTI.getRegisterClassName(pair.first) << ", " <<
 pair.second << " registers\n"; } dbgs() << "LV(REG): Found invariant usage: "
 << Invariant.size() << " item\n"; for (const auto
 &pair : Invariant) { dbgs() << "LV(REG): RegisterClass: "
 << TTI.getRegisterClassName(pair.first) << ", " <<
 pair.second << " registers\n"; } }; } } while (false)
             << " registers\n";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { { dbgs() << "LV(REG): VF = " <<
 VFs[i] << '\n'; dbgs() << "LV(REG): Found max usage: "
 << MaxUsages[i].size() << " item\n"; for (const auto
 &pair : MaxUsages[i]) { dbgs() << "LV(REG): RegisterClass: "
 << TTI.getRegisterClassName(pair.first) << ", " <<
 pair.second << " registers\n"; } dbgs() << "LV(REG): Found invariant usage: "
 << Invariant.size() << " item\n"; for (const auto
 &pair : Invariant) { dbgs() << "LV(REG): RegisterClass: "
 << TTI.getRegisterClassName(pair.first) << ", " <<
 pair.second << " registers\n"; } }; } } while (false)
    }do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { { dbgs() << "LV(REG): VF = " <<
 VFs[i] << '\n'; dbgs() << "LV(REG): Found max usage: "
 << MaxUsages[i].size() << " item\n"; for (const auto
 &pair : MaxUsages[i]) { dbgs() << "LV(REG): RegisterClass: "
 << TTI.getRegisterClassName(pair.first) << ", " <<
 pair.second << " registers\n"; } dbgs() << "LV(REG): Found invariant usage: "
 << Invariant.size() << " item\n"; for (const auto
 &pair : Invariant) { dbgs() << "LV(REG): RegisterClass: "
 << TTI.getRegisterClassName(pair.first) << ", " <<
 pair.second << " registers\n"; } }; } } while (false)
    dbgs() << "LV(REG): Found invariant usage: " << Invariant.size()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { { dbgs() << "LV(REG): VF = " <<
 VFs[i] << '\n'; dbgs() << "LV(REG): Found max usage: "
 << MaxUsages[i].size() << " item\n"; for (const auto
 &pair : MaxUsages[i]) { dbgs() << "LV(REG): RegisterClass: "
 << TTI.getRegisterClassName(pair.first) << ", " <<
 pair.second << " registers\n"; } dbgs() << "LV(REG): Found invariant usage: "
 << Invariant.size() << " item\n"; for (const auto
 &pair : Invariant) { dbgs() << "LV(REG): RegisterClass: "
 << TTI.getRegisterClassName(pair.first) << ", " <<
 pair.second << " registers\n"; } }; } } while (false)
           << " item\n";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { { dbgs() << "LV(REG): VF = " <<
 VFs[i] << '\n'; dbgs() << "LV(REG): Found max usage: "
 << MaxUsages[i].size() << " item\n"; for (const auto
 &pair : MaxUsages[i]) { dbgs() << "LV(REG): RegisterClass: "
 << TTI.getRegisterClassName(pair.first) << ", " <<
 pair.second << " registers\n"; } dbgs() << "LV(REG): Found invariant usage: "
 << Invariant.size() << " item\n"; for (const auto
 &pair : Invariant) { dbgs() << "LV(REG): RegisterClass: "
 << TTI.getRegisterClassName(pair.first) << ", " <<
 pair.second << " registers\n"; } }; } } while (false)
    for (const auto &pair : Invariant) {do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { { dbgs() << "LV(REG): VF = " <<
 VFs[i] << '\n'; dbgs() << "LV(REG): Found max usage: "
 << MaxUsages[i].size() << " item\n"; for (const auto
 &pair : MaxUsages[i]) { dbgs() << "LV(REG): RegisterClass: "
 << TTI.getRegisterClassName(pair.first) << ", " <<
 pair.second << " registers\n"; } dbgs() << "LV(REG): Found invariant usage: "
 << Invariant.size() << " item\n"; for (const auto
 &pair : Invariant) { dbgs() << "LV(REG): RegisterClass: "
 << TTI.getRegisterClassName(pair.first) << ", " <<
 pair.second << " registers\n"; } }; } } while (false)
      dbgs() << "LV(REG): RegisterClass: "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { { dbgs() << "LV(REG): VF = " <<
 VFs[i] << '\n'; dbgs() << "LV(REG): Found max usage: "
 << MaxUsages[i].size() << " item\n"; for (const auto
 &pair : MaxUsages[i]) { dbgs() << "LV(REG): RegisterClass: "
 << TTI.getRegisterClassName(pair.first) << ", " <<
 pair.second << " registers\n"; } dbgs() << "LV(REG): Found invariant usage: "
 << Invariant.size() << " item\n"; for (const auto
 &pair : Invariant) { dbgs() << "LV(REG): RegisterClass: "
 << TTI.getRegisterClassName(pair.first) << ", " <<
 pair.second << " registers\n"; } }; } } while (false)
             << TTI.getRegisterClassName(pair.first) << ", " << pair.seconddo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { { dbgs() << "LV(REG): VF = " <<
 VFs[i] << '\n'; dbgs() << "LV(REG): Found max usage: "
 << MaxUsages[i].size() << " item\n"; for (const auto
 &pair : MaxUsages[i]) { dbgs() << "LV(REG): RegisterClass: "
 << TTI.getRegisterClassName(pair.first) << ", " <<
 pair.second << " registers\n"; } dbgs() << "LV(REG): Found invariant usage: "
 << Invariant.size() << " item\n"; for (const auto
 &pair : Invariant) { dbgs() << "LV(REG): RegisterClass: "
 << TTI.getRegisterClassName(pair.first) << ", " <<
 pair.second << " registers\n"; } }; } } while (false)
             << " registers\n";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { { dbgs() << "LV(REG): VF = " <<
 VFs[i] << '\n'; dbgs() << "LV(REG): Found max usage: "
 << MaxUsages[i].size() << " item\n"; for (const auto
 &pair : MaxUsages[i]) { dbgs() << "LV(REG): RegisterClass: "
 << TTI.getRegisterClassName(pair.first) << ", " <<
 pair.second << " registers\n"; } dbgs() << "LV(REG): Found invariant usage: "
 << Invariant.size() << " item\n"; for (const auto
 &pair : Invariant) { dbgs() << "LV(REG): RegisterClass: "
 << TTI.getRegisterClassName(pair.first) << ", " <<
 pair.second << " registers\n"; } }; } } while (false)
    }do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { { dbgs() << "LV(REG): VF = " <<
 VFs[i] << '\n'; dbgs() << "LV(REG): Found max usage: "
 << MaxUsages[i].size() << " item\n"; for (const auto
 &pair : MaxUsages[i]) { dbgs() << "LV(REG): RegisterClass: "
 << TTI.getRegisterClassName(pair.first) << ", " <<
 pair.second << " registers\n"; } dbgs() << "LV(REG): Found invariant usage: "
 << Invariant.size() << " item\n"; for (const auto
 &pair : Invariant) { dbgs() << "LV(REG): RegisterClass: "
 << TTI.getRegisterClassName(pair.first) << ", " <<
 pair.second << " registers\n"; } }; } } while (false)
  })do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { { dbgs() << "LV(REG): VF = " <<
 VFs[i] << '\n'; dbgs() << "LV(REG): Found max usage: "
 << MaxUsages[i].size() << " item\n"; for (const auto
 &pair : MaxUsages[i]) { dbgs() << "LV(REG): RegisterClass: "
 << TTI.getRegisterClassName(pair.first) << ", " <<
 pair.second << " registers\n"; } dbgs() << "LV(REG): Found invariant usage: "
 << Invariant.size() << " item\n"; for (const auto
 &pair : Invariant) { dbgs() << "LV(REG): RegisterClass: "
 << TTI.getRegisterClassName(pair.first) << ", " <<
 pair.second << " registers\n"; } }; } } while (false);

  RU.LoopInvariantRegs = Invariant;
  RU.MaxLocalUsers = MaxUsages[i];
  RUs[i] = RU;
}

return RUs;
6530}

6532bool LoopVectorizationCostModel::useEmulatedMaskMemRefHack(Instruction *I){
// TODO: Cost model for emulated masked load/store is completely
// broken. This hack guides the cost model to use an artificially
// high enough value to practically disable vectorization with such
// operations, except where previously deployed legality hack allowed
// using very low cost values. This is to avoid regressions coming simply
// from moving "masked load/store" check from legality to cost model.
// Masked Load/Gather emulation was previously never allowed.
// Limited number of Masked Store/Scatter emulation was allowed.
assert(isPredicatedInst(I) && "Expecting a scalar emulated instruction")((isPredicatedInst(I) && "Expecting a scalar emulated instruction"
) ? static_cast<void> (0) : __assert_fail ("isPredicatedInst(I) && \"Expecting a scalar emulated instruction\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 6541, __PRETTY_FUNCTION__));
return isa<LoadInst>(I) ||
       (isa<StoreInst>(I) &&
        NumPredStores > NumberOfStoresToPredicate);
6545}

6547void LoopVectorizationCostModel::collectInstsToScalarize(ElementCount VF) {
// If we aren't vectorizing the loop, or if we've already collected the
// instructions to scalarize, there's nothing to do. Collection may already
// have occurred if we have a user-selected VF and are now computing the
// expected cost for interleaving.
if (VF.isScalar() || VF.isZero() ||
    InstsToScalarize.find(VF) != InstsToScalarize.end())
  return;

// Initialize a mapping for VF in InstsToScalalarize. If we find that it's
// not profitable to scalarize any instructions, the presence of VF in the
// map will indicate that we've analyzed it already.
ScalarCostsTy &ScalarCostsVF = InstsToScalarize[VF];

// Find all the instructions that are scalar with predication in the loop and
// determine if it would be better to not if-convert the blocks they are in.
// If so, we also record the instructions to scalarize.
for (BasicBlock *BB : TheLoop->blocks()) {
  if (!blockNeedsPredication(BB))
    continue;
  for (Instruction &I : *BB)
    if (isScalarWithPredication(&I)) {
      ScalarCostsTy ScalarCosts;
      // Do not apply discount logic if hacked cost is needed
      // for emulated masked memrefs.
      if (!useEmulatedMaskMemRefHack(&I) &&
          computePredInstDiscount(&I, ScalarCosts, VF) >= 0)
        ScalarCostsVF.insert(ScalarCosts.begin(), ScalarCosts.end());
      // Remember that BB will remain after vectorization.
      PredicatedBBsAfterVectorization.insert(BB);
    }
}
6579}

6581int LoopVectorizationCostModel::computePredInstDiscount(
  Instruction *PredInst, ScalarCostsTy &ScalarCosts, ElementCount VF) {
assert(!isUniformAfterVectorization(PredInst, VF) &&((!isUniformAfterVectorization(PredInst, VF) && "Instruction marked uniform-after-vectorization will be predicated"
) ? static_cast<void> (0) : __assert_fail ("!isUniformAfterVectorization(PredInst, VF) && \"Instruction marked uniform-after-vectorization will be predicated\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 6584, __PRETTY_FUNCTION__))
       "Instruction marked uniform-after-vectorization will be predicated")((!isUniformAfterVectorization(PredInst, VF) && "Instruction marked uniform-after-vectorization will be predicated"
) ? static_cast<void> (0) : __assert_fail ("!isUniformAfterVectorization(PredInst, VF) && \"Instruction marked uniform-after-vectorization will be predicated\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 6584, __PRETTY_FUNCTION__));

// Initialize the discount to zero, meaning that the scalar version and the
// vector version cost the same.
InstructionCost Discount = 0;

// Holds instructions to analyze. The instructions we visit are mapped in
// ScalarCosts. Those instructions are the ones that would be scalarized if
// we find that the scalar version costs less.
SmallVector<Instruction *, 8> Worklist;

// Returns true if the given instruction can be scalarized.
auto canBeScalarized = [&](Instruction *I) -> bool {
  // We only attempt to scalarize instructions forming a single-use chain
  // from the original predicated block that would otherwise be vectorized.
  // Although not strictly necessary, we give up on instructions we know will
  // already be scalar to avoid traversing chains that are unlikely to be
  // beneficial.
  if (!I->hasOneUse() || PredInst->getParent() != I->getParent() ||
      isScalarAfterVectorization(I, VF))
    return false;

  // If the instruction is scalar with predication, it will be analyzed
  // separately. We ignore it within the context of PredInst.
  if (isScalarWithPredication(I))
    return false;

  // If any of the instruction's operands are uniform after vectorization,
  // the instruction cannot be scalarized. This prevents, for example, a
  // masked load from being scalarized.
  //
  // We assume we will only emit a value for lane zero of an instruction
  // marked uniform after vectorization, rather than VF identical values.
  // Thus, if we scalarize an instruction that uses a uniform, we would
  // create uses of values corresponding to the lanes we aren't emitting code
  // for. This behavior can be changed by allowing getScalarValue to clone
  // the lane zero values for uniforms rather than asserting.
  for (Use &U : I->operands())
    if (auto *J = dyn_cast<Instruction>(U.get()))
      if (isUniformAfterVectorization(J, VF))
        return false;

  // Otherwise, we can scalarize the instruction.
  return true;
};

// Compute the expected cost discount from scalarizing the entire expression
// feeding the predicated instruction. We currently only consider expressions
// that are single-use instruction chains.
Worklist.push_back(PredInst);
while (!Worklist.empty()) {
  Instruction *I = Worklist.pop_back_val();

  // If we've already analyzed the instruction, there's nothing to do.
  if (ScalarCosts.find(I) != ScalarCosts.end())
    continue;

  // Compute the cost of the vector instruction. Note that this cost already
  // includes the scalarization overhead of the predicated instruction.
  InstructionCost VectorCost = getInstructionCost(I, VF).first;

  // Compute the cost of the scalarized instruction. This cost is the cost of
  // the instruction as if it wasn't if-converted and instead remained in the
  // predicated block. We will scale this cost by block probability after
  // computing the scalarization overhead.
  assert(!VF.isScalable() && "scalable vectors not yet supported.")((!VF.isScalable() && "scalable vectors not yet supported."
) ? static_cast<void> (0) : __assert_fail ("!VF.isScalable() && \"scalable vectors not yet supported.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 6649, __PRETTY_FUNCTION__));
  InstructionCost ScalarCost =
      VF.getKnownMinValue() *
      getInstructionCost(I, ElementCount::getFixed(1)).first;

  // Compute the scalarization overhead of needed insertelement instructions
  // and phi nodes.
  if (isScalarWithPredication(I) && !I->getType()->isVoidTy()) {
    ScalarCost += TTI.getScalarizationOverhead(
        cast<VectorType>(ToVectorTy(I->getType(), VF)),
        APInt::getAllOnesValue(VF.getKnownMinValue()), true, false);
    assert(!VF.isScalable() && "scalable vectors not yet supported.")((!VF.isScalable() && "scalable vectors not yet supported."
) ? static_cast<void> (0) : __assert_fail ("!VF.isScalable() && \"scalable vectors not yet supported.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 6660, __PRETTY_FUNCTION__));
    ScalarCost +=
        VF.getKnownMinValue() *
        TTI.getCFInstrCost(Instruction::PHI, TTI::TCK_RecipThroughput);
  }

  // Compute the scalarization overhead of needed extractelement
  // instructions. For each of the instruction's operands, if the operand can
  // be scalarized, add it to the worklist; otherwise, account for the
  // overhead.
  for (Use &U : I->operands())
    if (auto *J = dyn_cast<Instruction>(U.get())) {
      assert(VectorType::isValidElementType(J->getType()) &&((VectorType::isValidElementType(J->getType()) && "Instruction has non-scalar type"
) ? static_cast<void> (0) : __assert_fail ("VectorType::isValidElementType(J->getType()) && \"Instruction has non-scalar type\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 6673, __PRETTY_FUNCTION__))
             "Instruction has non-scalar type")((VectorType::isValidElementType(J->getType()) && "Instruction has non-scalar type"
) ? static_cast<void> (0) : __assert_fail ("VectorType::isValidElementType(J->getType()) && \"Instruction has non-scalar type\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 6673, __PRETTY_FUNCTION__));
      if (canBeScalarized(J))
        Worklist.push_back(J);
      else if (needsExtract(J, VF)) {
        assert(!VF.isScalable() && "scalable vectors not yet supported.")((!VF.isScalable() && "scalable vectors not yet supported."
) ? static_cast<void> (0) : __assert_fail ("!VF.isScalable() && \"scalable vectors not yet supported.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 6677, __PRETTY_FUNCTION__));
        ScalarCost += TTI.getScalarizationOverhead(
            cast<VectorType>(ToVectorTy(J->getType(), VF)),
            APInt::getAllOnesValue(VF.getKnownMinValue()), false, true);
      }
    }

  // Scale the total scalar cost by block probability.
  ScalarCost /= getReciprocalPredBlockProb();

  // Compute the discount. A non-negative discount means the vector version
  // of the instruction costs more, and scalarizing would be beneficial.
  Discount += VectorCost - ScalarCost;
  ScalarCosts[I] = ScalarCost;
}

return *Discount.getValue();
6694}

6696LoopVectorizationCostModel::VectorizationCostTy
6697LoopVectorizationCostModel::expectedCost(ElementCount VF) {
VectorizationCostTy Cost;

// For each block.
for (BasicBlock *BB : TheLoop->blocks()) {
  VectorizationCostTy BlockCost;

  // For each instruction in the old loop.
  for (Instruction &I : BB->instructionsWithoutDebug()) {
    // Skip ignored values.
    if (ValuesToIgnore.count(&I) ||
        (VF.isVector() && VecValuesToIgnore.count(&I)))
      continue;

    VectorizationCostTy C = getInstructionCost(&I, VF);

    // Check if we should override the cost.
    if (ForceTargetInstructionCost.getNumOccurrences() > 0)
      C.first = InstructionCost(ForceTargetInstructionCost);

    BlockCost.first += C.first;
    BlockCost.second |= C.second;
    LLVM_DEBUG(dbgs() << "LV: Found an estimated cost of " << C.firstdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Found an estimated cost of "
 << C.first << " for VF " << VF << " For instruction: "
 << I << '\n'; } } while (false)
                      << " for VF " << VF << " For instruction: " << Ido { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Found an estimated cost of "
 << C.first << " for VF " << VF << " For instruction: "
 << I << '\n'; } } while (false)
                      << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Found an estimated cost of "
 << C.first << " for VF " << VF << " For instruction: "
 << I << '\n'; } } while (false);
  }

  // If we are vectorizing a predicated block, it will have been
  // if-converted. This means that the block's instructions (aside from
  // stores and instructions that may divide by zero) will now be
  // unconditionally executed. For the scalar case, we may not always execute
  // the predicated block, if it is an if-else block. Thus, scale the block's
  // cost by the probability of executing it. blockNeedsPredication from
  // Legal is used so as to not include all blocks in tail folded loops.
  if (VF.isScalar() && Legal->blockNeedsPredication(BB))
    BlockCost.first /= getReciprocalPredBlockProb();

  Cost.first += BlockCost.first;
  Cost.second |= BlockCost.second;
}

return Cost;
6739}

6741/// Gets Address Access SCEV after verifying that the access pattern
6742/// is loop invariant except the induction variable dependence.
6743///
6744/// This SCEV can be sent to the Target in order to estimate the address
6745/// calculation cost.
6746static const SCEV *getAddressAccessSCEV(
            Value *Ptr,
            LoopVectorizationLegality *Legal,
            PredicatedScalarEvolution &PSE,
            const Loop *TheLoop) {

auto *Gep = dyn_cast<GetElementPtrInst>(Ptr);
if (!Gep)
  return nullptr;

// We are looking for a gep with all loop invariant indices except for one
// which should be an induction variable.
auto SE = PSE.getSE();
unsigned NumOperands = Gep->getNumOperands();
for (unsigned i = 1; i < NumOperands; ++i) {
  Value *Opd = Gep->getOperand(i);
  if (!SE->isLoopInvariant(SE->getSCEV(Opd), TheLoop) &&
      !Legal->isInductionVariable(Opd))
    return nullptr;
}

// Now we know we have a GEP ptr, %inv, %ind, %inv. return the Ptr SCEV.
return PSE.getSCEV(Ptr);
6769}

6771static bool isStrideMul(Instruction *I, LoopVectorizationLegality *Legal) {
return Legal->hasStride(I->getOperand(0)) ||
       Legal->hasStride(I->getOperand(1));
6774}

6776InstructionCost
6777LoopVectorizationCostModel::getMemInstScalarizationCost(Instruction *I,
                                                      ElementCount VF) {
assert(VF.isVector() &&((VF.isVector() && "Scalarization cost of instruction implies vectorization."
) ? static_cast<void> (0) : __assert_fail ("VF.isVector() && \"Scalarization cost of instruction implies vectorization.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 6780, __PRETTY_FUNCTION__))
       "Scalarization cost of instruction implies vectorization.")((VF.isVector() && "Scalarization cost of instruction implies vectorization."
) ? static_cast<void> (0) : __assert_fail ("VF.isVector() && \"Scalarization cost of instruction implies vectorization.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 6780, __PRETTY_FUNCTION__));
assert(!VF.isScalable() && "scalable vectors not yet supported.")((!VF.isScalable() && "scalable vectors not yet supported."
) ? static_cast<void> (0) : __assert_fail ("!VF.isScalable() && \"scalable vectors not yet supported.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 6781, __PRETTY_FUNCTION__));
Type *ValTy = getMemInstValueType(I);
auto SE = PSE.getSE();

unsigned AS = getLoadStoreAddressSpace(I);
Value *Ptr = getLoadStorePointerOperand(I);
Type *PtrTy = ToVectorTy(Ptr->getType(), VF);

// Figure out whether the access is strided and get the stride value
// if it's known in compile time
const SCEV *PtrSCEV = getAddressAccessSCEV(Ptr, Legal, PSE, TheLoop);

// Get the cost of the scalar memory instruction and address computation.
InstructionCost Cost =
    VF.getKnownMinValue() * TTI.getAddressComputationCost(PtrTy, SE, PtrSCEV);

// Don't pass *I here, since it is scalar but will actually be part of a
// vectorized loop where the user of it is a vectorized instruction.
const Align Alignment = getLoadStoreAlignment(I);
Cost += VF.getKnownMinValue() *
        TTI.getMemoryOpCost(I->getOpcode(), ValTy->getScalarType(), Alignment,
                            AS, TTI::TCK_RecipThroughput);

// Get the overhead of the extractelement and insertelement instructions
// we might create due to scalarization.
Cost += getScalarizationOverhead(I, VF);

// If we have a predicated load/store, it will need extra i1 extracts and
// conditional branches, but may not be executed for each vector lane. Scale
// the cost by the probability of executing the predicated block.
if (isPredicatedInst(I)) {
  Cost /= getReciprocalPredBlockProb();

  // Add the cost of an i1 extract and a branch
  auto *Vec_i1Ty =
      VectorType::get(IntegerType::getInt1Ty(ValTy->getContext()), VF);
  Cost += TTI.getScalarizationOverhead(
      Vec_i1Ty, APInt::getAllOnesValue(VF.getKnownMinValue()),
      /*Insert=*/false, /*Extract=*/true);
  Cost += TTI.getCFInstrCost(Instruction::Br, TTI::TCK_RecipThroughput);

  if (useEmulatedMaskMemRefHack(I))
    // Artificially setting to a high enough value to practically disable
    // vectorization with such operations.
    Cost = 3000000;
}

return Cost;
6829}

6831InstructionCost
6832LoopVectorizationCostModel::getConsecutiveMemOpCost(Instruction *I,
                                                  ElementCount VF) {
Type *ValTy = getMemInstValueType(I);
auto *VectorTy = cast<VectorType>(ToVectorTy(ValTy, VF));
Value *Ptr = getLoadStorePointerOperand(I);
unsigned AS = getLoadStoreAddressSpace(I);
int ConsecutiveStride = Legal->isConsecutivePtr(Ptr);
enum TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;

assert((ConsecutiveStride == 1 || ConsecutiveStride == -1) &&(((ConsecutiveStride == 1 || ConsecutiveStride == -1) &&
 "Stride should be 1 or -1 for consecutive memory access") ? static_cast
<void> (0) : __assert_fail ("(ConsecutiveStride == 1 || ConsecutiveStride == -1) && \"Stride should be 1 or -1 for consecutive memory access\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 6842, __PRETTY_FUNCTION__))
       "Stride should be 1 or -1 for consecutive memory access")(((ConsecutiveStride == 1 || ConsecutiveStride == -1) &&
 "Stride should be 1 or -1 for consecutive memory access") ? static_cast
<void> (0) : __assert_fail ("(ConsecutiveStride == 1 || ConsecutiveStride == -1) && \"Stride should be 1 or -1 for consecutive memory access\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 6842, __PRETTY_FUNCTION__));
const Align Alignment = getLoadStoreAlignment(I);
InstructionCost Cost = 0;
if (Legal->isMaskRequired(I))
  Cost += TTI.getMaskedMemoryOpCost(I->getOpcode(), VectorTy, Alignment, AS,
                                    CostKind);
else
  Cost += TTI.getMemoryOpCost(I->getOpcode(), VectorTy, Alignment, AS,
                              CostKind, I);

bool Reverse = ConsecutiveStride < 0;
if (Reverse)
  Cost +=
      TTI.getShuffleCost(TargetTransformInfo::SK_Reverse, VectorTy, None, 0);
return Cost;
6857}

6859InstructionCost
6860LoopVectorizationCostModel::getUniformMemOpCost(Instruction *I,
                                              ElementCount VF) {
assert(Legal->isUniformMemOp(*I))((Legal->isUniformMemOp(*I)) ? static_cast<void> (0)
 : __assert_fail ("Legal->isUniformMemOp(*I)", "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 6862, __PRETTY_FUNCTION__));

Type *ValTy = getMemInstValueType(I);
auto *VectorTy = cast<VectorType>(ToVectorTy(ValTy, VF));
const Align Alignment = getLoadStoreAlignment(I);
unsigned AS = getLoadStoreAddressSpace(I);
enum TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
if (isa<LoadInst>(I)) {
  return TTI.getAddressComputationCost(ValTy) +
         TTI.getMemoryOpCost(Instruction::Load, ValTy, Alignment, AS,
                             CostKind) +
         TTI.getShuffleCost(TargetTransformInfo::SK_Broadcast, VectorTy);
}
StoreInst *SI = cast<StoreInst>(I);

bool isLoopInvariantStoreValue = Legal->isUniform(SI->getValueOperand());
return TTI.getAddressComputationCost(ValTy) +
       TTI.getMemoryOpCost(Instruction::Store, ValTy, Alignment, AS,
                           CostKind) +
       (isLoopInvariantStoreValue
            ? 0
            : TTI.getVectorInstrCost(Instruction::ExtractElement, VectorTy,
                                     VF.getKnownMinValue() - 1));
6885}

6887InstructionCost
6888LoopVectorizationCostModel::getGatherScatterCost(Instruction *I,
                                               ElementCount VF) {
Type *ValTy = getMemInstValueType(I);
auto *VectorTy = cast<VectorType>(ToVectorTy(ValTy, VF));
const Align Alignment = getLoadStoreAlignment(I);
const Value *Ptr = getLoadStorePointerOperand(I);

return TTI.getAddressComputationCost(VectorTy) +
       TTI.getGatherScatterOpCost(
           I->getOpcode(), VectorTy, Ptr, Legal->isMaskRequired(I), Alignment,
           TargetTransformInfo::TCK_RecipThroughput, I);
6899}

6901InstructionCost
6902LoopVectorizationCostModel::getInterleaveGroupCost(Instruction *I,
                                                 ElementCount VF) {
// TODO: Once we have support for interleaving with scalable vectors
// we can calculate the cost properly here.
if (VF.isScalable())
  return InstructionCost::getInvalid();

Type *ValTy = getMemInstValueType(I);
auto *VectorTy = cast<VectorType>(ToVectorTy(ValTy, VF));
unsigned AS = getLoadStoreAddressSpace(I);

auto Group = getInterleavedAccessGroup(I);
assert(Group && "Fail to get an interleaved access group.")((Group && "Fail to get an interleaved access group."
) ? static_cast<void> (0) : __assert_fail ("Group && \"Fail to get an interleaved access group.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 6914, __PRETTY_FUNCTION__));

unsigned InterleaveFactor = Group->getFactor();
auto *WideVecTy = VectorType::get(ValTy, VF * InterleaveFactor);

// Holds the indices of existing members in an interleaved load group.
// An interleaved store group doesn't need this as it doesn't allow gaps.
SmallVector<unsigned, 4> Indices;
if (isa<LoadInst>(I)) {
  for (unsigned i = 0; i < InterleaveFactor; i++)
    if (Group->getMember(i))
      Indices.push_back(i);
}

// Calculate the cost of the whole interleaved group.
bool UseMaskForGaps =
    Group->requiresScalarEpilogue() && !isScalarEpilogueAllowed();
InstructionCost Cost = TTI.getInterleavedMemoryOpCost(
    I->getOpcode(), WideVecTy, Group->getFactor(), Indices, Group->getAlign(),
    AS, TTI::TCK_RecipThroughput, Legal->isMaskRequired(I), UseMaskForGaps);

if (Group->isReverse()) {
  // TODO: Add support for reversed masked interleaved access.
  assert(!Legal->isMaskRequired(I) &&((!Legal->isMaskRequired(I) && "Reverse masked interleaved access not supported."
) ? static_cast<void> (0) : __assert_fail ("!Legal->isMaskRequired(I) && \"Reverse masked interleaved access not supported.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 6938, __PRETTY_FUNCTION__))
         "Reverse masked interleaved access not supported.")((!Legal->isMaskRequired(I) && "Reverse masked interleaved access not supported."
) ? static_cast<void> (0) : __assert_fail ("!Legal->isMaskRequired(I) && \"Reverse masked interleaved access not supported.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 6938, __PRETTY_FUNCTION__));
  Cost +=
      Group->getNumMembers() *
      TTI.getShuffleCost(TargetTransformInfo::SK_Reverse, VectorTy, None, 0);
}
return Cost;
6944}

6946InstructionCost LoopVectorizationCostModel::getReductionPatternCost(
  Instruction *I, ElementCount VF, Type *Ty, TTI::TargetCostKind CostKind) {
// Early exit for no inloop reductions
if (InLoopReductionChains.empty() || VF.isScalar() || !isa<VectorType>(Ty))
  return InstructionCost::getInvalid();
auto *VectorTy = cast<VectorType>(Ty);

// We are looking for a pattern of, and finding the minimal acceptable cost:
//  reduce(mul(ext(A), ext(B))) or
//  reduce(mul(A, B)) or
//  reduce(ext(A)) or
//  reduce(A).
// The basic idea is that we walk down the tree to do that, finding the root
// reduction instruction in InLoopReductionImmediateChains. From there we find
// the pattern of mul/ext and test the cost of the entire pattern vs the cost
// of the components. If the reduction cost is lower then we return it for the
// reduction instruction and 0 for the other instructions in the pattern. If
// it is not we return an invalid cost specifying the orignal cost method
// should be used.
Instruction *RetI = I;
if ((RetI->getOpcode() == Instruction::SExt ||
     RetI->getOpcode() == Instruction::ZExt)) {
  if (!RetI->hasOneUser())
    return InstructionCost::getInvalid();
  RetI = RetI->user_back();
}
if (RetI->getOpcode() == Instruction::Mul &&
    RetI->user_back()->getOpcode() == Instruction::Add) {
  if (!RetI->hasOneUser())
    return InstructionCost::getInvalid();
  RetI = RetI->user_back();
}

// Test if the found instruction is a reduction, and if not return an invalid
// cost specifying the parent to use the original cost modelling.
if (!InLoopReductionImmediateChains.count(RetI))
  return InstructionCost::getInvalid();

// Find the reduction this chain is a part of and calculate the basic cost of
// the reduction on its own.
Instruction *LastChain = InLoopReductionImmediateChains[RetI];
Instruction *ReductionPhi = LastChain;
while (!isa<PHINode>(ReductionPhi))
  ReductionPhi = InLoopReductionImmediateChains[ReductionPhi];

RecurrenceDescriptor RdxDesc =
    Legal->getReductionVars()[cast<PHINode>(ReductionPhi)];
InstructionCost BaseCost = TTI.getArithmeticReductionCost(
    RdxDesc.getOpcode(), VectorTy, false, CostKind);

// Get the operand that was not the reduction chain and match it to one of the
// patterns, returning the better cost if it is found.
Instruction *RedOp = RetI->getOperand(1) == LastChain
                         ? dyn_cast<Instruction>(RetI->getOperand(0))
                         : dyn_cast<Instruction>(RetI->getOperand(1));

VectorTy = VectorType::get(I->getOperand(0)->getType(), VectorTy);

if (RedOp && (isa<SExtInst>(RedOp) || isa<ZExtInst>(RedOp)) &&
    !TheLoop->isLoopInvariant(RedOp)) {
  bool IsUnsigned = isa<ZExtInst>(RedOp);
  auto *ExtType = VectorType::get(RedOp->getOperand(0)->getType(), VectorTy);
  InstructionCost RedCost = TTI.getExtendedAddReductionCost(
      /*IsMLA=*/false, IsUnsigned, RdxDesc.getRecurrenceType(), ExtType,
      CostKind);

  InstructionCost ExtCost =
      TTI.getCastInstrCost(RedOp->getOpcode(), VectorTy, ExtType,
                           TTI::CastContextHint::None, CostKind, RedOp);
  if (RedCost.isValid() && RedCost < BaseCost + ExtCost)
    return I == RetI ? *RedCost.getValue() : 0;
} else if (RedOp && RedOp->getOpcode() == Instruction::Mul) {
  Instruction *Mul = RedOp;
  Instruction *Op0 = dyn_cast<Instruction>(Mul->getOperand(0));
  Instruction *Op1 = dyn_cast<Instruction>(Mul->getOperand(1));
  if (Op0 && Op1 && (isa<SExtInst>(Op0) || isa<ZExtInst>(Op0)) &&
      Op0->getOpcode() == Op1->getOpcode() &&
      Op0->getOperand(0)->getType() == Op1->getOperand(0)->getType() &&
      !TheLoop->isLoopInvariant(Op0) && !TheLoop->isLoopInvariant(Op1)) {
    bool IsUnsigned = isa<ZExtInst>(Op0);
    auto *ExtType = VectorType::get(Op0->getOperand(0)->getType(), VectorTy);
    // reduce(mul(ext, ext))
    InstructionCost ExtCost =
        TTI.getCastInstrCost(Op0->getOpcode(), VectorTy, ExtType,
                             TTI::CastContextHint::None, CostKind, Op0);
    InstructionCost MulCost =
        TTI.getArithmeticInstrCost(Mul->getOpcode(), VectorTy, CostKind);

    InstructionCost RedCost = TTI.getExtendedAddReductionCost(
        /*IsMLA=*/true, IsUnsigned, RdxDesc.getRecurrenceType(), ExtType,
        CostKind);

    if (RedCost.isValid() && RedCost < ExtCost * 2 + MulCost + BaseCost)
      return I == RetI ? *RedCost.getValue() : 0;
  } else {
    InstructionCost MulCost =
        TTI.getArithmeticInstrCost(Mul->getOpcode(), VectorTy, CostKind);

    InstructionCost RedCost = TTI.getExtendedAddReductionCost(
        /*IsMLA=*/true, true, RdxDesc.getRecurrenceType(), VectorTy,
        CostKind);

    if (RedCost.isValid() && RedCost < MulCost + BaseCost)
      return I == RetI ? *RedCost.getValue() : 0;
  }
}

return I == RetI ? BaseCost : InstructionCost::getInvalid();
7054}

7056InstructionCost
7057LoopVectorizationCostModel::getMemoryInstructionCost(Instruction *I,
                                                   ElementCount VF) {
// Calculate scalar cost only. Vectorization cost should be ready at this
// moment.
if (VF.isScalar()) {
  Type *ValTy = getMemInstValueType(I);
  const Align Alignment = getLoadStoreAlignment(I);
  unsigned AS = getLoadStoreAddressSpace(I);

  return TTI.getAddressComputationCost(ValTy) +
         TTI.getMemoryOpCost(I->getOpcode(), ValTy, Alignment, AS,
                             TTI::TCK_RecipThroughput, I);
}
return getWideningCost(I, VF);
7071}

7073LoopVectorizationCostModel::VectorizationCostTy
7074LoopVectorizationCostModel::getInstructionCost(Instruction *I,
                                             ElementCount VF) {
// If we know that this instruction will remain uniform, check the cost of
// the scalar version.
if (isUniformAfterVectorization(I, VF))
  VF = ElementCount::getFixed(1);

if (VF.isVector() && isProfitableToScalarize(I, VF))
  return VectorizationCostTy(InstsToScalarize[VF][I], false);

// Forced scalars do not have any scalarization overhead.
auto ForcedScalar = ForcedScalars.find(VF);
if (VF.isVector() && ForcedScalar != ForcedScalars.end()) {
  auto InstSet = ForcedScalar->second;
  if (InstSet.count(I))
    return VectorizationCostTy(
        (getInstructionCost(I, ElementCount::getFixed(1)).first *
         VF.getKnownMinValue()),
        false);
}

Type *VectorTy;
InstructionCost C = getInstructionCost(I, VF, VectorTy);

bool TypeNotScalarized =
    VF.isVector() && VectorTy->isVectorTy() &&
    TTI.getNumberOfParts(VectorTy) < VF.getKnownMinValue();
return VectorizationCostTy(C, TypeNotScalarized);
7102}

7104InstructionCost
7105LoopVectorizationCostModel::getScalarizationOverhead(Instruction *I,
                                                   ElementCount VF) const {

if (VF.isScalable())
  return InstructionCost::getInvalid();

if (VF.isScalar())
  return 0;

InstructionCost Cost = 0;
Type *RetTy = ToVectorTy(I->getType(), VF);
if (!RetTy->isVoidTy() &&
    (!isa<LoadInst>(I) || !TTI.supportsEfficientVectorElementLoadStore()))
  Cost += TTI.getScalarizationOverhead(
      cast<VectorType>(RetTy), APInt::getAllOnesValue(VF.getKnownMinValue()),
      true, false);

// Some targets keep addresses scalar.
if (isa<LoadInst>(I) && !TTI.prefersVectorizedAddressing())
  return Cost;

// Some targets support efficient element stores.
if (isa<StoreInst>(I) && TTI.supportsEfficientVectorElementLoadStore())
  return Cost;

// Collect operands to consider.
CallInst *CI = dyn_cast<CallInst>(I);
Instruction::op_range Ops = CI ? CI->arg_operands() : I->operands();

// Skip operands that do not require extraction/scalarization and do not incur
// any overhead.
SmallVector<Type *> Tys;
for (auto *V : filterExtractingOperands(Ops, VF))
  Tys.push_back(MaybeVectorizeType(V->getType(), VF));
return Cost + TTI.getOperandsScalarizationOverhead(
                  filterExtractingOperands(Ops, VF), Tys);
7141}

7143void LoopVectorizationCostModel::setCostBasedWideningDecision(ElementCount VF) {
if (VF.isScalar())
  return;
NumPredStores = 0;
for (BasicBlock *BB : TheLoop->blocks()) {
  // For each instruction in the old loop.
  for (Instruction &I : *BB) {
    Value *Ptr =  getLoadStorePointerOperand(&I);
    if (!Ptr)
      continue;

    // TODO: We should generate better code and update the cost model for
    // predicated uniform stores. Today they are treated as any other
    // predicated store (see added test cases in
    // invariant-store-vectorization.ll).
    if (isa<StoreInst>(&I) && isScalarWithPredication(&I))
      NumPredStores++;

    if (Legal->isUniformMemOp(I)) {
      // TODO: Avoid replicating loads and stores instead of
      // relying on instcombine to remove them.
      // Load: Scalar load + broadcast
      // Store: Scalar store + isLoopInvariantStoreValue ? 0 : extract
      InstructionCost Cost = getUniformMemOpCost(&I, VF);
      setWideningDecision(&I, VF, CM_Scalarize, Cost);
      continue;
    }

    // We assume that widening is the best solution when possible.
    if (memoryInstructionCanBeWidened(&I, VF)) {
      InstructionCost Cost = getConsecutiveMemOpCost(&I, VF);
      int ConsecutiveStride =
             Legal->isConsecutivePtr(getLoadStorePointerOperand(&I));
      assert((ConsecutiveStride == 1 || ConsecutiveStride == -1) &&(((ConsecutiveStride == 1 || ConsecutiveStride == -1) &&
 "Expected consecutive stride.") ? static_cast<void> (0
) : __assert_fail ("(ConsecutiveStride == 1 || ConsecutiveStride == -1) && \"Expected consecutive stride.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 7177, __PRETTY_FUNCTION__))
             "Expected consecutive stride.")(((ConsecutiveStride == 1 || ConsecutiveStride == -1) &&
 "Expected consecutive stride.") ? static_cast<void> (0
) : __assert_fail ("(ConsecutiveStride == 1 || ConsecutiveStride == -1) && \"Expected consecutive stride.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 7177, __PRETTY_FUNCTION__));
      InstWidening Decision =
          ConsecutiveStride == 1 ? CM_Widen : CM_Widen_Reverse;
      setWideningDecision(&I, VF, Decision, Cost);
      continue;
    }

    // Choose between Interleaving, Gather/Scatter or Scalarization.
    InstructionCost InterleaveCost = InstructionCost::getInvalid();
    unsigned NumAccesses = 1;
    if (isAccessInterleaved(&I)) {
      auto Group = getInterleavedAccessGroup(&I);
      assert(Group && "Fail to get an interleaved access group.")((Group && "Fail to get an interleaved access group."
) ? static_cast<void> (0) : __assert_fail ("Group && \"Fail to get an interleaved access group.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 7189, __PRETTY_FUNCTION__));

      // Make one decision for the whole group.
      if (getWideningDecision(&I, VF) != CM_Unknown)
        continue;

      NumAccesses = Group->getNumMembers();
      if (interleavedAccessCanBeWidened(&I, VF))
        InterleaveCost = getInterleaveGroupCost(&I, VF);
    }

    InstructionCost GatherScatterCost =
        isLegalGatherOrScatter(&I)
            ? getGatherScatterCost(&I, VF) * NumAccesses
            : InstructionCost::getInvalid();

    InstructionCost ScalarizationCost =
        !VF.isScalable() ? getMemInstScalarizationCost(&I, VF) * NumAccesses
                         : InstructionCost::getInvalid();

    // Choose better solution for the current VF,
    // write down this decision and use it during vectorization.
    InstructionCost Cost;
    InstWidening Decision;
    if (InterleaveCost <= GatherScatterCost &&
        InterleaveCost < ScalarizationCost) {
      Decision = CM_Interleave;
      Cost = InterleaveCost;
    } else if (GatherScatterCost < ScalarizationCost) {
      Decision = CM_GatherScatter;
      Cost = GatherScatterCost;
    } else {
      assert(!VF.isScalable() &&((!VF.isScalable() && "We cannot yet scalarise for scalable vectors"
) ? static_cast<void> (0) : __assert_fail ("!VF.isScalable() && \"We cannot yet scalarise for scalable vectors\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 7222, __PRETTY_FUNCTION__))
             "We cannot yet scalarise for scalable vectors")((!VF.isScalable() && "We cannot yet scalarise for scalable vectors"
) ? static_cast<void> (0) : __assert_fail ("!VF.isScalable() && \"We cannot yet scalarise for scalable vectors\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 7222, __PRETTY_FUNCTION__));
      Decision = CM_Scalarize;
      Cost = ScalarizationCost;
    }
    // If the instructions belongs to an interleave group, the whole group
    // receives the same decision. The whole group receives the cost, but
    // the cost will actually be assigned to one instruction.
    if (auto Group = getInterleavedAccessGroup(&I))
      setWideningDecision(Group, VF, Decision, Cost);
    else
      setWideningDecision(&I, VF, Decision, Cost);
  }
}

// Make sure that any load of address and any other address computation
// remains scalar unless there is gather/scatter support. This avoids
// inevitable extracts into address registers, and also has the benefit of
// activating LSR more, since that pass can't optimize vectorized
// addresses.
if (TTI.prefersVectorizedAddressing())
  return;

// Start with all scalar pointer uses.
SmallPtrSet<Instruction *, 8> AddrDefs;
for (BasicBlock *BB : TheLoop->blocks())
  for (Instruction &I : *BB) {
    Instruction *PtrDef =
      dyn_cast_or_null<Instruction>(getLoadStorePointerOperand(&I));
    if (PtrDef && TheLoop->contains(PtrDef) &&
        getWideningDecision(&I, VF) != CM_GatherScatter)
      AddrDefs.insert(PtrDef);
  }

// Add all instructions used to generate the addresses.
SmallVector<Instruction *, 4> Worklist;
append_range(Worklist, AddrDefs);
while (!Worklist.empty()) {
  Instruction *I = Worklist.pop_back_val();
  for (auto &Op : I->operands())
    if (auto *InstOp = dyn_cast<Instruction>(Op))
      if ((InstOp->getParent() == I->getParent()) && !isa<PHINode>(InstOp) &&
          AddrDefs.insert(InstOp).second)
        Worklist.push_back(InstOp);
}

for (auto *I : AddrDefs) {
  if (isa<LoadInst>(I)) {
    // Setting the desired widening decision should ideally be handled in
    // by cost functions, but since this involves the task of finding out
    // if the loaded register is involved in an address computation, it is
    // instead changed here when we know this is the case.
    InstWidening Decision = getWideningDecision(I, VF);
    if (Decision == CM_Widen || Decision == CM_Widen_Reverse)
      // Scalarize a widened load of address.
      setWideningDecision(
          I, VF, CM_Scalarize,
          (VF.getKnownMinValue() *
           getMemoryInstructionCost(I, ElementCount::getFixed(1))));
    else if (auto Group = getInterleavedAccessGroup(I)) {
      // Scalarize an interleave group of address loads.
      for (unsigned I = 0; I < Group->getFactor(); ++I) {
        if (Instruction *Member = Group->getMember(I))
          setWideningDecision(
              Member, VF, CM_Scalarize,
              (VF.getKnownMinValue() *
               getMemoryInstructionCost(Member, ElementCount::getFixed(1))));
      }
    }
  } else
    // Make sure I gets scalarized and a cost estimate without
    // scalarization overhead.
    ForcedScalars[VF].insert(I);
}
7295}

7297InstructionCost
7298LoopVectorizationCostModel::getInstructionCost(Instruction *I, ElementCount VF,
                                             Type *&VectorTy) {
Type *RetTy = I->getType();
if (canTruncateToMinimalBitwidth(I, VF))
  RetTy = IntegerType::get(RetTy->getContext(), MinBWs[I]);
VectorTy = isScalarAfterVectorization(I, VF) ? RetTy : ToVectorTy(RetTy, VF);
auto SE = PSE.getSE();
TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;

// TODO: We need to estimate the cost of intrinsic calls.
switch (I->getOpcode()) {
case Instruction::GetElementPtr:
  // We mark this instruction as zero-cost because the cost of GEPs in
  // vectorized code depends on whether the corresponding memory instruction
  // is scalarized or not. Therefore, we handle GEPs with the memory
  // instruction cost.
  return 0;
case Instruction::Br: {
  // In cases of scalarized and predicated instructions, there will be VF
  // predicated blocks in the vectorized loop. Each branch around these
  // blocks requires also an extract of its vector compare i1 element.
  bool ScalarPredicatedBB = false;
  BranchInst *BI = cast<BranchInst>(I);
  if (VF.isVector() && BI->isConditional() &&
      (PredicatedBBsAfterVectorization.count(BI->getSuccessor(0)) ||
       PredicatedBBsAfterVectorization.count(BI->getSuccessor(1))))
    ScalarPredicatedBB = true;

  if (ScalarPredicatedBB) {
    // Return cost for branches around scalarized and predicated blocks.
    assert(!VF.isScalable() && "scalable vectors not yet supported.")((!VF.isScalable() && "scalable vectors not yet supported."
) ? static_cast<void> (0) : __assert_fail ("!VF.isScalable() && \"scalable vectors not yet supported.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 7328, __PRETTY_FUNCTION__));
    auto *Vec_i1Ty =
        VectorType::get(IntegerType::getInt1Ty(RetTy->getContext()), VF);
    return (TTI.getScalarizationOverhead(
                Vec_i1Ty, APInt::getAllOnesValue(VF.getKnownMinValue()),
                false, true) +
            (TTI.getCFInstrCost(Instruction::Br, CostKind) *
             VF.getKnownMinValue()));
  } else if (I->getParent() == TheLoop->getLoopLatch() || VF.isScalar())
    // The back-edge branch will remain, as will all scalar branches.
    return TTI.getCFInstrCost(Instruction::Br, CostKind);
  else
    // This branch will be eliminated by if-conversion.
    return 0;
  // Note: We currently assume zero cost for an unconditional branch inside
  // a predicated block since it will become a fall-through, although we
  // may decide in the future to call TTI for all branches.
}
case Instruction::PHI: {
  auto *Phi = cast<PHINode>(I);

  // First-order recurrences are replaced by vector shuffles inside the loop.
  // NOTE: Don't use ToVectorTy as SK_ExtractSubvector expects a vector type.
  if (VF.isVector() && Legal->isFirstOrderRecurrence(Phi))
    return TTI.getShuffleCost(
        TargetTransformInfo::SK_ExtractSubvector, cast<VectorType>(VectorTy),
        None, VF.getKnownMinValue() - 1, FixedVectorType::get(RetTy, 1));

  // Phi nodes in non-header blocks (not inductions, reductions, etc.) are
  // converted into select instructions. We require N - 1 selects per phi
  // node, where N is the number of incoming values.
  if (VF.isVector() && Phi->getParent() != TheLoop->getHeader())
    return (Phi->getNumIncomingValues() - 1) *
           TTI.getCmpSelInstrCost(
               Instruction::Select, ToVectorTy(Phi->getType(), VF),
               ToVectorTy(Type::getInt1Ty(Phi->getContext()), VF),
               CmpInst::BAD_ICMP_PREDICATE, CostKind);

  return TTI.getCFInstrCost(Instruction::PHI, CostKind);
}
case Instruction::UDiv:
case Instruction::SDiv:
case Instruction::URem:
case Instruction::SRem:
  // If we have a predicated instruction, it may not be executed for each
  // vector lane. Get the scalarization cost and scale this amount by the
  // probability of executing the predicated block. If the instruction is not
  // predicated, we fall through to the next case.
  if (VF.isVector() && isScalarWithPredication(I)) {
    InstructionCost Cost = 0;

    // These instructions have a non-void type, so account for the phi nodes
    // that we will create. This cost is likely to be zero. The phi node
    // cost, if any, should be scaled by the block probability because it
    // models a copy at the end of each predicated block.
    Cost += VF.getKnownMinValue() *
            TTI.getCFInstrCost(Instruction::PHI, CostKind);

    // The cost of the non-predicated instruction.
    Cost += VF.getKnownMinValue() *
            TTI.getArithmeticInstrCost(I->getOpcode(), RetTy, CostKind);

    // The cost of insertelement and extractelement instructions needed for
    // scalarization.
    Cost += getScalarizationOverhead(I, VF);

    // Scale the cost by the probability of executing the predicated blocks.
    // This assumes the predicated block for each vector lane is equally
    // likely.
    return Cost / getReciprocalPredBlockProb();
  }
  LLVM_FALLTHROUGH[[gnu::fallthrough]];
case Instruction::Add:
case Instruction::FAdd:
case Instruction::Sub:
case Instruction::FSub:
case Instruction::Mul:
case Instruction::FMul:
case Instruction::FDiv:
case Instruction::FRem:
case Instruction::Shl:
case Instruction::LShr:
case Instruction::AShr:
case Instruction::And:
case Instruction::Or:
case Instruction::Xor: {
  // Since we will replace the stride by 1 the multiplication should go away.
  if (I->getOpcode() == Instruction::Mul && isStrideMul(I, Legal))
    return 0;

  // Detect reduction patterns
  InstructionCost RedCost;
  if ((RedCost = getReductionPatternCost(I, VF, VectorTy, CostKind))
          .isValid())
    return RedCost;

  // Certain instructions can be cheaper to vectorize if they have a constant
  // second vector operand. One example of this are shifts on x86.
  Value *Op2 = I->getOperand(1);
  TargetTransformInfo::OperandValueProperties Op2VP;
  TargetTransformInfo::OperandValueKind Op2VK =
      TTI.getOperandInfo(Op2, Op2VP);
  if (Op2VK == TargetTransformInfo::OK_AnyValue && Legal->isUniform(Op2))
    Op2VK = TargetTransformInfo::OK_UniformValue;

  SmallVector<const Value *, 4> Operands(I->operand_values());
  unsigned N = isScalarAfterVectorization(I, VF) ? VF.getKnownMinValue() : 1;
  return N * TTI.getArithmeticInstrCost(
                 I->getOpcode(), VectorTy, CostKind,
                 TargetTransformInfo::OK_AnyValue,
                 Op2VK, TargetTransformInfo::OP_None, Op2VP, Operands, I);
}
case Instruction::FNeg: {
  assert(!VF.isScalable() && "VF is assumed to be non scalable.")((!VF.isScalable() && "VF is assumed to be non scalable."
) ? static_cast<void> (0) : __assert_fail ("!VF.isScalable() && \"VF is assumed to be non scalable.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 7441, __PRETTY_FUNCTION__));
  unsigned N = isScalarAfterVectorization(I, VF) ? VF.getKnownMinValue() : 1;
  return N * TTI.getArithmeticInstrCost(
                 I->getOpcode(), VectorTy, CostKind,
                 TargetTransformInfo::OK_AnyValue,
                 TargetTransformInfo::OK_AnyValue,
                 TargetTransformInfo::OP_None, TargetTransformInfo::OP_None,
                 I->getOperand(0), I);
}
case Instruction::Select: {
  SelectInst *SI = cast<SelectInst>(I);
  const SCEV *CondSCEV = SE->getSCEV(SI->getCondition());
  bool ScalarCond = (SE->isLoopInvariant(CondSCEV, TheLoop));

  const Value *Op0, *Op1;
  using namespace llvm::PatternMatch;
  if (!ScalarCond && (match(I, m_LogicalAnd(m_Value(Op0), m_Value(Op1))) ||
                      match(I, m_LogicalOr(m_Value(Op0), m_Value(Op1))))) {
    // select x, y, false --> x & y
    // select x, true, y --> x | y
    TTI::OperandValueProperties Op1VP = TTI::OP_None;
    TTI::OperandValueProperties Op2VP = TTI::OP_None;
    TTI::OperandValueKind Op1VK = TTI::getOperandInfo(Op0, Op1VP);
    TTI::OperandValueKind Op2VK = TTI::getOperandInfo(Op1, Op2VP);
    assert(Op0->getType()->getScalarSizeInBits() == 1 &&((Op0->getType()->getScalarSizeInBits() == 1 &&
 Op1->getType()->getScalarSizeInBits() == 1) ? static_cast
<void> (0) : __assert_fail ("Op0->getType()->getScalarSizeInBits() == 1 && Op1->getType()->getScalarSizeInBits() == 1"
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 7466, __PRETTY_FUNCTION__))
            Op1->getType()->getScalarSizeInBits() == 1)((Op0->getType()->getScalarSizeInBits() == 1 &&
 Op1->getType()->getScalarSizeInBits() == 1) ? static_cast
<void> (0) : __assert_fail ("Op0->getType()->getScalarSizeInBits() == 1 && Op1->getType()->getScalarSizeInBits() == 1"
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 7466, __PRETTY_FUNCTION__));

    SmallVector<const Value *, 2> Operands{Op0, Op1};
    return TTI.getArithmeticInstrCost(
        match(I, m_LogicalOr()) ? Instruction::Or : Instruction::And, VectorTy,
        CostKind, Op1VK, Op2VK, Op1VP, Op2VP, Operands, I);
  }

  Type *CondTy = SI->getCondition()->getType();
  if (!ScalarCond)
    CondTy = VectorType::get(CondTy, VF);
  return TTI.getCmpSelInstrCost(I->getOpcode(), VectorTy, CondTy,
                                CmpInst::BAD_ICMP_PREDICATE, CostKind, I);
}
case Instruction::ICmp:
case Instruction::FCmp: {
  Type *ValTy = I->getOperand(0)->getType();
  Instruction *Op0AsInstruction = dyn_cast<Instruction>(I->getOperand(0));
  if (canTruncateToMinimalBitwidth(Op0AsInstruction, VF))
    ValTy = IntegerType::get(ValTy->getContext(), MinBWs[Op0AsInstruction]);
  VectorTy = ToVectorTy(ValTy, VF);
  return TTI.getCmpSelInstrCost(I->getOpcode(), VectorTy, nullptr,
                                CmpInst::BAD_ICMP_PREDICATE, CostKind, I);
}
case Instruction::Store:
case Instruction::Load: {
  ElementCount Width = VF;
  if (Width.isVector()) {
    InstWidening Decision = getWideningDecision(I, Width);
    assert(Decision != CM_Unknown &&((Decision != CM_Unknown && "CM decision should be taken at this point"
) ? static_cast<void> (0) : __assert_fail ("Decision != CM_Unknown && \"CM decision should be taken at this point\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 7496, __PRETTY_FUNCTION__))
           "CM decision should be taken at this point")((Decision != CM_Unknown && "CM decision should be taken at this point"
) ? static_cast<void> (0) : __assert_fail ("Decision != CM_Unknown && \"CM decision should be taken at this point\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 7496, __PRETTY_FUNCTION__));
    if (Decision == CM_Scalarize)
      Width = ElementCount::getFixed(1);
  }
  VectorTy = ToVectorTy(getMemInstValueType(I), Width);
  return getMemoryInstructionCost(I, VF);
}
case Instruction::ZExt:
case Instruction::SExt:
case Instruction::FPToUI:
case Instruction::FPToSI:
case Instruction::FPExt:
case Instruction::PtrToInt:
case Instruction::IntToPtr:
case Instruction::SIToFP:
case Instruction::UIToFP:
case Instruction::Trunc:
case Instruction::FPTrunc:
case Instruction::BitCast: {
  // Computes the CastContextHint from a Load/Store instruction.
  auto ComputeCCH = [&](Instruction *I) -> TTI::CastContextHint {
    assert((isa<LoadInst>(I) || isa<StoreInst>(I)) &&(((isa<LoadInst>(I) || isa<StoreInst>(I)) &&
 "Expected a load or a store!") ? static_cast<void> (0)
 : __assert_fail ("(isa<LoadInst>(I) || isa<StoreInst>(I)) && \"Expected a load or a store!\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 7518, __PRETTY_FUNCTION__))
           "Expected a load or a store!")(((isa<LoadInst>(I) || isa<StoreInst>(I)) &&
 "Expected a load or a store!") ? static_cast<void> (0)
 : __assert_fail ("(isa<LoadInst>(I) || isa<StoreInst>(I)) && \"Expected a load or a store!\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 7518, __PRETTY_FUNCTION__));

    if (VF.isScalar() || !TheLoop->contains(I))
      return TTI::CastContextHint::Normal;

    switch (getWideningDecision(I, VF)) {
    case LoopVectorizationCostModel::CM_GatherScatter:
      return TTI::CastContextHint::GatherScatter;
    case LoopVectorizationCostModel::CM_Interleave:
      return TTI::CastContextHint::Interleave;
    case LoopVectorizationCostModel::CM_Scalarize:
    case LoopVectorizationCostModel::CM_Widen:
      return Legal->isMaskRequired(I) ? TTI::CastContextHint::Masked
                                      : TTI::CastContextHint::Normal;
    case LoopVectorizationCostModel::CM_Widen_Reverse:
      return TTI::CastContextHint::Reversed;
    case LoopVectorizationCostModel::CM_Unknown:
      llvm_unreachable("Instr did not go through cost modelling?")::llvm::llvm_unreachable_internal("Instr did not go through cost modelling?"
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 7535);
    }

    llvm_unreachable("Unhandled case!")::llvm::llvm_unreachable_internal("Unhandled case!", "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 7538);
  };

  unsigned Opcode = I->getOpcode();
  TTI::CastContextHint CCH = TTI::CastContextHint::None;
  // For Trunc, the context is the only user, which must be a StoreInst.
  if (Opcode == Instruction::Trunc || Opcode == Instruction::FPTrunc) {
    if (I->hasOneUse())
      if (StoreInst *Store = dyn_cast<StoreInst>(*I->user_begin()))
        CCH = ComputeCCH(Store);
  }
  // For Z/Sext, the context is the operand, which must be a LoadInst.
  else if (Opcode == Instruction::ZExt || Opcode == Instruction::SExt ||
           Opcode == Instruction::FPExt) {
    if (LoadInst *Load = dyn_cast<LoadInst>(I->getOperand(0)))
      CCH = ComputeCCH(Load);
  }

  // We optimize the truncation of induction variables having constant
  // integer steps. The cost of these truncations is the same as the scalar
  // operation.
  if (isOptimizableIVTruncate(I, VF)) {
    auto *Trunc = cast<TruncInst>(I);
    return TTI.getCastInstrCost(Instruction::Trunc, Trunc->getDestTy(),
                                Trunc->getSrcTy(), CCH, CostKind, Trunc);
  }

  // Detect reduction patterns
  InstructionCost RedCost;
  if ((RedCost = getReductionPatternCost(I, VF, VectorTy, CostKind))
          .isValid())
    return RedCost;

  Type *SrcScalarTy = I->getOperand(0)->getType();
  Type *SrcVecTy =
      VectorTy->isVectorTy() ? ToVectorTy(SrcScalarTy, VF) : SrcScalarTy;
  if (canTruncateToMinimalBitwidth(I, VF)) {
    // This cast is going to be shrunk. This may remove the cast or it might
    // turn it into slightly different cast. For example, if MinBW == 16,
    // "zext i8 %1 to i32" becomes "zext i8 %1 to i16".
    //
    // Calculate the modified src and dest types.
    Type *MinVecTy = VectorTy;
    if (Opcode == Instruction::Trunc) {
      SrcVecTy = smallestIntegerVectorType(SrcVecTy, MinVecTy);
      VectorTy =
          largestIntegerVectorType(ToVectorTy(I->getType(), VF), MinVecTy);
    } else if (Opcode == Instruction::ZExt || Opcode == Instruction::SExt) {
      SrcVecTy = largestIntegerVectorType(SrcVecTy, MinVecTy);
      VectorTy =
          smallestIntegerVectorType(ToVectorTy(I->getType(), VF), MinVecTy);
    }
  }

  unsigned N;
  if (isScalarAfterVectorization(I, VF)) {
    assert(!VF.isScalable() && "VF is assumed to be non scalable")((!VF.isScalable() && "VF is assumed to be non scalable"
) ? static_cast<void> (0) : __assert_fail ("!VF.isScalable() && \"VF is assumed to be non scalable\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 7594, __PRETTY_FUNCTION__));
    N = VF.getKnownMinValue();
  } else
    N = 1;
  return N *
         TTI.getCastInstrCost(Opcode, VectorTy, SrcVecTy, CCH, CostKind, I);
}
case Instruction::Call: {
  bool NeedToScalarize;
  CallInst *CI = cast<CallInst>(I);
  InstructionCost CallCost = getVectorCallCost(CI, VF, NeedToScalarize);
  if (getVectorIntrinsicIDForCall(CI, TLI)) {
    InstructionCost IntrinsicCost = getVectorIntrinsicCost(CI, VF);
    return std::min(CallCost, IntrinsicCost);
  }
  return CallCost;
}
case Instruction::ExtractValue:
  return TTI.getInstructionCost(I, TTI::TCK_RecipThroughput);
default:
  // The cost of executing VF copies of the scalar instruction. This opcode
  // is unknown. Assume that it is the same as 'mul'.
  return VF.getKnownMinValue() * TTI.getArithmeticInstrCost(
                                     Instruction::Mul, VectorTy, CostKind) +
         getScalarizationOverhead(I, VF);
} // end of switch.
7620}

7622char LoopVectorize::ID = 0;

7624static const char lv_name[] = "Loop Vectorization";

7626INITIALIZE_PASS_BEGIN(LoopVectorize, LV_NAME, lv_name, false, false)static void *initializeLoopVectorizePassOnce(PassRegistry &
Registry) {
7627INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)initializeTargetTransformInfoWrapperPassPass(Registry);
7628INITIALIZE_PASS_DEPENDENCY(BasicAAWrapperPass)initializeBasicAAWrapperPassPass(Registry);
7629INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass)initializeAAResultsWrapperPassPass(Registry);
7630INITIALIZE_PASS_DEPENDENCY(GlobalsAAWrapperPass)initializeGlobalsAAWrapperPassPass(Registry);
7631INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)initializeAssumptionCacheTrackerPass(Registry);
7632INITIALIZE_PASS_DEPENDENCY(BlockFrequencyInfoWrapperPass)initializeBlockFrequencyInfoWrapperPassPass(Registry);
7633INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)initializeDominatorTreeWrapperPassPass(Registry);
7634INITIALIZE_PASS_DEPENDENCY(ScalarEvolutionWrapperPass)initializeScalarEvolutionWrapperPassPass(Registry);
7635INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass)initializeLoopInfoWrapperPassPass(Registry);
7636INITIALIZE_PASS_DEPENDENCY(LoopAccessLegacyAnalysis)initializeLoopAccessLegacyAnalysisPass(Registry);
7637INITIALIZE_PASS_DEPENDENCY(DemandedBitsWrapperPass)initializeDemandedBitsWrapperPassPass(Registry);
7638INITIALIZE_PASS_DEPENDENCY(OptimizationRemarkEmitterWrapperPass)initializeOptimizationRemarkEmitterWrapperPassPass(Registry);
7639INITIALIZE_PASS_DEPENDENCY(ProfileSummaryInfoWrapperPass)initializeProfileSummaryInfoWrapperPassPass(Registry);
7640INITIALIZE_PASS_DEPENDENCY(InjectTLIMappingsLegacy)initializeInjectTLIMappingsLegacyPass(Registry);
7641INITIALIZE_PASS_END(LoopVectorize, LV_NAME, lv_name, false, false)PassInfo *PI = new PassInfo( lv_name, "loop-vectorize", &
LoopVectorize::ID, PassInfo::NormalCtor_t(callDefaultCtor<
LoopVectorize>), false, false); Registry.registerPass(*PI,
 true); return PI; } static llvm::once_flag InitializeLoopVectorizePassFlag
; void llvm::initializeLoopVectorizePass(PassRegistry &Registry
) { llvm::call_once(InitializeLoopVectorizePassFlag, initializeLoopVectorizePassOnce
, std::ref(Registry)); }

7643namespace llvm {

7645Pass *createLoopVectorizePass() { return new LoopVectorize(); }

7647Pass *createLoopVectorizePass(bool InterleaveOnlyWhenForced,
                            bool VectorizeOnlyWhenForced) {
return new LoopVectorize(InterleaveOnlyWhenForced, VectorizeOnlyWhenForced);
7650}

7652} // end namespace llvm

7654bool LoopVectorizationCostModel::isConsecutiveLoadOrStore(Instruction *Inst) {
// Check if the pointer operand of a load or store instruction is
// consecutive.
if (auto *Ptr = getLoadStorePointerOperand(Inst))
  return Legal->isConsecutivePtr(Ptr);
return false;
7660}

7662void LoopVectorizationCostModel::collectValuesToIgnore() {
// Ignore ephemeral values.
CodeMetrics::collectEphemeralValues(TheLoop, AC, ValuesToIgnore);

// Ignore type-promoting instructions we identified during reduction
// detection.
for (auto &Reduction : Legal->getReductionVars()) {
  RecurrenceDescriptor &RedDes = Reduction.second;
  const SmallPtrSetImpl<Instruction *> &Casts = RedDes.getCastInsts();
  VecValuesToIgnore.insert(Casts.begin(), Casts.end());
}
// Ignore type-casting instructions we identified during induction
// detection.
for (auto &Induction : Legal->getInductionVars()) {
  InductionDescriptor &IndDes = Induction.second;
  const SmallVectorImpl<Instruction *> &Casts = IndDes.getCastInsts();
  VecValuesToIgnore.insert(Casts.begin(), Casts.end());
}
7680}

7682void LoopVectorizationCostModel::collectInLoopReductions() {
for (auto &Reduction : Legal->getReductionVars()) {
  PHINode *Phi = Reduction.first;
  RecurrenceDescriptor &RdxDesc = Reduction.second;

  // We don't collect reductions that are type promoted (yet).
  if (RdxDesc.getRecurrenceType() != Phi->getType())
    continue;

  // If the target would prefer this reduction to happen "in-loop", then we
  // want to record it as such.
  unsigned Opcode = RdxDesc.getOpcode();
  if (!PreferInLoopReductions && !useOrderedReductions(RdxDesc) &&
      !TTI.preferInLoopReduction(Opcode, Phi->getType(),
                                 TargetTransformInfo::ReductionFlags()))
    continue;

  // Check that we can correctly put the reductions into the loop, by
  // finding the chain of operations that leads from the phi to the loop
  // exit value.
  SmallVector<Instruction *, 4> ReductionOperations =
      RdxDesc.getReductionOpChain(Phi, TheLoop);
  bool InLoop = !ReductionOperations.empty();
  if (InLoop) {
    InLoopReductionChains[Phi] = ReductionOperations;
    // Add the elements to InLoopReductionImmediateChains for cost modelling.
    Instruction *LastChain = Phi;
    for (auto *I : ReductionOperations) {
      InLoopReductionImmediateChains[I] = LastChain;
      LastChain = I;
    }
  }
  LLVM_DEBUG(dbgs() << "LV: Using " << (InLoop ? "inloop" : "out of loop")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Using " << (
InLoop ? "inloop" : "out of loop") << " reduction for phi: "
 << *Phi << "\n"; } } while (false)
                    << " reduction for phi: " << *Phi << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Using " << (
InLoop ? "inloop" : "out of loop") << " reduction for phi: "
 << *Phi << "\n"; } } while (false);
}
7717}

7719// TODO: we could return a pair of values that specify the max VF and
7720// min VF, to be used in `buildVPlans(MinVF, MaxVF)` instead of
7721// `buildVPlans(VF, VF)`. We cannot do it because VPLAN at the moment
7722// doesn't have a cost model that can choose which plan to execute if
7723// more than one is generated.
7724static unsigned determineVPlanVF(const unsigned WidestVectorRegBits,
                               LoopVectorizationCostModel &CM) {
unsigned WidestType;
std::tie(std::ignore, WidestType) = CM.getSmallestAndWidestTypes();
return WidestVectorRegBits / WidestType;
7729}

7731VectorizationFactor
7732LoopVectorizationPlanner::planInVPlanNativePath(ElementCount UserVF) {
assert(!UserVF.isScalable() && "scalable vectors not yet supported")((!UserVF.isScalable() && "scalable vectors not yet supported"
) ? static_cast<void> (0) : __assert_fail ("!UserVF.isScalable() && \"scalable vectors not yet supported\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 7733, __PRETTY_FUNCTION__));
ElementCount VF = UserVF;
// Outer loop handling: They may require CFG and instruction level
// transformations before even evaluating whether vectorization is profitable.
// Since we cannot modify the incoming IR, we need to build VPlan upfront in
// the vectorization pipeline.
if (!OrigLoop->isInnermost()) {
  // If the user doesn't provide a vectorization factor, determine a
  // reasonable one.
  if (UserVF.isZero()) {
    VF = ElementCount::getFixed(determineVPlanVF(
        TTI->getRegisterBitWidth(TargetTransformInfo::RGK_FixedWidthVector)
            .getFixedSize(),
        CM));
    LLVM_DEBUG(dbgs() << "LV: VPlan computed VF " << VF << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: VPlan computed VF "
 << VF << ".\n"; } } while (false);

    // Make sure we have a VF > 1 for stress testing.
    if (VPlanBuildStressTest && (VF.isScalar() || VF.isZero())) {
      LLVM_DEBUG(dbgs() << "LV: VPlan stress testing: "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: VPlan stress testing: "
 << "overriding computed VF.\n"; } } while (false)
                        << "overriding computed VF.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: VPlan stress testing: "
 << "overriding computed VF.\n"; } } while (false);
      VF = ElementCount::getFixed(4);
    }
  }
  assert(EnableVPlanNativePath && "VPlan-native path is not enabled.")((EnableVPlanNativePath && "VPlan-native path is not enabled."
) ? static_cast<void> (0) : __assert_fail ("EnableVPlanNativePath && \"VPlan-native path is not enabled.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 7756, __PRETTY_FUNCTION__));
  assert(isPowerOf2_32(VF.getKnownMinValue()) &&((isPowerOf2_32(VF.getKnownMinValue()) && "VF needs to be a power of two"
) ? static_cast<void> (0) : __assert_fail ("isPowerOf2_32(VF.getKnownMinValue()) && \"VF needs to be a power of two\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 7758, __PRETTY_FUNCTION__))
         "VF needs to be a power of two")((isPowerOf2_32(VF.getKnownMinValue()) && "VF needs to be a power of two"
) ? static_cast<void> (0) : __assert_fail ("isPowerOf2_32(VF.getKnownMinValue()) && \"VF needs to be a power of two\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 7758, __PRETTY_FUNCTION__));
  LLVM_DEBUG(dbgs() << "LV: Using " << (!UserVF.isZero() ? "user " : "")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Using " << (
!UserVF.isZero() ? "user " : "") << "VF " << VF <<
 " to build VPlans.\n"; } } while (false)
                    << "VF " << VF << " to build VPlans.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Using " << (
!UserVF.isZero() ? "user " : "") << "VF " << VF <<
 " to build VPlans.\n"; } } while (false);
  buildVPlans(VF, VF);

  // For VPlan build stress testing, we bail out after VPlan construction.
  if (VPlanBuildStressTest)
    return VectorizationFactor::Disabled();

  return {VF, 0 /*Cost*/};
}

LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Not vectorizing. Inner loops aren't supported in the "
 "VPlan-native path.\n"; } } while (false)
    dbgs() << "LV: Not vectorizing. Inner loops aren't supported in the "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Not vectorizing. Inner loops aren't supported in the "
 "VPlan-native path.\n"; } } while (false)
              "VPlan-native path.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Not vectorizing. Inner loops aren't supported in the "
 "VPlan-native path.\n"; } } while (false);
return VectorizationFactor::Disabled();
7774}

7776Optional<VectorizationFactor>
7777LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC) {
assert(OrigLoop->isInnermost() && "Inner loop expected.")((OrigLoop->isInnermost() && "Inner loop expected."
) ? static_cast<void> (0) : __assert_fail ("OrigLoop->isInnermost() && \"Inner loop expected.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 7778, __PRETTY_FUNCTION__));
24
←
Assuming the condition is true→
25
←
'?' condition is true→
Optional<ElementCount> MaybeMaxVF = CM.computeMaxVF(UserVF, UserIC);
if (!MaybeMaxVF) // Cases that should not to be vectorized nor interleaved.
26
←
Taking false branch→
  return None;

// Invalidate interleave groups if all blocks of loop will be predicated.
if (CM.blockNeedsPredication(OrigLoop->getHeader()) &&
27
←
Assuming the condition is false→
    !useMaskedInterleavedAccesses(*TTI)) {
  LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Invalidate all interleaved groups due to fold-tail by masking "
 "which requires masked-interleaved support.\n"; } } while (false
)
      dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Invalidate all interleaved groups due to fold-tail by masking "
 "which requires masked-interleaved support.\n"; } } while (false
)
      << "LV: Invalidate all interleaved groups due to fold-tail by masking "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Invalidate all interleaved groups due to fold-tail by masking "
 "which requires masked-interleaved support.\n"; } } while (false
)
         "which requires masked-interleaved support.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Invalidate all interleaved groups due to fold-tail by masking "
 "which requires masked-interleaved support.\n"; } } while (false
);
  if (CM.InterleaveInfo.invalidateGroups())
    // Invalidating interleave groups also requires invalidating all decisions
    // based on them, which includes widening decisions and uniform and scalar
    // values.
    CM.invalidateCostModelingDecisions();
}

ElementCount MaxVF = MaybeMaxVF.getValue();
assert(MaxVF.isNonZero() && "MaxVF is zero.")((MaxVF.isNonZero() && "MaxVF is zero.") ? static_cast
<void> (0) : __assert_fail ("MaxVF.isNonZero() && \"MaxVF is zero.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 7798, __PRETTY_FUNCTION__));
28
←
'?' condition is true→

bool UserVFIsLegal = ElementCount::isKnownLE(UserVF, MaxVF);
if (!UserVF.isZero() &&
    (UserVFIsLegal28.1
'UserVFIsLegal' is true
 || (UserVF.isScalable() && MaxVF.isScalable()))) {
  // FIXME: MaxVF is temporarily used inplace of UserVF for illegal scalable
  // VFs here, this should be reverted to only use legal UserVFs once the
  // loop below supports scalable VFs.
  ElementCount VF = UserVFIsLegal28.2
'UserVFIsLegal' is true
 ? UserVF : MaxVF;
29
←
'?' condition is true→
  LLVM_DEBUG(dbgs() << "LV: Using " << (UserVFIsLegal ? "user" : "max")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Using " << (
UserVFIsLegal ? "user" : "max") << " VF " << VF <<
 ".\n"; } } while (false)
30
←
Assuming 'DebugFlag' is false→
31
←
Loop condition is false.  Exiting loop→
                    << " VF " << VF << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Using " << (
UserVFIsLegal ? "user" : "max") << " VF " << VF <<
 ".\n"; } } while (false);
  assert(isPowerOf2_32(VF.getKnownMinValue()) &&((isPowerOf2_32(VF.getKnownMinValue()) && "VF needs to be a power of two"
) ? static_cast<void> (0) : __assert_fail ("isPowerOf2_32(VF.getKnownMinValue()) && \"VF needs to be a power of two\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 7810, __PRETTY_FUNCTION__))
32
←
'?' condition is true→
         "VF needs to be a power of two")((isPowerOf2_32(VF.getKnownMinValue()) && "VF needs to be a power of two"
) ? static_cast<void> (0) : __assert_fail ("isPowerOf2_32(VF.getKnownMinValue()) && \"VF needs to be a power of two\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 7810, __PRETTY_FUNCTION__));
  // Collect the instructions (and their associated costs) that will be more
  // profitable to scalarize.
  CM.selectUserVectorizationFactor(VF);
  CM.collectInLoopReductions();
  buildVPlansWithVPRecipes(VF, VF);
33
←
Calling 'LoopVectorizationPlanner::buildVPlansWithVPRecipes'→
  LLVM_DEBUG(printPlans(dbgs()))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { printPlans(dbgs()); } } while (false);
  return {{VF, 0}};
}

assert(!MaxVF.isScalable() &&((!MaxVF.isScalable() && "Scalable vectors not yet supported beyond this point"
) ? static_cast<void> (0) : __assert_fail ("!MaxVF.isScalable() && \"Scalable vectors not yet supported beyond this point\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 7821, __PRETTY_FUNCTION__))
       "Scalable vectors not yet supported beyond this point")((!MaxVF.isScalable() && "Scalable vectors not yet supported beyond this point"
) ? static_cast<void> (0) : __assert_fail ("!MaxVF.isScalable() && \"Scalable vectors not yet supported beyond this point\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 7821, __PRETTY_FUNCTION__));

for (ElementCount VF = ElementCount::getFixed(1);
     ElementCount::isKnownLE(VF, MaxVF); VF *= 2) {
  // Collect Uniform and Scalar instructions after vectorization with VF.
  CM.collectUniformsAndScalars(VF);

  // Collect the instructions (and their associated costs) that will be more
  // profitable to scalarize.
  if (VF.isVector())
    CM.collectInstsToScalarize(VF);
}

CM.collectInLoopReductions();

buildVPlansWithVPRecipes(ElementCount::getFixed(1), MaxVF);
LLVM_DEBUG(printPlans(dbgs()))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { printPlans(dbgs()); } } while (false);
if (MaxVF.isScalar())
  return VectorizationFactor::Disabled();

// Select the optimal vectorization factor.
auto SelectedVF = CM.selectVectorizationFactor(MaxVF);

// Check if it is profitable to vectorize with runtime checks.
unsigned NumRuntimePointerChecks = Requirements.getNumRuntimePointerChecks();
if (SelectedVF.Width.getKnownMinValue() > 1 && NumRuntimePointerChecks) {
  bool PragmaThresholdReached =
      NumRuntimePointerChecks > PragmaVectorizeMemoryCheckThreshold;
  bool ThresholdReached =
      NumRuntimePointerChecks > VectorizerParams::RuntimeMemoryCheckThreshold;
  if ((ThresholdReached && !Hints.allowReordering()) ||
      PragmaThresholdReached) {
    ORE->emit([&]() {
      return OptimizationRemarkAnalysisAliasing(
                 DEBUG_TYPE"loop-vectorize", "CantReorderMemOps", OrigLoop->getStartLoc(),
                 OrigLoop->getHeader())
             << "loop not vectorized: cannot prove it is safe to reorder "
                "memory operations";
    });
    LLVM_DEBUG(dbgs() << "LV: Too many memory checks needed.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Too many memory checks needed.\n"
; } } while (false);
    Hints.emitRemarkWithHints();
    return VectorizationFactor::Disabled();
  }
}
return SelectedVF;
7866}

7868void LoopVectorizationPlanner::setBestPlan(ElementCount VF, unsigned UF) {
LLVM_DEBUG(dbgs() << "Setting best plan to VF=" << VF << ", UF=" << UFdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "Setting best plan to VF="
 << VF << ", UF=" << UF << '\n'; } } while
 (false)
                  << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "Setting best plan to VF="
 << VF << ", UF=" << UF << '\n'; } } while
 (false);
BestVF = VF;
BestUF = UF;

erase_if(VPlans, [VF](const VPlanPtr &Plan) {
  return !Plan->hasVF(VF);
});
assert(VPlans.size() == 1 && "Best VF has not a single VPlan.")((VPlans.size() == 1 && "Best VF has not a single VPlan."
) ? static_cast<void> (0) : __assert_fail ("VPlans.size() == 1 && \"Best VF has not a single VPlan.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 7877, __PRETTY_FUNCTION__));
7878}

7880void LoopVectorizationPlanner::executePlan(InnerLoopVectorizer &ILV,
                                         DominatorTree *DT) {
// Perform the actual loop transformation.

// 1. Create a new empty loop. Unlink the old loop and connect the new one.
assert(BestVF.hasValue() && "Vectorization Factor is missing")((BestVF.hasValue() && "Vectorization Factor is missing"
) ? static_cast<void> (0) : __assert_fail ("BestVF.hasValue() && \"Vectorization Factor is missing\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 7885, __PRETTY_FUNCTION__));
assert(VPlans.size() == 1 && "Not a single VPlan to execute.")((VPlans.size() == 1 && "Not a single VPlan to execute."
) ? static_cast<void> (0) : __assert_fail ("VPlans.size() == 1 && \"Not a single VPlan to execute.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 7886, __PRETTY_FUNCTION__));

VPTransformState State{
    *BestVF, BestUF, LI, DT, ILV.Builder, &ILV, VPlans.front().get()};
State.CFG.PrevBB = ILV.createVectorizedLoopSkeleton();
State.TripCount = ILV.getOrCreateTripCount(nullptr);
State.CanonicalIV = ILV.Induction;

ILV.printDebugTracesAtStart();

//===------------------------------------------------===//
//
// Notice: any optimization or new instruction that go
// into the code below should also be implemented in
// the cost-model.
//
//===------------------------------------------------===//

// 2. Copy and widen instructions from the old loop into the new loop.
VPlans.front()->execute(&State);

// 3. Fix the vectorized code: take care of header phi's, live-outs,
//    predication, updating analyses.
ILV.fixVectorizedLoop(State);

ILV.printDebugTracesAtEnd();
7912}

7914#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
7915void LoopVectorizationPlanner::printPlans(raw_ostream &O) {
for (const auto &Plan : VPlans)
  if (PrintVPlansInDotFormat)
    Plan->printDOT(O);
  else
    Plan->print(O);
7921}
7922#endif

7924void LoopVectorizationPlanner::collectTriviallyDeadInstructions(
  SmallPtrSetImpl<Instruction *> &DeadInstructions) {

// We create new control-flow for the vectorized loop, so the original exit
// conditions will be dead after vectorization if it's only used by the
// terminator
SmallVector<BasicBlock*> ExitingBlocks;
OrigLoop->getExitingBlocks(ExitingBlocks);
for (auto *BB : ExitingBlocks) {
  auto *Cmp = dyn_cast<Instruction>(BB->getTerminator()->getOperand(0));
  if (!Cmp || !Cmp->hasOneUse())
    continue;

  // TODO: we should introduce a getUniqueExitingBlocks on Loop
  if (!DeadInstructions.insert(Cmp).second)
    continue;

  // The operands of the icmp is often a dead trunc, used by IndUpdate.
  // TODO: can recurse through operands in general
  for (Value *Op : Cmp->operands()) {
    if (isa<TruncInst>(Op) && Op->hasOneUse())
        DeadInstructions.insert(cast<Instruction>(Op));
  }
}

// We create new "steps" for induction variable updates to which the original
// induction variables map. An original update instruction will be dead if
// all its users except the induction variable are dead.
auto *Latch = OrigLoop->getLoopLatch();
for (auto &Induction : Legal->getInductionVars()) {
  PHINode *Ind = Induction.first;
  auto *IndUpdate = cast<Instruction>(Ind->getIncomingValueForBlock(Latch));

  // If the tail is to be folded by masking, the primary induction variable,
  // if exists, isn't dead: it will be used for masking. Don't kill it.
  if (CM.foldTailByMasking() && IndUpdate == Legal->getPrimaryInduction())
    continue;

  if (llvm::all_of(IndUpdate->users(), [&](User *U) -> bool {
        return U == Ind || DeadInstructions.count(cast<Instruction>(U));
      }))
    DeadInstructions.insert(IndUpdate);

  // We record as "Dead" also the type-casting instructions we had identified
  // during induction analysis. We don't need any handling for them in the
  // vectorized loop because we have proven that, under a proper runtime
  // test guarding the vectorized loop, the value of the phi, and the casted
  // value of the phi, are the same. The last instruction in this casting chain
  // will get its scalar/vector/widened def from the scalar/vector/widened def
  // of the respective phi node. Any other casts in the induction def-use chain
  // have no other uses outside the phi update chain, and will be ignored.
  InductionDescriptor &IndDes = Induction.second;
  const SmallVectorImpl<Instruction *> &Casts = IndDes.getCastInsts();
  DeadInstructions.insert(Casts.begin(), Casts.end());
}
7979}

7981Value *InnerLoopUnroller::reverseVector(Value *Vec) { return Vec; }

7983Value *InnerLoopUnroller::getBroadcastInstrs(Value *V) { return V; }

7985Value *InnerLoopUnroller::getStepVector(Value *Val, int StartIdx, Value *Step,
                                      Instruction::BinaryOps BinOp) {
// When unrolling and the VF is 1, we only need to add a simple scalar.
Type *Ty = Val->getType();
assert(!Ty->isVectorTy() && "Val must be a scalar")((!Ty->isVectorTy() && "Val must be a scalar") ? static_cast
<void> (0) : __assert_fail ("!Ty->isVectorTy() && \"Val must be a scalar\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 7989, __PRETTY_FUNCTION__));

if (Ty->isFloatingPointTy()) {
  Constant *C = ConstantFP::get(Ty, (double)StartIdx);

  // Floating-point operations inherit FMF via the builder's flags.
  Value *MulOp = Builder.CreateFMul(C, Step);
  return Builder.CreateBinOp(BinOp, Val, MulOp);
}
Constant *C = ConstantInt::get(Ty, StartIdx);
return Builder.CreateAdd(Val, Builder.CreateMul(C, Step), "induction");
8000}

8002static void AddRuntimeUnrollDisableMetaData(Loop *L) {
SmallVector<Metadata *, 4> MDs;
// Reserve first location for self reference to the LoopID metadata node.
MDs.push_back(nullptr);
bool IsUnrollMetadata = false;
MDNode *LoopID = L->getLoopID();
if (LoopID) {
  // First find existing loop unrolling disable metadata.
  for (unsigned i = 1, ie = LoopID->getNumOperands(); i < ie; ++i) {
    auto *MD = dyn_cast<MDNode>(LoopID->getOperand(i));
    if (MD) {
      const auto *S = dyn_cast<MDString>(MD->getOperand(0));
      IsUnrollMetadata =
          S && S->getString().startswith("llvm.loop.unroll.disable");
    }
    MDs.push_back(LoopID->getOperand(i));
  }
}

if (!IsUnrollMetadata) {
  // Add runtime unroll disable metadata.
  LLVMContext &Context = L->getHeader()->getContext();
  SmallVector<Metadata *, 1> DisableOperands;
  DisableOperands.push_back(
      MDString::get(Context, "llvm.loop.unroll.runtime.disable"));
  MDNode *DisableNode = MDNode::get(Context, DisableOperands);
  MDs.push_back(DisableNode);
  MDNode *NewLoopID = MDNode::get(Context, MDs);
  // Set operand 0 to refer to the loop id itself.
  NewLoopID->replaceOperandWith(0, NewLoopID);
  L->setLoopID(NewLoopID);
}
8034}

8036//===--------------------------------------------------------------------===//
8037// EpilogueVectorizerMainLoop
8038//===--------------------------------------------------------------------===//

8040/// This function is partially responsible for generating the control flow
8041/// depicted in https://llvm.org/docs/Vectorizers.html#epilogue-vectorization.
8042BasicBlock *EpilogueVectorizerMainLoop::createEpilogueVectorizedLoopSkeleton() {
MDNode *OrigLoopID = OrigLoop->getLoopID();
Loop *Lp = createVectorLoopSkeleton("");

// Generate the code to check the minimum iteration count of the vector
// epilogue (see below).
EPI.EpilogueIterationCountCheck =
    emitMinimumIterationCountCheck(Lp, LoopScalarPreHeader, true);
EPI.EpilogueIterationCountCheck->setName("iter.check");

// Generate the code to check any assumptions that we've made for SCEV
// expressions.
EPI.SCEVSafetyCheck = emitSCEVChecks(Lp, LoopScalarPreHeader);

// Generate the code that checks at runtime if arrays overlap. We put the
// checks into a separate block to make the more common case of few elements
// faster.
EPI.MemSafetyCheck = emitMemRuntimeChecks(Lp, LoopScalarPreHeader);

// Generate the iteration count check for the main loop, *after* the check
// for the epilogue loop, so that the path-length is shorter for the case
// that goes directly through the vector epilogue. The longer-path length for
// the main loop is compensated for, by the gain from vectorizing the larger
// trip count. Note: the branch will get updated later on when we vectorize
// the epilogue.
EPI.MainLoopIterationCountCheck =
    emitMinimumIterationCountCheck(Lp, LoopScalarPreHeader, false);

// Generate the induction variable.
OldInduction = Legal->getPrimaryInduction();
Type *IdxTy = Legal->getWidestInductionType();
Value *StartIdx = ConstantInt::get(IdxTy, 0);
Constant *Step = ConstantInt::get(IdxTy, VF.getKnownMinValue() * UF);
Value *CountRoundDown = getOrCreateVectorTripCount(Lp);
EPI.VectorTripCount = CountRoundDown;
Induction =
    createInductionVariable(Lp, StartIdx, CountRoundDown, Step,
                            getDebugLocFromInstOrOperands(OldInduction));

// Skip induction resume value creation here because they will be created in
// the second pass. If we created them here, they wouldn't be used anyway,
// because the vplan in the second pass still contains the inductions from the
// original loop.

return completeLoopSkeleton(Lp, OrigLoopID);
8087}

8089void EpilogueVectorizerMainLoop::printDebugTracesAtStart() {
LLVM_DEBUG({do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { { dbgs() << "Create Skeleton for epilogue vectorized loop (first pass)\n"
 << "Main Loop VF:" << EPI.MainLoopVF.getKnownMinValue
() << ", Main Loop UF:" << EPI.MainLoopUF <<
 ", Epilogue Loop VF:" << EPI.EpilogueVF.getKnownMinValue
() << ", Epilogue Loop UF:" << EPI.EpilogueUF <<
 "\n"; }; } } while (false)
  dbgs() << "Create Skeleton for epilogue vectorized loop (first pass)\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { { dbgs() << "Create Skeleton for epilogue vectorized loop (first pass)\n"
 << "Main Loop VF:" << EPI.MainLoopVF.getKnownMinValue
() << ", Main Loop UF:" << EPI.MainLoopUF <<
 ", Epilogue Loop VF:" << EPI.EpilogueVF.getKnownMinValue
() << ", Epilogue Loop UF:" << EPI.EpilogueUF <<
 "\n"; }; } } while (false)
         << "Main Loop VF:" << EPI.MainLoopVF.getKnownMinValue()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { { dbgs() << "Create Skeleton for epilogue vectorized loop (first pass)\n"
 << "Main Loop VF:" << EPI.MainLoopVF.getKnownMinValue
() << ", Main Loop UF:" << EPI.MainLoopUF <<
 ", Epilogue Loop VF:" << EPI.EpilogueVF.getKnownMinValue
() << ", Epilogue Loop UF:" << EPI.EpilogueUF <<
 "\n"; }; } } while (false)
         << ", Main Loop UF:" << EPI.MainLoopUFdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { { dbgs() << "Create Skeleton for epilogue vectorized loop (first pass)\n"
 << "Main Loop VF:" << EPI.MainLoopVF.getKnownMinValue
() << ", Main Loop UF:" << EPI.MainLoopUF <<
 ", Epilogue Loop VF:" << EPI.EpilogueVF.getKnownMinValue
() << ", Epilogue Loop UF:" << EPI.EpilogueUF <<
 "\n"; }; } } while (false)
         << ", Epilogue Loop VF:" << EPI.EpilogueVF.getKnownMinValue()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { { dbgs() << "Create Skeleton for epilogue vectorized loop (first pass)\n"
 << "Main Loop VF:" << EPI.MainLoopVF.getKnownMinValue
() << ", Main Loop UF:" << EPI.MainLoopUF <<
 ", Epilogue Loop VF:" << EPI.EpilogueVF.getKnownMinValue
() << ", Epilogue Loop UF:" << EPI.EpilogueUF <<
 "\n"; }; } } while (false)
         << ", Epilogue Loop UF:" << EPI.EpilogueUF << "\n";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { { dbgs() << "Create Skeleton for epilogue vectorized loop (first pass)\n"
 << "Main Loop VF:" << EPI.MainLoopVF.getKnownMinValue
() << ", Main Loop UF:" << EPI.MainLoopUF <<
 ", Epilogue Loop VF:" << EPI.EpilogueVF.getKnownMinValue
() << ", Epilogue Loop UF:" << EPI.EpilogueUF <<
 "\n"; }; } } while (false)
})do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { { dbgs() << "Create Skeleton for epilogue vectorized loop (first pass)\n"
 << "Main Loop VF:" << EPI.MainLoopVF.getKnownMinValue
() << ", Main Loop UF:" << EPI.MainLoopUF <<
 ", Epilogue Loop VF:" << EPI.EpilogueVF.getKnownMinValue
() << ", Epilogue Loop UF:" << EPI.EpilogueUF <<
 "\n"; }; } } while (false);
8097}

8099void EpilogueVectorizerMainLoop::printDebugTracesAtEnd() {
DEBUG_WITH_TYPE(VerboseDebug, {do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
(VerboseDebug)) { { dbgs() << "intermediate fn:\n" <<
 *Induction->getFunction() << "\n"; }; } } while (false
)
  dbgs() << "intermediate fn:\n" << *Induction->getFunction() << "\n";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
(VerboseDebug)) { { dbgs() << "intermediate fn:\n" <<
 *Induction->getFunction() << "\n"; }; } } while (false
)
})do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
(VerboseDebug)) { { dbgs() << "intermediate fn:\n" <<
 *Induction->getFunction() << "\n"; }; } } while (false
);
8103}

8105BasicBlock *EpilogueVectorizerMainLoop::emitMinimumIterationCountCheck(
  Loop *L, BasicBlock *Bypass, bool ForEpilogue) {
assert(L && "Expected valid Loop.")((L && "Expected valid Loop.") ? static_cast<void>
 (0) : __assert_fail ("L && \"Expected valid Loop.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 8107, __PRETTY_FUNCTION__));
assert(Bypass && "Expected valid bypass basic block.")((Bypass && "Expected valid bypass basic block.") ? static_cast
<void> (0) : __assert_fail ("Bypass && \"Expected valid bypass basic block.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 8108, __PRETTY_FUNCTION__));
unsigned VFactor =
    ForEpilogue ? EPI.EpilogueVF.getKnownMinValue() : VF.getKnownMinValue();
unsigned UFactor = ForEpilogue ? EPI.EpilogueUF : UF;
Value *Count = getOrCreateTripCount(L);
// Reuse existing vector loop preheader for TC checks.
// Note that new preheader block is generated for vector loop.
BasicBlock *const TCCheckBlock = LoopVectorPreHeader;
IRBuilder<> Builder(TCCheckBlock->getTerminator());

// Generate code to check if the loop's trip count is less than VF * UF of the
// main vector loop.
auto P =
    Cost->requiresScalarEpilogue() ? ICmpInst::ICMP_ULE : ICmpInst::ICMP_ULT;

Value *CheckMinIters = Builder.CreateICmp(
    P, Count, ConstantInt::get(Count->getType(), VFactor * UFactor),
    "min.iters.check");

if (!ForEpilogue)
  TCCheckBlock->setName("vector.main.loop.iter.check");

// Create new preheader for vector loop.
LoopVectorPreHeader = SplitBlock(TCCheckBlock, TCCheckBlock->getTerminator(),
                                 DT, LI, nullptr, "vector.ph");

if (ForEpilogue) {
  assert(DT->properlyDominates(DT->getNode(TCCheckBlock),((DT->properlyDominates(DT->getNode(TCCheckBlock), DT->
getNode(Bypass)->getIDom()) && "TC check is expected to dominate Bypass"
) ? static_cast<void> (0) : __assert_fail ("DT->properlyDominates(DT->getNode(TCCheckBlock), DT->getNode(Bypass)->getIDom()) && \"TC check is expected to dominate Bypass\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 8137, __PRETTY_FUNCTION__))
                               DT->getNode(Bypass)->getIDom()) &&((DT->properlyDominates(DT->getNode(TCCheckBlock), DT->
getNode(Bypass)->getIDom()) && "TC check is expected to dominate Bypass"
) ? static_cast<void> (0) : __assert_fail ("DT->properlyDominates(DT->getNode(TCCheckBlock), DT->getNode(Bypass)->getIDom()) && \"TC check is expected to dominate Bypass\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 8137, __PRETTY_FUNCTION__))
         "TC check is expected to dominate Bypass")((DT->properlyDominates(DT->getNode(TCCheckBlock), DT->
getNode(Bypass)->getIDom()) && "TC check is expected to dominate Bypass"
) ? static_cast<void> (0) : __assert_fail ("DT->properlyDominates(DT->getNode(TCCheckBlock), DT->getNode(Bypass)->getIDom()) && \"TC check is expected to dominate Bypass\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 8137, __PRETTY_FUNCTION__));

  // Update dominator for Bypass & LoopExit.
  DT->changeImmediateDominator(Bypass, TCCheckBlock);
  DT->changeImmediateDominator(LoopExitBlock, TCCheckBlock);

  LoopBypassBlocks.push_back(TCCheckBlock);

  // Save the trip count so we don't have to regenerate it in the
  // vec.epilog.iter.check. This is safe to do because the trip count
  // generated here dominates the vector epilog iter check.
  EPI.TripCount = Count;
}

ReplaceInstWithInst(
    TCCheckBlock->getTerminator(),
    BranchInst::Create(Bypass, LoopVectorPreHeader, CheckMinIters));

return TCCheckBlock;
8156}

8158//===--------------------------------------------------------------------===//
8159// EpilogueVectorizerEpilogueLoop
8160//===--------------------------------------------------------------------===//

8162/// This function is partially responsible for generating the control flow
8163/// depicted in https://llvm.org/docs/Vectorizers.html#epilogue-vectorization.
8164BasicBlock *
8165EpilogueVectorizerEpilogueLoop::createEpilogueVectorizedLoopSkeleton() {
MDNode *OrigLoopID = OrigLoop->getLoopID();
Loop *Lp = createVectorLoopSkeleton("vec.epilog.");

// Now, compare the remaining count and if there aren't enough iterations to
// execute the vectorized epilogue skip to the scalar part.
BasicBlock *VecEpilogueIterationCountCheck = LoopVectorPreHeader;
VecEpilogueIterationCountCheck->setName("vec.epilog.iter.check");
LoopVectorPreHeader =
    SplitBlock(LoopVectorPreHeader, LoopVectorPreHeader->getTerminator(), DT,
               LI, nullptr, "vec.epilog.ph");
emitMinimumVectorEpilogueIterCountCheck(Lp, LoopScalarPreHeader,
                                        VecEpilogueIterationCountCheck);

// Adjust the control flow taking the state info from the main loop
// vectorization into account.
assert(EPI.MainLoopIterationCountCheck && EPI.EpilogueIterationCountCheck &&((EPI.MainLoopIterationCountCheck && EPI.EpilogueIterationCountCheck
 && "expected this to be saved from the previous pass."
) ? static_cast<void> (0) : __assert_fail ("EPI.MainLoopIterationCountCheck && EPI.EpilogueIterationCountCheck && \"expected this to be saved from the previous pass.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 8182, __PRETTY_FUNCTION__))
       "expected this to be saved from the previous pass.")((EPI.MainLoopIterationCountCheck && EPI.EpilogueIterationCountCheck
 && "expected this to be saved from the previous pass."
) ? static_cast<void> (0) : __assert_fail ("EPI.MainLoopIterationCountCheck && EPI.EpilogueIterationCountCheck && \"expected this to be saved from the previous pass.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 8182, __PRETTY_FUNCTION__));
EPI.MainLoopIterationCountCheck->getTerminator()->replaceUsesOfWith(
    VecEpilogueIterationCountCheck, LoopVectorPreHeader);

DT->changeImmediateDominator(LoopVectorPreHeader,
                             EPI.MainLoopIterationCountCheck);

EPI.EpilogueIterationCountCheck->getTerminator()->replaceUsesOfWith(
    VecEpilogueIterationCountCheck, LoopScalarPreHeader);

if (EPI.SCEVSafetyCheck)
  EPI.SCEVSafetyCheck->getTerminator()->replaceUsesOfWith(
      VecEpilogueIterationCountCheck, LoopScalarPreHeader);
if (EPI.MemSafetyCheck)
  EPI.MemSafetyCheck->getTerminator()->replaceUsesOfWith(
      VecEpilogueIterationCountCheck, LoopScalarPreHeader);

DT->changeImmediateDominator(
    VecEpilogueIterationCountCheck,
    VecEpilogueIterationCountCheck->getSinglePredecessor());

DT->changeImmediateDominator(LoopScalarPreHeader,
                             EPI.EpilogueIterationCountCheck);
DT->changeImmediateDominator(LoopExitBlock, EPI.EpilogueIterationCountCheck);

// Keep track of bypass blocks, as they feed start values to the induction
// phis in the scalar loop preheader.
if (EPI.SCEVSafetyCheck)
  LoopBypassBlocks.push_back(EPI.SCEVSafetyCheck);
if (EPI.MemSafetyCheck)
  LoopBypassBlocks.push_back(EPI.MemSafetyCheck);
LoopBypassBlocks.push_back(EPI.EpilogueIterationCountCheck);

// Generate a resume induction for the vector epilogue and put it in the
// vector epilogue preheader
Type *IdxTy = Legal->getWidestInductionType();
PHINode *EPResumeVal = PHINode::Create(IdxTy, 2, "vec.epilog.resume.val",
                                       LoopVectorPreHeader->getFirstNonPHI());
EPResumeVal->addIncoming(EPI.VectorTripCount, VecEpilogueIterationCountCheck);
EPResumeVal->addIncoming(ConstantInt::get(IdxTy, 0),
                         EPI.MainLoopIterationCountCheck);

// Generate the induction variable.
OldInduction = Legal->getPrimaryInduction();
Value *CountRoundDown = getOrCreateVectorTripCount(Lp);
Constant *Step = ConstantInt::get(IdxTy, VF.getKnownMinValue() * UF);
Value *StartIdx = EPResumeVal;
Induction =
    createInductionVariable(Lp, StartIdx, CountRoundDown, Step,
                            getDebugLocFromInstOrOperands(OldInduction));

// Generate induction resume values. These variables save the new starting
// indexes for the scalar loop. They are used to test if there are any tail
// iterations left once the vector loop has completed.
// Note that when the vectorized epilogue is skipped due to iteration count
// check, then the resume value for the induction variable comes from
// the trip count of the main vector loop, hence passing the AdditionalBypass
// argument.
createInductionResumeValues(Lp, CountRoundDown,
                            {VecEpilogueIterationCountCheck,
                             EPI.VectorTripCount} /* AdditionalBypass */);

AddRuntimeUnrollDisableMetaData(Lp);
return completeLoopSkeleton(Lp, OrigLoopID);
8246}

8248BasicBlock *
8249EpilogueVectorizerEpilogueLoop::emitMinimumVectorEpilogueIterCountCheck(
  Loop *L, BasicBlock *Bypass, BasicBlock *Insert) {

assert(EPI.TripCount &&((EPI.TripCount && "Expected trip count to have been safed in the first pass."
) ? static_cast<void> (0) : __assert_fail ("EPI.TripCount && \"Expected trip count to have been safed in the first pass.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 8253, __PRETTY_FUNCTION__))
       "Expected trip count to have been safed in the first pass.")((EPI.TripCount && "Expected trip count to have been safed in the first pass."
) ? static_cast<void> (0) : __assert_fail ("EPI.TripCount && \"Expected trip count to have been safed in the first pass.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 8253, __PRETTY_FUNCTION__));
assert((((!isa<Instruction>(EPI.TripCount) || DT->dominates
(cast<Instruction>(EPI.TripCount)->getParent(), Insert
)) && "saved trip count does not dominate insertion point."
) ? static_cast<void> (0) : __assert_fail ("(!isa<Instruction>(EPI.TripCount) || DT->dominates(cast<Instruction>(EPI.TripCount)->getParent(), Insert)) && \"saved trip count does not dominate insertion point.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 8257, __PRETTY_FUNCTION__))
    (!isa<Instruction>(EPI.TripCount) ||(((!isa<Instruction>(EPI.TripCount) || DT->dominates
(cast<Instruction>(EPI.TripCount)->getParent(), Insert
)) && "saved trip count does not dominate insertion point."
) ? static_cast<void> (0) : __assert_fail ("(!isa<Instruction>(EPI.TripCount) || DT->dominates(cast<Instruction>(EPI.TripCount)->getParent(), Insert)) && \"saved trip count does not dominate insertion point.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 8257, __PRETTY_FUNCTION__))
     DT->dominates(cast<Instruction>(EPI.TripCount)->getParent(), Insert)) &&(((!isa<Instruction>(EPI.TripCount) || DT->dominates
(cast<Instruction>(EPI.TripCount)->getParent(), Insert
)) && "saved trip count does not dominate insertion point."
) ? static_cast<void> (0) : __assert_fail ("(!isa<Instruction>(EPI.TripCount) || DT->dominates(cast<Instruction>(EPI.TripCount)->getParent(), Insert)) && \"saved trip count does not dominate insertion point.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 8257, __PRETTY_FUNCTION__))
    "saved trip count does not dominate insertion point.")(((!isa<Instruction>(EPI.TripCount) || DT->dominates
(cast<Instruction>(EPI.TripCount)->getParent(), Insert
)) && "saved trip count does not dominate insertion point."
) ? static_cast<void> (0) : __assert_fail ("(!isa<Instruction>(EPI.TripCount) || DT->dominates(cast<Instruction>(EPI.TripCount)->getParent(), Insert)) && \"saved trip count does not dominate insertion point.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 8257, __PRETTY_FUNCTION__));
Value *TC = EPI.TripCount;
IRBuilder<> Builder(Insert->getTerminator());
Value *Count = Builder.CreateSub(TC, EPI.VectorTripCount, "n.vec.remaining");

// Generate code to check if the loop's trip count is less than VF * UF of the
// vector epilogue loop.
auto P =
    Cost->requiresScalarEpilogue() ? ICmpInst::ICMP_ULE : ICmpInst::ICMP_ULT;

Value *CheckMinIters = Builder.CreateICmp(
    P, Count,
    ConstantInt::get(Count->getType(),
                     EPI.EpilogueVF.getKnownMinValue() * EPI.EpilogueUF),
    "min.epilog.iters.check");

ReplaceInstWithInst(
    Insert->getTerminator(),
    BranchInst::Create(Bypass, LoopVectorPreHeader, CheckMinIters));

LoopBypassBlocks.push_back(Insert);
return Insert;
8279}

8281void EpilogueVectorizerEpilogueLoop::printDebugTracesAtStart() {
LLVM_DEBUG({do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { { dbgs() << "Create Skeleton for epilogue vectorized loop (second pass)\n"
 << "Main Loop VF:" << EPI.MainLoopVF.getKnownMinValue
() << ", Main Loop UF:" << EPI.MainLoopUF <<
 ", Epilogue Loop VF:" << EPI.EpilogueVF.getKnownMinValue
() << ", Epilogue Loop UF:" << EPI.EpilogueUF <<
 "\n"; }; } } while (false)
  dbgs() << "Create Skeleton for epilogue vectorized loop (second pass)\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { { dbgs() << "Create Skeleton for epilogue vectorized loop (second pass)\n"
 << "Main Loop VF:" << EPI.MainLoopVF.getKnownMinValue
() << ", Main Loop UF:" << EPI.MainLoopUF <<
 ", Epilogue Loop VF:" << EPI.EpilogueVF.getKnownMinValue
() << ", Epilogue Loop UF:" << EPI.EpilogueUF <<
 "\n"; }; } } while (false)
         << "Main Loop VF:" << EPI.MainLoopVF.getKnownMinValue()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { { dbgs() << "Create Skeleton for epilogue vectorized loop (second pass)\n"
 << "Main Loop VF:" << EPI.MainLoopVF.getKnownMinValue
() << ", Main Loop UF:" << EPI.MainLoopUF <<
 ", Epilogue Loop VF:" << EPI.EpilogueVF.getKnownMinValue
() << ", Epilogue Loop UF:" << EPI.EpilogueUF <<
 "\n"; }; } } while (false)
         << ", Main Loop UF:" << EPI.MainLoopUFdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { { dbgs() << "Create Skeleton for epilogue vectorized loop (second pass)\n"
 << "Main Loop VF:" << EPI.MainLoopVF.getKnownMinValue
() << ", Main Loop UF:" << EPI.MainLoopUF <<
 ", Epilogue Loop VF:" << EPI.EpilogueVF.getKnownMinValue
() << ", Epilogue Loop UF:" << EPI.EpilogueUF <<
 "\n"; }; } } while (false)
         << ", Epilogue Loop VF:" << EPI.EpilogueVF.getKnownMinValue()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { { dbgs() << "Create Skeleton for epilogue vectorized loop (second pass)\n"
 << "Main Loop VF:" << EPI.MainLoopVF.getKnownMinValue
() << ", Main Loop UF:" << EPI.MainLoopUF <<
 ", Epilogue Loop VF:" << EPI.EpilogueVF.getKnownMinValue
() << ", Epilogue Loop UF:" << EPI.EpilogueUF <<
 "\n"; }; } } while (false)
         << ", Epilogue Loop UF:" << EPI.EpilogueUF << "\n";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { { dbgs() << "Create Skeleton for epilogue vectorized loop (second pass)\n"
 << "Main Loop VF:" << EPI.MainLoopVF.getKnownMinValue
() << ", Main Loop UF:" << EPI.MainLoopUF <<
 ", Epilogue Loop VF:" << EPI.EpilogueVF.getKnownMinValue
() << ", Epilogue Loop UF:" << EPI.EpilogueUF <<
 "\n"; }; } } while (false)
})do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { { dbgs() << "Create Skeleton for epilogue vectorized loop (second pass)\n"
 << "Main Loop VF:" << EPI.MainLoopVF.getKnownMinValue
() << ", Main Loop UF:" << EPI.MainLoopUF <<
 ", Epilogue Loop VF:" << EPI.EpilogueVF.getKnownMinValue
() << ", Epilogue Loop UF:" << EPI.EpilogueUF <<
 "\n"; }; } } while (false);
8289}

8291void EpilogueVectorizerEpilogueLoop::printDebugTracesAtEnd() {
DEBUG_WITH_TYPE(VerboseDebug, {do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
(VerboseDebug)) { { dbgs() << "final fn:\n" << *Induction
->getFunction() << "\n"; }; } } while (false)
  dbgs() << "final fn:\n" << *Induction->getFunction() << "\n";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
(VerboseDebug)) { { dbgs() << "final fn:\n" << *Induction
->getFunction() << "\n"; }; } } while (false)
})do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
(VerboseDebug)) { { dbgs() << "final fn:\n" << *Induction
->getFunction() << "\n"; }; } } while (false);
8295}

8297bool LoopVectorizationPlanner::getDecisionAndClampRange(
  const std::function<bool(ElementCount)> &Predicate, VFRange &Range) {
assert(!Range.isEmpty() && "Trying to test an empty VF range.")((!Range.isEmpty() && "Trying to test an empty VF range."
) ? static_cast<void> (0) : __assert_fail ("!Range.isEmpty() && \"Trying to test an empty VF range.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 8299, __PRETTY_FUNCTION__));
bool PredicateAtRangeStart = Predicate(Range.Start);

for (ElementCount TmpVF = Range.Start * 2;
     ElementCount::isKnownLT(TmpVF, Range.End); TmpVF *= 2)
  if (Predicate(TmpVF) != PredicateAtRangeStart) {
    Range.End = TmpVF;
    break;
  }

return PredicateAtRangeStart;
8310}

8312/// Build VPlans for the full range of feasible VF's = {\p MinVF, 2 * \p MinVF,
8313/// 4 * \p MinVF, ..., \p MaxVF} by repeatedly building a VPlan for a sub-range
8314/// of VF's starting at a given VF and extending it as much as possible. Each
8315/// vectorization decision can potentially shorten this sub-range during
8316/// buildVPlan().
8317void LoopVectorizationPlanner::buildVPlans(ElementCount MinVF,
                                         ElementCount MaxVF) {
auto MaxVFPlusOne = MaxVF.getWithIncrement(1);
for (ElementCount VF = MinVF; ElementCount::isKnownLT(VF, MaxVFPlusOne);) {
  VFRange SubRange = {VF, MaxVFPlusOne};
  VPlans.push_back(buildVPlan(SubRange));
  VF = SubRange.End;
}
8325}

8327VPValue *VPRecipeBuilder::createEdgeMask(BasicBlock *Src, BasicBlock *Dst,
                                       VPlanPtr &Plan) {
assert(is_contained(predecessors(Dst), Src) && "Invalid edge")((is_contained(predecessors(Dst), Src) && "Invalid edge"
) ? static_cast<void> (0) : __assert_fail ("is_contained(predecessors(Dst), Src) && \"Invalid edge\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 8329, __PRETTY_FUNCTION__));

// Look for cached value.
std::pair<BasicBlock *, BasicBlock *> Edge(Src, Dst);
EdgeMaskCacheTy::iterator ECEntryIt = EdgeMaskCache.find(Edge);
if (ECEntryIt != EdgeMaskCache.end())
  return ECEntryIt->second;

VPValue *SrcMask = createBlockInMask(Src, Plan);

// The terminator has to be a branch inst!
BranchInst *BI = dyn_cast<BranchInst>(Src->getTerminator());
assert(BI && "Unexpected terminator found")((BI && "Unexpected terminator found") ? static_cast<
void> (0) : __assert_fail ("BI && \"Unexpected terminator found\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 8341, __PRETTY_FUNCTION__));

if (!BI->isConditional() || BI->getSuccessor(0) == BI->getSuccessor(1))
  return EdgeMaskCache[Edge] = SrcMask;

// If source is an exiting block, we know the exit edge is dynamically dead
// in the vector loop, and thus we don't need to restrict the mask.  Avoid
// adding uses of an otherwise potentially dead instruction.
if (OrigLoop->isLoopExiting(Src))
  return EdgeMaskCache[Edge] = SrcMask;

VPValue *EdgeMask = Plan->getOrAddVPValue(BI->getCondition());
assert(EdgeMask && "No Edge Mask found for condition")((EdgeMask && "No Edge Mask found for condition") ? static_cast
<void> (0) : __assert_fail ("EdgeMask && \"No Edge Mask found for condition\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 8353, __PRETTY_FUNCTION__));

if (BI->getSuccessor(0) != Dst)
  EdgeMask = Builder.createNot(EdgeMask);

if (SrcMask) { // Otherwise block in-mask is all-one, no need to AND.
  // The condition is 'SrcMask && EdgeMask', which is equivalent to
  // 'select i1 SrcMask, i1 EdgeMask, i1 false'.
  // The select version does not introduce new UB if SrcMask is false and
  // EdgeMask is poison. Using 'and' here introduces undefined behavior.
  VPValue *False = Plan->getOrAddVPValue(
      ConstantInt::getFalse(BI->getCondition()->getType()));
  EdgeMask = Builder.createSelect(SrcMask, EdgeMask, False);
}

return EdgeMaskCache[Edge] = EdgeMask;
8369}

8371VPValue *VPRecipeBuilder::createBlockInMask(BasicBlock *BB, VPlanPtr &Plan) {
assert(OrigLoop->contains(BB) && "Block is not a part of a loop")((OrigLoop->contains(BB) && "Block is not a part of a loop"
) ? static_cast<void> (0) : __assert_fail ("OrigLoop->contains(BB) && \"Block is not a part of a loop\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 8372, __PRETTY_FUNCTION__));

// Look for cached value.
BlockMaskCacheTy::iterator BCEntryIt = BlockMaskCache.find(BB);
if (BCEntryIt != BlockMaskCache.end())
  return BCEntryIt->second;

// All-one mask is modelled as no-mask following the convention for masked
// load/store/gather/scatter. Initialize BlockMask to no-mask.
VPValue *BlockMask = nullptr;

if (OrigLoop->getHeader() == BB) {
  if (!CM.blockNeedsPredication(BB))
    return BlockMaskCache[BB] = BlockMask; // Loop incoming mask is all-one.

  // Create the block in mask as the first non-phi instruction in the block.
  VPBuilder::InsertPointGuard Guard(Builder);
  auto NewInsertionPoint = Builder.getInsertBlock()->getFirstNonPhi();
  Builder.setInsertPoint(Builder.getInsertBlock(), NewInsertionPoint);

  // Introduce the early-exit compare IV <= BTC to form header block mask.
  // This is used instead of IV < TC because TC may wrap, unlike BTC.
  // Start by constructing the desired canonical IV.
  VPValue *IV = nullptr;
  if (Legal->getPrimaryInduction())
    IV = Plan->getOrAddVPValue(Legal->getPrimaryInduction());
  else {
    auto IVRecipe = new VPWidenCanonicalIVRecipe();
    Builder.getInsertBlock()->insert(IVRecipe, NewInsertionPoint);
    IV = IVRecipe->getVPValue();
  }
  VPValue *BTC = Plan->getOrCreateBackedgeTakenCount();
  bool TailFolded = !CM.isScalarEpilogueAllowed();

  if (TailFolded && CM.TTI.emitGetActiveLaneMask()) {
    // While ActiveLaneMask is a binary op that consumes the loop tripcount
    // as a second argument, we only pass the IV here and extract the
    // tripcount from the transform state where codegen of the VP instructions
    // happen.
    BlockMask = Builder.createNaryOp(VPInstruction::ActiveLaneMask, {IV});
  } else {
    BlockMask = Builder.createNaryOp(VPInstruction::ICmpULE, {IV, BTC});
  }
  return BlockMaskCache[BB] = BlockMask;
}

// This is the block mask. We OR all incoming edges.
for (auto *Predecessor : predecessors(BB)) {
  VPValue *EdgeMask = createEdgeMask(Predecessor, BB, Plan);
  if (!EdgeMask) // Mask of predecessor is all-one so mask of block is too.
    return BlockMaskCache[BB] = EdgeMask;

  if (!BlockMask) { // BlockMask has its initialized nullptr value.
    BlockMask = EdgeMask;
    continue;
  }

  BlockMask = Builder.createOr(BlockMask, EdgeMask);
}

return BlockMaskCache[BB] = BlockMask;
8433}

8435VPRecipeBase *VPRecipeBuilder::tryToWidenMemory(Instruction *I,
                                              ArrayRef<VPValue *> Operands,
                                              VFRange &Range,
                                              VPlanPtr &Plan) {
assert((isa<LoadInst>(I) || isa<StoreInst>(I)) &&(((isa<LoadInst>(I) || isa<StoreInst>(I)) &&
 "Must be called with either a load or store") ? static_cast<
void> (0) : __assert_fail ("(isa<LoadInst>(I) || isa<StoreInst>(I)) && \"Must be called with either a load or store\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 8440, __PRETTY_FUNCTION__))
       "Must be called with either a load or store")(((isa<LoadInst>(I) || isa<StoreInst>(I)) &&
 "Must be called with either a load or store") ? static_cast<
void> (0) : __assert_fail ("(isa<LoadInst>(I) || isa<StoreInst>(I)) && \"Must be called with either a load or store\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 8440, __PRETTY_FUNCTION__));

auto willWiden = [&](ElementCount VF) -> bool {
  if (VF.isScalar())
    return false;
  LoopVectorizationCostModel::InstWidening Decision =
      CM.getWideningDecision(I, VF);
  assert(Decision != LoopVectorizationCostModel::CM_Unknown &&((Decision != LoopVectorizationCostModel::CM_Unknown &&
 "CM decision should be taken at this point.") ? static_cast<
void> (0) : __assert_fail ("Decision != LoopVectorizationCostModel::CM_Unknown && \"CM decision should be taken at this point.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 8448, __PRETTY_FUNCTION__))
         "CM decision should be taken at this point.")((Decision != LoopVectorizationCostModel::CM_Unknown &&
 "CM decision should be taken at this point.") ? static_cast<
void> (0) : __assert_fail ("Decision != LoopVectorizationCostModel::CM_Unknown && \"CM decision should be taken at this point.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 8448, __PRETTY_FUNCTION__));
  if (Decision == LoopVectorizationCostModel::CM_Interleave)
    return true;
  if (CM.isScalarAfterVectorization(I, VF) ||
      CM.isProfitableToScalarize(I, VF))
    return false;
  return Decision != LoopVectorizationCostModel::CM_Scalarize;
};

if (!LoopVectorizationPlanner::getDecisionAndClampRange(willWiden, Range))
  return nullptr;

VPValue *Mask = nullptr;
if (Legal->isMaskRequired(I))
  Mask = createBlockInMask(I->getParent(), Plan);

if (LoadInst *Load = dyn_cast<LoadInst>(I))
  return new VPWidenMemoryInstructionRecipe(*Load, Operands[0], Mask);

StoreInst *Store = cast<StoreInst>(I);
return new VPWidenMemoryInstructionRecipe(*Store, Operands[1], Operands[0],
                                          Mask);
8470}

8472VPWidenIntOrFpInductionRecipe *
8473VPRecipeBuilder::tryToOptimizeInductionPHI(PHINode *Phi,
                                         ArrayRef<VPValue *> Operands) const {
// Check if this is an integer or fp induction. If so, build the recipe that
// produces its scalar and vector values.
InductionDescriptor II = Legal->getInductionVars().lookup(Phi);
if (II.getKind() == InductionDescriptor::IK_IntInduction ||
    II.getKind() == InductionDescriptor::IK_FpInduction) {
  assert(II.getStartValue() ==((II.getStartValue() == Phi->getIncomingValueForBlock(OrigLoop
->getLoopPreheader())) ? static_cast<void> (0) : __assert_fail
 ("II.getStartValue() == Phi->getIncomingValueForBlock(OrigLoop->getLoopPreheader())"
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 8481, __PRETTY_FUNCTION__))
         Phi->getIncomingValueForBlock(OrigLoop->getLoopPreheader()))((II.getStartValue() == Phi->getIncomingValueForBlock(OrigLoop
->getLoopPreheader())) ? static_cast<void> (0) : __assert_fail
 ("II.getStartValue() == Phi->getIncomingValueForBlock(OrigLoop->getLoopPreheader())"
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 8481, __PRETTY_FUNCTION__));
  const SmallVectorImpl<Instruction *> &Casts = II.getCastInsts();
  return new VPWidenIntOrFpInductionRecipe(
      Phi, Operands[0], Casts.empty() ? nullptr : Casts.front());
}

return nullptr;
8488}

8490VPWidenIntOrFpInductionRecipe *VPRecipeBuilder::tryToOptimizeInductionTruncate(
  TruncInst *I, ArrayRef<VPValue *> Operands, VFRange &Range,
  VPlan &Plan) const {
// Optimize the special case where the source is a constant integer
// induction variable. Notice that we can only optimize the 'trunc' case
// because (a) FP conversions lose precision, (b) sext/zext may wrap, and
// (c) other casts depend on pointer size.

// Determine whether \p K is a truncation based on an induction variable that
// can be optimized.
auto isOptimizableIVTruncate =
    [&](Instruction *K) -> std::function<bool(ElementCount)> {
  return [=](ElementCount VF) -> bool {
    return CM.isOptimizableIVTruncate(K, VF);
  };
};

if (LoopVectorizationPlanner::getDecisionAndClampRange(
        isOptimizableIVTruncate(I), Range)) {

  InductionDescriptor II =
      Legal->getInductionVars().lookup(cast<PHINode>(I->getOperand(0)));
  VPValue *Start = Plan.getOrAddVPValue(II.getStartValue());
  return new VPWidenIntOrFpInductionRecipe(cast<PHINode>(I->getOperand(0)),
                                           Start, nullptr, I);
}
return nullptr;
8517}

8519VPRecipeOrVPValueTy VPRecipeBuilder::tryToBlend(PHINode *Phi,
                                              ArrayRef<VPValue *> Operands,
                                              VPlanPtr &Plan) {
// If all incoming values are equal, the incoming VPValue can be used directly
// instead of creating a new VPBlendRecipe.
VPValue *FirstIncoming = Operands[0];
if (all_of(Operands, [FirstIncoming](const VPValue *Inc) {
      return FirstIncoming == Inc;
    })) {
  return Operands[0];
}

// We know that all PHIs in non-header blocks are converted into selects, so
// we don't have to worry about the insertion order and we can just use the
// builder. At this point we generate the predication tree. There may be
// duplications since this is a simple recursive scan, but future
// optimizations will clean it up.
SmallVector<VPValue *, 2> OperandsWithMask;
unsigned NumIncoming = Phi->getNumIncomingValues();

for (unsigned In = 0; In < NumIncoming; In++) {
  VPValue *EdgeMask =
    createEdgeMask(Phi->getIncomingBlock(In), Phi->getParent(), Plan);
  assert((EdgeMask || NumIncoming == 1) &&(((EdgeMask || NumIncoming == 1) && "Multiple predecessors with one having a full mask"
) ? static_cast<void> (0) : __assert_fail ("(EdgeMask || NumIncoming == 1) && \"Multiple predecessors with one having a full mask\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 8543, __PRETTY_FUNCTION__))
         "Multiple predecessors with one having a full mask")(((EdgeMask || NumIncoming == 1) && "Multiple predecessors with one having a full mask"
) ? static_cast<void> (0) : __assert_fail ("(EdgeMask || NumIncoming == 1) && \"Multiple predecessors with one having a full mask\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 8543, __PRETTY_FUNCTION__));
  OperandsWithMask.push_back(Operands[In]);
  if (EdgeMask)
    OperandsWithMask.push_back(EdgeMask);
}
return toVPRecipeResult(new VPBlendRecipe(Phi, OperandsWithMask));
8549}

8551VPWidenCallRecipe *VPRecipeBuilder::tryToWidenCall(CallInst *CI,
                                                 ArrayRef<VPValue *> Operands,
                                                 VFRange &Range) const {

bool IsPredicated = LoopVectorizationPlanner::getDecisionAndClampRange(
    [this, CI](ElementCount VF) {
      return CM.isScalarWithPredication(CI, VF);
    },
    Range);

if (IsPredicated)
  return nullptr;

Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);
if (ID && (ID == Intrinsic::assume || ID == Intrinsic::lifetime_end ||
           ID == Intrinsic::lifetime_start || ID == Intrinsic::sideeffect ||
           ID == Intrinsic::pseudoprobe ||
           ID == Intrinsic::experimental_noalias_scope_decl))
  return nullptr;

auto willWiden = [&](ElementCount VF) -> bool {
  Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);
  // The following case may be scalarized depending on the VF.
  // The flag shows whether we use Intrinsic or a usual Call for vectorized
  // version of the instruction.
  // Is it beneficial to perform intrinsic call compared to lib call?
  bool NeedToScalarize = false;
  InstructionCost CallCost = CM.getVectorCallCost(CI, VF, NeedToScalarize);
  InstructionCost IntrinsicCost = ID ? CM.getVectorIntrinsicCost(CI, VF) : 0;
  bool UseVectorIntrinsic = ID && IntrinsicCost <= CallCost;
  assert((IntrinsicCost.isValid() || CallCost.isValid()) &&(((IntrinsicCost.isValid() || CallCost.isValid()) && "Either the intrinsic cost or vector call cost must be valid"
) ? static_cast<void> (0) : __assert_fail ("(IntrinsicCost.isValid() || CallCost.isValid()) && \"Either the intrinsic cost or vector call cost must be valid\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 8582, __PRETTY_FUNCTION__))
         "Either the intrinsic cost or vector call cost must be valid")(((IntrinsicCost.isValid() || CallCost.isValid()) && "Either the intrinsic cost or vector call cost must be valid"
) ? static_cast<void> (0) : __assert_fail ("(IntrinsicCost.isValid() || CallCost.isValid()) && \"Either the intrinsic cost or vector call cost must be valid\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 8582, __PRETTY_FUNCTION__));
  return UseVectorIntrinsic || !NeedToScalarize;
};

if (!LoopVectorizationPlanner::getDecisionAndClampRange(willWiden, Range))
  return nullptr;

ArrayRef<VPValue *> Ops = Operands.take_front(CI->getNumArgOperands());
return new VPWidenCallRecipe(*CI, make_range(Ops.begin(), Ops.end()));
8591}

8593bool VPRecipeBuilder::shouldWiden(Instruction *I, VFRange &Range) const {
assert(!isa<BranchInst>(I) && !isa<PHINode>(I) && !isa<LoadInst>(I) &&((!isa<BranchInst>(I) && !isa<PHINode>(I)
 && !isa<LoadInst>(I) && !isa<StoreInst
>(I) && "Instruction should have been handled earlier"
) ? static_cast<void> (0) : __assert_fail ("!isa<BranchInst>(I) && !isa<PHINode>(I) && !isa<LoadInst>(I) && !isa<StoreInst>(I) && \"Instruction should have been handled earlier\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 8595, __PRETTY_FUNCTION__))
       !isa<StoreInst>(I) && "Instruction should have been handled earlier")((!isa<BranchInst>(I) && !isa<PHINode>(I)
 && !isa<LoadInst>(I) && !isa<StoreInst
>(I) && "Instruction should have been handled earlier"
) ? static_cast<void> (0) : __assert_fail ("!isa<BranchInst>(I) && !isa<PHINode>(I) && !isa<LoadInst>(I) && !isa<StoreInst>(I) && \"Instruction should have been handled earlier\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 8595, __PRETTY_FUNCTION__));
// Instruction should be widened, unless it is scalar after vectorization,
// scalarization is profitable or it is predicated.
auto WillScalarize = [this, I](ElementCount VF) -> bool {
  return CM.isScalarAfterVectorization(I, VF) ||
         CM.isProfitableToScalarize(I, VF) ||
         CM.isScalarWithPredication(I, VF);
};
return !LoopVectorizationPlanner::getDecisionAndClampRange(WillScalarize,
                                                           Range);
8605}

8607VPWidenRecipe *VPRecipeBuilder::tryToWiden(Instruction *I,
                                         ArrayRef<VPValue *> Operands) const {
auto IsVectorizableOpcode = [](unsigned Opcode) {
  switch (Opcode) {
  case Instruction::Add:
  case Instruction::And:
  case Instruction::AShr:
  case Instruction::BitCast:
  case Instruction::FAdd:
  case Instruction::FCmp:
  case Instruction::FDiv:
  case Instruction::FMul:
  case Instruction::FNeg:
  case Instruction::FPExt:
  case Instruction::FPToSI:
  case Instruction::FPToUI:
  case Instruction::FPTrunc:
  case Instruction::FRem:
  case Instruction::FSub:
  case Instruction::ICmp:
  case Instruction::IntToPtr:
  case Instruction::LShr:
  case Instruction::Mul:
  case Instruction::Or:
  case Instruction::PtrToInt:
  case Instruction::SDiv:
  case Instruction::Select:
  case Instruction::SExt:
  case Instruction::Shl:
  case Instruction::SIToFP:
  case Instruction::SRem:
  case Instruction::Sub:
  case Instruction::Trunc:
  case Instruction::UDiv:
  case Instruction::UIToFP:
  case Instruction::URem:
  case Instruction::Xor:
  case Instruction::ZExt:
    return true;
  }
  return false;
};

if (!IsVectorizableOpcode(I->getOpcode()))
  return nullptr;

// Success: widen this instruction.
return new VPWidenRecipe(*I, make_range(Operands.begin(), Operands.end()));
8655}

8657VPBasicBlock *VPRecipeBuilder::handleReplication(
  Instruction *I, VFRange &Range, VPBasicBlock *VPBB,
  VPlanPtr &Plan) {
bool IsUniform = LoopVectorizationPlanner::getDecisionAndClampRange(
    [&](ElementCount VF) { return CM.isUniformAfterVectorization(I, VF); },
    Range);

bool IsPredicated = LoopVectorizationPlanner::getDecisionAndClampRange(
    [&](ElementCount VF) { return CM.isScalarWithPredication(I, VF); },
    Range);

auto *Recipe = new VPReplicateRecipe(I, Plan->mapToVPValues(I->operands()),
                                     IsUniform, IsPredicated);
setRecipe(I, Recipe);
Plan->addVPValue(I, Recipe);

// Find if I uses a predicated instruction. If so, it will use its scalar
// value. Avoid hoisting the insert-element which packs the scalar value into
// a vector value, as that happens iff all users use the vector value.
for (VPValue *Op : Recipe->operands()) {
45
←
Assuming '__begin1' is not equal to '__end1'→
  auto *PredR = dyn_cast_or_null<VPPredInstPHIRecipe>(Op->getDef());
46
←
Assuming the object is a 'VPPredInstPHIRecipe'→
  if (!PredR46.1
'PredR' is non-null
)
47
←
Taking false branch→
    continue;
  auto *RepR =
49
←
'RepR' initialized to a null pointer value→
      cast_or_null<VPReplicateRecipe>(PredR->getOperand(0)->getDef());
48
←
Assuming null pointer is passed into cast→
  assert(RepR->isPredicated() &&((RepR->isPredicated() && "expected Replicate recipe to be predicated"
) ? static_cast<void> (0) : __assert_fail ("RepR->isPredicated() && \"expected Replicate recipe to be predicated\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 8683, __PRETTY_FUNCTION__))
50
←
Called C++ object pointer is null
         "expected Replicate recipe to be predicated")((RepR->isPredicated() && "expected Replicate recipe to be predicated"
) ? static_cast<void> (0) : __assert_fail ("RepR->isPredicated() && \"expected Replicate recipe to be predicated\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 8683, __PRETTY_FUNCTION__));
  RepR->setAlsoPack(false);
}

// Finalize the recipe for Instr, first if it is not predicated.
if (!IsPredicated) {
  LLVM_DEBUG(dbgs() << "LV: Scalarizing:" << *I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Scalarizing:" <<
 *I << "\n"; } } while (false);
  VPBB->appendRecipe(Recipe);
  return VPBB;
}
LLVM_DEBUG(dbgs() << "LV: Scalarizing and predicating:" << *I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Scalarizing and predicating:"
 << *I << "\n"; } } while (false);
assert(VPBB->getSuccessors().empty() &&((VPBB->getSuccessors().empty() && "VPBB has successors when handling predicated replication."
) ? static_cast<void> (0) : __assert_fail ("VPBB->getSuccessors().empty() && \"VPBB has successors when handling predicated replication.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 8695, __PRETTY_FUNCTION__))
       "VPBB has successors when handling predicated replication.")((VPBB->getSuccessors().empty() && "VPBB has successors when handling predicated replication."
) ? static_cast<void> (0) : __assert_fail ("VPBB->getSuccessors().empty() && \"VPBB has successors when handling predicated replication.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 8695, __PRETTY_FUNCTION__));
// Record predicated instructions for above packing optimizations.
VPBlockBase *Region = createReplicateRegion(I, Recipe, Plan);
VPBlockUtils::insertBlockAfter(Region, VPBB);
auto *RegSucc = new VPBasicBlock();
VPBlockUtils::insertBlockAfter(RegSucc, Region);
return RegSucc;
8702}

8704VPRegionBlock *VPRecipeBuilder::createReplicateRegion(Instruction *Instr,
                                                    VPRecipeBase *PredRecipe,
                                                    VPlanPtr &Plan) {
// Instructions marked for predication are replicated and placed under an
// if-then construct to prevent side-effects.

// Generate recipes to compute the block mask for this region.
VPValue *BlockInMask = createBlockInMask(Instr->getParent(), Plan);

// Build the triangular if-then region.
std::string RegionName = (Twine("pred.") + Instr->getOpcodeName()).str();
assert(Instr->getParent() && "Predicated instruction not in any basic block")((Instr->getParent() && "Predicated instruction not in any basic block"
) ? static_cast<void> (0) : __assert_fail ("Instr->getParent() && \"Predicated instruction not in any basic block\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 8715, __PRETTY_FUNCTION__));
auto *BOMRecipe = new VPBranchOnMaskRecipe(BlockInMask);
auto *Entry = new VPBasicBlock(Twine(RegionName) + ".entry", BOMRecipe);
auto *PHIRecipe = Instr->getType()->isVoidTy()
                      ? nullptr
                      : new VPPredInstPHIRecipe(Plan->getOrAddVPValue(Instr));
if (PHIRecipe) {
  Plan->removeVPValueFor(Instr);
  Plan->addVPValue(Instr, PHIRecipe);
}
auto *Exit = new VPBasicBlock(Twine(RegionName) + ".continue", PHIRecipe);
auto *Pred = new VPBasicBlock(Twine(RegionName) + ".if", PredRecipe);
VPRegionBlock *Region = new VPRegionBlock(Entry, Exit, RegionName, true);

// Note: first set Entry as region entry and then connect successors starting
// from it in order, to propagate the "parent" of each VPBasicBlock.
VPBlockUtils::insertTwoBlocksAfter(Pred, Exit, BlockInMask, Entry);
VPBlockUtils::connectBlocks(Pred, Exit);

return Region;
8735}

8737VPRecipeOrVPValueTy
8738VPRecipeBuilder::tryToCreateWidenRecipe(Instruction *Instr,
                                      ArrayRef<VPValue *> Operands,
                                      VFRange &Range, VPlanPtr &Plan) {
// First, check for specific widening recipes that deal with calls, memory
// operations, inductions and Phi nodes.
if (auto *CI = dyn_cast<CallInst>(Instr))
  return toVPRecipeResult(tryToWidenCall(CI, Operands, Range));

if (isa<LoadInst>(Instr) || isa<StoreInst>(Instr))
  return toVPRecipeResult(tryToWidenMemory(Instr, Operands, Range, Plan));

VPRecipeBase *Recipe;
if (auto Phi = dyn_cast<PHINode>(Instr)) {
  if (Phi->getParent() != OrigLoop->getHeader())
    return tryToBlend(Phi, Operands, Plan);
  if ((Recipe = tryToOptimizeInductionPHI(Phi, Operands)))
    return toVPRecipeResult(Recipe);

  if (Legal->isReductionVariable(Phi)) {
    RecurrenceDescriptor &RdxDesc = Legal->getReductionVars()[Phi];
    assert(RdxDesc.getRecurrenceStartValue() ==((RdxDesc.getRecurrenceStartValue() == Phi->getIncomingValueForBlock
(OrigLoop->getLoopPreheader())) ? static_cast<void> (
0) : __assert_fail ("RdxDesc.getRecurrenceStartValue() == Phi->getIncomingValueForBlock(OrigLoop->getLoopPreheader())"
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 8759, __PRETTY_FUNCTION__))
           Phi->getIncomingValueForBlock(OrigLoop->getLoopPreheader()))((RdxDesc.getRecurrenceStartValue() == Phi->getIncomingValueForBlock
(OrigLoop->getLoopPreheader())) ? static_cast<void> (
0) : __assert_fail ("RdxDesc.getRecurrenceStartValue() == Phi->getIncomingValueForBlock(OrigLoop->getLoopPreheader())"
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 8759, __PRETTY_FUNCTION__));
    VPValue *StartV = Operands[0];
    return toVPRecipeResult(new VPWidenPHIRecipe(Phi, RdxDesc, *StartV));
  }

  return toVPRecipeResult(new VPWidenPHIRecipe(Phi));
}

if (isa<TruncInst>(Instr) &&
    (Recipe = tryToOptimizeInductionTruncate(cast<TruncInst>(Instr), Operands,
                                             Range, *Plan)))
  return toVPRecipeResult(Recipe);

if (!shouldWiden(Instr, Range))
  return nullptr;

if (auto GEP = dyn_cast<GetElementPtrInst>(Instr))
  return toVPRecipeResult(new VPWidenGEPRecipe(
      GEP, make_range(Operands.begin(), Operands.end()), OrigLoop));

if (auto *SI = dyn_cast<SelectInst>(Instr)) {
  bool InvariantCond =
      PSE.getSE()->isLoopInvariant(PSE.getSCEV(SI->getOperand(0)), OrigLoop);
  return toVPRecipeResult(new VPWidenSelectRecipe(
      *SI, make_range(Operands.begin(), Operands.end()), InvariantCond));
}

return toVPRecipeResult(tryToWiden(Instr, Operands));
8787}

8789void LoopVectorizationPlanner::buildVPlansWithVPRecipes(ElementCount MinVF,
                                                      ElementCount MaxVF) {
assert(OrigLoop->isInnermost() && "Inner loop expected.")((OrigLoop->isInnermost() && "Inner loop expected."
) ? static_cast<void> (0) : __assert_fail ("OrigLoop->isInnermost() && \"Inner loop expected.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 8791, __PRETTY_FUNCTION__));
34
←
Assuming the condition is true→
35
←
'?' condition is true→

// Collect instructions from the original loop that will become trivially dead
// in the vectorized loop. We don't need to vectorize these instructions. For
// example, original induction update instructions can become dead because we
// separately emit induction "steps" when generating code for the new loop.
// Similarly, we create a new latch condition when setting up the structure
// of the new loop, so the old one can become dead.
SmallPtrSet<Instruction *, 4> DeadInstructions;
collectTriviallyDeadInstructions(DeadInstructions);

// Add assume instructions we need to drop to DeadInstructions, to prevent
// them from being added to the VPlan.
// TODO: We only need to drop assumes in blocks that get flattend. If the
// control flow is preserved, we should keep them.
auto &ConditionalAssumes = Legal->getConditionalAssumes();
DeadInstructions.insert(ConditionalAssumes.begin(), ConditionalAssumes.end());

DenseMap<Instruction *, Instruction *> &SinkAfter = Legal->getSinkAfter();
// Dead instructions do not need sinking. Remove them from SinkAfter.
for (Instruction *I : DeadInstructions)
  SinkAfter.erase(I);

auto MaxVFPlusOne = MaxVF.getWithIncrement(1);
for (ElementCount VF = MinVF; ElementCount::isKnownLT(VF, MaxVFPlusOne);) {
36
←
Loop condition is true.  Entering loop body→
  VFRange SubRange = {VF, MaxVFPlusOne};
  VPlans.push_back(
      buildVPlanWithVPRecipes(SubRange, DeadInstructions, SinkAfter));
37
←
Calling 'LoopVectorizationPlanner::buildVPlanWithVPRecipes'→
  VF = SubRange.End;
}
8821}

8823VPlanPtr LoopVectorizationPlanner::buildVPlanWithVPRecipes(
  VFRange &Range, SmallPtrSetImpl<Instruction *> &DeadInstructions,
  const DenseMap<Instruction *, Instruction *> &SinkAfter) {

SmallPtrSet<const InterleaveGroup<Instruction> *, 1> InterleaveGroups;

VPRecipeBuilder RecipeBuilder(OrigLoop, TLI, Legal, CM, PSE, Builder);

// ---------------------------------------------------------------------------
// Pre-construction: record ingredients whose recipes we'll need to further
// process after constructing the initial VPlan.
// ---------------------------------------------------------------------------

// Mark instructions we'll need to sink later and their targets as
// ingredients whose recipe we'll need to record.
for (auto &Entry : SinkAfter) {
  RecipeBuilder.recordRecipeOf(Entry.first);
  RecipeBuilder.recordRecipeOf(Entry.second);
}
for (auto &Reduction : CM.getInLoopReductionChains()) {
38
←
Assuming '__begin1' is equal to '__end1'→
  PHINode *Phi = Reduction.first;
  RecurKind Kind = Legal->getReductionVars()[Phi].getRecurrenceKind();
  const SmallVector<Instruction *, 4> &ReductionOperations = Reduction.second;

  RecipeBuilder.recordRecipeOf(Phi);
  for (auto &R : ReductionOperations) {
    RecipeBuilder.recordRecipeOf(R);
    // For min/max reducitons, where we have a pair of icmp/select, we also
    // need to record the ICmp recipe, so it can be removed later.
    if (RecurrenceDescriptor::isMinMaxRecurrenceKind(Kind))
      RecipeBuilder.recordRecipeOf(cast<Instruction>(R->getOperand(0)));
  }
}

// For each interleave group which is relevant for this (possibly trimmed)
// Range, add it to the set of groups to be later applied to the VPlan and add
// placeholders for its members' Recipes which we'll be replacing with a
// single VPInterleaveRecipe.
for (InterleaveGroup<Instruction> *IG : IAI.getInterleaveGroups()) {
  auto applyIG = [IG, this](ElementCount VF) -> bool {
    return (VF.isVector() && // Query is illegal for VF == 1
            CM.getWideningDecision(IG->getInsertPos(), VF) ==
                LoopVectorizationCostModel::CM_Interleave);
  };
  if (!getDecisionAndClampRange(applyIG, Range))
    continue;
  InterleaveGroups.insert(IG);
  for (unsigned i = 0; i < IG->getFactor(); i++)
    if (Instruction *Member = IG->getMember(i))
      RecipeBuilder.recordRecipeOf(Member);
};

// ---------------------------------------------------------------------------
// Build initial VPlan: Scan the body of the loop in a topological order to
// visit each basic block after having visited its predecessor basic blocks.
// ---------------------------------------------------------------------------

// Create a dummy pre-entry VPBasicBlock to start building the VPlan.
auto Plan = std::make_unique<VPlan>();
VPBasicBlock *VPBB = new VPBasicBlock("Pre-Entry");
Plan->setEntry(VPBB);

// Scan the body of the loop in a topological order to visit each basic block
// after having visited its predecessor basic blocks.
LoopBlocksDFS DFS(OrigLoop);
DFS.perform(LI);

for (BasicBlock *BB : make_range(DFS.beginRPO(), DFS.endRPO())) {
  // Relevant instructions from basic block BB will be grouped into VPRecipe
  // ingredients and fill a new VPBasicBlock.
  unsigned VPBBsForBB = 0;
  auto *FirstVPBBForBB = new VPBasicBlock(BB->getName());
  VPBlockUtils::insertBlockAfter(FirstVPBBForBB, VPBB);
  VPBB = FirstVPBBForBB;
  Builder.setInsertPoint(VPBB);

  // Introduce each ingredient into VPlan.
  // TODO: Model and preserve debug instrinsics in VPlan.
  for (Instruction &I : BB->instructionsWithoutDebug()) {
    Instruction *Instr = &I;

    // First filter out irrelevant instructions, to ensure no recipes are
    // built for them.
    if (isa<BranchInst>(Instr) || DeadInstructions.count(Instr))
39
←
Assuming 'Instr' is not a 'BranchInst'→
40
←
Assuming the condition is false→
41
←
Taking false branch→
      continue;

    SmallVector<VPValue *, 4> Operands;
    auto *Phi = dyn_cast<PHINode>(Instr);
42
←
Assuming 'Instr' is not a 'PHINode'→
    if (Phi42.1
'Phi' is null
 && Phi->getParent() == OrigLoop->getHeader()) {
      Operands.push_back(Plan->getOrAddVPValue(
          Phi->getIncomingValueForBlock(OrigLoop->getLoopPreheader())));
    } else {
      auto OpRange = Plan->mapToVPValues(Instr->operands());
      Operands = {OpRange.begin(), OpRange.end()};
    }
    if (auto RecipeOrValue = RecipeBuilder.tryToCreateWidenRecipe(
43
←
Taking false branch→
            Instr, Operands, Range, Plan)) {
      // If Instr can be simplified to an existing VPValue, use it.
      if (RecipeOrValue.is<VPValue *>()) {
        Plan->addVPValue(Instr, RecipeOrValue.get<VPValue *>());
        continue;
      }
      // Otherwise, add the new recipe.
      VPRecipeBase *Recipe = RecipeOrValue.get<VPRecipeBase *>();
      for (auto *Def : Recipe->definedValues()) {
        auto *UV = Def->getUnderlyingValue();
        Plan->addVPValue(UV, Def);
      }

      RecipeBuilder.setRecipe(Instr, Recipe);
      VPBB->appendRecipe(Recipe);
      continue;
    }

    // Otherwise, if all widening options failed, Instruction is to be
    // replicated. This may create a successor for VPBB.
    VPBasicBlock *NextVPBB =
        RecipeBuilder.handleReplication(Instr, Range, VPBB, Plan);
44
←
Calling 'VPRecipeBuilder::handleReplication'→
    if (NextVPBB != VPBB) {
      VPBB = NextVPBB;
      VPBB->setName(BB->hasName() ? BB->getName() + "." + Twine(VPBBsForBB++)
                                  : "");
    }
  }
}

// Discard empty dummy pre-entry VPBasicBlock. Note that other VPBasicBlocks
// may also be empty, such as the last one VPBB, reflecting original
// basic-blocks with no recipes.
VPBasicBlock *PreEntry = cast<VPBasicBlock>(Plan->getEntry());
assert(PreEntry->empty() && "Expecting empty pre-entry block.")((PreEntry->empty() && "Expecting empty pre-entry block."
) ? static_cast<void> (0) : __assert_fail ("PreEntry->empty() && \"Expecting empty pre-entry block.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 8953, __PRETTY_FUNCTION__));
VPBlockBase *Entry = Plan->setEntry(PreEntry->getSingleSuccessor());
VPBlockUtils::disconnectBlocks(PreEntry, Entry);
delete PreEntry;

// ---------------------------------------------------------------------------
// Transform initial VPlan: Apply previously taken decisions, in order, to
// bring the VPlan to its final state.
// ---------------------------------------------------------------------------

// Apply Sink-After legal constraints.
for (auto &Entry : SinkAfter) {
  VPRecipeBase *Sink = RecipeBuilder.getRecipe(Entry.first);
  VPRecipeBase *Target = RecipeBuilder.getRecipe(Entry.second);
  // If the target is in a replication region, make sure to move Sink to the
  // block after it, not into the replication region itself.
  if (auto *Region =
          dyn_cast_or_null<VPRegionBlock>(Target->getParent()->getParent())) {
    if (Region->isReplicator()) {
      assert(Region->getNumSuccessors() == 1 && "Expected SESE region!")((Region->getNumSuccessors() == 1 && "Expected SESE region!"
) ? static_cast<void> (0) : __assert_fail ("Region->getNumSuccessors() == 1 && \"Expected SESE region!\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 8972, __PRETTY_FUNCTION__));
      VPBasicBlock *NextBlock =
          cast<VPBasicBlock>(Region->getSuccessors().front());
      Sink->moveBefore(*NextBlock, NextBlock->getFirstNonPhi());
      continue;
    }
  }
  Sink->moveAfter(Target);
}

// Interleave memory: for each Interleave Group we marked earlier as relevant
// for this VPlan, replace the Recipes widening its memory instructions with a
// single VPInterleaveRecipe at its insertion point.
for (auto IG : InterleaveGroups) {
  auto *Recipe = cast<VPWidenMemoryInstructionRecipe>(
      RecipeBuilder.getRecipe(IG->getInsertPos()));
  SmallVector<VPValue *, 4> StoredValues;
  for (unsigned i = 0; i < IG->getFactor(); ++i)
    if (auto *SI = dyn_cast_or_null<StoreInst>(IG->getMember(i)))
      StoredValues.push_back(Plan->getOrAddVPValue(SI->getOperand(0)));

  auto *VPIG = new VPInterleaveRecipe(IG, Recipe->getAddr(), StoredValues,
                                      Recipe->getMask());
  VPIG->insertBefore(Recipe);
  unsigned J = 0;
  for (unsigned i = 0; i < IG->getFactor(); ++i)
    if (Instruction *Member = IG->getMember(i)) {
      if (!Member->getType()->isVoidTy()) {
        VPValue *OriginalV = Plan->getVPValue(Member);
        Plan->removeVPValueFor(Member);
        Plan->addVPValue(Member, VPIG->getVPValue(J));
        OriginalV->replaceAllUsesWith(VPIG->getVPValue(J));
        J++;
      }
      RecipeBuilder.getRecipe(Member)->eraseFromParent();
    }
}

// Adjust the recipes for any inloop reductions.
if (Range.Start.isVector())
  adjustRecipesForInLoopReductions(Plan, RecipeBuilder);

// Finally, if tail is folded by masking, introduce selects between the phi
// and the live-out instruction of each reduction, at the end of the latch.
if (CM.foldTailByMasking() && !Legal->getReductionVars().empty()) {
  Builder.setInsertPoint(VPBB);
  auto *Cond = RecipeBuilder.createBlockInMask(OrigLoop->getHeader(), Plan);
  for (auto &Reduction : Legal->getReductionVars()) {
    if (CM.isInLoopReduction(Reduction.first))
      continue;
    VPValue *Phi = Plan->getOrAddVPValue(Reduction.first);
    VPValue *Red = Plan->getOrAddVPValue(Reduction.second.getLoopExitInstr());
    Builder.createNaryOp(Instruction::Select, {Cond, Red, Phi});
  }
}

std::string PlanName;
raw_string_ostream RSO(PlanName);
ElementCount VF = Range.Start;
Plan->addVF(VF);
RSO << "Initial VPlan for VF={" << VF;
for (VF *= 2; ElementCount::isKnownLT(VF, Range.End); VF *= 2) {
  Plan->addVF(VF);
  RSO << "," << VF;
}
RSO << "},UF>=1";
RSO.flush();
Plan->setName(PlanName);

return Plan;
9042}

9044VPlanPtr LoopVectorizationPlanner::buildVPlan(VFRange &Range) {
// Outer loop handling: They may require CFG and instruction level
// transformations before even evaluating whether vectorization is profitable.
// Since we cannot modify the incoming IR, we need to build VPlan upfront in
// the vectorization pipeline.
assert(!OrigLoop->isInnermost())((!OrigLoop->isInnermost()) ? static_cast<void> (0) :
 __assert_fail ("!OrigLoop->isInnermost()", "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 9049, __PRETTY_FUNCTION__));
assert(EnableVPlanNativePath && "VPlan-native path is not enabled.")((EnableVPlanNativePath && "VPlan-native path is not enabled."
) ? static_cast<void> (0) : __assert_fail ("EnableVPlanNativePath && \"VPlan-native path is not enabled.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 9050, __PRETTY_FUNCTION__));

// Create new empty VPlan
auto Plan = std::make_unique<VPlan>();

// Build hierarchical CFG
VPlanHCFGBuilder HCFGBuilder(OrigLoop, LI, *Plan);
HCFGBuilder.buildHierarchicalCFG();

for (ElementCount VF = Range.Start; ElementCount::isKnownLT(VF, Range.End);
     VF *= 2)
  Plan->addVF(VF);

if (EnableVPlanPredication) {
  VPlanPredicator VPP(*Plan);
  VPP.predicate();

  // Avoid running transformation to recipes until masked code generation in
  // VPlan-native path is in place.
  return Plan;
}

SmallPtrSet<Instruction *, 1> DeadInstructions;
VPlanTransforms::VPInstructionsToVPRecipes(OrigLoop, Plan,
                                           Legal->getInductionVars(),
                                           DeadInstructions, *PSE.getSE());
return Plan;
9077}

9079// Adjust the recipes for any inloop reductions. The chain of instructions
9080// leading from the loop exit instr to the phi need to be converted to
9081// reductions, with one operand being vector and the other being the scalar
9082// reduction chain.
9083void LoopVectorizationPlanner::adjustRecipesForInLoopReductions(
  VPlanPtr &Plan, VPRecipeBuilder &RecipeBuilder) {
for (auto &Reduction : CM.getInLoopReductionChains()) {
  PHINode *Phi = Reduction.first;
  RecurrenceDescriptor &RdxDesc = Legal->getReductionVars()[Phi];
  const SmallVector<Instruction *, 4> &ReductionOperations = Reduction.second;

  // ReductionOperations are orders top-down from the phi's use to the
  // LoopExitValue. We keep a track of the previous item (the Chain) to tell
  // which of the two operands will remain scalar and which will be reduced.
  // For minmax the chain will be the select instructions.
  Instruction *Chain = Phi;
  for (Instruction *R : ReductionOperations) {
    VPRecipeBase *WidenRecipe = RecipeBuilder.getRecipe(R);
    RecurKind Kind = RdxDesc.getRecurrenceKind();

    VPValue *ChainOp = Plan->getVPValue(Chain);
    unsigned FirstOpId;
    if (RecurrenceDescriptor::isMinMaxRecurrenceKind(Kind)) {
      assert(isa<VPWidenSelectRecipe>(WidenRecipe) &&((isa<VPWidenSelectRecipe>(WidenRecipe) && "Expected to replace a VPWidenSelectSC"
) ? static_cast<void> (0) : __assert_fail ("isa<VPWidenSelectRecipe>(WidenRecipe) && \"Expected to replace a VPWidenSelectSC\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 9103, __PRETTY_FUNCTION__))
             "Expected to replace a VPWidenSelectSC")((isa<VPWidenSelectRecipe>(WidenRecipe) && "Expected to replace a VPWidenSelectSC"
) ? static_cast<void> (0) : __assert_fail ("isa<VPWidenSelectRecipe>(WidenRecipe) && \"Expected to replace a VPWidenSelectSC\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 9103, __PRETTY_FUNCTION__));
      FirstOpId = 1;
    } else {
      assert(isa<VPWidenRecipe>(WidenRecipe) &&((isa<VPWidenRecipe>(WidenRecipe) && "Expected to replace a VPWidenSC"
) ? static_cast<void> (0) : __assert_fail ("isa<VPWidenRecipe>(WidenRecipe) && \"Expected to replace a VPWidenSC\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 9107, __PRETTY_FUNCTION__))
             "Expected to replace a VPWidenSC")((isa<VPWidenRecipe>(WidenRecipe) && "Expected to replace a VPWidenSC"
) ? static_cast<void> (0) : __assert_fail ("isa<VPWidenRecipe>(WidenRecipe) && \"Expected to replace a VPWidenSC\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 9107, __PRETTY_FUNCTION__));
      FirstOpId = 0;
    }
    unsigned VecOpId =
        R->getOperand(FirstOpId) == Chain ? FirstOpId + 1 : FirstOpId;
    VPValue *VecOp = Plan->getVPValue(R->getOperand(VecOpId));

    auto *CondOp = CM.foldTailByMasking()
                       ? RecipeBuilder.createBlockInMask(R->getParent(), Plan)
                       : nullptr;
    VPReductionRecipe *RedRecipe = new VPReductionRecipe(
        &RdxDesc, R, ChainOp, VecOp, CondOp, TTI);
    WidenRecipe->getVPValue()->replaceAllUsesWith(RedRecipe);
    Plan->removeVPValueFor(R);
    Plan->addVPValue(R, RedRecipe);
    WidenRecipe->getParent()->insert(RedRecipe, WidenRecipe->getIterator());
    WidenRecipe->getVPValue()->replaceAllUsesWith(RedRecipe);
    WidenRecipe->eraseFromParent();

    if (RecurrenceDescriptor::isMinMaxRecurrenceKind(Kind)) {
      VPRecipeBase *CompareRecipe =
          RecipeBuilder.getRecipe(cast<Instruction>(R->getOperand(0)));
      assert(isa<VPWidenRecipe>(CompareRecipe) &&((isa<VPWidenRecipe>(CompareRecipe) && "Expected to replace a VPWidenSC"
) ? static_cast<void> (0) : __assert_fail ("isa<VPWidenRecipe>(CompareRecipe) && \"Expected to replace a VPWidenSC\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 9130, __PRETTY_FUNCTION__))
             "Expected to replace a VPWidenSC")((isa<VPWidenRecipe>(CompareRecipe) && "Expected to replace a VPWidenSC"
) ? static_cast<void> (0) : __assert_fail ("isa<VPWidenRecipe>(CompareRecipe) && \"Expected to replace a VPWidenSC\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 9130, __PRETTY_FUNCTION__));
      assert(cast<VPWidenRecipe>(CompareRecipe)->getNumUsers() == 0 &&((cast<VPWidenRecipe>(CompareRecipe)->getNumUsers() ==
&& "Expected no remaining users") ? static_cast<
void> (0) : __assert_fail ("cast<VPWidenRecipe>(CompareRecipe)->getNumUsers() == 0 && \"Expected no remaining users\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 9132, __PRETTY_FUNCTION__))
             "Expected no remaining users")((cast<VPWidenRecipe>(CompareRecipe)->getNumUsers() ==
&& "Expected no remaining users") ? static_cast<
void> (0) : __assert_fail ("cast<VPWidenRecipe>(CompareRecipe)->getNumUsers() == 0 && \"Expected no remaining users\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 9132, __PRETTY_FUNCTION__));
      CompareRecipe->eraseFromParent();
    }
    Chain = R;
  }
}
9138}

9140#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
9141void VPInterleaveRecipe::print(raw_ostream &O, const Twine &Indent,
                             VPSlotTracker &SlotTracker) const {
O << Indent << "INTERLEAVE-GROUP with factor " << IG->getFactor() << " at ";
IG->getInsertPos()->printAsOperand(O, false);
O << ", ";
getAddr()->printAsOperand(O, SlotTracker);
VPValue *Mask = getMask();
if (Mask) {
  O << ", ";
  Mask->printAsOperand(O, SlotTracker);
}
for (unsigned i = 0; i < IG->getFactor(); ++i)
  if (Instruction *I = IG->getMember(i))
    O << "\n" << Indent << "  " << VPlanIngredient(I) << " " << i;
9155}
9156#endif

9158void VPWidenCallRecipe::execute(VPTransformState &State) {
State.ILV->widenCallInstruction(*cast<CallInst>(getUnderlyingInstr()), this,
                                *this, State);
9161}

9163void VPWidenSelectRecipe::execute(VPTransformState &State) {
State.ILV->widenSelectInstruction(*cast<SelectInst>(getUnderlyingInstr()),
                                  this, *this, InvariantCond, State);
9166}

9168void VPWidenRecipe::execute(VPTransformState &State) {
State.ILV->widenInstruction(*getUnderlyingInstr(), this, *this, State);
9170}

9172void VPWidenGEPRecipe::execute(VPTransformState &State) {
State.ILV->widenGEP(cast<GetElementPtrInst>(getUnderlyingInstr()), this,
                    *this, State.UF, State.VF, IsPtrLoopInvariant,
                    IsIndexLoopInvariant, State);
9176}

9178void VPWidenIntOrFpInductionRecipe::execute(VPTransformState &State) {
assert(!State.Instance && "Int or FP induction being replicated.")((!State.Instance && "Int or FP induction being replicated."
) ? static_cast<void> (0) : __assert_fail ("!State.Instance && \"Int or FP induction being replicated.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 9179, __PRETTY_FUNCTION__));
State.ILV->widenIntOrFpInduction(IV, getStartValue()->getLiveInIRValue(),
                                 getTruncInst(), getVPValue(0),
                                 getCastValue(), State);
9183}

9185void VPWidenPHIRecipe::execute(VPTransformState &State) {
State.ILV->widenPHIInstruction(cast<PHINode>(getUnderlyingValue()), RdxDesc,
                               this, State);
9188}

9190void VPBlendRecipe::execute(VPTransformState &State) {
State.ILV->setDebugLocFromInst(State.Builder, Phi);
// We know that all PHIs in non-header blocks are converted into
// selects, so we don't have to worry about the insertion order and we
// can just use the builder.
// At this point we generate the predication tree. There may be
// duplications since this is a simple recursive scan, but future
// optimizations will clean it up.

unsigned NumIncoming = getNumIncomingValues();

// Generate a sequence of selects of the form:
// SELECT(Mask3, In3,
//        SELECT(Mask2, In2,
//               SELECT(Mask1, In1,
//                      In0)))
// Note that Mask0 is never used: lanes for which no path reaches this phi and
// are essentially undef are taken from In0.
InnerLoopVectorizer::VectorParts Entry(State.UF);
for (unsigned In = 0; In < NumIncoming; ++In) {
  for (unsigned Part = 0; Part < State.UF; ++Part) {
    // We might have single edge PHIs (blocks) - use an identity
    // 'select' for the first PHI operand.
    Value *In0 = State.get(getIncomingValue(In), Part);
    if (In == 0)
      Entry[Part] = In0; // Initialize with the first incoming value.
    else {
      // Select between the current value and the previous incoming edge
      // based on the incoming mask.
      Value *Cond = State.get(getMask(In), Part);
      Entry[Part] =
          State.Builder.CreateSelect(Cond, In0, Entry[Part], "predphi");
    }
  }
}
for (unsigned Part = 0; Part < State.UF; ++Part)
  State.set(this, Entry[Part], Part);
9227}

9229void VPInterleaveRecipe::execute(VPTransformState &State) {
assert(!State.Instance && "Interleave group being replicated.")((!State.Instance && "Interleave group being replicated."
) ? static_cast<void> (0) : __assert_fail ("!State.Instance && \"Interleave group being replicated.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 9230, __PRETTY_FUNCTION__));
State.ILV->vectorizeInterleaveGroup(IG, definedValues(), State, getAddr(),
                                    getStoredValues(), getMask());
9233}

9235void VPReductionRecipe::execute(VPTransformState &State) {
assert(!State.Instance && "Reduction being replicated.")((!State.Instance && "Reduction being replicated.") ?
 static_cast<void> (0) : __assert_fail ("!State.Instance && \"Reduction being replicated.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 9236, __PRETTY_FUNCTION__));
Value *PrevInChain = State.get(getChainOp(), 0);
for (unsigned Part = 0; Part < State.UF; ++Part) {
  RecurKind Kind = RdxDesc->getRecurrenceKind();
  bool IsOrdered = useOrderedReductions(*RdxDesc);
  Value *NewVecOp = State.get(getVecOp(), Part);
  if (VPValue *Cond = getCondOp()) {
    Value *NewCond = State.get(Cond, Part);
    VectorType *VecTy = cast<VectorType>(NewVecOp->getType());
    Constant *Iden = RecurrenceDescriptor::getRecurrenceIdentity(
        Kind, VecTy->getElementType(), RdxDesc->getFastMathFlags());
    Constant *IdenVec =
        ConstantVector::getSplat(VecTy->getElementCount(), Iden);
    Value *Select = State.Builder.CreateSelect(NewCond, NewVecOp, IdenVec);
    NewVecOp = Select;
  }
  Value *NewRed;
  Value *NextInChain;
  if (IsOrdered) {
    NewRed = createOrderedReduction(State.Builder, *RdxDesc, NewVecOp,
                                    PrevInChain);
    PrevInChain = NewRed;
  } else {
    PrevInChain = State.get(getChainOp(), Part);
    NewRed = createTargetReduction(State.Builder, TTI, *RdxDesc, NewVecOp);
  }
  if (RecurrenceDescriptor::isMinMaxRecurrenceKind(Kind)) {
    NextInChain =
        createMinMaxOp(State.Builder, RdxDesc->getRecurrenceKind(),
                       NewRed, PrevInChain);
  } else if (IsOrdered)
    NextInChain = NewRed;
  else {
    NextInChain = State.Builder.CreateBinOp(
        (Instruction::BinaryOps)getUnderlyingInstr()->getOpcode(), NewRed,
        PrevInChain);
  }
  State.set(this, NextInChain, Part);
}
9275}

9277void VPReplicateRecipe::execute(VPTransformState &State) {
if (State.Instance) { // Generate a single instance.
  assert(!State.VF.isScalable() && "Can't scalarize a scalable vector")((!State.VF.isScalable() && "Can't scalarize a scalable vector"
) ? static_cast<void> (0) : __assert_fail ("!State.VF.isScalable() && \"Can't scalarize a scalable vector\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 9279, __PRETTY_FUNCTION__));
  State.ILV->scalarizeInstruction(getUnderlyingInstr(), this, *this,
                                  *State.Instance, IsPredicated, State);
  // Insert scalar instance packing it into a vector.
  if (AlsoPack && State.VF.isVector()) {
    // If we're constructing lane 0, initialize to start from poison.
    if (State.Instance->Lane.isFirstLane()) {
      assert(!State.VF.isScalable() && "VF is assumed to be non scalable.")((!State.VF.isScalable() && "VF is assumed to be non scalable."
) ? static_cast<void> (0) : __assert_fail ("!State.VF.isScalable() && \"VF is assumed to be non scalable.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 9286, __PRETTY_FUNCTION__));
      Value *Poison = PoisonValue::get(
          VectorType::get(getUnderlyingValue()->getType(), State.VF));
      State.set(this, Poison, State.Instance->Part);
    }
    State.ILV->packScalarIntoVectorValue(this, *State.Instance, State);
  }
  return;
}

// Generate scalar instances for all VF lanes of all UF parts, unless the
// instruction is uniform inwhich case generate only the first lane for each
// of the UF parts.
unsigned EndLane = IsUniform ? 1 : State.VF.getKnownMinValue();
assert((!State.VF.isScalable() || IsUniform) &&(((!State.VF.isScalable() || IsUniform) && "Can't scalarize a scalable vector"
) ? static_cast<void> (0) : __assert_fail ("(!State.VF.isScalable() || IsUniform) && \"Can't scalarize a scalable vector\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 9301, __PRETTY_FUNCTION__))
       "Can't scalarize a scalable vector")(((!State.VF.isScalable() || IsUniform) && "Can't scalarize a scalable vector"
) ? static_cast<void> (0) : __assert_fail ("(!State.VF.isScalable() || IsUniform) && \"Can't scalarize a scalable vector\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 9301, __PRETTY_FUNCTION__));
for (unsigned Part = 0; Part < State.UF; ++Part)
  for (unsigned Lane = 0; Lane < EndLane; ++Lane)
    State.ILV->scalarizeInstruction(getUnderlyingInstr(), this, *this,
                                    VPIteration(Part, Lane), IsPredicated,
                                    State);
9307}

9309void VPBranchOnMaskRecipe::execute(VPTransformState &State) {
assert(State.Instance && "Branch on Mask works only on single instance.")((State.Instance && "Branch on Mask works only on single instance."
) ? static_cast<void> (0) : __assert_fail ("State.Instance && \"Branch on Mask works only on single instance.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 9310, __PRETTY_FUNCTION__));

unsigned Part = State.Instance->Part;
unsigned Lane = State.Instance->Lane.getKnownLane();

Value *ConditionBit = nullptr;
VPValue *BlockInMask = getMask();
if (BlockInMask) {
  ConditionBit = State.get(BlockInMask, Part);
  if (ConditionBit->getType()->isVectorTy())
    ConditionBit = State.Builder.CreateExtractElement(
        ConditionBit, State.Builder.getInt32(Lane));
} else // Block in mask is all-one.
  ConditionBit = State.Builder.getTrue();

// Replace the temporary unreachable terminator with a new conditional branch,
// whose two destinations will be set later when they are created.
auto *CurrentTerminator = State.CFG.PrevBB->getTerminator();
assert(isa<UnreachableInst>(CurrentTerminator) &&((isa<UnreachableInst>(CurrentTerminator) && "Expected to replace unreachable terminator with conditional branch."
) ? static_cast<void> (0) : __assert_fail ("isa<UnreachableInst>(CurrentTerminator) && \"Expected to replace unreachable terminator with conditional branch.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 9329, __PRETTY_FUNCTION__))
       "Expected to replace unreachable terminator with conditional branch.")((isa<UnreachableInst>(CurrentTerminator) && "Expected to replace unreachable terminator with conditional branch."
) ? static_cast<void> (0) : __assert_fail ("isa<UnreachableInst>(CurrentTerminator) && \"Expected to replace unreachable terminator with conditional branch.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 9329, __PRETTY_FUNCTION__));
auto *CondBr = BranchInst::Create(State.CFG.PrevBB, nullptr, ConditionBit);
CondBr->setSuccessor(0, nullptr);
ReplaceInstWithInst(CurrentTerminator, CondBr);
9333}

9335void VPPredInstPHIRecipe::execute(VPTransformState &State) {
assert(State.Instance && "Predicated instruction PHI works per instance.")((State.Instance && "Predicated instruction PHI works per instance."
) ? static_cast<void> (0) : __assert_fail ("State.Instance && \"Predicated instruction PHI works per instance.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 9336, __PRETTY_FUNCTION__));
Instruction *ScalarPredInst =
    cast<Instruction>(State.get(getOperand(0), *State.Instance));
BasicBlock *PredicatedBB = ScalarPredInst->getParent();
BasicBlock *PredicatingBB = PredicatedBB->getSinglePredecessor();
assert(PredicatingBB && "Predicated block has no single predecessor.")((PredicatingBB && "Predicated block has no single predecessor."
) ? static_cast<void> (0) : __assert_fail ("PredicatingBB && \"Predicated block has no single predecessor.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 9341, __PRETTY_FUNCTION__));
assert(isa<VPReplicateRecipe>(getOperand(0)) &&((isa<VPReplicateRecipe>(getOperand(0)) && "operand must be VPReplicateRecipe"
) ? static_cast<void> (0) : __assert_fail ("isa<VPReplicateRecipe>(getOperand(0)) && \"operand must be VPReplicateRecipe\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 9343, __PRETTY_FUNCTION__))
       "operand must be VPReplicateRecipe")((isa<VPReplicateRecipe>(getOperand(0)) && "operand must be VPReplicateRecipe"
) ? static_cast<void> (0) : __assert_fail ("isa<VPReplicateRecipe>(getOperand(0)) && \"operand must be VPReplicateRecipe\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 9343, __PRETTY_FUNCTION__));

// By current pack/unpack logic we need to generate only a single phi node: if
// a vector value for the predicated instruction exists at this point it means
// the instruction has vector users only, and a phi for the vector value is
// needed. In this case the recipe of the predicated instruction is marked to
// also do that packing, thereby "hoisting" the insert-element sequence.
// Otherwise, a phi node for the scalar value is needed.
unsigned Part = State.Instance->Part;
if (State.hasVectorValue(getOperand(0), Part)) {
  Value *VectorValue = State.get(getOperand(0), Part);
  InsertElementInst *IEI = cast<InsertElementInst>(VectorValue);
  PHINode *VPhi = State.Builder.CreatePHI(IEI->getType(), 2);
  VPhi->addIncoming(IEI->getOperand(0), PredicatingBB); // Unmodified vector.
  VPhi->addIncoming(IEI, PredicatedBB); // New vector with inserted element.
  if (State.hasVectorValue(this, Part))
    State.reset(this, VPhi, Part);
  else
    State.set(this, VPhi, Part);
  // NOTE: Currently we need to update the value of the operand, so the next
  // predicated iteration inserts its generated value in the correct vector.
  State.reset(getOperand(0), VPhi, Part);
} else {
  Type *PredInstType = getOperand(0)->getUnderlyingValue()->getType();
  PHINode *Phi = State.Builder.CreatePHI(PredInstType, 2);
  Phi->addIncoming(PoisonValue::get(ScalarPredInst->getType()),
                   PredicatingBB);
  Phi->addIncoming(ScalarPredInst, PredicatedBB);
  if (State.hasScalarValue(this, *State.Instance))
    State.reset(this, Phi, *State.Instance);
  else
    State.set(this, Phi, *State.Instance);
  // NOTE: Currently we need to update the value of the operand, so the next
  // predicated iteration inserts its generated value in the correct vector.
  State.reset(getOperand(0), Phi, *State.Instance);
}
9379}

9381void VPWidenMemoryInstructionRecipe::execute(VPTransformState &State) {
VPValue *StoredValue = isStore() ? getStoredValue() : nullptr;
State.ILV->vectorizeMemoryInstruction(&Ingredient, State,
                                      StoredValue ? nullptr : getVPValue(),
                                      getAddr(), StoredValue, getMask());
9386}

9388// Determine how to lower the scalar epilogue, which depends on 1) optimising
9389// for minimum code-size, 2) predicate compiler options, 3) loop hints forcing
9390// predication, and 4) a TTI hook that analyses whether the loop is suitable
9391// for predication.
9392static ScalarEpilogueLowering getScalarEpilogueLowering(
  Function *F, Loop *L, LoopVectorizeHints &Hints, ProfileSummaryInfo *PSI,
  BlockFrequencyInfo *BFI, TargetTransformInfo *TTI, TargetLibraryInfo *TLI,
  AssumptionCache *AC, LoopInfo *LI, ScalarEvolution *SE, DominatorTree *DT,
  LoopVectorizationLegality &LVL) {
// 1) OptSize takes precedence over all other options, i.e. if this is set,
// don't look at hints or options, and don't request a scalar epilogue.
// (For PGSO, as shouldOptimizeForSize isn't currently accessible from
// LoopAccessInfo (due to code dependency and not being able to reliably get
// PSI/BFI from a loop analysis under NPM), we cannot suppress the collection
// of strides in LoopAccessInfo::analyzeLoop() and vectorize without
// versioning when the vectorization is forced, unlike hasOptSize. So revert
// back to the old way and vectorize with versioning when forced. See D81345.)
if (F->hasOptSize() || (llvm::shouldOptimizeForSize(L->getHeader(), PSI, BFI,
                                                    PGSOQueryType::IRPass) &&
                        Hints.getForce() != LoopVectorizeHints::FK_Enabled))
  return CM_ScalarEpilogueNotAllowedOptSize;

// 2) If set, obey the directives
if (PreferPredicateOverEpilogue.getNumOccurrences()) {
  switch (PreferPredicateOverEpilogue) {
  case PreferPredicateTy::ScalarEpilogue:
    return CM_ScalarEpilogueAllowed;
  case PreferPredicateTy::PredicateElseScalarEpilogue:
    return CM_ScalarEpilogueNotNeededUsePredicate;
  case PreferPredicateTy::PredicateOrDontVectorize:
    return CM_ScalarEpilogueNotAllowedUsePredicate;
  };
}

// 3) If set, obey the hints
switch (Hints.getPredicate()) {
case LoopVectorizeHints::FK_Enabled:
  return CM_ScalarEpilogueNotNeededUsePredicate;
case LoopVectorizeHints::FK_Disabled:
  return CM_ScalarEpilogueAllowed;
};

// 4) if the TTI hook indicates this is profitable, request predication.
if (TTI->preferPredicateOverEpilogue(L, LI, *SE, *AC, TLI, DT,
                                     LVL.getLAI()))
  return CM_ScalarEpilogueNotNeededUsePredicate;

return CM_ScalarEpilogueAllowed;
9436}

9438Value *VPTransformState::get(VPValue *Def, unsigned Part) {
// If Values have been set for this Def return the one relevant for \p Part.
if (hasVectorValue(Def, Part))
  return Data.PerPartOutput[Def][Part];

if (!hasScalarValue(Def, {Part, 0})) {
  Value *IRV = Def->getLiveInIRValue();
  Value *B = ILV->getBroadcastInstrs(IRV);
  set(Def, B, Part);
  return B;
}

Value *ScalarValue = get(Def, {Part, 0});
// If we aren't vectorizing, we can just copy the scalar map values over
// to the vector map.
if (VF.isScalar()) {
  set(Def, ScalarValue, Part);
  return ScalarValue;
}

auto *RepR = dyn_cast<VPReplicateRecipe>(Def);
bool IsUniform = RepR && RepR->isUniform();

unsigned LastLane = IsUniform ? 0 : VF.getKnownMinValue() - 1;
// Check if there is a scalar value for the selected lane.
if (!hasScalarValue(Def, {Part, LastLane})) {
  // At the moment, VPWidenIntOrFpInductionRecipes can also be uniform.
  assert(isa<VPWidenIntOrFpInductionRecipe>(Def->getDef()) &&((isa<VPWidenIntOrFpInductionRecipe>(Def->getDef()) &&
 "unexpected recipe found to be invariant") ? static_cast<
void> (0) : __assert_fail ("isa<VPWidenIntOrFpInductionRecipe>(Def->getDef()) && \"unexpected recipe found to be invariant\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 9466, __PRETTY_FUNCTION__))
         "unexpected recipe found to be invariant")((isa<VPWidenIntOrFpInductionRecipe>(Def->getDef()) &&
 "unexpected recipe found to be invariant") ? static_cast<
void> (0) : __assert_fail ("isa<VPWidenIntOrFpInductionRecipe>(Def->getDef()) && \"unexpected recipe found to be invariant\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 9466, __PRETTY_FUNCTION__));
  IsUniform = true;
  LastLane = 0;
}

auto *LastInst = cast<Instruction>(get(Def, {Part, LastLane}));

// Set the insert point after the last scalarized instruction. This
// ensures the insertelement sequence will directly follow the scalar
// definitions.
auto OldIP = Builder.saveIP();
auto NewIP = std::next(BasicBlock::iterator(LastInst));
Builder.SetInsertPoint(&*NewIP);

// However, if we are vectorizing, we need to construct the vector values.
// If the value is known to be uniform after vectorization, we can just
// broadcast the scalar value corresponding to lane zero for each unroll
// iteration. Otherwise, we construct the vector values using
// insertelement instructions. Since the resulting vectors are stored in
// State, we will only generate the insertelements once.
Value *VectorValue = nullptr;
if (IsUniform) {
  VectorValue = ILV->getBroadcastInstrs(ScalarValue);
  set(Def, VectorValue, Part);
} else {
  // Initialize packing with insertelements to start from undef.
  assert(!VF.isScalable() && "VF is assumed to be non scalable.")((!VF.isScalable() && "VF is assumed to be non scalable."
) ? static_cast<void> (0) : __assert_fail ("!VF.isScalable() && \"VF is assumed to be non scalable.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 9492, __PRETTY_FUNCTION__));
  Value *Undef = PoisonValue::get(VectorType::get(LastInst->getType(), VF));
  set(Def, Undef, Part);
  for (unsigned Lane = 0; Lane < VF.getKnownMinValue(); ++Lane)
    ILV->packScalarIntoVectorValue(Def, {Part, Lane}, *this);
  VectorValue = get(Def, Part);
}
Builder.restoreIP(OldIP);
return VectorValue;
9501}

9503// Process the loop in the VPlan-native vectorization path. This path builds
9504// VPlan upfront in the vectorization pipeline, which allows to apply
9505// VPlan-to-VPlan transformations from the very beginning without modifying the
9506// input LLVM IR.
9507static bool processLoopInVPlanNativePath(
  Loop *L, PredicatedScalarEvolution &PSE, LoopInfo *LI, DominatorTree *DT,
  LoopVectorizationLegality *LVL, TargetTransformInfo *TTI,
  TargetLibraryInfo *TLI, DemandedBits *DB, AssumptionCache *AC,
  OptimizationRemarkEmitter *ORE, BlockFrequencyInfo *BFI,
  ProfileSummaryInfo *PSI, LoopVectorizeHints &Hints,
  LoopVectorizationRequirements &Requirements) {

if (isa<SCEVCouldNotCompute>(PSE.getBackedgeTakenCount())) {
  LLVM_DEBUG(dbgs() << "LV: cannot compute the outer-loop trip count\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: cannot compute the outer-loop trip count\n"
; } } while (false);
  return false;
}
assert(EnableVPlanNativePath && "VPlan-native path is disabled.")((EnableVPlanNativePath && "VPlan-native path is disabled."
) ? static_cast<void> (0) : __assert_fail ("EnableVPlanNativePath && \"VPlan-native path is disabled.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 9519, __PRETTY_FUNCTION__));
Function *F = L->getHeader()->getParent();
InterleavedAccessInfo IAI(PSE, L, DT, LI, LVL->getLAI());

ScalarEpilogueLowering SEL = getScalarEpilogueLowering(
    F, L, Hints, PSI, BFI, TTI, TLI, AC, LI, PSE.getSE(), DT, *LVL);

LoopVectorizationCostModel CM(SEL, L, PSE, LI, LVL, *TTI, TLI, DB, AC, ORE, F,
                              &Hints, IAI);
// Use the planner for outer loop vectorization.
// TODO: CM is not used at this point inside the planner. Turn CM into an
// optional argument if we don't need it in the future.
LoopVectorizationPlanner LVP(L, LI, TLI, TTI, LVL, CM, IAI, PSE, Hints,
                             Requirements, ORE);

// Get user vectorization factor.
ElementCount UserVF = Hints.getWidth();

// Plan how to best vectorize, return the best VF and its cost.
const VectorizationFactor VF = LVP.planInVPlanNativePath(UserVF);

// If we are stress testing VPlan builds, do not attempt to generate vector
// code. Masked vector code generation support will follow soon.
// Also, do not attempt to vectorize if no vector code will be produced.
if (VPlanBuildStressTest || EnableVPlanPredication ||
    VectorizationFactor::Disabled() == VF)
  return false;

LVP.setBestPlan(VF.Width, 1);

{
  GeneratedRTChecks Checks(*PSE.getSE(), DT, LI,
                           F->getParent()->getDataLayout());
  InnerLoopVectorizer LB(L, PSE, LI, DT, TLI, TTI, AC, ORE, VF.Width, 1, LVL,
                         &CM, BFI, PSI, Checks);
  LLVM_DEBUG(dbgs() << "Vectorizing outer loop in \""do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "Vectorizing outer loop in \""
 << L->getHeader()->getParent()->getName() <<
 "\"\n"; } } while (false)
                    << L->getHeader()->getParent()->getName() << "\"\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "Vectorizing outer loop in \""
 << L->getHeader()->getParent()->getName() <<
 "\"\n"; } } while (false);
  LVP.executePlan(LB, DT);
}

// Mark the loop as already vectorized to avoid vectorizing again.
Hints.setAlreadyVectorized();
assert(!verifyFunction(*L->getHeader()->getParent(), &dbgs()))((!verifyFunction(*L->getHeader()->getParent(), &dbgs
())) ? static_cast<void> (0) : __assert_fail ("!verifyFunction(*L->getHeader()->getParent(), &dbgs())"
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 9561, __PRETTY_FUNCTION__));
return true;
9563}

9565// Emit a remark if there are stores to floats that required a floating point
9566// extension. If the vectorized loop was generated with floating point there
9567// will be a performance penalty from the conversion overhead and the change in
9568// the vector width.
9569static void checkMixedPrecision(Loop *L, OptimizationRemarkEmitter *ORE) {
SmallVector<Instruction *, 4> Worklist;
for (BasicBlock *BB : L->getBlocks()) {
  for (Instruction &Inst : *BB) {
    if (auto *S = dyn_cast<StoreInst>(&Inst)) {
      if (S->getValueOperand()->getType()->isFloatTy())
        Worklist.push_back(S);
    }
  }
}

// Traverse the floating point stores upwards searching, for floating point
// conversions.
SmallPtrSet<const Instruction *, 4> Visited;
SmallPtrSet<const Instruction *, 4> EmittedRemark;
while (!Worklist.empty()) {
  auto *I = Worklist.pop_back_val();
  if (!L->contains(I))
    continue;
  if (!Visited.insert(I).second)
    continue;

  // Emit a remark if the floating point store required a floating
  // point conversion.
  // TODO: More work could be done to identify the root cause such as a
  // constant or a function return type and point the user to it.
  if (isa<FPExtInst>(I) && EmittedRemark.insert(I).second)
    ORE->emit([&]() {
      return OptimizationRemarkAnalysis(LV_NAME"loop-vectorize", "VectorMixedPrecision",
                                        I->getDebugLoc(), L->getHeader())
             << "floating point conversion changes vector width. "
             << "Mixed floating point precision requires an up/down "
             << "cast that will negatively impact performance.";
    });

  for (Use &Op : I->operands())
    if (auto *OpI = dyn_cast<Instruction>(Op))
      Worklist.push_back(OpI);
}
9608}

9610LoopVectorizePass::LoopVectorizePass(LoopVectorizeOptions Opts)
  : InterleaveOnlyWhenForced(Opts.InterleaveOnlyWhenForced ||
                             !EnableLoopInterleaving),
    VectorizeOnlyWhenForced(Opts.VectorizeOnlyWhenForced ||
                            !EnableLoopVectorization) {}

9616bool LoopVectorizePass::processLoop(Loop *L) {
assert((EnableVPlanNativePath || L->isInnermost()) &&(((EnableVPlanNativePath || L->isInnermost()) && "VPlan-native path is not enabled. Only process inner loops."
) ? static_cast<void> (0) : __assert_fail ("(EnableVPlanNativePath || L->isInnermost()) && \"VPlan-native path is not enabled. Only process inner loops.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 9618, __PRETTY_FUNCTION__))
1
Assuming the condition is false→
2
←
Assuming the condition is true→
3
←
'?' condition is true→
       "VPlan-native path is not enabled. Only process inner loops.")(((EnableVPlanNativePath || L->isInnermost()) && "VPlan-native path is not enabled. Only process inner loops."
) ? static_cast<void> (0) : __assert_fail ("(EnableVPlanNativePath || L->isInnermost()) && \"VPlan-native path is not enabled. Only process inner loops.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 9618, __PRETTY_FUNCTION__));

9620#ifndef NDEBUG
const std::string DebugLocStr = getDebugLocString(L);
9622#endif /* NDEBUG */

LLVM_DEBUG(dbgs() << "\nLV: Checking a loop in \""do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "\nLV: Checking a loop in \""
 << L->getHeader()->getParent()->getName() <<
 "\" from " << DebugLocStr << "\n"; } } while (false
)
4
←
Assuming 'DebugFlag' is false→
5
←
Loop condition is false.  Exiting loop→
                  << L->getHeader()->getParent()->getName() << "\" from "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "\nLV: Checking a loop in \""
 << L->getHeader()->getParent()->getName() <<
 "\" from " << DebugLocStr << "\n"; } } while (false
)
                  << DebugLocStr << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "\nLV: Checking a loop in \""
 << L->getHeader()->getParent()->getName() <<
 "\" from " << DebugLocStr << "\n"; } } while (false
);

LoopVectorizeHints Hints(L, InterleaveOnlyWhenForced, *ORE);

LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Loop hints:" <<
 " force=" << (Hints.getForce() == LoopVectorizeHints::
FK_Disabled ? "disabled" : (Hints.getForce() == LoopVectorizeHints
::FK_Enabled ? "enabled" : "?")) << " width=" << Hints
.getWidth() << " unroll=" << Hints.getInterleave(
) << "\n"; } } while (false)
6
←
Assuming 'DebugFlag' is false→
7
←
Loop condition is false.  Exiting loop→
    dbgs() << "LV: Loop hints:"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Loop hints:" <<
 " force=" << (Hints.getForce() == LoopVectorizeHints::
FK_Disabled ? "disabled" : (Hints.getForce() == LoopVectorizeHints
::FK_Enabled ? "enabled" : "?")) << " width=" << Hints
.getWidth() << " unroll=" << Hints.getInterleave(
) << "\n"; } } while (false)
           << " force="do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Loop hints:" <<
 " force=" << (Hints.getForce() == LoopVectorizeHints::
FK_Disabled ? "disabled" : (Hints.getForce() == LoopVectorizeHints
::FK_Enabled ? "enabled" : "?")) << " width=" << Hints
.getWidth() << " unroll=" << Hints.getInterleave(
) << "\n"; } } while (false)
           << (Hints.getForce() == LoopVectorizeHints::FK_Disableddo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Loop hints:" <<
 " force=" << (Hints.getForce() == LoopVectorizeHints::
FK_Disabled ? "disabled" : (Hints.getForce() == LoopVectorizeHints
::FK_Enabled ? "enabled" : "?")) << " width=" << Hints
.getWidth() << " unroll=" << Hints.getInterleave(
) << "\n"; } } while (false)
                   ? "disabled"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Loop hints:" <<
 " force=" << (Hints.getForce() == LoopVectorizeHints::
FK_Disabled ? "disabled" : (Hints.getForce() == LoopVectorizeHints
::FK_Enabled ? "enabled" : "?")) << " width=" << Hints
.getWidth() << " unroll=" << Hints.getInterleave(
) << "\n"; } } while (false)
                   : (Hints.getForce() == LoopVectorizeHints::FK_Enableddo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Loop hints:" <<
 " force=" << (Hints.getForce() == LoopVectorizeHints::
FK_Disabled ? "disabled" : (Hints.getForce() == LoopVectorizeHints
::FK_Enabled ? "enabled" : "?")) << " width=" << Hints
.getWidth() << " unroll=" << Hints.getInterleave(
) << "\n"; } } while (false)
                          ? "enabled"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Loop hints:" <<
 " force=" << (Hints.getForce() == LoopVectorizeHints::
FK_Disabled ? "disabled" : (Hints.getForce() == LoopVectorizeHints
::FK_Enabled ? "enabled" : "?")) << " width=" << Hints
.getWidth() << " unroll=" << Hints.getInterleave(
) << "\n"; } } while (false)
                          : "?"))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Loop hints:" <<
 " force=" << (Hints.getForce() == LoopVectorizeHints::
FK_Disabled ? "disabled" : (Hints.getForce() == LoopVectorizeHints
::FK_Enabled ? "enabled" : "?")) << " width=" << Hints
.getWidth() << " unroll=" << Hints.getInterleave(
) << "\n"; } } while (false)
           << " width=" << Hints.getWidth()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Loop hints:" <<
 " force=" << (Hints.getForce() == LoopVectorizeHints::
FK_Disabled ? "disabled" : (Hints.getForce() == LoopVectorizeHints
::FK_Enabled ? "enabled" : "?")) << " width=" << Hints
.getWidth() << " unroll=" << Hints.getInterleave(
) << "\n"; } } while (false)
           << " unroll=" << Hints.getInterleave() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Loop hints:" <<
 " force=" << (Hints.getForce() == LoopVectorizeHints::
FK_Disabled ? "disabled" : (Hints.getForce() == LoopVectorizeHints
::FK_Enabled ? "enabled" : "?")) << " width=" << Hints
.getWidth() << " unroll=" << Hints.getInterleave(
) << "\n"; } } while (false);

// Function containing loop
Function *F = L->getHeader()->getParent();

// Looking at the diagnostic output is the only way to determine if a loop
// was vectorized (other than looking at the IR or machine code), so it
// is important to generate an optimization remark for each loop. Most of
// these messages are generated as OptimizationRemarkAnalysis. Remarks
// generated as OptimizationRemark and OptimizationRemarkMissed are
// less verbose reporting vectorized loops and unvectorized loops that may
// benefit from vectorization, respectively.

if (!Hints.allowVectorization(F, L, VectorizeOnlyWhenForced)) {
8
←
Assuming the condition is false→
9
←
Taking false branch→
  LLVM_DEBUG(dbgs() << "LV: Loop hints prevent vectorization.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Loop hints prevent vectorization.\n"
; } } while (false);
  return false;
}

PredicatedScalarEvolution PSE(*SE, *L);

// Check if it is legal to vectorize the loop.
LoopVectorizationRequirements Requirements;
LoopVectorizationLegality LVL(L, PSE, DT, TTI, TLI, AA, F, GetLAA, LI, ORE,
                              &Requirements, &Hints, DB, AC, BFI, PSI);
if (!LVL.canVectorize(EnableVPlanNativePath)) {
10
←
Assuming the condition is false→
11
←
Taking false branch→
  LLVM_DEBUG(dbgs() << "LV: Not vectorizing: Cannot prove legality.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Not vectorizing: Cannot prove legality.\n"
; } } while (false);
  Hints.emitRemarkWithHints();
  return false;
}

// Check the function attributes and profiles to find out if this function
// should be optimized for size.
ScalarEpilogueLowering SEL = getScalarEpilogueLowering(
    F, L, Hints, PSI, BFI, TTI, TLI, AC, LI, PSE.getSE(), DT, LVL);

// Entrance to the VPlan-native vectorization path. Outer loops are processed
// here. They may require CFG and instruction level transformations before
// even evaluating whether vectorization is profitable. Since we cannot modify
// the incoming IR, we need to build VPlan upfront in the vectorization
// pipeline.
if (!L->isInnermost())
12
←
Assuming the condition is false→
13
←
Taking false branch→
  return processLoopInVPlanNativePath(L, PSE, LI, DT, &LVL, TTI, TLI, DB, AC,
                                      ORE, BFI, PSI, Hints, Requirements);

assert(L->isInnermost() && "Inner loop expected.")((L->isInnermost() && "Inner loop expected.") ? static_cast
<void> (0) : __assert_fail ("L->isInnermost() && \"Inner loop expected.\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 9683, __PRETTY_FUNCTION__));
14
←
Assuming the condition is true→
15
←
'?' condition is true→

// Check the loop for a trip count threshold: vectorize loops with a tiny trip
// count by optimizing for size, to minimize overheads.
auto ExpectedTC = getSmallBestKnownTC(*SE, L);
if (ExpectedTC && *ExpectedTC < TinyTripCountVectorThreshold) {
  LLVM_DEBUG(dbgs() << "LV: Found a loop with a very small trip count. "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Found a loop with a very small trip count. "
 << "This loop is worth vectorizing only if no scalar "
 << "iteration overheads are incurred."; } } while (false
)
                    << "This loop is worth vectorizing only if no scalar "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Found a loop with a very small trip count. "
 << "This loop is worth vectorizing only if no scalar "
 << "iteration overheads are incurred."; } } while (false
)
                    << "iteration overheads are incurred.")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Found a loop with a very small trip count. "
 << "This loop is worth vectorizing only if no scalar "
 << "iteration overheads are incurred."; } } while (false
);
  if (Hints.getForce() == LoopVectorizeHints::FK_Enabled)
    LLVM_DEBUG(dbgs() << " But vectorizing was explicitly forced.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << " But vectorizing was explicitly forced.\n"
; } } while (false);
  else {
    LLVM_DEBUG(dbgs() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "\n"; } } while (false);
    SEL = CM_ScalarEpilogueNotAllowedLowTripLoop;
  }
}

// Check the function attributes to see if implicit floats are allowed.
// FIXME: This check doesn't seem possibly correct -- what if the loop is
// an integer loop and the vector instructions selected are purely integer
// vector instructions?
if (F->hasFnAttribute(Attribute::NoImplicitFloat)) {
16
←
Assuming the condition is false→
17
←
Taking false branch→
  reportVectorizationFailure(
      "Can't vectorize when the NoImplicitFloat attribute is used",
      "loop not vectorized due to NoImplicitFloat attribute",
      "NoImplicitFloat", ORE, L);
  Hints.emitRemarkWithHints();
  return false;
}

// Check if the target supports potentially unsafe FP vectorization.
// FIXME: Add a check for the type of safety issue (denormal, signaling)
// for the target we're vectorizing for, to make sure none of the
// additional fp-math flags can help.
if (Hints.isPotentiallyUnsafe() &&
    TTI->isFPVectorizationPotentiallyUnsafe()) {
  reportVectorizationFailure(
      "Potentially unsafe FP op prevents vectorization",
      "loop not vectorized due to unsafe FP support.",
      "UnsafeFP", ORE, L);
  Hints.emitRemarkWithHints();
  return false;
}

if (!Requirements.canVectorizeFPMath(Hints)) {
18
←
Taking false branch→
  ORE->emit([&]() {
    auto *ExactFPMathInst = Requirements.getExactFPInst();
    return OptimizationRemarkAnalysisFPCommute(DEBUG_TYPE"loop-vectorize", "CantReorderFPOps",
                                               ExactFPMathInst->getDebugLoc(),
                                               ExactFPMathInst->getParent())
           << "loop not vectorized: cannot prove it is safe to reorder "
              "floating-point operations";
  });
  LLVM_DEBUG(dbgs() << "LV: loop not vectorized: cannot prove it is safe to "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: loop not vectorized: cannot prove it is safe to "
 "reorder floating-point operations\n"; } } while (false)
                       "reorder floating-point operations\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: loop not vectorized: cannot prove it is safe to "
 "reorder floating-point operations\n"; } } while (false);
  Hints.emitRemarkWithHints();
  return false;
}

bool UseInterleaved = TTI->enableInterleavedAccessVectorization();
InterleavedAccessInfo IAI(PSE, L, DT, LI, LVL.getLAI());

// If an override option has been passed in for interleaved accesses, use it.
if (EnableInterleavedMemAccesses.getNumOccurrences() > 0)
19
←
Assuming the condition is false→
20
←
Taking false branch→
  UseInterleaved = EnableInterleavedMemAccesses;

// Analyze interleaved memory accesses.
if (UseInterleaved) {
21
←
Assuming 'UseInterleaved' is false→
22
←
Taking false branch→
  IAI.analyzeInterleaving(useMaskedInterleavedAccesses(*TTI));
}

// Use the cost model.
LoopVectorizationCostModel CM(SEL, L, PSE, LI, &LVL, *TTI, TLI, DB, AC, ORE,
                              F, &Hints, IAI);
CM.collectValuesToIgnore();

// Use the planner for vectorization.
LoopVectorizationPlanner LVP(L, LI, TLI, TTI, &LVL, CM, IAI, PSE, Hints,
                             Requirements, ORE);

// Get user vectorization factor and interleave count.
ElementCount UserVF = Hints.getWidth();
unsigned UserIC = Hints.getInterleave();

// Plan how to best vectorize, return the best VF and its cost.
Optional<VectorizationFactor> MaybeVF = LVP.plan(UserVF, UserIC);
23
←
Calling 'LoopVectorizationPlanner::plan'→

VectorizationFactor VF = VectorizationFactor::Disabled();
unsigned IC = 1;

if (MaybeVF) {
  VF = *MaybeVF;
  // Select the interleave count.
  IC = CM.selectInterleaveCount(VF.Width, VF.Cost);
}

// Identify the diagnostic messages that should be produced.
std::pair<StringRef, std::string> VecDiagMsg, IntDiagMsg;
bool VectorizeLoop = true, InterleaveLoop = true;
if (VF.Width.isScalar()) {
  LLVM_DEBUG(dbgs() << "LV: Vectorization is possible but not beneficial.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Vectorization is possible but not beneficial.\n"
; } } while (false);
  VecDiagMsg = std::make_pair(
      "VectorizationNotBeneficial",
      "the cost-model indicates that vectorization is not beneficial");
  VectorizeLoop = false;
}

if (!MaybeVF && UserIC > 1) {
  // Tell the user interleaving was avoided up-front, despite being explicitly
  // requested.
  LLVM_DEBUG(dbgs() << "LV: Ignoring UserIC, because vectorization and "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Ignoring UserIC, because vectorization and "
 "interleaving should be avoided up front\n"; } } while (false
)
                       "interleaving should be avoided up front\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Ignoring UserIC, because vectorization and "
 "interleaving should be avoided up front\n"; } } while (false
);
  IntDiagMsg = std::make_pair(
      "InterleavingAvoided",
      "Ignoring UserIC, because interleaving was avoided up front");
  InterleaveLoop = false;
} else if (IC == 1 && UserIC <= 1) {
  // Tell the user interleaving is not beneficial.
  LLVM_DEBUG(dbgs() << "LV: Interleaving is not beneficial.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Interleaving is not beneficial.\n"
; } } while (false);
  IntDiagMsg = std::make_pair(
      "InterleavingNotBeneficial",
      "the cost-model indicates that interleaving is not beneficial");
  InterleaveLoop = false;
  if (UserIC == 1) {
    IntDiagMsg.first = "InterleavingNotBeneficialAndDisabled";
    IntDiagMsg.second +=
        " and is explicitly disabled or interleave count is set to 1";
  }
} else if (IC > 1 && UserIC == 1) {
  // Tell the user interleaving is beneficial, but it explicitly disabled.
  LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Interleaving is beneficial but is explicitly disabled."
; } } while (false)
      dbgs() << "LV: Interleaving is beneficial but is explicitly disabled.")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Interleaving is beneficial but is explicitly disabled."
; } } while (false);
  IntDiagMsg = std::make_pair(
      "InterleavingBeneficialButDisabled",
      "the cost-model indicates that interleaving is beneficial "
      "but is explicitly disabled or interleave count is set to 1");
  InterleaveLoop = false;
}

// Override IC if user provided an interleave count.
IC = UserIC > 0 ? UserIC : IC;

// Emit diagnostic messages, if any.
const char *VAPassName = Hints.vectorizeAnalysisPassName();
if (!VectorizeLoop && !InterleaveLoop) {
  // Do not vectorize or interleaving the loop.
  ORE->emit([&]() {
    return OptimizationRemarkMissed(VAPassName, VecDiagMsg.first,
                                    L->getStartLoc(), L->getHeader())
           << VecDiagMsg.second;
  });
  ORE->emit([&]() {
    return OptimizationRemarkMissed(LV_NAME"loop-vectorize", IntDiagMsg.first,
                                    L->getStartLoc(), L->getHeader())
           << IntDiagMsg.second;
  });
  return false;
} else if (!VectorizeLoop && InterleaveLoop) {
  LLVM_DEBUG(dbgs() << "LV: Interleave Count is " << IC << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Interleave Count is "
 << IC << '\n'; } } while (false);
  ORE->emit([&]() {
    return OptimizationRemarkAnalysis(VAPassName, VecDiagMsg.first,
                                      L->getStartLoc(), L->getHeader())
           << VecDiagMsg.second;
  });
} else if (VectorizeLoop && !InterleaveLoop) {
  LLVM_DEBUG(dbgs() << "LV: Found a vectorizable loop (" << VF.Widthdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Found a vectorizable loop ("
 << VF.Width << ") in " << DebugLocStr <<
 '\n'; } } while (false)
                    << ") in " << DebugLocStr << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Found a vectorizable loop ("
 << VF.Width << ") in " << DebugLocStr <<
 '\n'; } } while (false);
  ORE->emit([&]() {
    return OptimizationRemarkAnalysis(LV_NAME"loop-vectorize", IntDiagMsg.first,
                                      L->getStartLoc(), L->getHeader())
           << IntDiagMsg.second;
  });
} else if (VectorizeLoop && InterleaveLoop) {
  LLVM_DEBUG(dbgs() << "LV: Found a vectorizable loop (" << VF.Widthdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Found a vectorizable loop ("
 << VF.Width << ") in " << DebugLocStr <<
 '\n'; } } while (false)
                    << ") in " << DebugLocStr << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Found a vectorizable loop ("
 << VF.Width << ") in " << DebugLocStr <<
 '\n'; } } while (false);
  LLVM_DEBUG(dbgs() << "LV: Interleave Count is " << IC << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Interleave Count is "
 << IC << '\n'; } } while (false);
}

bool DisableRuntimeUnroll = false;
MDNode *OrigLoopID = L->getLoopID();
{
  // Optimistically generate runtime checks. Drop them if they turn out to not
  // be profitable. Limit the scope of Checks, so the cleanup happens
  // immediately after vector codegeneration is done.
  GeneratedRTChecks Checks(*PSE.getSE(), DT, LI,
                           F->getParent()->getDataLayout());
  if (!VF.Width.isScalar() || IC > 1)
    Checks.Create(L, *LVL.getLAI(), PSE.getUnionPredicate());
  LVP.setBestPlan(VF.Width, IC);

  using namespace ore;
  if (!VectorizeLoop) {
    assert(IC > 1 && "interleave count should not be 1 or 0")((IC > 1 && "interleave count should not be 1 or 0"
) ? static_cast<void> (0) : __assert_fail ("IC > 1 && \"interleave count should not be 1 or 0\""
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 9875, __PRETTY_FUNCTION__));
    // If we decided that it is not legal to vectorize the loop, then
    // interleave it.
    InnerLoopUnroller Unroller(L, PSE, LI, DT, TLI, TTI, AC, ORE, IC, &LVL,
                               &CM, BFI, PSI, Checks);
    LVP.executePlan(Unroller, DT);

    ORE->emit([&]() {
      return OptimizationRemark(LV_NAME"loop-vectorize", "Interleaved", L->getStartLoc(),
                                L->getHeader())
             << "interleaved loop (interleaved count: "
             << NV("InterleaveCount", IC) << ")";
    });
  } else {
    // If we decided that it is *legal* to vectorize the loop, then do it.

    // Consider vectorizing the epilogue too if it's profitable.
    VectorizationFactor EpilogueVF =
        CM.selectEpilogueVectorizationFactor(VF.Width, LVP);
    if (EpilogueVF.Width.isVector()) {

      // The first pass vectorizes the main loop and creates a scalar epilogue
      // to be vectorized by executing the plan (potentially with a different
      // factor) again shortly afterwards.
      EpilogueLoopVectorizationInfo EPI(VF.Width.getKnownMinValue(), IC,
                                        EpilogueVF.Width.getKnownMinValue(),
                                        1);
      EpilogueVectorizerMainLoop MainILV(L, PSE, LI, DT, TLI, TTI, AC, ORE,
                                         EPI, &LVL, &CM, BFI, PSI, Checks);

      LVP.setBestPlan(EPI.MainLoopVF, EPI.MainLoopUF);
      LVP.executePlan(MainILV, DT);
      ++LoopsVectorized;

      simplifyLoop(L, DT, LI, SE, AC, nullptr, false /* PreserveLCSSA */);
      formLCSSARecursively(*L, *DT, LI, SE);

      // Second pass vectorizes the epilogue and adjusts the control flow
      // edges from the first pass.
      LVP.setBestPlan(EPI.EpilogueVF, EPI.EpilogueUF);
      EPI.MainLoopVF = EPI.EpilogueVF;
      EPI.MainLoopUF = EPI.EpilogueUF;
      EpilogueVectorizerEpilogueLoop EpilogILV(L, PSE, LI, DT, TLI, TTI, AC,
                                               ORE, EPI, &LVL, &CM, BFI, PSI,
                                               Checks);
      LVP.executePlan(EpilogILV, DT);
      ++LoopsEpilogueVectorized;

      if (!MainILV.areSafetyChecksAdded())
        DisableRuntimeUnroll = true;
    } else {
      InnerLoopVectorizer LB(L, PSE, LI, DT, TLI, TTI, AC, ORE, VF.Width, IC,
                             &LVL, &CM, BFI, PSI, Checks);
      LVP.executePlan(LB, DT);
      ++LoopsVectorized;

      // Add metadata to disable runtime unrolling a scalar loop when there
      // are no runtime checks about strides and memory. A scalar loop that is
      // rarely used is not worth unrolling.
      if (!LB.areSafetyChecksAdded())
        DisableRuntimeUnroll = true;
    }
    // Report the vectorization decision.
    ORE->emit([&]() {
      return OptimizationRemark(LV_NAME"loop-vectorize", "Vectorized", L->getStartLoc(),
                                L->getHeader())
             << "vectorized loop (vectorization width: "
             << NV("VectorizationFactor", VF.Width)
             << ", interleaved count: " << NV("InterleaveCount", IC) << ")";
    });
  }

  if (ORE->allowExtraAnalysis(LV_NAME"loop-vectorize"))
    checkMixedPrecision(L, ORE);
}

Optional<MDNode *> RemainderLoopID =
    makeFollowupLoopID(OrigLoopID, {LLVMLoopVectorizeFollowupAll,
                                    LLVMLoopVectorizeFollowupEpilogue});
if (RemainderLoopID.hasValue()) {
  L->setLoopID(RemainderLoopID.getValue());
} else {
  if (DisableRuntimeUnroll)
    AddRuntimeUnrollDisableMetaData(L);

  // Mark the loop as already vectorized to avoid vectorizing again.
  Hints.setAlreadyVectorized();
}

assert(!verifyFunction(*L->getHeader()->getParent(), &dbgs()))((!verifyFunction(*L->getHeader()->getParent(), &dbgs
())) ? static_cast<void> (0) : __assert_fail ("!verifyFunction(*L->getHeader()->getParent(), &dbgs())"
, "/build/llvm-toolchain-snapshot-13~++20210413100635+64c24f493e5f/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 9964, __PRETTY_FUNCTION__));
return true;
9966}

9968LoopVectorizeResult LoopVectorizePass::runImpl(
  Function &F, ScalarEvolution &SE_, LoopInfo &LI_, TargetTransformInfo &TTI_,
  DominatorTree &DT_, BlockFrequencyInfo &BFI_, TargetLibraryInfo *TLI_,
  DemandedBits &DB_, AAResults &AA_, AssumptionCache &AC_,
  std::function<const LoopAccessInfo &(Loop &)> &GetLAA_,
  OptimizationRemarkEmitter &ORE_, ProfileSummaryInfo *PSI_) {
SE = &SE_;
LI = &LI_;
TTI = &TTI_;
DT = &DT_;
BFI = &BFI_;
TLI = TLI_;
AA = &AA_;
AC = &AC_;
GetLAA = &GetLAA_;
DB = &DB_;
ORE = &ORE_;
PSI = PSI_;

// Don't attempt if
// 1. the target claims to have no vector registers, and
// 2. interleaving won't help ILP.
//
// The second condition is necessary because, even if the target has no
// vector registers, loop vectorization may still enable scalar
// interleaving.
if (!TTI->getNumberOfRegisters(TTI->getRegisterClassForType(true)) &&
    TTI->getMaxInterleaveFactor(1) < 2)
  return LoopVectorizeResult(false, false);

bool Changed = false, CFGChanged = false;

// The vectorizer requires loops to be in simplified form.
// Since simplification may add new inner loops, it has to run before the
// legality and profitability checks. This means running the loop vectorizer
// will simplify all loops, regardless of whether anything end up being
// vectorized.
for (auto &L : *LI)
  Changed |= CFGChanged |=
      simplifyLoop(L, DT, LI, SE, AC, nullptr, false /* PreserveLCSSA */);

// Build up a worklist of inner-loops to vectorize. This is necessary as
// the act of vectorizing or partially unrolling a loop creates new loops
// and can invalidate iterators across the loops.
SmallVector<Loop *, 8> Worklist;

for (Loop *L : *LI)
  collectSupportedLoops(*L, LI, ORE, Worklist);

LoopsAnalyzed += Worklist.size();

// Now walk the identified inner loops.
while (!Worklist.empty()) {
  Loop *L = Worklist.pop_back_val();

  // For the inner loops we actually process, form LCSSA to simplify the
  // transform.
  Changed |= formLCSSARecursively(*L, *DT, LI, SE);

  Changed |= CFGChanged |= processLoop(L);
}

// Process each loop nest in the function.
return LoopVectorizeResult(Changed, CFGChanged);
10032}

10034PreservedAnalyses LoopVectorizePass::run(Function &F,
                                       FunctionAnalysisManager &AM) {
  auto &SE = AM.getResult<ScalarEvolutionAnalysis>(F);
  auto &LI = AM.getResult<LoopAnalysis>(F);
  auto &TTI = AM.getResult<TargetIRAnalysis>(F);
  auto &DT = AM.getResult<DominatorTreeAnalysis>(F);
  auto &BFI = AM.getResult<BlockFrequencyAnalysis>(F);
  auto &TLI = AM.getResult<TargetLibraryAnalysis>(F);
  auto &AA = AM.getResult<AAManager>(F);
  auto &AC = AM.getResult<AssumptionAnalysis>(F);
  auto &DB = AM.getResult<DemandedBitsAnalysis>(F);
  auto &ORE = AM.getResult<OptimizationRemarkEmitterAnalysis>(F);
  MemorySSA *MSSA = EnableMSSALoopDependency
                        ? &AM.getResult<MemorySSAAnalysis>(F).getMSSA()
                        : nullptr;

  auto &LAM = AM.getResult<LoopAnalysisManagerFunctionProxy>(F).getManager();
  std::function<const LoopAccessInfo &(Loop &)> GetLAA =
      [&](Loop &L) -> const LoopAccessInfo & {
    LoopStandardAnalysisResults AR = {AA,  AC,  DT,      LI,  SE,
                                      TLI, TTI, nullptr, MSSA};
    return LAM.getResult<LoopAccessAnalysis>(L, AR);
  };
  auto &MAMProxy = AM.getResult<ModuleAnalysisManagerFunctionProxy>(F);
  ProfileSummaryInfo *PSI =
      MAMProxy.getCachedResult<ProfileSummaryAnalysis>(*F.getParent());
  LoopVectorizeResult Result =
      runImpl(F, SE, LI, TTI, DT, BFI, &TLI, DB, AA, AC, GetLAA, ORE, PSI);
  if (!Result.MadeAnyChange)
    return PreservedAnalyses::all();
  PreservedAnalyses PA;

  // We currently do not preserve loopinfo/dominator analyses with outer loop
  // vectorization. Until this is addressed, mark these analyses as preserved
  // only for non-VPlan-native path.
  // TODO: Preserve Loop and Dominator analyses for VPlan-native path.
  if (!EnableVPlanNativePath) {
    PA.preserve<LoopAnalysis>();
    PA.preserve<DominatorTreeAnalysis>();
  }
  PA.preserve<BasicAA>();
  PA.preserve<GlobalsAA>();
  if (!Result.MadeCFGChange)
    PA.preserveSet<CFGAnalyses>();
  return PA;
10079}