LLVM  4.0.0
LoopAccessAnalysis.h
Go to the documentation of this file.
1 //===- llvm/Analysis/LoopAccessAnalysis.h -----------------------*- C++ -*-===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file defines the interface for the loop memory dependence framework that
11 // was originally developed for the Loop Vectorizer.
12 //
13 //===----------------------------------------------------------------------===//
14 
15 #ifndef LLVM_ANALYSIS_LOOPACCESSANALYSIS_H
16 #define LLVM_ANALYSIS_LOOPACCESSANALYSIS_H
17 
19 #include "llvm/ADT/Optional.h"
20 #include "llvm/ADT/SetVector.h"
25 #include "llvm/IR/DiagnosticInfo.h"
26 #include "llvm/IR/ValueHandle.h"
27 #include "llvm/Pass.h"
29 
30 namespace llvm {
31 
32 class Value;
33 class DataLayout;
34 class ScalarEvolution;
35 class Loop;
36 class SCEV;
37 class SCEVUnionPredicate;
38 class LoopAccessInfo;
39 class OptimizationRemarkEmitter;
40 
41 /// Optimization analysis message produced during vectorization. Messages inform
42 /// the user why vectorization did not occur.
44  std::string Message;
45  const Instruction *Instr;
46 
47 protected:
48  LoopAccessReport(const Twine &Message, const Instruction *I)
49  : Message(Message.str()), Instr(I) {}
50 
51 public:
52  LoopAccessReport(const Instruction *I = nullptr) : Instr(I) {}
53 
54  template <typename A> LoopAccessReport &operator<<(const A &Value) {
55  raw_string_ostream Out(Message);
56  Out << Value;
57  return *this;
58  }
59 
60  const Instruction *getInstr() const { return Instr; }
61 
62  std::string &str() { return Message; }
63  const std::string &str() const { return Message; }
64  operator Twine() { return Message; }
65 
66  /// \brief Emit an analysis note for \p PassName with the debug location from
67  /// the instruction in \p Message if available. Otherwise use the location of
68  /// \p TheLoop.
69  static void emitAnalysis(const LoopAccessReport &Message, const Loop *TheLoop,
70  const char *PassName,
72 };
73 
74 /// \brief Collection of parameters shared beetween the Loop Vectorizer and the
75 /// Loop Access Analysis.
77  /// \brief Maximum SIMD width.
78  static const unsigned MaxVectorWidth;
79 
80  /// \brief VF as overridden by the user.
81  static unsigned VectorizationFactor;
82  /// \brief Interleave factor as overridden by the user.
83  static unsigned VectorizationInterleave;
84  /// \brief True if force-vector-interleave was specified by the user.
85  static bool isInterleaveForced();
86 
87  /// \\brief When performing memory disambiguation checks at runtime do not
88  /// make more than this number of comparisons.
89  static unsigned RuntimeMemoryCheckThreshold;
90 };
91 
92 /// \brief Checks memory dependences among accesses to the same underlying
93 /// object to determine whether there vectorization is legal or not (and at
94 /// which vectorization factor).
95 ///
96 /// Note: This class will compute a conservative dependence for access to
97 /// different underlying pointers. Clients, such as the loop vectorizer, will
98 /// sometimes deal these potential dependencies by emitting runtime checks.
99 ///
100 /// We use the ScalarEvolution framework to symbolically evalutate access
101 /// functions pairs. Since we currently don't restructure the loop we can rely
102 /// on the program order of memory accesses to determine their safety.
103 /// At the moment we will only deem accesses as safe for:
104 /// * A negative constant distance assuming program order.
105 ///
106 /// Safe: tmp = a[i + 1]; OR a[i + 1] = x;
107 /// a[i] = tmp; y = a[i];
108 ///
109 /// The latter case is safe because later checks guarantuee that there can't
110 /// be a cycle through a phi node (that is, we check that "x" and "y" is not
111 /// the same variable: a header phi can only be an induction or a reduction, a
112 /// reduction can't have a memory sink, an induction can't have a memory
113 /// source). This is important and must not be violated (or we have to
114 /// resort to checking for cycles through memory).
115 ///
116 /// * A positive constant distance assuming program order that is bigger
117 /// than the biggest memory access.
118 ///
119 /// tmp = a[i] OR b[i] = x
120 /// a[i+2] = tmp y = b[i+2];
121 ///
122 /// Safe distance: 2 x sizeof(a[0]), and 2 x sizeof(b[0]), respectively.
123 ///
124 /// * Zero distances and all accesses have the same size.
125 ///
127 public:
130  /// \brief Set of potential dependent memory accesses.
132 
133  /// \brief Dependece between memory access instructions.
134  struct Dependence {
135  /// \brief The type of the dependence.
136  enum DepType {
137  // No dependence.
139  // We couldn't determine the direction or the distance.
141  // Lexically forward.
142  //
143  // FIXME: If we only have loop-independent forward dependences (e.g. a
144  // read and write of A[i]), LAA will locally deem the dependence "safe"
145  // without querying the MemoryDepChecker. Therefore we can miss
146  // enumerating loop-independent forward dependences in
147  // getDependences. Note that as soon as there are different
148  // indices used to access the same array, the MemoryDepChecker *is*
149  // queried and the dependence list is complete.
151  // Forward, but if vectorized, is likely to prevent store-to-load
152  // forwarding.
154  // Lexically backward.
156  // Backward, but the distance allows a vectorization factor of
157  // MaxSafeDepDistBytes.
159  // Same, but may prevent store-to-load forwarding.
161  };
162 
163  /// \brief String version of the types.
164  static const char *DepName[];
165 
166  /// \brief Index of the source of the dependence in the InstMap vector.
167  unsigned Source;
168  /// \brief Index of the destination of the dependence in the InstMap vector.
169  unsigned Destination;
170  /// \brief The type of the dependence.
172 
173  Dependence(unsigned Source, unsigned Destination, DepType Type)
174  : Source(Source), Destination(Destination), Type(Type) {}
175 
176  /// \brief Return the source instruction of the dependence.
177  Instruction *getSource(const LoopAccessInfo &LAI) const;
178  /// \brief Return the destination instruction of the dependence.
179  Instruction *getDestination(const LoopAccessInfo &LAI) const;
180 
181  /// \brief Dependence types that don't prevent vectorization.
182  static bool isSafeForVectorization(DepType Type);
183 
184  /// \brief Lexically forward dependence.
185  bool isForward() const;
186  /// \brief Lexically backward dependence.
187  bool isBackward() const;
188 
189  /// \brief May be a lexically backward dependence type (includes Unknown).
190  bool isPossiblyBackward() const;
191 
192  /// \brief Print the dependence. \p Instr is used to map the instruction
193  /// indices to instructions.
194  void print(raw_ostream &OS, unsigned Depth,
195  const SmallVectorImpl<Instruction *> &Instrs) const;
196  };
197 
199  : PSE(PSE), InnermostLoop(L), AccessIdx(0),
200  ShouldRetryWithRuntimeCheck(false), SafeForVectorization(true),
201  RecordDependences(true) {}
202 
203  /// \brief Register the location (instructions are given increasing numbers)
204  /// of a write access.
205  void addAccess(StoreInst *SI) {
206  Value *Ptr = SI->getPointerOperand();
207  Accesses[MemAccessInfo(Ptr, true)].push_back(AccessIdx);
208  InstMap.push_back(SI);
209  ++AccessIdx;
210  }
211 
212  /// \brief Register the location (instructions are given increasing numbers)
213  /// of a write access.
214  void addAccess(LoadInst *LI) {
215  Value *Ptr = LI->getPointerOperand();
216  Accesses[MemAccessInfo(Ptr, false)].push_back(AccessIdx);
217  InstMap.push_back(LI);
218  ++AccessIdx;
219  }
220 
221  /// \brief Check whether the dependencies between the accesses are safe.
222  ///
223  /// Only checks sets with elements in \p CheckDeps.
224  bool areDepsSafe(DepCandidates &AccessSets, MemAccessInfoSet &CheckDeps,
225  const ValueToValueMap &Strides);
226 
227  /// \brief No memory dependence was encountered that would inhibit
228  /// vectorization.
229  bool isSafeForVectorization() const { return SafeForVectorization; }
230 
231  /// \brief The maximum number of bytes of a vector register we can vectorize
232  /// the accesses safely with.
233  uint64_t getMaxSafeDepDistBytes() { return MaxSafeDepDistBytes; }
234 
235  /// \brief In same cases when the dependency check fails we can still
236  /// vectorize the loop with a dynamic array access check.
237  bool shouldRetryWithRuntimeCheck() { return ShouldRetryWithRuntimeCheck; }
238 
239  /// \brief Returns the memory dependences. If null is returned we exceeded
240  /// the MaxDependences threshold and this information is not
241  /// available.
243  return RecordDependences ? &Dependences : nullptr;
244  }
245 
246  void clearDependences() { Dependences.clear(); }
247 
248  /// \brief The vector of memory access instructions. The indices are used as
249  /// instruction identifiers in the Dependence class.
251  return InstMap;
252  }
253 
254  /// \brief Generate a mapping between the memory instructions and their
255  /// indices according to program order.
258 
259  for (unsigned I = 0; I < InstMap.size(); ++I)
260  OrderMap[InstMap[I]] = I;
261 
262  return OrderMap;
263  }
264 
265  /// \brief Find the set of instructions that read or write via \p Ptr.
267  bool isWrite) const;
268 
269 private:
270  /// A wrapper around ScalarEvolution, used to add runtime SCEV checks, and
271  /// applies dynamic knowledge to simplify SCEV expressions and convert them
272  /// to a more usable form. We need this in case assumptions about SCEV
273  /// expressions need to be made in order to avoid unknown dependences. For
274  /// example we might assume a unit stride for a pointer in order to prove
275  /// that a memory access is strided and doesn't wrap.
277  const Loop *InnermostLoop;
278 
279  /// \brief Maps access locations (ptr, read/write) to program order.
281 
282  /// \brief Memory access instructions in program order.
284 
285  /// \brief The program order index to be used for the next instruction.
286  unsigned AccessIdx;
287 
288  // We can access this many bytes in parallel safely.
289  uint64_t MaxSafeDepDistBytes;
290 
291  /// \brief If we see a non-constant dependence distance we can still try to
292  /// vectorize this loop with runtime checks.
293  bool ShouldRetryWithRuntimeCheck;
294 
295  /// \brief No memory dependence was encountered that would inhibit
296  /// vectorization.
297  bool SafeForVectorization;
298 
299  //// \brief True if Dependences reflects the dependences in the
300  //// loop. If false we exceeded MaxDependences and
301  //// Dependences is invalid.
302  bool RecordDependences;
303 
304  /// \brief Memory dependences collected during the analysis. Only valid if
305  /// RecordDependences is true.
306  SmallVector<Dependence, 8> Dependences;
307 
308  /// \brief Check whether there is a plausible dependence between the two
309  /// accesses.
310  ///
311  /// Access \p A must happen before \p B in program order. The two indices
312  /// identify the index into the program order map.
313  ///
314  /// This function checks whether there is a plausible dependence (or the
315  /// absence of such can't be proved) between the two accesses. If there is a
316  /// plausible dependence but the dependence distance is bigger than one
317  /// element access it records this distance in \p MaxSafeDepDistBytes (if this
318  /// distance is smaller than any other distance encountered so far).
319  /// Otherwise, this function returns true signaling a possible dependence.
320  Dependence::DepType isDependent(const MemAccessInfo &A, unsigned AIdx,
321  const MemAccessInfo &B, unsigned BIdx,
322  const ValueToValueMap &Strides);
323 
324  /// \brief Check whether the data dependence could prevent store-load
325  /// forwarding.
326  ///
327  /// \return false if we shouldn't vectorize at all or avoid larger
328  /// vectorization factors by limiting MaxSafeDepDistBytes.
329  bool couldPreventStoreLoadForward(uint64_t Distance, uint64_t TypeByteSize);
330 };
331 
332 /// \brief Holds information about the memory runtime legality checks to verify
333 /// that a group of pointers do not overlap.
335 public:
336  struct PointerInfo {
337  /// Holds the pointer value that we need to check.
339  /// Holds the smallest byte address accessed by the pointer throughout all
340  /// iterations of the loop.
341  const SCEV *Start;
342  /// Holds the largest byte address accessed by the pointer throughout all
343  /// iterations of the loop, plus 1.
344  const SCEV *End;
345  /// Holds the information if this pointer is used for writing to memory.
347  /// Holds the id of the set of pointers that could be dependent because of a
348  /// shared underlying object.
349  unsigned DependencySetId;
350  /// Holds the id of the disjoint alias set to which this pointer belongs.
351  unsigned AliasSetId;
352  /// SCEV for the access.
353  const SCEV *Expr;
354 
356  bool IsWritePtr, unsigned DependencySetId, unsigned AliasSetId,
357  const SCEV *Expr)
358  : PointerValue(PointerValue), Start(Start), End(End),
359  IsWritePtr(IsWritePtr), DependencySetId(DependencySetId),
360  AliasSetId(AliasSetId), Expr(Expr) {}
361  };
362 
364 
365  /// Reset the state of the pointer runtime information.
366  void reset() {
367  Need = false;
368  Pointers.clear();
369  Checks.clear();
370  }
371 
372  /// Insert a pointer and calculate the start and end SCEVs.
373  /// We need \p PSE in order to compute the SCEV expression of the pointer
374  /// according to the assumptions that we've made during the analysis.
375  /// The method might also version the pointer stride according to \p Strides,
376  /// and add new predicates to \p PSE.
377  void insert(Loop *Lp, Value *Ptr, bool WritePtr, unsigned DepSetId,
378  unsigned ASId, const ValueToValueMap &Strides,
380 
381  /// \brief No run-time memory checking is necessary.
382  bool empty() const { return Pointers.empty(); }
383 
384  /// A grouping of pointers. A single memcheck is required between
385  /// two groups.
387  /// \brief Create a new pointer checking group containing a single
388  /// pointer, with index \p Index in RtCheck.
390  : RtCheck(RtCheck), High(RtCheck.Pointers[Index].End),
391  Low(RtCheck.Pointers[Index].Start) {
392  Members.push_back(Index);
393  }
394 
395  /// \brief Tries to add the pointer recorded in RtCheck at index
396  /// \p Index to this pointer checking group. We can only add a pointer
397  /// to a checking group if we will still be able to get
398  /// the upper and lower bounds of the check. Returns true in case
399  /// of success, false otherwise.
400  bool addPointer(unsigned Index);
401 
402  /// Constitutes the context of this pointer checking group. For each
403  /// pointer that is a member of this group we will retain the index
404  /// at which it appears in RtCheck.
406  /// The SCEV expression which represents the upper bound of all the
407  /// pointers in this group.
408  const SCEV *High;
409  /// The SCEV expression which represents the lower bound of all the
410  /// pointers in this group.
411  const SCEV *Low;
412  /// Indices of all the pointers that constitute this grouping.
414  };
415 
416  /// \brief A memcheck which made up of a pair of grouped pointers.
417  ///
418  /// These *have* to be const for now, since checks are generated from
419  /// CheckingPtrGroups in LAI::addRuntimeChecks which is a const member
420  /// function. FIXME: once check-generation is moved inside this class (after
421  /// the PtrPartition hack is removed), we could drop const.
422  typedef std::pair<const CheckingPtrGroup *, const CheckingPtrGroup *>
424 
425  /// \brief Generate the checks and store it. This also performs the grouping
426  /// of pointers to reduce the number of memchecks necessary.
428  bool UseDependencies);
429 
430  /// \brief Returns the checks that generateChecks created.
431  const SmallVector<PointerCheck, 4> &getChecks() const { return Checks; }
432 
433  /// \brief Decide if we need to add a check between two groups of pointers,
434  /// according to needsChecking.
435  bool needsChecking(const CheckingPtrGroup &M,
436  const CheckingPtrGroup &N) const;
437 
438  /// \brief Returns the number of run-time checks required according to
439  /// needsChecking.
440  unsigned getNumberOfChecks() const { return Checks.size(); }
441 
442  /// \brief Print the list run-time memory checks necessary.
443  void print(raw_ostream &OS, unsigned Depth = 0) const;
444 
445  /// Print \p Checks.
446  void printChecks(raw_ostream &OS, const SmallVectorImpl<PointerCheck> &Checks,
447  unsigned Depth = 0) const;
448 
449  /// This flag indicates if we need to add the runtime check.
450  bool Need;
451 
452  /// Information about the pointers that may require checking.
454 
455  /// Holds a partitioning of pointers into "check groups".
457 
458  /// \brief Check if pointers are in the same partition
459  ///
460  /// \p PtrToPartition contains the partition number for pointers (-1 if the
461  /// pointer belongs to multiple partitions).
462  static bool
463  arePointersInSamePartition(const SmallVectorImpl<int> &PtrToPartition,
464  unsigned PtrIdx1, unsigned PtrIdx2);
465 
466  /// \brief Decide whether we need to issue a run-time check for pointer at
467  /// index \p I and \p J to prove their independence.
468  bool needsChecking(unsigned I, unsigned J) const;
469 
470  /// \brief Return PointerInfo for pointer at index \p PtrIdx.
471  const PointerInfo &getPointerInfo(unsigned PtrIdx) const {
472  return Pointers[PtrIdx];
473  }
474 
475 private:
476  /// \brief Groups pointers such that a single memcheck is required
477  /// between two different groups. This will clear the CheckingGroups vector
478  /// and re-compute it. We will only group dependecies if \p UseDependencies
479  /// is true, otherwise we will create a separate group for each pointer.
480  void groupChecks(MemoryDepChecker::DepCandidates &DepCands,
481  bool UseDependencies);
482 
483  /// Generate the checks and return them.
485  generateChecks() const;
486 
487  /// Holds a pointer to the ScalarEvolution analysis.
488  ScalarEvolution *SE;
489 
490  /// \brief Set of run-time checks required to establish independence of
491  /// otherwise may-aliasing pointers in the loop.
493 };
494 
495 /// \brief Drive the analysis of memory accesses in the loop
496 ///
497 /// This class is responsible for analyzing the memory accesses of a loop. It
498 /// collects the accesses and then its main helper the AccessAnalysis class
499 /// finds and categorizes the dependences in buildDependenceSets.
500 ///
501 /// For memory dependences that can be analyzed at compile time, it determines
502 /// whether the dependence is part of cycle inhibiting vectorization. This work
503 /// is delegated to the MemoryDepChecker class.
504 ///
505 /// For memory dependences that cannot be determined at compile time, it
506 /// generates run-time checks to prove independence. This is done by
507 /// AccessAnalysis::canCheckPtrAtRT and the checks are maintained by the
508 /// RuntimePointerCheck class.
509 ///
510 /// If pointers can wrap or can't be expressed as affine AddRec expressions by
511 /// ScalarEvolution, we will generate run-time checks by emitting a
512 /// SCEVUnionPredicate.
513 ///
514 /// Checks for both memory dependences and the SCEV predicates contained in the
515 /// PSE must be emitted in order for the results of this analysis to be valid.
517 public:
519  AliasAnalysis *AA, DominatorTree *DT, LoopInfo *LI);
520 
521  /// Return true we can analyze the memory accesses in the loop and there are
522  /// no memory dependence cycles.
523  bool canVectorizeMemory() const { return CanVecMem; }
524 
526  return PtrRtChecking.get();
527  }
528 
529  /// \brief Number of memchecks required to prove independence of otherwise
530  /// may-alias pointers.
531  unsigned getNumRuntimePointerChecks() const {
532  return PtrRtChecking->getNumberOfChecks();
533  }
534 
535  /// Return true if the block BB needs to be predicated in order for the loop
536  /// to be vectorized.
537  static bool blockNeedsPredication(BasicBlock *BB, Loop *TheLoop,
538  DominatorTree *DT);
539 
540  /// Returns true if the value V is uniform within the loop.
541  bool isUniform(Value *V) const;
542 
543  uint64_t getMaxSafeDepDistBytes() const { return MaxSafeDepDistBytes; }
544  unsigned getNumStores() const { return NumStores; }
545  unsigned getNumLoads() const { return NumLoads;}
546 
547  /// \brief Add code that checks at runtime if the accessed arrays overlap.
548  ///
549  /// Returns a pair of instructions where the first element is the first
550  /// instruction generated in possibly a sequence of instructions and the
551  /// second value is the final comparator value or NULL if no check is needed.
552  std::pair<Instruction *, Instruction *>
553  addRuntimeChecks(Instruction *Loc) const;
554 
555  /// \brief Generete the instructions for the checks in \p PointerChecks.
556  ///
557  /// Returns a pair of instructions where the first element is the first
558  /// instruction generated in possibly a sequence of instructions and the
559  /// second value is the final comparator value or NULL if no check is needed.
560  std::pair<Instruction *, Instruction *>
563  &PointerChecks) const;
564 
565  /// \brief The diagnostics report generated for the analysis. E.g. why we
566  /// couldn't analyze the loop.
567  const OptimizationRemarkAnalysis *getReport() const { return Report.get(); }
568 
569  /// \brief the Memory Dependence Checker which can determine the
570  /// loop-independent and loop-carried dependences between memory accesses.
571  const MemoryDepChecker &getDepChecker() const { return *DepChecker; }
572 
573  /// \brief Return the list of instructions that use \p Ptr to read or write
574  /// memory.
576  bool isWrite) const {
577  return DepChecker->getInstructionsForAccess(Ptr, isWrite);
578  }
579 
580  /// \brief If an access has a symbolic strides, this maps the pointer value to
581  /// the stride symbol.
582  const ValueToValueMap &getSymbolicStrides() const { return SymbolicStrides; }
583 
584  /// \brief Pointer has a symbolic stride.
585  bool hasStride(Value *V) const { return StrideSet.count(V); }
586 
587  /// \brief Print the information about the memory accesses in the loop.
588  void print(raw_ostream &OS, unsigned Depth = 0) const;
589 
590  /// \brief Checks existence of store to invariant address inside loop.
591  /// If the loop has any store to invariant address, then it returns true,
592  /// else returns false.
594  return StoreToLoopInvariantAddress;
595  }
596 
597  /// Used to add runtime SCEV checks. Simplifies SCEV expressions and converts
598  /// them to a more usable form. All SCEV expressions during the analysis
599  /// should be re-written (and therefore simplified) according to PSE.
600  /// A user of LoopAccessAnalysis will need to emit the runtime checks
601  /// associated with this predicate.
602  const PredicatedScalarEvolution &getPSE() const { return *PSE; }
603 
604 private:
605  /// \brief Analyze the loop.
606  void analyzeLoop(AliasAnalysis *AA, LoopInfo *LI,
607  const TargetLibraryInfo *TLI, DominatorTree *DT);
608 
609  /// \brief Check if the structure of the loop allows it to be analyzed by this
610  /// pass.
611  bool canAnalyzeLoop();
612 
613  /// \brief Save the analysis remark.
614  ///
615  /// LAA does not directly emits the remarks. Instead it stores it which the
616  /// client can retrieve and presents as its own analysis
617  /// (e.g. -Rpass-analysis=loop-vectorize).
618  OptimizationRemarkAnalysis &recordAnalysis(StringRef RemarkName,
619  Instruction *Instr = nullptr);
620 
621  /// \brief Collect memory access with loop invariant strides.
622  ///
623  /// Looks for accesses like "a[i * StrideA]" where "StrideA" is loop
624  /// invariant.
625  void collectStridedAccess(Value *LoadOrStoreInst);
626 
627  std::unique_ptr<PredicatedScalarEvolution> PSE;
628 
629  /// We need to check that all of the pointers in this list are disjoint
630  /// at runtime. Using std::unique_ptr to make using move ctor simpler.
631  std::unique_ptr<RuntimePointerChecking> PtrRtChecking;
632 
633  /// \brief the Memory Dependence Checker which can determine the
634  /// loop-independent and loop-carried dependences between memory accesses.
635  std::unique_ptr<MemoryDepChecker> DepChecker;
636 
637  Loop *TheLoop;
638 
639  unsigned NumLoads;
640  unsigned NumStores;
641 
642  uint64_t MaxSafeDepDistBytes;
643 
644  /// \brief Cache the result of analyzeLoop.
645  bool CanVecMem;
646 
647  /// \brief Indicator for storing to uniform addresses.
648  /// If a loop has write to a loop invariant address then it should be true.
649  bool StoreToLoopInvariantAddress;
650 
651  /// \brief The diagnostics report generated for the analysis. E.g. why we
652  /// couldn't analyze the loop.
653  std::unique_ptr<OptimizationRemarkAnalysis> Report;
654 
655  /// \brief If an access has a symbolic strides, this maps the pointer value to
656  /// the stride symbol.
657  ValueToValueMap SymbolicStrides;
658 
659  /// \brief Set of symbolic strides values.
660  SmallPtrSet<Value *, 8> StrideSet;
661 };
662 
663 Value *stripIntegerCast(Value *V);
664 
665 /// \brief Return the SCEV corresponding to a pointer with the symbolic stride
666 /// replaced with constant one, assuming the SCEV predicate associated with
667 /// \p PSE is true.
668 ///
669 /// If necessary this method will version the stride of the pointer according
670 /// to \p PtrToStride and therefore add further predicates to \p PSE.
671 ///
672 /// If \p OrigPtr is not null, use it to look up the stride value instead of \p
673 /// Ptr. \p PtrToStride provides the mapping between the pointer value and its
674 /// stride as collected by LoopVectorizationLegality::collectStridedAccess.
675 const SCEV *replaceSymbolicStrideSCEV(PredicatedScalarEvolution &PSE,
676  const ValueToValueMap &PtrToStride,
677  Value *Ptr, Value *OrigPtr = nullptr);
678 
679 /// \brief If the pointer has a constant stride return it in units of its
680 /// element size. Otherwise return zero.
681 ///
682 /// Ensure that it does not wrap in the address space, assuming the predicate
683 /// associated with \p PSE is true.
684 ///
685 /// If necessary this method will version the stride of the pointer according
686 /// to \p PtrToStride and therefore add further predicates to \p PSE.
687 /// The \p Assume parameter indicates if we are allowed to make additional
688 /// run-time assumptions.
689 int64_t getPtrStride(PredicatedScalarEvolution &PSE, Value *Ptr, const Loop *Lp,
690  const ValueToValueMap &StridesMap = ValueToValueMap(),
691  bool Assume = false, bool ShouldCheckWrap = true);
692 
693 /// \brief Returns true if the memory operations \p A and \p B are consecutive.
694 /// This is a simple API that does not depend on the analysis pass.
695 bool isConsecutiveAccess(Value *A, Value *B, const DataLayout &DL,
696  ScalarEvolution &SE, bool CheckType = true);
697 
698 /// \brief This analysis provides dependence information for the memory accesses
699 /// of a loop.
700 ///
701 /// It runs the analysis for a loop on demand. This can be initiated by
702 /// querying the loop access info via LAA::getInfo. getInfo return a
703 /// LoopAccessInfo object. See this class for the specifics of what information
704 /// is provided.
706 public:
707  static char ID;
708 
711  }
712 
713  bool runOnFunction(Function &F) override;
714 
715  void getAnalysisUsage(AnalysisUsage &AU) const override;
716 
717  /// \brief Query the result of the loop access information for the loop \p L.
718  ///
719  /// If there is no cached result available run the analysis.
720  const LoopAccessInfo &getInfo(Loop *L);
721 
722  void releaseMemory() override {
723  // Invalidate the cache when the pass is freed.
724  LoopAccessInfoMap.clear();
725  }
726 
727  /// \brief Print the result of the analysis when invoked with -analyze.
728  void print(raw_ostream &OS, const Module *M = nullptr) const override;
729 
730 private:
731  /// \brief The cache.
733 
734  // The used analysis passes.
735  ScalarEvolution *SE;
736  const TargetLibraryInfo *TLI;
737  AliasAnalysis *AA;
738  DominatorTree *DT;
739  LoopInfo *LI;
740 };
741 
742 /// \brief This analysis provides dependence information for the memory
743 /// accesses of a loop.
744 ///
745 /// It runs the analysis for a loop on demand. This can be initiated by
746 /// querying the loop access info via AM.getResult<LoopAccessAnalysis>.
747 /// getResult return a LoopAccessInfo object. See this class for the
748 /// specifics of what information is provided.
750  : public AnalysisInfoMixin<LoopAccessAnalysis> {
752  static AnalysisKey Key;
753 
754 public:
756 
758 };
759 
761  const LoopAccessInfo &LAI) const {
763 }
764 
766  const LoopAccessInfo &LAI) const {
767  return LAI.getDepChecker().getMemoryInstructions()[Destination];
768 }
769 
770 } // End llvm namespace
771 
772 #endif
MachineLoop * L
bool isPossiblyBackward() const
May be a lexically backward dependence type (includes Unknown).
void print(raw_ostream &OS, unsigned Depth, const SmallVectorImpl< Instruction * > &Instrs) const
Print the dependence.
void print(raw_ostream &OS, unsigned Depth=0) const
Print the information about the memory accesses in the loop.
static unsigned RuntimeMemoryCheckThreshold
\brief When performing memory disambiguation checks at runtime do not make more than this number of c...
TrackingVH< Value > PointerValue
Holds the pointer value that we need to check.
void push_back(const T &Elt)
Definition: SmallVector.h:211
SmallVector< CheckingPtrGroup, 2 > CheckingGroups
Holds a partitioning of pointers into "check groups".
void print(raw_ostream &OS, unsigned Depth=0) const
Print the list run-time memory checks necessary.
uint64_t getMaxSafeDepDistBytes() const
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
bool hasStride(Value *V) const
Pointer has a symbolic stride.
void initializeLoopAccessLegacyAnalysisPass(PassRegistry &)
unsigned getNumLoads() const
const SmallVectorImpl< Instruction * > & getMemoryInstructions() const
The vector of memory access instructions.
A Module instance is used to store all the information related to an LLVM module. ...
Definition: Module.h:52
DepType Type
The type of the dependence.
bool isSafeForVectorization() const
No memory dependence was encountered that would inhibit vectorization.
The main scalar evolution driver.
int64_t getPtrStride(PredicatedScalarEvolution &PSE, Value *Ptr, const Loop *Lp, const ValueToValueMap &StridesMap=ValueToValueMap(), bool Assume=false, bool ShouldCheckWrap=true)
If the pointer has a constant stride return it in units of its element size.
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - This function should be overriden by passes that need analysis information to do t...
size_type count(PtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
Definition: SmallPtrSet.h:380
bool IsWritePtr
Holds the information if this pointer is used for writing to memory.
std::pair< Instruction *, Instruction * > addRuntimeChecks(Instruction *Loc) const
Add code that checks at runtime if the accessed arrays overlap.
void reset()
Reset the state of the pointer runtime information.
The adaptor from a function pass to a loop pass computes these analyses and makes them available to t...
An instruction for reading from memory.
Definition: Instructions.h:164
Checks memory dependences among accesses to the same underlying object to determine whether there vec...
static unsigned VectorizationFactor
VF as overridden by the user.
DenseMap< const Value *, Value * > ValueToValueMap
void printChecks(raw_ostream &OS, const SmallVectorImpl< PointerCheck > &Checks, unsigned Depth=0) const
Print Checks.
void insert(Loop *Lp, Value *Ptr, bool WritePtr, unsigned DepSetId, unsigned ASId, const ValueToValueMap &Strides, PredicatedScalarEvolution &PSE)
Insert a pointer and calculate the start and end SCEVs.
bool empty() const
No run-time memory checking is necessary.
uint64_t getMaxSafeDepDistBytes()
The maximum number of bytes of a vector register we can vectorize the accesses safely with...
static const unsigned MaxVectorWidth
Maximum SIMD width.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:81
LoopAccessReport(const Twine &Message, const Instruction *I)
const std::string & str() const
static bool arePointersInSamePartition(const SmallVectorImpl< int > &PtrToPartition, unsigned PtrIdx1, unsigned PtrIdx2)
Check if pointers are in the same partition.
Diagnostic information for optimization analysis remarks.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: APFloat.h:32
const PredicatedScalarEvolution & getPSE() const
Used to add runtime SCEV checks.
const Instruction * getInstr() const
static const char * DepName[]
String version of the types.
#define F(x, y, z)
Definition: MD5.cpp:51
const RuntimePointerChecking * getRuntimePointerChecking() const
unsigned Source
Index of the source of the dependence in the InstMap vector.
Function Alias Analysis false
RuntimePointerChecking(ScalarEvolution *SE)
This header provides classes for managing per-loop analyses.
static GCRegistry::Add< OcamlGC > B("ocaml","ocaml 3.10-compatible GC")
An instruction for storing to memory.
Definition: Instructions.h:300
Instruction * getDestination(const LoopAccessInfo &LAI) const
Return the destination instruction of the dependence.
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree...
Definition: Dominators.h:96
Optimization analysis message produced during vectorization.
bool isConsecutiveAccess(Value *A, Value *B, const DataLayout &DL, ScalarEvolution &SE, bool CheckType=true)
Returns true if the memory operations A and B are consecutive.
Instruction * getSource(const LoopAccessInfo &LAI) const
Return the source instruction of the dependence.
LLVM Basic Block Representation.
Definition: BasicBlock.h:51
PointerIntPair - This class implements a pair of a pointer and small integer.
The instances of the Type class are immutable: once they are created, they are never changed...
Definition: Type.h:45
Value handle that tracks a Value across RAUW.
Definition: ValueHandle.h:275
PointerInfo(Value *PointerValue, const SCEV *Start, const SCEV *End, bool IsWritePtr, unsigned DependencySetId, unsigned AliasSetId, const SCEV *Expr)
This analysis provides dependence information for the memory accesses of a loop.
SmallVector< Instruction *, 4 > getInstructionsForAccess(Value *Ptr, bool isWrite) const
Return the list of instructions that use Ptr to read or write memory.
const SCEV * End
Holds the largest byte address accessed by the pointer throughout all iterations of the loop...
const PointerInfo & getPointerInfo(unsigned PtrIdx) const
Return PointerInfo for pointer at index PtrIdx.
bool addPointer(unsigned Index)
Tries to add the pointer recorded in RtCheck at index Index to this pointer checking group...
Result run(Loop &L, LoopAnalysisManager &AM, LoopStandardAnalysisResults &AR)
A CRTP mix-in that provides informational APIs needed for analysis passes.
Definition: PassManager.h:328
SmallVector< Instruction *, 4 > getInstructionsForAccess(Value *Ptr, bool isWrite) const
Find the set of instructions that read or write via Ptr.
EquivalenceClasses - This represents a collection of equivalence classes and supports three efficient...
Represent the analysis usage information of a pass.
unsigned getNumStores() const
void addAccess(StoreInst *SI)
Register the location (instructions are given increasing numbers) of a write access.
const SmallVector< PointerCheck, 4 > & getChecks() const
Returns the checks that generateChecks created.
static const unsigned End
FunctionPass class - This class is used to implement most global optimizations.
Definition: Pass.h:298
const SCEV * replaceSymbolicStrideSCEV(PredicatedScalarEvolution &PSE, const ValueToValueMap &PtrToStride, Value *Ptr, Value *OrigPtr=nullptr)
Return the SCEV corresponding to a pointer with the symbolic stride replaced with constant one...
Value * getPointerOperand()
Definition: Instructions.h:270
std::pair< const CheckingPtrGroup *, const CheckingPtrGroup * > PointerCheck
A memcheck which made up of a pair of grouped pointers.
const OptimizationRemarkAnalysis * getReport() const
The diagnostics report generated for the analysis.
Value * stripIntegerCast(Value *V)
LoopAccessReport & operator<<(const A &Value)
unsigned AliasSetId
Holds the id of the disjoint alias set to which this pointer belongs.
DepType
The type of the dependence.
SmallVector< unsigned, 2 > Members
Indices of all the pointers that constitute this grouping.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements...
Definition: SmallPtrSet.h:425
const ValueToValueMap & getSymbolicStrides() const
If an access has a symbolic strides, this maps the pointer value to the stride symbol.
bool isBackward() const
Lexically backward dependence.
LoopAccessReport(const Instruction *I=nullptr)
bool runOnFunction(Function &F) override
runOnFunction - Virtual method overriden by subclasses to do the per-function processing of the pass...
bool isUniform(Value *V) const
Returns true if the value V is uniform within the loop.
Dependence(unsigned Source, unsigned Destination, DepType Type)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:843
Provides information about what library functions are available for the current target.
An interface layer with SCEV used to manage how we see SCEV expressions for values in the context of ...
bool isForward() const
Lexically forward dependence.
const SCEV * Low
The SCEV expression which represents the lower bound of all the pointers in this group.
const MemoryDepChecker & getDepChecker() const
the Memory Dependence Checker which can determine the loop-independent and loop-carried dependences b...
Drive the analysis of memory accesses in the loop.
const SmallVectorImpl< Dependence > * getDependences() const
Returns the memory dependences.
const LoopAccessInfo & getInfo(Loop *L)
Query the result of the loop access information for the loop L.
void generateChecks(MemoryDepChecker::DepCandidates &DepCands, bool UseDependencies)
Generate the checks and store it.
Holds information about the memory runtime legality checks to verify that a group of pointers do not ...
bool hasStoreToLoopInvariantAddress() const
Checks existence of store to invariant address inside loop.
static bool isSafeForVectorization(DepType Type)
Dependence types that don't prevent vectorization.
unsigned getNumberOfChecks() const
Returns the number of run-time checks required according to needsChecking.
void releaseMemory() override
releaseMemory() - This member can be implemented by a pass if it wants to be able to release its memo...
unsigned getNumRuntimePointerChecks() const
Number of memchecks required to prove independence of otherwise may-alias pointers.
Basic Alias true
void addAccess(LoadInst *LI)
Register the location (instructions are given increasing numbers) of a write access.
This analysis provides dependence information for the memory accesses of a loop.
Dependece between memory access instructions.
const SCEV * Start
Holds the smallest byte address accessed by the pointer throughout all iterations of the loop...
This class represents an analyzed expression in the program.
const SCEV * High
The SCEV expression which represents the upper bound of all the pointers in this group.
SmallPtrSet< MemAccessInfo, 8 > MemAccessInfoSet
Represents a single loop in the control flow graph.
Definition: LoopInfo.h:368
static void emitAnalysis(const LoopAccessReport &Message, const Loop *TheLoop, const char *PassName, OptimizationRemarkEmitter &ORE)
Emit an analysis note for PassName with the debug location from the instruction in Message if availab...
#define I(x, y, z)
Definition: MD5.cpp:54
#define N
LLVM_ATTRIBUTE_ALWAYS_INLINE size_type size() const
Definition: SmallVector.h:135
SmallVector< PointerInfo, 2 > Pointers
Information about the pointers that may require checking.
PointerIntPair< Value *, 1, bool > MemAccessInfo
DenseMap< Instruction *, unsigned > generateInstructionOrderMap() const
Generate a mapping between the memory instructions and their indices according to program order...
Collection of parameters shared beetween the Loop Vectorizer and the Loop Access Analysis.
bool areDepsSafe(DepCandidates &AccessSets, MemAccessInfoSet &CheckDeps, const ValueToValueMap &Strides)
Check whether the dependencies between the accesses are safe.
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckType(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N, const TargetLowering *TLI, const DataLayout &DL)
RuntimePointerChecking & RtCheck
Constitutes the context of this pointer checking group.
A raw_ostream that writes to an std::string.
Definition: raw_ostream.h:463
LLVM Value Representation.
Definition: Value.h:71
static bool blockNeedsPredication(BasicBlock *BB, Loop *TheLoop, DominatorTree *DT)
Return true if the block BB needs to be predicated in order for the loop to be vectorized.
CheckingPtrGroup(unsigned Index, RuntimePointerChecking &RtCheck)
Create a new pointer checking group containing a single pointer, with index Index in RtCheck...
bool canVectorizeMemory() const
Return true we can analyze the memory accesses in the loop and there are no memory dependence cycles...
bool needsChecking(const CheckingPtrGroup &M, const CheckingPtrGroup &N) const
Decide if we need to add a check between two groups of pointers, according to needsChecking.
This class implements an extremely fast bulk output stream that can only output to a stream...
Definition: raw_ostream.h:44
static unsigned VectorizationInterleave
Interleave factor as overridden by the user.
MemoryDepChecker(PredicatedScalarEvolution &PSE, const Loop *L)
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:47
A container for analyses that lazily runs them and caches their results.
int * Ptr
static bool isInterleaveForced()
True if force-vector-interleave was specified by the user.
static GCRegistry::Add< ErlangGC > A("erlang","erlang-compatible garbage collector")
Value * getPointerOperand()
Definition: Instructions.h:394
The optimization diagnostic interface.
A special type used by analysis passes to provide an address that identifies that particular analysis...
Definition: PassManager.h:64
EquivalenceClasses< MemAccessInfo > DepCandidates
Set of potential dependent memory accesses.
void print(raw_ostream &OS, const Module *M=nullptr) const override
Print the result of the analysis when invoked with -analyze.
unsigned DependencySetId
Holds the id of the set of pointers that could be dependent because of a shared underlying object...
const SCEV * Expr
SCEV for the access.
bool shouldRetryWithRuntimeCheck()
In same cases when the dependency check fails we can still vectorize the loop with a dynamic array ac...
unsigned Destination
Index of the destination of the dependence in the InstMap vector.
bool Need
This flag indicates if we need to add the runtime check.
LoopAccessInfo(Loop *L, ScalarEvolution *SE, const TargetLibraryInfo *TLI, AliasAnalysis *AA, DominatorTree *DT, LoopInfo *LI)