LLVM  16.0.0git
VPlan.h
Go to the documentation of this file.
1 //===- VPlan.h - Represent A Vectorizer Plan --------------------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 /// \file
10 /// This file contains the declarations of the Vectorization Plan base classes:
11 /// 1. VPBasicBlock and VPRegionBlock that inherit from a common pure virtual
12 /// VPBlockBase, together implementing a Hierarchical CFG;
13 /// 2. Specializations of GraphTraits that allow VPBlockBase graphs to be
14 /// treated as proper graphs for generic algorithms;
15 /// 3. Pure virtual VPRecipeBase serving as the base class for recipes contained
16 /// within VPBasicBlocks;
17 /// 4. VPInstruction, a concrete Recipe and VPUser modeling a single planned
18 /// instruction;
19 /// 5. The VPlan class holding a candidate for vectorization;
20 /// 6. The VPlanPrinter class providing a way to print a plan in dot format;
21 /// These are documented in docs/VectorizationPlan.rst.
22 //
23 //===----------------------------------------------------------------------===//
24 
25 #ifndef LLVM_TRANSFORMS_VECTORIZE_VPLAN_H
26 #define LLVM_TRANSFORMS_VECTORIZE_VPLAN_H
27 
28 #include "VPlanValue.h"
29 #include "llvm/ADT/DenseMap.h"
31 #include "llvm/ADT/GraphTraits.h"
32 #include "llvm/ADT/MapVector.h"
33 #include "llvm/ADT/Optional.h"
35 #include "llvm/ADT/SmallPtrSet.h"
36 #include "llvm/ADT/SmallVector.h"
37 #include "llvm/ADT/Twine.h"
38 #include "llvm/ADT/ilist.h"
39 #include "llvm/ADT/ilist_node.h"
40 #include "llvm/Analysis/LoopInfo.h"
42 #include "llvm/IR/DebugLoc.h"
43 #include "llvm/IR/FMF.h"
45 #include <algorithm>
46 #include <cassert>
47 #include <cstddef>
48 #include <string>
49 
50 namespace llvm {
51 
52 class BasicBlock;
53 class DominatorTree;
54 class InductionDescriptor;
55 class InnerLoopVectorizer;
56 class IRBuilderBase;
57 class LoopInfo;
58 class raw_ostream;
59 class RecurrenceDescriptor;
60 class Value;
61 class VPBasicBlock;
62 class VPRegionBlock;
63 class VPlan;
64 class VPReplicateRecipe;
65 class VPlanSlp;
66 
67 namespace Intrinsic {
68 typedef unsigned ID;
69 }
70 
71 /// Returns a calculation for the total number of elements for a given \p VF.
72 /// For fixed width vectors this value is a constant, whereas for scalable
73 /// vectors it is an expression determined at runtime.
74 Value *getRuntimeVF(IRBuilderBase &B, Type *Ty, ElementCount VF);
75 
76 /// Return a value for Step multiplied by VF.
77 Value *createStepForVF(IRBuilderBase &B, Type *Ty, ElementCount VF,
78  int64_t Step);
79 
80 /// A range of powers-of-2 vectorization factors with fixed start and
81 /// adjustable end. The range includes start and excludes end, e.g.,:
82 /// [1, 9) = {1, 2, 4, 8}
83 struct VFRange {
84  // A power of 2.
86 
87  // Need not be a power of 2. If End <= Start range is empty.
89 
90  bool isEmpty() const {
92  }
93 
95  : Start(Start), End(End) {
97  "Both Start and End should have the same scalable flag");
99  "Expected Start to be a power of 2");
100  }
101 };
102 
103 using VPlanPtr = std::unique_ptr<VPlan>;
104 
105 /// In what follows, the term "input IR" refers to code that is fed into the
106 /// vectorizer whereas the term "output IR" refers to code that is generated by
107 /// the vectorizer.
108 
109 /// VPLane provides a way to access lanes in both fixed width and scalable
110 /// vectors, where for the latter the lane index sometimes needs calculating
111 /// as a runtime expression.
112 class VPLane {
113 public:
114  /// Kind describes how to interpret Lane.
115  enum class Kind : uint8_t {
116  /// For First, Lane is the index into the first N elements of a
117  /// fixed-vector <N x <ElTy>> or a scalable vector <vscale x N x <ElTy>>.
118  First,
119  /// For ScalableLast, Lane is the offset from the start of the last
120  /// N-element subvector in a scalable vector <vscale x N x <ElTy>>. For
121  /// example, a Lane of 0 corresponds to lane `(vscale - 1) * N`, a Lane of
122  /// 1 corresponds to `((vscale - 1) * N) + 1`, etc.
124  };
125 
126 private:
127  /// in [0..VF)
128  unsigned Lane;
129 
130  /// Indicates how the Lane should be interpreted, as described above.
131  Kind LaneKind;
132 
133 public:
134  VPLane(unsigned Lane, Kind LaneKind) : Lane(Lane), LaneKind(LaneKind) {}
135 
137 
139  unsigned LaneOffset = VF.getKnownMinValue() - 1;
140  Kind LaneKind;
141  if (VF.isScalable())
142  // In this case 'LaneOffset' refers to the offset from the start of the
143  // last subvector with VF.getKnownMinValue() elements.
144  LaneKind = VPLane::Kind::ScalableLast;
145  else
146  LaneKind = VPLane::Kind::First;
147  return VPLane(LaneOffset, LaneKind);
148  }
149 
150  /// Returns a compile-time known value for the lane index and asserts if the
151  /// lane can only be calculated at runtime.
152  unsigned getKnownLane() const {
153  assert(LaneKind == Kind::First);
154  return Lane;
155  }
156 
157  /// Returns an expression describing the lane index that can be used at
158  /// runtime.
160 
161  /// Returns the Kind of lane offset.
162  Kind getKind() const { return LaneKind; }
163 
164  /// Returns true if this is the first lane of the whole vector.
165  bool isFirstLane() const { return Lane == 0 && LaneKind == Kind::First; }
166 
167  /// Maps the lane to a cache index based on \p VF.
168  unsigned mapToCacheIndex(const ElementCount &VF) const {
169  switch (LaneKind) {
171  assert(VF.isScalable() && Lane < VF.getKnownMinValue());
172  return VF.getKnownMinValue() + Lane;
173  default:
174  assert(Lane < VF.getKnownMinValue());
175  return Lane;
176  }
177  }
178 
179  /// Returns the maxmimum number of lanes that we are able to consider
180  /// caching for \p VF.
181  static unsigned getNumCachedLanes(const ElementCount &VF) {
182  return VF.getKnownMinValue() * (VF.isScalable() ? 2 : 1);
183  }
184 };
185 
186 /// VPIteration represents a single point in the iteration space of the output
187 /// (vectorized and/or unrolled) IR loop.
188 struct VPIteration {
189  /// in [0..UF)
190  unsigned Part;
191 
193 
194  VPIteration(unsigned Part, unsigned Lane,
196  : Part(Part), Lane(Lane, Kind) {}
197 
198  VPIteration(unsigned Part, const VPLane &Lane) : Part(Part), Lane(Lane) {}
199 
200  bool isFirstIteration() const { return Part == 0 && Lane.isFirstLane(); }
201 };
202 
203 /// VPTransformState holds information passed down when "executing" a VPlan,
204 /// needed for generating the output IR.
209  : VF(VF), UF(UF), LI(LI), DT(DT), Builder(Builder), ILV(ILV), Plan(Plan),
210  LVer(nullptr) {}
211 
212  /// The chosen Vectorization and Unroll Factors of the loop being vectorized.
214  unsigned UF;
215 
216  /// Hold the indices to generate specific scalar instructions. Null indicates
217  /// that all instances are to be generated, using either scalar or vector
218  /// instructions.
220 
221  struct DataState {
222  /// A type for vectorized values in the new loop. Each value from the
223  /// original loop, when vectorized, is represented by UF vector values in
224  /// the new unrolled loop, where UF is the unroll factor.
226 
228 
231  } Data;
232 
233  /// Get the generated Value for a given VPValue and a given Part. Note that
234  /// as some Defs are still created by ILV and managed in its ValueMap, this
235  /// method will delegate the call to ILV in such cases in order to provide
236  /// callers a consistent API.
237  /// \see set.
238  Value *get(VPValue *Def, unsigned Part);
239 
240  /// Get the generated Value for a given VPValue and given Part and Lane.
242 
243  bool hasVectorValue(VPValue *Def, unsigned Part) {
244  auto I = Data.PerPartOutput.find(Def);
245  return I != Data.PerPartOutput.end() && Part < I->second.size() &&
246  I->second[Part];
247  }
248 
250  return Data.PerPartOutput.find(Def) != Data.PerPartOutput.end();
251  }
252 
254  auto I = Data.PerPartScalars.find(Def);
255  if (I == Data.PerPartScalars.end())
256  return false;
257  unsigned CacheIdx = Instance.Lane.mapToCacheIndex(VF);
258  return Instance.Part < I->second.size() &&
259  CacheIdx < I->second[Instance.Part].size() &&
260  I->second[Instance.Part][CacheIdx];
261  }
262 
263  /// Set the generated Value for a given VPValue and a given Part.
264  void set(VPValue *Def, Value *V, unsigned Part) {
265  if (!Data.PerPartOutput.count(Def)) {
267  Data.PerPartOutput[Def] = Entry;
268  }
269  Data.PerPartOutput[Def][Part] = V;
270  }
271  /// Reset an existing vector value for \p Def and a given \p Part.
272  void reset(VPValue *Def, Value *V, unsigned Part) {
273  auto Iter = Data.PerPartOutput.find(Def);
274  assert(Iter != Data.PerPartOutput.end() &&
275  "need to overwrite existing value");
276  Iter->second[Part] = V;
277  }
278 
279  /// Set the generated scalar \p V for \p Def and the given \p Instance.
280  void set(VPValue *Def, Value *V, const VPIteration &Instance) {
281  auto Iter = Data.PerPartScalars.insert({Def, {}});
282  auto &PerPartVec = Iter.first->second;
283  while (PerPartVec.size() <= Instance.Part)
284  PerPartVec.emplace_back();
285  auto &Scalars = PerPartVec[Instance.Part];
286  unsigned CacheIdx = Instance.Lane.mapToCacheIndex(VF);
287  while (Scalars.size() <= CacheIdx)
288  Scalars.push_back(nullptr);
289  assert(!Scalars[CacheIdx] && "should overwrite existing value");
290  Scalars[CacheIdx] = V;
291  }
292 
293  /// Reset an existing scalar value for \p Def and a given \p Instance.
294  void reset(VPValue *Def, Value *V, const VPIteration &Instance) {
295  auto Iter = Data.PerPartScalars.find(Def);
296  assert(Iter != Data.PerPartScalars.end() &&
297  "need to overwrite existing value");
298  assert(Instance.Part < Iter->second.size() &&
299  "need to overwrite existing value");
300  unsigned CacheIdx = Instance.Lane.mapToCacheIndex(VF);
301  assert(CacheIdx < Iter->second[Instance.Part].size() &&
302  "need to overwrite existing value");
303  Iter->second[Instance.Part][CacheIdx] = V;
304  }
305 
306  /// Add additional metadata to \p To that was not present on \p Orig.
307  ///
308  /// Currently this is used to add the noalias annotations based on the
309  /// inserted memchecks. Use this for instructions that are *cloned* into the
310  /// vector loop.
311  void addNewMetadata(Instruction *To, const Instruction *Orig);
312 
313  /// Add metadata from one instruction to another.
314  ///
315  /// This includes both the original MDs from \p From and additional ones (\see
316  /// addNewMetadata). Use this for *newly created* instructions in the vector
317  /// loop.
319 
320  /// Similar to the previous function but it adds the metadata to a
321  /// vector of instructions.
323 
324  /// Set the debug location in the builder using the debug location in \p V.
325  void setDebugLocFromInst(const Value *V);
326 
327  /// Hold state information used when constructing the CFG of the output IR,
328  /// traversing the VPBasicBlocks and generating corresponding IR BasicBlocks.
329  struct CFGState {
330  /// The previous VPBasicBlock visited. Initially set to null.
331  VPBasicBlock *PrevVPBB = nullptr;
332 
333  /// The previous IR BasicBlock created or used. Initially set to the new
334  /// header BasicBlock.
335  BasicBlock *PrevBB = nullptr;
336 
337  /// The last IR BasicBlock in the output IR. Set to the exit block of the
338  /// vector loop.
339  BasicBlock *ExitBB = nullptr;
340 
341  /// A mapping of each VPBasicBlock to the corresponding BasicBlock. In case
342  /// of replication, maps the BasicBlock of the last replica created.
344 
345  CFGState() = default;
346 
347  /// Returns the BasicBlock* mapped to the pre-header of the loop region
348  /// containing \p R.
350  } CFG;
351 
352  /// Hold a pointer to LoopInfo to register new basic blocks in the loop.
354 
355  /// Hold a pointer to Dominator Tree to register new basic blocks in the loop.
357 
358  /// Hold a reference to the IRBuilder used to generate output IR code.
360 
362 
363  /// Hold the canonical scalar IV of the vector loop (start=0, step=VF*UF).
364  Value *CanonicalIV = nullptr;
365 
366  /// Hold a pointer to InnerLoopVectorizer to reuse its IR generation methods.
368 
369  /// Pointer to the VPlan code is generated for.
371 
372  /// Holds recipes that may generate a poison value that is used after
373  /// vectorization, even when their operands are not poison.
375 
376  /// The loop object for the current parent region, or nullptr.
378 
379  /// LoopVersioning. It's only set up (non-null) if memchecks were
380  /// used.
381  ///
382  /// This is currently only used to add no-alias metadata based on the
383  /// memchecks. The actually versioning is performed manually.
384  std::unique_ptr<LoopVersioning> LVer;
385 };
386 
387 /// VPBlockBase is the building block of the Hierarchical Control-Flow Graph.
388 /// A VPBlockBase can be either a VPBasicBlock or a VPRegionBlock.
389 class VPBlockBase {
390  friend class VPBlockUtils;
391 
392  const unsigned char SubclassID; ///< Subclass identifier (for isa/dyn_cast).
393 
394  /// An optional name for the block.
395  std::string Name;
396 
397  /// The immediate VPRegionBlock which this VPBlockBase belongs to, or null if
398  /// it is a topmost VPBlockBase.
399  VPRegionBlock *Parent = nullptr;
400 
401  /// List of predecessor blocks.
402  SmallVector<VPBlockBase *, 1> Predecessors;
403 
404  /// List of successor blocks.
406 
407  /// VPlan containing the block. Can only be set on the entry block of the
408  /// plan.
409  VPlan *Plan = nullptr;
410 
411  /// Add \p Successor as the last successor to this block.
412  void appendSuccessor(VPBlockBase *Successor) {
413  assert(Successor && "Cannot add nullptr successor!");
414  Successors.push_back(Successor);
415  }
416 
417  /// Add \p Predecessor as the last predecessor to this block.
418  void appendPredecessor(VPBlockBase *Predecessor) {
419  assert(Predecessor && "Cannot add nullptr predecessor!");
420  Predecessors.push_back(Predecessor);
421  }
422 
423  /// Remove \p Predecessor from the predecessors of this block.
424  void removePredecessor(VPBlockBase *Predecessor) {
425  auto Pos = find(Predecessors, Predecessor);
426  assert(Pos && "Predecessor does not exist");
427  Predecessors.erase(Pos);
428  }
429 
430  /// Remove \p Successor from the successors of this block.
431  void removeSuccessor(VPBlockBase *Successor) {
432  auto Pos = find(Successors, Successor);
433  assert(Pos && "Successor does not exist");
434  Successors.erase(Pos);
435  }
436 
437 protected:
438  VPBlockBase(const unsigned char SC, const std::string &N)
439  : SubclassID(SC), Name(N) {}
440 
441 public:
442  /// An enumeration for keeping track of the concrete subclass of VPBlockBase
443  /// that are actually instantiated. Values of this enumeration are kept in the
444  /// SubclassID field of the VPBlockBase objects. They are used for concrete
445  /// type identification.
446  using VPBlockTy = enum { VPBasicBlockSC, VPRegionBlockSC };
447 
449 
450  virtual ~VPBlockBase() = default;
451 
452  const std::string &getName() const { return Name; }
453 
454  void setName(const Twine &newName) { Name = newName.str(); }
455 
456  /// \return an ID for the concrete type of this object.
457  /// This is used to implement the classof checks. This should not be used
458  /// for any other purpose, as the values may change as LLVM evolves.
459  unsigned getVPBlockID() const { return SubclassID; }
460 
461  VPRegionBlock *getParent() { return Parent; }
462  const VPRegionBlock *getParent() const { return Parent; }
463 
464  /// \return A pointer to the plan containing the current block.
465  VPlan *getPlan();
466  const VPlan *getPlan() const;
467 
468  /// Sets the pointer of the plan containing the block. The block must be the
469  /// entry block into the VPlan.
470  void setPlan(VPlan *ParentPlan);
471 
472  void setParent(VPRegionBlock *P) { Parent = P; }
473 
474  /// \return the VPBasicBlock that is the entry of this VPBlockBase,
475  /// recursively, if the latter is a VPRegionBlock. Otherwise, if this
476  /// VPBlockBase is a VPBasicBlock, it is returned.
477  const VPBasicBlock *getEntryBasicBlock() const;
479 
480  /// \return the VPBasicBlock that is the exiting this VPBlockBase,
481  /// recursively, if the latter is a VPRegionBlock. Otherwise, if this
482  /// VPBlockBase is a VPBasicBlock, it is returned.
483  const VPBasicBlock *getExitingBasicBlock() const;
485 
486  const VPBlocksTy &getSuccessors() const { return Successors; }
487  VPBlocksTy &getSuccessors() { return Successors; }
488 
490 
491  const VPBlocksTy &getPredecessors() const { return Predecessors; }
492  VPBlocksTy &getPredecessors() { return Predecessors; }
493 
494  /// \return the successor of this VPBlockBase if it has a single successor.
495  /// Otherwise return a null pointer.
497  return (Successors.size() == 1 ? *Successors.begin() : nullptr);
498  }
499 
500  /// \return the predecessor of this VPBlockBase if it has a single
501  /// predecessor. Otherwise return a null pointer.
503  return (Predecessors.size() == 1 ? *Predecessors.begin() : nullptr);
504  }
505 
506  size_t getNumSuccessors() const { return Successors.size(); }
507  size_t getNumPredecessors() const { return Predecessors.size(); }
508 
509  /// An Enclosing Block of a block B is any block containing B, including B
510  /// itself. \return the closest enclosing block starting from "this", which
511  /// has successors. \return the root enclosing block if all enclosing blocks
512  /// have no successors.
514 
515  /// \return the closest enclosing block starting from "this", which has
516  /// predecessors. \return the root enclosing block if all enclosing blocks
517  /// have no predecessors.
519 
520  /// \return the successors either attached directly to this VPBlockBase or, if
521  /// this VPBlockBase is the exit block of a VPRegionBlock and has no
522  /// successors of its own, search recursively for the first enclosing
523  /// VPRegionBlock that has successors and return them. If no such
524  /// VPRegionBlock exists, return the (empty) successors of the topmost
525  /// VPBlockBase reached.
528  }
529 
530  /// \return the hierarchical successor of this VPBlockBase if it has a single
531  /// hierarchical successor. Otherwise return a null pointer.
534  }
535 
536  /// \return the predecessors either attached directly to this VPBlockBase or,
537  /// if this VPBlockBase is the entry block of a VPRegionBlock and has no
538  /// predecessors of its own, search recursively for the first enclosing
539  /// VPRegionBlock that has predecessors and return them. If no such
540  /// VPRegionBlock exists, return the (empty) predecessors of the topmost
541  /// VPBlockBase reached.
544  }
545 
546  /// \return the hierarchical predecessor of this VPBlockBase if it has a
547  /// single hierarchical predecessor. Otherwise return a null pointer.
550  }
551 
552  /// Set a given VPBlockBase \p Successor as the single successor of this
553  /// VPBlockBase. This VPBlockBase is not added as predecessor of \p Successor.
554  /// This VPBlockBase must have no successors.
556  assert(Successors.empty() && "Setting one successor when others exist.");
557  appendSuccessor(Successor);
558  }
559 
560  /// Set two given VPBlockBases \p IfTrue and \p IfFalse to be the two
561  /// successors of this VPBlockBase. This VPBlockBase is not added as
562  /// predecessor of \p IfTrue or \p IfFalse. This VPBlockBase must have no
563  /// successors.
564  void setTwoSuccessors(VPBlockBase *IfTrue, VPBlockBase *IfFalse) {
565  assert(Successors.empty() && "Setting two successors when others exist.");
566  appendSuccessor(IfTrue);
567  appendSuccessor(IfFalse);
568  }
569 
570  /// Set each VPBasicBlock in \p NewPreds as predecessor of this VPBlockBase.
571  /// This VPBlockBase must have no predecessors. This VPBlockBase is not added
572  /// as successor of any VPBasicBlock in \p NewPreds.
574  assert(Predecessors.empty() && "Block predecessors already set.");
575  for (auto *Pred : NewPreds)
576  appendPredecessor(Pred);
577  }
578 
579  /// Remove all the predecessor of this block.
580  void clearPredecessors() { Predecessors.clear(); }
581 
582  /// Remove all the successors of this block.
583  void clearSuccessors() { Successors.clear(); }
584 
585  /// The method which generates the output IR that correspond to this
586  /// VPBlockBase, thereby "executing" the VPlan.
587  virtual void execute(VPTransformState *State) = 0;
588 
589  /// Delete all blocks reachable from a given VPBlockBase, inclusive.
590  static void deleteCFG(VPBlockBase *Entry);
591 
592  /// Return true if it is legal to hoist instructions into this block.
594  // There are currently no constraints that prevent an instruction to be
595  // hoisted into a VPBlockBase.
596  return true;
597  }
598 
599  /// Replace all operands of VPUsers in the block with \p NewValue and also
600  /// replaces all uses of VPValues defined in the block with NewValue.
601  virtual void dropAllReferences(VPValue *NewValue) = 0;
602 
603 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
604  void printAsOperand(raw_ostream &OS, bool PrintType) const {
605  OS << getName();
606  }
607 
608  /// Print plain-text dump of this VPBlockBase to \p O, prefixing all lines
609  /// with \p Indent. \p SlotTracker is used to print unnamed VPValue's using
610  /// consequtive numbers.
611  ///
612  /// Note that the numbering is applied to the whole VPlan, so printing
613  /// individual blocks is consistent with the whole VPlan printing.
614  virtual void print(raw_ostream &O, const Twine &Indent,
615  VPSlotTracker &SlotTracker) const = 0;
616 
617  /// Print plain-text dump of this VPlan to \p O.
618  void print(raw_ostream &O) const {
620  print(O, "", SlotTracker);
621  }
622 
623  /// Print the successors of this block to \p O, prefixing all lines with \p
624  /// Indent.
625  void printSuccessors(raw_ostream &O, const Twine &Indent) const;
626 
627  /// Dump this VPBlockBase to dbgs().
628  LLVM_DUMP_METHOD void dump() const { print(dbgs()); }
629 #endif
630 };
631 
632 /// A value that is used outside the VPlan. The operand of the user needs to be
633 /// added to the associated LCSSA phi node.
634 class VPLiveOut : public VPUser {
635  PHINode *Phi;
636 
637 public:
639  : VPUser({Op}, VPUser::VPUserID::LiveOut), Phi(Phi) {}
640 
641  /// Fixup the wrapped LCSSA phi node in the unique exit block. This simply
642  /// means we need to add the appropriate incoming value from the middle
643  /// block as exiting edges from the scalar epilogue loop (if present) are
644  /// already in place, and we exit the vector loop exclusively to the middle
645  /// block.
646  void fixPhi(VPlan &Plan, VPTransformState &State);
647 
648  /// Returns true if the VPLiveOut uses scalars of operand \p Op.
649  bool usesScalars(const VPValue *Op) const override {
651  "Op must be an operand of the recipe");
652  return true;
653  }
654 
655  PHINode *getPhi() const { return Phi; }
656 };
657 
658 /// VPRecipeBase is a base class modeling a sequence of one or more output IR
659 /// instructions. VPRecipeBase owns the the VPValues it defines through VPDef
660 /// and is responsible for deleting its defined values. Single-value
661 /// VPRecipeBases that also inherit from VPValue must make sure to inherit from
662 /// VPRecipeBase before VPValue.
663 class VPRecipeBase : public ilist_node_with_parent<VPRecipeBase, VPBasicBlock>,
664  public VPDef,
665  public VPUser {
666  friend VPBasicBlock;
667  friend class VPBlockUtils;
668 
669  /// Each VPRecipe belongs to a single VPBasicBlock.
670  VPBasicBlock *Parent = nullptr;
671 
672 public:
674  : VPDef(SC), VPUser(Operands, VPUser::VPUserID::Recipe) {}
675 
676  template <typename IterT>
678  : VPDef(SC), VPUser(Operands, VPUser::VPUserID::Recipe) {}
679  virtual ~VPRecipeBase() = default;
680 
681  /// \return the VPBasicBlock which this VPRecipe belongs to.
682  VPBasicBlock *getParent() { return Parent; }
683  const VPBasicBlock *getParent() const { return Parent; }
684 
685  /// The method which generates the output IR instructions that correspond to
686  /// this VPRecipe, thereby "executing" the VPlan.
687  virtual void execute(VPTransformState &State) = 0;
688 
689  /// Insert an unlinked recipe into a basic block immediately before
690  /// the specified recipe.
691  void insertBefore(VPRecipeBase *InsertPos);
692  /// Insert an unlinked recipe into \p BB immediately before the insertion
693  /// point \p IP;
695 
696  /// Insert an unlinked Recipe into a basic block immediately after
697  /// the specified Recipe.
698  void insertAfter(VPRecipeBase *InsertPos);
699 
700  /// Unlink this recipe from its current VPBasicBlock and insert it into
701  /// the VPBasicBlock that MovePos lives in, right after MovePos.
702  void moveAfter(VPRecipeBase *MovePos);
703 
704  /// Unlink this recipe and insert into BB before I.
705  ///
706  /// \pre I is a valid iterator into BB.
708 
709  /// This method unlinks 'this' from the containing basic block, but does not
710  /// delete it.
711  void removeFromParent();
712 
713  /// This method unlinks 'this' from the containing basic block and deletes it.
714  ///
715  /// \returns an iterator pointing to the element after the erased one
717 
718  /// Returns the underlying instruction, if the recipe is a VPValue or nullptr
719  /// otherwise.
721  return cast<Instruction>(getVPSingleValue()->getUnderlyingValue());
722  }
724  return cast<Instruction>(getVPSingleValue()->getUnderlyingValue());
725  }
726 
727  /// Method to support type inquiry through isa, cast, and dyn_cast.
728  static inline bool classof(const VPDef *D) {
729  // All VPDefs are also VPRecipeBases.
730  return true;
731  }
732 
733  static inline bool classof(const VPUser *U) {
734  return U->getVPUserID() == VPUser::VPUserID::Recipe;
735  }
736 
737  /// Returns true if the recipe may have side-effects.
738  bool mayHaveSideEffects() const;
739 
740  /// Returns true for PHI-like recipes.
741  bool isPhi() const {
742  return getVPDefID() >= VPFirstPHISC && getVPDefID() <= VPLastPHISC;
743  }
744 
745  /// Returns true if the recipe may read from memory.
746  bool mayReadFromMemory() const;
747 
748  /// Returns true if the recipe may write to memory.
749  bool mayWriteToMemory() const;
750 
751  /// Returns true if the recipe may read from or write to memory.
752  bool mayReadOrWriteMemory() const {
753  return mayReadFromMemory() || mayWriteToMemory();
754  }
755 };
756 
757 inline bool VPUser::classof(const VPDef *Def) {
758  return Def->getVPDefID() == VPRecipeBase::VPInstructionSC ||
759  Def->getVPDefID() == VPRecipeBase::VPWidenSC ||
760  Def->getVPDefID() == VPRecipeBase::VPWidenCallSC ||
761  Def->getVPDefID() == VPRecipeBase::VPWidenSelectSC ||
762  Def->getVPDefID() == VPRecipeBase::VPWidenGEPSC ||
763  Def->getVPDefID() == VPRecipeBase::VPBlendSC ||
764  Def->getVPDefID() == VPRecipeBase::VPInterleaveSC ||
765  Def->getVPDefID() == VPRecipeBase::VPReplicateSC ||
766  Def->getVPDefID() == VPRecipeBase::VPReductionSC ||
767  Def->getVPDefID() == VPRecipeBase::VPBranchOnMaskSC ||
768  Def->getVPDefID() == VPRecipeBase::VPWidenMemoryInstructionSC;
769 }
770 
771 /// This is a concrete Recipe that models a single VPlan-level instruction.
772 /// While as any Recipe it may generate a sequence of IR instructions when
773 /// executed, these instructions would always form a single-def expression as
774 /// the VPInstruction is also a single def-use vertex.
775 class VPInstruction : public VPRecipeBase, public VPValue {
776  friend class VPlanSlp;
777 
778 public:
779  /// VPlan opcodes, extending LLVM IR with idiomatics instructions.
780  enum {
782  Instruction::OtherOpsEnd + 1, // Combines the incoming and previous
783  // values of a first-order recurrence.
791  // The next two are similar to the above, but instead increment the
792  // canonical IV separately for each unrolled part.
797  };
798 
799 private:
800  typedef unsigned char OpcodeTy;
801  OpcodeTy Opcode;
802  FastMathFlags FMF;
803  DebugLoc DL;
804 
805  /// An optional name that can be used for the generated IR instruction.
806  const std::string Name;
807 
808  /// Utility method serving execute(): generates a single instance of the
809  /// modeled instruction.
810  void generateInstruction(VPTransformState &State, unsigned Part);
811 
812 protected:
814 
815 public:
817  const Twine &Name = "")
818  : VPRecipeBase(VPRecipeBase::VPInstructionSC, Operands),
819  VPValue(VPValue::VPVInstructionSC, nullptr, this), Opcode(Opcode),
820  DL(DL), Name(Name.str()) {}
821 
822  VPInstruction(unsigned Opcode, std::initializer_list<VPValue *> Operands,
823  DebugLoc DL = {}, const Twine &Name = "")
825 
826  /// Method to support type inquiry through isa, cast, and dyn_cast.
827  static inline bool classof(const VPValue *V) {
829  }
830 
831  VPInstruction *clone() const {
833  return new VPInstruction(Opcode, Operands, DL, Name);
834  }
835 
836  /// Method to support type inquiry through isa, cast, and dyn_cast.
837  static inline bool classof(const VPDef *R) {
838  return R->getVPDefID() == VPRecipeBase::VPInstructionSC;
839  }
840 
841  /// Extra classof implementations to allow directly casting from VPUser ->
842  /// VPInstruction.
843  static inline bool classof(const VPUser *U) {
844  auto *R = dyn_cast<VPRecipeBase>(U);
845  return R && R->getVPDefID() == VPRecipeBase::VPInstructionSC;
846  }
847  static inline bool classof(const VPRecipeBase *R) {
848  return R->getVPDefID() == VPRecipeBase::VPInstructionSC;
849  }
850 
851  unsigned getOpcode() const { return Opcode; }
852 
853  /// Generate the instruction.
854  /// TODO: We currently execute only per-part unless a specific instance is
855  /// provided.
856  void execute(VPTransformState &State) override;
857 
858 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
859  /// Print the VPInstruction to \p O.
860  void print(raw_ostream &O, const Twine &Indent,
861  VPSlotTracker &SlotTracker) const override;
862 
863  /// Print the VPInstruction to dbgs() (for debugging).
864  LLVM_DUMP_METHOD void dump() const;
865 #endif
866 
867  /// Return true if this instruction may modify memory.
868  bool mayWriteToMemory() const {
869  // TODO: we can use attributes of the called function to rule out memory
870  // modifications.
871  return Opcode == Instruction::Store || Opcode == Instruction::Call ||
872  Opcode == Instruction::Invoke || Opcode == SLPStore;
873  }
874 
875  bool hasResult() const {
876  // CallInst may or may not have a result, depending on the called function.
877  // Conservatively return calls have results for now.
878  switch (getOpcode()) {
879  case Instruction::Ret:
880  case Instruction::Br:
881  case Instruction::Store:
882  case Instruction::Switch:
883  case Instruction::IndirectBr:
884  case Instruction::Resume:
885  case Instruction::CatchRet:
886  case Instruction::Unreachable:
887  case Instruction::Fence:
888  case Instruction::AtomicRMW:
891  return false;
892  default:
893  return true;
894  }
895  }
896 
897  /// Set the fast-math flags.
898  void setFastMathFlags(FastMathFlags FMFNew);
899 
900  /// Returns true if the recipe only uses the first lane of operand \p Op.
901  bool onlyFirstLaneUsed(const VPValue *Op) const override {
903  "Op must be an operand of the recipe");
904  if (getOperand(0) != Op)
905  return false;
906  switch (getOpcode()) {
907  default:
908  return false;
915  return true;
916  };
917  llvm_unreachable("switch should return");
918  }
919 };
920 
921 /// VPWidenRecipe is a recipe for producing a copy of vector type its
922 /// ingredient. This recipe covers most of the traditional vectorization cases
923 /// where each ingredient transforms into a vectorized version of itself.
924 class VPWidenRecipe : public VPRecipeBase, public VPValue {
925 public:
926  template <typename IterT>
928  : VPRecipeBase(VPRecipeBase::VPWidenSC, Operands),
930 
931  ~VPWidenRecipe() override = default;
932 
933  /// Method to support type inquiry through isa, cast, and dyn_cast.
934  static inline bool classof(const VPDef *D) {
935  return D->getVPDefID() == VPRecipeBase::VPWidenSC;
936  }
937  static inline bool classof(const VPValue *V) {
938  return V->getVPValueID() == VPValue::VPVWidenSC;
939  }
940 
941  /// Produce widened copies of all Ingredients.
942  void execute(VPTransformState &State) override;
943 
944 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
945  /// Print the recipe.
946  void print(raw_ostream &O, const Twine &Indent,
947  VPSlotTracker &SlotTracker) const override;
948 #endif
949 };
950 
951 /// A recipe for widening Call instructions.
952 class VPWidenCallRecipe : public VPRecipeBase, public VPValue {
953  /// ID of the vector intrinsic to call when widening the call. If set the
954  /// Intrinsic::not_intrinsic, a library call will be used instead.
955  Intrinsic::ID VectorIntrinsicID;
956 
957 public:
958  template <typename IterT>
960  Intrinsic::ID VectorIntrinsicID)
961  : VPRecipeBase(VPRecipeBase::VPWidenCallSC, CallArguments),
963  VectorIntrinsicID(VectorIntrinsicID) {}
964 
965  ~VPWidenCallRecipe() override = default;
966 
967  /// Method to support type inquiry through isa, cast, and dyn_cast.
968  static inline bool classof(const VPDef *D) {
969  return D->getVPDefID() == VPRecipeBase::VPWidenCallSC;
970  }
971 
972  /// Produce a widened version of the call instruction.
973  void execute(VPTransformState &State) override;
974 
975 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
976  /// Print the recipe.
977  void print(raw_ostream &O, const Twine &Indent,
978  VPSlotTracker &SlotTracker) const override;
979 #endif
980 };
981 
982 /// A recipe for widening select instructions.
983 class VPWidenSelectRecipe : public VPRecipeBase, public VPValue {
984 
985  /// Is the condition of the select loop invariant?
986  bool InvariantCond;
987 
988 public:
989  template <typename IterT>
991  bool InvariantCond)
992  : VPRecipeBase(VPRecipeBase::VPWidenSelectSC, Operands),
994  InvariantCond(InvariantCond) {}
995 
996  ~VPWidenSelectRecipe() override = default;
997 
998  /// Method to support type inquiry through isa, cast, and dyn_cast.
999  static inline bool classof(const VPDef *D) {
1000  return D->getVPDefID() == VPRecipeBase::VPWidenSelectSC;
1001  }
1002 
1003  /// Produce a widened version of the select instruction.
1004  void execute(VPTransformState &State) override;
1005 
1006 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1007  /// Print the recipe.
1008  void print(raw_ostream &O, const Twine &Indent,
1009  VPSlotTracker &SlotTracker) const override;
1010 #endif
1011 };
1012 
1013 /// A recipe for handling GEP instructions.
1014 class VPWidenGEPRecipe : public VPRecipeBase, public VPValue {
1015  bool IsPtrLoopInvariant;
1016  SmallBitVector IsIndexLoopInvariant;
1017 
1018 public:
1019  template <typename IterT>
1021  : VPRecipeBase(VPRecipeBase::VPWidenGEPSC, Operands),
1022  VPValue(VPWidenGEPSC, GEP, this),
1023  IsIndexLoopInvariant(GEP->getNumIndices(), false) {}
1024 
1025  template <typename IterT>
1027  Loop *OrigLoop)
1028  : VPRecipeBase(VPRecipeBase::VPWidenGEPSC, Operands),
1030  IsIndexLoopInvariant(GEP->getNumIndices(), false) {
1031  IsPtrLoopInvariant = OrigLoop->isLoopInvariant(GEP->getPointerOperand());
1032  for (auto Index : enumerate(GEP->indices()))
1033  IsIndexLoopInvariant[Index.index()] =
1034  OrigLoop->isLoopInvariant(Index.value().get());
1035  }
1036  ~VPWidenGEPRecipe() override = default;
1037 
1038  /// Method to support type inquiry through isa, cast, and dyn_cast.
1039  static inline bool classof(const VPDef *D) {
1040  return D->getVPDefID() == VPRecipeBase::VPWidenGEPSC;
1041  }
1042 
1043  /// Generate the gep nodes.
1044  void execute(VPTransformState &State) override;
1045 
1046 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1047  /// Print the recipe.
1048  void print(raw_ostream &O, const Twine &Indent,
1049  VPSlotTracker &SlotTracker) const override;
1050 #endif
1051 };
1052 
1053 /// A recipe for handling phi nodes of integer and floating-point inductions,
1054 /// producing their vector values.
1056  PHINode *IV;
1057  const InductionDescriptor &IndDesc;
1058  bool NeedsVectorIV;
1059 
1060 public:
1062  const InductionDescriptor &IndDesc,
1063  bool NeedsVectorIV)
1064  : VPRecipeBase(VPWidenIntOrFpInductionSC, {Start, Step}),
1065  VPValue(IV, this), IV(IV), IndDesc(IndDesc),
1066  NeedsVectorIV(NeedsVectorIV) {}
1067 
1069  const InductionDescriptor &IndDesc,
1070  TruncInst *Trunc, bool NeedsVectorIV)
1071  : VPRecipeBase(VPWidenIntOrFpInductionSC, {Start, Step}),
1072  VPValue(Trunc, this), IV(IV), IndDesc(IndDesc),
1073  NeedsVectorIV(NeedsVectorIV) {}
1074 
1075  ~VPWidenIntOrFpInductionRecipe() override = default;
1076 
1077  /// Method to support type inquiry through isa, cast, and dyn_cast.
1078  static inline bool classof(const VPDef *D) {
1079  return D->getVPDefID() == VPRecipeBase::VPWidenIntOrFpInductionSC;
1080  }
1081 
1082  /// Generate the vectorized and scalarized versions of the phi node as
1083  /// needed by their users.
1084  void execute(VPTransformState &State) override;
1085 
1086 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1087  /// Print the recipe.
1088  void print(raw_ostream &O, const Twine &Indent,
1089  VPSlotTracker &SlotTracker) const override;
1090 #endif
1091 
1092  /// Returns the start value of the induction.
1094  const VPValue *getStartValue() const { return getOperand(0); }
1095 
1096  /// Returns the step value of the induction.
1097  VPValue *getStepValue() { return getOperand(1); }
1098  const VPValue *getStepValue() const { return getOperand(1); }
1099 
1100  /// Returns the first defined value as TruncInst, if it is one or nullptr
1101  /// otherwise.
1103  return dyn_cast_or_null<TruncInst>(getVPValue(0)->getUnderlyingValue());
1104  }
1105  const TruncInst *getTruncInst() const {
1106  return dyn_cast_or_null<TruncInst>(getVPValue(0)->getUnderlyingValue());
1107  }
1108 
1109  PHINode *getPHINode() { return IV; }
1110 
1111  /// Returns the induction descriptor for the recipe.
1112  const InductionDescriptor &getInductionDescriptor() const { return IndDesc; }
1113 
1114  /// Returns true if the induction is canonical, i.e. starting at 0 and
1115  /// incremented by UF * VF (= the original IV is incremented by 1).
1116  bool isCanonical() const;
1117 
1118  /// Returns the scalar type of the induction.
1119  const Type *getScalarType() const {
1120  const TruncInst *TruncI = getTruncInst();
1121  return TruncI ? TruncI->getType() : IV->getType();
1122  }
1123 
1124  /// Returns true if a vector phi needs to be created for the induction.
1125  bool needsVectorIV() const { return NeedsVectorIV; }
1126 };
1127 
1128 /// A pure virtual base class for all recipes modeling header phis, including
1129 /// phis for first order recurrences, pointer inductions and reductions. The
1130 /// start value is the first operand of the recipe and the incoming value from
1131 /// the backedge is the second operand.
1132 class VPHeaderPHIRecipe : public VPRecipeBase, public VPValue {
1133 protected:
1134  VPHeaderPHIRecipe(unsigned char VPVID, unsigned char VPDefID, PHINode *Phi,
1135  VPValue *Start = nullptr)
1136  : VPRecipeBase(VPDefID, {}), VPValue(VPVID, Phi, this) {
1137  if (Start)
1138  addOperand(Start);
1139  }
1140 
1141 public:
1142  ~VPHeaderPHIRecipe() override = default;
1143 
1144  /// Method to support type inquiry through isa, cast, and dyn_cast.
1145  static inline bool classof(const VPRecipeBase *B) {
1146  return B->getVPDefID() >= VPRecipeBase::VPFirstHeaderPHISC &&
1147  B->getVPDefID() <= VPRecipeBase::VPLastPHISC;
1148  }
1149  static inline bool classof(const VPValue *V) {
1150  return V->getVPValueID() >= VPValue::VPVFirstHeaderPHISC &&
1152  }
1153 
1154  /// Generate the phi nodes.
1155  void execute(VPTransformState &State) override = 0;
1156 
1157 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1158  /// Print the recipe.
1159  void print(raw_ostream &O, const Twine &Indent,
1160  VPSlotTracker &SlotTracker) const override = 0;
1161 #endif
1162 
1163  /// Returns the start value of the phi, if one is set.
1165  return getNumOperands() == 0 ? nullptr : getOperand(0);
1166  }
1168  return getNumOperands() == 0 ? nullptr : getOperand(0);
1169  }
1170 
1171  /// Returns the incoming value from the loop backedge.
1173  return getOperand(1);
1174  }
1175 
1176  /// Returns the backedge value as a recipe. The backedge value is guaranteed
1177  /// to be a recipe.
1179  return cast<VPRecipeBase>(getBackedgeValue()->getDef());
1180  }
1181 };
1182 
1184  const InductionDescriptor &IndDesc;
1185 
1186  bool IsScalarAfterVectorization;
1187 
1188 public:
1189  /// Create a new VPWidenPointerInductionRecipe for \p Phi with start value \p
1190  /// Start.
1192  const InductionDescriptor &IndDesc,
1193  bool IsScalarAfterVectorization)
1194  : VPHeaderPHIRecipe(VPVWidenPointerInductionSC, VPWidenPointerInductionSC,
1195  Phi),
1196  IndDesc(IndDesc),
1197  IsScalarAfterVectorization(IsScalarAfterVectorization) {
1198  addOperand(Start);
1199  addOperand(Step);
1200  }
1201 
1202  ~VPWidenPointerInductionRecipe() override = default;
1203 
1204  /// Method to support type inquiry through isa, cast, and dyn_cast.
1205  static inline bool classof(const VPRecipeBase *B) {
1206  return B->getVPDefID() == VPRecipeBase::VPWidenPointerInductionSC;
1207  }
1208  static inline bool classof(const VPHeaderPHIRecipe *R) {
1209  return R->getVPDefID() == VPRecipeBase::VPWidenPointerInductionSC;
1210  }
1211  static inline bool classof(const VPValue *V) {
1213  }
1214 
1215  /// Generate vector values for the pointer induction.
1216  void execute(VPTransformState &State) override;
1217 
1218  /// Returns true if only scalar values will be generated.
1220 
1221 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1222  /// Print the recipe.
1223  void print(raw_ostream &O, const Twine &Indent,
1224  VPSlotTracker &SlotTracker) const override;
1225 #endif
1226 };
1227 
1228 /// A recipe for handling header phis that are widened in the vector loop.
1229 /// In the VPlan native path, all incoming VPValues & VPBasicBlock pairs are
1230 /// managed in the recipe directly.
1232  /// List of incoming blocks. Only used in the VPlan native path.
1233  SmallVector<VPBasicBlock *, 2> IncomingBlocks;
1234 
1235 public:
1236  /// Create a new VPWidenPHIRecipe for \p Phi with start value \p Start.
1237  VPWidenPHIRecipe(PHINode *Phi, VPValue *Start = nullptr)
1238  : VPHeaderPHIRecipe(VPVWidenPHISC, VPWidenPHISC, Phi) {
1239  if (Start)
1240  addOperand(Start);
1241  }
1242 
1243  ~VPWidenPHIRecipe() override = default;
1244 
1245  /// Method to support type inquiry through isa, cast, and dyn_cast.
1246  static inline bool classof(const VPRecipeBase *B) {
1247  return B->getVPDefID() == VPRecipeBase::VPWidenPHISC;
1248  }
1249  static inline bool classof(const VPHeaderPHIRecipe *R) {
1250  return R->getVPDefID() == VPRecipeBase::VPWidenPHISC;
1251  }
1252  static inline bool classof(const VPValue *V) {
1253  return V->getVPValueID() == VPValue::VPVWidenPHISC;
1254  }
1255 
1256  /// Generate the phi/select nodes.
1257  void execute(VPTransformState &State) override;
1258 
1259 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1260  /// Print the recipe.
1261  void print(raw_ostream &O, const Twine &Indent,
1262  VPSlotTracker &SlotTracker) const override;
1263 #endif
1264 
1265  /// Adds a pair (\p IncomingV, \p IncomingBlock) to the phi.
1266  void addIncoming(VPValue *IncomingV, VPBasicBlock *IncomingBlock) {
1267  addOperand(IncomingV);
1268  IncomingBlocks.push_back(IncomingBlock);
1269  }
1270 
1271  /// Returns the \p I th incoming VPBasicBlock.
1272  VPBasicBlock *getIncomingBlock(unsigned I) { return IncomingBlocks[I]; }
1273 
1274  /// Returns the \p I th incoming VPValue.
1275  VPValue *getIncomingValue(unsigned I) { return getOperand(I); }
1276 };
1277 
1278 /// A recipe for handling first-order recurrence phis. The start value is the
1279 /// first operand of the recipe and the incoming value from the backedge is the
1280 /// second operand.
1284  VPFirstOrderRecurrencePHISC, Phi, &Start) {}
1285 
1286  /// Method to support type inquiry through isa, cast, and dyn_cast.
1287  static inline bool classof(const VPRecipeBase *R) {
1288  return R->getVPDefID() == VPRecipeBase::VPFirstOrderRecurrencePHISC;
1289  }
1290  static inline bool classof(const VPHeaderPHIRecipe *R) {
1291  return R->getVPDefID() == VPRecipeBase::VPFirstOrderRecurrencePHISC;
1292  }
1293  static inline bool classof(const VPValue *V) {
1295  }
1296 
1297  void execute(VPTransformState &State) override;
1298 
1299 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1300  /// Print the recipe.
1301  void print(raw_ostream &O, const Twine &Indent,
1302  VPSlotTracker &SlotTracker) const override;
1303 #endif
1304 };
1305 
1306 /// A recipe for handling reduction phis. The start value is the first operand
1307 /// of the recipe and the incoming value from the backedge is the second
1308 /// operand.
1310  /// Descriptor for the reduction.
1311  const RecurrenceDescriptor &RdxDesc;
1312 
1313  /// The phi is part of an in-loop reduction.
1314  bool IsInLoop;
1315 
1316  /// The phi is part of an ordered reduction. Requires IsInLoop to be true.
1317  bool IsOrdered;
1318 
1319 public:
1320  /// Create a new VPReductionPHIRecipe for the reduction \p Phi described by \p
1321  /// RdxDesc.
1323  VPValue &Start, bool IsInLoop = false,
1324  bool IsOrdered = false)
1325  : VPHeaderPHIRecipe(VPVReductionPHISC, VPReductionPHISC, Phi, &Start),
1326  RdxDesc(RdxDesc), IsInLoop(IsInLoop), IsOrdered(IsOrdered) {
1327  assert((!IsOrdered || IsInLoop) && "IsOrdered requires IsInLoop");
1328  }
1329 
1330  ~VPReductionPHIRecipe() override = default;
1331 
1332  /// Method to support type inquiry through isa, cast, and dyn_cast.
1333  static inline bool classof(const VPRecipeBase *R) {
1334  return R->getVPDefID() == VPRecipeBase::VPReductionPHISC;
1335  }
1336  static inline bool classof(const VPHeaderPHIRecipe *R) {
1337  return R->getVPDefID() == VPRecipeBase::VPReductionPHISC;
1338  }
1339  static inline bool classof(const VPValue *V) {
1341  }
1342 
1343  /// Generate the phi/select nodes.
1344  void execute(VPTransformState &State) override;
1345 
1346 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1347  /// Print the recipe.
1348  void print(raw_ostream &O, const Twine &Indent,
1349  VPSlotTracker &SlotTracker) const override;
1350 #endif
1351 
1353  return RdxDesc;
1354  }
1355 
1356  /// Returns true, if the phi is part of an ordered reduction.
1357  bool isOrdered() const { return IsOrdered; }
1358 
1359  /// Returns true, if the phi is part of an in-loop reduction.
1360  bool isInLoop() const { return IsInLoop; }
1361 };
1362 
1363 /// A recipe for vectorizing a phi-node as a sequence of mask-based select
1364 /// instructions.
1365 class VPBlendRecipe : public VPRecipeBase, public VPValue {
1366  PHINode *Phi;
1367 
1368 public:
1369  /// The blend operation is a User of the incoming values and of their
1370  /// respective masks, ordered [I0, M0, I1, M1, ...]. Note that a single value
1371  /// might be incoming with a full mask for which there is no VPValue.
1373  : VPRecipeBase(VPBlendSC, Operands),
1374  VPValue(VPValue::VPVBlendSC, Phi, this), Phi(Phi) {
1375  assert(Operands.size() > 0 &&
1376  ((Operands.size() == 1) || (Operands.size() % 2 == 0)) &&
1377  "Expected either a single incoming value or a positive even number "
1378  "of operands");
1379  }
1380 
1381  /// Method to support type inquiry through isa, cast, and dyn_cast.
1382  static inline bool classof(const VPDef *D) {
1383  return D->getVPDefID() == VPRecipeBase::VPBlendSC;
1384  }
1385 
1386  /// Return the number of incoming values, taking into account that a single
1387  /// incoming value has no mask.
1388  unsigned getNumIncomingValues() const { return (getNumOperands() + 1) / 2; }
1389 
1390  /// Return incoming value number \p Idx.
1391  VPValue *getIncomingValue(unsigned Idx) const { return getOperand(Idx * 2); }
1392 
1393  /// Return mask number \p Idx.
1394  VPValue *getMask(unsigned Idx) const { return getOperand(Idx * 2 + 1); }
1395 
1396  /// Generate the phi/select nodes.
1397  void execute(VPTransformState &State) override;
1398 
1399 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1400  /// Print the recipe.
1401  void print(raw_ostream &O, const Twine &Indent,
1402  VPSlotTracker &SlotTracker) const override;
1403 #endif
1404 
1405  /// Returns true if the recipe only uses the first lane of operand \p Op.
1406  bool onlyFirstLaneUsed(const VPValue *Op) const override {
1408  "Op must be an operand of the recipe");
1409  // Recursing through Blend recipes only, must terminate at header phi's the
1410  // latest.
1411  return all_of(users(),
1412  [this](VPUser *U) { return U->onlyFirstLaneUsed(this); });
1413  }
1414 };
1415 
1416 /// VPInterleaveRecipe is a recipe for transforming an interleave group of load
1417 /// or stores into one wide load/store and shuffles. The first operand of a
1418 /// VPInterleave recipe is the address, followed by the stored values, followed
1419 /// by an optional mask.
1421  const InterleaveGroup<Instruction> *IG;
1422 
1423  bool HasMask = false;
1424 
1425 public:
1427  ArrayRef<VPValue *> StoredValues, VPValue *Mask)
1428  : VPRecipeBase(VPInterleaveSC, {Addr}), IG(IG) {
1429  for (unsigned i = 0; i < IG->getFactor(); ++i)
1430  if (Instruction *I = IG->getMember(i)) {
1431  if (I->getType()->isVoidTy())
1432  continue;
1433  new VPValue(I, this);
1434  }
1435 
1436  for (auto *SV : StoredValues)
1437  addOperand(SV);
1438  if (Mask) {
1439  HasMask = true;
1440  addOperand(Mask);
1441  }
1442  }
1443  ~VPInterleaveRecipe() override = default;
1444 
1445  /// Method to support type inquiry through isa, cast, and dyn_cast.
1446  static inline bool classof(const VPDef *D) {
1447  return D->getVPDefID() == VPRecipeBase::VPInterleaveSC;
1448  }
1449 
1450  /// Return the address accessed by this recipe.
1451  VPValue *getAddr() const {
1452  return getOperand(0); // Address is the 1st, mandatory operand.
1453  }
1454 
1455  /// Return the mask used by this recipe. Note that a full mask is represented
1456  /// by a nullptr.
1457  VPValue *getMask() const {
1458  // Mask is optional and therefore the last, currently 2nd operand.
1459  return HasMask ? getOperand(getNumOperands() - 1) : nullptr;
1460  }
1461 
1462  /// Return the VPValues stored by this interleave group. If it is a load
1463  /// interleave group, return an empty ArrayRef.
1465  // The first operand is the address, followed by the stored values, followed
1466  // by an optional mask.
1468  .slice(1, getNumStoreOperands());
1469  }
1470 
1471  /// Generate the wide load or store, and shuffles.
1472  void execute(VPTransformState &State) override;
1473 
1474 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1475  /// Print the recipe.
1476  void print(raw_ostream &O, const Twine &Indent,
1477  VPSlotTracker &SlotTracker) const override;
1478 #endif
1479 
1481 
1482  /// Returns the number of stored operands of this interleave group. Returns 0
1483  /// for load interleave groups.
1484  unsigned getNumStoreOperands() const {
1485  return getNumOperands() - (HasMask ? 2 : 1);
1486  }
1487 
1488  /// The recipe only uses the first lane of the address.
1489  bool onlyFirstLaneUsed(const VPValue *Op) const override {
1491  "Op must be an operand of the recipe");
1492  return Op == getAddr() && !llvm::is_contained(getStoredValues(), Op);
1493  }
1494 };
1495 
1496 /// A recipe to represent inloop reduction operations, performing a reduction on
1497 /// a vector operand into a scalar value, and adding the result to a chain.
1498 /// The Operands are {ChainOp, VecOp, [Condition]}.
1499 class VPReductionRecipe : public VPRecipeBase, public VPValue {
1500  /// The recurrence decriptor for the reduction in question.
1501  const RecurrenceDescriptor *RdxDesc;
1502  /// Pointer to the TTI, needed to create the target reduction
1503  const TargetTransformInfo *TTI;
1504 
1505 public:
1507  VPValue *ChainOp, VPValue *VecOp, VPValue *CondOp,
1508  const TargetTransformInfo *TTI)
1509  : VPRecipeBase(VPRecipeBase::VPReductionSC, {ChainOp, VecOp}),
1510  VPValue(VPValue::VPVReductionSC, I, this), RdxDesc(R), TTI(TTI) {
1511  if (CondOp)
1512  addOperand(CondOp);
1513  }
1514 
1515  ~VPReductionRecipe() override = default;
1516 
1517  /// Method to support type inquiry through isa, cast, and dyn_cast.
1518  static inline bool classof(const VPValue *V) {
1519  return V->getVPValueID() == VPValue::VPVReductionSC;
1520  }
1521 
1522  /// Generate the reduction in the loop
1523  void execute(VPTransformState &State) override;
1524 
1525 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1526  /// Print the recipe.
1527  void print(raw_ostream &O, const Twine &Indent,
1528  VPSlotTracker &SlotTracker) const override;
1529 #endif
1530 
1531  /// The VPValue of the scalar Chain being accumulated.
1532  VPValue *getChainOp() const { return getOperand(0); }
1533  /// The VPValue of the vector value to be reduced.
1534  VPValue *getVecOp() const { return getOperand(1); }
1535  /// The VPValue of the condition for the block.
1536  VPValue *getCondOp() const {
1537  return getNumOperands() > 2 ? getOperand(2) : nullptr;
1538  }
1539 };
1540 
1541 /// VPReplicateRecipe replicates a given instruction producing multiple scalar
1542 /// copies of the original scalar type, one per lane, instead of producing a
1543 /// single copy of widened type for all lanes. If the instruction is known to be
1544 /// uniform only one copy, per lane zero, will be generated.
1545 class VPReplicateRecipe : public VPRecipeBase, public VPValue {
1546  /// Indicator if only a single replica per lane is needed.
1547  bool IsUniform;
1548 
1549  /// Indicator if the replicas are also predicated.
1550  bool IsPredicated;
1551 
1552  /// Indicator if the scalar values should also be packed into a vector.
1553  bool AlsoPack;
1554 
1555 public:
1556  template <typename IterT>
1558  bool IsUniform, bool IsPredicated = false)
1559  : VPRecipeBase(VPReplicateSC, Operands), VPValue(VPVReplicateSC, I, this),
1560  IsUniform(IsUniform), IsPredicated(IsPredicated) {
1561  // Retain the previous behavior of predicateInstructions(), where an
1562  // insert-element of a predicated instruction got hoisted into the
1563  // predicated basic block iff it was its only user. This is achieved by
1564  // having predicated instructions also pack their values into a vector by
1565  // default unless they have a replicated user which uses their scalar value.
1566  AlsoPack = IsPredicated && !I->use_empty();
1567  }
1568 
1569  ~VPReplicateRecipe() override = default;
1570 
1571  /// Method to support type inquiry through isa, cast, and dyn_cast.
1572  static inline bool classof(const VPDef *D) {
1573  return D->getVPDefID() == VPRecipeBase::VPReplicateSC;
1574  }
1575 
1576  static inline bool classof(const VPValue *V) {
1577  return V->getVPValueID() == VPValue::VPVReplicateSC;
1578  }
1579 
1580  /// Generate replicas of the desired Ingredient. Replicas will be generated
1581  /// for all parts and lanes unless a specific part and lane are specified in
1582  /// the \p State.
1583  void execute(VPTransformState &State) override;
1584 
1585  void setAlsoPack(bool Pack) { AlsoPack = Pack; }
1586 
1587 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1588  /// Print the recipe.
1589  void print(raw_ostream &O, const Twine &Indent,
1590  VPSlotTracker &SlotTracker) const override;
1591 #endif
1592 
1593  bool isUniform() const { return IsUniform; }
1594 
1595  bool isPacked() const { return AlsoPack; }
1596 
1597  bool isPredicated() const { return IsPredicated; }
1598 
1599  /// Returns true if the recipe only uses the first lane of operand \p Op.
1600  bool onlyFirstLaneUsed(const VPValue *Op) const override {
1602  "Op must be an operand of the recipe");
1603  return isUniform();
1604  }
1605 
1606  /// Returns true if the recipe uses scalars of operand \p Op.
1607  bool usesScalars(const VPValue *Op) const override {
1609  "Op must be an operand of the recipe");
1610  return true;
1611  }
1612 };
1613 
1614 /// A recipe for generating conditional branches on the bits of a mask.
1616 public:
1618  : VPRecipeBase(VPBranchOnMaskSC, {}) {
1619  if (BlockInMask) // nullptr means all-one mask.
1620  addOperand(BlockInMask);
1621  }
1622 
1623  /// Method to support type inquiry through isa, cast, and dyn_cast.
1624  static inline bool classof(const VPDef *D) {
1625  return D->getVPDefID() == VPRecipeBase::VPBranchOnMaskSC;
1626  }
1627 
1628  /// Generate the extraction of the appropriate bit from the block mask and the
1629  /// conditional branch.
1630  void execute(VPTransformState &State) override;
1631 
1632 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1633  /// Print the recipe.
1634  void print(raw_ostream &O, const Twine &Indent,
1635  VPSlotTracker &SlotTracker) const override {
1636  O << Indent << "BRANCH-ON-MASK ";
1637  if (VPValue *Mask = getMask())
1638  Mask->printAsOperand(O, SlotTracker);
1639  else
1640  O << " All-One";
1641  }
1642 #endif
1643 
1644  /// Return the mask used by this recipe. Note that a full mask is represented
1645  /// by a nullptr.
1646  VPValue *getMask() const {
1647  assert(getNumOperands() <= 1 && "should have either 0 or 1 operands");
1648  // Mask is optional.
1649  return getNumOperands() == 1 ? getOperand(0) : nullptr;
1650  }
1651 
1652  /// Returns true if the recipe uses scalars of operand \p Op.
1653  bool usesScalars(const VPValue *Op) const override {
1655  "Op must be an operand of the recipe");
1656  return true;
1657  }
1658 };
1659 
1660 /// VPPredInstPHIRecipe is a recipe for generating the phi nodes needed when
1661 /// control converges back from a Branch-on-Mask. The phi nodes are needed in
1662 /// order to merge values that are set under such a branch and feed their uses.
1663 /// The phi nodes can be scalar or vector depending on the users of the value.
1664 /// This recipe works in concert with VPBranchOnMaskRecipe.
1665 class VPPredInstPHIRecipe : public VPRecipeBase, public VPValue {
1666 public:
1667  /// Construct a VPPredInstPHIRecipe given \p PredInst whose value needs a phi
1668  /// nodes after merging back from a Branch-on-Mask.
1670  : VPRecipeBase(VPPredInstPHISC, PredV),
1671  VPValue(VPValue::VPVPredInstPHI, nullptr, this) {}
1672  ~VPPredInstPHIRecipe() override = default;
1673 
1674  /// Method to support type inquiry through isa, cast, and dyn_cast.
1675  static inline bool classof(const VPDef *D) {
1676  return D->getVPDefID() == VPRecipeBase::VPPredInstPHISC;
1677  }
1678 
1679  /// Generates phi nodes for live-outs as needed to retain SSA form.
1680  void execute(VPTransformState &State) override;
1681 
1682 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1683  /// Print the recipe.
1684  void print(raw_ostream &O, const Twine &Indent,
1685  VPSlotTracker &SlotTracker) const override;
1686 #endif
1687 
1688  /// Returns true if the recipe uses scalars of operand \p Op.
1689  bool usesScalars(const VPValue *Op) const override {
1691  "Op must be an operand of the recipe");
1692  return true;
1693  }
1694 };
1695 
1696 /// A Recipe for widening load/store operations.
1697 /// The recipe uses the following VPValues:
1698 /// - For load: Address, optional mask
1699 /// - For store: Address, stored value, optional mask
1700 /// TODO: We currently execute only per-part unless a specific instance is
1701 /// provided.
1703  Instruction &Ingredient;
1704 
1705  // Whether the loaded-from / stored-to addresses are consecutive.
1706  bool Consecutive;
1707 
1708  // Whether the consecutive loaded/stored addresses are in reverse order.
1709  bool Reverse;
1710 
1711  void setMask(VPValue *Mask) {
1712  if (!Mask)
1713  return;
1714  addOperand(Mask);
1715  }
1716 
1717  bool isMasked() const {
1718  return isStore() ? getNumOperands() == 3 : getNumOperands() == 2;
1719  }
1720 
1721 public:
1723  bool Consecutive, bool Reverse)
1724  : VPRecipeBase(VPWidenMemoryInstructionSC, {Addr}), Ingredient(Load),
1725  Consecutive(Consecutive), Reverse(Reverse) {
1726  assert((Consecutive || !Reverse) && "Reverse implies consecutive");
1728  setMask(Mask);
1729  }
1730 
1732  VPValue *StoredValue, VPValue *Mask,
1733  bool Consecutive, bool Reverse)
1734  : VPRecipeBase(VPWidenMemoryInstructionSC, {Addr, StoredValue}),
1735  Ingredient(Store), Consecutive(Consecutive), Reverse(Reverse) {
1736  assert((Consecutive || !Reverse) && "Reverse implies consecutive");
1737  setMask(Mask);
1738  }
1739 
1740  /// Method to support type inquiry through isa, cast, and dyn_cast.
1741  static inline bool classof(const VPDef *D) {
1742  return D->getVPDefID() == VPRecipeBase::VPWidenMemoryInstructionSC;
1743  }
1744 
1745  /// Return the address accessed by this recipe.
1746  VPValue *getAddr() const {
1747  return getOperand(0); // Address is the 1st, mandatory operand.
1748  }
1749 
1750  /// Return the mask used by this recipe. Note that a full mask is represented
1751  /// by a nullptr.
1752  VPValue *getMask() const {
1753  // Mask is optional and therefore the last operand.
1754  return isMasked() ? getOperand(getNumOperands() - 1) : nullptr;
1755  }
1756 
1757  /// Returns true if this recipe is a store.
1758  bool isStore() const { return isa<StoreInst>(Ingredient); }
1759 
1760  /// Return the address accessed by this recipe.
1762  assert(isStore() && "Stored value only available for store instructions");
1763  return getOperand(1); // Stored value is the 2nd, mandatory operand.
1764  }
1765 
1766  // Return whether the loaded-from / stored-to addresses are consecutive.
1767  bool isConsecutive() const { return Consecutive; }
1768 
1769  // Return whether the consecutive loaded/stored addresses are in reverse
1770  // order.
1771  bool isReverse() const { return Reverse; }
1772 
1773  /// Generate the wide load/store.
1774  void execute(VPTransformState &State) override;
1775 
1776 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1777  /// Print the recipe.
1778  void print(raw_ostream &O, const Twine &Indent,
1779  VPSlotTracker &SlotTracker) const override;
1780 #endif
1781 
1782  /// Returns true if the recipe only uses the first lane of operand \p Op.
1783  bool onlyFirstLaneUsed(const VPValue *Op) const override {
1785  "Op must be an operand of the recipe");
1786 
1787  // Widened, consecutive memory operations only demand the first lane of
1788  // their address, unless the same operand is also stored. That latter can
1789  // happen with opaque pointers.
1790  return Op == getAddr() && isConsecutive() &&
1791  (!isStore() || Op != getStoredValue());
1792  }
1793 
1794  Instruction &getIngredient() const { return Ingredient; }
1795 };
1796 
1797 /// Recipe to expand a SCEV expression.
1798 class VPExpandSCEVRecipe : public VPRecipeBase, public VPValue {
1799  const SCEV *Expr;
1800  ScalarEvolution &SE;
1801 
1802 public:
1804  : VPRecipeBase(VPExpandSCEVSC, {}), VPValue(nullptr, this), Expr(Expr),
1805  SE(SE) {}
1806 
1807  ~VPExpandSCEVRecipe() override = default;
1808 
1809  /// Method to support type inquiry through isa, cast, and dyn_cast.
1810  static inline bool classof(const VPDef *D) {
1811  return D->getVPDefID() == VPExpandSCEVSC;
1812  }
1813 
1814  /// Generate a canonical vector induction variable of the vector loop, with
1815  void execute(VPTransformState &State) override;
1816 
1817 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1818  /// Print the recipe.
1819  void print(raw_ostream &O, const Twine &Indent,
1820  VPSlotTracker &SlotTracker) const override;
1821 #endif
1822 
1823  const SCEV *getSCEV() const { return Expr; }
1824 };
1825 
1826 /// Canonical scalar induction phi of the vector loop. Starting at the specified
1827 /// start value (either 0 or the resume value when vectorizing the epilogue
1828 /// loop). VPWidenCanonicalIVRecipe represents the vector version of the
1829 /// canonical induction variable.
1831  DebugLoc DL;
1832 
1833 public:
1835  : VPHeaderPHIRecipe(VPValue::VPVCanonicalIVPHISC, VPCanonicalIVPHISC,
1836  nullptr, StartV),
1837  DL(DL) {}
1838 
1839  ~VPCanonicalIVPHIRecipe() override = default;
1840 
1841  /// Method to support type inquiry through isa, cast, and dyn_cast.
1842  static inline bool classof(const VPDef *D) {
1843  return D->getVPDefID() == VPCanonicalIVPHISC;
1844  }
1845  static inline bool classof(const VPHeaderPHIRecipe *D) {
1846  return D->getVPDefID() == VPCanonicalIVPHISC;
1847  }
1848  static inline bool classof(const VPValue *V) {
1850  }
1851 
1852  /// Generate the canonical scalar induction phi of the vector loop.
1853  void execute(VPTransformState &State) override;
1854 
1855 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1856  /// Print the recipe.
1857  void print(raw_ostream &O, const Twine &Indent,
1858  VPSlotTracker &SlotTracker) const override;
1859 #endif
1860 
1861  /// Returns the scalar type of the induction.
1862  const Type *getScalarType() const {
1863  return getOperand(0)->getLiveInIRValue()->getType();
1864  }
1865 
1866  /// Returns true if the recipe only uses the first lane of operand \p Op.
1867  bool onlyFirstLaneUsed(const VPValue *Op) const override {
1869  "Op must be an operand of the recipe");
1870  return true;
1871  }
1872 };
1873 
1874 /// A recipe for generating the active lane mask for the vector loop that is
1875 /// used to predicate the vector operations.
1876 /// TODO: It would be good to use the existing VPWidenPHIRecipe instead and
1877 /// remove VPActiveLaneMaskPHIRecipe.
1879  DebugLoc DL;
1880 
1881 public:
1884  VPActiveLaneMaskPHISC, nullptr, StartMask),
1885  DL(DL) {}
1886 
1887  ~VPActiveLaneMaskPHIRecipe() override = default;
1888 
1889  /// Method to support type inquiry through isa, cast, and dyn_cast.
1890  static inline bool classof(const VPDef *D) {
1891  return D->getVPDefID() == VPActiveLaneMaskPHISC;
1892  }
1893  static inline bool classof(const VPHeaderPHIRecipe *D) {
1894  return D->getVPDefID() == VPActiveLaneMaskPHISC;
1895  }
1896  static inline bool classof(const VPValue *V) {
1898  }
1899 
1900  /// Generate the active lane mask phi of the vector loop.
1901  void execute(VPTransformState &State) override;
1902 
1903 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1904  /// Print the recipe.
1905  void print(raw_ostream &O, const Twine &Indent,
1906  VPSlotTracker &SlotTracker) const override;
1907 #endif
1908 };
1909 
1910 /// A Recipe for widening the canonical induction variable of the vector loop.
1912 public:
1914  : VPRecipeBase(VPWidenCanonicalIVSC, {CanonicalIV}),
1915  VPValue(VPValue::VPVWidenCanonicalIVSC, nullptr, this) {}
1916 
1917  ~VPWidenCanonicalIVRecipe() override = default;
1918 
1919  /// Method to support type inquiry through isa, cast, and dyn_cast.
1920  static inline bool classof(const VPDef *D) {
1921  return D->getVPDefID() == VPRecipeBase::VPWidenCanonicalIVSC;
1922  }
1923 
1924  /// Extra classof implementations to allow directly casting from VPUser ->
1925  /// VPWidenCanonicalIVRecipe.
1926  static inline bool classof(const VPUser *U) {
1927  auto *R = dyn_cast<VPRecipeBase>(U);
1928  return R && R->getVPDefID() == VPRecipeBase::VPWidenCanonicalIVSC;
1929  }
1930  static inline bool classof(const VPRecipeBase *R) {
1931  return R->getVPDefID() == VPRecipeBase::VPWidenCanonicalIVSC;
1932  }
1933 
1934  /// Generate a canonical vector induction variable of the vector loop, with
1935  /// start = {<Part*VF, Part*VF+1, ..., Part*VF+VF-1> for 0 <= Part < UF}, and
1936  /// step = <VF*UF, VF*UF, ..., VF*UF>.
1937  void execute(VPTransformState &State) override;
1938 
1939 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1940  /// Print the recipe.
1941  void print(raw_ostream &O, const Twine &Indent,
1942  VPSlotTracker &SlotTracker) const override;
1943 #endif
1944 
1945  /// Returns the scalar type of the induction.
1946  const Type *getScalarType() const {
1947  return cast<VPCanonicalIVPHIRecipe>(getOperand(0)->getDef())
1948  ->getScalarType();
1949  }
1950 };
1951 
1952 /// A recipe for handling phi nodes of integer and floating-point inductions,
1953 /// producing their scalar values.
1955  /// Scalar type to use for the generated values.
1956  Type *Ty;
1957  /// If not nullptr, truncate the generated values to TruncToTy.
1958  Type *TruncToTy;
1959  const InductionDescriptor &IndDesc;
1960 
1961 public:
1963  VPValue *CanonicalIV, VPValue *Start, VPValue *Step,
1964  Type *TruncToTy)
1965  : VPRecipeBase(VPScalarIVStepsSC, {CanonicalIV, Start, Step}),
1966  VPValue(nullptr, this), Ty(Ty), TruncToTy(TruncToTy), IndDesc(IndDesc) {
1967  }
1968 
1969  ~VPScalarIVStepsRecipe() override = default;
1970 
1971  /// Method to support type inquiry through isa, cast, and dyn_cast.
1972  static inline bool classof(const VPDef *D) {
1973  return D->getVPDefID() == VPRecipeBase::VPScalarIVStepsSC;
1974  }
1975  /// Extra classof implementations to allow directly casting from VPUser ->
1976  /// VPScalarIVStepsRecipe.
1977  static inline bool classof(const VPUser *U) {
1978  auto *R = dyn_cast<VPRecipeBase>(U);
1979  return R && R->getVPDefID() == VPRecipeBase::VPScalarIVStepsSC;
1980  }
1981  static inline bool classof(const VPRecipeBase *R) {
1982  return R->getVPDefID() == VPRecipeBase::VPScalarIVStepsSC;
1983  }
1984 
1985  /// Generate the scalarized versions of the phi node as needed by their users.
1986  void execute(VPTransformState &State) override;
1987 
1988 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1989  /// Print the recipe.
1990  void print(raw_ostream &O, const Twine &Indent,
1991  VPSlotTracker &SlotTracker) const override;
1992 #endif
1993 
1994  /// Returns true if the induction is canonical, i.e. starting at 0 and
1995  /// incremented by UF * VF (= the original IV is incremented by 1).
1996  bool isCanonical() const;
1997 
1999  VPValue *getStartValue() const { return getOperand(1); }
2000  VPValue *getStepValue() const { return getOperand(2); }
2001 
2002  /// Returns true if the recipe only uses the first lane of operand \p Op.
2003  bool onlyFirstLaneUsed(const VPValue *Op) const override {
2005  "Op must be an operand of the recipe");
2006  return true;
2007  }
2008 };
2009 
2010 /// VPBasicBlock serves as the leaf of the Hierarchical Control-Flow Graph. It
2011 /// holds a sequence of zero or more VPRecipe's each representing a sequence of
2012 /// output IR instructions. All PHI-like recipes must come before any non-PHI recipes.
2013 class VPBasicBlock : public VPBlockBase {
2014 public:
2016 
2017 private:
2018  /// The VPRecipes held in the order of output instructions to generate.
2019  RecipeListTy Recipes;
2020 
2021 public:
2022  VPBasicBlock(const Twine &Name = "", VPRecipeBase *Recipe = nullptr)
2023  : VPBlockBase(VPBasicBlockSC, Name.str()) {
2024  if (Recipe)
2025  appendRecipe(Recipe);
2026  }
2027 
2028  ~VPBasicBlock() override {
2029  while (!Recipes.empty())
2030  Recipes.pop_back();
2031  }
2032 
2033  /// Instruction iterators...
2038 
2039  //===--------------------------------------------------------------------===//
2040  /// Recipe iterator methods
2041  ///
2042  inline iterator begin() { return Recipes.begin(); }
2043  inline const_iterator begin() const { return Recipes.begin(); }
2044  inline iterator end() { return Recipes.end(); }
2045  inline const_iterator end() const { return Recipes.end(); }
2046 
2047  inline reverse_iterator rbegin() { return Recipes.rbegin(); }
2048  inline const_reverse_iterator rbegin() const { return Recipes.rbegin(); }
2049  inline reverse_iterator rend() { return Recipes.rend(); }
2050  inline const_reverse_iterator rend() const { return Recipes.rend(); }
2051 
2052  inline size_t size() const { return Recipes.size(); }
2053  inline bool empty() const { return Recipes.empty(); }
2054  inline const VPRecipeBase &front() const { return Recipes.front(); }
2055  inline VPRecipeBase &front() { return Recipes.front(); }
2056  inline const VPRecipeBase &back() const { return Recipes.back(); }
2057  inline VPRecipeBase &back() { return Recipes.back(); }
2058 
2059  /// Returns a reference to the list of recipes.
2060  RecipeListTy &getRecipeList() { return Recipes; }
2061 
2062  /// Returns a pointer to a member of the recipe list.
2064  return &VPBasicBlock::Recipes;
2065  }
2066 
2067  /// Method to support type inquiry through isa, cast, and dyn_cast.
2068  static inline bool classof(const VPBlockBase *V) {
2069  return V->getVPBlockID() == VPBlockBase::VPBasicBlockSC;
2070  }
2071 
2072  void insert(VPRecipeBase *Recipe, iterator InsertPt) {
2073  assert(Recipe && "No recipe to append.");
2074  assert(!Recipe->Parent && "Recipe already in VPlan");
2075  Recipe->Parent = this;
2076  Recipes.insert(InsertPt, Recipe);
2077  }
2078 
2079  /// Augment the existing recipes of a VPBasicBlock with an additional
2080  /// \p Recipe as the last recipe.
2081  void appendRecipe(VPRecipeBase *Recipe) { insert(Recipe, end()); }
2082 
2083  /// The method which generates the output IR instructions that correspond to
2084  /// this VPBasicBlock, thereby "executing" the VPlan.
2085  void execute(VPTransformState *State) override;
2086 
2087  /// Return the position of the first non-phi node recipe in the block.
2089 
2090  /// Returns an iterator range over the PHI-like recipes in the block.
2092  return make_range(begin(), getFirstNonPhi());
2093  }
2094 
2095  void dropAllReferences(VPValue *NewValue) override;
2096 
2097  /// Split current block at \p SplitAt by inserting a new block between the
2098  /// current block and its successors and moving all recipes starting at
2099  /// SplitAt to the new block. Returns the new block.
2100  VPBasicBlock *splitAt(iterator SplitAt);
2101 
2103 
2104 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2105  /// Print this VPBsicBlock to \p O, prefixing all lines with \p Indent. \p
2106  /// SlotTracker is used to print unnamed VPValue's using consequtive numbers.
2107  ///
2108  /// Note that the numbering is applied to the whole VPlan, so printing
2109  /// individual blocks is consistent with the whole VPlan printing.
2110  void print(raw_ostream &O, const Twine &Indent,
2111  VPSlotTracker &SlotTracker) const override;
2112  using VPBlockBase::print; // Get the print(raw_stream &O) version.
2113 #endif
2114 
2115  /// If the block has multiple successors, return the branch recipe terminating
2116  /// the block. If there are no or only a single successor, return nullptr;
2118  const VPRecipeBase *getTerminator() const;
2119 
2120  /// Returns true if the block is exiting it's parent region.
2121  bool isExiting() const;
2122 
2123 private:
2124  /// Create an IR BasicBlock to hold the output instructions generated by this
2125  /// VPBasicBlock, and return it. Update the CFGState accordingly.
2126  BasicBlock *createEmptyBasicBlock(VPTransformState::CFGState &CFG);
2127 };
2128 
2129 /// VPRegionBlock represents a collection of VPBasicBlocks and VPRegionBlocks
2130 /// which form a Single-Entry-Single-Exiting subgraph of the output IR CFG.
2131 /// A VPRegionBlock may indicate that its contents are to be replicated several
2132 /// times. This is designed to support predicated scalarization, in which a
2133 /// scalar if-then code structure needs to be generated VF * UF times. Having
2134 /// this replication indicator helps to keep a single model for multiple
2135 /// candidate VF's. The actual replication takes place only once the desired VF
2136 /// and UF have been determined.
2137 class VPRegionBlock : public VPBlockBase {
2138  /// Hold the Single Entry of the SESE region modelled by the VPRegionBlock.
2139  VPBlockBase *Entry;
2140 
2141  /// Hold the Single Exiting block of the SESE region modelled by the
2142  /// VPRegionBlock.
2143  VPBlockBase *Exiting;
2144 
2145  /// An indicator whether this region is to generate multiple replicated
2146  /// instances of output IR corresponding to its VPBlockBases.
2147  bool IsReplicator;
2148 
2149 public:
2151  const std::string &Name = "", bool IsReplicator = false)
2152  : VPBlockBase(VPRegionBlockSC, Name), Entry(Entry), Exiting(Exiting),
2153  IsReplicator(IsReplicator) {
2154  assert(Entry->getPredecessors().empty() && "Entry block has predecessors.");
2155  assert(Exiting->getSuccessors().empty() && "Exit block has successors.");
2156  Entry->setParent(this);
2157  Exiting->setParent(this);
2158  }
2159  VPRegionBlock(const std::string &Name = "", bool IsReplicator = false)
2160  : VPBlockBase(VPRegionBlockSC, Name), Entry(nullptr), Exiting(nullptr),
2161  IsReplicator(IsReplicator) {}
2162 
2163  ~VPRegionBlock() override {
2164  if (Entry) {
2165  VPValue DummyValue;
2166  Entry->dropAllReferences(&DummyValue);
2167  deleteCFG(Entry);
2168  }
2169  }
2170 
2171  /// Method to support type inquiry through isa, cast, and dyn_cast.
2172  static inline bool classof(const VPBlockBase *V) {
2173  return V->getVPBlockID() == VPBlockBase::VPRegionBlockSC;
2174  }
2175 
2176  const VPBlockBase *getEntry() const { return Entry; }
2177  VPBlockBase *getEntry() { return Entry; }
2178 
2179  /// Set \p EntryBlock as the entry VPBlockBase of this VPRegionBlock. \p
2180  /// EntryBlock must have no predecessors.
2181  void setEntry(VPBlockBase *EntryBlock) {
2182  assert(EntryBlock->getPredecessors().empty() &&
2183  "Entry block cannot have predecessors.");
2184  Entry = EntryBlock;
2185  EntryBlock->setParent(this);
2186  }
2187 
2188  // FIXME: DominatorTreeBase is doing 'A->getParent()->front()'. 'front' is a
2189  // specific interface of llvm::Function, instead of using
2190  // GraphTraints::getEntryNode. We should add a new template parameter to
2191  // DominatorTreeBase representing the Graph type.
2192  VPBlockBase &front() const { return *Entry; }
2193 
2194  const VPBlockBase *getExiting() const { return Exiting; }
2195  VPBlockBase *getExiting() { return Exiting; }
2196 
2197  /// Set \p ExitingBlock as the exiting VPBlockBase of this VPRegionBlock. \p
2198  /// ExitingBlock must have no successors.
2199  void setExiting(VPBlockBase *ExitingBlock) {
2200  assert(ExitingBlock->getSuccessors().empty() &&
2201  "Exit block cannot have successors.");
2202  Exiting = ExitingBlock;
2203  ExitingBlock->setParent(this);
2204  }
2205 
2206  /// Returns the pre-header VPBasicBlock of the loop region.
2208  assert(!isReplicator() && "should only get pre-header of loop regions");
2210  }
2211 
2212  /// An indicator whether this region is to generate multiple replicated
2213  /// instances of output IR corresponding to its VPBlockBases.
2214  bool isReplicator() const { return IsReplicator; }
2215 
2216  /// The method which generates the output IR instructions that correspond to
2217  /// this VPRegionBlock, thereby "executing" the VPlan.
2218  void execute(VPTransformState *State) override;
2219 
2220  void dropAllReferences(VPValue *NewValue) override;
2221 
2222 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2223  /// Print this VPRegionBlock to \p O (recursively), prefixing all lines with
2224  /// \p Indent. \p SlotTracker is used to print unnamed VPValue's using
2225  /// consequtive numbers.
2226  ///
2227  /// Note that the numbering is applied to the whole VPlan, so printing
2228  /// individual regions is consistent with the whole VPlan printing.
2229  void print(raw_ostream &O, const Twine &Indent,
2230  VPSlotTracker &SlotTracker) const override;
2231  using VPBlockBase::print; // Get the print(raw_stream &O) version.
2232 #endif
2233 };
2234 
2235 //===----------------------------------------------------------------------===//
2236 // GraphTraits specializations for VPlan Hierarchical Control-Flow Graphs //
2237 //===----------------------------------------------------------------------===//
2238 
2239 // The following set of template specializations implement GraphTraits to treat
2240 // any VPBlockBase as a node in a graph of VPBlockBases. It's important to note
2241 // that VPBlockBase traits don't recurse into VPRegioBlocks, i.e., if the
2242 // VPBlockBase is a VPRegionBlock, this specialization provides access to its
2243 // successors/predecessors but not to the blocks inside the region.
2244 
2245 template <> struct GraphTraits<VPBlockBase *> {
2248 
2249  static NodeRef getEntryNode(NodeRef N) { return N; }
2250 
2252  return N->getSuccessors().begin();
2253  }
2254 
2256  return N->getSuccessors().end();
2257  }
2258 };
2259 
2260 template <> struct GraphTraits<const VPBlockBase *> {
2261  using NodeRef = const VPBlockBase *;
2263 
2264  static NodeRef getEntryNode(NodeRef N) { return N; }
2265 
2267  return N->getSuccessors().begin();
2268  }
2269 
2271  return N->getSuccessors().end();
2272  }
2273 };
2274 
2275 // Inverse order specialization for VPBasicBlocks. Predecessors are used instead
2276 // of successors for the inverse traversal.
2277 template <> struct GraphTraits<Inverse<VPBlockBase *>> {
2280 
2281  static NodeRef getEntryNode(Inverse<NodeRef> B) { return B.Graph; }
2282 
2284  return N->getPredecessors().begin();
2285  }
2286 
2288  return N->getPredecessors().end();
2289  }
2290 };
2291 
2292 // The following set of template specializations implement GraphTraits to
2293 // treat VPRegionBlock as a graph and recurse inside its nodes. It's important
2294 // to note that the blocks inside the VPRegionBlock are treated as VPBlockBases
2295 // (i.e., no dyn_cast is performed, VPBlockBases specialization is used), so
2296 // there won't be automatic recursion into other VPBlockBases that turn to be
2297 // VPRegionBlocks.
2298 
2299 template <>
2303 
2304  static NodeRef getEntryNode(GraphRef N) { return N->getEntry(); }
2305 
2307  return nodes_iterator::begin(N->getEntry());
2308  }
2309 
2311  // df_iterator::end() returns an empty iterator so the node used doesn't
2312  // matter.
2313  return nodes_iterator::end(N);
2314  }
2315 };
2316 
2317 template <>
2320  using GraphRef = const VPRegionBlock *;
2322 
2323  static NodeRef getEntryNode(GraphRef N) { return N->getEntry(); }
2324 
2326  return nodes_iterator::begin(N->getEntry());
2327  }
2328 
2330  // df_iterator::end() returns an empty iterator so the node used doesn't
2331  // matter.
2332  return nodes_iterator::end(N);
2333  }
2334 };
2335 
2336 template <>
2341 
2343  return N.Graph->getExiting();
2344  }
2345 
2347  return nodes_iterator::begin(N->getExiting());
2348  }
2349 
2351  // df_iterator::end() returns an empty iterator so the node used doesn't
2352  // matter.
2353  return nodes_iterator::end(N);
2354  }
2355 };
2356 
2357 /// Iterator to traverse all successors of a VPBlockBase node. This includes the
2358 /// entry node of VPRegionBlocks. Exit blocks of a region implicitly have their
2359 /// parent region's successors. This ensures all blocks in a region are visited
2360 /// before any blocks in a successor region when doing a reverse post-order
2361 // traversal of the graph.
2362 template <typename BlockPtrTy>
2364  : public iterator_facade_base<VPAllSuccessorsIterator<BlockPtrTy>,
2365  std::forward_iterator_tag, VPBlockBase> {
2366  BlockPtrTy Block;
2367  /// Index of the current successor. For VPBasicBlock nodes, this simply is the
2368  /// index for the successor array. For VPRegionBlock, SuccessorIdx == 0 is
2369  /// used for the region's entry block, and SuccessorIdx - 1 are the indices
2370  /// for the successor array.
2371  size_t SuccessorIdx;
2372 
2373  static BlockPtrTy getBlockWithSuccs(BlockPtrTy Current) {
2374  while (Current && Current->getNumSuccessors() == 0)
2375  Current = Current->getParent();
2376  return Current;
2377  }
2378 
2379  /// Templated helper to dereference successor \p SuccIdx of \p Block. Used by
2380  /// both the const and non-const operator* implementations.
2381  template <typename T1> static T1 deref(T1 Block, unsigned SuccIdx) {
2382  if (auto *R = dyn_cast<VPRegionBlock>(Block)) {
2383  if (SuccIdx == 0)
2384  return R->getEntry();
2385  SuccIdx--;
2386  }
2387 
2388  // For exit blocks, use the next parent region with successors.
2389  return getBlockWithSuccs(Block)->getSuccessors()[SuccIdx];
2390  }
2391 
2392 public:
2393  VPAllSuccessorsIterator(BlockPtrTy Block, size_t Idx = 0)
2394  : Block(Block), SuccessorIdx(Idx) {}
2396  : Block(Other.Block), SuccessorIdx(Other.SuccessorIdx) {}
2397 
2399  Block = R.Block;
2400  SuccessorIdx = R.SuccessorIdx;
2401  return *this;
2402  }
2403 
2404  static VPAllSuccessorsIterator end(BlockPtrTy Block) {
2405  BlockPtrTy ParentWithSuccs = getBlockWithSuccs(Block);
2406  unsigned NumSuccessors = ParentWithSuccs
2407  ? ParentWithSuccs->getNumSuccessors()
2408  : Block->getNumSuccessors();
2409 
2410  if (auto *R = dyn_cast<VPRegionBlock>(Block))
2411  return {R, NumSuccessors + 1};
2412  return {Block, NumSuccessors};
2413  }
2414 
2415  bool operator==(const VPAllSuccessorsIterator &R) const {
2416  return Block == R.Block && SuccessorIdx == R.SuccessorIdx;
2417  }
2418 
2419  const VPBlockBase *operator*() const { return deref(Block, SuccessorIdx); }
2420 
2421  BlockPtrTy operator*() { return deref(Block, SuccessorIdx); }
2422 
2424  SuccessorIdx++;
2425  return *this;
2426  }
2427 
2429  VPAllSuccessorsIterator Orig = *this;
2430  SuccessorIdx++;
2431  return Orig;
2432  }
2433 };
2434 
2435 /// Helper for GraphTraits specialization that traverses through VPRegionBlocks.
2436 template <typename BlockTy> class VPBlockRecursiveTraversalWrapper {
2437  BlockTy Entry;
2438 
2439 public:
2441  BlockTy getEntry() { return Entry; }
2442 };
2443 
2444 /// GraphTraits specialization to recursively traverse VPBlockBase nodes,
2445 /// including traversing through VPRegionBlocks. Exit blocks of a region
2446 /// implicitly have their parent region's successors. This ensures all blocks in
2447 /// a region are visited before any blocks in a successor region when doing a
2448 /// reverse post-order traversal of the graph.
2449 template <>
2453 
2454  static NodeRef
2456  return N.getEntry();
2457  }
2458 
2460  return ChildIteratorType(N);
2461  }
2462 
2464  return ChildIteratorType::end(N);
2465  }
2466 };
2467 
2468 template <>
2470  using NodeRef = const VPBlockBase *;
2472 
2473  static NodeRef
2475  return N.getEntry();
2476  }
2477 
2479  return ChildIteratorType(N);
2480  }
2481 
2483  return ChildIteratorType::end(N);
2484  }
2485 };
2486 
2487 /// VPlan models a candidate for vectorization, encoding various decisions take
2488 /// to produce efficient output IR, including which branches, basic-blocks and
2489 /// output IR instructions to generate, and their cost. VPlan holds a
2490 /// Hierarchical-CFG of VPBasicBlocks and VPRegionBlocks rooted at an Entry
2491 /// VPBlock.
2492 class VPlan {
2493  friend class VPlanPrinter;
2494  friend class VPSlotTracker;
2495 
2496  /// Hold the single entry to the Hierarchical CFG of the VPlan.
2497  VPBlockBase *Entry;
2498 
2499  /// Holds the VFs applicable to this VPlan.
2501 
2502  /// Holds the name of the VPlan, for printing.
2503  std::string Name;
2504 
2505  /// Holds all the external definitions created for this VPlan. External
2506  /// definitions must be immutable and hold a pointer to their underlying IR.
2507  DenseMap<Value *, VPValue *> VPExternalDefs;
2508 
2509  /// Represents the trip count of the original loop, for folding
2510  /// the tail.
2511  VPValue *TripCount = nullptr;
2512 
2513  /// Represents the backedge taken count of the original loop, for folding
2514  /// the tail. It equals TripCount - 1.
2515  VPValue *BackedgeTakenCount = nullptr;
2516 
2517  /// Represents the vector trip count.
2518  VPValue VectorTripCount;
2519 
2520  /// Holds a mapping between Values and their corresponding VPValue inside
2521  /// VPlan.
2522  Value2VPValueTy Value2VPValue;
2523 
2524  /// Contains all VPValues that been allocated by addVPValue directly and need
2525  /// to be free when the plan's destructor is called.
2526  SmallVector<VPValue *, 16> VPValuesToFree;
2527 
2528  /// Indicates whether it is safe use the Value2VPValue mapping or if the
2529  /// mapping cannot be used any longer, because it is stale.
2530  bool Value2VPValueEnabled = true;
2531 
2532  /// Values used outside the plan.
2534 
2535 public:
2536  VPlan(VPBlockBase *Entry = nullptr) : Entry(Entry) {
2537  if (Entry)
2538  Entry->setPlan(this);
2539  }
2540 
2542  clearLiveOuts();
2543 
2544  if (Entry) {
2545  VPValue DummyValue;
2546  for (VPBlockBase *Block : depth_first(Entry))
2547  Block->dropAllReferences(&DummyValue);
2548 
2549  VPBlockBase::deleteCFG(Entry);
2550  }
2551  for (VPValue *VPV : VPValuesToFree)
2552  delete VPV;
2553  if (TripCount)
2554  delete TripCount;
2555  if (BackedgeTakenCount)
2556  delete BackedgeTakenCount;
2557  for (auto &P : VPExternalDefs)
2558  delete P.second;
2559  }
2560 
2561  /// Prepare the plan for execution, setting up the required live-in values.
2562  void prepareToExecute(Value *TripCount, Value *VectorTripCount,
2563  Value *CanonicalIVStartValue, VPTransformState &State,
2564  bool IsEpilogueVectorization);
2565 
2566  /// Generate the IR code for this VPlan.
2567  void execute(VPTransformState *State);
2568 
2569  VPBlockBase *getEntry() { return Entry; }
2570  const VPBlockBase *getEntry() const { return Entry; }
2571 
2573  Entry = Block;
2574  Block->setPlan(this);
2575  return Entry;
2576  }
2577 
2578  /// The trip count of the original loop.
2580  if (!TripCount)
2581  TripCount = new VPValue();
2582  return TripCount;
2583  }
2584 
2585  /// The backedge taken count of the original loop.
2587  if (!BackedgeTakenCount)
2588  BackedgeTakenCount = new VPValue();
2589  return BackedgeTakenCount;
2590  }
2591 
2592  /// The vector trip count.
2593  VPValue &getVectorTripCount() { return VectorTripCount; }
2594 
2595  /// Mark the plan to indicate that using Value2VPValue is not safe any
2596  /// longer, because it may be stale.
2597  void disableValue2VPValue() { Value2VPValueEnabled = false; }
2598 
2599  void addVF(ElementCount VF) { VFs.insert(VF); }
2600 
2601  bool hasVF(ElementCount VF) { return VFs.count(VF); }
2602 
2603  const std::string &getName() const { return Name; }
2604 
2605  void setName(const Twine &newName) { Name = newName.str(); }
2606 
2607  /// Get the existing or add a new external definition for \p V.
2609  auto I = VPExternalDefs.insert({V, nullptr});
2610  if (I.second)
2611  I.first->second = new VPValue(V);
2612  return I.first->second;
2613  }
2614 
2615  void addVPValue(Value *V) {
2616  assert(Value2VPValueEnabled &&
2617  "IR value to VPValue mapping may be out of date!");
2618  assert(V && "Trying to add a null Value to VPlan");
2619  assert(!Value2VPValue.count(V) && "Value already exists in VPlan");
2620  VPValue *VPV = new VPValue(V);
2621  Value2VPValue[V] = VPV;
2622  VPValuesToFree.push_back(VPV);
2623  }
2624 
2625  void addVPValue(Value *V, VPValue *VPV) {
2626  assert(Value2VPValueEnabled && "Value2VPValue mapping may be out of date!");
2627  assert(V && "Trying to add a null Value to VPlan");
2628  assert(!Value2VPValue.count(V) && "Value already exists in VPlan");
2629  Value2VPValue[V] = VPV;
2630  }
2631 
2632  /// Returns the VPValue for \p V. \p OverrideAllowed can be used to disable
2633  /// checking whether it is safe to query VPValues using IR Values.
2634  VPValue *getVPValue(Value *V, bool OverrideAllowed = false) {
2635  assert((OverrideAllowed || isa<Constant>(V) || Value2VPValueEnabled) &&
2636  "Value2VPValue mapping may be out of date!");
2637  assert(V && "Trying to get the VPValue of a null Value");
2638  assert(Value2VPValue.count(V) && "Value does not exist in VPlan");
2639  return Value2VPValue[V];
2640  }
2641 
2642  /// Gets the VPValue or adds a new one (if none exists yet) for \p V. \p
2643  /// OverrideAllowed can be used to disable checking whether it is safe to
2644  /// query VPValues using IR Values.
2645  VPValue *getOrAddVPValue(Value *V, bool OverrideAllowed = false) {
2646  assert((OverrideAllowed || isa<Constant>(V) || Value2VPValueEnabled) &&
2647  "Value2VPValue mapping may be out of date!");
2648  assert(V && "Trying to get or add the VPValue of a null Value");
2649  if (!Value2VPValue.count(V))
2650  addVPValue(V);
2651  return getVPValue(V);
2652  }
2653 
2655  assert(Value2VPValueEnabled &&
2656  "IR value to VPValue mapping may be out of date!");
2657  Value2VPValue.erase(V);
2658  }
2659 
2660 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2661  /// Print this VPlan to \p O.
2662  void print(raw_ostream &O) const;
2663 
2664  /// Print this VPlan in DOT format to \p O.
2665  void printDOT(raw_ostream &O) const;
2666 
2667  /// Dump the plan to stderr (for debugging).
2668  LLVM_DUMP_METHOD void dump() const;
2669 #endif
2670 
2671  /// Returns a range mapping the values the range \p Operands to their
2672  /// corresponding VPValues.
2675  std::function<VPValue *(Value *)> Fn = [this](Value *Op) {
2676  return getOrAddVPValue(Op);
2677  };
2678  return map_range(Operands, Fn);
2679  }
2680 
2681  /// Returns the VPRegionBlock of the vector loop.
2683  return cast<VPRegionBlock>(getEntry()->getSingleSuccessor());
2684  }
2686  return cast<VPRegionBlock>(getEntry()->getSingleSuccessor());
2687  }
2688 
2689  /// Returns the canonical induction recipe of the vector loop.
2692  if (EntryVPBB->empty()) {
2693  // VPlan native path.
2694  EntryVPBB = cast<VPBasicBlock>(EntryVPBB->getSingleSuccessor());
2695  }
2696  return cast<VPCanonicalIVPHIRecipe>(&*EntryVPBB->begin());
2697  }
2698 
2699  /// Find and return the VPActiveLaneMaskPHIRecipe from the header - there
2700  /// be only one at most. If there isn't one, then return nullptr.
2702 
2703  void addLiveOut(PHINode *PN, VPValue *V);
2704 
2705  void clearLiveOuts() {
2706  for (auto &KV : LiveOuts)
2707  delete KV.second;
2708  LiveOuts.clear();
2709  }
2710 
2712  delete LiveOuts[PN];
2713  LiveOuts.erase(PN);
2714  }
2715 
2717  return LiveOuts;
2718  }
2719 
2720 private:
2721  /// Add to the given dominator tree the header block and every new basic block
2722  /// that was created between it and the latch block, inclusive.
2723  static void updateDominatorTree(DominatorTree *DT, BasicBlock *LoopLatchBB,
2724  BasicBlock *LoopPreHeaderBB,
2725  BasicBlock *LoopExitBB);
2726 };
2727 
2728 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2729 /// VPlanPrinter prints a given VPlan to a given output stream. The printing is
2730 /// indented and follows the dot format.
2732  raw_ostream &OS;
2733  const VPlan &Plan;
2734  unsigned Depth = 0;
2735  unsigned TabWidth = 2;
2736  std::string Indent;
2737  unsigned BID = 0;
2739 
2741 
2742  /// Handle indentation.
2743  void bumpIndent(int b) { Indent = std::string((Depth += b) * TabWidth, ' '); }
2744 
2745  /// Print a given \p Block of the Plan.
2746  void dumpBlock(const VPBlockBase *Block);
2747 
2748  /// Print the information related to the CFG edges going out of a given
2749  /// \p Block, followed by printing the successor blocks themselves.
2750  void dumpEdges(const VPBlockBase *Block);
2751 
2752  /// Print a given \p BasicBlock, including its VPRecipes, followed by printing
2753  /// its successor blocks.
2754  void dumpBasicBlock(const VPBasicBlock *BasicBlock);
2755 
2756  /// Print a given \p Region of the Plan.
2757  void dumpRegion(const VPRegionBlock *Region);
2758 
2759  unsigned getOrCreateBID(const VPBlockBase *Block) {
2760  return BlockID.count(Block) ? BlockID[Block] : BlockID[Block] = BID++;
2761  }
2762 
2763  Twine getOrCreateName(const VPBlockBase *Block);
2764 
2765  Twine getUID(const VPBlockBase *Block);
2766 
2767  /// Print the information related to a CFG edge between two VPBlockBases.
2768  void drawEdge(const VPBlockBase *From, const VPBlockBase *To, bool Hidden,
2769  const Twine &Label);
2770 
2771 public:
2773  : OS(O), Plan(P), SlotTracker(&P) {}
2774 
2775  LLVM_DUMP_METHOD void dump();
2776 };
2777 
2779  const Value *V;
2780 
2781  VPlanIngredient(const Value *V) : V(V) {}
2782 
2783  void print(raw_ostream &O) const;
2784 };
2785 
2787  I.print(OS);
2788  return OS;
2789 }
2790 
2791 inline raw_ostream &operator<<(raw_ostream &OS, const VPlan &Plan) {
2792  Plan.print(OS);
2793  return OS;
2794 }
2795 #endif
2796 
2797 //===----------------------------------------------------------------------===//
2798 // VPlan Utilities
2799 //===----------------------------------------------------------------------===//
2800 
2801 /// Class that provides utilities for VPBlockBases in VPlan.
2803 public:
2804  VPBlockUtils() = delete;
2805 
2806  /// Insert disconnected VPBlockBase \p NewBlock after \p BlockPtr. Add \p
2807  /// NewBlock as successor of \p BlockPtr and \p BlockPtr as predecessor of \p
2808  /// NewBlock, and propagate \p BlockPtr parent to \p NewBlock. \p BlockPtr's
2809  /// successors are moved from \p BlockPtr to \p NewBlock. \p NewBlock must
2810  /// have neither successors nor predecessors.
2811  static void insertBlockAfter(VPBlockBase *NewBlock, VPBlockBase *BlockPtr) {
2812  assert(NewBlock->getSuccessors().empty() &&
2813  NewBlock->getPredecessors().empty() &&
2814  "Can't insert new block with predecessors or successors.");
2815  NewBlock->setParent(BlockPtr->getParent());
2816  SmallVector<VPBlockBase *> Succs(BlockPtr->successors());
2817  for (VPBlockBase *Succ : Succs) {
2818  disconnectBlocks(BlockPtr, Succ);
2819  connectBlocks(NewBlock, Succ);
2820  }
2821  connectBlocks(BlockPtr, NewBlock);
2822  }
2823 
2824  /// Insert disconnected VPBlockBases \p IfTrue and \p IfFalse after \p
2825  /// BlockPtr. Add \p IfTrue and \p IfFalse as succesors of \p BlockPtr and \p
2826  /// BlockPtr as predecessor of \p IfTrue and \p IfFalse. Propagate \p BlockPtr
2827  /// parent to \p IfTrue and \p IfFalse. \p BlockPtr must have no successors
2828  /// and \p IfTrue and \p IfFalse must have neither successors nor
2829  /// predecessors.
2830  static void insertTwoBlocksAfter(VPBlockBase *IfTrue, VPBlockBase *IfFalse,
2831  VPBlockBase *BlockPtr) {
2832  assert(IfTrue->getSuccessors().empty() &&
2833  "Can't insert IfTrue with successors.");
2834  assert(IfFalse->getSuccessors().empty() &&
2835  "Can't insert IfFalse with successors.");
2836  BlockPtr->setTwoSuccessors(IfTrue, IfFalse);
2837  IfTrue->setPredecessors({BlockPtr});
2838  IfFalse->setPredecessors({BlockPtr});
2839  IfTrue->setParent(BlockPtr->getParent());
2840  IfFalse->setParent(BlockPtr->getParent());
2841  }
2842 
2843  /// Connect VPBlockBases \p From and \p To bi-directionally. Append \p To to
2844  /// the successors of \p From and \p From to the predecessors of \p To. Both
2845  /// VPBlockBases must have the same parent, which can be null. Both
2846  /// VPBlockBases can be already connected to other VPBlockBases.
2848  assert((From->getParent() == To->getParent()) &&
2849  "Can't connect two block with different parents");
2850  assert(From->getNumSuccessors() < 2 &&
2851  "Blocks can't have more than two successors.");
2852  From->appendSuccessor(To);
2853  To->appendPredecessor(From);
2854  }
2855 
2856  /// Disconnect VPBlockBases \p From and \p To bi-directionally. Remove \p To
2857  /// from the successors of \p From and \p From from the predecessors of \p To.
2859  assert(To && "Successor to disconnect is null.");
2860  From->removeSuccessor(To);
2861  To->removePredecessor(From);
2862  }
2863 
2864  /// Try to merge \p Block into its single predecessor, if \p Block is a
2865  /// VPBasicBlock and its predecessor has a single successor. Returns a pointer
2866  /// to the predecessor \p Block was merged into or nullptr otherwise.
2868  auto *VPBB = dyn_cast<VPBasicBlock>(Block);
2869  auto *PredVPBB =
2870  dyn_cast_or_null<VPBasicBlock>(Block->getSinglePredecessor());
2871  if (!VPBB || !PredVPBB || PredVPBB->getNumSuccessors() != 1)
2872  return nullptr;
2873 
2874  for (VPRecipeBase &R : make_early_inc_range(*VPBB))
2875  R.moveBefore(*PredVPBB, PredVPBB->end());
2876  VPBlockUtils::disconnectBlocks(PredVPBB, VPBB);
2877  auto *ParentRegion = cast<VPRegionBlock>(Block->getParent());
2878  if (ParentRegion->getExiting() == Block)
2879  ParentRegion->setExiting(PredVPBB);
2880  SmallVector<VPBlockBase *> Successors(Block->successors());
2881  for (auto *Succ : Successors) {
2882  VPBlockUtils::disconnectBlocks(Block, Succ);
2883  VPBlockUtils::connectBlocks(PredVPBB, Succ);
2884  }
2885  delete Block;
2886  return PredVPBB;
2887  }
2888 
2889  /// Return an iterator range over \p Range which only includes \p BlockTy
2890  /// blocks. The accesses are casted to \p BlockTy.
2891  template <typename BlockTy, typename T>
2892  static auto blocksOnly(const T &Range) {
2893  // Create BaseTy with correct const-ness based on BlockTy.
2894  using BaseTy = std::conditional_t<std::is_const<BlockTy>::value,
2895  const VPBlockBase, VPBlockBase>;
2896 
2897  // We need to first create an iterator range over (const) BlocktTy & instead
2898  // of (const) BlockTy * for filter_range to work properly.
2899  auto Mapped =
2900  map_range(Range, [](BaseTy *Block) -> BaseTy & { return *Block; });
2901  auto Filter = make_filter_range(
2902  Mapped, [](BaseTy &Block) { return isa<BlockTy>(&Block); });
2903  return map_range(Filter, [](BaseTy &Block) -> BlockTy * {
2904  return cast<BlockTy>(&Block);
2905  });
2906  }
2907 };
2908 
2911  InterleaveGroupMap;
2912 
2913  /// Type for mapping of instruction based interleave groups to VPInstruction
2914  /// interleave groups
2917 
2918  /// Recursively \p Region and populate VPlan based interleave groups based on
2919  /// \p IAI.
2920  void visitRegion(VPRegionBlock *Region, Old2NewTy &Old2New,
2921  InterleavedAccessInfo &IAI);
2922  /// Recursively traverse \p Block and populate VPlan based interleave groups
2923  /// based on \p IAI.
2924  void visitBlock(VPBlockBase *Block, Old2NewTy &Old2New,
2925  InterleavedAccessInfo &IAI);
2926 
2927 public:
2929 
2932  // Avoid releasing a pointer twice.
2933  for (auto &I : InterleaveGroupMap)
2934  DelSet.insert(I.second);
2935  for (auto *Ptr : DelSet)
2936  delete Ptr;
2937  }
2938 
2939  /// Get the interleave group that \p Instr belongs to.
2940  ///
2941  /// \returns nullptr if doesn't have such group.
2944  return InterleaveGroupMap.lookup(Instr);
2945  }
2946 };
2947 
2948 /// Class that maps (parts of) an existing VPlan to trees of combined
2949 /// VPInstructions.
2950 class VPlanSlp {
2951  enum class OpMode { Failed, Load, Opcode };
2952 
2953  /// A DenseMapInfo implementation for using SmallVector<VPValue *, 4> as
2954  /// DenseMap keys.
2955  struct BundleDenseMapInfo {
2956  static SmallVector<VPValue *, 4> getEmptyKey() {
2957  return {reinterpret_cast<VPValue *>(-1)};
2958  }
2959 
2960  static SmallVector<VPValue *, 4> getTombstoneKey() {
2961  return {reinterpret_cast<VPValue *>(-2)};
2962  }
2963 
2964  static unsigned getHashValue(const SmallVector<VPValue *, 4> &V) {
2965  return static_cast<unsigned>(hash_combine_range(V.begin(), V.end()));
2966  }
2967 
2968  static bool isEqual(const SmallVector<VPValue *, 4> &LHS,
2969  const SmallVector<VPValue *, 4> &RHS) {
2970  return LHS == RHS;
2971  }
2972  };
2973 
2974  /// Mapping of values in the original VPlan to a combined VPInstruction.
2975  DenseMap<SmallVector<VPValue *, 4>, VPInstruction *, BundleDenseMapInfo>
2976  BundleToCombined;
2977 
2979 
2980  /// Basic block to operate on. For now, only instructions in a single BB are
2981  /// considered.
2982  const VPBasicBlock &BB;
2983 
2984  /// Indicates whether we managed to combine all visited instructions or not.
2985  bool CompletelySLP = true;
2986 
2987  /// Width of the widest combined bundle in bits.
2988  unsigned WidestBundleBits = 0;
2989 
2990  using MultiNodeOpTy =
2991  typename std::pair<VPInstruction *, SmallVector<VPValue *, 4>>;
2992 
2993  // Input operand bundles for the current multi node. Each multi node operand
2994  // bundle contains values not matching the multi node's opcode. They will
2995  // be reordered in reorderMultiNodeOps, once we completed building a
2996  // multi node.
2997  SmallVector<MultiNodeOpTy, 4> MultiNodeOps;
2998 
2999  /// Indicates whether we are building a multi node currently.
3000  bool MultiNodeActive = false;
3001 
3002  /// Check if we can vectorize Operands together.
3003  bool areVectorizable(ArrayRef<VPValue *> Operands) const;
3004 
3005  /// Add combined instruction \p New for the bundle \p Operands.
3006  void addCombined(ArrayRef<VPValue *> Operands, VPInstruction *New);
3007 
3008  /// Indicate we hit a bundle we failed to combine. Returns nullptr for now.
3009  VPInstruction *markFailed();
3010 
3011  /// Reorder operands in the multi node to maximize sequential memory access
3012  /// and commutative operations.
3013  SmallVector<MultiNodeOpTy, 4> reorderMultiNodeOps();
3014 
3015  /// Choose the best candidate to use for the lane after \p Last. The set of
3016  /// candidates to choose from are values with an opcode matching \p Last's
3017  /// or loads consecutive to \p Last.
3018  std::pair<OpMode, VPValue *> getBest(OpMode Mode, VPValue *Last,
3019  SmallPtrSetImpl<VPValue *> &Candidates,
3021 
3022 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
3023  /// Print bundle \p Values to dbgs().
3024  void dumpBundle(ArrayRef<VPValue *> Values);
3025 #endif
3026 
3027 public:
3029 
3030  ~VPlanSlp() = default;
3031 
3032  /// Tries to build an SLP tree rooted at \p Operands and returns a
3033  /// VPInstruction combining \p Operands, if they can be combined.
3035 
3036  /// Return the width of the widest combined bundle in bits.
3037  unsigned getWidestBundleBits() const { return WidestBundleBits; }
3038 
3039  /// Return true if all visited instruction can be combined.
3040  bool isCompletelySLP() const { return CompletelySLP; }
3041 };
3042 
3043 namespace vputils {
3044 
3045 /// Returns true if only the first lane of \p Def is used.
3047 
3048 /// Get or create a VPValue that corresponds to the expansion of \p Expr. If \p
3049 /// Expr is a SCEVConstant or SCEVUnknown, return a VPValue wrapping the live-in
3050 /// value. Otherwise return a VPExpandSCEVRecipe to expand \p Expr. If \p Plan's
3051 /// pre-header already contains a recipe expanding \p Expr, return it. If not,
3052 /// create a new one.
3053 VPValue *getOrCreateVPValueForSCEVExpr(VPlan &Plan, const SCEV *Expr,
3054  ScalarEvolution &SE);
3055 
3056 /// Returns true if \p VPV is uniform after vectorization.
3058  if (auto *Def = VPV->getDef()) {
3059  if (auto Rep = dyn_cast<VPReplicateRecipe>(Def))
3060  return Rep->isUniform();
3061  return false;
3062  }
3063  // A value without a def is external to vplan and thus uniform.
3064  return true;
3065 }
3066 } // end namespace vputils
3067 
3068 } // end namespace llvm
3069 
3070 #endif // LLVM_TRANSFORMS_VECTORIZE_VPLAN_H
llvm::VPBasicBlock::getRecipeList
RecipeListTy & getRecipeList()
Returns a reference to the list of recipes.
Definition: VPlan.h:2060
llvm::VPRegionBlock::front
VPBlockBase & front() const
Definition: VPlan.h:2192
llvm::VPInstruction::getOpcode
unsigned getOpcode() const
Definition: VPlan.h:851
i
i
Definition: README.txt:29
llvm::VPWidenGEPRecipe::~VPWidenGEPRecipe
~VPWidenGEPRecipe() override=default
llvm::VFRange::VFRange
VFRange(const ElementCount &Start, const ElementCount &End)
Definition: VPlan.h:94
llvm::VPUser::getVPUserID
VPUserID getVPUserID() const
Definition: VPlanValue.h:252
llvm::VPRecipeBase::moveAfter
void moveAfter(VPRecipeBase *MovePos)
Unlink this recipe from its current VPBasicBlock and insert it into the VPBasicBlock that MovePos liv...
Definition: VPlanRecipes.cpp:178
llvm::VPBlockBase::getSinglePredecessor
VPBlockBase * getSinglePredecessor() const
Definition: VPlan.h:502
llvm::VPBlockUtils::tryToMergeBlockIntoPredecessor
static VPBasicBlock * tryToMergeBlockIntoPredecessor(VPBlockBase *Block)
Try to merge Block into its single predecessor, if Block is a VPBasicBlock and its predecessor has a ...
Definition: VPlan.h:2867
llvm::VPlan::getActiveLaneMaskPhi
VPActiveLaneMaskPHIRecipe * getActiveLaneMaskPhi()
Find and return the VPActiveLaneMaskPHIRecipe from the header - there be only one at most.
Definition: VPlan.cpp:571
llvm::VPExpandSCEVRecipe::getSCEV
const SCEV * getSCEV() const
Definition: VPlan.h:1823
llvm::VPReductionRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlanRecipes.cpp:888
llvm::VPBlockBase::getPlan
VPlan * getPlan()
Definition: VPlan.cpp:134
llvm::VPWidenMemoryInstructionRecipe::getMask
VPValue * getMask() const
Return the mask used by this recipe.
Definition: VPlan.h:1752
llvm::VPAllSuccessorsIterator::operator*
const VPBlockBase * operator*() const
Definition: VPlan.h:2419
llvm::VPWidenIntOrFpInductionRecipe::getStepValue
VPValue * getStepValue()
Returns the step value of the induction.
Definition: VPlan.h:1097
llvm::VPWidenMemoryInstructionRecipe::VPWidenMemoryInstructionRecipe
VPWidenMemoryInstructionRecipe(LoadInst &Load, VPValue *Addr, VPValue *Mask, bool Consecutive, bool Reverse)
Definition: VPlan.h:1722
llvm::VPBlendRecipe::getNumIncomingValues
unsigned getNumIncomingValues() const
Return the number of incoming values, taking into account that a single incoming value has no mask.
Definition: VPlan.h:1388
llvm::VPlanIngredient::print
void print(raw_ostream &O) const
Definition: VPlan.cpp:958
llvm::Loop::isLoopInvariant
bool isLoopInvariant(const Value *V) const
Return true if the specified value is loop invariant.
Definition: LoopInfo.cpp:60
llvm::VPlanSlp
Class that maps (parts of) an existing VPlan to trees of combined VPInstructions.
Definition: VPlan.h:2950
llvm::VPBlockRecursiveTraversalWrapper::getEntry
BlockTy getEntry()
Definition: VPlan.h:2441
llvm::VPWidenMemoryInstructionRecipe::onlyFirstLaneUsed
bool onlyFirstLaneUsed(const VPValue *Op) const override
Returns true if the recipe only uses the first lane of operand Op.
Definition: VPlan.h:1783
llvm::GraphTraits< const VPBlockBase * >::getEntryNode
static NodeRef getEntryNode(NodeRef N)
Definition: VPlan.h:2264
llvm::VPWidenSelectRecipe::~VPWidenSelectRecipe
~VPWidenSelectRecipe() override=default
llvm::VPWidenSelectRecipe::classof
static bool classof(const VPDef *D)
Method to support type inquiry through isa, cast, and dyn_cast.
Definition: VPlan.h:999
llvm::VPLane::VPLane
VPLane(unsigned Lane, Kind LaneKind)
Definition: VPlan.h:134
LLVM_DUMP_METHOD
#define LLVM_DUMP_METHOD
Mark debug helper function definitions like dump() that should not be stripped from debug builds.
Definition: Compiler.h:492
llvm::VPBlendRecipe::getIncomingValue
VPValue * getIncomingValue(unsigned Idx) const
Return incoming value number Idx.
Definition: VPlan.h:1391
llvm
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
llvm::SmallVectorImpl::erase
iterator erase(const_iterator CI)
Definition: SmallVector.h:723
llvm::tgtok::Def
@ Def
Definition: TGLexer.h:50
FMF.h
llvm::VPRecipeBase::getUnderlyingInstr
Instruction * getUnderlyingInstr()
Returns the underlying instruction, if the recipe is a VPValue or nullptr otherwise.
Definition: VPlan.h:720
llvm::VPLiveOut::usesScalars
bool usesScalars(const VPValue *Op) const override
Returns true if the VPLiveOut uses scalars of operand Op.
Definition: VPlan.h:649
llvm::VPWidenCallRecipe::~VPWidenCallRecipe
~VPWidenCallRecipe() override=default
llvm::GraphTraits< VPBlockBase * >::getEntryNode
static NodeRef getEntryNode(NodeRef N)
Definition: VPlan.h:2249
llvm::VPlanIngredient::VPlanIngredient
VPlanIngredient(const Value *V)
Definition: VPlan.h:2781
llvm::VPBlockBase::dropAllReferences
virtual void dropAllReferences(VPValue *NewValue)=0
Replace all operands of VPUsers in the block with NewValue and also replaces all uses of VPValues def...
llvm::VPCanonicalIVPHIRecipe
Canonical scalar induction phi of the vector loop.
Definition: VPlan.h:1830
llvm::VPWidenPHIRecipe::VPWidenPHIRecipe
VPWidenPHIRecipe(PHINode *Phi, VPValue *Start=nullptr)
Create a new VPWidenPHIRecipe for Phi with start value Start.
Definition: VPlan.h:1237
llvm::make_range
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
Definition: iterator_range.h:53
llvm::VPTransformState::CFG
struct llvm::VPTransformState::CFGState CFG
llvm::VPRecipeBase::eraseFromParent
iplist< VPRecipeBase >::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
Definition: VPlanRecipes.cpp:173
llvm::VPLane::getFirstLane
static VPLane getFirstLane()
Definition: VPlan.h:136
llvm::VPActiveLaneMaskPHIRecipe::VPActiveLaneMaskPHIRecipe
VPActiveLaneMaskPHIRecipe(VPValue *StartMask, DebugLoc DL)
Definition: VPlan.h:1882
Optional.h
llvm::VPRegionBlock::execute
void execute(VPTransformState *State) override
The method which generates the output IR instructions that correspond to this VPRegionBlock,...
Definition: VPlan.cpp:506
llvm::VPWidenCanonicalIVRecipe::execute
void execute(VPTransformState &State) override
Generate a canonical vector induction variable of the vector loop, with start = {<Part*VF,...
Definition: VPlanRecipes.cpp:1087
llvm::VPPredInstPHIRecipe::execute
void execute(VPTransformState &State) override
Generates phi nodes for live-outs as needed to retain SSA form.
Definition: VPlanRecipes.cpp:960
llvm::VPDef::VPValue
friend class VPValue
Definition: VPlanValue.h:316
llvm::VPIteration::VPIteration
VPIteration(unsigned Part, const VPLane &Lane)
Definition: VPlan.h:198
llvm::VPBlendRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlanRecipes.cpp:868
llvm::VPBlockBase::getEnclosingBlockWithPredecessors
VPBlockBase * getEnclosingBlockWithPredecessors()
Definition: VPlan.cpp:182
llvm::VPAllSuccessorsIterator::operator*
BlockPtrTy operator*()
Definition: VPlan.h:2421
llvm::VPInstruction::classof
static bool classof(const VPUser *U)
Extra classof implementations to allow directly casting from VPUser -> VPInstruction.
Definition: VPlan.h:843
llvm::VPBasicBlock::back
const VPRecipeBase & back() const
Definition: VPlan.h:2056
llvm::VPLiveOut::getPhi
PHINode * getPhi() const
Definition: VPlan.h:655
llvm::VPBlockBase::isLegalToHoistInto
bool isLegalToHoistInto()
Return true if it is legal to hoist instructions into this block.
Definition: VPlan.h:593
llvm::VPBlockBase::getName
const std::string & getName() const
Definition: VPlan.h:452
llvm::VPlan::disableValue2VPValue
void disableValue2VPValue()
Mark the plan to indicate that using Value2VPValue is not safe any longer, because it may be stale.
Definition: VPlan.h:2597
llvm::VPTransformState::Builder
IRBuilderBase & Builder
Hold a reference to the IRBuilder used to generate output IR code.
Definition: VPlan.h:359
llvm::VPlan::getOrAddVPValue
VPValue * getOrAddVPValue(Value *V, bool OverrideAllowed=false)
Gets the VPValue or adds a new one (if none exists yet) for V.
Definition: VPlan.h:2645
llvm::ElementCount
Definition: TypeSize.h:404
llvm::VPWidenCanonicalIVRecipe::VPWidenCanonicalIVRecipe
VPWidenCanonicalIVRecipe(VPCanonicalIVPHIRecipe *CanonicalIV)
Definition: VPlan.h:1913
llvm::VPBasicBlock
VPBasicBlock serves as the leaf of the Hierarchical Control-Flow Graph.
Definition: VPlan.h:2013
llvm::VPScalarIVStepsRecipe::classof
static bool classof(const VPUser *U)
Extra classof implementations to allow directly casting from VPUser -> VPScalarIVStepsRecipe.
Definition: VPlan.h:1977
llvm::VPInstruction::FirstOrderRecurrenceSplice
@ FirstOrderRecurrenceSplice
Definition: VPlan.h:781
llvm::Loop
Represents a single loop in the control flow graph.
Definition: LoopInfo.h:546
llvm::DenseMapBase< DenseMap< KeyT, ValueT, DenseMapInfo< KeyT >, llvm::detail::DenseMapPair< KeyT, ValueT > >, KeyT, ValueT, DenseMapInfo< KeyT >, llvm::detail::DenseMapPair< KeyT, ValueT > >::lookup
ValueT lookup(const_arg_type_t< KeyT > Val) const
lookup - Return the entry for the specified key, or a default constructed value if no such entry exis...
Definition: DenseMap.h:197
llvm::VPBranchOnMaskRecipe::usesScalars
bool usesScalars(const VPValue *Op) const override
Returns true if the recipe uses scalars of operand Op.
Definition: VPlan.h:1653
P
This currently compiles esp xmm0 movsd esp eax eax esp ret We should use not the dag combiner This is because dagcombine2 needs to be able to see through the X86ISD::Wrapper which DAGCombine can t really do The code for turning x load into a single vector load is target independent and should be moved to the dag combiner The code for turning x load into a vector load can only handle a direct load from a global or a direct load from the stack It should be generalized to handle any load from P
Definition: README-SSE.txt:411
llvm::VPWidenPHIRecipe
A recipe for handling header phis that are widened in the vector loop.
Definition: VPlan.h:1231
BlockTy
std::pair< BasicBlock *, unsigned > BlockTy
A pair of (basic block, score).
Definition: HotColdSplitting.cpp:398
llvm::VPHeaderPHIRecipe
A pure virtual base class for all recipes modeling header phis, including phis for first order recurr...
Definition: VPlan.h:1132
llvm::VPWidenIntOrFpInductionRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlanRecipes.cpp:690
llvm::VPValue::getLiveInIRValue
Value * getLiveInIRValue()
Returns the underlying IR value, if this VPValue is defined outside the scope of VPlan.
Definition: VPlanValue.h:191
llvm::Type::getScalarType
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
Definition: Type.h:314
llvm::VPlanPrinter::dump
LLVM_DUMP_METHOD void dump()
Definition: VPlan.cpp:849
llvm::VPPredInstPHIRecipe::classof
static bool classof(const VPDef *D)
Method to support type inquiry through isa, cast, and dyn_cast.
Definition: VPlan.h:1675
llvm::SmallVector
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1181
llvm::VPCanonicalIVPHIRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlanRecipes.cpp:1042
llvm::VPWidenIntOrFpInductionRecipe::~VPWidenIntOrFpInductionRecipe
~VPWidenIntOrFpInductionRecipe() override=default
llvm::VPInterleaveRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: LoopVectorize.cpp:9303
ilist.h
llvm::VPInstruction::CanonicalIVIncrement
@ CanonicalIVIncrement
Definition: VPlan.h:789
llvm::VPDef::getVPSingleValue
VPValue * getVPSingleValue()
Returns the only VPValue defined by the VPDef.
Definition: VPlanValue.h:393
llvm::VPBasicBlock::end
const_iterator end() const
Definition: VPlan.h:2045
llvm::VPTransformState::get
Value * get(VPValue *Def, unsigned Part)
Get the generated Value for a given VPValue and a given Part.
Definition: LoopVectorize.cpp:9881
llvm::VPlan::getOrAddExternalDef
VPValue * getOrAddExternalDef(Value *V)
Get the existing or add a new external definition for V.
Definition: VPlan.h:2608
llvm::enumerate
detail::enumerator< R > enumerate(R &&TheRange)
Given an input range, returns a new range whose values are are pair (A,B) such that A is the 0-based ...
Definition: STLExtras.h:2068
llvm::VPRegionBlock::dropAllReferences
void dropAllReferences(VPValue *NewValue) override
Replace all operands of VPUsers in the block with NewValue and also replaces all uses of VPValues def...
Definition: VPlan.cpp:499
llvm::ilist_node_with_parent
An ilist node that can access its parent list.
Definition: ilist_node.h:257
llvm::VPInterleavedAccessInfo
Definition: VPlan.h:2909
llvm::VPBranchOnMaskRecipe::execute
void execute(VPTransformState &State) override
Generate the extraction of the appropriate bit from the block mask and the conditional branch.
Definition: VPlanRecipes.cpp:934
llvm::VPBlockBase::VPBlockBase
VPBlockBase(const unsigned char SC, const std::string &N)
Definition: VPlan.h:438
llvm::GraphTraits< VPRegionBlock * >::nodes_begin
static nodes_iterator nodes_begin(GraphRef N)
Definition: VPlan.h:2306
llvm::VPLane::Kind::First
@ First
For First, Lane is the index into the first N elements of a fixed-vector <N x <ElTy>> or a scalable v...
llvm::TargetTransformInfo
This pass provides access to the codegen interfaces that are needed for IR-level transformations.
Definition: TargetTransformInfo.h:172
llvm::VPInterleaveRecipe::VPInterleaveRecipe
VPInterleaveRecipe(const InterleaveGroup< Instruction > *IG, VPValue *Addr, ArrayRef< VPValue * > StoredValues, VPValue *Mask)
Definition: VPlan.h:1426
llvm::VPInstruction::ActiveLaneMask
@ ActiveLaneMask
Definition: VPlan.h:788
llvm::VPLane::Kind::ScalableLast
@ ScalableLast
For ScalableLast, Lane is the offset from the start of the last N-element subvector in a scalable vec...
llvm::VPWidenPointerInductionRecipe::onlyScalarsGenerated
bool onlyScalarsGenerated(ElementCount VF)
Returns true if only scalar values will be generated.
Definition: VPlanRecipes.cpp:1050
llvm::GraphTraits< const VPBlockBase * >::child_end
static ChildIteratorType child_end(NodeRef N)
Definition: VPlan.h:2270
MapVector.h
llvm::SmallDenseMap
Definition: DenseMap.h:880
llvm::ScalarEvolution
The main scalar evolution driver.
Definition: ScalarEvolution.h:449
llvm::GraphTraits< VPBlockRecursiveTraversalWrapper< VPBlockBase * > >::child_begin
static ChildIteratorType child_begin(NodeRef N)
Definition: VPlan.h:2459
llvm::VPInstruction::setFastMathFlags
void setFastMathFlags(FastMathFlags FMFNew)
Set the fast-math flags.
Definition: VPlanRecipes.cpp:427
llvm::VPReductionPHIRecipe
A recipe for handling reduction phis.
Definition: VPlan.h:1309
llvm::VPWidenIntOrFpInductionRecipe::needsVectorIV
bool needsVectorIV() const
Returns true if a vector phi needs to be created for the induction.
Definition: VPlan.h:1125
llvm::VPTransformState::set
void set(VPValue *Def, Value *V, unsigned Part)
Set the generated Value for a given VPValue and a given Part.
Definition: VPlan.h:264
llvm::DominatorTree
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
Definition: Dominators.h:166
llvm::VPTransformState::set
void set(VPValue *Def, Value *V, const VPIteration &Instance)
Set the generated scalar V for Def and the given Instance.
Definition: VPlan.h:280
llvm::VPValue::VPVWidenCanonicalIVSC
@ VPVWidenCanonicalIVSC
Definition: VPlanValue.h:99
llvm::iplist_impl< simple_ilist< VPRecipeBase, Options... >, ilist_traits< VPRecipeBase > >::const_reverse_iterator
base_list_type::const_reverse_iterator const_reverse_iterator
Definition: ilist.h:182
llvm::VPInterleaveRecipe::~VPInterleaveRecipe
~VPInterleaveRecipe() override=default
llvm::VPRegionBlock::getPreheaderVPBB
VPBasicBlock * getPreheaderVPBB()
Returns the pre-header VPBasicBlock of the loop region.
Definition: VPlan.h:2207
llvm::VPRecipeBase::getUnderlyingInstr
const Instruction * getUnderlyingInstr() const
Definition: VPlan.h:723
llvm::DenseMapBase::erase
bool erase(const KeyT &Val)
Definition: DenseMap.h:302
llvm::VPTransformState::CFGState::getPreheaderBBFor
BasicBlock * getPreheaderBBFor(VPRecipeBase *R)
Returns the BasicBlock* mapped to the pre-header of the loop region containing R.
Definition: VPlan.cpp:225
llvm::VPBlendRecipe::execute
void execute(VPTransformState &State) override
Generate the phi/select nodes.
Definition: VPlanRecipes.cpp:828
llvm::VPReductionRecipe::VPReductionRecipe
VPReductionRecipe(const RecurrenceDescriptor *R, Instruction *I, VPValue *ChainOp, VPValue *VecOp, VPValue *CondOp, const TargetTransformInfo *TTI)
Definition: VPlan.h:1506
llvm::Depth
@ Depth
Definition: SIMachineScheduler.h:36
llvm::VPBlockBase::~VPBlockBase
virtual ~VPBlockBase()=default
llvm::VPPredInstPHIRecipe
VPPredInstPHIRecipe is a recipe for generating the phi nodes needed when control converges back from ...
Definition: VPlan.h:1665
llvm::VPBasicBlock::dropAllReferences
void dropAllReferences(VPValue *NewValue) override
Replace all operands of VPUsers in the block with NewValue and also replaces all uses of VPValues def...
Definition: VPlan.cpp:382
llvm::VPLane::getKind
Kind getKind() const
Returns the Kind of lane offset.
Definition: VPlan.h:162
llvm::VPlan::getCanonicalIV
VPCanonicalIVPHIRecipe * getCanonicalIV()
Returns the canonical induction recipe of the vector loop.
Definition: VPlan.h:2690
llvm::Type
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
DenseMap.h
llvm::VPWidenIntOrFpInductionRecipe::getTruncInst
TruncInst * getTruncInst()
Returns the first defined value as TruncInst, if it is one or nullptr otherwise.
Definition: VPlan.h:1102
llvm::VPIteration::isFirstIteration
bool isFirstIteration() const
Definition: VPlan.h:200
llvm::VPlanPtr
std::unique_ptr< VPlan > VPlanPtr
Definition: VPlan.h:103
llvm::sys::path::end
const_iterator end(StringRef path)
Get end iterator over path.
Definition: Path.cpp:235
llvm::sys::path::begin
const_iterator begin(StringRef path, Style style=Style::native)
Get begin iterator over path.
Definition: Path.cpp:226
llvm::GraphTraits< Inverse< VPRegionBlock * > >::getEntryNode
static NodeRef getEntryNode(Inverse< GraphRef > N)
Definition: VPlan.h:2342
llvm::VPBlockBase::getHierarchicalPredecessors
const VPBlocksTy & getHierarchicalPredecessors()
Definition: VPlan.h:542
llvm::VPBasicBlock::getTerminator
VPRecipeBase * getTerminator()
If the block has multiple successors, return the branch recipe terminating the block.
Definition: VPlan.cpp:456
T1
#define T1
Definition: Mips16ISelLowering.cpp:340
llvm::VPInterleaveRecipe::classof
static bool classof(const VPDef *D)
Method to support type inquiry through isa, cast, and dyn_cast.
Definition: VPlan.h:1446
llvm::VPBlockUtils::insertTwoBlocksAfter
static void insertTwoBlocksAfter(VPBlockBase *IfTrue, VPBlockBase *IfFalse, VPBlockBase *BlockPtr)
Insert disconnected VPBlockBases IfTrue and IfFalse after BlockPtr.
Definition: VPlan.h:2830
llvm::VPWidenIntOrFpInductionRecipe::getScalarType
const Type * getScalarType() const
Returns the scalar type of the induction.
Definition: VPlan.h:1119
llvm::VPBlockUtils
Class that provides utilities for VPBlockBases in VPlan.
Definition: VPlan.h:2802
llvm::Optional
Definition: APInt.h:33
llvm::VPUser::op_begin
operand_iterator op_begin()
Definition: VPlanValue.h:281
llvm::VPInstruction::CanonicalIVIncrementNUW
@ CanonicalIVIncrementNUW
Definition: VPlan.h:790
llvm::VPScalarIVStepsRecipe::execute
void execute(VPTransformState &State) override
Generate the scalarized versions of the phi node as needed by their users.
Definition: LoopVectorize.cpp:9530
llvm::DenseMapBase::count
size_type count(const_arg_type_t< KeyT > Val) const
Return 1 if the specified key is in the map, 0 otherwise.
Definition: DenseMap.h:145
llvm::VPlan::getEntry
const VPBlockBase * getEntry() const
Definition: VPlan.h:2570
llvm::VPWidenIntOrFpInductionRecipe::VPWidenIntOrFpInductionRecipe
VPWidenIntOrFpInductionRecipe(PHINode *IV, VPValue *Start, VPValue *Step, const InductionDescriptor &IndDesc, TruncInst *Trunc, bool NeedsVectorIV)
Definition: VPlan.h:1068
llvm::VPWidenMemoryInstructionRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlanRecipes.cpp:1015
llvm::VPWidenPHIRecipe::~VPWidenPHIRecipe
~VPWidenPHIRecipe() override=default
llvm::mapped_iterator
Definition: STLExtras.h:286
llvm::MapVector
This class implements a map that also provides access to all stored values in a deterministic order.
Definition: MapVector.h:37
llvm::SmallPtrSet
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
Definition: SmallPtrSet.h:450
llvm::VPReplicateRecipe::classof
static bool classof(const VPDef *D)
Method to support type inquiry through isa, cast, and dyn_cast.
Definition: VPlan.h:1572
llvm::VPWidenCallRecipe
A recipe for widening Call instructions.
Definition: VPlan.h:952
llvm::VPInstruction::BranchOnCond
@ BranchOnCond
Definition: VPlan.h:796
llvm::VPPredInstPHIRecipe::usesScalars
bool usesScalars(const VPValue *Op) const override
Returns true if the recipe uses scalars of operand Op.
Definition: VPlan.h:1689
llvm::VPBasicBlock::isExiting
bool isExiting() const
Returns true if the block is exiting it's parent region.
Definition: VPlan.cpp:468
llvm::VPValue::VPVInstructionSC
@ VPVInstructionSC
Definition: VPlanValue.h:93
llvm::VPWidenMemoryInstructionRecipe::VPWidenMemoryInstructionRecipe
VPWidenMemoryInstructionRecipe(StoreInst &Store, VPValue *Addr, VPValue *StoredValue, VPValue *Mask, bool Consecutive, bool Reverse)
Definition: VPlan.h:1731
llvm::MipsISD::Ret
@ Ret
Definition: MipsISelLowering.h:119
llvm::VPReplicateRecipe::onlyFirstLaneUsed
bool onlyFirstLaneUsed(const VPValue *Op) const override
Returns true if the recipe only uses the first lane of operand Op.
Definition: VPlan.h:1600
llvm::VPUser::operands
operand_range operands()
Definition: VPlanValue.h:285
llvm::VPUser::VPUserID
VPUserID
Subclass identifier (for isa/dyn_cast).
Definition: VPlanValue.h:213
llvm::VPValue::VPVWidenPointerInductionSC
@ VPVWidenPointerInductionSC
Definition: VPlanValue.h:112
llvm::VPBasicBlock::execute
void execute(VPTransformState *State) override
The method which generates the output IR instructions that correspond to this VPBasicBlock,...
Definition: VPlan.cpp:315
RHS
Value * RHS
Definition: X86PartialReduction.cpp:76
llvm::VPValue::VPVReductionSC
@ VPVReductionSC
Definition: VPlanValue.h:95
llvm::VPScalarIVStepsRecipe::classof
static bool classof(const VPRecipeBase *R)
Definition: VPlan.h:1981
llvm::isPowerOf2_32
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition: MathExtras.h:458
llvm::VPRegionBlock::getEntry
const VPBlockBase * getEntry() const
Definition: VPlan.h:2176
llvm::VPAllSuccessorsIterator
Iterator to traverse all successors of a VPBlockBase node.
Definition: VPlan.h:2363
llvm::FastMathFlags
Convenience struct for specifying and reasoning about fast-math flags.
Definition: FMF.h:21
llvm::VPTransformState::hasScalarValue
bool hasScalarValue(VPValue *Def, VPIteration Instance)
Definition: VPlan.h:253
llvm::SPII::Load
@ Load
Definition: SparcInstrInfo.h:32
llvm::VPInstruction::mayWriteToMemory
bool mayWriteToMemory() const
Return true if this instruction may modify memory.
Definition: VPlan.h:868
VPlanValue.h
llvm::VPReplicateRecipe::isPacked
bool isPacked() const
Definition: VPlan.h:1595
llvm::InterleaveGroup
The group of interleaved loads/stores sharing the same stride and close to each other.
Definition: VectorUtils.h:284
llvm::LinearPolySize::isScalable
bool isScalable() const
Returns whether the size is scaled by a runtime quantity (vscale).
Definition: TypeSize.h:298
llvm::GraphTraits< VPBlockRecursiveTraversalWrapper< const VPBlockBase * > >::getEntryNode
static NodeRef getEntryNode(VPBlockRecursiveTraversalWrapper< const VPBlockBase * > N)
Definition: VPlan.h:2474
llvm::VPInstruction::SLPLoad
@ SLPLoad
Definition: VPlan.h:786
llvm::VPBasicBlock::getEnclosingLoopRegion
VPRegionBlock * getEnclosingLoopRegion()
Definition: VPlan.cpp:417
llvm::iplist
An intrusive list with ownership and callbacks specified/controlled by ilist_traits,...
Definition: ilist.h:391
llvm::VPReductionRecipe
A recipe to represent inloop reduction operations, performing a reduction on a vector operand into a ...
Definition: VPlan.h:1499
llvm::VPReductionPHIRecipe::~VPReductionPHIRecipe
~VPReductionPHIRecipe() override=default
llvm::isEqual
bool isEqual(const GCNRPTracker::LiveRegSet &S1, const GCNRPTracker::LiveRegSet &S2)
Definition: GCNRegPressure.cpp:55
llvm::VPScalarIVStepsRecipe::classof
static bool classof(const VPDef *D)
Method to support type inquiry through isa, cast, and dyn_cast.
Definition: VPlan.h:1972
llvm::VPScalarIVStepsRecipe::isCanonical
bool isCanonical() const
Returns true if the induction is canonical, i.e.
Definition: VPlanRecipes.cpp:716
llvm::VPValue::users
user_range users()
Definition: VPlanValue.h:166
DepthFirstIterator.h
llvm::VPWidenPHIRecipe::execute
void execute(VPTransformState &State) override
Generate the phi/select nodes.
Definition: VPlanRecipes.cpp:1230
llvm::VPScalarIVStepsRecipe::getCanonicalIV
VPCanonicalIVPHIRecipe * getCanonicalIV() const
Definition: VPlanRecipes.cpp:712
llvm::VPInterleaveRecipe::getInterleaveGroup
const InterleaveGroup< Instruction > * getInterleaveGroup()
Definition: VPlan.h:1480
llvm::RISCVFenceField::R
@ R
Definition: RISCVBaseInfo.h:265
llvm::VFRange
A range of powers-of-2 vectorization factors with fixed start and adjustable end.
Definition: VPlan.h:83
llvm::VPTransformState::VPTransformState
VPTransformState(ElementCount VF, unsigned UF, LoopInfo *LI, DominatorTree *DT, IRBuilderBase &Builder, InnerLoopVectorizer *ILV, VPlan *Plan)
Definition: VPlan.h:206
llvm::BasicBlock
LLVM Basic Block Representation.
Definition: BasicBlock.h:55
llvm::VPLiveOut::fixPhi
void fixPhi(VPlan &Plan, VPTransformState &State)
Fixup the wrapped LCSSA phi node in the unique exit block.
Definition: VPlanRecipes.cpp:134
llvm::VPBranchOnMaskRecipe::classof
static bool classof(const VPDef *D)
Method to support type inquiry through isa, cast, and dyn_cast.
Definition: VPlan.h:1624
llvm::VPWidenIntOrFpInductionRecipe::getInductionDescriptor
const InductionDescriptor & getInductionDescriptor() const
Returns the induction descriptor for the recipe.
Definition: VPlan.h:1112
llvm::VPDef
This class augments a recipe with a set of VPValues defined by the recipe.
Definition: VPlanValue.h:315
llvm::VPCanonicalIVPHIRecipe::getScalarType
const Type * getScalarType() const
Returns the scalar type of the induction.
Definition: VPlan.h:1862
llvm::VPWidenPointerInductionRecipe::VPWidenPointerInductionRecipe
VPWidenPointerInductionRecipe(PHINode *Phi, VPValue *Start, VPValue *Step, const InductionDescriptor &IndDesc, bool IsScalarAfterVectorization)
Create a new VPWidenPointerInductionRecipe for Phi with start value Start.
Definition: VPlan.h:1191
llvm::VPRecipeBase::mayWriteToMemory
bool mayWriteToMemory() const
Returns true if the recipe may write to memory.
Definition: VPlanRecipes.cpp:42
llvm::dbgs
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
GraphTraits.h
llvm::VPTransformState::DataState::PerPartOutput
DenseMap< VPValue *, PerPartValuesTy > PerPartOutput
Definition: VPlan.h:227
llvm::VPInterleaveRecipe::execute
void execute(VPTransformState &State) override
Generate the wide load or store, and shuffles.
Definition: LoopVectorize.cpp:9589
llvm::SmallBitVector
This is a 'bitvector' (really, a variable-sized bit array), optimized for the case when the array is ...
Definition: SmallBitVector.h:35
llvm::VPPredInstPHIRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlanRecipes.cpp:1007
llvm::VPReplicateRecipe
VPReplicateRecipe replicates a given instruction producing multiple scalar copies of the original sca...
Definition: VPlan.h:1545
llvm::VPWidenIntOrFpInductionRecipe::getStartValue
const VPValue * getStartValue() const
Definition: VPlan.h:1094
llvm::BitmaskEnumDetail::Mask
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
Definition: BitmaskEnum.h:80
llvm::VPTransformState
VPTransformState holds information passed down when "executing" a VPlan, needed for generating the ou...
Definition: VPlan.h:205
llvm::VPBasicBlock::iterator
RecipeListTy::iterator iterator
Instruction iterators...
Definition: VPlan.h:2034
llvm::VPReductionPHIRecipe::classof
static bool classof(const VPHeaderPHIRecipe *R)
Definition: VPlan.h:1336
LHS
Value * LHS
Definition: X86PartialReduction.cpp:75
llvm::VPBlockBase
VPBlockBase is the building block of the Hierarchical Control-Flow Graph.
Definition: VPlan.h:389
llvm::VPReplicateRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlanRecipes.cpp:909
llvm::VPBasicBlock::appendRecipe
void appendRecipe(VPRecipeBase *Recipe)
Augment the existing recipes of a VPBasicBlock with an additional Recipe as the last recipe.
Definition: VPlan.h:2081
llvm::VPValue::VPVFirstOrderRecurrencePHISC
@ VPVFirstOrderRecurrencePHISC
Definition: VPlanValue.h:109
llvm::VPlan::hasVF
bool hasVF(ElementCount VF)
Definition: VPlan.h:2601
llvm::all_of
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1590
llvm::VPReductionPHIRecipe::isInLoop
bool isInLoop() const
Returns true, if the phi is part of an in-loop reduction.
Definition: VPlan.h:1360
llvm::VPWidenCanonicalIVRecipe::classof
static bool classof(const VPRecipeBase *R)
Definition: VPlan.h:1930
llvm::InductionDescriptor
A struct for saving information about induction variables.
Definition: IVDescriptors.h:306
llvm::VPWidenPHIRecipe::classof
static bool classof(const VPValue *V)
Definition: VPlan.h:1252
llvm::VPTransformState::DataState::PerPartValuesTy
SmallVector< Value *, 2 > PerPartValuesTy
A type for vectorized values in the new loop.
Definition: VPlan.h:225
llvm::VPInstruction::classof
static bool classof(const VPValue *V)
Method to support type inquiry through isa, cast, and dyn_cast.
Definition: VPlan.h:827
llvm::VPLane::mapToCacheIndex
unsigned mapToCacheIndex(const ElementCount &VF) const
Maps the lane to a cache index based on VF.
Definition: VPlan.h:168
llvm::VPWidenPHIRecipe::getIncomingBlock
VPBasicBlock * getIncomingBlock(unsigned I)
Returns the I th incoming VPBasicBlock.
Definition: VPlan.h:1272
llvm::VPInstruction
This is a concrete Recipe that models a single VPlan-level instruction.
Definition: VPlan.h:775
llvm::VPTransformState::ILV
InnerLoopVectorizer * ILV
Hold a pointer to InnerLoopVectorizer to reuse its IR generation methods.
Definition: VPlan.h:367
llvm::VFRange::isEmpty
bool isEmpty() const
Definition: VPlan.h:90
llvm::PPCISD::SC
@ SC
CHAIN = SC CHAIN, Imm128 - System call.
Definition: PPCISelLowering.h:418
llvm::Failed
testing::Matcher< const detail::ErrorHolder & > Failed()
Definition: Error.h:198
llvm::VPlan::print
void print(raw_ostream &O) const
Print this VPlan to O.
Definition: VPlan.cpp:747
llvm::vputils::getOrCreateVPValueForSCEVExpr
VPValue * getOrCreateVPValueForSCEVExpr(VPlan &Plan, const SCEV *Expr, ScalarEvolution &SE)
Get or create a VPValue that corresponds to the expansion of Expr.
Definition: VPlan.cpp:1098
llvm::VPBlockBase::setPredecessors
void setPredecessors(ArrayRef< VPBlockBase * > NewPreds)
Set each VPBasicBlock in NewPreds as predecessor of this VPBlockBase.
Definition: VPlan.h:573
llvm::createStepForVF
Value * createStepForVF(IRBuilderBase &B, Type *Ty, ElementCount VF, int64_t Step)
Return a value for Step multiplied by VF.
Definition: LoopVectorize.cpp:954
llvm::VPValue::VPVPredInstPHI
@ VPVPredInstPHI
Definition: VPlanValue.h:105
llvm::VPScalarIVStepsRecipe::onlyFirstLaneUsed
bool onlyFirstLaneUsed(const VPValue *Op) const override
Returns true if the recipe only uses the first lane of operand Op.
Definition: VPlan.h:2003
llvm::VPInterleavedAccessInfo::getInterleaveGroup
InterleaveGroup< VPInstruction > * getInterleaveGroup(VPInstruction *Instr) const
Get the interleave group that Instr belongs to.
Definition: VPlan.h:2943
llvm::VPIteration::Lane
VPLane Lane
Definition: VPlan.h:192
llvm::VPTransformState::LI
LoopInfo * LI
Hold a pointer to LoopInfo to register new basic blocks in the loop.
Definition: VPlan.h:353
llvm::VPlanSlp::buildGraph
VPInstruction * buildGraph(ArrayRef< VPValue * > Operands)
Tries to build an SLP tree rooted at Operands and returns a VPInstruction combining Operands,...
Definition: VPlanSLP.cpp:358
llvm::VPValue::VPVBlendSC
@ VPVBlendSC
Definition: VPlanValue.h:104
llvm::VPReplicateRecipe::classof
static bool classof(const VPValue *V)
Definition: VPlan.h:1576
llvm::VPLane::isFirstLane
bool isFirstLane() const
Returns true if this is the first lane of the whole vector.
Definition: VPlan.h:165
Twine.h
llvm::VPBlockBase::getSuccessors
VPBlocksTy & getSuccessors()
Definition: VPlan.h:487
llvm::VPBlockUtils::insertBlockAfter
static void insertBlockAfter(VPBlockBase *NewBlock, VPBlockBase *BlockPtr)
Insert disconnected VPBlockBase NewBlock after BlockPtr.
Definition: VPlan.h:2811
llvm::VPWidenPointerInductionRecipe::execute
void execute(VPTransformState &State) override
Generate vector values for the pointer induction.
Definition: LoopVectorize.cpp:9434
llvm::VPBasicBlock::getSublistAccess
static RecipeListTy VPBasicBlock::* getSublistAccess(VPRecipeBase *)
Returns a pointer to a member of the recipe list.
Definition: VPlan.h:2063
llvm::VPWidenMemoryInstructionRecipe::isReverse
bool isReverse() const
Definition: VPlan.h:1771
llvm::VPScalarIVStepsRecipe::VPScalarIVStepsRecipe
VPScalarIVStepsRecipe(Type *Ty, const InductionDescriptor &IndDesc, VPValue *CanonicalIV, VPValue *Start, VPValue *Step, Type *TruncToTy)
Definition: VPlan.h:1962
llvm::GraphTraits< const VPBlockBase * >::ChildIteratorType
SmallVectorImpl< VPBlockBase * >::const_iterator ChildIteratorType
Definition: VPlan.h:2262
llvm::VPlanPrinter::VPlanPrinter
VPlanPrinter(raw_ostream &O, const VPlan &P)
Definition: VPlan.h:2772
llvm::VPlan::getVPValue
VPValue * getVPValue(Value *V, bool OverrideAllowed=false)
Returns the VPValue for V.
Definition: VPlan.h:2634
llvm::GraphTraits< Inverse< VPBlockBase * > >::child_end
static ChildIteratorType child_end(NodeRef N)
Definition: VPlan.h:2287
llvm::VPInstruction::ICmpULE
@ ICmpULE
Definition: VPlan.h:785
b
the resulting code requires compare and branches when and if the revised code is with conditional branches instead of More there is a byte word extend before each where there should be only and the condition codes are not remembered when the same two values are compared twice More LSR enhancements i8 and i32 load store addressing modes are identical int b
Definition: README.txt:418
llvm::VPRecipeBase::moveBefore
void moveBefore(VPBasicBlock &BB, iplist< VPRecipeBase >::iterator I)
Unlink this recipe and insert into BB before I.
Definition: VPlanRecipes.cpp:183
llvm::VPBlockBase::getPredecessors
VPBlocksTy & getPredecessors()
Definition: VPlan.h:492
llvm::VPInterleaveRecipe
VPInterleaveRecipe is a recipe for transforming an interleave group of load or stores into one wide l...
Definition: VPlan.h:1420
IP
Definition: NVPTXLowerArgs.cpp:168
llvm::VPBranchOnMaskRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlan.h:1634
llvm::SPII::Store
@ Store
Definition: SparcInstrInfo.h:33
llvm::VPBlockBase::VPBlockTy
enum { VPBasicBlockSC, VPRegionBlockSC } VPBlockTy
An enumeration for keeping track of the concrete subclass of VPBlockBase that are actually instantiat...
Definition: VPlan.h:446
false
Definition: StackSlotColoring.cpp:141
llvm::dwarf::Index
Index
Definition: Dwarf.h:472
llvm::VPBlockBase::getEnclosingBlockWithSuccessors
VPBlockBase * getEnclosingBlockWithSuccessors()
An Enclosing Block of a block B is any block containing B, including B itself.
Definition: VPlan.cpp:174
llvm::VPCanonicalIVPHIRecipe::classof
static bool classof(const VPHeaderPHIRecipe *D)
Definition: VPlan.h:1845
llvm::VPWidenPHIRecipe::getIncomingValue
VPValue * getIncomingValue(unsigned I)
Returns the I th incoming VPValue.
Definition: VPlan.h:1275
llvm::VPWidenPointerInductionRecipe::classof
static bool classof(const VPRecipeBase *B)
Method to support type inquiry through isa, cast, and dyn_cast.
Definition: VPlan.h:1205
B
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
llvm::VPWidenCanonicalIVRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlanRecipes.cpp:1108
llvm::VPReductionPHIRecipe::classof
static bool classof(const VPValue *V)
Definition: VPlan.h:1339
llvm::VPUser::VPUserID::Recipe
@ Recipe
llvm::GraphTraits< const VPRegionBlock * >::nodes_begin
static nodes_iterator nodes_begin(GraphRef N)
Definition: VPlan.h:2325
llvm::VPHeaderPHIRecipe::getBackedgeRecipe
VPRecipeBase * getBackedgeRecipe()
Returns the backedge value as a recipe.
Definition: VPlan.h:1178
llvm::VPTransformState::DataState::PerPartScalars
DenseMap< VPValue *, ScalarsPerPartValuesTy > PerPartScalars
Definition: VPlan.h:230
llvm::Instruction
Definition: Instruction.h:42
llvm::VPInstruction::VPInstruction
VPInstruction(unsigned Opcode, std::initializer_list< VPValue * > Operands, DebugLoc DL={}, const Twine &Name="")
Definition: VPlan.h:822
llvm::VPReplicateRecipe::~VPReplicateRecipe
~VPReplicateRecipe() override=default
llvm::InterleavedAccessInfo
Drive the analysis of interleaved memory accesses in the loop.
Definition: VectorUtils.h:751
llvm::VPBlockBase::printSuccessors
void printSuccessors(raw_ostream &O, const Twine &Indent) const
Print the successors of this block to O, prefixing all lines with Indent.
Definition: VPlan.cpp:473
llvm::VPReductionRecipe::execute
void execute(VPTransformState &State) override
Generate the reduction in the loop.
Definition: LoopVectorize.cpp:9595
llvm::raw_ostream
This class implements an extremely fast bulk output stream that can only output to a stream.
Definition: raw_ostream.h:52
llvm::VPScalarIVStepsRecipe::getStartValue
VPValue * getStartValue() const
Definition: VPlan.h:1999
llvm::VPBlockUtils::blocksOnly
static auto blocksOnly(const T &Range)
Return an iterator range over Range which only includes BlockTy blocks.
Definition: VPlan.h:2892
llvm::VPValue
Definition: VPlanValue.h:44
llvm::VPRegionBlock::~VPRegionBlock
~VPRegionBlock() override
Definition: VPlan.h:2163
llvm::operator<<
raw_ostream & operator<<(raw_ostream &OS, const APFixedPoint &FX)
Definition: APFixedPoint.h:230
llvm::VPWidenCallRecipe::execute
void execute(VPTransformState &State) override
Produce a widened version of the call instruction.
Definition: VPlanRecipes.cpp:437
llvm::VPlanSlp::~VPlanSlp
~VPlanSlp()=default
llvm::VPlan::clearLiveOuts
void clearLiveOuts()
Definition: VPlan.h:2705
llvm::VPBlockBase::successors
iterator_range< VPBlockBase ** > successors()
Definition: VPlan.h:489
llvm::VPlanSlp::getWidestBundleBits
unsigned getWidestBundleBits() const
Return the width of the widest combined bundle in bits.
Definition: VPlan.h:3037
llvm::VPlan::getName
const std::string & getName() const
Definition: VPlan.h:2603
DebugLoc.h
SmallPtrSet.h
llvm::VPTransformState::CFGState::ExitBB
BasicBlock * ExitBB
The last IR BasicBlock in the output IR.
Definition: VPlan.h:339
llvm::VPWidenCanonicalIVRecipe
A Recipe for widening the canonical induction variable of the vector loop.
Definition: VPlan.h:1911
llvm::VPWidenMemoryInstructionRecipe::getIngredient
Instruction & getIngredient() const
Definition: VPlan.h:1794
llvm::VPAllSuccessorsIterator::operator++
VPAllSuccessorsIterator & operator++()
Definition: VPlan.h:2423
llvm::VPlan::removeLiveOut
void removeLiveOut(PHINode *PN)
Definition: VPlan.h:2711
llvm::InnerLoopVectorizer
InnerLoopVectorizer vectorizes loops which contain only one basic block to a specified vectorization ...
Definition: LoopVectorize.cpp:443
llvm::VPValue::VPVActiveLaneMaskPHISC
@ VPVActiveLaneMaskPHISC
Definition: VPlanValue.h:108
llvm::VPValue::VPVCanonicalIVPHISC
@ VPVCanonicalIVPHISC
Definition: VPlanValue.h:107
llvm::VPBasicBlock::end
iterator end()
Definition: VPlan.h:2044
llvm::MCID::Call
@ Call
Definition: MCInstrDesc.h:155
llvm::InterleaveGroup::getMember
InstTy * getMember(uint32_t Index) const
Get the member with the given index Index.
Definition: VectorUtils.h:679
llvm::VPReductionPHIRecipe::classof
static bool classof(const VPRecipeBase *R)
Method to support type inquiry through isa, cast, and dyn_cast.
Definition: VPlan.h:1333
llvm::GraphTraits< const VPBlockBase * >::child_begin
static ChildIteratorType child_begin(NodeRef N)
Definition: VPlan.h:2266
llvm::ArrayRef::slice
ArrayRef< T > slice(size_t N, size_t M) const
slice(n, m) - Chop off the first N elements of the array, and keep M elements in the array.
Definition: ArrayRef.h:194
llvm::VPWidenCanonicalIVRecipe::~VPWidenCanonicalIVRecipe
~VPWidenCanonicalIVRecipe() override=default
llvm::VPUser::getNumOperands
unsigned getNumOperands() const
Definition: VPlanValue.h:259
llvm::VPRegionBlock::VPRegionBlock
VPRegionBlock(VPBlockBase *Entry, VPBlockBase *Exiting, const std::string &Name="", bool IsReplicator=false)
Definition: VPlan.h:2150
llvm::lltok::Kind
Kind
Definition: LLToken.h:18
llvm::iplist_impl< simple_ilist< VPRecipeBase, Options... >, ilist_traits< VPRecipeBase > >::reverse_iterator
base_list_type::reverse_iterator reverse_iterator
Definition: ilist.h:180
X
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
llvm::VPBasicBlock::empty
bool empty() const
Definition: VPlan.h:2053
llvm::codeview::ClassOptions::Intrinsic
@ Intrinsic
llvm::VPWidenIntOrFpInductionRecipe::VPWidenIntOrFpInductionRecipe
VPWidenIntOrFpInductionRecipe(PHINode *IV, VPValue *Start, VPValue *Step, const InductionDescriptor &IndDesc, bool NeedsVectorIV)
Definition: VPlan.h:1061
llvm::VPCanonicalIVPHIRecipe::~VPCanonicalIVPHIRecipe
~VPCanonicalIVPHIRecipe() override=default
llvm::VPWidenMemoryInstructionRecipe::getAddr
VPValue * getAddr() const
Return the address accessed by this recipe.
Definition: VPlan.h:1746
llvm::VPRegionBlock
VPRegionBlock represents a collection of VPBasicBlocks and VPRegionBlocks which form a Single-Entry-S...
Definition: VPlan.h:2137
llvm::VPWidenMemoryInstructionRecipe::isStore
bool isStore() const
Returns true if this recipe is a store.
Definition: VPlan.h:1758
llvm::VPBlockBase::getPredecessors
const VPBlocksTy & getPredecessors() const
Definition: VPlan.h:491
LoopInfo.h
llvm::VPReductionPHIRecipe::VPReductionPHIRecipe
VPReductionPHIRecipe(PHINode *Phi, const RecurrenceDescriptor &RdxDesc, VPValue &Start, bool IsInLoop=false, bool IsOrdered=false)
Create a new VPReductionPHIRecipe for the reduction Phi described by RdxDesc.
Definition: VPlan.h:1322
llvm::GraphTraits< VPBlockRecursiveTraversalWrapper< const VPBlockBase * > >::child_end
static ChildIteratorType child_end(NodeRef N)
Definition: VPlan.h:2482
llvm::Twine::str
std::string str() const
Return the twine contents as a std::string.
Definition: Twine.cpp:17
Operands
mir Rename Register Operands
Definition: MIRNamerPass.cpp:74
llvm::GraphTraits< Inverse< VPBlockBase * > >::getEntryNode
static NodeRef getEntryNode(Inverse< NodeRef > B)
Definition: VPlan.h:2281
llvm::GraphTraits< VPBlockBase * >::child_begin
static ChildIteratorType child_begin(NodeRef N)
Definition: VPlan.h:2251
llvm::VPInterleaveRecipe::getAddr
VPValue * getAddr() const
Return the address accessed by this recipe.
Definition: VPlan.h:1451
llvm::VPReplicateRecipe::execute
void execute(VPTransformState &State) override
Generate replicas of the desired Ingredient.
Definition: LoopVectorize.cpp:9644
llvm::VPInterleaveRecipe::getStoredValues
ArrayRef< VPValue * > getStoredValues() const
Return the VPValues stored by this interleave group.
Definition: VPlan.h:1464
llvm::VPRecipeBase::VPRecipeBase
VPRecipeBase(const unsigned char SC, ArrayRef< VPValue * > Operands)
Definition: VPlan.h:673
llvm::VPTransformState::setDebugLocFromInst
void setDebugLocFromInst(const Value *V)
Set the debug location in the builder using the debug location in V.
Definition: VPlan.cpp:250
llvm::VPWidenCanonicalIVRecipe::classof
static bool classof(const VPDef *D)
Method to support type inquiry through isa, cast, and dyn_cast.
Definition: VPlan.h:1920
llvm::VPBlockBase::clearPredecessors
void clearPredecessors()
Remove all the predecessor of this block.
Definition: VPlan.h:580
llvm::VPBlockBase::getNumSuccessors
size_t getNumSuccessors() const
Definition: VPlan.h:506
llvm::GraphTraits< VPBlockRecursiveTraversalWrapper< const VPBlockBase * > >::child_begin
static ChildIteratorType child_begin(NodeRef N)
Definition: VPlan.h:2478
llvm::VPLane::getNumCachedLanes
static unsigned getNumCachedLanes(const ElementCount &VF)
Returns the maxmimum number of lanes that we are able to consider caching for VF.
Definition: VPlan.h:181
llvm::VPlan::getVectorLoopRegion
const VPRegionBlock * getVectorLoopRegion() const
Definition: VPlan.h:2685
llvm::VPInstruction::classof
static bool classof(const VPRecipeBase *R)
Definition: VPlan.h:847
llvm::VPValue::VPVWidenSC
@ VPVWidenSC
Definition: VPlanValue.h:97
VectorUtils.h
llvm::VPActiveLaneMaskPHIRecipe::execute
void execute(VPTransformState &State) override
Generate the active lane mask phi of the vector loop.
Definition: VPlanRecipes.cpp:1279
llvm::ClrHandlerType::Filter
@ Filter
llvm::VPInterleaveRecipe::getMask
VPValue * getMask() const
Return the mask used by this recipe.
Definition: VPlan.h:1457
llvm::VPTransformState::CanonicalIV
Value * CanonicalIV
Hold the canonical scalar IV of the vector loop (start=0, step=VF*UF).
Definition: VPlan.h:364
llvm::VPRecipeBase::getParent
const VPBasicBlock * getParent() const
Definition: VPlan.h:683
llvm::SCEV
This class represents an analyzed expression in the program.
Definition: ScalarEvolution.h:75
llvm::VPBlendRecipe::onlyFirstLaneUsed
bool onlyFirstLaneUsed(const VPValue *Op) const override
Returns true if the recipe only uses the first lane of operand Op.
Definition: VPlan.h:1406
llvm::VPBasicBlock::rbegin
const_reverse_iterator rbegin() const
Definition: VPlan.h:2048
llvm::VPlan::setName
void setName(const Twine &newName)
Definition: VPlan.h:2605
llvm::VPRecipeBase::removeFromParent
void removeFromParent()
This method unlinks 'this' from the containing basic block, but does not delete it.
Definition: VPlanRecipes.cpp:167
llvm::VPBlockBase::getHierarchicalSuccessors
const VPBlocksTy & getHierarchicalSuccessors()
Definition: VPlan.h:526
llvm::VPWidenMemoryInstructionRecipe::classof
static bool classof(const VPDef *D)
Method to support type inquiry through isa, cast, and dyn_cast.
Definition: VPlan.h:1741
llvm::VPPredInstPHIRecipe::~VPPredInstPHIRecipe
~VPPredInstPHIRecipe() override=default
llvm::RISCVFenceField::O
@ O
Definition: RISCVBaseInfo.h:264
llvm::VPBranchOnMaskRecipe::VPBranchOnMaskRecipe
VPBranchOnMaskRecipe(VPValue *BlockInMask)
Definition: VPlan.h:1617
llvm::VPRecipeBase::mayHaveSideEffects
bool mayHaveSideEffects() const
Returns true if the recipe may have side-effects.
Definition: VPlanRecipes.cpp:104
llvm::StoreInst
An instruction for storing to memory.
Definition: Instructions.h:298
llvm::VPExpandSCEVRecipe::execute
void execute(VPTransformState &State) override
Generate a canonical vector induction variable of the vector loop, with.
Definition: VPlanRecipes.cpp:1066
llvm::VPLane::getAsRuntimeExpr
Value * getAsRuntimeExpr(IRBuilderBase &Builder, const ElementCount &VF) const
Returns an expression describing the lane index that can be used at runtime.
Definition: VPlan.cpp:62
llvm::VPInstruction::Not
@ Not
Definition: VPlan.h:784
llvm::VPInterleaveRecipe::getNumStoreOperands
unsigned getNumStoreOperands() const
Returns the number of stored operands of this interleave group.
Definition: VPlan.h:1484
llvm::VPReductionRecipe::getCondOp
VPValue * getCondOp() const
The VPValue of the condition for the block.
Definition: VPlan.h:1536
llvm::VPTransformState::reset
void reset(VPValue *Def, Value *V, const VPIteration &Instance)
Reset an existing scalar value for Def and a given Instance.
Definition: VPlan.h:294
llvm::VPInterleaveRecipe::onlyFirstLaneUsed
bool onlyFirstLaneUsed(const VPValue *Op) const override
The recipe only uses the first lane of the address.
Definition: VPlan.h:1489
llvm::VPBlockBase::clearSuccessors
void clearSuccessors()
Remove all the successors of this block.
Definition: VPlan.h:583
llvm::VPBlockBase::getSingleHierarchicalSuccessor
VPBlockBase * getSingleHierarchicalSuccessor()
Definition: VPlan.h:532
llvm::VPBlockBase::getSingleHierarchicalPredecessor
VPBlockBase * getSingleHierarchicalPredecessor()
Definition: VPlan.h:548
llvm::VPWidenGEPRecipe::execute
void execute(VPTransformState &State) override
Generate the gep nodes.
Definition: VPlanRecipes.cpp:739
D
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
llvm::VPTransformState::DT
DominatorTree * DT
Hold a pointer to Dominator Tree to register new basic blocks in the loop.
Definition: VPlan.h:356
llvm::VPWidenIntOrFpInductionRecipe::execute
void execute(VPTransformState &State) override
Generate the vectorized and scalarized versions of the phi node as needed by their users.
Definition: LoopVectorize.cpp:9333
llvm::VPWidenRecipe
VPWidenRecipe is a recipe for producing a copy of vector type its ingredient.
Definition: VPlan.h:924
llvm::find
auto find(R &&Range, const T &Val)
Provide wrappers to std::find which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1610
llvm::VPValue::VPVFirstHeaderPHISC
@ VPVFirstHeaderPHISC
Definition: VPlanValue.h:114
const
aarch64 promote const
Definition: AArch64PromoteConstant.cpp:232
llvm::VPInstruction::execute
void execute(VPTransformState &State) override
Generate the instruction.
Definition: VPlanRecipes.cpp:349
llvm::VPReductionPHIRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlanRecipes.cpp:1220
llvm::InterleaveGroup::getFactor
uint32_t getFactor() const
Definition: VectorUtils.h:625
Addr
uint64_t Addr
Definition: ELFObjHandler.cpp:78
llvm::TruncInst
This class represents a truncation of integer types.
Definition: Instructions.h:4811
llvm::iplist_impl< simple_ilist< VPRecipeBase, Options... >, ilist_traits< VPRecipeBase > >::iterator
base_list_type::iterator iterator
Definition: ilist.h:178
llvm::VPBlockBase::getVPBlockID
unsigned getVPBlockID() const
Definition: VPlan.h:459
llvm::VPBlockBase::dump
LLVM_DUMP_METHOD void dump() const
Dump this VPBlockBase to dbgs().
Definition: VPlan.h:628
llvm::VPValue::VPVReductionPHISC
@ VPVReductionPHISC
Definition: VPlanValue.h:113
llvm::VPLane::getLastLaneForVF
static VPLane getLastLaneForVF(const ElementCount &VF)
Definition: VPlan.h:138
llvm::VPTransformState::UF
unsigned UF
Definition: VPlan.h:214
llvm::DenseMap
Definition: DenseMap.h:714
llvm::VPInstruction::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the VPInstruction to O.
Definition: VPlanRecipes.cpp:363
llvm::VPIteration::VPIteration
VPIteration(unsigned Part, unsigned Lane, VPLane::Kind Kind=VPLane::Kind::First)
Definition: VPlan.h:194
llvm::VPTransformState::DataState
Definition: VPlan.h:221
llvm::VPBasicBlock::size
size_t size() const
Definition: VPlan.h:2052
I
#define I(x, y, z)
Definition: MD5.cpp:58
llvm::VPIteration
VPIteration represents a single point in the iteration space of the output (vectorized and/or unrolle...
Definition: VPlan.h:188
llvm::VPBlockUtils::connectBlocks
static void connectBlocks(VPBlockBase *From, VPBlockBase *To)
Connect VPBlockBases From and To bi-directionally.
Definition: VPlan.h:2847
llvm::GraphTraits< Inverse< VPBlockBase * > >::child_begin
static ChildIteratorType child_begin(NodeRef N)
Definition: VPlan.h:2283
llvm::iterator_facade_base
CRTP base class which implements the entire standard iterator facade in terms of a minimal subset of ...
Definition: iterator.h:80
llvm::VPHeaderPHIRecipe::getStartValue
VPValue * getStartValue()
Returns the start value of the phi, if one is set.
Definition: VPlan.h:1164
llvm::GetElementPtrInst
an instruction for type-safe pointer arithmetic to access elements of arrays and structs
Definition: Instructions.h:929
llvm::VPRegionBlock::getExiting
const VPBlockBase * getExiting() const
Definition: VPlan.h:2194
llvm::make_early_inc_range
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Definition: STLExtras.h:596
llvm::VPHeaderPHIRecipe::classof
static bool classof(const VPValue *V)
Definition: VPlan.h:1149
llvm::VPBlendRecipe::getMask
VPValue * getMask(unsigned Idx) const
Return mask number Idx.
Definition: VPlan.h:1394
llvm::VPTransformState::CFGState::PrevVPBB
VPBasicBlock * PrevVPBB
The previous VPBasicBlock visited. Initially set to null.
Definition: VPlan.h:331
llvm::is_contained
bool is_contained(R &&Range, const E &Element)
Wrapper function around std::find to detect if an element exists in a container.
Definition: STLExtras.h:1673
llvm::VPBlockBase::getNumPredecessors
size_t getNumPredecessors() const
Definition: VPlan.h:507
llvm::VPlan::removeVPValueFor
void removeVPValueFor(Value *V)
Definition: VPlan.h:2654
llvm::VPlan::addVPValue
void addVPValue(Value *V)
Definition: VPlan.h:2615
llvm::VPWidenGEPRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlanRecipes.cpp:813
llvm::VPWidenMemoryInstructionRecipe::getStoredValue
VPValue * getStoredValue() const
Return the address accessed by this recipe.
Definition: VPlan.h:1761
llvm::VPExpandSCEVRecipe::VPExpandSCEVRecipe
VPExpandSCEVRecipe(const SCEV *Expr, ScalarEvolution &SE)
Definition: VPlan.h:1803
llvm::VPTransformState::hasAnyVectorValue
bool hasAnyVectorValue(VPValue *Def) const
Definition: VPlan.h:249
llvm::VPBlockBase::setTwoSuccessors
void setTwoSuccessors(VPBlockBase *IfTrue, VPBlockBase *IfFalse)
Set two given VPBlockBases IfTrue and IfFalse to be the two successors of this VPBlockBase.
Definition: VPlan.h:564
llvm::GraphTraits< VPBlockBase * >
Definition: VPlan.h:2245
llvm::VPBasicBlock::VPBasicBlock
VPBasicBlock(const Twine &Name="", VPRecipeBase *Recipe=nullptr)
Definition: VPlan.h:2022
llvm::VPBlockUtils::disconnectBlocks
static void disconnectBlocks(VPBlockBase *From, VPBlockBase *To)
Disconnect VPBlockBases From and To bi-directionally.
Definition: VPlan.h:2858
llvm::VPWidenPointerInductionRecipe::~VPWidenPointerInductionRecipe
~VPWidenPointerInductionRecipe() override=default
llvm::VPlan::getEntry
VPBlockBase * getEntry()
Definition: VPlan.h:2569
assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
llvm::VPWidenIntOrFpInductionRecipe::getPHINode
PHINode * getPHINode()
Definition: VPlan.h:1109
llvm::VPFirstOrderRecurrencePHIRecipe::classof
static bool classof(const VPHeaderPHIRecipe *R)
Definition: VPlan.h:1290
llvm::VPHeaderPHIRecipe::classof
static bool classof(const VPRecipeBase *B)
Method to support type inquiry through isa, cast, and dyn_cast.
Definition: VPlan.h:1145
llvm::VPBasicBlock::back
VPRecipeBase & back()
Definition: VPlan.h:2057
llvm::VPWidenIntOrFpInductionRecipe
A recipe for handling phi nodes of integer and floating-point inductions, producing their vector valu...
Definition: VPlan.h:1055
llvm::GraphTraits< VPRegionBlock * >::getEntryNode
static NodeRef getEntryNode(GraphRef N)
Definition: VPlan.h:2304
llvm::GraphTraits< Inverse< VPRegionBlock * > >::nodes_end
static nodes_iterator nodes_end(GraphRef N)
Definition: VPlan.h:2350
llvm::VPReplicateRecipe::isUniform
bool isUniform() const
Definition: VPlan.h:1593
llvm::VPWidenPHIRecipe::classof
static bool classof(const VPRecipeBase *B)
Method to support type inquiry through isa, cast, and dyn_cast.
Definition: VPlan.h:1246
llvm::VPWidenGEPRecipe
A recipe for handling GEP instructions.
Definition: VPlan.h:1014
llvm::VPInstruction::hasResult
bool hasResult() const
Definition: VPlan.h:875
Ptr
@ Ptr
Definition: TargetLibraryInfo.cpp:60
llvm::VPLiveOut
A value that is used outside the VPlan.
Definition: VPlan.h:634
llvm::SelectInst
This class represents the LLVM 'select' instruction.
Definition: Instructions.h:1737
llvm::ISD::BasicBlock
@ BasicBlock
Various leaf nodes.
Definition: ISDOpcodes.h:71
Mode
SI Whole Quad Mode
Definition: SIWholeQuadMode.cpp:262
function
print Print MemDeps of function
Definition: MemDepPrinter.cpp:82
llvm::VPReplicateRecipe::VPReplicateRecipe
VPReplicateRecipe(Instruction *I, iterator_range< IterT > Operands, bool IsUniform, bool IsPredicated=false)
Definition: VPlan.h:1557
llvm::VPlan::~VPlan
~VPlan()
Definition: VPlan.h:2541
llvm::VPWidenPointerInductionRecipe::classof
static bool classof(const VPHeaderPHIRecipe *R)
Definition: VPlan.h:1208
llvm::VPWidenPointerInductionRecipe
Definition: VPlan.h:1183
llvm::VPInstruction::CanonicalIVIncrementForPart
@ CanonicalIVIncrementForPart
Definition: VPlan.h:793
llvm::VPWidenMemoryInstructionRecipe
A Recipe for widening load/store operations.
Definition: VPlan.h:1702
llvm::VPInstruction::clone
VPInstruction * clone() const
Definition: VPlan.h:831
llvm::VPBasicBlock::~VPBasicBlock
~VPBasicBlock() override
Definition: VPlan.h:2028
llvm::VPValue::VPVWidenSelectSC
@ VPVWidenSelectSC
Definition: VPlanValue.h:101
llvm::LinearPolySize::getKnownMinValue
ScalarTy getKnownMinValue() const
Returns the minimum value this size can represent.
Definition: TypeSize.h:296
llvm::VPTransformState::addNewMetadata
void addNewMetadata(Instruction *To, const Instruction *Orig)
Add additional metadata to To that was not present on Orig.
Definition: VPlan.cpp:230
llvm::df_iterator
Definition: DepthFirstIterator.h:86
llvm::VPWidenIntOrFpInductionRecipe::classof
static bool classof(const VPDef *D)
Method to support type inquiry through isa, cast, and dyn_cast.
Definition: VPlan.h:1078
llvm::VPWidenRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlanRecipes.cpp:682
llvm::VPRegionBlock::setExiting
void setExiting(VPBlockBase *ExitingBlock)
Set ExitingBlock as the exiting VPBlockBase of this VPRegionBlock.
Definition: VPlan.h:2199
llvm::VPWidenSelectRecipe
A recipe for widening select instructions.
Definition: VPlan.h:983
llvm::GraphTraits< const VPRegionBlock * >::nodes_end
static nodes_iterator nodes_end(GraphRef N)
Definition: VPlan.h:2329
Builder
assume Assume Builder
Definition: AssumeBundleBuilder.cpp:651
llvm::VPInstruction::classof
static bool classof(const VPDef *R)
Method to support type inquiry through isa, cast, and dyn_cast.
Definition: VPlan.h:837
llvm::VPWidenCanonicalIVRecipe::classof
static bool classof(const VPUser *U)
Extra classof implementations to allow directly casting from VPUser -> VPWidenCanonicalIVRecipe.
Definition: VPlan.h:1926
llvm::VPCanonicalIVPHIRecipe::VPCanonicalIVPHIRecipe
VPCanonicalIVPHIRecipe(VPValue *StartV, DebugLoc DL)
Definition: VPlan.h:1834
llvm::SetVector< T, SmallVector< T, N >, SmallDenseSet< T, N > >::insert
bool insert(const value_type &X)
Insert a new element into the SetVector.
Definition: SetVector.h:141
llvm::VPWidenIntOrFpInductionRecipe::isCanonical
bool isCanonical() const
Returns true if the induction is canonical, i.e.
Definition: VPlanRecipes.cpp:706
llvm::VPActiveLaneMaskPHIRecipe::classof
static bool classof(const VPValue *V)
Definition: VPlan.h:1896
llvm::VPWidenCallRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlanRecipes.cpp:496
llvm::VPBlockBase::printAsOperand
void printAsOperand(raw_ostream &OS, bool PrintType) const
Definition: VPlan.h:604
llvm::GraphTraits< const VPBlockBase * >
Definition: VPlan.h:2260
llvm::VPlan::getVectorLoopRegion
VPRegionBlock * getVectorLoopRegion()
Returns the VPRegionBlock of the vector loop.
Definition: VPlan.h:2682
llvm::VPBasicBlock::front
VPRecipeBase & front()
Definition: VPlan.h:2055
llvm::VPCanonicalIVPHIRecipe::onlyFirstLaneUsed
bool onlyFirstLaneUsed(const VPValue *Op) const override
Returns true if the recipe only uses the first lane of operand Op.
Definition: VPlan.h:1867
llvm::VPTransformState::reset
void reset(VPValue *Def, Value *V, unsigned Part)
Reset an existing vector value for Def and a given Part.
Definition: VPlan.h:272
llvm::VPScalarIVStepsRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlanRecipes.cpp:730
llvm::VPWidenCallRecipe::VPWidenCallRecipe
VPWidenCallRecipe(CallInst &I, iterator_range< IterT > CallArguments, Intrinsic::ID VectorIntrinsicID)
Definition: VPlan.h:959
llvm::VPTransformState::addMetadata
void addMetadata(Instruction *To, Instruction *From)
Add metadata from one instruction to another.
Definition: VPlan.cpp:238
llvm::VPlanIngredient::V
const Value * V
Definition: VPlan.h:2779
llvm::getRuntimeVF
Value * getRuntimeVF(IRBuilderBase &B, Type *Ty, ElementCount VF)
Return the runtime value for VF.
Definition: LoopVectorize.cpp:962
llvm::VPFirstOrderRecurrencePHIRecipe::VPFirstOrderRecurrencePHIRecipe
VPFirstOrderRecurrencePHIRecipe(PHINode *Phi, VPValue &Start)
Definition: VPlan.h:1282
llvm::ArrayRef
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: APInt.h:32
llvm::LoopInfo
Definition: LoopInfo.h:1105
llvm::VPReductionRecipe::getVecOp
VPValue * getVecOp() const
The VPValue of the vector value to be reduced.
Definition: VPlan.h:1534
llvm::VPBlockBase::getSingleSuccessor
VPBlockBase * getSingleSuccessor() const
Definition: VPlan.h:496
this
Analysis the ScalarEvolution expression for r is this
Definition: README.txt:8
llvm::VPUser::addOperand
void addOperand(VPValue *Operand)
Definition: VPlanValue.h:254
llvm::VPUser::onlyFirstLaneUsed
virtual bool onlyFirstLaneUsed(const VPValue *Op) const
Returns true if the VPUser only uses the first lane of operand Op.
Definition: VPlanValue.h:303
llvm::VPLane::getKnownLane
unsigned getKnownLane() const
Returns a compile-time known value for the lane index and asserts if the lane can only be calculated ...
Definition: VPlan.h:152
llvm_unreachable
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition: ErrorHandling.h:143
llvm::Value::getType
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:255
llvm::VPWidenCallRecipe::classof
static bool classof(const VPDef *D)
Method to support type inquiry through isa, cast, and dyn_cast.
Definition: VPlan.h:968
llvm::VPLane::Kind
Kind
Kind describes how to interpret Lane.
Definition: VPlan.h:115
llvm::VPExpandSCEVRecipe::classof
static bool classof(const VPDef *D)
Method to support type inquiry through isa, cast, and dyn_cast.
Definition: VPlan.h:1810
llvm::VPRecipeBase::getParent
VPBasicBlock * getParent()
Definition: VPlan.h:682
llvm::IRBuilderBase
Common base class shared among various IRBuilders.
Definition: IRBuilder.h:93
DL
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Definition: AArch64SLSHardening.cpp:76
llvm::VPIteration::Part
unsigned Part
in [0..UF)
Definition: VPlan.h:190
llvm::VPCanonicalIVPHIRecipe::execute
void execute(VPTransformState &State) override
Generate the canonical scalar induction phi of the vector loop.
Definition: VPlanRecipes.cpp:1029
llvm::VPFirstOrderRecurrencePHIRecipe
A recipe for handling first-order recurrence phis.
Definition: VPlan.h:1281
llvm::VPExpandSCEVRecipe::~VPExpandSCEVRecipe
~VPExpandSCEVRecipe() override=default
llvm::VPWidenPointerInductionRecipe::classof
static bool classof(const VPValue *V)
Definition: VPlan.h:1211
llvm::VPScalarIVStepsRecipe::getStepValue
VPValue * getStepValue() const
Definition: VPlan.h:2000
llvm::VPAllSuccessorsIterator::VPAllSuccessorsIterator
VPAllSuccessorsIterator(BlockPtrTy Block, size_t Idx=0)
Definition: VPlan.h:2393
llvm::VPBranchOnMaskRecipe
A recipe for generating conditional branches on the bits of a mask.
Definition: VPlan.h:1615
llvm::VPWidenSelectRecipe::execute
void execute(VPTransformState &State) override
Produce a widened version of the select instruction.
Definition: VPlanRecipes.cpp:532
llvm::VPBasicBlock::splitAt
VPBasicBlock * splitAt(iterator SplitAt)
Split current block at SplitAt by inserting a new block between the current block and its successors ...
Definition: VPlan.cpp:392
llvm::VPHeaderPHIRecipe::VPHeaderPHIRecipe
VPHeaderPHIRecipe(unsigned char VPVID, unsigned char VPDefID, PHINode *Phi, VPValue *Start=nullptr)
Definition: VPlan.h:1134
llvm::VPBlockBase::setOneSuccessor
void setOneSuccessor(VPBlockBase *Successor)
Set a given VPBlockBase Successor as the single successor of this VPBlockBase.
Definition: VPlan.h:555
llvm::VPInstruction::BranchOnCount
@ BranchOnCount
Definition: VPlan.h:795
llvm::VPlan::execute
void execute(VPTransformState *State)
Generate the IR code for this VPlan.
Definition: VPlan.cpp:666
llvm::VPTransformState::VF
ElementCount VF
The chosen Vectorization and Unroll Factors of the loop being vectorized.
Definition: VPlan.h:213
llvm::LoadInst
An instruction for reading from memory.
Definition: Instructions.h:174
llvm::VPWidenPHIRecipe::classof
static bool classof(const VPHeaderPHIRecipe *R)
Definition: VPlan.h:1249
llvm::VPValue::VPInstruction
friend class VPInstruction
Definition: VPlanValue.h:47
llvm::DenseMapBase::insert
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Definition: DenseMap.h:207
llvm::VPlan::addLiveOut
void addLiveOut(PHINode *PN, VPValue *V)
Definition: VPlan.cpp:792
llvm::make_filter_range
iterator_range< filter_iterator< detail::IterOfRange< RangeT >, PredicateT > > make_filter_range(RangeT &&Range, PredicateT Pred)
Convenience function that takes a range of elements and a predicate, and return a new filter_iterator...
Definition: STLExtras.h:512
llvm::VPBlockBase::getEntryBasicBlock
const VPBasicBlock * getEntryBasicBlock() const
Definition: VPlan.cpp:139
llvm::vputils::onlyFirstLaneUsed
bool onlyFirstLaneUsed(VPValue *Def)
Returns true if only the first lane of Def is used.
Definition: VPlan.cpp:1093
llvm::VPRegionBlock::getEntry
VPBlockBase * getEntry()
Definition: VPlan.h:2177
llvm::VPBasicBlock::classof
static bool classof(const VPBlockBase *V)
Method to support type inquiry through isa, cast, and dyn_cast.
Definition: VPlan.h:2068
llvm::VPTransformState::Data
struct llvm::VPTransformState::DataState Data
llvm::VPDef::getVPValue
VPValue * getVPValue(unsigned I)
Returns the VPValue with index I defined by the VPDef.
Definition: VPlanValue.h:405
llvm::vputils::isUniformAfterVectorization
bool isUniformAfterVectorization(VPValue *VPV)
Returns true if VPV is uniform after vectorization.
Definition: VPlan.h:3057
llvm::VPInstruction::onlyFirstLaneUsed
bool onlyFirstLaneUsed(const VPValue *Op) const override
Returns true if the recipe only uses the first lane of operand Op.
Definition: VPlan.h:901
llvm::VPRecipeBase::classof
static bool classof(const VPUser *U)
Definition: VPlan.h:733
llvm::VPLiveOut::VPLiveOut
VPLiveOut(PHINode *Phi, VPValue *Op)
Definition: VPlan.h:638
llvm::VPFirstOrderRecurrencePHIRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlanRecipes.cpp:1146
llvm::VPValue::setUnderlyingValue
void setUnderlyingValue(Value *Val)
Definition: VPlanValue.h:77
llvm::VPReplicateRecipe::setAlsoPack
void setAlsoPack(bool Pack)
Definition: VPlan.h:1585
llvm::VPReductionPHIRecipe::getRecurrenceDescriptor
const RecurrenceDescriptor & getRecurrenceDescriptor() const
Definition: VPlan.h:1352
llvm::depth_first
iterator_range< df_iterator< T > > depth_first(const T &G)
Definition: DepthFirstIterator.h:230
CFG
Common register allocation spilling lr str ldr sxth r3 ldr mla r4 can lr mov lr str ldr sxth r3 mla r4 and then merge mul and lr str ldr sxth r3 mla r4 It also increase the likelihood the store may become dead bb27 Successors according to CFG
Definition: README.txt:39
llvm::VPTransformState::LVer
std::unique_ptr< LoopVersioning > LVer
LoopVersioning.
Definition: VPlan.h:384
llvm::VPlan::addVPValue
void addVPValue(Value *V, VPValue *VPV)
Definition: VPlan.h:2625
llvm::GraphTraits< VPRegionBlock * >::nodes_end
static nodes_iterator nodes_end(GraphRef N)
Definition: VPlan.h:2310
llvm::VPBlockBase::setParent
void setParent(VPRegionBlock *P)
Definition: VPlan.h:472
llvm::VPWidenPHIRecipe::addIncoming
void addIncoming(VPValue *IncomingV, VPBasicBlock *IncomingBlock)
Adds a pair (IncomingV, IncomingBlock) to the phi.
Definition: VPlan.h:1266
llvm::VPRecipeBase::VPRecipeBase
VPRecipeBase(const unsigned char SC, iterator_range< IterT > Operands)
Definition: VPlan.h:677
llvm::VPBasicBlock::rend
const_reverse_iterator rend() const
Definition: VPlan.h:2050
llvm::VPlan::mapToVPValues
iterator_range< mapped_iterator< Use *, std::function< VPValue *(Value *)> > > mapToVPValues(User::op_range Operands)
Returns a range mapping the values the range Operands to their corresponding VPValues.
Definition: VPlan.h:2674
llvm::VPUser::classof
static bool classof(const VPDef *Recipe)
Method to support type inquiry through isa, cast, and dyn_cast.
Definition: VPlan.h:757
llvm::VPReductionPHIRecipe::execute
void execute(VPTransformState &State) override
Generate the phi/select nodes.
Definition: VPlanRecipes.cpp:1155
llvm::Twine
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:81
llvm::VPBlockBase::print
void print(raw_ostream &O) const
Print plain-text dump of this VPlan to O.
Definition: VPlan.h:618
llvm::VPSlotTracker
This class can be used to assign consecutive numbers to all VPValues in a VPlan and allows querying t...
Definition: VPlanValue.h:443
llvm::map_range
auto map_range(ContainerTy &&C, FuncTy F)
Definition: STLExtras.h:315
llvm::VPBasicBlock::begin
const_iterator begin() const
Definition: VPlan.h:2043
llvm::VPInstruction::CanonicalIVIncrementForPartNUW
@ CanonicalIVIncrementForPartNUW
Definition: VPlan.h:794
llvm::GraphProgram::Name
Name
Definition: GraphWriter.h:50
llvm::VPRecipeBase::~VPRecipeBase
virtual ~VPRecipeBase()=default
llvm::VPWidenGEPRecipe::VPWidenGEPRecipe
VPWidenGEPRecipe(GetElementPtrInst *GEP, iterator_range< IterT > Operands, Loop *OrigLoop)
Definition: VPlan.h:1026
llvm::VPlanSlp::VPlanSlp
VPlanSlp(VPInterleavedAccessInfo &IAI, VPBasicBlock &BB)
Definition: VPlan.h:3028
llvm::VPBlockBase::getParent
VPRegionBlock * getParent()
Definition: VPlan.h:461
llvm::VPAllSuccessorsIterator::operator==
bool operator==(const VPAllSuccessorsIterator &R) const
Definition: VPlan.h:2415
llvm::VPTransformState::CurrentVectorLoop
Loop * CurrentVectorLoop
The loop object for the current parent region, or nullptr.
Definition: VPlan.h:377
llvm::VPBasicBlock::rbegin
reverse_iterator rbegin()
Definition: VPlan.h:2047
llvm::VPAllSuccessorsIterator::end
static VPAllSuccessorsIterator end(BlockPtrTy Block)
Definition: VPlan.h:2404
llvm::VPWidenGEPRecipe::classof
static bool classof(const VPDef *D)
Method to support type inquiry through isa, cast, and dyn_cast.
Definition: VPlan.h:1039
llvm::Successor
@ Successor
Definition: SIMachineScheduler.h:35
llvm::AMDGPU::SendMsg::Op
Op
Definition: SIDefines.h:348
llvm::VPTransformState::MayGeneratePoisonRecipes
SmallPtrSet< VPRecipeBase *, 16 > MayGeneratePoisonRecipes
Holds recipes that may generate a poison value that is used after vectorization, even when their oper...
Definition: VPlan.h:374
llvm::VPValue::getDef
VPDef * getDef()
Definition: VPlanValue.h:185
llvm::ilist_iterator
Iterator for intrusive lists based on ilist_node.
Definition: ilist_iterator.h:57
llvm::VPRegionBlock::VPRegionBlock
VPRegionBlock(const std::string &Name="", bool IsReplicator=false)
Definition: VPlan.h:2159
llvm::VPRecipeBase::classof
static bool classof(const VPDef *D)
Method to support type inquiry through isa, cast, and dyn_cast.
Definition: VPlan.h:728
llvm::VPCanonicalIVPHIRecipe::classof
static bool classof(const VPDef *D)
Method to support type inquiry through isa, cast, and dyn_cast.
Definition: VPlan.h:1842
llvm::VPWidenGEPRecipe::VPWidenGEPRecipe
VPWidenGEPRecipe(GetElementPtrInst *GEP, iterator_range< IterT > Operands)
Definition: VPlan.h:1020
llvm::VPScalarIVStepsRecipe
A recipe for handling phi nodes of integer and floating-point inductions, producing their scalar valu...
Definition: VPlan.h:1954
llvm::VPRecipeBase
VPRecipeBase is a base class modeling a sequence of one or more output IR instructions.
Definition: VPlan.h:663
llvm::iplist_impl::pop_back
void pop_back()
Definition: ilist.h:319
llvm::VPRegionBlock::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print this VPRegionBlock to O (recursively), prefixing all lines with Indent.
Definition: VPlan.cpp:557
llvm::VPlan::setEntry
VPBlockBase * setEntry(VPBlockBase *Block)
Definition: VPlan.h:2572
llvm::VPReductionRecipe::~VPReductionRecipe
~VPReductionRecipe() override=default
llvm::VPActiveLaneMaskPHIRecipe::~VPActiveLaneMaskPHIRecipe
~VPActiveLaneMaskPHIRecipe() override=default
llvm::TargetStackID::Value
Value
Definition: TargetFrameLowering.h:27
llvm::SetVector< T, SmallVector< T, N >, SmallDenseSet< T, N > >::count
size_type count(const key_type &key) const
Count the number of elements of a given key in the SetVector.
Definition: SetVector.h:215
llvm::VPAllSuccessorsIterator::VPAllSuccessorsIterator
VPAllSuccessorsIterator(const VPAllSuccessorsIterator &Other)
Definition: VPlan.h:2395
llvm::SlotTracker
This class provides computation of slot numbers for LLVM Assembly writing.
Definition: AsmWriter.cpp:664
llvm::VPReplicateRecipe::usesScalars
bool usesScalars(const VPValue *Op) const override
Returns true if the recipe uses scalars of operand Op.
Definition: VPlan.h:1607
llvm::VPRecipeBase::mayReadOrWriteMemory
bool mayReadOrWriteMemory() const
Returns true if the recipe may read from or write to memory.
Definition: VPlan.h:752
llvm::VPlan::getVectorTripCount
VPValue & getVectorTripCount()
The vector trip count.
Definition: VPlan.h:2593
llvm::SmallVectorImpl::clear
void clear()
Definition: SmallVector.h:596
llvm::VPInstruction::setUnderlyingInstr
void setUnderlyingInstr(Instruction *I)
Definition: VPlan.h:813
llvm::VPTransformState::CFGState
Hold state information used when constructing the CFG of the output IR, traversing the VPBasicBlocks ...
Definition: VPlan.h:329
llvm::VPWidenCanonicalIVRecipe::getScalarType
const Type * getScalarType() const
Returns the scalar type of the induction.
Definition: VPlan.h:1946
llvm::VPBlockBase::setName
void setName(const Twine &newName)
Definition: VPlan.h:454
llvm::Inverse
Definition: GraphTraits.h:97
LoopVersioning.h
llvm::VPUser::VPUserID::LiveOut
@ LiveOut
llvm::VPWidenPointerInductionRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlanRecipes.cpp:1056
llvm::VPTransformState::CFGState::CFGState
CFGState()=default
llvm::VPBasicBlock::insert
void insert(VPRecipeBase *Recipe, iterator InsertPt)
Definition: VPlan.h:2072
llvm::VPValue::VPVLastPHISC
@ VPVLastPHISC
Definition: VPlanValue.h:115
llvm::VPBlendRecipe
A recipe for vectorizing a phi-node as a sequence of mask-based select instructions.
Definition: VPlan.h:1365
llvm::iplist_impl::insert
iterator insert(iterator where, pointer New)
Definition: ilist.h:229
llvm::VPReplicateRecipe::isPredicated
bool isPredicated() const
Definition: VPlan.h:1597
llvm::VPUser
This class augments VPValue with operands which provide the inverse def-use edges from VPValue's user...
Definition: VPlanValue.h:210
llvm::VPFirstOrderRecurrencePHIRecipe::classof
static bool classof(const VPValue *V)
Definition: VPlan.h:1293
llvm::VPInstruction::SLPStore
@ SLPStore
Definition: VPlan.h:787
llvm::RecurrenceDescriptor
The RecurrenceDescriptor is used to identify recurrences variables in a loop.
Definition: IVDescriptors.h:69
llvm::VPlan::prepareToExecute
void prepareToExecute(Value *TripCount, Value *VectorTripCount, Value *CanonicalIVStartValue, VPTransformState &State, bool IsEpilogueVectorization)
Prepare the plan for execution, setting up the required live-in values.
Definition: VPlan.cpp:589
llvm::VPlan::VPlan
VPlan(VPBlockBase *Entry=nullptr)
Definition: VPlan.h:2536
llvm::VPValue::VPVWidenGEPSC
@ VPVWidenGEPSC
Definition: VPlanValue.h:100
llvm::VPBlockBase::print
virtual void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const =0
Print plain-text dump of this VPBlockBase to O, prefixing all lines with Indent.
llvm::VPReductionRecipe::classof
static bool classof(const VPValue *V)
Method to support type inquiry through isa, cast, and dyn_cast.
Definition: VPlan.h:1518
llvm::VPInstruction::VPInstruction
VPInstruction(unsigned Opcode, ArrayRef< VPValue * > Operands, DebugLoc DL, const Twine &Name="")
Definition: VPlan.h:816
llvm::VPBlendRecipe::VPBlendRecipe
VPBlendRecipe(PHINode *Phi, ArrayRef< VPValue * > Operands)
The blend operation is a User of the incoming values and of their respective masks,...
Definition: VPlan.h:1372
SmallVector.h
llvm::VPBasicBlock::front
const VPRecipeBase & front() const
Definition: VPlan.h:2054
llvm::VPTransformState::VPValue2Value
VPValue2ValueTy VPValue2Value
Definition: VPlan.h:361
llvm::VPBasicBlock::begin
iterator begin()
Recipe iterator methods.
Definition: VPlan.h:2042
llvm::VPlanSlp::isCompletelySLP
bool isCompletelySLP() const
Return true if all visited instruction can be combined.
Definition: VPlan.h:3040
llvm::VPAllSuccessorsIterator::operator++
VPAllSuccessorsIterator operator++(int X)
Definition: VPlan.h:2428
llvm::VPValue::VPVWidenPHISC
@ VPVWidenPHISC
Definition: VPlanValue.h:110
llvm::VPHeaderPHIRecipe::getStartValue
VPValue * getStartValue() const
Definition: VPlan.h:1167
llvm::VPRegionBlock::setEntry
void setEntry(VPBlockBase *EntryBlock)
Set EntryBlock as the entry VPBlockBase of this VPRegionBlock.
Definition: VPlan.h:2181
llvm::VPAllSuccessorsIterator::operator=
VPAllSuccessorsIterator & operator=(const VPAllSuccessorsIterator &R)
Definition: VPlan.h:2398
llvm::VPlan::getLiveOuts
const MapVector< PHINode *, VPLiveOut * > & getLiveOuts() const
Definition: VPlan.h:2716
llvm::VPWidenMemoryInstructionRecipe::execute
void execute(VPTransformState &State) override
Generate the wide load/store.
Definition: LoopVectorize.cpp:9705
llvm::VPBlendRecipe::classof
static bool classof(const VPDef *D)
Method to support type inquiry through isa, cast, and dyn_cast.
Definition: VPlan.h:1382
llvm::VPActiveLaneMaskPHIRecipe
A recipe for generating the active lane mask for the vector loop that is used to predicate the vector...
Definition: VPlan.h:1878
SmallBitVector.h
llvm::VPUser::getOperand
VPValue * getOperand(unsigned N) const
Definition: VPlanValue.h:260
llvm::VPTransformState::Instance
Optional< VPIteration > Instance
Hold the indices to generate specific scalar instructions.
Definition: VPlan.h:219
N
#define N
llvm::VPRecipeBase::mayReadFromMemory
bool mayReadFromMemory() const
Returns true if the recipe may read from memory.
Definition: VPlanRecipes.cpp:73
llvm::iplist_impl< simple_ilist< VPRecipeBase, Options... >, ilist_traits< VPRecipeBase > >::const_iterator
base_list_type::const_iterator const_iterator
Definition: ilist.h:179
llvm::hash_combine_range
hash_code hash_combine_range(InputIteratorT first, InputIteratorT last)
Compute a hash_code for a sequence of values.
Definition: Hashing.h:483
llvm::VPRegionBlock::classof
static bool classof(const VPBlockBase *V)
Method to support type inquiry through isa, cast, and dyn_cast.
Definition: VPlan.h:2172
llvm::VPlan::printDOT
void printDOT(raw_ostream &O) const
Print this VPlan in DOT format to O.
Definition: VPlan.cpp:783
llvm::VPlan::getOrCreateBackedgeTakenCount
VPValue * getOrCreateBackedgeTakenCount()
The backedge taken count of the original loop.
Definition: VPlan.h:2586
llvm::VPBasicBlock::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print this VPBsicBlock to O, prefixing all lines with Indent.
Definition: VPlan.cpp:485
llvm::VPBlockUtils::VPBlockUtils
VPBlockUtils()=delete
llvm::iterator_range
A range adaptor for a pair of iterators.
Definition: iterator_range.h:30
llvm::VPExpandSCEVRecipe
Recipe to expand a SCEV expression.
Definition: VPlan.h:1798
llvm::VPActiveLaneMaskPHIRecipe::classof
static bool classof(const VPHeaderPHIRecipe *D)
Definition: VPlan.h:1893
llvm::VPActiveLaneMaskPHIRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlanRecipes.cpp:1292
llvm::PHINode
Definition: Instructions.h:2699
llvm::VPWidenRecipe::VPWidenRecipe
VPWidenRecipe(Instruction &I, iterator_range< IterT > Operands)
Definition: VPlan.h:927
llvm::VPWidenMemoryInstructionRecipe::isConsecutive
bool isConsecutive() const
Definition: VPlan.h:1767
ilist_node.h
llvm::GraphTraits< VPBlockRecursiveTraversalWrapper< VPBlockBase * > >::child_end
static ChildIteratorType child_end(NodeRef N)
Definition: VPlan.h:2463
llvm::VPBlockBase::getExitingBasicBlock
const VPBasicBlock * getExitingBasicBlock() const
Definition: VPlan.cpp:160
llvm::VPFirstOrderRecurrencePHIRecipe::execute
void execute(VPTransformState &State) override
Generate the phi nodes.
Definition: VPlanRecipes.cpp:1117
llvm::SmallVectorImpl< VPBlockBase * >
llvm::VPTransformState::CFGState::VPBB2IRBB
SmallDenseMap< VPBasicBlock *, BasicBlock * > VPBB2IRBB
A mapping of each VPBasicBlock to the corresponding BasicBlock.
Definition: VPlan.h:343
llvm::VPBlockBase::getParent
const VPRegionBlock * getParent() const
Definition: VPlan.h:462
llvm::VPTransformState::Plan
VPlan * Plan
Pointer to the VPlan code is generated for.
Definition: VPlan.h:370
llvm::VPLane
In what follows, the term "input IR" refers to code that is fed into the vectorizer whereas the term ...
Definition: VPlan.h:112
llvm::SmallPtrSetImpl
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
Definition: SmallPtrSet.h:344
llvm::SmallSetVector
A SetVector that performs no allocations if smaller than a certain size.
Definition: SetVector.h:307
llvm::VPHeaderPHIRecipe::getBackedgeValue
VPValue * getBackedgeValue()
Returns the incoming value from the loop backedge.
Definition: VPlan.h:1172
llvm::VPFirstOrderRecurrencePHIRecipe::classof
static bool classof(const VPRecipeBase *R)
Method to support type inquiry through isa, cast, and dyn_cast.
Definition: VPlan.h:1287
llvm::VPPredInstPHIRecipe::VPPredInstPHIRecipe
VPPredInstPHIRecipe(VPValue *PredV)
Construct a VPPredInstPHIRecipe given PredInst whose value needs a phi nodes after merging back from ...
Definition: VPlan.h:1669
llvm::CallInst
This class represents a function call, abstracting a target machine's calling convention.
Definition: Instructions.h:1474
llvm::VPValue::getVPValueID
unsigned getVPValueID() const
Definition: VPlanValue.h:128
llvm::VPWidenSelectRecipe::VPWidenSelectRecipe
VPWidenSelectRecipe(SelectInst &I, iterator_range< IterT > Operands, bool InvariantCond)
Definition: VPlan.h:990
llvm::VPBranchOnMaskRecipe::getMask
VPValue * getMask() const
Return the mask used by this recipe.
Definition: VPlan.h:1646
BB
Common register allocation spilling lr str ldr sxth r3 ldr mla r4 can lr mov lr str ldr sxth r3 mla r4 and then merge mul and lr str ldr sxth r3 mla r4 It also increase the likelihood the store may become dead bb27 Successors according to LLVM BB
Definition: README.txt:39
GEP
Hexagon Common GEP
Definition: HexagonCommonGEP.cpp:171
llvm::VPBlockBase::deleteCFG
static void deleteCFG(VPBlockBase *Entry)
Delete all blocks reachable from a given VPBlockBase, inclusive.
Definition: VPlan.cpp:190
llvm::VPRegionBlock::isReplicator
bool isReplicator() const
An indicator whether this region is to generate multiple replicated instances of output IR correspond...
Definition: VPlan.h:2214
llvm::VPValue::VPVReplicateSC
@ VPVReplicateSC
Definition: VPlanValue.h:96
llvm::GraphTraits
Definition: GraphTraits.h:37
From
BlockVerifier::State From
Definition: BlockVerifier.cpp:55
llvm::DebugLoc
A debug info location.
Definition: DebugLoc.h:33
llvm::VPWidenPHIRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlanRecipes.cpp:1257
llvm::VPWidenIntOrFpInductionRecipe::getTruncInst
const TruncInst * getTruncInst() const
Definition: VPlan.h:1105
llvm::Region
Definition: RegionInfo.h:889
llvm::VFRange::End
ElementCount End
Definition: VPlan.h:88
llvm::VPBasicBlock::phis
iterator_range< iterator > phis()
Returns an iterator range over the PHI-like recipes in the block.
Definition: VPlan.h:2091
llvm::VPBasicBlock::getFirstNonPhi
iterator getFirstNonPhi()
Return the position of the first non-phi node recipe in the block.
Definition: VPlan.cpp:197
llvm::VPReductionRecipe::getChainOp
VPValue * getChainOp() const
The VPValue of the scalar Chain being accumulated.
Definition: VPlan.h:1532
llvm::VPInterleavedAccessInfo::VPInterleavedAccessInfo
VPInterleavedAccessInfo(VPlan &Plan, InterleavedAccessInfo &IAI)
Definition: VPlan.cpp:1062
llvm::VPBlockRecursiveTraversalWrapper
Helper for GraphTraits specialization that traverses through VPRegionBlocks.
Definition: VPlan.h:2436
llvm::VPWidenRecipe::classof
static bool classof(const VPValue *V)
Definition: VPlan.h:937
llvm::VPWidenRecipe::classof
static bool classof(const VPDef *D)
Method to support type inquiry through isa, cast, and dyn_cast.
Definition: VPlan.h:934
llvm::VPBlockBase::getSuccessors
const VPBlocksTy & getSuccessors() const
Definition: VPlan.h:486
llvm::VPHeaderPHIRecipe::~VPHeaderPHIRecipe
~VPHeaderPHIRecipe() override=default
llvm::VPExpandSCEVRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlanRecipes.cpp:1079
llvm::GraphTraits< Inverse< VPBlockBase * > >::ChildIteratorType
SmallVectorImpl< VPBlockBase * >::iterator ChildIteratorType
Definition: VPlan.h:2279
llvm::VPValue::VPVWidenCallSC
@ VPVWidenCallSC
Definition: VPlanValue.h:98
llvm::VPBlockRecursiveTraversalWrapper::VPBlockRecursiveTraversalWrapper
VPBlockRecursiveTraversalWrapper(BlockTy Entry)
Definition: VPlan.h:2440
llvm::VPWidenRecipe::~VPWidenRecipe
~VPWidenRecipe() override=default
llvm::VPWidenRecipe::execute
void execute(VPTransformState &State) override
Produce widened copies of all Ingredients.
Definition: VPlanRecipes.cpp:553
llvm::VPDef::getVPDefID
unsigned getVPDefID() const
Definition: VPlanValue.h:425
llvm::VPInstruction::dump
LLVM_DUMP_METHOD void dump() const
Print the VPInstruction to dbgs() (for debugging).
Definition: VPlanRecipes.cpp:358
llvm::VPValue::VPVMemoryInstructionSC
@ VPVMemoryInstructionSC
Definition: VPlanValue.h:94
llvm::GraphTraits< VPBlockRecursiveTraversalWrapper< VPBlockBase * > >::getEntryNode
static NodeRef getEntryNode(VPBlockRecursiveTraversalWrapper< VPBlockBase * > N)
Definition: VPlan.h:2455
llvm::VPBasicBlock::rend
reverse_iterator rend()
Definition: VPlan.h:2049
llvm::VPValue::getUnderlyingValue
Value * getUnderlyingValue()
Return the underlying Value attached to this VPValue.
Definition: VPlanValue.h:84
llvm::GraphTraits< VPBlockBase * >::child_end
static ChildIteratorType child_end(NodeRef N)
Definition: VPlan.h:2255
llvm::VPlan
VPlan models a candidate for vectorization, encoding various decisions take to produce efficient outp...
Definition: VPlan.h:2492
llvm::GraphTraits< Inverse< VPRegionBlock * > >::nodes_begin
static nodes_iterator nodes_begin(GraphRef N)
Definition: VPlan.h:2346
llvm::VPWidenIntOrFpInductionRecipe::getStepValue
const VPValue * getStepValue() const
Definition: VPlan.h:1098
llvm::VPInterleavedAccessInfo::~VPInterleavedAccessInfo
~VPInterleavedAccessInfo()
Definition: VPlan.h:2930
llvm::VPBlockBase::setPlan
void setPlan(VPlan *ParentPlan)
Sets the pointer of the plan containing the block.
Definition: VPlan.cpp:153
llvm::VPRecipeBase::execute
virtual void execute(VPTransformState &State)=0
The method which generates the output IR instructions that correspond to this VPRecipe,...
llvm::Value
LLVM Value Representation.
Definition: Value.h:74
llvm::VPRecipeBase::insertBefore
void insertBefore(VPRecipeBase *InsertPos)
Insert an unlinked recipe into a basic block immediately before the specified recipe.
Definition: VPlanRecipes.cpp:143
llvm::VPScalarIVStepsRecipe::~VPScalarIVStepsRecipe
~VPScalarIVStepsRecipe() override=default
llvm::deref
Binary functor that adapts to any other binary functor after dereferencing operands.
Definition: STLExtras.h:1931
llvm::MapVector::erase
VectorType::iterator erase(typename VectorType::iterator Iterator)
Remove the element given by Iterator.
Definition: MapVector.h:174
llvm::VPCanonicalIVPHIRecipe::classof
static bool classof(const VPValue *V)
Definition: VPlan.h:1848
llvm::VPHeaderPHIRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override=0
Print the recipe.
llvm::VPReductionPHIRecipe::isOrdered
bool isOrdered() const
Returns true, if the phi is part of an ordered reduction.
Definition: VPlan.h:1357
llvm::VPActiveLaneMaskPHIRecipe::classof
static bool classof(const VPDef *D)
Method to support type inquiry through isa, cast, and dyn_cast.
Definition: VPlan.h:1890
llvm::GraphTraits< const VPRegionBlock * >::getEntryNode
static NodeRef getEntryNode(GraphRef N)
Definition: VPlan.h:2323
llvm::VPlan::dump
LLVM_DUMP_METHOD void dump() const
Dump the plan to stderr (for debugging).
Definition: VPlan.cpp:789
llvm::VPlanIngredient
Definition: VPlan.h:2778
llvm::VPWidenSelectRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlanRecipes.cpp:518
llvm::VPRecipeBase::isPhi
bool isPhi() const
Returns true for PHI-like recipes.
Definition: VPlan.h:741
llvm::VPWidenIntOrFpInductionRecipe::getStartValue
VPValue * getStartValue()
Returns the start value of the induction.
Definition: VPlan.h:1093
llvm::VPlanPrinter
VPlanPrinter prints a given VPlan to a given output stream.
Definition: VPlan.h:2731
llvm::VPTransformState::CFGState::PrevBB
BasicBlock * PrevBB
The previous IR BasicBlock created or used.
Definition: VPlan.h:335
llvm::GraphTraits< VPBlockBase * >::ChildIteratorType
SmallVectorImpl< VPBlockBase * >::iterator ChildIteratorType
Definition: VPlan.h:2247
llvm::VPRegionBlock::getExiting
VPBlockBase * getExiting()
Definition: VPlan.h:2195
llvm::VPRecipeBase::insertAfter
void insertAfter(VPRecipeBase *InsertPos)
Insert an unlinked Recipe into a basic block immediately after the specified Recipe.
Definition: VPlanRecipes.cpp:159
llvm::VFRange::Start
const ElementCount Start
Definition: VPlan.h:85
Other
Optional< std::vector< StOtherPiece > > Other
Definition: ELFYAML.cpp:1247
llvm::VPTransformState::hasVectorValue
bool hasVectorValue(VPValue *Def, unsigned Part)
Definition: VPlan.h:243
llvm::Use
A Use represents the edge between a Value definition and its users.
Definition: Use.h:43
llvm::VPlan::getOrCreateTripCount
VPValue * getOrCreateTripCount()
The trip count of the original loop.
Definition: VPlan.h:2579
llvm::VPHeaderPHIRecipe::execute
void execute(VPTransformState &State) override=0
Generate the phi nodes.
llvm::SmallPtrSetImpl::insert
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:365
llvm::VPBlockBase::execute
virtual void execute(VPTransformState *State)=0
The method which generates the output IR that correspond to this VPBlockBase, thereby "executing" the...
llvm::Intrinsic::ID
unsigned ID
Definition: TargetTransformInfo.h:38
llvm::VPlan::addVF
void addVF(ElementCount VF)
Definition: VPlan.h:2599