LLVM  15.0.0git
VPlan.h
Go to the documentation of this file.
1 //===- VPlan.h - Represent A Vectorizer Plan --------------------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 /// \file
10 /// This file contains the declarations of the Vectorization Plan base classes:
11 /// 1. VPBasicBlock and VPRegionBlock that inherit from a common pure virtual
12 /// VPBlockBase, together implementing a Hierarchical CFG;
13 /// 2. Specializations of GraphTraits that allow VPBlockBase graphs to be
14 /// treated as proper graphs for generic algorithms;
15 /// 3. Pure virtual VPRecipeBase serving as the base class for recipes contained
16 /// within VPBasicBlocks;
17 /// 4. VPInstruction, a concrete Recipe and VPUser modeling a single planned
18 /// instruction;
19 /// 5. The VPlan class holding a candidate for vectorization;
20 /// 6. The VPlanPrinter class providing a way to print a plan in dot format;
21 /// These are documented in docs/VectorizationPlan.rst.
22 //
23 //===----------------------------------------------------------------------===//
24 
25 #ifndef LLVM_TRANSFORMS_VECTORIZE_VPLAN_H
26 #define LLVM_TRANSFORMS_VECTORIZE_VPLAN_H
27 
28 #include "VPlanLoopInfo.h"
29 #include "VPlanValue.h"
30 #include "llvm/ADT/DenseMap.h"
32 #include "llvm/ADT/GraphTraits.h"
33 #include "llvm/ADT/Optional.h"
35 #include "llvm/ADT/SmallPtrSet.h"
36 #include "llvm/ADT/SmallVector.h"
37 #include "llvm/ADT/Twine.h"
38 #include "llvm/ADT/ilist.h"
39 #include "llvm/ADT/ilist_node.h"
41 #include "llvm/IR/DebugLoc.h"
42 #include "llvm/IR/FMF.h"
43 #include <algorithm>
44 #include <cassert>
45 #include <cstddef>
46 #include <string>
47 
48 namespace llvm {
49 
50 class BasicBlock;
51 class DominatorTree;
52 class InductionDescriptor;
53 class InnerLoopVectorizer;
54 class IRBuilderBase;
55 class LoopInfo;
56 class raw_ostream;
57 class RecurrenceDescriptor;
58 class Value;
59 class VPBasicBlock;
60 class VPRegionBlock;
61 class VPlan;
62 class VPReplicateRecipe;
63 class VPlanSlp;
64 
65 /// Returns a calculation for the total number of elements for a given \p VF.
66 /// For fixed width vectors this value is a constant, whereas for scalable
67 /// vectors it is an expression determined at runtime.
68 Value *getRuntimeVF(IRBuilderBase &B, Type *Ty, ElementCount VF);
69 
70 /// Return a value for Step multiplied by VF.
71 Value *createStepForVF(IRBuilderBase &B, Type *Ty, ElementCount VF,
72  int64_t Step);
73 
74 /// A range of powers-of-2 vectorization factors with fixed start and
75 /// adjustable end. The range includes start and excludes end, e.g.,:
76 /// [1, 9) = {1, 2, 4, 8}
77 struct VFRange {
78  // A power of 2.
80 
81  // Need not be a power of 2. If End <= Start range is empty.
83 
84  bool isEmpty() const {
86  }
87 
89  : Start(Start), End(End) {
91  "Both Start and End should have the same scalable flag");
93  "Expected Start to be a power of 2");
94  }
95 };
96 
97 using VPlanPtr = std::unique_ptr<VPlan>;
98 
99 /// In what follows, the term "input IR" refers to code that is fed into the
100 /// vectorizer whereas the term "output IR" refers to code that is generated by
101 /// the vectorizer.
102 
103 /// VPLane provides a way to access lanes in both fixed width and scalable
104 /// vectors, where for the latter the lane index sometimes needs calculating
105 /// as a runtime expression.
106 class VPLane {
107 public:
108  /// Kind describes how to interpret Lane.
109  enum class Kind : uint8_t {
110  /// For First, Lane is the index into the first N elements of a
111  /// fixed-vector <N x <ElTy>> or a scalable vector <vscale x N x <ElTy>>.
112  First,
113  /// For ScalableLast, Lane is the offset from the start of the last
114  /// N-element subvector in a scalable vector <vscale x N x <ElTy>>. For
115  /// example, a Lane of 0 corresponds to lane `(vscale - 1) * N`, a Lane of
116  /// 1 corresponds to `((vscale - 1) * N) + 1`, etc.
118  };
119 
120 private:
121  /// in [0..VF)
122  unsigned Lane;
123 
124  /// Indicates how the Lane should be interpreted, as described above.
125  Kind LaneKind;
126 
127 public:
128  VPLane(unsigned Lane, Kind LaneKind) : Lane(Lane), LaneKind(LaneKind) {}
129 
131 
133  unsigned LaneOffset = VF.getKnownMinValue() - 1;
134  Kind LaneKind;
135  if (VF.isScalable())
136  // In this case 'LaneOffset' refers to the offset from the start of the
137  // last subvector with VF.getKnownMinValue() elements.
138  LaneKind = VPLane::Kind::ScalableLast;
139  else
140  LaneKind = VPLane::Kind::First;
141  return VPLane(LaneOffset, LaneKind);
142  }
143 
144  /// Returns a compile-time known value for the lane index and asserts if the
145  /// lane can only be calculated at runtime.
146  unsigned getKnownLane() const {
147  assert(LaneKind == Kind::First);
148  return Lane;
149  }
150 
151  /// Returns an expression describing the lane index that can be used at
152  /// runtime.
154 
155  /// Returns the Kind of lane offset.
156  Kind getKind() const { return LaneKind; }
157 
158  /// Returns true if this is the first lane of the whole vector.
159  bool isFirstLane() const { return Lane == 0 && LaneKind == Kind::First; }
160 
161  /// Maps the lane to a cache index based on \p VF.
162  unsigned mapToCacheIndex(const ElementCount &VF) const {
163  switch (LaneKind) {
165  assert(VF.isScalable() && Lane < VF.getKnownMinValue());
166  return VF.getKnownMinValue() + Lane;
167  default:
168  assert(Lane < VF.getKnownMinValue());
169  return Lane;
170  }
171  }
172 
173  /// Returns the maxmimum number of lanes that we are able to consider
174  /// caching for \p VF.
175  static unsigned getNumCachedLanes(const ElementCount &VF) {
176  return VF.getKnownMinValue() * (VF.isScalable() ? 2 : 1);
177  }
178 };
179 
180 /// VPIteration represents a single point in the iteration space of the output
181 /// (vectorized and/or unrolled) IR loop.
182 struct VPIteration {
183  /// in [0..UF)
184  unsigned Part;
185 
187 
188  VPIteration(unsigned Part, unsigned Lane,
190  : Part(Part), Lane(Lane, Kind) {}
191 
192  VPIteration(unsigned Part, const VPLane &Lane) : Part(Part), Lane(Lane) {}
193 
194  bool isFirstIteration() const { return Part == 0 && Lane.isFirstLane(); }
195 };
196 
197 /// VPTransformState holds information passed down when "executing" a VPlan,
198 /// needed for generating the output IR.
203  : VF(VF), UF(UF), LI(LI), DT(DT), Builder(Builder), ILV(ILV), Plan(Plan) {
204  }
205 
206  /// The chosen Vectorization and Unroll Factors of the loop being vectorized.
208  unsigned UF;
209 
210  /// Hold the indices to generate specific scalar instructions. Null indicates
211  /// that all instances are to be generated, using either scalar or vector
212  /// instructions.
214 
215  struct DataState {
216  /// A type for vectorized values in the new loop. Each value from the
217  /// original loop, when vectorized, is represented by UF vector values in
218  /// the new unrolled loop, where UF is the unroll factor.
220 
222 
225  } Data;
226 
227  /// Get the generated Value for a given VPValue and a given Part. Note that
228  /// as some Defs are still created by ILV and managed in its ValueMap, this
229  /// method will delegate the call to ILV in such cases in order to provide
230  /// callers a consistent API.
231  /// \see set.
232  Value *get(VPValue *Def, unsigned Part);
233 
234  /// Get the generated Value for a given VPValue and given Part and Lane.
236 
237  bool hasVectorValue(VPValue *Def, unsigned Part) {
238  auto I = Data.PerPartOutput.find(Def);
239  return I != Data.PerPartOutput.end() && Part < I->second.size() &&
240  I->second[Part];
241  }
242 
244  return Data.PerPartOutput.find(Def) != Data.PerPartOutput.end();
245  }
246 
248  auto I = Data.PerPartScalars.find(Def);
249  if (I == Data.PerPartScalars.end())
250  return false;
251  unsigned CacheIdx = Instance.Lane.mapToCacheIndex(VF);
252  return Instance.Part < I->second.size() &&
253  CacheIdx < I->second[Instance.Part].size() &&
254  I->second[Instance.Part][CacheIdx];
255  }
256 
257  /// Set the generated Value for a given VPValue and a given Part.
258  void set(VPValue *Def, Value *V, unsigned Part) {
259  if (!Data.PerPartOutput.count(Def)) {
261  Data.PerPartOutput[Def] = Entry;
262  }
263  Data.PerPartOutput[Def][Part] = V;
264  }
265  /// Reset an existing vector value for \p Def and a given \p Part.
266  void reset(VPValue *Def, Value *V, unsigned Part) {
267  auto Iter = Data.PerPartOutput.find(Def);
268  assert(Iter != Data.PerPartOutput.end() &&
269  "need to overwrite existing value");
270  Iter->second[Part] = V;
271  }
272 
273  /// Set the generated scalar \p V for \p Def and the given \p Instance.
274  void set(VPValue *Def, Value *V, const VPIteration &Instance) {
275  auto Iter = Data.PerPartScalars.insert({Def, {}});
276  auto &PerPartVec = Iter.first->second;
277  while (PerPartVec.size() <= Instance.Part)
278  PerPartVec.emplace_back();
279  auto &Scalars = PerPartVec[Instance.Part];
280  unsigned CacheIdx = Instance.Lane.mapToCacheIndex(VF);
281  while (Scalars.size() <= CacheIdx)
282  Scalars.push_back(nullptr);
283  assert(!Scalars[CacheIdx] && "should overwrite existing value");
284  Scalars[CacheIdx] = V;
285  }
286 
287  /// Reset an existing scalar value for \p Def and a given \p Instance.
288  void reset(VPValue *Def, Value *V, const VPIteration &Instance) {
289  auto Iter = Data.PerPartScalars.find(Def);
290  assert(Iter != Data.PerPartScalars.end() &&
291  "need to overwrite existing value");
292  assert(Instance.Part < Iter->second.size() &&
293  "need to overwrite existing value");
294  unsigned CacheIdx = Instance.Lane.mapToCacheIndex(VF);
295  assert(CacheIdx < Iter->second[Instance.Part].size() &&
296  "need to overwrite existing value");
297  Iter->second[Instance.Part][CacheIdx] = V;
298  }
299 
300  /// Hold state information used when constructing the CFG of the output IR,
301  /// traversing the VPBasicBlocks and generating corresponding IR BasicBlocks.
302  struct CFGState {
303  /// The previous VPBasicBlock visited. Initially set to null.
304  VPBasicBlock *PrevVPBB = nullptr;
305 
306  /// The previous IR BasicBlock created or used. Initially set to the new
307  /// header BasicBlock.
308  BasicBlock *PrevBB = nullptr;
309 
310  /// The last IR BasicBlock in the output IR. Set to the exit block of the
311  /// vector loop.
312  BasicBlock *ExitBB = nullptr;
313 
314  /// A mapping of each VPBasicBlock to the corresponding BasicBlock. In case
315  /// of replication, maps the BasicBlock of the last replica created.
317 
318  /// Vector of VPBasicBlocks whose terminator instruction needs to be fixed
319  /// up at the end of vector code generation.
321 
322  CFGState() = default;
323 
324  /// Returns the BasicBlock* mapped to the pre-header of the loop region
325  /// containing \p R.
327  } CFG;
328 
329  /// Hold a pointer to LoopInfo to register new basic blocks in the loop.
331 
332  /// Hold a pointer to Dominator Tree to register new basic blocks in the loop.
334 
335  /// Hold a reference to the IRBuilder used to generate output IR code.
337 
339 
340  /// Hold the canonical scalar IV of the vector loop (start=0, step=VF*UF).
341  Value *CanonicalIV = nullptr;
342 
343  /// Hold a pointer to InnerLoopVectorizer to reuse its IR generation methods.
345 
346  /// Pointer to the VPlan code is generated for.
348 
349  /// Holds recipes that may generate a poison value that is used after
350  /// vectorization, even when their operands are not poison.
352 
353  /// The loop object for the current parent region, or nullptr.
355 };
356 
357 /// VPUsers instance used by VPBlockBase to manage CondBit and the block
358 /// predicate. Currently VPBlockUsers are used in VPBlockBase for historical
359 /// reasons, but in the future the only VPUsers should either be recipes or
360 /// live-outs.VPBlockBase uses.
361 struct VPBlockUser : public VPUser {
363 
365  if (getNumOperands() == 1)
366  return getOperand(0);
367 
368  return nullptr;
369  }
371  if (getNumOperands() == 1)
372  return getOperand(0);
373 
374  return nullptr;
375  }
376 
377  void resetSingleOpUser(VPValue *NewVal) {
378  assert(getNumOperands() <= 1 && "Didn't expect more than one operand!");
379  if (!NewVal) {
380  if (getNumOperands() == 1)
382  return;
383  }
384 
385  if (getNumOperands() == 1)
386  setOperand(0, NewVal);
387  else
388  addOperand(NewVal);
389  }
390 };
391 
392 /// VPBlockBase is the building block of the Hierarchical Control-Flow Graph.
393 /// A VPBlockBase can be either a VPBasicBlock or a VPRegionBlock.
394 class VPBlockBase {
395  friend class VPBlockUtils;
396 
397  const unsigned char SubclassID; ///< Subclass identifier (for isa/dyn_cast).
398 
399  /// An optional name for the block.
400  std::string Name;
401 
402  /// The immediate VPRegionBlock which this VPBlockBase belongs to, or null if
403  /// it is a topmost VPBlockBase.
404  VPRegionBlock *Parent = nullptr;
405 
406  /// List of predecessor blocks.
407  SmallVector<VPBlockBase *, 1> Predecessors;
408 
409  /// List of successor blocks.
411 
412  /// Successor selector managed by a VPUser. For blocks with zero or one
413  /// successors, there is no operand. Otherwise there is exactly one operand
414  /// which is the branch condition.
415  VPBlockUser CondBitUser;
416 
417  /// If the block is predicated, its predicate is stored as an operand of this
418  /// VPUser to maintain the def-use relations. Otherwise there is no operand
419  /// here.
420  VPBlockUser PredicateUser;
421 
422  /// VPlan containing the block. Can only be set on the entry block of the
423  /// plan.
424  VPlan *Plan = nullptr;
425 
426  /// Add \p Successor as the last successor to this block.
427  void appendSuccessor(VPBlockBase *Successor) {
428  assert(Successor && "Cannot add nullptr successor!");
429  Successors.push_back(Successor);
430  }
431 
432  /// Add \p Predecessor as the last predecessor to this block.
433  void appendPredecessor(VPBlockBase *Predecessor) {
434  assert(Predecessor && "Cannot add nullptr predecessor!");
435  Predecessors.push_back(Predecessor);
436  }
437 
438  /// Remove \p Predecessor from the predecessors of this block.
439  void removePredecessor(VPBlockBase *Predecessor) {
440  auto Pos = find(Predecessors, Predecessor);
441  assert(Pos && "Predecessor does not exist");
442  Predecessors.erase(Pos);
443  }
444 
445  /// Remove \p Successor from the successors of this block.
446  void removeSuccessor(VPBlockBase *Successor) {
447  auto Pos = find(Successors, Successor);
448  assert(Pos && "Successor does not exist");
449  Successors.erase(Pos);
450  }
451 
452 protected:
453  VPBlockBase(const unsigned char SC, const std::string &N)
454  : SubclassID(SC), Name(N) {}
455 
456 public:
457  /// An enumeration for keeping track of the concrete subclass of VPBlockBase
458  /// that are actually instantiated. Values of this enumeration are kept in the
459  /// SubclassID field of the VPBlockBase objects. They are used for concrete
460  /// type identification.
461  using VPBlockTy = enum { VPBasicBlockSC, VPRegionBlockSC };
462 
464 
465  virtual ~VPBlockBase() = default;
466 
467  const std::string &getName() const { return Name; }
468 
469  void setName(const Twine &newName) { Name = newName.str(); }
470 
471  /// \return an ID for the concrete type of this object.
472  /// This is used to implement the classof checks. This should not be used
473  /// for any other purpose, as the values may change as LLVM evolves.
474  unsigned getVPBlockID() const { return SubclassID; }
475 
476  VPRegionBlock *getParent() { return Parent; }
477  const VPRegionBlock *getParent() const { return Parent; }
478 
479  /// \return A pointer to the plan containing the current block.
480  VPlan *getPlan();
481  const VPlan *getPlan() const;
482 
483  /// Sets the pointer of the plan containing the block. The block must be the
484  /// entry block into the VPlan.
485  void setPlan(VPlan *ParentPlan);
486 
487  void setParent(VPRegionBlock *P) { Parent = P; }
488 
489  /// \return the VPBasicBlock that is the entry of this VPBlockBase,
490  /// recursively, if the latter is a VPRegionBlock. Otherwise, if this
491  /// VPBlockBase is a VPBasicBlock, it is returned.
492  const VPBasicBlock *getEntryBasicBlock() const;
494 
495  /// \return the VPBasicBlock that is the exit of this VPBlockBase,
496  /// recursively, if the latter is a VPRegionBlock. Otherwise, if this
497  /// VPBlockBase is a VPBasicBlock, it is returned.
498  const VPBasicBlock *getExitBasicBlock() const;
500 
501  const VPBlocksTy &getSuccessors() const { return Successors; }
502  VPBlocksTy &getSuccessors() { return Successors; }
503 
505 
506  const VPBlocksTy &getPredecessors() const { return Predecessors; }
507  VPBlocksTy &getPredecessors() { return Predecessors; }
508 
509  /// \return the successor of this VPBlockBase if it has a single successor.
510  /// Otherwise return a null pointer.
512  return (Successors.size() == 1 ? *Successors.begin() : nullptr);
513  }
514 
515  /// \return the predecessor of this VPBlockBase if it has a single
516  /// predecessor. Otherwise return a null pointer.
518  return (Predecessors.size() == 1 ? *Predecessors.begin() : nullptr);
519  }
520 
521  size_t getNumSuccessors() const { return Successors.size(); }
522  size_t getNumPredecessors() const { return Predecessors.size(); }
523 
524  /// An Enclosing Block of a block B is any block containing B, including B
525  /// itself. \return the closest enclosing block starting from "this", which
526  /// has successors. \return the root enclosing block if all enclosing blocks
527  /// have no successors.
529 
530  /// \return the closest enclosing block starting from "this", which has
531  /// predecessors. \return the root enclosing block if all enclosing blocks
532  /// have no predecessors.
534 
535  /// \return the successors either attached directly to this VPBlockBase or, if
536  /// this VPBlockBase is the exit block of a VPRegionBlock and has no
537  /// successors of its own, search recursively for the first enclosing
538  /// VPRegionBlock that has successors and return them. If no such
539  /// VPRegionBlock exists, return the (empty) successors of the topmost
540  /// VPBlockBase reached.
543  }
544 
545  /// \return the hierarchical successor of this VPBlockBase if it has a single
546  /// hierarchical successor. Otherwise return a null pointer.
549  }
550 
551  /// \return the predecessors either attached directly to this VPBlockBase or,
552  /// if this VPBlockBase is the entry block of a VPRegionBlock and has no
553  /// predecessors of its own, search recursively for the first enclosing
554  /// VPRegionBlock that has predecessors and return them. If no such
555  /// VPRegionBlock exists, return the (empty) predecessors of the topmost
556  /// VPBlockBase reached.
559  }
560 
561  /// \return the hierarchical predecessor of this VPBlockBase if it has a
562  /// single hierarchical predecessor. Otherwise return a null pointer.
565  }
566 
567  /// \return the condition bit selecting the successor.
568  VPValue *getCondBit();
569  /// \return the condition bit selecting the successor.
570  const VPValue *getCondBit() const;
571  /// Set the condition bit selecting the successor.
572  void setCondBit(VPValue *CV);
573 
574  /// \return the block's predicate.
576  /// \return the block's predicate.
577  const VPValue *getPredicate() const;
578  /// Set the block's predicate.
579  void setPredicate(VPValue *Pred);
580 
581  /// Set a given VPBlockBase \p Successor as the single successor of this
582  /// VPBlockBase. This VPBlockBase is not added as predecessor of \p Successor.
583  /// This VPBlockBase must have no successors.
585  assert(Successors.empty() && "Setting one successor when others exist.");
586  appendSuccessor(Successor);
587  }
588 
589  /// Set two given VPBlockBases \p IfTrue and \p IfFalse to be the two
590  /// successors of this VPBlockBase. \p Condition is set as the successor
591  /// selector. This VPBlockBase is not added as predecessor of \p IfTrue or \p
592  /// IfFalse. This VPBlockBase must have no successors.
593  void setTwoSuccessors(VPBlockBase *IfTrue, VPBlockBase *IfFalse,
594  VPValue *Condition) {
595  assert(Successors.empty() && "Setting two successors when others exist.");
596  assert(Condition && "Setting two successors without condition!");
597  setCondBit(Condition);
598  appendSuccessor(IfTrue);
599  appendSuccessor(IfFalse);
600  }
601 
602  /// Set each VPBasicBlock in \p NewPreds as predecessor of this VPBlockBase.
603  /// This VPBlockBase must have no predecessors. This VPBlockBase is not added
604  /// as successor of any VPBasicBlock in \p NewPreds.
606  assert(Predecessors.empty() && "Block predecessors already set.");
607  for (auto *Pred : NewPreds)
608  appendPredecessor(Pred);
609  }
610 
611  /// Remove all the predecessor of this block.
612  void clearPredecessors() { Predecessors.clear(); }
613 
614  /// Remove all the successors of this block and set to null its condition bit
616  Successors.clear();
617  setCondBit(nullptr);
618  }
619 
620  /// The method which generates the output IR that correspond to this
621  /// VPBlockBase, thereby "executing" the VPlan.
622  virtual void execute(struct VPTransformState *State) = 0;
623 
624  /// Delete all blocks reachable from a given VPBlockBase, inclusive.
625  static void deleteCFG(VPBlockBase *Entry);
626 
627  /// Return true if it is legal to hoist instructions into this block.
629  // There are currently no constraints that prevent an instruction to be
630  // hoisted into a VPBlockBase.
631  return true;
632  }
633 
634  /// Replace all operands of VPUsers in the block with \p NewValue and also
635  /// replaces all uses of VPValues defined in the block with NewValue.
636  virtual void dropAllReferences(VPValue *NewValue) = 0;
637 
638 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
639  void printAsOperand(raw_ostream &OS, bool PrintType) const {
640  OS << getName();
641  }
642 
643  /// Print plain-text dump of this VPBlockBase to \p O, prefixing all lines
644  /// with \p Indent. \p SlotTracker is used to print unnamed VPValue's using
645  /// consequtive numbers.
646  ///
647  /// Note that the numbering is applied to the whole VPlan, so printing
648  /// individual blocks is consistent with the whole VPlan printing.
649  virtual void print(raw_ostream &O, const Twine &Indent,
650  VPSlotTracker &SlotTracker) const = 0;
651 
652  /// Print plain-text dump of this VPlan to \p O.
653  void print(raw_ostream &O) const {
655  print(O, "", SlotTracker);
656  }
657 
658  /// Print the successors of this block to \p O, prefixing all lines with \p
659  /// Indent.
660  void printSuccessors(raw_ostream &O, const Twine &Indent) const;
661 
662  /// Dump this VPBlockBase to dbgs().
663  LLVM_DUMP_METHOD void dump() const { print(dbgs()); }
664 #endif
665 };
666 
667 /// VPRecipeBase is a base class modeling a sequence of one or more output IR
668 /// instructions. VPRecipeBase owns the the VPValues it defines through VPDef
669 /// and is responsible for deleting its defined values. Single-value
670 /// VPRecipeBases that also inherit from VPValue must make sure to inherit from
671 /// VPRecipeBase before VPValue.
672 class VPRecipeBase : public ilist_node_with_parent<VPRecipeBase, VPBasicBlock>,
673  public VPDef,
674  public VPUser {
675  friend VPBasicBlock;
676  friend class VPBlockUtils;
677 
678  /// Each VPRecipe belongs to a single VPBasicBlock.
679  VPBasicBlock *Parent = nullptr;
680 
681 public:
683  : VPDef(SC), VPUser(Operands, VPUser::VPUserID::Recipe) {}
684 
685  template <typename IterT>
687  : VPDef(SC), VPUser(Operands, VPUser::VPUserID::Recipe) {}
688  virtual ~VPRecipeBase() = default;
689 
690  /// \return the VPBasicBlock which this VPRecipe belongs to.
691  VPBasicBlock *getParent() { return Parent; }
692  const VPBasicBlock *getParent() const { return Parent; }
693 
694  /// The method which generates the output IR instructions that correspond to
695  /// this VPRecipe, thereby "executing" the VPlan.
696  virtual void execute(struct VPTransformState &State) = 0;
697 
698  /// Insert an unlinked recipe into a basic block immediately before
699  /// the specified recipe.
700  void insertBefore(VPRecipeBase *InsertPos);
701  /// Insert an unlinked recipe into \p BB immediately before the insertion
702  /// point \p IP;
704 
705  /// Insert an unlinked Recipe into a basic block immediately after
706  /// the specified Recipe.
707  void insertAfter(VPRecipeBase *InsertPos);
708 
709  /// Unlink this recipe from its current VPBasicBlock and insert it into
710  /// the VPBasicBlock that MovePos lives in, right after MovePos.
711  void moveAfter(VPRecipeBase *MovePos);
712 
713  /// Unlink this recipe and insert into BB before I.
714  ///
715  /// \pre I is a valid iterator into BB.
717 
718  /// This method unlinks 'this' from the containing basic block, but does not
719  /// delete it.
720  void removeFromParent();
721 
722  /// This method unlinks 'this' from the containing basic block and deletes it.
723  ///
724  /// \returns an iterator pointing to the element after the erased one
726 
727  /// Returns the underlying instruction, if the recipe is a VPValue or nullptr
728  /// otherwise.
730  return cast<Instruction>(getVPSingleValue()->getUnderlyingValue());
731  }
733  return cast<Instruction>(getVPSingleValue()->getUnderlyingValue());
734  }
735 
736  /// Method to support type inquiry through isa, cast, and dyn_cast.
737  static inline bool classof(const VPDef *D) {
738  // All VPDefs are also VPRecipeBases.
739  return true;
740  }
741 
742  static inline bool classof(const VPUser *U) {
743  return U->getVPUserID() == VPUser::VPUserID::Recipe;
744  }
745 
746  /// Returns true if the recipe may have side-effects.
747  bool mayHaveSideEffects() const;
748 
749  /// Returns true for PHI-like recipes.
750  bool isPhi() const {
751  return getVPDefID() >= VPFirstPHISC && getVPDefID() <= VPLastPHISC;
752  }
753 
754  /// Returns true if the recipe may read from memory.
755  bool mayReadFromMemory() const;
756 
757  /// Returns true if the recipe may write to memory.
758  bool mayWriteToMemory() const;
759 
760  /// Returns true if the recipe may read from or write to memory.
761  bool mayReadOrWriteMemory() const {
762  return mayReadFromMemory() || mayWriteToMemory();
763  }
764 };
765 
766 inline bool VPUser::classof(const VPDef *Def) {
767  return Def->getVPDefID() == VPRecipeBase::VPInstructionSC ||
768  Def->getVPDefID() == VPRecipeBase::VPWidenSC ||
769  Def->getVPDefID() == VPRecipeBase::VPWidenCallSC ||
770  Def->getVPDefID() == VPRecipeBase::VPWidenSelectSC ||
771  Def->getVPDefID() == VPRecipeBase::VPWidenGEPSC ||
772  Def->getVPDefID() == VPRecipeBase::VPBlendSC ||
773  Def->getVPDefID() == VPRecipeBase::VPInterleaveSC ||
774  Def->getVPDefID() == VPRecipeBase::VPReplicateSC ||
775  Def->getVPDefID() == VPRecipeBase::VPReductionSC ||
776  Def->getVPDefID() == VPRecipeBase::VPBranchOnMaskSC ||
777  Def->getVPDefID() == VPRecipeBase::VPWidenMemoryInstructionSC;
778 }
779 
780 /// This is a concrete Recipe that models a single VPlan-level instruction.
781 /// While as any Recipe it may generate a sequence of IR instructions when
782 /// executed, these instructions would always form a single-def expression as
783 /// the VPInstruction is also a single def-use vertex.
784 class VPInstruction : public VPRecipeBase, public VPValue {
785  friend class VPlanSlp;
786 
787 public:
788  /// VPlan opcodes, extending LLVM IR with idiomatics instructions.
789  enum {
791  Instruction::OtherOpsEnd + 1, // Combines the incoming and previous
792  // values of a first-order recurrence.
801  };
802 
803 private:
804  typedef unsigned char OpcodeTy;
805  OpcodeTy Opcode;
806  FastMathFlags FMF;
807  DebugLoc DL;
808 
809  /// Utility method serving execute(): generates a single instance of the
810  /// modeled instruction.
811  void generateInstruction(VPTransformState &State, unsigned Part);
812 
813 protected:
815 
816 public:
818  : VPRecipeBase(VPRecipeBase::VPInstructionSC, Operands),
819  VPValue(VPValue::VPVInstructionSC, nullptr, this), Opcode(Opcode),
820  DL(DL) {}
821 
822  VPInstruction(unsigned Opcode, std::initializer_list<VPValue *> Operands,
823  DebugLoc DL = {})
825 
826  /// Method to support type inquiry through isa, cast, and dyn_cast.
827  static inline bool classof(const VPValue *V) {
829  }
830 
831  VPInstruction *clone() const {
833  return new VPInstruction(Opcode, Operands, DL);
834  }
835 
836  /// Method to support type inquiry through isa, cast, and dyn_cast.
837  static inline bool classof(const VPDef *R) {
838  return R->getVPDefID() == VPRecipeBase::VPInstructionSC;
839  }
840 
841  /// Extra classof implementations to allow directly casting from VPUser ->
842  /// VPInstruction.
843  static inline bool classof(const VPUser *U) {
844  auto *R = dyn_cast<VPRecipeBase>(U);
845  return R && R->getVPDefID() == VPRecipeBase::VPInstructionSC;
846  }
847  static inline bool classof(const VPRecipeBase *R) {
848  return R->getVPDefID() == VPRecipeBase::VPInstructionSC;
849  }
850 
851  unsigned getOpcode() const { return Opcode; }
852 
853  /// Generate the instruction.
854  /// TODO: We currently execute only per-part unless a specific instance is
855  /// provided.
856  void execute(VPTransformState &State) override;
857 
858 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
859  /// Print the VPInstruction to \p O.
860  void print(raw_ostream &O, const Twine &Indent,
861  VPSlotTracker &SlotTracker) const override;
862 
863  /// Print the VPInstruction to dbgs() (for debugging).
864  LLVM_DUMP_METHOD void dump() const;
865 #endif
866 
867  /// Return true if this instruction may modify memory.
868  bool mayWriteToMemory() const {
869  // TODO: we can use attributes of the called function to rule out memory
870  // modifications.
871  return Opcode == Instruction::Store || Opcode == Instruction::Call ||
872  Opcode == Instruction::Invoke || Opcode == SLPStore;
873  }
874 
875  bool hasResult() const {
876  // CallInst may or may not have a result, depending on the called function.
877  // Conservatively return calls have results for now.
878  switch (getOpcode()) {
879  case Instruction::Ret:
880  case Instruction::Br:
881  case Instruction::Store:
882  case Instruction::Switch:
883  case Instruction::IndirectBr:
884  case Instruction::Resume:
885  case Instruction::CatchRet:
886  case Instruction::Unreachable:
887  case Instruction::Fence:
888  case Instruction::AtomicRMW:
890  return false;
891  default:
892  return true;
893  }
894  }
895 
896  /// Set the fast-math flags.
897  void setFastMathFlags(FastMathFlags FMFNew);
898 
899  /// Returns true if the recipe only uses the first lane of operand \p Op.
900  bool onlyFirstLaneUsed(const VPValue *Op) const override {
902  "Op must be an operand of the recipe");
903  if (getOperand(0) != Op)
904  return false;
905  switch (getOpcode()) {
906  default:
907  return false;
912  return true;
913  };
914  llvm_unreachable("switch should return");
915  }
916 };
917 
918 /// VPWidenRecipe is a recipe for producing a copy of vector type its
919 /// ingredient. This recipe covers most of the traditional vectorization cases
920 /// where each ingredient transforms into a vectorized version of itself.
921 class VPWidenRecipe : public VPRecipeBase, public VPValue {
922 public:
923  template <typename IterT>
925  : VPRecipeBase(VPRecipeBase::VPWidenSC, Operands),
927 
928  ~VPWidenRecipe() override = default;
929 
930  /// Method to support type inquiry through isa, cast, and dyn_cast.
931  static inline bool classof(const VPDef *D) {
932  return D->getVPDefID() == VPRecipeBase::VPWidenSC;
933  }
934  static inline bool classof(const VPValue *V) {
935  return V->getVPValueID() == VPValue::VPVWidenSC;
936  }
937 
938  /// Produce widened copies of all Ingredients.
939  void execute(VPTransformState &State) override;
940 
941 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
942  /// Print the recipe.
943  void print(raw_ostream &O, const Twine &Indent,
944  VPSlotTracker &SlotTracker) const override;
945 #endif
946 };
947 
948 /// A recipe for widening Call instructions.
949 class VPWidenCallRecipe : public VPRecipeBase, public VPValue {
950 
951 public:
952  template <typename IterT>
954  : VPRecipeBase(VPRecipeBase::VPWidenCallSC, CallArguments),
956 
957  ~VPWidenCallRecipe() override = default;
958 
959  /// Method to support type inquiry through isa, cast, and dyn_cast.
960  static inline bool classof(const VPDef *D) {
961  return D->getVPDefID() == VPRecipeBase::VPWidenCallSC;
962  }
963 
964  /// Produce a widened version of the call instruction.
965  void execute(VPTransformState &State) override;
966 
967 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
968  /// Print the recipe.
969  void print(raw_ostream &O, const Twine &Indent,
970  VPSlotTracker &SlotTracker) const override;
971 #endif
972 };
973 
974 /// A recipe for widening select instructions.
975 class VPWidenSelectRecipe : public VPRecipeBase, public VPValue {
976 
977  /// Is the condition of the select loop invariant?
978  bool InvariantCond;
979 
980 public:
981  template <typename IterT>
983  bool InvariantCond)
984  : VPRecipeBase(VPRecipeBase::VPWidenSelectSC, Operands),
986  InvariantCond(InvariantCond) {}
987 
988  ~VPWidenSelectRecipe() override = default;
989 
990  /// Method to support type inquiry through isa, cast, and dyn_cast.
991  static inline bool classof(const VPDef *D) {
992  return D->getVPDefID() == VPRecipeBase::VPWidenSelectSC;
993  }
994 
995  /// Produce a widened version of the select instruction.
996  void execute(VPTransformState &State) override;
997 
998 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
999  /// Print the recipe.
1000  void print(raw_ostream &O, const Twine &Indent,
1001  VPSlotTracker &SlotTracker) const override;
1002 #endif
1003 };
1004 
1005 /// A recipe for handling GEP instructions.
1006 class VPWidenGEPRecipe : public VPRecipeBase, public VPValue {
1007  bool IsPtrLoopInvariant;
1008  SmallBitVector IsIndexLoopInvariant;
1009 
1010 public:
1011  template <typename IterT>
1013  : VPRecipeBase(VPRecipeBase::VPWidenGEPSC, Operands),
1014  VPValue(VPWidenGEPSC, GEP, this),
1015  IsIndexLoopInvariant(GEP->getNumIndices(), false) {}
1016 
1017  template <typename IterT>
1019  Loop *OrigLoop)
1020  : VPRecipeBase(VPRecipeBase::VPWidenGEPSC, Operands),
1022  IsIndexLoopInvariant(GEP->getNumIndices(), false) {
1023  IsPtrLoopInvariant = OrigLoop->isLoopInvariant(GEP->getPointerOperand());
1024  for (auto Index : enumerate(GEP->indices()))
1025  IsIndexLoopInvariant[Index.index()] =
1026  OrigLoop->isLoopInvariant(Index.value().get());
1027  }
1028  ~VPWidenGEPRecipe() override = default;
1029 
1030  /// Method to support type inquiry through isa, cast, and dyn_cast.
1031  static inline bool classof(const VPDef *D) {
1032  return D->getVPDefID() == VPRecipeBase::VPWidenGEPSC;
1033  }
1034 
1035  /// Generate the gep nodes.
1036  void execute(VPTransformState &State) override;
1037 
1038 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1039  /// Print the recipe.
1040  void print(raw_ostream &O, const Twine &Indent,
1041  VPSlotTracker &SlotTracker) const override;
1042 #endif
1043 };
1044 
1045 /// A recipe for handling phi nodes of integer and floating-point inductions,
1046 /// producing their vector values.
1048  PHINode *IV;
1049  const InductionDescriptor &IndDesc;
1050  bool NeedsScalarIV;
1051  bool NeedsVectorIV;
1052 
1053 public:
1055  const InductionDescriptor &IndDesc,
1056  bool NeedsScalarIV, bool NeedsVectorIV)
1057  : VPRecipeBase(VPWidenIntOrFpInductionSC, {Start, Step}),
1058  VPValue(IV, this), IV(IV), IndDesc(IndDesc),
1059  NeedsScalarIV(NeedsScalarIV), NeedsVectorIV(NeedsVectorIV) {}
1060 
1062  const InductionDescriptor &IndDesc,
1063  TruncInst *Trunc, bool NeedsScalarIV,
1064  bool NeedsVectorIV)
1065  : VPRecipeBase(VPWidenIntOrFpInductionSC, {Start, Step}),
1066  VPValue(Trunc, this), IV(IV), IndDesc(IndDesc),
1067  NeedsScalarIV(NeedsScalarIV), NeedsVectorIV(NeedsVectorIV) {}
1068 
1069  ~VPWidenIntOrFpInductionRecipe() override = default;
1070 
1071  /// Method to support type inquiry through isa, cast, and dyn_cast.
1072  static inline bool classof(const VPDef *D) {
1073  return D->getVPDefID() == VPRecipeBase::VPWidenIntOrFpInductionSC;
1074  }
1075 
1076  /// Generate the vectorized and scalarized versions of the phi node as
1077  /// needed by their users.
1078  void execute(VPTransformState &State) override;
1079 
1080 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1081  /// Print the recipe.
1082  void print(raw_ostream &O, const Twine &Indent,
1083  VPSlotTracker &SlotTracker) const override;
1084 #endif
1085 
1086  /// Returns the start value of the induction.
1088  const VPValue *getStartValue() const { return getOperand(0); }
1089 
1090  /// Returns the step value of the induction.
1091  VPValue *getStepValue() { return getOperand(1); }
1092  const VPValue *getStepValue() const { return getOperand(1); }
1093 
1094  /// Returns the first defined value as TruncInst, if it is one or nullptr
1095  /// otherwise.
1097  return dyn_cast_or_null<TruncInst>(getVPValue(0)->getUnderlyingValue());
1098  }
1099  const TruncInst *getTruncInst() const {
1100  return dyn_cast_or_null<TruncInst>(getVPValue(0)->getUnderlyingValue());
1101  }
1102 
1103  PHINode *getPHINode() { return IV; }
1104 
1105  /// Returns the induction descriptor for the recipe.
1106  const InductionDescriptor &getInductionDescriptor() const { return IndDesc; }
1107 
1108  /// Returns true if the induction is canonical, i.e. starting at 0 and
1109  /// incremented by UF * VF (= the original IV is incremented by 1).
1110  bool isCanonical() const;
1111 
1112  /// Returns the scalar type of the induction.
1113  const Type *getScalarType() const {
1114  const TruncInst *TruncI = getTruncInst();
1115  return TruncI ? TruncI->getType() : IV->getType();
1116  }
1117 
1118  /// Returns true if a scalar phi needs to be created for the induction.
1119  bool needsScalarIV() const { return NeedsScalarIV; }
1120 
1121  /// Returns true if a vector phi needs to be created for the induction.
1122  bool needsVectorIV() const { return NeedsVectorIV; }
1123 };
1124 
1125 /// A pure virtual base class for all recipes modeling header phis, including
1126 /// phis for first order recurrences, pointer inductions and reductions. The
1127 /// start value is the first operand of the recipe and the incoming value from
1128 /// the backedge is the second operand.
1129 class VPHeaderPHIRecipe : public VPRecipeBase, public VPValue {
1130 protected:
1131  VPHeaderPHIRecipe(unsigned char VPVID, unsigned char VPDefID, PHINode *Phi,
1132  VPValue *Start = nullptr)
1133  : VPRecipeBase(VPDefID, {}), VPValue(VPVID, Phi, this) {
1134  if (Start)
1135  addOperand(Start);
1136  }
1137 
1138 public:
1139  ~VPHeaderPHIRecipe() override = default;
1140 
1141  /// Method to support type inquiry through isa, cast, and dyn_cast.
1142  static inline bool classof(const VPRecipeBase *B) {
1143  return B->getVPDefID() == VPRecipeBase::VPCanonicalIVPHISC ||
1144  B->getVPDefID() == VPRecipeBase::VPFirstOrderRecurrencePHISC ||
1145  B->getVPDefID() == VPRecipeBase::VPReductionPHISC ||
1146  B->getVPDefID() == VPRecipeBase::VPWidenIntOrFpInductionSC ||
1147  B->getVPDefID() == VPRecipeBase::VPWidenPHISC;
1148  }
1149  static inline bool classof(const VPValue *V) {
1150  return V->getVPValueID() == VPValue::VPVCanonicalIVPHISC ||
1155  }
1156 
1157  /// Generate the phi nodes.
1158  void execute(VPTransformState &State) override = 0;
1159 
1160 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1161  /// Print the recipe.
1162  void print(raw_ostream &O, const Twine &Indent,
1163  VPSlotTracker &SlotTracker) const override = 0;
1164 #endif
1165 
1166  /// Returns the start value of the phi, if one is set.
1168  return getNumOperands() == 0 ? nullptr : getOperand(0);
1169  }
1171  return getNumOperands() == 0 ? nullptr : getOperand(0);
1172  }
1173 
1174  /// Returns the incoming value from the loop backedge.
1176  return getOperand(1);
1177  }
1178 
1179  /// Returns the backedge value as a recipe. The backedge value is guaranteed
1180  /// to be a recipe.
1182  return cast<VPRecipeBase>(getBackedgeValue()->getDef());
1183  }
1184 };
1185 
1187  const InductionDescriptor &IndDesc;
1188 
1189  /// SCEV used to expand step.
1190  /// FIXME: move expansion of step to the pre-header, once it is modeled
1191  /// explicitly.
1192  ScalarEvolution &SE;
1193 
1194 public:
1195  /// Create a new VPWidenPointerInductionRecipe for \p Phi with start value \p
1196  /// Start.
1198  const InductionDescriptor &IndDesc,
1199  ScalarEvolution &SE)
1200  : VPHeaderPHIRecipe(VPVWidenPointerInductionSC, VPWidenPointerInductionSC,
1201  Phi),
1202  IndDesc(IndDesc), SE(SE) {
1203  addOperand(Start);
1204  }
1205 
1206  ~VPWidenPointerInductionRecipe() override = default;
1207 
1208  /// Method to support type inquiry through isa, cast, and dyn_cast.
1209  static inline bool classof(const VPRecipeBase *B) {
1210  return B->getVPDefID() == VPRecipeBase::VPWidenPointerInductionSC;
1211  }
1212  static inline bool classof(const VPHeaderPHIRecipe *R) {
1213  return R->getVPDefID() == VPRecipeBase::VPWidenPointerInductionSC;
1214  }
1215  static inline bool classof(const VPValue *V) {
1217  }
1218 
1219  /// Generate vector values for the pointer induction.
1220  void execute(VPTransformState &State) override;
1221 
1222 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1223  /// Print the recipe.
1224  void print(raw_ostream &O, const Twine &Indent,
1225  VPSlotTracker &SlotTracker) const override;
1226 #endif
1227 };
1228 
1229 /// A recipe for handling header phis that are widened in the vector loop.
1230 /// In the VPlan native path, all incoming VPValues & VPBasicBlock pairs are
1231 /// managed in the recipe directly.
1233  /// List of incoming blocks. Only used in the VPlan native path.
1234  SmallVector<VPBasicBlock *, 2> IncomingBlocks;
1235 
1236 public:
1237  /// Create a new VPWidenPHIRecipe for \p Phi with start value \p Start.
1238  VPWidenPHIRecipe(PHINode *Phi, VPValue *Start = nullptr)
1239  : VPHeaderPHIRecipe(VPVWidenPHISC, VPWidenPHISC, Phi) {
1240  if (Start)
1241  addOperand(Start);
1242  }
1243 
1244  ~VPWidenPHIRecipe() override = default;
1245 
1246  /// Method to support type inquiry through isa, cast, and dyn_cast.
1247  static inline bool classof(const VPRecipeBase *B) {
1248  return B->getVPDefID() == VPRecipeBase::VPWidenPHISC;
1249  }
1250  static inline bool classof(const VPHeaderPHIRecipe *R) {
1251  return R->getVPDefID() == VPRecipeBase::VPWidenPHISC;
1252  }
1253  static inline bool classof(const VPValue *V) {
1254  return V->getVPValueID() == VPValue::VPVWidenPHISC;
1255  }
1256 
1257  /// Generate the phi/select nodes.
1258  void execute(VPTransformState &State) override;
1259 
1260 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1261  /// Print the recipe.
1262  void print(raw_ostream &O, const Twine &Indent,
1263  VPSlotTracker &SlotTracker) const override;
1264 #endif
1265 
1266  /// Adds a pair (\p IncomingV, \p IncomingBlock) to the phi.
1267  void addIncoming(VPValue *IncomingV, VPBasicBlock *IncomingBlock) {
1268  addOperand(IncomingV);
1269  IncomingBlocks.push_back(IncomingBlock);
1270  }
1271 
1272  /// Returns the \p I th incoming VPBasicBlock.
1273  VPBasicBlock *getIncomingBlock(unsigned I) { return IncomingBlocks[I]; }
1274 
1275  /// Returns the \p I th incoming VPValue.
1276  VPValue *getIncomingValue(unsigned I) { return getOperand(I); }
1277 };
1278 
1279 /// A recipe for handling first-order recurrence phis. The start value is the
1280 /// first operand of the recipe and the incoming value from the backedge is the
1281 /// second operand.
1285  VPFirstOrderRecurrencePHISC, Phi, &Start) {}
1286 
1287  /// Method to support type inquiry through isa, cast, and dyn_cast.
1288  static inline bool classof(const VPRecipeBase *R) {
1289  return R->getVPDefID() == VPRecipeBase::VPFirstOrderRecurrencePHISC;
1290  }
1291  static inline bool classof(const VPHeaderPHIRecipe *R) {
1292  return R->getVPDefID() == VPRecipeBase::VPFirstOrderRecurrencePHISC;
1293  }
1294  static inline bool classof(const VPValue *V) {
1296  }
1297 
1298  void execute(VPTransformState &State) override;
1299 
1300 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1301  /// Print the recipe.
1302  void print(raw_ostream &O, const Twine &Indent,
1303  VPSlotTracker &SlotTracker) const override;
1304 #endif
1305 };
1306 
1307 /// A recipe for handling reduction phis. The start value is the first operand
1308 /// of the recipe and the incoming value from the backedge is the second
1309 /// operand.
1311  /// Descriptor for the reduction.
1312  const RecurrenceDescriptor &RdxDesc;
1313 
1314  /// The phi is part of an in-loop reduction.
1315  bool IsInLoop;
1316 
1317  /// The phi is part of an ordered reduction. Requires IsInLoop to be true.
1318  bool IsOrdered;
1319 
1320 public:
1321  /// Create a new VPReductionPHIRecipe for the reduction \p Phi described by \p
1322  /// RdxDesc.
1324  VPValue &Start, bool IsInLoop = false,
1325  bool IsOrdered = false)
1326  : VPHeaderPHIRecipe(VPVReductionPHISC, VPReductionPHISC, Phi, &Start),
1327  RdxDesc(RdxDesc), IsInLoop(IsInLoop), IsOrdered(IsOrdered) {
1328  assert((!IsOrdered || IsInLoop) && "IsOrdered requires IsInLoop");
1329  }
1330 
1331  ~VPReductionPHIRecipe() override = default;
1332 
1333  /// Method to support type inquiry through isa, cast, and dyn_cast.
1334  static inline bool classof(const VPRecipeBase *R) {
1335  return R->getVPDefID() == VPRecipeBase::VPReductionPHISC;
1336  }
1337  static inline bool classof(const VPHeaderPHIRecipe *R) {
1338  return R->getVPDefID() == VPRecipeBase::VPReductionPHISC;
1339  }
1340  static inline bool classof(const VPValue *V) {
1342  }
1343 
1344  /// Generate the phi/select nodes.
1345  void execute(VPTransformState &State) override;
1346 
1347 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1348  /// Print the recipe.
1349  void print(raw_ostream &O, const Twine &Indent,
1350  VPSlotTracker &SlotTracker) const override;
1351 #endif
1352 
1354  return RdxDesc;
1355  }
1356 
1357  /// Returns true, if the phi is part of an ordered reduction.
1358  bool isOrdered() const { return IsOrdered; }
1359 
1360  /// Returns true, if the phi is part of an in-loop reduction.
1361  bool isInLoop() const { return IsInLoop; }
1362 };
1363 
1364 /// A recipe for vectorizing a phi-node as a sequence of mask-based select
1365 /// instructions.
1366 class VPBlendRecipe : public VPRecipeBase, public VPValue {
1367  PHINode *Phi;
1368 
1369 public:
1370  /// The blend operation is a User of the incoming values and of their
1371  /// respective masks, ordered [I0, M0, I1, M1, ...]. Note that a single value
1372  /// might be incoming with a full mask for which there is no VPValue.
1374  : VPRecipeBase(VPBlendSC, Operands),
1375  VPValue(VPValue::VPVBlendSC, Phi, this), Phi(Phi) {
1376  assert(Operands.size() > 0 &&
1377  ((Operands.size() == 1) || (Operands.size() % 2 == 0)) &&
1378  "Expected either a single incoming value or a positive even number "
1379  "of operands");
1380  }
1381 
1382  /// Method to support type inquiry through isa, cast, and dyn_cast.
1383  static inline bool classof(const VPDef *D) {
1384  return D->getVPDefID() == VPRecipeBase::VPBlendSC;
1385  }
1386 
1387  /// Return the number of incoming values, taking into account that a single
1388  /// incoming value has no mask.
1389  unsigned getNumIncomingValues() const { return (getNumOperands() + 1) / 2; }
1390 
1391  /// Return incoming value number \p Idx.
1392  VPValue *getIncomingValue(unsigned Idx) const { return getOperand(Idx * 2); }
1393 
1394  /// Return mask number \p Idx.
1395  VPValue *getMask(unsigned Idx) const { return getOperand(Idx * 2 + 1); }
1396 
1397  /// Generate the phi/select nodes.
1398  void execute(VPTransformState &State) override;
1399 
1400 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1401  /// Print the recipe.
1402  void print(raw_ostream &O, const Twine &Indent,
1403  VPSlotTracker &SlotTracker) const override;
1404 #endif
1405 
1406  /// Returns true if the recipe only uses the first lane of operand \p Op.
1407  bool onlyFirstLaneUsed(const VPValue *Op) const override {
1409  "Op must be an operand of the recipe");
1410  // Recursing through Blend recipes only, must terminate at header phi's the
1411  // latest.
1412  return all_of(users(),
1413  [this](VPUser *U) { return U->onlyFirstLaneUsed(this); });
1414  }
1415 };
1416 
1417 /// VPInterleaveRecipe is a recipe for transforming an interleave group of load
1418 /// or stores into one wide load/store and shuffles. The first operand of a
1419 /// VPInterleave recipe is the address, followed by the stored values, followed
1420 /// by an optional mask.
1422  const InterleaveGroup<Instruction> *IG;
1423 
1424  bool HasMask = false;
1425 
1426 public:
1428  ArrayRef<VPValue *> StoredValues, VPValue *Mask)
1429  : VPRecipeBase(VPInterleaveSC, {Addr}), IG(IG) {
1430  for (unsigned i = 0; i < IG->getFactor(); ++i)
1431  if (Instruction *I = IG->getMember(i)) {
1432  if (I->getType()->isVoidTy())
1433  continue;
1434  new VPValue(I, this);
1435  }
1436 
1437  for (auto *SV : StoredValues)
1438  addOperand(SV);
1439  if (Mask) {
1440  HasMask = true;
1441  addOperand(Mask);
1442  }
1443  }
1444  ~VPInterleaveRecipe() override = default;
1445 
1446  /// Method to support type inquiry through isa, cast, and dyn_cast.
1447  static inline bool classof(const VPDef *D) {
1448  return D->getVPDefID() == VPRecipeBase::VPInterleaveSC;
1449  }
1450 
1451  /// Return the address accessed by this recipe.
1452  VPValue *getAddr() const {
1453  return getOperand(0); // Address is the 1st, mandatory operand.
1454  }
1455 
1456  /// Return the mask used by this recipe. Note that a full mask is represented
1457  /// by a nullptr.
1458  VPValue *getMask() const {
1459  // Mask is optional and therefore the last, currently 2nd operand.
1460  return HasMask ? getOperand(getNumOperands() - 1) : nullptr;
1461  }
1462 
1463  /// Return the VPValues stored by this interleave group. If it is a load
1464  /// interleave group, return an empty ArrayRef.
1466  // The first operand is the address, followed by the stored values, followed
1467  // by an optional mask.
1469  .slice(1, getNumStoreOperands());
1470  }
1471 
1472  /// Generate the wide load or store, and shuffles.
1473  void execute(VPTransformState &State) override;
1474 
1475 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1476  /// Print the recipe.
1477  void print(raw_ostream &O, const Twine &Indent,
1478  VPSlotTracker &SlotTracker) const override;
1479 #endif
1480 
1482 
1483  /// Returns the number of stored operands of this interleave group. Returns 0
1484  /// for load interleave groups.
1485  unsigned getNumStoreOperands() const {
1486  return getNumOperands() - (HasMask ? 2 : 1);
1487  }
1488 
1489  /// The recipe only uses the first lane of the address.
1490  bool onlyFirstLaneUsed(const VPValue *Op) const override {
1492  "Op must be an operand of the recipe");
1493  return Op == getAddr() && all_of(getStoredValues(), [Op](VPValue *StoredV) {
1494  return Op != StoredV;
1495  });
1496  }
1497 };
1498 
1499 /// A recipe to represent inloop reduction operations, performing a reduction on
1500 /// a vector operand into a scalar value, and adding the result to a chain.
1501 /// The Operands are {ChainOp, VecOp, [Condition]}.
1502 class VPReductionRecipe : public VPRecipeBase, public VPValue {
1503  /// The recurrence decriptor for the reduction in question.
1504  const RecurrenceDescriptor *RdxDesc;
1505  /// Pointer to the TTI, needed to create the target reduction
1506  const TargetTransformInfo *TTI;
1507 
1508 public:
1510  VPValue *ChainOp, VPValue *VecOp, VPValue *CondOp,
1511  const TargetTransformInfo *TTI)
1512  : VPRecipeBase(VPRecipeBase::VPReductionSC, {ChainOp, VecOp}),
1513  VPValue(VPValue::VPVReductionSC, I, this), RdxDesc(R), TTI(TTI) {
1514  if (CondOp)
1515  addOperand(CondOp);
1516  }
1517 
1518  ~VPReductionRecipe() override = default;
1519 
1520  /// Method to support type inquiry through isa, cast, and dyn_cast.
1521  static inline bool classof(const VPValue *V) {
1522  return V->getVPValueID() == VPValue::VPVReductionSC;
1523  }
1524 
1525  /// Generate the reduction in the loop
1526  void execute(VPTransformState &State) override;
1527 
1528 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1529  /// Print the recipe.
1530  void print(raw_ostream &O, const Twine &Indent,
1531  VPSlotTracker &SlotTracker) const override;
1532 #endif
1533 
1534  /// The VPValue of the scalar Chain being accumulated.
1535  VPValue *getChainOp() const { return getOperand(0); }
1536  /// The VPValue of the vector value to be reduced.
1537  VPValue *getVecOp() const { return getOperand(1); }
1538  /// The VPValue of the condition for the block.
1539  VPValue *getCondOp() const {
1540  return getNumOperands() > 2 ? getOperand(2) : nullptr;
1541  }
1542 };
1543 
1544 /// VPReplicateRecipe replicates a given instruction producing multiple scalar
1545 /// copies of the original scalar type, one per lane, instead of producing a
1546 /// single copy of widened type for all lanes. If the instruction is known to be
1547 /// uniform only one copy, per lane zero, will be generated.
1548 class VPReplicateRecipe : public VPRecipeBase, public VPValue {
1549  /// Indicator if only a single replica per lane is needed.
1550  bool IsUniform;
1551 
1552  /// Indicator if the replicas are also predicated.
1553  bool IsPredicated;
1554 
1555  /// Indicator if the scalar values should also be packed into a vector.
1556  bool AlsoPack;
1557 
1558 public:
1559  template <typename IterT>
1561  bool IsUniform, bool IsPredicated = false)
1562  : VPRecipeBase(VPReplicateSC, Operands), VPValue(VPVReplicateSC, I, this),
1563  IsUniform(IsUniform), IsPredicated(IsPredicated) {
1564  // Retain the previous behavior of predicateInstructions(), where an
1565  // insert-element of a predicated instruction got hoisted into the
1566  // predicated basic block iff it was its only user. This is achieved by
1567  // having predicated instructions also pack their values into a vector by
1568  // default unless they have a replicated user which uses their scalar value.
1569  AlsoPack = IsPredicated && !I->use_empty();
1570  }
1571 
1572  ~VPReplicateRecipe() override = default;
1573 
1574  /// Method to support type inquiry through isa, cast, and dyn_cast.
1575  static inline bool classof(const VPDef *D) {
1576  return D->getVPDefID() == VPRecipeBase::VPReplicateSC;
1577  }
1578 
1579  static inline bool classof(const VPValue *V) {
1580  return V->getVPValueID() == VPValue::VPVReplicateSC;
1581  }
1582 
1583  /// Generate replicas of the desired Ingredient. Replicas will be generated
1584  /// for all parts and lanes unless a specific part and lane are specified in
1585  /// the \p State.
1586  void execute(VPTransformState &State) override;
1587 
1588  void setAlsoPack(bool Pack) { AlsoPack = Pack; }
1589 
1590 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1591  /// Print the recipe.
1592  void print(raw_ostream &O, const Twine &Indent,
1593  VPSlotTracker &SlotTracker) const override;
1594 #endif
1595 
1596  bool isUniform() const { return IsUniform; }
1597 
1598  bool isPacked() const { return AlsoPack; }
1599 
1600  bool isPredicated() const { return IsPredicated; }
1601 
1602  /// Returns true if the recipe only uses the first lane of operand \p Op.
1603  bool onlyFirstLaneUsed(const VPValue *Op) const override {
1605  "Op must be an operand of the recipe");
1606  return isUniform();
1607  }
1608 
1609  /// Returns true if the recipe uses scalars of operand \p Op.
1610  bool usesScalars(const VPValue *Op) const override {
1612  "Op must be an operand of the recipe");
1613  return true;
1614  }
1615 };
1616 
1617 /// A recipe for generating conditional branches on the bits of a mask.
1619 public:
1621  : VPRecipeBase(VPBranchOnMaskSC, {}) {
1622  if (BlockInMask) // nullptr means all-one mask.
1623  addOperand(BlockInMask);
1624  }
1625 
1626  /// Method to support type inquiry through isa, cast, and dyn_cast.
1627  static inline bool classof(const VPDef *D) {
1628  return D->getVPDefID() == VPRecipeBase::VPBranchOnMaskSC;
1629  }
1630 
1631  /// Generate the extraction of the appropriate bit from the block mask and the
1632  /// conditional branch.
1633  void execute(VPTransformState &State) override;
1634 
1635 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1636  /// Print the recipe.
1637  void print(raw_ostream &O, const Twine &Indent,
1638  VPSlotTracker &SlotTracker) const override {
1639  O << Indent << "BRANCH-ON-MASK ";
1640  if (VPValue *Mask = getMask())
1641  Mask->printAsOperand(O, SlotTracker);
1642  else
1643  O << " All-One";
1644  }
1645 #endif
1646 
1647  /// Return the mask used by this recipe. Note that a full mask is represented
1648  /// by a nullptr.
1649  VPValue *getMask() const {
1650  assert(getNumOperands() <= 1 && "should have either 0 or 1 operands");
1651  // Mask is optional.
1652  return getNumOperands() == 1 ? getOperand(0) : nullptr;
1653  }
1654 };
1655 
1656 /// VPPredInstPHIRecipe is a recipe for generating the phi nodes needed when
1657 /// control converges back from a Branch-on-Mask. The phi nodes are needed in
1658 /// order to merge values that are set under such a branch and feed their uses.
1659 /// The phi nodes can be scalar or vector depending on the users of the value.
1660 /// This recipe works in concert with VPBranchOnMaskRecipe.
1661 class VPPredInstPHIRecipe : public VPRecipeBase, public VPValue {
1662 public:
1663  /// Construct a VPPredInstPHIRecipe given \p PredInst whose value needs a phi
1664  /// nodes after merging back from a Branch-on-Mask.
1666  : VPRecipeBase(VPPredInstPHISC, PredV),
1667  VPValue(VPValue::VPVPredInstPHI, nullptr, this) {}
1668  ~VPPredInstPHIRecipe() override = default;
1669 
1670  /// Method to support type inquiry through isa, cast, and dyn_cast.
1671  static inline bool classof(const VPDef *D) {
1672  return D->getVPDefID() == VPRecipeBase::VPPredInstPHISC;
1673  }
1674 
1675  /// Generates phi nodes for live-outs as needed to retain SSA form.
1676  void execute(VPTransformState &State) override;
1677 
1678 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1679  /// Print the recipe.
1680  void print(raw_ostream &O, const Twine &Indent,
1681  VPSlotTracker &SlotTracker) const override;
1682 #endif
1683 
1684  /// Returns true if the recipe uses scalars of operand \p Op.
1685  bool usesScalars(const VPValue *Op) const override {
1687  "Op must be an operand of the recipe");
1688  return true;
1689  }
1690 };
1691 
1692 /// A Recipe for widening load/store operations.
1693 /// The recipe uses the following VPValues:
1694 /// - For load: Address, optional mask
1695 /// - For store: Address, stored value, optional mask
1696 /// TODO: We currently execute only per-part unless a specific instance is
1697 /// provided.
1699  Instruction &Ingredient;
1700 
1701  // Whether the loaded-from / stored-to addresses are consecutive.
1702  bool Consecutive;
1703 
1704  // Whether the consecutive loaded/stored addresses are in reverse order.
1705  bool Reverse;
1706 
1707  void setMask(VPValue *Mask) {
1708  if (!Mask)
1709  return;
1710  addOperand(Mask);
1711  }
1712 
1713  bool isMasked() const {
1714  return isStore() ? getNumOperands() == 3 : getNumOperands() == 2;
1715  }
1716 
1717 public:
1719  bool Consecutive, bool Reverse)
1720  : VPRecipeBase(VPWidenMemoryInstructionSC, {Addr}), Ingredient(Load),
1721  Consecutive(Consecutive), Reverse(Reverse) {
1722  assert((Consecutive || !Reverse) && "Reverse implies consecutive");
1724  setMask(Mask);
1725  }
1726 
1728  VPValue *StoredValue, VPValue *Mask,
1729  bool Consecutive, bool Reverse)
1730  : VPRecipeBase(VPWidenMemoryInstructionSC, {Addr, StoredValue}),
1731  Ingredient(Store), Consecutive(Consecutive), Reverse(Reverse) {
1732  assert((Consecutive || !Reverse) && "Reverse implies consecutive");
1733  setMask(Mask);
1734  }
1735 
1736  /// Method to support type inquiry through isa, cast, and dyn_cast.
1737  static inline bool classof(const VPDef *D) {
1738  return D->getVPDefID() == VPRecipeBase::VPWidenMemoryInstructionSC;
1739  }
1740 
1741  /// Return the address accessed by this recipe.
1742  VPValue *getAddr() const {
1743  return getOperand(0); // Address is the 1st, mandatory operand.
1744  }
1745 
1746  /// Return the mask used by this recipe. Note that a full mask is represented
1747  /// by a nullptr.
1748  VPValue *getMask() const {
1749  // Mask is optional and therefore the last operand.
1750  return isMasked() ? getOperand(getNumOperands() - 1) : nullptr;
1751  }
1752 
1753  /// Returns true if this recipe is a store.
1754  bool isStore() const { return isa<StoreInst>(Ingredient); }
1755 
1756  /// Return the address accessed by this recipe.
1758  assert(isStore() && "Stored value only available for store instructions");
1759  return getOperand(1); // Stored value is the 2nd, mandatory operand.
1760  }
1761 
1762  // Return whether the loaded-from / stored-to addresses are consecutive.
1763  bool isConsecutive() const { return Consecutive; }
1764 
1765  // Return whether the consecutive loaded/stored addresses are in reverse
1766  // order.
1767  bool isReverse() const { return Reverse; }
1768 
1769  /// Generate the wide load/store.
1770  void execute(VPTransformState &State) override;
1771 
1772 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1773  /// Print the recipe.
1774  void print(raw_ostream &O, const Twine &Indent,
1775  VPSlotTracker &SlotTracker) const override;
1776 #endif
1777 
1778  /// Returns true if the recipe only uses the first lane of operand \p Op.
1779  bool onlyFirstLaneUsed(const VPValue *Op) const override {
1781  "Op must be an operand of the recipe");
1782 
1783  // Widened, consecutive memory operations only demand the first lane of
1784  // their address, unless the same operand is also stored. That latter can
1785  // happen with opaque pointers.
1786  return Op == getAddr() && isConsecutive() &&
1787  (!isStore() || Op != getStoredValue());
1788  }
1789 
1790  Instruction &getIngredient() const { return Ingredient; }
1791 };
1792 
1793 /// Recipe to expand a SCEV expression.
1794 class VPExpandSCEVRecipe : public VPRecipeBase, public VPValue {
1795  const SCEV *Expr;
1796  ScalarEvolution &SE;
1797 
1798 public:
1800  : VPRecipeBase(VPExpandSCEVSC, {}), VPValue(nullptr, this), Expr(Expr),
1801  SE(SE) {}
1802 
1803  ~VPExpandSCEVRecipe() override = default;
1804 
1805  /// Method to support type inquiry through isa, cast, and dyn_cast.
1806  static inline bool classof(const VPDef *D) {
1807  return D->getVPDefID() == VPExpandSCEVSC;
1808  }
1809 
1810  /// Generate a canonical vector induction variable of the vector loop, with
1811  void execute(VPTransformState &State) override;
1812 
1813 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1814  /// Print the recipe.
1815  void print(raw_ostream &O, const Twine &Indent,
1816  VPSlotTracker &SlotTracker) const override;
1817 #endif
1818 
1819  const SCEV *getSCEV() const { return Expr; }
1820 };
1821 
1822 /// Canonical scalar induction phi of the vector loop. Starting at the specified
1823 /// start value (either 0 or the resume value when vectorizing the epilogue
1824 /// loop). VPWidenCanonicalIVRecipe represents the vector version of the
1825 /// canonical induction variable.
1827  DebugLoc DL;
1828 
1829 public:
1831  : VPHeaderPHIRecipe(VPValue::VPVCanonicalIVPHISC, VPCanonicalIVPHISC,
1832  nullptr, StartV),
1833  DL(DL) {}
1834 
1835  ~VPCanonicalIVPHIRecipe() override = default;
1836 
1837  /// Method to support type inquiry through isa, cast, and dyn_cast.
1838  static inline bool classof(const VPDef *D) {
1839  return D->getVPDefID() == VPCanonicalIVPHISC;
1840  }
1841  static inline bool classof(const VPHeaderPHIRecipe *D) {
1842  return D->getVPDefID() == VPCanonicalIVPHISC;
1843  }
1844  static inline bool classof(const VPValue *V) {
1846  }
1847 
1848  /// Generate the canonical scalar induction phi of the vector loop.
1849  void execute(VPTransformState &State) override;
1850 
1851 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1852  /// Print the recipe.
1853  void print(raw_ostream &O, const Twine &Indent,
1854  VPSlotTracker &SlotTracker) const override;
1855 #endif
1856 
1857  /// Returns the scalar type of the induction.
1858  const Type *getScalarType() const {
1859  return getOperand(0)->getLiveInIRValue()->getType();
1860  }
1861 
1862  /// Returns true if the recipe only uses the first lane of operand \p Op.
1863  bool onlyFirstLaneUsed(const VPValue *Op) const override {
1865  "Op must be an operand of the recipe");
1866  return true;
1867  }
1868 };
1869 
1870 /// A Recipe for widening the canonical induction variable of the vector loop.
1872 public:
1874  : VPRecipeBase(VPWidenCanonicalIVSC, {CanonicalIV}),
1875  VPValue(VPValue::VPVWidenCanonicalIVSC, nullptr, this) {}
1876 
1877  ~VPWidenCanonicalIVRecipe() override = default;
1878 
1879  /// Method to support type inquiry through isa, cast, and dyn_cast.
1880  static inline bool classof(const VPDef *D) {
1881  return D->getVPDefID() == VPRecipeBase::VPWidenCanonicalIVSC;
1882  }
1883 
1884  /// Extra classof implementations to allow directly casting from VPUser ->
1885  /// VPWidenCanonicalIVRecipe.
1886  static inline bool classof(const VPUser *U) {
1887  auto *R = dyn_cast<VPRecipeBase>(U);
1888  return R && R->getVPDefID() == VPRecipeBase::VPWidenCanonicalIVSC;
1889  }
1890  static inline bool classof(const VPRecipeBase *R) {
1891  return R->getVPDefID() == VPRecipeBase::VPWidenCanonicalIVSC;
1892  }
1893 
1894  /// Generate a canonical vector induction variable of the vector loop, with
1895  /// start = {<Part*VF, Part*VF+1, ..., Part*VF+VF-1> for 0 <= Part < UF}, and
1896  /// step = <VF*UF, VF*UF, ..., VF*UF>.
1897  void execute(VPTransformState &State) override;
1898 
1899 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1900  /// Print the recipe.
1901  void print(raw_ostream &O, const Twine &Indent,
1902  VPSlotTracker &SlotTracker) const override;
1903 #endif
1904 
1905  /// Returns the scalar type of the induction.
1906  const Type *getScalarType() const {
1907  return cast<VPCanonicalIVPHIRecipe>(getOperand(0)->getDef())
1908  ->getScalarType();
1909  }
1910 };
1911 
1912 /// A recipe for handling phi nodes of integer and floating-point inductions,
1913 /// producing their scalar values.
1915  /// Scalar type to use for the generated values.
1916  Type *Ty;
1917  /// If not nullptr, truncate the generated values to TruncToTy.
1918  Type *TruncToTy;
1919  const InductionDescriptor &IndDesc;
1920 
1921 public:
1923  VPValue *CanonicalIV, VPValue *Start, VPValue *Step,
1924  Type *TruncToTy)
1925  : VPRecipeBase(VPScalarIVStepsSC, {CanonicalIV, Start, Step}),
1926  VPValue(nullptr, this), Ty(Ty), TruncToTy(TruncToTy), IndDesc(IndDesc) {
1927  }
1928 
1929  ~VPScalarIVStepsRecipe() override = default;
1930 
1931  /// Method to support type inquiry through isa, cast, and dyn_cast.
1932  static inline bool classof(const VPDef *D) {
1933  return D->getVPDefID() == VPRecipeBase::VPScalarIVStepsSC;
1934  }
1935  /// Extra classof implementations to allow directly casting from VPUser ->
1936  /// VPScalarIVStepsRecipe.
1937  static inline bool classof(const VPUser *U) {
1938  auto *R = dyn_cast<VPRecipeBase>(U);
1939  return R && R->getVPDefID() == VPRecipeBase::VPScalarIVStepsSC;
1940  }
1941  static inline bool classof(const VPRecipeBase *R) {
1942  return R->getVPDefID() == VPRecipeBase::VPScalarIVStepsSC;
1943  }
1944 
1945  /// Generate the scalarized versions of the phi node as needed by their users.
1946  void execute(VPTransformState &State) override;
1947 
1948 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1949  /// Print the recipe.
1950  void print(raw_ostream &O, const Twine &Indent,
1951  VPSlotTracker &SlotTracker) const override;
1952 #endif
1953 
1954  /// Returns true if the induction is canonical, i.e. starting at 0 and
1955  /// incremented by UF * VF (= the original IV is incremented by 1).
1956  bool isCanonical() const;
1957 
1959  VPValue *getStartValue() const { return getOperand(1); }
1960  VPValue *getStepValue() const { return getOperand(2); }
1961 
1962  /// Returns true if the recipe only uses the first lane of operand \p Op.
1963  bool onlyFirstLaneUsed(const VPValue *Op) const override {
1965  "Op must be an operand of the recipe");
1966  return true;
1967  }
1968 };
1969 
1970 /// VPBasicBlock serves as the leaf of the Hierarchical Control-Flow Graph. It
1971 /// holds a sequence of zero or more VPRecipe's each representing a sequence of
1972 /// output IR instructions. All PHI-like recipes must come before any non-PHI recipes.
1973 class VPBasicBlock : public VPBlockBase {
1974 public:
1976 
1977 private:
1978  /// The VPRecipes held in the order of output instructions to generate.
1979  RecipeListTy Recipes;
1980 
1981 public:
1982  VPBasicBlock(const Twine &Name = "", VPRecipeBase *Recipe = nullptr)
1983  : VPBlockBase(VPBasicBlockSC, Name.str()) {
1984  if (Recipe)
1985  appendRecipe(Recipe);
1986  }
1987 
1988  ~VPBasicBlock() override {
1989  while (!Recipes.empty())
1990  Recipes.pop_back();
1991  }
1992 
1993  /// Instruction iterators...
1998 
1999  //===--------------------------------------------------------------------===//
2000  /// Recipe iterator methods
2001  ///
2002  inline iterator begin() { return Recipes.begin(); }
2003  inline const_iterator begin() const { return Recipes.begin(); }
2004  inline iterator end() { return Recipes.end(); }
2005  inline const_iterator end() const { return Recipes.end(); }
2006 
2007  inline reverse_iterator rbegin() { return Recipes.rbegin(); }
2008  inline const_reverse_iterator rbegin() const { return Recipes.rbegin(); }
2009  inline reverse_iterator rend() { return Recipes.rend(); }
2010  inline const_reverse_iterator rend() const { return Recipes.rend(); }
2011 
2012  inline size_t size() const { return Recipes.size(); }
2013  inline bool empty() const { return Recipes.empty(); }
2014  inline const VPRecipeBase &front() const { return Recipes.front(); }
2015  inline VPRecipeBase &front() { return Recipes.front(); }
2016  inline const VPRecipeBase &back() const { return Recipes.back(); }
2017  inline VPRecipeBase &back() { return Recipes.back(); }
2018 
2019  /// Returns a reference to the list of recipes.
2020  RecipeListTy &getRecipeList() { return Recipes; }
2021 
2022  /// Returns a pointer to a member of the recipe list.
2024  return &VPBasicBlock::Recipes;
2025  }
2026 
2027  /// Method to support type inquiry through isa, cast, and dyn_cast.
2028  static inline bool classof(const VPBlockBase *V) {
2029  return V->getVPBlockID() == VPBlockBase::VPBasicBlockSC;
2030  }
2031 
2032  void insert(VPRecipeBase *Recipe, iterator InsertPt) {
2033  assert(Recipe && "No recipe to append.");
2034  assert(!Recipe->Parent && "Recipe already in VPlan");
2035  Recipe->Parent = this;
2036  Recipes.insert(InsertPt, Recipe);
2037  }
2038 
2039  /// Augment the existing recipes of a VPBasicBlock with an additional
2040  /// \p Recipe as the last recipe.
2041  void appendRecipe(VPRecipeBase *Recipe) { insert(Recipe, end()); }
2042 
2043  /// The method which generates the output IR instructions that correspond to
2044  /// this VPBasicBlock, thereby "executing" the VPlan.
2045  void execute(struct VPTransformState *State) override;
2046 
2047  /// Return the position of the first non-phi node recipe in the block.
2049 
2050  /// Returns an iterator range over the PHI-like recipes in the block.
2052  return make_range(begin(), getFirstNonPhi());
2053  }
2054 
2055  void dropAllReferences(VPValue *NewValue) override;
2056 
2057  /// Split current block at \p SplitAt by inserting a new block between the
2058  /// current block and its successors and moving all recipes starting at
2059  /// SplitAt to the new block. Returns the new block.
2060  VPBasicBlock *splitAt(iterator SplitAt);
2061 
2063 
2064 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2065  /// Print this VPBsicBlock to \p O, prefixing all lines with \p Indent. \p
2066  /// SlotTracker is used to print unnamed VPValue's using consequtive numbers.
2067  ///
2068  /// Note that the numbering is applied to the whole VPlan, so printing
2069  /// individual blocks is consistent with the whole VPlan printing.
2070  void print(raw_ostream &O, const Twine &Indent,
2071  VPSlotTracker &SlotTracker) const override;
2072  using VPBlockBase::print; // Get the print(raw_stream &O) version.
2073 #endif
2074 
2075 private:
2076  /// Create an IR BasicBlock to hold the output instructions generated by this
2077  /// VPBasicBlock, and return it. Update the CFGState accordingly.
2078  BasicBlock *createEmptyBasicBlock(VPTransformState::CFGState &CFG);
2079 };
2080 
2081 /// VPRegionBlock represents a collection of VPBasicBlocks and VPRegionBlocks
2082 /// which form a Single-Entry-Single-Exit subgraph of the output IR CFG.
2083 /// A VPRegionBlock may indicate that its contents are to be replicated several
2084 /// times. This is designed to support predicated scalarization, in which a
2085 /// scalar if-then code structure needs to be generated VF * UF times. Having
2086 /// this replication indicator helps to keep a single model for multiple
2087 /// candidate VF's. The actual replication takes place only once the desired VF
2088 /// and UF have been determined.
2089 class VPRegionBlock : public VPBlockBase {
2090  /// Hold the Single Entry of the SESE region modelled by the VPRegionBlock.
2091  VPBlockBase *Entry;
2092 
2093  /// Hold the Single Exit of the SESE region modelled by the VPRegionBlock.
2094  VPBlockBase *Exit;
2095 
2096  /// An indicator whether this region is to generate multiple replicated
2097  /// instances of output IR corresponding to its VPBlockBases.
2098  bool IsReplicator;
2099 
2100 public:
2102  const std::string &Name = "", bool IsReplicator = false)
2103  : VPBlockBase(VPRegionBlockSC, Name), Entry(Entry), Exit(Exit),
2104  IsReplicator(IsReplicator) {
2105  assert(Entry->getPredecessors().empty() && "Entry block has predecessors.");
2106  assert(Exit->getSuccessors().empty() && "Exit block has successors.");
2107  Entry->setParent(this);
2108  Exit->setParent(this);
2109  }
2110  VPRegionBlock(const std::string &Name = "", bool IsReplicator = false)
2111  : VPBlockBase(VPRegionBlockSC, Name), Entry(nullptr), Exit(nullptr),
2112  IsReplicator(IsReplicator) {}
2113 
2114  ~VPRegionBlock() override {
2115  if (Entry) {
2116  VPValue DummyValue;
2117  Entry->dropAllReferences(&DummyValue);
2118  deleteCFG(Entry);
2119  }
2120  }
2121 
2122  /// Method to support type inquiry through isa, cast, and dyn_cast.
2123  static inline bool classof(const VPBlockBase *V) {
2124  return V->getVPBlockID() == VPBlockBase::VPRegionBlockSC;
2125  }
2126 
2127  const VPBlockBase *getEntry() const { return Entry; }
2128  VPBlockBase *getEntry() { return Entry; }
2129 
2130  /// Set \p EntryBlock as the entry VPBlockBase of this VPRegionBlock. \p
2131  /// EntryBlock must have no predecessors.
2132  void setEntry(VPBlockBase *EntryBlock) {
2133  assert(EntryBlock->getPredecessors().empty() &&
2134  "Entry block cannot have predecessors.");
2135  Entry = EntryBlock;
2136  EntryBlock->setParent(this);
2137  }
2138 
2139  // FIXME: DominatorTreeBase is doing 'A->getParent()->front()'. 'front' is a
2140  // specific interface of llvm::Function, instead of using
2141  // GraphTraints::getEntryNode. We should add a new template parameter to
2142  // DominatorTreeBase representing the Graph type.
2143  VPBlockBase &front() const { return *Entry; }
2144 
2145  const VPBlockBase *getExit() const { return Exit; }
2146  VPBlockBase *getExit() { return Exit; }
2147 
2148  /// Set \p ExitBlock as the exit VPBlockBase of this VPRegionBlock. \p
2149  /// ExitBlock must have no successors.
2150  void setExit(VPBlockBase *ExitBlock) {
2151  assert(ExitBlock->getSuccessors().empty() &&
2152  "Exit block cannot have successors.");
2153  Exit = ExitBlock;
2154  ExitBlock->setParent(this);
2155  }
2156 
2157  /// Returns the pre-header VPBasicBlock of the loop region.
2159  assert(!isReplicator() && "should only get pre-header of loop regions");
2161  }
2162 
2163  /// An indicator whether this region is to generate multiple replicated
2164  /// instances of output IR corresponding to its VPBlockBases.
2165  bool isReplicator() const { return IsReplicator; }
2166 
2167  /// The method which generates the output IR instructions that correspond to
2168  /// this VPRegionBlock, thereby "executing" the VPlan.
2169  void execute(struct VPTransformState *State) override;
2170 
2171  void dropAllReferences(VPValue *NewValue) override;
2172 
2173 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2174  /// Print this VPRegionBlock to \p O (recursively), prefixing all lines with
2175  /// \p Indent. \p SlotTracker is used to print unnamed VPValue's using
2176  /// consequtive numbers.
2177  ///
2178  /// Note that the numbering is applied to the whole VPlan, so printing
2179  /// individual regions is consistent with the whole VPlan printing.
2180  void print(raw_ostream &O, const Twine &Indent,
2181  VPSlotTracker &SlotTracker) const override;
2182  using VPBlockBase::print; // Get the print(raw_stream &O) version.
2183 #endif
2184 };
2185 
2186 //===----------------------------------------------------------------------===//
2187 // GraphTraits specializations for VPlan Hierarchical Control-Flow Graphs //
2188 //===----------------------------------------------------------------------===//
2189 
2190 // The following set of template specializations implement GraphTraits to treat
2191 // any VPBlockBase as a node in a graph of VPBlockBases. It's important to note
2192 // that VPBlockBase traits don't recurse into VPRegioBlocks, i.e., if the
2193 // VPBlockBase is a VPRegionBlock, this specialization provides access to its
2194 // successors/predecessors but not to the blocks inside the region.
2195 
2196 template <> struct GraphTraits<VPBlockBase *> {
2199 
2200  static NodeRef getEntryNode(NodeRef N) { return N; }
2201 
2203  return N->getSuccessors().begin();
2204  }
2205 
2207  return N->getSuccessors().end();
2208  }
2209 };
2210 
2211 template <> struct GraphTraits<const VPBlockBase *> {
2212  using NodeRef = const VPBlockBase *;
2214 
2215  static NodeRef getEntryNode(NodeRef N) { return N; }
2216 
2218  return N->getSuccessors().begin();
2219  }
2220 
2222  return N->getSuccessors().end();
2223  }
2224 };
2225 
2226 // Inverse order specialization for VPBasicBlocks. Predecessors are used instead
2227 // of successors for the inverse traversal.
2228 template <> struct GraphTraits<Inverse<VPBlockBase *>> {
2231 
2232  static NodeRef getEntryNode(Inverse<NodeRef> B) { return B.Graph; }
2233 
2235  return N->getPredecessors().begin();
2236  }
2237 
2239  return N->getPredecessors().end();
2240  }
2241 };
2242 
2243 // The following set of template specializations implement GraphTraits to
2244 // treat VPRegionBlock as a graph and recurse inside its nodes. It's important
2245 // to note that the blocks inside the VPRegionBlock are treated as VPBlockBases
2246 // (i.e., no dyn_cast is performed, VPBlockBases specialization is used), so
2247 // there won't be automatic recursion into other VPBlockBases that turn to be
2248 // VPRegionBlocks.
2249 
2250 template <>
2254 
2255  static NodeRef getEntryNode(GraphRef N) { return N->getEntry(); }
2256 
2258  return nodes_iterator::begin(N->getEntry());
2259  }
2260 
2262  // df_iterator::end() returns an empty iterator so the node used doesn't
2263  // matter.
2264  return nodes_iterator::end(N);
2265  }
2266 };
2267 
2268 template <>
2271  using GraphRef = const VPRegionBlock *;
2273 
2274  static NodeRef getEntryNode(GraphRef N) { return N->getEntry(); }
2275 
2277  return nodes_iterator::begin(N->getEntry());
2278  }
2279 
2281  // df_iterator::end() returns an empty iterator so the node used doesn't
2282  // matter.
2283  return nodes_iterator::end(N);
2284  }
2285 };
2286 
2287 template <>
2292 
2294  return N.Graph->getExit();
2295  }
2296 
2298  return nodes_iterator::begin(N->getExit());
2299  }
2300 
2302  // df_iterator::end() returns an empty iterator so the node used doesn't
2303  // matter.
2304  return nodes_iterator::end(N);
2305  }
2306 };
2307 
2308 /// Iterator to traverse all successors of a VPBlockBase node. This includes the
2309 /// entry node of VPRegionBlocks. Exit blocks of a region implicitly have their
2310 /// parent region's successors. This ensures all blocks in a region are visited
2311 /// before any blocks in a successor region when doing a reverse post-order
2312 // traversal of the graph.
2313 template <typename BlockPtrTy>
2315  : public iterator_facade_base<VPAllSuccessorsIterator<BlockPtrTy>,
2316  std::forward_iterator_tag, VPBlockBase> {
2317  BlockPtrTy Block;
2318  /// Index of the current successor. For VPBasicBlock nodes, this simply is the
2319  /// index for the successor array. For VPRegionBlock, SuccessorIdx == 0 is
2320  /// used for the region's entry block, and SuccessorIdx - 1 are the indices
2321  /// for the successor array.
2322  size_t SuccessorIdx;
2323 
2324  static BlockPtrTy getBlockWithSuccs(BlockPtrTy Current) {
2325  while (Current && Current->getNumSuccessors() == 0)
2326  Current = Current->getParent();
2327  return Current;
2328  }
2329 
2330  /// Templated helper to dereference successor \p SuccIdx of \p Block. Used by
2331  /// both the const and non-const operator* implementations.
2332  template <typename T1> static T1 deref(T1 Block, unsigned SuccIdx) {
2333  if (auto *R = dyn_cast<VPRegionBlock>(Block)) {
2334  if (SuccIdx == 0)
2335  return R->getEntry();
2336  SuccIdx--;
2337  }
2338 
2339  // For exit blocks, use the next parent region with successors.
2340  return getBlockWithSuccs(Block)->getSuccessors()[SuccIdx];
2341  }
2342 
2343 public:
2344  VPAllSuccessorsIterator(BlockPtrTy Block, size_t Idx = 0)
2345  : Block(Block), SuccessorIdx(Idx) {}
2347  : Block(Other.Block), SuccessorIdx(Other.SuccessorIdx) {}
2348 
2350  Block = R.Block;
2351  SuccessorIdx = R.SuccessorIdx;
2352  return *this;
2353  }
2354 
2355  static VPAllSuccessorsIterator end(BlockPtrTy Block) {
2356  BlockPtrTy ParentWithSuccs = getBlockWithSuccs(Block);
2357  unsigned NumSuccessors = ParentWithSuccs
2358  ? ParentWithSuccs->getNumSuccessors()
2359  : Block->getNumSuccessors();
2360 
2361  if (auto *R = dyn_cast<VPRegionBlock>(Block))
2362  return {R, NumSuccessors + 1};
2363  return {Block, NumSuccessors};
2364  }
2365 
2366  bool operator==(const VPAllSuccessorsIterator &R) const {
2367  return Block == R.Block && SuccessorIdx == R.SuccessorIdx;
2368  }
2369 
2370  const VPBlockBase *operator*() const { return deref(Block, SuccessorIdx); }
2371 
2372  BlockPtrTy operator*() { return deref(Block, SuccessorIdx); }
2373 
2375  SuccessorIdx++;
2376  return *this;
2377  }
2378 
2380  VPAllSuccessorsIterator Orig = *this;
2381  SuccessorIdx++;
2382  return Orig;
2383  }
2384 };
2385 
2386 /// Helper for GraphTraits specialization that traverses through VPRegionBlocks.
2387 template <typename BlockTy> class VPBlockRecursiveTraversalWrapper {
2388  BlockTy Entry;
2389 
2390 public:
2392  BlockTy getEntry() { return Entry; }
2393 };
2394 
2395 /// GraphTraits specialization to recursively traverse VPBlockBase nodes,
2396 /// including traversing through VPRegionBlocks. Exit blocks of a region
2397 /// implicitly have their parent region's successors. This ensures all blocks in
2398 /// a region are visited before any blocks in a successor region when doing a
2399 /// reverse post-order traversal of the graph.
2400 template <>
2404 
2405  static NodeRef
2407  return N.getEntry();
2408  }
2409 
2411  return ChildIteratorType(N);
2412  }
2413 
2415  return ChildIteratorType::end(N);
2416  }
2417 };
2418 
2419 template <>
2421  using NodeRef = const VPBlockBase *;
2423 
2424  static NodeRef
2426  return N.getEntry();
2427  }
2428 
2430  return ChildIteratorType(N);
2431  }
2432 
2434  return ChildIteratorType::end(N);
2435  }
2436 };
2437 
2438 /// VPlan models a candidate for vectorization, encoding various decisions take
2439 /// to produce efficient output IR, including which branches, basic-blocks and
2440 /// output IR instructions to generate, and their cost. VPlan holds a
2441 /// Hierarchical-CFG of VPBasicBlocks and VPRegionBlocks rooted at an Entry
2442 /// VPBlock.
2443 class VPlan {
2444  friend class VPlanPrinter;
2445  friend class VPSlotTracker;
2446 
2447  /// Hold the single entry to the Hierarchical CFG of the VPlan.
2448  VPBlockBase *Entry;
2449 
2450  /// Holds the VFs applicable to this VPlan.
2452 
2453  /// Holds the name of the VPlan, for printing.
2454  std::string Name;
2455 
2456  /// Holds all the external definitions created for this VPlan. External
2457  /// definitions must be immutable and hold a pointer to their underlying IR.
2458  DenseMap<Value *, VPValue *> VPExternalDefs;
2459 
2460  /// Represents the trip count of the original loop, for folding
2461  /// the tail.
2462  VPValue *TripCount = nullptr;
2463 
2464  /// Represents the backedge taken count of the original loop, for folding
2465  /// the tail. It equals TripCount - 1.
2466  VPValue *BackedgeTakenCount = nullptr;
2467 
2468  /// Represents the vector trip count.
2469  VPValue VectorTripCount;
2470 
2471  /// Holds a mapping between Values and their corresponding VPValue inside
2472  /// VPlan.
2473  Value2VPValueTy Value2VPValue;
2474 
2475  /// Contains all VPValues that been allocated by addVPValue directly and need
2476  /// to be free when the plan's destructor is called.
2477  SmallVector<VPValue *, 16> VPValuesToFree;
2478 
2479  /// Holds the VPLoopInfo analysis for this VPlan.
2480  VPLoopInfo VPLInfo;
2481 
2482  /// Indicates whether it is safe use the Value2VPValue mapping or if the
2483  /// mapping cannot be used any longer, because it is stale.
2484  bool Value2VPValueEnabled = true;
2485 
2486 public:
2487  VPlan(VPBlockBase *Entry = nullptr) : Entry(Entry) {
2488  if (Entry)
2489  Entry->setPlan(this);
2490  }
2491 
2493  if (Entry) {
2494  VPValue DummyValue;
2495  for (VPBlockBase *Block : depth_first(Entry))
2496  Block->dropAllReferences(&DummyValue);
2497 
2498  VPBlockBase::deleteCFG(Entry);
2499  }
2500  for (VPValue *VPV : VPValuesToFree)
2501  delete VPV;
2502  if (TripCount)
2503  delete TripCount;
2504  if (BackedgeTakenCount)
2505  delete BackedgeTakenCount;
2506  for (auto &P : VPExternalDefs)
2507  delete P.second;
2508  }
2509 
2510  /// Prepare the plan for execution, setting up the required live-in values.
2511  void prepareToExecute(Value *TripCount, Value *VectorTripCount,
2512  Value *CanonicalIVStartValue, VPTransformState &State);
2513 
2514  /// Generate the IR code for this VPlan.
2515  void execute(struct VPTransformState *State);
2516 
2517  VPBlockBase *getEntry() { return Entry; }
2518  const VPBlockBase *getEntry() const { return Entry; }
2519 
2521  Entry = Block;
2522  Block->setPlan(this);
2523  return Entry;
2524  }
2525 
2526  /// The trip count of the original loop.
2528  if (!TripCount)
2529  TripCount = new VPValue();
2530  return TripCount;
2531  }
2532 
2533  /// The backedge taken count of the original loop.
2535  if (!BackedgeTakenCount)
2536  BackedgeTakenCount = new VPValue();
2537  return BackedgeTakenCount;
2538  }
2539 
2540  /// The vector trip count.
2541  VPValue &getVectorTripCount() { return VectorTripCount; }
2542 
2543  /// Mark the plan to indicate that using Value2VPValue is not safe any
2544  /// longer, because it may be stale.
2545  void disableValue2VPValue() { Value2VPValueEnabled = false; }
2546 
2547  void addVF(ElementCount VF) { VFs.insert(VF); }
2548 
2549  bool hasVF(ElementCount VF) { return VFs.count(VF); }
2550 
2551  const std::string &getName() const { return Name; }
2552 
2553  void setName(const Twine &newName) { Name = newName.str(); }
2554 
2555  /// Get the existing or add a new external definition for \p V.
2557  auto I = VPExternalDefs.insert({V, nullptr});
2558  if (I.second)
2559  I.first->second = new VPValue(V);
2560  return I.first->second;
2561  }
2562 
2563  void addVPValue(Value *V) {
2564  assert(Value2VPValueEnabled &&
2565  "IR value to VPValue mapping may be out of date!");
2566  assert(V && "Trying to add a null Value to VPlan");
2567  assert(!Value2VPValue.count(V) && "Value already exists in VPlan");
2568  VPValue *VPV = new VPValue(V);
2569  Value2VPValue[V] = VPV;
2570  VPValuesToFree.push_back(VPV);
2571  }
2572 
2573  void addVPValue(Value *V, VPValue *VPV) {
2574  assert(Value2VPValueEnabled && "Value2VPValue mapping may be out of date!");
2575  assert(V && "Trying to add a null Value to VPlan");
2576  assert(!Value2VPValue.count(V) && "Value already exists in VPlan");
2577  Value2VPValue[V] = VPV;
2578  }
2579 
2580  /// Returns the VPValue for \p V. \p OverrideAllowed can be used to disable
2581  /// checking whether it is safe to query VPValues using IR Values.
2582  VPValue *getVPValue(Value *V, bool OverrideAllowed = false) {
2583  assert((OverrideAllowed || isa<Constant>(V) || Value2VPValueEnabled) &&
2584  "Value2VPValue mapping may be out of date!");
2585  assert(V && "Trying to get the VPValue of a null Value");
2586  assert(Value2VPValue.count(V) && "Value does not exist in VPlan");
2587  return Value2VPValue[V];
2588  }
2589 
2590  /// Gets the VPValue or adds a new one (if none exists yet) for \p V. \p
2591  /// OverrideAllowed can be used to disable checking whether it is safe to
2592  /// query VPValues using IR Values.
2593  VPValue *getOrAddVPValue(Value *V, bool OverrideAllowed = false) {
2594  assert((OverrideAllowed || isa<Constant>(V) || Value2VPValueEnabled) &&
2595  "Value2VPValue mapping may be out of date!");
2596  assert(V && "Trying to get or add the VPValue of a null Value");
2597  if (!Value2VPValue.count(V))
2598  addVPValue(V);
2599  return getVPValue(V);
2600  }
2601 
2603  assert(Value2VPValueEnabled &&
2604  "IR value to VPValue mapping may be out of date!");
2605  Value2VPValue.erase(V);
2606  }
2607 
2608  /// Return the VPLoopInfo analysis for this VPlan.
2609  VPLoopInfo &getVPLoopInfo() { return VPLInfo; }
2610  const VPLoopInfo &getVPLoopInfo() const { return VPLInfo; }
2611 
2612 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2613  /// Print this VPlan to \p O.
2614  void print(raw_ostream &O) const;
2615 
2616  /// Print this VPlan in DOT format to \p O.
2617  void printDOT(raw_ostream &O) const;
2618 
2619  /// Dump the plan to stderr (for debugging).
2620  LLVM_DUMP_METHOD void dump() const;
2621 #endif
2622 
2623  /// Returns a range mapping the values the range \p Operands to their
2624  /// corresponding VPValues.
2627  std::function<VPValue *(Value *)> Fn = [this](Value *Op) {
2628  return getOrAddVPValue(Op);
2629  };
2630  return map_range(Operands, Fn);
2631  }
2632 
2633  /// Returns true if \p VPV is uniform after vectorization.
2635  auto RepR = dyn_cast_or_null<VPReplicateRecipe>(VPV->getDef());
2636  return !VPV->getDef() || (RepR && RepR->isUniform());
2637  }
2638 
2639  /// Returns the VPRegionBlock of the vector loop.
2641  if (auto *R = dyn_cast<VPRegionBlock>(getEntry()))
2642  return R;
2643  return cast<VPRegionBlock>(getEntry()->getSingleSuccessor());
2644  }
2646  if (auto *R = dyn_cast<VPRegionBlock>(getEntry()))
2647  return R;
2648  return cast<VPRegionBlock>(getEntry()->getSingleSuccessor());
2649  }
2650 
2651  /// Returns the canonical induction recipe of the vector loop.
2654  if (EntryVPBB->empty()) {
2655  // VPlan native path.
2656  EntryVPBB = cast<VPBasicBlock>(EntryVPBB->getSingleSuccessor());
2657  }
2658  return cast<VPCanonicalIVPHIRecipe>(&*EntryVPBB->begin());
2659  }
2660 
2661 private:
2662  /// Add to the given dominator tree the header block and every new basic block
2663  /// that was created between it and the latch block, inclusive.
2664  static void updateDominatorTree(DominatorTree *DT, BasicBlock *LoopLatchBB,
2665  BasicBlock *LoopPreHeaderBB,
2666  BasicBlock *LoopExitBB);
2667 };
2668 
2669 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2670 /// VPlanPrinter prints a given VPlan to a given output stream. The printing is
2671 /// indented and follows the dot format.
2673  raw_ostream &OS;
2674  const VPlan &Plan;
2675  unsigned Depth = 0;
2676  unsigned TabWidth = 2;
2677  std::string Indent;
2678  unsigned BID = 0;
2680 
2682 
2683  /// Handle indentation.
2684  void bumpIndent(int b) { Indent = std::string((Depth += b) * TabWidth, ' '); }
2685 
2686  /// Print a given \p Block of the Plan.
2687  void dumpBlock(const VPBlockBase *Block);
2688 
2689  /// Print the information related to the CFG edges going out of a given
2690  /// \p Block, followed by printing the successor blocks themselves.
2691  void dumpEdges(const VPBlockBase *Block);
2692 
2693  /// Print a given \p BasicBlock, including its VPRecipes, followed by printing
2694  /// its successor blocks.
2695  void dumpBasicBlock(const VPBasicBlock *BasicBlock);
2696 
2697  /// Print a given \p Region of the Plan.
2698  void dumpRegion(const VPRegionBlock *Region);
2699 
2700  unsigned getOrCreateBID(const VPBlockBase *Block) {
2701  return BlockID.count(Block) ? BlockID[Block] : BlockID[Block] = BID++;
2702  }
2703 
2704  Twine getOrCreateName(const VPBlockBase *Block);
2705 
2706  Twine getUID(const VPBlockBase *Block);
2707 
2708  /// Print the information related to a CFG edge between two VPBlockBases.
2709  void drawEdge(const VPBlockBase *From, const VPBlockBase *To, bool Hidden,
2710  const Twine &Label);
2711 
2712 public:
2714  : OS(O), Plan(P), SlotTracker(&P) {}
2715 
2716  LLVM_DUMP_METHOD void dump();
2717 };
2718 
2720  const Value *V;
2721 
2722  VPlanIngredient(const Value *V) : V(V) {}
2723 
2724  void print(raw_ostream &O) const;
2725 };
2726 
2728  I.print(OS);
2729  return OS;
2730 }
2731 
2732 inline raw_ostream &operator<<(raw_ostream &OS, const VPlan &Plan) {
2733  Plan.print(OS);
2734  return OS;
2735 }
2736 #endif
2737 
2738 //===----------------------------------------------------------------------===//
2739 // VPlan Utilities
2740 //===----------------------------------------------------------------------===//
2741 
2742 /// Class that provides utilities for VPBlockBases in VPlan.
2744 public:
2745  VPBlockUtils() = delete;
2746 
2747  /// Insert disconnected VPBlockBase \p NewBlock after \p BlockPtr. Add \p
2748  /// NewBlock as successor of \p BlockPtr and \p BlockPtr as predecessor of \p
2749  /// NewBlock, and propagate \p BlockPtr parent to \p NewBlock. \p BlockPtr's
2750  /// successors are moved from \p BlockPtr to \p NewBlock and \p BlockPtr's
2751  /// conditional bit is propagated to \p NewBlock. \p NewBlock must have
2752  /// neither successors nor predecessors.
2753  static void insertBlockAfter(VPBlockBase *NewBlock, VPBlockBase *BlockPtr) {
2754  assert(NewBlock->getSuccessors().empty() &&
2755  NewBlock->getPredecessors().empty() &&
2756  "Can't insert new block with predecessors or successors.");
2757  NewBlock->setParent(BlockPtr->getParent());
2758  SmallVector<VPBlockBase *> Succs(BlockPtr->successors());
2759  for (VPBlockBase *Succ : Succs) {
2760  disconnectBlocks(BlockPtr, Succ);
2761  connectBlocks(NewBlock, Succ);
2762  }
2763  NewBlock->setCondBit(BlockPtr->getCondBit());
2764  BlockPtr->setCondBit(nullptr);
2765  connectBlocks(BlockPtr, NewBlock);
2766  }
2767 
2768  /// Insert disconnected VPBlockBases \p IfTrue and \p IfFalse after \p
2769  /// BlockPtr. Add \p IfTrue and \p IfFalse as succesors of \p BlockPtr and \p
2770  /// BlockPtr as predecessor of \p IfTrue and \p IfFalse. Propagate \p BlockPtr
2771  /// parent to \p IfTrue and \p IfFalse. \p Condition is set as the successor
2772  /// selector. \p BlockPtr must have no successors and \p IfTrue and \p IfFalse
2773  /// must have neither successors nor predecessors.
2774  static void insertTwoBlocksAfter(VPBlockBase *IfTrue, VPBlockBase *IfFalse,
2775  VPValue *Condition, VPBlockBase *BlockPtr) {
2776  assert(IfTrue->getSuccessors().empty() &&
2777  "Can't insert IfTrue with successors.");
2778  assert(IfFalse->getSuccessors().empty() &&
2779  "Can't insert IfFalse with successors.");
2780  BlockPtr->setTwoSuccessors(IfTrue, IfFalse, Condition);
2781  IfTrue->setPredecessors({BlockPtr});
2782  IfFalse->setPredecessors({BlockPtr});
2783  IfTrue->setParent(BlockPtr->getParent());
2784  IfFalse->setParent(BlockPtr->getParent());
2785  }
2786 
2787  /// Connect VPBlockBases \p From and \p To bi-directionally. Append \p To to
2788  /// the successors of \p From and \p From to the predecessors of \p To. Both
2789  /// VPBlockBases must have the same parent, which can be null. Both
2790  /// VPBlockBases can be already connected to other VPBlockBases.
2792  assert((From->getParent() == To->getParent()) &&
2793  "Can't connect two block with different parents");
2794  assert(From->getNumSuccessors() < 2 &&
2795  "Blocks can't have more than two successors.");
2796  From->appendSuccessor(To);
2797  To->appendPredecessor(From);
2798  }
2799 
2800  /// Disconnect VPBlockBases \p From and \p To bi-directionally. Remove \p To
2801  /// from the successors of \p From and \p From from the predecessors of \p To.
2803  assert(To && "Successor to disconnect is null.");
2804  From->removeSuccessor(To);
2805  To->removePredecessor(From);
2806  }
2807 
2808  /// Try to merge \p Block into its single predecessor, if \p Block is a
2809  /// VPBasicBlock and its predecessor has a single successor. Returns a pointer
2810  /// to the predecessor \p Block was merged into or nullptr otherwise.
2812  auto *VPBB = dyn_cast<VPBasicBlock>(Block);
2813  auto *PredVPBB =
2814  dyn_cast_or_null<VPBasicBlock>(Block->getSinglePredecessor());
2815  if (!VPBB || !PredVPBB || PredVPBB->getNumSuccessors() != 1)
2816  return nullptr;
2817 
2818  for (VPRecipeBase &R : make_early_inc_range(*VPBB))
2819  R.moveBefore(*PredVPBB, PredVPBB->end());
2820  VPBlockUtils::disconnectBlocks(PredVPBB, VPBB);
2821  auto *ParentRegion = cast<VPRegionBlock>(Block->getParent());
2822  if (ParentRegion->getExit() == Block)
2823  ParentRegion->setExit(PredVPBB);
2824  SmallVector<VPBlockBase *> Successors(Block->successors());
2825  for (auto *Succ : Successors) {
2826  VPBlockUtils::disconnectBlocks(Block, Succ);
2827  VPBlockUtils::connectBlocks(PredVPBB, Succ);
2828  }
2829  delete Block;
2830  return PredVPBB;
2831  }
2832 
2833  /// Returns true if the edge \p FromBlock -> \p ToBlock is a back-edge.
2834  static bool isBackEdge(const VPBlockBase *FromBlock,
2835  const VPBlockBase *ToBlock, const VPLoopInfo *VPLI) {
2836  assert(FromBlock->getParent() == ToBlock->getParent() &&
2837  FromBlock->getParent() && "Must be in same region");
2838  const VPLoop *FromLoop = VPLI->getLoopFor(FromBlock);
2839  const VPLoop *ToLoop = VPLI->getLoopFor(ToBlock);
2840  if (!FromLoop || !ToLoop || FromLoop != ToLoop)
2841  return false;
2842 
2843  // A back-edge is a branch from the loop latch to its header.
2844  return ToLoop->isLoopLatch(FromBlock) && ToBlock == ToLoop->getHeader();
2845  }
2846 
2847  /// Returns true if \p Block is a loop latch
2848  static bool blockIsLoopLatch(const VPBlockBase *Block,
2849  const VPLoopInfo *VPLInfo) {
2850  if (const VPLoop *ParentVPL = VPLInfo->getLoopFor(Block))
2851  return ParentVPL->isLoopLatch(Block);
2852 
2853  return false;
2854  }
2855 
2856  /// Count and return the number of succesors of \p PredBlock excluding any
2857  /// backedges.
2858  static unsigned countSuccessorsNoBE(VPBlockBase *PredBlock,
2859  VPLoopInfo *VPLI) {
2860  unsigned Count = 0;
2861  for (VPBlockBase *SuccBlock : PredBlock->getSuccessors()) {
2862  if (!VPBlockUtils::isBackEdge(PredBlock, SuccBlock, VPLI))
2863  Count++;
2864  }
2865  return Count;
2866  }
2867 
2868  /// Return an iterator range over \p Range which only includes \p BlockTy
2869  /// blocks. The accesses are casted to \p BlockTy.
2870  template <typename BlockTy, typename T>
2871  static auto blocksOnly(const T &Range) {
2872  // Create BaseTy with correct const-ness based on BlockTy.
2873  using BaseTy =
2874  typename std::conditional<std::is_const<BlockTy>::value,
2876 
2877  // We need to first create an iterator range over (const) BlocktTy & instead
2878  // of (const) BlockTy * for filter_range to work properly.
2879  auto Mapped =
2880  map_range(Range, [](BaseTy *Block) -> BaseTy & { return *Block; });
2881  auto Filter = make_filter_range(
2882  Mapped, [](BaseTy &Block) { return isa<BlockTy>(&Block); });
2883  return map_range(Filter, [](BaseTy &Block) -> BlockTy * {
2884  return cast<BlockTy>(&Block);
2885  });
2886  }
2887 };
2888 
2891  InterleaveGroupMap;
2892 
2893  /// Type for mapping of instruction based interleave groups to VPInstruction
2894  /// interleave groups
2897 
2898  /// Recursively \p Region and populate VPlan based interleave groups based on
2899  /// \p IAI.
2900  void visitRegion(VPRegionBlock *Region, Old2NewTy &Old2New,
2901  InterleavedAccessInfo &IAI);
2902  /// Recursively traverse \p Block and populate VPlan based interleave groups
2903  /// based on \p IAI.
2904  void visitBlock(VPBlockBase *Block, Old2NewTy &Old2New,
2905  InterleavedAccessInfo &IAI);
2906 
2907 public:
2909 
2912  // Avoid releasing a pointer twice.
2913  for (auto &I : InterleaveGroupMap)
2914  DelSet.insert(I.second);
2915  for (auto *Ptr : DelSet)
2916  delete Ptr;
2917  }
2918 
2919  /// Get the interleave group that \p Instr belongs to.
2920  ///
2921  /// \returns nullptr if doesn't have such group.
2924  return InterleaveGroupMap.lookup(Instr);
2925  }
2926 };
2927 
2928 /// Class that maps (parts of) an existing VPlan to trees of combined
2929 /// VPInstructions.
2930 class VPlanSlp {
2931  enum class OpMode { Failed, Load, Opcode };
2932 
2933  /// A DenseMapInfo implementation for using SmallVector<VPValue *, 4> as
2934  /// DenseMap keys.
2935  struct BundleDenseMapInfo {
2936  static SmallVector<VPValue *, 4> getEmptyKey() {
2937  return {reinterpret_cast<VPValue *>(-1)};
2938  }
2939 
2940  static SmallVector<VPValue *, 4> getTombstoneKey() {
2941  return {reinterpret_cast<VPValue *>(-2)};
2942  }
2943 
2944  static unsigned getHashValue(const SmallVector<VPValue *, 4> &V) {
2945  return static_cast<unsigned>(hash_combine_range(V.begin(), V.end()));
2946  }
2947 
2948  static bool isEqual(const SmallVector<VPValue *, 4> &LHS,
2949  const SmallVector<VPValue *, 4> &RHS) {
2950  return LHS == RHS;
2951  }
2952  };
2953 
2954  /// Mapping of values in the original VPlan to a combined VPInstruction.
2955  DenseMap<SmallVector<VPValue *, 4>, VPInstruction *, BundleDenseMapInfo>
2956  BundleToCombined;
2957 
2959 
2960  /// Basic block to operate on. For now, only instructions in a single BB are
2961  /// considered.
2962  const VPBasicBlock &BB;
2963 
2964  /// Indicates whether we managed to combine all visited instructions or not.
2965  bool CompletelySLP = true;
2966 
2967  /// Width of the widest combined bundle in bits.
2968  unsigned WidestBundleBits = 0;
2969 
2970  using MultiNodeOpTy =
2971  typename std::pair<VPInstruction *, SmallVector<VPValue *, 4>>;
2972 
2973  // Input operand bundles for the current multi node. Each multi node operand
2974  // bundle contains values not matching the multi node's opcode. They will
2975  // be reordered in reorderMultiNodeOps, once we completed building a
2976  // multi node.
2977  SmallVector<MultiNodeOpTy, 4> MultiNodeOps;
2978 
2979  /// Indicates whether we are building a multi node currently.
2980  bool MultiNodeActive = false;
2981 
2982  /// Check if we can vectorize Operands together.
2983  bool areVectorizable(ArrayRef<VPValue *> Operands) const;
2984 
2985  /// Add combined instruction \p New for the bundle \p Operands.
2986  void addCombined(ArrayRef<VPValue *> Operands, VPInstruction *New);
2987 
2988  /// Indicate we hit a bundle we failed to combine. Returns nullptr for now.
2989  VPInstruction *markFailed();
2990 
2991  /// Reorder operands in the multi node to maximize sequential memory access
2992  /// and commutative operations.
2993  SmallVector<MultiNodeOpTy, 4> reorderMultiNodeOps();
2994 
2995  /// Choose the best candidate to use for the lane after \p Last. The set of
2996  /// candidates to choose from are values with an opcode matching \p Last's
2997  /// or loads consecutive to \p Last.
2998  std::pair<OpMode, VPValue *> getBest(OpMode Mode, VPValue *Last,
2999  SmallPtrSetImpl<VPValue *> &Candidates,
3001 
3002 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
3003  /// Print bundle \p Values to dbgs().
3004  void dumpBundle(ArrayRef<VPValue *> Values);
3005 #endif
3006 
3007 public:
3009 
3010  ~VPlanSlp() = default;
3011 
3012  /// Tries to build an SLP tree rooted at \p Operands and returns a
3013  /// VPInstruction combining \p Operands, if they can be combined.
3015 
3016  /// Return the width of the widest combined bundle in bits.
3017  unsigned getWidestBundleBits() const { return WidestBundleBits; }
3018 
3019  /// Return true if all visited instruction can be combined.
3020  bool isCompletelySLP() const { return CompletelySLP; }
3021 };
3022 
3023 namespace vputils {
3024 
3025 /// Returns true if only the first lane of \p Def is used.
3027 
3028 /// Get or create a VPValue that corresponds to the expansion of \p Expr. If \p
3029 /// Expr is a SCEVConstant or SCEVUnknown, return a VPValue wrapping the live-in
3030 /// value. Otherwise return a VPExpandSCEVRecipe to expand \p Expr. If \p Plan's
3031 /// pre-header already contains a recipe expanding \p Expr, return it. If not,
3032 /// create a new one.
3033 VPValue *getOrCreateVPValueForSCEVExpr(VPlan &Plan, const SCEV *Expr,
3034  ScalarEvolution &SE);
3035 
3036 } // end namespace vputils
3037 
3038 } // end namespace llvm
3039 
3040 #endif // LLVM_TRANSFORMS_VECTORIZE_VPLAN_H
llvm::VPBasicBlock::getRecipeList
RecipeListTy & getRecipeList()
Returns a reference to the list of recipes.
Definition: VPlan.h:2020
llvm::VPRegionBlock::front
VPBlockBase & front() const
Definition: VPlan.h:2143
llvm::VPInstruction::getOpcode
unsigned getOpcode() const
Definition: VPlan.h:851
i
i
Definition: README.txt:29
llvm::VPWidenGEPRecipe::~VPWidenGEPRecipe
~VPWidenGEPRecipe() override=default
llvm::VPWidenIntOrFpInductionRecipe::VPWidenIntOrFpInductionRecipe
VPWidenIntOrFpInductionRecipe(PHINode *IV, VPValue *Start, VPValue *Step, const InductionDescriptor &IndDesc, TruncInst *Trunc, bool NeedsScalarIV, bool NeedsVectorIV)
Definition: VPlan.h:1061
llvm::VFRange::VFRange
VFRange(const ElementCount &Start, const ElementCount &End)
Definition: VPlan.h:88
llvm::VPUser::getVPUserID
VPUserID getVPUserID() const
Definition: VPlanValue.h:250
llvm::VPRecipeBase::moveAfter
void moveAfter(VPRecipeBase *MovePos)
Unlink this recipe from its current VPBasicBlock and insert it into the VPBasicBlock that MovePos liv...
Definition: VPlan.cpp:685
llvm::VPBlockBase::getSinglePredecessor
VPBlockBase * getSinglePredecessor() const
Definition: VPlan.h:517
llvm::VPBlockUtils::tryToMergeBlockIntoPredecessor
static VPBasicBlock * tryToMergeBlockIntoPredecessor(VPBlockBase *Block)
Try to merge Block into its single predecessor, if Block is a VPBasicBlock and its predecessor has a ...
Definition: VPlan.h:2811
llvm::VPExpandSCEVRecipe::getSCEV
const SCEV * getSCEV() const
Definition: VPlan.h:1819
llvm::VPValue::VPVWidenGEPSC
@ VPVWidenGEPSC
Definition: VPlanValue.h:100
llvm::VPReductionRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlan.cpp:1407
llvm::VPWidenIntOrFpInductionRecipe::needsScalarIV
bool needsScalarIV() const
Returns true if a scalar phi needs to be created for the induction.
Definition: VPlan.h:1119
llvm::VPBlockBase::getPlan
VPlan * getPlan()
Definition: VPlan.cpp:135
llvm::VPWidenMemoryInstructionRecipe::getMask
VPValue * getMask() const
Return the mask used by this recipe.
Definition: VPlan.h:1748
llvm::VPAllSuccessorsIterator::operator*
const VPBlockBase * operator*() const
Definition: VPlan.h:2370
llvm::VPWidenIntOrFpInductionRecipe::getStepValue
VPValue * getStepValue()
Returns the step value of the induction.
Definition: VPlan.h:1091
llvm::VPWidenMemoryInstructionRecipe::VPWidenMemoryInstructionRecipe
VPWidenMemoryInstructionRecipe(LoadInst &Load, VPValue *Addr, VPValue *Mask, bool Consecutive, bool Reverse)
Definition: VPlan.h:1718
llvm::VPBlendRecipe::getNumIncomingValues
unsigned getNumIncomingValues() const
Return the number of incoming values, taking into account that a single incoming value has no mask.
Definition: VPlan.h:1389
llvm::VPlan::execute
void execute(struct VPTransformState *State)
Generate the IR code for this VPlan.
Definition: VPlan.cpp:947
llvm::VPlanIngredient::print
void print(raw_ostream &O) const
Definition: VPlan.cpp:1241
llvm::Loop::isLoopInvariant
bool isLoopInvariant(const Value *V) const
Return true if the specified value is loop invariant.
Definition: LoopInfo.cpp:60
llvm::VPlanSlp
Class that maps (parts of) an existing VPlan to trees of combined VPInstructions.
Definition: VPlan.h:2930
llvm::VPBlockRecursiveTraversalWrapper::getEntry
BlockTy getEntry()
Definition: VPlan.h:2392
llvm::VPWidenMemoryInstructionRecipe::onlyFirstLaneUsed
bool onlyFirstLaneUsed(const VPValue *Op) const override
Returns true if the recipe only uses the first lane of operand Op.
Definition: VPlan.h:1779
llvm::GraphTraits< const VPBlockBase * >::getEntryNode
static NodeRef getEntryNode(NodeRef N)
Definition: VPlan.h:2215
llvm::VPWidenSelectRecipe::~VPWidenSelectRecipe
~VPWidenSelectRecipe() override=default
llvm::VPWidenSelectRecipe::classof
static bool classof(const VPDef *D)
Method to support type inquiry through isa, cast, and dyn_cast.
Definition: VPlan.h:991
llvm::VPLane::VPLane
VPLane(unsigned Lane, Kind LaneKind)
Definition: VPlan.h:128
LLVM_DUMP_METHOD
#define LLVM_DUMP_METHOD
Mark debug helper function definitions like dump() that should not be stripped from debug builds.
Definition: Compiler.h:494
llvm::VPRegionBlock::VPRegionBlock
VPRegionBlock(VPBlockBase *Entry, VPBlockBase *Exit, const std::string &Name="", bool IsReplicator=false)
Definition: VPlan.h:2101
llvm::VPBlendRecipe::getIncomingValue
VPValue * getIncomingValue(unsigned Idx) const
Return incoming value number Idx.
Definition: VPlan.h:1392
llvm
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:17
llvm::SmallVectorImpl::erase
iterator erase(const_iterator CI)
Definition: SmallVector.h:724
llvm::tgtok::Def
@ Def
Definition: TGLexer.h:50
FMF.h
llvm::VPRecipeBase::getUnderlyingInstr
Instruction * getUnderlyingInstr()
Returns the underlying instruction, if the recipe is a VPValue or nullptr otherwise.
Definition: VPlan.h:729
llvm::VPWidenCallRecipe::~VPWidenCallRecipe
~VPWidenCallRecipe() override=default
llvm::VPLoop
Hold analysis information for every loop detected by VPLoopInfo.
Definition: VPlanLoopInfo.h:27
llvm::GraphTraits< VPBlockBase * >::getEntryNode
static NodeRef getEntryNode(NodeRef N)
Definition: VPlan.h:2200
llvm::VPlanIngredient::VPlanIngredient
VPlanIngredient(const Value *V)
Definition: VPlan.h:2722
llvm::VPBlockBase::dropAllReferences
virtual void dropAllReferences(VPValue *NewValue)=0
Replace all operands of VPUsers in the block with NewValue and also replaces all uses of VPValues def...
llvm::VPCanonicalIVPHIRecipe
Canonical scalar induction phi of the vector loop.
Definition: VPlan.h:1826
llvm::VPWidenPHIRecipe::VPWidenPHIRecipe
VPWidenPHIRecipe(PHINode *Phi, VPValue *Start=nullptr)
Create a new VPWidenPHIRecipe for Phi with start value Start.
Definition: VPlan.h:1238
llvm::make_range
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
Definition: iterator_range.h:53
llvm::VPTransformState::CFG
struct llvm::VPTransformState::CFGState CFG
llvm::VPRecipeBase::eraseFromParent
iplist< VPRecipeBase >::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
Definition: VPlan.cpp:680
llvm::VPLane::getFirstLane
static VPLane getFirstLane()
Definition: VPlan.h:130
Optional.h
llvm::VPWidenCanonicalIVRecipe::execute
void execute(VPTransformState &State) override
Generate a canonical vector induction variable of the vector loop, with start = {<Part*VF,...
Definition: VPlan.cpp:1516
llvm::VPPredInstPHIRecipe::execute
void execute(VPTransformState &State) override
Generates phi nodes for live-outs as needed to retain SSA form.
Definition: LoopVectorize.cpp:9917
llvm::VPDef::VPValue
friend class VPValue
Definition: VPlanValue.h:314
llvm::VPIteration::VPIteration
VPIteration(unsigned Part, const VPLane &Lane)
Definition: VPlan.h:192
llvm::VPBlendRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlan.cpp:1387
llvm::VPBlockBase::getEnclosingBlockWithPredecessors
VPBlockBase * getEnclosingBlockWithPredecessors()
Definition: VPlan.cpp:183
llvm::VPAllSuccessorsIterator::operator*
BlockPtrTy operator*()
Definition: VPlan.h:2372
llvm::VPInstruction::classof
static bool classof(const VPUser *U)
Extra classof implementations to allow directly casting from VPUser -> VPInstruction.
Definition: VPlan.h:843
llvm::VPBasicBlock::back
const VPRecipeBase & back() const
Definition: VPlan.h:2016
llvm::VPBlockBase::isLegalToHoistInto
bool isLegalToHoistInto()
Return true if it is legal to hoist instructions into this block.
Definition: VPlan.h:628
llvm::VPBlockBase::getName
const std::string & getName() const
Definition: VPlan.h:467
llvm::VPlan::disableValue2VPValue
void disableValue2VPValue()
Mark the plan to indicate that using Value2VPValue is not safe any longer, because it may be stale.
Definition: VPlan.h:2545
llvm::VPTransformState::Builder
IRBuilderBase & Builder
Hold a reference to the IRBuilder used to generate output IR code.
Definition: VPlan.h:336
llvm::VPlan::getOrAddVPValue
VPValue * getOrAddVPValue(Value *V, bool OverrideAllowed=false)
Gets the VPValue or adds a new one (if none exists yet) for V.
Definition: VPlan.h:2593
llvm::VPInstruction::Not
@ Not
Definition: VPlan.h:793
llvm::ElementCount
Definition: TypeSize.h:390
llvm::VPUser::removeLastOperand
void removeLastOperand()
Definition: VPlanValue.h:269
llvm::VPWidenCanonicalIVRecipe::VPWidenCanonicalIVRecipe
VPWidenCanonicalIVRecipe(VPCanonicalIVPHIRecipe *CanonicalIV)
Definition: VPlan.h:1873
llvm::VPBasicBlock
VPBasicBlock serves as the leaf of the Hierarchical Control-Flow Graph.
Definition: VPlan.h:1973
llvm::VPScalarIVStepsRecipe::classof
static bool classof(const VPUser *U)
Extra classof implementations to allow directly casting from VPUser -> VPScalarIVStepsRecipe.
Definition: VPlan.h:1937
llvm::Loop
Represents a single loop in the control flow graph.
Definition: LoopInfo.h:530
llvm::DenseMapBase< DenseMap< KeyT, ValueT, DenseMapInfo< KeyT >, llvm::detail::DenseMapPair< KeyT, ValueT > >, KeyT, ValueT, DenseMapInfo< KeyT >, llvm::detail::DenseMapPair< KeyT, ValueT > >::lookup
ValueT lookup(const_arg_type_t< KeyT > Val) const
lookup - Return the entry for the specified key, or a default constructed value if no such entry exis...
Definition: DenseMap.h:199
P
This currently compiles esp xmm0 movsd esp eax eax esp ret We should use not the dag combiner This is because dagcombine2 needs to be able to see through the X86ISD::Wrapper which DAGCombine can t really do The code for turning x load into a single vector load is target independent and should be moved to the dag combiner The code for turning x load into a vector load can only handle a direct load from a global or a direct load from the stack It should be generalized to handle any load from P
Definition: README-SSE.txt:411
llvm::VPWidenPHIRecipe
A recipe for handling header phis that are widened in the vector loop.
Definition: VPlan.h:1232
BlockTy
std::pair< BasicBlock *, unsigned > BlockTy
A pair of (basic block, score).
Definition: HotColdSplitting.cpp:397
llvm::VPBlockBase::getExitBasicBlock
const VPBasicBlock * getExitBasicBlock() const
Definition: VPlan.cpp:161
llvm::VPInstruction::SLPStore
@ SLPStore
Definition: VPlan.h:796
llvm::VPHeaderPHIRecipe
A pure virtual base class for all recipes modeling header phis, including phis for first order recurr...
Definition: VPlan.h:1129
llvm::VPlan::prepareToExecute
void prepareToExecute(Value *TripCount, Value *VectorTripCount, Value *CanonicalIVStartValue, VPTransformState &State)
Prepare the plan for execution, setting up the required live-in values.
Definition: VPlan.cpp:897
llvm::VPWidenIntOrFpInductionRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlan.cpp:1296
llvm::VPValue::VPVWidenSC
@ VPVWidenSC
Definition: VPlanValue.h:97
llvm::VPValue::getLiveInIRValue
Value * getLiveInIRValue()
Returns the underlying IR value, if this VPValue is defined outside the scope of VPlan.
Definition: VPlanValue.h:187
llvm::Type::getScalarType
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
Definition: Type.h:309
llvm::VPlanPrinter::dump
LLVM_DUMP_METHOD void dump()
Definition: VPlan.cpp:1132
llvm::VPPredInstPHIRecipe::classof
static bool classof(const VPDef *D)
Method to support type inquiry through isa, cast, and dyn_cast.
Definition: VPlan.h:1671
llvm::SmallVector
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1185
llvm::VPCanonicalIVPHIRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlan.cpp:1487
llvm::VPWidenIntOrFpInductionRecipe::~VPWidenIntOrFpInductionRecipe
~VPWidenIntOrFpInductionRecipe() override=default
llvm::VPInterleaveRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: LoopVectorize.cpp:9243
ilist.h
llvm::VPInstruction::BranchOnCount
@ BranchOnCount
Definition: VPlan.h:800
llvm::VPDef::getVPSingleValue
VPValue * getVPSingleValue()
Returns the only VPValue defined by the VPDef.
Definition: VPlanValue.h:388
llvm::VPBasicBlock::end
const_iterator end() const
Definition: VPlan.h:2005
llvm::VPTransformState::get
Value * get(VPValue *Def, unsigned Part)
Get the generated Value for a given VPValue and a given Part.
Definition: LoopVectorize.cpp:10140
llvm::VPlan::getOrAddExternalDef
VPValue * getOrAddExternalDef(Value *V)
Get the existing or add a new external definition for V.
Definition: VPlan.h:2556
llvm::enumerate
detail::enumerator< R > enumerate(R &&TheRange)
Given an input range, returns a new range whose values are are pair (A,B) such that A is the 0-based ...
Definition: STLExtras.h:2045
llvm::VPRegionBlock::dropAllReferences
void dropAllReferences(VPValue *NewValue) override
Replace all operands of VPUsers in the block with NewValue and also replaces all uses of VPValues def...
Definition: VPlan.cpp:486
llvm::ilist_node_with_parent
An ilist node that can access its parent list.
Definition: ilist_node.h:257
llvm::VPInterleavedAccessInfo
Definition: VPlan.h:2889
llvm::VPBranchOnMaskRecipe::execute
void execute(VPTransformState &State) override
Generate the extraction of the appropriate bit from the block mask and the conditional branch.
Definition: LoopVectorize.cpp:9891
llvm::VPBlockBase::VPBlockBase
VPBlockBase(const unsigned char SC, const std::string &N)
Definition: VPlan.h:453
llvm::GraphTraits< VPRegionBlock * >::nodes_begin
static nodes_iterator nodes_begin(GraphRef N)
Definition: VPlan.h:2257
llvm::VPLane::Kind::First
@ First
For First, Lane is the index into the first N elements of a fixed-vector <N x <ElTy>> or a scalable v...
llvm::TargetTransformInfo
This pass provides access to the codegen interfaces that are needed for IR-level transformations.
Definition: TargetTransformInfo.h:167
llvm::VPInterleaveRecipe::VPInterleaveRecipe
VPInterleaveRecipe(const InterleaveGroup< Instruction > *IG, VPValue *Addr, ArrayRef< VPValue * > StoredValues, VPValue *Mask)
Definition: VPlan.h:1427
llvm::VPLane::Kind::ScalableLast
@ ScalableLast
For ScalableLast, Lane is the offset from the start of the last N-element subvector in a scalable vec...
llvm::GraphTraits< const VPBlockBase * >::child_end
static ChildIteratorType child_end(NodeRef N)
Definition: VPlan.h:2221
llvm::SmallDenseMap
Definition: DenseMap.h:882
llvm::ScalarEvolution
The main scalar evolution driver.
Definition: ScalarEvolution.h:449
llvm::GraphTraits< VPBlockRecursiveTraversalWrapper< VPBlockBase * > >::child_begin
static ChildIteratorType child_begin(NodeRef N)
Definition: VPlan.h:2410
llvm::VPInstruction::setFastMathFlags
void setFastMathFlags(FastMathFlags FMFNew)
Set the fast-math flags.
Definition: VPlan.cpp:887
llvm::VPReductionPHIRecipe
A recipe for handling reduction phis.
Definition: VPlan.h:1310
llvm::VPWidenIntOrFpInductionRecipe::needsVectorIV
bool needsVectorIV() const
Returns true if a vector phi needs to be created for the induction.
Definition: VPlan.h:1122
llvm::VPTransformState::set
void set(VPValue *Def, Value *V, unsigned Part)
Set the generated Value for a given VPValue and a given Part.
Definition: VPlan.h:258
llvm::DominatorTree
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
Definition: Dominators.h:166
llvm::VPTransformState::set
void set(VPValue *Def, Value *V, const VPIteration &Instance)
Set the generated scalar V for Def and the given Instance.
Definition: VPlan.h:274
llvm::iplist_impl< simple_ilist< VPRecipeBase, Options... >, ilist_traits< VPRecipeBase > >::const_reverse_iterator
base_list_type::const_reverse_iterator const_reverse_iterator
Definition: ilist.h:182
llvm::VPInterleaveRecipe::~VPInterleaveRecipe
~VPInterleaveRecipe() override=default
llvm::VPRegionBlock::getPreheaderVPBB
VPBasicBlock * getPreheaderVPBB()
Returns the pre-header VPBasicBlock of the loop region.
Definition: VPlan.h:2158
llvm::VPRecipeBase::getUnderlyingInstr
const Instruction * getUnderlyingInstr() const
Definition: VPlan.h:732
llvm::DenseMapBase::erase
bool erase(const KeyT &Val)
Definition: DenseMap.h:304
llvm::VPTransformState::CFGState::getPreheaderBBFor
BasicBlock * getPreheaderBBFor(VPRecipeBase *R)
Returns the BasicBlock* mapped to the pre-header of the loop region containing R.
Definition: VPlan.cpp:248
llvm::VPBlendRecipe::execute
void execute(VPTransformState &State) override
Generate the phi/select nodes.
Definition: LoopVectorize.cpp:9765
llvm::VPReductionRecipe::VPReductionRecipe
VPReductionRecipe(const RecurrenceDescriptor *R, Instruction *I, VPValue *ChainOp, VPValue *VecOp, VPValue *CondOp, const TargetTransformInfo *TTI)
Definition: VPlan.h:1509
llvm::Depth
@ Depth
Definition: SIMachineScheduler.h:36
llvm::VPBlockBase::~VPBlockBase
virtual ~VPBlockBase()=default
llvm::VPPredInstPHIRecipe
VPPredInstPHIRecipe is a recipe for generating the phi nodes needed when control converges back from ...
Definition: VPlan.h:1661
llvm::VPBasicBlock::dropAllReferences
void dropAllReferences(VPValue *NewValue) override
Replace all operands of VPUsers in the block with NewValue and also replaces all uses of VPValues def...
Definition: VPlan.cpp:399
llvm::VPLane::getKind
Kind getKind() const
Returns the Kind of lane offset.
Definition: VPlan.h:156
llvm::VPlan::getCanonicalIV
VPCanonicalIVPHIRecipe * getCanonicalIV()
Returns the canonical induction recipe of the vector loop.
Definition: VPlan.h:2652
llvm::Type
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
DenseMap.h
llvm::VPWidenIntOrFpInductionRecipe::getTruncInst
TruncInst * getTruncInst()
Returns the first defined value as TruncInst, if it is one or nullptr otherwise.
Definition: VPlan.h:1096
llvm::VPIteration::isFirstIteration
bool isFirstIteration() const
Definition: VPlan.h:194
llvm::VPlanPtr
std::unique_ptr< VPlan > VPlanPtr
Definition: VPlan.h:97
llvm::sys::path::end
const_iterator end(StringRef path)
Get end iterator over path.
Definition: Path.cpp:235
llvm::sys::path::begin
const_iterator begin(StringRef path, Style style=Style::native)
Get begin iterator over path.
Definition: Path.cpp:226
llvm::GraphTraits< Inverse< VPRegionBlock * > >::getEntryNode
static NodeRef getEntryNode(Inverse< GraphRef > N)
Definition: VPlan.h:2293
llvm::VPBlockBase::getHierarchicalPredecessors
const VPBlocksTy & getHierarchicalPredecessors()
Definition: VPlan.h:557
T1
#define T1
Definition: Mips16ISelLowering.cpp:340
llvm::VPInterleaveRecipe::classof
static bool classof(const VPDef *D)
Method to support type inquiry through isa, cast, and dyn_cast.
Definition: VPlan.h:1447
llvm::SPII::Load
@ Load
Definition: SparcInstrInfo.h:32
llvm::VPValue::VPVBlendSC
@ VPVBlendSC
Definition: VPlanValue.h:104
llvm::VPWidenIntOrFpInductionRecipe::getScalarType
const Type * getScalarType() const
Returns the scalar type of the induction.
Definition: VPlan.h:1113
llvm::VPBlockUtils
Class that provides utilities for VPBlockBases in VPlan.
Definition: VPlan.h:2743
llvm::Optional
Definition: APInt.h:33
llvm::VPUser::op_begin
operand_iterator op_begin()
Definition: VPlanValue.h:279
llvm::VPScalarIVStepsRecipe::execute
void execute(VPTransformState &State) override
Generate the scalarized versions of the phi node as needed by their users.
Definition: LoopVectorize.cpp:9701
llvm::DenseMapBase::count
size_type count(const_arg_type_t< KeyT > Val) const
Return 1 if the specified key is in the map, 0 otherwise.
Definition: DenseMap.h:147
llvm::VPlan::getEntry
const VPBlockBase * getEntry() const
Definition: VPlan.h:2518
llvm::VPWidenMemoryInstructionRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlan.cpp:1460
llvm::VPWidenPHIRecipe::~VPWidenPHIRecipe
~VPWidenPHIRecipe() override=default
llvm::mapped_iterator
Definition: STLExtras.h:298
llvm::SmallPtrSet
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
Definition: SmallPtrSet.h:450
llvm::VPReplicateRecipe::classof
static bool classof(const VPDef *D)
Method to support type inquiry through isa, cast, and dyn_cast.
Definition: VPlan.h:1575
llvm::VPWidenCallRecipe
A recipe for widening Call instructions.
Definition: VPlan.h:949
llvm::VPPredInstPHIRecipe::usesScalars
bool usesScalars(const VPValue *Op) const override
Returns true if the recipe uses scalars of operand Op.
Definition: VPlan.h:1685
llvm::VPWidenMemoryInstructionRecipe::VPWidenMemoryInstructionRecipe
VPWidenMemoryInstructionRecipe(StoreInst &Store, VPValue *Addr, VPValue *StoredValue, VPValue *Mask, bool Consecutive, bool Reverse)
Definition: VPlan.h:1727
llvm::MipsISD::Ret
@ Ret
Definition: MipsISelLowering.h:119
llvm::LoopBase::isLoopLatch
bool isLoopLatch(const BlockT *BB) const
Definition: LoopInfo.h:239
llvm::VPReplicateRecipe::onlyFirstLaneUsed
bool onlyFirstLaneUsed(const VPValue *Op) const override
Returns true if the recipe only uses the first lane of operand Op.
Definition: VPlan.h:1603
llvm::VPUser::operands
operand_range operands()
Definition: VPlanValue.h:283
llvm::VPUser::VPUserID
VPUserID
Subclass identifier (for isa/dyn_cast).
Definition: VPlanValue.h:209
RHS
Value * RHS
Definition: X86PartialReduction.cpp:76
llvm::VPScalarIVStepsRecipe::classof
static bool classof(const VPRecipeBase *R)
Definition: VPlan.h:1941
llvm::isPowerOf2_32
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition: MathExtras.h:491
llvm::VPRegionBlock::getEntry
const VPBlockBase * getEntry() const
Definition: VPlan.h:2127
llvm::VPAllSuccessorsIterator
Iterator to traverse all successors of a VPBlockBase node.
Definition: VPlan.h:2314
llvm::FastMathFlags
Convenience struct for specifying and reasoning about fast-math flags.
Definition: FMF.h:21
llvm::VPTransformState::hasScalarValue
bool hasScalarValue(VPValue *Def, VPIteration Instance)
Definition: VPlan.h:247
llvm::VPInstruction::mayWriteToMemory
bool mayWriteToMemory() const
Return true if this instruction may modify memory.
Definition: VPlan.h:868
VPlanValue.h
llvm::VPReplicateRecipe::isPacked
bool isPacked() const
Definition: VPlan.h:1598
llvm::InterleaveGroup
The group of interleaved loads/stores sharing the same stride and close to each other.
Definition: VectorUtils.h:284
llvm::LinearPolySize::isScalable
bool isScalable() const
Returns whether the size is scaled by a runtime quantity (vscale).
Definition: TypeSize.h:298
llvm::GraphTraits< VPBlockRecursiveTraversalWrapper< const VPBlockBase * > >::getEntryNode
static NodeRef getEntryNode(VPBlockRecursiveTraversalWrapper< const VPBlockBase * > N)
Definition: VPlan.h:2425
llvm::VPBasicBlock::getEnclosingLoopRegion
VPRegionBlock * getEnclosingLoopRegion()
Definition: VPlan.cpp:434
llvm::iplist
An intrusive list with ownership and callbacks specified/controlled by ilist_traits,...
Definition: ilist.h:391
llvm::VPReductionRecipe
A recipe to represent inloop reduction operations, performing a reduction on a vector operand into a ...
Definition: VPlan.h:1502
llvm::VPReductionPHIRecipe::~VPReductionPHIRecipe
~VPReductionPHIRecipe() override=default
llvm::VPInstruction::ICmpULE
@ ICmpULE
Definition: VPlan.h:794
llvm::isEqual
bool isEqual(const GCNRPTracker::LiveRegSet &S1, const GCNRPTracker::LiveRegSet &S2)
Definition: GCNRegPressure.cpp:55
llvm::VPScalarIVStepsRecipe::classof
static bool classof(const VPDef *D)
Method to support type inquiry through isa, cast, and dyn_cast.
Definition: VPlan.h:1932
llvm::VPScalarIVStepsRecipe::isCanonical
bool isCanonical() const
Returns true if the induction is canonical, i.e.
Definition: VPlan.cpp:1332
llvm::VPValue::users
user_range users()
Definition: VPlanValue.h:162
DepthFirstIterator.h
llvm::VPWidenPHIRecipe::execute
void execute(VPTransformState &State) override
Generate the phi/select nodes.
Definition: LoopVectorize.cpp:9760
llvm::VPBlockBase::execute
virtual void execute(struct VPTransformState *State)=0
The method which generates the output IR that correspond to this VPBlockBase, thereby "executing" the...
llvm::VPBlockBase::setPredicate
void setPredicate(VPValue *Pred)
Set the block's predicate.
Definition: VPlan.cpp:209
llvm::VPScalarIVStepsRecipe::getCanonicalIV
VPCanonicalIVPHIRecipe * getCanonicalIV() const
Definition: VPlan.cpp:1328
llvm::VPInterleaveRecipe::getInterleaveGroup
const InterleaveGroup< Instruction > * getInterleaveGroup()
Definition: VPlan.h:1481
llvm::RISCVFenceField::R
@ R
Definition: RISCVBaseInfo.h:240
llvm::VFRange
A range of powers-of-2 vectorization factors with fixed start and adjustable end.
Definition: VPlan.h:77
llvm::VPInstruction::ActiveLaneMask
@ ActiveLaneMask
Definition: VPlan.h:797
llvm::VPTransformState::VPTransformState
VPTransformState(ElementCount VF, unsigned UF, LoopInfo *LI, DominatorTree *DT, IRBuilderBase &Builder, InnerLoopVectorizer *ILV, VPlan *Plan)
Definition: VPlan.h:200
llvm::BasicBlock
LLVM Basic Block Representation.
Definition: BasicBlock.h:55
llvm::VPBranchOnMaskRecipe::classof
static bool classof(const VPDef *D)
Method to support type inquiry through isa, cast, and dyn_cast.
Definition: VPlan.h:1627
llvm::VPWidenIntOrFpInductionRecipe::getInductionDescriptor
const InductionDescriptor & getInductionDescriptor() const
Returns the induction descriptor for the recipe.
Definition: VPlan.h:1106
llvm::VPDef
This class augments a recipe with a set of VPValues defined by the recipe.
Definition: VPlanValue.h:313
llvm::VPCanonicalIVPHIRecipe::getScalarType
const Type * getScalarType() const
Returns the scalar type of the induction.
Definition: VPlan.h:1858
llvm::VPRecipeBase::mayWriteToMemory
bool mayWriteToMemory() const
Returns true if the recipe may write to memory.
Definition: VPlan.cpp:558
llvm::dbgs
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
GraphTraits.h
llvm::VPTransformState::DataState::PerPartOutput
DenseMap< VPValue *, PerPartValuesTy > PerPartOutput
Definition: VPlan.h:221
llvm::VPInterleaveRecipe::execute
void execute(VPTransformState &State) override
Generate the wide load or store, and shuffles.
Definition: LoopVectorize.cpp:9804
llvm::SmallBitVector
This is a 'bitvector' (really, a variable-sized bit array), optimized for the case when the array is ...
Definition: SmallBitVector.h:35
llvm::VPPredInstPHIRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlan.cpp:1452
llvm::VPValue::VPVInstructionSC
@ VPVInstructionSC
Definition: VPlanValue.h:93
llvm::VPReplicateRecipe
VPReplicateRecipe replicates a given instruction producing multiple scalar copies of the original sca...
Definition: VPlan.h:1548
llvm::VPWidenIntOrFpInductionRecipe::getStartValue
const VPValue * getStartValue() const
Definition: VPlan.h:1088
llvm::BitmaskEnumDetail::Mask
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
Definition: BitmaskEnum.h:80
llvm::VPTransformState
VPTransformState holds information passed down when "executing" a VPlan, needed for generating the ou...
Definition: VPlan.h:199
llvm::VPBasicBlock::iterator
RecipeListTy::iterator iterator
Instruction iterators...
Definition: VPlan.h:1994
llvm::VPReductionPHIRecipe::classof
static bool classof(const VPHeaderPHIRecipe *R)
Definition: VPlan.h:1337
llvm::VPValue::VPVWidenPointerInductionSC
@ VPVWidenPointerInductionSC
Definition: VPlanValue.h:109
LHS
Value * LHS
Definition: X86PartialReduction.cpp:75
llvm::VPBlockBase
VPBlockBase is the building block of the Hierarchical Control-Flow Graph.
Definition: VPlan.h:394
llvm::VPReplicateRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlan.cpp:1428
llvm::VPBasicBlock::appendRecipe
void appendRecipe(VPRecipeBase *Recipe)
Augment the existing recipes of a VPBasicBlock with an additional Recipe as the last recipe.
Definition: VPlan.h:2041
llvm::VPlan::hasVF
bool hasVF(ElementCount VF)
Definition: VPlan.h:2549
llvm::all_of
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1605
llvm::VPReductionPHIRecipe::isInLoop
bool isInLoop() const
Returns true, if the phi is part of an in-loop reduction.
Definition: VPlan.h:1361
llvm::VPWidenCanonicalIVRecipe::classof
static bool classof(const VPRecipeBase *R)
Definition: VPlan.h:1890
llvm::InductionDescriptor
A struct for saving information about induction variables.
Definition: IVDescriptors.h:306
llvm::VPWidenPHIRecipe::classof
static bool classof(const VPValue *V)
Definition: VPlan.h:1253
llvm::VPTransformState::DataState::PerPartValuesTy
SmallVector< Value *, 2 > PerPartValuesTy
A type for vectorized values in the new loop.
Definition: VPlan.h:219
llvm::VPInstruction::classof
static bool classof(const VPValue *V)
Method to support type inquiry through isa, cast, and dyn_cast.
Definition: VPlan.h:827
llvm::VPLane::mapToCacheIndex
unsigned mapToCacheIndex(const ElementCount &VF) const
Maps the lane to a cache index based on VF.
Definition: VPlan.h:162
llvm::VPWidenPHIRecipe::getIncomingBlock
VPBasicBlock * getIncomingBlock(unsigned I)
Returns the I th incoming VPBasicBlock.
Definition: VPlan.h:1273
llvm::VPInstruction
This is a concrete Recipe that models a single VPlan-level instruction.
Definition: VPlan.h:784
llvm::VPBlockUser::VPBlockUser
VPBlockUser()
Definition: VPlan.h:362
llvm::VPTransformState::ILV
InnerLoopVectorizer * ILV
Hold a pointer to InnerLoopVectorizer to reuse its IR generation methods.
Definition: VPlan.h:344
llvm::VFRange::isEmpty
bool isEmpty() const
Definition: VPlan.h:84
llvm::PPCISD::SC
@ SC
CHAIN = SC CHAIN, Imm128 - System call.
Definition: PPCISelLowering.h:418
llvm::Failed
testing::Matcher< const detail::ErrorHolder & > Failed()
Definition: Error.h:198
llvm::VPlan::print
void print(raw_ostream &O) const
Print this VPlan to O.
Definition: VPlan.cpp:1047
llvm::vputils::getOrCreateVPValueForSCEVExpr
VPValue * getOrCreateVPValueForSCEVExpr(VPlan &Plan, const SCEV *Expr, ScalarEvolution &SE)
Get or create a VPValue that corresponds to the expansion of Expr.
Definition: VPlan.cpp:1776
llvm::VPBlockBase::setPredecessors
void setPredecessors(ArrayRef< VPBlockBase * > NewPreds)
Set each VPBasicBlock in NewPreds as predecessor of this VPBlockBase.
Definition: VPlan.h:605
llvm::createStepForVF
Value * createStepForVF(IRBuilderBase &B, Type *Ty, ElementCount VF, int64_t Step)
Return a value for Step multiplied by VF.
Definition: LoopVectorize.cpp:1013
llvm::VPScalarIVStepsRecipe::onlyFirstLaneUsed
bool onlyFirstLaneUsed(const VPValue *Op) const override
Returns true if the recipe only uses the first lane of operand Op.
Definition: VPlan.h:1963
llvm::VPInterleavedAccessInfo::getInterleaveGroup
InterleaveGroup< VPInstruction > * getInterleaveGroup(VPInstruction *Instr) const
Get the interleave group that Instr belongs to.
Definition: VPlan.h:2923
llvm::VPIteration::Lane
VPLane Lane
Definition: VPlan.h:186
llvm::VPTransformState::LI
LoopInfo * LI
Hold a pointer to LoopInfo to register new basic blocks in the loop.
Definition: VPlan.h:330
llvm::VPlanSlp::buildGraph
VPInstruction * buildGraph(ArrayRef< VPValue * > Operands)
Tries to build an SLP tree rooted at Operands and returns a VPInstruction combining Operands,...
Definition: VPlanSLP.cpp:358
llvm::VPReplicateRecipe::classof
static bool classof(const VPValue *V)
Definition: VPlan.h:1579
llvm::VPLane::isFirstLane
bool isFirstLane() const
Returns true if this is the first lane of the whole vector.
Definition: VPlan.h:159
Twine.h
llvm::VPBlockBase::getSuccessors
VPBlocksTy & getSuccessors()
Definition: VPlan.h:502
llvm::VPBlockUtils::insertBlockAfter
static void insertBlockAfter(VPBlockBase *NewBlock, VPBlockBase *BlockPtr)
Insert disconnected VPBlockBase NewBlock after BlockPtr.
Definition: VPlan.h:2753
llvm::VPWidenPointerInductionRecipe::execute
void execute(VPTransformState &State) override
Generate vector values for the pointer induction.
Definition: LoopVectorize.cpp:9602
llvm::VPBasicBlock::getSublistAccess
static RecipeListTy VPBasicBlock::* getSublistAccess(VPRecipeBase *)
Returns a pointer to a member of the recipe list.
Definition: VPlan.h:2023
llvm::VPWidenMemoryInstructionRecipe::isReverse
bool isReverse() const
Definition: VPlan.h:1767
llvm::VPScalarIVStepsRecipe::VPScalarIVStepsRecipe
VPScalarIVStepsRecipe(Type *Ty, const InductionDescriptor &IndDesc, VPValue *CanonicalIV, VPValue *Start, VPValue *Step, Type *TruncToTy)
Definition: VPlan.h:1922
llvm::GraphTraits< const VPBlockBase * >::ChildIteratorType
SmallVectorImpl< VPBlockBase * >::const_iterator ChildIteratorType
Definition: VPlan.h:2213
llvm::VPlanPrinter::VPlanPrinter
VPlanPrinter(raw_ostream &O, const VPlan &P)
Definition: VPlan.h:2713
llvm::VPlan::getVPValue
VPValue * getVPValue(Value *V, bool OverrideAllowed=false)
Returns the VPValue for V.
Definition: VPlan.h:2582
llvm::GraphTraits< Inverse< VPBlockBase * > >::child_end
static ChildIteratorType child_end(NodeRef N)
Definition: VPlan.h:2238
b
the resulting code requires compare and branches when and if the revised code is with conditional branches instead of More there is a byte word extend before each where there should be only and the condition codes are not remembered when the same two values are compared twice More LSR enhancements i8 and i32 load store addressing modes are identical int b
Definition: README.txt:418
llvm::VPRecipeBase::moveBefore
void moveBefore(VPBasicBlock &BB, iplist< VPRecipeBase >::iterator I)
Unlink this recipe and insert into BB before I.
Definition: VPlan.cpp:690
llvm::VPBlockBase::getPredecessors
VPBlocksTy & getPredecessors()
Definition: VPlan.h:507
llvm::VPInterleaveRecipe
VPInterleaveRecipe is a recipe for transforming an interleave group of load or stores into one wide l...
Definition: VPlan.h:1421
IP
Definition: NVPTXLowerArgs.cpp:167
llvm::VPBranchOnMaskRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlan.h:1637
llvm::VPBlockBase::VPBlockTy
enum { VPBasicBlockSC, VPRegionBlockSC } VPBlockTy
An enumeration for keeping track of the concrete subclass of VPBlockBase that are actually instantiat...
Definition: VPlan.h:461
false
Definition: StackSlotColoring.cpp:141
llvm::VPBlockBase::getEnclosingBlockWithSuccessors
VPBlockBase * getEnclosingBlockWithSuccessors()
An Enclosing Block of a block B is any block containing B, including B itself.
Definition: VPlan.cpp:175
llvm::VPCanonicalIVPHIRecipe::classof
static bool classof(const VPHeaderPHIRecipe *D)
Definition: VPlan.h:1841
llvm::VPWidenPHIRecipe::getIncomingValue
VPValue * getIncomingValue(unsigned I)
Returns the I th incoming VPValue.
Definition: VPlan.h:1276
llvm::VPWidenPointerInductionRecipe::classof
static bool classof(const VPRecipeBase *B)
Method to support type inquiry through isa, cast, and dyn_cast.
Definition: VPlan.h:1209
B
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
llvm::VPWidenCanonicalIVRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlan.cpp:1536
llvm::VPReductionPHIRecipe::classof
static bool classof(const VPValue *V)
Definition: VPlan.h:1340
llvm::VPUser::VPUserID::Recipe
@ Recipe
llvm::GraphTraits< const VPRegionBlock * >::nodes_begin
static nodes_iterator nodes_begin(GraphRef N)
Definition: VPlan.h:2276
llvm::VPHeaderPHIRecipe::getBackedgeRecipe
VPRecipeBase * getBackedgeRecipe()
Returns the backedge value as a recipe.
Definition: VPlan.h:1181
llvm::VPTransformState::DataState::PerPartScalars
DenseMap< VPValue *, ScalarsPerPartValuesTy > PerPartScalars
Definition: VPlan.h:224
llvm::Instruction
Definition: Instruction.h:42
llvm::VPReplicateRecipe::~VPReplicateRecipe
~VPReplicateRecipe() override=default
llvm::InterleavedAccessInfo
Drive the analysis of interleaved memory accesses in the loop.
Definition: VectorUtils.h:751
llvm::SPII::Store
@ Store
Definition: SparcInstrInfo.h:33
llvm::VPBlockBase::printSuccessors
void printSuccessors(raw_ostream &O, const Twine &Indent) const
Print the successors of this block to O, prefixing all lines with Indent.
Definition: VPlan.cpp:445
llvm::VPReductionRecipe::execute
void execute(VPTransformState &State) override
Generate the reduction in the loop.
Definition: LoopVectorize.cpp:9810
llvm::VPRegionBlock::execute
void execute(struct VPTransformState *State) override
The method which generates the output IR instructions that correspond to this VPRegionBlock,...
Definition: VPlan.cpp:493
llvm::raw_ostream
This class implements an extremely fast bulk output stream that can only output to a stream.
Definition: raw_ostream.h:54
llvm::VPScalarIVStepsRecipe::getStartValue
VPValue * getStartValue() const
Definition: VPlan.h:1959
llvm::VPBlockUtils::blocksOnly
static auto blocksOnly(const T &Range)
Return an iterator range over Range which only includes BlockTy blocks.
Definition: VPlan.h:2871
llvm::VPValue
Definition: VPlanValue.h:44
llvm::VPRegionBlock::~VPRegionBlock
~VPRegionBlock() override
Definition: VPlan.h:2114
llvm::operator<<
raw_ostream & operator<<(raw_ostream &OS, const APFixedPoint &FX)
Definition: APFixedPoint.h:230
llvm::VPWidenCallRecipe::execute
void execute(VPTransformState &State) override
Produce a widened version of the call instruction.
Definition: LoopVectorize.cpp:9273
llvm::VPTransformState::CFGState::VPBBsToFix
SmallVector< VPBasicBlock *, 8 > VPBBsToFix
Vector of VPBasicBlocks whose terminator instruction needs to be fixed up at the end of vector code g...
Definition: VPlan.h:320
llvm::VPlanSlp::~VPlanSlp
~VPlanSlp()=default
llvm::VPBlockBase::successors
iterator_range< VPBlockBase ** > successors()
Definition: VPlan.h:504
llvm::VPlanSlp::getWidestBundleBits
unsigned getWidestBundleBits() const
Return the width of the widest combined bundle in bits.
Definition: VPlan.h:3017
llvm::VPlan::getName
const std::string & getName() const
Definition: VPlan.h:2551
DebugLoc.h
SmallPtrSet.h
llvm::VPTransformState::CFGState::ExitBB
BasicBlock * ExitBB
The last IR BasicBlock in the output IR.
Definition: VPlan.h:312
llvm::VPWidenCanonicalIVRecipe
A Recipe for widening the canonical induction variable of the vector loop.
Definition: VPlan.h:1871
llvm::VPWidenMemoryInstructionRecipe::getIngredient
Instruction & getIngredient() const
Definition: VPlan.h:1790
llvm::VPAllSuccessorsIterator::operator++
VPAllSuccessorsIterator & operator++()
Definition: VPlan.h:2374
llvm::InnerLoopVectorizer
InnerLoopVectorizer vectorizes loops which contain only one basic block to a specified vectorization ...
Definition: LoopVectorize.cpp:446
llvm::VPBasicBlock::end
iterator end()
Definition: VPlan.h:2004
llvm::MCID::Call
@ Call
Definition: MCInstrDesc.h:155
llvm::InterleaveGroup::getMember
InstTy * getMember(uint32_t Index) const
Get the member with the given index Index.
Definition: VectorUtils.h:679
llvm::VPReductionPHIRecipe::classof
static bool classof(const VPRecipeBase *R)
Method to support type inquiry through isa, cast, and dyn_cast.
Definition: VPlan.h:1334
llvm::GraphTraits< const VPBlockBase * >::child_begin
static ChildIteratorType child_begin(NodeRef N)
Definition: VPlan.h:2217
llvm::ArrayRef::slice
ArrayRef< T > slice(size_t N, size_t M) const
slice(n, m) - Chop off the first N elements of the array, and keep M elements in the array.
Definition: ArrayRef.h:194
llvm::VPWidenCanonicalIVRecipe::~VPWidenCanonicalIVRecipe
~VPWidenCanonicalIVRecipe() override=default
llvm::VPUser::getNumOperands
unsigned getNumOperands() const
Definition: VPlanValue.h:257
llvm::lltok::Kind
Kind
Definition: LLToken.h:18
llvm::iplist_impl< simple_ilist< VPRecipeBase, Options... >, ilist_traits< VPRecipeBase > >::reverse_iterator
base_list_type::reverse_iterator reverse_iterator
Definition: ilist.h:180
X
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
llvm::VPBasicBlock::empty
bool empty() const
Definition: VPlan.h:2013
llvm::VPCanonicalIVPHIRecipe::~VPCanonicalIVPHIRecipe
~VPCanonicalIVPHIRecipe() override=default
llvm::VPWidenMemoryInstructionRecipe::getAddr
VPValue * getAddr() const
Return the address accessed by this recipe.
Definition: VPlan.h:1742
llvm::VPRegionBlock
VPRegionBlock represents a collection of VPBasicBlocks and VPRegionBlocks which form a Single-Entry-S...
Definition: VPlan.h:2089
llvm::VPWidenMemoryInstructionRecipe::isStore
bool isStore() const
Returns true if this recipe is a store.
Definition: VPlan.h:1754
llvm::VPValue::VPVPredInstPHI
@ VPVPredInstPHI
Definition: VPlanValue.h:110
llvm::VPBlockBase::getPredecessors
const VPBlocksTy & getPredecessors() const
Definition: VPlan.h:506
llvm::VPReductionPHIRecipe::VPReductionPHIRecipe
VPReductionPHIRecipe(PHINode *Phi, const RecurrenceDescriptor &RdxDesc, VPValue &Start, bool IsInLoop=false, bool IsOrdered=false)
Create a new VPReductionPHIRecipe for the reduction Phi described by RdxDesc.
Definition: VPlan.h:1323
llvm::VPValue::VPVWidenCallSC
@ VPVWidenCallSC
Definition: VPlanValue.h:98
llvm::GraphTraits< VPBlockRecursiveTraversalWrapper< const VPBlockBase * > >::child_end
static ChildIteratorType child_end(NodeRef N)
Definition: VPlan.h:2433
llvm::Twine::str
std::string str() const
Return the twine contents as a std::string.
Definition: Twine.cpp:17
Operands
mir Rename Register Operands
Definition: MIRNamerPass.cpp:74
llvm::VPBlockBase::getCondBit
VPValue * getCondBit()
Definition: VPlan.cpp:191
llvm::GraphTraits< Inverse< VPBlockBase * > >::getEntryNode
static NodeRef getEntryNode(Inverse< NodeRef > B)
Definition: VPlan.h:2232
llvm::VPValue::VPVReductionSC
@ VPVReductionSC
Definition: VPlanValue.h:95
llvm::GraphTraits< VPBlockBase * >::child_begin
static ChildIteratorType child_begin(NodeRef N)
Definition: VPlan.h:2202
llvm::VPInterleaveRecipe::getAddr
VPValue * getAddr() const
Return the address accessed by this recipe.
Definition: VPlan.h:1452
llvm::VPWidenPointerInductionRecipe::VPWidenPointerInductionRecipe
VPWidenPointerInductionRecipe(PHINode *Phi, VPValue *Start, const InductionDescriptor &IndDesc, ScalarEvolution &SE)
Create a new VPWidenPointerInductionRecipe for Phi with start value Start.
Definition: VPlan.h:1197
llvm::VPReplicateRecipe::execute
void execute(VPTransformState &State) override
Generate replicas of the desired Ingredient.
Definition: LoopVectorize.cpp:9859
llvm::VPInterleaveRecipe::getStoredValues
ArrayRef< VPValue * > getStoredValues() const
Return the VPValues stored by this interleave group.
Definition: VPlan.h:1465
llvm::VPRecipeBase::VPRecipeBase
VPRecipeBase(const unsigned char SC, ArrayRef< VPValue * > Operands)
Definition: VPlan.h:682
llvm::VPUser::setOperand
void setOperand(unsigned I, VPValue *New)
Definition: VPlanValue.h:263
llvm::VPWidenCanonicalIVRecipe::classof
static bool classof(const VPDef *D)
Method to support type inquiry through isa, cast, and dyn_cast.
Definition: VPlan.h:1880
llvm::VPBlockBase::clearPredecessors
void clearPredecessors()
Remove all the predecessor of this block.
Definition: VPlan.h:612
llvm::VPBlockUser::getSingleOperandOrNull
const VPValue * getSingleOperandOrNull() const
Definition: VPlan.h:370
llvm::VPValue::VPVWidenPHISC
@ VPVWidenPHISC
Definition: VPlanValue.h:107
llvm::VPBlockBase::getNumSuccessors
size_t getNumSuccessors() const
Definition: VPlan.h:521
llvm::GraphTraits< VPBlockRecursiveTraversalWrapper< const VPBlockBase * > >::child_begin
static ChildIteratorType child_begin(NodeRef N)
Definition: VPlan.h:2429
llvm::VPLane::getNumCachedLanes
static unsigned getNumCachedLanes(const ElementCount &VF)
Returns the maxmimum number of lanes that we are able to consider caching for VF.
Definition: VPlan.h:175
llvm::VPRecipeBase::execute
virtual void execute(struct VPTransformState &State)=0
The method which generates the output IR instructions that correspond to this VPRecipe,...
llvm::VPlan::getVectorLoopRegion
const VPRegionBlock * getVectorLoopRegion() const
Definition: VPlan.h:2645
llvm::VPInstruction::classof
static bool classof(const VPRecipeBase *R)
Definition: VPlan.h:847
VectorUtils.h
llvm::VPValue::VPVCanonicalIVPHISC
@ VPVCanonicalIVPHISC
Definition: VPlanValue.h:105
llvm::ClrHandlerType::Filter
@ Filter
llvm::VPInterleaveRecipe::getMask
VPValue * getMask() const
Return the mask used by this recipe.
Definition: VPlan.h:1458
llvm::VPTransformState::CanonicalIV
Value * CanonicalIV
Hold the canonical scalar IV of the vector loop (start=0, step=VF*UF).
Definition: VPlan.h:341
llvm::VPRecipeBase::getParent
const VPBasicBlock * getParent() const
Definition: VPlan.h:692
llvm::SCEV
This class represents an analyzed expression in the program.
Definition: ScalarEvolution.h:75
llvm::VPBlendRecipe::onlyFirstLaneUsed
bool onlyFirstLaneUsed(const VPValue *Op) const override
Returns true if the recipe only uses the first lane of operand Op.
Definition: VPlan.h:1407
llvm::VPBasicBlock::rbegin
const_reverse_iterator rbegin() const
Definition: VPlan.h:2008
llvm::VPlan::setName
void setName(const Twine &newName)
Definition: VPlan.h:2553
llvm::VPRecipeBase::removeFromParent
void removeFromParent()
This method unlinks 'this' from the containing basic block, but does not delete it.
Definition: VPlan.cpp:674
llvm::VPBlockBase::getHierarchicalSuccessors
const VPBlocksTy & getHierarchicalSuccessors()
Definition: VPlan.h:541
llvm::VPWidenMemoryInstructionRecipe::classof
static bool classof(const VPDef *D)
Method to support type inquiry through isa, cast, and dyn_cast.
Definition: VPlan.h:1737
llvm::VPPredInstPHIRecipe::~VPPredInstPHIRecipe
~VPPredInstPHIRecipe() override=default
llvm::RISCVFenceField::O
@ O
Definition: RISCVBaseInfo.h:239
llvm::VPBranchOnMaskRecipe::VPBranchOnMaskRecipe
VPBranchOnMaskRecipe(VPValue *BlockInMask)
Definition: VPlan.h:1620
llvm::VPRecipeBase::mayHaveSideEffects
bool mayHaveSideEffects() const
Returns true if the recipe may have side-effects.
Definition: VPlan.cpp:620
llvm::StoreInst
An instruction for storing to memory.
Definition: Instructions.h:305
llvm::VPExpandSCEVRecipe::execute
void execute(VPTransformState &State) override
Generate a canonical vector induction variable of the vector loop, with.
Definition: VPlan.cpp:1495
type
AMD64 Optimization Manual has some nice information about optimizing integer multiplication by a constant How much of it applies to Intel s X86 implementation There are definite trade offs to xmm0 cvttss2siq rdx jb L3 subss xmm0 rax cvttss2siq rdx xorq rdx rax ret instead of xmm1 cvttss2siq rcx movaps xmm2 subss xmm2 cvttss2siq rax rdx xorq rax ucomiss xmm0 cmovb rax ret Seems like the jb branch has high likelihood of being taken It would have saved a few instructions It s not possible to reference and DH registers in an instruction requiring REX prefix divb and mulb both produce results in AH If isel emits a CopyFromReg which gets turned into a movb and that can be allocated a r8b r15b To get around isel emits a CopyFromReg from AX and then right shift it down by and truncate it It s not pretty but it works We need some register allocation magic to make the hack go which would often require a callee saved register Callees usually need to keep this value live for most of their body so it doesn t add a significant burden on them We currently implement this in however this is suboptimal because it means that it would be quite awkward to implement the optimization for callers A better implementation would be to relax the LLVM IR rules for sret arguments to allow a function with an sret argument to have a non void return type
Definition: README-X86-64.txt:70
llvm::VPLane::getAsRuntimeExpr
Value * getAsRuntimeExpr(IRBuilderBase &Builder, const ElementCount &VF) const
Returns an expression describing the lane index that can be used at runtime.
Definition: VPlan.cpp:63
llvm::VPInterleaveRecipe::getNumStoreOperands
unsigned getNumStoreOperands() const
Returns the number of stored operands of this interleave group.
Definition: VPlan.h:1485
llvm::VPReductionRecipe::getCondOp
VPValue * getCondOp() const
The VPValue of the condition for the block.
Definition: VPlan.h:1539
llvm::VPTransformState::reset
void reset(VPValue *Def, Value *V, const VPIteration &Instance)
Reset an existing scalar value for Def and a given Instance.
Definition: VPlan.h:288
VPlanLoopInfo.h
llvm::VPInterleaveRecipe::onlyFirstLaneUsed
bool onlyFirstLaneUsed(const VPValue *Op) const override
The recipe only uses the first lane of the address.
Definition: VPlan.h:1490
llvm::VPlan::isUniformAfterVectorization
bool isUniformAfterVectorization(VPValue *VPV) const
Returns true if VPV is uniform after vectorization.
Definition: VPlan.h:2634
llvm::VPBlockBase::clearSuccessors
void clearSuccessors()
Remove all the successors of this block and set to null its condition bit.
Definition: VPlan.h:615
llvm::VPBlockBase::getSingleHierarchicalSuccessor
VPBlockBase * getSingleHierarchicalSuccessor()
Definition: VPlan.h:547
llvm::VPBlockBase::getSingleHierarchicalPredecessor
VPBlockBase * getSingleHierarchicalPredecessor()
Definition: VPlan.h:563
llvm::VPWidenGEPRecipe::execute
void execute(VPTransformState &State) override
Generate the gep nodes.
Definition: LoopVectorize.cpp:9428
D
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
llvm::VPRegionBlock::setExit
void setExit(VPBlockBase *ExitBlock)
Set ExitBlock as the exit VPBlockBase of this VPRegionBlock.
Definition: VPlan.h:2150
llvm::VPTransformState::DT
DominatorTree * DT
Hold a pointer to Dominator Tree to register new basic blocks in the loop.
Definition: VPlan.h:333
llvm::VPWidenIntOrFpInductionRecipe::execute
void execute(VPTransformState &State) override
Generate the vectorized and scalarized versions of the phi node as needed by their users.
Definition: LoopVectorize.cpp:9501
llvm::VPWidenRecipe
VPWidenRecipe is a recipe for producing a copy of vector type its ingredient.
Definition: VPlan.h:921
llvm::find
auto find(R &&Range, const T &Val)
Provide wrappers to std::find which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1625
const
aarch64 promote const
Definition: AArch64PromoteConstant.cpp:232
llvm::VPInstruction::execute
void execute(VPTransformState &State) override
Generate the instruction.
Definition: VPlan.cpp:818
llvm::VPReductionPHIRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlan.cpp:1648
llvm::InterleaveGroup::getFactor
uint32_t getFactor() const
Definition: VectorUtils.h:625
Addr
uint64_t Addr
Definition: ELFObjHandler.cpp:78
llvm::TruncInst
This class represents a truncation of integer types.
Definition: Instructions.h:4780
llvm::iplist_impl< simple_ilist< VPRecipeBase, Options... >, ilist_traits< VPRecipeBase > >::iterator
base_list_type::iterator iterator
Definition: ilist.h:178
llvm::VPBlockBase::getVPBlockID
unsigned getVPBlockID() const
Definition: VPlan.h:474
llvm::VPBlockBase::dump
LLVM_DUMP_METHOD void dump() const
Dump this VPBlockBase to dbgs().
Definition: VPlan.h:663
llvm::VPLane::getLastLaneForVF
static VPLane getLastLaneForVF(const ElementCount &VF)
Definition: VPlan.h:132
llvm::VPTransformState::UF
unsigned UF
Definition: VPlan.h:208
llvm::DenseMap
Definition: DenseMap.h:716
llvm::VPInstruction::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the VPInstruction to O.
Definition: VPlan.cpp:832
llvm::VPIteration::VPIteration
VPIteration(unsigned Part, unsigned Lane, VPLane::Kind Kind=VPLane::Kind::First)
Definition: VPlan.h:188
llvm::VPTransformState::DataState
Definition: VPlan.h:215
llvm::VPBasicBlock::size
size_t size() const
Definition: VPlan.h:2012
llvm::LoopInfoBase::getLoopFor
LoopT * getLoopFor(const BlockT *BB) const
Return the inner most loop that BB lives in.
Definition: LoopInfo.h:970
llvm::VPValue::VPVWidenSelectSC
@ VPVWidenSelectSC
Definition: VPlanValue.h:101
I
#define I(x, y, z)
Definition: MD5.cpp:58
llvm::VPIteration
VPIteration represents a single point in the iteration space of the output (vectorized and/or unrolle...
Definition: VPlan.h:182
llvm::VPBlockUtils::connectBlocks
static void connectBlocks(VPBlockBase *From, VPBlockBase *To)
Connect VPBlockBases From and To bi-directionally.
Definition: VPlan.h:2791
llvm::GraphTraits< Inverse< VPBlockBase * > >::child_begin
static ChildIteratorType child_begin(NodeRef N)
Definition: VPlan.h:2234
llvm::iterator_facade_base
CRTP base class which implements the entire standard iterator facade in terms of a minimal subset of ...
Definition: iterator.h:80
llvm::VPHeaderPHIRecipe::getStartValue
VPValue * getStartValue()
Returns the start value of the phi, if one is set.
Definition: VPlan.h:1167
llvm::GetElementPtrInst
an instruction for type-safe pointer arithmetic to access elements of arrays and structs
Definition: Instructions.h:929
llvm::make_early_inc_range
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Definition: STLExtras.h:608
llvm::VPHeaderPHIRecipe::classof
static bool classof(const VPValue *V)
Definition: VPlan.h:1149
llvm::VPBlendRecipe::getMask
VPValue * getMask(unsigned Idx) const
Return mask number Idx.
Definition: VPlan.h:1395
llvm::VPTransformState::CFGState::PrevVPBB
VPBasicBlock * PrevVPBB
The previous VPBasicBlock visited. Initially set to null.
Definition: VPlan.h:304
llvm::is_contained
bool is_contained(R &&Range, const E &Element)
Wrapper function around std::find to detect if an element exists in a container.
Definition: STLExtras.h:1670
llvm::VPBlockBase::getNumPredecessors
size_t getNumPredecessors() const
Definition: VPlan.h:522
llvm::VPlan::removeVPValueFor
void removeVPValueFor(Value *V)
Definition: VPlan.h:2602
llvm::VPlan::addVPValue
void addVPValue(Value *V)
Definition: VPlan.h:2563
llvm::VPWidenGEPRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlan.cpp:1354
TemplateParamKind::Type
@ Type
llvm::VPWidenMemoryInstructionRecipe::getStoredValue
VPValue * getStoredValue() const
Return the address accessed by this recipe.
Definition: VPlan.h:1757
llvm::VPExpandSCEVRecipe::VPExpandSCEVRecipe
VPExpandSCEVRecipe(const SCEV *Expr, ScalarEvolution &SE)
Definition: VPlan.h:1799
llvm::VPTransformState::hasAnyVectorValue
bool hasAnyVectorValue(VPValue *Def) const
Definition: VPlan.h:243
llvm::GraphTraits< VPBlockBase * >
Definition: VPlan.h:2196
llvm::VPBasicBlock::VPBasicBlock
VPBasicBlock(const Twine &Name="", VPRecipeBase *Recipe=nullptr)
Definition: VPlan.h:1982
llvm::VPBlockUtils::disconnectBlocks
static void disconnectBlocks(VPBlockBase *From, VPBlockBase *To)
Disconnect VPBlockBases From and To bi-directionally.
Definition: VPlan.h:2802
llvm::VPInstruction::CanonicalIVIncrement
@ CanonicalIVIncrement
Definition: VPlan.h:798
llvm::VPWidenPointerInductionRecipe::~VPWidenPointerInductionRecipe
~VPWidenPointerInductionRecipe() override=default
llvm::VPlan::getEntry
VPBlockBase * getEntry()
Definition: VPlan.h:2517
assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
llvm::VPWidenIntOrFpInductionRecipe::getPHINode
PHINode * getPHINode()
Definition: VPlan.h:1103
llvm::VPFirstOrderRecurrencePHIRecipe::classof
static bool classof(const VPHeaderPHIRecipe *R)
Definition: VPlan.h:1291
llvm::VPHeaderPHIRecipe::classof
static bool classof(const VPRecipeBase *B)
Method to support type inquiry through isa, cast, and dyn_cast.
Definition: VPlan.h:1142
llvm::VPBasicBlock::back
VPRecipeBase & back()
Definition: VPlan.h:2017
llvm::VPWidenIntOrFpInductionRecipe
A recipe for handling phi nodes of integer and floating-point inductions, producing their vector valu...
Definition: VPlan.h:1047
llvm::GraphTraits< VPRegionBlock * >::getEntryNode
static NodeRef getEntryNode(GraphRef N)
Definition: VPlan.h:2255
llvm::GraphTraits< Inverse< VPRegionBlock * > >::nodes_end
static nodes_iterator nodes_end(GraphRef N)
Definition: VPlan.h:2301
llvm::VPReplicateRecipe::isUniform
bool isUniform() const
Definition: VPlan.h:1596
llvm::VPWidenPHIRecipe::classof
static bool classof(const VPRecipeBase *B)
Method to support type inquiry through isa, cast, and dyn_cast.
Definition: VPlan.h:1247
llvm::VPWidenGEPRecipe
A recipe for handling GEP instructions.
Definition: VPlan.h:1006
llvm::VPBasicBlock::execute
void execute(struct VPTransformState *State) override
The method which generates the output IR instructions that correspond to this VPBasicBlock,...
Definition: VPlan.cpp:319
llvm::VPWidenIntOrFpInductionRecipe::VPWidenIntOrFpInductionRecipe
VPWidenIntOrFpInductionRecipe(PHINode *IV, VPValue *Start, VPValue *Step, const InductionDescriptor &IndDesc, bool NeedsScalarIV, bool NeedsVectorIV)
Definition: VPlan.h:1054
llvm::VPInstruction::hasResult
bool hasResult() const
Definition: VPlan.h:875
llvm::SelectInst
This class represents the LLVM 'select' instruction.
Definition: Instructions.h:1737
llvm::ISD::BasicBlock
@ BasicBlock
Various leaf nodes.
Definition: ISDOpcodes.h:71
Mode
SI Whole Quad Mode
Definition: SIWholeQuadMode.cpp:262
function
print Print MemDeps of function
Definition: MemDepPrinter.cpp:82
llvm::VPReplicateRecipe::VPReplicateRecipe
VPReplicateRecipe(Instruction *I, iterator_range< IterT > Operands, bool IsUniform, bool IsPredicated=false)
Definition: VPlan.h:1560
llvm::VPUser::VPUserID::Block
@ Block
llvm::VPlan::~VPlan
~VPlan()
Definition: VPlan.h:2492
llvm::VPWidenPointerInductionRecipe::classof
static bool classof(const VPHeaderPHIRecipe *R)
Definition: VPlan.h:1212
llvm::VPWidenPointerInductionRecipe
Definition: VPlan.h:1186
llvm::VPWidenMemoryInstructionRecipe
A Recipe for widening load/store operations.
Definition: VPlan.h:1698
llvm::VPInstruction::clone
VPInstruction * clone() const
Definition: VPlan.h:831
llvm::VPBasicBlock::~VPBasicBlock
~VPBasicBlock() override
Definition: VPlan.h:1988
llvm::LinearPolySize::getKnownMinValue
ScalarTy getKnownMinValue() const
Returns the minimum value this size can represent.
Definition: TypeSize.h:296
llvm::df_iterator
Definition: DepthFirstIterator.h:86
llvm::VPWidenIntOrFpInductionRecipe::classof
static bool classof(const VPDef *D)
Method to support type inquiry through isa, cast, and dyn_cast.
Definition: VPlan.h:1072
llvm::VPWidenRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlan.cpp:1288
llvm::VPWidenSelectRecipe
A recipe for widening select instructions.
Definition: VPlan.h:975
llvm::GraphTraits< const VPRegionBlock * >::nodes_end
static nodes_iterator nodes_end(GraphRef N)
Definition: VPlan.h:2280
Builder
assume Assume Builder
Definition: AssumeBundleBuilder.cpp:651
llvm::VPInstruction::classof
static bool classof(const VPDef *R)
Method to support type inquiry through isa, cast, and dyn_cast.
Definition: VPlan.h:837
llvm::VPWidenCanonicalIVRecipe::classof
static bool classof(const VPUser *U)
Extra classof implementations to allow directly casting from VPUser -> VPWidenCanonicalIVRecipe.
Definition: VPlan.h:1886
llvm::VPCanonicalIVPHIRecipe::VPCanonicalIVPHIRecipe
VPCanonicalIVPHIRecipe(VPValue *StartV, DebugLoc DL)
Definition: VPlan.h:1830
llvm::SetVector< T, SmallVector< T, N >, SmallDenseSet< T, N > >::insert
bool insert(const value_type &X)
Insert a new element into the SetVector.
Definition: SetVector.h:141
llvm::VPWidenIntOrFpInductionRecipe::isCanonical
bool isCanonical() const
Returns true if the induction is canonical, i.e.
Definition: VPlan.cpp:1322
llvm::VPWidenCallRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlan.cpp:1258
llvm::VPBlockBase::printAsOperand
void printAsOperand(raw_ostream &OS, bool PrintType) const
Definition: VPlan.h:639
llvm::GraphTraits< const VPBlockBase * >
Definition: VPlan.h:2211
llvm::VPlan::getVectorLoopRegion
VPRegionBlock * getVectorLoopRegion()
Returns the VPRegionBlock of the vector loop.
Definition: VPlan.h:2640
llvm::VPBasicBlock::front
VPRecipeBase & front()
Definition: VPlan.h:2015
llvm::VPCanonicalIVPHIRecipe::onlyFirstLaneUsed
bool onlyFirstLaneUsed(const VPValue *Op) const override
Returns true if the recipe only uses the first lane of operand Op.
Definition: VPlan.h:1863
llvm::VPTransformState::reset
void reset(VPValue *Def, Value *V, unsigned Part)
Reset an existing vector value for Def and a given Part.
Definition: VPlan.h:266
llvm::VPScalarIVStepsRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlan.cpp:1346
llvm::VPlanIngredient::V
const Value * V
Definition: VPlan.h:2720
llvm::getRuntimeVF
Value * getRuntimeVF(IRBuilderBase &B, Type *Ty, ElementCount VF)
Return the runtime value for VF.
Definition: LoopVectorize.cpp:1021
llvm::VPFirstOrderRecurrencePHIRecipe::VPFirstOrderRecurrencePHIRecipe
VPFirstOrderRecurrencePHIRecipe(PHINode *Phi, VPValue &Start)
Definition: VPlan.h:1283
llvm::ArrayRef
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: APInt.h:32
llvm::LoopInfo
Definition: LoopInfo.h:1086
llvm::VPReductionRecipe::getVecOp
VPValue * getVecOp() const
The VPValue of the vector value to be reduced.
Definition: VPlan.h:1537
llvm::VPBlockBase::getSingleSuccessor
VPBlockBase * getSingleSuccessor() const
Definition: VPlan.h:511
this
Analysis the ScalarEvolution expression for r is this
Definition: README.txt:8
llvm::VPUser::addOperand
void addOperand(VPValue *Operand)
Definition: VPlanValue.h:252
llvm::VPUser::onlyFirstLaneUsed
virtual bool onlyFirstLaneUsed(const VPValue *Op) const
Returns true if the VPUser only uses the first lane of operand Op.
Definition: VPlanValue.h:301
llvm::VPLane::getKnownLane
unsigned getKnownLane() const
Returns a compile-time known value for the lane index and asserts if the lane can only be calculated ...
Definition: VPlan.h:146
llvm::VPWidenCallRecipe::VPWidenCallRecipe
VPWidenCallRecipe(CallInst &I, iterator_range< IterT > CallArguments)
Definition: VPlan.h:953
llvm_unreachable
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition: ErrorHandling.h:143
llvm::VPValue::VPVWidenIntOrFpInductionSC
@ VPVWidenIntOrFpInductionSC
Definition: VPlanValue.h:108
llvm::Value::getType
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:255
llvm::VPWidenCallRecipe::classof
static bool classof(const VPDef *D)
Method to support type inquiry through isa, cast, and dyn_cast.
Definition: VPlan.h:960
llvm::VPLane::Kind
Kind
Kind describes how to interpret Lane.
Definition: VPlan.h:109
llvm::VPExpandSCEVRecipe::classof
static bool classof(const VPDef *D)
Method to support type inquiry through isa, cast, and dyn_cast.
Definition: VPlan.h:1806
llvm::VPRecipeBase::getParent
VPBasicBlock * getParent()
Definition: VPlan.h:691
llvm::IRBuilderBase
Common base class shared among various IRBuilders.
Definition: IRBuilder.h:93
DL
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Definition: AArch64SLSHardening.cpp:76
llvm::VPIteration::Part
unsigned Part
in [0..UF)
Definition: VPlan.h:184
llvm::VPCanonicalIVPHIRecipe::execute
void execute(VPTransformState &State) override
Generate the canonical scalar induction phi of the vector loop.
Definition: VPlan.cpp:1474
llvm::VPValue::VPVReductionPHISC
@ VPVReductionPHISC
Definition: VPlanValue.h:111
llvm::VPBlockUtils::countSuccessorsNoBE
static unsigned countSuccessorsNoBE(VPBlockBase *PredBlock, VPLoopInfo *VPLI)
Count and return the number of succesors of PredBlock excluding any backedges.
Definition: VPlan.h:2858
llvm::VPFirstOrderRecurrencePHIRecipe
A recipe for handling first-order recurrence phis.
Definition: VPlan.h:1282
llvm::VPExpandSCEVRecipe::~VPExpandSCEVRecipe
~VPExpandSCEVRecipe() override=default
llvm::VPWidenPointerInductionRecipe::classof
static bool classof(const VPValue *V)
Definition: VPlan.h:1215
llvm::VPScalarIVStepsRecipe::getStepValue
VPValue * getStepValue() const
Definition: VPlan.h:1960
llvm::VPAllSuccessorsIterator::VPAllSuccessorsIterator
VPAllSuccessorsIterator(BlockPtrTy Block, size_t Idx=0)
Definition: VPlan.h:2344
llvm::VPBranchOnMaskRecipe
A recipe for generating conditional branches on the bits of a mask.
Definition: VPlan.h:1618
llvm::VPWidenSelectRecipe::execute
void execute(VPTransformState &State) override
Produce a widened version of the select instruction.
Definition: LoopVectorize.cpp:9278
llvm::VPBasicBlock::splitAt
VPBasicBlock * splitAt(iterator SplitAt)
Split current block at SplitAt by inserting a new block between the current block and its successors ...
Definition: VPlan.cpp:409
llvm::VPHeaderPHIRecipe::VPHeaderPHIRecipe
VPHeaderPHIRecipe(unsigned char VPVID, unsigned char VPDefID, PHINode *Phi, VPValue *Start=nullptr)
Definition: VPlan.h:1131
llvm::VPBlockUtils::insertTwoBlocksAfter
static void insertTwoBlocksAfter(VPBlockBase *IfTrue, VPBlockBase *IfFalse, VPValue *Condition, VPBlockBase *BlockPtr)
Insert disconnected VPBlockBases IfTrue and IfFalse after BlockPtr.
Definition: VPlan.h:2774
llvm::VPBlockBase::setOneSuccessor
void setOneSuccessor(VPBlockBase *Successor)
Set a given VPBlockBase Successor as the single successor of this VPBlockBase.
Definition: VPlan.h:584
llvm::VPRegionBlock::getExit
VPBlockBase * getExit()
Definition: VPlan.h:2146
llvm::VPTransformState::VF
ElementCount VF
The chosen Vectorization and Unroll Factors of the loop being vectorized.
Definition: VPlan.h:207
llvm::LoadInst
An instruction for reading from memory.
Definition: Instructions.h:176
llvm::VPWidenPHIRecipe::classof
static bool classof(const VPHeaderPHIRecipe *R)
Definition: VPlan.h:1250
llvm::VPValue::VPInstruction
friend class VPInstruction
Definition: VPlanValue.h:47
llvm::DenseMapBase::insert
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Definition: DenseMap.h:209
llvm::make_filter_range
iterator_range< filter_iterator< detail::IterOfRange< RangeT >, PredicateT > > make_filter_range(RangeT &&Range, PredicateT Pred)
Convenience function that takes a range of elements and a predicate, and return a new filter_iterator...
Definition: STLExtras.h:524
llvm::VPBlockBase::getEntryBasicBlock
const VPBasicBlock * getEntryBasicBlock() const
Definition: VPlan.cpp:140
llvm::vputils::onlyFirstLaneUsed
bool onlyFirstLaneUsed(VPValue *Def)
Returns true if only the first lane of Def is used.
Definition: VPlan.cpp:1771
llvm::VPRegionBlock::getEntry
VPBlockBase * getEntry()
Definition: VPlan.h:2128
llvm::VPValue::VPVMemoryInstructionSC
@ VPVMemoryInstructionSC
Definition: VPlanValue.h:94
llvm::VPBasicBlock::classof
static bool classof(const VPBlockBase *V)
Method to support type inquiry through isa, cast, and dyn_cast.
Definition: VPlan.h:2028
llvm::VPTransformState::Data
struct llvm::VPTransformState::DataState Data
llvm::VPDef::getVPValue
VPValue * getVPValue(unsigned I)
Returns the VPValue with index I defined by the VPDef.
Definition: VPlanValue.h:400
llvm::VPInstruction::onlyFirstLaneUsed
bool onlyFirstLaneUsed(const VPValue *Op) const override
Returns true if the recipe only uses the first lane of operand Op.
Definition: VPlan.h:900
llvm::VPRecipeBase::classof
static bool classof(const VPUser *U)
Definition: VPlan.h:742
llvm::VPFirstOrderRecurrencePHIRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlan.cpp:1574
llvm::VPValue::setUnderlyingValue
void setUnderlyingValue(Value *Val)
Definition: VPlanValue.h:77
llvm::VPReplicateRecipe::setAlsoPack
void setAlsoPack(bool Pack)
Definition: VPlan.h:1588
llvm::VPReductionPHIRecipe::getRecurrenceDescriptor
const RecurrenceDescriptor & getRecurrenceDescriptor() const
Definition: VPlan.h:1353
llvm::depth_first
iterator_range< df_iterator< T > > depth_first(const T &G)
Definition: DepthFirstIterator.h:230
CFG
Common register allocation spilling lr str ldr sxth r3 ldr mla r4 can lr mov lr str ldr sxth r3 mla r4 and then merge mul and lr str ldr sxth r3 mla r4 It also increase the likelihood the store may become dead bb27 Successors according to CFG
Definition: README.txt:39
llvm::VPInstruction::VPInstruction
VPInstruction(unsigned Opcode, ArrayRef< VPValue * > Operands, DebugLoc DL)
Definition: VPlan.h:817
llvm::VPBlockBase::setCondBit
void setCondBit(VPValue *CV)
Set the condition bit selecting the successor.
Definition: VPlan.cpp:199
llvm::VPlan::addVPValue
void addVPValue(Value *V, VPValue *VPV)
Definition: VPlan.h:2573
llvm::GraphTraits< VPRegionBlock * >::nodes_end
static nodes_iterator nodes_end(GraphRef N)
Definition: VPlan.h:2261
llvm::VPBlockBase::setParent
void setParent(VPRegionBlock *P)
Definition: VPlan.h:487
llvm::VPWidenPHIRecipe::addIncoming
void addIncoming(VPValue *IncomingV, VPBasicBlock *IncomingBlock)
Adds a pair (IncomingV, IncomingBlock) to the phi.
Definition: VPlan.h:1267
llvm::VPRecipeBase::VPRecipeBase
VPRecipeBase(const unsigned char SC, iterator_range< IterT > Operands)
Definition: VPlan.h:686
llvm::VPBasicBlock::rend
const_reverse_iterator rend() const
Definition: VPlan.h:2010
llvm::VPlan::mapToVPValues
iterator_range< mapped_iterator< Use *, std::function< VPValue *(Value *)> > > mapToVPValues(User::op_range Operands)
Returns a range mapping the values the range Operands to their corresponding VPValues.
Definition: VPlan.h:2626
llvm::VPUser::classof
static bool classof(const VPDef *Recipe)
Method to support type inquiry through isa, cast, and dyn_cast.
Definition: VPlan.h:766
llvm::VPReductionPHIRecipe::execute
void execute(VPTransformState &State) override
Generate the phi/select nodes.
Definition: VPlan.cpp:1583
llvm::VPBlockUtils::isBackEdge
static bool isBackEdge(const VPBlockBase *FromBlock, const VPBlockBase *ToBlock, const VPLoopInfo *VPLI)
Returns true if the edge FromBlock -> ToBlock is a back-edge.
Definition: VPlan.h:2834
llvm::Twine
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:83
llvm::VPBlockBase::print
void print(raw_ostream &O) const
Print plain-text dump of this VPlan to O.
Definition: VPlan.h:653
llvm::VPSlotTracker
This class can be used to assign consecutive numbers to all VPValues in a VPlan and allows querying t...
Definition: VPlanValue.h:438
llvm::map_range
auto map_range(ContainerTy &&C, FuncTy F)
Definition: STLExtras.h:327
llvm::VPBasicBlock::begin
const_iterator begin() const
Definition: VPlan.h:2003
llvm::VPRegionBlock::getExit
const VPBlockBase * getExit() const
Definition: VPlan.h:2145
llvm::GraphProgram::Name
Name
Definition: GraphWriter.h:50
llvm::VPRecipeBase::~VPRecipeBase
virtual ~VPRecipeBase()=default
llvm::VPWidenGEPRecipe::VPWidenGEPRecipe
VPWidenGEPRecipe(GetElementPtrInst *GEP, iterator_range< IterT > Operands, Loop *OrigLoop)
Definition: VPlan.h:1018
llvm::VPlanSlp::VPlanSlp
VPlanSlp(VPInterleavedAccessInfo &IAI, VPBasicBlock &BB)
Definition: VPlan.h:3008
llvm::VPBlockBase::getParent
VPRegionBlock * getParent()
Definition: VPlan.h:476
llvm::VPAllSuccessorsIterator::operator==
bool operator==(const VPAllSuccessorsIterator &R) const
Definition: VPlan.h:2366
llvm::VPTransformState::CurrentVectorLoop
Loop * CurrentVectorLoop
The loop object for the current parent region, or nullptr.
Definition: VPlan.h:354
llvm::VPBasicBlock::rbegin
reverse_iterator rbegin()
Definition: VPlan.h:2007
llvm::VPAllSuccessorsIterator::end
static VPAllSuccessorsIterator end(BlockPtrTy Block)
Definition: VPlan.h:2355
llvm::VPWidenGEPRecipe::classof
static bool classof(const VPDef *D)
Method to support type inquiry through isa, cast, and dyn_cast.
Definition: VPlan.h:1031
llvm::Successor
@ Successor
Definition: SIMachineScheduler.h:35
llvm::AMDGPU::SendMsg::Op
Op
Definition: SIDefines.h:341
llvm::VPTransformState::MayGeneratePoisonRecipes
SmallPtrSet< VPRecipeBase *, 16 > MayGeneratePoisonRecipes
Holds recipes that may generate a poison value that is used after vectorization, even when their oper...
Definition: VPlan.h:351
llvm::VPValue::getDef
VPDef * getDef()
Definition: VPlanValue.h:181
llvm::ilist_iterator
Iterator for intrusive lists based on ilist_node.
Definition: ilist_iterator.h:57
llvm::VPRegionBlock::VPRegionBlock
VPRegionBlock(const std::string &Name="", bool IsReplicator=false)
Definition: VPlan.h:2110
llvm::VPRecipeBase::classof
static bool classof(const VPDef *D)
Method to support type inquiry through isa, cast, and dyn_cast.
Definition: VPlan.h:737
llvm::VPCanonicalIVPHIRecipe::classof
static bool classof(const VPDef *D)
Method to support type inquiry through isa, cast, and dyn_cast.
Definition: VPlan.h:1838
llvm::LoopInfoBase< VPBlockBase, VPLoop >
llvm::VPWidenGEPRecipe::VPWidenGEPRecipe
VPWidenGEPRecipe(GetElementPtrInst *GEP, iterator_range< IterT > Operands)
Definition: VPlan.h:1012
llvm::VPScalarIVStepsRecipe
A recipe for handling phi nodes of integer and floating-point inductions, producing their scalar valu...
Definition: VPlan.h:1914
llvm::VPRecipeBase
VPRecipeBase is a base class modeling a sequence of one or more output IR instructions.
Definition: VPlan.h:672
llvm::iplist_impl::pop_back
void pop_back()
Definition: ilist.h:319
llvm::VPRegionBlock::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print this VPRegionBlock to O (recursively), prefixing all lines with Indent.
Definition: VPlan.cpp:544
llvm::VPlan::setEntry
VPBlockBase * setEntry(VPBlockBase *Block)
Definition: VPlan.h:2520
llvm::LoopBase::getHeader
BlockT * getHeader() const
Definition: LoopInfo.h:104
llvm::VPReductionRecipe::~VPReductionRecipe
~VPReductionRecipe() override=default
llvm::TargetStackID::Value
Value
Definition: TargetFrameLowering.h:27
llvm::SetVector< T, SmallVector< T, N >, SmallDenseSet< T, N > >::count
size_type count(const key_type &key) const
Count the number of elements of a given key in the SetVector.
Definition: SetVector.h:215
llvm::VPBlockBase::getPredicate
VPValue * getPredicate()
Definition: VPlan.cpp:201
llvm::VPAllSuccessorsIterator::VPAllSuccessorsIterator
VPAllSuccessorsIterator(const VPAllSuccessorsIterator &Other)
Definition: VPlan.h:2346
llvm::SlotTracker
This class provides computation of slot numbers for LLVM Assembly writing.
Definition: AsmWriter.cpp:658
llvm::VPReplicateRecipe::usesScalars
bool usesScalars(const VPValue *Op) const override
Returns true if the recipe uses scalars of operand Op.
Definition: VPlan.h:1610
llvm::VPRecipeBase::mayReadOrWriteMemory
bool mayReadOrWriteMemory() const
Returns true if the recipe may read from or write to memory.
Definition: VPlan.h:761
llvm::VPlan::getVectorTripCount
VPValue & getVectorTripCount()
The vector trip count.
Definition: VPlan.h:2541
llvm::SmallVectorImpl::clear
void clear()
Definition: SmallVector.h:591
llvm::VPInstruction::setUnderlyingInstr
void setUnderlyingInstr(Instruction *I)
Definition: VPlan.h:814
llvm::VPTransformState::CFGState
Hold state information used when constructing the CFG of the output IR, traversing the VPBasicBlocks ...
Definition: VPlan.h:302
llvm::VPValue::VPVFirstOrderRecurrencePHISC
@ VPVFirstOrderRecurrencePHISC
Definition: VPlanValue.h:106
llvm::VPWidenCanonicalIVRecipe::getScalarType
const Type * getScalarType() const
Returns the scalar type of the induction.
Definition: VPlan.h:1906
llvm::VPBlockBase::setName
void setName(const Twine &newName)
Definition: VPlan.h:469
llvm::Inverse
Definition: GraphTraits.h:97
llvm::VPBlockUtils::blockIsLoopLatch
static bool blockIsLoopLatch(const VPBlockBase *Block, const VPLoopInfo *VPLInfo)
Returns true if Block is a loop latch.
Definition: VPlan.h:2848
llvm::VPWidenPointerInductionRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlan.cpp:1311
llvm::VPTransformState::CFGState::CFGState
CFGState()=default
llvm::VPBasicBlock::insert
void insert(VPRecipeBase *Recipe, iterator InsertPt)
Definition: VPlan.h:2032
llvm::VPlan::getVPLoopInfo
VPLoopInfo & getVPLoopInfo()
Return the VPLoopInfo analysis for this VPlan.
Definition: VPlan.h:2609
llvm::VPBlendRecipe
A recipe for vectorizing a phi-node as a sequence of mask-based select instructions.
Definition: VPlan.h:1366
llvm::iplist_impl::insert
iterator insert(iterator where, pointer New)
Definition: ilist.h:229
llvm::VPReplicateRecipe::isPredicated
bool isPredicated() const
Definition: VPlan.h:1600
llvm::VPUser
This class augments VPValue with operands which provide the inverse def-use edges from VPValue's user...
Definition: VPlanValue.h:206
llvm::VPBlockUser::getSingleOperandOrNull
VPValue * getSingleOperandOrNull()
Definition: VPlan.h:364
llvm::VPFirstOrderRecurrencePHIRecipe::classof
static bool classof(const VPValue *V)
Definition: VPlan.h:1294
llvm::RecurrenceDescriptor
The RecurrenceDescriptor is used to identify recurrences variables in a loop.
Definition: IVDescriptors.h:69
llvm::VPlan::VPlan
VPlan(VPBlockBase *Entry=nullptr)
Definition: VPlan.h:2487
llvm::VPBlockBase::print
virtual void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const =0
Print plain-text dump of this VPBlockBase to O, prefixing all lines with Indent.
llvm::VPReductionRecipe::classof
static bool classof(const VPValue *V)
Method to support type inquiry through isa, cast, and dyn_cast.
Definition: VPlan.h:1521
llvm::VPInstruction::FirstOrderRecurrenceSplice
@ FirstOrderRecurrenceSplice
Definition: VPlan.h:790
llvm::VPBlendRecipe::VPBlendRecipe
VPBlendRecipe(PHINode *Phi, ArrayRef< VPValue * > Operands)
The blend operation is a User of the incoming values and of their respective masks,...
Definition: VPlan.h:1373
SmallVector.h
llvm::VPBasicBlock::front
const VPRecipeBase & front() const
Definition: VPlan.h:2014
llvm::VPTransformState::VPValue2Value
VPValue2ValueTy VPValue2Value
Definition: VPlan.h:338
llvm::VPBasicBlock::begin
iterator begin()
Recipe iterator methods.
Definition: VPlan.h:2002
llvm::VPlanSlp::isCompletelySLP
bool isCompletelySLP() const
Return true if all visited instruction can be combined.
Definition: VPlan.h:3020
llvm::VPAllSuccessorsIterator::operator++
VPAllSuccessorsIterator operator++(int X)
Definition: VPlan.h:2379
llvm::VPHeaderPHIRecipe::getStartValue
VPValue * getStartValue() const
Definition: VPlan.h:1170
llvm::VPRegionBlock::setEntry
void setEntry(VPBlockBase *EntryBlock)
Set EntryBlock as the entry VPBlockBase of this VPRegionBlock.
Definition: VPlan.h:2132
llvm::VPAllSuccessorsIterator::operator=
VPAllSuccessorsIterator & operator=(const VPAllSuccessorsIterator &R)
Definition: VPlan.h:2349
llvm::VPWidenMemoryInstructionRecipe::execute
void execute(VPTransformState &State) override
Generate the wide load/store.
Definition: LoopVectorize.cpp:9963
llvm::VPBlendRecipe::classof
static bool classof(const VPDef *D)
Method to support type inquiry through isa, cast, and dyn_cast.
Definition: VPlan.h:1383
SmallBitVector.h
llvm::VPUser::getOperand
VPValue * getOperand(unsigned N) const
Definition: VPlanValue.h:258
llvm::VPTransformState::Instance
Optional< VPIteration > Instance
Hold the indices to generate specific scalar instructions.
Definition: VPlan.h:213
N
#define N
llvm::VPRecipeBase::mayReadFromMemory
bool mayReadFromMemory() const
Returns true if the recipe may read from memory.
Definition: VPlan.cpp:589
llvm::iplist_impl< simple_ilist< VPRecipeBase, Options... >, ilist_traits< VPRecipeBase > >::const_iterator
base_list_type::const_iterator const_iterator
Definition: ilist.h:179
llvm::hash_combine_range
hash_code hash_combine_range(InputIteratorT first, InputIteratorT last)
Compute a hash_code for a sequence of values.
Definition: Hashing.h:483
llvm::VPRegionBlock::classof
static bool classof(const VPBlockBase *V)
Method to support type inquiry through isa, cast, and dyn_cast.
Definition: VPlan.h:2123
llvm::VPlan::printDOT
void printDOT(raw_ostream &O) const
Print this VPlan in DOT format to O.
Definition: VPlan.cpp:1072
llvm::VPlan::getOrCreateBackedgeTakenCount
VPValue * getOrCreateBackedgeTakenCount()
The backedge taken count of the original loop.
Definition: VPlan.h:2534
llvm::VPBasicBlock::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print this VPBsicBlock to O, prefixing all lines with Indent.
Definition: VPlan.cpp:457
llvm::VPBlockUtils::VPBlockUtils
VPBlockUtils()=delete
llvm::iterator_range
A range adaptor for a pair of iterators.
Definition: iterator_range.h:30
llvm::VPExpandSCEVRecipe
Recipe to expand a SCEV expression.
Definition: VPlan.h:1794
llvm::PHINode
Definition: Instructions.h:2664
llvm::VPWidenRecipe::VPWidenRecipe
VPWidenRecipe(Instruction &I, iterator_range< IterT > Operands)
Definition: VPlan.h:924
llvm::VPWidenMemoryInstructionRecipe::isConsecutive
bool isConsecutive() const
Definition: VPlan.h:1763
ilist_node.h
llvm::GraphTraits< VPBlockRecursiveTraversalWrapper< VPBlockBase * > >::child_end
static ChildIteratorType child_end(NodeRef N)
Definition: VPlan.h:2414
llvm::VPFirstOrderRecurrencePHIRecipe::execute
void execute(VPTransformState &State) override
Generate the phi nodes.
Definition: VPlan.cpp:1545
llvm::SmallVectorImpl< VPBlockBase * >
llvm::VPTransformState::CFGState::VPBB2IRBB
SmallDenseMap< VPBasicBlock *, BasicBlock * > VPBB2IRBB
A mapping of each VPBasicBlock to the corresponding BasicBlock.
Definition: VPlan.h:316
llvm::VPBlockBase::getParent
const VPRegionBlock * getParent() const
Definition: VPlan.h:477
llvm::VPTransformState::Plan
VPlan * Plan
Pointer to the VPlan code is generated for.
Definition: VPlan.h:347
llvm::VPLane
In what follows, the term "input IR" refers to code that is fed into the vectorizer whereas the term ...
Definition: VPlan.h:106
llvm::SmallPtrSetImpl
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
Definition: SmallPtrSet.h:344
llvm::SmallSetVector
A SetVector that performs no allocations if smaller than a certain size.
Definition: SetVector.h:307
llvm::VPHeaderPHIRecipe::getBackedgeValue
VPValue * getBackedgeValue()
Returns the incoming value from the loop backedge.
Definition: VPlan.h:1175
llvm::VPFirstOrderRecurrencePHIRecipe::classof
static bool classof(const VPRecipeBase *R)
Method to support type inquiry through isa, cast, and dyn_cast.
Definition: VPlan.h:1288
llvm::VPlan::getVPLoopInfo
const VPLoopInfo & getVPLoopInfo() const
Definition: VPlan.h:2610
llvm::VPPredInstPHIRecipe::VPPredInstPHIRecipe
VPPredInstPHIRecipe(VPValue *PredV)
Construct a VPPredInstPHIRecipe given PredInst whose value needs a phi nodes after merging back from ...
Definition: VPlan.h:1665
llvm::CallInst
This class represents a function call, abstracting a target machine's calling convention.
Definition: Instructions.h:1474
llvm::VPValue::VPVWidenCanonicalIVSC
@ VPVWidenCanonicalIVSC
Definition: VPlanValue.h:99
llvm::VPValue::getVPValueID
unsigned getVPValueID() const
Definition: VPlanValue.h:124
llvm::VPWidenSelectRecipe::VPWidenSelectRecipe
VPWidenSelectRecipe(SelectInst &I, iterator_range< IterT > Operands, bool InvariantCond)
Definition: VPlan.h:982
llvm::VPBranchOnMaskRecipe::getMask
VPValue * getMask() const
Return the mask used by this recipe.
Definition: VPlan.h:1649
BB
Common register allocation spilling lr str ldr sxth r3 ldr mla r4 can lr mov lr str ldr sxth r3 mla r4 and then merge mul and lr str ldr sxth r3 mla r4 It also increase the likelihood the store may become dead bb27 Successors according to LLVM BB
Definition: README.txt:39
GEP
Hexagon Common GEP
Definition: HexagonCommonGEP.cpp:172
llvm::VPBlockBase::deleteCFG
static void deleteCFG(VPBlockBase *Entry)
Delete all blocks reachable from a given VPBlockBase, inclusive.
Definition: VPlan.cpp:213
llvm::VPRegionBlock::isReplicator
bool isReplicator() const
An indicator whether this region is to generate multiple replicated instances of output IR correspond...
Definition: VPlan.h:2165
llvm::VPValue::VPVReplicateSC
@ VPVReplicateSC
Definition: VPlanValue.h:96
llvm::GraphTraits
Definition: GraphTraits.h:37
From
BlockVerifier::State From
Definition: BlockVerifier.cpp:55
llvm::DebugLoc
A debug info location.
Definition: DebugLoc.h:33
llvm::VPWidenPHIRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlan.cpp:1368
llvm::VPInstruction::CanonicalIVIncrementNUW
@ CanonicalIVIncrementNUW
Definition: VPlan.h:799
llvm::VPInstruction::VPInstruction
VPInstruction(unsigned Opcode, std::initializer_list< VPValue * > Operands, DebugLoc DL={})
Definition: VPlan.h:822
llvm::VPWidenIntOrFpInductionRecipe::getTruncInst
const TruncInst * getTruncInst() const
Definition: VPlan.h:1099
llvm::Region
Definition: RegionInfo.h:889
llvm::VFRange::End
ElementCount End
Definition: VPlan.h:82
llvm::VPBasicBlock::phis
iterator_range< iterator > phis()
Returns an iterator range over the PHI-like recipes in the block.
Definition: VPlan.h:2051
llvm::VPBasicBlock::getFirstNonPhi
iterator getFirstNonPhi()
Return the position of the first non-phi node recipe in the block.
Definition: VPlan.cpp:220
llvm::VPReductionRecipe::getChainOp
VPValue * getChainOp() const
The VPValue of the scalar Chain being accumulated.
Definition: VPlan.h:1535
llvm::VPInterleavedAccessInfo::VPInterleavedAccessInfo
VPInterleavedAccessInfo(VPlan &Plan, InterleavedAccessInfo &IAI)
Definition: VPlan.cpp:1740
llvm::VPBlockRecursiveTraversalWrapper
Helper for GraphTraits specialization that traverses through VPRegionBlocks.
Definition: VPlan.h:2387
llvm::VPWidenRecipe::classof
static bool classof(const VPValue *V)
Definition: VPlan.h:934
llvm::VPWidenRecipe::classof
static bool classof(const VPDef *D)
Method to support type inquiry through isa, cast, and dyn_cast.
Definition: VPlan.h:931
llvm::VPBlockBase::getSuccessors
const VPBlocksTy & getSuccessors() const
Definition: VPlan.h:501
llvm::VPHeaderPHIRecipe::~VPHeaderPHIRecipe
~VPHeaderPHIRecipe() override=default
llvm::VPExpandSCEVRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlan.cpp:1508
llvm::GraphTraits< Inverse< VPBlockBase * > >::ChildIteratorType
SmallVectorImpl< VPBlockBase * >::iterator ChildIteratorType
Definition: VPlan.h:2230
llvm::VPBlockRecursiveTraversalWrapper::VPBlockRecursiveTraversalWrapper
VPBlockRecursiveTraversalWrapper(BlockTy Entry)
Definition: VPlan.h:2391
llvm::VPWidenRecipe::~VPWidenRecipe
~VPWidenRecipe() override=default
llvm::VPWidenRecipe::execute
void execute(VPTransformState &State) override
Produce widened copies of all Ingredients.
Definition: LoopVectorize.cpp:9299
llvm::VPDef::getVPDefID
unsigned getVPDefID() const
Definition: VPlanValue.h:420
llvm::VPInstruction::dump
LLVM_DUMP_METHOD void dump() const
Print the VPInstruction to dbgs() (for debugging).
Definition: VPlan.cpp:827
llvm::GraphTraits< VPBlockRecursiveTraversalWrapper< VPBlockBase * > >::getEntryNode
static NodeRef getEntryNode(VPBlockRecursiveTraversalWrapper< VPBlockBase * > N)
Definition: VPlan.h:2406
llvm::VPBasicBlock::rend
reverse_iterator rend()
Definition: VPlan.h:2009
llvm::VPValue::getUnderlyingValue
Value * getUnderlyingValue()
Return the underlying Value attached to this VPValue.
Definition: VPlanValue.h:84
llvm::GraphTraits< VPBlockBase * >::child_end
static ChildIteratorType child_end(NodeRef N)
Definition: VPlan.h:2206
llvm::VPlan
VPlan models a candidate for vectorization, encoding various decisions take to produce efficient outp...
Definition: VPlan.h:2443
llvm::GraphTraits< Inverse< VPRegionBlock * > >::nodes_begin
static nodes_iterator nodes_begin(GraphRef N)
Definition: VPlan.h:2297
llvm::VPWidenIntOrFpInductionRecipe::getStepValue
const VPValue * getStepValue() const
Definition: VPlan.h:1092
llvm::VPBlockBase::setTwoSuccessors
void setTwoSuccessors(VPBlockBase *IfTrue, VPBlockBase *IfFalse, VPValue *Condition)
Set two given VPBlockBases IfTrue and IfFalse to be the two successors of this VPBlockBase.
Definition: VPlan.h:593
llvm::VPInterleavedAccessInfo::~VPInterleavedAccessInfo
~VPInterleavedAccessInfo()
Definition: VPlan.h:2910
llvm::VPBlockBase::setPlan
void setPlan(VPlan *ParentPlan)
Sets the pointer of the plan containing the block.
Definition: VPlan.cpp:154
llvm::Value
LLVM Value Representation.
Definition: Value.h:74
llvm::VPRecipeBase::insertBefore
void insertBefore(VPRecipeBase *InsertPos)
Insert an unlinked recipe into a basic block immediately before the specified recipe.
Definition: VPlan.cpp:650
llvm::VPScalarIVStepsRecipe::~VPScalarIVStepsRecipe
~VPScalarIVStepsRecipe() override=default
llvm::deref
Binary functor that adapts to any other binary functor after dereferencing operands.
Definition: STLExtras.h:1924
llvm::VPCanonicalIVPHIRecipe::classof
static bool classof(const VPValue *V)
Definition: VPlan.h:1844
llvm::VPHeaderPHIRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override=0
Print the recipe.
llvm::VPReductionPHIRecipe::isOrdered
bool isOrdered() const
Returns true, if the phi is part of an ordered reduction.
Definition: VPlan.h:1358
llvm::GraphTraits< const VPRegionBlock * >::getEntryNode
static NodeRef getEntryNode(GraphRef N)
Definition: VPlan.h:2274
llvm::VPlan::dump
LLVM_DUMP_METHOD void dump() const
Dump the plan to stderr (for debugging).
Definition: VPlan.cpp:1078
llvm::VPlanIngredient
Definition: VPlan.h:2719
llvm::VPWidenSelectRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlan.cpp:1275
llvm::VPRecipeBase::isPhi
bool isPhi() const
Returns true for PHI-like recipes.
Definition: VPlan.h:750
llvm::VPWidenIntOrFpInductionRecipe::getStartValue
VPValue * getStartValue()
Returns the start value of the induction.
Definition: VPlan.h:1087
llvm::VPlanPrinter
VPlanPrinter prints a given VPlan to a given output stream.
Definition: VPlan.h:2672
llvm::VPTransformState::CFGState::PrevBB
BasicBlock * PrevBB
The previous IR BasicBlock created or used.
Definition: VPlan.h:308
llvm::GraphTraits< VPBlockBase * >::ChildIteratorType
SmallVectorImpl< VPBlockBase * >::iterator ChildIteratorType
Definition: VPlan.h:2198
llvm::VPBlockUser
VPUsers instance used by VPBlockBase to manage CondBit and the block predicate.
Definition: VPlan.h:361
llvm::VPRecipeBase::insertAfter
void insertAfter(VPRecipeBase *InsertPos)
Insert an unlinked Recipe into a basic block immediately after the specified Recipe.
Definition: VPlan.cpp:666
llvm::VFRange::Start
const ElementCount Start
Definition: VPlan.h:79
Other
Optional< std::vector< StOtherPiece > > Other
Definition: ELFYAML.cpp:1225
llvm::VPBlockUser::resetSingleOpUser
void resetSingleOpUser(VPValue *NewVal)
Definition: VPlan.h:377
llvm::VPTransformState::hasVectorValue
bool hasVectorValue(VPValue *Def, unsigned Part)
Definition: VPlan.h:237
llvm::Use
A Use represents the edge between a Value definition and its users.
Definition: Use.h:43
llvm::VPlan::getOrCreateTripCount
VPValue * getOrCreateTripCount()
The trip count of the original loop.
Definition: VPlan.h:2527
llvm::VPHeaderPHIRecipe::execute
void execute(VPTransformState &State) override=0
Generate the phi nodes.
llvm::SmallPtrSetImpl::insert
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:365
llvm::VPlan::addVF
void addVF(ElementCount VF)
Definition: VPlan.h:2547
llvm::VPInstruction::SLPLoad
@ SLPLoad
Definition: VPlan.h:795