LLVM  6.0.0svn
VPlan.h
Go to the documentation of this file.
1 //===- VPlan.h - Represent A Vectorizer Plan --------------------*- C++ -*-===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 /// \file
11 /// This file contains the declarations of the Vectorization Plan base classes:
12 /// 1. VPBasicBlock and VPRegionBlock that inherit from a common pure virtual
13 /// VPBlockBase, together implementing a Hierarchical CFG;
14 /// 2. Specializations of GraphTraits that allow VPBlockBase graphs to be
15 /// treated as proper graphs for generic algorithms;
16 /// 3. Pure virtual VPRecipeBase serving as the base class for recipes contained
17 /// within VPBasicBlocks;
18 /// 4. VPInstruction, a concrete Recipe and VPUser modeling a single planned
19 /// instruction;
20 /// 5. The VPlan class holding a candidate for vectorization;
21 /// 6. The VPlanPrinter class providing a way to print a plan in dot format;
22 /// These are documented in docs/VectorizationPlan.rst.
23 //
24 //===----------------------------------------------------------------------===//
25 
26 #ifndef LLVM_TRANSFORMS_VECTORIZE_VPLAN_H
27 #define LLVM_TRANSFORMS_VECTORIZE_VPLAN_H
28 
29 #include "VPlanValue.h"
30 #include "llvm/ADT/DenseMap.h"
31 #include "llvm/ADT/GraphTraits.h"
32 #include "llvm/ADT/Optional.h"
33 #include "llvm/ADT/SmallSet.h"
34 #include "llvm/ADT/SmallVector.h"
35 #include "llvm/ADT/Twine.h"
36 #include "llvm/ADT/ilist.h"
37 #include "llvm/ADT/ilist_node.h"
38 #include "llvm/IR/IRBuilder.h"
39 #include <algorithm>
40 #include <cassert>
41 #include <cstddef>
42 #include <map>
43 #include <string>
44 
45 // The (re)use of existing LoopVectorize classes is subject to future VPlan
46 // refactoring.
47 namespace {
48 class LoopVectorizationLegality;
49 class LoopVectorizationCostModel;
50 } // namespace
51 
52 namespace llvm {
53 
54 class BasicBlock;
55 class DominatorTree;
56 class InnerLoopVectorizer;
57 class LoopInfo;
58 class raw_ostream;
59 class Value;
60 class VPBasicBlock;
61 class VPRegionBlock;
62 
63 /// In what follows, the term "input IR" refers to code that is fed into the
64 /// vectorizer whereas the term "output IR" refers to code that is generated by
65 /// the vectorizer.
66 
67 /// VPIteration represents a single point in the iteration space of the output
68 /// (vectorized and/or unrolled) IR loop.
69 struct VPIteration {
70  /// in [0..UF)
71  unsigned Part;
72 
73  /// in [0..VF)
74  unsigned Lane;
75 };
76 
77 /// This is a helper struct for maintaining vectorization state. It's used for
78 /// mapping values from the original loop to their corresponding values in
79 /// the new loop. Two mappings are maintained: one for vectorized values and
80 /// one for scalarized values. Vectorized values are represented with UF
81 /// vector values in the new loop, and scalarized values are represented with
82 /// UF x VF scalar values in the new loop. UF and VF are the unroll and
83 /// vectorization factors, respectively.
84 ///
85 /// Entries can be added to either map with setVectorValue and setScalarValue,
86 /// which assert that an entry was not already added before. If an entry is to
87 /// replace an existing one, call resetVectorValue and resetScalarValue. This is
88 /// currently needed to modify the mapped values during "fix-up" operations that
89 /// occur once the first phase of widening is complete. These operations include
90 /// type truncation and the second phase of recurrence widening.
91 ///
92 /// Entries from either map can be retrieved using the getVectorValue and
93 /// getScalarValue functions, which assert that the desired value exists.
95  friend struct VPTransformState;
96 
97 private:
98  /// The unroll factor. Each entry in the vector map contains UF vector values.
99  unsigned UF;
100 
101  /// The vectorization factor. Each entry in the scalar map contains UF x VF
102  /// scalar values.
103  unsigned VF;
104 
105  /// The vector and scalar map storage. We use std::map and not DenseMap
106  /// because insertions to DenseMap invalidate its iterators.
109  std::map<Value *, VectorParts> VectorMapStorage;
110  std::map<Value *, ScalarParts> ScalarMapStorage;
111 
112 public:
113  /// Construct an empty map with the given unroll and vectorization factors.
114  VectorizerValueMap(unsigned UF, unsigned VF) : UF(UF), VF(VF) {}
115 
116  /// \return True if the map has any vector entry for \p Key.
117  bool hasAnyVectorValue(Value *Key) const {
118  return VectorMapStorage.count(Key);
119  }
120 
121  /// \return True if the map has a vector entry for \p Key and \p Part.
122  bool hasVectorValue(Value *Key, unsigned Part) const {
123  assert(Part < UF && "Queried Vector Part is too large.");
124  if (!hasAnyVectorValue(Key))
125  return false;
126  const VectorParts &Entry = VectorMapStorage.find(Key)->second;
127  assert(Entry.size() == UF && "VectorParts has wrong dimensions.");
128  return Entry[Part] != nullptr;
129  }
130 
131  /// \return True if the map has any scalar entry for \p Key.
132  bool hasAnyScalarValue(Value *Key) const {
133  return ScalarMapStorage.count(Key);
134  }
135 
136  /// \return True if the map has a scalar entry for \p Key and \p Instance.
137  bool hasScalarValue(Value *Key, const VPIteration &Instance) const {
138  assert(Instance.Part < UF && "Queried Scalar Part is too large.");
139  assert(Instance.Lane < VF && "Queried Scalar Lane is too large.");
140  if (!hasAnyScalarValue(Key))
141  return false;
142  const ScalarParts &Entry = ScalarMapStorage.find(Key)->second;
143  assert(Entry.size() == UF && "ScalarParts has wrong dimensions.");
144  assert(Entry[Instance.Part].size() == VF &&
145  "ScalarParts has wrong dimensions.");
146  return Entry[Instance.Part][Instance.Lane] != nullptr;
147  }
148 
149  /// Retrieve the existing vector value that corresponds to \p Key and
150  /// \p Part.
152  assert(hasVectorValue(Key, Part) && "Getting non-existent value.");
153  return VectorMapStorage[Key][Part];
154  }
155 
156  /// Retrieve the existing scalar value that corresponds to \p Key and
157  /// \p Instance.
158  Value *getScalarValue(Value *Key, const VPIteration &Instance) {
159  assert(hasScalarValue(Key, Instance) && "Getting non-existent value.");
160  return ScalarMapStorage[Key][Instance.Part][Instance.Lane];
161  }
162 
163  /// Set a vector value associated with \p Key and \p Part. Assumes such a
164  /// value is not already set. If it is, use resetVectorValue() instead.
165  void setVectorValue(Value *Key, unsigned Part, Value *Vector) {
166  assert(!hasVectorValue(Key, Part) && "Vector value already set for part");
167  if (!VectorMapStorage.count(Key)) {
168  VectorParts Entry(UF);
169  VectorMapStorage[Key] = Entry;
170  }
171  VectorMapStorage[Key][Part] = Vector;
172  }
173 
174  /// Set a scalar value associated with \p Key and \p Instance. Assumes such a
175  /// value is not already set.
176  void setScalarValue(Value *Key, const VPIteration &Instance, Value *Scalar) {
177  assert(!hasScalarValue(Key, Instance) && "Scalar value already set");
178  if (!ScalarMapStorage.count(Key)) {
179  ScalarParts Entry(UF);
180  // TODO: Consider storing uniform values only per-part, as they occupy
181  // lane 0 only, keeping the other VF-1 redundant entries null.
182  for (unsigned Part = 0; Part < UF; ++Part)
183  Entry[Part].resize(VF, nullptr);
184  ScalarMapStorage[Key] = Entry;
185  }
186  ScalarMapStorage[Key][Instance.Part][Instance.Lane] = Scalar;
187  }
188 
189  /// Reset the vector value associated with \p Key for the given \p Part.
190  /// This function can be used to update values that have already been
191  /// vectorized. This is the case for "fix-up" operations including type
192  /// truncation and the second phase of recurrence vectorization.
193  void resetVectorValue(Value *Key, unsigned Part, Value *Vector) {
194  assert(hasVectorValue(Key, Part) && "Vector value not set for part");
195  VectorMapStorage[Key][Part] = Vector;
196  }
197 
198  /// Reset the scalar value associated with \p Key for \p Part and \p Lane.
199  /// This function can be used to update values that have already been
200  /// scalarized. This is the case for "fix-up" operations including scalar phi
201  /// nodes for scalarized and predicated instructions.
202  void resetScalarValue(Value *Key, const VPIteration &Instance,
203  Value *Scalar) {
204  assert(hasScalarValue(Key, Instance) &&
205  "Scalar value not set for part and lane");
206  ScalarMapStorage[Key][Instance.Part][Instance.Lane] = Scalar;
207  }
208 };
209 
210 /// This class is used to enable the VPlan to invoke a method of ILV. This is
211 /// needed until the method is refactored out of ILV and becomes reusable.
212 struct VPCallback {
213  virtual ~VPCallback() {}
214  virtual Value *getOrCreateVectorValues(Value *V, unsigned Part) = 0;
215 };
216 
217 /// VPTransformState holds information passed down when "executing" a VPlan,
218 /// needed for generating the output IR.
220  VPTransformState(unsigned VF, unsigned UF, LoopInfo *LI, DominatorTree *DT,
222  InnerLoopVectorizer *ILV, VPCallback &Callback)
223  : VF(VF), UF(UF), Instance(), LI(LI), DT(DT), Builder(Builder),
224  ValueMap(ValueMap), ILV(ILV), Callback(Callback) {}
225 
226  /// The chosen Vectorization and Unroll Factors of the loop being vectorized.
227  unsigned VF;
228  unsigned UF;
229 
230  /// Hold the indices to generate specific scalar instructions. Null indicates
231  /// that all instances are to be generated, using either scalar or vector
232  /// instructions.
234 
235  struct DataState {
236  /// A type for vectorized values in the new loop. Each value from the
237  /// original loop, when vectorized, is represented by UF vector values in
238  /// the new unrolled loop, where UF is the unroll factor.
240 
242  } Data;
243 
244  /// Get the generated Value for a given VPValue and a given Part. Note that
245  /// as some Defs are still created by ILV and managed in its ValueMap, this
246  /// method will delegate the call to ILV in such cases in order to provide
247  /// callers a consistent API.
248  /// \see set.
249  Value *get(VPValue *Def, unsigned Part) {
250  // If Values have been set for this Def return the one relevant for \p Part.
251  if (Data.PerPartOutput.count(Def))
252  return Data.PerPartOutput[Def][Part];
253  // Def is managed by ILV: bring the Values from ValueMap.
254  return Callback.getOrCreateVectorValues(VPValue2Value[Def], Part);
255  }
256 
257  /// Set the generated Value for a given VPValue and a given Part.
258  void set(VPValue *Def, Value *V, unsigned Part) {
259  if (!Data.PerPartOutput.count(Def)) {
260  DataState::PerPartValuesTy Entry(UF);
261  Data.PerPartOutput[Def] = Entry;
262  }
263  Data.PerPartOutput[Def][Part] = V;
264  }
265 
266  /// Hold state information used when constructing the CFG of the output IR,
267  /// traversing the VPBasicBlocks and generating corresponding IR BasicBlocks.
268  struct CFGState {
269  /// The previous VPBasicBlock visited. Initially set to null.
270  VPBasicBlock *PrevVPBB = nullptr;
271 
272  /// The previous IR BasicBlock created or used. Initially set to the new
273  /// header BasicBlock.
274  BasicBlock *PrevBB = nullptr;
275 
276  /// The last IR BasicBlock in the output IR. Set to the new latch
277  /// BasicBlock, used for placing the newly created BasicBlocks.
278  BasicBlock *LastBB = nullptr;
279 
280  /// A mapping of each VPBasicBlock to the corresponding BasicBlock. In case
281  /// of replication, maps the BasicBlock of the last replica created.
283 
284  CFGState() = default;
285  } CFG;
286 
287  /// Hold a pointer to LoopInfo to register new basic blocks in the loop.
289 
290  /// Hold a pointer to Dominator Tree to register new basic blocks in the loop.
292 
293  /// Hold a reference to the IRBuilder used to generate output IR code.
295 
296  /// Hold a reference to the Value state information used when generating the
297  /// Values of the output IR.
299 
300  /// Hold a reference to a mapping between VPValues in VPlan and original
301  /// Values they correspond to.
303 
304  /// Hold a pointer to InnerLoopVectorizer to reuse its IR generation methods.
306 
308 };
309 
310 /// VPBlockBase is the building block of the Hierarchical Control-Flow Graph.
311 /// A VPBlockBase can be either a VPBasicBlock or a VPRegionBlock.
312 class VPBlockBase {
313 private:
314  const unsigned char SubclassID; ///< Subclass identifier (for isa/dyn_cast).
315 
316  /// An optional name for the block.
317  std::string Name;
318 
319  /// The immediate VPRegionBlock which this VPBlockBase belongs to, or null if
320  /// it is a topmost VPBlockBase.
321  VPRegionBlock *Parent = nullptr;
322 
323  /// List of predecessor blocks.
324  SmallVector<VPBlockBase *, 1> Predecessors;
325 
326  /// List of successor blocks.
328 
329  /// Add \p Successor as the last successor to this block.
330  void appendSuccessor(VPBlockBase *Successor) {
331  assert(Successor && "Cannot add nullptr successor!");
332  Successors.push_back(Successor);
333  }
334 
335  /// Add \p Predecessor as the last predecessor to this block.
336  void appendPredecessor(VPBlockBase *Predecessor) {
337  assert(Predecessor && "Cannot add nullptr predecessor!");
338  Predecessors.push_back(Predecessor);
339  }
340 
341  /// Remove \p Predecessor from the predecessors of this block.
342  void removePredecessor(VPBlockBase *Predecessor) {
343  auto Pos = std::find(Predecessors.begin(), Predecessors.end(), Predecessor);
344  assert(Pos && "Predecessor does not exist");
345  Predecessors.erase(Pos);
346  }
347 
348  /// Remove \p Successor from the successors of this block.
349  void removeSuccessor(VPBlockBase *Successor) {
350  auto Pos = std::find(Successors.begin(), Successors.end(), Successor);
351  assert(Pos && "Successor does not exist");
352  Successors.erase(Pos);
353  }
354 
355 protected:
356  VPBlockBase(const unsigned char SC, const std::string &N)
357  : SubclassID(SC), Name(N) {}
358 
359 public:
360  /// An enumeration for keeping track of the concrete subclass of VPBlockBase
361  /// that are actually instantiated. Values of this enumeration are kept in the
362  /// SubclassID field of the VPBlockBase objects. They are used for concrete
363  /// type identification.
364  using VPBlockTy = enum { VPBasicBlockSC, VPRegionBlockSC };
365 
367 
368  virtual ~VPBlockBase() = default;
369 
370  const std::string &getName() const { return Name; }
371 
372  void setName(const Twine &newName) { Name = newName.str(); }
373 
374  /// \return an ID for the concrete type of this object.
375  /// This is used to implement the classof checks. This should not be used
376  /// for any other purpose, as the values may change as LLVM evolves.
377  unsigned getVPBlockID() const { return SubclassID; }
378 
379  const VPRegionBlock *getParent() const { return Parent; }
380 
381  void setParent(VPRegionBlock *P) { Parent = P; }
382 
383  /// \return the VPBasicBlock that is the entry of this VPBlockBase,
384  /// recursively, if the latter is a VPRegionBlock. Otherwise, if this
385  /// VPBlockBase is a VPBasicBlock, it is returned.
386  const VPBasicBlock *getEntryBasicBlock() const;
387  VPBasicBlock *getEntryBasicBlock();
388 
389  /// \return the VPBasicBlock that is the exit of this VPBlockBase,
390  /// recursively, if the latter is a VPRegionBlock. Otherwise, if this
391  /// VPBlockBase is a VPBasicBlock, it is returned.
392  const VPBasicBlock *getExitBasicBlock() const;
393  VPBasicBlock *getExitBasicBlock();
394 
395  const VPBlocksTy &getSuccessors() const { return Successors; }
396  VPBlocksTy &getSuccessors() { return Successors; }
397 
398  const VPBlocksTy &getPredecessors() const { return Predecessors; }
399  VPBlocksTy &getPredecessors() { return Predecessors; }
400 
401  /// \return the successor of this VPBlockBase if it has a single successor.
402  /// Otherwise return a null pointer.
404  return (Successors.size() == 1 ? *Successors.begin() : nullptr);
405  }
406 
407  /// \return the predecessor of this VPBlockBase if it has a single
408  /// predecessor. Otherwise return a null pointer.
410  return (Predecessors.size() == 1 ? *Predecessors.begin() : nullptr);
411  }
412 
413  /// An Enclosing Block of a block B is any block containing B, including B
414  /// itself. \return the closest enclosing block starting from "this", which
415  /// has successors. \return the root enclosing block if all enclosing blocks
416  /// have no successors.
417  VPBlockBase *getEnclosingBlockWithSuccessors();
418 
419  /// \return the closest enclosing block starting from "this", which has
420  /// predecessors. \return the root enclosing block if all enclosing blocks
421  /// have no predecessors.
422  VPBlockBase *getEnclosingBlockWithPredecessors();
423 
424  /// \return the successors either attached directly to this VPBlockBase or, if
425  /// this VPBlockBase is the exit block of a VPRegionBlock and has no
426  /// successors of its own, search recursively for the first enclosing
427  /// VPRegionBlock that has successors and return them. If no such
428  /// VPRegionBlock exists, return the (empty) successors of the topmost
429  /// VPBlockBase reached.
431  return getEnclosingBlockWithSuccessors()->getSuccessors();
432  }
433 
434  /// \return the hierarchical successor of this VPBlockBase if it has a single
435  /// hierarchical successor. Otherwise return a null pointer.
437  return getEnclosingBlockWithSuccessors()->getSingleSuccessor();
438  }
439 
440  /// \return the predecessors either attached directly to this VPBlockBase or,
441  /// if this VPBlockBase is the entry block of a VPRegionBlock and has no
442  /// predecessors of its own, search recursively for the first enclosing
443  /// VPRegionBlock that has predecessors and return them. If no such
444  /// VPRegionBlock exists, return the (empty) predecessors of the topmost
445  /// VPBlockBase reached.
447  return getEnclosingBlockWithPredecessors()->getPredecessors();
448  }
449 
450  /// \return the hierarchical predecessor of this VPBlockBase if it has a
451  /// single hierarchical predecessor. Otherwise return a null pointer.
453  return getEnclosingBlockWithPredecessors()->getSinglePredecessor();
454  }
455 
456  /// Sets a given VPBlockBase \p Successor as the single successor and \return
457  /// \p Successor. The parent of this Block is copied to be the parent of
458  /// \p Successor.
460  assert(Successors.empty() && "Setting one successor when others exist.");
461  appendSuccessor(Successor);
462  Successor->appendPredecessor(this);
463  Successor->Parent = Parent;
464  return Successor;
465  }
466 
467  /// Sets two given VPBlockBases \p IfTrue and \p IfFalse to be the two
468  /// successors. The parent of this Block is copied to be the parent of both
469  /// \p IfTrue and \p IfFalse.
470  void setTwoSuccessors(VPBlockBase *IfTrue, VPBlockBase *IfFalse) {
471  assert(Successors.empty() && "Setting two successors when others exist.");
472  appendSuccessor(IfTrue);
473  appendSuccessor(IfFalse);
474  IfTrue->appendPredecessor(this);
475  IfFalse->appendPredecessor(this);
476  IfTrue->Parent = Parent;
477  IfFalse->Parent = Parent;
478  }
479 
480  void disconnectSuccessor(VPBlockBase *Successor) {
481  assert(Successor && "Successor to disconnect is null.");
482  removeSuccessor(Successor);
483  Successor->removePredecessor(this);
484  }
485 
486  /// The method which generates the output IR that correspond to this
487  /// VPBlockBase, thereby "executing" the VPlan.
488  virtual void execute(struct VPTransformState *State) = 0;
489 
490  /// Delete all blocks reachable from a given VPBlockBase, inclusive.
491  static void deleteCFG(VPBlockBase *Entry);
492 };
493 
494 /// VPRecipeBase is a base class modeling a sequence of one or more output IR
495 /// instructions.
496 class VPRecipeBase : public ilist_node_with_parent<VPRecipeBase, VPBasicBlock> {
497  friend VPBasicBlock;
498 
499 private:
500  const unsigned char SubclassID; ///< Subclass identifier (for isa/dyn_cast).
501 
502  /// Each VPRecipe belongs to a single VPBasicBlock.
503  VPBasicBlock *Parent = nullptr;
504 
505 public:
506  /// An enumeration for keeping track of the concrete subclass of VPRecipeBase
507  /// that is actually instantiated. Values of this enumeration are kept in the
508  /// SubclassID field of the VPRecipeBase objects. They are used for concrete
509  /// type identification.
510  using VPRecipeTy = enum {
511  VPBlendSC,
512  VPBranchOnMaskSC,
513  VPInstructionSC,
514  VPInterleaveSC,
515  VPPredInstPHISC,
516  VPReplicateSC,
517  VPWidenIntOrFpInductionSC,
518  VPWidenMemoryInstructionSC,
519  VPWidenPHISC,
520  VPWidenSC,
521  };
522 
523  VPRecipeBase(const unsigned char SC) : SubclassID(SC) {}
524  virtual ~VPRecipeBase() = default;
525 
526  /// \return an ID for the concrete type of this object.
527  /// This is used to implement the classof checks. This should not be used
528  /// for any other purpose, as the values may change as LLVM evolves.
529  unsigned getVPRecipeID() const { return SubclassID; }
530 
531  /// \return the VPBasicBlock which this VPRecipe belongs to.
532  VPBasicBlock *getParent() { return Parent; }
533  const VPBasicBlock *getParent() const { return Parent; }
534 
535  /// The method which generates the output IR instructions that correspond to
536  /// this VPRecipe, thereby "executing" the VPlan.
537  virtual void execute(struct VPTransformState &State) = 0;
538 
539  /// Each recipe prints itself.
540  virtual void print(raw_ostream &O, const Twine &Indent) const = 0;
541 };
542 
543 /// This is a concrete Recipe that models a single VPlan-level instruction.
544 /// While as any Recipe it may generate a sequence of IR instructions when
545 /// executed, these instructions would always form a single-def expression as
546 /// the VPInstruction is also a single def-use vertex.
547 class VPInstruction : public VPUser, public VPRecipeBase {
548 public:
549  /// VPlan opcodes, extending LLVM IR with idiomatics instructions.
550  enum { Not = Instruction::OtherOpsEnd + 1 };
551 
552 private:
553  typedef unsigned char OpcodeTy;
554  OpcodeTy Opcode;
555 
556  /// Utility method serving execute(): generates a single instance of the
557  /// modeled instruction.
558  void generateInstruction(VPTransformState &State, unsigned Part);
559 
560 public:
561  VPInstruction(unsigned Opcode, std::initializer_list<VPValue *> Operands)
562  : VPUser(VPValue::VPInstructionSC, Operands),
563  VPRecipeBase(VPRecipeBase::VPInstructionSC), Opcode(Opcode) {}
564 
565  /// Method to support type inquiry through isa, cast, and dyn_cast.
566  static inline bool classof(const VPValue *V) {
567  return V->getVPValueID() == VPValue::VPInstructionSC;
568  }
569 
570  /// Method to support type inquiry through isa, cast, and dyn_cast.
571  static inline bool classof(const VPRecipeBase *R) {
572  return R->getVPRecipeID() == VPRecipeBase::VPInstructionSC;
573  }
574 
575  unsigned getOpcode() const { return Opcode; }
576 
577  /// Generate the instruction.
578  /// TODO: We currently execute only per-part unless a specific instance is
579  /// provided.
580  void execute(VPTransformState &State) override;
581 
582  /// Print the Recipe.
583  void print(raw_ostream &O, const Twine &Indent) const override;
584 
585  /// Print the VPInstruction.
586  void print(raw_ostream &O) const;
587 };
588 
589 /// VPBasicBlock serves as the leaf of the Hierarchical Control-Flow Graph. It
590 /// holds a sequence of zero or more VPRecipe's each representing a sequence of
591 /// output IR instructions.
592 class VPBasicBlock : public VPBlockBase {
593 public:
595 
596 private:
597  /// The VPRecipes held in the order of output instructions to generate.
598  RecipeListTy Recipes;
599 
600 public:
601  VPBasicBlock(const Twine &Name = "", VPRecipeBase *Recipe = nullptr)
602  : VPBlockBase(VPBasicBlockSC, Name.str()) {
603  if (Recipe)
604  appendRecipe(Recipe);
605  }
606 
607  ~VPBasicBlock() override { Recipes.clear(); }
608 
609  /// Instruction iterators...
614 
615  //===--------------------------------------------------------------------===//
616  /// Recipe iterator methods
617  ///
618  inline iterator begin() { return Recipes.begin(); }
619  inline const_iterator begin() const { return Recipes.begin(); }
620  inline iterator end() { return Recipes.end(); }
621  inline const_iterator end() const { return Recipes.end(); }
622 
623  inline reverse_iterator rbegin() { return Recipes.rbegin(); }
624  inline const_reverse_iterator rbegin() const { return Recipes.rbegin(); }
625  inline reverse_iterator rend() { return Recipes.rend(); }
626  inline const_reverse_iterator rend() const { return Recipes.rend(); }
627 
628  inline size_t size() const { return Recipes.size(); }
629  inline bool empty() const { return Recipes.empty(); }
630  inline const VPRecipeBase &front() const { return Recipes.front(); }
631  inline VPRecipeBase &front() { return Recipes.front(); }
632  inline const VPRecipeBase &back() const { return Recipes.back(); }
633  inline VPRecipeBase &back() { return Recipes.back(); }
634 
635  /// \brief Returns a pointer to a member of the recipe list.
637  return &VPBasicBlock::Recipes;
638  }
639 
640  /// Method to support type inquiry through isa, cast, and dyn_cast.
641  static inline bool classof(const VPBlockBase *V) {
642  return V->getVPBlockID() == VPBlockBase::VPBasicBlockSC;
643  }
644 
645  void insert(VPRecipeBase *Recipe, iterator InsertPt) {
646  assert(Recipe && "No recipe to append.");
647  assert(!Recipe->Parent && "Recipe already in VPlan");
648  Recipe->Parent = this;
649  Recipes.insert(InsertPt, Recipe);
650  }
651 
652  /// Augment the existing recipes of a VPBasicBlock with an additional
653  /// \p Recipe as the last recipe.
654  void appendRecipe(VPRecipeBase *Recipe) { insert(Recipe, end()); }
655 
656  /// The method which generates the output IR instructions that correspond to
657  /// this VPBasicBlock, thereby "executing" the VPlan.
658  void execute(struct VPTransformState *State) override;
659 
660 private:
661  /// Create an IR BasicBlock to hold the output instructions generated by this
662  /// VPBasicBlock, and return it. Update the CFGState accordingly.
663  BasicBlock *createEmptyBasicBlock(VPTransformState::CFGState &CFG);
664 };
665 
666 /// VPRegionBlock represents a collection of VPBasicBlocks and VPRegionBlocks
667 /// which form a Single-Entry-Single-Exit subgraph of the output IR CFG.
668 /// A VPRegionBlock may indicate that its contents are to be replicated several
669 /// times. This is designed to support predicated scalarization, in which a
670 /// scalar if-then code structure needs to be generated VF * UF times. Having
671 /// this replication indicator helps to keep a single model for multiple
672 /// candidate VF's. The actual replication takes place only once the desired VF
673 /// and UF have been determined.
674 class VPRegionBlock : public VPBlockBase {
675 private:
676  /// Hold the Single Entry of the SESE region modelled by the VPRegionBlock.
677  VPBlockBase *Entry;
678 
679  /// Hold the Single Exit of the SESE region modelled by the VPRegionBlock.
680  VPBlockBase *Exit;
681 
682  /// An indicator whether this region is to generate multiple replicated
683  /// instances of output IR corresponding to its VPBlockBases.
684  bool IsReplicator;
685 
686 public:
688  const std::string &Name = "", bool IsReplicator = false)
689  : VPBlockBase(VPRegionBlockSC, Name), Entry(Entry), Exit(Exit),
690  IsReplicator(IsReplicator) {
691  assert(Entry->getPredecessors().empty() && "Entry block has predecessors.");
692  assert(Exit->getSuccessors().empty() && "Exit block has successors.");
693  Entry->setParent(this);
694  Exit->setParent(this);
695  }
696 
697  ~VPRegionBlock() override {
698  if (Entry)
699  deleteCFG(Entry);
700  }
701 
702  /// Method to support type inquiry through isa, cast, and dyn_cast.
703  static inline bool classof(const VPBlockBase *V) {
704  return V->getVPBlockID() == VPBlockBase::VPRegionBlockSC;
705  }
706 
707  const VPBlockBase *getEntry() const { return Entry; }
708  VPBlockBase *getEntry() { return Entry; }
709 
710  const VPBlockBase *getExit() const { return Exit; }
711  VPBlockBase *getExit() { return Exit; }
712 
713  /// An indicator whether this region is to generate multiple replicated
714  /// instances of output IR corresponding to its VPBlockBases.
715  bool isReplicator() const { return IsReplicator; }
716 
717  /// The method which generates the output IR instructions that correspond to
718  /// this VPRegionBlock, thereby "executing" the VPlan.
719  void execute(struct VPTransformState *State) override;
720 };
721 
722 /// VPlan models a candidate for vectorization, encoding various decisions take
723 /// to produce efficient output IR, including which branches, basic-blocks and
724 /// output IR instructions to generate, and their cost. VPlan holds a
725 /// Hierarchical-CFG of VPBasicBlocks and VPRegionBlocks rooted at an Entry
726 /// VPBlock.
727 class VPlan {
728  friend class VPlanPrinter;
729 
730 private:
731  /// Hold the single entry to the Hierarchical CFG of the VPlan.
732  VPBlockBase *Entry;
733 
734  /// Holds the VFs applicable to this VPlan.
736 
737  /// Holds the name of the VPlan, for printing.
738  std::string Name;
739 
740  /// Holds a mapping between Values and their corresponding VPValue inside
741  /// VPlan.
742  Value2VPValueTy Value2VPValue;
743 
744 public:
745  VPlan(VPBlockBase *Entry = nullptr) : Entry(Entry) {}
746 
747  ~VPlan() {
748  if (Entry)
749  VPBlockBase::deleteCFG(Entry);
750  for (auto &MapEntry : Value2VPValue)
751  delete MapEntry.second;
752  }
753 
754  /// Generate the IR code for this VPlan.
755  void execute(struct VPTransformState *State);
756 
757  VPBlockBase *getEntry() { return Entry; }
758  const VPBlockBase *getEntry() const { return Entry; }
759 
760  VPBlockBase *setEntry(VPBlockBase *Block) { return Entry = Block; }
761 
762  void addVF(unsigned VF) { VFs.insert(VF); }
763 
764  bool hasVF(unsigned VF) { return VFs.count(VF); }
765 
766  const std::string &getName() const { return Name; }
767 
768  void setName(const Twine &newName) { Name = newName.str(); }
769 
770  void addVPValue(Value *V) {
771  assert(V && "Trying to add a null Value to VPlan");
772  assert(!Value2VPValue.count(V) && "Value already exists in VPlan");
773  Value2VPValue[V] = new VPValue();
774  }
775 
777  assert(V && "Trying to get the VPValue of a null Value");
778  assert(Value2VPValue.count(V) && "Value does not exist in VPlan");
779  return Value2VPValue[V];
780  }
781 
782 private:
783  /// Add to the given dominator tree the header block and every new basic block
784  /// that was created between it and the latch block, inclusive.
785  static void updateDominatorTree(DominatorTree *DT,
786  BasicBlock *LoopPreHeaderBB,
787  BasicBlock *LoopLatchBB);
788 };
789 
790 /// VPlanPrinter prints a given VPlan to a given output stream. The printing is
791 /// indented and follows the dot format.
793  friend inline raw_ostream &operator<<(raw_ostream &OS, VPlan &Plan);
794  friend inline raw_ostream &operator<<(raw_ostream &OS,
795  const struct VPlanIngredient &I);
796 
797 private:
798  raw_ostream &OS;
799  VPlan &Plan;
800  unsigned Depth;
801  unsigned TabWidth = 2;
802  std::string Indent;
803  unsigned BID = 0;
805 
806  VPlanPrinter(raw_ostream &O, VPlan &P) : OS(O), Plan(P) {}
807 
808  /// Handle indentation.
809  void bumpIndent(int b) { Indent = std::string((Depth += b) * TabWidth, ' '); }
810 
811  /// Print a given \p Block of the Plan.
812  void dumpBlock(const VPBlockBase *Block);
813 
814  /// Print the information related to the CFG edges going out of a given
815  /// \p Block, followed by printing the successor blocks themselves.
816  void dumpEdges(const VPBlockBase *Block);
817 
818  /// Print a given \p BasicBlock, including its VPRecipes, followed by printing
819  /// its successor blocks.
820  void dumpBasicBlock(const VPBasicBlock *BasicBlock);
821 
822  /// Print a given \p Region of the Plan.
823  void dumpRegion(const VPRegionBlock *Region);
824 
825  unsigned getOrCreateBID(const VPBlockBase *Block) {
826  return BlockID.count(Block) ? BlockID[Block] : BlockID[Block] = BID++;
827  }
828 
829  const Twine getOrCreateName(const VPBlockBase *Block);
830 
831  const Twine getUID(const VPBlockBase *Block);
832 
833  /// Print the information related to a CFG edge between two VPBlockBases.
834  void drawEdge(const VPBlockBase *From, const VPBlockBase *To, bool Hidden,
835  const Twine &Label);
836 
837  void dump();
838 
839  static void printAsIngredient(raw_ostream &O, Value *V);
840 };
841 
844 
845  VPlanIngredient(Value *V) : V(V) {}
846 };
847 
849  VPlanPrinter::printAsIngredient(OS, I.V);
850  return OS;
851 }
852 
854  VPlanPrinter Printer(OS, Plan);
855  Printer.dump();
856  return OS;
857 }
858 
859 //===--------------------------------------------------------------------===//
860 // GraphTraits specializations for VPlan/VPRegionBlock Control-Flow Graphs //
861 //===--------------------------------------------------------------------===//
862 
863 // Provide specializations of GraphTraits to be able to treat a VPBlockBase as a
864 // graph of VPBlockBase nodes...
865 
866 template <> struct GraphTraits<VPBlockBase *> {
867  using NodeRef = VPBlockBase *;
869 
870  static NodeRef getEntryNode(NodeRef N) { return N; }
871 
873  return N->getSuccessors().begin();
874  }
875 
877  return N->getSuccessors().end();
878  }
879 };
880 
881 template <> struct GraphTraits<const VPBlockBase *> {
882  using NodeRef = const VPBlockBase *;
884 
885  static NodeRef getEntryNode(NodeRef N) { return N; }
886 
888  return N->getSuccessors().begin();
889  }
890 
892  return N->getSuccessors().end();
893  }
894 };
895 
896 // Provide specializations of GraphTraits to be able to treat a VPBlockBase as a
897 // graph of VPBlockBase nodes... and to walk it in inverse order. Inverse order
898 // for a VPBlockBase is considered to be when traversing the predecessors of a
899 // VPBlockBase instead of its successors.
900 template <> struct GraphTraits<Inverse<VPBlockBase *>> {
901  using NodeRef = VPBlockBase *;
903 
905  return B;
906  }
907 
909  return N->getPredecessors().begin();
910  }
911 
913  return N->getPredecessors().end();
914  }
915 };
916 
917 } // end namespace llvm
918 
919 #endif // LLVM_TRANSFORMS_VECTORIZE_VPLAN_H
const std::string & getName() const
Definition: VPlan.h:370
LoopInfo * LI
Hold a pointer to LoopInfo to register new basic blocks in the loop.
Definition: VPlan.h:288
const_iterator end(StringRef path)
Get end iterator over path.
Definition: Path.cpp:244
const VPRegionBlock * getParent() const
Definition: VPlan.h:379
enum { VPBasicBlockSC, VPRegionBlockSC } VPBlockTy
An enumeration for keeping track of the concrete subclass of VPBlockBase that are actually instantiat...
Definition: VPlan.h:364
typename SuperClass::const_iterator const_iterator
Definition: SmallVector.h:329
VectorizerValueMap(unsigned UF, unsigned VF)
Construct an empty map with the given unroll and vectorization factors.
Definition: VPlan.h:114
~VPBasicBlock() override
Definition: VPlan.h:607
SmallVectorImpl< VPBlockBase * >::const_iterator ChildIteratorType
Definition: VPlan.h:883
Compute iterated dominance frontiers using a linear time algorithm.
Definition: AllocatorList.h:24
InnerLoopVectorizer vectorizes loops which contain only one basic block to a specified vectorization ...
Various leaf nodes.
Definition: ISDOpcodes.h:60
VPRegionBlock(VPBlockBase *Entry, VPBlockBase *Exit, const std::string &Name="", bool IsReplicator=false)
Definition: VPlan.h:687
const_reverse_iterator rbegin() const
Definition: VPlan.h:624
Optional< VPIteration > Instance
Hold the indices to generate specific scalar instructions.
Definition: VPlan.h:233
LLVM_ATTRIBUTE_ALWAYS_INLINE size_type size() const
Definition: SmallVector.h:136
VPlan models a candidate for vectorization, encoding various decisions take to produce efficient outp...
Definition: VPlan.h:727
DenseMap< VPValue *, PerPartValuesTy > PerPartOutput
Definition: VPlan.h:241
Value * getScalarValue(Value *Key, const VPIteration &Instance)
Retrieve the existing scalar value that corresponds to Key and Instance.
Definition: VPlan.h:158
VPlanIngredient(Value *V)
Definition: VPlan.h:845
VPRegionBlock represents a collection of VPBasicBlocks and VPRegionBlocks which form a Single-Entry-S...
Definition: VPlan.h:674
enum { VPBlendSC, VPBranchOnMaskSC, VPInstructionSC, VPInterleaveSC, VPPredInstPHISC, VPReplicateSC, VPWidenIntOrFpInductionSC, VPWidenMemoryInstructionSC, VPWidenPHISC, VPWidenSC, } VPRecipeTy
An enumeration for keeping track of the concrete subclass of VPRecipeBase that is actually instantiat...
Definition: VPlan.h:521
IRBuilder & Builder
Hold a reference to the IRBuilder used to generate output IR code.
Definition: VPlan.h:294
VPRecipeBase & back()
Definition: VPlan.h:633
This is a helper struct for maintaining vectorization state.
Definition: VPlan.h:94
bool hasAnyScalarValue(Value *Key) const
Definition: VPlan.h:132
VPBlockBase * getSingleSuccessor() const
Definition: VPlan.h:403
void addVPValue(Value *V)
Definition: VPlan.h:770
print alias Alias Set Printer
VPRecipeBase & front()
Definition: VPlan.h:631
VPBlockBase * setOneSuccessor(VPBlockBase *Successor)
Sets a given VPBlockBase Successor as the single successor and.
Definition: VPlan.h:459
VPRecipeBase is a base class modeling a sequence of one or more output IR instructions.
Definition: VPlan.h:496
unsigned getVPRecipeID() const
Definition: VPlan.h:529
VPValue * getVPValue(Value *V)
Definition: VPlan.h:776
bool hasAnyVectorValue(Value *Key) const
Definition: VPlan.h:117
reverse_iterator rend()
Definition: VPlan.h:625
static bool classof(const VPRecipeBase *R)
Method to support type inquiry through isa, cast, and dyn_cast.
Definition: VPlan.h:571
unsigned VF
The chosen Vectorization and Unroll Factors of the loop being vectorized.
Definition: VPlan.h:227
const VPBlocksTy & getHierarchicalSuccessors()
Definition: VPlan.h:430
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:81
virtual ~VPCallback()
Definition: VPlan.h:213
VPBlockBase * getEntry()
Definition: VPlan.h:708
size_t size() const
Definition: VPlan.h:628
bool hasVectorValue(Value *Key, unsigned Part) const
Definition: VPlan.h:122
VPBlocksTy & getSuccessors()
Definition: VPlan.h:396
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: APFloat.h:42
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition: IRBuilder.h:668
VectorizerValueMap & ValueMap
Hold a reference to the Value state information used when generating the Values of the output IR...
Definition: VPlan.h:298
~VPRegionBlock() override
Definition: VPlan.h:697
void setName(const Twine &newName)
Definition: VPlan.h:372
DominatorTree * DT
Hold a pointer to Dominator Tree to register new basic blocks in the loop.
Definition: VPlan.h:291
void insert(VPRecipeBase *Recipe, iterator InsertPt)
Definition: VPlan.h:645
const std::string & getName() const
Definition: VPlan.h:766
VPBlocksTy & getPredecessors()
Definition: VPlan.h:399
static RecipeListTy VPBasicBlock::* getSublistAccess(VPRecipeBase *)
Returns a pointer to a member of the recipe list.
Definition: VPlan.h:636
static ChildIteratorType child_end(NodeRef N)
Definition: VPlan.h:912
void setName(const Twine &newName)
Definition: VPlan.h:768
static ChildIteratorType child_begin(NodeRef N)
Definition: VPlan.h:887
Key
PAL metadata keys.
VPBlockBase * getSinglePredecessor() const
Definition: VPlan.h:409
VPlanPrinter prints a given VPlan to a given output stream.
Definition: VPlan.h:792
static ChildIteratorType child_end(NodeRef N)
Definition: VPlan.h:891
bool isReplicator() const
An indicator whether this region is to generate multiple replicated instances of output IR correspond...
Definition: VPlan.h:715
This class is used to enable the VPlan to invoke a method of ILV.
Definition: VPlan.h:212
const VPBlockBase * getExit() const
Definition: VPlan.h:710
VPTransformState holds information passed down when "executing" a VPlan, needed for generating the ou...
Definition: VPlan.h:219
VPBlockBase * getSingleHierarchicalSuccessor()
Definition: VPlan.h:436
VPBlockBase * getSingleHierarchicalPredecessor()
Definition: VPlan.h:452
unsigned getVPBlockID() const
Definition: VPlan.h:377
static bool classof(const VPValue *V)
Method to support type inquiry through isa, cast, and dyn_cast.
Definition: VPlan.h:566
unsigned getVPValueID() const
Definition: VPlanValue.h:62
VPInstruction(unsigned Opcode, std::initializer_list< VPValue *> Operands)
Definition: VPlan.h:561
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree...
Definition: Dominators.h:140
iterator begin()
Recipe iterator methods.
Definition: VPlan.h:618
void resetScalarValue(Value *Key, const VPIteration &Instance, Value *Scalar)
Reset the scalar value associated with Key for Part and Lane.
Definition: VPlan.h:202
SmallVectorImpl< VPBlockBase * >::iterator ChildIteratorType
Definition: VPlan.h:868
void appendRecipe(VPRecipeBase *Recipe)
Augment the existing recipes of a VPBasicBlock with an additional Recipe as the last recipe...
Definition: VPlan.h:654
const VPBlocksTy & getHierarchicalPredecessors()
Definition: VPlan.h:446
#define P(N)
This class augments VPValue with operands which provide the inverse def-use edges from VPValue&#39;s user...
Definition: VPlanValue.h:93
An ilist node that can access its parent list.
Definition: ilist_node.h:257
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
static ChildIteratorType child_begin(NodeRef N)
Definition: VPlan.h:908
reverse_iterator rbegin()
Definition: VPlan.h:623
LLVM Basic Block Representation.
Definition: BasicBlock.h:59
SmallVector< Value *, 2 > PerPartValuesTy
A type for vectorized values in the new loop.
Definition: VPlan.h:239
bool hasScalarValue(Value *Key, const VPIteration &Instance) const
Definition: VPlan.h:137
LLVM_ATTRIBUTE_ALWAYS_INLINE iterator begin()
Definition: SmallVector.h:116
SmallDenseMap< VPBasicBlock *, BasicBlock * > VPBB2IRBB
A mapping of each VPBasicBlock to the corresponding BasicBlock.
Definition: VPlan.h:282
static NodeRef getEntryNode(NodeRef N)
Definition: VPlan.h:870
static ChildIteratorType child_end(NodeRef N)
Definition: VPlan.h:876
Hold state information used when constructing the CFG of the output IR, traversing the VPBasicBlocks ...
Definition: VPlan.h:268
~VPlan()
Definition: VPlan.h:747
VPCallback & Callback
Definition: VPlan.h:307
VPBlockBase * setEntry(VPBlockBase *Block)
Definition: VPlan.h:760
static void print(raw_ostream &Out, object::Archive::Kind Kind, T Val)
bool hasVF(unsigned VF)
Definition: VPlan.h:764
VPValue2ValueTy VPValue2Value
Hold a reference to a mapping between VPValues in VPlan and original Values they correspond to...
Definition: VPlan.h:302
std::pair< NoneType, bool > insert(const T &V)
insert - Insert an element into the set if it isn&#39;t already there.
Definition: SmallSet.h:81
static bool classof(const VPBlockBase *V)
Method to support type inquiry through isa, cast, and dyn_cast.
Definition: VPlan.h:641
SmallVectorImpl< VPBlockBase * >::iterator ChildIteratorType
Definition: VPlan.h:902
iterator erase(const_iterator CI)
Definition: SmallVector.h:449
auto find(R &&Range, const T &Val) -> decltype(adl_begin(Range))
Provide wrappers to std::find which take ranges instead of having to pass begin/end explicitly...
Definition: STLExtras.h:834
void setScalarValue(Value *Key, const VPIteration &Instance, Value *Scalar)
Set a scalar value associated with Key and Instance.
Definition: VPlan.h:176
static void deleteCFG(VPBlockBase *Entry)
Delete all blocks reachable from a given VPBlockBase, inclusive.
Definition: VPlan.cpp:103
VPBasicBlock serves as the leaf of the Hierarchical Control-Flow Graph.
Definition: VPlan.h:592
VPBlockBase(const unsigned char SC, const std::string &N)
Definition: VPlan.h:356
const_iterator begin() const
Definition: VPlan.h:619
unsigned Lane
in [0..VF)
Definition: VPlan.h:74
void setTwoSuccessors(VPBlockBase *IfTrue, VPBlockBase *IfFalse)
Sets two given VPBlockBases IfTrue and IfFalse to be the two successors.
Definition: VPlan.h:470
const VPRecipeBase & back() const
Definition: VPlan.h:632
Iterator for intrusive lists based on ilist_node.
See the file comment.
Definition: ValueMap.h:86
const VPRecipeBase & front() const
Definition: VPlan.h:630
VPBlockBase is the building block of the Hierarchical Control-Flow Graph.
Definition: VPlan.h:312
This is a &#39;vector&#39; (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:864
static NodeRef getEntryNode(NodeRef N)
Definition: VPlan.h:885
CHAIN = SC CHAIN, Imm128 - System call.
static bool classof(const VPBlockBase *V)
Method to support type inquiry through isa, cast, and dyn_cast.
Definition: VPlan.h:703
const VPBlocksTy & getSuccessors() const
Definition: VPlan.h:395
const VPBlockBase * getEntry() const
Definition: VPlan.h:707
void addVF(unsigned VF)
Definition: VPlan.h:762
static Inverse< VPBlockBase * > getEntryNode(Inverse< VPBlockBase *> B)
Definition: VPlan.h:904
void resetVectorValue(Value *Key, unsigned Part, Value *Vector)
Reset the vector value associated with Key for the given Part.
Definition: VPlan.h:193
VPBlockBase * getExit()
Definition: VPlan.h:711
typename SuperClass::iterator iterator
Definition: SmallVector.h:328
InnerLoopVectorizer * ILV
Hold a pointer to InnerLoopVectorizer to reuse its IR generation methods.
Definition: VPlan.h:305
const VPBasicBlock * getParent() const
Definition: VPlan.h:533
bool empty() const
Definition: VPlan.h:629
LLVM_ATTRIBUTE_ALWAYS_INLINE iterator end()
Definition: SmallVector.h:120
iterator insert(iterator where, pointer New)
Definition: ilist.h:241
const VPBlocksTy & getPredecessors() const
Definition: VPlan.h:398
Flatten the CFG
Value * getVectorValue(Value *Key, unsigned Part)
Retrieve the existing vector value that corresponds to Key and Part.
Definition: VPlan.h:151
void clear()
Definition: ilist.h:322
VPBlockBase * getEntry()
Definition: VPlan.h:757
LLVM_NODISCARD bool empty() const
Definition: SmallVector.h:61
VPTransformState(unsigned VF, unsigned UF, LoopInfo *LI, DominatorTree *DT, IRBuilder<> &Builder, VectorizerValueMap &ValueMap, InnerLoopVectorizer *ILV, VPCallback &Callback)
Definition: VPlan.h:220
#define I(x, y, z)
Definition: MD5.cpp:58
#define N
VPRecipeBase(const unsigned char SC)
Definition: VPlan.h:523
const VPBlockBase * getEntry() const
Definition: VPlan.h:758
VPlan(VPBlockBase *Entry=nullptr)
Definition: VPlan.h:745
unsigned getOpcode() const
Definition: VPlan.h:575
const_iterator end() const
Definition: VPlan.h:621
raw_ostream & operator<<(raw_ostream &OS, const APInt &I)
Definition: APInt.h:2018
size_type count(const_arg_type_t< KeyT > Val) const
Return 1 if the specified key is in the map, 0 otherwise.
Definition: DenseMap.h:141
std::string str() const
Return the twine contents as a std::string.
Definition: Twine.cpp:17
void setVectorValue(Value *Key, unsigned Part, Value *Vector)
Set a vector value associated with Key and Part.
Definition: VPlan.h:165
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
void disconnectSuccessor(VPBlockBase *Successor)
Definition: VPlan.h:480
iterator end()
Definition: VPlan.h:620
VPBasicBlock * getParent()
Definition: VPlan.h:532
aarch64 promote const
This file contains the declarations of the entities induced by Vectorization Plans, e.g.
LLVM Value Representation.
Definition: Value.h:73
static ChildIteratorType child_begin(NodeRef N)
Definition: VPlan.h:872
This class implements an extremely fast bulk output stream that can only output to a stream...
Definition: raw_ostream.h:44
In what follows, the term "input IR" refers to code that is fed into the vectorizer whereas the term ...
Definition: VPlan.h:69
unsigned Part
in [0..UF)
Definition: VPlan.h:71
This is a concrete Recipe that models a single VPlan-level instruction.
Definition: VPlan.h:547
const_reverse_iterator rend() const
Definition: VPlan.h:626
VPBasicBlock(const Twine &Name="", VPRecipeBase *Recipe=nullptr)
Definition: VPlan.h:601
void setParent(VPRegionBlock *P)
Definition: VPlan.h:381
size_type count(const T &V) const
count - Return 1 if the element is in the set, 0 otherwise.
Definition: SmallSet.h:65