LLVM 19.0.0git
VPlan.h
Go to the documentation of this file.
1//===- VPlan.h - Represent A Vectorizer Plan --------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// This file contains the declarations of the Vectorization Plan base classes:
11/// 1. VPBasicBlock and VPRegionBlock that inherit from a common pure virtual
12/// VPBlockBase, together implementing a Hierarchical CFG;
13/// 2. Pure virtual VPRecipeBase serving as the base class for recipes contained
14/// within VPBasicBlocks;
15/// 3. Pure virtual VPSingleDefRecipe serving as a base class for recipes that
16/// also inherit from VPValue.
17/// 4. VPInstruction, a concrete Recipe and VPUser modeling a single planned
18/// instruction;
19/// 5. The VPlan class holding a candidate for vectorization;
20/// 6. The VPlanPrinter class providing a way to print a plan in dot format;
21/// These are documented in docs/VectorizationPlan.rst.
22//
23//===----------------------------------------------------------------------===//
24
25#ifndef LLVM_TRANSFORMS_VECTORIZE_VPLAN_H
26#define LLVM_TRANSFORMS_VECTORIZE_VPLAN_H
27
28#include "VPlanAnalysis.h"
29#include "VPlanValue.h"
30#include "llvm/ADT/DenseMap.h"
31#include "llvm/ADT/MapVector.h"
35#include "llvm/ADT/Twine.h"
36#include "llvm/ADT/ilist.h"
37#include "llvm/ADT/ilist_node.h"
41#include "llvm/IR/DebugLoc.h"
42#include "llvm/IR/FMF.h"
43#include "llvm/IR/Operator.h"
44#include <algorithm>
45#include <cassert>
46#include <cstddef>
47#include <string>
48
49namespace llvm {
50
51class BasicBlock;
52class DominatorTree;
53class InnerLoopVectorizer;
54class IRBuilderBase;
55class LoopInfo;
56class raw_ostream;
57class RecurrenceDescriptor;
58class SCEV;
59class Type;
60class VPBasicBlock;
61class VPRegionBlock;
62class VPlan;
63class VPReplicateRecipe;
64class VPlanSlp;
65class Value;
66class LoopVersioning;
67
68namespace Intrinsic {
69typedef unsigned ID;
70}
71
72/// Returns a calculation for the total number of elements for a given \p VF.
73/// For fixed width vectors this value is a constant, whereas for scalable
74/// vectors it is an expression determined at runtime.
75Value *getRuntimeVF(IRBuilderBase &B, Type *Ty, ElementCount VF);
76
77/// Return a value for Step multiplied by VF.
78Value *createStepForVF(IRBuilderBase &B, Type *Ty, ElementCount VF,
79 int64_t Step);
80
81const SCEV *createTripCountSCEV(Type *IdxTy, PredicatedScalarEvolution &PSE,
82 Loop *CurLoop = nullptr);
83
84/// A range of powers-of-2 vectorization factors with fixed start and
85/// adjustable end. The range includes start and excludes end, e.g.,:
86/// [1, 16) = {1, 2, 4, 8}
87struct VFRange {
88 // A power of 2.
90
91 // A power of 2. If End <= Start range is empty.
93
94 bool isEmpty() const {
96 }
97
99 : Start(Start), End(End) {
101 "Both Start and End should have the same scalable flag");
103 "Expected Start to be a power of 2");
105 "Expected End to be a power of 2");
106 }
107
108 /// Iterator to iterate over vectorization factors in a VFRange.
110 : public iterator_facade_base<iterator, std::forward_iterator_tag,
111 ElementCount> {
112 ElementCount VF;
113
114 public:
115 iterator(ElementCount VF) : VF(VF) {}
116
117 bool operator==(const iterator &Other) const { return VF == Other.VF; }
118
119 ElementCount operator*() const { return VF; }
120
122 VF *= 2;
123 return *this;
124 }
125 };
126
130 return iterator(End);
131 }
132};
133
134using VPlanPtr = std::unique_ptr<VPlan>;
135
136/// In what follows, the term "input IR" refers to code that is fed into the
137/// vectorizer whereas the term "output IR" refers to code that is generated by
138/// the vectorizer.
139
140/// VPLane provides a way to access lanes in both fixed width and scalable
141/// vectors, where for the latter the lane index sometimes needs calculating
142/// as a runtime expression.
143class VPLane {
144public:
145 /// Kind describes how to interpret Lane.
146 enum class Kind : uint8_t {
147 /// For First, Lane is the index into the first N elements of a
148 /// fixed-vector <N x <ElTy>> or a scalable vector <vscale x N x <ElTy>>.
149 First,
150 /// For ScalableLast, Lane is the offset from the start of the last
151 /// N-element subvector in a scalable vector <vscale x N x <ElTy>>. For
152 /// example, a Lane of 0 corresponds to lane `(vscale - 1) * N`, a Lane of
153 /// 1 corresponds to `((vscale - 1) * N) + 1`, etc.
155 };
156
157private:
158 /// in [0..VF)
159 unsigned Lane;
160
161 /// Indicates how the Lane should be interpreted, as described above.
162 Kind LaneKind;
163
164public:
165 VPLane(unsigned Lane, Kind LaneKind) : Lane(Lane), LaneKind(LaneKind) {}
166
168
170 unsigned LaneOffset = VF.getKnownMinValue() - 1;
171 Kind LaneKind;
172 if (VF.isScalable())
173 // In this case 'LaneOffset' refers to the offset from the start of the
174 // last subvector with VF.getKnownMinValue() elements.
176 else
177 LaneKind = VPLane::Kind::First;
178 return VPLane(LaneOffset, LaneKind);
179 }
180
181 /// Returns a compile-time known value for the lane index and asserts if the
182 /// lane can only be calculated at runtime.
183 unsigned getKnownLane() const {
184 assert(LaneKind == Kind::First);
185 return Lane;
186 }
187
188 /// Returns an expression describing the lane index that can be used at
189 /// runtime.
190 Value *getAsRuntimeExpr(IRBuilderBase &Builder, const ElementCount &VF) const;
191
192 /// Returns the Kind of lane offset.
193 Kind getKind() const { return LaneKind; }
194
195 /// Returns true if this is the first lane of the whole vector.
196 bool isFirstLane() const { return Lane == 0 && LaneKind == Kind::First; }
197
198 /// Maps the lane to a cache index based on \p VF.
199 unsigned mapToCacheIndex(const ElementCount &VF) const {
200 switch (LaneKind) {
202 assert(VF.isScalable() && Lane < VF.getKnownMinValue());
203 return VF.getKnownMinValue() + Lane;
204 default:
205 assert(Lane < VF.getKnownMinValue());
206 return Lane;
207 }
208 }
209
210 /// Returns the maxmimum number of lanes that we are able to consider
211 /// caching for \p VF.
212 static unsigned getNumCachedLanes(const ElementCount &VF) {
213 return VF.getKnownMinValue() * (VF.isScalable() ? 2 : 1);
214 }
215};
216
217/// VPIteration represents a single point in the iteration space of the output
218/// (vectorized and/or unrolled) IR loop.
220 /// in [0..UF)
221 unsigned Part;
222
224
225 VPIteration(unsigned Part, unsigned Lane,
227 : Part(Part), Lane(Lane, Kind) {}
228
229 VPIteration(unsigned Part, const VPLane &Lane) : Part(Part), Lane(Lane) {}
230
231 bool isFirstIteration() const { return Part == 0 && Lane.isFirstLane(); }
232};
233
234/// VPTransformState holds information passed down when "executing" a VPlan,
235/// needed for generating the output IR.
240
241 /// The chosen Vectorization and Unroll Factors of the loop being vectorized.
243 unsigned UF;
244
245 /// Hold the indices to generate specific scalar instructions. Null indicates
246 /// that all instances are to be generated, using either scalar or vector
247 /// instructions.
248 std::optional<VPIteration> Instance;
249
250 struct DataState {
251 /// A type for vectorized values in the new loop. Each value from the
252 /// original loop, when vectorized, is represented by UF vector values in
253 /// the new unrolled loop, where UF is the unroll factor.
255
257
261
262 /// Get the generated vector Value for a given VPValue \p Def and a given \p
263 /// Part if \p IsScalar is false, otherwise return the generated scalar
264 /// for \p Part. \See set.
265 Value *get(VPValue *Def, unsigned Part, bool IsScalar = false);
266
267 /// Get the generated Value for a given VPValue and given Part and Lane.
268 Value *get(VPValue *Def, const VPIteration &Instance);
269
270 bool hasVectorValue(VPValue *Def, unsigned Part) {
271 auto I = Data.PerPartOutput.find(Def);
272 return I != Data.PerPartOutput.end() && Part < I->second.size() &&
273 I->second[Part];
274 }
275
277 auto I = Data.PerPartScalars.find(Def);
278 if (I == Data.PerPartScalars.end())
279 return false;
280 unsigned CacheIdx = Instance.Lane.mapToCacheIndex(VF);
281 return Instance.Part < I->second.size() &&
282 CacheIdx < I->second[Instance.Part].size() &&
283 I->second[Instance.Part][CacheIdx];
284 }
285
286 /// Set the generated vector Value for a given VPValue and a given Part, if \p
287 /// IsScalar is false. If \p IsScalar is true, set the scalar in (Part, 0).
288 void set(VPValue *Def, Value *V, unsigned Part, bool IsScalar = false) {
289 if (IsScalar) {
290 set(Def, V, VPIteration(Part, 0));
291 return;
292 }
293 assert((VF.isScalar() || V->getType()->isVectorTy()) &&
294 "scalar values must be stored as (Part, 0)");
295 if (!Data.PerPartOutput.count(Def)) {
297 Data.PerPartOutput[Def] = Entry;
298 }
299 Data.PerPartOutput[Def][Part] = V;
300 }
301
302 /// Reset an existing vector value for \p Def and a given \p Part.
303 void reset(VPValue *Def, Value *V, unsigned Part) {
304 auto Iter = Data.PerPartOutput.find(Def);
305 assert(Iter != Data.PerPartOutput.end() &&
306 "need to overwrite existing value");
307 Iter->second[Part] = V;
308 }
309
310 /// Set the generated scalar \p V for \p Def and the given \p Instance.
311 void set(VPValue *Def, Value *V, const VPIteration &Instance) {
312 auto Iter = Data.PerPartScalars.insert({Def, {}});
313 auto &PerPartVec = Iter.first->second;
314 if (PerPartVec.size() <= Instance.Part)
315 PerPartVec.resize(Instance.Part + 1);
316 auto &Scalars = PerPartVec[Instance.Part];
317 unsigned CacheIdx = Instance.Lane.mapToCacheIndex(VF);
318 if (Scalars.size() <= CacheIdx)
319 Scalars.resize(CacheIdx + 1);
320 assert(!Scalars[CacheIdx] && "should overwrite existing value");
321 Scalars[CacheIdx] = V;
322 }
323
324 /// Reset an existing scalar value for \p Def and a given \p Instance.
325 void reset(VPValue *Def, Value *V, const VPIteration &Instance) {
326 auto Iter = Data.PerPartScalars.find(Def);
327 assert(Iter != Data.PerPartScalars.end() &&
328 "need to overwrite existing value");
329 assert(Instance.Part < Iter->second.size() &&
330 "need to overwrite existing value");
331 unsigned CacheIdx = Instance.Lane.mapToCacheIndex(VF);
332 assert(CacheIdx < Iter->second[Instance.Part].size() &&
333 "need to overwrite existing value");
334 Iter->second[Instance.Part][CacheIdx] = V;
335 }
336
337 /// Add additional metadata to \p To that was not present on \p Orig.
338 ///
339 /// Currently this is used to add the noalias annotations based on the
340 /// inserted memchecks. Use this for instructions that are *cloned* into the
341 /// vector loop.
342 void addNewMetadata(Instruction *To, const Instruction *Orig);
343
344 /// Add metadata from one instruction to another.
345 ///
346 /// This includes both the original MDs from \p From and additional ones (\see
347 /// addNewMetadata). Use this for *newly created* instructions in the vector
348 /// loop.
350
351 /// Similar to the previous function but it adds the metadata to a
352 /// vector of instructions.
354
355 /// Set the debug location in the builder using the debug location \p DL.
357
358 /// Construct the vector value of a scalarized value \p V one lane at a time.
360
361 /// Hold state information used when constructing the CFG of the output IR,
362 /// traversing the VPBasicBlocks and generating corresponding IR BasicBlocks.
363 struct CFGState {
364 /// The previous VPBasicBlock visited. Initially set to null.
366
367 /// The previous IR BasicBlock created or used. Initially set to the new
368 /// header BasicBlock.
369 BasicBlock *PrevBB = nullptr;
370
371 /// The last IR BasicBlock in the output IR. Set to the exit block of the
372 /// vector loop.
373 BasicBlock *ExitBB = nullptr;
374
375 /// A mapping of each VPBasicBlock to the corresponding BasicBlock. In case
376 /// of replication, maps the BasicBlock of the last replica created.
378
379 CFGState() = default;
380
381 /// Returns the BasicBlock* mapped to the pre-header of the loop region
382 /// containing \p R.
385
386 /// Hold a pointer to LoopInfo to register new basic blocks in the loop.
388
389 /// Hold a pointer to Dominator Tree to register new basic blocks in the loop.
391
392 /// Hold a reference to the IRBuilder used to generate output IR code.
394
395 /// Hold a pointer to InnerLoopVectorizer to reuse its IR generation methods.
397
398 /// Pointer to the VPlan code is generated for.
400
401 /// The loop object for the current parent region, or nullptr.
403
404 /// LoopVersioning. It's only set up (non-null) if memchecks were
405 /// used.
406 ///
407 /// This is currently only used to add no-alias metadata based on the
408 /// memchecks. The actually versioning is performed manually.
410
411 /// Map SCEVs to their expanded values. Populated when executing
412 /// VPExpandSCEVRecipes.
414
415 /// VPlan-based type analysis.
417};
418
419/// VPBlockBase is the building block of the Hierarchical Control-Flow Graph.
420/// A VPBlockBase can be either a VPBasicBlock or a VPRegionBlock.
422 friend class VPBlockUtils;
423
424 const unsigned char SubclassID; ///< Subclass identifier (for isa/dyn_cast).
425
426 /// An optional name for the block.
427 std::string Name;
428
429 /// The immediate VPRegionBlock which this VPBlockBase belongs to, or null if
430 /// it is a topmost VPBlockBase.
431 VPRegionBlock *Parent = nullptr;
432
433 /// List of predecessor blocks.
435
436 /// List of successor blocks.
438
439 /// VPlan containing the block. Can only be set on the entry block of the
440 /// plan.
441 VPlan *Plan = nullptr;
442
443 /// Add \p Successor as the last successor to this block.
444 void appendSuccessor(VPBlockBase *Successor) {
445 assert(Successor && "Cannot add nullptr successor!");
446 Successors.push_back(Successor);
447 }
448
449 /// Add \p Predecessor as the last predecessor to this block.
450 void appendPredecessor(VPBlockBase *Predecessor) {
451 assert(Predecessor && "Cannot add nullptr predecessor!");
452 Predecessors.push_back(Predecessor);
453 }
454
455 /// Remove \p Predecessor from the predecessors of this block.
456 void removePredecessor(VPBlockBase *Predecessor) {
457 auto Pos = find(Predecessors, Predecessor);
458 assert(Pos && "Predecessor does not exist");
459 Predecessors.erase(Pos);
460 }
461
462 /// Remove \p Successor from the successors of this block.
463 void removeSuccessor(VPBlockBase *Successor) {
464 auto Pos = find(Successors, Successor);
465 assert(Pos && "Successor does not exist");
466 Successors.erase(Pos);
467 }
468
469protected:
470 VPBlockBase(const unsigned char SC, const std::string &N)
471 : SubclassID(SC), Name(N) {}
472
473public:
474 /// An enumeration for keeping track of the concrete subclass of VPBlockBase
475 /// that are actually instantiated. Values of this enumeration are kept in the
476 /// SubclassID field of the VPBlockBase objects. They are used for concrete
477 /// type identification.
478 using VPBlockTy = enum { VPBasicBlockSC, VPRegionBlockSC };
479
481
482 virtual ~VPBlockBase() = default;
483
484 const std::string &getName() const { return Name; }
485
486 void setName(const Twine &newName) { Name = newName.str(); }
487
488 /// \return an ID for the concrete type of this object.
489 /// This is used to implement the classof checks. This should not be used
490 /// for any other purpose, as the values may change as LLVM evolves.
491 unsigned getVPBlockID() const { return SubclassID; }
492
493 VPRegionBlock *getParent() { return Parent; }
494 const VPRegionBlock *getParent() const { return Parent; }
495
496 /// \return A pointer to the plan containing the current block.
497 VPlan *getPlan();
498 const VPlan *getPlan() const;
499
500 /// Sets the pointer of the plan containing the block. The block must be the
501 /// entry block into the VPlan.
502 void setPlan(VPlan *ParentPlan);
503
504 void setParent(VPRegionBlock *P) { Parent = P; }
505
506 /// \return the VPBasicBlock that is the entry of this VPBlockBase,
507 /// recursively, if the latter is a VPRegionBlock. Otherwise, if this
508 /// VPBlockBase is a VPBasicBlock, it is returned.
509 const VPBasicBlock *getEntryBasicBlock() const;
511
512 /// \return the VPBasicBlock that is the exiting this VPBlockBase,
513 /// recursively, if the latter is a VPRegionBlock. Otherwise, if this
514 /// VPBlockBase is a VPBasicBlock, it is returned.
515 const VPBasicBlock *getExitingBasicBlock() const;
517
518 const VPBlocksTy &getSuccessors() const { return Successors; }
519 VPBlocksTy &getSuccessors() { return Successors; }
520
522
523 const VPBlocksTy &getPredecessors() const { return Predecessors; }
524 VPBlocksTy &getPredecessors() { return Predecessors; }
525
526 /// \return the successor of this VPBlockBase if it has a single successor.
527 /// Otherwise return a null pointer.
529 return (Successors.size() == 1 ? *Successors.begin() : nullptr);
530 }
531
532 /// \return the predecessor of this VPBlockBase if it has a single
533 /// predecessor. Otherwise return a null pointer.
535 return (Predecessors.size() == 1 ? *Predecessors.begin() : nullptr);
536 }
537
538 size_t getNumSuccessors() const { return Successors.size(); }
539 size_t getNumPredecessors() const { return Predecessors.size(); }
540
541 /// An Enclosing Block of a block B is any block containing B, including B
542 /// itself. \return the closest enclosing block starting from "this", which
543 /// has successors. \return the root enclosing block if all enclosing blocks
544 /// have no successors.
546
547 /// \return the closest enclosing block starting from "this", which has
548 /// predecessors. \return the root enclosing block if all enclosing blocks
549 /// have no predecessors.
551
552 /// \return the successors either attached directly to this VPBlockBase or, if
553 /// this VPBlockBase is the exit block of a VPRegionBlock and has no
554 /// successors of its own, search recursively for the first enclosing
555 /// VPRegionBlock that has successors and return them. If no such
556 /// VPRegionBlock exists, return the (empty) successors of the topmost
557 /// VPBlockBase reached.
560 }
561
562 /// \return the hierarchical successor of this VPBlockBase if it has a single
563 /// hierarchical successor. Otherwise return a null pointer.
566 }
567
568 /// \return the predecessors either attached directly to this VPBlockBase or,
569 /// if this VPBlockBase is the entry block of a VPRegionBlock and has no
570 /// predecessors of its own, search recursively for the first enclosing
571 /// VPRegionBlock that has predecessors and return them. If no such
572 /// VPRegionBlock exists, return the (empty) predecessors of the topmost
573 /// VPBlockBase reached.
576 }
577
578 /// \return the hierarchical predecessor of this VPBlockBase if it has a
579 /// single hierarchical predecessor. Otherwise return a null pointer.
582 }
583
584 /// Set a given VPBlockBase \p Successor as the single successor of this
585 /// VPBlockBase. This VPBlockBase is not added as predecessor of \p Successor.
586 /// This VPBlockBase must have no successors.
588 assert(Successors.empty() && "Setting one successor when others exist.");
589 assert(Successor->getParent() == getParent() &&
590 "connected blocks must have the same parent");
591 appendSuccessor(Successor);
592 }
593
594 /// Set two given VPBlockBases \p IfTrue and \p IfFalse to be the two
595 /// successors of this VPBlockBase. This VPBlockBase is not added as
596 /// predecessor of \p IfTrue or \p IfFalse. This VPBlockBase must have no
597 /// successors.
598 void setTwoSuccessors(VPBlockBase *IfTrue, VPBlockBase *IfFalse) {
599 assert(Successors.empty() && "Setting two successors when others exist.");
600 appendSuccessor(IfTrue);
601 appendSuccessor(IfFalse);
602 }
603
604 /// Set each VPBasicBlock in \p NewPreds as predecessor of this VPBlockBase.
605 /// This VPBlockBase must have no predecessors. This VPBlockBase is not added
606 /// as successor of any VPBasicBlock in \p NewPreds.
608 assert(Predecessors.empty() && "Block predecessors already set.");
609 for (auto *Pred : NewPreds)
610 appendPredecessor(Pred);
611 }
612
613 /// Remove all the predecessor of this block.
614 void clearPredecessors() { Predecessors.clear(); }
615
616 /// Remove all the successors of this block.
617 void clearSuccessors() { Successors.clear(); }
618
619 /// The method which generates the output IR that correspond to this
620 /// VPBlockBase, thereby "executing" the VPlan.
621 virtual void execute(VPTransformState *State) = 0;
622
623 /// Delete all blocks reachable from a given VPBlockBase, inclusive.
624 static void deleteCFG(VPBlockBase *Entry);
625
626 /// Return true if it is legal to hoist instructions into this block.
628 // There are currently no constraints that prevent an instruction to be
629 // hoisted into a VPBlockBase.
630 return true;
631 }
632
633 /// Replace all operands of VPUsers in the block with \p NewValue and also
634 /// replaces all uses of VPValues defined in the block with NewValue.
635 virtual void dropAllReferences(VPValue *NewValue) = 0;
636
637#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
638 void printAsOperand(raw_ostream &OS, bool PrintType) const {
639 OS << getName();
640 }
641
642 /// Print plain-text dump of this VPBlockBase to \p O, prefixing all lines
643 /// with \p Indent. \p SlotTracker is used to print unnamed VPValue's using
644 /// consequtive numbers.
645 ///
646 /// Note that the numbering is applied to the whole VPlan, so printing
647 /// individual blocks is consistent with the whole VPlan printing.
648 virtual void print(raw_ostream &O, const Twine &Indent,
649 VPSlotTracker &SlotTracker) const = 0;
650
651 /// Print plain-text dump of this VPlan to \p O.
652 void print(raw_ostream &O) const {
654 print(O, "", SlotTracker);
655 }
656
657 /// Print the successors of this block to \p O, prefixing all lines with \p
658 /// Indent.
659 void printSuccessors(raw_ostream &O, const Twine &Indent) const;
660
661 /// Dump this VPBlockBase to dbgs().
662 LLVM_DUMP_METHOD void dump() const { print(dbgs()); }
663#endif
664
665 /// Clone the current block and it's recipes without updating the operands of
666 /// the cloned recipes, including all blocks in the single-entry single-exit
667 /// region for VPRegionBlocks.
668 virtual VPBlockBase *clone() = 0;
669};
670
671/// A value that is used outside the VPlan. The operand of the user needs to be
672/// added to the associated LCSSA phi node.
673class VPLiveOut : public VPUser {
674 PHINode *Phi;
675
676public:
678 : VPUser({Op}, VPUser::VPUserID::LiveOut), Phi(Phi) {}
679
680 static inline bool classof(const VPUser *U) {
681 return U->getVPUserID() == VPUser::VPUserID::LiveOut;
682 }
683
684 /// Fixup the wrapped LCSSA phi node in the unique exit block. This simply
685 /// means we need to add the appropriate incoming value from the middle
686 /// block as exiting edges from the scalar epilogue loop (if present) are
687 /// already in place, and we exit the vector loop exclusively to the middle
688 /// block.
689 void fixPhi(VPlan &Plan, VPTransformState &State);
690
691 /// Returns true if the VPLiveOut uses scalars of operand \p Op.
692 bool usesScalars(const VPValue *Op) const override {
694 "Op must be an operand of the recipe");
695 return true;
696 }
697
698 PHINode *getPhi() const { return Phi; }
699
700#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
701 /// Print the VPLiveOut to \p O.
703#endif
704};
705
706/// VPRecipeBase is a base class modeling a sequence of one or more output IR
707/// instructions. VPRecipeBase owns the VPValues it defines through VPDef
708/// and is responsible for deleting its defined values. Single-value
709/// recipes must inherit from VPSingleDef instead of inheriting from both
710/// VPRecipeBase and VPValue separately.
711class VPRecipeBase : public ilist_node_with_parent<VPRecipeBase, VPBasicBlock>,
712 public VPDef,
713 public VPUser {
714 friend VPBasicBlock;
715 friend class VPBlockUtils;
716
717 /// Each VPRecipe belongs to a single VPBasicBlock.
718 VPBasicBlock *Parent = nullptr;
719
720 /// The debug location for the recipe.
721 DebugLoc DL;
722
723public:
725 DebugLoc DL = {})
727
728 template <typename IterT>
730 DebugLoc DL = {})
732 virtual ~VPRecipeBase() = default;
733
734 /// Clone the current recipe.
735 virtual VPRecipeBase *clone() = 0;
736
737 /// \return the VPBasicBlock which this VPRecipe belongs to.
738 VPBasicBlock *getParent() { return Parent; }
739 const VPBasicBlock *getParent() const { return Parent; }
740
741 /// The method which generates the output IR instructions that correspond to
742 /// this VPRecipe, thereby "executing" the VPlan.
743 virtual void execute(VPTransformState &State) = 0;
744
745 /// Insert an unlinked recipe into a basic block immediately before
746 /// the specified recipe.
747 void insertBefore(VPRecipeBase *InsertPos);
748 /// Insert an unlinked recipe into \p BB immediately before the insertion
749 /// point \p IP;
751
752 /// Insert an unlinked Recipe into a basic block immediately after
753 /// the specified Recipe.
754 void insertAfter(VPRecipeBase *InsertPos);
755
756 /// Unlink this recipe from its current VPBasicBlock and insert it into
757 /// the VPBasicBlock that MovePos lives in, right after MovePos.
758 void moveAfter(VPRecipeBase *MovePos);
759
760 /// Unlink this recipe and insert into BB before I.
761 ///
762 /// \pre I is a valid iterator into BB.
764
765 /// This method unlinks 'this' from the containing basic block, but does not
766 /// delete it.
767 void removeFromParent();
768
769 /// This method unlinks 'this' from the containing basic block and deletes it.
770 ///
771 /// \returns an iterator pointing to the element after the erased one
773
774 /// Method to support type inquiry through isa, cast, and dyn_cast.
775 static inline bool classof(const VPDef *D) {
776 // All VPDefs are also VPRecipeBases.
777 return true;
778 }
779
780 static inline bool classof(const VPUser *U) {
781 return U->getVPUserID() == VPUser::VPUserID::Recipe;
782 }
783
784 /// Returns true if the recipe may have side-effects.
785 bool mayHaveSideEffects() const;
786
787 /// Returns true for PHI-like recipes.
788 bool isPhi() const {
789 return getVPDefID() >= VPFirstPHISC && getVPDefID() <= VPLastPHISC;
790 }
791
792 /// Returns true if the recipe may read from memory.
793 bool mayReadFromMemory() const;
794
795 /// Returns true if the recipe may write to memory.
796 bool mayWriteToMemory() const;
797
798 /// Returns true if the recipe may read from or write to memory.
799 bool mayReadOrWriteMemory() const {
801 }
802
803 /// Returns the debug location of the recipe.
804 DebugLoc getDebugLoc() const { return DL; }
805};
806
807// Helper macro to define common classof implementations for recipes.
808#define VP_CLASSOF_IMPL(VPDefID) \
809 static inline bool classof(const VPDef *D) { \
810 return D->getVPDefID() == VPDefID; \
811 } \
812 static inline bool classof(const VPValue *V) { \
813 auto *R = V->getDefiningRecipe(); \
814 return R && R->getVPDefID() == VPDefID; \
815 } \
816 static inline bool classof(const VPUser *U) { \
817 auto *R = dyn_cast<VPRecipeBase>(U); \
818 return R && R->getVPDefID() == VPDefID; \
819 } \
820 static inline bool classof(const VPRecipeBase *R) { \
821 return R->getVPDefID() == VPDefID; \
822 } \
823 static inline bool classof(const VPSingleDefRecipe *R) { \
824 return R->getVPDefID() == VPDefID; \
825 }
826
827/// VPSingleDef is a base class for recipes for modeling a sequence of one or
828/// more output IR that define a single result VPValue.
829/// Note that VPRecipeBase must be inherited from before VPValue.
830class VPSingleDefRecipe : public VPRecipeBase, public VPValue {
831public:
832 template <typename IterT>
833 VPSingleDefRecipe(const unsigned char SC, IterT Operands, DebugLoc DL = {})
834 : VPRecipeBase(SC, Operands, DL), VPValue(this) {}
835
836 VPSingleDefRecipe(const unsigned char SC, ArrayRef<VPValue *> Operands,
837 DebugLoc DL = {})
838 : VPRecipeBase(SC, Operands, DL), VPValue(this) {}
839
840 template <typename IterT>
841 VPSingleDefRecipe(const unsigned char SC, IterT Operands, Value *UV,
842 DebugLoc DL = {})
843 : VPRecipeBase(SC, Operands, DL), VPValue(this, UV) {}
844
845 static inline bool classof(const VPRecipeBase *R) {
846 switch (R->getVPDefID()) {
847 case VPRecipeBase::VPDerivedIVSC:
848 case VPRecipeBase::VPExpandSCEVSC:
849 case VPRecipeBase::VPInstructionSC:
850 case VPRecipeBase::VPReductionSC:
851 case VPRecipeBase::VPReplicateSC:
852 case VPRecipeBase::VPScalarIVStepsSC:
853 case VPRecipeBase::VPVectorPointerSC:
854 case VPRecipeBase::VPWidenCallSC:
855 case VPRecipeBase::VPWidenCanonicalIVSC:
856 case VPRecipeBase::VPWidenCastSC:
857 case VPRecipeBase::VPWidenGEPSC:
858 case VPRecipeBase::VPWidenSC:
859 case VPRecipeBase::VPWidenSelectSC:
860 case VPRecipeBase::VPBlendSC:
861 case VPRecipeBase::VPPredInstPHISC:
862 case VPRecipeBase::VPCanonicalIVPHISC:
863 case VPRecipeBase::VPActiveLaneMaskPHISC:
864 case VPRecipeBase::VPFirstOrderRecurrencePHISC:
865 case VPRecipeBase::VPWidenPHISC:
866 case VPRecipeBase::VPWidenIntOrFpInductionSC:
867 case VPRecipeBase::VPWidenPointerInductionSC:
868 case VPRecipeBase::VPReductionPHISC:
869 case VPRecipeBase::VPScalarCastSC:
870 return true;
871 case VPRecipeBase::VPInterleaveSC:
872 case VPRecipeBase::VPBranchOnMaskSC:
873 case VPRecipeBase::VPWidenMemoryInstructionSC:
874 // TODO: Widened stores don't define a value, but widened loads do. Split
875 // the recipes to be able to make widened loads VPSingleDefRecipes.
876 return false;
877 }
878 llvm_unreachable("Unhandled VPDefID");
879 }
880
881 static inline bool classof(const VPUser *U) {
882 auto *R = dyn_cast<VPRecipeBase>(U);
883 return R && classof(R);
884 }
885
886 /// Returns the underlying instruction.
888 return cast<Instruction>(getUnderlyingValue());
889 }
891 return cast<Instruction>(getUnderlyingValue());
892 }
893};
894
895/// Class to record LLVM IR flag for a recipe along with it.
897 enum class OperationType : unsigned char {
898 Cmp,
899 OverflowingBinOp,
900 DisjointOp,
901 PossiblyExactOp,
902 GEPOp,
903 FPMathOp,
904 NonNegOp,
905 Other
906 };
907
908public:
909 struct WrapFlagsTy {
910 char HasNUW : 1;
911 char HasNSW : 1;
912
914 };
915
916protected:
917 struct GEPFlagsTy {
918 char IsInBounds : 1;
920 };
921
922private:
923 struct DisjointFlagsTy {
924 char IsDisjoint : 1;
925 };
926 struct ExactFlagsTy {
927 char IsExact : 1;
928 };
929 struct NonNegFlagsTy {
930 char NonNeg : 1;
931 };
932 struct FastMathFlagsTy {
933 char AllowReassoc : 1;
934 char NoNaNs : 1;
935 char NoInfs : 1;
936 char NoSignedZeros : 1;
937 char AllowReciprocal : 1;
938 char AllowContract : 1;
939 char ApproxFunc : 1;
940
941 FastMathFlagsTy(const FastMathFlags &FMF);
942 };
943
944 OperationType OpType;
945
946 union {
949 DisjointFlagsTy DisjointFlags;
950 ExactFlagsTy ExactFlags;
952 NonNegFlagsTy NonNegFlags;
953 FastMathFlagsTy FMFs;
954 unsigned AllFlags;
955 };
956
957protected:
959 OpType = Other.OpType;
960 AllFlags = Other.AllFlags;
961 }
962
963public:
964 template <typename IterT>
965 VPRecipeWithIRFlags(const unsigned char SC, IterT Operands, DebugLoc DL = {})
966 : VPSingleDefRecipe(SC, Operands, DL) {
967 OpType = OperationType::Other;
968 AllFlags = 0;
969 }
970
971 template <typename IterT>
972 VPRecipeWithIRFlags(const unsigned char SC, IterT Operands, Instruction &I)
974 if (auto *Op = dyn_cast<CmpInst>(&I)) {
975 OpType = OperationType::Cmp;
976 CmpPredicate = Op->getPredicate();
977 } else if (auto *Op = dyn_cast<PossiblyDisjointInst>(&I)) {
978 OpType = OperationType::DisjointOp;
979 DisjointFlags.IsDisjoint = Op->isDisjoint();
980 } else if (auto *Op = dyn_cast<OverflowingBinaryOperator>(&I)) {
981 OpType = OperationType::OverflowingBinOp;
982 WrapFlags = {Op->hasNoUnsignedWrap(), Op->hasNoSignedWrap()};
983 } else if (auto *Op = dyn_cast<PossiblyExactOperator>(&I)) {
984 OpType = OperationType::PossiblyExactOp;
985 ExactFlags.IsExact = Op->isExact();
986 } else if (auto *GEP = dyn_cast<GetElementPtrInst>(&I)) {
987 OpType = OperationType::GEPOp;
988 GEPFlags.IsInBounds = GEP->isInBounds();
989 } else if (auto *PNNI = dyn_cast<PossiblyNonNegInst>(&I)) {
990 OpType = OperationType::NonNegOp;
991 NonNegFlags.NonNeg = PNNI->hasNonNeg();
992 } else if (auto *Op = dyn_cast<FPMathOperator>(&I)) {
993 OpType = OperationType::FPMathOp;
994 FMFs = Op->getFastMathFlags();
995 } else {
996 OpType = OperationType::Other;
997 AllFlags = 0;
998 }
999 }
1000
1001 template <typename IterT>
1002 VPRecipeWithIRFlags(const unsigned char SC, IterT Operands,
1003 CmpInst::Predicate Pred, DebugLoc DL = {})
1004 : VPSingleDefRecipe(SC, Operands, DL), OpType(OperationType::Cmp),
1005 CmpPredicate(Pred) {}
1006
1007 template <typename IterT>
1008 VPRecipeWithIRFlags(const unsigned char SC, IterT Operands,
1010 : VPSingleDefRecipe(SC, Operands, DL),
1011 OpType(OperationType::OverflowingBinOp), WrapFlags(WrapFlags) {}
1012
1013 template <typename IterT>
1014 VPRecipeWithIRFlags(const unsigned char SC, IterT Operands,
1015 FastMathFlags FMFs, DebugLoc DL = {})
1016 : VPSingleDefRecipe(SC, Operands, DL), OpType(OperationType::FPMathOp),
1017 FMFs(FMFs) {}
1018
1019protected:
1020 template <typename IterT>
1021 VPRecipeWithIRFlags(const unsigned char SC, IterT Operands,
1022 GEPFlagsTy GEPFlags, DebugLoc DL = {})
1023 : VPSingleDefRecipe(SC, Operands, DL), OpType(OperationType::GEPOp),
1024 GEPFlags(GEPFlags) {}
1025
1026public:
1027 static inline bool classof(const VPRecipeBase *R) {
1028 return R->getVPDefID() == VPRecipeBase::VPInstructionSC ||
1029 R->getVPDefID() == VPRecipeBase::VPWidenSC ||
1030 R->getVPDefID() == VPRecipeBase::VPWidenGEPSC ||
1031 R->getVPDefID() == VPRecipeBase::VPWidenCastSC ||
1032 R->getVPDefID() == VPRecipeBase::VPReplicateSC ||
1033 R->getVPDefID() == VPRecipeBase::VPVectorPointerSC;
1034 }
1035
1036 /// Drop all poison-generating flags.
1038 // NOTE: This needs to be kept in-sync with
1039 // Instruction::dropPoisonGeneratingFlags.
1040 switch (OpType) {
1041 case OperationType::OverflowingBinOp:
1042 WrapFlags.HasNUW = false;
1043 WrapFlags.HasNSW = false;
1044 break;
1045 case OperationType::DisjointOp:
1046 DisjointFlags.IsDisjoint = false;
1047 break;
1048 case OperationType::PossiblyExactOp:
1049 ExactFlags.IsExact = false;
1050 break;
1051 case OperationType::GEPOp:
1052 GEPFlags.IsInBounds = false;
1053 break;
1054 case OperationType::FPMathOp:
1055 FMFs.NoNaNs = false;
1056 FMFs.NoInfs = false;
1057 break;
1058 case OperationType::NonNegOp:
1059 NonNegFlags.NonNeg = false;
1060 break;
1061 case OperationType::Cmp:
1062 case OperationType::Other:
1063 break;
1064 }
1065 }
1066
1067 /// Set the IR flags for \p I.
1068 void setFlags(Instruction *I) const {
1069 switch (OpType) {
1070 case OperationType::OverflowingBinOp:
1071 I->setHasNoUnsignedWrap(WrapFlags.HasNUW);
1072 I->setHasNoSignedWrap(WrapFlags.HasNSW);
1073 break;
1074 case OperationType::DisjointOp:
1075 cast<PossiblyDisjointInst>(I)->setIsDisjoint(DisjointFlags.IsDisjoint);
1076 break;
1077 case OperationType::PossiblyExactOp:
1078 I->setIsExact(ExactFlags.IsExact);
1079 break;
1080 case OperationType::GEPOp:
1081 cast<GetElementPtrInst>(I)->setIsInBounds(GEPFlags.IsInBounds);
1082 break;
1083 case OperationType::FPMathOp:
1084 I->setHasAllowReassoc(FMFs.AllowReassoc);
1085 I->setHasNoNaNs(FMFs.NoNaNs);
1086 I->setHasNoInfs(FMFs.NoInfs);
1087 I->setHasNoSignedZeros(FMFs.NoSignedZeros);
1088 I->setHasAllowReciprocal(FMFs.AllowReciprocal);
1089 I->setHasAllowContract(FMFs.AllowContract);
1090 I->setHasApproxFunc(FMFs.ApproxFunc);
1091 break;
1092 case OperationType::NonNegOp:
1093 I->setNonNeg(NonNegFlags.NonNeg);
1094 break;
1095 case OperationType::Cmp:
1096 case OperationType::Other:
1097 break;
1098 }
1099 }
1100
1102 assert(OpType == OperationType::Cmp &&
1103 "recipe doesn't have a compare predicate");
1104 return CmpPredicate;
1105 }
1106
1107 bool isInBounds() const {
1108 assert(OpType == OperationType::GEPOp &&
1109 "recipe doesn't have inbounds flag");
1110 return GEPFlags.IsInBounds;
1111 }
1112
1113 /// Returns true if the recipe has fast-math flags.
1114 bool hasFastMathFlags() const { return OpType == OperationType::FPMathOp; }
1115
1117
1118 bool hasNoUnsignedWrap() const {
1119 assert(OpType == OperationType::OverflowingBinOp &&
1120 "recipe doesn't have a NUW flag");
1121 return WrapFlags.HasNUW;
1122 }
1123
1124 bool hasNoSignedWrap() const {
1125 assert(OpType == OperationType::OverflowingBinOp &&
1126 "recipe doesn't have a NSW flag");
1127 return WrapFlags.HasNSW;
1128 }
1129
1130#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1131 void printFlags(raw_ostream &O) const;
1132#endif
1133};
1134
1135/// This is a concrete Recipe that models a single VPlan-level instruction.
1136/// While as any Recipe it may generate a sequence of IR instructions when
1137/// executed, these instructions would always form a single-def expression as
1138/// the VPInstruction is also a single def-use vertex.
1140 friend class VPlanSlp;
1141
1142public:
1143 /// VPlan opcodes, extending LLVM IR with idiomatics instructions.
1144 enum {
1146 Instruction::OtherOpsEnd + 1, // Combines the incoming and previous
1147 // values of a first-order recurrence.
1153 // Increment the canonical IV separately for each unrolled part.
1158 };
1159
1160private:
1161 typedef unsigned char OpcodeTy;
1162 OpcodeTy Opcode;
1163
1164 /// An optional name that can be used for the generated IR instruction.
1165 const std::string Name;
1166
1167 /// Utility method serving execute(): generates a single instance of the
1168 /// modeled instruction. \returns the generated value for \p Part.
1169 /// In some cases an existing value is returned rather than a generated
1170 /// one.
1171 Value *generateInstruction(VPTransformState &State, unsigned Part);
1172
1173#if !defined(NDEBUG)
1174 /// Return true if the VPInstruction is a floating point math operation, i.e.
1175 /// has fast-math flags.
1176 bool isFPMathOp() const;
1177#endif
1178
1179public:
1181 const Twine &Name = "")
1182 : VPRecipeWithIRFlags(VPDef::VPInstructionSC, Operands, DL),
1183 Opcode(Opcode), Name(Name.str()) {}
1184
1185 VPInstruction(unsigned Opcode, std::initializer_list<VPValue *> Operands,
1186 DebugLoc DL = {}, const Twine &Name = "")
1188
1189 VPInstruction(unsigned Opcode, CmpInst::Predicate Pred, VPValue *A,
1190 VPValue *B, DebugLoc DL = {}, const Twine &Name = "");
1191
1192 VPInstruction(unsigned Opcode, std::initializer_list<VPValue *> Operands,
1193 WrapFlagsTy WrapFlags, DebugLoc DL = {}, const Twine &Name = "")
1194 : VPRecipeWithIRFlags(VPDef::VPInstructionSC, Operands, WrapFlags, DL),
1195 Opcode(Opcode), Name(Name.str()) {}
1196
1197 VPInstruction(unsigned Opcode, std::initializer_list<VPValue *> Operands,
1198 FastMathFlags FMFs, DebugLoc DL = {}, const Twine &Name = "");
1199
1200 VP_CLASSOF_IMPL(VPDef::VPInstructionSC)
1201
1202 VPRecipeBase *clone() override {
1204 auto *New = new VPInstruction(Opcode, Operands, getDebugLoc(), Name);
1205 New->transferFlags(*this);
1206 return New;
1207 }
1208
1209 unsigned getOpcode() const { return Opcode; }
1210
1211 /// Generate the instruction.
1212 /// TODO: We currently execute only per-part unless a specific instance is
1213 /// provided.
1214 void execute(VPTransformState &State) override;
1215
1216#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1217 /// Print the VPInstruction to \p O.
1218 void print(raw_ostream &O, const Twine &Indent,
1219 VPSlotTracker &SlotTracker) const override;
1220
1221 /// Print the VPInstruction to dbgs() (for debugging).
1222 LLVM_DUMP_METHOD void dump() const;
1223#endif
1224
1225 /// Return true if this instruction may modify memory.
1226 bool mayWriteToMemory() const {
1227 // TODO: we can use attributes of the called function to rule out memory
1228 // modifications.
1229 return Opcode == Instruction::Store || Opcode == Instruction::Call ||
1230 Opcode == Instruction::Invoke || Opcode == SLPStore;
1231 }
1232
1233 bool hasResult() const {
1234 // CallInst may or may not have a result, depending on the called function.
1235 // Conservatively return calls have results for now.
1236 switch (getOpcode()) {
1237 case Instruction::Ret:
1238 case Instruction::Br:
1239 case Instruction::Store:
1240 case Instruction::Switch:
1241 case Instruction::IndirectBr:
1242 case Instruction::Resume:
1243 case Instruction::CatchRet:
1244 case Instruction::Unreachable:
1245 case Instruction::Fence:
1246 case Instruction::AtomicRMW:
1249 return false;
1250 default:
1251 return true;
1252 }
1253 }
1254
1255 /// Returns true if the recipe only uses the first lane of operand \p Op.
1256 bool onlyFirstLaneUsed(const VPValue *Op) const override;
1257
1258 /// Returns true if the recipe only uses the first part of operand \p Op.
1259 bool onlyFirstPartUsed(const VPValue *Op) const override {
1261 "Op must be an operand of the recipe");
1262 if (getOperand(0) != Op)
1263 return false;
1264 switch (getOpcode()) {
1265 default:
1266 return false;
1268 return true;
1269 };
1270 llvm_unreachable("switch should return");
1271 }
1272};
1273
1274/// VPWidenRecipe is a recipe for producing a copy of vector type its
1275/// ingredient. This recipe covers most of the traditional vectorization cases
1276/// where each ingredient transforms into a vectorized version of itself.
1278 unsigned Opcode;
1279
1280public:
1281 template <typename IterT>
1283 : VPRecipeWithIRFlags(VPDef::VPWidenSC, Operands, I),
1284 Opcode(I.getOpcode()) {}
1285
1286 ~VPWidenRecipe() override = default;
1287
1288 VPRecipeBase *clone() override {
1289 auto *R = new VPWidenRecipe(*getUnderlyingInstr(), operands());
1290 R->transferFlags(*this);
1291 return R;
1292 }
1293
1294 VP_CLASSOF_IMPL(VPDef::VPWidenSC)
1295
1296 /// Produce widened copies of all Ingredients.
1297 void execute(VPTransformState &State) override;
1298
1299 unsigned getOpcode() const { return Opcode; }
1300
1301#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1302 /// Print the recipe.
1303 void print(raw_ostream &O, const Twine &Indent,
1304 VPSlotTracker &SlotTracker) const override;
1305#endif
1306};
1307
1308/// VPWidenCastRecipe is a recipe to create vector cast instructions.
1310 /// Cast instruction opcode.
1311 Instruction::CastOps Opcode;
1312
1313 /// Result type for the cast.
1314 Type *ResultTy;
1315
1316public:
1318 CastInst &UI)
1319 : VPRecipeWithIRFlags(VPDef::VPWidenCastSC, Op, UI), Opcode(Opcode),
1320 ResultTy(ResultTy) {
1321 assert(UI.getOpcode() == Opcode &&
1322 "opcode of underlying cast doesn't match");
1323 assert(UI.getType() == ResultTy &&
1324 "result type of underlying cast doesn't match");
1325 }
1326
1328 : VPRecipeWithIRFlags(VPDef::VPWidenCastSC, Op), Opcode(Opcode),
1329 ResultTy(ResultTy) {}
1330
1331 ~VPWidenCastRecipe() override = default;
1332
1333 VPRecipeBase *clone() override {
1334 if (auto *UV = getUnderlyingValue())
1335 return new VPWidenCastRecipe(Opcode, getOperand(0), ResultTy,
1336 *cast<CastInst>(UV));
1337
1338 return new VPWidenCastRecipe(Opcode, getOperand(0), ResultTy);
1339 }
1340
1341 VP_CLASSOF_IMPL(VPDef::VPWidenCastSC)
1342
1343 /// Produce widened copies of the cast.
1344 void execute(VPTransformState &State) override;
1345
1346#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1347 /// Print the recipe.
1348 void print(raw_ostream &O, const Twine &Indent,
1349 VPSlotTracker &SlotTracker) const override;
1350#endif
1351
1352 Instruction::CastOps getOpcode() const { return Opcode; }
1353
1354 /// Returns the result type of the cast.
1355 Type *getResultType() const { return ResultTy; }
1356};
1357
1358/// VPScalarCastRecipe is a recipe to create scalar cast instructions.
1360 Instruction::CastOps Opcode;
1361
1362 Type *ResultTy;
1363
1364 Value *generate(VPTransformState &State, unsigned Part);
1365
1366public:
1368 : VPSingleDefRecipe(VPDef::VPScalarCastSC, {Op}), Opcode(Opcode),
1369 ResultTy(ResultTy) {}
1370
1371 ~VPScalarCastRecipe() override = default;
1372
1373 VPRecipeBase *clone() override {
1374 return new VPScalarCastRecipe(Opcode, getOperand(0), ResultTy);
1375 }
1376
1377 VP_CLASSOF_IMPL(VPDef::VPScalarCastSC)
1378
1379 void execute(VPTransformState &State) override;
1380
1381#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1382 void print(raw_ostream &O, const Twine &Indent,
1383 VPSlotTracker &SlotTracker) const override;
1384#endif
1385
1386 /// Returns the result type of the cast.
1387 Type *getResultType() const { return ResultTy; }
1388
1389 bool onlyFirstLaneUsed(const VPValue *Op) const override {
1390 // At the moment, only uniform codegen is implemented.
1392 "Op must be an operand of the recipe");
1393 return true;
1394 }
1395};
1396
1397/// A recipe for widening Call instructions.
1399 /// ID of the vector intrinsic to call when widening the call. If set the
1400 /// Intrinsic::not_intrinsic, a library call will be used instead.
1401 Intrinsic::ID VectorIntrinsicID;
1402 /// If this recipe represents a library call, Variant stores a pointer to
1403 /// the chosen function. There is a 1:1 mapping between a given VF and the
1404 /// chosen vectorized variant, so there will be a different vplan for each
1405 /// VF with a valid variant.
1406 Function *Variant;
1407
1408public:
1409 template <typename IterT>
1411 Intrinsic::ID VectorIntrinsicID, DebugLoc DL = {},
1412 Function *Variant = nullptr)
1413 : VPSingleDefRecipe(VPDef::VPWidenCallSC, CallArguments, &I, DL),
1414 VectorIntrinsicID(VectorIntrinsicID), Variant(Variant) {}
1415
1416 ~VPWidenCallRecipe() override = default;
1417
1418 VPRecipeBase *clone() override {
1419 return new VPWidenCallRecipe(*cast<CallInst>(getUnderlyingInstr()),
1420 operands(), VectorIntrinsicID, getDebugLoc(),
1421 Variant);
1422 }
1423
1424 VP_CLASSOF_IMPL(VPDef::VPWidenCallSC)
1425
1426 /// Produce a widened version of the call instruction.
1427 void execute(VPTransformState &State) override;
1428
1429#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1430 /// Print the recipe.
1431 void print(raw_ostream &O, const Twine &Indent,
1432 VPSlotTracker &SlotTracker) const override;
1433#endif
1434};
1435
1436/// A recipe for widening select instructions.
1438 template <typename IterT>
1440 : VPSingleDefRecipe(VPDef::VPWidenSelectSC, Operands, &I,
1441 I.getDebugLoc()) {}
1442
1443 ~VPWidenSelectRecipe() override = default;
1444
1445 VPRecipeBase *clone() override {
1446 return new VPWidenSelectRecipe(*cast<SelectInst>(getUnderlyingInstr()),
1447 operands());
1448 }
1449
1450 VP_CLASSOF_IMPL(VPDef::VPWidenSelectSC)
1451
1452 /// Produce a widened version of the select instruction.
1453 void execute(VPTransformState &State) override;
1454
1455#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1456 /// Print the recipe.
1457 void print(raw_ostream &O, const Twine &Indent,
1458 VPSlotTracker &SlotTracker) const override;
1459#endif
1460
1461 VPValue *getCond() const {
1462 return getOperand(0);
1463 }
1464
1465 bool isInvariantCond() const {
1467 }
1468};
1469
1470/// A recipe for handling GEP instructions.
1472 bool isPointerLoopInvariant() const {
1474 }
1475
1476 bool isIndexLoopInvariant(unsigned I) const {
1478 }
1479
1480 bool areAllOperandsInvariant() const {
1481 return all_of(operands(), [](VPValue *Op) {
1482 return Op->isDefinedOutsideVectorRegions();
1483 });
1484 }
1485
1486public:
1487 template <typename IterT>
1489 : VPRecipeWithIRFlags(VPDef::VPWidenGEPSC, Operands, *GEP) {}
1490
1491 ~VPWidenGEPRecipe() override = default;
1492
1493 VPRecipeBase *clone() override {
1494 return new VPWidenGEPRecipe(cast<GetElementPtrInst>(getUnderlyingInstr()),
1495 operands());
1496 }
1497
1498 VP_CLASSOF_IMPL(VPDef::VPWidenGEPSC)
1499
1500 /// Generate the gep nodes.
1501 void execute(VPTransformState &State) override;
1502
1503#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1504 /// Print the recipe.
1505 void print(raw_ostream &O, const Twine &Indent,
1506 VPSlotTracker &SlotTracker) const override;
1507#endif
1508};
1509
1510/// A recipe to compute the pointers for widened memory accesses of IndexTy for
1511/// all parts. If IsReverse is true, compute pointers for accessing the input in
1512/// reverse order per part.
1514 Type *IndexedTy;
1515 bool IsReverse;
1516
1517public:
1518 VPVectorPointerRecipe(VPValue *Ptr, Type *IndexedTy, bool IsReverse,
1519 bool IsInBounds, DebugLoc DL)
1520 : VPRecipeWithIRFlags(VPDef::VPVectorPointerSC, ArrayRef<VPValue *>(Ptr),
1521 GEPFlagsTy(IsInBounds), DL),
1522 IndexedTy(IndexedTy), IsReverse(IsReverse) {}
1523
1524 VP_CLASSOF_IMPL(VPDef::VPVectorPointerSC)
1525
1526 void execute(VPTransformState &State) override;
1527
1528 bool onlyFirstLaneUsed(const VPValue *Op) const override {
1530 "Op must be an operand of the recipe");
1531 return true;
1532 }
1533
1534 VPRecipeBase *clone() override {
1535 return new VPVectorPointerRecipe(getOperand(0), IndexedTy, IsReverse,
1536 isInBounds(), getDebugLoc());
1537 }
1538
1539#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1540 /// Print the recipe.
1541 void print(raw_ostream &O, const Twine &Indent,
1542 VPSlotTracker &SlotTracker) const override;
1543#endif
1544};
1545
1546/// A pure virtual base class for all recipes modeling header phis, including
1547/// phis for first order recurrences, pointer inductions and reductions. The
1548/// start value is the first operand of the recipe and the incoming value from
1549/// the backedge is the second operand.
1550///
1551/// Inductions are modeled using the following sub-classes:
1552/// * VPCanonicalIVPHIRecipe: Canonical scalar induction of the vector loop,
1553/// starting at a specified value (zero for the main vector loop, the resume
1554/// value for the epilogue vector loop) and stepping by 1. The induction
1555/// controls exiting of the vector loop by comparing against the vector trip
1556/// count. Produces a single scalar PHI for the induction value per
1557/// iteration.
1558/// * VPWidenIntOrFpInductionRecipe: Generates vector values for integer and
1559/// floating point inductions with arbitrary start and step values. Produces
1560/// a vector PHI per-part.
1561/// * VPDerivedIVRecipe: Converts the canonical IV value to the corresponding
1562/// value of an IV with different start and step values. Produces a single
1563/// scalar value per iteration
1564/// * VPScalarIVStepsRecipe: Generates scalar values per-lane based on a
1565/// canonical or derived induction.
1566/// * VPWidenPointerInductionRecipe: Generate vector and scalar values for a
1567/// pointer induction. Produces either a vector PHI per-part or scalar values
1568/// per-lane based on the canonical induction.
1570protected:
1571 VPHeaderPHIRecipe(unsigned char VPDefID, Instruction *UnderlyingInstr,
1572 VPValue *Start = nullptr, DebugLoc DL = {})
1573 : VPSingleDefRecipe(VPDefID, ArrayRef<VPValue *>(), UnderlyingInstr, DL) {
1574 if (Start)
1575 addOperand(Start);
1576 }
1577
1578public:
1579 ~VPHeaderPHIRecipe() override = default;
1580
1581 /// Method to support type inquiry through isa, cast, and dyn_cast.
1582 static inline bool classof(const VPRecipeBase *B) {
1583 return B->getVPDefID() >= VPDef::VPFirstHeaderPHISC &&
1584 B->getVPDefID() <= VPDef::VPLastHeaderPHISC;
1585 }
1586 static inline bool classof(const VPValue *V) {
1587 auto *B = V->getDefiningRecipe();
1588 return B && B->getVPDefID() >= VPRecipeBase::VPFirstHeaderPHISC &&
1589 B->getVPDefID() <= VPRecipeBase::VPLastHeaderPHISC;
1590 }
1591
1592 /// Generate the phi nodes.
1593 void execute(VPTransformState &State) override = 0;
1594
1595#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1596 /// Print the recipe.
1597 void print(raw_ostream &O, const Twine &Indent,
1598 VPSlotTracker &SlotTracker) const override = 0;
1599#endif
1600
1601 /// Returns the start value of the phi, if one is set.
1603 return getNumOperands() == 0 ? nullptr : getOperand(0);
1604 }
1606 return getNumOperands() == 0 ? nullptr : getOperand(0);
1607 }
1608
1609 /// Update the start value of the recipe.
1611
1612 /// Returns the incoming value from the loop backedge.
1614 return getOperand(1);
1615 }
1616
1617 /// Returns the backedge value as a recipe. The backedge value is guaranteed
1618 /// to be a recipe.
1621 }
1622};
1623
1624/// A recipe for handling phi nodes of integer and floating-point inductions,
1625/// producing their vector values.
1627 PHINode *IV;
1628 TruncInst *Trunc;
1629 const InductionDescriptor &IndDesc;
1630
1631public:
1633 const InductionDescriptor &IndDesc)
1634 : VPHeaderPHIRecipe(VPDef::VPWidenIntOrFpInductionSC, IV, Start), IV(IV),
1635 Trunc(nullptr), IndDesc(IndDesc) {
1636 addOperand(Step);
1637 }
1638
1640 const InductionDescriptor &IndDesc,
1641 TruncInst *Trunc)
1642 : VPHeaderPHIRecipe(VPDef::VPWidenIntOrFpInductionSC, Trunc, Start),
1643 IV(IV), Trunc(Trunc), IndDesc(IndDesc) {
1644 addOperand(Step);
1645 }
1646
1648
1649 VPRecipeBase *clone() override {
1651 getStepValue(), IndDesc, Trunc);
1652 }
1653
1654 VP_CLASSOF_IMPL(VPDef::VPWidenIntOrFpInductionSC)
1655
1656 /// Generate the vectorized and scalarized versions of the phi node as
1657 /// needed by their users.
1658 void execute(VPTransformState &State) override;
1659
1660#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1661 /// Print the recipe.
1662 void print(raw_ostream &O, const Twine &Indent,
1663 VPSlotTracker &SlotTracker) const override;
1664#endif
1665
1667 // TODO: All operands of base recipe must exist and be at same index in
1668 // derived recipe.
1670 "VPWidenIntOrFpInductionRecipe generates its own backedge value");
1671 }
1672
1674 // TODO: All operands of base recipe must exist and be at same index in
1675 // derived recipe.
1677 "VPWidenIntOrFpInductionRecipe generates its own backedge value");
1678 }
1679
1680 /// Returns the step value of the induction.
1682 const VPValue *getStepValue() const { return getOperand(1); }
1683
1684 /// Returns the first defined value as TruncInst, if it is one or nullptr
1685 /// otherwise.
1686 TruncInst *getTruncInst() { return Trunc; }
1687 const TruncInst *getTruncInst() const { return Trunc; }
1688
1689 PHINode *getPHINode() { return IV; }
1690
1691 /// Returns the induction descriptor for the recipe.
1692 const InductionDescriptor &getInductionDescriptor() const { return IndDesc; }
1693
1694 /// Returns true if the induction is canonical, i.e. starting at 0 and
1695 /// incremented by UF * VF (= the original IV is incremented by 1).
1696 bool isCanonical() const;
1697
1698 /// Returns the scalar type of the induction.
1700 return Trunc ? Trunc->getType() : IV->getType();
1701 }
1702};
1703
1705 const InductionDescriptor &IndDesc;
1706
1707 bool IsScalarAfterVectorization;
1708
1709public:
1710 /// Create a new VPWidenPointerInductionRecipe for \p Phi with start value \p
1711 /// Start.
1713 const InductionDescriptor &IndDesc,
1714 bool IsScalarAfterVectorization)
1715 : VPHeaderPHIRecipe(VPDef::VPWidenPointerInductionSC, Phi),
1716 IndDesc(IndDesc),
1717 IsScalarAfterVectorization(IsScalarAfterVectorization) {
1718 addOperand(Start);
1719 addOperand(Step);
1720 }
1721
1723
1724 VPRecipeBase *clone() override {
1726 cast<PHINode>(getUnderlyingInstr()), getOperand(0), getOperand(1),
1727 IndDesc, IsScalarAfterVectorization);
1728 }
1729
1730 VP_CLASSOF_IMPL(VPDef::VPWidenPointerInductionSC)
1731
1732 /// Generate vector values for the pointer induction.
1733 void execute(VPTransformState &State) override;
1734
1735 /// Returns true if only scalar values will be generated.
1736 bool onlyScalarsGenerated(bool IsScalable);
1737
1738 /// Returns the induction descriptor for the recipe.
1739 const InductionDescriptor &getInductionDescriptor() const { return IndDesc; }
1740
1741#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1742 /// Print the recipe.
1743 void print(raw_ostream &O, const Twine &Indent,
1744 VPSlotTracker &SlotTracker) const override;
1745#endif
1746};
1747
1748/// A recipe for handling phis that are widened in the vector loop.
1749/// In the VPlan native path, all incoming VPValues & VPBasicBlock pairs are
1750/// managed in the recipe directly.
1752 /// List of incoming blocks. Only used in the VPlan native path.
1753 SmallVector<VPBasicBlock *, 2> IncomingBlocks;
1754
1755public:
1756 /// Create a new VPWidenPHIRecipe for \p Phi with start value \p Start.
1757 VPWidenPHIRecipe(PHINode *Phi, VPValue *Start = nullptr)
1758 : VPSingleDefRecipe(VPDef::VPWidenPHISC, ArrayRef<VPValue *>(), Phi) {
1759 if (Start)
1760 addOperand(Start);
1761 }
1762
1763 VPRecipeBase *clone() override {
1764 llvm_unreachable("cloning not implemented yet");
1765 }
1766
1767 ~VPWidenPHIRecipe() override = default;
1768
1769 VP_CLASSOF_IMPL(VPDef::VPWidenPHISC)
1770
1771 /// Generate the phi/select nodes.
1772 void execute(VPTransformState &State) override;
1773
1774#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1775 /// Print the recipe.
1776 void print(raw_ostream &O, const Twine &Indent,
1777 VPSlotTracker &SlotTracker) const override;
1778#endif
1779
1780 /// Adds a pair (\p IncomingV, \p IncomingBlock) to the phi.
1781 void addIncoming(VPValue *IncomingV, VPBasicBlock *IncomingBlock) {
1782 addOperand(IncomingV);
1783 IncomingBlocks.push_back(IncomingBlock);
1784 }
1785
1786 /// Returns the \p I th incoming VPBasicBlock.
1787 VPBasicBlock *getIncomingBlock(unsigned I) { return IncomingBlocks[I]; }
1788
1789 /// Returns the \p I th incoming VPValue.
1790 VPValue *getIncomingValue(unsigned I) { return getOperand(I); }
1791};
1792
1793/// A recipe for handling first-order recurrence phis. The start value is the
1794/// first operand of the recipe and the incoming value from the backedge is the
1795/// second operand.
1798 : VPHeaderPHIRecipe(VPDef::VPFirstOrderRecurrencePHISC, Phi, &Start) {}
1799
1800 VP_CLASSOF_IMPL(VPDef::VPFirstOrderRecurrencePHISC)
1801
1803 return R->getVPDefID() == VPDef::VPFirstOrderRecurrencePHISC;
1804 }
1805
1806 VPRecipeBase *clone() override {
1808 cast<PHINode>(getUnderlyingInstr()), *getOperand(0));
1809 }
1810
1811 void execute(VPTransformState &State) override;
1812
1813#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1814 /// Print the recipe.
1815 void print(raw_ostream &O, const Twine &Indent,
1816 VPSlotTracker &SlotTracker) const override;
1817#endif
1818};
1819
1820/// A recipe for handling reduction phis. The start value is the first operand
1821/// of the recipe and the incoming value from the backedge is the second
1822/// operand.
1824 /// Descriptor for the reduction.
1825 const RecurrenceDescriptor &RdxDesc;
1826
1827 /// The phi is part of an in-loop reduction.
1828 bool IsInLoop;
1829
1830 /// The phi is part of an ordered reduction. Requires IsInLoop to be true.
1831 bool IsOrdered;
1832
1833public:
1834 /// Create a new VPReductionPHIRecipe for the reduction \p Phi described by \p
1835 /// RdxDesc.
1837 VPValue &Start, bool IsInLoop = false,
1838 bool IsOrdered = false)
1839 : VPHeaderPHIRecipe(VPDef::VPReductionPHISC, Phi, &Start),
1840 RdxDesc(RdxDesc), IsInLoop(IsInLoop), IsOrdered(IsOrdered) {
1841 assert((!IsOrdered || IsInLoop) && "IsOrdered requires IsInLoop");
1842 }
1843
1844 ~VPReductionPHIRecipe() override = default;
1845
1846 VPRecipeBase *clone() override {
1847 auto *R =
1848 new VPReductionPHIRecipe(cast<PHINode>(getUnderlyingInstr()), RdxDesc,
1849 *getOperand(0), IsInLoop, IsOrdered);
1850 R->addOperand(getBackedgeValue());
1851 return R;
1852 }
1853
1854 VP_CLASSOF_IMPL(VPDef::VPReductionPHISC)
1855
1857 return R->getVPDefID() == VPDef::VPReductionPHISC;
1858 }
1859
1860 /// Generate the phi/select nodes.
1861 void execute(VPTransformState &State) override;
1862
1863#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1864 /// Print the recipe.
1865 void print(raw_ostream &O, const Twine &Indent,
1866 VPSlotTracker &SlotTracker) const override;
1867#endif
1868
1870 return RdxDesc;
1871 }
1872
1873 /// Returns true, if the phi is part of an ordered reduction.
1874 bool isOrdered() const { return IsOrdered; }
1875
1876 /// Returns true, if the phi is part of an in-loop reduction.
1877 bool isInLoop() const { return IsInLoop; }
1878};
1879
1880/// A recipe for vectorizing a phi-node as a sequence of mask-based select
1881/// instructions.
1883public:
1884 /// The blend operation is a User of the incoming values and of their
1885 /// respective masks, ordered [I0, M0, I1, M1, ...]. Note that a single value
1886 /// might be incoming with a full mask for which there is no VPValue.
1888 : VPSingleDefRecipe(VPDef::VPBlendSC, Operands, Phi, Phi->getDebugLoc()) {
1889 assert(Operands.size() > 0 &&
1890 ((Operands.size() == 1) || (Operands.size() % 2 == 0)) &&
1891 "Expected either a single incoming value or a positive even number "
1892 "of operands");
1893 }
1894
1895 VPRecipeBase *clone() override {
1897 return new VPBlendRecipe(cast<PHINode>(getUnderlyingValue()), Ops);
1898 }
1899
1900 VP_CLASSOF_IMPL(VPDef::VPBlendSC)
1901
1902 /// Return the number of incoming values, taking into account that a single
1903 /// incoming value has no mask.
1904 unsigned getNumIncomingValues() const { return (getNumOperands() + 1) / 2; }
1905
1906 /// Return incoming value number \p Idx.
1907 VPValue *getIncomingValue(unsigned Idx) const { return getOperand(Idx * 2); }
1908
1909 /// Return mask number \p Idx.
1910 VPValue *getMask(unsigned Idx) const { return getOperand(Idx * 2 + 1); }
1911
1912 /// Generate the phi/select nodes.
1913 void execute(VPTransformState &State) override;
1914
1915#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1916 /// Print the recipe.
1917 void print(raw_ostream &O, const Twine &Indent,
1918 VPSlotTracker &SlotTracker) const override;
1919#endif
1920
1921 /// Returns true if the recipe only uses the first lane of operand \p Op.
1922 bool onlyFirstLaneUsed(const VPValue *Op) const override {
1924 "Op must be an operand of the recipe");
1925 // Recursing through Blend recipes only, must terminate at header phi's the
1926 // latest.
1927 return all_of(users(),
1928 [this](VPUser *U) { return U->onlyFirstLaneUsed(this); });
1929 }
1930};
1931
1932/// VPInterleaveRecipe is a recipe for transforming an interleave group of load
1933/// or stores into one wide load/store and shuffles. The first operand of a
1934/// VPInterleave recipe is the address, followed by the stored values, followed
1935/// by an optional mask.
1938
1939 /// Indicates if the interleave group is in a conditional block and requires a
1940 /// mask.
1941 bool HasMask = false;
1942
1943 /// Indicates if gaps between members of the group need to be masked out or if
1944 /// unusued gaps can be loaded speculatively.
1945 bool NeedsMaskForGaps = false;
1946
1947public:
1949 ArrayRef<VPValue *> StoredValues, VPValue *Mask,
1950 bool NeedsMaskForGaps)
1951 : VPRecipeBase(VPDef::VPInterleaveSC, {Addr}), IG(IG),
1952 NeedsMaskForGaps(NeedsMaskForGaps) {
1953 for (unsigned i = 0; i < IG->getFactor(); ++i)
1954 if (Instruction *I = IG->getMember(i)) {
1955 if (I->getType()->isVoidTy())
1956 continue;
1957 new VPValue(I, this);
1958 }
1959
1960 for (auto *SV : StoredValues)
1961 addOperand(SV);
1962 if (Mask) {
1963 HasMask = true;
1964 addOperand(Mask);
1965 }
1966 }
1967 ~VPInterleaveRecipe() override = default;
1968
1969 VPRecipeBase *clone() override {
1970 return new VPInterleaveRecipe(IG, getAddr(), getStoredValues(), getMask(),
1971 NeedsMaskForGaps);
1972 }
1973
1974 VP_CLASSOF_IMPL(VPDef::VPInterleaveSC)
1975
1976 /// Return the address accessed by this recipe.
1977 VPValue *getAddr() const {
1978 return getOperand(0); // Address is the 1st, mandatory operand.
1979 }
1980
1981 /// Return the mask used by this recipe. Note that a full mask is represented
1982 /// by a nullptr.
1983 VPValue *getMask() const {
1984 // Mask is optional and therefore the last, currently 2nd operand.
1985 return HasMask ? getOperand(getNumOperands() - 1) : nullptr;
1986 }
1987
1988 /// Return the VPValues stored by this interleave group. If it is a load
1989 /// interleave group, return an empty ArrayRef.
1991 // The first operand is the address, followed by the stored values, followed
1992 // by an optional mask.
1995 }
1996
1997 /// Generate the wide load or store, and shuffles.
1998 void execute(VPTransformState &State) override;
1999
2000#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2001 /// Print the recipe.
2002 void print(raw_ostream &O, const Twine &Indent,
2003 VPSlotTracker &SlotTracker) const override;
2004#endif
2005
2007
2008 /// Returns the number of stored operands of this interleave group. Returns 0
2009 /// for load interleave groups.
2010 unsigned getNumStoreOperands() const {
2011 return getNumOperands() - (HasMask ? 2 : 1);
2012 }
2013
2014 /// The recipe only uses the first lane of the address.
2015 bool onlyFirstLaneUsed(const VPValue *Op) const override {
2017 "Op must be an operand of the recipe");
2018 return Op == getAddr() && !llvm::is_contained(getStoredValues(), Op);
2019 }
2020};
2021
2022/// A recipe to represent inloop reduction operations, performing a reduction on
2023/// a vector operand into a scalar value, and adding the result to a chain.
2024/// The Operands are {ChainOp, VecOp, [Condition]}.
2026 /// The recurrence decriptor for the reduction in question.
2027 const RecurrenceDescriptor &RdxDesc;
2028
2029public:
2031 VPValue *ChainOp, VPValue *VecOp, VPValue *CondOp)
2032 : VPSingleDefRecipe(VPDef::VPReductionSC,
2033 ArrayRef<VPValue *>({ChainOp, VecOp}), I),
2034 RdxDesc(R) {
2035 if (CondOp)
2036 addOperand(CondOp);
2037 }
2038
2039 ~VPReductionRecipe() override = default;
2040
2041 VPRecipeBase *clone() override {
2042 return new VPReductionRecipe(RdxDesc, getUnderlyingInstr(), getChainOp(),
2043 getVecOp(), getCondOp());
2044 }
2045
2046 VP_CLASSOF_IMPL(VPDef::VPReductionSC)
2047
2048 /// Generate the reduction in the loop
2049 void execute(VPTransformState &State) override;
2050
2051#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2052 /// Print the recipe.
2053 void print(raw_ostream &O, const Twine &Indent,
2054 VPSlotTracker &SlotTracker) const override;
2055#endif
2056
2057 /// The VPValue of the scalar Chain being accumulated.
2058 VPValue *getChainOp() const { return getOperand(0); }
2059 /// The VPValue of the vector value to be reduced.
2060 VPValue *getVecOp() const { return getOperand(1); }
2061 /// The VPValue of the condition for the block.
2063 return getNumOperands() > 2 ? getOperand(2) : nullptr;
2064 }
2065};
2066
2067/// VPReplicateRecipe replicates a given instruction producing multiple scalar
2068/// copies of the original scalar type, one per lane, instead of producing a
2069/// single copy of widened type for all lanes. If the instruction is known to be
2070/// uniform only one copy, per lane zero, will be generated.
2072 /// Indicator if only a single replica per lane is needed.
2073 bool IsUniform;
2074
2075 /// Indicator if the replicas are also predicated.
2076 bool IsPredicated;
2077
2078public:
2079 template <typename IterT>
2081 bool IsUniform, VPValue *Mask = nullptr)
2082 : VPRecipeWithIRFlags(VPDef::VPReplicateSC, Operands, *I),
2083 IsUniform(IsUniform), IsPredicated(Mask) {
2084 if (Mask)
2085 addOperand(Mask);
2086 }
2087
2088 ~VPReplicateRecipe() override = default;
2089
2090 VPRecipeBase *clone() override {
2091 auto *Copy =
2092 new VPReplicateRecipe(getUnderlyingInstr(), operands(), IsUniform,
2093 isPredicated() ? getMask() : nullptr);
2094 Copy->transferFlags(*this);
2095 return Copy;
2096 }
2097
2098 VP_CLASSOF_IMPL(VPDef::VPReplicateSC)
2099
2100 /// Generate replicas of the desired Ingredient. Replicas will be generated
2101 /// for all parts and lanes unless a specific part and lane are specified in
2102 /// the \p State.
2103 void execute(VPTransformState &State) override;
2104
2105#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2106 /// Print the recipe.
2107 void print(raw_ostream &O, const Twine &Indent,
2108 VPSlotTracker &SlotTracker) const override;
2109#endif
2110
2111 bool isUniform() const { return IsUniform; }
2112
2113 bool isPredicated() const { return IsPredicated; }
2114
2115 /// Returns true if the recipe only uses the first lane of operand \p Op.
2116 bool onlyFirstLaneUsed(const VPValue *Op) const override {
2118 "Op must be an operand of the recipe");
2119 return isUniform();
2120 }
2121
2122 /// Returns true if the recipe uses scalars of operand \p Op.
2123 bool usesScalars(const VPValue *Op) const override {
2125 "Op must be an operand of the recipe");
2126 return true;
2127 }
2128
2129 /// Returns true if the recipe is used by a widened recipe via an intervening
2130 /// VPPredInstPHIRecipe. In this case, the scalar values should also be packed
2131 /// in a vector.
2132 bool shouldPack() const;
2133
2134 /// Return the mask of a predicated VPReplicateRecipe.
2136 assert(isPredicated() && "Trying to get the mask of a unpredicated recipe");
2137 return getOperand(getNumOperands() - 1);
2138 }
2139};
2140
2141/// A recipe for generating conditional branches on the bits of a mask.
2143public:
2145 : VPRecipeBase(VPDef::VPBranchOnMaskSC, {}) {
2146 if (BlockInMask) // nullptr means all-one mask.
2147 addOperand(BlockInMask);
2148 }
2149
2150 VPRecipeBase *clone() override {
2151 return new VPBranchOnMaskRecipe(getOperand(0));
2152 }
2153
2154 VP_CLASSOF_IMPL(VPDef::VPBranchOnMaskSC)
2155
2156 /// Generate the extraction of the appropriate bit from the block mask and the
2157 /// conditional branch.
2158 void execute(VPTransformState &State) override;
2159
2160#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2161 /// Print the recipe.
2162 void print(raw_ostream &O, const Twine &Indent,
2163 VPSlotTracker &SlotTracker) const override {
2164 O << Indent << "BRANCH-ON-MASK ";
2165 if (VPValue *Mask = getMask())
2166 Mask->printAsOperand(O, SlotTracker);
2167 else
2168 O << " All-One";
2169 }
2170#endif
2171
2172 /// Return the mask used by this recipe. Note that a full mask is represented
2173 /// by a nullptr.
2174 VPValue *getMask() const {
2175 assert(getNumOperands() <= 1 && "should have either 0 or 1 operands");
2176 // Mask is optional.
2177 return getNumOperands() == 1 ? getOperand(0) : nullptr;
2178 }
2179
2180 /// Returns true if the recipe uses scalars of operand \p Op.
2181 bool usesScalars(const VPValue *Op) const override {
2183 "Op must be an operand of the recipe");
2184 return true;
2185 }
2186};
2187
2188/// VPPredInstPHIRecipe is a recipe for generating the phi nodes needed when
2189/// control converges back from a Branch-on-Mask. The phi nodes are needed in
2190/// order to merge values that are set under such a branch and feed their uses.
2191/// The phi nodes can be scalar or vector depending on the users of the value.
2192/// This recipe works in concert with VPBranchOnMaskRecipe.
2194public:
2195 /// Construct a VPPredInstPHIRecipe given \p PredInst whose value needs a phi
2196 /// nodes after merging back from a Branch-on-Mask.
2198 : VPSingleDefRecipe(VPDef::VPPredInstPHISC, PredV) {}
2199 ~VPPredInstPHIRecipe() override = default;
2200
2201 VPRecipeBase *clone() override {
2202 return new VPPredInstPHIRecipe(getOperand(0));
2203 }
2204
2205 VP_CLASSOF_IMPL(VPDef::VPPredInstPHISC)
2206
2207 /// Generates phi nodes for live-outs as needed to retain SSA form.
2208 void execute(VPTransformState &State) override;
2209
2210#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2211 /// Print the recipe.
2212 void print(raw_ostream &O, const Twine &Indent,
2213 VPSlotTracker &SlotTracker) const override;
2214#endif
2215
2216 /// Returns true if the recipe uses scalars of operand \p Op.
2217 bool usesScalars(const VPValue *Op) const override {
2219 "Op must be an operand of the recipe");
2220 return true;
2221 }
2222};
2223
2224/// A Recipe for widening load/store operations.
2225/// The recipe uses the following VPValues:
2226/// - For load: Address, optional mask
2227/// - For store: Address, stored value, optional mask
2228/// TODO: We currently execute only per-part unless a specific instance is
2229/// provided.
2231 Instruction &Ingredient;
2232
2233 // Whether the loaded-from / stored-to addresses are consecutive.
2234 bool Consecutive;
2235
2236 // Whether the consecutive loaded/stored addresses are in reverse order.
2237 bool Reverse;
2238
2239 void setMask(VPValue *Mask) {
2240 if (!Mask)
2241 return;
2242 addOperand(Mask);
2243 }
2244
2245 bool isMasked() const {
2246 return isStore() ? getNumOperands() == 3 : getNumOperands() == 2;
2247 }
2248
2249public:
2251 bool Consecutive, bool Reverse)
2252 : VPRecipeBase(VPDef::VPWidenMemoryInstructionSC, {Addr}),
2253 Ingredient(Load), Consecutive(Consecutive), Reverse(Reverse) {
2254 assert((Consecutive || !Reverse) && "Reverse implies consecutive");
2255 new VPValue(this, &Load);
2256 setMask(Mask);
2257 }
2258
2260 VPValue *StoredValue, VPValue *Mask,
2261 bool Consecutive, bool Reverse)
2262 : VPRecipeBase(VPDef::VPWidenMemoryInstructionSC, {Addr, StoredValue}),
2263 Ingredient(Store), Consecutive(Consecutive), Reverse(Reverse) {
2264 assert((Consecutive || !Reverse) && "Reverse implies consecutive");
2265 setMask(Mask);
2266 }
2267
2268 VPRecipeBase *clone() override {
2269 if (isStore())
2271 cast<StoreInst>(Ingredient), getAddr(), getStoredValue(), getMask(),
2272 Consecutive, Reverse);
2273
2275 cast<LoadInst>(Ingredient), getAddr(), getMask(), Consecutive, Reverse);
2276 }
2277
2278 VP_CLASSOF_IMPL(VPDef::VPWidenMemoryInstructionSC)
2279
2280 /// Return the address accessed by this recipe.
2281 VPValue *getAddr() const {
2282 return getOperand(0); // Address is the 1st, mandatory operand.
2283 }
2284
2285 /// Return the mask used by this recipe. Note that a full mask is represented
2286 /// by a nullptr.
2287 VPValue *getMask() const {
2288 // Mask is optional and therefore the last operand.
2289 return isMasked() ? getOperand(getNumOperands() - 1) : nullptr;
2290 }
2291
2292 /// Returns true if this recipe is a store.
2293 bool isStore() const { return isa<StoreInst>(Ingredient); }
2294
2295 /// Return the address accessed by this recipe.
2297 assert(isStore() && "Stored value only available for store instructions");
2298 return getOperand(1); // Stored value is the 2nd, mandatory operand.
2299 }
2300
2301 // Return whether the loaded-from / stored-to addresses are consecutive.
2302 bool isConsecutive() const { return Consecutive; }
2303
2304 // Return whether the consecutive loaded/stored addresses are in reverse
2305 // order.
2306 bool isReverse() const { return Reverse; }
2307
2308 /// Generate the wide load/store.
2309 void execute(VPTransformState &State) override;
2310
2311#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2312 /// Print the recipe.
2313 void print(raw_ostream &O, const Twine &Indent,
2314 VPSlotTracker &SlotTracker) const override;
2315#endif
2316
2317 /// Returns true if the recipe only uses the first lane of operand \p Op.
2318 bool onlyFirstLaneUsed(const VPValue *Op) const override {
2320 "Op must be an operand of the recipe");
2321
2322 // Widened, consecutive memory operations only demand the first lane of
2323 // their address, unless the same operand is also stored. That latter can
2324 // happen with opaque pointers.
2325 return Op == getAddr() && isConsecutive() &&
2326 (!isStore() || Op != getStoredValue());
2327 }
2328
2329 Instruction &getIngredient() const { return Ingredient; }
2330};
2331
2332/// Recipe to expand a SCEV expression.
2334 const SCEV *Expr;
2335 ScalarEvolution &SE;
2336
2337public:
2339 : VPSingleDefRecipe(VPDef::VPExpandSCEVSC, {}), Expr(Expr), SE(SE) {}
2340
2341 ~VPExpandSCEVRecipe() override = default;
2342
2343 VPRecipeBase *clone() override { return new VPExpandSCEVRecipe(Expr, SE); }
2344
2345 VP_CLASSOF_IMPL(VPDef::VPExpandSCEVSC)
2346
2347 /// Generate a canonical vector induction variable of the vector loop, with
2348 void execute(VPTransformState &State) override;
2349
2350#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2351 /// Print the recipe.
2352 void print(raw_ostream &O, const Twine &Indent,
2353 VPSlotTracker &SlotTracker) const override;
2354#endif
2355
2356 const SCEV *getSCEV() const { return Expr; }
2357};
2358
2359/// Canonical scalar induction phi of the vector loop. Starting at the specified
2360/// start value (either 0 or the resume value when vectorizing the epilogue
2361/// loop). VPWidenCanonicalIVRecipe represents the vector version of the
2362/// canonical induction variable.
2364public:
2366 : VPHeaderPHIRecipe(VPDef::VPCanonicalIVPHISC, nullptr, StartV, DL) {}
2367
2368 ~VPCanonicalIVPHIRecipe() override = default;
2369
2370 VPRecipeBase *clone() override {
2371 auto *R = new VPCanonicalIVPHIRecipe(getOperand(0), getDebugLoc());
2372 R->addOperand(getBackedgeValue());
2373 return R;
2374 }
2375
2376 VP_CLASSOF_IMPL(VPDef::VPCanonicalIVPHISC)
2377
2379 return D->getVPDefID() == VPDef::VPCanonicalIVPHISC;
2380 }
2381
2382 /// Generate the canonical scalar induction phi of the vector loop.
2383 void execute(VPTransformState &State) override;
2384
2385#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2386 /// Print the recipe.
2387 void print(raw_ostream &O, const Twine &Indent,
2388 VPSlotTracker &SlotTracker) const override;
2389#endif
2390
2391 /// Returns the scalar type of the induction.
2393 return getStartValue()->getLiveInIRValue()->getType();
2394 }
2395
2396 /// Returns true if the recipe only uses the first lane of operand \p Op.
2397 bool onlyFirstLaneUsed(const VPValue *Op) const override {
2399 "Op must be an operand of the recipe");
2400 return true;
2401 }
2402
2403 /// Returns true if the recipe only uses the first part of operand \p Op.
2404 bool onlyFirstPartUsed(const VPValue *Op) const override {
2406 "Op must be an operand of the recipe");
2407 return true;
2408 }
2409
2410 /// Check if the induction described by \p Kind, /p Start and \p Step is
2411 /// canonical, i.e. has the same start and step (of 1) as the canonical IV.
2413 VPValue *Step) const;
2414};
2415
2416/// A recipe for generating the active lane mask for the vector loop that is
2417/// used to predicate the vector operations.
2418/// TODO: It would be good to use the existing VPWidenPHIRecipe instead and
2419/// remove VPActiveLaneMaskPHIRecipe.
2421public:
2423 : VPHeaderPHIRecipe(VPDef::VPActiveLaneMaskPHISC, nullptr, StartMask,
2424 DL) {}
2425
2426 ~VPActiveLaneMaskPHIRecipe() override = default;
2427
2428 VPRecipeBase *clone() override {
2430 }
2431
2432 VP_CLASSOF_IMPL(VPDef::VPActiveLaneMaskPHISC)
2433
2435 return D->getVPDefID() == VPDef::VPActiveLaneMaskPHISC;
2436 }
2437
2438 /// Generate the active lane mask phi of the vector loop.
2439 void execute(VPTransformState &State) override;
2440
2441#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2442 /// Print the recipe.
2443 void print(raw_ostream &O, const Twine &Indent,
2444 VPSlotTracker &SlotTracker) const override;
2445#endif
2446};
2447
2448/// A Recipe for widening the canonical induction variable of the vector loop.
2450public:
2452 : VPSingleDefRecipe(VPDef::VPWidenCanonicalIVSC, {CanonicalIV}) {}
2453
2454 ~VPWidenCanonicalIVRecipe() override = default;
2455
2456 VPRecipeBase *clone() override {
2457 return new VPWidenCanonicalIVRecipe(
2458 cast<VPCanonicalIVPHIRecipe>(getOperand(0)));
2459 }
2460
2461 VP_CLASSOF_IMPL(VPDef::VPWidenCanonicalIVSC)
2462
2463 /// Generate a canonical vector induction variable of the vector loop, with
2464 /// start = {<Part*VF, Part*VF+1, ..., Part*VF+VF-1> for 0 <= Part < UF}, and
2465 /// step = <VF*UF, VF*UF, ..., VF*UF>.
2466 void execute(VPTransformState &State) override;
2467
2468#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2469 /// Print the recipe.
2470 void print(raw_ostream &O, const Twine &Indent,
2471 VPSlotTracker &SlotTracker) const override;
2472#endif
2473
2474 /// Returns the scalar type of the induction.
2475 const Type *getScalarType() const {
2476 return cast<VPCanonicalIVPHIRecipe>(getOperand(0)->getDefiningRecipe())
2477 ->getScalarType();
2478 }
2479};
2480
2481/// A recipe for converting the canonical IV value to the corresponding value of
2482/// an IV with different start and step values, using Start + CanonicalIV *
2483/// Step.
2485 /// Kind of the induction.
2487 /// If not nullptr, the floating point induction binary operator. Must be set
2488 /// for floating point inductions.
2489 const FPMathOperator *FPBinOp;
2490
2492 const FPMathOperator *FPBinOp, VPValue *Start,
2493 VPCanonicalIVPHIRecipe *CanonicalIV, VPValue *Step)
2494 : VPSingleDefRecipe(VPDef::VPDerivedIVSC, {Start, CanonicalIV, Step}),
2495 Kind(Kind), FPBinOp(FPBinOp) {}
2496
2497public:
2499 VPCanonicalIVPHIRecipe *CanonicalIV, VPValue *Step)
2501 IndDesc.getKind(),
2502 dyn_cast_or_null<FPMathOperator>(IndDesc.getInductionBinOp()),
2503 Start, CanonicalIV, Step) {}
2504
2505 ~VPDerivedIVRecipe() override = default;
2506
2507 VPRecipeBase *clone() override {
2508 return new VPDerivedIVRecipe(Kind, FPBinOp, getStartValue(),
2510 }
2511
2512 VP_CLASSOF_IMPL(VPDef::VPDerivedIVSC)
2513
2514 /// Generate the transformed value of the induction at offset StartValue (1.
2515 /// operand) + IV (2. operand) * StepValue (3, operand).
2516 void execute(VPTransformState &State) override;
2517
2518#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2519 /// Print the recipe.
2520 void print(raw_ostream &O, const Twine &Indent,
2521 VPSlotTracker &SlotTracker) const override;
2522#endif
2523
2525 return getStartValue()->getLiveInIRValue()->getType();
2526 }
2527
2528 VPValue *getStartValue() const { return getOperand(0); }
2530 return cast<VPCanonicalIVPHIRecipe>(getOperand(1));
2531 }
2532 VPValue *getStepValue() const { return getOperand(2); }
2533
2534 /// Returns true if the recipe only uses the first lane of operand \p Op.
2535 bool onlyFirstLaneUsed(const VPValue *Op) const override {
2537 "Op must be an operand of the recipe");
2538 return true;
2539 }
2540};
2541
2542/// A recipe for handling phi nodes of integer and floating-point inductions,
2543/// producing their scalar values.
2545 Instruction::BinaryOps InductionOpcode;
2546
2547public:
2550 : VPRecipeWithIRFlags(VPDef::VPScalarIVStepsSC,
2551 ArrayRef<VPValue *>({IV, Step}), FMFs),
2552 InductionOpcode(Opcode) {}
2553
2555 VPValue *Step)
2557 IV, Step, IndDesc.getInductionOpcode(),
2558 dyn_cast_or_null<FPMathOperator>(IndDesc.getInductionBinOp())
2559 ? IndDesc.getInductionBinOp()->getFastMathFlags()
2560 : FastMathFlags()) {}
2561
2562 ~VPScalarIVStepsRecipe() override = default;
2563
2564 VPRecipeBase *clone() override {
2565 return new VPScalarIVStepsRecipe(
2566 getOperand(0), getOperand(1), InductionOpcode,
2568 }
2569
2570 VP_CLASSOF_IMPL(VPDef::VPScalarIVStepsSC)
2571
2572 /// Generate the scalarized versions of the phi node as needed by their users.
2573 void execute(VPTransformState &State) override;
2574
2575#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2576 /// Print the recipe.
2577 void print(raw_ostream &O, const Twine &Indent,
2578 VPSlotTracker &SlotTracker) const override;
2579#endif
2580
2581 VPValue *getStepValue() const { return getOperand(1); }
2582
2583 /// Returns true if the recipe only uses the first lane of operand \p Op.
2584 bool onlyFirstLaneUsed(const VPValue *Op) const override {
2586 "Op must be an operand of the recipe");
2587 return true;
2588 }
2589};
2590
2591/// VPBasicBlock serves as the leaf of the Hierarchical Control-Flow Graph. It
2592/// holds a sequence of zero or more VPRecipe's each representing a sequence of
2593/// output IR instructions. All PHI-like recipes must come before any non-PHI recipes.
2595public:
2597
2598private:
2599 /// The VPRecipes held in the order of output instructions to generate.
2600 RecipeListTy Recipes;
2601
2602public:
2603 VPBasicBlock(const Twine &Name = "", VPRecipeBase *Recipe = nullptr)
2604 : VPBlockBase(VPBasicBlockSC, Name.str()) {
2605 if (Recipe)
2606 appendRecipe(Recipe);
2607 }
2608
2609 ~VPBasicBlock() override {
2610 while (!Recipes.empty())
2611 Recipes.pop_back();
2612 }
2613
2614 /// Instruction iterators...
2619
2620 //===--------------------------------------------------------------------===//
2621 /// Recipe iterator methods
2622 ///
2623 inline iterator begin() { return Recipes.begin(); }
2624 inline const_iterator begin() const { return Recipes.begin(); }
2625 inline iterator end() { return Recipes.end(); }
2626 inline const_iterator end() const { return Recipes.end(); }
2627
2628 inline reverse_iterator rbegin() { return Recipes.rbegin(); }
2629 inline const_reverse_iterator rbegin() const { return Recipes.rbegin(); }
2630 inline reverse_iterator rend() { return Recipes.rend(); }
2631 inline const_reverse_iterator rend() const { return Recipes.rend(); }
2632
2633 inline size_t size() const { return Recipes.size(); }
2634 inline bool empty() const { return Recipes.empty(); }
2635 inline const VPRecipeBase &front() const { return Recipes.front(); }
2636 inline VPRecipeBase &front() { return Recipes.front(); }
2637 inline const VPRecipeBase &back() const { return Recipes.back(); }
2638 inline VPRecipeBase &back() { return Recipes.back(); }
2639
2640 /// Returns a reference to the list of recipes.
2641 RecipeListTy &getRecipeList() { return Recipes; }
2642
2643 /// Returns a pointer to a member of the recipe list.
2645 return &VPBasicBlock::Recipes;
2646 }
2647
2648 /// Method to support type inquiry through isa, cast, and dyn_cast.
2649 static inline bool classof(const VPBlockBase *V) {
2650 return V->getVPBlockID() == VPBlockBase::VPBasicBlockSC;
2651 }
2652
2653 void insert(VPRecipeBase *Recipe, iterator InsertPt) {
2654 assert(Recipe && "No recipe to append.");
2655 assert(!Recipe->Parent && "Recipe already in VPlan");
2656 Recipe->Parent = this;
2657 Recipes.insert(InsertPt, Recipe);
2658 }
2659
2660 /// Augment the existing recipes of a VPBasicBlock with an additional
2661 /// \p Recipe as the last recipe.
2662 void appendRecipe(VPRecipeBase *Recipe) { insert(Recipe, end()); }
2663
2664 /// The method which generates the output IR instructions that correspond to
2665 /// this VPBasicBlock, thereby "executing" the VPlan.
2666 void execute(VPTransformState *State) override;
2667
2668 /// Return the position of the first non-phi node recipe in the block.
2670
2671 /// Returns an iterator range over the PHI-like recipes in the block.
2673 return make_range(begin(), getFirstNonPhi());
2674 }
2675
2676 void dropAllReferences(VPValue *NewValue) override;
2677
2678 /// Split current block at \p SplitAt by inserting a new block between the
2679 /// current block and its successors and moving all recipes starting at
2680 /// SplitAt to the new block. Returns the new block.
2681 VPBasicBlock *splitAt(iterator SplitAt);
2682
2684
2685#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2686 /// Print this VPBsicBlock to \p O, prefixing all lines with \p Indent. \p
2687 /// SlotTracker is used to print unnamed VPValue's using consequtive numbers.
2688 ///
2689 /// Note that the numbering is applied to the whole VPlan, so printing
2690 /// individual blocks is consistent with the whole VPlan printing.
2691 void print(raw_ostream &O, const Twine &Indent,
2692 VPSlotTracker &SlotTracker) const override;
2693 using VPBlockBase::print; // Get the print(raw_stream &O) version.
2694#endif
2695
2696 /// If the block has multiple successors, return the branch recipe terminating
2697 /// the block. If there are no or only a single successor, return nullptr;
2699 const VPRecipeBase *getTerminator() const;
2700
2701 /// Returns true if the block is exiting it's parent region.
2702 bool isExiting() const;
2703
2704 /// Clone the current block and it's recipes, without updating the operands of
2705 /// the cloned recipes.
2706 VPBasicBlock *clone() override {
2707 auto *NewBlock = new VPBasicBlock(getName());
2708 for (VPRecipeBase &R : *this)
2709 NewBlock->appendRecipe(R.clone());
2710 return NewBlock;
2711 }
2712
2713private:
2714 /// Create an IR BasicBlock to hold the output instructions generated by this
2715 /// VPBasicBlock, and return it. Update the CFGState accordingly.
2716 BasicBlock *createEmptyBasicBlock(VPTransformState::CFGState &CFG);
2717};
2718
2719/// VPRegionBlock represents a collection of VPBasicBlocks and VPRegionBlocks
2720/// which form a Single-Entry-Single-Exiting subgraph of the output IR CFG.
2721/// A VPRegionBlock may indicate that its contents are to be replicated several
2722/// times. This is designed to support predicated scalarization, in which a
2723/// scalar if-then code structure needs to be generated VF * UF times. Having
2724/// this replication indicator helps to keep a single model for multiple
2725/// candidate VF's. The actual replication takes place only once the desired VF
2726/// and UF have been determined.
2728 /// Hold the Single Entry of the SESE region modelled by the VPRegionBlock.
2729 VPBlockBase *Entry;
2730
2731 /// Hold the Single Exiting block of the SESE region modelled by the
2732 /// VPRegionBlock.
2733 VPBlockBase *Exiting;
2734
2735 /// An indicator whether this region is to generate multiple replicated
2736 /// instances of output IR corresponding to its VPBlockBases.
2737 bool IsReplicator;
2738
2739public:
2741 const std::string &Name = "", bool IsReplicator = false)
2742 : VPBlockBase(VPRegionBlockSC, Name), Entry(Entry), Exiting(Exiting),
2743 IsReplicator(IsReplicator) {
2744 assert(Entry->getPredecessors().empty() && "Entry block has predecessors.");
2745 assert(Exiting->getSuccessors().empty() && "Exit block has successors.");
2746 Entry->setParent(this);
2747 Exiting->setParent(this);
2748 }
2749 VPRegionBlock(const std::string &Name = "", bool IsReplicator = false)
2750 : VPBlockBase(VPRegionBlockSC, Name), Entry(nullptr), Exiting(nullptr),
2751 IsReplicator(IsReplicator) {}
2752
2753 ~VPRegionBlock() override {
2754 if (Entry) {
2755 VPValue DummyValue;
2756 Entry->dropAllReferences(&DummyValue);
2757 deleteCFG(Entry);
2758 }
2759 }
2760
2761 /// Method to support type inquiry through isa, cast, and dyn_cast.
2762 static inline bool classof(const VPBlockBase *V) {
2763 return V->getVPBlockID() == VPBlockBase::VPRegionBlockSC;
2764 }
2765
2766 const VPBlockBase *getEntry() const { return Entry; }
2767 VPBlockBase *getEntry() { return Entry; }
2768
2769 /// Set \p EntryBlock as the entry VPBlockBase of this VPRegionBlock. \p
2770 /// EntryBlock must have no predecessors.
2771 void setEntry(VPBlockBase *EntryBlock) {
2772 assert(EntryBlock->getPredecessors().empty() &&
2773 "Entry block cannot have predecessors.");
2774 Entry = EntryBlock;
2775 EntryBlock->setParent(this);
2776 }
2777
2778 const VPBlockBase *getExiting() const { return Exiting; }
2779 VPBlockBase *getExiting() { return Exiting; }
2780
2781 /// Set \p ExitingBlock as the exiting VPBlockBase of this VPRegionBlock. \p
2782 /// ExitingBlock must have no successors.
2783 void setExiting(VPBlockBase *ExitingBlock) {
2784 assert(ExitingBlock->getSuccessors().empty() &&
2785 "Exit block cannot have successors.");
2786 Exiting = ExitingBlock;
2787 ExitingBlock->setParent(this);
2788 }
2789
2790 /// Returns the pre-header VPBasicBlock of the loop region.
2792 assert(!isReplicator() && "should only get pre-header of loop regions");
2794 }
2795
2796 /// An indicator whether this region is to generate multiple replicated
2797 /// instances of output IR corresponding to its VPBlockBases.
2798 bool isReplicator() const { return IsReplicator; }
2799
2800 /// The method which generates the output IR instructions that correspond to
2801 /// this VPRegionBlock, thereby "executing" the VPlan.
2802 void execute(VPTransformState *State) override;
2803
2804 void dropAllReferences(VPValue *NewValue) override;
2805
2806#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2807 /// Print this VPRegionBlock to \p O (recursively), prefixing all lines with
2808 /// \p Indent. \p SlotTracker is used to print unnamed VPValue's using
2809 /// consequtive numbers.
2810 ///
2811 /// Note that the numbering is applied to the whole VPlan, so printing
2812 /// individual regions is consistent with the whole VPlan printing.
2813 void print(raw_ostream &O, const Twine &Indent,
2814 VPSlotTracker &SlotTracker) const override;
2815 using VPBlockBase::print; // Get the print(raw_stream &O) version.
2816#endif
2817
2818 /// Clone all blocks in the single-entry single-exit region of the block and
2819 /// their recipes without updating the operands of the cloned recipes.
2820 VPRegionBlock *clone() override;
2821};
2822
2823/// VPlan models a candidate for vectorization, encoding various decisions take
2824/// to produce efficient output IR, including which branches, basic-blocks and
2825/// output IR instructions to generate, and their cost. VPlan holds a
2826/// Hierarchical-CFG of VPBasicBlocks and VPRegionBlocks rooted at an Entry
2827/// VPBasicBlock.
2828class VPlan {
2829 friend class VPlanPrinter;
2830 friend class VPSlotTracker;
2831
2832 /// Hold the single entry to the Hierarchical CFG of the VPlan, i.e. the
2833 /// preheader of the vector loop.
2834 VPBasicBlock *Entry;
2835
2836 /// VPBasicBlock corresponding to the original preheader. Used to place
2837 /// VPExpandSCEV recipes for expressions used during skeleton creation and the
2838 /// rest of VPlan execution.
2839 VPBasicBlock *Preheader;
2840
2841 /// Holds the VFs applicable to this VPlan.
2843
2844 /// Holds the UFs applicable to this VPlan. If empty, the VPlan is valid for
2845 /// any UF.
2847
2848 /// Holds the name of the VPlan, for printing.
2849 std::string Name;
2850
2851 /// Represents the trip count of the original loop, for folding
2852 /// the tail.
2853 VPValue *TripCount = nullptr;
2854
2855 /// Represents the backedge taken count of the original loop, for folding
2856 /// the tail. It equals TripCount - 1.
2857 VPValue *BackedgeTakenCount = nullptr;
2858
2859 /// Represents the vector trip count.
2860 VPValue VectorTripCount;
2861
2862 /// Represents the loop-invariant VF * UF of the vector loop region.
2863 VPValue VFxUF;
2864
2865 /// Holds a mapping between Values and their corresponding VPValue inside
2866 /// VPlan.
2867 Value2VPValueTy Value2VPValue;
2868
2869 /// Contains all the external definitions created for this VPlan. External
2870 /// definitions are VPValues that hold a pointer to their underlying IR.
2871 SmallVector<VPValue *, 16> VPLiveInsToFree;
2872
2873 /// Indicates whether it is safe use the Value2VPValue mapping or if the
2874 /// mapping cannot be used any longer, because it is stale.
2875 bool Value2VPValueEnabled = true;
2876
2877 /// Values used outside the plan.
2879
2880 /// Mapping from SCEVs to the VPValues representing their expansions.
2881 /// NOTE: This mapping is temporary and will be removed once all users have
2882 /// been modeled in VPlan directly.
2883 DenseMap<const SCEV *, VPValue *> SCEVToExpansion;
2884
2885public:
2886 /// Construct a VPlan with original preheader \p Preheader, trip count \p TC
2887 /// and \p Entry to the plan. At the moment, \p Preheader and \p Entry need to
2888 /// be disconnected, as the bypass blocks between them are not yet modeled in
2889 /// VPlan.
2890 VPlan(VPBasicBlock *Preheader, VPValue *TC, VPBasicBlock *Entry)
2891 : VPlan(Preheader, Entry) {
2892 TripCount = TC;
2893 }
2894
2895 /// Construct a VPlan with original preheader \p Preheader and \p Entry to
2896 /// the plan. At the moment, \p Preheader and \p Entry need to be
2897 /// disconnected, as the bypass blocks between them are not yet modeled in
2898 /// VPlan.
2899 VPlan(VPBasicBlock *Preheader, VPBasicBlock *Entry)
2900 : Entry(Entry), Preheader(Preheader) {
2901 Entry->setPlan(this);
2902 Preheader->setPlan(this);
2903 assert(Preheader->getNumSuccessors() == 0 &&
2904 Preheader->getNumPredecessors() == 0 &&
2905 "preheader must be disconnected");
2906 }
2907
2908 ~VPlan();
2909
2910 /// Create initial VPlan skeleton, having an "entry" VPBasicBlock (wrapping
2911 /// original scalar pre-header) which contains SCEV expansions that need to
2912 /// happen before the CFG is modified; a VPBasicBlock for the vector
2913 /// pre-header, followed by a region for the vector loop, followed by the
2914 /// middle VPBasicBlock.
2915 static VPlanPtr createInitialVPlan(const SCEV *TripCount,
2916 ScalarEvolution &PSE);
2917
2918 /// Prepare the plan for execution, setting up the required live-in values.
2919 void prepareToExecute(Value *TripCount, Value *VectorTripCount,
2920 Value *CanonicalIVStartValue, VPTransformState &State);
2921
2922 /// Generate the IR code for this VPlan.
2923 void execute(VPTransformState *State);
2924
2925 VPBasicBlock *getEntry() { return Entry; }
2926 const VPBasicBlock *getEntry() const { return Entry; }
2927
2928 /// The trip count of the original loop.
2930 assert(TripCount && "trip count needs to be set before accessing it");
2931 return TripCount;
2932 }
2933
2934 /// Resets the trip count for the VPlan. The caller must make sure all uses of
2935 /// the original trip count have been replaced.
2936 void resetTripCount(VPValue *NewTripCount) {
2937 assert(TripCount && NewTripCount && TripCount->getNumUsers() == 0 &&
2938 "TripCount always must be set");
2939 TripCount = NewTripCount;
2940 }
2941
2942 /// The backedge taken count of the original loop.
2944 if (!BackedgeTakenCount)
2945 BackedgeTakenCount = new VPValue();
2946 return BackedgeTakenCount;
2947 }
2948
2949 /// The vector trip count.
2950 VPValue &getVectorTripCount() { return VectorTripCount; }
2951
2952 /// Returns VF * UF of the vector loop region.
2953 VPValue &getVFxUF() { return VFxUF; }
2954
2955 /// Mark the plan to indicate that using Value2VPValue is not safe any
2956 /// longer, because it may be stale.
2957 void disableValue2VPValue() { Value2VPValueEnabled = false; }
2958
2959 void addVF(ElementCount VF) { VFs.insert(VF); }
2960
2962 assert(hasVF(VF) && "Cannot set VF not already in plan");
2963 VFs.clear();
2964 VFs.insert(VF);
2965 }
2966
2967 bool hasVF(ElementCount VF) { return VFs.count(VF); }
2969 return any_of(VFs, [](ElementCount VF) { return VF.isScalable(); });
2970 }
2971
2972 bool hasScalarVFOnly() const { return VFs.size() == 1 && VFs[0].isScalar(); }
2973
2974 bool hasUF(unsigned UF) const { return UFs.empty() || UFs.contains(UF); }
2975
2976 void setUF(unsigned UF) {
2977 assert(hasUF(UF) && "Cannot set the UF not already in plan");
2978 UFs.clear();
2979 UFs.insert(UF);
2980 }
2981
2982 /// Return a string with the name of the plan and the applicable VFs and UFs.
2983 std::string getName() const;
2984
2985 void setName(const Twine &newName) { Name = newName.str(); }
2986
2987 void addVPValue(Value *V, VPValue *VPV) {
2988 assert((Value2VPValueEnabled || VPV->isLiveIn()) &&
2989 "Value2VPValue mapping may be out of date!");
2990 assert(V && "Trying to add a null Value to VPlan");
2991 assert(!Value2VPValue.count(V) && "Value already exists in VPlan");
2992 Value2VPValue[V] = VPV;
2993 }
2994
2995 /// Returns the VPValue for \p V.
2997 assert(V && "Trying to get the VPValue of a null Value");
2998 assert(Value2VPValue.count(V) && "Value does not exist in VPlan");
2999 assert((Value2VPValueEnabled || Value2VPValue[V]->isLiveIn()) &&
3000 "Value2VPValue mapping may be out of date!");
3001 return Value2VPValue[V];
3002 }
3003
3004 /// Gets the VPValue for \p V or adds a new live-in (if none exists yet) for
3005 /// \p V.
3007 assert(V && "Trying to get or add the VPValue of a null Value");
3008 if (!Value2VPValue.count(V)) {
3009 VPValue *VPV = new VPValue(V);
3010 VPLiveInsToFree.push_back(VPV);
3011 addVPValue(V, VPV);
3012 }
3013
3014 return getVPValue(V);
3015 }
3016
3017#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
3018 /// Print the live-ins of this VPlan to \p O.
3019 void printLiveIns(raw_ostream &O) const;
3020
3021 /// Print this VPlan to \p O.
3022 void print(raw_ostream &O) const;
3023
3024 /// Print this VPlan in DOT format to \p O.
3025 void printDOT(raw_ostream &O) const;
3026
3027 /// Dump the plan to stderr (for debugging).
3028 LLVM_DUMP_METHOD void dump() const;
3029#endif
3030
3031 /// Returns a range mapping the values the range \p Operands to their
3032 /// corresponding VPValues.
3033 iterator_range<mapped_iterator<Use *, std::function<VPValue *(Value *)>>>
3035 std::function<VPValue *(Value *)> Fn = [this](Value *Op) {
3036 return getVPValueOrAddLiveIn(Op);
3037 };
3038 return map_range(Operands, Fn);
3039 }
3040
3041 /// Returns the VPRegionBlock of the vector loop.
3043 return cast<VPRegionBlock>(getEntry()->getSingleSuccessor());
3044 }
3046 return cast<VPRegionBlock>(getEntry()->getSingleSuccessor());
3047 }
3048
3049 /// Returns the canonical induction recipe of the vector loop.
3052 if (EntryVPBB->empty()) {
3053 // VPlan native path.
3054 EntryVPBB = cast<VPBasicBlock>(EntryVPBB->getSingleSuccessor());
3055 }
3056 return cast<VPCanonicalIVPHIRecipe>(&*EntryVPBB->begin());
3057 }
3058
3059 void addLiveOut(PHINode *PN, VPValue *V);
3060
3062 delete LiveOuts[PN];
3063 LiveOuts.erase(PN);
3064 }
3065
3067 return LiveOuts;
3068 }
3069
3070 VPValue *getSCEVExpansion(const SCEV *S) const {
3071 return SCEVToExpansion.lookup(S);
3072 }
3073
3074 void addSCEVExpansion(const SCEV *S, VPValue *V) {
3075 assert(!SCEVToExpansion.contains(S) && "SCEV already expanded");
3076 SCEVToExpansion[S] = V;
3077 }
3078
3079 /// \return The block corresponding to the original preheader.
3080 VPBasicBlock *getPreheader() { return Preheader; }
3081 const VPBasicBlock *getPreheader() const { return Preheader; }
3082
3083 /// Clone the current VPlan, update all VPValues of the new VPlan and cloned
3084 /// recipes to refer to the clones, and return it.
3085 VPlan *duplicate();
3086
3087private:
3088 /// Add to the given dominator tree the header block and every new basic block
3089 /// that was created between it and the latch block, inclusive.
3090 static void updateDominatorTree(DominatorTree *DT, BasicBlock *LoopLatchBB,
3091 BasicBlock *LoopPreHeaderBB,
3092 BasicBlock *LoopExitBB);
3093};
3094
3095#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
3096/// VPlanPrinter prints a given VPlan to a given output stream. The printing is
3097/// indented and follows the dot format.
3099 raw_ostream &OS;
3100 const VPlan &Plan;
3101 unsigned Depth = 0;
3102 unsigned TabWidth = 2;
3103 std::string Indent;
3104 unsigned BID = 0;
3106
3108
3109 /// Handle indentation.
3110 void bumpIndent(int b) { Indent = std::string((Depth += b) * TabWidth, ' '); }
3111
3112 /// Print a given \p Block of the Plan.
3113 void dumpBlock(const VPBlockBase *Block);
3114
3115 /// Print the information related to the CFG edges going out of a given
3116 /// \p Block, followed by printing the successor blocks themselves.
3117 void dumpEdges(const VPBlockBase *Block);
3118
3119 /// Print a given \p BasicBlock, including its VPRecipes, followed by printing
3120 /// its successor blocks.
3121 void dumpBasicBlock(const VPBasicBlock *BasicBlock);
3122
3123 /// Print a given \p Region of the Plan.
3124 void dumpRegion(const VPRegionBlock *Region);
3125
3126 unsigned getOrCreateBID(const VPBlockBase *Block) {
3127 return BlockID.count(Block) ? BlockID[Block] : BlockID[Block] = BID++;
3128 }
3129
3130 Twine getOrCreateName(const VPBlockBase *Block);
3131
3132 Twine getUID(const VPBlockBase *Block);
3133
3134 /// Print the information related to a CFG edge between two VPBlockBases.
3135 void drawEdge(const VPBlockBase *From, const VPBlockBase *To, bool Hidden,
3136 const Twine &Label);
3137
3138public:
3140 : OS(O), Plan(P), SlotTracker(&P) {}
3141
3142 LLVM_DUMP_METHOD void dump();
3143};
3144
3146 const Value *V;
3147
3148 VPlanIngredient(const Value *V) : V(V) {}
3149
3150 void print(raw_ostream &O) const;
3151};
3152
3154 I.print(OS);
3155 return OS;
3156}
3157
3159 Plan.print(OS);
3160 return OS;
3161}
3162#endif
3163
3164//===----------------------------------------------------------------------===//
3165// VPlan Utilities
3166//===----------------------------------------------------------------------===//
3167
3168/// Class that provides utilities for VPBlockBases in VPlan.
3170public:
3171 VPBlockUtils() = delete;
3172
3173 /// Insert disconnected VPBlockBase \p NewBlock after \p BlockPtr. Add \p
3174 /// NewBlock as successor of \p BlockPtr and \p BlockPtr as predecessor of \p
3175 /// NewBlock, and propagate \p BlockPtr parent to \p NewBlock. \p BlockPtr's
3176 /// successors are moved from \p BlockPtr to \p NewBlock. \p NewBlock must
3177 /// have neither successors nor predecessors.
3178 static void insertBlockAfter(VPBlockBase *NewBlock, VPBlockBase *BlockPtr) {
3179 assert(NewBlock->getSuccessors().empty() &&
3180 NewBlock->getPredecessors().empty() &&
3181 "Can't insert new block with predecessors or successors.");
3182 NewBlock->setParent(BlockPtr->getParent());
3183 SmallVector<VPBlockBase *> Succs(BlockPtr->successors());
3184 for (VPBlockBase *Succ : Succs) {
3185 disconnectBlocks(BlockPtr, Succ);
3186 connectBlocks(NewBlock, Succ);
3187 }
3188 connectBlocks(BlockPtr, NewBlock);
3189 }
3190
3191 /// Insert disconnected VPBlockBases \p IfTrue and \p IfFalse after \p
3192 /// BlockPtr. Add \p IfTrue and \p IfFalse as succesors of \p BlockPtr and \p
3193 /// BlockPtr as predecessor of \p IfTrue and \p IfFalse. Propagate \p BlockPtr
3194 /// parent to \p IfTrue and \p IfFalse. \p BlockPtr must have no successors
3195 /// and \p IfTrue and \p IfFalse must have neither successors nor
3196 /// predecessors.
3197 static void insertTwoBlocksAfter(VPBlockBase *IfTrue, VPBlockBase *IfFalse,
3198 VPBlockBase *BlockPtr) {
3199 assert(IfTrue->getSuccessors().empty() &&
3200 "Can't insert IfTrue with successors.");
3201 assert(IfFalse->getSuccessors().empty() &&
3202 "Can't insert IfFalse with successors.");
3203 BlockPtr->setTwoSuccessors(IfTrue, IfFalse);
3204 IfTrue->setPredecessors({BlockPtr});
3205 IfFalse->setPredecessors({BlockPtr});
3206 IfTrue->setParent(BlockPtr->getParent());
3207 IfFalse->setParent(BlockPtr->getParent());
3208 }
3209
3210 /// Connect VPBlockBases \p From and \p To bi-directionally. Append \p To to
3211 /// the successors of \p From and \p From to the predecessors of \p To. Both
3212 /// VPBlockBases must have the same parent, which can be null. Both
3213 /// VPBlockBases can be already connected to other VPBlockBases.
3215 assert((From->getParent() == To->getParent()) &&
3216 "Can't connect two block with different parents");
3217 assert(From->getNumSuccessors() < 2 &&
3218 "Blocks can't have more than two successors.");
3219 From->appendSuccessor(To);
3220 To->appendPredecessor(From);
3221 }
3222
3223 /// Disconnect VPBlockBases \p From and \p To bi-directionally. Remove \p To
3224 /// from the successors of \p From and \p From from the predecessors of \p To.
3226 assert(To && "Successor to disconnect is null.");
3227 From->removeSuccessor(To);
3228 To->removePredecessor(From);
3229 }
3230
3231 /// Return an iterator range over \p Range which only includes \p BlockTy
3232 /// blocks. The accesses are casted to \p BlockTy.
3233 template <typename BlockTy, typename T>
3234 static auto blocksOnly(const T &Range) {
3235 // Create BaseTy with correct const-ness based on BlockTy.
3236 using BaseTy = std::conditional_t<std::is_const<BlockTy>::value,
3237 const VPBlockBase, VPBlockBase>;
3238
3239 // We need to first create an iterator range over (const) BlocktTy & instead
3240 // of (const) BlockTy * for filter_range to work properly.
3241 auto Mapped =
3242 map_range(Range, [](BaseTy *Block) -> BaseTy & { return *Block; });
3244 Mapped, [](BaseTy &Block) { return isa<BlockTy>(&Block); });
3245 return map_range(Filter, [](BaseTy &Block) -> BlockTy * {
3246 return cast<BlockTy>(&Block);
3247 });
3248 }
3249};
3250
3253 InterleaveGroupMap;
3254
3255 /// Type for mapping of instruction based interleave groups to VPInstruction
3256 /// interleave groups
3259
3260 /// Recursively \p Region and populate VPlan based interleave groups based on
3261 /// \p IAI.
3262 void visitRegion(VPRegionBlock *Region, Old2NewTy &Old2New,
3264 /// Recursively traverse \p Block and populate VPlan based interleave groups
3265 /// based on \p IAI.
3266 void visitBlock(VPBlockBase *Block, Old2NewTy &Old2New,
3268
3269public:
3271
3274 // Avoid releasing a pointer twice.
3275 for (auto &I : InterleaveGroupMap)
3276 DelSet.insert(I.second);
3277 for (auto *Ptr : DelSet)
3278 delete Ptr;
3279 }
3280
3281 /// Get the interleave group that \p Instr belongs to.
3282 ///
3283 /// \returns nullptr if doesn't have such group.
3286 return InterleaveGroupMap.lookup(Instr);
3287 }
3288};
3289
3290/// Class that maps (parts of) an existing VPlan to trees of combined
3291/// VPInstructions.
3293 enum class OpMode { Failed, Load, Opcode };
3294
3295 /// A DenseMapInfo implementation for using SmallVector<VPValue *, 4> as
3296 /// DenseMap keys.
3297 struct BundleDenseMapInfo {
3298 static SmallVector<VPValue *, 4> getEmptyKey() {
3299 return {reinterpret_cast<VPValue *>(-1)};
3300 }
3301
3302 static SmallVector<VPValue *, 4> getTombstoneKey() {
3303 return {reinterpret_cast<VPValue *>(-2)};
3304 }
3305
3306 static unsigned getHashValue(const SmallVector<VPValue *, 4> &V) {
3307 return static_cast<unsigned>(hash_combine_range(V.begin(), V.end()));
3308 }
3309
3310 static bool isEqual(const SmallVector<VPValue *, 4> &LHS,
3312 return LHS == RHS;
3313 }
3314 };
3315
3316 /// Mapping of values in the original VPlan to a combined VPInstruction.
3318 BundleToCombined;
3319
3321
3322 /// Basic block to operate on. For now, only instructions in a single BB are
3323 /// considered.
3324 const VPBasicBlock &BB;
3325
3326 /// Indicates whether we managed to combine all visited instructions or not.
3327 bool CompletelySLP = true;
3328
3329 /// Width of the widest combined bundle in bits.
3330 unsigned WidestBundleBits = 0;
3331
3332 using MultiNodeOpTy =
3333 typename std::pair<VPInstruction *, SmallVector<VPValue *, 4>>;
3334
3335 // Input operand bundles for the current multi node. Each multi node operand
3336 // bundle contains values not matching the multi node's opcode. They will
3337 // be reordered in reorderMultiNodeOps, once we completed building a
3338 // multi node.
3339 SmallVector<MultiNodeOpTy, 4> MultiNodeOps;
3340
3341 /// Indicates whether we are building a multi node currently.
3342 bool MultiNodeActive = false;
3343
3344 /// Check if we can vectorize Operands together.
3345 bool areVectorizable(ArrayRef<VPValue *> Operands) const;
3346
3347 /// Add combined instruction \p New for the bundle \p Operands.
3348 void addCombined(ArrayRef<VPValue *> Operands, VPInstruction *New);
3349
3350 /// Indicate we hit a bundle we failed to combine. Returns nullptr for now.
3351 VPInstruction *markFailed();
3352
3353 /// Reorder operands in the multi node to maximize sequential memory access
3354 /// and commutative operations.
3355 SmallVector<MultiNodeOpTy, 4> reorderMultiNodeOps();
3356
3357 /// Choose the best candidate to use for the lane after \p Last. The set of
3358 /// candidates to choose from are values with an opcode matching \p Last's
3359 /// or loads consecutive to \p Last.
3360 std::pair<OpMode, VPValue *> getBest(OpMode Mode, VPValue *Last,
3361 SmallPtrSetImpl<VPValue *> &Candidates,
3363
3364#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
3365 /// Print bundle \p Values to dbgs().
3366 void dumpBundle(ArrayRef<VPValue *> Values);
3367#endif
3368
3369public:
3370 VPlanSlp(VPInterleavedAccessInfo &IAI, VPBasicBlock &BB) : IAI(IAI), BB(BB) {}
3371
3372 ~VPlanSlp() = default;
3373
3374 /// Tries to build an SLP tree rooted at \p Operands and returns a
3375 /// VPInstruction combining \p Operands, if they can be combined.
3377
3378 /// Return the width of the widest combined bundle in bits.
3379 unsigned getWidestBundleBits() const { return WidestBundleBits; }
3380
3381 /// Return true if all visited instruction can be combined.
3382 bool isCompletelySLP() const { return CompletelySLP; }
3383};
3384
3385namespace vputils {
3386
3387/// Returns true if only the first lane of \p Def is used.
3388bool onlyFirstLaneUsed(const VPValue *Def);
3389
3390/// Returns true if only the first part of \p Def is used.
3391bool onlyFirstPartUsed(const VPValue *Def);
3392
3393/// Get or create a VPValue that corresponds to the expansion of \p Expr. If \p
3394/// Expr is a SCEVConstant or SCEVUnknown, return a VPValue wrapping the live-in
3395/// value. Otherwise return a VPExpandSCEVRecipe to expand \p Expr. If \p Plan's
3396/// pre-header already contains a recipe expanding \p Expr, return it. If not,
3397/// create a new one.
3399 ScalarEvolution &SE);
3400
3401/// Returns true if \p VPV is uniform after vectorization.
3403 // A value defined outside the vector region must be uniform after
3404 // vectorization inside a vector region.
3406 return true;
3407 VPRecipeBase *Def = VPV->getDefiningRecipe();
3408 assert(Def && "Must have definition for value defined inside vector region");
3409 if (auto Rep = dyn_cast<VPReplicateRecipe>(Def))
3410 return Rep->isUniform();
3411 if (auto *GEP = dyn_cast<VPWidenGEPRecipe>(Def))
3412 return all_of(GEP->operands(), isUniformAfterVectorization);
3413 if (auto *VPI = dyn_cast<VPInstruction>(Def))
3414 return VPI->getOpcode() == VPInstruction::ComputeReductionResult;
3415 return false;
3416}
3417} // end namespace vputils
3418
3419} // end namespace llvm
3420
3421#endif // LLVM_TRANSFORMS_VECTORIZE_VPLAN_H
aarch64 promote const
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static MCDisassembler::DecodeStatus addOperand(MCInst &Inst, const MCOperand &Opnd)
always inline
BlockVerifier::State From
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
RelocType Type
Definition: COFFYAML.cpp:391
#define LLVM_DUMP_METHOD
Mark debug helper function definitions like dump() that should not be stripped from debug builds.
Definition: Compiler.h:529
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
This file defines the DenseMap class.
uint64_t Addr
std::string Name
std::optional< std::vector< StOtherPiece > > Other
Definition: ELFYAML.cpp:1290
Flatten the CFG
Hexagon Common GEP
std::pair< BasicBlock *, unsigned > BlockTy
A pair of (basic block, score).
#define I(x, y, z)
Definition: MD5.cpp:58
mir Rename Register Operands
This file implements a map that provides insertion order iteration.
#define P(N)
static cl::opt< RegAllocEvictionAdvisorAnalysis::AdvisorMode > Mode("regalloc-enable-advisor", cl::Hidden, cl::init(RegAllocEvictionAdvisorAnalysis::AdvisorMode::Default), cl::desc("Enable regalloc advisor mode"), cl::values(clEnumValN(RegAllocEvictionAdvisorAnalysis::AdvisorMode::Default, "default", "Default"), clEnumValN(RegAllocEvictionAdvisorAnalysis::AdvisorMode::Release, "release", "precompiled"), clEnumValN(RegAllocEvictionAdvisorAnalysis::AdvisorMode::Development, "development", "for training")))
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
raw_pwrite_stream & OS
This file implements the SmallBitVector class.
This file defines the SmallPtrSet class.
This file defines the SmallVector class.
This file contains the declarations of the entities induced by Vectorization Plans,...
#define VP_CLASSOF_IMPL(VPDefID)
Definition: VPlan.h:808
Value * RHS
Value * LHS
static const uint32_t IV[8]
Definition: blake3_impl.h:78
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
ArrayRef< T > slice(size_t N, size_t M) const
slice(n, m) - Chop off the first N elements of the array, and keep M elements in the array.
Definition: ArrayRef.h:195
LLVM Basic Block Representation.
Definition: BasicBlock.h:60
This class represents a function call, abstracting a target machine's calling convention.
This is the base class for all instructions that perform data casts.
Definition: InstrTypes.h:579
Instruction::CastOps getOpcode() const
Return the opcode of this CastInst.
Definition: InstrTypes.h:908
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition: InstrTypes.h:965
This class represents an Operation in the Expression.
A debug info location.
Definition: DebugLoc.h:33
ValueT lookup(const_arg_type_t< KeyT > Val) const
lookup - Return the entry for the specified key, or a default constructed value if no such entry exis...
Definition: DenseMap.h:202
size_type count(const_arg_type_t< KeyT > Val) const
Return 1 if the specified key is in the map, 0 otherwise.
Definition: DenseMap.h:151
bool contains(const_arg_type_t< KeyT > Val) const
Return true if the specified key is in the map, false otherwise.
Definition: DenseMap.h:145
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
Definition: Dominators.h:162
constexpr bool isScalar() const
Exactly one element.
Definition: TypeSize.h:307
Utility class for floating point operations which can have information about relaxed accuracy require...
Definition: Operator.h:200
Convenience struct for specifying and reasoning about fast-math flags.
Definition: FMF.h:20
an instruction for type-safe pointer arithmetic to access elements of arrays and structs
Definition: Instructions.h:973
Common base class shared among various IRBuilders.
Definition: IRBuilder.h:94
A struct for saving information about induction variables.
InductionKind
This enum represents the kinds of inductions that we support.
InnerLoopVectorizer vectorizes loops which contain only one basic block to a specified vectorization ...
The group of interleaved loads/stores sharing the same stride and close to each other.
Definition: VectorUtils.h:444
uint32_t getFactor() const
Definition: VectorUtils.h:460
InstTy * getMember(uint32_t Index) const
Get the member with the given index Index.
Definition: VectorUtils.h:514
Drive the analysis of interleaved memory accesses in the loop.
Definition: VectorUtils.h:586
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:67
An instruction for reading from memory.
Definition: Instructions.h:184
This class emits a version of the loop where run-time checks ensure that may-alias pointers can't ove...
Represents a single loop in the control flow graph.
Definition: LoopInfo.h:44
This class implements a map that also provides access to all stored values in a deterministic order.
Definition: MapVector.h:36
VectorType::iterator erase(typename VectorType::iterator Iterator)
Remove the element given by Iterator.
Definition: MapVector.h:193
The RecurrenceDescriptor is used to identify recurrences variables in a loop.
Definition: IVDescriptors.h:71
This class represents an analyzed expression in the program.
The main scalar evolution driver.
This class represents the LLVM 'select' instruction.
size_type size() const
Determine the number of elements in the SetVector.
Definition: SetVector.h:98
void clear()
Completely clear the SetVector.
Definition: SetVector.h:273
size_type count(const key_type &key) const
Count the number of elements of a given key in the SetVector.
Definition: SetVector.h:264
bool empty() const
Determine if the SetVector is empty or not.
Definition: SetVector.h:93
bool insert(const value_type &X)
Insert a new element into the SetVector.
Definition: SetVector.h:162
bool contains(const key_type &key) const
Check if the SetVector contains the given key.
Definition: SetVector.h:254
This class provides computation of slot numbers for LLVM Assembly writing.
Definition: AsmWriter.cpp:690
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
Definition: SmallPtrSet.h:321
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:342
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
Definition: SmallPtrSet.h:427
A SetVector that performs no allocations if smaller than a certain size.
Definition: SetVector.h:370
bool empty() const
Definition: SmallVector.h:94
size_t size() const
Definition: SmallVector.h:91
iterator erase(const_iterator CI)
Definition: SmallVector.h:750
void push_back(const T &Elt)
Definition: SmallVector.h:426
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1209
An instruction for storing to memory.
Definition: Instructions.h:317
This class represents a truncation of integer types.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:81
std::string str() const
Return the twine contents as a std::string.
Definition: Twine.cpp:17
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
Definition: Type.h:348
A Use represents the edge between a Value definition and its users.
Definition: Use.h:43
Iterator to iterate over vectorization factors in a VFRange.
Definition: VPlan.h:111
ElementCount operator*() const
Definition: VPlan.h:119
iterator & operator++()
Definition: VPlan.h:121
iterator(ElementCount VF)
Definition: VPlan.h:115
bool operator==(const iterator &Other) const
Definition: VPlan.h:117
A recipe for generating the active lane mask for the vector loop that is used to predicate the vector...
Definition: VPlan.h:2420
void execute(VPTransformState &State) override
Generate the active lane mask phi of the vector loop.
VPRecipeBase * clone() override
Clone the current recipe.
Definition: VPlan.h:2428
static bool classof(const VPHeaderPHIRecipe *D)
Definition: VPlan.h:2434
VPActiveLaneMaskPHIRecipe(VPValue *StartMask, DebugLoc DL)
Definition: VPlan.h:2422
~VPActiveLaneMaskPHIRecipe() override=default
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPBasicBlock serves as the leaf of the Hierarchical Control-Flow Graph.
Definition: VPlan.h:2594
RecipeListTy::const_iterator const_iterator
Definition: VPlan.h:2616
void appendRecipe(VPRecipeBase *Recipe)
Augment the existing recipes of a VPBasicBlock with an additional Recipe as the last recipe.
Definition: VPlan.h:2662
VPBasicBlock * clone() override
Clone the current block and it's recipes, without updating the operands of the cloned recipes.
Definition: VPlan.h:2706
RecipeListTy::const_reverse_iterator const_reverse_iterator
Definition: VPlan.h:2618
RecipeListTy::iterator iterator
Instruction iterators...
Definition: VPlan.h:2615
void execute(VPTransformState *State) override
The method which generates the output IR instructions that correspond to this VPBasicBlock,...
Definition: VPlan.cpp:452
RecipeListTy & getRecipeList()
Returns a reference to the list of recipes.
Definition: VPlan.h:2641
iterator end()
Definition: VPlan.h:2625
VPBasicBlock(const Twine &Name="", VPRecipeBase *Recipe=nullptr)
Definition: VPlan.h:2603
iterator begin()
Recipe iterator methods.
Definition: VPlan.h:2623
RecipeListTy::reverse_iterator reverse_iterator
Definition: VPlan.h:2617
iterator_range< iterator > phis()
Returns an iterator range over the PHI-like recipes in the block.
Definition: VPlan.h:2672
iterator getFirstNonPhi()
Return the position of the first non-phi node recipe in the block.
Definition: VPlan.cpp:210
~VPBasicBlock() override
Definition: VPlan.h:2609
VPRegionBlock * getEnclosingLoopRegion()
Definition: VPlan.cpp:555
void dropAllReferences(VPValue *NewValue) override
Replace all operands of VPUsers in the block with NewValue and also replaces all uses of VPValues def...
Definition: VPlan.cpp:520
const_reverse_iterator rbegin() const
Definition: VPlan.h:2629
reverse_iterator rend()
Definition: VPlan.h:2630
VPBasicBlock * splitAt(iterator SplitAt)
Split current block at SplitAt by inserting a new block between the current block and its successors ...
Definition: VPlan.cpp:530
VPRecipeBase & back()
Definition: VPlan.h:2638
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print this VPBsicBlock to O, prefixing all lines with Indent.
Definition: VPlan.cpp:622
const VPRecipeBase & front() const
Definition: VPlan.h:2635
const_iterator begin() const
Definition: VPlan.h:2624
VPRecipeBase & front()
Definition: VPlan.h:2636
bool isExiting() const
Returns true if the block is exiting it's parent region.
Definition: VPlan.cpp:605
VPRecipeBase * getTerminator()
If the block has multiple successors, return the branch recipe terminating the block.
Definition: VPlan.cpp:593
const VPRecipeBase & back() const
Definition: VPlan.h:2637
void insert(VPRecipeBase *Recipe, iterator InsertPt)
Definition: VPlan.h:2653
bool empty() const
Definition: VPlan.h:2634
const_iterator end() const
Definition: VPlan.h:2626
static bool classof(const VPBlockBase *V)
Method to support type inquiry through isa, cast, and dyn_cast.
Definition: VPlan.h:2649
static RecipeListTy VPBasicBlock::* getSublistAccess(VPRecipeBase *)
Returns a pointer to a member of the recipe list.
Definition: VPlan.h:2644
reverse_iterator rbegin()
Definition: VPlan.h:2628
size_t size() const
Definition: VPlan.h:2633
const_reverse_iterator rend() const
Definition: VPlan.h:2631
A recipe for vectorizing a phi-node as a sequence of mask-based select instructions.
Definition: VPlan.h:1882
VPBlendRecipe(PHINode *Phi, ArrayRef< VPValue * > Operands)
The blend operation is a User of the incoming values and of their respective masks,...
Definition: VPlan.h:1887
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
bool onlyFirstLaneUsed(const VPValue *Op) const override
Returns true if the recipe only uses the first lane of operand Op.
Definition: VPlan.h:1922
VPValue * getIncomingValue(unsigned Idx) const
Return incoming value number Idx.
Definition: VPlan.h:1907
VPRecipeBase * clone() override
Clone the current recipe.
Definition: VPlan.h:1895
VPValue * getMask(unsigned Idx) const
Return mask number Idx.
Definition: VPlan.h:1910
unsigned getNumIncomingValues() const
Return the number of incoming values, taking into account that a single incoming value has no mask.
Definition: VPlan.h:1904
void execute(VPTransformState &State) override
Generate the phi/select nodes.
VPBlockBase is the building block of the Hierarchical Control-Flow Graph.
Definition: VPlan.h:421
VPRegionBlock * getParent()
Definition: VPlan.h:493
VPBlocksTy & getPredecessors()
Definition: VPlan.h:524
const VPBasicBlock * getExitingBasicBlock() const
Definition: VPlan.cpp:175
LLVM_DUMP_METHOD void dump() const
Dump this VPBlockBase to dbgs().
Definition: VPlan.h:662
void setName(const Twine &newName)
Definition: VPlan.h:486
size_t getNumSuccessors() const
Definition: VPlan.h:538
iterator_range< VPBlockBase ** > successors()
Definition: VPlan.h:521
virtual void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const =0
Print plain-text dump of this VPBlockBase to O, prefixing all lines with Indent.
void printSuccessors(raw_ostream &O, const Twine &Indent) const
Print the successors of this block to O, prefixing all lines with Indent.
Definition: VPlan.cpp:610
bool isLegalToHoistInto()
Return true if it is legal to hoist instructions into this block.
Definition: VPlan.h:627
virtual ~VPBlockBase()=default
void print(raw_ostream &O) const
Print plain-text dump of this VPlan to O.
Definition: VPlan.h:652
const VPBlocksTy & getHierarchicalPredecessors()
Definition: VPlan.h:574
size_t getNumPredecessors() const
Definition: VPlan.h:539
void setPredecessors(ArrayRef< VPBlockBase * > NewPreds)
Set each VPBasicBlock in NewPreds as predecessor of this VPBlockBase.
Definition: VPlan.h:607
VPBlockBase * getEnclosingBlockWithPredecessors()
Definition: VPlan.cpp:197
const VPBlocksTy & getPredecessors() const
Definition: VPlan.h:523
virtual VPBlockBase * clone()=0
Clone the current block and it's recipes without updating the operands of the cloned recipes,...
static void deleteCFG(VPBlockBase *Entry)
Delete all blocks reachable from a given VPBlockBase, inclusive.
Definition: VPlan.cpp:205
VPlan * getPlan()
Definition: VPlan.cpp:148
void setPlan(VPlan *ParentPlan)
Sets the pointer of the plan containing the block.
Definition: VPlan.cpp:167
const VPRegionBlock * getParent() const
Definition: VPlan.h:494
void printAsOperand(raw_ostream &OS, bool PrintType) const
Definition: VPlan.h:638
const std::string & getName() const
Definition: VPlan.h:484
void clearSuccessors()
Remove all the successors of this block.
Definition: VPlan.h:617
VPBlockBase * getSingleHierarchicalSuccessor()
Definition: VPlan.h:564
void setTwoSuccessors(VPBlockBase *IfTrue, VPBlockBase *IfFalse)
Set two given VPBlockBases IfTrue and IfFalse to be the two successors of this VPBlockBase.
Definition: VPlan.h:598
VPBlockBase * getSinglePredecessor() const
Definition: VPlan.h:534
virtual void execute(VPTransformState *State)=0
The method which generates the output IR that correspond to this VPBlockBase, thereby "executing" the...
const VPBlocksTy & getHierarchicalSuccessors()
Definition: VPlan.h:558
void clearPredecessors()
Remove all the predecessor of this block.
Definition: VPlan.h:614
enum { VPBasicBlockSC, VPRegionBlockSC } VPBlockTy
An enumeration for keeping track of the concrete subclass of VPBlockBase that are actually instantiat...
Definition: VPlan.h:478
unsigned getVPBlockID() const
Definition: VPlan.h:491
VPBlockBase(const unsigned char SC, const std::string &N)
Definition: VPlan.h:470
VPBlocksTy & getSuccessors()
Definition: VPlan.h:519
VPBlockBase * getEnclosingBlockWithSuccessors()
An Enclosing Block of a block B is any block containing B, including B itself.
Definition: VPlan.cpp:189
const VPBasicBlock * getEntryBasicBlock() const
Definition: VPlan.cpp:153
void setOneSuccessor(VPBlockBase *Successor)
Set a given VPBlockBase Successor as the single successor of this VPBlockBase.
Definition: VPlan.h:587
void setParent(VPRegionBlock *P)
Definition: VPlan.h:504
virtual void dropAllReferences(VPValue *NewValue)=0
Replace all operands of VPUsers in the block with NewValue and also replaces all uses of VPValues def...
VPBlockBase * getSingleHierarchicalPredecessor()
Definition: VPlan.h:580
VPBlockBase * getSingleSuccessor() const
Definition: VPlan.h:528
const VPBlocksTy & getSuccessors() const
Definition: VPlan.h:518
Class that provides utilities for VPBlockBases in VPlan.
Definition: VPlan.h:3169
static auto blocksOnly(const T &Range)
Return an iterator range over Range which only includes BlockTy blocks.
Definition: VPlan.h:3234
static void insertBlockAfter(VPBlockBase *NewBlock, VPBlockBase *BlockPtr)
Insert disconnected VPBlockBase NewBlock after BlockPtr.
Definition: VPlan.h:3178
static void insertTwoBlocksAfter(VPBlockBase *IfTrue, VPBlockBase *IfFalse, VPBlockBase *BlockPtr)
Insert disconnected VPBlockBases IfTrue and IfFalse after BlockPtr.
Definition: VPlan.h:3197
static void disconnectBlocks(VPBlockBase *From, VPBlockBase *To)
Disconnect VPBlockBases From and To bi-directionally.
Definition: VPlan.h:3225
static void connectBlocks(VPBlockBase *From, VPBlockBase *To)
Connect VPBlockBases From and To bi-directionally.
Definition: VPlan.h:3214
A recipe for generating conditional branches on the bits of a mask.
Definition: VPlan.h:2142
VPValue * getMask() const
Return the mask used by this recipe.
Definition: VPlan.h:2174
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlan.h:2162
VPBranchOnMaskRecipe(VPValue *BlockInMask)
Definition: VPlan.h:2144
bool usesScalars(const VPValue *Op) const override
Returns true if the recipe uses scalars of operand Op.
Definition: VPlan.h:2181
void execute(VPTransformState &State) override
Generate the extraction of the appropriate bit from the block mask and the conditional branch.
VPRecipeBase * clone() override
Clone the current recipe.
Definition: VPlan.h:2150
Canonical scalar induction phi of the vector loop.
Definition: VPlan.h:2363
VPRecipeBase * clone() override
Clone the current recipe.
Definition: VPlan.h:2370
bool onlyFirstPartUsed(const VPValue *Op) const override
Returns true if the recipe only uses the first part of operand Op.
Definition: VPlan.h:2404
~VPCanonicalIVPHIRecipe() override=default
static bool classof(const VPHeaderPHIRecipe *D)
Definition: VPlan.h:2378
VPCanonicalIVPHIRecipe(VPValue *StartV, DebugLoc DL)
Definition: VPlan.h:2365
bool onlyFirstLaneUsed(const VPValue *Op) const override
Returns true if the recipe only uses the first lane of operand Op.
Definition: VPlan.h:2397
void execute(VPTransformState &State) override
Generate the canonical scalar induction phi of the vector loop.
Type * getScalarType() const
Returns the scalar type of the induction.
Definition: VPlan.h:2392
bool isCanonical(InductionDescriptor::InductionKind Kind, VPValue *Start, VPValue *Step) const
Check if the induction described by Kind, /p Start and Step is canonical, i.e.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
This class augments a recipe with a set of VPValues defined by the recipe.
Definition: VPlanValue.h:314
unsigned getVPDefID() const
Definition: VPlanValue.h:430
A recipe for converting the canonical IV value to the corresponding value of an IV with different sta...
Definition: VPlan.h:2484
void execute(VPTransformState &State) override
Generate the transformed value of the induction at offset StartValue (1.
VPCanonicalIVPHIRecipe * getCanonicalIV() const
Definition: VPlan.h:2529
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPValue * getStepValue() const
Definition: VPlan.h:2532
VPDerivedIVRecipe(const InductionDescriptor &IndDesc, VPValue *Start, VPCanonicalIVPHIRecipe *CanonicalIV, VPValue *Step)
Definition: VPlan.h:2498
VPRecipeBase * clone() override
Clone the current recipe.
Definition: VPlan.h:2507
Type * getScalarType() const
Definition: VPlan.h:2524
~VPDerivedIVRecipe() override=default
bool onlyFirstLaneUsed(const VPValue *Op) const override
Returns true if the recipe only uses the first lane of operand Op.
Definition: VPlan.h:2535
VPValue * getStartValue() const
Definition: VPlan.h:2528
Recipe to expand a SCEV expression.
Definition: VPlan.h:2333
VPExpandSCEVRecipe(const SCEV *Expr, ScalarEvolution &SE)
Definition: VPlan.h:2338
VPRecipeBase * clone() override
Clone the current recipe.
Definition: VPlan.h:2343
const SCEV * getSCEV() const
Definition: VPlan.h:2356
void execute(VPTransformState &State) override
Generate a canonical vector induction variable of the vector loop, with.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
~VPExpandSCEVRecipe() override=default
A pure virtual base class for all recipes modeling header phis, including phis for first order recurr...
Definition: VPlan.h:1569
static bool classof(const VPValue *V)
Definition: VPlan.h:1586
VPHeaderPHIRecipe(unsigned char VPDefID, Instruction *UnderlyingInstr, VPValue *Start=nullptr, DebugLoc DL={})
Definition: VPlan.h:1571
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override=0
Print the recipe.
virtual VPValue * getBackedgeValue()
Returns the incoming value from the loop backedge.
Definition: VPlan.h:1613
VPValue * getStartValue()
Returns the start value of the phi, if one is set.
Definition: VPlan.h:1602
void setStartValue(VPValue *V)
Update the start value of the recipe.
Definition: VPlan.h:1610
VPValue * getStartValue() const
Definition: VPlan.h:1605
static bool classof(const VPRecipeBase *B)
Method to support type inquiry through isa, cast, and dyn_cast.
Definition: VPlan.h:1582
void execute(VPTransformState &State) override=0
Generate the phi nodes.
virtual VPRecipeBase & getBackedgeRecipe()
Returns the backedge value as a recipe.
Definition: VPlan.h:1619
~VPHeaderPHIRecipe() override=default
This is a concrete Recipe that models a single VPlan-level instruction.
Definition: VPlan.h:1139
bool onlyFirstPartUsed(const VPValue *Op) const override
Returns true if the recipe only uses the first part of operand Op.
Definition: VPlan.h:1259
@ FirstOrderRecurrenceSplice
Definition: VPlan.h:1145
@ CanonicalIVIncrementForPart
Definition: VPlan.h:1154
@ CalculateTripCountMinusVF
Definition: VPlan.h:1152
VPRecipeBase * clone() override
Clone the current recipe.
Definition: VPlan.h:1202
VPInstruction(unsigned Opcode, ArrayRef< VPValue * > Operands, DebugLoc DL, const Twine &Name="")
Definition: VPlan.h:1180
bool hasResult() const
Definition: VPlan.h:1233
LLVM_DUMP_METHOD void dump() const
Print the VPInstruction to dbgs() (for debugging).
unsigned getOpcode() const
Definition: VPlan.h:1209
VPInstruction(unsigned Opcode, std::initializer_list< VPValue * > Operands, WrapFlagsTy WrapFlags, DebugLoc DL={}, const Twine &Name="")
Definition: VPlan.h:1192
VPInstruction(unsigned Opcode, std::initializer_list< VPValue * > Operands, DebugLoc DL={}, const Twine &Name="")
Definition: VPlan.h:1185
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the VPInstruction to O.
bool onlyFirstLaneUsed(const VPValue *Op) const override
Returns true if the recipe only uses the first lane of operand Op.
bool mayWriteToMemory() const
Return true if this instruction may modify memory.
Definition: VPlan.h:1226
void execute(VPTransformState &State) override
Generate the instruction.
VPInterleaveRecipe is a recipe for transforming an interleave group of load or stores into one wide l...
Definition: VPlan.h:1936
bool onlyFirstLaneUsed(const VPValue *Op) const override
The recipe only uses the first lane of the address.
Definition: VPlan.h:2015
~VPInterleaveRecipe() override=default
VPValue * getAddr() const
Return the address accessed by this recipe.
Definition: VPlan.h:1977
VPInterleaveRecipe(const InterleaveGroup< Instruction > *IG, VPValue *Addr, ArrayRef< VPValue * > StoredValues, VPValue *Mask, bool NeedsMaskForGaps)
Definition: VPlan.h:1948
VPValue * getMask() const
Return the mask used by this recipe.
Definition: VPlan.h:1983
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
Generate the wide load or store, and shuffles.
ArrayRef< VPValue * > getStoredValues() const
Return the VPValues stored by this interleave group.
Definition: VPlan.h:1990
const InterleaveGroup< Instruction > * getInterleaveGroup()
Definition: VPlan.h:2006
unsigned getNumStoreOperands() const
Returns the number of stored operands of this interleave group.
Definition: VPlan.h:2010
VPRecipeBase * clone() override
Clone the current recipe.
Definition: VPlan.h:1969
InterleaveGroup< VPInstruction > * getInterleaveGroup(VPInstruction *Instr) const
Get the interleave group that Instr belongs to.
Definition: VPlan.h:3285
In what follows, the term "input IR" refers to code that is fed into the vectorizer whereas the term ...
Definition: VPlan.h:143
static VPLane getLastLaneForVF(const ElementCount &VF)
Definition: VPlan.h:169
static unsigned getNumCachedLanes(const ElementCount &VF)
Returns the maxmimum number of lanes that we are able to consider caching for VF.
Definition: VPlan.h:212
Value * getAsRuntimeExpr(IRBuilderBase &Builder, const ElementCount &VF) const
Returns an expression describing the lane index that can be used at runtime.
Definition: VPlan.cpp:68
VPLane(unsigned Lane, Kind LaneKind)
Definition: VPlan.h:165
Kind getKind() const
Returns the Kind of lane offset.
Definition: VPlan.h:193
bool isFirstLane() const
Returns true if this is the first lane of the whole vector.
Definition: VPlan.h:196
unsigned getKnownLane() const
Returns a compile-time known value for the lane index and asserts if the lane can only be calculated ...
Definition: VPlan.h:183
static VPLane getFirstLane()
Definition: VPlan.h:167
Kind
Kind describes how to interpret Lane.
Definition: VPlan.h:146
@ ScalableLast
For ScalableLast, Lane is the offset from the start of the last N-element subvector in a scalable vec...
@ First
For First, Lane is the index into the first N elements of a fixed-vector <N x <ElTy>> or a scalable v...
unsigned mapToCacheIndex(const ElementCount &VF) const
Maps the lane to a cache index based on VF.
Definition: VPlan.h:199
A value that is used outside the VPlan.
Definition: VPlan.h:673
VPLiveOut(PHINode *Phi, VPValue *Op)
Definition: VPlan.h:677
static bool classof(const VPUser *U)
Definition: VPlan.h:680
bool usesScalars(const VPValue *Op) const override
Returns true if the VPLiveOut uses scalars of operand Op.
Definition: VPlan.h:692
void print(raw_ostream &O, VPSlotTracker &SlotTracker) const
Print the VPLiveOut to O.
PHINode * getPhi() const
Definition: VPlan.h:698
void fixPhi(VPlan &Plan, VPTransformState &State)
Fixup the wrapped LCSSA phi node in the unique exit block.
VPPredInstPHIRecipe is a recipe for generating the phi nodes needed when control converges back from ...
Definition: VPlan.h:2193
~VPPredInstPHIRecipe() override=default
bool usesScalars(const VPValue *Op) const override
Returns true if the recipe uses scalars of operand Op.
Definition: VPlan.h:2217
VPPredInstPHIRecipe(VPValue *PredV)
Construct a VPPredInstPHIRecipe given PredInst whose value needs a phi nodes after merging back from ...
Definition: VPlan.h:2197
void execute(VPTransformState &State) override
Generates phi nodes for live-outs as needed to retain SSA form.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPRecipeBase * clone() override
Clone the current recipe.
Definition: VPlan.h:2201
VPRecipeBase is a base class modeling a sequence of one or more output IR instructions.
Definition: VPlan.h:713
bool mayReadFromMemory() const
Returns true if the recipe may read from memory.
bool mayReadOrWriteMemory() const
Returns true if the recipe may read from or write to memory.
Definition: VPlan.h:799
bool mayHaveSideEffects() const
Returns true if the recipe may have side-effects.
bool mayWriteToMemory() const
Returns true if the recipe may write to memory.
virtual ~VPRecipeBase()=default
VPBasicBlock * getParent()
Definition: VPlan.h:738
DebugLoc getDebugLoc() const
Returns the debug location of the recipe.
Definition: VPlan.h:804
virtual void execute(VPTransformState &State)=0
The method which generates the output IR instructions that correspond to this VPRecipe,...
void moveBefore(VPBasicBlock &BB, iplist< VPRecipeBase >::iterator I)
Unlink this recipe and insert into BB before I.
void insertBefore(VPRecipeBase *InsertPos)
Insert an unlinked recipe into a basic block immediately before the specified recipe.
void insertAfter(VPRecipeBase *InsertPos)
Insert an unlinked Recipe into a basic block immediately after the specified Recipe.
static bool classof(const VPDef *D)
Method to support type inquiry through isa, cast, and dyn_cast.
Definition: VPlan.h:775
iplist< VPRecipeBase >::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
VPRecipeBase(const unsigned char SC, ArrayRef< VPValue * > Operands, DebugLoc DL={})
Definition: VPlan.h:724
virtual VPRecipeBase * clone()=0
Clone the current recipe.
const VPBasicBlock * getParent() const
Definition: VPlan.h:739
static bool classof(const VPUser *U)
Definition: VPlan.h:780
VPRecipeBase(const unsigned char SC, iterator_range< IterT > Operands, DebugLoc DL={})
Definition: VPlan.h:729
void removeFromParent()
This method unlinks 'this' from the containing basic block, but does not delete it.
bool isPhi() const
Returns true for PHI-like recipes.
Definition: VPlan.h:788
void moveAfter(VPRecipeBase *MovePos)
Unlink this recipe from its current VPBasicBlock and insert it into the VPBasicBlock that MovePos liv...
Class to record LLVM IR flag for a recipe along with it.
Definition: VPlan.h:896
ExactFlagsTy ExactFlags
Definition: VPlan.h:950
FastMathFlagsTy FMFs
Definition: VPlan.h:953
VPRecipeWithIRFlags(const unsigned char SC, IterT Operands, GEPFlagsTy GEPFlags, DebugLoc DL={})
Definition: VPlan.h:1021
NonNegFlagsTy NonNegFlags
Definition: VPlan.h:952
CmpInst::Predicate CmpPredicate
Definition: VPlan.h:947
VPRecipeWithIRFlags(const unsigned char SC, IterT Operands, CmpInst::Predicate Pred, DebugLoc DL={})
Definition: VPlan.h:1002
void setFlags(Instruction *I) const
Set the IR flags for I.
Definition: VPlan.h:1068
bool isInBounds() const
Definition: VPlan.h:1107
GEPFlagsTy GEPFlags
Definition: VPlan.h:951
static bool classof(const VPRecipeBase *R)
Definition: VPlan.h:1027
VPRecipeWithIRFlags(const unsigned char SC, IterT Operands, FastMathFlags FMFs, DebugLoc DL={})
Definition: VPlan.h:1014
void dropPoisonGeneratingFlags()
Drop all poison-generating flags.
Definition: VPlan.h:1037
bool hasFastMathFlags() const
Returns true if the recipe has fast-math flags.
Definition: VPlan.h:1114
VPRecipeWithIRFlags(const unsigned char SC, IterT Operands, Instruction &I)
Definition: VPlan.h:972
DisjointFlagsTy DisjointFlags
Definition: VPlan.h:949
VPRecipeWithIRFlags(const unsigned char SC, IterT Operands, WrapFlagsTy WrapFlags, DebugLoc DL={})
Definition: VPlan.h:1008
void transferFlags(VPRecipeWithIRFlags &Other)
Definition: VPlan.h:958
WrapFlagsTy WrapFlags
Definition: VPlan.h:948
bool hasNoUnsignedWrap() const
Definition: VPlan.h:1118
void printFlags(raw_ostream &O) const
CmpInst::Predicate getPredicate() const
Definition: VPlan.h:1101
bool hasNoSignedWrap() const
Definition: VPlan.h:1124
FastMathFlags getFastMathFlags() const
VPRecipeWithIRFlags(const unsigned char SC, IterT Operands, DebugLoc DL={})
Definition: VPlan.h:965
A recipe for handling reduction phis.
Definition: VPlan.h:1823
VPReductionPHIRecipe(PHINode *Phi, const RecurrenceDescriptor &RdxDesc, VPValue &Start, bool IsInLoop=false, bool IsOrdered=false)
Create a new VPReductionPHIRecipe for the reduction Phi described by RdxDesc.
Definition: VPlan.h:1836
bool isOrdered() const
Returns true, if the phi is part of an ordered reduction.
Definition: VPlan.h:1874
~VPReductionPHIRecipe() override=default
VPRecipeBase * clone() override
Clone the current recipe.
Definition: VPlan.h:1846
bool isInLoop() const
Returns true, if the phi is part of an in-loop reduction.
Definition: VPlan.h:1877
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
Generate the phi/select nodes.
static bool classof(const VPHeaderPHIRecipe *R)
Definition: VPlan.h:1856
const RecurrenceDescriptor & getRecurrenceDescriptor() const
Definition: VPlan.h:1869
A recipe to represent inloop reduction operations, performing a reduction on a vector operand into a ...
Definition: VPlan.h:2025
VPRecipeBase * clone() override
Clone the current recipe.
Definition: VPlan.h:2041
VPValue * getVecOp() const
The VPValue of the vector value to be reduced.
Definition: VPlan.h:2060
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPValue * getCondOp() const
The VPValue of the condition for the block.
Definition: VPlan.h:2062
VPReductionRecipe(const RecurrenceDescriptor &R, Instruction *I, VPValue *ChainOp, VPValue *VecOp, VPValue *CondOp)
Definition: VPlan.h:2030
~VPReductionRecipe() override=default
VPValue * getChainOp() const
The VPValue of the scalar Chain being accumulated.
Definition: VPlan.h:2058
void execute(VPTransformState &State) override
Generate the reduction in the loop.
VPRegionBlock represents a collection of VPBasicBlocks and VPRegionBlocks which form a Single-Entry-S...
Definition: VPlan.h:2727
VPRegionBlock * clone() override
Clone all blocks in the single-entry single-exit region of the block and their recipes without updati...
Definition: VPlan.cpp:675
const VPBlockBase * getEntry() const
Definition: VPlan.h:2766
bool isReplicator() const
An indicator whether this region is to generate multiple replicated instances of output IR correspond...
Definition: VPlan.h:2798
void dropAllReferences(VPValue *NewValue) override
Replace all operands of VPUsers in the block with NewValue and also replaces all uses of VPValues def...
Definition: VPlan.cpp:684
void setExiting(VPBlockBase *ExitingBlock)
Set ExitingBlock as the exiting VPBlockBase of this VPRegionBlock.
Definition: VPlan.h:2783
VPBlockBase * getExiting()
Definition: VPlan.h:2779
void setEntry(VPBlockBase *EntryBlock)
Set EntryBlock as the entry VPBlockBase of this VPRegionBlock.
Definition: VPlan.h:2771
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print this VPRegionBlock to O (recursively), prefixing all lines with Indent.
Definition: VPlan.cpp:743
VPRegionBlock(const std::string &Name="", bool IsReplicator=false)
Definition: VPlan.h:2749
VPRegionBlock(VPBlockBase *Entry, VPBlockBase *Exiting, const std::string &Name="", bool IsReplicator=false)
Definition: VPlan.h:2740
void execute(VPTransformState *State) override
The method which generates the output IR instructions that correspond to this VPRegionBlock,...
Definition: VPlan.cpp:691
const VPBlockBase * getExiting() const
Definition: VPlan.h:2778
VPBlockBase * getEntry()
Definition: VPlan.h:2767
VPBasicBlock * getPreheaderVPBB()
Returns the pre-header VPBasicBlock of the loop region.
Definition: VPlan.h:2791
~VPRegionBlock() override
Definition: VPlan.h:2753
static bool classof(const VPBlockBase *V)
Method to support type inquiry through isa, cast, and dyn_cast.
Definition: VPlan.h:2762
VPReplicateRecipe replicates a given instruction producing multiple scalar copies of the original sca...
Definition: VPlan.h:2071
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
Generate replicas of the desired Ingredient.
~VPReplicateRecipe() override=default
VPRecipeBase * clone() override
Clone the current recipe.
Definition: VPlan.h:2090
bool onlyFirstLaneUsed(const VPValue *Op) const override
Returns true if the recipe only uses the first lane of operand Op.
Definition: VPlan.h:2116
bool usesScalars(const VPValue *Op) const override
Returns true if the recipe uses scalars of operand Op.
Definition: VPlan.h:2123
bool isUniform() const
Definition: VPlan.h:2111
bool isPredicated() const
Definition: VPlan.h:2113
VPReplicateRecipe(Instruction *I, iterator_range< IterT > Operands, bool IsUniform, VPValue *Mask=nullptr)
Definition: VPlan.h:2080
VPValue * getMask()
Return the mask of a predicated VPReplicateRecipe.
Definition: VPlan.h:2135
bool shouldPack() const
Returns true if the recipe is used by a widened recipe via an intervening VPPredInstPHIRecipe.
VPScalarCastRecipe is a recipe to create scalar cast instructions.
Definition: VPlan.h:1359
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Each concrete VPDef prints itself.
~VPScalarCastRecipe() override=default
bool onlyFirstLaneUsed(const VPValue *Op) const override
Returns true if the VPUser only uses the first lane of operand Op.
Definition: VPlan.h:1389
VPRecipeBase * clone() override
Clone the current recipe.
Definition: VPlan.h:1373
void execute(VPTransformState &State) override
The method which generates the output IR instructions that correspond to this VPRecipe,...
Type * getResultType() const
Returns the result type of the cast.
Definition: VPlan.h:1387
VPScalarCastRecipe(Instruction::CastOps Opcode, VPValue *Op, Type *ResultTy)
Definition: VPlan.h:1367
A recipe for handling phi nodes of integer and floating-point inductions, producing their scalar valu...
Definition: VPlan.h:2544
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
bool onlyFirstLaneUsed(const VPValue *Op) const override
Returns true if the recipe only uses the first lane of operand Op.
Definition: VPlan.h:2584
VPValue * getStepValue() const
Definition: VPlan.h:2581
VPScalarIVStepsRecipe(const InductionDescriptor &IndDesc, VPValue *IV, VPValue *Step)
Definition: VPlan.h:2554
VPScalarIVStepsRecipe(VPValue *IV, VPValue *Step, Instruction::BinaryOps Opcode, FastMathFlags FMFs)
Definition: VPlan.h:2548
~VPScalarIVStepsRecipe() override=default
void execute(VPTransformState &State) override
Generate the scalarized versions of the phi node as needed by their users.
VPRecipeBase * clone() override
Clone the current recipe.
Definition: VPlan.h:2564
VPSingleDef is a base class for recipes for modeling a sequence of one or more output IR that define ...
Definition: VPlan.h:830
VPSingleDefRecipe(const unsigned char SC, ArrayRef< VPValue * > Operands, DebugLoc DL={})
Definition: VPlan.h:836
Instruction * getUnderlyingInstr()
Returns the underlying instruction.
Definition: VPlan.h:887
static bool classof(const VPRecipeBase *R)
Definition: VPlan.h:845
const Instruction * getUnderlyingInstr() const
Definition: VPlan.h:890
VPSingleDefRecipe(const unsigned char SC, IterT Operands, DebugLoc DL={})
Definition: VPlan.h:833
static bool classof(const VPUser *U)
Definition: VPlan.h:881
VPSingleDefRecipe(const unsigned char SC, IterT Operands, Value *UV, DebugLoc DL={})
Definition: VPlan.h:841
This class can be used to assign consecutive numbers to all VPValues in a VPlan and allows querying t...
Definition: VPlanValue.h:448
An analysis for type-inference for VPValues.
Definition: VPlanAnalysis.h:36
This class augments VPValue with operands which provide the inverse def-use edges from VPValue's user...
Definition: VPlanValue.h:204
operand_range operands()
Definition: VPlanValue.h:279
void setOperand(unsigned I, VPValue *New)
Definition: VPlanValue.h:259
unsigned getNumOperands() const
Definition: VPlanValue.h:253
operand_iterator op_begin()
Definition: VPlanValue.h:275
VPValue * getOperand(unsigned N) const
Definition: VPlanValue.h:254
VPUser()=delete
void addOperand(VPValue *Operand)
Definition: VPlanValue.h:248
Value * getUnderlyingValue()
Return the underlying Value attached to this VPValue.
Definition: VPlanValue.h:78
VPRecipeBase * getDefiningRecipe()
Returns the recipe defining this VPValue or nullptr if it is not defined by a recipe,...
Definition: VPlan.cpp:118
unsigned getNumUsers() const
Definition: VPlanValue.h:113
Value * getLiveInIRValue()
Returns the underlying IR value, if this VPValue is defined outside the scope of VPlan.
Definition: VPlanValue.h:174
bool isLiveIn() const
Returns true if this VPValue is a live-in, i.e. defined outside the VPlan.
Definition: VPlanValue.h:169
friend class VPRecipeBase
Definition: VPlanValue.h:52
user_range users()
Definition: VPlanValue.h:134
bool isDefinedOutsideVectorRegions() const
Returns true if the VPValue is defined outside any vector regions, i.e.
Definition: VPlanValue.h:188
A recipe to compute the pointers for widened memory accesses of IndexTy for all parts.
Definition: VPlan.h:1513
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
The method which generates the output IR instructions that correspond to this VPRecipe,...
VPRecipeBase * clone() override
Clone the current recipe.
Definition: VPlan.h:1534
VPVectorPointerRecipe(VPValue *Ptr, Type *IndexedTy, bool IsReverse, bool IsInBounds, DebugLoc DL)
Definition: VPlan.h:1518
bool onlyFirstLaneUsed(const VPValue *Op) const override
Returns true if the VPUser only uses the first lane of operand Op.
Definition: VPlan.h:1528
A recipe for widening Call instructions.
Definition: VPlan.h:1398
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPWidenCallRecipe(CallInst &I, iterator_range< IterT > CallArguments, Intrinsic::ID VectorIntrinsicID, DebugLoc DL={}, Function *Variant=nullptr)
Definition: VPlan.h:1410
VPRecipeBase * clone() override
Clone the current recipe.
Definition: VPlan.h:1418
void execute(VPTransformState &State) override
Produce a widened version of the call instruction.
~VPWidenCallRecipe() override=default
A Recipe for widening the canonical induction variable of the vector loop.
Definition: VPlan.h:2449
void execute(VPTransformState &State) override
Generate a canonical vector induction variable of the vector loop, with start = {<Part*VF,...
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
~VPWidenCanonicalIVRecipe() override=default
VPWidenCanonicalIVRecipe(VPCanonicalIVPHIRecipe *CanonicalIV)
Definition: VPlan.h:2451
const Type * getScalarType() const
Returns the scalar type of the induction.
Definition: VPlan.h:2475
VPRecipeBase * clone() override
Clone the current recipe.
Definition: VPlan.h:2456
VPWidenCastRecipe is a recipe to create vector cast instructions.
Definition: VPlan.h:1309
VPWidenCastRecipe(Instruction::CastOps Opcode, VPValue *Op, Type *ResultTy, CastInst &UI)
Definition: VPlan.h:1317
Instruction::CastOps getOpcode() const
Definition: VPlan.h:1352
VPRecipeBase * clone() override
Clone the current recipe.
Definition: VPlan.h:1333
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Type * getResultType() const
Returns the result type of the cast.
Definition: VPlan.h:1355
VPWidenCastRecipe(Instruction::CastOps Opcode, VPValue *Op, Type *ResultTy)
Definition: VPlan.h:1327
void execute(VPTransformState &State) override
Produce widened copies of the cast.
~VPWidenCastRecipe() override=default
A recipe for handling GEP instructions.
Definition: VPlan.h:1471
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
Generate the gep nodes.
~VPWidenGEPRecipe() override=default
VPRecipeBase * clone() override
Clone the current recipe.
Definition: VPlan.h:1493
VPWidenGEPRecipe(GetElementPtrInst *GEP, iterator_range< IterT > Operands)
Definition: VPlan.h:1488
A recipe for handling phi nodes of integer and floating-point inductions, producing their vector valu...
Definition: VPlan.h:1626
VPWidenIntOrFpInductionRecipe(PHINode *IV, VPValue *Start, VPValue *Step, const InductionDescriptor &IndDesc, TruncInst *Trunc)
Definition: VPlan.h:1639
const TruncInst * getTruncInst() const
Definition: VPlan.h:1687
VPRecipeBase & getBackedgeRecipe() override
Returns the backedge value as a recipe.
Definition: VPlan.h:1673
~VPWidenIntOrFpInductionRecipe() override=default
TruncInst * getTruncInst()
Returns the first defined value as TruncInst, if it is one or nullptr otherwise.
Definition: VPlan.h:1686
void execute(VPTransformState &State) override
Generate the vectorized and scalarized versions of the phi node as needed by their users.
VPValue * getStepValue()
Returns the step value of the induction.
Definition: VPlan.h:1681
VPWidenIntOrFpInductionRecipe(PHINode *IV, VPValue *Start, VPValue *Step, const InductionDescriptor &IndDesc)
Definition: VPlan.h:1632
const VPValue * getStepValue() const
Definition: VPlan.h:1682
Type * getScalarType() const
Returns the scalar type of the induction.
Definition: VPlan.h:1699
VPValue * getBackedgeValue() override
Returns the incoming value from the loop backedge.
Definition: VPlan.h:1666
VPRecipeBase * clone() override
Clone the current recipe.
Definition: VPlan.h:1649
bool isCanonical() const
Returns true if the induction is canonical, i.e.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
const InductionDescriptor & getInductionDescriptor() const
Returns the induction descriptor for the recipe.
Definition: VPlan.h:1692
A Recipe for widening load/store operations.
Definition: VPlan.h:2230
VPValue * getMask() const
Return the mask used by this recipe.
Definition: VPlan.h:2287
VPValue * getAddr() const
Return the address accessed by this recipe.
Definition: VPlan.h:2281
Instruction & getIngredient() const
Definition: VPlan.h:2329
VPWidenMemoryInstructionRecipe(StoreInst &Store, VPValue *Addr, VPValue *StoredValue, VPValue *Mask, bool Consecutive, bool Reverse)
Definition: VPlan.h:2259
void execute(VPTransformState &State) override
Generate the wide load/store.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPWidenMemoryInstructionRecipe(LoadInst &Load, VPValue *Addr, VPValue *Mask, bool Consecutive, bool Reverse)
Definition: VPlan.h:2250
VPValue * getStoredValue() const
Return the address accessed by this recipe.
Definition: VPlan.h:2296
bool onlyFirstLaneUsed(const VPValue *Op) const override
Returns true if the recipe only uses the first lane of operand Op.
Definition: VPlan.h:2318
bool isStore() const
Returns true if this recipe is a store.
Definition: VPlan.h:2293
VPRecipeBase * clone() override
Clone the current recipe.
Definition: VPlan.h:2268
A recipe for handling phis that are widened in the vector loop.
Definition: VPlan.h:1751
void addIncoming(VPValue *IncomingV, VPBasicBlock *IncomingBlock)
Adds a pair (IncomingV, IncomingBlock) to the phi.
Definition: VPlan.h:1781
VPValue * getIncomingValue(unsigned I)
Returns the I th incoming VPValue.
Definition: VPlan.h:1790
VPWidenPHIRecipe(PHINode *Phi, VPValue *Start=nullptr)
Create a new VPWidenPHIRecipe for Phi with start value Start.
Definition: VPlan.h:1757
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
~VPWidenPHIRecipe() override=default
VPRecipeBase * clone() override
Clone the current recipe.
Definition: VPlan.h:1763
VPBasicBlock * getIncomingBlock(unsigned I)
Returns the I th incoming VPBasicBlock.
Definition: VPlan.h:1787
void execute(VPTransformState &State) override
Generate the phi/select nodes.
VPRecipeBase * clone() override
Clone the current recipe.
Definition: VPlan.h:1724
const InductionDescriptor & getInductionDescriptor() const
Returns the induction descriptor for the recipe.
Definition: VPlan.h:1739
~VPWidenPointerInductionRecipe() override=default
bool onlyScalarsGenerated(bool IsScalable)
Returns true if only scalar values will be generated.
void execute(VPTransformState &State) override
Generate vector values for the pointer induction.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPWidenPointerInductionRecipe(PHINode *Phi, VPValue *Start, VPValue *Step, const InductionDescriptor &IndDesc, bool IsScalarAfterVectorization)
Create a new VPWidenPointerInductionRecipe for Phi with start value Start.
Definition: VPlan.h:1712
VPWidenRecipe is a recipe for producing a copy of vector type its ingredient.
Definition: VPlan.h:1277
void execute(VPTransformState &State) override
Produce widened copies of all Ingredients.
VPRecipeBase * clone() override
Clone the current recipe.
Definition: VPlan.h:1288
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
~VPWidenRecipe() override=default
VPWidenRecipe(Instruction &I, iterator_range< IterT > Operands)
Definition: VPlan.h:1282
unsigned getOpcode() const
Definition: VPlan.h:1299
VPlanPrinter prints a given VPlan to a given output stream.
Definition: VPlan.h:3098
VPlanPrinter(raw_ostream &O, const VPlan &P)
Definition: VPlan.h:3139
LLVM_DUMP_METHOD void dump()
Definition: VPlan.cpp:1141
Class that maps (parts of) an existing VPlan to trees of combined VPInstructions.
Definition: VPlan.h:3292
VPInstruction * buildGraph(ArrayRef< VPValue * > Operands)
Tries to build an SLP tree rooted at Operands and returns a VPInstruction combining Operands,...
Definition: VPlanSLP.cpp:359
bool isCompletelySLP() const
Return true if all visited instruction can be combined.
Definition: VPlan.h:3382
~VPlanSlp()=default
VPlanSlp(VPInterleavedAccessInfo &IAI, VPBasicBlock &BB)
Definition: VPlan.h:3370
unsigned getWidestBundleBits() const
Return the width of the widest combined bundle in bits.
Definition: VPlan.h:3379
VPlan models a candidate for vectorization, encoding various decisions take to produce efficient outp...
Definition: VPlan.h:2828
void printDOT(raw_ostream &O) const
Print this VPlan in DOT format to O.
Definition: VPlan.cpp:994
VPValue * getVPValue(Value *V)
Returns the VPValue for V.
Definition: VPlan.h:2996
std::string getName() const
Return a string with the name of the plan and the applicable VFs and UFs.
Definition: VPlan.cpp:970
void prepareToExecute(Value *TripCount, Value *VectorTripCount, Value *CanonicalIVStartValue, VPTransformState &State)
Prepare the plan for execution, setting up the required live-in values.
Definition: VPlan.cpp:792
bool hasScalableVF()
Definition: VPlan.h:2968
VPBasicBlock * getEntry()
Definition: VPlan.h:2925
void addVPValue(Value *V, VPValue *VPV)
Definition: VPlan.h:2987
VPValue & getVectorTripCount()
The vector trip count.
Definition: VPlan.h:2950
void setName(const Twine &newName)
Definition: VPlan.h:2985
VPValue & getVFxUF()
Returns VF * UF of the vector loop region.
Definition: VPlan.h:2953
VPValue * getTripCount() const
The trip count of the original loop.
Definition: VPlan.h:2929
VPValue * getOrCreateBackedgeTakenCount()
The backedge taken count of the original loop.
Definition: VPlan.h:2943
void removeLiveOut(PHINode *PN)
Definition: VPlan.h:3061
void addLiveOut(PHINode *PN, VPValue *V)
Definition: VPlan.cpp:1003
const VPBasicBlock * getEntry() const
Definition: VPlan.h:2926
VPlan(VPBasicBlock *Preheader, VPValue *TC, VPBasicBlock *Entry)
Construct a VPlan with original preheader Preheader, trip count TC and Entry to the plan.
Definition: VPlan.h:2890
VPBasicBlock * getPreheader()
Definition: VPlan.h:3080
VPValue * getVPValueOrAddLiveIn(Value *V)
Gets the VPValue for V or adds a new live-in (if none exists yet) for V.
Definition: VPlan.h:3006
VPRegionBlock * getVectorLoopRegion()
Returns the VPRegionBlock of the vector loop.
Definition: VPlan.h:3042
const VPRegionBlock * getVectorLoopRegion() const
Definition: VPlan.h:3045
static VPlanPtr createInitialVPlan(const SCEV *TripCount, ScalarEvolution &PSE)
Create initial VPlan skeleton, having an "entry" VPBasicBlock (wrapping original scalar pre-header) w...
Definition: VPlan.cpp:778
bool hasVF(ElementCount VF)
Definition: VPlan.h:2967
void addSCEVExpansion(const SCEV *S, VPValue *V)
Definition: VPlan.h:3074
bool hasUF(unsigned UF) const
Definition: VPlan.h:2974
void setVF(ElementCount VF)
Definition: VPlan.h:2961
void resetTripCount(VPValue *NewTripCount)
Resets the trip count for the VPlan.
Definition: VPlan.h:2936
VPlan(VPBasicBlock *Preheader, VPBasicBlock *Entry)
Construct a VPlan with original preheader Preheader and Entry to the plan.
Definition: VPlan.h:2899
void disableValue2VPValue()
Mark the plan to indicate that using Value2VPValue is not safe any longer, because it may be stale.
Definition: VPlan.h:2957
const VPBasicBlock * getPreheader() const
Definition: VPlan.h:3081
LLVM_DUMP_METHOD void dump() const
Dump the plan to stderr (for debugging).
Definition: VPlan.cpp:1000
bool hasScalarVFOnly() const
Definition: VPlan.h:2972
iterator_range< mapped_iterator< Use *, std::function< VPValue *(Value *)> > > mapToVPValues(User::op_range Operands)
Returns a range mapping the values the range Operands to their corresponding VPValues.
Definition: VPlan.h:3034
void execute(VPTransformState *State)
Generate the IR code for this VPlan.
Definition: VPlan.cpp:834
VPCanonicalIVPHIRecipe * getCanonicalIV()
Returns the canonical induction recipe of the vector loop.
Definition: VPlan.h:3050
const MapVector< PHINode *, VPLiveOut * > & getLiveOuts() const
Definition: VPlan.h:3066
void print(raw_ostream &O) const
Print this VPlan to O.
Definition: VPlan.cpp:944
void addVF(ElementCount VF)
Definition: VPlan.h:2959
VPValue * getSCEVExpansion(const SCEV *S) const
Definition: VPlan.h:3070
void printLiveIns(raw_ostream &O) const
Print the live-ins of this VPlan to O.
Definition: VPlan.cpp:914
void setUF(unsigned UF)
Definition: VPlan.h:2976
VPlan * duplicate()
Clone the current VPlan, update all VPValues of the new VPlan and cloned recipes to refer to the clon...
Definition: VPlan.cpp:1084
LLVM Value Representation.
Definition: Value.h:74
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:255
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
Definition: TypeSize.h:171
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
Definition: TypeSize.h:168
An ilist node that can access its parent list.
Definition: ilist_node.h:284
base_list_type::const_reverse_iterator const_reverse_iterator
Definition: ilist.h:125
void pop_back()
Definition: ilist.h:255
base_list_type::reverse_iterator reverse_iterator
Definition: ilist.h:123
base_list_type::const_iterator const_iterator
Definition: ilist.h:122
iterator insert(iterator where, pointer New)
Definition: ilist.h:165
CRTP base class which implements the entire standard iterator facade in terms of a minimal subset of ...
Definition: iterator.h:80
A range adaptor for a pair of iterators.
This class implements an extremely fast bulk output stream that can only output to a stream.
Definition: raw_ostream.h:52
This file defines classes to implement an intrusive doubly linked list class (i.e.
This file defines the ilist_node class template, which is a convenient base class for creating classe...
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ BasicBlock
Various leaf nodes.
Definition: ISDOpcodes.h:71
VPValue * getOrCreateVPValueForSCEVExpr(VPlan &Plan, const SCEV *Expr, ScalarEvolution &SE)
Get or create a VPValue that corresponds to the expansion of Expr.
Definition: VPlan.cpp:1422
bool isUniformAfterVectorization(VPValue *VPV)
Returns true if VPV is uniform after vectorization.
Definition: VPlan.h:3402
bool onlyFirstPartUsed(const VPValue *Def)
Returns true if only the first part of Def is used.
Definition: VPlan.cpp:1417
bool onlyFirstLaneUsed(const VPValue *Def)
Returns true if only the first lane of Def is used.
Definition: VPlan.cpp:1412
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
auto find(R &&Range, const T &Val)
Provide wrappers to std::find which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1751
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1731
bool isEqual(const GCNRPTracker::LiveRegSet &S1, const GCNRPTracker::LiveRegSet &S2)
const SCEV * createTripCountSCEV(Type *IdxTy, PredicatedScalarEvolution &PSE, Loop *OrigLoop)
testing::Matcher< const detail::ErrorHolder & > Failed()
Definition: Error.h:198
Value * getRuntimeVF(IRBuilderBase &B, Type *Ty, ElementCount VF)
Return the runtime value for VF.
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
auto map_range(ContainerTy &&C, FuncTy F)
Definition: STLExtras.h:377
auto dyn_cast_or_null(const Y &Val)
Definition: Casting.h:759
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1738
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition: MathExtras.h:264
std::unique_ptr< VPlan > VPlanPtr
Definition: VPlan.h:134
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
iterator_range< filter_iterator< detail::IterOfRange< RangeT >, PredicateT > > make_filter_range(RangeT &&Range, PredicateT Pred)
Convenience function that takes a range of elements and a predicate, and return a new filter_iterator...
Definition: STLExtras.h:581
@ Other
Any other memory.
raw_ostream & operator<<(raw_ostream &OS, const APFixedPoint &FX)
Definition: APFixedPoint.h:293
Value * createStepForVF(IRBuilderBase &B, Type *Ty, ElementCount VF, int64_t Step)
Return a value for Step multiplied by VF.
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
Definition: STLExtras.h:1888
hash_code hash_combine_range(InputIteratorT first, InputIteratorT last)
Compute a hash_code for a sequence of values.
Definition: Hashing.h:491
#define N
A range of powers-of-2 vectorization factors with fixed start and adjustable end.
Definition: VPlan.h:87
iterator end()
Definition: VPlan.h:128
const ElementCount Start
Definition: VPlan.h:89
ElementCount End
Definition: VPlan.h:92
iterator begin()
Definition: VPlan.h:127
bool isEmpty() const
Definition: VPlan.h:94
VFRange(const ElementCount &Start, const ElementCount &End)
Definition: VPlan.h:98
A recipe for handling first-order recurrence phis.
Definition: VPlan.h:1796
void execute(VPTransformState &State) override
Generate the phi nodes.
VPFirstOrderRecurrencePHIRecipe(PHINode *Phi, VPValue &Start)
Definition: VPlan.h:1797
VPRecipeBase * clone() override
Clone the current recipe.
Definition: VPlan.h:1806
static bool classof(const VPHeaderPHIRecipe *R)
Definition: VPlan.h:1802
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPIteration represents a single point in the iteration space of the output (vectorized and/or unrolle...
Definition: VPlan.h:219
VPIteration(unsigned Part, const VPLane &Lane)
Definition: VPlan.h:229
unsigned Part
in [0..UF)
Definition: VPlan.h:221
VPLane Lane
Definition: VPlan.h:223
VPIteration(unsigned Part, unsigned Lane, VPLane::Kind Kind=VPLane::Kind::First)
Definition: VPlan.h:225
bool isFirstIteration() const
Definition: VPlan.h:231
WrapFlagsTy(bool HasNUW, bool HasNSW)
Definition: VPlan.h:913
Hold state information used when constructing the CFG of the output IR, traversing the VPBasicBlocks ...
Definition: VPlan.h:363
BasicBlock * PrevBB
The previous IR BasicBlock created or used.
Definition: VPlan.h:369
SmallDenseMap< VPBasicBlock *, BasicBlock * > VPBB2IRBB
A mapping of each VPBasicBlock to the corresponding BasicBlock.
Definition: VPlan.h:377
VPBasicBlock * PrevVPBB
The previous VPBasicBlock visited. Initially set to null.
Definition: VPlan.h:365
BasicBlock * ExitBB
The last IR BasicBlock in the output IR.
Definition: VPlan.h:373
BasicBlock * getPreheaderBBFor(VPRecipeBase *R)
Returns the BasicBlock* mapped to the pre-header of the loop region containing R.
Definition: VPlan.cpp:348
SmallVector< Value *, 2 > PerPartValuesTy
A type for vectorized values in the new loop.
Definition: VPlan.h:254
DenseMap< VPValue *, ScalarsPerPartValuesTy > PerPartScalars
Definition: VPlan.h:259
DenseMap< VPValue *, PerPartValuesTy > PerPartOutput
Definition: VPlan.h:256
VPTransformState holds information passed down when "executing" a VPlan, needed for generating the ou...
Definition: VPlan.h:236
Value * get(VPValue *Def, unsigned Part, bool IsScalar=false)
Get the generated vector Value for a given VPValue Def and a given Part if IsScalar is false,...
Definition: VPlan.cpp:247
LoopInfo * LI
Hold a pointer to LoopInfo to register new basic blocks in the loop.
Definition: VPlan.h:387
DenseMap< const SCEV *, Value * > ExpandedSCEVs
Map SCEVs to their expanded values.
Definition: VPlan.h:413
void addMetadata(Instruction *To, Instruction *From)
Add metadata from one instruction to another.
Definition: VPlan.cpp:361
VPTypeAnalysis TypeAnalysis
VPlan-based type analysis.
Definition: VPlan.h:416
struct llvm::VPTransformState::DataState Data
void reset(VPValue *Def, Value *V, unsigned Part)
Reset an existing vector value for Def and a given Part.
Definition: VPlan.h:303
struct llvm::VPTransformState::CFGState CFG
void reset(VPValue *Def, Value *V, const VPIteration &Instance)
Reset an existing scalar value for Def and a given Instance.
Definition: VPlan.h:325
LoopVersioning * LVer
LoopVersioning.
Definition: VPlan.h:409
void addNewMetadata(Instruction *To, const Instruction *Orig)
Add additional metadata to To that was not present on Orig.
Definition: VPlan.cpp:353
void packScalarIntoVectorValue(VPValue *Def, const VPIteration &Instance)
Construct the vector value of a scalarized value V one lane at a time.
Definition: VPlan.cpp:402
void set(VPValue *Def, Value *V, const VPIteration &Instance)
Set the generated scalar V for Def and the given Instance.
Definition: VPlan.h:311
void set(VPValue *Def, Value *V, unsigned Part, bool IsScalar=false)
Set the generated vector Value for a given VPValue and a given Part, if IsScalar is false.
Definition: VPlan.h:288
std::optional< VPIteration > Instance
Hold the indices to generate specific scalar instructions.
Definition: VPlan.h:248
IRBuilderBase & Builder
Hold a reference to the IRBuilder used to generate output IR code.
Definition: VPlan.h:393
DominatorTree * DT
Hold a pointer to Dominator Tree to register new basic blocks in the loop.
Definition: VPlan.h:390
bool hasScalarValue(VPValue *Def, VPIteration Instance)
Definition: VPlan.h:276
VPlan * Plan
Pointer to the VPlan code is generated for.
Definition: VPlan.h:399
InnerLoopVectorizer * ILV
Hold a pointer to InnerLoopVectorizer to reuse its IR generation methods.
Definition: VPlan.h:396
bool hasVectorValue(VPValue *Def, unsigned Part)
Definition: VPlan.h:270
ElementCount VF
The chosen Vectorization and Unroll Factors of the loop being vectorized.
Definition: VPlan.h:242
Loop * CurrentVectorLoop
The loop object for the current parent region, or nullptr.
Definition: VPlan.h:402
void setDebugLocFrom(DebugLoc DL)
Set the debug location in the builder using the debug location DL.
Definition: VPlan.cpp:381
A recipe for widening select instructions.
Definition: VPlan.h:1437
bool isInvariantCond() const
Definition: VPlan.h:1465
VPWidenSelectRecipe(SelectInst &I, iterator_range< IterT > Operands)
Definition: VPlan.h:1439
VPValue * getCond() const
Definition: VPlan.h:1461
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
Produce a widened version of the select instruction.
VPRecipeBase * clone() override
Clone the current recipe.
Definition: VPlan.h:1445
~VPWidenSelectRecipe() override=default
VPlanIngredient(const Value *V)
Definition: VPlan.h:3148
const Value * V
Definition: VPlan.h:3146
void print(raw_ostream &O) const
Definition: VPlan.cpp:1259