LLVM 22.0.0git
VPlan.h
Go to the documentation of this file.
1//===- VPlan.h - Represent A Vectorizer Plan --------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// This file contains the declarations of the Vectorization Plan base classes:
11/// 1. VPBasicBlock and VPRegionBlock that inherit from a common pure virtual
12/// VPBlockBase, together implementing a Hierarchical CFG;
13/// 2. Pure virtual VPRecipeBase serving as the base class for recipes contained
14/// within VPBasicBlocks;
15/// 3. Pure virtual VPSingleDefRecipe serving as a base class for recipes that
16/// also inherit from VPValue.
17/// 4. VPInstruction, a concrete Recipe and VPUser modeling a single planned
18/// instruction;
19/// 5. The VPlan class holding a candidate for vectorization;
20/// These are documented in docs/VectorizationPlan.rst.
21//
22//===----------------------------------------------------------------------===//
23
24#ifndef LLVM_TRANSFORMS_VECTORIZE_VPLAN_H
25#define LLVM_TRANSFORMS_VECTORIZE_VPLAN_H
26
27#include "VPlanAnalysis.h"
28#include "VPlanValue.h"
29#include "llvm/ADT/DenseMap.h"
33#include "llvm/ADT/Twine.h"
34#include "llvm/ADT/ilist.h"
35#include "llvm/ADT/ilist_node.h"
38#include "llvm/IR/DebugLoc.h"
39#include "llvm/IR/FMF.h"
40#include "llvm/IR/Operator.h"
43#include <algorithm>
44#include <cassert>
45#include <cstddef>
46#include <string>
47
48namespace llvm {
49
50class BasicBlock;
51class DominatorTree;
53class IRBuilderBase;
54struct VPTransformState;
55class raw_ostream;
57class SCEV;
58class Type;
59class VPBasicBlock;
60class VPBuilder;
61class VPDominatorTree;
62class VPRegionBlock;
63class VPlan;
64class VPLane;
66class VPlanSlp;
67class Value;
69class LoopVersioning;
70
71struct VPCostContext;
72
73namespace Intrinsic {
74typedef unsigned ID;
75}
76
77using VPlanPtr = std::unique_ptr<VPlan>;
78
79/// VPBlockBase is the building block of the Hierarchical Control-Flow Graph.
80/// A VPBlockBase can be either a VPBasicBlock or a VPRegionBlock.
82 friend class VPBlockUtils;
83
84 const unsigned char SubclassID; ///< Subclass identifier (for isa/dyn_cast).
85
86 /// An optional name for the block.
87 std::string Name;
88
89 /// The immediate VPRegionBlock which this VPBlockBase belongs to, or null if
90 /// it is a topmost VPBlockBase.
91 VPRegionBlock *Parent = nullptr;
92
93 /// List of predecessor blocks.
95
96 /// List of successor blocks.
98
99 /// VPlan containing the block. Can only be set on the entry block of the
100 /// plan.
101 VPlan *Plan = nullptr;
102
103 /// Add \p Successor as the last successor to this block.
104 void appendSuccessor(VPBlockBase *Successor) {
105 assert(Successor && "Cannot add nullptr successor!");
106 Successors.push_back(Successor);
107 }
108
109 /// Add \p Predecessor as the last predecessor to this block.
110 void appendPredecessor(VPBlockBase *Predecessor) {
111 assert(Predecessor && "Cannot add nullptr predecessor!");
112 Predecessors.push_back(Predecessor);
113 }
114
115 /// Remove \p Predecessor from the predecessors of this block.
116 void removePredecessor(VPBlockBase *Predecessor) {
117 auto Pos = find(Predecessors, Predecessor);
118 assert(Pos && "Predecessor does not exist");
119 Predecessors.erase(Pos);
120 }
121
122 /// Remove \p Successor from the successors of this block.
123 void removeSuccessor(VPBlockBase *Successor) {
124 auto Pos = find(Successors, Successor);
125 assert(Pos && "Successor does not exist");
126 Successors.erase(Pos);
127 }
128
129 /// This function replaces one predecessor with another, useful when
130 /// trying to replace an old block in the CFG with a new one.
131 void replacePredecessor(VPBlockBase *Old, VPBlockBase *New) {
132 auto I = find(Predecessors, Old);
133 assert(I != Predecessors.end());
134 assert(Old->getParent() == New->getParent() &&
135 "replaced predecessor must have the same parent");
136 *I = New;
137 }
138
139 /// This function replaces one successor with another, useful when
140 /// trying to replace an old block in the CFG with a new one.
141 void replaceSuccessor(VPBlockBase *Old, VPBlockBase *New) {
142 auto I = find(Successors, Old);
143 assert(I != Successors.end());
144 assert(Old->getParent() == New->getParent() &&
145 "replaced successor must have the same parent");
146 *I = New;
147 }
148
149protected:
150 VPBlockBase(const unsigned char SC, const std::string &N)
151 : SubclassID(SC), Name(N) {}
152
153public:
154 /// An enumeration for keeping track of the concrete subclass of VPBlockBase
155 /// that are actually instantiated. Values of this enumeration are kept in the
156 /// SubclassID field of the VPBlockBase objects. They are used for concrete
157 /// type identification.
158 using VPBlockTy = enum { VPRegionBlockSC, VPBasicBlockSC, VPIRBasicBlockSC };
159
161
162 virtual ~VPBlockBase() = default;
163
164 const std::string &getName() const { return Name; }
165
166 void setName(const Twine &newName) { Name = newName.str(); }
167
168 /// \return an ID for the concrete type of this object.
169 /// This is used to implement the classof checks. This should not be used
170 /// for any other purpose, as the values may change as LLVM evolves.
171 unsigned getVPBlockID() const { return SubclassID; }
172
173 VPRegionBlock *getParent() { return Parent; }
174 const VPRegionBlock *getParent() const { return Parent; }
175
176 /// \return A pointer to the plan containing the current block.
177 VPlan *getPlan();
178 const VPlan *getPlan() const;
179
180 /// Sets the pointer of the plan containing the block. The block must be the
181 /// entry block into the VPlan.
182 void setPlan(VPlan *ParentPlan);
183
184 void setParent(VPRegionBlock *P) { Parent = P; }
185
186 /// \return the VPBasicBlock that is the entry of this VPBlockBase,
187 /// recursively, if the latter is a VPRegionBlock. Otherwise, if this
188 /// VPBlockBase is a VPBasicBlock, it is returned.
189 const VPBasicBlock *getEntryBasicBlock() const;
190 VPBasicBlock *getEntryBasicBlock();
191
192 /// \return the VPBasicBlock that is the exiting this VPBlockBase,
193 /// recursively, if the latter is a VPRegionBlock. Otherwise, if this
194 /// VPBlockBase is a VPBasicBlock, it is returned.
195 const VPBasicBlock *getExitingBasicBlock() const;
196 VPBasicBlock *getExitingBasicBlock();
197
198 const VPBlocksTy &getSuccessors() const { return Successors; }
199 VPBlocksTy &getSuccessors() { return Successors; }
200
203
204 const VPBlocksTy &getPredecessors() const { return Predecessors; }
205 VPBlocksTy &getPredecessors() { return Predecessors; }
206
207 /// \return the successor of this VPBlockBase if it has a single successor.
208 /// Otherwise return a null pointer.
210 return (Successors.size() == 1 ? *Successors.begin() : nullptr);
211 }
212
213 /// \return the predecessor of this VPBlockBase if it has a single
214 /// predecessor. Otherwise return a null pointer.
216 return (Predecessors.size() == 1 ? *Predecessors.begin() : nullptr);
217 }
218
219 size_t getNumSuccessors() const { return Successors.size(); }
220 size_t getNumPredecessors() const { return Predecessors.size(); }
221
222 /// Returns true if this block has any predecessors.
223 bool hasPredecessors() const { return !Predecessors.empty(); }
224
225 /// An Enclosing Block of a block B is any block containing B, including B
226 /// itself. \return the closest enclosing block starting from "this", which
227 /// has successors. \return the root enclosing block if all enclosing blocks
228 /// have no successors.
229 VPBlockBase *getEnclosingBlockWithSuccessors();
230
231 /// \return the closest enclosing block starting from "this", which has
232 /// predecessors. \return the root enclosing block if all enclosing blocks
233 /// have no predecessors.
234 VPBlockBase *getEnclosingBlockWithPredecessors();
235
236 /// \return the successors either attached directly to this VPBlockBase or, if
237 /// this VPBlockBase is the exit block of a VPRegionBlock and has no
238 /// successors of its own, search recursively for the first enclosing
239 /// VPRegionBlock that has successors and return them. If no such
240 /// VPRegionBlock exists, return the (empty) successors of the topmost
241 /// VPBlockBase reached.
243 return getEnclosingBlockWithSuccessors()->getSuccessors();
244 }
245
246 /// \return the hierarchical successor of this VPBlockBase if it has a single
247 /// hierarchical successor. Otherwise return a null pointer.
249 return getEnclosingBlockWithSuccessors()->getSingleSuccessor();
250 }
251
252 /// \return the predecessors either attached directly to this VPBlockBase or,
253 /// if this VPBlockBase is the entry block of a VPRegionBlock and has no
254 /// predecessors of its own, search recursively for the first enclosing
255 /// VPRegionBlock that has predecessors and return them. If no such
256 /// VPRegionBlock exists, return the (empty) predecessors of the topmost
257 /// VPBlockBase reached.
259 return getEnclosingBlockWithPredecessors()->getPredecessors();
260 }
261
262 /// \return the hierarchical predecessor of this VPBlockBase if it has a
263 /// single hierarchical predecessor. Otherwise return a null pointer.
267
268 /// Set a given VPBlockBase \p Successor as the single successor of this
269 /// VPBlockBase. This VPBlockBase is not added as predecessor of \p Successor.
270 /// This VPBlockBase must have no successors.
272 assert(Successors.empty() && "Setting one successor when others exist.");
273 assert(Successor->getParent() == getParent() &&
274 "connected blocks must have the same parent");
275 appendSuccessor(Successor);
276 }
277
278 /// Set two given VPBlockBases \p IfTrue and \p IfFalse to be the two
279 /// successors of this VPBlockBase. This VPBlockBase is not added as
280 /// predecessor of \p IfTrue or \p IfFalse. This VPBlockBase must have no
281 /// successors.
282 void setTwoSuccessors(VPBlockBase *IfTrue, VPBlockBase *IfFalse) {
283 assert(Successors.empty() && "Setting two successors when others exist.");
284 appendSuccessor(IfTrue);
285 appendSuccessor(IfFalse);
286 }
287
288 /// Set each VPBasicBlock in \p NewPreds as predecessor of this VPBlockBase.
289 /// This VPBlockBase must have no predecessors. This VPBlockBase is not added
290 /// as successor of any VPBasicBlock in \p NewPreds.
292 assert(Predecessors.empty() && "Block predecessors already set.");
293 for (auto *Pred : NewPreds)
294 appendPredecessor(Pred);
295 }
296
297 /// Set each VPBasicBlock in \p NewSuccss as successor of this VPBlockBase.
298 /// This VPBlockBase must have no successors. This VPBlockBase is not added
299 /// as predecessor of any VPBasicBlock in \p NewSuccs.
301 assert(Successors.empty() && "Block successors already set.");
302 for (auto *Succ : NewSuccs)
303 appendSuccessor(Succ);
304 }
305
306 /// Remove all the predecessor of this block.
307 void clearPredecessors() { Predecessors.clear(); }
308
309 /// Remove all the successors of this block.
310 void clearSuccessors() { Successors.clear(); }
311
312 /// Swap predecessors of the block. The block must have exactly 2
313 /// predecessors.
315 assert(Predecessors.size() == 2 && "must have 2 predecessors to swap");
316 std::swap(Predecessors[0], Predecessors[1]);
317 }
318
319 /// Swap successors of the block. The block must have exactly 2 successors.
320 // TODO: This should be part of introducing conditional branch recipes rather
321 // than being independent.
323 assert(Successors.size() == 2 && "must have 2 successors to swap");
324 std::swap(Successors[0], Successors[1]);
325 }
326
327 /// Returns the index for \p Pred in the blocks predecessors list.
328 unsigned getIndexForPredecessor(const VPBlockBase *Pred) const {
329 assert(count(Predecessors, Pred) == 1 &&
330 "must have Pred exactly once in Predecessors");
331 return std::distance(Predecessors.begin(), find(Predecessors, Pred));
332 }
333
334 /// Returns the index for \p Succ in the blocks successor list.
335 unsigned getIndexForSuccessor(const VPBlockBase *Succ) const {
336 assert(count(Successors, Succ) == 1 &&
337 "must have Succ exactly once in Successors");
338 return std::distance(Successors.begin(), find(Successors, Succ));
339 }
340
341 /// The method which generates the output IR that correspond to this
342 /// VPBlockBase, thereby "executing" the VPlan.
343 virtual void execute(VPTransformState *State) = 0;
344
345 /// Return the cost of the block.
347
348 /// Return true if it is legal to hoist instructions into this block.
350 // There are currently no constraints that prevent an instruction to be
351 // hoisted into a VPBlockBase.
352 return true;
353 }
354
355#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
356 void printAsOperand(raw_ostream &OS, bool PrintType = false) const {
357 OS << getName();
358 }
359
360 /// Print plain-text dump of this VPBlockBase to \p O, prefixing all lines
361 /// with \p Indent. \p SlotTracker is used to print unnamed VPValue's using
362 /// consequtive numbers.
363 ///
364 /// Note that the numbering is applied to the whole VPlan, so printing
365 /// individual blocks is consistent with the whole VPlan printing.
366 virtual void print(raw_ostream &O, const Twine &Indent,
367 VPSlotTracker &SlotTracker) const = 0;
368
369 /// Print plain-text dump of this VPlan to \p O.
370 void print(raw_ostream &O) const;
371
372 /// Print the successors of this block to \p O, prefixing all lines with \p
373 /// Indent.
374 void printSuccessors(raw_ostream &O, const Twine &Indent) const;
375
376 /// Dump this VPBlockBase to dbgs().
377 LLVM_DUMP_METHOD void dump() const { print(dbgs()); }
378#endif
379
380 /// Clone the current block and it's recipes without updating the operands of
381 /// the cloned recipes, including all blocks in the single-entry single-exit
382 /// region for VPRegionBlocks.
383 virtual VPBlockBase *clone() = 0;
384};
385
386/// VPRecipeBase is a base class modeling a sequence of one or more output IR
387/// instructions. VPRecipeBase owns the VPValues it defines through VPDef
388/// and is responsible for deleting its defined values. Single-value
389/// recipes must inherit from VPSingleDef instead of inheriting from both
390/// VPRecipeBase and VPValue separately.
392 : public ilist_node_with_parent<VPRecipeBase, VPBasicBlock>,
393 public VPDef,
394 public VPUser {
395 friend VPBasicBlock;
396 friend class VPBlockUtils;
397
398 /// Each VPRecipe belongs to a single VPBasicBlock.
399 VPBasicBlock *Parent = nullptr;
400
401 /// The debug location for the recipe.
402 DebugLoc DL;
403
404public:
405 VPRecipeBase(const unsigned char SC, ArrayRef<VPValue *> Operands,
407 : VPDef(SC), VPUser(Operands), DL(DL) {}
408
409 virtual ~VPRecipeBase() = default;
410
411 /// Clone the current recipe.
412 virtual VPRecipeBase *clone() = 0;
413
414 /// \return the VPBasicBlock which this VPRecipe belongs to.
415 VPBasicBlock *getParent() { return Parent; }
416 const VPBasicBlock *getParent() const { return Parent; }
417
418 /// The method which generates the output IR instructions that correspond to
419 /// this VPRecipe, thereby "executing" the VPlan.
420 virtual void execute(VPTransformState &State) = 0;
421
422 /// Return the cost of this recipe, taking into account if the cost
423 /// computation should be skipped and the ForceTargetInstructionCost flag.
424 /// Also takes care of printing the cost for debugging.
426
427 /// Insert an unlinked recipe into a basic block immediately before
428 /// the specified recipe.
429 void insertBefore(VPRecipeBase *InsertPos);
430 /// Insert an unlinked recipe into \p BB immediately before the insertion
431 /// point \p IP;
432 void insertBefore(VPBasicBlock &BB, iplist<VPRecipeBase>::iterator IP);
433
434 /// Insert an unlinked Recipe into a basic block immediately after
435 /// the specified Recipe.
436 void insertAfter(VPRecipeBase *InsertPos);
437
438 /// Unlink this recipe from its current VPBasicBlock and insert it into
439 /// the VPBasicBlock that MovePos lives in, right after MovePos.
440 void moveAfter(VPRecipeBase *MovePos);
441
442 /// Unlink this recipe and insert into BB before I.
443 ///
444 /// \pre I is a valid iterator into BB.
445 void moveBefore(VPBasicBlock &BB, iplist<VPRecipeBase>::iterator I);
446
447 /// This method unlinks 'this' from the containing basic block, but does not
448 /// delete it.
449 void removeFromParent();
450
451 /// This method unlinks 'this' from the containing basic block and deletes it.
452 ///
453 /// \returns an iterator pointing to the element after the erased one
455
456 /// Method to support type inquiry through isa, cast, and dyn_cast.
457 static inline bool classof(const VPDef *D) {
458 // All VPDefs are also VPRecipeBases.
459 return true;
460 }
461
462 static inline bool classof(const VPUser *U) { return true; }
463
464 /// Returns true if the recipe may have side-effects.
465 bool mayHaveSideEffects() const;
466
467 /// Returns true for PHI-like recipes.
468 bool isPhi() const;
469
470 /// Returns true if the recipe may read from memory.
471 bool mayReadFromMemory() const;
472
473 /// Returns true if the recipe may write to memory.
474 bool mayWriteToMemory() const;
475
476 /// Returns true if the recipe may read from or write to memory.
477 bool mayReadOrWriteMemory() const {
479 }
480
481 /// Returns the debug location of the recipe.
482 DebugLoc getDebugLoc() const { return DL; }
483
484 /// Return true if the recipe is a scalar cast.
485 bool isScalarCast() const;
486
487 /// Set the recipe's debug location to \p NewDL.
488 void setDebugLoc(DebugLoc NewDL) { DL = NewDL; }
489
490protected:
491 /// Compute the cost of this recipe either using a recipe's specialized
492 /// implementation or using the legacy cost model and the underlying
493 /// instructions.
494 virtual InstructionCost computeCost(ElementCount VF,
495 VPCostContext &Ctx) const;
496};
497
498// Helper macro to define common classof implementations for recipes.
499#define VP_CLASSOF_IMPL(VPDefID) \
500 static inline bool classof(const VPDef *D) { \
501 return D->getVPDefID() == VPDefID; \
502 } \
503 static inline bool classof(const VPValue *V) { \
504 auto *R = V->getDefiningRecipe(); \
505 return R && R->getVPDefID() == VPDefID; \
506 } \
507 static inline bool classof(const VPUser *U) { \
508 auto *R = dyn_cast<VPRecipeBase>(U); \
509 return R && R->getVPDefID() == VPDefID; \
510 } \
511 static inline bool classof(const VPRecipeBase *R) { \
512 return R->getVPDefID() == VPDefID; \
513 } \
514 static inline bool classof(const VPSingleDefRecipe *R) { \
515 return R->getVPDefID() == VPDefID; \
516 }
517
518/// VPSingleDef is a base class for recipes for modeling a sequence of one or
519/// more output IR that define a single result VPValue.
520/// Note that VPRecipeBase must be inherited from before VPValue.
521class VPSingleDefRecipe : public VPRecipeBase, public VPValue {
522public:
523 VPSingleDefRecipe(const unsigned char SC, ArrayRef<VPValue *> Operands,
525 : VPRecipeBase(SC, Operands, DL), VPValue(this) {}
526
527 VPSingleDefRecipe(const unsigned char SC, ArrayRef<VPValue *> Operands,
529 : VPRecipeBase(SC, Operands, DL), VPValue(this, UV) {}
530
531 static inline bool classof(const VPRecipeBase *R) {
532 switch (R->getVPDefID()) {
533 case VPRecipeBase::VPDerivedIVSC:
534 case VPRecipeBase::VPEVLBasedIVPHISC:
535 case VPRecipeBase::VPExpandSCEVSC:
536 case VPRecipeBase::VPExpressionSC:
537 case VPRecipeBase::VPInstructionSC:
538 case VPRecipeBase::VPReductionEVLSC:
539 case VPRecipeBase::VPReductionSC:
540 case VPRecipeBase::VPReplicateSC:
541 case VPRecipeBase::VPScalarIVStepsSC:
542 case VPRecipeBase::VPVectorPointerSC:
543 case VPRecipeBase::VPVectorEndPointerSC:
544 case VPRecipeBase::VPWidenCallSC:
545 case VPRecipeBase::VPWidenCanonicalIVSC:
546 case VPRecipeBase::VPWidenCastSC:
547 case VPRecipeBase::VPWidenGEPSC:
548 case VPRecipeBase::VPWidenIntrinsicSC:
549 case VPRecipeBase::VPWidenSC:
550 case VPRecipeBase::VPWidenSelectSC:
551 case VPRecipeBase::VPBlendSC:
552 case VPRecipeBase::VPPredInstPHISC:
553 case VPRecipeBase::VPCanonicalIVPHISC:
554 case VPRecipeBase::VPActiveLaneMaskPHISC:
555 case VPRecipeBase::VPFirstOrderRecurrencePHISC:
556 case VPRecipeBase::VPWidenPHISC:
557 case VPRecipeBase::VPWidenIntOrFpInductionSC:
558 case VPRecipeBase::VPWidenPointerInductionSC:
559 case VPRecipeBase::VPReductionPHISC:
560 case VPRecipeBase::VPPartialReductionSC:
561 return true;
562 case VPRecipeBase::VPBranchOnMaskSC:
563 case VPRecipeBase::VPInterleaveEVLSC:
564 case VPRecipeBase::VPInterleaveSC:
565 case VPRecipeBase::VPIRInstructionSC:
566 case VPRecipeBase::VPWidenLoadEVLSC:
567 case VPRecipeBase::VPWidenLoadSC:
568 case VPRecipeBase::VPWidenStoreEVLSC:
569 case VPRecipeBase::VPWidenStoreSC:
570 case VPRecipeBase::VPHistogramSC:
571 // TODO: Widened stores don't define a value, but widened loads do. Split
572 // the recipes to be able to make widened loads VPSingleDefRecipes.
573 return false;
574 }
575 llvm_unreachable("Unhandled VPDefID");
576 }
577
578 static inline bool classof(const VPUser *U) {
579 auto *R = dyn_cast<VPRecipeBase>(U);
580 return R && classof(R);
581 }
582
583 virtual VPSingleDefRecipe *clone() override = 0;
584
585 /// Returns the underlying instruction.
592
593#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
594 /// Print this VPSingleDefRecipe to dbgs() (for debugging).
595 LLVM_DUMP_METHOD void dump() const;
596#endif
597};
598
599/// Class to record and manage LLVM IR flags.
601 enum class OperationType : unsigned char {
602 Cmp,
603 OverflowingBinOp,
604 Trunc,
605 DisjointOp,
606 PossiblyExactOp,
607 GEPOp,
608 FPMathOp,
609 NonNegOp,
610 Other
611 };
612
613public:
614 struct WrapFlagsTy {
615 char HasNUW : 1;
616 char HasNSW : 1;
617
619 };
620
622 char HasNUW : 1;
623 char HasNSW : 1;
624
626 };
627
632
634 char NonNeg : 1;
635 NonNegFlagsTy(bool IsNonNeg) : NonNeg(IsNonNeg) {}
636 };
637
638private:
639 struct ExactFlagsTy {
640 char IsExact : 1;
641 };
642 struct FastMathFlagsTy {
643 char AllowReassoc : 1;
644 char NoNaNs : 1;
645 char NoInfs : 1;
646 char NoSignedZeros : 1;
647 char AllowReciprocal : 1;
648 char AllowContract : 1;
649 char ApproxFunc : 1;
650
651 LLVM_ABI_FOR_TEST FastMathFlagsTy(const FastMathFlags &FMF);
652 };
653
654 OperationType OpType;
655
656 union {
661 ExactFlagsTy ExactFlags;
664 FastMathFlagsTy FMFs;
665 unsigned AllFlags;
666 };
667
668public:
669 VPIRFlags() : OpType(OperationType::Other), AllFlags(0) {}
670
672 if (auto *Op = dyn_cast<CmpInst>(&I)) {
673 OpType = OperationType::Cmp;
674 CmpPredicate = Op->getPredicate();
675 } else if (auto *Op = dyn_cast<PossiblyDisjointInst>(&I)) {
676 OpType = OperationType::DisjointOp;
677 DisjointFlags.IsDisjoint = Op->isDisjoint();
678 } else if (auto *Op = dyn_cast<OverflowingBinaryOperator>(&I)) {
679 OpType = OperationType::OverflowingBinOp;
680 WrapFlags = {Op->hasNoUnsignedWrap(), Op->hasNoSignedWrap()};
681 } else if (auto *Op = dyn_cast<TruncInst>(&I)) {
682 OpType = OperationType::Trunc;
683 TruncFlags = {Op->hasNoUnsignedWrap(), Op->hasNoSignedWrap()};
684 } else if (auto *Op = dyn_cast<PossiblyExactOperator>(&I)) {
685 OpType = OperationType::PossiblyExactOp;
686 ExactFlags.IsExact = Op->isExact();
687 } else if (auto *GEP = dyn_cast<GetElementPtrInst>(&I)) {
688 OpType = OperationType::GEPOp;
689 GEPFlags = GEP->getNoWrapFlags();
690 } else if (auto *PNNI = dyn_cast<PossiblyNonNegInst>(&I)) {
691 OpType = OperationType::NonNegOp;
692 NonNegFlags.NonNeg = PNNI->hasNonNeg();
693 } else if (auto *Op = dyn_cast<FPMathOperator>(&I)) {
694 OpType = OperationType::FPMathOp;
695 FMFs = Op->getFastMathFlags();
696 } else {
697 OpType = OperationType::Other;
698 AllFlags = 0;
699 }
700 }
701
703 : OpType(OperationType::Cmp), CmpPredicate(Pred) {}
704
706 : OpType(OperationType::OverflowingBinOp), WrapFlags(WrapFlags) {}
707
709 : OpType(OperationType::Trunc), TruncFlags(TruncFlags) {}
710
711 VPIRFlags(FastMathFlags FMFs) : OpType(OperationType::FPMathOp), FMFs(FMFs) {}
712
714 : OpType(OperationType::DisjointOp), DisjointFlags(DisjointFlags) {}
715
717 : OpType(OperationType::NonNegOp), NonNegFlags(NonNegFlags) {}
718
720 : OpType(OperationType::GEPOp), GEPFlags(GEPFlags) {}
721
723 OpType = Other.OpType;
724 AllFlags = Other.AllFlags;
725 }
726
727 /// Only keep flags also present in \p Other. \p Other must have the same
728 /// OpType as the current object.
729 void intersectFlags(const VPIRFlags &Other);
730
731 /// Drop all poison-generating flags.
733 // NOTE: This needs to be kept in-sync with
734 // Instruction::dropPoisonGeneratingFlags.
735 switch (OpType) {
736 case OperationType::OverflowingBinOp:
737 WrapFlags.HasNUW = false;
738 WrapFlags.HasNSW = false;
739 break;
740 case OperationType::Trunc:
741 TruncFlags.HasNUW = false;
742 TruncFlags.HasNSW = false;
743 break;
744 case OperationType::DisjointOp:
745 DisjointFlags.IsDisjoint = false;
746 break;
747 case OperationType::PossiblyExactOp:
748 ExactFlags.IsExact = false;
749 break;
750 case OperationType::GEPOp:
752 break;
753 case OperationType::FPMathOp:
754 FMFs.NoNaNs = false;
755 FMFs.NoInfs = false;
756 break;
757 case OperationType::NonNegOp:
758 NonNegFlags.NonNeg = false;
759 break;
760 case OperationType::Cmp:
761 case OperationType::Other:
762 break;
763 }
764 }
765
766 /// Apply the IR flags to \p I.
767 void applyFlags(Instruction &I) const {
768 switch (OpType) {
769 case OperationType::OverflowingBinOp:
770 I.setHasNoUnsignedWrap(WrapFlags.HasNUW);
771 I.setHasNoSignedWrap(WrapFlags.HasNSW);
772 break;
773 case OperationType::Trunc:
774 I.setHasNoUnsignedWrap(TruncFlags.HasNUW);
775 I.setHasNoSignedWrap(TruncFlags.HasNSW);
776 break;
777 case OperationType::DisjointOp:
778 cast<PossiblyDisjointInst>(&I)->setIsDisjoint(DisjointFlags.IsDisjoint);
779 break;
780 case OperationType::PossiblyExactOp:
781 I.setIsExact(ExactFlags.IsExact);
782 break;
783 case OperationType::GEPOp:
784 cast<GetElementPtrInst>(&I)->setNoWrapFlags(GEPFlags);
785 break;
786 case OperationType::FPMathOp:
787 I.setHasAllowReassoc(FMFs.AllowReassoc);
788 I.setHasNoNaNs(FMFs.NoNaNs);
789 I.setHasNoInfs(FMFs.NoInfs);
790 I.setHasNoSignedZeros(FMFs.NoSignedZeros);
791 I.setHasAllowReciprocal(FMFs.AllowReciprocal);
792 I.setHasAllowContract(FMFs.AllowContract);
793 I.setHasApproxFunc(FMFs.ApproxFunc);
794 break;
795 case OperationType::NonNegOp:
796 I.setNonNeg(NonNegFlags.NonNeg);
797 break;
798 case OperationType::Cmp:
799 case OperationType::Other:
800 break;
801 }
802 }
803
805 assert(OpType == OperationType::Cmp &&
806 "recipe doesn't have a compare predicate");
807 return CmpPredicate;
808 }
809
811 assert(OpType == OperationType::Cmp &&
812 "recipe doesn't have a compare predicate");
813 CmpPredicate = Pred;
814 }
815
817
818 /// Returns true if the recipe has a comparison predicate.
819 bool hasPredicate() const { return OpType == OperationType::Cmp; }
820
821 /// Returns true if the recipe has fast-math flags.
822 bool hasFastMathFlags() const { return OpType == OperationType::FPMathOp; }
823
825
826 /// Returns true if the recipe has non-negative flag.
827 bool hasNonNegFlag() const { return OpType == OperationType::NonNegOp; }
828
829 bool isNonNeg() const {
830 assert(OpType == OperationType::NonNegOp &&
831 "recipe doesn't have a NNEG flag");
832 return NonNegFlags.NonNeg;
833 }
834
835 bool hasNoUnsignedWrap() const {
836 switch (OpType) {
837 case OperationType::OverflowingBinOp:
838 return WrapFlags.HasNUW;
839 case OperationType::Trunc:
840 return TruncFlags.HasNUW;
841 default:
842 llvm_unreachable("recipe doesn't have a NUW flag");
843 }
844 }
845
846 bool hasNoSignedWrap() const {
847 switch (OpType) {
848 case OperationType::OverflowingBinOp:
849 return WrapFlags.HasNSW;
850 case OperationType::Trunc:
851 return TruncFlags.HasNSW;
852 default:
853 llvm_unreachable("recipe doesn't have a NSW flag");
854 }
855 }
856
857 bool isDisjoint() const {
858 assert(OpType == OperationType::DisjointOp &&
859 "recipe cannot have a disjoing flag");
860 return DisjointFlags.IsDisjoint;
861 }
862
863#if !defined(NDEBUG)
864 /// Returns true if the set flags are valid for \p Opcode.
865 bool flagsValidForOpcode(unsigned Opcode) const;
866#endif
867
868#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
869 void printFlags(raw_ostream &O) const;
870#endif
871};
872
873/// A pure-virtual common base class for recipes defining a single VPValue and
874/// using IR flags.
876 VPRecipeWithIRFlags(const unsigned char SC, ArrayRef<VPValue *> Operands,
878 : VPSingleDefRecipe(SC, Operands, DL), VPIRFlags() {}
879
880 VPRecipeWithIRFlags(const unsigned char SC, ArrayRef<VPValue *> Operands,
881 Instruction &I)
882 : VPSingleDefRecipe(SC, Operands, &I, I.getDebugLoc()), VPIRFlags(I) {}
883
884 VPRecipeWithIRFlags(const unsigned char SC, ArrayRef<VPValue *> Operands,
885 const VPIRFlags &Flags,
887 : VPSingleDefRecipe(SC, Operands, DL), VPIRFlags(Flags) {}
888
889 static inline bool classof(const VPRecipeBase *R) {
890 return R->getVPDefID() == VPRecipeBase::VPInstructionSC ||
891 R->getVPDefID() == VPRecipeBase::VPWidenSC ||
892 R->getVPDefID() == VPRecipeBase::VPWidenGEPSC ||
893 R->getVPDefID() == VPRecipeBase::VPWidenCallSC ||
894 R->getVPDefID() == VPRecipeBase::VPWidenCastSC ||
895 R->getVPDefID() == VPRecipeBase::VPWidenIntrinsicSC ||
896 R->getVPDefID() == VPRecipeBase::VPWidenSelectSC ||
897 R->getVPDefID() == VPRecipeBase::VPReductionSC ||
898 R->getVPDefID() == VPRecipeBase::VPReductionEVLSC ||
899 R->getVPDefID() == VPRecipeBase::VPReplicateSC ||
900 R->getVPDefID() == VPRecipeBase::VPVectorEndPointerSC ||
901 R->getVPDefID() == VPRecipeBase::VPVectorPointerSC;
902 }
903
904 static inline bool classof(const VPUser *U) {
905 auto *R = dyn_cast<VPRecipeBase>(U);
906 return R && classof(R);
907 }
908
909 static inline bool classof(const VPValue *V) {
910 auto *R = dyn_cast_or_null<VPRecipeBase>(V->getDefiningRecipe());
911 return R && classof(R);
912 }
913
914 virtual VPRecipeWithIRFlags *clone() override = 0;
915
916 static inline bool classof(const VPSingleDefRecipe *U) {
917 auto *R = dyn_cast<VPRecipeBase>(U);
918 return R && classof(R);
919 }
920
921 void execute(VPTransformState &State) override = 0;
922
923 /// Compute the cost for this recipe for \p VF, using \p Opcode and \p Ctx.
925 VPCostContext &Ctx) const;
926};
927
928/// Helper to access the operand that contains the unroll part for this recipe
929/// after unrolling.
930template <unsigned PartOpIdx> class LLVM_ABI_FOR_TEST VPUnrollPartAccessor {
931protected:
932 /// Return the VPValue operand containing the unroll part or null if there is
933 /// no such operand.
934 VPValue *getUnrollPartOperand(const VPUser &U) const;
935
936 /// Return the unroll part.
937 unsigned getUnrollPart(const VPUser &U) const;
938};
939
940/// Helper to manage IR metadata for recipes. It filters out metadata that
941/// cannot be propagated.
944
945public:
947
948 /// Adds metatadata that can be preserved from the original instruction
949 /// \p I.
951
952 /// Adds metatadata that can be preserved from the original instruction
953 /// \p I and noalias metadata guaranteed by runtime checks using \p LVer.
955
956 /// Copy constructor for cloning.
957 VPIRMetadata(const VPIRMetadata &Other) : Metadata(Other.Metadata) {}
958
960 Metadata = Other.Metadata;
961 return *this;
962 }
963
964 /// Add all metadata to \p I.
965 void applyMetadata(Instruction &I) const;
966
967 /// Add metadata with kind \p Kind and \p Node.
968 void addMetadata(unsigned Kind, MDNode *Node) {
969 Metadata.emplace_back(Kind, Node);
970 }
971
972 /// Intersect this VPIRMetada object with \p MD, keeping only metadata
973 /// nodes that are common to both.
974 void intersect(const VPIRMetadata &MD);
975};
976
977/// This is a concrete Recipe that models a single VPlan-level instruction.
978/// While as any Recipe it may generate a sequence of IR instructions when
979/// executed, these instructions would always form a single-def expression as
980/// the VPInstruction is also a single def-use vertex.
982 public VPIRMetadata,
983 public VPUnrollPartAccessor<1> {
984 friend class VPlanSlp;
985
986public:
987 /// VPlan opcodes, extending LLVM IR with idiomatics instructions.
988 enum {
990 Instruction::OtherOpsEnd + 1, // Combines the incoming and previous
991 // values of a first-order recurrence.
995 // Creates a mask where each lane is active (true) whilst the current
996 // counter (first operand + index) is less than the second operand. i.e.
997 // mask[i] = icmpt ult (op0 + i), op1
998 // The size of the mask returned is VF * Multiplier (UF, third op).
1002 // Increment the canonical IV separately for each unrolled part.
1007 /// Given operands of (the same) struct type, creates a struct of fixed-
1008 /// width vectors each containing a struct field of all operands. The
1009 /// number of operands matches the element count of every vector.
1011 /// Creates a fixed-width vector containing all operands. The number of
1012 /// operands matches the vector element count.
1014 /// Compute the final result of a AnyOf reduction with select(cmp(),x,y),
1015 /// where one of (x,y) is loop invariant, and both x and y are integer type.
1019 // Extracts the last lane from its operand if it is a vector, or the last
1020 // part if scalar. In the latter case, the recipe will be removed during
1021 // unrolling.
1023 // Extracts the second-to-last lane from its operand or the second-to-last
1024 // part if it is scalar. In the latter case, the recipe will be removed
1025 // during unrolling.
1027 LogicalAnd, // Non-poison propagating logical And.
1028 // Add an offset in bytes (second operand) to a base pointer (first
1029 // operand). Only generates scalar values (either for the first lane only or
1030 // for all lanes, depending on its uses).
1032 // Add a vector offset in bytes (second operand) to a scalar base pointer
1033 // (first operand).
1035 // Returns a scalar boolean value, which is true if any lane of its
1036 // (boolean) vector operands is true. It produces the reduced value across
1037 // all unrolled iterations. Unrolling will add all copies of its original
1038 // operand as additional operands. AnyOf is poison-safe as all operands
1039 // will be frozen.
1041 // Calculates the first active lane index of the vector predicate operands.
1042 // It produces the lane index across all unrolled iterations. Unrolling will
1043 // add all copies of its original operand as additional operands.
1045
1046 // The opcodes below are used for VPInstructionWithType.
1047 //
1048 /// Scale the first operand (vector step) by the second operand
1049 /// (scalar-step). Casts both operands to the result type if needed.
1051 /// Start vector for reductions with 3 operands: the original start value,
1052 /// the identity value for the reduction and an integer indicating the
1053 /// scaling factor.
1055 // Creates a step vector starting from 0 to VF with a step of 1.
1057 /// Extracts a single lane (first operand) from a set of vector operands.
1058 /// The lane specifies an index into a vector formed by combining all vector
1059 /// operands (all operands after the first one).
1061 /// Explicit user for the resume phi of the canonical induction in the main
1062 /// VPlan, used by the epilogue vector loop.
1064 /// Returns the value for vscale.
1066 };
1067
1068 /// Returns true if this VPInstruction generates scalar values for all lanes.
1069 /// Most VPInstructions generate a single value per part, either vector or
1070 /// scalar. VPReplicateRecipe takes care of generating multiple (scalar)
1071 /// values per all lanes, stemming from an original ingredient. This method
1072 /// identifies the (rare) cases of VPInstructions that do so as well, w/o an
1073 /// underlying ingredient.
1074 bool doesGeneratePerAllLanes() const;
1075
1076private:
1077 typedef unsigned char OpcodeTy;
1078 OpcodeTy Opcode;
1079
1080 /// An optional name that can be used for the generated IR instruction.
1081 const std::string Name;
1082
1083 /// Returns true if we can generate a scalar for the first lane only if
1084 /// needed.
1085 bool canGenerateScalarForFirstLane() const;
1086
1087 /// Utility methods serving execute(): generates a single vector instance of
1088 /// the modeled instruction. \returns the generated value. . In some cases an
1089 /// existing value is returned rather than a generated one.
1090 Value *generate(VPTransformState &State);
1091
1092#if !defined(NDEBUG)
1093 /// Return the number of operands determined by the opcode of the
1094 /// VPInstruction. Returns -1u if the number of operands cannot be determined
1095 /// directly by the opcode.
1096 static unsigned getNumOperandsForOpcode(unsigned Opcode);
1097#endif
1098
1099public:
1101 DebugLoc DL = DebugLoc::getUnknown(), const Twine &Name = "")
1102 : VPRecipeWithIRFlags(VPDef::VPInstructionSC, Operands, DL),
1103 VPIRMetadata(), Opcode(Opcode), Name(Name.str()) {}
1104
1106 const VPIRFlags &Flags, DebugLoc DL = DebugLoc::getUnknown(),
1107 const Twine &Name = "");
1108
1109 VP_CLASSOF_IMPL(VPDef::VPInstructionSC)
1110
1111 VPInstruction *clone() override {
1113 auto *New = new VPInstruction(Opcode, Operands, *this, getDebugLoc(), Name);
1114 if (getUnderlyingValue())
1115 New->setUnderlyingValue(getUnderlyingInstr());
1116 return New;
1117 }
1118
1119 unsigned getOpcode() const { return Opcode; }
1120
1121 /// Generate the instruction.
1122 /// TODO: We currently execute only per-part unless a specific instance is
1123 /// provided.
1124 void execute(VPTransformState &State) override;
1125
1126 /// Return the cost of this VPInstruction.
1127 InstructionCost computeCost(ElementCount VF,
1128 VPCostContext &Ctx) const override;
1129
1130#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1131 /// Print the VPInstruction to \p O.
1132 void print(raw_ostream &O, const Twine &Indent,
1133 VPSlotTracker &SlotTracker) const override;
1134
1135 /// Print the VPInstruction to dbgs() (for debugging).
1136 LLVM_DUMP_METHOD void dump() const;
1137#endif
1138
1139 bool hasResult() const {
1140 // CallInst may or may not have a result, depending on the called function.
1141 // Conservatively return calls have results for now.
1142 switch (getOpcode()) {
1143 case Instruction::Ret:
1144 case Instruction::Br:
1145 case Instruction::Store:
1146 case Instruction::Switch:
1147 case Instruction::IndirectBr:
1148 case Instruction::Resume:
1149 case Instruction::CatchRet:
1150 case Instruction::Unreachable:
1151 case Instruction::Fence:
1152 case Instruction::AtomicRMW:
1155 return false;
1156 default:
1157 return true;
1158 }
1159 }
1160
1161 /// Returns true if the underlying opcode may read from or write to memory.
1162 bool opcodeMayReadOrWriteFromMemory() const;
1163
1164 /// Returns true if the recipe only uses the first lane of operand \p Op.
1165 bool onlyFirstLaneUsed(const VPValue *Op) const override;
1166
1167 /// Returns true if the recipe only uses the first part of operand \p Op.
1168 bool onlyFirstPartUsed(const VPValue *Op) const override;
1169
1170 /// Returns true if this VPInstruction produces a scalar value from a vector,
1171 /// e.g. by performing a reduction or extracting a lane.
1172 bool isVectorToScalar() const;
1173
1174 /// Returns true if this VPInstruction's operands are single scalars and the
1175 /// result is also a single scalar.
1176 bool isSingleScalar() const;
1177
1178 /// Returns the symbolic name assigned to the VPInstruction.
1179 StringRef getName() const { return Name; }
1180};
1181
1182/// A specialization of VPInstruction augmenting it with a dedicated result
1183/// type, to be used when the opcode and operands of the VPInstruction don't
1184/// directly determine the result type. Note that there is no separate VPDef ID
1185/// for VPInstructionWithType; it shares the same ID as VPInstruction and is
1186/// distinguished purely by the opcode.
1188 /// Scalar result type produced by the recipe.
1189 Type *ResultTy;
1190
1191public:
1193 Type *ResultTy, const VPIRFlags &Flags, DebugLoc DL,
1194 const Twine &Name = "")
1195 : VPInstruction(Opcode, Operands, Flags, DL, Name), ResultTy(ResultTy) {}
1196
1197 static inline bool classof(const VPRecipeBase *R) {
1198 // VPInstructionWithType are VPInstructions with specific opcodes requiring
1199 // type information.
1200 if (R->isScalarCast())
1201 return true;
1202 auto *VPI = dyn_cast<VPInstruction>(R);
1203 if (!VPI)
1204 return false;
1205 switch (VPI->getOpcode()) {
1209 return true;
1210 default:
1211 return false;
1212 }
1213 }
1214
1215 static inline bool classof(const VPUser *R) {
1217 }
1218
1219 VPInstruction *clone() override {
1221 auto *New =
1223 getDebugLoc(), getName());
1224 New->setUnderlyingValue(getUnderlyingValue());
1225 return New;
1226 }
1227
1228 void execute(VPTransformState &State) override;
1229
1230 /// Return the cost of this VPInstruction.
1232 VPCostContext &Ctx) const override {
1233 // TODO: Compute accurate cost after retiring the legacy cost model.
1234 return 0;
1235 }
1236
1237 Type *getResultType() const { return ResultTy; }
1238
1239#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1240 /// Print the recipe.
1241 void print(raw_ostream &O, const Twine &Indent,
1242 VPSlotTracker &SlotTracker) const override;
1243#endif
1244};
1245
1246/// Helper type to provide functions to access incoming values and blocks for
1247/// phi-like recipes.
1249protected:
1250 /// Return a VPRecipeBase* to the current object.
1251 virtual const VPRecipeBase *getAsRecipe() const = 0;
1252
1253public:
1254 virtual ~VPPhiAccessors() = default;
1255
1256 /// Returns the incoming VPValue with index \p Idx.
1257 VPValue *getIncomingValue(unsigned Idx) const {
1258 return getAsRecipe()->getOperand(Idx);
1259 }
1260
1261 /// Returns the incoming block with index \p Idx.
1262 const VPBasicBlock *getIncomingBlock(unsigned Idx) const;
1263
1264 /// Returns the number of incoming values, also number of incoming blocks.
1265 virtual unsigned getNumIncoming() const {
1266 return getAsRecipe()->getNumOperands();
1267 }
1268
1269 /// Returns an interator range over the incoming values.
1271 return make_range(getAsRecipe()->op_begin(),
1272 getAsRecipe()->op_begin() + getNumIncoming());
1273 }
1274
1276 detail::index_iterator, std::function<const VPBasicBlock *(size_t)>>>;
1277
1278 /// Returns an iterator range over the incoming blocks.
1280 std::function<const VPBasicBlock *(size_t)> GetBlock = [this](size_t Idx) {
1281 return getIncomingBlock(Idx);
1282 };
1283 return map_range(index_range(0, getNumIncoming()), GetBlock);
1284 }
1285
1286 /// Returns an iterator range over pairs of incoming values and corresponding
1287 /// incoming blocks.
1293
1294 /// Removes the incoming value for \p IncomingBlock, which must be a
1295 /// predecessor.
1296 void removeIncomingValueFor(VPBlockBase *IncomingBlock) const;
1297
1298#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1299 /// Print the recipe.
1301#endif
1302};
1303
1307
1308 static inline bool classof(const VPUser *U) {
1309 auto *VPI = dyn_cast<VPInstruction>(U);
1310 return VPI && VPI->getOpcode() == Instruction::PHI;
1311 }
1312
1313 static inline bool classof(const VPValue *V) {
1314 auto *VPI = dyn_cast<VPInstruction>(V);
1315 return VPI && VPI->getOpcode() == Instruction::PHI;
1316 }
1317
1318 static inline bool classof(const VPSingleDefRecipe *SDR) {
1319 auto *VPI = dyn_cast<VPInstruction>(SDR);
1320 return VPI && VPI->getOpcode() == Instruction::PHI;
1321 }
1322
1323 VPPhi *clone() override {
1324 auto *PhiR = new VPPhi(operands(), getDebugLoc(), getName());
1325 PhiR->setUnderlyingValue(getUnderlyingValue());
1326 return PhiR;
1327 }
1328
1329 void execute(VPTransformState &State) override;
1330
1331#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1332 /// Print the recipe.
1333 void print(raw_ostream &O, const Twine &Indent,
1334 VPSlotTracker &SlotTracker) const override;
1335#endif
1336
1337protected:
1338 const VPRecipeBase *getAsRecipe() const override { return this; }
1339};
1340
1341/// A recipe to wrap on original IR instruction not to be modified during
1342/// execution, except for PHIs. PHIs are modeled via the VPIRPhi subclass.
1343/// Expect PHIs, VPIRInstructions cannot have any operands.
1345 Instruction &I;
1346
1347protected:
1348 /// VPIRInstruction::create() should be used to create VPIRInstructions, as
1349 /// subclasses may need to be created, e.g. VPIRPhi.
1351 : VPRecipeBase(VPDef::VPIRInstructionSC, ArrayRef<VPValue *>()), I(I) {}
1352
1353public:
1354 ~VPIRInstruction() override = default;
1355
1356 /// Create a new VPIRPhi for \p \I, if it is a PHINode, otherwise create a
1357 /// VPIRInstruction.
1359
1360 VP_CLASSOF_IMPL(VPDef::VPIRInstructionSC)
1361
1363 auto *R = create(I);
1364 for (auto *Op : operands())
1365 R->addOperand(Op);
1366 return R;
1367 }
1368
1369 void execute(VPTransformState &State) override;
1370
1371 /// Return the cost of this VPIRInstruction.
1373 computeCost(ElementCount VF, VPCostContext &Ctx) const override;
1374
1375 Instruction &getInstruction() const { return I; }
1376
1377#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1378 /// Print the recipe.
1379 void print(raw_ostream &O, const Twine &Indent,
1380 VPSlotTracker &SlotTracker) const override;
1381#endif
1382
1383 bool usesScalars(const VPValue *Op) const override {
1385 "Op must be an operand of the recipe");
1386 return true;
1387 }
1388
1389 bool onlyFirstPartUsed(const VPValue *Op) const override {
1391 "Op must be an operand of the recipe");
1392 return true;
1393 }
1394
1395 bool onlyFirstLaneUsed(const VPValue *Op) const override {
1397 "Op must be an operand of the recipe");
1398 return true;
1399 }
1400
1401 /// Update the recipes first operand to the last lane of the operand using \p
1402 /// Builder. Must only be used for VPIRInstructions with at least one operand
1403 /// wrapping a PHINode.
1405};
1406
1407/// An overlay for VPIRInstructions wrapping PHI nodes enabling convenient use
1408/// cast/dyn_cast/isa and execute() implementation. A single VPValue operand is
1409/// allowed, and it is used to add a new incoming value for the single
1410/// predecessor VPBB.
1412 public VPPhiAccessors {
1414
1415 static inline bool classof(const VPRecipeBase *U) {
1416 auto *R = dyn_cast<VPIRInstruction>(U);
1417 return R && isa<PHINode>(R->getInstruction());
1418 }
1419
1421
1422 void execute(VPTransformState &State) override;
1423
1424#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1425 /// Print the recipe.
1426 void print(raw_ostream &O, const Twine &Indent,
1427 VPSlotTracker &SlotTracker) const override;
1428#endif
1429
1430protected:
1431 const VPRecipeBase *getAsRecipe() const override { return this; }
1432};
1433
1434/// VPWidenRecipe is a recipe for producing a widened instruction using the
1435/// opcode and operands of the recipe. This recipe covers most of the
1436/// traditional vectorization cases where each recipe transforms into a
1437/// vectorized version of itself.
1439 public VPIRMetadata {
1440 unsigned Opcode;
1441
1442public:
1444 const VPIRFlags &Flags, const VPIRMetadata &Metadata,
1445 DebugLoc DL)
1446 : VPRecipeWithIRFlags(VPDef::VPWidenSC, Operands, Flags, DL),
1447 VPIRMetadata(Metadata), Opcode(Opcode) {}
1448
1452
1453 ~VPWidenRecipe() override = default;
1454
1455 VPWidenRecipe *clone() override {
1456 auto *R =
1457 new VPWidenRecipe(getOpcode(), operands(), *this, *this, getDebugLoc());
1458 R->setUnderlyingValue(getUnderlyingValue());
1459 return R;
1460 }
1461
1462 VP_CLASSOF_IMPL(VPDef::VPWidenSC)
1463
1464 /// Produce a widened instruction using the opcode and operands of the recipe,
1465 /// processing State.VF elements.
1466 void execute(VPTransformState &State) override;
1467
1468 /// Return the cost of this VPWidenRecipe.
1469 InstructionCost computeCost(ElementCount VF,
1470 VPCostContext &Ctx) const override;
1471
1472 unsigned getOpcode() const { return Opcode; }
1473
1474#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1475 /// Print the recipe.
1476 void print(raw_ostream &O, const Twine &Indent,
1477 VPSlotTracker &SlotTracker) const override;
1478#endif
1479};
1480
1481/// VPWidenCastRecipe is a recipe to create vector cast instructions.
1483 /// Cast instruction opcode.
1484 Instruction::CastOps Opcode;
1485
1486 /// Result type for the cast.
1487 Type *ResultTy;
1488
1489public:
1491 CastInst &UI)
1492 : VPRecipeWithIRFlags(VPDef::VPWidenCastSC, Op, UI), VPIRMetadata(UI),
1493 Opcode(Opcode), ResultTy(ResultTy) {
1494 assert(UI.getOpcode() == Opcode &&
1495 "opcode of underlying cast doesn't match");
1496 }
1497
1499 const VPIRFlags &Flags = {},
1500 const VPIRMetadata &Metadata = {},
1502 : VPRecipeWithIRFlags(VPDef::VPWidenCastSC, Op, Flags, DL),
1503 VPIRMetadata(Metadata), Opcode(Opcode), ResultTy(ResultTy) {
1504 assert(flagsValidForOpcode(Opcode) &&
1505 "Set flags not supported for the provided opcode");
1506 }
1507
1508 ~VPWidenCastRecipe() override = default;
1509
1511 auto *New = new VPWidenCastRecipe(Opcode, getOperand(0), ResultTy, *this,
1512 *this, getDebugLoc());
1513 if (auto *UV = getUnderlyingValue())
1514 New->setUnderlyingValue(UV);
1515 return New;
1516 }
1517
1518 VP_CLASSOF_IMPL(VPDef::VPWidenCastSC)
1519
1520 /// Produce widened copies of the cast.
1521 void execute(VPTransformState &State) override;
1522
1523 /// Return the cost of this VPWidenCastRecipe.
1525 VPCostContext &Ctx) const override;
1526
1527#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1528 /// Print the recipe.
1529 void print(raw_ostream &O, const Twine &Indent,
1530 VPSlotTracker &SlotTracker) const override;
1531#endif
1532
1533 Instruction::CastOps getOpcode() const { return Opcode; }
1534
1535 /// Returns the result type of the cast.
1536 Type *getResultType() const { return ResultTy; }
1537};
1538
1539/// A recipe for widening vector intrinsics.
1541 /// ID of the vector intrinsic to widen.
1542 Intrinsic::ID VectorIntrinsicID;
1543
1544 /// Scalar return type of the intrinsic.
1545 Type *ResultTy;
1546
1547 /// True if the intrinsic may read from memory.
1548 bool MayReadFromMemory;
1549
1550 /// True if the intrinsic may read write to memory.
1551 bool MayWriteToMemory;
1552
1553 /// True if the intrinsic may have side-effects.
1554 bool MayHaveSideEffects;
1555
1556public:
1558 ArrayRef<VPValue *> CallArguments, Type *Ty,
1560 : VPRecipeWithIRFlags(VPDef::VPWidenIntrinsicSC, CallArguments, CI),
1561 VPIRMetadata(CI), VectorIntrinsicID(VectorIntrinsicID), ResultTy(Ty),
1562 MayReadFromMemory(CI.mayReadFromMemory()),
1563 MayWriteToMemory(CI.mayWriteToMemory()),
1564 MayHaveSideEffects(CI.mayHaveSideEffects()) {}
1565
1567 ArrayRef<VPValue *> CallArguments, Type *Ty,
1569 : VPRecipeWithIRFlags(VPDef::VPWidenIntrinsicSC, CallArguments, DL),
1570 VPIRMetadata(), VectorIntrinsicID(VectorIntrinsicID), ResultTy(Ty) {
1571 LLVMContext &Ctx = Ty->getContext();
1572 AttributeSet Attrs = Intrinsic::getFnAttributes(Ctx, VectorIntrinsicID);
1573 MemoryEffects ME = Attrs.getMemoryEffects();
1574 MayReadFromMemory = !ME.onlyWritesMemory();
1575 MayWriteToMemory = !ME.onlyReadsMemory();
1576 MayHaveSideEffects = MayWriteToMemory ||
1577 !Attrs.hasAttribute(Attribute::NoUnwind) ||
1578 !Attrs.hasAttribute(Attribute::WillReturn);
1579 }
1580
1581 ~VPWidenIntrinsicRecipe() override = default;
1582
1584 if (Value *CI = getUnderlyingValue())
1585 return new VPWidenIntrinsicRecipe(*cast<CallInst>(CI), VectorIntrinsicID,
1586 operands(), ResultTy, getDebugLoc());
1587 return new VPWidenIntrinsicRecipe(VectorIntrinsicID, operands(), ResultTy,
1588 getDebugLoc());
1589 }
1590
1591 VP_CLASSOF_IMPL(VPDef::VPWidenIntrinsicSC)
1592
1593 /// Produce a widened version of the vector intrinsic.
1594 void execute(VPTransformState &State) override;
1595
1596 /// Return the cost of this vector intrinsic.
1598 VPCostContext &Ctx) const override;
1599
1600 /// Return the ID of the intrinsic.
1601 Intrinsic::ID getVectorIntrinsicID() const { return VectorIntrinsicID; }
1602
1603 /// Return the scalar return type of the intrinsic.
1604 Type *getResultType() const { return ResultTy; }
1605
1606 /// Return to name of the intrinsic as string.
1608
1609 /// Returns true if the intrinsic may read from memory.
1610 bool mayReadFromMemory() const { return MayReadFromMemory; }
1611
1612 /// Returns true if the intrinsic may write to memory.
1613 bool mayWriteToMemory() const { return MayWriteToMemory; }
1614
1615 /// Returns true if the intrinsic may have side-effects.
1616 bool mayHaveSideEffects() const { return MayHaveSideEffects; }
1617
1618#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1619 /// Print the recipe.
1620 void print(raw_ostream &O, const Twine &Indent,
1621 VPSlotTracker &SlotTracker) const override;
1622#endif
1623
1624 bool onlyFirstLaneUsed(const VPValue *Op) const override;
1625};
1626
1627/// A recipe for widening Call instructions using library calls.
1629 public VPIRMetadata {
1630 /// Variant stores a pointer to the chosen function. There is a 1:1 mapping
1631 /// between a given VF and the chosen vectorized variant, so there will be a
1632 /// different VPlan for each VF with a valid variant.
1633 Function *Variant;
1634
1635public:
1637 ArrayRef<VPValue *> CallArguments,
1639 : VPRecipeWithIRFlags(VPDef::VPWidenCallSC, CallArguments,
1640 *cast<Instruction>(UV)),
1641 VPIRMetadata(*cast<Instruction>(UV)), Variant(Variant) {
1642 assert(
1644 "last operand must be the called function");
1645 }
1646
1647 ~VPWidenCallRecipe() override = default;
1648
1650 return new VPWidenCallRecipe(getUnderlyingValue(), Variant, operands(),
1651 getDebugLoc());
1652 }
1653
1654 VP_CLASSOF_IMPL(VPDef::VPWidenCallSC)
1655
1656 /// Produce a widened version of the call instruction.
1657 void execute(VPTransformState &State) override;
1658
1659 /// Return the cost of this VPWidenCallRecipe.
1660 InstructionCost computeCost(ElementCount VF,
1661 VPCostContext &Ctx) const override;
1662
1666
1669
1670#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1671 /// Print the recipe.
1672 void print(raw_ostream &O, const Twine &Indent,
1673 VPSlotTracker &SlotTracker) const override;
1674#endif
1675};
1676
1677/// A recipe representing a sequence of load -> update -> store as part of
1678/// a histogram operation. This means there may be aliasing between vector
1679/// lanes, which is handled by the llvm.experimental.vector.histogram family
1680/// of intrinsics. The only update operations currently supported are
1681/// 'add' and 'sub' where the other term is loop-invariant.
1683 /// Opcode of the update operation, currently either add or sub.
1684 unsigned Opcode;
1685
1686public:
1687 VPHistogramRecipe(unsigned Opcode, ArrayRef<VPValue *> Operands,
1689 : VPRecipeBase(VPDef::VPHistogramSC, Operands, DL), Opcode(Opcode) {}
1690
1691 ~VPHistogramRecipe() override = default;
1692
1694 return new VPHistogramRecipe(Opcode, operands(), getDebugLoc());
1695 }
1696
1697 VP_CLASSOF_IMPL(VPDef::VPHistogramSC);
1698
1699 /// Produce a vectorized histogram operation.
1700 void execute(VPTransformState &State) override;
1701
1702 /// Return the cost of this VPHistogramRecipe.
1704 VPCostContext &Ctx) const override;
1705
1706 unsigned getOpcode() const { return Opcode; }
1707
1708 /// Return the mask operand if one was provided, or a null pointer if all
1709 /// lanes should be executed unconditionally.
1710 VPValue *getMask() const {
1711 return getNumOperands() == 3 ? getOperand(2) : nullptr;
1712 }
1713
1714#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1715 /// Print the recipe
1716 void print(raw_ostream &O, const Twine &Indent,
1717 VPSlotTracker &SlotTracker) const override;
1718#endif
1719};
1720
1721/// A recipe for widening select instructions.
1723 public VPIRMetadata {
1727
1728 ~VPWidenSelectRecipe() override = default;
1729
1734
1735 VP_CLASSOF_IMPL(VPDef::VPWidenSelectSC)
1736
1737 /// Produce a widened version of the select instruction.
1738 void execute(VPTransformState &State) override;
1739
1740 /// Return the cost of this VPWidenSelectRecipe.
1741 InstructionCost computeCost(ElementCount VF,
1742 VPCostContext &Ctx) const override;
1743
1744#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1745 /// Print the recipe.
1746 void print(raw_ostream &O, const Twine &Indent,
1747 VPSlotTracker &SlotTracker) const override;
1748#endif
1749
1750 unsigned getOpcode() const { return Instruction::Select; }
1751
1752 VPValue *getCond() const {
1753 return getOperand(0);
1754 }
1755
1756 bool isInvariantCond() const {
1757 return getCond()->isDefinedOutsideLoopRegions();
1758 }
1759
1760 /// Returns true if the recipe only uses the first lane of operand \p Op.
1761 bool onlyFirstLaneUsed(const VPValue *Op) const override {
1763 "Op must be an operand of the recipe");
1764 return Op == getCond() && isInvariantCond();
1765 }
1766};
1767
1768/// A recipe for handling GEP instructions.
1770 Type *SourceElementTy;
1771
1772 bool isPointerLoopInvariant() const {
1773 return getOperand(0)->isDefinedOutsideLoopRegions();
1774 }
1775
1776 bool isIndexLoopInvariant(unsigned I) const {
1777 return getOperand(I + 1)->isDefinedOutsideLoopRegions();
1778 }
1779
1780 bool areAllOperandsInvariant() const {
1781 return all_of(operands(), [](VPValue *Op) {
1782 return Op->isDefinedOutsideLoopRegions();
1783 });
1784 }
1785
1786public:
1788 : VPRecipeWithIRFlags(VPDef::VPWidenGEPSC, Operands, *GEP),
1789 SourceElementTy(GEP->getSourceElementType()) {
1791 (void)Metadata;
1793 assert(Metadata.empty() && "unexpected metadata on GEP");
1794 }
1795
1796 ~VPWidenGEPRecipe() override = default;
1797
1802
1803 VP_CLASSOF_IMPL(VPDef::VPWidenGEPSC)
1804
1805 /// This recipe generates a GEP instruction.
1806 unsigned getOpcode() const { return Instruction::GetElementPtr; }
1807
1808 /// Generate the gep nodes.
1809 void execute(VPTransformState &State) override;
1810
1811 Type *getSourceElementType() const { return SourceElementTy; }
1812
1813 /// Return the cost of this VPWidenGEPRecipe.
1815 VPCostContext &Ctx) const override {
1816 // TODO: Compute accurate cost after retiring the legacy cost model.
1817 return 0;
1818 }
1819
1820#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1821 /// Print the recipe.
1822 void print(raw_ostream &O, const Twine &Indent,
1823 VPSlotTracker &SlotTracker) const override;
1824#endif
1825
1826 /// Returns true if the recipe only uses the first lane of operand \p Op.
1827 bool onlyFirstLaneUsed(const VPValue *Op) const override {
1829 "Op must be an operand of the recipe");
1830 if (Op == getOperand(0))
1831 return isPointerLoopInvariant();
1832 else
1833 return !isPointerLoopInvariant() && Op->isDefinedOutsideLoopRegions();
1834 }
1835};
1836
1837/// A recipe to compute a pointer to the last element of each part of a widened
1838/// memory access for widened memory accesses of IndexedTy. Used for
1839/// VPWidenMemoryRecipes or VPInterleaveRecipes that are reversed.
1841 public VPUnrollPartAccessor<2> {
1842 Type *IndexedTy;
1843
1844 /// The constant stride of the pointer computed by this recipe, expressed in
1845 /// units of IndexedTy.
1846 int64_t Stride;
1847
1848public:
1850 int64_t Stride, GEPNoWrapFlags GEPFlags, DebugLoc DL)
1851 : VPRecipeWithIRFlags(VPDef::VPVectorEndPointerSC,
1852 ArrayRef<VPValue *>({Ptr, VF}), GEPFlags, DL),
1853 IndexedTy(IndexedTy), Stride(Stride) {
1854 assert(Stride < 0 && "Stride must be negative");
1855 }
1856
1857 VP_CLASSOF_IMPL(VPDef::VPVectorEndPointerSC)
1858
1860 const VPValue *getVFValue() const { return getOperand(1); }
1861
1862 void execute(VPTransformState &State) override;
1863
1864 bool onlyFirstLaneUsed(const VPValue *Op) const override {
1866 "Op must be an operand of the recipe");
1867 return true;
1868 }
1869
1870 /// Return the cost of this VPVectorPointerRecipe.
1872 VPCostContext &Ctx) const override {
1873 // TODO: Compute accurate cost after retiring the legacy cost model.
1874 return 0;
1875 }
1876
1877 /// Returns true if the recipe only uses the first part of operand \p Op.
1878 bool onlyFirstPartUsed(const VPValue *Op) const override {
1880 "Op must be an operand of the recipe");
1881 assert(getNumOperands() <= 2 && "must have at most two operands");
1882 return true;
1883 }
1884
1886 return new VPVectorEndPointerRecipe(getOperand(0), getVFValue(), IndexedTy,
1887 Stride, getGEPNoWrapFlags(),
1888 getDebugLoc());
1889 }
1890
1891#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1892 /// Print the recipe.
1893 void print(raw_ostream &O, const Twine &Indent,
1894 VPSlotTracker &SlotTracker) const override;
1895#endif
1896};
1897
1898/// A recipe to compute the pointers for widened memory accesses of IndexTy.
1900 public VPUnrollPartAccessor<1> {
1901 Type *SourceElementTy;
1902
1903public:
1906 : VPRecipeWithIRFlags(VPDef::VPVectorPointerSC, ArrayRef<VPValue *>(Ptr),
1907 GEPFlags, DL),
1908 SourceElementTy(SourceElementTy) {}
1909
1910 VP_CLASSOF_IMPL(VPDef::VPVectorPointerSC)
1911
1912 void execute(VPTransformState &State) override;
1913
1914 Type *getSourceElementType() const { return SourceElementTy; }
1915
1916 bool onlyFirstLaneUsed(const VPValue *Op) const override {
1918 "Op must be an operand of the recipe");
1919 return true;
1920 }
1921
1922 /// Returns true if the recipe only uses the first part of operand \p Op.
1923 bool onlyFirstPartUsed(const VPValue *Op) const override {
1925 "Op must be an operand of the recipe");
1926 assert(getNumOperands() <= 2 && "must have at most two operands");
1927 return true;
1928 }
1929
1931 return new VPVectorPointerRecipe(getOperand(0), SourceElementTy,
1933 }
1934
1935 /// Return true if this VPVectorPointerRecipe corresponds to part 0. Note that
1936 /// this is only accurate after the VPlan has been unrolled.
1937 bool isFirstPart() const { return getUnrollPart(*this) == 0; }
1938
1939 /// Return the cost of this VPHeaderPHIRecipe.
1941 VPCostContext &Ctx) const override {
1942 // TODO: Compute accurate cost after retiring the legacy cost model.
1943 return 0;
1944 }
1945
1946#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1947 /// Print the recipe.
1948 void print(raw_ostream &O, const Twine &Indent,
1949 VPSlotTracker &SlotTracker) const override;
1950#endif
1951};
1952
1953/// A pure virtual base class for all recipes modeling header phis, including
1954/// phis for first order recurrences, pointer inductions and reductions. The
1955/// start value is the first operand of the recipe and the incoming value from
1956/// the backedge is the second operand.
1957///
1958/// Inductions are modeled using the following sub-classes:
1959/// * VPCanonicalIVPHIRecipe: Canonical scalar induction of the vector loop,
1960/// starting at a specified value (zero for the main vector loop, the resume
1961/// value for the epilogue vector loop) and stepping by 1. The induction
1962/// controls exiting of the vector loop by comparing against the vector trip
1963/// count. Produces a single scalar PHI for the induction value per
1964/// iteration.
1965/// * VPWidenIntOrFpInductionRecipe: Generates vector values for integer and
1966/// floating point inductions with arbitrary start and step values. Produces
1967/// a vector PHI per-part.
1968/// * VPDerivedIVRecipe: Converts the canonical IV value to the corresponding
1969/// value of an IV with different start and step values. Produces a single
1970/// scalar value per iteration
1971/// * VPScalarIVStepsRecipe: Generates scalar values per-lane based on a
1972/// canonical or derived induction.
1973/// * VPWidenPointerInductionRecipe: Generate vector and scalar values for a
1974/// pointer induction. Produces either a vector PHI per-part or scalar values
1975/// per-lane based on the canonical induction.
1977 public VPPhiAccessors {
1978protected:
1979 VPHeaderPHIRecipe(unsigned char VPDefID, Instruction *UnderlyingInstr,
1980 VPValue *Start, DebugLoc DL = DebugLoc::getUnknown())
1981 : VPSingleDefRecipe(VPDefID, ArrayRef<VPValue *>({Start}),
1982 UnderlyingInstr, DL) {}
1983
1984 const VPRecipeBase *getAsRecipe() const override { return this; }
1985
1986public:
1987 ~VPHeaderPHIRecipe() override = default;
1988
1989 /// Method to support type inquiry through isa, cast, and dyn_cast.
1990 static inline bool classof(const VPRecipeBase *B) {
1991 return B->getVPDefID() >= VPDef::VPFirstHeaderPHISC &&
1992 B->getVPDefID() <= VPDef::VPLastHeaderPHISC;
1993 }
1994 static inline bool classof(const VPValue *V) {
1995 auto *B = V->getDefiningRecipe();
1996 return B && B->getVPDefID() >= VPRecipeBase::VPFirstHeaderPHISC &&
1997 B->getVPDefID() <= VPRecipeBase::VPLastHeaderPHISC;
1998 }
1999
2000 /// Generate the phi nodes.
2001 void execute(VPTransformState &State) override = 0;
2002
2003 /// Return the cost of this header phi recipe.
2005 VPCostContext &Ctx) const override;
2006
2007#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2008 /// Print the recipe.
2009 void print(raw_ostream &O, const Twine &Indent,
2010 VPSlotTracker &SlotTracker) const override = 0;
2011#endif
2012
2013 /// Returns the start value of the phi, if one is set.
2015 return getNumOperands() == 0 ? nullptr : getOperand(0);
2016 }
2018 return getNumOperands() == 0 ? nullptr : getOperand(0);
2019 }
2020
2021 /// Update the start value of the recipe.
2023
2024 /// Returns the incoming value from the loop backedge.
2026 return getOperand(1);
2027 }
2028
2029 /// Update the incoming value from the loop backedge.
2031
2032 /// Returns the backedge value as a recipe. The backedge value is guaranteed
2033 /// to be a recipe.
2035 return *getBackedgeValue()->getDefiningRecipe();
2036 }
2037};
2038
2039/// Base class for widened induction (VPWidenIntOrFpInductionRecipe and
2040/// VPWidenPointerInductionRecipe), providing shared functionality, including
2041/// retrieving the step value, induction descriptor and original phi node.
2043 const InductionDescriptor &IndDesc;
2044
2045public:
2046 VPWidenInductionRecipe(unsigned char Kind, PHINode *IV, VPValue *Start,
2047 VPValue *Step, const InductionDescriptor &IndDesc,
2048 DebugLoc DL)
2049 : VPHeaderPHIRecipe(Kind, IV, Start, DL), IndDesc(IndDesc) {
2050 addOperand(Step);
2051 }
2052
2053 static inline bool classof(const VPRecipeBase *R) {
2054 return R->getVPDefID() == VPDef::VPWidenIntOrFpInductionSC ||
2055 R->getVPDefID() == VPDef::VPWidenPointerInductionSC;
2056 }
2057
2058 static inline bool classof(const VPValue *V) {
2059 auto *R = V->getDefiningRecipe();
2060 return R && classof(R);
2061 }
2062
2063 static inline bool classof(const VPHeaderPHIRecipe *R) {
2064 return classof(static_cast<const VPRecipeBase *>(R));
2065 }
2066
2067 virtual void execute(VPTransformState &State) override = 0;
2068
2069 /// Returns the step value of the induction.
2071 const VPValue *getStepValue() const { return getOperand(1); }
2072
2073 /// Update the step value of the recipe.
2074 void setStepValue(VPValue *V) { setOperand(1, V); }
2075
2077 const VPValue *getVFValue() const { return getOperand(2); }
2078
2079 /// Returns the number of incoming values, also number of incoming blocks.
2080 /// Note that at the moment, VPWidenPointerInductionRecipe only has a single
2081 /// incoming value, its start value.
2082 unsigned getNumIncoming() const override { return 1; }
2083
2085
2086 /// Returns the induction descriptor for the recipe.
2087 const InductionDescriptor &getInductionDescriptor() const { return IndDesc; }
2088
2090 // TODO: All operands of base recipe must exist and be at same index in
2091 // derived recipe.
2093 "VPWidenIntOrFpInductionRecipe generates its own backedge value");
2094 }
2095
2097 // TODO: All operands of base recipe must exist and be at same index in
2098 // derived recipe.
2100 "VPWidenIntOrFpInductionRecipe generates its own backedge value");
2101 }
2102
2103 /// Returns true if the recipe only uses the first lane of operand \p Op.
2104 bool onlyFirstLaneUsed(const VPValue *Op) const override {
2106 "Op must be an operand of the recipe");
2107 // The recipe creates its own wide start value, so it only requests the
2108 // first lane of the operand.
2109 // TODO: Remove once creating the start value is modeled separately.
2110 return Op == getStartValue() || Op == getStepValue();
2111 }
2112};
2113
2114/// A recipe for handling phi nodes of integer and floating-point inductions,
2115/// producing their vector values. This is an abstract recipe and must be
2116/// converted to concrete recipes before executing.
2118 TruncInst *Trunc;
2119
2120 // If this recipe is unrolled it will have 2 additional operands.
2121 bool isUnrolled() const { return getNumOperands() == 5; }
2122
2123public:
2125 VPValue *VF, const InductionDescriptor &IndDesc,
2126 DebugLoc DL)
2127 : VPWidenInductionRecipe(VPDef::VPWidenIntOrFpInductionSC, IV, Start,
2128 Step, IndDesc, DL),
2129 Trunc(nullptr) {
2130 addOperand(VF);
2131 }
2132
2134 VPValue *VF, const InductionDescriptor &IndDesc,
2135 TruncInst *Trunc, DebugLoc DL)
2136 : VPWidenInductionRecipe(VPDef::VPWidenIntOrFpInductionSC, IV, Start,
2137 Step, IndDesc, DL),
2138 Trunc(Trunc) {
2139 addOperand(VF);
2141 (void)Metadata;
2142 if (Trunc)
2144 assert(Metadata.empty() && "unexpected metadata on Trunc");
2145 }
2146
2148
2154
2155 VP_CLASSOF_IMPL(VPDef::VPWidenIntOrFpInductionSC)
2156
2157 void execute(VPTransformState &State) override {
2158 llvm_unreachable("cannot execute this recipe, should be expanded via "
2159 "expandVPWidenIntOrFpInductionRecipe");
2160 }
2161
2162#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2163 /// Print the recipe.
2164 void print(raw_ostream &O, const Twine &Indent,
2165 VPSlotTracker &SlotTracker) const override;
2166#endif
2167
2169 // If the recipe has been unrolled return the VPValue for the induction
2170 // increment.
2171 return isUnrolled() ? getOperand(getNumOperands() - 2) : nullptr;
2172 }
2173
2174 /// Returns the number of incoming values, also number of incoming blocks.
2175 /// Note that at the moment, VPWidenIntOrFpInductionRecipes only have a single
2176 /// incoming value, its start value.
2177 unsigned getNumIncoming() const override { return 1; }
2178
2179 /// Returns the first defined value as TruncInst, if it is one or nullptr
2180 /// otherwise.
2181 TruncInst *getTruncInst() { return Trunc; }
2182 const TruncInst *getTruncInst() const { return Trunc; }
2183
2184 /// Returns true if the induction is canonical, i.e. starting at 0 and
2185 /// incremented by UF * VF (= the original IV is incremented by 1) and has the
2186 /// same type as the canonical induction.
2187 bool isCanonical() const;
2188
2189 /// Returns the scalar type of the induction.
2191 return Trunc ? Trunc->getType()
2193 }
2194
2195 /// Returns the VPValue representing the value of this induction at
2196 /// the last unrolled part, if it exists. Returns itself if unrolling did not
2197 /// take place.
2199 return isUnrolled() ? getOperand(getNumOperands() - 1) : this;
2200 }
2201};
2202
2204 bool IsScalarAfterVectorization;
2205
2206public:
2207 /// Create a new VPWidenPointerInductionRecipe for \p Phi with start value \p
2208 /// Start and the number of elements unrolled \p NumUnrolledElems, typically
2209 /// VF*UF.
2211 VPValue *NumUnrolledElems,
2212 const InductionDescriptor &IndDesc,
2213 bool IsScalarAfterVectorization, DebugLoc DL)
2214 : VPWidenInductionRecipe(VPDef::VPWidenPointerInductionSC, Phi, Start,
2215 Step, IndDesc, DL),
2216 IsScalarAfterVectorization(IsScalarAfterVectorization) {
2217 addOperand(NumUnrolledElems);
2218 }
2219
2221
2225 getOperand(2), getInductionDescriptor(), IsScalarAfterVectorization,
2226 getDebugLoc());
2227 }
2228
2229 VP_CLASSOF_IMPL(VPDef::VPWidenPointerInductionSC)
2230
2231 /// Generate vector values for the pointer induction.
2232 void execute(VPTransformState &State) override {
2233 llvm_unreachable("cannot execute this recipe, should be expanded via "
2234 "expandVPWidenPointerInduction");
2235 };
2236
2237 /// Returns true if only scalar values will be generated.
2238 bool onlyScalarsGenerated(bool IsScalable);
2239
2240#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2241 /// Print the recipe.
2242 void print(raw_ostream &O, const Twine &Indent,
2243 VPSlotTracker &SlotTracker) const override;
2244#endif
2245};
2246
2247/// A recipe for widened phis. Incoming values are operands of the recipe and
2248/// their operand index corresponds to the incoming predecessor block. If the
2249/// recipe is placed in an entry block to a (non-replicate) region, it must have
2250/// exactly 2 incoming values, the first from the predecessor of the region and
2251/// the second from the exiting block of the region.
2253 public VPPhiAccessors {
2254 /// Name to use for the generated IR instruction for the widened phi.
2255 std::string Name;
2256
2257protected:
2258 const VPRecipeBase *getAsRecipe() const override { return this; }
2259
2260public:
2261 /// Create a new VPWidenPHIRecipe for \p Phi with start value \p Start and
2262 /// debug location \p DL.
2263 VPWidenPHIRecipe(PHINode *Phi, VPValue *Start = nullptr,
2264 DebugLoc DL = DebugLoc::getUnknown(), const Twine &Name = "")
2265 : VPSingleDefRecipe(VPDef::VPWidenPHISC, ArrayRef<VPValue *>(), Phi, DL),
2266 Name(Name.str()) {
2267 if (Start)
2268 addOperand(Start);
2269 }
2270
2273 getOperand(0), getDebugLoc(), Name);
2275 C->addOperand(Op);
2276 return C;
2277 }
2278
2279 ~VPWidenPHIRecipe() override = default;
2280
2281 VP_CLASSOF_IMPL(VPDef::VPWidenPHISC)
2282
2283 /// Generate the phi/select nodes.
2284 void execute(VPTransformState &State) override;
2285
2286#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2287 /// Print the recipe.
2288 void print(raw_ostream &O, const Twine &Indent,
2289 VPSlotTracker &SlotTracker) const override;
2290#endif
2291};
2292
2293/// A recipe for handling first-order recurrence phis. The start value is the
2294/// first operand of the recipe and the incoming value from the backedge is the
2295/// second operand.
2298 : VPHeaderPHIRecipe(VPDef::VPFirstOrderRecurrencePHISC, Phi, &Start) {}
2299
2300 VP_CLASSOF_IMPL(VPDef::VPFirstOrderRecurrencePHISC)
2301
2306
2307 void execute(VPTransformState &State) override;
2308
2309 /// Return the cost of this first-order recurrence phi recipe.
2311 VPCostContext &Ctx) const override;
2312
2313#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2314 /// Print the recipe.
2315 void print(raw_ostream &O, const Twine &Indent,
2316 VPSlotTracker &SlotTracker) const override;
2317#endif
2318
2319 /// Returns true if the recipe only uses the first lane of operand \p Op.
2320 bool onlyFirstLaneUsed(const VPValue *Op) const override {
2322 "Op must be an operand of the recipe");
2323 return Op == getStartValue();
2324 }
2325};
2326
2327/// A recipe for handling reduction phis. The start value is the first operand
2328/// of the recipe and the incoming value from the backedge is the second
2329/// operand.
2331 public VPUnrollPartAccessor<2> {
2332 /// The recurrence kind of the reduction.
2333 const RecurKind Kind;
2334
2335 /// The phi is part of an in-loop reduction.
2336 bool IsInLoop;
2337
2338 /// The phi is part of an ordered reduction. Requires IsInLoop to be true.
2339 bool IsOrdered;
2340
2341 /// When expanding the reduction PHI, the plan's VF element count is divided
2342 /// by this factor to form the reduction phi's VF.
2343 unsigned VFScaleFactor = 1;
2344
2345public:
2346 /// Create a new VPReductionPHIRecipe for the reduction \p Phi.
2348 bool IsInLoop = false, bool IsOrdered = false,
2349 unsigned VFScaleFactor = 1)
2350 : VPHeaderPHIRecipe(VPDef::VPReductionPHISC, Phi, &Start), Kind(Kind),
2351 IsInLoop(IsInLoop), IsOrdered(IsOrdered), VFScaleFactor(VFScaleFactor) {
2352 assert((!IsOrdered || IsInLoop) && "IsOrdered requires IsInLoop");
2353 }
2354
2355 ~VPReductionPHIRecipe() override = default;
2356
2358 auto *R = new VPReductionPHIRecipe(
2360 *getOperand(0), IsInLoop, IsOrdered, VFScaleFactor);
2361 R->addOperand(getBackedgeValue());
2362 return R;
2363 }
2364
2365 VP_CLASSOF_IMPL(VPDef::VPReductionPHISC)
2366
2367 /// Generate the phi/select nodes.
2368 void execute(VPTransformState &State) override;
2369
2370 /// Get the factor that the VF of this recipe's output should be scaled by.
2371 unsigned getVFScaleFactor() const { return VFScaleFactor; }
2372
2373#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2374 /// Print the recipe.
2375 void print(raw_ostream &O, const Twine &Indent,
2376 VPSlotTracker &SlotTracker) const override;
2377#endif
2378
2379 /// Returns the number of incoming values, also number of incoming blocks.
2380 /// Note that at the moment, VPWidenPointerInductionRecipe only has a single
2381 /// incoming value, its start value.
2382 unsigned getNumIncoming() const override { return 2; }
2383
2384 /// Returns the recurrence kind of the reduction.
2385 RecurKind getRecurrenceKind() const { return Kind; }
2386
2387 /// Returns true, if the phi is part of an ordered reduction.
2388 bool isOrdered() const { return IsOrdered; }
2389
2390 /// Returns true, if the phi is part of an in-loop reduction.
2391 bool isInLoop() const { return IsInLoop; }
2392
2393 /// Returns true if the recipe only uses the first lane of operand \p Op.
2394 bool onlyFirstLaneUsed(const VPValue *Op) const override {
2396 "Op must be an operand of the recipe");
2397 return isOrdered() || isInLoop();
2398 }
2399};
2400
2401/// A recipe for vectorizing a phi-node as a sequence of mask-based select
2402/// instructions.
2404public:
2405 /// The blend operation is a User of the incoming values and of their
2406 /// respective masks, ordered [I0, M0, I1, M1, I2, M2, ...]. Note that M0 can
2407 /// be omitted (implied by passing an odd number of operands) in which case
2408 /// all other incoming values are merged into it.
2410 : VPSingleDefRecipe(VPDef::VPBlendSC, Operands, Phi, DL) {
2411 assert(Operands.size() > 0 && "Expected at least one operand!");
2412 }
2413
2418
2419 VP_CLASSOF_IMPL(VPDef::VPBlendSC)
2420
2421 /// A normalized blend is one that has an odd number of operands, whereby the
2422 /// first operand does not have an associated mask.
2423 bool isNormalized() const { return getNumOperands() % 2; }
2424
2425 /// Return the number of incoming values, taking into account when normalized
2426 /// the first incoming value will have no mask.
2427 unsigned getNumIncomingValues() const {
2428 return (getNumOperands() + isNormalized()) / 2;
2429 }
2430
2431 /// Return incoming value number \p Idx.
2432 VPValue *getIncomingValue(unsigned Idx) const {
2433 return Idx == 0 ? getOperand(0) : getOperand(Idx * 2 - isNormalized());
2434 }
2435
2436 /// Return mask number \p Idx.
2437 VPValue *getMask(unsigned Idx) const {
2438 assert((Idx > 0 || !isNormalized()) && "First index has no mask!");
2439 return Idx == 0 ? getOperand(1) : getOperand(Idx * 2 + !isNormalized());
2440 }
2441
2442 /// Set mask number \p Idx to \p V.
2443 void setMask(unsigned Idx, VPValue *V) {
2444 assert((Idx > 0 || !isNormalized()) && "First index has no mask!");
2445 Idx == 0 ? setOperand(1, V) : setOperand(Idx * 2 + !isNormalized(), V);
2446 }
2447
2448 void execute(VPTransformState &State) override {
2449 llvm_unreachable("VPBlendRecipe should be expanded by simplifyBlends");
2450 }
2451
2452 /// Return the cost of this VPWidenMemoryRecipe.
2453 InstructionCost computeCost(ElementCount VF,
2454 VPCostContext &Ctx) const override;
2455
2456#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2457 /// Print the recipe.
2458 void print(raw_ostream &O, const Twine &Indent,
2459 VPSlotTracker &SlotTracker) const override;
2460#endif
2461
2462 /// Returns true if the recipe only uses the first lane of operand \p Op.
2463 bool onlyFirstLaneUsed(const VPValue *Op) const override {
2465 "Op must be an operand of the recipe");
2466 // Recursing through Blend recipes only, must terminate at header phi's the
2467 // latest.
2468 return all_of(users(),
2469 [this](VPUser *U) { return U->onlyFirstLaneUsed(this); });
2470 }
2471};
2472
2473/// A common base class for interleaved memory operations.
2474/// An Interleaved memory operation is a memory access method that combines
2475/// multiple strided loads/stores into a single wide load/store with shuffles.
2476/// The first operand is the start address. The optional operands are, in order,
2477/// the stored values and the mask.
2479 public VPIRMetadata {
2481
2482 /// Indicates if the interleave group is in a conditional block and requires a
2483 /// mask.
2484 bool HasMask = false;
2485
2486 /// Indicates if gaps between members of the group need to be masked out or if
2487 /// unusued gaps can be loaded speculatively.
2488 bool NeedsMaskForGaps = false;
2489
2490protected:
2491 VPInterleaveBase(const unsigned char SC,
2493 ArrayRef<VPValue *> Operands,
2494 ArrayRef<VPValue *> StoredValues, VPValue *Mask,
2495 bool NeedsMaskForGaps, const VPIRMetadata &MD, DebugLoc DL)
2496 : VPRecipeBase(SC, Operands, DL), VPIRMetadata(MD), IG(IG),
2497 NeedsMaskForGaps(NeedsMaskForGaps) {
2498 // TODO: extend the masked interleaved-group support to reversed access.
2499 assert((!Mask || !IG->isReverse()) &&
2500 "Reversed masked interleave-group not supported.");
2501 for (unsigned I = 0; I < IG->getFactor(); ++I)
2502 if (Instruction *Inst = IG->getMember(I)) {
2503 if (Inst->getType()->isVoidTy())
2504 continue;
2505 new VPValue(Inst, this);
2506 }
2507
2508 for (auto *SV : StoredValues)
2509 addOperand(SV);
2510 if (Mask) {
2511 HasMask = true;
2512 addOperand(Mask);
2513 }
2514 }
2515
2516public:
2517 VPInterleaveBase *clone() override = 0;
2518
2519 static inline bool classof(const VPRecipeBase *R) {
2520 return R->getVPDefID() == VPRecipeBase::VPInterleaveSC ||
2521 R->getVPDefID() == VPRecipeBase::VPInterleaveEVLSC;
2522 }
2523
2524 static inline bool classof(const VPUser *U) {
2525 auto *R = dyn_cast<VPRecipeBase>(U);
2526 return R && classof(R);
2527 }
2528
2529 /// Return the address accessed by this recipe.
2530 VPValue *getAddr() const {
2531 return getOperand(0); // Address is the 1st, mandatory operand.
2532 }
2533
2534 /// Return the mask used by this recipe. Note that a full mask is represented
2535 /// by a nullptr.
2536 VPValue *getMask() const {
2537 // Mask is optional and the last operand.
2538 return HasMask ? getOperand(getNumOperands() - 1) : nullptr;
2539 }
2540
2541 /// Return true if the access needs a mask because of the gaps.
2542 bool needsMaskForGaps() const { return NeedsMaskForGaps; }
2543
2545
2546 Instruction *getInsertPos() const { return IG->getInsertPos(); }
2547
2548 void execute(VPTransformState &State) override {
2549 llvm_unreachable("VPInterleaveBase should not be instantiated.");
2550 }
2551
2552 /// Return the cost of this recipe.
2553 InstructionCost computeCost(ElementCount VF,
2554 VPCostContext &Ctx) const override;
2555
2556 /// Returns true if the recipe only uses the first lane of operand \p Op.
2557 virtual bool onlyFirstLaneUsed(const VPValue *Op) const override = 0;
2558
2559 /// Returns the number of stored operands of this interleave group. Returns 0
2560 /// for load interleave groups.
2561 virtual unsigned getNumStoreOperands() const = 0;
2562
2563 /// Return the VPValues stored by this interleave group. If it is a load
2564 /// interleave group, return an empty ArrayRef.
2566 return ArrayRef<VPValue *>(op_end() -
2567 (getNumStoreOperands() + (HasMask ? 1 : 0)),
2569 }
2570};
2571
2572/// VPInterleaveRecipe is a recipe for transforming an interleave group of load
2573/// or stores into one wide load/store and shuffles. The first operand of a
2574/// VPInterleave recipe is the address, followed by the stored values, followed
2575/// by an optional mask.
2577public:
2579 ArrayRef<VPValue *> StoredValues, VPValue *Mask,
2580 bool NeedsMaskForGaps, const VPIRMetadata &MD, DebugLoc DL)
2581 : VPInterleaveBase(VPDef::VPInterleaveSC, IG, Addr, StoredValues, Mask,
2582 NeedsMaskForGaps, MD, DL) {}
2583
2584 ~VPInterleaveRecipe() override = default;
2585
2589 needsMaskForGaps(), *this, getDebugLoc());
2590 }
2591
2592 VP_CLASSOF_IMPL(VPDef::VPInterleaveSC)
2593
2594 /// Generate the wide load or store, and shuffles.
2595 void execute(VPTransformState &State) override;
2596
2597#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2598 /// Print the recipe.
2599 void print(raw_ostream &O, const Twine &Indent,
2600 VPSlotTracker &SlotTracker) const override;
2601#endif
2602
2603 bool onlyFirstLaneUsed(const VPValue *Op) const override {
2605 "Op must be an operand of the recipe");
2606 return Op == getAddr() && !llvm::is_contained(getStoredValues(), Op);
2607 }
2608
2609 unsigned getNumStoreOperands() const override {
2610 return getNumOperands() - (getMask() ? 2 : 1);
2611 }
2612};
2613
2614/// A recipe for interleaved memory operations with vector-predication
2615/// intrinsics. The first operand is the address, the second operand is the
2616/// explicit vector length. Stored values and mask are optional operands.
2618public:
2620 : VPInterleaveBase(VPDef::VPInterleaveEVLSC, R.getInterleaveGroup(),
2621 ArrayRef<VPValue *>({R.getAddr(), &EVL}),
2622 R.getStoredValues(), Mask, R.needsMaskForGaps(), R,
2623 R.getDebugLoc()) {
2624 assert(!getInterleaveGroup()->isReverse() &&
2625 "Reversed interleave-group with tail folding is not supported.");
2626 assert(!needsMaskForGaps() && "Interleaved access with gap mask is not "
2627 "supported for scalable vector.");
2628 }
2629
2630 ~VPInterleaveEVLRecipe() override = default;
2631
2633 llvm_unreachable("cloning not implemented yet");
2634 }
2635
2636 VP_CLASSOF_IMPL(VPDef::VPInterleaveEVLSC)
2637
2638 /// The VPValue of the explicit vector length.
2639 VPValue *getEVL() const { return getOperand(1); }
2640
2641 /// Generate the wide load or store, and shuffles.
2642 void execute(VPTransformState &State) override;
2643
2644#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2645 /// Print the recipe.
2646 void print(raw_ostream &O, const Twine &Indent,
2647 VPSlotTracker &SlotTracker) const override;
2648#endif
2649
2650 /// The recipe only uses the first lane of the address, and EVL operand.
2651 bool onlyFirstLaneUsed(const VPValue *Op) const override {
2653 "Op must be an operand of the recipe");
2654 return (Op == getAddr() && !llvm::is_contained(getStoredValues(), Op)) ||
2655 Op == getEVL();
2656 }
2657
2658 unsigned getNumStoreOperands() const override {
2659 return getNumOperands() - (getMask() ? 3 : 2);
2660 }
2661};
2662
2663/// A recipe to represent inloop reduction operations, performing a reduction on
2664/// a vector operand into a scalar value, and adding the result to a chain.
2665/// The Operands are {ChainOp, VecOp, [Condition]}.
2667 /// The recurrence kind for the reduction in question.
2668 RecurKind RdxKind;
2669 bool IsOrdered;
2670 /// Whether the reduction is conditional.
2671 bool IsConditional = false;
2672
2673protected:
2674 VPReductionRecipe(const unsigned char SC, RecurKind RdxKind,
2677 bool IsOrdered, DebugLoc DL)
2678 : VPRecipeWithIRFlags(SC, Operands, FMFs, DL), RdxKind(RdxKind),
2679 IsOrdered(IsOrdered) {
2680 if (CondOp) {
2681 IsConditional = true;
2682 addOperand(CondOp);
2683 }
2685 }
2686
2687public:
2689 VPValue *ChainOp, VPValue *VecOp, VPValue *CondOp,
2690 bool IsOrdered, DebugLoc DL = DebugLoc::getUnknown())
2691 : VPReductionRecipe(VPDef::VPReductionSC, RdxKind, FMFs, I,
2692 ArrayRef<VPValue *>({ChainOp, VecOp}), CondOp,
2693 IsOrdered, DL) {}
2694
2696 VPValue *ChainOp, VPValue *VecOp, VPValue *CondOp,
2697 bool IsOrdered, DebugLoc DL = DebugLoc::getUnknown())
2698 : VPReductionRecipe(VPDef::VPReductionSC, RdxKind, FMFs, nullptr,
2699 ArrayRef<VPValue *>({ChainOp, VecOp}), CondOp,
2700 IsOrdered, DL) {}
2701
2702 ~VPReductionRecipe() override = default;
2703
2705 return new VPReductionRecipe(RdxKind, getFastMathFlags(),
2707 getCondOp(), IsOrdered, getDebugLoc());
2708 }
2709
2710 static inline bool classof(const VPRecipeBase *R) {
2711 return R->getVPDefID() == VPRecipeBase::VPReductionSC ||
2712 R->getVPDefID() == VPRecipeBase::VPReductionEVLSC;
2713 }
2714
2715 static inline bool classof(const VPUser *U) {
2716 auto *R = dyn_cast<VPRecipeBase>(U);
2717 return R && classof(R);
2718 }
2719
2720 /// Generate the reduction in the loop.
2721 void execute(VPTransformState &State) override;
2722
2723 /// Return the cost of VPReductionRecipe.
2724 InstructionCost computeCost(ElementCount VF,
2725 VPCostContext &Ctx) const override;
2726
2727#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2728 /// Print the recipe.
2729 void print(raw_ostream &O, const Twine &Indent,
2730 VPSlotTracker &SlotTracker) const override;
2731#endif
2732
2733 /// Return the recurrence kind for the in-loop reduction.
2734 RecurKind getRecurrenceKind() const { return RdxKind; }
2735 /// Return true if the in-loop reduction is ordered.
2736 bool isOrdered() const { return IsOrdered; };
2737 /// Return true if the in-loop reduction is conditional.
2738 bool isConditional() const { return IsConditional; };
2739 /// The VPValue of the scalar Chain being accumulated.
2740 VPValue *getChainOp() const { return getOperand(0); }
2741 /// The VPValue of the vector value to be reduced.
2742 VPValue *getVecOp() const { return getOperand(1); }
2743 /// The VPValue of the condition for the block.
2745 return isConditional() ? getOperand(getNumOperands() - 1) : nullptr;
2746 }
2747};
2748
2749/// A recipe for forming partial reductions. In the loop, an accumulator and
2750/// vector operand are added together and passed to the next iteration as the
2751/// next accumulator. After the loop body, the accumulator is reduced to a
2752/// scalar value.
2754 unsigned Opcode;
2755
2756 /// The divisor by which the VF of this recipe's output should be divided
2757 /// during execution.
2758 unsigned VFScaleFactor;
2759
2760public:
2762 VPValue *Op1, VPValue *Cond, unsigned VFScaleFactor)
2763 : VPPartialReductionRecipe(ReductionInst->getOpcode(), Op0, Op1, Cond,
2764 VFScaleFactor, ReductionInst) {}
2765 VPPartialReductionRecipe(unsigned Opcode, VPValue *Op0, VPValue *Op1,
2766 VPValue *Cond, unsigned ScaleFactor,
2767 Instruction *ReductionInst = nullptr)
2768 : VPReductionRecipe(VPDef::VPPartialReductionSC, RecurKind::Add,
2769 FastMathFlags(), ReductionInst,
2770 ArrayRef<VPValue *>({Op0, Op1}), Cond, false, {}),
2771 Opcode(Opcode), VFScaleFactor(ScaleFactor) {
2772 [[maybe_unused]] auto *AccumulatorRecipe =
2774 assert((isa<VPReductionPHIRecipe>(AccumulatorRecipe) ||
2775 isa<VPPartialReductionRecipe>(AccumulatorRecipe)) &&
2776 "Unexpected operand order for partial reduction recipe");
2777 }
2778 ~VPPartialReductionRecipe() override = default;
2779
2781 return new VPPartialReductionRecipe(Opcode, getOperand(0), getOperand(1),
2782 getCondOp(), VFScaleFactor,
2784 }
2785
2786 VP_CLASSOF_IMPL(VPDef::VPPartialReductionSC)
2787
2788 /// Generate the reduction in the loop.
2789 void execute(VPTransformState &State) override;
2790
2791 /// Return the cost of this VPPartialReductionRecipe.
2793 VPCostContext &Ctx) const override;
2794
2795 /// Get the binary op's opcode.
2796 unsigned getOpcode() const { return Opcode; }
2797
2798 /// Get the factor that the VF of this recipe's output should be scaled by.
2799 unsigned getVFScaleFactor() const { return VFScaleFactor; }
2800
2801#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2802 /// Print the recipe.
2803 void print(raw_ostream &O, const Twine &Indent,
2804 VPSlotTracker &SlotTracker) const override;
2805#endif
2806};
2807
2808/// A recipe to represent inloop reduction operations with vector-predication
2809/// intrinsics, performing a reduction on a vector operand with the explicit
2810/// vector length (EVL) into a scalar value, and adding the result to a chain.
2811/// The Operands are {ChainOp, VecOp, EVL, [Condition]}.
2813public:
2817 VPDef::VPReductionEVLSC, R.getRecurrenceKind(),
2818 R.getFastMathFlags(),
2820 ArrayRef<VPValue *>({R.getChainOp(), R.getVecOp(), &EVL}), CondOp,
2821 R.isOrdered(), DL) {}
2822
2823 ~VPReductionEVLRecipe() override = default;
2824
2826 llvm_unreachable("cloning not implemented yet");
2827 }
2828
2829 VP_CLASSOF_IMPL(VPDef::VPReductionEVLSC)
2830
2831 /// Generate the reduction in the loop
2832 void execute(VPTransformState &State) override;
2833
2834#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2835 /// Print the recipe.
2836 void print(raw_ostream &O, const Twine &Indent,
2837 VPSlotTracker &SlotTracker) const override;
2838#endif
2839
2840 /// The VPValue of the explicit vector length.
2841 VPValue *getEVL() const { return getOperand(2); }
2842
2843 /// Returns true if the recipe only uses the first lane of operand \p Op.
2844 bool onlyFirstLaneUsed(const VPValue *Op) const override {
2846 "Op must be an operand of the recipe");
2847 return Op == getEVL();
2848 }
2849};
2850
2851/// VPReplicateRecipe replicates a given instruction producing multiple scalar
2852/// copies of the original scalar type, one per lane, instead of producing a
2853/// single copy of widened type for all lanes. If the instruction is known to be
2854/// a single scalar, only one copy, per lane zero, will be generated.
2856 public VPIRMetadata {
2857 /// Indicator if only a single replica per lane is needed.
2858 bool IsSingleScalar;
2859
2860 /// Indicator if the replicas are also predicated.
2861 bool IsPredicated;
2862
2863public:
2865 bool IsSingleScalar, VPValue *Mask = nullptr,
2866 VPIRMetadata Metadata = {})
2867 : VPRecipeWithIRFlags(VPDef::VPReplicateSC, Operands, *I),
2868 VPIRMetadata(Metadata), IsSingleScalar(IsSingleScalar),
2869 IsPredicated(Mask) {
2870 if (Mask)
2871 addOperand(Mask);
2872 }
2873
2874 ~VPReplicateRecipe() override = default;
2875
2877 auto *Copy =
2878 new VPReplicateRecipe(getUnderlyingInstr(), operands(), IsSingleScalar,
2879 isPredicated() ? getMask() : nullptr, *this);
2880 Copy->transferFlags(*this);
2881 return Copy;
2882 }
2883
2884 VP_CLASSOF_IMPL(VPDef::VPReplicateSC)
2885
2886 /// Generate replicas of the desired Ingredient. Replicas will be generated
2887 /// for all parts and lanes unless a specific part and lane are specified in
2888 /// the \p State.
2889 void execute(VPTransformState &State) override;
2890
2891 /// Return the cost of this VPReplicateRecipe.
2892 InstructionCost computeCost(ElementCount VF,
2893 VPCostContext &Ctx) const override;
2894
2895#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2896 /// Print the recipe.
2897 void print(raw_ostream &O, const Twine &Indent,
2898 VPSlotTracker &SlotTracker) const override;
2899#endif
2900
2901 bool isSingleScalar() const { return IsSingleScalar; }
2902
2903 bool isPredicated() const { return IsPredicated; }
2904
2905 /// Returns true if the recipe only uses the first lane of operand \p Op.
2906 bool onlyFirstLaneUsed(const VPValue *Op) const override {
2908 "Op must be an operand of the recipe");
2909 return isSingleScalar();
2910 }
2911
2912 /// Returns true if the recipe uses scalars of operand \p Op.
2913 bool usesScalars(const VPValue *Op) const override {
2915 "Op must be an operand of the recipe");
2916 return true;
2917 }
2918
2919 /// Returns true if the recipe is used by a widened recipe via an intervening
2920 /// VPPredInstPHIRecipe. In this case, the scalar values should also be packed
2921 /// in a vector.
2922 bool shouldPack() const;
2923
2924 /// Return the mask of a predicated VPReplicateRecipe.
2926 assert(isPredicated() && "Trying to get the mask of a unpredicated recipe");
2927 return getOperand(getNumOperands() - 1);
2928 }
2929
2930 unsigned getOpcode() const { return getUnderlyingInstr()->getOpcode(); }
2931};
2932
2933/// A recipe for generating conditional branches on the bits of a mask.
2935public:
2937 : VPRecipeBase(VPDef::VPBranchOnMaskSC, {BlockInMask}, DL) {}
2938
2941 }
2942
2943 VP_CLASSOF_IMPL(VPDef::VPBranchOnMaskSC)
2944
2945 /// Generate the extraction of the appropriate bit from the block mask and the
2946 /// conditional branch.
2947 void execute(VPTransformState &State) override;
2948
2949 /// Return the cost of this VPBranchOnMaskRecipe.
2950 InstructionCost computeCost(ElementCount VF,
2951 VPCostContext &Ctx) const override;
2952
2953#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2954 /// Print the recipe.
2955 void print(raw_ostream &O, const Twine &Indent,
2956 VPSlotTracker &SlotTracker) const override {
2957 O << Indent << "BRANCH-ON-MASK ";
2959 }
2960#endif
2961
2962 /// Returns true if the recipe uses scalars of operand \p Op.
2963 bool usesScalars(const VPValue *Op) const override {
2965 "Op must be an operand of the recipe");
2966 return true;
2967 }
2968};
2969
2970/// A recipe to combine multiple recipes into a single 'expression' recipe,
2971/// which should be considered a single entity for cost-modeling and transforms.
2972/// The recipe needs to be 'decomposed', i.e. replaced by its individual
2973/// expression recipes, before execute. The individual expression recipes are
2974/// completely disconnected from the def-use graph of other recipes not part of
2975/// the expression. Def-use edges between pairs of expression recipes remain
2976/// intact, whereas every edge between an expression recipe and a recipe outside
2977/// the expression is elevated to connect the non-expression recipe with the
2978/// VPExpressionRecipe itself.
2979class VPExpressionRecipe : public VPSingleDefRecipe {
2980 /// Recipes included in this VPExpressionRecipe.
2981 SmallVector<VPSingleDefRecipe *> ExpressionRecipes;
2982
2983 /// Temporary VPValues used for external operands of the expression, i.e.
2984 /// operands not defined by recipes in the expression.
2985 SmallVector<VPValue *> LiveInPlaceholders;
2986
2987 enum class ExpressionTypes {
2988 /// Represents an inloop extended reduction operation, performing a
2989 /// reduction on an extended vector operand into a scalar value, and adding
2990 /// the result to a chain.
2991 ExtendedReduction,
2992 /// Represent an inloop multiply-accumulate reduction, multiplying the
2993 /// extended vector operands, performing a reduction.add on the result, and
2994 /// adding the scalar result to a chain.
2995 ExtMulAccReduction,
2996 /// Represent an inloop multiply-accumulate reduction, multiplying the
2997 /// vector operands, performing a reduction.add on the result, and adding
2998 /// the scalar result to a chain.
2999 MulAccReduction,
3000 };
3001
3002 /// Type of the expression.
3003 ExpressionTypes ExpressionType;
3004
3005 /// Construct a new VPExpressionRecipe by internalizing recipes in \p
3006 /// ExpressionRecipes. External operands (i.e. not defined by another recipe
3007 /// in the expression) are replaced by temporary VPValues and the original
3008 /// operands are transferred to the VPExpressionRecipe itself. Clone recipes
3009 /// as needed (excluding last) to ensure they are only used by other recipes
3010 /// in the expression.
3011 VPExpressionRecipe(ExpressionTypes ExpressionType,
3012 ArrayRef<VPSingleDefRecipe *> ExpressionRecipes);
3013
3014public:
3016 : VPExpressionRecipe(ExpressionTypes::ExtendedReduction, {Ext, Red}) {}
3018 : VPExpressionRecipe(ExpressionTypes::MulAccReduction, {Mul, Red}) {}
3021 : VPExpressionRecipe(ExpressionTypes::ExtMulAccReduction,
3022 {Ext0, Ext1, Mul, Red}) {}
3023
3025 for (auto *R : reverse(ExpressionRecipes))
3026 delete R;
3027 for (VPValue *T : LiveInPlaceholders)
3028 delete T;
3029 }
3030
3031 VP_CLASSOF_IMPL(VPDef::VPExpressionSC)
3032
3033 VPExpressionRecipe *clone() override {
3034 assert(!ExpressionRecipes.empty() && "empty expressions should be removed");
3035 SmallVector<VPSingleDefRecipe *> NewExpressiondRecipes;
3036 for (auto *R : ExpressionRecipes)
3037 NewExpressiondRecipes.push_back(R->clone());
3038 for (auto *New : NewExpressiondRecipes) {
3039 for (const auto &[Idx, Old] : enumerate(ExpressionRecipes))
3040 New->replaceUsesOfWith(Old, NewExpressiondRecipes[Idx]);
3041 // Update placeholder operands in the cloned recipe to use the external
3042 // operands, to be internalized when the cloned expression is constructed.
3043 for (const auto &[Placeholder, OutsideOp] :
3044 zip(LiveInPlaceholders, operands()))
3045 New->replaceUsesOfWith(Placeholder, OutsideOp);
3046 }
3047 return new VPExpressionRecipe(ExpressionType, NewExpressiondRecipes);
3048 }
3049
3050 /// Return the VPValue to use to infer the result type of the recipe.
3052 unsigned OpIdx =
3053 cast<VPReductionRecipe>(ExpressionRecipes.back())->isConditional() ? 2
3054 : 1;
3055 return getOperand(getNumOperands() - OpIdx);
3056 }
3057
3058 /// Insert the recipes of the expression back into the VPlan, directly before
3059 /// the current recipe. Leaves the expression recipe empty, which must be
3060 /// removed before codegen.
3061 void decompose();
3062
3063 /// Method for generating code, must not be called as this recipe is abstract.
3064 void execute(VPTransformState &State) override {
3065 llvm_unreachable("recipe must be removed before execute");
3066 }
3067
3069 VPCostContext &Ctx) const override;
3070
3071#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
3072 /// Print the recipe.
3073 void print(raw_ostream &O, const Twine &Indent,
3074 VPSlotTracker &SlotTracker) const override;
3075#endif
3076
3077 /// Returns true if this expression contains recipes that may read from or
3078 /// write to memory.
3079 bool mayReadOrWriteMemory() const;
3080
3081 /// Returns true if this expression contains recipes that may have side
3082 /// effects.
3083 bool mayHaveSideEffects() const;
3084};
3085
3086/// VPPredInstPHIRecipe is a recipe for generating the phi nodes needed when
3087/// control converges back from a Branch-on-Mask. The phi nodes are needed in
3088/// order to merge values that are set under such a branch and feed their uses.
3089/// The phi nodes can be scalar or vector depending on the users of the value.
3090/// This recipe works in concert with VPBranchOnMaskRecipe.
3092public:
3093 /// Construct a VPPredInstPHIRecipe given \p PredInst whose value needs a phi
3094 /// nodes after merging back from a Branch-on-Mask.
3096 : VPSingleDefRecipe(VPDef::VPPredInstPHISC, PredV, DL) {}
3097 ~VPPredInstPHIRecipe() override = default;
3098
3100 return new VPPredInstPHIRecipe(getOperand(0), getDebugLoc());
3101 }
3102
3103 VP_CLASSOF_IMPL(VPDef::VPPredInstPHISC)
3104
3105 /// Generates phi nodes for live-outs (from a replicate region) as needed to
3106 /// retain SSA form.
3107 void execute(VPTransformState &State) override;
3108
3109 /// Return the cost of this VPPredInstPHIRecipe.
3111 VPCostContext &Ctx) const override {
3112 // TODO: Compute accurate cost after retiring the legacy cost model.
3113 return 0;
3114 }
3115
3116#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
3117 /// Print the recipe.
3118 void print(raw_ostream &O, const Twine &Indent,
3119 VPSlotTracker &SlotTracker) const override;
3120#endif
3121
3122 /// Returns true if the recipe uses scalars of operand \p Op.
3123 bool usesScalars(const VPValue *Op) const override {
3125 "Op must be an operand of the recipe");
3126 return true;
3127 }
3128};
3129
3130/// A common base class for widening memory operations. An optional mask can be
3131/// provided as the last operand.
3133 public VPIRMetadata {
3134protected:
3136
3137 /// Whether the accessed addresses are consecutive.
3139
3140 /// Whether the consecutive accessed addresses are in reverse order.
3142
3143 /// Whether the memory access is masked.
3144 bool IsMasked = false;
3145
3146 void setMask(VPValue *Mask) {
3147 assert(!IsMasked && "cannot re-set mask");
3148 if (!Mask)
3149 return;
3150 addOperand(Mask);
3151 IsMasked = true;
3152 }
3153
3154 VPWidenMemoryRecipe(const char unsigned SC, Instruction &I,
3155 std::initializer_list<VPValue *> Operands,
3156 bool Consecutive, bool Reverse,
3157 const VPIRMetadata &Metadata, DebugLoc DL)
3158 : VPRecipeBase(SC, Operands, DL), VPIRMetadata(Metadata), Ingredient(I),
3160 assert((Consecutive || !Reverse) && "Reverse implies consecutive");
3161 }
3162
3163public:
3165 llvm_unreachable("cloning not supported");
3166 }
3167
3168 static inline bool classof(const VPRecipeBase *R) {
3169 return R->getVPDefID() == VPRecipeBase::VPWidenLoadSC ||
3170 R->getVPDefID() == VPRecipeBase::VPWidenStoreSC ||
3171 R->getVPDefID() == VPRecipeBase::VPWidenLoadEVLSC ||
3172 R->getVPDefID() == VPRecipeBase::VPWidenStoreEVLSC;
3173 }
3174
3175 static inline bool classof(const VPUser *U) {
3176 auto *R = dyn_cast<VPRecipeBase>(U);
3177 return R && classof(R);
3178 }
3179
3180 /// Return whether the loaded-from / stored-to addresses are consecutive.
3181 bool isConsecutive() const { return Consecutive; }
3182
3183 /// Return whether the consecutive loaded/stored addresses are in reverse
3184 /// order.
3185 bool isReverse() const { return Reverse; }
3186
3187 /// Return the address accessed by this recipe.
3188 VPValue *getAddr() const { return getOperand(0); }
3189
3190 /// Returns true if the recipe is masked.
3191 bool isMasked() const { return IsMasked; }
3192
3193 /// Return the mask used by this recipe. Note that a full mask is represented
3194 /// by a nullptr.
3195 VPValue *getMask() const {
3196 // Mask is optional and therefore the last operand.
3197 return isMasked() ? getOperand(getNumOperands() - 1) : nullptr;
3198 }
3199
3200 /// Generate the wide load/store.
3201 void execute(VPTransformState &State) override {
3202 llvm_unreachable("VPWidenMemoryRecipe should not be instantiated.");
3203 }
3204
3205 /// Return the cost of this VPWidenMemoryRecipe.
3206 InstructionCost computeCost(ElementCount VF,
3207 VPCostContext &Ctx) const override;
3208
3210};
3211
3212/// A recipe for widening load operations, using the address to load from and an
3213/// optional mask.
3215 public VPValue {
3217 bool Consecutive, bool Reverse,
3218 const VPIRMetadata &Metadata, DebugLoc DL)
3219 : VPWidenMemoryRecipe(VPDef::VPWidenLoadSC, Load, {Addr}, Consecutive,
3220 Reverse, Metadata, DL),
3221 VPValue(this, &Load) {
3222 setMask(Mask);
3223 }
3224
3227 getMask(), Consecutive, Reverse, *this,
3228 getDebugLoc());
3229 }
3230
3231 VP_CLASSOF_IMPL(VPDef::VPWidenLoadSC);
3232
3233 /// Generate a wide load or gather.
3234 void execute(VPTransformState &State) override;
3235
3236#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
3237 /// Print the recipe.
3238 void print(raw_ostream &O, const Twine &Indent,
3239 VPSlotTracker &SlotTracker) const override;
3240#endif
3241
3242 /// Returns true if the recipe only uses the first lane of operand \p Op.
3243 bool onlyFirstLaneUsed(const VPValue *Op) const override {
3245 "Op must be an operand of the recipe");
3246 // Widened, consecutive loads operations only demand the first lane of
3247 // their address.
3248 return Op == getAddr() && isConsecutive();
3249 }
3250};
3251
3252/// A recipe for widening load operations with vector-predication intrinsics,
3253/// using the address to load from, the explicit vector length and an optional
3254/// mask.
3255struct VPWidenLoadEVLRecipe final : public VPWidenMemoryRecipe, public VPValue {
3257 VPValue *Mask)
3258 : VPWidenMemoryRecipe(VPDef::VPWidenLoadEVLSC, L.getIngredient(),
3259 {Addr, &EVL}, L.isConsecutive(), L.isReverse(), L,
3260 L.getDebugLoc()),
3261 VPValue(this, &getIngredient()) {
3262 setMask(Mask);
3263 }
3264
3265 VP_CLASSOF_IMPL(VPDef::VPWidenLoadEVLSC)
3266
3267 /// Return the EVL operand.
3268 VPValue *getEVL() const { return getOperand(1); }
3269
3270 /// Generate the wide load or gather.
3271 void execute(VPTransformState &State) override;
3272
3273 /// Return the cost of this VPWidenLoadEVLRecipe.
3275 VPCostContext &Ctx) const override;
3276
3277#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
3278 /// Print the recipe.
3279 void print(raw_ostream &O, const Twine &Indent,
3280 VPSlotTracker &SlotTracker) const override;
3281#endif
3282
3283 /// Returns true if the recipe only uses the first lane of operand \p Op.
3284 bool onlyFirstLaneUsed(const VPValue *Op) const override {
3286 "Op must be an operand of the recipe");
3287 // Widened loads only demand the first lane of EVL and consecutive loads
3288 // only demand the first lane of their address.
3289 return Op == getEVL() || (Op == getAddr() && isConsecutive());
3290 }
3291};
3292
3293/// A recipe for widening store operations, using the stored value, the address
3294/// to store to and an optional mask.
3296 VPWidenStoreRecipe(StoreInst &Store, VPValue *Addr, VPValue *StoredVal,
3297 VPValue *Mask, bool Consecutive, bool Reverse,
3298 const VPIRMetadata &Metadata, DebugLoc DL)
3299 : VPWidenMemoryRecipe(VPDef::VPWidenStoreSC, Store, {Addr, StoredVal},
3300 Consecutive, Reverse, Metadata, DL) {
3301 setMask(Mask);
3302 }
3303
3309
3310 VP_CLASSOF_IMPL(VPDef::VPWidenStoreSC);
3311
3312 /// Return the value stored by this recipe.
3313 VPValue *getStoredValue() const { return getOperand(1); }
3314
3315 /// Generate a wide store or scatter.
3316 void execute(VPTransformState &State) override;
3317
3318#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
3319 /// Print the recipe.
3320 void print(raw_ostream &O, const Twine &Indent,
3321 VPSlotTracker &SlotTracker) const override;
3322#endif
3323
3324 /// Returns true if the recipe only uses the first lane of operand \p Op.
3325 bool onlyFirstLaneUsed(const VPValue *Op) const override {
3327 "Op must be an operand of the recipe");
3328 // Widened, consecutive stores only demand the first lane of their address,
3329 // unless the same operand is also stored.
3330 return Op == getAddr() && isConsecutive() && Op != getStoredValue();
3331 }
3332};
3333
3334/// A recipe for widening store operations with vector-predication intrinsics,
3335/// using the value to store, the address to store to, the explicit vector
3336/// length and an optional mask.
3339 VPValue *Mask)
3340 : VPWidenMemoryRecipe(VPDef::VPWidenStoreEVLSC, S.getIngredient(),
3341 {Addr, S.getStoredValue(), &EVL}, S.isConsecutive(),
3342 S.isReverse(), S, S.getDebugLoc()) {
3343 setMask(Mask);
3344 }
3345
3346 VP_CLASSOF_IMPL(VPDef::VPWidenStoreEVLSC)
3347
3348 /// Return the address accessed by this recipe.
3349 VPValue *getStoredValue() const { return getOperand(1); }
3350
3351 /// Return the EVL operand.
3352 VPValue *getEVL() const { return getOperand(2); }
3353
3354 /// Generate the wide store or scatter.
3355 void execute(VPTransformState &State) override;
3356
3357 /// Return the cost of this VPWidenStoreEVLRecipe.
3359 VPCostContext &Ctx) const override;
3360
3361#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
3362 /// Print the recipe.
3363 void print(raw_ostream &O, const Twine &Indent,
3364 VPSlotTracker &SlotTracker) const override;
3365#endif
3366
3367 /// Returns true if the recipe only uses the first lane of operand \p Op.
3368 bool onlyFirstLaneUsed(const VPValue *Op) const override {
3370 "Op must be an operand of the recipe");
3371 if (Op == getEVL()) {
3372 assert(getStoredValue() != Op && "unexpected store of EVL");
3373 return true;
3374 }
3375 // Widened, consecutive memory operations only demand the first lane of
3376 // their address, unless the same operand is also stored. That latter can
3377 // happen with opaque pointers.
3378 return Op == getAddr() && isConsecutive() && Op != getStoredValue();
3379 }
3380};
3381
3382/// Recipe to expand a SCEV expression.
3384 const SCEV *Expr;
3385
3386public:
3388 : VPSingleDefRecipe(VPDef::VPExpandSCEVSC, {}), Expr(Expr) {}
3389
3390 ~VPExpandSCEVRecipe() override = default;
3391
3392 VPExpandSCEVRecipe *clone() override { return new VPExpandSCEVRecipe(Expr); }
3393
3394 VP_CLASSOF_IMPL(VPDef::VPExpandSCEVSC)
3395
3396 void execute(VPTransformState &State) override {
3397 llvm_unreachable("SCEV expressions must be expanded before final execute");
3398 }
3399
3400 /// Return the cost of this VPExpandSCEVRecipe.
3402 VPCostContext &Ctx) const override {
3403 // TODO: Compute accurate cost after retiring the legacy cost model.
3404 return 0;
3405 }
3406
3407#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
3408 /// Print the recipe.
3409 void print(raw_ostream &O, const Twine &Indent,
3410 VPSlotTracker &SlotTracker) const override;
3411#endif
3412
3413 const SCEV *getSCEV() const { return Expr; }
3414};
3415
3416/// Canonical scalar induction phi of the vector loop. Starting at the specified
3417/// start value (either 0 or the resume value when vectorizing the epilogue
3418/// loop). VPWidenCanonicalIVRecipe represents the vector version of the
3419/// canonical induction variable.
3421public:
3423 : VPHeaderPHIRecipe(VPDef::VPCanonicalIVPHISC, nullptr, StartV, DL) {}
3424
3425 ~VPCanonicalIVPHIRecipe() override = default;
3426
3428 auto *R = new VPCanonicalIVPHIRecipe(getOperand(0), getDebugLoc());
3429 R->addOperand(getBackedgeValue());
3430 return R;
3431 }
3432
3433 VP_CLASSOF_IMPL(VPDef::VPCanonicalIVPHISC)
3434
3435 void execute(VPTransformState &State) override {
3436 llvm_unreachable("cannot execute this recipe, should be replaced by a "
3437 "scalar phi recipe");
3438 }
3439
3440#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
3441 /// Print the recipe.
3442 void print(raw_ostream &O, const Twine &Indent,
3443 VPSlotTracker &SlotTracker) const override;
3444#endif
3445
3446 /// Returns the scalar type of the induction.
3448 return getStartValue()->getLiveInIRValue()->getType();
3449 }
3450
3451 /// Returns true if the recipe only uses the first lane of operand \p Op.
3452 bool onlyFirstLaneUsed(const VPValue *Op) const override {
3454 "Op must be an operand of the recipe");
3455 return true;
3456 }
3457
3458 /// Returns true if the recipe only uses the first part of operand \p Op.
3459 bool onlyFirstPartUsed(const VPValue *Op) const override {
3461 "Op must be an operand of the recipe");
3462 return true;
3463 }
3464
3465 /// Return the cost of this VPCanonicalIVPHIRecipe.
3467 VPCostContext &Ctx) const override {
3468 // For now, match the behavior of the legacy cost model.
3469 return 0;
3470 }
3471};
3472
3473/// A recipe for generating the active lane mask for the vector loop that is
3474/// used to predicate the vector operations.
3475/// TODO: It would be good to use the existing VPWidenPHIRecipe instead and
3476/// remove VPActiveLaneMaskPHIRecipe.
3478public:
3480 : VPHeaderPHIRecipe(VPDef::VPActiveLaneMaskPHISC, nullptr, StartMask,
3481 DL) {}
3482
3483 ~VPActiveLaneMaskPHIRecipe() override = default;
3484
3487 if (getNumOperands() == 2)
3488 R->addOperand(getOperand(1));
3489 return R;
3490 }
3491
3492 VP_CLASSOF_IMPL(VPDef::VPActiveLaneMaskPHISC)
3493
3494 /// Generate the active lane mask phi of the vector loop.
3495 void execute(VPTransformState &State) override;
3496
3497#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
3498 /// Print the recipe.
3499 void print(raw_ostream &O, const Twine &Indent,
3500 VPSlotTracker &SlotTracker) const override;
3501#endif
3502};
3503
3504/// A recipe for generating the phi node for the current index of elements,
3505/// adjusted in accordance with EVL value. It starts at the start value of the
3506/// canonical induction and gets incremented by EVL in each iteration of the
3507/// vector loop.
3509public:
3511 : VPHeaderPHIRecipe(VPDef::VPEVLBasedIVPHISC, nullptr, StartIV, DL) {}
3512
3513 ~VPEVLBasedIVPHIRecipe() override = default;
3514
3516 llvm_unreachable("cloning not implemented yet");
3517 }
3518
3519 VP_CLASSOF_IMPL(VPDef::VPEVLBasedIVPHISC)
3520
3521 void execute(VPTransformState &State) override {
3522 llvm_unreachable("cannot execute this recipe, should be replaced by a "
3523 "scalar phi recipe");
3524 }
3525
3526 /// Return the cost of this VPEVLBasedIVPHIRecipe.
3528 VPCostContext &Ctx) const override {
3529 // For now, match the behavior of the legacy cost model.
3530 return 0;
3531 }
3532
3533 /// Returns true if the recipe only uses the first lane of operand \p Op.
3534 bool onlyFirstLaneUsed(const VPValue *Op) const override {
3536 "Op must be an operand of the recipe");
3537 return true;
3538 }
3539
3540#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
3541 /// Print the recipe.
3542 void print(raw_ostream &O, const Twine &Indent,
3543 VPSlotTracker &SlotTracker) const override;
3544#endif
3545};
3546
3547/// A Recipe for widening the canonical induction variable of the vector loop.
3549 public VPUnrollPartAccessor<1> {
3550public:
3552 : VPSingleDefRecipe(VPDef::VPWidenCanonicalIVSC, {CanonicalIV}) {}
3553
3554 ~VPWidenCanonicalIVRecipe() override = default;
3555
3560
3561 VP_CLASSOF_IMPL(VPDef::VPWidenCanonicalIVSC)
3562
3563 /// Generate a canonical vector induction variable of the vector loop, with
3564 /// start = {<Part*VF, Part*VF+1, ..., Part*VF+VF-1> for 0 <= Part < UF}, and
3565 /// step = <VF*UF, VF*UF, ..., VF*UF>.
3566 void execute(VPTransformState &State) override;
3567
3568 /// Return the cost of this VPWidenCanonicalIVPHIRecipe.
3570 VPCostContext &Ctx) const override {
3571 // TODO: Compute accurate cost after retiring the legacy cost model.
3572 return 0;
3573 }
3574
3575#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
3576 /// Print the recipe.
3577 void print(raw_ostream &O, const Twine &Indent,
3578 VPSlotTracker &SlotTracker) const override;
3579#endif
3580};
3581
3582/// A recipe for converting the input value \p IV value to the corresponding
3583/// value of an IV with different start and step values, using Start + IV *
3584/// Step.
3586 /// Kind of the induction.
3588 /// If not nullptr, the floating point induction binary operator. Must be set
3589 /// for floating point inductions.
3590 const FPMathOperator *FPBinOp;
3591
3592 /// Name to use for the generated IR instruction for the derived IV.
3593 std::string Name;
3594
3595public:
3597 VPCanonicalIVPHIRecipe *CanonicalIV, VPValue *Step,
3598 const Twine &Name = "")
3600 IndDesc.getKind(),
3601 dyn_cast_or_null<FPMathOperator>(IndDesc.getInductionBinOp()),
3602 Start, CanonicalIV, Step, Name) {}
3603
3605 const FPMathOperator *FPBinOp, VPValue *Start, VPValue *IV,
3606 VPValue *Step, const Twine &Name = "")
3607 : VPSingleDefRecipe(VPDef::VPDerivedIVSC, {Start, IV, Step}), Kind(Kind),
3608 FPBinOp(FPBinOp), Name(Name.str()) {}
3609
3610 ~VPDerivedIVRecipe() override = default;
3611
3613 return new VPDerivedIVRecipe(Kind, FPBinOp, getStartValue(), getOperand(1),
3614 getStepValue());
3615 }
3616
3617 VP_CLASSOF_IMPL(VPDef::VPDerivedIVSC)
3618
3619 /// Generate the transformed value of the induction at offset StartValue (1.
3620 /// operand) + IV (2. operand) * StepValue (3, operand).
3621 void execute(VPTransformState &State) override;
3622
3623 /// Return the cost of this VPDerivedIVRecipe.
3625 VPCostContext &Ctx) const override {
3626 // TODO: Compute accurate cost after retiring the legacy cost model.
3627 return 0;
3628 }
3629
3630#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
3631 /// Print the recipe.
3632 void print(raw_ostream &O, const Twine &Indent,
3633 VPSlotTracker &SlotTracker) const override;
3634#endif
3635
3637 return getStartValue()->getLiveInIRValue()->getType();
3638 }
3639
3640 VPValue *getStartValue() const { return getOperand(0); }
3641 VPValue *getStepValue() const { return getOperand(2); }
3642
3643 /// Returns true if the recipe only uses the first lane of operand \p Op.
3644 bool onlyFirstLaneUsed(const VPValue *Op) const override {
3646 "Op must be an operand of the recipe");
3647 return true;
3648 }
3649};
3650
3651/// A recipe for handling phi nodes of integer and floating-point inductions,
3652/// producing their scalar values.
3654 public VPUnrollPartAccessor<3> {
3655 Instruction::BinaryOps InductionOpcode;
3656
3657public:
3660 DebugLoc DL)
3661 : VPRecipeWithIRFlags(VPDef::VPScalarIVStepsSC,
3662 ArrayRef<VPValue *>({IV, Step, VF}), FMFs, DL),
3663 InductionOpcode(Opcode) {}
3664
3666 VPValue *Step, VPValue *VF,
3669 IV, Step, VF, IndDesc.getInductionOpcode(),
3670 dyn_cast_or_null<FPMathOperator>(IndDesc.getInductionBinOp())
3671 ? IndDesc.getInductionBinOp()->getFastMathFlags()
3672 : FastMathFlags(),
3673 DL) {}
3674
3675 ~VPScalarIVStepsRecipe() override = default;
3676
3678 return new VPScalarIVStepsRecipe(
3679 getOperand(0), getOperand(1), getOperand(2), InductionOpcode,
3681 getDebugLoc());
3682 }
3683
3684 /// Return true if this VPScalarIVStepsRecipe corresponds to part 0. Note that
3685 /// this is only accurate after the VPlan has been unrolled.
3686 bool isPart0() const { return getUnrollPart(*this) == 0; }
3687
3688 VP_CLASSOF_IMPL(VPDef::VPScalarIVStepsSC)
3689
3690 /// Generate the scalarized versions of the phi node as needed by their users.
3691 void execute(VPTransformState &State) override;
3692
3693 /// Return the cost of this VPScalarIVStepsRecipe.
3695 VPCostContext &Ctx) const override {
3696 // TODO: Compute accurate cost after retiring the legacy cost model.
3697 return 0;
3698 }
3699
3700#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
3701 /// Print the recipe.
3702 void print(raw_ostream &O, const Twine &Indent,
3703 VPSlotTracker &SlotTracker) const override;
3704#endif
3705
3706 VPValue *getStepValue() const { return getOperand(1); }
3707
3708 /// Returns true if the recipe only uses the first lane of operand \p Op.
3709 bool onlyFirstLaneUsed(const VPValue *Op) const override {
3711 "Op must be an operand of the recipe");
3712 return true;
3713 }
3714};
3715
3716/// Casting from VPRecipeBase -> VPPhiAccessors is supported for all recipe
3717/// types implementing VPPhiAccessors. Used by isa<> & co.
3719 static inline bool isPossible(const VPRecipeBase *f) {
3720 // TODO: include VPPredInstPHIRecipe too, once it implements VPPhiAccessors.
3722 }
3723};
3724/// Support casting from VPRecipeBase -> VPPhiAccessors, by down-casting to the
3725/// recipe types implementing VPPhiAccessors. Used by cast<>, dyn_cast<> & co.
3726template <typename SrcTy>
3727struct CastInfoVPPhiAccessors : public CastIsPossible<VPPhiAccessors, SrcTy> {
3728
3730
3731 /// doCast is used by cast<>.
3732 static inline VPPhiAccessors *doCast(SrcTy R) {
3733 return const_cast<VPPhiAccessors *>([R]() -> const VPPhiAccessors * {
3734 switch (R->getVPDefID()) {
3735 case VPDef::VPInstructionSC:
3736 return cast<VPPhi>(R);
3737 case VPDef::VPIRInstructionSC:
3738 return cast<VPIRPhi>(R);
3739 case VPDef::VPWidenPHISC:
3740 return cast<VPWidenPHIRecipe>(R);
3741 default:
3742 return cast<VPHeaderPHIRecipe>(R);
3743 }
3744 }());
3745 }
3746
3747 /// doCastIfPossible is used by dyn_cast<>.
3748 static inline VPPhiAccessors *doCastIfPossible(SrcTy f) {
3749 if (!Self::isPossible(f))
3750 return nullptr;
3751 return doCast(f);
3752 }
3753};
3754template <>
3757template <>
3760
3761/// VPBasicBlock serves as the leaf of the Hierarchical Control-Flow Graph. It
3762/// holds a sequence of zero or more VPRecipe's each representing a sequence of
3763/// output IR instructions. All PHI-like recipes must come before any non-PHI recipes.
3764class LLVM_ABI_FOR_TEST VPBasicBlock : public VPBlockBase {
3765 friend class VPlan;
3766
3767 /// Use VPlan::createVPBasicBlock to create VPBasicBlocks.
3768 VPBasicBlock(const Twine &Name = "", VPRecipeBase *Recipe = nullptr)
3769 : VPBlockBase(VPBasicBlockSC, Name.str()) {
3770 if (Recipe)
3771 appendRecipe(Recipe);
3772 }
3773
3774public:
3776
3777protected:
3778 /// The VPRecipes held in the order of output instructions to generate.
3780
3781 VPBasicBlock(const unsigned char BlockSC, const Twine &Name = "")
3782 : VPBlockBase(BlockSC, Name.str()) {}
3783
3784public:
3785 ~VPBasicBlock() override {
3786 while (!Recipes.empty())
3787 Recipes.pop_back();
3788 }
3789
3790 /// Instruction iterators...
3795
3796 //===--------------------------------------------------------------------===//
3797 /// Recipe iterator methods
3798 ///
3799 inline iterator begin() { return Recipes.begin(); }
3800 inline const_iterator begin() const { return Recipes.begin(); }
3801 inline iterator end() { return Recipes.end(); }
3802 inline const_iterator end() const { return Recipes.end(); }
3803
3804 inline reverse_iterator rbegin() { return Recipes.rbegin(); }
3805 inline const_reverse_iterator rbegin() const { return Recipes.rbegin(); }
3806 inline reverse_iterator rend() { return Recipes.rend(); }
3807 inline const_reverse_iterator rend() const { return Recipes.rend(); }
3808
3809 inline size_t size() const { return Recipes.size(); }
3810 inline bool empty() const { return Recipes.empty(); }
3811 inline const VPRecipeBase &front() const { return Recipes.front(); }
3812 inline VPRecipeBase &front() { return Recipes.front(); }
3813 inline const VPRecipeBase &back() const { return Recipes.back(); }
3814 inline VPRecipeBase &back() { return Recipes.back(); }
3815
3816 /// Returns a reference to the list of recipes.
3818
3819 /// Returns a pointer to a member of the recipe list.
3820 static RecipeListTy VPBasicBlock::*getSublistAccess(VPRecipeBase *) {
3821 return &VPBasicBlock::Recipes;
3822 }
3823
3824 /// Method to support type inquiry through isa, cast, and dyn_cast.
3825 static inline bool classof(const VPBlockBase *V) {
3826 return V->getVPBlockID() == VPBlockBase::VPBasicBlockSC ||
3827 V->getVPBlockID() == VPBlockBase::VPIRBasicBlockSC;
3828 }
3829
3830 void insert(VPRecipeBase *Recipe, iterator InsertPt) {
3831 assert(Recipe && "No recipe to append.");
3832 assert(!Recipe->Parent && "Recipe already in VPlan");
3833 Recipe->Parent = this;
3834 Recipes.insert(InsertPt, Recipe);
3835 }
3836
3837 /// Augment the existing recipes of a VPBasicBlock with an additional
3838 /// \p Recipe as the last recipe.
3839 void appendRecipe(VPRecipeBase *Recipe) { insert(Recipe, end()); }
3840
3841 /// The method which generates the output IR instructions that correspond to
3842 /// this VPBasicBlock, thereby "executing" the VPlan.
3843 void execute(VPTransformState *State) override;
3844
3845 /// Return the cost of this VPBasicBlock.
3846 InstructionCost cost(ElementCount VF, VPCostContext &Ctx) override;
3847
3848 /// Return the position of the first non-phi node recipe in the block.
3849 iterator getFirstNonPhi();
3850
3851 /// Returns an iterator range over the PHI-like recipes in the block.
3855
3856 /// Split current block at \p SplitAt by inserting a new block between the
3857 /// current block and its successors and moving all recipes starting at
3858 /// SplitAt to the new block. Returns the new block.
3859 VPBasicBlock *splitAt(iterator SplitAt);
3860
3861 VPRegionBlock *getEnclosingLoopRegion();
3862 const VPRegionBlock *getEnclosingLoopRegion() const;
3863
3864#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
3865 /// Print this VPBsicBlock to \p O, prefixing all lines with \p Indent. \p
3866 /// SlotTracker is used to print unnamed VPValue's using consequtive numbers.
3867 ///
3868 /// Note that the numbering is applied to the whole VPlan, so printing
3869 /// individual blocks is consistent with the whole VPlan printing.
3870 void print(raw_ostream &O, const Twine &Indent,
3871 VPSlotTracker &SlotTracker) const override;
3872 using VPBlockBase::print; // Get the print(raw_stream &O) version.
3873#endif
3874
3875 /// If the block has multiple successors, return the branch recipe terminating
3876 /// the block. If there are no or only a single successor, return nullptr;
3877 VPRecipeBase *getTerminator();
3878 const VPRecipeBase *getTerminator() const;
3879
3880 /// Returns true if the block is exiting it's parent region.
3881 bool isExiting() const;
3882
3883 /// Clone the current block and it's recipes, without updating the operands of
3884 /// the cloned recipes.
3885 VPBasicBlock *clone() override;
3886
3887 /// Returns the predecessor block at index \p Idx with the predecessors as per
3888 /// the corresponding plain CFG. If the block is an entry block to a region,
3889 /// the first predecessor is the single predecessor of a region, and the
3890 /// second predecessor is the exiting block of the region.
3891 const VPBasicBlock *getCFGPredecessor(unsigned Idx) const;
3892
3893protected:
3894 /// Execute the recipes in the IR basic block \p BB.
3895 void executeRecipes(VPTransformState *State, BasicBlock *BB);
3896
3897 /// Connect the VPBBs predecessors' in the VPlan CFG to the IR basic block
3898 /// generated for this VPBB.
3899 void connectToPredecessors(VPTransformState &State);
3900
3901private:
3902 /// Create an IR BasicBlock to hold the output instructions generated by this
3903 /// VPBasicBlock, and return it. Update the CFGState accordingly.
3904 BasicBlock *createEmptyBasicBlock(VPTransformState &State);
3905};
3906
3907inline const VPBasicBlock *
3909 return getAsRecipe()->getParent()->getCFGPredecessor(Idx);
3910}
3911
3912/// A special type of VPBasicBlock that wraps an existing IR basic block.
3913/// Recipes of the block get added before the first non-phi instruction in the
3914/// wrapped block.
3915/// Note: At the moment, VPIRBasicBlock can only be used to wrap VPlan's
3916/// preheader block.
3917class VPIRBasicBlock : public VPBasicBlock {
3918 friend class VPlan;
3919
3920 BasicBlock *IRBB;
3921
3922 /// Use VPlan::createVPIRBasicBlock to create VPIRBasicBlocks.
3923 VPIRBasicBlock(BasicBlock *IRBB)
3924 : VPBasicBlock(VPIRBasicBlockSC,
3925 (Twine("ir-bb<") + IRBB->getName() + Twine(">")).str()),
3926 IRBB(IRBB) {}
3927
3928public:
3929 ~VPIRBasicBlock() override {}
3930
3931 static inline bool classof(const VPBlockBase *V) {
3932 return V->getVPBlockID() == VPBlockBase::VPIRBasicBlockSC;
3933 }
3934
3935 /// The method which generates the output IR instructions that correspond to
3936 /// this VPBasicBlock, thereby "executing" the VPlan.
3937 void execute(VPTransformState *State) override;
3938
3939 VPIRBasicBlock *clone() override;
3940
3941 BasicBlock *getIRBasicBlock() const { return IRBB; }
3942};
3943
3944/// VPRegionBlock represents a collection of VPBasicBlocks and VPRegionBlocks
3945/// which form a Single-Entry-Single-Exiting subgraph of the output IR CFG.
3946/// A VPRegionBlock may indicate that its contents are to be replicated several
3947/// times. This is designed to support predicated scalarization, in which a
3948/// scalar if-then code structure needs to be generated VF * UF times. Having
3949/// this replication indicator helps to keep a single model for multiple
3950/// candidate VF's. The actual replication takes place only once the desired VF
3951/// and UF have been determined.
3952class LLVM_ABI_FOR_TEST VPRegionBlock : public VPBlockBase {
3953 friend class VPlan;
3954
3955 /// Hold the Single Entry of the SESE region modelled by the VPRegionBlock.
3956 VPBlockBase *Entry;
3957
3958 /// Hold the Single Exiting block of the SESE region modelled by the
3959 /// VPRegionBlock.
3960 VPBlockBase *Exiting;
3961
3962 /// An indicator whether this region is to generate multiple replicated
3963 /// instances of output IR corresponding to its VPBlockBases.
3964 bool IsReplicator;
3965
3966 /// Use VPlan::createVPRegionBlock to create VPRegionBlocks.
3967 VPRegionBlock(VPBlockBase *Entry, VPBlockBase *Exiting,
3968 const std::string &Name = "", bool IsReplicator = false)
3969 : VPBlockBase(VPRegionBlockSC, Name), Entry(Entry), Exiting(Exiting),
3970 IsReplicator(IsReplicator) {
3971 assert(Entry->getPredecessors().empty() && "Entry block has predecessors.");
3972 assert(Exiting->getSuccessors().empty() && "Exit block has successors.");
3973 Entry->setParent(this);
3974 Exiting->setParent(this);
3975 }
3976 VPRegionBlock(const std::string &Name = "", bool IsReplicator = false)
3977 : VPBlockBase(VPRegionBlockSC, Name), Entry(nullptr), Exiting(nullptr),
3978 IsReplicator(IsReplicator) {}
3979
3980public:
3981 ~VPRegionBlock() override {}
3982
3983 /// Method to support type inquiry through isa, cast, and dyn_cast.
3984 static inline bool classof(const VPBlockBase *V) {
3985 return V->getVPBlockID() == VPBlockBase::VPRegionBlockSC;
3986 }
3987
3988 const VPBlockBase *getEntry() const { return Entry; }
3989 VPBlockBase *getEntry() { return Entry; }
3990
3991 /// Set \p EntryBlock as the entry VPBlockBase of this VPRegionBlock. \p
3992 /// EntryBlock must have no predecessors.
3993 void setEntry(VPBlockBase *EntryBlock) {
3994 assert(EntryBlock->getPredecessors().empty() &&
3995 "Entry block cannot have predecessors.");
3996 Entry = EntryBlock;
3997 EntryBlock->setParent(this);
3998 }
3999
4000 const VPBlockBase *getExiting() const { return Exiting; }
4001 VPBlockBase *getExiting() { return Exiting; }
4002
4003 /// Set \p ExitingBlock as the exiting VPBlockBase of this VPRegionBlock. \p
4004 /// ExitingBlock must have no successors.
4005 void setExiting(VPBlockBase *ExitingBlock) {
4006 assert(ExitingBlock->getSuccessors().empty() &&
4007 "Exit block cannot have successors.");
4008 Exiting = ExitingBlock;
4009 ExitingBlock->setParent(this);
4010 }
4011
4012 /// Returns the pre-header VPBasicBlock of the loop region.
4014 assert(!isReplicator() && "should only get pre-header of loop regions");
4015 return getSinglePredecessor()->getExitingBasicBlock();
4016 }
4017
4018 /// An indicator whether this region is to generate multiple replicated
4019 /// instances of output IR corresponding to its VPBlockBases.
4020 bool isReplicator() const { return IsReplicator; }
4021
4022 /// The method which generates the output IR instructions that correspond to
4023 /// this VPRegionBlock, thereby "executing" the VPlan.
4024 void execute(VPTransformState *State) override;
4025
4026 // Return the cost of this region.
4027 InstructionCost cost(ElementCount VF, VPCostContext &Ctx) override;
4028
4029#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
4030 /// Print this VPRegionBlock to \p O (recursively), prefixing all lines with
4031 /// \p Indent. \p SlotTracker is used to print unnamed VPValue's using
4032 /// consequtive numbers.
4033 ///
4034 /// Note that the numbering is applied to the whole VPlan, so printing
4035 /// individual regions is consistent with the whole VPlan printing.
4036 void print(raw_ostream &O, const Twine &Indent,
4037 VPSlotTracker &SlotTracker) const override;
4038 using VPBlockBase::print; // Get the print(raw_stream &O) version.
4039#endif
4040
4041 /// Clone all blocks in the single-entry single-exit region of the block and
4042 /// their recipes without updating the operands of the cloned recipes.
4043 VPRegionBlock *clone() override;
4044
4045 /// Remove the current region from its VPlan, connecting its predecessor to
4046 /// its entry, and its exiting block to its successor.
4047 void dissolveToCFGLoop();
4048};
4049
4050/// VPlan models a candidate for vectorization, encoding various decisions take
4051/// to produce efficient output IR, including which branches, basic-blocks and
4052/// output IR instructions to generate, and their cost. VPlan holds a
4053/// Hierarchical-CFG of VPBasicBlocks and VPRegionBlocks rooted at an Entry
4054/// VPBasicBlock.
4055class VPlan {
4056 friend class VPlanPrinter;
4057 friend class VPSlotTracker;
4058
4059 /// VPBasicBlock corresponding to the original preheader. Used to place
4060 /// VPExpandSCEV recipes for expressions used during skeleton creation and the
4061 /// rest of VPlan execution.
4062 /// When this VPlan is used for the epilogue vector loop, the entry will be
4063 /// replaced by a new entry block created during skeleton creation.
4064 VPBasicBlock *Entry;
4065
4066 /// VPIRBasicBlock wrapping the header of the original scalar loop.
4067 VPIRBasicBlock *ScalarHeader;
4068
4069 /// Immutable list of VPIRBasicBlocks wrapping the exit blocks of the original
4070 /// scalar loop. Note that some exit blocks may be unreachable at the moment,
4071 /// e.g. if the scalar epilogue always executes.
4073
4074 /// Holds the VFs applicable to this VPlan.
4076
4077 /// Holds the UFs applicable to this VPlan. If empty, the VPlan is valid for
4078 /// any UF.
4080
4081 /// Holds the name of the VPlan, for printing.
4082 std::string Name;
4083
4084 /// Represents the trip count of the original loop, for folding
4085 /// the tail.
4086 VPValue *TripCount = nullptr;
4087
4088 /// Represents the backedge taken count of the original loop, for folding
4089 /// the tail. It equals TripCount - 1.
4090 VPValue *BackedgeTakenCount = nullptr;
4091
4092 /// Represents the vector trip count.
4093 VPValue VectorTripCount;
4094
4095 /// Represents the vectorization factor of the loop.
4096 VPValue VF;
4097
4098 /// Represents the loop-invariant VF * UF of the vector loop region.
4099 VPValue VFxUF;
4100
4101 /// Holds a mapping between Values and their corresponding VPValue inside
4102 /// VPlan.
4103 Value2VPValueTy Value2VPValue;
4104
4105 /// Contains all the external definitions created for this VPlan. External
4106 /// definitions are VPValues that hold a pointer to their underlying IR.
4108
4109 /// Mapping from SCEVs to the VPValues representing their expansions.
4110 /// NOTE: This mapping is temporary and will be removed once all users have
4111 /// been modeled in VPlan directly.
4112 DenseMap<const SCEV *, VPValue *> SCEVToExpansion;
4113
4114 /// Blocks allocated and owned by the VPlan. They will be deleted once the
4115 /// VPlan is destroyed.
4116 SmallVector<VPBlockBase *> CreatedBlocks;
4117
4118 /// Construct a VPlan with \p Entry to the plan and with \p ScalarHeader
4119 /// wrapping the original header of the scalar loop.
4120 VPlan(VPBasicBlock *Entry, VPIRBasicBlock *ScalarHeader)
4121 : Entry(Entry), ScalarHeader(ScalarHeader) {
4122 Entry->setPlan(this);
4123 assert(ScalarHeader->getNumSuccessors() == 0 &&
4124 "scalar header must be a leaf node");
4125 }
4126
4127public:
4128 /// Construct a VPlan for \p L. This will create VPIRBasicBlocks wrapping the
4129 /// original preheader and scalar header of \p L, to be used as entry and
4130 /// scalar header blocks of the new VPlan.
4131 VPlan(Loop *L);
4132
4133 /// Construct a VPlan with a new VPBasicBlock as entry, a VPIRBasicBlock
4134 /// wrapping \p ScalarHeaderBB and a trip count of \p TC.
4135 VPlan(BasicBlock *ScalarHeaderBB, VPValue *TC) {
4136 setEntry(createVPBasicBlock("preheader"));
4137 ScalarHeader = createVPIRBasicBlock(ScalarHeaderBB);
4138 TripCount = TC;
4139 }
4140
4142
4144 Entry = VPBB;
4145 VPBB->setPlan(this);
4146 }
4147
4148 /// Generate the IR code for this VPlan.
4149 void execute(VPTransformState *State);
4150
4151 /// Return the cost of this plan.
4153
4154 VPBasicBlock *getEntry() { return Entry; }
4155 const VPBasicBlock *getEntry() const { return Entry; }
4156
4157 /// Returns the preheader of the vector loop region, if one exists, or null
4158 /// otherwise.
4160 VPRegionBlock *VectorRegion = getVectorLoopRegion();
4161 return VectorRegion
4162 ? cast<VPBasicBlock>(VectorRegion->getSinglePredecessor())
4163 : nullptr;
4164 }
4165
4166 /// Returns the VPRegionBlock of the vector loop.
4169
4170 /// Returns the 'middle' block of the plan, that is the block that selects
4171 /// whether to execute the scalar tail loop or the exit block from the loop
4172 /// latch. If there is an early exit from the vector loop, the middle block
4173 /// conceptully has the early exit block as third successor, split accross 2
4174 /// VPBBs. In that case, the second VPBB selects whether to execute the scalar
4175 /// tail loop or the exit bock. If the scalar tail loop or exit block are
4176 /// known to always execute, the middle block may branch directly to that
4177 /// block. This function cannot be called once the vector loop region has been
4178 /// removed.
4180 VPRegionBlock *LoopRegion = getVectorLoopRegion();
4181 assert(
4182 LoopRegion &&
4183 "cannot call the function after vector loop region has been removed");
4184 auto *RegionSucc = cast<VPBasicBlock>(LoopRegion->getSingleSuccessor());
4185 if (RegionSucc->getSingleSuccessor() ||
4186 is_contained(RegionSucc->getSuccessors(), getScalarPreheader()))
4187 return RegionSucc;
4188 // There is an early exit. The successor of RegionSucc is the middle block.
4189 return cast<VPBasicBlock>(RegionSucc->getSuccessors()[1]);
4190 }
4191
4193 return const_cast<VPlan *>(this)->getMiddleBlock();
4194 }
4195
4196 /// Return the VPBasicBlock for the preheader of the scalar loop.
4198 return cast<VPBasicBlock>(getScalarHeader()->getSinglePredecessor());
4199 }
4200
4201 /// Return the VPIRBasicBlock wrapping the header of the scalar loop.
4202 VPIRBasicBlock *getScalarHeader() const { return ScalarHeader; }
4203
4204 /// Return an ArrayRef containing VPIRBasicBlocks wrapping the exit blocks of
4205 /// the original scalar loop.
4206 ArrayRef<VPIRBasicBlock *> getExitBlocks() const { return ExitBlocks; }
4207
4208 /// Return the VPIRBasicBlock corresponding to \p IRBB. \p IRBB must be an
4209 /// exit block.
4211
4212 /// Returns true if \p VPBB is an exit block.
4213 bool isExitBlock(VPBlockBase *VPBB);
4214
4215 /// The trip count of the original loop.
4217 assert(TripCount && "trip count needs to be set before accessing it");
4218 return TripCount;
4219 }
4220
4221 /// Set the trip count assuming it is currently null; if it is not - use
4222 /// resetTripCount().
4223 void setTripCount(VPValue *NewTripCount) {
4224 assert(!TripCount && NewTripCount && "TripCount should not be set yet.");
4225 TripCount = NewTripCount;
4226 }
4227
4228 /// Resets the trip count for the VPlan. The caller must make sure all uses of
4229 /// the original trip count have been replaced.
4230 void resetTripCount(VPValue *NewTripCount) {
4231 assert(TripCount && NewTripCount && TripCount->getNumUsers() == 0 &&
4232 "TripCount must be set when resetting");
4233 TripCount = NewTripCount;
4234 }
4235
4236 /// The backedge taken count of the original loop.
4238 if (!BackedgeTakenCount)
4239 BackedgeTakenCount = new VPValue();
4240 return BackedgeTakenCount;
4241 }
4242
4243 /// The vector trip count.
4244 VPValue &getVectorTripCount() { return VectorTripCount; }
4245
4246 /// Returns the VF of the vector loop region.
4247 VPValue &getVF() { return VF; };
4248
4249 /// Returns VF * UF of the vector loop region.
4250 VPValue &getVFxUF() { return VFxUF; }
4251
4254 }
4255
4256 void addVF(ElementCount VF) { VFs.insert(VF); }
4257
4259 assert(hasVF(VF) && "Cannot set VF not already in plan");
4260 VFs.clear();
4261 VFs.insert(VF);
4262 }
4263
4264 bool hasVF(ElementCount VF) const { return VFs.count(VF); }
4265 bool hasScalableVF() const {
4266 return any_of(VFs, [](ElementCount VF) { return VF.isScalable(); });
4267 }
4268
4269 /// Returns an iterator range over all VFs of the plan.
4272 return VFs;
4273 }
4274
4275 bool hasScalarVFOnly() const {
4276 bool HasScalarVFOnly = VFs.size() == 1 && VFs[0].isScalar();
4277 assert(HasScalarVFOnly == hasVF(ElementCount::getFixed(1)) &&
4278 "Plan with scalar VF should only have a single VF");
4279 return HasScalarVFOnly;
4280 }
4281
4282 bool hasUF(unsigned UF) const { return UFs.empty() || UFs.contains(UF); }
4283
4284 unsigned getUF() const {
4285 assert(UFs.size() == 1 && "Expected a single UF");
4286 return UFs[0];
4287 }
4288
4289 void setUF(unsigned UF) {
4290 assert(hasUF(UF) && "Cannot set the UF not already in plan");
4291 UFs.clear();
4292 UFs.insert(UF);
4293 }
4294
4295 /// Returns true if the VPlan already has been unrolled, i.e. it has a single
4296 /// concrete UF.
4297 bool isUnrolled() const { return UFs.size() == 1; }
4298
4299 /// Return a string with the name of the plan and the applicable VFs and UFs.
4300 std::string getName() const;
4301
4302 void setName(const Twine &newName) { Name = newName.str(); }
4303
4304 /// Gets the live-in VPValue for \p V or adds a new live-in (if none exists
4305 /// yet) for \p V.
4307 assert(V && "Trying to get or add the VPValue of a null Value");
4308 auto [It, Inserted] = Value2VPValue.try_emplace(V);
4309 if (Inserted) {
4310 VPValue *VPV = new VPValue(V);
4311 VPLiveIns.push_back(VPV);
4312 assert(VPV->isLiveIn() && "VPV must be a live-in.");
4313 It->second = VPV;
4314 }
4315
4316 assert(It->second->isLiveIn() && "Only live-ins should be in mapping");
4317 return It->second;
4318 }
4319
4320 /// Return a VPValue wrapping i1 true.
4322 LLVMContext &Ctx = getContext();
4324 }
4325
4326 /// Return a VPValue wrapping i1 false.
4328 LLVMContext &Ctx = getContext();
4330 }
4331
4332 /// Return the live-in VPValue for \p V, if there is one or nullptr otherwise.
4333 VPValue *getLiveIn(Value *V) const { return Value2VPValue.lookup(V); }
4334
4335 /// Return the list of live-in VPValues available in the VPlan.
4337 assert(all_of(Value2VPValue,
4338 [this](const auto &P) {
4339 return is_contained(VPLiveIns, P.second);
4340 }) &&
4341 "all VPValues in Value2VPValue must also be in VPLiveIns");
4342 return VPLiveIns;
4343 }
4344
4345#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
4346 /// Print the live-ins of this VPlan to \p O.
4347 void printLiveIns(raw_ostream &O) const;
4348
4349 /// Print this VPlan to \p O.
4350 void print(raw_ostream &O) const;
4351
4352 /// Print this VPlan in DOT format to \p O.
4353 void printDOT(raw_ostream &O) const;
4354
4355 /// Dump the plan to stderr (for debugging).
4356 LLVM_DUMP_METHOD void dump() const;
4357#endif
4358
4359 /// Returns the canonical induction recipe of the vector loop.
4362 if (EntryVPBB->empty()) {
4363 // VPlan native path.
4364 EntryVPBB = cast<VPBasicBlock>(EntryVPBB->getSingleSuccessor());
4365 }
4366 return cast<VPCanonicalIVPHIRecipe>(&*EntryVPBB->begin());
4367 }
4368
4369 VPValue *getSCEVExpansion(const SCEV *S) const {
4370 return SCEVToExpansion.lookup(S);
4371 }
4372
4373 void addSCEVExpansion(const SCEV *S, VPValue *V) {
4374 assert(!SCEVToExpansion.contains(S) && "SCEV already expanded");
4375 SCEVToExpansion[S] = V;
4376 }
4377
4378 /// Clone the current VPlan, update all VPValues of the new VPlan and cloned
4379 /// recipes to refer to the clones, and return it.
4380 VPlan *duplicate();
4381
4382 /// Create a new VPBasicBlock with \p Name and containing \p Recipe if
4383 /// present. The returned block is owned by the VPlan and deleted once the
4384 /// VPlan is destroyed.
4386 VPRecipeBase *Recipe = nullptr) {
4387 auto *VPB = new VPBasicBlock(Name, Recipe);
4388 CreatedBlocks.push_back(VPB);
4389 return VPB;
4390 }
4391
4392 /// Create a new VPRegionBlock with \p Entry, \p Exiting and \p Name. If \p
4393 /// IsReplicator is true, the region is a replicate region. The returned block
4394 /// is owned by the VPlan and deleted once the VPlan is destroyed.
4396 const std::string &Name = "",
4397 bool IsReplicator = false) {
4398 auto *VPB = new VPRegionBlock(Entry, Exiting, Name, IsReplicator);
4399 CreatedBlocks.push_back(VPB);
4400 return VPB;
4401 }
4402
4403 /// Create a new loop VPRegionBlock with \p Name and entry and exiting blocks set
4404 /// to nullptr. The returned block is owned by the VPlan and deleted once the
4405 /// VPlan is destroyed.
4406 VPRegionBlock *createVPRegionBlock(const std::string &Name = "") {
4407 auto *VPB = new VPRegionBlock(Name);
4408 CreatedBlocks.push_back(VPB);
4409 return VPB;
4410 }
4411
4412 /// Create a VPIRBasicBlock wrapping \p IRBB, but do not create
4413 /// VPIRInstructions wrapping the instructions in t\p IRBB. The returned
4414 /// block is owned by the VPlan and deleted once the VPlan is destroyed.
4416
4417 /// Create a VPIRBasicBlock from \p IRBB containing VPIRInstructions for all
4418 /// instructions in \p IRBB, except its terminator which is managed by the
4419 /// successors of the block in VPlan. The returned block is owned by the VPlan
4420 /// and deleted once the VPlan is destroyed.
4422
4423 /// Returns true if the VPlan is based on a loop with an early exit. That is
4424 /// the case if the VPlan has either more than one exit block or a single exit
4425 /// block with multiple predecessors (one for the exit via the latch and one
4426 /// via the other early exit).
4427 bool hasEarlyExit() const {
4428 return count_if(ExitBlocks,
4429 [](VPIRBasicBlock *EB) { return EB->hasPredecessors(); }) >
4430 1 ||
4431 (ExitBlocks.size() == 1 && ExitBlocks[0]->getNumPredecessors() > 1);
4432 }
4433
4434 /// Returns true if the scalar tail may execute after the vector loop. Note
4435 /// that this relies on unneeded branches to the scalar tail loop being
4436 /// removed.
4437 bool hasScalarTail() const {
4438 return !(!getScalarPreheader()->hasPredecessors() ||
4440 }
4441};
4442
4443#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
4444inline raw_ostream &operator<<(raw_ostream &OS, const VPlan &Plan) {
4445 Plan.print(OS);
4446 return OS;
4447}
4448#endif
4449
4450} // end namespace llvm
4451
4452#endif // LLVM_TRANSFORMS_VECTORIZE_VPLAN_H
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
aarch64 promote const
static MCDisassembler::DecodeStatus addOperand(MCInst &Inst, const MCOperand &Opnd)
Rewrite undef for PHI
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static void print(raw_ostream &Out, object::Archive::Kind Kind, T Val)
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
#define LLVM_DUMP_METHOD
Mark debug helper function definitions like dump() that should not be stripped from debug builds.
Definition Compiler.h:638
#define LLVM_ABI_FOR_TEST
Definition Compiler.h:218
dxil translate DXIL Translate Metadata
This file defines the DenseMap class.
Hexagon Common GEP
iv users
Definition IVUsers.cpp:48
This file defines an InstructionCost class that is used when calculating the cost of an instruction,...
static std::pair< Value *, APInt > getMask(Value *WideMask, unsigned Factor, ElementCount LeafValueEC)
#define I(x, y, z)
Definition MD5.cpp:58
mir Rename Register Operands
static DebugLoc getDebugLoc(MachineBasicBlock::instr_iterator FirstMI, MachineBasicBlock::instr_iterator LastMI)
Return the first found DebugLoc that has a DILocation, given a range of instructions.
#define T
MachineInstr unsigned OpIdx
#define P(N)
static StringRef getName(Value *V)
const SmallVectorImpl< MachineOperand > & Cond
static bool mayHaveSideEffects(MachineInstr &MI)
This file implements the SmallBitVector class.
This file defines the SmallPtrSet class.
This file defines the SmallVector class.
static const BasicSubtargetSubTypeKV * find(StringRef S, ArrayRef< BasicSubtargetSubTypeKV > A)
Find KV in array using binary search.
static std::optional< unsigned > getOpcode(ArrayRef< VPValue * > Values)
Returns the opcode of Values or ~0 if they do not all agree.
Definition VPlanSLP.cpp:247
This file contains the declarations of the entities induced by Vectorization Plans,...
#define VP_CLASSOF_IMPL(VPDefID)
Definition VPlan.h:499
static const uint32_t IV[8]
Definition blake3_impl.h:83
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition ArrayRef.h:41
This class holds the attributes for a particular argument, parameter, function, or return value.
Definition Attributes.h:361
LLVM Basic Block Representation.
Definition BasicBlock.h:62
LLVM_ABI LLVMContext & getContext() const
Get the context in which this basic block lives.
This class represents a function call, abstracting a target machine's calling convention.
This is the base class for all instructions that perform data casts.
Definition InstrTypes.h:448
Instruction::CastOps getOpcode() const
Return the opcode of this CastInst.
Definition InstrTypes.h:612
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition InstrTypes.h:678
static LLVM_ABI ConstantInt * getTrue(LLVMContext &Context)
static LLVM_ABI ConstantInt * getFalse(LLVMContext &Context)
A debug info location.
Definition DebugLoc.h:124
static DebugLoc getUnknown()
Definition DebugLoc.h:162
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
Definition Dominators.h:165
static constexpr ElementCount getFixed(ScalarTy MinVal)
Definition TypeSize.h:309
Utility class for floating point operations which can have information about relaxed accuracy require...
Definition Operator.h:200
Convenience struct for specifying and reasoning about fast-math flags.
Definition FMF.h:22
Represents flags for the getelementptr instruction/expression.
static GEPNoWrapFlags none()
an instruction for type-safe pointer arithmetic to access elements of arrays and structs
Common base class shared among various IRBuilders.
Definition IRBuilder.h:114
A struct for saving information about induction variables.
InductionKind
This enum represents the kinds of inductions that we support.
InnerLoopVectorizer vectorizes loops which contain only one basic block to a specified vectorization ...
The group of interleaved loads/stores sharing the same stride and close to each other.
This is an important class for using LLVM in a threaded context.
Definition LLVMContext.h:68
An instruction for reading from memory.
LoopVectorizationCostModel - estimates the expected speedups due to vectorization.
This class emits a version of the loop where run-time checks ensure that may-alias pointers can't ove...
Represents a single loop in the control flow graph.
Definition LoopInfo.h:40
Metadata node.
Definition Metadata.h:1077
bool onlyWritesMemory() const
Whether this function only (at most) writes memory.
Definition ModRef.h:221
bool onlyReadsMemory() const
Whether this function only (at most) reads memory.
Definition ModRef.h:218
Root of the metadata hierarchy.
Definition Metadata.h:63
The RecurrenceDescriptor is used to identify recurrences variables in a loop.
This class represents an analyzed expression in the program.
This class represents the LLVM 'select' instruction.
This class provides computation of slot numbers for LLVM Assembly writing.
A SetVector that performs no allocations if smaller than a certain size.
Definition SetVector.h:356
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
iterator erase(const_iterator CI)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
An instruction for storing to memory.
StringRef - Represent a constant reference to a string, i.e.
Definition StringRef.h:55
This class represents a truncation of integer types.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition Twine.h:82
LLVM_ABI std::string str() const
Return the twine contents as a std::string.
Definition Twine.cpp:17
The instances of the Type class are immutable: once they are created, they are never changed.
Definition Type.h:45
void execute(VPTransformState &State) override
Generate the active lane mask phi of the vector loop.
VPActiveLaneMaskPHIRecipe * clone() override
Clone the current recipe.
Definition VPlan.h:3485
VPActiveLaneMaskPHIRecipe(VPValue *StartMask, DebugLoc DL)
Definition VPlan.h:3479
~VPActiveLaneMaskPHIRecipe() override=default
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPBasicBlock serves as the leaf of the Hierarchical Control-Flow Graph.
Definition VPlan.h:3764
RecipeListTy::const_iterator const_iterator
Definition VPlan.h:3792
void appendRecipe(VPRecipeBase *Recipe)
Augment the existing recipes of a VPBasicBlock with an additional Recipe as the last recipe.
Definition VPlan.h:3839
RecipeListTy::const_reverse_iterator const_reverse_iterator
Definition VPlan.h:3794
RecipeListTy::iterator iterator
Instruction iterators...
Definition VPlan.h:3791
RecipeListTy & getRecipeList()
Returns a reference to the list of recipes.
Definition VPlan.h:3817
iplist< VPRecipeBase > RecipeListTy
Definition VPlan.h:3775
VPBasicBlock(const unsigned char BlockSC, const Twine &Name="")
Definition VPlan.h:3781
iterator end()
Definition VPlan.h:3801
iterator begin()
Recipe iterator methods.
Definition VPlan.h:3799
RecipeListTy::reverse_iterator reverse_iterator
Definition VPlan.h:3793
iterator_range< iterator > phis()
Returns an iterator range over the PHI-like recipes in the block.
Definition VPlan.h:3852
const VPBasicBlock * getCFGPredecessor(unsigned Idx) const
Returns the predecessor block at index Idx with the predecessors as per the corresponding plain CFG.
Definition VPlan.cpp:807
iterator getFirstNonPhi()
Return the position of the first non-phi node recipe in the block.
Definition VPlan.cpp:246
~VPBasicBlock() override
Definition VPlan.h:3785
const_reverse_iterator rbegin() const
Definition VPlan.h:3805
reverse_iterator rend()
Definition VPlan.h:3806
RecipeListTy Recipes
The VPRecipes held in the order of output instructions to generate.
Definition VPlan.h:3779
VPRecipeBase & back()
Definition VPlan.h:3814
const VPRecipeBase & front() const
Definition VPlan.h:3811
const_iterator begin() const
Definition VPlan.h:3800
VPRecipeBase & front()
Definition VPlan.h:3812
const VPRecipeBase & back() const
Definition VPlan.h:3813
void insert(VPRecipeBase *Recipe, iterator InsertPt)
Definition VPlan.h:3830
bool empty() const
Definition VPlan.h:3810
const_iterator end() const
Definition VPlan.h:3802
static bool classof(const VPBlockBase *V)
Method to support type inquiry through isa, cast, and dyn_cast.
Definition VPlan.h:3825
static RecipeListTy VPBasicBlock::* getSublistAccess(VPRecipeBase *)
Returns a pointer to a member of the recipe list.
Definition VPlan.h:3820
reverse_iterator rbegin()
Definition VPlan.h:3804
friend class VPlan
Definition VPlan.h:3765
size_t size() const
Definition VPlan.h:3809
const_reverse_iterator rend() const
Definition VPlan.h:3807
bool onlyFirstLaneUsed(const VPValue *Op) const override
Returns true if the recipe only uses the first lane of operand Op.
Definition VPlan.h:2463
VPValue * getIncomingValue(unsigned Idx) const
Return incoming value number Idx.
Definition VPlan.h:2432
VPValue * getMask(unsigned Idx) const
Return mask number Idx.
Definition VPlan.h:2437
unsigned getNumIncomingValues() const
Return the number of incoming values, taking into account when normalized the first incoming value wi...
Definition VPlan.h:2427
void execute(VPTransformState &State) override
The method which generates the output IR instructions that correspond to this VPRecipe,...
Definition VPlan.h:2448
VPBlendRecipe * clone() override
Clone the current recipe.
Definition VPlan.h:2414
VPBlendRecipe(PHINode *Phi, ArrayRef< VPValue * > Operands, DebugLoc DL)
The blend operation is a User of the incoming values and of their respective masks,...
Definition VPlan.h:2409
void setMask(unsigned Idx, VPValue *V)
Set mask number Idx to V.
Definition VPlan.h:2443
bool isNormalized() const
A normalized blend is one that has an odd number of operands, whereby the first operand does not have...
Definition VPlan.h:2423
VPBlockBase is the building block of the Hierarchical Control-Flow Graph.
Definition VPlan.h:81
void setSuccessors(ArrayRef< VPBlockBase * > NewSuccs)
Set each VPBasicBlock in NewSuccss as successor of this VPBlockBase.
Definition VPlan.h:300
VPRegionBlock * getParent()
Definition VPlan.h:173
VPBlocksTy & getPredecessors()
Definition VPlan.h:205
iterator_range< VPBlockBase ** > predecessors()
Definition VPlan.h:202
LLVM_DUMP_METHOD void dump() const
Dump this VPBlockBase to dbgs().
Definition VPlan.h:377
void setName(const Twine &newName)
Definition VPlan.h:166
size_t getNumSuccessors() const
Definition VPlan.h:219
iterator_range< VPBlockBase ** > successors()
Definition VPlan.h:201
virtual void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const =0
Print plain-text dump of this VPBlockBase to O, prefixing all lines with Indent.
bool hasPredecessors() const
Returns true if this block has any predecessors.
Definition VPlan.h:223
void swapSuccessors()
Swap successors of the block. The block must have exactly 2 successors.
Definition VPlan.h:322
void printSuccessors(raw_ostream &O, const Twine &Indent) const
Print the successors of this block to O, prefixing all lines with Indent.
Definition VPlan.cpp:686
SmallVectorImpl< VPBlockBase * > VPBlocksTy
Definition VPlan.h:160
bool isLegalToHoistInto()
Return true if it is legal to hoist instructions into this block.
Definition VPlan.h:349
virtual ~VPBlockBase()=default
const VPBlocksTy & getHierarchicalPredecessors()
Definition VPlan.h:258
unsigned getIndexForSuccessor(const VPBlockBase *Succ) const
Returns the index for Succ in the blocks successor list.
Definition VPlan.h:335
size_t getNumPredecessors() const
Definition VPlan.h:220
void setPredecessors(ArrayRef< VPBlockBase * > NewPreds)
Set each VPBasicBlock in NewPreds as predecessor of this VPBlockBase.
Definition VPlan.h:291
VPBlockBase * getEnclosingBlockWithPredecessors()
Definition VPlan.cpp:212
unsigned getIndexForPredecessor(const VPBlockBase *Pred) const
Returns the index for Pred in the blocks predecessors list.
Definition VPlan.h:328
const VPBlocksTy & getPredecessors() const
Definition VPlan.h:204
virtual VPBlockBase * clone()=0
Clone the current block and it's recipes without updating the operands of the cloned recipes,...
enum { VPRegionBlockSC, VPBasicBlockSC, VPIRBasicBlockSC } VPBlockTy
An enumeration for keeping track of the concrete subclass of VPBlockBase that are actually instantiat...
Definition VPlan.h:158
virtual InstructionCost cost(ElementCount VF, VPCostContext &Ctx)=0
Return the cost of the block.
void setPlan(VPlan *ParentPlan)
Sets the pointer of the plan containing the block.
Definition VPlan.cpp:184
const VPRegionBlock * getParent() const
Definition VPlan.h:174
const std::string & getName() const
Definition VPlan.h:164
void clearSuccessors()
Remove all the successors of this block.
Definition VPlan.h:310
VPBlockBase * getSingleHierarchicalSuccessor()
Definition VPlan.h:248
void setTwoSuccessors(VPBlockBase *IfTrue, VPBlockBase *IfFalse)
Set two given VPBlockBases IfTrue and IfFalse to be the two successors of this VPBlockBase.
Definition VPlan.h:282
VPBlockBase * getSinglePredecessor() const
Definition VPlan.h:215
virtual void execute(VPTransformState *State)=0
The method which generates the output IR that correspond to this VPBlockBase, thereby "executing" the...
const VPBlocksTy & getHierarchicalSuccessors()
Definition VPlan.h:242
void clearPredecessors()
Remove all the predecessor of this block.
Definition VPlan.h:307
friend class VPBlockUtils
Definition VPlan.h:82
unsigned getVPBlockID() const
Definition VPlan.h:171
void printAsOperand(raw_ostream &OS, bool PrintType=false) const
Definition VPlan.h:356
void swapPredecessors()
Swap predecessors of the block.
Definition VPlan.h:314
VPBlockBase(const unsigned char SC, const std::string &N)
Definition VPlan.h:150
VPBlocksTy & getSuccessors()
Definition VPlan.h:199
VPBlockBase * getEnclosingBlockWithSuccessors()
An Enclosing Block of a block B is any block containing B, including B itself.
Definition VPlan.cpp:204
const VPBasicBlock * getEntryBasicBlock() const
Definition VPlan.cpp:170
void setOneSuccessor(VPBlockBase *Successor)
Set a given VPBlockBase Successor as the single successor of this VPBlockBase.
Definition VPlan.h:271
void setParent(VPRegionBlock *P)
Definition VPlan.h:184
VPBlockBase * getSingleHierarchicalPredecessor()
Definition VPlan.h:264
VPBlockBase * getSingleSuccessor() const
Definition VPlan.h:209
const VPBlocksTy & getSuccessors() const
Definition VPlan.h:198
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition VPlan.h:2955
VPBranchOnMaskRecipe * clone() override
Clone the current recipe.
Definition VPlan.h:2939
bool usesScalars(const VPValue *Op) const override
Returns true if the recipe uses scalars of operand Op.
Definition VPlan.h:2963
VPBranchOnMaskRecipe(VPValue *BlockInMask, DebugLoc DL)
Definition VPlan.h:2936
VPlan-based builder utility analogous to IRBuilder.
Canonical scalar induction phi of the vector loop.
Definition VPlan.h:3420
bool onlyFirstPartUsed(const VPValue *Op) const override
Returns true if the recipe only uses the first part of operand Op.
Definition VPlan.h:3459
~VPCanonicalIVPHIRecipe() override=default
VPCanonicalIVPHIRecipe * clone() override
Clone the current recipe.
Definition VPlan.h:3427
VPCanonicalIVPHIRecipe(VPValue *StartV, DebugLoc DL)
Definition VPlan.h:3422
bool onlyFirstLaneUsed(const VPValue *Op) const override
Returns true if the recipe only uses the first lane of operand Op.
Definition VPlan.h:3452
Type * getScalarType() const
Returns the scalar type of the induction.
Definition VPlan.h:3447
void execute(VPTransformState &State) override
Generate the phi nodes.
Definition VPlan.h:3435
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this VPCanonicalIVPHIRecipe.
Definition VPlan.h:3466
This class augments a recipe with a set of VPValues defined by the recipe.
Definition VPlanValue.h:302
friend class VPValue
Definition VPlanValue.h:303
VPDef(const unsigned char SC)
Definition VPlanValue.h:382
void execute(VPTransformState &State) override
Generate the transformed value of the induction at offset StartValue (1.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this VPDerivedIVRecipe.
Definition VPlan.h:3624
VPValue * getStepValue() const
Definition VPlan.h:3641
Type * getScalarType() const
Definition VPlan.h:3636
VPDerivedIVRecipe * clone() override
Clone the current recipe.
Definition VPlan.h:3612
VPDerivedIVRecipe(InductionDescriptor::InductionKind Kind, const FPMathOperator *FPBinOp, VPValue *Start, VPValue *IV, VPValue *Step, const Twine &Name="")
Definition VPlan.h:3604
~VPDerivedIVRecipe() override=default
bool onlyFirstLaneUsed(const VPValue *Op) const override
Returns true if the recipe only uses the first lane of operand Op.
Definition VPlan.h:3644
VPValue * getStartValue() const
Definition VPlan.h:3640
VPDerivedIVRecipe(const InductionDescriptor &IndDesc, VPValue *Start, VPCanonicalIVPHIRecipe *CanonicalIV, VPValue *Step, const Twine &Name="")
Definition VPlan.h:3596
Template specialization of the standard LLVM dominator tree utility for VPBlockBases.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPEVLBasedIVPHIRecipe * clone() override
Clone the current recipe.
Definition VPlan.h:3515
~VPEVLBasedIVPHIRecipe() override=default
void execute(VPTransformState &State) override
Generate the phi nodes.
Definition VPlan.h:3521
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this VPEVLBasedIVPHIRecipe.
Definition VPlan.h:3527
VPEVLBasedIVPHIRecipe(VPValue *StartIV, DebugLoc DL)
Definition VPlan.h:3510
bool onlyFirstLaneUsed(const VPValue *Op) const override
Returns true if the recipe only uses the first lane of operand Op.
Definition VPlan.h:3534
void execute(VPTransformState &State) override
The method which generates the output IR instructions that correspond to this VPRecipe,...
Definition VPlan.h:3396
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this VPExpandSCEVRecipe.
Definition VPlan.h:3401
VPExpandSCEVRecipe(const SCEV *Expr)
Definition VPlan.h:3387
const SCEV * getSCEV() const
Definition VPlan.h:3413
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPExpandSCEVRecipe * clone() override
Clone the current recipe.
Definition VPlan.h:3392
~VPExpandSCEVRecipe() override=default
void execute(VPTransformState &State) override
Method for generating code, must not be called as this recipe is abstract.
Definition VPlan.h:3064
VPValue * getOperandOfResultType() const
Return the VPValue to use to infer the result type of the recipe.
Definition VPlan.h:3051
VPExpressionRecipe * clone() override
Clone the current recipe.
Definition VPlan.h:3033
void decompose()
Insert the recipes of the expression back into the VPlan, directly before the current recipe.
~VPExpressionRecipe() override
Definition VPlan.h:3024
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPExpressionRecipe(VPWidenCastRecipe *Ext, VPReductionRecipe *Red)
Definition VPlan.h:3015
bool mayHaveSideEffects() const
Returns true if this expression contains recipes that may have side effects.
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Compute the cost of this recipe either using a recipe's specialized implementation or using the legac...
bool mayReadOrWriteMemory() const
Returns true if this expression contains recipes that may read from or write to memory.
VPExpressionRecipe(VPWidenCastRecipe *Ext0, VPWidenCastRecipe *Ext1, VPWidenRecipe *Mul, VPReductionRecipe *Red)
Definition VPlan.h:3019
VPExpressionRecipe(VPWidenRecipe *Mul, VPReductionRecipe *Red)
Definition VPlan.h:3017
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this header phi recipe.
const VPRecipeBase * getAsRecipe() const override
Return a VPRecipeBase* to the current object.
Definition VPlan.h:1984
static bool classof(const VPValue *V)
Definition VPlan.h:1994
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override=0
Print the recipe.
virtual VPValue * getBackedgeValue()
Returns the incoming value from the loop backedge.
Definition VPlan.h:2025
void setBackedgeValue(VPValue *V)
Update the incoming value from the loop backedge.
Definition VPlan.h:2030
VPValue * getStartValue()
Returns the start value of the phi, if one is set.
Definition VPlan.h:2014
void setStartValue(VPValue *V)
Update the start value of the recipe.
Definition VPlan.h:2022
VPValue * getStartValue() const
Definition VPlan.h:2017
static bool classof(const VPRecipeBase *B)
Method to support type inquiry through isa, cast, and dyn_cast.
Definition VPlan.h:1990
void execute(VPTransformState &State) override=0
Generate the phi nodes.
virtual VPRecipeBase & getBackedgeRecipe()
Returns the backedge value as a recipe.
Definition VPlan.h:2034
VPHeaderPHIRecipe(unsigned char VPDefID, Instruction *UnderlyingInstr, VPValue *Start, DebugLoc DL=DebugLoc::getUnknown())
Definition VPlan.h:1979
~VPHeaderPHIRecipe() override=default
void execute(VPTransformState &State) override
Produce a vectorized histogram operation.
VP_CLASSOF_IMPL(VPDef::VPHistogramSC)
VPHistogramRecipe * clone() override
Clone the current recipe.
Definition VPlan.h:1693
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this VPHistogramRecipe.
VPValue * getMask() const
Return the mask operand if one was provided, or a null pointer if all lanes should be executed uncond...
Definition VPlan.h:1710
unsigned getOpcode() const
Definition VPlan.h:1706
VPHistogramRecipe(unsigned Opcode, ArrayRef< VPValue * > Operands, DebugLoc DL=DebugLoc::getUnknown())
Definition VPlan.h:1687
~VPHistogramRecipe() override=default
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
A special type of VPBasicBlock that wraps an existing IR basic block.
Definition VPlan.h:3917
void execute(VPTransformState *State) override
The method which generates the output IR instructions that correspond to this VPBasicBlock,...
Definition VPlan.cpp:487
BasicBlock * getIRBasicBlock() const
Definition VPlan.h:3941
~VPIRBasicBlock() override
Definition VPlan.h:3929
static bool classof(const VPBlockBase *V)
Definition VPlan.h:3931
friend class VPlan
Definition VPlan.h:3918
VPIRBasicBlock * clone() override
Clone the current block and it's recipes, without updating the operands of the cloned recipes.
Definition VPlan.cpp:512
Class to record and manage LLVM IR flags.
Definition VPlan.h:600
FastMathFlagsTy FMFs
Definition VPlan.h:664
bool flagsValidForOpcode(unsigned Opcode) const
Returns true if the set flags are valid for Opcode.
VPIRFlags(DisjointFlagsTy DisjointFlags)
Definition VPlan.h:713
VPIRFlags(WrapFlagsTy WrapFlags)
Definition VPlan.h:705
WrapFlagsTy WrapFlags
Definition VPlan.h:658
CmpInst::Predicate CmpPredicate
Definition VPlan.h:657
void printFlags(raw_ostream &O) const
GEPNoWrapFlags GEPFlags
Definition VPlan.h:662
bool hasFastMathFlags() const
Returns true if the recipe has fast-math flags.
Definition VPlan.h:822
LLVM_ABI_FOR_TEST FastMathFlags getFastMathFlags() const
TruncFlagsTy TruncFlags
Definition VPlan.h:659
CmpInst::Predicate getPredicate() const
Definition VPlan.h:804
bool hasNonNegFlag() const
Returns true if the recipe has non-negative flag.
Definition VPlan.h:827
void transferFlags(VPIRFlags &Other)
Definition VPlan.h:722
ExactFlagsTy ExactFlags
Definition VPlan.h:661
bool hasNoSignedWrap() const
Definition VPlan.h:846
void intersectFlags(const VPIRFlags &Other)
Only keep flags also present in Other.
bool isDisjoint() const
Definition VPlan.h:857
VPIRFlags(TruncFlagsTy TruncFlags)
Definition VPlan.h:708
VPIRFlags(FastMathFlags FMFs)
Definition VPlan.h:711
VPIRFlags(NonNegFlagsTy NonNegFlags)
Definition VPlan.h:716
VPIRFlags(CmpInst::Predicate Pred)
Definition VPlan.h:702
bool isNonNeg() const
Definition VPlan.h:829
GEPNoWrapFlags getGEPNoWrapFlags() const
Definition VPlan.h:816
bool hasPredicate() const
Returns true if the recipe has a comparison predicate.
Definition VPlan.h:819
DisjointFlagsTy DisjointFlags
Definition VPlan.h:660
unsigned AllFlags
Definition VPlan.h:665
void setPredicate(CmpInst::Predicate Pred)
Definition VPlan.h:810
bool hasNoUnsignedWrap() const
Definition VPlan.h:835
NonNegFlagsTy NonNegFlags
Definition VPlan.h:663
void dropPoisonGeneratingFlags()
Drop all poison-generating flags.
Definition VPlan.h:732
void applyFlags(Instruction &I) const
Apply the IR flags to I.
Definition VPlan.h:767
VPIRFlags(GEPNoWrapFlags GEPFlags)
Definition VPlan.h:719
VPIRFlags(Instruction &I)
Definition VPlan.h:671
Instruction & getInstruction() const
Definition VPlan.h:1375
bool onlyFirstPartUsed(const VPValue *Op) const override
Returns true if the VPUser only uses the first part of operand Op.
Definition VPlan.h:1389
~VPIRInstruction() override=default
void execute(VPTransformState &State) override
The method which generates the output IR instructions that correspond to this VPRecipe,...
void extractLastLaneOfFirstOperand(VPBuilder &Builder)
Update the recipes first operand to the last lane of the operand using Builder.
bool onlyFirstLaneUsed(const VPValue *Op) const override
Returns true if the VPUser only uses the first lane of operand Op.
Definition VPlan.h:1395
VPIRInstruction * clone() override
Clone the current recipe.
Definition VPlan.h:1362
static LLVM_ABI_FOR_TEST VPIRInstruction * create(Instruction &I)
Create a new VPIRPhi for \I , if it is a PHINode, otherwise create a VPIRInstruction.
LLVM_ABI_FOR_TEST InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this VPIRInstruction.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
bool usesScalars(const VPValue *Op) const override
Returns true if the VPUser uses scalars of operand Op.
Definition VPlan.h:1383
VPIRInstruction(Instruction &I)
VPIRInstruction::create() should be used to create VPIRInstructions, as subclasses may need to be cre...
Definition VPlan.h:1350
Helper to manage IR metadata for recipes.
Definition VPlan.h:942
VPIRMetadata(Instruction &I)
Adds metatadata that can be preserved from the original instruction I.
Definition VPlan.h:950
void intersect(const VPIRMetadata &MD)
Intersect this VPIRMetada object with MD, keeping only metadata nodes that are common to both.
VPIRMetadata & operator=(const VPIRMetadata &Other)
Definition VPlan.h:959
VPIRMetadata(const VPIRMetadata &Other)
Copy constructor for cloning.
Definition VPlan.h:957
void addMetadata(unsigned Kind, MDNode *Node)
Add metadata with kind Kind and Node.
Definition VPlan.h:968
void applyMetadata(Instruction &I) const
Add all metadata to I.
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this VPInstruction.
Definition VPlan.h:1231
static bool classof(const VPUser *R)
Definition VPlan.h:1215
static bool classof(const VPRecipeBase *R)
Definition VPlan.h:1197
Type * getResultType() const
Definition VPlan.h:1237
VPInstructionWithType(unsigned Opcode, ArrayRef< VPValue * > Operands, Type *ResultTy, const VPIRFlags &Flags, DebugLoc DL, const Twine &Name="")
Definition VPlan.h:1192
VPInstruction * clone() override
Clone the current recipe.
Definition VPlan.h:1219
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
Generate the instruction.
This is a concrete Recipe that models a single VPlan-level instruction.
Definition VPlan.h:983
VPInstruction(unsigned Opcode, ArrayRef< VPValue * > Operands, DebugLoc DL=DebugLoc::getUnknown(), const Twine &Name="")
Definition VPlan.h:1100
VPInstruction * clone() override
Clone the current recipe.
Definition VPlan.h:1111
@ ExtractLane
Extracts a single lane (first operand) from a set of vector operands.
Definition VPlan.h:1060
@ ComputeAnyOfResult
Compute the final result of a AnyOf reduction with select(cmp(),x,y), where one of (x,...
Definition VPlan.h:1016
@ WideIVStep
Scale the first operand (vector step) by the second operand (scalar-step).
Definition VPlan.h:1050
@ ResumeForEpilogue
Explicit user for the resume phi of the canonical induction in the main VPlan, used by the epilogue v...
Definition VPlan.h:1063
@ FirstOrderRecurrenceSplice
Definition VPlan.h:989
@ ReductionStartVector
Start vector for reductions with 3 operands: the original start value, the identity value for the red...
Definition VPlan.h:1054
@ BuildVector
Creates a fixed-width vector containing all operands.
Definition VPlan.h:1013
@ BuildStructVector
Given operands of (the same) struct type, creates a struct of fixed- width vectors each containing a ...
Definition VPlan.h:1010
@ VScale
Returns the value for vscale.
Definition VPlan.h:1065
@ CanonicalIVIncrementForPart
Definition VPlan.h:1003
bool hasResult() const
Definition VPlan.h:1139
StringRef getName() const
Returns the symbolic name assigned to the VPInstruction.
Definition VPlan.h:1179
unsigned getOpcode() const
Definition VPlan.h:1119
friend class VPlanSlp
Definition VPlan.h:984
virtual unsigned getNumStoreOperands() const =0
Returns the number of stored operands of this interleave group.
bool needsMaskForGaps() const
Return true if the access needs a mask because of the gaps.
Definition VPlan.h:2542
void execute(VPTransformState &State) override
The method which generates the output IR instructions that correspond to this VPRecipe,...
Definition VPlan.h:2548
static bool classof(const VPUser *U)
Definition VPlan.h:2524
virtual bool onlyFirstLaneUsed(const VPValue *Op) const override=0
Returns true if the recipe only uses the first lane of operand Op.
VPInterleaveBase(const unsigned char SC, const InterleaveGroup< Instruction > *IG, ArrayRef< VPValue * > Operands, ArrayRef< VPValue * > StoredValues, VPValue *Mask, bool NeedsMaskForGaps, const VPIRMetadata &MD, DebugLoc DL)
Definition VPlan.h:2491
Instruction * getInsertPos() const
Definition VPlan.h:2546
static bool classof(const VPRecipeBase *R)
Definition VPlan.h:2519
const InterleaveGroup< Instruction > * getInterleaveGroup() const
Definition VPlan.h:2544
VPValue * getMask() const
Return the mask used by this recipe.
Definition VPlan.h:2536
ArrayRef< VPValue * > getStoredValues() const
Return the VPValues stored by this interleave group.
Definition VPlan.h:2565
VPInterleaveBase * clone() override=0
Clone the current recipe.
VPValue * getAddr() const
Return the address accessed by this recipe.
Definition VPlan.h:2530
VPValue * getEVL() const
The VPValue of the explicit vector length.
Definition VPlan.h:2639
~VPInterleaveEVLRecipe() override=default
bool onlyFirstLaneUsed(const VPValue *Op) const override
The recipe only uses the first lane of the address, and EVL operand.
Definition VPlan.h:2651
unsigned getNumStoreOperands() const override
Returns the number of stored operands of this interleave group.
Definition VPlan.h:2658
VPInterleaveEVLRecipe * clone() override
Clone the current recipe.
Definition VPlan.h:2632
VPInterleaveEVLRecipe(VPInterleaveRecipe &R, VPValue &EVL, VPValue *Mask)
Definition VPlan.h:2619
VPInterleaveRecipe is a recipe for transforming an interleave group of load or stores into one wide l...
Definition VPlan.h:2576
unsigned getNumStoreOperands() const override
Returns the number of stored operands of this interleave group.
Definition VPlan.h:2609
bool onlyFirstLaneUsed(const VPValue *Op) const override
Returns true if the recipe only uses the first lane of operand Op.
Definition VPlan.h:2603
~VPInterleaveRecipe() override=default
VPInterleaveRecipe * clone() override
Clone the current recipe.
Definition VPlan.h:2586
VPInterleaveRecipe(const InterleaveGroup< Instruction > *IG, VPValue *Addr, ArrayRef< VPValue * > StoredValues, VPValue *Mask, bool NeedsMaskForGaps, const VPIRMetadata &MD, DebugLoc DL)
Definition VPlan.h:2578
In what follows, the term "input IR" refers to code that is fed into the vectorizer whereas the term ...
VPPartialReductionRecipe(Instruction *ReductionInst, VPValue *Op0, VPValue *Op1, VPValue *Cond, unsigned VFScaleFactor)
Definition VPlan.h:2761
VPPartialReductionRecipe(unsigned Opcode, VPValue *Op0, VPValue *Op1, VPValue *Cond, unsigned ScaleFactor, Instruction *ReductionInst=nullptr)
Definition VPlan.h:2765
~VPPartialReductionRecipe() override=default
unsigned getVFScaleFactor() const
Get the factor that the VF of this recipe's output should be scaled by.
Definition VPlan.h:2799
void execute(VPTransformState &State) override
Generate the reduction in the loop.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this VPPartialReductionRecipe.
unsigned getOpcode() const
Get the binary op's opcode.
Definition VPlan.h:2796
VPPartialReductionRecipe * clone() override
Clone the current recipe.
Definition VPlan.h:2780
Helper type to provide functions to access incoming values and blocks for phi-like recipes.
Definition VPlan.h:1248
virtual const VPRecipeBase * getAsRecipe() const =0
Return a VPRecipeBase* to the current object.
VPUser::const_operand_range incoming_values() const
Returns an interator range over the incoming values.
Definition VPlan.h:1270
virtual unsigned getNumIncoming() const
Returns the number of incoming values, also number of incoming blocks.
Definition VPlan.h:1265
void removeIncomingValueFor(VPBlockBase *IncomingBlock) const
Removes the incoming value for IncomingBlock, which must be a predecessor.
const VPBasicBlock * getIncomingBlock(unsigned Idx) const
Returns the incoming block with index Idx.
Definition VPlan.h:3908
detail::zippy< llvm::detail::zip_first, VPUser::const_operand_range, const_incoming_blocks_range > incoming_values_and_blocks() const
Returns an iterator range over pairs of incoming values and corresponding incoming blocks.
Definition VPlan.h:1290
VPValue * getIncomingValue(unsigned Idx) const
Returns the incoming VPValue with index Idx.
Definition VPlan.h:1257
virtual ~VPPhiAccessors()=default
void printPhiOperands(raw_ostream &O, VPSlotTracker &SlotTracker) const
Print the recipe.
iterator_range< mapped_iterator< detail::index_iterator, std::function< const VPBasicBlock *(size_t)> > > const_incoming_blocks_range
Definition VPlan.h:1275
const_incoming_blocks_range incoming_blocks() const
Returns an iterator range over the incoming blocks.
Definition VPlan.h:1279
~VPPredInstPHIRecipe() override=default
bool usesScalars(const VPValue *Op) const override
Returns true if the recipe uses scalars of operand Op.
Definition VPlan.h:3123
VPPredInstPHIRecipe * clone() override
Clone the current recipe.
Definition VPlan.h:3099
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this VPPredInstPHIRecipe.
Definition VPlan.h:3110
VPPredInstPHIRecipe(VPValue *PredV, DebugLoc DL)
Construct a VPPredInstPHIRecipe given PredInst whose value needs a phi nodes after merging back from ...
Definition VPlan.h:3095
VPRecipeBase is a base class modeling a sequence of one or more output IR instructions.
Definition VPlan.h:394
bool mayReadFromMemory() const
Returns true if the recipe may read from memory.
bool mayReadOrWriteMemory() const
Returns true if the recipe may read from or write to memory.
Definition VPlan.h:477
void setDebugLoc(DebugLoc NewDL)
Set the recipe's debug location to NewDL.
Definition VPlan.h:488
bool mayWriteToMemory() const
Returns true if the recipe may write to memory.
virtual ~VPRecipeBase()=default
VPBasicBlock * getParent()
Definition VPlan.h:415
DebugLoc getDebugLoc() const
Returns the debug location of the recipe.
Definition VPlan.h:482
virtual void execute(VPTransformState &State)=0
The method which generates the output IR instructions that correspond to this VPRecipe,...
void moveBefore(VPBasicBlock &BB, iplist< VPRecipeBase >::iterator I)
Unlink this recipe and insert into BB before I.
void insertBefore(VPRecipeBase *InsertPos)
Insert an unlinked recipe into a basic block immediately before the specified recipe.
void insertAfter(VPRecipeBase *InsertPos)
Insert an unlinked Recipe into a basic block immediately after the specified Recipe.
static bool classof(const VPDef *D)
Method to support type inquiry through isa, cast, and dyn_cast.
Definition VPlan.h:457
iplist< VPRecipeBase >::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
virtual VPRecipeBase * clone()=0
Clone the current recipe.
friend class VPBlockUtils
Definition VPlan.h:396
const VPBasicBlock * getParent() const
Definition VPlan.h:416
InstructionCost cost(ElementCount VF, VPCostContext &Ctx)
Return the cost of this recipe, taking into account if the cost computation should be skipped and the...
static bool classof(const VPUser *U)
Definition VPlan.h:462
void removeFromParent()
This method unlinks 'this' from the containing basic block, but does not delete it.
void moveAfter(VPRecipeBase *MovePos)
Unlink this recipe from its current VPBasicBlock and insert it into the VPBasicBlock that MovePos liv...
VPRecipeBase(const unsigned char SC, ArrayRef< VPValue * > Operands, DebugLoc DL=DebugLoc::getUnknown())
Definition VPlan.h:405
bool onlyFirstLaneUsed(const VPValue *Op) const override
Returns true if the recipe only uses the first lane of operand Op.
Definition VPlan.h:2844
VPValue * getEVL() const
The VPValue of the explicit vector length.
Definition VPlan.h:2841
VPReductionEVLRecipe(VPReductionRecipe &R, VPValue &EVL, VPValue *CondOp, DebugLoc DL=DebugLoc::getUnknown())
Definition VPlan.h:2814
VPReductionEVLRecipe * clone() override
Clone the current recipe.
Definition VPlan.h:2825
~VPReductionEVLRecipe() override=default
bool isOrdered() const
Returns true, if the phi is part of an ordered reduction.
Definition VPlan.h:2388
VPReductionPHIRecipe * clone() override
Clone the current recipe.
Definition VPlan.h:2357
unsigned getVFScaleFactor() const
Get the factor that the VF of this recipe's output should be scaled by.
Definition VPlan.h:2371
~VPReductionPHIRecipe() override=default
bool onlyFirstLaneUsed(const VPValue *Op) const override
Returns true if the recipe only uses the first lane of operand Op.
Definition VPlan.h:2394
VPReductionPHIRecipe(PHINode *Phi, RecurKind Kind, VPValue &Start, bool IsInLoop=false, bool IsOrdered=false, unsigned VFScaleFactor=1)
Create a new VPReductionPHIRecipe for the reduction Phi.
Definition VPlan.h:2347
unsigned getNumIncoming() const override
Returns the number of incoming values, also number of incoming blocks.
Definition VPlan.h:2382
bool isInLoop() const
Returns true, if the phi is part of an in-loop reduction.
Definition VPlan.h:2391
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
Generate the phi/select nodes.
RecurKind getRecurrenceKind() const
Returns the recurrence kind of the reduction.
Definition VPlan.h:2385
A recipe to represent inloop reduction operations, performing a reduction on a vector operand into a ...
Definition VPlan.h:2666
bool isConditional() const
Return true if the in-loop reduction is conditional.
Definition VPlan.h:2738
static bool classof(const VPRecipeBase *R)
Definition VPlan.h:2710
VPReductionRecipe(const RecurKind RdxKind, FastMathFlags FMFs, VPValue *ChainOp, VPValue *VecOp, VPValue *CondOp, bool IsOrdered, DebugLoc DL=DebugLoc::getUnknown())
Definition VPlan.h:2695
VPValue * getVecOp() const
The VPValue of the vector value to be reduced.
Definition VPlan.h:2742
VPValue * getCondOp() const
The VPValue of the condition for the block.
Definition VPlan.h:2744
RecurKind getRecurrenceKind() const
Return the recurrence kind for the in-loop reduction.
Definition VPlan.h:2734
bool isOrdered() const
Return true if the in-loop reduction is ordered.
Definition VPlan.h:2736
~VPReductionRecipe() override=default
VPValue * getChainOp() const
The VPValue of the scalar Chain being accumulated.
Definition VPlan.h:2740
VPReductionRecipe(RecurKind RdxKind, FastMathFlags FMFs, Instruction *I, VPValue *ChainOp, VPValue *VecOp, VPValue *CondOp, bool IsOrdered, DebugLoc DL=DebugLoc::getUnknown())
Definition VPlan.h:2688
VPReductionRecipe * clone() override
Clone the current recipe.
Definition VPlan.h:2704
VPReductionRecipe(const unsigned char SC, RecurKind RdxKind, FastMathFlags FMFs, Instruction *I, ArrayRef< VPValue * > Operands, VPValue *CondOp, bool IsOrdered, DebugLoc DL)
Definition VPlan.h:2674
static bool classof(const VPUser *U)
Definition VPlan.h:2715
VPRegionBlock represents a collection of VPBasicBlocks and VPRegionBlocks which form a Single-Entry-S...
Definition VPlan.h:3952
const VPBlockBase * getEntry() const
Definition VPlan.h:3988
bool isReplicator() const
An indicator whether this region is to generate multiple replicated instances of output IR correspond...
Definition VPlan.h:4020
void setExiting(VPBlockBase *ExitingBlock)
Set ExitingBlock as the exiting VPBlockBase of this VPRegionBlock.
Definition VPlan.h:4005
VPBlockBase * getExiting()
Definition VPlan.h:4001
void setEntry(VPBlockBase *EntryBlock)
Set EntryBlock as the entry VPBlockBase of this VPRegionBlock.
Definition VPlan.h:3993
const VPBlockBase * getExiting() const
Definition VPlan.h:4000
VPBlockBase * getEntry()
Definition VPlan.h:3989
VPBasicBlock * getPreheaderVPBB()
Returns the pre-header VPBasicBlock of the loop region.
Definition VPlan.h:4013
~VPRegionBlock() override
Definition VPlan.h:3981
friend class VPlan
Definition VPlan.h:3953
static bool classof(const VPBlockBase *V)
Method to support type inquiry through isa, cast, and dyn_cast.
Definition VPlan.h:3984
VPReplicateRecipe replicates a given instruction producing multiple scalar copies of the original sca...
Definition VPlan.h:2856
VPReplicateRecipe(Instruction *I, ArrayRef< VPValue * > Operands, bool IsSingleScalar, VPValue *Mask=nullptr, VPIRMetadata Metadata={})
Definition VPlan.h:2864
bool isSingleScalar() const
Definition VPlan.h:2901
~VPReplicateRecipe() override=default
bool onlyFirstLaneUsed(const VPValue *Op) const override
Returns true if the recipe only uses the first lane of operand Op.
Definition VPlan.h:2906
bool usesScalars(const VPValue *Op) const override
Returns true if the recipe uses scalars of operand Op.
Definition VPlan.h:2913
bool isPredicated() const
Definition VPlan.h:2903
VPReplicateRecipe * clone() override
Clone the current recipe.
Definition VPlan.h:2876
unsigned getOpcode() const
Definition VPlan.h:2930
VPValue * getMask()
Return the mask of a predicated VPReplicateRecipe.
Definition VPlan.h:2925
bool onlyFirstLaneUsed(const VPValue *Op) const override
Returns true if the recipe only uses the first lane of operand Op.
Definition VPlan.h:3709
VPValue * getStepValue() const
Definition VPlan.h:3706
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this VPScalarIVStepsRecipe.
Definition VPlan.h:3694
VPScalarIVStepsRecipe(const InductionDescriptor &IndDesc, VPValue *IV, VPValue *Step, VPValue *VF, DebugLoc DL=DebugLoc::getUnknown())
Definition VPlan.h:3665
bool isPart0() const
Return true if this VPScalarIVStepsRecipe corresponds to part 0.
Definition VPlan.h:3686
VPScalarIVStepsRecipe * clone() override
Clone the current recipe.
Definition VPlan.h:3677
VPScalarIVStepsRecipe(VPValue *IV, VPValue *Step, VPValue *VF, Instruction::BinaryOps Opcode, FastMathFlags FMFs, DebugLoc DL)
Definition VPlan.h:3658
~VPScalarIVStepsRecipe() override=default
VPSingleDef is a base class for recipes for modeling a sequence of one or more output IR that define ...
Definition VPlan.h:521
VPSingleDefRecipe(const unsigned char SC, ArrayRef< VPValue * > Operands, Value *UV, DebugLoc DL=DebugLoc::getUnknown())
Definition VPlan.h:527
Instruction * getUnderlyingInstr()
Returns the underlying instruction.
Definition VPlan.h:586
static bool classof(const VPRecipeBase *R)
Definition VPlan.h:531
const Instruction * getUnderlyingInstr() const
Definition VPlan.h:589
static bool classof(const VPUser *U)
Definition VPlan.h:578
LLVM_DUMP_METHOD void dump() const
Print this VPSingleDefRecipe to dbgs() (for debugging).
VPSingleDefRecipe(const unsigned char SC, ArrayRef< VPValue * > Operands, DebugLoc DL=DebugLoc::getUnknown())
Definition VPlan.h:523
virtual VPSingleDefRecipe * clone() override=0
Clone the current recipe.
This class can be used to assign names to VPValues.
Helper to access the operand that contains the unroll part for this recipe after unrolling.
Definition VPlan.h:930
VPValue * getUnrollPartOperand(const VPUser &U) const
Return the VPValue operand containing the unroll part or null if there is no such operand.
unsigned getUnrollPart(const VPUser &U) const
Return the unroll part.
This class augments VPValue with operands which provide the inverse def-use edges from VPValue's user...
Definition VPlanValue.h:199
void printOperands(raw_ostream &O, VPSlotTracker &SlotTracker) const
Print the operands to O.
Definition VPlan.cpp:1455
operand_range operands()
Definition VPlanValue.h:267
void setOperand(unsigned I, VPValue *New)
Definition VPlanValue.h:243
unsigned getNumOperands() const
Definition VPlanValue.h:237
operand_iterator op_end()
Definition VPlanValue.h:265
VPValue * getOperand(unsigned N) const
Definition VPlanValue.h:238
VPUser(ArrayRef< VPValue * > Operands)
Definition VPlanValue.h:218
iterator_range< const_operand_iterator > const_operand_range
Definition VPlanValue.h:261
iterator_range< operand_iterator > operand_range
Definition VPlanValue.h:260
VPRecipeBase * getDefiningRecipe()
Returns the recipe defining this VPValue or nullptr if it is not defined by a recipe,...
Definition VPlan.cpp:135
friend class VPExpressionRecipe
Definition VPlanValue.h:53
Value * getLiveInIRValue() const
Returns the underlying IR value, if this VPValue is defined outside the scope of VPlan.
Definition VPlanValue.h:176
friend class VPDef
Definition VPlanValue.h:49
Value * getUnderlyingValue() const
Return the underlying Value attached to this VPValue.
Definition VPlanValue.h:85
VPValue(const unsigned char SC, Value *UV=nullptr, VPDef *Def=nullptr)
Definition VPlan.cpp:98
void setUnderlyingValue(Value *Val)
Definition VPlanValue.h:186
unsigned getNumUsers() const
Definition VPlanValue.h:113
bool isLiveIn() const
Returns true if this VPValue is a live-in, i.e. defined outside the VPlan.
Definition VPlanValue.h:171
void execute(VPTransformState &State) override
The method which generates the output IR instructions that correspond to this VPRecipe,...
bool onlyFirstPartUsed(const VPValue *Op) const override
Returns true if the recipe only uses the first part of operand Op.
Definition VPlan.h:1878
bool onlyFirstLaneUsed(const VPValue *Op) const override
Returns true if the VPUser only uses the first lane of operand Op.
Definition VPlan.h:1864
VPVectorEndPointerRecipe * clone() override
Clone the current recipe.
Definition VPlan.h:1885
const VPValue * getVFValue() const
Definition VPlan.h:1860
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this VPVectorPointerRecipe.
Definition VPlan.h:1871
VPVectorEndPointerRecipe(VPValue *Ptr, VPValue *VF, Type *IndexedTy, int64_t Stride, GEPNoWrapFlags GEPFlags, DebugLoc DL)
Definition VPlan.h:1849
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
bool isFirstPart() const
Return true if this VPVectorPointerRecipe corresponds to part 0.
Definition VPlan.h:1937
Type * getSourceElementType() const
Definition VPlan.h:1914
void execute(VPTransformState &State) override
The method which generates the output IR instructions that correspond to this VPRecipe,...
bool onlyFirstPartUsed(const VPValue *Op) const override
Returns true if the recipe only uses the first part of operand Op.
Definition VPlan.h:1923
VPVectorPointerRecipe(VPValue *Ptr, Type *SourceElementTy, GEPNoWrapFlags GEPFlags, DebugLoc DL)
Definition VPlan.h:1904
bool onlyFirstLaneUsed(const VPValue *Op) const override
Returns true if the VPUser only uses the first lane of operand Op.
Definition VPlan.h:1916
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this VPHeaderPHIRecipe.
Definition VPlan.h:1940
VPVectorPointerRecipe * clone() override
Clone the current recipe.
Definition VPlan.h:1930
const_operand_range args() const
Definition VPlan.h:1668
VPWidenCallRecipe * clone() override
Clone the current recipe.
Definition VPlan.h:1649
VPWidenCallRecipe(Value *UV, Function *Variant, ArrayRef< VPValue * > CallArguments, DebugLoc DL=DebugLoc::getUnknown())
Definition VPlan.h:1636
operand_range args()
Definition VPlan.h:1667
Function * getCalledScalarFunction() const
Definition VPlan.h:1663
~VPWidenCallRecipe() override=default
void execute(VPTransformState &State) override
Generate a canonical vector induction variable of the vector loop, with start = {<Part*VF,...
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
~VPWidenCanonicalIVRecipe() override=default
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this VPWidenCanonicalIVPHIRecipe.
Definition VPlan.h:3569
VPWidenCanonicalIVRecipe * clone() override
Clone the current recipe.
Definition VPlan.h:3556
VPWidenCanonicalIVRecipe(VPCanonicalIVPHIRecipe *CanonicalIV)
Definition VPlan.h:3551
VPWidenCastRecipe is a recipe to create vector cast instructions.
Definition VPlan.h:1482
VPWidenCastRecipe(Instruction::CastOps Opcode, VPValue *Op, Type *ResultTy, CastInst &UI)
Definition VPlan.h:1490
Instruction::CastOps getOpcode() const
Definition VPlan.h:1533
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPWidenCastRecipe(Instruction::CastOps Opcode, VPValue *Op, Type *ResultTy, const VPIRFlags &Flags={}, const VPIRMetadata &Metadata={}, DebugLoc DL=DebugLoc::getUnknown())
Definition VPlan.h:1498
Type * getResultType() const
Returns the result type of the cast.
Definition VPlan.h:1536
void execute(VPTransformState &State) override
Produce widened copies of the cast.
~VPWidenCastRecipe() override=default
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this VPWidenCastRecipe.
VPWidenCastRecipe * clone() override
Clone the current recipe.
Definition VPlan.h:1510
unsigned getOpcode() const
This recipe generates a GEP instruction.
Definition VPlan.h:1806
bool onlyFirstLaneUsed(const VPValue *Op) const override
Returns true if the recipe only uses the first lane of operand Op.
Definition VPlan.h:1827
Type * getSourceElementType() const
Definition VPlan.h:1811
VPWidenGEPRecipe(GetElementPtrInst *GEP, ArrayRef< VPValue * > Operands)
Definition VPlan.h:1787
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this VPWidenGEPRecipe.
Definition VPlan.h:1814
VPWidenGEPRecipe * clone() override
Clone the current recipe.
Definition VPlan.h:1798
~VPWidenGEPRecipe() override=default
bool onlyFirstLaneUsed(const VPValue *Op) const override
Returns true if the recipe only uses the first lane of operand Op.
Definition VPlan.h:2104
static bool classof(const VPValue *V)
Definition VPlan.h:2058
void setStepValue(VPValue *V)
Update the step value of the recipe.
Definition VPlan.h:2074
VPValue * getBackedgeValue() override
Returns the incoming value from the loop backedge.
Definition VPlan.h:2089
unsigned getNumIncoming() const override
Returns the number of incoming values, also number of incoming blocks.
Definition VPlan.h:2082
PHINode * getPHINode() const
Definition VPlan.h:2084
VPWidenInductionRecipe(unsigned char Kind, PHINode *IV, VPValue *Start, VPValue *Step, const InductionDescriptor &IndDesc, DebugLoc DL)
Definition VPlan.h:2046
VPValue * getStepValue()
Returns the step value of the induction.
Definition VPlan.h:2070
const InductionDescriptor & getInductionDescriptor() const
Returns the induction descriptor for the recipe.
Definition VPlan.h:2087
VPRecipeBase & getBackedgeRecipe() override
Returns the backedge value as a recipe.
Definition VPlan.h:2096
static bool classof(const VPRecipeBase *R)
Definition VPlan.h:2053
static bool classof(const VPHeaderPHIRecipe *R)
Definition VPlan.h:2063
const VPValue * getVFValue() const
Definition VPlan.h:2077
const VPValue * getStepValue() const
Definition VPlan.h:2071
virtual void execute(VPTransformState &State) override=0
Generate the phi nodes.
const TruncInst * getTruncInst() const
Definition VPlan.h:2182
void execute(VPTransformState &State) override
Generate the phi nodes.
Definition VPlan.h:2157
~VPWidenIntOrFpInductionRecipe() override=default
VPWidenIntOrFpInductionRecipe(PHINode *IV, VPValue *Start, VPValue *Step, VPValue *VF, const InductionDescriptor &IndDesc, TruncInst *Trunc, DebugLoc DL)
Definition VPlan.h:2133
VPWidenIntOrFpInductionRecipe * clone() override
Clone the current recipe.
Definition VPlan.h:2149
TruncInst * getTruncInst()
Returns the first defined value as TruncInst, if it is one or nullptr otherwise.
Definition VPlan.h:2181
VPWidenIntOrFpInductionRecipe(PHINode *IV, VPValue *Start, VPValue *Step, VPValue *VF, const InductionDescriptor &IndDesc, DebugLoc DL)
Definition VPlan.h:2124
VPValue * getLastUnrolledPartOperand()
Returns the VPValue representing the value of this induction at the last unrolled part,...
Definition VPlan.h:2198
unsigned getNumIncoming() const override
Returns the number of incoming values, also number of incoming blocks.
Definition VPlan.h:2177
Type * getScalarType() const
Returns the scalar type of the induction.
Definition VPlan.h:2190
bool isCanonical() const
Returns true if the induction is canonical, i.e.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
bool onlyFirstLaneUsed(const VPValue *Op) const override
Returns true if the VPUser only uses the first lane of operand Op.
VPWidenIntrinsicRecipe(Intrinsic::ID VectorIntrinsicID, ArrayRef< VPValue * > CallArguments, Type *Ty, DebugLoc DL=DebugLoc::getUnknown())
Definition VPlan.h:1566
Intrinsic::ID getVectorIntrinsicID() const
Return the ID of the intrinsic.
Definition VPlan.h:1601
bool mayReadFromMemory() const
Returns true if the intrinsic may read from memory.
Definition VPlan.h:1610
StringRef getIntrinsicName() const
Return to name of the intrinsic as string.
VPWidenIntrinsicRecipe(CallInst &CI, Intrinsic::ID VectorIntrinsicID, ArrayRef< VPValue * > CallArguments, Type *Ty, DebugLoc DL=DebugLoc::getUnknown())
Definition VPlan.h:1557
bool mayHaveSideEffects() const
Returns true if the intrinsic may have side-effects.
Definition VPlan.h:1616
VPWidenIntrinsicRecipe * clone() override
Clone the current recipe.
Definition VPlan.h:1583
bool mayWriteToMemory() const
Returns true if the intrinsic may write to memory.
Definition VPlan.h:1613
~VPWidenIntrinsicRecipe() override=default
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Type * getResultType() const
Return the scalar return type of the intrinsic.
Definition VPlan.h:1604
void execute(VPTransformState &State) override
Produce a widened version of the vector intrinsic.
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this vector intrinsic.
bool IsMasked
Whether the memory access is masked.
Definition VPlan.h:3144
bool Reverse
Whether the consecutive accessed addresses are in reverse order.
Definition VPlan.h:3141
bool isConsecutive() const
Return whether the loaded-from / stored-to addresses are consecutive.
Definition VPlan.h:3181
static bool classof(const VPUser *U)
Definition VPlan.h:3175
void execute(VPTransformState &State) override
Generate the wide load/store.
Definition VPlan.h:3201
Instruction & Ingredient
Definition VPlan.h:3135
VPWidenMemoryRecipe * clone() override
Clone the current recipe.
Definition VPlan.h:3164
Instruction & getIngredient() const
Definition VPlan.h:3209
bool Consecutive
Whether the accessed addresses are consecutive.
Definition VPlan.h:3138
static bool classof(const VPRecipeBase *R)
Definition VPlan.h:3168
VPValue * getMask() const
Return the mask used by this recipe.
Definition VPlan.h:3195
bool isMasked() const
Returns true if the recipe is masked.
Definition VPlan.h:3191
VPWidenMemoryRecipe(const char unsigned SC, Instruction &I, std::initializer_list< VPValue * > Operands, bool Consecutive, bool Reverse, const VPIRMetadata &Metadata, DebugLoc DL)
Definition VPlan.h:3154
void setMask(VPValue *Mask)
Definition VPlan.h:3146
VPValue * getAddr() const
Return the address accessed by this recipe.
Definition VPlan.h:3188
bool isReverse() const
Return whether the consecutive loaded/stored addresses are in reverse order.
Definition VPlan.h:3185
const VPRecipeBase * getAsRecipe() const override
Return a VPRecipeBase* to the current object.
Definition VPlan.h:2258
VPWidenPHIRecipe(PHINode *Phi, VPValue *Start=nullptr, DebugLoc DL=DebugLoc::getUnknown(), const Twine &Name="")
Create a new VPWidenPHIRecipe for Phi with start value Start and debug location DL.
Definition VPlan.h:2263
VPWidenPHIRecipe * clone() override
Clone the current recipe.
Definition VPlan.h:2271
~VPWidenPHIRecipe() override=default
void execute(VPTransformState &State) override
Generate the phi/select nodes.
VPWidenPointerInductionRecipe * clone() override
Clone the current recipe.
Definition VPlan.h:2222
~VPWidenPointerInductionRecipe() override=default
bool onlyScalarsGenerated(bool IsScalable)
Returns true if only scalar values will be generated.
void execute(VPTransformState &State) override
Generate vector values for the pointer induction.
Definition VPlan.h:2232
VPWidenPointerInductionRecipe(PHINode *Phi, VPValue *Start, VPValue *Step, VPValue *NumUnrolledElems, const InductionDescriptor &IndDesc, bool IsScalarAfterVectorization, DebugLoc DL)
Create a new VPWidenPointerInductionRecipe for Phi with start value Start and the number of elements ...
Definition VPlan.h:2210
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPWidenRecipe is a recipe for producing a widened instruction using the opcode and operands of the re...
Definition VPlan.h:1439
VPWidenRecipe * clone() override
Clone the current recipe.
Definition VPlan.h:1455
VPWidenRecipe(Instruction &I, ArrayRef< VPValue * > Operands)
Definition VPlan.h:1449
VPWidenRecipe(unsigned Opcode, ArrayRef< VPValue * > Operands, const VPIRFlags &Flags, const VPIRMetadata &Metadata, DebugLoc DL)
Definition VPlan.h:1443
~VPWidenRecipe() override=default
unsigned getOpcode() const
Definition VPlan.h:1472
Class that maps (parts of) an existing VPlan to trees of combined VPInstructions.
Definition VPlanSLP.h:74
VPlan models a candidate for vectorization, encoding various decisions take to produce efficient outp...
Definition VPlan.h:4055
void printDOT(raw_ostream &O) const
Print this VPlan in DOT format to O.
Definition VPlan.cpp:1141
friend class VPSlotTracker
Definition VPlan.h:4057
std::string getName() const
Return a string with the name of the plan and the applicable VFs and UFs.
Definition VPlan.cpp:1117
bool hasVF(ElementCount VF) const
Definition VPlan.h:4264
LLVMContext & getContext() const
Definition VPlan.h:4252
VPBasicBlock * getEntry()
Definition VPlan.h:4154
VPRegionBlock * createVPRegionBlock(VPBlockBase *Entry, VPBlockBase *Exiting, const std::string &Name="", bool IsReplicator=false)
Create a new VPRegionBlock with Entry, Exiting and Name.
Definition VPlan.h:4395
VPValue & getVectorTripCount()
The vector trip count.
Definition VPlan.h:4244
void setName(const Twine &newName)
Definition VPlan.h:4302
bool hasScalableVF() const
Definition VPlan.h:4265
VPValue & getVFxUF()
Returns VF * UF of the vector loop region.
Definition VPlan.h:4250
VPValue & getVF()
Returns the VF of the vector loop region.
Definition VPlan.h:4247
VPValue * getTripCount() const
The trip count of the original loop.
Definition VPlan.h:4216
VPValue * getTrue()
Return a VPValue wrapping i1 true.
Definition VPlan.h:4321
VPValue * getOrCreateBackedgeTakenCount()
The backedge taken count of the original loop.
Definition VPlan.h:4237
iterator_range< SmallSetVector< ElementCount, 2 >::iterator > vectorFactors() const
Returns an iterator range over all VFs of the plan.
Definition VPlan.h:4271
VPlan(BasicBlock *ScalarHeaderBB, VPValue *TC)
Construct a VPlan with a new VPBasicBlock as entry, a VPIRBasicBlock wrapping ScalarHeaderBB and a tr...
Definition VPlan.h:4135
VPIRBasicBlock * getExitBlock(BasicBlock *IRBB) const
Return the VPIRBasicBlock corresponding to IRBB.
Definition VPlan.cpp:937
LLVM_ABI_FOR_TEST ~VPlan()
Definition VPlan.cpp:914
bool isExitBlock(VPBlockBase *VPBB)
Returns true if VPBB is an exit block.
Definition VPlan.cpp:945
const VPBasicBlock * getEntry() const
Definition VPlan.h:4155
friend class VPlanPrinter
Definition VPlan.h:4056
unsigned getUF() const
Definition VPlan.h:4284
VPRegionBlock * createVPRegionBlock(const std::string &Name="")
Create a new loop VPRegionBlock with Name and entry and exiting blocks set to nullptr.
Definition VPlan.h:4406
VPIRBasicBlock * createEmptyVPIRBasicBlock(BasicBlock *IRBB)
Create a VPIRBasicBlock wrapping IRBB, but do not create VPIRInstructions wrapping the instructions i...
Definition VPlan.cpp:1255
void addSCEVExpansion(const SCEV *S, VPValue *V)
Definition VPlan.h:4373
bool hasUF(unsigned UF) const
Definition VPlan.h:4282
ArrayRef< VPIRBasicBlock * > getExitBlocks() const
Return an ArrayRef containing VPIRBasicBlocks wrapping the exit blocks of the original scalar loop.
Definition VPlan.h:4206
void setVF(ElementCount VF)
Definition VPlan.h:4258
bool isUnrolled() const
Returns true if the VPlan already has been unrolled, i.e.
Definition VPlan.h:4297
LLVM_ABI_FOR_TEST VPRegionBlock * getVectorLoopRegion()
Returns the VPRegionBlock of the vector loop.
Definition VPlan.cpp:1046
bool hasEarlyExit() const
Returns true if the VPlan is based on a loop with an early exit.
Definition VPlan.h:4427
InstructionCost cost(ElementCount VF, VPCostContext &Ctx)
Return the cost of this plan.
Definition VPlan.cpp:1028
const VPBasicBlock * getMiddleBlock() const
Definition VPlan.h:4192
void setTripCount(VPValue *NewTripCount)
Set the trip count assuming it is currently null; if it is not - use resetTripCount().
Definition VPlan.h:4223
void resetTripCount(VPValue *NewTripCount)
Resets the trip count for the VPlan.
Definition VPlan.h:4230
VPBasicBlock * getMiddleBlock()
Returns the 'middle' block of the plan, that is the block that selects whether to execute the scalar ...
Definition VPlan.h:4179
void setEntry(VPBasicBlock *VPBB)
Definition VPlan.h:4143
VPBasicBlock * createVPBasicBlock(const Twine &Name, VPRecipeBase *Recipe=nullptr)
Create a new VPBasicBlock with Name and containing Recipe if present.
Definition VPlan.h:4385
LLVM_ABI_FOR_TEST VPIRBasicBlock * createVPIRBasicBlock(BasicBlock *IRBB)
Create a VPIRBasicBlock from IRBB containing VPIRInstructions for all instructions in IRBB,...
Definition VPlan.cpp:1261
VPValue * getFalse()
Return a VPValue wrapping i1 false.
Definition VPlan.h:4327
VPValue * getOrAddLiveIn(Value *V)
Gets the live-in VPValue for V or adds a new live-in (if none exists yet) for V.
Definition VPlan.h:4306
LLVM_DUMP_METHOD void dump() const
Dump the plan to stderr (for debugging).
Definition VPlan.cpp:1147
bool hasScalarVFOnly() const
Definition VPlan.h:4275
VPBasicBlock * getScalarPreheader() const
Return the VPBasicBlock for the preheader of the scalar loop.
Definition VPlan.h:4197
void execute(VPTransformState *State)
Generate the IR code for this VPlan.
Definition VPlan.cpp:952
ArrayRef< VPValue * > getLiveIns() const
Return the list of live-in VPValues available in the VPlan.
Definition VPlan.h:4336
VPCanonicalIVPHIRecipe * getCanonicalIV()
Returns the canonical induction recipe of the vector loop.
Definition VPlan.h:4360
void print(raw_ostream &O) const
Print this VPlan to O.
Definition VPlan.cpp:1100
void addVF(ElementCount VF)
Definition VPlan.h:4256
VPIRBasicBlock * getScalarHeader() const
Return the VPIRBasicBlock wrapping the header of the scalar loop.
Definition VPlan.h:4202
VPValue * getLiveIn(Value *V) const
Return the live-in VPValue for V, if there is one or nullptr otherwise.
Definition VPlan.h:4333
VPValue * getSCEVExpansion(const SCEV *S) const
Definition VPlan.h:4369
void printLiveIns(raw_ostream &O) const
Print the live-ins of this VPlan to O.
Definition VPlan.cpp:1062
VPBasicBlock * getVectorPreheader()
Returns the preheader of the vector loop region, if one exists, or null otherwise.
Definition VPlan.h:4159
void setUF(unsigned UF)
Definition VPlan.h:4289
bool hasScalarTail() const
Returns true if the scalar tail may execute after the vector loop.
Definition VPlan.h:4437
VPlan * duplicate()
Clone the current VPlan, update all VPValues of the new VPlan and cloned recipes to refer to the clon...
Definition VPlan.cpp:1188
LLVM Value Representation.
Definition Value.h:75
Type * getType() const
All values are typed, get the type of this value.
Definition Value.h:256
Increasing range of size_t indices.
Definition STLExtras.h:2405
base_list_type::const_reverse_iterator const_reverse_iterator
Definition ilist.h:125
base_list_type::reverse_iterator reverse_iterator
Definition ilist.h:123
base_list_type::const_iterator const_iterator
Definition ilist.h:122
An intrusive list with ownership and callbacks specified/controlled by ilist_traits,...
Definition ilist.h:328
A range adaptor for a pair of iterators.
This class implements an extremely fast bulk output stream that can only output to a stream.
Definition raw_ostream.h:53
This file defines classes to implement an intrusive doubly linked list class (i.e.
This file defines the ilist_node class template, which is a convenient base class for creating classe...
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ C
The default llvm calling convention, compatible with C.
Definition CallingConv.h:34
This namespace contains an enum with a value for every intrinsic/builtin function known by LLVM.
LLVM_ABI AttributeSet getFnAttributes(LLVMContext &C, ID id)
Return the function attributes for an intrinsic.
This is an optimization pass for GlobalISel generic memory operations.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
Definition STLExtras.h:318
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
detail::zippy< detail::zip_shortest, T, U, Args... > zip(T &&t, U &&u, Args &&...args)
zip iterator for two or more iteratable types.
Definition STLExtras.h:831
auto find(R &&Range, const T &Val)
Provide wrappers to std::find which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1731
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1705
detail::zippy< detail::zip_first, T, U, Args... > zip_equal(T &&t, U &&u, Args &&...args)
zip iterator that assumes that all iteratees have the same length.
Definition STLExtras.h:841
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are tuples (A, B,...
Definition STLExtras.h:2452
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:649
LLVM_ABI void getMetadataToPropagate(Instruction *Inst, SmallVectorImpl< std::pair< unsigned, MDNode * > > &Metadata)
Add metadata from Inst to Metadata, if it can be preserved after vectorization.
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
auto cast_or_null(const Y &Val)
Definition Casting.h:720
MemoryEffectsBase< IRMemLocation > MemoryEffects
Summary of how a function affects memory in the program.
Definition ModRef.h:296
auto map_range(ContainerTy &&C, FuncTy F)
Definition STLExtras.h:366
auto dyn_cast_or_null(const Y &Val)
Definition Casting.h:759
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1712
auto reverse(ContainerTy &&C)
Definition STLExtras.h:408
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition Debug.cpp:207
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
Definition Casting.h:548
auto drop_end(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the last N elements excluded.
Definition STLExtras.h:325
@ Other
Any other memory.
Definition ModRef.h:68
RecurKind
These are the kinds of recurrences that we support.
@ Mul
Product of integers.
@ Add
Sum of integers.
auto count(R &&Range, const E &Element)
Wrapper function around std::count to count the number of times an element Element occurs in the give...
Definition STLExtras.h:1934
DWARFExpression::Operation Op
raw_ostream & operator<<(raw_ostream &OS, const APFixedPoint &FX)
auto count_if(R &&Range, UnaryPredicate P)
Wrapper function around std::count_if to count the number of times an element satisfying a given pred...
Definition STLExtras.h:1941
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:565
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
Definition STLExtras.h:1877
DenseMap< Value *, VPValue * > Value2VPValueTy
Definition VPlanValue.h:192
std::unique_ptr< VPlan > VPlanPtr
Definition VPlan.h:77
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition BitVector.h:853
#define N
Support casting from VPRecipeBase -> VPPhiAccessors, by down-casting to the recipe types implementing...
Definition VPlan.h:3727
static VPPhiAccessors * doCastIfPossible(SrcTy f)
doCastIfPossible is used by dyn_cast<>.
Definition VPlan.h:3748
CastInfo< VPPhiAccessors, SrcTy > Self
Definition VPlan.h:3729
static VPPhiAccessors * doCast(SrcTy R)
doCast is used by cast<>.
Definition VPlan.h:3732
This struct provides a method for customizing the way a cast is performed.
Definition Casting.h:476
static bool isPossible(const VPRecipeBase *f)
Definition VPlan.h:3719
This struct provides a way to check if a given cast is possible.
Definition Casting.h:253
static bool isPossible(const SrcTy &f)
Definition Casting.h:254
Struct to hold various analysis needed for cost computations.
void execute(VPTransformState &State) override
Generate the phi nodes.
VPFirstOrderRecurrencePHIRecipe * clone() override
Clone the current recipe.
Definition VPlan.h:2302
VPFirstOrderRecurrencePHIRecipe(PHINode *Phi, VPValue &Start)
Definition VPlan.h:2297
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this first-order recurrence phi recipe.
bool onlyFirstLaneUsed(const VPValue *Op) const override
Returns true if the recipe only uses the first lane of operand Op.
Definition VPlan.h:2320
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
DisjointFlagsTy(bool IsDisjoint)
Definition VPlan.h:630
NonNegFlagsTy(bool IsNonNeg)
Definition VPlan.h:635
TruncFlagsTy(bool HasNUW, bool HasNSW)
Definition VPlan.h:625
WrapFlagsTy(bool HasNUW, bool HasNSW)
Definition VPlan.h:618
PHINode & getIRPhi()
Definition VPlan.h:1420
VPIRPhi(PHINode &PN)
Definition VPlan.h:1413
static bool classof(const VPRecipeBase *U)
Definition VPlan.h:1415
const VPRecipeBase * getAsRecipe() const override
Return a VPRecipeBase* to the current object.
Definition VPlan.h:1431
static bool classof(const VPUser *U)
Definition VPlan.h:1308
VPPhi * clone() override
Clone the current recipe.
Definition VPlan.h:1323
const VPRecipeBase * getAsRecipe() const override
Return a VPRecipeBase* to the current object.
Definition VPlan.h:1338
VPPhi(ArrayRef< VPValue * > Operands, DebugLoc DL, const Twine &Name="")
Definition VPlan.h:1305
static bool classof(const VPSingleDefRecipe *SDR)
Definition VPlan.h:1318
static bool classof(const VPValue *V)
Definition VPlan.h:1313
A pure-virtual common base class for recipes defining a single VPValue and using IR flags.
Definition VPlan.h:875
static bool classof(const VPRecipeBase *R)
Definition VPlan.h:889
InstructionCost getCostForRecipeWithOpcode(unsigned Opcode, ElementCount VF, VPCostContext &Ctx) const
Compute the cost for this recipe for VF, using Opcode and Ctx.
VPRecipeWithIRFlags(const unsigned char SC, ArrayRef< VPValue * > Operands, Instruction &I)
Definition VPlan.h:880
VPRecipeWithIRFlags(const unsigned char SC, ArrayRef< VPValue * > Operands, const VPIRFlags &Flags, DebugLoc DL=DebugLoc::getUnknown())
Definition VPlan.h:884
virtual VPRecipeWithIRFlags * clone() override=0
Clone the current recipe.
static bool classof(const VPValue *V)
Definition VPlan.h:909
static bool classof(const VPSingleDefRecipe *U)
Definition VPlan.h:916
void execute(VPTransformState &State) override=0
The method which generates the output IR instructions that correspond to this VPRecipe,...
static bool classof(const VPUser *U)
Definition VPlan.h:904
VPRecipeWithIRFlags(const unsigned char SC, ArrayRef< VPValue * > Operands, DebugLoc DL=DebugLoc::getUnknown())
Definition VPlan.h:876
VPTransformState holds information passed down when "executing" a VPlan, needed for generating the ou...
void execute(VPTransformState &State) override
Generate the wide load or gather.
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this VPWidenLoadEVLRecipe.
VPValue * getEVL() const
Return the EVL operand.
Definition VPlan.h:3268
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPWidenLoadEVLRecipe(VPWidenLoadRecipe &L, VPValue *Addr, VPValue &EVL, VPValue *Mask)
Definition VPlan.h:3256
bool onlyFirstLaneUsed(const VPValue *Op) const override
Returns true if the recipe only uses the first lane of operand Op.
Definition VPlan.h:3284
A recipe for widening load operations, using the address to load from and an optional mask.
Definition VPlan.h:3215
VP_CLASSOF_IMPL(VPDef::VPWidenLoadSC)
bool onlyFirstLaneUsed(const VPValue *Op) const override
Returns true if the recipe only uses the first lane of operand Op.
Definition VPlan.h:3243
void execute(VPTransformState &State) override
Generate a wide load or gather.
VPWidenLoadRecipe(LoadInst &Load, VPValue *Addr, VPValue *Mask, bool Consecutive, bool Reverse, const VPIRMetadata &Metadata, DebugLoc DL)
Definition VPlan.h:3216
VPWidenLoadRecipe * clone() override
Clone the current recipe.
Definition VPlan.h:3225
bool isInvariantCond() const
Definition VPlan.h:1756
VPWidenSelectRecipe * clone() override
Clone the current recipe.
Definition VPlan.h:1730
VPWidenSelectRecipe(SelectInst &I, ArrayRef< VPValue * > Operands)
Definition VPlan.h:1724
VPValue * getCond() const
Definition VPlan.h:1752
unsigned getOpcode() const
Definition VPlan.h:1750
bool onlyFirstLaneUsed(const VPValue *Op) const override
Returns true if the recipe only uses the first lane of operand Op.
Definition VPlan.h:1761
~VPWidenSelectRecipe() override=default
VPValue * getStoredValue() const
Return the address accessed by this recipe.
Definition VPlan.h:3349
void execute(VPTransformState &State) override
Generate the wide store or scatter.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
bool onlyFirstLaneUsed(const VPValue *Op) const override
Returns true if the recipe only uses the first lane of operand Op.
Definition VPlan.h:3368
VPWidenStoreEVLRecipe(VPWidenStoreRecipe &S, VPValue *Addr, VPValue &EVL, VPValue *Mask)
Definition VPlan.h:3338
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this VPWidenStoreEVLRecipe.
VPValue * getEVL() const
Return the EVL operand.
Definition VPlan.h:3352
A recipe for widening store operations, using the stored value, the address to store to and an option...
Definition VPlan.h:3295
bool onlyFirstLaneUsed(const VPValue *Op) const override
Returns true if the recipe only uses the first lane of operand Op.
Definition VPlan.h:3325
VP_CLASSOF_IMPL(VPDef::VPWidenStoreSC)
VPValue * getStoredValue() const
Return the value stored by this recipe.
Definition VPlan.h:3313
VPWidenStoreRecipe * clone() override
Clone the current recipe.
Definition VPlan.h:3304
VPWidenStoreRecipe(StoreInst &Store, VPValue *Addr, VPValue *StoredVal, VPValue *Mask, bool Consecutive, bool Reverse, const VPIRMetadata &Metadata, DebugLoc DL)
Definition VPlan.h:3296