23#ifndef LLVM_TRANSFORMS_VECTORIZE_VPLAN_H
24#define LLVM_TRANSFORMS_VECTORIZE_VPLAN_H
50class InnerLoopVectorizer;
53class PredicateScalarEvolution;
55class RecurrenceDescriptor;
61class VPReplicateRecipe;
80 Loop *CurLoop =
nullptr);
99 "Both Start and End should have the same scalable flag");
101 "Expected Start to be a power of 2");
103 "Expected End to be a power of 2");
163 VPLane(
unsigned Lane,
Kind LaneKind) : Lane(Lane), LaneKind(LaneKind) {}
176 return VPLane(LaneOffset, LaneKind);
286 unsigned CacheIdx =
Instance.Lane.mapToCacheIndex(
VF);
287 return Instance.Part <
I->second.size() &&
288 CacheIdx <
I->second[
Instance.Part].size() &&
304 "need to overwrite existing value");
305 Iter->second[Part] = V;
311 auto &PerPartVec = Iter.first->second;
312 while (PerPartVec.size() <=
Instance.Part)
313 PerPartVec.emplace_back();
314 auto &Scalars = PerPartVec[
Instance.Part];
315 unsigned CacheIdx =
Instance.Lane.mapToCacheIndex(
VF);
316 while (Scalars.size() <= CacheIdx)
317 Scalars.push_back(
nullptr);
318 assert(!Scalars[CacheIdx] &&
"should overwrite existing value");
319 Scalars[CacheIdx] = V;
326 "need to overwrite existing value");
328 "need to overwrite existing value");
329 unsigned CacheIdx =
Instance.Lane.mapToCacheIndex(
VF);
331 "need to overwrite existing value");
332 Iter->second[
Instance.Part][CacheIdx] = V;
421 const unsigned char SubclassID;
438 VPlan *Plan =
nullptr;
448 assert(Predecessor &&
"Cannot add nullptr predecessor!");
453 void removePredecessor(VPBlockBase *Predecessor) {
454 auto Pos =
find(Predecessors, Predecessor);
455 assert(Pos &&
"Predecessor does not exist");
456 Predecessors.
erase(Pos);
460 void removeSuccessor(VPBlockBase *
Successor) {
462 assert(Pos &&
"Successor does not exist");
463 Successors.
erase(Pos);
468 : SubclassID(SC),
Name(
N) {}
475 using VPBlockTy =
enum { VPBasicBlockSC, VPRegionBlockSC };
526 return (Successors.
size() == 1 ? *Successors.
begin() :
nullptr);
532 return (Predecessors.
size() == 1 ? *Predecessors.
begin() :
nullptr);
585 assert(Successors.
empty() &&
"Setting one successor when others exist.");
594 assert(Successors.
empty() &&
"Setting two successors when others exist.");
595 appendSuccessor(IfTrue);
596 appendSuccessor(IfFalse);
603 assert(Predecessors.
empty() &&
"Block predecessors already set.");
604 for (
auto *Pred : NewPreds)
605 appendPredecessor(Pred);
632#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
684 "Op must be an operand of the recipe");
690#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
714 template <
typename IterT>
796#define VP_CLASSOF_IMPL(VPDefID) \
797 static inline bool classof(const VPDef *D) { \
798 return D->getVPDefID() == VPDefID; \
800 static inline bool classof(const VPValue *V) { \
801 auto *R = V->getDefiningRecipe(); \
802 return R && R->getVPDefID() == VPDefID; \
804 static inline bool classof(const VPUser *U) { \
805 auto *R = dyn_cast<VPRecipeBase>(U); \
806 return R && R->getVPDefID() == VPDefID; \
808 static inline bool classof(const VPRecipeBase *R) { \
809 return R->getVPDefID() == VPDefID; \
823 Instruction::OtherOpsEnd + 1,
842 typedef unsigned char OpcodeTy;
848 const std::string
Name;
881#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
894 return Opcode == Instruction::Store || Opcode == Instruction::Call ||
895 Opcode == Instruction::Invoke || Opcode ==
SLPStore;
902 case Instruction::Ret:
903 case Instruction::Br:
904 case Instruction::Store:
905 case Instruction::Switch:
906 case Instruction::IndirectBr:
907 case Instruction::Resume:
908 case Instruction::CatchRet:
909 case Instruction::Unreachable:
910 case Instruction::Fence:
911 case Instruction::AtomicRMW:
926 "Op must be an operand of the recipe");
947 enum class OperationType :
unsigned char {
958 struct ExactFlagsTy {
964 struct FastMathFlagsTy {
965 char AllowReassoc : 1;
968 char NoSignedZeros : 1;
969 char AllowReciprocal : 1;
970 char AllowContract : 1;
974 OperationType OpType;
985 template <
typename IterT>
988 OpType = OperationType::Other;
992 template <
typename IterT>
996 if (
auto *Op = dyn_cast<OverflowingBinaryOperator>(&
I)) {
997 OpType = OperationType::OverflowingBinOp;
998 WrapFlags.HasNUW = Op->hasNoUnsignedWrap();
999 WrapFlags.HasNSW = Op->hasNoSignedWrap();
1000 }
else if (
auto *Op = dyn_cast<PossiblyExactOperator>(&
I)) {
1001 OpType = OperationType::PossiblyExactOp;
1003 }
else if (
auto *
GEP = dyn_cast<GetElementPtrInst>(&
I)) {
1004 OpType = OperationType::GEPOp;
1006 }
else if (
auto *Op = dyn_cast<FPMathOperator>(&
I)) {
1007 OpType = OperationType::FPMathOp;
1020 return R->getVPDefID() == VPRecipeBase::VPWidenSC ||
1021 R->getVPDefID() == VPRecipeBase::VPWidenGEPSC ||
1022 R->getVPDefID() == VPRecipeBase::VPReplicateSC;
1030 case OperationType::OverflowingBinOp:
1034 case OperationType::PossiblyExactOp:
1037 case OperationType::GEPOp:
1040 case OperationType::FPMathOp:
1041 FMFs.NoNaNs =
false;
1042 FMFs.NoInfs =
false;
1044 case OperationType::Other:
1052 case OperationType::OverflowingBinOp:
1056 case OperationType::PossiblyExactOp:
1059 case OperationType::GEPOp:
1060 cast<GetElementPtrInst>(
I)->setIsInBounds(
GEPFlags.IsInBounds);
1062 case OperationType::FPMathOp:
1063 I->setHasAllowReassoc(
FMFs.AllowReassoc);
1064 I->setHasNoNaNs(
FMFs.NoNaNs);
1065 I->setHasNoInfs(
FMFs.NoInfs);
1066 I->setHasNoSignedZeros(
FMFs.NoSignedZeros);
1067 I->setHasAllowReciprocal(
FMFs.AllowReciprocal);
1068 I->setHasAllowContract(
FMFs.AllowContract);
1069 I->setHasApproxFunc(
FMFs.ApproxFunc);
1071 case OperationType::Other:
1077 assert(OpType == OperationType::GEPOp &&
1078 "recipe doesn't have inbounds flag");
1082#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1105 template <
typename IterT>
1116#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1135 Opcode(Opcode), ResultTy(ResultTy) {
1136 assert((!UI || UI->getOpcode() == Opcode) &&
1137 "opcode of underlying cast doesn't match");
1138 assert((!UI || UI->getType() == ResultTy) &&
1139 "result type of underlying cast doesn't match");
1149#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1173 template <
typename IterT>
1178 VectorIntrinsicID(VectorIntrinsicID), Variant(Variant) {}
1187#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1196 template <
typename IterT>
1207#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1224 bool isPointerLoopInvariant()
const {
1228 bool isIndexLoopInvariant(
unsigned I)
const {
1232 bool areAllOperandsInvariant()
const {
1234 return Op->isDefinedOutsideVectorRegions();
1239 template <
typename IterT>
1251#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1295 return B->getVPDefID() >= VPDef::VPFirstHeaderPHISC &&
1296 B->getVPDefID() <= VPDef::VPLastHeaderPHISC;
1299 auto *
B = V->getDefiningRecipe();
1300 return B &&
B->getVPDefID() >= VPRecipeBase::VPFirstHeaderPHISC &&
1301 B->getVPDefID() <= VPRecipeBase::VPLastHeaderPHISC;
1307#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1347 Trunc(nullptr), IndDesc(IndDesc) {
1355 IV(IV), Trunc(Trunc), IndDesc(IndDesc) {
1367#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1377 "VPWidenIntOrFpInductionRecipe generates its own backedge value");
1384 "VPWidenIntOrFpInductionRecipe generates its own backedge value");
1414 bool IsScalarAfterVectorization;
1421 bool IsScalarAfterVectorization)
1424 IsScalarAfterVectorization(IsScalarAfterVectorization) {
1442#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1471#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1480 IncomingBlocks.
push_back(IncomingBlock);
1500 return R->getVPDefID() == VPDef::VPFirstOrderRecurrencePHISC;
1505#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1529 VPValue &Start,
bool IsInLoop =
false,
1530 bool IsOrdered =
false)
1532 RdxDesc(RdxDesc), IsInLoop(IsInLoop), IsOrdered(IsOrdered) {
1533 assert((!IsOrdered || IsInLoop) &&
"IsOrdered requires IsInLoop");
1541 return R->getVPDefID() == VPDef::VPReductionPHISC;
1547#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1577 "Expected either a single incoming value or a positive even number "
1596#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1605 "Op must be an operand of the recipe");
1609 [
this](
VPUser *U) {
return U->onlyFirstLaneUsed(
this); });
1622 bool HasMask =
false;
1626 bool NeedsMaskForGaps =
false;
1631 bool NeedsMaskForGaps)
1633 NeedsMaskForGaps(NeedsMaskForGaps) {
1634 for (
unsigned i = 0; i < IG->
getFactor(); ++i)
1636 if (
I->getType()->isVoidTy())
1641 for (
auto *SV : StoredValues)
1676#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1693 "Op must be an operand of the recipe");
1724#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1752 template <
typename IterT>
1754 bool IsUniform,
VPValue *Mask =
nullptr)
1756 VPValue(this,
I), IsUniform(IsUniform), IsPredicated(Mask) {
1770#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1783 "Op must be an operand of the recipe");
1790 "Op must be an operand of the recipe");
1819 void execute(VPTransformState &State)
override;
1821#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1825 O << Indent <<
"BRANCH-ON-MASK ";
1844 "Op must be an operand of the recipe");
1867#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1876 "Op must be an operand of the recipe");
1902 bool isMasked()
const {
1908 bool Consecutive,
bool Reverse)
1910 Ingredient(Load), Consecutive(Consecutive),
Reverse(Reverse) {
1911 assert((Consecutive || !Reverse) &&
"Reverse implies consecutive");
1918 bool Consecutive,
bool Reverse)
1920 Ingredient(Store), Consecutive(Consecutive),
Reverse(Reverse) {
1921 assert((Consecutive || !Reverse) &&
"Reverse implies consecutive");
1940 bool isStore()
const {
return isa<StoreInst>(Ingredient); }
1944 assert(
isStore() &&
"Stored value only available for store instructions");
1958#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1967 "Op must be an operand of the recipe");
1996#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2021 return D->getVPDefID() == VPDef::VPCanonicalIVPHISC;
2027#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2041 "Op must be an operand of the recipe");
2069 return D->getVPDefID() == VPDef::VPActiveLaneMaskPHISC;
2075#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2098#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2127 VPValue(
this), ResultTy(ResultTy), IndDesc(IndDesc) {}
2137#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2150 "Op must be an operand of the recipe");
2173#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2184 "Op must be an operand of the recipe");
2208 while (!Recipes.empty())
2231 inline size_t size()
const {
return Recipes.size(); }
2232 inline bool empty()
const {
return Recipes.empty(); }
2243 return &VPBasicBlock::Recipes;
2248 return V->getVPBlockID() == VPBlockBase::VPBasicBlockSC;
2252 assert(Recipe &&
"No recipe to append.");
2253 assert(!Recipe->Parent &&
"Recipe already in VPlan");
2254 Recipe->Parent =
this;
2255 Recipes.
insert(InsertPt, Recipe);
2283#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2330 const std::string &Name =
"",
bool IsReplicator =
false)
2331 :
VPBlockBase(VPRegionBlockSC, Name), Entry(Entry), Exiting(Exiting),
2332 IsReplicator(IsReplicator) {
2333 assert(Entry->getPredecessors().empty() &&
"Entry block has predecessors.");
2335 Entry->setParent(
this);
2339 :
VPBlockBase(VPRegionBlockSC, Name), Entry(nullptr), Exiting(nullptr),
2340 IsReplicator(IsReplicator) {}
2345 Entry->dropAllReferences(&DummyValue);
2352 return V->getVPBlockID() == VPBlockBase::VPRegionBlockSC;
2362 "Entry block cannot have predecessors.");
2374 "Exit block cannot have successors.");
2375 Exiting = ExitingBlock;
2395#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2442 VPValue *BackedgeTakenCount =
nullptr;
2457 bool Value2VPValueEnabled =
true;
2473 :
VPlan(Preheader, Entry) {
2482 : Entry(Entry), Preheader(Preheader) {
2483 Entry->setPlan(
this);
2487 "preheader must be disconnected");
2500 bool IsEpilogueVectorization);
2510 assert(TripCount &&
"trip count needs to be set before accessing it");
2516 if (!BackedgeTakenCount)
2517 BackedgeTakenCount =
new VPValue();
2518 return BackedgeTakenCount;
2531 assert(
hasVF(VF) &&
"Cannot set VF not already in plan");
2543 assert(
hasUF(UF) &&
"Cannot set the UF not already in plan");
2555 "Value2VPValue mapping may be out of date!");
2556 assert(V &&
"Trying to add a null Value to VPlan");
2557 assert(!Value2VPValue.
count(V) &&
"Value already exists in VPlan");
2558 Value2VPValue[V] = VPV;
2564 assert(V &&
"Trying to get the VPValue of a null Value");
2565 assert(Value2VPValue.
count(V) &&
"Value does not exist in VPlan");
2566 assert((Value2VPValueEnabled || OverrideAllowed ||
2567 Value2VPValue[V]->isLiveIn()) &&
2568 "Value2VPValue mapping may be out of date!");
2569 return Value2VPValue[V];
2575 assert(V &&
"Trying to get or add the VPValue of a null Value");
2576 if (!Value2VPValue.
count(V)) {
2586 assert(Value2VPValueEnabled &&
2587 "IR value to VPValue mapping may be out of date!");
2588 Value2VPValue.
erase(V);
2591#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2614 return cast<VPRegionBlock>(
getEntry()->getSingleSuccessor());
2617 return cast<VPRegionBlock>(
getEntry()->getSingleSuccessor());
2623 if (EntryVPBB->
empty()) {
2627 return cast<VPCanonicalIVPHIRecipe>(&*EntryVPBB->
begin());
2637 delete LiveOuts[PN];
2646 auto I = SCEVToExpansion.
find(S);
2647 if (
I == SCEVToExpansion.
end())
2654 SCEVToExpansion[S] = V;
2669#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2676 unsigned TabWidth = 2;
2684 void bumpIndent(
int b) { Indent = std::string((
Depth += b) * TabWidth,
' '); }
2710 const Twine &Label);
2755 "Can't insert new block with predecessors or successors.");
2774 "Can't insert IfTrue with successors.");
2776 "Can't insert IfFalse with successors.");
2790 "Can't connect two block with different parents");
2792 "Blocks can't have more than two successors.");
2793 From->appendSuccessor(To);
2794 To->appendPredecessor(
From);
2800 assert(To &&
"Successor to disconnect is null.");
2801 From->removeSuccessor(To);
2802 To->removePredecessor(
From);
2807 template <
typename BlockTy,
typename T>
2810 using BaseTy = std::conditional_t<std::is_const<BlockTy>::value,
2820 return cast<BlockTy>(&
Block);
2849 for (
auto &
I : InterleaveGroupMap)
2851 for (
auto *
Ptr : DelSet)
2860 return InterleaveGroupMap.
lookup(Instr);
2867 enum class OpMode {
Failed, Load, Opcode };
2871 struct BundleDenseMapInfo {
2873 return {
reinterpret_cast<VPValue *
>(-1)};
2877 return {
reinterpret_cast<VPValue *
>(-2)};
2901 bool CompletelySLP =
true;
2904 unsigned WidestBundleBits = 0;
2906 using MultiNodeOpTy =
2907 typename std::pair<VPInstruction *, SmallVector<VPValue *, 4>>;
2916 bool MultiNodeActive =
false;
2938#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2979 assert(Def &&
"Must have definition for value defined inside vector region");
2980 if (
auto Rep = dyn_cast<VPReplicateRecipe>(Def))
2981 return Rep->isUniform();
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
BlockVerifier::State From
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
#define LLVM_DUMP_METHOD
Mark debug helper function definitions like dump() that should not be stripped from debug builds.
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
This file defines the DenseMap class.
std::optional< std::vector< StOtherPiece > > Other
std::pair< BasicBlock *, unsigned > BlockTy
A pair of (basic block, score).
mir Rename Register Operands
This file implements a map that provides insertion order iteration.
static cl::opt< RegAllocEvictionAdvisorAnalysis::AdvisorMode > Mode("regalloc-enable-advisor", cl::Hidden, cl::init(RegAllocEvictionAdvisorAnalysis::AdvisorMode::Default), cl::desc("Enable regalloc advisor mode"), cl::values(clEnumValN(RegAllocEvictionAdvisorAnalysis::AdvisorMode::Default, "default", "Default"), clEnumValN(RegAllocEvictionAdvisorAnalysis::AdvisorMode::Release, "release", "precompiled"), clEnumValN(RegAllocEvictionAdvisorAnalysis::AdvisorMode::Development, "development", "for training")))
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file implements the SmallBitVector class.
This file defines the SmallPtrSet class.
This file defines the SmallVector class.
This file contains the declarations of the entities induced by Vectorization Plans,...
#define VP_CLASSOF_IMPL(VPDefID)
static const uint32_t IV[8]
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
ArrayRef< T > slice(size_t N, size_t M) const
slice(n, m) - Chop off the first N elements of the array, and keep M elements in the array.
LLVM Basic Block Representation.
This class represents a function call, abstracting a target machine's calling convention.
This is the base class for all instructions that perform data casts.
ValueT lookup(const_arg_type_t< KeyT > Val) const
lookup - Return the entry for the specified key, or a default constructed value if no such entry exis...
iterator find(const_arg_type_t< KeyT > Val)
bool erase(const KeyT &Val)
size_type count(const_arg_type_t< KeyT > Val) const
Return 1 if the specified key is in the map, 0 otherwise.
bool contains(const_arg_type_t< KeyT > Val) const
Return true if the specified key is in the map, false otherwise.
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
Convenience struct for specifying and reasoning about fast-math flags.
void setAllowContract(bool B=true)
bool noSignedZeros() const
void setAllowReciprocal(bool B=true)
bool allowReciprocal() const
void setNoSignedZeros(bool B=true)
bool allowReassoc() const
Flag queries.
void setNoNaNs(bool B=true)
void setAllowReassoc(bool B=true)
Flag setters.
void setApproxFunc(bool B=true)
void setNoInfs(bool B=true)
bool allowContract() const
an instruction for type-safe pointer arithmetic to access elements of arrays and structs
Common base class shared among various IRBuilders.
A struct for saving information about induction variables.
InductionKind
This enum represents the kinds of inductions that we support.
InnerLoopVectorizer vectorizes loops which contain only one basic block to a specified vectorization ...
The group of interleaved loads/stores sharing the same stride and close to each other.
uint32_t getFactor() const
InstTy * getMember(uint32_t Index) const
Get the member with the given index Index.
Drive the analysis of interleaved memory accesses in the loop.
An instruction for reading from memory.
This class emits a version of the loop where run-time checks ensure that may-alias pointers can't ove...
Represents a single loop in the control flow graph.
This class implements a map that also provides access to all stored values in a deterministic order.
VectorType::iterator erase(typename VectorType::iterator Iterator)
Remove the element given by Iterator.
The RecurrenceDescriptor is used to identify recurrences variables in a loop.
This class represents an analyzed expression in the program.
The main scalar evolution driver.
This class represents the LLVM 'select' instruction.
size_type size() const
Determine the number of elements in the SetVector.
size_type count(const key_type &key) const
Count the number of elements of a given key in the SetVector.
bool contains(const key_type &key) const
Check if the SetVector contains the given key.
bool insert(const value_type &X)
Insert a new element into the SetVector.
void clear()
Completely clear the SetVector.
bool empty() const
Determine if the SetVector is empty or not.
This class provides computation of slot numbers for LLVM Assembly writing.
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
A SetVector that performs no allocations if smaller than a certain size.
iterator erase(const_iterator CI)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
An instruction for storing to memory.
This class represents a truncation of integer types.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
std::string str() const
Return the twine contents as a std::string.
The instances of the Type class are immutable: once they are created, they are never changed.
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
A Use represents the edge between a Value definition and its users.
Iterator to iterate over vectorization factors in a VFRange.
ElementCount operator*() const
iterator(ElementCount VF)
bool operator==(const iterator &Other) const
A recipe for generating the active lane mask for the vector loop that is used to predicate the vector...
void execute(VPTransformState &State) override
Generate the active lane mask phi of the vector loop.
static bool classof(const VPHeaderPHIRecipe *D)
VPActiveLaneMaskPHIRecipe(VPValue *StartMask, DebugLoc DL)
~VPActiveLaneMaskPHIRecipe() override=default
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPBasicBlock serves as the leaf of the Hierarchical Control-Flow Graph.
RecipeListTy::const_iterator const_iterator
void appendRecipe(VPRecipeBase *Recipe)
Augment the existing recipes of a VPBasicBlock with an additional Recipe as the last recipe.
RecipeListTy::const_reverse_iterator const_reverse_iterator
RecipeListTy::iterator iterator
Instruction iterators...
void execute(VPTransformState *State) override
The method which generates the output IR instructions that correspond to this VPBasicBlock,...
RecipeListTy & getRecipeList()
Returns a reference to the list of recipes.
VPBasicBlock(const Twine &Name="", VPRecipeBase *Recipe=nullptr)
iterator begin()
Recipe iterator methods.
RecipeListTy::reverse_iterator reverse_iterator
iterator_range< iterator > phis()
Returns an iterator range over the PHI-like recipes in the block.
iterator getFirstNonPhi()
Return the position of the first non-phi node recipe in the block.
VPRegionBlock * getEnclosingLoopRegion()
void dropAllReferences(VPValue *NewValue) override
Replace all operands of VPUsers in the block with NewValue and also replaces all uses of VPValues def...
const_reverse_iterator rbegin() const
VPBasicBlock * splitAt(iterator SplitAt)
Split current block at SplitAt by inserting a new block between the current block and its successors ...
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print this VPBsicBlock to O, prefixing all lines with Indent.
const VPRecipeBase & front() const
const_iterator begin() const
bool isExiting() const
Returns true if the block is exiting it's parent region.
VPRecipeBase * getTerminator()
If the block has multiple successors, return the branch recipe terminating the block.
const VPRecipeBase & back() const
void insert(VPRecipeBase *Recipe, iterator InsertPt)
const_iterator end() const
static bool classof(const VPBlockBase *V)
Method to support type inquiry through isa, cast, and dyn_cast.
static RecipeListTy VPBasicBlock::* getSublistAccess(VPRecipeBase *)
Returns a pointer to a member of the recipe list.
reverse_iterator rbegin()
const_reverse_iterator rend() const
A recipe for vectorizing a phi-node as a sequence of mask-based select instructions.
VPBlendRecipe(PHINode *Phi, ArrayRef< VPValue * > Operands)
The blend operation is a User of the incoming values and of their respective masks,...
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
bool onlyFirstLaneUsed(const VPValue *Op) const override
Returns true if the recipe only uses the first lane of operand Op.
VPValue * getIncomingValue(unsigned Idx) const
Return incoming value number Idx.
VPValue * getMask(unsigned Idx) const
Return mask number Idx.
unsigned getNumIncomingValues() const
Return the number of incoming values, taking into account that a single incoming value has no mask.
void execute(VPTransformState &State) override
Generate the phi/select nodes.
VPBlockBase is the building block of the Hierarchical Control-Flow Graph.
VPRegionBlock * getParent()
VPBlocksTy & getPredecessors()
const VPBasicBlock * getExitingBasicBlock() const
LLVM_DUMP_METHOD void dump() const
Dump this VPBlockBase to dbgs().
void setName(const Twine &newName)
size_t getNumSuccessors() const
iterator_range< VPBlockBase ** > successors()
virtual void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const =0
Print plain-text dump of this VPBlockBase to O, prefixing all lines with Indent.
void printSuccessors(raw_ostream &O, const Twine &Indent) const
Print the successors of this block to O, prefixing all lines with Indent.
bool isLegalToHoistInto()
Return true if it is legal to hoist instructions into this block.
virtual ~VPBlockBase()=default
void print(raw_ostream &O) const
Print plain-text dump of this VPlan to O.
const VPBlocksTy & getHierarchicalPredecessors()
size_t getNumPredecessors() const
void setPredecessors(ArrayRef< VPBlockBase * > NewPreds)
Set each VPBasicBlock in NewPreds as predecessor of this VPBlockBase.
VPBlockBase * getEnclosingBlockWithPredecessors()
const VPBlocksTy & getPredecessors() const
static void deleteCFG(VPBlockBase *Entry)
Delete all blocks reachable from a given VPBlockBase, inclusive.
void setPlan(VPlan *ParentPlan)
Sets the pointer of the plan containing the block.
const VPRegionBlock * getParent() const
void printAsOperand(raw_ostream &OS, bool PrintType) const
const std::string & getName() const
void clearSuccessors()
Remove all the successors of this block.
VPBlockBase * getSingleHierarchicalSuccessor()
void setTwoSuccessors(VPBlockBase *IfTrue, VPBlockBase *IfFalse)
Set two given VPBlockBases IfTrue and IfFalse to be the two successors of this VPBlockBase.
VPBlockBase * getSinglePredecessor() const
virtual void execute(VPTransformState *State)=0
The method which generates the output IR that correspond to this VPBlockBase, thereby "executing" the...
const VPBlocksTy & getHierarchicalSuccessors()
void clearPredecessors()
Remove all the predecessor of this block.
enum { VPBasicBlockSC, VPRegionBlockSC } VPBlockTy
An enumeration for keeping track of the concrete subclass of VPBlockBase that are actually instantiat...
unsigned getVPBlockID() const
VPBlockBase(const unsigned char SC, const std::string &N)
VPBlocksTy & getSuccessors()
VPBlockBase * getEnclosingBlockWithSuccessors()
An Enclosing Block of a block B is any block containing B, including B itself.
const VPBasicBlock * getEntryBasicBlock() const
void setOneSuccessor(VPBlockBase *Successor)
Set a given VPBlockBase Successor as the single successor of this VPBlockBase.
void setParent(VPRegionBlock *P)
virtual void dropAllReferences(VPValue *NewValue)=0
Replace all operands of VPUsers in the block with NewValue and also replaces all uses of VPValues def...
VPBlockBase * getSingleHierarchicalPredecessor()
VPBlockBase * getSingleSuccessor() const
const VPBlocksTy & getSuccessors() const
Class that provides utilities for VPBlockBases in VPlan.
static auto blocksOnly(const T &Range)
Return an iterator range over Range which only includes BlockTy blocks.
static void insertBlockAfter(VPBlockBase *NewBlock, VPBlockBase *BlockPtr)
Insert disconnected VPBlockBase NewBlock after BlockPtr.
static void insertTwoBlocksAfter(VPBlockBase *IfTrue, VPBlockBase *IfFalse, VPBlockBase *BlockPtr)
Insert disconnected VPBlockBases IfTrue and IfFalse after BlockPtr.
static void disconnectBlocks(VPBlockBase *From, VPBlockBase *To)
Disconnect VPBlockBases From and To bi-directionally.
static void connectBlocks(VPBlockBase *From, VPBlockBase *To)
Connect VPBlockBases From and To bi-directionally.
A recipe for generating conditional branches on the bits of a mask.
VPValue * getMask() const
Return the mask used by this recipe.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPBranchOnMaskRecipe(VPValue *BlockInMask)
bool usesScalars(const VPValue *Op) const override
Returns true if the recipe uses scalars of operand Op.
void execute(VPTransformState &State) override
Generate the extraction of the appropriate bit from the block mask and the conditional branch.
Canonical scalar induction phi of the vector loop.
bool isCanonical(InductionDescriptor::InductionKind Kind, VPValue *Start, VPValue *Step, Type *Ty) const
Check if the induction described by Kind, /p Start and Step is canonical, i.e.
~VPCanonicalIVPHIRecipe() override=default
static bool classof(const VPHeaderPHIRecipe *D)
VPCanonicalIVPHIRecipe(VPValue *StartV, DebugLoc DL)
bool onlyFirstLaneUsed(const VPValue *Op) const override
Returns true if the recipe only uses the first lane of operand Op.
void execute(VPTransformState &State) override
Generate the canonical scalar induction phi of the vector loop.
const Type * getScalarType() const
Returns the scalar type of the induction.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
This class augments a recipe with a set of VPValues defined by the recipe.
VPValue * getVPSingleValue()
Returns the only VPValue defined by the VPDef.
unsigned getVPDefID() const
A recipe for converting the canonical IV value to the corresponding value of an IV with different sta...
void execute(VPTransformState &State) override
Generate the transformed value of the induction at offset StartValue (1.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPDerivedIVRecipe(const InductionDescriptor &IndDesc, VPValue *Start, VPCanonicalIVPHIRecipe *CanonicalIV, VPValue *Step, Type *ResultTy)
VPValue * getStepValue() const
VPValue * getCanonicalIV() const
~VPDerivedIVRecipe() override=default
bool onlyFirstLaneUsed(const VPValue *Op) const override
Returns true if the recipe only uses the first lane of operand Op.
VPValue * getStartValue() const
Recipe to expand a SCEV expression.
VPExpandSCEVRecipe(const SCEV *Expr, ScalarEvolution &SE)
const SCEV * getSCEV() const
void execute(VPTransformState &State) override
Generate a canonical vector induction variable of the vector loop, with.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
~VPExpandSCEVRecipe() override=default
This is a concrete Recipe that models a single VPlan-level instruction.
bool onlyFirstLaneUsed(const VPValue *Op) const override
Returns true if the recipe only uses the first lane of operand Op.
VPInstruction * clone() const
VPInstruction(unsigned Opcode, ArrayRef< VPValue * > Operands, DebugLoc DL, const Twine &Name="")
@ CanonicalIVIncrementForPartNUW
@ FirstOrderRecurrenceSplice
@ CanonicalIVIncrementNUW
@ CanonicalIVIncrementForPart
@ CalculateTripCountMinusVF
void setUnderlyingInstr(Instruction *I)
LLVM_DUMP_METHOD void dump() const
Print the VPInstruction to dbgs() (for debugging).
unsigned getOpcode() const
VPInstruction(unsigned Opcode, std::initializer_list< VPValue * > Operands, DebugLoc DL={}, const Twine &Name="")
void setFastMathFlags(FastMathFlags FMFNew)
Set the fast-math flags.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the VPInstruction to O.
bool mayWriteToMemory() const
Return true if this instruction may modify memory.
void execute(VPTransformState &State) override
Generate the instruction.
VPInterleaveRecipe is a recipe for transforming an interleave group of load or stores into one wide l...
bool onlyFirstLaneUsed(const VPValue *Op) const override
The recipe only uses the first lane of the address.
~VPInterleaveRecipe() override=default
VPValue * getAddr() const
Return the address accessed by this recipe.
VPInterleaveRecipe(const InterleaveGroup< Instruction > *IG, VPValue *Addr, ArrayRef< VPValue * > StoredValues, VPValue *Mask, bool NeedsMaskForGaps)
VPValue * getMask() const
Return the mask used by this recipe.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
Generate the wide load or store, and shuffles.
ArrayRef< VPValue * > getStoredValues() const
Return the VPValues stored by this interleave group.
const InterleaveGroup< Instruction > * getInterleaveGroup()
unsigned getNumStoreOperands() const
Returns the number of stored operands of this interleave group.
~VPInterleavedAccessInfo()
InterleaveGroup< VPInstruction > * getInterleaveGroup(VPInstruction *Instr) const
Get the interleave group that Instr belongs to.
In what follows, the term "input IR" refers to code that is fed into the vectorizer whereas the term ...
static VPLane getLastLaneForVF(const ElementCount &VF)
static unsigned getNumCachedLanes(const ElementCount &VF)
Returns the maxmimum number of lanes that we are able to consider caching for VF.
Value * getAsRuntimeExpr(IRBuilderBase &Builder, const ElementCount &VF) const
Returns an expression describing the lane index that can be used at runtime.
VPLane(unsigned Lane, Kind LaneKind)
Kind getKind() const
Returns the Kind of lane offset.
bool isFirstLane() const
Returns true if this is the first lane of the whole vector.
unsigned getKnownLane() const
Returns a compile-time known value for the lane index and asserts if the lane can only be calculated ...
static VPLane getFirstLane()
Kind
Kind describes how to interpret Lane.
@ ScalableLast
For ScalableLast, Lane is the offset from the start of the last N-element subvector in a scalable vec...
@ First
For First, Lane is the index into the first N elements of a fixed-vector <N x <ElTy>> or a scalable v...
unsigned mapToCacheIndex(const ElementCount &VF) const
Maps the lane to a cache index based on VF.
A value that is used outside the VPlan.
VPLiveOut(PHINode *Phi, VPValue *Op)
static bool classof(const VPUser *U)
bool usesScalars(const VPValue *Op) const override
Returns true if the VPLiveOut uses scalars of operand Op.
void print(raw_ostream &O, VPSlotTracker &SlotTracker) const
Print the VPLiveOut to O.
void fixPhi(VPlan &Plan, VPTransformState &State)
Fixup the wrapped LCSSA phi node in the unique exit block.
VPPredInstPHIRecipe is a recipe for generating the phi nodes needed when control converges back from ...
~VPPredInstPHIRecipe() override=default
bool usesScalars(const VPValue *Op) const override
Returns true if the recipe uses scalars of operand Op.
VPPredInstPHIRecipe(VPValue *PredV)
Construct a VPPredInstPHIRecipe given PredInst whose value needs a phi nodes after merging back from ...
void execute(VPTransformState &State) override
Generates phi nodes for live-outs as needed to retain SSA form.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPRecipeBase is a base class modeling a sequence of one or more output IR instructions.
bool mayReadFromMemory() const
Returns true if the recipe may read from memory.
VPRecipeBase(const unsigned char SC, ArrayRef< VPValue * > Operands)
bool mayReadOrWriteMemory() const
Returns true if the recipe may read from or write to memory.
bool mayHaveSideEffects() const
Returns true if the recipe may have side-effects.
Instruction * getUnderlyingInstr()
Returns the underlying instruction, if the recipe is a VPValue or nullptr otherwise.
bool mayWriteToMemory() const
Returns true if the recipe may write to memory.
virtual ~VPRecipeBase()=default
VPBasicBlock * getParent()
virtual void execute(VPTransformState &State)=0
The method which generates the output IR instructions that correspond to this VPRecipe,...
void moveBefore(VPBasicBlock &BB, iplist< VPRecipeBase >::iterator I)
Unlink this recipe and insert into BB before I.
void insertBefore(VPRecipeBase *InsertPos)
Insert an unlinked recipe into a basic block immediately before the specified recipe.
void insertAfter(VPRecipeBase *InsertPos)
Insert an unlinked Recipe into a basic block immediately after the specified Recipe.
static bool classof(const VPDef *D)
Method to support type inquiry through isa, cast, and dyn_cast.
iplist< VPRecipeBase >::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
VPRecipeBase(const unsigned char SC, iterator_range< IterT > Operands)
const VPBasicBlock * getParent() const
const Instruction * getUnderlyingInstr() const
static bool classof(const VPUser *U)
void removeFromParent()
This method unlinks 'this' from the containing basic block, but does not delete it.
bool isPhi() const
Returns true for PHI-like recipes.
void moveAfter(VPRecipeBase *MovePos)
Unlink this recipe from its current VPBasicBlock and insert it into the VPBasicBlock that MovePos liv...
Class to record LLVM IR flag for a recipe along with it.
VPRecipeWithIRFlags(const unsigned char SC, iterator_range< IterT > Operands)
void setFlags(Instruction *I) const
Set the IR flags for I.
static bool classof(const VPRecipeBase *R)
void dropPoisonGeneratingFlags()
Drop all poison-generating flags.
VPRecipeWithIRFlags(const unsigned char SC, iterator_range< IterT > Operands, Instruction &I)
FastMathFlags getFastMathFlags() const
void printFlags(raw_ostream &O) const
A recipe for handling reduction phis.
VPReductionPHIRecipe(PHINode *Phi, const RecurrenceDescriptor &RdxDesc, VPValue &Start, bool IsInLoop=false, bool IsOrdered=false)
Create a new VPReductionPHIRecipe for the reduction Phi described by RdxDesc.
bool isOrdered() const
Returns true, if the phi is part of an ordered reduction.
~VPReductionPHIRecipe() override=default
bool isInLoop() const
Returns true, if the phi is part of an in-loop reduction.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
Generate the phi/select nodes.
static bool classof(const VPHeaderPHIRecipe *R)
const RecurrenceDescriptor & getRecurrenceDescriptor() const
A recipe to represent inloop reduction operations, performing a reduction on a vector operand into a ...
VPValue * getVecOp() const
The VPValue of the vector value to be reduced.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPValue * getCondOp() const
The VPValue of the condition for the block.
~VPReductionRecipe() override=default
VPValue * getChainOp() const
The VPValue of the scalar Chain being accumulated.
VPReductionRecipe(const RecurrenceDescriptor *R, Instruction *I, VPValue *ChainOp, VPValue *VecOp, VPValue *CondOp, const TargetTransformInfo *TTI)
void execute(VPTransformState &State) override
Generate the reduction in the loop.
VPRegionBlock represents a collection of VPBasicBlocks and VPRegionBlocks which form a Single-Entry-S...
const VPBlockBase * getEntry() const
bool isReplicator() const
An indicator whether this region is to generate multiple replicated instances of output IR correspond...
void dropAllReferences(VPValue *NewValue) override
Replace all operands of VPUsers in the block with NewValue and also replaces all uses of VPValues def...
void setExiting(VPBlockBase *ExitingBlock)
Set ExitingBlock as the exiting VPBlockBase of this VPRegionBlock.
VPBlockBase * getExiting()
void setEntry(VPBlockBase *EntryBlock)
Set EntryBlock as the entry VPBlockBase of this VPRegionBlock.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print this VPRegionBlock to O (recursively), prefixing all lines with Indent.
VPRegionBlock(const std::string &Name="", bool IsReplicator=false)
VPRegionBlock(VPBlockBase *Entry, VPBlockBase *Exiting, const std::string &Name="", bool IsReplicator=false)
void execute(VPTransformState *State) override
The method which generates the output IR instructions that correspond to this VPRegionBlock,...
const VPBlockBase * getExiting() const
VPBasicBlock * getPreheaderVPBB()
Returns the pre-header VPBasicBlock of the loop region.
~VPRegionBlock() override
static bool classof(const VPBlockBase *V)
Method to support type inquiry through isa, cast, and dyn_cast.
VPReplicateRecipe replicates a given instruction producing multiple scalar copies of the original sca...
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
Generate replicas of the desired Ingredient.
~VPReplicateRecipe() override=default
bool onlyFirstLaneUsed(const VPValue *Op) const override
Returns true if the recipe only uses the first lane of operand Op.
bool usesScalars(const VPValue *Op) const override
Returns true if the recipe uses scalars of operand Op.
bool isPredicated() const
VPReplicateRecipe(Instruction *I, iterator_range< IterT > Operands, bool IsUniform, VPValue *Mask=nullptr)
VPValue * getMask()
Return the mask of a predicated VPReplicateRecipe.
bool shouldPack() const
Returns true if the recipe is used by a widened recipe via an intervening VPPredInstPHIRecipe.
A recipe for handling phi nodes of integer and floating-point inductions, producing their scalar valu...
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
bool onlyFirstLaneUsed(const VPValue *Op) const override
Returns true if the recipe only uses the first lane of operand Op.
VPValue * getStepValue() const
VPScalarIVStepsRecipe(const InductionDescriptor &IndDesc, VPValue *IV, VPValue *Step)
~VPScalarIVStepsRecipe() override=default
void execute(VPTransformState &State) override
Generate the scalarized versions of the phi node as needed by their users.
This class can be used to assign consecutive numbers to all VPValues in a VPlan and allows querying t...
This class augments VPValue with operands which provide the inverse def-use edges from VPValue's user...
void setOperand(unsigned I, VPValue *New)
unsigned getNumOperands() const
operand_iterator op_begin()
VPValue * getOperand(unsigned N) const
VPUserID
Subclass identifier (for isa/dyn_cast).
void addOperand(VPValue *Operand)
VPRecipeBase * getDefiningRecipe()
Returns the recipe defining this VPValue or nullptr if it is not defined by a recipe,...
friend class VPInstruction
void setUnderlyingValue(Value *Val)
Value * getLiveInIRValue()
Returns the underlying IR value, if this VPValue is defined outside the scope of VPlan.
bool isLiveIn() const
Returns true if this VPValue is a live-in, i.e. defined outside the VPlan.
bool isDefinedOutsideVectorRegions() const
Returns true if the VPValue is defined outside any vector regions, i.e.
A recipe for widening Call instructions.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
Produce a widened version of the call instruction.
VPWidenCallRecipe(CallInst &I, iterator_range< IterT > CallArguments, Intrinsic::ID VectorIntrinsicID, Function *Variant=nullptr)
~VPWidenCallRecipe() override=default
A Recipe for widening the canonical induction variable of the vector loop.
void execute(VPTransformState &State) override
Generate a canonical vector induction variable of the vector loop, with start = {<Part*VF,...
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
~VPWidenCanonicalIVRecipe() override=default
VPWidenCanonicalIVRecipe(VPCanonicalIVPHIRecipe *CanonicalIV)
const Type * getScalarType() const
Returns the scalar type of the induction.
VPWidenCastRecipe is a recipe to create vector cast instructions.
Instruction::CastOps getOpcode() const
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Type * getResultType() const
Returns the result type of the cast.
void execute(VPTransformState &State) override
Produce widened copies of the cast.
VPWidenCastRecipe(Instruction::CastOps Opcode, VPValue *Op, Type *ResultTy, CastInst *UI=nullptr)
~VPWidenCastRecipe() override=default
A recipe for handling GEP instructions.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
Generate the gep nodes.
~VPWidenGEPRecipe() override=default
VPWidenGEPRecipe(GetElementPtrInst *GEP, iterator_range< IterT > Operands)
A recipe for handling phi nodes of integer and floating-point inductions, producing their vector valu...
VPWidenIntOrFpInductionRecipe(PHINode *IV, VPValue *Start, VPValue *Step, const InductionDescriptor &IndDesc, TruncInst *Trunc)
const TruncInst * getTruncInst() const
VPRecipeBase & getBackedgeRecipe() override
Returns the backedge value as a recipe.
~VPWidenIntOrFpInductionRecipe() override=default
TruncInst * getTruncInst()
Returns the first defined value as TruncInst, if it is one or nullptr otherwise.
void execute(VPTransformState &State) override
Generate the vectorized and scalarized versions of the phi node as needed by their users.
VPValue * getStepValue()
Returns the step value of the induction.
const Type * getScalarType() const
Returns the scalar type of the induction.
VPWidenIntOrFpInductionRecipe(PHINode *IV, VPValue *Start, VPValue *Step, const InductionDescriptor &IndDesc)
const VPValue * getStepValue() const
VPValue * getBackedgeValue() override
Returns the incoming value from the loop backedge.
bool isCanonical() const
Returns true if the induction is canonical, i.e.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
const InductionDescriptor & getInductionDescriptor() const
Returns the induction descriptor for the recipe.
A Recipe for widening load/store operations.
VPValue * getMask() const
Return the mask used by this recipe.
VPValue * getAddr() const
Return the address accessed by this recipe.
Instruction & getIngredient() const
VPWidenMemoryInstructionRecipe(StoreInst &Store, VPValue *Addr, VPValue *StoredValue, VPValue *Mask, bool Consecutive, bool Reverse)
void execute(VPTransformState &State) override
Generate the wide load/store.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPWidenMemoryInstructionRecipe(LoadInst &Load, VPValue *Addr, VPValue *Mask, bool Consecutive, bool Reverse)
VPValue * getStoredValue() const
Return the address accessed by this recipe.
bool onlyFirstLaneUsed(const VPValue *Op) const override
Returns true if the recipe only uses the first lane of operand Op.
bool isStore() const
Returns true if this recipe is a store.
bool isConsecutive() const
A recipe for handling header phis that are widened in the vector loop.
void addIncoming(VPValue *IncomingV, VPBasicBlock *IncomingBlock)
Adds a pair (IncomingV, IncomingBlock) to the phi.
VPValue * getIncomingValue(unsigned I)
Returns the I th incoming VPValue.
VPWidenPHIRecipe(PHINode *Phi, VPValue *Start=nullptr)
Create a new VPWidenPHIRecipe for Phi with start value Start.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
~VPWidenPHIRecipe() override=default
VPBasicBlock * getIncomingBlock(unsigned I)
Returns the I th incoming VPBasicBlock.
void execute(VPTransformState &State) override
Generate the phi/select nodes.
bool onlyScalarsGenerated(ElementCount VF)
Returns true if only scalar values will be generated.
const InductionDescriptor & getInductionDescriptor() const
Returns the induction descriptor for the recipe.
~VPWidenPointerInductionRecipe() override=default
void execute(VPTransformState &State) override
Generate vector values for the pointer induction.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPWidenPointerInductionRecipe(PHINode *Phi, VPValue *Start, VPValue *Step, const InductionDescriptor &IndDesc, bool IsScalarAfterVectorization)
Create a new VPWidenPointerInductionRecipe for Phi with start value Start.
VPWidenRecipe is a recipe for producing a copy of vector type its ingredient.
void execute(VPTransformState &State) override
Produce widened copies of all Ingredients.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
~VPWidenRecipe() override=default
VPWidenRecipe(Instruction &I, iterator_range< IterT > Operands)
VPlanPrinter prints a given VPlan to a given output stream.
VPlanPrinter(raw_ostream &O, const VPlan &P)
LLVM_DUMP_METHOD void dump()
Class that maps (parts of) an existing VPlan to trees of combined VPInstructions.
VPInstruction * buildGraph(ArrayRef< VPValue * > Operands)
Tries to build an SLP tree rooted at Operands and returns a VPInstruction combining Operands,...
bool isCompletelySLP() const
Return true if all visited instruction can be combined.
VPlanSlp(VPInterleavedAccessInfo &IAI, VPBasicBlock &BB)
unsigned getWidestBundleBits() const
Return the width of the widest combined bundle in bits.
VPlan models a candidate for vectorization, encoding various decisions take to produce efficient outp...
void printDOT(raw_ostream &O) const
Print this VPlan in DOT format to O.
std::string getName() const
Return a string with the name of the plan and the applicable VFs and UFs.
VPBasicBlock * getEntry()
void addVPValue(Value *V, VPValue *VPV)
VPValue & getVectorTripCount()
The vector trip count.
void setName(const Twine &newName)
VPValue * getTripCount() const
The trip count of the original loop.
void removeVPValueFor(Value *V)
VPValue * getOrCreateBackedgeTakenCount()
The backedge taken count of the original loop.
void removeLiveOut(PHINode *PN)
void addLiveOut(PHINode *PN, VPValue *V)
const VPBasicBlock * getEntry() const
void prepareToExecute(Value *TripCount, Value *VectorTripCount, Value *CanonicalIVStartValue, VPTransformState &State, bool IsEpilogueVectorization)
Prepare the plan for execution, setting up the required live-in values.
VPlan(VPBasicBlock *Preheader, VPValue *TC, VPBasicBlock *Entry)
Construct a VPlan with original preheader Preheader, trip count TC and Entry to the plan.
VPBasicBlock * getPreheader()
VPValue * getVPValueOrAddLiveIn(Value *V)
Gets the VPValue for V or adds a new live-in (if none exists yet) for V.
VPRegionBlock * getVectorLoopRegion()
Returns the VPRegionBlock of the vector loop.
const VPRegionBlock * getVectorLoopRegion() const
static VPlanPtr createInitialVPlan(const SCEV *TripCount, ScalarEvolution &PSE)
Create an initial VPlan with preheader and entry blocks.
bool hasVF(ElementCount VF)
void addSCEVExpansion(const SCEV *S, VPValue *V)
bool hasUF(unsigned UF) const
void setVF(ElementCount VF)
VPValue * getVPValue(Value *V, bool OverrideAllowed=false)
Returns the VPValue for V.
VPlan(VPBasicBlock *Preheader, VPBasicBlock *Entry)
Construct a VPlan with original preheader Preheader and Entry to the plan.
void disableValue2VPValue()
Mark the plan to indicate that using Value2VPValue is not safe any longer, because it may be stale.
const VPBasicBlock * getPreheader() const
LLVM_DUMP_METHOD void dump() const
Dump the plan to stderr (for debugging).
bool hasScalarVFOnly() const
iterator_range< mapped_iterator< Use *, std::function< VPValue *(Value *)> > > mapToVPValues(User::op_range Operands)
Returns a range mapping the values the range Operands to their corresponding VPValues.
void execute(VPTransformState *State)
Generate the IR code for this VPlan.
VPCanonicalIVPHIRecipe * getCanonicalIV()
Returns the canonical induction recipe of the vector loop.
const MapVector< PHINode *, VPLiveOut * > & getLiveOuts() const
VPActiveLaneMaskPHIRecipe * getActiveLaneMaskPhi()
Find and return the VPActiveLaneMaskPHIRecipe from the header - there be only one at most.
void print(raw_ostream &O) const
Print this VPlan to O.
void addVF(ElementCount VF)
VPValue * getSCEVExpansion(const SCEV *S) const
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
Iterator for intrusive lists based on ilist_node.
An ilist node that can access its parent list.
base_list_type::const_reverse_iterator const_reverse_iterator
base_list_type::reverse_iterator reverse_iterator
base_list_type::const_iterator const_iterator
base_list_type::iterator iterator
iterator insert(iterator where, pointer New)
CRTP base class which implements the entire standard iterator facade in terms of a minimal subset of ...
A range adaptor for a pair of iterators.
This class implements an extremely fast bulk output stream that can only output to a stream.
This file defines classes to implement an intrusive doubly linked list class (i.e.
This file defines the ilist_node class template, which is a convenient base class for creating classe...
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ BasicBlock
Various leaf nodes.
VPValue * getOrCreateVPValueForSCEVExpr(VPlan &Plan, const SCEV *Expr, ScalarEvolution &SE)
Get or create a VPValue that corresponds to the expansion of Expr.
bool isUniformAfterVectorization(VPValue *VPV)
Returns true if VPV is uniform after vectorization.
bool onlyFirstLaneUsed(VPValue *Def)
Returns true if only the first lane of Def is used.
This is an optimization pass for GlobalISel generic memory operations.
auto find(R &&Range, const T &Val)
Provide wrappers to std::find which take ranges instead of having to pass begin/end explicitly.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
bool isEqual(const GCNRPTracker::LiveRegSet &S1, const GCNRPTracker::LiveRegSet &S2)
const SCEV * createTripCountSCEV(Type *IdxTy, PredicatedScalarEvolution &PSE, Loop *OrigLoop)
testing::Matcher< const detail::ErrorHolder & > Failed()
Value * getRuntimeVF(IRBuilderBase &B, Type *Ty, ElementCount VF)
Return the runtime value for VF.
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
auto map_range(ContainerTy &&C, FuncTy F)
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
std::unique_ptr< VPlan > VPlanPtr
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
iterator_range< filter_iterator< detail::IterOfRange< RangeT >, PredicateT > > make_filter_range(RangeT &&Range, PredicateT Pred)
Convenience function that takes a range of elements and a predicate, and return a new filter_iterator...
raw_ostream & operator<<(raw_ostream &OS, const APFixedPoint &FX)
Value * createStepForVF(IRBuilderBase &B, Type *Ty, ElementCount VF, int64_t Step)
Return a value for Step multiplied by VF.
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
hash_code hash_combine_range(InputIteratorT first, InputIteratorT last)
Compute a hash_code for a sequence of values.
A range of powers-of-2 vectorization factors with fixed start and adjustable end.
VFRange(const ElementCount &Start, const ElementCount &End)
A recipe for handling first-order recurrence phis.
void execute(VPTransformState &State) override
Generate the phi nodes.
VPFirstOrderRecurrencePHIRecipe(PHINode *Phi, VPValue &Start)
static bool classof(const VPHeaderPHIRecipe *R)
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPIteration represents a single point in the iteration space of the output (vectorized and/or unrolle...
VPIteration(unsigned Part, const VPLane &Lane)
VPIteration(unsigned Part, unsigned Lane, VPLane::Kind Kind=VPLane::Kind::First)
bool isFirstIteration() const
A recipe for widening select instructions.
bool isInvariantCond() const
VPWidenSelectRecipe(SelectInst &I, iterator_range< IterT > Operands)
VPValue * getCond() const
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
Produce a widened version of the select instruction.
~VPWidenSelectRecipe() override=default
VPlanIngredient(const Value *V)
void print(raw_ostream &O) const