54#define DEBUG_TYPE "vplan"
56#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
80 : SubclassID(SC), UnderlyingVal(UV), Def(Def) {
82 Def->addDefinedValue(
this);
86 assert(Users.empty() &&
"trying to delete a VPValue with remaining users");
88 Def->removeDefinedValue(
this);
91#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
108 const VPRecipeBase *Instr = dyn_cast_or_null<VPRecipeBase>(
this);
117 return cast_or_null<VPRecipeBase>(
Def);
121 return cast_or_null<VPRecipeBase>(
Def);
129 while ((Next = Next->getParent()))
135 for (
unsigned i = 0; i < WorkList.
size(); i++) {
136 T *Current = WorkList[i];
137 if (Current->getNumPredecessors() == 0)
139 auto &Predecessors = Current->getPredecessors();
140 WorkList.
insert(Predecessors.begin(), Predecessors.end());
155 return cast<VPBasicBlock>(
Block);
162 return cast<VPBasicBlock>(
Block);
168 "Can only set plan on its entry or preheader block.");
177 return cast<VPBasicBlock>(
Block);
184 return cast<VPBasicBlock>(
Block);
188 if (!Successors.empty() || !Parent)
191 "Block w/o successors not the exiting block of its parent.");
196 if (!Predecessors.empty() || !Parent)
199 "Block w/o predecessors not the entry of its parent.");
210 while (It !=
end() && It->isPhi())
217 return Def->getLiveInIRValue();
226 if (!VecPart->getType()->isVectorTy()) {
227 assert(
Instance.Lane.isFirstLane() &&
"cannot get lane > 0 for scalar");
237 VPRegionBlock *LoopRegion = R->getParent()->getEnclosingLoopRegion();
245 if (
LVer && (isa<LoadInst>(Orig) || isa<StoreInst>(Orig)))
263 for (
Value *V : To) {
270 const Instruction *Inst = dyn_cast<Instruction>(V);
288 << DIL->getFilename() <<
" Line: " << DIL->getLine());
303 for (
VPBlockBase *PredVPBlock : getHierarchicalPredecessors()) {
308 assert(PredBB &&
"Predecessor basic-block not found building successor.");
312 auto *TermBr = dyn_cast<BranchInst>(PredBBTerminator);
313 if (isa<UnreachableInst>(PredBBTerminator)) {
314 assert(PredVPSuccessors.size() == 1 &&
315 "Predecessor ending w/o branch must have single successor.");
316 DebugLoc DL = PredBBTerminator->getDebugLoc();
320 }
else if (TermBr && !TermBr->isConditional()) {
321 TermBr->setSuccessor(0, NewBB);
325 unsigned idx = PredVPSuccessors.front() ==
this ? 0 : 1;
326 assert(!TermBr->getSuccessor(idx) &&
327 "Trying to reset an existing successor block.");
328 TermBr->setSuccessor(idx, NewBB);
341 auto *R = dyn_cast<VPRegionBlock>(BB);
342 return R && !R->isReplicator();
346 if (getPlan()->getVectorLoopRegion()->getSingleSuccessor() ==
this) {
352 VPBlockBase *PredVPB = getSingleHierarchicalPredecessor();
355 "predecessor must have the current block as only successor");
359 cast<BranchInst>(ExitingBB->
getTerminator())->setSuccessor(0, NewBB);
360 }
else if (PrevVPBB &&
361 !((SingleHPred = getSingleHierarchicalPredecessor()) &&
364 (SingleHPred->
getParent() == getEnclosingLoopRegion() &&
365 !IsLoopRegion(SingleHPred))) &&
366 !(Replica && getPredecessors().empty())) {
376 NewBB = createEmptyBasicBlock(State->
CFG);
390 <<
" in BB:" << NewBB->
getName() <<
'\n');
403 for (
auto *Def : R.definedValues())
404 Def->replaceAllUsesWith(NewValue);
406 for (
unsigned I = 0,
E = R.getNumOperands();
I !=
E;
I++)
407 R.setOperand(
I, NewValue);
412 assert((SplitAt == end() || SplitAt->getParent() ==
this) &&
413 "can only split at a position in the same block");
438 if (
P &&
P->isReplicator()) {
440 assert(!cast<VPRegionBlock>(
P)->isReplicator() &&
441 "unexpected nested replicate regions");
450 "block with multiple successors doesn't have a recipe as terminator");
455 auto *VPI = dyn_cast<VPInstruction>(R);
457 isa<VPBranchOnMaskRecipe>(R) ||
464 "conditional branch recipe");
471 "block with 0 or 1 successors terminated by conditional branch recipe");
488 return getParent()->getExitingBasicBlock() ==
this;
491#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
493 if (getSuccessors().empty()) {
494 O << Indent <<
"No successors\n";
496 O << Indent <<
"Successor(s): ";
498 for (
auto *Succ : getSuccessors())
499 O << LS << Succ->getName();
506 O << Indent <<
getName() <<
":\n";
508 auto RecipeIndent = Indent +
" ";
514 printSuccessors(O, Indent);
522 Block->dropAllReferences(NewValue);
529 if (!isReplicator()) {
546 Block->execute(State);
553 assert(!State->
Instance &&
"Replicating a Region with non-null instance.");
558 for (
unsigned Part = 0,
UF = State->
UF; Part <
UF; ++Part) {
567 Block->execute(State);
576#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
579 O << Indent << (isReplicator() ?
"<xVFxUF> " :
"<x1> ") <<
getName() <<
": {";
580 auto NewIndent = Indent +
" ";
585 O << Indent <<
"}\n";
587 printSuccessors(O, Indent);
592 for (
auto &KV : LiveOuts)
599 Block->dropAllReferences(&DummyValue);
603 Preheader->dropAllReferences(&DummyValue);
606 for (
VPValue *VPV : VPLiveInsToFree)
608 if (BackedgeTakenCount)
609 delete BackedgeTakenCount;
615 auto Plan = std::make_unique<VPlan>(Preheader, VecPreheader);
622 VPBasicBlock *Header = getVectorLoopRegion()->getEntryBasicBlock();
624 if (isa<VPActiveLaneMaskPHIRecipe>(&R))
625 return cast<VPActiveLaneMaskPHIRecipe>(&R);
631 Value *CanonicalIVStartValue,
633 bool IsEpilogueVectorization) {
635 if (BackedgeTakenCount && BackedgeTakenCount->getNumUsers()) {
639 "trip.count.minus.1");
643 for (
unsigned Part = 0,
UF = State.
UF; Part <
UF; ++Part)
644 State.
set(BackedgeTakenCount, VTCMO, Part);
647 for (
unsigned Part = 0,
UF = State.
UF; Part <
UF; ++Part)
648 State.
set(&VectorTripCount, VectorTripCountV, Part);
653 if (CanonicalIVStartValue) {
654 VPValue *VPV = getVPValueOrAddLiveIn(CanonicalIVStartValue);
655 auto *
IV = getCanonicalIV();
658 if (isa<VPScalarIVStepsRecipe>(U) ||
659 isa<VPDerivedIVRecipe>(U))
661 auto *VPI = cast<VPInstruction>(U);
662 return VPI->getOpcode() ==
663 VPInstruction::CanonicalIVIncrement ||
665 VPInstruction::CanonicalIVIncrementNUW;
667 "the canonical IV should only be used by its increments or "
668 "ScalarIVSteps when resetting the start value");
669 IV->setOperand(0, VPV);
678 for (
auto &Entry : Value2VPValue)
689 Block->execute(State);
696 VPBasicBlock *Header = getVectorLoopRegion()->getEntryBasicBlock();
699 if (isa<VPWidenPHIRecipe>(&R))
702 if (isa<VPWidenPointerInductionRecipe>(&R) ||
703 isa<VPWidenIntOrFpInductionRecipe>(&R)) {
705 if (isa<VPWidenIntOrFpInductionRecipe>(&R)) {
706 Phi = cast<PHINode>(State->
get(R.getVPSingleValue(), 0));
708 auto *WidenPhi = cast<VPWidenPointerInductionRecipe>(&R);
711 if (WidenPhi->onlyScalarsGenerated(State->
VF))
714 auto *
GEP = cast<GetElementPtrInst>(State->
get(WidenPhi, 0));
715 Phi = cast<PHINode>(
GEP->getPointerOperand());
727 auto *PhiR = cast<VPHeaderPHIRecipe>(&R);
732 bool SinglePartNeeded = isa<VPCanonicalIVPHIRecipe>(PhiR) ||
733 isa<VPFirstOrderRecurrencePHIRecipe>(PhiR) ||
734 (isa<VPReductionPHIRecipe>(PhiR) &&
735 cast<VPReductionPHIRecipe>(PhiR)->isOrdered());
736 unsigned LastPartForNewPhi = SinglePartNeeded ? 1 : State->
UF;
738 for (
unsigned Part = 0; Part < LastPartForNewPhi; ++Part) {
739 Value *Phi = State->
get(PhiR, Part);
740 Value *Val = State->
get(PhiR->getBackedgeValue(),
741 SinglePartNeeded ? State->
UF - 1 : Part);
742 cast<PHINode>(Phi)->addIncoming(Val, VectorLatchBB);
750 updateDominatorTree(State->
DT, VectorHeaderBB, VectorLatchBB,
755#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
760 O <<
"VPlan '" <<
getName() <<
"' {";
762 if (VectorTripCount.getNumUsers() > 0) {
765 O <<
" = vector-trip-count";
768 if (BackedgeTakenCount && BackedgeTakenCount->getNumUsers()) {
770 BackedgeTakenCount->printAsOperand(O,
SlotTracker);
771 O <<
" = backedge-taken count";
775 if (TripCount->isLiveIn())
778 O <<
" = original trip-count";
781 if (!getPreheader()->empty()) {
791 if (!LiveOuts.empty())
793 for (
const auto &KV : LiveOuts) {
803 RSO <<
Name <<
" for ";
805 RSO <<
"VF={" << VFs[0];
814 RSO <<
"UF={" << UFs[0];
834 assert(LiveOuts.count(PN) == 0 &&
"an exit value for PN already exists");
835 LiveOuts.insert({PN,
new VPLiveOut(PN, V)});
845 for (
auto *BB = LoopHeaderBB; BB != LoopLatchBB; BB = PostDomSucc) {
848 assert(Succs.size() <= 2 &&
849 "Basic block in vector loop has more than 2 successors.");
850 PostDomSucc = Succs[0];
851 if (Succs.size() == 1) {
853 "PostDom successor has more than one predecessor.");
859 PostDomSucc = Succs[1];
860 InterimSucc = Succs[0];
863 "One successor of a basic block does not lead to the other.");
865 "Interim successor has more than one predecessor.");
867 "PostDom successor has more than two predecessors.");
876#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
879 return (isa<VPRegionBlock>(
Block) ?
"cluster_N" :
"N") +
884 const std::string &
Name =
Block->getName();
893 OS <<
"digraph VPlan {\n";
894 OS <<
"graph [labelloc=t, fontsize=30; label=\"Vectorization Plan";
897 if (
Plan.BackedgeTakenCount) {
900 OS <<
" := BackedgeTakenCount";
903 OS <<
"node [shape=rect, fontname=Courier, fontsize=30]\n";
904 OS <<
"edge [fontname=Courier, fontsize=30]\n";
905 OS <<
"compound=true\n";
925 bool Hidden,
const Twine &Label) {
930 OS << Indent << getUID(
Tail) <<
" -> " << getUID(Head);
931 OS <<
" [ label=\"" << Label <<
'\"';
933 OS <<
" ltail=" << getUID(
From);
935 OS <<
" lhead=" << getUID(To);
937 OS <<
"; splines=none";
942 auto &Successors =
Block->getSuccessors();
943 if (Successors.size() == 1)
944 drawEdge(
Block, Successors.front(),
false,
"");
945 else if (Successors.size() == 2) {
946 drawEdge(
Block, Successors.front(),
false,
"T");
947 drawEdge(
Block, Successors.back(),
false,
"F");
949 unsigned SuccessorNumber = 0;
976 EmitLine(Line,
" +\n");
977 EmitLine(
Lines.back(),
"\n");
980 OS << Indent <<
"]\n";
986 OS << Indent <<
"subgraph " << getUID(
Region) <<
" {\n";
988 OS << Indent <<
"fontname=Courier\n"
989 << Indent <<
"label=\""
997 OS << Indent <<
"}\n";
1002 if (
auto *Inst = dyn_cast<Instruction>(V)) {
1003 if (!Inst->getType()->isVoidTy()) {
1004 Inst->printAsOperand(O,
false);
1007 O << Inst->getOpcodeName() <<
" ";
1008 unsigned E = Inst->getNumOperands();
1010 Inst->getOperand(0)->printAsOperand(O,
false);
1011 for (
unsigned I = 1;
I <
E; ++
I)
1012 Inst->getOperand(
I)->printAsOperand(O <<
", ",
false);
1015 V->printAsOperand(O,
false);
1023 for (
unsigned J = 0; J < getNumUsers();) {
1025 unsigned NumUsers = getNumUsers();
1032 if (NumUsers == getNumUsers())
1037#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1039 if (
const Value *UV = getUnderlyingValue()) {
1041 UV->printAsOperand(
OS,
false);
1046 unsigned Slot = Tracker.
getSlot(
this);
1047 if (Slot ==
unsigned(-1))
1050 OS <<
"vp<%" << Tracker.
getSlot(
this) <<
">";
1066 visitBlock(
Base, Old2New, IAI);
1070void VPInterleavedAccessInfo::visitBlock(
VPBlockBase *
Block, Old2NewTy &Old2New,
1074 if (isa<VPHeaderPHIRecipe>(&VPI))
1076 assert(isa<VPInstruction>(&VPI) &&
"Can only handle VPInstructions");
1077 auto *VPInst = cast<VPInstruction>(&VPI);
1079 auto *Inst = dyn_cast_or_null<Instruction>(VPInst->getUnderlyingValue());
1086 auto NewIGIter = Old2New.find(IG);
1087 if (NewIGIter == Old2New.end())
1089 IG->getFactor(), IG->isReverse(), IG->getAlign());
1091 if (Inst == IG->getInsertPos())
1092 Old2New[IG]->setInsertPos(VPInst);
1094 InterleaveGroupMap[VPInst] = Old2New[IG];
1095 InterleaveGroupMap[VPInst]->insertMember(
1096 VPInst, IG->getIndex(Inst),
1097 Align(IG->isReverse() ? (-1) *
int(IG->getFactor())
1098 : IG->getFactor()));
1101 visitRegion(
Region, Old2New, IAI);
1112void VPSlotTracker::assignSlot(
const VPValue *V) {
1113 assert(!Slots.contains(V) &&
"VPValue already has a slot!");
1114 Slots[V] = NextSlot++;
1117void VPSlotTracker::assignSlots(
const VPlan &
Plan) {
1118 assignSlot(&
Plan.VectorTripCount);
1119 if (
Plan.BackedgeTakenCount)
1120 assignSlot(
Plan.BackedgeTakenCount);
1126 VPBlockUtils::blocksOnly<const VPBasicBlock>(RPOT))
1130void VPSlotTracker::assignSlots(
const VPBasicBlock *VPBB) {
1132 for (
VPValue *Def : Recipe.definedValues())
1137 return all_of(Def->users(),
1138 [Def](
VPUser *U) { return U->onlyFirstLaneUsed(Def); });
1146 if (
auto *
E = dyn_cast<SCEVConstant>(Expr))
1148 else if (
auto *
E = dyn_cast<SCEVUnknown>(Expr))
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static const Function * getParent(const Value *V)
BlockVerifier::State From
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
#define LLVM_DUMP_METHOD
Mark debug helper function definitions like dump() that should not be stripped from debug builds.
This file builds on the ADT/GraphTraits.h file to build generic depth first graph iterator.
static void dumpEdges(CFGMST< Edge, BBInfo > &MST, GCOVFunction &GF)
Generic dominator tree construction - this file provides routines to construct immediate dominator in...
This file provides various utilities for inspecting and working with the control flow graph in LLVM I...
iv Induction Variable Users
Memory true print Memory SSA Printer
This file builds on the ADT/GraphTraits.h file to build a generic graph post order iterator.
static StringRef getName(Value *V)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the SmallVector class.
This file implements dominator tree analysis for a single level of a VPlan's H-CFG.
static T * getPlanEntry(T *Start)
static bool hasConditionalTerminator(const VPBasicBlock *VPBB)
This file contains the declarations of the Vectorization Plan base classes:
static bool IsCondBranch(unsigned BrOpc)
static const uint32_t IV[8]
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
LLVM Basic Block Representation.
void print(raw_ostream &OS, AssemblyAnnotationWriter *AAW=nullptr, bool ShouldPreserveUseListOrder=false, bool IsForDebug=false) const
Print the basic block to an output stream with an optional AssemblyAnnotationWriter.
static BasicBlock * Create(LLVMContext &Context, const Twine &Name="", Function *Parent=nullptr, BasicBlock *InsertBefore=nullptr)
Creates a new BasicBlock.
bool hasNPredecessors(unsigned N) const
Return true if this block has exactly N predecessors.
const BasicBlock * getSinglePredecessor() const
Return the predecessor of this block if it has a single predecessor block.
const BasicBlock * getSingleSuccessor() const
Return the successor of this block if it has a single successor.
const Function * getParent() const
Return the enclosing method, or null if none.
LLVMContext & getContext() const
Get the context in which this basic block lives.
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
static BranchInst * Create(BasicBlock *IfTrue, Instruction *InsertBefore=nullptr)
static Constant * get(Type *Ty, uint64_t V, bool IsSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
std::optional< const DILocation * > cloneByMultiplyingDuplicationFactor(unsigned DF) const
Returns a new DILocation with duplication factor DF * current duplication factor encoded in the discr...
Core dominator tree base class.
bool verify(VerificationLevel VL=VerificationLevel::Full) const
verify - checks if the tree is correct.
void changeImmediateDominator(DomTreeNodeBase< NodeT > *N, DomTreeNodeBase< NodeT > *NewIDom)
changeImmediateDominator - This method is used to update the dominator tree information when a node's...
DomTreeNodeBase< NodeT > * addNewBlock(NodeT *BB, NodeT *DomBB)
Add a new node to the dominator tree information.
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
constexpr bool isScalar() const
Exactly one element.
bool shouldEmitDebugInfoForProfiling() const
Returns true if we should emit debug info for profiling.
Common base class shared among various IRBuilders.
Value * CreateExtractElement(Value *Vec, Value *Idx, const Twine &Name="")
UnreachableInst * CreateUnreachable()
Value * CreateVectorSplat(unsigned NumElts, Value *V, const Twine &Name="")
Return a vector value that contains.
void SetCurrentDebugLocation(DebugLoc L)
Set location information used by debugging information.
Value * CreateSub(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
void SetInsertPoint(BasicBlock *TheBB)
This specifies that created instructions should be appended to the end of the specified block.
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
bool isDebugOrPseudoInst() const LLVM_READONLY
Return true if the instruction is a DbgInfoIntrinsic or PseudoProbeInst.
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
const Function * getFunction() const
Return the function this instruction belongs to.
void moveBefore(Instruction *MovePos)
Unlink this instruction from its current basic block and insert it into the basic block that MovePos ...
The group of interleaved loads/stores sharing the same stride and close to each other.
Drive the analysis of interleaved memory accesses in the loop.
InterleaveGroup< Instruction > * getInterleaveGroup(const Instruction *Instr) const
Get the interleave group that Instr belongs to.
void addBasicBlockToLoop(BlockT *NewBB, LoopInfoBase< BlockT, LoopT > &LI)
This method is used by other analyses to update loop information.
void addChildLoop(LoopT *NewChild)
Add the specified loop to be a child of this loop.
void addTopLevelLoop(LoopT *New)
This adds the specified loop to the collection of top-level loops.
LoopT * AllocateLoop(ArgsTy &&...Args)
LoopT * getLoopFor(const BlockT *BB) const
Return the inner most loop that BB lives in.
void annotateInstWithNoAlias(Instruction *VersionedInst, const Instruction *OrigInst)
Add the noalias annotations to VersionedInst.
Represents a single loop in the control flow graph.
void eraseFromParent()
This method unlinks 'this' from the containing function and deletes it.
void setIncomingBlock(unsigned i, BasicBlock *BB)
Value * getIncomingValue(unsigned i) const
Return incoming value number x.
BlockT * getEntry() const
Get the entry BasicBlock of the Region.
This class represents an analyzed expression in the program.
The main scalar evolution driver.
size_type size() const
Determine the number of elements in the SetVector.
bool insert(const value_type &X)
Insert a new element into the SetVector.
This class provides computation of slot numbers for LLVM Assembly writing.
A SetVector that performs no allocations if smaller than a certain size.
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StringRef - Represent a constant reference to a string, i.e.
std::pair< StringRef, StringRef > split(char Separator) const
Split into two substrings around the first occurrence of a separator character.
StringRef rtrim(char Char) const
Return string with consecutive Char characters starting from the right removed.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
This function has undefined behavior.
void setOperand(unsigned i, Value *Val)
Value * getOperand(unsigned i) const
unsigned getNumOperands() const
A recipe for generating the active lane mask for the vector loop that is used to predicate the vector...
VPBasicBlock serves as the leaf of the Hierarchical Control-Flow Graph.
void appendRecipe(VPRecipeBase *Recipe)
Augment the existing recipes of a VPBasicBlock with an additional Recipe as the last recipe.
RecipeListTy::iterator iterator
Instruction iterators...
void execute(VPTransformState *State) override
The method which generates the output IR instructions that correspond to this VPBasicBlock,...
iterator begin()
Recipe iterator methods.
iterator getFirstNonPhi()
Return the position of the first non-phi node recipe in the block.
VPRegionBlock * getEnclosingLoopRegion()
void dropAllReferences(VPValue *NewValue) override
Replace all operands of VPUsers in the block with NewValue and also replaces all uses of VPValues def...
VPBasicBlock * splitAt(iterator SplitAt)
Split current block at SplitAt by inserting a new block between the current block and its successors ...
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print this VPBsicBlock to O, prefixing all lines with Indent.
bool isExiting() const
Returns true if the block is exiting it's parent region.
VPRecipeBase * getTerminator()
If the block has multiple successors, return the branch recipe terminating the block.
const VPRecipeBase & back() const
VPBlockBase is the building block of the Hierarchical Control-Flow Graph.
VPRegionBlock * getParent()
const VPBasicBlock * getExitingBasicBlock() const
size_t getNumSuccessors() const
void printSuccessors(raw_ostream &O, const Twine &Indent) const
Print the successors of this block to O, prefixing all lines with Indent.
VPBlockBase * getEnclosingBlockWithPredecessors()
static void deleteCFG(VPBlockBase *Entry)
Delete all blocks reachable from a given VPBlockBase, inclusive.
void setPlan(VPlan *ParentPlan)
Sets the pointer of the plan containing the block.
VPBlockBase * getSingleHierarchicalSuccessor()
const VPBlocksTy & getHierarchicalSuccessors()
VPBlockBase * getEnclosingBlockWithSuccessors()
An Enclosing Block of a block B is any block containing B, including B itself.
const VPBasicBlock * getEntryBasicBlock() const
VPBlockBase * getSingleSuccessor() const
Helper for GraphTraits specialization that traverses through VPRegionBlocks.
static void insertBlockAfter(VPBlockBase *NewBlock, VPBlockBase *BlockPtr)
Insert disconnected VPBlockBase NewBlock after BlockPtr.
static void disconnectBlocks(VPBlockBase *From, VPBlockBase *To)
Disconnect VPBlockBases From and To bi-directionally.
static void connectBlocks(VPBlockBase *From, VPBlockBase *To)
Connect VPBlockBases From and To bi-directionally.
This class augments a recipe with a set of VPValues defined by the recipe.
void dump() const
Dump the VPDef to stderr (for debugging).
virtual void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const =0
Each concrete VPDef prints itself.
Recipe to expand a SCEV expression.
This is a concrete Recipe that models a single VPlan-level instruction.
VPInterleavedAccessInfo(VPlan &Plan, InterleavedAccessInfo &IAI)
In what follows, the term "input IR" refers to code that is fed into the vectorizer whereas the term ...
Value * getAsRuntimeExpr(IRBuilderBase &Builder, const ElementCount &VF) const
Returns an expression describing the lane index that can be used at runtime.
@ ScalableLast
For ScalableLast, Lane is the offset from the start of the last N-element subvector in a scalable vec...
@ First
For First, Lane is the index into the first N elements of a fixed-vector <N x <ElTy>> or a scalable v...
A value that is used outside the VPlan.
VPRecipeBase is a base class modeling a sequence of one or more output IR instructions.
VPBasicBlock * getParent()
VPRegionBlock represents a collection of VPBasicBlocks and VPRegionBlocks which form a Single-Entry-S...
const VPBlockBase * getEntry() const
void dropAllReferences(VPValue *NewValue) override
Replace all operands of VPUsers in the block with NewValue and also replaces all uses of VPValues def...
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print this VPRegionBlock to O (recursively), prefixing all lines with Indent.
void execute(VPTransformState *State) override
The method which generates the output IR instructions that correspond to this VPRegionBlock,...
const VPBlockBase * getExiting() const
VPBasicBlock * getPreheaderVPBB()
Returns the pre-header VPBasicBlock of the loop region.
This class can be used to assign consecutive numbers to all VPValues in a VPlan and allows querying t...
unsigned getSlot(const VPValue *V) const
This class augments VPValue with operands which provide the inverse def-use edges from VPValue's user...
void printOperands(raw_ostream &O, VPSlotTracker &SlotTracker) const
Print the operands to O.
VPRecipeBase * getDefiningRecipe()
Returns the recipe defining this VPValue or nullptr if it is not defined by a recipe,...
void printAsOperand(raw_ostream &OS, VPSlotTracker &Tracker) const
void dump() const
Dump the value to stderr (for debugging).
VPValue(const unsigned char SC, Value *UV=nullptr, VPDef *Def=nullptr)
void print(raw_ostream &OS, VPSlotTracker &Tracker) const
void replaceAllUsesWith(VPValue *New)
VPDef * Def
Pointer to the VPDef that defines this VPValue.
VPlanPrinter prints a given VPlan to a given output stream.
LLVM_DUMP_METHOD void dump()
VPlan models a candidate for vectorization, encoding various decisions take to produce efficient outp...
void printDOT(raw_ostream &O) const
Print this VPlan in DOT format to O.
std::string getName() const
Return a string with the name of the plan and the applicable VFs and UFs.
VPBasicBlock * getEntry()
void addLiveOut(PHINode *PN, VPValue *V)
void prepareToExecute(Value *TripCount, Value *VectorTripCount, Value *CanonicalIVStartValue, VPTransformState &State, bool IsEpilogueVectorization)
Prepare the plan for execution, setting up the required live-in values.
VPBasicBlock * getPreheader()
VPValue * getVPValueOrAddLiveIn(Value *V)
Gets the VPValue for V or adds a new live-in (if none exists yet) for V.
VPRegionBlock * getVectorLoopRegion()
Returns the VPRegionBlock of the vector loop.
static VPlanPtr createInitialVPlan(const SCEV *TripCount, ScalarEvolution &PSE)
Create an initial VPlan with preheader and entry blocks.
void addSCEVExpansion(const SCEV *S, VPValue *V)
LLVM_DUMP_METHOD void dump() const
Dump the plan to stderr (for debugging).
void execute(VPTransformState *State)
Generate the IR code for this VPlan.
VPActiveLaneMaskPHIRecipe * getActiveLaneMaskPhi()
Find and return the VPActiveLaneMaskPHIRecipe from the header - there be only one at most.
void print(raw_ostream &O) const
Print this VPlan to O.
VPValue * getSCEVExpansion(const SCEV *S) const
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
StringRef getName() const
Return a constant reference to the value's name.
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
This class implements an extremely fast bulk output stream that can only output to a stream.
A raw_ostream that writes to an std::string.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ Tail
Attemps to make calls as fast as possible while guaranteeing that tail call optimization can always b...
std::string EscapeString(const std::string &Label)
VPValue * getOrCreateVPValueForSCEVExpr(VPlan &Plan, const SCEV *Expr, ScalarEvolution &SE)
Get or create a VPValue that corresponds to the expansion of Expr.
bool onlyFirstLaneUsed(VPValue *Def)
Returns true if only the first lane of Def is used.
This is an optimization pass for GlobalISel generic memory operations.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
Interval::succ_iterator succ_end(Interval *I)
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
auto successors(const MachineBasicBlock *BB)
Value * getRuntimeVF(IRBuilderBase &B, Type *Ty, ElementCount VF)
Return the runtime value for VF.
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
Interval::succ_iterator succ_begin(Interval *I)
succ_begin/succ_end - define methods so that Intervals may be used just like BasicBlocks can with the...
void interleaveComma(const Container &c, StreamT &os, UnaryFunctor each_fn)
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
iterator_range< df_iterator< VPBlockShallowTraversalWrapper< VPBlockBase * > > > vp_depth_first_shallow(VPBlockBase *G)
Returns an iterator range to traverse the graph starting at G in depth-first order.
Instruction * propagateMetadata(Instruction *I, ArrayRef< Value * > VL)
Specifically, let Kinds = [MD_tbaa, MD_alias_scope, MD_noalias, MD_fpmath, MD_nontemporal,...
cl::opt< bool > EnableFSDiscriminator
Printable print(const GCNRegPressure &RP, const GCNSubtarget *ST=nullptr)
cl::opt< bool > EnableVPlanNativePath("enable-vplan-native-path", cl::Hidden, cl::desc("Enable VPlan-native vectorization path with " "support for outer loop vectorization."))
std::unique_ptr< VPlan > VPlanPtr
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
SmallVector< ValueTypeFromRangeType< R >, Size > to_vector(R &&Range)
Given a range of type R, iterate the entire range and return a SmallVector with elements of the vecto...
raw_ostream & operator<<(raw_ostream &OS, const APFixedPoint &FX)
BasicBlock * SplitBlock(BasicBlock *Old, Instruction *SplitPt, DominatorTree *DT, LoopInfo *LI=nullptr, MemorySSAUpdater *MSSAU=nullptr, const Twine &BBName="", bool Before=false)
Split the specified block at the specified instruction.
This struct is a compact representation of a valid (non-zero power of two) alignment.
VPIteration represents a single point in the iteration space of the output (vectorized and/or unrolle...
void print(raw_ostream &O) const