51#define DEBUG_TYPE "vplan"
53#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
77 : SubclassID(SC), UnderlyingVal(UV), Def(Def) {
79 Def->addDefinedValue(
this);
83 assert(Users.empty() &&
"trying to delete a VPValue with remaining users");
85 Def->removeDefinedValue(
this);
88#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
105 const VPRecipeBase *Instr = dyn_cast_or_null<VPRecipeBase>(
this);
114 return cast_or_null<VPRecipeBase>(
Def);
118 return cast_or_null<VPRecipeBase>(
Def);
126 while ((Next = Next->getParent()))
132 for (
unsigned i = 0; i < WorkList.
size(); i++) {
133 T *Current = WorkList[i];
134 if (Current->getNumPredecessors() == 0)
136 auto &Predecessors = Current->getPredecessors();
137 WorkList.
insert(Predecessors.begin(), Predecessors.end());
152 return cast<VPBasicBlock>(
Block);
159 return cast<VPBasicBlock>(
Block);
164 "Can only set plan on its entry block.");
173 return cast<VPBasicBlock>(
Block);
180 return cast<VPBasicBlock>(
Block);
184 if (!Successors.empty() || !Parent)
187 "Block w/o successors not the exiting block of its parent.");
192 if (!Predecessors.empty() || !Parent)
195 "Block w/o predecessors not the entry of its parent.");
206 while (It !=
end() && It->isPhi())
212 if (!Def->hasDefiningRecipe())
213 return Def->getLiveInIRValue();
222 if (!VecPart->getType()->isVectorTy()) {
223 assert(
Instance.Lane.isFirstLane() &&
"cannot get lane > 0 for scalar");
233 VPRegionBlock *LoopRegion = R->getParent()->getEnclosingLoopRegion();
241 if (
LVer && (isa<LoadInst>(Orig) || isa<StoreInst>(Orig)))
242 LVer->annotateInstWithNoAlias(To, Orig);
251 for (
Value *V : To) {
258 const Instruction *Inst = dyn_cast<Instruction>(V);
276 << DIL->getFilename() <<
" Line: " << DIL->getLine());
291 for (
VPBlockBase *PredVPBlock : getHierarchicalPredecessors()) {
296 assert(PredBB &&
"Predecessor basic-block not found building successor.");
300 auto *TermBr = dyn_cast<BranchInst>(PredBBTerminator);
301 if (isa<UnreachableInst>(PredBBTerminator)) {
302 assert(PredVPSuccessors.size() == 1 &&
303 "Predecessor ending w/o branch must have single successor.");
304 DebugLoc DL = PredBBTerminator->getDebugLoc();
308 }
else if (TermBr && !TermBr->isConditional()) {
309 TermBr->setSuccessor(0, NewBB);
313 unsigned idx = PredVPSuccessors.front() ==
this ? 0 : 1;
314 assert(!TermBr->getSuccessor(idx) &&
315 "Trying to reset an existing successor block.");
316 TermBr->setSuccessor(idx, NewBB);
329 auto *R = dyn_cast<VPRegionBlock>(BB);
330 return R && !R->isReplicator();
334 if (getPlan()->getVectorLoopRegion()->getSingleSuccessor() ==
this) {
340 VPBlockBase *PredVPB = getSingleHierarchicalPredecessor();
343 "predecessor must have the current block as only successor");
347 cast<BranchInst>(ExitingBB->
getTerminator())->setSuccessor(0, NewBB);
348 }
else if (PrevVPBB &&
349 !((SingleHPred = getSingleHierarchicalPredecessor()) &&
352 (SingleHPred->
getParent() == getEnclosingLoopRegion() &&
353 !IsLoopRegion(SingleHPred))) &&
354 !(Replica && getPredecessors().empty())) {
364 NewBB = createEmptyBasicBlock(State->
CFG);
378 <<
" in BB:" << NewBB->
getName() <<
'\n');
391 for (
auto *Def : R.definedValues())
392 Def->replaceAllUsesWith(NewValue);
394 for (
unsigned I = 0,
E = R.getNumOperands();
I !=
E;
I++)
395 R.setOperand(
I, NewValue);
400 assert((SplitAt == end() || SplitAt->getParent() ==
this) &&
401 "can only split at a position in the same block");
426 if (
P &&
P->isReplicator()) {
428 assert(!cast<VPRegionBlock>(
P)->isReplicator() &&
429 "unexpected nested replicate regions");
438 "block with multiple successors doesn't have a recipe as terminator");
443 auto *VPI = dyn_cast<VPInstruction>(R);
445 isa<VPBranchOnMaskRecipe>(R) ||
452 "conditional branch recipe");
459 "block with 0 or 1 successors terminated by conditional branch recipe");
476 return getParent()->getExitingBasicBlock() ==
this;
479#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
481 if (getSuccessors().empty()) {
482 O << Indent <<
"No successors\n";
484 O << Indent <<
"Successor(s): ";
486 for (
auto *Succ : getSuccessors())
487 O << LS << Succ->getName();
494 O << Indent <<
getName() <<
":\n";
496 auto RecipeIndent = Indent +
" ";
502 printSuccessors(O, Indent);
510 Block->dropAllReferences(NewValue);
517 if (!isReplicator()) {
534 Block->execute(State);
541 assert(!State->
Instance &&
"Replicating a Region with non-null instance.");
546 for (
unsigned Part = 0,
UF = State->
UF; Part <
UF; ++Part) {
555 Block->execute(State);
564#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
567 O << Indent << (isReplicator() ?
"<xVFxUF> " :
"<x1> ") <<
getName() <<
": {";
568 auto NewIndent = Indent +
" ";
573 O << Indent <<
"}\n";
575 printSuccessors(O, Indent);
585 Block->dropAllReferences(&DummyValue);
589 for (
VPValue *VPV : VPValuesToFree)
593 if (BackedgeTakenCount)
594 delete BackedgeTakenCount;
595 for (
auto &
P : VPExternalDefs)
600 VPBasicBlock *Header = getVectorLoopRegion()->getEntryBasicBlock();
602 if (isa<VPActiveLaneMaskPHIRecipe>(&R))
603 return cast<VPActiveLaneMaskPHIRecipe>(&R);
609 Value *CanonicalIVStartValue,
611 bool IsEpilogueVectorization) {
614 if (TripCount && TripCount->getNumUsers()) {
615 for (
unsigned Part = 0,
UF = State.
UF; Part <
UF; ++Part)
616 State.
set(TripCount, TripCountV, Part);
620 if (BackedgeTakenCount && BackedgeTakenCount->getNumUsers()) {
624 "trip.count.minus.1");
628 for (
unsigned Part = 0,
UF = State.
UF; Part <
UF; ++Part)
629 State.
set(BackedgeTakenCount, VTCMO, Part);
632 for (
unsigned Part = 0,
UF = State.
UF; Part <
UF; ++Part)
633 State.
set(&VectorTripCount, VectorTripCountV, Part);
638 if (CanonicalIVStartValue) {
639 VPValue *VPV = getOrAddExternalDef(CanonicalIVStartValue);
640 auto *
IV = getCanonicalIV();
643 if (isa<VPScalarIVStepsRecipe>(U) ||
644 isa<VPDerivedIVRecipe>(U))
646 auto *VPI = cast<VPInstruction>(U);
647 return VPI->getOpcode() ==
648 VPInstruction::CanonicalIVIncrement ||
650 VPInstruction::CanonicalIVIncrementNUW;
652 "the canonical IV should only be used by its increments or "
653 "ScalarIVSteps when "
654 "resetting the start value");
655 IV->setOperand(0, VPV);
664 for (
auto &Entry : Value2VPValue)
675 Block->execute(State);
682 VPBasicBlock *Header = getVectorLoopRegion()->getEntryBasicBlock();
685 if (isa<VPWidenPHIRecipe>(&R))
688 if (isa<VPWidenPointerInductionRecipe>(&R) ||
689 isa<VPWidenIntOrFpInductionRecipe>(&R)) {
691 if (isa<VPWidenIntOrFpInductionRecipe>(&R)) {
692 Phi = cast<PHINode>(State->
get(R.getVPSingleValue(), 0));
694 auto *WidenPhi = cast<VPWidenPointerInductionRecipe>(&R);
697 if (WidenPhi->onlyScalarsGenerated(State->
VF))
700 auto *
GEP = cast<GetElementPtrInst>(State->
get(WidenPhi, 0));
701 Phi = cast<PHINode>(
GEP->getPointerOperand());
713 auto *PhiR = cast<VPHeaderPHIRecipe>(&R);
718 bool SinglePartNeeded = isa<VPCanonicalIVPHIRecipe>(PhiR) ||
719 isa<VPFirstOrderRecurrencePHIRecipe>(PhiR) ||
720 (isa<VPReductionPHIRecipe>(PhiR) &&
721 cast<VPReductionPHIRecipe>(PhiR)->isOrdered());
722 unsigned LastPartForNewPhi = SinglePartNeeded ? 1 : State->
UF;
724 for (
unsigned Part = 0; Part < LastPartForNewPhi; ++Part) {
725 Value *Phi = State->
get(PhiR, Part);
726 Value *Val = State->
get(PhiR->getBackedgeValue(),
727 SinglePartNeeded ? State->
UF - 1 : Part);
728 cast<PHINode>(Phi)->addIncoming(Val, VectorLatchBB);
736 updateDominatorTree(State->
DT, VectorHeaderBB, VectorLatchBB,
741#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
746 O <<
"VPlan '" <<
getName() <<
"' {";
748 if (VectorTripCount.getNumUsers() > 0) {
751 O <<
" = vector-trip-count\n";
754 if (BackedgeTakenCount && BackedgeTakenCount->getNumUsers()) {
756 BackedgeTakenCount->printAsOperand(O,
SlotTracker);
757 O <<
" = backedge-taken count\n";
765 if (!LiveOuts.empty())
767 for (
const auto &KV : LiveOuts) {
769 KV.second->getPhi()->printAsOperand(O);
771 KV.second->getOperand(0)->printAsOperand(O,
SlotTracker);
781 RSO <<
Name <<
" for ";
783 RSO <<
"VF={" << VFs[0];
792 RSO <<
"UF={" << UFs[0];
812 assert(LiveOuts.count(PN) == 0 &&
"an exit value for PN already exists");
813 LiveOuts.insert({PN,
new VPLiveOut(PN, V)});
823 for (
auto *BB = LoopHeaderBB; BB != LoopLatchBB; BB = PostDomSucc) {
826 assert(Succs.size() <= 2 &&
827 "Basic block in vector loop has more than 2 successors.");
828 PostDomSucc = Succs[0];
829 if (Succs.size() == 1) {
831 "PostDom successor has more than one predecessor.");
837 PostDomSucc = Succs[1];
838 InterimSucc = Succs[0];
841 "One successor of a basic block does not lead to the other.");
843 "Interim successor has more than one predecessor.");
845 "PostDom successor has more than two predecessors.");
854#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
857 return (isa<VPRegionBlock>(
Block) ?
"cluster_N" :
"N") +
862 const std::string &
Name =
Block->getName();
871 OS <<
"digraph VPlan {\n";
872 OS <<
"graph [labelloc=t, fontsize=30; label=\"Vectorization Plan";
875 if (
Plan.BackedgeTakenCount) {
878 OS <<
" := BackedgeTakenCount";
881 OS <<
"node [shape=rect, fontname=Courier, fontsize=30]\n";
882 OS <<
"edge [fontname=Courier, fontsize=30]\n";
883 OS <<
"compound=true\n";
901 bool Hidden,
const Twine &Label) {
906 OS << Indent << getUID(
Tail) <<
" -> " << getUID(Head);
907 OS <<
" [ label=\"" << Label <<
'\"';
909 OS <<
" ltail=" << getUID(
From);
911 OS <<
" lhead=" << getUID(To);
913 OS <<
"; splines=none";
918 auto &Successors =
Block->getSuccessors();
919 if (Successors.size() == 1)
920 drawEdge(
Block, Successors.front(),
false,
"");
921 else if (Successors.size() == 2) {
922 drawEdge(
Block, Successors.front(),
false,
"T");
923 drawEdge(
Block, Successors.back(),
false,
"F");
925 unsigned SuccessorNumber = 0;
934 OS << Indent << getUID(
BasicBlock) <<
" [label =\n";
952 EmitLine(Line,
" +\n");
953 EmitLine(
Lines.back(),
"\n");
956 OS << Indent <<
"]\n";
962 OS << Indent <<
"subgraph " << getUID(
Region) <<
" {\n";
964 OS << Indent <<
"fontname=Courier\n"
965 << Indent <<
"label=\""
973 OS << Indent <<
"}\n";
978 if (
auto *Inst = dyn_cast<Instruction>(V)) {
979 if (!Inst->getType()->isVoidTy()) {
980 Inst->printAsOperand(O,
false);
983 O << Inst->getOpcodeName() <<
" ";
984 unsigned E = Inst->getNumOperands();
986 Inst->getOperand(0)->printAsOperand(O,
false);
987 for (
unsigned I = 1;
I <
E; ++
I)
988 Inst->getOperand(
I)->printAsOperand(O <<
", ",
false);
991 V->printAsOperand(O,
false);
999 for (
unsigned J = 0; J < getNumUsers();) {
1001 unsigned NumUsers = getNumUsers();
1008 if (NumUsers == getNumUsers())
1013#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1015 if (
const Value *UV = getUnderlyingValue()) {
1017 UV->printAsOperand(OS,
false);
1022 unsigned Slot = Tracker.
getSlot(
this);
1023 if (Slot ==
unsigned(-1))
1026 OS <<
"vp<%" << Tracker.
getSlot(
this) <<
">";
1042 visitBlock(
Base, Old2New, IAI);
1046void VPInterleavedAccessInfo::visitBlock(
VPBlockBase *
Block, Old2NewTy &Old2New,
1050 if (isa<VPHeaderPHIRecipe>(&VPI))
1052 assert(isa<VPInstruction>(&VPI) &&
"Can only handle VPInstructions");
1053 auto *VPInst = cast<VPInstruction>(&VPI);
1055 auto *Inst = dyn_cast_or_null<Instruction>(VPInst->getUnderlyingValue());
1062 auto NewIGIter = Old2New.find(IG);
1063 if (NewIGIter == Old2New.end())
1065 IG->getFactor(), IG->isReverse(), IG->getAlign());
1067 if (Inst == IG->getInsertPos())
1068 Old2New[IG]->setInsertPos(VPInst);
1070 InterleaveGroupMap[VPInst] = Old2New[IG];
1071 InterleaveGroupMap[VPInst]->insertMember(
1072 VPInst, IG->getIndex(Inst),
1073 Align(IG->isReverse() ? (-1) *
int(IG->getFactor())
1074 : IG->getFactor()));
1077 visitRegion(
Region, Old2New, IAI);
1088void VPSlotTracker::assignSlot(
const VPValue *V) {
1089 assert(Slots.find(V) == Slots.end() &&
"VPValue already has a slot!");
1090 Slots[V] = NextSlot++;
1093void VPSlotTracker::assignSlots(
const VPlan &
Plan) {
1095 for (
const auto &
P :
Plan.VPExternalDefs)
1096 assignSlot(
P.second);
1098 assignSlot(&
Plan.VectorTripCount);
1099 if (
Plan.BackedgeTakenCount)
1100 assignSlot(
Plan.BackedgeTakenCount);
1105 VPBlockUtils::blocksOnly<const VPBasicBlock>(RPOT))
1107 for (
VPValue *Def : Recipe.definedValues())
1112 return all_of(Def->users(),
1113 [Def](
VPUser *U) { return U->onlyFirstLaneUsed(Def); });
1118 if (
auto *
E = dyn_cast<SCEVConstant>(Expr))
1120 if (
auto *
E = dyn_cast<SCEVUnknown>(Expr))
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static const Function * getParent(const Value *V)
BlockVerifier::State From
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
#define LLVM_DUMP_METHOD
Mark debug helper function definitions like dump() that should not be stripped from debug builds.
This file builds on the ADT/GraphTraits.h file to build generic depth first graph iterator.
static void dumpEdges(CFGMST< Edge, BBInfo > &MST, GCOVFunction &GF)
Generic dominator tree construction - this file provides routines to construct immediate dominator in...
This file provides various utilities for inspecting and working with the control flow graph in LLVM I...
iv Induction Variable Users
cl::opt< bool > EnableVPlanNativePath("enable-vplan-native-path", cl::init(false), cl::Hidden, cl::desc("Enable VPlan-native vectorization path with " "support for outer loop vectorization."))
Memory true print Memory SSA Printer
This file builds on the ADT/GraphTraits.h file to build a generic graph post order iterator.
static StringRef getName(Value *V)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the SmallVector class.
This file implements dominator tree analysis for a single level of a VPlan's H-CFG.
static T * getPlanEntry(T *Start)
cl::opt< bool > EnableVPlanNativePath
static bool hasConditionalTerminator(const VPBasicBlock *VPBB)
This file contains the declarations of the Vectorization Plan base classes:
static bool IsCondBranch(unsigned BrOpc)
static const uint32_t IV[8]
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
LLVM Basic Block Representation.
void print(raw_ostream &OS, AssemblyAnnotationWriter *AAW=nullptr, bool ShouldPreserveUseListOrder=false, bool IsForDebug=false) const
Print the basic block to an output stream with an optional AssemblyAnnotationWriter.
static BasicBlock * Create(LLVMContext &Context, const Twine &Name="", Function *Parent=nullptr, BasicBlock *InsertBefore=nullptr)
Creates a new BasicBlock.
bool hasNPredecessors(unsigned N) const
Return true if this block has exactly N predecessors.
const BasicBlock * getSinglePredecessor() const
Return the predecessor of this block if it has a single predecessor block.
const BasicBlock * getSingleSuccessor() const
Return the successor of this block if it has a single successor.
const Function * getParent() const
Return the enclosing method, or null if none.
LLVMContext & getContext() const
Get the context in which this basic block lives.
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
static BranchInst * Create(BasicBlock *IfTrue, Instruction *InsertBefore=nullptr)
static Constant * get(Type *Ty, uint64_t V, bool IsSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
std::optional< const DILocation * > cloneByMultiplyingDuplicationFactor(unsigned DF) const
Returns a new DILocation with duplication factor DF * current duplication factor encoded in the discr...
Core dominator tree base class.
bool verify(VerificationLevel VL=VerificationLevel::Full) const
verify - checks if the tree is correct.
void changeImmediateDominator(DomTreeNodeBase< NodeT > *N, DomTreeNodeBase< NodeT > *NewIDom)
changeImmediateDominator - This method is used to update the dominator tree information when a node's...
DomTreeNodeBase< NodeT > * addNewBlock(NodeT *BB, NodeT *DomBB)
Add a new node to the dominator tree information.
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
constexpr bool isScalar() const
Exactly one element.
bool shouldEmitDebugInfoForProfiling() const
Returns true if we should emit debug info for profiling.
Common base class shared among various IRBuilders.
Value * CreateExtractElement(Value *Vec, Value *Idx, const Twine &Name="")
UnreachableInst * CreateUnreachable()
Value * CreateVectorSplat(unsigned NumElts, Value *V, const Twine &Name="")
Return a vector value that contains.
void SetCurrentDebugLocation(DebugLoc L)
Set location information used by debugging information.
Value * CreateSub(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
void SetInsertPoint(BasicBlock *TheBB)
This specifies that created instructions should be appended to the end of the specified block.
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
const Function * getFunction() const
Return the function this instruction belongs to.
void moveBefore(Instruction *MovePos)
Unlink this instruction from its current basic block and insert it into the basic block that MovePos ...
The group of interleaved loads/stores sharing the same stride and close to each other.
Drive the analysis of interleaved memory accesses in the loop.
InterleaveGroup< Instruction > * getInterleaveGroup(const Instruction *Instr) const
Get the interleave group that Instr belongs to.
void addBasicBlockToLoop(BlockT *NewBB, LoopInfoBase< BlockT, LoopT > &LI)
This method is used by other analyses to update loop information.
void addChildLoop(LoopT *NewChild)
Add the specified loop to be a child of this loop.
LoopT * AllocateLoop(ArgsTy &&... Args)
void addTopLevelLoop(LoopT *New)
This adds the specified loop to the collection of top-level loops.
LoopT * getLoopFor(const BlockT *BB) const
Return the inner most loop that BB lives in.
Represents a single loop in the control flow graph.
void eraseFromParent()
This method unlinks 'this' from the containing function and deletes it.
void setIncomingBlock(unsigned i, BasicBlock *BB)
Value * getIncomingValue(unsigned i) const
Return incoming value number x.
BlockT * getEntry() const
Get the entry BasicBlock of the Region.
This class represents an analyzed expression in the program.
The main scalar evolution driver.
size_type size() const
Determine the number of elements in the SetVector.
bool insert(const value_type &X)
Insert a new element into the SetVector.
This class provides computation of slot numbers for LLVM Assembly writing.
A SetVector that performs no allocations if smaller than a certain size.
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StringRef - Represent a constant reference to a string, i.e.
std::pair< StringRef, StringRef > split(char Separator) const
Split into two substrings around the first occurrence of a separator character.
StringRef rtrim(char Char) const
Return string with consecutive Char characters starting from the right removed.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
This function has undefined behavior.
void setOperand(unsigned i, Value *Val)
Value * getOperand(unsigned i) const
unsigned getNumOperands() const
A recipe for generating the active lane mask for the vector loop that is used to predicate the vector...
VPBasicBlock serves as the leaf of the Hierarchical Control-Flow Graph.
void appendRecipe(VPRecipeBase *Recipe)
Augment the existing recipes of a VPBasicBlock with an additional Recipe as the last recipe.
RecipeListTy::iterator iterator
Instruction iterators...
void execute(VPTransformState *State) override
The method which generates the output IR instructions that correspond to this VPBasicBlock,...
iterator begin()
Recipe iterator methods.
iterator getFirstNonPhi()
Return the position of the first non-phi node recipe in the block.
VPRegionBlock * getEnclosingLoopRegion()
void dropAllReferences(VPValue *NewValue) override
Replace all operands of VPUsers in the block with NewValue and also replaces all uses of VPValues def...
VPBasicBlock * splitAt(iterator SplitAt)
Split current block at SplitAt by inserting a new block between the current block and its successors ...
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print this VPBsicBlock to O, prefixing all lines with Indent.
bool isExiting() const
Returns true if the block is exiting it's parent region.
VPRecipeBase * getTerminator()
If the block has multiple successors, return the branch recipe terminating the block.
const VPRecipeBase & back() const
VPBlockBase is the building block of the Hierarchical Control-Flow Graph.
VPRegionBlock * getParent()
const VPBasicBlock * getExitingBasicBlock() const
size_t getNumSuccessors() const
void printSuccessors(raw_ostream &O, const Twine &Indent) const
Print the successors of this block to O, prefixing all lines with Indent.
VPBlockBase * getEnclosingBlockWithPredecessors()
static void deleteCFG(VPBlockBase *Entry)
Delete all blocks reachable from a given VPBlockBase, inclusive.
void setPlan(VPlan *ParentPlan)
Sets the pointer of the plan containing the block.
VPBlockBase * getSingleHierarchicalSuccessor()
const VPBlocksTy & getHierarchicalSuccessors()
VPBlockBase * getEnclosingBlockWithSuccessors()
An Enclosing Block of a block B is any block containing B, including B itself.
const VPBasicBlock * getEntryBasicBlock() const
VPBlockBase * getSingleSuccessor() const
Helper for GraphTraits specialization that traverses through VPRegionBlocks.
static void insertBlockAfter(VPBlockBase *NewBlock, VPBlockBase *BlockPtr)
Insert disconnected VPBlockBase NewBlock after BlockPtr.
static void disconnectBlocks(VPBlockBase *From, VPBlockBase *To)
Disconnect VPBlockBases From and To bi-directionally.
static void connectBlocks(VPBlockBase *From, VPBlockBase *To)
Connect VPBlockBases From and To bi-directionally.
This class augments a recipe with a set of VPValues defined by the recipe.
void dump() const
Dump the VPDef to stderr (for debugging).
virtual void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const =0
Each concrete VPDef prints itself.
Recipe to expand a SCEV expression.
This is a concrete Recipe that models a single VPlan-level instruction.
VPInterleavedAccessInfo(VPlan &Plan, InterleavedAccessInfo &IAI)
In what follows, the term "input IR" refers to code that is fed into the vectorizer whereas the term ...
Value * getAsRuntimeExpr(IRBuilderBase &Builder, const ElementCount &VF) const
Returns an expression describing the lane index that can be used at runtime.
@ ScalableLast
For ScalableLast, Lane is the offset from the start of the last N-element subvector in a scalable vec...
@ First
For First, Lane is the index into the first N elements of a fixed-vector <N x <ElTy>> or a scalable v...
A value that is used outside the VPlan.
VPRecipeBase is a base class modeling a sequence of one or more output IR instructions.
VPBasicBlock * getParent()
VPRegionBlock represents a collection of VPBasicBlocks and VPRegionBlocks which form a Single-Entry-S...
const VPBlockBase * getEntry() const
void dropAllReferences(VPValue *NewValue) override
Replace all operands of VPUsers in the block with NewValue and also replaces all uses of VPValues def...
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print this VPRegionBlock to O (recursively), prefixing all lines with Indent.
void execute(VPTransformState *State) override
The method which generates the output IR instructions that correspond to this VPRegionBlock,...
const VPBlockBase * getExiting() const
VPBasicBlock * getPreheaderVPBB()
Returns the pre-header VPBasicBlock of the loop region.
This class can be used to assign consecutive numbers to all VPValues in a VPlan and allows querying t...
unsigned getSlot(const VPValue *V) const
This class augments VPValue with operands which provide the inverse def-use edges from VPValue's user...
void printOperands(raw_ostream &O, VPSlotTracker &SlotTracker) const
Print the operands to O.
VPRecipeBase * getDefiningRecipe()
Returns the recipe defining this VPValue or nullptr if it is not defined by a recipe,...
void printAsOperand(raw_ostream &OS, VPSlotTracker &Tracker) const
void dump() const
Dump the value to stderr (for debugging).
VPValue(const unsigned char SC, Value *UV=nullptr, VPDef *Def=nullptr)
void print(raw_ostream &OS, VPSlotTracker &Tracker) const
void replaceAllUsesWith(VPValue *New)
VPDef * Def
Pointer to the VPDef that defines this VPValue.
VPlanPrinter prints a given VPlan to a given output stream.
LLVM_DUMP_METHOD void dump()
VPlan models a candidate for vectorization, encoding various decisions take to produce efficient outp...
void printDOT(raw_ostream &O) const
Print this VPlan in DOT format to O.
std::string getName() const
Return a string with the name of the plan and the applicable VFs and UFs.
void addLiveOut(PHINode *PN, VPValue *V)
void prepareToExecute(Value *TripCount, Value *VectorTripCount, Value *CanonicalIVStartValue, VPTransformState &State, bool IsEpilogueVectorization)
Prepare the plan for execution, setting up the required live-in values.
VPValue * getOrAddExternalDef(Value *V)
Get the existing or add a new external definition for V.
VPRegionBlock * getVectorLoopRegion()
Returns the VPRegionBlock of the vector loop.
LLVM_DUMP_METHOD void dump() const
Dump the plan to stderr (for debugging).
void execute(VPTransformState *State)
Generate the IR code for this VPlan.
VPActiveLaneMaskPHIRecipe * getActiveLaneMaskPhi()
Find and return the VPActiveLaneMaskPHIRecipe from the header - there be only one at most.
void print(raw_ostream &O) const
Print this VPlan to O.
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
StringRef getName() const
Return a constant reference to the value's name.
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
This class implements an extremely fast bulk output stream that can only output to a stream.
A raw_ostream that writes to an std::string.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ Tail
Attemps to make calls as fast as possible while guaranteeing that tail call optimization can always b...
std::string EscapeString(const std::string &Label)
VPValue * getOrCreateVPValueForSCEVExpr(VPlan &Plan, const SCEV *Expr, ScalarEvolution &SE)
Get or create a VPValue that corresponds to the expansion of Expr.
bool onlyFirstLaneUsed(VPValue *Def)
Returns true if only the first lane of Def is used.
This is an optimization pass for GlobalISel generic memory operations.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
Interval::succ_iterator succ_end(Interval *I)
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
auto successors(const MachineBasicBlock *BB)
Value * getRuntimeVF(IRBuilderBase &B, Type *Ty, ElementCount VF)
Return the runtime value for VF.
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
Interval::succ_iterator succ_begin(Interval *I)
succ_begin/succ_end - define methods so that Intervals may be used just like BasicBlocks can with the...
void interleaveComma(const Container &c, StreamT &os, UnaryFunctor each_fn)
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
iterator_range< df_iterator< VPBlockShallowTraversalWrapper< VPBlockBase * > > > vp_depth_first_shallow(VPBlockBase *G)
Returns an iterator range to traverse the graph starting at G in depth-first order.
Instruction * propagateMetadata(Instruction *I, ArrayRef< Value * > VL)
Specifically, let Kinds = [MD_tbaa, MD_alias_scope, MD_noalias, MD_fpmath, MD_nontemporal,...
cl::opt< bool > EnableFSDiscriminator
Printable print(const GCNRegPressure &RP, const GCNSubtarget *ST=nullptr)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
SmallVector< ValueTypeFromRangeType< R >, Size > to_vector(R &&Range)
Given a range of type R, iterate the entire range and return a SmallVector with elements of the vecto...
raw_ostream & operator<<(raw_ostream &OS, const APFixedPoint &FX)
BasicBlock * SplitBlock(BasicBlock *Old, Instruction *SplitPt, DominatorTree *DT, LoopInfo *LI=nullptr, MemorySSAUpdater *MSSAU=nullptr, const Twine &BBName="", bool Before=false)
Split the specified block at the specified instruction.
This struct is a compact representation of a valid (non-zero power of two) alignment.
VPIteration represents a single point in the iteration space of the output (vectorized and/or unrolle...
void print(raw_ostream &O) const