Go to the documentation of this file.
22 #define DEBUG_TYPE "loop-vectorize"
35 for (
const auto *Block : VPBlockVec) {
36 if (VPBlockSet.
count(Block))
50 assert(VPB->getParent() ==
Region &&
"VPBlockBase has wrong parent");
52 auto *VPBB = dyn_cast<VPBasicBlock>(VPB);
54 if (VPB->getNumSuccessors() > 1 || (VPBB && VPBB->isExiting()))
55 assert(VPBB && VPBB->getTerminator() &&
56 "Block has multiple successors but doesn't "
57 "have a proper branch recipe!");
59 assert((!VPBB || !VPBB->getTerminator()) &&
"Unexpected branch recipe!");
62 const auto &Successors = VPB->getSuccessors();
66 "Multiple instances of the same successor.");
70 const auto &SuccPreds = Succ->getPredecessors();
76 const auto &Predecessors = VPB->getPredecessors();
81 "Multiple instances of the same predecessor.");
85 assert(Pred->getParent() == VPB->getParent() &&
86 "Predecessor is not in the same region.");
89 const auto &PredSuccs = Pred->getSuccessors();
103 assert(!Entry->getNumPredecessors() &&
"Region entry has predecessors.");
105 "Region exiting block has successors.");
121 if (
const auto *SubRegion = dyn_cast<VPRegionBlock>(VPB))
132 assert(!TopRegion->
getParent() &&
"VPlan Top Region should have no parent.");
140 auto RecipeI = VPBB->
begin();
141 auto End = VPBB->
end();
142 unsigned NumActiveLaneMaskPhiRecipes = 0;
144 bool IsHeaderVPBB = ParentR && !ParentR->
isReplicator() &&
146 while (RecipeI != End && RecipeI->isPhi()) {
147 if (isa<VPActiveLaneMaskPHIRecipe>(RecipeI))
148 NumActiveLaneMaskPhiRecipes++;
150 if (IsHeaderVPBB && !isa<VPHeaderPHIRecipe>(*RecipeI)) {
151 errs() <<
"Found non-header PHI recipe in header VPBB";
152 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
159 if (!IsHeaderVPBB && isa<VPHeaderPHIRecipe>(*RecipeI)) {
160 errs() <<
"Found header PHI recipe in non-header VPBB";
161 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
171 if (NumActiveLaneMaskPhiRecipes > 1) {
172 errs() <<
"There should be no more than one VPActiveLaneMaskPHIRecipe";
176 while (RecipeI != End) {
177 if (RecipeI->isPhi() && !isa<VPBlendRecipe>(&*RecipeI)) {
178 errs() <<
"Found phi-like recipe after non-phi recipe";
180 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
184 std::prev(RecipeI)->dump();
203 RecipeNumbering[&R] = Cnt++;
206 for (
const VPValue *V : R.definedValues()) {
207 for (
const VPUser *U : V->users()) {
208 auto *UI = dyn_cast<VPRecipeBase>(U);
210 if (!UI || isa<VPHeaderPHIRecipe>(UI) || isa<VPPredInstPHIRecipe>(UI))
215 if (UI->getParent() == VPBB) {
216 if (RecipeNumbering[UI] < RecipeNumbering[&R]) {
217 errs() <<
"Use before def!\n";
223 if (!VPDT.
dominates(VPBB, UI->getParent())) {
224 errs() <<
"Use before def!\n";
239 VPBlockUtils::blocksOnly<const VPBasicBlock>(Iter)) {
247 errs() <<
"VPlan entry block is not a VPBasicBlock\n";
251 if (!isa<VPCanonicalIVPHIRecipe>(&*Entry->begin())) {
252 errs() <<
"VPlan vector loop header does not start with a "
253 "VPCanonicalIVPHIRecipe\n";
259 errs() <<
"VPlan exiting block is not a VPBasicBlock\n";
263 if (Exiting->
empty()) {
264 errs() <<
"VPlan vector loop exiting block must end with BranchOnCount or "
265 "BranchOnCond VPInstruction but is empty\n";
269 auto *LastInst = dyn_cast<VPInstruction>(std::prev(Exiting->
end()));
272 errs() <<
"VPlan vector loop exit must end with BranchOnCount or "
273 "BranchOnCond VPInstruction\n";
278 VPBlockUtils::blocksOnly<const VPRegionBlock>(
281 errs() <<
"region entry block has predecessors\n";
284 if (
Region->getExiting()->getNumSuccessors() != 0) {
285 errs() <<
"region exiting block has successors\n";
291 if (KV.second->getNumOperands() != 1) {
292 errs() <<
"live outs must have a single operand\n";
This is an optimization pass for GlobalISel generic memory operations.
static bool verifyPlanIsValid(const VPlan &Plan)
Verify invariants for general VPlans.
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
static bool verifyVPBasicBlock(const VPBasicBlock *VPBB, VPDominatorTree &VPDT)
static void verifyRegion(const VPRegionBlock *Region)
Verify the CFG invariants of VPRegionBlock Region and its nested VPBlockBases.
VPBasicBlock serves as the leaf of the Hierarchical Control-Flow Graph.
static void verifyBlocksInRegion(const VPRegionBlock *Region)
Helper function that verifies the CFG invariants of the VPBlockBases within Region.
Implements a dense probed hash-table based set with some number of buckets stored inline.
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
const VPBlockBase * getEntry() const
std::pair< iterator, bool > insert(const ValueT &V)
size_type count(const_arg_type_t< ValueT > V) const
Return 1 if the specified key is in the set, 0 otherwise.
static bool verifyPhiRecipes(const VPBasicBlock *VPBB)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
VPBlockBase is the building block of the Hierarchical Control-Flow Graph.
iterator_range< df_iterator< VPBlockShallowTraversalWrapper< VPBlockBase * > > > vp_depth_first_shallow(VPBlockBase *G)
Returns an iterator range to traverse the graph starting at G in depth-first order.
iterator_range< df_iterator< VPBlockDeepTraversalWrapper< VPBlockBase * > > > vp_depth_first_deep(VPBlockBase *G)
Returns an iterator range to traverse the graph starting at G in depth-first order while traversing t...
VPRegionBlock represents a collection of VPBasicBlocks and VPRegionBlocks which form a Single-Entry-S...
size_t getNumSuccessors() const
BlockT * getEntry() const
Get the entry BasicBlock of the Region.
const VPBlockBase * getExiting() const
initializer< Ty > init(const Ty &Val)
static cl::opt< bool > EnableHCFGVerifier("vplan-verify-hcfg", cl::init(false), cl::Hidden, cl::desc("Verify VPlan H-CFG."))
bool is_contained(R &&Range, const E &Element)
Wrapper function around std::find to detect if an element exists in a container.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
void recalculate(ParentType &Func)
recalculate - compute a dominator tree for the given function
VPRegionBlock * getVectorLoopRegion()
Returns the VPRegionBlock of the vector loop.
Core dominator tree base class.
bool dominates(const DomTreeNodeBase< NodeT > *A, const DomTreeNodeBase< NodeT > *B) const
dominates - Returns true iff A dominates B.
const VPBasicBlock * getEntryBasicBlock() const
static bool hasDuplicates(const SmallVectorImpl< VPBlockBase * > &VPBlockVec)
Utility function that checks whether VPBlockVec has duplicate VPBlockBases.
VPRegionBlock * getParent()
VPRecipeBase is a base class modeling a sequence of one or more output IR instructions.
This class augments VPValue with operands which provide the inverse def-use edges from VPValue's user...
iterator begin()
Recipe iterator methods.
const MapVector< PHINode *, VPLiveOut * > & getLiveOuts() const
static void verifyRegionRec(const VPRegionBlock *Region)
Verify the CFG invariants of VPRegionBlock Region and its nested VPBlockBases.
void verifyHierarchicalCFG(const VPRegionBlock *TopRegion) const
Verify the invariants of the H-CFG starting from TopRegion.
bool isReplicator() const
An indicator whether this region is to generate multiple replicated instances of output IR correspond...
VPlan models a candidate for vectorization, encoding various decisions take to produce efficient outp...