22#define DEBUG_TYPE "loop-vectorize"
35 for (
const auto *
Block : VPBlockVec) {
50 assert(VPB->getParent() ==
Region &&
"VPBlockBase has wrong parent");
52 auto *VPBB = dyn_cast<VPBasicBlock>(VPB);
54 if (VPB->getNumSuccessors() > 1 || (VPBB && VPBB->isExiting()))
55 assert(VPBB && VPBB->getTerminator() &&
56 "Block has multiple successors but doesn't "
57 "have a proper branch recipe!");
59 assert((!VPBB || !VPBB->getTerminator()) &&
"Unexpected branch recipe!");
62 const auto &Successors = VPB->getSuccessors();
66 "Multiple instances of the same successor.");
70 const auto &SuccPreds = Succ->getPredecessors();
76 const auto &Predecessors = VPB->getPredecessors();
81 "Multiple instances of the same predecessor.");
85 assert(Pred->getParent() == VPB->getParent() &&
86 "Predecessor is not in the same region.");
89 const auto &PredSuccs = Pred->getSuccessors();
103 assert(!Entry->getNumPredecessors() &&
"Region entry has predecessors.");
105 "Region exiting block has successors.");
121 if (
const auto *SubRegion = dyn_cast<VPRegionBlock>(VPB))
132 assert(!TopRegion->
getParent() &&
"VPlan Top Region should have no parent.");
140 auto RecipeI = VPBB->
begin();
142 unsigned NumActiveLaneMaskPhiRecipes = 0;
144 bool IsHeaderVPBB = ParentR && !ParentR->
isReplicator() &&
146 while (RecipeI !=
End && RecipeI->isPhi()) {
147 if (isa<VPActiveLaneMaskPHIRecipe>(RecipeI))
148 NumActiveLaneMaskPhiRecipes++;
150 if (IsHeaderVPBB && !isa<VPHeaderPHIRecipe>(*RecipeI)) {
151 errs() <<
"Found non-header PHI recipe in header VPBB";
152#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
159 if (!IsHeaderVPBB && isa<VPHeaderPHIRecipe>(*RecipeI)) {
160 errs() <<
"Found header PHI recipe in non-header VPBB";
161#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
171 if (NumActiveLaneMaskPhiRecipes > 1) {
172 errs() <<
"There should be no more than one VPActiveLaneMaskPHIRecipe";
176 while (RecipeI !=
End) {
177 if (RecipeI->isPhi() && !isa<VPBlendRecipe>(&*RecipeI)) {
178 errs() <<
"Found phi-like recipe after non-phi recipe";
180#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
184 std::prev(RecipeI)->dump();
203 RecipeNumbering[&R] = Cnt++;
206 for (
const VPValue *V : R.definedValues()) {
207 for (
const VPUser *U : V->users()) {
208 auto *UI = dyn_cast<VPRecipeBase>(U);
210 if (!UI || isa<VPHeaderPHIRecipe>(UI) || isa<VPPredInstPHIRecipe>(UI))
215 if (UI->getParent() == VPBB) {
216 if (RecipeNumbering[UI] < RecipeNumbering[&R]) {
217 errs() <<
"Use before def!\n";
224 errs() <<
"Use before def!\n";
239 VPBlockUtils::blocksOnly<const VPBasicBlock>(Iter)) {
247 errs() <<
"VPlan entry block is not a VPBasicBlock\n";
251 if (!isa<VPCanonicalIVPHIRecipe>(&*Entry->begin())) {
252 errs() <<
"VPlan vector loop header does not start with a "
253 "VPCanonicalIVPHIRecipe\n";
259 errs() <<
"VPlan exiting block is not a VPBasicBlock\n";
263 if (Exiting->
empty()) {
264 errs() <<
"VPlan vector loop exiting block must end with BranchOnCount or "
265 "BranchOnCond VPInstruction but is empty\n";
269 auto *LastInst = dyn_cast<VPInstruction>(std::prev(Exiting->
end()));
272 errs() <<
"VPlan vector loop exit must end with BranchOnCount or "
273 "BranchOnCond VPInstruction\n";
278 VPBlockUtils::blocksOnly<const VPRegionBlock>(
281 errs() <<
"region entry block has predecessors\n";
284 if (
Region->getExiting()->getNumSuccessors() != 0) {
285 errs() <<
"region exiting block has successors\n";
291 if (KV.second->getNumOperands() != 1) {
292 errs() <<
"live outs must have a single operand\n";
This file builds on the ADT/GraphTraits.h file to build generic depth first graph iterator.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file implements dominator tree analysis for a single level of a VPlan's H-CFG.
static void verifyRegionRec(const VPRegionBlock *Region)
Verify the CFG invariants of VPRegionBlock Region and its nested VPBlockBases.
static cl::opt< bool > EnableHCFGVerifier("vplan-verify-hcfg", cl::init(false), cl::Hidden, cl::desc("Verify VPlan H-CFG."))
static bool verifyPhiRecipes(const VPBasicBlock *VPBB)
static bool verifyVPBasicBlock(const VPBasicBlock *VPBB, VPDominatorTree &VPDT)
static void verifyRegion(const VPRegionBlock *Region)
Verify the CFG invariants of VPRegionBlock Region and its nested VPBlockBases.
static bool hasDuplicates(const SmallVectorImpl< VPBlockBase * > &VPBlockVec)
Utility function that checks whether VPBlockVec has duplicate VPBlockBases.
static void verifyBlocksInRegion(const VPRegionBlock *Region)
Helper function that verifies the CFG invariants of the VPBlockBases within Region.
This file declares the class VPlanVerifier, which contains utility functions to check the consistency...
This file contains the declarations of the Vectorization Plan base classes:
Core dominator tree base class.
bool dominates(const DomTreeNodeBase< NodeT > *A, const DomTreeNodeBase< NodeT > *B) const
dominates - Returns true iff A dominates B.
void recalculate(ParentType &Func)
recalculate - compute a dominator tree for the given function
BlockT * getEntry() const
Get the entry BasicBlock of the Region.
Implements a dense probed hash-table based set with some number of buckets stored inline.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
VPBasicBlock serves as the leaf of the Hierarchical Control-Flow Graph.
iterator begin()
Recipe iterator methods.
VPBlockBase is the building block of the Hierarchical Control-Flow Graph.
VPRegionBlock * getParent()
size_t getNumSuccessors() const
const VPBasicBlock * getEntryBasicBlock() const
VPRecipeBase is a base class modeling a sequence of one or more output IR instructions.
VPRegionBlock represents a collection of VPBasicBlocks and VPRegionBlocks which form a Single-Entry-S...
const VPBlockBase * getEntry() const
bool isReplicator() const
An indicator whether this region is to generate multiple replicated instances of output IR correspond...
const VPBlockBase * getExiting() const
This class augments VPValue with operands which provide the inverse def-use edges from VPValue's user...
VPlan models a candidate for vectorization, encoding various decisions take to produce efficient outp...
VPBasicBlock * getEntry()
VPRegionBlock * getVectorLoopRegion()
Returns the VPRegionBlock of the vector loop.
const MapVector< PHINode *, VPLiveOut * > & getLiveOuts() const
std::pair< iterator, bool > insert(const ValueT &V)
size_type count(const_arg_type_t< ValueT > V) const
Return 1 if the specified key is in the set, 0 otherwise.
initializer< Ty > init(const Ty &Val)
This is an optimization pass for GlobalISel generic memory operations.
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
iterator_range< df_iterator< VPBlockShallowTraversalWrapper< VPBlockBase * > > > vp_depth_first_shallow(VPBlockBase *G)
Returns an iterator range to traverse the graph starting at G in depth-first order.
iterator_range< df_iterator< VPBlockDeepTraversalWrapper< VPBlockBase * > > > vp_depth_first_deep(VPBlockBase *G)
Returns an iterator range to traverse the graph starting at G in depth-first order while traversing t...
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
static bool verifyPlanIsValid(const VPlan &Plan)
Verify invariants for general VPlans.
void verifyHierarchicalCFG(const VPRegionBlock *TopRegion) const
Verify the invariants of the H-CFG starting from TopRegion.