22#define DEBUG_TYPE "loop-vectorize"
30 auto RecipeI = VPBB->
begin();
32 unsigned NumActiveLaneMaskPhiRecipes = 0;
34 bool IsHeaderVPBB = ParentR && !ParentR->
isReplicator() &&
36 while (RecipeI !=
End && RecipeI->isPhi()) {
37 if (isa<VPActiveLaneMaskPHIRecipe>(RecipeI))
38 NumActiveLaneMaskPhiRecipes++;
40 if (IsHeaderVPBB && !isa<VPHeaderPHIRecipe, VPWidenPHIRecipe>(*RecipeI)) {
41 errs() <<
"Found non-header PHI recipe in header VPBB";
42#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
49 if (!IsHeaderVPBB && isa<VPHeaderPHIRecipe>(*RecipeI)) {
50 errs() <<
"Found header PHI recipe in non-header VPBB";
51#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
61 if (NumActiveLaneMaskPhiRecipes > 1) {
62 errs() <<
"There should be no more than one VPActiveLaneMaskPHIRecipe";
66 while (RecipeI !=
End) {
67 if (RecipeI->isPhi() && !isa<VPBlendRecipe>(&*RecipeI)) {
68 errs() <<
"Found phi-like recipe after non-phi recipe";
70#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
74 std::prev(RecipeI)->dump();
93 RecipeNumbering[&R] = Cnt++;
96 for (
const VPValue *V : R.definedValues()) {
97 for (
const VPUser *U : V->users()) {
98 auto *UI = dyn_cast<VPRecipeBase>(U);
101 isa<VPHeaderPHIRecipe, VPWidenPHIRecipe, VPPredInstPHIRecipe>(UI))
106 if (UI->getParent() == VPBB) {
107 if (RecipeNumbering[UI] < RecipeNumbering[&R]) {
108 errs() <<
"Use before def!\n";
115 errs() <<
"Use before def!\n";
128 for (
const auto *
Block : VPBlockVec) {
137 auto *VPBB = dyn_cast<VPBasicBlock>(VPB);
140 (VPBB && VPBB->getParent() && VPBB->isExiting() &&
141 !VPBB->getParent()->isReplicator())) {
142 if (!VPBB || !VPBB->getTerminator()) {
143 errs() <<
"Block has multiple successors but doesn't "
144 "have a proper branch recipe!\n";
148 if (VPBB && VPBB->getTerminator()) {
149 errs() <<
"Unexpected branch recipe!\n";
159 errs() <<
"Multiple instances of the same successor.\n";
165 const auto &SuccPreds = Succ->getPredecessors();
167 errs() <<
"Missing predecessor link.\n";
178 errs() <<
"Multiple instances of the same predecessor.\n";
184 if (Pred->getParent() != VPB->
getParent()) {
185 errs() <<
"Predecessor is not in the same region.\n";
190 const auto &PredSuccs = Pred->getSuccessors();
192 errs() <<
"Missing successor link.\n";
206 if (VPB->getParent() !=
Region) {
207 errs() <<
"VPBlockBase has wrong parent\n";
225 if (Entry->getNumPredecessors() != 0) {
226 errs() <<
"region entry block has predecessors\n";
230 errs() <<
"region exiting block has successors\n";
245 const auto *SubRegion = dyn_cast<VPRegionBlock>(VPB);
246 return !SubRegion || verifyRegionRec(SubRegion, VPDT);
256 [&VPDT](
const VPBlockBase *VPB) { return !verifyBlock(VPB, VPDT); }))
264 errs() <<
"VPlan Top Region should have no parent.\n";
270 errs() <<
"VPlan entry block is not a VPBasicBlock\n";
274 if (!isa<VPCanonicalIVPHIRecipe>(&*Entry->begin())) {
275 errs() <<
"VPlan vector loop header does not start with a "
276 "VPCanonicalIVPHIRecipe\n";
282 errs() <<
"VPlan exiting block is not a VPBasicBlock\n";
286 if (Exiting->
empty()) {
287 errs() <<
"VPlan vector loop exiting block must end with BranchOnCount or "
288 "BranchOnCond VPInstruction but is empty\n";
292 auto *LastInst = dyn_cast<VPInstruction>(std::prev(Exiting->
end()));
295 errs() <<
"VPlan vector loop exit must end with BranchOnCount or "
296 "BranchOnCond VPInstruction\n";
301 if (KV.second->getNumOperands() != 1) {
302 errs() <<
"live outs must have a single operand\n";
This file builds on the ADT/GraphTraits.h file to build generic depth first graph iterator.
This file implements dominator tree analysis for a single level of a VPlan's H-CFG.
static bool verifyPhiRecipes(const VPBasicBlock *VPBB)
static bool verifyRegion(const VPRegionBlock *Region, const VPDominatorTree &VPDT)
Verify the CFG invariants of VPRegionBlock Region and its nested VPBlockBases.
static bool verifyRegionRec(const VPRegionBlock *Region, const VPDominatorTree &VPDT)
Verify the CFG invariants of VPRegionBlock Region and its nested VPBlockBases.
static bool verifyVPBasicBlock(const VPBasicBlock *VPBB, const VPDominatorTree &VPDT)
static bool verifyBlock(const VPBlockBase *VPB, const VPDominatorTree &VPDT)
static bool hasDuplicates(const SmallVectorImpl< VPBlockBase * > &VPBlockVec)
Utility function that checks whether VPBlockVec has duplicate VPBlockBases.
static bool verifyBlocksInRegion(const VPRegionBlock *Region, const VPDominatorTree &VPDT)
Helper function that verifies the CFG invariants of the VPBlockBases within Region.
This file declares the class VPlanVerifier, which contains utility functions to check the consistency...
This file contains the declarations of the Vectorization Plan base classes:
Core dominator tree base class.
bool dominates(const DomTreeNodeBase< NodeT > *A, const DomTreeNodeBase< NodeT > *B) const
dominates - Returns true iff A dominates B.
void recalculate(ParentType &Func)
recalculate - compute a dominator tree for the given function
BlockT * getEntry() const
Get the entry BasicBlock of the Region.
Implements a dense probed hash-table based set with some number of buckets stored inline.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
VPBasicBlock serves as the leaf of the Hierarchical Control-Flow Graph.
iterator begin()
Recipe iterator methods.
VPBlockBase is the building block of the Hierarchical Control-Flow Graph.
VPRegionBlock * getParent()
size_t getNumSuccessors() const
const VPBlocksTy & getPredecessors() const
const VPBasicBlock * getEntryBasicBlock() const
const VPBlocksTy & getSuccessors() const
VPRecipeBase is a base class modeling a sequence of one or more output IR instructions.
VPRegionBlock represents a collection of VPBasicBlocks and VPRegionBlocks which form a Single-Entry-S...
const VPBlockBase * getEntry() const
bool isReplicator() const
An indicator whether this region is to generate multiple replicated instances of output IR correspond...
const VPBlockBase * getExiting() const
This class augments VPValue with operands which provide the inverse def-use edges from VPValue's user...
VPlan models a candidate for vectorization, encoding various decisions take to produce efficient outp...
VPBasicBlock * getEntry()
VPRegionBlock * getVectorLoopRegion()
Returns the VPRegionBlock of the vector loop.
const MapVector< PHINode *, VPLiveOut * > & getLiveOuts() const
std::pair< iterator, bool > insert(const ValueT &V)
size_type count(const_arg_type_t< ValueT > V) const
Return 1 if the specified key is in the set, 0 otherwise.
This is an optimization pass for GlobalISel generic memory operations.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
iterator_range< df_iterator< VPBlockShallowTraversalWrapper< VPBlockBase * > > > vp_depth_first_shallow(VPBlockBase *G)
Returns an iterator range to traverse the graph starting at G in depth-first order.
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
bool verifyVPlanIsValid(const VPlan &Plan)
Verify invariants for general VPlans.