23#define DEBUG_TYPE "loop-vectorize"
63bool VPlanVerifier::verifyPhiRecipes(
const VPBasicBlock *VPBB) {
64 auto RecipeI = VPBB->
begin();
66 unsigned NumActiveLaneMaskPhiRecipes = 0;
68 bool IsHeaderVPBB = ParentR && !ParentR->
isReplicator() &&
70 while (RecipeI !=
End && RecipeI->isPhi()) {
71 if (isa<VPActiveLaneMaskPHIRecipe>(RecipeI))
72 NumActiveLaneMaskPhiRecipes++;
74 if (IsHeaderVPBB && !isa<VPHeaderPHIRecipe, VPWidenPHIRecipe>(*RecipeI)) {
75 errs() <<
"Found non-header PHI recipe in header VPBB";
76#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
83 if (!IsHeaderVPBB && isa<VPHeaderPHIRecipe>(*RecipeI)) {
84 errs() <<
"Found header PHI recipe in non-header VPBB";
85#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
95 if (NumActiveLaneMaskPhiRecipes > 1) {
96 errs() <<
"There should be no more than one VPActiveLaneMaskPHIRecipe";
100 while (RecipeI !=
End) {
101 if (RecipeI->isPhi() && !isa<VPBlendRecipe>(&*RecipeI)) {
102 errs() <<
"Found phi-like recipe after non-phi recipe";
104#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
108 std::prev(RecipeI)->dump();
117bool VPlanVerifier::verifyVPBasicBlock(
const VPBasicBlock *VPBB) {
118 if (!verifyPhiRecipes(VPBB))
126 RecipeNumbering[&
R] = Cnt++;
129 for (
const VPValue *V :
R.definedValues()) {
130 for (
const VPUser *U :
V->users()) {
131 auto *UI = dyn_cast<VPRecipeBase>(U);
134 isa<VPHeaderPHIRecipe, VPWidenPHIRecipe, VPPredInstPHIRecipe>(UI))
139 if (UI->getParent() == VPBB) {
140 if (RecipeNumbering[UI] < RecipeNumbering[&R]) {
141 errs() <<
"Use before def!\n";
147 if (!VPDT.dominates(VPBB, UI->getParent())) {
148 errs() <<
"Use before def!\n";
155 auto *IRBB = dyn_cast<VPIRBasicBlock>(VPBB);
159 if (!WrappedIRBBs.insert(IRBB->getIRBasicBlock()).second) {
160 errs() <<
"Same IR basic block used by multiple wrapper blocks!\n";
163 if (IRBB != IRBB->getPlan()->getPreheader()) {
164 errs() <<
"VPIRBasicBlock can only be used as pre-header at the moment!\n";
174 for (
const auto *
Block : VPBlockVec) {
182bool VPlanVerifier::verifyBlock(
const VPBlockBase *VPB) {
183 auto *VPBB = dyn_cast<VPBasicBlock>(VPB);
186 (VPBB && VPBB->getParent() && VPBB->isExiting() &&
187 !VPBB->getParent()->isReplicator())) {
188 if (!VPBB || !VPBB->getTerminator()) {
189 errs() <<
"Block has multiple successors but doesn't "
190 "have a proper branch recipe!\n";
194 if (VPBB && VPBB->getTerminator()) {
195 errs() <<
"Unexpected branch recipe!\n";
205 errs() <<
"Multiple instances of the same successor.\n";
211 const auto &SuccPreds = Succ->getPredecessors();
213 errs() <<
"Missing predecessor link.\n";
224 errs() <<
"Multiple instances of the same predecessor.\n";
230 if (Pred->getParent() != VPB->
getParent()) {
231 errs() <<
"Predecessor is not in the same region.\n";
236 const auto &PredSuccs = Pred->getSuccessors();
238 errs() <<
"Missing successor link.\n";
242 return !VPBB || verifyVPBasicBlock(VPBB);
249 errs() <<
"VPBlockBase has wrong parent\n";
253 if (!verifyBlock(VPB))
264 if (
Entry->getNumPredecessors() != 0) {
265 errs() <<
"region entry block has predecessors\n";
269 errs() <<
"region exiting block has successors\n";
273 return verifyBlocksInRegion(
Region);
278 return verifyRegion(
Region) &&
281 const auto *SubRegion = dyn_cast<VPRegionBlock>(VPB);
282 return !SubRegion || verifyRegionRec(SubRegion);
286bool VPlanVerifier::verify(
const VPlan &Plan) {
288 [
this](
const VPBlockBase *VPB) { return !verifyBlock(VPB); }))
292 if (!verifyRegionRec(TopRegion))
296 errs() <<
"VPlan Top Region should have no parent.\n";
302 errs() <<
"VPlan entry block is not a VPBasicBlock\n";
306 if (!isa<VPCanonicalIVPHIRecipe>(&*
Entry->begin())) {
307 errs() <<
"VPlan vector loop header does not start with a "
308 "VPCanonicalIVPHIRecipe\n";
314 errs() <<
"VPlan exiting block is not a VPBasicBlock\n";
318 if (Exiting->
empty()) {
319 errs() <<
"VPlan vector loop exiting block must end with BranchOnCount or "
320 "BranchOnCond VPInstruction but is empty\n";
324 auto *LastInst = dyn_cast<VPInstruction>(std::prev(Exiting->
end()));
327 errs() <<
"VPlan vector loop exit must end with BranchOnCount or "
328 "BranchOnCond VPInstruction\n";
333 if (KV.second->getNumOperands() != 1) {
334 errs() <<
"live outs must have a single operand\n";
This file builds on the ADT/GraphTraits.h file to build generic depth first graph iterator.
verify safepoint Safepoint IR Verifier
This file defines the SmallPtrSet class.
This file implements dominator tree analysis for a single level of a VPlan's H-CFG.
static bool hasDuplicates(const SmallVectorImpl< VPBlockBase * > &VPBlockVec)
Utility function that checks whether VPBlockVec has duplicate VPBlockBases.
This file declares the class VPlanVerifier, which contains utility functions to check the consistency...
This file contains the declarations of the Vectorization Plan base classes:
Core dominator tree base class.
void recalculate(ParentType &Func)
recalculate - compute a dominator tree for the given function
BlockT * getEntry() const
Get the entry BasicBlock of the Region.
Implements a dense probed hash-table based set with some number of buckets stored inline.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
VPBasicBlock serves as the leaf of the Hierarchical Control-Flow Graph.
iterator begin()
Recipe iterator methods.
VPBlockBase is the building block of the Hierarchical Control-Flow Graph.
VPRegionBlock * getParent()
size_t getNumSuccessors() const
const VPBlocksTy & getPredecessors() const
const VPBasicBlock * getEntryBasicBlock() const
const VPBlocksTy & getSuccessors() const
VPRecipeBase is a base class modeling a sequence of one or more output IR instructions.
VPRegionBlock represents a collection of VPBasicBlocks and VPRegionBlocks which form a Single-Entry-S...
const VPBlockBase * getEntry() const
bool isReplicator() const
An indicator whether this region is to generate multiple replicated instances of output IR correspond...
const VPBlockBase * getExiting() const
This class augments VPValue with operands which provide the inverse def-use edges from VPValue's user...
VPlan models a candidate for vectorization, encoding various decisions take to produce efficient outp...
VPBasicBlock * getEntry()
VPRegionBlock * getVectorLoopRegion()
Returns the VPRegionBlock of the vector loop.
const MapVector< PHINode *, VPLiveOut * > & getLiveOuts() const
std::pair< iterator, bool > insert(const ValueT &V)
size_type count(const_arg_type_t< ValueT > V) const
Return 1 if the specified key is in the set, 0 otherwise.
This is an optimization pass for GlobalISel generic memory operations.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
iterator_range< df_iterator< VPBlockShallowTraversalWrapper< VPBlockBase * > > > vp_depth_first_shallow(VPBlockBase *G)
Returns an iterator range to traverse the graph starting at G in depth-first order.
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
bool verifyVPlanIsValid(const VPlan &Plan)
Verify invariants for general VPlans.