LLVM 24.0.0git
VPlanVerifier.cpp
Go to the documentation of this file.
1//===-- VPlanVerifier.cpp -------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// This file defines the class VPlanVerifier, which contains utility functions
11/// to check the consistency and invariants of a VPlan.
12///
13//===----------------------------------------------------------------------===//
14
15#include "VPlanVerifier.h"
16#include "VPlan.h"
17#include "VPlanCFG.h"
18#include "VPlanDominatorTree.h"
19#include "VPlanHelpers.h"
20#include "VPlanPatternMatch.h"
21#include "VPlanUtils.h"
23
24#define DEBUG_TYPE "loop-vectorize"
25
26using namespace llvm;
27using namespace VPlanPatternMatch;
28
29namespace {
30class VPlanVerifier {
31 const VPDominatorTree &VPDT;
32
34
35 // Verify that phi-like recipes are at the beginning of \p VPBB, with no
36 // other recipes in between. Also check that only header blocks contain
37 // VPHeaderPHIRecipes.
38 bool verifyPhiRecipes(const VPBasicBlock *VPBB);
39
40 /// Verify that \p LastActiveLane's operand is guaranteed to be a prefix-mask.
41 bool verifyLastActiveLaneRecipe(const VPInstruction &LastActiveLane) const;
42
43 bool verifyVPBasicBlock(const VPBasicBlock *VPBB);
44
45 bool verifyBlock(const VPBlockBase *VPB);
46
47 /// Helper function that verifies the CFG invariants of the VPBlockBases
48 /// within
49 /// \p Region. Checks in this function are generic for VPBlockBases. They are
50 /// not specific for VPBasicBlocks or VPRegionBlocks.
51 bool verifyBlocksInRegion(const VPRegionBlock *Region);
52
53 /// Verify the CFG invariants of VPRegionBlock \p Region and its nested
54 /// VPBlockBases. Do not recurse inside nested VPRegionBlocks.
55 bool verifyRegion(const VPRegionBlock *Region);
56
57 /// Verify the CFG invariants of VPRegionBlock \p Region and its nested
58 /// VPBlockBases. Recurse inside nested VPRegionBlocks.
59 bool verifyRegionRec(const VPRegionBlock *Region);
60
61public:
62 VPlanVerifier(VPDominatorTree &VPDT) : VPDT(VPDT) {}
63
64 bool verify(const VPlan &Plan);
65};
66} // namespace
67
68bool VPlanVerifier::verifyPhiRecipes(const VPBasicBlock *VPBB) {
69 auto RecipeI = VPBB->begin();
70 auto End = VPBB->end();
71 unsigned NumActiveLaneMaskPhiRecipes = 0;
72 bool IsHeaderVPBB = VPBlockUtils::isHeader(VPBB, VPDT);
73 while (RecipeI != End && RecipeI->isPhi()) {
75 NumActiveLaneMaskPhiRecipes++;
76
77 if (IsHeaderVPBB &&
79 errs() << "Found non-header PHI recipe in header VPBB";
80#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
81 errs() << ": ";
82 RecipeI->dump();
83#endif
84 return false;
85 }
86
87 if (!IsHeaderVPBB && isa<VPHeaderPHIRecipe>(*RecipeI)) {
88 errs() << "Found header PHI recipe in non-header VPBB";
89#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
90 errs() << ": ";
91 RecipeI->dump();
92#endif
93 return false;
94 }
95
96 // In region form, VPCurrentIterationPHIRecipe must be the first header phi
97 // recipe. In a plain CFG VPlan, it must either be the first or second.
100 ? RecipeI->getIterator() != VPBB->begin()
101 : RecipeI->getIterator() != VPBB->begin() &&
102 RecipeI->getIterator() != std::next(VPBB->begin()))) {
103 errs() << "CurrentIteration PHI is not the first/second recipe\n";
104 return false;
105 }
106
107 // Check if the recipe operands match the number of predecessors.
108 // TODO Extend to other phi-like recipes.
109 if (auto *PhiIRI = dyn_cast<VPIRPhi>(&*RecipeI)) {
110 if (PhiIRI->getNumOperands() != VPBB->getNumPredecessors()) {
111 errs() << "Phi-like recipe with different number of operands and "
112 "predecessors.\n";
113 // TODO: Print broken recipe. At the moment printing an ill-formed
114 // phi-like recipe may crash.
115 return false;
116 }
117 }
118
119 RecipeI++;
120 }
121
122 if (!VPBB->getPlan()->isUnrolled() && NumActiveLaneMaskPhiRecipes > 1) {
123 errs() << "There should be no more than one VPActiveLaneMaskPHIRecipe";
124 return false;
125 }
126
127 while (RecipeI != End) {
128 if (RecipeI->isPhi() && !isa<VPBlendRecipe>(&*RecipeI)) {
129 errs() << "Found phi-like recipe after non-phi recipe";
130
131#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
132 errs() << ": ";
133 RecipeI->dump();
134 errs() << "after\n";
135 std::prev(RecipeI)->dump();
136#endif
137 return false;
138 }
139 RecipeI++;
140 }
141 return true;
142}
143
145 VPValue *X, *Y;
146 if (match(V, m_Add(m_VPValue(X), m_VPValue(Y))))
147 return cast<VPRecipeWithIRFlags>(V)->hasNoUnsignedWrap() &&
149 if (match(V, m_StepVector()))
150 return true;
151 // Only handle a subset of IVs until we can guarantee there's no overflow.
152 if (auto *WidenIV = dyn_cast<VPWidenIntOrFpInductionRecipe>(V))
153 return WidenIV->isCanonical() || WidenIV->hasNoUnsignedWrap();
154 if (auto *Steps = dyn_cast<VPScalarIVStepsRecipe>(V))
155 return match(Steps->getOperand(0),
159 match(Steps->getStepValue(), m_One());
161 return true;
163}
164
165bool VPlanVerifier::verifyLastActiveLaneRecipe(
166 const VPInstruction &LastActiveLane) const {
167 assert(LastActiveLane.getOpcode() == VPInstruction::LastActiveLane &&
168 "must be called with VPInstruction::LastActiveLane");
169
170 if (LastActiveLane.getNumOperands() < 1) {
171 errs() << "LastActiveLane must have at least one operand\n";
172 return false;
173 }
174
175 // All operands must be prefix-mask. This means an icmp ult/ule LHS, RHS where
176 // the LHS is monotonically increasing and RHS is uniform across VFs and UF.
177 for (VPValue *Op : LastActiveLane.operands()) {
178 VPValue *Mask = Op;
179 VPValue *HeaderMask;
180
181 // Look through any `and`s with the incoming alias mask or a
182 // loop_dependence_war_mask, which are always prefix masks.
183 // TODO: Verify the full loop.dependence.mask chain.
184 if (match(Op,
186 m_VPValue(HeaderMask),
190 m_VPValue()),
193 Mask = HeaderMask;
194
195 // The header mask is a prefix mask. Before being materialized it is the
196 // loop region's abstract header mask; afterwards it is an active lane mask
197 // (an intrinsic, phi, or an extract from a phi), or the icmp checked below.
198 if (match(Mask, m_CombineOr(
203 continue;
204
205 CmpPredicate Pred;
206 VPValue *LHS, *RHS;
207 if (match(Mask, m_ICmp(Pred, m_VPValue(LHS), m_VPValue(RHS))) &&
211 match(RHS, m_EVL(m_VPValue()))))
212 continue;
213
214 errs() << "LastActiveLane operand ";
215#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
216 VPSlotTracker Tracker(LastActiveLane.getParent()->getPlan());
217 Op->printAsOperand(errs(), Tracker);
218#endif
219 errs() << " must be prefix mask (a header mask or an "
220 "EVL-derived mask currently)\n";
221 return false;
222 }
223
224 return true;
225}
226
227bool VPlanVerifier::verifyVPBasicBlock(const VPBasicBlock *VPBB) {
228 if (!verifyPhiRecipes(VPBB))
229 return false;
230
231 // Verify that defs in VPBB dominate all their uses.
232 DenseMap<const VPRecipeBase *, unsigned> RecipeNumbering;
233 unsigned Cnt = 0;
234 for (const VPRecipeBase &R : *VPBB)
235 RecipeNumbering[&R] = Cnt++;
236
237 for (const VPRecipeBase &R : *VPBB) {
238 if (isa<VPIRInstruction>(&R) && !isa<VPIRBasicBlock>(VPBB)) {
239 errs() << "VPIRInstructions ";
240#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
241 R.dump();
242 errs() << " ";
243#endif
244 errs() << "not in a VPIRBasicBlock!\n";
245 return false;
246 }
247 for (const VPValue *V : R.definedValues()) {
248 // Verify that each defined value has a scalar type.
249 if (!V->getScalarType()) {
250 errs() << "VPValue without scalar type!\n";
251 return false;
252 }
253
254 for (const VPUser *U : V->users()) {
255 auto *UI = cast<VPRecipeBase>(U);
256 if (isa<VPIRPhi>(UI) &&
257 UI->getNumOperands() != UI->getParent()->getNumPredecessors()) {
258 errs() << "Phi-like recipe with different number of operands and "
259 "predecessors.\n";
260 return false;
261 }
262
263 if (auto *Phi = dyn_cast<VPPhiAccessors>(UI)) {
264 for (const auto &[IncomingVPV, IncomingVPBB] :
265 Phi->incoming_values_and_blocks()) {
266 if (IncomingVPV != V)
267 continue;
268
269 if (VPDT.dominates(VPBB, IncomingVPBB))
270 continue;
271
272 errs() << "Incoming def does not dominate incoming block!\n";
273#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
274 VPSlotTracker Tracker(VPBB->getPlan());
275 IncomingVPV->getDefiningRecipe()->print(errs(), " ", Tracker);
276 errs() << "\n does not dominate " << IncomingVPBB->getName()
277 << " for\n";
278 UI->print(errs(), " ", Tracker);
279#endif
280 return false;
281 }
282 continue;
283 }
284 // TODO: Also verify VPPredInstPHIRecipe.
286 continue;
287
288 // If the user is in the same block, check it comes after R in the
289 // block.
290 if (UI->getParent() == VPBB) {
291 if (RecipeNumbering[UI] >= RecipeNumbering[&R])
292 continue;
293 } else {
294 // MaskedCond may be used from blocks it don't dominate; the block
295 // will be linearized and it will dominate its users after
296 // linearization.
298 VPDT.dominates(VPBB, UI->getParent()))
299 continue;
300 }
301
302 // Recipes in blocks with a MaskedCond may be used in exit blocks; the
303 // block will be linearized and its recipes will dominate their users
304 // after linearization.
305 bool BlockHasMaskedCond = any_of(*VPBB, [](const VPRecipeBase &R) {
307 });
308 if (BlockHasMaskedCond &&
309 any_of(VPBB->getPlan()->getExitBlocks(), [UI](VPIRBasicBlock *EB) {
310 return is_contained(EB->getPredecessors(), UI->getParent());
311 })) {
312 continue;
313 }
314
315 errs() << "Use before def!\n";
316#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
317 VPSlotTracker Tracker(VPBB->getPlan());
318 UI->print(errs(), " ", Tracker);
319 errs() << "\n before\n";
320 R.print(errs(), " ", Tracker);
321 errs() << "\n";
322#endif
323 return false;
324 }
325 }
326 if (const auto *VPI = dyn_cast<VPInstruction>(&R)) {
327 switch (VPI->getOpcode()) {
329 if (!verifyLastActiveLaneRecipe(*VPI))
330 return false;
331 break;
332 default:
333 break;
334 }
335 }
336 if (const auto *DIV = dyn_cast<VPDerivedIVRecipe>(&R)) {
337 if (!DIV->getStartValue()->isDefinedOutsideLoopRegions()) {
338 errs() << "VPDerivedIVRecipe must have start value defined outside "
339 "loop regions\n";
340 return false;
341 }
342 }
343 if (const auto *ScalarIVSteps = dyn_cast<VPScalarIVStepsRecipe>(&R)) {
344 unsigned NumOps = ScalarIVSteps->getNumOperands();
345 if (NumOps != 3 && NumOps != 4) {
346 errs() << "VPScalarIVStepsRecipe must have 3 or 4 operands\n";
347 return false;
348 }
349 }
350 }
351
352 auto *IRBB = dyn_cast<VPIRBasicBlock>(VPBB);
353 if (!IRBB)
354 return true;
355
356 if (!WrappedIRBBs.insert(IRBB->getIRBasicBlock()).second) {
357 errs() << "Same IR basic block used by multiple wrapper blocks!\n";
358 return false;
359 }
360
361 return true;
362}
363
364bool VPlanVerifier::verifyBlock(const VPBlockBase *VPB) {
365 auto *VPBB = dyn_cast<VPBasicBlock>(VPB);
366 // Check block's condition bit.
367 if (VPBB && !isa<VPIRBasicBlock>(VPB)) {
368 // For plain CFG VPlans, verify header and latch block structure.
369 if (!VPBB->getParent()) {
370 if (VPBlockUtils::isHeader(VPBB, VPDT)) {
371 if (VPB->getNumPredecessors() != 2) {
372 errs()
373 << "Header block in plain CFG VPlan must have 2 predecessors!\n";
374 return false;
375 }
376 // Predecessor 0 is preheader, predecessor 1 is latch.
377 if (!VPBlockUtils::isLatch(VPB->getPredecessors()[1], VPDT)) {
378 errs() << "Header's second predecessor must be the latch!\n";
379 return false;
380 }
381 }
382
383 if (VPBlockUtils::isLatch(VPBB, VPDT)) {
384 if (!match(VPBB->getTerminator(), m_Branch())) {
385 errs() << "Latch block must have a branch terminator!\n";
386 return false;
387 }
388 // Successor 0 is middle block, successor 1 is header.
389 if (VPBlockUtils::isHeader(VPB->getSuccessors()[0], VPDT)) {
390 errs() << "Latch's first successor must not be the header (must be "
391 "middle block)!\n";
392 return false;
393 }
394 }
395 } else if (VPB->getNumSuccessors() > 1 ||
396 (VPBB->isExiting() && !VPBB->getParent()->isReplicator())) {
397 if (!VPBB->getTerminator()) {
398 errs() << "Block has multiple successors but doesn't "
399 "have a proper branch recipe!\n";
400 return false;
401 }
402 } else if (VPBB->getTerminator()) {
403 errs() << "Unexpected branch recipe!\n";
404 return false;
405 }
406 }
407
408 // Check block's successors.
409 const auto &Successors = VPB->getSuccessors();
410 for (const VPBlockBase *Succ : Successors) {
411 // There must be a bi-directional link between block and successor.
412 const auto &SuccPreds = Succ->getPredecessors();
413 if (!is_contained(SuccPreds, VPB)) {
414 errs() << "Missing predecessor link.\n";
415 return false;
416 }
417 }
418
419 // Check block's predecessors.
420 const auto &Predecessors = VPB->getPredecessors();
421
422 for (const VPBlockBase *Pred : Predecessors) {
423 // Block and predecessor must be inside the same region.
424 if (Pred->getParent() != VPB->getParent()) {
425 errs() << "Predecessor is not in the same region.\n";
426 return false;
427 }
428
429 // There must be a bi-directional link between block and predecessor.
430 const auto &PredSuccs = Pred->getSuccessors();
431 if (!is_contained(PredSuccs, VPB)) {
432 errs() << "Missing successor link.\n";
433 return false;
434 }
435 }
436 return !VPBB || verifyVPBasicBlock(VPBB);
437}
438
439bool VPlanVerifier::verifyBlocksInRegion(const VPRegionBlock *Region) {
440 for (const VPBlockBase *VPB : vp_depth_first_shallow(Region->getEntry())) {
441 // Check block's parent.
442 if (VPB->getParent() != Region) {
443 errs() << "VPBlockBase has wrong parent\n";
444 return false;
445 }
446
447 if (!verifyBlock(VPB))
448 return false;
449 }
450 return true;
451}
452
453bool VPlanVerifier::verifyRegion(const VPRegionBlock *Region) {
454 const VPBlockBase *Entry = Region->getEntry();
455 const VPBlockBase *Exiting = Region->getExiting();
456
457 // Entry and Exiting shouldn't have any predecessor/successor, respectively.
458 if (Entry->hasPredecessors()) {
459 errs() << "region entry block has predecessors\n";
460 return false;
461 }
462 if (Exiting->getNumSuccessors() != 0) {
463 errs() << "region exiting block has successors\n";
464 return false;
465 }
466
467 return verifyBlocksInRegion(Region);
468}
469
470bool VPlanVerifier::verifyRegionRec(const VPRegionBlock *Region) {
471 // Recurse inside nested regions and check all blocks inside the region.
472 return verifyRegion(Region) &&
474 [this](const VPBlockBase *VPB) {
475 const auto *SubRegion = dyn_cast<VPRegionBlock>(VPB);
476 return !SubRegion || verifyRegionRec(SubRegion);
477 });
478}
479
480bool VPlanVerifier::verify(const VPlan &Plan) {
482 [this](const VPBlockBase *VPB) { return !verifyBlock(VPB); }))
483 return false;
484
485 // Check that the plan has a single loop region reachable from entry, and it
486 // matches the one returned by getVectorLoopRegion.
487 const VPRegionBlock *TopRegion = Plan.getVectorLoopRegion();
490 [TopRegion](const VPRegionBlock *R) {
491 return !R->isReplicator() && R != TopRegion;
492 })) {
493 errs() << "VPlan must have a single top-level loop region, reachable from "
494 "the entry by following the last successor of each block\n";
495 return false;
496 }
497
498 // TODO: Verify all blocks using vp_depth_first_deep iterators.
499 if (!TopRegion)
500 return true;
501
502 if (!verifyRegionRec(TopRegion))
503 return false;
504
505 if (TopRegion->getParent()) {
506 errs() << "VPlan Top Region should have no parent.\n";
507 return false;
508 }
509
510 const VPBasicBlock *Entry = dyn_cast<VPBasicBlock>(TopRegion->getEntry());
511 if (!Entry) {
512 errs() << "VPlan entry block is not a VPBasicBlock\n";
513 return false;
514 }
515
516 const VPBasicBlock *Exiting = dyn_cast<VPBasicBlock>(TopRegion->getExiting());
517 if (!Exiting) {
518 errs() << "VPlan exiting block is not a VPBasicBlock\n";
519 return false;
520 }
521
522 if (Exiting->empty()) {
523 errs() << "VPlan vector loop exiting block must end with BranchOnCount, "
524 "BranchOnCond, or BranchOnTwoConds VPInstruction but is empty\n";
525 return false;
526 }
527
528 auto *LastInst = dyn_cast<VPInstruction>(std::prev(Exiting->end()));
529 if (!match(LastInst, m_Branch())) {
530 errs() << "VPlan vector loop exit must end with BranchOnCount, "
531 "BranchOnCond, or BranchOnTwoConds VPInstruction\n";
532 return false;
533 }
534
535 return true;
536}
537
539 // The entry must be the root of the plan's top-level CFG: the dominator tree
540 // constructed below and the verifier's block walks all start there.
541 if (Plan.getEntry()->hasPredecessors()) {
542 errs() << "VPlan entry block has predecessors\n";
543 return false;
544 }
545
546 VPDominatorTree VPDT(const_cast<VPlan &>(Plan));
547 VPlanVerifier Verifier(VPDT);
548 return Verifier.verify(Plan);
549}
SDValue Pred
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
#define X(NUM, ENUM, NAME)
Definition ELF.h:665
const size_t AbstractManglingParser< Derived, Alloc >::NumOps
ppc ctr loops verify
verify safepoint Safepoint IR Verifier
This file defines the SmallPtrSet class.
static TableGen::Emitter::Opt Y("gen-skeleton-entry", EmitSkeleton, "Generate example skeleton entry")
This file implements dominator tree analysis for a single level of a VPlan's H-CFG.
This file contains the declarations of different VPlan-related auxiliary helpers.
static bool isKnownMonotonic(VPValue *V)
This file declares the class VPlanVerifier, which contains utility functions to check the consistency...
This file contains the declarations of the Vectorization Plan base classes:
Value * RHS
Value * LHS
@ ICMP_ULT
unsigned less than
Definition InstrTypes.h:765
@ ICMP_ULE
unsigned less or equal
Definition InstrTypes.h:766
bool dominates(const DomTreeNodeBase< NodeT > *A, const DomTreeNodeBase< NodeT > *B) const
dominates - Returns true iff A dominates B.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
VPBasicBlock serves as the leaf of the Hierarchical Control-Flow Graph.
Definition VPlan.h:4540
iterator end()
Definition VPlan.h:4581
iterator begin()
Recipe iterator methods.
Definition VPlan.h:4579
bool empty() const
Definition VPlan.h:4590
VPBlockBase is the building block of the Hierarchical Control-Flow Graph.
Definition VPlan.h:95
VPRegionBlock * getParent()
Definition VPlan.h:193
size_t getNumSuccessors() const
Definition VPlan.h:243
bool hasPredecessors() const
Returns true if this block has any predecessors.
Definition VPlan.h:223
size_t getNumPredecessors() const
Definition VPlan.h:244
const VPBlocksTy & getPredecessors() const
Definition VPlan.h:228
VPlan * getPlan()
Definition VPlan.h:197
const VPBlocksTy & getSuccessors() const
Definition VPlan.h:217
static bool isLatch(const VPBlockBase *VPB, const VPDominatorTree &VPDT)
Returns true if VPB is a loop latch, using isHeader().
static bool isHeader(const VPBlockBase *VPB, const VPDominatorTree &VPDT)
Returns true if VPB is a loop header, based on regions or VPDT in their absence.
static auto blocksOnly(T &&Range)
Return an iterator range over Range which only includes BlockTy blocks.
Definition VPlanUtils.h:457
Template specialization of the standard LLVM dominator tree utility for VPBlockBases.
This is a concrete Recipe that models a single VPlan-level instruction.
Definition VPlan.h:1322
unsigned getOpcode() const
Definition VPlan.h:1525
VPRegionBlock represents a collection of VPBasicBlocks and VPRegionBlocks which form a Single-Entry-S...
Definition VPlan.h:4771
const VPBlockBase * getEntry() const
Definition VPlan.h:4815
const VPBlockBase * getExiting() const
Definition VPlan.h:4827
operand_range operands()
Definition VPlanValue.h:472
unsigned getNumOperands() const
Definition VPlanValue.h:436
This is the base class of the VPlan Def/Use graph, used for modeling the data flow into,...
Definition VPlanValue.h:50
VPlan models a candidate for vectorization, encoding various decisions take to produce efficient outp...
Definition VPlan.h:4958
VPBasicBlock * getEntry()
Definition VPlan.h:5050
bool isUnrolled() const
Returns true if the VPlan already has been unrolled, i.e.
Definition VPlan.h:5222
LLVM_ABI_FOR_TEST VPRegionBlock * getVectorLoopRegion()
Returns the VPRegionBlock of the vector loop.
Definition VPlan.cpp:1062
const ParentTy * getParent() const
Definition ilist_node.h:34
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ Entry
Definition COFF.h:862
SpecificConstantMatch m_ZeroInt()
Convenience matchers for specific integer values.
match_combine_or< Ty... > m_CombineOr(const Ty &...Ps)
Combine pattern matchers matching any of Ps patterns.
BinaryOp_match< LHS, RHS, Instruction::Add > m_Add(const LHS &L, const RHS &R)
bool match(Val *V, const Pattern &P)
cst_pred_ty< is_one > m_One()
Match an integer 1 or a vector with all elements equal to 1.
auto m_Intrinsic(const Ts &...Ops)
Match intrinsic calls like this: m_Intrinsic<Intrinsic::fabs>(m_Value(X))
CmpClass_match< LHS, RHS, ICmpInst > m_ICmp(CmpPredicate &Pred, const LHS &L, const RHS &R)
AllRecipe_commutative_match< Instruction::And, Op0_t, Op1_t > m_c_BinaryAnd(const Op0_t &Op0, const Op1_t &Op1)
Match a binary AND operation.
VPInstruction_match< VPInstruction::StepVector > m_StepVector()
canonical_iv_match m_CanonicalIV()
auto m_VPValue()
Match an arbitrary VPValue and ignore it.
header_mask_match m_HeaderMask()
VPInstruction_match< VPInstruction::ExplicitVectorLength, Op0_t > m_EVL(const Op0_t &Op0)
match_bind< VPInstruction > m_VPInstruction(VPInstruction *&V)
Match a VPInstruction, capturing if we match.
VPInstruction_match< VPInstruction::ExtractVectorForPart, Op_t... > m_ExtractVectorForPart(const Op_t &...Op)
auto m_DerivedIV(const Op0_t &Op0, const Op1_t &Op1, const Op2_t &Op2)
NodeAddr< PhiNode * > Phi
Definition RDFGraph.h:390
LLVM_ABI_FOR_TEST bool isUniformAcrossVFsAndUFs(const VPValue *V)
Checks if V is uniform across all VF lanes and UF parts.
This is an optimization pass for GlobalISel generic memory operations.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1755
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:643
iterator_range< df_iterator< VPBlockShallowTraversalWrapper< VPBlockBase * > > > vp_depth_first_shallow(VPBlockBase *G)
Returns an iterator range to traverse the graph starting at G in depth-first order.
Definition VPlanCFG.h:250
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1762
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
Definition Casting.h:547
LLVM_ABI raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
DWARFExpression::Operation Op
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:559
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
Definition STLExtras.h:1963
@ End
Run the machine verifier once, at the end of the pipeline.
Definition CodeGen.h:320
LLVM_ABI_FOR_TEST bool verifyVPlanIsValid(const VPlan &Plan)
Verify invariants for general VPlans.