48 SmallPtrSet<VPRecipeBase *, 8> ToSkip;
52 DenseMap<VPValue *, SmallVector<VPValue *>> VPV2Parts;
55 void unrollReplicateRegionByUF(VPRegionBlock *VPR);
59 void unrollRecipeByUF(VPRecipeBase &R);
64 void unrollHeaderPHIByUF(VPHeaderPHIRecipe *R,
69 void unrollWidenInductionByUF(VPWidenInductionRecipe *
IV,
73 Type *CanIVIntTy = Plan.getVectorLoopRegion()->getCanonicalIVType();
74 return Plan.getConstantInt(CanIVIntTy, Part);
78 UnrollState(VPlan &Plan,
unsigned UF) : Plan(Plan), UF(UF) {}
80 void unrollBlock(VPBlockBase *VPB);
82 VPValue *getValueForPart(VPValue *
V,
unsigned Part) {
85 assert((VPV2Parts.contains(
V) && VPV2Parts[
V].size() >= Part) &&
86 "accessed value does not exist");
87 return VPV2Parts[
V][Part - 1];
93 void addRecipeForPart(VPRecipeBase *OrigR, VPRecipeBase *CopyR,
96 const auto &[
V,
_] = VPV2Parts.try_emplace(VPV);
97 assert(
V->second.size() == Part - 1 &&
"earlier parts not set");
103 void addUniformForAllParts(VPSingleDefRecipe *R) {
104 const auto &[
V,
Inserted] = VPV2Parts.try_emplace(R);
105 assert(Inserted &&
"uniform value already added");
106 for (
unsigned Part = 0; Part != UF; ++Part)
107 V->second.push_back(R);
110 bool contains(VPValue *VPV)
const {
return VPV2Parts.contains(VPV); }
114 void remapOperand(VPRecipeBase *R,
unsigned OpIdx,
unsigned Part) {
115 auto *
Op =
R->getOperand(OpIdx);
116 R->setOperand(OpIdx, getValueForPart(
Op, Part));
121 for (
const auto &[OpIdx,
Op] :
enumerate(
R->operands()))
122 R->setOperand(OpIdx, getValueForPart(
Op, Part));
128 unsigned Part,
VPlan &Plan) {
138 StartIndex = Builder.createOverflowingOp(
142 StartIndex = Builder.createScalarSExtOrTrunc(StartIndex, IntStepTy,
146 StartIndex = Builder.createScalarCast(Instruction::SIToFP, StartIndex,
152void UnrollState::unrollReplicateRegionByUF(
VPRegionBlock *VPR) {
154 for (
unsigned Part = 1; Part !=
UF; ++Part) {
160 for (
const auto &[PartIVPBB, Part0VPBB] :
163 for (
const auto &[PartIR, Part0R] :
zip(*PartIVPBB, *Part0VPBB)) {
168 addRecipeForPart(&Part0R, &PartIR, Part);
174void UnrollState::unrollWidenInductionByUF(
177 IV->getParent()->getEnclosingLoopRegion()->getSinglePredecessor());
178 Type *IVTy =
IV->getScalarType();
179 auto &
ID =
IV->getInductionDescriptor();
181 VPIRFlags::WrapFlagsTy WrapFlags(
false,
false);
183 FMF = IntOrFPInd->getFastMathFlagsOrNone();
184 WrapFlags = IntOrFPInd->getNoWrapFlagsOrNone();
187 VPValue *ScalarStep =
IV->getStepValue();
190 VPInstruction *VectorStep = Builder.createNaryOp(
194 ToSkip.
insert(VectorStep);
209 Builder.setInsertPoint(InsertPtForPhi);
216 AddOpc =
ID.getInductionOpcode();
219 AddOpc = Instruction::Add;
220 AddFlags = WrapFlags;
222 AddFlags = VPIRFlags::WrapFlagsTy(
true,
false);
224 for (
unsigned Part = 1; Part !=
UF; ++Part) {
226 Part > 1 ?
"step.add." + std::to_string(Part) :
"step.add";
229 Builder.createNaryOp(AddOpc,
234 AddFlags,
IV->getDebugLoc(), Name);
236 addRecipeForPart(
IV,
Add, Part);
239 IV->addUnrolledPartOperands(VectorStep, Prev);
242void UnrollState::unrollHeaderPHIByUF(VPHeaderPHIRecipe *R,
251 unrollWidenInductionByUF(
IV, InsertPtForPhi);
256 if (RdxPhi && RdxPhi->isOrdered())
259 auto InsertPt = std::next(
R->getIterator());
260 for (
unsigned Part = 1; Part !=
UF; ++Part) {
261 VPRecipeBase *
Copy =
R->clone();
262 Copy->insertBefore(InsertPt);
263 addRecipeForPart(R, Copy, Part);
271 "unexpected start VPInstruction");
276 StartV = VPI->getOperand(1);
278 auto *
C = VPI->clone();
279 C->setOperand(0,
C->getOperand(1));
283 for (
unsigned Part = 1; Part !=
UF; ++Part)
284 VPV2Parts[VPI][Part - 1] = StartV;
288 "unexpected header phi recipe not needing unrolled part");
294void UnrollState::unrollRecipeByUF(VPRecipeBase &R) {
302 addUniformForAllParts(VPI);
309 RepR->getOperand(1)->isDefinedOutsideLoopRegions()) {
316 addUniformForAllParts(RepR);
322 auto InsertPt = std::next(
R.getIterator());
323 for (
unsigned Part = 1; Part !=
UF; ++Part) {
324 VPRecipeBase *
Copy =
R.clone();
325 Copy->insertBefore(InsertPt);
326 addRecipeForPart(&R, Copy, Part);
335 Copy->setOperand(0, getValueForPart(
Op, Part - 1));
336 Copy->setOperand(1, getValueForPart(
Op, Part));
351 :
DL.getIndexType(
R.getVPSingleValue()->getScalarType());
352 VPValue *VF = Builder.createScalarZExtOrTrunc(&Plan.
getVF(), IndexTy,
355 VPValue *VFxPart = Builder.createOverflowingOp(
359 VecPtr->addPerPartOffset(VFxPart);
366 if (Phi &&
Phi->isOrdered()) {
367 auto &Parts = VPV2Parts[
Phi];
370 Parts.push_back(Red);
372 Parts.push_back(
Copy->getVPSingleValue());
373 Phi->setOperand(1,
Copy->getVPSingleValue());
378 VEPR->setOperand(0,
R.getOperand(0));
379 VEPR->setOperand(1,
R.getOperand(1));
380 VEPR->materializeOffset(Part);
392 VPValue *ScaledByPart = Builder.createOverflowingOp(
394 Copy->setOperand(1, ScaledByPart);
399 VEPR->materializeOffset();
407void UnrollState::unrollBlock(VPBlockBase *VPB) {
411 return unrollReplicateRegionByUF(VPR);
415 ReversePostOrderTraversal<VPBlockShallowTraversalWrapper<VPBlockBase *>>
417 for (VPBlockBase *VPB : RPOT)
424 auto InsertPtForPhi = VPBB->getFirstNonPhi();
440 addUniformForAllParts(VPI);
441 for (
unsigned Part = 1; Part !=
UF; ++Part)
442 VPI->addOperand(getValueForPart(Op1, Part));
448 addUniformForAllParts(VPI);
449 for (
unsigned Part = 1; Part !=
UF; ++Part)
450 VPI->addOperand(getValueForPart(Op1, Part));
458 addUniformForAllParts(VPI);
459 for (
unsigned Part = 1; Part !=
UF; ++Part) {
460 VPI->addOperand(getValueForPart(Op1, Part));
461 VPI->addOperand(getValueForPart(Op2, Part));
470 bool IsPenultimatePart =
472 unsigned PartIdx = IsPenultimatePart ?
UF - 2 :
UF - 1;
474 I->replaceAllUsesWith(getValueForPart(Op0, PartIdx));
482 R.setOperand(0, getValueForPart(Op0, UF - 1));
494 if (ALM &&
match(ALM->getBackedgeValue(),
496 addUniformForAllParts(ALM);
509 addUniformForAllParts(SingleDef);
514 unrollHeaderPHIByUF(
H, InsertPtForPhi);
523 assert(UF > 0 &&
"Unroll factor must be positive");
536 VPI.replaceAllUsesWith(
Op);
537 assert(VPI.user_empty() &&
538 "ExtractLastPart must be resolved by unrolling");
539 VPI.eraseFromParent();
543 VPI.getOperand(1) == &Plan.
getVF()) {
544 VPI.replaceAllUsesWith(VPI.getOperand(0));
545 VPI.eraseFromParent();
557 UnrollState Unroller(Plan, UF);
565 Unroller.unrollBlock(VPB);
577 Unroller.remapOperand(&
H, 1, UF - 1);
580 if (Unroller.contains(
H.getVPSingleValue())) {
584 Unroller.remapOperands(&
H, Part);
594 assert(Lane > 0 &&
"Zero lane adds no offset to start index");
605 LaneOffset = Plan.
getOrAddLiveIn(ConstantFP::get(BaseIVTy, Lane));
606 AddOpcode = Instruction::FAdd;
611 APInt(BaseIVBits, Lane,
false,
true));
612 AddOpcode = Instruction::Add;
616 VPValue *NewStartIndex = LaneOffset;
620 Builder.createNaryOp(AddOpcode, {OldStartIndex, LaneOffset}, Flags);
633 "DefR must be a VPReplicateRecipe, VPInstruction or "
634 "VPScalarIVStepsRecipe");
637 auto LaneDefs = Def2LaneDefs.find(
Op);
638 if (LaneDefs != Def2LaneDefs.end())
639 return LaneDefs->second[Lane];
642 return Builder.createNaryOp(Instruction::ExtractElement, {
Op, Idx});
650 auto LaneDefs = Def2LaneDefs.find(
Op);
651 if (LaneDefs != Def2LaneDefs.end()) {
652 NewOps.
push_back(LaneDefs->second[Lane]);
661 VPValue *Ext = Builder.createNaryOp(Instruction::ExtractElement, {
Op, Idx});
671 RepR->getOpcode(), NewOps,
nullptr, *RepR, *RepR,
672 RepR->getDebugLoc(), RepR->getScalarType(), RepR->getUnderlyingInstr());
676 New->setOperand(Idx,
Op);
684 New->insertBefore(DefR);
694 if (!Freq || Freq->IsEstimated)
701 uint32_t NotTaken =
P.getCompl().getNumerator();
702 uint32_t GCD = std::gcd(Taken, NotTaken);
723 "must not contain wide phis, inserts or extracts before conversion");
726 DebugLoc OldDL = OldR.getDebugLoc();
729 for (
const auto &[
I,
Op] :
enumerate(OldR.operands())) {
734 auto *DefR =
Op->getDefiningRecipe();
736 DefR->getParent() == VPB) ||
741 VPValue *Extract = Builder.createNaryOp(Instruction::ExtractElement,
743 OldR.setOperand(
I, Extract);
749 RepR->getOpcode(),
to_vector(RepR->operands()),
nullptr,
750 *RepR, *RepR, OldDL, RepR->getScalarType(),
751 RepR->getUnderlyingInstr());
752 NewR->insertBefore(RepR);
753 RepR->replaceAllUsesWith(NewR);
754 RepR->eraseFromParent();
758 {BranchOnMask->getOperand(0)}, OldDL);
760 BranchOnMask->getExecutionFrequency(), Plan.
getContext()))
761 BOC->setMetadata(LLVMContext::MD_prof, Weights);
762 BranchOnMask->eraseFromParent();
764 VPValue *PredOp = PredPhi->getOperand(0);
767 VPPhi *NewPhi = Builder.createScalarPhi({Poison, PredOp}, OldDL);
769 PredPhi->eraseFromParent();
775 "unexpected unhandled recipe");
788 for (
const auto &[OldBB, NewBB] :
791 for (
auto &&[OldR, NewR] :
793 for (
const auto &[OldV, NewV] :
794 zip_equal(OldR.definedValues(), NewR.definedValues()))
795 Old2NewVPValues[OldV] = NewV;
798 for (
const auto &[
I, OldOp] :
enumerate(NewR.operands())) {
800 if (
auto *NewOp = Old2NewVPValues.
lookup(OldOp))
801 NewR.setOperand(
I, NewOp);
808 "extract indices must be zero");
809 NewR.setOperand(1, IdxLane);
813 "VPPhis expected to have only first lane used");
816 assert(BVUser->getOperand(0) == OldPhi &&
817 "Unexpected first operand of build vector user");
818 BVUser->setOperand(Lane, NewPhi);
835 assert(Predecessor &&
"Replicate region must have a single predecessor");
871 Type *ScalarTy = Phi.getScalarType();
877 Phi.replaceAllUsesWith(BV);
878 BV->setOperand(0, &Phi);
879 BV->insertBefore(
Successor->getFirstNonPhi());
885 for (
int Lane = NumLanes - 1; Lane > 0; --Lane) {
886 const auto &[CurrentLaneEntry, CurrentLaneExiting] =
894 NextLaneEntry = CurrentLaneEntry;
908 assert(BV->getNumOperands() == NumLanes &&
909 "BuildVector must have one operand per lane");
910 for (
const auto &[Idx,
Op] :
enumerate(BV->operands())) {
912 auto DL = ScalarPhi->getDebugLoc();
914 VPValue *Poison = ScalarPhi->getOperand(0);
915 VPValue *PrevVal = Idx == 0 ? Poison : BV->getOperand(Idx - 1);
917 auto *Insert = Builder.createNaryOp(
918 Instruction::InsertElement,
920 Builder.setInsertPoint(ScalarPhi);
921 auto *NewPhi = Builder.createWidenPhi({PrevVal, Insert},
DL);
922 ScalarPhi->replaceAllUsesWith(NewPhi);
923 ScalarPhi->eraseFromParent();
925 BV->replaceAllUsesWith(BV->getOperand(NumLanes - 1));
926 BV->eraseFromParent();
938 if (
Region->isReplicator())
943 "cannot replicate across scalable VFs");
987 if (DefR->user_empty()) {
991 DefR->eraseFromParent();
1000 Def2LaneDefs[DefR] = LaneDefs;
1003 DefR->replaceUsesWithIf(
1005 [DefR](
VPUser *U) { return !U->usesScalars(DefR); });
1011 DefR->replaceUsesWithIf(LaneDefs[0], [DefR, IsSingleScalar](
VPUser *U) {
1012 if (U->usesFirstLaneOnly(DefR) ||
1013 (IsSingleScalar && U->usesScalars(DefR)))
1027 [[maybe_unused]]
bool Match =
1029 assert(Match &&
"unexpected remaining user");
1031 Ext->replaceAllUsesWith(Def2LaneDefs.
at(DefR)[Lane]);
1032 Ext->eraseFromParent();
1034 DefR->eraseFromParent();
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
ReachingDefInfo InstSet & ToRemove
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
This file contains the declarations for the subclasses of Constant, which represent the different fla...
ManagedStatic< HTTPClientCleanup > Cleanup
This file builds on the ADT/GraphTraits.h file to build a generic graph post order iterator.
static bool contains(SmallPtrSetImpl< ConstantExpr * > &Cache, ConstantExpr *Expr, Constant *C)
This file defines the scope_exit class, which executes user-defined cleanup logic at scope exit.
static ConstantInt * getConstantInt(Value *V, const DataLayout &DL)
Extract ConstantInt from value, looking through IntToPtr and PointerNullValue.
This file contains the declarations of different VPlan-related auxiliary helpers.
static void addLaneToStartIndex(VPScalarIVStepsRecipe *Steps, unsigned Lane, VPlan &Plan, VPRecipeBase *InsertPt)
Add a lane offset to the start index of Steps.
static void replicateReplicateRegionsByVF(VPlan &Plan, ElementCount VF, Type *IdxTy)
Collect and dissolve all replicate regions in the vector loop, replicating their blocks and recipes f...
static VPValue * cloneForLane(VPlan &Plan, VPBuilder &Builder, Type *IdxTy, VPSingleDefRecipe *DefR, unsigned Lane, const DenseMap< VPValue *, SmallVector< VPValue * > > &Def2LaneDefs)
Create a single-scalar clone of DefR (must be a VPReplicateRecipe, VPInstruction or VPScalarIVStepsRe...
static MDNode * convertFrequencyToBranchWeights(std::optional< VPExecutionFrequency > Freq, LLVMContext &Ctx)
Converts the frequency Freq with which a block is entered to branch weights for the branch guarding i...
static void addStartIndexForScalarSteps(VPScalarIVStepsRecipe *Steps, unsigned Part, VPlan &Plan)
static void convertRecipesInRegionBlocksToSingleScalar(VPlan &Plan, Type *IdxTy, VPBlockBase *Entry, ElementCount VF)
Convert recipes in region blocks to operate on a single lane 0.
static void dissolveReplicateRegion(VPRegionBlock *Region, ElementCount VF, VPlan &Plan, Type *IdxTy)
Dissolve a single replicate region by replicating its blocks for each lane of VF.
static void processLaneForReplicateRegion(VPlan &Plan, Type *IdxTy, unsigned Lane, VPBasicBlock *OldEntry, VPBasicBlock *NewEntry)
Update recipes in the cloned blocks rooted at NewEntry to match Lane, using the original blocks roote...
static void remapOperands(VPBlockBase *Entry, VPBlockBase *NewEntry, DenseMap< VPValue *, VPValue * > &Old2NewVPValues)
This file contains the declarations of the Vectorization Plan base classes:
static const uint32_t IV[8]
Class for arbitrary precision integers.
LLVM_ABI LLVMContext & getContext() const
Get the context in which this basic block lives.
static DebugLoc getUnknown()
ValueT & at(const_arg_type_t< KeyT > Val)
Return the entry for the specified key, or abort if no such entry exists.
ValueT lookup(const_arg_type_t< KeyT > Val) const
Return the entry for the specified key, or a default constructed value if no such entry exists.
constexpr bool isScalar() const
Exactly one element.
Convenience struct for specifying and reasoning about fast-math flags.
static GEPNoWrapFlags none()
static LLVM_ABI IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
This is an important class for using LLVM in a threaded context.
LLVM_ABI MDNode * createBranchWeights(uint32_t TrueWeight, uint32_t FalseWeight, bool IsExpected=false)
Return metadata containing two branch weights.
RegionT * getParent() const
Get the parent of the Region.
BlockT * getEntry() const
Get the entry BasicBlock of the Region.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
bool contains(ConstPtrType Ptr) const
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
The instances of the Type class are immutable: once they are created, they are never changed.
bool isPointerTy() const
True if this is an instance of PointerType.
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
bool isFloatingPointTy() const
Return true if this is one of the floating-point types.
VPBasicBlock serves as the leaf of the Hierarchical Control-Flow Graph.
RecipeListTy::iterator iterator
Instruction iterators...
iterator_range< iterator > phis()
Returns an iterator range over the PHI-like recipes in the block.
VPBlockBase is the building block of the Hierarchical Control-Flow Graph.
const VPBasicBlock * getEntryBasicBlock() const
void setParent(VPRegionBlock *P)
VPBlockBase * getSingleSuccessor() const
static auto blocksAs(T &&Range)
Return an iterator range over Range with each block cast to BlockTy.
static void connectBlocks(VPBlockBase *From, VPBlockBase *To, unsigned PredIdx=-1u, unsigned SuccIdx=-1u)
Connect VPBlockBases From and To bi-directionally.
static void disconnectBlocks(VPBlockBase *From, VPBlockBase *To)
Disconnect VPBlockBases From and To bi-directionally.
static void insertBlockBefore(VPBlockBase *NewBlock, VPBlockBase *BlockPtr)
Insert disconnected block NewBlock before Blockptr.
static auto blocksOnly(T &&Range)
Return an iterator range over Range which only includes BlockTy blocks.
static std::pair< VPBlockBase *, VPBlockBase * > cloneFrom(VPBlockBase *Entry)
Clone the CFG for all nodes reachable from Entry, including cloning the blocks and their recipes.
static VPBuilderBase getToInsertAfter(VPRecipeBase *R)
static VPSingleDefRecipe * createSingleScalarOp(unsigned Opcode, ArrayRef< VPValue * > Operands, VPValue *Mask, const VPIRFlags &Flags, const VPIRMetadata &Metadata, DebugLoc DL, Type *ResultTy, Instruction *UV)
VPValue * getVPValue(unsigned I)
Returns the VPValue with index I defined by the VPDef.
ArrayRef< VPRecipeValue * > definedValues()
Returns an ArrayRef of the values defined by the VPDef.
BasicBlock * getIRBasicBlock() const
Class to record and manage LLVM IR flags.
This is a concrete Recipe that models a single VPlan-level instruction.
@ WideIVStep
Scale the first operand (vector step) by the second operand (scalar-step).
@ ExtractPenultimateElement
@ ReductionStartVector
Start vector for reductions with 3 operands: the original start value, the identity value for the red...
@ BuildVector
Creates a fixed-width vector containing all operands.
@ CanonicalIVIncrementForPart
VPRecipeBase is a base class modeling a sequence of one or more output IR instructions.
DebugLoc getDebugLoc() const
Returns the debug location of the recipe.
VPRegionBlock represents a collection of VPBasicBlocks and VPRegionBlocks which form a Single-Entry-S...
VPRegionBlock * clone() override
Clone all blocks in the single-entry single-exit region of the block and their recipes without updati...
const VPBlockBase * getEntry() const
bool isReplicator() const
An indicator whether this region is to generate multiple replicated instances of output IR correspond...
Type * getCanonicalIVType() const
Return the type of the canonical IV for loop regions.
A recipe for handling phi nodes of integer and floating-point inductions, producing their scalar valu...
void setStartIndex(VPValue *StartIndex)
Set or add the StartIndex operand.
VPValue * getStartIndex() const
Return the StartIndex, or null if known to be zero, valid only after unrolling.
VPValue * getVFValue() const
Return the number of scalars to produce per unroll part, used to compute StartIndex during unrolling.
VPSingleDefRecipe is a base class for recipes that model a sequence of one or more output IR that def...
VPSingleDefRecipe * clone() override=0
Clone the current recipe.
This class augments VPValue with operands which provide the inverse def-use edges from VPValue's user...
VPValue * getOperand(unsigned N) const
This is the base class of the VPlan Def/Use graph, used for modeling the data flow into,...
Type * getScalarType() const
Returns the scalar type of this VPValue, dispatching based on the concrete subclass.
void replaceAllUsesWith(VPValue *New)
VPlan models a candidate for vectorization, encoding various decisions take to produce efficient outp...
const DataLayout & getDataLayout() const
LLVMContext & getContext() const
VPBasicBlock * getEntry()
VPValue * getTripCount() const
The trip count of the original loop.
VPIRValue * getPoison(Type *Ty)
Return a VPIRValue wrapping a poison value of type Ty.
VPIRValue * getOrAddLiveIn(Value *V)
Gets the live-in VPIRValue for V or adds a new live-in (if none exists yet) for V.
VPIRValue * getZero(Type *Ty)
Return a VPIRValue wrapping the null value of type Ty.
LLVM_ABI_FOR_TEST VPRegionBlock * getVectorLoopRegion()
Returns the VPRegionBlock of the vector loop.
VPSymbolicValue & getUF()
Returns the UF of the vector loop region.
bool hasScalarVFOnly() const
VPIRBasicBlock * getScalarHeader() const
Return the VPIRBasicBlock wrapping the header of the scalar loop.
VPSymbolicValue & getVF()
Returns the VF of the vector loop region.
VPIRValue * getConstantInt(Type *Ty, uint64_t Val, bool IsSigned=false)
Return a VPIRValue wrapping a ConstantInt with the given type and value.
constexpr ScalarTy getFixedValue() const
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
SpecificConstantMatch m_ZeroInt()
Convenience matchers for specific integer values.
match_combine_or< Ty... > m_CombineOr(const Ty &...Ps)
Combine pattern matchers matching any of Ps patterns.
bool match(Val *V, const Pattern &P)
specificval_ty m_Specific(const Value *V)
Match if we have a specific specified value.
cst_pred_ty< is_one > m_One()
Match an integer 1 or a vector with all elements equal to 1.
auto m_Intrinsic(const Ts &...Ops)
Match intrinsic calls like this: m_Intrinsic<Intrinsic::fabs>(m_Value(X))
auto m_ConstantInt()
Match an arbitrary ConstantInt and ignore it.
VPInstruction_match< VPInstruction::ExtractLastLane, VPInstruction_match< VPInstruction::ExtractLastPart, Op0_t > > m_ExtractLastLaneOfLastPart(const Op0_t &Op0)
VPInstruction_match< VPInstruction::ComputeReductionResult, Op0_t > m_ComputeReductionResult(const Op0_t &Op0)
VPInstruction_match< Instruction::InsertElement, Op0_t, Op1_t, Op2_t > m_InsertElement(const Op0_t &Op0, const Op1_t &Op1, const Op2_t &Op2)
VPInstruction_match< VPInstruction::LastActiveLane, Op0_t > m_LastActiveLane(const Op0_t &Op0)
VPInstruction_match< VPInstruction::ExtractLastActive, Op0_t, Op1_t, Op2_t > m_ExtractLastActive(const Op0_t &Op0, const Op1_t &Op1, const Op2_t &Op2)
VPInstruction_match< Instruction::ExtractElement, Op0_t, Op1_t > m_ExtractElement(const Op0_t &Op0, const Op1_t &Op1)
auto m_VPValue()
Match an arbitrary VPValue and ignore it.
VPInstruction_match< VPInstruction::ExtractLastPart, Op0_t > m_ExtractLastPart(const Op0_t &Op0)
VPInstruction_match< VPInstruction::BuildVector > m_BuildVector()
BuildVector is matches only its opcode, w/o matching its operands as the number of operands is not fi...
VPInstruction_match< VPInstruction::ExtractPenultimateElement, Op0_t > m_ExtractPenultimateElement(const Op0_t &Op0)
match_bind< VPInstruction > m_VPInstruction(VPInstruction *&V)
Match a VPInstruction, capturing if we match.
VPInstruction_match< VPInstruction::FirstActiveLane, Op0_t > m_FirstActiveLane(const Op0_t &Op0)
VPInstruction_match< VPInstruction::WideActiveLaneMask, Op_t... > m_WideActiveLaneMask(const Op_t &...Op)
VPInstruction_match< VPInstruction::ExtractLane, Op0_t, Op1_t > m_ExtractLane(const Op0_t &Op0, const Op1_t &Op1)
NodeAddr< PhiNode * > Phi
BranchProbability getExecutionProbability(BlockFrequency Freq)
Returns Freq as a BranchProbability, relative to the full mass.
bool isSingleScalar(const VPValue *VPV)
Returns true if VPV is a single scalar, either because it produces the same value for all lanes or on...
bool onlyFirstPartUsed(const VPValue *Def)
Returns true if only the first part of Def is used by all users.
bool onlyFirstLaneUsed(const VPValue *Def)
Returns true if only the first lane of Def is used by all users.
bool doesGeneratePerAllLanes(const VPRecipeBase *R)
Returns true if R produces scalar values for all VF lanes.
bool onlyScalarValuesOrLanesUsed(const VPValue *Def)
Returns true if only scalar values of Def are used by all users, either directly or by extracting a c...
bool onlyScalarValuesUsed(const VPValue *Def)
Returns true if only scalar values of Def are used by all users.
LLVM_ABI_FOR_TEST bool isUniformAcrossVFsAndUFs(const VPValue *V)
Checks if V is uniform across all VF lanes and UF parts.
This is an optimization pass for GlobalISel generic memory operations.
detail::zippy< detail::zip_shortest, T, U, Args... > zip(T &&t, U &&u, Args &&...args)
zip iterator for two or more iteratable types.
detail::zippy< detail::zip_first, T, U, Args... > zip_equal(T &&t, U &&u, Args &&...args)
zip iterator that assumes that all iteratees have the same length.
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are tuples (A, B,...
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
VPBuilderBase<> VPBuilder
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
iterator_range< df_iterator< VPBlockShallowTraversalWrapper< VPBlockBase * > > > vp_depth_first_shallow(VPBlockBase *G)
Returns an iterator range to traverse the graph starting at G in depth-first order.
iterator_range< df_iterator< VPBlockDeepTraversalWrapper< VPBlockBase * > > > vp_depth_first_deep(VPBlockBase *G)
Returns an iterator range to traverse the graph starting at G in depth-first order while traversing t...
auto map_range(ContainerTy &&C, FuncTy F)
Return a range that applies F to the elements of C.
detail::concat_range< ValueT, RangeTs... > concat(RangeTs &&...Ranges)
Returns a concatenated range across two or more ranges.
auto make_isa_range(RangeT &&Range)
Return a range over Range containing only elements for which isa<T> holds, casting each of them to T.
auto dyn_cast_or_null(const Y &Val)
auto reverse(ContainerTy &&C)
bool isa_and_present(const Y &Val)
isa_and_present<X> - Functionally identical to isa, except that a null value is accepted.
SmallVector< ValueTypeFromRangeType< R >, Size > to_vector(R &&Range)
Given a range of type R, iterate the entire range and return a SmallVector with elements of the vecto...
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
DWARFExpression::Operation Op
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
constexpr detail::CastFunc< To > CastTo