34 GetIntOrFpInductionDescriptor,
39 for (
VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(RPOT)) {
41 auto EndIter = Term ? Term->getIterator() : VPBB->end();
46 VPValue *VPV = Ingredient.getVPSingleValue();
50 if (
auto *VPPhi = dyn_cast<VPWidenPHIRecipe>(&Ingredient)) {
51 auto *Phi = cast<PHINode>(VPPhi->getUnderlyingValue());
52 if (
const auto *II = GetIntOrFpInductionDescriptor(Phi)) {
53 VPValue *Start = Plan->getVPValueOrAddLiveIn(II->getStartValue());
58 Plan->addVPValue(Phi, VPPhi);
62 assert(isa<VPInstruction>(&Ingredient) &&
63 "only VPInstructions expected here");
64 assert(!isa<PHINode>(Inst) &&
"phis should be handled above");
66 if (
LoadInst *Load = dyn_cast<LoadInst>(Inst)) {
68 *Load, Ingredient.getOperand(0),
nullptr ,
70 }
else if (
StoreInst *Store = dyn_cast<StoreInst>(Inst)) {
72 *Store, Ingredient.getOperand(1), Ingredient.getOperand(0),
73 nullptr ,
false ,
false );
76 }
else if (
CallInst *CI = dyn_cast<CallInst>(Inst)) {
80 }
else if (
SelectInst *SI = dyn_cast<SelectInst>(Inst)) {
82 }
else if (
auto *CI = dyn_cast<CastInst>(Inst)) {
84 CI->getOpcode(), Ingredient.getOperand(0), CI->getType(), CI);
95 "Only recpies with zero or one defined values expected");
96 Ingredient.eraseFromParent();
103 bool Changed =
false;
107 for (
VPRegionBlock *VPR : VPBlockUtils::blocksOnly<VPRegionBlock>(Iter)) {
114 for (
auto &Recipe : *VPBB) {
116 if (
auto *Def =
Op->getDefiningRecipe())
117 WorkList.
insert(std::make_pair(VPBB, Def));
123 for (
unsigned I = 0;
I != WorkList.
size(); ++
I) {
126 std::tie(SinkTo, SinkCandidate) = WorkList[
I];
127 if (SinkCandidate->
getParent() == SinkTo ||
131 if (
auto *RepR = dyn_cast<VPReplicateRecipe>(SinkCandidate)) {
132 if (!ScalarVFOnly && RepR->isUniform())
134 }
else if (!isa<VPScalarIVStepsRecipe>(SinkCandidate))
137 bool NeedsDuplicating =
false;
142 auto CanSinkWithUser = [SinkTo, &NeedsDuplicating,
143 SinkCandidate](
VPUser *U) {
144 auto *UI = dyn_cast<VPRecipeBase>(U);
147 if (UI->getParent() == SinkTo)
152 return NeedsDuplicating && isa<VPReplicateRecipe>(SinkCandidate);
157 if (NeedsDuplicating) {
161 cast<VPReplicateRecipe>(SinkCandidate)->getUnderlyingValue());
165 Clone->insertBefore(SinkCandidate);
167 Clone, [SinkTo](
VPUser &U,
unsigned) {
168 return cast<VPRecipeBase>(&U)->getParent() != SinkTo;
173 if (
auto *Def =
Op->getDefiningRecipe())
174 WorkList.
insert(std::make_pair(SinkTo, Def));
183 auto *EntryBB = dyn_cast<VPBasicBlock>(R->getEntry());
184 if (!EntryBB || EntryBB->size() != 1 ||
185 !isa<VPBranchOnMaskRecipe>(EntryBB->begin()))
188 return cast<VPBranchOnMaskRecipe>(&*EntryBB->begin())->getOperand(0);
193 auto *EntryBB = cast<VPBasicBlock>(R->getEntry());
194 if (EntryBB->getNumSuccessors() != 2)
197 auto *Succ0 = dyn_cast<VPBasicBlock>(EntryBB->getSuccessors()[0]);
198 auto *Succ1 = dyn_cast<VPBasicBlock>(EntryBB->getSuccessors()[1]);
199 if (!Succ0 || !Succ1)
202 if (Succ0->getNumSuccessors() + Succ1->getNumSuccessors() != 1)
204 if (Succ0->getSingleSuccessor() == Succ1)
206 if (Succ1->getSingleSuccessor() == Succ0)
221 for (
VPRegionBlock *Region1 : VPBlockUtils::blocksOnly<VPRegionBlock>(
223 if (!Region1->isReplicator())
225 auto *MiddleBasicBlock =
226 dyn_cast_or_null<VPBasicBlock>(Region1->getSingleSuccessor());
227 if (!MiddleBasicBlock || !MiddleBasicBlock->empty())
231 dyn_cast_or_null<VPRegionBlock>(MiddleBasicBlock->getSingleSuccessor());
232 if (!Region2 || !Region2->isReplicator())
237 if (!Mask1 || Mask1 != Mask2)
240 assert(Mask1 && Mask2 &&
"both region must have conditions");
246 if (DeletedRegions.
contains(Region1))
248 auto *MiddleBasicBlock = cast<VPBasicBlock>(Region1->getSingleSuccessor());
249 auto *Region2 = cast<VPRegionBlock>(MiddleBasicBlock->getSingleSuccessor());
253 if (!Then1 || !Then2)
272 cast<VPPredInstPHIRecipe>(&Phi1ToMove)->getOperand(0);
273 VPValue *Phi1ToMoveV = Phi1ToMove.getVPSingleValue();
275 auto *UI = dyn_cast<VPRecipeBase>(&U);
276 return UI && UI->getParent() == Then2;
279 Phi1ToMove.moveBefore(*Merge2, Merge2->begin());
288 DeletedRegions.
insert(Region1);
293 return !DeletedRegions.
empty();
300 std::string RegionName = (
Twine(
"pred.") + Instr->getOpcodeName()).str();
301 assert(Instr->getParent() &&
"Predicated instruction not in any basic block");
302 auto *BlockInMask = PredRecipe->
getMask();
334 for (
VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
337 if (
auto *RepR = dyn_cast<VPReplicateRecipe>(&R)) {
338 if (RepR->isPredicated())
364 bool ShouldSimplify =
true;
365 while (ShouldSimplify) {
368 ShouldSimplify |= VPlanTransforms::mergeBlocksIntoPredecessors(Plan);
371bool VPlanTransforms::mergeBlocksIntoPredecessors(
VPlan &Plan) {
373 for (
VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
376 dyn_cast_or_null<VPBasicBlock>(VPBB->getSinglePredecessor());
377 if (PredVPBB && PredVPBB->getNumSuccessors() == 1)
382 VPBasicBlock *PredVPBB = cast<VPBasicBlock>(VPBB->getSinglePredecessor());
384 R.moveBefore(*PredVPBB, PredVPBB->
end());
386 auto *ParentRegion = cast_or_null<VPRegionBlock>(VPBB->getParent());
387 if (ParentRegion && ParentRegion->getExiting() == VPBB)
388 ParentRegion->setExiting(PredVPBB);
389 for (
auto *Succ :
to_vector(VPBB->successors())) {
395 return !WorkList.empty();
398void VPlanTransforms::removeRedundantInductionCasts(
VPlan &Plan) {
400 auto *
IV = dyn_cast<VPWidenIntOrFpInductionRecipe>(&Phi);
401 if (!
IV ||
IV->getTruncInst())
412 auto &Casts =
IV->getInductionDescriptor().getCastInsts();
416 for (
auto *U : FindMyCast->
users()) {
417 auto *UserCast = cast<VPRecipeBase>(U);
418 if (UserCast->getNumDefinedValues() == 1 &&
419 UserCast->getVPSingleValue()->getUnderlyingValue() == IRCast) {
420 FoundUserCast = UserCast;
430void VPlanTransforms::removeRedundantCanonicalIVs(
VPlan &Plan) {
434 WidenNewIV = dyn_cast<VPWidenCanonicalIVRecipe>(U);
444 auto *WidenOriginalIV = dyn_cast<VPWidenIntOrFpInductionRecipe>(&Phi);
446 if (!WidenOriginalIV || !WidenOriginalIV->isCanonical() ||
447 WidenOriginalIV->getScalarType() != WidenNewIV->
getScalarType())
454 if (
any_of(WidenOriginalIV->users(),
455 [WidenOriginalIV](
VPUser *U) {
456 return !U->usesScalars(WidenOriginalIV);
466void VPlanTransforms::removeDeadRecipes(
VPlan &Plan) {
476 [](
VPValue *V) { return V->getNumUsers(); }))
481 auto *RepR = dyn_cast<VPReplicateRecipe>(&R);
482 bool IsConditionalAssume =
483 RepR && RepR->isPredicated() &&
484 match(RepR->getUnderlyingInstr(), m_Intrinsic<Intrinsic::assume>());
485 if (
R.mayHaveSideEffects() && !IsConditionalAssume)
502 if (!CanonicalIV->
isCanonical(
ID.getKind(), StartV, Step, TruncTy)) {
504 TruncI ? TruncI->
getType() :
nullptr);
509 HeaderVPBB->
insert(Steps, IP);
518 auto *WideIV = dyn_cast<VPWidenIntOrFpInductionRecipe>(&Phi);
521 if (HasOnlyVectorVFs &&
none_of(WideIV->users(), [WideIV](
VPUser *U) {
522 return U->usesScalars(WideIV);
528 Plan,
ID, SE, WideIV->getTruncInst(), WideIV->getPHINode()->getType(),
529 WideIV->getStartValue(), WideIV->getStepValue());
533 WideIV->replaceUsesWithIf(
534 Steps, [HasOnlyVectorVFs, WideIV](
VPUser &U,
unsigned) {
535 return !HasOnlyVectorVFs ||
U.usesScalars(WideIV);
540void VPlanTransforms::removeRedundantExpandSCEVRecipes(
VPlan &Plan) {
545 auto *ExpR = dyn_cast<VPExpandSCEVRecipe>(&R);
549 auto I = SCEV2VPV.
insert({ExpR->getSCEV(), ExpR});
552 ExpR->replaceAllUsesWith(
I.first->second);
553 ExpR->eraseFromParent();
558 VPInstruction *Not = dyn_cast<VPInstruction>(Term->getOperand(0));
569 assert(Plan.
hasVF(BestVF) &&
"BestVF is not available in Plan");
570 assert(Plan.
hasUF(BestUF) &&
"BestUF is not available in Plan");
573 auto *Term = dyn_cast<VPInstruction>(&ExitingVPBB->
back());
590 if (TripCount->
isZero() ||
598 Term->eraseFromParent();
609 auto *
Region = dyn_cast_or_null<VPRegionBlock>(R->getParent()->getParent());
612 Region->getNumPredecessors() == 1 &&
"Expected SESE region!");
613 assert(R->getParent()->size() == 1 &&
614 "A recipe in an original replicator region must be the only "
615 "recipe in its block");
628 for (
auto &R : *
A->getParent()) {
638 if (ParentA == ParentB)
639 return LocalComesBefore(
A,
B);
642 "No replicate regions expected at this point");
644 "No replicate regions expected at this point");
659 auto TryToPushSinkCandidate = [&](
VPRecipeBase *SinkCandidate) {
662 if (SinkCandidate == Previous)
665 if (isa<VPHeaderPHIRecipe>(SinkCandidate) ||
666 !Seen.
insert(SinkCandidate).second ||
670 if (SinkCandidate->mayHaveSideEffects())
679 for (
unsigned I = 0;
I != WorkList.
size(); ++
I) {
682 "only recipes with a single defined value expected");
685 if (
auto *R = dyn_cast<VPRecipeBase>(
User))
686 if (!TryToPushSinkCandidate(R))
698 if (SinkCandidate == FOR)
701 SinkCandidate->moveAfter(Previous);
702 Previous = SinkCandidate;
715 if (
auto *FOR = dyn_cast<VPFirstOrderRecurrencePHIRecipe>(&R))
720 VPRecipeBase *Previous = FOR->getBackedgeValue()->getDefiningRecipe();
723 while (
auto *PrevPhi =
724 dyn_cast_or_null<VPFirstOrderRecurrencePHIRecipe>(Previous)) {
725 assert(PrevPhi->getParent() == FOR->getParent());
727 Previous = PrevPhi->getBackedgeValue()->getDefiningRecipe();
736 if (isa<VPHeaderPHIRecipe>(Previous))
741 auto *RecurSplice = cast<VPInstruction>(
743 {FOR, FOR->getBackedgeValue()}));
745 FOR->replaceAllUsesWith(RecurSplice);
748 RecurSplice->setOperand(0, FOR);
756 auto *PhiR = dyn_cast<VPReductionPHIRecipe>(&R);
767 for (
unsigned I = 0;
I != Worklist.
size(); ++
I) {
769 if (
auto *RecWithFlags =
771 RecWithFlags->dropPoisonGeneratingFlags();
775 auto *UserRecipe = dyn_cast<VPRecipeBase>(U);
778 for (
VPValue *V : UserRecipe->definedValues())
789 auto *
C = dyn_cast<ConstantInt>(V->getLiveInIRValue());
790 return C &&
C->isOne();
795 if (
auto *WidenR = dyn_cast<VPWidenRecipe>(&R))
796 return WidenR->getUnderlyingInstr()->getOpcode();
797 if (
auto *WidenC = dyn_cast<VPWidenCastRecipe>(&R))
798 return WidenC->getOpcode();
799 if (
auto *RepR = dyn_cast<VPReplicateRecipe>(&R))
800 return RepR->getUnderlyingInstr()->getOpcode();
801 if (
auto *VPI = dyn_cast<VPInstruction>(&R))
802 return VPI->getOpcode();
809 case Instruction::Mul: {
813 return R.getVPSingleValue()->replaceAllUsesWith(
B);
815 return R.getVPSingleValue()->replaceAllUsesWith(
A);
818 case Instruction::Trunc: {
819 VPRecipeBase *Ext = R.getOperand(0)->getDefiningRecipe();
823 if (ExtOpcode != Instruction::ZExt && ExtOpcode != Instruction::SExt)
826 VPValue *Trunc = R.getVPSingleValue();
829 if (TruncTy == ATy) {
834 VPC->insertBefore(&R);
838 VPC->insertBefore(&R);
847 auto *R = dyn_cast<VPRecipeBase>(U);
850 for (
VPValue *VPV : R->definedValues())
866 for (
VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(RPOT)) {
879 unsigned NumProcessedRecipes = 0;
888 for (
VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
895 VPValue *ResultVPV = R.getVPSingleValue();
897 unsigned NewResSizeInBits = MinBWs.
lookup(UI);
898 if (!NewResSizeInBits)
902 NumProcessedRecipes++;
908 if (isa<VPReplicateRecipe, VPWidenCastRecipe>(&R)) {
920 auto *UV = dyn_cast_or_null<Instruction>(
Op->getUnderlyingValue());
923 return !isa<VPWidenRecipe, VPWidenSelectRecipe>(U);
927 ProcessedTruncs[
Op] =
nullptr;
928 NumProcessedRecipes += 1;
938 if (OldResSizeInBits == NewResSizeInBits)
940 assert(OldResSizeInBits > NewResSizeInBits &&
"Nothing to shrink?");
941 (void)OldResSizeInBits;
946 unsigned StartIdx = isa<VPWidenSelectRecipe>(&R) ? 1 : 0;
947 for (
unsigned Idx = StartIdx;
Idx != R.getNumOperands(); ++
Idx) {
948 auto *
Op = R.getOperand(
Idx);
949 unsigned OpSizeInBits =
951 if (OpSizeInBits == NewResSizeInBits)
953 assert(OpSizeInBits > NewResSizeInBits &&
"nothing to truncate");
954 auto [ProcessedIter, IterIsEmpty] =
955 ProcessedTruncs.
insert({
Op,
nullptr});
959 : ProcessedIter->second;
960 R.setOperand(
Idx, NewOp);
963 ProcessedIter->second = NewOp;
964 if (!
Op->isLiveIn()) {
969 auto *OpInst = dyn_cast<Instruction>(
Op->getLiveInIRValue());
970 bool IsContained = MinBWs.
contains(OpInst);
971 NumProcessedRecipes += IsContained;
979 if (
auto *VPW = dyn_cast<VPRecipeWithIRFlags>(&R))
980 VPW->dropPoisonGeneratingFlags();
984 Ext->insertAfter(&R);
986 Ext->setOperand(0, ResultVPV);
990 assert(MinBWs.
size() == NumProcessedRecipes &&
991 "some entries in MinBWs haven't been processed");
995 removeRedundantCanonicalIVs(Plan);
996 removeRedundantInductionCasts(Plan);
998 optimizeInductions(Plan, SE);
1000 removeDeadRecipes(Plan);
1004 removeRedundantExpandSCEVRecipes(Plan);
1005 mergeBlocksIntoPredecessors(Plan);
1046 VPValue *StartV = CanonicalIVPHI->getStartValue();
1048 auto *CanonicalIVIncrement =
1049 cast<VPInstruction>(CanonicalIVPHI->getBackedgeValue());
1052 CanonicalIVIncrement->dropPoisonGeneratingFlags();
1053 DebugLoc DL = CanonicalIVIncrement->getDebugLoc();
1063 VPValue *TripCount, *IncrementValue;
1068 IncrementValue = CanonicalIVIncrement;
1074 IncrementValue = CanonicalIVPHI;
1085 DL,
"active.lane.mask.entry");
1090 LaneMaskPhi->insertAfter(CanonicalIVPHI);
1096 auto *InLoopIncrement =
1098 {IncrementValue}, {
false,
false},
DL);
1100 {InLoopIncrement, TripCount},
DL,
1101 "active.lane.mask.next");
1102 LaneMaskPhi->addOperand(ALM);
1113 VPlan &Plan,
bool UseActiveLaneMaskForControlFlow,
1116 UseActiveLaneMaskForControlFlow) &&
1117 "DataAndControlFlowWithoutRuntimeCheck implies "
1118 "UseActiveLaneMaskForControlFlow");
1120 auto FoundWidenCanonicalIVUser =
1122 [](
VPUser *U) { return isa<VPWidenCanonicalIVRecipe>(U); });
1123 assert(FoundWidenCanonicalIVUser &&
1124 "Must have widened canonical IV when tail folding!");
1125 auto *WideCanonicalIV =
1126 cast<VPWidenCanonicalIVRecipe>(*FoundWidenCanonicalIVUser);
1128 if (UseActiveLaneMaskForControlFlow) {
1134 nullptr,
"active.lane.mask");
1143 auto *CompareToReplace = dyn_cast<VPInstruction>(U);
1144 if (!CompareToReplace ||
1145 CompareToReplace->getOpcode() != Instruction::ICmp ||
1147 CompareToReplace->getOperand(1) != BTC)
1150 assert(CompareToReplace->getOperand(0) == WideCanonicalIV &&
1151 "WidenCanonicalIV must be the first operand of the compare");
1153 CompareToReplace->eraseFromParent();
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
ReachingDefAnalysis InstSet & ToRemove
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
static bool isConstantOne(const Value *Val)
isConstantOne - Return true only if val is constant int 1
This file builds on the ADT/GraphTraits.h file to build a generic graph post order iterator.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file implements a set that has insertion order iteration characteristics.
This file implements dominator tree analysis for a single level of a VPlan's H-CFG.
static const uint32_t IV[8]
LLVM Basic Block Representation.
const Function * getParent() const
Return the enclosing method, or null if none.
This class represents a function call, abstracting a target machine's calling convention.
@ ICMP_ULE
unsigned less or equal
static ConstantInt * getTrue(LLVMContext &Context)
This class represents an Operation in the Expression.
bool contains(const_arg_type_t< KeyT > Val) const
Return true if the specified key is in the map, false otherwise.
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Core dominator tree base class.
void recalculate(ParentType &Func)
recalculate - compute a dominator tree for the given function
bool properlyDominates(const DomTreeNodeBase< NodeT > *A, const DomTreeNodeBase< NodeT > *B) const
properlyDominates - Returns true iff A dominates B and A != B.
static constexpr ElementCount getFixed(ScalarTy MinVal)
an instruction for type-safe pointer arithmetic to access elements of arrays and structs
A struct for saving information about induction variables.
static IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
This is an important class for using LLVM in a threaded context.
An instruction for reading from memory.
This class implements a map that also provides access to all stored values in a deterministic order.
bool contains(const KeyT &Key) const
ValueT lookup(const KeyT &Key) const
An interface layer with SCEV used to manage how we see SCEV expressions for values in the context of ...
ScalarEvolution * getSE() const
Returns the ScalarEvolution analysis used.
The RecurrenceDescriptor is used to identify recurrences variables in a loop.
RecurKind getRecurrenceKind() const
This class represents an analyzed expression in the program.
bool isZero() const
Return true if the expression is a constant zero.
Type * getType() const
Return the LLVM type of this SCEV expression.
The main scalar evolution driver.
const SCEV * getConstant(ConstantInt *V)
bool isKnownPredicate(ICmpInst::Predicate Pred, const SCEV *LHS, const SCEV *RHS)
Test if the given expression is known to satisfy the condition described by Pred, LHS,...
LLVMContext & getContext() const
This class represents the LLVM 'select' instruction.
A vector that has set insertion semantics.
size_type size() const
Determine the number of elements in the SetVector.
bool empty() const
Determine if the SetVector is empty or not.
bool insert(const value_type &X)
Insert a new element into the SetVector.
bool contains(const key_type &key) const
Check if the SetVector contains the given key.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
A SetVector that performs no allocations if smaller than a certain size.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
An instruction for storing to memory.
Provides information about what library functions are available for the current target.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
The instances of the Type class are immutable: once they are created, they are never changed.
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
bool isIntegerTy() const
True if this is an instance of IntegerType.
A recipe for generating the active lane mask for the vector loop that is used to predicate the vector...
VPBasicBlock serves as the leaf of the Hierarchical Control-Flow Graph.
void appendRecipe(VPRecipeBase *Recipe)
Augment the existing recipes of a VPBasicBlock with an additional Recipe as the last recipe.
iterator_range< iterator > phis()
Returns an iterator range over the PHI-like recipes in the block.
iterator getFirstNonPhi()
Return the position of the first non-phi node recipe in the block.
VPBasicBlock * splitAt(iterator SplitAt)
Split current block at SplitAt by inserting a new block between the current block and its successors ...
VPRecipeBase * getTerminator()
If the block has multiple successors, return the branch recipe terminating the block.
const VPRecipeBase & back() const
void insert(VPRecipeBase *Recipe, iterator InsertPt)
VPBlockBase is the building block of the Hierarchical Control-Flow Graph.
VPRegionBlock * getParent()
const VPBasicBlock * getExitingBasicBlock() const
VPBlockBase * getSinglePredecessor() const
const VPBasicBlock * getEntryBasicBlock() const
VPBlockBase * getSingleSuccessor() const
const VPBlocksTy & getSuccessors() const
static void insertTwoBlocksAfter(VPBlockBase *IfTrue, VPBlockBase *IfFalse, VPBlockBase *BlockPtr)
Insert disconnected VPBlockBases IfTrue and IfFalse after BlockPtr.
static void disconnectBlocks(VPBlockBase *From, VPBlockBase *To)
Disconnect VPBlockBases From and To bi-directionally.
static void connectBlocks(VPBlockBase *From, VPBlockBase *To)
Connect VPBlockBases From and To bi-directionally.
A recipe for generating conditional branches on the bits of a mask.
VPlan-based builder utility analogous to IRBuilder.
VPInstruction * createOverflowingOp(unsigned Opcode, std::initializer_list< VPValue * > Operands, VPRecipeWithIRFlags::WrapFlagsTy WrapFlags, DebugLoc DL, const Twine &Name="")
VPValue * createNot(VPValue *Operand, DebugLoc DL, const Twine &Name="")
VPValue * createNaryOp(unsigned Opcode, ArrayRef< VPValue * > Operands, Instruction *Inst=nullptr, const Twine &Name="")
Create an N-ary operation with Opcode, Operands and set Inst as its underlying Instruction.
void setInsertPoint(VPBasicBlock *TheBB)
This specifies that created VPInstructions should be appended to the end of the specified block.
Canonical scalar induction phi of the vector loop.
bool isCanonical(InductionDescriptor::InductionKind Kind, VPValue *Start, VPValue *Step, Type *Ty) const
Check if the induction described by Kind, /p Start and Step is canonical, i.e.
unsigned getNumDefinedValues() const
Returns the number of values defined by the VPDef.
VPValue * getVPSingleValue()
Returns the only VPValue defined by the VPDef.
A recipe for converting the canonical IV value to the corresponding value of an IV with different sta...
This is a concrete Recipe that models a single VPlan-level instruction.
unsigned getOpcode() const
@ FirstOrderRecurrenceSplice
@ CanonicalIVIncrementForPart
@ CalculateTripCountMinusVF
VPPredInstPHIRecipe is a recipe for generating the phi nodes needed when control converges back from ...
VPRecipeBase is a base class modeling a sequence of one or more output IR instructions.
bool mayReadOrWriteMemory() const
Returns true if the recipe may read from or write to memory.
bool mayHaveSideEffects() const
Returns true if the recipe may have side-effects.
Instruction * getUnderlyingInstr()
Returns the underlying instruction, if the recipe is a VPValue or nullptr otherwise.
VPBasicBlock * getParent()
void moveBefore(VPBasicBlock &BB, iplist< VPRecipeBase >::iterator I)
Unlink this recipe and insert into BB before I.
void insertBefore(VPRecipeBase *InsertPos)
Insert an unlinked recipe into a basic block immediately before the specified recipe.
void insertAfter(VPRecipeBase *InsertPos)
Insert an unlinked Recipe into a basic block immediately after the specified Recipe.
iplist< VPRecipeBase >::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
VPRegionBlock represents a collection of VPBasicBlocks and VPRegionBlocks which form a Single-Entry-S...
const VPBlockBase * getEntry() const
VPReplicateRecipe replicates a given instruction producing multiple scalar copies of the original sca...
VPValue * getMask()
Return the mask of a predicated VPReplicateRecipe.
A recipe for handling phi nodes of integer and floating-point inductions, producing their scalar valu...
An analysis for type-inference for VPValues.
LLVMContext & getContext()
Return the LLVMContext used by the analysis.
Type * inferScalarType(const VPValue *V)
Infer the type of V. Returns the scalar type of V.
This class augments VPValue with operands which provide the inverse def-use edges from VPValue's user...
void setOperand(unsigned I, VPValue *New)
operand_iterator op_end()
operand_iterator op_begin()
VPValue * getOperand(unsigned N) const
Value * getUnderlyingValue()
Return the underlying Value attached to this VPValue.
VPRecipeBase * getDefiningRecipe()
Returns the recipe defining this VPValue or nullptr if it is not defined by a recipe,...
void replaceAllUsesWith(VPValue *New)
unsigned getNumUsers() const
Value * getLiveInIRValue()
Returns the underlying IR value, if this VPValue is defined outside the scope of VPlan.
void replaceUsesWithIf(VPValue *New, llvm::function_ref< bool(VPUser &U, unsigned Idx)> ShouldReplace)
Go through the uses list for this VPValue and make each use point to New if the callback ShouldReplac...
A recipe for widening Call instructions.
A Recipe for widening the canonical induction variable of the vector loop.
const Type * getScalarType() const
Returns the scalar type of the induction.
VPWidenCastRecipe is a recipe to create vector cast instructions.
A recipe for handling GEP instructions.
A recipe for handling phi nodes of integer and floating-point inductions, producing their vector valu...
A Recipe for widening load/store operations.
VPWidenRecipe is a recipe for producing a copy of vector type its ingredient.
VPlan models a candidate for vectorization, encoding various decisions take to produce efficient outp...
VPBasicBlock * getEntry()
VPValue * getTripCount() const
The trip count of the original loop.
VPValue * getOrCreateBackedgeTakenCount()
The backedge taken count of the original loop.
VPValue * getVPValueOrAddLiveIn(Value *V)
Gets the VPValue for V or adds a new live-in (if none exists yet) for V.
VPRegionBlock * getVectorLoopRegion()
Returns the VPRegionBlock of the vector loop.
bool hasVF(ElementCount VF)
bool hasUF(unsigned UF) const
void setVF(ElementCount VF)
bool hasScalarVFOnly() const
VPCanonicalIVPHIRecipe * getCanonicalIV()
Returns the canonical induction recipe of the vector loop.
Type * getType() const
All values are typed, get the type of this value.
void setName(const Twine &Name)
Change the name of the value.
StringRef getName() const
Return a constant reference to the value's name.
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
An efficient, type-erasing, non-owning reference to a callable.
self_iterator getIterator()
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ C
The default llvm calling convention, compatible with C.
bool match(Val *V, const Pattern &P)
VPValue * getOrCreateVPValueForSCEVExpr(VPlan &Plan, const SCEV *Expr, ScalarEvolution &SE)
Get or create a VPValue that corresponds to the expansion of Expr.
bool onlyFirstLaneUsed(VPValue *Def)
Returns true if only the first lane of Def is used.
This is an optimization pass for GlobalISel generic memory operations.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Intrinsic::ID getVectorIntrinsicIDForCall(const CallInst *CI, const TargetLibraryInfo *TLI)
Returns intrinsic ID for call.
const SCEV * createTripCountSCEV(Type *IdxTy, PredicatedScalarEvolution &PSE, Loop *OrigLoop)
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
iterator_range< df_iterator< VPBlockDeepTraversalWrapper< VPBlockBase * > > > vp_depth_first_deep(VPBlockBase *G)
Returns an iterator range to traverse the graph starting at G in depth-first order while traversing t...
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
auto reverse(ContainerTy &&C)
void sort(IteratorTy Start, IteratorTy End)
std::unique_ptr< VPlan > VPlanPtr
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
SmallVector< ValueTypeFromRangeType< R >, Size > to_vector(R &&Range)
Given a range of type R, iterate the entire range and return a SmallVector with elements of the vecto...
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
auto drop_end(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the last N elements excluded.
RecurKind
These are the kinds of recurrences that we support.
@ Mul
Product of integers.
DWARFExpression::Operation Op
BasicBlock * SplitBlock(BasicBlock *Old, BasicBlock::iterator SplitPt, DominatorTree *DT, LoopInfo *LI=nullptr, MemorySSAUpdater *MSSAU=nullptr, const Twine &BBName="", bool Before=false)
Split the specified block at the specified instruction.
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
@ DataAndControlFlowWithoutRuntimeCheck
Use predicate to control both data and control flow, but modify the trip count so that a runtime over...
A recipe for handling first-order recurrence phis.
A recipe for widening select instructions.