Go to the documentation of this file.
25 #ifndef LLVM_TRANSFORMS_VECTORIZE_VPLAN_H
26 #define LLVM_TRANSFORMS_VECTORIZE_VPLAN_H
52 class InductionDescriptor;
53 class InnerLoopVectorizer;
57 class RecurrenceDescriptor;
62 class VPReplicateRecipe;
91 "Both Start and End should have the same scalable flag");
93 "Expected Start to be a power of 2");
128 VPLane(
unsigned Lane,
Kind LaneKind) : Lane(Lane), LaneKind(LaneKind) {}
141 return VPLane(LaneOffset, LaneKind);
251 unsigned CacheIdx =
Instance.Lane.mapToCacheIndex(
VF);
252 return Instance.Part <
I->second.size() &&
253 CacheIdx <
I->second[
Instance.Part].size() &&
269 "need to overwrite existing value");
270 Iter->second[Part] = V;
276 auto &PerPartVec = Iter.first->second;
277 while (PerPartVec.size() <=
Instance.Part)
278 PerPartVec.emplace_back();
279 auto &Scalars = PerPartVec[
Instance.Part];
280 unsigned CacheIdx =
Instance.Lane.mapToCacheIndex(
VF);
281 while (Scalars.size() <= CacheIdx)
282 Scalars.push_back(
nullptr);
283 assert(!Scalars[CacheIdx] &&
"should overwrite existing value");
284 Scalars[CacheIdx] = V;
291 "need to overwrite existing value");
293 "need to overwrite existing value");
294 unsigned CacheIdx =
Instance.Lane.mapToCacheIndex(
VF);
296 "need to overwrite existing value");
297 Iter->second[
Instance.Part][CacheIdx] = V;
397 const unsigned char SubclassID;
424 VPlan *Plan =
nullptr;
434 assert(Predecessor &&
"Cannot add nullptr predecessor!");
435 Predecessors.push_back(Predecessor);
440 auto Pos =
find(Predecessors, Predecessor);
441 assert(Pos &&
"Predecessor does not exist");
442 Predecessors.
erase(Pos);
448 assert(Pos &&
"Successor does not exist");
449 Successors.
erase(Pos);
461 using VPBlockTy =
enum { VPBasicBlockSC, VPRegionBlockSC };
512 return (Successors.size() == 1 ? *Successors.begin() :
nullptr);
518 return (Predecessors.size() == 1 ? *Predecessors.begin() :
nullptr);
585 assert(Successors.empty() &&
"Setting one successor when others exist.");
595 assert(Successors.empty() &&
"Setting two successors when others exist.");
596 assert(Condition &&
"Setting two successors without condition!");
598 appendSuccessor(IfTrue);
599 appendSuccessor(IfFalse);
606 assert(Predecessors.empty() &&
"Block predecessors already set.");
607 for (
auto *Pred : NewPreds)
608 appendPredecessor(Pred);
638 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
685 template <
typename IterT>
767 return Def->getVPDefID() == VPRecipeBase::VPInstructionSC ||
768 Def->getVPDefID() == VPRecipeBase::VPWidenSC ||
769 Def->getVPDefID() == VPRecipeBase::VPWidenCallSC ||
770 Def->getVPDefID() == VPRecipeBase::VPWidenSelectSC ||
771 Def->getVPDefID() == VPRecipeBase::VPWidenGEPSC ||
772 Def->getVPDefID() == VPRecipeBase::VPBlendSC ||
773 Def->getVPDefID() == VPRecipeBase::VPInterleaveSC ||
774 Def->getVPDefID() == VPRecipeBase::VPReplicateSC ||
775 Def->getVPDefID() == VPRecipeBase::VPReductionSC ||
776 Def->getVPDefID() == VPRecipeBase::VPBranchOnMaskSC ||
777 Def->getVPDefID() == VPRecipeBase::VPWidenMemoryInstructionSC;
791 Instruction::OtherOpsEnd + 1,
804 typedef unsigned char OpcodeTy;
838 return R->getVPDefID() == VPRecipeBase::VPInstructionSC;
844 auto *
R = dyn_cast<VPRecipeBase>(U);
845 return R &&
R->getVPDefID() == VPRecipeBase::VPInstructionSC;
848 return R->getVPDefID() == VPRecipeBase::VPInstructionSC;
858 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
872 Opcode == Instruction::Invoke || Opcode ==
SLPStore;
880 case Instruction::Br:
882 case Instruction::Switch:
883 case Instruction::IndirectBr:
884 case Instruction::Resume:
885 case Instruction::CatchRet:
886 case Instruction::Unreachable:
887 case Instruction::Fence:
888 case Instruction::AtomicRMW:
902 "Op must be an operand of the recipe");
923 template <
typename IterT>
932 return D->getVPDefID() == VPRecipeBase::VPWidenSC;
941 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
952 template <
typename IterT>
961 return D->getVPDefID() == VPRecipeBase::VPWidenCallSC;
967 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
981 template <
typename IterT>
986 InvariantCond(InvariantCond) {}
992 return D->getVPDefID() == VPRecipeBase::VPWidenSelectSC;
998 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1007 bool IsPtrLoopInvariant;
1011 template <
typename IterT>
1015 IsIndexLoopInvariant(
GEP->getNumIndices(),
false) {}
1017 template <
typename IterT>
1022 IsIndexLoopInvariant(
GEP->getNumIndices(),
false) {
1025 IsIndexLoopInvariant[Index.index()] =
1032 return D->getVPDefID() == VPRecipeBase::VPWidenGEPSC;
1038 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1056 bool NeedsScalarIV,
bool NeedsVectorIV)
1057 :
VPRecipeBase(VPWidenIntOrFpInductionSC, {Start, Step}),
1058 VPValue(IV,
this), IV(IV), IndDesc(IndDesc),
1059 NeedsScalarIV(NeedsScalarIV), NeedsVectorIV(NeedsVectorIV) {}
1065 :
VPRecipeBase(VPWidenIntOrFpInductionSC, {Start, Step}),
1066 VPValue(Trunc,
this), IV(IV), IndDesc(IndDesc),
1067 NeedsScalarIV(NeedsScalarIV), NeedsVectorIV(NeedsVectorIV) {}
1073 return D->getVPDefID() == VPRecipeBase::VPWidenIntOrFpInductionSC;
1080 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1143 return B->getVPDefID() == VPRecipeBase::VPCanonicalIVPHISC ||
1144 B->getVPDefID() == VPRecipeBase::VPFirstOrderRecurrencePHISC ||
1145 B->getVPDefID() == VPRecipeBase::VPReductionPHISC ||
1146 B->getVPDefID() == VPRecipeBase::VPWidenIntOrFpInductionSC ||
1147 B->getVPDefID() == VPRecipeBase::VPWidenPHISC;
1160 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1202 IndDesc(IndDesc), SE(SE) {
1210 return B->getVPDefID() == VPRecipeBase::VPWidenPointerInductionSC;
1213 return R->getVPDefID() == VPRecipeBase::VPWidenPointerInductionSC;
1222 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1248 return B->getVPDefID() == VPRecipeBase::VPWidenPHISC;
1251 return R->getVPDefID() == VPRecipeBase::VPWidenPHISC;
1260 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1269 IncomingBlocks.push_back(IncomingBlock);
1285 VPFirstOrderRecurrencePHISC, Phi, &Start) {}
1289 return R->getVPDefID() == VPRecipeBase::VPFirstOrderRecurrencePHISC;
1292 return R->getVPDefID() == VPRecipeBase::VPFirstOrderRecurrencePHISC;
1300 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1324 VPValue &Start,
bool IsInLoop =
false,
1325 bool IsOrdered =
false)
1327 RdxDesc(RdxDesc), IsInLoop(IsInLoop), IsOrdered(IsOrdered) {
1328 assert((!IsOrdered || IsInLoop) &&
"IsOrdered requires IsInLoop");
1335 return R->getVPDefID() == VPRecipeBase::VPReductionPHISC;
1338 return R->getVPDefID() == VPRecipeBase::VPReductionPHISC;
1347 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1378 "Expected either a single incoming value or a positive even number "
1384 return D->getVPDefID() == VPRecipeBase::VPBlendSC;
1400 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1409 "Op must be an operand of the recipe");
1424 bool HasMask =
false;
1432 if (
I->getType()->isVoidTy())
1437 for (
auto *SV : StoredValues)
1448 return D->getVPDefID() == VPRecipeBase::VPInterleaveSC;
1475 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1492 "Op must be an operand of the recipe");
1494 return Op != StoredV;
1528 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1559 template <
typename IterT>
1561 bool IsUniform,
bool IsPredicated =
false)
1563 IsUniform(IsUniform), IsPredicated(IsPredicated) {
1569 AlsoPack = IsPredicated && !
I->use_empty();
1576 return D->getVPDefID() == VPRecipeBase::VPReplicateSC;
1590 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1605 "Op must be an operand of the recipe");
1612 "Op must be an operand of the recipe");
1628 return D->getVPDefID() == VPRecipeBase::VPBranchOnMaskSC;
1635 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1639 O << Indent <<
"BRANCH-ON-MASK ";
1672 return D->getVPDefID() == VPRecipeBase::VPPredInstPHISC;
1678 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1687 "Op must be an operand of the recipe");
1713 bool isMasked()
const {
1719 bool Consecutive,
bool Reverse)
1721 Consecutive(Consecutive), Reverse(Reverse) {
1722 assert((Consecutive || !Reverse) &&
"Reverse implies consecutive");
1729 bool Consecutive,
bool Reverse)
1731 Ingredient(
Store), Consecutive(Consecutive), Reverse(Reverse) {
1732 assert((Consecutive || !Reverse) &&
"Reverse implies consecutive");
1738 return D->getVPDefID() == VPRecipeBase::VPWidenMemoryInstructionSC;
1754 bool isStore()
const {
return isa<StoreInst>(Ingredient); }
1758 assert(
isStore() &&
"Stored value only available for store instructions");
1772 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1781 "Op must be an operand of the recipe");
1807 return D->getVPDefID() == VPExpandSCEVSC;
1813 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1839 return D->getVPDefID() == VPCanonicalIVPHISC;
1842 return D->getVPDefID() == VPCanonicalIVPHISC;
1851 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1865 "Op must be an operand of the recipe");
1881 return D->getVPDefID() == VPRecipeBase::VPWidenCanonicalIVSC;
1887 auto *R = dyn_cast<VPRecipeBase>(U);
1888 return R && R->getVPDefID() == VPRecipeBase::VPWidenCanonicalIVSC;
1891 return R->getVPDefID() == VPRecipeBase::VPWidenCanonicalIVSC;
1899 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1925 :
VPRecipeBase(VPScalarIVStepsSC, {CanonicalIV, Start, Step}),
1926 VPValue(
nullptr,
this), Ty(Ty), TruncToTy(TruncToTy), IndDesc(IndDesc) {
1933 return D->getVPDefID() == VPRecipeBase::VPScalarIVStepsSC;
1938 auto *R = dyn_cast<VPRecipeBase>(U);
1939 return R && R->getVPDefID() == VPRecipeBase::VPScalarIVStepsSC;
1942 return R->getVPDefID() == VPRecipeBase::VPScalarIVStepsSC;
1948 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1965 "Op must be an operand of the recipe");
1989 while (!Recipes.empty())
2012 inline size_t size()
const {
return Recipes.size(); }
2013 inline bool empty()
const {
return Recipes.empty(); }
2024 return &VPBasicBlock::Recipes;
2029 return V->
getVPBlockID() == VPBlockBase::VPBasicBlockSC;
2033 assert(Recipe &&
"No recipe to append.");
2034 assert(!Recipe->Parent &&
"Recipe already in VPlan");
2035 Recipe->Parent =
this;
2036 Recipes.
insert(InsertPt, Recipe);
2064 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2102 const std::string &Name =
"",
bool IsReplicator =
false)
2103 :
VPBlockBase(VPRegionBlockSC, Name), Entry(Entry), Exit(Exit),
2104 IsReplicator(IsReplicator) {
2105 assert(Entry->getPredecessors().empty() &&
"Entry block has predecessors.");
2107 Entry->setParent(
this);
2111 :
VPBlockBase(VPRegionBlockSC, Name), Entry(nullptr), Exit(nullptr),
2112 IsReplicator(IsReplicator) {}
2117 Entry->dropAllReferences(&DummyValue);
2124 return V->
getVPBlockID() == VPBlockBase::VPRegionBlockSC;
2134 "Entry block cannot have predecessors.");
2152 "Exit block cannot have successors.");
2173 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2203 return N->getSuccessors().begin();
2207 return N->getSuccessors().end();
2218 return N->getSuccessors().begin();
2222 return N->getSuccessors().end();
2235 return N->getPredecessors().begin();
2239 return N->getPredecessors().end();
2294 return N.Graph->getExit();
2313 template <
typename BlockPtrTy>
2316 std::forward_iterator_tag, VPBlockBase> {
2322 size_t SuccessorIdx;
2324 static BlockPtrTy getBlockWithSuccs(BlockPtrTy Current) {
2325 while (Current && Current->getNumSuccessors() == 0)
2326 Current = Current->getParent();
2332 template <
typename T1>
static T1 deref(
T1 Block,
unsigned SuccIdx) {
2333 if (
auto *R = dyn_cast<VPRegionBlock>(Block)) {
2335 return R->getEntry();
2340 return getBlockWithSuccs(Block)->getSuccessors()[SuccIdx];
2345 : Block(Block), SuccessorIdx(Idx) {}
2347 : Block(
Other.Block), SuccessorIdx(
Other.SuccessorIdx) {}
2351 SuccessorIdx = R.SuccessorIdx;
2356 BlockPtrTy ParentWithSuccs = getBlockWithSuccs(Block);
2357 unsigned NumSuccessors = ParentWithSuccs
2358 ? ParentWithSuccs->getNumSuccessors()
2359 : Block->getNumSuccessors();
2361 if (
auto *R = dyn_cast<VPRegionBlock>(Block))
2362 return {R, NumSuccessors + 1};
2363 return {Block, NumSuccessors};
2367 return Block == R.Block && SuccessorIdx == R.SuccessorIdx;
2407 return N.getEntry();
2426 return N.getEntry();
2466 VPValue *BackedgeTakenCount =
nullptr;
2484 bool Value2VPValueEnabled =
true;
2489 Entry->setPlan(
this);
2496 Block->dropAllReferences(&DummyValue);
2500 for (
VPValue *VPV : VPValuesToFree)
2504 if (BackedgeTakenCount)
2505 delete BackedgeTakenCount;
2506 for (
auto &
P : VPExternalDefs)
2522 Block->setPlan(
this);
2535 if (!BackedgeTakenCount)
2536 BackedgeTakenCount =
new VPValue();
2537 return BackedgeTakenCount;
2557 auto I = VPExternalDefs.
insert({V,
nullptr});
2560 return I.first->second;
2564 assert(Value2VPValueEnabled &&
2565 "IR value to VPValue mapping may be out of date!");
2566 assert(V &&
"Trying to add a null Value to VPlan");
2567 assert(!Value2VPValue.
count(V) &&
"Value already exists in VPlan");
2569 Value2VPValue[V] = VPV;
2570 VPValuesToFree.push_back(VPV);
2574 assert(Value2VPValueEnabled &&
"Value2VPValue mapping may be out of date!");
2575 assert(V &&
"Trying to add a null Value to VPlan");
2576 assert(!Value2VPValue.
count(V) &&
"Value already exists in VPlan");
2577 Value2VPValue[V] = VPV;
2583 assert((OverrideAllowed || isa<Constant>(V) || Value2VPValueEnabled) &&
2584 "Value2VPValue mapping may be out of date!");
2585 assert(V &&
"Trying to get the VPValue of a null Value");
2586 assert(Value2VPValue.
count(V) &&
"Value does not exist in VPlan");
2587 return Value2VPValue[V];
2594 assert((OverrideAllowed || isa<Constant>(V) || Value2VPValueEnabled) &&
2595 "Value2VPValue mapping may be out of date!");
2596 assert(V &&
"Trying to get or add the VPValue of a null Value");
2597 if (!Value2VPValue.
count(V))
2603 assert(Value2VPValueEnabled &&
2604 "IR value to VPValue mapping may be out of date!");
2605 Value2VPValue.
erase(V);
2612 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2635 auto RepR = dyn_cast_or_null<VPReplicateRecipe>(VPV->
getDef());
2636 return !VPV->
getDef() || (RepR && RepR->isUniform());
2641 if (
auto *
R = dyn_cast<VPRegionBlock>(
getEntry()))
2643 return cast<VPRegionBlock>(
getEntry()->getSingleSuccessor());
2646 if (
auto *
R = dyn_cast<VPRegionBlock>(
getEntry()))
2648 return cast<VPRegionBlock>(
getEntry()->getSingleSuccessor());
2654 if (EntryVPBB->
empty()) {
2658 return cast<VPCanonicalIVPHIRecipe>(&*EntryVPBB->
begin());
2669 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2676 unsigned TabWidth = 2;
2684 void bumpIndent(
int b) { Indent = std::string((
Depth +=
b) * TabWidth,
' '); }
2700 unsigned getOrCreateBID(
const VPBlockBase *Block) {
2701 return BlockID.
count(Block) ? BlockID[Block] : BlockID[Block] = BID++;
2710 const Twine &Label);
2756 "Can't insert new block with predecessors or successors.");
2777 "Can't insert IfTrue with successors.");
2779 "Can't insert IfFalse with successors.");
2793 "Can't connect two block with different parents");
2795 "Blocks can't have more than two successors.");
2796 From->appendSuccessor(To);
2797 To->appendPredecessor(
From);
2803 assert(To &&
"Successor to disconnect is null.");
2804 From->removeSuccessor(To);
2805 To->removePredecessor(
From);
2812 auto *VPBB = dyn_cast<VPBasicBlock>(Block);
2814 dyn_cast_or_null<VPBasicBlock>(Block->getSinglePredecessor());
2815 if (!VPBB || !PredVPBB || PredVPBB->getNumSuccessors() != 1)
2819 R.moveBefore(*PredVPBB, PredVPBB->end());
2821 auto *ParentRegion = cast<VPRegionBlock>(Block->getParent());
2822 if (ParentRegion->getExit() == Block)
2823 ParentRegion->setExit(PredVPBB);
2825 for (
auto *Succ : Successors) {
2837 FromBlock->
getParent() &&
"Must be in same region");
2840 if (!FromLoop || !ToLoop || FromLoop != ToLoop)
2851 return ParentVPL->isLoopLatch(Block);
2870 template <
typename BlockTy,
typename T>
2874 typename std::conditional<std::is_const<BlockTy>::value,
2880 map_range(Range, [](BaseTy *Block) -> BaseTy & {
return *Block; });
2882 Mapped, [](BaseTy &Block) {
return isa<BlockTy>(&Block); });
2884 return cast<BlockTy>(&Block);
2913 for (
auto &
I : InterleaveGroupMap)
2915 for (
auto *Ptr : DelSet)
2924 return InterleaveGroupMap.
lookup(Instr);
2931 enum class OpMode {
Failed, Load, Opcode };
2935 struct BundleDenseMapInfo {
2937 return {
reinterpret_cast<VPValue *
>(-1)};
2941 return {
reinterpret_cast<VPValue *
>(-2)};
2965 bool CompletelySLP =
true;
2968 unsigned WidestBundleBits = 0;
2970 using MultiNodeOpTy =
2971 typename std::pair<VPInstruction *, SmallVector<VPValue *, 4>>;
2980 bool MultiNodeActive =
false;
2998 std::pair<OpMode, VPValue *> getBest(OpMode
Mode,
VPValue *Last,
3002 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
3040 #endif // LLVM_TRANSFORMS_VECTORIZE_VPLAN_H
RecipeListTy & getRecipeList()
Returns a reference to the list of recipes.
VPBlockBase & front() const
unsigned getOpcode() const
~VPWidenGEPRecipe() override=default
VPWidenIntOrFpInductionRecipe(PHINode *IV, VPValue *Start, VPValue *Step, const InductionDescriptor &IndDesc, TruncInst *Trunc, bool NeedsScalarIV, bool NeedsVectorIV)
VFRange(const ElementCount &Start, const ElementCount &End)
VPUserID getVPUserID() const
void moveAfter(VPRecipeBase *MovePos)
Unlink this recipe from its current VPBasicBlock and insert it into the VPBasicBlock that MovePos liv...
VPBlockBase * getSinglePredecessor() const
static VPBasicBlock * tryToMergeBlockIntoPredecessor(VPBlockBase *Block)
Try to merge Block into its single predecessor, if Block is a VPBasicBlock and its predecessor has a ...
const SCEV * getSCEV() const
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
bool needsScalarIV() const
Returns true if a scalar phi needs to be created for the induction.
VPValue * getMask() const
Return the mask used by this recipe.
const VPBlockBase * operator*() const
VPValue * getStepValue()
Returns the step value of the induction.
VPWidenMemoryInstructionRecipe(LoadInst &Load, VPValue *Addr, VPValue *Mask, bool Consecutive, bool Reverse)
unsigned getNumIncomingValues() const
Return the number of incoming values, taking into account that a single incoming value has no mask.
void execute(struct VPTransformState *State)
Generate the IR code for this VPlan.
void print(raw_ostream &O) const
bool isLoopInvariant(const Value *V) const
Return true if the specified value is loop invariant.
Class that maps (parts of) an existing VPlan to trees of combined VPInstructions.
bool onlyFirstLaneUsed(const VPValue *Op) const override
Returns true if the recipe only uses the first lane of operand Op.
static NodeRef getEntryNode(NodeRef N)
~VPWidenSelectRecipe() override=default
static bool classof(const VPDef *D)
Method to support type inquiry through isa, cast, and dyn_cast.
VPLane(unsigned Lane, Kind LaneKind)
#define LLVM_DUMP_METHOD
Mark debug helper function definitions like dump() that should not be stripped from debug builds.
VPRegionBlock(VPBlockBase *Entry, VPBlockBase *Exit, const std::string &Name="", bool IsReplicator=false)
VPValue * getIncomingValue(unsigned Idx) const
Return incoming value number Idx.
This is an optimization pass for GlobalISel generic memory operations.
iterator erase(const_iterator CI)
Instruction * getUnderlyingInstr()
Returns the underlying instruction, if the recipe is a VPValue or nullptr otherwise.
~VPWidenCallRecipe() override=default
Hold analysis information for every loop detected by VPLoopInfo.
static NodeRef getEntryNode(NodeRef N)
VPlanIngredient(const Value *V)
virtual void dropAllReferences(VPValue *NewValue)=0
Replace all operands of VPUsers in the block with NewValue and also replaces all uses of VPValues def...
Canonical scalar induction phi of the vector loop.
VPWidenPHIRecipe(PHINode *Phi, VPValue *Start=nullptr)
Create a new VPWidenPHIRecipe for Phi with start value Start.
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
iplist< VPRecipeBase >::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
static VPLane getFirstLane()
void execute(VPTransformState &State) override
Generate a canonical vector induction variable of the vector loop, with start = {<Part*VF,...
void execute(VPTransformState &State) override
Generates phi nodes for live-outs as needed to retain SSA form.
VPIteration(unsigned Part, const VPLane &Lane)
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPBlockBase * getEnclosingBlockWithPredecessors()
static bool classof(const VPUser *U)
Extra classof implementations to allow directly casting from VPUser -> VPInstruction.
const VPRecipeBase & back() const
bool isLegalToHoistInto()
Return true if it is legal to hoist instructions into this block.
const std::string & getName() const
void disableValue2VPValue()
Mark the plan to indicate that using Value2VPValue is not safe any longer, because it may be stale.
VPValue * getOrAddVPValue(Value *V, bool OverrideAllowed=false)
Gets the VPValue or adds a new one (if none exists yet) for V.
VPWidenCanonicalIVRecipe(VPCanonicalIVPHIRecipe *CanonicalIV)
VPBasicBlock serves as the leaf of the Hierarchical Control-Flow Graph.
static bool classof(const VPUser *U)
Extra classof implementations to allow directly casting from VPUser -> VPScalarIVStepsRecipe.
Represents a single loop in the control flow graph.
ValueT lookup(const_arg_type_t< KeyT > Val) const
lookup - Return the entry for the specified key, or a default constructed value if no such entry exis...
This currently compiles esp xmm0 movsd esp eax eax esp ret We should use not the dag combiner This is because dagcombine2 needs to be able to see through the X86ISD::Wrapper which DAGCombine can t really do The code for turning x load into a single vector load is target independent and should be moved to the dag combiner The code for turning x load into a vector load can only handle a direct load from a global or a direct load from the stack It should be generalized to handle any load from P
A recipe for handling header phis that are widened in the vector loop.
std::pair< BasicBlock *, unsigned > BlockTy
A pair of (basic block, score).
const VPBasicBlock * getExitBasicBlock() const
void prepareToExecute(Value *TripCount, Value *VectorTripCount, Value *CanonicalIVStartValue, VPTransformState &State)
Prepare the plan for execution, setting up the required live-in values.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Value * getLiveInIRValue()
Returns the underlying IR value, if this VPValue is defined outside the scope of VPlan.
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
LLVM_DUMP_METHOD void dump()
static bool classof(const VPDef *D)
Method to support type inquiry through isa, cast, and dyn_cast.
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
~VPWidenIntOrFpInductionRecipe() override=default
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPValue * getVPSingleValue()
Returns the only VPValue defined by the VPDef.
const_iterator end() const
VPValue * getOrAddExternalDef(Value *V)
Get the existing or add a new external definition for V.
detail::enumerator< R > enumerate(R &&TheRange)
Given an input range, returns a new range whose values are are pair (A,B) such that A is the 0-based ...
void dropAllReferences(VPValue *NewValue) override
Replace all operands of VPUsers in the block with NewValue and also replaces all uses of VPValues def...
An ilist node that can access its parent list.
void execute(VPTransformState &State) override
Generate the extraction of the appropriate bit from the block mask and the conditional branch.
VPBlockBase(const unsigned char SC, const std::string &N)
static nodes_iterator nodes_begin(GraphRef N)
@ First
For First, Lane is the index into the first N elements of a fixed-vector <N x <ElTy>> or a scalable v...
VPInterleaveRecipe(const InterleaveGroup< Instruction > *IG, VPValue *Addr, ArrayRef< VPValue * > StoredValues, VPValue *Mask)
@ ScalableLast
For ScalableLast, Lane is the offset from the start of the last N-element subvector in a scalable vec...
static ChildIteratorType child_end(NodeRef N)
The main scalar evolution driver.
static ChildIteratorType child_begin(NodeRef N)
void setFastMathFlags(FastMathFlags FMFNew)
Set the fast-math flags.
A recipe for handling reduction phis.
bool needsVectorIV() const
Returns true if a vector phi needs to be created for the induction.
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
base_list_type::const_reverse_iterator const_reverse_iterator
~VPInterleaveRecipe() override=default
VPBasicBlock * getPreheaderVPBB()
Returns the pre-header VPBasicBlock of the loop region.
const Instruction * getUnderlyingInstr() const
bool erase(const KeyT &Val)
void execute(VPTransformState &State) override
Generate the phi/select nodes.
VPReductionRecipe(const RecurrenceDescriptor *R, Instruction *I, VPValue *ChainOp, VPValue *VecOp, VPValue *CondOp, const TargetTransformInfo *TTI)
virtual ~VPBlockBase()=default
VPPredInstPHIRecipe is a recipe for generating the phi nodes needed when control converges back from ...
void dropAllReferences(VPValue *NewValue) override
Replace all operands of VPUsers in the block with NewValue and also replaces all uses of VPValues def...
Kind getKind() const
Returns the Kind of lane offset.
VPCanonicalIVPHIRecipe * getCanonicalIV()
Returns the canonical induction recipe of the vector loop.
The instances of the Type class are immutable: once they are created, they are never changed.
TruncInst * getTruncInst()
Returns the first defined value as TruncInst, if it is one or nullptr otherwise.
bool isFirstIteration() const
std::unique_ptr< VPlan > VPlanPtr
const_iterator end(StringRef path)
Get end iterator over path.
const_iterator begin(StringRef path, Style style=Style::native)
Get begin iterator over path.
static NodeRef getEntryNode(Inverse< GraphRef > N)
const VPBlocksTy & getHierarchicalPredecessors()
static bool classof(const VPDef *D)
Method to support type inquiry through isa, cast, and dyn_cast.
const Type * getScalarType() const
Returns the scalar type of the induction.
Class that provides utilities for VPBlockBases in VPlan.
operand_iterator op_begin()
void execute(VPTransformState &State) override
Generate the scalarized versions of the phi node as needed by their users.
size_type count(const_arg_type_t< KeyT > Val) const
Return 1 if the specified key is in the map, 0 otherwise.
const VPBlockBase * getEntry() const
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
~VPWidenPHIRecipe() override=default
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
static bool classof(const VPDef *D)
Method to support type inquiry through isa, cast, and dyn_cast.
A recipe for widening Call instructions.
bool usesScalars(const VPValue *Op) const override
Returns true if the recipe uses scalars of operand Op.
VPWidenMemoryInstructionRecipe(StoreInst &Store, VPValue *Addr, VPValue *StoredValue, VPValue *Mask, bool Consecutive, bool Reverse)
bool isLoopLatch(const BlockT *BB) const
bool onlyFirstLaneUsed(const VPValue *Op) const override
Returns true if the recipe only uses the first lane of operand Op.
VPUserID
Subclass identifier (for isa/dyn_cast).
static bool classof(const VPRecipeBase *R)
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
const VPBlockBase * getEntry() const
Iterator to traverse all successors of a VPBlockBase node.
Convenience struct for specifying and reasoning about fast-math flags.
bool mayWriteToMemory() const
Return true if this instruction may modify memory.
The group of interleaved loads/stores sharing the same stride and close to each other.
bool isScalable() const
Returns whether the size is scaled by a runtime quantity (vscale).
static NodeRef getEntryNode(VPBlockRecursiveTraversalWrapper< const VPBlockBase * > N)
VPRegionBlock * getEnclosingLoopRegion()
An intrusive list with ownership and callbacks specified/controlled by ilist_traits,...
A recipe to represent inloop reduction operations, performing a reduction on a vector operand into a ...
~VPReductionPHIRecipe() override=default
bool isEqual(const GCNRPTracker::LiveRegSet &S1, const GCNRPTracker::LiveRegSet &S2)
static bool classof(const VPDef *D)
Method to support type inquiry through isa, cast, and dyn_cast.
bool isCanonical() const
Returns true if the induction is canonical, i.e.
void execute(VPTransformState &State) override
Generate the phi/select nodes.
virtual void execute(struct VPTransformState *State)=0
The method which generates the output IR that correspond to this VPBlockBase, thereby "executing" the...
void setPredicate(VPValue *Pred)
Set the block's predicate.
VPCanonicalIVPHIRecipe * getCanonicalIV() const
const InterleaveGroup< Instruction > * getInterleaveGroup()
A range of powers-of-2 vectorization factors with fixed start and adjustable end.
LLVM Basic Block Representation.
static bool classof(const VPDef *D)
Method to support type inquiry through isa, cast, and dyn_cast.
const InductionDescriptor & getInductionDescriptor() const
Returns the induction descriptor for the recipe.
This class augments a recipe with a set of VPValues defined by the recipe.
const Type * getScalarType() const
Returns the scalar type of the induction.
bool mayWriteToMemory() const
Returns true if the recipe may write to memory.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void execute(VPTransformState &State) override
Generate the wide load or store, and shuffles.
This is a 'bitvector' (really, a variable-sized bit array), optimized for the case when the array is ...
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPReplicateRecipe replicates a given instruction producing multiple scalar copies of the original sca...
const VPValue * getStartValue() const
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
RecipeListTy::iterator iterator
Instruction iterators...
static bool classof(const VPHeaderPHIRecipe *R)
@ VPVWidenPointerInductionSC
VPBlockBase is the building block of the Hierarchical Control-Flow Graph.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void appendRecipe(VPRecipeBase *Recipe)
Augment the existing recipes of a VPBasicBlock with an additional Recipe as the last recipe.
bool hasVF(ElementCount VF)
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
bool isInLoop() const
Returns true, if the phi is part of an in-loop reduction.
static bool classof(const VPRecipeBase *R)
A struct for saving information about induction variables.
static bool classof(const VPValue *V)
static bool classof(const VPValue *V)
Method to support type inquiry through isa, cast, and dyn_cast.
unsigned mapToCacheIndex(const ElementCount &VF) const
Maps the lane to a cache index based on VF.
VPBasicBlock * getIncomingBlock(unsigned I)
Returns the I th incoming VPBasicBlock.
This is a concrete Recipe that models a single VPlan-level instruction.
@ SC
CHAIN = SC CHAIN, Imm128 - System call.
testing::Matcher< const detail::ErrorHolder & > Failed()
void print(raw_ostream &O) const
Print this VPlan to O.
VPValue * getOrCreateVPValueForSCEVExpr(VPlan &Plan, const SCEV *Expr, ScalarEvolution &SE)
Get or create a VPValue that corresponds to the expansion of Expr.
void setPredecessors(ArrayRef< VPBlockBase * > NewPreds)
Set each VPBasicBlock in NewPreds as predecessor of this VPBlockBase.
Value * createStepForVF(IRBuilderBase &B, Type *Ty, ElementCount VF, int64_t Step)
Return a value for Step multiplied by VF.
bool onlyFirstLaneUsed(const VPValue *Op) const override
Returns true if the recipe only uses the first lane of operand Op.
InterleaveGroup< VPInstruction > * getInterleaveGroup(VPInstruction *Instr) const
Get the interleave group that Instr belongs to.
VPInstruction * buildGraph(ArrayRef< VPValue * > Operands)
Tries to build an SLP tree rooted at Operands and returns a VPInstruction combining Operands,...
static bool classof(const VPValue *V)
bool isFirstLane() const
Returns true if this is the first lane of the whole vector.
VPBlocksTy & getSuccessors()
static void insertBlockAfter(VPBlockBase *NewBlock, VPBlockBase *BlockPtr)
Insert disconnected VPBlockBase NewBlock after BlockPtr.
void execute(VPTransformState &State) override
Generate vector values for the pointer induction.
static RecipeListTy VPBasicBlock::* getSublistAccess(VPRecipeBase *)
Returns a pointer to a member of the recipe list.
VPScalarIVStepsRecipe(Type *Ty, const InductionDescriptor &IndDesc, VPValue *CanonicalIV, VPValue *Start, VPValue *Step, Type *TruncToTy)
SmallVectorImpl< VPBlockBase * >::const_iterator ChildIteratorType
VPlanPrinter(raw_ostream &O, const VPlan &P)
VPValue * getVPValue(Value *V, bool OverrideAllowed=false)
Returns the VPValue for V.
static ChildIteratorType child_end(NodeRef N)
the resulting code requires compare and branches when and if the revised code is with conditional branches instead of More there is a byte word extend before each where there should be only and the condition codes are not remembered when the same two values are compared twice More LSR enhancements i8 and i32 load store addressing modes are identical int b
void moveBefore(VPBasicBlock &BB, iplist< VPRecipeBase >::iterator I)
Unlink this recipe and insert into BB before I.
VPBlocksTy & getPredecessors()
VPInterleaveRecipe is a recipe for transforming an interleave group of load or stores into one wide l...
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
enum { VPBasicBlockSC, VPRegionBlockSC } VPBlockTy
An enumeration for keeping track of the concrete subclass of VPBlockBase that are actually instantiat...
VPBlockBase * getEnclosingBlockWithSuccessors()
An Enclosing Block of a block B is any block containing B, including B itself.
static bool classof(const VPHeaderPHIRecipe *D)
VPValue * getIncomingValue(unsigned I)
Returns the I th incoming VPValue.
static bool classof(const VPRecipeBase *B)
Method to support type inquiry through isa, cast, and dyn_cast.
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
static bool classof(const VPValue *V)
static nodes_iterator nodes_begin(GraphRef N)
~VPReplicateRecipe() override=default
Drive the analysis of interleaved memory accesses in the loop.
void printSuccessors(raw_ostream &O, const Twine &Indent) const
Print the successors of this block to O, prefixing all lines with Indent.
void execute(VPTransformState &State) override
Generate the reduction in the loop.
void execute(struct VPTransformState *State) override
The method which generates the output IR instructions that correspond to this VPRegionBlock,...
This class implements an extremely fast bulk output stream that can only output to a stream.
VPValue * getStartValue() const
static auto blocksOnly(const T &Range)
Return an iterator range over Range which only includes BlockTy blocks.
~VPRegionBlock() override
raw_ostream & operator<<(raw_ostream &OS, const APFixedPoint &FX)
void execute(VPTransformState &State) override
Produce a widened version of the call instruction.
iterator_range< VPBlockBase ** > successors()
unsigned getWidestBundleBits() const
Return the width of the widest combined bundle in bits.
const std::string & getName() const
A Recipe for widening the canonical induction variable of the vector loop.
Instruction & getIngredient() const
VPAllSuccessorsIterator & operator++()
InnerLoopVectorizer vectorizes loops which contain only one basic block to a specified vectorization ...
InstTy * getMember(uint32_t Index) const
Get the member with the given index Index.
static bool classof(const VPRecipeBase *R)
Method to support type inquiry through isa, cast, and dyn_cast.
static ChildIteratorType child_begin(NodeRef N)
ArrayRef< T > slice(size_t N, size_t M) const
slice(n, m) - Chop off the first N elements of the array, and keep M elements in the array.
~VPWidenCanonicalIVRecipe() override=default
unsigned getNumOperands() const
base_list_type::reverse_iterator reverse_iterator
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
~VPCanonicalIVPHIRecipe() override=default
VPValue * getAddr() const
Return the address accessed by this recipe.
VPRegionBlock represents a collection of VPBasicBlocks and VPRegionBlocks which form a Single-Entry-S...
bool isStore() const
Returns true if this recipe is a store.
const VPBlocksTy & getPredecessors() const
VPReductionPHIRecipe(PHINode *Phi, const RecurrenceDescriptor &RdxDesc, VPValue &Start, bool IsInLoop=false, bool IsOrdered=false)
Create a new VPReductionPHIRecipe for the reduction Phi described by RdxDesc.
static ChildIteratorType child_end(NodeRef N)
std::string str() const
Return the twine contents as a std::string.
mir Rename Register Operands
static NodeRef getEntryNode(Inverse< NodeRef > B)
static ChildIteratorType child_begin(NodeRef N)
VPValue * getAddr() const
Return the address accessed by this recipe.
VPWidenPointerInductionRecipe(PHINode *Phi, VPValue *Start, const InductionDescriptor &IndDesc, ScalarEvolution &SE)
Create a new VPWidenPointerInductionRecipe for Phi with start value Start.
void execute(VPTransformState &State) override
Generate replicas of the desired Ingredient.
ArrayRef< VPValue * > getStoredValues() const
Return the VPValues stored by this interleave group.
VPRecipeBase(const unsigned char SC, ArrayRef< VPValue * > Operands)
void setOperand(unsigned I, VPValue *New)
static bool classof(const VPDef *D)
Method to support type inquiry through isa, cast, and dyn_cast.
void clearPredecessors()
Remove all the predecessor of this block.
const VPValue * getSingleOperandOrNull() const
size_t getNumSuccessors() const
static ChildIteratorType child_begin(NodeRef N)
static unsigned getNumCachedLanes(const ElementCount &VF)
Returns the maxmimum number of lanes that we are able to consider caching for VF.
virtual void execute(struct VPTransformState &State)=0
The method which generates the output IR instructions that correspond to this VPRecipe,...
const VPRegionBlock * getVectorLoopRegion() const
static bool classof(const VPRecipeBase *R)
VPValue * getMask() const
Return the mask used by this recipe.
const VPBasicBlock * getParent() const
This class represents an analyzed expression in the program.
bool onlyFirstLaneUsed(const VPValue *Op) const override
Returns true if the recipe only uses the first lane of operand Op.
const_reverse_iterator rbegin() const
void setName(const Twine &newName)
void removeFromParent()
This method unlinks 'this' from the containing basic block, but does not delete it.
const VPBlocksTy & getHierarchicalSuccessors()
static bool classof(const VPDef *D)
Method to support type inquiry through isa, cast, and dyn_cast.
~VPPredInstPHIRecipe() override=default
VPBranchOnMaskRecipe(VPValue *BlockInMask)
bool mayHaveSideEffects() const
Returns true if the recipe may have side-effects.
An instruction for storing to memory.
void execute(VPTransformState &State) override
Generate a canonical vector induction variable of the vector loop, with.
AMD64 Optimization Manual has some nice information about optimizing integer multiplication by a constant How much of it applies to Intel s X86 implementation There are definite trade offs to xmm0 cvttss2siq rdx jb L3 subss xmm0 rax cvttss2siq rdx xorq rdx rax ret instead of xmm1 cvttss2siq rcx movaps xmm2 subss xmm2 cvttss2siq rax rdx xorq rax ucomiss xmm0 cmovb rax ret Seems like the jb branch has high likelihood of being taken It would have saved a few instructions It s not possible to reference and DH registers in an instruction requiring REX prefix divb and mulb both produce results in AH If isel emits a CopyFromReg which gets turned into a movb and that can be allocated a r8b r15b To get around isel emits a CopyFromReg from AX and then right shift it down by and truncate it It s not pretty but it works We need some register allocation magic to make the hack go which would often require a callee saved register Callees usually need to keep this value live for most of their body so it doesn t add a significant burden on them We currently implement this in however this is suboptimal because it means that it would be quite awkward to implement the optimization for callers A better implementation would be to relax the LLVM IR rules for sret arguments to allow a function with an sret argument to have a non void return type
Value * getAsRuntimeExpr(IRBuilderBase &Builder, const ElementCount &VF) const
Returns an expression describing the lane index that can be used at runtime.
unsigned getNumStoreOperands() const
Returns the number of stored operands of this interleave group.
VPValue * getCondOp() const
The VPValue of the condition for the block.
bool onlyFirstLaneUsed(const VPValue *Op) const override
The recipe only uses the first lane of the address.
bool isUniformAfterVectorization(VPValue *VPV) const
Returns true if VPV is uniform after vectorization.
void clearSuccessors()
Remove all the successors of this block and set to null its condition bit.
VPBlockBase * getSingleHierarchicalSuccessor()
VPBlockBase * getSingleHierarchicalPredecessor()
void execute(VPTransformState &State) override
Generate the gep nodes.
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
void setExit(VPBlockBase *ExitBlock)
Set ExitBlock as the exit VPBlockBase of this VPRegionBlock.
void execute(VPTransformState &State) override
Generate the vectorized and scalarized versions of the phi node as needed by their users.
VPWidenRecipe is a recipe for producing a copy of vector type its ingredient.
auto find(R &&Range, const T &Val)
Provide wrappers to std::find which take ranges instead of having to pass begin/end explicitly.
void execute(VPTransformState &State) override
Generate the instruction.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
uint32_t getFactor() const
This class represents a truncation of integer types.
base_list_type::iterator iterator
unsigned getVPBlockID() const
LLVM_DUMP_METHOD void dump() const
Dump this VPBlockBase to dbgs().
static VPLane getLastLaneForVF(const ElementCount &VF)
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the VPInstruction to O.
VPIteration(unsigned Part, unsigned Lane, VPLane::Kind Kind=VPLane::Kind::First)
LoopT * getLoopFor(const BlockT *BB) const
Return the inner most loop that BB lives in.
VPIteration represents a single point in the iteration space of the output (vectorized and/or unrolle...
static void connectBlocks(VPBlockBase *From, VPBlockBase *To)
Connect VPBlockBases From and To bi-directionally.
static ChildIteratorType child_begin(NodeRef N)
CRTP base class which implements the entire standard iterator facade in terms of a minimal subset of ...
an instruction for type-safe pointer arithmetic to access elements of arrays and structs
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
VPValue * getMask(unsigned Idx) const
Return mask number Idx.
bool is_contained(R &&Range, const E &Element)
Wrapper function around std::find to detect if an element exists in a container.
size_t getNumPredecessors() const
void removeVPValueFor(Value *V)
void addVPValue(Value *V)
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPValue * getStoredValue() const
Return the address accessed by this recipe.
VPExpandSCEVRecipe(const SCEV *Expr, ScalarEvolution &SE)
VPBasicBlock(const Twine &Name="", VPRecipeBase *Recipe=nullptr)
static void disconnectBlocks(VPBlockBase *From, VPBlockBase *To)
Disconnect VPBlockBases From and To bi-directionally.
~VPWidenPointerInductionRecipe() override=default
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static bool classof(const VPHeaderPHIRecipe *R)
A recipe for handling phi nodes of integer and floating-point inductions, producing their vector valu...
static NodeRef getEntryNode(GraphRef N)
static nodes_iterator nodes_end(GraphRef N)
static bool classof(const VPRecipeBase *B)
Method to support type inquiry through isa, cast, and dyn_cast.
A recipe for handling GEP instructions.
void execute(struct VPTransformState *State) override
The method which generates the output IR instructions that correspond to this VPBasicBlock,...
VPWidenIntOrFpInductionRecipe(PHINode *IV, VPValue *Start, VPValue *Step, const InductionDescriptor &IndDesc, bool NeedsScalarIV, bool NeedsVectorIV)
This class represents the LLVM 'select' instruction.
@ BasicBlock
Various leaf nodes.
print Print MemDeps of function
VPReplicateRecipe(Instruction *I, iterator_range< IterT > Operands, bool IsUniform, bool IsPredicated=false)
static bool classof(const VPHeaderPHIRecipe *R)
A Recipe for widening load/store operations.
VPInstruction * clone() const
ScalarTy getKnownMinValue() const
Returns the minimum value this size can represent.
static bool classof(const VPDef *D)
Method to support type inquiry through isa, cast, and dyn_cast.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
A recipe for widening select instructions.
static nodes_iterator nodes_end(GraphRef N)
static bool classof(const VPDef *R)
Method to support type inquiry through isa, cast, and dyn_cast.
static bool classof(const VPUser *U)
Extra classof implementations to allow directly casting from VPUser -> VPWidenCanonicalIVRecipe.
VPCanonicalIVPHIRecipe(VPValue *StartV, DebugLoc DL)
bool insert(const value_type &X)
Insert a new element into the SetVector.
bool isCanonical() const
Returns true if the induction is canonical, i.e.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void printAsOperand(raw_ostream &OS, bool PrintType) const
VPRegionBlock * getVectorLoopRegion()
Returns the VPRegionBlock of the vector loop.
bool onlyFirstLaneUsed(const VPValue *Op) const override
Returns true if the recipe only uses the first lane of operand Op.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Value * getRuntimeVF(IRBuilderBase &B, Type *Ty, ElementCount VF)
Return the runtime value for VF.
VPFirstOrderRecurrencePHIRecipe(PHINode *Phi, VPValue &Start)
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
VPValue * getVecOp() const
The VPValue of the vector value to be reduced.
VPBlockBase * getSingleSuccessor() const
Analysis the ScalarEvolution expression for r is this
void addOperand(VPValue *Operand)
virtual bool onlyFirstLaneUsed(const VPValue *Op) const
Returns true if the VPUser only uses the first lane of operand Op.
unsigned getKnownLane() const
Returns a compile-time known value for the lane index and asserts if the lane can only be calculated ...
VPWidenCallRecipe(CallInst &I, iterator_range< IterT > CallArguments)
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ VPVWidenIntOrFpInductionSC
Type * getType() const
All values are typed, get the type of this value.
static bool classof(const VPDef *D)
Method to support type inquiry through isa, cast, and dyn_cast.
Kind
Kind describes how to interpret Lane.
static bool classof(const VPDef *D)
Method to support type inquiry through isa, cast, and dyn_cast.
VPBasicBlock * getParent()
Common base class shared among various IRBuilders.
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
void execute(VPTransformState &State) override
Generate the canonical scalar induction phi of the vector loop.
static unsigned countSuccessorsNoBE(VPBlockBase *PredBlock, VPLoopInfo *VPLI)
Count and return the number of succesors of PredBlock excluding any backedges.
A recipe for handling first-order recurrence phis.
~VPExpandSCEVRecipe() override=default
static bool classof(const VPValue *V)
VPValue * getStepValue() const
VPAllSuccessorsIterator(BlockPtrTy Block, size_t Idx=0)
A recipe for generating conditional branches on the bits of a mask.
void execute(VPTransformState &State) override
Produce a widened version of the select instruction.
VPBasicBlock * splitAt(iterator SplitAt)
Split current block at SplitAt by inserting a new block between the current block and its successors ...
static void insertTwoBlocksAfter(VPBlockBase *IfTrue, VPBlockBase *IfFalse, VPValue *Condition, VPBlockBase *BlockPtr)
Insert disconnected VPBlockBases IfTrue and IfFalse after BlockPtr.
void setOneSuccessor(VPBlockBase *Successor)
Set a given VPBlockBase Successor as the single successor of this VPBlockBase.
An instruction for reading from memory.
static bool classof(const VPHeaderPHIRecipe *R)
friend class VPInstruction
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
iterator_range< filter_iterator< detail::IterOfRange< RangeT >, PredicateT > > make_filter_range(RangeT &&Range, PredicateT Pred)
Convenience function that takes a range of elements and a predicate, and return a new filter_iterator...
const VPBasicBlock * getEntryBasicBlock() const
bool onlyFirstLaneUsed(VPValue *Def)
Returns true if only the first lane of Def is used.
static bool classof(const VPBlockBase *V)
Method to support type inquiry through isa, cast, and dyn_cast.
VPValue * getVPValue(unsigned I)
Returns the VPValue with index I defined by the VPDef.
bool onlyFirstLaneUsed(const VPValue *Op) const override
Returns true if the recipe only uses the first lane of operand Op.
static bool classof(const VPUser *U)
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void setUnderlyingValue(Value *Val)
void setAlsoPack(bool Pack)
const RecurrenceDescriptor & getRecurrenceDescriptor() const
iterator_range< df_iterator< T > > depth_first(const T &G)
Common register allocation spilling lr str ldr sxth r3 ldr mla r4 can lr mov lr str ldr sxth r3 mla r4 and then merge mul and lr str ldr sxth r3 mla r4 It also increase the likelihood the store may become dead bb27 Successors according to CFG
VPInstruction(unsigned Opcode, ArrayRef< VPValue * > Operands, DebugLoc DL)
void setCondBit(VPValue *CV)
Set the condition bit selecting the successor.
void addVPValue(Value *V, VPValue *VPV)
static nodes_iterator nodes_end(GraphRef N)
void setParent(VPRegionBlock *P)
void addIncoming(VPValue *IncomingV, VPBasicBlock *IncomingBlock)
Adds a pair (IncomingV, IncomingBlock) to the phi.
VPRecipeBase(const unsigned char SC, iterator_range< IterT > Operands)
const_reverse_iterator rend() const
iterator_range< mapped_iterator< Use *, std::function< VPValue *(Value *)> > > mapToVPValues(User::op_range Operands)
Returns a range mapping the values the range Operands to their corresponding VPValues.
static bool classof(const VPDef *Recipe)
Method to support type inquiry through isa, cast, and dyn_cast.
void execute(VPTransformState &State) override
Generate the phi/select nodes.
static bool isBackEdge(const VPBlockBase *FromBlock, const VPBlockBase *ToBlock, const VPLoopInfo *VPLI)
Returns true if the edge FromBlock -> ToBlock is a back-edge.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
void print(raw_ostream &O) const
Print plain-text dump of this VPlan to O.
This class can be used to assign consecutive numbers to all VPValues in a VPlan and allows querying t...
auto map_range(ContainerTy &&C, FuncTy F)
const_iterator begin() const
const VPBlockBase * getExit() const
virtual ~VPRecipeBase()=default
VPWidenGEPRecipe(GetElementPtrInst *GEP, iterator_range< IterT > Operands, Loop *OrigLoop)
VPlanSlp(VPInterleavedAccessInfo &IAI, VPBasicBlock &BB)
VPRegionBlock * getParent()
bool operator==(const VPAllSuccessorsIterator &R) const
reverse_iterator rbegin()
static VPAllSuccessorsIterator end(BlockPtrTy Block)
static bool classof(const VPDef *D)
Method to support type inquiry through isa, cast, and dyn_cast.
Iterator for intrusive lists based on ilist_node.
VPRegionBlock(const std::string &Name="", bool IsReplicator=false)
static bool classof(const VPDef *D)
Method to support type inquiry through isa, cast, and dyn_cast.
static bool classof(const VPDef *D)
Method to support type inquiry through isa, cast, and dyn_cast.
VPWidenGEPRecipe(GetElementPtrInst *GEP, iterator_range< IterT > Operands)
A recipe for handling phi nodes of integer and floating-point inductions, producing their scalar valu...
VPRecipeBase is a base class modeling a sequence of one or more output IR instructions.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print this VPRegionBlock to O (recursively), prefixing all lines with Indent.
VPBlockBase * setEntry(VPBlockBase *Block)
BlockT * getHeader() const
~VPReductionRecipe() override=default
size_type count(const key_type &key) const
Count the number of elements of a given key in the SetVector.
VPAllSuccessorsIterator(const VPAllSuccessorsIterator &Other)
This class provides computation of slot numbers for LLVM Assembly writing.
bool usesScalars(const VPValue *Op) const override
Returns true if the recipe uses scalars of operand Op.
bool mayReadOrWriteMemory() const
Returns true if the recipe may read from or write to memory.
VPValue & getVectorTripCount()
The vector trip count.
void setUnderlyingInstr(Instruction *I)
@ VPVFirstOrderRecurrencePHISC
const Type * getScalarType() const
Returns the scalar type of the induction.
void setName(const Twine &newName)
static bool blockIsLoopLatch(const VPBlockBase *Block, const VPLoopInfo *VPLInfo)
Returns true if Block is a loop latch.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void insert(VPRecipeBase *Recipe, iterator InsertPt)
VPLoopInfo & getVPLoopInfo()
Return the VPLoopInfo analysis for this VPlan.
A recipe for vectorizing a phi-node as a sequence of mask-based select instructions.
iterator insert(iterator where, pointer New)
bool isPredicated() const
This class augments VPValue with operands which provide the inverse def-use edges from VPValue's user...
VPValue * getSingleOperandOrNull()
static bool classof(const VPValue *V)
The RecurrenceDescriptor is used to identify recurrences variables in a loop.
VPlan(VPBlockBase *Entry=nullptr)
virtual void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const =0
Print plain-text dump of this VPBlockBase to O, prefixing all lines with Indent.
static bool classof(const VPValue *V)
Method to support type inquiry through isa, cast, and dyn_cast.
@ FirstOrderRecurrenceSplice
VPBlendRecipe(PHINode *Phi, ArrayRef< VPValue * > Operands)
The blend operation is a User of the incoming values and of their respective masks,...
const VPRecipeBase & front() const
iterator begin()
Recipe iterator methods.
bool isCompletelySLP() const
Return true if all visited instruction can be combined.
VPAllSuccessorsIterator operator++(int X)
void setEntry(VPBlockBase *EntryBlock)
Set EntryBlock as the entry VPBlockBase of this VPRegionBlock.
VPAllSuccessorsIterator & operator=(const VPAllSuccessorsIterator &R)
void execute(VPTransformState &State) override
Generate the wide load/store.
static bool classof(const VPDef *D)
Method to support type inquiry through isa, cast, and dyn_cast.
VPValue * getOperand(unsigned N) const
bool mayReadFromMemory() const
Returns true if the recipe may read from memory.
base_list_type::const_iterator const_iterator
hash_code hash_combine_range(InputIteratorT first, InputIteratorT last)
Compute a hash_code for a sequence of values.
static bool classof(const VPBlockBase *V)
Method to support type inquiry through isa, cast, and dyn_cast.
void printDOT(raw_ostream &O) const
Print this VPlan in DOT format to O.
VPValue * getOrCreateBackedgeTakenCount()
The backedge taken count of the original loop.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print this VPBsicBlock to O, prefixing all lines with Indent.
A range adaptor for a pair of iterators.
Recipe to expand a SCEV expression.
VPWidenRecipe(Instruction &I, iterator_range< IterT > Operands)
bool isConsecutive() const
static ChildIteratorType child_end(NodeRef N)
void execute(VPTransformState &State) override
Generate the phi nodes.
const VPRegionBlock * getParent() const
In what follows, the term "input IR" refers to code that is fed into the vectorizer whereas the term ...
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
A SetVector that performs no allocations if smaller than a certain size.
static bool classof(const VPRecipeBase *R)
Method to support type inquiry through isa, cast, and dyn_cast.
const VPLoopInfo & getVPLoopInfo() const
VPPredInstPHIRecipe(VPValue *PredV)
Construct a VPPredInstPHIRecipe given PredInst whose value needs a phi nodes after merging back from ...
This class represents a function call, abstracting a target machine's calling convention.
unsigned getVPValueID() const
VPWidenSelectRecipe(SelectInst &I, iterator_range< IterT > Operands, bool InvariantCond)
VPValue * getMask() const
Return the mask used by this recipe.
Common register allocation spilling lr str ldr sxth r3 ldr mla r4 can lr mov lr str ldr sxth r3 mla r4 and then merge mul and lr str ldr sxth r3 mla r4 It also increase the likelihood the store may become dead bb27 Successors according to LLVM BB
static void deleteCFG(VPBlockBase *Entry)
Delete all blocks reachable from a given VPBlockBase, inclusive.
bool isReplicator() const
An indicator whether this region is to generate multiple replicated instances of output IR correspond...
BlockVerifier::State From
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
@ CanonicalIVIncrementNUW
VPInstruction(unsigned Opcode, std::initializer_list< VPValue * > Operands, DebugLoc DL={})
const TruncInst * getTruncInst() const
iterator_range< iterator > phis()
Returns an iterator range over the PHI-like recipes in the block.
iterator getFirstNonPhi()
Return the position of the first non-phi node recipe in the block.
VPValue * getChainOp() const
The VPValue of the scalar Chain being accumulated.
VPInterleavedAccessInfo(VPlan &Plan, InterleavedAccessInfo &IAI)
Helper for GraphTraits specialization that traverses through VPRegionBlocks.
static bool classof(const VPValue *V)
static bool classof(const VPDef *D)
Method to support type inquiry through isa, cast, and dyn_cast.
const VPBlocksTy & getSuccessors() const
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
SmallVectorImpl< VPBlockBase * >::iterator ChildIteratorType
VPBlockRecursiveTraversalWrapper(BlockTy Entry)
~VPWidenRecipe() override=default
void execute(VPTransformState &State) override
Produce widened copies of all Ingredients.
unsigned getVPDefID() const
LLVM_DUMP_METHOD void dump() const
Print the VPInstruction to dbgs() (for debugging).
static NodeRef getEntryNode(VPBlockRecursiveTraversalWrapper< VPBlockBase * > N)
Value * getUnderlyingValue()
Return the underlying Value attached to this VPValue.
static ChildIteratorType child_end(NodeRef N)
VPlan models a candidate for vectorization, encoding various decisions take to produce efficient outp...
static nodes_iterator nodes_begin(GraphRef N)
const VPValue * getStepValue() const
void setTwoSuccessors(VPBlockBase *IfTrue, VPBlockBase *IfFalse, VPValue *Condition)
Set two given VPBlockBases IfTrue and IfFalse to be the two successors of this VPBlockBase.
~VPInterleavedAccessInfo()
void setPlan(VPlan *ParentPlan)
Sets the pointer of the plan containing the block.
LLVM Value Representation.
void insertBefore(VPRecipeBase *InsertPos)
Insert an unlinked recipe into a basic block immediately before the specified recipe.
~VPScalarIVStepsRecipe() override=default
Binary functor that adapts to any other binary functor after dereferencing operands.
static bool classof(const VPValue *V)
bool isOrdered() const
Returns true, if the phi is part of an ordered reduction.
static NodeRef getEntryNode(GraphRef N)
LLVM_DUMP_METHOD void dump() const
Dump the plan to stderr (for debugging).
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
bool isPhi() const
Returns true for PHI-like recipes.
VPValue * getStartValue()
Returns the start value of the induction.
VPlanPrinter prints a given VPlan to a given output stream.
SmallVectorImpl< VPBlockBase * >::iterator ChildIteratorType
VPUsers instance used by VPBlockBase to manage CondBit and the block predicate.
void insertAfter(VPRecipeBase *InsertPos)
Insert an unlinked Recipe into a basic block immediately after the specified Recipe.
Optional< std::vector< StOtherPiece > > Other
void resetSingleOpUser(VPValue *NewVal)
A Use represents the edge between a Value definition and its users.
VPValue * getOrCreateTripCount()
The trip count of the original loop.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
void addVF(ElementCount VF)