LLVM 20.0.0git
|
This is a concrete Recipe that models a single VPlan-level instruction. More...
#include "Transforms/Vectorize/VPlan.h"
Public Types | |
enum | { FirstOrderRecurrenceSplice , Not , SLPLoad , SLPStore , ActiveLaneMask , ExplicitVectorLength , ResumePhi , CalculateTripCountMinusVF , CanonicalIVIncrementForPart , BranchOnCount , BranchOnCond , ComputeReductionResult , ExtractFromEnd , LogicalAnd , PtrAdd , AnyOf } |
VPlan opcodes, extending LLVM IR with idiomatics instructions. More... | |
Public Types inherited from llvm::VPDef | |
using | VPRecipeTy = enum { VPBranchOnMaskSC, VPDerivedIVSC, VPExpandSCEVSC, VPIRInstructionSC, VPInstructionSC, VPInterleaveSC, VPReductionEVLSC, VPReductionSC, VPReplicateSC, VPScalarCastSC, VPScalarIVStepsSC, VPVectorPointerSC, VPReverseVectorPointerSC, VPWidenCallSC, VPWidenCanonicalIVSC, VPWidenCastSC, VPWidenGEPSC, VPWidenIntrinsicSC, VPWidenLoadEVLSC, VPWidenLoadSC, VPWidenStoreEVLSC, VPWidenStoreSC, VPWidenSC, VPWidenEVLSC, VPWidenSelectSC, VPBlendSC, VPHistogramSC, VPWidenPHISC, VPPredInstPHISC, VPCanonicalIVPHISC, VPActiveLaneMaskPHISC, VPEVLBasedIVPHISC, VPFirstOrderRecurrencePHISC, VPWidenIntOrFpInductionSC, VPWidenPointerInductionSC, VPScalarPHISC, VPReductionPHISC, VPFirstPHISC=VPWidenPHISC, VPFirstHeaderPHISC=VPCanonicalIVPHISC, VPLastHeaderPHISC=VPReductionPHISC, VPLastPHISC=VPReductionPHISC, } |
An enumeration for keeping track of the concrete subclass of VPRecipeBase that is actually instantiated. | |
Public Types inherited from llvm::VPUser | |
typedef SmallVectorImpl< VPValue * >::iterator | operand_iterator |
typedef SmallVectorImpl< VPValue * >::const_iterator | const_operand_iterator |
typedef iterator_range< operand_iterator > | operand_range |
typedef iterator_range< const_operand_iterator > | const_operand_range |
Public Types inherited from llvm::VPValue | |
enum | { VPValueSC , VPVRecipeSC } |
An enumeration for keeping track of the concrete subclass of VPValue that are actually instantiated. More... | |
typedef SmallVectorImpl< VPUser * >::iterator | user_iterator |
typedef SmallVectorImpl< VPUser * >::const_iterator | const_user_iterator |
typedef iterator_range< user_iterator > | user_range |
typedef iterator_range< const_user_iterator > | const_user_range |
Public Member Functions | |
VPInstruction (unsigned Opcode, ArrayRef< VPValue * > Operands, DebugLoc DL, const Twine &Name="") | |
VPInstruction (unsigned Opcode, std::initializer_list< VPValue * > Operands, DebugLoc DL={}, const Twine &Name="") | |
VPInstruction (unsigned Opcode, CmpInst::Predicate Pred, VPValue *A, VPValue *B, DebugLoc DL={}, const Twine &Name="") | |
VPInstruction (unsigned Opcode, std::initializer_list< VPValue * > Operands, WrapFlagsTy WrapFlags, DebugLoc DL={}, const Twine &Name="") | |
VPInstruction (unsigned Opcode, std::initializer_list< VPValue * > Operands, DisjointFlagsTy DisjointFlag, DebugLoc DL={}, const Twine &Name="") | |
VPInstruction (VPValue *Ptr, VPValue *Offset, GEPNoWrapFlags Flags, DebugLoc DL={}, const Twine &Name="") | |
VPInstruction (unsigned Opcode, std::initializer_list< VPValue * > Operands, FastMathFlags FMFs, DebugLoc DL={}, const Twine &Name="") | |
VPInstruction * | clone () override |
Clone the current recipe. | |
unsigned | getOpcode () const |
void | execute (VPTransformState &State) override |
Generate the instruction. | |
InstructionCost | computeCost (ElementCount VF, VPCostContext &Ctx) const override |
Return the cost of this VPInstruction. | |
void | print (raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override |
Print the VPInstruction to O . | |
LLVM_DUMP_METHOD void | dump () const |
Print the VPInstruction to dbgs() (for debugging). | |
bool | hasResult () const |
bool | onlyFirstLaneUsed (const VPValue *Op) const override |
Returns true if the recipe only uses the first lane of operand Op . | |
bool | onlyFirstPartUsed (const VPValue *Op) const override |
Returns true if the recipe only uses the first part of operand Op . | |
bool | isVectorToScalar () const |
Returns true if this VPInstruction produces a scalar value from a vector, e.g. | |
bool | isSingleScalar () const |
Returns true if this VPInstruction's operands are single scalars and the result is also a single scalar. | |
StringRef | getName () const |
Returns the symbolic name assigned to the VPInstruction. | |
Public Member Functions inherited from llvm::VPRecipeWithIRFlags | |
template<typename IterT > | |
VPRecipeWithIRFlags (const unsigned char SC, IterT Operands, DebugLoc DL={}) | |
template<typename IterT > | |
VPRecipeWithIRFlags (const unsigned char SC, IterT Operands, Instruction &I) | |
template<typename IterT > | |
VPRecipeWithIRFlags (const unsigned char SC, IterT Operands, CmpInst::Predicate Pred, DebugLoc DL={}) | |
template<typename IterT > | |
VPRecipeWithIRFlags (const unsigned char SC, IterT Operands, WrapFlagsTy WrapFlags, DebugLoc DL={}) | |
template<typename IterT > | |
VPRecipeWithIRFlags (const unsigned char SC, IterT Operands, FastMathFlags FMFs, DebugLoc DL={}) | |
template<typename IterT > | |
VPRecipeWithIRFlags (const unsigned char SC, IterT Operands, DisjointFlagsTy DisjointFlags, DebugLoc DL={}) | |
void | dropPoisonGeneratingFlags () |
Drop all poison-generating flags. | |
void | setFlags (Instruction *I) const |
Set the IR flags for I . | |
CmpInst::Predicate | getPredicate () const |
GEPNoWrapFlags | getGEPNoWrapFlags () const |
bool | hasFastMathFlags () const |
Returns true if the recipe has fast-math flags. | |
FastMathFlags | getFastMathFlags () const |
bool | hasNoUnsignedWrap () const |
bool | hasNoSignedWrap () const |
bool | isDisjoint () const |
void | printFlags (raw_ostream &O) const |
Public Member Functions inherited from llvm::VPSingleDefRecipe | |
template<typename IterT > | |
VPSingleDefRecipe (const unsigned char SC, IterT Operands, DebugLoc DL={}) | |
VPSingleDefRecipe (const unsigned char SC, ArrayRef< VPValue * > Operands, DebugLoc DL={}) | |
template<typename IterT > | |
VPSingleDefRecipe (const unsigned char SC, IterT Operands, Value *UV, DebugLoc DL={}) | |
virtual VPSingleDefRecipe * | clone () override=0 |
Clone the current recipe. | |
Instruction * | getUnderlyingInstr () |
Returns the underlying instruction. | |
const Instruction * | getUnderlyingInstr () const |
LLVM_DUMP_METHOD void | dump () const |
Print this VPSingleDefRecipe to dbgs() (for debugging). | |
Public Member Functions inherited from llvm::VPRecipeBase | |
VPRecipeBase (const unsigned char SC, ArrayRef< VPValue * > Operands, DebugLoc DL={}) | |
template<typename IterT > | |
VPRecipeBase (const unsigned char SC, iterator_range< IterT > Operands, DebugLoc DL={}) | |
virtual | ~VPRecipeBase ()=default |
virtual VPRecipeBase * | clone ()=0 |
Clone the current recipe. | |
VPBasicBlock * | getParent () |
const VPBasicBlock * | getParent () const |
virtual void | execute (VPTransformState &State)=0 |
The method which generates the output IR instructions that correspond to this VPRecipe, thereby "executing" the VPlan. | |
InstructionCost | cost (ElementCount VF, VPCostContext &Ctx) |
Return the cost of this recipe, taking into account if the cost computation should be skipped and the ForceTargetInstructionCost flag. | |
void | insertBefore (VPRecipeBase *InsertPos) |
Insert an unlinked recipe into a basic block immediately before the specified recipe. | |
void | insertBefore (VPBasicBlock &BB, iplist< VPRecipeBase >::iterator IP) |
Insert an unlinked recipe into BB immediately before the insertion point IP ;. | |
void | insertAfter (VPRecipeBase *InsertPos) |
Insert an unlinked Recipe into a basic block immediately after the specified Recipe. | |
void | moveAfter (VPRecipeBase *MovePos) |
Unlink this recipe from its current VPBasicBlock and insert it into the VPBasicBlock that MovePos lives in, right after MovePos. | |
void | moveBefore (VPBasicBlock &BB, iplist< VPRecipeBase >::iterator I) |
Unlink this recipe and insert into BB before I. | |
void | removeFromParent () |
This method unlinks 'this' from the containing basic block, but does not delete it. | |
iplist< VPRecipeBase >::iterator | eraseFromParent () |
This method unlinks 'this' from the containing basic block and deletes it. | |
bool | mayHaveSideEffects () const |
Returns true if the recipe may have side-effects. | |
bool | isPhi () const |
Returns true for PHI-like recipes. | |
bool | mayReadFromMemory () const |
Returns true if the recipe may read from memory. | |
bool | mayWriteToMemory () const |
Returns true if the recipe may write to memory. | |
bool | mayReadOrWriteMemory () const |
Returns true if the recipe may read from or write to memory. | |
DebugLoc | getDebugLoc () const |
Returns the debug location of the recipe. | |
Public Member Functions inherited from llvm::ilist_node_with_parent< VPRecipeBase, VPBasicBlock > | |
VPRecipeBase * | getPrevNode () |
const VPRecipeBase * | getPrevNode () const |
Get the previous node, or nullptr for the list head. | |
VPRecipeBase * | getNextNode () |
Get the next node, or nullptr for the list tail. | |
const VPRecipeBase * | getNextNode () const |
Get the next node, or nullptr for the list tail. | |
Public Member Functions inherited from llvm::ilist_node_impl< ilist_detail::compute_node_options< T, Options... >::type > | |
self_iterator | getIterator () |
const_self_iterator | getIterator () const |
reverse_self_iterator | getReverseIterator () |
const_reverse_self_iterator | getReverseIterator () const |
bool | isSentinel () const |
Check whether this is the sentinel node. | |
Public Member Functions inherited from llvm::ilist_detail::node_parent_access< NodeTy, ParentTy > | |
const ParentTy * | getParent () const |
ParentTy * | getParent () |
void | setParent (ParentTy *Parent) |
Public Member Functions inherited from llvm::VPDef | |
VPDef (const unsigned char SC) | |
virtual | ~VPDef () |
VPValue * | getVPSingleValue () |
Returns the only VPValue defined by the VPDef. | |
const VPValue * | getVPSingleValue () const |
VPValue * | getVPValue (unsigned I) |
Returns the VPValue with index I defined by the VPDef. | |
const VPValue * | getVPValue (unsigned I) const |
ArrayRef< VPValue * > | definedValues () |
Returns an ArrayRef of the values defined by the VPDef. | |
ArrayRef< VPValue * > | definedValues () const |
Returns an ArrayRef of the values defined by the VPDef. | |
unsigned | getNumDefinedValues () const |
Returns the number of values defined by the VPDef. | |
unsigned | getVPDefID () const |
void | dump () const |
Dump the VPDef to stderr (for debugging). | |
virtual void | print (raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const =0 |
Each concrete VPDef prints itself. | |
Public Member Functions inherited from llvm::VPUser | |
VPUser ()=delete | |
VPUser (const VPUser &)=delete | |
VPUser & | operator= (const VPUser &)=delete |
virtual | ~VPUser () |
void | addOperand (VPValue *Operand) |
unsigned | getNumOperands () const |
VPValue * | getOperand (unsigned N) const |
void | setOperand (unsigned I, VPValue *New) |
operand_iterator | op_begin () |
const_operand_iterator | op_begin () const |
operand_iterator | op_end () |
const_operand_iterator | op_end () const |
operand_range | operands () |
const_operand_range | operands () const |
virtual bool | usesScalars (const VPValue *Op) const |
Returns true if the VPUser uses scalars of operand Op . | |
virtual bool | onlyFirstLaneUsed (const VPValue *Op) const |
Returns true if the VPUser only uses the first lane of operand Op . | |
virtual bool | onlyFirstPartUsed (const VPValue *Op) const |
Returns true if the VPUser only uses the first part of operand Op . | |
Public Member Functions inherited from llvm::VPValue | |
Value * | getUnderlyingValue () const |
Return the underlying Value attached to this VPValue. | |
VPValue (Value *UV=nullptr) | |
Create a live-in VPValue. | |
VPValue (VPDef *Def, Value *UV=nullptr) | |
Create a VPValue for a Def which is a subclass of VPValue. | |
VPValue (Value *UV, VPDef *Def) | |
Create a VPValue for a Def which defines multiple values. | |
VPValue (const VPValue &)=delete | |
VPValue & | operator= (const VPValue &)=delete |
virtual | ~VPValue () |
unsigned | getVPValueID () const |
void | printAsOperand (raw_ostream &OS, VPSlotTracker &Tracker) const |
void | print (raw_ostream &OS, VPSlotTracker &Tracker) const |
void | dump () const |
Dump the value to stderr (for debugging). | |
unsigned | getNumUsers () const |
void | addUser (VPUser &User) |
void | removeUser (VPUser &User) |
Remove a single User from the list of users. | |
user_iterator | user_begin () |
const_user_iterator | user_begin () const |
user_iterator | user_end () |
const_user_iterator | user_end () const |
user_range | users () |
const_user_range | users () const |
bool | hasMoreThanOneUniqueUser () const |
Returns true if the value has more than one unique user. | |
void | replaceAllUsesWith (VPValue *New) |
void | replaceUsesWithIf (VPValue *New, llvm::function_ref< bool(VPUser &U, unsigned Idx)> ShouldReplace) |
Go through the uses list for this VPValue and make each use point to New if the callback ShouldReplace returns true for the given use specified by a pair of (VPUser, the use index). | |
VPRecipeBase * | getDefiningRecipe () |
Returns the recipe defining this VPValue or nullptr if it is not defined by a recipe, i.e. | |
const VPRecipeBase * | getDefiningRecipe () const |
bool | hasDefiningRecipe () const |
Returns true if this VPValue is defined by a recipe. | |
bool | isLiveIn () const |
Returns true if this VPValue is a live-in, i.e. defined outside the VPlan. | |
Value * | getLiveInIRValue () |
Returns the underlying IR value, if this VPValue is defined outside the scope of VPlan. | |
const Value * | getLiveInIRValue () const |
bool | isDefinedOutsideLoopRegions () const |
Returns true if the VPValue is defined outside any loop region. | |
void | setUnderlyingValue (Value *Val) |
Friends | |
class | VPlanSlp |
Additional Inherited Members | |
Static Public Member Functions inherited from llvm::VPRecipeWithIRFlags | |
static bool | classof (const VPRecipeBase *R) |
static bool | classof (const VPUser *U) |
Static Public Member Functions inherited from llvm::VPSingleDefRecipe | |
static bool | classof (const VPRecipeBase *R) |
static bool | classof (const VPUser *U) |
Static Public Member Functions inherited from llvm::VPRecipeBase | |
static bool | classof (const VPDef *D) |
Method to support type inquiry through isa, cast, and dyn_cast. | |
static bool | classof (const VPUser *U) |
Protected Types inherited from llvm::ilist_node_impl< ilist_detail::compute_node_options< T, Options... >::type > | |
using | self_iterator = typename ilist_select_iterator_type< OptionsT::has_iterator_bits, ilist_detail::compute_node_options< T, Options... >::type, false, false >::type |
using | const_self_iterator = typename ilist_select_iterator_type< OptionsT::has_iterator_bits, ilist_detail::compute_node_options< T, Options... >::type, false, true >::type |
using | reverse_self_iterator = typename ilist_select_iterator_type< OptionsT::has_iterator_bits, ilist_detail::compute_node_options< T, Options... >::type, true, false >::type |
using | const_reverse_self_iterator = typename ilist_select_iterator_type< OptionsT::has_iterator_bits, ilist_detail::compute_node_options< T, Options... >::type, true, true >::type |
Protected Member Functions inherited from llvm::VPRecipeWithIRFlags | |
void | transferFlags (VPRecipeWithIRFlags &Other) |
template<typename IterT > | |
VPRecipeWithIRFlags (const unsigned char SC, IterT Operands, GEPNoWrapFlags GEPFlags, DebugLoc DL={}) | |
virtual InstructionCost | computeCost (ElementCount VF, VPCostContext &Ctx) const |
Compute the cost of this recipe either using a recipe's specialized implementation or using the legacy cost model and the underlying instructions. | |
Protected Member Functions inherited from llvm::ilist_node_with_parent< VPRecipeBase, VPBasicBlock > | |
ilist_node_with_parent ()=default | |
Protected Member Functions inherited from llvm::ilist_node_impl< ilist_detail::compute_node_options< T, Options... >::type > | |
ilist_node_impl ()=default | |
Protected Member Functions inherited from llvm::VPUser | |
void | printOperands (raw_ostream &O, VPSlotTracker &SlotTracker) const |
Print the operands to O . | |
VPUser (ArrayRef< VPValue * > Operands) | |
VPUser (std::initializer_list< VPValue * > Operands) | |
template<typename IterT > | |
VPUser (iterator_range< IterT > Operands) | |
Protected Member Functions inherited from llvm::VPValue | |
VPValue (const unsigned char SC, Value *UV=nullptr, VPDef *Def=nullptr) | |
Protected Member Functions inherited from llvm::VPUnrollPartAccessor< 1 > | |
VPValue * | getUnrollPartOperand (VPUser &U) const |
Return the VPValue operand containing the unroll part or null if there is no such operand. | |
unsigned | getUnrollPart (VPUser &U) const |
Return the unroll part. | |
Protected Attributes inherited from llvm::VPValue | |
Value * | UnderlyingVal |
VPDef * | Def |
Pointer to the VPDef that defines this VPValue. | |
This is a concrete Recipe that models a single VPlan-level instruction.
While as any Recipe it may generate a sequence of IR instructions when executed, these instructions would always form a single-def expression as the VPInstruction is also a single def-use vertex.
anonymous enum |
VPlan opcodes, extending LLVM IR with idiomatics instructions.
Enumerator | |
---|---|
FirstOrderRecurrenceSplice | |
Not | |
SLPLoad | |
SLPStore | |
ActiveLaneMask | |
ExplicitVectorLength | |
ResumePhi | Creates a scalar phi in a leaf VPBB with a single predecessor in VPlan. The first operand is the incoming value from the predecessor in VPlan, the second operand is the incoming value for all other predecessors (which are currently not modeled in VPlan). |
CalculateTripCountMinusVF | |
CanonicalIVIncrementForPart | |
BranchOnCount | |
BranchOnCond | |
ComputeReductionResult | |
ExtractFromEnd | |
LogicalAnd | |
PtrAdd | |
AnyOf |
VPInstruction::VPInstruction | ( | unsigned | Opcode, |
CmpInst::Predicate | Pred, | ||
VPValue * | A, | ||
VPValue * | B, | ||
DebugLoc | DL = {} , |
||
const Twine & | Name = "" |
||
) |
Definition at line 328 of file VPlanRecipes.cpp.
VPInstruction::VPInstruction | ( | unsigned | Opcode, |
std::initializer_list< VPValue * > | Operands, | ||
FastMathFlags | FMFs, | ||
DebugLoc | DL = {} , |
||
const Twine & | Name = "" |
||
) |
Definition at line 338 of file VPlanRecipes.cpp.
References assert().
|
inlineoverridevirtual |
Clone the current recipe.
Implements llvm::VPSingleDefRecipe.
Definition at line 1308 of file VPlan.h.
References llvm::VPRecipeBase::getDebugLoc(), Name, Operands, and llvm::VPUser::operands().
|
inlineoverridevirtual |
Return the cost of this VPInstruction.
Reimplemented from llvm::VPRecipeBase.
void VPInstruction::dump | ( | ) | const |
Print the VPInstruction to dbgs() (for debugging).
Definition at line 756 of file VPlanRecipes.cpp.
References llvm::dbgs(), llvm::VPRecipeBase::getParent(), and print().
|
overridevirtual |
Generate the instruction.
TODO: We currently execute only per-part unless a specific instance is provided.
Implements llvm::VPRecipeBase.
Definition at line 675 of file VPlanRecipes.cpp.
References assert(), llvm::VPTransformState::Builder, llvm::VPRecipeBase::getDebugLoc(), llvm::VPRecipeWithIRFlags::getFastMathFlags(), llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getKnownMinValue(), getOpcode(), llvm::Value::getType(), llvm::VPRecipeWithIRFlags::hasFastMathFlags(), hasResult(), llvm::ElementCount::isScalar(), isSingleScalar(), isVectorToScalar(), llvm::Type::isVectorTy(), llvm::VPTransformState::Lane, llvm::vputils::onlyFirstLaneUsed(), llvm::VPTransformState::set(), llvm::VPTransformState::setDebugLocFrom(), llvm::IRBuilderBase::setFastMathFlags(), and llvm::VPTransformState::VF.
|
inline |
Returns the symbolic name assigned to the VPInstruction.
Definition at line 1375 of file VPlan.h.
References Name.
|
inline |
Definition at line 1315 of file VPlan.h.
Referenced by execute(), hasResult(), isSingleScalar(), isVectorToScalar(), onlyFirstLaneUsed(), onlyFirstPartUsed(), and print().
|
inline |
Definition at line 1338 of file VPlan.h.
References BranchOnCond, BranchOnCount, and getOpcode().
bool VPInstruction::isSingleScalar | ( | ) | const |
Returns true if this VPInstruction's operands are single scalars and the result is also a single scalar.
Definition at line 660 of file VPlanRecipes.cpp.
References getOpcode(), and ResumePhi.
Referenced by execute().
bool VPInstruction::isVectorToScalar | ( | ) | const |
Returns true if this VPInstruction produces a scalar value from a vector, e.g.
by performing a reduction or extracting a lane.
Definition at line 654 of file VPlanRecipes.cpp.
References AnyOf, ComputeReductionResult, ExtractFromEnd, and getOpcode().
Referenced by execute().
Returns true if the recipe only uses the first lane of operand Op
.
Reimplemented from llvm::VPUser.
Definition at line 710 of file VPlanRecipes.cpp.
References ActiveLaneMask, assert(), BranchOnCond, BranchOnCount, CalculateTripCountMinusVF, CanonicalIVIncrementForPart, ExplicitVectorLength, getOpcode(), llvm::is_contained(), llvm::Instruction::isBinaryOp(), llvm_unreachable, llvm::vputils::onlyFirstLaneUsed(), llvm::VPUser::operands(), PtrAdd, and ResumePhi.
Returns true if the recipe only uses the first part of operand Op
.
Reimplemented from llvm::VPUser.
Definition at line 736 of file VPlanRecipes.cpp.
References assert(), BranchOnCond, BranchOnCount, CanonicalIVIncrementForPart, getOpcode(), llvm::is_contained(), llvm::Instruction::isBinaryOp(), llvm_unreachable, llvm::vputils::onlyFirstPartUsed(), and llvm::VPUser::operands().
|
overridevirtual |
Print the VPInstruction to O
.
Implements llvm::VPDef.
Definition at line 761 of file VPlanRecipes.cpp.
References ActiveLaneMask, AnyOf, BranchOnCond, BranchOnCount, CalculateTripCountMinusVF, CanonicalIVIncrementForPart, ComputeReductionResult, DL, ExplicitVectorLength, ExtractFromEnd, FirstOrderRecurrenceSplice, llvm::VPRecipeBase::getDebugLoc(), getOpcode(), llvm::Instruction::getOpcodeName(), hasResult(), LogicalAnd, Not, llvm::MachineBasicBlock::print(), llvm::VPValue::printAsOperand(), llvm::VPRecipeWithIRFlags::printFlags(), llvm::VPUser::printOperands(), PtrAdd, ResumePhi, SLPLoad, and SLPStore.
Referenced by dump().