37#define VPINTERNAL_VPLEGAL_CASES \
38 VPINTERNAL_CASE(Legal) \
39 VPINTERNAL_CASE(Discard) \
40 VPINTERNAL_CASE(Convert)
42#define VPINTERNAL_CASE(X) "|" #X
48 ". If non-empty, ignore "
49 "TargetTransformInfo and "
50 "always use this transformation for the %evl parameter (Used in "
56 ". If non-empty, Ignore "
57 "TargetTransformInfo and "
58 "always use this transformation for the %mask parameter (Used in "
62#define VPINTERNAL_CASE(X) .Case(#X, VPLegalization::X)
68#undef VPINTERNAL_VPLEGAL_CASES
75#define DEBUG_TYPE "expandvp"
77STATISTIC(NumFoldedVL,
"Number of folded vector length params");
78STATISTIC(NumLoweredVPOps,
"Number of folded vector predication operations");
86 return ConstValue->isAllOnesValue();
94 return ConstantInt::get(DivTy, 1u,
false);
107 NewInst->setFastMathFlags(OldFMOp->getFastMathFlags());
136struct CachingVPExpander {
137 const TargetTransformInfo &TTI;
150 ElementCount ElemCount);
154 bool foldEVLIntoMask(VPIntrinsic &VPI);
159 bool discardEVLParameter(VPIntrinsic &PI);
162 bool expandPredicationInBinaryOperator(
IRBuilder<> &Builder, VPIntrinsic &PI);
165 bool expandPredicationToIntCall(
IRBuilder<> &Builder, VPIntrinsic &PI);
168 bool expandPredicationToFPCall(
IRBuilder<> &Builder, VPIntrinsic &PI,
169 unsigned UnpredicatedIntrinsicID);
172 bool expandPredicationInReduction(
IRBuilder<> &Builder,
173 VPReductionIntrinsic &PI);
176 bool expandPredicationToCastIntrinsic(
IRBuilder<> &Builder, VPIntrinsic &VPI);
179 bool expandPredicationInMemoryIntrinsic(
IRBuilder<> &Builder,
183 bool expandPredicationInComparison(
IRBuilder<> &Builder, VPCmpIntrinsic &PI);
186 bool expandPredication(VPIntrinsic &PI);
190 VPLegalization getVPLegalizationStrategy(
const VPIntrinsic &VPI)
const;
191 bool UsingTTIOverrides;
194 CachingVPExpander(
const TargetTransformInfo &TTI)
210 Type *BoolVecTy = VectorType::get(Builder.
getInt1Ty(), ElemCount);
214 {BoolVecTy, EVLParam->
getType()},
215 {ConstZero, EVLParam});
226bool CachingVPExpander::expandPredicationInBinaryOperator(
IRBuilder<> &Builder,
229 "Implicitly dropping %evl in non-speculatable operator!");
246 case Instruction::UDiv:
247 case Instruction::SDiv:
248 case Instruction::URem:
249 case Instruction::SRem:
262bool CachingVPExpander::expandPredicationToIntCall(
IRBuilder<> &Builder,
277bool CachingVPExpander::expandPredicationToFPCall(
278 IRBuilder<> &Builder, VPIntrinsic &VPI,
unsigned UnpredicatedIntrinsicID) {
280 "Implicitly dropping %evl in non-speculatable operator!");
282 switch (UnpredicatedIntrinsicID) {
283 case Intrinsic::fabs:
284 case Intrinsic::sqrt:
285 case Intrinsic::maxnum:
286 case Intrinsic::minnum: {
298 case Intrinsic::fmuladd:
299 case Intrinsic::experimental_constrained_fma:
300 case Intrinsic::experimental_constrained_fmuladd: {
305 VPI.
getModule(), UnpredicatedIntrinsicID, {VPI.getType()});
320static Value *getNeutralReductionElement(
const VPReductionIntrinsic &VPI,
329bool CachingVPExpander::expandPredicationInReduction(
332 "Implicitly dropping %evl in non-speculatable operator!");
339 auto *NeutralElt = getNeutralReductionElement(VPI, VPI.
getType());
342 RedOp = Builder.
CreateSelect(Mask, RedOp, NeutralVector);
351 case Intrinsic::vp_reduce_add:
352 case Intrinsic::vp_reduce_mul:
353 case Intrinsic::vp_reduce_and:
354 case Intrinsic::vp_reduce_or:
355 case Intrinsic::vp_reduce_xor: {
364 case Intrinsic::vp_reduce_smax:
365 case Intrinsic::vp_reduce_smin:
366 case Intrinsic::vp_reduce_umax:
367 case Intrinsic::vp_reduce_umin:
368 case Intrinsic::vp_reduce_fmax:
369 case Intrinsic::vp_reduce_fmin:
370 case Intrinsic::vp_reduce_fmaximum:
371 case Intrinsic::vp_reduce_fminimum: {
379 case Intrinsic::vp_reduce_fadd:
382 case Intrinsic::vp_reduce_fmul:
391bool CachingVPExpander::expandPredicationToCastIntrinsic(
IRBuilder<> &Builder,
404bool CachingVPExpander::expandPredicationInMemoryIntrinsic(
IRBuilder<> &Builder,
417 Value *NewMemoryInst =
nullptr;
421 case Intrinsic::vp_store:
423 StoreInst *NewStore =
425 if (AlignOpt.has_value())
427 NewMemoryInst = NewStore;
430 DataParam, PtrParam, AlignOpt.
valueOrOne(), MaskParam);
433 case Intrinsic::vp_load:
437 if (AlignOpt.has_value())
439 NewMemoryInst = NewLoad;
445 case Intrinsic::vp_scatter: {
450 AlignOpt.value_or(
DL.getPrefTypeAlign(ElementType)), MaskParam);
453 case Intrinsic::vp_gather: {
457 AlignOpt.value_or(
DL.getPrefTypeAlign(ElementType)), MaskParam,
nullptr,
468bool CachingVPExpander::expandPredicationInComparison(
IRBuilder<> &Builder,
469 VPCmpIntrinsic &VPI) {
471 "Implicitly dropping %evl in non-speculatable operator!");
480 auto *NewCmp = Builder.
CreateCmp(Pred, Op0, Op1);
486bool CachingVPExpander::discardEVLParameter(VPIntrinsic &VPI) {
497 Value *MaxEVL =
nullptr;
504 MaxEVL = Builder.
CreateNUWMul(VScale, FactorConst,
"scalable_size");
512bool CachingVPExpander::foldEVLIntoMask(VPIntrinsic &VPI) {
526 "Unexpected VP intrinsic without mask operand");
531 assert(OldMaskParam &&
"no mask param to fold the vl param into");
532 assert(OldEVLParam &&
"no EVL param to fold away");
539 Value *VLMask = convertEVLToMask(Builder, OldEVLParam, ElemCount);
548 discardEVLParameter(VPI);
550 "transformation did not render the evl param ineffective!");
556bool CachingVPExpander::expandPredication(VPIntrinsic &VPI) {
557 LLVM_DEBUG(
dbgs() <<
"Lowering to unpredicated op: " << VPI <<
'\n');
565 return expandPredicationInBinaryOperator(Builder, VPI);
568 return expandPredicationInReduction(Builder, *VPRI);
571 return expandPredicationInComparison(Builder, *VPCmp);
574 return expandPredicationToCastIntrinsic(Builder, VPI);
579 case Intrinsic::vp_fneg: {
584 case Intrinsic::vp_select:
585 case Intrinsic::vp_merge: {
592 case Intrinsic::vp_abs:
593 case Intrinsic::vp_smax:
594 case Intrinsic::vp_smin:
595 case Intrinsic::vp_umax:
596 case Intrinsic::vp_umin:
597 case Intrinsic::vp_bswap:
598 case Intrinsic::vp_bitreverse:
599 case Intrinsic::vp_ctpop:
600 case Intrinsic::vp_ctlz:
601 case Intrinsic::vp_cttz:
602 case Intrinsic::vp_sadd_sat:
603 case Intrinsic::vp_uadd_sat:
604 case Intrinsic::vp_ssub_sat:
605 case Intrinsic::vp_usub_sat:
606 case Intrinsic::vp_fshl:
607 case Intrinsic::vp_fshr:
608 return expandPredicationToIntCall(Builder, VPI);
609 case Intrinsic::vp_fabs:
610 case Intrinsic::vp_sqrt:
611 case Intrinsic::vp_maxnum:
612 case Intrinsic::vp_minnum:
613 case Intrinsic::vp_maximum:
614 case Intrinsic::vp_minimum:
615 case Intrinsic::vp_fma:
616 case Intrinsic::vp_fmuladd:
617 return expandPredicationToFPCall(Builder, VPI,
619 case Intrinsic::vp_load:
620 case Intrinsic::vp_store:
621 case Intrinsic::vp_gather:
622 case Intrinsic::vp_scatter:
623 return expandPredicationInMemoryIntrinsic(Builder, VPI);
627 if (expandPredicationToFPCall(Builder, VPI, *CID))
635void sanitizeStrategy(VPIntrinsic &VPI,
VPLegalization &LegalizeStrat) {
657CachingVPExpander::getVPLegalizationStrategy(
const VPIntrinsic &VPI)
const {
672CachingVPExpander::expandVectorPredication(VPIntrinsic &VPI) {
673 auto Strategy = getVPLegalizationStrategy(VPI);
674 sanitizeStrategy(VPI, Strategy);
679 switch (Strategy.EVLParamStrategy) {
683 if (discardEVLParameter(VPI))
684 Changed = VPExpansionDetails::IntrinsicUpdated;
687 if (foldEVLIntoMask(VPI)) {
688 Changed = VPExpansionDetails::IntrinsicUpdated;
695 switch (Strategy.OpStrategy) {
701 if (expandPredication(VPI)) {
703 Changed = VPExpansionDetails::IntrinsicReplaced;
715 return CachingVPExpander(
TTI).expandVectorPredication(VPI);
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
#define LLVM_LIKELY(EXPR)
This file contains the declarations for the subclasses of Constant, which represent the different fla...
static VPTransform parseOverrideOption(const std::string &TextOpt)
static cl::opt< std::string > MaskTransformOverride("expandvp-override-mask-transform", cl::init(""), cl::Hidden, cl::desc("Options: <empty>" VPINTERNAL_VPLEGAL_CASES ". If non-empty, Ignore " "TargetTransformInfo and " "always use this transformation for the %mask parameter (Used in " "testing)."))
static cl::opt< std::string > EVLTransformOverride("expandvp-override-evl-transform", cl::init(""), cl::Hidden, cl::desc("Options: <empty>" VPINTERNAL_VPLEGAL_CASES ". If non-empty, ignore " "TargetTransformInfo and " "always use this transformation for the %evl parameter (Used in " "testing)."))
static void replaceOperation(Value &NewOp, VPIntrinsic &OldOp)
Transfer all properties from OldOp to NewOp and replace all uses.
static bool isAllTrueMask(Value *MaskVal)
static void transferDecorations(Value &NewVal, VPIntrinsic &VPI)
Transfer operation properties from OldVPI to NewVal.
TargetTransformInfo::VPLegalization VPLegalization
TargetTransformInfo::VPLegalization::VPTransform VPTransform
static bool anyExpandVPOverridesSet()
static bool maySpeculateLanes(VPIntrinsic &VPI)
static Constant * getSafeDivisor(Type *DivTy)
#define VPINTERNAL_VPLEGAL_CASES
loop Loop Strength Reduction
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
LLVM_ABI bool hasAttribute(Attribute::AttrKind Kind) const
Return true if the attribute exists in this set.
Value * getArgOperand(unsigned i) const
void setArgOperand(unsigned i, Value *v)
@ ICMP_ULT
unsigned less than
This is an important base class in LLVM.
Value * CreateNUWMul(Value *LHS, Value *RHS, const Twine &Name="")
LLVM_ABI CallInst * CreateFAddReduce(Value *Acc, Value *Src)
Create a sequential vector fadd reduction intrinsic of the source vector.
IntegerType * getInt1Ty()
Fetch the type representing a single bit.
LLVM_ABI Value * CreateVectorSplat(unsigned NumElts, Value *V, const Twine &Name="")
Return a vector value that contains.
LLVM_ABI CallInst * CreateMaskedLoad(Type *Ty, Value *Ptr, Align Alignment, Value *Mask, Value *PassThru=nullptr, const Twine &Name="")
Create a call to Masked Load intrinsic.
LLVM_ABI CallInst * CreateConstrainedFPCall(Function *Callee, ArrayRef< Value * > Args, const Twine &Name="", std::optional< RoundingMode > Rounding=std::nullopt, std::optional< fp::ExceptionBehavior > Except=std::nullopt)
LLVM_ABI Value * CreateSelect(Value *C, Value *True, Value *False, const Twine &Name="", Instruction *MDFrom=nullptr)
Value * CreateCast(Instruction::CastOps Op, Value *V, Type *DestTy, const Twine &Name="", MDNode *FPMathTag=nullptr, FMFSource FMFSource={})
Value * CreateVScale(Type *Ty, const Twine &Name="")
Create a call to llvm.vscale.<Ty>().
LLVM_ABI Value * CreateBinaryIntrinsic(Intrinsic::ID ID, Value *LHS, Value *RHS, FMFSource FMFSource={}, const Twine &Name="")
Create a call to intrinsic ID with 2 operands which is mangled on the first type.
LLVM_ABI CallInst * CreateIntrinsic(Intrinsic::ID ID, ArrayRef< Type * > Types, ArrayRef< Value * > Args, FMFSource FMFSource={}, const Twine &Name="")
Create a call to intrinsic ID with Args, mangled using Types.
ConstantInt * getInt32(uint32_t C)
Get a constant 32-bit value.
Value * CreateCmp(CmpInst::Predicate Pred, Value *LHS, Value *RHS, const Twine &Name="", MDNode *FPMathTag=nullptr)
LLVM_ABI CallInst * CreateUnaryIntrinsic(Intrinsic::ID ID, Value *V, FMFSource FMFSource={}, const Twine &Name="")
Create a call to intrinsic ID with 1 operand which is mangled on its type.
LoadInst * CreateLoad(Type *Ty, Value *Ptr, const char *Name)
Provided to resolve 'CreateLoad(Ty, Ptr, "...")' correctly, instead of converting the string to 'bool...
Value * CreateAnd(Value *LHS, Value *RHS, const Twine &Name="")
StoreInst * CreateStore(Value *Val, Value *Ptr, bool isVolatile=false)
LLVM_ABI CallInst * CreateMaskedStore(Value *Val, Value *Ptr, Align Alignment, Value *Mask)
Create a call to Masked Store intrinsic.
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args={}, const Twine &Name="", MDNode *FPMathTag=nullptr)
Value * CreateBinOp(Instruction::BinaryOps Opc, Value *LHS, Value *RHS, const Twine &Name="", MDNode *FPMathTag=nullptr)
LLVM_ABI CallInst * CreateFMulReduce(Value *Acc, Value *Src)
Create a sequential vector fmul reduction intrinsic of the source vector.
Value * CreateICmp(CmpInst::Predicate P, Value *LHS, Value *RHS, const Twine &Name="")
Value * CreateFNeg(Value *V, const Twine &Name="", MDNode *FPMathTag=nullptr)
LLVM_ABI Value * CreateStepVector(Type *DstType, const Twine &Name="")
Creates a vector of type DstType with the linear sequence <0, 1, ...>
LLVM_ABI CallInst * CreateMaskedScatter(Value *Val, Value *Ptrs, Align Alignment, Value *Mask=nullptr)
Create a call to Masked Scatter intrinsic.
LLVM_ABI CallInst * CreateMaskedGather(Type *Ty, Value *Ptrs, Align Alignment, Value *Mask=nullptr, Value *PassThru=nullptr, const Twine &Name="")
Create a call to Masked Gather intrinsic.
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
LLVM_ABI const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
LLVM_ABI InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
LLVM_ABI FastMathFlags getFastMathFlags() const LLVM_READONLY
Convenience function for getting all the fast-math flags, which must be an operator which supports th...
LLVM_ABI const DataLayout & getDataLayout() const
Get the data layout of the module this instruction belongs to.
Intrinsic::ID getIntrinsicID() const
Return the intrinsic ID of this intrinsic.
void setAlignment(Align Align)
void setAlignment(Align Align)
A switch()-like statement whose cases are string literals.
The instances of the Type class are immutable: once they are created, they are never changed.
bool isIntOrIntVectorTy() const
Return true if this is an integer type or a vector of integer types.
Value * getOperand(unsigned i) const
unsigned getNumOperands() const
static LLVM_ABI bool isVPCast(Intrinsic::ID ID)
LLVM_ABI CmpInst::Predicate getPredicate() const
This is the common base class for vector predication intrinsics.
std::optional< unsigned > getFunctionalIntrinsicID() const
LLVM_ABI bool canIgnoreVectorLengthParam() const
LLVM_ABI void setMaskParam(Value *)
static LLVM_ABI std::optional< unsigned > getFunctionalOpcodeForVP(Intrinsic::ID ID)
LLVM_ABI Value * getVectorLengthParam() const
LLVM_ABI void setVectorLengthParam(Value *)
LLVM_ABI Value * getMemoryDataParam() const
LLVM_ABI Value * getMemoryPointerParam() const
std::optional< unsigned > getConstrainedIntrinsicID() const
LLVM_ABI MaybeAlign getPointerAlignment() const
LLVM_ABI Value * getMaskParam() const
LLVM_ABI ElementCount getStaticVectorLength() const
std::optional< unsigned > getFunctionalOpcode() const
LLVM_ABI unsigned getStartParamPos() const
LLVM_ABI unsigned getVectorParamPos() const
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
LLVM_ABI void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
LLVM_ABI LLVMContext & getContext() const
All values hold a context through their type.
LLVM_ABI StringRef getName() const
Return a constant reference to the value's name.
constexpr ScalarTy getFixedValue() const
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
const ParentTy * getParent() const
self_iterator getIterator()
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
LLVM_ABI Function * getOrInsertDeclaration(Module *M, ID id, ArrayRef< Type * > Tys={})
Look up the Function declaration of the intrinsic id in the Module M.
LLVM_ABI AttributeSet getFnAttributes(LLVMContext &C, ID id)
Return the function attributes for an intrinsic.
LLVM_ABI bool isConstrainedFPIntrinsic(ID QID)
Returns true if the intrinsic ID is for one of the "ConstrainedFloating-Point Intrinsics".
initializer< Ty > init(const Ty &Val)
ElementType
The element type of an SRV or UAV resource.
This is an optimization pass for GlobalISel generic memory operations.
FunctionAddr VTableAddr Value
LLVM_ABI Intrinsic::ID getMinMaxReductionIntrinsicOp(Intrinsic::ID RdxID)
Returns the min/max intrinsic used when expanding a min/max reduction.
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
FunctionAddr VTableAddr uintptr_t uintptr_t Int32Ty
LLVM_ABI Value * getReductionIdentity(Intrinsic::ID RdxID, Type *Ty, FastMathFlags FMF)
Given information about an @llvm.vector.reduce.
LLVM_ABI unsigned getArithmeticReductionInstruction(Intrinsic::ID RdxID)
Returns the arithmetic instruction opcode used when expanding a reduction.
LLVM_ABI Value * getSplatValue(const Value *V)
Get splat value if the input is a splat vector or return nullptr.
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
VPExpansionDetails expandVectorPredicationIntrinsic(VPIntrinsic &VPI, const TargetTransformInfo &TTI)
Expand a vector predication intrinsic.
LLVM_ABI bool isSafeToSpeculativelyExecuteWithOpcode(unsigned Opcode, const Instruction *Inst, const Instruction *CtxI=nullptr, AssumptionCache *AC=nullptr, const DominatorTree *DT=nullptr, const TargetLibraryInfo *TLI=nullptr, bool UseVariableInfo=true, bool IgnoreUBImplyingAttrs=true)
This returns the same result as isSafeToSpeculativelyExecute if Opcode is the actual opcode of Inst.
IRBuilder(LLVMContext &, FolderTy, InserterTy, MDNode *, ArrayRef< OperandBundleDef >) -> IRBuilder< FolderTy, InserterTy >
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
VPExpansionDetails
Represents the details the expansion of a VP intrinsic.
Align valueOrOne() const
For convenience, returns a valid alignment or 1 if undefined.