16 #ifndef LLVM_CODEGEN_BASICTTIIMPL_H
17 #define LLVM_CODEGEN_BASICTTIIMPL_H
47 unsigned getScalarizationOverhead(
Type *Ty,
bool Insert,
bool Extract) {
53 Cost +=
static_cast<T *
>(
this)
56 Cost +=
static_cast<T *
>(
this)
65 unsigned getPermuteShuffleOverhead(
Type *Ty) {
76 Cost +=
static_cast<T *
>(
this)
78 Cost +=
static_cast<T *
>(
this)
86 return static_cast<const T *
>(
this)->getST();
91 return static_cast<const T *
>(
this)->getTLI();
105 unsigned Alignment,
bool *
Fast)
const {
123 bool HasBaseReg, int64_t Scale,
124 unsigned AddrSpace) {
134 bool HasBaseReg, int64_t Scale,
unsigned AddrSpace) {
172 if (IID == Intrinsic::cttz) {
173 if (getTLI()->isCheapToSpeculateCttz())
178 if (IID == Intrinsic::ctlz) {
179 if (getTLI()->isCheapToSpeculateCtlz())
214 case Instruction::Trunc: {
219 case Instruction::ZExt: {
270 if (isa<CallInst>(J) || isa<InvokeInst>(J)) {
273 if (!static_cast<T *>(
this)->isLoweredToCall(
F))
307 unsigned Opcode,
Type *Ty,
316 assert(ISD &&
"Invalid opcode");
323 unsigned OpCost = (IsFloat ? 2 : 1);
328 return LT.first * OpCost;
334 return LT.first * 2 * OpCost;
342 unsigned Cost =
static_cast<T *
>(
this)
347 return getScalarizationOverhead(Ty,
true,
true) + Num * Cost;
358 return getPermuteShuffleOverhead(Tp);
366 assert(ISD &&
"Invalid opcode");
371 if (SrcLT.first == DstLT.first &&
372 SrcLT.second.getSizeInBits() == DstLT.second.getSizeInBits()) {
375 if (Opcode == Instruction::BitCast || Opcode == Instruction::Trunc)
379 if (Opcode == Instruction::Trunc &&
383 if (Opcode == Instruction::ZExt &&
387 if (Opcode == Instruction::AddrSpaceCast &&
393 if (SrcLT.first == DstLT.first &&
401 if (Opcode == Instruction::BitCast)
417 if (SrcLT.first == DstLT.first &&
418 SrcLT.second.getSizeInBits() == DstLT.second.getSizeInBits()) {
421 if (Opcode == Instruction::ZExt)
425 if (Opcode == Instruction::SExt)
432 return SrcLT.first * 1;
447 T *
TTI =
static_cast<T *
>(
this);
448 return TTI->getVectorSplitCost() +
449 (2 * TTI->getCastInstrCost(Opcode, SplitDst, SplitSrc));
460 return getScalarizationOverhead(Dst,
true,
true) + Num * Cost;
467 if (Opcode == Instruction::BitCast)
469 return (Src->
isVectorTy() ? getScalarizationOverhead(Src,
false,
true)
471 (Dst->
isVectorTy() ? getScalarizationOverhead(Dst,
true,
false)
480 Instruction::ExtractElement, VecTy, Index) +
493 assert(ISD &&
"Invalid opcode");
497 assert(CondTy &&
"CondTy must exist");
503 if (!(ValTy->
isVectorTy() && !LT.second.isVector()) &&
522 return getScalarizationOverhead(ValTy,
true,
false) + Num * Cost;
530 std::pair<unsigned, MVT>
LT =
542 unsigned Cost = LT.first;
573 assert(VT &&
"Expect a vector type for interleaved memory op");
576 assert(Factor > 1 && NumElts % Factor == 0 &&
"Invalid interleave factor");
578 unsigned NumSubElts = NumElts / Factor;
583 Opcode, VecTy, Alignment, AddressSpace);
593 auto ceil = [](
unsigned A,
unsigned B) {
return (A +
B - 1) /
B; };
613 unsigned NumLegalInsts = ceil(VecTySize, VecTyLTSize);
617 unsigned NumEltsPerLegalInst = ceil(NumElts, NumLegalInsts);
620 BitVector UsedInsts(NumLegalInsts,
false);
621 for (
unsigned Index : Indices)
622 for (
unsigned Elt = 0; Elt < NumSubElts; ++Elt)
623 UsedInsts.
set((Index + Elt * Factor) / NumEltsPerLegalInst);
627 Cost *= UsedInsts.
count() / NumLegalInsts;
642 "Interleaved memory op has too many members");
644 for (
unsigned Index : Indices) {
645 assert(Index < Factor &&
"Invalid index for interleaved memory op");
648 for (
unsigned i = 0;
i < NumSubElts;
i++)
649 Cost += static_cast<T *>(
this)->getVectorInstrCost(
650 Instruction::ExtractElement, VT, Index +
i * Factor);
653 unsigned InsSubCost = 0;
654 for (
unsigned i = 0;
i < NumSubElts;
i++)
655 InsSubCost += static_cast<T *>(
this)->getVectorInstrCost(
656 Instruction::InsertElement, SubVT,
i);
658 Cost += Indices.size() * InsSubCost;
669 unsigned ExtSubCost = 0;
670 for (
unsigned i = 0;
i < NumSubElts;
i++)
671 ExtSubCost += static_cast<T *>(
this)->getVectorInstrCost(
672 Instruction::ExtractElement, SubVT,
i);
673 Cost += ExtSubCost * Factor;
675 for (
unsigned i = 0;
i < NumElts;
i++)
676 Cost += static_cast<T *>(
this)
677 ->getVectorInstrCost(Instruction::InsertElement, VT,
i);
694 case Intrinsic::masked_scatter: {
696 bool VarMask = !isa<Constant>(
Mask);
697 unsigned Alignment = cast<ConstantInt>(Args[2])->getZExtValue();
704 case Intrinsic::masked_gather: {
706 bool VarMask = !isa<Constant>(
Mask);
707 unsigned Alignment = cast<ConstantInt>(Args[1])->getZExtValue();
710 RetTy, Args[0], VarMask,
720 unsigned SingleCallCost = 10;
724 unsigned ScalarizationCost = 0;
725 unsigned ScalarCalls = 1;
726 Type *ScalarRetTy = RetTy;
728 ScalarizationCost = getScalarizationOverhead(RetTy,
true,
false);
733 for (
unsigned i = 0, ie = Tys.
size();
i != ie; ++
i) {
736 ScalarizationCost += getScalarizationOverhead(Ty,
false,
true);
742 if (ScalarCalls == 1)
746 IID, ScalarRetTy, ScalarTys, FMF);
748 return ScalarCalls * ScalarCost + ScalarizationCost;
752 case Intrinsic::sqrt:
764 case Intrinsic::exp2:
770 case Intrinsic::log10:
773 case Intrinsic::log2:
776 case Intrinsic::fabs:
789 case Intrinsic::copysign:
792 case Intrinsic::floor:
795 case Intrinsic::ceil:
798 case Intrinsic::trunc:
801 case Intrinsic::nearbyint:
804 case Intrinsic::rint:
816 case Intrinsic::fmuladd:
820 case Intrinsic::lifetime_start:
821 case Intrinsic::lifetime_end:
823 case Intrinsic::masked_store:
824 return static_cast<T *
>(
this)
826 case Intrinsic::masked_load:
827 return static_cast<T *
>(
this)
829 case Intrinsic::ctpop:
843 for (
unsigned ISD : ISDs) {
845 if (IID == Intrinsic::fabs && TLI->
isFAbsFree(LT.second)) {
864 auto MinLegalCostI = std::min_element(LegalCost.
begin(), LegalCost.
end());
865 if (MinLegalCostI != LegalCost.
end())
866 return *MinLegalCostI;
868 auto MinCustomCostI = std::min_element(CustomCost.
begin(), CustomCost.
end());
869 if (MinCustomCostI != CustomCost.
end())
870 return *MinCustomCostI;
874 if (IID == Intrinsic::fmuladd)
875 return static_cast<T *
>(
this)
877 static_cast<T *
>(
this)
884 unsigned ScalarizationCost = getScalarizationOverhead(RetTy,
true,
false);
887 for (
unsigned i = 0, ie = Tys.
size();
i != ie; ++
i) {
895 for (
unsigned i = 0, ie = Tys.
size();
i != ie; ++
i) {
896 if (Tys[
i]->isVectorTy()) {
897 ScalarizationCost += getScalarizationOverhead(Tys[
i],
false,
true);
898 ScalarCalls = std::max(ScalarCalls, Tys[i]->getVectorNumElements());
902 return ScalarCalls * ScalarCost + ScalarizationCost;
906 return SingleCallCost;
938 unsigned NumReduxLevels =
Log2_32(NumVecElts);
974 unsigned ArithCost = 0;
975 unsigned ShuffleCost = 0;
976 auto *ConcreteTTI =
static_cast<T *
>(
this);
977 std::pair<unsigned, MVT>
LT =
978 ConcreteTTI->getTLI()->getTypeLegalizationCost(
DL, Ty);
979 unsigned LongVectorCount = 0;
981 LT.second.isVector() ? LT.second.getVectorNumElements() : 1;
982 while (NumVecElts > MVTLen) {
985 ShuffleCost += (IsPairwise + 1) *
988 ArithCost += ConcreteTTI->getArithmeticInstrCost(Opcode, Ty);
996 ShuffleCost += (NumReduxLevels - LongVectorCount) * (IsPairwise + 1) *
999 ArithCost += (NumReduxLevels - LongVectorCount) *
1000 ConcreteTTI->getArithmeticInstrCost(Opcode, Ty);
1001 return ShuffleCost + ArithCost + getScalarizationOverhead(Ty,
false,
true);
void push_back(const T &Elt)
A parsed version of the target data layout string in and methods for querying it. ...
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
This represents an addressing mode of: BaseGV + BaseOffs + BaseReg + Scale*ScaleReg If BaseGV is null...
virtual bool isZExtFree(Type *FromTy, Type *ToTy) const
Return true if any actual instruction that defines a value of type FromTy implicitly zero-extends the...
unsigned getMemoryOpCost(unsigned Opcode, Type *Src, unsigned Alignment, unsigned AddressSpace)
LegalizeTypeAction getTypeAction(LLVMContext &Context, EVT VT) const
Return how we should legalize values of this type, either it is already legal (return 'Legal') or we ...
The main scalar evolution driver.
unsigned getIntrinsicCost(Intrinsic::ID IID, Type *RetTy, ArrayRef< Type * > ParamTys)
static uint64_t round(uint64_t Acc, uint64_t Input)
virtual bool isFAbsFree(EVT VT) const
Return true if an fabs operation is free to the point where it is never worthwhile to replace it with...
virtual bool isLegalICmpImmediate(int64_t) const
Return true if the specified immediate is legal icmp immediate, that is the target has icmp instructi...
BasicTTIImplBase(const TargetMachine *TM, const DataLayout &DL)
virtual bool allowsMisalignedMemoryAccesses(EVT, unsigned AddrSpace=0, unsigned Align=1, bool *=nullptr) const
Determine if the target supports unaligned memory accesses.
int getGEPCost(Type *PointeeType, const Value *Ptr, ArrayRef< const Value * > Operands)
bool isProfitableToHoist(Instruction *I)
FunctionType * getType(LLVMContext &Context, ID id, ArrayRef< Type * > Tys=None)
Return the function type for an intrinsic.
unsigned getCallInstrCost(Function *F, Type *RetTy, ArrayRef< Type * > Tys)
Compute a cost of the given call instruction.
Base class which can be used to help build a TTI implementation.
unsigned getMaxInterleaveFactor(unsigned VF)
iterator begin()
Instruction iterator methods.
unsigned getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src)
unsigned getAddressComputationCost(Type *Ty, ScalarEvolution *, const SCEV *)
virtual bool isLegalAddImmediate(int64_t) const
Return true if the specified immediate is legal add immediate, that is the target has add instruction...
unsigned getExtractWithExtendCost(unsigned Opcode, Type *Dst, VectorType *VecTy, unsigned Index)
int getScalingFactorCost(Type *Ty, GlobalValue *BaseGV, int64_t BaseOffset, bool HasBaseReg, int64_t Scale, unsigned AddrSpace)
virtual bool isNoopAddrSpaceCast(unsigned SrcAS, unsigned DestAS) const
Returns true if a cast between SrcAS and DestAS is a noop.
unsigned getJumpBufSize() const
Returns the target's jmp_buf size in bytes (if never set, the default is 200)
const MCSchedModel & getSchedModel() const
Get the machine model for this subtarget's CPU.
unsigned getStoreSize() const
getStoreSize - Return the number of bytes overwritten by a store of the specified value type...
Type * getVectorElementType() const
BasicTTIImpl(const TargetMachine *ST, const Function &F)
int InstructionOpcodeToISD(unsigned Opcode) const
Get the ISD node that corresponds to the Instruction class opcode.
unsigned getInterleavedMemoryOpCost(unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef< unsigned > Indices, unsigned Alignment, unsigned AddressSpace)
unsigned getJumpBufSize()
unsigned getReductionCost(unsigned Opcode, Type *Ty, bool IsPairwise)
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
unsigned getIntrinsicInstrCost(Intrinsic::ID IID, Type *RetTy, ArrayRef< Type * > Tys, FastMathFlags FMF)
Get intrinsic cost based on argument types.
LegalizeAction getTruncStoreAction(EVT ValVT, EVT MemVT) const
Return how this store with truncation should be treated: either it is legal, needs to be promoted to ...
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory)...
bool isFloatingPointTy() const
Return true if this is one of the six floating-point types.
static GCRegistry::Add< OcamlGC > B("ocaml","ocaml 3.10-compatible GC")
EVT getValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the EVT corresponding to this LLVM type.
bool isOperationLegalOrPromote(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target or can be made legal using promotion...
Type * getScalarType() const LLVM_READONLY
If this is a vector type, return the element type, otherwise return 'this'.
Type * getElementType() const
size_t size() const
size - Get the array size.
LegalizeAction getLoadExtAction(unsigned ExtType, EVT ValVT, EVT MemVT) const
Return how this load with extension should be treated: either it is legal, needs to be promoted to a ...
static GCRegistry::Add< CoreCLRGC > E("coreclr","CoreCLR-compatible GC")
unsigned getFPOpCost(Type *Ty)
bool isTruncateFree(Type *Ty1, Type *Ty2)
unsigned getCmpSelInstrCost(unsigned Opcode, Type *ValTy, Type *CondTy)
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
size_type count() const
count - Returns the number of bits which are set.
MVT - Machine Value Type.
Concrete BasicTTIImpl that can be used if no further customization is needed.
LLVM Basic Block Representation.
The instances of the Type class are immutable: once they are created, they are never changed...
This is an important class for using LLVM in a threaded context.
unsigned getRegisterBitWidth(bool Vector)
bool isVectorTy() const
True if this is an instance of VectorType.
bool isOperationLegalOrCustom(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
LLVM_ATTRIBUTE_ALWAYS_INLINE iterator begin()
uint64_t getNumElements() const
bool isLegalAddressingMode(Type *Ty, GlobalValue *BaseGV, int64_t BaseOffset, bool HasBaseReg, int64_t Scale, unsigned AddrSpace)
unsigned getNumberOfParts(Type *Tp)
virtual int getScalingFactorCost(const DataLayout &DL, const AddrMode &AM, Type *Ty, unsigned AS=0) const
Return the cost of the scaling factor used in the addressing mode represented by AM for this target...
EVT - Extended Value Type.
bool isFoldableMemAccessOffset(Instruction *I, int64_t Offset)
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
LLVM_READONLY APFloat maxnum(const APFloat &A, const APFloat &B)
Implements IEEE maxNum semantics.
bool isLegalAddImmediate(int64_t imm)
This base class for TargetLowering contains the SelectionDAG-independent parts that can be used from ...
LegalizeAction
This enum indicates whether operations are valid for a target, and if not, what action should be used...
unsigned getCFInstrCost(unsigned Opcode)
unsigned getShuffleCost(TTI::ShuffleKind Kind, Type *Tp, int Index, Type *SubTp)
unsigned getNumberOfRegisters(bool Vector)
Iterator for intrusive lists based on ilist_node.
unsigned getArithmeticInstrCost(unsigned Opcode, Type *Ty, TTI::OperandValueKind Opd1Info=TTI::OK_AnyValue, TTI::OperandValueKind Opd2Info=TTI::OK_AnyValue, TTI::OperandValueProperties Opd1PropInfo=TTI::OP_None, TTI::OperandValueProperties Opd2PropInfo=TTI::OP_None, ArrayRef< const Value * > Args=ArrayRef< const Value * >())
unsigned getVectorSplitCost()
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
cl::opt< unsigned > PartialUnrollingThreshold
virtual bool isProfitableToHoist(Instruction *I) const
unsigned Log2_32(uint32_t Value)
Log2_32 - This function returns the floor log base 2 of the specified value, -1 if the value is zero...
Class to represent vector types.
Select(COND, TRUEVAL, FALSEVAL).
std::pair< int, MVT > getTypeLegalizationCost(const DataLayout &DL, Type *Ty) const
Estimate the cost of type-legalization and the legalized type.
unsigned getVectorNumElements() const
std::vector< BlockT * >::const_iterator block_iterator
unsigned LoopMicroOpBufferSize
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
FMINNAN/FMAXNAN - Behave identically to FMINNUM/FMAXNUM, except that when a single input is NaN...
TargetSubtargetInfo - Generic base class for all target subtargets.
block_iterator block_end() const
BR_JT - Jumptable branch.
LLVM_ATTRIBUTE_ALWAYS_INLINE iterator end()
unsigned getIntrinsicCost(Intrinsic::ID IID, Type *RetTy, ArrayRef< const Value * > Arguments)
unsigned getOperationCost(unsigned Opcode, Type *Ty, Type *OpTy)
This class represents an analyzed expression in the program.
bool allowsMisalignedMemoryAccesses(LLVMContext &Context, unsigned BitWidth, unsigned AddressSpace, unsigned Alignment, bool *Fast) const
bool hasBranchDivergence()
unsigned getJumpBufAlignment() const
Returns the target's jmp_buf alignment in bytes (if never set, the default is 0)
Represents a single loop in the control flow graph.
virtual bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty, unsigned AddrSpace) const
Return true if the addressing mode represented by AM is legal for this target, for a load/store of th...
unsigned getJumpBufAlignment()
bool shouldBuildLookupTables()
ImmutableCallSite - establish a view to a call site for examination.
unsigned getInliningThresholdMultiplier()
LLVM_NODISCARD std::enable_if<!is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
unsigned getVectorInstrCost(unsigned Opcode, Type *Val, unsigned Index)
uint64_t getTypeStoreSize(Type *Ty) const
Returns the maximum number of bytes that may be overwritten by storing the specified type...
virtual bool isFoldableMemAccessOffset(Instruction *I, int64_t Offset) const
virtual bool isTruncateFree(Type *FromTy, Type *ToTy) const
Return true if it's free to truncate a value of type FromTy to type ToTy.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
FunTy * getCalledFunction() const
getCalledFunction - Return the function being called if this is a direct call, otherwise return null ...
bool isSourceOfDivergence(const Value *V)
unsigned getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
LLVM Value Representation.
FMA - Perform a * b + c with no intermediate rounding step.
static VectorType * get(Type *ElementType, unsigned NumElements)
This static method is the primary way to construct an VectorType.
std::underlying_type< E >::type Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
bool isOperationExpand(unsigned Op, EVT VT) const
Return true if the specified operation is illegal on this target or unlikely to be made legal with cu...
Primary interface to the complete machine description for the target machine.
bool isTypeLegal(Type *Ty)
block_iterator block_begin() const
Convenience struct for specifying and reasoning about fast-math flags.
unsigned getIntrinsicInstrCost(Intrinsic::ID IID, Type *RetTy, ArrayRef< Value * > Args, FastMathFlags FMF)
Get intrinsic cost based on arguments.
bool haveFastSqrt(Type *Ty)
void getUnrollingPreferences(Loop *L, TTI::UnrollingPreferences &UP)
bool isLegalICmpImmediate(int64_t imm)
unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
static GCRegistry::Add< ErlangGC > A("erlang","erlang-compatible garbage collector")
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
getIntegerVT - Returns the EVT that represents an integer with the given number of bits...
Fast - This calling convention attempts to make calls as fast as possible (e.g.
LLVM_READONLY APFloat minnum(const APFloat &A, const APFloat &B)
Implements IEEE minNum semantics.
bool noNaNs() const
Flag queries.
This file describes how to lower LLVM code to machine code.
bool isVoidTy() const
Return true if this is 'void'.