AArch64TargetTransformInfo.cpp File Reference

#include "AArch64TargetTransformInfo.h"
#include "AArch64ExpandImm.h"
#include "AArch64PerfectShuffle.h"
#include "MCTargetDesc/AArch64AddressingModes.h"
#include "llvm/Analysis/IVDescriptors.h"
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/CodeGen/BasicTTIImpl.h"
#include "llvm/CodeGen/CostTable.h"
#include "llvm/CodeGen/TargetLowering.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/IntrinsicsAArch64.h"
#include "llvm/IR/PatternMatch.h"
#include "llvm/Support/Debug.h"
#include "llvm/Transforms/InstCombine/InstCombiner.h"
#include "llvm/Transforms/Vectorize/LoopVectorizationLegality.h"
#include <algorithm>
#include <optional>

Go to the source code of this file.

Macros
#define	DEBUG_TYPE   "aarch64tti"

Functions
static bool	isSMEABIRoutineCall (const CallInst &CI)
static bool	hasPossibleIncompatibleOps (const Function *F)
	Returns true if the function has explicit operations that can only be lowered using incompatible instructions for the selected mode.
static bool	isUnpackedVectorVT (EVT VecVT)
static InstructionCost	getHistogramCost (const IntrinsicCostAttributes &ICA)
static std::optional< Instruction * >	processPhiNode (InstCombiner &IC, IntrinsicInst &II)
	The function will remove redundant reinterprets casting in the presence of the control flow.
static std::optional< Instruction * >	tryCombineFromSVBoolBinOp (InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * >	instCombineConvertFromSVBool (InstCombiner &IC, IntrinsicInst &II)
static bool	isAllActivePredicate (Value *Pred)
static std::optional< Instruction * >	instCombineSVENoActiveUnaryErase (InstCombiner &IC, IntrinsicInst &II, int PredPos)
static std::optional< Instruction * >	instCombineSVENoActiveUnaryZero (InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * >	instCombineSVESel (InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * >	instCombineSVEDup (InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * >	instCombineSVEDupX (InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * >	instCombineSVECmpNE (InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * >	instCombineSVELast (InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * >	instCombineSVECondLast (InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * >	instCombineRDFFR (InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * >	instCombineSVECntElts (InstCombiner &IC, IntrinsicInst &II, unsigned NumElts)
static std::optional< Instruction * >	instCombineSVEPTest (InstCombiner &IC, IntrinsicInst &II)
template<Intrinsic::ID MulOpc, typename Intrinsic::ID FuseOpc>
static std::optional< Instruction * >	instCombineSVEVectorFuseMulAddSub (InstCombiner &IC, IntrinsicInst &II, bool MergeIntoAddendOp)
static std::optional< Instruction * >	instCombineSVELD1 (InstCombiner &IC, IntrinsicInst &II, const DataLayout &DL)
static std::optional< Instruction * >	instCombineSVEST1 (InstCombiner &IC, IntrinsicInst &II, const DataLayout &DL)
static Instruction::BinaryOps	intrinsicIDToBinOpCode (unsigned Intrinsic)
static std::optional< Instruction * >	instCombineSVEVectorBinOp (InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * >	instCombineSVEAllActive (IntrinsicInst &II, Intrinsic::ID IID)
static std::optional< Instruction * >	instCombineSVEAllOrNoActive (InstCombiner &IC, IntrinsicInst &II, Intrinsic::ID IID)
static std::optional< Instruction * >	instCombineSVEVectorAdd (InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * >	instCombineSVEVectorFAdd (InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * >	instCombineSVEVectorFAddU (InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * >	instCombineSVEVectorFSub (InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * >	instCombineSVEVectorFSubU (InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * >	instCombineSVEVectorSub (InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * >	instCombineSVEVectorMul (InstCombiner &IC, IntrinsicInst &II, Intrinsic::ID IID)
static std::optional< Instruction * >	instCombineSVEUnpack (InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * >	instCombineSVETBL (InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * >	instCombineSVEUzp1 (InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * >	instCombineSVEZip (InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * >	instCombineLD1GatherIndex (InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * >	instCombineST1ScatterIndex (InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * >	instCombineSVESDIV (InstCombiner &IC, IntrinsicInst &II)
bool	SimplifyValuePattern (SmallVector< Value * > &Vec, bool AllowPoison)
static std::optional< Instruction * >	instCombineSVEDupqLane (InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * >	instCombineMaxMinNM (InstCombiner &IC, IntrinsicInst &II)
static std::optional< Instruction * >	instCombineSVESrshl (InstCombiner &IC, IntrinsicInst &II)
static unsigned	getSVEGatherScatterOverhead (unsigned Opcode, const AArch64Subtarget *ST)
static void	getFalkorUnrollingPreferences (Loop *L, ScalarEvolution &SE, TargetTransformInfo::UnrollingPreferences &UP)
static bool	containsDecreasingPointers (Loop *TheLoop, PredicatedScalarEvolution *PSE)

Variables
static cl::opt< bool >	EnableFalkorHWPFUnrollFix ("enable-falkor-hwpf-unroll-fix", cl::init(true), cl::Hidden)
static cl::opt< unsigned >	SVEGatherOverhead ("sve-gather-overhead", cl::init(10), cl::Hidden)
static cl::opt< unsigned >	SVEScatterOverhead ("sve-scatter-overhead", cl::init(10), cl::Hidden)
static cl::opt< unsigned >	SVETailFoldInsnThreshold ("sve-tail-folding-insn-threshold", cl::init(15), cl::Hidden)
static cl::opt< unsigned >	NeonNonConstStrideOverhead ("neon-nonconst-stride-overhead", cl::init(10), cl::Hidden)
static cl::opt< unsigned >	CallPenaltyChangeSM ("call-penalty-sm-change", cl::init(5), cl::Hidden, cl::desc("Penalty of calling a function that requires a change to PSTATE.SM"))
static cl::opt< unsigned >	InlineCallPenaltyChangeSM ("inline-call-penalty-sm-change", cl::init(10), cl::Hidden, cl::desc("Penalty of inlining a call that requires a change to PSTATE.SM"))
static cl::opt< bool >	EnableOrLikeSelectOpt ("enable-aarch64-or-like-select", cl::init(true), cl::Hidden)
static cl::opt< bool >	EnableLSRCostOpt ("enable-aarch64-lsr-cost-opt", cl::init(true), cl::Hidden)
static cl::opt< unsigned >	BaseHistCntCost ("aarch64-base-histcnt-cost", cl::init(8), cl::Hidden, cl::desc("The cost of a histcnt instruction"))
TailFoldingOption	TailFoldingOptionLoc
cl::opt< TailFoldingOption, true, cl::parser< std::string > >	SVETailFolding ("sve-tail-folding", cl::desc("Control the use of vectorisation using tail-folding for SVE where the" " option is specified in the form (Initial)[+(Flag1|Flag2|...)]:" "\ndisabled (Initial) No loop types will vectorize using " "tail-folding" "\ndefault (Initial) Uses the default tail-folding settings for " "the target CPU" "\nall (Initial) All legal loop types will vectorize using " "tail-folding" "\nsimple (Initial) Use tail-folding for simple loops (not " "reductions or recurrences)" "\nreductions Use tail-folding for loops containing reductions" "\nnoreductions Inverse of above" "\nrecurrences Use tail-folding for loops containing fixed order " "recurrences" "\nnorecurrences Inverse of above" "\nreverse Use tail-folding for loops requiring reversed " "predicates" "\nnoreverse Inverse of above"), cl::location(TailFoldingOptionLoc))
static cl::opt< bool >	EnableFixedwidthAutovecInStreamingMode ("enable-fixedwidth-autovec-in-streaming-mode", cl::init(false), cl::Hidden)
static cl::opt< bool >	EnableScalableAutovecInStreamingMode ("enable-scalable-autovec-in-streaming-mode", cl::init(false), cl::Hidden)

Macro Definition Documentation

DEBUG_TYPE

#define DEBUG_TYPE   "aarch64tti"

Definition at line 31 of file AArch64TargetTransformInfo.cpp.

Function Documentation

containsDecreasingPointers()

static bool containsDecreasingPointers ( Loop *  TheLoop, PredicatedScalarEvolution *  PSE  )

static

Definition at line 4393 of file AArch64TargetTransformInfo.cpp.

References llvm::LoopBase< BlockT, LoopT >::blocks(), llvm::getLoadStorePointerOperand(), llvm::getLoadStoreType(), llvm::getPtrStride(), I, and Ptr.

Referenced by llvm::AArch64TTIImpl::preferPredicateOverEpilogue().

getFalkorUnrollingPreferences()

static void getFalkorUnrollingPreferences ( Loop *  L, ScalarEvolution &  SE, TargetTransformInfo::UnrollingPreferences &  UP  )

static

Definition at line 3647 of file AArch64TargetTransformInfo.cpp.

References llvm::dbgs(), llvm::LoadInst::getPointerOperand(), llvm::ScalarEvolution::getSCEV(), I, llvm::SCEVAddRecExpr::isAffine(), LLVM_DEBUG, llvm::Log2_32(), and llvm::TargetTransformInfo::UnrollingPreferences::MaxCount.

Referenced by llvm::AArch64TTIImpl::getUnrollingPreferences().

getHistogramCost()

static InstructionCost getHistogramCost ( const IntrinsicCostAttributes &  ICA )

static

Definition at line 517 of file AArch64TargetTransformInfo.cpp.

References BaseHistCntCost, llvm::IntrinsicCostAttributes::getArgTypes(), llvm::InstructionCost::getInvalid(), llvm::Type::getScalarSizeInBits(), llvm::Type::isIntegerTy(), and llvm::Type::isPointerTy().

Referenced by llvm::AArch64TTIImpl::getIntrinsicInstrCost().

getSVEGatherScatterOverhead()

static unsigned getSVEGatherScatterOverhead ( unsigned  Opcode, const AArch64Subtarget *  ST  )

static

Definition at line 3434 of file AArch64TargetTransformInfo.cpp.

References assert(), llvm_unreachable, SVEGatherOverhead, and SVEScatterOverhead.

Referenced by llvm::AArch64TTIImpl::getGatherScatterOpCost().

hasPossibleIncompatibleOps()

static bool hasPossibleIncompatibleOps ( const Function *  F )

static

Returns true if the function has explicit operations that can only be lowered using incompatible instructions for the selected mode.

This also returns true if the function F may use or modify ZA state.

Definition at line 226 of file AArch64TargetTransformInfo.cpp.

References F, I, and isSMEABIRoutineCall().

Referenced by llvm::AArch64TTIImpl::areInlineCompatible().

instCombineConvertFromSVBool()

static std::optional< Instruction * > instCombineConvertFromSVBool ( InstCombiner &  IC, IntrinsicInst &  II  )

static

Definition at line 1008 of file AArch64TargetTransformInfo.cpp.

References llvm::SmallVectorTemplateCommon< T, typename >::begin(), llvm::Value::getType(), II, llvm::SmallVectorImpl< T >::insert(), processPhiNode(), llvm::InstCombiner::replaceInstUsesWith(), and tryCombineFromSVBoolBinOp().

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

instCombineLD1GatherIndex()

static std::optional< Instruction * > instCombineLD1GatherIndex ( InstCombiner &  IC, IntrinsicInst &  II  )

static

Definition at line 1870 of file AArch64TargetTransformInfo.cpp.

References llvm::InstCombiner::Builder, llvm::IRBuilderBase::CreateBitCast(), llvm::IRBuilderBase::CreateGEP(), llvm::IRBuilderBase::CreateMaskedLoad(), llvm::ConstantAggregateZero::get(), llvm::PointerType::getUnqual(), II, instCombineSVENoActiveUnaryZero(), llvm::PatternMatch::m_SpecificInt(), llvm::PatternMatch::m_Value(), llvm::PatternMatch::match(), Ptr, llvm::InstCombiner::replaceInstUsesWith(), and llvm::Value::takeName().

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

instCombineMaxMinNM()

static std::optional< Instruction * > instCombineMaxMinNM ( InstCombiner &  IC, IntrinsicInst &  II  )

static

Definition at line 2056 of file AArch64TargetTransformInfo.cpp.

References A, B, II, and llvm::InstCombiner::replaceInstUsesWith().

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

instCombineRDFFR()

static std::optional< Instruction * > instCombineRDFFR ( InstCombiner &  IC, IntrinsicInst &  II  )

static

Definition at line 1374 of file AArch64TargetTransformInfo.cpp.

References llvm::InstCombiner::Builder, llvm::IRBuilderBase::CreateIntrinsic(), llvm::Type::getInt32Ty(), II, llvm::InstCombiner::replaceInstUsesWith(), and llvm::Value::takeName().

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

instCombineST1ScatterIndex()

static std::optional< Instruction * > instCombineST1ScatterIndex ( InstCombiner &  IC, IntrinsicInst &  II  )

static

Definition at line 1904 of file AArch64TargetTransformInfo.cpp.

References llvm::InstCombiner::Builder, llvm::IRBuilderBase::CreateBitCast(), llvm::IRBuilderBase::CreateGEP(), llvm::IRBuilderBase::CreateMaskedStore(), llvm::InstCombiner::eraseInstFromFunction(), llvm::Value::getType(), llvm::PointerType::getUnqual(), II, llvm::PatternMatch::m_SpecificInt(), llvm::PatternMatch::m_Value(), llvm::PatternMatch::match(), and Ptr.

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

instCombineSVEAllActive()

static std::optional< Instruction * > instCombineSVEAllActive ( IntrinsicInst &  II, Intrinsic::ID  IID  )

static

Definition at line 1595 of file AArch64TargetTransformInfo.cpp.

References llvm::Intrinsic::getDeclaration(), II, llvm::PatternMatch::match(), and llvm::Mod.

Referenced by instCombineSVEAllOrNoActive().

instCombineSVEAllOrNoActive()

static std::optional< Instruction * > instCombineSVEAllOrNoActive ( InstCombiner &  IC, IntrinsicInst &  II, Intrinsic::ID  IID  )

static

Definition at line 1612 of file AArch64TargetTransformInfo.cpp.

References II, instCombineSVEAllActive(), llvm::PatternMatch::m_ZeroInt(), llvm::PatternMatch::match(), and llvm::InstCombiner::replaceInstUsesWith().

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic(), instCombineSVEVectorAdd(), instCombineSVEVectorFAdd(), instCombineSVEVectorFSub(), and instCombineSVEVectorSub().

instCombineSVECmpNE()

static std::optional< Instruction * > instCombineSVECmpNE ( InstCombiner &  IC, IntrinsicInst &  II  )

static

Definition at line 1159 of file AArch64TargetTransformInfo.cpp.

References llvm::InstCombiner::Builder, llvm::IRBuilderBase::CreateIntrinsic(), llvm::ScalableVectorType::get(), llvm::Type::getInt1Ty(), llvm::Type::getInt32Ty(), llvm::Constant::getNullValue(), llvm::getSplatValue(), I, II, llvm::InstCombiner::replaceInstUsesWith(), llvm::AArch64::SVEBitsPerBlock, and llvm::Value::takeName().

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

instCombineSVECntElts()

static std::optional< Instruction * > instCombineSVECntElts ( InstCombiner &  IC, IntrinsicInst &  II, unsigned  NumElts  )

static

Definition at line 1390 of file AArch64TargetTransformInfo.cpp.

References llvm::InstCombiner::Builder, llvm::IRBuilderBase::CreateVScale(), llvm::getNumElementsFromSVEPredPattern(), II, llvm::InstCombiner::replaceInstUsesWith(), and llvm::Value::takeName().

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

instCombineSVECondLast()

static std::optional< Instruction * > instCombineSVECondLast ( InstCombiner &  IC, IntrinsicInst &  II  )

static

Definition at line 1332 of file AArch64TargetTransformInfo.cpp.

References llvm::InstCombiner::Builder, llvm::IRBuilderBase::CreateBitCast(), llvm::IRBuilderBase::CreateIntrinsic(), Fallback, llvm::VectorType::get(), getBitWidth(), llvm::IRBuilderBase::getDoubleTy(), llvm::IRBuilderBase::getFloatTy(), llvm::IRBuilderBase::getHalfTy(), llvm::Value::getType(), II, llvm::Type::isIntegerTy(), and llvm::InstCombiner::replaceInstUsesWith().

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

instCombineSVEDup()

static std::optional< Instruction * > instCombineSVEDup ( InstCombiner &  IC, IntrinsicInst &  II  )

static

Definition at line 1125 of file AArch64TargetTransformInfo.cpp.

References llvm::InsertElementInst::Create(), llvm::Type::getInt64Ty(), llvm::IntrinsicInst::getIntrinsicID(), llvm::User::getOperand(), II, and llvm::InstCombiner::replaceInstUsesWith().

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

instCombineSVEDupqLane()

static std::optional< Instruction * > instCombineSVEDupqLane ( InstCombiner &  IC, IntrinsicInst &  II  )

static

Definition at line 1994 of file AArch64TargetTransformInfo.cpp.

References llvm::InstCombiner::Builder, llvm::IRBuilderBase::CreateBitOrPointerCast(), llvm::IRBuilderBase::CreateInsertElement(), llvm::IRBuilderBase::CreateInsertVector(), llvm::IRBuilderBase::CreateShuffleVector(), llvm::Default, llvm::ScalableVectorType::get(), llvm::PoisonValue::get(), llvm::ConstantAggregateZero::get(), llvm::IRBuilderBase::getInt32Ty(), llvm::IRBuilderBase::getInt64(), llvm::IRBuilderBase::getInt64Ty(), llvm::IRBuilderBase::getIntNTy(), llvm::Value::getType(), I, Idx, II, llvm::PatternMatch::m_Value(), llvm::PatternMatch::match(), llvm::InstCombiner::replaceInstUsesWith(), SimplifyValuePattern(), and llvm::SmallVectorBase< Size_T >::size().

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

instCombineSVEDupX()

static std::optional< Instruction * > instCombineSVEDupX ( InstCombiner &  IC, IntrinsicInst &  II  )

static

Definition at line 1149 of file AArch64TargetTransformInfo.cpp.

References llvm::InstCombiner::Builder, llvm::IRBuilderBase::CreateVectorSplat(), II, llvm::InstCombiner::replaceInstUsesWith(), RetTy, and llvm::Splat.

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

instCombineSVELast()

static std::optional< Instruction * > instCombineSVELast ( InstCombiner &  IC, IntrinsicInst &  II  )

static

Definition at line 1258 of file AArch64TargetTransformInfo.cpp.

References llvm::InstCombiner::Builder, llvm::CallingConv::C, llvm::ExtractElementInst::Create(), llvm::IRBuilderBase::CreateIntrinsic(), llvm::BinaryOperator::CreateWithCopiedFlags(), llvm::Type::getInt64Ty(), llvm::getNumElementsFromSVEPredPattern(), llvm::getSplatValue(), llvm::Value::getType(), Idx, II, llvm::isSplatValue(), LHS, llvm::PatternMatch::m_BinOp(), llvm::PatternMatch::m_OneUse(), llvm::PatternMatch::m_Value(), llvm::PatternMatch::match(), llvm::InstCombiner::replaceInstUsesWith(), and RHS.

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

instCombineSVELD1()

static std::optional< Instruction * > instCombineSVELD1 ( InstCombiner &  IC, IntrinsicInst &  II, const DataLayout &  DL  )

static

Definition at line 1521 of file AArch64TargetTransformInfo.cpp.

References llvm::InstCombiner::Builder, llvm::Instruction::copyMetadata(), llvm::IRBuilderBase::CreateLoad(), llvm::IRBuilderBase::CreateMaskedLoad(), DL, llvm::ConstantAggregateZero::get(), llvm::Value::getPointerAlignment(), II, instCombineSVENoActiveUnaryZero(), isAllActivePredicate(), and llvm::InstCombiner::replaceInstUsesWith().

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

instCombineSVENoActiveUnaryErase()

static std::optional< Instruction * > instCombineSVENoActiveUnaryErase ( InstCombiner &  IC, IntrinsicInst &  II, int  PredPos  )

static

Definition at line 1078 of file AArch64TargetTransformInfo.cpp.

References llvm::InstCombiner::eraseInstFromFunction(), II, llvm::PatternMatch::m_ZeroInt(), and llvm::PatternMatch::match().

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

instCombineSVENoActiveUnaryZero()

static std::optional< Instruction * > instCombineSVENoActiveUnaryZero ( InstCombiner &  IC, IntrinsicInst &  II  )

static

Definition at line 1089 of file AArch64TargetTransformInfo.cpp.

References llvm::InstCombiner::eraseInstFromFunction(), llvm::ConstantStruct::get(), II, llvm::PatternMatch::m_ZeroInt(), llvm::PatternMatch::match(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::InstCombiner::replaceInstUsesWith(), and RetTy.

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic(), instCombineLD1GatherIndex(), and instCombineSVELD1().

instCombineSVEPTest()

static std::optional< Instruction * > instCombineSVEPTest ( InstCombiner &  IC, IntrinsicInst &  II  )

static

Definition at line 1408 of file AArch64TargetTransformInfo.cpp.

References llvm::InstCombiner::Builder, llvm::IRBuilderBase::CreateIntrinsic(), llvm::CallBase::getArgOperand(), llvm::IntrinsicInst::getIntrinsicID(), llvm::Value::getType(), II, llvm::InstCombiner::replaceInstUsesWith(), and llvm::Value::takeName().

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

instCombineSVESDIV()

static std::optional< Instruction * > instCombineSVESDIV ( InstCombiner &  IC, IntrinsicInst &  II  )

static

Definition at line 1933 of file AArch64TargetTransformInfo.cpp.

References llvm::InstCombiner::Builder, llvm::IRBuilderBase::CreateIntrinsic(), llvm::IRBuilderBase::getInt32Ty(), llvm::getSplatValue(), llvm::ConstantInt::getValue(), II, llvm::APInt::isNegatedPowerOf2(), llvm::APInt::isPowerOf2(), llvm::APInt::logBase2(), llvm::APInt::negate(), and llvm::InstCombiner::replaceInstUsesWith().

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

instCombineSVESel()

static std::optional< Instruction * > instCombineSVESel ( InstCombiner &  IC, IntrinsicInst &  II  )

static

Definition at line 1113 of file AArch64TargetTransformInfo.cpp.

References llvm::InstCombiner::Builder, llvm::IRBuilderBase::CreateSelect(), II, isAllActivePredicate(), llvm::InstCombiner::replaceInstUsesWith(), and Select.

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

instCombineSVESrshl()

static std::optional< Instruction * > instCombineSVESrshl ( InstCombiner &  IC, IntrinsicInst &  II  )

static

Definition at line 2066 of file AArch64TargetTransformInfo.cpp.

References llvm::InstCombiner::Builder, llvm::IRBuilderBase::CreateIntrinsic(), II, isAllActivePredicate(), llvm::PatternMatch::m_NonNegative(), llvm::PatternMatch::m_Value(), llvm::PatternMatch::match(), and llvm::InstCombiner::replaceInstUsesWith().

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

instCombineSVEST1()

static std::optional< Instruction * > instCombineSVEST1 ( InstCombiner &  IC, IntrinsicInst &  II, const DataLayout &  DL  )

static

Definition at line 1544 of file AArch64TargetTransformInfo.cpp.

References llvm::InstCombiner::Builder, llvm::Instruction::copyMetadata(), llvm::IRBuilderBase::CreateMaskedStore(), llvm::IRBuilderBase::CreateStore(), DL, llvm::InstCombiner::eraseInstFromFunction(), llvm::Value::getPointerAlignment(), II, and isAllActivePredicate().

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

instCombineSVETBL()

static std::optional< Instruction * > instCombineSVETBL ( InstCombiner &  IC, IntrinsicInst &  II  )

static

Definition at line 1801 of file AArch64TargetTransformInfo.cpp.

References llvm::InstCombiner::Builder, llvm::IRBuilderBase::CreateExtractElement(), llvm::IRBuilderBase::CreateVectorSplat(), llvm::getSplatValue(), II, llvm::InstCombiner::replaceInstUsesWith(), and llvm::Value::takeName().

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

instCombineSVEUnpack()

static std::optional< Instruction * > instCombineSVEUnpack ( InstCombiner &  IC, IntrinsicInst &  II  )

static

Definition at line 1781 of file AArch64TargetTransformInfo.cpp.

References llvm::InstCombiner::Builder, llvm::IRBuilderBase::CreateIntCast(), llvm::IRBuilderBase::CreateVectorSplat(), llvm::getSplatValue(), II, llvm::InstCombiner::replaceInstUsesWith(), RetTy, and llvm::Value::takeName().

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

instCombineSVEUzp1()

static std::optional< Instruction * > instCombineSVEUzp1 ( InstCombiner &  IC, IntrinsicInst &  II  )

static

Definition at line 1824 of file AArch64TargetTransformInfo.cpp.

References A, B, llvm::InstCombiner::Builder, llvm::IRBuilderBase::CreateInsertVector(), llvm::PoisonValue::get(), llvm::ScalableVectorType::getDoubleElementsVectorType(), llvm::IRBuilderBase::getInt64(), II, llvm::PatternMatch::m_Value(), llvm::PatternMatch::match(), llvm::InstCombiner::replaceInstUsesWith(), RetTy, and llvm::Value::takeName().

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

instCombineSVEVectorAdd()

static std::optional< Instruction * > instCombineSVEVectorAdd ( InstCombiner &  IC, IntrinsicInst &  II  )

static

Definition at line 1622 of file AArch64TargetTransformInfo.cpp.

References II, instCombineSVEAllOrNoActive(), and instCombineSVEVectorFuseMulAddSub().

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

instCombineSVEVectorBinOp()

static std::optional< Instruction * > instCombineSVEVectorBinOp ( InstCombiner &  IC, IntrinsicInst &  II  )

static

Definition at line 1575 of file AArch64TargetTransformInfo.cpp.

References llvm::InstCombiner::Builder, llvm::IRBuilderBase::CreateBinOp(), II, intrinsicIDToBinOpCode(), llvm::PatternMatch::match(), llvm::InstCombiner::replaceInstUsesWith(), and llvm::IRBuilderBase::setFastMathFlags().

Referenced by instCombineSVEVectorFAddU(), instCombineSVEVectorFSubU(), and instCombineSVEVectorMul().

instCombineSVEVectorFAdd()

static std::optional< Instruction * > instCombineSVEVectorFAdd ( InstCombiner &  IC, IntrinsicInst &  II  )

static

Definition at line 1639 of file AArch64TargetTransformInfo.cpp.

References II, instCombineSVEAllOrNoActive(), and instCombineSVEVectorFuseMulAddSub().

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

instCombineSVEVectorFAddU()

static std::optional< Instruction * > instCombineSVEVectorFAddU ( InstCombiner &  IC, IntrinsicInst &  II  )

static

Definition at line 1662 of file AArch64TargetTransformInfo.cpp.

References II, instCombineSVEVectorBinOp(), and instCombineSVEVectorFuseMulAddSub().

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

instCombineSVEVectorFSub()

static std::optional< Instruction * > instCombineSVEVectorFSub ( InstCombiner &  IC, IntrinsicInst &  II  )

static

Definition at line 1682 of file AArch64TargetTransformInfo.cpp.

References II, instCombineSVEAllOrNoActive(), and instCombineSVEVectorFuseMulAddSub().

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

instCombineSVEVectorFSubU()

static std::optional< Instruction * > instCombineSVEVectorFSubU ( InstCombiner &  IC, IntrinsicInst &  II  )

static

Definition at line 1705 of file AArch64TargetTransformInfo.cpp.

References II, instCombineSVEVectorBinOp(), and instCombineSVEVectorFuseMulAddSub().

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

instCombineSVEVectorFuseMulAddSub()

template<Intrinsic::ID MulOpc, typename Intrinsic::ID FuseOpc>

static std::optional< Instruction * > instCombineSVEVectorFuseMulAddSub ( InstCombiner &  IC, IntrinsicInst &  II, bool  MergeIntoAddendOp  )

static

Definition at line 1478 of file AArch64TargetTransformInfo.cpp.

References llvm::FastMathFlags::allowContract(), llvm::InstCombiner::Builder, llvm::IRBuilderBase::CreateIntrinsic(), getFastMathFlags(), II, llvm::PatternMatch::m_Specific(), llvm::PatternMatch::m_Value(), llvm::PatternMatch::match(), llvm::Mul, P, and llvm::InstCombiner::replaceInstUsesWith().

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic(), instCombineSVEVectorAdd(), instCombineSVEVectorFAdd(), instCombineSVEVectorFAddU(), instCombineSVEVectorFSub(), instCombineSVEVectorFSubU(), and instCombineSVEVectorSub().

instCombineSVEVectorMul()

static std::optional< Instruction * > instCombineSVEVectorMul ( InstCombiner &  IC, IntrinsicInst &  II, Intrinsic::ID  IID  )

static

Definition at line 1736 of file AArch64TargetTransformInfo.cpp.

References llvm::getSplatValue(), I, II, instCombineSVEVectorBinOp(), llvm::PatternMatch::m_FPOne(), llvm::PatternMatch::m_One(), llvm::PatternMatch::match(), and llvm::InstCombiner::replaceInstUsesWith().

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

instCombineSVEVectorSub()

static std::optional< Instruction * > instCombineSVEVectorSub ( InstCombiner &  IC, IntrinsicInst &  II  )

static

Definition at line 1724 of file AArch64TargetTransformInfo.cpp.

References II, instCombineSVEAllOrNoActive(), and instCombineSVEVectorFuseMulAddSub().

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

instCombineSVEZip()

static std::optional< Instruction * > instCombineSVEZip ( InstCombiner &  IC, IntrinsicInst &  II  )

static

Definition at line 1854 of file AArch64TargetTransformInfo.cpp.

References A, B, II, llvm::PatternMatch::m_Specific(), llvm::PatternMatch::m_Value(), llvm::PatternMatch::match(), and llvm::InstCombiner::replaceInstUsesWith().

Referenced by llvm::AArch64TTIImpl::instCombineIntrinsic().

intrinsicIDToBinOpCode()

static Instruction::BinaryOps intrinsicIDToBinOpCode ( unsigned  Intrinsic )

static

Definition at line 1561 of file AArch64TargetTransformInfo.cpp.

Referenced by instCombineSVEVectorBinOp().

isAllActivePredicate()

static bool isAllActivePredicate ( Value *  Pred )

static

Definition at line 1060 of file AArch64TargetTransformInfo.cpp.

References llvm::Value::getType(), llvm::PatternMatch::m_Value(), and llvm::PatternMatch::match().

isSMEABIRoutineCall()

static bool isSMEABIRoutineCall ( const CallInst &  CI )

static

Definition at line 213 of file AArch64TargetTransformInfo.cpp.

References llvm::StringSwitch< T, R >::Case(), llvm::StringSwitch< T, R >::Default(), F, and llvm::CallBase::getCalledFunction().

Referenced by hasPossibleIncompatibleOps().

isUnpackedVectorVT()

static bool isUnpackedVectorVT ( EVT  VecVT )

static

Definition at line 512 of file AArch64TargetTransformInfo.cpp.

References llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getKnownMinValue(), llvm::EVT::getSizeInBits(), llvm::EVT::isScalableVector(), and llvm::AArch64::SVEBitsPerBlock.

Referenced by llvm::AArch64TTIImpl::getIntrinsicInstrCost().

processPhiNode()

static std::optional< Instruction * > processPhiNode ( InstCombiner &  IC, IntrinsicInst &  II  )

static

The function will remove redundant reinterprets casting in the presence of the control flow.

Definition at line 913 of file AArch64TargetTransformInfo.cpp.

References llvm::PHINode::addIncoming(), assert(), llvm::InstCombiner::Builder, llvm::IRBuilderBase::CreatePHI(), I, II, llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::InstCombiner::replaceInstUsesWith(), and llvm::IRBuilderBase::SetInsertPoint().

Referenced by instCombineConvertFromSVBool().

SimplifyValuePattern()

bool SimplifyValuePattern	(	SmallVector< Value * > &	Vec,
		bool	AllowPoison
	)

Definition at line 1965 of file AArch64TargetTransformInfo.cpp.

References llvm::SmallVectorTemplateCommon< T, typename >::begin(), llvm::SmallVectorTemplateCommon< T, typename >::end(), llvm::isPowerOf2_64(), LHS, llvm::SmallVectorImpl< T >::resize(), RHS, SimplifyValuePattern(), and llvm::SmallVectorBase< Size_T >::size().

Referenced by instCombineSVEDupqLane(), and SimplifyValuePattern().

tryCombineFromSVBoolBinOp()

static std::optional< Instruction * > tryCombineFromSVBoolBinOp ( InstCombiner &  IC, IntrinsicInst &  II  )

static

Definition at line 959 of file AArch64TargetTransformInfo.cpp.

References llvm::InstCombiner::Builder, llvm::IRBuilderBase::CreateIntrinsic(), II, llvm::SmallVectorTemplateBase< T, bool >::push_back(), and llvm::InstCombiner::replaceInstUsesWith().

Referenced by instCombineConvertFromSVBool().

Variable Documentation

BaseHistCntCost

cl::opt< unsigned > BaseHistCntCost("aarch64-base-histcnt-cost", cl::init(8), cl::Hidden, cl::desc("The cost of a histcnt instruction")) ( "aarch64-base-histcnt-cost"  , cl::init(8)  , cl::Hidden  , cl::desc("The cost of a histcnt instruction")    )

static

Referenced by getHistogramCost().

CallPenaltyChangeSM

cl::opt< unsigned > CallPenaltyChangeSM("call-penalty-sm-change", cl::init(5), cl::Hidden, cl::desc( "Penalty of calling a function that requires a change to PSTATE.SM")) ( "call-penalty-sm-change"  , cl::init(5)  , cl::Hidden  , cl::desc( "Penalty of calling a function that requires a change to PSTATE.SM")    )

static

Referenced by llvm::AArch64TTIImpl::getInlineCallPenalty().

EnableFalkorHWPFUnrollFix

cl::opt< bool > EnableFalkorHWPFUnrollFix("enable-falkor-hwpf-unroll-fix", cl::init(true), cl::Hidden) ( "enable-falkor-hwpf-unroll-fix"  , cl::init(true)  , cl::Hidden    )

static

Referenced by llvm::AArch64TTIImpl::getUnrollingPreferences().

EnableFixedwidthAutovecInStreamingMode

cl::opt< bool > EnableFixedwidthAutovecInStreamingMode("enable-fixedwidth-autovec-in-streaming-mode", cl::init(false), cl::Hidden) ( "enable-fixedwidth-autovec-in-streaming-mode"  , cl::init(false)  , cl::Hidden    )

static

Referenced by llvm::AArch64TTIImpl::getRegisterBitWidth().

EnableLSRCostOpt

cl::opt< bool > EnableLSRCostOpt("enable-aarch64-lsr-cost-opt", cl::init(true), cl::Hidden) ( "enable-aarch64-lsr-cost-opt"  , cl::init(true)  , cl::Hidden    )

static

Referenced by llvm::AArch64TTIImpl::isLSRCostLess().

EnableOrLikeSelectOpt

cl::opt< bool > EnableOrLikeSelectOpt("enable-aarch64-or-like-select", cl::init(true), cl::Hidden) ( "enable-aarch64-or-like-select"  , cl::init(true)  , cl::Hidden    )

static

Referenced by llvm::AArch64TTIImpl::shouldTreatInstructionLikeSelect().

EnableScalableAutovecInStreamingMode

cl::opt< bool > EnableScalableAutovecInStreamingMode("enable-scalable-autovec-in-streaming-mode", cl::init(false), cl::Hidden) ( "enable-scalable-autovec-in-streaming-mode"  , cl::init(false)  , cl::Hidden    )

static

Referenced by llvm::AArch64TTIImpl::enableScalableVectorization(), and llvm::AArch64TTIImpl::getRegisterBitWidth().

InlineCallPenaltyChangeSM

cl::opt< unsigned > InlineCallPenaltyChangeSM("inline-call-penalty-sm-change", cl::init(10), cl::Hidden, cl::desc("Penalty of inlining a call that requires a change to PSTATE.SM")) ( "inline-call-penalty-sm-change"  , cl::init(10)  , cl::Hidden  , cl::desc("Penalty of inlining a call that requires a change to PSTATE.SM")    )

static

Referenced by llvm::AArch64TTIImpl::getInlineCallPenalty().

NeonNonConstStrideOverhead

cl::opt< unsigned > NeonNonConstStrideOverhead("neon-nonconst-stride-overhead", cl::init(10), cl::Hidden) ( "neon-nonconst-stride-overhead"  , cl::init(10)  , cl::Hidden    )

static

Referenced by llvm::AArch64TTIImpl::getAddressComputationCost().

SVEGatherOverhead

cl::opt< unsigned > SVEGatherOverhead("sve-gather-overhead", cl::init(10), cl::Hidden) ( "sve-gather-overhead"  , cl::init(10)  , cl::Hidden    )

static

Referenced by getSVEGatherScatterOverhead().

SVEScatterOverhead

cl::opt< unsigned > SVEScatterOverhead("sve-scatter-overhead", cl::init(10), cl::Hidden) ( "sve-scatter-overhead"  , cl::init(10)  , cl::Hidden    )

static

Referenced by getSVEGatherScatterOverhead().

SVETailFolding

cl::opt< TailFoldingOption, true, cl::parser< std::string > > SVETailFolding("sve-tail-folding", cl::desc( "Control the use of vectorisation using tail-folding for SVE where the" " option is specified in the form (Initial)[+(Flag1|Flag2|...)]:" "\ndisabled (Initial) No loop types will vectorize using " "tail-folding" "\ndefault (Initial) Uses the default tail-folding settings for " "the target CPU" "\nall (Initial) All legal loop types will vectorize using " "tail-folding" "\nsimple (Initial) Use tail-folding for simple loops (not " "reductions or recurrences)" "\nreductions Use tail-folding for loops containing reductions" "\nnoreductions Inverse of above" "\nrecurrences Use tail-folding for loops containing fixed order " "recurrences" "\nnorecurrences Inverse of above" "\nreverse Use tail-folding for loops requiring reversed " "predicates" "\nnoreverse Inverse of above"), cl::location(TailFoldingOptionLoc))	(	"sve-tail-folding"	,
		cl::desc( "Control the use of vectorisation using tail-folding for SVE where the" " option is specified in the form (Initial)[+(Flag1|Flag2|...)]:" "\ndisabled (Initial) No loop types will vectorize using " "tail-folding" "\ndefault (Initial) Uses the default tail-folding settings for " "the target CPU" "\nall (Initial) All legal loop types will vectorize using " "tail-folding" "\nsimple (Initial) Use tail-folding for simple loops (not " "reductions or recurrences)" "\nreductions Use tail-folding for loops containing reductions" "\nnoreductions Inverse of above" "\nrecurrences Use tail-folding for loops containing fixed order " "recurrences" "\nnorecurrences Inverse of above" "\nreverse Use tail-folding for loops requiring reversed " "predicates" "\nnoreverse Inverse of above")	,
		cl::location(TailFoldingOptionLoc)
	)

SVETailFoldInsnThreshold

cl::opt< unsigned > SVETailFoldInsnThreshold("sve-tail-folding-insn-threshold", cl::init(15), cl::Hidden) ( "sve-tail-folding-insn-threshold"  , cl::init(15)  , cl::Hidden    )

static

Referenced by llvm::AArch64TTIImpl::preferPredicateOverEpilogue().

TailFoldingOptionLoc

TailFoldingOption TailFoldingOptionLoc

Definition at line 176 of file AArch64TargetTransformInfo.cpp.

Referenced by llvm::AArch64TTIImpl::preferPredicateOverEpilogue().