AArch64ISelLowering.cpp File Reference

#include "AArch64ISelLowering.h"
#include "AArch64CallingConvention.h"
#include "AArch64ExpandImm.h"
#include "AArch64MachineFunctionInfo.h"
#include "AArch64PerfectShuffle.h"
#include "AArch64RegisterInfo.h"
#include "AArch64SMEAttributes.h"
#include "AArch64Subtarget.h"
#include "AArch64TargetMachine.h"
#include "MCTargetDesc/AArch64AddressingModes.h"
#include "Utils/AArch64BaseInfo.h"
#include "llvm/ADT/APFloat.h"
#include "llvm/ADT/APInt.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SmallSet.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/SmallVectorExtras.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/ADT/StringSwitch.h"
#include "llvm/ADT/Twine.h"
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/Analysis/MemoryLocation.h"
#include "llvm/Analysis/ObjCARCUtil.h"
#include "llvm/Analysis/OptimizationRemarkEmitter.h"
#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/Analysis/VectorUtils.h"
#include "llvm/CodeGen/Analysis.h"
#include "llvm/CodeGen/CallingConvLower.h"
#include "llvm/CodeGen/CodeGenCommonISel.h"
#include "llvm/CodeGen/ComplexDeinterleavingPass.h"
#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h"
#include "llvm/CodeGen/GlobalISel/Utils.h"
#include "llvm/CodeGen/ISDOpcodes.h"
#include "llvm/CodeGen/MachineBasicBlock.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineInstr.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
#include "llvm/CodeGen/MachineMemOperand.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/CodeGen/SDPatternMatch.h"
#include "llvm/CodeGen/SelectionDAG.h"
#include "llvm/CodeGen/SelectionDAGNodes.h"
#include "llvm/CodeGen/TargetCallingConv.h"
#include "llvm/CodeGen/TargetInstrInfo.h"
#include "llvm/CodeGen/TargetOpcodes.h"
#include "llvm/CodeGen/ValueTypes.h"
#include "llvm/CodeGenTypes/MachineValueType.h"
#include "llvm/IR/Attributes.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/DebugLoc.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/GetElementPtrTypeIterator.h"
#include "llvm/IR/GlobalValue.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/Instruction.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/IntrinsicsAArch64.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/PatternMatch.h"
#include "llvm/IR/Type.h"
#include "llvm/IR/Use.h"
#include "llvm/IR/Value.h"
#include "llvm/Support/AtomicOrdering.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/CodeGen.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/InstructionCost.h"
#include "llvm/Support/KnownBits.h"
#include "llvm/Support/MathExtras.h"
#include "llvm/Support/SipHash.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Target/TargetMachine.h"
#include "llvm/Target/TargetOptions.h"
#include "llvm/TargetParser/Triple.h"
#include <algorithm>
#include <bitset>
#include <cassert>
#include <cctype>
#include <cstdint>
#include <cstdlib>
#include <iterator>
#include <limits>
#include <optional>
#include <tuple>
#include <utility>
#include <vector>
#include "AArch64GenAsmMatcher.inc"

Go to the source code of this file.

Classes
struct	SVEMultiVectorInfo
struct	uitofp_tbl_match
struct	deinterleaving_tbl_match
struct	GenericSetCCInfo
	Helper structure to keep track of ISD::SET_CC operands. More...
struct	AArch64SetCCInfo
	Helper structure to keep track of a SET_CC lowered into AArch64 code. More...
union	SetCCInfo
	Helper structure to keep track of SetCC information. More...
struct	SetCCInfoAndKind
	Helper structure to be able to read SetCC information. More...

Macros
#define	DEBUG_TYPE   "aarch64-lower"
#define	GET_REGISTER_MATCHER

Enumerations
enum class	PredicateConstraint { Uph , Upl , Upa }
enum class	ReducedGprConstraint { Uci , Ucj }

Functions
	STATISTIC (NumTailCalls, "Number of tail calls")
	STATISTIC (NumOptimizedImms, "Number of times immediates were optimized")
static EVT	getPackedSVEVectorVT (EVT VT)
static EVT	getPackedSVEVectorVT (ElementCount EC)
static EVT	getPromotedVTForPredicate (EVT VT)
static bool	isPackedVectorType (EVT VT, SelectionDAG &DAG)
	Returns true if VT's elements occupy the lowest bit positions of its associated register class without any intervening space.
static bool	isPackedPredicateType (EVT VT, SelectionDAG &DAG)
static bool	isUnpackedType (EVT VT, SelectionDAG &DAG)
	Returns true if the conceptual representation for VT does not map directly to its physical register representation, meaning there are gaps between elements in the register.
static bool	isMergePassthruOpcode (unsigned Opc)
static bool	isZeroingInactiveLanes (SDValue Op)
static std::tuple< SDValue, SDValue >	extractPtrauthBlendDiscriminators (SDValue Disc, SelectionDAG *DAG)
static bool	isIntImmediate (const SDNode *N, uint64_t &Imm)
static bool	isVectorizedBinOp (unsigned Opcode)
static bool	isOpcWithIntImmediate (const SDNode *N, unsigned Opc, uint64_t &Imm)
static bool	optimizeLogicalImm (SDValue Op, unsigned Size, uint64_t Imm, const APInt &Demanded, TargetLowering::TargetLoweringOpt &TLO, unsigned NewOpc)
static const MachineInstr *	stripVRegCopies (const MachineRegisterInfo &MRI, Register Reg)
static EVT	getContainerForFixedLengthVector (SelectionDAG &DAG, EVT VT)
static SDValue	convertToScalableVector (SelectionDAG &DAG, EVT VT, SDValue V)
static SDValue	convertFromScalableVector (SelectionDAG &DAG, EVT VT, SDValue V)
static SDValue	convertFixedMaskToScalableVector (SDValue Mask, SelectionDAG &DAG)
static SDValue	getPredicateForVector (SelectionDAG &DAG, SDLoc &DL, EVT VT)
static SDValue	getPredicateForFixedLengthVector (SelectionDAG &DAG, SDLoc &DL, EVT VT)
static SDValue	getPredicateForScalableVector (SelectionDAG &DAG, SDLoc &DL, EVT VT)
static SDValue	getSVEPredicateBitCast (EVT VT, SDValue Op, SelectionDAG &DAG)
static bool	isZerosVector (const SDNode *N)
	isZerosVector - Check whether SDNode N is a zero-filled vector.
static bool	isOneVector (SDValue V)
static AArch64CC::CondCode	changeIntCCToAArch64CC (ISD::CondCode CC, SDValue RHS={})
	changeIntCCToAArch64CC - Convert a DAG integer condition code to an AArch64 CC
static void	changeFPCCToAArch64CC (ISD::CondCode CC, AArch64CC::CondCode &CondCode, AArch64CC::CondCode &CondCode2)
	changeFPCCToAArch64CC - Convert a DAG fp condition code to an AArch64 CC.
static void	changeFPCCToANDAArch64CC (ISD::CondCode CC, AArch64CC::CondCode &CondCode, AArch64CC::CondCode &CondCode2)
	Convert a DAG fp condition code to an AArch64 CC.
static void	changeVectorFPCCToAArch64CC (ISD::CondCode CC, AArch64CC::CondCode &CondCode, AArch64CC::CondCode &CondCode2, bool &Invert)
	changeVectorFPCCToAArch64CC - Convert a DAG fp condition code to an AArch64 CC usable with the vector instructions.
static SDValue	getCondCode (SelectionDAG &DAG, AArch64CC::CondCode CC)
	Like SelectionDAG::getCondCode(), but for AArch64 condition codes.
static unsigned	numberOfInstrToLoadImm (const APInt &C)
static bool	isSafeSignedCMN (SDValue Op, SelectionDAG &DAG)
static bool	isCMN (SDValue Op, ISD::CondCode CC, SelectionDAG &DAG)
static SDValue	emitStrictFPComparison (SDValue LHS, SDValue RHS, const SDLoc &DL, SelectionDAG &DAG, SDValue Chain, bool IsSignaling)
static SDValue	emitComparison (SDValue LHS, SDValue RHS, ISD::CondCode CC, const SDLoc &DL, SelectionDAG &DAG)
static bool	isSupportedExtend (SDValue V)
static unsigned	getCmpOperandFoldingProfit (SDValue Op, bool AllowExtend)
	Returns how profitable it is to fold a comparison's operand's shift and/or extension operations.
static unsigned	getCmpOrCmnOperandFoldingProfit (SDValue Op, ISD::CondCode CC, SelectionDAG &DAG)
static bool	shouldBeAdjustedToZero (SDValue LHS, const APInt &C, ISD::CondCode &CC)
static SDValue	getAArch64Cmp (SDValue LHS, SDValue RHS, ISD::CondCode CC, SDValue &AArch64cc, SelectionDAG &DAG, const SDLoc &DL)
static std::pair< SDValue, SDValue >	getAArch64XALUOOp (AArch64CC::CondCode &CC, SDValue Op, SelectionDAG &DAG)
static SDValue	valueToCarryFlag (SDValue Value, SelectionDAG &DAG, bool Invert)
static SDValue	carryFlagToValue (SDValue Glue, EVT VT, SelectionDAG &DAG, bool Invert)
static SDValue	overflowFlagToValue (SDValue Glue, EVT VT, SelectionDAG &DAG)
static SDValue	lowerADDSUBO_CARRY (SDValue Op, SelectionDAG &DAG, unsigned Opcode, bool IsSigned)
static SDValue	lowerIntNeonIntrinsic (SDValue Op, unsigned Opcode, SelectionDAG &DAG, bool LastOperandIsImm=false)
static SDValue	LowerXALUO (SDValue Op, SelectionDAG &DAG)
static SDValue	LowerPREFETCH (SDValue Op, SelectionDAG &DAG)
static void	simplifySetCCIntoEq (ISD::CondCode &CC, SDValue &LHS, SDValue &RHS, SelectionDAG &DAG, const SDLoc DL)
static MVT	getSVEContainerType (EVT ContentTy)
static std::optional< uint64_t >	getConstantLaneNumOfExtractHalfOperand (SDValue &Op)
static bool	isExtendedBUILD_VECTOR (SDValue N, SelectionDAG &DAG, bool isSigned)
static SDValue	skipExtensionForVectorMULL (SDValue N, SelectionDAG &DAG)
static bool	isSignExtended (SDValue N, SelectionDAG &DAG)
static bool	isZeroExtended (SDValue N, SelectionDAG &DAG)
static bool	isAddSubSExt (SDValue N, SelectionDAG &DAG)
static bool	isAddSubZExt (SDValue N, SelectionDAG &DAG)
static unsigned	selectUmullSmull (SDValue &N0, SDValue &N1, SelectionDAG &DAG, SDLoc DL, bool &IsMLA)
static SDValue	convertMulToShlAdd (SDNode *N, SelectionDAG &DAG)
static SDValue	getPTrue (SelectionDAG &DAG, SDLoc DL, EVT VT, int Pattern)
static SDValue	getPTrueAsCounter (SelectionDAG &DAG, SDLoc DL, EVT VT)
static SDValue	optimizeIncrementingWhile (SDNode *N, SelectionDAG &DAG, bool IsSigned, bool IsEqual)
static SDValue	optimizeBrk (SDNode *N, SelectionDAG &DAG)
static SDValue	LowerSMELdrStr (SDValue N, SelectionDAG &DAG, bool IsLoad)
static SDValue	LowerVectorMatch (SDValue Op, SelectionDAG &DAG)
static bool	isEligibleForSmallVectorLoadOpt (LoadSDNode *LD, const AArch64Subtarget &Subtarget)
	Helper function to check if a small vector load can be optimized.
static unsigned	getGatherVecOpcode (bool IsScaled, bool IsSigned, bool NeedsExtend)
static unsigned	getSignExtendedGatherOpcode (unsigned Opcode)
static SDValue	LowerTruncateVectorStore (SDLoc DL, StoreSDNode *ST, EVT VT, EVT MemVT, SelectionDAG &DAG)
static SDValue	LowerADDRSPACECAST (SDValue Op, SelectionDAG &DAG)
static bool	isLegalNTStore (Type *DataType, Align Alignment, const DataLayout &DL)
static SDValue	LowerNTStore (StoreSDNode *StoreNode, EVT VT, EVT MemVT, const SDLoc &DL, SelectionDAG &DAG)
static SVEMultiVectorInfo	getSVEMultiVectorInfo (MVT VT)
static bool	isSimpleScalableLoadOrStore (const LSBaseSDNode *LSNode, EVT VT)
static SDValue	tryLowerMultiVectorStore (StoreSDNode *StoreNode, SelectionDAG &DAG)
static bool	tryLowerMultiVectorLoad (LoadSDNode *LoadNode, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG)
static SDValue	tryLowerSmallVectorExtLoad (LoadSDNode *Load, SelectionDAG &DAG)
	Helper function to optimize loads of extended small vectors.
static SDValue	LowerBRCOND (SDValue Op, SelectionDAG &DAG)
static SDValue	LowerFunnelShift (SDValue Op, SelectionDAG &DAG)
static SDValue	LowerFLDEXP (SDValue Op, SelectionDAG &DAG)
static SDValue	DoCrossByteCLMUL (SDValue OpA, SDValue OpB, SDLoc DL, SelectionDAG &DAG, EVT VT)
static unsigned	getIntrinsicID (const SDNode *N)
static bool	isPassedInFPR (EVT VT)
static bool	canGuaranteeTCO (CallingConv::ID CC, bool GuaranteeTailCalls)
	Return true if the calling convention is one that we can guarantee TCO for.
static bool	mayTailCallThisCC (CallingConv::ID CC)
	Return true if we might ever do TCO for calls with this calling convention.
static bool	callConvSupportsVarArgs (CallingConv::ID CC)
	Return true if the call convention supports varargs Currently only those that pass varargs like the C calling convention does are eligible Calling conventions listed in this function must also be properly handled in AArch64Subtarget::isCallingConvWin64.
static void	analyzeCallOperands (const AArch64TargetLowering &TLI, const AArch64Subtarget *Subtarget, const TargetLowering::CallLoweringInfo &CLI, CCState &CCInfo)
static SMECallAttrs	getSMECallAttrs (const Function &Caller, const RTLIB::RuntimeLibcallsInfo &RTLCI, const TargetLowering::CallLoweringInfo &CLI)
static bool	checkZExtBool (SDValue Arg, const SelectionDAG &DAG)
static AArch64SME::ToggleCondition	getSMToggleCondition (const SMECallAttrs &CallAttrs)
static bool	shouldLowerTailCallStackArg (const MachineFunction &MF, const CCValAssign &VA, SDValue Arg, ISD::ArgFlagsTy Flags, int CallOffset)
	Check whether a stack argument requires lowering in a tail call.
static std::optional< unsigned >	getZAMarkerForCall (const SMECallAttrs &CallAttrs)
	Returns the ZA state save/use marker node necessary for CallAttrs.
static SDValue	LowerPtrAuthGlobalAddressStatically (SDValue TGA, SDLoc DL, EVT VT, AArch64PACKey::ID KeyC, SDValue Discriminator, SDValue AddrDiscriminator, SelectionDAG &DAG)
static std::pair< SDValue, uint64_t >	lookThroughSignExtension (SDValue Val)
static SDValue	optimizeBitTest (SDLoc DL, SDValue Op, SDValue Chain, SDValue Dest, unsigned Opcode, SelectionDAG &DAG)
static bool	isLegalCondCmpImmediate (const APInt &Imm)
static bool	hasLegalCmpImmediate (SDValue LHS, SDValue RHS)
static bool	hasLegalCondCmpImmediate (SDValue LHS, SDValue RHS)
static bool	hasCheapXorImmediateThatNeedsCondCmpReg (const std::pair< SDValue, SDValue > &Pair)
static bool	isOrXorChain (SDValue N, SelectionDAG &DAG, unsigned &NumLeaves, unsigned &NumXors, bool &SawXor, bool RequireLegalCmpImmediates, SmallVectorImpl< std::pair< SDValue, SDValue > > &WorkList)
static SDValue	performOrXorChainCombine (SDNode *N, SelectionDAG &DAG)
static SDValue	emitVectorComparison (SDValue LHS, SDValue RHS, AArch64CC::CondCode CC, bool NoNans, EVT VT, const SDLoc &DL, SelectionDAG &DAG)
	Emit vector comparison for floating-point values, producing a mask.
static SDValue	emitFloatCompareMask (SDValue LHS, SDValue RHS, SDValue TVal, SDValue FVal, ISD::CondCode CC, bool NoNaNs, const SDLoc &DL, SelectionDAG &DAG)
	For SELECT_CC, when the true/false values are (-1, 0) and the compared values are scalars, try to emit a mask generating vector instruction.
static SDValue	getEstimate (const AArch64Subtarget *ST, unsigned Opcode, SDValue Operand, SelectionDAG &DAG, int &ExtraSteps)
static std::optional< std::pair< unsigned, const TargetRegisterClass * > >	parseSVERegAsConstraint (StringRef Constraint)
static std::optional< PredicateConstraint >	parsePredicateConstraint (StringRef Constraint)
static const TargetRegisterClass *	getPredicateRegisterClass (PredicateConstraint Constraint, EVT VT)
static std::optional< ReducedGprConstraint >	parseReducedGprConstraint (StringRef Constraint)
static const TargetRegisterClass *	getReducedGprRegisterClass (ReducedGprConstraint Constraint, EVT VT)
static AArch64CC::CondCode	parseConstraintCode (llvm::StringRef Constraint)
static SDValue	getSETCC (AArch64CC::CondCode CC, SDValue NZCV, const SDLoc &DL, SelectionDAG &DAG)
	Helper function to create 'CSET', which is equivalent to 'CSINC <Wd>, WZR, WZR, invert(<cond>)'.
static SDValue	WidenVector (SDValue V64Reg, SelectionDAG &DAG)
	WidenVector - Given a value in the V64 register class, produce the equivalent value in the V128 register class.
static unsigned	getExtFactor (SDValue &V)
	getExtFactor - Determine the adjustment factor for the position when generating an "extract from vector registers" instruction.
SDValue	ReconstructShuffleWithRuntimeMask (SDValue Op, SelectionDAG &DAG)
static SDValue	ReconstructTruncateFromBuildVector (SDValue V, SelectionDAG &DAG)
static bool	isWideDUPMask (ArrayRef< int > M, EVT VT, unsigned BlockSize, unsigned &DupLaneOp)
	Check if a vector shuffle corresponds to a DUP instructions with a larger element width than the vector lane type.
static unsigned	checkLaneSlide (ArrayRef< int > Mask, unsigned LaneStart, unsigned LaneElts, unsigned NumElts, bool &IsLeftSlide)
	Flag slide shuffle patterns where one operand is zeros.
static SDValue	isSlideWithZerosMask (ArrayRef< int > M, EVT VT, SDValue V1, SDValue V2, unsigned &ShiftAmount, bool &IsRightShift, unsigned &MatchedLaneSize)
static bool	isEXTMaskWithSplat (ArrayRef< int > M, EVT VT, unsigned SplatOperand, bool &ReverseEXT, unsigned &Imm)
static bool	isINSMask (ArrayRef< int > M, int NumInputElements, bool &DstIsLeft, int &Anomaly)
static bool	isConcatMask (ArrayRef< int > Mask, EVT VT, bool SplitLHS)
static SDValue	tryFormConcatFromShuffle (SDValue Op, SelectionDAG &DAG)
static SDValue	GenerateTBL (SDValue Op, ArrayRef< int > ShuffleMask, SelectionDAG &DAG)
static unsigned	getDUPLANEOp (EVT EltType)
static SDValue	constructDup (SDValue V, int Lane, SDLoc DL, EVT VT, unsigned Opcode, SelectionDAG &DAG)
static SDValue	tryWidenMaskForShuffle (SDValue Op, SelectionDAG &DAG)
static SDValue	tryToConvertShuffleOfTbl2ToTbl4 (SDValue Op, ArrayRef< int > ShuffleMask, SelectionDAG &DAG)
static bool	resolveBuildVector (BuildVectorSDNode *BVN, APInt &CnstBits, APInt &UndefBits)
static SDValue	tryAdvSIMDModImm64 (unsigned NewOp, SDValue Op, SelectionDAG &DAG, const APInt &Bits)
static SDValue	tryAdvSIMDModImm32 (unsigned NewOp, SDValue Op, SelectionDAG &DAG, const APInt &Bits, const SDValue *LHS=nullptr)
static SDValue	tryAdvSIMDModImm16 (unsigned NewOp, SDValue Op, SelectionDAG &DAG, const APInt &Bits, const SDValue *LHS=nullptr)
static SDValue	tryAdvSIMDModImm321s (unsigned NewOp, SDValue Op, SelectionDAG &DAG, const APInt &Bits)
static SDValue	tryAdvSIMDModImm8 (unsigned NewOp, SDValue Op, SelectionDAG &DAG, const APInt &Bits)
static SDValue	tryAdvSIMDModImmFP (unsigned NewOp, SDValue Op, SelectionDAG &DAG, const APInt &Bits)
static bool	isAllConstantBuildVector (const SDValue &PotentialBVec, uint64_t &ConstVal)
static bool	isAllInactivePredicate (SDValue N)
static bool	isAllActivePredicate (const SelectionDAG &DAG, SDValue N)
static SDValue	tryLowerToSLI (SDNode *N, SelectionDAG &DAG)
static SDValue	tryLowerToBSL (SDValue N, SelectionDAG &DAG)
static SDValue	NormalizeBuildVector (SDValue Op, SelectionDAG &DAG)
static SDValue	trySVESplat64 (SDValue Op, SelectionDAG &DAG, const AArch64Subtarget *ST, APInt &DefBits)
static SDValue	ConstantBuildVector (SDValue Op, SelectionDAG &DAG, const AArch64Subtarget *ST)
static bool	isPow2Splat (SDValue Op, uint64_t &SplatVal, bool &Negated)
static bool	getVShiftImm (SDValue Op, unsigned ElementBits, int64_t &Cnt)
	getVShiftImm - Check if this is a valid build_vector for the immediate operand of a vector shift operation, where all the elements of the build_vector must have the same constant integer value.
static bool	isVShiftLImm (SDValue Op, EVT VT, bool isLong, int64_t &Cnt)
	isVShiftLImm - Check if this is a valid build_vector for the immediate operand of a vector shift left operation.
static bool	isVShiftRImm (SDValue Op, EVT VT, bool isNarrow, int64_t &Cnt)
	isVShiftRImm - Check if this is a valid build_vector for the immediate operand of a vector shift right operation.
static bool	canLowerSRLToRoundingShiftForVT (SDValue Shift, EVT ResVT, SelectionDAG &DAG, unsigned &ShiftValue, SDValue &RShOperand)
static SDValue	getReductionSDNode (unsigned Op, SDLoc DL, SDValue ScalarOp, SelectionDAG &DAG)
static SDValue	getVectorBitwiseReduce (unsigned Opcode, SDValue Vec, EVT VT, SDLoc DL, SelectionDAG &DAG)
template<unsigned NumVecs>
static void	setInfoSVEStN (const AArch64TargetLowering &TLI, const DataLayout &DL, AArch64TargetLowering::IntrinsicInfo &Info, const CallBase &CI)
	Set the IntrinsicInfo for the aarch64_sve_st<N> intrinsics.
static bool	createTblShuffleMask (unsigned SrcWidth, unsigned DstWidth, unsigned NumElts, bool IsLittleEndian, SmallVectorImpl< int > &Mask)
static Value *	createTblShuffleForZExt (IRBuilderBase &Builder, Value *Op, FixedVectorType *ZExtTy, FixedVectorType *DstTy, bool IsLittleEndian)
static Value *	createTblShuffleForSExt (IRBuilderBase &Builder, Value *Op, FixedVectorType *DstTy, bool IsLittleEndian)
static void	createTblForTrunc (TruncInst *TI, bool IsLittleEndian)
static auto	m_Tbl (Instruction *&Tbl, Value *&Mask)
static auto	m_UIToFPTbl (Instruction *&Tbl, Value *&Mask)
static auto	m_DeinterleavingTbl (Instruction *&Tbl, unsigned &Idx)
static bool	foldAdjacentReversesIntoTbls (IntrinsicInst *Rev, Loop *L)
static ScalableVectorType *	getSVEContainerIRType (FixedVectorType *VTy)
static Function *	getStructuredLoadFunction (Module *M, unsigned Factor, bool Scalable, Type *LDVTy, Type *PtrTy)
static Function *	getStructuredStoreFunction (Module *M, unsigned Factor, bool Scalable, Type *STVTy, Type *PtrTy)
template<typename Iter>
bool	hasNearbyPairedStore (Iter It, Iter End, Value *Ptr, const DataLayout &DL)
static SDValue	foldVectorXorShiftIntoCmp (SDNode *N, SelectionDAG &DAG, const AArch64Subtarget *Subtarget)
	Turn vector tests of the signbit in the form of: xor (sra X, elt_size(X)-1), -1 into: cmge X, X, #0.
static SDValue	performVecReduceAddCombineWithUADDLP (SDNode *N, SelectionDAG &DAG)
static SDValue	performActiveLaneMaskCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const AArch64Subtarget *ST)
static SDValue	performVecReduceAddCntpCombine (SDNode *N, SelectionDAG &DAG, const AArch64Subtarget *ST)
static SDValue	performVecReduceAddCombine (SDNode *N, SelectionDAG &DAG, const AArch64Subtarget *ST)
static SDValue	performUADDVAddCombine (SDValue A, SelectionDAG &DAG)
static SDValue	performUADDVZextCombine (SDValue A, SelectionDAG &DAG)
static SDValue	performUADDVCombine (SDNode *N, SelectionDAG &DAG)
static SDValue	performBICiCombine (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
static SDValue	performXorCombine (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const AArch64Subtarget *Subtarget)
static bool	IsSVECntIntrinsic (SDValue S)
static std::optional< ElementCount >	getMaxValueForSVECntIntrinsic (SDValue Op)
static EVT	calculatePreExtendType (SDValue Extend)
	Calculates what the pre-extend type is, based on the extension operation node provided by Extend.
static SDValue	performBuildShuffleExtendCombine (SDValue BV, SelectionDAG &DAG)
	Combines a buildvector(sext/zext) or shuffle(sext/zext, undef) node pattern into sext/zext(buildvector) or sext/zext(shuffle) making use of the vector SExt/ZExt rather than the scalar SExt/ZExt.
static SDValue	performMulVectorExtendCombine (SDNode *Mul, SelectionDAG &DAG)
	Combines a mul(dup(sext/zext)) node pattern into mul(sext/zext(dup)) making use of the vector SExt/ZExt rather than the scalar SExt/ZExt.
static SDValue	performMulRdsvlCombine (SDNode *Mul, SelectionDAG &DAG)
static SDValue	performMulVectorCmpZeroCombine (SDNode *N, SelectionDAG &DAG)
static SDValue	performVectorExtCombine (SDNode *N, SelectionDAG &DAG)
static bool	isOnlyUsedAsMemoryAddress (SDNode *N)
static SDValue	performMulCombine (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const AArch64Subtarget *Subtarget)
static SDValue	performVectorCompareAndMaskUnaryOpCombine (SDNode *N, SelectionDAG &DAG)
static SDValue	tryToReplaceScalarFPConversionWithSVE (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const AArch64Subtarget *Subtarget)
	Tries to replace scalar FP <-> INT conversions with SVE in streaming functions, this can help to reduce the number of fmovs to/from GPRs.
static SDValue	performIntToFpCombine (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const AArch64Subtarget *Subtarget)
static SDValue	performFpToIntCombine (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const AArch64Subtarget *Subtarget)
	Fold a floating-point multiply by power of two into floating-point to fixed-point conversion.
static SDValue	performANDORCSELCombine (SDNode *N, SelectionDAG &DAG)
static SDValue	tryCombineToREV (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
static SDValue	performANDORDUPNOTCombine (SDNode *N, SelectionDAG &DAG)
static SDValue	performORCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static bool	isConstantSplatVectorMaskForType (SDNode *N, EVT MemVT)
static SDValue	performReinterpretCastCombine (SDNode *N)
static SDValue	performSVEAndCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SDValue	performANDSETCCCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SDValue	performANDCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SDValue	performFADDCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SDValue	performFADDPREDCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static bool	hasPairwiseAdd (unsigned Opcode, EVT VT, bool FullFP16)
static SDValue	getPTest (SelectionDAG &DAG, EVT VT, SDValue Pg, SDValue Op, AArch64CC::CondCode Cond)
static bool	isPredicateCCSettingOp (SDValue N)
static SDValue	performFirstTrueTestVectorCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const AArch64Subtarget *Subtarget)
static SDValue	performLastTrueTestVectorCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const AArch64Subtarget *Subtarget)
static SDValue	performExtractLastActiveCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const AArch64Subtarget *Subtarget)
static bool	hasSVEMultiVectorOps (const AArch64Subtarget *Subtarget)
static auto	m_PredicateAsCounterWhile ()
static SDValue	perfomPextFirstTrueVectorCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const AArch64Subtarget *Subtarget)
	Folds extracting the first lane from the first segment of a predicate-as-counter while to a conditional set (CSET) based on the "FIRST_ACTIVE" status flag from the while.
static SDValue	performExtractVectorEltCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const AArch64Subtarget *Subtarget)
static SDValue	performConcatVectorsCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
static SDValue	performExtractSubvectorCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
static SDValue	performInsertSubvectorCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
static SDValue	tryCombineFixedPointConvert (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
static SDValue	tryExtendDUPToExtractHigh (SDValue N, SelectionDAG &DAG)
static bool	isEssentiallyExtractHighSubvector (SDValue N)
static bool	isSetCC (SDValue Op, SetCCInfoAndKind &SetCCInfo)
	Check whether or not Op is a SET_CC operation, either a generic or an AArch64 lowered one.
static bool	isSetCCOrZExtSetCC (const SDValue &Op, SetCCInfoAndKind &Info)
static SDValue	performSetccAddFolding (SDNode *Op, TargetLowering::DAGCombinerInfo &DCI)
static SDValue	performAddUADDVCombine (SDNode *N, SelectionDAG &DAG)
static SDValue	performAddCSelIntoCSinc (SDNode *N, SelectionDAG &DAG)
	Perform the scalar expression combine in the form of: CSEL(c, 1, cc) + b => CSINC(b+c, b, cc) CSNEG(c, -1, cc) + b => CSINC(b+c, b, cc)
static SDValue	performAddDotCombine (SDNode *N, SelectionDAG &DAG)
static bool	isNegatedInteger (SDValue Op)
static SDValue	performNegCSelCombine (SDNode *N, SelectionDAG &DAG)
static SDValue	reassociateAddSubLong (SDNode *N, SelectionDAG &DAG)
static SDValue	performAddSubLongCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static bool	isCMP (SDValue Op)
static std::optional< AArch64CC::CondCode >	getCSETCondCode (SDValue Op)
static SDValue	foldOverflowCheck (SDNode *Op, SelectionDAG &DAG, bool IsAdd)
static SDValue	foldADCToCINC (SDNode *N, SelectionDAG &DAG)
static SDValue	performAddSubCarryCombine (SDNode *N, SelectionDAG &DAG, unsigned NewOpcode)
	Takes advantage of the negation of the second operand by the with-carry-in instructions to fold:
static SDValue	performBuildVectorCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
static SDValue	trySQDMULHCombine (SDNode *N, SelectionDAG &DAG)
static SDValue	performTruncateCombine (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
static bool	isExtendOrShiftOperand (SDValue N)
static SDValue	performAddCombineSubShift (SDNode *N, SDValue SUB, SDValue Z, SelectionDAG &DAG)
static SDValue	performAddCombineForShiftedOperands (SDNode *N, SelectionDAG &DAG)
static SDValue	performNegSignMaskedAndCombine (SDNode *N, SelectionDAG &DAG)
static SDValue	performSubAddMULCombine (SDNode *N, SelectionDAG &DAG)
static SDValue	performSVEMulAddSubCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SDValue	performAddSubIntoVectorOp (SDNode *N, SelectionDAG &DAG)
static bool	isLoadOrMultipleLoads (SDValue B, SmallVector< LoadSDNode * > &Loads)
static bool	areLoadedOffsetButOtherwiseSame (SDValue Op0, SDValue Op1, SelectionDAG &DAG, unsigned &NumSubLoads)
static SDValue	performExtBinopLoadFold (SDNode *N, SelectionDAG &DAG)
static SDValue	performAddMulZextCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SDValue	performSubWithBorrowCombine (SDNode *N, SelectionDAG &DAG)
static SDValue	performAddTruncShiftCombine (SDNode *N, SelectionDAG &DAG)
static SDValue	performSubNegAndOneCombine (SDNode *N, SelectionDAG &DAG)
static SDValue	performAddAddCombine (SDNode *N, SelectionDAG &DAG)
static SDValue	performAddWithSBCCombine (SDNode *N, SelectionDAG &DAG)
static SDValue	performConditionalSubCombine (SDNode *N, SelectionDAG &DAG)
static SDValue	performAddSubCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SDValue	tryCombineLongOpWithDup (unsigned IID, SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
static SDValue	tryCombineShiftImm (unsigned IID, SDNode *N, SelectionDAG &DAG)
static SDValue	tryCombineCRC32 (unsigned Mask, SDNode *N, SelectionDAG &DAG)
static SDValue	combineAcrossLanesIntrinsic (unsigned Opc, SDNode *N, SelectionDAG &DAG)
static SDValue	LowerSVEIntrinsicIndex (SDNode *N, SelectionDAG &DAG)
static SDValue	LowerSVEIntrinsicDUP (SDNode *N, SelectionDAG &DAG)
static SDValue	LowerSVEIntrinsicEXT (SDNode *N, SelectionDAG &DAG)
static SDValue	tryConvertSVEWideCompare (SDNode *N, ISD::CondCode CC, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
static SDValue	combineSVEReductionInt (SDNode *N, unsigned Opc, SelectionDAG &DAG)
static SDValue	combineSVEReductionFP (SDNode *N, unsigned Opc, SelectionDAG &DAG)
static SDValue	combineSVEReductionOrderedFP (SDNode *N, unsigned Opc, SelectionDAG &DAG)
static SDValue	tryCombineNeonFcvtFP16ToI16 (SDNode *N, unsigned Opcode, SelectionDAG &DAG)
static SDValue	convertMergedOpToPredOp (SDNode *N, unsigned Opc, SelectionDAG &DAG, bool UnpredOp=false, bool SwapOperands=false)
static SDValue	combineSVEBitSel (unsigned IID, SDNode *N, SelectionDAG &DAG)
static SDValue	tryCombineFADDReductionWithZero (SDNode *N, SelectionDAG &DAG, const AArch64Subtarget *Subtarget, SDValue Vec, SDValue Start=SDValue())
	Optimize patterns where we insert zeros into vector lanes before a floating-point add reduction.
static unsigned	getPredCounterWhileElementSize (Intrinsic::ID IID)
	Returns the element size of a predicate-as-counter while intrinsic.
static unsigned	getPredCounterWhileOpcode (Intrinsic::ID IID)
	Returns the ISD opcode for a predicate-as-counter while intrinsic.
static SDValue	lowerPredCounterWhile (Intrinsic::ID IID, SDNode *N, SelectionDAG &DAG)
	Lower a predicate-as-counter while intrinsic to an AArch64ISD node.
static SDValue	performIntrinsicCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const AArch64Subtarget *Subtarget)
static bool	isCheapToExtend (const SDValue &N)
static SDValue	performSignExtendSetCCCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
static SDValue	performZExtDeinterleaveShuffleCombine (SDNode *N, SelectionDAG &DAG)
static SDValue	performZExtUZPCombine (SDNode *N, SelectionDAG &DAG)
static SDValue	combineToExtendBoolVectorInReg (unsigned Opcode, const SDLoc &DL, EVT VT, SDValue N0, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const AArch64Subtarget &Subtarget)
static SDValue	performExtendDuplaneTruncCombine (SDNode *N, SelectionDAG &DAG)
static SDValue	performExtendToBoolVectorLoadCombine (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const AArch64Subtarget &Subtarget)
static SDValue	performExtendCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG, const AArch64Subtarget *Subtarget)
static SDValue	splitStoreSplat (SelectionDAG &DAG, StoreSDNode &St, SDValue SplatVal, unsigned NumVecElts)
static SDValue	performLD1Combine (SDNode *N, SelectionDAG &DAG, unsigned Opc)
static SDValue	performLDNT1Combine (SDNode *N, SelectionDAG &DAG)
template<unsigned Opcode>
static SDValue	performLD1ReplicateCombine (SDNode *N, SelectionDAG &DAG)
static SDValue	performST1Combine (SDNode *N, SelectionDAG &DAG)
static SDValue	performSTNT1Combine (SDNode *N, SelectionDAG &DAG)
static SDValue	replaceZeroVectorStore (SelectionDAG &DAG, StoreSDNode &St)
	Replace a splat of zeros to a vector store by scalar stores of WZR/XZR.
static SDValue	replaceSplatVectorStore (SelectionDAG &DAG, StoreSDNode &St)
	Replace a splat of a scalar to a vector store by scalar stores of the scalar value.
static SDValue	splitStores (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG, const AArch64Subtarget *Subtarget)
static SDValue	performSpliceCombine (SDNode *N, SelectionDAG &DAG)
static SDValue	performUnpackCombine (SDNode *N, SelectionDAG &DAG, const AArch64Subtarget *Subtarget)
static bool	isHalvingTruncateAndConcatOfLegalIntScalableType (SDNode *N)
static SDValue	tryCombineExtendRShTrunc (SDNode *N, SelectionDAG &DAG)
static SDValue	trySimplifySrlAddToRshrnb (SDValue Srl, SelectionDAG &DAG, const AArch64Subtarget *Subtarget)
static SDValue	isNVCastToHalfWidthElements (SDValue V, const AArch64Subtarget *Subtarget)
static SDValue	performUzpCombine (SDNode *N, SelectionDAG &DAG, const AArch64Subtarget *Subtarget)
static SDValue	performGLD1Combine (SDNode *N, SelectionDAG &DAG)
static SDValue	performVectorShiftCombine (SDNode *N, const AArch64TargetLowering &TLI, TargetLowering::DAGCombinerInfo &DCI)
	Optimize a vector shift instruction and its operand if shifted out bits are not used.
static SDValue	performSunpkloCombine (SDNode *N, SelectionDAG &DAG)
static SDValue	performPostLD1Combine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, bool IsLaneOp)
	Target-specific DAG combine function for post-increment LD1 (lane) and post-increment LD1R.
return	SDValue ()
else	if (!isSplatVectorInterleaveOps(DAG, DL, WideValue, ValueInterleaveOps))
	if (NumParts !=2 &&NumParts !=3 &&NumParts !=4) return SDValue()
	if (!TLI.isTypeLegal(SubVecTy)) return SDValue()
	if (IsScalable &&SubBits !=128) return SDValue()
	if (IsScalable)
	if (!IsMasked &&(SubBits==64||SubBits==128) &&Subtarget.isNeonAvailable())
	if (!Subtarget.isSVEorStreamingSVEAvailable()) return SDValue()
	if (IsMasked)
Ops	append ({Chain, DAG.getConstant(SVEStores[NumParts - 2], DL, MVT::i32)})
	for (SDValue V :ValueInterleaveOps) Ops.push_back(convertToScalableVector(DAG
Ops	append ({Pred, BasePtr})
return DAG	getMemIntrinsicNode (ISD::INTRINSIC_VOID, DL, DAG.getVTList(MVT::Other), Ops, MemN->getMemoryVT(), MemN->getMemOperand())
static SDValue	performLegalizedInterleavedStoreCombine (StoreSDNode *ST, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
static SDValue	expandScalableCompressingStore (MaskedStoreSDNode *Store, SelectionDAG &DAG)
	Expand a scalable compressing store to a VECTOR_COMPRESS + a masked store.
static SDValue	performMSTORECombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG, const AArch64Subtarget *Subtarget, const AArch64TargetLowering &TLI)
static bool	foldIndexIntoBase (SDValue &BasePtr, SDValue &Index, SDValue Scale, SDLoc DL, SelectionDAG &DAG)
static bool	findMoreOptimalIndexType (const MaskedGatherScatterSDNode *N, SDValue &BasePtr, SDValue &Index, SelectionDAG &DAG)
static SDValue	performMaskedGatherScatterCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
static SDValue	performNEONPostLDSTCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
	Target-specific DAG combine function for NEON load/store intrinsics to merge base address updates.
static bool	checkValueWidth (SDValue V, unsigned width, ISD::LoadExtType &ExtType)
static bool	isEquivalentMaskless (unsigned CC, unsigned width, ISD::LoadExtType ExtType, int AddConstant, int CompConstant)
static SDValue	performSubsToAndsCombine (SDNode *N, SDNode *SubsNode, SDNode *AndNode, SelectionDAG &DAG, unsigned CCIndex, unsigned CmpIndex, unsigned CC)
static SDValue	performCONDCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG, unsigned CCIndex, unsigned CmpIndex)
static SDValue	performBRCONDCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
static SDValue	foldCSELofCTTZ (SDNode *N, SelectionDAG &DAG)
static SDValue	foldCSELOfCSEL (SDNode *Op, SelectionDAG &DAG)
static SDValue	reassociateCSELOperandsForCSE (SDNode *N, SelectionDAG &DAG)
static SDValue	foldCSELofLASTB (SDNode *Op, SelectionDAG &DAG)
static SDValue	foldCSELOfCSELSameFlags (SDNode *N, SelectionDAG &DAG)
static SDValue	performCSELCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
static SDValue	tryToWidenSetCCOperands (SDNode *Op, SelectionDAG &DAG)
static SDValue	performVecReduceBitwiseCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
static SDValue	performSETCCCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
static SDValue	performSETCCCARRYCombine (SDNode *N, SelectionDAG &DAG)
static SDValue	performSELECT_CCCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
static SDValue	performFlagSettingCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, unsigned GenericOpcode)
static SDValue	performANDSCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SDValue	performSetCCPunpkCombine (SDNode *N, SelectionDAG &DAG)
static bool	isSignExtInReg (const SDValue &V)
static SDValue	performSetccMergeZeroCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SDValue	getTestBitOperand (SDValue Op, unsigned &Bit, bool &Invert, SelectionDAG &DAG)
static SDValue	performTBZCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
static SDValue	trySwapVSelectOperands (SDNode *N, SelectionDAG &DAG)
static SDValue	performVselectPowCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SDValue	matchZeroSelectArm (SDValue TVal, SDValue FVal, ISD::CondCode &RequiredCC)
static SDValue	foldMaskedShiftToUSHL (SelectionDAG &DAG, const AArch64Subtarget *Subtarget, SDNode *N, SDValue X, SDValue Amt, SDValue Cond, ISD::CondCode RequiredCC, bool IsSRL)
static SDValue	performVSelectMaskedShiftCombine (SDNode *N, SelectionDAG &DAG, const AArch64Subtarget *Subtarget)
static SDValue	performVSelectCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const AArch64Subtarget *Subtarget)
static SDValue	performSelectCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
	A vector select: "(select vL, vR, (setcc LHS, RHS))" is best performed with the compare-mask instructions rather than going via NZCV, even if LHS and RHS are really scalar. This replaces any scalar setcc in the above pattern with a vector one followed by a DUP shuffle on the result.
static SDValue	performDUPCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const AArch64Subtarget *Subtarget)
static SDValue	performNVCASTCombine (SDNode *N, SelectionDAG &DAG, const AArch64Subtarget *Subtarget)
	Get rid of unnecessary NVCASTs (that don't change the type). Under LE a NVCAST is converted to a bitcast.
static SDValue	performGlobalAddressCombine (SDNode *N, SelectionDAG &DAG, const AArch64Subtarget *Subtarget, const TargetMachine &TM)
static SDValue	performCTLZCombine (SDNode *N, SelectionDAG &DAG, const AArch64Subtarget *Subtarget)
static SDValue	getScaledOffsetForBitWidth (SelectionDAG &DAG, SDValue Offset, SDLoc DL, unsigned BitWidth)
static bool	isValidImmForSVEVecImmAddrMode (unsigned OffsetInBytes, unsigned ScalarSizeInBytes)
	Check if the value of OffsetInBytes can be used as an immediate for the gather load/prefetch and scatter store instructions with vector base and immediate offset addressing mode:
static bool	isValidImmForSVEVecImmAddrMode (SDValue Offset, unsigned ScalarSizeInBytes)
	Check if the value of Offset represents a valid immediate for the SVE gather load/prefetch and scatter store instructiona with vector base and immediate offset addressing mode:
static SDValue	performScatterStoreCombine (SDNode *N, SelectionDAG &DAG, unsigned Opcode, bool OnlyPackedOffsets=true)
static SDValue	performGatherLoadCombine (SDNode *N, SelectionDAG &DAG, unsigned Opcode, bool OnlyPackedOffsets=true)
static SDValue	performSignExtendInRegCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
static SDValue	legalizeSVEGatherPrefetchOffsVec (SDNode *N, SelectionDAG &DAG)
	Legalize the gather prefetch (scalar + vector addressing mode) when the offset vector is an unpacked 32-bit scalable vector. The other cases (Offset != nxv2i32) do not need legalization.
static SDValue	combineSVEPrefetchVecBaseImmOff (SDNode *N, SelectionDAG &DAG, unsigned ScalarSizeInBytes)
	Combines a node carrying the intrinsic aarch64_sve_prf<T>_gather_scalar_offset into a node that uses aarch64_sve_prfb_gather_uxtw_index when the scalar offset passed to aarch64_sve_prf<T>_gather_scalar_offset is not a valid immediate for the sve gather prefetch instruction with vector plus immediate addressing mode.
static bool	isLanes1toNKnownZero (SDValue Op)
static bool	isLane0KnownActive (SDValue Op)
static SDValue	removeRedundantInsertVectorElt (SDNode *N)
static SDValue	performInsertVectorEltCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SDValue	performFPExtendCombine (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const AArch64Subtarget *Subtarget)
static SDValue	performBSPExpandForSVE (SDNode *N, SelectionDAG &DAG, const AArch64Subtarget *Subtarget)
static SDValue	performDupLane128Combine (SDNode *N, SelectionDAG &DAG)
static SDValue	tryCombineMULLWithUZP1 (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
static SDValue	performMULLCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
static SDValue	performPTestFirstCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
static SDValue	performScalarToVectorCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
static bool	isDeinterleave4ForWideningUToFP (SDNode *N)
static SDValue	performLegalizedVectorDeinterleaveCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
static SDValue	performVectorDeinterleaveCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
static SDValue	performSHLCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
	If the operand is a bitwise AND with a constant RHS, and the shift has a constant RHS and is the only use, we can pull it out of the shift, i.e.
static SDValue	performRNDRCombine (SDNode *N, SelectionDAG &DAG)
static SDValue	performCTPOPCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
static unsigned	getReductionForOpcode (unsigned Op)
static SDValue	trySQABSCombine (SDNode *N, SelectionDAG &DAG, const AArch64TargetLowering &TLI)
static SDValue	performMINMAXCombine (SDNode *N, SelectionDAG &DAG, const AArch64TargetLowering &TLI)
static SDValue	performPredicateLoadCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
static void	replaceBoolVectorBitcast (SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG)
static void	CustomNonLegalBITCASTResults (SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG, EVT ExtendVT, EVT CastVT)
static void	ReplaceAddWithADDP (SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG, const AArch64Subtarget *Subtarget)
static void	ReplaceReductionResults (SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG, unsigned InterOp, unsigned AcrossOp)
static SDValue	createGPRPairNode (SelectionDAG &DAG, SDValue V)
static void	ReplaceCMP_SWAP_128Results (SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG, const AArch64Subtarget *Subtarget)
static unsigned	getAtomicLoad128Opcode (unsigned ISDOpcode, AtomicOrdering Ordering)
static void	ReplaceATOMIC_LOAD_128Results (SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG, const AArch64Subtarget *Subtarget)
static bool	rmwOpMayLowerToLibcall (const AArch64Subtarget &Subtarget, const AtomicRMWInst *RMW)
static Value *	UseTlsOffset (IRBuilderBase &IRB, unsigned Offset)
static std::optional< Intrinsic::ID >	getPairwiseOpForReduction (unsigned Op)
	Returns the pairwise SVE2 op that could be used for a v2<ty> reduction.
static unsigned	getPredicatedReductionOpcode (unsigned Op)
	Returns the corresponding predicated SVE reduction opcode for a VECREDUCE_*.
static SDValue	lowerPredicateInterleaveWithPackedVectors (SDValue Op, SelectionDAG &DAG)
static unsigned	getSVEOpcodeForFPToInt (unsigned Opc)
static SDValue	GenerateFixedLengthSVETBL (SDValue Op, SDValue Op1, SDValue Op2, ArrayRef< int > ShuffleMask, EVT VT, EVT ContainerVT, SelectionDAG &DAG)
static SDValue	emitConditionalComparison (SDValue LHS, SDValue RHS, ISD::CondCode CC, SDValue CCOp, AArch64CC::CondCode Predicate, AArch64CC::CondCode OutCC, const SDLoc &DL, SelectionDAG &DAG)
	can be transformed to: not (and (not (and (setCC (cmp C)) (setCD (cmp D)))) (and (not (setCA (cmp A)) (not (setCB (cmp B))))))" which can be implemented as: cmp C ccmp D, inv(CD), CC ccmp A, CA, inv(CD) ccmp B, CB, inv(CA) check for CB flags
static bool	canEmitConjunction (SelectionDAG &DAG, const SDValue Val, bool &CanNegate, bool &MustBeFirst, bool &PreferFirst, bool WillNegate, unsigned Depth=0)
	Returns true if Val is a tree of AND/OR/SETCC operations that can be expressed as a conjunction.
static SDValue	emitConjunctionRec (SelectionDAG &DAG, SDValue Val, AArch64CC::CondCode &OutCC, bool Negate, SDValue CCOp, AArch64CC::CondCode Predicate)
	Emit conjunction or disjunction tree with the CMP/FCMP followed by a chain of CCMP/CFCMP ops.
static SDValue	emitConjunction (SelectionDAG &DAG, SDValue Val, AArch64CC::CondCode &OutCC)
	Emit expression as a conjunction (a series of CCMP/CFCMP ops).

Variables
constexpr MVT	CondCodeVT = MVT::i32
	Value type used for condition codes.
constexpr MVT	FlagsVT = MVT::i32
	Value type used for NZCV flags.
static const MCPhysReg	GPRArgRegs []
static const MCPhysReg	FPRArgRegs []
	true
	Simplify Addr given that the top byte of it is ignored by HW during / address translation.
unsigned	NumParts = ValueInterleaveOps.size()
EVT	SubVecTy = ValueInterleaveOps[0].getValueType()
const TargetLowering &	TLI = DAG.getTargetLoweringInfo()
bool	IsScalable = SubVecTy.isScalableVector()
unsigned	SubBits = SubVecTy.getSizeInBits().getKnownMinValue()
auto *	MemN = cast<MemSDNode>(N)
const AArch64Subtarget &	Subtarget = DAG.getSubtarget<AArch64Subtarget>()
EVT	ContainerVT = getContainerForFixedLengthVector(DAG, SubVecTy)
SDValue	Pred
	else
static constexpr Intrinsic::ID	SVEStores []
SmallVector< SDValue, 8 >	Ops
	V

Macro Definition Documentation

DEBUG_TYPE

#define DEBUG_TYPE   "aarch64-lower"

Definition at line 111 of file AArch64ISelLowering.cpp.

GET_REGISTER_MATCHER

#define GET_REGISTER_MATCHER

Definition at line 13933 of file AArch64ISelLowering.cpp.

Enumeration Type Documentation

PredicateConstraint

enum class PredicateConstraint

strong

Enumerator
Uph
Upl
Upa

Definition at line 14257 of file AArch64ISelLowering.cpp.

ReducedGprConstraint

enum class ReducedGprConstraint

strong

Enumerator
Uci
Ucj

Definition at line 14320 of file AArch64ISelLowering.cpp.

Function Documentation

analyzeCallOperands()

void analyzeCallOperands ( const AArch64TargetLowering & TLI, const AArch64Subtarget * Subtarget, const TargetLowering::CallLoweringInfo & CLI, CCState & CCInfo )

static

Definition at line 10041 of file AArch64ISelLowering.cpp.

References llvm::CCState::AllocateStack(), llvm::CallingConv::ARM64EC_Thunk_X64, assert(), llvm::TargetLowering::CallLoweringInfo::CallConv, llvm::TargetLowering::CallLoweringInfo::CB, llvm::CCValAssign::Full, llvm::EVT::getSimpleVT(), llvm::CallBase::isMustTailCall(), llvm::EVT::isSimple(), llvm::TargetLowering::CallLoweringInfo::IsVarArg, llvm::ISD::ArgFlagsTy::isVarArg(), llvm::TargetLowering::CallLoweringInfo::Outs, llvm::report_fatal_error(), llvm::SmallVectorTemplateCommon< T, typename >::size(), Subtarget, and TLI.

append() [1/2]

Ops append

(

{Chain, DAG.getConstant(SVEStores[NumParts - 2], DL, MVT::i32)}

)

References AbstractManglingParser< Derived, Alloc >::Ops, DL, llvm::SelectionDAG::getConstant(), NumParts, and SVEStores.

Referenced by llvm::opt::DerivedArgList::AddFlagArg(), llvm::opt::DerivedArgList::AddJoinedArg(), llvm::opt::DerivedArgList::AddPositionalArg(), llvm::opt::DerivedArgList::AddSeparateArg(), llvm::SmallVector< BitWord >::SmallVector(), llvm::SmallVector< BitWord >::SmallVector(), llvm::SmallVector< BitWord >::SmallVector(), llvm::SmallVector< BitWord >::SmallVector(), and llvm::detail::IEEEFloat::toString().

append() [2/2]

Ops append

(

{Pred, BasePtr}

)

References AbstractManglingParser< Derived, Alloc >::Ops, and Pred.

areLoadedOffsetButOtherwiseSame()

bool areLoadedOffsetButOtherwiseSame ( SDValue Op0, SDValue Op1, SelectionDAG & DAG, unsigned & NumSubLoads )

static

Definition at line 24599 of file AArch64ISelLowering.cpp.

References llvm::ISD::ADD, llvm::all_of(), llvm::ISD::ANY_EXTEND, areLoadedOffsetButOtherwiseSame(), llvm::SelectionDAG::areNonVolatileConsecutiveLoads(), llvm::get(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::SDValue::getValueType(), llvm::SDValue::hasOneUse(), isLoadOrMultipleLoads(), llvm::ISD::SIGN_EXTEND, llvm::Size, llvm::SmallVectorTemplateCommon< T, typename >::size(), llvm::ISD::SUB, llvm::ISD::ZERO_EXTEND, and llvm::zip().

Referenced by areLoadedOffsetButOtherwiseSame(), and performExtBinopLoadFold().

calculatePreExtendType()

EVT calculatePreExtendType ( SDValue Extend )

static

Calculates what the pre-extend type is, based on the extension operation node provided by Extend.

In the case that Extend is a SIGN_EXTEND or a ZERO_EXTEND, the pre-extend type is pulled directly from the operand, while other extend operations need a bit more inspection to get this information.

Parameters

Extend

The SDNode from the DAG that represents the extend operation

Returns

The type representing the Extend source type, or MVT::Other if no valid type can be determined

Definition at line 21443 of file AArch64ISelLowering.cpp.

References llvm::ISD::AND, llvm::ISD::ANY_EXTEND, llvm::ISD::AssertSext, llvm::ISD::AssertZext, llvm::dyn_cast(), llvm::SDValue::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValueType(), llvm::VTSDNode::getVT(), llvm::ISD::SIGN_EXTEND, llvm::ISD::SIGN_EXTEND_INREG, uint64_t, and llvm::ISD::ZERO_EXTEND.

Referenced by performBuildShuffleExtendCombine().

callConvSupportsVarArgs()

bool callConvSupportsVarArgs ( CallingConv::ID CC )

static

Return true if the call convention supports varargs Currently only those that pass varargs like the C calling convention does are eligible Calling conventions listed in this function must also be properly handled in AArch64Subtarget::isCallingConvWin64.

Definition at line 10027 of file AArch64ISelLowering.cpp.

References llvm::CallingConv::AArch64_SVE_VectorCall, llvm::CallingConv::C, CC, and llvm::CallingConv::PreserveNone.

canEmitConjunction()

bool canEmitConjunction ( SelectionDAG & DAG, const SDValue Val, bool & CanNegate, bool & MustBeFirst, bool & PreferFirst, bool WillNegate, unsigned Depth = 0 )

static

Returns true if Val is a tree of AND/OR/SETCC operations that can be expressed as a conjunction.

See CMP;CCMP matching.

Parameters

CanNegate	Set to true if we can negate the whole sub-tree just by changing the conditions on the SETCC tests. (this means we can call emitConjunctionRec() with Negate==true on this sub-tree)
MustBeFirst	Set to true if this subtree needs to be negated and we cannot do the negation naturally. We are required to emit the subtree first in this case.
PreferFirst	Set to true if processing this subtree first may result in more efficient code.
WillNegate	Is true if are called when the result of this subexpression must be negated. This happens when the outer expression is an OR. We can use this fact to know that we have a double negation (or (or ...) ...) that can be implemented for free.

Definition at line 4155 of file AArch64ISelLowering.cpp.

References llvm::APInt::abs(), llvm::ISD::AND, assert(), C(), canEmitConjunction(), llvm::Depth, llvm::SelectionDAG::doesNodeExist(), llvm::dyn_cast(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getVTList(), llvm::APInt::getZExtValue(), llvm::SDValue::hasOneUse(), llvm::EVT::isInteger(), llvm::AArch64_AM::isLegalArithImmed(), llvm::O0, llvm::O1, llvm::ISD::OR, llvm::ISD::SETCC, llvm::ISD::SUB, and llvm::APInt::ugt().

Referenced by canEmitConjunction(), canEmitConjunction(), emitConjunction(), and emitConjunctionRec().

canGuaranteeTCO()

bool canGuaranteeTCO ( CallingConv::ID CC, bool GuaranteeTailCalls )

static

Return true if the calling convention is one that we can guarantee TCO for.

Definition at line 9999 of file AArch64ISelLowering.cpp.

References CC, llvm::CallingConv::Fast, llvm::CallingConv::SwiftTail, and llvm::CallingConv::Tail.

Referenced by llvm::AArch64CallLowering::isEligibleForTailCallOptimization(), mayTailCallThisCC(), mayTailCallThisCC(), shouldGuaranteeTCO(), and shouldGuaranteeTCO().

canLowerSRLToRoundingShiftForVT()

bool canLowerSRLToRoundingShiftForVT ( SDValue Shift, EVT ResVT, SelectionDAG & DAG, unsigned & ShiftValue, SDValue & RShOperand )

static

Definition at line 18095 of file AArch64ISelLowering.cpp.

References llvm::ISD::ADD, llvm::Add, assert(), llvm::dyn_cast_or_null(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::SelectionDAG::getSplatValue(), llvm::SDValue::getValueType(), llvm::EVT::isScalableVT(), llvm::ISD::SRL, and uint64_t.

Referenced by trySimplifySrlAddToRshrnb().

carryFlagToValue()

SDValue carryFlagToValue ( SDValue Glue, EVT VT, SelectionDAG & DAG, bool Invert )

static

Definition at line 4768 of file AArch64ISelLowering.cpp.

References assert(), CC, Cond, DL, getCondCode(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getResNo(), llvm::AArch64CC::HS, llvm::AArch64CC::LO, and Zero.

Referenced by LowerADDSUBO_CARRY(), and lowerADDSUBO_CARRY().

changeFPCCToAArch64CC()

void changeFPCCToAArch64CC ( ISD::CondCode CC, AArch64CC::CondCode & CondCode, AArch64CC::CondCode & CondCode2 )

static

changeFPCCToAArch64CC - Convert a DAG fp condition code to an AArch64 CC.

Definition at line 3795 of file AArch64ISelLowering.cpp.

References llvm::AArch64CC::AL, CC, llvm::AArch64CC::EQ, llvm::AArch64CC::GE, llvm::AArch64CC::GT, llvm::AArch64CC::HI, llvm::AArch64CC::LE, llvm_unreachable, llvm::AArch64CC::LS, llvm::AArch64CC::LT, llvm::AArch64CC::MI, llvm::AArch64CC::NE, llvm::AArch64CC::PL, llvm::ISD::SETEQ, llvm::ISD::SETGE, llvm::ISD::SETGT, llvm::ISD::SETLE, llvm::ISD::SETLT, llvm::ISD::SETNE, llvm::ISD::SETO, llvm::ISD::SETOEQ, llvm::ISD::SETOGE, llvm::ISD::SETOGT, llvm::ISD::SETOLE, llvm::ISD::SETOLT, llvm::ISD::SETONE, llvm::ISD::SETUEQ, llvm::ISD::SETUGE, llvm::ISD::SETUGT, llvm::ISD::SETULE, llvm::ISD::SETULT, llvm::ISD::SETUNE, llvm::ISD::SETUO, llvm::AArch64CC::VC, and llvm::AArch64CC::VS.

Referenced by changeFPCCToANDAArch64CC(), and changeVectorFPCCToAArch64CC().

changeFPCCToANDAArch64CC()

void changeFPCCToANDAArch64CC ( ISD::CondCode CC, AArch64CC::CondCode & CondCode, AArch64CC::CondCode & CondCode2 )

static

Convert a DAG fp condition code to an AArch64 CC.

This differs from changeFPCCToAArch64CC in that it returns cond codes that should be AND'ed instead of OR'ed.

Definition at line 3858 of file AArch64ISelLowering.cpp.

References llvm::AArch64CC::AL, assert(), CC, changeFPCCToAArch64CC(), llvm::AArch64CC::LE, llvm::AArch64CC::NE, llvm::AArch64CC::PL, llvm::ISD::SETONE, llvm::ISD::SETUEQ, and llvm::AArch64CC::VC.

Referenced by emitConjunctionRec().

changeIntCCToAArch64CC()

AArch64CC::CondCode changeIntCCToAArch64CC ( ISD::CondCode CC, SDValue RHS = {} )

static

changeIntCCToAArch64CC - Convert a DAG integer condition code to an AArch64 CC

Definition at line 3766 of file AArch64ISelLowering.cpp.

References CC, and RHS.

Referenced by emitConjunctionRec(), and getAArch64Cmp().

changeVectorFPCCToAArch64CC()

void changeVectorFPCCToAArch64CC ( ISD::CondCode CC, AArch64CC::CondCode & CondCode, AArch64CC::CondCode & CondCode2, bool & Invert )

static

changeVectorFPCCToAArch64CC - Convert a DAG fp condition code to an AArch64 CC usable with the vector instructions.

Fewer operations are available without a real NZCV register, so we have to use less efficient combinations to get the same effect.

Definition at line 3888 of file AArch64ISelLowering.cpp.

References llvm::AArch64CC::AL, CC, changeFPCCToAArch64CC(), llvm::AArch64CC::GE, llvm::AArch64CC::LS, llvm::AArch64CC::MI, llvm::ISD::SETLE, llvm::ISD::SETLT, llvm::ISD::SETO, llvm::ISD::SETUEQ, llvm::ISD::SETUGE, llvm::ISD::SETUGT, llvm::ISD::SETULE, llvm::ISD::SETULT, and llvm::ISD::SETUO.

Referenced by emitFloatCompareMask().

checkLaneSlide()

unsigned checkLaneSlide ( ArrayRef< int > Mask, unsigned LaneStart, unsigned LaneElts, unsigned NumElts, bool & IsLeftSlide )

static

Flag slide shuffle patterns where one operand is zeros.

Left slide: shufflevector v, zeros, <1,2,3,...> -> ushr Right slide: shufflevector zeros, v, <N-1,N,N+1,...> -> shl Check if a single 64-bit lane has a valid slide pattern. LaneStart: first element index of this lane in the full vector LaneElts: number of elements in the lane Returns slide amount in elements, or 0 if not a valid slide.

Definition at line 15357 of file AArch64ISelLowering.cpp.

Referenced by isSlideWithZerosMask().

checkValueWidth()

bool checkValueWidth ( SDValue V, unsigned width, ISD::LoadExtType & ExtType )

static

Definition at line 29059 of file AArch64ISelLowering.cpp.

References llvm::ISD::AssertSext, llvm::ISD::AssertZext, llvm::cast(), llvm::ISD::Constant, llvm::LoadSDNode::getExtensionType(), llvm::MemSDNode::getMemoryVT(), llvm::VTSDNode::getVT(), llvm::ISD::LOAD, llvm::ISD::NON_EXTLOAD, llvm::ISD::SEXTLOAD, llvm::ISD::TargetConstant, V, and llvm::ISD::ZEXTLOAD.

Referenced by performCONDCombine(), and performSETCCCombine().

checkZExtBool()

bool checkZExtBool ( SDValue Arg, const SelectionDAG & DAG )

static

Definition at line 10315 of file AArch64ISelLowering.cpp.

References llvm::SelectionDAG::computeKnownBits(), llvm::EVT::getSizeInBits(), and llvm::SDValue::getValueType().

combineAcrossLanesIntrinsic()

SDValue combineAcrossLanesIntrinsic ( unsigned Opc, SDNode * N, SelectionDAG & DAG )

static

Definition at line 25173 of file AArch64ISelLowering.cpp.

References DL, llvm::SelectionDAG::getExtractVectorElt(), llvm::SelectionDAG::getNode(), N, and Opc.

Referenced by performIntrinsicCombine().

combineSVEBitSel()

SDValue combineSVEBitSel ( unsigned IID, SDNode * N, SelectionDAG & DAG )

static

Definition at line 25432 of file AArch64ISelLowering.cpp.

References DL, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNOT(), llvm_unreachable, and N.

Referenced by performIntrinsicCombine().

combineSVEPrefetchVecBaseImmOff()

SDValue combineSVEPrefetchVecBaseImmOff ( SDNode * N, SelectionDAG & DAG, unsigned ScalarSizeInBytes )

static

Combines a node carrying the intrinsic aarch64_sve_prf<T>_gather_scalar_offset into a node that uses aarch64_sve_prfb_gather_uxtw_index when the scalar offset passed to aarch64_sve_prf<T>_gather_scalar_offset is not a valid immediate for the sve gather prefetch instruction with vector plus immediate addressing mode.

Definition at line 31347 of file AArch64ISelLowering.cpp.

References AbstractManglingParser< Derived, Alloc >::Ops, DL, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getVTList(), isValidImmForSVEVecImmAddrMode(), N, SDValue(), and std::swap().

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

combineSVEReductionFP()

SDValue combineSVEReductionFP ( SDNode * N, unsigned Opc, SelectionDAG & DAG )

static

Definition at line 25362 of file AArch64ISelLowering.cpp.

References DL, llvm::SelectionDAG::getExtractVectorElt(), llvm::SelectionDAG::getNode(), llvm::SDValue::getValueType(), N, Opc, and Pred.

Referenced by performIntrinsicCombine().

combineSVEReductionInt()

SDValue combineSVEReductionInt ( SDNode * N, unsigned Opc, SelectionDAG & DAG )

static

Definition at line 25345 of file AArch64ISelLowering.cpp.

References DL, llvm::SelectionDAG::getExtractVectorElt(), llvm::SelectionDAG::getNode(), getPackedSVEVectorVT(), N, Opc, and Pred.

Referenced by performIntrinsicCombine().

combineSVEReductionOrderedFP()

SDValue combineSVEReductionOrderedFP ( SDNode * N, unsigned Opc, SelectionDAG & DAG )

static

Definition at line 25377 of file AArch64ISelLowering.cpp.

References DL, llvm::SelectionDAG::getExtractVectorElt(), llvm::SelectionDAG::getInsertVectorElt(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getPOISON(), llvm::SDValue::getValueType(), N, Opc, and Pred.

Referenced by performIntrinsicCombine().

combineToExtendBoolVectorInReg()

SDValue combineToExtendBoolVectorInReg ( unsigned Opcode, const SDLoc & DL, EVT VT, SDValue N0, SelectionDAG & DAG, TargetLowering::DAGCombinerInfo & DCI, const AArch64Subtarget & Subtarget )

static

Definition at line 26258 of file AArch64ISelLowering.cpp.

References llvm::ISD::AND, llvm::ISD::ANY_EXTEND, assert(), llvm::ISD::BITCAST, DL, llvm::SelectionDAG::getAnyExtOrTrunc(), llvm::SelectionDAG::getBitcast(), llvm::APInt::getBitsSet(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarType(), llvm::SelectionDAG::getSetCC(), llvm::SelectionDAG::getSExtOrTrunc(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getSplat(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::EVT::getVectorVT(), I, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::DataLayout::isBigEndian(), llvm::EVT::isScalarInteger(), llvm::EVT::isVector(), llvm::ISD::SCALAR_TO_VECTOR, SDValue(), llvm::ISD::SETEQ, llvm::ISD::SIGN_EXTEND, llvm::ISD::SRL, Subtarget, and llvm::ISD::ZERO_EXTEND.

Referenced by canonicalizeBoolMask(), combineSelect(), combineSext(), combineZext(), performExtendCombine(), performExtendToBoolVectorLoadCombine(), and performVSelectCombine().

ConstantBuildVector()

SDValue ConstantBuildVector ( SDValue Op, SelectionDAG & DAG, const AArch64Subtarget * ST )

static

Definition at line 16958 of file AArch64ISelLowering.cpp.

References assert(), llvm::cast(), DL, llvm::ISD::FNEG, llvm::APInt::getHighBitsSet(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::MVT::getVectorVT(), resolveBuildVector(), SDValue(), tryAdvSIMDModImm16(), tryAdvSIMDModImm32(), tryAdvSIMDModImm321s(), tryAdvSIMDModImm64(), tryAdvSIMDModImm8(), tryAdvSIMDModImmFP(), trySVESplat64(), and llvm::APInt::zext().

constructDup()

SDValue constructDup ( SDValue V, int Lane, SDLoc DL, EVT VT, unsigned Opcode, SelectionDAG & DAG )

static

Definition at line 15707 of file AArch64ISelLowering.cpp.

References llvm::ISD::BITCAST, llvm::ISD::CONCAT_VECTORS, DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getConstantOperandVal(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::MVT::getScalarType(), llvm::SDValue::getScalarValueSizeInBits(), llvm::SDValue::getSimpleValueType(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::EVT::is128BitVector(), V, and WidenVector().

convertFixedMaskToScalableVector()

SDValue convertFixedMaskToScalableVector ( SDValue Mask, SelectionDAG & DAG )

static

Definition at line 34714 of file AArch64ISelLowering.cpp.

References CC, ContainerVT, convertToScalableVector(), DL, llvm::SelectionDAG::getConstant(), getContainerForFixedLengthVector(), llvm::SelectionDAG::getNode(), getPredicateForFixedLengthVector(), llvm::SDValue::getValueType(), llvm::isBitwiseNot(), llvm::ISD::isBuildVectorAllOnes(), llvm::ISD::SETCC, and llvm::ISD::SETNE.

Referenced by if(), llvm::AArch64TargetLowering::LowerOperation(), LowerVectorMatch(), and performExtractLastActiveCombine().

convertFromScalableVector()

SDValue convertFromScalableVector ( SelectionDAG & DAG, EVT VT, SDValue V )

static

Definition at line 34666 of file AArch64ISelLowering.cpp.

References assert(), llvm::SelectionDAG::getExtractSubvector(), llvm::EVT::isFixedLengthVector(), and V.

Referenced by convertLocVTToValVT(), GenerateFixedLengthSVETBL(), getVCIXISDNodeWCHAIN(), getWideningInterleave(), llvm::RISCVTargetLowering::joinRegisterPartsIntoValue(), lowerBUILD_VECTOR(), lowerBuildVectorOfConstants(), lowerBuildVectorViaDominantValues(), lowerBuildVectorViaVID(), lowerCONVERT_FROM_ARBITRARY_FP(), lowerFixedVectorSegLoadIntrinsics(), lowerFMAXIMUM_FMINIMUM(), lowerFP_TO_INT_SAT(), llvm::RISCVTargetLowering::LowerOperation(), lowerVECTOR_SHUFFLEAsVSlide1(), lowerVECTOR_SHUFFLEAsVSlidedown(), lowerVECTOR_SHUFFLEAsVSlideup(), lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND(), LowerVectorMatch(), lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND(), lowerVectorXRINT_XROUND(), lowerZvzipVPAIR(), lowerZvzipVUNZIP(), lowerZvzipVZIP(), matchSplatAsGather(), performDUPCombine(), performFP_TO_INTCombine(), performLegalizedVectorDeinterleaveCombine(), and performSVEMulAddSubCombine().

convertMergedOpToPredOp()

SDValue convertMergedOpToPredOp ( SDNode * N, unsigned Opc, SelectionDAG & DAG, bool UnpredOp = false, bool SwapOperands = false )

static

Definition at line 25411 of file AArch64ISelLowering.cpp.

References assert(), llvm::SelectionDAG::getNode(), llvm::ISD::INTRINSIC_WO_CHAIN, isAllActivePredicate(), N, Opc, and SDValue().

Referenced by performIntrinsicCombine().

convertMulToShlAdd()

SDValue convertMulToShlAdd ( SDNode * N, SelectionDAG & DAG )

static

Definition at line 6143 of file AArch64ISelLowering.cpp.

References llvm::APInt::abs(), llvm::ISD::ADD, llvm::APInt::countr_zero(), llvm::ISD::DELETED_NODE, DL, llvm::APInt::getBoolValue(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNegative(), llvm::SelectionDAG::getNode(), llvm::ISD::isConstantSplatVector(), llvm::APInt::isNegative(), LHS, llvm::APInt::lshrInPlace(), N, SDValue(), llvm::ISD::SHL, and llvm::ISD::SUB.

convertToScalableVector()

SDValue convertToScalableVector ( SelectionDAG & DAG, EVT VT, SDValue V )

static

Definition at line 34657 of file AArch64ISelLowering.cpp.

References assert(), llvm::SelectionDAG::getInsertSubvector(), llvm::SelectionDAG::getPOISON(), llvm::EVT::isScalableVector(), and V.

Referenced by combineToVCPOP(), convertFixedMaskToScalableVector(), convertValVTToLocVT(), GenerateFixedLengthSVETBL(), getWideningInterleave(), lowerBUILD_VECTOR(), lowerBuildVectorOfConstants(), lowerBuildVectorViaDominantValues(), lowerCONVERT_FROM_ARBITRARY_FP(), lowerCttzElts(), lowerFixedVectorSegLoadIntrinsics(), lowerFixedVectorSegStoreIntrinsics(), lowerFMAXIMUM_FMINIMUM(), lowerFP_TO_INT_SAT(), llvm::RISCVTargetLowering::LowerOperation(), lowerScalarInsert(), lowerShuffleViaVRegSplitting(), lowerVECTOR_SHUFFLEAsVSlide1(), lowerVECTOR_SHUFFLEAsVSlidedown(), lowerVECTOR_SHUFFLEAsVSlideup(), lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND(), LowerVectorMatch(), lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND(), lowerVectorXRINT_XROUND(), lowerZvzipVPAIR(), lowerZvzipVUNZIP(), lowerZvzipVZIP(), matchSplatAsGather(), llvm::RISCVTargetLowering::PerformDAGCombine(), performExtractLastActiveCombine(), performFP_TO_INTCombine(), performLegalizedInterleavedStoreCombine(), performSVEMulAddSubCombine(), processVCIXOperands(), llvm::RISCVTargetLowering::ReplaceNodeResults(), and llvm::RISCVTargetLowering::splitValueIntoRegisterParts().

createGPRPairNode()

SDValue createGPRPairNode ( SelectionDAG & DAG, SDValue V )

static

Definition at line 33061 of file AArch64ISelLowering.cpp.

References AbstractManglingParser< Derived, Alloc >::Ops, DL, llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getMachineNode(), llvm::SelectionDAG::getTargetConstant(), llvm::DataLayout::isBigEndian(), SDValue(), llvm::SelectionDAG::SplitScalar(), std::swap(), and V.

Referenced by ReplaceCMP_SWAP_128Results().

createTblForTrunc()

void createTblForTrunc ( TruncInst * TI, bool IsLittleEndian )

static

Definition at line 19090 of file AArch64ISelLowering.cpp.

References assert(), llvm::SmallVectorTemplateCommon< T, typename >::back(), llvm::SmallVectorTemplateCommon< T, typename >::begin(), llvm::cast(), llvm::SmallVectorImpl< T >::clear(), llvm::SmallVectorTemplateCommon< T, typename >::empty(), llvm::SmallVectorTemplateCommon< T, typename >::end(), llvm::Instruction::eraseFromParent(), llvm::ConstantVector::get(), llvm::FixedVectorType::get(), llvm::User::getOperand(), llvm::Value::getType(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::Value::replaceAllUsesWith(), Results, and llvm::SmallVectorTemplateCommon< T, typename >::size().

Referenced by llvm::AArch64TargetLowering::optimizeExtendOrTruncateConversion().

createTblShuffleForSExt()

Value * createTblShuffleForSExt ( IRBuilderBase & Builder, Value * Op, FixedVectorType * DstTy, bool IsLittleEndian )

static

Definition at line 19072 of file AArch64ISelLowering.cpp.

References llvm::cast(), createTblShuffleMask(), llvm::PoisonValue::get(), llvm::VectorType::getElementType(), and uint64_t.

Referenced by llvm::AArch64TargetLowering::optimizeExtendOrTruncateConversion().

createTblShuffleForZExt()

Value * createTblShuffleForZExt ( IRBuilderBase & Builder, Value * Op, FixedVectorType * ZExtTy, FixedVectorType * DstTy, bool IsLittleEndian )

static

Definition at line 19050 of file AArch64ISelLowering.cpp.

References llvm::cast(), createTblShuffleMask(), llvm::PoisonValue::get(), llvm::VectorType::getElementType(), and uint64_t.

Referenced by llvm::AArch64TargetLowering::optimizeExtendOrTruncateConversion().

createTblShuffleMask()

bool createTblShuffleMask ( unsigned SrcWidth, unsigned DstWidth, unsigned NumElts, bool IsLittleEndian, SmallVectorImpl< int > & Mask )

static

Definition at line 19027 of file AArch64ISelLowering.cpp.

References assert(), and I.

Referenced by createTblShuffleForSExt(), and createTblShuffleForZExt().

CustomNonLegalBITCASTResults()

void CustomNonLegalBITCASTResults ( SDNode * N, SmallVectorImpl< SDValue > & Results, SelectionDAG & DAG, EVT ExtendVT, EVT CastVT )

static

Definition at line 32844 of file AArch64ISelLowering.cpp.

References llvm::ISD::BITCAST, DL, llvm::SelectionDAG::getExtractSubvector(), llvm::SelectionDAG::getNode(), N, Results, and llvm::ISD::SCALAR_TO_VECTOR.

DoCrossByteCLMUL()

SDValue DoCrossByteCLMUL ( SDValue OpA, SDValue OpB, SDLoc DL, SelectionDAG & DAG, EVT VT )

static

Definition at line 8592 of file AArch64ISelLowering.cpp.

References llvm::ISD::CLMUL, DL, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetConstant(), and llvm::ISD::INTRINSIC_WO_CHAIN.

emitComparison()

SDValue emitComparison ( SDValue LHS, SDValue RHS, ISD::CondCode CC, const SDLoc & DL, SelectionDAG & DAG )

static

Definition at line 3993 of file AArch64ISelLowering.cpp.

References llvm::ISD::AND, assert(), CC, DL, FlagsVT, llvm::ISD::FP_EXTEND, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getSubtarget(), llvm::SDValue::getValue(), llvm::SelectionDAG::getVTList(), isCMN(), llvm::EVT::isFloatingPoint(), llvm::isNullConstant(), LHS, llvm::SelectionDAG::ReplaceAllUsesWith(), RHS, and llvm::ISD::SUB.

Referenced by emitConjunctionRec(), and getAArch64Cmp().

emitConditionalComparison()

SDValue emitConditionalComparison ( SDValue LHS, SDValue RHS, ISD::CondCode CC, SDValue CCOp, AArch64CC::CondCode Predicate, AArch64CC::CondCode OutCC, const SDLoc & DL, SelectionDAG & DAG )

static

can be transformed to: not (and (not (and (setCC (cmp C)) (setCD (cmp D)))) (and (not (setCA (cmp A)) (not (setCB (cmp B))))))" which can be implemented as: cmp C ccmp D, inv(CD), CC ccmp A, CA, inv(CD) ccmp B, CB, inv(CA) check for CB flags

A counterexample is "or (and A B) (and C D)" which translates to not (and (not (and (not A) (not B))) (not (and (not C) (not D)))), we can only implement 1 of the inner (not) operations, but not both! Create a conditional comparison; Use CCMP, CCMN or FCCMP as appropriate.

Definition at line 4097 of file AArch64ISelLowering.cpp.

References assert(), CC, DL, llvm::dyn_cast(), FlagsVT, llvm::ISD::FP_EXTEND, getCondCode(), llvm::SelectionDAG::getConstant(), llvm::AArch64CC::getInvertedCondCode(), llvm::SelectionDAG::getNode(), llvm::AArch64CC::getNZCVToSatisfyCondCode(), llvm::SelectionDAG::getSubtarget(), Imm, isCMN(), llvm::isNullConstant(), LHS, RHS, and llvm::ISD::SUB.

Referenced by emitConjunctionRec().

emitConjunction()

SDValue emitConjunction ( SelectionDAG & DAG, SDValue Val, AArch64CC::CondCode & OutCC )

static

Emit expression as a conjunction (a series of CCMP/CFCMP ops).

In some cases this is even possible with OR operations in the expression. See CMP;CCMP matching.

See also

emitConjunctionRec().

Definition at line 4358 of file AArch64ISelLowering.cpp.

References llvm::AArch64CC::AL, canEmitConjunction(), emitConjunctionRec(), and SDValue().

Referenced by getAArch64Cmp(), LowerBRCOND(), and performANDSETCCCombine().

emitConjunctionRec()

SDValue emitConjunctionRec ( SelectionDAG & DAG, SDValue Val, AArch64CC::CondCode & OutCC, bool Negate, SDValue CCOp, AArch64CC::CondCode Predicate )

static

Emit conjunction or disjunction tree with the CMP/FCMP followed by a chain of CCMP/CFCMP ops.

See CMP;CCMP matching. Tries to transform the given i1 producing node Val to a series compare and conditional compare operations.

Returns

an NZCV flags producing node and sets OutCC to the flags that should be tested or returns SDValue() if transformation was not possible. Negate is true if we want this sub-tree being negated just by changing SETCC conditions.

Definition at line 4240 of file AArch64ISelLowering.cpp.

References llvm::AArch64CC::AL, llvm::ISD::AND, assert(), canEmitConjunction(), CC, changeFPCCToANDAArch64CC(), changeIntCCToAArch64CC(), DL, emitComparison(), emitConditionalComparison(), emitConjunctionRec(), llvm::AArch64CC::getInvertedCondCode(), llvm::SDValue::getNode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::SDNode::getOperandCondCode(), llvm::SDNode::hasOneUse(), LHS, llvm::ISD::OR, RHS, llvm::ISD::SETCC, and std::swap().

Referenced by emitConjunction(), and emitConjunctionRec().

emitFloatCompareMask()

SDValue emitFloatCompareMask ( SDValue LHS, SDValue RHS, SDValue TVal, SDValue FVal, ISD::CondCode CC, bool NoNaNs, const SDLoc & DL, SelectionDAG & DAG )

static

For SELECT_CC, when the true/false values are (-1, 0) and the compared values are scalars, try to emit a mask generating vector instruction.

Definition at line 13053 of file AArch64ISelLowering.cpp.

References llvm::AArch64CC::AL, assert(), CC, changeVectorFPCCToAArch64CC(), DL, llvm::dyn_cast(), emitVectorComparison(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getExtractVectorElt(), llvm::SelectionDAG::getInsertVectorElt(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNOT(), llvm::SelectionDAG::getPOISON(), llvm::ISD::getSetCCInverse(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorVT(), llvm::ConstantSDNode::isAllOnes(), llvm::SelectionDAG::isKnownNeverNaN(), llvm::ConstantSDNode::isZero(), LHS, llvm::ISD::OR, RHS, llvm::ISD::SETO, llvm::ISD::SETOEQ, llvm::ISD::SETUNE, and llvm::ISD::SETUO.

emitStrictFPComparison()

SDValue emitStrictFPComparison ( SDValue LHS, SDValue RHS, const SDLoc & DL, SelectionDAG & DAG, SDValue Chain, bool IsSignaling )

static

Definition at line 3973 of file AArch64ISelLowering.cpp.

References assert(), DL, FlagsVT, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getSubtarget(), LHS, RHS, and llvm::ISD::STRICT_FP_EXTEND.

emitVectorComparison()

SDValue emitVectorComparison ( SDValue LHS, SDValue RHS, AArch64CC::CondCode CC, bool NoNans, EVT VT, const SDLoc & DL, SelectionDAG & DAG )

static

Emit vector comparison for floating-point values, producing a mask.

Definition at line 13005 of file AArch64ISelLowering.cpp.

References assert(), CC, DL, llvm::AArch64CC::EQ, llvm::AArch64CC::GE, llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getExtractVectorElt(), llvm::SelectionDAG::getInsertVectorElt(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNOT(), llvm::SelectionDAG::getPOISON(), llvm::EVT::getSizeInBits(), llvm::EVT::getVectorVT(), llvm::AArch64CC::GT, llvm::AArch64CC::LE, LHS, llvm::AArch64CC::LS, llvm::AArch64CC::LT, llvm::AArch64CC::MI, llvm::AArch64CC::NE, RHS, and SDValue().

Referenced by emitFloatCompareMask().

expandScalableCompressingStore()

SDValue expandScalableCompressingStore ( MaskedStoreSDNode * Store, SelectionDAG & DAG )

static

Expand a scalable compressing store to a VECTOR_COMPRESS + a masked store.

Definition at line 28660 of file AArch64ISelLowering.cpp.

References assert(), DL, llvm::ISD::GET_ACTIVE_LANE_MASK, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getMaskedStore(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getPOISON(), llvm::SelectionDAG::getTargetConstant(), llvm::ISD::INTRINSIC_WO_CHAIN, llvm::EVT::isScalableVector(), llvm::Store, llvm::ISD::VECTOR_COMPRESS, and Zero.

Referenced by performMSTORECombine().

extractPtrauthBlendDiscriminators()

std::tuple< SDValue, SDValue > extractPtrauthBlendDiscriminators ( SDValue Disc, SelectionDAG * DAG )

static

Definition at line 342 of file AArch64ISelLowering.cpp.

References DL, llvm::dyn_cast(), llvm::SDNode::getConstantOperandVal(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getRegister(), llvm::SelectionDAG::getTargetConstant(), llvm::ISD::INTRINSIC_WO_CHAIN, llvm::isUInt(), and Register.

findMoreOptimalIndexType()

bool findMoreOptimalIndexType ( const MaskedGatherScatterSDNode * N, SDValue & BasePtr, SDValue & Index, SelectionDAG & DAG )

static

Definition at line 28809 of file AArch64ISelLowering.cpp.

References llvm::cast(), Changed, llvm::EVT::changeVectorElementType(), foldIndexIntoBase(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), llvm::SelectionDAG::getStepVector(), llvm::SelectionDAG::getSubtarget(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorMinNumElements(), llvm::EVT::isFixedLengthVector(), llvm::ISD::isVectorShrinkable(), N, llvm::ISD::STEP_VECTOR, Subtarget, llvm::AArch64::SVEBitsPerBlock, and llvm::ISD::TRUNCATE.

Referenced by performMaskedGatherScatterCombine().

foldADCToCINC()

SDValue foldADCToCINC ( SDNode * N, SelectionDAG & DAG )

static

Definition at line 23960 of file AArch64ISelLowering.cpp.

References CC, Cond, DL, getCondCode(), llvm::SelectionDAG::getNode(), llvm::isNullConstant(), LHS, llvm::AArch64CC::LO, N, RHS, and SDValue().

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

foldAdjacentReversesIntoTbls()

bool foldAdjacentReversesIntoTbls ( IntrinsicInst * Rev, Loop * L )

static

Definition at line 19453 of file AArch64ISelLowering.cpp.

References assert(), llvm::cast(), llvm::SmallVectorTemplateCommon< T, typename >::empty(), llvm::Instruction::eraseFromParent(), llvm::APInt::getAllOnes(), llvm::Function::getDataLayout(), llvm::Instruction::getFunction(), llvm::VectorType::getInteger(), llvm::IntrinsicInst::getIntrinsicID(), llvm::User::getOperand(), llvm::Value::getSingleUndroppableUse(), llvm::BasicBlock::getTerminator(), llvm::Value::getType(), llvm::Invalid, llvm::DataLayout::isLittleEndian(), llvm::PatternMatch::m_c_BinOp(), m_DeinterleavingTbl(), llvm::PatternMatch::m_Intrinsic(), llvm::MIPatternMatch::m_OneUse(), llvm::PatternMatch::m_Specific(), llvm::PatternMatch::match(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), and llvm::Value::uses().

Referenced by llvm::AArch64TargetLowering::optimizeVectorCrossLaneOperation().

foldCSELOfCSEL()

SDValue foldCSELOfCSEL ( SDNode * Op, SelectionDAG & DAG )

static

Definition at line 29517 of file AArch64ISelLowering.cpp.

References llvm::cast(), CC, Cond, DL, llvm::AArch64CC::EQ, llvm::ConstantSDNode::getAPIntValue(), getCondCode(), llvm::SDNode::getConstantOperandVal(), llvm::AArch64CC::getInvertedCondCode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::SDValue::getOperand(), llvm::isa(), isCMP(), llvm::AArch64CC::NE, SDValue(), std::swap(), X, and Y.

Referenced by performCSELCombine().

foldCSELOfCSELSameFlags()

SDValue foldCSELOfCSELSameFlags ( SDNode * N, SelectionDAG & DAG )

static

Definition at line 29755 of file AArch64ISelLowering.cpp.

References AbstractManglingParser< Derived, Alloc >::Ops, llvm::ISD::ADD, llvm::AArch64CC::AL, llvm::any_of(), CC, llvm::SDNode::getConstantOperandVal(), llvm::SDNode::getFlags(), llvm::AArch64CC::getInvertedCondCode(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getNumOperands(), llvm::SDNode::getNumValues(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValueType(), llvm::SDValue::hasOneUse(), llvm::isAllOnesConstant(), llvm::isNullConstant(), llvm::isOneConstant(), llvm::SelectionDAG::isSafeToSpeculativelyExecuteNode(), N, llvm::AArch64CC::NV, llvm::SDNode::op_values(), Opc, SDValue(), llvm::ISD::SUB, and llvm::ISD::XOR.

Referenced by performCSELCombine().

foldCSELofCTTZ()

SDValue foldCSELofCTTZ ( SDNode * N, SelectionDAG & DAG )

static

Definition at line 29469 of file AArch64ISelLowering.cpp.

References llvm::ISD::AND, assert(), llvm::BitWidth, CC, llvm::ISD::CTTZ, llvm::AArch64CC::EQ, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::isNullConstant(), N, llvm::AArch64CC::NE, SDValue(), llvm::ISD::TRUNCATE, X, and Zero.

Referenced by performCSELCombine().

foldCSELofLASTB()

SDValue foldCSELofLASTB ( SDNode * Op, SelectionDAG & DAG )

static

Definition at line 29712 of file AArch64ISelLowering.cpp.

References Default, llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), isAllActivePredicate(), llvm::AArch64CC::NE, and SDValue().

Referenced by performCSELCombine().

foldIndexIntoBase()

bool foldIndexIntoBase ( SDValue & BasePtr, SDValue & Index, SDValue Scale, SDLoc DL, SelectionDAG & DAG )

static

Returns

true if part of the index was folded into the Base.

Definition at line 28759 of file AArch64ISelLowering.cpp.

References llvm::ISD::ADD, llvm::Add, DL, llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getSplatValue(), llvm::EVT::isVectorOf(), llvm::ISD::MUL, llvm::Offset, and llvm::ISD::SHL.

Referenced by findMoreOptimalIndexType().

foldMaskedShiftToUSHL()

SDValue foldMaskedShiftToUSHL ( SelectionDAG & DAG, const AArch64Subtarget * Subtarget, SDNode * N, SDValue X, SDValue Amt, SDValue Cond, ISD::CondCode RequiredCC, bool IsSRL )

static

Definition at line 30502 of file AArch64ISelLowering.cpp.

References llvm::SelectionDAG::computeKnownBits(), Cond, DL, llvm::SelectionDAG::getConstant(), llvm::KnownBits::getMaxValue(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::ISD::INTRINSIC_WO_CHAIN, llvm::EVT::is128BitVector(), llvm::EVT::is64BitVector(), llvm::EVT::isFixedLengthVector(), llvm::EVT::isInteger(), llvm::TargetLoweringBase::isTypeLegal(), llvm::PatternMatch::m_Specific(), llvm::PatternMatch::m_SpecificInt(), llvm::SDPatternMatch::m_SpecificSetCC(), N, llvm::SDPatternMatch::sd_match(), SDValue(), llvm::ISD::SUB, Subtarget, llvm::APInt::ule(), llvm::ISD::UMIN, and X.

Referenced by performVSelectMaskedShiftCombine().

foldOverflowCheck()

SDValue foldOverflowCheck ( SDNode * Op, SelectionDAG & DAG, bool IsAdd )

static

Definition at line 23936 of file AArch64ISelLowering.cpp.

References CC, getCSETCondCode(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::SDValue::getOperand(), llvm::AArch64CC::HS, isCMP(), llvm::isNullConstant(), llvm::isOneConstant(), llvm::AArch64CC::LO, and SDValue().

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

foldVectorXorShiftIntoCmp()

SDValue foldVectorXorShiftIntoCmp ( SDNode * N, SelectionDAG & DAG, const AArch64Subtarget * Subtarget )

static

Turn vector tests of the signbit in the form of: xor (sra X, elt_size(X)-1), -1 into: cmge X, X, #0.

Definition at line 20789 of file AArch64ISelLowering.cpp.

References DL, llvm::dyn_cast(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getSetCC(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::SDValue::hasOneUse(), llvm::ISD::isBuildVectorAllOnes(), llvm::EVT::isVector(), N, SDValue(), llvm::ISD::SETGE, Subtarget, and Zero.

Referenced by combineXor(), and performXorCombine().

for()

for

(

SDValue V :ValueInterleaveOps

)

References V.

GenerateFixedLengthSVETBL()

SDValue GenerateFixedLengthSVETBL ( SDValue Op, SDValue Op1, SDValue Op2, ArrayRef< int > ShuffleMask, EVT VT, EVT ContainerVT, SelectionDAG & DAG )

static

Definition at line 36255 of file AArch64ISelLowering.cpp.

References llvm::ISD::ADD, llvm::ArrayRef(), assert(), llvm::ISD::BITCAST, llvm::EVT::changeTypeToInteger(), ContainerVT, convertFromScalableVector(), convertToScalableVector(), llvm::SmallVectorTemplateCommon< T, typename >::data(), DL, llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), getContainerForFixedLengthVector(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getSubtarget(), llvm::SelectionDAG::getTargetConstant(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::SelectionDAG::getVScale(), llvm::ISD::INTRINSIC_WO_CHAIN, llvm::ShuffleVectorInst::isSingleSourceMask(), llvm::maxUIntN(), llvm::ISD::MUL, llvm::SmallVectorTemplateBase< T, bool >::push_back(), SDValue(), llvm::ISD::SPLAT_VECTOR, Subtarget, llvm::AArch64::SVEBitsPerBlock, and uint64_t.

GenerateTBL()

SDValue GenerateTBL ( SDValue Op, ArrayRef< int > ShuffleMask, SelectionDAG & DAG )

static

Definition at line 15618 of file AArch64ISelLowering.cpp.

References llvm::ISD::BITCAST, llvm::ISD::CONCAT_VECTORS, llvm::SmallVectorTemplateCommon< T, typename >::data(), DL, llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetConstant(), llvm::ISD::INTRINSIC_WO_CHAIN, llvm::SDValue::isUndef(), isZerosVector(), llvm::Offset, llvm::SmallVectorTemplateBase< T, bool >::push_back(), Swap, std::swap(), and llvm::V1.

getAArch64Cmp()

SDValue getAArch64Cmp ( SDValue LHS, SDValue RHS, ISD::CondCode CC, SDValue & AArch64cc, SelectionDAG & DAG, const SDLoc & DL )

static

Definition at line 4436 of file AArch64ISelLowering.cpp.

References assert(), C(), llvm::cast(), CC, changeIntCCToAArch64CC(), DL, llvm::dyn_cast(), emitComparison(), emitConjunction(), getCmpOperandFoldingProfit(), getCmpOrCmnOperandFoldingProfit(), getCondCode(), llvm::SelectionDAG::getConstant(), llvm::AArch64CC::getInvertedCondCode(), llvm::SelectionDAG::getNode(), llvm::ISD::getSetCCSwappedOperands(), llvm::SelectionDAG::getSignedConstant(), llvm::SelectionDAG::getValueType(), llvm::ConstantSDNode::getZExtValue(), llvm::isa(), llvm::AArch64_AM::isLegalArithImmed(), llvm::AArch64_AM::isLegalCmpImmed(), llvm::ConstantSDNode::isOne(), llvm::ConstantSDNode::isZero(), LHS, numberOfInstrToLoadImm(), RHS, llvm::ISD::SETGE, llvm::ISD::SETGT, llvm::ISD::SETLE, llvm::ISD::SETLT, llvm::ISD::SETNE, llvm::ISD::SETUGE, llvm::ISD::SETUGT, llvm::ISD::SETULE, llvm::ISD::SETULT, shouldBeAdjustedToZero(), llvm::ISD::SIGN_EXTEND_INREG, std::swap(), and llvm::ISD::ZEXTLOAD.

Referenced by performSetccAddFolding().

getAArch64XALUOOp()

std::pair< SDValue, SDValue > getAArch64XALUOOp ( AArch64CC::CondCode & CC, SDValue Op, SelectionDAG & DAG )

static

Definition at line 4580 of file AArch64ISelLowering.cpp.

References assert(), CC, DL, FlagsVT, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getValue(), llvm::SelectionDAG::getVTList(), llvm::AArch64CC::HS, LHS, llvm_unreachable, llvm::AArch64CC::LO, llvm::ISD::MUL, Mul, llvm::ISD::MULHS, llvm::ISD::MULHU, llvm::AArch64CC::NE, Opc, RHS, llvm::ISD::SADDO, llvm::ISD::SIGN_EXTEND, llvm::ISD::SMULO, llvm::ISD::SRA, llvm::ISD::SSUBO, llvm::ISD::TRUNCATE, llvm::ISD::UADDO, llvm::ISD::UMULO, llvm::ISD::USUBO, llvm::AArch64CC::VS, and llvm::ISD::ZERO_EXTEND.

Referenced by LowerXALUO().

getAtomicLoad128Opcode()

unsigned getAtomicLoad128Opcode ( unsigned ISDOpcode, AtomicOrdering Ordering )

static

Definition at line 33163 of file AArch64ISelLowering.cpp.

References llvm::Acquire, llvm::AcquireRelease, assert(), llvm::ISD::ATOMIC_LOAD_ADD, llvm::ISD::ATOMIC_LOAD_AND, llvm::ISD::ATOMIC_LOAD_CLR, llvm::ISD::ATOMIC_LOAD_OR, llvm::ISD::ATOMIC_LOAD_SUB, llvm::ISD::ATOMIC_SWAP, llvm_unreachable, llvm::Monotonic, llvm::Release, and llvm::SequentiallyConsistent.

Referenced by ReplaceATOMIC_LOAD_128Results().

getCmpOperandFoldingProfit()

unsigned getCmpOperandFoldingProfit ( SDValue Op, bool AllowExtend )

static

Returns how profitable it is to fold a comparison's operand's shift and/or extension operations.

Definition at line 4386 of file AArch64ISelLowering.cpp.

References llvm::dyn_cast(), isSupportedExtend(), Opc, llvm::ISD::SHL, llvm::ISD::SRA, llvm::ISD::SRL, and uint64_t.

Referenced by getAArch64Cmp(), and getCmpOrCmnOperandFoldingProfit().

getCmpOrCmnOperandFoldingProfit()

unsigned getCmpOrCmnOperandFoldingProfit ( SDValue Op, ISD::CondCode CC, SelectionDAG & DAG )

static

Definition at line 4407 of file AArch64ISelLowering.cpp.

References CC, getCmpOperandFoldingProfit(), and isCMN().

Referenced by getAArch64Cmp().

getCondCode()

SDValue getCondCode ( SelectionDAG & DAG, AArch64CC::CondCode CC )

static

Like SelectionDAG::getCondCode(), but for AArch64 condition codes.

Definition at line 3928 of file AArch64ISelLowering.cpp.

References CC, CondCodeVT, and llvm::SelectionDAG::getConstant().

Referenced by carryFlagToValue(), emitConditionalComparison(), foldADCToCINC(), foldCSELOfCSEL(), getAArch64Cmp(), getPTest(), getSETCC(), LowerBRCOND(), LowerXALUO(), overflowFlagToValue(), performAddCSelIntoCSinc(), performANDORCSELCombine(), performANDSETCCCombine(), performBRCONDCombine(), performCSELCombine(), performRNDRCombine(), performSETCCCombine(), and reassociateCSELOperandsForCSE().

getConstantLaneNumOfExtractHalfOperand()

std::optional< uint64_t > getConstantLaneNumOfExtractHalfOperand ( SDValue & Op )

static

Definition at line 5861 of file AArch64ISelLowering.cpp.

References C(), llvm::dyn_cast(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), and llvm::EVT::isFixedLengthVector().

getContainerForFixedLengthVector()

EVT getContainerForFixedLengthVector ( SelectionDAG & DAG, EVT VT )

static

Definition at line 34579 of file AArch64ISelLowering.cpp.

References assert(), llvm::EVT::getSimpleVT(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::EVT::getVectorElementType(), llvm::EVT::isFixedLengthVector(), llvm::TargetLoweringBase::isTypeLegal(), llvm_unreachable, and llvm::MVT::SimpleTy.

Referenced by combineToVCPOP(), convertFixedMaskToScalableVector(), GenerateFixedLengthSVETBL(), getVCIXISDNodeWCHAIN(), getWideningInterleave(), isLegalVTForZvzipDeinterleavedOperand(), isLegalVTForZvzipInterleavedOperand(), lowerBUILD_VECTOR(), lowerBuildVectorOfConstants(), lowerBuildVectorViaDominantValues(), lowerBuildVectorViaVID(), lowerCONVERT_FROM_ARBITRARY_FP(), lowerCttzElts(), lowerFixedVectorSegLoadIntrinsics(), lowerFixedVectorSegStoreIntrinsics(), lowerFMAXIMUM_FMINIMUM(), lowerFP_TO_INT_SAT(), lowerScalarInsert(), lowerShuffleViaVRegSplitting(), lowerVECTOR_SHUFFLEAsVSlide1(), lowerVECTOR_SHUFFLEAsVSlidedown(), lowerVECTOR_SHUFFLEAsVSlideup(), lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND(), LowerVectorMatch(), lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND(), lowerVectorXRINT_XROUND(), lowerZvzipVPAIR(), lowerZvzipVUNZIP(), lowerZvzipVZIP(), matchSplatAsGather(), performDUPCombine(), performExtractLastActiveCombine(), performFP_TO_INTCombine(), performLegalizedInterleavedStoreCombine(), performLegalizedVectorDeinterleaveCombine(), and processVCIXOperands().

getCSETCondCode()

std::optional< AArch64CC::CondCode > getCSETCondCode ( SDValue Op )

static

Definition at line 23918 of file AArch64ISelLowering.cpp.

References llvm::AArch64CC::AL, CC, llvm::isNullConstant(), llvm::isOneConstant(), and llvm::AArch64CC::NV.

Referenced by foldOverflowCheck(), performBRCONDCombine(), performExtendCombine(), and performSubWithBorrowCombine().

getDUPLANEOp()

unsigned getDUPLANEOp ( EVT EltType )

static

Definition at line 15694 of file AArch64ISelLowering.cpp.

References llvm_unreachable.

Referenced by performDUPCombine().

getEstimate()

SDValue getEstimate ( const AArch64Subtarget * ST, unsigned Opcode, SDValue Operand, SelectionDAG & DAG, int & ExtraSteps )

static

Definition at line 14102 of file AArch64ISelLowering.cpp.

References llvm::EVT::getFltSemantics(), llvm::SelectionDAG::getNode(), llvm::SDValue::getValueType(), llvm::Log2_64_Ceil(), SDValue(), llvm::APFloatBase::semanticsPrecision(), and llvm::TargetLoweringBase::Unspecified.

getExtFactor()

unsigned getExtFactor ( SDValue & V )

static

getExtFactor - Determine the adjustment factor for the position when generating an "extract from vector registers" instruction.

Definition at line 14829 of file AArch64ISelLowering.cpp.

References llvm::EVT::getSizeInBits(), and V.

Referenced by llvm::AArch64TargetLowering::ReconstructShuffle().

getGatherVecOpcode()

unsigned getGatherVecOpcode ( bool IsScaled, bool IsSigned, bool NeedsExtend )

static

Definition at line 7468 of file AArch64ISelLowering.cpp.

References llvm::InnerAnalysisManagerProxy< AnalysisManagerT, IRUnitT, ExtraArgTs... >::Key.

Referenced by performGLD1Combine().

getIntrinsicID()

unsigned getIntrinsicID ( const SDNode * N )

static

Definition at line 9235 of file AArch64ISelLowering.cpp.

References llvm::ISD::INTRINSIC_WO_CHAIN, N, and llvm::Intrinsic::not_intrinsic.

Referenced by llvm::AMDGPURegisterBankInfo::applyMappingImpl(), collectBitParts(), FindPreallocatedCall(), llvm::AMDGPURegisterBankInfo::getInstrAlternativeMappingsIntrinsic(), llvm::AMDGPURegisterBankInfo::getInstrAlternativeMappingsIntrinsicWSideEffects(), llvm::AArch64RegisterBankInfo::getInstrMapping(), llvm::AMDGPURegisterBankInfo::getInstrMapping(), getMaxValueForSVECntIntrinsic(), llvm::VPRecipeBuilder::handleReplication(), llvm::SITargetLowering::isCanonicalized(), isFPIntrinsic(), isFPIntrinsic(), llvm::AArch64TargetLowering::isReassocProfitable(), IsSVECntIntrinsic(), llvm::MipsLegalizerInfo::legalizeIntrinsic(), performIntrinsicCombine(), llvm::SubsumingPositionIterator::SubsumingPositionIterator(), tryConvertSVEWideCompare(), llvm::SPIRVGlobalRegistry::updateAssignType(), and valueIsKnownNeverF32Denorm().

getMaxValueForSVECntIntrinsic()

std::optional< ElementCount > getMaxValueForSVECntIntrinsic ( SDValue Op )

static

Definition at line 21414 of file AArch64ISelLowering.cpp.

References getIntrinsicID(), and llvm::ElementCount::getScalable().

getMemIntrinsicNode()

return DAG getMemIntrinsicNode	(	ISD::INTRINSIC_VOID	,
		DL	,
		DAG.	getVTListMVT::Other,
		Ops	,
		MemN->	getMemoryVT(),
		MemN->	getMemOperand() )

References AbstractManglingParser< Derived, Alloc >::Ops, DL, llvm::ISD::INTRINSIC_VOID, and MemN.

getPackedSVEVectorVT() [1/2]

EVT getPackedSVEVectorVT ( ElementCount EC )

inlinestatic

Definition at line 158 of file AArch64ISelLowering.cpp.

References llvm_unreachable.

getPackedSVEVectorVT() [2/2]

EVT getPackedSVEVectorVT ( EVT VT )

inlinestatic

Definition at line 133 of file AArch64ISelLowering.cpp.

References llvm::EVT::getSimpleVT(), llvm_unreachable, and llvm::MVT::SimpleTy.

Referenced by combineSVEReductionInt(), getSVEMultiVectorInfo(), lowerPredicateInterleaveWithPackedVectors(), LowerVectorMatch(), performDupLane128Combine(), and tryToReplaceScalarFPConversionWithSVE().

getPairwiseOpForReduction()

std::optional< Intrinsic::ID > getPairwiseOpForReduction ( unsigned Op )

static

Returns the pairwise SVE2 op that could be used for a v2<ty> reduction.

Definition at line 35255 of file AArch64ISelLowering.cpp.

References llvm::ISD::VECREDUCE_SMAX, llvm::ISD::VECREDUCE_SMIN, llvm::ISD::VECREDUCE_UMAX, and llvm::ISD::VECREDUCE_UMIN.

Referenced by llvm::AArch64TargetLowering::shouldLowerReductionToSVE().

getPredCounterWhileElementSize()

unsigned getPredCounterWhileElementSize ( Intrinsic::ID IID )

static

Returns the element size of a predicate-as-counter while intrinsic.

Definition at line 25574 of file AArch64ISelLowering.cpp.

References llvm_unreachable.

Referenced by lowerPredCounterWhile().

getPredCounterWhileOpcode()

unsigned getPredCounterWhileOpcode ( Intrinsic::ID IID )

static

Returns the ISD opcode for a predicate-as-counter while intrinsic.

Definition at line 25618 of file AArch64ISelLowering.cpp.

References llvm_unreachable.

Referenced by lowerPredCounterWhile().

getPredicatedReductionOpcode()

unsigned getPredicatedReductionOpcode ( unsigned Op )

static

Returns the corresponding predicated SVE reduction opcode for a VECREDUCE_*.

Definition at line 35271 of file AArch64ISelLowering.cpp.

References llvm_unreachable, llvm::ISD::VECREDUCE_ADD, llvm::ISD::VECREDUCE_AND, llvm::ISD::VECREDUCE_FADD, llvm::ISD::VECREDUCE_FMAX, llvm::ISD::VECREDUCE_FMAXIMUM, llvm::ISD::VECREDUCE_FMIN, llvm::ISD::VECREDUCE_FMINIMUM, llvm::ISD::VECREDUCE_OR, llvm::ISD::VECREDUCE_SMAX, llvm::ISD::VECREDUCE_SMIN, llvm::ISD::VECREDUCE_UMAX, llvm::ISD::VECREDUCE_UMIN, and llvm::ISD::VECREDUCE_XOR.

getPredicateForFixedLengthVector()

SDValue getPredicateForFixedLengthVector ( SelectionDAG & DAG, SDLoc & DL, EVT VT )

static

Definition at line 34606 of file AArch64ISelLowering.cpp.

References assert(), DL, getPTrue(), llvm::EVT::getSimpleVT(), llvm::getSVEPredPatternFromNumElements(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::EVT::isFixedLengthVector(), llvm::TargetLoweringBase::isTypeLegal(), llvm_unreachable, and llvm::MVT::SimpleTy.

Referenced by convertFixedMaskToScalableVector(), getPredicateForVector(), performLegalizedInterleavedStoreCombine(), and performLegalizedVectorDeinterleaveCombine().

getPredicateForScalableVector()

SDValue getPredicateForScalableVector ( SelectionDAG & DAG, SDLoc & DL, EVT VT )

static

Definition at line 34641 of file AArch64ISelLowering.cpp.

References assert(), llvm::EVT::changeVectorElementType(), DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::EVT::isScalableVector(), and llvm::TargetLoweringBase::isTypeLegal().

Referenced by getPredicateForVector().

getPredicateForVector()

SDValue getPredicateForVector ( SelectionDAG & DAG, SDLoc & DL, EVT VT )

static

Definition at line 34649 of file AArch64ISelLowering.cpp.

References DL, getPredicateForFixedLengthVector(), getPredicateForScalableVector(), and llvm::EVT::isFixedLengthVector().

Referenced by llvm::AArch64TargetLowering::LowerOperation(), tryCombineExtendRShTrunc(), and trySQABSCombine().

getPredicateRegisterClass()

const TargetRegisterClass * getPredicateRegisterClass ( PredicateConstraint Constraint, EVT VT )

static

Definition at line 14301 of file AArch64ISelLowering.cpp.

References llvm::EVT::isScalableVectorOf(), llvm_unreachable, Upa, Uph, and Upl.

getPromotedVTForPredicate()

EVT getPromotedVTForPredicate ( EVT VT )

inlinestatic

Definition at line 173 of file AArch64ISelLowering.cpp.

References assert(), llvm::EVT::getVectorMinNumElements(), llvm::EVT::isScalableVectorOf(), and llvm_unreachable.

getPTest()

SDValue getPTest ( SelectionDAG & DAG, EVT VT, SDValue Pg, SDValue Op, AArch64CC::CondCode Cond )

static

Definition at line 25303 of file AArch64ISelLowering.cpp.

References llvm::AArch64CC::ANY_ACTIVE, assert(), CC, Cond, DL, llvm::AArch64CC::FIRST_ACTIVE, getCondCode(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getNode(), getSVEPredicateBitCast(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getZExtOrTrunc(), isZeroingInactiveLanes(), llvm::AArch64CC::NONE_ACTIVE, llvm::Test, and TLI.

Referenced by performFirstTrueTestVectorCombine(), performIntrinsicCombine(), and performLastTrueTestVectorCombine().

getPTrue()

SDValue getPTrue ( SelectionDAG & DAG, SDLoc DL, EVT VT, int Pattern )

inlinestatic

Definition at line 6277 of file AArch64ISelLowering.cpp.

References DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::getNumElementsFromSVEPredPattern(), llvm::SelectionDAG::getSubtarget(), llvm::SelectionDAG::getTargetConstant(), llvm::EVT::getVectorMinNumElements(), and Subtarget.

Referenced by getPredicateForFixedLengthVector(), optimizeIncrementingWhile(), performMSTORECombine(), and performUnpackCombine().

getPTrueAsCounter()

SDValue getPTrueAsCounter ( SelectionDAG & DAG, SDLoc DL, EVT VT )

inlinestatic

Definition at line 6296 of file AArch64ISelLowering.cpp.

References DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), llvm::ISD::INTRINSIC_WO_CHAIN, and llvm_unreachable.

Referenced by tryLowerMultiVectorLoad(), and tryLowerMultiVectorStore().

getReducedGprRegisterClass()

const TargetRegisterClass * getReducedGprRegisterClass ( ReducedGprConstraint Constraint, EVT VT )

static

Definition at line 14331 of file AArch64ISelLowering.cpp.

References llvm::EVT::getFixedSizeInBits(), llvm::EVT::isScalarInteger(), llvm_unreachable, Uci, and Ucj.

getReductionForOpcode()

unsigned getReductionForOpcode ( unsigned Op )

static

Definition at line 32147 of file AArch64ISelLowering.cpp.

References llvm_unreachable, llvm::ISD::SMAX, llvm::ISD::SMIN, llvm::ISD::UMAX, llvm::ISD::UMIN, llvm::ISD::VECREDUCE_SMAX, llvm::ISD::VECREDUCE_SMIN, llvm::ISD::VECREDUCE_UMAX, and llvm::ISD::VECREDUCE_UMIN.

Referenced by performMINMAXCombine().

getReductionSDNode()

SDValue getReductionSDNode ( unsigned Op, SDLoc DL, SDValue ScalarOp, SelectionDAG & DAG )

static

Definition at line 18276 of file AArch64ISelLowering.cpp.

References DL, llvm::SelectionDAG::getExtractVectorElt(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), and llvm::SDValue::getValueType().

getScaledOffsetForBitWidth()

SDValue getScaledOffsetForBitWidth ( SelectionDAG & DAG, SDValue Offset, SDLoc DL, unsigned BitWidth )

static

Definition at line 30907 of file AArch64ISelLowering.cpp.

References assert(), llvm::BitWidth, DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::Log2_32(), llvm::Offset, llvm::ISD::SHL, and llvm::ISD::SPLAT_VECTOR.

Referenced by performGatherLoadCombine(), and performScatterStoreCombine().

getSETCC()

SDValue getSETCC ( AArch64CC::CondCode CC, SDValue NZCV, const SDLoc & DL, SelectionDAG & DAG )

static

Helper function to create 'CSET', which is equivalent to 'CSINC <Wd>, WZR, WZR, invert(<cond>)'.

Definition at line 14371 of file AArch64ISelLowering.cpp.

References CC, DL, getCondCode(), llvm::SelectionDAG::getConstant(), and llvm::SelectionDAG::getNode().

Referenced by combineAnd(), combineCMov(), combineM68kSetCC(), combineOr(), combineSetCC(), combineSubSetcc(), combineVECREDUCE_LOGIC(), combineVectorSizedSetCCEquality(), combineX86SetCC(), foldXor1SetCC(), LowerADDSUBO_CARRY(), llvm::X86TargetLowering::LowerAsmOutputForConstraint(), LowerCMP_SWAP(), LowerINTRINSIC_W_CHAIN(), LowerPARITY(), LowerXALUO(), perfomPextFirstTrueVectorCombine(), llvm::X86TargetLowering::ReplaceNodeResults(), and llvm::X86TargetLowering::SimplifyDemandedBitsForTargetNode().

getSignExtendedGatherOpcode()

unsigned getSignExtendedGatherOpcode ( unsigned Opcode )

static

Definition at line 7492 of file AArch64ISelLowering.cpp.

References llvm_unreachable.

Referenced by performGLD1Combine().

getSMECallAttrs()

SMECallAttrs getSMECallAttrs ( const Function & Caller, const RTLIB::RuntimeLibcallsInfo & RTLCI, const TargetLowering::CallLoweringInfo & CLI )

static

Definition at line 10101 of file AArch64ISelLowering.cpp.

References llvm::TargetLowering::CallLoweringInfo::Callee, llvm::TargetLowering::CallLoweringInfo::CB, llvm::dyn_cast(), and llvm::SMEAttrs::Normal.

getSMToggleCondition()

AArch64SME::ToggleCondition getSMToggleCondition ( const SMECallAttrs & CallAttrs )

static

Definition at line 10433 of file AArch64ISelLowering.cpp.

References llvm::AArch64SME::Always, CallAttrs, llvm::AArch64SME::IfCallerIsNonStreaming, llvm::AArch64SME::IfCallerIsStreaming, and llvm_unreachable.

getStructuredLoadFunction()

Function * getStructuredLoadFunction ( Module * M, unsigned Factor, bool Scalable, Type * LDVTy, Type * PtrTy )

static

Definition at line 19685 of file AArch64ISelLowering.cpp.

References assert(), and llvm::Intrinsic::getOrInsertDeclaration().

Referenced by llvm::AArch64TargetLowering::lowerDeinterleaveIntrinsicToLoad(), and llvm::AArch64TargetLowering::lowerInterleavedLoad().

getStructuredStoreFunction()

Function * getStructuredStoreFunction ( Module * M, unsigned Factor, bool Scalable, Type * STVTy, Type * PtrTy )

static

Definition at line 19703 of file AArch64ISelLowering.cpp.

References assert(), llvm::Intrinsic::getOrInsertDeclaration(), and SVEStores.

Referenced by llvm::AArch64TargetLowering::lowerInterleavedStore(), and llvm::AArch64TargetLowering::lowerInterleaveIntrinsicToStore().

getSVEContainerIRType()

ScalableVectorType * getSVEContainerIRType ( FixedVectorType * VTy )

static

Definition at line 19657 of file AArch64ISelLowering.cpp.

References llvm::ScalableVectorType::get(), llvm::Type::getBFloatTy(), llvm::Type::getContext(), llvm::Type::getDoubleTy(), llvm::VectorType::getElementType(), llvm::Type::getFloatTy(), llvm::Type::getHalfTy(), llvm::Type::getInt16Ty(), llvm::Type::getInt32Ty(), llvm::Type::getInt64Ty(), llvm::Type::getInt8Ty(), and llvm_unreachable.

Referenced by llvm::AArch64TargetLowering::lowerInterleavedLoad(), and llvm::AArch64TargetLowering::lowerInterleavedStore().

getSVEContainerType()

MVT getSVEContainerType ( EVT ContentTy )

static

Definition at line 26568 of file AArch64ISelLowering.cpp.

References assert(), llvm::EVT::getSimpleVT(), llvm::EVT::isSimple(), llvm_unreachable, and llvm::MVT::SimpleTy.

Referenced by performGatherLoadCombine(), performLD1Combine(), performScatterStoreCombine(), and performST1Combine().

getSVEMultiVectorInfo()

SVEMultiVectorInfo getSVEMultiVectorInfo ( MVT VT )

static

Definition at line 7918 of file AArch64ISelLowering.cpp.

References assert(), getPackedSVEVectorVT(), llvm::TypeSize::getScalable(), llvm::EVT::getSimpleVT(), llvm::MVT::getSizeInBits(), llvm::MVT::getVectorElementType(), and llvm::Size.

Referenced by tryLowerMultiVectorLoad(), and tryLowerMultiVectorStore().

getSVEOpcodeForFPToInt()

unsigned getSVEOpcodeForFPToInt ( unsigned Opc )

static

Definition at line 36132 of file AArch64ISelLowering.cpp.

References llvm::ISD::FP_TO_SINT, llvm::ISD::FP_TO_SINT_SAT, llvm::ISD::FP_TO_UINT, llvm::ISD::FP_TO_UINT_SAT, llvm_unreachable, Opc, llvm::ISD::STRICT_FP_TO_SINT, and llvm::ISD::STRICT_FP_TO_UINT.

getSVEPredicateBitCast()

SDValue getSVEPredicateBitCast ( EVT VT, SDValue Op, SelectionDAG & DAG )

static

Definition at line 6410 of file AArch64ISelLowering.cpp.

References llvm::ISD::AND, assert(), llvm::EVT::bitsGT(), DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::EVT::getVectorElementType(), llvm::ISD::INTRINSIC_WO_CHAIN, llvm::EVT::isScalableVector(), llvm::TargetLoweringBase::isTypeLegal(), and isZeroingInactiveLanes().

Referenced by getPTest(), and optimizeBrk().

getTestBitOperand()

SDValue getTestBitOperand ( SDValue Op, unsigned & Bit, bool & Invert, SelectionDAG & DAG )

static

Definition at line 30290 of file AArch64ISelLowering.cpp.

References llvm::ISD::AND, llvm::ISD::ANY_EXTEND, C(), llvm::dyn_cast(), getTestBitOperand(), llvm::ISD::SHL, llvm::ISD::SRA, llvm::ISD::SRL, llvm::ISD::TRUNCATE, and llvm::ISD::XOR.

Referenced by getTestBitOperand(), and performTBZCombine().

getVectorBitwiseReduce()

SDValue getVectorBitwiseReduce ( unsigned Opcode, SDValue Vec, EVT VT, SDLoc DL, SelectionDAG & DAG )

static

Definition at line 18283 of file AArch64ISelLowering.cpp.

References llvm::ISD::AND, llvm::ISD::ANY_EXTEND, assert(), llvm::EVT::changeVectorElementType(), DL, llvm::SelectionDAG::getAnyExtOrTrunc(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::EVT::getIntegerVT(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), getVectorBitwiseReduce(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::Hi, llvm::EVT::isFixedLengthVector(), llvm::EVT::isPow2VectorType(), llvm_unreachable, Lo, llvm::ISD::OR, SDValue(), llvm::ISD::SETCC, llvm::ISD::SIGN_EXTEND, llvm::SelectionDAG::SplitVector(), llvm::ISD::SRL, llvm::ISD::VECREDUCE_ADD, llvm::ISD::VECREDUCE_AND, llvm::ISD::VECREDUCE_OR, llvm::ISD::VECREDUCE_UMAX, llvm::ISD::VECREDUCE_UMIN, llvm::ISD::VECREDUCE_XOR, and llvm::ISD::XOR.

Referenced by getVectorBitwiseReduce(), and performVecReduceBitwiseCombine().

getVShiftImm()

bool getVShiftImm ( SDValue Op, unsigned ElementBits, int64_t & Cnt )

static

getVShiftImm - Check if this is a valid build_vector for the immediate operand of a vector shift operation, where all the elements of the build_vector must have the same constant integer value.

Definition at line 18025 of file AArch64ISelLowering.cpp.

References llvm::ISD::BITCAST, llvm::dyn_cast(), llvm::APInt::getSExtValue(), and llvm::BuildVectorSDNode::isConstantSplat().

Referenced by isVShiftLImm(), isVShiftLImm(), isVShiftRImm(), and isVShiftRImm().

getZAMarkerForCall()

std::optional< unsigned > getZAMarkerForCall ( const SMECallAttrs & CallAttrs )

static

Returns the ZA state save/use marker node necessary for CallAttrs.

Definition at line 10486 of file AArch64ISelLowering.cpp.

References CallAttrs.

hasCheapXorImmediateThatNeedsCondCmpReg()

bool hasCheapXorImmediateThatNeedsCondCmpReg ( const std::pair< SDValue, SDValue > & Pair )

static

Definition at line 12742 of file AArch64ISelLowering.cpp.

References llvm::BitWidth, C(), llvm::dyn_cast(), Imm, isLegalCondCmpImmediate(), llvm::AArch64_AM::isLogicalImmediate(), and V.

Referenced by performOrXorChainCombine().

hasLegalCmpImmediate()

bool hasLegalCmpImmediate ( SDValue LHS, SDValue RHS )

static

Definition at line 12717 of file AArch64ISelLowering.cpp.

References C(), llvm::dyn_cast(), llvm::AArch64_AM::isLegalCmpImmed(), LHS, and RHS.

Referenced by isOrXorChain(), and performOrXorChainCombine().

hasLegalCondCmpImmediate()

bool hasLegalCondCmpImmediate ( SDValue LHS, SDValue RHS )

static

Definition at line 12728 of file AArch64ISelLowering.cpp.

References C(), llvm::dyn_cast(), isLegalCondCmpImmediate(), LHS, and RHS.

Referenced by performOrXorChainCombine().

hasNearbyPairedStore()

template<typename Iter>

bool hasNearbyPairedStore	(	Iter	It,
		Iter	End,
		Value *	Ptr,
		const DataLayout &	DL )

Definition at line 19864 of file AArch64ISelLowering.cpp.

References DL, llvm::dyn_cast(), llvm::End, llvm::isa(), llvm::APInt::sextOrTrunc(), and llvm::Value::stripAndAccumulateConstantOffsets().

Referenced by llvm::AArch64TargetLowering::lowerInterleavedStore().

hasPairwiseAdd()

bool hasPairwiseAdd ( unsigned Opcode, EVT VT, bool FullFP16 )

static

Definition at line 22700 of file AArch64ISelLowering.cpp.

References llvm::ISD::ADD, llvm::ISD::FADD, and llvm::ISD::STRICT_FADD.

Referenced by performExtractVectorEltCombine().

hasSVEMultiVectorOps()

bool hasSVEMultiVectorOps ( const AArch64Subtarget * Subtarget )

static

Definition at line 22840 of file AArch64ISelLowering.cpp.

References Subtarget.

Referenced by perfomPextFirstTrueVectorCombine().

if() [1/8]

if

(

!IsMasked &&

SubBits==64||SubBits==128) &&Subtarget.isNeonAvailable(

)

Definition at line 28531 of file AArch64ISelLowering.cpp.

References AbstractManglingParser< Derived, Alloc >::Ops, DL, llvm::SelectionDAG::getMemIntrinsicNode(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getVTList(), llvm::ISD::INTRINSIC_VOID, MemN, NumParts, SubBits, and Subtarget.

if() [2/8]

else if

(

!

isSplatVectorInterleaveOpsDAG, DL, WideValue, ValueInterleaveOps

)

Definition at line 28478 of file AArch64ISelLowering.cpp.

if() [3/8]

if

(

!Subtarget.

isSVEorStreamingSVEAvailable()

)

References Subtarget.

if() [4/8]

if

(

!TLI.

isTypeLegalSubVecTy

)

References SubVecTy, and TLI.

if() [5/8]

if

(

IsMasked

)

Definition at line 28553 of file AArch64ISelLowering.cpp.

References convertFixedMaskToScalableVector(), DL, llvm::SelectionDAG::getNode(), NumParts, Pred, SDValue(), llvm::ISD::SIGN_EXTEND, and SubVecTy.

if() [6/8]

if

(

IsScalable &&SubBits !

= 128

)

References IsScalable, and SubBits.

if() [7/8]

if

(

IsScalable

)

Definition at line 28497 of file AArch64ISelLowering.cpp.

References AbstractManglingParser< Derived, Alloc >::Ops, DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getMemIntrinsicNode(), llvm::SelectionDAG::getVTList(), llvm::ISD::INTRINSIC_VOID, IsScalable, MemN, NumParts, Pred, SDValue(), and SubVecTy.

if() [8/8]

if

(

NumParts !

= 2 &&NumParts !=3 &&NumParts !=4

)

References NumParts.

isAddSubSExt()

bool isAddSubSExt ( SDValue N, SelectionDAG & DAG )

static

Definition at line 5922 of file AArch64ISelLowering.cpp.

References llvm::ISD::ADD, llvm::SDNode::hasOneUse(), isSignExtended(), N, and llvm::ISD::SUB.

Referenced by LowerMUL(), and selectUmullSmull().

isAddSubZExt()

bool isAddSubZExt ( SDValue N, SelectionDAG & DAG )

static

Definition at line 5933 of file AArch64ISelLowering.cpp.

References llvm::ISD::ADD, llvm::SDNode::hasOneUse(), isZeroExtended(), N, and llvm::ISD::SUB.

Referenced by LowerMUL(), and selectUmullSmull().

isAllActivePredicate()

bool isAllActivePredicate ( const SelectionDAG & DAG, SDValue N )

static

Definition at line 16633 of file AArch64ISelLowering.cpp.

References llvm::ISD::isConstantSplatVectorAllOnes(), and N.

Referenced by convertMergedOpToPredOp(), foldCSELofLASTB(), instCombineSVECmpNE(), instCombineSVEDup(), instCombineSVELD1(), instCombineSVESel(), instCombineSVESrshl(), instCombineSVEST1(), instCombineSVEUxt(), instCombineSVEVectorBinOp(), instCombineZExtSVECmpNE(), performSetccMergeZeroCombine(), performSVEAndCombine(), performVSelectCombine(), simplifySVEIntrinsic(), and tryLowerToSLI().

isAllConstantBuildVector()

bool isAllConstantBuildVector ( const SDValue & PotentialBVec, uint64_t & ConstVal )

static

Definition at line 16608 of file AArch64ISelLowering.cpp.

References llvm::dyn_cast(), llvm::SDNode::getOperand(), llvm::SDNode::getValueType(), llvm::EVT::getVectorNumElements(), llvm::ConstantSDNode::getZExtValue(), and uint64_t.

Referenced by isPow2Splat().

isAllInactivePredicate()

bool isAllInactivePredicate ( SDValue N )

static

Definition at line 16625 of file AArch64ISelLowering.cpp.

References llvm::ISD::isConstantSplatVectorAllZeros(), and N.

Referenced by performVSelectCombine().

isCheapToExtend()

bool isCheapToExtend ( const SDValue & N )

static

Definition at line 26081 of file AArch64ISelLowering.cpp.

References llvm::ISD::isConstantSplatVectorAllZeros(), llvm::ISD::LOAD, llvm::ISD::MLOAD, and N.

Referenced by performSignExtendSetCCCombine().

isCMN()

bool isCMN ( SDValue Op, ISD::CondCode CC, SelectionDAG & DAG )

static

Definition at line 3966 of file AArch64ISelLowering.cpp.

References CC, llvm::SelectionDAG::isKnownNeverZero(), llvm::isNullConstant(), isSafeSignedCMN(), and llvm::ISD::SUB.

Referenced by emitComparison(), emitConditionalComparison(), and getCmpOrCmnOperandFoldingProfit().

isCMP()

bool isCMP ( SDValue Op )

static

Definition at line 23912 of file AArch64ISelLowering.cpp.

Referenced by foldCSELOfCSEL(), foldOverflowCheck(), and performBRCONDCombine().

isConcatMask()

bool isConcatMask ( ArrayRef< int > Mask, EVT VT, bool SplitLHS )

static

Definition at line 15572 of file AArch64ISelLowering.cpp.

References E(), llvm::EVT::getSizeInBits(), llvm::EVT::getVectorNumElements(), I, and llvm::Offset.

Referenced by llvm::AArch64TargetLowering::isShuffleMaskLegal(), and tryFormConcatFromShuffle().

isConstantSplatVectorMaskForType()

bool isConstantSplatVectorMaskForType ( SDNode * N, EVT MemVT )

static

Definition at line 22377 of file AArch64ISelLowering.cpp.

References llvm::dyn_cast(), llvm::EVT::getSimpleVT(), llvm::EVT::getVectorElementType(), llvm::EVT::isSimple(), N, llvm::MVT::SimpleTy, llvm::ISD::SPLAT_VECTOR, and uint64_t.

isDeinterleave4ForWideningUToFP()

bool isDeinterleave4ForWideningUToFP ( SDNode * N )

static

Definition at line 31797 of file AArch64ISelLowering.cpp.

References llvm::all_of(), llvm::Use::getUser(), N, llvm::ISD::UINT_TO_FP, and Users.

Referenced by performVectorDeinterleaveCombine().

isEligibleForSmallVectorLoadOpt()

bool isEligibleForSmallVectorLoadOpt ( LoadSDNode * LD, const AArch64Subtarget & Subtarget )

static

Helper function to check if a small vector load can be optimized.

Definition at line 7410 of file AArch64ISelLowering.cpp.

References llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getFixedValue(), llvm::EVT::getStoreSize(), and Subtarget.

Referenced by tryLowerSmallVectorExtLoad().

isEquivalentMaskless()

bool isEquivalentMaskless ( unsigned CC, unsigned width, ISD::LoadExtType ExtType, int AddConstant, int CompConstant )

static

Definition at line 29165 of file AArch64ISelLowering.cpp.

References llvm::AArch64CC::AL, CC, llvm::AArch64CC::EQ, llvm::AArch64CC::GE, llvm::AArch64CC::GT, llvm::AArch64CC::HI, llvm::AArch64CC::HS, llvm::AArch64CC::Invalid, llvm::AArch64CC::LE, llvm::AArch64CC::LO, llvm::AArch64CC::LS, llvm::AArch64CC::LT, llvm::AArch64CC::MI, llvm::AArch64CC::NE, llvm::AArch64CC::NV, llvm::AArch64CC::PL, llvm::ISD::SEXTLOAD, llvm::AArch64CC::VC, and llvm::AArch64CC::VS.

Referenced by performCONDCombine().

isEssentiallyExtractHighSubvector()

bool isEssentiallyExtractHighSubvector ( SDValue N )

static

Definition at line 23488 of file AArch64ISelLowering.cpp.

References llvm::ISD::BITCAST, llvm::ISD::EXTRACT_SUBVECTOR, and N.

Referenced by performAddSubLongCombine(), reassociateAddSubLong(), tryCombineLongOpWithDup(), and tryCombineMULLWithUZP1().

isExtendedBUILD_VECTOR()

bool isExtendedBUILD_VECTOR ( SDValue N, SelectionDAG & DAG, bool isSigned )

static

Definition at line 5874 of file AArch64ISelLowering.cpp.

References llvm::ISD::BUILD_VECTOR, C(), llvm::dyn_cast(), llvm::EVT::getScalarSizeInBits(), llvm::isIntN(), isSigned(), llvm::isUIntN(), and N.

Referenced by isSignExtended(), isSignExtended(), isZeroExtended(), and isZeroExtended().

isExtendOrShiftOperand()

bool isExtendOrShiftOperand ( SDValue N )

static

Definition at line 24281 of file AArch64ISelLowering.cpp.

References llvm::ISD::AND, llvm::cast(), llvm::dyn_cast(), llvm::ConstantSDNode::getZExtValue(), llvm::isa(), llvm::ISD::isExtOpcode(), N, llvm::ISD::SHL, llvm::ISD::SIGN_EXTEND_INREG, llvm::ISD::SRA, llvm::ISD::SRL, and uint64_t.

Referenced by performAddCombineSubShift().

isEXTMaskWithSplat()

bool isEXTMaskWithSplat ( ArrayRef< int > M, EVT VT, unsigned SplatOperand, bool & ReverseEXT, unsigned & Imm )

static

Definition at line 15478 of file AArch64ISelLowering.cpp.

References llvm::EVT::getVectorNumElements(), I, and Imm.

isHalvingTruncateAndConcatOfLegalIntScalableType()

bool isHalvingTruncateAndConcatOfLegalIntScalableType ( SDNode * N )

static

Definition at line 26980 of file AArch64ISelLowering.cpp.

References llvm::SDNode::getValueType(), and N.

Referenced by performUzpCombine().

isINSMask()

bool isINSMask ( ArrayRef< int > M, int NumInputElements, bool & DstIsLeft, int & Anomaly )

static

Definition at line 15533 of file AArch64ISelLowering.cpp.

Referenced by llvm::AArch64TargetLowering::isShuffleMaskLegal().

isIntImmediate()

bool isIntImmediate ( const SDNode * N, uint64_t & Imm )

static

Definition at line 2720 of file AArch64ISelLowering.cpp.

References C(), llvm::dyn_cast(), Imm, N, and uint64_t.

isLane0KnownActive()

bool isLane0KnownActive ( SDValue Op )

static

Definition at line 31393 of file AArch64ISelLowering.cpp.

References isLane0KnownActive(), llvm::isOneConstant(), and llvm::ISD::SPLAT_VECTOR.

Referenced by isLane0KnownActive(), and performPTestFirstCombine().

isLanes1toNKnownZero()

bool isLanes1toNKnownZero ( SDValue Op )

static

Definition at line 31368 of file AArch64ISelLowering.cpp.

Referenced by removeRedundantInsertVectorElt().

isLegalCondCmpImmediate()

bool isLegalCondCmpImmediate ( const APInt & Imm )

static

Definition at line 12710 of file AArch64ISelLowering.cpp.

References Imm.

Referenced by hasCheapXorImmediateThatNeedsCondCmpReg(), and hasLegalCondCmpImmediate().

isLegalNTStore()

bool isLegalNTStore ( Type * DataType, Align Alignment, const DataLayout & DL )

static

Definition at line 7825 of file AArch64ISelLowering.cpp.

References DL, llvm::dyn_cast(), and llvm::isPowerOf2_64().

Referenced by llvm::AArch64TargetLowering::lowerInterleavedStore(), and LowerNTStore().

isLoadOrMultipleLoads()

bool isLoadOrMultipleLoads ( SDValue B, SmallVector< LoadSDNode * > & Loads )

static

Definition at line 24528 of file AArch64ISelLowering.cpp.

References B(), llvm::ISD::BUILD_VECTOR, llvm::cast(), llvm::ISD::CONCAT_VECTORS, llvm::dyn_cast(), llvm::SDValue::getNumOperands(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::hasOneUse(), llvm::SDValue::hasOneUse(), I, llvm::peekThroughOneUseBitcasts(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), and llvm::ISD::VECTOR_SHUFFLE.

Referenced by areLoadedOffsetButOtherwiseSame(), and performExtBinopLoadFold().

isMergePassthruOpcode()

bool isMergePassthruOpcode ( unsigned Opc )

static

Definition at line 224 of file AArch64ISelLowering.cpp.

References Opc.

Referenced by performVSelectCombine().

isNegatedInteger()

bool isNegatedInteger ( SDValue Op )

static

Definition at line 23773 of file AArch64ISelLowering.cpp.

References llvm::isNullConstant(), and llvm::ISD::SUB.

Referenced by performNegCSelCombine().

isNVCastToHalfWidthElements()

SDValue isNVCastToHalfWidthElements ( SDValue V, const AArch64Subtarget * Subtarget )

static

Definition at line 27061 of file AArch64ISelLowering.cpp.

References llvm::ISD::BITCAST, SDValue(), Subtarget, and V.

Referenced by performUzpCombine().

isOneVector()

bool isOneVector ( SDValue V )

static

Definition at line 3759 of file AArch64ISelLowering.cpp.

References llvm::isOneConstant(), llvm::isOneOrOneSplat(), and V.

Referenced by performSubNegAndOneCombine().

isOnlyUsedAsMemoryAddress()

bool isOnlyUsedAsMemoryAddress ( SDNode * N )

static

Definition at line 21719 of file AArch64ISelLowering.cpp.

References llvm::ISD::ADD, assert(), llvm::cast(), llvm::Use::getOperandNo(), llvm::Use::getUser(), llvm::ISD::LOAD, llvm::Load, N, llvm::ISD::STORE, llvm::Store, and llvm::ISD::SUB.

Referenced by performMulCombine().

isOpcWithIntImmediate()

bool isOpcWithIntImmediate ( const SDNode * N, unsigned Opc, uint64_t & Imm )

static

Definition at line 2740 of file AArch64ISelLowering.cpp.

References Imm, isIntImmediate(), N, Opc, and uint64_t.

isOrXorChain()

bool isOrXorChain ( SDValue N, SelectionDAG & DAG, unsigned & NumLeaves, unsigned & NumXors, bool & SawXor, bool RequireLegalCmpImmediates, SmallVectorImpl< std::pair< SDValue, SDValue > > & WorkList )

static

Definition at line 12759 of file AArch64ISelLowering.cpp.

References llvm::AArch64Subtarget::getCLOpts(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getSubtarget(), hasLegalCmpImmediate(), isOrXorChain(), llvm::EVT::isScalarInteger(), N, llvm::ISD::OR, llvm::ISD::XOR, and llvm::ISD::ZERO_EXTEND.

Referenced by isOrXorChain(), and performOrXorChainCombine().

isPackedPredicateType()

bool isPackedPredicateType ( EVT VT, SelectionDAG & DAG )

inlinestatic

Definition at line 202 of file AArch64ISelLowering.cpp.

References assert(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::TargetLoweringBase::isTypeLegal(), and llvm::EVT::isVector().

Referenced by isUnpackedType().

isPackedVectorType()

bool isPackedVectorType ( EVT VT, SelectionDAG & DAG )

inlinestatic

Returns true if VT's elements occupy the lowest bit positions of its associated register class without any intervening space.

For example, nxv2f16, nxv4f16 and nxv8f16 are legal types that belong to the same register class, but only nxv8f16 can be treated as a packed vector.

Definition at line 195 of file AArch64ISelLowering.cpp.

References assert(), llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getKnownMinValue(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::EVT::isFixedLengthVector(), llvm::TargetLoweringBase::isTypeLegal(), llvm::EVT::isVector(), and llvm::AArch64::SVEBitsPerBlock.

Referenced by isUnpackedType().

isPassedInFPR()

bool isPassedInFPR ( EVT VT )

static

Definition at line 9344 of file AArch64ISelLowering.cpp.

References llvm::EVT::isFixedLengthVector(), llvm::EVT::isFloatingPoint(), and llvm::EVT::isScalableVector().

isPow2Splat()

bool isPow2Splat ( SDValue Op, uint64_t & SplatVal, bool & Negated )

static

Definition at line 17888 of file AArch64ISelLowering.cpp.

References llvm::ISD::BUILD_VECTOR, llvm::isa(), isAllConstantBuildVector(), llvm::isPowerOf2_64(), llvm::ISD::SPLAT_VECTOR, and uint64_t.

isPredicateCCSettingOp()

bool isPredicateCCSettingOp ( SDValue N )

static

Definition at line 22715 of file AArch64ISelLowering.cpp.

References llvm::ISD::GET_ACTIVE_LANE_MASK, llvm::ISD::INTRINSIC_WO_CHAIN, N, and llvm::ISD::SETCC.

Referenced by performFirstTrueTestVectorCombine().

isSafeSignedCMN()

bool isSafeSignedCMN ( SDValue Op, SelectionDAG & DAG )

static

Definition at line 3940 of file AArch64ISelLowering.cpp.

References llvm::SelectionDAG::computeKnownBits(), llvm::KnownBits::getSignedMinValue(), and llvm::APInt::isMinSignedValue().

Referenced by isCMN(), and isCMN().

isSetCC()

bool isSetCC ( SDValue Op, SetCCInfoAndKind & SetCCInfo )

static

Check whether or not Op is a SET_CC operation, either a generic or an AArch64 lowered one.

SetCCInfo is filled accordingly.

Postcondition

SetCCInfo is meanginfull only when this function returns true.

Returns

True when Op is a kind of SET_CC operation.

Definition at line 23532 of file AArch64ISelLowering.cpp.

References SetCCInfo::AArch64, AArch64SetCCInfo::CC, GenericSetCCInfo::CC, AArch64SetCCInfo::Cmp, llvm::dyn_cast(), SetCCInfo::Generic, llvm::AArch64CC::getInvertedCondCode(), llvm::ConstantSDNode::isOne(), llvm::ConstantSDNode::isZero(), GenericSetCCInfo::Opnd0, GenericSetCCInfo::Opnd1, llvm::ISD::SETCC, and std::swap().

Referenced by isSetCCOrZExtSetCC().

isSetCCOrZExtSetCC()

bool isSetCCOrZExtSetCC ( const SDValue & Op, SetCCInfoAndKind & Info )

static

Definition at line 23575 of file AArch64ISelLowering.cpp.

References isSetCC(), and llvm::ISD::ZERO_EXTEND.

Referenced by performSetccAddFolding().

isSignExtended()

bool isSignExtended ( SDValue N, SelectionDAG & DAG )

static

Definition at line 5910 of file AArch64ISelLowering.cpp.

References llvm::ISD::ANY_EXTEND, isExtendedBUILD_VECTOR(), N, and llvm::ISD::SIGN_EXTEND.

Referenced by isAddSubSExt(), isAddSubSExt(), LowerMUL(), performMulCombine(), selectUmullSmull(), and signExtendOperandIfUnknown().

isSignExtInReg()

bool isSignExtInReg ( const SDValue & V )

static

Definition at line 30204 of file AArch64ISelLowering.cpp.

References llvm::BitWidth, and V.

Referenced by performSetccMergeZeroCombine().

isSimpleScalableLoadOrStore()

bool isSimpleScalableLoadOrStore ( const LSBaseSDNode * LSNode, EVT VT )

static

Definition at line 7942 of file AArch64ISelLowering.cpp.

References llvm::MemSDNode::getMemoryVT(), llvm::LSBaseSDNode::getOffset(), llvm::EVT::isScalableVector(), llvm::EVT::isSimple(), llvm::MemSDNode::isSimple(), llvm::SDValue::isUndef(), and llvm::LSBaseSDNode::isUnindexed().

Referenced by tryLowerMultiVectorLoad(), and tryLowerMultiVectorStore().

isSlideWithZerosMask()

SDValue isSlideWithZerosMask ( ArrayRef< int > M, EVT VT, SDValue V1, SDValue V2, unsigned & ShiftAmount, bool & IsRightShift, unsigned & MatchedLaneSize )

static

Definition at line 15410 of file AArch64ISelLowering.cpp.

References checkLaneSlide(), llvm::ShuffleVectorSDNode::commuteMask(), llvm::SDValue::getNode(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getSizeInBits(), llvm::EVT::getVectorNumElements(), llvm::ISD::isBuildVectorAllZeros(), SDValue(), and llvm::V1.

isSupportedExtend()

bool isSupportedExtend ( SDValue V )

static

Definition at line 4372 of file AArch64ISelLowering.cpp.

References llvm::ISD::AND, llvm::dyn_cast(), llvm::ISD::SIGN_EXTEND_INREG, uint64_t, and V.

Referenced by getCmpOperandFoldingProfit().

IsSVECntIntrinsic()

bool IsSVECntIntrinsic ( SDValue S )

static

Definition at line 21399 of file AArch64ISelLowering.cpp.

References getIntrinsicID(), and llvm::SDValue::getNode().

Referenced by performMulCombine().

isUnpackedType()

bool isUnpackedType ( EVT VT, SelectionDAG & DAG )

inlinestatic

Returns true if the conceptual representation for VT does not map directly to its physical register representation, meaning there are gaps between elements in the register.

In practice, the vector elements will be strided by a power of two and placed starting from lane 0. For example, nxv8i1 or nxv2f32 are unpacked types.

Precondition

VT is a legal type.

Definition at line 215 of file AArch64ISelLowering.cpp.

References assert(), isPackedPredicateType(), isPackedVectorType(), and llvm::EVT::isScalableVector().

Referenced by performConcatVectorsCombine().

isValidImmForSVEVecImmAddrMode() [1/2]

bool isValidImmForSVEVecImmAddrMode ( SDValue Offset, unsigned ScalarSizeInBytes )

static

Check if the value of Offset represents a valid immediate for the SVE gather load/prefetch and scatter store instructiona with vector base and immediate offset addressing mode:

 [<Zn>.[S|D]{, #<imm>}]

where <imm> = sizeof(<T>) * k, for k = 0, 1, ..., 31.

Definition at line 30945 of file AArch64ISelLowering.cpp.

References llvm::dyn_cast(), llvm::ConstantSDNode::getZExtValue(), isValidImmForSVEVecImmAddrMode(), and llvm::Offset.

isValidImmForSVEVecImmAddrMode() [2/2]

bool isValidImmForSVEVecImmAddrMode ( unsigned OffsetInBytes, unsigned ScalarSizeInBytes )

static

Check if the value of OffsetInBytes can be used as an immediate for the gather load/prefetch and scatter store instructions with vector base and immediate offset addressing mode:

 [<Zn>.[S|D]{, #<imm>}]

where <imm> = sizeof(<T>) * k, for k = 0, 1, ..., 31.

Definition at line 30925 of file AArch64ISelLowering.cpp.

Referenced by combineSVEPrefetchVecBaseImmOff(), isValidImmForSVEVecImmAddrMode(), performGatherLoadCombine(), and performScatterStoreCombine().

isVectorizedBinOp()

bool isVectorizedBinOp ( unsigned Opcode )

static

Definition at line 2728 of file AArch64ISelLowering.cpp.

Referenced by performConcatVectorsCombine().

isVShiftLImm()

bool isVShiftLImm ( SDValue Op, EVT VT, bool isLong, int64_t & Cnt )

static

isVShiftLImm - Check if this is a valid build_vector for the immediate operand of a vector shift left operation.

That value must be in the range: 0 <= Value < ElementBits for a left shift; or 0 <= Value <= ElementBits for a long left shift.

Definition at line 18045 of file AArch64ISelLowering.cpp.

References assert(), llvm::EVT::getScalarSizeInBits(), getVShiftImm(), and llvm::EVT::isVector().

Referenced by LowerShift(), llvm::ARMTargetLowering::PerformIntrinsicCombine(), and PerformShiftCombine().

isVShiftRImm()

bool isVShiftRImm ( SDValue Op, EVT VT, bool isNarrow, int64_t & Cnt )

static

isVShiftRImm - Check if this is a valid build_vector for the immediate operand of a vector shift right operation.

The value must be in the range: 1 <= Value <= ElementBits for a right shift; or

Definition at line 18056 of file AArch64ISelLowering.cpp.

References assert(), llvm::EVT::getScalarSizeInBits(), getVShiftImm(), and llvm::EVT::isVector().

Referenced by LowerShift(), llvm::ARMTargetLowering::PerformIntrinsicCombine(), and PerformShiftCombine().

isWideDUPMask()

bool isWideDUPMask ( ArrayRef< int > M, EVT VT, unsigned BlockSize, unsigned & DupLaneOp )

static

Check if a vector shuffle corresponds to a DUP instructions with a larger element width than the vector lane type.

If that is the case the function returns true and writes the value of the DUP instruction lane operand into DupLaneOp

Definition at line 15279 of file AArch64ISelLowering.cpp.

References assert(), llvm::SmallVectorTemplateCommon< T, typename >::begin(), BlockSize, llvm::SmallVectorTemplateCommon< T, typename >::end(), llvm::find_if(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getSizeInBits(), llvm::EVT::getVectorNumElements(), and I.

isZeroExtended()

bool isZeroExtended ( SDValue N, SelectionDAG & DAG )

static

Definition at line 5916 of file AArch64ISelLowering.cpp.

References llvm::ISD::ANY_EXTEND, isExtendedBUILD_VECTOR(), N, and llvm::ISD::ZERO_EXTEND.

Referenced by isAddSubZExt(), isAddSubZExt(), LowerMUL(), performMulCombine(), and selectUmullSmull().

isZeroingInactiveLanes()

bool isZeroingInactiveLanes ( SDValue Op )

static

Definition at line 279 of file AArch64ISelLowering.cpp.

References llvm::ISD::GET_ACTIVE_LANE_MASK, llvm::ISD::INTRINSIC_WO_CHAIN, and llvm::ISD::SPLAT_VECTOR.

Referenced by getPTest(), and getSVEPredicateBitCast().

isZerosVector()

bool isZerosVector ( const SDNode * N )

static

isZerosVector - Check whether SDNode N is a zero-filled vector.

Definition at line 3744 of file AArch64ISelLowering.cpp.

References llvm::ISD::BITCAST, llvm::ISD::isConstantSplatVectorAllZeros(), llvm::isNullConstant(), llvm::isNullFPConstant(), and N.

Referenced by GenerateTBL(), performAddDotCombine(), performSETCCCombine(), performSetccMergeZeroCombine(), performSetCCPunpkCombine(), performSubNegAndOneCombine(), performUnpackCombine(), and performVectorDeinterleaveCombine().

legalizeSVEGatherPrefetchOffsVec()

SDValue legalizeSVEGatherPrefetchOffsVec ( SDNode * N, SelectionDAG & DAG )

static

Legalize the gather prefetch (scalar + vector addressing mode) when the offset vector is an unpacked 32-bit scalable vector. The other cases (Offset != nxv2i32) do not need legalization.

Definition at line 31324 of file AArch64ISelLowering.cpp.

References AbstractManglingParser< Derived, Alloc >::Ops, llvm::ISD::ANY_EXTEND, DL, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getVTList(), N, llvm::Offset, and SDValue().

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

lookThroughSignExtension()

std::pair< SDValue, uint64_t > lookThroughSignExtension ( SDValue Val )

static

Definition at line 12106 of file AArch64ISelLowering.cpp.

References llvm::cast(), llvm::EVT::getFixedSizeInBits(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValueSizeInBits(), llvm::SDNode::getValueType(), llvm::ISD::SIGN_EXTEND, and llvm::ISD::SIGN_EXTEND_INREG.

LowerADDRSPACECAST()

SDValue LowerADDRSPACECAST ( SDValue Op, SelectionDAG & DAG )

static

Definition at line 7794 of file AArch64ISelLowering.cpp.

References assert(), llvm::cast(), DL, llvm::SelectionDAG::getAnyExtOrTrunc(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getNode(), llvm::DataLayout::getPointerSize(), llvm::SelectionDAG::getZeroExtendInReg(), N, llvm::ARM64AS::PTR32_SPTR, llvm::ARM64AS::PTR32_UPTR, llvm::ISD::SIGN_EXTEND, and llvm::ISD::ZERO_EXTEND.

Referenced by llvm::AArch64TargetLowering::LowerOperation(), llvm::X86TargetLowering::LowerOperation(), and llvm::X86TargetLowering::ReplaceNodeResults().

lowerADDSUBO_CARRY()

SDValue lowerADDSUBO_CARRY ( SDValue Op, SelectionDAG & DAG, unsigned Opcode, bool IsSigned )

static

Definition at line 4791 of file AArch64ISelLowering.cpp.

References carryFlagToValue(), DL, FlagsVT, llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), llvm::SDValue::getValue(), llvm::SelectionDAG::getVTList(), overflowFlagToValue(), SDValue(), and valueToCarryFlag().

Referenced by llvm::AArch64TargetLowering::LowerOperation().

LowerBRCOND()

SDValue LowerBRCOND ( SDValue Op, SelectionDAG & DAG )

static

Definition at line 8301 of file AArch64ISelLowering.cpp.

References CC, Cond, DL, emitConjunction(), getCondCode(), llvm::SelectionDAG::getNode(), and SDValue().

Referenced by llvm::AArch64TargetLowering::LowerOperation().

LowerFLDEXP()

SDValue LowerFLDEXP ( SDValue Op, SelectionDAG & DAG )

static

Definition at line 8355 of file AArch64ISelLowering.cpp.

References llvm::EVT::changeVectorElementType(), DL, llvm::ISD::FP_EXTEND, llvm::ISD::FP_ROUND, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getExtractVectorElt(), llvm::SelectionDAG::getInsertVectorElt(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getPOISON(), llvm::SelectionDAG::getTargetConstant(), llvm::ISD::INTRINSIC_WO_CHAIN, SDValue(), llvm::ISD::SIGN_EXTEND, and X.

Referenced by llvm::AArch64TargetLowering::LowerOperation(), and llvm::X86TargetLowering::LowerOperation().

LowerFunnelShift()

SDValue LowerFunnelShift ( SDValue Op, SelectionDAG & DAG )

static

Definition at line 8319 of file AArch64ISelLowering.cpp.

References DL, llvm::dyn_cast(), llvm::ISD::FSHL, llvm::ISD::FSHR, llvm::SelectionDAG::getConstant(), llvm::MVT::getFixedSizeInBits(), llvm::SelectionDAG::getNode(), llvm::SDValue::getValueType(), and SDValue().

Referenced by llvm::AArch64TargetLowering::LowerOperation(), and llvm::X86TargetLowering::LowerOperation().

lowerIntNeonIntrinsic()

SDValue lowerIntNeonIntrinsic ( SDValue Op, unsigned Opcode, SelectionDAG & DAG, bool LastOperandIsImm = false )

static

Definition at line 4816 of file AArch64ISelLowering.cpp.

References assert(), DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), I, llvm::SmallVectorTemplateBase< T, bool >::push_back(), and SDValue().

LowerNTStore()

SDValue LowerNTStore ( StoreSDNode * StoreNode, EVT VT, EVT MemVT, const SDLoc & DL, SelectionDAG & DAG )

static

Definition at line 7867 of file AArch64ISelLowering.cpp.

References assert(), DL, llvm::MemSDNode::getAlign(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getExtractSubvector(), llvm::EVT::getHalfNumVectorElementsVT(), llvm::SelectionDAG::getMemIntrinsicNode(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getMemoryVT(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getSizeInBits(), llvm::EVT::getTypeForEVT(), llvm::StoreSDNode::getValue(), llvm::EVT::getVectorElementCount(), llvm::SelectionDAG::getVTList(), llvm::Hi, isLegalNTStore(), llvm::DataLayout::isLittleEndian(), llvm::MemSDNode::isNonTemporal(), llvm::EVT::isVector(), Lo, SDValue(), and llvm::SelectionDAG::shouldOptForSize().

lowerPredCounterWhile()

SDValue lowerPredCounterWhile ( Intrinsic::ID IID, SDNode * N, SelectionDAG & DAG )

static

Lower a predicate-as-counter while intrinsic to an AArch64ISD node.

Definition at line 25666 of file AArch64ISelLowering.cpp.

References DL, FlagsVT, llvm::SelectionDAG::getNode(), getPredCounterWhileElementSize(), getPredCounterWhileOpcode(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getVTList(), and N.

Referenced by performIntrinsicCombine().

lowerPredicateInterleaveWithPackedVectors()

SDValue lowerPredicateInterleaveWithPackedVectors ( SDValue Op, SelectionDAG & DAG )

static

Definition at line 35601 of file AArch64ISelLowering.cpp.

References assert(), DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), getPackedSVEVectorVT(), llvm::SelectionDAG::getSetCC(), llvm::SDValue::getValue(), llvm::EVT::getVectorElementCount(), llvm::EVT::getVectorElementType(), llvm::SelectionDAG::getVTList(), I, llvm::EVT::isScalableVector(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), Results, llvm::ISD::SETNE, V, Zero, and llvm::ISD::ZERO_EXTEND.

LowerPREFETCH()

SDValue LowerPREFETCH ( SDValue Op, SelectionDAG & DAG )

static

Definition at line 4874 of file AArch64ISelLowering.cpp.

References assert(), DL, llvm::SelectionDAG::getNode(), and llvm::SelectionDAG::getTargetConstant().

Referenced by llvm::AArch64TargetLowering::LowerOperation(), llvm::ARMTargetLowering::LowerOperation(), llvm::RISCVTargetLowering::LowerOperation(), and llvm::X86TargetLowering::LowerOperation().

LowerPtrAuthGlobalAddressStatically()

SDValue LowerPtrAuthGlobalAddressStatically ( SDValue TGA, SDLoc DL, EVT VT, AArch64PACKey::ID KeyC, SDValue Discriminator, SDValue AddrDiscriminator, SelectionDAG & DAG )

static

Definition at line 12006 of file AArch64ISelLowering.cpp.

References assert(), llvm::cast(), DL, llvm::SelectionDAG::getMachineNode(), llvm::SDValue::getNode(), llvm::SelectionDAG::getTargetConstant(), llvm::InnerAnalysisManagerProxy< AnalysisManagerT, IRUnitT, ExtraArgTs... >::Key, llvm::isNullConstant(), llvm::report_fatal_error(), and SDValue().

LowerSMELdrStr()

SDValue LowerSMELdrStr ( SDValue N, SelectionDAG & DAG, bool IsLoad )

static

Definition at line 6511 of file AArch64ISelLowering.cpp.

References llvm::ISD::ADD, llvm::sampleprof::Base, C(), llvm::cast(), DL, llvm::dyn_cast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getTargetConstant(), llvm::isa(), llvm::ISD::MUL, Mul, N, SDValue(), and llvm::ISD::SIGN_EXTEND.

LowerSVEIntrinsicDUP()

SDValue LowerSVEIntrinsicDUP ( SDNode * N, SelectionDAG & DAG )

static

Definition at line 25199 of file AArch64ISelLowering.cpp.

References llvm::ISD::ANY_EXTEND, DL, llvm::SelectionDAG::getNode(), N, and Pred.

Referenced by performIntrinsicCombine().

LowerSVEIntrinsicEXT()

SDValue LowerSVEIntrinsicEXT ( SDNode * N, SelectionDAG & DAG )

static

Definition at line 25213 of file AArch64ISelLowering.cpp.

References assert(), llvm::ISD::BITCAST, DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getKnownMinValue(), llvm::SelectionDAG::getNode(), llvm::ElementCount::getScalable(), llvm::EVT::getSizeInBits(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorVT(), llvm::EVT::isScalableVector(), llvm::ISD::MUL, N, SDValue(), and llvm::AArch64::SVEBitsPerBlock.

Referenced by performIntrinsicCombine().

LowerSVEIntrinsicIndex()

SDValue LowerSVEIntrinsicIndex ( SDNode * N, SelectionDAG & DAG )

static

Definition at line 25183 of file AArch64ISelLowering.cpp.

References llvm::ISD::ADD, llvm::sampleprof::Base, DL, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getStepVector(), llvm::SDValue::getValueType(), llvm::ISD::MUL, Mul, N, and llvm::ISD::SPLAT_VECTOR.

Referenced by performIntrinsicCombine().

LowerTruncateVectorStore()

SDValue LowerTruncateVectorStore ( SDLoc DL, StoreSDNode * ST, EVT VT, EVT MemVT, SelectionDAG & DAG )

static

Definition at line 7764 of file AArch64ISelLowering.cpp.

References assert(), llvm::ISD::BITCAST, llvm::ISD::CONCAT_VECTORS, DL, llvm::SelectionDAG::getContext(), llvm::EVT::getDoubleNumVectorElementsVT(), llvm::SelectionDAG::getExtractVectorElt(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getPOISON(), llvm::SelectionDAG::getStore(), llvm::EVT::isVector(), and llvm::ISD::TRUNCATE.

LowerVectorMatch()

SDValue LowerVectorMatch ( SDValue Op, SelectionDAG & DAG )

static

Definition at line 6558 of file AArch64ISelLowering.cpp.

References convertFixedMaskToScalableVector(), convertFromScalableVector(), convertToScalableVector(), DL, llvm::SelectionDAG::getBitcast(), getContainerForFixedLengthVector(), llvm::SelectionDAG::getExtractVectorElt(), llvm::EVT::getFixedSizeInBits(), llvm::SelectionDAG::getInsertSubvector(), llvm::SelectionDAG::getInsertVectorElt(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getNode(), getPackedSVEVectorVT(), llvm::SelectionDAG::getPOISON(), llvm::SelectionDAG::getSplatVector(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::MVT::getVectorVT(), llvm::ISD::INTRINSIC_WO_CHAIN, llvm::EVT::is128BitVector(), llvm::EVT::isScalableVector(), llvm::TargetLoweringBase::isTypeLegal(), llvm::ISD::OR, SDValue(), llvm::ISD::SIGN_EXTEND, llvm::SelectionDAG::SplitVector(), llvm::ISD::TRUNCATE, and llvm::ISD::VECTOR_MATCH.

Referenced by llvm::AArch64TargetLowering::LowerOperation().

LowerXALUO()

SDValue LowerXALUO ( SDValue Op, SelectionDAG & DAG )

static

Definition at line 4844 of file AArch64ISelLowering.cpp.

References CC, DL, getAArch64XALUOOp(), getCondCode(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::TargetLoweringBase::isTypeLegal(), and SDValue().

Referenced by LowerMULO(), llvm::AArch64TargetLowering::LowerOperation(), and llvm::X86TargetLowering::LowerOperation().

m_DeinterleavingTbl()

auto m_DeinterleavingTbl ( Instruction *& Tbl, unsigned & Idx )

static

Definition at line 19424 of file AArch64ISelLowering.cpp.

Referenced by foldAdjacentReversesIntoTbls().

m_PredicateAsCounterWhile()

auto m_PredicateAsCounterWhile ( )

static

Definition at line 22846 of file AArch64ISelLowering.cpp.

References llvm::SDPatternMatch::m_AnyOf(), and llvm::SDPatternMatch::m_SpecificOpc().

Referenced by perfomPextFirstTrueVectorCombine().

m_Tbl()

auto m_Tbl ( Instruction *& Tbl, Value *& Mask )

static

Definition at line 19351 of file AArch64ISelLowering.cpp.

References llvm::PatternMatch::m_BitCast(), llvm::PatternMatch::m_Instruction(), llvm::PatternMatch::m_Intrinsic(), llvm::MIPatternMatch::m_OneUse(), and llvm::PatternMatch::m_Value().

Referenced by deinterleaving_tbl_match::match(), and uitofp_tbl_match::match().

m_UIToFPTbl()

auto m_UIToFPTbl ( Instruction *& Tbl, Value *& Mask )

static

Definition at line 19378 of file AArch64ISelLowering.cpp.

Referenced by deinterleaving_tbl_match::match().

matchZeroSelectArm()

SDValue matchZeroSelectArm ( SDValue TVal, SDValue FVal, ISD::CondCode & RequiredCC )

static

Definition at line 30484 of file AArch64ISelLowering.cpp.

References llvm::SDValue::getNode(), llvm::ISD::isConstantSplatVectorAllZeros(), SDValue(), llvm::ISD::SETUGE, and llvm::ISD::SETULT.

Referenced by performVSelectMaskedShiftCombine().

mayTailCallThisCC()

bool mayTailCallThisCC ( CallingConv::ID CC )

static

Return true if we might ever do TCO for calls with this calling convention.

Definition at line 10005 of file AArch64ISelLowering.cpp.

References llvm::CallingConv::AArch64_SVE_VectorCall, llvm::CallingConv::C, CC, llvm::CallingConv::Fast, llvm::CallingConv::PreserveAll, llvm::CallingConv::PreserveMost, llvm::CallingConv::PreserveNone, llvm::CallingConv::Swift, llvm::CallingConv::SwiftTail, and llvm::CallingConv::Tail.

Referenced by llvm::AArch64CallLowering::isEligibleForTailCallOptimization().

NormalizeBuildVector()

SDValue NormalizeBuildVector ( SDValue Op, SelectionDAG & DAG )

static

Definition at line 16901 of file AArch64ISelLowering.cpp.

References AbstractManglingParser< Derived, Alloc >::Ops, assert(), llvm::ISD::BUILD_VECTOR, DL, llvm::dyn_cast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SelectionDAG::getPOISON(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::isFloatingPoint(), llvm::ISD::POISON, llvm::ISD::TRUNCATE, and llvm::ISD::UNDEF.

numberOfInstrToLoadImm()

unsigned numberOfInstrToLoadImm ( const APInt & C )

static

Definition at line 3933 of file AArch64ISelLowering.cpp.

References C(), llvm::AArch64_IMM::expandMOVImm(), Imm, llvm::SmallVectorTemplateCommon< T, typename >::size(), and uint64_t.

Referenced by getAArch64Cmp().

optimizeBitTest()

SDValue optimizeBitTest ( SDLoc DL, SDValue Op, SDValue Chain, SDValue Dest, unsigned Opcode, SelectionDAG & DAG )

static

Definition at line 12121 of file AArch64ISelLowering.cpp.

References llvm::ISD::AND, DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::isa(), llvm::isPowerOf2_64(), llvm::Log2_64(), SDValue(), llvm::ISD::SHL, llvm::ISD::SRL, llvm::Test, and uint64_t.

optimizeBrk()

SDValue optimizeBrk ( SDNode * N, SelectionDAG & DAG )

static

Definition at line 6372 of file AArch64ISelLowering.cpp.

References llvm::ISD::ADD, DL, llvm::SelectionDAG::getNode(), getSVEPredicateBitCast(), llvm::SelectionDAG::getTargetConstant(), llvm::ISD::INTRINSIC_WO_CHAIN, llvm::isOneOrOneSplat(), llvm::EVT::isScalableVector(), llvm::isZeroOrZeroSplat(), N, SDValue(), and llvm::Upper.

Referenced by performActiveLaneMaskCombine().

optimizeIncrementingWhile()

SDValue optimizeIncrementingWhile ( SDNode * N, SelectionDAG & DAG, bool IsSigned, bool IsEqual )

static

Definition at line 6320 of file AArch64ISelLowering.cpp.

References DL, llvm::APInt::getBitWidth(), llvm::SelectionDAG::getConstant(), llvm::AArch64Subtarget::getMinSVEVectorSizeInBits(), getPTrue(), llvm::SelectionDAG::getSubtarget(), llvm::getSVEPredPatternFromNumElements(), llvm::APInt::getZExtValue(), llvm::ISD::INTRINSIC_WO_CHAIN, llvm::isa(), N, llvm::APInt::sadd_ov(), SDValue(), llvm::APInt::uadd_ov(), X, and Y.

Referenced by performActiveLaneMaskCombine().

optimizeLogicalImm()

bool optimizeLogicalImm ( SDValue Op, unsigned Size, uint64_t Imm, const APInt & Demanded, TargetLowering::TargetLoweringOpt & TLO, unsigned NewOpc )

static

Definition at line 2746 of file AArch64ISelLowering.cpp.

References assert(), llvm::TargetLowering::TargetLoweringOpt::CombineTo(), llvm::TargetLowering::TargetLoweringOpt::DAG, DL, llvm::AArch64_AM::encodeLogicalImmediate(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getMachineNode(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetConstant(), llvm::APInt::getZExtValue(), llvm::Hi, Imm, llvm::AArch64_AM::isLogicalImmediate(), llvm::isShiftedMask_64(), SDValue(), llvm::Size, and uint64_t.

Referenced by llvm::AArch64TargetLowering::targetShrinkDemandedConstant(), and llvm::ARMTargetLowering::targetShrinkDemandedConstant().

overflowFlagToValue()

SDValue overflowFlagToValue ( SDValue Glue, EVT VT, SelectionDAG & DAG )

static

Definition at line 4780 of file AArch64ISelLowering.cpp.

References assert(), CC, DL, getCondCode(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getResNo(), llvm::AArch64CC::VS, and Zero.

Referenced by LowerADDSUBO_CARRY(), and lowerADDSUBO_CARRY().

parseConstraintCode()

AArch64CC::CondCode parseConstraintCode ( llvm::StringRef Constraint )

static

Definition at line 14347 of file AArch64ISelLowering.cpp.

References llvm::StringSwitch< T, R >::Case(), Cond, llvm::StringSwitch< T, R >::Default(), llvm::AArch64CC::EQ, llvm::AArch64CC::GE, llvm::AArch64CC::GT, llvm::AArch64CC::HI, llvm::AArch64CC::HS, llvm::AArch64CC::Invalid, llvm::AArch64CC::LE, llvm::AArch64CC::LO, llvm::AArch64CC::LS, llvm::AArch64CC::LT, llvm::AArch64CC::MI, llvm::AArch64CC::NE, llvm::AArch64CC::PL, llvm::AArch64CC::VC, and llvm::AArch64CC::VS.

Referenced by llvm::X86TargetLowering::getConstraintType(), llvm::X86TargetLowering::getRegForInlineAsmConstraint(), and llvm::X86TargetLowering::LowerAsmOutputForConstraint().

parsePredicateConstraint()

std::optional< PredicateConstraint > parsePredicateConstraint ( StringRef Constraint )

static

Definition at line 14292 of file AArch64ISelLowering.cpp.

References llvm::StringSwitch< T, R >::Case(), llvm::StringSwitch< T, R >::Default(), Upa, Uph, and Upl.

parseReducedGprConstraint()

std::optional< ReducedGprConstraint > parseReducedGprConstraint ( StringRef Constraint )

static

Definition at line 14323 of file AArch64ISelLowering.cpp.

References llvm::StringSwitch< T, R >::Case(), llvm::StringSwitch< T, R >::Default(), Uci, and Ucj.

parseSVERegAsConstraint()

std::optional< std::pair< unsigned, const TargetRegisterClass * > > parseSVERegAsConstraint ( StringRef Constraint )

static

Definition at line 14269 of file AArch64ISelLowering.cpp.

References llvm::StringRef::drop_front(), llvm::StringRef::ends_with(), llvm::StringRef::getAsInteger(), llvm::StringRef::size(), llvm::StringRef::starts_with(), llvm::StringRef::substr(), and V.

perfomPextFirstTrueVectorCombine()

SDValue perfomPextFirstTrueVectorCombine ( SDNode * N, TargetLowering::DAGCombinerInfo & DCI, const AArch64Subtarget * Subtarget )

static

Folds extracting the first lane from the first segment of a predicate-as-counter while to a conditional set (CSET) based on the "FIRST_ACTIVE" status flag from the while.

while, flags = WHILE_*_PRED_COUNTER .. ; predicate-as-counter while first.segment = pext(while, 0) ; predicate extract of segment 0 first.active = extract_elt(first.segment, 0) ; extract first lane

->

while, flags = WHILE_*_PRED_COUNTER .. ; predicate-as-counter while first.active = cset(flags, FIRST_ACTIVE)

Definition at line 22871 of file AArch64ISelLowering.cpp.

References assert(), llvm::TargetLowering::DAGCombinerInfo::DAG, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::AArch64CC::FIRST_ACTIVE, llvm::SDValue::getNode(), llvm::SDNode::getOperand(), getSETCC(), llvm::SDValue::getValueType(), hasSVEMultiVectorOps(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::isNullConstant(), llvm::EVT::isScalableVectorOf(), llvm::SDPatternMatch::m_IntrinsicWOChain(), m_PredicateAsCounterWhile(), llvm::PatternMatch::m_Zero(), N, llvm::SDPatternMatch::sd_match(), SDValue(), and Subtarget.

Referenced by performExtractVectorEltCombine().

performActiveLaneMaskCombine()

SDValue performActiveLaneMaskCombine ( SDNode * N, TargetLowering::DAGCombinerInfo & DCI, const AArch64Subtarget * ST )

static

Definition at line 20922 of file AArch64ISelLowering.cpp.

References assert(), llvm::TargetLowering::DAGCombinerInfo::CombineTo(), llvm::ISD::CONCAT_VECTORS, llvm::count_if(), llvm::TargetLowering::DAGCombinerInfo::DAG, llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::divideCoefficientBy(), DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::ISD::GET_ACTIVE_LANE_MASK, llvm::SelectionDAG::getContext(), llvm::EVT::getDoubleNumVectorElementsVT(), llvm::SelectionDAG::getElementCount(), llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getKnownMinValue(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetConstant(), llvm::EVT::getVectorElementCount(), I, llvm::ISD::INTRINSIC_WO_CHAIN, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), N, llvm::Offset, optimizeBrk(), optimizeIncrementingWhile(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), SDValue(), llvm::ISD::UADDSAT, and llvm::ISD::ZERO_EXTEND.

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performAddAddCombine()

SDValue performAddAddCombine ( SDNode * N, SelectionDAG & DAG )

static

Definition at line 24894 of file AArch64ISelLowering.cpp.

References A(), llvm::ISD::ADD, B(), DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::hasOneUse(), llvm::EVT::isScalarInteger(), N, SDValue(), and llvm::ISD::SHL.

Referenced by performAddSubCombine().

performAddCombineForShiftedOperands()

SDValue performAddCombineForShiftedOperands ( SDNode * N, SelectionDAG & DAG )

static

Definition at line 24334 of file AArch64ISelLowering.cpp.

References llvm::ISD::ADD, DL, llvm::SelectionDAG::getNode(), isOpcWithIntImmediate(), LHS, N, performAddCombineSubShift(), RHS, SDValue(), llvm::ISD::SHL, and uint64_t.

Referenced by performAddSubCombine().

performAddCombineSubShift()

SDValue performAddCombineSubShift ( SDNode * N, SDValue SUB, SDValue Z, SelectionDAG & DAG )

static

Definition at line 24306 of file AArch64ISelLowering.cpp.

References llvm::ISD::ADD, DL, llvm::SelectionDAG::getNode(), llvm::isa(), isExtendOrShiftOperand(), N, SDValue(), llvm::ISD::SUB, and Y.

Referenced by performAddCombineForShiftedOperands().

performAddCSelIntoCSinc()

SDValue performAddCSelIntoCSinc ( SDNode * N, SelectionDAG & DAG )

static

Perform the scalar expression combine in the form of: CSEL(c, 1, cc) + b => CSINC(b+c, b, cc) CSNEG(c, -1, cc) + b => CSINC(b+c, b, cc)

Definition at line 23677 of file AArch64ISelLowering.cpp.

References llvm::ISD::ADD, assert(), C(), llvm::cast(), DL, llvm::dyn_cast(), llvm::SelectionDAG::getAllOnesConstant(), llvm::ConstantSDNode::getAPIntValue(), getCondCode(), llvm::SelectionDAG::getConstant(), llvm::AArch64CC::getInvertedCondCode(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::ConstantSDNode::isAllOnes(), llvm::ConstantSDNode::isOne(), llvm::EVT::isScalarInteger(), LHS, N, RHS, SDValue(), std::swap(), and TLI.

Referenced by performAddSubCombine().

performAddDotCombine()

SDValue performAddDotCombine ( SDNode * N, SelectionDAG & DAG )

static

Definition at line 23750 of file AArch64ISelLowering.cpp.

References A(), llvm::ISD::ADD, llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), isZerosVector(), N, SDValue(), and std::swap().

Referenced by performAddSubCombine().

performAddMulZextCombine()

SDValue performAddMulZextCombine ( SDNode * N, TargetLowering::DAGCombinerInfo & DCI )

static

Definition at line 24770 of file AArch64ISelLowering.cpp.

References llvm::ISD::ADD, C(), llvm::TargetLowering::DAGCombinerInfo::DAG, DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::isa(), llvm::TargetLowering::DAGCombinerInfo::isAfterLegalizeDAG(), llvm::ISD::MUL, Mul, N, llvm::Other, SDValue(), llvm::ISD::SUB, std::swap(), and llvm::ISD::ZERO_EXTEND.

Referenced by performAddSubCombine().

performAddSubCarryCombine()

SDValue performAddSubCarryCombine ( SDNode * N, SelectionDAG & DAG, unsigned NewOpcode )

static

Takes advantage of the negation of the second operand by the with-carry-in instructions to fold:

AddWithCarry(X, -1, C) => SubWithCarry(X, 0, C) SubWithCarry(X, -1, C) => AddWithCarry(X, 0, C)

This allows using the zero register (xzr/wzr) to encode the immediate operand (rather than a MOV to a GPR).

Definition at line 23984 of file AArch64ISelLowering.cpp.

References DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::isAllOnesConstant(), N, RHS, and SDValue().

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performAddSubCombine()

SDValue performAddSubCombine ( SDNode * N, TargetLowering::DAGCombinerInfo & DCI )

static

Definition at line 24983 of file AArch64ISelLowering.cpp.

References llvm::TargetLowering::DAGCombinerInfo::DAG, N, performAddAddCombine(), performAddCombineForShiftedOperands(), performAddCSelIntoCSinc(), performAddDotCombine(), performAddMulZextCombine(), performAddSubIntoVectorOp(), performAddSubLongCombine(), performAddTruncShiftCombine(), performAddUADDVCombine(), performAddWithSBCCombine(), performConditionalSubCombine(), performExtBinopLoadFold(), performNegCSelCombine(), performNegSignMaskedAndCombine(), performSetccAddFolding(), performSubAddMULCombine(), performSubNegAndOneCombine(), performSubWithBorrowCombine(), performSVEMulAddSubCombine(), and performVectorExtCombine().

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performAddSubIntoVectorOp()

SDValue performAddSubIntoVectorOp ( SDNode * N, SelectionDAG & DAG )

static

Definition at line 24495 of file AArch64ISelLowering.cpp.

References DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getExtractVectorElt(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::TargetLoweringBase::isOperationExpand(), llvm::ISD::LOAD, N, llvm::ISD::SCALAR_TO_VECTOR, and SDValue().

Referenced by performAddSubCombine().

performAddSubLongCombine()

SDValue performAddSubLongCombine ( SDNode * N, TargetLowering::DAGCombinerInfo & DCI )

static

Definition at line 23870 of file AArch64ISelLowering.cpp.

References llvm::TargetLowering::DAGCombinerInfo::DAG, llvm::SelectionDAG::getNode(), llvm::MVT::is128BitVector(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), isEssentiallyExtractHighSubvector(), LHS, N, reassociateAddSubLong(), RHS, SDValue(), llvm::ISD::SIGN_EXTEND, tryExtendDUPToExtractHigh(), and llvm::ISD::ZERO_EXTEND.

Referenced by performAddSubCombine().

performAddTruncShiftCombine()

SDValue performAddTruncShiftCombine ( SDNode * N, SelectionDAG & DAG )

static

Definition at line 24851 of file AArch64ISelLowering.cpp.

References llvm::ISD::ADD, DL, llvm::SDValue::getConstantOperandVal(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::SelectionDAG::getTargetConstant(), llvm::PatternMatch::m_Add(), llvm::PatternMatch::m_Trunc(), llvm::PatternMatch::m_Value(), N, llvm::SDPatternMatch::sd_match(), SDValue(), std::swap(), and llvm::ISD::TRUNCATE.

Referenced by performAddSubCombine().

performAddUADDVCombine()

SDValue performAddUADDVCombine ( SDNode * N, SelectionDAG & DAG )

static

Definition at line 23639 of file AArch64ISelLowering.cpp.

References llvm::ISD::ADD, DL, llvm::dyn_cast(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getExtractVectorElt(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getValueType(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::isScalarInteger(), LHS, N, RHS, and SDValue().

Referenced by performAddSubCombine().

performAddWithSBCCombine()

SDValue performAddWithSBCCombine ( SDNode * N, SelectionDAG & DAG )

static

Definition at line 24926 of file AArch64ISelLowering.cpp.

References llvm::ISD::ADD, C(), DL, llvm::SelectionDAG::getNegative(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::hasOneUse(), llvm::isNullConstant(), N, SDValue(), and std::swap().

Referenced by performAddSubCombine().

performANDCombine()

SDValue performANDCombine ( SDNode * N, TargetLowering::DAGCombinerInfo & DCI )

static

Definition at line 22567 of file AArch64ISelLowering.cpp.

References llvm::SelectionDAG::computeKnownBits(), llvm::TargetLowering::DAGCombinerInfo::DAG, llvm::dyn_cast(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), I, llvm::EVT::is128BitVector(), llvm::EVT::is64BitVector(), llvm::EVT::isScalableVector(), llvm::TargetLoweringBase::isTypeLegal(), llvm::Known, LHS, N, performANDORCSELCombine(), performANDORDUPNOTCombine(), performANDSETCCCombine(), performSVEAndCombine(), resolveBuildVector(), RHS, SDValue(), tryAdvSIMDModImm16(), and tryAdvSIMDModImm32().

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine(), llvm::LoongArchTargetLowering::PerformDAGCombine(), llvm::MipsSETargetLowering::PerformDAGCombine(), llvm::MipsTargetLowering::PerformDAGCombine(), and llvm::RISCVTargetLowering::PerformDAGCombine().

performANDORCSELCombine()

SDValue performANDORCSELCombine ( SDNode * N, SelectionDAG & DAG )

static

Definition at line 22176 of file AArch64ISelLowering.cpp.

References llvm::ISD::AND, DL, llvm::dyn_cast(), FlagsVT, getCondCode(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getConstantOperandVal(), llvm::AArch64CC::getInvertedCondCode(), llvm::SelectionDAG::getNode(), llvm::AArch64CC::getNZCVToSatisfyCondCode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getValueType(), llvm::SDNode::hasOneUse(), llvm::isNullConstant(), llvm::isOneConstant(), N, SDValue(), and std::swap().

Referenced by performANDCombine(), performANDSCombine(), and performORCombine().

performANDORDUPNOTCombine()

SDValue performANDORDUPNOTCombine ( SDNode * N, SelectionDAG & DAG )

static

Definition at line 22344 of file AArch64ISelLowering.cpp.

References llvm::ISD::AND, assert(), DL, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNOT(), llvm::SelectionDAG::getSplat(), llvm::PatternMatch::m_c_BinOp(), llvm::PatternMatch::m_InsertElt(), llvm::MIPatternMatch::m_Not(), llvm::PatternMatch::m_Poison(), llvm::PatternMatch::m_Shuffle(), llvm::PatternMatch::m_Value(), llvm::PatternMatch::m_Zero(), N, Opc, llvm::ISD::OR, llvm::SDPatternMatch::sd_match(), SDValue(), X, and Y.

Referenced by performANDCombine(), and performORCombine().

performANDSCombine()

SDValue performANDSCombine ( SDNode * N, TargetLowering::DAGCombinerInfo & DCI )

static

Definition at line 30148 of file AArch64ISelLowering.cpp.

References llvm::ISD::AND, llvm::TargetLowering::DAGCombinerInfo::DAG, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), N, performANDORCSELCombine(), performFlagSettingCombine(), and SDValue().

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performANDSETCCCombine()

SDValue performANDSETCCCombine ( SDNode * N, TargetLowering::DAGCombinerInfo & DCI )

static

Definition at line 22528 of file AArch64ISelLowering.cpp.

References CC, llvm::TargetLowering::DAGCombinerInfo::DAG, DL, emitConjunction(), getCondCode(), llvm::SelectionDAG::getConstant(), llvm::AArch64CC::getInvertedCondCode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValueType(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), N, SDValue(), llvm::ISD::SELECT, and llvm::ISD::SETCC.

Referenced by performANDCombine().

performBICiCombine()

SDValue performBICiCombine ( SDNode * N, SelectionDAG & DAG, TargetLowering::DAGCombinerInfo & DCI )

static

Definition at line 21293 of file AArch64ISelLowering.cpp.

References llvm::APInt::getAllOnes(), llvm::SelectionDAG::getTargetLoweringInfo(), N, SDValue(), and llvm::TargetLowering::SimplifyDemandedBits().

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performBRCONDCombine()

SDValue performBRCONDCombine ( SDNode * N, TargetLowering::DAGCombinerInfo & DCI, SelectionDAG & DAG )

static

Definition at line 29395 of file AArch64ISelLowering.cpp.

References assert(), CC, llvm::TargetLowering::DAGCombinerInfo::CombineTo(), DL, llvm::AArch64CC::EQ, llvm::SDNode::getAsZExtVal(), getCondCode(), getCSETCondCode(), llvm::MachineFunction::getFunction(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::Function::hasFnAttribute(), llvm::isa(), isCMP(), llvm::isNullConstant(), llvm::isOneConstant(), LHS, N, llvm::AArch64CC::NE, performCONDCombine(), RHS, SDValue(), llvm::ISD::SHL, llvm::ISD::SRA, llvm::ISD::SRL, and std::swap().

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performBSPExpandForSVE()

SDValue performBSPExpandForSVE ( SDNode * N, SelectionDAG & DAG, const AArch64Subtarget * Subtarget )

static

Definition at line 31499 of file AArch64ISelLowering.cpp.

References llvm::ISD::AND, DL, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNOT(), llvm::EVT::isScalableVector(), N, llvm::ISD::OR, SDValue(), and Subtarget.

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performBuildShuffleExtendCombine()

SDValue performBuildShuffleExtendCombine ( SDValue BV, SelectionDAG & DAG )

static

Combines a buildvector(sext/zext) or shuffle(sext/zext, undef) node pattern into sext/zext(buildvector) or sext/zext(shuffle) making use of the vector SExt/ZExt rather than the scalar SExt/ZExt.

Definition at line 21482 of file AArch64ISelLowering.cpp.

References llvm::ISD::AND, llvm::ISD::ANY_EXTEND, llvm::ISD::AssertSext, llvm::ISD::AssertZext, llvm::ISD::BUILD_VECTOR, calculatePreExtendType(), llvm::cast(), llvm::EVT::changeVectorElementType(), DL, llvm::drop_begin(), llvm::SelectionDAG::getAnyExtOrTrunc(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getPOISON(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getVectorShuffle(), llvm::SDValue::isUndef(), Opc, llvm::SDNode::ops(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), SDValue(), llvm::ISD::SIGN_EXTEND, llvm::ISD::SIGN_EXTEND_INREG, llvm::ISD::VECTOR_SHUFFLE, and llvm::ISD::ZERO_EXTEND.

Referenced by performMulVectorExtendCombine().

performBuildVectorCombine()

SDValue performBuildVectorCombine ( SDNode * N, TargetLowering::DAGCombinerInfo & DCI, SelectionDAG & DAG )

static

Definition at line 23995 of file AArch64ISelLowering.cpp.

References llvm::ISD::ANY_EXTEND, llvm::EVT::changeVectorElementType(), Concat, llvm::ISD::CONCAT_VECTORS, DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::ISD::FP_EXTEND, llvm::ISD::FP_ROUND, llvm::SDNode::getConstantOperandVal(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getExtractSubvector(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getPOISON(), llvm::SelectionDAG::getSubtarget(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::EVT::getVectorMinNumElements(), llvm::isa(), llvm::AArch64Subtarget::isNeonAvailable(), llvm::TargetLoweringBase::isTypeLegal(), N, and SDValue().

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performConcatVectorsCombine()

SDValue performConcatVectorsCombine ( SDNode * N, TargetLowering::DAGCombinerInfo & DCI, SelectionDAG & DAG )

static

Definition at line 23031 of file AArch64ISelLowering.cpp.

References AbstractManglingParser< Derived, Alloc >::Ops, llvm::ISD::ADD, llvm::Add, llvm::all_of(), assert(), llvm::ISD::BITCAST, llvm::cast(), CC, llvm::ISD::CONCAT_VECTORS, llvm::TargetLowering::DAGCombinerInfo::DAG, llvm::dbgs(), DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SDNode::getConstantOperandVal(), llvm::SDValue::getConstantOperandVal(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getLoad(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNOT(), llvm::SDNode::getOpcode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getPOISON(), llvm::EVT::getScalarSizeInBits(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::MVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::EVT::getVectorVT(), llvm::MVT::getVectorVT(), llvm::SDNode::hasOneUse(), Imm, llvm::EVT::is128BitVector(), llvm::isa(), llvm::TargetLowering::DAGCombinerInfo::isAfterLegalizeDAG(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::TargetLoweringBase::isBinOp(), llvm::ISD::isConstantSplatVectorAllOnes(), llvm::SDNode::isOnlyUserOf(), llvm::EVT::isScalableVector(), llvm::SDValue::isUndef(), isUnpackedType(), llvm::MVT::isVector(), isVectorizedBinOp(), LLVM_DEBUG, N, llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), RHS, SDValue(), llvm::ISD::TRUNCATE, uint64_t, V, WidenVector(), X, llvm::ISD::XOR, and Y.

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performCONDCombine()

SDValue performCONDCombine ( SDNode * N, TargetLowering::DAGCombinerInfo & DCI, SelectionDAG & DAG, unsigned CCIndex, unsigned CmpIndex )

static

Definition at line 29289 of file AArch64ISelLowering.cpp.

References AbstractManglingParser< Derived, Alloc >::Ops, llvm::ISD::ADD, llvm::ISD::AND, C(), llvm::cast(), CC, checkValueWidth(), llvm::TargetLowering::DAGCombinerInfo::CombineTo(), DL, llvm::dyn_cast(), llvm::AArch64CC::EQ, llvm::SelectionDAG::getConstant(), llvm::SDNode::getConstantOperandAPInt(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getShiftAmountConstant(), llvm::SDValue::getValue(), llvm::SDNode::getValueType(), llvm::SDNode::getVTList(), llvm::SelectionDAG::getVTList(), llvm::SDNode::hasAnyUseOfValue(), llvm::SDNode::hasOneUse(), llvm::isa(), isEquivalentMaskless(), llvm::AArch64_AM::isLegalArithImmed(), N, llvm::AArch64CC::NE, performSubsToAndsCombine(), llvm::SelectionDAG::ReplaceAllUsesWith(), SDValue(), llvm::ISD::SHL, and X.

Referenced by performBRCONDCombine(), and performCSELCombine().

performConditionalSubCombine()

SDValue performConditionalSubCombine ( SDNode * N, SelectionDAG & DAG )

static

Definition at line 24953 of file AArch64ISelLowering.cpp.

References llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValue(), llvm::SDValue::hasOneUse(), llvm::isNullConstant(), N, SDValue(), llvm::ISD::SUB, and X.

Referenced by performAddSubCombine().

performCSELCombine()

SDValue performCSELCombine ( SDNode * N, TargetLowering::DAGCombinerInfo & DCI, SelectionDAG & DAG )

static

Definition at line 29836 of file AArch64ISelLowering.cpp.

References llvm::AArch64CC::AL, Cond, DL, llvm::SelectionDAG::doesNodeExist(), foldCSELOfCSEL(), foldCSELOfCSELSameFlags(), foldCSELofCTTZ(), foldCSELofLASTB(), Folded, getCondCode(), llvm::SelectionDAG::getNode(), getSwappedCondition(), llvm::TargetLowering::DAGCombinerInfo::isAfterLegalizeDAG(), llvm::isNullConstant(), N, performCONDCombine(), reassociateCSELOperandsForCSE(), llvm::ISD::SUB, and llvm::Sub.

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performCTLZCombine()

SDValue performCTLZCombine ( SDNode * N, SelectionDAG & DAG, const AArch64Subtarget * Subtarget )

static

Definition at line 30894 of file AArch64ISelLowering.cpp.

References llvm::ISD::BITREVERSE, llvm::ISD::CTTZ, DL, llvm::SelectionDAG::getNode(), N, SDValue(), and Subtarget.

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performCTPOPCombine()

SDValue performCTPOPCombine ( SDNode * N, TargetLowering::DAGCombinerInfo & DCI, SelectionDAG & DAG )

static

Definition at line 32106 of file AArch64ISelLowering.cpp.

References llvm::EVT::changeVectorElementTypeToInteger(), DL, llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getNegative(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::SelectionDAG::getSExtOrTrunc(), llvm::EVT::getSizeInBits(), llvm::EVT::getVectorElementCount(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorVT(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::EVT::isFixedLengthVector(), llvm::EVT::isVector(), llvm::Log2_64_Ceil(), llvm::PatternMatch::m_BitCast(), llvm::SDPatternMatch::m_SetCC(), llvm::PatternMatch::m_Value(), llvm::SDPatternMatch::m_VT(), llvm::PatternMatch::m_ZExtOrSelf(), N, llvm::SDPatternMatch::sd_match(), SDValue(), llvm::ISD::SIGN_EXTEND, and llvm::ISD::VECREDUCE_ADD.

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performDUPCombine()

SDValue performDUPCombine ( SDNode * N, TargetLowering::DAGCombinerInfo & DCI, const AArch64Subtarget * Subtarget )

static

Definition at line 30711 of file AArch64ISelLowering.cpp.

References AbstractManglingParser< Derived, Alloc >::Ops, llvm::ISD::BITCAST, llvm::EVT::changeVectorElementTypeToInteger(), llvm::SelectionDAG::computeKnownBits(), convertFromScalableVector(), llvm::TargetLowering::DAGCombinerInfo::DAG, DL, llvm::dyn_cast(), llvm::ISD::EXTLOAD, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), getContainerForFixedLengthVector(), llvm::SelectionDAG::getContext(), llvm::EVT::getDoubleNumVectorElementsVT(), getDUPLANEOp(), llvm::SelectionDAG::getExtractSubvector(), llvm::SelectionDAG::getInsertVectorElt(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNodeIfExists(), llvm::APInt::getOneBitSet(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getPOISON(), llvm::EVT::getScalarType(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorVT(), llvm::SelectionDAG::getVTList(), llvm::EVT::is64BitVector(), llvm::TargetLowering::DAGCombinerInfo::isAfterLegalizeDAG(), llvm::EVT::isInteger(), llvm::Known, N, performPostLD1Combine(), llvm::ISD::SCALAR_TO_VECTOR, SDValue(), llvm::ISD::SPLAT_VECTOR, Subtarget, Zero, and llvm::ISD::ZEXTLOAD.

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performDupLane128Combine()

SDValue performDupLane128Combine ( SDNode * N, SelectionDAG & DAG )

static

Definition at line 31519 of file AArch64ISelLowering.cpp.

References llvm::ISD::BITCAST, DL, llvm::SelectionDAG::getNode(), getPackedSVEVectorVT(), llvm::SelectionDAG::getPOISON(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::ISD::INSERT_SUBVECTOR, llvm::EVT::is128BitVector(), N, SDValue(), llvm::ISD::SPLAT_VECTOR, and uint64_t.

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performExtBinopLoadFold()

SDValue performExtBinopLoadFold ( SDNode * N, SelectionDAG & DAG )

static

Definition at line 24654 of file AArch64ISelLowering.cpp.

References AbstractManglingParser< Derived, Alloc >::Ops, areLoadedOffsetButOtherwiseSame(), llvm::ISD::CONCAT_VECTORS, DL, llvm::SelectionDAG::getContext(), llvm::EVT::getDoubleNumVectorElementsVT(), llvm::SelectionDAG::getExtractSubvector(), llvm::SelectionDAG::getLoad(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::EVT::getVectorVT(), llvm::SDValue::hasOneUse(), llvm::Hi, llvm::ISD::isConstantSplatVector(), llvm::ISD::isExtOpcode(), llvm::EVT::isFixedLengthVector(), isLoadOrMultipleLoads(), llvm::TargetLoweringBase::isTypeLegal(), Lo, llvm::Load, llvm::SelectionDAG::makeEquivalentMemoryOrdering(), N, llvm::O0, llvm::O1, llvm::SDNode::op_values(), llvm::Other, llvm::SmallVectorTemplateBase< T, bool >::push_back(), SDValue(), llvm::ISD::SHL, llvm::SmallVectorTemplateCommon< T, typename >::size(), llvm::ISD::SUB, std::swap(), and llvm::zip().

Referenced by performAddSubCombine().

performExtendCombine()

SDValue performExtendCombine ( SDNode * N, TargetLowering::DAGCombinerInfo & DCI, SelectionDAG & DAG, const AArch64Subtarget * Subtarget )

static

Definition at line 26446 of file AArch64ISelLowering.cpp.

References llvm::ISD::ABDS, llvm::ISD::ABDU, llvm::ISD::ANY_EXTEND, llvm::ISD::BSWAP, combineToExtendBoolVectorInReg(), DL, getCSETCondCode(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::SDValue::getOperand(), llvm::SDValue::getValueType(), llvm::SDValue::hasOneUse(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), N, llvm::Intrinsic::not_intrinsic, performExtendDuplaneTruncCombine(), performExtendToBoolVectorLoadCombine(), performSignExtendSetCCCombine(), performZExtDeinterleaveShuffleCombine(), performZExtUZPCombine(), SDValue(), llvm::ISD::SETCC, llvm::ISD::SIGN_EXTEND, Subtarget, tryCombineLongOpWithDup(), V, and llvm::ISD::ZERO_EXTEND.

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performExtendDuplaneTruncCombine()

SDValue performExtendDuplaneTruncCombine ( SDNode * N, SelectionDAG & DAG )

static

Definition at line 26345 of file AArch64ISelLowering.cpp.

References llvm::ISD::ANY_EXTEND, assert(), llvm::SelectionDAG::ComputeNumSignBits(), llvm::APInt::getHighBitsSet(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::SDValue::getValueType(), llvm::SDNode::hasOneUse(), llvm::SDValue::hasOneUse(), llvm::ISD::INSERT_SUBVECTOR, llvm::EVT::is128BitVector(), llvm::isNullConstant(), llvm::SelectionDAG::MaskedValueIsZero(), N, SDValue(), llvm::ISD::SIGN_EXTEND, llvm::ISD::TRUNCATE, and llvm::ISD::ZERO_EXTEND.

Referenced by performExtendCombine().

performExtendToBoolVectorLoadCombine()

SDValue performExtendToBoolVectorLoadCombine ( SDNode * N, SelectionDAG & DAG, TargetLowering::DAGCombinerInfo & DCI, const AArch64Subtarget & Subtarget )

static

Definition at line 26403 of file AArch64ISelLowering.cpp.

References combineToExtendBoolVectorInReg(), DL, llvm::dyn_cast(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getContext(), llvm::EVT::getIntegerVT(), llvm::SelectionDAG::getLoad(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::SDValue::hasOneUse(), llvm::EVT::isFixedLengthVector(), llvm::ISD::isNormalLoad(), N, llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), SDValue(), Subtarget, and V.

Referenced by performExtendCombine().

performExtractLastActiveCombine()

SDValue performExtractLastActiveCombine ( SDNode * N, TargetLowering::DAGCombinerInfo & DCI, const AArch64Subtarget * Subtarget )

static

Definition at line 22801 of file AArch64ISelLowering.cpp.

References llvm::ArrayRef(), assert(), llvm::EVT::changeTypeToInteger(), ContainerVT, convertFixedMaskToScalableVector(), convertToScalableVector(), llvm::TargetLowering::DAGCombinerInfo::DAG, llvm::ISD::EXTRACT_VECTOR_ELT, getContainerForFixedLengthVector(), llvm::SelectionDAG::getNode(), llvm::EVT::getSimpleVT(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::is_contained(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::TargetLoweringBase::isOperationLegal(), N, SDValue(), llvm::ISD::SIGN_EXTEND, Subtarget, and llvm::ISD::VECTOR_FIND_LAST_ACTIVE.

Referenced by performExtractVectorEltCombine().

performExtractSubvectorCombine()

SDValue performExtractSubvectorCombine ( SDNode * N, TargetLowering::DAGCombinerInfo & DCI, SelectionDAG & DAG )

static

Definition at line 23309 of file AArch64ISelLowering.cpp.

References llvm::SelectionDAG::getNode(), llvm::isa(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::EVT::isScalableVector(), N, SDValue(), llvm::ISD::SPLAT_VECTOR, and V.

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performExtractVectorEltCombine()

SDValue performExtractVectorEltCombine ( SDNode * N, TargetLowering::DAGCombinerInfo & DCI, const AArch64Subtarget * Subtarget )

static

Definition at line 22897 of file AArch64ISelLowering.cpp.

References llvm::all_of(), llvm::ISD::ANY_EXTEND, assert(), llvm::commonAlignment(), llvm::TargetLowering::DAGCombinerInfo::DAG, DL, llvm::dyn_cast(), llvm::ISD::EXTLOAD, llvm::ISD::EXTRACT_SUBVECTOR, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getConstant(), llvm::SDValue::getConstantOperandVal(), llvm::SelectionDAG::getExtLoad(), llvm::SelectionDAG::getExtractVectorElt(), llvm::ShuffleVectorSDNode::getMaskElt(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getObjectPtrOffset(), llvm::SDNode::getOpcode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::SDValue::getOperand(), llvm::SDValue::getValue(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::SDValue::hasOneUse(), hasPairwiseAdd(), llvm::ISD::INSERT_SUBVECTOR, llvm::isa(), llvm::TargetLowering::DAGCombinerInfo::isAfterLegalizeDAG(), llvm::EVT::isInteger(), llvm::ISD::isNormalLoad(), llvm::isNullConstant(), llvm::SDNode::isStrictFPOpcode(), llvm::SDValue::isUndef(), llvm::Load, llvm::SelectionDAG::makeEquivalentMemoryOrdering(), N, llvm::Offset, llvm::Other, perfomPextFirstTrueVectorCombine(), performExtractLastActiveCombine(), performFirstTrueTestVectorCombine(), performLastTrueTestVectorCombine(), llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), SDValue(), llvm::ISD::SEXTLOAD, llvm::ISD::SIGN_EXTEND, Subtarget, llvm::SDNode::uses(), llvm::ISD::ZERO_EXTEND, and llvm::ISD::ZEXTLOAD.

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performFADDCombine()

SDValue performFADDCombine ( SDNode * N, TargetLowering::DAGCombinerInfo & DCI )

static

Definition at line 22633 of file AArch64ISelLowering.cpp.

References A(), B(), llvm::TargetLowering::DAGCombinerInfo::DAG, DL, llvm::ISD::FADD, llvm::SelectionDAG::getNode(), llvm::ISD::INTRINSIC_WO_CHAIN, LHS, N, Opc, RHS, SDValue(), and llvm::SDNode::setFlags().

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performFADDPREDCombine()

SDValue performFADDPREDCombine ( SDNode * N, TargetLowering::DAGCombinerInfo & DCI )

static

Definition at line 22669 of file AArch64ISelLowering.cpp.

References llvm::TargetLowering::DAGCombinerInfo::DAG, DL, llvm::FMul, llvm::SelectionDAG::getNode(), N, SDValue(), and std::swap().

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performFirstTrueTestVectorCombine()

SDValue performFirstTrueTestVectorCombine ( SDNode * N, TargetLowering::DAGCombinerInfo & DCI, const AArch64Subtarget * Subtarget )

static

Definition at line 22744 of file AArch64ISelLowering.cpp.

References assert(), llvm::TargetLowering::DAGCombinerInfo::DAG, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::AArch64CC::FIRST_ACTIVE, llvm::SelectionDAG::getConstant(), getPTest(), llvm::SDValue::getResNo(), llvm::SDValue::getValueType(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::isNullConstant(), isPredicateCCSettingOp(), llvm::EVT::isScalableVectorOf(), N, SDValue(), and Subtarget.

Referenced by performExtractVectorEltCombine().

performFlagSettingCombine()

SDValue performFlagSettingCombine ( SDNode * N, TargetLowering::DAGCombinerInfo & DCI, unsigned GenericOpcode )

static

Definition at line 30125 of file AArch64ISelLowering.cpp.

References llvm::TargetLowering::DAGCombinerInfo::CombineTo(), llvm::TargetLowering::DAGCombinerInfo::DAG, DL, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNodeIfExists(), llvm::SelectionDAG::getVTList(), LHS, N, RHS, and SDValue().

Referenced by performANDSCombine(), and llvm::AArch64TargetLowering::PerformDAGCombine().

performFPExtendCombine()

SDValue performFPExtendCombine ( SDNode * N, SelectionDAG & DAG, TargetLowering::DAGCombinerInfo & DCI, const AArch64Subtarget * Subtarget )

static

Definition at line 31461 of file AArch64ISelLowering.cpp.

References llvm::cast(), llvm::TargetLowering::DAGCombinerInfo::CombineTo(), llvm::ISD::EXTLOAD, llvm::ISD::FP_ROUND, llvm::LoadSDNode::getBasePtr(), llvm::MemSDNode::getChain(), llvm::SelectionDAG::getExtLoad(), llvm::EVT::getFixedSizeInBits(), llvm::SelectionDAG::getIntPtrConstant(), llvm::MemSDNode::getMemOperand(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::SDValue::hasOneUse(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::EVT::isFixedLengthVector(), llvm::ISD::isNormalLoad(), N, SDValue(), and Subtarget.

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performFpToIntCombine()

SDValue performFpToIntCombine ( SDNode * N, SelectionDAG & DAG, TargetLowering::DAGCombinerInfo & DCI, const AArch64Subtarget * Subtarget )

static

Fold a floating-point multiply by power of two into floating-point to fixed-point conversion.

Definition at line 22156 of file AArch64ISelLowering.cpp.

References N, SDValue(), Subtarget, and tryToReplaceScalarFPConversionWithSVE().

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performGatherLoadCombine()

SDValue performGatherLoadCombine ( SDNode * N, SelectionDAG & DAG, unsigned Opcode, bool OnlyPackedOffsets = true )

static

Definition at line 31065 of file AArch64ISelLowering.cpp.

References AbstractManglingParser< Derived, Alloc >::Ops, llvm::ISD::ANY_EXTEND, assert(), llvm::sampleprof::Base, llvm::ISD::BITCAST, DL, llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getKnownMinValue(), llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), getScaledOffsetForBitWidth(), llvm::EVT::getSizeInBits(), getSVEContainerType(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValue(), llvm::SelectionDAG::getValueType(), llvm::SelectionDAG::getVTList(), llvm::EVT::isFloatingPoint(), llvm::EVT::isInteger(), llvm::EVT::isScalableVector(), isValidImmForSVEVecImmAddrMode(), llvm::Load, N, llvm::Offset, SDValue(), llvm::AArch64::SVEBitsPerBlock, std::swap(), TLI, and llvm::ISD::TRUNCATE.

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performGLD1Combine()

SDValue performGLD1Combine ( SDNode * N, SelectionDAG & DAG )

static

Definition at line 27287 of file AArch64ISelLowering.cpp.

References llvm::sampleprof::Base, llvm::cast(), DL, getGatherVecOpcode(), llvm::SelectionDAG::getNode(), getSignExtendedGatherOpcode(), llvm::EVT::getVectorElementType(), llvm::VTSDNode::getVT(), N, llvm::Offset, Opc, Scaled, SDValue(), and llvm::Signed.

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performGlobalAddressCombine()

SDValue performGlobalAddressCombine ( SDNode * N, SelectionDAG & DAG, const AArch64Subtarget * Subtarget, const TargetMachine & TM )

static

Definition at line 30843 of file AArch64ISelLowering.cpp.

References llvm::ISD::ADD, C(), llvm::cast(), DL, llvm::dyn_cast(), llvm::SelectionDAG::getConstant(), llvm::GlobalValue::getDataLayout(), llvm::GlobalAddressSDNode::getGlobal(), llvm::SelectionDAG::getGlobalAddress(), llvm::SelectionDAG::getNode(), llvm::GlobalAddressSDNode::getOffset(), llvm::DataLayout::getTypeAllocSize(), llvm::GlobalValue::getValueType(), llvm::AArch64II::MO_NO_FLAG, N, llvm::Offset, SDValue(), llvm::ISD::SUB, Subtarget, T, uint64_t, and llvm::SDNode::users().

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performInsertSubvectorCombine()

SDValue performInsertSubvectorCombine ( SDNode * N, TargetLowering::DAGCombinerInfo & DCI, SelectionDAG & DAG )

static

Definition at line 23335 of file AArch64ISelLowering.cpp.

References C(), llvm::ISD::CONCAT_VECTORS, DL, llvm::dyn_cast_or_null(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getConstantFP(), llvm::SelectionDAG::getExtractSubvector(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getSplatValue(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::Hi, llvm::EVT::isFixedLengthVector(), llvm::EVT::isScalableVector(), llvm::TargetLoweringBase::isTypeLegal(), llvm::SDValue::isUndef(), Lo, N, SDValue(), and uint64_t.

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performInsertVectorEltCombine()

SDValue performInsertVectorEltCombine ( SDNode * N, TargetLowering::DAGCombinerInfo & DCI )

static

Definition at line 31443 of file AArch64ISelLowering.cpp.

References llvm::TargetLowering::DAGCombinerInfo::DAG, DL, llvm::SelectionDAG::getNode(), llvm::SDValue::getValueType(), llvm::isNullConstant(), llvm::EVT::isScalableVector(), llvm::SDNode::isUndef(), N, performPostLD1Combine(), removeRedundantInsertVectorElt(), and llvm::ISD::SPLAT_VECTOR.

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performIntrinsicCombine()

SDValue performIntrinsicCombine ( SDNode * N, TargetLowering::DAGCombinerInfo & DCI, const AArch64Subtarget * Subtarget )

static

Definition at line 25676 of file AArch64ISelLowering.cpp.

References llvm::ISD::ABDS, llvm::ISD::ABDU, AbstractManglingParser< Derived, Alloc >::Ops, llvm::ISD::ADD, llvm::AArch64CC::ANY_ACTIVE, combineAcrossLanesIntrinsic(), combineSVEBitSel(), combineSVEReductionFP(), combineSVEReductionInt(), combineSVEReductionOrderedFP(), convertMergedOpToPredOp(), llvm::TargetLowering::DAGCombinerInfo::DAG, DL, llvm::AArch64CC::FIRST_ACTIVE, llvm::ISD::FMAXIMUM, llvm::ISD::FMAXNUM, llvm::ISD::FMINIMUM, llvm::ISD::FMINNUM, llvm::ISD::GET_ACTIVE_LANE_MASK, llvm::SelectionDAG::getCondCode(), llvm::SelectionDAG::getConstant(), getIntrinsicID(), llvm::SelectionDAG::getNode(), getPTest(), I, llvm::ISD::INTRINSIC_WO_CHAIN, llvm::ISD::isBuildVectorAllZeros(), llvm::AArch64CC::LAST_ACTIVE, llvm_unreachable, lowerPredCounterWhile(), LowerSVEIntrinsicDUP(), LowerSVEIntrinsicEXT(), LowerSVEIntrinsicIndex(), N, llvm::ISD::OR, llvm::ISD::SADDSAT, SDValue(), llvm::ISD::SETEQ, llvm::ISD::SETGE, llvm::ISD::SETGT, llvm::ISD::SETLE, llvm::ISD::SETLT, llvm::ISD::SETNE, llvm::ISD::SETOEQ, llvm::ISD::SETOGE, llvm::ISD::SETOGT, llvm::ISD::SETUGE, llvm::ISD::SETUGT, llvm::ISD::SETULE, llvm::ISD::SETULT, llvm::ISD::SETUNE, llvm::ISD::SETUO, llvm::ISD::SPLAT_VECTOR, llvm::ISD::SSUBSAT, llvm::ISD::SUB, Subtarget, tryCombineCRC32(), tryCombineFADDReductionWithZero(), tryCombineFixedPointConvert(), tryCombineLongOpWithDup(), tryCombineNeonFcvtFP16ToI16(), tryCombineShiftImm(), tryConvertSVEWideCompare(), llvm::ISD::UADDSAT, llvm::ISD::USUBSAT, V, llvm::ISD::VSELECT, and llvm::ISD::XOR.

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performIntToFpCombine()

SDValue performIntToFpCombine ( SDNode * N, SelectionDAG & DAG, TargetLowering::DAGCombinerInfo & DCI, const AArch64Subtarget * Subtarget )

static

Definition at line 22107 of file AArch64ISelLowering.cpp.

References llvm::cast(), llvm::MemSDNode::getAlign(), llvm::LoadSDNode::getBasePtr(), llvm::MemSDNode::getChain(), llvm::MachineMemOperand::getFlags(), llvm::SelectionDAG::getLoad(), llvm::MemSDNode::getMemOperand(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::MemSDNode::getPointerInfo(), llvm::EVT::getSizeInBits(), llvm::SDValue::hasOneUse(), llvm::ISD::isNormalLoad(), llvm::Load, N, performVectorCompareAndMaskUnaryOpCombine(), llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), SDValue(), llvm::ISD::SINT_TO_FP, Subtarget, and tryToReplaceScalarFPConversionWithSVE().

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performLastTrueTestVectorCombine()

SDValue performLastTrueTestVectorCombine ( SDNode * N, TargetLowering::DAGCombinerInfo & DCI, const AArch64Subtarget * Subtarget )

static

Definition at line 22773 of file AArch64ISelLowering.cpp.

References assert(), llvm::TargetLowering::DAGCombinerInfo::DAG, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getConstant(), llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getKnownMinValue(), getPTest(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementCount(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::EVT::isScalableVectorOf(), llvm::AArch64CC::LAST_ACTIVE, llvm::PatternMatch::m_Add(), llvm::MIPatternMatch::m_AllOnes(), llvm::PatternMatch::m_SpecificInt(), llvm::PatternMatch::m_VScale(), llvm::PatternMatch::m_ZExtOrSelf(), N, llvm::SDPatternMatch::sd_match(), SDValue(), and Subtarget.

Referenced by performExtractVectorEltCombine().

performLD1Combine()

SDValue performLD1Combine ( SDNode * N, SelectionDAG & DAG, unsigned Opc )

static

Definition at line 26600 of file AArch64ISelLowering.cpp.

References AbstractManglingParser< Derived, Alloc >::Ops, ContainerVT, DL, llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getKnownMinValue(), llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), getSVEContainerType(), llvm::SelectionDAG::getValueType(), llvm::SelectionDAG::getVTList(), llvm::Load, N, Opc, SDValue(), llvm::AArch64::SVEBitsPerBlock, and llvm::ISD::TRUNCATE.

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performLD1ReplicateCombine()

template<unsigned Opcode>

SDValue performLD1ReplicateCombine ( SDNode * N, SelectionDAG & DAG )

static

Definition at line 26652 of file AArch64ISelLowering.cpp.

References AbstractManglingParser< Derived, Alloc >::Ops, llvm::ISD::BITCAST, llvm::EVT::changeTypeToInteger(), DL, llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), llvm::EVT::isFloatingPoint(), llvm::Load, N, and SDValue().

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performLDNT1Combine()

SDValue performLDNT1Combine ( SDNode * N, SelectionDAG & DAG )

static

Definition at line 26626 of file AArch64ISelLowering.cpp.

References AbstractManglingParser< Derived, Alloc >::Ops, llvm::ISD::BITCAST, llvm::cast(), llvm::EVT::changeTypeToInteger(), DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getMaskedLoad(), llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getPOISON(), llvm::EVT::isFloatingPoint(), N, llvm::ISD::NON_EXTLOAD, and llvm::ISD::UNINDEXED.

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performLegalizedInterleavedStoreCombine()

SDValue performLegalizedInterleavedStoreCombine ( StoreSDNode * ST, TargetLowering::DAGCombinerInfo & DCI, SelectionDAG & DAG )

static

Definition at line 28579 of file AArch64ISelLowering.cpp.

References AbstractManglingParser< Derived, Alloc >::Ops, llvm::AfterLegalizeTypes, llvm::SelectionDAG::areNonVolatileConsecutiveStores(), llvm::TargetLowering::DAGCombinerInfo::CombineTo(), ContainerVT, convertToScalableVector(), DL, llvm::dyn_cast(), llvm::enumerate(), llvm::StoreSDNode::getBasePtr(), llvm::MemSDNode::getChain(), llvm::SelectionDAG::getConstant(), getContainerForFixedLengthVector(), llvm::SelectionDAG::getContext(), llvm::TargetLowering::DAGCombinerInfo::getDAGCombineLevel(), llvm::SelectionDAG::getMachineFunction(), llvm::MachineFunction::getMachineMemOperand(), llvm::SelectionDAG::getMemIntrinsicNode(), llvm::MemSDNode::getMemOperand(), llvm::SDValue::getNode(), llvm::SDNode::getNumOperands(), llvm::SDNode::getOpcode(), getPredicateForFixedLengthVector(), llvm::SelectionDAG::getSubtarget(), llvm::SelectionDAG::getTargetConstant(), llvm::Use::getUser(), llvm::SDNode::getValueType(), llvm::EVT::getVectorVT(), llvm::SelectionDAG::getVTList(), llvm::ISD::INTRINSIC_VOID, llvm::is_contained(), NumParts, llvm::SDNode::op_begin(), llvm::SDNode::op_end(), llvm::SDNode::op_values(), SDValue(), llvm::Store, Subtarget, SubVecTy, SVEStores, llvm::SDNode::uses(), V, and llvm::ISD::VECTOR_INTERLEAVE.

performLegalizedVectorDeinterleaveCombine()

SDValue performLegalizedVectorDeinterleaveCombine ( SDNode * N, TargetLowering::DAGCombinerInfo & DCI, SelectionDAG & DAG )

static

Definition at line 31821 of file AArch64ISelLowering.cpp.

References llvm::AfterLegalizeTypes, llvm::SelectionDAG::areNonVolatileConsecutiveLoads(), llvm::TargetLowering::DAGCombinerInfo::CombineTo(), ContainerVT, convertFromScalableVector(), DL, llvm::dyn_cast(), llvm::enumerate(), getContainerForFixedLengthVector(), llvm::SelectionDAG::getContext(), llvm::TargetLowering::DAGCombinerInfo::getDAGCombineLevel(), llvm::SelectionDAG::getMachineFunction(), llvm::MachineFunction::getMachineMemOperand(), llvm::SelectionDAG::getMemIntrinsicNode(), llvm::MemSDNode::getMemOperand(), getPredicateForFixedLengthVector(), llvm::SelectionDAG::getSubtarget(), llvm::SDValue::getValue(), llvm::EVT::getVectorVT(), llvm::SelectionDAG::getVTList(), llvm::SDValue::hasOneUse(), I, llvm::ISD::INTRINSIC_W_CHAIN, llvm::Load, N, NumParts, Pred, llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), SDValue(), Subtarget, SubVecTy, and llvm::ISD::VECTOR_DEINTERLEAVE.

Referenced by performVectorDeinterleaveCombine().

performMaskedGatherScatterCombine()

SDValue performMaskedGatherScatterCombine ( SDNode * N, TargetLowering::DAGCombinerInfo & DCI, SelectionDAG & DAG )

static

Definition at line 28869 of file AArch64ISelLowering.cpp.

References AbstractManglingParser< Derived, Alloc >::Ops, llvm::cast(), llvm::Data, DL, llvm::dyn_cast(), findMoreOptimalIndexType(), llvm::MaskedGatherScatterSDNode::getBasePtr(), llvm::MemSDNode::getChain(), llvm::MaskedGatherSDNode::getExtensionType(), llvm::MaskedGatherScatterSDNode::getIndex(), llvm::MaskedGatherScatterSDNode::getIndexType(), llvm::MaskedGatherScatterSDNode::getMask(), llvm::SelectionDAG::getMaskedGather(), llvm::SelectionDAG::getMaskedHistogram(), llvm::SelectionDAG::getMaskedScatter(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getMemoryVT(), llvm::MaskedGatherSDNode::getPassThru(), llvm::MaskedGatherScatterSDNode::getScale(), llvm::MaskedScatterSDNode::getValue(), llvm::SelectionDAG::getVTList(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::MaskedScatterSDNode::isTruncatingStore(), N, and SDValue().

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performMINMAXCombine()

SDValue performMINMAXCombine ( SDNode * N, SelectionDAG & DAG, const AArch64TargetLowering & TLI )

static

Definition at line 32194 of file AArch64ISelLowering.cpp.

References llvm::SelectionDAG::getNode(), getReductionForOpcode(), llvm::SDPatternMatch::m_ConstInt(), llvm::PatternMatch::m_ExtractElt(), llvm::MIPatternMatch::m_OneUse(), llvm::PatternMatch::m_Specific(), llvm::PatternMatch::m_SpecificInt(), llvm::SDPatternMatch::m_SpecificVT(), llvm::PatternMatch::m_Value(), N, llvm::SDPatternMatch::sd_match(), SDValue(), TLI, trySQABSCombine(), trySQDMULHCombine(), and V.

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performMSTORECombine()

SDValue performMSTORECombine ( SDNode * N, TargetLowering::DAGCombinerInfo & DCI, SelectionDAG & DAG, const AArch64Subtarget * Subtarget, const AArch64TargetLowering & TLI )

static

Definition at line 28688 of file AArch64ISelLowering.cpp.

References llvm::ISD::BITCAST, llvm::cast(), llvm::EVT::changeVectorElementType(), DL, expandScalableCompressingStore(), llvm::MaskedLoadStoreSDNode::getAddressingMode(), llvm::MaskedStoreSDNode::getBasePtr(), llvm::MemSDNode::getChain(), llvm::SelectionDAG::getContext(), llvm::MaskedStoreSDNode::getMask(), llvm::SelectionDAG::getMaskedStore(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getMemoryVT(), llvm::getNumElementsFromSVEPredPattern(), llvm::MaskedStoreSDNode::getOffset(), getPTrue(), llvm::EVT::getSizeInBits(), llvm::MaskedStoreSDNode::getValue(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::Value::hasOneUse(), llvm::MaskedStoreSDNode::isCompressingStore(), llvm::EVT::isScalableVector(), llvm::MaskedStoreSDNode::isTruncatingStore(), llvm::MaskedLoadStoreSDNode::isUnindexed(), N, SDValue(), Subtarget, TLI, trySimplifySrlAddToRshrnb(), llvm::ISD::VECTOR_COMPRESS, and llvm::EVT::widenIntegerVectorElementType().

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performMulCombine()

SDValue performMulCombine ( SDNode * N, SelectionDAG & DAG, TargetLowering::DAGCombinerInfo & DCI, const AArch64Subtarget * Subtarget )

static

Definition at line 21750 of file AArch64ISelLowering.cpp.

References llvm::ISD::ADD, llvm::Add, llvm::APInt::ashr(), llvm::BitWidth, C(), llvm::cast(), llvm::APInt::countr_zero(), DL, llvm::dyn_cast(), llvm::APInt::getBitWidth(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::SDValue::getValueSizeInBits(), llvm::APInt::getZExtValue(), llvm::SDNode::hasOneUse(), llvm::isa(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::APInt::isNegative(), llvm::APInt::isNonNegative(), isOnlyUsedAsMemoryAddress(), llvm::APInt::isPowerOf2(), isSignExtended(), IsSVECntIntrinsic(), isZeroExtended(), llvm::APInt::logBase2(), llvm::ISD::MUL, N, performMulRdsvlCombine(), performMulVectorCmpZeroCombine(), performMulVectorExtendCombine(), performVectorExtCombine(), RHS, llvm::APInt::sdivrem(), SDValue(), llvm::APInt::sge(), llvm::ISD::SHL, llvm::APInt::sle(), llvm::ISD::SUB, llvm::Sub, Subtarget, std::swap(), llvm::ISD::TRUNCATE, V, X, and Zero.

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performMULLCombine()

SDValue performMULLCombine ( SDNode * N, TargetLowering::DAGCombinerInfo & DCI, SelectionDAG & DAG )

static

Definition at line 31695 of file AArch64ISelLowering.cpp.

References N, llvm::Intrinsic::not_intrinsic, SDValue(), tryCombineLongOpWithDup(), and tryCombineMULLWithUZP1().

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performMulRdsvlCombine()

SDValue performMulRdsvlCombine ( SDNode * Mul, SelectionDAG & DAG )

static

Definition at line 21597 of file AArch64ISelLowering.cpp.

References llvm::abs(), B(), llvm::cast(), llvm::countr_zero(), DL, llvm::SDNodeFlags::Exact, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getSignedConstant(), llvm::Log2_32_Ceil(), Mul, SDValue(), llvm::ISD::SHL, and llvm::ISD::SRL.

Referenced by performMulCombine().

performMulVectorCmpZeroCombine()

SDValue performMulVectorCmpZeroCombine ( SDNode * N, SelectionDAG & DAG )

static

Definition at line 21644 of file AArch64ISelLowering.cpp.

References llvm::ISD::AND, DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::EVT::getIntegerVT(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::SelectionDAG::getSetCC(), llvm::EVT::getVectorElementCount(), llvm::EVT::getVectorVT(), llvm::ISD::isConstantSplatVector(), N, SDValue(), llvm::ISD::SETGT, llvm::ISD::SRL, llvm::V1, llvm::V3, and Zero.

Referenced by performMulCombine().

performMulVectorExtendCombine()

SDValue performMulVectorExtendCombine ( SDNode * Mul, SelectionDAG & DAG )

static

Combines a mul(dup(sext/zext)) node pattern into mul(sext/zext(dup)) making use of the vector SExt/ZExt rather than the scalar SExt/ZExt.

Definition at line 21569 of file AArch64ISelLowering.cpp.

References DL, llvm::SelectionDAG::getNode(), Mul, performBuildShuffleExtendCombine(), and SDValue().

Referenced by performMulCombine().

performNegCSelCombine()

SDValue performNegCSelCombine ( SDNode * N, SelectionDAG & DAG )

static

Definition at line 23784 of file AArch64ISelLowering.cpp.

References DL, llvm::SelectionDAG::getNegative(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValueType(), llvm::SDNode::hasOneUse(), isNegatedInteger(), N, and SDValue().

Referenced by performAddSubCombine().

performNegSignMaskedAndCombine()

SDValue performNegSignMaskedAndCombine ( SDNode * N, SelectionDAG & DAG )

static

Definition at line 24375 of file AArch64ISelLowering.cpp.

References llvm::ISD::AND, llvm::any_of(), llvm::SelectionDAG::ComputeNumSignBits(), DL, llvm::SelectionDAG::getNegative(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), llvm::isNullConstant(), N, SDValue(), llvm::ISD::SUB, std::swap(), and X.

Referenced by performAddSubCombine().

performNEONPostLDSTCombine()

SDValue performNEONPostLDSTCombine ( SDNode * N, TargetLowering::DAGCombinerInfo & DCI, SelectionDAG & DAG )

static

Target-specific DAG combine function for NEON load/store intrinsics to merge base address updates.

Definition at line 28922 of file AArch64ISelLowering.cpp.

References AbstractManglingParser< Derived, Alloc >::Ops, llvm::ISD::ADD, llvm::ArrayRef(), llvm::cast(), llvm::TargetLowering::DAGCombinerInfo::CombineTo(), llvm::dyn_cast(), llvm::SelectionDAG::getMemIntrinsicNode(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getMemoryVT(), llvm::SDValue::getNode(), llvm::User::getOperand(), llvm::SelectionDAG::getRegister(), llvm::SDValue::getResNo(), llvm::EVT::getSizeInBits(), llvm::Use::getUser(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVTList(), llvm::SDNode::hasPredecessorHelper(), llvm::SmallPtrSetImpl< PtrType >::insert(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::TargetLowering::DAGCombinerInfo::isCalledByLegalizer(), llvm_unreachable, N, llvm::SmallVectorTemplateBase< T, bool >::push_back(), SDValue(), uint64_t, and llvm::SDNode::uses().

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performNVCASTCombine()

SDValue performNVCASTCombine ( SDNode * N, SelectionDAG & DAG, const AArch64Subtarget * Subtarget )

static

Get rid of unnecessary NVCASTs (that don't change the type). Under LE a NVCAST is converted to a bitcast.

Definition at line 30820 of file AArch64ISelLowering.cpp.

References llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getUNDEF(), N, SDValue(), and Subtarget.

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performORCombine()

SDValue performORCombine ( SDNode * N, TargetLowering::DAGCombinerInfo & DCI )

static

Definition at line 22361 of file AArch64ISelLowering.cpp.

References llvm::TargetLowering::DAGCombinerInfo::DAG, N, performANDORCSELCombine(), performANDORDUPNOTCombine(), SDValue(), and tryCombineToREV().

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine(), llvm::LoongArchTargetLowering::PerformDAGCombine(), llvm::MipsSETargetLowering::PerformDAGCombine(), llvm::MipsTargetLowering::PerformDAGCombine(), and llvm::RISCVTargetLowering::PerformDAGCombine().

performOrXorChainCombine()

SDValue performOrXorChainCombine ( SDNode * N, SelectionDAG & DAG )

static

Definition at line 12803 of file AArch64ISelLowering.cpp.

References llvm::ISD::AND, llvm::any_of(), llvm::SmallVectorTemplateCommon< T, typename >::begin(), Cond, DL, llvm::SmallVectorTemplateCommon< T, typename >::end(), F, llvm::find_if(), llvm::First, llvm::MachineFunction::getFunction(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getSetCC(), llvm::SelectionDAG::getSubtarget(), hasCheapXorImmediateThatNeedsCondCmpReg(), hasLegalCmpImmediate(), hasLegalCondCmpImmediate(), I, llvm::isNullConstant(), isOrXorChain(), LHS, N, llvm::ISD::OR, RHS, SDValue(), llvm::ISD::SETCC, llvm::ISD::SETEQ, llvm::ISD::SETNE, and llvm::SmallVectorTemplateCommon< T, typename >::size().

Referenced by performSETCCCombine().

performPostLD1Combine()

SDValue performPostLD1Combine ( SDNode * N, TargetLowering::DAGCombinerInfo & DCI, bool IsLaneOp )

static

Target-specific DAG combine function for post-increment LD1 (lane) and post-increment LD1R.

Definition at line 27390 of file AArch64ISelLowering.cpp.

References AbstractManglingParser< Derived, Alloc >::Ops, llvm::ISD::ADD, llvm::cast(), llvm::TargetLowering::DAGCombinerInfo::CombineTo(), llvm::TargetLowering::DAGCombinerInfo::DAG, llvm::dyn_cast(), llvm::ISD::FMA, llvm::ISD::FMUL, llvm::SelectionDAG::getMemIntrinsicNode(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getMemoryVT(), llvm::SDValue::getNode(), llvm::User::getOperand(), llvm::SelectionDAG::getRegister(), llvm::SDValue::getResNo(), llvm::EVT::getScalarSizeInBits(), llvm::Use::getUser(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVTList(), llvm::SDNode::hasPredecessorHelper(), llvm::SmallPtrSetImpl< PtrType >::insert(), llvm::EVT::is128BitVector(), llvm::EVT::is64BitVector(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::isNullOrNullSplat(), N, llvm::SmallVectorTemplateBase< T, bool >::push_back(), SDValue(), uint64_t, llvm::SDNode::uses(), and llvm::Vector.

Referenced by performDUPCombine(), and performInsertVectorEltCombine().

performPredicateLoadCombine()

SDValue performPredicateLoadCombine ( SDNode * N, TargetLowering::DAGCombinerInfo & DCI, SelectionDAG & DAG )

static

Definition at line 32226 of file AArch64ISelLowering.cpp.

References llvm::cast(), DL, llvm::SelectionDAG::getLoad(), llvm::SDValue::getNode(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getSubtarget(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::EVT::isFixedLengthVectorOf(), llvm::ISD::isNormalLoad(), llvm::Load, llvm::PatternMatch::m_BitCast(), llvm::SDPatternMatch::m_InsertSubvector(), llvm::PatternMatch::m_SpecificInt(), llvm::SDPatternMatch::m_SpecificVT(), llvm::PatternMatch::m_Undef(), llvm::PatternMatch::m_Value(), llvm::SelectionDAG::makeEquivalentMemoryOrdering(), N, llvm::SDPatternMatch::sd_match(), SDValue(), and Subtarget.

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performPTestFirstCombine()

SDValue performPTestFirstCombine ( SDNode * N, TargetLowering::DAGCombinerInfo & DCI, SelectionDAG & DAG )

static

Definition at line 31708 of file AArch64ISelLowering.cpp.

References llvm::ISD::CONCAT_VECTORS, DL, llvm::SelectionDAG::getNode(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), isLane0KnownActive(), N, Pred, and SDValue().

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performReinterpretCastCombine()

SDValue performReinterpretCastCombine ( SDNode * N )

static

Definition at line 22404 of file AArch64ISelLowering.cpp.

References llvm::SDValue::getValueType(), N, and SDValue().

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performRNDRCombine()

SDValue performRNDRCombine ( SDNode * N, SelectionDAG & DAG )

static

Definition at line 32089 of file AArch64ISelLowering.cpp.

References A(), B(), DL, FlagsVT, getCondCode(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getVTList(), llvm::SelectionDAG::getZExtOrTrunc(), N, and llvm::AArch64CC::NE.

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performScalarToVectorCombine()

SDValue performScalarToVectorCombine ( SDNode * N, TargetLowering::DAGCombinerInfo & DCI, SelectionDAG & DAG )

static

Definition at line 31735 of file AArch64ISelLowering.cpp.

References AbstractManglingParser< Derived, Alloc >::Ops, llvm::TargetLowering::DAGCombinerInfo::DAG, DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getExtractSubvector(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNodeIfExists(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getPOISON(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::TargetLowering::DAGCombinerInfo::isAfterLegalizeDAG(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::EVT::isFixedLengthVector(), llvm::isIntOrFPConstant(), llvm::isNullConstant(), N, SDValue(), llvm::Undef, and llvm::ISD::ZERO_EXTEND.

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performScatterStoreCombine()

SDValue performScatterStoreCombine ( SDNode * N, SelectionDAG & DAG, unsigned Opcode, bool OnlyPackedOffsets = true )

static

Definition at line 30952 of file AArch64ISelLowering.cpp.

References AbstractManglingParser< Derived, Alloc >::Ops, llvm::ISD::ANY_EXTEND, assert(), llvm::sampleprof::Base, llvm::ISD::BITCAST, DL, llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getKnownMinValue(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), getScaledOffsetForBitWidth(), llvm::EVT::getSimpleVT(), llvm::EVT::getSizeInBits(), llvm::MVT::getSizeInBits(), getSVEContainerType(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValue(), llvm::SelectionDAG::getValueType(), llvm::EVT::getVectorElementType(), llvm::SelectionDAG::getVTList(), llvm::EVT::isFloatingPoint(), llvm::MVT::isFloatingPoint(), llvm::EVT::isScalableVector(), isValidImmForSVEVecImmAddrMode(), N, llvm::Offset, SDValue(), llvm::AArch64::SVEBitsPerBlock, std::swap(), and TLI.

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performSELECT_CCCombine()

SDValue performSELECT_CCCombine ( SDNode * N, TargetLowering::DAGCombinerInfo & DCI, SelectionDAG & DAG )

static

Definition at line 30086 of file AArch64ISelLowering.cpp.

References assert(), llvm::ISD::BITCAST, Cond, DL, llvm::SelectionDAG::getNode(), llvm::isAllOnesConstant(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::EVT::isFixedLengthVectorOf(), llvm::isNullConstant(), LHS, N, RHS, SDValue(), llvm::ISD::SELECT_CC, llvm::ISD::SETEQ, llvm::ISD::SETNE, and std::swap().

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine(), and llvm::LoongArchTargetLowering::PerformDAGCombine().

performSelectCombine()

SDValue performSelectCombine ( SDNode * N, TargetLowering::DAGCombinerInfo & DCI )

static

A vector select: "(select vL, vR, (setcc LHS, RHS))" is best performed with the compare-mask instructions rather than going via NZCV, even if LHS and RHS are really scalar. This replaces any scalar setcc in the above pattern with a vector one followed by a DUP shuffle on the result.

Definition at line 30643 of file AArch64ISelLowering.cpp.

References assert(), llvm::ISD::BITCAST, llvm::EVT::changeVectorElementTypeToInteger(), llvm::TargetLowering::DAGCombinerInfo::DAG, DL, llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getSelect(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::EVT::getVectorVT(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::EVT::isFloatingPoint(), llvm::EVT::isScalableVT(), llvm::TargetLoweringBase::isTypeLegal(), llvm::EVT::isVector(), LHS, N, RHS, llvm::ISD::SCALAR_TO_VECTOR, SDValue(), and llvm::ISD::SETCC.

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performSetccAddFolding()

SDValue performSetccAddFolding ( SDNode * Op, TargetLowering::DAGCombinerInfo & DCI )

static

Definition at line 23588 of file AArch64ISelLowering.cpp.

References SetCCInfo::AArch64, llvm::ISD::ADD, AArch64SetCCInfo::CC, GenericSetCCInfo::CC, AArch64SetCCInfo::Cmp, llvm::TargetLowering::DAGCombinerInfo::DAG, DL, SetCCInfo::Generic, getAArch64Cmp(), llvm::SelectionDAG::getConstant(), llvm::AArch64CC::getInvertedCondCode(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::ISD::getSetCCInverse(), llvm::SDNode::getValueType(), llvm::SDValue::getValueType(), SetCCInfoAndKind::Info, SetCCInfoAndKind::IsAArch64, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), isSetCCOrZExtSetCC(), LHS, GenericSetCCInfo::Opnd0, GenericSetCCInfo::Opnd1, RHS, SDValue(), and std::swap().

Referenced by performAddSubCombine().

performSETCCCARRYCombine()

SDValue performSETCCCARRYCombine ( SDNode * N, SelectionDAG & DAG )

static

Definition at line 30052 of file AArch64ISelLowering.cpp.

References llvm::ISD::AND, assert(), Cond, DL, llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getResNo(), llvm::SelectionDAG::getSetCC(), llvm::isa(), llvm::isNullConstant(), N, llvm::ISD::OR, SDValue(), llvm::ISD::SETCCCARRY, llvm::ISD::SETEQ, llvm::ISD::SETNE, llvm::ISD::SETULT, and llvm::ISD::USUBO.

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performSETCCCombine()

SDValue performSETCCCombine ( SDNode * N, TargetLowering::DAGCombinerInfo & DCI, SelectionDAG & DAG )

static

Definition at line 29954 of file AArch64ISelLowering.cpp.

References llvm::ISD::AND, assert(), llvm::ISD::BITCAST, Cond, DL, getCondCode(), llvm::SelectionDAG::getConstant(), llvm::EVT::getFixedSizeInBits(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNOT(), llvm::SelectionDAG::getSetCC(), llvm::SelectionDAG::getSignedConstant(), llvm::EVT::getVectorElementType(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::isa(), llvm::TargetLowering::DAGCombinerInfo::isAfterLegalizeDAG(), llvm::isAllOnesConstant(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::ISD::isConstantSplatVector(), llvm::EVT::isFixedLengthVector(), llvm::EVT::isInteger(), llvm::isNullConstant(), llvm::APInt::isOne(), llvm::isOneConstant(), llvm::EVT::isScalarInteger(), isZerosVector(), LHS, N, performOrXorChainCombine(), RHS, SDValue(), llvm::ISD::SETCC, llvm::ISD::SETEQ, llvm::ISD::SETGE, llvm::ISD::SETGT, llvm::ISD::SETNE, llvm::ISD::SIGN_EXTEND, llvm::ISD::SRL, tryToWidenSetCCOperands(), uint64_t, V, llvm::ISD::VECREDUCE_AND, llvm::ISD::VECREDUCE_OR, and llvm::ISD::ZERO_EXTEND.

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine(), llvm::LoongArchTargetLowering::PerformDAGCombine(), llvm::MipsSETargetLowering::PerformDAGCombine(), and llvm::RISCVTargetLowering::PerformDAGCombine().

performSetccMergeZeroCombine()

SDValue performSetccMergeZeroCombine ( SDNode * N, TargetLowering::DAGCombinerInfo & DCI )

static

Definition at line 30216 of file AArch64ISelLowering.cpp.

References llvm::ISD::AND, assert(), llvm::SelectionDAG::ComputeNumSignBits(), Cond, llvm::TargetLowering::DAGCombinerInfo::DAG, DL, llvm::SDValue::getConstantOperandVal(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::ISD::INSERT_SUBVECTOR, llvm::TargetLowering::DAGCombinerInfo::isAfterLegalizeDAG(), isAllActivePredicate(), llvm::isNullConstant(), isSignExtInReg(), llvm::SDValue::isUndef(), isZerosVector(), LHS, N, performSetCCPunpkCombine(), Pred, RHS, SDValue(), llvm::ISD::SETNE, llvm::ISD::SIGN_EXTEND, and V.

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performSetCCPunpkCombine()

SDValue performSetCCPunpkCombine ( SDNode * N, SelectionDAG & DAG )

static

Definition at line 30166 of file AArch64ISelLowering.cpp.

References Cond, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SDNode::getConstantOperandVal(), llvm::SDValue::getConstantOperandVal(), llvm::SDNode::getOpcode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::SDValue::getOperand(), llvm::SDNode::getValueType(), isZerosVector(), LHS, N, Pred, RHS, SDValue(), llvm::ISD::SETNE, and llvm::ISD::SIGN_EXTEND.

Referenced by performSetccMergeZeroCombine().

performSHLCombine()

SDValue performSHLCombine ( SDNode * N, TargetLowering::DAGCombinerInfo & DCI, SelectionDAG & DAG )

static

If the operand is a bitwise AND with a constant RHS, and the shift has a constant RHS and is the only use, we can pull it out of the shift, i.e.

(shl (and X, C1), C2) -> (and (shl X, C2), (shl C1, C2))

We prefer this canonical form to match existing isel patterns.

Definition at line 32051 of file AArch64ISelLowering.cpp.

References llvm::ISD::ADD, llvm::ISD::AND, C1, DL, llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::SDValue::hasOneUse(), llvm::isa(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), N, SDValue(), llvm::ISD::SETCC, llvm::ISD::SHL, llvm::ISD::SUB, and X.

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine(), llvm::LoongArchTargetLowering::PerformDAGCombine(), llvm::MipsSETargetLowering::PerformDAGCombine(), llvm::MipsTargetLowering::PerformDAGCombine(), and llvm::RISCVTargetLowering::PerformDAGCombine().

performSignExtendInRegCombine()

SDValue performSignExtendInRegCombine ( SDNode * N, TargetLowering::DAGCombinerInfo & DCI, SelectionDAG & DAG )

static

Definition at line 31172 of file AArch64ISelLowering.cpp.

References AbstractManglingParser< Derived, Alloc >::Ops, llvm::cast(), llvm::TargetLowering::DAGCombinerInfo::CombineTo(), DL, llvm::SelectionDAG::getAllOnesConstant(), llvm::AArch64Subtarget::getCLOpts(), llvm::SelectionDAG::getContext(), llvm::EVT::getDoubleNumVectorElementsVT(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::SDValue::getOperand(), llvm::EVT::getScalarType(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getSubtarget(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getValueType(), llvm::EVT::getVectorElementType(), llvm::SelectionDAG::getVTList(), I, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::isNullConstant(), llvm::isOneConstant(), N, Opc, SDValue(), and llvm::ISD::SIGN_EXTEND_INREG.

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performSignExtendSetCCCombine()

SDValue performSignExtendSetCCCombine ( SDNode * N, TargetLowering::DAGCombinerInfo & DCI, SelectionDAG & DAG )

static

Definition at line 26088 of file AArch64ISelLowering.cpp.

References assert(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperandCondCode(), llvm::SelectionDAG::getSetCC(), llvm::SDNode::getValueType(), isCheapToExtend(), llvm::EVT::isInteger(), N, SDValue(), llvm::ISD::SETCC, llvm::ISD::SIGN_EXTEND, and llvm::ISD::ZERO_EXTEND.

Referenced by performExtendCombine().

performSpliceCombine()

SDValue performSpliceCombine ( SDNode * N, SelectionDAG & DAG )

static

Definition at line 26920 of file AArch64ISelLowering.cpp.

References assert(), N, and SDValue().

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performST1Combine()

SDValue performST1Combine ( SDNode * N, SelectionDAG & DAG )

static

Definition at line 26673 of file AArch64ISelLowering.cpp.

References AbstractManglingParser< Derived, Alloc >::Ops, llvm::ISD::ANY_EXTEND, llvm::ISD::BITCAST, llvm::Data, DL, llvm::SelectionDAG::getNode(), getSVEContainerType(), llvm::SelectionDAG::getValueType(), llvm::EVT::isFloatingPoint(), and N.

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performSTNT1Combine()

SDValue performSTNT1Combine ( SDNode * N, SelectionDAG & DAG )

static

Definition at line 26699 of file AArch64ISelLowering.cpp.

References llvm::ISD::BITCAST, llvm::cast(), llvm::EVT::changeTypeToInteger(), llvm::Data, DL, llvm::SelectionDAG::getMaskedStore(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getPOISON(), llvm::EVT::isFloatingPoint(), N, and llvm::ISD::UNINDEXED.

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performSubAddMULCombine()

SDValue performSubAddMULCombine ( SDNode * N, SelectionDAG & DAG )

static

Definition at line 24410 of file AArch64ISelLowering.cpp.

References llvm::ISD::ADD, llvm::Add, llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SelectionDAG::isConstantIntBuildVectorOrConstantInt(), llvm::M1(), llvm::ISD::MUL, N, SDValue(), llvm::ISD::SUB, llvm::Sub, and X.

Referenced by performAddSubCombine().

performSubNegAndOneCombine()

SDValue performSubNegAndOneCombine ( SDNode * N, SelectionDAG & DAG )

static

Definition at line 24877 of file AArch64ISelLowering.cpp.

References llvm::ISD::AND, DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getSetCC(), llvm::EVT::isFixedLengthVector(), isOneVector(), isZerosVector(), N, SDValue(), llvm::ISD::SETNE, llvm::ISD::SUB, and Zero.

Referenced by performAddSubCombine().

performSubsToAndsCombine()

SDValue performSubsToAndsCombine ( SDNode * N, SDNode * SubsNode, SDNode * AndNode, SelectionDAG & DAG, unsigned CCIndex, unsigned CmpIndex, unsigned CC )

static

Definition at line 29242 of file AArch64ISelLowering.cpp.

References AbstractManglingParser< Derived, Alloc >::Ops, assert(), CC, DL, llvm::dyn_cast(), llvm::AArch64CC::EQ, llvm::ConstantSDNode::getAPIntValue(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::SDValue::getValue(), llvm::SDNode::getValueType(), llvm::SDNode::getVTList(), llvm::AArch64CC::HI, llvm::APInt::isMask(), llvm::APInt::isPowerOf2(), llvm::AArch64CC::LO, N, llvm::AArch64CC::NE, and SDValue().

Referenced by performCONDCombine().

performSubWithBorrowCombine()

SDValue performSubWithBorrowCombine ( SDNode * N, SelectionDAG & DAG )

static

Definition at line 24801 of file AArch64ISelLowering.cpp.

References CC, DL, llvm::dyn_cast(), llvm::ConstantSDNode::getAPIntValue(), llvm::SelectionDAG::getConstant(), getCSETCondCode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValue(), llvm::SDValue::hasOneUse(), llvm::AArch64CC::HI, llvm::AArch64_AM::isLegalCmpImmed(), llvm::AArch64CC::LO, N, SDValue(), llvm::ISD::SUB, and llvm::ISD::ZERO_EXTEND.

Referenced by performAddSubCombine().

performSunpkloCombine()

SDValue performSunpkloCombine ( SDNode * N, SelectionDAG & DAG )

static

Definition at line 27372 of file AArch64ISelLowering.cpp.

References CC, llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getExtractSubvector(), llvm::SelectionDAG::getNode(), N, SDValue(), and llvm::ISD::SIGN_EXTEND.

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performSVEAndCombine()

SDValue performSVEAndCombine ( SDNode * N, TargetLowering::DAGCombinerInfo & DCI )

static

Definition at line 22415 of file AArch64ISelLowering.cpp.

References llvm::ISD::AND, C(), llvm::cast(), llvm::TargetLowering::DAGCombinerInfo::DAG, DL, llvm::dyn_cast(), llvm::ISD::EXTLOAD, llvm::AArch64Subtarget::getCLOpts(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getSubtarget(), llvm::SDNode::getValueType(), llvm::EVT::getVectorElementType(), isAllActivePredicate(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), isConstantSplatVectorMaskForType(), N, Opc, SDValue(), llvm::ISD::SPLAT_VECTOR, uint64_t, and llvm::ISD::ZEXTLOAD.

Referenced by performANDCombine().

performSVEMulAddSubCombine()

SDValue performSVEMulAddSubCombine ( SDNode * N, TargetLowering::DAGCombinerInfo & DCI )

static

Definition at line 24446 of file AArch64ISelLowering.cpp.

References llvm::ISD::ADD, llvm::cast(), convertFromScalableVector(), convertToScalableVector(), llvm::TargetLowering::DAGCombinerInfo::DAG, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getSubtarget(), llvm::SDValue::getValueType(), llvm::SDValue::hasOneUse(), llvm::TargetLowering::DAGCombinerInfo::isAfterLegalizeDAG(), llvm::EVT::isScalableVector(), N, SDValue(), and llvm::ISD::SUB.

Referenced by performAddSubCombine().

performTBZCombine()

SDValue performTBZCombine ( SDNode * N, TargetLowering::DAGCombinerInfo & DCI, SelectionDAG & DAG )

static

Definition at line 30363 of file AArch64ISelLowering.cpp.

References assert(), DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), getTestBitOperand(), N, and SDValue().

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performTruncateCombine()

SDValue performTruncateCombine ( SDNode * N, SelectionDAG & DAG, TargetLowering::DAGCombinerInfo & DCI )

static

Definition at line 24214 of file AArch64ISelLowering.cpp.

References assert(), llvm::ISD::BUILD_VECTOR, llvm::cast(), llvm::SelectionDAG::ComputeNumSignBits(), DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getExtractVectorElt(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::SDValue::getValueType(), llvm::SDValue::hasOneUse(), llvm::EVT::is64BitVector(), llvm::isa(), llvm::TargetLowering::DAGCombinerInfo::isAfterLegalizeDAG(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::EVT::isFixedLengthVector(), llvm::EVT::isScalableVector(), N, llvm::SDNode::ops(), llvm::ISD::OR, SDValue(), and llvm::ISD::TRUNCATE.

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performUADDVAddCombine()

SDValue performUADDVAddCombine ( SDValue A, SelectionDAG & DAG )

static

Definition at line 21183 of file AArch64ISelLowering.cpp.

References A(), llvm::ISD::ADD, assert(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::SDValue::getConstantOperandVal(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::EVT::isScalableVector(), performUADDVAddCombine(), SDValue(), llvm::ISD::SIGN_EXTEND, and llvm::ISD::ZERO_EXTEND.

Referenced by performUADDVAddCombine(), and performUADDVCombine().

performUADDVCombine()

SDValue performUADDVCombine ( SDNode * N, SelectionDAG & DAG )

static

Definition at line 21271 of file AArch64ISelLowering.cpp.

References A(), llvm::ISD::ADD, assert(), llvm::SelectionDAG::computeKnownBits(), llvm::APInt::getAllOnes(), llvm::SelectionDAG::getNode(), llvm::MVT::getVectorNumElements(), llvm::KnownBits::isZero(), N, performUADDVAddCombine(), performUADDVZextCombine(), and SDValue().

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performUADDVZextCombine()

SDValue performUADDVZextCombine ( SDValue A, SelectionDAG & DAG )

static

Definition at line 21233 of file AArch64ISelLowering.cpp.

References A(), llvm::ISD::ADD, assert(), Concat, llvm::ISD::CONCAT_VECTORS, llvm::SelectionDAG::getContext(), llvm::EVT::getDoubleNumVectorElementsVT(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getSimpleVT(), llvm::SDValue::getValueType(), llvm_unreachable, SDValue(), llvm::MVT::SimpleTy, and llvm::ISD::ZERO_EXTEND.

Referenced by performUADDVCombine().

performUnpackCombine()

SDValue performUnpackCombine ( SDNode * N, SelectionDAG & DAG, const AArch64Subtarget * Subtarget )

static

Definition at line 26930 of file AArch64ISelLowering.cpp.

References assert(), llvm::cast(), llvm::EVT::changeVectorElementType(), DL, llvm::MaskedLoadStoreSDNode::getAddressingMode(), llvm::MaskedLoadSDNode::getBasePtr(), llvm::MemSDNode::getChain(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::MaskedLoadSDNode::getExtensionType(), llvm::MaskedLoadSDNode::getMask(), llvm::SelectionDAG::getMaskedLoad(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getMemoryVT(), llvm::SDValue::getNode(), llvm::getNumElementsFromSVEPredPattern(), llvm::MaskedLoadSDNode::getOffset(), llvm::MaskedLoadSDNode::getPassThru(), getPTrue(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValue(), llvm::EVT::getVectorElementType(), llvm::SDNode::isUndef(), llvm::MaskedLoadStoreSDNode::isUnindexed(), isZerosVector(), llvm::ISD::MLOAD, N, llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), SDValue(), llvm::ISD::SEXTLOAD, Subtarget, and llvm::ISD::ZEXTLOAD.

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performUzpCombine()

SDValue performUzpCombine ( SDNode * N, SelectionDAG & DAG, const AArch64Subtarget * Subtarget )

static

Definition at line 27076 of file AArch64ISelLowering.cpp.

References llvm::ISD::BITCAST, Concat, llvm::ISD::CONCAT_VECTORS, DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SDNode::getAsZExtVal(), llvm::SelectionDAG::getBitcast(), llvm::SDValue::getConstantOperandVal(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::EVT::getDoubleNumVectorElementsVT(), llvm::SelectionDAG::getExtractSubvector(), llvm::EVT::getFixedSizeInBits(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getPOISON(), llvm::EVT::getScalarSizeInBits(), llvm::SDValue::getSimpleValueType(), llvm::EVT::getSimpleVT(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::EVT::getVectorMinNumElements(), llvm::isConstOrConstSplat(), isHalvingTruncateAndConcatOfLegalIntScalableType(), llvm::DataLayout::isLittleEndian(), isNVCastToHalfWidthElements(), llvm::EVT::isScalableVector(), llvm::EVT::isSimple(), llvm::SDValue::isUndef(), llvm_unreachable, N, llvm::peekThroughBitcasts(), SDValue(), llvm::MVT::SimpleTy, llvm::ISD::SRL, Subtarget, TLI, llvm::ISD::TRUNCATE, tryCombineExtendRShTrunc(), trySimplifySrlAddToRshrnb(), uint64_t, and X.

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performVecReduceAddCntpCombine()

SDValue performVecReduceAddCntpCombine ( SDNode * N, SelectionDAG & DAG, const AArch64Subtarget * ST )

static

Definition at line 21032 of file AArch64ISelLowering.cpp.

References DL, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::ISD::INTRINSIC_WO_CHAIN, llvm::EVT::isScalableVectorOf(), llvm::TargetLoweringBase::isTypeLegal(), N, SDValue(), and llvm::ISD::ZERO_EXTEND.

Referenced by performVecReduceAddCombine().

performVecReduceAddCombine()

SDValue performVecReduceAddCombine ( SDNode * N, SelectionDAG & DAG, const AArch64Subtarget * ST )

static

Definition at line 21052 of file AArch64ISelLowering.cpp.

References A(), llvm::ISD::ADD, B(), llvm::ISD::CONCAT_VECTORS, DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getExtractSubvector(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::EVT::getVectorVT(), I, llvm::EVT::isScalableVT(), llvm::ISD::MUL, N, performVecReduceAddCntpCombine(), performVecReduceAddCombineWithUADDLP(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), SDValue(), llvm::ISD::SIGN_EXTEND, std::swap(), llvm::ISD::VECREDUCE_ADD, and llvm::ISD::ZERO_EXTEND.

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performVecReduceAddCombineWithUADDLP()

SDValue performVecReduceAddCombineWithUADDLP ( SDNode * N, SelectionDAG & DAG )

static

Definition at line 20838 of file AArch64ISelLowering.cpp.

References llvm::ISD::ABDS, llvm::ISD::ABDU, llvm::ISD::ADD, DL, llvm::SelectionDAG::getExtractSubvector(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::SDValue::getOperand(), llvm::SDNode::getValueType(), llvm::ISD::isAbsOpcode(), N, SDValue(), llvm::ISD::SIGN_EXTEND, llvm::ISD::SUB, llvm::ISD::VECREDUCE_ADD, and llvm::ISD::ZERO_EXTEND.

Referenced by performVecReduceAddCombine().

performVecReduceBitwiseCombine()

SDValue performVecReduceBitwiseCombine ( SDNode * N, TargetLowering::DAGCombinerInfo & DCI, SelectionDAG & DAG )

static

Definition at line 29939 of file AArch64ISelLowering.cpp.

References DL, llvm::SDValue::getValueType(), getVectorBitwiseReduce(), llvm::EVT::getVectorElementType(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::EVT::isFixedLengthVector(), llvm::EVT::isPow2VectorType(), N, and SDValue().

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performVectorCompareAndMaskUnaryOpCombine()

SDValue performVectorCompareAndMaskUnaryOpCombine ( SDNode * N, SelectionDAG & DAG )

static

Definition at line 22005 of file AArch64ISelLowering.cpp.

References llvm::ISD::AND, llvm::ISD::BITCAST, DL, llvm::dyn_cast(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::EVT::isVector(), N, SDValue(), and llvm::ISD::SETCC.

Referenced by performIntToFpCombine().

performVectorDeinterleaveCombine()

SDValue performVectorDeinterleaveCombine ( SDNode * N, TargetLowering::DAGCombinerInfo & DCI, SelectionDAG & DAG )

static

Definition at line 31910 of file AArch64ISelLowering.cpp.

References llvm::TargetLowering::DAGCombinerInfo::CombineTo(), DL, llvm::dyn_cast(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getConstant(), llvm::SDNode::getConstantOperandVal(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getMemIntrinsicNode(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getMemoryVT(), llvm::SDValue::getNode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getSubtarget(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValue(), llvm::SelectionDAG::getVTList(), I, llvm::ISD::INTRINSIC_W_CHAIN, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), isDeinterleave4ForWideningUToFP(), llvm::ISD::isNormalLoad(), llvm::ISD::isNormalMaskedLoad(), IsScalable, isZerosVector(), llvm::Load, N, NumParts, performLegalizedVectorDeinterleaveCombine(), Pred, llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), SDValue(), Subtarget, SubVecTy, and TLI.

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performVectorExtCombine()

SDValue performVectorExtCombine ( SDNode * N, SelectionDAG & DAG )

static

Definition at line 21682 of file AArch64ISelLowering.cpp.

References DL, llvm::SelectionDAG::getContext(), llvm::EVT::getHalfSizedIntegerVT(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarType(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementCount(), llvm::EVT::getVectorVT(), llvm::EVT::isFixedLengthVector(), llvm::ISD::MUL, N, S1, SDValue(), llvm::ISD::SIGN_EXTEND, and llvm::ISD::ZERO_EXTEND.

Referenced by performAddSubCombine(), and performMulCombine().

performVectorShiftCombine()

SDValue performVectorShiftCombine ( SDNode * N, const AArch64TargetLowering & TLI, TargetLowering::DAGCombinerInfo & DCI )

static

Optimize a vector shift instruction and its operand if shifted out bits are not used.

Definition at line 27340 of file AArch64ISelLowering.cpp.

References assert(), llvm::SelectionDAG::ComputeNumSignBits(), llvm::TargetLowering::DAGCombinerInfo::DAG, llvm::APInt::getLowBitsSet(), N, SDValue(), and TLI.

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performVSelectCombine()

SDValue performVSelectCombine ( SDNode * N, TargetLowering::DAGCombinerInfo & DCI, const AArch64Subtarget * Subtarget )

static

Definition at line 30562 of file AArch64ISelLowering.cpp.

References AbstractManglingParser< Derived, Alloc >::Ops, llvm::ISD::BITCAST, llvm::EVT::changeVectorElementTypeToInteger(), combineToExtendBoolVectorInReg(), llvm::TargetLowering::DAGCombinerInfo::DAG, DL, llvm::ElementCount::getFixed(), llvm::SelectionDAG::getNode(), llvm::SDValue::getNumOperands(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::SDValue::getOperand(), llvm::SDValue::getOperandCondCode(), llvm::SelectionDAG::getSelect(), llvm::SelectionDAG::getSetCC(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementCount(), llvm::EVT::getVectorElementType(), llvm::SDValue::hasOneUse(), isAllActivePredicate(), isAllInactivePredicate(), llvm::EVT::isFloatingPoint(), isMergePassthruOpcode(), llvm::SDValue::isUndef(), llvm::EVT::isVector(), N, llvm::SDNode::op_values(), performVSelectMaskedShiftCombine(), performVselectPowCombine(), SDValue(), llvm::ISD::SETCC, llvm::ISD::SIGN_EXTEND, Subtarget, llvm::ISD::TRUNCATE, trySwapVSelectOperands(), and llvm::ISD::VSELECT.

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performVSelectMaskedShiftCombine()

SDValue performVSelectMaskedShiftCombine ( SDNode * N, SelectionDAG & DAG, const AArch64Subtarget * Subtarget )

static

Definition at line 30543 of file AArch64ISelLowering.cpp.

References foldMaskedShiftToUSHL(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), matchZeroSelectArm(), N, SDValue(), llvm::ISD::SHL, llvm::ISD::SRL, and Subtarget.

Referenced by performVSelectCombine().

performVselectPowCombine()

SDValue performVselectPowCombine ( SDNode * N, TargetLowering::DAGCombinerInfo & DCI )

static

Definition at line 30427 of file AArch64ISelLowering.cpp.

References assert(), Cond, llvm::TargetLowering::DAGCombinerInfo::DAG, DL, llvm::ISD::FPOW, llvm::SelectionDAG::getConstantFP(), llvm::SDNode::getFlags(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getTargetLoweringInfo(), N, SDValue(), TLI, and llvm::ISD::VSELECT.

Referenced by performVSelectCombine().

performXorCombine()

SDValue performXorCombine ( SDNode * N, SelectionDAG & DAG, TargetLowering::DAGCombinerInfo & DCI, const AArch64Subtarget * Subtarget )

static

Definition at line 21306 of file AArch64ISelLowering.cpp.

References foldVectorXorShiftIntoCmp(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), N, SDValue(), and Subtarget.

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performZExtDeinterleaveShuffleCombine()

SDValue performZExtDeinterleaveShuffleCombine ( SDNode * N, SelectionDAG & DAG )

static

Definition at line 26128 of file AArch64ISelLowering.cpp.

References llvm::all_of(), llvm::ISD::AND, assert(), DL, llvm::dyn_cast(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getVectorNumElements(), llvm::ShuffleVectorInst::isDeInterleaveMaskOfFactor(), llvm::EVT::isFixedLengthVector(), N, SDValue(), llvm::ISD::SRL, and llvm::ISD::ZERO_EXTEND.

Referenced by performExtendCombine().

performZExtUZPCombine()

SDValue performZExtUZPCombine ( SDNode * N, SelectionDAG & DAG )

static

Definition at line 26194 of file AArch64ISelLowering.cpp.

References llvm::ISD::AND, DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getConstant(), llvm::APInt::getLowBitsSet(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), llvm::SelectionDAG::getTargetConstant(), llvm::EVT::getVectorNumElements(), llvm::ISD::isConstantSplatVector(), N, SDValue(), and llvm::ISD::ZERO_EXTEND.

Referenced by performExtendCombine().

reassociateAddSubLong()

SDValue reassociateAddSubLong ( SDNode * N, SelectionDAG & DAG )

static

Definition at line 23819 of file AArch64ISelLowering.cpp.

References A(), llvm::ISD::ADD, B(), DL, llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), isEssentiallyExtractHighSubvector(), llvm::PatternMatch::m_Add(), llvm::SDPatternMatch::m_Node(), llvm::MIPatternMatch::m_OneUse(), llvm::SDPatternMatch::m_SpecificVectorElementVT(), llvm::PatternMatch::m_Sub(), llvm::PatternMatchHelpers::m_Unless(), llvm::PatternMatch::m_Value(), N, llvm::SDPatternMatch::sd_match(), SDValue(), llvm::ISD::SIGN_EXTEND, llvm::ISD::SUB, X, and llvm::ISD::ZERO_EXTEND.

Referenced by performAddSubLongCombine().

reassociateCSELOperandsForCSE()

SDValue reassociateCSELOperandsForCSE ( SDNode * N, SelectionDAG & DAG )

static

Definition at line 29575 of file AArch64ISelLowering.cpp.

References llvm::ISD::ADD, CC, Check, llvm::dyn_cast(), llvm::AArch64CC::EQ, FlagsVT, llvm::AArch64CC::GE, getCondCode(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), getSwappedCondition(), llvm::SDValue::getValue(), llvm::SelectionDAG::getVTList(), llvm::AArch64CC::GT, llvm::SDValue::hasOneUse(), llvm::AArch64CC::HI, llvm::AArch64CC::HS, llvm::AArch64CC::LE, llvm::AArch64CC::LO, llvm::AArch64CC::LS, llvm::AArch64CC::LT, N, llvm::AArch64CC::NE, llvm::SelectionDAG::ReplaceAllUsesWith(), SDValue(), llvm::ISD::SUB, std::swap(), X, and Y.

Referenced by performCSELCombine().

ReconstructShuffleWithRuntimeMask()

SDValue ReconstructShuffleWithRuntimeMask	(	SDValue	Op,
		SelectionDAG &	DAG )

Definition at line 14837 of file AArch64ISelLowering.cpp.

References llvm::ISD::AND, llvm::ISD::ANY_EXTEND, assert(), llvm::ISD::BUILD_VECTOR, llvm::cast(), llvm::ISD::CONCAT_VECTORS, DL, llvm::SmallVectorTemplateCommon< T, typename >::empty(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getPOISON(), llvm::SelectionDAG::getTargetConstant(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::ISD::INTRINSIC_WO_CHAIN, llvm::isa(), llvm::EVT::isScalableVector(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), SDValue(), and V.

ReconstructTruncateFromBuildVector()

SDValue ReconstructTruncateFromBuildVector ( SDValue V, SelectionDAG & DAG )

static

Definition at line 15228 of file AArch64ISelLowering.cpp.

References assert(), llvm::sampleprof::Base, llvm::ISD::CONCAT_VECTORS, DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SDValue::getConstantOperandVal(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValueType(), llvm::isa(), SDValue(), llvm::ISD::TRUNCATE, V, X, and Y.

removeRedundantInsertVectorElt()

SDValue removeRedundantInsertVectorElt ( SDNode * N )

static

Definition at line 31406 of file AArch64ISelLowering.cpp.

References assert(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SDValue::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValueType(), llvm::ISD::INSERT_VECTOR_ELT, llvm::ISD::isConstantSplatVectorAllZeros(), isLanes1toNKnownZero(), llvm::isNullConstant(), N, and SDValue().

Referenced by performInsertVectorEltCombine().

ReplaceAddWithADDP()

void ReplaceAddWithADDP ( SDNode * N, SmallVectorImpl< SDValue > & Results, SelectionDAG & DAG, const AArch64Subtarget * Subtarget )

static

Definition at line 32909 of file AArch64ISelLowering.cpp.

References assert(), llvm::ISD::CONCAT_VECTORS, DL, llvm::dyn_cast(), E(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getPOISON(), llvm::EVT::getScalarType(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), I, llvm::EVT::is256BitVector(), llvm::EVT::isFloatingPoint(), N, llvm::SmallVectorTemplateBase< T, bool >::push_back(), Results, llvm::SelectionDAG::SplitVector(), Subtarget, and X.

ReplaceATOMIC_LOAD_128Results()

void ReplaceATOMIC_LOAD_128Results ( SDNode * N, SmallVectorImpl< SDValue > & Results, SelectionDAG & DAG, const AArch64Subtarget * Subtarget )

static

Definition at line 33238 of file AArch64ISelLowering.cpp.

References AbstractManglingParser< Derived, Alloc >::Ops, assert(), llvm::ISD::ATOMIC_LOAD_AND, llvm::ISD::BUILD_PAIR, llvm::cast(), DL, llvm::SelectionDAG::getAllOnesConstant(), getAtomicLoad128Opcode(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getMachineNode(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getVTList(), llvm::Hi, llvm::DataLayout::isBigEndian(), Lo, N, Results, SDValue(), llvm::SelectionDAG::setNodeMemRefs(), llvm::SelectionDAG::SplitScalar(), Subtarget, std::swap(), and llvm::ISD::XOR.

replaceBoolVectorBitcast()

void replaceBoolVectorBitcast ( SDNode * N, SmallVectorImpl< SDValue > & Results, SelectionDAG & DAG )

static

Definition at line 32818 of file AArch64ISelLowering.cpp.

References assert(), llvm::ISD::CONCAT_VECTORS, DL, llvm::SelectionDAG::getZExtOrTrunc(), I, llvm::EVT::isVectorOf(), N, and Results.

ReplaceCMP_SWAP_128Results()

void ReplaceCMP_SWAP_128Results ( SDNode * N, SmallVectorImpl< SDValue > & Results, SelectionDAG & DAG, const AArch64Subtarget * Subtarget )

static

Definition at line 33075 of file AArch64ISelLowering.cpp.

References AbstractManglingParser< Derived, Alloc >::Ops, llvm::Acquire, llvm::AcquireRelease, assert(), llvm::ISD::BUILD_PAIR, llvm::cast(), createGPRPairNode(), DL, llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getMachineNode(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetExtractSubreg(), llvm::SelectionDAG::getVTList(), llvm::Hi, llvm::DataLayout::isBigEndian(), llvm_unreachable, Lo, llvm::Monotonic, N, llvm::Release, Results, SDValue(), llvm::SequentiallyConsistent, llvm::SelectionDAG::setNodeMemRefs(), llvm::SelectionDAG::SplitScalar(), Subtarget, and std::swap().

ReplaceReductionResults()

void ReplaceReductionResults ( SDNode * N, SmallVectorImpl< SDValue > & Results, SelectionDAG & DAG, unsigned InterOp, unsigned AcrossOp )

static

Definition at line 32957 of file AArch64ISelLowering.cpp.

References DL, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::GetSplitDestVTs(), llvm::Hi, Lo, N, Results, and llvm::SelectionDAG::SplitVectorOperand().

replaceSplatVectorStore()

SDValue replaceSplatVectorStore ( SelectionDAG & DAG, StoreSDNode & St )

static

Replace a splat of a scalar to a vector store by scalar stores of the scalar value.

The load store optimizer pass will merge them to store pair stores. This has better performance than a splat of the scalar followed by a split vector store. Even if the stores are not merged it is four stores vs a dup, followed by an ext.b and two stores.

Definition at line 26798 of file AArch64ISelLowering.cpp.

References llvm::dyn_cast(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::ConstantSDNode::getZExtValue(), I, llvm::ISD::INSERT_VECTOR_ELT, llvm::EVT::isFloatingPoint(), SDValue(), splitStoreSplat(), and uint64_t.

Referenced by splitStores().

replaceZeroVectorStore()

SDValue replaceZeroVectorStore ( SelectionDAG & DAG, StoreSDNode & St )

static

Replace a splat of zeros to a vector store by scalar stores of WZR/XZR.

The load store optimizer pass will merge them to store pair stores. This should be better than a movi to create the vector zero followed by a vector store if the zero constant is not re-used, since one instructions and one register live range will be removed.

For example, the final generated code should be:

stp xzr, xzr, [x0]

instead of:

movi v0.2d, #0 str q0, [x0]

Definition at line 26731 of file AArch64ISelLowering.cpp.

References llvm::ISD::BUILD_VECTOR, DL, llvm::SelectionDAG::getCopyFromReg(), llvm::SelectionDAG::getEntryNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::SDValue::hasOneUse(), I, llvm::SelectionDAG::isBaseWithConstantOffset(), llvm::isNullConstant(), llvm::isNullFPConstant(), llvm::EVT::isScalableVector(), llvm::Offset, SDValue(), and splitStoreSplat().

Referenced by splitStores().

resolveBuildVector()

bool resolveBuildVector ( BuildVectorSDNode * BVN, APInt & CnstBits, APInt & UndefBits )

static

Definition at line 16381 of file AArch64ISelLowering.cpp.

References llvm::EVT::getSizeInBits(), llvm::SDNode::getValueType(), llvm::BuildVectorSDNode::isConstantSplat(), and llvm::APInt::zextOrTrunc().

Referenced by ConstantBuildVector(), and performANDCombine().

rmwOpMayLowerToLibcall()

bool rmwOpMayLowerToLibcall ( const AArch64Subtarget & Subtarget, const AtomicRMWInst * RMW )

static

Definition at line 33757 of file AArch64ISelLowering.cpp.

References llvm::Type::BFloatTyID, llvm::Type::DoubleTyID, llvm::Type::FloatTyID, llvm::Type::getScalarType(), llvm::Value::getType(), llvm::Type::getTypeID(), llvm::Type::HalfTyID, llvm::AtomicRMWInst::isFloatingPointOperation(), llvm_unreachable, and Subtarget.

Referenced by llvm::AArch64TargetLowering::shouldExpandAtomicRMWInIR().

SDValue()

return SDValue

(

)

Referenced by AddCombineBUILD_VECTORToVPADDL(), AddCombineTo64bitMLAL(), AddCombineTo64BitSMLAL16(), AddCombineTo64bitUMAAL(), AddCombineToVPADD(), AddCombineVUZPToVPADDL(), llvm::InstrEmitter::AddDbgValueLocationOps(), llvm::AMDGPUTargetLowering::addTokenForArgument(), adjustBitcastSrcVectorSSE1(), adjustForFNeg(), adjustForLTGFR(), adjustForStackGuardCompare(), adjustForSubtraction(), adjustForTestUnderMask(), adjustSubwordCmp(), llvm::RISCVDAGToDAGISel::areOffsetsWithinAlignment(), AVRDAGToDAGISel::select< AVRISD::CALL >(), AVRDAGToDAGISel::select< ISD::BRIND >(), AVRDAGToDAGISel::select< ISD::LOAD >(), AVRDAGToDAGISel::select< ISD::STORE >(), BuildExactSDIV(), BuildExactUDIV(), buildFromShuffleMostly(), buildGPRPair(), llvm::TargetLowering::buildLegalVectorShuffle(), llvm::TargetLowering::BuildSDIV(), llvm::PPCTargetLowering::BuildSDIVPow2(), llvm::TargetLowering::BuildSDIVPow2(), buildSMovImm32(), llvm::TargetLowering::BuildSREMPow2(), llvm::TargetLowering::BuildUDIV(), canonicalizeBitSelect(), canonicalizeBoolMask(), canonicalizeLaneShuffleWithRepeatedOps(), canonicalizeMaskForVLPredicate(), canonicalizePRMTInput(), canonicalizeShuffleMaskWithHorizOp(), canonicalizeShuffleWithOp(), canonicalizeVSelectTrueToOneUse(), llvm::AArch64TargetLowering::changeStreamingMode(), checkBoolTestSetCCCombine(), checkIntrinsicImmArg(), checkSignTestSetCCCombine(), combineADC(), combineAddMulh(), combineAddMulParts(), combineAddOfBooleanXor(), combineAddOfExts(), combineAddOfExtsAlreadyExtended(), combineAddOfPMADDWD(), combineAddOrSubToADCOrSBB(), combineAddOrSubToADCOrSBB(), combineAddPackedByteDot(), combineADDRSPACECAST(), combineADDToADDZE(), combineADDToMAT_PCREL_ADDR(), combineADDToSUB(), combineADDX(), combineADOX(), combineAnd(), combineAndMaskToShift(), combineAndNotIntoANDNP(), combineAndNotIntoVANDN(), combineAndNotOrIntoAndNotAnd(), combineAndnp(), combineAndOnGF2P8AFFINEQBOperand(), combineAndOrForCcmpCtest(), CombineANDShift(), combineAndShuffleNot(), combineAndXorSubWithBMI(), combineArithReduction(), combineAtomicLoad(), combineAVG(), combineAVX512SetCCToKMOV(), combineBallotPattern(), CombineBaseUpdate(), combineBasicSADPattern(), combineBinOpOfExt(), combineBinOpOfExtractToReduceTree(), combineBinOpToReduce(), combineBitcast(), combineBitcastToBoolVector(), combineBitcastvxi1(), combineBitOpWithMOVMSK(), combineBitOpWithPACK(), combineBitOpWithShift(), combineBITREVERSE(), combineBlendOfPermutes(), combineBMI(), combineBMILogicOp(), combineBrCond(), combineBROADCAST_LOAD(), combineBT(), combineBTToBitOpFlag(), combineBVOfConsecutiveLoads(), combineBVOfVecSExt(), combineBVZEXTLOAD(), combineCarryDiamond(), combineCarryThroughADD(), combineCarryThroughADD(), combineCastedMaskArithmetic(), combineCCMask(), combineCMov(), combineCMP(), combineCommutableSHUFP(), combineCompareEqual(), combineCONCAT_VECTORS(), combineConcatVectorOfCasts(), combineConcatVectorOfConcatVectors(), combineConcatVectorOfExtracts(), combineConcatVectorOfScalars(), combineConcatVectorOfShuffleAndItsOperands(), combineConcatVectorOfShuffles(), combineConcatVectorOfSplats(), combineConcatVectorOps(), combineConstantPoolLoads(), combineCVTP2I_CVTTP2I(), combineCVTPH2PS(), combineDeMorganOfBoolean(), combineEXTEND_VECTOR_INREG(), combineEXTRACT_SUBVECTOR(), combineExtractFromVectorLoad(), combineExtractVectorElt(), combineExtractWithShuffle(), combineExtSetcc(), combineFaddCFmul(), combineFaddFsub(), combineFAddWithNeg(), combineFAndFNotToFAndn(), combineFMA(), combineFMADDSUB(), combineFMinFMax(), llvm::AMDGPUTargetLowering::combineFMinMaxLegacy(), llvm::AMDGPUTargetLowering::combineFMinMaxLegacyImpl(), combineFMinNumFMaxNum(), combineFneg(), combineFP16_TO_FP(), combineFP_EXTEND(), combineFP_ROUND(), combineFP_ROUND(), combineFP_TO_xINT_SAT(), combineFPToSInt(), combineFunnelShift(), combineGatherScatter(), combineGF2P8AFFINEQB(), combineHorizOpWithShuffle(), combinei64TruncSrlConstant(), combineI8AndNotIntoI32AndNot(), combineINSERT_SUBVECTOR(), combineInt64DivRemViaFPReciprocal(), combineIntDivRem(), combineIntDivRemViaExactFPDiv(), combineINTRINSIC_VOID(), combineINTRINSIC_W_CHAIN(), combineINTRINSIC_WO_CHAIN(), combineIntrinsicWOChain(), combineKSHIFT(), combineLoad(), combineLogicBlendIntoConditionalNegate(), combineLogicBlendIntoPBLENDV(), combineLRINT_LLRINT(), combineM68kBrCond(), combineM68kSetCC(), combineMADConstOne(), combineMaskBeforeFirst(), combineMaskBitOp(), combineMaskedLoad(), combineMaskedLoadConstantMask(), combineMaskedStore(), combineMinMaxToSat(), combineMinNumMaxNumImpl(), combineMOVDQ2Q(), combineMOVMSK(), combineMul(), combineMulSelectConstOne(), combineMulSpecial(), combineMulToPMADD52(), combineMulToPMADDWD(), combineMulToPMULDQ(), combineNarrowableShiftedLoad(), combineNegOfSelectCCSubs(), combineNestedGF2P8AFFINEQB(), combineOp_VLToVWOp_VL(), combineOr(), combineOrAndToBitfieldInsert(), combineOrCmpEqZeroToCtlzSrl(), combineOrOfCZERO(), combineOrOfSetCCToUSUBOCarry(), combineOrToBitfieldInsert(), combineOrWithGF2P8AFFINEQB(), combineOrXorWithSETCC(), combinePackingMovIntoStore(), combinePCLMULQDQ(), combinePExtTruncate(), combinePExtWideningAddAcc(), combinePExtWideningAddSub(), combinePExtWideningMul(), combinePExtWideningSubAcc(), combinePMULDQ(), combinePMULH(), combinePRMT(), combineProxyReg(), combinePTESTCC(), combineRedundantDWordShuffle(), combineRotate(), combineSBB(), combineSCALAR_TO_VECTOR(), combineScalarAndWithMaskSetcc(), combineScalarMinMaxAbsStore(), combineSelect(), llvm::VETargetLowering::combineSelect(), combineSELECT_CCBitFloor(), combineSelectAndUse(), combineSelectAndUse(), combineSelectAndUse(), combineSelectAndUseCommutative(), combineSelectAndUseCommutative(), combineSelectAndUseCommutative(), combineSelectAsExtAnd(), llvm::VETargetLowering::combineSelectCC(), combineSelectOfTwoConstants(), combineSelectToBinOp(), combineSelectToPseudoMinMax(), combineSetCC(), combineSetCCAtomicArith(), combineSetCCCCR(), combineSetCCMOVMSK(), combineSext(), combineSextInRegCmov(), combineShiftAnd1ToBitTest(), combineShiftLeft(), combineShiftOfLogicOp(), combineShiftOfShiftedLogic(), combineShiftRightArithmetic(), combineShiftRightLogical(), combineShiftToAVG(), combineShiftToAVGSUB(), combineShiftToMULH(), combineShiftToPMULH(), combineShlAddIAdd(), combineShlAddIAddImpl(), combineShuffle(), combineShuffleOfBitcast(), combineShuffleOfScalars(), combineShuffleOfSplatVal(), combineShuffleToAddSubOrFMAddSub(), combineShuffleToAnyExtendVectorInreg(), combineShuffleToFMAddSub(), combineShuffleToZeroExtendVectorInReg(), combineSignExtendInReg(), combineSIntToFP(), combineSTORE(), combineStore(), combineSubABS(), combineSubOfBoolean(), combineSubSetcc(), combineSubShiftToOrcB(), combineSUBX(), combineSVEPrefetchVecBaseImmOff(), combineSZExtToMulWide(), combineTargetShuffle(), combineTESTP(), combineToConsecutiveLoads(), combineToExtendBoolVectorInReg(), combineToExtendBoolVectorInReg(), combineToExtendCMOV(), combineToFPTruncExtElt(), combineToHorizontalAddSub(), combineToVCPOP(), combineToVWMACC(), llvm::VETargetLowering::combineTRUNCATE(), combineTruncate(), combineTruncatedArithmetic(), combineTruncateWithSat(), combineTruncationShuffle(), combineTruncOfSraSext(), combineUADDO_CARRYDiamond(), combineUIntToFP(), combineUnpackingMovIntoLoad(), combineVdot4aAccum(), combineVECREDUCE_LOGIC(), combineVECREDUCE_MUL(), combineVectorCompare(), combineVectorCompareAndMaskUnaryOp(), combineVectorHADDSUB(), combineVectorInsert(), combineVectorMulToSraBitcast(), combineVectorPack(), combineVectorShiftImm(), combineVectorShiftUniformVar(), combineVectorSizedSetCCEquality(), combineVectorSizedSetCCEquality(), combineVectorSizedSetCCEquality(), combineVEXTRACT_STORE(), combineVFMADD_VLWithVFNEG_VL(), CombineVLDDUP(), combineVMNOTOfSetCC(), CombineVMOVDRRCandidateWithVecOp(), combineVPDPBUSDPattern(), combineVPMADD(), combineVPMADD52LH(), combineVSelectToBLENDV(), combineVSelectWithAllOnesOrZeros(), combineVTRUNC(), combineVTRUNCSAT(), combineVWADDSUBWSelect(), combineX86AddSub(), combineX86CloadCstore(), combineX86FPLogicOp(), combineX86GatherScatter(), combineX86INT_TO_FP(), combineX86SetCC(), combineX86ShuffleChain(), combineX86ShuffleChainWithExtract(), combineX86ShufflesConstants(), combineX86ShufflesRecursively(), combineX86SubCmpForFlags(), combineX86XOR(), combineXor(), combineXorOnGF2P8AFFINEQBOperand(), combineXorSelectCC(), combineXorSubCTLZ(), combineXorToBitfieldInsert(), combineXorWithGF2P8AFFINEQB(), combineXorWithTwoGF2P8AFFINEQB(), combineZext(), combineZextSetccWithZero(), commuteSelect(), compressShuffleOfShuffles(), ConstantBuildVector(), constructOperandInfo(), constructRetValue(), convertFPToInt(), convertFromF16(), convertIntLogicToFPLogic(), convertMergedOpToPredOp(), convertMulToShlAdd(), ConvertSelectToConcatVector(), convertShiftLeftToScale(), convertTwoLoadsAndCmpToVCMPEQUB(), llvm::SITargetLowering::copyToM0(), createGPRPairNode(), createGPRPairNode2xi32(), llvm::TargetLowering::createSelectForFMINNUM_FMAXNUM(), createVariablePermute(), llvm::TargetLowering::CTTZTableLookup(), DAGCombineAddc(), DAGCombineSube(), detectPMADDUBSW(), detectSSatPattern(), detectSSatSPattern(), detectSSatUPattern(), detectUSatPattern(), detectUSatUPattern(), llvm::HexagonDAGToDAGISel::DetectUseSxtw(), determineConstraints(), llvm::SDNode::DropOperands(), eliminateFPCastPair(), EltsFromConsecutiveLoads(), EmitAVX512Test(), emitCmp(), emitConjunction(), emitConjunctionForCCMP(), emitConstantSizeRepmov(), emitConstantSizeRepstos(), llvm::InstrEmitter::EmitDbgInstrRef(), emitIntrinsicWithCCAndChain(), emitLockedStackOp(), llvm::AArch64SelectionDAGInfo::EmitMOPS(), llvm::ARMSelectionDAGInfo::EmitSpecializedLibcall(), llvm::X86TargetLowering::emitStackGuardMixFP(), llvm::AArch64SelectionDAGInfo::EmitStreamingCompatibleMemLibCall(), llvm::SelectionDAGTargetInfo::EmitTargetCodeForMemccpy(), llvm::AArch64SelectionDAGInfo::EmitTargetCodeForMemchr(), llvm::SelectionDAGTargetInfo::EmitTargetCodeForMemchr(), llvm::SelectionDAGTargetInfo::EmitTargetCodeForMemcmp(), llvm::AArch64SelectionDAGInfo::EmitTargetCodeForMemcpy(), llvm::ARMSelectionDAGInfo::EmitTargetCodeForMemcpy(), llvm::BPFSelectionDAGInfo::EmitTargetCodeForMemcpy(), llvm::HexagonSelectionDAGInfo::EmitTargetCodeForMemcpy(), llvm::LanaiSelectionDAGInfo::EmitTargetCodeForMemcpy(), llvm::SelectionDAGTargetInfo::EmitTargetCodeForMemcpy(), llvm::SystemZSelectionDAGInfo::EmitTargetCodeForMemcpy(), llvm::WebAssemblySelectionDAGInfo::EmitTargetCodeForMemcpy(), llvm::X86SelectionDAGInfo::EmitTargetCodeForMemcpy(), llvm::XCoreSelectionDAGInfo::EmitTargetCodeForMemcpy(), llvm::AArch64SelectionDAGInfo::EmitTargetCodeForMemmove(), llvm::SelectionDAGTargetInfo::EmitTargetCodeForMemmove(), llvm::SystemZSelectionDAGInfo::EmitTargetCodeForMemmove(), llvm::AArch64SelectionDAGInfo::EmitTargetCodeForMemset(), llvm::ARMSelectionDAGInfo::EmitTargetCodeForMemset(), llvm::RISCVSelectionDAGInfo::EmitTargetCodeForMemset(), llvm::SelectionDAGTargetInfo::EmitTargetCodeForMemset(), llvm::SystemZSelectionDAGInfo::EmitTargetCodeForMemset(), llvm::WebAssemblySelectionDAGInfo::EmitTargetCodeForMemset(), llvm::X86SelectionDAGInfo::EmitTargetCodeForMemset(), llvm::AArch64SelectionDAGInfo::EmitTargetCodeForSetTag(), llvm::SelectionDAGTargetInfo::EmitTargetCodeForSetTag(), llvm::SelectionDAGTargetInfo::EmitTargetCodeForStrcmp(), llvm::SelectionDAGTargetInfo::EmitTargetCodeForStrcpy(), llvm::SelectionDAGTargetInfo::EmitTargetCodeForStrlen(), llvm::SelectionDAGTargetInfo::EmitTargetCodeForStrnlen(), llvm::SelectionDAGTargetInfo::EmitTargetCodeForStrstr(), EmitTest(), emitVectorComparison(), Expand64BitShift(), llvm::TargetLowering::expandABS(), expandBitCastI128ToF128(), llvm::TargetLowering::expandBSWAP(), llvm::TargetLowering::expandCONVERT_FROM_ARBITRARY_FP(), llvm::TargetLowering::expandCONVERT_TO_ARBITRARY_FP(), llvm::TargetLowering::expandCTLZ(), llvm::TargetLowering::expandCTPOP(), llvm::TargetLowering::expandCTTZ(), llvm::TargetLowering::expandFixedPointDiv(), llvm::TargetLowering::expandFixedPointMul(), llvm::TargetLowering::expandFMINNUM_FMAXNUM(), llvm::TargetLowering::expandFP_ROUND(), llvm::TargetLowering::expandFP_TO_INT_SAT(), llvm::TargetLowering::expandFP_TO_SINT(), llvm::TargetLowering::expandFP_TO_UINT(), expandFSH64(), llvm::TargetLowering::expandFunnelShift(), expandIntrinsicWChainHelper(), expandMul(), llvm::TargetLowering::expandMULH(), llvm::TargetLowering::expandMultipleResultFPLibCall(), expandMulToAddOrSubOfShl(), expandMulToShlAddShlAdd(), llvm::TargetLowering::expandROT(), llvm::TargetLowering::expandUADDSUBO(), llvm::TargetLowering::expandUINT_TO_FP(), llvm::TargetLowering::expandVectorNaryOpBySplitting(), llvm::PPCTargetLowering::expandVSXLoadForLE(), llvm::PPCTargetLowering::expandVSXStoreForLE(), ExtendSrcToDst(), extractBooleanFlip(), extractShiftForRotate(), llvm::HexagonDAGToDAGISel::FastFDiv(), llvm::HexagonDAGToDAGISel::FDiv(), FindBFIToCombineWith(), findCCUse(), findMemSDNode(), findMUL_LOHI(), findVSplat(), FixupMMXIntrinsicTypes(), foldADCToCINC(), foldAddSubBoolOfMaskedVal(), foldAddSubMasked1(), foldAddSubOfSignBit(), foldAndOrOfSETCC(), foldAndToUsubsat(), foldBinOpIntoSelectIfProfitable(), foldBinOpIntoSelectIfProfitable(), foldBitOrderCrossLogicOp(), foldBoolSelectToLogic(), FoldBUILD_VECTOR(), foldCONCAT_VECTORS(), foldConcatVector(), llvm::SelectionDAG::FoldConstantArithmetic(), llvm::SelectionDAG::FoldConstantArithmetic(), llvm::SelectionDAG::FoldConstantBuildVector(), llvm::SelectionDAG::foldConstantFPMath(), foldCSELOfCSEL(), foldCSELOfCSELSameFlags(), foldCSELofCTTZ(), foldCSELofLASTB(), foldExtendedSignBitTest(), foldExtendVectorInregToExtendOfSubvector(), foldExtractSubvectorFromShuffleVector(), foldFPToIntToFP(), llvm::AMDGPUTargetLowering::foldFreeOpFromSelect(), FoldIntToFPToInt(), foldLogicOfShifts(), foldLogicTreeOfShifts(), foldMaskedMerge(), foldMaskedShiftToUSHL(), foldOverflowCheck(), foldReduceOperandViaVDOT4A(), foldRemainderIdiom(), foldSelectOfConstantsUsingSra(), foldSelectOfCTTZOrCTLZ(), foldSelectOfSelectCmp(), foldSelectToUSATI(), foldSelectWithIdentityConstant(), llvm::SelectionDAG::FoldSetCC(), foldSetCCWithFunnelShift(), foldSetCCWithRotate(), foldShuffleOfConcatUndefs(), FoldSTEP_VECTOR(), foldSubCtlzNot(), llvm::SelectionDAG::FoldSymbolOffset(), foldToMaskedStore(), foldToSaturated(), foldVectorXorShiftIntoCmp(), foldVectorXorShiftIntoCmp(), foldVMskZeroTest(), foldVSelectToSignBitSplatMask(), foldXor1SetCC(), foldXorTruncShiftIntoCmp(), formSplatFromShuffles(), GenerateFixedLengthSVETBL(), llvm::PPC::get_VSPLTI_elt(), getAddOneOp(), llvm::SelectionDAG::getAddrSpaceCast(), llvm::getAnnotatedNodeAVL(), getAsCarry(), llvm::SelectionDAG::getAssertAlign(), llvm::SelectionDAG::getAtomic(), getAVX2GatherNode(), getAVX512Node(), getAVX512TruncNode(), llvm::SelectionDAG::getBasicBlock(), llvm::getBitwiseNotOperand(), llvm::SelectionDAG::getBlockAddress(), getBMIMatchingOp(), getBROADCAST_LOAD(), getBT(), llvm::TargetLowering::getCheaperOrNeutralNegatedExpression(), getCircRegister(), llvm::SelectionDAG::getCondCode(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getConstantFP(), llvm::SelectionDAG::getConstantPool(), llvm::SelectionDAG::getConstantPool(), llvm::RegsForValue::getCopyFromRegs(), getDataClassTest(), llvm::SelectionDAG::getDeactivationSymbol(), llvm::SelectionDAG::getEntryNode(), getEstimate(), GetExtendHigh(), llvm::SelectionDAG::getExternalSymbol(), getFlagsOfCmpZeroFori1(), llvm::SelectionDAG::getFrameIndex(), getGatherNode(), llvm::getGatherScatterIndex(), llvm::getGatherScatterScale(), llvm::SelectionDAG::getGatherVP(), llvm::SelectionDAG::getGetFPEnv(), llvm::SelectionDAG::getGlobalAddress(), llvm::SelectionDAG::getIdentityElement(), getInputChainForNode(), getIntrinsicCmp(), getInvertedVectorForFMA(), llvm::SelectionDAG::getJumpTable(), getKnownUndefForVectorBinop(), llvm::SelectionDAG::getLabelNode(), getLeftShift(), getLegalMaskAndStepVector(), llvm::SelectionDAG::getLifetimeNode(), llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getLoadFFVP(), getLoadStackGuard(), llvm::getLoadStoreStride(), llvm::SelectionDAG::getLoadVP(), llvm::StatepointLoweringState::getLocation(), getLowSubvectorMaskCompare(), llvm::SelectionDAG::getMachineNode(), llvm::SelectionDAG::getMaskedGather(), llvm::SelectionDAG::getMaskedHistogram(), llvm::SelectionDAG::getMaskedLoad(), llvm::SelectionDAG::getMaskedScatter(), llvm::SelectionDAG::getMaskedStore(), llvm::SelectionDAG::getMCSymbol(), llvm::SelectionDAG::getMDNode(), getMemcpyLoadsAndStores(), llvm::SelectionDAG::getMemIntrinsicNode(), getMemmoveLoadsAndStores(), llvm::getMemoryPtr(), getMemsetStores(), getMemsetStringVal(), llvm::AMDGPUTargetLowering::getNegatedExpression(), llvm::PPCTargetLowering::getNegatedExpression(), llvm::TargetLowering::getNegatedExpression(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNode(), llvm::getNodeAVL(), llvm::getNodeChain(), llvm::getNodeMask(), llvm::getNodePassthru(), getNullFPConstForNullVal(), getPrefetchNode(), llvm::SelectionDAG::getPseudoProbeNode(), llvm::AMDGPUTargetLowering::getRecipEstimate(), llvm::LoongArchTargetLowering::getRecipEstimate(), llvm::TargetLowering::getRecipEstimate(), llvm::SelectionDAG::getRegister(), llvm::SelectionDAG::getRegisterMask(), llvm::BuildVectorSDNode::getRepeatedSequence(), getScalarValueForVectorElement(), getScatterNode(), llvm::SelectionDAG::getScatterVP(), llvm::SelectionDAG::getSetFPEnv(), getShiftedLowMaskZeroTest(), getShuffleScalarElt(), getSingleShuffleSrc(), llvm::SelectionDAG::getSplatSourceVector(), getSplatValue(), llvm::BuildVectorSDNode::getSplatValue(), llvm::SelectionDAG::getSplatValue(), getSplitVectorSrc(), llvm::AMDGPUTargetLowering::getSqrtEstimate(), llvm::LoongArchTargetLowering::getSqrtEstimate(), llvm::NVPTXTargetLowering::getSqrtEstimate(), llvm::TargetLowering::getSqrtEstimate(), llvm::SelectionDAG::getSrcValue(), llvm::SelectionDAG::getStackArgumentTokenFactor(), llvm::SelectionDAG::getStore(), llvm::getStoredValue(), llvm::SelectionDAG::getStoreVP(), llvm::SelectionDAG::getStrictFPExtendOrRound(), llvm::SelectionDAG::getStridedLoadVP(), llvm::SelectionDAG::getStridedStoreVP(), getSubVectorSrc(), llvm::SelectionDAG::getTargetExternalSymbol(), llvm::SelectionDAG::getTargetExtractSubreg(), llvm::SelectionDAG::getTargetInsertSubreg(), GetTLSADDR(), getTruncatedUSUBSAT(), llvm::SelectionDAG::getTruncStoreVP(), getUsefulBitsForUse(), llvm::SelectionDAGBuilder::getValueImpl(), llvm::SelectionDAG::getValueType(), getVectorBitwiseReduce(), llvm::SelectionDAG::getVectorShuffle(), getVLOperand(), HandleMergeInputChains(), if(), if(), incDecVectorConstant(), INITIALIZE_PASS(), InvertCarryFlag(), isAddSubOrSubAdd(), isCFMulFromFMAddSub(), IsCMPZCSINC(), isFNEG(), llvm::TargetLowering::isGAPlusOffset(), isHorizontalBinOp(), llvm::GCNTTIImpl::isInlineAsmSourceOfDivergence(), IsNOT(), isNOT(), isNVCastToHalfWidthElements(), isOnlyUsedByStores(), IsOperandAMemoryOperand(), isSaturatingMinMax(), isScalarToVec(), IsSingleInstrConstant(), isSlideWithZerosMask(), isTargetShuffleEquivalent(), isUpperSubvectorUndef(), isVMOVModifiedImm(), isWorthFoldingIntoOrrWithShift(), isWorthFoldingIntoRegRegScale(), llvm::RISCVTargetLowering::joinRegisterPartsIntoValue(), llvm::SystemZTargetLowering::joinRegisterPartsIntoValue(), llvm::TargetLowering::joinRegisterPartsIntoValue(), llvm::TargetLowering::LegalizeSetCCCondCode(), legalizeSVEGatherPrefetchOffsVec(), llvm::SITargetLowering::legalizeTargetIndependentNode(), lower128BitShuffle(), lower1BitShuffle(), lower1BitShuffleAsKSHIFTR(), lower256BitShuffle(), LowerABD(), LowerABS(), LowerADDSAT_SUBSAT(), lowerADDSUBO_CARRY(), LowerADDSUBSAT(), LowerAndToBT(), LowerAndToBTST(), llvm::SystemZTargetLowering::LowerAsmOutputForConstraint(), llvm::TargetLowering::LowerAsmOutputForConstraint(), llvm::X86TargetLowering::LowerAsmOutputForConstraint(), LowerAsSplatVectorLoad(), llvm::SelectionDAGBuilder::LowerAsSTATEPOINT(), llvm::VETargetLowering::lowerATOMIC_FENCE(), LowerATOMIC_LOAD_STORE(), LowerAtomicLoadStore(), LowerAVG(), lowerBALLOTIntrinsic(), llvm::HexagonTargetLowering::LowerBITCAST(), LowerBITCAST(), lowerBitreverseShuffle(), LowerBRCOND(), llvm::HexagonTargetLowering::LowerBUILD_VECTOR(), llvm::VETargetLowering::lowerBUILD_VECTOR(), lowerBUILD_VECTOR(), LowerBUILD_VECTOR_i1(), lowerBUILD_VECTORAsBroadCastLoad(), LowerBUILD_VECTORAsVariablePermute(), LowerBUILD_VECTORToVIDUP(), lowerBuildVectorAsBlend(), lowerBuildVectorAsBroadcast(), lowerBuildVectorOfConstants(), LowerBuildVectorOfFPExt(), LowerBuildVectorOfFPTrunc(), lowerBuildVectorToBitOp(), LowerBuildVectorv16i8(), LowerBuildVectorv4x32(), LowerBuildVectorv8i16(), lowerBuildVectorViaDominantValues(), lowerBuildVectorViaPacking(), lowerBuildVectorViaVID(), llvm::HexagonTargetLowering::LowerCall(), llvm::NVPTXTargetLowering::LowerCall(), llvm::SITargetLowering::LowerCall(), llvm::VETargetLowering::LowerCall(), LowerCallResult(), lowerCallResult(), llvm::TargetLowering::LowerCallTo(), LowerCLMUL(), LowerClusterLaunchControlQueryCancel(), llvm::TargetLowering::lowerCmpEqZeroToCtlzSrl(), llvm::HexagonTargetLowering::LowerCONCAT_VECTORS(), lowerConstant(), lowerCONVERT_FROM_ARBITRARY_FP(), LowerConvertLow(), LowerCTPOP(), LowerCTTZ(), LowerCTTZ(), lowerDisjointIndicesShuffle(), llvm::AMDGPUTargetLowering::LowerDIVREMToFloat(), lowerDSPIntr(), llvm::NVPTXTargetLowering::LowerDYNAMIC_STACKALLOC(), llvm::SITargetLowering::LowerDYNAMIC_STACKALLOC(), llvm::TargetLowering::lowerEHPadEntry(), LowerEXTEND_VECTOR_INREG(), LowerEXTRACT_VECTOR_ELT(), llvm::VETargetLowering::lowerEXTRACT_VECTOR_ELT(), LowerEXTRACT_VECTOR_ELT_SSE4(), LowerF128_FPEXTEND(), LowerF128_FPROUND(), LowerF128Load(), llvm::SparcTargetLowering::LowerF128Op(), LowerF64Op(), llvm::HexagonTargetLowering::LowerFDIV(), llvm::AMDGPUTargetLowering::lowerFEXP(), LowerFLDEXP(), LowerFLDEXP(), LowerFNEGorFABS(), llvm::VETargetLowering::LowerFormalArguments(), llvm::AMDGPUTargetLowering::LowerFP_TO_FP16(), llvm::AMDGPUTargetLowering::LowerFP_TO_INT(), llvm::AMDGPUTargetLowering::LowerFP_TO_INT_SAT(), LowerFP_TO_INT_SAT(), lowerFP_TO_INT_SAT(), LowerFP_TO_SINT(), lowerFP_TO_SINT_STORE(), LowerFP_TO_UINT(), lowerFPToIntToFP(), lowerFTRUNC_FCEIL_FFLOOR_FROUND(), LowerFunnelShift(), LowerFunnelShift(), llvm::AMDGPUTargetLowering::LowerGlobalAddress(), LowerI64IntToFP16(), LowerI64IntToFP_AVX512DQ(), llvm::VETargetLowering::lowerINSERT_VECTOR_ELT(), lowerINT_TO_FP_vXi64(), lowerIntNeonIntrinsic(), llvm::HexagonTargetLowering::LowerINTRINSIC_VOID(), LowerINTRINSIC_W_CHAIN(), llvm::HexagonTargetLowering::LowerINTRINSIC_WO_CHAIN(), llvm::SparcTargetLowering::LowerINTRINSIC_WO_CHAIN(), llvm::VETargetLowering::lowerINTRINSIC_WO_CHAIN(), lowerIntrinsicWChain(), lowerLaneOp(), llvm::HexagonTargetLowering::LowerLoad(), LowerLoad(), llvm::MipsTargetLowering::lowerLOAD(), lowerLoadF128(), lowerLoadI1(), lowerLoadVector(), LowerMGATHER(), LowerMINMAX(), LowerMINMAX_REDUCE(), LowerMSCATTER(), llvm::LanaiTargetLowering::LowerMUL(), LowerMUL(), LowerNTStore(), llvm::AArch64TargetLowering::LowerOperation(), llvm::ARMTargetLowering::LowerOperation(), llvm::AVRTargetLowering::LowerOperation(), llvm::HexagonTargetLowering::LowerOperation(), llvm::LoongArchTargetLowering::LowerOperation(), llvm::MipsTargetLowering::LowerOperation(), llvm::NVPTXTargetLowering::LowerOperation(), llvm::PPCTargetLowering::LowerOperation(), llvm::R600TargetLowering::LowerOperation(), llvm::RISCVTargetLowering::LowerOperation(), llvm::SITargetLowering::LowerOperation(), llvm::SystemZTargetLowering::LowerOperationWrapper(), llvm::TargetLowering::LowerOperationWrapper(), LowerPARITY(), LowerPDEPPEXT(), llvm::SITargetLowering::lowerPREFETCH(), LowerPtrAuthGlobalAddressStatically(), llvm::VETargetLowering::LowerReturn(), llvm::SparcTargetLowering::LowerReturn_32(), LowerRotate(), llvm::HexagonTargetLowering::LowerROTL(), LowerSaturatingConditional(), lowerSelectToBinOp(), LowerSELECTWithCmpZero(), llvm::HexagonTargetLowering::LowerSETCC(), LowerShift(), LowerShift(), LowerShiftByScalarImmediate(), LowerShiftByScalarVariable(), lowerShuffleAsBitBlend(), lowerShuffleAsBitMask(), lowerShuffleAsBitRotate(), lowerShuffleAsBlend(), lowerShuffleAsBlendAndPermute(), lowerShuffleAsBroadcast(), lowerShuffleAsByteRotate(), lowerShuffleAsByteRotateAndPermute(), lowerShuffleAsByteShiftMask(), lowerShuffleAsElementInsertion(), lowerShuffleAsInsertPS(), lowerShuffleAsLanePermuteAndPermute(), lowerShuffleAsLanePermuteAndRepeatedMask(), lowerShuffleAsPermuteAndUnpack(), lowerShuffleAsRepeatedMaskAndLanePermute(), lowerShuffleAsShift(), lowerShuffleAsSpecificExtension(), lowerShuffleAsTruncBroadcast(), lowerShuffleAsUNPCKAndPermute(), lowerShuffleAsVALIGN(), lowerShuffleAsVTRUNC(), lowerShuffleAsVTRUNCAndUnpack(), lowerShuffleAsZeroOrAnyExtend(), lowerShuffleOfExtractsAsVperm(), lowerShufflePairAsUNPCKAndPermute(), lowerShuffleViaVRegSplitting(), lowerShuffleWithEXPAND(), lowerShuffleWithPACK(), lowerShuffleWithPSHUFB(), lowerShuffleWithSHUFPD(), lowerShuffleWithSSE4A(), lowerShuffleWithUndefHalf(), lowerShuffleWithUNPCK(), lowerShuffleWithUNPCK256(), lowerShuffleWithVPMOV(), LowerSINT_TO_FP(), LowerSMELdrStr(), llvm::NVPTXTargetLowering::LowerSTACKSAVE(), LowerSTORE(), llvm::HexagonTargetLowering::LowerStore(), LowerStore(), llvm::VETargetLowering::lowerSTORE(), lowerStoreI1(), lowerSTOREVector(), LowerSVEIntrinsicEXT(), llvm::VETargetLowering::lowerToVVP(), LowerTruncate(), LowerTruncateToBTST(), LowerTruncateVecPack(), LowerTruncateVecPackWithSignBits(), llvm::HexagonTargetLowering::LowerUAddSubO(), LowerUINT_TO_FP(), lowerUINT_TO_FP_v2i32(), lowerUINT_TO_FP_vXi32(), llvm::AMDGPUTargetLowering::lowerUnhandledCall(), lowerV16I8Shuffle(), lowerV2X128Shuffle(), lowerV4X128Shuffle(), LowerVECREDUCE(), LowerVecReduce(), LowerVecReduceF(), LowerVecReduceMinMax(), lowerVECTOR_COMPRESS(), llvm::HexagonTargetLowering::LowerVECTOR_SHIFT(), llvm::HexagonTargetLowering::LowerVECTOR_SHUFFLE(), LowerVECTOR_SHUFFLE(), lowerVECTOR_SHUFFLE_ILVEV(), lowerVECTOR_SHUFFLE_ILVL(), lowerVECTOR_SHUFFLE_ILVOD(), lowerVECTOR_SHUFFLE_ILVR(), lowerVECTOR_SHUFFLE_IsReverse(), lowerVECTOR_SHUFFLE_PCKEV(), lowerVECTOR_SHUFFLE_PCKOD(), lowerVECTOR_SHUFFLE_SHF(), lowerVECTOR_SHUFFLE_VEXTRINS(), lowerVECTOR_SHUFFLE_VILVH(), lowerVECTOR_SHUFFLE_VILVL(), lowerVECTOR_SHUFFLE_VPACKEV(), lowerVECTOR_SHUFFLE_VPACKOD(), lowerVECTOR_SHUFFLE_VPERMI(), lowerVECTOR_SHUFFLE_VPICKEV(), lowerVECTOR_SHUFFLE_VPICKOD(), lowerVECTOR_SHUFFLE_VREPLVEI(), lowerVECTOR_SHUFFLE_VSHUF4I(), lowerVECTOR_SHUFFLE_XVEXTRINS(), lowerVECTOR_SHUFFLE_XVILVH(), lowerVECTOR_SHUFFLE_XVILVL(), lowerVECTOR_SHUFFLE_XVINSVE0(), lowerVECTOR_SHUFFLE_XVPERM(), lowerVECTOR_SHUFFLE_XVPERMI(), lowerVECTOR_SHUFFLE_XVPERMIQ(), lowerVECTOR_SHUFFLE_XVPICKEV(), lowerVECTOR_SHUFFLE_XVPICKOD(), lowerVECTOR_SHUFFLE_XVREPLVE0(), lowerVECTOR_SHUFFLE_XVREPLVEI(), lowerVECTOR_SHUFFLE_XVSHUF(), lowerVECTOR_SHUFFLE_XVSHUF4I(), lowerVECTOR_SHUFFLEAsByteRotate(), lowerVECTOR_SHUFFLEAsLanePermuteAndShuffle(), lowerVECTOR_SHUFFLEAsPPair(), lowerVECTOR_SHUFFLEAsPSlide1(), lowerVECTOR_SHUFFLEAsPUnzip(), lowerVECTOR_SHUFFLEAsPZip(), lowerVECTOR_SHUFFLEAsRotate(), lowerVECTOR_SHUFFLEAsRV32PNarrowingShift(), lowerVECTOR_SHUFFLEAsShift(), lowerVECTOR_SHUFFLEAsVRGatherVX(), lowerVECTOR_SHUFFLEAsVSlide1(), lowerVECTOR_SHUFFLEAsVSlidedown(), lowerVECTOR_SHUFFLEAsVSlideup(), lowerVECTOR_SHUFFLEAsZeroOrAnyExtend(), LowerVECTOR_SHUFFLEUsingMovs(), LowerVECTOR_SHUFFLEUsingOneOff(), LowerVectorAllEqual(), LowerVectorCTPOP(), LowerVectorExtend(), lowerVectorIntrinsicScalars(), LowerVectorMatch(), llvm::HexagonTargetLowering::LowerVSELECT(), lowerVSELECTtoVectorShuffle(), LowerVSETCC(), LowerVSETCC(), LowerVSETCCWithSUBUS(), llvm::VETargetLowering::lowerVVP_GATHER_SCATTER(), llvm::VETargetLowering::lowerVVP_LOAD_STORE(), lowerWaveShuffle(), LowerXALUO(), llvm::SelectionDAG::makeEquivalentMemoryOrdering(), matchBF16FPExtendLike(), matchBinaryPermuteShuffle(), llvm::SelectionDAG::matchBinOpReduction(), matchBinOpWithSharedOperand(), matchBSwapHWordOrAndAnd(), matchCSET(), matchDeinterleaveBuildVector(), matchExtFromI32orI32(), matchHalfOf128BitLanes(), llvm::AMDGPUDAGToDAGISel::matchLoadD16FromBuildVector(), matchLSNode(), matchMADConstOnePattern(), matchMergedBFX(), matchPERM(), matchPMADDWD(), matchPMADDWD_2(), matchSplatAsGather(), matchTruncateWithPACK(), MatchVectorAllEqualTest(), matchVPMADD52(), matchZeroSelectArm(), materializeVectorConstant(), mergeI16Unpacks(), MergeInputChains(), llvm::SelectionDAG::MorphNodeTo(), moveBelowOrigChain(), llvm::SelectionDAG::mutateStrictFPToFP(), narrowBitOpRMW(), narrowExtractedVectorBinOp(), narrowExtractedVectorLoad(), narrowExtractedVectorSelect(), narrowInsertExtractVectorBinOp(), narrowLoadToVZLoad(), narrowShuffle(), narrowVectorSelect(), llvm::DAGTypeLegalizer::NoteDeletion(), optimizeBitTest(), optimizeBrk(), optimizeIncrementingWhile(), optimizeLogicalImm(), partitionShuffleOfConcats(), perfomPextFirstTrueVectorCombine(), performActiveLaneMaskCombine(), performAddAddCombine(), PerformADDCombine(), performADDCombine(), performADDCombine(), performAddCombineForShiftedOperands(), performAddCombineSubShift(), PerformADDCombineWithOperands(), PerformADDCombineWithOperands(), performAddCSelIntoCSinc(), PerformAddcSubcCombine(), performAddDotCombine(), PerformADDECombine(), PerformAddeSubeCombine(), performAddMulZextCombine(), performAddSubCarryCombine(), performAddSubIntoVectorOp(), performAddSubLongCombine(), performAddTruncShiftCombine(), performAddUADDVCombine(), PerformADDVecReduce(), performAddWithSBCCombine(), PerformANDCombine(), performANDCombine(), performANDCombine(), performANDCombine(), performANDCombine(), performANDORCSELCombine(), performANDORDUPNOTCombine(), performANDSCombine(), performANDSETCCCombine(), performAnyAllCombine(), PerformARMBUILD_VECTORCombine(), llvm::AMDGPUTargetLowering::performAssertSZExtCombine(), PerformBFICombine(), performBICiCombine(), llvm::SparcTargetLowering::PerformBITCASTCombine(), PerformBITCASTCombine(), performBITCASTCombine(), performBitcastCombine(), performBitmaskCombine(), performBITREV_WCombine(), performBITREVERSECombine(), performBR_CCCombine(), llvm::ARMTargetLowering::PerformBRCONDCombine(), performBRCONDCombine(), performBSPExpandForSVE(), llvm::SparcTargetLowering::PerformBSWAPCombine(), PerformBUILD_VECTORCombine(), PerformBUILD_VECTORCombine(), performBUILD_VECTORCombine(), performBuildShuffleExtendCombine(), performBuildVectorCombine(), llvm::ARMTargetLowering::PerformCMOVCombine(), performCMovFPCombine(), llvm::ARMTargetLowering::PerformCMOVToBFICombine(), PerformCMPZCombine(), performCONCAT_VECTORSCombine(), performCONCAT_VECTORSCombine(), performConcatVectorsCombine(), performCONDCombine(), performConditionalSubCombine(), performConvertFPCombine(), PerformCSETCombine(), llvm::AMDGPUTargetLowering::performCtlz_CttzCombine(), performCTLZCombine(), performCTPOPCombine(), llvm::AArch64TargetLowering::PerformDAGCombine(), llvm::AMDGPUTargetLowering::PerformDAGCombine(), llvm::ARMTargetLowering::PerformDAGCombine(), llvm::HexagonTargetLowering::PerformDAGCombine(), llvm::LanaiTargetLowering::PerformDAGCombine(), llvm::LoongArchTargetLowering::PerformDAGCombine(), llvm::MipsTargetLowering::PerformDAGCombine(), llvm::PPCTargetLowering::PerformDAGCombine(), llvm::R600TargetLowering::PerformDAGCombine(), llvm::RISCVTargetLowering::PerformDAGCombine(), llvm::SITargetLowering::PerformDAGCombine(), llvm::SparcTargetLowering::PerformDAGCombine(), llvm::SystemZTargetLowering::PerformDAGCombine(), llvm::TargetLowering::PerformDAGCombine(), llvm::VETargetLowering::PerformDAGCombine(), llvm::X86TargetLowering::PerformDAGCombine(), performDemandedBitsCombine(), performDivRemCombine(), performDSPShiftCombine(), performDUPCombine(), performDupLane128Combine(), performExtBinopLoadFold(), PerformExtendCombine(), performEXTENDCombine(), performExtendCombine(), performExtendDuplaneTruncCombine(), performExtendToBoolVectorLoadCombine(), PerformEXTRACTCombine(), PerformExtractEltCombine(), PerformExtractEltToVMOVRRD(), PerformExtractFpToIntStores(), performExtractLastActiveCombine(), performExtractSubvectorCombine(), performExtractVectorEltCombine(), llvm::AMDGPUTargetLowering::performFAbsCombine(), PerformFADDCombine(), performFADDCombine(), performFADDPREDCombine(), PerformFADDVCMLACombine(), PerformFAddVSelectCombine(), performFirstTrueTestVectorCombine(), performFlagSettingCombine(), PerformFMinMaxCombine(), llvm::AMDGPUTargetLowering::performFNegCombine(), performFP_TO_INT_SATCombine(), performFP_TO_INTCombine(), performFP_TO_INTCombine(), performFP_TO_UINTCombine(), PerformFPExtendCombine(), performFPExtendCombine(), performFpToIntCombine(), performGatherLoadCombine(), performGLD1Combine(), performGlobalAddressCombine(), performHorizWideningCombine(), PerformHWLoopCombine(), performINSERT_VECTOR_ELTCombine(), PerformInsertEltCombine(), PerformInsertSubvectorCombine(), performInsertSubvectorCombine(), PerformInterleavedStoreCombine(), performINTRINSIC_WO_CHAINCombine(), llvm::ARMTargetLowering::PerformIntrinsicCombine(), performIntrinsicCombine(), llvm::AMDGPUTargetLowering::performIntrinsicWOChainCombine(), performIntToFpCombine(), performLastTrueTestVectorCombine(), performLD1Combine(), performLD1ReplicateCombine(), PerformLegalizedInterleavedStoreCombine(), performLegalizedInterleavedStoreCombine(), PerformLegalizedVECTOR_DEINTERLEAVECombine(), performLegalizedVectorDeinterleaveCombine(), PerformLOADCombine(), llvm::AMDGPUTargetLowering::performLoadCombine(), PerformLongShiftCombine(), performMADD_MSUBCombine(), performMaskedGatherScatterCombine(), performMaskedLoadToVPLoadCombine(), performMemPairCombine(), PerformMinMaxCombine(), performMINMAXCombine(), performMinMaxF128Combine(), PerformMinMaxFpToSatCombine(), PerformMinMaxToSatCombine(), performMOVFR2GR_SCombine(), performMOVGR2FR_WCombine(), performMSTORECombine(), PerformMULCombine(), PerformMULCombine(), performMULCombine(), performMULCombine(), llvm::AMDGPUTargetLowering::performMulCombine(), performMulCombine(), performMulCombine(), PerformMULCombineWithOperands(), llvm::AMDGPUTargetLowering::performMulhsCombine(), llvm::AMDGPUTargetLowering::performMulhuCombine(), performMULLCombine(), llvm::AMDGPUTargetLowering::performMulLoHiCombine(), performMulRdsvlCombine(), performMulVectorCmpZeroCombine(), performMulVectorExtendCombine(), llvm::ARMTargetLowering::PerformMVEExtCombine(), llvm::ARMTargetLowering::PerformMVETruncCombine(), PerformMVEVLDCombine(), PerformMVEVMULLCombine(), performNanGuardFpToSatCombine(), performNegCMovCombine(), performNegCSelCombine(), performNegSignMaskedAndCombine(), performNEONPostLDSTCombine(), performNVCASTCombine(), PerformORCombine(), performORCombine(), performORCombine(), performORCombine(), performORCombine(), PerformORCombine_i1(), PerformORCombineToBFI(), PerformORCombineToShiftInsert(), PerformORCombineToSMULWBT(), performOrXorChainCombine(), performP_BITCASTCombine(), performPostLD1Combine(), PerformPREDICATE_CASTCombine(), performPredicateLoadCombine(), performPTestFirstCombine(), llvm::AMDGPUTargetLowering::performRcpCombine(), PerformReduceShuffleCombine(), performReinterpretCastCombine(), performReverseEVLCombine(), performScalarToVectorCombine(), performScatterStoreCombine(), performSELECT_CCCombine(), performSELECT_CCCombine(), PerformSELECTCombine(), performSELECTCombine(), performSELECTCombine(), performSELECTCombine(), llvm::AMDGPUTargetLowering::performSelectCombine(), performSelectCombine(), PerformSELECTShiftCombine(), performSETCC_BITCASTCombine(), performSetccAddFolding(), performSETCCCARRYCombine(), PerformSETCCCombine(), performSETCCCombine(), performSETCCCombine(), performSETCCCombine(), performSETCCCombine(), performSETCCCombine(), performSetccMergeZeroCombine(), performSetCCPunpkCombine(), PerformShiftCombine(), performShiftCombine(), PerformSHLCombine(), performSHLCombine(), performSHLCombine(), performSHLCombine(), performSHLCombine(), performSHLCombine(), llvm::AMDGPUTargetLowering::performShlCombine(), PerformSHLSimplify(), PerformShuffleVMOVNCombine(), performSIGN_EXTEND_INREGCombine(), performSignExtendCombine(), PerformSignExtendInregCombine(), performSignExtendInRegCombine(), performSignExtendSetCCCombine(), performSINT_TO_FPCombine(), performSpliceCombine(), performSPLIT_PAIR_F64Combine(), PerformSplittingMVEEXTToWideningLoad(), PerformSplittingMVETruncToNarrowingStores(), PerformSplittingToNarrowingStores(), PerformSplittingToWideningLoad(), performSRACombine(), performSRACombine(), llvm::AMDGPUTargetLowering::performSraCombine(), performSRLCombine(), performSRLCombine(), performSRLCombine(), llvm::AMDGPUTargetLowering::performSrlCombine(), llvm::SparcTargetLowering::PerformSTORECombine(), PerformSTORECombine(), llvm::AMDGPUTargetLowering::performStoreCombine(), performSubAddMULCombine(), PerformSUBCombine(), PerformSUBCombine(), performSUBCombine(), performSUBCombine(), PerformSubCSINCCombine(), performSubNegAndOneCombine(), performSubsToAndsCombine(), performSubWithBorrowCombine(), performSunpkloCombine(), performSVEAndCombine(), performSVEMulAddSubCombine(), performTBZCombine(), performTRUNCATECombine(), llvm::AMDGPUTargetLowering::performTruncateCombine(), performTruncateCombine(), performTruncateCombine(), PerformTruncatingStoreCombine(), performUADDVAddCombine(), performUADDVCombine(), performUADDVZextCombine(), performUINT_TO_FPCombine(), PerformUMinFpToSatCombine(), PerformUMLALCombine(), performUnpackCombine(), performUzpCombine(), performVANDNCombine(), PerformVCMPCombine(), PerformVCVTCombine(), PerformVDUPCombine(), PerformVDUPLANECombine(), PerformVECREDUCE_ADDCombine(), performVecReduceAddCntpCombine(), performVecReduceAddCombine(), performVecReduceAddCombineWithUADDLP(), performVecReduceBitwiseCombine(), performVECREDUCECombine(), PerformVECTOR_DEINTERLEAVECombine(), performVECTOR_INTERLEAVECombine(), PerformVECTOR_REG_CASTCombine(), PerformVECTOR_SHUFFLECombine(), performVECTOR_SHUFFLECombine(), performVECTOR_SHUFFLECombine(), performVectorCompareAndMaskUnaryOpCombine(), performVectorDeinterleaveCombine(), performVectorExtCombine(), performVectorExtendCombine(), performVectorExtendToFPCombine(), performVectorNonNegToFPCombine(), performVectorShiftCombine(), performVectorTruncZeroCombine(), performVEXT_VLCombine(), performVFMADD_VLCombine(), PerformVLDCombine(), PerformVMOVDRRCombine(), PerformVMOVhrCombine(), PerformVMOVNCombine(), PerformVMOVrhCombine(), PerformVMOVRRDCombine(), PerformVMULCombine(), PerformVMulVCTPCombine(), performVP_STORECombine(), PerformVQDMULHCombine(), PerformVQDMULHCombine(), PerformVQMOVNCombine(), PerformVSELECTCombine(), PerformVSELECTCombine(), performVSELECTCombine(), performVSELECTCombine(), performVSELECTCombine(), performVSelectCombine(), performVSelectMaskedShiftCombine(), performVselectPowCombine(), PerformVSetCCToVCTPCombine(), performVSlideUpDownCombine(), performVWABDACombine(), performVWABDACombineWV(), PerformXORCombine(), performXORCombine(), performXorCombine(), performZExtDeinterleaveShuffleCombine(), performZExtUZPCombine(), llvm::SITargetLowering::PostISelFolding(), PrepareTailCall(), llvm::RISCVDAGToDAGISel::PreprocessISelDAG(), promoteExtBeforeAdd(), PromoteMaskArithmetic(), PromoteMaskArithmetic(), PromoteMaskArithmetic(), PromoteMaskArithmetic(), promoteToConstantPool(), propagateUnaryUndef(), pushAddIntoCmovOfConsts(), reassociateAddSubLong(), reassociateCSELOperandsForCSE(), llvm::AArch64TargetLowering::ReconstructShuffle(), ReconstructShuffleWithRuntimeMask(), ReconstructTruncateFromBuildVector(), reduceANDOfAtomicLoad(), reduceBuildVecToShuffleWithZero(), reduceMaskedLoadToScalarLoad(), reduceMaskedStoreToScalarStore(), reduceVMULWidth(), reduceVSXSwap(), llvm::SelectionDAG::RemoveDeadNodes(), removeRedundantInsertVectorElt(), llvm::SelectionDAG::ReplaceAllUsesWith(), llvm::SelectionDAG::ReplaceAllUsesWith(), ReplaceATOMIC_LOAD_128Results(), ReplaceCMP_SWAP_128Results(), ReplaceCMP_SWAP_64Results(), replaceCMP_XCHG_128Results(), replaceInChain(), llvm::AMDGPUTargetLowering::ReplaceNodeResults(), llvm::ARMTargetLowering::ReplaceNodeResults(), llvm::AVRTargetLowering::ReplaceNodeResults(), llvm::LoongArchTargetLowering::ReplaceNodeResults(), llvm::PPCTargetLowering::ReplaceNodeResults(), llvm::R600TargetLowering::ReplaceNodeResults(), llvm::RISCVTargetLowering::ReplaceNodeResults(), llvm::SITargetLowering::ReplaceNodeResults(), llvm::SparcTargetLowering::ReplaceNodeResults(), llvm::X86TargetLowering::ReplaceNodeResults(), replaceSplatVectorStore(), replaceZeroVectorStore(), llvm::SITargetLowering::requiresUniformRegister(), reverseZExtICmpCombine(), llvm::DAGTypeLegalizer::run(), scalarizeBinOpOfSplats(), scalarizeExtEltFP(), scalarizeExtractedBinOp(), llvm::TargetLowering::scalarizeExtractedVectorLoad(), scalarizeVectorStore(), llvm::SDPatternMatch::sd_match(), SearchLoopIntrinsic(), llvm::RISCVDAGToDAGISel::Select(), llvm::PPCTargetLowering::SelectAddressRegImm(), llvm::RISCVDAGToDAGISel::SelectAddrRegImm(), llvm::RISCVDAGToDAGISel::SelectAddrRegImm26(), llvm::RISCVDAGToDAGISel::SelectAddrRegImmLsb00000(), llvm::RISCVDAGToDAGISel::SelectAddrRegRegScale(), llvm::HexagonDAGToDAGISel::SelectBrevLdIntrinsic(), llvm::AMDGPUDAGToDAGISel::SelectBuildVector(), llvm::SelectionDAGISel::SelectCodeCommon(), selectConstantAddr(), llvm::HexagonDAGToDAGISel::SelectHVXDualOutput(), selectI64Imm(), selectI64ImmDirect(), selectI64ImmDirectPrefix(), selectImm(), selectImmSeq(), llvm::HexagonDAGToDAGISel::SelectIndexedLoad(), llvm::HexagonDAGToDAGISel::SelectIndexedStore(), llvm::HexagonDAGToDAGISel::SelectNewCircIntrinsic(), llvm::PPCTargetLowering::SelectOptimalAddrMode(), SelectSAddrFI(), llvm::RISCVDAGToDAGISel::selectSETCC(), llvm::LoongArchDAGToDAGISel::selectShiftMask(), llvm::RISCVDAGToDAGISel::selectShiftMask(), llvm::RISCVDAGToDAGISel::selectSHXADD_UWOp(), llvm::RISCVDAGToDAGISel::selectSHXADDOp(), llvm::HexagonDAGToDAGISel::SelectVAlign(), llvm::AMDGPUDAGToDAGISel::SelectVectorShuffle(), llvm::RISCVDAGToDAGISel::selectVLSEG(), llvm::RISCVDAGToDAGISel::selectVLSEGFF(), llvm::RISCVDAGToDAGISel::selectVLXSEG(), llvm::FunctionLoweringInfo::set(), llvm::X86TargetLowering::shouldReduceLoadWidth(), ShrinkLoadReplaceStoreWithStore(), simplifyDemandedBitsForPRMT(), llvm::X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode(), simplifyDivRem(), llvm::SelectionDAG::simplifyFPBinop(), simplifyMul24(), llvm::TargetLowering::SimplifyMultipleUseDemandedBits(), llvm::TargetLowering::SimplifyMultipleUseDemandedBitsForTargetNode(), simplifyOp_VL(), llvm::SelectionDAG::simplifySelect(), llvm::TargetLowering::SimplifySetCC(), simplifySetCCWithCTPOP(), llvm::SelectionDAG::simplifyShift(), simplifyShuffleOfShuffle(), sinkProxyReg(), SkipExtensionForVMULL(), llvm::TargetLowering::softenSetCCOperands(), splitAndLowerShuffle(), llvm::VETargetLowering::splitPackedLoadStore(), splitStores(), splitVectorOp(), splitVectorStore(), llvm::HexagonDAGToDAGISel::StoreInstrForLoadIntrinsic(), strictFPExtFromF16(), stripModuloOnShift(), stripTruncAndExt(), takeInexpensiveLog2(), transformAddImmMulImm(), transformAddShlImm(), transformCallee(), llvm::HexagonDAGToDAGISel::translateIEEEIntrinsicToQFloat(), truncateAVX512SetCCNoBWI(), truncateVectorWithNARROW(), truncateVectorWithPACK(), tryAdvSIMDModImm16(), tryAdvSIMDModImm32(), tryAdvSIMDModImm321s(), tryAdvSIMDModImm64(), tryAdvSIMDModImm8(), tryAdvSIMDModImmFP(), tryBitfieldInsertOpFromOr(), tryBitfieldInsertOpFromOrAndImm(), tryBuildVectorShuffle(), TryCombineBaseUpdate(), tryCombineCRC32(), tryCombineExtendRShTrunc(), tryCombineFADDReductionWithZero(), tryCombineFixedPointConvert(), tryCombineLongOpWithDup(), tryCombineMULLWithUZP1(), tryCombineNeonFcvtFP16ToI16(), tryCombineShiftImm(), tryCombineToREV(), tryConvertSVEWideCompare(), tryDemorganOfBooleanCondition(), TryDistrubutionADDVecReduce(), tryExtendDUPToExtractHigh(), tryFoldMADwithSRL(), tryFoldSelectIntoOp(), tryFoldToZero(), tryFormConcatFromShuffle(), llvm::HexagonDAGToDAGISel::tryLoadOfLoadIntrinsic(), tryLowerMultiVectorStore(), tryLowerSmallVectorExtLoad(), tryLowerToBSL(), tryLowerToSLI(), tryMemPairCombine(), TryMULWIDECombine(), llvm::RISCVDAGToDAGISel::tryShrinkShlLogicImm(), trySimplifySrlAddToRshrnb(), trySQABSCombine(), trySQDMULHCombine(), trySVESplat64(), trySwapVSelectOperands(), tryToConvertShuffleOfTbl2ToTbl4(), tryToFoldExtendOfConstant(), tryToFoldExtendSelectLoad(), tryToFoldExtOfAtomicLoad(), tryToFoldExtOfExtload(), tryToFoldExtOfLoad(), tryToFoldExtOfMaskedLoad(), tryToReplaceScalarFPConversionWithSVE(), tryToWidenSetCCOperands(), TryWideExtMulCombine(), llvm::RISCVDAGToDAGISel::tryWideningMulAcc(), tryWidenMaskForShuffle(), tryWidenMaskForShuffle(), llvm::X86InstrInfo::unfoldMemoryOperand(), useInversedSetcc(), vectorizeExtractedCast(), visitORCommutative(), Widen(), widenAbs(), widenBuildVec(), widenBuildVector(), widenCtPop(), widenShuffleMask(), widenVectorToPartType(), and llvm::SITargetLowering::wrapAddr64Rsrc().

selectUmullSmull()

unsigned selectUmullSmull ( SDValue & N0, SDValue & N1, SelectionDAG & DAG, SDLoc DL, bool & IsMLA )

static

Definition at line 6072 of file AArch64ISelLowering.cpp.

References llvm::SelectionDAG::ComputeNumSignBits(), DL, llvm::APInt::getHighBitsSet(), llvm::EVT::getScalarSizeInBits(), llvm::SDValue::getValueType(), isAddSubSExt(), isAddSubZExt(), isSignExtended(), isZeroExtended(), llvm::SelectionDAG::MaskedValueIsZero(), and std::swap().

setInfoSVEStN()

template<unsigned NumVecs>

void setInfoSVEStN ( const AArch64TargetLowering & TLI, const DataLayout & DL, AArch64TargetLowering::IntrinsicInfo & Info, const CallBase & CI )

static

Set the IntrinsicInfo for the aarch64_sve_st<N> intrinsics.

Definition at line 18620 of file AArch64ISelLowering.cpp.

References llvm::CallBase::arg_size(), assert(), DL, llvm::dyn_cast(), llvm::CallBase::getArgOperand(), llvm::Type::getContext(), llvm::EVT::getScalarType(), llvm::Value::getType(), llvm::EVT::getVectorElementCount(), llvm::EVT::getVectorVT(), I, llvm::ISD::INTRINSIC_VOID, llvm::MachineMemOperand::MOStore, Pred, TLI, and llvm::MemoryLocation::UnknownSize.

Referenced by llvm::AArch64TargetLowering::getTgtMemIntrinsic().

shouldBeAdjustedToZero()

bool shouldBeAdjustedToZero ( SDValue LHS, const APInt & C, ISD::CondCode & CC )

static

Definition at line 4417 of file AArch64ISelLowering.cpp.

References llvm::ISD::AND, C(), CC, LHS, llvm::ISD::SETGE, llvm::ISD::SETGT, llvm::ISD::SETLE, and llvm::ISD::SETLT.

Referenced by getAArch64Cmp().

shouldLowerTailCallStackArg()

bool shouldLowerTailCallStackArg ( const MachineFunction & MF, const CCValAssign & VA, SDValue Arg, ISD::ArgFlagsTy Flags, int CallOffset )

static

Check whether a stack argument requires lowering in a tail call.

Definition at line 10446 of file AArch64ISelLowering.cpp.

References llvm::ISD::ANY_EXTEND, llvm::ISD::BITCAST, llvm::dyn_cast(), llvm::LoadSDNode::getBasePtr(), llvm::EVT::getFixedSizeInBits(), llvm::MachineFunction::getFrameInfo(), llvm::MemSDNode::getMemoryVT(), llvm::MachineFrameInfo::getObjectOffset(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::MVT::getSizeInBits(), llvm::CCValAssign::getValVT(), llvm::SDNode::isAssert(), llvm::MachineFrameInfo::isImmutableObjectIndex(), uint64_t, and llvm::ISD::ZERO_EXTEND.

simplifySetCCIntoEq()

void simplifySetCCIntoEq ( ISD::CondCode & CC, SDValue & LHS, SDValue & RHS, SelectionDAG & DAG, const SDLoc DL )

static

Definition at line 4904 of file AArch64ISelLowering.cpp.

References llvm::ISD::AND, CC, DL, llvm::dyn_cast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::ConstantSDNode::getZExtValue(), llvm::isPowerOf2_64(), LHS, RHS, llvm::ISD::SETEQ, llvm::ISD::SETULT, and uint64_t.

skipExtensionForVectorMULL()

SDValue skipExtensionForVectorMULL ( SDValue N, SelectionDAG & DAG )

static

Definition at line 5900 of file AArch64ISelLowering.cpp.

References assert(), llvm::SelectionDAG::getContext(), llvm::EVT::getHalfSizedIntegerVT(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarType(), llvm::EVT::getVectorElementCount(), llvm::EVT::getVectorVT(), llvm::EVT::is128BitVector(), N, and llvm::ISD::TRUNCATE.

splitStores()

SDValue splitStores ( SDNode * N, TargetLowering::DAGCombinerInfo & DCI, SelectionDAG & DAG, const AArch64Subtarget * Subtarget )

static

Definition at line 26851 of file AArch64ISelLowering.cpp.

References llvm::ISD::ADD, llvm::cast(), DL, llvm::MemSDNode::getAlign(), llvm::StoreSDNode::getBasePtr(), llvm::MemSDNode::getChain(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getExtractSubvector(), llvm::MachineMemOperand::getFlags(), llvm::MachineFunction::getFunction(), llvm::EVT::getHalfNumVectorElementsVT(), llvm::SelectionDAG::getMachineFunction(), llvm::MemSDNode::getMemOperand(), llvm::SelectionDAG::getNode(), llvm::MemSDNode::getPointerInfo(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getStore(), llvm::SDValue::getValue(), llvm::StoreSDNode::getValue(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::Function::hasMinSize(), llvm::EVT::isFixedLengthVector(), llvm::LSBaseSDNode::isIndexed(), llvm::MemSDNode::isVolatile(), N, replaceSplatVectorStore(), replaceZeroVectorStore(), SDValue(), and Subtarget.

splitStoreSplat()

SDValue splitStoreSplat ( SelectionDAG & DAG, StoreSDNode & St, SDValue SplatVal, unsigned NumVecElts )

static

Definition at line 26526 of file AArch64ISelLowering.cpp.

References llvm::ISD::ADD, assert(), llvm::cast(), llvm::commonAlignment(), DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getStore(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::MachinePointerInfo::getWithOffset(), llvm::isa(), llvm::Offset, and uint64_t.

Referenced by replaceSplatVectorStore(), and replaceZeroVectorStore().

STATISTIC() [1/2]

STATISTIC	(	NumOptimizedImms	,
		"Number of times immediates were optimized"	)

STATISTIC() [2/2]

STATISTIC	(	NumTailCalls	,
		"Number of tail calls"	)

stripVRegCopies()

const MachineInstr * stripVRegCopies ( const MachineRegisterInfo & MRI, Register Reg )

static

Definition at line 3539 of file AArch64ISelLowering.cpp.

References assert(), DefMI, MRI, and Reg.

Referenced by llvm::AArch64TargetLowering::fixupPtrauthDiscriminator().

tryAdvSIMDModImm16()

SDValue tryAdvSIMDModImm16 ( unsigned NewOp, SDValue Op, SelectionDAG & DAG, const APInt & Bits, const SDValue * LHS = nullptr )

static

Definition at line 16478 of file AArch64ISelLowering.cpp.

References DL, llvm::AArch64_AM::encodeAdvSIMDModImmType5(), llvm::AArch64_AM::encodeAdvSIMDModImmType6(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getSubtarget(), llvm::AArch64_AM::isAdvSIMDModImmType5(), llvm::AArch64_AM::isAdvSIMDModImmType6(), llvm::EVT::isFixedLengthVector(), llvm::AArch64Subtarget::isNeonAvailable(), LHS, SDValue(), and uint64_t.

Referenced by ConstantBuildVector(), and performANDCombine().

tryAdvSIMDModImm32()

SDValue tryAdvSIMDModImm32 ( unsigned NewOp, SDValue Op, SelectionDAG & DAG, const APInt & Bits, const SDValue * LHS = nullptr )

static

Definition at line 16425 of file AArch64ISelLowering.cpp.

References DL, llvm::AArch64_AM::encodeAdvSIMDModImmType1(), llvm::AArch64_AM::encodeAdvSIMDModImmType2(), llvm::AArch64_AM::encodeAdvSIMDModImmType3(), llvm::AArch64_AM::encodeAdvSIMDModImmType4(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getSubtarget(), llvm::AArch64_AM::isAdvSIMDModImmType1(), llvm::AArch64_AM::isAdvSIMDModImmType2(), llvm::AArch64_AM::isAdvSIMDModImmType3(), llvm::AArch64_AM::isAdvSIMDModImmType4(), llvm::EVT::isFixedLengthVector(), llvm::AArch64Subtarget::isNeonAvailable(), LHS, SDValue(), and uint64_t.

Referenced by ConstantBuildVector(), and performANDCombine().

tryAdvSIMDModImm321s()

SDValue tryAdvSIMDModImm321s ( unsigned NewOp, SDValue Op, SelectionDAG & DAG, const APInt & Bits )

static

Definition at line 16523 of file AArch64ISelLowering.cpp.

References DL, llvm::AArch64_AM::encodeAdvSIMDModImmType7(), llvm::AArch64_AM::encodeAdvSIMDModImmType8(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::AArch64_AM::isAdvSIMDModImmType7(), llvm::AArch64_AM::isAdvSIMDModImmType8(), SDValue(), and uint64_t.

Referenced by ConstantBuildVector().

tryAdvSIMDModImm64()

SDValue tryAdvSIMDModImm64 ( unsigned NewOp, SDValue Op, SelectionDAG & DAG, const APInt & Bits )

static

Definition at line 16404 of file AArch64ISelLowering.cpp.

References DL, llvm::AArch64_AM::encodeAdvSIMDModImmType10(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::AArch64_AM::isAdvSIMDModImmType10(), SDValue(), and uint64_t.

Referenced by ConstantBuildVector().

tryAdvSIMDModImm8()

SDValue tryAdvSIMDModImm8 ( unsigned NewOp, SDValue Op, SelectionDAG & DAG, const APInt & Bits )

static

Definition at line 16554 of file AArch64ISelLowering.cpp.

References DL, llvm::AArch64_AM::encodeAdvSIMDModImmType9(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::AArch64_AM::isAdvSIMDModImmType9(), SDValue(), and uint64_t.

Referenced by ConstantBuildVector().

tryAdvSIMDModImmFP()

SDValue tryAdvSIMDModImmFP ( unsigned NewOp, SDValue Op, SelectionDAG & DAG, const APInt & Bits )

static

Definition at line 16575 of file AArch64ISelLowering.cpp.

References DL, llvm::AArch64_AM::encodeAdvSIMDModImmType11(), llvm::AArch64_AM::encodeAdvSIMDModImmType12(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::AArch64_AM::isAdvSIMDModImmType11(), llvm::AArch64_AM::isAdvSIMDModImmType12(), SDValue(), and uint64_t.

Referenced by ConstantBuildVector().

tryCombineCRC32()

SDValue tryCombineCRC32 ( unsigned Mask, SDNode * N, SelectionDAG & DAG )

static

Definition at line 25160 of file AArch64ISelLowering.cpp.

References llvm::ISD::AND, llvm::dyn_cast(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::ConstantSDNode::getZExtValue(), llvm::ISD::INTRINSIC_WO_CHAIN, N, and SDValue().

Referenced by performIntrinsicCombine().

tryCombineExtendRShTrunc()

SDValue tryCombineExtendRShTrunc ( SDNode * N, SelectionDAG & DAG )

static

Definition at line 26994 of file AArch64ISelLowering.cpp.

References assert(), DL, llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), getPredicateForVector(), llvm::Hi, Lo, N, and SDValue().

Referenced by performUzpCombine().

tryCombineFADDReductionWithZero()

SDValue tryCombineFADDReductionWithZero ( SDNode * N, SelectionDAG & DAG, const AArch64Subtarget * Subtarget, SDValue Vec, SDValue Start = SDValue() )

static

Optimize patterns where we insert zeros into vector lanes before a floating-point add reduction.

This handles both the aarch64.neon.faddv intrinsic and ISD::VECREDUCE_FADD/VECREDUCE_SEQ_FADD. When Start is set, the reduction is sequential (left-to-right), otherwise it is pairwise (tree).

Definition at line 25462 of file AArch64ISelLowering.cpp.

References llvm::SelectionDAG::canIgnoreSignBitOfZeroResult(), llvm::SelectionDAG::computeKnownBits(), DL, llvm::ISD::FADD, llvm::APInt::getAllOnes(), llvm::SelectionDAG::getConstantFP(), llvm::SelectionDAG::getExtractVectorElt(), llvm::SelectionDAG::getNode(), llvm::APInt::getOneBitSet(), llvm::APInt::getSignedMaxValue(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::APInt::getZero(), I, llvm::EVT::isFixedLengthVector(), llvm::isPowerOf2_32(), llvm::APInt::isSubsetOf(), N, llvm::APInt::popcount(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), SDValue(), llvm::APInt::setBit(), llvm::SmallVectorTemplateCommon< T, typename >::size(), Subtarget, and llvm::KnownBits::Zero.

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine(), and performIntrinsicCombine().

tryCombineFixedPointConvert()

SDValue tryCombineFixedPointConvert ( SDNode * N, TargetLowering::DAGCombinerInfo & DCI, SelectionDAG & DAG )

static

Definition at line 23385 of file AArch64ISelLowering.cpp.

References DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::ISD::INTRINSIC_WO_CHAIN, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), N, and SDValue().

Referenced by performIntrinsicCombine().

tryCombineLongOpWithDup()

SDValue tryCombineLongOpWithDup ( unsigned IID, SDNode * N, TargetLowering::DAGCombinerInfo & DCI, SelectionDAG & DAG )

static

Definition at line 25035 of file AArch64ISelLowering.cpp.

References assert(), llvm::SelectionDAG::getNode(), llvm::ISD::INTRINSIC_WO_CHAIN, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), isEssentiallyExtractHighSubvector(), LHS, N, llvm::Intrinsic::not_intrinsic, RHS, SDValue(), and tryExtendDUPToExtractHigh().

Referenced by performExtendCombine(), performIntrinsicCombine(), and performMULLCombine().

tryCombineMULLWithUZP1()

SDValue tryCombineMULLWithUZP1 ( SDNode * N, TargetLowering::DAGCombinerInfo & DCI, SelectionDAG & DAG )

static

Definition at line 31559 of file AArch64ISelLowering.cpp.

References llvm::ISD::BITCAST, DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getContext(), llvm::EVT::getDoubleNumVectorElementsVT(), llvm::SelectionDAG::getExtractSubvector(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::SDValue::getOperand(), llvm::User::getOperand(), llvm::SelectionDAG::getPOISON(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::SDNode::hasOneUse(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), isEssentiallyExtractHighSubvector(), llvm::isNullConstant(), llvm::SDNode::isPredecessorOf(), llvm::SelectionDAG::isSplatValue(), LHS, N, llvm::SelectionDAG::ReplaceAllUsesWith(), RHS, SDValue(), llvm::SDValue::setNode(), llvm::ISD::TRUNCATE, llvm::SDNode::use_size(), llvm::SDNode::user_begin(), and llvm::SDNode::users().

Referenced by performMULLCombine().

tryCombineNeonFcvtFP16ToI16()

SDValue tryCombineNeonFcvtFP16ToI16 ( SDNode * N, unsigned Opcode, SelectionDAG & DAG )

static

Definition at line 25397 of file AArch64ISelLowering.cpp.

References DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), N, SDValue(), and llvm::ISD::TRUNCATE.

Referenced by performIntrinsicCombine().

tryCombineShiftImm()

SDValue tryCombineShiftImm ( unsigned IID, SDNode * N, SelectionDAG & DAG )

static

Definition at line 25068 of file AArch64ISelLowering.cpp.

References DL, llvm::dyn_cast(), llvm::SelectionDAG::getExtractVectorElt(), llvm::SelectionDAG::getNode(), llvm::APInt::getSExtValue(), llvm::SelectionDAG::getSignedConstant(), llvm::SelectionDAG::getTargetConstant(), llvm::BuildVectorSDNode::isConstantSplat(), llvm_unreachable, N, llvm::ISD::SCALAR_TO_VECTOR, and SDValue().

Referenced by performIntrinsicCombine().

tryCombineToREV()

SDValue tryCombineToREV ( SDNode * N, SelectionDAG & DAG, TargetLowering::DAGCombinerInfo & DCI )

static

Definition at line 22254 of file AArch64ISelLowering.cpp.

References assert(), DL, llvm::SDValue::getConstantOperandVal(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getPOISON(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::is64BitVector(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::ISD::isConstantSplatVector(), llvm::EVT::isFixedLengthVector(), llvm::EVT::isScalableVector(), llvm::EVT::isVector(), N, llvm::ISD::OR, SDValue(), and std::swap().

Referenced by performORCombine().

tryConvertSVEWideCompare()

SDValue tryConvertSVEWideCompare ( SDNode * N, ISD::CondCode CC, TargetLowering::DAGCombinerInfo & DCI, SelectionDAG & DAG )

static

Definition at line 25239 of file AArch64ISelLowering.cpp.

References CC, DL, llvm::dyn_cast(), llvm::SelectionDAG::getCondCode(), llvm::SelectionDAG::getConstant(), getIntrinsicID(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getSignedConstant(), Imm, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm_unreachable, N, Pred, SDValue(), llvm::Splat, llvm::ISD::SPLAT_VECTOR, and uint64_t.

Referenced by performIntrinsicCombine().

tryExtendDUPToExtractHigh()

SDValue tryExtendDUPToExtractHigh ( SDValue N, SelectionDAG & DAG )

static

Definition at line 23448 of file AArch64ISelLowering.cpp.

References DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getExtractSubvector(), llvm::SelectionDAG::getNode(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::MVT::is64BitVector(), N, and SDValue().

Referenced by performAddSubLongCombine(), and tryCombineLongOpWithDup().

tryFormConcatFromShuffle()

SDValue tryFormConcatFromShuffle ( SDValue Op, SelectionDAG & DAG )

static

Definition at line 15592 of file AArch64ISelLowering.cpp.

References llvm::cast(), llvm::ISD::CONCAT_VECTORS, DL, llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getExtractSubvector(), llvm::EVT::getHalfNumVectorElementsVT(), llvm::SelectionDAG::getNode(), llvm::EVT::getVectorElementType(), isConcatMask(), SDValue(), and llvm::V1.

tryLowerMultiVectorLoad()

bool tryLowerMultiVectorLoad ( LoadSDNode * LoadNode, SmallVectorImpl< SDValue > & Results, SelectionDAG & DAG )

static

Definition at line 7984 of file AArch64ISelLowering.cpp.

References llvm::ISD::CONCAT_VECTORS, DL, llvm::LoadSDNode::getBasePtr(), llvm::MemSDNode::getChain(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), getPTrueAsCounter(), llvm::EVT::getSimpleVT(), getSVEMultiVectorInfo(), llvm::SDValue::getValue(), llvm::SDNode::getValueType(), I, llvm::ISD::INTRINSIC_W_CHAIN, isSimpleScalableLoadOrStore(), SVEMultiVectorInfo::LoadIntID, SVEMultiVectorInfo::NumVecs, llvm::SmallVectorTemplateBase< T, bool >::push_back(), SVEMultiVectorInfo::RegVT, and Results.

tryLowerMultiVectorStore()

SDValue tryLowerMultiVectorStore ( StoreSDNode * StoreNode, SelectionDAG & DAG )

static

Definition at line 7950 of file AArch64ISelLowering.cpp.

References AbstractManglingParser< Derived, Alloc >::Ops, DL, llvm::StoreSDNode::getBasePtr(), llvm::MemSDNode::getChain(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getExtractSubvector(), llvm::SelectionDAG::getMemIntrinsicNode(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getMemoryVT(), getPTrueAsCounter(), llvm::EVT::getSimpleVT(), getSVEMultiVectorInfo(), llvm::StoreSDNode::getValue(), llvm::MVT::getVectorMinNumElements(), llvm::SelectionDAG::getVTList(), llvm::ISD::INTRINSIC_VOID, isSimpleScalableLoadOrStore(), SVEMultiVectorInfo::NumVecs, SVEMultiVectorInfo::RegVT, SDValue(), and SVEMultiVectorInfo::StoreIntID.

tryLowerSmallVectorExtLoad()

SDValue tryLowerSmallVectorExtLoad ( LoadSDNode * Load, SelectionDAG & DAG )

static

Helper function to optimize loads of extended small vectors.

These patterns would otherwise get scalarized into inefficient sequences.

Definition at line 8113 of file AArch64ISelLowering.cpp.

References llvm::ISD::BITCAST, DL, llvm::ISD::EXTLOAD, llvm::SelectionDAG::getExtractSubvector(), llvm::MVT::getFloatingPointVT(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getStoreSizeInBits(), llvm::SelectionDAG::getSubtarget(), llvm::SDValue::getValue(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorVT(), isEligibleForSmallVectorLoadOpt(), llvm::Load, llvm::ISD::NON_EXTLOAD, llvm::ISD::SCALAR_TO_VECTOR, SDValue(), llvm::ISD::SEXTLOAD, llvm::ISD::SIGN_EXTEND, Subtarget, llvm::ISD::ZERO_EXTEND, and llvm::ISD::ZEXTLOAD.

tryLowerToBSL()

SDValue tryLowerToBSL ( SDValue N, SelectionDAG & DAG )

static

Definition at line 16757 of file AArch64ISelLowering.cpp.

References llvm::ISD::ADD, llvm::Add, llvm::ISD::AND, assert(), DL, llvm::dyn_cast(), llvm::ConstantSDNode::getAPIntValue(), llvm::SDNode::getAsAPIntVal(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::SelectionDAG::getSubtarget(), llvm::EVT::getVectorNumElements(), llvm::ISD::isConstantSplatVector(), llvm::ISD::isConstantSplatVectorAllOnes(), llvm::ISD::isConstantSplatVectorAllZeros(), llvm::EVT::isScalableVector(), llvm::EVT::isVector(), N, llvm::O0, llvm::O1, SDValue(), llvm::ISD::SUB, llvm::Sub, Subtarget, and llvm::APInt::trunc().

tryLowerToSLI()

SDValue tryLowerToSLI ( SDNode * N, SelectionDAG & DAG )

static

Definition at line 16664 of file AArch64ISelLowering.cpp.

References llvm::ISD::AND, assert(), C(), DL, llvm::dyn_cast(), llvm::ConstantSDNode::getAPIntValue(), llvm::APInt::getHighBitsSet(), llvm::APInt::getLowBitsSet(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::SelectionDAG::getTargetConstant(), Imm, isAllActivePredicate(), llvm::ISD::isConstantSplatVector(), llvm::EVT::isVector(), N, SDValue(), uint64_t, X, Y, and llvm::APInt::zextOrTrunc().

trySimplifySrlAddToRshrnb()

SDValue trySimplifySrlAddToRshrnb ( SDValue Srl, SelectionDAG & DAG, const AArch64Subtarget * Subtarget )

static

Definition at line 27034 of file AArch64ISelLowering.cpp.

References canLowerSRLToRoundingShiftForVT(), DL, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetConstant(), llvm::SDNode::getValueType(), llvm::EVT::isScalableVector(), SDValue(), and Subtarget.

Referenced by performMSTORECombine(), and performUzpCombine().

trySQABSCombine()

SDValue trySQABSCombine ( SDNode * N, SelectionDAG & DAG, const AArch64TargetLowering & TLI )

static

Definition at line 32164 of file AArch64ISelLowering.cpp.

References DL, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getPOISON(), getPredicateForVector(), llvm::EVT::getScalarSizeInBits(), llvm::APInt::getSignedMaxValue(), llvm::SelectionDAG::getSubtarget(), llvm::SelectionDAG::getTargetConstant(), llvm::ISD::INTRINSIC_WO_CHAIN, llvm::EVT::is128BitVector(), llvm::EVT::is64BitVector(), llvm::EVT::isInteger(), llvm::EVT::isScalableVector(), llvm::EVT::isVector(), llvm::EVT::isVectorOf(), llvm::SDPatternMatch::m_Abs(), llvm::PatternMatch::m_SpecificInt(), llvm::PatternMatch::m_UMin(), llvm::PatternMatch::m_Value(), N, llvm::SDPatternMatch::sd_match(), SDValue(), Subtarget, TLI, and X.

Referenced by performMINMAXCombine().

trySQDMULHCombine()

SDValue trySQDMULHCombine ( SDNode * N, SelectionDAG & DAG )

static

Definition at line 24133 of file AArch64ISelLowering.cpp.

References DL, llvm::EVT::getFixedSizeInBits(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::ConstantSDNode::getSExtValue(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::SDValue::hasOneUse(), llvm::isConstOrConstSplat(), llvm::EVT::isPow2VectorType(), llvm::EVT::isScalableVector(), llvm::EVT::isVector(), llvm::ISD::MUL, Mul, N, SDValue(), llvm::ISD::SIGN_EXTEND, llvm::ISD::SMIN, llvm::ISD::SRA, and llvm::V1.

Referenced by performMINMAXCombine().

trySVESplat64()

SDValue trySVESplat64 ( SDValue Op, SelectionDAG & DAG, const AArch64Subtarget * ST, APInt & DefBits )

static

Definition at line 16936 of file AArch64ISelLowering.cpp.

References DL, llvm::SelectionDAG::getConstant(), llvm::APInt::getHiBits(), llvm::APInt::getLoBits(), llvm::SelectionDAG::getNode(), llvm::APInt::getSExtValue(), llvm::APInt::getZExtValue(), llvm::EVT::is128BitVector(), llvm::AArch64_AM::isSVECpyDupImm(), llvm::AArch64_AM::isSVELogicalImm(), SDValue(), llvm::APInt::trunc(), and uint64_t.

Referenced by ConstantBuildVector().

trySwapVSelectOperands()

SDValue trySwapVSelectOperands ( SDNode * N, SelectionDAG & DAG )

static

Definition at line 30394 of file AArch64ISelLowering.cpp.

References CC, llvm::ISD::FADD, llvm::ISD::FMUL, llvm::ISD::FSUB, llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getOperandCondCode(), llvm::SelectionDAG::getSetCC(), llvm::ISD::getSetCCInverse(), llvm::SDValue::getValueType(), llvm::SDValue::hasOneUse(), N, SDValue(), llvm::ISD::SETCC, and llvm::ISD::VSELECT.

Referenced by performVSelectCombine().

tryToConvertShuffleOfTbl2ToTbl4()

SDValue tryToConvertShuffleOfTbl2ToTbl4 ( SDValue Op, ArrayRef< int > ShuffleMask, SelectionDAG & DAG )

static

Definition at line 15809 of file AArch64ISelLowering.cpp.

References llvm::ISD::BUILD_VECTOR, C(), DL, llvm::dyn_cast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getTargetConstant(), llvm::SDValue::getValueType(), I, llvm::ISD::INTRINSIC_WO_CHAIN, and SDValue().

tryToReplaceScalarFPConversionWithSVE()

SDValue tryToReplaceScalarFPConversionWithSVE ( SDNode * N, SelectionDAG & DAG, TargetLowering::DAGCombinerInfo & DCI, const AArch64Subtarget * Subtarget )

static

Tries to replace scalar FP <-> INT conversions with SVE in streaming functions, this can help to reduce the number of fmovs to/from GPRs.

Definition at line 22054 of file AArch64ISelLowering.cpp.

References llvm::EVT::bitsGT(), llvm::EVT::changeVectorElementType(), DL, llvm::ISD::FP_TO_SINT_SAT, llvm::ISD::FP_TO_UINT_SAT, llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getExtractVectorElt(), llvm::SelectionDAG::getInsertVectorElt(), llvm::SelectionDAG::getNode(), getPackedSVEVectorVT(), llvm::SelectionDAG::getPOISON(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getValueType(), llvm::EVT::getVectorElementType(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), isSupportedType(), llvm::EVT::isVector(), N, llvm::SmallVectorTemplateBase< T, bool >::push_back(), SDValue(), and Subtarget.

Referenced by performFpToIntCombine(), and performIntToFpCombine().

tryToWidenSetCCOperands()

SDValue tryToWidenSetCCOperands ( SDNode * Op, SelectionDAG & DAG )

static

Definition at line 29892 of file AArch64ISelLowering.cpp.

References llvm::any_of(), CC, llvm::EVT::changeVectorElementType(), DL, llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNodeIfExists(), llvm::SDNode::getOpcode(), llvm::EVT::getScalarSizeInBits(), llvm::SDNode::getValueType(), llvm::SelectionDAG::getVTList(), llvm::ISD::isConstantSplatVector(), llvm::EVT::isVector(), N, SDValue(), llvm::ISD::SETCC, llvm::ISD::SIGN_EXTEND, V, llvm::ISD::VSELECT, and llvm::ISD::ZERO_EXTEND.

Referenced by performSETCCCombine().

tryWidenMaskForShuffle()

SDValue tryWidenMaskForShuffle ( SDValue Op, SelectionDAG & DAG )

static

Definition at line 15776 of file AArch64ISelLowering.cpp.

References llvm::cast(), DL, llvm::SelectionDAG::getBitcast(), llvm::EVT::getFixedSizeInBits(), llvm::MVT::getFloatingPointVT(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), llvm::EVT::isFloatingPoint(), llvm::TargetLoweringBase::isTypeLegal(), SDValue(), llvm::V1, and llvm::widenShuffleMaskElts().

UseTlsOffset()

Value * UseTlsOffset ( IRBuilderBase & IRB, unsigned Offset )

static

Definition at line 34045 of file AArch64ISelLowering.cpp.

References llvm::IRBuilderBase::CreateCall(), llvm::IRBuilderBase::CreateConstGEP1_32(), llvm::IRBuilderBase::CreatePointerCast(), llvm::IRBuilderBase::getInt8Ty(), llvm::IRBuilderBase::getModule(), llvm::Intrinsic::getOrInsertDeclaration(), llvm::IRBuilderBase::getPtrTy(), and llvm::Offset.

Referenced by llvm::AArch64TargetLowering::getIRStackGuard(), and llvm::AArch64TargetLowering::getSafeStackPointerLocation().

valueToCarryFlag()

SDValue valueToCarryFlag ( SDValue Value, SelectionDAG & DAG, bool Invert )

static

Definition at line 4756 of file AArch64ISelLowering.cpp.

References DL, FlagsVT, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), and llvm::SelectionDAG::getVTList().

Referenced by LowerADDSUBO_CARRY(), lowerADDSUBO_CARRY(), and LowerSETCCCARRY().

WidenVector()

SDValue WidenVector ( SDValue V64Reg, SelectionDAG & DAG )

static

WidenVector - Given a value in the V64 register class, produce the equivalent value in the V128 register class.

Definition at line 14817 of file AArch64ISelLowering.cpp.

References DL, llvm::SelectionDAG::getInsertSubvector(), llvm::SelectionDAG::getPOISON(), llvm::EVT::getSimpleVT(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), and llvm::MVT::getVectorVT().

Referenced by constructDup(), and performConcatVectorsCombine().

Variable Documentation

CondCodeVT

MVT CondCodeVT = MVT::i32

constexpr

Value type used for condition codes.

Definition at line 117 of file AArch64ISelLowering.cpp.

Referenced by getCondCode().

ContainerVT

ContainerVT = getContainerForFixedLengthVector(DAG, SubVecTy)

Definition at line 28551 of file AArch64ISelLowering.cpp.

Referenced by combineToVCPOP(), convertFixedMaskToScalableVector(), GenerateFixedLengthSVETBL(), getDefaultVLOps(), getDefaultVLOps(), getLMULOctuple(), llvm::RISCVTargetLowering::getRegForInlineAsmConstraint(), llvm::RISCVTTIImpl::getScalarizationOverhead(), llvm::RISCVTTIImpl::getShuffleCost(), llvm::RISCVTargetLowering::isLegalInterleavedAccessType(), isLegalVTForZvzipDeinterleavedOperand(), isLegalVTForZvzipInterleavedOperand(), lowerBUILD_VECTOR(), lowerBuildVectorOfConstants(), lowerBuildVectorViaDominantValues(), lowerBuildVectorViaVID(), lowerCttzElts(), lowerFixedVectorSegLoadIntrinsics(), lowerFixedVectorSegStoreIntrinsics(), lowerFMAXIMUM_FMINIMUM(), llvm::RISCVTargetLowering::LowerOperation(), lowerShuffleViaVRegSplitting(), lowerVECTOR_SHUFFLEAsVSlide1(), lowerVECTOR_SHUFFLEAsVSlidedown(), lowerVECTOR_SHUFFLEAsVSlideup(), lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND(), lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND(), lowerZvzipVPAIR(), lowerZvzipVUNZIP(), lowerZvzipVZIP(), matchSplatAsGather(), performExtractLastActiveCombine(), performFP_TO_INTCombine(), performLD1Combine(), performLegalizedInterleavedStoreCombine(), performLegalizedVectorDeinterleaveCombine(), llvm::RISCVTargetLowering::ReplaceNodeResults(), llvm::RISCVTargetLowering::RISCVTargetLowering(), and llvm::RISCVDAGToDAGISel::Select().

else

else

Initial value:

{
    Pred = getPredicateForFixedLengthVector(DAG, DL, SubVecTy)

Definition at line 28562 of file AArch64ISelLowering.cpp.

FlagsVT

MVT FlagsVT = MVT::i32

constexpr

Value type used for NZCV flags.

Definition at line 120 of file AArch64ISelLowering.cpp.

Referenced by emitComparison(), emitConditionalComparison(), emitStrictFPComparison(), Expand64BitShift(), getAArch64XALUOOp(), lowerADDSUBO_CARRY(), lowerPredCounterWhile(), performANDORCSELCombine(), performRNDRCombine(), reassociateCSELOperandsForCSE(), and valueToCarryFlag().

FPRArgRegs

const MCPhysReg FPRArgRegs[]

static

Initial value:

= {AArch64::Q0, AArch64::Q1, AArch64::Q2,
                                       AArch64::Q3, AArch64::Q4, AArch64::Q5,
                                       AArch64::Q6, AArch64::Q7}

Definition at line 125 of file AArch64ISelLowering.cpp.

Referenced by llvm::AArch64::getFPRArgRegs().

GPRArgRegs

const MCPhysReg GPRArgRegs[]

static

Initial value:

= {AArch64::X0, AArch64::X1, AArch64::X2,
                                       AArch64::X3, AArch64::X4, AArch64::X5,
                                       AArch64::X6, AArch64::X7}

Definition at line 122 of file AArch64ISelLowering.cpp.

Referenced by f64AssignAAPCS(), and llvm::AArch64::getGPRArgRegs().

IsScalable

bool IsScalable = SubVecTy.isScalableVector()

Definition at line 28491 of file AArch64ISelLowering.cpp.

Referenced by canNarrowLoad(), canNarrowOps(), llvm::TargetLoweringBase::computeRegisterProperties(), llvm::AArch64TargetLowering::createComplexDeinterleavingIR(), llvm::HexagonTTIImpl::getMinimumVF(), llvm::TargetTransformInfo::getMinimumVF(), llvm::TargetTransformInfoImplBase::getMinimumVF(), llvm::getOptionalElementCountLoopAttribute(), llvm::Intrinsic::IITDescriptor::getVector(), llvm::EVT::getVectorVT(), llvm::MVT::getVectorVT(), if(), if(), llvm::VPWidenPointerInductionRecipe::onlyScalarsGenerated(), performVectorDeinterleaveCombine(), and llvm::TargetLoweringBase::shouldExpandGetVectorLength().

MemN

auto* MemN = cast<MemSDNode>(N)

Definition at line 28496 of file AArch64ISelLowering.cpp.

Referenced by llvm::getIdiomaticVectorType(), getMemIntrinsicNode(), llvm::getMemoryPtr(), llvm::getNodeChain(), if(), if(), PerformMVEVLDCombine(), reportInvalidTensormapReplaceUsage(), and TryCombineBaseUpdate().

NumParts

unsigned NumParts = ValueInterleaveOps.size()

Definition at line 28483 of file AArch64ISelLowering.cpp.

Referenced by slpvectorizer::BoUpSLP::ShuffleCostEstimator::add(), slpvectorizer::BoUpSLP::ShuffleCostEstimator::add(), slpvectorizer::BoUpSLP::ShuffleCostEstimator::adjustExtracts(), slpvectorizer::BoUpSLP::ShuffleInstructionBuilder::adjustExtracts(), analyzeArgSplit(), append(), llvm::LegalizerHelper::buildConstantShiftPart(), llvm::AMDGPURegisterBankInfo::buildReadFirstLane(), slpvectorizer::BoUpSLP::canBuildSplitNode(), castBufferRsrcFromV4I32(), castBufferRsrcToV4I32(), slpvectorizer::BoUpSLP::computeMinimumValueSizes(), llvm::CallLowering::determineAssignments(), ensureMatchedVecRegParts(), llvm::AMDGPURegisterBankInfo::executeInWaterfallLoop(), llvm::extractParts(), llvm::extractParts(), llvm::LegalizerHelper::fewerElementsVectorMerge(), llvm::LegalizerHelper::fewerElementsVectorReductions(), llvm::LegalizerHelper::fewerElementsVectorSeqReductions(), slpvectorizer::BoUpSLP::findReusedOrderedScalars(), getCopyFromParts(), getCopyFromPartsVector(), getCopyToParts(), getCopyToPartsVector(), llvm::RegsForValue::getCopyToRegs(), llvm::slpvectorizer::getFloorFullVectorNumberOfElements(), llvm::slpvectorizer::getFullVectorNumberOfElements(), getNarrowTypeBreakDown(), llvm::slpvectorizer::getNumberOfPartsOrRegs(), llvm::slpvectorizer::getPartNumElems(), llvm::SIRegisterInfo::getRegSplitParts(), slpvectorizer::BoUpSLP::getReorderingData(), llvm::GetReturnInfo(), llvm::CallLowering::getReturnInfo(), llvm::SystemZTTIImpl::getVectorTruncCost(), llvm::CallLowering::handleAssignments(), llvm::slpvectorizer::hasFullVectorsOrPowerOf2(), if(), if(), if(), if(), isSequentialConcatOfVectorInterleave(), llvm::RISCVTargetLowering::joinRegisterPartsIntoValue(), llvm::SystemZTargetLowering::joinRegisterPartsIntoValue(), llvm::TargetLowering::joinRegisterPartsIntoValue(), llvm::AMDGPULegalizerInfo::legalizeLaneOp(), llvm::AMDGPULegalizerInfo::legalizeMul(), llvm::SystemZTargetLowering::LowerCall(), llvm::WebAssemblyCallLowering::lowerCall(), llvm::TargetLowering::LowerCallTo(), llvm::SystemZTargetLowering::LowerFormalArguments(), llvm::WebAssemblyCallLowering::lowerFormalArguments(), llvm::WebAssemblyCallLowering::lowerReturn(), makeDstOps(), llvm::LegalizerHelper::narrowScalar(), llvm::LegalizerHelper::narrowScalarExtract(), llvm::LegalizerHelper::narrowScalarMul(), llvm::LegalizerHelper::narrowScalarShift(), llvm::LegalizerHelper::narrowScalarShiftByConstantMultiway(), llvm::LegalizerHelper::narrowScalarShiftMultiway(), PerformInterleavedStoreCombine(), PerformLegalizedInterleavedStoreCombine(), performLegalizedInterleavedStoreCombine(), PerformLegalizedVECTOR_DEINTERLEAVECombine(), performLegalizedVectorDeinterleaveCombine(), PerformVECTOR_DEINTERLEAVECombine(), performVectorDeinterleaveCombine(), PerformVQDMULHCombine(), slpvectorizer::BoUpSLP::processBuildVector(), llvm::LegalizerHelper::reduceLoadStoreWidth(), llvm::AMDGPUTargetMachine::splitModule(), llvm::TargetMachine::splitModule(), llvm::PPCTargetLowering::splitValueIntoRegisterParts(), llvm::RISCVTargetLowering::splitValueIntoRegisterParts(), llvm::SystemZTargetLowering::splitValueIntoRegisterParts(), llvm::TargetLowering::splitValueIntoRegisterParts(), SRAGlobal(), and tryToFindDuplicates().

Ops

SmallVector<SDValue, 8> Ops

Definition at line 28569 of file AArch64ISelLowering.cpp.

Pred

SDValue Pred

Definition at line 28552 of file AArch64ISelLowering.cpp.

Referenced by llvm::jitlink::BlockAddressMap::addBlock(), llvm::jitlink::BlockAddressMap::addBlocks(), llvm::MachineInstrBuilder::addPredicate(), llvm::MIPatternMatch::And< Pred, Preds... >::And(), llvm::SDPatternMatch::And< Pred, Preds... >::And(), AntiDepEdges(), llvm::fuzzerop::anyAggregateType(), llvm::fuzzerop::anyFloatOrVecFloatType(), llvm::fuzzerop::anyFloatType(), llvm::fuzzerop::anyIntOrVecIntType(), llvm::fuzzerop::anyIntType(), llvm::fuzzerop::anyPtrType(), llvm::fuzzerop::anyType(), llvm::fuzzerop::anyVectorType(), append(), BBIsJumpedOver(), BBIsJumpedOver(), bbIsJumpedOver(), llvm::fuzzerop::boolOrVecBoolType(), llvm::SSAUpdaterImpl< UpdaterT >::BuildBlockList(), buildClonedLoops(), buildNonAtomicBinOp(), calcMaxScratches(), calcMaxScratches(), CalcNodeSethiUllmanNumber(), CalcNodeSethiUllmanNumber(), calculateUpperBound(), CanGenerateTest(), CanLowerPriorityDirectlyInPredecessors(), canoncalizeSelectICmpMinMax(), canonicalizeCmpWithConstant(), canonicalizeForInvariantConditionInjection(), canonicalizeSaturatedAddSigned(), canonicalizeSaturatedAddUnsigned(), canonicalizeSaturatedSubtractSigned(), canonicalizeSaturatedSubtractUnsigned(), canProveExitOnFirstIteration(), CanProveNotTakenFirstIteration(), CanRedirectPredsOfEmptyBBToSucc(), checkAndReplaceCondition(), checkAndReplaceMinMax(), checkCondition(), llvm::AAMemoryLocation::checkForAllAccessesToMemoryKind(), llvm::Attributor::checkForAllCallees(), llvm::Attributor::checkForAllCallLikeInstructions(), llvm::Attributor::checkForAllCallSites(), llvm::MustBeExecutedContextExplorer::checkForAllContext(), checkForAllInstructionsImpl(), llvm::Attributor::checkForAllReadWriteInstructions(), llvm::Attributor::checkForAllReturnedValues(), llvm::Attributor::checkForAllUses(), checkOrAndOpImpliedByOther(), llvm::LegalizeRuleSet::clampScalar(), llvm::DISubprogram::cleanupRetainedNodesIf(), llvm::TargetInstrInfo::ClobbersPredicate(), llvm::VPlanPatternMatch::Cmp_match< Op0_t, Op1_t, Opcodes >::Cmp_match(), llvm::PatternMatch::CmpClass_match< LHS_t, RHS_t, Class, Commutable >::CmpClass_match(), cmpExcludesZero(), llvm::DDGNode::collectInstructions(), collectPHIKills(), collectTransitivePredecessors(), collectUnswitchCandidatesWithInjections(), combineRecipe(), combineSVEReductionFP(), combineSVEReductionInt(), combineSVEReductionOrderedFP(), llvm::sandboxir::FCmpInst::compare(), llvm::sandboxir::ICmpInst::compare(), llvm::MIPatternMatch::CompareOp_match< Pred_P, LHS_P, RHS_P, Opcode, Commutable >::CompareOp_match(), computeKnownBitsFromCmp(), computeKnownBitsFromICmpCond(), computeKnownBitsFromOperator(), computeKnownExponentRangeFromContext(), computeKnownFPClassFromCond(), computePointerICmp(), llvm::RegionBase< RegionTraits< MachineFunction > >::contains(), containsUnconditionalCallSafepoint(), countToEliminateCompares(), countUnexecutedTerminators(), llvm::IRBuilderBase::CreateCmp(), llvm::VPBuilderBase<>::createFCmp(), llvm::VPBuilderBase<>::createICmp(), llvm::MachineOperand::CreatePredicate(), createReplacement(), createReplicateRegion(), CriticalPathStep(), llvm::LegalizeRuleSet::customFor(), llvm::LegalizeRuleSet::customFor(), llvm::LegalizeRuleSet::customForCartesianProduct(), DecodeTSTInstruction(), decomposeBitTest(), detectShiftUntilBitTestIdiom(), detectShiftUntilZeroIdiom(), llvm::ArrayRef< llvm::cfg::Update< MachineBasicBlock * > >::drop_until(), llvm::MutableArrayRef< char >::drop_until(), llvm::ArrayRef< llvm::cfg::Update< MachineBasicBlock * > >::drop_while(), llvm::MutableArrayRef< char >::drop_while(), llvm::RegionBase< RegionTraits< MachineFunction > >::dump(), dumpUnpackedICmp(), emitRegPlusImmediate(), emitSPUpdate(), emitThumb1LoadConstPool(), emitThumb2LoadConstPool(), llvm::InlineOrder::erase_if(), llvm::Instruction::eraseMetadataIf(), llvm::Value::eraseMetadataIf(), examinePred(), explicitlyBranchesTo(), extendZeroIfEqual(), extractEquivalentCondition(), llvm::fcmpImpliesClass(), llvm::fcmpImpliesClass(), llvm::fcmpImpliesClass(), llvm::fcmpImpliesClass(), llvm::GenericFloatingPointPredicateUtils< SSAContext >::fcmpImpliesClass(), llvm::GenericFloatingPointPredicateUtils< SSAContext >::fcmpImpliesClass(), llvm::GenericFloatingPointPredicateUtils< SSAContext >::fcmpImpliesClass(), llvm::FCmpInst::FCmpInst(), fcmpPredExcludesEqual(), llvm::fcmpToClassTest(), llvm::fcmpToClassTest(), llvm::GenericFloatingPointPredicateUtils< SSAContext >::fcmpToClassTest(), llvm::GenericFloatingPointPredicateUtils< SSAContext >::fcmpToClassTest(), llvm::MDTuple::filter(), llvm::filter_iterator_impl< WrappedIteratorT, PredicateT, detail::fwd_or_bidi_tag< WrappedIteratorT > >< const_target_iterator, std::function< bool(const Target &)> >::filter_iterator_impl(), llvm::filter_to_vector(), llvm::SSAUpdaterImpl< UpdaterT >::FindAvailableVals(), llvm::SSAUpdaterImpl< UpdaterT >::FindDominators(), findIncDecAfter(), findIncDecBefore(), findLoopComponents(), findReaching(), llvm::vputils::findRecipe(), FindWidenableTerminatorAboveLoop(), fixupBlock(), foldAbsDiff(), foldAddToAshr(), foldAndOrOfICmpsWithPow2AndWithZero(), foldClampRangeOfTwo(), foldCondBranchOnValueKnownInPredecessorImpl(), foldCtpopPow2Test(), foldFCmpFAbsFSubIntToFP(), foldFCmpFNegCommonOp(), foldFCmpFpTrunc(), foldFCmpFSubIntoFCmp(), foldFCmpReciprocalAndZero(), foldFCmpWithFloorAndCeil(), foldICmpAndXX(), foldICmpIntrinsicWithIntrinsic(), foldICmpOfCmpIntrinsicWithConstant(), foldICmpOfUAddOv(), foldICmpOrChain(), foldICmpOrXorSubChain(), foldICmpOrXX(), foldICmpPow2Test(), foldICmpShlLHSC(), foldICmpUSubSatOrUAddSatWithConstant(), foldICmpWithDominatingICmp(), foldICmpWithHighBitMask(), foldICmpWithLowBitMaskedVal(), foldICmpXNegX(), foldICmpXorXX(), foldSelectBinOpIdentity(), foldSelectICmpAndAnd(), foldSelectICmpAndZeroShl(), foldSelectICmpLshrAshr(), foldSelectICmpMinMax(), foldSelectSplitCTLZCTTZ(), foldSignedTruncationCheck(), foldTwoEntryPHINode(), foldVectorCmp(), llvm::InstCombinerImpl::foldVectorSelect(), llvm::AAUnderlyingObjects::forallUnderlyingObjects(), formLCSSAForInstructionsImpl(), gatherIncomingValuesToPhi(), generateKeySubkey(), getAppleRuntimeUnrollPreferences(), getBoolCmpOpcode(), getCaseResults(), llvm::ValueLatticeElement::getCompare(), getCompareCC(), getComparePred(), getComparePred(), getComparePreds(), getConstraintFromMemoryAccess(), llvm::GenericCycleInfo< ContextT >::getCyclePredecessor(), getDwarfOpForIcmpPred(), llvm::RegionBase< Tr >::getEnteringBlock(), llvm::RegionBase< Tr >::getExitingBlock(), llvm::RegionBase< Tr >::getExitingBlocks(), getFCMPLibcallDesc(), getFCmpOpcode(), getFCmpOpcode(), getHashValueImpl(), getICmpOpcode(), llvm::VPBlockBase::getIndexForPredecessor(), llvm::BasicTTIImplBase< BasicTTIImpl >::getIntrinsicInstrCost(), llvm::ICmpInst::getInverseCmpPredicate(), llvm::LoopBase< BlockT, LoopT >::getLoopLatch(), llvm::LoopBase< BlockT, LoopT >::getLoopLatches(), llvm::LoopBase< BlockT, LoopT >::getLoopPredecessor(), getMaskedICmpType(), llvm::RegionInfoBase< Tr >::getMaxRegionExit(), llvm::ICmpInst::getNonStrictCmpPredicate(), llvm::LoopBase< BlockT, LoopT >::getNumBackEdges(), getOperandsForBranch(), llvm::CmpInst::getOrderedPredicate(), getPHIDeps(), getPHIStoreToRewrite(), getPotentialCopiesOfMemoryValue(), getPredicateResult(), getPreStartForExtend(), getPtrCmpOpcode(), getRangeForNUWMulSquare(), getRangeViaSLT(), getSignedOverflowLimitForStep(), getSingleSchedPred(), llvm::ICmpInst::getSwappedCmpPredicate(), llvm::BasicTTIImplBase< BasicTTIImpl >::getTypeBasedIntrinsicInstrCost(), llvm::getUniqueExitBlocksHelper(), llvm::CmpInst::getUnorderedPredicate(), getUnsignedOverflowLimitForStep(), getValueFromICmpCtpop(), getWaitStatesSince(), llvm::MIPatternMatch::GFCstPredMatch(), llvm::MIPatternMatch::GFCstPredMatch< Pred >::GFCstPredMatch(), hasCallsInBlocksBetween(), hasHazard(), llvm::HexagonShuffler::HasInstWith(), llvm::hasNItems(), llvm::hasNItemsOrLess(), llvm::hasNItemsOrMore(), hasNoWrapProblem(), hasOnlyLiveInOpers(), llvm::DomTreeBuilder::SemiNCAInfo< DomTreeT >::HasProperSupport(), hasReachableLoopEntryToHeader(), hasVRegCycleUse(), hoistAdd(), hoistAddSub(), hoistSub(), if(), if(), llvm::MachineBasicBlock::ilist_callback_traits< MachineBasicBlock >, initVRegCycle(), injectPendingInvariantConditions(), llvm::VPBlockUtils::insertBlockBefore(), insertCandidatesWithPendingInjections(), InsertLDR_STR(), insertTrivialPHIs(), instCombineSVELD1(), instCombineSVESDIV(), instCombineSVESrshl(), instCombineSVEST1(), llvm::VPlanPatternMatch::int_pred_ty< is_specific_int, Bitwidth >::int_pred_ty(), isAllActivePredicate(), llvm::InstCombiner::isCanonicalPredicate(), llvm::FCmpInst::isCommutative(), llvm::PHINode::isComplete(), llvm::SSAUpdaterImpl< UpdaterT >::IsDefInDomFrontier(), llvm::FCmpInst::isEquality(), isICmpTrue(), isImpliedCondCommonOperandWithCR(), isImpliedCondOperands(), isImpliedToBeAPowerOfTwoFromCond(), isIncrementOrDecrement(), isIndirectBrTarget(), isKnownNonNullFromDominatingCondition(), isKnownNonZeroFromAssume(), isKnownNonZeroFromOperator(), isKnownPredicateExtendIdiom(), IsKnownPredicateViaAddRecMonotonicity(), IsKnownPredicateViaAddRecStart(), IsKnownPredicateViaMinOrMax(), isLoopEntryGuardedByCond(), isLoopNeverExecuted(), isOnlyReachableViaThisEdge(), llvm::SUnit::isPred(), isRightShiftBitTestKnownToTerminate(), isSafeDecreasingBound(), isSafeIncreasingBound(), isSameCompare(), llvm::CmpInst::isSigned(), isSignificantBitCheckWellFormed(), isSignTest(), isSingleUnscheduledPred(), isTruePredicate(), isUniformlyReached(), llvm::CmpInst::isUnsigned(), jumpTableHasOtherUses(), llvm::LegalizeRuleSet::legalFor(), llvm::LegalizeRuleSet::legalFor(), llvm::LegalizeRuleSet::legalFor(), llvm::LegalizeRuleSet::legalForTypesWithMemDesc(), legalizeFCmpPredicate(), llvm::LegalizeRuleSet::libcallFor(), llvm::LegalizeRuleSet::libcallFor(), LLVMOrcCreateDynamicLibrarySearchGeneratorForPath(), LLVMOrcCreateDynamicLibrarySearchGeneratorForProcess(), lowerConstrainedFPCmpIntrinsic(), llvm::LegalizeRuleSet::lowerFor(), LowerPredicateLoad(), LowerSVEIntrinsicDUP(), llvm::MIPatternMatch::m_c_GFCmp(), llvm::MIPatternMatch::m_c_GICmp(), llvm::PatternMatch::m_c_ICmp(), llvm::PatternMatch::m_Cmp(), llvm::VPlanPatternMatch::m_Cmp(), llvm::PatternMatch::m_FCmp(), llvm::MIPatternMatch::m_GFCmp(), llvm::MIPatternMatch::m_GICmp(), llvm::PatternMatch::m_ICmp(), llvm::VPlanPatternMatch::m_ICmp(), llvm::PatternMatch::m_ICmpLike(), llvm::SDPatternMatch::m_Unless(), llvm::make_filter_range(), makeGreaterThan(), makeLessThan(), llvm::PatternMatch::FMaxMin_match< LHS_t, RHS_t, Pred_t >::match(), llvm::PatternMatch::ofmax_pred_ty::match(), llvm::PatternMatch::ofmin_pred_ty::match(), llvm::PatternMatch::UAddWithOverflow_match< LHS_t, RHS_t, Sum_t >::match(), llvm::PatternMatch::ufmax_pred_ty::match(), llvm::PatternMatch::ufmin_pred_ty::match(), matchAddCC(), matchAndCC(), matchClamp(), matchCondition(), matchEquivZeroRHS(), llvm::fuzzerop::SourcePred::matches(), matchFastFloatClamp(), llvm::fuzzerop::matchFirstLengthWAnyType(), llvm::fuzzerop::matchFirstType(), matchICmpOperand(), matchMinMax(), matchOrCC(), matchScalarInAggregate(), llvm::fuzzerop::matchScalarOfFirstType(), llvm::fuzzerop::matchSecondType(), matchSelectPattern(), matchShiftULTCondition(), matchSubCC(), matchUAddWithOverflowConstantEdgeCases(), matchXorCC(), memoryIsNotModifiedBetween(), mergeBlocksIntoPredecessors(), mergeReplicateRegionsIntoSuccessors(), llvm::LegalizeRuleSet::minScalar(), moveLCSSAPhis(), mustBeFiniteCountedLoop(), needsLFTR(), llvm::SDPatternMatch::Not(), llvm::SDPatternMatch::Not< Pred >::Not(), numberCtrlPredInSU(), llvm::fuzzerop::onlyType(), llvm::GenericCycleInfo< ContextT >::operator=(), llvm::GVNValueTable::operator=(), optimizeUniformIntrinsic(), llvm::MIPatternMatch::Or< Pred, Preds... >::Or(), llvm::SDPatternMatch::Or< Pred, Preds... >::Or(), performLegalizedVectorDeinterleaveCombine(), performPTestFirstCombine(), performSetccMergeZeroCombine(), performSetCCPunpkCombine(), performVectorDeinterleaveCombine(), llvm::phys_regs_and_masks(), placeSplitBlockCarefully(), llvm::StringSwitch< T, R >::Predicate(), llvm::predOps(), llvm::sandboxir::MemDGNode::print(), PromoteMVEPredVector(), llvm::SampleProfileLoaderBaseImpl< FT >::propagateThroughEdges(), reachesUncontrolledConvergentCallBeforeBlock(), llvm::VPBlockUtils::reassociateBlocks(), reassociateMinMaxWithConstants(), recordCondition(), recordConditions(), redirectAllPredecessorsTo(), reduceToAnyAllTrue(), llvm::DenseMapBase< densemap::detail::DenseMapStorage< BucketT >, KeyT, ValueT, KeyInfoT, BucketT >::remove_if(), llvm::detail::DenseSetImpl< ValueT, DenseMap< ValueT, DenseSetEmpty, ValueInfoT, DenseSetPair< ValueT > > >< ValueT, ValueInfoT >::remove_if(), llvm::MapVector< StringRef, HashData >::remove_if(), llvm::MapVector< StringRef, HashData >::remove_if(), llvm::orc::detail::WaitingOnGraph< ContainerIdT, ElementIdT >::ElementSet::remove_if(), llvm::orc::SymbolLookupSet::remove_if(), llvm::sampleprof::HashKeyMap< MapT, KeyT, ValueT, MapTArgs >::remove_if(), llvm::StringMap< EmptyStringSetTag, AllocatorTy >::remove_if(), llvm::LoopInfoBase< BasicBlock, Loop >::removeBlocksIf(), removeEmptyBlocks(), removeEmptyCleanup(), resetVRegCycle(), reuseTableCompare(), rewritePHINodeStore(), rewritePHIs(), rewritePHIsForCleanupPad(), runImpl(), runLowerI1Comparisons(), runMoveAutoInit(), llvm::ScalarEvolution::LoopGuards::ScalarEvolution, llvm::SCEVExprContains(), searchSuccBBs(), llvm::object::SectionFilter::SectionFilter(), llvm::object::SectionFilterIterator::SectionFilterIterator(), llvm::set_intersect(), llvm::set_subtract(), llvm::set_subtract(), setInfoSVEStN(), setNaNField(), llvm::VPBlockBase::setPredecessors(), llvm::VPIRFlags::setPredicate(), shouldPeelLastIteration(), shouldSplitOnPredicatedArgument(), shouldTryInjectInvariantCondition(), simplifyAndOrWithICmpEq(), simplifyBinaryIntrinsic(), simplifyCmpSelCase(), simplifyCmpSelFalseCase(), simplifyCmpSelOfMaxMin(), simplifyCmpSelTrueCase(), simplifyFCmpInst(), simplifyICmpInst(), simplifyICmpOfBools(), simplifyICmpUsingMonotonicValues(), simplifyICmpWithBinOp(), simplifyICmpWithBinOpOnLHS(), simplifyICmpWithConstant(), simplifyICmpWithIntrinsicOnLHS(), simplifyICmpWithMinMax(), simplifyICmpWithZero(), simplifyRecipe(), simplifySelectWithFCmp(), simplifySelectWithICmpCond(), simplifySuspendPoint(), simplifySwitchOfCmpIntrinsic(), simplifySwitchOnSelectUsingRanges(), simplifyUsingControlFlow(), sinkCommonCodeFromPredecessors(), llvm::fuzzerop::sizedPtrType(), sortBlocks(), llvm::fuzzerop::SourcePred::SourcePred(), llvm::VPlanPatternMatch::SpecificCmp_match< Op0_t, Op1_t, Opcodes >::SpecificCmp_match(), llvm::PatternMatch::SpecificCmpClass_match< LHS_t, RHS_t, Class, Commutable >::SpecificCmpClass_match(), speculatePHINodeLoads(), llvm::DominatorTreeBase< BlockT, false >::Split(), SplitBlockPredecessorsImpl(), llvm::GenericCycleInfo< ContextT >::splitCriticalEdge(), SplitLandingPadPredecessorsImpl(), splitLoopBound(), STATISTIC(), llvm::ArrayRef< llvm::cfg::Update< MachineBasicBlock * > >::take_until(), llvm::MutableArrayRef< char >::take_until(), llvm::ArrayRef< llvm::cfg::Update< MachineBasicBlock * > >::take_while(), llvm::MutableArrayRef< char >::take_while(), llvm::LoopInfoBase< BasicBlock, Loop >::takeChildrenIf(), threadCmpOverPHI(), threadCmpOverSelect(), llvm::SDPatternMatch::TLI_pred_match(), llvm::SDPatternMatch::TLI_pred_match< Pattern, PredFuncT >::TLI_pred_match(), tryCompressMaskProducer(), tryConvertSVEWideCompare(), trySimplifyICmpWithAdds(), tryToMergeLandingPad(), tryToMoveFreeBeforeNullTest(), llvm::MemoryPhi::unorderedDeleteIncomingIf(), UpdateAnalysisInformation(), updatePredecessors(), upgradeAMDGCNIntrinsicCall(), upgradeMaskedCompare(), upgradeX86IntrinsicCall(), upgradeX86vpcom(), validExtractValueIndex(), validInsertValueIndex(), validShuffleVectorIndex(), validThroughout(), llvm::SDPatternMatch::ValueType_match(), llvm::SDPatternMatch::ValueType_match< Pattern, PredFuncT >::ValueType_match(), llvm::GenericCycleInfo< ContextT >::verifyCycle(), llvm::VPIRFlags::VPIRFlags(), llvm::VPIRFlags::VPIRFlags(), and llvm::BlockFrequencyInfoImplBase::~BlockFrequencyInfoImplBase().

SubBits

unsigned SubBits = SubVecTy.getSizeInBits().getKnownMinValue()

Definition at line 28492 of file AArch64ISelLowering.cpp.

Referenced by extractConstantBits(), if(), if(), and llvm::KnownBits::insertBits().

Subtarget

const AArch64Subtarget& Subtarget = DAG.getSubtarget<AArch64Subtarget>()

Definition at line 28529 of file AArch64ISelLowering.cpp.

Referenced by AddCombineBUILD_VECTORToVPADDL(), AddCombineTo64bitMLAL(), AddCombineTo64BitSMLAL16(), AddCombineTo64bitUMAAL(), AddCombineToVPADD(), AddCombineVUZPToVPADDL(), addOpAccessChainReqs(), adjustDefLatency(), analyzeCallOperands(), llvm::ARMFunctionInfo::ARMFunctionInfo(), llvm::AArch64FrameLowering::assignCalleeSavedSpillSlots(), llvm::SystemZXPLINKFrameLowering::assignCalleeSavedSpillSlots(), attachMEMCPYScratchRegs(), AVRDAGToDAGISel::select< AVRISD::CALL >(), AVRDAGToDAGISel::select< ISD::LOAD >(), buildCallOperands(), canChangeToInt(), canCombineAsMaskOperation(), llvm::WebAssembly::canLowerMultivalueReturn(), llvm::WebAssembly::canLowerReturn(), canLowerToLDG(), canonicalizeBitSelect(), canonicalizeBoolMask(), canonicalizeLaneShuffleWithRepeatedOps(), canonicalizeShuffleMaskWithHorizOp(), canonicalizeShuffleVectorByLane(), canonicalizeVSelectTrueToOneUse(), llvm::MipsRegisterInfo::canRealignStack(), llvm::AArch64FrameLowering::canUseAsPrologue(), llvm::AArch64FrameLowering::canUseRedZone(), CC_AArch64_Custom_Block(), CC_AIX(), CC_MipsO32(), CC_RISCV_FastCC(), CC_RISCV_GHC(), CC_RISCV_Impl(), CC_RISCVAssign2XLen(), llvm::CC_SystemZ_I128Indirect(), combine_CC(), combine_CC(), combineAdd(), combineAddMulh(), combineAddMulParts(), combineAddOfExts(), combineAddOfExtsAlreadyExtended(), combineAddPackedByteDot(), combineADDToADDZE(), combineADDToMAT_PCREL_ADDR(), combineADDToSUB(), combineAnd(), combineAndMaskToShift(), combineAndnp(), CombineANDShift(), combineAndShuffleNot(), combineAndXorSubWithBMI(), combineArithReduction(), combineAVG(), combineAVX512SetCCToKMOV(), combineBasicSADPattern(), combineBinOpOfExtractToReduceTree(), combineBinOpToReduce(), combineBitcast(), combineBitcastToBoolVector(), combineBitcastvxi1(), combineBITREVERSE(), combineBlendOfPermutes(), combineBMILogicOp(), combineBrCond(), combineCastedMaskArithmetic(), combineCMov(), combineCMP(), combineCommutableSHUFP(), combineCompareEqual(), combineCONCAT_VECTORS(), combineConcatVectorOps(), combineConstantPoolLoads(), combineEXTEND_VECTOR_INREG(), combineEXTRACT_SUBVECTOR(), combineExtractVectorElt(), combineExtractWithShuffle(), combineExtSetcc(), combineFaddCFmul(), combineFaddFsub(), combineFAnd(), combineFAndFNotToFAndn(), combineFAndn(), combineFMA(), combineFMADDSUB(), combineFMinNumFMaxNum(), combineFMulcFCMulc(), combineFneg(), combineFOr(), combineFP16_TO_FP(), combineFP_EXTEND(), combineFP_ROUND(), combineFP_ROUND(), combineFP_TO_xINT_SAT(), combineFPToSInt(), combineFunnelShift(), combineGatherScatter(), combineHorizOpWithShuffle(), combineI8AndNotIntoI32AndNot(), combineINSERT_SUBVECTOR(), combineInt64DivRemViaFPReciprocal(), combineIntDivRem(), combineIntDivRemViaExactFPDiv(), combineLoad(), combineLogicBlendIntoConditionalNegate(), combineLogicBlendIntoPBLENDV(), combineLRINT_LLRINT(), combineM68kBrCond(), combineM68kSetCC(), combineMaskedLoad(), combineMaskedStore(), combineMinMaxToSat(), combineMOVMSK(), combineMul(), combineMulToPMADD52(), combineMulToPMADDWD(), combineMulToPMULDQ(), combineOp_VLToVWOp_VL(), combineOr(), combineOrAndToBitfieldInsert(), combineOrCmpEqZeroToCtlzSrl(), combineOrToBitfieldInsert(), combinePExtTruncate(), combinePExtWideningAddAcc(), combinePExtWideningAddSub(), combinePExtWideningMul(), combinePExtWideningSubAcc(), combinePMULDQ(), combinePMULH(), combinePTESTCC(), combineSCALAR_TO_VECTOR(), combineScalarAndWithMaskSetcc(), combineScalarMinMaxAbsStore(), combineSelect(), combineSelectAndUse(), combineSelectAndUseCommutative(), combineSelectToBinOp(), combineSetCC(), combineSetCCAtomicArith(), combineSetCCCCR(), combineSetCCEFLAGS(), combineSetCCMOVMSK(), combineSext(), combineShiftLeft(), combineShiftRightArithmetic(), combineShiftRightLogical(), combineShiftToPMULH(), combineShlAddIAdd(), combineShuffle(), combineShuffleToAddSubOrFMAddSub(), combineShuffleToFMAddSub(), combineSignExtendInReg(), combineSIntToFP(), combineStore(), combineSub(), combineSubShiftToOrcB(), combineTargetShuffle(), combineTESTP(), combineToConsecutiveLoads(), combineToExtendBoolVectorInReg(), combineToExtendBoolVectorInReg(), combineToHorizontalAddSub(), combineToVCPOP(), combineToVWMACC(), combineTruncate(), combineTruncatedArithmetic(), combineTruncateWithSat(), combineUIntToFP(), combineVdot4aAccum(), combineVECREDUCE_LOGIC(), combineVECREDUCE_MUL(), combineVectorCompare(), combineVectorHADDSUB(), combineVectorInsert(), combineVectorPack(), combineVectorShiftImm(), combineVectorShiftUniformVar(), combineVectorSizedSetCCEquality(), combineVectorSizedSetCCEquality(), combineVectorSizedSetCCEquality(), combineVEXTRACT_STORE(), combineVPDPBUSDPattern(), combineVSelectToBLENDV(), combineVTRUNCSAT(), combineX86FPLogicOp(), combineX86SetCC(), combineX86ShuffleChain(), combineX86ShuffleChainWithExtract(), combineX86ShufflesConstants(), combineX86ShufflesRecursively(), combineX86ShufflesRecursively(), combineXor(), combineXorSubCTLZ(), combineXorToBitfieldInsert(), combineZext(), commuteSelect(), compressShuffleOfShuffles(), computeBytesPoppedByCalleeForSRet(), llvm::mir2vec::MIREmbedder::computeEmbeddings(), llvm::ConstantMaterializationCost(), convertFPToInt(), convertFromScalableVector(), convertIntLogicToFPLogic(), convertIntToFP(), convertLocVTToValVT(), convertShiftLeftToScale(), convertToScalableVector(), convertValVTToLocVT(), CopyToFromAsymmetricReg(), llvm::createAArch64InstructionSelector(), llvm::createBPFInstructionSelector(), llvm::PPC::createFastISel(), llvm::createM68kInstructionSelector(), llvm::createMipsInstructionSelector(), createMMXBuildVector(), llvm::createPPCInstructionSelector(), createPSADBW(), llvm::createRISCVInstructionSelector(), createSetFPEnvNodes(), llvm::createSPIRVCtorDtorLoweringLegacyPass(), llvm::createSPIRVInstructionSelector(), createVariablePermute(), createVPDPBUSD(), llvm::createWebAssemblyInstructionSelector(), llvm::createX86InstructionSelector(), DecodeNEONSplat(), delayHasHazard(), detectEvenOddMultiplyOperand(), detectPMADDUBSW(), detectRoundChangeOnMachineFunction(), llvm::AArch64FrameLowering::determineCalleeSaves(), llvm::SparcFrameLowering::determineCalleeSaves(), llvm::SystemZXPLINKFrameLowering::determineCalleeSaves(), llvm::SystemZXPLINKFrameLowering::determineFrameLayout(), llvm::mca::DispatchStage::DispatchStage(), llvm::BPFAsmPrinter::doFinalization(), llvm::AArch64FrameLowering::eliminateCallFramePseudoInstr(), llvm::PPCRegisterInfo::eliminateFrameIndex(), llvm::SparcRegisterInfo::eliminateFrameIndex(), llvm::VERegisterInfo::eliminateFrameIndex(), EltsFromConsecutiveLoads(), emitAtomicCmpSwapHardware(), emitAtomicCmpSwapSoftware(), EmitAVX512Test(), emitBuildPairF64Pseudo(), emitBuildPairF64Pseudo(), EmitCmp(), emitConjunctionForCCMP(), emitConjunctionForCCMPRec(), emitConstantSizeRepmov(), emitConstantSizeRepstos(), llvm::SystemZXPLINKFrameLowering::emitEpilogue(), emitFROUND(), llvm::SystemZXPLINKAsmPrinter::emitInstruction(), emitLockedStackOp(), EmitLoweredCascadedSelect(), emitNop(), llvm::AArch64FrameLowering::emitPacRetPlusLeafHardening(), llvm::RISCVFrameLowering::emitPrologue(), llvm::SparcFrameLowering::emitPrologue(), llvm::SystemZXPLINKFrameLowering::emitPrologue(), emitPseudoCTPOP(), emitPseudoVMSKCOND(), emitPseudoXVINSGR2VR(), emitQuietFCMP(), emitRepmovs(), emitRepmovsB(), emitRepstos(), emitRepstosB(), emitSelect(), emitSelectPseudo(), emitSelectPseudo(), llvm::ARMSelectionDAGInfo::EmitSpecializedLibcall(), emitSplitF64Pseudo(), emitSplitPairF64Pseudo(), emitStackProbeInline(), EmitTailCallStoreFPAndRetAddr(), llvm::ARMSelectionDAGInfo::EmitTargetCodeForMemcpy(), llvm::X86SelectionDAGInfo::EmitTargetCodeForMemcpy(), llvm::SystemZSelectionDAGInfo::EmitTargetCodeForMemmove(), llvm::ARMSelectionDAGInfo::EmitTargetCodeForMemset(), llvm::RISCVSelectionDAGInfo::EmitTargetCodeForMemset(), llvm::X86SelectionDAGInfo::EmitTargetCodeForMemset(), EmitTest(), emitX86Nops(), enableMultiVectorSpillFill(), expandFP_TO_UINT_SSE(), expandIntrinsicWChainHelper(), ExpandMOVImmSExti8(), expandMul(), expandXorFP(), ExtractBitFromMaskVector(), fillSubVectorFromBuildVector(), fillVector(), findDelayInstr(), findMoreOptimalIndexType(), llvm::RISCVFrameLowering::findScratchNonCalleeSaveRegister(), finishStackBlock(), fixAllFDIVSQRTInMachineFunction(), fixupShuffleMaskForPermutedSToV(), foldBinOpIntoSelectIfProfitable(), foldBinOpIntoSelectIfProfitable(), foldMaskedShiftToBEXTR(), foldMaskedShiftToUSHL(), foldReduceOperandViaVDOT4A(), foldSelectToUSATI(), foldVectorXorShiftIntoCmp(), foldVectorXorShiftIntoCmp(), foldVMskZeroTest(), GenerateFixedLengthSVETBL(), generateSToVPermutedForVecShuffle(), get64BitArgumentGPRs(), get64BitArgumentXMMs(), getAVX2GatherNode(), getAVX512Node(), getAVX512TruncNode(), llvm::PPCRegisterInfo::getBaseRegister(), GetBranchTargetEnforcement(), llvm::LoongArchRegisterInfo::getCalleeSavedRegs(), llvm::MipsRegisterInfo::getCalleeSavedRegs(), llvm::PPCRegisterInfo::getCalleeSavedRegs(), llvm::RISCVRegisterInfo::getCalleeSavedRegs(), llvm::SystemZELFRegisters::getCalleeSavedRegs(), llvm::SystemZRegisterInfo::getCalleeSavedRegs(), llvm::SystemZXPLINK64Registers::getCalleeSavedRegs(), llvm::X86RegisterInfo::getCalleeSavedRegs(), getCallOpcode(), llvm::LoongArchRegisterInfo::getCallPreservedMask(), llvm::MipsRegisterInfo::getCallPreservedMask(), llvm::PPCRegisterInfo::getCallPreservedMask(), llvm::RISCVRegisterInfo::getCallPreservedMask(), llvm::SystemZELFRegisters::getCallPreservedMask(), llvm::SystemZRegisterInfo::getCallPreservedMask(), llvm::SystemZXPLINK64Registers::getCallPreservedMask(), llvm::X86RegisterInfo::getCallPreservedMask(), getCodeModel(), getContainerForFixedLengthVector(), getDataClassTest(), getDefaultScalableVLOps(), getDefaultVLOps(), getDefaultVLOps(), getEstimateRefinementSteps(), getEstimateRefinementSteps(), getFRAMEADDR(), llvm::SparcFrameLowering::getFrameIndexReference(), llvm::MipsRegisterInfo::getFrameRegister(), llvm::SystemZRegisterInfo::getFrameRegister(), getGatherNode(), getGlobalMemoryFPAtomicLegality(), getIndirectThunkSymbol(), llvm::RISCVRegisterBankInfo::getInstrMapping(), getLabelAccessInfo(), llvm::PPCRegisterInfo::getLargestLegalSuperClass(), llvm::X86RegisterInfo::getLargestLegalSuperClass(), llvm::getLibcallLowering(), llvm::LibcallLoweringInfoWrapper::getLibcallLowering(), llvm::WebAssembly::getLibcallSignature(), llvm::WebAssembly::getLibcallSignature(), getLoadStoreOffsetSizeInBits(), getLowSubvectorMaskCompare(), getMaskNode(), getMClassRegisterMask(), llvm::AMDGPU::getMCOpcode(), getMOVMSK(), GetNEONSplatValue(), getNullFPConstForNullVal(), getOptimalRepType(), llvm::WebAssembly::getOrCreateFuncrefCallTableSymbol(), llvm::WebAssembly::getOrCreateFunctionTableSymbol(), getPack(), getPACKOpcode(), getPredicateForSetCC(), getPrefetchNode(), llvm::X86RegisterInfo::getPtrSizedFrameRegister(), llvm::X86RegisterInfo::getPtrSizedStackRegister(), getPTrue(), getReadTimeStampCounter(), llvm::RISCVRegisterInfo::getRegAllocationHints(), llvm::SystemZRegisterInfo::getRegAllocationHints(), llvm::PPCRegisterInfo::getRegPressureLimit(), llvm::SystemZELFFrameLowering::getRegSpillOffset(), llvm::MipsRegisterInfo::getReservedRegs(), llvm::PPCRegisterInfo::getReservedRegs(), llvm::RISCVRegisterInfo::getReservedRegs(), llvm::SparcRegisterInfo::getReservedRegs(), llvm::SystemZRegisterInfo::getReservedRegs(), getRetOpcode(), getRVVFPReductionOpAndOperands(), getScalarMaskingNode(), getScatterNode(), getShuffleVectorZeroOrUndef(), getSmallestVTForIndex(), llvm::MipsCCState::getSpecialCallingConvForCallee(), getSToVPermuted(), GetSymbolRef(), getTargetVShiftNode(), GetTLSADDR(), llvm::WebAssembly::getTLSBase(), getv64i1Argument(), getVCIXISDNodeWCHAIN(), getVectorCompareInfo(), getVectorMaskingNode(), llvm::MIR2VecVocabProvider::getVocabulary(), getVSlidedown(), getVSlideup(), getWideningInterleave(), getZeroVector(), handleMaskRegisterForCallingConv(), hasCalleePopSRet(), llvm::AArch64FrameLowering::hasFPImpl(), hasFPType(), llvm::HasLowerConstantMaterializationCost(), hasPartialRegUpdate(), hasSVEMultiVectorOps(), if(), if(), INITIALIZE_PASS(), INITIALIZE_PASS(), INITIALIZE_PASS(), INITIALIZE_PASS(), llvm::SelectionDAGISel::initializeAnalysisResults(), llvm::SelectionDAGISel::initializeAnalysisResults(), insert1BitVector(), InsertBitToMaskVector(), insertDefsUses(), insertNOPLoadInMachineFunction(), insertSEH(), InvertCarryFlag(), isAddSubOrSubAdd(), isBF16orSoftF16(), isCachePolicySupported(), llvm::PPCRegisterInfo::isCallerPreservedPhysReg(), isClusterableLdStOpcPair(), isEligibleForSmallVectorLoadOpt(), isEligibleToFoldADDIForFasterLocalAccesses(), llvm::X86::isExtendedSwiftAsyncFrameSupported(), isFMAddSubOrFMSubAdd(), isHorizontalBinOp(), isIndirectCall(), isInterleaveShuffle(), isL1EvictionSupported(), isL2EvictionSupported(), isL2PrefetchSupported(), isLegalAddressImmediate(), isLegalBitRotate(), isLegalConversion(), isLegalElementTypeForRVV(), isLegalLogicalImmediate(), isLegalMaskCompare(), isLegalT2AddressImmediate(), isLegalVTForZvzipDeinterleavedOperand(), isLegalVTForZvzipInterleavedOperand(), isMaskableNode(), isNVCastToHalfWidthElements(), isPromotedOpNeedingSplit(), IsRegInSet(), isRegRegScaleLoadOrStore(), isSoftF16(), isSupportedArgumentType(), isSupportedReciprocalEstimateType(), isSupportedReturnType(), isThreadPointerAcquisitionNode(), isTOCSaveRestoreRequired(), isValidEGW(), isValidSplatLoad(), llvm::SystemZVectorConstantInfo::isVectorConstantLegal(), llvm::PPC::isVPKUDUMShuffleMask(), isWorthFoldingIntoRegRegScale(), isXPLeafCandidate(), llvm::LanaiFrameLowering::LanaiFrameLowering(), legalizeIntrinsicImmArg(), lower128BitShuffle(), lower128BitShuffle(), lower1BitShuffle(), lower1BitShuffleAsKSHIFTR(), lower256BitShuffle(), lower256BitShuffle(), lower512BitShuffle(), LowerABD(), LowerABS(), llvm::PPCRegisterInfo::lowerACCRestore(), llvm::PPCRegisterInfo::lowerACCSpilling(), LowerADDSAT_SUBSAT(), lowerAddSub(), LowerADDSUBSAT(), lowerAddSubToHorizontalOp(), LowerANY_EXTEND(), LowerATOMIC_FENCE(), LowerATOMIC_FENCE(), LowerATOMIC_FENCE(), LowerATOMIC_STORE(), lowerAtomicArith(), lowerAtomicArithWithLOCK(), LowerAVG(), LowerAVXCONCAT_VECTORS(), LowerAVXExtend(), LowerBITCAST(), LowerBITREVERSE(), lowerBitreverseShuffle(), lowerBUILD_VECTOR(), LowerBUILD_VECTORAsVariablePermute(), LowerBUILD_VECTORvXbf16(), LowerBUILD_VECTORvXi1(), lowerBuildVectorAsBlend(), lowerBuildVectorAsBroadcast(), LowerBuildVectorAsInsert(), lowerBuildVectorOfConstants(), lowerBuildVectorToBitOp(), LowerBuildVectorv16i8(), LowerBuildVectorv4x32(), LowerBuildVectorv8i16(), lowerBuildVectorViaDominantValues(), lowerBuildVectorViaPacking(), lowerBuildVectorViaVID(), llvm::AArch64CallLowering::lowerCall(), llvm::RISCVCallLowering::lowerCall(), LowerCallResults(), LowerCLMUL(), LowerCMP_SWAP(), LowerCONCAT_VECTORS(), LowerCONCAT_VECTORSvXi1(), lowerConstant(), lowerCONVERT_FROM_ARBITRARY_FP(), llvm::PPCRegisterInfo::lowerCRBitRestore(), llvm::PPCRegisterInfo::lowerCRBitSpilling(), llvm::PPCRegisterInfo::lowerCRRestore(), llvm::PPCRegisterInfo::lowerCRSpilling(), LowerCTLZ(), LowerCTPOP(), LowerCTTZ(), lowerCttzElts(), lowerDisjointIndicesShuffle(), llvm::PPCRegisterInfo::lowerDMRRestore(), llvm::PPCRegisterInfo::lowerDMRSpilling(), LowerDYNAMIC_STACKALLOC(), llvm::PPCRegisterInfo::lowerDynamicAlloc(), llvm::PPCRegisterInfo::lowerDynamicAreaOffset(), LowerEXTEND_VECTOR_INREG(), LowerEXTRACT_SUBVECTOR(), lowerFABSorFNEG(), lowerFCOPYSIGN(), lowerFixedVectorSegLoadIntrinsics(), lowerFixedVectorSegStoreIntrinsics(), LowerFLDEXP(), lowerFMAXIMUM_FMINIMUM(), LowerFMINIMUM_FMAXIMUM(), llvm::AArch64CallLowering::lowerFormalArguments(), llvm::AMDGPUCallLowering::lowerFormalArguments(), llvm::ARMCallLowering::lowerFormalArguments(), llvm::RISCVCallLowering::lowerFormalArguments(), llvm::AMDGPUCallLowering::lowerFormalArgumentsKernel(), lowerFP_TO_INT(), LowerFP_TO_INT_SAT(), lowerFP_TO_INT_SAT(), lowerFPToIntToFP(), LowerFRAMEADDR(), lowerFRAMEADDR(), lowerFTRUNC_FCEIL_FFLOOR_FROUND(), LowerFunnelShift(), lowerGetVectorLength(), LowerHorizontalByteSum(), LowerI64IntToFP16(), LowerI64IntToFP_AVX512DQ(), LowerINSERT_SUBVECTOR(), llvm::AVRMCInstLower::lowerInstruction(), lowerINT_TO_FP(), lowerINT_TO_FP_vXi64(), LowerINTRINSIC_W_CHAIN(), LowerLoad(), LowerMGATHER(), LowerMINMAX(), LowerMINMAX_REDUCE(), LowerMLOAD(), lowerMSALoadIntr(), lowerMSAStoreIntr(), LowerMSCATTER(), LowerMSTORE(), LowerMUL(), LowerMULH(), LowerMULO(), llvm::PPCRegisterInfo::lowerOctWordSpilling(), LowerPARITY(), LowerPARTIAL_REDUCE_FMLA(), LowerPARTIAL_REDUCE_MLA(), LowerPDEPPEXT(), LowerPREFETCH(), LowerPREFETCH(), LowerPREFETCH(), llvm::PPCRegisterInfo::lowerPrepareProbedAlloca(), lowerPZExt(), llvm::PPCRegisterInfo::lowerQuadwordRestore(), llvm::PPCRegisterInfo::lowerQuadwordSpilling(), LowerREADCYCLECOUNTER(), lowerReductionSeq(), llvm::AArch64CallLowering::lowerReturn(), llvm::RISCVCallLowering::lowerReturn(), LowerRETURNADDR(), lowerRETURNADDR(), LowerRotate(), LowerSCALAR_TO_VECTOR(), lowerScalarInsert(), lowerScalarSplat(), lowerSelectToBinOp(), LowerSELECTWithCmpZero(), LowerShift(), LowerShiftByScalarImmediate(), LowerShiftByScalarVariable(), lowerShuffleAsBitRotate(), lowerShuffleAsBlend(), lowerShuffleAsBroadcast(), lowerShuffleAsByteRotate(), lowerShuffleAsByteRotateAndPermute(), lowerShuffleAsByteShiftMask(), lowerShuffleAsDecomposedShuffleMerge(), lowerShuffleAsElementInsertion(), lowerShuffleAsLanePermuteAndPermute(), lowerShuffleAsLanePermuteAndRepeatedMask(), lowerShuffleAsLanePermuteAndShuffle(), lowerShuffleAsPermuteAndUnpack(), lowerShuffleAsRepeatedMaskAndLanePermute(), lowerShuffleAsShift(), lowerShuffleAsSpecificExtension(), lowerShuffleAsSplitOrBlend(), lowerShuffleAsTruncBroadcast(), lowerShuffleAsVALIGN(), lowerShuffleAsVTRUNC(), lowerShuffleAsZeroOrAnyExtend(), lowerShuffleViaVRegSplitting(), lowerShuffleWithEXPAND(), lowerShuffleWithPACK(), lowerShuffleWithPERMV(), lowerShuffleWithPSHUFB(), lowerShuffleWithSHUFPD(), lowerShuffleWithUndefHalf(), lowerShuffleWithVPMOV(), LowerSIGN_EXTEND(), LowerSIGN_EXTEND_Mask(), LowerSTACKADDRESS(), LowerStore(), llvm::AVRMCInstLower::lowerSymbolOperand(), LowerTruncate(), LowerTruncateVecI1(), LowerTruncateVecPack(), LowerTruncateVecPackWithSignBits(), LowerUINT_TO_FP_i32(), LowerUINT_TO_FP_i64(), lowerUINT_TO_FP_v2i32(), lowerUINT_TO_FP_vec(), lowerUINT_TO_FP_vXi32(), lowerV16F32Shuffle(), lowerV16I16Shuffle(), lowerV16I32Shuffle(), lowerV16I8Shuffle(), lowerV2F64Shuffle(), lowerV2I64Shuffle(), lowerV2X128Shuffle(), lowerV32I16Shuffle(), lowerV32I8Shuffle(), lowerV4F32Shuffle(), lowerV4F64Shuffle(), lowerV4I32Shuffle(), lowerV4I64Shuffle(), lowerV4X128Shuffle(), lowerV64I8Shuffle(), lowerV8F16Shuffle(), lowerV8F32Shuffle(), lowerV8F64Shuffle(), lowerV8I16GeneralSingleInputShuffle(), lowerV8I16Shuffle(), lowerV8I32Shuffle(), lowerV8I64Shuffle(), LowerVACOPY(), LowerVECREDUCE(), lowerVECTOR_COMPRESS(), lowerVECTOR_SHUFFLE(), lowerVECTOR_SHUFFLE_IsReverse(), lowerVECTOR_SHUFFLE_VEXTRINS(), lowerVECTOR_SHUFFLE_VPERMI(), lowerVECTOR_SHUFFLE_VREPLVEI(), lowerVECTOR_SHUFFLE_VSHUF(), lowerVECTOR_SHUFFLE_VSHUF4I(), lowerVECTOR_SHUFFLE_XVEXTRINS(), lowerVECTOR_SHUFFLE_XVINSVE0(), lowerVECTOR_SHUFFLE_XVPERM(), lowerVECTOR_SHUFFLE_XVPERMI(), lowerVECTOR_SHUFFLE_XVPERMIQ(), lowerVECTOR_SHUFFLE_XVREPLVEI(), lowerVECTOR_SHUFFLE_XVSHUF4I(), lowerVECTOR_SHUFFLEAsByteRotate(), lowerVECTOR_SHUFFLEAsPSlide1(), lowerVECTOR_SHUFFLEAsPZip(), lowerVECTOR_SHUFFLEAsRotate(), lowerVECTOR_SHUFFLEAsRV32PNarrowingShift(), lowerVECTOR_SHUFFLEAsShift(), lowerVECTOR_SHUFFLEAsVRGatherVX(), lowerVECTOR_SHUFFLEAsVSlide1(), lowerVECTOR_SHUFFLEAsVSlidedown(), lowerVECTOR_SHUFFLEAsVSlideup(), LowerVectorAllEqual(), LowerVectorCTLZ(), LowerVectorCTLZ_AVX512CDI(), LowerVectorCTLZInRegLUT(), LowerVectorCTPOP(), LowerVectorCTPOPInRegLUT(), LowerVectorExtend(), lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND(), lowerVectorIntrinsicScalars(), lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND(), lowerVectorXRINT_XROUND(), lowerVSELECTtoVectorShuffle(), LowerVSETCC(), LowerVSETCCWithSUBUS(), LowervXi8MulWithUNPCK(), llvm::PPCRegisterInfo::lowerWACCRestore(), llvm::PPCRegisterInfo::lowerWACCSpilling(), LowerZERO_EXTEND(), LowerZERO_EXTEND_Mask(), lowerZvzipVPAIR(), lowerZvzipVUNZIP(), lowerZvzipVZIP(), matchBinaryPermuteShuffle(), matchBinaryShuffle(), matchCSET(), matchPMADDWD(), matchPMADDWD_2(), matchShuffleAsBitRotate(), matchShuffleAsShift(), matchShuffleAsVTRUNC(), matchShuffleWithPACK(), matchShuffleWithUNPCK(), matchSplatAsGather(), matchTruncateWithPACK(), matchUnaryPermuteShuffle(), matchUnaryShuffle(), MatchVectorAllEqualTest(), matchVPMADD52(), llvm::PPCRegisterInfo::materializeFrameBaseRegister(), materializeVectorConstant(), mayFoldIntoVector(), llvm::X86::mayFoldLoad(), llvm::X86::mayFoldLoadIntoBroadcastFromMem(), mustAvoidNeonAtMBBI(), narrowBitOpRMW(), narrowVectorSelect(), llvm::RISCVRegisterInfo::needsFrameBaseReg(), needStackSlotPassParameters(), optimizeLogicalImm(), optimizeNZCVDefs(), Passv64i1ArgInRegs(), peephole(), perfomPextFirstTrueVectorCombine(), PerformADDCombine(), performADDCombine(), performADDCombine(), performADDCombine(), PerformADDCombineWithOperands(), PerformAddcSubcCombine(), PerformADDECombine(), PerformAddeSubeCombine(), PerformADDVecReduce(), PerformANDCombine(), performANDCombine(), performANDCombine(), performANDCombine(), performANDCombine(), performBITCASTCombine(), performBITREV_WCombine(), performBITREVERSECombine(), performBR_CCCombine(), performBSPExpandForSVE(), PerformBUILD_VECTORCombine(), performBUILD_VECTORCombine(), performCMovFPCombine(), performCONCAT_VECTORSCombine(), performCONCAT_VECTORSCombine(), performCTLZCombine(), performDivRemCombine(), performDSPShiftCombine(), performDUPCombine(), performEXTENDCombine(), performExtendCombine(), performExtendToBoolVectorLoadCombine(), performExtractLastActiveCombine(), performExtractVectorEltCombine(), PerformFADDCombine(), PerformFAddVSelectCombine(), performFirstTrueTestVectorCombine(), performFP_TO_INT_SATCombine(), performFP_TO_INTCombine(), performFP_TO_INTCombine(), performFPExtendCombine(), performFpToIntCombine(), performGlobalAddressCombine(), performHorizWideningCombine(), performINSERT_VECTOR_ELTCombine(), PerformInterleavedStoreCombine(), performINTRINSIC_WO_CHAINCombine(), performIntrinsicCombine(), performIntToFpCombine(), performLastTrueTestVectorCombine(), PerformLegalizedInterleavedStoreCombine(), performLegalizedInterleavedStoreCombine(), PerformLegalizedVECTOR_DEINTERLEAVECombine(), performLegalizedVectorDeinterleaveCombine(), PerformLOADCombine(), performMADD_MSUBCombine(), performMemPairCombine(), PerformMinMaxToSatCombine(), performMOVFR2GR_SCombine(), performMOVGR2FR_WCombine(), performMSTORECombine(), PerformMULCombine(), performMULCombine(), performMULCombine(), performMulCombine(), PerformMVEVMULLCombine(), performNVCASTCombine(), PerformORCombine(), performORCombine(), performORCombine(), performORCombine(), performORCombine(), PerformORCombine_i1(), PerformORCombineToBFI(), PerformORCombineToSMULWBT(), performP_BITCASTCombine(), performPredicateLoadCombine(), performReverseEVLCombine(), performSELECT_CCCombine(), PerformSELECTCombine(), performSELECTCombine(), performSELECTCombine(), performSELECTCombine(), performSETCC_BITCASTCombine(), performSETCCCombine(), performSETCCCombine(), performSETCCCombine(), performSHLCombine(), performSHLCombine(), performSHLCombine(), performSHLCombine(), performSIGN_EXTEND_INREGCombine(), performSignExtendCombine(), performSINT_TO_FPCombine(), performSPLIT_PAIR_F64Combine(), performSRACombine(), performSRACombine(), performSRLCombine(), performSRLCombine(), performSRLCombine(), PerformSTORECombine(), PerformSUBCombine(), performSUBCombine(), performSUBCombine(), performSUBCombine(), performTRUNCATECombine(), performUINT_TO_FPCombine(), PerformUMLALCombine(), performUnpackCombine(), performUzpCombine(), performVANDNCombine(), PerformVCMPCombine(), PerformVCVTCombine(), PerformVDUPCombine(), PerformVDUPLANECombine(), performVECREDUCECombine(), PerformVECTOR_DEINTERLEAVECombine(), performVECTOR_SHUFFLECombine(), performVectorDeinterleaveCombine(), performVectorExtendToFPCombine(), performVEXT_VLCombine(), performVFMADD_VLCombine(), PerformVMOVRRDCombine(), PerformVMULCombine(), PerformVMulVCTPCombine(), performVP_STORECombine(), PerformVSELECTCombine(), performVSELECTCombine(), performVSELECTCombine(), performVSelectCombine(), performVSelectMaskedShiftCombine(), PerformVSetCCToVCTPCombine(), performVSlideUpDownCombine(), performVWABDACombine(), performVWABDACombineWV(), performVWADDSUBW_VLCombine(), PerformXORCombine(), performXORCombine(), performXORCombine(), performXorCombine(), prepareDescriptorIndirectCall(), llvm::PPCRegisterInfo::prepareDynamicAlloca(), prepareOutOfLineGlueCall(), llvm::SystemZXPLINKFrameLowering::processFunctionBeforeFrameFinalized(), processVCIXOperands(), produceCompactUnwindFrame(), promoteExtBeforeAdd(), PromoteMaskArithmetic(), PromoteMaskArithmetic(), PromoteMaskArithmetic(), PromoteMaskArithmetic(), promoteVCIXScalar(), pushAddIntoCmovOfConsts(), recoverFramePointer(), reduceMaskedLoadToScalarLoad(), reduceMaskedStoreToScalarStore(), reduceVMULWidth(), ReplaceAddWithADDP(), ReplaceATOMIC_LOAD_128Results(), ReplaceCMP_SWAP_128Results(), replaceINTRINSIC_WO_CHAINResults(), ReplaceREADCYCLECOUNTER(), replaceVecCondBranchResults(), replaceVPICKVE2GRResults(), llvm::ARMFrameLowering::requiresAAPCSFrameRecord(), llvm::PPCRegisterInfo::requiresFrameIndexScavenging(), llvm::PPCRegisterInfo::requiresVirtualBaseRegisters(), llvm::AArch64FrameLowering::resetCFIToInitialState(), llvm::PPCRegisterInfo::resolveFrameIndex(), llvm::AArch64FrameLowering::resolveFrameOffsetReference(), llvm::AArch64FrameLowering::restoreCalleeSavedRegisters(), llvm::SystemZXPLINKFrameLowering::restoreCalleeSavedRegisters(), reverseZExtICmpCombine(), rmwOpMayLowerToLibcall(), llvm::DwarfEHPreparePass::run(), llvm::IRTranslatorPass::run(), llvm::LegalizerPass::run(), llvm::SafeStackPass::run(), llvm::WebAssemblyMemIntrinsicResultsPass::run(), llvm::WebAssemblyPeepholePass::run(), runIndirectBranchTracking(), runLegalizerOnMachineFunction(), runLVLGenOnMachineFunction(), runMipsSizeReduce(), llvm::StackProtector::runOnFunction(), runOnMachineBasicBlock(), llvm::IRTranslatorImpl::runOnMachineFunction(), llvm::IRTranslatorLegacy::runOnMachineFunction(), llvm::LegalizerLegacy::runOnMachineFunction(), runX86LoadValueInjectionRetHardening(), runX86SpeculativeExecutionSideEffectSuppression(), scalarizeExtEltFP(), selectConstantAddr(), selectI64Imm(), selectImm(), selectSOffset(), selectVSplatImmHelper(), shouldGenerateInlineTPLoop(), shouldTransformMulToShiftsAddsSubs(), shouldUseHorizontalOp(), llvm::AArch64FrameLowering::spillCalleeSavedRegisters(), llvm::SystemZXPLINKFrameLowering::spillCalleeSavedRegisters(), SplitOpsAndApply(), splitStores(), STATISTIC(), supportedVectorShiftWithBaseAmnt(), supportedVectorShiftWithImm(), supportedVectorVarShift(), supportsMin3Max3(), transformAddImmMulImm(), transformAddShlImm(), transformCallee(), translateSetCCForBranch(), truncateAVX512SetCCNoBWI(), truncateVectorWithPACK(), truncateVectorWithPACKSS(), truncateVectorWithPACKUS(), tryCombineFADDReductionWithZero(), tryCombineRestoreWithPrevInst(), llvm::tryFoldSPUpdateIntoPushPop(), tryLowerSmallVectorExtLoad(), tryLowerToBSL(), tryMemPairCombine(), trySimplifySrlAddToRshrnb(), trySQABSCombine(), tryToReplaceScalarFPConversionWithSVE(), unpackF64OnRV32DSoftABI(), unpackFromRegLoc(), useEGPRInlineAsm(), useInversedSetcc(), llvm::SystemZELFFrameLowering::usePackedStack(), useRVVForFixedLengthVectorVT(), useVectorCast(), useVPTERNLOG(), vectorizeExtractedCast(), llvm::X86SelectionDAGInfo::verifyTargetNode(), widenBuildVector(), widenMaskVector(), widenMaskVectorType(), widenSubVector(), widenSubVector(), and X86ChooseCmpOpcode().

SubVecTy

EVT SubVecTy = ValueInterleaveOps[0].getValueType()

Definition at line 28487 of file AArch64ISelLowering.cpp.

Referenced by llvm::LegalizerHelper::bitcastInsertSubvector(), llvm::MachineIRBuilder::buildInstr(), slpvectorizer::BoUpSLP::canBuildSplitNode(), slpvectorizer::BoUpSLP::canVectorizeLoads(), llvm::AArch64TTIImpl::getInterleavedMemoryOpCost(), llvm::ARMTTIImpl::getInterleavedMemoryOpCost(), llvm::RISCVTTIImpl::getInterleavedMemoryOpCost(), llvm::WebAssemblyTTIImpl::getInterleavedMemoryOpCost(), if(), if(), if(), llvm::AArch64TargetLowering::lowerInterleavedStore(), llvm::ARMTargetLowering::lowerInterleavedStore(), PerformInterleavedStoreCombine(), PerformLegalizedInterleavedStoreCombine(), performLegalizedInterleavedStoreCombine(), PerformLegalizedVECTOR_DEINTERLEAVECombine(), performLegalizedVectorDeinterleaveCombine(), PerformVECTOR_DEINTERLEAVECombine(), performVectorDeinterleaveCombine(), promoteAllocaUserToVector(), llvm::InstCombinerImpl::visitCallInst(), and llvm::LegalizerHelper::widenScalar().

SVEStores

Intrinsic::ID SVEStores[]

staticconstexpr

Initial value:

= {Intrinsic::aarch64_sve_st2,
                                                Intrinsic::aarch64_sve_st3,
                                                Intrinsic::aarch64_sve_st4}

Definition at line 28566 of file AArch64ISelLowering.cpp.

Referenced by append(), getStructuredStoreFunction(), and performLegalizedInterleavedStoreCombine().

TLI

const TargetLowering& TLI = DAG.getTargetLoweringInfo()

Examples

/work/as-worker-4/publish-doxygen-docs/llvm-project/llvm/include/llvm/Transforms/Utils/Local.h.

Definition at line 28488 of file AArch64ISelLowering.cpp.

Referenced by addBoundsChecking(), AddCombineBUILD_VECTORToVPADDL(), AddCombineToVPADD(), AddCombineVUZPToVPADDL(), llvm::RegsForValue::AddInlineAsmOperands(), addMappingsFromTLI(), addUsedSymbolToPreservedGUID(), adjustCostForPairing(), adjustInliningThresholdUsingCallee(), AlignBlocks(), alignReassociatedOperandsByKey(), allocateRVVReg(), analyzeCallOperands(), annotateAllFunctions(), llvm::slpvectorizer::areCompatibleCmpOps(), llvm::slpvectorizer::arePointersCompatible(), atomicSizeSupported(), llvm::attributesPermitTailCall(), llvm::BranchProbabilityInfo::BranchProbabilityInfo(), llvm::MachineIRBuilder::buildBoolExtInReg(), BuildExactSDIV(), BuildExactUDIV(), llvm::MachineIRBuilder::buildExtractVectorElementConstant(), buildFromShuffleMostly(), llvm::BranchProbabilityInfo::calculate(), llvm::calculateDbgEntityHistory(), llvm::callsGCLeafFunction(), llvm::CombinerHelper::canCombineFMadOrFMA(), canCombineShuffleToExtendVectorInreg(), canConstantEvolve(), canConstantFold(), llvm::canConstantFoldCallTo(), canConvertToFMA(), canExpandVectorCTPOP(), canFoldInAddressingMode(), canFoldInAddressingMode(), canFuseFMulWithFAddSub(), llvm::canLowerMemCpyFamily(), canNarrowCLMULToLegal(), canonicalizeShuffleWithOp(), canonicalizeToIntegerIV(), llvm::AArch64FrameLowering::canUseAsPrologue(), llvm::X86FrameLowering::canUseAsPrologue(), llvm::slpvectorizer::canWidenLoadsOverUndefLanes(), CC_RISCV_FastCC(), CC_RISCV_Impl(), checkAddrSpaceIsValidForLibcall(), CheckChildType(), llvm::slpvectorizer::checkEVsForVecCalls(), CheckType(), CheckValueType(), collectMemAccessInfo(), CollectOpsToWiden(), combine_CC(), combineAddMulh(), combineAnd(), combineAndnp(), combineAndShuffleNot(), combineBitcast(), combineBitcastToBoolVector(), combineBMI(), combineCarryDiamond(), combineCMP(), combineCONCAT_VECTORS(), combineConcatVectorOfCasts(), combineConcatVectorOfExtracts(), combineConcatVectorOfScalars(), combineConcatVectorOfShuffleAndItsOperands(), combineConcatVectorOfShuffles(), combineConcatVectorOfSplats(), combineConcatVectorOps(), combineCVTPH2PS(), combineEXTEND_VECTOR_INREG(), combineEXTRACT_SUBVECTOR(), combineExtractFromVectorLoad(), combineExtractVectorElt(), combineFMA(), combineFMADDSUB(), combineFMinNumFMaxNum(), combineFneg(), combineGatherScatter(), combineINSERT_SUBVECTOR(), combineInstructionsOverFunction(), combineKSHIFT(), combineLoad(), combineLRINT_LLRINT(), combineMaskBitOp(), combineMaskedLoad(), combineMaskedStore(), combineMinNumMaxNumImpl(), combineMOVMSK(), combineOr(), combineOrOfSetCCToUSUBOCarry(), combinePCLMULQDQ(), combinePMULDQ(), combinePTESTCC(), combineScalarAndWithMaskSetcc(), combineSelect(), combineSelectAsExtAnd(), combineSelectCCToPseudoMinMax(), combineSetCC(), combineShiftAnd1ToBitTest(), combineShiftToAVG(), combineShiftToAVGSUB(), combineShiftToMULH(), combineShuffle(), combineShuffleOfBitcast(), combineShuffleOfScalars(), combineShuffleToAnyExtendVectorInreg(), combineShuffleToFMAddSub(), combineShuffleToZeroExtendVectorInReg(), combineStore(), combineTargetShuffle(), combineTESTP(), combineToVCPOP(), combineTruncatedArithmetic(), combineTruncateWithSat(), combineVECREDUCE_LOGIC(), combineVectorInsert(), combineVectorMulToSraBitcast(), combineVectorShiftImm(), combineVectorShiftUniformVar(), combineVEXTRACT_STORE(), combineVPMADD(), combineVPMADD52LH(), combineVSelectToBLENDV(), combineVSelectWithAllOnesOrZeros(), combineVTRUNC(), combineX86GatherScatter(), combineX86INT_TO_FP(), combineX86ShuffleChain(), combineX86ShufflesRecursively(), combineXor(), compareCmp(), computeConstraintToUse(), computeConstraintToUse(), llvm::computeKnownFPClass(), llvm::computeLegalValueVTs(), llvm::computeLegalValueVTs(), computePointerICmp(), ComputePTXValueVTs(), llvm::memprof::computeUndriftMap(), llvm::ComputeValueVTs(), llvm::ComputeValueVTs(), llvm::ConstantFoldCall(), llvm::ConstantFoldCompareInstOperands(), llvm::ConstantFoldConstant(), llvm::ConstantFoldInstOperands(), llvm::ConstantFoldInstruction(), llvm::ConstantFoldTerminator(), constructOperandInfo(), containsUnconditionalCallSafepoint(), llvm::AArch64PrologueEpilogueCommon::convertCalleeSaveRestoreToSPPrePostIncDec(), llvm::SelectionDAGBuilder::CopyValueToVirtualRegister(), llvm::createDefaultScheduler(), llvm::createHybridListDAGScheduler(), llvm::createILPListDAGScheduler(), llvm::ARMBaseTargetMachine::createMachineFunctionInfo(), createMMXBuildVector(), CreatePrologue(), DCEInstruction(), llvm::DeleteDeadPHIs(), despeculateCountZeros(), determineConstraints(), doCallSiteSplitting(), llvm::slpvectorizer::doesInTreeUserNeedToExtract(), earlyExpandDIVFIX(), llvm::AArch64FrameLowering::eliminateCallFramePseudoInstr(), llvm::LoongArchFrameLowering::eliminateCallFramePseudoInstr(), llvm::RISCVFrameLowering::eliminateCallFramePseudoInstr(), eliminateConstraints(), eliminateDeadCode(), eliminateDeadStores(), EltsFromConsecutiveLoads(), llvm::emitBCmp(), llvm::emitBinaryFloatFnCall(), llvm::emitBinaryFloatFnCall(), emitBinaryFloatFnCallHelper(), llvm::emitCalloc(), llvm::LoongArchTargetLowering::emitDynamicProbedAlloc(), llvm::RISCVTargetLowering::emitDynamicProbedAlloc(), llvm::emitFPutC(), llvm::emitFPutS(), llvm::emitFWrite(), llvm::WebAssemblyAsmPrinter::emitGlobalVariable(), llvm::emitHotColdNew(), llvm::emitHotColdNewAligned(), llvm::emitHotColdNewAlignedNoThrow(), llvm::emitHotColdNewNoThrow(), llvm::emitHotColdSizeReturningNew(), llvm::emitHotColdSizeReturningNewAligned(), llvm::AsmPrinter::emitJumpTableEntry(), emitLibCall(), llvm::emitMalloc(), llvm::emitMemCCpy(), llvm::emitMemChr(), llvm::emitMemCmp(), llvm::emitMemCpyChk(), llvm::emitMemPCpy(), llvm::emitMemRChr(), llvm::AArch64PrologueEmitter::emitPrologue(), llvm::ARMFrameLowering::emitPrologue(), llvm::LoongArchFrameLowering::emitPrologue(), llvm::PPCFrameLowering::emitPrologue(), llvm::RISCVFrameLowering::emitPrologue(), llvm::SystemZELFFrameLowering::emitPrologue(), llvm::emitPutChar(), llvm::emitPutS(), llvm::emitSNPrintf(), llvm::ARMSelectionDAGInfo::EmitSpecializedLibcall(), llvm::emitSPrintf(), llvm::X86FrameLowering::emitSPUpdate(), emitStackProbeInline(), emitStackProbeInline(), llvm::emitStpCpy(), llvm::emitStpNCpy(), llvm::emitStrCat(), llvm::emitStrChr(), llvm::emitStrCpy(), llvm::emitStrDup(), llvm::AArch64SelectionDAGInfo::EmitStreamingCompatibleMemLibCall(), llvm::emitStrLCat(), llvm::emitStrLCpy(), llvm::emitStrLen(), llvm::emitStrNCat(), llvm::emitStrNCmp(), llvm::emitStrNCpy(), llvm::HexagonSelectionDAGInfo::EmitTargetCodeForMemcpy(), llvm::XCoreSelectionDAGInfo::EmitTargetCodeForMemcpy(), llvm::emitUnaryFloatFnCall(), llvm::emitUnaryFloatFnCall(), emitUnaryFloatFnCallHelper(), llvm::emitVSNPrintf(), llvm::emitVSPrintf(), llvm::emitWcsLen(), EvaluateExpression(), EvaluateStaticConstructor(), ExpandBVWithShuffles(), expandDivFix(), expandExp(), expandExp2(), expandLog(), expandLog10(), expandLog2(), expandPow(), ExtendUsesToFormExtLoad(), extractBooleanFlip(), llvm::memprof::extractCallsFromIR(), FindAllMemoryUses(), FindAllMemoryUses(), FindAtExitLibFunc(), findGISelOptimalMemOpLowering(), findMemType(), finishStackBlock(), foldAndOrOfSETCC(), foldExtendedSignBitTest(), foldExtendVectorInregToExtendOfSubvector(), foldExtractSubvectorFromShuffleVector(), foldFCmpToFPClassTest(), foldFPToIntToFP(), foldICmpWithDominatingICmp(), foldLibCalls(), foldMaskedMerge(), foldMemSetZeroOrOneLength(), foldReduceOperandViaVDOT4A(), foldSelectBinOpIdentity(), foldSelectWithIdentityConstant(), foldShuffleOfConcatUndefs(), foldSqrt(), foldToMaskedStore(), foldToSaturated(), foldUnusualPatterns(), foldVSelectToSignBitSplatMask(), foldXorTruncShiftIntoCmp(), generateKeySubkey(), getAddressForMemoryInput(), llvm::getAllocAlignment(), getAllocationData(), getAllocationDataForFunction(), llvm::getAllocationFamily(), getAllocationSize(), llvm::getAllocSize(), getAsCarry(), getAssignFnsForCC(), getAssignFnsForCC(), getAVX2GatherNode(), llvm::getBaseObjectSize(), llvm::getBestSimplifyQuery(), llvm::getBestSimplifyQuery(), llvm::slpvectorizer::getBitPackCost(), llvm::MachineIRBuilder::getBoolExtOp(), getBoundsCheckCond(), llvm::AArch64RegisterInfo::getCalleeSavedRegs(), llvm::BasicTTIImplBase< BasicTTIImpl >::getCastInstrCost(), llvm::AsmPrinter::getCodeViewJumpTableInfo(), getCombineLoadStoreParts(), getConstantEvolvingPHI(), getConstantEvolvingPHIOperands(), getConstraintFromMemoryAccess(), getCopyFromParts(), getCopyFromPartsVector(), llvm::RegsForValue::getCopyFromRegs(), getCopyToParts(), getCopyToPartsVector(), llvm::RegsForValue::getCopyToRegs(), GetCostForDef(), getEpilogueLowering(), llvm::BasicTTIImplBase< BasicTTIImpl >::getEstimatedNumberOfCaseClusters(), GetExponent(), llvm::getFloatFn(), llvm::MemoryLocation::getForArgument(), llvm::MemoryLocation::getForArgument(), llvm::MemoryLocation::getForDest(), llvm::BasicTTIImplBase< BasicTTIImpl >::getFPOpCost(), llvm::getFreedOperand(), getFusedFMul(), getGatherNode(), llvm::getICmpTrueVal(), llvm::AA::getInitialValueForObj(), llvm::getInitialValueOfAllocation(), llvm::getIntrinsicForCallSite(), getIntrinsicFromLibfunc(), getIntTy(), getLegalMaskAndStepVector(), llvm::lto::LTO::getLibFuncSymbols(), getLoadStackGuard(), llvm::WebAssemblyFrameLowering::getLocalForStackObject(), GetLocation(), getMemcpyLoadsAndStores(), getMemIntrinsicSize(), getMemmoveLoadsAndStores(), getMemSetPattern16Value(), getMemsetStores(), getMemsetStringVal(), getMinMaxOpcodeForClamp(), getNoopInput(), getObjectSize(), llvm::getObjectSize(), llvm::SDNode::getOperationName(), llvm::getOrInsertLibFunc(), llvm::getOrInsertLibFunc(), llvm::getOrInsertLibFunc(), llvm::getOrInsertLibFunc(), llvm::getOrInsertLibFunc(), getOrInsertValueProfilingCall(), getPointerSize(), getPostIndexedLoadStoreOp(), getPotentialCopiesOfMemoryValue(), getPrefetchNode(), getPromotedVectorElementType(), getPTest(), getRegistersForValue(), getRegistersForValue(), llvm::GetReturnInfo(), getRuntimeCallSDValueHelper(), llvm::slpvectorizer::getSameOpcode(), getScatterNode(), getSizeTTy(), getSqrtCall(), getStackGuard(), getStateLibraryFunctionFor(), getTagSymNode(), getTM(), llvm::BasicTTIImplBase< BasicTTIImpl >::getTypeBasedIntrinsicInstrCost(), getUnfusedBinOpCost(), getUniformBase(), llvm::SelectionDAGBuilder::getValueImpl(), llvm::slpvectorizer::getVectorCallCosts(), llvm::getVectorIntrinsicIDForCall(), getVectorLibraryVariantFor(), llvm::slpvectorizer::getVectorOpCost(), getVShift(), getZeroVector(), llvm::SelectionDAGBuilder::handleDebugValue(), llvm::hasFloatFn(), hasFMAddSubForType(), hasIntrinsicVectorMapping(), hasSameExtUse(), hasSwiftErrorArg(), hasVectorLibraryVariantFor(), hasVectorMapping(), llvm::BasicTTIImplBase< BasicTTIImpl >::haveFastSqrt(), if(), llvm::inferNonMandatoryLibFuncAttrs(), llvm::inferNonMandatoryLibFuncAttrs(), initialize(), INITIALIZE_PASS(), initializeBase(), initializeLibCalls(), llvm::GenericScheduler::initPolicy(), llvm::HexagonFrameLowering::inlineStackProbe(), llvm::PPCFrameLowering::inlineStackProbe(), llvm::SystemZELFFrameLowering::inlineStackProbe(), InsertSafepointPoll(), insertSinCosCall(), InsertStackProtectors(), instrumentAllFunctions(), interpretValues(), isAddSubOrSubAdd(), llvm::isAllocationFn(), isAllocationWithHotColdVariant(), llvm::isAllocLikeFn(), isAllocSiteRemovable(), llvm::slpvectorizer::isAlternateInstruction(), llvm::slpvectorizer::isCmpSameOrSwapped(), isConsecutiveLSLoc(), llvm::isConstFalseVal(), llvm::isConstTrueVal(), isConstValidTrue(), isFoldedIntoRsqrtEstimate(), isFormingBranchFromSelectProfitable(), isFusedFMul(), llvm::isInstructionTriviallyDead(), isKnownLibFunction(), isKnownLibFunction(), isLegalToCombineMinNumMaxNum(), llvm::lto::InputFile::Symbol::isLibcall(), llvm::isLibFuncEmittable(), llvm::isLibFuncEmittable(), isLoadStoreSizeLegal(), isMainInstruction(), llvm::slpvectorizer::isMaskedLoadCompress(), llvm::slpvectorizer::isMaskedLoadCompress(), llvm::isMathLibCallNoop(), isNeverEqualToUnescapedAlloc(), isNonVectorizableInst(), isNoopBitcast(), isObjectSize(), isObjectSmallerThan(), IsOperandAMemoryOperand(), llvm::slpvectorizer::isPoorThroughputOp(), isPromotedInstructionLegal(), isPromotedOpNeedingSplit(), isRegOtherThanSPAndFP(), llvm::isRemovableAlloc(), isRemovableWrite(), isSafeToExecuteUnconditionally(), llvm::isSafeToSpeculativelyExecute(), llvm::isSafeToSpeculativelyExecute(), llvm::isSafeToSpeculativelyExecuteWithOpcode(), llvm::TargetInstrInfo::isSchedulingBoundary(), isSpecificZeroAfterMaybeRounding(), isSupportedType(), llvm::isTLIScalarize(), llvm::AArch64PrologueEpilogueCommon::isVGInstruction(), llvm::AMDGPULegalizerInfo::legalizeGlobalValue(), llvm::AMDGPULegalizerInfo::legalizeStackSave(), llvm::MachineBasicBlock::liveout_begin(), LLVMAddTargetLibraryInfo(), llvm::LoopAccessInfo::LoopAccessInfo(), llvm::WebAssemblyMCInstLower::lower(), LowerABD(), LowerADDSAT_SUBSAT(), lowerBALLOTIntrinsic(), LowerBR_CC(), lowerBuildVectorAsBroadcast(), lowerBuildVectorToBitOp(), llvm::SelectionDAGBuilder::LowerCallTo(), lowerCanLoadSpeculatively(), llvm::lowerConstantIntrinsics(), LowerCTPOP(), llvm::SelectionDAGBuilder::LowerDeoptimizeCall(), LowerF128_FPEXTEND(), LowerF128_FPROUND(), LowerFMINIMUM_FMAXIMUM(), LowerFP_TO_SINT(), LowerFP_TO_UINT(), lowerFRAMEADDR(), lowerImmediateIfPossible(), llvm::SelectionDAGBuilder::lowerInvokable(), lowerLaneOp(), lowerMSACopyIntr(), LowerMSCATTER(), LowerMULH(), LowerMULO(), llvm::lowerObjectSizeCall(), llvm::lowerObjectSizeCall(), LowerRETURNADDR(), lowerRETURNADDR(), LowerSELECT_CC(), LowerSINT_TO_FP(), LowerStore(), LowerSTRICT_FSETCC(), LowerUINT_TO_FP(), LowerVASTART(), LowerVECREDUCE(), LowerVSETCC(), lowerWaveShuffle(), matchBSwapHWordOrAndAnd(), llvm::CombinerHelper::matchCombineFAddFpExtFMulToFMadOrFMA(), llvm::CombinerHelper::matchCombineFAddFpExtFMulToFMadOrFMAAggressive(), llvm::CombinerHelper::matchCombineFSubFpExtFNegFMulToFMadOrFMA(), llvm::CombinerHelper::matchHoistLogicOpWithSameOpcodeHands(), llvm::matchLibcall(), matchMergedBFX(), llvm::CombinerHelper::matchNarrowBinopFeedingAnd(), llvm::CombinerHelper::matchNonNegZext(), llvm::CombinerHelper::matchNotCmp(), llvm::CombinerHelper::matchPtrAddImmedChain(), matchRotateSub(), llvm::CombinerHelper::matchSDivOrSRemByConst(), llvm::CombinerHelper::matchUDivOrURemByConst(), maybeLoadStoreLocation(), llvm::maybeMarkSanitizerLibraryCallNoBuiltin(), memVTFromLoadIntrData(), memVTFromLoadIntrReturn(), mergeComparisons(), narrowExtractedVectorBinOp(), narrowExtractedVectorLoad(), narrowInsertExtractVectorBinOp(), needsStatepoint(), llvm::ScheduleDAGSDNodes::newSUnit(), llvm::TargetLibraryInfo::operator=(), llvm::TargetLibraryInfo::operator=(), llvm::TargetLibraryInfoImpl::operator=(), llvm::TargetLibraryInfoImpl::operator=(), optimizeBinaryDoubleFP(), optimizeBitCast(), optimizeBranch(), optimizeDoubleFP(), OptimizeExtractBits(), OptimizeNoopCopyExpression(), optimizeUnaryDoubleFP(), patchMatchingInput(), peekThroughDemandedElts(), performAddCSelIntoCSinc(), PerformARMBUILD_VECTORCombine(), performBUILD_VECTORCombine(), performCONCAT_VECTORSCombine(), llvm::AMDGPUTargetLowering::PerformDAGCombine(), performDemandedBitsCombine(), PerformExtendCombine(), performFP_TO_INT_SATCombine(), performFP_TO_INTCombine(), performGatherLoadCombine(), performHorizWideningCombine(), performINSERT_VECTOR_ELTCombine(), performMINMAXCombine(), performMinMaxF128Combine(), performMSTORECombine(), PerformPREDICATE_CASTCombine(), performScatterStoreCombine(), PerformShiftCombine(), PerformTruncatingStoreCombine(), performUzpCombine(), performVECREDUCECombine(), PerformVECTOR_SHUFFLECombine(), performVECTOR_SHUFFLECombine(), performVectorDeinterleaveCombine(), performVectorShiftCombine(), PerformVMOVhrCombine(), PerformVMOVNCombine(), PerformVQMOVNCombine(), performVselectPowCombine(), PerformXORCombine(), performXORCombine(), pgoMemOPSizeOptImpl(), preferGPRForFPImm(), prepareDAGLevelOperands(), processPhi(), promotConstantsInFunction(), promoteConstantsInModule(), PromoteMaskArithmetic(), PromoteMaskArithmetic(), promoteToConstantPool(), readMemprof(), llvm::RecursivelyDeleteDeadPHINode(), llvm::RecursivelyDeleteTriviallyDeadInstructions(), llvm::RecursivelyDeleteTriviallyDeadInstructions(), llvm::RecursivelyDeleteTriviallyDeadInstructionsPermissive(), reduceBuildVecToShuffleWithZero(), ReduceLoopStrength(), refineIndexType(), llvm::RegsForValue::RegsForValue(), llvm::RegsForValue::RegsForValue(), llvm::replaceAndRecursivelySimplify(), replaceAndRecursivelySimplifyImpl(), llvm::LoongArchTargetLowering::ReplaceNodeResults(), replaceProxyReg(), replaceWithCallToVeclib(), llvm::returnTypeIsEligibleForTailCall(), llvm::rewriteLoopExitValues(), llvm::AddressSanitizerPass::run(), llvm::AggressiveInstCombinePass::run(), llvm::AMDGPUCodeGenPreparePass::run(), llvm::AnnotationRemarksPass::run(), llvm::BasicAA::run(), llvm::BoundsCheckingPass::run(), llvm::BranchProbabilityAnalysis::run(), llvm::CallSiteSplittingPass::run(), llvm::ComplexDeinterleavingPass::run(), llvm::ConstraintEliminationPass::run(), llvm::DSEPass::run(), llvm::EarlyCSEPass::run(), llvm::ExpandIRInstsPass::run(), llvm::ExpandMemCmpPass::run(), llvm::GVNPass::run(), llvm::HardwareLoopsPass::run(), llvm::IndirectBrExpandPass::run(), llvm::InjectTLIMappings::run(), llvm::InstCombinePass::run(), llvm::InstSimplifyPass::run(), llvm::InterleavedAccessPass::run(), llvm::LibCallsShrinkWrapPass::run(), llvm::LintPass::run(), llvm::LoopAccessAnalysis::run(), llvm::MemoryDependenceAnalysis::run(), llvm::MemProfUsePass::run(), llvm::MergeICmpsPass::run(), llvm::NewGVNPass::run(), llvm::PartiallyInlineLibCallsPass::run(), llvm::PGOMemOPSizeOpt::run(), llvm::PlaceSafepointsPass::run(), llvm::ReplaceWithVeclib::run(), llvm::RewriteStatepointsForGC::run(), llvm::ScalarEvolutionAnalysis::run(), llvm::SCCPPass::run(), llvm::SeparateConstOffsetFromGEPPass::run(), llvm::StackProtectorPass::run(), llvm::TypeSanitizerPass::run(), llvm::X86LowerAMXTypePass::run(), llvm::PlaceSafepointsPass::runImpl(), runImpl(), runImpl(), runImpl(), runImpl(), runImpl(), runImpl(), runImpl(), runImpl(), runImpl(), runNewPMPasses(), llvm::BranchProbabilityInfoWrapperPass::runOnFunction(), llvm::InstructionCombiningPass::runOnFunction(), llvm::LazyBranchProbabilityInfoPass::runOnFunction(), llvm::MemoryDependenceWrapperPass::runOnFunction(), llvm::ReplaceWithVeclibLegacy::runOnFunction(), llvm::RewriteStatepointsForGC::runOnFunction(), llvm::StackProtector::runOnFunction(), runPartiallyInlineLibCalls(), runSCCP(), RunTermFold(), SaturateWidenedDIVFIX(), scalarizeBinOpOfSplats(), scalarizeExtractedBinOp(), llvm::slpvectorizer::scanAltAssociativeOperands(), scanAssociativeOperands(), llvm::InstructionSelectImpl::selectMachineFunction(), setArgExtAttr(), setI8ArgExtAttr(), setInfoSVEStN(), setRetExtAttr(), llvm::BasicTTIImplBase< BasicTTIImpl >::shouldBuildLookupTables(), shouldCheckArgs(), shouldCombineToPostInc(), shouldConvertSelectOfConstantsToMath(), shouldWidenLoad(), ShrinkLoadReplaceStoreWithStore(), simplifyAndDCEInstruction(), simplifyDemandedBitsForPRMT(), llvm::SimplifyInstructionsInBlock(), simplifyLoopInst(), simplifyMul24(), simplifySetCCWithCTPOP(), sinkAndCmp0Expression(), sinkCmpExpression(), SinkShiftAndTruncate(), slotOnlyDiscardsData(), SoleWriteToDeadLocal(), llvm::AArch64FrameLowering::spillCalleeSavedRegisters(), splitMergedValStore(), STATISTIC(), STATISTIC(), STATISTIC(), STATISTIC(), STATISTIC(), stripModuloOnShift(), llvm::TargetLibraryInfo::TargetLibraryInfo(), llvm::TargetLibraryInfo::TargetLibraryInfo(), llvm::TargetLibraryInfoImpl::TargetLibraryInfoImpl(), llvm::TargetLibraryInfoImpl::TargetLibraryInfoImpl(), llvm::TargetLibraryInfoImpl::TargetLibraryInfoImpl(), tryEmitAutoInitRemark(), tryFoldToZero(), trySplitVectorSinCos(), trySQABSCombine(), llvm::VPlanTransforms::tryToConvertVPInstructionsToVPRecipes(), tryToFindDuplicates(), tryToFoldExtendOfConstant(), tryToFoldExtendSelectLoad(), tryToFoldExtOfAtomicLoad(), tryToFoldExtOfExtload(), tryToFoldExtOfLoad(), tryToFoldExtOfMaskedLoad(), unpackFromMemLoc(), unpackFromRegLoc(), unpackFromRegLoc(), llvm::ValueProfileCollector::ValueProfileCollector(), llvm::SelectionDAGBuilder::visitBitTestCase(), llvm::SelectionDAGBuilder::visitBitTestHeader(), llvm::SelectionDAGBuilder::visitJumpTableHeader(), llvm::SelectionDAGBuilder::visitSPDescriptorFailure(), llvm::SelectionDAGBuilder::visitSPDescriptorParent(), llvm::SelectionDAGBuilder::visitSwitchCase(), widenAbs(), widenBuildVec(), widenCtPop(), llvm::wouldInstructionBeTriviallyDead(), and writeUnsupportedAtomicSizeReason().

true

true

Simplify Addr given that the top byte of it is ignored by HW during / address translation.

static bool performTBISimplification(SDValue Addr, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG) { const auto &Subtarget = DAG.getSubtarget<AArch64Subtarget>(); If MTE is enabled, TBI only applies to the top 4 bits. Both arm64 and arm64e processes on Darwin may run with MTE enabled. unsigned NumIgnoreBits = Subtarget.hasMTE() || Subtarget.isTargetDarwin() ? 4 : 8; APInt DemandedMask = APInt::getLowBitsSet(64, 64 - NumIgnoreBits); KnownBits Known; TargetLowering::TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(), !DCI.isBeforeLegalizeOps()); const TargetLowering &TLI = DAG.getTargetLoweringInfo(); if (TLI.SimplifyDemandedBits(Addr, DemandedMask, Known, TLO)) { DCI.CommitTargetLoweringOpt(TLO); return true; } return false; }

static SDValue foldTruncStoreOfExt(SelectionDAG &DAG, SDNode *N) { assert((N->getOpcode() == ISD::STORE || N->getOpcode() == ISD::MSTORE) && "Expected STORE dag node in input!");

if (auto Store = dyn_cast<StoreSDNode>(N)) { if (!Store->isTruncatingStore() || Store->isIndexed()) return SDValue(); SDValue Ext = Store->getValue(); auto ExtOpCode = Ext.getOpcode(); if (ExtOpCode != ISD::ZERO_EXTEND && ExtOpCode != ISD::SIGN_EXTEND && ExtOpCode != ISD::ANY_EXTEND) return SDValue(); SDValue Orig = Ext->getOperand(0); if (Store->getMemoryVT() != Orig.getValueType()) return SDValue(); return DAG.getStore(Store->getChain(), SDLoc(Store), Orig, Store->getBasePtr(), Store->getMemOperand()); }

return SDValue(); }

A custom combine to lower load <3 x i8> as the more efficient sequence below: ldrb wX, [x0, #2] ldrh wY, [x0] orr wX, wY, wX, lsl #16 fmov s0, wX

Note that an alternative sequence with even fewer (although usually more complex/expensive) instructions would be: ld1r.4h { v0 }, [x0], #2 ld1.b { v0 }[2], [x0]

Generating this sequence unfortunately results in noticeably worse codegen for code that extends the loaded v3i8, due to legalization breaking vector shuffle detection in a way that is very difficult to work around. TODO: Revisit once v3i8 legalization has been improved in general. static SDValue combineV3I8LoadExt(LoadSDNode *LD, SelectionDAG &DAG) { EVT MemVT = LD->getMemoryVT(); if (MemVT != EVT::getVectorVT(*DAG.getContext(), MVT::i8, 3) || LD->getBaseAlign() >= 4) return SDValue();

SDLoc DL(LD); MachineFunction &MF = DAG.getMachineFunction(); SDValue Chain = LD->getChain(); SDValue BasePtr = LD->getBasePtr(); MachineMemOperand *MMO = LD->getMemOperand(); assert(LD->getOffset().isUndef() && "undef offset expected");

Load 2 x i8, then 1 x i8. SDValue L16 = DAG.getLoad(MVT::i16, DL, Chain, BasePtr, MMO); TypeSize Offset2 = TypeSize::getFixed(2); SDValue L8 = DAG.getLoad(MVT::i8, DL, Chain, DAG.getMemBasePlusOffset(BasePtr, Offset2, DL), MF.getMachineMemOperand(MMO, 2, 1));

Extend to i32. SDValue Ext16 = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, L16); SDValue Ext8 = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, L8);

Pack 2 x i8 and 1 x i8 in an i32 and convert to v4i8. SDValue Shl = DAG.getNode(ISD::SHL, DL, MVT::i32, Ext8, DAG.getConstant(16, DL, MVT::i32)); SDValue Or = DAG.getNode(ISD::OR, DL, MVT::i32, Ext16, Shl); SDValue Cast = DAG.getNode(ISD::BITCAST, DL, MVT::v4i8, Or);

Extract v3i8 again. SDValue Extract = DAG.getExtractSubvector(DL, MemVT, Cast, 0); SDValue TokenFactor = DAG.getNode( ISD::TokenFactor, DL, MVT::Other, {SDValue(cast<SDNode>(L16), 1), SDValue(cast<SDNode>(L8), 1)}); return DAG.getMergeValues({Extract, TokenFactor}, DL); }

Perform TBI simplification if supported by the target and try to break up nontemporal loads larger than 256-bits loads for odd types so LDNPQ 256-bit load instructions can be selected. static SDValue performLOADCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG, const AArch64Subtarget *Subtarget) { if (Subtarget->supportsAddressTopByteIgnored()) performTBISimplification(N->getOperand(1), DCI, DAG);

LoadSDNode *LD = cast<LoadSDNode>(N); EVT RegVT = LD->getValueType(0); EVT MemVT = LD->getMemoryVT(); const TargetLowering &TLI = DAG.getTargetLoweringInfo(); SDLoc DL(LD);

Cast ptr32 and ptr64 pointers to the default address space before a load. unsigned AddrSpace = LD->getAddressSpace(); if (AddrSpace == ARM64AS::PTR64 || AddrSpace == ARM64AS::PTR32_SPTR || AddrSpace == ARM64AS::PTR32_UPTR) { MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout()); if (PtrVT != LD->getBasePtr().getSimpleValueType()) { SDValue Cast = DAG.getAddrSpaceCast(DL, PtrVT, LD->getBasePtr(), AddrSpace, 0); return DAG.getExtLoad(LD->getExtensionType(), DL, RegVT, LD->getChain(), Cast, LD->getPointerInfo(), MemVT, LD->getBaseAlign(), LD->getMemOperand()->getFlags()); } }

if (LD->isVolatile() || !Subtarget->isLittleEndian()) return SDValue(N, 0);

if (SDValue Res = combineV3I8LoadExt(LD, DAG)) return Res;

if (!LD->isNonTemporal()) return SDValue(N, 0);

if (MemVT.isScalableVector() || MemVT.getSizeInBits() <= 256 || MemVT.getSizeInBits() % 256 == 0 || 256 % MemVT.getScalarSizeInBits() != 0) return SDValue(N, 0);

SDValue Chain = LD->getChain(); SDValue BasePtr = LD->getBasePtr(); SDNodeFlags Flags = LD->getFlags(); SmallVector<SDValue, 4> LoadOps; SmallVector<SDValue, 4> LoadOpsChain; Replace any non temporal load over 256-bit with a series of 256 bit loads and a scalar/vector load less than 256. This way we can utilize 256-bit loads and reduce the amount of load instructions generated. MVT NewVT = MVT::getVectorVT(MemVT.getVectorElementType().getSimpleVT(), 256 / MemVT.getVectorElementType().getSizeInBits()); unsigned Num256Loads = MemVT.getSizeInBits() / 256; Create all 256-bit loads starting from offset 0 and up to Num256Loads-1*32. for (unsigned I = 0; I < Num256Loads; I++) { unsigned PtrOffset = I * 32; SDValue NewPtr = DAG.getMemBasePlusOffset( BasePtr, TypeSize::getFixed(PtrOffset), DL, Flags); Align NewAlign = commonAlignment(LD->getAlign(), PtrOffset); SDValue NewLoad = DAG.getLoad( NewVT, DL, Chain, NewPtr, LD->getPointerInfo().getWithOffset(PtrOffset), NewAlign, LD->getMemOperand()->getFlags(), LD->getAAInfo()); LoadOps.push_back(NewLoad); LoadOpsChain.push_back(SDValue(cast<SDNode>(NewLoad), 1)); }

Process remaining bits of the load operation. This is done by creating an UNDEF vector to match the size of the 256-bit loads and inserting the remaining load to it. We extract the original load type at the end using EXTRACT_SUBVECTOR instruction. unsigned BitsRemaining = MemVT.getSizeInBits() % 256; unsigned PtrOffset = (MemVT.getSizeInBits() - BitsRemaining) / 8; MVT RemainingVT = MVT::getVectorVT( MemVT.getVectorElementType().getSimpleVT(), BitsRemaining / MemVT.getVectorElementType().getSizeInBits()); SDValue NewPtr = DAG.getMemBasePlusOffset( BasePtr, TypeSize::getFixed(PtrOffset), DL, Flags); Align NewAlign = commonAlignment(LD->getAlign(), PtrOffset); SDValue RemainingLoad = DAG.getLoad(RemainingVT, DL, Chain, NewPtr, LD->getPointerInfo().getWithOffset(PtrOffset), NewAlign, LD->getMemOperand()->getFlags(), LD->getAAInfo()); SDValue PoisonVector = DAG.getPOISON(NewVT); SDValue ExtendedRemainingLoad = DAG.getInsertSubvector(DL, PoisonVector, RemainingLoad, 0); LoadOps.push_back(ExtendedRemainingLoad); LoadOpsChain.push_back(SDValue(cast<SDNode>(RemainingLoad), 1)); EVT ConcatVT = EVT::getVectorVT(*DAG.getContext(), MemVT.getScalarType(), LoadOps.size() * NewVT.getVectorNumElements()); SDValue ConcatVectors = DAG.getNode(ISD::CONCAT_VECTORS, DL, ConcatVT, LoadOps); Extract the original vector type size. SDValue ExtractSubVector = DAG.getExtractSubvector(DL, MemVT, ConcatVectors, 0); SDValue TokenFactor = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, LoadOpsChain); return DAG.getMergeValues({ExtractSubVector, TokenFactor}, DL); }

static EVT tryGetOriginalBoolVectorType(SDValue Op, int Depth = 0) { EVT VecVT = Op.getValueType(); assert(VecVT.isVectorOf(MVT::i1) && "Need boolean vector type.");

if (Depth > 3) return MVT::INVALID_SIMPLE_VALUE_TYPE;

We can get the base type from a vector compare or truncate. if (Op.getOpcode() == ISD::SETCC || Op.getOpcode() == ISD::TRUNCATE) return Op.getOperand(0).getValueType();

If an operand is a bool vector, continue looking. EVT BaseVT = MVT::INVALID_SIMPLE_VALUE_TYPE; for (SDValue Operand : Op->op_values()) { if (Operand.getValueType() != VecVT) continue;

EVT OperandVT = tryGetOriginalBoolVectorType(Operand, Depth + 1); if (!BaseVT.isSimple()) BaseVT = OperandVT; else if (OperandVT != BaseVT) return MVT::INVALID_SIMPLE_VALUE_TYPE; }

return BaseVT; }

static bool getBoolVectorBitcastCompare(SDValue Vec, SDValue RHS, const SDLoc &DL, SelectionDAG &DAG, SDValue &CompareLHS, SDValue &CompareRHS) { if (DAG.getDataLayout().isBigEndian()) return false;

EVT VecVT = Vec.getValueType(); assert(VecVT.isFixedLengthVector() && VecVT.getVectorElementType() == MVT::i1 && "Expected a fixed-length bool vector");

unsigned NumElts = VecVT.getVectorNumElements(); if (NumElts != 2 && NumElts != 4 && NumElts != 8 && NumElts != 16) return false;

auto getCanonicalCompareVecVT = [&]() { unsigned BitsPerElement = std::max(64 / NumElts, 8u); return MVT::getVectorVT(MVT::getIntegerVT(BitsPerElement), NumElts); };

EVT CompareVecVT = tryGetOriginalBoolVectorType(Vec); if (!CompareVecVT.isSimple() || CompareVecVT.getSizeInBits() > 128) { CompareVecVT = getCanonicalCompareVecVT(); } CompareVecVT = CompareVecVT.changeVectorElementTypeToInteger();

if (CompareVecVT.getSizeInBits() > 128) return false;

SDValue CompareBits = DAG.getSExtOrTrunc(Vec, DL, CompareVecVT); unsigned CompareBitsSize = CompareBits.getValueSizeInBits();

Use a canonical 64/128-bit vector representation before bitcasting to a scalar view. Some legal original vector types are smaller than 64-bit, which would make the direct scalar bitcasts below invalid. if (CompareBitsSize != 64 && CompareBitsSize != 128) { CompareVecVT = getCanonicalCompareVecVT(); CompareBits = DAG.getSExtOrTrunc(Vec, DL, CompareVecVT); CompareBitsSize = CompareBits.getValueSizeInBits(); }

if (CompareBitsSize != 64 && CompareBitsSize != 128) return false;

bool IsNull = isNullConstant(RHS); if (CompareBitsSize == 64) { CompareLHS = DAG.getBitcast(MVT::i64, CompareBits); CompareRHS = IsNull ? DAG.getConstant(0, DL, MVT::i64) : DAG.getAllOnesConstant(DL, MVT::i64); } else { SDValue PairwiseBits = DAG.getBitcast(MVT::v2i64, CompareBits); SDValue Lo = DAG.getExtractVectorElt(DL, MVT::i64, PairwiseBits, 0); SDValue Hi = DAG.getExtractVectorElt(DL, MVT::i64, PairwiseBits, 1); CompareLHS = DAG.getNode(ISD::ADD, DL, MVT::i64, Lo, Hi); CompareRHS = IsNull ? DAG.getConstant(0, DL, MVT::i64) : DAG.getSignedConstant(-2, DL, MVT::i64); }

return true; }

When converting a <N x iX> vector to <N x i1> to store or use as a scalar iN, we can use a trick that extracts the i^th bit from the i^th element and then performs a vector add to get a scalar bitmask. This requires that each element's bits are either all 1 or all 0. static SDValue vectorToScalarBitmask(SDValue ComparisonResult, SelectionDAG &DAG) { SDLoc DL(ComparisonResult); EVT VecVT = ComparisonResult.getValueType(); assert(VecVT.isVector() && "Must be a vector type");

unsigned NumElts = VecVT.getVectorNumElements(); if (NumElts != 2 && NumElts != 4 && NumElts != 8 && NumElts != 16) return SDValue();

if (VecVT.getVectorElementType() != MVT::i1 && !DAG.getTargetLoweringInfo().isTypeLegal(VecVT)) return SDValue();

If we can find the original types to work on instead of a vector of i1, we can avoid extend/extract conversion instructions. if (VecVT.getVectorElementType() == MVT::i1) { VecVT = tryGetOriginalBoolVectorType(ComparisonResult); if (!VecVT.isSimple()) { unsigned BitsPerElement = std::max(64 / NumElts, 8u); // >= 64-bit vector VecVT = MVT::getVectorVT(MVT::getIntegerVT(BitsPerElement), NumElts); } } VecVT = VecVT.changeVectorElementTypeToInteger();

The bitmask depends only on the lane count, not the lane width so when the preferred type exceeds 128 bits narrow the lanes to fill one 128-bit register instead of spilling through the stack. if (VecVT.getSizeInBits() > 128) { assert(128 % NumElts == 0 && "Expected NumElts to evenly divide 128 bits"); unsigned NarrowedEltBits = 128 / NumElts; if (NarrowedEltBits < 8) return SDValue(); VecVT = MVT::getVectorVT(MVT::getIntegerVT(NarrowedEltBits), NumElts); }

Ensure that all elements' bits are either 0s or 1s. ComparisonResult = DAG.getSExtOrTrunc(ComparisonResult, DL, VecVT);

bool IsLE = DAG.getDataLayout().isLittleEndian(); SmallVector<SDValue, 16> MaskConstants; if (DAG.getSubtarget<AArch64Subtarget>().isNeonAvailable() && VecVT == MVT::v16i8) { v16i8 is a special case: we have 16 entries but only 8 positional bits per i16 of bitmask. ANDing with a mask whose two 8-byte halves both hold the powers-of-two 1,2,...,128 leaves at most one bit set per byte lane within each 8-byte group, so pairwise addition equals OR and cannot carry. Three levels of ADDP then losslessly pack the 16 per-lane bits into the low i16 of the result vector. for (unsigned Half = 0; Half < 2; ++Half) { for (unsigned I = 0; I < 8; ++I) { On big-endian targets, the lane order in sub-byte vector elements gets reversed, so we need to flip the bit index. unsigned MaskBit = IsLE ? (1u << I) : (1u << (7 - I)); MaskConstants.push_back(DAG.getConstant(MaskBit, DL, MVT::i32)); } } SDValue Mask = DAG.getNode(ISD::BUILD_VECTOR, DL, VecVT, MaskConstants); SDValue RepresentativeBits = DAG.getNode(ISD::AND, DL, VecVT, ComparisonResult, Mask);

SDValue V = RepresentativeBits; V = DAG.getNode(AArch64ISD::ADDP, DL, VecVT, V, V); V = DAG.getNode(AArch64ISD::ADDP, DL, VecVT, V, V); V = DAG.getNode(AArch64ISD::ADDP, DL, VecVT, V, V); V = DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, V); return DAG.getExtractVectorElt(DL, MVT::i32, V, 0); }

All other vector sizes. unsigned NumEl = VecVT.getVectorNumElements(); for (unsigned I = 0; I < NumEl; ++I) { unsigned MaskBit = IsLE ? (1u << I) : (1u << (NumEl - 1 - I)); MaskConstants.push_back(DAG.getConstant(MaskBit, DL, MVT::i64)); }

SDValue Mask = DAG.getNode(ISD::BUILD_VECTOR, DL, VecVT, MaskConstants); SDValue RepresentativeBits = DAG.getNode(ISD::AND, DL, VecVT, ComparisonResult, Mask); EVT ResultVT = MVT::getIntegerVT(std::max<unsigned>( NumElts, VecVT.getVectorElementType().getSizeInBits())); return DAG.getNode(ISD::VECREDUCE_ADD, DL, ResultVT, RepresentativeBits); }

When taking the trailing-zero count of a bitcast from <N x i1> to scalar iN, we can pack each lane into a small bit-slot of a 64-bit scalar with shrn (for byte lanes) or xtn (for wider lanes) then run a scalar cttz on the result. static SDValue performCTTZCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG) { Run before vectorToScalarBitmask() expands it. if (!DCI.isBeforeLegalize()) return SDValue();

EVT VT = N->getValueType(0); if (!VT.isScalarInteger()) return SDValue(); SDValue BitcastNode = N->getOperand(0); if (BitcastNode.getOpcode() != ISD::BITCAST) return SDValue(); SDValue Predicate = BitcastNode.getOperand(0); EVT PredVT = Predicate.getValueType(); if (!PredVT.isFixedLengthVector() || PredVT.getVectorElementType() != MVT::i1) return SDValue();

unsigned NumLanes = PredVT.getVectorNumElements(); if (!isPowerOf2_32(NumLanes) || NumLanes < 2 || NumLanes > 32) return SDValue();

constexpr unsigned DRegBits = 64; constexpr unsigned QRegBits = 128; constexpr unsigned MinLaneBits = 8;

Pick a lane width to sext the predicate to. Prefer the original compare input width if we can find it and it fits in at most two NEON registers but otherwise pick a minimum-width type that fits. EVT MaskVT = tryGetOriginalBoolVectorType(Predicate); if (!MaskVT.isSimple() || MaskVT.changeVectorElementTypeToInteger().getSizeInBits() > 2 * QRegBits || MaskVT.changeVectorElementTypeToInteger().getSizeInBits() < DRegBits) { unsigned LaneBits = std::max(DRegBits / NumLanes, MinLaneBits); MaskVT = MVT::getVectorVT(MVT::getIntegerVT(LaneBits), NumLanes); } MaskVT = MaskVT.changeVectorElementTypeToInteger();

unsigned MaskBits = MaskVT.getSizeInBits(); if (MaskBits != DRegBits && MaskBits != QRegBits && MaskBits != 2 * QRegBits) return SDValue();

Target 64 bits when lanes fit, otherwise 128. unsigned TargetBits = DRegBits; if (MaskBits == 2 * QRegBits && NumLanes > 16) TargetBits = QRegBits; unsigned CompressedBits = std::min(MaskBits, TargetBits);

SDLoc DL(N); SDValue Mask = DAG.getSExtOrTrunc(Predicate, DL, MaskVT);

LLVMContext &Ctx = *DAG.getContext();

NEON's smallest vector lane type is i8 so we can't narrow further. Instead, bitcast the mask to i16 lanes and shrn by 4 to pack each input byte into 4 bits of the output auto ShrnPair = [&](SDValue V) -> SDValue { EVT InVT = V.getValueType(); EVT OutVT = InVT.getHalfNumVectorElementsVT(Ctx); EVT PairedVT = OutVT.widenIntegerVectorElementType(Ctx); SDValue Paired = DAG.getNode(AArch64ISD::NVCAST, DL, PairedVT, V); SDValue ShAmt = DAG.getConstant(MinLaneBits / 2, DL, PairedVT); SDValue Shifted = DAG.getNode(ISD::SRL, DL, PairedVT, Paired, ShAmt); return DAG.getNode(ISD::TRUNCATE, DL, OutVT, Shifted); };

Trailing-zero count of the compressed result as an i64. auto CompressedCttz = [&](SDValue V) -> SDValue { if (V.getValueType().getSizeInBits() == DRegBits) { SDValue Nv = DAG.getNode(AArch64ISD::NVCAST, DL, MVT::v1i64, V); SDValue Scalar = DAG.getExtractVectorElt(DL, MVT::i64, Nv, 0); return DAG.getNode(ISD::CTTZ, DL, MVT::i64, Scalar); } SDValue Compressed = DAG.getBitcast(MVT::i128, V); BITCAST puts lane 0 at the high end of the scalar on BE (unlike NVCAST above), so use clz instead of cttz to find the first match unsigned Op = DAG.getDataLayout().isLittleEndian() ? ISD::CTTZ : ISD::CTLZ; SDValue Ctz = DAG.getNode(Op, DL, MVT::i128, Compressed); return DAG.getNode(ISD::TRUNCATE, DL, MVT::i64, Ctz); };

SDValue Cur = Mask; Narrow wide lanes toward CompressedBits with xtn. while (Cur.getValueType().getScalarSizeInBits() > MinLaneBits && Cur.getValueType().getSizeInBits() > CompressedBits) { EVT CurVT = Cur.getValueType(); unsigned CurLaneBits = CurVT.getScalarSizeInBits(); EVT NarrowVT = CurVT.changeVectorElementType( Ctx, EVT::getIntegerVT(Ctx, CurLaneBits / 2)); Cur = DAG.getNode(ISD::TRUNCATE, DL, NarrowVT, Cur); }

If still too wide, pack two lanes per byte with shrn. if (Cur.getValueType().getSizeInBits() > CompressedBits) { EVT CurVT = Cur.getValueType(); if (CurVT.getSizeInBits() <= QRegBits) { Cur = ShrnPair(Cur); } else { Split into Q-reg-sized halves so each ShrnPair stays in range. EVT HalfVT = CurVT.getHalfNumVectorElementsVT(Ctx); SDValue Lo = DAG.getExtractSubvector(DL, HalfVT, Cur, 0); SDValue Hi = DAG.getNode( ISD::EXTRACT_SUBVECTOR, DL, HalfVT, Cur, DAG.getConstant(HalfVT.getVectorNumElements(), DL, MVT::i64)); Cur = DAG.getNode(ISD::CONCAT_VECTORS, DL, HalfVT, ShrnPair(Lo), ShrnPair(Hi)); } }

Cur has been narrowed to CompressedBits. assert(Cur.getValueType().getSizeInBits() == CompressedBits && "Unexpected mask width!");

SDValue Ctz = CompressedCttz(Cur); unsigned LaneIndexShift = Log2_32(CompressedBits / NumLanes); if (LaneIndexShift) Ctz = DAG.getNode(ISD::SRL, DL, MVT::i64, Ctz, DAG.getShiftAmountConstant(LaneIndexShift, MVT::i64, DL)); return DAG.getSExtOrTrunc(Ctz, DL, VT); }

static SDValue combineBoolVectorAndTruncateStore(SelectionDAG &DAG, StoreSDNode *Store) { if (!Store->isTruncatingStore()) return SDValue();

SDLoc DL(Store); SDValue VecOp = Store->getValue(); EVT VT = VecOp.getValueType(); EVT MemVT = Store->getMemoryVT();

if (!MemVT.isVector() || !VT.isVector() || MemVT.getVectorElementType() != MVT::i1) return SDValue();

If we are storing a vector that we are currently building, let scalarizeVectorStore() handle this more efficiently. if (VecOp.getOpcode() == ISD::BUILD_VECTOR) return SDValue();

VecOp = DAG.getNode(ISD::TRUNCATE, DL, MemVT, VecOp); SDValue VectorBits = vectorToScalarBitmask(VecOp, DAG); if (!VectorBits) return SDValue();

EVT StoreVT = EVT::getIntegerVT(*DAG.getContext(), MemVT.getStoreSizeInBits()); SDValue ExtendedBits = DAG.getZExtOrTrunc(VectorBits, DL, StoreVT); return DAG.getStore(Store->getChain(), DL, ExtendedBits, Store->getBasePtr(), Store->getMemOperand()); }

Combine store (fp_to_int X) to use vector semantics around the conversion when NEON is available. This allows us to store the in-vector result directly without transferring the result into a GPR in the process. static SDValue combineStoreValueFPToInt(StoreSDNode *ST, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG, const AArch64Subtarget *Subtarget) { if (!Subtarget->isNeonAvailable()) return SDValue(); Source operand is already a vector. SDValue Value = ST->getValue(); if (Value.getValueType().isVector()) return SDValue();

Look through potential assertions. while (Value->isAssert()) Value = Value.getOperand(0);

if (Value.getOpcode() != ISD::FP_TO_SINT && Value.getOpcode() != ISD::FP_TO_UINT) return SDValue(); if (!Value->hasOneUse()) return SDValue();

SDValue FPSrc = Value.getOperand(0); EVT SrcVT = FPSrc.getValueType(); if (SrcVT != MVT::f32 && SrcVT != MVT::f64) return SDValue();

No support for assignments such as i64 = fp_to_sint i32 EVT VT = Value.getSimpleValueType(); if (VT != SrcVT.changeTypeToInteger()) return SDValue();

Create a 128-bit element vector to avoid widening. The floating point conversion is transformed into a single element conversion via a pattern. unsigned NumElements = 128 / SrcVT.getFixedSizeInBits(); EVT VecSrcVT = EVT::getVectorVT(*DAG.getContext(), SrcVT, NumElements); EVT VecDstVT = VecSrcVT.changeTypeToInteger(); SDLoc DL(ST); SDValue VecFP = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecSrcVT, FPSrc); SDValue VecConv = DAG.getNode(Value.getOpcode(), DL, VecDstVT, VecFP); SDValue Extracted = DAG.getExtractVectorElt(DL, VT, VecConv, 0);

DCI.CombineTo(ST->getValue().getNode(), Extracted); return SDValue(ST, 0); }

static bool isHalvingTruncateOfLegalScalableType(EVT SrcVT, EVT DstVT) { return (SrcVT == MVT::nxv8i16 && DstVT == MVT::nxv8i8) || (SrcVT == MVT::nxv4i32 && DstVT == MVT::nxv4i16) || (SrcVT == MVT::nxv2i64 && DstVT == MVT::nxv2i32); }

Combine store (trunc X to <3 x i8>) to sequence of ST1.b. static SDValue combineI8TruncStore(StoreSDNode *ST, SelectionDAG &DAG, const AArch64Subtarget *Subtarget) { SDValue Value = ST->getValue(); EVT ValueVT = Value.getValueType();

if (ST->isVolatile() || !Subtarget->isLittleEndian() || Value.getOpcode() != ISD::TRUNCATE || ValueVT != EVT::getVectorVT(*DAG.getContext(), MVT::i8, 3)) return SDValue();

assert(ST->getOffset().isUndef() && "undef offset expected"); SDLoc DL(ST); EVT WideVT = EVT::getVectorVT( DAG.getContext(), Value->getOperand(0).getValueType().getVectorElementType(), 4); if (!DAG.getTargetLoweringInfo().isTypeLegal(WideVT)) return SDValue(); SDValue PoisonVector = DAG.getPOISON(WideVT); SDValue WideTrunc = DAG.getNode( ISD::INSERT_SUBVECTOR, DL, WideVT, {PoisonVector, Value->getOperand(0), DAG.getVectorIdxConstant(0, DL)}); SDValue Cast = DAG.getNode( ISD::BITCAST, DL, WideVT.getSizeInBits() == 64 ? MVT::v8i8 : MVT::v16i8, WideTrunc);

MachineFunction &MF = DAG.getMachineFunction(); SDValue Chain = ST->getChain(); MachineMemOperand *MMO = ST->getMemOperand(); unsigned IdxScale = WideVT.getScalarSizeInBits() / 8; SDValue E2 = DAG.getExtractVectorElt(DL, MVT::i8, Cast, IdxScale * 2); TypeSize Offset2 = TypeSize::getFixed(2); SDValue Ptr2 = DAG.getMemBasePlusOffset(ST->getBasePtr(), Offset2, DL); Chain = DAG.getStore(Chain, DL, E2, Ptr2, MF.getMachineMemOperand(MMO, 2, 1));

SDValue E1 = DAG.getExtractVectorElt(DL, MVT::i8, Cast, IdxScale); TypeSize Offset1 = TypeSize::getFixed(1); SDValue Ptr1 = DAG.getMemBasePlusOffset(ST->getBasePtr(), Offset1, DL); Chain = DAG.getStore(Chain, DL, E1, Ptr1, MF.getMachineMemOperand(MMO, 1, 1));

SDValue E0 = DAG.getExtractVectorElt(DL, MVT::i8, Cast, 0); Chain = DAG.getStore(Chain, DL, E0, ST->getBasePtr(), MF.getMachineMemOperand(MMO, 0, 1)); return Chain; }

static unsigned getFPSubregForVT(EVT VT) { assert(VT.isSimple() && "Expected simple VT"); switch (VT.getSimpleVT().SimpleTy) { case MVT::aarch64mfp8: return AArch64::bsub; case MVT::f16: return AArch64::hsub; case MVT::f32: return AArch64::ssub; case MVT::f64: return AArch64::dsub; default: llvm_unreachable("Unexpected VT!"); } }

static SDValue performInterleavedStoreCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG);

static SDValue performLegalizedInterleavedStoreCombine( StoreSDNode *ST, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG);

static SDValue performSTORECombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG, const AArch64Subtarget *Subtarget) { using namespace llvm::SDPatternMatch; StoreSDNode *ST = cast<StoreSDNode>(N); SDValue Chain = ST->getChain(); SDValue Value = ST->getValue(); SDValue Ptr = ST->getBasePtr(); EVT ValueVT = Value.getValueType(); EVT MemVT = ST->getMemoryVT(); const TargetLowering &TLI = DAG.getTargetLoweringInfo(); SDLoc DL(ST);

Fixed length svbool_t store operations use an i8 vector as the underlying memory type which is cast from a scalable boolean vector. This combine replaces such sequences with a direct store of an SVE predicate. SDValue SVBool; if (DCI.isBeforeLegalize() && Subtarget->isSVEorStreamingSVEAvailable() && ISD::isNormalStore(N) && ValueVT.isFixedLengthVectorOf(MVT::i8) && ValueVT.getSizeInBits() == Subtarget->getSVEPredicateSizeInBits() && sd_match(Value, m_ExtractSubvector(m_BitCast(m_SpecificVT( MVT::nxv16i1, m_Value(SVBool))), m_SpecificInt(0)))) return DAG.getStore(Chain, DL, SVBool, Ptr, ST->getPointerInfo(), ST->getBaseAlign(), ST->getMemOperand()->getFlags(), ST->getAAInfo());

if (SDValue Res = combineStoreValueFPToInt(ST, DCI, DAG, Subtarget)) return Res;

if (SDValue Res = performInterleavedStoreCombine(N, DCI, DAG)) return Res;

if (SDValue Res = performLegalizedInterleavedStoreCombine(ST, DCI, DAG)) return Res;

Cast ptr32 and ptr64 pointers to the default address space before a store. unsigned AddrSpace = ST->getAddressSpace(); if (AddrSpace == ARM64AS::PTR64 || AddrSpace == ARM64AS::PTR32_SPTR || AddrSpace == ARM64AS::PTR32_UPTR) { MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout()); if (PtrVT != Ptr.getSimpleValueType()) { SDValue Cast = DAG.getAddrSpaceCast(DL, PtrVT, Ptr, AddrSpace, 0); if (ST->isTruncatingStore()) return DAG.getTruncStore(Chain, DL, Value, Cast, MemVT, ST->getMemOperand()); return DAG.getStore(Chain, DL, Value, Cast, ST->getMemOperand()); } }

if (SDValue Res = combineI8TruncStore(ST, DAG, Subtarget)) return Res;

auto hasValidElementTypeForFPTruncStore = [](EVT DstVT, EVT SrcVT) { if (DstVT.getScalarType() == MVT::bf16) return false; EVT EltVT = SrcVT.getVectorElementType(); return EltVT == MVT::f32 || EltVT == MVT::f64; };

If this is an FP_ROUND followed by a store, fold this into a truncating store. We can do this even if this is already a truncstore. We purposefully don't care about legality of the nodes here as we know they can be split down into something legal. if (DCI.isBeforeLegalizeOps() && Value.getOpcode() == ISD::FP_ROUND && Value->hasOneUse() && ST->isUnindexed() && Subtarget->useSVEForFixedLengthVectors() && ValueVT.isFixedLengthVector() && ValueVT.getFixedSizeInBits() >= Subtarget->getMinSVEVectorSizeInBits() && hasValidElementTypeForFPTruncStore(Value.getValueType(), Value.getOperand(0).getValueType())) return DAG.getTruncStore(Chain, DL, Value.getOperand(0), Ptr, MemVT, ST->getMemOperand());

if (SDValue Split = splitStores(N, DCI, DAG, Subtarget)) return Split;

if (Subtarget->supportsAddressTopByteIgnored() && performTBISimplification(N->getOperand(2), DCI, DAG)) return SDValue(N, 0);

if (SDValue Store = foldTruncStoreOfExt(DAG, N)) return Store;

if (SDValue Store = combineBoolVectorAndTruncateStore(DAG, ST)) return Store;

if (ST->isTruncatingStore() && isHalvingTruncateOfLegalScalableType(ValueVT, MemVT)) { if (SDValue Rshrnb = trySimplifySrlAddToRshrnb(ST->getOperand(1), DAG, Subtarget)) { return DAG.getTruncStore(ST->getChain(), ST, Rshrnb, ST->getBasePtr(), MemVT, ST->getMemOperand()); } }

This is an integer vector_extract_elt followed by a (possibly truncating) store. We may be able to replace this with a store of an FP subregister. if (DCI.isAfterLegalizeDAG() && ST->isUnindexed() && Value.getOpcode() == ISD::EXTRACT_VECTOR_ELT) {

SDValue Vector = Value.getOperand(0); SDValue ExtIdx = Value.getOperand(1); EVT VectorVT = Vector.getValueType(); EVT ElemVT = VectorVT.getVectorElementType();

if (!ValueVT.isInteger()) return SDValue();

Propagate zero constants (applying this fold may miss optimizations). if (ISD::isConstantSplatVectorAllZeros(Vector.getNode())) { SDValue ZeroElt = DAG.getConstant(0, DL, ValueVT); DAG.ReplaceAllUsesWith(Value, ZeroElt); return SDValue(); }

if (ValueVT != MemVT && !ST->isTruncatingStore()) return SDValue();

This could generate an additional extract if the index is non-zero and the extracted value has multiple uses. auto *ExtCst = dyn_cast<ConstantSDNode>(ExtIdx); if ((!ExtCst || !ExtCst->isZero()) && !Value.hasOneUse()) return SDValue();

These can lower to st1, which is preferable if we're unlikely to fold the addressing into the store. if (Subtarget->isNeonAvailable() && ElemVT == MemVT && (VectorVT.is64BitVector() || VectorVT.is128BitVector()) && ExtCst && !ExtCst->isZero() && ST->getBasePtr().getOpcode() != ISD::ADD) return SDValue();

if (MemVT == MVT::i64 || MemVT == MVT::i32) { Heuristic: If there are other users of w/x integer scalars extracted from this vector that won't fold into the store – abandon folding. Applying this fold may disrupt paired stores. for (const auto &Use : Vector->uses()) { if (Use.getResNo() != Vector.getResNo()) continue; const SDNode *User = Use.getUser(); if (User->getOpcode() == ISD::EXTRACT_VECTOR_ELT && (!User->hasOneUse() || (*User->user_begin())->getOpcode() != ISD::STORE)) return SDValue(); } }

SDValue ExtVector = Vector; if (!ExtCst || !ExtCst->isZero()) { Handle extracting from lanes != 0. SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, Value.getValueType(), Vector, ExtIdx); ExtVector = DAG.getInsertVectorElt(DL, DAG.getPOISON(VectorVT), Ext, 0); }

EVT FPMemVT = MemVT == MVT::i8 ? MVT::aarch64mfp8 : EVT::getFloatingPointVT(MemVT.getSizeInBits()); SDValue FPSubreg = DAG.getTargetExtractSubreg(getFPSubregForVT(FPMemVT), DL, FPMemVT, ExtVector);

return DAG.getStore(ST->getChain(), DL, FPSubreg, ST->getBasePtr(), ST->getMemOperand()); }

return SDValue(); }

static bool isSequentialConcatOfVectorInterleave(SDNode *N, SmallVectorImpl<SDValue> &Ops) { if (N->getOpcode() != ISD::CONCAT_VECTORS) return false;

unsigned NumParts = N->getNumOperands();

We should be concatenating each sequential result from a VECTOR_INTERLEAVE. SDNode *InterleaveOp = N->getOperand(0).getNode(); if (InterleaveOp->getOpcode() != ISD::VECTOR_INTERLEAVE || InterleaveOp->getNumOperands() != NumParts) return false;

for (unsigned I = 0; I < NumParts; I++) if (N->getOperand(I) != SDValue(InterleaveOp, I)) return false;

Ops.append(InterleaveOp->op_begin(), InterleaveOp->op_end()); return true; }

static bool isSplatVectorInterleaveOps(SelectionDAG &DAG, SDLoc DL, SDValue WideValue, SmallVectorImpl<SDValue> &Ops) { EVT WideVT = WideValue.getValueType(); if (!WideVT.isScalableVector()) return false;

SDValue SplatValue = DAG.getSplatValue(WideValue); if (!SplatValue) return false;

TypeSize WideSize = WideVT.getSizeInBits(); if (!WideSize.isKnownMultipleOf(128)) return false;

unsigned NumParts = WideSize.getKnownMinValue() / 128; if (NumParts != 2 && NumParts != 3 && NumParts != 4) return false;

ElementCount WideEC = WideVT.getVectorElementCount(); if (!WideEC.isKnownMultipleOf(NumParts)) return false;

EVT SubVecTy = EVT::getVectorVT(*DAG.getContext(), WideVT.getVectorElementType(), WideEC.divideCoefficientBy(NumParts));

if (SubVecTy.getSizeInBits().getKnownMinValue() != 128 || !DAG.getTargetLoweringInfo().isTypeLegal(SubVecTy)) return false;

SDValue NarrowSplat = DAG.getNode(ISD::SPLAT_VECTOR, DL, SubVecTy, SplatValue); Ops.append(NumParts, NarrowSplat); return true; }

static SDValue getNarrowMaskForInterleavedOps(SelectionDAG &DAG, SDLoc &DL, SDValue WideMask, unsigned RequiredNumParts) { if (WideMask->getOpcode() == ISD::CONCAT_VECTORS) { SmallVector<SDValue, 4> MaskInterleaveOps; if (!isSequentialConcatOfVectorInterleave(WideMask.getNode(), MaskInterleaveOps)) return SDValue();

if (MaskInterleaveOps.size() != RequiredNumParts) return SDValue();

Make sure the inputs to the vector interleave are identical. if (!llvm::all_equal(MaskInterleaveOps)) return SDValue();

return MaskInterleaveOps[0]; }

if (WideMask->getOpcode() != ISD::SPLAT_VECTOR) return SDValue();

ElementCount EC = WideMask.getValueType().getVectorElementCount(); assert(EC.isKnownMultipleOf(RequiredNumParts) && "Expected element count divisible by number of parts"); EC = EC.divideCoefficientBy(RequiredNumParts); return DAG.getNode(ISD::SPLAT_VECTOR, DL, MVT::getVectorVT(MVT::i1, EC), WideMask->getOperand(0)); }

static SDValue performInterleavedStoreCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG) { if (!DCI.isBeforeLegalize()) return SDValue();

SDValue WideValue, Chain, BasePtr, Mask; bool IsMasked = false; if (auto *MST = dyn_cast<MaskedStoreSDNode>(N)) { if (!ISD::isNormalMaskedStore(MST) || !MST->isSimple() || !MST->getOffset().isUndef()) return SDValue(); WideValue = MST->getValue(); Chain = MST->getChain(); BasePtr = MST->getBasePtr(); Mask = MST->getMask(); IsMasked = true; } else { auto *ST = cast<StoreSDNode>(N); if (!ISD::isNormalStore(ST) || !ST->isSimple() || !ST->getOffset().isUndef()) return SDValue(); WideValue = ST->getValue(); Chain = ST->getChain(); BasePtr = ST->getBasePtr(); }

SDLoc DL(N); SmallVector<SDValue, 4> ValueInterleaveOps; if (isSequentialConcatOfVectorInterleave(WideValue.getNode(), ValueInterleaveOps)) { Multiple ordinary stores can each fold the interleave if no other user needs the wide value. if (!WideValue.hasOneUse()) { Single Zip pair + multiple stp is often preferable to multiple st2 if ((WideValue.getValueType().isFixedLengthVector() && ValueInterleaveOps.size() == 2 && !DAG.shouldOptForSize()) || !llvmall_of(WideValue->users(), [&](SDNode *User) { auto *Store = dyn_cast<StoreSDNode>(User); return Store && ISD::isNormalStore(Store) && Store->isSimple() && Store->getValue() == WideValue; })) return SDValue();

Reusing the wide vector is cheaper than duplicating structured stores when every fixed-length input is already a splat. if (WideValue.getValueType().isFixedLengthVector() && ValueInterleaveOps.size() == 4 && llvm::all_of(ValueInterleaveOps, [&](SDValue Op) { return DAG.isSplatValue(Op, /*AllowUndefs=

Definition at line 28474 of file AArch64ISelLowering.cpp.

V

V

Examples

/work/as-worker-4/publish-doxygen-docs/llvm-project/llvm/include/llvm/Transforms/Utils/Local.h.

Definition at line 28572 of file AArch64ISelLowering.cpp.

Referenced by llvm::json::abbreviate(), llvm::json::abbreviateChildren(), llvm::AbsoluteDifference(), accumulateGEPOffset(), llvm::HashBuilder< hashbuilder_detail::HashCodeHasher, llvm::endianness::native >::add(), llvm::ImmutableSet< ValT, ValInfo, Canonicalize >::Factory::add(), llvm::ImmutableSetRef< ValT, ValInfo, Canonicalize >::add(), llvm::ImutAVLFactory< ImutInfo, Canonicalize >::add(), llvm::NumberedValues< T >::add(), llvm::SPIRVIRMapping::add(), llvm::ImutAVLFactory< ImutInfo, Canonicalize >::add_internal(), AddAliasScopeMetadata(), addAllGlobalValueUsers(), llvm::MachineIDFSSAUpdater::addAvailableValue(), llvm::VPHeaderPHIRecipe::addBackedgeValue(), AddCombineBUILD_VECTORToVPADDL(), llvm::SPIRVGlobalRegistry::addGlobalObject(), llvm::MemoryPhi::addIncoming(), llvm::PHINode::addIncoming(), llvm::cl::parser< DataType >::addLiteralOption(), llvm::CCState::addLoc(), addMissingPrototypes(), llvm::SymbolTableListTraits< ValueSubClass, Args >::addNodeToList(), addPipelineStateValidationInfo(), addScaledStencilTerm(), llvm::cl::list_storage< DataType, StorageClass >::addValue(), llvm::cl::list_storage< DataType, bool >::addValue(), llvm::DIEValueList::addValue(), llvm::InstructionWorklist::addValue(), addValueAffectedByCondition(), llvm::AsmParserContext::addValueReferenceAtLocation(), llvm::DebugLocEntry::addValues(), llvm::ImmutableGraphBuilder< MachineGadgetGraph >::addVertex(), adjustDown(), adjustUp(), affectedValuesAreEphemeral(), aggregatePossibleConstantValues(), aggregatePossibleConstantValuesImpl(), llvm::alignDown(), alignReassociatedOperandsByKey(), llvm::alignTo(), llvm::alignTo(), llvm::alignTo(), llvm::alignToPowerOf2(), allStructUsersAreExtractValueInsts(), allUsersAreInFunction(), alwaysAvailable(), llvm::NextUseDistance::AMDGPUNextUseAnalysisImpl, analyzeGlobalAux(), analyzeLoopUnrollCost(), analyzeModule(), analyzeParsePointLiveness(), llvm::fuzzerop::anyAggregateType(), llvm::fuzzerop::anyFloatOrVecFloatType(), llvm::fuzzerop::anyFloatType(), llvm::fuzzerop::anyIntOrVecIntType(), llvm::fuzzerop::anyIntType(), llvm::fuzzerop::anyPtrType(), llvm::fuzzerop::anyType(), anyTypeContainsFP(), llvm::fuzzerop::anyVectorType(), llvm::APFloat::APFloat(), llvm::orc::shared::TrivialSPSSequenceDeserialization< SPSElementTagT, SmallVectorImpl< T > >::append(), llvm::orc::shared::TrivialSPSSequenceDeserialization< SPSElementTagT, std::vector< T > >::append(), llvm::DWARFTypePrinter< DieType >::appendConstVolatileQualifierAfter(), llvm::DWARFTypePrinter< DieType >::appendConstVolatileQualifierBefore(), appendSpeculatableOperands(), llvm::DWARFTypePrinter< DieType >::appendTemplateParameters(), appendToUsedList(), llvm::DIExpression::ExprOperand::appendToVector(), llvm::xray::Record::apply(), llvm::jitlink::hexagon::applyFixup(), llvm::jitlink::mips::applyFixup(), llvm::sandboxir::VecUtils::areConsecutive(), llvm::PatternMatch::Argument_match< Opnd_t >::Argument_match(), llvm::VPlanPatternMatch::Argument_match< Opnd_t >::Argument_match(), llvm::ArrayRef< llvm::cfg::Update< MachineBasicBlock * > >::ArrayRef(), assertBranchOrSelectConditionHoisted(), llvm::AssumptionCache::assumptionsFor(), llvm::ImutAVLFactory< ImutInfo, Canonicalize >::balanceTree(), llvm::MIPatternMatch::bind_helper< BindTy >::bind(), llvm::PatternMatch::bind_const_intval_ty::bind_const_intval_ty(), llvm::PatternMatch::bind_immconstant_ty::bind_immconstant_ty(), llvm::MIPatternMatch::bind_ty< Class >::bind_ty(), llvm::BitCodeAbbrevOp::BitCodeAbbrevOp(), llvm::yaml::ScalarBitSetTraits< support::detail::packed_endian_specific_integral< value_type, endian, alignment >, std::enable_if_t< has_ScalarBitSetTraits< value_type >::value > >::bitset(), llvm::sandboxir::VecUtils::BndlTypeClassifier::BndlTypeClassifier(), llvm::fuzzerop::boolOrVecBoolType(), breakSelfRecursivePHI(), llvm::memprof::CallStackRadixTreeBuilder< FrameIdTy >::build(), BuildConstantFromSCEV(), buildFrameDebugInfo(), buildLogBase2(), buildSignature(), BuildSubAggregate(), llvm::byteswap(), cacheDIVar(), canCombineAsMaskOperation(), canConvertSETCCToXori(), canConvertToFMA(), canEvaluateShuffled(), llvm::InstCombiner::canFreelyInvertAllUsersOf(), canNarrowOps(), llvm::VPlanPatternMatch::canonical_widen_iv_match::canonical_widen_iv_match(), canonicalizeLaneShuffleWithRepeatedOps(), canonicalizeSaturatedAdd(), canonicalizeSaturatedSubtract(), canonicalizeShuffleMaskWithHorizOp(), canReplaceGEPIdxWithZero(), canRewriteGEPAsOffset(), canSafelyConvertTo16Bit(), canThrow(), llvm::sandboxir::LegalityAnalysis::canVectorize(), llvm::Attributor::changeAfterManifest(), llvm::Attributor::changeUseAfterManifest(), cheapToScalarize(), llvm::PatternMatch::undef_match::check(), checkAsyncFuncPointer(), checkBasicSSA(), checkConcrete(), checkConstantInt(), checkFnAllocKind(), CheckForMaskedLoad(), checkHoistValue(), checkIVUsers(), checkOverflow(), checkValueWidth(), checkValueWidth(), checkVars(), checkWFAlloc(), checkWFDealloc(), checkWFRetconPrototype(), llvm::AddrSpaceCastInst::classof(), llvm::AllocaInst::classof(), llvm::AMDGPUGWSResourcePseudoSourceValue::classof(), llvm::AnyCoroEndInst::classof(), llvm::AnyCoroIdInst::classof(), llvm::AnyCoroIdRetconInst::classof(), llvm::AnyCoroSuspendInst::classof(), llvm::AnyMemCpyInst::classof(), llvm::AnyMemIntrinsic::classof(), llvm::AnyMemMoveInst::classof(), llvm::AnyMemSetInst::classof(), llvm::AnyMemTransferInst::classof(), llvm::Argument::classof(), llvm::AssumeInst::classof(), llvm::AtomicCmpXchgInst::classof(), llvm::AtomicRMWInst::classof(), llvm::BasicBlock::classof(), llvm::BinaryConstantExpr::classof(), llvm::BinaryOperator::classof(), llvm::BinaryOpIntrinsic::classof(), llvm::BitCastInst::classof(), llvm::BitExtractInst::classof(), llvm::BitInsertInst::classof(), llvm::BlockAddress::classof(), llvm::BTF::ArrayType::classof(), llvm::BTF::Enum64Type::classof(), llvm::BTF::EnumType::classof(), llvm::BTF::StructType::classof(), llvm::CallBase::classof(), llvm::CallBrInst::classof(), llvm::CallInst::classof(), llvm::CastConstantExpr::classof(), llvm::CastInst::classof(), llvm::CatchPadInst::classof(), llvm::CatchReturnInst::classof(), llvm::CatchSwitchInst::classof(), llvm::CleanupPadInst::classof(), llvm::CleanupReturnInst::classof(), llvm::CmpInst::classof(), llvm::CmpIntrinsic::classof(), llvm::ConcreteOperator< SuperClass, Opc >::classof(), llvm::CondBrInst::classof(), llvm::Constant::classof(), llvm::ConstantAggregate::classof(), llvm::ConstantAggregateZero::classof(), llvm::ConstantArray::classof(), llvm::ConstantByte::classof(), llvm::ConstantData::classof(), llvm::ConstantDataArray::classof(), llvm::ConstantDataSequential::classof(), llvm::ConstantDataVector::classof(), llvm::ConstantExpr::classof(), llvm::ConstantFP::classof(), llvm::ConstantInt::classof(), llvm::ConstantPointerNull::classof(), llvm::ConstantPtrAuth::classof(), llvm::ConstantStruct::classof(), llvm::ConstantTargetNone::classof(), llvm::ConstantTokenNone::classof(), llvm::ConstantVector::classof(), llvm::ConstrainedFPCmpIntrinsic::classof(), llvm::ConstrainedFPIntrinsic::classof(), llvm::ConvergenceControlInst::classof(), llvm::CoroAlignInst::classof(), llvm::CoroAllocaAllocInst::classof(), llvm::CoroAllocaFreeInst::classof(), llvm::CoroAllocaGetInst::classof(), llvm::CoroAllocInst::classof(), llvm::CoroAsyncContextAllocInst::classof(), llvm::CoroAsyncContextDeallocInst::classof(), llvm::CoroAsyncEndInst::classof(), llvm::CoroAsyncResumeInst::classof(), llvm::CoroAsyncSizeReplace::classof(), llvm::CoroAwaitSuspendInst::classof(), llvm::CoroBeginInst::classof(), llvm::CoroDeadInst::classof(), llvm::CoroEndInst::classof(), llvm::CoroEndResults::classof(), llvm::CoroFrameInst::classof(), llvm::CoroFreeInst::classof(), llvm::CoroIdAsyncInst::classof(), llvm::CoroIdInst::classof(), llvm::CoroIdRetconInst::classof(), llvm::CoroIdRetconOnceInst::classof(), llvm::CoroIsInRampInst::classof(), llvm::CoroPromiseInst::classof(), llvm::CoroSaveInst::classof(), llvm::CoroSizeInst::classof(), llvm::CoroSubFnInst::classof(), llvm::CoroSuspendAsyncInst::classof(), llvm::CoroSuspendInst::classof(), llvm::CoroSuspendRetconInst::classof(), llvm::DbgAssignIntrinsic::classof(), llvm::DbgDeclareInst::classof(), llvm::DbgInfoIntrinsic::classof(), llvm::DbgLabelInst::classof(), llvm::DbgValueInst::classof(), llvm::DbgVariableIntrinsic::classof(), llvm::DSOLocalEquivalent::classof(), llvm::ExternalSymbolPseudoSourceValue::classof(), llvm::ExtractElementConstantExpr::classof(), llvm::ExtractElementInst::classof(), llvm::ExtractValueInst::classof(), llvm::FCmpInst::classof(), llvm::FenceInst::classof(), llvm::FixedStackPseudoSourceValue::classof(), llvm::FPBinaryOperator::classof(), llvm::FPExtInst::classof(), llvm::FPMathOperator::classof(), llvm::FPToSIInst::classof(), llvm::FPToUIInst::classof(), llvm::FPTruncInst::classof(), llvm::FPUnaryOperator::classof(), llvm::FreezeInst::classof(), llvm::FuncletPadInst::classof(), llvm::Function::classof(), llvm::GCProjectionInst::classof(), llvm::GCRelocateInst::classof(), llvm::GCResultInst::classof(), llvm::GCStatepointInst::classof(), llvm::GetElementPtrConstantExpr::classof(), llvm::GetElementPtrInst::classof(), llvm::GlobalAlias::classof(), llvm::GlobalIFunc::classof(), llvm::GlobalObject::classof(), llvm::GlobalValue::classof(), llvm::GlobalValuePseudoSourceValue::classof(), llvm::GlobalVariable::classof(), llvm::ICmpInst::classof(), llvm::IndirectBrInst::classof(), llvm::InlineAsm::classof(), llvm::InsertElementConstantExpr::classof(), llvm::InsertElementInst::classof(), llvm::InsertValueInst::classof(), llvm::InstrProfCallsite::classof(), llvm::InstrProfCntrInstBase::classof(), llvm::InstrProfCoverInst::classof(), llvm::InstrProfIncrementInst::classof(), llvm::InstrProfIncrementInstStep::classof(), llvm::InstrProfInstBase::classof(), llvm::InstrProfMCDCBitmapInstBase::classof(), llvm::InstrProfMCDCBitmapParameters::classof(), llvm::InstrProfMCDCTVBitmapUpdate::classof(), llvm::InstrProfTimestampInst::classof(), llvm::InstrProfValueProfileInst::classof(), llvm::Instruction::classof(), llvm::IntrinsicInst::classof(), llvm::IntToPtrInst::classof(), llvm::InvokeInst::classof(), llvm::LandingPadInst::classof(), llvm::LifetimeIntrinsic::classof(), llvm::LoadInst::classof(), llvm::MemCpyInst::classof(), llvm::MemIntrinsic::classof(), llvm::MemMoveInst::classof(), llvm::MemoryAccess::classof(), llvm::MemoryPhi::classof(), llvm::MemSetInst::classof(), llvm::MemSetPatternInst::classof(), llvm::MemTransferInst::classof(), llvm::MetadataAsValue::classof(), llvm::MinMaxIntrinsic::classof(), llvm::NoAliasScopeDeclInst::classof(), llvm::NoCFIValue::classof(), llvm::object::COFFImportFile::classof(), llvm::object::DirectX::DescriptorTableView::classof(), llvm::object::DirectX::RootConstantView::classof(), llvm::object::DirectX::RootDescriptorView::classof(), llvm::object::GOFFObjectFile::classof(), llvm::object::MachOUniversalBinary::classof(), llvm::object::OffloadBinary::classof(), llvm::object::WindowsResource::classof(), llvm::Operator::classof(), llvm::OverflowingBinaryOperator::classof(), llvm::PHINode::classof(), llvm::PoisonValue::classof(), llvm::PossiblyDisjointInst::classof(), llvm::PossiblyExactOperator::classof(), llvm::PossiblyNonNegInst::classof(), llvm::PseudoProbeInst::classof(), llvm::PtrToAddrInst::classof(), llvm::PtrToIntInst::classof(), llvm::ResumeInst::classof(), llvm::ReturnInst::classof(), llvm::sandboxir::FPMathOperator::classof(), llvm::sandboxir::IntrinsicInst::classof(), llvm::sandboxir::UnaryInstruction::classof(), llvm::SaturatingInst::classof(), llvm::SelectInst::classof(), llvm::SExtInst::classof(), llvm::ShuffleVectorConstantExpr::classof(), llvm::ShuffleVectorInst::classof(), llvm::SIToFPInst::classof(), llvm::StoreInst::classof(), llvm::StructuredAllocaInst::classof(), llvm::StructuredGEPInst::classof(), llvm::SwitchInst::classof(), llvm::TruncInst::classof(), llvm::UIToFPInst::classof(), llvm::UnaryInstruction::classof(), llvm::UnaryOperator::classof(), llvm::UncondBrInst::classof(), llvm::UndefValue::classof(), llvm::UnreachableInst::classof(), llvm::User::classof(), llvm::VAArgInst::classof(), llvm::VACopyInst::classof(), llvm::VAEndInst::classof(), llvm::VAStartInst::classof(), llvm::VPBasicBlock::classof(), llvm::VPConstant::classof(), llvm::VPConstantInt::classof(), llvm::VPHeaderPHIRecipe::classof(), llvm::VPIntrinsic::classof(), llvm::VPIRBasicBlock::classof(), llvm::VPIRValue::classof(), llvm::VPMultiDefValue::classof(), llvm::VPPhi::classof(), llvm::VPRecipeValue::classof(), llvm::VPRecipeWithIRFlags::classof(), llvm::VPReductionIntrinsic::classof(), llvm::VPRegionBlock::classof(), llvm::VPRegionValue::classof(), llvm::VPSingleDefRecipe::classof(), llvm::VPSingleDefValue::classof(), llvm::VPSymbolicValue::classof(), llvm::VPWidenInductionRecipe::classof(), llvm::VPWidenIntrinsicRecipe::classof(), llvm::WithOverflowInst::classof(), llvm::ZExtInst::classof(), CleanupPointerRootUsers(), llvm::iplist_impl< IntrusiveListT, TraitsT >::cloneFrom(), llvm::simple_ilist< Node >::cloneFrom(), cloneInstForMustTail(), CloneLoopBlocks(), collectBitParts(), collectInsertionElements(), collectInstructionDeps(), collectLeaves(), llvm::SDPatternMatch::ReassociatableOpc_match< PatternTs >::collectLeaves(), collectMinMaxFMF(), collectNocallEntryArgRegs(), collectOffsetOp(), collectPointerReturningCalls(), llvm::sandboxir::VecUtils::DeadInstructionMorgue::collectPotentiallyDeadInstrs(), collectPrivatizingConstructs(), collectShuffleElements(), collectSingleShuffleElements(), collectSRATypes(), collectSupportedLoops(), collectUsedHandles(), collectVersions(), combineAdd(), combineAnd(), combineAndOrForCcmpCtest(), combineAndShuffleNot(), combineArithReduction(), combineBinOpToReduce(), combineBitcast(), combineBitcastToBoolVector(), combineBitcastvxi1(), combineBT(), combineCommutableSHUFP(), combineConcatVectorOps(), combineEXTRACT_SUBVECTOR(), combineExtractVectorElt(), combineExtractWithShuffle(), combineFAnd(), combineFAndFNotToFAndn(), combineFAndn(), combineFMA(), combineInt64DivRemViaFPReciprocal(), combineIntDivRem(), combineLoad(), combineLogicBlendIntoConditionalNegate(), combineMul(), combinePExtWideningAddAcc(), combinePMULH(), combineProxyReg(), combineRecipe(), combineRedundantDWordShuffle(), combineScalarAndWithMaskSetcc(), combineSelect(), combineSetCC(), combineSetCCMOVMSK(), combineSext(), combineShiftOfShiftedLogic(), combineShiftRightArithmetic(), combineShiftRightLogical(), combineShlAddIAdd(), combineShuffle(), combineShuffleToAddSubOrFMAddSub(), combineSignExtendInReg(), combineSIntToFP(), combineStore(), combineStoreToNewValue(), combineStoreToValueType(), combineSub(), combineTargetShuffle(), combineTruncate(), combineTruncateWithSat(), combineTruncOfSraSext(), combineVECREDUCE_LOGIC(), combineVectorHADDSUB(), combineVectorPack(), combineVectorShiftImm(), combineVectorSizedSetCCEquality(), combineVFMADD_VLWithVFNEG_VL(), combineVSelectToBLENDV(), combineX86ShuffleChain(), combineX86ShuffleChainWithExtract(), combineZext(), combineZextSetccWithZero(), llvm::cl::GenericOptionValue::compare(), llvm::cl::OptionValueCopy< DataType >::compare(), llvm::cl::OptionValueCopy< DataType >::compare(), compareCmp(), completeEphemeralValues(), computeKnownBits(), llvm::InstCombiner::computeKnownBits(), llvm::InstCombiner::computeKnownBits(), computeKnownBitsFromCmp(), computeKnownBitsFromCond(), computeKnownBitsFromICmpCond(), computeKnownBitsFromLerpPattern(), computeKnownBitsFromOperator(), computeKnownExponentRangeFromContext(), computeKnownFPClass(), computeKnownFPClass(), computeKnownFPClassFromCond(), computeKnownFPClassFromContext(), computeLiveInValues(), computeLiveOutSeed(), ComputeNumSignBits(), ComputeNumSignBits(), ComputeNumSignBitsImpl(), computeNumSignBitsVectorConstant(), computePointerICmp(), computeUnrollAndJamCount(), llvm::JumpThreadingPass::computeValueKnownInPredecessors(), computeVariableSummary(), computeVTableFuncs(), computeZeroableShuffleElements(), concatSubVectors(), llvm::DomConditionCache::conditionsFor(), ConnectEpilog(), ConnectProlog(), llvm::ConstantArray::Constant, llvm::ConstantStruct::Constant, llvm::ConstantVector::Constant, llvm::SDPatternMatch::Constant64_match< T >::Constant64_match(), llvm::ConstantByte::ConstantByte(), llvm::ConstantFP::ConstantFP(), llvm::ConstantInt::ConstantInt(), llvm::SDPatternMatch::ConstantInt_match::ConstantInt_match(), llvm::ConstantByte::ConstantVector, llvm::ConstantFP::ConstantVector, llvm::ConstantInt::ConstantVector, constructDup(), constructSVEIntrinsicInfo(), constructTriples(), llvm::detail::DenseSetImpl< ValueT, DenseMap< ValueT, DenseSetEmpty, ValueInfoT, DenseSetPair< ValueT > > >< ValueT, ValueInfoT >::contains(), llvm::directive::VersionRange::contains(), llvm::EquivalenceClasses< BasicBlock * >::contains(), llvm::ImmutableList< T >::contains(), llvm::ImmutableSet< ValT, ValInfo, Canonicalize >::contains(), llvm::ImmutableSetRef< ValT, ValInfo, Canonicalize >::contains(), llvm::SmallSet< Register, 16 >::contains(), containsBufferFatPointers(), llvm::sandboxir::ConstantArray::Context, llvm::sandboxir::ConstantFP::Context, llvm::sandboxir::ConstantStruct::Context, llvm::sandboxir::ConstantVector::Context, convertFromScalableVector(), convertFromScalableVector(), ConvertSETCCToXori(), convertTo16Bit(), convertToScalableVector(), convertToScalableVector(), copyLifetimeIntrinsicFor(), copyMustTailReturn(), correctParamType(), llvm::detail::DenseSetImpl< ValueT, DenseMap< ValueT, DenseSetEmpty, ValueInfoT, DenseSetPair< ValueT > > >< ValueT, ValueInfoT >::count(), llvm::SmallSet< Register, 16 >::count(), llvm::ScopedHashTableVal< K, V >::Create(), llvm::sandboxir::StoreInst::create(), llvm::IRBuilderBase::CreateAdd(), llvm::IRBuilderBase::CreateAddrSpaceCast(), llvm::IRBuilderBase::CreateAggregateRet(), llvm::IRBuilderBase::CreateAnd(), llvm::IRBuilderBase::CreateAShr(), llvm::IRBuilderBase::CreateBinOpFMF(), llvm::IRBuilderBase::CreateBitCast(), llvm::IRBuilderBase::CreateBitExtract(), llvm::IRBuilderBase::CreateBitInsert(), llvm::IRBuilderBase::CreateBitOrPointerCast(), createBitOrPointerCast(), llvm::IRBuilderBase::CreateCast(), createCfiFunctionsMetadata(), llvm::IRBuilderBase::CreateExactBinOp(), llvm::JobSlot::createExplicit(), llvm::IRBuilderBase::CreateExtractElement(), llvm::IRBuilderBase::CreateExtractValue(), llvm::IRBuilderBase::CreateFAbs(), llvm::IRBuilderBase::CreateFAddFMF(), llvm::IRBuilderBase::CreateFDivFMF(), llvm::IRBuilderBase::CreateFMulFMF(), llvm::IRBuilderBase::CreateFNeg(), llvm::IRBuilderBase::CreateFNegFMF(), llvm::IRBuilderBase::CreateFPCast(), llvm::IRBuilderBase::CreateFPExt(), llvm::IRBuilderBase::CreateFPExtFMF(), llvm::IRBuilderBase::CreateFPToSI(), llvm::IRBuilderBase::CreateFPToUI(), llvm::IRBuilderBase::CreateFPTrunc(), llvm::IRBuilderBase::CreateFPTruncFMF(), llvm::IRBuilderBase::CreateFreeze(), llvm::IRBuilderBase::CreateFRemFMF(), llvm::IRBuilderBase::CreateFSubFMF(), llvm::IRBuilderBase::CreateGEP(), createGPRPairNode(), createGPRPairNodei64(), llvm::IRBuilderBase::CreateICmp(), llvm::IRBuilderBase::CreateInsertElement(), llvm::IRBuilderBase::CreateInsertValue(), llvm::IRBuilderBase::CreateIntCast(), llvm::IRBuilderBase::CreateIntCast(), llvm::IRBuilderBase::CreateIntToPtr(), llvm::IRBuilderBase::CreateLShr(), createMMXBuildVector(), llvm::IRBuilderBase::CreateMul(), llvm::IRBuilderBase::CreateNeg(), llvm::ImutAVLFactory< ImutInfo, Canonicalize >::createNode(), llvm::IRBuilderBase::CreateNot(), llvm::IRBuilderBase::CreateNoWrapBinOp(), llvm::IRBuilderBase::CreateNSWNeg(), llvm::IRBuilderBase::CreateOr(), createOverflowTuple(), createPlaceholderForShuffleVector(), llvm::IRBuilderBase::CreatePointerBitCastOrAddrSpaceCast(), llvm::IRBuilderBase::CreatePointerCast(), llvm::IRBuilderBase::CreatePtrToAddr(), llvm::IRBuilderBase::CreatePtrToInt(), createRawLoads(), createRawStores(), llvm::IRBuilderBase::CreateRet(), llvm::IRBuilderBase::CreateSDiv(), llvm::IRBuilderBase::CreateSExt(), llvm::IRBuilderBase::CreateSExtOrBitCast(), llvm::IRBuilderBase::CreateSExtOrTrunc(), llvm::IRBuilderBase::CreateShl(), llvm::IRBuilderBase::CreateShuffleVector(), llvm::IRBuilderBase::CreateShuffleVector(), llvm::IRBuilderBase::CreateSIToFP(), createSlotTracker(), llvm::IRBuilderBase::CreateSRem(), createStringMetadata(), llvm::IRBuilderBase::CreateSub(), llvm::IRBuilderBase::CreateSwitch(), createTaskWithPrivatesTy(), createTextureLoad(), createTextureStore(), llvm::IRBuilderBase::CreateTrunc(), llvm::IRBuilderBase::CreateTruncOrBitCast(), createTypedBufferLoad(), createTypedBufferStore(), llvm::IRBuilderBase::CreateUDiv(), llvm::IRBuilderBase::CreateUIToFP(), llvm::IRBuilderBase::CreateUnOp(), llvm::IRBuilderBase::CreateUnOpFMF(), llvm::IRBuilderBase::CreateURem(), createValuePack(), llvm::UnaryOperator::CreateWithCopiedFlags(), llvm::IRBuilderBase::CreateXor(), llvm::IRBuilderBase::CreateZExt(), llvm::IRBuilderBase::CreateZExtOrBitCast(), llvm::IRBuilderBase::CreateZExtOrTrunc(), llvm::SCEVPatternMatch::cst_pred_ty< Predicate >::cst_pred_ty(), llvm::ImutKeyValueInfo< T, S >::DataOfValue(), llvm::DbgVariable::DbgVariable(), llvm::DebugVariableAggregate::DebugVariableAggregate(), llvm::object::BBAddrMap::BBEntry::Metadata::decode(), llvm::BitCodeAbbrevOp::DecodeChar6(), decodeLRW16Imm8(), llvm::decodeVersion(), decompose(), llvm::DWARFTypePrinter< DieType >::decomposeConstVolatile(), decomposeImpl(), defaultComponentBroadcast(), llvm::MIPatternMatch::deferred_ty< Class >::deferred_ty(), definedInCaller(), definedInRegion(), llvm::ilist_alloc_traits< NodeTy >::deleteNode(), llvm::ilist_alloc_traits< Instruction >::deleteNode(), llvm::ilist_noalloc_traits< NodeTy >::deleteNode(), dependsOnLocalPhi(), llvm::orc::shared::SPSSerializationTraits< SPSSequence< SPSTuple< SPSString, SPSValueT > >, StringMap< ValueT > >::deserialize(), llvm::orc::shared::SPSSerializationTraits< SPSSymbolLookupSetElement, RemoteSymbolLookupSetElement >::deserialize(), llvm::orc::shared::SPSArgList<>::deserializeFromSmallVector(), deserializeSanitizerMetadata(), llvm::sandboxir::Context::detach(), llvm::sandboxir::Context::detachLLVMValue(), llvm::df_ext_iterator< T, SetTy >::df_ext_iterator(), diagnosePossiblyInvalidConstraint(), llvm::DIEAbbrevData::DIEAbbrevData(), llvm::sandboxir::LegalityAnalysis::differentBlock(), llvm::omp::DirectiveNameParser::DirectiveNameParser(), directlyImpliesPoison(), llvm::ModuleSummaryIndex::discoverNodes(), llvm::divideCeil(), llvm::divideCeilSigned(), llvm::divideFloorSigned(), llvm::divideNearest(), llvm::divideSignedWouldOverflow(), doesHistoryAllowICP(), DoFlattenLoopPair(), dominatesInsertPoint(), dominatesMergePoint(), doPromotion(), llvm::APInt::doubleToBits(), dropIntrinsicWithUnknownMetadataArgument(), llvm::sandboxir::dumpImpl(), llvm::gsym::GsymReader::dumpStatistics(), llvm::mdconst::dyn_extract(), llvm::mdconst::dyn_extract_or_null(), EltsFromConsecutiveLoads(), llvm::ir2vec::Embedding::Embedding(), llvm::ir2vec::Embedding::Embedding(), llvm::memprof::RecordWriterTrait::EmitData(), emitDebugLabelComment(), emitDebugValueComment(), llvm::memprof::RecordWriterTrait::EmitKeyDataLength(), emitRawLoad(), emitRawStore(), emitSetSwiftErrorValue(), emitSignature(), emitSignedInt64(), llvm::CodeViewDebug::LocalVarDef::emptyValue(), llvm::object::BBAddrMap::BBEntry::Metadata::encode(), llvm::object::BBAddrMap::Features::encode(), encodeFunctionSignature(), ensureValueAvailableInSuccessor(), llvm::EnumeratedArray< ValueType, Enumeration, LargestEnum, IndexType, Size >::EnumeratedArray(), llvm::dxil::ValueEnumerator::EnumerateType(), llvm::yaml::ScalarEnumerationTraits< support::detail::packed_endian_specific_integral< value_type, endian, alignment >, std::enable_if_t< has_ScalarEnumerationTraits< value_type >::value > >::enumeration(), llvm::ConstantInt::equalsInt(), llvm::sandboxir::ConstantInt::equalsInt(), llvm::detail::DenseSetImpl< ValueT, DenseMap< ValueT, DenseSetEmpty, ValueInfoT, DenseSetPair< ValueT > > >< ValueT, ValueInfoT >::erase(), llvm::EquivalenceClasses< BasicBlock * >::erase(), llvm::erase(), llvm::ScopedHashTableScope< K, V, KInfo, AllocatorTy >::erase(), llvm::SetVector< EdgeType * >::erase(), llvm::SmallSet< Register, 16 >::erase(), llvm::StringMap< EmptyStringSetTag, AllocatorTy >::erase(), llvm::EquivalenceClasses< BasicBlock * >::eraseClass(), llvm::sandboxir::Context::EraseFromParent, llvm::DomTreeBuilder::SemiNCAInfo< DomTreeT >::eval(), evalLaneExpr(), EvaluateExpression(), evaluateICmpRelation(), evaluateInDifferentElementOrder(), EvaluateInDifferentTypeImpl(), llvm::GIMatchTableExecutor::executeMatchTable(), expandAbs(), expandBinOp(), ExpandBVWithShuffles(), expandCommutativeBinOp(), expandMul(), expandUser(), llvm::mdconst::extract_or_null(), extractBooleanFlip(), extractEquivalentCondition(), extractInteger(), extractIntPart(), llvm::PatternMatch::ExtractValue_match< Ind, Opnd_t >::ExtractValue_match(), extractValues(), extractVector(), fail(), llvm::omp::FallbackVersion(), fflagsToString(), llvm::detail::DenseSetImpl< ValueT, DenseMap< ValueT, DenseSetEmpty, ValueInfoT, DenseSetPair< ValueT > > >< ValueT, ValueInfoT >::find(), llvm::detail::DenseSetImpl< ValueT, DenseMap< ValueT, DenseSetEmpty, ValueInfoT, DenseSetPair< ValueT > > >< ValueT, ValueInfoT >::find(), llvm::SPIRVIRMapping::find(), findAffectedValues(), findAffectedValues(), llvm::findAllocaForValue(), findArgParts(), findArgumentCopyElisionCandidates(), findAssumesThroughMergesForTypeTest(), findBasePointer(), findBasePointers(), FindBFIToCombineWith(), findDbgIntrinsics(), findDemandedEltsByAllUsers(), findDemandedEltsBySingleUser(), findGlobalCtors(), llvm::LiveRange::findIndexesLiveAt(), findInnerReductionPhi(), llvm::EquivalenceClasses< BasicBlock * >::findLeader(), findMemSDNode(), findMUL_LOHI(), findPHIToPartitionLoops(), llvm::vputils::findRecipe(), findRefEdges(), findSelectThroughCast(), FindSingleBitChange(), FindSingleUseMultiplyFactors(), llvm::VPlanPatternMatch::findSingleUser(), llvm::VPlanPatternMatch::findSingleUser(), findUses(), first_use_of_in_block(), llvm::yaml::CustomMappingTraits< GlobalValueSummaryMapTy >::fixAliaseeLinks(), fixupPHINodeForUnwindDest(), llvm::APInt::floatToBits(), llvm::InstCombinerImpl::fmulByZeroIsZero(), FoldBitCast(), foldBitOrderCrossLogicOp(), llvm::ConstantFolder::FoldCast(), llvm::InstSimplifyFolder::FoldCast(), llvm::IRBuilderFolder::FoldCast(), llvm::NoFolder::FoldCast(), llvm::TargetFolder::FoldCast(), foldCONCAT_VECTORS(), foldCondBranchOnValueKnownInPredecessorImpl(), llvm::ConstantFolder::FoldGEP(), llvm::TargetFolder::FoldGEP(), foldGuardedFunnelShift(), foldICmpEqualityWithOffset(), foldLoadsRecursive(), foldLogOpOfMaskedICmps(), foldLogOpOfMaskedICmpsAsymmetric(), foldMaybeUndesirableCast(), foldNeonShift(), foldPowerOf2AndShiftedMask(), foldSelectICmpAnd(), foldSelectICmpAndBinOp(), foldSelectICmpMinMax(), foldShiftOfShiftedBinOp(), foldSpliceBinOp(), foldTwoEntryPHINode(), llvm::ConstantFolder::FoldUnOpFMF(), llvm::InstSimplifyFolder::FoldUnOpFMF(), llvm::IRBuilderFolder::FoldUnOpFMF(), llvm::NoFolder::FoldUnOpFMF(), llvm::TargetFolder::FoldUnOpFMF(), foldVectorCmp(), for(), llvm::GenericUniformityAnalysisImpl< ContextT >::forgetValue(), llvm::GenericUniformityInfo< ContextT >::forgetValue(), llvm::GenericUniformityInfo< SSAContext >::forgetValue(), llvm::format_provider< codeview::GloballyHashedType >::format(), llvm::format_provider< codeview::GUID >::format(), llvm::format_provider< codeview::LocallyHashedType >::format(), llvm::format_provider< codeview::TypeIndex >::format(), llvm::format_provider< llvm::iterator_range< IterT > >::format(), llvm::format_provider< T, std::enable_if_t< support::detail::use_char_formatter< T >::value > >::format(), llvm::format_provider< T, std::enable_if_t< support::detail::use_double_formatter< T >::value > >::format(), llvm::format_provider< T, std::enable_if_t< support::detail::use_integral_formatter< T >::value > >::format(), llvm::format_provider< T, std::enable_if_t< support::detail::use_pointer_formatter< T >::value > >::format(), llvm::format_provider< T, std::enable_if_t< support::detail::use_string_formatter< T >::value > >::format(), llvm::format_provider< Twine >::format(), formLCSSAForInstructionsImpl(), llvm::SmallVectorTemplateBase< T, bool >::forward_value_param(), llvm::SmallVectorTemplateBase< T, bool >::forward_value_param(), llvm::SmallVectorTemplateBase< T, true >::forward_value_param(), forwardHandleAccesses(), gatherIncomingValuesToPhi(), generateKeySubkey(), generateNewInstTree(), generateReproducer(), llvm::GenericValue::GenericValue(), llvm::DagInit::get(), llvm::DagInit::get(), llvm::ImmutableGraphBuilder< MachineGadgetGraph >::get(), llvm::sandboxir::ConstantArray::get(), llvm::sandboxir::ConstantFP::get(), llvm::sandboxir::ConstantFP::get(), llvm::sandboxir::ConstantFP::get(), llvm::sandboxir::ConstantInt::get(), llvm::sandboxir::ConstantInt::get(), llvm::sandboxir::ConstantInt::get(), llvm::sandboxir::ConstantInt::get(), llvm::sandboxir::ConstantStruct::get(), llvm::sandboxir::ConstantVector::get(), getAccessIndices(), getAddOneOp(), getAddressSpace(), llvm::AVR::getAddressSpace(), getAlign(), llvm::SCEVExpander::getAllInsertedInstructions(), getAllocationData(), getAllocationData(), getAllocFnKind(), llvm::IRPosition::getAnchorScope(), llvm::ConstantStruct::getAnon(), llvm::ConstantStruct::getAnon(), llvm::sandboxir::ConstantStruct::getAnon(), llvm::sandboxir::ConstantStruct::getAnon(), getAnyNonZeroConstInt(), getAppleRuntimeUnrollPreferences(), llvm::TargetTransformInfoImplBase::getArithmeticInstrCost(), llvm::json::Object::getArray(), llvm::json::Object::getArray(), getAsCarry(), getAsConstantStr(), getAsMetadata(), llvm::SelectionDAG::getAssertAlign(), llvm::TargetLibraryInfo::getAsSizeT(), llvm::omp::descriptor::Directive::getAssociation(), llvm::BasicTTIImplBase< BasicTTIImpl >::getAssumedAddrSpace(), llvm::TargetMachine::getAssumedAddrSpace(), llvm::TargetTransformInfoImplBase::getAssumedAddrSpace(), llvm::Attributor::getAssumedConstant(), llvm::Attributor::getAssumedSimplified(), llvm::TrackingVH< ValueTy >::GetAsValue(), llvm::TrackingVH< ValueTy >::GetAsValue(), llvm::SelectionDAG::getAtomic(), getBaseType(), getBaseValues(), llvm::objcarc::GetBasicARCInstKind(), getBLENDIBlendMask(), llvm::sandboxir::ConstantInt::getBool(), llvm::sandboxir::ConstantInt::getBool(), llvm::json::Object::getBoolean(), getBrevLdObject(), getByteSequenceIfElementsMatch(), getCalledFunction(), llvm::AbstractCallSite::getCalledFunction(), llvm::omp::descriptor::Directive::getCategory(), llvm::omp::descriptor::ClauseSet::getClauses(), llvm::omp::descriptor::Directive::getClauses(), llvm::omp::descriptor::Modifier::getClauses(), llvm::omp::descriptor::ModifierSet::getClauses(), llvm::omp::descriptor::Directive::getClauseSets(), llvm::PBQP::Matrix::getColAsVector(), llvm::sandboxir::VecUtils::getCombinedVectorTypeFor(), getCommonSplatValue(), llvm::InstCombiner::getComplexity(), getConstantEvolvingPHI(), getConstantFP(), getConstantInt(), getConstVector(), getCopyFromParts(), getCopyFromPartsVector(), getCopyOf(), getCopyToParts(), getCopyToPartsVector(), llvm::IRPosition::getCtxI(), getDecompositionLinkRow(), getDemandAllEltsMask(), getDenseSwitchRangeReductionShift(), llvm::omp::getDescriptor(), getDIEStringAttr(), llvm::omp::descriptor::Clause::getDirectives(), llvm::omp::descriptor::ClauseSet::getDirectives(), llvm::GraphTraits< ValueInfo >::getEntryNode(), getExpandableCallee(), llvm::sandboxir::Utils::getExpectedType(), getExpressionIV(), getExtFactor(), getExtractVectorizedValue(), getFlagsOfCmpZeroFori1(), getFnAttrParsedVector(), getFPSequenceIfElementsMatch(), llvm::InstCombiner::getFreelyInverted(), llvm::InstCombiner::getFreelyInverted(), getFreezeInsertPt(), llvm::PointerIntPair< PointerTy, IntBits, IntType >::getFromOpaqueValue(), llvm::PointerIntPair< PointerTy, IntBits, IntType >::getFromOpaqueValue(), llvm::PointerUnion< PTs... >::getFromOpaqueValue(), llvm::SelectionDAG::getGatherVP(), getGetActiveLaneMask(), llvm::SelectionDAG::getGetFPEnv(), getGlobalAddressDbgOperand(), llvm::dxil::ValueEnumerator::getGlobalBasicBlockID(), llvm::instrumentor::InstrumentationConfig::getGlobalString(), llvm::Module::getGUID(), llvm::DenseMapInfo< BasicBlockEdge >::getHashValue(), llvm::DenseMapInfo< BitVector >::getHashValue(), llvm::DenseMapInfo< coverage::CounterExpression >::getHashValue(), llvm::DenseMapInfo< CSKYTargetStreamer::SymbolIndex >::getHashValue(), llvm::DenseMapInfo< directive::Version >::getHashValue(), llvm::DenseMapInfo< llvm::SmallVector< T, N > >::getHashValue(), llvm::DenseMapInfo< orc::NonOwningSymbolStringPtr >::getHashValue(), llvm::DenseMapInfo< orc::SymbolStringPtr >::getHashValue(), llvm::DenseMapInfo< PointerIntPair< PointerTy, IntBits, IntType >, void >::getHashValue(), llvm::DenseMapInfo< SmallBitVector >::getHashValue(), getHexUint(), getIdentityValue(), getIndexedTypeInternal(), llvm::IRBuilderBase::getInt1(), llvm::json::Object::getInteger(), getIntSequenceIfElementsMatch(), getInvertedVectorForFMA(), llvm::getKnownAlignment(), getKnownUndefForVectorBinop(), getKnownValueOnEdge(), llvm::EquivalenceClasses< BasicBlock * >::getLeaderValue(), llvm::SelectionDAG::getLifetimeNode(), llvm::VPlan::getLiveIn(), llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getLoadFFVP(), llvm::getLoadStorePointerOperand(), llvm::getLoadStorePointerOperand(), llvm::SelectionDAG::getLoadVP(), getLocalFunctionMetadata(), GetLocation(), getLoopEntryValue(), getLoopVariantGEPOperand(), llvm::sandboxir::VecUtils::getLowest(), getLowSubvectorMaskCompare(), llvm::SelectionDAG::getMaskedGather(), llvm::SelectionDAG::getMaskedHistogram(), llvm::SelectionDAG::getMaskedLoad(), llvm::SelectionDAG::getMaskedScatter(), llvm::SelectionDAG::getMaskedStore(), llvm::sandboxir::getMatchingBundle(), llvm::SelectionDAG::getMemIntrinsicNode(), GetMemOpSize(), getMemSetPattern16Value(), getMemSetPatternValue(), getMinimalExtentFrom(), getMinimumFPType(), getMinMaxRecurrence(), llvm::omp::descriptor::Clause::getModifiers(), llvm::omp::descriptor::ModifierSet::getModifiers(), llvm::omp::descriptor::Clause::getModifierSets(), getModuleFromVal(), GetMOForConstDbgOp(), getMOVMSK(), getName(), getNegatibleInsts(), getNegativeIsTrueBoolVec(), getNegOperand(), llvm::msgpack::Document::getNode(), llvm::msgpack::Document::getNode(), llvm::msgpack::Document::getNode(), llvm::msgpack::Document::getNode(), llvm::msgpack::Document::getNode(), llvm::msgpack::Document::getNode(), llvm::msgpack::Document::getNode(), llvm::msgpack::Document::getNode(), llvm::msgpack::Document::getNode(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNode(), getNoopInput(), getNotValue(), llvm::json::Object::getNull(), getNullFPConstForNullVal(), llvm::json::Object::getNumber(), llvm::sandboxir::Utils::getNumBits(), llvm::sandboxir::VecUtils::getNumLanes(), llvm::sandboxir::VecUtils::getNumLanes(), llvm::AArch64CC::getNZCVToSatisfyCondCode(), llvm::json::Object::getObject(), llvm::json::Object::getObject(), llvm::safestack::StackLayout::getObjectAlignment(), llvm::safestack::StackLayout::getObjectOffset(), getObjectSize(), getOffloadingArrayType(), getOneTrueElt(), llvm::Operator::getOpcode(), getOptimizationFlags(), llvm::cl::generic_parser_base::getOptionValue(), llvm::VPlan::getOrAddLiveIn(), llvm::VPlan::getOrAddLiveIn(), llvm::ConstantUniqueMap< ConstantArray >::getOrCreate(), getOrCreateCachedOptional(), llvm::EquivalenceClasses< BasicBlock * >::getOrInsertLeaderValue(), getParent(), getPermuteMask(), llvm::getPointerOperand(), llvm::getPointerOperand(), llvm::TargetTransformInfoImplCRTPBase< T >::getPointersChainCost(), getPointerSize(), llvm::IRPosition::getPositionKind(), getPotentialCopiesOfMemoryValue(), llvm::BasicTTIImplBase< BasicTTIImpl >::getPredicatedAddrSpace(), llvm::TargetMachine::getPredicatedAddrSpace(), llvm::TargetTransformInfoImplBase::getPredicatedAddrSpace(), llvm::PredicateInfo::getPredicateInfoFor(), getPRMTSelector(), llvm::AVR::getProgramMemoryBank(), llvm::omp::descriptor::Clause::getProperties(), llvm::omp::descriptor::ClauseSet::getProperties(), llvm::omp::descriptor::Directive::getProperties(), llvm::omp::descriptor::Modifier::getProperties(), llvm::omp::descriptor::ModifierSet::getProperties(), llvm::omp::getProperties(), llvm::SelectionDAG::getPseudoProbeNode(), getRange(), getRangeForNUWMulSquare(), GetRangeFromMetadata(), llvm::objcarc::GetRCIdentityRoot(), llvm::objcarc::GetRCIdentityRoot(), getReassocColumnsQuality(), getRecipesForUncountableExit(), getReducedType(), llvm::PBQP::Matrix::getRowAsVector(), getRowForLessEqual(), getRowFromCol(), GetRunOfOnesStartingAt(), getSavedReg(), getScalarValueForVectorElement(), llvm::SelectionDAG::getScatterVP(), llvm::ScalarEvolution::getSCEVAtExit(), getSequenceIfElementsMatch(), llvm::SelectionDAG::getSetFPEnv(), getShape(), llvm::object::ELFFile< ELFT >::getSHNDXTable(), getShuffleHalfVectors(), getSign32(), llvm::ConstantByte::getSigned(), llvm::ConstantByte::getSigned(), llvm::ConstantInt::getSigned(), llvm::ConstantInt::getSigned(), llvm::sandboxir::ConstantInt::getSigned(), llvm::sandboxir::ConstantInt::getSigned(), llvm::ARM_AM::getSOImmTwoPartFirst(), llvm::ARM_AM::getSOImmTwoPartSecond(), llvm::BaseRecord::getSpellingForIdentifier(), getSqrtCall(), llvm::SelectionDAG::getStore(), llvm::SelectionDAG::getStoreVP(), getStrideAndModOffsetOfGEP(), llvm::SelectionDAG::getStridedLoadVP(), llvm::SelectionDAG::getStridedStoreVP(), getStrideFromPointer(), llvm::json::Object::getString(), GetStringLengthH(), llvm::object::ELFFile< ELFT >::getStringTable(), llvm::getStringValueFromReg(), getSubVectorSrc(), llvm::ARM_AM::getT2SOImmTwoPartFirst(), llvm::ARM_AM::getT2SOImmValRotate(), llvm::ARM_AM::getT2SOImmValRotateVal(), llvm::ARM_AM::getT2SOImmValSplatVal(), getTargetShuffleAndZeroables(), llvm::ARM_AM::getThumbImmNonShiftedVal(), getTrueOrFalseValue(), getTrueOrFalseValue(), llvm::SelectionDAG::getTruncStoreVP(), llvm::sandboxir::ConstantStruct::getTypeForElements(), llvm::sandboxir::ConstantStruct::getTypeForElements(), llvm::objcarc::GetUnderlyingObjCPtr(), llvm::objcarc::GetUnderlyingObjCPtrCached(), llvm::getUnderlyingObject(), getUnderLyingObjectForBrevLdIntr(), getUnderlyingObjectFromInt(), getUnderlyingObjectsForInstr(), getUnsignedMonotonicValues(), llvm::SSAUpdaterImpl< UpdaterT >::GetValue(), llvm::instrumentor::InstrumentationOpportunity::getValue(), llvm::sandboxir::Context::getValue(), llvm::sandboxir::Context::getValue(), llvm::ScopedHashTableVal< K, V >::getValue(), llvm::ScopedHashTableVal< K, V >::getValue(), getValueFwdRef(), llvm::dxil::ValueEnumerator::getValueID(), getValueOnFirstIteration(), llvm::TargetTransformInfoImplBase::getValueUniformity(), getVectorProduct(), getVectorShuffle(), llvm::omp::descriptor::Descriptor< details::Clause >::getVersions(), llvm::slpvectorizer::BaseShuffleAnalysis::getVF(), getWideningSpread(), globalHasNVVMAnnotation(), llvm::GlobalStatus::GlobalStatus(), llvm::SPIRV::handle(), handleDanglingVariadicDebugInfo(), HandleInlinedEHPad(), handleIntToPtr(), HandleMergeInputChains(), handleOtherCmpSelSimplifications(), handlePhiDef(), llvm::InstructionWorklist::handleUseCountDecrement(), llvm::mdconst::hasa(), llvm::AArch64TargetLowering::hasAndNotCompare(), llvm::ARMTargetLowering::hasAndNotCompare(), llvm::CallBase::hasArgument(), llvm::sandboxir::CallBase::hasArgument(), hasCFUser(), hasCheapXorImmediateThatNeedsCondCmpReg(), hasConcreteDef(), hasConcreteDefImpl(), llvm::PBQP::hash_value(), llvm::PBQP::hash_value(), hasOnlyArtificialUses(), hasOnlySelectUsers(), llvm::PBQP::hasRegisterOptions(), hasUndefContents(), hasZeroSignBit(), haveEfficientBuildVectorPattern(), haveNoCommonBitsSetSpecialCases(), hoistConditionalLoadsStores(), hoistGEP(), hoistValue(), identifyCallee(), llvm::idf_ext_iterator< T, SetTy >::idf_ext_iterator(), llvm::idf_ext_iterator< T, SetTy >::idf_ext_iterator(), llvm::idf_iterator< T, SetTy, External >::idf_iterator(), impliesPoison(), InBlock(), incDecVectorConstant(), INITIALIZE_PASS(), INITIALIZE_PASS(), llvm::yaml::ScalarTraits< support::detail::packed_endian_specific_integral< value_type, endian, alignment >, std::enable_if_t< has_ScalarTraits< value_type >::value > >::input(), llvm::yaml::CustomMappingTraits< GlobalValueSummaryMapTy >::inputOne(), llvm::yaml::CustomMappingTraits< std::map< std::vector< uint64_t >, WholeProgramDevirtResolution::ByArg > >::inputOne(), llvm::yaml::CustomMappingTraits< std::map< uint64_t, WholeProgramDevirtResolution > >::inputOne(), llvm::yaml::CustomMappingTraits< TypeIdSummaryMapTy >::inputOne(), llvm::IRBuilderBase::Insert(), llvm::AllocatorList< T, BumpPtrAllocator >< Addend >::insert(), llvm::AllocatorList< T, BumpPtrAllocator >< Addend >::insert(), llvm::detail::DenseSetImpl< ValueT, DenseMap< ValueT, DenseSetEmpty, ValueInfoT, DenseSetPair< ValueT > > >< ValueT, ValueInfoT >::insert(), llvm::detail::DenseSetImpl< ValueT, DenseMap< ValueT, DenseSetEmpty, ValueInfoT, DenseSetPair< ValueT > > >< ValueT, ValueInfoT >::insert(), llvm::ScopedHashTable< MachineInstr *, unsigned, MachineInstrExpressionTrait, AllocatorTy >::insert(), llvm::SmallSet< Register, 16 >::insert(), llvm::SmallSet< Register, 16 >::insert(), llvm::TinyPtrVector< ReachingDef >::insert(), llvm::TinyPtrVector< ReachingDef >::insert(), insert1BitVector(), llvm::detail::DenseSetImpl< ValueT, DenseMap< ValueT, DenseSetEmpty, ValueInfoT, DenseSetPair< ValueT > > >< ValueT, ValueInfoT >::insert_as(), llvm::detail::DenseSetImpl< ValueT, DenseMap< ValueT, DenseSetEmpty, ValueInfoT, DenseSetPair< ValueT > > >< ValueT, ValueInfoT >::insert_as(), llvm::DenseMapBase< densemap::detail::DenseMapStorage< BucketT >, KeyT, ValueT, KeyInfoT, BucketT >::insert_or_assign(), llvm::DenseMapBase< densemap::detail::DenseMapStorage< BucketT >, KeyT, ValueT, KeyInfoT, BucketT >::insert_or_assign(), llvm::MapVector< StringRef, HashData >::insert_or_assign(), llvm::MapVector< StringRef, HashData >::insert_or_assign(), llvm::StringMap< EmptyStringSetTag, AllocatorTy >::insert_or_assign(), llvm::Module::insertGUID(), insertIfNamed(), insertInteger(), llvm::ScopedHashTable< MachineInstr *, unsigned, MachineInstrExpressionTrait, AllocatorTy >::insertIntoScope(), InsertNewValueIntoMap(), insertParsePoints(), insertVector(), llvm::RawInstrProf::ProfileData< IntPtrT >::INSTR_PROF_DATA(), llvm::instrumentor::InstrumentationOpportunity::instrument(), llvm::InstCostVisitor::InstVisitor< InstCostVisitor, Constant * >, llvm::is_contained(), isAbsoluteValueULEOne(), isAddLike(), isAggrConstForceInt32(), isAllocDisjoint(), llvm::GenericSSAContext< Function >::isAlwaysUniform(), isAMXIntrinsic(), isAnyConstantBuildVector(), isArgumentOrArgumentLike(), isAspectNeeded(), isAvailableAtEndOf(), isBigEndianBitShift(), isBswapIntrinsicCall(), isByValArg(), isCFMulFromFMAddSub(), isConstantPowerOf2(), isConstantPowerOfTwo(), isCopyOfAPHI(), isCopyOfPHI(), llvm::omp::isDataSharingAttributeClause(), llvm::SuspendCrossingInfo::isDefinitionAcrossSuspend(), llvm::SuspendCrossingInfo::isDefinitionAcrossSuspend(), isDereferenceableAndAlignedPointer(), isDereferenceableArg(), isDereferenceableForAllocaSize(), isDisabledEnv(), llvm::GenericUniformityAnalysisImpl< ContextT >::isDivergent(), llvm::GenericUniformityInfo< ContextT >::isDivergentAtDef(), llvm::GenericUniformityInfo< SSAContext >::isDivergentAtDef(), llvm::ImutAVLTree< ValInfo, Canonicalize >::isElementEqual(), llvm::IVUsers::isEphemeral(), isEphemeralValueOf(), llvm::APFloat::isExactlyValue(), llvm::ConstantFP::isExactlyValue(), llvm::ConstantFPSDNode::isExactlyValue(), llvm::sandboxir::ConstantFP::isExactlyValue(), llvm::sandboxir::ConstantFP::isExactlyValue(), llvm::FunctionLoweringInfo::isExportedInst(), isFMulAddCandidate(), isFoldableAsShuffle(), isFoldableRMW(), isFoldedIntoRsqrtEstimate(), isFreeConcat(), llvm::InstCombiner::isFreeToInvert(), llvm::InstCombiner::isFreeToInvert(), isFreeToSplitVector(), isFrexpExp(), isGCValue(), llvm::isGuaranteedNotToBePoison(), isGuaranteedNotToBeUndefOrPoison(), isICmpTrue(), llvm::PatternMatch::immconstant_ty::isImmConstant(), isImplicitDef(), isImpliedToBeAPowerOfTwoFromCond(), isIncomingOfPHI(), isInertARCValue(), isInterestingPHIIncomingValue(), llvm::isIntOrFPConstant(), isIntOrIntVectorValue(), isInvariantLoad(), isIVIncrement(), llvm::RawLocationWrapper::isKillLocation(), isKnownBase(), isKnownMonotonic(), isKnownNonConstant(), isKnownNonEqualFromContext(), isKnownNonNaN(), isKnownNonNullFromDominatingCondition(), isKnownNonPositive(), isKnownNonZero(), isKnownNonZero(), isKnownNonZeroFromAssume(), isKnownNoWrap(), llvm::InstCombiner::isKnownToBeAPowerOfTwo(), isKnownTypeIdMember(), isLegalAddressImmediate(), isLegalT1AddressImmediate(), isLegalT2AddressImmediate(), isLoadCombineCandidate(), isLoopExitTestBasedOn(), isLoopHeaderPHI(), isLowerSaturatingConditional(), isMaskableNode(), isMaskOrZero(), isMImm(), isMultipleOfN(), isMultiRegisterAggregate(), isNeg(), isNeverEqualToUnescapedAlloc(), isNoAliasOrByValArgument(), isNonEscapingGlobalNoAliasWithLoad(), IsNonLocalValue(), IsNOT(), isNOT(), isNotExclusivelyConstantDerived(), isNullFPScalarOrVectorConst(), llvm::objcarc::IsNullOrUndef(), isNVCastToHalfWidthElements(), llvm::objcarc::IsObjCIdentifiedObject(), isObjectSize(), isObjectSizeLessThanOrEq(), isObjectSmallerThan(), isOneVector(), isOnlyCopiedFromConstantMemory(), isOnlyUsedInComparisonWithZero(), isOnlyUsedInEqualityComparison(), isOriginalBaseResult(), isPointerAlwaysReplaceable(), isPointerOperand(), isPointerPairOperand(), llvm::CastInfo< To, PointerUnion< PTs... > >::isPossible(), isPowerOf2Constant(), isPreserveArrayIndex(), isPreserveStructIndex(), isPreserveUnionIndex(), llvm::omp::isPrivatizingClause(), llvm::AVR::isProgramMemoryAccess(), llvm::AVR::isProgramMemoryAddress(), isPtrKnownNeverNull(), isReassociableOp(), isReassociableOp(), llvm::SmallVectorTemplateCommon< T, typename >::isReferenceToRange(), llvm::SmallVectorTemplateCommon< T, typename >::isReferenceToStorage(), isRepeatedByteSequence(), isRepeatedByteSequence(), IsSafeComputationToRemove(), isSameCompare(), isShapePreserving(), isShuffleFoldableLoad(), isSignExtInReg(), isSimm12Constant(), isSimm7(), llvm::IsSingleCodeUnitUTF16Codepoint(), llvm::IsSingleCodeUnitUTF32Codepoint(), llvm::IsSingleCodeUnitUTF8Codepoint(), llvm::ARM_AM::isSOImmTwoPartVal(), llvm::ARM_AM::isSOImmTwoPartValNeg(), isSplat(), isSplatShuffle(), isSpvAggrPlaceholder(), isSupportedExtend(), isSwiftError(), llvm::ARM_AM::isT2SOImmTwoPartVal(), isTargetConstant(), isThreadID(), llvm::ARM_AM::isThumbImm16ShiftedVal(), llvm::ARM_AM::isThumbImmShiftedVal(), isTriviallyUniform(), isTruncToI1(), isTruncWithZeroHighBitsInput(), isTruncWithZeroHighBitsInput(), isUniformAcrossVFsAndUFsImpl(), llvm::GenericUniformityInfo< SSAContext >::isUniformAtDef(), isUpperSubvectorUndef(), isUsedAsMemCpySource(), isUsedWithinShuffleVector(), llvm::ValueHandleBase::isValid(), isValidEnumValue(), isValidFlags(), isValidVisibilityForLinkage(), isValInBlock(), llvm::sandboxir::ConstantFP::isValueValidForType(), llvm::sandboxir::ConstantInt::isValueValidForType(), llvm::sandboxir::ConstantInt::isValueValidForType(), isVECTOR_SHUFFLE_SPLATI(), isVectorPromotionViable(), isWorthFoldingSHL(), isZero(), isZero(), llvm::JobSlot::JobserverClientImpl, llvm::ImutAVLFactory< ImutInfo, Canonicalize >::joinTrees(), llvm::ImutKeyValueInfo< T, S >::KeyOfValue(), llvm::FunctionTypeKeyInfo::KeyTy::KeyTy(), llvm::OffsetSpan::known(), llvm::SizeOffsetAPInt::known(), llvm::SizeOffsetValue::known(), llvm::SizeOffsetWeakTrackingVH::known(), llvm::LaneBitmask::LaneBitmask(), legacyCSE(), legalizeI8Memory(), legalizeInteger(), llvm::LetRecord::LetRecord(), likeBitCastFromVector(), LinearizeExprTree(), llvm::LiveRange::LiveRangeUpdater, LLVM_ATTRIBUTE_C_DEPRECATED(), LLVM_ATTRIBUTE_C_DEPRECATED(), LLVM_ATTRIBUTE_C_DEPRECATED(), LLVMAddTargetDependentFunctionAttr(), LLVMBuildFNeg(), LLVMBuildNeg(), LLVMBuildNot(), LLVMBuildNSWNeg(), LLVMBuildNUWNeg(), LLVMBuildRet(), LLVMBuildSwitch(), LLVMCanValueUseFastMathFlags(), LLVMConstArray(), LLVMConstArray2(), LLVMCreateStringAttribute(), LLVMGetAlignment(), LLVMGetComdat(), LLVMGetFirstUse(), LLVMGetMDNodeNumOperands(), LLVMGetMDNodeOperands(), LLVMGetMDString(), LLVMGetNumOperands(), LLVMGetOperand(), LLVMGetOperandUse(), LLVMGetParamParent(), LLVMGetValueName2(), LLVMInitializeAArch64AsmParser(), LLVMInitializeAArch64Target(), LLVMIsABranchInst(), LLVMMDNodeInContext(), LLVMReplaceMDNodeOperandWith(), LLVMSetAlignment(), LLVMSetComdat(), LLVMValueAsMetadata(), llvm::longestCommonSequence(), lookThroughShuffles(), llvm::ScopedHashTable< MachineInstr *, unsigned, MachineInstrExpressionTrait, AllocatorTy >::lookup(), lookupConstant(), lower1BitShuffle(), lower256BitShuffle(), lower512BitShuffle(), lowerAllowChecks(), LowerBITCAST(), lowerBitreverseShuffle(), LowerBSWAP(), lowerBUILD_VECTOR(), LowerBUILD_VECTOR_i1(), LowerBUILD_VECTORAsVariablePermute(), lowerBuildVectorAsBroadcast(), LowerBuildVectorAsInsert(), lowerBuildVectorOfConstants(), lowerBuildVectorToBitOp(), LowerBuildVectorv16i8(), lowerBuildVectorViaDominantValues(), LowerCTLZ(), lowerCTLZCTPOP(), LowerCTPOP(), lowerDXILDebugInfo(), LowerHorizontalByteSum(), lowerPredicateInterleaveWithPackedVectors(), lowerRV32HorizontalMul64(), LowerShift(), lowerShuffleAsBitMask(), lowerShuffleAsBlendAndPermute(), lowerShuffleAsBlendOfPSHUFBs(), lowerShuffleAsBroadcast(), lowerShuffleAsLanePermuteAndPermute(), lowerShuffleAsShift(), lowerShuffleAsSpecificExtension(), lowerShuffleAsZeroOrAnyExtend(), lowerShuffleWithPSHUFB(), LowerSIGN_EXTEND_Mask(), lowerStatepointMetaArgs(), lowerStoreI1(), lowerSTOREVector(), lowerUINT_TO_FP_vXi32(), lowerV16F32Shuffle(), lowerV16I16Shuffle(), lowerV16I32Shuffle(), lowerV16I8Shuffle(), lowerV2F64Shuffle(), lowerV2I64Shuffle(), lowerV32I16Shuffle(), lowerV32I8Shuffle(), lowerV4F32Shuffle(), lowerV4F64Shuffle(), lowerV4I32Shuffle(), lowerV4I64Shuffle(), lowerV64I8Shuffle(), lowerV8F16Shuffle(), lowerV8F32Shuffle(), lowerV8F64Shuffle(), lowerV8I16GeneralSingleInputShuffle(), lowerV8I16Shuffle(), lowerV8I32Shuffle(), lowerV8I64Shuffle(), LowerVECTOR_SHUFFLE(), lowerVECTOR_SHUFFLE(), lowerVECTOR_SHUFFLEAsShift(), lowerVECTOR_SHUFFLEAsZeroOrAnyExtend(), LowerVectorAllEqual(), LowerVectorArith(), LowerVSETCC(), llvm::PatternMatch::m_BasicBlock(), llvm::PatternMatch::m_BasicBlock(), llvm::VPlanPatternMatch::m_CanonicalWidenIV(), llvm::PatternMatch::m_CheckedFp(), llvm::PatternMatch::m_CheckedInt(), llvm::PatternMatch::m_ConstantInt(), llvm::SDPatternMatch::m_ConstInt(), llvm::SDPatternMatch::m_ConstInt(), llvm::SDPatternMatch::m_ConstInt(), llvm::PatternMatch::m_Deferred(), llvm::PatternMatch::m_Deferred(), llvm::SDPatternMatch::m_Deferred(), llvm::VPlanPatternMatch::m_Deferred(), llvm::SDPatternMatch::m_ElementCount(), llvm::PatternMatch::m_ExtractValue(), llvm::PatternMatch::m_Finite(), llvm::PatternMatch::m_FiniteNonZero(), llvm::PatternMatch::m_LowBitMask(), llvm::PatternMatch::m_LowBitMaskOrZero(), llvm::PatternMatch::m_MaxSignedValue(), llvm::PatternMatch::m_Neg(), llvm::SDPatternMatch::m_Neg(), llvm::PatternMatch::m_NegatedPower2(), llvm::PatternMatch::m_NegatedPower2OrZero(), llvm::PatternMatch::m_Negative(), llvm::PatternMatch::m_NonNegative(), llvm::PatternMatch::m_NonPositive(), llvm::PatternMatch::m_Not(), llvm::SDPatternMatch::m_Not(), llvm::PatternMatch::m_NotForbidPoison(), llvm::PatternMatch::m_NSWNeg(), llvm::MIPatternMatch::m_PosZeroFP(), llvm::PatternMatch::m_Power2(), llvm::PatternMatch::m_Power2OrZero(), llvm::VPlanPatternMatch::m_ReductionPhi(), llvm::SCEVPatternMatch::m_SCEV(), llvm::SCEVPatternMatch::m_SCEV(), llvm::SCEVPatternMatch::m_scev_Add(), llvm::SCEVPatternMatch::m_scev_Mul(), llvm::SCEVPatternMatch::m_scev_SpecificInt(), llvm::SCEVPatternMatch::m_scev_SpecificSInt(), llvm::SCEVPatternMatch::m_SCEVConstant(), llvm::SCEVPatternMatch::m_SCEVUnknown(), llvm::PatternMatch::m_Signum(), llvm::PatternMatch::m_Specific(), llvm::PatternMatch::m_SpecificFP(), llvm::SDPatternMatch::m_SpecificFP(), llvm::SDPatternMatch::m_SpecificFP(), llvm::PatternMatch::m_SpecificInt(), llvm::PatternMatch::m_SpecificInt(), llvm::SDPatternMatch::m_SpecificInt(), llvm::SDPatternMatch::m_SpecificInt(), llvm::VPlanPatternMatch::m_SpecificInt(), llvm::PatternMatch::m_SpecificIntAllowPoison(), llvm::PatternMatch::m_SpecificIntAllowPoison(), llvm::SDPatternMatch::m_SpecificNeg(), llvm::VPlanPatternMatch::m_SpecificSInt(), llvm::PatternMatch::m_SpecificType(), llvm::PatternMatch::m_SpecificType(), llvm::PatternMatch::m_StrictlyPositive(), llvm::PatternMatch::m_Value(), llvm::PatternMatch::m_Value(), llvm::PatternMatch::m_Value(), llvm::PatternMatch::m_Value(), llvm::VPlanPatternMatch::m_VPInstruction(), llvm::VPlanPatternMatch::m_VPIRValue(), llvm::VPlanPatternMatch::m_VPSingleDefRecipe(), llvm::VPlanPatternMatch::m_VPValue(), llvm::VPlanPatternMatch::m_VPValue(), llvm::MachineConstantPoolEntry::MachineConstantPoolEntry(), llvm::MachineConstantPoolEntry::MachineConstantPoolEntry(), makeExactMulNSWRegion(), makeExactMulNUWRegion(), makeGreaterThan(), makeLessThan(), mapOptOrNull(), llvm::MapValue(), llvm::MapValue(), mapValueToSlot(), llvm::ValueLatticeElement::markConstant(), llvm::ValueLatticeElement::markNotConstant(), llvm::InstCombiner::MaskedValueIsZero(), deinterleaving_tbl_match::match(), llvm::MIPatternMatch::bind_ty< Class >::match(), llvm::MIPatternMatch::deferred_helper< BindTy >::match(), llvm::MIPatternMatch::deferred_ty< Class >::match(), llvm::PatternMatch::AllowFmf_match< SubPattern_t, Flag >::match(), llvm::PatternMatch::AnyBinaryOp_match< LHS_t, RHS_t, Commutable >::match(), llvm::PatternMatch::AnyOps_match< Opcode, OperandTypes >::match(), llvm::PatternMatch::AnyUnaryOp_match< OP_t >::match(), llvm::PatternMatch::ap_match< APTy >::match(), llvm::PatternMatch::apf_pred_ty< Predicate >::match(), llvm::PatternMatch::api_pred_ty< Predicate >::match(), llvm::PatternMatch::Argument_match< Opnd_t >::match(), llvm::PatternMatch::BinaryOp_match< LHS_t, RHS_t, Opcode, Commutable >::match(), llvm::PatternMatch::BinaryOp_match< LHS_t, RHS_t, Opcode, Commutable >::match(), llvm::PatternMatch::bind_const_intval_ty::match(), llvm::PatternMatch::bind_immconstant_ty::match(), llvm::PatternMatch::BinOpPred_match< LHS_t, RHS_t, Predicate, Commutable >::match(), llvm::PatternMatch::br_match::match(), llvm::PatternMatch::brc_match< Cond_t, TrueBlock_t, FalseBlock_t >::match(), llvm::PatternMatch::CastInst_match< Op_t, Class >::match(), llvm::PatternMatch::CastOperator_match< Op_t, Opcode >::match(), llvm::PatternMatch::CmpClass_match< LHS_t, RHS_t, Class, Commutable >::match(), llvm::PatternMatch::CommutativeBinaryIntrinsic_match< IntrID, LHS, RHS >::match(), llvm::PatternMatch::constantint_match< Val >::match(), llvm::PatternMatch::ContainsMatchingVectorElement_match< SPTy >::match(), llvm::PatternMatch::cstval_pred_ty< Predicate, ConstantInt, AllowPoison >::match(), llvm::PatternMatch::DisjointOr_match< LHS, RHS, Commutable >::match(), llvm::PatternMatch::ElementWiseBitCast_match< Op_t >::match(), llvm::PatternMatch::Exact_match< SubPattern_t >::match(), llvm::PatternMatch::ExtractValue_match< Ind, Opnd_t >::match(), llvm::PatternMatch::FMaxMin_match< LHS_t, RHS_t, Pred_t >::match(), llvm::PatternMatch::FNeg_match< Op_t, AllowFSubNegZero >::match(), llvm::PatternMatch::ICmpLike_match< LHS_t, RHS_t >::match(), llvm::PatternMatch::InsertValue_match< Ind, T0, T1 >::match(), llvm::PatternMatch::IntrinsicID_match::match(), llvm::PatternMatch::IntrinsicIDs_match< IntrIDs >::match(), llvm::PatternMatch::is_zero< AllowPoison >::match(), llvm::PatternMatch::LoadSimple_match< OpTy >::match(), llvm::PatternMatch::LogicalOp_match< LHS, RHS, Opcode, Commutable >::match(), llvm::PatternMatch::match(), llvm::PatternMatch::match_immconstant_ty::match(), llvm::PatternMatch::NNegZExt_match< Op_t >::match(), llvm::PatternMatch::NoWrapTrunc_match< Op_t, WrapFlags >::match(), llvm::PatternMatch::OneOps_match< T0, Opcode >::match(), llvm::PatternMatch::OneUse_match< SubPattern_t >::match(), llvm::PatternMatch::OverflowingBinaryOp_match< LHS_t, RHS_t, Opcode, WrapFlags, Commutable >::match(), llvm::PatternMatch::PtrAdd_match< PointerOpTy, OffsetOpTy >::match(), llvm::PatternMatch::PtrToIntSameSize_match< Op_t >::match(), llvm::PatternMatch::SelectLike_match< CondTy, LTy, RTy >::match(), llvm::PatternMatch::ShiftLike_match< LHS_t, Opcode >::match(), llvm::PatternMatch::Shuffle_match< T0, T1, T2 >::match(), llvm::PatternMatch::Signum_match< Opnd_t >::match(), llvm::PatternMatch::specific_bbval::match(), llvm::PatternMatch::specific_fpval::match(), llvm::PatternMatch::specific_intval64< AllowPoison >::match(), llvm::PatternMatch::specific_intval< AllowPoison >::match(), llvm::PatternMatch::SpecificBinaryOp_match< LHS_t, RHS_t, Commutable >::match(), llvm::PatternMatch::SpecificCmpClass_match< LHS_t, RHS_t, Class, Commutable >::match(), llvm::PatternMatch::SpecificType_match< Pattern >::match(), llvm::PatternMatch::specificval_ty::match(), llvm::PatternMatch::Splat_match< SubPattern_t >::match(), llvm::PatternMatch::ThreeOps_match< T0, T1, T2, Opcode, CommutableOp2Op3 >::match(), llvm::PatternMatch::TwoOps_match< T0, T1, Opcode >::match(), llvm::PatternMatch::UAddWithOverflow_match< LHS_t, RHS_t, Sum_t >::match(), llvm::PatternMatch::undef_match::match(), llvm::PatternMatch::XorLike_match< LHS, RHS >::match(), llvm::PatternMatchHelpers::match_bind< Ty >::match(), llvm::PatternMatchHelpers::match_bind< SCEVUseT< SCEVPtrT > >::match(), llvm::PatternMatchHelpers::match_combine_and< Ty, Tys... >::match(), llvm::PatternMatchHelpers::match_combine_or< Ty, Tys... >::match(), llvm::PatternMatchHelpers::match_deferred< Ty >::match(), llvm::PatternMatchHelpers::match_isa< To >::match(), llvm::PatternMatchHelpers::match_unless< Ty >::match(), llvm::SDPatternMatch::Constant64_match< T >::match(), llvm::SDPatternMatch::SpecificFP_match::match(), llvm::VPlanPatternMatch::Argument_match< Opnd_t >::match(), llvm::VPlanPatternMatch::canonical_iv_match::match(), llvm::VPlanPatternMatch::canonical_widen_iv_match::match(), llvm::VPlanPatternMatch::Cmp_match< Op0_t, Op1_t, Opcodes >::match(), llvm::VPlanPatternMatch::Cmp_match< Op0_t, Op1_t, Opcodes >::match(), llvm::VPlanPatternMatch::GetElementPtr_match::match(), llvm::VPlanPatternMatch::header_mask_match::match(), llvm::VPlanPatternMatch::IntrinsicID_match::match(), llvm::VPlanPatternMatch::LiveIn_match::match(), llvm::VPlanPatternMatch::Load_match< Addr_t, Mask_t >::match(), llvm::VPlanPatternMatch::match(), llvm::VPlanPatternMatch::match_poison::match(), llvm::VPlanPatternMatch::OneUse_match< SubPattern_t >::match(), llvm::VPlanPatternMatch::Recipe_match< std::tuple< OpTys... >, Opcode, false, VPWidenRecipe, VPReplicateRecipe, VPWidenCastRecipe, VPInstruction >::match(), llvm::VPlanPatternMatch::RemoveMask_match< Op0_t, Op1_t >::match(), llvm::VPlanPatternMatch::SpecificCmp_match< Op0_t, Op1_t, Opcodes >::match(), llvm::VPlanPatternMatch::SpecificCmp_match< Op0_t, Op1_t, Opcodes >::match(), llvm::VPlanPatternMatch::Store_match< Addr_t, Val_t, Mask_t >::match(), uitofp_tbl_match::match(), llvm::PatternMatchHelpers::match_bind< Ty >::match_bind(), llvm::PatternMatchHelpers::match_bind< SCEVUseT< SCEVPtrT > >::match_bind(), llvm::PatternMatchHelpers::match_deferred< Ty >::match_deferred(), llvm::VPlanPatternMatch::match_fn(), llvm::PatternMatch::cstval_pred_ty< Predicate, ConstantInt, AllowPoison >::match_impl(), matchAndOrChain(), MatchBinaryOp(), matchBinaryShuffle(), matchBitmaskMul(), llvm::fuzzerop::matchFirstLengthWAnyType(), llvm::fuzzerop::matchFirstType(), matchIntPart(), matchMergedBFX(), matchPopCountBytes(), matchScalarInAggregate(), llvm::fuzzerop::matchScalarOfFirstType(), llvm::fuzzerop::matchSecondType(), llvm::matchSelectPattern(), matchSelectWithOptionalNotCond(), matchShuffleAsEXTRQ(), matchSubIntegerPackFromVector(), MatchVectorAllEqualTest(), matchZExtedSubInteger(), llvm::ValueMaterializer::materialize(), MaximallyReplicateSubImmediate(), mayBeVisibleThroughUnwinding(), mayHaveSignedWrap(), mayLookThrough(), llvm::BitTracker::BitValue::meet(), llvm::EquivalenceClasses< BasicBlock * >::members(), mergeI16Unpacks(), mergeTwoVectors(), llvm::MinAlign(), llvm::mod(), llvm::Module::ModuleFlagEntry::ModuleFlagEntry(), movePHIValuesToInsertedBlock(), llvm::MutableArrayRef< char >::MutableArrayRef(), nameOrNone(), llvm::sys::unicode::nameToGeneratedCodePoint(), llvm::sys::unicode::nameToHangulCodePoint(), narrowExtractedVectorBinOp(), narrowInterleaveGroupOp(), NegateValue(), NewSDValueDbgMsg(), nextByte(), llvm::NextUseDistance::NextUseDistance(), llvm::NextUseDistance::NextUseDistance(), llvm::Registry< T, CtorParamTypes >::node::node(), llvm::dwarf_linker::parallel::CompileUnit::noteStmtSeqListAttribute(), llvm::json::Value::Object, llvm::json::ObjectKey::ObjectKey(), llvm::json::ObjectKey::ObjectKey(), llvm::fuzzerop::onlyType(), OnlyUsedBy(), llvm::sandboxir::OpaqueValue::OpaqueValue(), llvm::BitTracker::BitValue::operator!=(), llvm::SDUse::operator!=(), llvm::ValueDeleter::operator()(), llvm::ScopedHashTableIterator< K, V, KInfo >::operator*(), llvm::ValueHandleBase::operator*(), llvm::NoopStatistic::operator+=(), llvm::PBQP::Vector::operator+=(), llvm::TrackingStatistic::operator+=(), llvm::NoopStatistic::operator-=(), llvm::TrackingStatistic::operator-=(), llvm::ScopedHashTableIterator< K, V, KInfo >::operator->(), llvm::operator<(), llvm::operator<(), llvm::sampleprof::SampleContext::operator<(), llvm::SDUse::operator<(), llvm::DiagnosticPrinter::operator<<(), llvm::json::operator<<(), llvm::operator<<(), llvm::operator<<(), llvm::operator<<(), llvm::operator<<(), llvm::operator<<(), llvm::operator<<(), llvm::operator<<(), llvm::operator<<(), llvm::operator<<(), llvm::PBQP::operator<<(), llvm::RISCV::operator<<(), llvm::sandboxir::Value::operator<<, llvm::vfs::OutputFile::operator<<(), llvm::cl::OptionValue< DataType >::operator=(), llvm::cl::OptionValue< cl::boolOrDefault >::operator=(), llvm::cl::OptionValue< std::string >::operator=(), llvm::detail::PunnedPointer< Ptr >::operator=(), llvm::GVNValueTable::operator=(), llvm::MachineSSAUpdater::operator=(), llvm::NextUseDistance::operator=(), llvm::NextUseDistance::operator=(), llvm::pdb::DbiStreamBuilder::operator=(), llvm::pdb::InfoStreamBuilder::operator=(), llvm::pointer_union_detail::PointerUnionMembers< Derived, Idx, Type, Types... >::operator=(), llvm::SDDbgInfo::operator=(), llvm::SSAUpdater::operator=(), llvm::TinyPtrVector< ReachingDef >::operator=(), llvm::ValueEnumerator::operator=(), llvm::ValueMapper::operator=(), llvm::BitTracker::BitValue::operator==(), llvm::PBQP::Vector::operator==(), llvm::SDUse::operator==(), OptimizeAwayTrappingUsesOfValue(), optimizeCallInst(), optimizeDoubleFP(), optimizeLoopExitWithUnknownExitCount(), optimizeMaskToEVL(), OptimizeNonTrivialIFuncs(), optimizeShiftInOrChain(), llvm::cl::OptionValue< DataType >::OptionValue(), llvm::cl::OptionValue< cl::boolOrDefault >::OptionValue(), llvm::cl::OptionValue< std::string >::OptionValue(), or32le(), orderModule(), orderModule(), orderModule(), orderRegForMul(), orderValue(), orderValue(), orderValue(), llvm::yaml::CustomMappingTraits< GlobalValueSummaryMapTy >::output(), llvm::yaml::CustomMappingTraits< std::map< std::vector< uint64_t >, WholeProgramDevirtResolution::ByArg > >::output(), llvm::yaml::CustomMappingTraits< std::map< uint64_t, WholeProgramDevirtResolution > >::output(), llvm::yaml::CustomMappingTraits< TypeIdSummaryMapTy >::output(), llvm::PseudoProbeDwarfDiscriminator::packProbeData(), llvm::parallelForEachError(), llvm::cl::parser< DataType >::parse(), llvm::json::parse(), llvm::remarks::HotnessThresholdParser::parse(), llvm::opt::parseArgValue(), llvm::opt::parseArgValue(), llvm::opt::parseArgValue(), llvm::opt::parseArgValue(), llvm::opt::parseArgValue(), llvm::opt::parseArgValue(), llvm::opt::parseArgValue(), parseGlobalCtors(), parseIRValue(), parseSVERegAsConstraint(), parseVersion(), passingValueIsAlwaysUndefined(), llvm::patchAddrBase(), llvm::patchStmtList(), llvm::InstCombiner::peekThroughBitcast(), peekThroughBitcasts(), peekThroughBitPosExtTrunc(), peekThroughDemandedElts(), llvm::peekThroughFreeze(), llvm::peekThroughOneUseFreeze(), llvm::slpvectorizer::BaseShuffleAnalysis::peekThroughShuffles(), performADDCombine(), performADDCombine(), performANDCombine(), PerformARMBUILD_VECTORCombine(), performBITCASTCombine(), performBR_CCCombine(), PerformBUILD_VECTORCombine(), performConcatVectorsCombine(), performExtendCombine(), performExtendToBoolVectorLoadCombine(), PerformExtractEltToVMOVRRD(), performExtractSubvectorCombine(), PerformInsertEltCombine(), performIntrinsicCombine(), performLegalizedInterleavedStoreCombine(), PerformMinMaxCombine(), performMINMAXCombine(), PerformMULCombine(), performMULCombine(), performMulCombine(), performORCombine(), PerformORCombine_i1(), performSELECTCombine(), performSETCC_BITCASTCombine(), performSETCCCombine(), performSETCCCombine(), performSETCCCombine(), performSETCCCombine(), performSetccMergeZeroCombine(), performSHLCombine(), performSPLIT_PAIR_F64Combine(), PerformSTORECombine(), performSUBCombine(), performSUBCombine(), performVECREDUCECombine(), performVECTOR_INTERLEAVECombine(), performVECTOR_SHUFFLECombine(), performVFMADD_VLCombine(), PerformVMOVrhCombine(), PerformVSELECTCombine(), performVSELECTCombine(), performVWADDSUBW_VLCombine(), performXORCombine(), PHIAreRefEachOther(), llvm::xray::PipelineConsumer::PipelineConsumer(), llvm::pointer_union_detail::PointerUnionMembers< Derived, Idx, Type, Types... >::PointerUnionMembers(), poisonGuaranteesUB(), llvm::popcount(), predictUseListOrder(), predictValueUseListOrder(), predictValueUseListOrder(), predictValueUseListOrder(), predictValueUseListOrderImpl(), predictValueUseListOrderImpl(), llvm::cl::OptionDiffPrinter< DT, DT >::print(), llvm::sandboxir::Action::print(), printAsOperandImpl(), llvm::ScopedPrinter::printBinary(), llvm::ScopedPrinter::printBinary(), llvm::ScopedPrinter::printBinary(), llvm::ScopedPrinter::printBinaryBlock(), llvm::json::Path::Root::printErrorContext(), printExtendedName(), llvm::AbstractLatticeFunction< LatticeKey, LatticeVal >::PrintLatticeVal(), printLLVMName(), llvm::cl::generic_parser_base::printOptionDiff(), llvm::cl::parser< bool >::printOptionDiff(), llvm::cl::parser< boolOrDefault >::printOptionDiff(), llvm::cl::parser< char >::printOptionDiff(), llvm::cl::parser< double >::printOptionDiff(), llvm::cl::parser< ElementCount >::printOptionDiff(), llvm::cl::parser< float >::printOptionDiff(), llvm::cl::parser< int >::printOptionDiff(), llvm::cl::parser< long >::printOptionDiff(), llvm::cl::parser< long long >::printOptionDiff(), llvm::cl::parser< std::optional< std::string > >::printOptionDiff(), llvm::cl::parser< std::string >::printOptionDiff(), llvm::cl::parser< unsigned >::printOptionDiff(), llvm::cl::parser< unsigned long >::printOptionDiff(), llvm::cl::parser< unsigned long long >::printOptionDiff(), llvm::cl::printOptionDiff(), llvm::cl::printOptionDiff(), printValues(), printWithoutType(), processHeaderPhiOperands(), processVCIXOperands(), llvm::ImutKeyValueInfo< T, S >::Profile(), programUndefinedIfUndefOrPoison(), promoteSingleBlockAlloca(), promoteToConstantPool(), PropagateInlinedFromMetadata(), llvm::sandboxir::Utils::propagateMetadata(), llvm::AllocatorList< T, BumpPtrAllocator >< Addend >::push_back(), llvm::AllocatorList< T, BumpPtrAllocator >< Addend >::push_back(), llvm::BitcodeReaderValueList::push_back(), llvm::TinyPtrVector< ReachingDef >::push_back(), llvm::AllocatorList< T, BumpPtrAllocator >< Addend >::push_front(), llvm::AllocatorList< T, BumpPtrAllocator >< Addend >::push_front(), pushAddIntoCmovOfConsts(), PushArgMD(), llvm::InstructionWorklist::pushValue(), readValuePack(), recomputeLiveInValues(), reconnectPhis(), ReconstructShuffleWithRuntimeMask(), ReconstructTruncateFromBuildVector(), llvm::xray::RecordInitializer::RecordInitializer(), llvm::memprof::RecordLookupTrait::RecordLookupTrait(), recordTypeIdCompatibleVtableReferences(), llvm::memprof::RecordWriterTrait::RecordWriterTrait(), reduceVSXSwap(), llvm::BitTracker::BitValue::ref(), llvm::sandboxir::Context::registerValue(), llvm::Attributor::registerVirtualUseCallback(), relocationViaAlloca(), llvm::ImmutableSet< ValT, ValInfo, Canonicalize >::Factory::remove(), llvm::ImmutableSetRef< ValT, ValInfo, Canonicalize >::remove(), llvm::ImutAVLFactory< ImutInfo, Canonicalize >::remove(), llvm::SetVector< EdgeType * >::remove_if(), removeASCastIfPresent(), removeEntryFromStack(), RemoveInstInputs(), llvm::SymbolTableListTraits< ValueSubClass, Args >::removeNodeFromList(), removePAICalls(), removeUndefDbgLocsFromEntryBlock(), llvm::DomConditionCache::removeValue(), llvm::Record::removeValue(), llvm::InstCombinerImpl::replaceInInstruction(), llvm::InstCombiner::replaceInstUsesWith(), llvm::LoadAndStorePromoter::replaceLoadWithValue(), llvm::InstCombiner::replaceOperand(), replaceUndefValuesInPhi(), replaceValuesPerBlockEntry(), replaceWithTileLoad(), llvm::orc::shared::TrivialSPSSequenceDeserialization< SPSElementTagT, SmallVectorImpl< T > >::reserve(), llvm::orc::shared::TrivialSPSSequenceDeserialization< SPSElementTagT, std::vector< T > >::reserve(), restoreUsedList(), rewriteAccessChain(), llvm::rotl(), llvm::rotr(), rsrcPartRoot(), llvm::DomTreeBuilder::SemiNCAInfo< DomTreeT >::runDFS(), runHardFloatOnModule(), runImpl(), llvm::SelectionDAGISel::RunSDNodeXForm(), safeCtxI(), SaturateWidenedDIVFIX(), llvm::SCEVUnknown::ScalarEvolution, ScaleVectorOffset(), scanAssociativeOperands(), llvm::SCEVTraversal< SV >::SCEVTraversal(), llvm::SDLoc::SDLoc(), searchConstantExprForGlobalVariables(), llvm::LiveRange::Segment::Segment(), llvm::orc::shared::SPSSerializationTraits< SPSSymbolLookupSetElement, RemoteSymbolLookupSetElement >::serialize(), llvm::orc::shared::SPSSerializationTraits< SPSSymbolLookupSetElement, SymbolLookupSet::value_type >::serialize(), llvm::orc::shared::SPSArgList<>::serializeToSmallVector(), llvm::IndexedInstrProf::Summary::set(), llvm::PointerSumType< TagT, MemberTs... >::set(), llvm::Use::set(), llvm::VPTransformState::set(), llvm::VPTransformState::set(), llvm::pdb::const_iterator< SrcHeaderBlockEntry >::set_as(), llvm::DbgVariableRecord::setAddress(), llvm::IndirectBrInst::setAddress(), llvm::OffloadEntriesInfoManager::OffloadEntryInfo::setAddress(), llvm::MachineFrameInfo::setAdjustsStack(), llvm::MCAsmInfo::setAllowAtInName(), llvm::MCFragment::setAllowAutoPadding(), llvm::AsmLexer::setAllowHashInIdentifier(), llvm::MachO::InterfaceFile::setApplicationExtensionSafe(), llvm::VPHeaderPHIRecipe::setBackedgeValue(), llvm::MachineBasicBlock::setBBID(), llvm::MachineFunction::setBBSectionsType(), llvm::SPIRVGlobalRegistry::setBound(), llvm::CallBase::setCalledOperand(), llvm::sandboxir::CallBase::setCalledOperand(), llvm::ValueLatticeElement::MergeOptions::setCheckWiden(), llvm::LandingPadInst::setCleanup(), llvm::CondBrInst::setCondition(), llvm::SelectInst::setCondition(), llvm::SwitchInst::setCondition(), llvm::sandboxir::GlobalVariable::setConstant(), llvm::GVNExpression::ConstantExpression::setConstantValue(), setDeducedOverflowingFlags(), llvm::MCStreamer::setDefaultWinCFIUnwindVersion(), llvm::AtomicRMWInst::setElementwise(), llvm::LoadInst::setElementwise(), llvm::StoreInst::setElementwise(), llvm::sandboxir::GlobalVariable::setExternallyInitialized(), llvm::SelectInst::setFalseValue(), llvm::cl::Option::setFormattingFlag(), llvm::sampleprof::SampleProfileWriter::setFormatVersion(), llvm::X86InstrInfo::setFrameAdjustment(), llvm::MCAsmInfo::setFullRegisterNames(), llvm::GlobalValue::setGlobalValueSubClassData(), llvm::MachineFrameInfo::setHasCalls(), llvm::MachineFunction::setHasEHContTarget(), llvm::MachineFunction::setHasEHFunclets(), llvm::MachineFunction::setHasEHScopes(), llvm::MachineFunction::setHasFakeUses(), llvm::X86MachineFunctionInfo::setHasSEHFramePtrSave(), llvm::MachineFrameInfo::setHasTailCall(), llvm::OffloadEntriesInfoManager::OffloadEntryInfoTargetRegion::setID(), llvm::MemoryPhi::setIncomingValue(), llvm::PHINode::setIncomingValue(), llvm::PHINode::setIncomingValueForBlock(), llvm::cl::opt< DataType, ExternalStorage, ParserClass >::setInitialValue(), setInsertionPoint(), llvm::MachineBasicBlock::setIsBeginSection(), llvm::MachineBasicBlock::setIsCleanupFuncletEntry(), llvm::MachineBasicBlock::setIsEHContTarget(), llvm::MachineBasicBlock::setIsEHFuncletEntry(), llvm::MachineBasicBlock::setIsEHPad(), llvm::MachineBasicBlock::setIsEHScopeEntry(), llvm::MachineBasicBlock::setIsEndSection(), llvm::MachineBasicBlock::setIsInlineAsmBrIndirectTarget(), llvm::DWPWriter::setIsLittleEndian(), llvm::Function::setIsMaterializable(), llvm::MachineFunction::setIsOutlined(), llvm::MCSymbol::setIsResolving(), llvm::DWPWriter::setIsWASM(), llvm::MachO::GlobalRecord::setKind(), setKnownBase(), llvm::AsmLexer::setLexHLASMIntegers(), llvm::AsmLexer::setLexHLASMStrings(), llvm::AsmLexer::setLexMasmHexFloats(), llvm::AsmLexer::setLexMasmIntegers(), llvm::AsmLexer::setLexMasmStrings(), llvm::AsmLexer::setLexMotorolaIntegers(), llvm::VPBlendRecipe::setMask(), llvm::ValueLatticeElement::setMayHaveDifferentProvenance(), llvm::ValueLatticeElement::MergeOptions::setMayIncludeUndef(), llvm::InstrProfInstBase::setNameValue(), llvm::GVNExpression::BasicExpression::setOperand(), llvm::MachO::InterfaceFile::setOSLibNotForSharedCache(), llvm::ELFObjectWriter::setOverrideABIVersion(), llvm::MCAsmParser::setParsingMSInlineAsm(), llvm::MCSectionWasm::setPassive(), llvm::MachineFunction::setPrefetchTargets(), setPrefetchTargets(), llvm::LoopUnrollOptions::setPrepareForLTO(), llvm::sampleprofutil::SampleCoverageTracker::setProfAccForSymsInList(), llvm::ISD::ArgFlagsTy::setReturned(), llvm::MachineBasicBlock::setSectionID(), llvm::MachO::InterfaceFile::setSimulatorSupport(), llvm::VPHeaderPHIRecipe::setStartValue(), llvm::AllocaInst::setSwiftError(), llvm::SymbolTableListTraits< T, Args... >::setSymTabObject(), llvm::SelectInst::setTrueValue(), llvm::MachO::InterfaceFile::setTwoLevelNamespace(), llvm::sandboxir::GlobalValue::setUnnamedAddr(), llvm::SelectionDAGBuilder::setUnusedArgValue(), setUsedInitializer(), llvm::AllocaInst::setUsedWithInAlloca(), llvm::DwarfDebug::setUseInlineStrings(), llvm::DwarfDebug::setUseRangesSection(), llvm::DwarfDebug::setUseSectionsAsReferences(), llvm::X86MachineFunctionInfo::setUsesRedZone(), SetValue(), llvm::cl::opt_storage< DataType, ExternalStorage, isClass >::setValue(), llvm::cl::opt_storage< DataType, false, false >::setValue(), llvm::cl::opt_storage< DataType, false, true >::setValue(), llvm::cl::OptionValueCopy< DataType >::setValue(), llvm::SelectionDAGBuilder::setValue(), llvm::StringMapEntryStorage< ValueTy >::setValue(), llvm::SwitchInst::CaseHandle::setValue(), llvm::instrumentor::BasePointerIO::setValueNoop(), llvm::GVNExpression::VariableExpression::setVariableValue(), llvm::MachO::Record::setVerify(), llvm::GlobalValue::setVisibility(), llvm::sandboxir::GlobalValue::setVisibility(), llvm::AtomicCmpXchgInst::setVolatile(), llvm::AtomicRMWInst::setVolatile(), llvm::LoadInst::setVolatile(), llvm::MemIntrinsic::setVolatile(), llvm::MemSetPatternInst::setVolatile(), llvm::StoreInst::setVolatile(), shouldCanonicalizeGEPToPtrAdd(), shouldConvertOrWithNoCommonBitsToAdd(), shouldSplit(), shrinkFPConstantVector(), llvm::PatternMatch::Signum_match< Opnd_t >::Signum_match(), simplifyAddInst(), simplifyAllocaArraySize(), simplifyAndCommutative(), simplifyAndInst(), simplifyAndOfICmpsWithAdd(), simplifyAndOrOfCmps(), simplifyAndOrOfFCmps(), simplifyAndOrWithOpReplaced(), simplifyAShrInst(), simplifyAssociativeBinOp(), simplifyBinaryIntrinsic(), llvm::InstCombiner::SimplifyDemandedVectorElts(), simplifyDiv(), simplifyDivRem(), simplifyExtractValueInst(), simplifyFCmpInst(), simplifyFPOp(), simplifyGEPInst(), simplifyICmpInst(), simplifyICmpOfBools(), simplifyICmpWithBinOp(), simplifyICmpWithMinMax(), simplifyInstructionWithOperands(), simplifyIRemMulShl(), simplifyLoopInst(), simplifyLShrInst(), simplifyMulInst(), simplifyOneLoop(), simplifyOperationIntoSelectOperand(), simplifyOrInst(), simplifyOrOfICmpsWithAdd(), simplifyRecipe(), simplifyReductionOperand(), simplifyRem(), simplifyRightShift(), simplifySelectInst(), simplifySelectWithFCmp(), simplifySelectWithICmpCond(), simplifyShift(), simplifyShlInst(), simplifySubInst(), simplifyTernarylogic(), simplifyValueKnownNonZero(), simplifyWasmSwizzle(), simplifyWithOpReplaced(), simplifyWithOpsReplaced(), simplifyX86pshufb(), simplifyX86vpermilvar(), simplifyX86vpermv(), simplifyX86vpermv3(), simplifyXorInst(), sinkCommonCodeFromPredecessors(), sinkMinMaxInBB(), sinkProxyReg(), sinkSelectOperand(), llvm::orc::shared::SPSSerializationTraits< SPSSymbolLookupSetElement, RemoteSymbolLookupSetElement >::size(), llvm::orc::shared::SPSSerializationTraits< SPSSymbolLookupSetElement, SymbolLookupSet::value_type >::size(), llvm::fuzzerop::sizedPtrType(), skipMetadataWrapper(), llvm::SparseSolver< LatticeKey, LatticeVal, KeyInfo >::Solve(), SolveQuadraticAddRecExact(), llvm::fuzzerop::SourcePred::SourcePred(), llvm::PatternMatch::specific_fpval::specific_fpval(), llvm::PatternMatch::specific_intval< AllowPoison >::specific_intval(), llvm::PatternMatch::specific_intval64< AllowPoison >::specific_intval64(), llvm::SDPatternMatch::SpecificFP_match::SpecificFP_match(), llvm::PatternMatch::specificval_ty::specificval_ty(), llvm::VPlanPatternMatch::specificval_ty::specificval_ty(), speculatePHINodeLoads(), speculateSelectInstLoads(), splitAndLowerShuffle(), splitCallSite(), splitMergedValStore(), SplitStrictFPVectorOp(), SRAGlobal(), llvm::SrcOp::SrcOp(), llvm::SrcOp::SrcOp(), STATISTIC(), stripAndAccumulateOffsets(), stripAndComputeConstantOffsets(), stripInactiveLanes(), StripPointerGEPsAndCasts(), stripSignExtendInReg(), StripSymtab(), suffixed_name_or(), takeInexpensiveLog2(), targetParallelCallback(), threadBinOpOverPHI(), threadCmpOverPHI(), llvm::TinyPtrVector< ReachingDef >::TinyPtrVector(), llvm::dwarf::toAddress(), llvm::dwarf::toAddress(), llvm::dwarf::toBlock(), TOCRestoreNeededForCallToImplementation(), llvm::dwarf::toDebugInfoReference(), llvm::dwarf::toDebugInfoReference(), llvm::symbolize::toHex(), llvm::dwarf::toRelativeReference(), llvm::dwarf::toRelativeReference(), llvm::dwarf::toSectionedAddress(), llvm::dwarf::toSectionOffset(), llvm::dwarf::toSectionOffset(), llvm::dwarf::toSignatureReference(), llvm::dwarf::toSignatureReference(), llvm::dwarf::toSigned(), llvm::dwarf::toSigned(), llvm::dwarf::toString(), llvm::dwarf::toString(), llvm::dwarf::toStringRef(), llvm::dwarf::toSupplementaryReference(), llvm::dwarf::toSupplementaryReference(), llvm::dwarf::toUnsigned(), llvm::dwarf::toUnsigned(), tracesToPointerAlloca(), llvm::sandboxir::VecUtils::DeadInstructionMorgue::tryEraseDeadInstrs(), tryFactorization(), llvm::sandboxir::VecUtils::tryGetCommonScalarType(), tryToFindDuplicates(), tryToRecognizeTableBasedLog2(), tryToVectorizeSequence(), tryToWidenSetCCOperands(), llvm::SDPatternMatch::TypeSize_match< TS >::TypeSize_match(), llvm::UnaryInstruction::UnaryInstruction(), llvm::ImmutableSet< ValT, ValInfo, Canonicalize >::Factory::unionSets(), unpackLoadToAggregate(), unpackStoreToAggregate(), unswitchTrivialBranch(), llvm::PGOContextualProfile::update(), updateDVIWithLocations(), updateIDTMetaData(), llvm::NoopStatistic::updateMax(), llvm::TrackingStatistic::updateMax(), UpdatePHINodes(), upgradeNVVMFnVectorAttr(), upgradeSingleNVVMAnnotation(), upgradeX86VPERMT2Intrinsics(), llvm::AsmLexer::useMasmDefaultRadix(), llvm::sandboxir::User::User(), validExtractValueIndex(), validInsertValueIndex(), validShuffleVectorIndex(), llvm::json::Value::Value(), llvm::json::Value::Value(), llvm::json::Value::Value(), llvm::json::Value::Value(), llvm::json::Value::Value(), llvm::json::Value::Value(), llvm::json::Value::Value(), llvm::IRPosition::value(), llvm::json::OStream::value(), llvm::AA::ValueAndContext::ValueAndContext(), llvm::AA::ValueAndContext::ValueAndContext(), valueDominatesPHI(), llvm::ValueHandleBase::ValueHandleBase(), llvm::DWARFFormValue::ValueType::ValueType(), llvm::DWARFFormValue::ValueType::ValueType(), llvm::DWARFFormValue::ValueType::ValueType(), llvm::GVNExpression::VariableExpression::VariableExpression(), llvm::pdb::Variant::Variant(), llvm::pdb::Variant::Variant(), llvm::pdb::Variant::Variant(), llvm::pdb::Variant::Variant(), llvm::pdb::Variant::Variant(), llvm::pdb::Variant::Variant(), llvm::pdb::Variant::Variant(), llvm::pdb::Variant::Variant(), llvm::pdb::Variant::Variant(), llvm::pdb::Variant::Variant(), llvm::pdb::Variant::Variant(), llvm::PBQP::Vector::Vector(), llvm::PBQP::Vector::Vector(), vectorizePackedAggregate(), llvm::verifyFunction(), llvm::verifyModule(), llvm::MachineFunctionProperties::verifyRequiredProperties(), llvm::memtag::StackInfoBuilder::visit(), llvm::orc::detail::WaitingOnGraph< ContainerIdT, ElementIdT >::ContainerElementsMap::visit(), llvm::PGOContextualProfile::visit(), llvm::codeview::detail::visitDebugSubsections(), llvm::codeview::visitDebugSubsections(), llvm::codeview::visitDebugSubsections(), llvm::codeview::visitDebugSubsections(), llvm::jitlink::visitEdge(), llvm::codeview::visitMemberRecord(), llvm::codeview::visitMemberRecordStream(), visitORCommutative(), llvm::codeview::visitTypeRecord(), llvm::codeview::visitTypeRecord(), llvm::codeview::visitTypeStream(), llvm::codeview::visitTypeStream(), llvm::codeview::visitTypeStream(), llvm::VPDef::VPMultiDefValue, widenPackedVectorWithZeros(), widenSubVector(), willShiftRightEliminate(), llvm::CGDataOStream::write(), llvm::support::endian::write(), llvm::support::endian::write(), write(), llvm::support::endian::write16(), llvm::support::endian::write16(), llvm::support::endian::write16be(), llvm::support::endian::write16le(), llvm::CGDataOStream::write32(), llvm::support::endian::write32(), llvm::support::endian::write32(), llvm::support::endian::write32be(), llvm::support::endian::write32le(), llvm::support::endian::write64(), llvm::support::endian::write64(), llvm::support::endian::write64be(), llvm::support::endian::write64le(), llvm::CGDataOStream::write8(), writeAsOperandInternal(), writeAsOperandInternal(), writeFragment(), writeMDTuple(), llvm::AMDGPUDisassembler::~AMDGPUDisassembler(), llvm::GlobalsAAResult::~GlobalsAAResult(), llvm::SCCPSolver::~SCCPSolver(), llvm::SSAUpdaterBulk::~SSAUpdaterBulk(), and llvm::TinyPtrVector< ReachingDef >::~TinyPtrVector().