AArch64ISelLowering.cpp File Reference

#include "AArch64ISelLowering.h"
#include "AArch64CallingConvention.h"
#include "AArch64ExpandImm.h"
#include "AArch64MachineFunctionInfo.h"
#include "AArch64PerfectShuffle.h"
#include "AArch64RegisterInfo.h"
#include "AArch64Subtarget.h"
#include "MCTargetDesc/AArch64AddressingModes.h"
#include "Utils/AArch64BaseInfo.h"
#include "llvm/ADT/APFloat.h"
#include "llvm/ADT/APInt.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SmallSet.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/ADT/Twine.h"
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/Analysis/MemoryLocation.h"
#include "llvm/Analysis/ObjCARCUtil.h"
#include "llvm/Analysis/OptimizationRemarkEmitter.h"
#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/Analysis/VectorUtils.h"
#include "llvm/CodeGen/Analysis.h"
#include "llvm/CodeGen/CallingConvLower.h"
#include "llvm/CodeGen/ComplexDeinterleavingPass.h"
#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
#include "llvm/CodeGen/GlobalISel/Utils.h"
#include "llvm/CodeGen/ISDOpcodes.h"
#include "llvm/CodeGen/MachineBasicBlock.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineInstr.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
#include "llvm/CodeGen/MachineMemOperand.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/CodeGen/RuntimeLibcallUtil.h"
#include "llvm/CodeGen/SelectionDAG.h"
#include "llvm/CodeGen/SelectionDAGNodes.h"
#include "llvm/CodeGen/TargetCallingConv.h"
#include "llvm/CodeGen/TargetInstrInfo.h"
#include "llvm/CodeGen/TargetOpcodes.h"
#include "llvm/CodeGen/ValueTypes.h"
#include "llvm/CodeGenTypes/MachineValueType.h"
#include "llvm/IR/Attributes.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/DebugLoc.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/GetElementPtrTypeIterator.h"
#include "llvm/IR/GlobalValue.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/Instruction.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/IntrinsicsAArch64.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/PatternMatch.h"
#include "llvm/IR/Type.h"
#include "llvm/IR/Use.h"
#include "llvm/IR/Value.h"
#include "llvm/MC/MCRegisterInfo.h"
#include "llvm/Support/AtomicOrdering.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/CodeGen.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/InstructionCost.h"
#include "llvm/Support/KnownBits.h"
#include "llvm/Support/MathExtras.h"
#include "llvm/Support/SipHash.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Target/TargetMachine.h"
#include "llvm/Target/TargetOptions.h"
#include "llvm/TargetParser/Triple.h"
#include <algorithm>
#include <bitset>
#include <cassert>
#include <cctype>
#include <cstdint>
#include <cstdlib>
#include <iterator>
#include <limits>
#include <optional>
#include <tuple>
#include <utility>
#include <vector>
#include "AArch64GenAsmMatcher.inc"

Go to the source code of this file.

Classes
struct	GenericSetCCInfo
	Helper structure to keep track of ISD::SET_CC operands. More...
struct	AArch64SetCCInfo
	Helper structure to keep track of a SET_CC lowered into AArch64 code. More...
union	SetCCInfo
	Helper structure to keep track of SetCC information. More...
struct	SetCCInfoAndKind
	Helper structure to be able to read SetCC information. More...

Macros
#define	DEBUG_TYPE   "aarch64-lower"
#define	LCALLNAMES(A, B, N)
#define	LCALLNAME4(A, B)
#define	LCALLNAME5(A, B)
#define	MAKE_CASE(V)
#define	GET_REGISTER_MATCHER

Enumerations
enum class	PredicateConstraint { Uph , Upl , Upa }
enum class	ReducedGprConstraint { Uci , Ucj }

Functions
	STATISTIC (NumTailCalls, "Number of tail calls")
	STATISTIC (NumShiftInserts, "Number of vector shift inserts")
	STATISTIC (NumOptimizedImms, "Number of times immediates were optimized")
static EVT	getPackedSVEVectorVT (EVT VT)
static EVT	getPackedSVEVectorVT (ElementCount EC)
static EVT	getPromotedVTForPredicate (EVT VT)
static bool	isPackedVectorType (EVT VT, SelectionDAG &DAG)
	Returns true if VT's elements occupy the lowest bit positions of its associated register class without any intervening space.
static bool	isMergePassthruOpcode (unsigned Opc)
static bool	isZeroingInactiveLanes (SDValue Op)
static std::tuple< SDValue, SDValue >	extractPtrauthBlendDiscriminators (SDValue Disc, SelectionDAG *DAG)
static bool	isIntImmediate (const SDNode *N, uint64_t &Imm)
static bool	isOpcWithIntImmediate (const SDNode *N, unsigned Opc, uint64_t &Imm)
static bool	optimizeLogicalImm (SDValue Op, unsigned Size, uint64_t Imm, const APInt &Demanded, TargetLowering::TargetLoweringOpt &TLO, unsigned NewOpc)
static EVT	getContainerForFixedLengthVector (SelectionDAG &DAG, EVT VT)
static SDValue	convertToScalableVector (SelectionDAG &DAG, EVT VT, SDValue V)
static SDValue	convertFromScalableVector (SelectionDAG &DAG, EVT VT, SDValue V)
static SDValue	convertFixedMaskToScalableVector (SDValue Mask, SelectionDAG &DAG)
static SDValue	getPredicateForVector (SelectionDAG &DAG, SDLoc &DL, EVT VT)
static SDValue	getPredicateForScalableVector (SelectionDAG &DAG, SDLoc &DL, EVT VT)
static bool	isZerosVector (const SDNode *N)
	isZerosVector - Check whether SDNode N is a zero-filled vector.
static AArch64CC::CondCode	changeIntCCToAArch64CC (ISD::CondCode CC)
	changeIntCCToAArch64CC - Convert a DAG integer condition code to an AArch64 CC
static void	changeFPCCToAArch64CC (ISD::CondCode CC, AArch64CC::CondCode &CondCode, AArch64CC::CondCode &CondCode2)
	changeFPCCToAArch64CC - Convert a DAG fp condition code to an AArch64 CC.
static void	changeFPCCToANDAArch64CC (ISD::CondCode CC, AArch64CC::CondCode &CondCode, AArch64CC::CondCode &CondCode2)
	Convert a DAG fp condition code to an AArch64 CC.
static void	changeVectorFPCCToAArch64CC (ISD::CondCode CC, AArch64CC::CondCode &CondCode, AArch64CC::CondCode &CondCode2, bool &Invert)
	changeVectorFPCCToAArch64CC - Convert a DAG fp condition code to an AArch64 CC usable with the vector instructions.
static bool	isLegalArithImmed (uint64_t C)
static bool	cannotBeIntMin (SDValue CheckedVal, SelectionDAG &DAG)
static bool	isCMN (SDValue Op, ISD::CondCode CC, SelectionDAG &DAG)
static SDValue	emitStrictFPComparison (SDValue LHS, SDValue RHS, const SDLoc &dl, SelectionDAG &DAG, SDValue Chain, bool IsSignaling)
static SDValue	emitComparison (SDValue LHS, SDValue RHS, ISD::CondCode CC, const SDLoc &dl, SelectionDAG &DAG)
static unsigned	getCmpOperandFoldingProfit (SDValue Op)
	Returns how profitable it is to fold a comparison's operand's shift and/or extension operations.
static SDValue	getAArch64Cmp (SDValue LHS, SDValue RHS, ISD::CondCode CC, SDValue &AArch64cc, SelectionDAG &DAG, const SDLoc &dl)
static std::pair< SDValue, SDValue >	getAArch64XALUOOp (AArch64CC::CondCode &CC, SDValue Op, SelectionDAG &DAG)
static SDValue	valueToCarryFlag (SDValue Value, SelectionDAG &DAG, bool Invert)
static SDValue	carryFlagToValue (SDValue Glue, EVT VT, SelectionDAG &DAG, bool Invert)
static SDValue	overflowFlagToValue (SDValue Glue, EVT VT, SelectionDAG &DAG)
static SDValue	lowerADDSUBO_CARRY (SDValue Op, SelectionDAG &DAG, unsigned Opcode, bool IsSigned)
static SDValue	LowerXALUO (SDValue Op, SelectionDAG &DAG)
static SDValue	LowerPREFETCH (SDValue Op, SelectionDAG &DAG)
static MVT	getSVEContainerType (EVT ContentTy)
static EVT	getExtensionTo64Bits (const EVT &OrigVT)
static SDValue	addRequiredExtensionForVectorMULL (SDValue N, SelectionDAG &DAG, const EVT &OrigTy, const EVT &ExtTy, unsigned ExtOpcode)
static std::optional< uint64_t >	getConstantLaneNumOfExtractHalfOperand (SDValue &Op)
static bool	isExtendedBUILD_VECTOR (SDValue N, SelectionDAG &DAG, bool isSigned)
static SDValue	skipExtensionForVectorMULL (SDValue N, SelectionDAG &DAG)
static bool	isSignExtended (SDValue N, SelectionDAG &DAG)
static bool	isZeroExtended (SDValue N, SelectionDAG &DAG)
static bool	isAddSubSExt (SDValue N, SelectionDAG &DAG)
static bool	isAddSubZExt (SDValue N, SelectionDAG &DAG)
static unsigned	selectUmullSmull (SDValue &N0, SDValue &N1, SelectionDAG &DAG, SDLoc DL, bool &IsMLA)
static SDValue	getPTrue (SelectionDAG &DAG, SDLoc DL, EVT VT, int Pattern)
static SDValue	optimizeIncrementingWhile (SDValue Op, SelectionDAG &DAG, bool IsSigned, bool IsEqual)
static SDValue	getSVEPredicateBitCast (EVT VT, SDValue Op, SelectionDAG &DAG)
SDValue	LowerSMELdrStr (SDValue N, SelectionDAG &DAG, bool IsLoad)
unsigned	getGatherVecOpcode (bool IsScaled, bool IsSigned, bool NeedsExtend)
unsigned	getSignExtendedGatherOpcode (unsigned Opcode)
static SDValue	LowerTruncateVectorStore (SDLoc DL, StoreSDNode *ST, EVT VT, EVT MemVT, SelectionDAG &DAG)
static SDValue	LowerBRCOND (SDValue Op, SelectionDAG &DAG)
static SDValue	LowerFunnelShift (SDValue Op, SelectionDAG &DAG)
static SDValue	LowerFLDEXP (SDValue Op, SelectionDAG &DAG)
static unsigned	getIntrinsicID (const SDNode *N)
static bool	isPassedInFPR (EVT VT)
static bool	canGuaranteeTCO (CallingConv::ID CC, bool GuaranteeTailCalls)
	Return true if the calling convention is one that we can guarantee TCO for.
static bool	mayTailCallThisCC (CallingConv::ID CC)
	Return true if we might ever do TCO for calls with this calling convention.
static bool	callConvSupportsVarArgs (CallingConv::ID CC)
	Return true if the call convention supports varargs Currently only those that pass varargs like the C calling convention does are eligible Calling conventions listed in this function must also be properly handled in AArch64Subtarget::isCallingConvWin64.
static void	analyzeCallOperands (const AArch64TargetLowering &TLI, const AArch64Subtarget *Subtarget, const TargetLowering::CallLoweringInfo &CLI, CCState &CCInfo)
static bool	checkZExtBool (SDValue Arg, const SelectionDAG &DAG)
static unsigned	getSMCondition (const SMEAttrs &CallerAttrs, const SMEAttrs &CalleeAttrs)
std::pair< SDValue, uint64_t >	lookThroughSignExtension (SDValue Val)
static bool	isOrXorChain (SDValue N, unsigned &Num, SmallVector< std::pair< SDValue, SDValue >, 16 > &WorkList)
static SDValue	performOrXorChainCombine (SDNode *N, SelectionDAG &DAG)
static SDValue	getEstimate (const AArch64Subtarget *ST, unsigned Opcode, SDValue Operand, SelectionDAG &DAG, int &ExtraSteps)
static std::optional< PredicateConstraint >	parsePredicateConstraint (StringRef Constraint)
static const TargetRegisterClass *	getPredicateRegisterClass (PredicateConstraint Constraint, EVT VT)
static std::optional< ReducedGprConstraint >	parseReducedGprConstraint (StringRef Constraint)
static const TargetRegisterClass *	getReducedGprRegisterClass (ReducedGprConstraint Constraint, EVT VT)
static AArch64CC::CondCode	parseConstraintCode (llvm::StringRef Constraint)
static SDValue	getSETCC (AArch64CC::CondCode CC, SDValue NZCV, const SDLoc &DL, SelectionDAG &DAG)
	Helper function to create 'CSET', which is equivalent to 'CSINC <Wd>, WZR, WZR, invert(<cond>)'.
static SDValue	WidenVector (SDValue V64Reg, SelectionDAG &DAG)
	WidenVector - Given a value in the V64 register class, produce the equivalent value in the V128 register class.
static unsigned	getExtFactor (SDValue &V)
	getExtFactor - Determine the adjustment factor for the position when generating an "extract from vector registers" instruction.
SDValue	ReconstructShuffleWithRuntimeMask (SDValue Op, SelectionDAG &DAG)
static bool	isSingletonEXTMask (ArrayRef< int > M, EVT VT, unsigned &Imm)
static SDValue	ReconstructTruncateFromBuildVector (SDValue V, SelectionDAG &DAG)
static bool	isWideDUPMask (ArrayRef< int > M, EVT VT, unsigned BlockSize, unsigned &DupLaneOp)
	Check if a vector shuffle corresponds to a DUP instructions with a larger element width than the vector lane type.
static bool	isEXTMask (ArrayRef< int > M, EVT VT, bool &ReverseEXT, unsigned &Imm)
static bool	isZIP_v_undef_Mask (ArrayRef< int > M, EVT VT, unsigned &WhichResult)
	isZIP_v_undef_Mask - Special case of isZIPMask for canonical form of "vector_shuffle v, v", i.e., "vector_shuffle v, undef".
static bool	isUZP_v_undef_Mask (ArrayRef< int > M, EVT VT, unsigned &WhichResult)
	isUZP_v_undef_Mask - Special case of isUZPMask for canonical form of "vector_shuffle v, v", i.e., "vector_shuffle v, undef".
static bool	isTRN_v_undef_Mask (ArrayRef< int > M, EVT VT, unsigned &WhichResult)
	isTRN_v_undef_Mask - Special case of isTRNMask for canonical form of "vector_shuffle v, v", i.e., "vector_shuffle v, undef".
static bool	isINSMask (ArrayRef< int > M, int NumInputElements, bool &DstIsLeft, int &Anomaly)
static bool	isConcatMask (ArrayRef< int > Mask, EVT VT, bool SplitLHS)
static SDValue	tryFormConcatFromShuffle (SDValue Op, SelectionDAG &DAG)
static SDValue	GeneratePerfectShuffle (unsigned ID, SDValue V1, SDValue V2, unsigned PFEntry, SDValue LHS, SDValue RHS, SelectionDAG &DAG, const SDLoc &dl)
	GeneratePerfectShuffle - Given an entry in the perfect-shuffle table, emit the specified operations to build the shuffle.
static SDValue	GenerateTBL (SDValue Op, ArrayRef< int > ShuffleMask, SelectionDAG &DAG)
static unsigned	getDUPLANEOp (EVT EltType)
static SDValue	constructDup (SDValue V, int Lane, SDLoc dl, EVT VT, unsigned Opcode, SelectionDAG &DAG)
static bool	isWideTypeMask (ArrayRef< int > M, EVT VT, SmallVectorImpl< int > &NewMask)
static SDValue	tryWidenMaskForShuffle (SDValue Op, SelectionDAG &DAG)
static SDValue	tryToConvertShuffleOfTbl2ToTbl4 (SDValue Op, ArrayRef< int > ShuffleMask, SelectionDAG &DAG)
static bool	resolveBuildVector (BuildVectorSDNode *BVN, APInt &CnstBits, APInt &UndefBits)
static SDValue	tryAdvSIMDModImm64 (unsigned NewOp, SDValue Op, SelectionDAG &DAG, const APInt &Bits)
static SDValue	tryAdvSIMDModImm32 (unsigned NewOp, SDValue Op, SelectionDAG &DAG, const APInt &Bits, const SDValue *LHS=nullptr)
static SDValue	tryAdvSIMDModImm16 (unsigned NewOp, SDValue Op, SelectionDAG &DAG, const APInt &Bits, const SDValue *LHS=nullptr)
static SDValue	tryAdvSIMDModImm321s (unsigned NewOp, SDValue Op, SelectionDAG &DAG, const APInt &Bits)
static SDValue	tryAdvSIMDModImm8 (unsigned NewOp, SDValue Op, SelectionDAG &DAG, const APInt &Bits)
static SDValue	tryAdvSIMDModImmFP (unsigned NewOp, SDValue Op, SelectionDAG &DAG, const APInt &Bits)
static bool	isAllConstantBuildVector (const SDValue &PotentialBVec, uint64_t &ConstVal)
static bool	isAllInactivePredicate (SDValue N)
static bool	isAllActivePredicate (SelectionDAG &DAG, SDValue N)
static SDValue	tryLowerToSLI (SDNode *N, SelectionDAG &DAG)
SDValue	tryWhileWRFromOR (SDValue Op, SelectionDAG &DAG, const AArch64Subtarget &Subtarget)
	Try to lower the construction of a pointer alias mask to a WHILEWR.
static SDValue	NormalizeBuildVector (SDValue Op, SelectionDAG &DAG)
static SDValue	ConstantBuildVector (SDValue Op, SelectionDAG &DAG, const AArch64Subtarget *ST)
static bool	isPow2Splat (SDValue Op, uint64_t &SplatVal, bool &Negated)
static bool	getVShiftImm (SDValue Op, unsigned ElementBits, int64_t &Cnt)
	getVShiftImm - Check if this is a valid build_vector for the immediate operand of a vector shift operation, where all the elements of the build_vector must have the same constant integer value.
static bool	isVShiftLImm (SDValue Op, EVT VT, bool isLong, int64_t &Cnt)
	isVShiftLImm - Check if this is a valid build_vector for the immediate operand of a vector shift left operation.
static bool	isVShiftRImm (SDValue Op, EVT VT, bool isNarrow, int64_t &Cnt)
	isVShiftRImm - Check if this is a valid build_vector for the immediate operand of a vector shift right operation.
static bool	canLowerSRLToRoundingShiftForVT (SDValue Shift, EVT ResVT, SelectionDAG &DAG, unsigned &ShiftValue, SDValue &RShOperand)
static SDValue	EmitVectorComparison (SDValue LHS, SDValue RHS, AArch64CC::CondCode CC, bool NoNans, EVT VT, const SDLoc &dl, SelectionDAG &DAG)
static SDValue	getReductionSDNode (unsigned Op, SDLoc DL, SDValue ScalarOp, SelectionDAG &DAG)
static SDValue	getVectorBitwiseReduce (unsigned Opcode, SDValue Vec, EVT VT, SDLoc DL, SelectionDAG &DAG)
template<unsigned NumVecs>
static bool	setInfoSVEStN (const AArch64TargetLowering &TLI, const DataLayout &DL, AArch64TargetLowering::IntrinsicInfo &Info, const CallInst &CI)
	Set the IntrinsicInfo for the aarch64_sve_st<N> intrinsics.
static bool	isSplatShuffle (Value *V)
static bool	areExtractShuffleVectors (Value *Op1, Value *Op2, bool AllowSplat=false)
	Check if both Op1 and Op2 are shufflevector extracts of either the lower or upper half of the vector elements.
static bool	areExtractExts (Value *Ext1, Value *Ext2)
	Check if Ext1 and Ext2 are extends of the same type, doubling the bitwidth of the vector elements.
static bool	isOperandOfVmullHighP64 (Value *Op)
	Check if Op could be used with vmull_high_p64 intrinsic.
static bool	areOperandsOfVmullHighP64 (Value *Op1, Value *Op2)
	Check if Op1 and Op2 could be used with vmull_high_p64 intrinsic.
static bool	shouldSinkVectorOfPtrs (Value *Ptrs, SmallVectorImpl< Use * > &Ops)
static bool	shouldSinkVScale (Value *Op, SmallVectorImpl< Use * > &Ops)
	We want to sink following cases: (add|sub|gep) A, ((mul|shl) vscale, imm); (add|sub|gep) A, vscale; (add|sub|gep) A, ((mul|shl) zext(vscale), imm);.
static bool	createTblShuffleMask (unsigned SrcWidth, unsigned DstWidth, unsigned NumElts, bool IsLittleEndian, SmallVectorImpl< int > &Mask)
static Value *	createTblShuffleForZExt (IRBuilderBase &Builder, Value *Op, FixedVectorType *ZExtTy, FixedVectorType *DstTy, bool IsLittleEndian)
static Value *	createTblShuffleForSExt (IRBuilderBase &Builder, Value *Op, FixedVectorType *DstTy, bool IsLittleEndian)
static void	createTblForTrunc (TruncInst *TI, bool IsLittleEndian)
static ScalableVectorType *	getSVEContainerIRType (FixedVectorType *VTy)
static Function *	getStructuredLoadFunction (Module *M, unsigned Factor, bool Scalable, Type *LDVTy, Type *PtrTy)
static Function *	getStructuredStoreFunction (Module *M, unsigned Factor, bool Scalable, Type *STVTy, Type *PtrTy)
template<typename Iter >
bool	hasNearbyPairedStore (Iter It, Iter End, Value *Ptr, const DataLayout &DL)
bool	getDeinterleave2Values (Value *DI, SmallVectorImpl< Instruction * > &DeinterleavedValues, SmallVectorImpl< Instruction * > &DeInterleaveDeadInsts)
bool	getDeinterleave4Values (Value *DI, SmallVectorImpl< Instruction * > &DeinterleavedValues, SmallVectorImpl< Instruction * > &DeInterleaveDeadInsts)
bool	getDeinterleavedValues (Value *DI, SmallVectorImpl< Instruction * > &DeinterleavedValues, SmallVectorImpl< Instruction * > &DeInterleaveDeadInsts)
bool	getValuesToInterleave (Value *II, SmallVectorImpl< Value * > &InterleavedValues, SmallVectorImpl< Instruction * > &InterleaveDeadInsts)
static SDValue	foldVectorXorShiftIntoCmp (SDNode *N, SelectionDAG &DAG, const AArch64Subtarget *Subtarget)
	Turn vector tests of the signbit in the form of: xor (sra X, elt_size(X)-1), -1 into: cmge X, X, #0.
static SDValue	performVecReduceAddCombineWithUADDLP (SDNode *N, SelectionDAG &DAG)
static SDValue	performVecReduceAddCombine (SDNode *N, SelectionDAG &DAG, const AArch64Subtarget *ST)
static SDValue	performUADDVAddCombine (SDValue A, SelectionDAG &DAG)
static SDValue	performUADDVZextCombine (SDValue A, SelectionDAG &DAG)
static SDValue	performUADDVCombine (SDNode *N, SelectionDAG &DAG)
static SDValue	performXorCombine (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const AArch64Subtarget *Subtarget)
static std::optional< unsigned >	IsSVECntIntrinsic (SDValue S)
static EVT	calculatePreExtendType (SDValue Extend)
	Calculates what the pre-extend type is, based on the extension operation node provided by Extend.
static SDValue	performBuildShuffleExtendCombine (SDValue BV, SelectionDAG &DAG)
	Combines a buildvector(sext/zext) or shuffle(sext/zext, undef) node pattern into sext/zext(buildvector) or sext/zext(shuffle) making use of the vector SExt/ZExt rather than the scalar SExt/ZExt.
static SDValue	performMulVectorExtendCombine (SDNode *Mul, SelectionDAG &DAG)
	Combines a mul(dup(sext/zext)) node pattern into mul(sext/zext(dup)) making use of the vector SExt/ZExt rather than the scalar SExt/ZExt.
static SDValue	performMulVectorCmpZeroCombine (SDNode *N, SelectionDAG &DAG)
static SDValue	performVectorExtCombine (SDNode *N, SelectionDAG &DAG)
static SDValue	performMulCombine (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const AArch64Subtarget *Subtarget)
static SDValue	performVectorCompareAndMaskUnaryOpCombine (SDNode *N, SelectionDAG &DAG)
static SDValue	performIntToFpCombine (SDNode *N, SelectionDAG &DAG, const AArch64Subtarget *Subtarget)
static SDValue	performFpToIntCombine (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const AArch64Subtarget *Subtarget)
	Fold a floating-point multiply by power of two into floating-point to fixed-point conversion.
static SDValue	tryCombineToBSL (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const AArch64TargetLowering &TLI)
static SDValue	performANDORCSELCombine (SDNode *N, SelectionDAG &DAG)
static SDValue	performORCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const AArch64Subtarget *Subtarget, const AArch64TargetLowering &TLI)
static bool	isConstantSplatVectorMaskForType (SDNode *N, EVT MemVT)
static SDValue	performReinterpretCastCombine (SDNode *N)
static SDValue	performSVEAndCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SDValue	performANDSETCCCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SDValue	performANDCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SDValue	performFADDCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static bool	hasPairwiseAdd (unsigned Opcode, EVT VT, bool FullFP16)
static SDValue	getPTest (SelectionDAG &DAG, EVT VT, SDValue Pg, SDValue Op, AArch64CC::CondCode Cond)
static bool	isPredicateCCSettingOp (SDValue N)
static SDValue	performFirstTrueTestVectorCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const AArch64Subtarget *Subtarget)
static SDValue	performLastTrueTestVectorCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const AArch64Subtarget *Subtarget)
static SDValue	performExtractVectorEltCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const AArch64Subtarget *Subtarget)
static SDValue	performConcatVectorsCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
static SDValue	performExtractSubvectorCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
static SDValue	performInsertSubvectorCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
static SDValue	tryCombineFixedPointConvert (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
static SDValue	tryExtendDUPToExtractHigh (SDValue N, SelectionDAG &DAG)
static bool	isEssentiallyExtractHighSubvector (SDValue N)
static bool	isSetCC (SDValue Op, SetCCInfoAndKind &SetCCInfo)
	Check whether or not Op is a SET_CC operation, either a generic or an AArch64 lowered one.
static bool	isSetCCOrZExtSetCC (const SDValue &Op, SetCCInfoAndKind &Info)
static SDValue	performSetccAddFolding (SDNode *Op, SelectionDAG &DAG)
static SDValue	performAddUADDVCombine (SDNode *N, SelectionDAG &DAG)
static SDValue	performAddCSelIntoCSinc (SDNode *N, SelectionDAG &DAG)
	Perform the scalar expression combine in the form of: CSEL(c, 1, cc) + b => CSINC(b+c, b, cc) CSNEG(c, -1, cc) + b => CSINC(b+c, b, cc)
static SDValue	performAddDotCombine (SDNode *N, SelectionDAG &DAG)
static bool	isNegatedInteger (SDValue Op)
static SDValue	getNegatedInteger (SDValue Op, SelectionDAG &DAG)
static SDValue	performNegCSelCombine (SDNode *N, SelectionDAG &DAG)
static SDValue	performAddSubLongCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static bool	isCMP (SDValue Op)
static std::optional< AArch64CC::CondCode >	getCSETCondCode (SDValue Op)
static SDValue	foldOverflowCheck (SDNode *Op, SelectionDAG &DAG, bool IsAdd)
static SDValue	foldADCToCINC (SDNode *N, SelectionDAG &DAG)
static SDValue	performBuildVectorCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
static SDValue	performTruncateCombine (SDNode *N, SelectionDAG &DAG)
static bool	isExtendOrShiftOperand (SDValue N)
static SDValue	performAddCombineSubShift (SDNode *N, SDValue SUB, SDValue Z, SelectionDAG &DAG)
static SDValue	performAddCombineForShiftedOperands (SDNode *N, SelectionDAG &DAG)
static SDValue	performSubAddMULCombine (SDNode *N, SelectionDAG &DAG)
static SDValue	performSVEMulAddSubCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SDValue	performAddSubIntoVectorOp (SDNode *N, SelectionDAG &DAG)
static bool	isLoadOrMultipleLoads (SDValue B, SmallVector< LoadSDNode * > &Loads)
static bool	areLoadedOffsetButOtherwiseSame (SDValue Op0, SDValue Op1, SelectionDAG &DAG, unsigned &NumSubLoads)
static SDValue	performExtBinopLoadFold (SDNode *N, SelectionDAG &DAG)
static SDValue	performAddSubCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SDValue	tryCombineLongOpWithDup (unsigned IID, SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
static SDValue	tryCombineShiftImm (unsigned IID, SDNode *N, SelectionDAG &DAG)
static SDValue	tryCombineCRC32 (unsigned Mask, SDNode *N, SelectionDAG &DAG)
static SDValue	combineAcrossLanesIntrinsic (unsigned Opc, SDNode *N, SelectionDAG &DAG)
static SDValue	LowerSVEIntrinsicIndex (SDNode *N, SelectionDAG &DAG)
static SDValue	LowerSVEIntrinsicDUP (SDNode *N, SelectionDAG &DAG)
static SDValue	LowerSVEIntrinsicEXT (SDNode *N, SelectionDAG &DAG)
static SDValue	tryConvertSVEWideCompare (SDNode *N, ISD::CondCode CC, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
static SDValue	combineSVEReductionInt (SDNode *N, unsigned Opc, SelectionDAG &DAG)
static SDValue	combineSVEReductionFP (SDNode *N, unsigned Opc, SelectionDAG &DAG)
static SDValue	combineSVEReductionOrderedFP (SDNode *N, unsigned Opc, SelectionDAG &DAG)
static SDValue	convertMergedOpToPredOp (SDNode *N, unsigned Opc, SelectionDAG &DAG, bool UnpredOp=false, bool SwapOperands=false)
static SDValue	tryCombineWhileLo (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const AArch64Subtarget *Subtarget)
static SDValue	performIntrinsicCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const AArch64Subtarget *Subtarget)
static bool	isCheapToExtend (const SDValue &N)
static SDValue	performSignExtendSetCCCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
static SDValue	performExtendCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
static SDValue	splitStoreSplat (SelectionDAG &DAG, StoreSDNode &St, SDValue SplatVal, unsigned NumVecElts)
static SDValue	performLD1Combine (SDNode *N, SelectionDAG &DAG, unsigned Opc)
static SDValue	performLDNT1Combine (SDNode *N, SelectionDAG &DAG)
template<unsigned Opcode>
static SDValue	performLD1ReplicateCombine (SDNode *N, SelectionDAG &DAG)
static SDValue	performST1Combine (SDNode *N, SelectionDAG &DAG)
static SDValue	performSTNT1Combine (SDNode *N, SelectionDAG &DAG)
static SDValue	replaceZeroVectorStore (SelectionDAG &DAG, StoreSDNode &St)
	Replace a splat of zeros to a vector store by scalar stores of WZR/XZR.
static SDValue	replaceSplatVectorStore (SelectionDAG &DAG, StoreSDNode &St)
	Replace a splat of a scalar to a vector store by scalar stores of the scalar value.
static SDValue	splitStores (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG, const AArch64Subtarget *Subtarget)
static SDValue	performSpliceCombine (SDNode *N, SelectionDAG &DAG)
static SDValue	performUnpackCombine (SDNode *N, SelectionDAG &DAG, const AArch64Subtarget *Subtarget)
static bool	isHalvingTruncateAndConcatOfLegalIntScalableType (SDNode *N)
static SDValue	tryCombineExtendRShTrunc (SDNode *N, SelectionDAG &DAG)
static SDValue	trySimplifySrlAddToRshrnb (SDValue Srl, SelectionDAG &DAG, const AArch64Subtarget *Subtarget)
static SDValue	performUzpCombine (SDNode *N, SelectionDAG &DAG, const AArch64Subtarget *Subtarget)
static SDValue	performGLD1Combine (SDNode *N, SelectionDAG &DAG)
static SDValue	performVectorShiftCombine (SDNode *N, const AArch64TargetLowering &TLI, TargetLowering::DAGCombinerInfo &DCI)
	Optimize a vector shift instruction and its operand if shifted out bits are not used.
static SDValue	performSunpkloCombine (SDNode *N, SelectionDAG &DAG)
static SDValue	performPostLD1Combine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, bool IsLaneOp)
	Target-specific DAG combine function for post-increment LD1 (lane) and post-increment LD1R.
static bool	performTBISimplification (SDValue Addr, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
	Simplify Addr given that the top byte of it is ignored by HW during address translation.
static SDValue	foldTruncStoreOfExt (SelectionDAG &DAG, SDNode *N)
static SDValue	combineV3I8LoadExt (LoadSDNode *LD, SelectionDAG &DAG)
static SDValue	performLOADCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG, const AArch64Subtarget *Subtarget)
static EVT	tryGetOriginalBoolVectorType (SDValue Op, int Depth=0)
static SDValue	vectorToScalarBitmask (SDNode *N, SelectionDAG &DAG)
static SDValue	combineBoolVectorAndTruncateStore (SelectionDAG &DAG, StoreSDNode *Store)
bool	isHalvingTruncateOfLegalScalableType (EVT SrcVT, EVT DstVT)
static SDValue	combineI8TruncStore (StoreSDNode *ST, SelectionDAG &DAG, const AArch64Subtarget *Subtarget)
static SDValue	performSTORECombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG, const AArch64Subtarget *Subtarget)
static SDValue	performMSTORECombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG, const AArch64Subtarget *Subtarget)
static bool	foldIndexIntoBase (SDValue &BasePtr, SDValue &Index, SDValue Scale, SDLoc DL, SelectionDAG &DAG)
static bool	findMoreOptimalIndexType (const MaskedGatherScatterSDNode *N, SDValue &BasePtr, SDValue &Index, SelectionDAG &DAG)
static SDValue	performMaskedGatherScatterCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
static SDValue	performNEONPostLDSTCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
	Target-specific DAG combine function for NEON load/store intrinsics to merge base address updates.
static bool	checkValueWidth (SDValue V, unsigned width, ISD::LoadExtType &ExtType)
static bool	isEquivalentMaskless (unsigned CC, unsigned width, ISD::LoadExtType ExtType, int AddConstant, int CompConstant)
static SDValue	performSubsToAndsCombine (SDNode *N, SDNode *SubsNode, SDNode *AndNode, SelectionDAG &DAG, unsigned CCIndex, unsigned CmpIndex, unsigned CC)
static SDValue	performCONDCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG, unsigned CCIndex, unsigned CmpIndex)
static SDValue	performBRCONDCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
static SDValue	foldCSELofCTTZ (SDNode *N, SelectionDAG &DAG)
static SDValue	foldCSELOfCSEL (SDNode *Op, SelectionDAG &DAG)
static SDValue	performCSELCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
static SDValue	tryToWidenSetCCOperands (SDNode *Op, SelectionDAG &DAG)
static SDValue	performVecReduceBitwiseCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
static SDValue	performSETCCCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
static SDValue	performFlagSettingCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, unsigned GenericOpcode)
static SDValue	performSetCCPunpkCombine (SDNode *N, SelectionDAG &DAG)
static SDValue	performSetccMergeZeroCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SDValue	getTestBitOperand (SDValue Op, unsigned &Bit, bool &Invert, SelectionDAG &DAG)
static SDValue	performTBZCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
static SDValue	trySwapVSelectOperands (SDNode *N, SelectionDAG &DAG)
static SDValue	performVSelectCombine (SDNode *N, SelectionDAG &DAG)
static SDValue	performSelectCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
	A vector select: "(select vL, vR, (setcc LHS, RHS))" is best performed with the compare-mask instructions rather than going via NZCV, even if LHS and RHS are really scalar.
static SDValue	performDUPCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SDValue	performNVCASTCombine (SDNode *N, SelectionDAG &DAG)
	Get rid of unnecessary NVCASTs (that don't change the type).
static SDValue	performGlobalAddressCombine (SDNode *N, SelectionDAG &DAG, const AArch64Subtarget *Subtarget, const TargetMachine &TM)
static SDValue	performCTLZCombine (SDNode *N, SelectionDAG &DAG, const AArch64Subtarget *Subtarget)
static SDValue	getScaledOffsetForBitWidth (SelectionDAG &DAG, SDValue Offset, SDLoc DL, unsigned BitWidth)
static bool	isValidImmForSVEVecImmAddrMode (unsigned OffsetInBytes, unsigned ScalarSizeInBytes)
	Check if the value of OffsetInBytes can be used as an immediate for the gather load/prefetch and scatter store instructions with vector base and immediate offset addressing mode:
static bool	isValidImmForSVEVecImmAddrMode (SDValue Offset, unsigned ScalarSizeInBytes)
	Check if the value of Offset represents a valid immediate for the SVE gather load/prefetch and scatter store instructiona with vector base and immediate offset addressing mode:
static SDValue	performScatterStoreCombine (SDNode *N, SelectionDAG &DAG, unsigned Opcode, bool OnlyPackedOffsets=true)
static SDValue	performGatherLoadCombine (SDNode *N, SelectionDAG &DAG, unsigned Opcode, bool OnlyPackedOffsets=true)
static SDValue	performSignExtendInRegCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
static SDValue	legalizeSVEGatherPrefetchOffsVec (SDNode *N, SelectionDAG &DAG)
	Legalize the gather prefetch (scalar + vector addressing mode) when the offset vector is an unpacked 32-bit scalable vector.
static SDValue	combineSVEPrefetchVecBaseImmOff (SDNode *N, SelectionDAG &DAG, unsigned ScalarSizeInBytes)
	Combines a node carrying the intrinsic aarch64_sve_prf<T>_gather_scalar_offset into a node that uses aarch64_sve_prfb_gather_uxtw_index when the scalar offset passed to aarch64_sve_prf<T>_gather_scalar_offset is not a valid immediate for the sve gather prefetch instruction with vector plus immediate addressing mode.
static bool	isLanes1toNKnownZero (SDValue Op)
static SDValue	removeRedundantInsertVectorElt (SDNode *N)
static SDValue	performInsertVectorEltCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SDValue	performFPExtendCombine (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const AArch64Subtarget *Subtarget)
static SDValue	performBSPExpandForSVE (SDNode *N, SelectionDAG &DAG, const AArch64Subtarget *Subtarget)
static SDValue	performDupLane128Combine (SDNode *N, SelectionDAG &DAG)
static SDValue	tryCombineMULLWithUZP1 (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
static SDValue	performMULLCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
static SDValue	performScalarToVectorCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
static void	replaceBoolVectorBitcast (SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG)
static void	CustomNonLegalBITCASTResults (SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG, EVT ExtendVT, EVT CastVT)
static void	ReplaceAddWithADDP (SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG, const AArch64Subtarget *Subtarget)
static void	ReplaceReductionResults (SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG, unsigned InterOp, unsigned AcrossOp)
static SDValue	createGPRPairNode (SelectionDAG &DAG, SDValue V)
static void	ReplaceCMP_SWAP_128Results (SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG, const AArch64Subtarget *Subtarget)
static unsigned	getAtomicLoad128Opcode (unsigned ISDOpcode, AtomicOrdering Ordering)
static void	ReplaceATOMIC_LOAD_128Results (SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG, const AArch64Subtarget *Subtarget)
static Value *	UseTlsOffset (IRBuilderBase &IRB, unsigned Offset)
static SDValue	getPredicateForFixedLengthVector (SelectionDAG &DAG, SDLoc &DL, EVT VT)
static SDValue	GenerateFixedLengthSVETBL (SDValue Op, SDValue Op1, SDValue Op2, ArrayRef< int > ShuffleMask, EVT VT, EVT ContainerVT, SelectionDAG &DAG)
static SDValue	emitConditionalComparison (SDValue LHS, SDValue RHS, ISD::CondCode CC, SDValue CCOp, AArch64CC::CondCode Predicate, AArch64CC::CondCode OutCC, const SDLoc &DL, SelectionDAG &DAG)
	can be transformed to: not (and (not (and (setCC (cmp C)) (setCD (cmp D)))) (and (not (setCA (cmp A)) (not (setCB (cmp B))))))" which can be implemented as: cmp C ccmp D, inv(CD), CC ccmp A, CA, inv(CD) ccmp B, CB, inv(CA) check for CB flags
static bool	canEmitConjunction (const SDValue Val, bool &CanNegate, bool &MustBeFirst, bool WillNegate, unsigned Depth=0)
	Returns true if Val is a tree of AND/OR/SETCC operations that can be expressed as a conjunction.
static SDValue	emitConjunctionRec (SelectionDAG &DAG, SDValue Val, AArch64CC::CondCode &OutCC, bool Negate, SDValue CCOp, AArch64CC::CondCode Predicate)
	Emit conjunction or disjunction tree with the CMP/FCMP followed by a chain of CCMP/CFCMP ops.
static SDValue	emitConjunction (SelectionDAG &DAG, SDValue Val, AArch64CC::CondCode &OutCC)
	Emit expression as a conjunction (a series of CCMP/CFCMP ops).

Variables
cl::opt< bool >	EnableAArch64ELFLocalDynamicTLSGeneration ("aarch64-elf-ldtls-generation", cl::Hidden, cl::desc("Allow AArch64 Local Dynamic TLS code generation"), cl::init(false))
static cl::opt< bool >	EnableOptimizeLogicalImm ("aarch64-enable-logical-imm", cl::Hidden, cl::desc("Enable AArch64 logical imm instruction " "optimization"), cl::init(true))
static cl::opt< bool >	EnableCombineMGatherIntrinsics ("aarch64-enable-mgather-combine", cl::Hidden, cl::desc("Combine extends of AArch64 masked " "gather intrinsics"), cl::init(true))
static cl::opt< bool >	EnableExtToTBL ("aarch64-enable-ext-to-tbl", cl::Hidden, cl::desc("Combine ext and trunc to TBL"), cl::init(true))
static cl::opt< unsigned >	MaxXors ("aarch64-max-xors", cl::init(16), cl::Hidden, cl::desc("Maximum of xors"))
cl::opt< bool >	EnableSVEGISel ("aarch64-enable-gisel-sve", cl::Hidden, cl::desc("Enable / disable SVE scalable vectors in Global ISel"), cl::init(false))
static const MVT	MVT_CC = MVT::i32
	Value type used for condition codes.
static const MCPhysReg	GPRArgRegs []
static const MCPhysReg	FPRArgRegs []

Macro Definition Documentation

DEBUG_TYPE

#define DEBUG_TYPE   "aarch64-lower"

Definition at line 109 of file AArch64ISelLowering.cpp.

GET_REGISTER_MATCHER

#define GET_REGISTER_MATCHER

Definition at line 11327 of file AArch64ISelLowering.cpp.

LCALLNAME4

#define LCALLNAME4	(		A,
			B
	)

Value:

  LCALLNAMES(A, B, 1)                                                          \
  LCALLNAMES(A, B, 2) LCALLNAMES(A, B, 4) LCALLNAMES(A, B, 8)

LCALLNAME5

#define LCALLNAME5	(		A,
			B
	)

Value:

  LCALLNAMES(A, B, 1)                                                          \
  LCALLNAMES(A, B, 2)                                                          \
  LCALLNAMES(A, B, 4) LCALLNAMES(A, B, 8) LCALLNAMES(A, B, 16)

LCALLNAMES

#define LCALLNAMES	(		A,
			B,
			N
	)

Value:

  setLibcallName(A##N##_RELAX, #B #N "_relax");                                \
  setLibcallName(A##N##_ACQ, #B #N "_acq");                                    \
  setLibcallName(A##N##_REL, #B #N "_rel");                                    \
  setLibcallName(A##N##_ACQ_REL, #B #N "_acq_rel");

MAKE_CASE

#define MAKE_CASE

(

V

)

Value:

  case V:                                                                      \
    return #V;

Enumeration Type Documentation

PredicateConstraint

enum class PredicateConstraint

strong

Enumerator
Uph
Upl
Upa

Definition at line 11618 of file AArch64ISelLowering.cpp.

ReducedGprConstraint

enum class ReducedGprConstraint

strong

Enumerator
Uci
Ucj

Definition at line 11650 of file AArch64ISelLowering.cpp.

Function Documentation

addRequiredExtensionForVectorMULL()

static SDValue addRequiredExtensionForVectorMULL ( SDValue  N, SelectionDAG &  DAG, const EVT &  OrigTy, const EVT &  ExtTy, unsigned  ExtOpcode  )

static

Definition at line 5014 of file AArch64ISelLowering.cpp.

References assert(), getExtensionTo64Bits(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::EVT::is128BitVector(), and N.

Referenced by skipExtensionForVectorMULL().

analyzeCallOperands()

static void analyzeCallOperands ( const AArch64TargetLowering &  TLI, const AArch64Subtarget *  Subtarget, const TargetLowering::CallLoweringInfo &  CLI, CCState &  CCInfo  )

static

Definition at line 8052 of file AArch64ISelLowering.cpp.

References llvm::CCState::AllocateStack(), llvm::TargetLowering::CallLoweringInfo::Args, llvm::CallingConv::ARM64EC_Thunk_X64, assert(), llvm::TargetLowering::CallLoweringInfo::CallConv, llvm::AArch64TargetLowering::CCAssignFnForCall(), llvm::TargetLowering::CallLoweringInfo::DAG, llvm::CCValAssign::Full, llvm::SelectionDAG::getDataLayout(), llvm::EVT::getSimpleVT(), llvm::TargetLoweringBase::getValueType(), llvm::AArch64Subtarget::isCallingConvWin64(), llvm::EVT::isSimple(), llvm::TargetLowering::CallLoweringInfo::IsVarArg, llvm::TargetLowering::CallLoweringInfo::Outs, and llvm::SmallVectorBase< Size_T >::size().

areExtractExts()

static bool areExtractExts ( Value *  Ext1, Value *  Ext2  )

static

Check if Ext1 and Ext2 are extends of the same type, doubling the bitwidth of the vector elements.

Definition at line 16206 of file AArch64ISelLowering.cpp.

References llvm::PatternMatch::m_Value(), llvm::PatternMatch::m_ZExtOrSExt(), and llvm::PatternMatch::match().

Referenced by llvm::AArch64TargetLowering::shouldSinkOperands(), and llvm::ARMTargetLowering::shouldSinkOperands().

areExtractShuffleVectors()

static bool areExtractShuffleVectors ( Value *  Op1, Value *  Op2, bool  AllowSplat = false  )

static

Check if both Op1 and Op2 are shufflevector extracts of either the lower or upper half of the vector elements.

Definition at line 16150 of file AArch64ISelLowering.cpp.

References llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getFixedValue(), llvm::Type::getPrimitiveSizeInBits(), llvm::Value::getType(), llvm::ShuffleVectorInst::isExtractSubvectorMask(), isSplatShuffle(), llvm::M1(), llvm::PatternMatch::m_Shuffle(), llvm::PatternMatch::m_Undef(), llvm::PatternMatch::m_Value(), and llvm::PatternMatch::match().

Referenced by llvm::AArch64TargetLowering::shouldSinkOperands().

areLoadedOffsetButOtherwiseSame()

static bool areLoadedOffsetButOtherwiseSame ( SDValue  Op0, SDValue  Op1, SelectionDAG &  DAG, unsigned &  NumSubLoads  )

static

Definition at line 21065 of file AArch64ISelLowering.cpp.

References llvm::ISD::ADD, llvm::all_of(), llvm::ISD::ANY_EXTEND, areLoadedOffsetButOtherwiseSame(), llvm::SelectionDAG::areNonVolatileConsecutiveLoads(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::SDValue::getValueType(), llvm::SDValue::hasOneUse(), isLoadOrMultipleLoads(), llvm::ISD::SIGN_EXTEND, llvm::SmallVectorBase< Size_T >::size(), Size, llvm::ISD::SUB, llvm::ISD::ZERO_EXTEND, and llvm::zip().

Referenced by areLoadedOffsetButOtherwiseSame(), and performExtBinopLoadFold().

areOperandsOfVmullHighP64()

static bool areOperandsOfVmullHighP64 ( Value *  Op1, Value *  Op2  )

static

Check if Op1 and Op2 could be used with vmull_high_p64 intrinsic.

Definition at line 16233 of file AArch64ISelLowering.cpp.

References isOperandOfVmullHighP64().

Referenced by llvm::AArch64TargetLowering::shouldSinkOperands().

calculatePreExtendType()

static EVT calculatePreExtendType ( SDValue  Extend )

static

Calculates what the pre-extend type is, based on the extension operation node provided by Extend.

In the case that Extend is a SIGN_EXTEND or a ZERO_EXTEND, the pre-extend type is pulled directly from the operand, while other extend operations need a bit more inspection to get this information.

Parameters

Extend

The SDNode from the DAG that represents the extend operation

Returns

The type representing the Extend source type, or MVT::Other if no valid type can be determined

Definition at line 18500 of file AArch64ISelLowering.cpp.

References llvm::ISD::AND, llvm::ISD::AssertSext, llvm::ISD::AssertZext, llvm::SDValue::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValueType(), llvm::VTSDNode::getVT(), llvm::ISD::SIGN_EXTEND, llvm::ISD::SIGN_EXTEND_INREG, and llvm::ISD::ZERO_EXTEND.

Referenced by performBuildShuffleExtendCombine().

callConvSupportsVarArgs()

static bool callConvSupportsVarArgs ( CallingConv::ID  CC )

static

Return true if the call convention supports varargs Currently only those that pass varargs like the C calling convention does are eligible Calling conventions listed in this function must also be properly handled in AArch64Subtarget::isCallingConvWin64.

Definition at line 8042 of file AArch64ISelLowering.cpp.

References llvm::CallingConv::C, CC, and llvm::CallingConv::PreserveNone.

canEmitConjunction()

static bool canEmitConjunction ( const SDValue  Val, bool &  CanNegate, bool &  MustBeFirst, bool  WillNegate, unsigned  Depth = 0  )

static

Returns true if Val is a tree of AND/OR/SETCC operations that can be expressed as a conjunction.

See CMP;CCMP matching.

Parameters

CanNegate	Set to true if we can negate the whole sub-tree just by changing the conditions on the SETCC tests. (this means we can call emitConjunctionRec() with Negate==true on this sub-tree)
MustBeFirst	Set to true if this subtree needs to be negated and we cannot do the negation naturally. We are required to emit the subtree first in this case.
WillNegate	Is true if are called when the result of this subexpression must be negated. This happens when the outer expression is an OR. We can use this fact to know that we have a double negation (or (or ...) ...) that can be implemented for free.

Definition at line 3639 of file AArch64ISelLowering.cpp.

References llvm::ISD::AND, assert(), canEmitConjunction(), llvm::Depth, llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::SDValue::getValueType(), llvm::SDValue::hasOneUse(), llvm::ISD::OR, and llvm::ISD::SETCC.

Referenced by canEmitConjunction(), emitConjunction(), and emitConjunctionRec().

canGuaranteeTCO()

static bool canGuaranteeTCO ( CallingConv::ID  CC, bool  GuaranteeTailCalls  )

static

Return true if the calling convention is one that we can guarantee TCO for.

Definition at line 8014 of file AArch64ISelLowering.cpp.

References CC, llvm::CallingConv::Fast, llvm::CallingConv::SwiftTail, and llvm::CallingConv::Tail.

Referenced by llvm::AArch64CallLowering::isEligibleForTailCallOptimization(), llvm::AMDGPUCallLowering::isEligibleForTailCallOptimization(), llvm::SITargetLowering::isEligibleForTailCallOptimization(), mayTailCallThisCC(), and shouldGuaranteeTCO().

canLowerSRLToRoundingShiftForVT()

static bool canLowerSRLToRoundingShiftForVT ( SDValue  Shift, EVT  ResVT, SelectionDAG &  DAG, unsigned &  ShiftValue, SDValue &  RShOperand  )

static

Definition at line 15115 of file AArch64ISelLowering.cpp.

References llvm::Add, llvm::ISD::ADD, assert(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::SelectionDAG::getSplatValue(), llvm::SDValue::getValueType(), llvm::EVT::isScalableVT(), and llvm::ISD::SRL.

Referenced by trySimplifySrlAddToRshrnb().

cannotBeIntMin()

static bool cannotBeIntMin ( SDValue  CheckedVal, SelectionDAG &  DAG  )

static

Definition at line 3434 of file AArch64ISelLowering.cpp.

References llvm::SelectionDAG::computeKnownBits(), llvm::KnownBits::getSignedMinValue(), and llvm::APInt::isMinSignedValue().

Referenced by isCMN().

carryFlagToValue()

static SDValue carryFlagToValue ( SDValue  Glue, EVT  VT, SelectionDAG &  DAG, bool  Invert  )

static

Definition at line 4185 of file AArch64ISelLowering.cpp.

References assert(), CC, Cond, llvm::AArch64ISD::CSEL, DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getResNo(), llvm::AArch64CC::HS, and llvm::AArch64CC::LO.

Referenced by lowerADDSUBO_CARRY().

changeFPCCToAArch64CC()

static void changeFPCCToAArch64CC ( ISD::CondCode  CC, AArch64CC::CondCode &  CondCode, AArch64CC::CondCode &  CondCode2  )

static

changeFPCCToAArch64CC - Convert a DAG fp condition code to an AArch64 CC.

Definition at line 3302 of file AArch64ISelLowering.cpp.

References llvm::AArch64CC::AL, CC, llvm::AArch64CC::EQ, llvm::AArch64CC::GE, llvm::AArch64CC::GT, llvm::AArch64CC::HI, llvm::AArch64CC::LE, llvm_unreachable, llvm::AArch64CC::LS, llvm::AArch64CC::LT, llvm::AArch64CC::MI, llvm::AArch64CC::NE, llvm::AArch64CC::PL, llvm::ISD::SETEQ, llvm::ISD::SETGE, llvm::ISD::SETGT, llvm::ISD::SETLE, llvm::ISD::SETLT, llvm::ISD::SETNE, llvm::ISD::SETO, llvm::ISD::SETOEQ, llvm::ISD::SETOGE, llvm::ISD::SETOGT, llvm::ISD::SETOLE, llvm::ISD::SETOLT, llvm::ISD::SETONE, llvm::ISD::SETUEQ, llvm::ISD::SETUGE, llvm::ISD::SETUGT, llvm::ISD::SETULE, llvm::ISD::SETULT, llvm::ISD::SETUNE, llvm::ISD::SETUO, llvm::AArch64CC::VC, and llvm::AArch64CC::VS.

Referenced by changeFPCCToANDAArch64CC(), and changeVectorFPCCToAArch64CC().

changeFPCCToANDAArch64CC()

static void changeFPCCToANDAArch64CC ( ISD::CondCode  CC, AArch64CC::CondCode &  CondCode, AArch64CC::CondCode &  CondCode2  )

static

Convert a DAG fp condition code to an AArch64 CC.

This differs from changeFPCCToAArch64CC in that it returns cond codes that should be AND'ed instead of OR'ed.

Definition at line 3365 of file AArch64ISelLowering.cpp.

References llvm::AArch64CC::AL, assert(), CC, changeFPCCToAArch64CC(), llvm::AArch64CC::LE, llvm::AArch64CC::NE, llvm::AArch64CC::PL, llvm::ISD::SETONE, llvm::ISD::SETUEQ, and llvm::AArch64CC::VC.

Referenced by emitConjunctionRec().

changeIntCCToAArch64CC()

static AArch64CC::CondCode changeIntCCToAArch64CC ( ISD::CondCode  CC )

static

changeIntCCToAArch64CC - Convert a DAG integer condition code to an AArch64 CC

Definition at line 3274 of file AArch64ISelLowering.cpp.

References CC, llvm::AArch64CC::EQ, llvm::AArch64CC::GE, llvm::AArch64CC::GT, llvm::AArch64CC::HI, llvm::AArch64CC::HS, llvm::AArch64CC::LE, llvm_unreachable, llvm::AArch64CC::LO, llvm::AArch64CC::LS, llvm::AArch64CC::LT, llvm::AArch64CC::NE, llvm::ISD::SETEQ, llvm::ISD::SETGE, llvm::ISD::SETGT, llvm::ISD::SETLE, llvm::ISD::SETLT, llvm::ISD::SETNE, llvm::ISD::SETUGE, llvm::ISD::SETUGT, llvm::ISD::SETULE, and llvm::ISD::SETULT.

Referenced by emitConjunctionRec(), and getAArch64Cmp().

changeVectorFPCCToAArch64CC()

static void changeVectorFPCCToAArch64CC ( ISD::CondCode  CC, AArch64CC::CondCode &  CondCode, AArch64CC::CondCode &  CondCode2, bool &  Invert  )

static

changeVectorFPCCToAArch64CC - Convert a DAG fp condition code to an AArch64 CC usable with the vector instructions.

Fewer operations are available without a real NZCV register, so we have to use less efficient combinations to get the same effect.

Definition at line 3395 of file AArch64ISelLowering.cpp.

References CC, changeFPCCToAArch64CC(), llvm::AArch64CC::GE, llvm::AArch64CC::MI, llvm::ISD::SETO, llvm::ISD::SETUEQ, llvm::ISD::SETUGE, llvm::ISD::SETUGT, llvm::ISD::SETULE, llvm::ISD::SETULT, and llvm::ISD::SETUO.

checkValueWidth()

static bool checkValueWidth ( SDValue  V, unsigned  width, ISD::LoadExtType &  ExtType  )

static

Definition at line 23811 of file AArch64ISelLowering.cpp.

References llvm::ISD::AssertSext, llvm::ISD::AssertZext, llvm::ISD::Constant, llvm::LoadSDNode::getExtensionType(), llvm::MemSDNode::getMemoryVT(), llvm::VTSDNode::getVT(), llvm::ISD::LOAD, llvm::ISD::NON_EXTLOAD, llvm::ISD::SEXTLOAD, llvm::ISD::TargetConstant, and llvm::ISD::ZEXTLOAD.

Referenced by performCONDCombine(), and performSETCCCombine().

checkZExtBool()

static bool checkZExtBool ( SDValue  Arg, const SelectionDAG &  DAG  )

static

Definition at line 8297 of file AArch64ISelLowering.cpp.

References llvm::SelectionDAG::computeKnownBits(), llvm::EVT::getSizeInBits(), and llvm::SDValue::getValueType().

combineAcrossLanesIntrinsic()

static SDValue combineAcrossLanesIntrinsic ( unsigned  Opc, SDNode *  N, SelectionDAG &  DAG  )

static

Definition at line 21413 of file AArch64ISelLowering.cpp.

References llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), and N.

Referenced by performIntrinsicCombine().

combineBoolVectorAndTruncateStore()

static SDValue combineBoolVectorAndTruncateStore ( SelectionDAG &  DAG, StoreSDNode *  Store  )

static

Definition at line 23314 of file AArch64ISelLowering.cpp.

References llvm::ISD::BUILD_VECTOR, DL, llvm::SelectionDAG::getContext(), llvm::EVT::getIntegerVT(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SelectionDAG::getStore(), llvm::EVT::getStoreSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::EVT::isVector(), llvm::ISD::TRUNCATE, and vectorToScalarBitmask().

Referenced by performSTORECombine().

combineI8TruncStore()

static SDValue combineI8TruncStore ( StoreSDNode *  ST, SelectionDAG &  DAG, const AArch64Subtarget *  Subtarget  )

static

Definition at line 23352 of file AArch64ISelLowering.cpp.

References assert(), llvm::ISD::BITCAST, DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::TypeSize::getFixed(), llvm::SelectionDAG::getMachineFunction(), llvm::MachineFunction::getMachineMemOperand(), llvm::SelectionDAG::getMemBasePlusOffset(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getStore(), llvm::SelectionDAG::getUNDEF(), llvm::SelectionDAG::getVectorIdxConstant(), llvm::EVT::getVectorVT(), llvm::ISD::INSERT_SUBVECTOR, llvm::AArch64Subtarget::isLittleEndian(), and llvm::ISD::TRUNCATE.

Referenced by performSTORECombine().

combineSVEPrefetchVecBaseImmOff()

static SDValue combineSVEPrefetchVecBaseImmOff ( SDNode *  N, SelectionDAG &  DAG, unsigned  ScalarSizeInBytes  )

static

Combines a node carrying the intrinsic aarch64_sve_prf<T>_gather_scalar_offset into a node that uses aarch64_sve_prfb_gather_uxtw_index when the scalar offset passed to aarch64_sve_prf<T>_gather_scalar_offset is not a valid immediate for the sve gather prefetch instruction with vector plus immediate addressing mode.

Definition at line 25364 of file AArch64ISelLowering.cpp.

References DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getVTList(), isValidImmForSVEVecImmAddrMode(), N, and std::swap().

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

combineSVEReductionFP()

static SDValue combineSVEReductionFP ( SDNode *  N, unsigned  Opc, SelectionDAG &  DAG  )

static

Definition at line 21600 of file AArch64ISelLowering.cpp.

References DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getValueType(), and N.

Referenced by performIntrinsicCombine().

combineSVEReductionInt()

static SDValue combineSVEReductionInt ( SDNode *  N, unsigned  Opc, SelectionDAG &  DAG  )

static

Definition at line 21581 of file AArch64ISelLowering.cpp.

References DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), getPackedSVEVectorVT(), and N.

Referenced by performIntrinsicCombine().

combineSVEReductionOrderedFP()

static SDValue combineSVEReductionOrderedFP ( SDNode *  N, unsigned  Opc, SelectionDAG &  DAG  )

static

Definition at line 21617 of file AArch64ISelLowering.cpp.

References DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::ISD::INSERT_VECTOR_ELT, and N.

Referenced by performIntrinsicCombine().

combineV3I8LoadExt()

static SDValue combineV3I8LoadExt ( LoadSDNode *  LD, SelectionDAG &  DAG  )

static

Definition at line 23082 of file AArch64ISelLowering.cpp.

References assert(), llvm::ISD::BITCAST, DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::TypeSize::getFixed(), llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getMachineFunction(), llvm::MachineFunction::getMachineMemOperand(), llvm::SelectionDAG::getMemBasePlusOffset(), llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), llvm::EVT::getVectorVT(), llvm::Or, llvm::ISD::OR, llvm::ISD::SHL, llvm::ISD::TokenFactor, and llvm::ISD::ZERO_EXTEND.

Referenced by performLOADCombine().

ConstantBuildVector()

static SDValue ConstantBuildVector ( SDValue  Op, SelectionDAG &  DAG, const AArch64Subtarget *  ST  )

static

Definition at line 14136 of file AArch64ISelLowering.cpp.

References assert(), DL, llvm::AArch64ISD::FMOV, llvm::ISD::FNEG, llvm::APInt::getHighBitsSet(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::MVT::getVectorVT(), llvm::AArch64ISD::MOVI, llvm::AArch64ISD::MOVIedit, llvm::AArch64ISD::MOVImsl, llvm::AArch64ISD::MOVIshift, llvm::AArch64ISD::MVNImsl, llvm::AArch64ISD::MVNIshift, llvm::AArch64ISD::NVCAST, resolveBuildVector(), tryAdvSIMDModImm16(), tryAdvSIMDModImm32(), tryAdvSIMDModImm321s(), tryAdvSIMDModImm64(), tryAdvSIMDModImm8(), tryAdvSIMDModImmFP(), and llvm::APInt::zext().

constructDup()

static SDValue constructDup ( SDValue  V, int  Lane, SDLoc  dl, EVT  VT, unsigned  Opcode, SelectionDAG &  DAG  )

static

Definition at line 13080 of file AArch64ISelLowering.cpp.

References llvm::ISD::BITCAST, llvm::ISD::CONCAT_VECTORS, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getConstantOperandVal(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::MVT::getScalarType(), llvm::SDValue::getScalarValueSizeInBits(), llvm::SDValue::getSimpleValueType(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorVT(), Idx, llvm::EVT::is128BitVector(), and WidenVector().

convertFixedMaskToScalableVector()

static SDValue convertFixedMaskToScalableVector ( SDValue  Mask, SelectionDAG &  DAG  )

static

Definition at line 27756 of file AArch64ISelLowering.cpp.

References convertToScalableVector(), DL, llvm::SelectionDAG::getConstant(), getContainerForFixedLengthVector(), llvm::SelectionDAG::getNode(), getPredicateForFixedLengthVector(), llvm::SDValue::getValueType(), llvm::ISD::isBuildVectorAllOnes(), and llvm::AArch64ISD::SETCC_MERGE_ZERO.

convertFromScalableVector()

static SDValue convertFromScalableVector ( SelectionDAG &  DAG, EVT  VT, SDValue  V  )

static

Definition at line 27706 of file AArch64ISelLowering.cpp.

References assert(), DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), and llvm::EVT::isFixedLengthVector().

Referenced by convertLocVTToValVT(), GenerateFixedLengthSVETBL(), getDeinterleaveViaVNSRL(), getVCIXISDNodeWCHAIN(), getWideningInterleave(), lowerBUILD_VECTOR(), lowerBuildVectorOfConstants(), lowerBuildVectorViaDominantValues(), lowerFMAXIMUM_FMINIMUM(), lowerFP_TO_INT_SAT(), llvm::RISCVTargetLowering::LowerOperation(), lowerShuffleViaVRegSplitting(), lowerVECTOR_SHUFFLE(), lowerVECTOR_SHUFFLEAsVSlide1(), lowerVECTOR_SHUFFLEAsVSlidedown(), lowerVECTOR_SHUFFLEAsVSlideup(), lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND(), lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND(), lowerVectorXRINT(), matchSplatAsGather(), performFP_TO_INTCombine(), and performSVEMulAddSubCombine().

convertMergedOpToPredOp()

static SDValue convertMergedOpToPredOp ( SDNode *  N, unsigned  Opc, SelectionDAG &  DAG, bool  UnpredOp = false, bool  SwapOperands = false  )

static

Definition at line 21643 of file AArch64ISelLowering.cpp.

References assert(), llvm::SelectionDAG::getNode(), llvm::ISD::INTRINSIC_WO_CHAIN, isAllActivePredicate(), and N.

Referenced by performIntrinsicCombine().

convertToScalableVector()

static SDValue convertToScalableVector ( SelectionDAG &  DAG, EVT  VT, SDValue  V  )

static

Definition at line 27695 of file AArch64ISelLowering.cpp.

References assert(), DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getUNDEF(), llvm::ISD::INSERT_SUBVECTOR, and llvm::EVT::isScalableVector().

Referenced by convertFixedMaskToScalableVector(), convertValVTToLocVT(), GenerateFixedLengthSVETBL(), getDeinterleaveViaVNSRL(), getWideningInterleave(), lowerBUILD_VECTOR(), lowerBuildVectorOfConstants(), lowerBuildVectorViaDominantValues(), lowerCttzElts(), lowerFMAXIMUM_FMINIMUM(), lowerFP_TO_INT_SAT(), llvm::RISCVTargetLowering::LowerOperation(), lowerScalarInsert(), lowerShuffleViaVRegSplitting(), lowerVECTOR_SHUFFLE(), lowerVECTOR_SHUFFLEAsVSlide1(), lowerVECTOR_SHUFFLEAsVSlidedown(), lowerVECTOR_SHUFFLEAsVSlideup(), lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND(), lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND(), lowerVectorXRINT(), matchSplatAsGather(), performFP_TO_INTCombine(), performSVEMulAddSubCombine(), processVCIXOperands(), and llvm::RISCVTargetLowering::ReplaceNodeResults().

createGPRPairNode()

static SDValue createGPRPairNode ( SelectionDAG &  DAG, SDValue  V  )

static

Definition at line 26435 of file AArch64ISelLowering.cpp.

References llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getMachineNode(), llvm::SelectionDAG::getTargetConstant(), llvm::DataLayout::isBigEndian(), llvm::SelectionDAG::SplitScalar(), and std::swap().

Referenced by ReplaceCMP_SWAP_128Results(), and ReplaceCMP_SWAP_64Results().

createTblForTrunc()

static void createTblForTrunc ( TruncInst *  TI, bool  IsLittleEndian  )

static

Definition at line 16617 of file AArch64ISelLowering.cpp.

References assert(), llvm::SmallVectorTemplateCommon< T, typename >::back(), llvm::SmallVectorTemplateCommon< T, typename >::begin(), llvm::SmallVectorImpl< T >::clear(), llvm::IRBuilderBase::CreateBitCast(), llvm::IRBuilderBase::CreateCall(), llvm::IRBuilderBase::CreateShuffleVector(), llvm::SmallVectorBase< Size_T >::empty(), llvm::Function::empty(), llvm::SmallVectorTemplateCommon< T, typename >::end(), llvm::Instruction::eraseFromParent(), F, llvm::ConstantVector::get(), llvm::FixedVectorType::get(), llvm::Intrinsic::getDeclaration(), llvm::IRBuilderBase::getInt8(), llvm::IRBuilderBase::getInt8Ty(), llvm::Instruction::getModule(), llvm::User::getOperand(), llvm::Value::getType(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::Value::replaceAllUsesWith(), Results, llvm::SmallVectorBase< Size_T >::size(), and llvm::Function::size().

Referenced by llvm::AArch64TargetLowering::optimizeExtendOrTruncateConversion().

createTblShuffleForSExt()

static Value * createTblShuffleForSExt ( IRBuilderBase &  Builder, Value *  Op, FixedVectorType *  DstTy, bool  IsLittleEndian  )

static

Definition at line 16599 of file AArch64ISelLowering.cpp.

References llvm::IRBuilderBase::CreateInsertElement(), llvm::IRBuilderBase::CreateShuffleVector(), createTblShuffleMask(), llvm::PoisonValue::get(), llvm::VectorType::getElementType(), and llvm::IRBuilderBase::getInt8().

Referenced by llvm::AArch64TargetLowering::optimizeExtendOrTruncateConversion().

createTblShuffleForZExt()

static Value * createTblShuffleForZExt ( IRBuilderBase &  Builder, Value *  Op, FixedVectorType *  ZExtTy, FixedVectorType *  DstTy, bool  IsLittleEndian  )

static

Definition at line 16577 of file AArch64ISelLowering.cpp.

References llvm::IRBuilderBase::CreateBitCast(), llvm::IRBuilderBase::CreateInsertElement(), llvm::IRBuilderBase::CreateShuffleVector(), createTblShuffleMask(), llvm::IRBuilderBase::CreateZExt(), llvm::PoisonValue::get(), llvm::VectorType::getElementType(), and llvm::IRBuilderBase::getInt8().

Referenced by llvm::AArch64TargetLowering::optimizeExtendOrTruncateConversion().

createTblShuffleMask()

static bool createTblShuffleMask ( unsigned  SrcWidth, unsigned  DstWidth, unsigned  NumElts, bool  IsLittleEndian, SmallVectorImpl< int > &  Mask  )

static

Definition at line 16554 of file AArch64ISelLowering.cpp.

References assert(), and I.

Referenced by createTblShuffleForSExt(), and createTblShuffleForZExt().

CustomNonLegalBITCASTResults()

static void CustomNonLegalBITCASTResults ( SDNode *  N, SmallVectorImpl< SDValue > &  Results, SelectionDAG &  DAG, EVT  ExtendVT, EVT  CastVT  )

static

Definition at line 26270 of file AArch64ISelLowering.cpp.

References llvm::ISD::BITCAST, DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getVectorIdxConstant(), N, Results, and llvm::ISD::SCALAR_TO_VECTOR.

emitComparison()

static SDValue emitComparison ( SDValue  LHS, SDValue  RHS, ISD::CondCode  CC, const SDLoc &  dl, SelectionDAG &  DAG  )

static

Definition at line 3477 of file AArch64ISelLowering.cpp.

References llvm::AArch64ISD::ADDS, llvm::ISD::AND, llvm::AArch64ISD::ANDS, assert(), CC, llvm::AArch64ISD::FCMP, llvm::ISD::FP_EXTEND, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getSubtarget(), llvm::SDValue::getValue(), llvm::SelectionDAG::getVTList(), isCMN(), llvm::EVT::isFloatingPoint(), llvm::isNullConstant(), LHS, MVT_CC, llvm::SelectionDAG::ReplaceAllUsesWith(), RHS, llvm::ISD::SUB, and llvm::AArch64ISD::SUBS.

Referenced by emitConjunctionRec(), and getAArch64Cmp().

emitConditionalComparison()

static SDValue emitConditionalComparison ( SDValue  LHS, SDValue  RHS, ISD::CondCode  CC, SDValue  CCOp, AArch64CC::CondCode  Predicate, AArch64CC::CondCode  OutCC, const SDLoc &  DL, SelectionDAG &  DAG  )

static

can be transformed to: not (and (not (and (setCC (cmp C)) (setCD (cmp D)))) (and (not (setCA (cmp A)) (not (setCB (cmp B))))))" which can be implemented as: cmp C ccmp D, inv(CD), CC ccmp A, CA, inv(CD) ccmp B, CB, inv(CA) check for CB flags

A counterexample is "or (and A B) (and C D)" which translates to not (and (not (and (not A) (not B))) (not (and (not C) (not D)))), we can only implement 1 of the inner (not) operations, but not both! Create a conditional comparison; Use CCMP, CCMN or FCCMP as appropriate.

Definition at line 3583 of file AArch64ISelLowering.cpp.

References assert(), CC, llvm::AArch64ISD::CCMN, llvm::AArch64ISD::CCMP, DL, llvm::AArch64ISD::FCCMP, llvm::ISD::FP_EXTEND, llvm::SelectionDAG::getConstant(), llvm::AArch64CC::getInvertedCondCode(), llvm::SelectionDAG::getNode(), llvm::AArch64CC::getNZCVToSatisfyCondCode(), llvm::SelectionDAG::getSubtarget(), isCMN(), llvm::isNullConstant(), LHS, MVT_CC, RHS, and llvm::ISD::SUB.

Referenced by emitConjunctionRec().

emitConjunction()

static SDValue emitConjunction ( SelectionDAG &  DAG, SDValue  Val, AArch64CC::CondCode &  OutCC  )

static

Emit expression as a conjunction (a series of CCMP/CFCMP ops).

In some cases this is even possible with OR operations in the expression. See CMP;CCMP matching.

See also

emitConjunctionRec().

Definition at line 3813 of file AArch64ISelLowering.cpp.

References llvm::AArch64CC::AL, canEmitConjunction(), and emitConjunctionRec().

Referenced by getAArch64Cmp(), LowerBRCOND(), and performANDSETCCCombine().

emitConjunctionRec()

static SDValue emitConjunctionRec ( SelectionDAG &  DAG, SDValue  Val, AArch64CC::CondCode &  OutCC, bool  Negate, SDValue  CCOp, AArch64CC::CondCode  Predicate  )

static

Emit conjunction or disjunction tree with the CMP/FCMP followed by a chain of CCMP/CFCMP ops.

See CMP;CCMP matching. Tries to transform the given i1 producing node Val to a series compare and conditional compare operations.

Returns

an NZCV flags producing node and sets OutCC to the flags that should be tested or returns SDValue() if transformation was not possible. Negate is true if we want this sub-tree being negated just by changing SETCC conditions.

Definition at line 3701 of file AArch64ISelLowering.cpp.

References llvm::AArch64CC::AL, llvm::ISD::AND, assert(), canEmitConjunction(), CC, changeFPCCToANDAArch64CC(), changeIntCCToAArch64CC(), DL, emitComparison(), emitConditionalComparison(), emitConjunctionRec(), llvm::AArch64CC::getInvertedCondCode(), llvm::SDValue::getNode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::SDNode::hasOneUse(), LHS, llvm::ISD::OR, RHS, llvm::ISD::SETCC, and std::swap().

Referenced by emitConjunction(), and emitConjunctionRec().

emitStrictFPComparison()

static SDValue emitStrictFPComparison ( SDValue  LHS, SDValue  RHS, const SDLoc &  dl, SelectionDAG &  DAG, SDValue  Chain, bool  IsSignaling  )

static

Definition at line 3456 of file AArch64ISelLowering.cpp.

References assert(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getSubtarget(), LHS, RHS, llvm::AArch64ISD::STRICT_FCMP, llvm::AArch64ISD::STRICT_FCMPE, and llvm::ISD::STRICT_FP_EXTEND.

EmitVectorComparison()

static SDValue EmitVectorComparison ( SDValue  LHS, SDValue  RHS, AArch64CC::CondCode  CC, bool  NoNans, EVT  VT, const SDLoc &  dl, SelectionDAG &  DAG  )

static

Definition at line 15226 of file AArch64ISelLowering.cpp.

References assert(), CC, llvm::AArch64ISD::CMEQ, llvm::AArch64ISD::CMEQz, llvm::AArch64ISD::CMGE, llvm::AArch64ISD::CMGEz, llvm::AArch64ISD::CMGT, llvm::AArch64ISD::CMGTz, llvm::AArch64ISD::CMHI, llvm::AArch64ISD::CMHS, llvm::AArch64ISD::CMLEz, llvm::AArch64ISD::CMLTz, llvm::AArch64CC::EQ, llvm::AArch64ISD::FCMEQ, llvm::AArch64ISD::FCMEQz, llvm::AArch64ISD::FCMGE, llvm::AArch64ISD::FCMGEz, llvm::AArch64ISD::FCMGT, llvm::AArch64ISD::FCMGTz, llvm::AArch64ISD::FCMLEz, llvm::AArch64ISD::FCMLTz, llvm::AArch64CC::GE, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNOT(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getSizeInBits(), llvm::EVT::getVectorElementType(), llvm::AArch64CC::GT, llvm::AArch64CC::HI, llvm::AArch64CC::HS, llvm::APInt::isAllOnes(), llvm::BuildVectorSDNode::isConstantSplat(), llvm::EVT::isFloatingPoint(), llvm::AArch64CC::LE, LHS, llvm::AArch64CC::LO, llvm::AArch64CC::LS, llvm::AArch64CC::LT, llvm::AArch64CC::MI, llvm::AArch64CC::NE, and RHS.

extractPtrauthBlendDiscriminators()

static std::tuple< SDValue, SDValue > extractPtrauthBlendDiscriminators ( SDValue  Disc, SelectionDAG *  DAG  )

static

Definition at line 343 of file AArch64ISelLowering.cpp.

References DL, llvm::SDNode::getConstantOperandVal(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getRegister(), llvm::SelectionDAG::getTargetConstant(), and llvm::ISD::INTRINSIC_WO_CHAIN.

findMoreOptimalIndexType()

static bool findMoreOptimalIndexType ( const MaskedGatherScatterSDNode *  N, SDValue &  BasePtr, SDValue &  Index, SelectionDAG &  DAG  )

static

Definition at line 23564 of file AArch64ISelLowering.cpp.

References llvm::EVT::changeVectorElementType(), foldIndexIntoBase(), llvm::SDValue::getConstantOperandVal(), llvm::AArch64Subtarget::getMaxSVEVectorSizeInBits(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::SelectionDAG::getSplatValue(), llvm::SelectionDAG::getStepVector(), llvm::SelectionDAG::getSubtarget(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorMinNumElements(), llvm::EVT::isFixedLengthVector(), llvm::ISD::isVectorShrinkable(), N, RHS, llvm::ISD::SHL, llvm::ISD::STEP_VECTOR, llvm::AArch64::SVEBitsPerBlock, and llvm::ISD::TRUNCATE.

Referenced by performMaskedGatherScatterCombine().

foldADCToCINC()

static SDValue foldADCToCINC ( SDNode *  N, SelectionDAG &  DAG  )

static

Definition at line 20612 of file AArch64ISelLowering.cpp.

References CC, Cond, llvm::AArch64ISD::CSINC, DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::isNullConstant(), LHS, llvm::AArch64CC::LO, N, and RHS.

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

foldCSELOfCSEL()

static SDValue foldCSELOfCSEL ( SDNode *  Op, SelectionDAG &  DAG  )

static

Definition at line 24227 of file AArch64ISelLowering.cpp.

References CC, Cond, llvm::AArch64ISD::CSEL, DL, llvm::AArch64CC::EQ, llvm::ConstantSDNode::getAPIntValue(), llvm::SelectionDAG::getConstant(), llvm::SDNode::getConstantOperandVal(), llvm::AArch64CC::getInvertedCondCode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), isCMP(), llvm::AArch64CC::NE, std::swap(), X, and Y.

Referenced by performCSELCombine().

foldCSELofCTTZ()

static SDValue foldCSELofCTTZ ( SDNode *  N, SelectionDAG &  DAG  )

static

Definition at line 24179 of file AArch64ISelLowering.cpp.

References llvm::ISD::AND, assert(), llvm::BitWidth, CC, llvm::ISD::CTTZ, llvm::AArch64CC::EQ, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::isNullConstant(), N, llvm::AArch64CC::NE, llvm::AArch64ISD::SUBS, llvm::ISD::TRUNCATE, and X.

Referenced by performCSELCombine().

foldIndexIntoBase()

static bool foldIndexIntoBase ( SDValue &  BasePtr, SDValue &  Index, SDValue  Scale, SDLoc  DL, SelectionDAG &  DAG  )

static

Returns

true if part of the index was folded into the Base.

Definition at line 23514 of file AArch64ISelLowering.cpp.

References llvm::Add, llvm::ISD::ADD, DL, llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getSplatValue(), llvm::EVT::getVectorElementType(), llvm::EVT::isVector(), llvm::ISD::MUL, llvm::Offset, and llvm::ISD::SHL.

Referenced by findMoreOptimalIndexType().

foldOverflowCheck()

static SDValue foldOverflowCheck ( SDNode *  Op, SelectionDAG &  DAG, bool  IsAdd  )

static

Definition at line 20588 of file AArch64ISelLowering.cpp.

References CC, getCSETCondCode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::AArch64CC::HS, isCMP(), llvm::isNullConstant(), llvm::isOneConstant(), and llvm::AArch64CC::LO.

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

foldTruncStoreOfExt()

static SDValue foldTruncStoreOfExt ( SelectionDAG &  DAG, SDNode *  N  )

static

Definition at line 23044 of file AArch64ISelLowering.cpp.

References llvm::ISD::ANY_EXTEND, assert(), llvm::SelectionDAG::getStore(), llvm::SDValue::getValueType(), llvm::ISD::MSTORE, N, llvm::ISD::SIGN_EXTEND, llvm::ISD::STORE, and llvm::ISD::ZERO_EXTEND.

Referenced by performSTORECombine().

foldVectorXorShiftIntoCmp()

static SDValue foldVectorXorShiftIntoCmp ( SDNode *  N, SelectionDAG &  DAG, const AArch64Subtarget *  Subtarget  )

static

Turn vector tests of the signbit in the form of: xor (sra X, elt_size(X)-1), -1 into: cmge X, X, #0.

Definition at line 18057 of file AArch64ISelLowering.cpp.

References llvm::AArch64ISD::CMGEz, llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::SDValue::hasOneUse(), llvm::ISD::isBuildVectorAllOnes(), llvm::EVT::isVector(), N, and llvm::AArch64ISD::VASHR.

Referenced by combineXor(), and performXorCombine().

GenerateFixedLengthSVETBL()

static SDValue GenerateFixedLengthSVETBL ( SDValue  Op, SDValue  Op1, SDValue  Op2, ArrayRef< int >  ShuffleMask, EVT  VT, EVT  ContainerVT, SelectionDAG &  DAG  )

static

Definition at line 28578 of file AArch64ISelLowering.cpp.

References llvm::ISD::ADD, assert(), llvm::ISD::BITCAST, llvm::EVT::changeTypeToInteger(), convertFromScalableVector(), convertToScalableVector(), llvm::SmallVectorTemplateCommon< T, typename >::data(), DL, llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), getContainerForFixedLengthVector(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getSubtarget(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::SelectionDAG::getVScale(), llvm::APInt::getZExtValue(), llvm::ISD::INTRINSIC_WO_CHAIN, llvm::ShuffleVectorInst::isSingleSourceMask(), llvm::ISD::MUL, llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::ArrayRef< T >::size(), llvm::ISD::SPLAT_VECTOR, and llvm::AArch64::SVEBitsPerBlock.

GeneratePerfectShuffle()

static SDValue GeneratePerfectShuffle ( unsigned  ID, SDValue  V1, SDValue  V2, unsigned  PFEntry, SDValue  LHS, SDValue  RHS, SelectionDAG &  DAG, const SDLoc &  dl  )

static

GeneratePerfectShuffle - Given an entry in the perfect-shuffle table, emit the specified operations to build the shuffle.

ID is the perfect-shuffle

Definition at line 12832 of file AArch64ISelLowering.cpp.

References assert(), llvm::AArch64ISD::DUPLANE16, llvm::AArch64ISD::DUPLANE32, llvm::AArch64ISD::DUPLANE64, llvm::AArch64ISD::DUPLANE8, llvm::AArch64ISD::EXT, llvm::ISD::EXTRACT_VECTOR_ELT, GeneratePerfectShuffle(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), getExtFactor(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::SelectionDAG::getVectorIdxConstant(), llvm::ISD::INSERT_VECTOR_ELT, LHS, llvm_unreachable, OP_COPY, OP_VDUP0, OP_VDUP1, OP_VDUP2, OP_VDUP3, OP_VEXT1, OP_VEXT2, OP_VEXT3, OP_VREV, OP_VTRNL, OP_VTRNR, OP_VUZPL, OP_VUZPR, OP_VZIPL, OP_VZIPR, llvm::PerfectShuffleTable, llvm::AArch64ISD::REV16, llvm::AArch64ISD::REV32, llvm::AArch64ISD::REV64, RHS, llvm::AArch64ISD::TRN1, llvm::AArch64ISD::TRN2, llvm::AArch64ISD::UZP1, llvm::AArch64ISD::UZP2, WidenVector(), llvm::AArch64ISD::ZIP1, and llvm::AArch64ISD::ZIP2.

Referenced by GeneratePerfectShuffle(), and LowerVECTOR_SHUFFLE().

GenerateTBL()

static SDValue GenerateTBL ( SDValue  Op, ArrayRef< int >  ShuffleMask, SelectionDAG &  DAG  )

static

Definition at line 12993 of file AArch64ISelLowering.cpp.

References llvm::ISD::BITCAST, llvm::ISD::CONCAT_VECTORS, llvm::SmallVectorTemplateCommon< T, typename >::data(), DL, llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::ISD::INTRINSIC_WO_CHAIN, llvm::SDValue::isUndef(), isZerosVector(), llvm::Offset, llvm::SmallVectorTemplateBase< T, bool >::push_back(), and std::swap().

getAArch64Cmp()

static SDValue getAArch64Cmp ( SDValue  LHS, SDValue  RHS, ISD::CondCode  CC, SDValue &  AArch64cc, SelectionDAG &  DAG, const SDLoc &  dl  )

static

Definition at line 3861 of file AArch64ISelLowering.cpp.

References llvm::CallingConv::C, CC, changeIntCCToAArch64CC(), emitComparison(), emitConjunction(), getCmpOperandFoldingProfit(), llvm::SelectionDAG::getConstant(), llvm::AArch64CC::getInvertedCondCode(), llvm::SelectionDAG::getNode(), llvm::ISD::getSetCCSwappedOperands(), llvm::SelectionDAG::getSignedConstant(), llvm::SelectionDAG::getValueType(), llvm::ConstantSDNode::getZExtValue(), INT64_MAX, isCMN(), isLegalArithImmed(), llvm::ConstantSDNode::isOne(), llvm::ConstantSDNode::isZero(), LHS, MVT_CC, RHS, llvm::ISD::SETGE, llvm::ISD::SETGT, llvm::ISD::SETLE, llvm::ISD::SETLT, llvm::ISD::SETNE, llvm::ISD::SETUGE, llvm::ISD::SETUGT, llvm::ISD::SETULE, llvm::ISD::SETULT, llvm::ISD::SIGN_EXTEND_INREG, std::swap(), UINT64_MAX, and llvm::ISD::ZEXTLOAD.

Referenced by performSetccAddFolding().

getAArch64XALUOOp()

static std::pair< SDValue, SDValue > getAArch64XALUOOp ( AArch64CC::CondCode &  CC, SDValue  Op, SelectionDAG &  DAG  )

static

Definition at line 3997 of file AArch64ISelLowering.cpp.

References llvm::AArch64ISD::ADDS, llvm::AArch64ISD::ANDS, assert(), CC, DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getValue(), llvm::SelectionDAG::getVTList(), llvm::AArch64CC::HS, LHS, llvm_unreachable, llvm::AArch64CC::LO, llvm::Mul, llvm::ISD::MUL, llvm::ISD::MULHS, llvm::ISD::MULHU, MVT_CC, llvm::AArch64CC::NE, RHS, llvm::ISD::SADDO, llvm::ISD::SIGN_EXTEND, llvm::ISD::SMULO, llvm::ISD::SRA, llvm::ISD::SSUBO, llvm::AArch64ISD::SUBS, llvm::ISD::TRUNCATE, llvm::ISD::UADDO, llvm::ISD::UMULO, llvm::ISD::USUBO, llvm::AArch64CC::VS, and llvm::ISD::ZERO_EXTEND.

Referenced by LowerXALUO().

getAtomicLoad128Opcode()

static unsigned getAtomicLoad128Opcode ( unsigned  ISDOpcode, AtomicOrdering  Ordering  )

static

Definition at line 26537 of file AArch64ISelLowering.cpp.

References llvm::Acquire, llvm::AcquireRelease, assert(), llvm::ISD::ATOMIC_LOAD_ADD, llvm::ISD::ATOMIC_LOAD_AND, llvm::ISD::ATOMIC_LOAD_CLR, llvm::ISD::ATOMIC_LOAD_OR, llvm::ISD::ATOMIC_LOAD_SUB, llvm::ISD::ATOMIC_SWAP, llvm_unreachable, llvm::Monotonic, llvm::Release, and llvm::SequentiallyConsistent.

Referenced by ReplaceATOMIC_LOAD_128Results().

getCmpOperandFoldingProfit()

static unsigned getCmpOperandFoldingProfit ( SDValue  Op )

static

Returns how profitable it is to fold a comparison's operand's shift and/or extension operations.

Definition at line 3827 of file AArch64ISelLowering.cpp.

References llvm::ISD::AND, llvm::ISD::SHL, llvm::ISD::SIGN_EXTEND_INREG, llvm::ISD::SRA, and llvm::ISD::SRL.

Referenced by getAArch64Cmp().

getConstantLaneNumOfExtractHalfOperand()

static std::optional< uint64_t > getConstantLaneNumOfExtractHalfOperand ( SDValue &  Op )

static

Definition at line 5034 of file AArch64ISelLowering.cpp.

References llvm::CallingConv::C, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), and llvm::EVT::isFixedLengthVector().

getContainerForFixedLengthVector()

static EVT getContainerForFixedLengthVector ( SelectionDAG &  DAG, EVT  VT  )

static

Definition at line 27607 of file AArch64ISelLowering.cpp.

References assert(), llvm::EVT::getSimpleVT(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::EVT::getVectorElementType(), llvm::EVT::isFixedLengthVector(), llvm::TargetLoweringBase::isTypeLegal(), llvm_unreachable, and llvm::MVT::SimpleTy.

Referenced by convertFixedMaskToScalableVector(), GenerateFixedLengthSVETBL(), getContainerForFixedLengthVector(), getDeinterleaveViaVNSRL(), getVCIXISDNodeWCHAIN(), getWideningInterleave(), lowerBUILD_VECTOR(), lowerBuildVectorOfConstants(), lowerBuildVectorViaDominantValues(), lowerCttzElts(), lowerFMAXIMUM_FMINIMUM(), lowerFP_TO_INT_SAT(), lowerScalarInsert(), lowerShuffleViaVRegSplitting(), lowerVECTOR_SHUFFLE(), lowerVECTOR_SHUFFLEAsVSlide1(), lowerVECTOR_SHUFFLEAsVSlidedown(), lowerVECTOR_SHUFFLEAsVSlideup(), lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND(), lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND(), lowerVectorXRINT(), matchSplatAsGather(), performFP_TO_INTCombine(), and processVCIXOperands().

getCSETCondCode()

static std::optional< AArch64CC::CondCode > getCSETCondCode ( SDValue  Op )

static

Definition at line 20570 of file AArch64ISelLowering.cpp.

References llvm::AArch64CC::AL, CC, llvm::AArch64ISD::CSEL, llvm::isNullConstant(), llvm::isOneConstant(), and llvm::AArch64CC::NV.

Referenced by foldOverflowCheck().

getDeinterleave2Values()

bool getDeinterleave2Values	(	Value *	DI,
		SmallVectorImpl< Instruction * > &	DeinterleavedValues,
		SmallVectorImpl< Instruction * > &	DeInterleaveDeadInsts
	)

Definition at line 17326 of file AArch64ISelLowering.cpp.

References llvm::SmallVectorTemplateCommon< T, typename >::begin(), llvm::dbgs(), llvm::SmallVectorTemplateCommon< T, typename >::end(), llvm::Value::hasNUses(), llvm::SmallVectorImpl< T >::insert(), LLVM_DEBUG, llvm::PatternMatch::m_Specific(), llvm::PatternMatch::match(), llvm::SmallVectorImpl< T >::resize(), and llvm::Value::user_begin().

Referenced by getDeinterleavedValues().

getDeinterleave4Values()

bool getDeinterleave4Values	(	Value *	DI,
		SmallVectorImpl< Instruction * > &	DeinterleavedValues,
		SmallVectorImpl< Instruction * > &	DeInterleaveDeadInsts
	)

Definition at line 17373 of file AArch64ISelLowering.cpp.

References A, B, llvm::SmallVectorTemplateCommon< T, typename >::begin(), llvm::CallingConv::C, D, llvm::dbgs(), llvm::SmallVectorTemplateCommon< T, typename >::end(), llvm::Value::hasNUses(), llvm::SmallVectorImpl< T >::insert(), LLVM_DEBUG, llvm::PatternMatch::m_Deinterleave2(), llvm::PatternMatch::m_Specific(), llvm::PatternMatch::match(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::SmallVectorImpl< T >::resize(), and llvm::Value::user_begin().

Referenced by getDeinterleavedValues().

getDeinterleavedValues()

bool getDeinterleavedValues	(	Value *	DI,
		SmallVectorImpl< Instruction * > &	DeinterleavedValues,
		SmallVectorImpl< Instruction * > &	DeInterleaveDeadInsts
	)

Definition at line 17440 of file AArch64ISelLowering.cpp.

References getDeinterleave2Values(), and getDeinterleave4Values().

Referenced by llvm::AArch64TargetLowering::lowerDeinterleaveIntrinsicToLoad().

getDUPLANEOp()

static unsigned getDUPLANEOp ( EVT  EltType )

static

Definition at line 13067 of file AArch64ISelLowering.cpp.

References llvm::AArch64ISD::DUPLANE16, llvm::AArch64ISD::DUPLANE32, llvm::AArch64ISD::DUPLANE64, llvm::AArch64ISD::DUPLANE8, and llvm_unreachable.

Referenced by performDUPCombine().

getEstimate()

static SDValue getEstimate ( const AArch64Subtarget *  ST, unsigned  Opcode, SDValue  Operand, SelectionDAG &  DAG, int &  ExtraSteps  )

static

Definition at line 11464 of file AArch64ISelLowering.cpp.

References llvm::EVT::getFltSemantics(), llvm::SelectionDAG::getNode(), llvm::SDValue::getValueType(), llvm::Log2_64_Ceil(), llvm::APFloatBase::semanticsPrecision(), and llvm::TargetLoweringBase::Unspecified.

getExtensionTo64Bits()

static EVT getExtensionTo64Bits ( const EVT &  OrigVT )

static

Definition at line 4997 of file AArch64ISelLowering.cpp.

References assert(), llvm::EVT::getSimpleVT(), llvm::EVT::getSizeInBits(), llvm::EVT::isSimple(), llvm_unreachable, and llvm::MVT::SimpleTy.

Referenced by addRequiredExtensionForVectorMULL(), AddRequiredExtensionForVMULL(), and SkipLoadExtensionForVMULL().

getExtFactor()

static unsigned getExtFactor ( SDValue &  V )

static

getExtFactor - Determine the adjustment factor for the position when generating an "extract from vector registers" instruction.

Definition at line 12093 of file AArch64ISelLowering.cpp.

References llvm::EVT::getSizeInBits().

Referenced by GeneratePerfectShuffle(), and llvm::AArch64TargetLowering::ReconstructShuffle().

getGatherVecOpcode()

unsigned getGatherVecOpcode	(	bool	IsScaled,
		bool	IsSigned,
		bool	NeedsExtend
	)

Definition at line 6188 of file AArch64ISelLowering.cpp.

References llvm::AArch64ISD::GLD1_MERGE_ZERO, llvm::AArch64ISD::GLD1_SCALED_MERGE_ZERO, llvm::AArch64ISD::GLD1_SXTW_MERGE_ZERO, llvm::AArch64ISD::GLD1_SXTW_SCALED_MERGE_ZERO, llvm::AArch64ISD::GLD1_UXTW_MERGE_ZERO, and llvm::AArch64ISD::GLD1_UXTW_SCALED_MERGE_ZERO.

Referenced by performGLD1Combine().

getIntrinsicID()

static unsigned getIntrinsicID ( const SDNode *  N )

static

Definition at line 7293 of file AArch64ISelLowering.cpp.

References llvm::ISD::INTRINSIC_WO_CHAIN, N, and llvm::Intrinsic::not_intrinsic.

Referenced by llvm::AMDGPURegisterBankInfo::applyMappingImpl(), collectBitParts(), llvm::SITargetLowering::CollectTargetIntrinsicOperands(), FindPreallocatedCall(), llvm::AMDGPURegisterBankInfo::getInstrAlternativeMappingsIntrinsic(), llvm::AMDGPURegisterBankInfo::getInstrAlternativeMappingsIntrinsicWSideEffects(), llvm::AMDGPURegisterBankInfo::getInstrMapping(), llvm::VPRecipeBuilder::handleReplication(), llvm::SITargetLowering::isCanonicalized(), isFPIntrinsic(), llvm::AArch64TargetLowering::isReassocProfitable(), IsSVECntIntrinsic(), performIntrinsicCombine(), tryConvertSVEWideCompare(), and valueIsKnownNeverF32Denorm().

getNegatedInteger()

static SDValue getNegatedInteger ( SDValue  Op, SelectionDAG &  DAG  )

static

Definition at line 20471 of file AArch64ISelLowering.cpp.

References DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), and llvm::ISD::SUB.

Referenced by performNegCSelCombine().

getPackedSVEVectorVT() [1/2]

static EVT getPackedSVEVectorVT ( ElementCount  EC )

inlinestatic

Definition at line 197 of file AArch64ISelLowering.cpp.

References llvm_unreachable.

getPackedSVEVectorVT() [2/2]

static EVT getPackedSVEVectorVT ( EVT  VT )

inlinestatic

Definition at line 172 of file AArch64ISelLowering.cpp.

References llvm::EVT::getSimpleVT(), llvm_unreachable, and llvm::MVT::SimpleTy.

Referenced by combineSVEReductionInt(), and performDupLane128Combine().

getPredicateForFixedLengthVector()

static SDValue getPredicateForFixedLengthVector ( SelectionDAG &  DAG, SDLoc &  DL, EVT  VT  )

static

Definition at line 27634 of file AArch64ISelLowering.cpp.

References assert(), DL, llvm::AArch64Subtarget::getMinSVEVectorSizeInBits(), getPTrue(), llvm::EVT::getSimpleVT(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getSubtarget(), llvm::getSVEPredPatternFromNumElements(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::EVT::isFixedLengthVector(), llvm::TargetLoweringBase::isTypeLegal(), llvm_unreachable, and llvm::MVT::SimpleTy.

Referenced by convertFixedMaskToScalableVector(), and getPredicateForVector().

getPredicateForScalableVector()

static SDValue getPredicateForScalableVector ( SelectionDAG &  DAG, SDLoc &  DL, EVT  VT  )

static

Definition at line 27679 of file AArch64ISelLowering.cpp.

References assert(), llvm::EVT::changeVectorElementType(), DL, getPTrue(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::EVT::isScalableVector(), and llvm::TargetLoweringBase::isTypeLegal().

Referenced by getPredicateForVector().

getPredicateForVector()

static SDValue getPredicateForVector ( SelectionDAG &  DAG, SDLoc &  DL, EVT  VT  )

static

Definition at line 27687 of file AArch64ISelLowering.cpp.

References DL, getPredicateForFixedLengthVector(), getPredicateForScalableVector(), and llvm::EVT::isFixedLengthVector().

Referenced by tryCombineExtendRShTrunc().

getPredicateRegisterClass()

static const TargetRegisterClass * getPredicateRegisterClass ( PredicateConstraint  Constraint, EVT  VT  )

static

Definition at line 11630 of file AArch64ISelLowering.cpp.

References llvm::EVT::getVectorElementType(), llvm::EVT::isScalableVector(), and llvm_unreachable.

getPromotedVTForPredicate()

static EVT getPromotedVTForPredicate ( EVT  VT )

inlinestatic

Definition at line 212 of file AArch64ISelLowering.cpp.

References assert(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorMinNumElements(), llvm::EVT::isScalableVector(), and llvm_unreachable.

getPTest()

static SDValue getPTest ( SelectionDAG &  DAG, EVT  VT, SDValue  Pg, SDValue  Op, AArch64CC::CondCode  Cond  )

static

Definition at line 21543 of file AArch64ISelLowering.cpp.

References llvm::AArch64CC::ANY_ACTIVE, assert(), CC, Cond, llvm::AArch64ISD::CSEL, DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getNode(), getSVEPredicateBitCast(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::TargetLoweringBase::getTypeToTransformTo(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::TargetLoweringBase::isTypeLegal(), isZeroingInactiveLanes(), llvm::AArch64CC::NONE_ACTIVE, llvm::AArch64ISD::PTEST, llvm::AArch64ISD::PTEST_ANY, llvm::AArch64ISD::REINTERPRET_CAST, and llvm::Test.

Referenced by performFirstTrueTestVectorCombine(), performIntrinsicCombine(), and performLastTrueTestVectorCombine().

getPTrue()

static SDValue getPTrue ( SelectionDAG &  DAG, SDLoc  DL, EVT  VT, int  Pattern  )

inlinestatic

Definition at line 5405 of file AArch64ISelLowering.cpp.

References DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetConstant(), and llvm::AArch64ISD::PTRUE.

Referenced by getPredicateForFixedLengthVector(), getPredicateForScalableVector(), LowerFLDEXP(), optimizeIncrementingWhile(), performFirstTrueTestVectorCombine(), performLastTrueTestVectorCombine(), performMSTORECombine(), and performUnpackCombine().

getReducedGprRegisterClass()

static const TargetRegisterClass * getReducedGprRegisterClass ( ReducedGprConstraint  Constraint, EVT  VT  )

static

Definition at line 11661 of file AArch64ISelLowering.cpp.

References llvm::EVT::getFixedSizeInBits(), llvm::EVT::isScalarInteger(), and llvm_unreachable.

getReductionSDNode()

static SDValue getReductionSDNode ( unsigned  Op, SDLoc  DL, SDValue  ScalarOp, SelectionDAG &  DAG  )

static

Definition at line 15428 of file AArch64ISelLowering.cpp.

References DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), and llvm::SDValue::getValueType().

getScaledOffsetForBitWidth()

static SDValue getScaledOffsetForBitWidth ( SelectionDAG &  DAG, SDValue  Offset, SDLoc  DL, unsigned  BitWidth  )

static

Definition at line 24944 of file AArch64ISelLowering.cpp.

References assert(), llvm::BitWidth, DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::Log2_32(), llvm::Offset, llvm::ISD::SHL, and llvm::ISD::SPLAT_VECTOR.

Referenced by performGatherLoadCombine(), and performScatterStoreCombine().

getSETCC()

static SDValue getSETCC ( AArch64CC::CondCode  CC, SDValue  NZCV, const SDLoc &  DL, SelectionDAG &  DAG  )

static

Helper function to create 'CSET', which is equivalent to 'CSINC <Wd>, WZR, WZR, invert(<cond>)'.

Definition at line 11701 of file AArch64ISelLowering.cpp.

References CC, llvm::AArch64ISD::CSINC, DL, llvm::SelectionDAG::getConstant(), and llvm::SelectionDAG::getNode().

Referenced by combineAnd(), combineCMov(), combineM68kSetCC(), combineOr(), combinePredicateReduction(), combineSetCC(), combineSubSetcc(), combineVectorSizedSetCCEquality(), combineX86SetCC(), foldXor1SetCC(), LowerADDSUBO_CARRY(), llvm::X86TargetLowering::LowerAsmOutputForConstraint(), LowerCMP_SWAP(), LowerINTRINSIC_W_CHAIN(), LowerPARITY(), LowerXALUO(), and llvm::X86TargetLowering::ReplaceNodeResults().

getSignExtendedGatherOpcode()

unsigned getSignExtendedGatherOpcode

(

unsigned

Opcode

)

Definition at line 6211 of file AArch64ISelLowering.cpp.

References llvm::AArch64ISD::GLD1_IMM_MERGE_ZERO, llvm::AArch64ISD::GLD1_MERGE_ZERO, llvm::AArch64ISD::GLD1_SCALED_MERGE_ZERO, llvm::AArch64ISD::GLD1_SXTW_MERGE_ZERO, llvm::AArch64ISD::GLD1_SXTW_SCALED_MERGE_ZERO, llvm::AArch64ISD::GLD1_UXTW_MERGE_ZERO, llvm::AArch64ISD::GLD1_UXTW_SCALED_MERGE_ZERO, llvm::AArch64ISD::GLD1S_IMM_MERGE_ZERO, llvm::AArch64ISD::GLD1S_MERGE_ZERO, llvm::AArch64ISD::GLD1S_SCALED_MERGE_ZERO, llvm::AArch64ISD::GLD1S_SXTW_MERGE_ZERO, llvm::AArch64ISD::GLD1S_SXTW_SCALED_MERGE_ZERO, llvm::AArch64ISD::GLD1S_UXTW_MERGE_ZERO, llvm::AArch64ISD::GLD1S_UXTW_SCALED_MERGE_ZERO, and llvm_unreachable.

Referenced by performGLD1Combine().

getSMCondition()

static unsigned getSMCondition ( const SMEAttrs &  CallerAttrs, const SMEAttrs &  CalleeAttrs  )

static

Definition at line 8378 of file AArch64ISelLowering.cpp.

References llvm::AArch64SME::Always, llvm::SMEAttrs::hasNonStreamingInterface(), llvm::SMEAttrs::hasStreamingBody(), llvm::SMEAttrs::hasStreamingCompatibleInterface(), llvm::SMEAttrs::hasStreamingInterface(), llvm::AArch64SME::IfCallerIsNonStreaming, llvm::AArch64SME::IfCallerIsStreaming, and llvm_unreachable.

getStructuredLoadFunction()

static Function * getStructuredLoadFunction ( Module *  M, unsigned  Factor, bool  Scalable, Type *  LDVTy, Type *  PtrTy  )

static

Definition at line 16959 of file AArch64ISelLowering.cpp.

References assert(), and llvm::Intrinsic::getDeclaration().

Referenced by llvm::AArch64TargetLowering::lowerDeinterleaveIntrinsicToLoad(), and llvm::AArch64TargetLowering::lowerInterleavedLoad().

getStructuredStoreFunction()

static Function * getStructuredStoreFunction ( Module *  M, unsigned  Factor, bool  Scalable, Type *  STVTy, Type *  PtrTy  )

static

Definition at line 16975 of file AArch64ISelLowering.cpp.

References assert(), and llvm::Intrinsic::getDeclaration().

Referenced by llvm::AArch64TargetLowering::lowerInterleavedStore(), and llvm::AArch64TargetLowering::lowerInterleaveIntrinsicToStore().

getSVEContainerIRType()

static ScalableVectorType * getSVEContainerIRType ( FixedVectorType *  VTy )

static

Definition at line 16931 of file AArch64ISelLowering.cpp.

References llvm::ScalableVectorType::get(), llvm::Type::getBFloatTy(), llvm::Type::getContext(), llvm::Type::getDoubleTy(), llvm::VectorType::getElementType(), llvm::Type::getFloatTy(), llvm::Type::getHalfTy(), llvm::Type::getInt16Ty(), llvm::Type::getInt32Ty(), llvm::Type::getInt64Ty(), llvm::Type::getInt8Ty(), and llvm_unreachable.

Referenced by llvm::AArch64TargetLowering::lowerInterleavedLoad(), and llvm::AArch64TargetLowering::lowerInterleavedStore().

getSVEContainerType()

static MVT getSVEContainerType ( EVT  ContentTy )

static

Definition at line 22143 of file AArch64ISelLowering.cpp.

References assert(), llvm::EVT::getSimpleVT(), llvm::EVT::isSimple(), llvm_unreachable, and llvm::MVT::SimpleTy.

Referenced by performGatherLoadCombine(), performLD1Combine(), performScatterStoreCombine(), and performST1Combine().

getSVEPredicateBitCast()

static SDValue getSVEPredicateBitCast ( EVT  VT, SDValue  Op, SelectionDAG &  DAG  )

static

Definition at line 5451 of file AArch64ISelLowering.cpp.

References llvm::ISD::AND, assert(), llvm::EVT::bitsGT(), DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::EVT::getVectorElementType(), llvm::EVT::isScalableVector(), llvm::TargetLoweringBase::isTypeLegal(), isZeroingInactiveLanes(), and llvm::AArch64ISD::REINTERPRET_CAST.

Referenced by getPTest().

getTestBitOperand()

static SDValue getTestBitOperand ( SDValue  Op, unsigned &  Bit, bool &  Invert, SelectionDAG &  DAG  )

static

Definition at line 24547 of file AArch64ISelLowering.cpp.

References llvm::ISD::AND, llvm::ISD::ANY_EXTEND, llvm::CallingConv::C, llvm::DWARFExpression::Operation::getNumOperands(), getTestBitOperand(), llvm::ISD::SHL, llvm::ISD::SRA, llvm::ISD::SRL, llvm::ISD::TRUNCATE, and llvm::ISD::XOR.

Referenced by getTestBitOperand(), and performTBZCombine().

getValuesToInterleave()

bool getValuesToInterleave	(	Value *	II,
		SmallVectorImpl< Value * > &	InterleavedValues,
		SmallVectorImpl< Instruction * > &	InterleaveDeadInsts
	)

Definition at line 17547 of file AArch64ISelLowering.cpp.

References A, B, llvm::CallingConv::C, D, II, llvm::PatternMatch::m_Interleave2(), llvm::PatternMatch::m_Value(), llvm::PatternMatch::match(), and llvm::SmallVectorTemplateBase< T, bool >::push_back().

Referenced by llvm::AArch64TargetLowering::lowerInterleaveIntrinsicToStore().

getVectorBitwiseReduce()

static SDValue getVectorBitwiseReduce ( unsigned  Opcode, SDValue  Vec, EVT  VT, SDLoc  DL, SelectionDAG &  DAG  )

static

Definition at line 15436 of file AArch64ISelLowering.cpp.

References llvm::ISD::AND, llvm::ISD::ANY_EXTEND, assert(), llvm::EVT::changeVectorElementType(), DL, llvm::SelectionDAG::getAnyExtOrTrunc(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::EVT::getIntegerVT(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), getVectorBitwiseReduce(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::Hi, llvm::EVT::isFixedLengthVector(), llvm::EVT::isPow2VectorType(), llvm_unreachable, llvm::Lo, llvm::ISD::OR, llvm::ISD::SIGN_EXTEND, llvm::SelectionDAG::SplitVector(), llvm::ISD::SRL, llvm::ISD::VECREDUCE_ADD, llvm::ISD::VECREDUCE_AND, llvm::ISD::VECREDUCE_OR, llvm::ISD::VECREDUCE_UMAX, llvm::ISD::VECREDUCE_UMIN, llvm::ISD::VECREDUCE_XOR, and llvm::ISD::XOR.

Referenced by getVectorBitwiseReduce(), and performVecReduceBitwiseCombine().

getVShiftImm()

static bool getVShiftImm ( SDValue  Op, unsigned  ElementBits, int64_t &  Cnt  )

static

getVShiftImm - Check if this is a valid build_vector for the immediate operand of a vector shift operation, where all the elements of the build_vector must have the same constant integer value.

Definition at line 15049 of file AArch64ISelLowering.cpp.

References llvm::ISD::BITCAST, llvm::APInt::getSExtValue(), and llvm::BuildVectorSDNode::isConstantSplat().

Referenced by isVShiftLImm(), and isVShiftRImm().

hasNearbyPairedStore()

template<typename Iter >

bool hasNearbyPairedStore	(	Iter	It,
		Iter	End,
		Value *	Ptr,
		const DataLayout &	DL
	)

Definition at line 17124 of file AArch64ISelLowering.cpp.

References DL, End, Ptr, and llvm::APInt::sextOrTrunc().

Referenced by llvm::AArch64TargetLowering::lowerInterleavedStore().

hasPairwiseAdd()

static bool hasPairwiseAdd ( unsigned  Opcode, EVT  VT, bool  FullFP16  )

static

Definition at line 19588 of file AArch64ISelLowering.cpp.

References llvm::ISD::ADD, llvm::ISD::FADD, and llvm::ISD::STRICT_FADD.

Referenced by performExtractVectorEltCombine().

isAddSubSExt()

static bool isAddSubSExt ( SDValue  N, SelectionDAG &  DAG  )

static

Definition at line 5115 of file AArch64ISelLowering.cpp.

References llvm::ISD::ADD, llvm::SDNode::hasOneUse(), isSignExtended(), N, and llvm::ISD::SUB.

Referenced by LowerMUL(), and selectUmullSmull().

isAddSubZExt()

static bool isAddSubZExt ( SDValue  N, SelectionDAG &  DAG  )

static

Definition at line 5126 of file AArch64ISelLowering.cpp.

References llvm::ISD::ADD, llvm::SDNode::hasOneUse(), isZeroExtended(), N, and llvm::ISD::SUB.

Referenced by LowerMUL(), and selectUmullSmull().

isAllActivePredicate()

static bool isAllActivePredicate ( SelectionDAG &  DAG, SDValue  N  )

static

Definition at line 13790 of file AArch64ISelLowering.cpp.

References llvm::AArch64Subtarget::getMinSVEVectorSizeInBits(), llvm::getNumElementsFromSVEPredPattern(), llvm::SelectionDAG::getSubtarget(), llvm::ISD::isConstantSplatVectorAllOnes(), N, llvm::AArch64ISD::PTRUE, llvm::AArch64ISD::REINTERPRET_CAST, and llvm::AArch64::SVEBitsPerBlock.

Referenced by convertMergedOpToPredOp(), instCombineSVELD1(), instCombineSVESel(), instCombineSVESrshl(), instCombineSVEST1(), performSetccMergeZeroCombine(), performSVEAndCombine(), performVSelectCombine(), and tryLowerToSLI().

isAllConstantBuildVector()

static bool isAllConstantBuildVector ( const SDValue &  PotentialBVec, uint64_t &  ConstVal  )

static

Definition at line 13765 of file AArch64ISelLowering.cpp.

References llvm::SDNode::getOperand(), llvm::SDNode::getValueType(), llvm::EVT::getVectorNumElements(), and llvm::ConstantSDNode::getZExtValue().

Referenced by isPow2Splat().

isAllInactivePredicate()

static bool isAllInactivePredicate ( SDValue  N )

static

Definition at line 13782 of file AArch64ISelLowering.cpp.

References llvm::ISD::isConstantSplatVectorAllZeros(), N, and llvm::AArch64ISD::REINTERPRET_CAST.

Referenced by performVSelectCombine().

isCheapToExtend()

static bool isCheapToExtend ( const SDValue &  N )

static

Definition at line 22027 of file AArch64ISelLowering.cpp.

References llvm::ISD::isConstantSplatVectorAllZeros(), llvm::ISD::LOAD, llvm::ISD::MLOAD, and N.

Referenced by performSignExtendSetCCCombine().

isCMN()

static bool isCMN ( SDValue  Op, ISD::CondCode  CC, SelectionDAG &  DAG  )

static

Definition at line 3449 of file AArch64ISelLowering.cpp.

References cannotBeIntMin(), CC, llvm::SelectionDAG::isKnownNeverZero(), llvm::isNullConstant(), and llvm::ISD::SUB.

Referenced by emitComparison(), emitConditionalComparison(), and getAArch64Cmp().

isCMP()

static bool isCMP ( SDValue  Op )

static

Definition at line 20563 of file AArch64ISelLowering.cpp.

References llvm::AArch64ISD::SUBS.

Referenced by foldCSELOfCSEL(), and foldOverflowCheck().

isConcatMask()

static bool isConcatMask ( ArrayRef< int >  Mask, EVT  VT, bool  SplitLHS  )

static

Definition at line 12779 of file AArch64ISelLowering.cpp.

References llvm::EVT::getSizeInBits(), llvm::EVT::getVectorNumElements(), I, and llvm::Offset.

Referenced by llvm::AArch64TargetLowering::isShuffleMaskLegal(), and tryFormConcatFromShuffle().

isConstantSplatVectorMaskForType()

static bool isConstantSplatVectorMaskForType ( SDNode *  N, EVT  MemVT  )

static

Definition at line 19299 of file AArch64ISelLowering.cpp.

References llvm::AArch64ISD::DUP, llvm::EVT::getSimpleVT(), llvm::EVT::getVectorElementType(), llvm::EVT::isSimple(), N, llvm::MVT::SimpleTy, and llvm::ISD::SPLAT_VECTOR.

isEquivalentMaskless()

static bool isEquivalentMaskless ( unsigned  CC, unsigned  width, ISD::LoadExtType  ExtType, int  AddConstant, int  CompConstant  )

static

Definition at line 23914 of file AArch64ISelLowering.cpp.

References llvm::AArch64CC::AL, CC, llvm::AArch64CC::EQ, llvm::AArch64CC::GE, llvm::AArch64CC::GT, llvm::AArch64CC::HI, llvm::AArch64CC::HS, llvm::AArch64CC::Invalid, llvm::AArch64CC::LE, llvm::AArch64CC::LO, llvm::AArch64CC::LS, llvm::AArch64CC::LT, llvm::AArch64CC::MI, llvm::AArch64CC::NE, llvm::AArch64CC::NV, llvm::AArch64CC::PL, llvm::ISD::SEXTLOAD, llvm::AArch64CC::VC, and llvm::AArch64CC::VS.

Referenced by performCONDCombine().

isEssentiallyExtractHighSubvector()

static bool isEssentiallyExtractHighSubvector ( SDValue  N )

static

Definition at line 20186 of file AArch64ISelLowering.cpp.

References llvm::ISD::BITCAST, llvm::ISD::EXTRACT_SUBVECTOR, and N.

Referenced by performAddSubLongCombine(), tryCombineLongOpWithDup(), and tryCombineMULLWithUZP1().

isExtendedBUILD_VECTOR()

static bool isExtendedBUILD_VECTOR ( SDValue  N, SelectionDAG &  DAG, bool  isSigned  )

static

Definition at line 5047 of file AArch64ISelLowering.cpp.

References llvm::ISD::BUILD_VECTOR, llvm::CallingConv::C, llvm::EVT::getScalarSizeInBits(), llvm::isIntN(), isSigned(), llvm::isUIntN(), and N.

Referenced by isSignExtended(), isZeroExtended(), and selectUmullSmull().

isExtendOrShiftOperand()

static bool isExtendOrShiftOperand ( SDValue  N )

static

Definition at line 20781 of file AArch64ISelLowering.cpp.

References llvm::ISD::AND, llvm::ConstantSDNode::getZExtValue(), llvm::ISD::isExtOpcode(), N, llvm::ISD::SHL, llvm::ISD::SIGN_EXTEND_INREG, llvm::ISD::SRA, and llvm::ISD::SRL.

Referenced by performAddCombineSubShift().

isEXTMask()

static bool isEXTMask ( ArrayRef< int >  M, EVT  VT, bool &  ReverseEXT, unsigned &  Imm  )

static

Definition at line 12647 of file AArch64ISelLowering.cpp.

References llvm::find_if(), llvm::EVT::getVectorNumElements(), llvm::APInt::getZExtValue(), and llvm::APInt::logBase2().

Referenced by llvm::AArch64TargetLowering::isShuffleMaskLegal().

isHalvingTruncateAndConcatOfLegalIntScalableType()

static bool isHalvingTruncateAndConcatOfLegalIntScalableType ( SDNode *  N )

static

Definition at line 22552 of file AArch64ISelLowering.cpp.

References llvm::SDNode::getValueType(), N, and llvm::AArch64ISD::UZP1.

Referenced by performUzpCombine().

isHalvingTruncateOfLegalScalableType()

bool isHalvingTruncateOfLegalScalableType	(	EVT	SrcVT,
		EVT	DstVT
	)

Definition at line 23345 of file AArch64ISelLowering.cpp.

Referenced by performMSTORECombine(), and performSTORECombine().

isINSMask()

static bool isINSMask ( ArrayRef< int >  M, int  NumInputElements, bool &  DstIsLeft, int &  Anomaly  )

static

Definition at line 12740 of file AArch64ISelLowering.cpp.

Referenced by llvm::AArch64TargetLowering::isShuffleMaskLegal().

isIntImmediate()

static bool isIntImmediate ( const SDNode *  N, uint64_t &  Imm  )

static

Definition at line 2172 of file AArch64ISelLowering.cpp.

References llvm::CallingConv::C, and N.

isLanes1toNKnownZero()

static bool isLanes1toNKnownZero ( SDValue  Op )

static

Definition at line 25385 of file AArch64ISelLowering.cpp.

References llvm::AArch64ISD::ANDV_PRED, llvm::AArch64ISD::EORV_PRED, llvm::AArch64ISD::FADDA_PRED, llvm::AArch64ISD::FADDV_PRED, llvm::AArch64ISD::FMAXNMV_PRED, llvm::AArch64ISD::FMAXV_PRED, llvm::AArch64ISD::FMINNMV_PRED, llvm::AArch64ISD::FMINV_PRED, llvm::AArch64ISD::ORV_PRED, llvm::AArch64ISD::SADDV_PRED, llvm::AArch64ISD::SMAXV_PRED, llvm::AArch64ISD::SMINV_PRED, llvm::AArch64ISD::UADDV_PRED, llvm::AArch64ISD::UMAXV_PRED, and llvm::AArch64ISD::UMINV_PRED.

Referenced by removeRedundantInsertVectorElt().

isLegalArithImmed()

static bool isLegalArithImmed ( uint64_t  C )

static

Definition at line 3426 of file AArch64ISelLowering.cpp.

References llvm::CallingConv::C, llvm::dbgs(), and LLVM_DEBUG.

Referenced by getAArch64Cmp().

isLoadOrMultipleLoads()

static bool isLoadOrMultipleLoads ( SDValue  B, SmallVector< LoadSDNode * > &  Loads  )

static

Definition at line 20994 of file AArch64ISelLowering.cpp.

References B, llvm::ISD::BUILD_VECTOR, llvm::ISD::CONCAT_VECTORS, llvm::SDValue::getNumOperands(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::hasOneUse(), llvm::SDNode::hasOneUse(), I, llvm::peekThroughOneUseBitcasts(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), and llvm::ISD::VECTOR_SHUFFLE.

Referenced by areLoadedOffsetButOtherwiseSame(), and performExtBinopLoadFold().

isMergePassthruOpcode()

static bool isMergePassthruOpcode ( unsigned  Opc )

static

Definition at line 243 of file AArch64ISelLowering.cpp.

References llvm::AArch64ISD::ABS_MERGE_PASSTHRU, llvm::AArch64ISD::BITREVERSE_MERGE_PASSTHRU, llvm::AArch64ISD::BSWAP_MERGE_PASSTHRU, llvm::AArch64ISD::CTLZ_MERGE_PASSTHRU, llvm::AArch64ISD::CTPOP_MERGE_PASSTHRU, llvm::AArch64ISD::DUP_MERGE_PASSTHRU, llvm::AArch64ISD::FABS_MERGE_PASSTHRU, llvm::AArch64ISD::FCEIL_MERGE_PASSTHRU, llvm::AArch64ISD::FCVTZS_MERGE_PASSTHRU, llvm::AArch64ISD::FCVTZU_MERGE_PASSTHRU, llvm::AArch64ISD::FFLOOR_MERGE_PASSTHRU, llvm::AArch64ISD::FNEARBYINT_MERGE_PASSTHRU, llvm::AArch64ISD::FNEG_MERGE_PASSTHRU, llvm::AArch64ISD::FP_EXTEND_MERGE_PASSTHRU, llvm::AArch64ISD::FP_ROUND_MERGE_PASSTHRU, llvm::AArch64ISD::FRECPX_MERGE_PASSTHRU, llvm::AArch64ISD::FRINT_MERGE_PASSTHRU, llvm::AArch64ISD::FROUND_MERGE_PASSTHRU, llvm::AArch64ISD::FROUNDEVEN_MERGE_PASSTHRU, llvm::AArch64ISD::FSQRT_MERGE_PASSTHRU, llvm::AArch64ISD::FTRUNC_MERGE_PASSTHRU, llvm::AArch64ISD::NEG_MERGE_PASSTHRU, llvm::AArch64ISD::REVD_MERGE_PASSTHRU, llvm::AArch64ISD::REVH_MERGE_PASSTHRU, llvm::AArch64ISD::REVW_MERGE_PASSTHRU, llvm::AArch64ISD::SIGN_EXTEND_INREG_MERGE_PASSTHRU, llvm::AArch64ISD::SINT_TO_FP_MERGE_PASSTHRU, llvm::AArch64ISD::UINT_TO_FP_MERGE_PASSTHRU, and llvm::AArch64ISD::ZERO_EXTEND_INREG_MERGE_PASSTHRU.

isNegatedInteger()

static bool isNegatedInteger ( SDValue  Op )

static

Definition at line 20467 of file AArch64ISelLowering.cpp.

References llvm::isNullConstant(), and llvm::ISD::SUB.

Referenced by performNegCSelCombine().

isOpcWithIntImmediate()

static bool isOpcWithIntImmediate ( const SDNode *  N, unsigned  Opc, uint64_t &  Imm  )

static

Definition at line 2183 of file AArch64ISelLowering.cpp.

References isIntImmediate(), and N.

isOperandOfVmullHighP64()

static bool isOperandOfVmullHighP64 ( Value *  Op )

static

Check if Op could be used with vmull_high_p64 intrinsic.

Definition at line 16222 of file AArch64ISelLowering.cpp.

References llvm::Value::getType(), llvm::ConstantInt::getValue(), llvm::PatternMatch::m_ConstantInt(), llvm::PatternMatch::m_ExtractElt(), llvm::PatternMatch::m_Value(), and llvm::PatternMatch::match().

Referenced by areOperandsOfVmullHighP64().

isOrXorChain()

static bool isOrXorChain ( SDValue  N, unsigned &  Num, SmallVector< std::pair< SDValue, SDValue >, 16 > &  WorkList  )

static

Definition at line 10396 of file AArch64ISelLowering.cpp.

References isOrXorChain(), MaxXors, N, llvm::ISD::OR, llvm::ISD::XOR, and llvm::ISD::ZERO_EXTEND.

Referenced by isOrXorChain(), and performOrXorChainCombine().

isPackedVectorType()

static bool isPackedVectorType ( EVT  VT, SelectionDAG &  DAG  )

inlinestatic

Returns true if VT's elements occupy the lowest bit positions of its associated register class without any intervening space.

For example, nxv2f16, nxv4f16 and nxv8f16 are legal types that belong to the same register class, but only nxv8f16 can be treated as a packed vector.

Definition at line 234 of file AArch64ISelLowering.cpp.

References assert(), llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getKnownMinValue(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::EVT::isFixedLengthVector(), llvm::TargetLoweringBase::isTypeLegal(), llvm::EVT::isVector(), and llvm::AArch64::SVEBitsPerBlock.

isPassedInFPR()

static bool isPassedInFPR ( EVT  VT )

static

Definition at line 7399 of file AArch64ISelLowering.cpp.

References llvm::EVT::isFixedLengthVector(), llvm::EVT::isFloatingPoint(), and llvm::EVT::isScalableVector().

isPow2Splat()

static bool isPow2Splat ( SDValue  Op, uint64_t &  SplatVal, bool &  Negated  )

static

Definition at line 14920 of file AArch64ISelLowering.cpp.

References llvm::ISD::BUILD_VECTOR, llvm::AArch64ISD::DUP, isAllConstantBuildVector(), llvm::isPowerOf2_64(), and llvm::ISD::SPLAT_VECTOR.

isPredicateCCSettingOp()

static bool isPredicateCCSettingOp ( SDValue  N )

static

Definition at line 19603 of file AArch64ISelLowering.cpp.

References llvm::ISD::INTRINSIC_WO_CHAIN, N, and llvm::ISD::SETCC.

Referenced by performFirstTrueTestVectorCombine().

isSetCC()

static bool isSetCC ( SDValue  Op, SetCCInfoAndKind &  SetCCInfo  )

static

Check whether or not Op is a SET_CC operation, either a generic or an AArch64 lowered one.

SetCCInfo is filled accordingly.

Postcondition

SetCCInfo is meanginfull only when this function returns true.

Returns

True when Op is a kind of SET_CC operation.

Definition at line 20230 of file AArch64ISelLowering.cpp.

References SetCCInfo::AArch64, GenericSetCCInfo::CC, AArch64SetCCInfo::CC, AArch64SetCCInfo::Cmp, llvm::AArch64ISD::CSEL, SetCCInfo::Generic, llvm::AArch64CC::getInvertedCondCode(), llvm::ConstantSDNode::isOne(), llvm::ConstantSDNode::isZero(), GenericSetCCInfo::Opnd0, GenericSetCCInfo::Opnd1, llvm::ISD::SETCC, and std::swap().

Referenced by isSetCCOrZExtSetCC().

isSetCCOrZExtSetCC()

static bool isSetCCOrZExtSetCC ( const SDValue &  Op, SetCCInfoAndKind &  Info  )

static

Definition at line 20273 of file AArch64ISelLowering.cpp.

References Info, isSetCC(), and llvm::ISD::ZERO_EXTEND.

Referenced by performSetccAddFolding().

isSignExtended()

static bool isSignExtended ( SDValue  N, SelectionDAG &  DAG  )

static

Definition at line 5103 of file AArch64ISelLowering.cpp.

References llvm::ISD::ANY_EXTEND, isExtendedBUILD_VECTOR(), N, and llvm::ISD::SIGN_EXTEND.

Referenced by llvm::PPCTargetLowering::EmitPartwordAtomicBinary(), isAddSubSExt(), LowerMUL(), performMulCombine(), and selectUmullSmull().

isSingletonEXTMask()

static bool isSingletonEXTMask ( ArrayRef< int >  M, EVT  VT, unsigned &  Imm  )

static

Definition at line 12491 of file AArch64ISelLowering.cpp.

References llvm::EVT::getVectorNumElements().

isSplatShuffle()

static bool isSplatShuffle ( Value *  V )

static

Definition at line 16142 of file AArch64ISelLowering.cpp.

References llvm::all_equal().

Referenced by areExtractShuffleVectors(), and llvm::AArch64TargetLowering::shouldSinkOperands().

IsSVECntIntrinsic()

static std::optional< unsigned > IsSVECntIntrinsic ( SDValue  S )

static

Definition at line 18473 of file AArch64ISelLowering.cpp.

References getIntrinsicID(), and llvm::SDValue::getNode().

Referenced by performMulCombine().

isTRN_v_undef_Mask()

static bool isTRN_v_undef_Mask ( ArrayRef< int >  M, EVT  VT, unsigned &  WhichResult  )

static

isTRN_v_undef_Mask - Special case of isTRNMask for canonical form of "vector_shuffle v, v", i.e., "vector_shuffle v, undef".

Mask is e.g., <0, 0, 2, 2> instead of <0, 4, 2, 6>.

Definition at line 12727 of file AArch64ISelLowering.cpp.

References llvm::EVT::getVectorNumElements().

Referenced by llvm::AArch64TargetLowering::isShuffleMaskLegal().

isUZP_v_undef_Mask()

static bool isUZP_v_undef_Mask ( ArrayRef< int >  M, EVT  VT, unsigned &  WhichResult  )

static

isUZP_v_undef_Mask - Special case of isUZPMask for canonical form of "vector_shuffle v, v", i.e., "vector_shuffle v, undef".

Mask is e.g., <0, 2, 0, 2> instead of <0, 2, 4, 6>,

Definition at line 12708 of file AArch64ISelLowering.cpp.

References llvm::EVT::getVectorNumElements(), and Idx.

Referenced by llvm::AArch64TargetLowering::isShuffleMaskLegal().

isValidImmForSVEVecImmAddrMode() [1/2]

static bool isValidImmForSVEVecImmAddrMode ( SDValue  Offset, unsigned  ScalarSizeInBytes  )

static

Check if the value of Offset represents a valid immediate for the SVE gather load/prefetch and scatter store instructiona with vector base and immediate offset addressing mode:

 [<Zn>.[S|D]{, #<imm>}]

where <imm> = sizeof(<T>) * k, for k = 0, 1, ..., 31.

Definition at line 24982 of file AArch64ISelLowering.cpp.

References llvm::ConstantSDNode::getZExtValue(), isValidImmForSVEVecImmAddrMode(), and llvm::Offset.

isValidImmForSVEVecImmAddrMode() [2/2]

static bool isValidImmForSVEVecImmAddrMode ( unsigned  OffsetInBytes, unsigned  ScalarSizeInBytes  )

inlinestatic

Check if the value of OffsetInBytes can be used as an immediate for the gather load/prefetch and scatter store instructions with vector base and immediate offset addressing mode:

 [<Zn>.[S|D]{, #<imm>}]

where <imm> = sizeof(<T>) * k, for k = 0, 1, ..., 31.

Definition at line 24962 of file AArch64ISelLowering.cpp.

Referenced by combineSVEPrefetchVecBaseImmOff(), isValidImmForSVEVecImmAddrMode(), performGatherLoadCombine(), and performScatterStoreCombine().

isVShiftLImm()

static bool isVShiftLImm ( SDValue  Op, EVT  VT, bool  isLong, int64_t &  Cnt  )

static

isVShiftLImm - Check if this is a valid build_vector for the immediate operand of a vector shift left operation.

That value must be in the range: 0 <= Value < ElementBits for a left shift; or 0 <= Value <= ElementBits for a long left shift.

Definition at line 15069 of file AArch64ISelLowering.cpp.

References assert(), llvm::EVT::getScalarSizeInBits(), getVShiftImm(), and llvm::EVT::isVector().

Referenced by LowerShift(), llvm::ARMTargetLowering::PerformIntrinsicCombine(), and PerformShiftCombine().

isVShiftRImm()

static bool isVShiftRImm ( SDValue  Op, EVT  VT, bool  isNarrow, int64_t &  Cnt  )

static

isVShiftRImm - Check if this is a valid build_vector for the immediate operand of a vector shift right operation.

The value must be in the range: 1 <= Value <= ElementBits for a right shift; or

Definition at line 15080 of file AArch64ISelLowering.cpp.

References assert(), llvm::EVT::getScalarSizeInBits(), getVShiftImm(), and llvm::EVT::isVector().

Referenced by LowerShift(), llvm::ARMTargetLowering::PerformIntrinsicCombine(), and PerformShiftCombine().

isWideDUPMask()

static bool isWideDUPMask ( ArrayRef< int >  M, EVT  VT, unsigned  BlockSize, unsigned &  DupLaneOp  )

static

Check if a vector shuffle corresponds to a DUP instructions with a larger element width than the vector lane type.

If that is the case the function returns true and writes the value of the DUP instruction lane operand into DupLaneOp

Definition at line 12574 of file AArch64ISelLowering.cpp.

References assert(), llvm::SmallVectorTemplateCommon< T, typename >::begin(), BlockSize, llvm::SmallVectorTemplateCommon< T, typename >::end(), llvm::find_if(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getSizeInBits(), llvm::EVT::getVectorNumElements(), and I.

isWideTypeMask()

static bool isWideTypeMask ( ArrayRef< int >  M, EVT  VT, SmallVectorImpl< int > &  NewMask  )

static

Definition at line 13142 of file AArch64ISelLowering.cpp.

References assert(), llvm::SmallVectorImpl< T >::clear(), llvm::EVT::getVectorNumElements(), llvm::M0(), llvm::M1(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), and llvm::SmallVectorBase< Size_T >::size().

Referenced by tryWidenMaskForShuffle().

isZeroExtended()

static bool isZeroExtended ( SDValue  N, SelectionDAG &  DAG  )

static

Definition at line 5109 of file AArch64ISelLowering.cpp.

References llvm::ISD::ANY_EXTEND, isExtendedBUILD_VECTOR(), N, and llvm::ISD::ZERO_EXTEND.

Referenced by isAddSubZExt(), LowerMUL(), performMulCombine(), and selectUmullSmull().

isZeroingInactiveLanes()

static bool isZeroingInactiveLanes ( SDValue  Op )

static

Definition at line 281 of file AArch64ISelLowering.cpp.

References llvm::ISD::INTRINSIC_WO_CHAIN, llvm::AArch64ISD::PTRUE, llvm::AArch64ISD::SETCC_MERGE_ZERO, and llvm::ISD::SPLAT_VECTOR.

Referenced by getPTest(), and getSVEPredicateBitCast().

isZerosVector()

static bool isZerosVector ( const SDNode *  N )

static

isZerosVector - Check whether SDNode N is a zero-filled vector.

Definition at line 3257 of file AArch64ISelLowering.cpp.

References llvm::ISD::BITCAST, llvm::AArch64ISD::DUP, llvm::ISD::isConstantSplatVectorAllZeros(), llvm::isNullConstant(), llvm::isNullFPConstant(), and N.

Referenced by GenerateTBL(), performAddDotCombine(), performSetccMergeZeroCombine(), performSetCCPunpkCombine(), and performUnpackCombine().

isZIP_v_undef_Mask()

static bool isZIP_v_undef_Mask ( ArrayRef< int >  M, EVT  VT, unsigned &  WhichResult  )

static

isZIP_v_undef_Mask - Special case of isZIPMask for canonical form of "vector_shuffle v, v", i.e., "vector_shuffle v, undef".

Mask is e.g., <0, 0, 1, 1> instead of <0, 4, 1, 5>.

Definition at line 12689 of file AArch64ISelLowering.cpp.

References llvm::EVT::getVectorNumElements(), and Idx.

Referenced by llvm::AArch64TargetLowering::isShuffleMaskLegal().

legalizeSVEGatherPrefetchOffsVec()

static SDValue legalizeSVEGatherPrefetchOffsVec ( SDNode *  N, SelectionDAG &  DAG  )

static

Legalize the gather prefetch (scalar + vector addressing mode) when the offset vector is an unpacked 32-bit scalable vector.

The other cases (Offset != nxv2i32) do not need legalization.

Definition at line 25341 of file AArch64ISelLowering.cpp.

References llvm::ISD::ANY_EXTEND, DL, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getVTList(), N, and llvm::Offset.

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

lookThroughSignExtension()

std::pair< SDValue, uint64_t > lookThroughSignExtension

(

SDValue

Val

)

Definition at line 9937 of file AArch64ISelLowering.cpp.

References llvm::EVT::getFixedSizeInBits(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValueSizeInBits(), llvm::SDNode::getValueType(), llvm::ISD::SIGN_EXTEND, and llvm::ISD::SIGN_EXTEND_INREG.

lowerADDSUBO_CARRY()

static SDValue lowerADDSUBO_CARRY ( SDValue  Op, SelectionDAG &  DAG, unsigned  Opcode, bool  IsSigned  )

static

Definition at line 4208 of file AArch64ISelLowering.cpp.

References carryFlagToValue(), DL, llvm::SelectionDAG::getNode(), llvm::SDValue::getValue(), llvm::SelectionDAG::getVTList(), llvm::ISD::MERGE_VALUES, overflowFlagToValue(), llvm::AArch64ISD::SBCS, and valueToCarryFlag().

Referenced by llvm::AArch64TargetLowering::LowerOperation().

LowerBRCOND()

static SDValue LowerBRCOND ( SDValue  Op, SelectionDAG &  DAG  )

static

Definition at line 6757 of file AArch64ISelLowering.cpp.

References llvm::AArch64ISD::BRCOND, CC, Cond, emitConjunction(), llvm::SelectionDAG::getConstant(), and llvm::SelectionDAG::getNode().

Referenced by llvm::AArch64TargetLowering::LowerOperation().

LowerFLDEXP()

static SDValue LowerFLDEXP ( SDValue  Op, SelectionDAG &  DAG  )

static

Definition at line 6797 of file AArch64ISelLowering.cpp.

References llvm::EVT::changeVectorElementType(), DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::ISD::FP_EXTEND, llvm::ISD::FP_ROUND, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), getPTrue(), llvm::SelectionDAG::getUNDEF(), llvm::ISD::INSERT_VECTOR_ELT, llvm::ISD::INTRINSIC_WO_CHAIN, llvm::ISD::SIGN_EXTEND, and X.

Referenced by llvm::AArch64TargetLowering::LowerOperation().

LowerFunnelShift()

static SDValue LowerFunnelShift ( SDValue  Op, SelectionDAG &  DAG  )

static

Definition at line 6775 of file AArch64ISelLowering.cpp.

References DL, llvm::ISD::FSHL, llvm::ISD::FSHR, llvm::SelectionDAG::getConstant(), llvm::MVT::getFixedSizeInBits(), llvm::SelectionDAG::getNode(), and llvm::SDValue::getValueType().

Referenced by llvm::AArch64TargetLowering::LowerOperation(), and llvm::X86TargetLowering::LowerOperation().

LowerPREFETCH()

static SDValue LowerPREFETCH ( SDValue  Op, SelectionDAG &  DAG  )

static

Definition at line 4265 of file AArch64ISelLowering.cpp.

References assert(), DL, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetConstant(), and llvm::AArch64ISD::PREFETCH.

Referenced by llvm::AArch64TargetLowering::LowerOperation(), llvm::ARMTargetLowering::LowerOperation(), and llvm::X86TargetLowering::LowerOperation().

LowerSMELdrStr()

SDValue LowerSMELdrStr	(	SDValue	N,
		SelectionDAG &	DAG,
		bool	IsLoad
	)

Definition at line 5543 of file AArch64ISelLowering.cpp.

References llvm::ISD::ADD, llvm::sampleprof::Base, llvm::CallingConv::C, DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getTargetConstant(), llvm::Mul, llvm::ISD::MUL, N, llvm::AArch64ISD::RDSVL, llvm::ISD::SIGN_EXTEND, llvm::AArch64ISD::SME_ZA_LDR, and llvm::AArch64ISD::SME_ZA_STR.

LowerSVEIntrinsicDUP()

static SDValue LowerSVEIntrinsicDUP ( SDNode *  N, SelectionDAG &  DAG  )

static

Definition at line 21439 of file AArch64ISelLowering.cpp.

References llvm::ISD::ANY_EXTEND, llvm::AArch64ISD::DUP_MERGE_PASSTHRU, llvm::SelectionDAG::getNode(), and N.

Referenced by performIntrinsicCombine().

LowerSVEIntrinsicEXT()

static SDValue LowerSVEIntrinsicEXT ( SDNode *  N, SelectionDAG &  DAG  )

static

Definition at line 21453 of file AArch64ISelLowering.cpp.

References assert(), llvm::ISD::BITCAST, llvm::AArch64ISD::EXT, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getKnownMinValue(), llvm::SelectionDAG::getNode(), llvm::ElementCount::getScalable(), llvm::EVT::getSizeInBits(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorVT(), llvm::EVT::isScalableVector(), llvm::ISD::MUL, N, and llvm::AArch64::SVEBitsPerBlock.

Referenced by performIntrinsicCombine().

LowerSVEIntrinsicIndex()

static SDValue LowerSVEIntrinsicIndex ( SDNode *  N, SelectionDAG &  DAG  )

static

Definition at line 21423 of file AArch64ISelLowering.cpp.

References llvm::ISD::ADD, llvm::sampleprof::Base, DL, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getStepVector(), llvm::SDValue::getValueType(), llvm::Mul, llvm::ISD::MUL, N, and llvm::ISD::SPLAT_VECTOR.

Referenced by performIntrinsicCombine().

LowerTruncateVectorStore()

static SDValue LowerTruncateVectorStore ( SDLoc  DL, StoreSDNode *  ST, EVT  VT, EVT  MemVT, SelectionDAG &  DAG  )

static

Definition at line 6440 of file AArch64ISelLowering.cpp.

References assert(), llvm::ISD::BITCAST, llvm::ISD::CONCAT_VECTORS, DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getStore(), llvm::SelectionDAG::getUNDEF(), llvm::EVT::isVector(), and llvm::ISD::TRUNCATE.

LowerXALUO()

static SDValue LowerXALUO ( SDValue  Op, SelectionDAG &  DAG  )

static

Definition at line 4234 of file AArch64ISelLowering.cpp.

References CC, llvm::AArch64ISD::CSEL, getAArch64XALUOOp(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getVTList(), llvm::TargetLoweringBase::isTypeLegal(), and llvm::ISD::MERGE_VALUES.

Referenced by LowerMULO(), llvm::AArch64TargetLowering::LowerOperation(), and llvm::X86TargetLowering::LowerOperation().

mayTailCallThisCC()

static bool mayTailCallThisCC ( CallingConv::ID  CC )

static

Return true if we might ever do TCO for calls with this calling convention.

Definition at line 8020 of file AArch64ISelLowering.cpp.

References llvm::CallingConv::AArch64_SVE_VectorCall, llvm::CallingConv::C, CC, llvm::CallingConv::Fast, llvm::CallingConv::PreserveAll, llvm::CallingConv::PreserveMost, llvm::CallingConv::PreserveNone, llvm::CallingConv::Swift, llvm::CallingConv::SwiftTail, and llvm::CallingConv::Tail.

Referenced by llvm::AArch64CallLowering::isEligibleForTailCallOptimization(), llvm::AMDGPUCallLowering::isEligibleForTailCallOptimization(), and llvm::SITargetLowering::isEligibleForTailCallOptimization().

NormalizeBuildVector()

static SDValue NormalizeBuildVector ( SDValue  Op, SelectionDAG &  DAG  )

static

Definition at line 14105 of file AArch64ISelLowering.cpp.

References assert(), llvm::ISD::BUILD_VECTOR, llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::EVT::getVectorElementType(), llvm::APInt::getZExtValue(), llvm::EVT::isFloatingPoint(), and llvm::SmallVectorTemplateBase< T, bool >::push_back().

optimizeIncrementingWhile()

static SDValue optimizeIncrementingWhile ( SDValue  Op, SelectionDAG &  DAG, bool  IsSigned, bool  IsEqual  )

static

Definition at line 5413 of file AArch64ISelLowering.cpp.

References llvm::APInt::getBitWidth(), llvm::AArch64Subtarget::getMinSVEVectorSizeInBits(), getPTrue(), llvm::SelectionDAG::getSubtarget(), llvm::getSVEPredPatternFromNumElements(), llvm::APInt::getZExtValue(), llvm::APInt::sadd_ov(), llvm::APInt::uadd_ov(), X, and Y.

optimizeLogicalImm()

static bool optimizeLogicalImm ( SDValue  Op, unsigned  Size, uint64_t  Imm, const APInt &  Demanded, TargetLowering::TargetLoweringOpt &  TLO, unsigned  NewOpc  )

static

Definition at line 2189 of file AArch64ISelLowering.cpp.

References assert(), llvm::TargetLowering::TargetLoweringOpt::CombineTo(), llvm::TargetLowering::TargetLoweringOpt::DAG, DL, llvm::AArch64_AM::encodeLogicalImmediate(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getMachineNode(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetConstant(), llvm::APInt::getZExtValue(), llvm::Hi, llvm::AArch64_AM::isLogicalImmediate(), llvm::isShiftedMask_64(), and Size.

Referenced by llvm::AArch64TargetLowering::targetShrinkDemandedConstant().

overflowFlagToValue()

static SDValue overflowFlagToValue ( SDValue  Glue, EVT  VT, SelectionDAG &  DAG  )

static

Definition at line 4197 of file AArch64ISelLowering.cpp.

References assert(), CC, llvm::AArch64ISD::CSEL, DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getResNo(), and llvm::AArch64CC::VS.

Referenced by lowerADDSUBO_CARRY().

parseConstraintCode()

static AArch64CC::CondCode parseConstraintCode ( llvm::StringRef  Constraint )

static

Definition at line 11677 of file AArch64ISelLowering.cpp.

References llvm::StringSwitch< T, R >::Case(), Cond, llvm::StringSwitch< T, R >::Default(), llvm::AArch64CC::EQ, llvm::AArch64CC::GE, llvm::AArch64CC::GT, llvm::AArch64CC::HI, llvm::AArch64CC::HS, llvm::AArch64CC::Invalid, llvm::AArch64CC::LE, llvm::AArch64CC::LO, llvm::AArch64CC::LS, llvm::AArch64CC::LT, llvm::AArch64CC::MI, llvm::AArch64CC::NE, llvm::AArch64CC::PL, llvm::AArch64CC::VC, and llvm::AArch64CC::VS.

Referenced by llvm::X86TargetLowering::getConstraintType(), llvm::X86TargetLowering::getRegForInlineAsmConstraint(), and llvm::X86TargetLowering::LowerAsmOutputForConstraint().

parsePredicateConstraint()

static std::optional< PredicateConstraint > parsePredicateConstraint ( StringRef  Constraint )

static

Definition at line 11621 of file AArch64ISelLowering.cpp.

References llvm::StringSwitch< T, R >::Case(), and llvm::StringSwitch< T, R >::Default().

parseReducedGprConstraint()

static std::optional< ReducedGprConstraint > parseReducedGprConstraint ( StringRef  Constraint )

static

Definition at line 11653 of file AArch64ISelLowering.cpp.

References llvm::StringSwitch< T, R >::Case(), and llvm::StringSwitch< T, R >::Default().

performAddCombineForShiftedOperands()

static SDValue performAddCombineForShiftedOperands ( SDNode *  N, SelectionDAG &  DAG  )

static

Definition at line 20834 of file AArch64ISelLowering.cpp.

References llvm::ISD::ADD, DL, llvm::SelectionDAG::getNode(), llvm::Value::hasOneUse(), isOpcWithIntImmediate(), LHS, N, performAddCombineSubShift(), RHS, and llvm::ISD::SHL.

Referenced by performAddSubCombine().

performAddCombineSubShift()

static SDValue performAddCombineSubShift ( SDNode *  N, SDValue  SUB, SDValue  Z, SelectionDAG &  DAG  )

static

Definition at line 20806 of file AArch64ISelLowering.cpp.

References llvm::ISD::ADD, DL, llvm::SelectionDAG::getNode(), isExtendOrShiftOperand(), N, llvm::ISD::SUB, and Y.

Referenced by performAddCombineForShiftedOperands().

performAddCSelIntoCSinc()

static SDValue performAddCSelIntoCSinc ( SDNode *  N, SelectionDAG &  DAG  )

static

Perform the scalar expression combine in the form of: CSEL(c, 1, cc) + b => CSINC(b+c, b, cc) CSNEG(c, -1, cc) + b => CSINC(b+c, b, cc)

Definition at line 20372 of file AArch64ISelLowering.cpp.

References llvm::ISD::ADD, assert(), llvm::CallingConv::C, llvm::AArch64ISD::CSEL, llvm::AArch64ISD::CSINC, llvm::AArch64ISD::CSNEG, DL, llvm::SelectionDAG::getAllOnesConstant(), llvm::ConstantSDNode::getAPIntValue(), llvm::SelectionDAG::getConstant(), llvm::AArch64CC::getInvertedCondCode(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::Value::hasOneUse(), llvm::ConstantSDNode::isAllOnes(), llvm::TargetLoweringBase::isLegalAddImmediate(), llvm::ConstantSDNode::isOne(), llvm::EVT::isScalarInteger(), LHS, N, RHS, and std::swap().

Referenced by performAddSubCombine().

performAddDotCombine()

static SDValue performAddDotCombine ( SDNode *  N, SelectionDAG &  DAG  )

static

Definition at line 20445 of file AArch64ISelLowering.cpp.

References A, llvm::ISD::ADD, llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), isZerosVector(), N, llvm::AArch64ISD::SDOT, std::swap(), and llvm::AArch64ISD::UDOT.

Referenced by performAddSubCombine().

performAddSubCombine()

static SDValue performAddSubCombine ( SDNode *  N, TargetLowering::DAGCombinerInfo &  DCI  )

static

Definition at line 21238 of file AArch64ISelLowering.cpp.

References llvm::TargetLowering::DAGCombinerInfo::DAG, N, performAddCombineForShiftedOperands(), performAddCSelIntoCSinc(), performAddDotCombine(), performAddSubIntoVectorOp(), performAddSubLongCombine(), performAddUADDVCombine(), performExtBinopLoadFold(), performNegCSelCombine(), performSubAddMULCombine(), performSVEMulAddSubCombine(), and performVectorExtCombine().

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performAddSubIntoVectorOp()

static SDValue performAddSubIntoVectorOp ( SDNode *  N, SelectionDAG &  DAG  )

static

Definition at line 20960 of file AArch64ISelLowering.cpp.

References DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::TargetLoweringBase::isOperationExpand(), llvm::ISD::LOAD, N, and llvm::ISD::SCALAR_TO_VECTOR.

Referenced by performAddSubCombine().

performAddSubLongCombine()

static SDValue performAddSubLongCombine ( SDNode *  N, TargetLowering::DAGCombinerInfo &  DCI  )

static

Definition at line 20521 of file AArch64ISelLowering.cpp.

References llvm::ISD::ADD, llvm::TargetLowering::DAGCombinerInfo::DAG, llvm::SelectionDAG::getNode(), llvm::MVT::is128BitVector(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), isEssentiallyExtractHighSubvector(), LHS, N, performSetccAddFolding(), RHS, llvm::ISD::SIGN_EXTEND, tryExtendDUPToExtractHigh(), and llvm::ISD::ZERO_EXTEND.

Referenced by performAddSubCombine().

performAddUADDVCombine()

static SDValue performAddUADDVCombine ( SDNode *  N, SelectionDAG &  DAG  )

static

Definition at line 20333 of file AArch64ISelLowering.cpp.

References llvm::ISD::ADD, DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::isScalarInteger(), LHS, N, RHS, and llvm::AArch64ISD::UADDV.

Referenced by performAddSubCombine().

performANDCombine()

static SDValue performANDCombine ( SDNode *  N, TargetLowering::DAGCombinerInfo &  DCI  )

static

Definition at line 19489 of file AArch64ISelLowering.cpp.

References llvm::AArch64ISD::BICi, llvm::SelectionDAG::computeKnownBits(), llvm::TargetLowering::DAGCombinerInfo::DAG, llvm::APInt::getBitWidth(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), I, llvm::EVT::is128BitVector(), llvm::EVT::is64BitVector(), llvm::EVT::isScalableVector(), llvm::TargetLoweringBase::isTypeLegal(), LHS, N, performANDORCSELCombine(), performANDSETCCCombine(), performSVEAndCombine(), resolveBuildVector(), RHS, tryAdvSIMDModImm16(), tryAdvSIMDModImm32(), llvm::KnownBits::Zero, and llvm::APInt::zext().

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine(), llvm::LoongArchTargetLowering::PerformDAGCombine(), llvm::MipsTargetLowering::PerformDAGCombine(), llvm::MipsSETargetLowering::PerformDAGCombine(), and llvm::RISCVTargetLowering::PerformDAGCombine().

performANDORCSELCombine()

static SDValue performANDORCSELCombine ( SDNode *  N, SelectionDAG &  DAG  )

static

Definition at line 19213 of file AArch64ISelLowering.cpp.

References llvm::ISD::AND, llvm::AArch64ISD::CCMN, llvm::AArch64ISD::CCMP, llvm::AArch64ISD::CSEL, DL, llvm::SelectionDAG::getConstant(), llvm::SDValue::getConstantOperandVal(), llvm::AArch64CC::getInvertedCondCode(), llvm::SelectionDAG::getNode(), llvm::AArch64CC::getNZCVToSatisfyCondCode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getValueType(), llvm::SDNode::hasOneUse(), llvm::isNullConstant(), llvm::isOneConstant(), MVT_CC, N, llvm::AArch64ISD::SUBS, and std::swap().

Referenced by performANDCombine(), and performORCombine().

performANDSETCCCombine()

static SDValue performANDSETCCCombine ( SDNode *  N, TargetLowering::DAGCombinerInfo &  DCI  )

static

Definition at line 19450 of file AArch64ISelLowering.cpp.

References CC, llvm::AArch64ISD::CSINC, llvm::TargetLowering::DAGCombinerInfo::DAG, DL, emitConjunction(), llvm::SelectionDAG::getConstant(), llvm::AArch64CC::getInvertedCondCode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValueType(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), N, llvm::ISD::SELECT, and llvm::ISD::SETCC.

Referenced by performANDCombine().

performBRCONDCombine()

static SDValue performBRCONDCombine ( SDNode *  N, TargetLowering::DAGCombinerInfo &  DCI, SelectionDAG &  DAG  )

static

Definition at line 24117 of file AArch64ISelLowering.cpp.

References llvm::AArch64ISD::ADDS, assert(), llvm::AArch64ISD::CBNZ, llvm::AArch64ISD::CBZ, CC, llvm::TargetLowering::DAGCombinerInfo::CombineTo(), llvm::AArch64CC::EQ, llvm::SDNode::getAsZExtVal(), llvm::MachineFunction::getFunction(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getNode(), llvm::Function::hasFnAttribute(), llvm::isNullConstant(), LHS, N, llvm::AArch64CC::NE, performCONDCombine(), RHS, llvm::ISD::SHL, llvm::ISD::SRA, llvm::ISD::SRL, llvm::AArch64ISD::SUBS, and std::swap().

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performBSPExpandForSVE()

static SDValue performBSPExpandForSVE ( SDNode *  N, SelectionDAG &  DAG, const AArch64Subtarget *  Subtarget  )

static

Definition at line 25490 of file AArch64ISelLowering.cpp.

References llvm::ISD::AND, DL, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNOT(), llvm::EVT::isScalableVector(), N, and llvm::ISD::OR.

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performBuildShuffleExtendCombine()

static SDValue performBuildShuffleExtendCombine ( SDValue  BV, SelectionDAG &  DAG  )

static

Combines a buildvector(sext/zext) or shuffle(sext/zext, undef) node pattern into sext/zext(buildvector) or sext/zext(shuffle) making use of the vector SExt/ZExt rather than the scalar SExt/ZExt.

Definition at line 18538 of file AArch64ISelLowering.cpp.

References llvm::ISD::AND, llvm::ISD::AssertSext, llvm::ISD::AssertZext, llvm::ISD::BUILD_VECTOR, calculatePreExtendType(), llvm::EVT::changeVectorElementType(), DL, llvm::drop_begin(), llvm::SelectionDAG::getAnyExtOrTrunc(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getVectorShuffle(), llvm::SDValue::isUndef(), llvm::SDNode::ops(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::ISD::SIGN_EXTEND, llvm::ISD::SIGN_EXTEND_INREG, llvm::ISD::VECTOR_SHUFFLE, and llvm::ISD::ZERO_EXTEND.

Referenced by performMulVectorExtendCombine().

performBuildVectorCombine()

static SDValue performBuildVectorCombine ( SDNode *  N, TargetLowering::DAGCombinerInfo &  DCI, SelectionDAG &  DAG  )

static

Definition at line 20628 of file AArch64ISelLowering.cpp.

References llvm::ISD::ANY_EXTEND, llvm::EVT::changeVectorElementType(), Concat, llvm::ISD::CONCAT_VECTORS, DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::AArch64ISD::FCVTXN, llvm::ISD::FP_EXTEND, llvm::ISD::FP_ROUND, llvm::SDNode::getConstantOperandVal(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getSubtarget(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getVectorIdxConstant(), llvm::EVT::getVectorMinNumElements(), llvm::AArch64Subtarget::isNeonAvailable(), llvm::TargetLoweringBase::isTypeLegal(), N, and llvm::ISD::UNDEF.

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performConcatVectorsCombine()

static SDValue performConcatVectorsCombine ( SDNode *  N, TargetLowering::DAGCombinerInfo &  DCI, SelectionDAG &  DAG  )

static

Definition at line 19757 of file AArch64ISelLowering.cpp.

References llvm::Add, llvm::ISD::ADD, llvm::all_of(), assert(), llvm::ISD::BITCAST, CC, llvm::ISD::CONCAT_VECTORS, llvm::TargetLowering::DAGCombinerInfo::DAG, llvm::dbgs(), llvm::AArch64ISD::DUP, llvm::AArch64ISD::DUPLANE64, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getConstantOperandVal(), llvm::SDNode::getConstantOperandVal(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getLoad(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNOT(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::MVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::EVT::getVectorVT(), llvm::MVT::getVectorVT(), llvm::SDNode::hasOneUse(), llvm::EVT::is128BitVector(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::TargetLoweringBase::isBinOp(), llvm::ISD::isConstantSplatVectorAllOnes(), llvm::SDNode::isOnlyUserOf(), llvm::EVT::isScalableVector(), llvm::SDValue::isUndef(), llvm::MVT::isVector(), LLVM_DEBUG, llvm::AArch64ISD::MOVIshift, N, llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), RHS, llvm::ISD::TRUNCATE, llvm::AArch64ISD::UZP2, llvm::AArch64ISD::VLSHR, WidenVector(), X, llvm::ISD::XOR, Y, llvm::AArch64ISD::ZIP1, and llvm::AArch64ISD::ZIP2.

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performCONDCombine()

static SDValue performCONDCombine ( SDNode *  N, TargetLowering::DAGCombinerInfo &  DCI, SelectionDAG &  DAG, unsigned  CCIndex, unsigned  CmpIndex  )

static

Definition at line 24038 of file AArch64ISelLowering.cpp.

References llvm::ISD::ADD, llvm::ISD::AND, CC, checkValueWidth(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::SDNode::getValueType(), llvm::SelectionDAG::getVTList(), llvm::SDNode::hasAnyUseOfValue(), llvm::SDNode::hasOneUse(), isEquivalentMaskless(), N, performSubsToAndsCombine(), llvm::SelectionDAG::ReplaceAllUsesWith(), and llvm::AArch64ISD::SUBS.

Referenced by performBRCONDCombine(), and performCSELCombine().

performCSELCombine()

static SDValue performCSELCombine ( SDNode *  N, TargetLowering::DAGCombinerInfo &  DCI, SelectionDAG &  DAG  )

static

Definition at line 24281 of file AArch64ISelLowering.cpp.

References foldCSELOfCSEL(), foldCSELofCTTZ(), N, and performCONDCombine().

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performCTLZCombine()

static SDValue performCTLZCombine ( SDNode *  N, SelectionDAG &  DAG, const AArch64Subtarget *  Subtarget  )

static

Definition at line 24931 of file AArch64ISelLowering.cpp.

References llvm::ISD::BITREVERSE, llvm::ISD::CTTZ, DL, llvm::SelectionDAG::getNode(), and N.

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performDUPCombine()

static SDValue performDUPCombine ( SDNode *  N, TargetLowering::DAGCombinerInfo &  DCI  )

static

Definition at line 24825 of file AArch64ISelLowering.cpp.

References llvm::TargetLowering::DAGCombinerInfo::DAG, DL, llvm::AArch64ISD::DUP, llvm::ISD::EXTRACT_SUBVECTOR, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::EVT::getDoubleNumVectorElementsVT(), getDUPLANEOp(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNodeIfExists(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::SelectionDAG::getVTList(), llvm::EVT::is64BitVector(), llvm::TargetLowering::DAGCombinerInfo::isAfterLegalizeDAG(), N, and performPostLD1Combine().

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performDupLane128Combine()

static SDValue performDupLane128Combine ( SDNode *  N, SelectionDAG &  DAG  )

static

Definition at line 25510 of file AArch64ISelLowering.cpp.

References llvm::ISD::BITCAST, DL, llvm::AArch64ISD::DUPLANE128, llvm::SelectionDAG::getNode(), getPackedSVEVectorVT(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::ISD::INSERT_SUBVECTOR, llvm::EVT::is128BitVector(), and N.

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performExtBinopLoadFold()

static SDValue performExtBinopLoadFold ( SDNode *  N, SelectionDAG &  DAG  )

static

Definition at line 21120 of file AArch64ISelLowering.cpp.

References areLoadedOffsetButOtherwiseSame(), llvm::ISD::CONCAT_VECTORS, DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::EVT::getDoubleNumVectorElementsVT(), llvm::SelectionDAG::getLoad(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::EVT::getVectorVT(), llvm::SDValue::hasOneUse(), llvm::Hi, llvm::ISD::isConstantSplatVector(), llvm::ISD::isExtOpcode(), llvm::EVT::isFixedLengthVector(), isLoadOrMultipleLoads(), llvm::TargetLoweringBase::isTypeLegal(), llvm::Lo, llvm::SelectionDAG::makeEquivalentMemoryOrdering(), N, llvm::SDNode::op_values(), llvm::Other, llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::ISD::SHL, llvm::SmallVectorBase< Size_T >::size(), llvm::ISD::SUB, std::swap(), and llvm::zip().

Referenced by performAddSubCombine().

performExtendCombine()

static SDValue performExtendCombine ( SDNode *  N, TargetLowering::DAGCombinerInfo &  DCI, SelectionDAG &  DAG  )

static

Definition at line 22073 of file AArch64ISelLowering.cpp.

References llvm::ISD::ABDS, llvm::ISD::ABDU, llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), N, llvm::Intrinsic::not_intrinsic, performSignExtendSetCCCombine(), llvm::ISD::SETCC, llvm::ISD::SIGN_EXTEND, tryCombineLongOpWithDup(), and llvm::ISD::ZERO_EXTEND.

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performExtractSubvectorCombine()

static SDValue performExtractSubvectorCombine ( SDNode *  N, TargetLowering::DAGCombinerInfo &  DCI, SelectionDAG &  DAG  )

static

Definition at line 20018 of file AArch64ISelLowering.cpp.

References llvm::SelectionDAG::getNode(), llvm::EVT::getVectorElementType(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::EVT::isScalableVector(), N, and llvm::ISD::SPLAT_VECTOR.

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performExtractVectorEltCombine()

static SDValue performExtractVectorEltCombine ( SDNode *  N, TargetLowering::DAGCombinerInfo &  DCI, const AArch64Subtarget *  Subtarget  )

static

Definition at line 19688 of file AArch64ISelLowering.cpp.

References assert(), llvm::TargetLowering::DAGCombinerInfo::DAG, DL, llvm::AArch64ISD::DUP, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getConstant(), llvm::ShuffleVectorSDNode::getMaskElt(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getSubtarget(), llvm::SDValue::getValue(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::SDValue::hasOneUse(), hasPairwiseAdd(), llvm::EVT::isInteger(), llvm::isNullConstant(), llvm::SDNode::isStrictFPOpcode(), N, llvm::Other, performFirstTrueTestVectorCombine(), performLastTrueTestVectorCombine(), and llvm::SelectionDAG::ReplaceAllUsesOfValueWith().

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performFADDCombine()

static SDValue performFADDCombine ( SDNode *  N, TargetLowering::DAGCombinerInfo &  DCI  )

static

Definition at line 19552 of file AArch64ISelLowering.cpp.

References A, B, llvm::TargetLowering::DAGCombinerInfo::DAG, DL, llvm::ISD::FADD, llvm::SelectionDAG::getNode(), llvm::ISD::INTRINSIC_WO_CHAIN, LHS, N, RHS, and llvm::SDNode::setFlags().

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performFirstTrueTestVectorCombine()

static SDValue performFirstTrueTestVectorCombine ( SDNode *  N, TargetLowering::DAGCombinerInfo &  DCI, const AArch64Subtarget *  Subtarget  )

static

Definition at line 19624 of file AArch64ISelLowering.cpp.

References assert(), llvm::TargetLowering::DAGCombinerInfo::DAG, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::AArch64CC::FIRST_ACTIVE, getPTest(), getPTrue(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::isNullConstant(), isPredicateCCSettingOp(), llvm::EVT::isScalableVector(), and N.

Referenced by performExtractVectorEltCombine().

performFlagSettingCombine()

static SDValue performFlagSettingCombine ( SDNode *  N, TargetLowering::DAGCombinerInfo &  DCI, unsigned  GenericOpcode  )

static

Definition at line 24439 of file AArch64ISelLowering.cpp.

References llvm::TargetLowering::DAGCombinerInfo::CombineTo(), llvm::TargetLowering::DAGCombinerInfo::DAG, DL, Generic, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNodeIfExists(), llvm::SelectionDAG::getVTList(), LHS, N, and RHS.

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performFPExtendCombine()

static SDValue performFPExtendCombine ( SDNode *  N, SelectionDAG &  DAG, TargetLowering::DAGCombinerInfo &  DCI, const AArch64Subtarget *  Subtarget  )

static

Definition at line 25452 of file AArch64ISelLowering.cpp.

References llvm::TargetLowering::DAGCombinerInfo::CombineTo(), llvm::ISD::EXTLOAD, llvm::ISD::FP_ROUND, llvm::LoadSDNode::getBasePtr(), llvm::MemSDNode::getChain(), llvm::SelectionDAG::getExtLoad(), llvm::EVT::getFixedSizeInBits(), llvm::SelectionDAG::getIntPtrConstant(), llvm::MemSDNode::getMemOperand(), llvm::AArch64Subtarget::getMinSVEVectorSizeInBits(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::SDValue::hasOneUse(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::EVT::isFixedLengthVector(), llvm::ISD::isNormalLoad(), N, and llvm::AArch64Subtarget::useSVEForFixedLengthVectors().

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performFpToIntCombine()

static SDValue performFpToIntCombine ( SDNode *  N, SelectionDAG &  DAG, TargetLowering::DAGCombinerInfo &  DCI, const AArch64Subtarget *  Subtarget  )

static

Fold a floating-point multiply by power of two into floating-point to fixed-point conversion.

Definition at line 19034 of file AArch64ISelLowering.cpp.

References llvm::CallingConv::C, DL, llvm::ISD::FMUL, llvm::ISD::FP_TO_SINT, llvm::ISD::FP_TO_SINT_SAT, llvm::ISD::FP_TO_UINT_SAT, llvm::SelectionDAG::getConstant(), llvm::BuildVectorSDNode::getConstantFPSplatPow2ToLog2Int(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::ISD::INTRINSIC_WO_CHAIN, llvm::AArch64Subtarget::isNeonAvailable(), llvm::TargetLoweringBase::isTypeLegal(), N, and llvm::ISD::TRUNCATE.

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performGatherLoadCombine()

static SDValue performGatherLoadCombine ( SDNode *  N, SelectionDAG &  DAG, unsigned  Opcode, bool  OnlyPackedOffsets = true  )

static

Definition at line 25102 of file AArch64ISelLowering.cpp.

References llvm::ISD::ANY_EXTEND, assert(), llvm::sampleprof::Base, llvm::ISD::BITCAST, DL, llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getKnownMinValue(), llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), getScaledOffsetForBitWidth(), llvm::EVT::getSizeInBits(), getSVEContainerType(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValue(), llvm::SelectionDAG::getValueType(), llvm::SelectionDAG::getVTList(), llvm::AArch64ISD::GLD1_IMM_MERGE_ZERO, llvm::AArch64ISD::GLD1_MERGE_ZERO, llvm::AArch64ISD::GLD1_UXTW_MERGE_ZERO, llvm::AArch64ISD::GLD1Q_INDEX_MERGE_ZERO, llvm::AArch64ISD::GLD1Q_MERGE_ZERO, llvm::AArch64ISD::GLDFF1_IMM_MERGE_ZERO, llvm::AArch64ISD::GLDFF1_MERGE_ZERO, llvm::AArch64ISD::GLDFF1_UXTW_MERGE_ZERO, llvm::AArch64ISD::GLDNT1_INDEX_MERGE_ZERO, llvm::AArch64ISD::GLDNT1_MERGE_ZERO, llvm::EVT::isFloatingPoint(), llvm::EVT::isInteger(), llvm::EVT::isScalableVector(), isValidImmForSVEVecImmAddrMode(), N, llvm::Offset, llvm::AArch64::SVEBitsPerBlock, std::swap(), and llvm::ISD::TRUNCATE.

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performGLD1Combine()

static SDValue performGLD1Combine ( SDNode *  N, SelectionDAG &  DAG  )

static

Definition at line 22803 of file AArch64ISelLowering.cpp.

References assert(), llvm::sampleprof::Base, DL, getGatherVecOpcode(), llvm::SelectionDAG::getNode(), getSignExtendedGatherOpcode(), llvm::EVT::getVectorElementType(), llvm::VTSDNode::getVT(), llvm::AArch64ISD::GLD1_IMM_MERGE_ZERO, llvm::AArch64ISD::GLD1_MERGE_ZERO, llvm::AArch64ISD::GLD1_SCALED_MERGE_ZERO, llvm::AArch64ISD::GLD1_SXTW_MERGE_ZERO, llvm::AArch64ISD::GLD1_SXTW_SCALED_MERGE_ZERO, llvm::AArch64ISD::GLD1_UXTW_MERGE_ZERO, llvm::AArch64ISD::GLD1_UXTW_SCALED_MERGE_ZERO, llvm::AArch64ISD::GLD1S_IMM_MERGE_ZERO, llvm::AArch64ISD::GLD1S_MERGE_ZERO, llvm::AArch64ISD::GLD1S_SCALED_MERGE_ZERO, N, llvm::Offset, Scaled, llvm::AArch64ISD::SIGN_EXTEND_INREG_MERGE_PASSTHRU, Signed, and llvm::AArch64ISD::ZERO_EXTEND_INREG_MERGE_PASSTHRU.

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performGlobalAddressCombine()

static SDValue performGlobalAddressCombine ( SDNode *  N, SelectionDAG &  DAG, const AArch64Subtarget *  Subtarget, const TargetMachine &  TM  )

static

Definition at line 24880 of file AArch64ISelLowering.cpp.

References llvm::ISD::ADD, llvm::CallingConv::C, llvm::AArch64Subtarget::ClassifyGlobalReference(), DL, llvm::SelectionDAG::getConstant(), llvm::GlobalValue::getDataLayout(), llvm::GlobalAddressSDNode::getGlobal(), llvm::SelectionDAG::getGlobalAddress(), llvm::SelectionDAG::getNode(), llvm::GlobalAddressSDNode::getOffset(), llvm::DataLayout::getTypeAllocSize(), llvm::GlobalValue::getValueType(), llvm::AArch64II::MO_NO_FLAG, N, llvm::Offset, llvm::ISD::SUB, and llvm::SDNode::uses().

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performInsertSubvectorCombine()

static SDValue performInsertSubvectorCombine ( SDNode *  N, TargetLowering::DAGCombinerInfo &  DCI, SelectionDAG &  DAG  )

static

Definition at line 20041 of file AArch64ISelLowering.cpp.

References llvm::ISD::CONCAT_VECTORS, DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getVectorIdxConstant(), llvm::EVT::getVectorNumElements(), llvm::Hi, llvm::EVT::isFixedLengthVector(), llvm::TargetLoweringBase::isTypeLegal(), llvm::SDValue::isUndef(), llvm::Lo, and N.

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performInsertVectorEltCombine()

static SDValue performInsertVectorEltCombine ( SDNode *  N, TargetLowering::DAGCombinerInfo &  DCI  )

static

Definition at line 25445 of file AArch64ISelLowering.cpp.

References N, performPostLD1Combine(), and removeRedundantInsertVectorElt().

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performIntrinsicCombine()

static SDValue performIntrinsicCombine ( SDNode *  N, TargetLowering::DAGCombinerInfo &  DCI, const AArch64Subtarget *  Subtarget  )

static

Definition at line 21724 of file AArch64ISelLowering.cpp.

References llvm::ISD::ABDS, llvm::ISD::ABDU, llvm::ISD::ADD, llvm::ISD::AND, llvm::AArch64ISD::ANDV_PRED, llvm::AArch64CC::ANY_ACTIVE, llvm::AArch64ISD::BIC, combineAcrossLanesIntrinsic(), combineSVEReductionFP(), combineSVEReductionInt(), combineSVEReductionOrderedFP(), convertMergedOpToPredOp(), llvm::TargetLowering::DAGCombinerInfo::DAG, llvm::AArch64ISD::EORV_PRED, llvm::AArch64ISD::FADD_PRED, llvm::AArch64ISD::FADDA_PRED, llvm::AArch64ISD::FADDV_PRED, llvm::AArch64ISD::FDIV_PRED, llvm::AArch64CC::FIRST_ACTIVE, llvm::AArch64ISD::FMA_PRED, llvm::AArch64ISD::FMAX_PRED, llvm::ISD::FMAXIMUM, llvm::AArch64ISD::FMAXNM_PRED, llvm::AArch64ISD::FMAXNMV_PRED, llvm::ISD::FMAXNUM, llvm::AArch64ISD::FMAXV_PRED, llvm::AArch64ISD::FMIN_PRED, llvm::ISD::FMINIMUM, llvm::AArch64ISD::FMINNM_PRED, llvm::AArch64ISD::FMINNMV_PRED, llvm::ISD::FMINNUM, llvm::AArch64ISD::FMINV_PRED, llvm::AArch64ISD::FMUL_PRED, llvm::AArch64ISD::FSUB_PRED, llvm::SelectionDAG::getCondCode(), getIntrinsicID(), llvm::SelectionDAG::getNode(), getPTest(), llvm::AArch64CC::LAST_ACTIVE, LowerSVEIntrinsicDUP(), LowerSVEIntrinsicEXT(), LowerSVEIntrinsicIndex(), llvm::AArch64ISD::MUL_PRED, llvm::AArch64ISD::MULHS_PRED, llvm::AArch64ISD::MULHU_PRED, N, llvm::ISD::OR, llvm::AArch64ISD::ORV_PRED, llvm::AArch64ISD::PMULL, llvm::ISD::SADDSAT, llvm::AArch64ISD::SADDV, llvm::AArch64ISD::SADDV_PRED, llvm::AArch64ISD::SDIV_PRED, llvm::AArch64ISD::SETCC_MERGE_ZERO, llvm::ISD::SETEQ, llvm::ISD::SETGE, llvm::ISD::SETGT, llvm::ISD::SETLE, llvm::ISD::SETLT, llvm::ISD::SETNE, llvm::ISD::SETUGE, llvm::ISD::SETUGT, llvm::ISD::SETULE, llvm::ISD::SETULT, llvm::ISD::SETUO, llvm::AArch64ISD::SHL_PRED, llvm::AArch64ISD::SMAX_PRED, llvm::AArch64ISD::SMAXV, llvm::AArch64ISD::SMAXV_PRED, llvm::AArch64ISD::SMIN_PRED, llvm::AArch64ISD::SMINV, llvm::AArch64ISD::SMINV_PRED, llvm::AArch64ISD::SMULL, llvm::ISD::SPLAT_VECTOR, llvm::AArch64ISD::SRA_PRED, llvm::AArch64ISD::SRAD_MERGE_OP1, llvm::AArch64ISD::SRL_PRED, llvm::ISD::SSUBSAT, llvm::ISD::SUB, tryCombineCRC32(), tryCombineFixedPointConvert(), tryCombineLongOpWithDup(), tryCombineShiftImm(), tryCombineWhileLo(), tryConvertSVEWideCompare(), llvm::ISD::UADDSAT, llvm::AArch64ISD::UADDV, llvm::AArch64ISD::UADDV_PRED, llvm::AArch64ISD::UDIV_PRED, llvm::AArch64ISD::UMAX_PRED, llvm::AArch64ISD::UMAXV, llvm::AArch64ISD::UMAXV_PRED, llvm::AArch64ISD::UMIN_PRED, llvm::AArch64ISD::UMINV, llvm::AArch64ISD::UMINV_PRED, llvm::AArch64ISD::UMULL, llvm::ISD::USUBSAT, llvm::ISD::VSELECT, and llvm::ISD::XOR.

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performIntToFpCombine()

static SDValue performIntToFpCombine ( SDNode *  N, SelectionDAG &  DAG, const AArch64Subtarget *  Subtarget  )

static

Definition at line 18992 of file AArch64ISelLowering.cpp.

References llvm::MemSDNode::getAlign(), llvm::LoadSDNode::getBasePtr(), llvm::MemSDNode::getChain(), llvm::MachineMemOperand::getFlags(), llvm::SelectionDAG::getLoad(), llvm::MemSDNode::getMemOperand(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::MemSDNode::getPointerInfo(), llvm::EVT::getSizeInBits(), llvm::SDValue::hasOneUse(), llvm::AArch64Subtarget::isNeonAvailable(), llvm::ISD::isNormalLoad(), N, performVectorCompareAndMaskUnaryOpCombine(), llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), llvm::ISD::SINT_TO_FP, llvm::AArch64ISD::SITOF, and llvm::AArch64ISD::UITOF.

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performLastTrueTestVectorCombine()

static SDValue performLastTrueTestVectorCombine ( SDNode *  N, TargetLowering::DAGCombinerInfo &  DCI, const AArch64Subtarget *  Subtarget  )

static

Definition at line 19654 of file AArch64ISelLowering.cpp.

References llvm::ISD::ADD, assert(), llvm::TargetLowering::DAGCombinerInfo::DAG, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getKnownMinValue(), getPTest(), getPTrue(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementCount(), llvm::EVT::getVectorElementType(), Idx, llvm::isAllOnesConstant(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::EVT::isScalableVector(), llvm::AArch64CC::LAST_ACTIVE, N, and llvm::ISD::VSCALE.

Referenced by performExtractVectorEltCombine().

performLD1Combine()

static SDValue performLD1Combine ( SDNode *  N, SelectionDAG &  DAG, unsigned  Opc  )

static

Definition at line 22171 of file AArch64ISelLowering.cpp.

References DL, llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getKnownMinValue(), llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), getSVEContainerType(), llvm::SelectionDAG::getValueType(), llvm::SelectionDAG::getVTList(), llvm::EVT::isInteger(), N, llvm::AArch64::SVEBitsPerBlock, and llvm::ISD::TRUNCATE.

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performLD1ReplicateCombine()

template<unsigned Opcode>

static SDValue performLD1ReplicateCombine ( SDNode *  N, SelectionDAG &  DAG  )

static

Definition at line 22223 of file AArch64ISelLowering.cpp.

References llvm::ISD::BITCAST, llvm::EVT::changeTypeToInteger(), DL, llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), llvm::EVT::isFloatingPoint(), llvm::AArch64ISD::LD1RO_MERGE_ZERO, llvm::AArch64ISD::LD1RQ_MERGE_ZERO, and N.

performLDNT1Combine()

static SDValue performLDNT1Combine ( SDNode *  N, SelectionDAG &  DAG  )

static

Definition at line 22197 of file AArch64ISelLowering.cpp.

References llvm::ISD::BITCAST, llvm::EVT::changeTypeToInteger(), DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getMaskedLoad(), llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getUNDEF(), llvm::EVT::isFloatingPoint(), N, llvm::ISD::NON_EXTLOAD, and llvm::ISD::UNINDEXED.

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performLOADCombine()

static SDValue performLOADCombine ( SDNode *  N, TargetLowering::DAGCombinerInfo &  DCI, SelectionDAG &  DAG, const AArch64Subtarget *  Subtarget  )

static

Definition at line 23124 of file AArch64ISelLowering.cpp.

References combineV3I8LoadExt(), llvm::commonAlignment(), llvm::ISD::CONCAT_VECTORS, DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getContext(), llvm::TypeSize::getFixed(), llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getMemBasePlusOffset(), llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::EVT::getSimpleVT(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::EVT::getVectorElementType(), llvm::SelectionDAG::getVectorIdxConstant(), llvm::MVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::MVT::getVectorVT(), I, llvm::ISD::INSERT_SUBVECTOR, llvm::AArch64Subtarget::isLittleEndian(), llvm::EVT::isScalableVector(), N, performTBISimplification(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::AArch64Subtarget::supportsAddressTopByteIgnored(), and llvm::ISD::TokenFactor.

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performMaskedGatherScatterCombine()

static SDValue performMaskedGatherScatterCombine ( SDNode *  N, TargetLowering::DAGCombinerInfo &  DCI, SelectionDAG &  DAG  )

static

Definition at line 23634 of file AArch64ISelLowering.cpp.

References assert(), llvm::Data, DL, findMoreOptimalIndexType(), llvm::MaskedGatherScatterSDNode::getBasePtr(), llvm::MemSDNode::getChain(), llvm::MaskedGatherSDNode::getExtensionType(), llvm::MaskedGatherScatterSDNode::getIndex(), llvm::MaskedGatherScatterSDNode::getIndexType(), llvm::MaskedGatherScatterSDNode::getMask(), llvm::SelectionDAG::getMaskedGather(), llvm::SelectionDAG::getMaskedScatter(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getMemoryVT(), llvm::MaskedGatherSDNode::getPassThru(), llvm::MaskedGatherScatterSDNode::getScale(), llvm::MaskedScatterSDNode::getValue(), llvm::SelectionDAG::getVTList(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::MaskedScatterSDNode::isTruncatingStore(), and N.

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performMSTORECombine()

static SDValue performMSTORECombine ( SDNode *  N, TargetLowering::DAGCombinerInfo &  DCI, SelectionDAG &  DAG, const AArch64Subtarget *  Subtarget  )

static

Definition at line 23456 of file AArch64ISelLowering.cpp.

References llvm::ISD::BITCAST, llvm::EVT::changeVectorElementType(), DL, llvm::MaskedLoadStoreSDNode::getAddressingMode(), llvm::MaskedStoreSDNode::getBasePtr(), llvm::MemSDNode::getChain(), llvm::SelectionDAG::getContext(), llvm::MaskedStoreSDNode::getMask(), llvm::SelectionDAG::getMaskedStore(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getMemoryVT(), llvm::AArch64Subtarget::getMinSVEVectorSizeInBits(), llvm::getNumElementsFromSVEPredPattern(), llvm::MaskedStoreSDNode::getOffset(), getPTrue(), llvm::EVT::getSizeInBits(), llvm::MaskedStoreSDNode::getValue(), llvm::EVT::getVectorElementType(), llvm::Value::hasOneUse(), isHalvingTruncateOfLegalScalableType(), llvm::MaskedStoreSDNode::isTruncatingStore(), llvm::MaskedLoadStoreSDNode::isUnindexed(), N, llvm::AArch64ISD::PTRUE, trySimplifySrlAddToRshrnb(), llvm::AArch64ISD::UZP1, and llvm::EVT::widenIntegerVectorElementType().

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performMulCombine()

static SDValue performMulCombine ( SDNode *  N, SelectionDAG &  DAG, TargetLowering::DAGCombinerInfo &  DCI, const AArch64Subtarget *  Subtarget  )

static

Definition at line 18696 of file AArch64ISelLowering.cpp.

References llvm::Add, llvm::ISD::ADD, llvm::APInt::ashr(), llvm::BitWidth, llvm::CallingConv::C, llvm::APInt::countr_zero(), DL, llvm::APInt::getBitWidth(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::SDValue::getValueSizeInBits(), llvm::APInt::getZExtValue(), llvm::SDNode::hasOneUse(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::APInt::isNegative(), llvm::APInt::isNonNegative(), llvm::APInt::isPowerOf2(), isSignExtended(), IsSVECntIntrinsic(), isZeroExtended(), llvm::APInt::logBase2(), llvm::ISD::MUL, N, performMulVectorCmpZeroCombine(), performMulVectorExtendCombine(), performVectorExtCombine(), RHS, llvm::APInt::sdivrem(), llvm::APInt::sge(), llvm::ISD::SHL, llvm::APInt::sle(), llvm::ISD::SUB, std::swap(), llvm::ISD::TRUNCATE, and X.

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performMULLCombine()

static SDValue performMULLCombine ( SDNode *  N, TargetLowering::DAGCombinerInfo &  DCI, SelectionDAG &  DAG  )

static

Definition at line 25680 of file AArch64ISelLowering.cpp.

References N, llvm::Intrinsic::not_intrinsic, tryCombineLongOpWithDup(), and tryCombineMULLWithUZP1().

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performMulVectorCmpZeroCombine()

static SDValue performMulVectorCmpZeroCombine ( SDNode *  N, SelectionDAG &  DAG  )

static

Definition at line 18622 of file AArch64ISelLowering.cpp.

References llvm::And, llvm::ISD::AND, llvm::AArch64ISD::CMLTz, DL, llvm::SelectionDAG::getContext(), llvm::EVT::getIntegerVT(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getVectorElementCount(), llvm::EVT::getVectorVT(), llvm::ISD::isConstantSplatVector(), llvm::APInt::isMask(), N, llvm::AArch64ISD::NVCAST, and llvm::ISD::SRL.

Referenced by performMulCombine().

performMulVectorExtendCombine()

static SDValue performMulVectorExtendCombine ( SDNode *  Mul, SelectionDAG &  DAG  )

static

Combines a mul(dup(sext/zext)) node pattern into mul(sext/zext(dup)) making use of the vector SExt/ZExt rather than the scalar SExt/ZExt.

Definition at line 18602 of file AArch64ISelLowering.cpp.

References DL, llvm::SelectionDAG::getNode(), llvm::Mul, and performBuildShuffleExtendCombine().

Referenced by performMulCombine().

performNegCSelCombine()

static SDValue performNegCSelCombine ( SDNode *  N, SelectionDAG &  DAG  )

static

Definition at line 20485 of file AArch64ISelLowering.cpp.

References llvm::AArch64ISD::CSEL, DL, getNegatedInteger(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValueType(), llvm::SDNode::hasOneUse(), isNegatedInteger(), and N.

Referenced by performAddSubCombine().

performNEONPostLDSTCombine()

static SDValue performNEONPostLDSTCombine ( SDNode *  N, TargetLowering::DAGCombinerInfo &  DCI, SelectionDAG &  DAG  )

static

Target-specific DAG combine function for NEON load/store intrinsics to merge base address updates.

Definition at line 23672 of file AArch64ISelLowering.cpp.

References llvm::ISD::ADD, Addr, llvm::TargetLowering::DAGCombinerInfo::CombineTo(), llvm::SelectionDAG::getMemIntrinsicNode(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getMemoryVT(), llvm::SDValue::getNode(), llvm::User::getOperand(), llvm::SelectionDAG::getRegister(), llvm::EVT::getSizeInBits(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVTList(), llvm::SDNode::hasPredecessorHelper(), llvm::SmallPtrSetImpl< PtrType >::insert(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::TargetLowering::DAGCombinerInfo::isCalledByLegalizer(), llvm::AArch64ISD::LD1x2post, llvm::AArch64ISD::LD1x3post, llvm::AArch64ISD::LD1x4post, llvm::AArch64ISD::LD2DUPpost, llvm::AArch64ISD::LD2LANEpost, llvm::AArch64ISD::LD2post, llvm::AArch64ISD::LD3DUPpost, llvm::AArch64ISD::LD3LANEpost, llvm::AArch64ISD::LD3post, llvm::AArch64ISD::LD4DUPpost, llvm::AArch64ISD::LD4LANEpost, llvm::AArch64ISD::LD4post, llvm_unreachable, N, llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::AArch64ISD::ST1x2post, llvm::AArch64ISD::ST1x3post, llvm::AArch64ISD::ST1x4post, llvm::AArch64ISD::ST2LANEpost, llvm::AArch64ISD::ST2post, llvm::AArch64ISD::ST3LANEpost, llvm::AArch64ISD::ST3post, llvm::AArch64ISD::ST4LANEpost, and llvm::AArch64ISD::ST4post.

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performNVCASTCombine()

static SDValue performNVCASTCombine ( SDNode *  N, SelectionDAG &  DAG  )

static

Get rid of unnecessary NVCASTs (that don't change the type).

Definition at line 24867 of file AArch64ISelLowering.cpp.

References llvm::SelectionDAG::getNode(), N, and llvm::AArch64ISD::NVCAST.

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performORCombine()

static SDValue performORCombine ( SDNode *  N, TargetLowering::DAGCombinerInfo &  DCI, const AArch64Subtarget *  Subtarget, const AArch64TargetLowering &  TLI  )

static

Definition at line 19281 of file AArch64ISelLowering.cpp.

References llvm::TargetLowering::DAGCombinerInfo::DAG, llvm::SelectionDAG::getTargetLoweringInfo(), llvm::TargetLoweringBase::isTypeLegal(), N, performANDORCSELCombine(), and tryCombineToBSL().

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine(), llvm::LoongArchTargetLowering::PerformDAGCombine(), llvm::MipsTargetLowering::PerformDAGCombine(), llvm::MipsSETargetLowering::PerformDAGCombine(), and llvm::RISCVTargetLowering::PerformDAGCombine().

performOrXorChainCombine()

static SDValue performOrXorChainCombine ( SDNode *  N, SelectionDAG &  DAG  )

static

Definition at line 10423 of file AArch64ISelLowering.cpp.

References llvm::ISD::AND, Cond, DL, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getSetCC(), llvm::Value::hasOneUse(), I, llvm::isNullConstant(), isOrXorChain(), LHS, N, llvm::ISD::OR, RHS, llvm::ISD::SETCC, llvm::ISD::SETEQ, llvm::ISD::SETNE, and llvm::SmallVectorBase< Size_T >::size().

Referenced by performSETCCCombine().

performPostLD1Combine()

static SDValue performPostLD1Combine ( SDNode *  N, TargetLowering::DAGCombinerInfo &  DCI, bool  IsLaneOp  )

static

Target-specific DAG combine function for post-increment LD1 (lane) and post-increment LD1R.

Definition at line 22913 of file AArch64ISelLowering.cpp.

References llvm::ISD::ADD, Addr, llvm::TargetLowering::DAGCombinerInfo::CombineTo(), llvm::TargetLowering::DAGCombinerInfo::DAG, llvm::ISD::FMA, llvm::ISD::FMUL, llvm::SelectionDAG::getMemIntrinsicNode(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getMemoryVT(), llvm::SDValue::getNode(), llvm::User::getOperand(), llvm::SelectionDAG::getRegister(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVTList(), llvm::SDNode::hasPredecessorHelper(), llvm::SmallPtrSetImpl< PtrType >::insert(), llvm::EVT::is128BitVector(), llvm::EVT::is64BitVector(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::AArch64ISD::LD1DUPpost, llvm::AArch64ISD::LD1LANEpost, llvm::ISD::LOAD, N, llvm::SmallVectorTemplateBase< T, bool >::push_back(), and llvm::Vector.

Referenced by performDUPCombine(), and performInsertVectorEltCombine().

performReinterpretCastCombine()

static SDValue performReinterpretCastCombine ( SDNode *  N )

static

Definition at line 19326 of file AArch64ISelLowering.cpp.

References llvm::SDValue::getValueType(), N, and llvm::AArch64ISD::REINTERPRET_CAST.

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performScalarToVectorCombine()

static SDValue performScalarToVectorCombine ( SDNode *  N, TargetLowering::DAGCombinerInfo &  DCI, SelectionDAG &  DAG  )

static

Definition at line 25694 of file AArch64ISelLowering.cpp.

References DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValueType(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::isNullConstant(), N, llvm::AArch64ISD::NVCAST, llvm::AArch64ISD::UADDLV, and llvm::ISD::ZERO_EXTEND.

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performScatterStoreCombine()

static SDValue performScatterStoreCombine ( SDNode *  N, SelectionDAG &  DAG, unsigned  Opcode, bool  OnlyPackedOffsets = true  )

static

Definition at line 24989 of file AArch64ISelLowering.cpp.

References llvm::ISD::ANY_EXTEND, assert(), llvm::sampleprof::Base, llvm::ISD::BITCAST, DL, llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getKnownMinValue(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), getScaledOffsetForBitWidth(), llvm::EVT::getSimpleVT(), llvm::EVT::getSizeInBits(), llvm::MVT::getSizeInBits(), getSVEContainerType(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValue(), llvm::SelectionDAG::getValueType(), llvm::EVT::getVectorElementType(), llvm::SelectionDAG::getVTList(), llvm::EVT::isFloatingPoint(), llvm::MVT::isFloatingPoint(), llvm::EVT::isScalableVector(), isValidImmForSVEVecImmAddrMode(), N, llvm::Offset, llvm::AArch64ISD::SST1_IMM_PRED, llvm::AArch64ISD::SST1_PRED, llvm::AArch64ISD::SST1_UXTW_PRED, llvm::AArch64ISD::SST1Q_INDEX_PRED, llvm::AArch64ISD::SST1Q_PRED, llvm::AArch64ISD::SSTNT1_INDEX_PRED, llvm::AArch64ISD::SSTNT1_PRED, llvm::AArch64::SVEBitsPerBlock, and std::swap().

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performSelectCombine()

static SDValue performSelectCombine ( SDNode *  N, TargetLowering::DAGCombinerInfo &  DCI  )

static

A vector select: "(select vL, vR, (setcc LHS, RHS))" is best performed with the compare-mask instructions rather than going via NZCV, even if LHS and RHS are really scalar.

This replaces any scalar setcc in the above pattern with a vector one followed by a DUP shuffle on the result.

Definition at line 24760 of file AArch64ISelLowering.cpp.

References assert(), llvm::ISD::BITCAST, llvm::EVT::changeVectorElementTypeToInteger(), llvm::TargetLowering::DAGCombinerInfo::DAG, DL, llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getSelect(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::EVT::getVectorVT(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::EVT::isFloatingPoint(), llvm::EVT::isScalableVT(), llvm::TargetLoweringBase::isTypeLegal(), llvm::EVT::isVector(), LHS, N, RHS, llvm::ISD::SCALAR_TO_VECTOR, and llvm::ISD::SETCC.

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performSetccAddFolding()

static SDValue performSetccAddFolding ( SDNode *  Op, SelectionDAG &  DAG  )

static

Definition at line 20286 of file AArch64ISelLowering.cpp.

References SetCCInfo::AArch64, llvm::ISD::ADD, assert(), GenericSetCCInfo::CC, AArch64SetCCInfo::CC, AArch64SetCCInfo::Cmp, llvm::AArch64ISD::CSEL, SetCCInfo::Generic, getAArch64Cmp(), llvm::SelectionDAG::getConstant(), llvm::AArch64CC::getInvertedCondCode(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::ISD::getSetCCInverse(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), SetCCInfoAndKind::Info, SetCCInfoAndKind::IsAArch64, isSetCCOrZExtSetCC(), LHS, GenericSetCCInfo::Opnd0, GenericSetCCInfo::Opnd1, RHS, and std::swap().

Referenced by performAddSubLongCombine().

performSETCCCombine()

static SDValue performSETCCCombine ( SDNode *  N, TargetLowering::DAGCombinerInfo &  DCI, SelectionDAG &  DAG  )

static

Definition at line 24363 of file AArch64ISelLowering.cpp.

References llvm::ISD::AND, assert(), llvm::ISD::BITCAST, Cond, llvm::AArch64ISD::CSEL, DL, llvm::SelectionDAG::getConstant(), llvm::EVT::getFixedSizeInBits(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), llvm::SelectionDAG::getSetCC(), llvm::SelectionDAG::getSignedConstant(), llvm::EVT::getVectorElementType(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::Value::hasOneUse(), llvm::isAllOnesConstant(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::EVT::isFixedLengthVector(), llvm::isNullConstant(), llvm::isOneConstant(), llvm::EVT::isScalarInteger(), LHS, N, performOrXorChainCombine(), RHS, llvm::ISD::SETCC, llvm::ISD::SETEQ, llvm::ISD::SETNE, llvm::ISD::SIGN_EXTEND, llvm::ISD::SRL, tryToWidenSetCCOperands(), llvm::ISD::VECREDUCE_AND, llvm::ISD::VECREDUCE_OR, and llvm::ISD::ZERO_EXTEND.

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine(), llvm::LoongArchTargetLowering::PerformDAGCombine(), llvm::MipsSETargetLowering::PerformDAGCombine(), and llvm::RISCVTargetLowering::PerformDAGCombine().

performSetccMergeZeroCombine()

static SDValue performSetccMergeZeroCombine ( SDNode *  N, TargetLowering::DAGCombinerInfo &  DCI  )

static

Definition at line 24501 of file AArch64ISelLowering.cpp.

References llvm::ISD::AND, assert(), Cond, llvm::TargetLowering::DAGCombinerInfo::DAG, llvm::SelectionDAG::getNode(), llvm::TargetLowering::DAGCombinerInfo::isAfterLegalizeDAG(), isAllActivePredicate(), isZerosVector(), LHS, N, performSetCCPunpkCombine(), RHS, llvm::AArch64ISD::SETCC_MERGE_ZERO, llvm::ISD::SETNE, and llvm::ISD::SIGN_EXTEND.

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performSetCCPunpkCombine()

static SDValue performSetCCPunpkCombine ( SDNode *  N, SelectionDAG &  DAG  )

static

Definition at line 24462 of file AArch64ISelLowering.cpp.

References Cond, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SDValue::getConstantOperandVal(), llvm::SDNode::getConstantOperandVal(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SDNode::getValueType(), isZerosVector(), LHS, N, llvm::AArch64ISD::PTRUE, RHS, llvm::AArch64ISD::SETCC_MERGE_ZERO, llvm::ISD::SETNE, and llvm::ISD::SIGN_EXTEND.

Referenced by performSetccMergeZeroCombine().

performSignExtendInRegCombine()

static SDValue performSignExtendInRegCombine ( SDNode *  N, TargetLowering::DAGCombinerInfo &  DCI, SelectionDAG &  DAG  )

static

Definition at line 25209 of file AArch64ISelLowering.cpp.

References assert(), llvm::TargetLowering::DAGCombinerInfo::CombineTo(), DL, EnableCombineMGatherIntrinsics, llvm::SelectionDAG::getContext(), llvm::EVT::getDoubleNumVectorElementsVT(), llvm::SelectionDAG::getNode(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getValueType(), llvm::EVT::getVectorElementType(), llvm::SelectionDAG::getVTList(), llvm::AArch64ISD::GLD1_IMM_MERGE_ZERO, llvm::AArch64ISD::GLD1_MERGE_ZERO, llvm::AArch64ISD::GLD1_SCALED_MERGE_ZERO, llvm::AArch64ISD::GLD1_SXTW_MERGE_ZERO, llvm::AArch64ISD::GLD1_SXTW_SCALED_MERGE_ZERO, llvm::AArch64ISD::GLD1_UXTW_MERGE_ZERO, llvm::AArch64ISD::GLD1_UXTW_SCALED_MERGE_ZERO, llvm::AArch64ISD::GLD1S_IMM_MERGE_ZERO, llvm::AArch64ISD::GLD1S_MERGE_ZERO, llvm::AArch64ISD::GLD1S_SCALED_MERGE_ZERO, llvm::AArch64ISD::GLD1S_SXTW_MERGE_ZERO, llvm::AArch64ISD::GLD1S_SXTW_SCALED_MERGE_ZERO, llvm::AArch64ISD::GLD1S_UXTW_MERGE_ZERO, llvm::AArch64ISD::GLD1S_UXTW_SCALED_MERGE_ZERO, llvm::AArch64ISD::GLDFF1_IMM_MERGE_ZERO, llvm::AArch64ISD::GLDFF1_MERGE_ZERO, llvm::AArch64ISD::GLDFF1_SCALED_MERGE_ZERO, llvm::AArch64ISD::GLDFF1_SXTW_MERGE_ZERO, llvm::AArch64ISD::GLDFF1_SXTW_SCALED_MERGE_ZERO, llvm::AArch64ISD::GLDFF1_UXTW_MERGE_ZERO, llvm::AArch64ISD::GLDFF1_UXTW_SCALED_MERGE_ZERO, llvm::AArch64ISD::GLDFF1S_IMM_MERGE_ZERO, llvm::AArch64ISD::GLDFF1S_MERGE_ZERO, llvm::AArch64ISD::GLDFF1S_SCALED_MERGE_ZERO, llvm::AArch64ISD::GLDFF1S_SXTW_MERGE_ZERO, llvm::AArch64ISD::GLDFF1S_SXTW_SCALED_MERGE_ZERO, llvm::AArch64ISD::GLDFF1S_UXTW_MERGE_ZERO, llvm::AArch64ISD::GLDFF1S_UXTW_SCALED_MERGE_ZERO, llvm::AArch64ISD::GLDNT1_MERGE_ZERO, llvm::AArch64ISD::GLDNT1S_MERGE_ZERO, I, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::AArch64ISD::LD1_MERGE_ZERO, llvm::AArch64ISD::LD1S_MERGE_ZERO, llvm::AArch64ISD::LDFF1_MERGE_ZERO, llvm::AArch64ISD::LDFF1S_MERGE_ZERO, llvm::AArch64ISD::LDNF1_MERGE_ZERO, llvm::AArch64ISD::LDNF1S_MERGE_ZERO, N, llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::ISD::SIGN_EXTEND_INREG, llvm::AArch64ISD::SUNPKHI, llvm::AArch64ISD::SUNPKLO, llvm::AArch64ISD::UUNPKHI, and llvm::AArch64ISD::UUNPKLO.

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performSignExtendSetCCCombine()

static SDValue performSignExtendSetCCCombine ( SDNode *  N, TargetLowering::DAGCombinerInfo &  DCI, SelectionDAG &  DAG  )

static

Definition at line 22034 of file AArch64ISelLowering.cpp.

References assert(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getSetCC(), llvm::SDNode::getValueType(), isCheapToExtend(), llvm::EVT::isInteger(), N, llvm::ISD::SETCC, llvm::ISD::SIGN_EXTEND, and llvm::ISD::ZERO_EXTEND.

Referenced by performExtendCombine().

performSpliceCombine()

static SDValue performSpliceCombine ( SDNode *  N, SelectionDAG &  DAG  )

static

Definition at line 22493 of file AArch64ISelLowering.cpp.

References assert(), N, and llvm::AArch64ISD::SPLICE.

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performST1Combine()

static SDValue performST1Combine ( SDNode *  N, SelectionDAG &  DAG  )

static

Definition at line 22244 of file AArch64ISelLowering.cpp.

References llvm::ISD::ANY_EXTEND, llvm::ISD::BITCAST, llvm::Data, DL, llvm::SelectionDAG::getNode(), getSVEContainerType(), llvm::SelectionDAG::getValueType(), llvm::EVT::isFloatingPoint(), N, and llvm::AArch64ISD::ST1_PRED.

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performSTNT1Combine()

static SDValue performSTNT1Combine ( SDNode *  N, SelectionDAG &  DAG  )

static

Definition at line 22270 of file AArch64ISelLowering.cpp.

References llvm::ISD::BITCAST, llvm::EVT::changeTypeToInteger(), llvm::Data, DL, llvm::SelectionDAG::getMaskedStore(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getUNDEF(), llvm::EVT::isFloatingPoint(), N, and llvm::ISD::UNINDEXED.

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performSTORECombine()

static SDValue performSTORECombine ( SDNode *  N, TargetLowering::DAGCombinerInfo &  DCI, SelectionDAG &  DAG, const AArch64Subtarget *  Subtarget  )

static

Definition at line 23398 of file AArch64ISelLowering.cpp.

References combineBoolVectorAndTruncateStore(), combineI8TruncStore(), foldTruncStoreOfExt(), llvm::ISD::FP_ROUND, llvm::EVT::getFixedSizeInBits(), llvm::AArch64Subtarget::getMinSVEVectorSizeInBits(), llvm::SelectionDAG::getTruncStore(), llvm::EVT::getVectorElementType(), llvm::Value::hasOneUse(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::EVT::isFixedLengthVector(), isHalvingTruncateOfLegalScalableType(), N, performTBISimplification(), Ptr, splitStores(), llvm::AArch64Subtarget::supportsAddressTopByteIgnored(), trySimplifySrlAddToRshrnb(), and llvm::AArch64Subtarget::useSVEForFixedLengthVectors().

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performSubAddMULCombine()

static SDValue performSubAddMULCombine ( SDNode *  N, SelectionDAG &  DAG  )

static

Definition at line 20875 of file AArch64ISelLowering.cpp.

References llvm::Add, llvm::ISD::ADD, llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SelectionDAG::isConstantIntBuildVectorOrConstantInt(), llvm::M1(), llvm::ISD::MUL, N, llvm::peekThroughBitcasts(), llvm::AArch64ISD::SMULL, llvm::ISD::SUB, llvm::AArch64ISD::UMULL, and X.

Referenced by performAddSubCombine().

performSubsToAndsCombine()

static SDValue performSubsToAndsCombine ( SDNode *  N, SDNode *  SubsNode, SDNode *  AndNode, SelectionDAG &  DAG, unsigned  CCIndex, unsigned  CmpIndex, unsigned  CC  )

static

Definition at line 23991 of file AArch64ISelLowering.cpp.

References llvm::AArch64ISD::ANDS, assert(), CC, DL, llvm::AArch64CC::EQ, llvm::ConstantSDNode::getAPIntValue(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::SDNode::getValueType(), llvm::SDNode::getVTList(), llvm::AArch64CC::HI, llvm::APInt::isMask(), llvm::APInt::isPowerOf2(), llvm::AArch64CC::LO, N, and llvm::AArch64CC::NE.

Referenced by performCONDCombine().

performSunpkloCombine()

static SDValue performSunpkloCombine ( SDNode *  N, SelectionDAG &  DAG  )

static

Definition at line 22894 of file AArch64ISelLowering.cpp.

References CC, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getVectorIdxConstant(), N, and llvm::ISD::SIGN_EXTEND.

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performSVEAndCombine()

static SDValue performSVEAndCombine ( SDNode *  N, TargetLowering::DAGCombinerInfo &  DCI  )

static

Definition at line 19337 of file AArch64ISelLowering.cpp.

References llvm::And, llvm::ISD::AND, llvm::CallingConv::C, llvm::TargetLowering::DAGCombinerInfo::DAG, DL, EnableCombineMGatherIntrinsics, llvm::ISD::EXTLOAD, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::EVT::getSizeInBits(), llvm::SDNode::getValueType(), llvm::EVT::getVectorElementType(), llvm::AArch64ISD::GLD1_IMM_MERGE_ZERO, llvm::AArch64ISD::GLD1_MERGE_ZERO, llvm::AArch64ISD::GLD1_SCALED_MERGE_ZERO, llvm::AArch64ISD::GLD1_SXTW_MERGE_ZERO, llvm::AArch64ISD::GLD1_SXTW_SCALED_MERGE_ZERO, llvm::AArch64ISD::GLD1_UXTW_MERGE_ZERO, llvm::AArch64ISD::GLD1_UXTW_SCALED_MERGE_ZERO, llvm::AArch64ISD::GLDFF1_IMM_MERGE_ZERO, llvm::AArch64ISD::GLDFF1_MERGE_ZERO, llvm::AArch64ISD::GLDFF1_SCALED_MERGE_ZERO, llvm::AArch64ISD::GLDFF1_SXTW_MERGE_ZERO, llvm::AArch64ISD::GLDFF1_SXTW_SCALED_MERGE_ZERO, llvm::AArch64ISD::GLDFF1_UXTW_MERGE_ZERO, llvm::AArch64ISD::GLDFF1_UXTW_SCALED_MERGE_ZERO, llvm::AArch64ISD::GLDNT1_MERGE_ZERO, isAllActivePredicate(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), isConstantSplatVectorMaskForType(), llvm::AArch64ISD::LD1_MERGE_ZERO, llvm::AArch64ISD::LDFF1_MERGE_ZERO, llvm::AArch64ISD::LDNF1_MERGE_ZERO, N, llvm::ISD::SPLAT_VECTOR, llvm::AArch64ISD::UUNPKHI, llvm::AArch64ISD::UUNPKLO, and llvm::ISD::ZEXTLOAD.

Referenced by performANDCombine().

performSVEMulAddSubCombine()

static SDValue performSVEMulAddSubCombine ( SDNode *  N, TargetLowering::DAGCombinerInfo &  DCI  )

static

Definition at line 20911 of file AArch64ISelLowering.cpp.

References llvm::ISD::ADD, convertFromScalableVector(), convertToScalableVector(), llvm::TargetLowering::DAGCombinerInfo::DAG, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getSubtarget(), llvm::SDValue::getValueType(), llvm::SDValue::hasOneUse(), llvm::TargetLowering::DAGCombinerInfo::isAfterLegalizeDAG(), llvm::EVT::isScalableVector(), llvm::AArch64ISD::MUL_PRED, N, and llvm::ISD::SUB.

Referenced by performAddSubCombine().

performTBISimplification()

static bool performTBISimplification ( SDValue  Addr, TargetLowering::DAGCombinerInfo &  DCI, SelectionDAG &  DAG  )

static

Simplify Addr given that the top byte of it is ignored by HW during address translation.

Definition at line 23029 of file AArch64ISelLowering.cpp.

References Addr, llvm::TargetLowering::DAGCombinerInfo::CommitTargetLoweringOpt(), llvm::APInt::getLowBitsSet(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), and llvm::TargetLowering::SimplifyDemandedBits().

Referenced by performLOADCombine(), and performSTORECombine().

performTBZCombine()

static SDValue performTBZCombine ( SDNode *  N, TargetLowering::DAGCombinerInfo &  DCI, SelectionDAG &  DAG  )

static

Definition at line 24620 of file AArch64ISelLowering.cpp.

References assert(), DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), getTestBitOperand(), N, llvm::AArch64ISD::TBNZ, and llvm::AArch64ISD::TBZ.

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performTruncateCombine()

static SDValue performTruncateCombine ( SDNode *  N, SelectionDAG &  DAG  )

static

Definition at line 20764 of file AArch64ISelLowering.cpp.

References llvm::AArch64ISD::DUP, llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarType(), llvm::SDValue::getValueType(), llvm::SDValue::hasOneUse(), llvm::EVT::is64BitVector(), llvm::EVT::isFixedLengthVector(), N, and llvm::ISD::TRUNCATE.

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performUADDVAddCombine()

static SDValue performUADDVAddCombine ( SDValue  A, SelectionDAG &  DAG  )

static

Definition at line 18282 of file AArch64ISelLowering.cpp.

References A, llvm::ISD::ADD, assert(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::SDValue::getConstantOperandVal(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), performUADDVAddCombine(), llvm::AArch64ISD::SADDLP, llvm::ISD::SIGN_EXTEND, llvm::AArch64ISD::UADDLP, and llvm::ISD::ZERO_EXTEND.

Referenced by performUADDVAddCombine(), and performUADDVCombine().

performUADDVCombine()

static SDValue performUADDVCombine ( SDNode *  N, SelectionDAG &  DAG  )

static

Definition at line 18369 of file AArch64ISelLowering.cpp.

References A, llvm::ISD::ADD, llvm::SelectionDAG::getNode(), N, performUADDVAddCombine(), and performUADDVZextCombine().

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performUADDVZextCombine()

static SDValue performUADDVZextCombine ( SDValue  A, SelectionDAG &  DAG  )

static

Definition at line 18331 of file AArch64ISelLowering.cpp.

References A, llvm::ISD::ADD, assert(), Concat, llvm::ISD::CONCAT_VECTORS, llvm::SelectionDAG::getContext(), llvm::EVT::getDoubleNumVectorElementsVT(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getSimpleVT(), llvm::SDValue::getValueType(), llvm_unreachable, llvm::AArch64ISD::NVCAST, llvm::MVT::SimpleTy, llvm::AArch64ISD::UADDLV, and llvm::ISD::ZERO_EXTEND.

Referenced by performUADDVCombine().

performUnpackCombine()

static SDValue performUnpackCombine ( SDNode *  N, SelectionDAG &  DAG, const AArch64Subtarget *  Subtarget  )

static

Definition at line 22503 of file AArch64ISelLowering.cpp.

References assert(), llvm::EVT::changeVectorElementType(), DL, llvm::MaskedLoadStoreSDNode::getAddressingMode(), llvm::MaskedLoadSDNode::getBasePtr(), llvm::MemSDNode::getChain(), llvm::SelectionDAG::getConstant(), llvm::MaskedLoadSDNode::getExtensionType(), llvm::MaskedLoadSDNode::getMask(), llvm::SelectionDAG::getMaskedLoad(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getMemoryVT(), llvm::AArch64Subtarget::getMinSVEVectorSizeInBits(), llvm::SDValue::getNode(), llvm::getNumElementsFromSVEPredPattern(), llvm::MaskedLoadSDNode::getOffset(), llvm::MaskedLoadSDNode::getPassThru(), getPTrue(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValue(), llvm::EVT::getVectorElementType(), llvm::SDNode::isUndef(), llvm::MaskedLoadStoreSDNode::isUnindexed(), isZerosVector(), llvm::ISD::MLOAD, N, llvm::AArch64ISD::PTRUE, llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), llvm::ISD::SEXTLOAD, llvm::AArch64ISD::UUNPKHI, llvm::AArch64ISD::UUNPKLO, and llvm::ISD::ZEXTLOAD.

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performUzpCombine()

static SDValue performUzpCombine ( SDNode *  N, SelectionDAG &  DAG, const AArch64Subtarget *  Subtarget  )

static

Definition at line 22633 of file AArch64ISelLowering.cpp.

References llvm::ISD::BITCAST, Concat, llvm::ISD::CONCAT_VECTORS, DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getConstantOperandVal(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::EVT::getDoubleNumVectorElementsVT(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::EVT::getSimpleVT(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::EVT::getVectorMinNumElements(), isHalvingTruncateAndConcatOfLegalIntScalableType(), llvm::DataLayout::isLittleEndian(), llvm::EVT::isSimple(), llvm_unreachable, N, llvm::peekThroughBitcasts(), llvm::MVT::SimpleTy, llvm::ISD::TRUNCATE, tryCombineExtendRShTrunc(), trySimplifySrlAddToRshrnb(), llvm::ISD::UNDEF, llvm::AArch64ISD::UUNPKHI, llvm::AArch64ISD::UUNPKLO, llvm::AArch64ISD::UZP1, llvm::AArch64ISD::UZP2, and X.

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performVecReduceAddCombine()

static SDValue performVecReduceAddCombine ( SDNode *  N, SelectionDAG &  DAG, const AArch64Subtarget *  ST  )

static

Definition at line 18176 of file AArch64ISelLowering.cpp.

References A, llvm::ISD::ADD, B, llvm::ISD::CONCAT_VECTORS, DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::EVT::getVectorVT(), I, llvm::EVT::isScalableVT(), llvm::ISD::MUL, N, performVecReduceAddCombineWithUADDLP(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::AArch64ISD::SDOT, llvm::ISD::SIGN_EXTEND, llvm::AArch64ISD::UDOT, llvm::ISD::VECREDUCE_ADD, and llvm::ISD::ZERO_EXTEND.

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performVecReduceAddCombineWithUADDLP()

static SDValue performVecReduceAddCombineWithUADDLP ( SDNode *  N, SelectionDAG &  DAG  )

static

Definition at line 18095 of file AArch64ISelLowering.cpp.

References llvm::ISD::ABDS, llvm::ISD::ABDU, llvm::ISD::ABS, llvm::ISD::ADD, DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::SDNode::getValueType(), N, llvm::ISD::SIGN_EXTEND, llvm::ISD::SUB, llvm::AArch64ISD::UADDLP, llvm::ISD::VECREDUCE_ADD, and llvm::ISD::ZERO_EXTEND.

Referenced by performVecReduceAddCombine().

performVecReduceBitwiseCombine()

static SDValue performVecReduceBitwiseCombine ( SDNode *  N, TargetLowering::DAGCombinerInfo &  DCI, SelectionDAG &  DAG  )

static

Definition at line 24348 of file AArch64ISelLowering.cpp.

References DL, llvm::SDValue::getValueType(), getVectorBitwiseReduce(), llvm::EVT::getVectorElementType(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::EVT::isFixedLengthVector(), llvm::EVT::isPow2VectorType(), and N.

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performVectorCompareAndMaskUnaryOpCombine()

static SDValue performVectorCompareAndMaskUnaryOpCombine ( SDNode *  N, SelectionDAG &  DAG  )

static

Definition at line 18946 of file AArch64ISelLowering.cpp.

References llvm::ISD::AND, llvm::ISD::BITCAST, DL, llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::EVT::isVector(), N, and llvm::ISD::SETCC.

Referenced by performIntToFpCombine().

performVectorExtCombine()

static SDValue performVectorExtCombine ( SDNode *  N, SelectionDAG &  DAG  )

static

Definition at line 18659 of file AArch64ISelLowering.cpp.

References DL, llvm::SelectionDAG::getContext(), llvm::EVT::getHalfSizedIntegerVT(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarType(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementCount(), llvm::EVT::getVectorVT(), llvm::EVT::isFixedLengthVector(), llvm::ISD::MUL, N, S1, llvm::ISD::SIGN_EXTEND, and llvm::ISD::ZERO_EXTEND.

Referenced by performAddSubCombine(), and performMulCombine().

performVectorShiftCombine()

static SDValue performVectorShiftCombine ( SDNode *  N, const AArch64TargetLowering &  TLI, TargetLowering::DAGCombinerInfo &  DCI  )

static

Optimize a vector shift instruction and its operand if shifted out bits are not used.

Definition at line 22862 of file AArch64ISelLowering.cpp.

References assert(), llvm::SelectionDAG::ComputeNumSignBits(), llvm::TargetLowering::DAGCombinerInfo::DAG, llvm::APInt::getLowBitsSet(), N, llvm::TargetLowering::SimplifyDemandedBits(), llvm::AArch64ISD::VASHR, llvm::AArch64ISD::VLSHR, and llvm::AArch64ISD::VSHL.

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performVSelectCombine()

static SDValue performVSelectCombine ( SDNode *  N, SelectionDAG &  DAG  )

static

Definition at line 24689 of file AArch64ISelLowering.cpp.

References llvm::EVT::changeVectorElementTypeToInteger(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getCondCode(), llvm::SelectionDAG::getConstant(), llvm::ElementCount::getFixed(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::SelectionDAG::getSetCC(), llvm::EVT::getSimpleVT(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementCount(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::is_contained(), isAllActivePredicate(), isAllInactivePredicate(), llvm::ISD::isConstantSplatVector(), llvm::ISD::isConstantSplatVectorAllOnes(), llvm::EVT::isFloatingPoint(), llvm::APInt::isOne(), llvm::EVT::isSimple(), N, llvm::Or, llvm::ISD::OR, llvm::ISD::SETCC, llvm::ISD::SETGT, llvm::MVT::SimpleTy, llvm::ISD::SRA, trySwapVSelectOperands(), and llvm::ISD::VSELECT.

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performXorCombine()

static SDValue performXorCombine ( SDNode *  N, SelectionDAG &  DAG, TargetLowering::DAGCombinerInfo &  DCI, const AArch64Subtarget *  Subtarget  )

static

Definition at line 18380 of file AArch64ISelLowering.cpp.

References foldVectorXorShiftIntoCmp(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), and N.

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

ReconstructShuffleWithRuntimeMask()

SDValue ReconstructShuffleWithRuntimeMask	(	SDValue	Op,
		SelectionDAG &	DAG
	)

Definition at line 12101 of file AArch64ISelLowering.cpp.

References llvm::ISD::AND, llvm::ISD::ANY_EXTEND, assert(), llvm::ISD::BUILD_VECTOR, llvm::ISD::CONCAT_VECTORS, llvm::SmallVectorBase< Size_T >::empty(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::ISD::INTRINSIC_WO_CHAIN, llvm::EVT::isScalableVector(), and llvm::SmallVectorTemplateBase< T, bool >::push_back().

ReconstructTruncateFromBuildVector()

static SDValue ReconstructTruncateFromBuildVector ( SDValue  V, SelectionDAG &  DAG  )

static

Definition at line 12523 of file AArch64ISelLowering.cpp.

References assert(), llvm::sampleprof::Base, llvm::ISD::CONCAT_VECTORS, DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SDValue::getConstantOperandVal(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValueType(), llvm::ISD::TRUNCATE, X, and Y.

removeRedundantInsertVectorElt()

static SDValue removeRedundantInsertVectorElt ( SDNode *  N )

static

Definition at line 25408 of file AArch64ISelLowering.cpp.

References assert(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SDValue::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValueType(), llvm::ISD::INSERT_VECTOR_ELT, llvm::ISD::isConstantSplatVectorAllZeros(), isLanes1toNKnownZero(), llvm::isNullConstant(), and N.

Referenced by performInsertVectorEltCombine().

ReplaceAddWithADDP()

static void ReplaceAddWithADDP ( SDNode *  N, SmallVectorImpl< SDValue > &  Results, SelectionDAG &  DAG, const AArch64Subtarget *  Subtarget  )

static

Definition at line 26338 of file AArch64ISelLowering.cpp.

References llvm::AArch64ISD::ADDP, assert(), llvm::ISD::CONCAT_VECTORS, DL, llvm::SelectionDAG::getNode(), llvm::EVT::getScalarType(), llvm::SelectionDAG::getUNDEF(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), I, llvm::EVT::is256BitVector(), llvm::EVT::isFloatingPoint(), N, llvm::SmallVectorTemplateBase< T, bool >::push_back(), Results, llvm::SelectionDAG::SplitVector(), and X.

ReplaceATOMIC_LOAD_128Results()

static void ReplaceATOMIC_LOAD_128Results ( SDNode *  N, SmallVectorImpl< SDValue > &  Results, SelectionDAG &  DAG, const AArch64Subtarget *  Subtarget  )

static

Definition at line 26612 of file AArch64ISelLowering.cpp.

References assert(), llvm::ISD::ATOMIC_LOAD_AND, llvm::ISD::BUILD_PAIR, getAtomicLoad128Opcode(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getMachineNode(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getVTList(), llvm::Hi, llvm::DataLayout::isBigEndian(), llvm::Lo, N, Ptr, Results, llvm::SelectionDAG::setNodeMemRefs(), llvm::SelectionDAG::SplitScalar(), std::swap(), and llvm::ISD::XOR.

replaceBoolVectorBitcast()

static void replaceBoolVectorBitcast ( SDNode *  N, SmallVectorImpl< SDValue > &  Results, SelectionDAG &  DAG  )

static

Definition at line 26243 of file AArch64ISelLowering.cpp.

References assert(), llvm::ISD::CONCAT_VECTORS, DL, llvm::DWARFExpression::Operation::getNumOperands(), llvm::EVT::getVectorElementType(), llvm::SelectionDAG::getZExtOrTrunc(), I, llvm::EVT::isVector(), N, Results, and vectorToScalarBitmask().

ReplaceCMP_SWAP_128Results()

static void ReplaceCMP_SWAP_128Results ( SDNode *  N, SmallVectorImpl< SDValue > &  Results, SelectionDAG &  DAG, const AArch64Subtarget *  Subtarget  )

static

Definition at line 26449 of file AArch64ISelLowering.cpp.

References llvm::Acquire, llvm::AcquireRelease, assert(), llvm::ISD::BUILD_PAIR, createGPRPairNode(), DL, llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getMachineNode(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetExtractSubreg(), llvm::SelectionDAG::getVTList(), llvm::Hi, llvm::DataLayout::isBigEndian(), llvm_unreachable, llvm::Lo, llvm::Monotonic, N, llvm::Release, Results, llvm::SequentiallyConsistent, llvm::SelectionDAG::setNodeMemRefs(), llvm::SelectionDAG::SplitScalar(), and std::swap().

ReplaceReductionResults()

static void ReplaceReductionResults ( SDNode *  N, SmallVectorImpl< SDValue > &  Results, SelectionDAG &  DAG, unsigned  InterOp, unsigned  AcrossOp  )

static

Definition at line 26386 of file AArch64ISelLowering.cpp.

References llvm::SelectionDAG::getNode(), llvm::SelectionDAG::GetSplitDestVTs(), llvm::Hi, llvm::Lo, N, Results, and llvm::SelectionDAG::SplitVectorOperand().

replaceSplatVectorStore()

static SDValue replaceSplatVectorStore ( SelectionDAG &  DAG, StoreSDNode &  St  )

static

Replace a splat of a scalar to a vector store by scalar stores of the scalar value.

The load store optimizer pass will merge them to store pair stores. This has better performance than a splat of the scalar followed by a split vector store. Even if the stores are not merged it is four stores vs a dup, followed by an ext.b and two stores.

Definition at line 22369 of file AArch64ISelLowering.cpp.

References llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::StoreSDNode::getValue(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::ConstantSDNode::getZExtValue(), I, llvm::ISD::INSERT_VECTOR_ELT, llvm::EVT::isFloatingPoint(), llvm::StoreSDNode::isTruncatingStore(), and splitStoreSplat().

Referenced by splitStores().

replaceZeroVectorStore()

static SDValue replaceZeroVectorStore ( SelectionDAG &  DAG, StoreSDNode &  St  )

static

Replace a splat of zeros to a vector store by scalar stores of WZR/XZR.

The load store optimizer pass will merge them to store pair stores. This should be better than a movi to create the vector zero followed by a vector store if the zero constant is not re-used, since one instructions and one register live range will be removed.

For example, the final generated code should be:

stp xzr, xzr, [x0]

instead of:

movi v0.2d, #0 str q0, [x0]

Definition at line 22302 of file AArch64ISelLowering.cpp.

References llvm::ISD::BUILD_VECTOR, DL, llvm::StoreSDNode::getBasePtr(), llvm::SDNode::getConstantOperandVal(), llvm::SelectionDAG::getCopyFromReg(), llvm::SelectionDAG::getEntryNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getSizeInBits(), llvm::StoreSDNode::getValue(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::SDValue::hasOneUse(), I, llvm::SelectionDAG::isBaseWithConstantOffset(), llvm::isNullConstant(), llvm::isNullFPConstant(), llvm::EVT::isScalableVector(), llvm::StoreSDNode::isTruncatingStore(), llvm::Offset, and splitStoreSplat().

Referenced by splitStores().

resolveBuildVector()

static bool resolveBuildVector ( BuildVectorSDNode *  BVN, APInt &  CnstBits, APInt &  UndefBits  )

static

Definition at line 13538 of file AArch64ISelLowering.cpp.

References llvm::EVT::getSizeInBits(), llvm::SDNode::getValueType(), llvm::BuildVectorSDNode::isConstantSplat(), and llvm::APInt::zextOrTrunc().

Referenced by ConstantBuildVector(), and performANDCombine().

selectUmullSmull()

static unsigned selectUmullSmull ( SDValue &  N0, SDValue &  N1, SelectionDAG &  DAG, SDLoc  DL, bool &  IsMLA  )

static

Definition at line 5263 of file AArch64ISelLowering.cpp.

References DL, llvm::APInt::getHighBitsSet(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::SelectionDAG::getSExtOrTrunc(), llvm::SDValue::getValueType(), isAddSubSExt(), isAddSubZExt(), isExtendedBUILD_VECTOR(), isSignExtended(), isZeroExtended(), llvm::SelectionDAG::MaskedValueIsZero(), llvm::SelectionDAG::SignBitIsZero(), llvm::AArch64ISD::SMULL, std::swap(), and llvm::AArch64ISD::UMULL.

setInfoSVEStN()

template<unsigned NumVecs>

static bool setInfoSVEStN ( const AArch64TargetLowering &  TLI, const DataLayout &  DL, AArch64TargetLowering::IntrinsicInfo &  Info, const CallInst &  CI  )

static

Set the IntrinsicInfo for the aarch64_sve_st<N> intrinsics.

Definition at line 15759 of file AArch64ISelLowering.cpp.

References llvm::CallBase::arg_size(), assert(), DL, llvm::CallBase::getArgOperand(), llvm::Type::getContext(), llvm::TargetLoweringBase::getMemValueType(), llvm::EVT::getScalarType(), llvm::Value::getType(), llvm::EVT::getVectorElementCount(), llvm::EVT::getVectorVT(), I, Info, llvm::ISD::INTRINSIC_VOID, and llvm::MachineMemOperand::MOStore.

shouldSinkVectorOfPtrs()

static bool shouldSinkVectorOfPtrs ( Value *  Ptrs, SmallVectorImpl< Use * > &  Ops  )

static

Definition at line 16237 of file AArch64ISelLowering.cpp.

References llvm::sampleprof::Base, GEP, and llvm::SmallVectorTemplateBase< T, bool >::push_back().

Referenced by llvm::AArch64TargetLowering::shouldSinkOperands().

shouldSinkVScale()

static bool shouldSinkVScale ( Value *  Op, SmallVectorImpl< Use * > &  Ops  )

static

We want to sink following cases: (add|sub|gep) A, ((mul|shl) vscale, imm); (add|sub|gep) A, vscale; (add|sub|gep) A, ((mul|shl) zext(vscale), imm);.

Definition at line 16265 of file AArch64ISelLowering.cpp.

References llvm::PatternMatch::m_ConstantInt(), llvm::PatternMatch::m_Mul(), llvm::PatternMatch::m_Shl(), llvm::PatternMatch::m_VScale(), llvm::PatternMatch::m_ZExt(), llvm::PatternMatch::match(), and llvm::SmallVectorTemplateBase< T, bool >::push_back().

Referenced by llvm::AArch64TargetLowering::shouldSinkOperands().

skipExtensionForVectorMULL()

static SDValue skipExtensionForVectorMULL ( SDValue  N, SelectionDAG &  DAG  )

static

Definition at line 5073 of file AArch64ISelLowering.cpp.

References addRequiredExtensionForVectorMULL(), assert(), llvm::ISD::BUILD_VECTOR, llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::APInt::getHighBitsSet(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::EVT::is128BitVector(), llvm::ISD::isExtOpcode(), llvm::SelectionDAG::MaskedValueIsZero(), N, llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::ISD::TRUNCATE, and llvm::APInt::zextOrTrunc().

splitStores()

static SDValue splitStores ( SDNode *  N, TargetLowering::DAGCombinerInfo &  DCI, SelectionDAG &  DAG, const AArch64Subtarget *  Subtarget  )

static

Definition at line 22422 of file AArch64ISelLowering.cpp.

References llvm::ISD::ADD, DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::MemSDNode::getAlign(), llvm::StoreSDNode::getBasePtr(), llvm::MemSDNode::getChain(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::MachineMemOperand::getFlags(), llvm::MachineFunction::getFunction(), llvm::EVT::getHalfNumVectorElementsVT(), llvm::SelectionDAG::getMachineFunction(), llvm::MemSDNode::getMemOperand(), llvm::SelectionDAG::getNode(), llvm::MemSDNode::getPointerInfo(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getStore(), llvm::StoreSDNode::getValue(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::Function::hasMinSize(), llvm::EVT::isFixedLengthVector(), llvm::LSBaseSDNode::isIndexed(), llvm::MemSDNode::isVolatile(), N, replaceSplatVectorStore(), and replaceZeroVectorStore().

Referenced by performSTORECombine().

splitStoreSplat()

static SDValue splitStoreSplat ( SelectionDAG &  DAG, StoreSDNode &  St, SDValue  SplatVal, unsigned  NumVecElts  )

static

Definition at line 22101 of file AArch64ISelLowering.cpp.

References llvm::ISD::ADD, assert(), llvm::commonAlignment(), DL, llvm::MemSDNode::getAlign(), llvm::StoreSDNode::getBasePtr(), llvm::MemSDNode::getChain(), llvm::SelectionDAG::getConstant(), llvm::MachineMemOperand::getFlags(), llvm::MemSDNode::getMemOperand(), llvm::SelectionDAG::getNode(), llvm::MemSDNode::getPointerInfo(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getStore(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::MachinePointerInfo::getWithOffset(), llvm::StoreSDNode::isTruncatingStore(), and llvm::Offset.

Referenced by replaceSplatVectorStore(), and replaceZeroVectorStore().

STATISTIC() [1/3]

STATISTIC	(	NumOptimizedImms	,
		"Number of times immediates were optimized"
	)

STATISTIC() [2/3]

STATISTIC	(	NumShiftInserts	,
		"Number of vector shift inserts"
	)

STATISTIC() [3/3]

STATISTIC	(	NumTailCalls	,
		"Number of tail calls"
	)

tryAdvSIMDModImm16()

static SDValue tryAdvSIMDModImm16 ( unsigned  NewOp, SDValue  Op, SelectionDAG &  DAG, const APInt &  Bits, const SDValue *  LHS = nullptr  )

static

Definition at line 13635 of file AArch64ISelLowering.cpp.

References llvm::AArch64_AM::encodeAdvSIMDModImmType5(), llvm::AArch64_AM::encodeAdvSIMDModImmType6(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getSubtarget(), llvm::AArch64_AM::isAdvSIMDModImmType5(), llvm::AArch64_AM::isAdvSIMDModImmType6(), llvm::EVT::isFixedLengthVector(), llvm::AArch64Subtarget::isNeonAvailable(), LHS, and llvm::AArch64ISD::NVCAST.

Referenced by ConstantBuildVector(), and performANDCombine().

tryAdvSIMDModImm32()

static SDValue tryAdvSIMDModImm32 ( unsigned  NewOp, SDValue  Op, SelectionDAG &  DAG, const APInt &  Bits, const SDValue *  LHS = nullptr  )

static

Definition at line 13582 of file AArch64ISelLowering.cpp.

References llvm::AArch64_AM::encodeAdvSIMDModImmType1(), llvm::AArch64_AM::encodeAdvSIMDModImmType2(), llvm::AArch64_AM::encodeAdvSIMDModImmType3(), llvm::AArch64_AM::encodeAdvSIMDModImmType4(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getSubtarget(), llvm::AArch64_AM::isAdvSIMDModImmType1(), llvm::AArch64_AM::isAdvSIMDModImmType2(), llvm::AArch64_AM::isAdvSIMDModImmType3(), llvm::AArch64_AM::isAdvSIMDModImmType4(), llvm::EVT::isFixedLengthVector(), llvm::AArch64Subtarget::isNeonAvailable(), LHS, and llvm::AArch64ISD::NVCAST.

Referenced by ConstantBuildVector(), and performANDCombine().

tryAdvSIMDModImm321s()

static SDValue tryAdvSIMDModImm321s ( unsigned  NewOp, SDValue  Op, SelectionDAG &  DAG, const APInt &  Bits  )

static

Definition at line 13680 of file AArch64ISelLowering.cpp.

References llvm::AArch64_AM::encodeAdvSIMDModImmType7(), llvm::AArch64_AM::encodeAdvSIMDModImmType8(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::AArch64_AM::isAdvSIMDModImmType7(), llvm::AArch64_AM::isAdvSIMDModImmType8(), and llvm::AArch64ISD::NVCAST.

Referenced by ConstantBuildVector().

tryAdvSIMDModImm64()

static SDValue tryAdvSIMDModImm64 ( unsigned  NewOp, SDValue  Op, SelectionDAG &  DAG, const APInt &  Bits  )

static

Definition at line 13561 of file AArch64ISelLowering.cpp.

References llvm::AArch64_AM::encodeAdvSIMDModImmType10(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::AArch64_AM::isAdvSIMDModImmType10(), and llvm::AArch64ISD::NVCAST.

Referenced by ConstantBuildVector().

tryAdvSIMDModImm8()

static SDValue tryAdvSIMDModImm8 ( unsigned  NewOp, SDValue  Op, SelectionDAG &  DAG, const APInt &  Bits  )

static

Definition at line 13711 of file AArch64ISelLowering.cpp.

References llvm::AArch64_AM::encodeAdvSIMDModImmType9(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::AArch64_AM::isAdvSIMDModImmType9(), and llvm::AArch64ISD::NVCAST.

Referenced by ConstantBuildVector().

tryAdvSIMDModImmFP()

static SDValue tryAdvSIMDModImmFP ( unsigned  NewOp, SDValue  Op, SelectionDAG &  DAG, const APInt &  Bits  )

static

Definition at line 13732 of file AArch64ISelLowering.cpp.

References llvm::AArch64_AM::encodeAdvSIMDModImmType11(), llvm::AArch64_AM::encodeAdvSIMDModImmType12(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::AArch64_AM::isAdvSIMDModImmType11(), llvm::AArch64_AM::isAdvSIMDModImmType12(), and llvm::AArch64ISD::NVCAST.

Referenced by ConstantBuildVector().

tryCombineCRC32()

static SDValue tryCombineCRC32 ( unsigned  Mask, SDNode *  N, SelectionDAG &  DAG  )

static

Definition at line 21400 of file AArch64ISelLowering.cpp.

References llvm::ISD::AND, llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::ConstantSDNode::getZExtValue(), llvm::ISD::INTRINSIC_WO_CHAIN, and N.

Referenced by performIntrinsicCombine().

tryCombineExtendRShTrunc()

static SDValue tryCombineExtendRShTrunc ( SDNode *  N, SelectionDAG &  DAG  )

static

Definition at line 22566 of file AArch64ISelLowering.cpp.

References assert(), DL, llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), getPredicateForVector(), llvm::Hi, llvm::Lo, N, llvm::AArch64ISD::RSHRNB_I, llvm::AArch64ISD::URSHR_I_PRED, llvm::AArch64ISD::UUNPKHI, llvm::AArch64ISD::UUNPKLO, and llvm::AArch64ISD::UZP1.

Referenced by performUzpCombine().

tryCombineFixedPointConvert()

static SDValue tryCombineFixedPointConvert ( SDNode *  N, TargetLowering::DAGCombinerInfo &  DCI, SelectionDAG &  DAG  )

static

Definition at line 20082 of file AArch64ISelLowering.cpp.

References DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::ISD::INTRINSIC_WO_CHAIN, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), and N.

Referenced by performIntrinsicCombine().

tryCombineLongOpWithDup()

static SDValue tryCombineLongOpWithDup ( unsigned  IID, SDNode *  N, TargetLowering::DAGCombinerInfo &  DCI, SelectionDAG &  DAG  )

static

Definition at line 21273 of file AArch64ISelLowering.cpp.

References assert(), llvm::SelectionDAG::getNode(), llvm::ISD::INTRINSIC_WO_CHAIN, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), isEssentiallyExtractHighSubvector(), LHS, N, llvm::Intrinsic::not_intrinsic, RHS, and tryExtendDUPToExtractHigh().

Referenced by performExtendCombine(), performIntrinsicCombine(), and performMULLCombine().

tryCombineMULLWithUZP1()

static SDValue tryCombineMULLWithUZP1 ( SDNode *  N, TargetLowering::DAGCombinerInfo &  DCI, SelectionDAG &  DAG  )

static

Definition at line 25546 of file AArch64ISelLowering.cpp.

References llvm::ISD::BITCAST, DL, llvm::AArch64ISD::DUP, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::EVT::getDoubleNumVectorElementsVT(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::User::getOperand(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::SDNode::hasOneUse(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), isEssentiallyExtractHighSubvector(), llvm::isNullConstant(), llvm::SelectionDAG::isSplatValue(), LHS, N, llvm::SelectionDAG::ReplaceAllUsesWith(), RHS, llvm::SDValue::setNode(), llvm::ISD::TRUNCATE, llvm::SDNode::use_begin(), llvm::SDNode::use_size(), llvm::SDNode::uses(), and llvm::AArch64ISD::UZP1.

Referenced by performMULLCombine().

tryCombineShiftImm()

static SDValue tryCombineShiftImm ( unsigned  IID, SDNode *  N, SelectionDAG &  DAG  )

static

Definition at line 21306 of file AArch64ISelLowering.cpp.

References llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::APInt::getSExtValue(), llvm::MVT::getSizeInBits(), llvm::BuildVectorSDNode::isConstantSplat(), llvm_unreachable, N, llvm::ISD::SCALAR_TO_VECTOR, llvm::AArch64ISD::SQSHL_I, llvm::AArch64ISD::SQSHLU_I, llvm::AArch64ISD::SRSHR_I, llvm::AArch64ISD::UQSHL_I, llvm::AArch64ISD::URSHR_I, llvm::AArch64ISD::VASHR, llvm::AArch64ISD::VLSHR, and llvm::AArch64ISD::VSHL.

Referenced by performIntrinsicCombine().

tryCombineToBSL()

static SDValue tryCombineToBSL ( SDNode *  N, TargetLowering::DAGCombinerInfo &  DCI, const AArch64TargetLowering &  TLI  )

static

Definition at line 19103 of file AArch64ISelLowering.cpp.

References llvm::Add, llvm::ISD::ADD, llvm::ISD::AND, llvm::AArch64ISD::BSP, llvm::TargetLowering::DAGCombinerInfo::DAG, DL, llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::SelectionDAG::getSubtarget(), llvm::EVT::getVectorNumElements(), llvm::APInt::getZExtValue(), llvm::ConstantSDNode::getZExtValue(), llvm::ISD::isConstantSplatVector(), llvm::ISD::isConstantSplatVectorAllOnes(), llvm::ISD::isConstantSplatVectorAllZeros(), llvm::EVT::isFixedLengthVector(), llvm::EVT::isScalableVector(), llvm::EVT::isVector(), N, llvm::ISD::SUB, and llvm::AArch64TargetLowering::useSVEForFixedLengthVectorVT().

Referenced by performORCombine().

tryCombineWhileLo()

static SDValue tryCombineWhileLo ( SDNode *  N, TargetLowering::DAGCombinerInfo &  DCI, const AArch64Subtarget *  Subtarget  )

static

Definition at line 21664 of file AArch64ISelLowering.cpp.

References llvm::TargetLowering::DAGCombinerInfo::CombineTo(), llvm::TargetLowering::DAGCombinerInfo::DAG, DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getNode(), llvm::ElementCount::getScalable(), llvm::SelectionDAG::getTargetConstant(), llvm::EVT::getVectorElementCount(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::Hi, Idx, llvm::ISD::INTRINSIC_WO_CHAIN, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::Lo, N, and std::swap().

Referenced by performIntrinsicCombine().

tryConvertSVEWideCompare()

static SDValue tryConvertSVEWideCompare ( SDNode *  N, ISD::CondCode  CC, TargetLowering::DAGCombinerInfo &  DCI, SelectionDAG &  DAG  )

static

Definition at line 21479 of file AArch64ISelLowering.cpp.

References CC, DL, llvm::AArch64ISD::DUP, llvm::SelectionDAG::getCondCode(), llvm::SelectionDAG::getConstant(), getIntrinsicID(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getSignedConstant(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm_unreachable, N, llvm::AArch64ISD::SETCC_MERGE_ZERO, llvm::Splat, and llvm::ISD::SPLAT_VECTOR.

Referenced by performIntrinsicCombine().

tryExtendDUPToExtractHigh()

static SDValue tryExtendDUPToExtractHigh ( SDValue  N, SelectionDAG &  DAG  )

static

Definition at line 20145 of file AArch64ISelLowering.cpp.

References DL, llvm::AArch64ISD::DUP, llvm::AArch64ISD::DUPLANE16, llvm::AArch64ISD::DUPLANE32, llvm::AArch64ISD::DUPLANE64, llvm::AArch64ISD::DUPLANE8, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::MVT::is64BitVector(), llvm::AArch64ISD::MOVI, llvm::AArch64ISD::MOVIedit, llvm::AArch64ISD::MOVImsl, llvm::AArch64ISD::MOVIshift, llvm::AArch64ISD::MVNImsl, llvm::AArch64ISD::MVNIshift, and N.

Referenced by performAddSubLongCombine(), and tryCombineLongOpWithDup().

tryFormConcatFromShuffle()

static SDValue tryFormConcatFromShuffle ( SDValue  Op, SelectionDAG &  DAG  )

static

Definition at line 12799 of file AArch64ISelLowering.cpp.

References llvm::ISD::CONCAT_VECTORS, DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::EVT::getHalfNumVectorElementsVT(), llvm::SelectionDAG::getNode(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), and isConcatMask().

tryGetOriginalBoolVectorType()

static EVT tryGetOriginalBoolVectorType ( SDValue  Op, int  Depth = 0  )

static

Definition at line 23210 of file AArch64ISelLowering.cpp.

References assert(), llvm::Depth, llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::MVT::INVALID_SIMPLE_VALUE_TYPE, llvm::EVT::isSimple(), llvm::EVT::isVector(), llvm::ISD::SETCC, llvm::ISD::TRUNCATE, and tryGetOriginalBoolVectorType().

Referenced by tryGetOriginalBoolVectorType(), and vectorToScalarBitmask().

tryLowerToSLI()

static SDValue tryLowerToSLI ( SDNode *  N, SelectionDAG &  DAG  )

static

Definition at line 13835 of file AArch64ISelLowering.cpp.

References llvm::And, llvm::ISD::AND, assert(), llvm::AArch64ISD::BICi, llvm::CallingConv::C, llvm::dbgs(), DL, llvm::SDNode::dump(), llvm::ConstantSDNode::getAPIntValue(), llvm::APInt::getHighBitsSet(), llvm::APInt::getLowBitsSet(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::SelectionDAG::getTargetConstant(), isAllActivePredicate(), llvm::ISD::isConstantSplatVector(), llvm::EVT::isVector(), LLVM_DEBUG, N, llvm::AArch64ISD::SHL_PRED, llvm::AArch64ISD::SRL_PRED, llvm::AArch64ISD::VLSHR, llvm::AArch64ISD::VSHL, llvm::AArch64ISD::VSLI, llvm::AArch64ISD::VSRI, X, Y, and llvm::APInt::zextOrTrunc().

trySimplifySrlAddToRshrnb()

static SDValue trySimplifySrlAddToRshrnb ( SDValue  Srl, SelectionDAG &  DAG, const AArch64Subtarget *  Subtarget  )

static

Definition at line 22606 of file AArch64ISelLowering.cpp.

References llvm::ISD::BITCAST, canLowerSRLToRoundingShiftForVT(), DL, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetConstant(), llvm::SDNode::getValueType(), llvm::EVT::isScalableVector(), and llvm::AArch64ISD::RSHRNB_I.

Referenced by performMSTORECombine(), performSTORECombine(), and performUzpCombine().

trySwapVSelectOperands()

static SDValue trySwapVSelectOperands ( SDNode *  N, SelectionDAG &  DAG  )

static

Definition at line 24651 of file AArch64ISelLowering.cpp.

References CC, llvm::ISD::FADD, llvm::ISD::FMUL, llvm::ISD::FSUB, llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getSetCC(), llvm::ISD::getSetCCInverse(), llvm::SDValue::getValueType(), llvm::SDValue::hasOneUse(), N, llvm::ISD::SETCC, and llvm::ISD::VSELECT.

Referenced by performVSelectCombine().

tryToConvertShuffleOfTbl2ToTbl4()

static SDValue tryToConvertShuffleOfTbl2ToTbl4 ( SDValue  Op, ArrayRef< int >  ShuffleMask, SelectionDAG &  DAG  )

static

Definition at line 13220 of file AArch64ISelLowering.cpp.

References llvm::CallingConv::C, llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getTargetConstant(), llvm::SDNode::getValueType(), I, and llvm::ISD::INTRINSIC_WO_CHAIN.

tryToWidenSetCCOperands()

static SDValue tryToWidenSetCCOperands ( SDNode *  Op, SelectionDAG &  DAG  )

static

Definition at line 24302 of file AArch64ISelLowering.cpp.

References llvm::any_of(), CC, llvm::EVT::changeVectorElementType(), DL, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNodeIfExists(), llvm::SDNode::getOpcode(), llvm::EVT::getScalarSizeInBits(), llvm::SDNode::getValueType(), llvm::SelectionDAG::getVTList(), llvm::ISD::isConstantSplatVector(), llvm::EVT::isVector(), N, llvm::ISD::SETCC, llvm::ISD::SIGN_EXTEND, llvm::ISD::VSELECT, and llvm::ISD::ZERO_EXTEND.

Referenced by performSETCCCombine().

tryWhileWRFromOR()

SDValue tryWhileWRFromOR	(	SDValue	Op,
		SelectionDAG &	DAG,
		const AArch64Subtarget &	Subtarget
	)

Try to lower the construction of a pointer alias mask to a WHILEWR.

The mask's enabled lanes represent the elements that will not overlap across one loop iteration. This tries to match: or (splat (setcc_lt (sub ptrA, ptrB), -(element_size - 1))), (get_active_lane_mask 0, (div (sub ptrA, ptrB), element_size))

Definition at line 13941 of file AArch64ISelLowering.cpp.

References llvm::Add, llvm::ISD::ADD, Cond, DL, llvm::SelectionDAG::getConstant(), llvm::SDValue::getConstantOperandVal(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValueType(), llvm::ISD::INTRINSIC_WO_CHAIN, llvm::isNullConstant(), llvm::isPowerOf2_64(), llvm::Log2_64(), Select, llvm::ISD::SELECT_CC, llvm::ISD::SETCC, llvm::ISD::SETLT, llvm::Splat, llvm::ISD::SPLAT_VECTOR, llvm::ISD::SRA, llvm::ISD::SRL, llvm::ISD::SUB, and std::swap().

tryWidenMaskForShuffle()

static SDValue tryWidenMaskForShuffle ( SDValue  Op, SelectionDAG &  DAG  )

static

Definition at line 13187 of file AArch64ISelLowering.cpp.

References DL, llvm::SelectionDAG::getBitcast(), llvm::EVT::getFixedSizeInBits(), llvm::MVT::getFloatingPointVT(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), llvm::EVT::isFloatingPoint(), llvm::TargetLoweringBase::isTypeLegal(), and isWideTypeMask().

UseTlsOffset()

static Value * UseTlsOffset ( IRBuilderBase &  IRB, unsigned  Offset  )

static

Definition at line 27240 of file AArch64ISelLowering.cpp.

References llvm::IRBuilderBase::CreateCall(), llvm::IRBuilderBase::CreateConstGEP1_32(), llvm::IRBuilderBase::CreatePointerCast(), llvm::Intrinsic::getDeclaration(), llvm::IRBuilderBase::GetInsertBlock(), llvm::IRBuilderBase::getInt8Ty(), llvm::GlobalValue::getParent(), llvm::BasicBlock::getParent(), llvm::IRBuilderBase::getPtrTy(), and llvm::Offset.

Referenced by llvm::AArch64TargetLowering::getIRStackGuard(), and llvm::AArch64TargetLowering::getSafeStackPointerLocation().

valueToCarryFlag()

static SDValue valueToCarryFlag ( SDValue  Value, SelectionDAG &  DAG, bool  Invert  )

static

Definition at line 4173 of file AArch64ISelLowering.cpp.

References DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getVTList(), and llvm::AArch64ISD::SUBS.

Referenced by lowerADDSUBO_CARRY().

vectorToScalarBitmask()

static SDValue vectorToScalarBitmask ( SDNode *  N, SelectionDAG &  DAG  )

static

Definition at line 23242 of file AArch64ISelLowering.cpp.

References llvm::ISD::AND, assert(), llvm::ISD::BITCAST, llvm::ISD::BUILD_VECTOR, llvm::EVT::changeVectorElementTypeToInteger(), DL, llvm::AArch64ISD::EXT, llvm::SelectionDAG::getConstant(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getSExtOrTrunc(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getSubtarget(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::AArch64Subtarget::isNeonAvailable(), llvm::EVT::isSimple(), llvm::TargetLoweringBase::isTypeLegal(), llvm::EVT::isVector(), N, llvm::SmallVectorTemplateBase< T, bool >::push_back(), tryGetOriginalBoolVectorType(), llvm::ISD::VECREDUCE_ADD, and llvm::AArch64ISD::ZIP1.

Referenced by combineBoolVectorAndTruncateStore(), and replaceBoolVectorBitcast().

WidenVector()

static SDValue WidenVector ( SDValue  V64Reg, SelectionDAG &  DAG  )

static

WidenVector - Given a value in the V64 register class, produce the equivalent value in the V128 register class.

Definition at line 12080 of file AArch64ISelLowering.cpp.

References DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::EVT::getSimpleVT(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::ISD::INSERT_SUBVECTOR, and WidenVector().

Referenced by constructDup(), GeneratePerfectShuffle(), performConcatVectorsCombine(), and WidenVector().

Variable Documentation

EnableAArch64ELFLocalDynamicTLSGeneration

cl::opt< bool > EnableAArch64ELFLocalDynamicTLSGeneration("aarch64-elf-ldtls-generation", cl::Hidden, cl::desc("Allow AArch64 Local Dynamic TLS code generation"), cl::init(false))	(	"aarch64-elf-ldtls-generation"	,
		cl::Hidden	,
		cl::desc("Allow AArch64 Local Dynamic TLS code generation")	,
		cl::init(false)
	)

Referenced by llvm::AArch64MCInstLower::lowerSymbolOperandELF().

EnableCombineMGatherIntrinsics

cl::opt< bool > EnableCombineMGatherIntrinsics("aarch64-enable-mgather-combine", cl::Hidden, cl::desc("Combine extends of AArch64 masked " "gather intrinsics"), cl::init(true)) ( "aarch64-enable-mgather-combine"  , cl::Hidden  , cl::desc("Combine extends of AArch64 masked " "gather intrinsics")  , cl::init(true)    )

static

Referenced by performSignExtendInRegCombine(), and performSVEAndCombine().

EnableExtToTBL

cl::opt< bool > EnableExtToTBL("aarch64-enable-ext-to-tbl", cl::Hidden, cl::desc("Combine ext and trunc to TBL"), cl::init(true)) ( "aarch64-enable-ext-to-tbl"  , cl::Hidden  , cl::desc("Combine ext and trunc to TBL")  , cl::init(true)    )

static

Referenced by llvm::AArch64TargetLowering::optimizeExtendOrTruncateConversion().

EnableOptimizeLogicalImm

cl::opt< bool > EnableOptimizeLogicalImm("aarch64-enable-logical-imm", cl::Hidden, cl::desc("Enable AArch64 logical imm instruction " "optimization"), cl::init(true)) ( "aarch64-enable-logical-imm"  , cl::Hidden  , cl::desc("Enable AArch64 logical imm instruction " "optimization")  , cl::init(true)    )

static

Referenced by llvm::AArch64TargetLowering::targetShrinkDemandedConstant().

EnableSVEGISel

cl::opt< bool > EnableSVEGISel("aarch64-enable-gisel-sve", cl::Hidden, cl::desc("Enable / disable SVE scalable vectors in Global ISel"), cl::init(false))	(	"aarch64-enable-gisel-sve"	,
		cl::Hidden	,
		cl::desc("Enable / disable SVE scalable vectors in Global ISel")	,
		cl::init(false)
	)

Referenced by llvm::AArch64TargetLowering::fallBackToDAGISel(), and llvm::AArch64CallLowering::fallBackToDAGISel().

FPRArgRegs

const MCPhysReg FPRArgRegs[]

static

Initial value:

= {AArch64::Q0, AArch64::Q1, AArch64::Q2,
                                       AArch64::Q3, AArch64::Q4, AArch64::Q5,
                                       AArch64::Q6, AArch64::Q7}

Definition at line 164 of file AArch64ISelLowering.cpp.

Referenced by llvm::AArch64::getFPRArgRegs().

GPRArgRegs

const MCPhysReg GPRArgRegs[]

static

Initial value:

= {AArch64::X0, AArch64::X1, AArch64::X2,
                                       AArch64::X3, AArch64::X4, AArch64::X5,
                                       AArch64::X6, AArch64::X7}

Definition at line 161 of file AArch64ISelLowering.cpp.

Referenced by f64AssignAAPCS(), and llvm::AArch64::getGPRArgRegs().

MaxXors

cl::opt< unsigned > MaxXors("aarch64-max-xors", cl::init(16), cl::Hidden, cl::desc("Maximum of xors")) ( "aarch64-max-xors"  , cl::init(16)  , cl::Hidden  , cl::desc("Maximum of xors")    )

static

Referenced by isOrXorChain().

MVT_CC

const MVT MVT_CC = MVT::i32

static

Value type used for condition codes.

Definition at line 159 of file AArch64ISelLowering.cpp.

Referenced by emitComparison(), emitConditionalComparison(), getAArch64Cmp(), getAArch64XALUOOp(), and performANDORCSELCombine().