|
LLVM
16.0.0git
|
#include "AArch64ISelLowering.h"#include "AArch64CallingConvention.h"#include "AArch64ExpandImm.h"#include "AArch64MachineFunctionInfo.h"#include "AArch64PerfectShuffle.h"#include "AArch64RegisterInfo.h"#include "AArch64Subtarget.h"#include "MCTargetDesc/AArch64AddressingModes.h"#include "Utils/AArch64BaseInfo.h"#include "llvm/ADT/APFloat.h"#include "llvm/ADT/APInt.h"#include "llvm/ADT/ArrayRef.h"#include "llvm/ADT/STLExtras.h"#include "llvm/ADT/SmallSet.h"#include "llvm/ADT/SmallVector.h"#include "llvm/ADT/Statistic.h"#include "llvm/ADT/StringRef.h"#include "llvm/ADT/Triple.h"#include "llvm/ADT/Twine.h"#include "llvm/Analysis/LoopInfo.h"#include "llvm/Analysis/MemoryLocation.h"#include "llvm/Analysis/ObjCARCUtil.h"#include "llvm/Analysis/TargetTransformInfo.h"#include "llvm/Analysis/VectorUtils.h"#include "llvm/CodeGen/Analysis.h"#include "llvm/CodeGen/CallingConvLower.h"#include "llvm/CodeGen/ISDOpcodes.h"#include "llvm/CodeGen/MachineBasicBlock.h"#include "llvm/CodeGen/MachineFrameInfo.h"#include "llvm/CodeGen/MachineFunction.h"#include "llvm/CodeGen/MachineInstr.h"#include "llvm/CodeGen/MachineInstrBuilder.h"#include "llvm/CodeGen/MachineMemOperand.h"#include "llvm/CodeGen/MachineRegisterInfo.h"#include "llvm/CodeGen/RuntimeLibcalls.h"#include "llvm/CodeGen/SelectionDAG.h"#include "llvm/CodeGen/SelectionDAGNodes.h"#include "llvm/CodeGen/TargetCallingConv.h"#include "llvm/CodeGen/TargetInstrInfo.h"#include "llvm/CodeGen/ValueTypes.h"#include "llvm/IR/Attributes.h"#include "llvm/IR/Constants.h"#include "llvm/IR/DataLayout.h"#include "llvm/IR/DebugLoc.h"#include "llvm/IR/DerivedTypes.h"#include "llvm/IR/Function.h"#include "llvm/IR/GetElementPtrTypeIterator.h"#include "llvm/IR/GlobalValue.h"#include "llvm/IR/IRBuilder.h"#include "llvm/IR/Instruction.h"#include "llvm/IR/Instructions.h"#include "llvm/IR/IntrinsicInst.h"#include "llvm/IR/Intrinsics.h"#include "llvm/IR/IntrinsicsAArch64.h"#include "llvm/IR/Module.h"#include "llvm/IR/OperandTraits.h"#include "llvm/IR/PatternMatch.h"#include "llvm/IR/Type.h"#include "llvm/IR/Use.h"#include "llvm/IR/Value.h"#include "llvm/MC/MCRegisterInfo.h"#include "llvm/Support/Casting.h"#include "llvm/Support/CodeGen.h"#include "llvm/Support/CommandLine.h"#include "llvm/Support/Compiler.h"#include "llvm/Support/Debug.h"#include "llvm/Support/ErrorHandling.h"#include "llvm/Support/InstructionCost.h"#include "llvm/Support/KnownBits.h"#include "llvm/Support/MachineValueType.h"#include "llvm/Support/MathExtras.h"#include "llvm/Support/raw_ostream.h"#include "llvm/Target/TargetMachine.h"#include "llvm/Target/TargetOptions.h"#include <algorithm>#include <bitset>#include <cassert>#include <cctype>#include <cstdint>#include <cstdlib>#include <iterator>#include <limits>#include <optional>#include <tuple>#include <utility>#include <vector>#include "AArch64GenAsmMatcher.inc"Go to the source code of this file.
Classes | |
| struct | GenericSetCCInfo |
| Helper structure to keep track of ISD::SET_CC operands. More... | |
| struct | AArch64SetCCInfo |
| Helper structure to keep track of a SET_CC lowered into AArch64 code. More... | |
| union | SetCCInfo |
| Helper structure to keep track of SetCC information. More... | |
| struct | SetCCInfoAndKind |
| Helper structure to be able to read SetCC information. More... | |
Macros | |
| #define | DEBUG_TYPE "aarch64-lower" |
| #define | LCALLNAMES(A, B, N) |
| #define | LCALLNAME4(A, B) |
| #define | LCALLNAME5(A, B) |
| #define | MAKE_CASE(V) |
| #define | GET_REGISTER_MATCHER |
Enumerations | |
| enum | PredicateConstraint { Upl, Upa, Invalid } |
Functions | |
| STATISTIC (NumTailCalls, "Number of tail calls") | |
| STATISTIC (NumShiftInserts, "Number of vector shift inserts") | |
| STATISTIC (NumOptimizedImms, "Number of times immediates were optimized") | |
| static EVT | getPackedSVEVectorVT (EVT VT) |
| static EVT | getPackedSVEVectorVT (ElementCount EC) |
| static EVT | getPromotedVTForPredicate (EVT VT) |
| static bool | isPackedVectorType (EVT VT, SelectionDAG &DAG) |
| Returns true if VT's elements occupy the lowest bit positions of its associated register class without any intervening space. More... | |
| static bool | isMergePassthruOpcode (unsigned Opc) |
| static bool | isZeroingInactiveLanes (SDValue Op) |
| static bool | isIntImmediate (const SDNode *N, uint64_t &Imm) |
| static bool | isOpcWithIntImmediate (const SDNode *N, unsigned Opc, uint64_t &Imm) |
| static bool | optimizeLogicalImm (SDValue Op, unsigned Size, uint64_t Imm, const APInt &Demanded, TargetLowering::TargetLoweringOpt &TLO, unsigned NewOpc) |
| static EVT | getContainerForFixedLengthVector (SelectionDAG &DAG, EVT VT) |
| static SDValue | convertToScalableVector (SelectionDAG &DAG, EVT VT, SDValue V) |
| static SDValue | convertFromScalableVector (SelectionDAG &DAG, EVT VT, SDValue V) |
| static SDValue | convertFixedMaskToScalableVector (SDValue Mask, SelectionDAG &DAG) |
| static SDValue | getPredicateForScalableVector (SelectionDAG &DAG, SDLoc &DL, EVT VT) |
| static bool | isZerosVector (const SDNode *N) |
| isZerosVector - Check whether SDNode N is a zero-filled vector. More... | |
| static AArch64CC::CondCode | changeIntCCToAArch64CC (ISD::CondCode CC) |
| changeIntCCToAArch64CC - Convert a DAG integer condition code to an AArch64 CC More... | |
| static void | changeFPCCToAArch64CC (ISD::CondCode CC, AArch64CC::CondCode &CondCode, AArch64CC::CondCode &CondCode2) |
| changeFPCCToAArch64CC - Convert a DAG fp condition code to an AArch64 CC. More... | |
| static void | changeFPCCToANDAArch64CC (ISD::CondCode CC, AArch64CC::CondCode &CondCode, AArch64CC::CondCode &CondCode2) |
| Convert a DAG fp condition code to an AArch64 CC. More... | |
| static void | changeVectorFPCCToAArch64CC (ISD::CondCode CC, AArch64CC::CondCode &CondCode, AArch64CC::CondCode &CondCode2, bool &Invert) |
| changeVectorFPCCToAArch64CC - Convert a DAG fp condition code to an AArch64 CC usable with the vector instructions. More... | |
| static bool | isLegalArithImmed (uint64_t C) |
| static bool | isCMN (SDValue Op, ISD::CondCode CC) |
| static SDValue | emitStrictFPComparison (SDValue LHS, SDValue RHS, const SDLoc &dl, SelectionDAG &DAG, SDValue Chain, bool IsSignaling) |
| static SDValue | emitComparison (SDValue LHS, SDValue RHS, ISD::CondCode CC, const SDLoc &dl, SelectionDAG &DAG) |
| static unsigned | getCmpOperandFoldingProfit (SDValue Op) |
| Returns how profitable it is to fold a comparison's operand's shift and/or extension operations. More... | |
| static SDValue | getAArch64Cmp (SDValue LHS, SDValue RHS, ISD::CondCode CC, SDValue &AArch64cc, SelectionDAG &DAG, const SDLoc &dl) |
| static std::pair< SDValue, SDValue > | getAArch64XALUOOp (AArch64CC::CondCode &CC, SDValue Op, SelectionDAG &DAG) |
| static SDValue | valueToCarryFlag (SDValue Value, SelectionDAG &DAG, bool Invert) |
| static SDValue | carryFlagToValue (SDValue Flag, EVT VT, SelectionDAG &DAG, bool Invert) |
| static SDValue | overflowFlagToValue (SDValue Flag, EVT VT, SelectionDAG &DAG) |
| static SDValue | lowerADDSUBCARRY (SDValue Op, SelectionDAG &DAG, unsigned Opcode, bool IsSigned) |
| static SDValue | LowerXALUO (SDValue Op, SelectionDAG &DAG) |
| static SDValue | LowerPREFETCH (SDValue Op, SelectionDAG &DAG) |
| static MVT | getSVEContainerType (EVT ContentTy) |
| static EVT | getExtensionTo64Bits (const EVT &OrigVT) |
| static SDValue | addRequiredExtensionForVectorMULL (SDValue N, SelectionDAG &DAG, const EVT &OrigTy, const EVT &ExtTy, unsigned ExtOpcode) |
| static Optional< uint64_t > | getConstantLaneNumOfExtractHalfOperand (SDValue &Op) |
| static bool | isExtendedBUILD_VECTOR (SDNode *N, SelectionDAG &DAG, bool isSigned) |
| static SDValue | skipExtensionForVectorMULL (SDNode *N, SelectionDAG &DAG) |
| static bool | isSignExtended (SDNode *N, SelectionDAG &DAG) |
| static bool | isZeroExtended (SDNode *N, SelectionDAG &DAG) |
| static bool | isAddSubSExt (SDNode *N, SelectionDAG &DAG) |
| static bool | isAddSubZExt (SDNode *N, SelectionDAG &DAG) |
| static unsigned | selectUmullSmull (SDNode *&N0, SDNode *&N1, SelectionDAG &DAG, bool &IsMLA) |
| static SDValue | getPTrue (SelectionDAG &DAG, SDLoc DL, EVT VT, int Pattern) |
| static SDValue | getSVEPredicateBitCast (EVT VT, SDValue Op, SelectionDAG &DAG) |
| static Optional< SMEAttrs > | getCalleeAttrsFromExternalFunction (SDValue V) |
| unsigned | getGatherVecOpcode (bool IsScaled, bool IsSigned, bool NeedsExtend) |
| unsigned | getSignExtendedGatherOpcode (unsigned Opcode) |
| static SDValue | LowerTruncateVectorStore (SDLoc DL, StoreSDNode *ST, EVT VT, EVT MemVT, SelectionDAG &DAG) |
| static SDValue | LowerBRCOND (SDValue Op, SelectionDAG &DAG) |
| static unsigned | getIntrinsicID (const SDNode *N) |
| static bool | canGuaranteeTCO (CallingConv::ID CC, bool GuaranteeTailCalls) |
| Return true if the calling convention is one that we can guarantee TCO for. More... | |
| static bool | mayTailCallThisCC (CallingConv::ID CC) |
| Return true if we might ever do TCO for calls with this calling convention. More... | |
| static void | analyzeCallOperands (const AArch64TargetLowering &TLI, const AArch64Subtarget *Subtarget, const TargetLowering::CallLoweringInfo &CLI, CCState &CCInfo) |
| static bool | checkZExtBool (SDValue Arg, const SelectionDAG &DAG) |
| std::pair< SDValue, uint64_t > | lookThroughSignExtension (SDValue Val) |
| static bool | isOrXorChain (SDValue N, unsigned &Num, SmallVector< std::pair< SDValue, SDValue >, 16 > &WorkList) |
| static SDValue | performOrXorChainCombine (SDNode *N, SelectionDAG &DAG) |
| static SDValue | getEstimate (const AArch64Subtarget *ST, unsigned Opcode, SDValue Operand, SelectionDAG &DAG, int &ExtraSteps) |
| static PredicateConstraint | parsePredicateConstraint (StringRef Constraint) |
| static SDValue | WidenVector (SDValue V64Reg, SelectionDAG &DAG) |
| WidenVector - Given a value in the V64 register class, produce the equivalent value in the V128 register class. More... | |
| static unsigned | getExtFactor (SDValue &V) |
| getExtFactor - Determine the adjustment factor for the position when generating an "extract from vector registers" instruction. More... | |
| static SDValue | NarrowVector (SDValue V128Reg, SelectionDAG &DAG) |
| NarrowVector - Given a value in the V128 register class, produce the equivalent value in the V64 register class. More... | |
| static bool | isSingletonEXTMask (ArrayRef< int > M, EVT VT, unsigned &Imm) |
| static SDValue | ReconstructTruncateFromBuildVector (SDValue V, SelectionDAG &DAG) |
| static bool | isWideDUPMask (ArrayRef< int > M, EVT VT, unsigned BlockSize, unsigned &DupLaneOp) |
| Check if a vector shuffle corresponds to a DUP instructions with a larger element width than the vector lane type. More... | |
| static bool | isEXTMask (ArrayRef< int > M, EVT VT, bool &ReverseEXT, unsigned &Imm) |
| static bool | isREVMask (ArrayRef< int > M, EVT VT, unsigned BlockSize) |
| isREVMask - Check if a vector shuffle corresponds to a REV instruction with the specified blocksize. More... | |
| static bool | isZIPMask (ArrayRef< int > M, EVT VT, unsigned &WhichResult) |
| static bool | isUZPMask (ArrayRef< int > M, EVT VT, unsigned &WhichResult) |
| static bool | isTRNMask (ArrayRef< int > M, EVT VT, unsigned &WhichResult) |
| static bool | isZIP_v_undef_Mask (ArrayRef< int > M, EVT VT, unsigned &WhichResult) |
| isZIP_v_undef_Mask - Special case of isZIPMask for canonical form of "vector_shuffle v, v", i.e., "vector_shuffle v, undef". More... | |
| static bool | isUZP_v_undef_Mask (ArrayRef< int > M, EVT VT, unsigned &WhichResult) |
| isUZP_v_undef_Mask - Special case of isUZPMask for canonical form of "vector_shuffle v, v", i.e., "vector_shuffle v, undef". More... | |
| static bool | isTRN_v_undef_Mask (ArrayRef< int > M, EVT VT, unsigned &WhichResult) |
| isTRN_v_undef_Mask - Special case of isTRNMask for canonical form of "vector_shuffle v, v", i.e., "vector_shuffle v, undef". More... | |
| static bool | isINSMask (ArrayRef< int > M, int NumInputElements, bool &DstIsLeft, int &Anomaly) |
| static bool | isConcatMask (ArrayRef< int > Mask, EVT VT, bool SplitLHS) |
| static SDValue | tryFormConcatFromShuffle (SDValue Op, SelectionDAG &DAG) |
| static SDValue | GeneratePerfectShuffle (unsigned ID, SDValue V1, SDValue V2, unsigned PFEntry, SDValue LHS, SDValue RHS, SelectionDAG &DAG, const SDLoc &dl) |
| GeneratePerfectShuffle - Given an entry in the perfect-shuffle table, emit the specified operations to build the shuffle. More... | |
| static SDValue | GenerateTBL (SDValue Op, ArrayRef< int > ShuffleMask, SelectionDAG &DAG) |
| static unsigned | getDUPLANEOp (EVT EltType) |
| static SDValue | constructDup (SDValue V, int Lane, SDLoc dl, EVT VT, unsigned Opcode, SelectionDAG &DAG) |
| static bool | isWideTypeMask (ArrayRef< int > M, EVT VT, SmallVectorImpl< int > &NewMask) |
| static SDValue | tryWidenMaskForShuffle (SDValue Op, SelectionDAG &DAG) |
| static SDValue | tryToConvertShuffleOfTbl2ToTbl4 (SDValue Op, ArrayRef< int > ShuffleMask, SelectionDAG &DAG) |
| static bool | resolveBuildVector (BuildVectorSDNode *BVN, APInt &CnstBits, APInt &UndefBits) |
| static SDValue | tryAdvSIMDModImm64 (unsigned NewOp, SDValue Op, SelectionDAG &DAG, const APInt &Bits) |
| static SDValue | tryAdvSIMDModImm32 (unsigned NewOp, SDValue Op, SelectionDAG &DAG, const APInt &Bits, const SDValue *LHS=nullptr) |
| static SDValue | tryAdvSIMDModImm16 (unsigned NewOp, SDValue Op, SelectionDAG &DAG, const APInt &Bits, const SDValue *LHS=nullptr) |
| static SDValue | tryAdvSIMDModImm321s (unsigned NewOp, SDValue Op, SelectionDAG &DAG, const APInt &Bits) |
| static SDValue | tryAdvSIMDModImm8 (unsigned NewOp, SDValue Op, SelectionDAG &DAG, const APInt &Bits) |
| static SDValue | tryAdvSIMDModImmFP (unsigned NewOp, SDValue Op, SelectionDAG &DAG, const APInt &Bits) |
| static bool | isAllConstantBuildVector (const SDValue &PotentialBVec, uint64_t &ConstVal) |
| static SDValue | tryLowerToSLI (SDNode *N, SelectionDAG &DAG) |
| static SDValue | NormalizeBuildVector (SDValue Op, SelectionDAG &DAG) |
| static SDValue | ConstantBuildVector (SDValue Op, SelectionDAG &DAG) |
| static bool | isPow2Splat (SDValue Op, uint64_t &SplatVal, bool &Negated) |
| static bool | getVShiftImm (SDValue Op, unsigned ElementBits, int64_t &Cnt) |
| getVShiftImm - Check if this is a valid build_vector for the immediate operand of a vector shift operation, where all the elements of the build_vector must have the same constant integer value. More... | |
| static bool | isVShiftLImm (SDValue Op, EVT VT, bool isLong, int64_t &Cnt) |
| isVShiftLImm - Check if this is a valid build_vector for the immediate operand of a vector shift left operation. More... | |
| static bool | isVShiftRImm (SDValue Op, EVT VT, bool isNarrow, int64_t &Cnt) |
| isVShiftRImm - Check if this is a valid build_vector for the immediate operand of a vector shift right operation. More... | |
| static SDValue | EmitVectorComparison (SDValue LHS, SDValue RHS, AArch64CC::CondCode CC, bool NoNans, EVT VT, const SDLoc &dl, SelectionDAG &DAG) |
| static SDValue | getReductionSDNode (unsigned Op, SDLoc DL, SDValue ScalarOp, SelectionDAG &DAG) |
| template<unsigned NumVecs> | |
| static bool | setInfoSVEStN (const AArch64TargetLowering &TLI, const DataLayout &DL, AArch64TargetLowering::IntrinsicInfo &Info, const CallInst &CI) |
Set the IntrinsicInfo for the aarch64_sve_st<N> intrinsics. More... | |
| static bool | isSplatShuffle (Value *V) |
| static bool | areExtractShuffleVectors (Value *Op1, Value *Op2, bool AllowSplat=false) |
| Check if both Op1 and Op2 are shufflevector extracts of either the lower or upper half of the vector elements. More... | |
| static bool | areExtractExts (Value *Ext1, Value *Ext2) |
| Check if Ext1 and Ext2 are extends of the same type, doubling the bitwidth of the vector elements. More... | |
| static bool | isOperandOfVmullHighP64 (Value *Op) |
| Check if Op could be used with vmull_high_p64 intrinsic. More... | |
| static bool | areOperandsOfVmullHighP64 (Value *Op1, Value *Op2) |
| Check if Op1 and Op2 could be used with vmull_high_p64 intrinsic. More... | |
| static void | createTblShuffleForZExt (ZExtInst *ZExt, bool IsLittleEndian) |
| static void | createTblForTrunc (TruncInst *TI, bool IsLittleEndian) |
| static ScalableVectorType * | getSVEContainerIRType (FixedVectorType *VTy) |
| static SDValue | foldVectorXorShiftIntoCmp (SDNode *N, SelectionDAG &DAG, const AArch64Subtarget *Subtarget) |
| Turn vector tests of the signbit in the form of: xor (sra X, elt_size(X)-1), -1 into: cmge X, X, #0. More... | |
| static SDValue | performVecReduceAddCombineWithUADDLP (SDNode *N, SelectionDAG &DAG) |
| static SDValue | performVecReduceAddCombine (SDNode *N, SelectionDAG &DAG, const AArch64Subtarget *ST) |
| static SDValue | performUADDVCombine (SDNode *N, SelectionDAG &DAG) |
| static SDValue | performXorCombine (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const AArch64Subtarget *Subtarget) |
| static std::optional< unsigned > | IsSVECntIntrinsic (SDValue S) |
| static EVT | calculatePreExtendType (SDValue Extend) |
Calculates what the pre-extend type is, based on the extension operation node provided by Extend. More... | |
| static SDValue | performBuildShuffleExtendCombine (SDValue BV, SelectionDAG &DAG) |
| Combines a buildvector(sext/zext) or shuffle(sext/zext, undef) node pattern into sext/zext(buildvector) or sext/zext(shuffle) making use of the vector SExt/ZExt rather than the scalar SExt/ZExt. More... | |
| static SDValue | performMulVectorExtendCombine (SDNode *Mul, SelectionDAG &DAG) |
| Combines a mul(dup(sext/zext)) node pattern into mul(sext/zext(dup)) making use of the vector SExt/ZExt rather than the scalar SExt/ZExt. More... | |
| static SDValue | performMulVectorCmpZeroCombine (SDNode *N, SelectionDAG &DAG) |
| static SDValue | performMulCombine (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const AArch64Subtarget *Subtarget) |
| static SDValue | performVectorCompareAndMaskUnaryOpCombine (SDNode *N, SelectionDAG &DAG) |
| static SDValue | performIntToFpCombine (SDNode *N, SelectionDAG &DAG, const AArch64Subtarget *Subtarget) |
| static SDValue | performFpToIntCombine (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const AArch64Subtarget *Subtarget) |
| Fold a floating-point multiply by power of two into floating-point to fixed-point conversion. More... | |
| static SDValue | performFDivCombine (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const AArch64Subtarget *Subtarget) |
| Fold a floating-point divide by power of two into fixed-point to floating-point conversion. More... | |
| static bool | findEXTRHalf (SDValue N, SDValue &Src, uint32_t &ShiftAmount, bool &FromHi) |
| An EXTR instruction is made up of two shifts, ORed together. More... | |
| static SDValue | tryCombineToEXTR (SDNode *N, TargetLowering::DAGCombinerInfo &DCI) |
| EXTR instruction extracts a contiguous chunk of bits from two existing registers viewed as a high/low pair. More... | |
| static SDValue | tryCombineToBSL (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const AArch64TargetLowering &TLI) |
| static SDValue | performANDORCSELCombine (SDNode *N, SelectionDAG &DAG) |
| static SDValue | performORCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const AArch64Subtarget *Subtarget, const AArch64TargetLowering &TLI) |
| static bool | isConstantSplatVectorMaskForType (SDNode *N, EVT MemVT) |
| static SDValue | performSVEAndCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI) |
| static SDValue | performANDCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI) |
| static bool | hasPairwiseAdd (unsigned Opcode, EVT VT, bool FullFP16) |
| static SDValue | getPTest (SelectionDAG &DAG, EVT VT, SDValue Pg, SDValue Op, AArch64CC::CondCode Cond) |
| static bool | isPredicateCCSettingOp (SDValue N) |
| static SDValue | performFirstTrueTestVectorCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const AArch64Subtarget *Subtarget) |
| static SDValue | performLastTrueTestVectorCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const AArch64Subtarget *Subtarget) |
| static SDValue | performExtractVectorEltCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const AArch64Subtarget *Subtarget) |
| static SDValue | performConcatVectorsCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG) |
| static SDValue | performExtractSubvectorCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG) |
| static SDValue | performInsertSubvectorCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG) |
| static SDValue | tryCombineFixedPointConvert (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG) |
| static SDValue | tryExtendDUPToExtractHigh (SDValue N, SelectionDAG &DAG) |
| static bool | isEssentiallyExtractHighSubvector (SDValue N) |
| static bool | isSetCC (SDValue Op, SetCCInfoAndKind &SetCCInfo) |
Check whether or not Op is a SET_CC operation, either a generic or an AArch64 lowered one. More... | |
| static bool | isSetCCOrZExtSetCC (const SDValue &Op, SetCCInfoAndKind &Info) |
| static SDValue | performSetccAddFolding (SDNode *Op, SelectionDAG &DAG) |
| static SDValue | performAddUADDVCombine (SDNode *N, SelectionDAG &DAG) |
| static SDValue | performAddCSelIntoCSinc (SDNode *N, SelectionDAG &DAG) |
| Perform the scalar expression combine in the form of: CSEL(c, 1, cc) + b => CSINC(b+c, b, cc) CSNEG(c, -1, cc) + b => CSINC(b+c, b, cc) More... | |
| static SDValue | performAddDotCombine (SDNode *N, SelectionDAG &DAG) |
| static bool | isNegatedInteger (SDValue Op) |
| static SDValue | getNegatedInteger (SDValue Op, SelectionDAG &DAG) |
| static SDValue | performNegCSelCombine (SDNode *N, SelectionDAG &DAG) |
| static SDValue | performAddSubLongCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG) |
| static bool | isCMP (SDValue Op) |
| static std::optional< AArch64CC::CondCode > | getCSETCondCode (SDValue Op) |
| static SDValue | foldOverflowCheck (SDNode *Op, SelectionDAG &DAG, bool IsAdd) |
| static SDValue | foldADCToCINC (SDNode *N, SelectionDAG &DAG) |
| static SDValue | performVectorAddSubExtCombine (SDNode *N, SelectionDAG &DAG) |
| static SDValue | performBuildVectorCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG) |
| static bool | isExtendOrShiftOperand (SDValue N) |
| static SDValue | performAddCombineSubShift (SDNode *N, SDValue SUB, SDValue Z, SelectionDAG &DAG) |
| static SDValue | performAddCombineForShiftedOperands (SDNode *N, SelectionDAG &DAG) |
| static SDValue | performAddSubCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG) |
| static SDValue | tryCombineLongOpWithDup (unsigned IID, SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG) |
| static SDValue | tryCombineShiftImm (unsigned IID, SDNode *N, SelectionDAG &DAG) |
| static SDValue | tryCombineCRC32 (unsigned Mask, SDNode *N, SelectionDAG &DAG) |
| static SDValue | combineAcrossLanesIntrinsic (unsigned Opc, SDNode *N, SelectionDAG &DAG) |
| static SDValue | LowerSVEIntrinsicIndex (SDNode *N, SelectionDAG &DAG) |
| static SDValue | LowerSVEIntrinsicDUP (SDNode *N, SelectionDAG &DAG) |
| static SDValue | LowerSVEIntrinsicEXT (SDNode *N, SelectionDAG &DAG) |
| static SDValue | tryConvertSVEWideCompare (SDNode *N, ISD::CondCode CC, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG) |
| static SDValue | combineSVEReductionInt (SDNode *N, unsigned Opc, SelectionDAG &DAG) |
| static SDValue | combineSVEReductionFP (SDNode *N, unsigned Opc, SelectionDAG &DAG) |
| static SDValue | combineSVEReductionOrderedFP (SDNode *N, unsigned Opc, SelectionDAG &DAG) |
| static bool | isAllInactivePredicate (SDValue N) |
| static bool | isAllActivePredicate (SelectionDAG &DAG, SDValue N) |
| static SDValue | convertMergedOpToPredOp (SDNode *N, unsigned Opc, SelectionDAG &DAG, bool UnpredOp=false, bool SwapOperands=false) |
| static SDValue | performIntrinsicCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const AArch64Subtarget *Subtarget) |
| static bool | isCheapToExtend (const SDValue &N) |
| static SDValue | performSignExtendSetCCCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG) |
| static SDValue | performExtendCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG) |
| static SDValue | splitStoreSplat (SelectionDAG &DAG, StoreSDNode &St, SDValue SplatVal, unsigned NumVecElts) |
| static SDValue | performLD1Combine (SDNode *N, SelectionDAG &DAG, unsigned Opc) |
| static SDValue | performLDNT1Combine (SDNode *N, SelectionDAG &DAG) |
| template<unsigned Opcode> | |
| static SDValue | performLD1ReplicateCombine (SDNode *N, SelectionDAG &DAG) |
| static SDValue | performST1Combine (SDNode *N, SelectionDAG &DAG) |
| static SDValue | performSTNT1Combine (SDNode *N, SelectionDAG &DAG) |
| static SDValue | replaceZeroVectorStore (SelectionDAG &DAG, StoreSDNode &St) |
| Replace a splat of zeros to a vector store by scalar stores of WZR/XZR. More... | |
| static SDValue | replaceSplatVectorStore (SelectionDAG &DAG, StoreSDNode &St) |
| Replace a splat of a scalar to a vector store by scalar stores of the scalar value. More... | |
| static SDValue | splitStores (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG, const AArch64Subtarget *Subtarget) |
| static SDValue | performSpliceCombine (SDNode *N, SelectionDAG &DAG) |
| static SDValue | performUnpackCombine (SDNode *N, SelectionDAG &DAG, const AArch64Subtarget *Subtarget) |
| static SDValue | performUzpCombine (SDNode *N, SelectionDAG &DAG) |
| static SDValue | performGLD1Combine (SDNode *N, SelectionDAG &DAG) |
| static SDValue | performVectorShiftCombine (SDNode *N, const AArch64TargetLowering &TLI, TargetLowering::DAGCombinerInfo &DCI) |
| Optimize a vector shift instruction and its operand if shifted out bits are not used. More... | |
| static SDValue | performSunpkloCombine (SDNode *N, SelectionDAG &DAG) |
| static SDValue | performPostLD1Combine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, bool IsLaneOp) |
| Target-specific DAG combine function for post-increment LD1 (lane) and post-increment LD1R. More... | |
| static bool | performTBISimplification (SDValue Addr, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG) |
Simplify Addr given that the top byte of it is ignored by HW during address translation. More... | |
| static SDValue | foldTruncStoreOfExt (SelectionDAG &DAG, SDNode *N) |
| static SDValue | performLOADCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG, const AArch64Subtarget *Subtarget) |
| static SDValue | performSTORECombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG, const AArch64Subtarget *Subtarget) |
| static SDValue | performMSTORECombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG, const AArch64Subtarget *Subtarget) |
| static bool | foldIndexIntoBase (SDValue &BasePtr, SDValue &Index, SDValue Scale, SDLoc DL, SelectionDAG &DAG) |
| static bool | findMoreOptimalIndexType (const MaskedGatherScatterSDNode *N, SDValue &BasePtr, SDValue &Index, SelectionDAG &DAG) |
| static SDValue | performMaskedGatherScatterCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG) |
| static SDValue | performNEONPostLDSTCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG) |
| Target-specific DAG combine function for NEON load/store intrinsics to merge base address updates. More... | |
| static bool | checkValueWidth (SDValue V, unsigned width, ISD::LoadExtType &ExtType) |
| static bool | isEquivalentMaskless (unsigned CC, unsigned width, ISD::LoadExtType ExtType, int AddConstant, int CompConstant) |
| static SDValue | performCONDCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG, unsigned CCIndex, unsigned CmpIndex) |
| static SDValue | performBRCONDCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG) |
| static SDValue | foldCSELofCTTZ (SDNode *N, SelectionDAG &DAG) |
| static SDValue | foldCSELOfCSEL (SDNode *Op, SelectionDAG &DAG) |
| static SDValue | performCSELCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG) |
| static SDValue | tryToWidenSetCCOperands (SDNode *Op, SelectionDAG &DAG) |
| static SDValue | performSETCCCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG) |
| static SDValue | performFlagSettingCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, unsigned GenericOpcode) |
| static SDValue | performSetCCPunpkCombine (SDNode *N, SelectionDAG &DAG) |
| static SDValue | performSetccMergeZeroCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI) |
| static SDValue | getTestBitOperand (SDValue Op, unsigned &Bit, bool &Invert, SelectionDAG &DAG) |
| static SDValue | performTBZCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG) |
| static SDValue | trySwapVSelectOperands (SDNode *N, SelectionDAG &DAG) |
| static SDValue | performVSelectCombine (SDNode *N, SelectionDAG &DAG) |
| static SDValue | performSelectCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI) |
| A vector select: "(select vL, vR, (setcc LHS, RHS))" is best performed with the compare-mask instructions rather than going via NZCV, even if LHS and RHS are really scalar. More... | |
| static SDValue | performDUPCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI) |
| static SDValue | performNVCASTCombine (SDNode *N) |
| Get rid of unnecessary NVCASTs (that don't change the type). More... | |
| static SDValue | performGlobalAddressCombine (SDNode *N, SelectionDAG &DAG, const AArch64Subtarget *Subtarget, const TargetMachine &TM) |
| static SDValue | getScaledOffsetForBitWidth (SelectionDAG &DAG, SDValue Offset, SDLoc DL, unsigned BitWidth) |
| static bool | isValidImmForSVEVecImmAddrMode (unsigned OffsetInBytes, unsigned ScalarSizeInBytes) |
Check if the value of OffsetInBytes can be used as an immediate for the gather load/prefetch and scatter store instructions with vector base and immediate offset addressing mode: More... | |
| static bool | isValidImmForSVEVecImmAddrMode (SDValue Offset, unsigned ScalarSizeInBytes) |
Check if the value of Offset represents a valid immediate for the SVE gather load/prefetch and scatter store instructiona with vector base and immediate offset addressing mode: More... | |
| static SDValue | performScatterStoreCombine (SDNode *N, SelectionDAG &DAG, unsigned Opcode, bool OnlyPackedOffsets=true) |
| static SDValue | performGatherLoadCombine (SDNode *N, SelectionDAG &DAG, unsigned Opcode, bool OnlyPackedOffsets=true) |
| static SDValue | performSignExtendInRegCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG) |
| static SDValue | legalizeSVEGatherPrefetchOffsVec (SDNode *N, SelectionDAG &DAG) |
| Legalize the gather prefetch (scalar + vector addressing mode) when the offset vector is an unpacked 32-bit scalable vector. More... | |
| static SDValue | combineSVEPrefetchVecBaseImmOff (SDNode *N, SelectionDAG &DAG, unsigned ScalarSizeInBytes) |
Combines a node carrying the intrinsic aarch64_sve_prf<T>_gather_scalar_offset into a node that uses aarch64_sve_prfb_gather_uxtw_index when the scalar offset passed to aarch64_sve_prf<T>_gather_scalar_offset is not a valid immediate for the sve gather prefetch instruction with vector plus immediate addressing mode. More... | |
| static bool | isLanes1toNKnownZero (SDValue Op) |
| static SDValue | removeRedundantInsertVectorElt (SDNode *N) |
| static SDValue | performInsertVectorEltCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI) |
| static SDValue | performSVESpliceCombine (SDNode *N, SelectionDAG &DAG) |
| static SDValue | performFPExtendCombine (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const AArch64Subtarget *Subtarget) |
| static SDValue | performBSPExpandForSVE (SDNode *N, SelectionDAG &DAG, const AArch64Subtarget *Subtarget) |
| static SDValue | performDupLane128Combine (SDNode *N, SelectionDAG &DAG) |
| static void | ReplaceAddWithADDP (SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG, const AArch64Subtarget *Subtarget) |
| static void | ReplaceReductionResults (SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG, unsigned InterOp, unsigned AcrossOp) |
| static std::pair< SDValue, SDValue > | splitInt128 (SDValue N, SelectionDAG &DAG) |
| static SDValue | createGPRPairNode (SelectionDAG &DAG, SDValue V) |
| static void | ReplaceCMP_SWAP_128Results (SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG, const AArch64Subtarget *Subtarget) |
| static Value * | UseTlsOffset (IRBuilderBase &IRB, unsigned Offset) |
| static SDValue | getPredicateForFixedLengthVector (SelectionDAG &DAG, SDLoc &DL, EVT VT) |
| static SDValue | getPredicateForVector (SelectionDAG &DAG, SDLoc &DL, EVT VT) |
| static SDValue | emitConditionalComparison (SDValue LHS, SDValue RHS, ISD::CondCode CC, SDValue CCOp, AArch64CC::CondCode Predicate, AArch64CC::CondCode OutCC, const SDLoc &DL, SelectionDAG &DAG) |
| can be transformed to: not (and (not (and (setCC (cmp C)) (setCD (cmp D)))) (and (not (setCA (cmp A)) (not (setCB (cmp B))))))" which can be implemented as: cmp C ccmp D, inv(CD), CC ccmp A, CA, inv(CD) ccmp B, CB, inv(CA) check for CB flags More... | |
| static bool | canEmitConjunction (const SDValue Val, bool &CanNegate, bool &MustBeFirst, bool WillNegate, unsigned Depth=0) |
Returns true if Val is a tree of AND/OR/SETCC operations that can be expressed as a conjunction. More... | |
| static SDValue | emitConjunctionRec (SelectionDAG &DAG, SDValue Val, AArch64CC::CondCode &OutCC, bool Negate, SDValue CCOp, AArch64CC::CondCode Predicate) |
| Emit conjunction or disjunction tree with the CMP/FCMP followed by a chain of CCMP/CFCMP ops. More... | |
| static SDValue | emitConjunction (SelectionDAG &DAG, SDValue Val, AArch64CC::CondCode &OutCC) |
| Emit expression as a conjunction (a series of CCMP/CFCMP ops). More... | |
Variables | |
| cl::opt< bool > | EnableAArch64ELFLocalDynamicTLSGeneration ("aarch64-elf-ldtls-generation", cl::Hidden, cl::desc("Allow AArch64 Local Dynamic TLS code generation"), cl::init(false)) |
| static cl::opt< bool > | EnableOptimizeLogicalImm ("aarch64-enable-logical-imm", cl::Hidden, cl::desc("Enable AArch64 logical imm instruction " "optimization"), cl::init(true)) |
| static cl::opt< bool > | EnableCombineMGatherIntrinsics ("aarch64-enable-mgather-combine", cl::Hidden, cl::desc("Combine extends of AArch64 masked " "gather intrinsics"), cl::init(true)) |
| static cl::opt< unsigned > | MaxXors ("aarch64-max-xors", cl::init(16), cl::Hidden, cl::desc("Maximum of xors")) |
| static const MVT | MVT_CC = MVT::i32 |
| Value type used for condition codes. More... | |
| #define DEBUG_TYPE "aarch64-lower" |
Definition at line 103 of file AArch64ISelLowering.cpp.
| #define GET_REGISTER_MATCHER |
Definition at line 9440 of file AArch64ISelLowering.cpp.
| #define MAKE_CASE | ( | V | ) |
| enum PredicateConstraint |
| Enumerator | |
|---|---|
| Upl | |
| Upa | |
| Invalid | |
Definition at line 9719 of file AArch64ISelLowering.cpp.
|
static |
Definition at line 4310 of file AArch64ISelLowering.cpp.
References assert(), getExtensionTo64Bits(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::EVT::is128BitVector(), and N.
Referenced by skipExtensionForVectorMULL().
|
static |
Definition at line 6654 of file AArch64ISelLowering.cpp.
References llvm::TargetLowering::CallLoweringInfo::Args, assert(), llvm::TargetLowering::CallLoweringInfo::CallConv, llvm::AArch64TargetLowering::CCAssignFnForCall(), llvm::TargetLowering::CallLoweringInfo::DAG, llvm::CCValAssign::Full, llvm::SelectionDAG::getDataLayout(), llvm::EVT::getSimpleVT(), llvm::TargetLoweringBase::getValueType(), i, llvm::MVT::i1, llvm::MVT::i16, llvm::MVT::i8, llvm::AArch64Subtarget::isCallingConvWin64(), llvm::EVT::isSimple(), llvm::TargetLowering::CallLoweringInfo::IsVarArg, and llvm::TargetLowering::CallLoweringInfo::Outs.
Check if Ext1 and Ext2 are extends of the same type, doubling the bitwidth of the vector elements.
Definition at line 13577 of file AArch64ISelLowering.cpp.
References llvm::MipsISD::Ext, llvm::PatternMatch::m_Value(), llvm::PatternMatch::m_ZExtOrSExt(), and llvm::PatternMatch::match().
Referenced by llvm::AArch64TargetLowering::shouldSinkOperands().
Check if both Op1 and Op2 are shufflevector extracts of either the lower or upper half of the vector elements.
Definition at line 13521 of file AArch64ISelLowering.cpp.
References llvm::TypeSize::getFixedSize(), llvm::Type::getPrimitiveSizeInBits(), llvm::Value::getType(), llvm::ShuffleVectorInst::isExtractSubvectorMask(), isSplatShuffle(), llvm::M1(), llvm::PatternMatch::m_Shuffle(), llvm::PatternMatch::m_Undef(), llvm::PatternMatch::m_Value(), and llvm::PatternMatch::match().
Referenced by llvm::AArch64TargetLowering::shouldSinkOperands().
Check if Op1 and Op2 could be used with vmull_high_p64 intrinsic.
Definition at line 13604 of file AArch64ISelLowering.cpp.
References isOperandOfVmullHighP64().
Referenced by llvm::AArch64TargetLowering::shouldSinkOperands().
Calculates what the pre-extend type is, based on the extension operation node provided by Extend.
In the case that Extend is a SIGN_EXTEND or a ZERO_EXTEND, the pre-extend type is pulled directly from the operand, while other extend operations need a bit more inspection to get this information.
| Extend | The SDNode from the DAG that represents the extend operation |
Extend source type, or MVT::Other if no valid type can be determined Definition at line 15042 of file AArch64ISelLowering.cpp.
References llvm::ISD::AND, llvm::ISD::AssertSext, llvm::ISD::AssertZext, llvm::SDValue::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValueType(), llvm::VTSDNode::getVT(), llvm::MVT::i16, llvm::MVT::i32, llvm::MVT::i8, llvm::BitmaskEnumDetail::Mask(), llvm::MVT::Other, llvm::ISD::SIGN_EXTEND, llvm::ISD::SIGN_EXTEND_INREG, and llvm::ISD::ZERO_EXTEND.
Referenced by performBuildShuffleExtendCombine().
|
static |
Returns true if Val is a tree of AND/OR/SETCC operations that can be expressed as a conjunction.
See CMP;CCMP matching.
| CanNegate | Set to true if we can negate the whole sub-tree just by changing the conditions on the SETCC tests. (this means we can call emitConjunctionRec() with Negate==true on this sub-tree) |
| MustBeFirst | Set to true if this subtree needs to be negated and we cannot do the negation naturally. We are required to emit the subtree first in this case. |
| WillNegate | Is true if are called when the result of this subexpression must be negated. This happens when the outer expression is an OR. We can use this fact to know that we have a double negation (or (or ...) ...) that can be implemented for free. |
Definition at line 3167 of file AArch64ISelLowering.cpp.
References llvm::ISD::AND, assert(), llvm::Depth, llvm::MVT::f128, llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::SDValue::getValueType(), llvm::SDValue::hasOneUse(), llvm::ISD::OR, and llvm::ISD::SETCC.
Referenced by emitConjunction(), and emitConjunctionRec().
|
static |
Return true if the calling convention is one that we can guarantee TCO for.
Definition at line 6633 of file AArch64ISelLowering.cpp.
References CC, llvm::CallingConv::Fast, llvm::CallingConv::SwiftTail, and llvm::CallingConv::Tail.
|
static |
Definition at line 3709 of file AArch64ISelLowering.cpp.
References assert(), CC, Cond, llvm::AArch64ISD::CSEL, DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::AArch64CC::HS, llvm::MVT::i32, and llvm::AArch64CC::LO.
Referenced by lowerADDSUBCARRY().
|
static |
changeFPCCToAArch64CC - Convert a DAG fp condition code to an AArch64 CC.
Definition at line 2847 of file AArch64ISelLowering.cpp.
References llvm::AArch64CC::AL, CC, llvm::AArch64CC::EQ, llvm::AArch64CC::GE, llvm::AArch64CC::GT, llvm::AArch64CC::HI, llvm::AArch64CC::LE, llvm_unreachable, llvm::AArch64CC::LS, llvm::AArch64CC::LT, llvm::AArch64CC::MI, llvm::AArch64CC::NE, llvm::AArch64CC::PL, llvm::ISD::SETEQ, llvm::ISD::SETGE, llvm::ISD::SETGT, llvm::ISD::SETLE, llvm::ISD::SETLT, llvm::ISD::SETNE, llvm::ISD::SETO, llvm::ISD::SETOEQ, llvm::ISD::SETOGE, llvm::ISD::SETOGT, llvm::ISD::SETOLE, llvm::ISD::SETOLT, llvm::ISD::SETONE, llvm::ISD::SETUEQ, llvm::ISD::SETUGE, llvm::ISD::SETUGT, llvm::ISD::SETULE, llvm::ISD::SETULT, llvm::ISD::SETUNE, llvm::ISD::SETUO, llvm::AArch64CC::VC, and llvm::AArch64CC::VS.
Referenced by changeFPCCToANDAArch64CC(), and changeVectorFPCCToAArch64CC().
|
static |
Convert a DAG fp condition code to an AArch64 CC.
This differs from changeFPCCToAArch64CC in that it returns cond codes that should be AND'ed instead of OR'ed.
Definition at line 2910 of file AArch64ISelLowering.cpp.
References llvm::AArch64CC::AL, assert(), CC, changeFPCCToAArch64CC(), llvm::AArch64CC::LE, llvm::AArch64CC::NE, llvm::AArch64CC::PL, llvm::ISD::SETONE, llvm::ISD::SETUEQ, and llvm::AArch64CC::VC.
Referenced by emitConjunctionRec().
|
static |
changeIntCCToAArch64CC - Convert a DAG integer condition code to an AArch64 CC
Definition at line 2819 of file AArch64ISelLowering.cpp.
References CC, llvm::AArch64CC::EQ, llvm::AArch64CC::GE, llvm::AArch64CC::GT, llvm::AArch64CC::HI, llvm::AArch64CC::HS, llvm::AArch64CC::LE, llvm_unreachable, llvm::AArch64CC::LO, llvm::AArch64CC::LS, llvm::AArch64CC::LT, llvm::AArch64CC::NE, llvm::ISD::SETEQ, llvm::ISD::SETGE, llvm::ISD::SETGT, llvm::ISD::SETLE, llvm::ISD::SETLT, llvm::ISD::SETNE, llvm::ISD::SETUGE, llvm::ISD::SETUGT, llvm::ISD::SETULE, and llvm::ISD::SETULT.
Referenced by emitConjunctionRec(), and getAArch64Cmp().
|
static |
changeVectorFPCCToAArch64CC - Convert a DAG fp condition code to an AArch64 CC usable with the vector instructions.
Fewer operations are available without a real NZCV register, so we have to use less efficient combinations to get the same effect.
Definition at line 2940 of file AArch64ISelLowering.cpp.
References CC, changeFPCCToAArch64CC(), llvm::MVT::f32, llvm::AArch64CC::GE, llvm::ISD::getSetCCInverse(), llvm::AArch64CC::MI, llvm::ISD::SETO, llvm::ISD::SETUEQ, llvm::ISD::SETUGE, llvm::ISD::SETUGT, llvm::ISD::SETULE, llvm::ISD::SETULT, and llvm::ISD::SETUO.
|
static |
Definition at line 19249 of file AArch64ISelLowering.cpp.
References llvm::abs(), llvm::ISD::AssertSext, llvm::ISD::AssertZext, llvm::ISD::Constant, llvm::LoadSDNode::getExtensionType(), llvm::MemSDNode::getMemoryVT(), llvm::SDValue::getNode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::VTSDNode::getVT(), llvm::MVT::i16, llvm::MVT::i8, llvm::ISD::LOAD, llvm::ISD::NON_EXTLOAD, llvm::ISD::SEXTLOAD, llvm::ISD::TargetConstant, and llvm::ISD::ZEXTLOAD.
Referenced by performCONDCombine().
|
static |
Definition at line 6891 of file AArch64ISelLowering.cpp.
References Arg, llvm::tgtok::Bits, and llvm::SelectionDAG::computeKnownBits().
|
static |
Definition at line 17323 of file AArch64ISelLowering.cpp.
References llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::MVT::i64, and N.
Referenced by performIntrinsicCombine().
|
static |
Combines a node carrying the intrinsic aarch64_sve_prf<T>_gather_scalar_offset into a node that uses aarch64_sve_prfb_gather_uxtw_index when the scalar offset passed to aarch64_sve_prf<T>_gather_scalar_offset is not a valid immediate for the sve gather prefetch instruction with vector plus immediate addressing mode.
Definition at line 20665 of file AArch64ISelLowering.cpp.
References DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getVTList(), llvm::MVT::i64, isValidImmForSVEVecImmAddrMode(), N, llvm::MVT::Other, and std::swap().
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 17510 of file AArch64ISelLowering.cpp.
References DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getValueType(), llvm::MVT::i64, and N.
Referenced by performIntrinsicCombine().
|
static |
Definition at line 17491 of file AArch64ISelLowering.cpp.
References DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), getPackedSVEVectorVT(), llvm::MVT::i64, and N.
Referenced by performIntrinsicCombine().
|
static |
Definition at line 17527 of file AArch64ISelLowering.cpp.
References DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::MVT::i64, llvm::ISD::INSERT_VECTOR_ELT, and N.
Referenced by performIntrinsicCombine().
|
static |
Definition at line 11902 of file AArch64ISelLowering.cpp.
References llvm::AArch64ISD::FMOV, llvm::EVT::getSizeInBits(), llvm::AArch64ISD::MOVI, llvm::AArch64ISD::MOVIedit, llvm::AArch64ISD::MOVImsl, llvm::AArch64ISD::MOVIshift, llvm::AArch64ISD::MVNImsl, llvm::AArch64ISD::MVNIshift, resolveBuildVector(), tryAdvSIMDModImm16(), tryAdvSIMDModImm32(), tryAdvSIMDModImm321s(), tryAdvSIMDModImm64(), tryAdvSIMDModImm8(), and tryAdvSIMDModImmFP().
|
static |
Definition at line 11056 of file AArch64ISelLowering.cpp.
References llvm::ISD::BITCAST, llvm::ISD::CONCAT_VECTORS, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getConstantOperandVal(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::MVT::getScalarType(), llvm::SDValue::getScalarValueSizeInBits(), llvm::SDValue::getSimpleValueType(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::MVT::i64, int, llvm::EVT::is128BitVector(), and WidenVector().
|
static |
Definition at line 22373 of file AArch64ISelLowering.cpp.
References convertToScalableVector(), DL, llvm::SelectionDAG::getConstant(), getContainerForFixedLengthVector(), llvm::SelectionDAG::getNode(), getPredicateForFixedLengthVector(), llvm::SDValue::getValueType(), llvm::ISD::isBuildVectorAllOnes(), llvm::BitmaskEnumDetail::Mask(), and llvm::AArch64ISD::SETCC_MERGE_ZERO.
|
static |
Definition at line 22323 of file AArch64ISelLowering.cpp.
References assert(), DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getValueType(), llvm::MVT::i64, llvm::EVT::isFixedLengthVector(), and llvm::EVT::isScalableVector().
|
static |
Definition at line 17600 of file AArch64ISelLowering.cpp.
References assert(), llvm::SelectionDAG::getNode(), llvm::ISD::INTRINSIC_WO_CHAIN, isAllActivePredicate(), and N.
Referenced by performIntrinsicCombine().
|
static |
Definition at line 22312 of file AArch64ISelLowering.cpp.
References assert(), DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::MVT::i64, llvm::ISD::INSERT_SUBVECTOR, llvm::EVT::isFixedLengthVector(), and llvm::EVT::isScalableVector().
Referenced by convertFixedMaskToScalableVector().
|
static |
Definition at line 21423 of file AArch64ISelLowering.cpp.
References llvm::SelectionDAG::getAnyExtOrTrunc(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getMachineNode(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetConstant(), llvm::MVT::i128, llvm::MVT::i32, llvm::MVT::i64, llvm::DataLayout::isBigEndian(), llvm::ISD::SRL, std::swap(), and llvm::MVT::Untyped.
Referenced by ReplaceCMP_SWAP_128Results().
|
static |
Definition at line 13807 of file AArch64ISelLowering.cpp.
References Builder, llvm::Instruction::eraseFromParent(), F, llvm::ConstantInt::get(), llvm::ConstantVector::get(), llvm::FixedVectorType::get(), llvm::Intrinsic::getDeclaration(), llvm::Instruction::getModule(), llvm::User::getOperand(), llvm::Value::getType(), and llvm::Value::replaceAllUsesWith().
Referenced by llvm::AArch64TargetLowering::optimizeExtendOrTruncateConversion().
|
static |
Definition at line 13782 of file AArch64ISelLowering.cpp.
References Builder, llvm::Instruction::eraseFromParent(), llvm::PoisonValue::get(), llvm::User::getOperand(), llvm::Value::getType(), i, llvm::BitmaskEnumDetail::Mask(), and llvm::Value::replaceAllUsesWith().
Referenced by llvm::AArch64TargetLowering::optimizeExtendOrTruncateConversion().
|
static |
Definition at line 3015 of file AArch64ISelLowering.cpp.
References llvm::AArch64ISD::ADDS, llvm::ISD::AND, llvm::AArch64ISD::ANDS, assert(), CC, llvm::MVT::f128, llvm::MVT::f16, llvm::MVT::f32, llvm::AArch64ISD::FCMP, llvm::ISD::FP_EXTEND, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getSubtarget(), llvm::SDValue::getValue(), llvm::SelectionDAG::getVTList(), isCMN(), llvm::EVT::isFloatingPoint(), llvm::isNullConstant(), llvm::ISD::isUnsignedIntSetCC(), LHS, MVT_CC, llvm::SelectionDAG::ReplaceAllUsesWith(), RHS, and llvm::AArch64ISD::SUBS.
Referenced by emitConjunctionRec(), and getAArch64Cmp().
|
static |
can be transformed to: not (and (not (and (setCC (cmp C)) (setCD (cmp D)))) (and (not (setCA (cmp A)) (not (setCB (cmp B))))))" which can be implemented as: cmp C ccmp D, inv(CD), CC ccmp A, CA, inv(CD) ccmp B, CB, inv(CA) check for CB flags
A counterexample is "or (and A B) (and C D)" which translates to not (and (not (and (not A) (not B))) (not (and (not C) (not D)))), we can only implement 1 of the inner (not) operations, but not both! Create a conditional comparison; Use CCMP, CCMN or FCCMP as appropriate.
Definition at line 3120 of file AArch64ISelLowering.cpp.
References assert(), CC, llvm::AArch64ISD::CCMN, llvm::AArch64ISD::CCMP, DL, llvm::MVT::f128, llvm::MVT::f16, llvm::MVT::f32, llvm::AArch64ISD::FCCMP, llvm::ISD::FP_EXTEND, llvm::SelectionDAG::getConstant(), llvm::AArch64CC::getInvertedCondCode(), llvm::SelectionDAG::getNode(), llvm::AArch64CC::getNZCVToSatisfyCondCode(), llvm::SelectionDAG::getSubtarget(), llvm::MVT::i32, llvm::isNullConstant(), LHS, MVT_CC, RHS, llvm::ISD::SETEQ, llvm::ISD::SETNE, and llvm::ISD::SUB.
Referenced by emitConjunctionRec().
|
static |
Emit expression as a conjunction (a series of CCMP/CFCMP ops).
In some cases this is even possible with OR operations in the expression. See CMP;CCMP matching.
Definition at line 3341 of file AArch64ISelLowering.cpp.
References llvm::AArch64CC::AL, canEmitConjunction(), and emitConjunctionRec().
Referenced by getAArch64Cmp(), and LowerBRCOND().
|
static |
Emit conjunction or disjunction tree with the CMP/FCMP followed by a chain of CCMP/CFCMP ops.
See CMP;CCMP matching. Tries to transform the given i1 producing node Val to a series compare and conditional compare operations.
OutCC to the flags that should be tested or returns SDValue() if transformation was not possible. Negate is true if we want this sub-tree being negated just by changing SETCC conditions. Definition at line 3229 of file AArch64ISelLowering.cpp.
References llvm::AArch64CC::AL, llvm::ISD::AND, assert(), canEmitConjunction(), CC, changeFPCCToANDAArch64CC(), changeIntCCToAArch64CC(), DL, emitComparison(), emitConditionalComparison(), llvm::AArch64CC::getInvertedCondCode(), llvm::SDValue::getNode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::ISD::getSetCCInverse(), llvm::SDNode::hasOneUse(), llvm::yaml::isInteger(), LHS, llvm::ISD::OR, RHS, llvm::ISD::SETCC, and std::swap().
Referenced by emitConjunction().
|
static |
Definition at line 2994 of file AArch64ISelLowering.cpp.
References assert(), llvm::MVT::f128, llvm::MVT::f16, llvm::MVT::f32, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getSubtarget(), LHS, llvm::MVT::Other, RHS, llvm::AArch64ISD::STRICT_FCMP, llvm::AArch64ISD::STRICT_FCMPE, and llvm::ISD::STRICT_FP_EXTEND.
|
static |
Definition at line 12805 of file AArch64ISelLowering.cpp.
References assert(), CC, llvm::AArch64ISD::CMEQ, llvm::AArch64ISD::CMEQz, llvm::AArch64ISD::CMGE, llvm::AArch64ISD::CMGEz, llvm::AArch64ISD::CMGT, llvm::AArch64ISD::CMGTz, llvm::AArch64ISD::CMHI, llvm::AArch64ISD::CMHS, llvm::AArch64ISD::CMLEz, llvm::AArch64ISD::CMLTz, llvm::AArch64CC::EQ, llvm::AArch64ISD::FCMEQ, llvm::AArch64ISD::FCMEQz, llvm::AArch64ISD::FCMGE, llvm::AArch64ISD::FCMGEz, llvm::AArch64ISD::FCMGT, llvm::AArch64ISD::FCMGTz, llvm::AArch64ISD::FCMLEz, llvm::AArch64ISD::FCMLTz, llvm::AArch64CC::GE, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNOT(), llvm::EVT::getSizeInBits(), llvm::EVT::getVectorElementType(), llvm::AArch64CC::GT, llvm::AArch64CC::HI, llvm::AArch64CC::HS, llvm::EVT::isFloatingPoint(), llvm::AArch64CC::LE, LHS, llvm::AArch64CC::LO, llvm::AArch64CC::LS, llvm::AArch64CC::LT, llvm::AArch64CC::MI, llvm::AArch64CC::NE, resolveBuildVector(), and RHS.
An EXTR instruction is made up of two shifts, ORed together.
This helper searches for and classifies those shifts.
Definition at line 15600 of file AArch64ISelLowering.cpp.
References N, llvm::ISD::SHL, and llvm::ISD::SRL.
Referenced by tryCombineToEXTR().
|
static |
Definition at line 19002 of file AArch64ISelLowering.cpp.
References llvm::EVT::changeVectorElementType(), foldIndexIntoBase(), llvm::SDValue::getConstantOperandVal(), llvm::AArch64Subtarget::getMaxSVEVectorSizeInBits(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::SelectionDAG::getSplatValue(), llvm::SelectionDAG::getStepVector(), llvm::SelectionDAG::getSubtarget(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorMinNumElements(), llvm::MVT::i32, llvm::MVT::i64, llvm::EVT::isFixedLengthVector(), llvm::ISD::isVectorShrinkable(), llvm::max(), llvm::min(), N, llvm::MVT::nxv2i64, RHS, Shift, llvm::ISD::SHL, llvm::ISD::STEP_VECTOR, llvm::AArch64::SVEBitsPerBlock, and llvm::ISD::TRUNCATE.
Referenced by performMaskedGatherScatterCombine().
|
static |
Definition at line 16988 of file AArch64ISelLowering.cpp.
References CC, Cond, llvm::AArch64ISD::CSINC, DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::MVT::i32, llvm::isNullConstant(), LHS, llvm::AArch64CC::LO, N, and RHS.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 19612 of file AArch64ISelLowering.cpp.
References CC, Cond, llvm::AArch64ISD::CSEL, DL, llvm::AArch64CC::EQ, llvm::SelectionDAG::getConstant(), llvm::SDNode::getConstantOperandVal(), llvm::AArch64CC::getInvertedCondCode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::MVT::i32, isCMP(), llvm::AArch64CC::NE, std::swap(), X, and Y.
Referenced by performCSELCombine().
|
static |
Definition at line 19564 of file AArch64ISelLowering.cpp.
References llvm::ISD::AND, assert(), llvm::BitWidth, CC, llvm::ISD::CTTZ, llvm::AArch64CC::EQ, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::MVT::i32, llvm::MVT::i64, llvm::isNullConstant(), N, llvm::AArch64CC::NE, llvm::AArch64ISD::SUBS, llvm::ISD::TRUNCATE, and X.
Referenced by performCSELCombine().
|
static |
Definition at line 18952 of file AArch64ISelLowering.cpp.
References llvm::ISD::ADD, DL, llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getSplatValue(), llvm::EVT::getVectorElementType(), llvm::MVT::i64, llvm::EVT::isVector(), llvm::ISD::MUL, Shift, and llvm::ISD::SHL.
Referenced by findMoreOptimalIndexType().
|
static |
Definition at line 16964 of file AArch64ISelLowering.cpp.
References CC, getCSETCondCode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::AArch64CC::HS, isCMP(), llvm::isNullConstant(), llvm::isOneConstant(), and llvm::AArch64CC::LO.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 18767 of file AArch64ISelLowering.cpp.
References llvm::ISD::ANY_EXTEND, assert(), llvm::MipsISD::Ext, llvm::SelectionDAG::getStore(), llvm::SDValue::getValueType(), llvm::ISD::MSTORE, N, llvm::ISD::SIGN_EXTEND, llvm::SPII::Store, llvm::ISD::STORE, and llvm::ISD::ZERO_EXTEND.
Referenced by performSTORECombine().
|
static |
Turn vector tests of the signbit in the form of: xor (sra X, elt_size(X)-1), -1 into: cmge X, X, #0.
Definition at line 14687 of file AArch64ISelLowering.cpp.
References llvm::AArch64ISD::CMGEz, llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::ISD::isBuildVectorAllOnes(), llvm::EVT::isVector(), N, Shift, and llvm::AArch64ISD::VASHR.
Referenced by performXorCombine().
|
static |
GeneratePerfectShuffle - Given an entry in the perfect-shuffle table, emit the specified operations to build the shuffle.
ID is the perfect-shuffle
Definition at line 10800 of file AArch64ISelLowering.cpp.
References assert(), llvm::MVT::bf16, llvm::AArch64ISD::DUPLANE16, llvm::AArch64ISD::DUPLANE32, llvm::AArch64ISD::DUPLANE64, llvm::AArch64ISD::DUPLANE8, llvm::MipsISD::Ext, llvm::AArch64ISD::EXT, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::MVT::f16, llvm::MVT::f32, llvm::MVT::f64, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), getExtFactor(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::SelectionDAG::getVectorIdxConstant(), llvm::SelectionDAG::getVTList(), llvm::MVT::i16, llvm::MVT::i32, llvm::MVT::i64, llvm::MVT::i8, llvm::RISCVMatInt::Imm, llvm::MipsISD::Ins, llvm::ISD::INSERT_VECTOR_ELT, LHS, llvm_unreachable, OP_COPY, OP_VDUP0, OP_VDUP1, OP_VDUP2, OP_VDUP3, OP_VEXT1, OP_VEXT2, OP_VEXT3, OP_VREV, OP_VTRNL, OP_VTRNR, OP_VUZPL, OP_VUZPR, OP_VZIPL, OP_VZIPR, PerfectShuffleTable, llvm::AArch64ISD::REV16, llvm::AArch64ISD::REV32, llvm::AArch64ISD::REV64, RHS, llvm::AArch64ISD::TRN1, llvm::AArch64ISD::TRN2, llvm::AArch64ISD::UZP1, llvm::AArch64ISD::UZP2, llvm::NVPTX::PTXLdStInstCode::V2, llvm::MVT::v2f32, llvm::MVT::v2f64, llvm::MVT::v4f16, llvm::MVT::v4i16, WidenVector(), x3, llvm::AArch64ISD::ZIP1, and llvm::AArch64ISD::ZIP2.
|
static |
Definition at line 10967 of file AArch64ISelLowering.cpp.
References llvm::ISD::BITCAST, llvm::ISD::CONCAT_VECTORS, DL, llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::MVT::i32, llvm::ISD::INTRINSIC_WO_CHAIN, llvm::SDValue::isUndef(), isZerosVector(), llvm::makeArrayRef(), llvm::Shuffle, std::swap(), llvm::MVT::v16i8, llvm::NVPTX::PTXLdStInstCode::V2, and llvm::MVT::v8i8.
|
static |
Definition at line 3389 of file AArch64ISelLowering.cpp.
References CC, changeIntCCToAArch64CC(), emitComparison(), emitConjunction(), getCmpOperandFoldingProfit(), llvm::SelectionDAG::getConstant(), llvm::AArch64CC::getInvertedCondCode(), llvm::SelectionDAG::getNode(), llvm::ISD::getSetCCSwappedOperands(), llvm::SelectionDAG::getValueType(), llvm::ConstantSDNode::getZExtValue(), llvm::MVT::i16, llvm::MVT::i32, llvm::MVT::i64, INT64_MAX, isCMN(), isLegalArithImmed(), llvm::ConstantSDNode::isOne(), llvm::ConstantSDNode::isZero(), LHS, MVT_CC, RHS, llvm::ISD::SETEQ, llvm::ISD::SETGE, llvm::ISD::SETGT, llvm::ISD::SETLE, llvm::ISD::SETLT, llvm::ISD::SETNE, llvm::ISD::SETUGE, llvm::ISD::SETUGT, llvm::ISD::SETULE, llvm::ISD::SETULT, llvm::ISD::SIGN_EXTEND_INREG, std::swap(), UINT64_MAX, and llvm::ISD::ZEXTLOAD.
Referenced by performSetccAddFolding().
|
static |
Definition at line 3521 of file AArch64ISelLowering.cpp.
References llvm::AArch64ISD::ADDS, llvm::AArch64ISD::ANDS, assert(), CC, DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getValue(), llvm::SelectionDAG::getVTList(), llvm::AArch64CC::HS, llvm::MVT::i32, llvm::MVT::i64, LHS, llvm_unreachable, llvm::AArch64CC::LO, Mul, llvm::ISD::MUL, llvm::ISD::MULHS, llvm::ISD::MULHU, MVT_CC, llvm::AArch64CC::NE, RHS, llvm::ISD::SADDO, llvm::ISD::SIGN_EXTEND, llvm::ISD::SMULO, llvm::ISD::SRA, llvm::ISD::SSUBO, llvm::AArch64ISD::SUBS, llvm::ISD::TRUNCATE, llvm::ISD::UADDO, llvm::ISD::UMULO, llvm::ISD::USUBO, llvm::AArch64CC::VS, and llvm::ISD::ZERO_EXTEND.
Referenced by LowerXALUO().
Definition at line 4652 of file AArch64ISelLowering.cpp.
References llvm::None, S, llvm::SMEAttrs::SM_Compatible, llvm::SMEAttrs::ZA_Preserved, and llvm::SMEAttrs::ZA_Shared.
|
static |
Returns how profitable it is to fold a comparison's operand's shift and/or extension operations.
Definition at line 3355 of file AArch64ISelLowering.cpp.
References llvm::ISD::AND, llvm::MVT::i32, llvm::MVT::i64, llvm::BitmaskEnumDetail::Mask(), Shift, llvm::ISD::SHL, llvm::ISD::SIGN_EXTEND_INREG, llvm::ISD::SRA, and llvm::ISD::SRL.
Referenced by getAArch64Cmp().
Definition at line 4329 of file AArch64ISelLowering.cpp.
References llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::EVT::isFixedLengthVector(), and llvm::None.
|
static |
Definition at line 22227 of file AArch64ISelLowering.cpp.
References assert(), llvm::MVT::f16, llvm::MVT::f32, llvm::MVT::f64, llvm::EVT::getSimpleVT(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::EVT::getVectorElementType(), llvm::MVT::i16, llvm::MVT::i32, llvm::MVT::i64, llvm::MVT::i8, llvm::EVT::isFixedLengthVector(), llvm::TargetLoweringBase::isTypeLegal(), llvm_unreachable, llvm::MVT::nxv16i8, llvm::MVT::nxv2f64, llvm::MVT::nxv2i64, llvm::MVT::nxv4f32, llvm::MVT::nxv4i32, llvm::MVT::nxv8f16, llvm::MVT::nxv8i16, and llvm::MVT::SimpleTy.
Referenced by convertFixedMaskToScalableVector().
|
static |
Definition at line 16946 of file AArch64ISelLowering.cpp.
References llvm::AArch64CC::AL, CC, llvm::AArch64ISD::CSEL, llvm::AArch64CC::getInvertedCondCode(), llvm::isNullConstant(), llvm::isOneConstant(), llvm::None, and llvm::AArch64CC::NV.
Referenced by foldOverflowCheck().
|
static |
Definition at line 11043 of file AArch64ISelLowering.cpp.
References llvm::MVT::bf16, llvm::AArch64ISD::DUPLANE16, llvm::AArch64ISD::DUPLANE32, llvm::AArch64ISD::DUPLANE64, llvm::AArch64ISD::DUPLANE8, llvm::MVT::f16, llvm::MVT::f32, llvm::MVT::f64, llvm::MVT::i16, llvm::MVT::i32, llvm::MVT::i64, llvm::MVT::i8, and llvm_unreachable.
|
static |
Definition at line 9570 of file AArch64ISelLowering.cpp.
References llvm::MVT::f32, llvm::MVT::f64, llvm::SelectionDAG::getNode(), llvm::EVT::getScalarType(), llvm::SDValue::getValueType(), llvm::MVT::nxv2f64, llvm::MVT::nxv4f32, llvm::MVT::nxv8f16, llvm::ARM_MB::ST, llvm::MVT::v1f32, llvm::MVT::v1f64, llvm::MVT::v2f32, llvm::MVT::v2f64, and llvm::MVT::v4f32.
Definition at line 4293 of file AArch64ISelLowering.cpp.
References assert(), llvm::EVT::getSimpleVT(), llvm::EVT::getSizeInBits(), llvm::EVT::isSimple(), llvm_unreachable, llvm::MVT::SimpleTy, llvm::MVT::v2i16, llvm::MVT::v2i32, llvm::MVT::v2i8, llvm::MVT::v4i16, and llvm::MVT::v4i8.
Referenced by addRequiredExtensionForVectorMULL().
|
static |
getExtFactor - Determine the adjustment factor for the position when generating an "extract from vector registers" instruction.
Definition at line 10079 of file AArch64ISelLowering.cpp.
References llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), and llvm::EVT::getVectorElementType().
Referenced by GeneratePerfectShuffle(), and llvm::AArch64TargetLowering::ReconstructShuffle().
| unsigned getGatherVecOpcode | ( | bool | IsScaled, |
| bool | IsSigned, | ||
| bool | NeedsExtend | ||
| ) |
Definition at line 5161 of file AArch64ISelLowering.cpp.
References llvm::AArch64ISD::GLD1_MERGE_ZERO, llvm::AArch64ISD::GLD1_SCALED_MERGE_ZERO, llvm::AArch64ISD::GLD1_SXTW_MERGE_ZERO, llvm::AArch64ISD::GLD1_SXTW_SCALED_MERGE_ZERO, llvm::AArch64ISD::GLD1_UXTW_MERGE_ZERO, and llvm::AArch64ISD::GLD1_UXTW_SCALED_MERGE_ZERO.
Referenced by performGLD1Combine().
Definition at line 5971 of file AArch64ISelLowering.cpp.
References llvm::ISD::INTRINSIC_WO_CHAIN, N, and llvm::Intrinsic::not_intrinsic.
Referenced by collectBitParts(), FindPreallocatedCall(), llvm::VPRecipeBuilder::handleReplication(), isKnownNonZero(), llvm::AArch64TargetLowering::isReassocProfitable(), IsSVECntIntrinsic(), performIntrinsicCombine(), and tryConvertSVEWideCompare().
|
static |
Definition at line 16847 of file AArch64ISelLowering.cpp.
References DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), and llvm::ISD::SUB.
Referenced by performNegCSelCombine().
|
inlinestatic |
Definition at line 167 of file AArch64ISelLowering.cpp.
References llvm_unreachable, llvm::MVT::nxv16i8, llvm::MVT::nxv2i64, llvm::MVT::nxv4i32, and llvm::MVT::nxv8i16.
Definition at line 142 of file AArch64ISelLowering.cpp.
References llvm::MVT::bf16, llvm::MVT::f16, llvm::MVT::f32, llvm::MVT::f64, llvm::EVT::getSimpleVT(), llvm::MVT::i16, llvm::MVT::i32, llvm::MVT::i64, llvm::MVT::i8, llvm_unreachable, llvm::MVT::nxv16i8, llvm::MVT::nxv2f64, llvm::MVT::nxv2i64, llvm::MVT::nxv4f32, llvm::MVT::nxv4i32, llvm::MVT::nxv8bf16, llvm::MVT::nxv8f16, llvm::MVT::nxv8i16, and llvm::MVT::SimpleTy.
Referenced by combineSVEReductionInt(), performDupLane128Combine(), and performSVESpliceCombine().
|
static |
Definition at line 22252 of file AArch64ISelLowering.cpp.
References llvm::LegalityPredicates::all(), assert(), DL, llvm::MVT::f16, llvm::MVT::f32, llvm::MVT::f64, llvm::AArch64Subtarget::getMinSVEVectorSizeInBits(), getPTrue(), llvm::EVT::getSimpleVT(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getSubtarget(), llvm::getSVEPredPatternFromNumElements(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::MVT::i16, llvm::MVT::i32, llvm::MVT::i64, llvm::MVT::i8, llvm::EVT::isFixedLengthVector(), llvm::TargetLoweringBase::isTypeLegal(), llvm_unreachable, llvm::MVT::nxv16i1, llvm::MVT::nxv2i1, llvm::MVT::nxv4i1, llvm::MVT::nxv8i1, and llvm::MVT::SimpleTy.
Referenced by convertFixedMaskToScalableVector(), and getPredicateForVector().
|
static |
Definition at line 22296 of file AArch64ISelLowering.cpp.
References llvm::LegalityPredicates::all(), assert(), llvm::EVT::changeVectorElementType(), DL, getPTrue(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::MVT::i1, llvm::EVT::isScalableVector(), and llvm::TargetLoweringBase::isTypeLegal().
Referenced by getPredicateForVector().
|
static |
Definition at line 22304 of file AArch64ISelLowering.cpp.
References DL, getPredicateForFixedLengthVector(), getPredicateForScalableVector(), and llvm::EVT::isFixedLengthVector().
Definition at line 182 of file AArch64ISelLowering.cpp.
References assert(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorMinNumElements(), llvm::MVT::i1, llvm::EVT::isScalableVector(), llvm_unreachable, llvm::MVT::nxv16i8, llvm::MVT::nxv2i64, llvm::MVT::nxv4i32, and llvm::MVT::nxv8i16.
Referenced by llvm::AArch64TargetLowering::getPromotedVTForPredicate(), and performIntrinsicCombine().
|
static |
Definition at line 17453 of file AArch64ISelLowering.cpp.
References llvm::AArch64CC::ANY_ACTIVE, assert(), CC, Cond, llvm::AArch64ISD::CSEL, DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::AArch64CC::getInvertedCondCode(), llvm::SelectionDAG::getNode(), getSVEPredicateBitCast(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::TargetLoweringBase::getTypeToTransformTo(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::MVT::i32, llvm::TargetLoweringBase::isTypeLegal(), isZeroingInactiveLanes(), llvm::AArch64CC::NONE_ACTIVE, llvm::MVT::nxv16i1, llvm::MVT::Other, llvm::AArch64ISD::PTEST, llvm::AArch64ISD::PTEST_ANY, llvm::AArch64ISD::REINTERPRET_CAST, and llvm::Test.
Referenced by performFirstTrueTestVectorCombine(), performIntrinsicCombine(), and performLastTrueTestVectorCombine().
|
inlinestatic |
Definition at line 4579 of file AArch64ISelLowering.cpp.
References llvm::LegalityPredicates::all(), DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetConstant(), llvm::MVT::i32, llvm::MVT::nxv1i1, and llvm::AArch64ISD::PTRUE.
Referenced by getPredicateForFixedLengthVector(), getPredicateForScalableVector(), performFirstTrueTestVectorCombine(), performLastTrueTestVectorCombine(), performMSTORECombine(), and performUnpackCombine().
|
static |
Definition at line 12975 of file AArch64ISelLowering.cpp.
References DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::SDValue::getValueType(), and llvm::MVT::i64.
|
static |
Definition at line 20259 of file AArch64ISelLowering.cpp.
References assert(), llvm::BitWidth, DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::MVT::i64, llvm::Log2_32(), llvm::MVT::nxv2i64, Shift, llvm::ISD::SHL, and llvm::ISD::SPLAT_VECTOR.
Referenced by performGatherLoadCombine(), and performScatterStoreCombine().
| unsigned getSignExtendedGatherOpcode | ( | unsigned | Opcode | ) |
Definition at line 5184 of file AArch64ISelLowering.cpp.
References llvm::AArch64ISD::GLD1_IMM_MERGE_ZERO, llvm::AArch64ISD::GLD1_MERGE_ZERO, llvm::AArch64ISD::GLD1_SCALED_MERGE_ZERO, llvm::AArch64ISD::GLD1_SXTW_MERGE_ZERO, llvm::AArch64ISD::GLD1_SXTW_SCALED_MERGE_ZERO, llvm::AArch64ISD::GLD1_UXTW_MERGE_ZERO, llvm::AArch64ISD::GLD1_UXTW_SCALED_MERGE_ZERO, llvm::AArch64ISD::GLD1S_IMM_MERGE_ZERO, llvm::AArch64ISD::GLD1S_MERGE_ZERO, llvm::AArch64ISD::GLD1S_SCALED_MERGE_ZERO, llvm::AArch64ISD::GLD1S_SXTW_MERGE_ZERO, llvm::AArch64ISD::GLD1S_SXTW_SCALED_MERGE_ZERO, llvm::AArch64ISD::GLD1S_UXTW_MERGE_ZERO, llvm::AArch64ISD::GLD1S_UXTW_SCALED_MERGE_ZERO, and llvm_unreachable.
Referenced by performGLD1Combine().
|
static |
Definition at line 13975 of file AArch64ISelLowering.cpp.
References llvm::ScalableVectorType::get(), llvm::Type::getBFloatTy(), llvm::Type::getContext(), llvm::Type::getDoubleTy(), llvm::VectorType::getElementType(), llvm::Type::getFloatTy(), llvm::Type::getHalfTy(), llvm::Type::getInt16Ty(), llvm::Type::getInt32Ty(), llvm::Type::getInt64Ty(), llvm::Type::getInt8Ty(), and llvm_unreachable.
Referenced by llvm::AArch64TargetLowering::lowerInterleavedLoad(), and llvm::AArch64TargetLowering::lowerInterleavedStore().
Definition at line 18016 of file AArch64ISelLowering.cpp.
References assert(), llvm::EVT::getSimpleVT(), llvm::EVT::isSimple(), llvm_unreachable, llvm::MVT::nxv16i8, llvm::MVT::nxv2f32, llvm::MVT::nxv2f64, llvm::MVT::nxv2i16, llvm::MVT::nxv2i32, llvm::MVT::nxv2i64, llvm::MVT::nxv2i8, llvm::MVT::nxv4f32, llvm::MVT::nxv4i16, llvm::MVT::nxv4i32, llvm::MVT::nxv4i8, llvm::MVT::nxv8bf16, llvm::MVT::nxv8f16, llvm::MVT::nxv8i16, llvm::MVT::nxv8i8, and llvm::MVT::SimpleTy.
Referenced by performGatherLoadCombine(), performLD1Combine(), performScatterStoreCombine(), and performST1Combine().
|
static |
Definition at line 4589 of file AArch64ISelLowering.cpp.
References llvm::ISD::AND, assert(), llvm::EVT::bitsGT(), DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::EVT::getVectorElementType(), llvm::MVT::i1, llvm::EVT::isScalableVector(), llvm::TargetLoweringBase::isTypeLegal(), isZeroingInactiveLanes(), llvm::BitmaskEnumDetail::Mask(), and llvm::AArch64ISD::REINTERPRET_CAST.
Referenced by getPTest().
|
static |
Definition at line 19901 of file AArch64ISelLowering.cpp.
References llvm::ISD::AND, llvm::ISD::ANY_EXTEND, llvm::tgtok::Bit, llvm::ISD::SHL, llvm::ISD::SRA, llvm::ISD::SRL, llvm::ISD::TRUNCATE, and llvm::ISD::XOR.
Referenced by performTBZCombine().
|
static |
getVShiftImm - Check if this is a valid build_vector for the immediate operand of a vector shift operation, where all the elements of the build_vector must have the same constant integer value.
Definition at line 12683 of file AArch64ISelLowering.cpp.
References llvm::ISD::BITCAST, llvm::APInt::getSExtValue(), and llvm::BuildVectorSDNode::isConstantSplat().
Referenced by isVShiftLImm(), and isVShiftRImm().
|
static |
Definition at line 16026 of file AArch64ISelLowering.cpp.
References llvm::ISD::ADD, llvm::MVT::f16, llvm::MVT::f32, llvm::MVT::f64, llvm::ISD::FADD, llvm::MVT::i64, and llvm::ISD::STRICT_FADD.
Referenced by performExtractVectorEltCombine().
|
static |
Definition at line 4405 of file AArch64ISelLowering.cpp.
References llvm::ISD::ADD, llvm::SDNode::hasOneUse(), isSignExtended(), N, and llvm::ISD::SUB.
Referenced by selectUmullSmull().
|
static |
Definition at line 4416 of file AArch64ISelLowering.cpp.
References llvm::ISD::ADD, llvm::SDNode::hasOneUse(), isZeroExtended(), N, and llvm::ISD::SUB.
Referenced by selectUmullSmull().
|
static |
Definition at line 17558 of file AArch64ISelLowering.cpp.
References llvm::LegalityPredicates::all(), llvm::AArch64Subtarget::getMinSVEVectorSizeInBits(), llvm::getNumElementsFromSVEPredPattern(), llvm::SelectionDAG::getSubtarget(), llvm::ISD::isConstantSplatVectorAllOnes(), N, llvm::AArch64ISD::PTRUE, llvm::AArch64ISD::REINTERPRET_CAST, and llvm::AArch64::SVEBitsPerBlock.
Referenced by convertMergedOpToPredOp(), llvm::AArch64TargetLowering::isAllActivePredicate(), performSetccMergeZeroCombine(), and performVSelectCombine().
Definition at line 11719 of file AArch64ISelLowering.cpp.
References llvm::SDNode::getOperand(), llvm::SDNode::getValueType(), llvm::EVT::getVectorNumElements(), llvm::ConstantSDNode::getZExtValue(), and i.
Referenced by isPow2Splat(), and tryLowerToSLI().
|
static |
Definition at line 17550 of file AArch64ISelLowering.cpp.
References llvm::ISD::isConstantSplatVectorAllZeros(), N, and llvm::AArch64ISD::REINTERPRET_CAST.
Referenced by performVSelectCombine().
Definition at line 17901 of file AArch64ISelLowering.cpp.
References llvm::ISD::isConstantSplatVectorAllZeros(), llvm::ISD::LOAD, llvm::ISD::MLOAD, N, and llvm::SystemZISD::OC.
Referenced by performSignExtendSetCCCombine().
|
static |
Definition at line 2989 of file AArch64ISelLowering.cpp.
References CC, llvm::isNullConstant(), llvm::ISD::SETEQ, llvm::ISD::SETNE, and llvm::ISD::SUB.
Referenced by emitComparison(), getAArch64Cmp(), and trySwapICmpOperands().
|
static |
Definition at line 16939 of file AArch64ISelLowering.cpp.
References llvm::AArch64ISD::SUBS.
Referenced by foldCSELOfCSEL(), and foldOverflowCheck().
Definition at line 10747 of file AArch64ISelLowering.cpp.
References E, llvm::EVT::getSizeInBits(), llvm::EVT::getVectorNumElements(), I, and llvm::BitmaskEnumDetail::Mask().
Referenced by llvm::AArch64TargetLowering::isShuffleMaskLegal(), and tryFormConcatFromShuffle().
Definition at line 15856 of file AArch64ISelLowering.cpp.
References llvm::AArch64ISD::DUP, llvm::EVT::getSimpleVT(), llvm::EVT::getVectorElementType(), llvm::MVT::i16, llvm::MVT::i32, llvm::MVT::i8, llvm::EVT::isSimple(), N, llvm::MVT::SimpleTy, and llvm::ISD::SPLAT_VECTOR.
Referenced by performSVEAndCombine().
|
static |
Definition at line 19352 of file AArch64ISelLowering.cpp.
References llvm::AArch64CC::AL, CC, llvm::AArch64CC::EQ, llvm::AArch64CC::GE, llvm::AArch64CC::GT, llvm::AArch64CC::HI, llvm::AArch64CC::HS, llvm::AArch64CC::Invalid, llvm::AArch64CC::LE, llvm::AArch64CC::LO, llvm::AArch64CC::LS, llvm::AArch64CC::LT, llvm::AArch64CC::MI, llvm::AArch64CC::NE, llvm::AArch64CC::NV, llvm::AArch64CC::PL, llvm::ISD::SEXTLOAD, llvm::AArch64CC::VC, and llvm::AArch64CC::VS.
Referenced by performCONDCombine().
|
static |
Definition at line 16562 of file AArch64ISelLowering.cpp.
References llvm::ISD::BITCAST, llvm::ISD::EXTRACT_SUBVECTOR, and N.
Referenced by performAddSubLongCombine(), and tryCombineLongOpWithDup().
|
static |
Definition at line 4342 of file AArch64ISelLowering.cpp.
References llvm::ISD::BUILD_VECTOR, llvm::EVT::getScalarSizeInBits(), llvm::isIntN(), isSigned(), llvm::isUIntN(), and N.
Referenced by isSignExtended(), and isZeroExtended().
|
static |
Definition at line 17087 of file AArch64ISelLowering.cpp.
References llvm::ISD::AND, llvm::ISD::ANY_EXTEND, llvm::ConstantSDNode::getZExtValue(), llvm::MVT::i16, llvm::MVT::i32, llvm::MVT::i8, N, llvm::ISD::SHL, llvm::ISD::SIGN_EXTEND, llvm::ISD::SIGN_EXTEND_INREG, llvm::ISD::SRA, llvm::ISD::SRL, and llvm::ISD::ZERO_EXTEND.
Referenced by performAddCombineSubShift().
Definition at line 10543 of file AArch64ISelLowering.cpp.
References llvm::any_of(), llvm::find_if(), llvm::EVT::getVectorNumElements(), llvm::APInt::getZExtValue(), llvm::RISCVMatInt::Imm, llvm::APInt::logBase2(), and M.
Referenced by llvm::AArch64TargetLowering::isShuffleMaskLegal().
|
static |
Definition at line 10708 of file AArch64ISelLowering.cpp.
Referenced by llvm::AArch64TargetLowering::isShuffleMaskLegal().
Definition at line 1837 of file AArch64ISelLowering.cpp.
References llvm::RISCVMatInt::Imm, and N.
Referenced by isOpcWithIntImmediate().
|
static |
Definition at line 20686 of file AArch64ISelLowering.cpp.
References llvm::AArch64ISD::ANDV_PRED, llvm::AArch64ISD::EORV_PRED, llvm::AArch64ISD::FADDA_PRED, llvm::AArch64ISD::FADDV_PRED, llvm::AArch64ISD::FMAXNMV_PRED, llvm::AArch64ISD::FMAXV_PRED, llvm::AArch64ISD::FMINNMV_PRED, llvm::AArch64ISD::FMINV_PRED, llvm::AArch64ISD::ORV_PRED, llvm::AArch64ISD::SADDV_PRED, llvm::AArch64ISD::SMAXV_PRED, llvm::AArch64ISD::SMINV_PRED, llvm::AArch64ISD::UADDV_PRED, llvm::AArch64ISD::UMAXV_PRED, and llvm::AArch64ISD::UMINV_PRED.
Referenced by removeRedundantInsertVectorElt().
|
static |
Definition at line 2971 of file AArch64ISelLowering.cpp.
References llvm::dbgs(), and LLVM_DEBUG.
Referenced by getAArch64Cmp(), tryAdjustICmpImmAndPred(), and trySwapICmpOperands().
|
static |
Definition at line 213 of file AArch64ISelLowering.cpp.
References llvm::AArch64ISD::ABS_MERGE_PASSTHRU, llvm::AArch64ISD::BITREVERSE_MERGE_PASSTHRU, llvm::AArch64ISD::BSWAP_MERGE_PASSTHRU, llvm::AArch64ISD::CTLZ_MERGE_PASSTHRU, llvm::AArch64ISD::CTPOP_MERGE_PASSTHRU, llvm::AArch64ISD::DUP_MERGE_PASSTHRU, llvm::AArch64ISD::FABS_MERGE_PASSTHRU, llvm::AArch64ISD::FCEIL_MERGE_PASSTHRU, llvm::AArch64ISD::FCVTZS_MERGE_PASSTHRU, llvm::AArch64ISD::FCVTZU_MERGE_PASSTHRU, llvm::AArch64ISD::FFLOOR_MERGE_PASSTHRU, llvm::AArch64ISD::FNEARBYINT_MERGE_PASSTHRU, llvm::AArch64ISD::FNEG_MERGE_PASSTHRU, llvm::AArch64ISD::FP_EXTEND_MERGE_PASSTHRU, llvm::AArch64ISD::FP_ROUND_MERGE_PASSTHRU, llvm::AArch64ISD::FRECPX_MERGE_PASSTHRU, llvm::AArch64ISD::FRINT_MERGE_PASSTHRU, llvm::AArch64ISD::FROUND_MERGE_PASSTHRU, llvm::AArch64ISD::FROUNDEVEN_MERGE_PASSTHRU, llvm::AArch64ISD::FSQRT_MERGE_PASSTHRU, llvm::AArch64ISD::FTRUNC_MERGE_PASSTHRU, llvm::AArch64ISD::NEG_MERGE_PASSTHRU, llvm::AArch64ISD::REVD_MERGE_PASSTHRU, llvm::AArch64ISD::REVH_MERGE_PASSTHRU, llvm::AArch64ISD::REVW_MERGE_PASSTHRU, llvm::AArch64ISD::SIGN_EXTEND_INREG_MERGE_PASSTHRU, llvm::AArch64ISD::SINT_TO_FP_MERGE_PASSTHRU, llvm::AArch64ISD::UINT_TO_FP_MERGE_PASSTHRU, and llvm::AArch64ISD::ZERO_EXTEND_INREG_MERGE_PASSTHRU.
|
static |
Definition at line 16843 of file AArch64ISelLowering.cpp.
References llvm::isNullConstant(), and llvm::ISD::SUB.
Referenced by performNegCSelCombine().
Definition at line 1848 of file AArch64ISelLowering.cpp.
References llvm::RISCVMatInt::Imm, isIntImmediate(), and N.
Referenced by performAddCombineForShiftedOperands().
|
static |
Check if Op could be used with vmull_high_p64 intrinsic.
Definition at line 13593 of file AArch64ISelLowering.cpp.
References llvm::Value::getType(), llvm::ConstantInt::getValue(), llvm::PatternMatch::m_ConstantInt(), llvm::PatternMatch::m_ExtractElt(), llvm::PatternMatch::m_Value(), and llvm::PatternMatch::match().
Referenced by areOperandsOfVmullHighP64().
|
static |
Definition at line 8601 of file AArch64ISelLowering.cpp.
References MaxXors, N, llvm::ISD::OR, llvm::ISD::XOR, and llvm::ISD::ZERO_EXTEND.
Referenced by performOrXorChainCombine().
|
inlinestatic |
Returns true if VT's elements occupy the lowest bit positions of its associated register class without any intervening space.
For example, nxv2f16, nxv4f16 and nxv8f16 are legal types that belong to the same register class, but only nxv8f16 can be treated as a packed vector.
Definition at line 204 of file AArch64ISelLowering.cpp.
References assert(), llvm::TypeSize::getKnownMinSize(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::EVT::isFixedLengthVector(), llvm::TargetLoweringBase::isTypeLegal(), llvm::EVT::isVector(), and llvm::AArch64::SVEBitsPerBlock.
Definition at line 12564 of file AArch64ISelLowering.cpp.
References llvm::ISD::BUILD_VECTOR, llvm::AArch64ISD::DUP, llvm::MVT::i64, isAllConstantBuildVector(), llvm::isPowerOf2_64(), and llvm::ISD::SPLAT_VECTOR.
|
static |
Definition at line 16041 of file AArch64ISelLowering.cpp.
References llvm::ISD::INTRINSIC_WO_CHAIN, N, and llvm::ISD::SETCC.
Referenced by performFirstTrueTestVectorCombine().
isREVMask - Check if a vector shuffle corresponds to a REV instruction with the specified blocksize.
(The order of the elements within each block of the vector is reversed.)
Definition at line 10585 of file AArch64ISelLowering.cpp.
References assert(), BlockSize, llvm::EVT::getScalarSizeInBits(), llvm::EVT::getVectorNumElements(), i, and M.
Referenced by llvm::AArch64TargetLowering::isShuffleMaskLegal().
|
static |
Check whether or not Op is a SET_CC operation, either a generic or an AArch64 lowered one.
SetCCInfo is filled accordingly.
Definition at line 16606 of file AArch64ISelLowering.cpp.
References SetCCInfo::AArch64, GenericSetCCInfo::CC, AArch64SetCCInfo::CC, AArch64SetCCInfo::Cmp, llvm::AArch64ISD::CSEL, SetCCInfo::Generic, llvm::AArch64CC::getInvertedCondCode(), llvm::ConstantSDNode::isOne(), llvm::ConstantSDNode::isZero(), GenericSetCCInfo::Opnd0, GenericSetCCInfo::Opnd1, llvm::ISD::SETCC, and std::swap().
Referenced by isSetCCOrZExtSetCC().
|
static |
Definition at line 16649 of file AArch64ISelLowering.cpp.
References Info, isSetCC(), and llvm::ISD::ZERO_EXTEND.
Referenced by performSetccAddFolding().
|
static |
Definition at line 4393 of file AArch64ISelLowering.cpp.
References llvm::ISD::ANY_EXTEND, isExtendedBUILD_VECTOR(), N, and llvm::ISD::SIGN_EXTEND.
Referenced by isAddSubSExt(), performMulCombine(), and selectUmullSmull().
Definition at line 10387 of file AArch64ISelLowering.cpp.
References llvm::EVT::getVectorNumElements(), i, llvm::RISCVMatInt::Imm, and M.
|
static |
Definition at line 13513 of file AArch64ISelLowering.cpp.
References llvm::all_equal().
Referenced by areExtractShuffleVectors(), and llvm::AArch64TargetLowering::shouldSinkOperands().
|
static |
Definition at line 15015 of file AArch64ISelLowering.cpp.
References getIntrinsicID(), and S.
Referenced by performMulCombine().
isTRN_v_undef_Mask - Special case of isTRNMask for canonical form of "vector_shuffle v, v", i.e., "vector_shuffle v, undef".
Mask is e.g., <0, 0, 2, 2> instead of <0, 4, 2, 6>.
Definition at line 10695 of file AArch64ISelLowering.cpp.
References llvm::EVT::getVectorNumElements(), i, and M.
Referenced by llvm::AArch64TargetLowering::isShuffleMaskLegal().
Definition at line 10641 of file AArch64ISelLowering.cpp.
References llvm::EVT::getVectorNumElements(), i, and M.
Referenced by llvm::AArch64TargetLowering::isShuffleMaskLegal().
isUZP_v_undef_Mask - Special case of isUZPMask for canonical form of "vector_shuffle v, v", i.e., "vector_shuffle v, undef".
Mask is e.g., <0, 2, 0, 2> instead of <0, 2, 4, 6>,
Definition at line 10676 of file AArch64ISelLowering.cpp.
References llvm::EVT::getVectorNumElements(), i, j(), and M.
Referenced by llvm::AArch64TargetLowering::isShuffleMaskLegal().
Definition at line 10628 of file AArch64ISelLowering.cpp.
References llvm::EVT::getVectorNumElements(), i, and M.
Referenced by llvm::AArch64TargetLowering::isShuffleMaskLegal().
|
static |
Check if the value of Offset represents a valid immediate for the SVE gather load/prefetch and scatter store instructiona with vector base and immediate offset addressing mode:
[<Zn>.[S|D]{, #<imm>}]
where <imm> = sizeof(<T>) * k, for k = 0, 1, ..., 31.
Definition at line 20297 of file AArch64ISelLowering.cpp.
References llvm::ConstantSDNode::getZExtValue(), and isValidImmForSVEVecImmAddrMode().
|
inlinestatic |
Check if the value of OffsetInBytes can be used as an immediate for the gather load/prefetch and scatter store instructions with vector base and immediate offset addressing mode:
[<Zn>.[S|D]{, #<imm>}]
where <imm> = sizeof(<T>) * k, for k = 0, 1, ..., 31.
Definition at line 20277 of file AArch64ISelLowering.cpp.
Referenced by combineSVEPrefetchVecBaseImmOff(), isValidImmForSVEVecImmAddrMode(), performGatherLoadCombine(), and performScatterStoreCombine().
isVShiftLImm - Check if this is a valid build_vector for the immediate operand of a vector shift left operation.
That value must be in the range: 0 <= Value < ElementBits for a left shift; or 0 <= Value <= ElementBits for a long left shift.
Definition at line 12703 of file AArch64ISelLowering.cpp.
References assert(), llvm::EVT::getScalarSizeInBits(), getVShiftImm(), and llvm::EVT::isVector().
isVShiftRImm - Check if this is a valid build_vector for the immediate operand of a vector shift right operation.
The value must be in the range: 1 <= Value <= ElementBits for a right shift; or
Definition at line 12714 of file AArch64ISelLowering.cpp.
References assert(), llvm::EVT::getScalarSizeInBits(), getVShiftImm(), and llvm::EVT::isVector().
|
static |
Check if a vector shuffle corresponds to a DUP instructions with a larger element width than the vector lane type.
If that is the case the function returns true and writes the value of the DUP instruction lane operand into DupLaneOp
Definition at line 10470 of file AArch64ISelLowering.cpp.
References assert(), BlockSize, llvm::find_if(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getSizeInBits(), llvm::EVT::getVectorNumElements(), I, and M.
|
static |
Definition at line 11118 of file AArch64ISelLowering.cpp.
References assert(), llvm::SmallVectorImpl< T >::clear(), llvm::EVT::getVectorNumElements(), i, M, llvm::M0(), and llvm::M1().
Referenced by tryWidenMaskForShuffle().
|
static |
Definition at line 4399 of file AArch64ISelLowering.cpp.
References llvm::ISD::ANY_EXTEND, isExtendedBUILD_VECTOR(), N, and llvm::ISD::ZERO_EXTEND.
Referenced by isAddSubZExt(), performMulCombine(), and selectUmullSmull().
|
static |
Definition at line 251 of file AArch64ISelLowering.cpp.
References llvm::ISD::INTRINSIC_WO_CHAIN, llvm::ISD::isConstantSplatVectorAllOnes(), llvm::AArch64ISD::PTRUE, and llvm::AArch64ISD::SETCC_MERGE_ZERO.
Referenced by getPTest(), and getSVEPredicateBitCast().
isZerosVector - Check whether SDNode N is a zero-filled vector.
Definition at line 2802 of file AArch64ISelLowering.cpp.
References llvm::ISD::BITCAST, llvm::AArch64ISD::DUP, llvm::ISD::isConstantSplatVectorAllZeros(), llvm::isNullConstant(), llvm::isNullFPConstant(), and N.
Referenced by GenerateTBL(), performAddDotCombine(), performSetccMergeZeroCombine(), performSetCCPunpkCombine(), and performUnpackCombine().
isZIP_v_undef_Mask - Special case of isZIPMask for canonical form of "vector_shuffle v, v", i.e., "vector_shuffle v, undef".
Mask is e.g., <0, 0, 1, 1> instead of <0, 4, 1, 5>.
Definition at line 10657 of file AArch64ISelLowering.cpp.
References llvm::EVT::getVectorNumElements(), i, and M.
Referenced by llvm::AArch64TargetLowering::isShuffleMaskLegal().
Definition at line 10612 of file AArch64ISelLowering.cpp.
References llvm::EVT::getVectorNumElements(), i, and M.
Referenced by llvm::AArch64TargetLowering::isShuffleMaskLegal().
|
static |
Legalize the gather prefetch (scalar + vector addressing mode) when the offset vector is an unpacked 32-bit scalable vector.
The other cases (Offset != nxv2i32) do not need legalization.
Definition at line 20642 of file AArch64ISelLowering.cpp.
References llvm::ISD::ANY_EXTEND, DL, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getVTList(), N, llvm::MVT::nxv2i32, llvm::MVT::nxv2i64, and llvm::MVT::Other.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
Definition at line 8164 of file AArch64ISelLowering.cpp.
References llvm::EVT::getFixedSizeInBits(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValueSizeInBits(), llvm::SDNode::getValueType(), llvm::ISD::SIGN_EXTEND, and llvm::ISD::SIGN_EXTEND_INREG.
|
static |
Definition at line 3732 of file AArch64ISelLowering.cpp.
References carryFlagToValue(), DL, llvm::SelectionDAG::getNode(), llvm::SDValue::getValue(), llvm::SelectionDAG::getVTList(), llvm::MVT::Glue, llvm::MVT::i32, llvm::MVT::i64, llvm::ISD::MERGE_VALUES, overflowFlagToValue(), llvm::AArch64ISD::SBCS, and valueToCarryFlag().
Referenced by llvm::AArch64TargetLowering::LowerOperation().
|
static |
Definition at line 5623 of file AArch64ISelLowering.cpp.
References llvm::AArch64ISD::BRCOND, CC, Cond, emitConjunction(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::MVT::i32, and llvm::MVT::Other.
Referenced by llvm::AArch64TargetLowering::LowerOperation().
|
static |
Definition at line 3789 of file AArch64ISelLowering.cpp.
References assert(), DL, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetConstant(), llvm::MVT::i32, llvm::MVT::Other, and llvm::AArch64ISD::PREFETCH.
Referenced by llvm::AArch64TargetLowering::LowerOperation().
|
static |
Definition at line 17349 of file AArch64ISelLowering.cpp.
References llvm::ISD::ANY_EXTEND, llvm::AArch64ISD::DUP_MERGE_PASSTHRU, llvm::SelectionDAG::getNode(), llvm::MVT::i16, llvm::MVT::i32, llvm::MVT::i8, and N.
Referenced by performIntrinsicCombine().
|
static |
Definition at line 17363 of file AArch64ISelLowering.cpp.
References assert(), llvm::ISD::BITCAST, llvm::AArch64ISD::EXT, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::TypeSize::getKnownMinSize(), llvm::SelectionDAG::getNode(), llvm::LinearPolySize< ElementCount >::getScalable(), llvm::EVT::getSizeInBits(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorVT(), llvm::MVT::i32, llvm::MVT::i8, llvm::EVT::isScalableVector(), llvm::ISD::MUL, N, and llvm::AArch64::SVEBitsPerBlock.
Referenced by performIntrinsicCombine().
|
static |
Definition at line 17333 of file AArch64ISelLowering.cpp.
References llvm::ISD::ADD, llvm::sampleprof::Base, DL, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getStepVector(), llvm::SDValue::getValueType(), llvm::MVT::i16, llvm::MVT::i32, llvm::MVT::i8, Mul, llvm::ISD::MUL, N, and llvm::ISD::SPLAT_VECTOR.
Referenced by performIntrinsicCombine().
|
static |
Definition at line 5415 of file AArch64ISelLowering.cpp.
References assert(), llvm::ISD::BITCAST, llvm::ISD::CONCAT_VECTORS, DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getStore(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::i16, llvm::MVT::i32, llvm::MVT::i64, llvm::EVT::isVector(), llvm::ARM_MB::ST, llvm::ISD::TRUNCATE, llvm::RegState::Undef, llvm::MVT::v2i32, llvm::MVT::v4i16, llvm::MVT::v4i8, llvm::MVT::v8i16, and llvm::MVT::v8i8.
|
static |
Definition at line 3758 of file AArch64ISelLowering.cpp.
References CC, llvm::AArch64ISD::CSEL, getAArch64XALUOOp(), llvm::SelectionDAG::getConstant(), llvm::AArch64CC::getInvertedCondCode(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getVTList(), llvm::MVT::i32, llvm::TargetLoweringBase::isTypeLegal(), and llvm::ISD::MERGE_VALUES.
Referenced by llvm::AArch64TargetLowering::LowerOperation().
|
static |
Return true if we might ever do TCO for calls with this calling convention.
Definition at line 6639 of file AArch64ISelLowering.cpp.
References llvm::CallingConv::AArch64_SVE_VectorCall, llvm::CallingConv::C, CC, llvm::CallingConv::Fast, llvm::CallingConv::PreserveMost, llvm::CallingConv::Swift, llvm::CallingConv::SwiftTail, and llvm::CallingConv::Tail.
|
static |
NarrowVector - Given a value in the V128 register class, produce the equivalent value in the V64 register class.
Definition at line 10086 of file AArch64ISelLowering.cpp.
References DL, llvm::EVT::getSimpleVT(), llvm::SelectionDAG::getTargetExtractSubreg(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), and llvm::MVT::getVectorVT().
|
static |
Definition at line 11871 of file AArch64ISelLowering.cpp.
References assert(), llvm::ISD::BUILD_VECTOR, llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::EVT::getVectorElementType(), llvm::APInt::getZExtValue(), llvm::MVT::i32, and llvm::EVT::isFloatingPoint().
|
static |
Definition at line 1854 of file AArch64ISelLowering.cpp.
References assert(), llvm::TargetLowering::TargetLoweringOpt::CombineTo(), llvm::TargetLowering::TargetLoweringOpt::DAG, DL, llvm::AArch64_AM::encodeLogicalImmediate(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getMachineNode(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetConstant(), llvm::APInt::getZExtValue(), llvm::RISCVMatInt::Imm, llvm::AArch64_AM::isLogicalImmediate(), llvm::isShiftedMask_64(), and llvm::BitmaskEnumDetail::Mask().
Referenced by llvm::AArch64TargetLowering::targetShrinkDemandedConstant().
|
static |
Definition at line 3721 of file AArch64ISelLowering.cpp.
References assert(), CC, llvm::AArch64ISD::CSEL, DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::MVT::i32, and llvm::AArch64CC::VS.
Referenced by lowerADDSUBCARRY().
|
static |
|
static |
Definition at line 17141 of file AArch64ISelLowering.cpp.
References llvm::ISD::ADD, DL, llvm::SelectionDAG::getNode(), llvm::Value::hasOneUse(), llvm::MVT::i32, llvm::MVT::i64, isOpcWithIntImmediate(), LHS, N, performAddCombineSubShift(), RHS, and llvm::ISD::SHL.
Referenced by performAddSubCombine().
|
static |
Definition at line 17113 of file AArch64ISelLowering.cpp.
References llvm::ISD::ADD, DL, llvm::SelectionDAG::getNode(), isExtendOrShiftOperand(), N, Shift, llvm::ISD::SUB, Y, and llvm::ARCCC::Z.
Referenced by performAddCombineForShiftedOperands().
|
static |
Perform the scalar expression combine in the form of: CSEL(c, 1, cc) + b => CSINC(b+c, b, cc) CSNEG(c, -1, cc) + b => CSINC(b+c, b, cc)
Definition at line 16748 of file AArch64ISelLowering.cpp.
References llvm::ISD::ADD, llvm::ISD::ADDC, assert(), llvm::AArch64ISD::CSEL, llvm::AArch64ISD::CSINC, llvm::AArch64ISD::CSNEG, DL, llvm::SelectionDAG::getAllOnesConstant(), llvm::ConstantSDNode::getAPIntValue(), llvm::SelectionDAG::getConstant(), llvm::AArch64CC::getInvertedCondCode(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::Value::hasOneUse(), llvm::MVT::i32, llvm::ConstantSDNode::isAllOnes(), llvm::TargetLoweringBase::isLegalAddImmediate(), llvm::ConstantSDNode::isOne(), llvm::EVT::isScalarInteger(), LHS, N, RHS, and std::swap().
Referenced by performAddSubCombine().
|
static |
Definition at line 16821 of file AArch64ISelLowering.cpp.
References llvm::ISD::ADD, llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), isZerosVector(), N, llvm::AArch64ISD::SDOT, std::swap(), and llvm::AArch64ISD::UDOT.
Referenced by performAddSubCombine().
|
static |
Definition at line 17180 of file AArch64ISelLowering.cpp.
References N, performAddCombineForShiftedOperands(), performAddCSelIntoCSinc(), performAddDotCombine(), performAddSubLongCombine(), performAddUADDVCombine(), performNegCSelCombine(), and performVectorAddSubExtCombine().
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 16897 of file AArch64ISelLowering.cpp.
References llvm::ISD::ADD, llvm::SelectionDAG::getNode(), llvm::MVT::is128BitVector(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), isEssentiallyExtractHighSubvector(), LHS, N, performSetccAddFolding(), RHS, llvm::ISD::SIGN_EXTEND, tryExtendDUPToExtractHigh(), and llvm::ISD::ZERO_EXTEND.
Referenced by performAddSubCombine().
|
static |
Definition at line 16709 of file AArch64ISelLowering.cpp.
References llvm::ISD::ADD, DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::EVT::getVectorElementType(), llvm::MVT::i64, llvm::EVT::isScalarInteger(), LHS, N, RHS, and llvm::AArch64ISD::UADDV.
Referenced by performAddSubCombine().
|
static |
Definition at line 15974 of file AArch64ISelLowering.cpp.
References llvm::AArch64ISD::BICi, llvm::TargetLowering::DAGCombinerInfo::DAG, llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::EVT::is128BitVector(), llvm::EVT::is64BitVector(), llvm::EVT::isScalableVector(), llvm::TargetLoweringBase::isTypeLegal(), LHS, N, performANDORCSELCombine(), performSVEAndCombine(), resolveBuildVector(), RHS, tryAdvSIMDModImm16(), and tryAdvSIMDModImm32().
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 15766 of file AArch64ISelLowering.cpp.
References llvm::ISD::AND, llvm::AArch64ISD::CCMN, llvm::AArch64ISD::CCMP, llvm::AArch64ISD::CSEL, DL, llvm::SelectionDAG::getConstant(), llvm::SDValue::getConstantOperandVal(), llvm::AArch64CC::getInvertedCondCode(), llvm::SelectionDAG::getNode(), llvm::AArch64CC::getNZCVToSatisfyCondCode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getValueType(), llvm::SDNode::hasOneUse(), llvm::MVT::i32, llvm::isNullConstant(), llvm::isOneConstant(), MVT_CC, N, llvm::AArch64ISD::SUBS, and std::swap().
Referenced by performANDCombine(), and performORCombine().
|
static |
Definition at line 19502 of file AArch64ISelLowering.cpp.
References llvm::AArch64ISD::ADDS, assert(), llvm::ISD::BR, llvm::AArch64ISD::CBNZ, llvm::AArch64ISD::CBZ, CC, llvm::TargetLowering::DAGCombinerInfo::CombineTo(), llvm::AArch64CC::EQ, llvm::MachineFunction::getFunction(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getNode(), llvm::Function::hasFnAttribute(), llvm::MVT::i32, llvm::MVT::i64, llvm::isNullConstant(), LHS, N, llvm::AArch64CC::NE, llvm::MVT::Other, performCONDCombine(), RHS, llvm::ISD::SHL, llvm::ISD::SRA, llvm::ISD::SRL, llvm::AArch64ISD::SUBS, and std::swap().
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 20808 of file AArch64ISelLowering.cpp.
References llvm::ISD::AND, DL, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNOT(), llvm::EVT::isScalableVector(), llvm::BitmaskEnumDetail::Mask(), N, and llvm::ISD::OR.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Combines a buildvector(sext/zext) or shuffle(sext/zext, undef) node pattern into sext/zext(buildvector) or sext/zext(shuffle) making use of the vector SExt/ZExt rather than the scalar SExt/ZExt.
Definition at line 15080 of file AArch64ISelLowering.cpp.
References llvm::ISD::AND, llvm::ISD::AssertSext, llvm::ISD::AssertZext, llvm::ISD::BUILD_VECTOR, calculatePreExtendType(), llvm::EVT::changeVectorElementType(), DL, llvm::drop_begin(), llvm::SelectionDAG::getAnyExtOrTrunc(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::i32, llvm::SDValue::isUndef(), llvm::SDNode::ops(), llvm::MVT::Other, llvm::ISD::SIGN_EXTEND, llvm::ISD::SIGN_EXTEND_INREG, llvm::ISD::VECTOR_SHUFFLE, and llvm::ISD::ZERO_EXTEND.
Referenced by performMulVectorExtendCombine().
|
static |
Definition at line 17039 of file AArch64ISelLowering.cpp.
References llvm::ISD::ANY_EXTEND, llvm::EVT::changeVectorElementType(), DL, llvm::MipsISD::Ext, llvm::ISD::EXTRACT_SUBVECTOR, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SDNode::getConstantOperandVal(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getVectorIdxConstant(), llvm::EVT::getVectorMinNumElements(), llvm::MVT::i32, llvm::TargetLoweringBase::isTypeLegal(), N, and llvm::MVT::v2i32.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 16196 of file AArch64ISelLowering.cpp.
References llvm::all_of(), assert(), llvm::ISD::AVGCEILS, llvm::ISD::AVGCEILU, llvm::ISD::AVGFLOORS, llvm::ISD::AVGFLOORU, llvm::ISD::BITCAST, llvm::ISD::CONCAT_VECTORS, llvm::dbgs(), llvm::AArch64ISD::DUPLANE64, llvm::ISD::EXTRACT_SUBVECTOR, llvm::MVT::f32, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getConstantOperandVal(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getLoad(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNOT(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::MVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::EVT::getVectorVT(), llvm::MVT::getVectorVT(), i, llvm::MVT::i64, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::ISD::isConstantSplatVectorAllOnes(), llvm::SDNode::isOnlyUserOf(), llvm::EVT::isScalableVector(), llvm::SDValue::isUndef(), llvm::MVT::isVector(), llvm::ARM_MB::LD, LLVM_DEBUG, llvm::BitmaskEnumDetail::Mask(), N, llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), RHS, llvm::ISD::TRUNCATE, llvm::MVT::v2i64, llvm::MVT::v4i32, llvm::MVT::v4i8, llvm::MVT::v8i16, WidenVector(), and llvm::ISD::XOR.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 19428 of file AArch64ISelLowering.cpp.
References llvm::ISD::ADD, llvm::ISD::AND, CC, checkValueWidth(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::SDNode::getValueType(), llvm::SelectionDAG::getVTList(), llvm::SDNode::hasAnyUseOfValue(), isEquivalentMaskless(), N, llvm::SelectionDAG::ReplaceAllUsesWith(), and llvm::AArch64ISD::SUBS.
Referenced by performBRCONDCombine(), and performCSELCombine().
|
static |
Definition at line 19658 of file AArch64ISelLowering.cpp.
References foldCSELOfCSEL(), foldCSELofCTTZ(), N, and performCONDCombine().
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 20177 of file AArch64ISelLowering.cpp.
References llvm::TargetLowering::DAGCombinerInfo::DAG, DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::EVT::getDoubleNumVectorElementsVT(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNodeIfExists(), llvm::SelectionDAG::getVTList(), llvm::MVT::i64, llvm::EVT::is64BitVector(), llvm::TargetLowering::DAGCombinerInfo::isAfterLegalizeDAG(), N, and performPostLD1Combine().
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 20828 of file AArch64ISelLowering.cpp.
References llvm::LegacyLegalizeActions::Bitcast, llvm::ISD::BITCAST, DL, llvm::AArch64ISD::DUPLANE128, llvm::SelectionDAG::getNode(), getPackedSVEVectorVT(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), Insert, llvm::ISD::INSERT_SUBVECTOR, llvm::EVT::is128BitVector(), and N.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 17947 of file AArch64ISelLowering.cpp.
References llvm::ISD::ABDS, llvm::ISD::ABDU, llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), N, llvm::Intrinsic::not_intrinsic, performSignExtendSetCCCombine(), llvm::ISD::SETCC, llvm::ISD::SIGN_EXTEND, tryCombineLongOpWithDup(), and llvm::ISD::ZERO_EXTEND.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 16394 of file AArch64ISelLowering.cpp.
References llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getVectorElementType(), llvm::MVT::i1, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::EVT::isScalableVector(), N, and llvm::ISD::SPLAT_VECTOR.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 16126 of file AArch64ISelLowering.cpp.
References assert(), llvm::TargetLowering::DAGCombinerInfo::DAG, DL, llvm::AArch64ISD::DUP, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getSubtarget(), llvm::SDValue::getValue(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::ConstantSDNode::getZExtValue(), llvm::SDValue::hasOneUse(), hasPairwiseAdd(), llvm::MVT::i64, llvm::SDNode::isStrictFPOpcode(), N, Other, performFirstTrueTestVectorCombine(), performLastTrueTestVectorCombine(), llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), llvm::MipsISD::Ret, and llvm::Shuffle.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Fold a floating-point divide by power of two into fixed-point to floating-point conversion.
Definition at line 15531 of file AArch64ISelLowering.cpp.
References DL, llvm::SelectionDAG::getConstant(), llvm::BuildVectorSDNode::getConstantFPSplatPow2ToLog2Int(), llvm::SelectionDAG::getNode(), llvm::MVT::getSizeInBits(), llvm::MVT::i32, llvm::ISD::INTRINSIC_WO_CHAIN, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), N, llvm::ISD::SIGN_EXTEND, llvm::ISD::SINT_TO_FP, llvm::ISD::UINT_TO_FP, llvm::MVT::v2i32, llvm::MVT::v2i64, llvm::MVT::v4i32, llvm::MVT::v4i64, and llvm::ISD::ZERO_EXTEND.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 16062 of file AArch64ISelLowering.cpp.
References llvm::LegalityPredicates::all(), assert(), llvm::TargetLowering::DAGCombinerInfo::DAG, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::AArch64CC::FIRST_ACTIVE, getPTest(), getPTrue(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::MVT::i1, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::isNullConstant(), isPredicateCCSettingOp(), llvm::EVT::isScalableVector(), and N.
Referenced by performExtractVectorEltCombine().
|
static |
Definition at line 19793 of file AArch64ISelLowering.cpp.
References llvm::TargetLowering::DAGCombinerInfo::CombineTo(), llvm::TargetLowering::DAGCombinerInfo::DAG, DL, Generic, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNodeIfExists(), llvm::SelectionDAG::getVTList(), llvm::MVT::i32, LHS, N, and RHS.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 20775 of file AArch64ISelLowering.cpp.
References llvm::TargetLowering::DAGCombinerInfo::CombineTo(), llvm::ISD::EXTLOAD, llvm::ISD::FP_ROUND, llvm::LoadSDNode::getBasePtr(), llvm::MemSDNode::getChain(), llvm::SelectionDAG::getExtLoad(), llvm::EVT::getFixedSizeInBits(), llvm::SelectionDAG::getIntPtrConstant(), llvm::MemSDNode::getMemOperand(), llvm::AArch64Subtarget::getMinSVEVectorSizeInBits(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::SDValue::hasOneUse(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::EVT::isFixedLengthVector(), llvm::ISD::isNormalLoad(), N, and llvm::AArch64Subtarget::useSVEForFixedLengthVectors().
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Fold a floating-point multiply by power of two into floating-point to fixed-point conversion.
Definition at line 15460 of file AArch64ISelLowering.cpp.
References llvm::tgtok::Bits, DL, llvm::ISD::FMUL, llvm::AArch64Subtarget::forceStreamingCompatibleSVE(), llvm::ISD::FP_TO_SINT, llvm::ISD::FP_TO_SINT_SAT, llvm::ISD::FP_TO_UINT_SAT, llvm::SelectionDAG::getConstant(), llvm::BuildVectorSDNode::getConstantFPSplatPow2ToLog2Int(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::MVT::i32, llvm::ISD::INTRINSIC_WO_CHAIN, llvm::TargetLoweringBase::isTypeLegal(), N, and llvm::ISD::TRUNCATE.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 20408 of file AArch64ISelLowering.cpp.
References llvm::ISD::ANY_EXTEND, assert(), llvm::sampleprof::Base, llvm::ISD::BITCAST, DL, llvm::TypeSize::getKnownMinSize(), llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), getScaledOffsetForBitWidth(), llvm::EVT::getSizeInBits(), getSVEContainerType(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValue(), llvm::SelectionDAG::getValueType(), llvm::SelectionDAG::getVTList(), llvm::AArch64ISD::GLD1_IMM_MERGE_ZERO, llvm::AArch64ISD::GLD1_MERGE_ZERO, llvm::AArch64ISD::GLD1_UXTW_MERGE_ZERO, llvm::AArch64ISD::GLDFF1_IMM_MERGE_ZERO, llvm::AArch64ISD::GLDFF1_MERGE_ZERO, llvm::AArch64ISD::GLDFF1_UXTW_MERGE_ZERO, llvm::AArch64ISD::GLDNT1_INDEX_MERGE_ZERO, llvm::AArch64ISD::GLDNT1_MERGE_ZERO, llvm::EVT::isFloatingPoint(), llvm::EVT::isInteger(), llvm::EVT::isScalableVector(), isValidImmForSVEVecImmAddrMode(), llvm::SPII::Load, N, llvm::MVT::nxv2i32, llvm::MVT::nxv2i64, llvm::MVT::nxv4i32, llvm::MVT::Other, llvm::AArch64::SVEBitsPerBlock, std::swap(), and llvm::ISD::TRUNCATE.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 18546 of file AArch64ISelLowering.cpp.
References assert(), llvm::sampleprof::Base, DL, getGatherVecOpcode(), llvm::SelectionDAG::getNode(), getSignExtendedGatherOpcode(), llvm::EVT::getVectorElementType(), llvm::VTSDNode::getVT(), llvm::AArch64ISD::GLD1_IMM_MERGE_ZERO, llvm::AArch64ISD::GLD1_MERGE_ZERO, llvm::AArch64ISD::GLD1_SCALED_MERGE_ZERO, llvm::AArch64ISD::GLD1_SXTW_MERGE_ZERO, llvm::AArch64ISD::GLD1_SXTW_SCALED_MERGE_ZERO, llvm::AArch64ISD::GLD1_UXTW_MERGE_ZERO, llvm::AArch64ISD::GLD1_UXTW_SCALED_MERGE_ZERO, llvm::AArch64ISD::GLD1S_IMM_MERGE_ZERO, llvm::AArch64ISD::GLD1S_MERGE_ZERO, llvm::AArch64ISD::GLD1S_SCALED_MERGE_ZERO, llvm::MVT::i32, N, llvm::MVT::Other, Scaled, llvm::AArch64ISD::SIGN_EXTEND_INREG_MERGE_PASSTHRU, Signed, and llvm::AArch64ISD::ZERO_EXTEND_INREG_MERGE_PASSTHRU.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 20206 of file AArch64ISelLowering.cpp.
References llvm::ISD::ADD, llvm::AArch64Subtarget::ClassifyGlobalReference(), DL, llvm::SelectionDAG::getConstant(), llvm::Module::getDataLayout(), llvm::GlobalAddressSDNode::getGlobal(), llvm::SelectionDAG::getGlobalAddress(), llvm::SelectionDAG::getNode(), llvm::GlobalAddressSDNode::getOffset(), llvm::GlobalValue::getParent(), llvm::DataLayout::getTypeAllocSize(), llvm::GlobalValue::getValueType(), llvm::MVT::i64, llvm::min(), llvm::AArch64II::MO_NO_FLAG, N, llvm::ISD::SUB, T, TM, and llvm::SDNode::uses().
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 16417 of file AArch64ISelLowering.cpp.
References llvm::ISD::CONCAT_VECTORS, DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getVectorIdxConstant(), llvm::EVT::getVectorNumElements(), llvm::EVT::isFixedLengthVector(), llvm::TargetLoweringBase::isTypeLegal(), llvm::SDValue::isUndef(), and N.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 20746 of file AArch64ISelLowering.cpp.
References N, performPostLD1Combine(), and removeRedundantInsertVectorElt().
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 17621 of file AArch64ISelLowering.cpp.
References llvm::ISD::ABDS, llvm::ISD::ABDU, llvm::ISD::ADD, llvm::ISD::AND, llvm::AArch64ISD::ANDV_PRED, llvm::AArch64CC::ANY_ACTIVE, llvm::AArch64ISD::BIC, combineAcrossLanesIntrinsic(), combineSVEReductionFP(), combineSVEReductionInt(), combineSVEReductionOrderedFP(), convertMergedOpToPredOp(), llvm::TargetLowering::DAGCombinerInfo::DAG, DL, llvm::AArch64ISD::EORV_PRED, llvm::ISD::EXTRACT_SUBVECTOR, llvm::AArch64ISD::FADD_PRED, llvm::AArch64ISD::FADDA_PRED, llvm::AArch64ISD::FADDV_PRED, llvm::AArch64CC::FIRST_ACTIVE, llvm::ISD::FMAXIMUM, llvm::AArch64ISD::FMAXNMV_PRED, llvm::ISD::FMAXNUM, llvm::AArch64ISD::FMAXV_PRED, llvm::ISD::FMINIMUM, llvm::AArch64ISD::FMINNMV_PRED, llvm::ISD::FMINNUM, llvm::AArch64ISD::FMINV_PRED, llvm::AArch64ISD::FMUL_PRED, llvm::AArch64ISD::FSUB_PRED, llvm::SelectionDAG::getCondCode(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), getIntrinsicID(), llvm::SelectionDAG::getNode(), getPromotedVTForPredicate(), getPTest(), llvm::LinearPolySize< ElementCount >::getScalable(), llvm::SelectionDAG::getTargetConstant(), llvm::EVT::getVectorElementCount(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::MVT::i1, llvm::MVT::i64, llvm::ISD::INTRINSIC_WO_CHAIN, llvm::EVT::isFixedLengthVector(), llvm::AArch64CC::LAST_ACTIVE, LowerSVEIntrinsicDUP(), LowerSVEIntrinsicEXT(), LowerSVEIntrinsicIndex(), llvm::AArch64ISD::MUL_PRED, llvm::AArch64ISD::MULHS_PRED, llvm::AArch64ISD::MULHU_PRED, N, llvm::ISD::OR, llvm::AArch64ISD::ORV_PRED, llvm::AArch64ISD::PMULL, llvm::ISD::SADDSAT, llvm::AArch64ISD::SADDV, llvm::AArch64ISD::SADDV_PRED, llvm::AArch64ISD::SETCC_MERGE_ZERO, llvm::ISD::SETEQ, llvm::ISD::SETGE, llvm::ISD::SETGT, llvm::ISD::SETLE, llvm::ISD::SETLT, llvm::ISD::SETNE, llvm::ISD::SETUGE, llvm::ISD::SETUGT, llvm::ISD::SETULE, llvm::ISD::SETULT, llvm::ISD::SETUO, llvm::AArch64ISD::SHL_PRED, llvm::ISD::SIGN_EXTEND, llvm::AArch64ISD::SMAX_PRED, llvm::AArch64ISD::SMAXV, llvm::AArch64ISD::SMAXV_PRED, llvm::AArch64ISD::SMIN_PRED, llvm::AArch64ISD::SMINV, llvm::AArch64ISD::SMINV_PRED, llvm::AArch64ISD::SMULL, llvm::ISD::SPLAT_VECTOR, llvm::AArch64ISD::SRA_PRED, llvm::AArch64ISD::SRAD_MERGE_OP1, llvm::AArch64ISD::SRL_PRED, llvm::ISD::SSUBSAT, llvm::ISD::SUB, llvm::ISD::TRUNCATE, tryCombineCRC32(), tryCombineFixedPointConvert(), tryCombineLongOpWithDup(), tryCombineShiftImm(), tryConvertSVEWideCompare(), llvm::ISD::UADDSAT, llvm::AArch64ISD::UADDV, llvm::AArch64ISD::UADDV_PRED, llvm::AArch64ISD::UMAX_PRED, llvm::AArch64ISD::UMAXV, llvm::AArch64ISD::UMAXV_PRED, llvm::AArch64ISD::UMIN_PRED, llvm::AArch64ISD::UMINV, llvm::AArch64ISD::UMINV_PRED, llvm::AArch64ISD::UMULL, llvm::ISD::USUBSAT, llvm::ISD::VSELECT, and llvm::ISD::XOR.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 15419 of file AArch64ISelLowering.cpp.
References llvm::MVT::f32, llvm::MVT::f64, llvm::MemSDNode::getAlign(), llvm::LoadSDNode::getBasePtr(), llvm::MemSDNode::getChain(), llvm::MachineMemOperand::getFlags(), llvm::SelectionDAG::getLoad(), llvm::MemSDNode::getMemOperand(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::MemSDNode::getPointerInfo(), llvm::EVT::getSizeInBits(), llvm::SDValue::hasOneUse(), llvm::ISD::isNormalLoad(), llvm::SPII::Load, N, performVectorCompareAndMaskUnaryOpCombine(), llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), llvm::ISD::SINT_TO_FP, llvm::AArch64ISD::SITOF, and llvm::AArch64ISD::UITOF.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 16092 of file AArch64ISelLowering.cpp.
References llvm::ISD::ADD, llvm::LegalityPredicates::all(), assert(), llvm::TargetLowering::DAGCombinerInfo::DAG, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::LinearPolySize< LeafTy >::getKnownMinValue(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), getPTest(), getPTrue(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementCount(), llvm::EVT::getVectorElementType(), llvm::MVT::i1, llvm::isAllOnesConstant(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::EVT::isScalableVector(), llvm::AArch64CC::LAST_ACTIVE, N, llvm::AArch64CC::VS, and llvm::ISD::VSCALE.
Referenced by performExtractVectorEltCombine().
|
static |
Definition at line 18044 of file AArch64ISelLowering.cpp.
References DL, llvm::TypeSize::getKnownMinSize(), llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), getSVEContainerType(), llvm::SelectionDAG::getValueType(), llvm::SelectionDAG::getVTList(), llvm::EVT::isInteger(), llvm::SPII::Load, N, llvm::MVT::Other, llvm::AArch64::SVEBitsPerBlock, and llvm::ISD::TRUNCATE.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 18096 of file AArch64ISelLowering.cpp.
References llvm::ISD::BITCAST, llvm::EVT::changeTypeToInteger(), DL, llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), llvm::EVT::isFloatingPoint(), llvm::AArch64ISD::LD1RO_MERGE_ZERO, llvm::AArch64ISD::LD1RQ_MERGE_ZERO, llvm::SPII::Load, N, and llvm::MVT::Other.
|
static |
Definition at line 18070 of file AArch64ISelLowering.cpp.
References llvm::ISD::BITCAST, llvm::EVT::changeTypeToInteger(), DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getMaskedLoad(), llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValue(), llvm::EVT::isFloatingPoint(), N, llvm::ISD::NON_EXTLOAD, and llvm::ISD::UNINDEXED.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 18792 of file AArch64ISelLowering.cpp.
References llvm::commonAlignment(), llvm::ISD::CONCAT_VECTORS, DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::TypeSize::Fixed(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getMemBasePlusOffset(), llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::EVT::getSimpleVT(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::EVT::getVectorElementType(), llvm::SelectionDAG::getVectorIdxConstant(), llvm::MVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::MVT::getVectorVT(), I, llvm::ISD::INSERT_SUBVECTOR, llvm::AArch64Subtarget::isLittleEndian(), llvm::EVT::isScalableVector(), llvm::ARM_MB::LD, N, llvm::MVT::Other, performTBISimplification(), llvm::AArch64Subtarget::supportsAddressTopByteIgnored(), and llvm::ISD::TokenFactor.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 19072 of file AArch64ISelLowering.cpp.
References assert(), llvm::Data, DL, findMoreOptimalIndexType(), llvm::MaskedGatherScatterSDNode::getBasePtr(), llvm::MemSDNode::getChain(), llvm::MaskedGatherSDNode::getExtensionType(), llvm::MaskedGatherScatterSDNode::getIndex(), llvm::MaskedGatherScatterSDNode::getIndexType(), llvm::MaskedGatherScatterSDNode::getMask(), llvm::SelectionDAG::getMaskedGather(), llvm::SelectionDAG::getMaskedScatter(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getMemoryVT(), llvm::MaskedGatherSDNode::getPassThru(), llvm::MaskedGatherScatterSDNode::getScale(), llvm::MaskedScatterSDNode::getValue(), llvm::SelectionDAG::getVTList(), Index, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::MaskedScatterSDNode::isTruncatingStore(), llvm::BitmaskEnumDetail::Mask(), N, and llvm::MVT::Other.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 18907 of file AArch64ISelLowering.cpp.
References llvm::ISD::BITCAST, llvm::EVT::changeVectorElementType(), DL, llvm::MaskedLoadStoreSDNode::getAddressingMode(), llvm::MaskedStoreSDNode::getBasePtr(), llvm::MemSDNode::getChain(), llvm::SelectionDAG::getContext(), llvm::MaskedStoreSDNode::getMask(), llvm::SelectionDAG::getMaskedStore(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getMemoryVT(), llvm::AArch64Subtarget::getMinSVEVectorSizeInBits(), llvm::getNumElementsFromSVEPredPattern(), llvm::MaskedStoreSDNode::getOffset(), getPTrue(), llvm::EVT::getSizeInBits(), llvm::MaskedStoreSDNode::getValue(), llvm::EVT::getVectorElementType(), llvm::Value::hasOneUse(), llvm::MVT::i1, llvm::MaskedLoadStoreSDNode::isUnindexed(), llvm::BitmaskEnumDetail::Mask(), N, llvm::AArch64ISD::PTRUE, llvm::AArch64ISD::UZP1, and llvm::EVT::widenIntegerVectorElementType().
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 15197 of file AArch64ISelLowering.cpp.
References llvm::ISD::ADD, llvm::APInt::ashr(), llvm::BitWidth, llvm::APInt::countTrailingZeros(), DL, llvm::MipsISD::Ext, llvm::SelectionDAG::getConstant(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::SDNode::hasOneUse(), i, llvm::MVT::i64, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::APInt::isNonNegative(), llvm::APInt::isPowerOf2(), isSignExtended(), IsSVECntIntrinsic(), isZeroExtended(), llvm::APInt::logBase2(), M, llvm::ISD::MUL, N, performMulVectorCmpZeroCombine(), performMulVectorExtendCombine(), RHS, llvm::APInt::sdivrem(), llvm::APInt::sge(), llvm::ISD::SHL, llvm::APInt::sle(), llvm::ISD::SUB, std::swap(), llvm::ISD::TRUNCATE, and X.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 15164 of file AArch64ISelLowering.cpp.
References llvm::And, llvm::ISD::AND, llvm::AArch64ISD::CMLTz, DL, llvm::SelectionDAG::getContext(), llvm::EVT::getIntegerVT(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getVectorElementCount(), llvm::EVT::getVectorVT(), llvm::tgtok::In, llvm::ISD::isConstantSplatVector(), llvm::APInt::isMask(), N, llvm::AArch64ISD::NVCAST, llvm::ISD::SRL, llvm::MVT::v1i64, llvm::NVPTX::PTXLdStInstCode::V2, llvm::MVT::v2i32, llvm::MVT::v2i64, llvm::MVT::v4i16, llvm::MVT::v4i32, and llvm::MVT::v8i16.
Referenced by performMulCombine().
|
static |
Combines a mul(dup(sext/zext)) node pattern into mul(sext/zext(dup)) making use of the vector SExt/ZExt rather than the scalar SExt/ZExt.
Definition at line 15144 of file AArch64ISelLowering.cpp.
References DL, llvm::SelectionDAG::getNode(), llvm::BinaryOperator::getOpcode(), llvm::User::getOperand(), Mul, performBuildShuffleExtendCombine(), llvm::MVT::v2i64, llvm::MVT::v4i32, and llvm::MVT::v8i16.
Referenced by performMulCombine().
|
static |
Definition at line 16861 of file AArch64ISelLowering.cpp.
References llvm::AArch64ISD::CSEL, DL, getNegatedInteger(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValueType(), llvm::SDNode::hasOneUse(), isNegatedInteger(), and N.
Referenced by performAddSubCombine().
|
static |
Target-specific DAG combine function for NEON load/store intrinsics to merge base address updates.
Definition at line 19110 of file AArch64ISelLowering.cpp.
References llvm::ISD::ADD, Addr, llvm::TargetLowering::DAGCombinerInfo::CombineTo(), llvm::SelectionDAG::getMemIntrinsicNode(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getMemoryVT(), llvm::SDValue::getNode(), llvm::User::getOperand(), llvm::SelectionDAG::getRegister(), llvm::EVT::getSizeInBits(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVTList(), llvm::SDNode::hasPredecessorHelper(), i, llvm::MVT::i64, llvm::SmallPtrSetImpl< PtrType >::insert(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::TargetLowering::DAGCombinerInfo::isCalledByLegalizer(), llvm::AArch64ISD::LD1x2post, llvm::AArch64ISD::LD1x3post, llvm::AArch64ISD::LD1x4post, llvm::AArch64ISD::LD2DUPpost, llvm::AArch64ISD::LD2LANEpost, llvm::AArch64ISD::LD2post, llvm::AArch64ISD::LD3DUPpost, llvm::AArch64ISD::LD3LANEpost, llvm::AArch64ISD::LD3post, llvm::AArch64ISD::LD4DUPpost, llvm::AArch64ISD::LD4LANEpost, llvm::AArch64ISD::LD4post, llvm_unreachable, llvm::makeArrayRef(), n, N, llvm::MVT::Other, llvm::AArch64ISD::ST1x2post, llvm::AArch64ISD::ST1x3post, llvm::AArch64ISD::ST1x4post, llvm::AArch64ISD::ST2LANEpost, llvm::AArch64ISD::ST2post, llvm::AArch64ISD::ST3LANEpost, llvm::AArch64ISD::ST3post, llvm::AArch64ISD::ST4LANEpost, and llvm::AArch64ISD::ST4post.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
Get rid of unnecessary NVCASTs (that don't change the type).
Definition at line 20196 of file AArch64ISelLowering.cpp.
References N.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 15834 of file AArch64ISelLowering.cpp.
References llvm::TargetLowering::DAGCombinerInfo::DAG, llvm::SelectionDAG::getTargetLoweringInfo(), llvm::TargetLoweringBase::isTypeLegal(), N, performANDORCSELCombine(), tryCombineToBSL(), and tryCombineToEXTR().
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 8628 of file AArch64ISelLowering.cpp.
References llvm::ISD::AND, Cond, DL, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getSetCC(), llvm::Value::hasOneUse(), I, llvm::isNullConstant(), isOrXorChain(), LHS, N, llvm::ISD::OR, RHS, llvm::ISD::SETCC, llvm::ISD::SETEQ, and llvm::ISD::SETNE.
Referenced by performSETCCCombine().
|
static |
Target-specific DAG combine function for post-increment LD1 (lane) and post-increment LD1R.
Definition at line 18645 of file AArch64ISelLowering.cpp.
References llvm::ISD::ADD, Addr, llvm::TargetLowering::DAGCombinerInfo::CombineTo(), llvm::TargetLowering::DAGCombinerInfo::DAG, llvm::SelectionDAG::getMemIntrinsicNode(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getMemoryVT(), llvm::SDValue::getNode(), llvm::User::getOperand(), llvm::SelectionDAG::getRegister(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVTList(), llvm::SDNode::hasPredecessorHelper(), llvm::MVT::i64, llvm::SmallPtrSetImpl< PtrType >::insert(), llvm::EVT::is128BitVector(), llvm::EVT::is64BitVector(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::ARM_MB::LD, llvm::AArch64ISD::LD1DUPpost, llvm::AArch64ISD::LD1LANEpost, llvm::ISD::LOAD, N, llvm::MVT::Other, and Vector.
Referenced by performDUPCombine(), and performInsertVectorEltCombine().
|
static |
Definition at line 20304 of file AArch64ISelLowering.cpp.
References llvm::ISD::ANY_EXTEND, assert(), llvm::sampleprof::Base, llvm::ISD::BITCAST, DL, llvm::TypeSize::getKnownMinSize(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), getScaledOffsetForBitWidth(), llvm::EVT::getSimpleVT(), llvm::EVT::getSizeInBits(), llvm::MVT::getSizeInBits(), getSVEContainerType(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValue(), llvm::SelectionDAG::getValueType(), llvm::EVT::getVectorElementType(), llvm::SelectionDAG::getVTList(), llvm::EVT::isFloatingPoint(), llvm::MVT::isFloatingPoint(), llvm::EVT::isScalableVector(), isValidImmForSVEVecImmAddrMode(), N, llvm::MVT::nxv2f64, llvm::MVT::nxv2i32, llvm::MVT::nxv2i64, llvm::MVT::nxv4f32, llvm::MVT::nxv4i32, llvm::MVT::Other, llvm::AArch64ISD::SST1_IMM_PRED, llvm::AArch64ISD::SST1_PRED, llvm::AArch64ISD::SST1_UXTW_PRED, llvm::AArch64ISD::SSTNT1_INDEX_PRED, llvm::AArch64ISD::SSTNT1_PRED, llvm::AArch64::SVEBitsPerBlock, and std::swap().
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
A vector select: "(select vL, vR, (setcc LHS, RHS))" is best performed with the compare-mask instructions rather than going via NZCV, even if LHS and RHS are really scalar.
This replaces any scalar setcc in the above pattern with a vector one followed by a DUP shuffle on the result.
Definition at line 20114 of file AArch64ISelLowering.cpp.
References assert(), llvm::ISD::BITCAST, llvm::EVT::changeVectorElementTypeToInteger(), llvm::TargetLowering::DAGCombinerInfo::DAG, DL, llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getSelect(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::EVT::getVectorVT(), llvm::MVT::i1, llvm::MVT::i32, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::EVT::isScalableVector(), llvm::TargetLoweringBase::isTypeLegal(), llvm::EVT::isVector(), LHS, llvm::BitmaskEnumDetail::Mask(), N, RHS, llvm::ISD::SCALAR_TO_VECTOR, and llvm::ISD::SETCC.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 16662 of file AArch64ISelLowering.cpp.
References SetCCInfo::AArch64, llvm::ISD::ADD, assert(), GenericSetCCInfo::CC, AArch64SetCCInfo::CC, AArch64SetCCInfo::Cmp, llvm::AArch64ISD::CSEL, SetCCInfo::Generic, getAArch64Cmp(), llvm::SelectionDAG::getConstant(), llvm::AArch64CC::getInvertedCondCode(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::ISD::getSetCCInverse(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::MVT::i32, llvm::MVT::i64, SetCCInfoAndKind::Info, SetCCInfoAndKind::IsAArch64, isSetCCOrZExtSetCC(), LHS, GenericSetCCInfo::Opnd0, GenericSetCCInfo::Opnd1, RHS, and std::swap().
Referenced by performAddSubLongCombine().
|
static |
Definition at line 19724 of file AArch64ISelLowering.cpp.
References llvm::ISD::AND, assert(), llvm::ISD::BITCAST, Cond, llvm::AArch64ISD::CSEL, DL, llvm::SelectionDAG::getConstant(), llvm::EVT::getFixedSizeInBits(), llvm::AArch64CC::getInvertedCondCode(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getSetCC(), llvm::EVT::getVectorElementType(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::Value::hasOneUse(), llvm::MVT::i1, llvm::MVT::i32, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::EVT::isFixedLengthVector(), llvm::isNullConstant(), llvm::isOneConstant(), llvm::EVT::isScalarInteger(), LHS, N, performOrXorChainCombine(), RHS, llvm::ISD::SETCC, llvm::ISD::SETEQ, llvm::ISD::SETNE, llvm::ISD::SRL, tryToWidenSetCCOperands(), llvm::AVRISD::TST, llvm::ISD::VECREDUCE_OR, and llvm::ISD::ZERO_EXTEND.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 19855 of file AArch64ISelLowering.cpp.
References llvm::ISD::AND, assert(), Cond, llvm::TargetLowering::DAGCombinerInfo::DAG, llvm::SelectionDAG::getNode(), llvm::TargetLowering::DAGCombinerInfo::isAfterLegalizeDAG(), isAllActivePredicate(), isZerosVector(), LHS, N, performSetCCPunpkCombine(), RHS, llvm::AArch64ISD::SETCC_MERGE_ZERO, llvm::ISD::SETNE, and llvm::ISD::SIGN_EXTEND.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 19816 of file AArch64ISelLowering.cpp.
References Cond, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SDValue::getConstantOperandVal(), llvm::SDNode::getConstantOperandVal(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SDNode::getValueType(), isZerosVector(), LHS, N, llvm::AArch64ISD::PTRUE, RHS, llvm::AArch64ISD::SETCC_MERGE_ZERO, llvm::ISD::SETNE, and llvm::ISD::SIGN_EXTEND.
Referenced by performSetccMergeZeroCombine().
|
static |
Definition at line 20510 of file AArch64ISelLowering.cpp.
References assert(), llvm::TargetLowering::DAGCombinerInfo::CombineTo(), DL, EnableCombineMGatherIntrinsics, llvm::MipsISD::Ext, llvm::SelectionDAG::getContext(), llvm::EVT::getDoubleNumVectorElementsVT(), llvm::SelectionDAG::getNode(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getValueType(), llvm::EVT::getVectorElementType(), llvm::SelectionDAG::getVTList(), llvm::AArch64ISD::GLD1_IMM_MERGE_ZERO, llvm::AArch64ISD::GLD1_MERGE_ZERO, llvm::AArch64ISD::GLD1_SCALED_MERGE_ZERO, llvm::AArch64ISD::GLD1_SXTW_MERGE_ZERO, llvm::AArch64ISD::GLD1_SXTW_SCALED_MERGE_ZERO, llvm::AArch64ISD::GLD1_UXTW_MERGE_ZERO, llvm::AArch64ISD::GLD1_UXTW_SCALED_MERGE_ZERO, llvm::AArch64ISD::GLD1S_IMM_MERGE_ZERO, llvm::AArch64ISD::GLD1S_MERGE_ZERO, llvm::AArch64ISD::GLD1S_SCALED_MERGE_ZERO, llvm::AArch64ISD::GLD1S_SXTW_MERGE_ZERO, llvm::AArch64ISD::GLD1S_SXTW_SCALED_MERGE_ZERO, llvm::AArch64ISD::GLD1S_UXTW_MERGE_ZERO, llvm::AArch64ISD::GLD1S_UXTW_SCALED_MERGE_ZERO, llvm::AArch64ISD::GLDFF1_IMM_MERGE_ZERO, llvm::AArch64ISD::GLDFF1_MERGE_ZERO, llvm::AArch64ISD::GLDFF1_SCALED_MERGE_ZERO, llvm::AArch64ISD::GLDFF1_SXTW_MERGE_ZERO, llvm::AArch64ISD::GLDFF1_SXTW_SCALED_MERGE_ZERO, llvm::AArch64ISD::GLDFF1_UXTW_MERGE_ZERO, llvm::AArch64ISD::GLDFF1_UXTW_SCALED_MERGE_ZERO, llvm::AArch64ISD::GLDFF1S_IMM_MERGE_ZERO, llvm::AArch64ISD::GLDFF1S_MERGE_ZERO, llvm::AArch64ISD::GLDFF1S_SCALED_MERGE_ZERO, llvm::AArch64ISD::GLDFF1S_SXTW_MERGE_ZERO, llvm::AArch64ISD::GLDFF1S_SXTW_SCALED_MERGE_ZERO, llvm::AArch64ISD::GLDFF1S_UXTW_MERGE_ZERO, llvm::AArch64ISD::GLDFF1S_UXTW_SCALED_MERGE_ZERO, llvm::AArch64ISD::GLDNT1_MERGE_ZERO, llvm::AArch64ISD::GLDNT1S_MERGE_ZERO, I, llvm::MVT::i16, llvm::MVT::i32, llvm::MVT::i8, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::AArch64ISD::LD1_MERGE_ZERO, llvm::AArch64ISD::LD1S_MERGE_ZERO, llvm::AArch64ISD::LDFF1_MERGE_ZERO, llvm::AArch64ISD::LDFF1S_MERGE_ZERO, llvm::AArch64ISD::LDNF1_MERGE_ZERO, llvm::AArch64ISD::LDNF1S_MERGE_ZERO, N, llvm::MVT::Other, llvm::ISD::SIGN_EXTEND_INREG, llvm::AArch64ISD::SUNPKHI, llvm::AArch64ISD::SUNPKLO, llvm::AArch64ISD::UUNPKHI, and llvm::AArch64ISD::UUNPKLO.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 17908 of file AArch64ISelLowering.cpp.
References assert(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getSetCC(), llvm::SDNode::getValueType(), isCheapToExtend(), llvm::EVT::isInteger(), llvm::ISD::isSignedIntSetCC(), N, llvm::ISD::SETCC, llvm::ISD::SIGN_EXTEND, and llvm::ISD::ZERO_EXTEND.
Referenced by performExtendCombine().
|
static |
Definition at line 18366 of file AArch64ISelLowering.cpp.
References assert(), N, and llvm::AArch64ISD::SPLICE.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 18117 of file AArch64ISelLowering.cpp.
References llvm::ISD::ANY_EXTEND, llvm::ISD::BITCAST, llvm::Data, DL, llvm::SelectionDAG::getNode(), getSVEContainerType(), llvm::SelectionDAG::getValueType(), llvm::EVT::isFloatingPoint(), N, and llvm::AArch64ISD::ST1_PRED.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 18143 of file AArch64ISelLowering.cpp.
References llvm::ISD::BITCAST, llvm::EVT::changeTypeToInteger(), llvm::Data, DL, llvm::SelectionDAG::getMaskedStore(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getUNDEF(), llvm::EVT::isFloatingPoint(), N, and llvm::ISD::UNINDEXED.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 18872 of file AArch64ISelLowering.cpp.
References foldTruncStoreOfExt(), llvm::ISD::FP_ROUND, llvm::AArch64Subtarget::getMinSVEVectorSizeInBits(), llvm::SelectionDAG::getTruncStore(), llvm::Value::hasOneUse(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), N, performTBISimplification(), Ptr, splitStores(), llvm::ARM_MB::ST, llvm::SPII::Store, llvm::AArch64Subtarget::supportsAddressTopByteIgnored(), and llvm::AArch64Subtarget::useSVEForFixedLengthVectors().
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 18626 of file AArch64ISelLowering.cpp.
References CC, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getVectorIdxConstant(), llvm::MVT::i1, N, and llvm::ISD::SIGN_EXTEND.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 15883 of file AArch64ISelLowering.cpp.
References llvm::And, llvm::ISD::AND, llvm::TargetLowering::DAGCombinerInfo::DAG, DL, EnableCombineMGatherIntrinsics, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::EVT::getSizeInBits(), llvm::SDNode::getValueType(), llvm::EVT::getVectorElementType(), llvm::AArch64ISD::GLD1_IMM_MERGE_ZERO, llvm::AArch64ISD::GLD1_MERGE_ZERO, llvm::AArch64ISD::GLD1_SCALED_MERGE_ZERO, llvm::AArch64ISD::GLD1_SXTW_MERGE_ZERO, llvm::AArch64ISD::GLD1_SXTW_SCALED_MERGE_ZERO, llvm::AArch64ISD::GLD1_UXTW_MERGE_ZERO, llvm::AArch64ISD::GLD1_UXTW_SCALED_MERGE_ZERO, llvm::AArch64ISD::GLDFF1_IMM_MERGE_ZERO, llvm::AArch64ISD::GLDFF1_MERGE_ZERO, llvm::AArch64ISD::GLDFF1_SCALED_MERGE_ZERO, llvm::AArch64ISD::GLDFF1_SXTW_MERGE_ZERO, llvm::AArch64ISD::GLDFF1_SXTW_SCALED_MERGE_ZERO, llvm::AArch64ISD::GLDFF1_UXTW_MERGE_ZERO, llvm::AArch64ISD::GLDFF1_UXTW_SCALED_MERGE_ZERO, llvm::AArch64ISD::GLDNT1_MERGE_ZERO, llvm::MVT::i16, llvm::MVT::i32, llvm::MVT::i8, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), isConstantSplatVectorMaskForType(), llvm::AArch64ISD::LD1_MERGE_ZERO, llvm::AArch64ISD::LDFF1_MERGE_ZERO, llvm::AArch64ISD::LDNF1_MERGE_ZERO, llvm::BitmaskEnumDetail::Mask(), N, llvm::ISD::SPLAT_VECTOR, llvm::AArch64ISD::UUNPKHI, and llvm::AArch64ISD::UUNPKLO.
Referenced by performANDCombine().
|
static |
Definition at line 20753 of file AArch64ISelLowering.cpp.
References llvm::EVT::changeVectorElementTypeToInteger(), DL, llvm::SelectionDAG::getAnyExtOrTrunc(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), getPackedSVEVectorVT(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getVectorElementCount(), llvm::EVT::getVectorElementType(), llvm::EVT::isInteger(), LHS, N, RHS, and llvm::ISD::VECTOR_SPLICE.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Simplify Addr given that the top byte of it is ignored by HW during address translation.
Definition at line 18752 of file AArch64ISelLowering.cpp.
References Addr, llvm::TargetLowering::DAGCombinerInfo::CommitTargetLoweringOpt(), llvm::APInt::getLowBitsSet(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), and llvm::TargetLowering::SimplifyDemandedBits().
Referenced by performLOADCombine(), and performSTORECombine().
|
static |
Definition at line 19974 of file AArch64ISelLowering.cpp.
References assert(), llvm::tgtok::Bit, DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), getTestBitOperand(), llvm::MVT::i64, N, llvm::MVT::Other, llvm::AArch64ISD::TBNZ, and llvm::AArch64ISD::TBZ.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 14852 of file AArch64ISelLowering.cpp.
References llvm::ISD::ADD, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SDValue::getConstantOperandVal(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), N, llvm::AArch64ISD::SADDLP, llvm::ISD::SIGN_EXTEND, llvm::AArch64ISD::UADDLP, and llvm::ISD::ZERO_EXTEND.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 18376 of file AArch64ISelLowering.cpp.
References assert(), llvm::EVT::changeVectorElementType(), DL, llvm::MaskedLoadStoreSDNode::getAddressingMode(), llvm::MaskedLoadSDNode::getBasePtr(), llvm::MemSDNode::getChain(), llvm::SelectionDAG::getConstant(), llvm::MaskedLoadSDNode::getExtensionType(), llvm::MaskedLoadSDNode::getMask(), llvm::SelectionDAG::getMaskedLoad(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getMemoryVT(), llvm::AArch64Subtarget::getMinSVEVectorSizeInBits(), llvm::SDValue::getNode(), llvm::getNumElementsFromSVEPredPattern(), llvm::MaskedLoadSDNode::getOffset(), llvm::MaskedLoadSDNode::getPassThru(), getPTrue(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValue(), llvm::EVT::getVectorElementType(), hasOneUse(), llvm::MVT::i1, llvm::SDNode::isUndef(), llvm::MaskedLoadStoreSDNode::isUnindexed(), isZerosVector(), llvm::BitmaskEnumDetail::Mask(), llvm::ISD::MLOAD, N, llvm::AArch64ISD::PTRUE, llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), llvm::ISD::SEXTLOAD, llvm::AArch64ISD::UUNPKHI, llvm::AArch64ISD::UUNPKLO, and llvm::ISD::ZEXTLOAD.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 18425 of file AArch64ISelLowering.cpp.
References llvm::ISD::BITCAST, llvm::ISD::CONCAT_VECTORS, DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::EVT::getSimpleVT(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::DataLayout::isLittleEndian(), llvm::EVT::isSimple(), llvm_unreachable, N, llvm::MVT::Other, llvm::MVT::SimpleTy, llvm::ISD::TRUNCATE, llvm::ISD::UNDEF, llvm::AArch64ISD::UUNPKHI, llvm::AArch64ISD::UUNPKLO, llvm::AArch64ISD::UZP1, llvm::MVT::v16i8, llvm::MVT::v2i32, llvm::MVT::v2i64, llvm::MVT::v4i16, llvm::MVT::v4i32, llvm::MVT::v8i16, llvm::MVT::v8i8, X, and llvm::ARCCC::Z.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 14803 of file AArch64ISelLowering.cpp.
References B, DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::MVT::i32, llvm::ISD::MUL, N, performVecReduceAddCombineWithUADDLP(), llvm::AArch64ISD::SDOT, llvm::ISD::SIGN_EXTEND, llvm::ARM_MB::ST, llvm::AArch64ISD::UDOT, llvm::MVT::v16i8, llvm::MVT::v2i32, llvm::MVT::v4i32, llvm::MVT::v8i8, llvm::ISD::VECREDUCE_ADD, and llvm::ISD::ZERO_EXTEND.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 14725 of file AArch64ISelLowering.cpp.
References llvm::ISD::ABDS, llvm::ISD::ABDU, llvm::ISD::ABS, llvm::ISD::ADD, DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::SDNode::getValueType(), llvm::MVT::i32, llvm::MVT::i64, N, llvm::ISD::SIGN_EXTEND, llvm::ISD::SUB, llvm::AArch64ISD::UADDLP, llvm::MVT::v16i32, llvm::MVT::v16i8, llvm::MVT::v4i32, llvm::MVT::v8i16, llvm::MVT::v8i8, llvm::ISD::VECREDUCE_ADD, and llvm::ISD::ZERO_EXTEND.
Referenced by performVecReduceAddCombine().
|
static |
Definition at line 17008 of file AArch64ISelLowering.cpp.
References DL, llvm::SelectionDAG::getContext(), llvm::EVT::getHalfSizedIntegerVT(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarType(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementCount(), llvm::EVT::getVectorVT(), llvm::MVT::i16, llvm::MVT::i32, llvm::MVT::i64, llvm::MVT::i8, llvm::EVT::isFixedLengthVector(), N, llvm::ISD::SIGN_EXTEND, and llvm::ISD::ZERO_EXTEND.
Referenced by performAddSubCombine().
|
static |
Definition at line 15373 of file AArch64ISelLowering.cpp.
References llvm::ISD::AND, llvm::ISD::BITCAST, DL, llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::EVT::isVector(), N, and llvm::ISD::SETCC.
Referenced by performIntToFpCombine().
|
static |
Optimize a vector shift instruction and its operand if shifted out bits are not used.
Definition at line 18605 of file AArch64ISelLowering.cpp.
References assert(), llvm::APInt::getLowBitsSet(), N, llvm::TargetLowering::SimplifyDemandedBits(), llvm::AArch64ISD::VASHR, and llvm::AArch64ISD::VLSHR.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 20043 of file AArch64ISelLowering.cpp.
References llvm::EVT::changeVectorElementTypeToInteger(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getCondCode(), llvm::SelectionDAG::getConstant(), llvm::LinearPolySize< ElementCount >::getFixed(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::SelectionDAG::getSetCC(), llvm::EVT::getSimpleVT(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementCount(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::MVT::i1, llvm::is_contained(), isAllActivePredicate(), isAllInactivePredicate(), llvm::ISD::isConstantSplatVector(), llvm::ISD::isConstantSplatVectorAllOnes(), llvm::APInt::isOne(), llvm::EVT::isSimple(), llvm::makeArrayRef(), N, llvm::Or, llvm::ISD::OR, llvm::ISD::SETCC, llvm::ISD::SETGT, Shift, llvm::MVT::SimpleTy, llvm::ISD::SRA, trySwapVSelectOperands(), llvm::MVT::v16i8, llvm::MVT::v2i32, llvm::MVT::v2i64, llvm::MVT::v4i16, llvm::MVT::v4i32, llvm::MVT::v8i16, llvm::MVT::v8i8, and llvm::ISD::VSELECT.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 14903 of file AArch64ISelLowering.cpp.
References foldVectorXorShiftIntoCmp(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), and N.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 10419 of file AArch64ISelLowering.cpp.
References assert(), llvm::sampleprof::Base, llvm::ISD::CONCAT_VECTORS, DL, llvm::MipsISD::Ext, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SDValue::getConstantOperandVal(), llvm::SelectionDAG::getNode(), llvm::SDValue::getNumOperands(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValueType(), llvm::ISD::TRUNCATE, llvm::MVT::v16i8, llvm::MVT::v4i16, llvm::MVT::v4i32, llvm::MVT::v8i16, llvm::MVT::v8i8, X, and Y.
Definition at line 20709 of file AArch64ISelLowering.cpp.
References assert(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SDValue::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValueType(), llvm::ISD::INSERT_VECTOR_ELT, llvm::ISD::isConstantSplatVectorAllZeros(), isLanes1toNKnownZero(), llvm::isNullConstant(), and N.
Referenced by performInsertVectorEltCombine().
|
static |
Definition at line 21318 of file AArch64ISelLowering.cpp.
References llvm::AArch64ISD::ADDP, assert(), llvm::ISD::CONCAT_VECTORS, DL, E, llvm::MVT::f16, llvm::SelectionDAG::getNode(), llvm::EVT::getScalarType(), llvm::SelectionDAG::getUNDEF(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), I, llvm::EVT::is256BitVector(), llvm::EVT::isFloatingPoint(), llvm::BitmaskEnumDetail::Mask(), N, Results, llvm::SelectionDAG::SplitVector(), and X.
|
static |
Definition at line 21440 of file AArch64ISelLowering.cpp.
References llvm::Acquire, llvm::AcquireRelease, assert(), llvm::ISD::BUILD_PAIR, createGPRPairNode(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getMachineNode(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetExtractSubreg(), llvm::SelectionDAG::getVTList(), llvm::MVT::i128, llvm::MVT::i32, llvm::MVT::i64, llvm::DataLayout::isBigEndian(), llvm_unreachable, llvm::Monotonic, N, llvm::MVT::Other, llvm::Release, Results, llvm::SequentiallyConsistent, llvm::SelectionDAG::setNodeMemRefs(), splitInt128(), std::swap(), and llvm::MVT::Untyped.
|
static |
Definition at line 21365 of file AArch64ISelLowering.cpp.
References llvm::SelectionDAG::getNode(), llvm::SelectionDAG::GetSplitDestVTs(), N, Results, and llvm::SelectionDAG::SplitVectorOperand().
|
static |
Replace a splat of a scalar to a vector store by scalar stores of the scalar value.
The load store optimizer pass will merge them to store pair stores. This has better performance than a splat of the scalar followed by a split vector store. Even if the stores are not merged it is four stores vs a dup, followed by an ext.b and two stores.
Definition at line 18242 of file AArch64ISelLowering.cpp.
References llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::StoreSDNode::getValue(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::ConstantSDNode::getZExtValue(), I, llvm::ISD::INSERT_VECTOR_ELT, llvm::EVT::isFloatingPoint(), llvm::StoreSDNode::isTruncatingStore(), and splitStoreSplat().
Referenced by splitStores().
|
static |
Replace a splat of zeros to a vector store by scalar stores of WZR/XZR.
The load store optimizer pass will merge them to store pair stores. This should be better than a movi to create the vector zero followed by a vector store if the zero constant is not re-used, since one instructions and one register live range will be removed.
For example, the final generated code should be:
stp xzr, xzr, [x0]
instead of:
movi v0.2d, #0 str q0, [x0]
Definition at line 18175 of file AArch64ISelLowering.cpp.
References llvm::ISD::BUILD_VECTOR, DL, llvm::StoreSDNode::getBasePtr(), llvm::SDNode::getConstantOperandVal(), llvm::SelectionDAG::getCopyFromReg(), llvm::SelectionDAG::getEntryNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getSizeInBits(), llvm::StoreSDNode::getValue(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::SDValue::hasOneUse(), I, llvm::MVT::i32, llvm::MVT::i64, llvm::SelectionDAG::isBaseWithConstantOffset(), llvm::isNullConstant(), llvm::isNullFPConstant(), llvm::EVT::isScalableVector(), llvm::StoreSDNode::isTruncatingStore(), and splitStoreSplat().
Referenced by splitStores().
|
static |
Definition at line 11494 of file AArch64ISelLowering.cpp.
References llvm::EVT::getSizeInBits(), llvm::SDNode::getValueType(), i, llvm::BuildVectorSDNode::isConstantSplat(), and llvm::APInt::zextOrTrunc().
Referenced by ConstantBuildVector(), EmitVectorComparison(), and performANDCombine().
|
static |
Definition at line 4494 of file AArch64ISelLowering.cpp.
References isAddSubSExt(), isAddSubZExt(), isSignExtended(), isZeroExtended(), llvm::AArch64ISD::SMULL, std::swap(), and llvm::AArch64ISD::UMULL.
|
static |
Set the IntrinsicInfo for the aarch64_sve_st<N> intrinsics.
Definition at line 13183 of file AArch64ISelLowering.cpp.
References llvm::CallBase::arg_size(), assert(), DL, llvm::CallBase::getArgOperand(), llvm::Type::getContext(), llvm::TargetLoweringBase::getMemValueType(), llvm::EVT::getScalarType(), llvm::Value::getType(), llvm::EVT::getVectorElementCount(), llvm::EVT::getVectorVT(), I, Info, llvm::ISD::INTRINSIC_VOID, and llvm::MachineMemOperand::MOStore.
|
static |
Definition at line 4368 of file AArch64ISelLowering.cpp.
References addRequiredExtensionForVectorMULL(), llvm::ISD::ANY_EXTEND, assert(), llvm::ISD::BUILD_VECTOR, llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::MVT::getIntegerVT(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorVT(), i, llvm::MVT::i32, N, llvm::ISD::SIGN_EXTEND, llvm::ISD::ZERO_EXTEND, and llvm::APInt::zextOrTrunc().
|
static |
Definition at line 21379 of file AArch64ISelLowering.cpp.
References DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::MVT::i128, llvm::MVT::i64, N, llvm::ISD::SRL, and llvm::ISD::TRUNCATE.
Referenced by ReplaceCMP_SWAP_128Results().
|
static |
Definition at line 18295 of file AArch64ISelLowering.cpp.
References llvm::ISD::ADD, DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::MachineFunction::getFunction(), llvm::EVT::getHalfNumVectorElementsVT(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getStore(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::Function::hasMinSize(), llvm::MVT::i64, llvm::EVT::isFixedLengthVector(), N, replaceSplatVectorStore(), replaceZeroVectorStore(), S, and llvm::MVT::v2i64.
Referenced by performSTORECombine().
|
static |
Definition at line 17974 of file AArch64ISelLowering.cpp.
References llvm::ISD::ADD, assert(), llvm::commonAlignment(), DL, llvm::MemSDNode::getAlign(), llvm::StoreSDNode::getBasePtr(), llvm::MemSDNode::getChain(), llvm::SelectionDAG::getConstant(), llvm::MachineMemOperand::getFlags(), llvm::MemSDNode::getMemOperand(), llvm::SelectionDAG::getNode(), llvm::MemSDNode::getPointerInfo(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getStore(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::MachinePointerInfo::getWithOffset(), llvm::MVT::i64, and llvm::StoreSDNode::isTruncatingStore().
Referenced by replaceSplatVectorStore(), and replaceZeroVectorStore().
| STATISTIC | ( | NumOptimizedImms | , |
| "Number of times immediates were optimized" | |||
| ) |
|
static |
Definition at line 11590 of file AArch64ISelLowering.cpp.
References llvm::tgtok::Bits, llvm::AArch64_AM::encodeAdvSIMDModImmType5(), llvm::AArch64_AM::encodeAdvSIMDModImmType6(), llvm::AArch64Subtarget::forceStreamingCompatibleSVE(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getSubtarget(), llvm::MVT::i32, llvm::AArch64_AM::isAdvSIMDModImmType5(), llvm::AArch64_AM::isAdvSIMDModImmType6(), llvm::EVT::isFixedLengthVector(), LHS, llvm::AArch64ISD::NVCAST, Shift, llvm::MVT::v4i16, and llvm::MVT::v8i16.
Referenced by ConstantBuildVector(), and performANDCombine().
|
static |
Definition at line 11538 of file AArch64ISelLowering.cpp.
References llvm::tgtok::Bits, llvm::AArch64_AM::encodeAdvSIMDModImmType1(), llvm::AArch64_AM::encodeAdvSIMDModImmType2(), llvm::AArch64_AM::encodeAdvSIMDModImmType3(), llvm::AArch64_AM::encodeAdvSIMDModImmType4(), llvm::AArch64Subtarget::forceStreamingCompatibleSVE(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getSubtarget(), llvm::MVT::i32, llvm::AArch64_AM::isAdvSIMDModImmType1(), llvm::AArch64_AM::isAdvSIMDModImmType2(), llvm::AArch64_AM::isAdvSIMDModImmType3(), llvm::AArch64_AM::isAdvSIMDModImmType4(), llvm::EVT::isFixedLengthVector(), LHS, llvm::AArch64ISD::NVCAST, Shift, llvm::MVT::v2i32, and llvm::MVT::v4i32.
Referenced by ConstantBuildVector(), and performANDCombine().
|
static |
Definition at line 11634 of file AArch64ISelLowering.cpp.
References llvm::tgtok::Bits, llvm::AArch64_AM::encodeAdvSIMDModImmType7(), llvm::AArch64_AM::encodeAdvSIMDModImmType8(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::MVT::i32, llvm::AArch64_AM::isAdvSIMDModImmType7(), llvm::AArch64_AM::isAdvSIMDModImmType8(), llvm::AArch64ISD::NVCAST, Shift, llvm::MVT::v2i32, and llvm::MVT::v4i32.
Referenced by ConstantBuildVector().
|
static |
Definition at line 11517 of file AArch64ISelLowering.cpp.
References llvm::tgtok::Bits, llvm::AArch64_AM::encodeAdvSIMDModImmType10(), llvm::MVT::f64, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::MVT::i32, llvm::AArch64_AM::isAdvSIMDModImmType10(), llvm::AArch64ISD::NVCAST, and llvm::MVT::v2i64.
Referenced by ConstantBuildVector().
|
static |
Definition at line 11665 of file AArch64ISelLowering.cpp.
References llvm::tgtok::Bits, llvm::AArch64_AM::encodeAdvSIMDModImmType9(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::MVT::i32, llvm::AArch64_AM::isAdvSIMDModImmType9(), llvm::AArch64ISD::NVCAST, llvm::MVT::v16i8, and llvm::MVT::v8i8.
Referenced by ConstantBuildVector().
|
static |
Definition at line 11686 of file AArch64ISelLowering.cpp.
References llvm::tgtok::Bits, llvm::AArch64_AM::encodeAdvSIMDModImmType11(), llvm::AArch64_AM::encodeAdvSIMDModImmType12(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::MVT::i32, llvm::AArch64_AM::isAdvSIMDModImmType11(), llvm::AArch64_AM::isAdvSIMDModImmType12(), llvm::AArch64ISD::NVCAST, llvm::MVT::v2f32, llvm::MVT::v2f64, and llvm::MVT::v4f32.
Referenced by ConstantBuildVector().
|
static |
Definition at line 17310 of file AArch64ISelLowering.cpp.
References llvm::ISD::AND, llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::ConstantSDNode::getZExtValue(), llvm::MVT::i32, llvm::ISD::INTRINSIC_WO_CHAIN, llvm::BitmaskEnumDetail::Mask(), and N.
Referenced by performIntrinsicCombine().
|
static |
Definition at line 16458 of file AArch64ISelLowering.cpp.
References DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::ISD::INTRINSIC_WO_CHAIN, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), N, Shift, llvm::MVT::v2f64, llvm::MVT::v2i64, llvm::MVT::v4f32, and llvm::MVT::v4i32.
Referenced by performIntrinsicCombine().
|
static |
Definition at line 17207 of file AArch64ISelLowering.cpp.
References assert(), llvm::SelectionDAG::getNode(), llvm::ISD::INTRINSIC_WO_CHAIN, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), isEssentiallyExtractHighSubvector(), LHS, N, llvm::Intrinsic::not_intrinsic, RHS, and tryExtendDUPToExtractHigh().
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine(), performExtendCombine(), and performIntrinsicCombine().
|
static |
Definition at line 17239 of file AArch64ISelLowering.cpp.
References llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::APInt::getSExtValue(), llvm::MVT::getSizeInBits(), llvm::MVT::i32, llvm::BuildVectorSDNode::isConstantSplat(), llvm_unreachable, N, llvm::AArch64ISD::SQSHL_I, llvm::AArch64ISD::SQSHLU_I, llvm::AArch64ISD::SRSHR_I, llvm::AArch64ISD::UQSHL_I, llvm::AArch64ISD::URSHR_I, and llvm::AArch64ISD::VSHL.
Referenced by performIntrinsicCombine().
|
static |
Definition at line 15662 of file AArch64ISelLowering.cpp.
References llvm::ISD::ADD, llvm::ISD::AND, llvm::tgtok::Bits, llvm::AArch64ISD::BSP, llvm::TargetLowering::DAGCombinerInfo::DAG, DL, llvm::AArch64Subtarget::forceStreamingCompatibleSVE(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::SelectionDAG::getSubtarget(), llvm::EVT::getVectorNumElements(), llvm::ConstantSDNode::getZExtValue(), i, llvm::ISD::isBuildVectorAllOnes(), llvm::ISD::isBuildVectorAllZeros(), llvm::EVT::isVector(), j(), N, llvm::ISD::SUB, and llvm::AArch64TargetLowering::useSVEForFixedLengthVectorVT().
Referenced by performORCombine().
|
static |
EXTR instruction extracts a contiguous chunk of bits from two existing registers viewed as a high/low pair.
This function looks for the pattern: (or (shl VAL1, #N), (srl VAL2, #RegWidth-N)) and replaces it with an EXTR. Can't quite be done in TableGen because the two immediates aren't independent.
Definition at line 15622 of file AArch64ISelLowering.cpp.
References assert(), llvm::TargetLowering::DAGCombinerInfo::DAG, DL, llvm::AArch64ISD::EXTR, findEXTRHalf(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::MVT::i32, llvm::MVT::i64, LHS, N, llvm::ISD::OR, RHS, and std::swap().
Referenced by performORCombine().
|
static |
Definition at line 17389 of file AArch64ISelLowering.cpp.
References CC, DL, llvm::AArch64ISD::DUP, llvm::SelectionDAG::getCondCode(), llvm::SelectionDAG::getConstant(), getIntrinsicID(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::MVT::i32, llvm::RISCVMatInt::Imm, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm_unreachable, N, llvm::AArch64ISD::SETCC_MERGE_ZERO, and llvm::ISD::SPLAT_VECTOR.
Referenced by performIntrinsicCombine().
|
static |
Definition at line 16521 of file AArch64ISelLowering.cpp.
References DL, llvm::AArch64ISD::DUP, llvm::AArch64ISD::DUPLANE16, llvm::AArch64ISD::DUPLANE32, llvm::AArch64ISD::DUPLANE64, llvm::AArch64ISD::DUPLANE8, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::MVT::i64, llvm::MVT::is64BitVector(), llvm::AArch64ISD::MOVI, llvm::AArch64ISD::MOVIedit, llvm::AArch64ISD::MOVImsl, llvm::AArch64ISD::MOVIshift, llvm::AArch64ISD::MVNImsl, llvm::AArch64ISD::MVNIshift, and N.
Referenced by performAddSubLongCombine(), and tryCombineLongOpWithDup().
|
static |
Definition at line 10767 of file AArch64ISelLowering.cpp.
References llvm::ISD::CONCAT_VECTORS, DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::EVT::getHalfNumVectorElementsVT(), llvm::SelectionDAG::getNode(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::MVT::i64, isConcatMask(), and llvm::BitmaskEnumDetail::Mask().
|
static |
Definition at line 11742 of file AArch64ISelLowering.cpp.
References llvm::And, llvm::ISD::AND, assert(), llvm::AArch64ISD::BICi, C1, llvm::dbgs(), DL, llvm::SDNode::dump(), llvm::APInt::getHighBitsSet(), llvm::APInt::getLowBitsSet(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::EVT::getScalarSizeInBits(), llvm::ConstantSDNode::getZExtValue(), isAllConstantBuildVector(), llvm::EVT::isVector(), LLVM_DEBUG, N, Shift, llvm::AArch64ISD::VLSHR, llvm::AArch64ISD::VSHL, llvm::AArch64ISD::VSLI, llvm::AArch64ISD::VSRI, X, and Y.
|
static |
Definition at line 20005 of file AArch64ISelLowering.cpp.
References CC, llvm::ISD::FADD, llvm::ISD::FMUL, llvm::ISD::FSUB, llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getSetCC(), llvm::ISD::getSetCCInverse(), llvm::SDValue::getValueType(), llvm::SDValue::hasOneUse(), N, llvm::ISD::SETCC, and llvm::ISD::VSELECT.
Referenced by performVSelectCombine().
|
static |
Definition at line 11196 of file AArch64ISelLowering.cpp.
References llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getTargetConstant(), llvm::SDNode::getValueType(), I, llvm::MVT::i32, llvm::MVT::i64, llvm::ISD::INTRINSIC_WO_CHAIN, and llvm::MVT::v16i8.
|
static |
Definition at line 19679 of file AArch64ISelLowering.cpp.
References llvm::any_of(), CC, llvm::EVT::changeVectorElementType(), DL, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNodeIfExists(), llvm::SDNode::getOpcode(), llvm::EVT::getScalarSizeInBits(), llvm::SDNode::getValueType(), llvm::SelectionDAG::getVTList(), llvm::MVT::i1, llvm::ISD::isConstantSplatVector(), llvm::ISD::isIntEqualitySetCC(), llvm::ISD::isSignedIntSetCC(), llvm::ISD::isUnsignedIntSetCC(), llvm::EVT::isVector(), N, llvm::ISD::SETCC, llvm::ISD::SIGN_EXTEND, llvm::ISD::VSELECT, and llvm::ISD::ZERO_EXTEND.
Referenced by performSETCCCombine().
|
static |
Definition at line 11163 of file AArch64ISelLowering.cpp.
References DL, llvm::SelectionDAG::getBitcast(), llvm::EVT::getFixedSizeInBits(), llvm::MVT::getFloatingPointVT(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), llvm::EVT::isFloatingPoint(), llvm::TargetLoweringBase::isTypeLegal(), isWideTypeMask(), and llvm::BitmaskEnumDetail::Mask().
|
static |
Definition at line 21926 of file AArch64ISelLowering.cpp.
References llvm::IRBuilderBase::CreateCall(), llvm::IRBuilderBase::CreateConstGEP1_32(), llvm::IRBuilderBase::CreatePointerCast(), llvm::Intrinsic::getDeclaration(), llvm::IRBuilderBase::GetInsertBlock(), llvm::IRBuilderBase::getInt8PtrTy(), llvm::IRBuilderBase::getInt8Ty(), llvm::BasicBlock::getParent(), llvm::GlobalValue::getParent(), llvm::Type::getPointerTo(), and M.
Referenced by llvm::AArch64TargetLowering::getIRStackGuard(), and llvm::AArch64TargetLowering::getSafeStackPointerLocation().
|
static |
Definition at line 3697 of file AArch64ISelLowering.cpp.
References DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getVTList(), llvm::MVT::Glue, and llvm::AArch64ISD::SUBS.
Referenced by lowerADDSUBCARRY().
|
static |
WidenVector - Given a value in the V64 register class, produce the equivalent value in the V128 register class.
Definition at line 10066 of file AArch64ISelLowering.cpp.
References DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::EVT::getSimpleVT(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::MVT::i64, and llvm::ISD::INSERT_SUBVECTOR.
Referenced by constructDup(), GeneratePerfectShuffle(), and performConcatVectorsCombine().
| cl::opt<bool> EnableAArch64ELFLocalDynamicTLSGeneration("aarch64-elf-ldtls-generation", cl::Hidden, cl::desc("Allow AArch64 Local Dynamic TLS code generation"), cl::init(false)) |
Referenced by llvm::AArch64MCInstLower::lowerSymbolOperandELF().
|
static |
Referenced by performSignExtendInRegCombine(), and performSVEAndCombine().
|
static |
Referenced by llvm::AArch64TargetLowering::targetShrinkDemandedConstant().
|
static |
Referenced by isOrXorChain().
Value type used for condition codes.
Definition at line 140 of file AArch64ISelLowering.cpp.
Referenced by emitComparison(), emitConditionalComparison(), getAArch64Cmp(), getAArch64XALUOOp(), and performANDORCSELCombine().
1.8.17