LLVM  3.7.0
Classes | Macros | Enumerations | Functions | Variables
AArch64ISelLowering.cpp File Reference
#include "AArch64ISelLowering.h"
#include "AArch64CallingConvention.h"
#include "AArch64MachineFunctionInfo.h"
#include "AArch64PerfectShuffle.h"
#include "AArch64Subtarget.h"
#include "AArch64TargetMachine.h"
#include "AArch64TargetObjectFile.h"
#include "MCTargetDesc/AArch64AddressingModes.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/CodeGen/CallingConvLower.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/GetElementPtrTypeIterator.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/Type.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Target/TargetOptions.h"
#include "AArch64GenCallingConv.inc"
Include dependency graph for AArch64ISelLowering.cpp:

Go to the source code of this file.

Classes

struct  GenericSetCCInfo
 Helper structure to keep track of ISD::SET_CC operands. More...
 
struct  AArch64SetCCInfo
 Helper structure to keep track of a SET_CC lowered into AArch64 code. More...
 
union  SetCCInfo
 Helper structure to keep track of SetCC information. More...
 
struct  SetCCInfoAndKind
 Helper structure to be able to read SetCC information. More...
 

Macros

#define DEBUG_TYPE   "aarch64-lower"
 

Enumerations

enum  AlignMode
 

Functions

 STATISTIC (NumTailCalls,"Number of tail calls")
 
 STATISTIC (NumShiftInserts,"Number of vector shift inserts")
 
static cl::opt< AlignModeAlign (cl::desc("Load/store alignment support"), cl::Hidden, cl::init(NoStrictAlign), cl::values(clEnumValN(StrictAlign,"aarch64-strict-align","Disallow all unaligned memory accesses"), clEnumValN(NoStrictAlign,"aarch64-no-strict-align","Allow unaligned memory accesses"), clEnumValEnd))
 
static AArch64CC::CondCode changeIntCCToAArch64CC (ISD::CondCode CC)
 changeIntCCToAArch64CC - Convert a DAG integer condition code to an AArch64 CC More...
 
static void changeFPCCToAArch64CC (ISD::CondCode CC, AArch64CC::CondCode &CondCode, AArch64CC::CondCode &CondCode2)
 changeFPCCToAArch64CC - Convert a DAG fp condition code to an AArch64 CC. More...
 
static void changeVectorFPCCToAArch64CC (ISD::CondCode CC, AArch64CC::CondCode &CondCode, AArch64CC::CondCode &CondCode2, bool &Invert)
 changeVectorFPCCToAArch64CC - Convert a DAG fp condition code to an AArch64 CC usable with the vector instructions. More...
 
static bool isLegalArithImmed (uint64_t C)
 
static SDValue emitComparison (SDValue LHS, SDValue RHS, ISD::CondCode CC, SDLoc dl, SelectionDAG &DAG)
 
static SDValue getAArch64Cmp (SDValue LHS, SDValue RHS, ISD::CondCode CC, SDValue &AArch64cc, SelectionDAG &DAG, SDLoc dl)
 
static std::pair< SDValue,
SDValue
getAArch64XALUOOp (AArch64CC::CondCode &CC, SDValue Op, SelectionDAG &DAG)
 
static SDValue LowerXOR (SDValue Op, SelectionDAG &DAG)
 
static SDValue LowerADDC_ADDE_SUBC_SUBE (SDValue Op, SelectionDAG &DAG)
 
static SDValue LowerXALUO (SDValue Op, SelectionDAG &DAG)
 
static SDValue LowerPREFETCH (SDValue Op, SelectionDAG &DAG)
 
static SDValue LowerVectorFP_TO_INT (SDValue Op, SelectionDAG &DAG)
 
static SDValue LowerVectorINT_TO_FP (SDValue Op, SelectionDAG &DAG)
 
static SDValue LowerBITCAST (SDValue Op, SelectionDAG &DAG)
 
static EVT getExtensionTo64Bits (const EVT &OrigVT)
 
static SDValue addRequiredExtensionForVectorMULL (SDValue N, SelectionDAG &DAG, const EVT &OrigTy, const EVT &ExtTy, unsigned ExtOpcode)
 
static bool isExtendedBUILD_VECTOR (SDNode *N, SelectionDAG &DAG, bool isSigned)
 
static SDValue skipExtensionForVectorMULL (SDNode *N, SelectionDAG &DAG)
 
static bool isSignExtended (SDNode *N, SelectionDAG &DAG)
 
static bool isZeroExtended (SDNode *N, SelectionDAG &DAG)
 
static bool isAddSubSExt (SDNode *N, SelectionDAG &DAG)
 
static bool isAddSubZExt (SDNode *N, SelectionDAG &DAG)
 
static SDValue LowerMUL (SDValue Op, SelectionDAG &DAG)
 
static bool selectCCOpsAreFMaxCompatible (SDValue Cmp, SDValue Result)
 A SELECT_CC operation is really some kind of max or min if both values being compared are, in some sense, equal to the results in either case. More...
 
static SDValue WidenVector (SDValue V64Reg, SelectionDAG &DAG)
 WidenVector - Given a value in the V64 register class, produce the equivalent value in the V128 register class. More...
 
static unsigned getExtFactor (SDValue &V)
 getExtFactor - Determine the adjustment factor for the position when generating an "extract from vector registers" instruction. More...
 
static SDValue NarrowVector (SDValue V128Reg, SelectionDAG &DAG)
 NarrowVector - Given a value in the V128 register class, produce the equivalent value in the V64 register class. More...
 
static bool isSingletonEXTMask (ArrayRef< int > M, EVT VT, unsigned &Imm)
 
static bool isEXTMask (ArrayRef< int > M, EVT VT, bool &ReverseEXT, unsigned &Imm)
 
static bool isREVMask (ArrayRef< int > M, EVT VT, unsigned BlockSize)
 isREVMask - Check if a vector shuffle corresponds to a REV instruction with the specified blocksize. More...
 
static bool isZIPMask (ArrayRef< int > M, EVT VT, unsigned &WhichResult)
 
static bool isUZPMask (ArrayRef< int > M, EVT VT, unsigned &WhichResult)
 
static bool isTRNMask (ArrayRef< int > M, EVT VT, unsigned &WhichResult)
 
static bool isZIP_v_undef_Mask (ArrayRef< int > M, EVT VT, unsigned &WhichResult)
 isZIP_v_undef_Mask - Special case of isZIPMask for canonical form of "vector_shuffle v, v", i.e., "vector_shuffle v, undef". More...
 
static bool isUZP_v_undef_Mask (ArrayRef< int > M, EVT VT, unsigned &WhichResult)
 isUZP_v_undef_Mask - Special case of isUZPMask for canonical form of "vector_shuffle v, v", i.e., "vector_shuffle v, undef". More...
 
static bool isTRN_v_undef_Mask (ArrayRef< int > M, EVT VT, unsigned &WhichResult)
 isTRN_v_undef_Mask - Special case of isTRNMask for canonical form of "vector_shuffle v, v", i.e., "vector_shuffle v, undef". More...
 
static bool isINSMask (ArrayRef< int > M, int NumInputElements, bool &DstIsLeft, int &Anomaly)
 
static bool isConcatMask (ArrayRef< int > Mask, EVT VT, bool SplitLHS)
 
static SDValue tryFormConcatFromShuffle (SDValue Op, SelectionDAG &DAG)
 
static SDValue GeneratePerfectShuffle (unsigned PFEntry, SDValue LHS, SDValue RHS, SelectionDAG &DAG, SDLoc dl)
 GeneratePerfectShuffle - Given an entry in the perfect-shuffle table, emit the specified operations to build the shuffle. More...
 
static SDValue GenerateTBL (SDValue Op, ArrayRef< int > ShuffleMask, SelectionDAG &DAG)
 
static unsigned getDUPLANEOp (EVT EltType)
 
static bool resolveBuildVector (BuildVectorSDNode *BVN, APInt &CnstBits, APInt &UndefBits)
 
static bool isAllConstantBuildVector (const SDValue &PotentialBVec, uint64_t &ConstVal)
 
static unsigned getIntrinsicID (const SDNode *N)
 
static SDValue tryLowerToSLI (SDNode *N, SelectionDAG &DAG)
 
static SDValue NormalizeBuildVector (SDValue Op, SelectionDAG &DAG)
 
static bool getVShiftImm (SDValue Op, unsigned ElementBits, int64_t &Cnt)
 getVShiftImm - Check if this is a valid build_vector for the immediate operand of a vector shift operation, where all the elements of the build_vector must have the same constant integer value. More...
 
static bool isVShiftLImm (SDValue Op, EVT VT, bool isLong, int64_t &Cnt)
 isVShiftLImm - Check if this is a valid build_vector for the immediate operand of a vector shift left operation. More...
 
static bool isVShiftRImm (SDValue Op, EVT VT, bool isNarrow, bool isIntrinsic, int64_t &Cnt)
 isVShiftRImm - Check if this is a valid build_vector for the immediate operand of a vector shift right operation. More...
 
static SDValue EmitVectorComparison (SDValue LHS, SDValue RHS, AArch64CC::CondCode CC, bool NoNans, EVT VT, SDLoc dl, SelectionDAG &DAG)
 
static ConstantgetSequentialMask (IRBuilder<> &Builder, unsigned Start, unsigned NumElts)
 Get a mask consisting of sequential integers starting from Start. More...
 
static bool memOpAlign (unsigned DstAlign, unsigned SrcAlign, unsigned AlignCheck)
 
static SDValue performIntegerAbsCombine (SDNode *N, SelectionDAG &DAG)
 
static SDValue performXorCombine (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const AArch64Subtarget *Subtarget)
 
static SDValue performMulCombine (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const AArch64Subtarget *Subtarget)
 
static SDValue performVectorCompareAndMaskUnaryOpCombine (SDNode *N, SelectionDAG &DAG)
 
static SDValue performIntToFpCombine (SDNode *N, SelectionDAG &DAG, const AArch64Subtarget *Subtarget)
 
static bool findEXTRHalf (SDValue N, SDValue &Src, uint32_t &ShiftAmount, bool &FromHi)
 An EXTR instruction is made up of two shifts, ORed together. More...
 
static SDValue tryCombineToEXTR (SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
 EXTR instruction extracts a contiguous chunk of bits from two existing registers viewed as a high/low pair. More...
 
static SDValue tryCombineToBSL (SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
 
static SDValue performORCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const AArch64Subtarget *Subtarget)
 
static SDValue performBitcastCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
 
static SDValue performConcatVectorsCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
 
static SDValue tryCombineFixedPointConvert (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
 
static SDValue tryExtendDUPToExtractHigh (SDValue N, SelectionDAG &DAG)
 
static bool isEssentiallyExtractSubvector (SDValue N)
 
static bool isSetCC (SDValue Op, SetCCInfoAndKind &SetCCInfo)
 Check whether or not Op is a SET_CC operation, either a generic or an AArch64 lowered one. More...
 
static bool isSetCCOrZExtSetCC (const SDValue &Op, SetCCInfoAndKind &Info)
 
static SDValue performSetccAddFolding (SDNode *Op, SelectionDAG &DAG)
 
static SDValue performAddSubLongCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
 
static SDValue tryCombineLongOpWithDup (unsigned IID, SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
 
static SDValue tryCombineShiftImm (unsigned IID, SDNode *N, SelectionDAG &DAG)
 
static SDValue tryCombineCRC32 (unsigned Mask, SDNode *N, SelectionDAG &DAG)
 
static SDValue combineAcrossLanesIntrinsic (unsigned Opc, SDNode *N, SelectionDAG &DAG)
 
static SDValue performIntrinsicCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const AArch64Subtarget *Subtarget)
 
static SDValue performExtendCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
 
static SDValue replaceSplatVectorStore (SelectionDAG &DAG, StoreSDNode *St)
 Replace a splat of a scalar to a vector store by scalar stores of the scalar value. More...
 
static SDValue performSTORECombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG, const AArch64Subtarget *Subtarget)
 
static SDValue performPostLD1Combine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, bool IsLaneOp)
 Target-specific DAG combine function for post-increment LD1 (lane) and post-increment LD1R. More...
 
static SDValue performNEONPostLDSTCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
 Target-specific DAG combine function for NEON load/store intrinsics to merge base address updates. More...
 
static bool checkValueWidth (SDValue V, unsigned width, ISD::LoadExtType &ExtType)
 
static bool isEquivalentMaskless (unsigned CC, unsigned width, ISD::LoadExtType ExtType, signed AddConstant, signed CompConstant)
 
static SDValue performCONDCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG, unsigned CCIndex, unsigned CmpIndex)
 
static SDValue performBRCONDCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
 
static SDValue performVSelectCombine (SDNode *N, SelectionDAG &DAG)
 
static SDValue performSelectCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
 A vector select: "(select vL, vR, (setcc LHS, RHS))" is best performed with the compare-mask instructions rather than going via NZCV, even if LHS and RHS are really scalar. More...
 
static SDValue performSelectCCCombine (SDNode *N, SelectionDAG &DAG)
 performSelectCCCombine - Target-specific DAG combining for ISD::SELECT_CC to match FMIN/FMAX patterns. More...
 
static SDValue performNVCASTCombine (SDNode *N)
 Get rid of unnecessary NVCASTs (that don't change the type). More...
 
static void ReplaceBITCASTResults (SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG)
 

Variables

static cl::opt< boolEnableAArch64ExtrGeneration ("aarch64-extr-generation", cl::Hidden, cl::desc("Allow AArch64 (or (shift)(shift))->extract"), cl::init(true))
 
static cl::opt< boolEnableAArch64SlrGeneration ("aarch64-shift-insert-generation", cl::Hidden, cl::desc("Allow AArch64 SLI/SRI formation"), cl::init(false))
 
cl::opt< boolEnableAArch64ELFLocalDynamicTLSGeneration ("aarch64-elf-ldtls-generation", cl::Hidden, cl::desc("Allow AArch64 Local Dynamic TLS code generation"), cl::init(false))
 

Macro Definition Documentation

#define DEBUG_TYPE   "aarch64-lower"

Definition at line 38 of file AArch64ISelLowering.cpp.

Enumeration Type Documentation

enum AlignMode

Definition at line 44 of file AArch64ISelLowering.cpp.

Function Documentation

static SDValue addRequiredExtensionForVectorMULL ( SDValue  N,
SelectionDAG DAG,
const EVT OrigTy,
const EVT ExtTy,
unsigned  ExtOpcode 
)
static
static cl::opt<AlignMode> Align ( cl::  desc"Load/store alignment support",
cl::Hidden  ,
cl::  initNoStrictAlign,
cl::  valuesclEnumValN(StrictAlign,"aarch64-strict-align","Disallow all unaligned memory accesses"), clEnumValN(NoStrictAlign,"aarch64-no-strict-align","Allow unaligned memory accesses"), clEnumValEnd 
)
static

Referenced by llvm::AArch64TargetLowering::AArch64TargetLowering(), AddAlignmentAssumptions(), llvm::AttrBuilder::addAlignmentAttr(), llvm::AttrBuilder::addStackAlignmentAttr(), AdjustStackOffset(), llvm::Recycler< llvm::MachineInstr >::Allocate(), llvm::ArrayRecycler< llvm::MachineOperand >::allocate(), llvm::MCContext::allocate(), llvm::HexagonFrameLowering::assignCalleeSavedSpillSlots(), CalculateStackSlotAlignment(), CalculateStackSlotUsed(), llvm::CC_ARM_AAPCS_Custom_Aggregate(), clampStackAlignment(), computeKnownBits(), llvm::SelectionDAG::computeKnownBits(), computeKnownBitsFromOperator(), llvm::MachineFrameInfo::CreateFixedObject(), llvm::MachineFrameInfo::CreateFixedSpillStackObject(), llvm::SelectionDAG::CreateStackTemporary(), llvm::AArch64FrameLowering::eliminateCallFramePseudoInstr(), llvm::XCoreFrameLowering::eliminateCallFramePseudoInstr(), llvm::Thumb1FrameLowering::eliminateCallFramePseudoInstr(), llvm::RuntimeDyldImpl::emitCommonSymbols(), llvm::AsmPrinter::EmitConstantPool(), llvm::AsmPrinter::EmitGlobalVariable(), llvm::ARMFrameLowering::emitPrologue(), llvm::X86SelectionDAGInfo::EmitTargetCodeForMemcpy(), llvm::X86SelectionDAGInfo::EmitTargetCodeForMemset(), enforceKnownAlignment(), llvm::MachineFrameInfo::ensureMaxAlignment(), llvm::MachineFrameInfo::estimateStackSize(), FindMemType(), fixupFuncForFI(), llvm::PPCTargetLowering::getByValTypeAlignment(), llvm::X86TargetLowering::getByValTypeAlignment(), llvm::MCSymbolMachO::getEncodedFlags(), llvm::SelectionDAG::getMemcpy(), getMemcpyLoadsAndStores(), llvm::SelectionDAG::getMemmove(), getMemmoveLoadsAndStores(), llvm::MipsInstrInfo::GetMemOperand(), llvm::SelectionDAG::getMemset(), getMemsetStores(), llvm::getOrEnforceKnownAlignment(), llvm::object::MachOObjectFile::getSectionAlignment(), llvm::SelectionDAG::getTargetConstantPool(), llvm::Attribute::getWithAlignment(), llvm::Attribute::getWithStackAlignment(), llvm::CCState::HandleByVal(), HandleByValArgument(), llvm::SelectionDAG::InferPtrAlignment(), llvm::isSafeToLoadUnconditionally(), llvm::HexagonInstrInfo::loadRegFromStackSlot(), llvm::AArch64InstrInfo::loadRegFromStackSlot(), llvm::ARMBaseInstrInfo::loadRegFromStackSlot(), llvm::SparcTargetLowering::LowerCall_32(), llvm::TargetLowering::LowerCallTo(), llvm::HexagonTargetLowering::LowerDYNAMIC_STACKALLOC(), memsetStore(), llvm::Recycler< llvm::MachineInstr >::PrintStats(), llvm::RoundUpToAlignment(), selectELFSectionForGlobal(), llvm::FunctionLoweringInfo::set(), llvm::MachineBasicBlock::setAlignment(), llvm::TargetLoweringBase::setJumpBufAlignment(), llvm::MachineFrameInfo::setLocalFrameMaxAlign(), llvm::TargetLoweringBase::setMinFunctionAlignment(), llvm::TargetLoweringBase::setMinStackArgumentAlignment(), llvm::MachineFrameInfo::setObjectAlignment(), llvm::TargetLoweringBase::setPrefFunctionAlignment(), llvm::TargetLoweringBase::setPrefLoopAlignment(), speculatePHINodeLoads(), llvm::HexagonInstrInfo::storeRegToStackSlot(), llvm::AArch64InstrInfo::storeRegToStackSlot(), llvm::ARMBaseInstrInfo::storeRegToStackSlot(), tryToMakeAllocaBePromotable(), llvm::InstCombiner::visitLoadInst(), and llvm::SelectionDAGBuilder::visitSPDescriptorParent().

static void changeFPCCToAArch64CC ( ISD::CondCode  CC,
AArch64CC::CondCode CondCode,
AArch64CC::CondCode CondCode2 
)
static
static AArch64CC::CondCode changeIntCCToAArch64CC ( ISD::CondCode  CC)
static
static void changeVectorFPCCToAArch64CC ( ISD::CondCode  CC,
AArch64CC::CondCode CondCode,
AArch64CC::CondCode CondCode2,
bool Invert 
)
static

changeVectorFPCCToAArch64CC - Convert a DAG fp condition code to an AArch64 CC usable with the vector instructions.

Fewer operations are available without a real NZCV register, so we have to use less efficient combinations to get the same effect.

Definition at line 1097 of file AArch64ISelLowering.cpp.

References changeFPCCToAArch64CC(), llvm::AArch64CC::GE, llvm::ISD::getSetCCInverse(), llvm::AArch64CC::MI, llvm::ISD::SETO, llvm::ISD::SETUEQ, llvm::ISD::SETUGE, llvm::ISD::SETUGT, llvm::ISD::SETULE, llvm::ISD::SETULT, and llvm::ISD::SETUO.

static bool checkValueWidth ( SDValue  V,
unsigned  width,
ISD::LoadExtType ExtType 
)
static
static SDValue combineAcrossLanesIntrinsic ( unsigned  Opc,
SDNode N,
SelectionDAG DAG 
)
static
static SDValue emitComparison ( SDValue  LHS,
SDValue  RHS,
ISD::CondCode  CC,
SDLoc  dl,
SelectionDAG DAG 
)
static
static SDValue EmitVectorComparison ( SDValue  LHS,
SDValue  RHS,
AArch64CC::CondCode  CC,
bool  NoNans,
EVT  VT,
SDLoc  dl,
SelectionDAG DAG 
)
static
static bool findEXTRHalf ( SDValue  N,
SDValue Src,
uint32_t &  ShiftAmount,
bool FromHi 
)
static

An EXTR instruction is made up of two shifts, ORed together.

This helper searches for and classifies those shifts.

Definition at line 7270 of file AArch64ISelLowering.cpp.

References llvm::SDNode::getConstantOperandVal(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::ISD::SHL, and llvm::ISD::SRL.

Referenced by tryCombineToEXTR().

static SDValue GeneratePerfectShuffle ( unsigned  PFEntry,
SDValue  LHS,
SDValue  RHS,
SelectionDAG DAG,
SDLoc  dl 
)
static
static SDValue GenerateTBL ( SDValue  Op,
ArrayRef< int ShuffleMask,
SelectionDAG DAG 
)
static
static SDValue getAArch64Cmp ( SDValue  LHS,
SDValue  RHS,
ISD::CondCode  CC,
SDValue AArch64cc,
SelectionDAG DAG,
SDLoc  dl 
)
static
static std::pair<SDValue, SDValue> getAArch64XALUOOp ( AArch64CC::CondCode CC,
SDValue  Op,
SelectionDAG DAG 
)
static
static unsigned getDUPLANEOp ( EVT  EltType)
static
static EVT getExtensionTo64Bits ( const EVT OrigVT)
static
static unsigned getExtFactor ( SDValue V)
static

getExtFactor - Determine the adjustment factor for the position when generating an "extract from vector registers" instruction.

Definition at line 4502 of file AArch64ISelLowering.cpp.

References llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), and llvm::EVT::getVectorElementType().

Referenced by GeneratePerfectShuffle(), and llvm::AArch64TargetLowering::ReconstructShuffle().

static unsigned getIntrinsicID ( const SDNode N)
static
static Constant* getSequentialMask ( IRBuilder<> &  Builder,
unsigned  Start,
unsigned  NumElts 
)
static

Get a mask consisting of sequential integers starting from Start.

I.e. <Start, Start + 1, ..., Start + NumElts - 1>

Definition at line 6770 of file AArch64ISelLowering.cpp.

References llvm::ConstantVector::get(), llvm::IRBuilderBase::getInt32(), and llvm::SmallVectorTemplateBase< T, isPodLike< T >::value >::push_back().

Referenced by llvm::AArch64TargetLowering::lowerInterleavedStore().

static bool getVShiftImm ( SDValue  Op,
unsigned  ElementBits,
int64_t &  Cnt 
)
static

getVShiftImm - Check if this is a valid build_vector for the immediate operand of a vector shift operation, where all the elements of the build_vector must have the same constant integer value.

Definition at line 6194 of file AArch64ISelLowering.cpp.

References llvm::ISD::BITCAST, llvm::dyn_cast(), llvm::SDValue::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::APInt::getSExtValue(), and llvm::BuildVectorSDNode::isConstantSplat().

Referenced by isVShiftLImm(), and isVShiftRImm().

static bool isAddSubSExt ( SDNode N,
SelectionDAG DAG 
)
static
static bool isAddSubZExt ( SDNode N,
SelectionDAG DAG 
)
static
static bool isAllConstantBuildVector ( const SDValue PotentialBVec,
uint64_t &  ConstVal 
)
static
static bool isConcatMask ( ArrayRef< int Mask,
EVT  VT,
bool  SplitLHS 
)
static
static bool isEquivalentMaskless ( unsigned  CC,
unsigned  width,
ISD::LoadExtType  ExtType,
signed  AddConstant,
signed  CompConstant 
)
static
static bool isEssentiallyExtractSubvector ( SDValue  N)
static
static bool isExtendedBUILD_VECTOR ( SDNode N,
SelectionDAG DAG,
bool  isSigned 
)
static
static bool isEXTMask ( ArrayRef< int M,
EVT  VT,
bool ReverseEXT,
unsigned Imm 
)
static
static bool isINSMask ( ArrayRef< int M,
int  NumInputElements,
bool DstIsLeft,
int Anomaly 
)
static
static bool isLegalArithImmed ( uint64_t  C)
static

Definition at line 1126 of file AArch64ISelLowering.cpp.

Referenced by getAArch64Cmp().

static bool isREVMask ( ArrayRef< int M,
EVT  VT,
unsigned  BlockSize 
)
static

isREVMask - Check if a vector shuffle corresponds to a REV instruction with the specified blocksize.

(The order of the elements within each block of the vector is reversed.)

Definition at line 4789 of file AArch64ISelLowering.cpp.

References llvm::EVT::getSizeInBits(), llvm::EVT::getVectorElementType(), and llvm::EVT::getVectorNumElements().

Referenced by llvm::AArch64TargetLowering::isShuffleMaskLegal().

static bool isSetCC ( SDValue  Op,
SetCCInfoAndKind SetCCInfo 
)
static

Check whether or not Op is a SET_CC operation, either a generic or an AArch64 lowered one.

SetCCInfo is filled accordingly.

Postcondition
SetCCInfo is meanginfull only when this function returns true.
Returns
True when Op is a kind of SET_CC operation.

Definition at line 7686 of file AArch64ISelLowering.cpp.

References SetCCInfo::AArch64, GenericSetCCInfo::CC, AArch64SetCCInfo::CC, AArch64SetCCInfo::Cmp, llvm::AArch64ISD::CSEL, llvm::dyn_cast(), SetCCInfo::Generic, llvm::AArch64CC::getInvertedCondCode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), SetCCInfoAndKind::Info, SetCCInfoAndKind::IsAArch64, llvm::ConstantSDNode::isOne(), GenericSetCCInfo::Opnd0, GenericSetCCInfo::Opnd1, llvm::ISD::SETCC, and std::swap().

Referenced by isSetCCOrZExtSetCC().

static bool isSetCCOrZExtSetCC ( const SDValue Op,
SetCCInfoAndKind Info 
)
static
static bool isSignExtended ( SDNode N,
SelectionDAG DAG 
)
static
static bool isSingletonEXTMask ( ArrayRef< int M,
EVT  VT,
unsigned Imm 
)
static

Definition at line 4716 of file AArch64ISelLowering.cpp.

References llvm::EVT::getVectorNumElements().

static bool isTRN_v_undef_Mask ( ArrayRef< int M,
EVT  VT,
unsigned WhichResult 
)
static

isTRN_v_undef_Mask - Special case of isTRNMask for canonical form of "vector_shuffle v, v", i.e., "vector_shuffle v, undef".

Mask is e.g., <0, 0, 2, 2> instead of <0, 4, 2, 6>.

Definition at line 4893 of file AArch64ISelLowering.cpp.

References llvm::EVT::getVectorNumElements().

Referenced by llvm::AArch64TargetLowering::isShuffleMaskLegal().

static bool isTRNMask ( ArrayRef< int M,
EVT  VT,
unsigned WhichResult 
)
static
static bool isUZP_v_undef_Mask ( ArrayRef< int M,
EVT  VT,
unsigned WhichResult 
)
static

isUZP_v_undef_Mask - Special case of isUZPMask for canonical form of "vector_shuffle v, v", i.e., "vector_shuffle v, undef".

Mask is e.g., <0, 2, 0, 2> instead of <0, 2, 4, 6>,

Definition at line 4874 of file AArch64ISelLowering.cpp.

References llvm::EVT::getVectorNumElements().

Referenced by llvm::AArch64TargetLowering::isShuffleMaskLegal().

static bool isUZPMask ( ArrayRef< int M,
EVT  VT,
unsigned WhichResult 
)
static
static bool isVShiftLImm ( SDValue  Op,
EVT  VT,
bool  isLong,
int64_t &  Cnt 
)
static

isVShiftLImm - Check if this is a valid build_vector for the immediate operand of a vector shift left operation.

That value must be in the range: 0 <= Value < ElementBits for a left shift; or 0 <= Value <= ElementBits for a long left shift.

Definition at line 6214 of file AArch64ISelLowering.cpp.

References llvm::EVT::getSizeInBits(), llvm::EVT::getVectorElementType(), getVShiftImm(), and llvm::EVT::isVector().

static bool isVShiftRImm ( SDValue  Op,
EVT  VT,
bool  isNarrow,
bool  isIntrinsic,
int64_t &  Cnt 
)
static

isVShiftRImm - Check if this is a valid build_vector for the immediate operand of a vector shift right operation.

For a shift opcode, the value is positive, but for an intrinsic the value count must be negative. The absolute value must be in the range: 1 <= |Value| <= ElementBits for a right shift; or 1 <= |Value| <= ElementBits/2 for a narrow right shift.

Definition at line 6228 of file AArch64ISelLowering.cpp.

References llvm::EVT::getSizeInBits(), llvm::EVT::getVectorElementType(), getVShiftImm(), and llvm::EVT::isVector().

static bool isZeroExtended ( SDNode N,
SelectionDAG DAG 
)
static
static bool isZIP_v_undef_Mask ( ArrayRef< int M,
EVT  VT,
unsigned WhichResult 
)
static

isZIP_v_undef_Mask - Special case of isZIPMask for canonical form of "vector_shuffle v, v", i.e., "vector_shuffle v, undef".

Mask is e.g., <0, 0, 1, 1> instead of <0, 4, 1, 5>.

Definition at line 4857 of file AArch64ISelLowering.cpp.

References llvm::EVT::getVectorNumElements().

Referenced by llvm::AArch64TargetLowering::isShuffleMaskLegal().

static bool isZIPMask ( ArrayRef< int M,
EVT  VT,
unsigned WhichResult 
)
static
static SDValue LowerADDC_ADDE_SUBC_SUBE ( SDValue  Op,
SelectionDAG DAG 
)
static
static SDValue LowerBITCAST ( SDValue  Op,
SelectionDAG DAG 
)
static
static SDValue LowerMUL ( SDValue  Op,
SelectionDAG DAG 
)
static
static SDValue LowerPREFETCH ( SDValue  Op,
SelectionDAG DAG 
)
static
static SDValue LowerVectorFP_TO_INT ( SDValue  Op,
SelectionDAG DAG 
)
static
static SDValue LowerVectorINT_TO_FP ( SDValue  Op,
SelectionDAG DAG 
)
static
static SDValue LowerXALUO ( SDValue  Op,
SelectionDAG DAG 
)
static
static SDValue LowerXOR ( SDValue  Op,
SelectionDAG DAG 
)
static
static bool memOpAlign ( unsigned  DstAlign,
unsigned  SrcAlign,
unsigned  AlignCheck 
)
static
static SDValue NarrowVector ( SDValue  V128Reg,
SelectionDAG DAG 
)
static

NarrowVector - Given a value in the V128 register class, produce the equivalent value in the V64 register class.

Definition at line 4509 of file AArch64ISelLowering.cpp.

References llvm::EVT::getSimpleVT(), llvm::SelectionDAG::getTargetExtractSubreg(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), and llvm::MVT::getVectorVT().

static SDValue NormalizeBuildVector ( SDValue  Op,
SelectionDAG DAG 
)
static
static SDValue performAddSubLongCombine ( SDNode N,
TargetLowering::DAGCombinerInfo DCI,
SelectionDAG DAG 
)
static
static SDValue performBitcastCombine ( SDNode N,
TargetLowering::DAGCombinerInfo DCI,
SelectionDAG DAG 
)
static
static SDValue performBRCONDCombine ( SDNode N,
TargetLowering::DAGCombinerInfo DCI,
SelectionDAG DAG 
)
static
static SDValue performConcatVectorsCombine ( SDNode N,
TargetLowering::DAGCombinerInfo DCI,
SelectionDAG DAG 
)
static
static SDValue performCONDCombine ( SDNode N,
TargetLowering::DAGCombinerInfo DCI,
SelectionDAG DAG,
unsigned  CCIndex,
unsigned  CmpIndex 
)
static
static SDValue performExtendCombine ( SDNode N,
TargetLowering::DAGCombinerInfo DCI,
SelectionDAG DAG 
)
static
static SDValue performIntegerAbsCombine ( SDNode N,
SelectionDAG DAG 
)
static
static SDValue performIntrinsicCombine ( SDNode N,
TargetLowering::DAGCombinerInfo DCI,
const AArch64Subtarget Subtarget 
)
static
static SDValue performIntToFpCombine ( SDNode N,
SelectionDAG DAG,
const AArch64Subtarget Subtarget 
)
static
static SDValue performMulCombine ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const AArch64Subtarget Subtarget 
)
static
static SDValue performNEONPostLDSTCombine ( SDNode N,
TargetLowering::DAGCombinerInfo DCI,
SelectionDAG DAG 
)
static

Target-specific DAG combine function for NEON load/store intrinsics to merge base address updates.

Definition at line 8331 of file AArch64ISelLowering.cpp.

References llvm::ISD::ADD, llvm::TargetLowering::DAGCombinerInfo::CombineTo(), llvm::SelectionDAG::getMemIntrinsicNode(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getMemoryVT(), llvm::SDValue::getNode(), llvm::SDNode::getNumOperands(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getRegister(), llvm::SDValue::getResNo(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVTList(), llvm::MVT::i64, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::TargetLowering::DAGCombinerInfo::isCalledByLegalizer(), llvm::SDNode::isPredecessorOf(), llvm::AArch64ISD::LD1x2post, llvm::AArch64ISD::LD1x3post, llvm::AArch64ISD::LD1x4post, llvm::AArch64ISD::LD2DUPpost, llvm::AArch64ISD::LD2LANEpost, llvm::AArch64ISD::LD2post, llvm::AArch64ISD::LD3DUPpost, llvm::AArch64ISD::LD3LANEpost, llvm::AArch64ISD::LD3post, llvm::AArch64ISD::LD4DUPpost, llvm::AArch64ISD::LD4LANEpost, llvm::AArch64ISD::LD4post, llvm_unreachable, llvm::makeArrayRef(), N, llvm::MVT::Other, llvm::SmallVectorTemplateBase< T, isPodLike< T >::value >::push_back(), llvm::AArch64ISD::ST1x2post, llvm::AArch64ISD::ST1x3post, llvm::AArch64ISD::ST1x4post, llvm::AArch64ISD::ST2LANEpost, llvm::AArch64ISD::ST2post, llvm::AArch64ISD::ST3LANEpost, llvm::AArch64ISD::ST3post, llvm::AArch64ISD::ST4LANEpost, llvm::AArch64ISD::ST4post, llvm::SDNode::use_begin(), and llvm::SDNode::use_end().

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

static SDValue performNVCASTCombine ( SDNode N)
static

Get rid of unnecessary NVCASTs (that don't change the type).

Definition at line 8941 of file AArch64ISelLowering.cpp.

References llvm::SDNode::getOperand(), llvm::SDValue::getValueType(), and llvm::SDNode::getValueType().

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

static SDValue performORCombine ( SDNode N,
TargetLowering::DAGCombinerInfo DCI,
const AArch64Subtarget Subtarget 
)
static
static SDValue performPostLD1Combine ( SDNode N,
TargetLowering::DAGCombinerInfo DCI,
bool  IsLaneOp 
)
static
static SDValue performSelectCCCombine ( SDNode N,
SelectionDAG DAG 
)
static
static SDValue performSelectCombine ( SDNode N,
TargetLowering::DAGCombinerInfo DCI 
)
static
static SDValue performSetccAddFolding ( SDNode Op,
SelectionDAG DAG 
)
static
static SDValue performSTORECombine ( SDNode N,
TargetLowering::DAGCombinerInfo DCI,
SelectionDAG DAG,
const AArch64Subtarget Subtarget 
)
static
static SDValue performVectorCompareAndMaskUnaryOpCombine ( SDNode N,
SelectionDAG DAG 
)
static
static SDValue performVSelectCombine ( SDNode N,
SelectionDAG DAG 
)
static
static SDValue performXorCombine ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const AArch64Subtarget Subtarget 
)
static
static void ReplaceBITCASTResults ( SDNode N,
SmallVectorImpl< SDValue > &  Results,
SelectionDAG DAG 
)
static
static SDValue replaceSplatVectorStore ( SelectionDAG DAG,
StoreSDNode St 
)
static

Replace a splat of a scalar to a vector store by scalar stores of the scalar value.

The load store optimizer pass will merge them to store pair stores. This has better performance than a splat of the scalar followed by a split vector store. Even if the stores are not merged it is four stores vs a dup, followed by an ext.b and two stores.

Definition at line 8110 of file AArch64ISelLowering.cpp.

References llvm::ISD::ADD, llvm::MemSDNode::getAlignment(), llvm::StoreSDNode::getBasePtr(), llvm::MemSDNode::getChain(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::MemSDNode::getPointerInfo(), llvm::SelectionDAG::getStore(), llvm::SDValue::getValue(), llvm::StoreSDNode::getValue(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::MVT::i64, llvm::ISD::INSERT_VECTOR_ELT, llvm::EVT::isFloatingPoint(), llvm::MemSDNode::isNonTemporal(), llvm::MemSDNode::isVolatile(), and fuzzer::min().

Referenced by performSTORECombine().

static bool resolveBuildVector ( BuildVectorSDNode BVN,
APInt CnstBits,
APInt UndefBits 
)
static
static bool selectCCOpsAreFMaxCompatible ( SDValue  Cmp,
SDValue  Result 
)
static

A SELECT_CC operation is really some kind of max or min if both values being compared are, in some sense, equal to the results in either case.

However, it is permissible to compare f32 values and produce directly extended f64 values.

Extending the comparison operands would also be allowed, but is less likely to happen in practice since their use is right here. Note that truncate operations would not be semantically equivalent.

Definition at line 3582 of file AArch64ISelLowering.cpp.

References llvm::APFloat::convert(), llvm::dyn_cast(), llvm::MVT::f64, llvm::ISD::FP_EXTEND, llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::ConstantFPSDNode::getValueAPF(), llvm::SDValue::getValueType(), llvm::APFloat::IEEEdouble, and llvm::APFloat::rmNearestTiesToEven.

Referenced by performSelectCCCombine().

static SDValue skipExtensionForVectorMULL ( SDNode N,
SelectionDAG DAG 
)
static
STATISTIC ( NumTailCalls  ,
"Number of tail calls"   
)
STATISTIC ( NumShiftInserts  ,
"Number of vector shift inserts"   
)
static SDValue tryCombineCRC32 ( unsigned  Mask,
SDNode N,
SelectionDAG DAG 
)
static
static SDValue tryCombineFixedPointConvert ( SDNode N,
TargetLowering::DAGCombinerInfo DCI,
SelectionDAG DAG 
)
static
static SDValue tryCombineLongOpWithDup ( unsigned  IID,
SDNode N,
TargetLowering::DAGCombinerInfo DCI,
SelectionDAG DAG 
)
static
static SDValue tryCombineShiftImm ( unsigned  IID,
SDNode N,
SelectionDAG DAG 
)
static
static SDValue tryCombineToBSL ( SDNode N,
TargetLowering::DAGCombinerInfo DCI 
)
static
static SDValue tryCombineToEXTR ( SDNode N,
TargetLowering::DAGCombinerInfo DCI 
)
static

EXTR instruction extracts a contiguous chunk of bits from two existing registers viewed as a high/low pair.

This function looks for the pattern: (or (shl VAL1, N), (srl VAL2, #RegWidth-N)) and replaces it with an EXTR. Can't quite be done in TableGen because the two immediates aren't independent.

Definition at line 7292 of file AArch64ISelLowering.cpp.

References llvm::TargetLowering::DAGCombinerInfo::DAG, llvm::AArch64ISD::EXTR, findEXTRHalf(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::EVT::getSizeInBits(), llvm::SDNode::getValueType(), llvm::MVT::i32, llvm::MVT::i64, llvm::ISD::OR, and std::swap().

Referenced by performORCombine().

static SDValue tryExtendDUPToExtractHigh ( SDValue  N,
SelectionDAG DAG 
)
static
static SDValue tryFormConcatFromShuffle ( SDValue  Op,
SelectionDAG DAG 
)
static
static SDValue tryLowerToSLI ( SDNode N,
SelectionDAG DAG 
)
static
static SDValue WidenVector ( SDValue  V64Reg,
SelectionDAG DAG 
)
static

Variable Documentation

cl::opt<bool> EnableAArch64ELFLocalDynamicTLSGeneration("aarch64-elf-ldtls-generation", cl::Hidden, cl::desc("Allow AArch64 Local Dynamic TLS code generation"), cl::init(false))
cl::opt<bool> EnableAArch64ExtrGeneration("aarch64-extr-generation", cl::Hidden, cl::desc("Allow AArch64 (or (shift)(shift))->extract"), cl::init(true))
static

Referenced by performORCombine().

cl::opt<bool> EnableAArch64SlrGeneration("aarch64-shift-insert-generation", cl::Hidden, cl::desc("Allow AArch64 SLI/SRI formation"), cl::init(false))
static