LLVM 20.0.0git
Macros | Enumerations | Functions
AArch64ISelDAGToDAG.cpp File Reference
#include "AArch64MachineFunctionInfo.h"
#include "AArch64TargetMachine.h"
#include "MCTargetDesc/AArch64AddressingModes.h"
#include "llvm/ADT/APSInt.h"
#include "llvm/CodeGen/ISDOpcodes.h"
#include "llvm/CodeGen/SelectionDAGISel.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/GlobalValue.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/IntrinsicsAArch64.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/KnownBits.h"
#include "llvm/Support/MathExtras.h"
#include "llvm/Support/raw_ostream.h"
#include "AArch64GenDAGISel.inc"

Go to the source code of this file.

Macros

#define DEBUG_TYPE   "aarch64-isel"
 
#define PASS_NAME   "AArch64 Instruction Selection"
 

Enumerations

enum class  SelectTypeKind { Int1 = 0 , Int = 1 , FP = 2 , AnyType = 3 }
 

Functions

static bool isIntImmediate (const SDNode *N, uint64_t &Imm)
 isIntImmediate - This method tests to see if the node is a constant operand.
 
static bool isIntImmediate (SDValue N, uint64_t &Imm)
 
static bool isOpcWithIntImmediate (const SDNode *N, unsigned Opc, uint64_t &Imm)
 
static bool isIntImmediateEq (SDValue N, const uint64_t ImmExpected)
 
static AArch64_AM::ShiftExtendType getShiftTypeForNode (SDValue N)
 getShiftTypeForNode - Translate a shift node to the corresponding ShiftType value.
 
static bool isWorthFoldingSHL (SDValue V)
 Determine whether it is worth it to fold SHL into the addressing mode.
 
static AArch64_AM::ShiftExtendType getExtendTypeForNode (SDValue N, bool IsLoadStore=false)
 getExtendTypeForNode - Translate an extend node to the corresponding ExtendType value.
 
static SDValue narrowIfNeeded (SelectionDAG *CurDAG, SDValue N)
 Instructions that accept extend modifiers like UXTW expect the register being extended to be a GPR32, but the incoming DAG might be acting on a GPR64 (either via SEXT_INREG or AND).
 
static bool isWorthFoldingADDlow (SDValue N)
 If there's a use of this ADDlow that's not itself a load/store then we'll need to create a real ADD instruction from it anyway and there's no point in folding it into the mem op.
 
static bool isValidAsScaledImmediate (int64_t Offset, unsigned Range, unsigned Size)
 Check if the immediate offset is valid as a scaled immediate.
 
static SDValue Widen (SelectionDAG *CurDAG, SDValue N)
 
static bool isPreferredADD (int64_t ImmOff)
 
static std::tuple< SDValue, SDValueextractPtrauthBlendDiscriminators (SDValue Disc, SelectionDAG *DAG)
 
template<SelectTypeKind Kind>
static unsigned SelectOpcodeFromVT (EVT VT, ArrayRef< unsigned > Opcodes)
 This function selects an opcode from a list of opcodes, which is expected to be the opcode for { 8-bit, 16-bit, 32-bit, 64-bit } element types, in this order.
 
bool SelectSMETile (unsigned &BaseReg, unsigned TileNum)
 
static SDValue NarrowVector (SDValue V128Reg, SelectionDAG &DAG)
 NarrowVector - Given a value in the V128 register class, produce the equivalent value in the V64 register class.
 
static bool isBitfieldExtractOpFromAnd (SelectionDAG *CurDAG, SDNode *N, unsigned &Opc, SDValue &Opd0, unsigned &LSB, unsigned &MSB, unsigned NumberOfIgnoredLowBits, bool BiggerPattern)
 
static bool isBitfieldExtractOpFromSExtInReg (SDNode *N, unsigned &Opc, SDValue &Opd0, unsigned &Immr, unsigned &Imms)
 
static bool isSeveralBitsExtractOpFromShr (SDNode *N, unsigned &Opc, SDValue &Opd0, unsigned &LSB, unsigned &MSB)
 
static bool isBitfieldExtractOpFromShr (SDNode *N, unsigned &Opc, SDValue &Opd0, unsigned &Immr, unsigned &Imms, bool BiggerPattern)
 
static bool isBitfieldExtractOp (SelectionDAG *CurDAG, SDNode *N, unsigned &Opc, SDValue &Opd0, unsigned &Immr, unsigned &Imms, unsigned NumberOfIgnoredLowBits=0, bool BiggerPattern=false)
 
static bool isBitfieldDstMask (uint64_t DstMask, const APInt &BitsToBeInserted, unsigned NumberOfIgnoredHighBits, EVT VT)
 Does DstMask form a complementary pair with the mask provided by BitsToBeInserted, suitable for use in a BFI instruction.
 
static void getUsefulBits (SDValue Op, APInt &UsefulBits, unsigned Depth=0)
 
static void getUsefulBitsFromAndWithImmediate (SDValue Op, APInt &UsefulBits, unsigned Depth)
 
static void getUsefulBitsFromBitfieldMoveOpd (SDValue Op, APInt &UsefulBits, uint64_t Imm, uint64_t MSB, unsigned Depth)
 
static void getUsefulBitsFromUBFM (SDValue Op, APInt &UsefulBits, unsigned Depth)
 
static void getUsefulBitsFromOrWithShiftedReg (SDValue Op, APInt &UsefulBits, unsigned Depth)
 
static void getUsefulBitsFromBFM (SDValue Op, SDValue Orig, APInt &UsefulBits, unsigned Depth)
 
static void getUsefulBitsForUse (SDNode *UserNode, APInt &UsefulBits, SDValue Orig, unsigned Depth)
 
static SDValue getLeftShift (SelectionDAG *CurDAG, SDValue Op, int ShlAmount)
 Create a machine node performing a notional SHL of Op by ShlAmount.
 
static bool isBitfieldPositioningOpFromAnd (SelectionDAG *CurDAG, SDValue Op, bool BiggerPattern, const uint64_t NonZeroBits, SDValue &Src, int &DstLSB, int &Width)
 
static bool isBitfieldPositioningOpFromShl (SelectionDAG *CurDAG, SDValue Op, bool BiggerPattern, const uint64_t NonZeroBits, SDValue &Src, int &DstLSB, int &Width)
 
static bool isBitfieldPositioningOp (SelectionDAG *CurDAG, SDValue Op, bool BiggerPattern, SDValue &Src, int &DstLSB, int &Width)
 Does this tree qualify as an attempt to move a bitfield into position, essentially "(and (shl VAL, N), Mask)" or (shl VAL, N).
 
static bool isSeveralBitsPositioningOpFromShl (const uint64_t ShlImm, SDValue Op, SDValue &Src, int &DstLSB, int &Width)
 
static bool isShiftedMask (uint64_t Mask, EVT VT)
 
static bool tryBitfieldInsertOpFromOrAndImm (SDNode *N, SelectionDAG *CurDAG)
 
static bool isWorthFoldingIntoOrrWithShift (SDValue Dst, SelectionDAG *CurDAG, SDValue &ShiftedOperand, uint64_t &EncodedShiftImm)
 
static bool tryOrrWithShift (SDNode *N, SDValue OrOpd0, SDValue OrOpd1, SDValue Src, SDValue Dst, SelectionDAG *CurDAG, const bool BiggerPattern)
 
static bool tryBitfieldInsertOpFromOr (SDNode *N, const APInt &UsefulBits, SelectionDAG *CurDAG)
 
static bool checkCVTFixedPointOperandWithFBits (SelectionDAG *CurDAG, SDValue N, SDValue &FixedPos, unsigned RegWidth, bool isReciprocal)
 
static int getIntOperandFromRegisterString (StringRef RegString)
 
static EVT getPackedVectorTypeFromPredicateType (LLVMContext &Ctx, EVT PredVT, unsigned NumVec)
 When PredVT is a scalable vector predicate in the form MVT::nx<M>xi1, it builds the correspondent scalable vector of integers MVT::nx<M>xi<bits> s.t.
 
static EVT getMemVTFromNode (LLVMContext &Ctx, SDNode *Root)
 Return the EVT of the data associated to a memory operation in Root.
 

Macro Definition Documentation

◆ DEBUG_TYPE

#define DEBUG_TYPE   "aarch64-isel"

Definition at line 31 of file AArch64ISelDAGToDAG.cpp.

◆ PASS_NAME

#define PASS_NAME   "AArch64 Instruction Selection"

Definition at line 32 of file AArch64ISelDAGToDAG.cpp.

Enumeration Type Documentation

◆ SelectTypeKind

enum class SelectTypeKind
strong
Enumerator
Int1 
Int 
FP 
AnyType 

Definition at line 1752 of file AArch64ISelDAGToDAG.cpp.

Function Documentation

◆ checkCVTFixedPointOperandWithFBits()

static bool checkCVTFixedPointOperandWithFBits ( SelectionDAG CurDAG,
SDValue  N,
SDValue FixedPos,
unsigned  RegWidth,
bool  isReciprocal 
)
static

◆ extractPtrauthBlendDiscriminators()

static std::tuple< SDValue, SDValue > extractPtrauthBlendDiscriminators ( SDValue  Disc,
SelectionDAG DAG 
)
static

◆ getExtendTypeForNode()

static AArch64_AM::ShiftExtendType getExtendTypeForNode ( SDValue  N,
bool  IsLoadStore = false 
)
static

◆ getIntOperandFromRegisterString()

static int getIntOperandFromRegisterString ( StringRef  RegString)
static

◆ getLeftShift()

static SDValue getLeftShift ( SelectionDAG CurDAG,
SDValue  Op,
int  ShlAmount 
)
static

Create a machine node performing a notional SHL of Op by ShlAmount.

If ShlAmount is negative, do a (logical) right-shift instead. If ShlAmount is 0, return Op unchanged.

Definition at line 2991 of file AArch64ISelDAGToDAG.cpp.

References assert(), llvm::BitWidth, llvm::SelectionDAG::getMachineNode(), llvm::EVT::getSizeInBits(), and llvm::SelectionDAG::getTargetConstant().

Referenced by isBitfieldPositioningOpFromAnd(), and isBitfieldPositioningOpFromShl().

◆ getMemVTFromNode()

static EVT getMemVTFromNode ( LLVMContext Ctx,
SDNode Root 
)
static

◆ getPackedVectorTypeFromPredicateType()

static EVT getPackedVectorTypeFromPredicateType ( LLVMContext Ctx,
EVT  PredVT,
unsigned  NumVec 
)
static

When PredVT is a scalable vector predicate in the form MVT::nx<M>xi1, it builds the correspondent scalable vector of integers MVT::nx<M>xi<bits> s.t.

M x bits = 128. When targeting structured vectors (NumVec >1), the output data type is MVT::nx<M*NumVec>xi<bits> s.t. M x bits = 128. If the input PredVT is not in the form MVT::nx<M>xi1, it returns an invalid EVT.

Definition at line 7107 of file AArch64ISelDAGToDAG.cpp.

References assert(), llvm::EVT::getIntegerVT(), llvm::EVT::getVectorElementCount(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorVT(), llvm::EVT::isScalableVector(), and llvm::AArch64::SVEBitsPerBlock.

Referenced by getMemVTFromNode().

◆ getShiftTypeForNode()

static AArch64_AM::ShiftExtendType getShiftTypeForNode ( SDValue  N)
static

getShiftTypeForNode - Translate a shift node to the corresponding ShiftType value.

Definition at line 643 of file AArch64ISelDAGToDAG.cpp.

References llvm::AArch64_AM::ASR, llvm::AArch64_AM::InvalidShiftExtend, llvm::AArch64_AM::LSL, llvm::AArch64_AM::LSR, N, llvm::AArch64_AM::ROR, llvm::ISD::ROTR, llvm::ISD::SHL, llvm::ISD::SRA, and llvm::ISD::SRL.

◆ getUsefulBits()

static void getUsefulBits ( SDValue  Op,
APInt UsefulBits,
unsigned  Depth = 0 
)
static

◆ getUsefulBitsForUse()

static void getUsefulBitsForUse ( SDNode UserNode,
APInt UsefulBits,
SDValue  Orig,
unsigned  Depth 
)
static

◆ getUsefulBitsFromAndWithImmediate()

static void getUsefulBitsFromAndWithImmediate ( SDValue  Op,
APInt UsefulBits,
unsigned  Depth 
)
static

◆ getUsefulBitsFromBFM()

static void getUsefulBitsFromBFM ( SDValue  Op,
SDValue  Orig,
APInt UsefulBits,
unsigned  Depth 
)
static

◆ getUsefulBitsFromBitfieldMoveOpd()

static void getUsefulBitsFromBitfieldMoveOpd ( SDValue  Op,
APInt UsefulBits,
uint64_t  Imm,
uint64_t  MSB,
unsigned  Depth 
)
static

◆ getUsefulBitsFromOrWithShiftedReg()

static void getUsefulBitsFromOrWithShiftedReg ( SDValue  Op,
APInt UsefulBits,
unsigned  Depth 
)
static

◆ getUsefulBitsFromUBFM()

static void getUsefulBitsFromUBFM ( SDValue  Op,
APInt UsefulBits,
unsigned  Depth 
)
static

Definition at line 2823 of file AArch64ISelDAGToDAG.cpp.

References llvm::Depth, and getUsefulBitsFromBitfieldMoveOpd().

Referenced by getUsefulBitsForUse().

◆ isBitfieldDstMask()

static bool isBitfieldDstMask ( uint64_t  DstMask,
const APInt BitsToBeInserted,
unsigned  NumberOfIgnoredHighBits,
EVT  VT 
)
static

Does DstMask form a complementary pair with the mask provided by BitsToBeInserted, suitable for use in a BFI instruction.

Roughly speaking, this asks whether DstMask zeroes precisely those bits that will be set by the other half.

Definition at line 2757 of file AArch64ISelDAGToDAG.cpp.

References assert(), llvm::BitWidth, llvm::EVT::getSizeInBits(), and llvm::APInt::zextOrTrunc().

Referenced by tryBitfieldInsertOpFromOr().

◆ isBitfieldExtractOp()

static bool isBitfieldExtractOp ( SelectionDAG CurDAG,
SDNode N,
unsigned Opc,
SDValue Opd0,
unsigned Immr,
unsigned Imms,
unsigned  NumberOfIgnoredLowBits = 0,
bool  BiggerPattern = false 
)
static

◆ isBitfieldExtractOpFromAnd()

static bool isBitfieldExtractOpFromAnd ( SelectionDAG CurDAG,
SDNode N,
unsigned Opc,
SDValue Opd0,
unsigned LSB,
unsigned MSB,
unsigned  NumberOfIgnoredLowBits,
bool  BiggerPattern 
)
static

◆ isBitfieldExtractOpFromSExtInReg()

static bool isBitfieldExtractOpFromSExtInReg ( SDNode N,
unsigned Opc,
SDValue Opd0,
unsigned Immr,
unsigned Imms 
)
static

◆ isBitfieldExtractOpFromShr()

static bool isBitfieldExtractOpFromShr ( SDNode N,
unsigned Opc,
SDValue Opd0,
unsigned Immr,
unsigned Imms,
bool  BiggerPattern 
)
static

◆ isBitfieldPositioningOp()

static bool isBitfieldPositioningOp ( SelectionDAG CurDAG,
SDValue  Op,
bool  BiggerPattern,
SDValue Src,
int &  DstLSB,
int &  Width 
)
static

Does this tree qualify as an attempt to move a bitfield into position, essentially "(and (shl VAL, N), Mask)" or (shl VAL, N).

Definition at line 3035 of file AArch64ISelDAGToDAG.cpp.

References llvm::ISD::AND, assert(), llvm::BitWidth, llvm::SelectionDAG::computeKnownBits(), llvm::EVT::getSizeInBits(), isBitfieldPositioningOpFromAnd(), isBitfieldPositioningOpFromShl(), llvm::isShiftedMask_64(), and llvm::ISD::SHL.

Referenced by tryBitfieldInsertOpFromOr().

◆ isBitfieldPositioningOpFromAnd()

static bool isBitfieldPositioningOpFromAnd ( SelectionDAG CurDAG,
SDValue  Op,
bool  BiggerPattern,
const uint64_t  NonZeroBits,
SDValue Src,
int &  DstLSB,
int &  Width 
)
static

◆ isBitfieldPositioningOpFromShl()

static bool isBitfieldPositioningOpFromShl ( SelectionDAG CurDAG,
SDValue  Op,
bool  BiggerPattern,
const uint64_t  NonZeroBits,
SDValue Src,
int &  DstLSB,
int &  Width 
)
static

◆ isIntImmediate() [1/2]

static bool isIntImmediate ( const SDNode N,
uint64_t Imm 
)
static

isIntImmediate - This method tests to see if the node is a constant operand.

If so Imm will receive the 32-bit value.

Definition at line 518 of file AArch64ISelDAGToDAG.cpp.

References llvm::CallingConv::C, and N.

Referenced by isBitfieldExtractOpFromShr(), isIntImmediate(), isIntImmediateEq(), isOpcWithIntImmediate(), and isSeveralBitsExtractOpFromShr().

◆ isIntImmediate() [2/2]

static bool isIntImmediate ( SDValue  N,
uint64_t Imm 
)
static

Definition at line 528 of file AArch64ISelDAGToDAG.cpp.

References isIntImmediate(), and N.

◆ isIntImmediateEq()

static bool isIntImmediateEq ( SDValue  N,
const uint64_t  ImmExpected 
)
static

Definition at line 544 of file AArch64ISelDAGToDAG.cpp.

References isIntImmediate(), and N.

Referenced by isSeveralBitsPositioningOpFromShl().

◆ isOpcWithIntImmediate()

static bool isOpcWithIntImmediate ( const SDNode N,
unsigned  Opc,
uint64_t Imm 
)
static

◆ isPreferredADD()

static bool isPreferredADD ( int64_t  ImmOff)
static

Definition at line 1299 of file AArch64ISelDAGToDAG.cpp.

◆ isSeveralBitsExtractOpFromShr()

static bool isSeveralBitsExtractOpFromShr ( SDNode N,
unsigned Opc,
SDValue Opd0,
unsigned LSB,
unsigned MSB 
)
static

◆ isSeveralBitsPositioningOpFromShl()

static bool isSeveralBitsPositioningOpFromShl ( const uint64_t  ShlImm,
SDValue  Op,
SDValue Src,
int &  DstLSB,
int &  Width 
)
static

◆ isShiftedMask()

static bool isShiftedMask ( uint64_t  Mask,
EVT  VT 
)
static

◆ isValidAsScaledImmediate()

static bool isValidAsScaledImmediate ( int64_t  Offset,
unsigned  Range,
unsigned  Size 
)
static

Check if the immediate offset is valid as a scaled immediate.

Definition at line 1024 of file AArch64ISelDAGToDAG.cpp.

References llvm::Log2_32(), llvm::Offset, Range, and Size.

◆ isWorthFoldingADDlow()

static bool isWorthFoldingADDlow ( SDValue  N)
static

If there's a use of this ADDlow that's not itself a load/store then we'll need to create a real ADD instruction from it anyway and there's no point in folding it into the mem op.

Theoretically, it shouldn't matter, but there's a single pseudo-instruction for an ADRP/ADD pair so over-aggressive folding leads to duplicated ADRP instructions.

Definition at line 1007 of file AArch64ISelDAGToDAG.cpp.

References llvm::ISD::ATOMIC_LOAD, llvm::ISD::ATOMIC_STORE, llvm::isStrongerThanMonotonic(), llvm::ISD::LOAD, N, and llvm::ISD::STORE.

◆ isWorthFoldingIntoOrrWithShift()

static bool isWorthFoldingIntoOrrWithShift ( SDValue  Dst,
SelectionDAG CurDAG,
SDValue ShiftedOperand,
uint64_t EncodedShiftImm 
)
static

◆ isWorthFoldingSHL()

static bool isWorthFoldingSHL ( SDValue  V)
static

Determine whether it is worth it to fold SHL into the addressing mode.

Definition at line 660 of file AArch64ISelDAGToDAG.cpp.

References assert(), llvm::ISD::SHL, and llvm::SDNode::uses().

◆ narrowIfNeeded()

static SDValue narrowIfNeeded ( SelectionDAG CurDAG,
SDValue  N 
)
static

Instructions that accept extend modifiers like UXTW expect the register being extended to be a GPR32, but the incoming DAG might be acting on a GPR64 (either via SEXT_INREG or AND).

Extract the appropriate low bits if this is the case.

Definition at line 898 of file AArch64ISelDAGToDAG.cpp.

References llvm::SelectionDAG::getTargetExtractSubreg(), and N.

◆ NarrowVector()

static SDValue NarrowVector ( SDValue  V128Reg,
SelectionDAG DAG 
)
static

NarrowVector - Given a value in the V128 register class, produce the equivalent value in the V64 register class.

Definition at line 2263 of file AArch64ISelDAGToDAG.cpp.

References llvm::EVT::getSimpleVT(), llvm::SelectionDAG::getTargetExtractSubreg(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), and llvm::MVT::getVectorVT().

◆ SelectOpcodeFromVT()

template<SelectTypeKind Kind>
static unsigned SelectOpcodeFromVT ( EVT  VT,
ArrayRef< unsigned Opcodes 
)
static

This function selects an opcode from a list of opcodes, which is expected to be the opcode for { 8-bit, 16-bit, 32-bit, 64-bit } element types, in this order.

Definition at line 1763 of file AArch64ISelDAGToDAG.cpp.

References AnyType, FP, llvm::EVT::getVectorElementType(), llvm::EVT::getVectorMinNumElements(), Int, Int1, llvm::EVT::isScalableVector(), llvm::Offset, and llvm::ArrayRef< T >::size().

◆ SelectSMETile()

bool SelectSMETile ( unsigned BaseReg,
unsigned  TileNum 
)

Definition at line 2027 of file AArch64ISelDAGToDAG.cpp.

◆ tryBitfieldInsertOpFromOr()

static bool tryBitfieldInsertOpFromOr ( SDNode N,
const APInt UsefulBits,
SelectionDAG CurDAG 
)
static

◆ tryBitfieldInsertOpFromOrAndImm()

static bool tryBitfieldInsertOpFromOrAndImm ( SDNode N,
SelectionDAG CurDAG 
)
static

◆ tryOrrWithShift()

static bool tryOrrWithShift ( SDNode N,
SDValue  OrOpd0,
SDValue  OrOpd1,
SDValue  Src,
SDValue  Dst,
SelectionDAG CurDAG,
const bool  BiggerPattern 
)
static

◆ Widen()

static SDValue Widen ( SelectionDAG CurDAG,
SDValue  N 
)
static