LLVM 24.0.0git
Classes | Macros | Functions | Variables
SIISelLowering.cpp File Reference

Custom DAG lowering for SI. More...

#include "SIISelLowering.h"
#include "AMDGPU.h"
#include "AMDGPUIGroupLP.h"
#include "AMDGPUInstrInfo.h"
#include "AMDGPULaneMaskUtils.h"
#include "AMDGPUMemoryUtils.h"
#include "AMDGPUSelectionDAGInfo.h"
#include "AMDGPUTargetMachine.h"
#include "GCNSubtarget.h"
#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
#include "SIMachineFunctionInfo.h"
#include "SIRegisterInfo.h"
#include "llvm/ADT/APFloat.h"
#include "llvm/ADT/APInt.h"
#include "llvm/ADT/FloatingPointMode.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/Analysis/OptimizationRemarkEmitter.h"
#include "llvm/Analysis/UniformityAnalysis.h"
#include "llvm/CodeGen/Analysis.h"
#include "llvm/CodeGen/ByteProvider.h"
#include "llvm/CodeGen/FunctionLoweringInfo.h"
#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h"
#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineLoopInfo.h"
#include "llvm/CodeGen/PseudoSourceValueManager.h"
#include "llvm/CodeGen/SDPatternMatch.h"
#include "llvm/IR/DiagnosticInfo.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/IntrinsicsAMDGPU.h"
#include "llvm/IR/IntrinsicsR600.h"
#include "llvm/IR/MDBuilder.h"
#include "llvm/Support/AMDGPUAddrSpace.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/KnownBits.h"
#include "llvm/Support/ModRef.h"
#include "llvm/TargetParser/AtomicScope.h"
#include "llvm/Transforms/Utils/LowerAtomic.h"
#include <optional>

Go to the source code of this file.

Classes

struct  DotSrc

Macros

#define DEBUG_TYPE   "si-lower"

Functions

 STATISTIC (NumTailCalls, "Number of tail calls")
static DenormalFPEnv getDenormalFPEnv (const MachineFunction &MF)
static bool denormalModeIsFlushAllF32 (const MachineFunction &MF)
static bool denormalModeIsFlushAllF64F16 (const MachineFunction &MF)
static unsigned findFirstFreeSGPR (CCState &CCInfo)
static EVT memVTFromLoadIntrData (const SITargetLowering &TLI, const DataLayout &DL, Type *Ty, unsigned MaxNumLanes)
static EVT memVTFromLoadIntrReturn (const SITargetLowering &TLI, const DataLayout &DL, Type *Ty, unsigned MaxNumLanes)
static unsigned getIntrMemWidth (unsigned IntrID)
static AtomicOrdering parseAtomicOrderingCABIArg (const CallBase &CI, unsigned ArgIdx)
static unsigned parseSyncscopeMDArg (const CallBase &CI, unsigned ArgIdx)
static void processPSInputArgs (SmallVectorImpl< ISD::InputArg > &Splits, CallingConv::ID CallConv, ArrayRef< ISD::InputArg > Ins, BitVector &Skipped, FunctionType *FType, SIMachineFunctionInfo *Info)
static ArgDescriptor allocateSGPR32InputImpl (CCState &CCInfo, const TargetRegisterClass *RC, unsigned NumArgRegs)
static void allocateFixedSGPRInputImpl (CCState &CCInfo, const TargetRegisterClass *RC, MCRegister Reg)
static void allocateSGPR32Input (CCState &CCInfo, ArgDescriptor &Arg)
static void allocateSGPR64Input (CCState &CCInfo, ArgDescriptor &Arg)
static void reservePrivateMemoryRegs (const TargetMachine &TM, MachineFunction &MF, const SIRegisterInfo &TRI, SIMachineFunctionInfo &Info)
static std::pair< MachineBasicBlock *, MachineBasicBlock * > splitBlockForLoop (MachineInstr &MI, MachineBasicBlock &MBB, bool InstInLoop)
static MachineBasicBlock::iterator emitLoadM0FromVGPRLoop (const SIInstrInfo *TII, MachineRegisterInfo &MRI, MachineBasicBlock &OrigBB, MachineBasicBlock &LoopBB, const DebugLoc &DL, const MachineOperand &Idx, unsigned InitReg, unsigned ResultReg, unsigned PhiReg, unsigned InitSaveExecReg, int Offset, bool UseGPRIdxMode, Register &SGPRIdxReg)
static MachineBasicBlock::iterator loadM0FromVGPR (const SIInstrInfo *TII, MachineBasicBlock &MBB, MachineInstr &MI, unsigned InitResultReg, unsigned PhiReg, int Offset, bool UseGPRIdxMode, Register &SGPRIdxReg)
static std::pair< unsigned, int > computeIndirectRegAndOffset (const SIRegisterInfo &TRI, const TargetRegisterClass *SuperRC, unsigned VecReg, int Offset)
static void setM0ToIndexFromSGPR (const SIInstrInfo *TII, MachineRegisterInfo &MRI, MachineInstr &MI, int Offset)
static Register getIndirectSGPRIdx (const SIInstrInfo *TII, MachineRegisterInfo &MRI, MachineInstr &MI, int Offset)
static MachineBasicBlock * emitIndirectSrc (MachineInstr &MI, MachineBasicBlock &MBB, const GCNSubtarget &ST)
static MachineBasicBlock * emitIndirectDst (MachineInstr &MI, MachineBasicBlock &MBB, const GCNSubtarget &ST)
static MachineBasicBlock * expand64BitScalarArithmetic (MachineInstr &MI, MachineBasicBlock *BB)
static void expand64BitV_CNDMASK (MachineInstr &MI, MachineBasicBlock *BB)
static uint64_t getIdentityValueForWaveReduction (unsigned Opc)
static bool is32bitWaveReduceOperation (unsigned Opc)
static bool isFloatingPointWaveReduceOperation (unsigned Opc)
static std::tuple< unsigned, unsigned > getDPPOpcForWaveReduction (unsigned Opc, const GCNSubtarget &ST)
static std::pair< Register, Register > ExtractSubRegs (MachineInstr &MI, MachineOperand &Op, const TargetRegisterClass *SrcRC, const GCNSubtarget &ST, MachineRegisterInfo &MRI)
static MachineBasicBlock * lowerWaveReduce (MachineInstr &MI, MachineBasicBlock &BB, const GCNSubtarget &ST, unsigned Opc)
static std::pair< SDValue, SDValue > splitTFEValueAndStatus (SDValue Op, EVT VT, const SDLoc &DL, SelectionDAG &DAG)
static SDValue adjustLoadValueTypeImpl (SDValue Result, EVT LoadVT, const SDLoc &DL, SelectionDAG &DAG, bool Unpacked)
static SDValue lowerBALLOTIntrinsic (const SITargetLowering &TLI, SDNode *N, SelectionDAG &DAG)
static SDValue lowerBFEIntrinsic (SDValue Op, SelectionDAG &DAG, Intrinsic::ID IntrinsicID)
static SDValue emitRemovedIntrinsicError (SelectionDAG &DAG, const SDLoc &DL, EVT VT)
static SDValue lowerLaneOp (const SITargetLowering &TLI, SDNode *N, SelectionDAG &DAG)
static SDValue lowerWaveShuffle (const SITargetLowering &TLI, SDNode *N, SelectionDAG &DAG)
static SDNode * findUser (SDValue Value, unsigned Opcode)
 Helper function for LowerBRCOND.
static unsigned getExtOpcodeForPromotedOp (SDValue Op)
static bool isKnownNonNull (SDValue Val, SelectionDAG &DAG, const AMDGPUTargetMachine &TM, unsigned AddrSpace)
 Return true if the value is a known valid address, such that a null check is not necessary.
static bool elementPairIsContiguous (ArrayRef< int > Mask, int Elt)
static bool elementPairIsOddToEven (ArrayRef< int > Mask, int Elt)
static SDValue buildPCRelGlobalAddress (SelectionDAG &DAG, const GlobalValue *GV, const SDLoc &DL, int64_t Offset, EVT PtrVT, unsigned GAFlags=SIInstrInfo::MO_NONE)
static SDValue emitNonHSAIntrinsicError (SelectionDAG &DAG, const SDLoc &DL, EVT VT)
static SDValue getBuildDwordsVector (SelectionDAG &DAG, SDLoc DL, ArrayRef< SDValue > Elts)
static SDValue padEltsToUndef (SelectionDAG &DAG, const SDLoc &DL, EVT CastVT, SDValue Src, int ExtraElts)
static SDValue constructRetValue (SelectionDAG &DAG, MachineSDNode *Result, ArrayRef< EVT > ResultTypes, bool IsTexFail, bool Unpacked, bool IsD16, int DMaskPop, int NumVDataDwords, bool IsAtomicPacked16Bit, const SDLoc &DL)
static bool parseTexFail (SDValue TexFailCtrl, SelectionDAG &DAG, SDValue *TFE, SDValue *LWE, bool &IsTexFail)
static void packImage16bitOpsToDwords (SelectionDAG &DAG, SDValue Op, MVT PackVectorVT, SmallVectorImpl< SDValue > &PackedAddrs, unsigned DimIdx, unsigned EndIdx, unsigned NumGradients)
static SDValue diagnoseUnsupportedImage (SelectionDAG &DAG, SDValue Op, ArrayRef< EVT > ResultTypes, const SDLoc &DL, const Twine &Msg)
 Emit a DiagnosticInfoUnsupported for an unsupported image intrinsic and return poison values of ResultTypes, preserving the chain if present.
static SDValue selectSOffset (SDValue SOffset, SelectionDAG &DAG, const GCNSubtarget *Subtarget)
static void initializeM0ToZeroForClusterLoad (SDValue Op, SelectionDAG &DAG, SDLoc DL)
static bool isAsyncLDSDMA (Intrinsic::ID Intr)
static bool isNoUnsignedWrap (SDValue Addr)
static SDValue getLoadExtOrTrunc (SelectionDAG &DAG, ISD::LoadExtType ExtType, SDValue Op, const SDLoc &SL, EVT VT)
static bool addressMayBeAccessedAsPrivate (const MachineMemOperand *MMO, const SIMachineFunctionInfo &Info)
static SDValue getFPBinOp (SelectionDAG &DAG, unsigned Opcode, const SDLoc &SL, EVT VT, SDValue A, SDValue B, SDValue GlueChain, SDNodeFlags Flags)
static SDValue getFPTernOp (SelectionDAG &DAG, unsigned Opcode, const SDLoc &SL, EVT VT, SDValue A, SDValue B, SDValue C, SDValue GlueChain, SDNodeFlags Flags)
static SDValue getSPDenormModeValue (uint32_t SPDenormMode, SelectionDAG &DAG, const SIMachineFunctionInfo *Info, const GCNSubtarget *ST)
static unsigned getBasePtrIndex (const MemSDNode *N)
 MemSDNode::getBasePtr() does not work for intrinsics, which needs to offset by the chain and intrinsic ID.
static bool bitOpWithConstantIsReducible (unsigned Opc, uint32_t Val)
static uint32_t getConstantPermuteMask (uint32_t C)
static uint32_t getPermuteMask (SDValue V)
static const std::optional< ByteProvider< SDValue > > calculateSrcByte (const SDValue Op, uint64_t DestByte, uint64_t SrcIndex=0, unsigned Depth=0)
static const std::optional< ByteProvider< SDValue > > calculateByteProvider (const SDValue &Op, unsigned Index, unsigned Depth, unsigned StartingIndex=0)
static bool isExtendedFrom16Bits (SDValue &Operand)
static bool addresses16Bits (int Mask)
static bool hasNon16BitAccesses (uint64_t PermMask, SDValue &Op, SDValue &OtherOp)
static SDValue getDWordFromOffset (SelectionDAG &DAG, SDLoc SL, SDValue Src, unsigned DWordOffset)
static SDValue matchPERM (SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static bool vectorEltWillFoldAway (SDValue Op)
static unsigned minMaxOpcToMin3Max3Opc (unsigned Opc)
static ConstantFPSDNode * getSplatConstantFP (SDValue Op)
static bool supportsMin3Max3 (const GCNSubtarget &Subtarget, unsigned Opc, EVT VT)
static bool isClampZeroToOne (SDValue A, SDValue B)
static SDValue strictFPExtFromF16 (SelectionDAG &DAG, SDValue Src)
 Return the source of an fp_extend from f16 to f32, or a converted FP constant.
static SDValue getMad64_32 (SelectionDAG &DAG, const SDLoc &SL, EVT VT, SDValue N0, SDValue N1, SDValue N2, bool Signed)
static SDValue tryFoldMADwithSRL (SelectionDAG &DAG, const SDLoc &SL, SDValue MulLHS, SDValue MulRHS, SDValue AddRHS)
static std::optional< ByteProvider< SDValue > > handleMulOperand (const SDValue &MulOperand)
static unsigned addPermMasks (unsigned First, unsigned Second)
static void placeSources (ByteProvider< SDValue > &Src0, ByteProvider< SDValue > &Src1, SmallVectorImpl< DotSrc > &Src0s, SmallVectorImpl< DotSrc > &Src1s, int Step)
static SDValue resolveSources (SelectionDAG &DAG, SDLoc SL, SmallVectorImpl< DotSrc > &Srcs, bool IsSigned, bool IsAny)
static void fixMasks (SmallVectorImpl< DotSrc > &Srcs, unsigned ChainLength)
static bool isMul (const SDValue Op)
static std::optional< bool > checkDot4MulSignedness (const SDValue &N, ByteProvider< SDValue > &Src0, ByteProvider< SDValue > &Src1, const SDValue &S0Op, const SDValue &S1Op, const SelectionDAG &DAG)
static bool isCtlzOpc (unsigned Opc)
static ISD::CondCode tryReduceF64CompareToHiHalf (const ISD::CondCode CC, const SDValue LHS, const SDValue RHS, const SelectionDAG &DAG)
static bool isFrexpExp (SDValue V, SDValue &FrexpInput)
static unsigned SubIdx2Lane (unsigned Idx)
 Helper function for adjustWritemask.
static bool isFrameIndexOp (SDValue Op)
static SDValue buildSMovImm32 (SelectionDAG &DAG, const SDLoc &DL, uint64_t Val)
static bool isImmConstraint (StringRef Constraint)
static uint64_t clearUnusedBits (uint64_t Val, unsigned Size)
static int getAlignedAGPRClassID (unsigned UnalignedClassID)
static void knownBitsForWorkitemID (const GCNSubtarget &ST, GISelValueTracking &VT, KnownBits &Known, unsigned Dim)
static void knownBitsForSBFE (const MachineInstr &MI, GISelValueTracking &VT, KnownBits &Known, const APInt &DemandedElts, unsigned BFEWidth, bool SExt, unsigned Depth)
static bool isCopyFromRegOfInlineAsm (const SDNode *N)
static AtomicFlushDenormalReason getAtomicFlushDenormalReason (const AtomicRMWInst *RMW)
static OptimizationRemark emitAtomicRMWLegalRemark (const AtomicRMWInst *RMW, GlobalFPAtomicLegality MemLegality, AtomicFlushDenormalReason DenormReason)
static bool isV2F16OrV2BF16 (Type *Ty)
static bool isV2F16 (Type *Ty)
static bool isV2BF16 (Type *Ty)
static bool isAtomicRMWLegalIntTy (Type *Ty)
static bool isAtomicRMWLegalXChgTy (const AtomicRMWInst *RMW)
static GlobalFPAtomicLegality getGlobalMemoryFPAtomicLegality (const GCNSubtarget &Subtarget, const AtomicRMWInst *RMW, bool HasSystemScope)
static TargetLowering::AtomicExpansionKind atomicSupportedIfLegalIntType (const AtomicRMWInst *RMW)
static bool flatInstrMayAccessPrivate (const Instruction *I)
 Return if a flat address space atomicrmw can access private memory.
static TargetLowering::AtomicExpansionKind getPrivateAtomicExpansionKind (const GCNSubtarget &STI)
static bool hasCFUser (const Value *V, SmallPtrSet< const Value *, 16 > &Visited, unsigned WaveSize)
static void convertScratchAtomicToFlatAtomic (Instruction *I, unsigned PtrOpIdx)

Variables

static cl::opt< bool > DisableLoopAlignment ("amdgpu-disable-loop-alignment", cl::desc("Do not align and prefetch loops"), cl::init(false))
static cl::opt< bool > UseDivergentRegisterIndexing ("amdgpu-use-divergent-register-indexing", cl::Hidden, cl::desc("Use indirect register addressing for divergent indexes"), cl::init(false))

Detailed Description

Custom DAG lowering for SI.

Definition in file SIISelLowering.cpp.

Macro Definition Documentation

◆ DEBUG_TYPE

#define DEBUG_TYPE   "si-lower"

Definition at line 60 of file SIISelLowering.cpp.

Function Documentation

◆ addPermMasks()

unsigned addPermMasks ( unsigned First,
unsigned Second )
static

Definition at line 17715 of file SIISelLowering.cpp.

References assert(), and llvm::First.

Referenced by placeSources(), and resolveSources().

◆ addresses16Bits()

bool addresses16Bits ( int Mask)
static

Definition at line 15627 of file SIISelLowering.cpp.

References assert().

Referenced by hasNon16BitAccesses().

◆ addressMayBeAccessedAsPrivate()

bool addressMayBeAccessedAsPrivate ( const MachineMemOperand * MMO,
const SIMachineFunctionInfo & Info )
static

Definition at line 13592 of file SIISelLowering.cpp.

◆ adjustLoadValueTypeImpl()

SDValue adjustLoadValueTypeImpl ( SDValue Result,
EVT LoadVT,
const SDLoc & DL,
SelectionDAG & DAG,
bool Unpacked )
static

Definition at line 7955 of file SIISelLowering.cpp.

References llvm::ISD::BITCAST, llvm::EVT::changeTypeToInteger(), DL, llvm::SelectionDAG::ExtractVectorElements(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getPOISON(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::EVT::isVector(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), and llvm::ISD::TRUNCATE.

Referenced by constructRetValue().

◆ allocateFixedSGPRInputImpl()

void allocateFixedSGPRInputImpl ( CCState & CCInfo,
const TargetRegisterClass * RC,
MCRegister Reg )
static

Definition at line 3058 of file SIISelLowering.cpp.

References llvm::MachineFunction::addLiveIn(), llvm::CCState::AllocateReg(), assert(), llvm::CCState::getMachineFunction(), and Reg.

Referenced by allocateSGPR32Input(), and allocateSGPR64Input().

◆ allocateSGPR32Input()

void allocateSGPR32Input ( CCState & CCInfo,
ArgDescriptor & Arg )
static

Definition at line 3067 of file SIISelLowering.cpp.

References allocateFixedSGPRInputImpl(), allocateSGPR32InputImpl(), and llvm::ArgDescriptor::getRegister().

Referenced by llvm::SITargetLowering::allocateSpecialInputSGPRs().

◆ allocateSGPR32InputImpl()

ArgDescriptor allocateSGPR32InputImpl ( CCState & CCInfo,
const TargetRegisterClass * RC,
unsigned NumArgRegs )
static

Definition at line 3038 of file SIISelLowering.cpp.

References llvm::MachineFunction::addLiveIn(), llvm::CCState::AllocateReg(), llvm::ArrayRef(), assert(), llvm::MCRegisterClass::begin(), llvm::ArgDescriptor::createRegister(), llvm::CCState::getFirstUnallocated(), llvm::CCState::getMachineFunction(), Reg, llvm::report_fatal_error(), and llvm::ArrayRef< T >::size().

Referenced by allocateSGPR32Input(), and allocateSGPR64Input().

◆ allocateSGPR64Input()

void allocateSGPR64Input ( CCState & CCInfo,
ArgDescriptor & Arg )
static

Definition at line 3075 of file SIISelLowering.cpp.

References allocateFixedSGPRInputImpl(), allocateSGPR32InputImpl(), and llvm::ArgDescriptor::getRegister().

Referenced by llvm::SITargetLowering::allocateSpecialInputSGPRs().

◆ atomicSupportedIfLegalIntType()

TargetLowering::AtomicExpansionKind atomicSupportedIfLegalIntType ( const AtomicRMWInst * RMW)
static
Returns
Action to perform on AtomicRMWInsts for integer operations.

Definition at line 21096 of file SIISelLowering.cpp.

References llvm::TargetLoweringBase::CmpXChg, llvm::Value::getType(), isAtomicRMWLegalIntTy(), and llvm::TargetLoweringBase::None.

Referenced by llvm::SITargetLowering::shouldExpandAtomicRMWInIR().

◆ bitOpWithConstantIsReducible()

bool bitOpWithConstantIsReducible ( unsigned Opc,
uint32_t Val )
static

Definition at line 14887 of file SIISelLowering.cpp.

References llvm::ISD::AND, Opc, llvm::ISD::OR, and llvm::ISD::XOR.

◆ buildPCRelGlobalAddress()

SDValue buildPCRelGlobalAddress ( SelectionDAG & DAG,
const GlobalValue * GV,
const SDLoc & DL,
int64_t Offset,
EVT PtrVT,
unsigned GAFlags = SIInstrInfo::MO_NONE )
static

Definition at line 10114 of file SIISelLowering.cpp.

References assert(), DL, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getSubtarget(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getTargetGlobalAddress(), llvm::isInt(), llvm::SIInstrInfo::MO_NONE, and llvm::Offset.

◆ buildSMovImm32()

SDValue buildSMovImm32 ( SelectionDAG & DAG,
const SDLoc & DL,
uint64_t Val )
static

Definition at line 20006 of file SIISelLowering.cpp.

References DL, llvm::SelectionDAG::getMachineNode(), llvm::SelectionDAG::getTargetConstant(), and uint64_t.

Referenced by llvm::SITargetLowering::buildRSRC(), and llvm::SITargetLowering::wrapAddr64Rsrc().

◆ calculateByteProvider()

const std::optional< ByteProvider< SDValue > > calculateByteProvider ( const SDValue & Op,
unsigned Index,
unsigned Depth,
unsigned StartingIndex = 0 )
static

Definition at line 15338 of file SIISelLowering.cpp.

References llvm::ISD::AND, llvm::ISD::ANY_EXTEND, llvm::ISD::AssertSext, llvm::ISD::AssertZext, llvm::BitWidth, llvm::ISD::BSWAP, calculateByteProvider(), calculateSrcByte(), llvm::cast(), llvm::ISD::CopyFromReg, llvm::Depth, llvm::dyn_cast(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::ISD::FSHR, llvm::ByteProvider< ISelOp >::getConstantZero(), llvm::SDValue::getValueSizeInBits(), LHS, llvm_unreachable, llvm::ISD::LOAD, llvm::ISD::OR, RHS, llvm::ISD::SHL, llvm::ISD::SIGN_EXTEND, llvm::ISD::SIGN_EXTEND_INREG, llvm::ISD::SRA, llvm::ISD::SRL, llvm::ISD::TRUNCATE, uint64_t, llvm::ISD::ZERO_EXTEND, and llvm::ISD::ZEXTLOAD.

◆ calculateSrcByte()

const std::optional< ByteProvider< SDValue > > calculateSrcByte ( const SDValue Op,
uint64_t DestByte,
uint64_t SrcIndex = 0,
unsigned Depth = 0 )
static

Definition at line 15269 of file SIISelLowering.cpp.

References llvm::ISD::ANY_EXTEND, calculateSrcByte(), llvm::cast(), llvm::Depth, llvm::dyn_cast(), llvm::ByteProvider< ISelOp >::getSrc(), llvm::EVT::getStoreSize(), llvm::SDValue::getValueType(), llvm::EVT::isByteSized(), llvm_unreachable, llvm::ISD::SIGN_EXTEND, llvm::ISD::SIGN_EXTEND_INREG, llvm::ISD::SRA, llvm::ISD::SRL, llvm::ISD::TRUNCATE, uint64_t, and llvm::ISD::ZERO_EXTEND.

Referenced by calculateByteProvider(), and calculateSrcByte().

◆ checkDot4MulSignedness()

std::optional< bool > checkDot4MulSignedness ( const SDValue & N,
ByteProvider< SDValue > & Src0,
ByteProvider< SDValue > & Src1,
const SDValue & S0Op,
const SDValue & S1Op,
const SelectionDAG & DAG )
static

Definition at line 17895 of file SIISelLowering.cpp.

References assert(), llvm::SelectionDAG::computeKnownBits(), llvm::KnownBits::countMinLeadingZeros(), llvm::SDValue::getValueSizeInBits(), llvm_unreachable, and N.

◆ clearUnusedBits()

uint64_t clearUnusedBits ( uint64_t Val,
unsigned Size )
static

Definition at line 20260 of file SIISelLowering.cpp.

References llvm::AMDGPU::isInlinableIntLiteral(), llvm::maskTrailingOnes(), Size, and uint64_t.

Referenced by llvm::APInt::APInt(), llvm::APInt::ashrInPlace(), llvm::SITargetLowering::checkAsmConstraintVal(), llvm::APInt::flipAllBits(), llvm::SITargetLowering::LowerAsmOperandForConstraint(), llvm::APInt::operator<<=(), llvm::APInt::operator=(), llvm::APInt::operator^=(), llvm::APInt::operator|=(), and llvm::APInt::setAllBits().

◆ computeIndirectRegAndOffset()

std::pair< unsigned, int > computeIndirectRegAndOffset ( const SIRegisterInfo & TRI,
const TargetRegisterClass * SuperRC,
unsigned VecReg,
int Offset )
static

Definition at line 5450 of file SIISelLowering.cpp.

References llvm::SIRegisterInfo::getSubRegFromChannel(), llvm::Offset, and TRI.

Referenced by emitIndirectDst(), and emitIndirectSrc().

◆ constructRetValue()

SDValue constructRetValue ( SelectionDAG & DAG,
MachineSDNode * Result,
ArrayRef< EVT > ResultTypes,
bool IsTexFail,
bool Unpacked,
bool IsD16,
int DMaskPop,
int NumVDataDwords,
bool IsAtomicPacked16Bit,
const SDLoc & DL )
static

Definition at line 10344 of file SIISelLowering.cpp.

References adjustLoadValueTypeImpl(), llvm::ISD::BITCAST, llvm::EVT::changeTypeToInteger(), llvm::Data, DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::MVT::getVectorVT(), llvm::EVT::isVector(), llvm::MVT::isVector(), padEltsToUndef(), and llvm::ISD::TRUNCATE.

◆ convertScratchAtomicToFlatAtomic()

void convertScratchAtomicToFlatAtomic ( Instruction * I,
unsigned PtrOpIdx )
static

Definition at line 21787 of file SIISelLowering.cpp.

References assert(), llvm::CastInst::CreatePointerCast(), llvm::AMDGPUAS::FLAT_ADDRESS, llvm::PointerType::get(), llvm::Type::getPointerAddressSpace(), llvm::Value::getType(), I, and llvm::AMDGPUAS::PRIVATE_ADDRESS.

Referenced by llvm::SITargetLowering::emitExpandAtomicCmpXchg(), llvm::SITargetLowering::emitExpandAtomicLoad(), llvm::SITargetLowering::emitExpandAtomicRMW(), and llvm::SITargetLowering::emitExpandAtomicStore().

◆ denormalModeIsFlushAllF32()

bool denormalModeIsFlushAllF32 ( const MachineFunction & MF)
static

Definition at line 78 of file SIISelLowering.cpp.

References llvm::SIModeRegisterDefaults::FP32Denormals, llvm::MachineFunction::getInfo(), llvm::SIMachineFunctionInfo::getMode(), and llvm::DenormalMode::getPreserveSign().

Referenced by llvm::SITargetLowering::denormalsEnabledForType(), llvm::SITargetLowering::denormalsEnabledForType(), llvm::SITargetLowering::isFPExtFoldable(), and llvm::SITargetLowering::isFPExtFoldable().

◆ denormalModeIsFlushAllF64F16()

bool denormalModeIsFlushAllF64F16 ( const MachineFunction & MF)
static

Definition at line 83 of file SIISelLowering.cpp.

References llvm::SIModeRegisterDefaults::FP64FP16Denormals, llvm::MachineFunction::getInfo(), llvm::SIMachineFunctionInfo::getMode(), and llvm::DenormalMode::getPreserveSign().

Referenced by llvm::SITargetLowering::denormalsEnabledForType(), and llvm::SITargetLowering::denormalsEnabledForType().

◆ diagnoseUnsupportedImage()

SDValue diagnoseUnsupportedImage ( SelectionDAG & DAG,
SDValue Op,
ArrayRef< EVT > ResultTypes,
const SDLoc & DL,
const Twine & Msg )
static

Emit a DiagnosticInfoUnsupported for an unsupported image intrinsic and return poison values of ResultTypes, preserving the chain if present.

Definition at line 10466 of file SIISelLowering.cpp.

References llvm::LLVMContext::diagnose(), DL, llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getErrorMergeValues(), llvm::MachineFunction::getFunction(), llvm::SelectionDAG::getMachineFunction(), and Msg.

◆ elementPairIsContiguous()

bool elementPairIsContiguous ( ArrayRef< int > Mask,
int Elt )
static

Definition at line 9889 of file SIISelLowering.cpp.

References assert().

◆ elementPairIsOddToEven()

bool elementPairIsOddToEven ( ArrayRef< int > Mask,
int Elt )
static

Definition at line 9894 of file SIISelLowering.cpp.

References assert().

◆ emitAtomicRMWLegalRemark()

OptimizationRemark emitAtomicRMWLegalRemark ( const AtomicRMWInst * RMW,
GlobalFPAtomicLegality MemLegality,
AtomicFlushDenormalReason DenormReason )
static

Definition at line 20970 of file SIISelLowering.cpp.

References DEBUG_TYPE, llvm::Value::getContext(), llvm::AtomicRMWInst::getOperation(), llvm::AtomicRMWInst::getOperationName(), llvm::AtomicRMWInst::getSyncScopeID(), and llvm_unreachable.

Referenced by llvm::SITargetLowering::shouldExpandAtomicRMWInIR().

◆ emitIndirectDst()

MachineBasicBlock * emitIndirectDst ( MachineInstr & MI,
MachineBasicBlock & MBB,
const GCNSubtarget & ST )
static

Definition at line 5593 of file SIISelLowering.cpp.

References llvm::MachineInstrBuilder::add(), llvm::MachineInstrBuilder::addImm(), llvm::MachineInstrBuilder::addReg(), assert(), llvm::BuildMI(), llvm::MachineRegisterInfo::clearKillFlags(), computeIndirectRegAndOffset(), llvm::MachineRegisterInfo::createVirtualRegister(), DL, getIndirectSGPRIdx(), llvm::MachineBasicBlock::getParent(), llvm::MachineOperand::getReg(), llvm::MachineRegisterInfo::getRegClass(), llvm::MachineFunction::getRegInfo(), I, llvm::MachineOperand::isReg(), loadM0FromVGPR(), MBB, MI, llvm::Offset, setM0ToIndexFromSGPR(), TII, and TRI.

Referenced by llvm::SITargetLowering::EmitInstrWithCustomInserter().

◆ emitIndirectSrc()

MachineBasicBlock * emitIndirectSrc ( MachineInstr & MI,
MachineBasicBlock & MBB,
const GCNSubtarget & ST )
static

Definition at line 5507 of file SIISelLowering.cpp.

References llvm::MachineInstrBuilder::addImm(), llvm::MachineInstrBuilder::addReg(), llvm::BuildMI(), computeIndirectRegAndOffset(), llvm::MachineRegisterInfo::createVirtualRegister(), DL, getIndirectSGPRIdx(), llvm::MachineBasicBlock::getParent(), llvm::MachineRegisterInfo::getRegClass(), llvm::MachineFunction::getRegInfo(), I, llvm::Implicit, loadM0FromVGPR(), MBB, MI, llvm::Offset, setM0ToIndexFromSGPR(), TII, and TRI.

Referenced by llvm::SITargetLowering::EmitInstrWithCustomInserter().

◆ emitLoadM0FromVGPRLoop()

MachineBasicBlock::iterator emitLoadM0FromVGPRLoop ( const SIInstrInfo * TII,
MachineRegisterInfo & MRI,
MachineBasicBlock & OrigBB,
MachineBasicBlock & LoopBB,
const DebugLoc & DL,
const MachineOperand & Idx,
unsigned InitReg,
unsigned ResultReg,
unsigned PhiReg,
unsigned InitSaveExecReg,
int Offset,
bool UseGPRIdxMode,
Register & SGPRIdxReg )
static

Definition at line 5304 of file SIISelLowering.cpp.

References llvm::MachineInstrBuilder::addImm(), llvm::MachineInstrBuilder::addMBB(), llvm::MachineInstrBuilder::addReg(), llvm::AMDGPU::LaneMaskConstants::AndSaveExecOpc, llvm::MachineBasicBlock::begin(), llvm::BuildMI(), llvm::MachineRegisterInfo::createVirtualRegister(), DL, llvm::AMDGPU::LaneMaskConstants::ExecReg, llvm::AMDGPU::LaneMaskConstants::get(), llvm::ilist_node_impl< OptionsT >::getIterator(), llvm::MachineBasicBlock::getParent(), llvm::MachineFunction::getSubtarget(), llvm::getUndefRegState(), I, llvm::Offset, llvm::MachineInstrBuilder::setOperandDead(), llvm::MachineRegisterInfo::setSimpleHint(), TII, TRI, and llvm::AMDGPU::LaneMaskConstants::XorTermOpc.

Referenced by loadM0FromVGPR().

◆ emitNonHSAIntrinsicError()

SDValue emitNonHSAIntrinsicError ( SelectionDAG & DAG,
const SDLoc & DL,
EVT VT )
static

Definition at line 10278 of file SIISelLowering.cpp.

References llvm::LLVMContext::diagnose(), DL, llvm::SelectionDAG::getContext(), llvm::MachineFunction::getFunction(), llvm::SelectionDAG::getMachineFunction(), and llvm::SelectionDAG::getPOISON().

◆ emitRemovedIntrinsicError()

SDValue emitRemovedIntrinsicError ( SelectionDAG & DAG,
const SDLoc & DL,
EVT VT )
static

Definition at line 10286 of file SIISelLowering.cpp.

References llvm::LLVMContext::diagnose(), DL, llvm::SelectionDAG::getContext(), llvm::MachineFunction::getFunction(), llvm::SelectionDAG::getMachineFunction(), and llvm::SelectionDAG::getPOISON().

◆ expand64BitScalarArithmetic()

MachineBasicBlock * expand64BitScalarArithmetic ( MachineInstr & MI,
MachineBasicBlock * BB )
static

Definition at line 5696 of file SIISelLowering.cpp.

References llvm::MachineInstrBuilder::add(), llvm::MachineInstrBuilder::addImm(), llvm::MachineInstrBuilder::addReg(), assert(), llvm::BuildMI(), DL, llvm::MachineBasicBlock::getParent(), llvm::MachineOperand::getReg(), llvm::MachineFunction::getRegInfo(), llvm::MachineFunction::getSubtarget(), llvm::Hi, llvm::MachineOperand::isDead(), llvm::MachineOperand::isDef(), MI, Opc, TII, and TRI.

Referenced by llvm::SITargetLowering::EmitInstrWithCustomInserter(), and lowerWaveReduce().

◆ expand64BitV_CNDMASK()

void expand64BitV_CNDMASK ( MachineInstr & MI,
MachineBasicBlock * BB )
static

Definition at line 5755 of file SIISelLowering.cpp.

References llvm::MachineInstrBuilder::add(), llvm::MachineInstrBuilder::addImm(), llvm::MachineInstrBuilder::addReg(), llvm::BuildMI(), llvm::MachineRegisterInfo::createVirtualRegister(), DL, llvm::MachineBasicBlock::getParent(), llvm::MachineFunction::getRegInfo(), llvm::MachineFunction::getSubtarget(), MI, TII, and TRI.

Referenced by llvm::SITargetLowering::EmitInstrWithCustomInserter(), and lowerWaveReduce().

◆ ExtractSubRegs()

std::pair< Register, Register > ExtractSubRegs ( MachineInstr & MI,
MachineOperand & Op,
const TargetRegisterClass * SrcRC,
const GCNSubtarget & ST,
MachineRegisterInfo & MRI )
static

Definition at line 5968 of file SIISelLowering.cpp.

References MI, TII, and TRI.

Referenced by lowerWaveReduce().

◆ findFirstFreeSGPR()

unsigned findFirstFreeSGPR ( CCState & CCInfo)
static

Definition at line 88 of file SIISelLowering.cpp.

References llvm::CCState::isAllocated(), llvm_unreachable, and Reg.

Referenced by llvm::SITargetLowering::allocateSystemSGPRs().

◆ findUser()

SDNode * findUser ( SDValue Value,
unsigned Opcode )
static

Helper function for LowerBRCOND.

Definition at line 8642 of file SIISelLowering.cpp.

References llvm::Value::uses().

◆ fixMasks()

void fixMasks ( SmallVectorImpl< DotSrc > & Srcs,
unsigned ChainLength )
static

Definition at line 17879 of file SIISelLowering.cpp.

◆ flatInstrMayAccessPrivate()

bool flatInstrMayAccessPrivate ( const Instruction * I)
static

Return if a flat address space atomicrmw can access private memory.

Definition at line 21103 of file SIISelLowering.cpp.

References llvm::AMDGPU::hasValueInRangeLikeMetadata(), I, and llvm::AMDGPUAS::PRIVATE_ADDRESS.

Referenced by llvm::SITargetLowering::shouldExpandAtomicCmpXchgInIR(), and llvm::SITargetLowering::shouldExpandAtomicRMWInIR().

◆ getAlignedAGPRClassID()

int getAlignedAGPRClassID ( unsigned UnalignedClassID)
static

Definition at line 20378 of file SIISelLowering.cpp.

Referenced by llvm::SITargetLowering::finalizeLowering().

◆ getAtomicFlushDenormalReason()

AtomicFlushDenormalReason getAtomicFlushDenormalReason ( const AtomicRMWInst * RMW)
static

Definition at line 20958 of file SIISelLowering.cpp.

References llvm::Function::getDenormalMode(), llvm::Type::getFltSemantics(), llvm::Instruction::getFunction(), llvm::DenormalMode::getPreserveSign(), llvm::Type::getScalarType(), llvm::Value::getType(), and llvm::Instruction::hasMetadata().

Referenced by llvm::SITargetLowering::shouldExpandAtomicRMWInIR().

◆ getBasePtrIndex()

unsigned getBasePtrIndex ( const MemSDNode * N)
static

MemSDNode::getBasePtr() does not work for intrinsics, which needs to offset by the chain and intrinsic ID.

Theoretically we would also need to check the specific intrinsic, but they all place the pointer operand first.

Definition at line 14854 of file SIISelLowering.cpp.

References llvm::ISD::INTRINSIC_VOID, llvm::ISD::INTRINSIC_W_CHAIN, N, and llvm::ISD::STORE.

Referenced by llvm::SITargetLowering::hasMemSDNodeUser().

◆ getBuildDwordsVector()

SDValue getBuildDwordsVector ( SelectionDAG & DAG,
SDLoc DL,
ArrayRef< SDValue > Elts )
static

Definition at line 10294 of file SIISelLowering.cpp.

References assert(), DL, llvm::ArrayRef< T >::empty(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getPOISON(), llvm::SDValue::getValueType(), llvm::MVT::getVectorVT(), and llvm::ArrayRef< T >::size().

◆ getConstantPermuteMask()

uint32_t getConstantPermuteMask ( uint32_t C)
static

Definition at line 14962 of file SIISelLowering.cpp.

References C().

Referenced by getPermuteMask().

◆ getDenormalFPEnv()

DenormalFPEnv getDenormalFPEnv ( const MachineFunction & MF)
static

Definition at line 74 of file SIISelLowering.cpp.

References llvm::MachineFunction::getInfo().

Referenced by llvm::SITargetLowering::isFMADLegal(), llvm::SITargetLowering::isFMADLegal(), and llvm::SITargetLowering::isFMAFasterThanFMulAndFAdd().

◆ getDPPOpcForWaveReduction()

std::tuple< unsigned, unsigned > getDPPOpcForWaveReduction ( unsigned Opc,
const GCNSubtarget & ST )
static

Definition at line 5890 of file SIISelLowering.cpp.

References llvm_unreachable, and Opc.

Referenced by lowerWaveReduce().

◆ getDWordFromOffset()

SDValue getDWordFromOffset ( SelectionDAG & DAG,
SDLoc SL,
SDValue Src,
unsigned DWordOffset )
static

Scalar Type

Definition at line 15668 of file SIISelLowering.cpp.

References assert(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::ExtractVectorElements(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBitcastedAnyExtOrTrunc(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getFixedValue(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::SDValue::getValueType(), llvm::EVT::getVectorVT(), llvm::MVT::getVectorVT(), and llvm::ISD::SRL.

Referenced by matchPERM(), and resolveSources().

◆ getExtOpcodeForPromotedOp()

unsigned getExtOpcodeForPromotedOp ( SDValue Op)
static

Definition at line 9010 of file SIISelLowering.cpp.

References llvm::ISD::ABS, llvm::ISD::ADD, llvm::ISD::AND, llvm::ISD::ANY_EXTEND, llvm::cast(), llvm::ISD::isSignedIntSetCC(), llvm_unreachable, llvm::ISD::MUL, llvm::ISD::OR, llvm::ISD::SELECT, llvm::ISD::SETCC, llvm::ISD::SHL, llvm::ISD::SIGN_EXTEND, llvm::ISD::SMAX, llvm::ISD::SMIN, llvm::ISD::SRA, llvm::ISD::SRL, llvm::ISD::SUB, llvm::ISD::UADDSAT, llvm::ISD::UMAX, llvm::ISD::UMIN, llvm::ISD::USUBSAT, llvm::ISD::XOR, and llvm::ISD::ZERO_EXTEND.

◆ getFPBinOp()

SDValue getFPBinOp ( SelectionDAG & DAG,
unsigned Opcode,
const SDLoc & SL,
EVT VT,
SDValue A,
SDValue B,
SDValue GlueChain,
SDNodeFlags Flags )
static

Definition at line 13894 of file SIISelLowering.cpp.

References A(), assert(), B(), llvm::ISD::FMUL, llvm::SelectionDAG::getNode(), llvm::SDNode::getNumValues(), llvm::SDValue::getValue(), llvm::SelectionDAG::getVTList(), and llvm_unreachable.

Referenced by llvm::VPDerivedIVRecipe::computeCost().

◆ getFPTernOp()

SDValue getFPTernOp ( SelectionDAG & DAG,
unsigned Opcode,
const SDLoc & SL,
EVT VT,
SDValue A,
SDValue B,
SDValue C,
SDValue GlueChain,
SDNodeFlags Flags )
static

Definition at line 13917 of file SIISelLowering.cpp.

References A(), assert(), B(), C(), llvm::ISD::FMA, llvm::SelectionDAG::getNode(), llvm::SDNode::getNumValues(), llvm::SDValue::getValue(), llvm::SelectionDAG::getVTList(), and llvm_unreachable.

◆ getGlobalMemoryFPAtomicLegality()

GlobalFPAtomicLegality getGlobalMemoryFPAtomicLegality ( const GCNSubtarget & Subtarget,
const AtomicRMWInst * RMW,
bool HasSystemScope )
static
Returns
whether it's valid to emit a native instruction for RMW, and why, based on the properties of the target memory.

Definition at line 21072 of file SIISelLowering.cpp.

References llvm::Instruction::hasMetadata().

Referenced by llvm::SITargetLowering::shouldExpandAtomicRMWInIR().

◆ getIdentityValueForWaveReduction()

uint64_t getIdentityValueForWaveReduction ( unsigned Opc)
static

Definition at line 5819 of file SIISelLowering.cpp.

References llvm_unreachable, Opc, and uint64_t.

Referenced by lowerWaveReduce().

◆ getIndirectSGPRIdx()

Register getIndirectSGPRIdx ( const SIInstrInfo * TII,
MachineRegisterInfo & MRI,
MachineInstr & MI,
int Offset )
static

Definition at line 5487 of file SIISelLowering.cpp.

References llvm::MachineInstrBuilder::add(), llvm::MachineInstrBuilder::addImm(), llvm::BuildMI(), llvm::MachineRegisterInfo::createVirtualRegister(), DL, I, MBB, MI, llvm::Offset, llvm::MachineInstrBuilder::setOperandDead(), and TII.

Referenced by emitIndirectDst(), and emitIndirectSrc().

◆ getIntrMemWidth()

unsigned getIntrMemWidth ( unsigned IntrID)
static

Definition at line 1440 of file SIISelLowering.cpp.

References llvm_unreachable.

Referenced by llvm::SITargetLowering::getTgtMemIntrinsic().

◆ getLoadExtOrTrunc()

SDValue getLoadExtOrTrunc ( SelectionDAG & DAG,
ISD::LoadExtType ExtType,
SDValue Op,
const SDLoc & SL,
EVT VT )
static

Definition at line 13504 of file SIISelLowering.cpp.

References llvm::ISD::ANY_EXTEND, llvm::EVT::bitsLT(), llvm::ISD::EXTLOAD, llvm::SelectionDAG::getNode(), llvm_unreachable, llvm::ISD::NON_EXTLOAD, llvm::ISD::SEXTLOAD, llvm::ISD::SIGN_EXTEND, llvm::ISD::TRUNCATE, llvm::ISD::ZERO_EXTEND, and llvm::ISD::ZEXTLOAD.

◆ getMad64_32()

SDValue getMad64_32 ( SelectionDAG & DAG,
const SDLoc & SL,
EVT VT,
SDValue N0,
SDValue N1,
SDValue N2,
bool Signed )
static

Definition at line 17493 of file SIISelLowering.cpp.

References llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getVTList(), llvm::AMDGPUISD::MAD_I64_I32, llvm::AMDGPUISD::MAD_U64_U32, llvm::Signed, and llvm::ISD::TRUNCATE.

Referenced by tryFoldMADwithSRL().

◆ getPermuteMask()

uint32_t getPermuteMask ( SDValue V)
static

Definition at line 14986 of file SIISelLowering.cpp.

References llvm::ISD::AND, assert(), C(), llvm::dyn_cast(), getConstantPermuteMask(), llvm::ConstantSDNode::getZExtValue(), llvm::ISD::OR, llvm::ISD::SHL, and llvm::ISD::SRL.

◆ getPrivateAtomicExpansionKind()

TargetLowering::AtomicExpansionKind getPrivateAtomicExpansionKind ( const GCNSubtarget & STI)
static

Definition at line 21110 of file SIISelLowering.cpp.

References llvm::TargetLoweringBase::CustomExpand, and llvm::TargetLoweringBase::NotAtomic.

Referenced by llvm::SITargetLowering::shouldExpandAtomicCmpXchgInIR(), llvm::SITargetLowering::shouldExpandAtomicLoadInIR(), llvm::SITargetLowering::shouldExpandAtomicRMWInIR(), and llvm::SITargetLowering::shouldExpandAtomicStoreInIR().

◆ getSPDenormModeValue()

SDValue getSPDenormModeValue ( uint32_t SPDenormMode,
SelectionDAG & DAG,
const SIMachineFunctionInfo * Info,
const GCNSubtarget * ST )
static

Definition at line 14038 of file SIISelLowering.cpp.

References assert(), llvm::SelectionDAG::getTargetConstant(), and Mode.

◆ getSplatConstantFP()

ConstantFPSDNode * getSplatConstantFP ( SDValue Op)
static

Definition at line 16801 of file SIISelLowering.cpp.

References C(), and llvm::dyn_cast().

◆ handleMulOperand()

std::optional< ByteProvider< SDValue > > handleMulOperand ( const SDValue & MulOperand)
static

Definition at line 17703 of file SIISelLowering.cpp.

References calculateByteProvider().

◆ hasCFUser()

bool hasCFUser ( const Value * V,
SmallPtrSet< const Value *, 16 > & Visited,
unsigned WaveSize )
static

Definition at line 21476 of file SIISelLowering.cpp.

References llvm::dyn_cast(), hasCFUser(), llvm::SmallPtrSetImpl< PtrType >::insert(), llvm::isa(), and IT.

Referenced by hasCFUser(), and llvm::SITargetLowering::requiresUniformRegister().

◆ hasNon16BitAccesses()

bool hasNon16BitAccesses ( uint64_t PermMask,
SDValue & Op,
SDValue & OtherOp )
static

Definition at line 15646 of file SIISelLowering.cpp.

References addresses16Bits(), isExtendedFrom16Bits(), llvm::peekThroughBitcasts(), and uint64_t.

Referenced by matchPERM().

◆ initializeM0ToZeroForClusterLoad()

void initializeM0ToZeroForClusterLoad ( SDValue Op,
SelectionDAG & DAG,
SDLoc DL )
static

Definition at line 11895 of file SIISelLowering.cpp.

References AbstractManglingParser< Derived, Alloc >::Ops, DL, llvm::SelectionDAG::getConstant(), N, and llvm::SelectionDAG::UpdateNodeOperands().

◆ is32bitWaveReduceOperation()

bool is32bitWaveReduceOperation ( unsigned Opc)
static

Definition at line 5871 of file SIISelLowering.cpp.

References Opc.

Referenced by lowerWaveReduce().

◆ isAsyncLDSDMA()

bool isAsyncLDSDMA ( Intrinsic::ID Intr)
static

Definition at line 12677 of file SIISelLowering.cpp.

◆ isAtomicRMWLegalIntTy()

bool isAtomicRMWLegalIntTy ( Type * Ty)
static
Returns
true if atomicrmw integer ops work for the type.

Definition at line 21037 of file SIISelLowering.cpp.

References llvm::dyn_cast(), and IT.

Referenced by atomicSupportedIfLegalIntType(), llvm::SITargetLowering::emitExpandAtomicRMW(), isAtomicRMWLegalXChgTy(), and llvm::SITargetLowering::shouldExpandAtomicRMWInIR().

◆ isAtomicRMWLegalXChgTy()

bool isAtomicRMWLegalXChgTy ( const AtomicRMWInst * RMW)
static
Returns
true if this atomicrmw xchg type can be selected.

Definition at line 21047 of file SIISelLowering.cpp.

References DL, llvm::dyn_cast(), llvm::Function::getDataLayout(), llvm::Instruction::getFunction(), llvm::Value::getType(), and isAtomicRMWLegalIntTy().

Referenced by llvm::SITargetLowering::shouldExpandAtomicRMWInIR().

◆ isClampZeroToOne()

bool isClampZeroToOne ( SDValue A,
SDValue B )
static

Definition at line 17056 of file SIISelLowering.cpp.

References A(), B(), and llvm::dyn_cast().

◆ isCopyFromRegOfInlineAsm()

bool isCopyFromRegOfInlineAsm ( const SDNode * N)
static

Definition at line 20817 of file SIISelLowering.cpp.

References assert(), llvm::ISD::CopyFromReg, llvm::ISD::INLINEASM, llvm::ISD::INLINEASM_BR, and N.

Referenced by llvm::SITargetLowering::isSDNodeSourceOfDivergence().

◆ isCtlzOpc()

bool isCtlzOpc ( unsigned Opc)
static

Definition at line 18244 of file SIISelLowering.cpp.

References llvm::ISD::CTLZ, llvm::ISD::CTLZ_ZERO_POISON, and Opc.

◆ isExtendedFrom16Bits()

bool isExtendedFrom16Bits ( SDValue & Operand)
static

Definition at line 15601 of file SIISelLowering.cpp.

References llvm::ISD::ANY_EXTEND, llvm::cast(), llvm::ISD::EXTLOAD, llvm::SDValue::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::isVector(), llvm::ISD::LOAD, llvm::ISD::SEXTLOAD, llvm::ISD::SIGN_EXTEND, llvm::ISD::ZERO_EXTEND, and llvm::ISD::ZEXTLOAD.

Referenced by hasNon16BitAccesses().

◆ isFloatingPointWaveReduceOperation()

bool isFloatingPointWaveReduceOperation ( unsigned Opc)
static

Definition at line 5881 of file SIISelLowering.cpp.

References Opc.

Referenced by lowerWaveReduce().

◆ isFrameIndexOp()

bool isFrameIndexOp ( SDValue Op)
static

Definition at line 19734 of file SIISelLowering.cpp.

References llvm::ISD::AssertZext, and llvm::isa().

Referenced by llvm::SITargetLowering::legalizeTargetIndependentNode().

◆ isFrexpExp()

bool isFrexpExp ( SDValue V,
SDValue & FrexpInput )
static

Definition at line 19155 of file SIISelLowering.cpp.

References llvm::ISD::FFREXP, llvm::SDValue::getOperand(), llvm::SDPatternMatch::m_IntrinsicWOChain(), llvm::PatternMatch::m_Value(), and llvm::SDPatternMatch::sd_match().

◆ isImmConstraint()

bool isImmConstraint ( StringRef Constraint)
static

Definition at line 20221 of file SIISelLowering.cpp.

References llvm::StringRef::size().

Referenced by llvm::SITargetLowering::getConstraintType(), and llvm::SITargetLowering::LowerAsmOperandForConstraint().

◆ isKnownNonNull()

bool isKnownNonNull ( SDValue Val,
SelectionDAG & DAG,
const AMDGPUTargetMachine & TM,
unsigned AddrSpace )
static

Return true if the value is a known valid address, such that a null check is not necessary.

Definition at line 9518 of file SIISelLowering.cpp.

References llvm::dyn_cast(), llvm::AMDGPU::getNullPointerValue(), and llvm::isa().

◆ isMul()

bool isMul ( const SDValue Op)
static

Definition at line 17887 of file SIISelLowering.cpp.

References llvm::ISD::MUL.

◆ isNoUnsignedWrap()

bool isNoUnsignedWrap ( SDValue Addr)
static

Definition at line 13246 of file SIISelLowering.cpp.

References llvm::ISD::ADD, llvm::SDNode::getFlags(), llvm::SDNode::getOpcode(), llvm::SDValue::getOpcode(), llvm::SDNodeFlags::hasNoUnsignedWrap(), and llvm::ISD::OR.

◆ isV2BF16()

bool isV2BF16 ( Type * Ty)
static

Definition at line 21031 of file SIISelLowering.cpp.

References llvm::dyn_cast(), llvm::VectorType::getElementType(), llvm::FixedVectorType::getNumElements(), and llvm::Type::isBFloatTy().

Referenced by llvm::SITargetLowering::shouldExpandAtomicRMWInIR().

◆ isV2F16()

bool isV2F16 ( Type * Ty)
static

Definition at line 21026 of file SIISelLowering.cpp.

References llvm::dyn_cast(), llvm::VectorType::getElementType(), llvm::FixedVectorType::getNumElements(), and llvm::Type::isHalfTy().

Referenced by llvm::SITargetLowering::shouldExpandAtomicRMWInIR().

◆ isV2F16OrV2BF16()

bool isV2F16OrV2BF16 ( Type * Ty)
static

Definition at line 21016 of file SIISelLowering.cpp.

References llvm::dyn_cast(), llvm::Type::isBFloatTy(), and llvm::Type::isHalfTy().

Referenced by llvm::SITargetLowering::shouldExpandAtomicRMWInIR().

◆ knownBitsForSBFE()

void knownBitsForSBFE ( const MachineInstr & MI,
GISelValueTracking & VT,
KnownBits & Known,
const APInt & DemandedElts,
unsigned BFEWidth,
bool SExt,
unsigned Depth )
static

Definition at line 20543 of file SIISelLowering.cpp.

References llvm::GISelValueTracking::computeKnownBitsImpl(), llvm::Depth, llvm::getIConstantVRegValWithLookThrough(), llvm::GISelValueTracking::getMachineFunction(), llvm::MachineFunction::getRegInfo(), llvm::Known, llvm::maskTrailingOnes(), MI, and llvm::Offset.

Referenced by llvm::SITargetLowering::computeKnownBitsForTargetInstr().

◆ knownBitsForWorkitemID()

void knownBitsForWorkitemID ( const GCNSubtarget & ST,
GISelValueTracking & VT,
KnownBits & Known,
unsigned Dim )
static

Definition at line 20535 of file SIISelLowering.cpp.

References llvm::countl_zero(), llvm::MachineFunction::getFunction(), llvm::GISelValueTracking::getMachineFunction(), and llvm::Known.

Referenced by llvm::SITargetLowering::computeKnownBitsForTargetInstr().

◆ loadM0FromVGPR()

MachineBasicBlock::iterator loadM0FromVGPR ( const SIInstrInfo * TII,
MachineBasicBlock & MBB,
MachineInstr & MI,
unsigned InitResultReg,
unsigned PhiReg,
int Offset,
bool UseGPRIdxMode,
Register & SGPRIdxReg )
static

Definition at line 5400 of file SIISelLowering.cpp.

References llvm::MachineInstrBuilder::addReg(), llvm::MachineBasicBlock::addSuccessor(), llvm::MachineBasicBlock::begin(), llvm::BuildMI(), llvm::MachineFunction::CreateMachineBasicBlock(), llvm::MachineRegisterInfo::createVirtualRegister(), DL, emitLoadM0FromVGPRLoop(), llvm::AMDGPU::LaneMaskConstants::ExecReg, llvm::First, llvm::AMDGPU::LaneMaskConstants::get(), llvm::MachineFunction::getRegInfo(), llvm::MachineFunction::getSubtarget(), I, llvm::MachineFunction::insert(), MBB, MBBI, MI, llvm::AMDGPU::LaneMaskConstants::MovOpc, llvm::Offset, splitBlockForLoop(), TII, and TRI.

Referenced by emitIndirectDst(), and emitIndirectSrc().

◆ lowerBALLOTIntrinsic()

SDValue lowerBALLOTIntrinsic ( const SITargetLowering & TLI,
SDNode * N,
SelectionDAG & DAG )
static

Definition at line 8105 of file SIISelLowering.cpp.

References llvm::dyn_cast(), llvm::ISD::FP_EXTEND, llvm::SelectionDAG::getCondCode(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getCopyFromReg(), llvm::SelectionDAG::getEntryNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getZExtOrTrunc(), N, llvm::ISD::SETCC, and llvm::ISD::SETNE.

◆ lowerBFEIntrinsic()

SDValue lowerBFEIntrinsic ( SDValue Op,
SelectionDAG & DAG,
Intrinsic::ID IntrinsicID )
static

Definition at line 8148 of file SIISelLowering.cpp.

References llvm::LLVMContext::diagnose(), DL, llvm::Intrinsic::getBaseName(), llvm::SelectionDAG::getContext(), llvm::MachineFunction::getFunction(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getPOISON(), llvm::Offset, and llvm::Signed.

◆ lowerLaneOp()

SDValue lowerLaneOp ( const SITargetLowering & TLI,
SDNode * N,
SelectionDAG & DAG )
static

Definition at line 8172 of file SIISelLowering.cpp.

References assert(), llvm::ISD::CONCAT_VECTORS, llvm::ISD::CONVERGENCECTRL_GLUE, llvm::ISD::EXTRACT_SUBVECTOR, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getAnyExtOrTrunc(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::MVT::getIntegerVT(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::EVT::getSimpleVT(), llvm::EVT::getSizeInBits(), llvm::SITargetLowering::getSubtarget(), llvm::SelectionDAG::getTargetConstant(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::SelectionDAG::getVectorIdxConstant(), llvm::EVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::MVT::getVectorVT(), llvm::ISD::INTRINSIC_WO_CHAIN, llvm::EVT::isFloatingPoint(), llvm::AMDGPU::isLegalDPALU_DPPControl(), llvm::EVT::isVector(), llvm_unreachable, N, Operands, llvm::SmallVectorTemplateBase< T, bool >::push_back(), and llvm::MVT::SimpleTy.

◆ lowerWaveReduce()

MachineBasicBlock * lowerWaveReduce ( MachineInstr & MI,
MachineBasicBlock & BB,
const GCNSubtarget & ST,
unsigned Opc )
static

Definition at line 5982 of file SIISelLowering.cpp.

References Accumulator, llvm::MachineInstrBuilder::addImm(), llvm::MachineInstrBuilder::addMBB(), llvm::MachineInstrBuilder::addReg(), assert(), llvm::AMDGPU::DPP::BCAST15, llvm::AMDGPU::DPP::BCAST31, llvm::BuildMI(), llvm::MachineOperand::CreateReg(), llvm::MachineRegisterInfo::createVirtualRegister(), llvm::Dead, DEFAULT, llvm::Define, DL, llvm::DPP, llvm::MachineBasicBlock::end(), expand64BitScalarArithmetic(), expand64BitV_CNDMASK(), ExtractSubRegs(), getDPPOpcForWaveReduction(), getIdentityValueForWaveReduction(), llvm::getImm(), llvm::MachineOperand::getImm(), llvm::MachineInstr::getOperand(), llvm::MachineBasicBlock::getParent(), llvm::MachineOperand::getReg(), llvm::MachineRegisterInfo::getRegClass(), llvm::MachineFunction::getRegInfo(), llvm::MachineRegisterInfo::getVRegDef(), llvm::AMDGPUSubtarget::GFX12, I, Imm, is32bitWaveReduceOperation(), isFloatingPointWaveReduceOperation(), llvm::MachineInstr::isMoveImmediate(), MI, llvm::SISrcMods::NEG, llvm::SISrcMods::NONE, Opc, Register, llvm::AMDGPU::DPP::ROW_SHR_FIRST, llvm::MachineInstrBuilder::setOperandDead(), splitBlockForLoop(), TII, TRI, and uint64_t.

Referenced by llvm::SITargetLowering::EmitInstrWithCustomInserter().

◆ lowerWaveShuffle()

SDValue lowerWaveShuffle ( const SITargetLowering & TLI,
SDNode * N,
SelectionDAG & DAG )
static

Definition at line 8384 of file SIISelLowering.cpp.

References llvm::ISD::AND, assert(), llvm::SelectionDAG::getAllOnesConstant(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getPOISON(), llvm::SelectionDAG::getSelect(), llvm::SelectionDAG::getSetCC(), llvm::SelectionDAG::getShiftAmountConstant(), llvm::EVT::getSizeInBits(), llvm::SITargetLowering::getSubtarget(), llvm::SelectionDAG::getTargetConstant(), llvm::SDNode::getValueType(), llvm::ISD::INTRINSIC_WO_CHAIN, llvm::GCNSubtarget::isWave64(), N, Operands, llvm::ISD::SETEQ, llvm::ISD::SHL, llvm::GCNSubtarget::supportsWaveWideBPermute(), and llvm::ISD::XOR.

◆ matchPERM()

SDValue matchPERM ( SDNode * N,
TargetLowering::DAGCombinerInfo & DCI )
static

Definition at line 15731 of file SIISelLowering.cpp.

References assert(), calculateByteProvider(), llvm::TargetLowering::DAGCombinerInfo::DAG, DL, llvm::ISD::FSHR, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBitcastedAnyExtOrTrunc(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getDataLayout(), getDWordFromOffset(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), hasNon16BitAccesses(), llvm::DataLayout::isBigEndian(), llvm::EVT::isByteSized(), N, P, llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::SmallVectorTemplateCommon< T, typename >::size(), and uint64_t.

Referenced by llvm::SITargetLowering::PerformDAGCombine().

◆ memVTFromLoadIntrData()

EVT memVTFromLoadIntrData ( const SITargetLowering & TLI,
const DataLayout & DL,
Type * Ty,
unsigned MaxNumLanes )
static

Definition at line 1382 of file SIISelLowering.cpp.

References assert(), DL, llvm::dyn_cast(), llvm::TargetLoweringBase::getValueType(), and llvm::EVT::getVectorVT().

Referenced by llvm::SITargetLowering::getTgtMemIntrinsic(), and memVTFromLoadIntrReturn().

◆ memVTFromLoadIntrReturn()

EVT memVTFromLoadIntrReturn ( const SITargetLowering & TLI,
const DataLayout & DL,
Type * Ty,
unsigned MaxNumLanes )
static

Definition at line 1398 of file SIISelLowering.cpp.

References assert(), DL, llvm::dyn_cast(), and memVTFromLoadIntrData().

Referenced by llvm::SITargetLowering::getTgtMemIntrinsic().

◆ minMaxOpcToMin3Max3Opc()

unsigned minMaxOpcToMin3Max3Opc ( unsigned Opc)
static

Definition at line 16736 of file SIISelLowering.cpp.

References llvm::ISD::FMAXIMUM, llvm::ISD::FMAXIMUMNUM, llvm::ISD::FMAXNUM, llvm::ISD::FMAXNUM_IEEE, llvm::ISD::FMINIMUM, llvm::ISD::FMINIMUMNUM, llvm::ISD::FMINNUM, llvm::ISD::FMINNUM_IEEE, llvm_unreachable, Opc, llvm::ISD::SMAX, llvm::ISD::SMIN, llvm::ISD::UMAX, and llvm::ISD::UMIN.

◆ packImage16bitOpsToDwords()

void packImage16bitOpsToDwords ( SelectionDAG & DAG,
SDValue Op,
MVT PackVectorVT,
SmallVectorImpl< SDValue > & PackedAddrs,
unsigned DimIdx,
unsigned EndIdx,
unsigned NumGradients )
static

Definition at line 10435 of file SIISelLowering.cpp.

References llvm::ISD::ANY_EXTEND, DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getNode(), llvm::SDValue::getValueType(), I, and llvm::SmallVectorTemplateBase< T, bool >::push_back().

◆ padEltsToUndef()

SDValue padEltsToUndef ( SelectionDAG & DAG,
const SDLoc & DL,
EVT CastVT,
SDValue Src,
int ExtraElts )
static

Definition at line 10323 of file SIISelLowering.cpp.

References DL, llvm::SelectionDAG::ExtractVectorElements(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getPOISON(), llvm::EVT::getScalarType(), llvm::EVT::isVector(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), and llvm::Undef.

Referenced by constructRetValue().

◆ parseAtomicOrderingCABIArg()

AtomicOrdering parseAtomicOrderingCABIArg ( const CallBase & CI,
unsigned ArgIdx )
static

Definition at line 1475 of file SIISelLowering.cpp.

References llvm::Acquire, llvm::acquire, llvm::cast(), llvm::CallBase::getArgOperand(), llvm::Monotonic, llvm::Release, llvm::release, llvm::seq_cst, and llvm::SequentiallyConsistent.

Referenced by llvm::SITargetLowering::getTgtMemIntrinsic().

◆ parseSyncscopeMDArg()

unsigned parseSyncscopeMDArg ( const CallBase & CI,
unsigned ArgIdx )
static

Definition at line 1494 of file SIISelLowering.cpp.

References llvm::cast(), llvm::CallBase::getArgOperand(), llvm::Value::getContext(), llvm::MDNode::getOperand(), and llvm::LLVMContext::getOrInsertSyncScopeID().

Referenced by llvm::SITargetLowering::getTgtMemIntrinsic().

◆ parseTexFail()

bool parseTexFail ( SDValue TexFailCtrl,
SelectionDAG & DAG,
SDValue * TFE,
SDValue * LWE,
bool & IsTexFail )
static

Definition at line 10417 of file SIISelLowering.cpp.

References llvm::cast(), DL, llvm::SDValue::getNode(), llvm::SelectionDAG::getTargetConstant(), and uint64_t.

◆ placeSources()

void placeSources ( ByteProvider< SDValue > & Src0,
ByteProvider< SDValue > & Src1,
SmallVectorImpl< DotSrc > & Src0s,
SmallVectorImpl< DotSrc > & Src1s,
int Step )
static

Definition at line 17735 of file SIISelLowering.cpp.

References addPermMasks(), assert(), llvm::SmallVectorTemplateCommon< T, typename >::end(), llvm::find_if(), I, and llvm::SmallVectorTemplateBase< T, bool >::push_back().

◆ processPSInputArgs()

void processPSInputArgs ( SmallVectorImpl< ISD::InputArg > & Splits,
CallingConv::ID CallConv,
ArrayRef< ISD::InputArg > Ins,
BitVector & Skipped,
FunctionType * FType,
SIMachineFunctionInfo * Info )
static

Definition at line 2945 of file SIISelLowering.cpp.

References llvm::CallingConv::AMDGPU_PS, assert(), E(), llvm::ISD::InputArg::Flags, llvm::ISD::InputArg::getOrigArgIndex(), llvm::MVT::getScalarSizeInBits(), I, llvm::ISD::ArgFlagsTy::isInReg(), llvm::ISD::ArgFlagsTy::isSplit(), llvm::ISD::ArgFlagsTy::isSplitEnd(), llvm::MVT::isVector(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::ArrayRef< T >::size(), llvm::ISD::InputArg::Used, and llvm::ISD::InputArg::VT.

Referenced by llvm::SITargetLowering::LowerFormalArguments().

◆ reservePrivateMemoryRegs()

void reservePrivateMemoryRegs ( const TargetMachine & TM,
MachineFunction & MF,
const SIRegisterInfo & TRI,
SIMachineFunctionInfo & Info )
static

Definition at line 3367 of file SIISelLowering.cpp.

References assert(), llvm::Function::getCallingConv(), llvm::MachineFunction::getFrameInfo(), llvm::MachineFunction::getFunction(), llvm::TargetMachine::getOptLevel(), llvm::MachineFunction::getRegInfo(), llvm::MachineFunction::getSubtarget(), llvm::MachineFrameInfo::hasCalls(), llvm::MachineFrameInfo::hasStackObjects(), llvm::MachineRegisterInfo::isLiveIn(), llvm::AMDGPU::isShader(), llvm::None, llvm::AMDGPUFunctionArgInfo::PRIVATE_SEGMENT_BUFFER, Reg, llvm::report_fatal_error(), and TRI.

Referenced by llvm::SITargetLowering::finalizeLowering().

◆ resolveSources()

SDValue resolveSources ( SelectionDAG & DAG,
SDLoc SL,
SmallVectorImpl< DotSrc > & Srcs,
bool IsSigned,
bool IsAny )
static

Definition at line 17811 of file SIISelLowering.cpp.

References addPermMasks(), assert(), llvm::SmallVectorTemplateCommon< T, typename >::begin(), llvm::SmallVectorTemplateCommon< T, typename >::end(), llvm::SelectionDAG::getConstant(), getDWordFromOffset(), llvm::SelectionDAG::getNode(), llvm::ISD::OR, llvm::SmallVectorTemplateBase< T, bool >::push_back(), and llvm::SmallVectorTemplateCommon< T, typename >::size().

◆ selectSOffset()

SDValue selectSOffset ( SDValue SOffset,
SelectionDAG & DAG,
const GCNSubtarget * Subtarget )
static

Definition at line 11822 of file SIISelLowering.cpp.

References llvm::SelectionDAG::getRegister(), and llvm::isNullConstant().

◆ setM0ToIndexFromSGPR()

void setM0ToIndexFromSGPR ( const SIInstrInfo * TII,
MachineRegisterInfo & MRI,
MachineInstr & MI,
int Offset )
static

Definition at line 5463 of file SIISelLowering.cpp.

References llvm::MachineInstrBuilder::add(), llvm::MachineInstrBuilder::addImm(), assert(), llvm::BuildMI(), DL, I, MBB, MI, llvm::Offset, llvm::MachineInstrBuilder::setOperandDead(), and TII.

Referenced by emitIndirectDst(), and emitIndirectSrc().

◆ splitBlockForLoop()

std::pair< MachineBasicBlock *, MachineBasicBlock * > splitBlockForLoop ( MachineInstr & MI,
MachineBasicBlock & MBB,
bool InstInLoop )
static
Returns
{ LoopBody, Remainder }

Definition at line 5200 of file SIISelLowering.cpp.

References llvm::MachineBasicBlock::addSuccessor(), llvm::MachineBasicBlock::begin(), llvm::MachineFunction::CreateMachineBasicBlock(), I, llvm::MachineFunction::insert(), MBB, MBBI, MI, llvm::Next, llvm::MachineBasicBlock::splice(), and llvm::MachineBasicBlock::transferSuccessorsAndUpdatePHIs().

Referenced by llvm::SITargetLowering::emitGWSMemViolTestLoop(), loadM0FromVGPR(), and lowerWaveReduce().

◆ splitTFEValueAndStatus()

std::pair< SDValue, SDValue > splitTFEValueAndStatus ( SDValue Op,
EVT VT,
const SDLoc & DL,
SelectionDAG & DAG )
static

Definition at line 7935 of file SIISelLowering.cpp.

References llvm::ISD::BITCAST, C(), llvm::EVT::changeTypeToInteger(), llvm::divideCeil(), DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getVectorIdxConstant(), llvm::EVT::getVectorVT(), llvm::EVT::isVector(), and llvm::ISD::TRUNCATE.

◆ STATISTIC()

STATISTIC ( NumTailCalls ,
"Number of tail calls"  )

◆ strictFPExtFromF16()

SDValue strictFPExtFromF16 ( SelectionDAG & DAG,
SDValue Src )
static

Return the source of an fp_extend from f16 to f32, or a converted FP constant.

Definition at line 17372 of file SIISelLowering.cpp.

References llvm::APFloat::convert(), llvm::dyn_cast(), llvm::ISD::FP_EXTEND, llvm::SelectionDAG::getConstantFP(), llvm::APFloatBase::IEEEhalf(), and llvm::APFloatBase::rmNearestTiesToEven.

◆ SubIdx2Lane()

unsigned SubIdx2Lane ( unsigned Idx)
static

Helper function for adjustWritemask.

Definition at line 19537 of file SIISelLowering.cpp.

◆ supportsMin3Max3()

bool supportsMin3Max3 ( const GCNSubtarget & Subtarget,
unsigned Opc,
EVT VT )
static
Returns
true if the subtarget supports minimum3 and maximum3 with the given base min/max opcode Opc for type VT.

Definition at line 16890 of file SIISelLowering.cpp.

References llvm::ISD::FMAXIMUM, llvm::ISD::FMAXIMUMNUM, llvm::ISD::FMAXNUM, llvm::ISD::FMAXNUM_IEEE, llvm::ISD::FMINIMUM, llvm::ISD::FMINIMUMNUM, llvm::ISD::FMINNUM, llvm::ISD::FMINNUM_IEEE, llvm::GCNSubtarget::hasMin3Max3_16(), llvm_unreachable, Opc, llvm::ISD::SMAX, llvm::ISD::SMIN, llvm::ISD::UMAX, and llvm::ISD::UMIN.

◆ tryFoldMADwithSRL()

SDValue tryFoldMADwithSRL ( SelectionDAG & DAG,
const SDLoc & SL,
SDValue MulLHS,
SDValue MulRHS,
SDValue AddRHS )
static

Definition at line 17507 of file SIISelLowering.cpp.

References llvm::dyn_cast(), llvm::SDNode::getAsZExtVal(), llvm::SelectionDAG::getConstant(), getMad64_32(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getZeroExtendInReg(), llvm::Hi_32(), llvm::Lo_32(), llvm::ISD::SRL, std::swap(), and llvm::ISD::TRUNCATE.

◆ tryReduceF64CompareToHiHalf()

ISD::CondCode tryReduceF64CompareToHiHalf ( const ISD::CondCode CC,
const SDValue LHS,
const SDValue RHS,
const SelectionDAG & DAG )
static

Definition at line 18660 of file SIISelLowering.cpp.

References assert(), llvm::KnownFPClass::bitcast(), llvm::SelectionDAG::computeKnownBits(), llvm::KnownBits::eq(), llvm::EVT::getFltSemantics(), llvm::KnownFPClass::isKnownNeverNaN(), llvm::APInt::isSignBitSet(), LHS, RHS, llvm::ISD::SETCC_INVALID, llvm::ISD::SETEQ, llvm::ISD::SETFALSE, llvm::ISD::SETGE, llvm::ISD::SETGT, llvm::ISD::SETLE, llvm::ISD::SETLT, llvm::ISD::SETNE, llvm::ISD::SETOEQ, llvm::ISD::SETOGE, llvm::ISD::SETOGT, llvm::ISD::SETOLE, llvm::ISD::SETOLT, llvm::ISD::SETONE, llvm::ISD::SETTRUE, llvm::ISD::SETUEQ, llvm::ISD::SETUGE, llvm::ISD::SETUGT, llvm::ISD::SETULE, llvm::ISD::SETULT, llvm::ISD::SETUNE, llvm::KnownBits::trunc(), llvm::KnownBits::uge(), llvm::KnownBits::ule(), and llvm::KnownBits::Zero.

◆ vectorEltWillFoldAway()

bool vectorEltWillFoldAway ( SDValue Op)
static

Definition at line 16659 of file SIISelLowering.cpp.

References llvm::isa().

Variable Documentation

◆ DisableLoopAlignment

cl::opt< bool > DisableLoopAlignment("amdgpu-disable-loop-alignment", cl::desc("Do not align and prefetch loops"), cl::init(false)) ( "amdgpu-disable-loop-alignment" ,
cl::desc("Do not align and prefetch loops") ,
cl::init(false)  )
static

Referenced by llvm::SITargetLowering::getPrefLoopAlignment().

◆ UseDivergentRegisterIndexing

cl::opt< bool > UseDivergentRegisterIndexing("amdgpu-use-divergent-register-indexing", cl::Hidden, cl::desc("Use indirect register addressing for divergent indexes"), cl::init(false)) ( "amdgpu-use-divergent-register-indexing" ,
cl::Hidden ,
cl::desc("Use indirect register addressing for divergent indexes") ,
cl::init(false)  )
static

Referenced by llvm::SITargetLowering::shouldExpandVectorDynExt().

Generated on for LLVM by doxygen 1.14.0