SIISelLowering.cpp File Reference

Custom DAG lowering for SI. More...

#include "SIISelLowering.h"
#include "AMDGPU.h"
#include "AMDGPUInstrInfo.h"
#include "AMDGPULaneMaskUtils.h"
#include "AMDGPUSelectionDAGInfo.h"
#include "AMDGPUTargetMachine.h"
#include "GCNSubtarget.h"
#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
#include "SIMachineFunctionInfo.h"
#include "SIRegisterInfo.h"
#include "llvm/ADT/APInt.h"
#include "llvm/ADT/FloatingPointMode.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/Analysis/OptimizationRemarkEmitter.h"
#include "llvm/Analysis/UniformityAnalysis.h"
#include "llvm/CodeGen/Analysis.h"
#include "llvm/CodeGen/ByteProvider.h"
#include "llvm/CodeGen/FunctionLoweringInfo.h"
#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h"
#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineLoopInfo.h"
#include "llvm/CodeGen/PseudoSourceValueManager.h"
#include "llvm/CodeGen/SDPatternMatch.h"
#include "llvm/IR/DiagnosticInfo.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/IntrinsicsAMDGPU.h"
#include "llvm/IR/IntrinsicsR600.h"
#include "llvm/IR/MDBuilder.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/KnownBits.h"
#include "llvm/Support/ModRef.h"
#include "llvm/Transforms/Utils/LowerAtomic.h"
#include <optional>

Go to the source code of this file.

Classes
struct	DotSrc

Macros
#define	DEBUG_TYPE   "si-lower"

Functions
	STATISTIC (NumTailCalls, "Number of tail calls")
static bool	denormalModeIsFlushAllF32 (const MachineFunction &MF)
static bool	denormalModeIsFlushAllF64F16 (const MachineFunction &MF)
static unsigned	findFirstFreeSGPR (CCState &CCInfo)
static EVT	memVTFromLoadIntrData (const SITargetLowering &TLI, const DataLayout &DL, Type *Ty, unsigned MaxNumLanes)
static EVT	memVTFromLoadIntrReturn (const SITargetLowering &TLI, const DataLayout &DL, Type *Ty, unsigned MaxNumLanes)
static unsigned	getIntrMemWidth (unsigned IntrID)
static AtomicOrdering	parseAtomicOrderingCABIArg (const CallBase &CI, unsigned ArgIdx)
static unsigned	parseSyncscopeMDArg (const CallBase &CI, unsigned ArgIdx)
static void	processPSInputArgs (SmallVectorImpl< ISD::InputArg > &Splits, CallingConv::ID CallConv, ArrayRef< ISD::InputArg > Ins, BitVector &Skipped, FunctionType *FType, SIMachineFunctionInfo *Info)
static ArgDescriptor	allocateVGPR32Input (CCState &CCInfo, unsigned Mask=~0u, ArgDescriptor Arg=ArgDescriptor())
static ArgDescriptor	allocateSGPR32InputImpl (CCState &CCInfo, const TargetRegisterClass *RC, unsigned NumArgRegs)
static void	allocateFixedSGPRInputImpl (CCState &CCInfo, const TargetRegisterClass *RC, MCRegister Reg)
static void	allocateSGPR32Input (CCState &CCInfo, ArgDescriptor &Arg)
static void	allocateSGPR64Input (CCState &CCInfo, ArgDescriptor &Arg)
static void	reservePrivateMemoryRegs (const TargetMachine &TM, MachineFunction &MF, const SIRegisterInfo &TRI, SIMachineFunctionInfo &Info)
static std::pair< MachineBasicBlock *, MachineBasicBlock * >	splitBlockForLoop (MachineInstr &MI, MachineBasicBlock &MBB, bool InstInLoop)
static MachineBasicBlock::iterator	emitLoadM0FromVGPRLoop (const SIInstrInfo *TII, MachineRegisterInfo &MRI, MachineBasicBlock &OrigBB, MachineBasicBlock &LoopBB, const DebugLoc &DL, const MachineOperand &Idx, unsigned InitReg, unsigned ResultReg, unsigned PhiReg, unsigned InitSaveExecReg, int Offset, bool UseGPRIdxMode, Register &SGPRIdxReg)
static MachineBasicBlock::iterator	loadM0FromVGPR (const SIInstrInfo *TII, MachineBasicBlock &MBB, MachineInstr &MI, unsigned InitResultReg, unsigned PhiReg, int Offset, bool UseGPRIdxMode, Register &SGPRIdxReg)
static std::pair< unsigned, int >	computeIndirectRegAndOffset (const SIRegisterInfo &TRI, const TargetRegisterClass *SuperRC, unsigned VecReg, int Offset)
static void	setM0ToIndexFromSGPR (const SIInstrInfo *TII, MachineRegisterInfo &MRI, MachineInstr &MI, int Offset)
static Register	getIndirectSGPRIdx (const SIInstrInfo *TII, MachineRegisterInfo &MRI, MachineInstr &MI, int Offset)
static MachineBasicBlock *	emitIndirectSrc (MachineInstr &MI, MachineBasicBlock &MBB, const GCNSubtarget &ST)
static MachineBasicBlock *	emitIndirectDst (MachineInstr &MI, MachineBasicBlock &MBB, const GCNSubtarget &ST)
static MachineBasicBlock *	expand64BitScalarArithmetic (MachineInstr &MI, MachineBasicBlock *BB)
static void	expand64BitV_CNDMASK (MachineInstr &MI, MachineBasicBlock *BB)
static uint64_t	getIdentityValueForWaveReduction (unsigned Opc)
static bool	is32bitWaveReduceOperation (unsigned Opc)
static bool	isFloatingPointWaveReduceOperation (unsigned Opc)
static std::tuple< unsigned, unsigned >	getDPPOpcForWaveReduction (unsigned Opc, const GCNSubtarget &ST)
static std::pair< Register, Register >	ExtractSubRegs (MachineInstr &MI, MachineOperand &Op, const TargetRegisterClass *SrcRC, const GCNSubtarget &ST, MachineRegisterInfo &MRI)
static MachineBasicBlock *	lowerWaveReduce (MachineInstr &MI, MachineBasicBlock &BB, const GCNSubtarget &ST, unsigned Opc)
static SDValue	adjustLoadValueTypeImpl (SDValue Result, EVT LoadVT, const SDLoc &DL, SelectionDAG &DAG, bool Unpacked)
static SDValue	lowerICMPIntrinsic (const SITargetLowering &TLI, SDNode *N, SelectionDAG &DAG)
static SDValue	lowerFCMPIntrinsic (const SITargetLowering &TLI, SDNode *N, SelectionDAG &DAG)
static SDValue	lowerBALLOTIntrinsic (const SITargetLowering &TLI, SDNode *N, SelectionDAG &DAG)
static SDValue	emitRemovedIntrinsicError (SelectionDAG &DAG, const SDLoc &DL, EVT VT)
static SDValue	lowerLaneOp (const SITargetLowering &TLI, SDNode *N, SelectionDAG &DAG)
static SDValue	lowerWaveShuffle (const SITargetLowering &TLI, SDNode *N, SelectionDAG &DAG)
static SDNode *	findUser (SDValue Value, unsigned Opcode)
	Helper function for LowerBRCOND.
static unsigned	getExtOpcodeForPromotedOp (SDValue Op)
static bool	isKnownNonNull (SDValue Val, SelectionDAG &DAG, const AMDGPUTargetMachine &TM, unsigned AddrSpace)
	Return true if the value is a known valid address, such that a null check is not necessary.
static bool	elementPairIsContiguous (ArrayRef< int > Mask, int Elt)
static bool	elementPairIsOddToEven (ArrayRef< int > Mask, int Elt)
static SDValue	buildPCRelGlobalAddress (SelectionDAG &DAG, const GlobalValue *GV, const SDLoc &DL, int64_t Offset, EVT PtrVT, unsigned GAFlags=SIInstrInfo::MO_NONE)
static SDValue	emitNonHSAIntrinsicError (SelectionDAG &DAG, const SDLoc &DL, EVT VT)
static SDValue	getBuildDwordsVector (SelectionDAG &DAG, SDLoc DL, ArrayRef< SDValue > Elts)
static SDValue	padEltsToUndef (SelectionDAG &DAG, const SDLoc &DL, EVT CastVT, SDValue Src, int ExtraElts)
static SDValue	constructRetValue (SelectionDAG &DAG, MachineSDNode *Result, ArrayRef< EVT > ResultTypes, bool IsTexFail, bool Unpacked, bool IsD16, int DMaskPop, int NumVDataDwords, bool IsAtomicPacked16Bit, const SDLoc &DL)
static bool	parseTexFail (SDValue TexFailCtrl, SelectionDAG &DAG, SDValue *TFE, SDValue *LWE, bool &IsTexFail)
static void	packImage16bitOpsToDwords (SelectionDAG &DAG, SDValue Op, MVT PackVectorVT, SmallVectorImpl< SDValue > &PackedAddrs, unsigned DimIdx, unsigned EndIdx, unsigned NumGradients)
static SDValue	selectSOffset (SDValue SOffset, SelectionDAG &DAG, const GCNSubtarget *Subtarget)
static bool	isAsyncLDSDMA (Intrinsic::ID Intr)
static bool	isNoUnsignedWrap (SDValue Addr)
static SDValue	getLoadExtOrTrunc (SelectionDAG &DAG, ISD::LoadExtType ExtType, SDValue Op, const SDLoc &SL, EVT VT)
static bool	addressMayBeAccessedAsPrivate (const MachineMemOperand *MMO, const SIMachineFunctionInfo &Info)
static SDValue	getFPBinOp (SelectionDAG &DAG, unsigned Opcode, const SDLoc &SL, EVT VT, SDValue A, SDValue B, SDValue GlueChain, SDNodeFlags Flags)
static SDValue	getFPTernOp (SelectionDAG &DAG, unsigned Opcode, const SDLoc &SL, EVT VT, SDValue A, SDValue B, SDValue C, SDValue GlueChain, SDNodeFlags Flags)
static SDValue	getSPDenormModeValue (uint32_t SPDenormMode, SelectionDAG &DAG, const SIMachineFunctionInfo *Info, const GCNSubtarget *ST)
static unsigned	getBasePtrIndex (const MemSDNode *N)
	MemSDNode::getBasePtr() does not work for intrinsics, which needs to offset by the chain and intrinsic ID.
static bool	bitOpWithConstantIsReducible (unsigned Opc, uint32_t Val)
static uint32_t	getConstantPermuteMask (uint32_t C)
static uint32_t	getPermuteMask (SDValue V)
static const std::optional< ByteProvider< SDValue > >	calculateSrcByte (const SDValue Op, uint64_t DestByte, uint64_t SrcIndex=0, unsigned Depth=0)
static const std::optional< ByteProvider< SDValue > >	calculateByteProvider (const SDValue &Op, unsigned Index, unsigned Depth, unsigned StartingIndex=0)
static bool	isExtendedFrom16Bits (SDValue &Operand)
static bool	addresses16Bits (int Mask)
static bool	hasNon16BitAccesses (uint64_t PermMask, SDValue &Op, SDValue &OtherOp)
static SDValue	getDWordFromOffset (SelectionDAG &DAG, SDLoc SL, SDValue Src, unsigned DWordOffset)
static SDValue	matchPERM (SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static bool	vectorEltWillFoldAway (SDValue Op)
static unsigned	minMaxOpcToMin3Max3Opc (unsigned Opc)
static ConstantFPSDNode *	getSplatConstantFP (SDValue Op)
static bool	supportsMin3Max3 (const GCNSubtarget &Subtarget, unsigned Opc, EVT VT)
static bool	isClampZeroToOne (SDValue A, SDValue B)
static SDValue	strictFPExtFromF16 (SelectionDAG &DAG, SDValue Src)
	Return the source of an fp_extend from f16 to f32, or a converted FP constant.
static SDValue	getMad64_32 (SelectionDAG &DAG, const SDLoc &SL, EVT VT, SDValue N0, SDValue N1, SDValue N2, bool Signed)
static SDValue	tryFoldMADwithSRL (SelectionDAG &DAG, const SDLoc &SL, SDValue MulLHS, SDValue MulRHS, SDValue AddRHS)
static std::optional< ByteProvider< SDValue > >	handleMulOperand (const SDValue &MulOperand)
static unsigned	addPermMasks (unsigned First, unsigned Second)
static void	placeSources (ByteProvider< SDValue > &Src0, ByteProvider< SDValue > &Src1, SmallVectorImpl< DotSrc > &Src0s, SmallVectorImpl< DotSrc > &Src1s, int Step)
static SDValue	resolveSources (SelectionDAG &DAG, SDLoc SL, SmallVectorImpl< DotSrc > &Srcs, bool IsSigned, bool IsAny)
static void	fixMasks (SmallVectorImpl< DotSrc > &Srcs, unsigned ChainLength)
static bool	isMul (const SDValue Op)
static std::optional< bool >	checkDot4MulSignedness (const SDValue &N, ByteProvider< SDValue > &Src0, ByteProvider< SDValue > &Src1, const SDValue &S0Op, const SDValue &S1Op, const SelectionDAG &DAG)
static bool	isCtlzOpc (unsigned Opc)
static ISD::CondCode	tryReduceF64CompareToHiHalf (const ISD::CondCode CC, const SDValue LHS, const SDValue RHS, const SelectionDAG &DAG)
static unsigned	SubIdx2Lane (unsigned Idx)
	Helper function for adjustWritemask.
static bool	isFrameIndexOp (SDValue Op)
static SDValue	buildSMovImm32 (SelectionDAG &DAG, const SDLoc &DL, uint64_t Val)
static bool	isImmConstraint (StringRef Constraint)
static uint64_t	clearUnusedBits (uint64_t Val, unsigned Size)
static int	getAlignedAGPRClassID (unsigned UnalignedClassID)
static void	knownBitsForWorkitemID (const GCNSubtarget &ST, GISelValueTracking &VT, KnownBits &Known, unsigned Dim)
static void	knownBitsForSBFE (const MachineInstr &MI, GISelValueTracking &VT, KnownBits &Known, const APInt &DemandedElts, unsigned BFEWidth, bool SExt, unsigned Depth)
static bool	isCopyFromRegOfInlineAsm (const SDNode *N)
static bool	atomicIgnoresDenormalModeOrFPModeIsFTZ (const AtomicRMWInst *RMW)
static OptimizationRemark	emitAtomicRMWLegalRemark (const AtomicRMWInst *RMW)
static bool	isV2F16OrV2BF16 (Type *Ty)
static bool	isV2F16 (Type *Ty)
static bool	isV2BF16 (Type *Ty)
static bool	isAtomicRMWLegalIntTy (Type *Ty)
static bool	isAtomicRMWLegalXChgTy (const AtomicRMWInst *RMW)
static bool	globalMemoryFPAtomicIsLegal (const GCNSubtarget &Subtarget, const AtomicRMWInst *RMW, bool HasSystemScope)
static TargetLowering::AtomicExpansionKind	atomicSupportedIfLegalIntType (const AtomicRMWInst *RMW)
static bool	flatInstrMayAccessPrivate (const Instruction *I)
	Return if a flat address space atomicrmw can access private memory.
static TargetLowering::AtomicExpansionKind	getPrivateAtomicExpansionKind (const GCNSubtarget &STI)
static bool	hasCFUser (const Value *V, SmallPtrSet< const Value *, 16 > &Visited, unsigned WaveSize)
static void	convertScratchAtomicToFlatAtomic (Instruction *I, unsigned PtrOpIdx)

Variables
static cl::opt< bool >	DisableLoopAlignment ("amdgpu-disable-loop-alignment", cl::desc("Do not align and prefetch loops"), cl::init(false))
static cl::opt< bool >	UseDivergentRegisterIndexing ("amdgpu-use-divergent-register-indexing", cl::Hidden, cl::desc("Use indirect register addressing for divergent indexes"), cl::init(false))

Detailed Description

Custom DAG lowering for SI.

Definition in file SIISelLowering.cpp.

Macro Definition Documentation

DEBUG_TYPE

#define DEBUG_TYPE   "si-lower"

Definition at line 55 of file SIISelLowering.cpp.

Function Documentation

addPermMasks()

unsigned addPermMasks ( unsigned First, unsigned Second )

static

Definition at line 17084 of file SIISelLowering.cpp.

References assert(), and llvm::First.

Referenced by placeSources(), and resolveSources().

addresses16Bits()

bool addresses16Bits ( int Mask )

static

Definition at line 15068 of file SIISelLowering.cpp.

References assert().

Referenced by hasNon16BitAccesses().

addressMayBeAccessedAsPrivate()

bool addressMayBeAccessedAsPrivate ( const MachineMemOperand * MMO, const SIMachineFunctionInfo & Info )

static

Definition at line 13073 of file SIISelLowering.cpp.

adjustLoadValueTypeImpl()

SDValue adjustLoadValueTypeImpl ( SDValue Result, EVT LoadVT, const SDLoc & DL, SelectionDAG & DAG, bool Unpacked )

static

Definition at line 7704 of file SIISelLowering.cpp.

References llvm::ISD::BITCAST, llvm::EVT::changeTypeToInteger(), DL, llvm::SelectionDAG::ExtractVectorElements(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getPOISON(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::EVT::isVector(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), and llvm::ISD::TRUNCATE.

Referenced by constructRetValue().

allocateFixedSGPRInputImpl()

void allocateFixedSGPRInputImpl ( CCState & CCInfo, const TargetRegisterClass * RC, MCRegister Reg )

static

Definition at line 2946 of file SIISelLowering.cpp.

References llvm::MachineFunction::addLiveIn(), llvm::CCState::AllocateReg(), assert(), llvm::CCState::getMachineFunction(), and Reg.

Referenced by allocateSGPR32Input(), and allocateSGPR64Input().

allocateSGPR32Input()

void allocateSGPR32Input ( CCState & CCInfo, ArgDescriptor & Arg )

static

Definition at line 2955 of file SIISelLowering.cpp.

References allocateFixedSGPRInputImpl(), allocateSGPR32InputImpl(), and llvm::ArgDescriptor::getRegister().

Referenced by llvm::SITargetLowering::allocateSpecialInputSGPRs().

allocateSGPR32InputImpl()

ArgDescriptor allocateSGPR32InputImpl ( CCState & CCInfo, const TargetRegisterClass * RC, unsigned NumArgRegs )

static

Definition at line 2926 of file SIISelLowering.cpp.

References llvm::MachineFunction::addLiveIn(), llvm::CCState::AllocateReg(), llvm::ArrayRef(), assert(), llvm::TargetRegisterClass::begin(), llvm::ArgDescriptor::createRegister(), llvm::CCState::getFirstUnallocated(), llvm::CCState::getMachineFunction(), Reg, llvm::report_fatal_error(), and llvm::ArrayRef< T >::size().

Referenced by allocateSGPR32Input(), and allocateSGPR64Input().

allocateSGPR64Input()

void allocateSGPR64Input ( CCState & CCInfo, ArgDescriptor & Arg )

static

Definition at line 2963 of file SIISelLowering.cpp.

References allocateFixedSGPRInputImpl(), allocateSGPR32InputImpl(), and llvm::ArgDescriptor::getRegister().

Referenced by llvm::SITargetLowering::allocateSpecialInputSGPRs().

allocateVGPR32Input()

ArgDescriptor allocateVGPR32Input ( CCState & CCInfo, unsigned Mask = ~0u, ArgDescriptor Arg = ArgDescriptor() )

static

Definition at line 2902 of file SIISelLowering.cpp.

References llvm::MachineFunction::addLiveIn(), llvm::CCState::AllocateReg(), llvm::CCState::AllocateStack(), llvm::ArrayRef(), assert(), llvm::ArgDescriptor::createArg(), llvm::ArgDescriptor::createRegister(), llvm::ArgDescriptor::createStack(), llvm::CCState::getFirstUnallocated(), llvm::CCState::getMachineFunction(), llvm::MachineFunction::getRegInfo(), llvm::Offset, Reg, llvm::LLT::scalar(), llvm::MachineRegisterInfo::setType(), and llvm::ArrayRef< T >::size().

Referenced by llvm::SITargetLowering::allocateSpecialInputVGPRs().

atomicIgnoresDenormalModeOrFPModeIsFTZ()

bool atomicIgnoresDenormalModeOrFPModeIsFTZ ( const AtomicRMWInst * RMW )

static

Definition at line 20085 of file SIISelLowering.cpp.

References llvm::Function::getDenormalMode(), llvm::Type::getFltSemantics(), llvm::Function::getFnAttribute(), llvm::Instruction::getFunction(), llvm::DenormalMode::getPreserveSign(), llvm::Type::getScalarType(), llvm::Value::getType(), llvm::Attribute::getValueAsBool(), and llvm::Instruction::hasMetadata().

Referenced by llvm::SITargetLowering::shouldExpandAtomicRMWInIR().

atomicSupportedIfLegalIntType()

TargetLowering::AtomicExpansionKind atomicSupportedIfLegalIntType ( const AtomicRMWInst * RMW )

static

Returns

Action to perform on AtomicRMWInsts for integer operations.

Definition at line 20189 of file SIISelLowering.cpp.

References llvm::TargetLoweringBase::CmpXChg, llvm::Value::getType(), isAtomicRMWLegalIntTy(), and llvm::TargetLoweringBase::None.

Referenced by llvm::SITargetLowering::shouldExpandAtomicRMWInIR().

bitOpWithConstantIsReducible()

bool bitOpWithConstantIsReducible ( unsigned Opc, uint32_t Val )

static

Definition at line 14344 of file SIISelLowering.cpp.

References llvm::ISD::AND, Opc, llvm::ISD::OR, and llvm::ISD::XOR.

buildPCRelGlobalAddress()

SDValue buildPCRelGlobalAddress ( SelectionDAG & DAG, const GlobalValue * GV, const SDLoc & DL, int64_t Offset, EVT PtrVT, unsigned GAFlags = SIInstrInfo::MO_NONE )

static

Definition at line 9857 of file SIISelLowering.cpp.

References assert(), DL, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getSubtarget(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getTargetGlobalAddress(), llvm::isInt(), llvm::SIInstrInfo::MO_NONE, and llvm::Offset.

buildSMovImm32()

SDValue buildSMovImm32 ( SelectionDAG & DAG, const SDLoc & DL, uint64_t Val )

static

Definition at line 19167 of file SIISelLowering.cpp.

References DL, llvm::SelectionDAG::getMachineNode(), llvm::SelectionDAG::getTargetConstant(), and SDValue().

Referenced by llvm::SITargetLowering::buildRSRC(), and llvm::SITargetLowering::wrapAddr64Rsrc().

calculateByteProvider()

const std::optional< ByteProvider< SDValue > > calculateByteProvider ( const SDValue & Op, unsigned Index, unsigned Depth, unsigned StartingIndex = 0 )

static

Definition at line 14779 of file SIISelLowering.cpp.

References llvm::ISD::AND, llvm::ISD::ANY_EXTEND, llvm::ISD::AssertSext, llvm::ISD::AssertZext, llvm::BitWidth, llvm::ISD::BSWAP, calculateByteProvider(), calculateSrcByte(), llvm::cast(), llvm::ISD::CopyFromReg, llvm::Depth, llvm::dyn_cast(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::ISD::FSHR, llvm::ByteProvider< ISelOp >::getConstantZero(), llvm::SDValue::getValueSizeInBits(), LHS, llvm_unreachable, llvm::ISD::LOAD, llvm::ISD::OR, RHS, llvm::ISD::SHL, llvm::ISD::SIGN_EXTEND, llvm::ISD::SIGN_EXTEND_INREG, llvm::ISD::SRA, llvm::ISD::SRL, llvm::ISD::TRUNCATE, llvm::ISD::ZERO_EXTEND, and llvm::ISD::ZEXTLOAD.

calculateSrcByte()

const std::optional< ByteProvider< SDValue > > calculateSrcByte ( const SDValue Op, uint64_t DestByte, uint64_t SrcIndex = 0, unsigned Depth = 0 )

static

Definition at line 14710 of file SIISelLowering.cpp.

References llvm::ISD::ANY_EXTEND, calculateSrcByte(), llvm::cast(), llvm::Depth, llvm::dyn_cast(), llvm::ByteProvider< ISelOp >::getSrc(), llvm::EVT::getStoreSize(), llvm::SDValue::getValueType(), llvm::EVT::isByteSized(), llvm_unreachable, llvm::ISD::SIGN_EXTEND, llvm::ISD::SIGN_EXTEND_INREG, llvm::ISD::SRA, llvm::ISD::SRL, llvm::ISD::TRUNCATE, and llvm::ISD::ZERO_EXTEND.

Referenced by calculateByteProvider(), and calculateSrcByte().

checkDot4MulSignedness()

std::optional< bool > checkDot4MulSignedness ( const SDValue & N, ByteProvider< SDValue > & Src0, ByteProvider< SDValue > & Src1, const SDValue & S0Op, const SDValue & S1Op, const SelectionDAG & DAG )

static

Definition at line 17264 of file SIISelLowering.cpp.

References assert(), llvm::SelectionDAG::computeKnownBits(), llvm::KnownBits::countMinLeadingZeros(), llvm::SDValue::getValueSizeInBits(), llvm_unreachable, and N.

clearUnusedBits()

uint64_t clearUnusedBits ( uint64_t Val, unsigned Size )

static

Definition at line 19423 of file SIISelLowering.cpp.

References llvm::AMDGPU::isInlinableIntLiteral(), llvm::maskTrailingOnes(), and Size.

Referenced by llvm::APInt::APInt(), llvm::APInt::ashrInPlace(), llvm::SITargetLowering::checkAsmConstraintVal(), llvm::APInt::flipAllBits(), llvm::SITargetLowering::LowerAsmOperandForConstraint(), llvm::APInt::operator<<=(), llvm::APInt::operator=(), llvm::APInt::operator^=(), llvm::APInt::operator|=(), and llvm::APInt::setAllBits().

computeIndirectRegAndOffset()

std::pair< unsigned, int > computeIndirectRegAndOffset ( const SIRegisterInfo & TRI, const TargetRegisterClass * SuperRC, unsigned VecReg, int Offset )

static

Definition at line 5351 of file SIISelLowering.cpp.

References llvm::SIRegisterInfo::getSubRegFromChannel(), llvm::Offset, and TRI.

Referenced by emitIndirectDst(), and emitIndirectSrc().

constructRetValue()

SDValue constructRetValue ( SelectionDAG & DAG, MachineSDNode * Result, ArrayRef< EVT > ResultTypes, bool IsTexFail, bool Unpacked, bool IsD16, int DMaskPop, int NumVDataDwords, bool IsAtomicPacked16Bit, const SDLoc & DL )

static

Definition at line 10082 of file SIISelLowering.cpp.

References adjustLoadValueTypeImpl(), llvm::ISD::BITCAST, llvm::EVT::changeTypeToInteger(), llvm::Data, DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::MVT::getVectorVT(), llvm::EVT::isVector(), llvm::MVT::isVector(), padEltsToUndef(), SDValue(), and llvm::ISD::TRUNCATE.

convertScratchAtomicToFlatAtomic()

void convertScratchAtomicToFlatAtomic ( Instruction * I, unsigned PtrOpIdx )

static

Definition at line 20861 of file SIISelLowering.cpp.

References assert(), llvm::CastInst::CreatePointerCast(), llvm::AMDGPUAS::FLAT_ADDRESS, llvm::PointerType::get(), llvm::Type::getPointerAddressSpace(), llvm::Value::getType(), I, and llvm::AMDGPUAS::PRIVATE_ADDRESS.

Referenced by llvm::SITargetLowering::emitExpandAtomicCmpXchg(), llvm::SITargetLowering::emitExpandAtomicLoad(), llvm::SITargetLowering::emitExpandAtomicRMW(), and llvm::SITargetLowering::emitExpandAtomicStore().

denormalModeIsFlushAllF32()

bool denormalModeIsFlushAllF32 ( const MachineFunction & MF )

static

Definition at line 69 of file SIISelLowering.cpp.

References llvm::SIModeRegisterDefaults::FP32Denormals, llvm::MachineFunction::getInfo(), llvm::SIMachineFunctionInfo::getMode(), and llvm::DenormalMode::getPreserveSign().

Referenced by llvm::SITargetLowering::denormalsEnabledForType(), llvm::SITargetLowering::denormalsEnabledForType(), llvm::SITargetLowering::isFMADLegal(), llvm::SITargetLowering::isFMADLegal(), llvm::SITargetLowering::isFMAFasterThanFMulAndFAdd(), llvm::SITargetLowering::isFPExtFoldable(), and llvm::SITargetLowering::isFPExtFoldable().

denormalModeIsFlushAllF64F16()

bool denormalModeIsFlushAllF64F16 ( const MachineFunction & MF )

static

Definition at line 74 of file SIISelLowering.cpp.

References llvm::SIModeRegisterDefaults::FP64FP16Denormals, llvm::MachineFunction::getInfo(), llvm::SIMachineFunctionInfo::getMode(), and llvm::DenormalMode::getPreserveSign().

Referenced by llvm::SITargetLowering::denormalsEnabledForType(), llvm::SITargetLowering::denormalsEnabledForType(), llvm::SITargetLowering::isFMADLegal(), llvm::SITargetLowering::isFMADLegal(), and llvm::SITargetLowering::isFMAFasterThanFMulAndFAdd().

elementPairIsContiguous()

bool elementPairIsContiguous ( ArrayRef< int > Mask, int Elt )

static

Definition at line 9639 of file SIISelLowering.cpp.

References assert().

elementPairIsOddToEven()

bool elementPairIsOddToEven ( ArrayRef< int > Mask, int Elt )

static

Definition at line 9644 of file SIISelLowering.cpp.

References assert().

emitAtomicRMWLegalRemark()

OptimizationRemark emitAtomicRMWLegalRemark ( const AtomicRMWInst * RMW )

static

Definition at line 20100 of file SIISelLowering.cpp.

References DEBUG_TYPE, llvm::Value::getContext(), llvm::AtomicRMWInst::getOperation(), llvm::AtomicRMWInst::getOperationName(), and llvm::AtomicRMWInst::getSyncScopeID().

Referenced by llvm::SITargetLowering::shouldExpandAtomicRMWInIR().

emitIndirectDst()

MachineBasicBlock * emitIndirectDst ( MachineInstr & MI, MachineBasicBlock & MBB, const GCNSubtarget & ST )

static

Definition at line 5492 of file SIISelLowering.cpp.

References llvm::MachineInstrBuilder::add(), llvm::MachineInstrBuilder::addImm(), llvm::MachineInstrBuilder::addReg(), assert(), llvm::BuildMI(), llvm::MachineRegisterInfo::clearKillFlags(), computeIndirectRegAndOffset(), llvm::MachineRegisterInfo::createVirtualRegister(), DL, getIndirectSGPRIdx(), llvm::MachineBasicBlock::getParent(), llvm::MachineOperand::getReg(), llvm::MachineRegisterInfo::getRegClass(), llvm::MachineFunction::getRegInfo(), I, llvm::MachineOperand::isReg(), loadM0FromVGPR(), MBB, MI, llvm::Offset, setM0ToIndexFromSGPR(), TII, and TRI.

Referenced by llvm::SITargetLowering::EmitInstrWithCustomInserter().

emitIndirectSrc()

MachineBasicBlock * emitIndirectSrc ( MachineInstr & MI, MachineBasicBlock & MBB, const GCNSubtarget & ST )

static

Definition at line 5406 of file SIISelLowering.cpp.

References llvm::MachineInstrBuilder::addImm(), llvm::MachineInstrBuilder::addReg(), llvm::BuildMI(), computeIndirectRegAndOffset(), llvm::MachineRegisterInfo::createVirtualRegister(), DL, getIndirectSGPRIdx(), llvm::MachineBasicBlock::getParent(), llvm::MachineOperand::getReg(), llvm::MachineRegisterInfo::getRegClass(), llvm::MachineFunction::getRegInfo(), I, llvm::Implicit, loadM0FromVGPR(), MBB, MI, llvm::Offset, setM0ToIndexFromSGPR(), TII, and TRI.

Referenced by llvm::SITargetLowering::EmitInstrWithCustomInserter().

emitLoadM0FromVGPRLoop()

MachineBasicBlock::iterator emitLoadM0FromVGPRLoop ( const SIInstrInfo * TII, MachineRegisterInfo & MRI, MachineBasicBlock & OrigBB, MachineBasicBlock & LoopBB, const DebugLoc & DL, const MachineOperand & Idx, unsigned InitReg, unsigned ResultReg, unsigned PhiReg, unsigned InitSaveExecReg, int Offset, bool UseGPRIdxMode, Register & SGPRIdxReg )

static

Definition at line 5209 of file SIISelLowering.cpp.

References llvm::MachineInstrBuilder::addImm(), llvm::MachineInstrBuilder::addMBB(), llvm::MachineInstrBuilder::addReg(), llvm::AMDGPU::LaneMaskConstants::AndSaveExecOpc, llvm::MachineBasicBlock::begin(), llvm::BuildMI(), llvm::MachineRegisterInfo::createVirtualRegister(), DL, llvm::AMDGPU::LaneMaskConstants::ExecReg, llvm::AMDGPU::LaneMaskConstants::get(), llvm::ilist_node_impl< OptionsT >::getIterator(), llvm::MachineBasicBlock::getParent(), llvm::MachineOperand::getReg(), llvm::MachineOperand::getSubReg(), llvm::MachineFunction::getSubtarget(), llvm::getUndefRegState(), I, llvm::MachineOperand::isUndef(), llvm::Kill, llvm::Offset, llvm::MachineRegisterInfo::setSimpleHint(), TII, TRI, and llvm::AMDGPU::LaneMaskConstants::XorTermOpc.

Referenced by loadM0FromVGPR().

emitNonHSAIntrinsicError()

SDValue emitNonHSAIntrinsicError ( SelectionDAG & DAG, const SDLoc & DL, EVT VT )

static

Definition at line 10016 of file SIISelLowering.cpp.

References llvm::LLVMContext::diagnose(), DL, llvm::SelectionDAG::getContext(), llvm::MachineFunction::getFunction(), llvm::SelectionDAG::getMachineFunction(), and llvm::SelectionDAG::getPOISON().

emitRemovedIntrinsicError()

SDValue emitRemovedIntrinsicError ( SelectionDAG & DAG, const SDLoc & DL, EVT VT )

static

Definition at line 10024 of file SIISelLowering.cpp.

References llvm::LLVMContext::diagnose(), DL, llvm::SelectionDAG::getContext(), llvm::MachineFunction::getFunction(), llvm::SelectionDAG::getMachineFunction(), and llvm::SelectionDAG::getPOISON().

Referenced by lowerLaneOp().

expand64BitScalarArithmetic()

MachineBasicBlock * expand64BitScalarArithmetic ( MachineInstr & MI, MachineBasicBlock * BB )

static

Definition at line 5595 of file SIISelLowering.cpp.

References llvm::MachineInstrBuilder::add(), llvm::MachineInstrBuilder::addImm(), llvm::MachineInstrBuilder::addReg(), llvm::BuildMI(), DL, llvm::MachineBasicBlock::getParent(), llvm::MachineOperand::getReg(), llvm::MachineFunction::getRegInfo(), llvm::MachineFunction::getSubtarget(), MI, Opc, TII, and TRI.

Referenced by llvm::SITargetLowering::EmitInstrWithCustomInserter(), and lowerWaveReduce().

expand64BitV_CNDMASK()

void expand64BitV_CNDMASK ( MachineInstr & MI, MachineBasicBlock * BB )

static

Definition at line 5646 of file SIISelLowering.cpp.

References llvm::MachineInstrBuilder::add(), llvm::MachineInstrBuilder::addImm(), llvm::MachineInstrBuilder::addReg(), llvm::BuildMI(), llvm::MachineRegisterInfo::createVirtualRegister(), DL, llvm::MachineBasicBlock::getParent(), llvm::MachineFunction::getRegInfo(), llvm::MachineFunction::getSubtarget(), MI, TII, and TRI.

Referenced by llvm::SITargetLowering::EmitInstrWithCustomInserter(), and lowerWaveReduce().

ExtractSubRegs()

std::pair< Register, Register > ExtractSubRegs ( MachineInstr & MI, MachineOperand & Op, const TargetRegisterClass * SrcRC, const GCNSubtarget & ST, MachineRegisterInfo & MRI )

static

Definition at line 5859 of file SIISelLowering.cpp.

References MI, TII, and TRI.

Referenced by lowerWaveReduce().

findFirstFreeSGPR()

unsigned findFirstFreeSGPR ( CCState & CCInfo )

static

Definition at line 79 of file SIISelLowering.cpp.

References llvm::CCState::isAllocated(), llvm_unreachable, and Reg.

Referenced by llvm::SITargetLowering::allocateSystemSGPRs().

findUser()

SDNode * findUser ( SDValue Value, unsigned Opcode )

static

Helper function for LowerBRCOND.

Definition at line 8411 of file SIISelLowering.cpp.

References llvm::Value::uses().

fixMasks()

void fixMasks ( SmallVectorImpl< DotSrc > & Srcs, unsigned ChainLength )

static

Definition at line 17248 of file SIISelLowering.cpp.

flatInstrMayAccessPrivate()

bool flatInstrMayAccessPrivate ( const Instruction * I )

static

Return if a flat address space atomicrmw can access private memory.

Definition at line 20196 of file SIISelLowering.cpp.

References llvm::AMDGPU::hasValueInRangeLikeMetadata(), I, and llvm::AMDGPUAS::PRIVATE_ADDRESS.

Referenced by llvm::SITargetLowering::shouldExpandAtomicCmpXchgInIR(), and llvm::SITargetLowering::shouldExpandAtomicRMWInIR().

getAlignedAGPRClassID()

int getAlignedAGPRClassID ( unsigned UnalignedClassID )

static

Definition at line 19541 of file SIISelLowering.cpp.

Referenced by llvm::SITargetLowering::finalizeLowering().

getBasePtrIndex()

unsigned getBasePtrIndex ( const MemSDNode * N )

static

MemSDNode::getBasePtr() does not work for intrinsics, which needs to offset by the chain and intrinsic ID.

Theoretically we would also need to check the specific intrinsic, but they all place the pointer operand first.

Definition at line 14311 of file SIISelLowering.cpp.

References llvm::ISD::INTRINSIC_VOID, llvm::ISD::INTRINSIC_W_CHAIN, N, and llvm::ISD::STORE.

Referenced by llvm::SITargetLowering::hasMemSDNodeUser().

getBuildDwordsVector()

SDValue getBuildDwordsVector ( SelectionDAG & DAG, SDLoc DL, ArrayRef< SDValue > Elts )

static

Definition at line 10032 of file SIISelLowering.cpp.

References assert(), DL, llvm::ArrayRef< T >::empty(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getPOISON(), llvm::SDValue::getValueType(), llvm::MVT::getVectorVT(), and llvm::ArrayRef< T >::size().

getConstantPermuteMask()

uint32_t getConstantPermuteMask ( uint32_t C )

static

Definition at line 14419 of file SIISelLowering.cpp.

References llvm::CallingConv::C.

Referenced by getPermuteMask().

getDPPOpcForWaveReduction()

std::tuple< unsigned, unsigned > getDPPOpcForWaveReduction ( unsigned Opc, const GCNSubtarget & ST )

static

Definition at line 5781 of file SIISelLowering.cpp.

References llvm_unreachable, and Opc.

Referenced by lowerWaveReduce().

getDWordFromOffset()

SDValue getDWordFromOffset ( SelectionDAG & DAG, SDLoc SL, SDValue Src, unsigned DWordOffset )

static

Scalar Type

Definition at line 15109 of file SIISelLowering.cpp.

References assert(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::ExtractVectorElements(), llvm::SelectionDAG::getBitcastedAnyExtOrTrunc(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getFixedValue(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::SDValue::getValueType(), llvm::MVT::getVectorVT(), and llvm::ISD::SRL.

Referenced by matchPERM(), and resolveSources().

getExtOpcodeForPromotedOp()

unsigned getExtOpcodeForPromotedOp ( SDValue Op )

static

Definition at line 8796 of file SIISelLowering.cpp.

References llvm::ISD::ADD, llvm::ISD::AND, llvm::ISD::ANY_EXTEND, llvm::cast(), llvm::ISD::isSignedIntSetCC(), llvm_unreachable, llvm::ISD::MUL, llvm::ISD::OR, llvm::ISD::SELECT, llvm::ISD::SETCC, llvm::ISD::SHL, llvm::ISD::SIGN_EXTEND, llvm::ISD::SMAX, llvm::ISD::SMIN, llvm::ISD::SRA, llvm::ISD::SRL, llvm::ISD::SUB, llvm::ISD::UMAX, llvm::ISD::UMIN, llvm::ISD::XOR, and llvm::ISD::ZERO_EXTEND.

getFPBinOp()

SDValue getFPBinOp ( SelectionDAG & DAG, unsigned Opcode, const SDLoc & SL, EVT VT, SDValue A, SDValue B, SDValue GlueChain, SDNodeFlags Flags )

static

Definition at line 13351 of file SIISelLowering.cpp.

References A(), assert(), B(), llvm::ISD::FMUL, llvm::SelectionDAG::getNode(), llvm::SDNode::getNumValues(), llvm::SDValue::getValue(), llvm::SelectionDAG::getVTList(), and llvm_unreachable.

getFPTernOp()

SDValue getFPTernOp ( SelectionDAG & DAG, unsigned Opcode, const SDLoc & SL, EVT VT, SDValue A, SDValue B, SDValue C, SDValue GlueChain, SDNodeFlags Flags )

static

Definition at line 13374 of file SIISelLowering.cpp.

References A(), assert(), B(), llvm::CallingConv::C, llvm::ISD::FMA, llvm::SelectionDAG::getNode(), llvm::SDNode::getNumValues(), llvm::SDValue::getValue(), llvm::SelectionDAG::getVTList(), and llvm_unreachable.

getIdentityValueForWaveReduction()

uint64_t getIdentityValueForWaveReduction ( unsigned Opc )

static

Definition at line 5710 of file SIISelLowering.cpp.

References llvm_unreachable, and Opc.

Referenced by lowerWaveReduce().

getIndirectSGPRIdx()

Register getIndirectSGPRIdx ( const SIInstrInfo * TII, MachineRegisterInfo & MRI, MachineInstr & MI, int Offset )

static

Definition at line 5387 of file SIISelLowering.cpp.

References llvm::MachineInstrBuilder::add(), llvm::MachineInstrBuilder::addImm(), llvm::BuildMI(), llvm::MachineRegisterInfo::createVirtualRegister(), DL, llvm::MachineOperand::getReg(), I, MBB, MI, llvm::Offset, and TII.

Referenced by emitIndirectDst(), and emitIndirectSrc().

getIntrMemWidth()

unsigned getIntrMemWidth ( unsigned IntrID )

static

Definition at line 1330 of file SIISelLowering.cpp.

References llvm_unreachable.

Referenced by llvm::SITargetLowering::getTgtMemIntrinsic().

getLoadExtOrTrunc()

SDValue getLoadExtOrTrunc ( SelectionDAG & DAG, ISD::LoadExtType ExtType, SDValue Op, const SDLoc & SL, EVT VT )

static

Definition at line 12984 of file SIISelLowering.cpp.

References llvm::ISD::ANY_EXTEND, llvm::EVT::bitsLT(), llvm::ISD::EXTLOAD, llvm::SelectionDAG::getNode(), llvm_unreachable, llvm::ISD::NON_EXTLOAD, llvm::ISD::SEXTLOAD, llvm::ISD::SIGN_EXTEND, llvm::ISD::TRUNCATE, llvm::ISD::ZERO_EXTEND, and llvm::ISD::ZEXTLOAD.

getMad64_32()

SDValue getMad64_32 ( SelectionDAG & DAG, const SDLoc & SL, EVT VT, SDValue N0, SDValue N1, SDValue N2, bool Signed )

static

Definition at line 16862 of file SIISelLowering.cpp.

References llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getVTList(), llvm::AMDGPUISD::MAD_I64_I32, llvm::AMDGPUISD::MAD_U64_U32, Signed, and llvm::ISD::TRUNCATE.

Referenced by tryFoldMADwithSRL().

getPermuteMask()

uint32_t getPermuteMask ( SDValue V )

static

Definition at line 14443 of file SIISelLowering.cpp.

References llvm::ISD::AND, assert(), llvm::CallingConv::C, llvm::dyn_cast(), getConstantPermuteMask(), llvm::ConstantSDNode::getZExtValue(), llvm::ISD::OR, llvm::ISD::SHL, and llvm::ISD::SRL.

getPrivateAtomicExpansionKind()

TargetLowering::AtomicExpansionKind getPrivateAtomicExpansionKind ( const GCNSubtarget & STI )

static

Definition at line 20203 of file SIISelLowering.cpp.

References llvm::TargetLoweringBase::CustomExpand, and llvm::TargetLoweringBase::NotAtomic.

Referenced by llvm::SITargetLowering::shouldExpandAtomicCmpXchgInIR(), llvm::SITargetLowering::shouldExpandAtomicLoadInIR(), llvm::SITargetLowering::shouldExpandAtomicRMWInIR(), and llvm::SITargetLowering::shouldExpandAtomicStoreInIR().

getSPDenormModeValue()

SDValue getSPDenormModeValue ( uint32_t SPDenormMode, SelectionDAG & DAG, const SIMachineFunctionInfo * Info, const GCNSubtarget * ST )

static

Definition at line 13495 of file SIISelLowering.cpp.

References assert(), llvm::SelectionDAG::getTargetConstant(), and Mode.

getSplatConstantFP()

ConstantFPSDNode * getSplatConstantFP ( SDValue Op )

static

Definition at line 16167 of file SIISelLowering.cpp.

References llvm::CallingConv::C, and llvm::dyn_cast().

globalMemoryFPAtomicIsLegal()

bool globalMemoryFPAtomicIsLegal ( const GCNSubtarget & Subtarget, const AtomicRMWInst * RMW, bool HasSystemScope )

static

Returns

true if it's valid to emit a native instruction for RMW, based on the properties of the target memory.

Definition at line 20166 of file SIISelLowering.cpp.

References llvm::Instruction::hasMetadata().

Referenced by llvm::SITargetLowering::shouldExpandAtomicRMWInIR().

handleMulOperand()

std::optional< ByteProvider< SDValue > > handleMulOperand ( const SDValue & MulOperand )

static

Definition at line 17072 of file SIISelLowering.cpp.

References calculateByteProvider().

hasCFUser()

bool hasCFUser ( const Value * V, SmallPtrSet< const Value *, 16 > & Visited, unsigned WaveSize )

static

Definition at line 20552 of file SIISelLowering.cpp.

References llvm::dyn_cast(), hasCFUser(), llvm::SmallPtrSetImpl< PtrType >::insert(), llvm::isa(), and IT.

Referenced by hasCFUser(), and llvm::SITargetLowering::requiresUniformRegister().

hasNon16BitAccesses()

bool hasNon16BitAccesses ( uint64_t PermMask, SDValue & Op, SDValue & OtherOp )

static

Definition at line 15087 of file SIISelLowering.cpp.

References addresses16Bits(), isExtendedFrom16Bits(), and llvm::peekThroughBitcasts().

Referenced by matchPERM().

is32bitWaveReduceOperation()

bool is32bitWaveReduceOperation ( unsigned Opc )

static

Definition at line 5762 of file SIISelLowering.cpp.

References Opc.

Referenced by lowerWaveReduce().

isAsyncLDSDMA()

bool isAsyncLDSDMA ( Intrinsic::ID Intr )

static

Definition at line 12158 of file SIISelLowering.cpp.

isAtomicRMWLegalIntTy()

bool isAtomicRMWLegalIntTy ( Type * Ty )

static

Returns

true if atomicrmw integer ops work for the type.

Definition at line 20132 of file SIISelLowering.cpp.

References llvm::dyn_cast(), and IT.

Referenced by atomicSupportedIfLegalIntType(), and isAtomicRMWLegalXChgTy().

isAtomicRMWLegalXChgTy()

bool isAtomicRMWLegalXChgTy ( const AtomicRMWInst * RMW )

static

Returns

true if this atomicrmw xchg type can be selected.

Definition at line 20142 of file SIISelLowering.cpp.

References DL, llvm::dyn_cast(), llvm::Module::getDataLayout(), llvm::Instruction::getFunction(), llvm::GlobalValue::getParent(), llvm::Value::getType(), and isAtomicRMWLegalIntTy().

Referenced by llvm::SITargetLowering::shouldExpandAtomicRMWInIR().

isClampZeroToOne()

bool isClampZeroToOne ( SDValue A, SDValue B )

static

Definition at line 16422 of file SIISelLowering.cpp.

References A(), B(), and llvm::dyn_cast().

isCopyFromRegOfInlineAsm()

bool isCopyFromRegOfInlineAsm ( const SDNode * N )

static

Definition at line 19968 of file SIISelLowering.cpp.

References assert(), llvm::ISD::CopyFromReg, llvm::ISD::INLINEASM, llvm::ISD::INLINEASM_BR, and N.

Referenced by llvm::SITargetLowering::isSDNodeSourceOfDivergence().

isCtlzOpc()

bool isCtlzOpc ( unsigned Opc )

static

Definition at line 17610 of file SIISelLowering.cpp.

References llvm::ISD::CTLZ, llvm::ISD::CTLZ_ZERO_POISON, and Opc.

isExtendedFrom16Bits()

bool isExtendedFrom16Bits ( SDValue & Operand )

static

Definition at line 15042 of file SIISelLowering.cpp.

References llvm::ISD::ANY_EXTEND, llvm::cast(), llvm::ISD::EXTLOAD, llvm::SDValue::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::isVector(), llvm::ISD::LOAD, llvm::ISD::SEXTLOAD, llvm::ISD::SIGN_EXTEND, llvm::ISD::ZERO_EXTEND, and llvm::ISD::ZEXTLOAD.

Referenced by hasNon16BitAccesses().

isFloatingPointWaveReduceOperation()

bool isFloatingPointWaveReduceOperation ( unsigned Opc )

static

Definition at line 5772 of file SIISelLowering.cpp.

References Opc.

Referenced by lowerWaveReduce().

isFrameIndexOp()

bool isFrameIndexOp ( SDValue Op )

static

Definition at line 18902 of file SIISelLowering.cpp.

References llvm::ISD::AssertZext, and llvm::isa().

Referenced by llvm::SITargetLowering::legalizeTargetIndependentNode().

isImmConstraint()

bool isImmConstraint ( StringRef Constraint )

static

Definition at line 19384 of file SIISelLowering.cpp.

References llvm::StringRef::size().

Referenced by llvm::SITargetLowering::getConstraintType(), and llvm::SITargetLowering::LowerAsmOperandForConstraint().

isKnownNonNull()

bool isKnownNonNull ( SDValue Val, SelectionDAG & DAG, const AMDGPUTargetMachine & TM, unsigned AddrSpace )

static

Return true if the value is a known valid address, such that a null check is not necessary.

Definition at line 9258 of file SIISelLowering.cpp.

References llvm::dyn_cast(), llvm::AMDGPU::getNullPointerValue(), and llvm::isa().

isMul()

bool isMul ( const SDValue Op )

static

Definition at line 17256 of file SIISelLowering.cpp.

References llvm::ISD::MUL.

isNoUnsignedWrap()

bool isNoUnsignedWrap ( SDValue Addr )

static

Definition at line 12729 of file SIISelLowering.cpp.

References llvm::ISD::ADD, llvm::SDNode::getFlags(), llvm::SDNode::getOpcode(), llvm::SDValue::getOpcode(), llvm::SDNodeFlags::hasNoUnsignedWrap(), and llvm::ISD::OR.

isV2BF16()

bool isV2BF16 ( Type * Ty )

static

Definition at line 20126 of file SIISelLowering.cpp.

References llvm::dyn_cast(), llvm::VectorType::getElementType(), llvm::FixedVectorType::getNumElements(), and llvm::Type::isBFloatTy().

Referenced by llvm::SITargetLowering::shouldExpandAtomicRMWInIR().

isV2F16()

bool isV2F16 ( Type * Ty )

static

Definition at line 20121 of file SIISelLowering.cpp.

References llvm::dyn_cast(), llvm::VectorType::getElementType(), llvm::FixedVectorType::getNumElements(), and llvm::Type::isHalfTy().

Referenced by llvm::SITargetLowering::shouldExpandAtomicRMWInIR().

isV2F16OrV2BF16()

bool isV2F16OrV2BF16 ( Type * Ty )

static

Definition at line 20111 of file SIISelLowering.cpp.

References llvm::dyn_cast(), llvm::Type::isBFloatTy(), and llvm::Type::isHalfTy().

Referenced by llvm::SITargetLowering::shouldExpandAtomicRMWInIR().

knownBitsForSBFE()

void knownBitsForSBFE ( const MachineInstr & MI, GISelValueTracking & VT, KnownBits & Known, const APInt & DemandedElts, unsigned BFEWidth, bool SExt, unsigned Depth )

static

Definition at line 19706 of file SIISelLowering.cpp.

References llvm::GISelValueTracking::computeKnownBitsImpl(), llvm::Depth, llvm::KnownBits::extractBits(), llvm::getIConstantVRegValWithLookThrough(), llvm::MachineOperand::getImm(), llvm::GISelValueTracking::getMachineFunction(), llvm::MachineOperand::getReg(), llvm::MachineFunction::getRegInfo(), llvm::MachineOperand::isImm(), llvm::MachineOperand::isReg(), llvm::maskTrailingOnes(), MI, llvm::Offset, llvm::KnownBits::sext(), and llvm::KnownBits::zext().

Referenced by llvm::SITargetLowering::computeKnownBitsForTargetInstr().

knownBitsForWorkitemID()

void knownBitsForWorkitemID ( const GCNSubtarget & ST, GISelValueTracking & VT, KnownBits & Known, unsigned Dim )

static

Definition at line 19698 of file SIISelLowering.cpp.

References llvm::countl_zero(), llvm::MachineFunction::getFunction(), llvm::GISelValueTracking::getMachineFunction(), llvm::APInt::setHighBits(), and llvm::KnownBits::Zero.

Referenced by llvm::SITargetLowering::computeKnownBitsForTargetInstr().

loadM0FromVGPR()

MachineBasicBlock::iterator loadM0FromVGPR ( const SIInstrInfo * TII, MachineBasicBlock & MBB, MachineInstr & MI, unsigned InitResultReg, unsigned PhiReg, int Offset, bool UseGPRIdxMode, Register & SGPRIdxReg )

static

Definition at line 5301 of file SIISelLowering.cpp.

References llvm::MachineInstrBuilder::addReg(), llvm::MachineBasicBlock::addSuccessor(), llvm::MachineBasicBlock::begin(), llvm::BuildMI(), llvm::MachineFunction::CreateMachineBasicBlock(), llvm::MachineRegisterInfo::createVirtualRegister(), DL, emitLoadM0FromVGPRLoop(), llvm::AMDGPU::LaneMaskConstants::ExecReg, llvm::First, llvm::AMDGPU::LaneMaskConstants::get(), llvm::MachineFunction::getRegInfo(), llvm::MachineFunction::getSubtarget(), I, llvm::MachineFunction::insert(), MBB, MBBI, MI, llvm::AMDGPU::LaneMaskConstants::MovOpc, llvm::Offset, splitBlockForLoop(), TII, and TRI.

Referenced by emitIndirectDst(), and emitIndirectSrc().

lowerBALLOTIntrinsic()

SDValue lowerBALLOTIntrinsic ( const SITargetLowering & TLI, SDNode * N, SelectionDAG & DAG )

static

Definition at line 7883 of file SIISelLowering.cpp.

References llvm::dyn_cast(), llvm::ISD::FP_EXTEND, llvm::SelectionDAG::getCondCode(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getCopyFromReg(), llvm::SelectionDAG::getEntryNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getZExtOrTrunc(), N, SDValue(), llvm::ISD::SETCC, and llvm::ISD::SETNE.

lowerFCMPIntrinsic()

SDValue lowerFCMPIntrinsic ( const SITargetLowering & TLI, SDNode * N, SelectionDAG & DAG )

static

Definition at line 7854 of file SIISelLowering.cpp.

References llvm::EVT::bitsEq(), llvm::ISD::FP_EXTEND, llvm::SelectionDAG::getCondCode(), llvm::SelectionDAG::getContext(), llvm::getFCmpCondCode(), llvm::EVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getPOISON(), llvm::SITargetLowering::getSubtarget(), llvm::SDValue::getValueType(), llvm::AMDGPUSubtarget::getWavefrontSize(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::CmpInst::isFPPredicate(), llvm::TargetLoweringBase::isTypeLegal(), and N.

lowerICMPIntrinsic()

SDValue lowerICMPIntrinsic ( const SITargetLowering & TLI, SDNode * N, SelectionDAG & DAG )

static

Definition at line 7820 of file SIISelLowering.cpp.

References llvm::EVT::bitsEq(), DL, llvm::SelectionDAG::getCondCode(), llvm::SelectionDAG::getContext(), llvm::getICmpCondCode(), llvm::EVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getPOISON(), llvm::SITargetLowering::getSubtarget(), llvm::AMDGPUSubtarget::getWavefrontSize(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::CmpInst::isIntPredicate(), llvm::CmpInst::isSigned(), llvm::TargetLoweringBase::isTypeLegal(), LHS, N, RHS, llvm::ISD::SIGN_EXTEND, and llvm::ISD::ZERO_EXTEND.

lowerLaneOp()

SDValue lowerLaneOp ( const SITargetLowering & TLI, SDNode * N, SelectionDAG & DAG )

static

Definition at line 7929 of file SIISelLowering.cpp.

References assert(), llvm::SmallVectorTemplateCommon< T, typename >::begin(), llvm::ISD::CONCAT_VECTORS, llvm::ISD::CONVERGENCECTRL_GLUE, emitRemovedIntrinsicError(), llvm::SmallVectorTemplateCommon< T, typename >::end(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getAnyExtOrTrunc(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::MVT::getIntegerVT(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::EVT::getSimpleVT(), llvm::EVT::getSizeInBits(), llvm::SITargetLowering::getSubtarget(), llvm::SelectionDAG::getTargetConstant(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::SelectionDAG::getVectorIdxConstant(), llvm::EVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::MVT::getVectorVT(), llvm::ISD::INTRINSIC_WO_CHAIN, llvm::EVT::isFloatingPoint(), llvm::AMDGPU::isLegalDPALU_DPPControl(), llvm::EVT::isVector(), llvm_unreachable, N, llvm::SmallVectorTemplateBase< T, bool >::push_back(), SDValue(), and llvm::MVT::SimpleTy.

lowerWaveReduce()

MachineBasicBlock * lowerWaveReduce ( MachineInstr & MI, MachineBasicBlock & BB, const GCNSubtarget & ST, unsigned Opc )

static

Definition at line 5873 of file SIISelLowering.cpp.

References Accumulator, llvm::MachineInstrBuilder::addImm(), llvm::MachineInstrBuilder::addMBB(), llvm::MachineInstrBuilder::addReg(), assert(), llvm::AMDGPU::DPP::BCAST15, llvm::AMDGPU::DPP::BCAST31, llvm::BuildMI(), llvm::MachineOperand::CreateReg(), llvm::MachineRegisterInfo::createVirtualRegister(), llvm::Dead, DEFAULT, llvm::Define, DL, llvm::DPP, llvm::MachineBasicBlock::end(), expand64BitScalarArithmetic(), expand64BitV_CNDMASK(), ExtractSubRegs(), getDPPOpcForWaveReduction(), getIdentityValueForWaveReduction(), llvm::getImm(), llvm::MachineInstr::getOperand(), llvm::MachineBasicBlock::getParent(), llvm::MachineOperand::getReg(), llvm::MachineRegisterInfo::getRegClass(), llvm::MachineFunction::getRegInfo(), llvm::AMDGPUSubtarget::GFX12, I, is32bitWaveReduceOperation(), isFloatingPointWaveReduceOperation(), llvm::Kill, MI, llvm::SISrcMods::NEG, llvm::SISrcMods::NONE, Opc, Register, llvm::AMDGPU::DPP::ROW_SHR_FIRST, llvm::MachineInstrBuilder::setOperandDead(), splitBlockForLoop(), TII, and TRI.

Referenced by llvm::SITargetLowering::EmitInstrWithCustomInserter().

lowerWaveShuffle()

SDValue lowerWaveShuffle ( const SITargetLowering & TLI, SDNode * N, SelectionDAG & DAG )

static

Definition at line 8145 of file SIISelLowering.cpp.

References llvm::ISD::AND, llvm::SmallVectorImpl< T >::append(), assert(), llvm::SelectionDAG::getAllOnesConstant(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getPOISON(), llvm::SelectionDAG::getSelect(), llvm::SelectionDAG::getSetCC(), llvm::SelectionDAG::getShiftAmountConstant(), llvm::EVT::getSizeInBits(), llvm::SITargetLowering::getSubtarget(), llvm::SelectionDAG::getTargetConstant(), llvm::SDNode::getValueType(), llvm::ISD::INTRINSIC_WO_CHAIN, llvm::GCNSubtarget::isWave64(), N, SDValue(), llvm::ISD::SETEQ, llvm::ISD::SHL, llvm::GCNSubtarget::supportsWaveWideBPermute(), and llvm::ISD::XOR.

matchPERM()

SDValue matchPERM ( SDNode * N, TargetLowering::DAGCombinerInfo & DCI )

static

Definition at line 15164 of file SIISelLowering.cpp.

References assert(), calculateByteProvider(), llvm::TargetLowering::DAGCombinerInfo::DAG, DL, llvm::ISD::FSHR, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBitcastedAnyExtOrTrunc(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getDataLayout(), getDWordFromOffset(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), hasNon16BitAccesses(), llvm::DataLayout::isBigEndian(), llvm::EVT::isByteSized(), N, P, llvm::SmallVectorTemplateBase< T, bool >::push_back(), SDValue(), and llvm::SmallVectorTemplateCommon< T, typename >::size().

Referenced by llvm::SITargetLowering::PerformDAGCombine().

memVTFromLoadIntrData()

EVT memVTFromLoadIntrData ( const SITargetLowering & TLI, const DataLayout & DL, Type * Ty, unsigned MaxNumLanes )

static

Definition at line 1272 of file SIISelLowering.cpp.

References assert(), DL, llvm::dyn_cast(), llvm::TargetLoweringBase::getValueType(), and llvm::EVT::getVectorVT().

Referenced by llvm::SITargetLowering::getTgtMemIntrinsic(), and memVTFromLoadIntrReturn().

memVTFromLoadIntrReturn()

EVT memVTFromLoadIntrReturn ( const SITargetLowering & TLI, const DataLayout & DL, Type * Ty, unsigned MaxNumLanes )

static

Definition at line 1288 of file SIISelLowering.cpp.

References assert(), DL, llvm::dyn_cast(), and memVTFromLoadIntrData().

Referenced by llvm::SITargetLowering::getTgtMemIntrinsic().

minMaxOpcToMin3Max3Opc()

unsigned minMaxOpcToMin3Max3Opc ( unsigned Opc )

static

Definition at line 16102 of file SIISelLowering.cpp.

References llvm::ISD::FMAXIMUM, llvm::ISD::FMAXIMUMNUM, llvm::ISD::FMAXNUM, llvm::ISD::FMAXNUM_IEEE, llvm::ISD::FMINIMUM, llvm::ISD::FMINIMUMNUM, llvm::ISD::FMINNUM, llvm::ISD::FMINNUM_IEEE, llvm_unreachable, Opc, llvm::ISD::SMAX, llvm::ISD::SMIN, llvm::ISD::UMAX, and llvm::ISD::UMIN.

packImage16bitOpsToDwords()

void packImage16bitOpsToDwords ( SelectionDAG & DAG, SDValue Op, MVT PackVectorVT, SmallVectorImpl< SDValue > & PackedAddrs, unsigned DimIdx, unsigned EndIdx, unsigned NumGradients )

static

Definition at line 10173 of file SIISelLowering.cpp.

References llvm::ISD::ANY_EXTEND, DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getNode(), llvm::SDValue::getValueType(), I, and llvm::SmallVectorTemplateBase< T, bool >::push_back().

padEltsToUndef()

SDValue padEltsToUndef ( SelectionDAG & DAG, const SDLoc & DL, EVT CastVT, SDValue Src, int ExtraElts )

static

Definition at line 10061 of file SIISelLowering.cpp.

References DL, llvm::SelectionDAG::ExtractVectorElements(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getPOISON(), llvm::EVT::getScalarType(), llvm::EVT::isVector(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), and llvm::Undef.

Referenced by constructRetValue().

parseAtomicOrderingCABIArg()

AtomicOrdering parseAtomicOrderingCABIArg ( const CallBase & CI, unsigned ArgIdx )

static

Definition at line 1365 of file SIISelLowering.cpp.

References llvm::Acquire, llvm::acquire, llvm::cast(), llvm::CallBase::getArgOperand(), llvm::Monotonic, llvm::Release, llvm::release, llvm::seq_cst, and llvm::SequentiallyConsistent.

Referenced by llvm::SITargetLowering::getTgtMemIntrinsic().

parseSyncscopeMDArg()

unsigned parseSyncscopeMDArg ( const CallBase & CI, unsigned ArgIdx )

static

Definition at line 1384 of file SIISelLowering.cpp.

References llvm::cast(), llvm::CallBase::getArgOperand(), llvm::Value::getContext(), llvm::MDNode::getOperand(), and llvm::LLVMContext::getOrInsertSyncScopeID().

Referenced by llvm::SITargetLowering::getTgtMemIntrinsic().

parseTexFail()

bool parseTexFail ( SDValue TexFailCtrl, SelectionDAG & DAG, SDValue * TFE, SDValue * LWE, bool & IsTexFail )

static

Definition at line 10155 of file SIISelLowering.cpp.

References llvm::cast(), DL, llvm::SDValue::getNode(), and llvm::SelectionDAG::getTargetConstant().

placeSources()

void placeSources ( ByteProvider< SDValue > & Src0, ByteProvider< SDValue > & Src1, SmallVectorImpl< DotSrc > & Src0s, SmallVectorImpl< DotSrc > & Src1s, int Step )

static

Definition at line 17104 of file SIISelLowering.cpp.

References addPermMasks(), assert(), llvm::SmallVectorTemplateCommon< T, typename >::end(), llvm::find_if(), I, llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::ByteProvider< ISelOp >::Src, and llvm::ByteProvider< ISelOp >::SrcOffset.

processPSInputArgs()

void processPSInputArgs ( SmallVectorImpl< ISD::InputArg > & Splits, CallingConv::ID CallConv, ArrayRef< ISD::InputArg > Ins, BitVector & Skipped, FunctionType * FType, SIMachineFunctionInfo * Info )

static

Definition at line 2805 of file SIISelLowering.cpp.

References llvm::CallingConv::AMDGPU_PS, assert(), E(), llvm::ISD::InputArg::Flags, llvm::ISD::InputArg::getOrigArgIndex(), llvm::MVT::getScalarSizeInBits(), I, llvm::ISD::ArgFlagsTy::isInReg(), llvm::ISD::ArgFlagsTy::isSplit(), llvm::ISD::ArgFlagsTy::isSplitEnd(), llvm::MVT::isVector(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::ArrayRef< T >::size(), llvm::ISD::InputArg::Used, and llvm::ISD::InputArg::VT.

Referenced by llvm::SITargetLowering::LowerFormalArguments().

reservePrivateMemoryRegs()

void reservePrivateMemoryRegs ( const TargetMachine & TM, MachineFunction & MF, const SIRegisterInfo & TRI, SIMachineFunctionInfo & Info )

static

Definition at line 3277 of file SIISelLowering.cpp.

References assert(), llvm::Function::getCallingConv(), llvm::MachineFunction::getFrameInfo(), llvm::MachineFunction::getFunction(), llvm::TargetMachine::getOptLevel(), llvm::MachineFunction::getRegInfo(), llvm::MachineFunction::getSubtarget(), llvm::MachineFrameInfo::hasCalls(), llvm::MachineFrameInfo::hasStackObjects(), llvm::MachineRegisterInfo::isLiveIn(), llvm::AMDGPU::isShader(), llvm::None, llvm::AMDGPUFunctionArgInfo::PRIVATE_SEGMENT_BUFFER, Reg, llvm::report_fatal_error(), and TRI.

Referenced by llvm::SITargetLowering::finalizeLowering().

resolveSources()

SDValue resolveSources ( SelectionDAG & DAG, SDLoc SL, SmallVectorImpl< DotSrc > & Srcs, bool IsSigned, bool IsAny )

static

Definition at line 17180 of file SIISelLowering.cpp.

References addPermMasks(), assert(), llvm::SmallVectorTemplateCommon< T, typename >::begin(), llvm::SmallVectorTemplateCommon< T, typename >::end(), llvm::SelectionDAG::getConstant(), getDWordFromOffset(), llvm::SelectionDAG::getNode(), llvm::ISD::OR, llvm::SmallVectorTemplateBase< T, bool >::push_back(), and llvm::SmallVectorTemplateCommon< T, typename >::size().

selectSOffset()

SDValue selectSOffset ( SDValue SOffset, SelectionDAG & DAG, const GCNSubtarget * Subtarget )

static

Definition at line 11325 of file SIISelLowering.cpp.

References llvm::SelectionDAG::getRegister(), and llvm::isNullConstant().

setM0ToIndexFromSGPR()

void setM0ToIndexFromSGPR ( const SIInstrInfo * TII, MachineRegisterInfo & MRI, MachineInstr & MI, int Offset )

static

Definition at line 5364 of file SIISelLowering.cpp.

References llvm::MachineInstrBuilder::add(), llvm::MachineInstrBuilder::addImm(), assert(), llvm::BuildMI(), DL, llvm::MachineOperand::getReg(), I, MBB, MI, llvm::Offset, and TII.

Referenced by emitIndirectDst(), and emitIndirectSrc().

splitBlockForLoop()

std::pair< MachineBasicBlock *, MachineBasicBlock * > splitBlockForLoop ( MachineInstr & MI, MachineBasicBlock & MBB, bool InstInLoop )

static

Returns

{ LoopBody, Remainder }

Definition at line 5105 of file SIISelLowering.cpp.

References llvm::MachineBasicBlock::addSuccessor(), llvm::MachineBasicBlock::begin(), llvm::MachineFunction::CreateMachineBasicBlock(), I, llvm::MachineFunction::insert(), MBB, MBBI, MI, llvm::Next, llvm::MachineBasicBlock::splice(), and llvm::MachineBasicBlock::transferSuccessorsAndUpdatePHIs().

Referenced by llvm::SITargetLowering::emitGWSMemViolTestLoop(), loadM0FromVGPR(), and lowerWaveReduce().

STATISTIC()

STATISTIC	(	NumTailCalls	,
		"Number of tail calls"	)

strictFPExtFromF16()

SDValue strictFPExtFromF16 ( SelectionDAG & DAG, SDValue Src )

static

Return the source of an fp_extend from f16 to f32, or a converted FP constant.

Definition at line 16738 of file SIISelLowering.cpp.

References llvm::APFloat::convert(), llvm::dyn_cast(), llvm::ISD::FP_EXTEND, llvm::SelectionDAG::getConstantFP(), llvm::APFloatBase::IEEEhalf(), llvm::APFloatBase::rmNearestTiesToEven, and SDValue().

SubIdx2Lane()

unsigned SubIdx2Lane ( unsigned Idx )

static

Helper function for adjustWritemask.

Definition at line 18705 of file SIISelLowering.cpp.

supportsMin3Max3()

bool supportsMin3Max3 ( const GCNSubtarget & Subtarget, unsigned Opc, EVT VT )

static

Returns

true if the subtarget supports minimum3 and maximum3 with the given base min/max opcode Opc for type VT.

Definition at line 16256 of file SIISelLowering.cpp.

References llvm::ISD::FMAXIMUM, llvm::ISD::FMAXIMUMNUM, llvm::ISD::FMAXNUM, llvm::ISD::FMAXNUM_IEEE, llvm::ISD::FMINIMUM, llvm::ISD::FMINIMUMNUM, llvm::ISD::FMINNUM, llvm::ISD::FMINNUM_IEEE, llvm::GCNSubtarget::hasMin3Max3_16(), llvm_unreachable, Opc, llvm::ISD::SMAX, llvm::ISD::SMIN, llvm::ISD::UMAX, and llvm::ISD::UMIN.

tryFoldMADwithSRL()

SDValue tryFoldMADwithSRL ( SelectionDAG & DAG, const SDLoc & SL, SDValue MulLHS, SDValue MulRHS, SDValue AddRHS )

static

Definition at line 16876 of file SIISelLowering.cpp.

References llvm::dyn_cast(), llvm::SDNode::getAsZExtVal(), llvm::SelectionDAG::getConstant(), getMad64_32(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getZeroExtendInReg(), llvm::Hi_32(), llvm::Lo_32(), SDValue(), llvm::ISD::SRL, std::swap(), and llvm::ISD::TRUNCATE.

tryReduceF64CompareToHiHalf()

ISD::CondCode tryReduceF64CompareToHiHalf ( const ISD::CondCode CC, const SDValue LHS, const SDValue RHS, const SelectionDAG & DAG )

static

Definition at line 17998 of file SIISelLowering.cpp.

References assert(), llvm::KnownFPClass::bitcast(), llvm::SelectionDAG::computeKnownBits(), llvm::KnownBits::eq(), llvm::EVT::getFltSemantics(), llvm::KnownFPClass::isKnownNeverNaN(), llvm::APInt::isSignBitSet(), LHS, RHS, llvm::ISD::SETCC_INVALID, llvm::ISD::SETEQ, llvm::ISD::SETFALSE, llvm::ISD::SETGE, llvm::ISD::SETGT, llvm::ISD::SETLE, llvm::ISD::SETLT, llvm::ISD::SETNE, llvm::ISD::SETOEQ, llvm::ISD::SETOGE, llvm::ISD::SETOGT, llvm::ISD::SETOLE, llvm::ISD::SETOLT, llvm::ISD::SETONE, llvm::ISD::SETTRUE, llvm::ISD::SETUEQ, llvm::ISD::SETUGE, llvm::ISD::SETUGT, llvm::ISD::SETULE, llvm::ISD::SETULT, llvm::ISD::SETUNE, llvm::KnownBits::trunc(), llvm::KnownBits::uge(), llvm::KnownBits::ule(), and llvm::KnownBits::Zero.

vectorEltWillFoldAway()

bool vectorEltWillFoldAway ( SDValue Op )

static

Definition at line 16028 of file SIISelLowering.cpp.

References llvm::isa().

Variable Documentation

DisableLoopAlignment

cl::opt< bool > DisableLoopAlignment("amdgpu-disable-loop-alignment", cl::desc("Do not align and prefetch loops"), cl::init(false)) ( "amdgpu-disable-loop-alignment" , cl::desc("Do not align and prefetch loops") , cl::init(false)  )

static

Referenced by llvm::SITargetLowering::getPrefLoopAlignment().

UseDivergentRegisterIndexing

cl::opt< bool > UseDivergentRegisterIndexing("amdgpu-use-divergent-register-indexing", cl::Hidden, cl::desc("Use indirect register addressing for divergent indexes"), cl::init(false)) ( "amdgpu-use-divergent-register-indexing" , cl::Hidden , cl::desc("Use indirect register addressing for divergent indexes") , cl::init(false)  )

static

Referenced by llvm::SITargetLowering::shouldExpandVectorDynExt().