SIISelLowering.cpp File Reference

Custom DAG lowering for SI. More...

#include "SIISelLowering.h"
#include "AMDGPU.h"
#include "AMDGPUInstrInfo.h"
#include "AMDGPUTargetMachine.h"
#include "GCNSubtarget.h"
#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
#include "SIMachineFunctionInfo.h"
#include "SIRegisterInfo.h"
#include "llvm/ADT/APInt.h"
#include "llvm/ADT/FloatingPointMode.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/Analysis/OptimizationRemarkEmitter.h"
#include "llvm/Analysis/UniformityAnalysis.h"
#include "llvm/CodeGen/Analysis.h"
#include "llvm/CodeGen/ByteProvider.h"
#include "llvm/CodeGen/FunctionLoweringInfo.h"
#include "llvm/CodeGen/GlobalISel/GISelKnownBits.h"
#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineLoopInfo.h"
#include "llvm/IR/DiagnosticInfo.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/IntrinsicsAMDGPU.h"
#include "llvm/IR/IntrinsicsR600.h"
#include "llvm/IR/MDBuilder.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/KnownBits.h"
#include "llvm/Support/ModRef.h"
#include "llvm/Transforms/Utils/LowerAtomic.h"
#include <optional>

Go to the source code of this file.

Classes
struct	DotSrc

Macros
#define	DEBUG_TYPE   "si-lower"

Functions
	STATISTIC (NumTailCalls, "Number of tail calls")
static bool	denormalModeIsFlushAllF32 (const MachineFunction &MF)
static bool	denormalModeIsFlushAllF64F16 (const MachineFunction &MF)
static unsigned	findFirstFreeSGPR (CCState &CCInfo)
static EVT	memVTFromLoadIntrData (const SITargetLowering &TLI, const DataLayout &DL, Type *Ty, unsigned MaxNumLanes)
static EVT	memVTFromLoadIntrReturn (const SITargetLowering &TLI, const DataLayout &DL, Type *Ty, unsigned MaxNumLanes)
static void	processPSInputArgs (SmallVectorImpl< ISD::InputArg > &Splits, CallingConv::ID CallConv, ArrayRef< ISD::InputArg > Ins, BitVector &Skipped, FunctionType *FType, SIMachineFunctionInfo *Info)
static ArgDescriptor	allocateVGPR32Input (CCState &CCInfo, unsigned Mask=~0u, ArgDescriptor Arg=ArgDescriptor())
static ArgDescriptor	allocateSGPR32InputImpl (CCState &CCInfo, const TargetRegisterClass *RC, unsigned NumArgRegs)
static void	allocateFixedSGPRInputImpl (CCState &CCInfo, const TargetRegisterClass *RC, MCRegister Reg)
static void	allocateSGPR32Input (CCState &CCInfo, ArgDescriptor &Arg)
static void	allocateSGPR64Input (CCState &CCInfo, ArgDescriptor &Arg)
static void	reservePrivateMemoryRegs (const TargetMachine &TM, MachineFunction &MF, const SIRegisterInfo &TRI, SIMachineFunctionInfo &Info)
static bool	canGuaranteeTCO (CallingConv::ID CC)
static bool	mayTailCallThisCC (CallingConv::ID CC)
	Return true if we might ever do TCO for calls with this calling convention.
static std::pair< MachineBasicBlock *, MachineBasicBlock * >	splitBlockForLoop (MachineInstr &MI, MachineBasicBlock &MBB, bool InstInLoop)
static MachineBasicBlock::iterator	emitLoadM0FromVGPRLoop (const SIInstrInfo *TII, MachineRegisterInfo &MRI, MachineBasicBlock &OrigBB, MachineBasicBlock &LoopBB, const DebugLoc &DL, const MachineOperand &Idx, unsigned InitReg, unsigned ResultReg, unsigned PhiReg, unsigned InitSaveExecReg, int Offset, bool UseGPRIdxMode, Register &SGPRIdxReg)
static MachineBasicBlock::iterator	loadM0FromVGPR (const SIInstrInfo *TII, MachineBasicBlock &MBB, MachineInstr &MI, unsigned InitResultReg, unsigned PhiReg, int Offset, bool UseGPRIdxMode, Register &SGPRIdxReg)
static std::pair< unsigned, int >	computeIndirectRegAndOffset (const SIRegisterInfo &TRI, const TargetRegisterClass *SuperRC, unsigned VecReg, int Offset)
static void	setM0ToIndexFromSGPR (const SIInstrInfo *TII, MachineRegisterInfo &MRI, MachineInstr &MI, int Offset)
static Register	getIndirectSGPRIdx (const SIInstrInfo *TII, MachineRegisterInfo &MRI, MachineInstr &MI, int Offset)
static MachineBasicBlock *	emitIndirectSrc (MachineInstr &MI, MachineBasicBlock &MBB, const GCNSubtarget &ST)
static MachineBasicBlock *	emitIndirectDst (MachineInstr &MI, MachineBasicBlock &MBB, const GCNSubtarget &ST)
static MachineBasicBlock *	lowerWaveReduce (MachineInstr &MI, MachineBasicBlock &BB, const GCNSubtarget &ST, unsigned Opc)
static SDValue	adjustLoadValueTypeImpl (SDValue Result, EVT LoadVT, const SDLoc &DL, SelectionDAG &DAG, bool Unpacked)
static SDValue	lowerICMPIntrinsic (const SITargetLowering &TLI, SDNode *N, SelectionDAG &DAG)
static SDValue	lowerFCMPIntrinsic (const SITargetLowering &TLI, SDNode *N, SelectionDAG &DAG)
static SDValue	lowerBALLOTIntrinsic (const SITargetLowering &TLI, SDNode *N, SelectionDAG &DAG)
static SDValue	lowerLaneOp (const SITargetLowering &TLI, SDNode *N, SelectionDAG &DAG)
static SDNode *	findUser (SDValue Value, unsigned Opcode)
	Helper function for LowerBRCOND.
static unsigned	getExtOpcodeForPromotedOp (SDValue Op)
static bool	isKnownNonNull (SDValue Val, SelectionDAG &DAG, const AMDGPUTargetMachine &TM, unsigned AddrSpace)
	Return true if the value is a known valid address, such that a null check is not necessary.
static bool	elementPairIsContiguous (ArrayRef< int > Mask, int Elt)
static SDValue	buildPCRelGlobalAddress (SelectionDAG &DAG, const GlobalValue *GV, const SDLoc &DL, int64_t Offset, EVT PtrVT, unsigned GAFlags=SIInstrInfo::MO_NONE)
static SDValue	emitNonHSAIntrinsicError (SelectionDAG &DAG, const SDLoc &DL, EVT VT)
static SDValue	emitRemovedIntrinsicError (SelectionDAG &DAG, const SDLoc &DL, EVT VT)
static SDValue	getBuildDwordsVector (SelectionDAG &DAG, SDLoc DL, ArrayRef< SDValue > Elts)
static SDValue	padEltsToUndef (SelectionDAG &DAG, const SDLoc &DL, EVT CastVT, SDValue Src, int ExtraElts)
static SDValue	constructRetValue (SelectionDAG &DAG, MachineSDNode *Result, ArrayRef< EVT > ResultTypes, bool IsTexFail, bool Unpacked, bool IsD16, int DMaskPop, int NumVDataDwords, bool IsAtomicPacked16Bit, const SDLoc &DL)
static bool	parseTexFail (SDValue TexFailCtrl, SelectionDAG &DAG, SDValue *TFE, SDValue *LWE, bool &IsTexFail)
static void	packImage16bitOpsToDwords (SelectionDAG &DAG, SDValue Op, MVT PackVectorVT, SmallVectorImpl< SDValue > &PackedAddrs, unsigned DimIdx, unsigned EndIdx, unsigned NumGradients)
static SDValue	selectSOffset (SDValue SOffset, SelectionDAG &DAG, const GCNSubtarget *Subtarget)
static SDValue	getLoadExtOrTrunc (SelectionDAG &DAG, ISD::LoadExtType ExtType, SDValue Op, const SDLoc &SL, EVT VT)
static bool	addressMayBeAccessedAsPrivate (const MachineMemOperand *MMO, const SIMachineFunctionInfo &Info)
static SDValue	getFPBinOp (SelectionDAG &DAG, unsigned Opcode, const SDLoc &SL, EVT VT, SDValue A, SDValue B, SDValue GlueChain, SDNodeFlags Flags)
static SDValue	getFPTernOp (SelectionDAG &DAG, unsigned Opcode, const SDLoc &SL, EVT VT, SDValue A, SDValue B, SDValue C, SDValue GlueChain, SDNodeFlags Flags)
static SDValue	getSPDenormModeValue (uint32_t SPDenormMode, SelectionDAG &DAG, const SIMachineFunctionInfo *Info, const GCNSubtarget *ST)
static unsigned	getBasePtrIndex (const MemSDNode *N)
	MemSDNode::getBasePtr() does not work for intrinsics, which needs to offset by the chain and intrinsic ID.
static bool	bitOpWithConstantIsReducible (unsigned Opc, uint32_t Val)
static uint32_t	getConstantPermuteMask (uint32_t C)
static uint32_t	getPermuteMask (SDValue V)
static const std::optional< ByteProvider< SDValue > >	calculateSrcByte (const SDValue Op, uint64_t DestByte, uint64_t SrcIndex=0, unsigned Depth=0)
static const std::optional< ByteProvider< SDValue > >	calculateByteProvider (const SDValue &Op, unsigned Index, unsigned Depth, unsigned StartingIndex=0)
static bool	isExtendedFrom16Bits (SDValue &Operand)
static bool	addresses16Bits (int Mask)
static bool	hasNon16BitAccesses (uint64_t PermMask, SDValue &Op, SDValue &OtherOp)
static SDValue	getDWordFromOffset (SelectionDAG &DAG, SDLoc SL, SDValue Src, unsigned DWordOffset)
static SDValue	matchPERM (SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static bool	vectorEltWillFoldAway (SDValue Op)
static unsigned	minMaxOpcToMin3Max3Opc (unsigned Opc)
static ConstantFPSDNode *	getSplatConstantFP (SDValue Op)
static bool	supportsMin3Max3 (const GCNSubtarget &Subtarget, unsigned Opc, EVT VT)
static bool	isClampZeroToOne (SDValue A, SDValue B)
static SDValue	strictFPExtFromF16 (SelectionDAG &DAG, SDValue Src)
	Return the source of an fp_extend from f16 to f32, or a converted FP constant.
static SDValue	getMad64_32 (SelectionDAG &DAG, const SDLoc &SL, EVT VT, SDValue N0, SDValue N1, SDValue N2, bool Signed)
static std::optional< ByteProvider< SDValue > >	handleMulOperand (const SDValue &MulOperand)
static unsigned	addPermMasks (unsigned First, unsigned Second)
static void	placeSources (ByteProvider< SDValue > &Src0, ByteProvider< SDValue > &Src1, SmallVectorImpl< DotSrc > &Src0s, SmallVectorImpl< DotSrc > &Src1s, int Step)
static SDValue	resolveSources (SelectionDAG &DAG, SDLoc SL, SmallVectorImpl< DotSrc > &Srcs, bool IsSigned, bool IsAny)
static void	fixMasks (SmallVectorImpl< DotSrc > &Srcs, unsigned ChainLength)
static bool	isMul (const SDValue Op)
static std::optional< bool >	checkDot4MulSignedness (const SDValue &N, ByteProvider< SDValue > &Src0, ByteProvider< SDValue > &Src1, const SDValue &S0Op, const SDValue &S1Op, const SelectionDAG &DAG)
static unsigned	SubIdx2Lane (unsigned Idx)
	Helper function for adjustWritemask.
static bool	isFrameIndexOp (SDValue Op)
static SDValue	buildSMovImm32 (SelectionDAG &DAG, const SDLoc &DL, uint64_t Val)
static bool	isImmConstraint (StringRef Constraint)
static uint64_t	clearUnusedBits (uint64_t Val, unsigned Size)
static int	getAlignedAGPRClassID (unsigned UnalignedClassID)
static void	knownBitsForWorkitemID (const GCNSubtarget &ST, GISelKnownBits &KB, KnownBits &Known, unsigned Dim)
static LLVM_ATTRIBUTE_UNUSED bool	isCopyFromRegOfInlineAsm (const SDNode *N)
static bool	atomicIgnoresDenormalModeOrFPModeIsFTZ (const AtomicRMWInst *RMW)
static OptimizationRemark	emitAtomicRMWLegalRemark (const AtomicRMWInst *RMW)
static bool	isV2F16OrV2BF16 (Type *Ty)
static bool	isV2F16 (Type *Ty)
static bool	isV2BF16 (Type *Ty)
static bool	isAtomicRMWLegalIntTy (Type *Ty)
static bool	isAtomicRMWLegalXChgTy (const AtomicRMWInst *RMW)
static bool	globalMemoryFPAtomicIsLegal (const GCNSubtarget &Subtarget, const AtomicRMWInst *RMW, bool HasSystemScope)
static TargetLowering::AtomicExpansionKind	atomicSupportedIfLegalIntType (const AtomicRMWInst *RMW)
static bool	flatInstrMayAccessPrivate (const Instruction *I)
	Return if a flat address space atomicrmw can access private memory.
static bool	hasCFUser (const Value *V, SmallPtrSet< const Value *, 16 > &Visited, unsigned WaveSize)

Variables
static cl::opt< bool >	DisableLoopAlignment ("amdgpu-disable-loop-alignment", cl::desc("Do not align and prefetch loops"), cl::init(false))
static cl::opt< bool >	UseDivergentRegisterIndexing ("amdgpu-use-divergent-register-indexing", cl::Hidden, cl::desc("Use indirect register addressing for divergent indexes"), cl::init(false))

Detailed Description

Custom DAG lowering for SI.

Definition in file SIISelLowering.cpp.

Macro Definition Documentation

DEBUG_TYPE

#define DEBUG_TYPE   "si-lower"

Definition at line 50 of file SIISelLowering.cpp.

Function Documentation

addPermMasks()

static unsigned addPermMasks ( unsigned  First, unsigned  Second  )

static

Definition at line 14000 of file SIISelLowering.cpp.

References assert(), and llvm::First.

Referenced by placeSources(), and resolveSources().

addresses16Bits()

static bool addresses16Bits ( int  Mask )

static

Definition at line 12301 of file SIISelLowering.cpp.

References assert().

Referenced by hasNon16BitAccesses().

addressMayBeAccessedAsPrivate()

static bool addressMayBeAccessedAsPrivate ( const MachineMemOperand *  MMO, const SIMachineFunctionInfo &  Info  )

static

Definition at line 10440 of file SIISelLowering.cpp.

References Info.

adjustLoadValueTypeImpl()

static SDValue adjustLoadValueTypeImpl ( SDValue  Result, EVT  LoadVT, const SDLoc &  DL, SelectionDAG &  DAG, bool  Unpacked  )

static

Definition at line 5975 of file SIISelLowering.cpp.

References llvm::ISD::BITCAST, llvm::EVT::changeTypeToInteger(), DL, llvm::SelectionDAG::ExtractVectorElements(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getUNDEF(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::EVT::isVector(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), and llvm::ISD::TRUNCATE.

Referenced by constructRetValue().

allocateFixedSGPRInputImpl()

static void allocateFixedSGPRInputImpl ( CCState &  CCInfo, const TargetRegisterClass *  RC, MCRegister  Reg  )

static

Definition at line 2333 of file SIISelLowering.cpp.

References llvm::MachineFunction::addLiveIn(), llvm::CCState::AllocateReg(), assert(), and llvm::CCState::getMachineFunction().

Referenced by allocateSGPR32Input(), and allocateSGPR64Input().

allocateSGPR32Input()

static void allocateSGPR32Input ( CCState &  CCInfo, ArgDescriptor &  Arg  )

static

Definition at line 2342 of file SIISelLowering.cpp.

References allocateFixedSGPRInputImpl(), allocateSGPR32InputImpl(), and llvm::ArgDescriptor::getRegister().

Referenced by llvm::SITargetLowering::allocateSpecialInputSGPRs().

allocateSGPR32InputImpl()

static ArgDescriptor allocateSGPR32InputImpl ( CCState &  CCInfo, const TargetRegisterClass *  RC, unsigned  NumArgRegs  )

static

Definition at line 2313 of file SIISelLowering.cpp.

References llvm::MachineFunction::addLiveIn(), llvm::CCState::AllocateReg(), assert(), llvm::TargetRegisterClass::begin(), llvm::ArgDescriptor::createRegister(), llvm::CCState::getFirstUnallocated(), llvm::CCState::getMachineFunction(), llvm::report_fatal_error(), and llvm::ArrayRef< T >::size().

Referenced by allocateSGPR32Input(), and allocateSGPR64Input().

allocateSGPR64Input()

static void allocateSGPR64Input ( CCState &  CCInfo, ArgDescriptor &  Arg  )

static

Definition at line 2350 of file SIISelLowering.cpp.

References allocateFixedSGPRInputImpl(), allocateSGPR32InputImpl(), and llvm::ArgDescriptor::getRegister().

Referenced by llvm::SITargetLowering::allocateSpecialInputSGPRs().

allocateVGPR32Input()

static ArgDescriptor allocateVGPR32Input ( CCState &  CCInfo, unsigned  Mask = ~0u, ArgDescriptor  Arg = ArgDescriptor()  )

static

Definition at line 2289 of file SIISelLowering.cpp.

References llvm::MachineFunction::addLiveIn(), llvm::CCState::AllocateReg(), llvm::CCState::AllocateStack(), assert(), llvm::ArgDescriptor::createArg(), llvm::ArgDescriptor::createRegister(), llvm::ArgDescriptor::createStack(), llvm::CCState::getFirstUnallocated(), llvm::CCState::getMachineFunction(), llvm::MachineFunction::getRegInfo(), llvm::Offset, llvm::LLT::scalar(), llvm::MachineRegisterInfo::setType(), and llvm::ArrayRef< T >::size().

Referenced by llvm::SITargetLowering::allocateSpecialInputVGPRs().

atomicIgnoresDenormalModeOrFPModeIsFTZ()

static bool atomicIgnoresDenormalModeOrFPModeIsFTZ ( const AtomicRMWInst *  RMW )

static

Definition at line 16393 of file SIISelLowering.cpp.

References Flt, llvm::Function::getDenormalMode(), llvm::Type::getFltSemantics(), llvm::Function::getFnAttribute(), llvm::Instruction::getFunction(), llvm::DenormalMode::getPreserveSign(), llvm::Type::getScalarType(), llvm::Value::getType(), llvm::Attribute::getValueAsBool(), and llvm::Instruction::hasMetadata().

Referenced by llvm::SITargetLowering::shouldExpandAtomicRMWInIR().

atomicSupportedIfLegalIntType()

static TargetLowering::AtomicExpansionKind atomicSupportedIfLegalIntType ( const AtomicRMWInst *  RMW )

static

Returns

Action to perform on AtomicRMWInsts for integer operations.

Definition at line 16495 of file SIISelLowering.cpp.

References llvm::TargetLoweringBase::CmpXChg, llvm::Value::getType(), isAtomicRMWLegalIntTy(), and llvm::TargetLoweringBase::None.

Referenced by llvm::SITargetLowering::shouldExpandAtomicRMWInIR().

bitOpWithConstantIsReducible()

static bool bitOpWithConstantIsReducible ( unsigned  Opc, uint32_t  Val  )

static

Definition at line 11605 of file SIISelLowering.cpp.

References llvm::ISD::AND, llvm::ISD::OR, and llvm::ISD::XOR.

buildPCRelGlobalAddress()

static SDValue buildPCRelGlobalAddress ( SelectionDAG &  DAG, const GlobalValue *  GV, const SDLoc &  DL, int64_t  Offset, EVT  PtrVT, unsigned  GAFlags = SIInstrInfo::MO_NONE  )

static

Definition at line 7679 of file SIISelLowering.cpp.

References assert(), DL, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getTargetGlobalAddress(), llvm::SIInstrInfo::MO_NONE, llvm::Offset, and llvm::AMDGPUISD::PC_ADD_REL_OFFSET.

buildSMovImm32()

static SDValue buildSMovImm32 ( SelectionDAG &  DAG, const SDLoc &  DL, uint64_t  Val  )

static

Definition at line 15605 of file SIISelLowering.cpp.

References DL, llvm::SelectionDAG::getMachineNode(), and llvm::SelectionDAG::getTargetConstant().

Referenced by llvm::SITargetLowering::buildRSRC(), and llvm::SITargetLowering::wrapAddr64Rsrc().

calculateByteProvider()

static const std::optional< ByteProvider< SDValue > > calculateByteProvider ( const SDValue &  Op, unsigned  Index, unsigned  Depth, unsigned  StartingIndex = 0  )

static

Definition at line 12011 of file SIISelLowering.cpp.

References llvm::ISD::AND, llvm::ISD::ANY_EXTEND, llvm::ISD::AssertSext, llvm::ISD::AssertZext, llvm::BitWidth, llvm::ISD::BSWAP, calculateByteProvider(), calculateSrcByte(), llvm::ISD::CopyFromReg, llvm::Depth, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::ISD::FSHR, llvm::ByteProvider< ISelOp >::getConstantZero(), llvm::SDValue::getValueSizeInBits(), LHS, llvm_unreachable, llvm::ISD::LOAD, llvm::ISD::OR, llvm::AMDGPUISD::PERM, RHS, llvm::ISD::SHL, llvm::ISD::SIGN_EXTEND, llvm::ISD::SIGN_EXTEND_INREG, llvm::ISD::SRA, llvm::ISD::SRL, llvm::ISD::TRUNCATE, llvm::ISD::ZERO_EXTEND, and llvm::ISD::ZEXTLOAD.

calculateSrcByte()

static const std::optional< ByteProvider< SDValue > > calculateSrcByte ( const SDValue  Op, uint64_t  DestByte, uint64_t  SrcIndex = 0, unsigned  Depth = 0  )

static

Definition at line 11946 of file SIISelLowering.cpp.

References calculateSrcByte(), llvm::Depth, llvm::ByteProvider< ISelOp >::getSrc(), llvm::EVT::getStoreSize(), llvm::SDValue::getValueType(), llvm::EVT::isByteSized(), llvm_unreachable, llvm::ISD::SIGN_EXTEND, llvm::ISD::SIGN_EXTEND_INREG, llvm::ISD::SRA, llvm::ISD::SRL, llvm::ISD::TRUNCATE, and llvm::ISD::ZERO_EXTEND.

Referenced by calculateByteProvider(), and calculateSrcByte().

canGuaranteeTCO()

static bool canGuaranteeTCO ( CallingConv::ID  CC )

static

Definition at line 3512 of file SIISelLowering.cpp.

References CC, and llvm::CallingConv::Fast.

checkDot4MulSignedness()

static std::optional< bool > checkDot4MulSignedness ( const SDValue &  N, ByteProvider< SDValue > &  Src0, ByteProvider< SDValue > &  Src1, const SDValue &  S0Op, const SDValue &  S1Op, const SelectionDAG &  DAG  )

static

Definition at line 14180 of file SIISelLowering.cpp.

References assert(), llvm::SelectionDAG::computeKnownBits(), llvm::KnownBits::countMinLeadingZeros(), llvm::SDValue::getValueSizeInBits(), and llvm_unreachable.

clearUnusedBits()

static uint64_t clearUnusedBits ( uint64_t  Val, unsigned  Size  )

static

Definition at line 15828 of file SIISelLowering.cpp.

References llvm::AMDGPU::isInlinableIntLiteral(), and Size.

Referenced by llvm::APInt::APInt(), llvm::APInt::ashrInPlace(), llvm::SITargetLowering::checkAsmConstraintVal(), llvm::APInt::flipAllBits(), llvm::SITargetLowering::LowerAsmOperandForConstraint(), llvm::APInt::operator<<=(), llvm::APInt::operator=(), llvm::APInt::operator^=(), llvm::APInt::operator|=(), and llvm::APInt::setAllBits().

computeIndirectRegAndOffset()

static std::pair< unsigned, int > computeIndirectRegAndOffset ( const SIRegisterInfo &  TRI, const TargetRegisterClass *  SuperRC, unsigned  VecReg, int  Offset  )

static

Definition at line 4676 of file SIISelLowering.cpp.

References llvm::SIRegisterInfo::getSubRegFromChannel(), llvm::Offset, and TRI.

Referenced by emitIndirectDst(), and emitIndirectSrc().

constructRetValue()

static SDValue constructRetValue ( SelectionDAG &  DAG, MachineSDNode *  Result, ArrayRef< EVT >  ResultTypes, bool  IsTexFail, bool  Unpacked, bool  IsD16, int  DMaskPop, int  NumVDataDwords, bool  IsAtomicPacked16Bit, const SDLoc &  DL  )

static

Definition at line 7883 of file SIISelLowering.cpp.

References adjustLoadValueTypeImpl(), llvm::ISD::BITCAST, llvm::EVT::changeTypeToInteger(), llvm::Data, DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::MVT::getVectorVT(), llvm::EVT::isVector(), llvm::MVT::isVector(), padEltsToUndef(), and llvm::ISD::TRUNCATE.

denormalModeIsFlushAllF32()

static bool denormalModeIsFlushAllF32 ( const MachineFunction &  MF )

static

Definition at line 64 of file SIISelLowering.cpp.

References llvm::MachineFunction::getInfo(), llvm::DenormalMode::getPreserveSign(), and Info.

Referenced by llvm::SITargetLowering::denormalsEnabledForType(), llvm::SITargetLowering::isFMADLegal(), llvm::SITargetLowering::isFMAFasterThanFMulAndFAdd(), and llvm::SITargetLowering::isFPExtFoldable().

denormalModeIsFlushAllF64F16()

static bool denormalModeIsFlushAllF64F16 ( const MachineFunction &  MF )

static

Definition at line 69 of file SIISelLowering.cpp.

References llvm::MachineFunction::getInfo(), llvm::DenormalMode::getPreserveSign(), and Info.

Referenced by llvm::SITargetLowering::denormalsEnabledForType(), llvm::SITargetLowering::isFMADLegal(), and llvm::SITargetLowering::isFMAFasterThanFMulAndFAdd().

elementPairIsContiguous()

static bool elementPairIsContiguous ( ArrayRef< int >  Mask, int  Elt  )

static

Definition at line 7535 of file SIISelLowering.cpp.

References assert().

emitAtomicRMWLegalRemark()

static OptimizationRemark emitAtomicRMWLegalRemark ( const AtomicRMWInst *  RMW )

static

Definition at line 16408 of file SIISelLowering.cpp.

References DEBUG_TYPE, llvm::Value::getContext(), llvm::AtomicRMWInst::getOperation(), llvm::AtomicRMWInst::getOperationName(), llvm::AtomicRMWInst::getSyncScopeID(), and llvm::LLVMContext::getSyncScopeName().

Referenced by llvm::SITargetLowering::shouldExpandAtomicRMWInIR().

emitIndirectDst()

static MachineBasicBlock * emitIndirectDst ( MachineInstr &  MI, MachineBasicBlock &  MBB, const GCNSubtarget &  ST  )

static

Definition at line 4817 of file SIISelLowering.cpp.

References llvm::MachineInstrBuilder::add(), llvm::MachineInstrBuilder::addImm(), llvm::MachineInstrBuilder::addReg(), assert(), llvm::BuildMI(), computeIndirectRegAndOffset(), DL, getIndirectSGPRIdx(), llvm::MachineBasicBlock::getParent(), llvm::MachineOperand::getReg(), llvm::MachineFunction::getRegInfo(), I, Idx, llvm::MachineOperand::isReg(), loadM0FromVGPR(), MBB, MI, MRI, llvm::Offset, setM0ToIndexFromSGPR(), SubReg, TII, and TRI.

Referenced by llvm::SITargetLowering::EmitInstrWithCustomInserter().

emitIndirectSrc()

static MachineBasicBlock * emitIndirectSrc ( MachineInstr &  MI, MachineBasicBlock &  MBB, const GCNSubtarget &  ST  )

static

Definition at line 4731 of file SIISelLowering.cpp.

References llvm::MachineInstrBuilder::addImm(), llvm::MachineInstrBuilder::addReg(), llvm::BuildMI(), computeIndirectRegAndOffset(), DL, getIndirectSGPRIdx(), llvm::MachineBasicBlock::getParent(), llvm::MachineFunction::getRegInfo(), I, Idx, llvm::RegState::Implicit, loadM0FromVGPR(), MBB, MI, MRI, llvm::Offset, setM0ToIndexFromSGPR(), SubReg, TII, and TRI.

Referenced by llvm::SITargetLowering::EmitInstrWithCustomInserter().

emitLoadM0FromVGPRLoop()

static MachineBasicBlock::iterator emitLoadM0FromVGPRLoop ( const SIInstrInfo *  TII, MachineRegisterInfo &  MRI, MachineBasicBlock &  OrigBB, MachineBasicBlock &  LoopBB, const DebugLoc &  DL, const MachineOperand &  Idx, unsigned  InitReg, unsigned  ResultReg, unsigned  PhiReg, unsigned  InitSaveExecReg, int  Offset, bool  UseGPRIdxMode, Register &  SGPRIdxReg  )

static

Definition at line 4528 of file SIISelLowering.cpp.

References llvm::MachineInstrBuilder::addImm(), llvm::MachineInstrBuilder::addMBB(), llvm::MachineInstrBuilder::addReg(), llvm::MachineBasicBlock::begin(), llvm::BuildMI(), DL, llvm::ilist_node_impl< OptionsT >::getIterator(), llvm::MachineBasicBlock::getParent(), llvm::MachineFunction::getSubtarget(), llvm::getUndefRegState(), I, Idx, llvm::RegState::Kill, MRI, llvm::Offset, TII, and TRI.

Referenced by loadM0FromVGPR().

emitNonHSAIntrinsicError()

static SDValue emitNonHSAIntrinsicError ( SelectionDAG &  DAG, const SDLoc &  DL, EVT  VT  )

static

Definition at line 7815 of file SIISelLowering.cpp.

References llvm::LLVMContext::diagnose(), DL, llvm::SelectionDAG::getContext(), llvm::MachineFunction::getFunction(), llvm::SelectionDAG::getMachineFunction(), and llvm::SelectionDAG::getUNDEF().

emitRemovedIntrinsicError()

static SDValue emitRemovedIntrinsicError ( SelectionDAG &  DAG, const SDLoc &  DL, EVT  VT  )

static

Definition at line 7824 of file SIISelLowering.cpp.

References llvm::LLVMContext::diagnose(), DL, llvm::SelectionDAG::getContext(), llvm::MachineFunction::getFunction(), llvm::SelectionDAG::getMachineFunction(), and llvm::SelectionDAG::getUNDEF().

findFirstFreeSGPR()

static unsigned findFirstFreeSGPR ( CCState &  CCInfo )

static

Definition at line 74 of file SIISelLowering.cpp.

References llvm::CCState::isAllocated(), and llvm_unreachable.

Referenced by llvm::SITargetLowering::allocateSystemSGPRs().

findUser()

static SDNode * findUser ( SDValue  Value, unsigned  Opcode  )

static

Helper function for LowerBRCOND.

Definition at line 6572 of file SIISelLowering.cpp.

References llvm::Value::uses().

fixMasks()

static void fixMasks ( SmallVectorImpl< DotSrc > &  Srcs, unsigned  ChainLength  )

static

Definition at line 14164 of file SIISelLowering.cpp.

flatInstrMayAccessPrivate()

static bool flatInstrMayAccessPrivate ( const Instruction *  I )

static

Return if a flat address space atomicrmw can access private memory.

Definition at line 16502 of file SIISelLowering.cpp.

References llvm::MDNode::getNumOperands(), llvm::MDNode::getOperand(), High, I, llvm::Low, and llvm::AMDGPUAS::PRIVATE_ADDRESS.

Referenced by llvm::SITargetLowering::shouldExpandAtomicCmpXchgInIR(), and llvm::SITargetLowering::shouldExpandAtomicRMWInIR().

getAlignedAGPRClassID()

static int getAlignedAGPRClassID ( unsigned  UnalignedClassID )

static

Definition at line 15946 of file SIISelLowering.cpp.

Referenced by llvm::SITargetLowering::finalizeLowering().

getBasePtrIndex()

static unsigned getBasePtrIndex ( const MemSDNode *  N )

static

MemSDNode::getBasePtr() does not work for intrinsics, which needs to offset by the chain and intrinsic ID.

Theoretically we would also need to check the specific intrinsic, but they all place the pointer operand first.

Definition at line 11571 of file SIISelLowering.cpp.

References llvm::ISD::INTRINSIC_VOID, llvm::ISD::INTRINSIC_W_CHAIN, N, and llvm::ISD::STORE.

Referenced by llvm::SITargetLowering::hasMemSDNodeUser().

getBuildDwordsVector()

static SDValue getBuildDwordsVector ( SelectionDAG &  DAG, SDLoc  DL, ArrayRef< SDValue >  Elts  )

static

Definition at line 7833 of file SIISelLowering.cpp.

References assert(), DL, llvm::ArrayRef< T >::empty(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::MVT::getVectorVT(), and llvm::ArrayRef< T >::size().

getConstantPermuteMask()

static uint32_t getConstantPermuteMask ( uint32_t  C )

static

Definition at line 11655 of file SIISelLowering.cpp.

References llvm::CallingConv::C.

Referenced by getPermuteMask().

getDWordFromOffset()

static SDValue getDWordFromOffset ( SelectionDAG &  DAG, SDLoc  SL, SDValue  Src, unsigned  DWordOffset  )

static

Scalar Type

Definition at line 12342 of file SIISelLowering.cpp.

References assert(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::ExtractVectorElements(), llvm::SelectionDAG::getBitcastedAnyExtOrTrunc(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::MVT::getVectorVT(), and llvm::ISD::SRL.

Referenced by matchPERM(), and resolveSources().

getExtOpcodeForPromotedOp()

static unsigned getExtOpcodeForPromotedOp ( SDValue  Op )

static

Definition at line 6825 of file SIISelLowering.cpp.

References llvm::ISD::ADD, llvm::ISD::AND, llvm::ISD::ANY_EXTEND, CC, llvm::ISD::isSignedIntSetCC(), llvm_unreachable, llvm::ISD::MUL, llvm::ISD::OR, llvm::ISD::SELECT, llvm::ISD::SETCC, llvm::ISD::SHL, llvm::ISD::SIGN_EXTEND, llvm::ISD::SMAX, llvm::ISD::SMIN, llvm::ISD::SRA, llvm::ISD::SRL, llvm::ISD::SUB, llvm::ISD::UMAX, llvm::ISD::UMIN, llvm::ISD::XOR, and llvm::ISD::ZERO_EXTEND.

getFPBinOp()

static SDValue getFPBinOp ( SelectionDAG &  DAG, unsigned  Opcode, const SDLoc &  SL, EVT  VT, SDValue  A, SDValue  B, SDValue  GlueChain, SDNodeFlags  Flags  )

static

Definition at line 10703 of file SIISelLowering.cpp.

References A, assert(), B, llvm::ISD::FMUL, llvm::AMDGPUISD::FMUL_W_CHAIN, llvm::SelectionDAG::getNode(), llvm::SDNode::getNumValues(), llvm::SDValue::getValue(), llvm::SelectionDAG::getVTList(), and llvm_unreachable.

getFPTernOp()

static SDValue getFPTernOp ( SelectionDAG &  DAG, unsigned  Opcode, const SDLoc &  SL, EVT  VT, SDValue  A, SDValue  B, SDValue  C, SDValue  GlueChain, SDNodeFlags  Flags  )

static

Definition at line 10726 of file SIISelLowering.cpp.

References A, assert(), B, llvm::CallingConv::C, llvm::ISD::FMA, llvm::AMDGPUISD::FMA_W_CHAIN, llvm::SelectionDAG::getNode(), llvm::SDNode::getNumValues(), llvm::SDValue::getValue(), llvm::SelectionDAG::getVTList(), and llvm_unreachable.

getIndirectSGPRIdx()

static Register getIndirectSGPRIdx ( const SIInstrInfo *  TII, MachineRegisterInfo &  MRI, MachineInstr &  MI, int  Offset  )

static

Definition at line 4712 of file SIISelLowering.cpp.

References llvm::MachineInstrBuilder::add(), llvm::MachineInstrBuilder::addImm(), llvm::BuildMI(), DL, I, Idx, MBB, MI, MRI, llvm::Offset, and TII.

Referenced by emitIndirectDst(), and emitIndirectSrc().

getLoadExtOrTrunc()

static SDValue getLoadExtOrTrunc ( SelectionDAG &  DAG, ISD::LoadExtType  ExtType, SDValue  Op, const SDLoc &  SL, EVT  VT  )

static

Definition at line 10351 of file SIISelLowering.cpp.

References llvm::ISD::ANY_EXTEND, llvm::EVT::bitsLT(), llvm::ISD::EXTLOAD, llvm::SelectionDAG::getNode(), llvm_unreachable, llvm::ISD::NON_EXTLOAD, llvm::ISD::SEXTLOAD, llvm::ISD::SIGN_EXTEND, llvm::ISD::TRUNCATE, llvm::ISD::ZERO_EXTEND, and llvm::ISD::ZEXTLOAD.

getMad64_32()

static SDValue getMad64_32 ( SelectionDAG &  DAG, const SDLoc &  SL, EVT  VT, SDValue  N0, SDValue  N1, SDValue  N2, bool  Signed  )

static

Definition at line 13852 of file SIISelLowering.cpp.

References llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getVTList(), llvm::AMDGPUISD::MAD_I64_I32, llvm::AMDGPUISD::MAD_U64_U32, Signed, and llvm::ISD::TRUNCATE.

getPermuteMask()

static uint32_t getPermuteMask ( SDValue  V )

static

Definition at line 11679 of file SIISelLowering.cpp.

References llvm::ISD::AND, assert(), llvm::CallingConv::C, getConstantPermuteMask(), llvm::ConstantSDNode::getZExtValue(), llvm::ISD::OR, llvm::ISD::SHL, and llvm::ISD::SRL.

getSPDenormModeValue()

static SDValue getSPDenormModeValue ( uint32_t  SPDenormMode, SelectionDAG &  DAG, const SIMachineFunctionInfo *  Info, const GCNSubtarget *  ST  )

static

Definition at line 10834 of file SIISelLowering.cpp.

References assert(), llvm::SelectionDAG::getTargetConstant(), and Info.

getSplatConstantFP()

static ConstantFPSDNode * getSplatConstantFP ( SDValue  Op )

static

Definition at line 13295 of file SIISelLowering.cpp.

References llvm::CallingConv::C.

globalMemoryFPAtomicIsLegal()

static bool globalMemoryFPAtomicIsLegal ( const GCNSubtarget &  Subtarget, const AtomicRMWInst *  RMW, bool  HasSystemScope  )

static

Returns

true if it's valid to emit a native instruction for RMW, based on the properties of the target memory.

Definition at line 16474 of file SIISelLowering.cpp.

References llvm::Instruction::hasMetadata(), and llvm::GCNSubtarget::supportsAgentScopeFineGrainedRemoteMemoryAtomics().

Referenced by llvm::SITargetLowering::shouldExpandAtomicRMWInIR().

handleMulOperand()

static std::optional< ByteProvider< SDValue > > handleMulOperand ( const SDValue &  MulOperand )

static

Definition at line 13988 of file SIISelLowering.cpp.

References calculateByteProvider().

hasCFUser()

static bool hasCFUser ( const Value *  V, SmallPtrSet< const Value *, 16 > &  Visited, unsigned  WaveSize  )

static

Definition at line 16804 of file SIISelLowering.cpp.

References hasCFUser(), llvm::SmallPtrSetImpl< PtrType >::insert(), and IT.

Referenced by hasCFUser(), and llvm::SITargetLowering::requiresUniformRegister().

hasNon16BitAccesses()

static bool hasNon16BitAccesses ( uint64_t  PermMask, SDValue &  Op, SDValue &  OtherOp  )

static

Definition at line 12320 of file SIISelLowering.cpp.

References addresses16Bits(), isExtendedFrom16Bits(), and llvm::peekThroughBitcasts().

Referenced by matchPERM().

isAtomicRMWLegalIntTy()

static bool isAtomicRMWLegalIntTy ( Type *  Ty )

static

Returns

true if atomicrmw integer ops work for the type.

Definition at line 16440 of file SIISelLowering.cpp.

References IT.

Referenced by atomicSupportedIfLegalIntType(), and isAtomicRMWLegalXChgTy().

isAtomicRMWLegalXChgTy()

static bool isAtomicRMWLegalXChgTy ( const AtomicRMWInst *  RMW )

static

Returns

true if this atomicrmw xchg type can be selected.

Definition at line 16450 of file SIISelLowering.cpp.

References DL, llvm::Module::getDataLayout(), llvm::Instruction::getFunction(), llvm::GlobalValue::getParent(), llvm::Value::getType(), isAtomicRMWLegalIntTy(), llvm::Type::isDoubleTy(), and llvm::Type::isFloatTy().

Referenced by llvm::SITargetLowering::shouldExpandAtomicRMWInIR().

isClampZeroToOne()

static bool isClampZeroToOne ( SDValue  A, SDValue  B  )

static

Definition at line 13456 of file SIISelLowering.cpp.

References A, and B.

isCopyFromRegOfInlineAsm()

static LLVM_ATTRIBUTE_UNUSED bool isCopyFromRegOfInlineAsm ( const SDNode *  N )

static

Definition at line 16276 of file SIISelLowering.cpp.

References assert(), llvm::ISD::CopyFromReg, llvm::ISD::INLINEASM, llvm::ISD::INLINEASM_BR, and N.

Referenced by llvm::SITargetLowering::isSDNodeSourceOfDivergence().

isExtendedFrom16Bits()

static bool isExtendedFrom16Bits ( SDValue &  Operand )

static

Definition at line 12275 of file SIISelLowering.cpp.

References llvm::ISD::ANY_EXTEND, llvm::ISD::EXTLOAD, llvm::SDValue::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::isVector(), llvm::ISD::LOAD, llvm::ISD::SEXTLOAD, llvm::ISD::SIGN_EXTEND, llvm::ISD::ZERO_EXTEND, and llvm::ISD::ZEXTLOAD.

Referenced by hasNon16BitAccesses().

isFrameIndexOp()

static bool isFrameIndexOp ( SDValue  Op )

static

Definition at line 15292 of file SIISelLowering.cpp.

References llvm::ISD::AssertZext.

Referenced by llvm::SITargetLowering::legalizeTargetIndependentNode().

isImmConstraint()

static bool isImmConstraint ( StringRef  Constraint )

static

Definition at line 15792 of file SIISelLowering.cpp.

References llvm::StringRef::size().

Referenced by llvm::SITargetLowering::getConstraintType(), and llvm::SITargetLowering::LowerAsmOperandForConstraint().

isKnownNonNull()

static bool isKnownNonNull ( SDValue  Val, SelectionDAG &  DAG, const AMDGPUTargetMachine &  TM, unsigned  AddrSpace  )

static

Return true if the value is a known valid address, such that a null check is not necessary.

Definition at line 7194 of file SIISelLowering.cpp.

isMul()

static bool isMul ( const SDValue  Op )

static

Definition at line 14172 of file SIISelLowering.cpp.

References llvm::ISD::MUL, llvm::AMDGPUISD::MUL_I24, and llvm::AMDGPUISD::MUL_U24.

isV2BF16()

static bool isV2BF16 ( Type *  Ty )

static

Definition at line 16434 of file SIISelLowering.cpp.

References llvm::VectorType::getElementType(), llvm::FixedVectorType::getNumElements(), and llvm::Type::isBFloatTy().

Referenced by llvm::SITargetLowering::shouldExpandAtomicRMWInIR().

isV2F16()

static bool isV2F16 ( Type *  Ty )

static

Definition at line 16429 of file SIISelLowering.cpp.

References llvm::VectorType::getElementType(), llvm::FixedVectorType::getNumElements(), and llvm::Type::isHalfTy().

Referenced by llvm::SITargetLowering::shouldExpandAtomicRMWInIR().

isV2F16OrV2BF16()

static bool isV2F16OrV2BF16 ( Type *  Ty )

static

Definition at line 16419 of file SIISelLowering.cpp.

References llvm::Type::isBFloatTy(), and llvm::Type::isHalfTy().

Referenced by llvm::SITargetLowering::shouldExpandAtomicRMWInIR().

knownBitsForWorkitemID()

static void knownBitsForWorkitemID ( const GCNSubtarget &  ST, GISelKnownBits &  KB, KnownBits &  Known, unsigned  Dim  )

static

Definition at line 16103 of file SIISelLowering.cpp.

References llvm::countl_zero(), llvm::MachineFunction::getFunction(), llvm::GISelKnownBits::getMachineFunction(), llvm::APInt::setHighBits(), and llvm::KnownBits::Zero.

Referenced by llvm::SITargetLowering::computeKnownBitsForTargetInstr().

loadM0FromVGPR()

static MachineBasicBlock::iterator loadM0FromVGPR ( const SIInstrInfo *  TII, MachineBasicBlock &  MBB, MachineInstr &  MI, unsigned  InitResultReg, unsigned  PhiReg, int  Offset, bool  UseGPRIdxMode, Register &  SGPRIdxReg  )

static

Definition at line 4625 of file SIISelLowering.cpp.

References llvm::MachineInstrBuilder::addReg(), llvm::MachineBasicBlock::addSuccessor(), llvm::MachineBasicBlock::begin(), llvm::BuildMI(), llvm::MachineFunction::CreateMachineBasicBlock(), DL, emitLoadM0FromVGPRLoop(), llvm::First, llvm::MachineBasicBlock::getParent(), llvm::MachineFunction::getRegInfo(), llvm::MachineFunction::getSubtarget(), I, Idx, llvm::MachineFunction::insert(), MBB, MBBI, MI, MRI, llvm::Offset, splitBlockForLoop(), TII, and TRI.

Referenced by emitIndirectDst(), and emitIndirectSrc().

lowerBALLOTIntrinsic()

static SDValue lowerBALLOTIntrinsic ( const SITargetLowering &  TLI, SDNode *  N, SelectionDAG &  DAG  )

static

Definition at line 6154 of file SIISelLowering.cpp.

References llvm::SelectionDAG::getCondCode(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getCopyFromReg(), llvm::SelectionDAG::getEntryNode(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), llvm::SelectionDAG::getZExtOrTrunc(), N, llvm::ISD::SETCC, llvm::AMDGPUISD::SETCC, and llvm::ISD::SETNE.

lowerFCMPIntrinsic()

static SDValue lowerFCMPIntrinsic ( const SITargetLowering &  TLI, SDNode *  N, SelectionDAG &  DAG  )

static

Definition at line 6125 of file SIISelLowering.cpp.

References llvm::EVT::bitsEq(), llvm::ISD::FP_EXTEND, llvm::SelectionDAG::getCondCode(), llvm::SelectionDAG::getContext(), llvm::getFCmpCondCode(), llvm::EVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::SITargetLowering::getSubtarget(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::AMDGPUSubtarget::getWavefrontSize(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::CmpInst::isFPPredicate(), llvm::TargetLoweringBase::isTypeLegal(), N, and llvm::AMDGPUISD::SETCC.

lowerICMPIntrinsic()

static SDValue lowerICMPIntrinsic ( const SITargetLowering &  TLI, SDNode *  N, SelectionDAG &  DAG  )

static

Definition at line 6091 of file SIISelLowering.cpp.

References llvm::EVT::bitsEq(), DL, llvm::SelectionDAG::getCondCode(), llvm::SelectionDAG::getContext(), llvm::getICmpCondCode(), llvm::EVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::SITargetLowering::getSubtarget(), llvm::SelectionDAG::getUNDEF(), llvm::AMDGPUSubtarget::getWavefrontSize(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::CmpInst::isIntPredicate(), llvm::CmpInst::isSigned(), llvm::TargetLoweringBase::isTypeLegal(), LHS, N, RHS, llvm::AMDGPUISD::SETCC, llvm::ISD::SIGN_EXTEND, and llvm::ISD::ZERO_EXTEND.

lowerLaneOp()

static SDValue lowerLaneOp ( const SITargetLowering &  TLI, SDNode *  N, SelectionDAG &  DAG  )

static

Definition at line 6191 of file SIISelLowering.cpp.

References assert(), llvm::ISD::CONCAT_VECTORS, llvm::ISD::CONVERGENCECTRL_GLUE, llvm::ISD::EXTRACT_SUBVECTOR, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getAnyExtOrTrunc(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::MVT::getIntegerVT(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::EVT::getSimpleVT(), llvm::EVT::getSizeInBits(), llvm::SITargetLowering::getSubtarget(), llvm::SelectionDAG::getTargetConstant(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::SelectionDAG::getVectorIdxConstant(), llvm::EVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::MVT::getVectorVT(), llvm::ISD::INTRINSIC_WO_CHAIN, llvm::EVT::isFloatingPoint(), llvm::AMDGPU::isLegalDPALU_DPPControl(), llvm::EVT::isVector(), llvm_unreachable, N, Operands, llvm::SmallVectorTemplateBase< T, bool >::push_back(), and llvm::MVT::SimpleTy.

lowerWaveReduce()

static MachineBasicBlock * lowerWaveReduce ( MachineInstr &  MI, MachineBasicBlock &  BB, const GCNSubtarget &  ST, unsigned  Opc  )

static

Definition at line 4920 of file SIISelLowering.cpp.

References Accumulator, llvm::MachineInstrBuilder::addImm(), llvm::MachineInstrBuilder::addMBB(), llvm::MachineInstrBuilder::addReg(), llvm::BuildMI(), DL, llvm::MachineBasicBlock::end(), llvm::MachineBasicBlock::getParent(), llvm::MachineFunction::getRegInfo(), I, MI, MRI, splitBlockForLoop(), TII, and TRI.

Referenced by llvm::SITargetLowering::EmitInstrWithCustomInserter().

matchPERM()

static SDValue matchPERM ( SDNode *  N, TargetLowering::DAGCombinerInfo &  DCI  )

static

Definition at line 12397 of file SIISelLowering.cpp.

References assert(), calculateByteProvider(), llvm::TargetLowering::DAGCombinerInfo::DAG, DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBitcastedAnyExtOrTrunc(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getDataLayout(), getDWordFromOffset(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), hasNon16BitAccesses(), llvm::DataLayout::isBigEndian(), llvm::EVT::isByteSized(), N, P, llvm::AMDGPUISD::PERM, llvm::SmallVectorTemplateBase< T, bool >::push_back(), and llvm::SmallVectorBase< Size_T >::size().

Referenced by llvm::SITargetLowering::PerformDAGCombine().

mayTailCallThisCC()

static bool mayTailCallThisCC ( CallingConv::ID  CC )

static

Return true if we might ever do TCO for calls with this calling convention.

Definition at line 3517 of file SIISelLowering.cpp.

References llvm::CallingConv::AMDGPU_Gfx, llvm::CallingConv::C, canGuaranteeTCO(), and CC.

memVTFromLoadIntrData()

static EVT memVTFromLoadIntrData ( const SITargetLowering &  TLI, const DataLayout &  DL, Type *  Ty, unsigned  MaxNumLanes  )

static

Definition at line 1140 of file SIISelLowering.cpp.

References assert(), DL, llvm::Type::getContext(), llvm::TargetLoweringBase::getValueType(), and llvm::EVT::getVectorVT().

Referenced by llvm::SITargetLowering::getTgtMemIntrinsic(), and memVTFromLoadIntrReturn().

memVTFromLoadIntrReturn()

static EVT memVTFromLoadIntrReturn ( const SITargetLowering &  TLI, const DataLayout &  DL, Type *  Ty, unsigned  MaxNumLanes  )

static

Definition at line 1156 of file SIISelLowering.cpp.

References assert(), DL, and memVTFromLoadIntrData().

Referenced by llvm::SITargetLowering::getTgtMemIntrinsic().

minMaxOpcToMin3Max3Opc()

static unsigned minMaxOpcToMin3Max3Opc ( unsigned  Opc )

static

Definition at line 13232 of file SIISelLowering.cpp.

References llvm::AMDGPUISD::FMAX3, llvm::ISD::FMAXIMUM, llvm::AMDGPUISD::FMAXIMUM3, llvm::ISD::FMAXNUM, llvm::ISD::FMAXNUM_IEEE, llvm::AMDGPUISD::FMIN3, llvm::ISD::FMINIMUM, llvm::AMDGPUISD::FMINIMUM3, llvm::ISD::FMINNUM, llvm::ISD::FMINNUM_IEEE, llvm_unreachable, llvm::ISD::SMAX, llvm::AMDGPUISD::SMAX3, llvm::ISD::SMIN, llvm::AMDGPUISD::SMIN3, llvm::ISD::UMAX, llvm::AMDGPUISD::UMAX3, llvm::ISD::UMIN, and llvm::AMDGPUISD::UMIN3.

packImage16bitOpsToDwords()

static void packImage16bitOpsToDwords ( SelectionDAG &  DAG, SDValue  Op, MVT  PackVectorVT, SmallVectorImpl< SDValue > &  PackedAddrs, unsigned  DimIdx, unsigned  EndIdx, unsigned  NumGradients  )

static

Definition at line 7974 of file SIISelLowering.cpp.

References Addr, llvm::ISD::ANY_EXTEND, DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getNode(), I, and llvm::SmallVectorTemplateBase< T, bool >::push_back().

padEltsToUndef()

static SDValue padEltsToUndef ( SelectionDAG &  DAG, const SDLoc &  DL, EVT  CastVT, SDValue  Src, int  ExtraElts  )

static

Definition at line 7862 of file SIISelLowering.cpp.

References DL, llvm::SelectionDAG::ExtractVectorElements(), llvm::SelectionDAG::getBuildVector(), llvm::EVT::getScalarType(), llvm::SelectionDAG::getUNDEF(), llvm::EVT::isVector(), and llvm::SmallVectorTemplateBase< T, bool >::push_back().

Referenced by constructRetValue().

parseTexFail()

static bool parseTexFail ( SDValue  TexFailCtrl, SelectionDAG &  DAG, SDValue *  TFE, SDValue *  LWE, bool &  IsTexFail  )

static

Definition at line 7956 of file SIISelLowering.cpp.

References DL, llvm::SDValue::getNode(), and llvm::SelectionDAG::getTargetConstant().

placeSources()

static void placeSources ( ByteProvider< SDValue > &  Src0, ByteProvider< SDValue > &  Src1, SmallVectorImpl< DotSrc > &  Src0s, SmallVectorImpl< DotSrc > &  Src1s, int  Step  )

static

Definition at line 14020 of file SIISelLowering.cpp.

References addPermMasks(), assert(), llvm::SmallVectorTemplateCommon< T, typename >::end(), llvm::find_if(), I, llvm::Match, llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::ByteProvider< ISelOp >::Src, and llvm::ByteProvider< ISelOp >::SrcOffset.

processPSInputArgs()

static void processPSInputArgs ( SmallVectorImpl< ISD::InputArg > &  Splits, CallingConv::ID  CallConv, ArrayRef< ISD::InputArg >  Ins, BitVector &  Skipped, FunctionType *  FType, SIMachineFunctionInfo *  Info  )

static

Definition at line 2192 of file SIISelLowering.cpp.

References llvm::CallingConv::AMDGPU_PS, assert(), llvm::ISD::InputArg::Flags, llvm::ISD::InputArg::getOrigArgIndex(), llvm::MVT::getScalarSizeInBits(), I, Info, llvm::ISD::ArgFlagsTy::isInReg(), llvm::ISD::ArgFlagsTy::isSplit(), llvm::ISD::ArgFlagsTy::isSplitEnd(), llvm::MVT::isVector(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::BitVector::set(), llvm::ISD::InputArg::Used, and llvm::ISD::InputArg::VT.

Referenced by llvm::SITargetLowering::LowerFormalArguments().

reservePrivateMemoryRegs()

static void reservePrivateMemoryRegs ( const TargetMachine &  TM, MachineFunction &  MF, const SIRegisterInfo &  TRI, SIMachineFunctionInfo &  Info  )

static

Definition at line 2664 of file SIISelLowering.cpp.

References assert(), llvm::Function::getCallingConv(), llvm::MachineFunction::getFrameInfo(), llvm::MachineFunction::getFunction(), llvm::MachineFunction::getRegInfo(), llvm::MachineFunction::getSubtarget(), llvm::MachineFrameInfo::hasCalls(), llvm::MachineFrameInfo::hasStackObjects(), Info, llvm::AMDGPU::isShader(), MRI, llvm::None, llvm::AMDGPUFunctionArgInfo::PRIVATE_SEGMENT_BUFFER, llvm::report_fatal_error(), and TRI.

Referenced by llvm::SITargetLowering::finalizeLowering().

resolveSources()

static SDValue resolveSources ( SelectionDAG &  DAG, SDLoc  SL, SmallVectorImpl< DotSrc > &  Srcs, bool  IsSigned, bool  IsAny  )

static

Definition at line 14096 of file SIISelLowering.cpp.

References addPermMasks(), assert(), llvm::SmallVectorTemplateCommon< T, typename >::begin(), llvm::SmallVectorTemplateCommon< T, typename >::end(), llvm::SelectionDAG::getConstant(), getDWordFromOffset(), llvm::SelectionDAG::getNode(), llvm::ISD::OR, llvm::AMDGPUISD::PERM, llvm::SmallVectorTemplateBase< T, bool >::push_back(), and llvm::SmallVectorBase< Size_T >::size().

selectSOffset()

static SDValue selectSOffset ( SDValue  SOffset, SelectionDAG &  DAG, const GCNSubtarget *  Subtarget  )

static

Definition at line 8898 of file SIISelLowering.cpp.

References llvm::SelectionDAG::getRegister(), llvm::GCNSubtarget::hasRestrictedSOffset(), and llvm::isNullConstant().

setM0ToIndexFromSGPR()

static void setM0ToIndexFromSGPR ( const SIInstrInfo *  TII, MachineRegisterInfo &  MRI, MachineInstr &  MI, int  Offset  )

static

Definition at line 4689 of file SIISelLowering.cpp.

References llvm::MachineInstrBuilder::add(), llvm::MachineInstrBuilder::addImm(), assert(), llvm::BuildMI(), DL, I, Idx, MBB, MI, llvm::Offset, and TII.

Referenced by emitIndirectDst(), and emitIndirectSrc().

splitBlockForLoop()

static std::pair< MachineBasicBlock *, MachineBasicBlock * > splitBlockForLoop ( MachineInstr &  MI, MachineBasicBlock &  MBB, bool  InstInLoop  )

static

Returns

{ LoopBody, Remainder }

Definition at line 4424 of file SIISelLowering.cpp.

References llvm::MachineBasicBlock::addSuccessor(), llvm::MachineBasicBlock::begin(), llvm::MachineFunction::CreateMachineBasicBlock(), llvm::MachineBasicBlock::end(), llvm::MachineBasicBlock::getParent(), I, llvm::MachineFunction::insert(), MBB, MBBI, MI, llvm::MachineBasicBlock::splice(), and llvm::MachineBasicBlock::transferSuccessorsAndUpdatePHIs().

Referenced by llvm::SITargetLowering::emitGWSMemViolTestLoop(), loadM0FromVGPR(), and lowerWaveReduce().

STATISTIC()

STATISTIC	(	NumTailCalls	,
		"Number of tail calls"
	)

strictFPExtFromF16()

static SDValue strictFPExtFromF16 ( SelectionDAG &  DAG, SDValue  Src  )

static

Return the source of an fp_extend from f16 to f32, or a converted FP constant.

Definition at line 13728 of file SIISelLowering.cpp.

References llvm::APFloat::convert(), llvm::ISD::FP_EXTEND, llvm::SelectionDAG::getConstantFP(), llvm::APFloatBase::IEEEhalf(), and llvm::APFloatBase::rmNearestTiesToEven.

SubIdx2Lane()

static unsigned SubIdx2Lane ( unsigned  Idx )

static

Helper function for adjustWritemask.

Definition at line 15093 of file SIISelLowering.cpp.

References Idx.

supportsMin3Max3()

static bool supportsMin3Max3 ( const GCNSubtarget &  Subtarget, unsigned  Opc, EVT  VT  )

static

Returns

true if the subtarget supports minimum3 and maximum3 with the given base min/max opcode Opc for type VT.

Definition at line 13359 of file SIISelLowering.cpp.

References llvm::AMDGPUISD::FMAX_LEGACY, llvm::ISD::FMAXIMUM, llvm::ISD::FMAXNUM, llvm::ISD::FMAXNUM_IEEE, llvm::AMDGPUISD::FMIN_LEGACY, llvm::ISD::FMINIMUM, llvm::ISD::FMINNUM, llvm::ISD::FMINNUM_IEEE, llvm::GCNSubtarget::hasMin3Max3_16(), llvm::GCNSubtarget::hasMinimum3Maximum3F16(), llvm::GCNSubtarget::hasMinimum3Maximum3F32(), llvm_unreachable, llvm::ISD::SMAX, llvm::ISD::SMIN, llvm::ISD::UMAX, and llvm::ISD::UMIN.

vectorEltWillFoldAway()

static bool vectorEltWillFoldAway ( SDValue  Op )

static

Definition at line 13158 of file SIISelLowering.cpp.

Variable Documentation

DisableLoopAlignment

cl::opt< bool > DisableLoopAlignment("amdgpu-disable-loop-alignment", cl::desc("Do not align and prefetch loops"), cl::init(false)) ( "amdgpu-disable-loop-alignment"  , cl::desc("Do not align and prefetch loops")  , cl::init(false)    )

static

Referenced by llvm::SITargetLowering::getPrefLoopAlignment().

UseDivergentRegisterIndexing

cl::opt< bool > UseDivergentRegisterIndexing("amdgpu-use-divergent-register-indexing", cl::Hidden, cl::desc("Use indirect register addressing for divergent indexes"), cl::init(false)) ( "amdgpu-use-divergent-register-indexing"  , cl::Hidden  , cl::desc("Use indirect register addressing for divergent indexes")  , cl::init(false)    )

static

Referenced by llvm::SITargetLowering::shouldExpandVectorDynExt().