LLVM  6.0.0svn
Macros | Functions | Variables
SIISelLowering.cpp File Reference

Custom DAG lowering for SI. More...

#include "SIISelLowering.h"
#include "AMDGPU.h"
#include "AMDGPUIntrinsicInfo.h"
#include "AMDGPUSubtarget.h"
#include "AMDGPUTargetMachine.h"
#include "SIDefines.h"
#include "SIInstrInfo.h"
#include "SIMachineFunctionInfo.h"
#include "SIRegisterInfo.h"
#include "Utils/AMDGPUBaseInfo.h"
#include "llvm/ADT/APFloat.h"
#include "llvm/ADT/APInt.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/BitVector.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/ADT/StringSwitch.h"
#include "llvm/ADT/Twine.h"
#include "llvm/CodeGen/Analysis.h"
#include "llvm/CodeGen/CallingConvLower.h"
#include "llvm/CodeGen/DAGCombine.h"
#include "llvm/CodeGen/ISDOpcodes.h"
#include "llvm/CodeGen/MachineBasicBlock.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineInstr.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
#include "llvm/CodeGen/MachineMemOperand.h"
#include "llvm/CodeGen/MachineModuleInfo.h"
#include "llvm/CodeGen/MachineOperand.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/CodeGen/MachineValueType.h"
#include "llvm/CodeGen/SelectionDAG.h"
#include "llvm/CodeGen/SelectionDAGNodes.h"
#include "llvm/CodeGen/ValueTypes.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/DebugLoc.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/DiagnosticInfo.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/GlobalValue.h"
#include "llvm/IR/InstrTypes.h"
#include "llvm/IR/Instruction.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/Type.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/CodeGen.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Compiler.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/KnownBits.h"
#include "llvm/Support/MathExtras.h"
#include "llvm/Target/TargetCallingConv.h"
#include "llvm/Target/TargetOptions.h"
#include "llvm/Target/TargetRegisterInfo.h"
#include <cassert>
#include <cmath>
#include <cstdint>
#include <iterator>
#include <tuple>
#include <utility>
#include <vector>

Go to the source code of this file.

Macros

#define DEBUG_TYPE   "si-lower"
 

Functions

 STATISTIC (NumTailCalls, "Number of tail calls")
 
static unsigned findFirstFreeSGPR (CCState &CCInfo)
 
static bool isFlatGlobalAddrSpace (unsigned AS, AMDGPUAS AMDGPUASI)
 
static void processShaderInputArgs (SmallVectorImpl< ISD::InputArg > &Splits, CallingConv::ID CallConv, ArrayRef< ISD::InputArg > Ins, BitVector &Skipped, FunctionType *FType, SIMachineFunctionInfo *Info)
 
static void allocateSpecialEntryInputVGPRs (CCState &CCInfo, MachineFunction &MF, const SIRegisterInfo &TRI, SIMachineFunctionInfo &Info)
 
static ArgDescriptor allocateVGPR32Input (CCState &CCInfo)
 
static ArgDescriptor allocateSGPR32InputImpl (CCState &CCInfo, const TargetRegisterClass *RC, unsigned NumArgRegs)
 
static ArgDescriptor allocateSGPR32Input (CCState &CCInfo)
 
static ArgDescriptor allocateSGPR64Input (CCState &CCInfo)
 
static void allocateSpecialInputVGPRs (CCState &CCInfo, MachineFunction &MF, const SIRegisterInfo &TRI, SIMachineFunctionInfo &Info)
 
static void allocateSpecialInputSGPRs (CCState &CCInfo, MachineFunction &MF, const SIRegisterInfo &TRI, SIMachineFunctionInfo &Info)
 
static void allocateHSAUserSGPRs (CCState &CCInfo, MachineFunction &MF, const SIRegisterInfo &TRI, SIMachineFunctionInfo &Info)
 
static void allocateSystemSGPRs (CCState &CCInfo, MachineFunction &MF, SIMachineFunctionInfo &Info, CallingConv::ID CallConv, bool IsShader)
 
static void reservePrivateMemoryRegs (const TargetMachine &TM, MachineFunction &MF, const SIRegisterInfo &TRI, SIMachineFunctionInfo &Info)
 
static bool canGuaranteeTCO (CallingConv::ID CC)
 
static bool mayTailCallThisCC (CallingConv::ID CC)
 Return true if we might ever do TCO for calls with this calling convention. More...
 
static MachineBasicBlock::iterator emitLoadM0FromVGPRLoop (const SIInstrInfo *TII, MachineRegisterInfo &MRI, MachineBasicBlock &OrigBB, MachineBasicBlock &LoopBB, const DebugLoc &DL, const MachineOperand &IdxReg, unsigned InitReg, unsigned ResultReg, unsigned PhiReg, unsigned InitSaveExecReg, int Offset, bool UseGPRIdxMode)
 
static MachineBasicBlock::iterator loadM0FromVGPR (const SIInstrInfo *TII, MachineBasicBlock &MBB, MachineInstr &MI, unsigned InitResultReg, unsigned PhiReg, int Offset, bool UseGPRIdxMode)
 
static std::pair< unsigned, int > computeIndirectRegAndOffset (const SIRegisterInfo &TRI, const TargetRegisterClass *SuperRC, unsigned VecReg, int Offset)
 
static bool setM0ToIndexFromSGPR (const SIInstrInfo *TII, MachineRegisterInfo &MRI, MachineInstr &MI, int Offset, bool UseGPRIdxMode, bool IsIndirectSrc)
 
static MachineBasicBlockemitIndirectSrc (MachineInstr &MI, MachineBasicBlock &MBB, const SISubtarget &ST)
 
static unsigned getMOVRELDPseudo (const SIRegisterInfo &TRI, const TargetRegisterClass *VecRC)
 
static MachineBasicBlockemitIndirectDst (MachineInstr &MI, MachineBasicBlock &MBB, const SISubtarget &ST)
 
static SDNodefindUser (SDValue Value, unsigned Opcode)
 Helper function for LowerBRCOND. More...
 
static SDValue buildPCRelGlobalAddress (SelectionDAG &DAG, const GlobalValue *GV, const SDLoc &DL, unsigned Offset, EVT PtrVT, unsigned GAFlags=SIInstrInfo::MO_NONE)
 
static SDValue emitNonHSAIntrinsicError (SelectionDAG &DAG, const SDLoc &DL, EVT VT)
 
static SDValue emitRemovedIntrinsicError (SelectionDAG &DAG, const SDLoc &DL, EVT VT)
 
static SDValue getFPBinOp (SelectionDAG &DAG, unsigned Opcode, const SDLoc &SL, EVT VT, SDValue A, SDValue B, SDValue GlueChain)
 
static SDValue getFPTernOp (SelectionDAG &DAG, unsigned Opcode, const SDLoc &SL, EVT VT, SDValue A, SDValue B, SDValue C, SDValue GlueChain)
 
static bool canFoldOffset (unsigned OffsetSize, unsigned AS, const SISubtarget &STI)
 Return true if the given offset Size in bytes can be folded into the immediate offsets of a memory instruction for the given address space. More...
 
static bool bitOpWithConstantIsReducible (unsigned Opc, uint32_t Val)
 
static bool isBoolSGPR (SDValue V)
 
static bool fp16SrcZerosHighBits (unsigned Opc)
 
static bool isKnownNeverSNan (SelectionDAG &DAG, SDValue Op)
 
static bool isCanonicalized (SelectionDAG &DAG, SDValue Op, const SISubtarget *ST, unsigned MaxDepth=5)
 
static unsigned minMaxOpcToMin3Max3Opc (unsigned Opc)
 
static ConstantFPSDNodegetSplatConstantFP (SDValue Op)
 
static bool isClampZeroToOne (SDValue A, SDValue B)
 
static bool convertBuildVectorCastElt (SelectionDAG &DAG, SDValue &Lo, SDValue &Hi)
 
static unsigned SubIdx2Lane (unsigned Idx)
 Helper function for adjustWritemask. More...
 
static bool isFrameIndexOp (SDValue Op)
 
static SDValue buildSMovImm32 (SelectionDAG &DAG, const SDLoc &DL, uint64_t Val)
 

Variables

static cl::opt< boolEnableVGPRIndexMode ("amdgpu-vgpr-index-mode", cl::desc("Use GPR indexing mode instead of movrel for vector indexing"), cl::init(false))
 

Detailed Description

Custom DAG lowering for SI.

Definition in file SIISelLowering.cpp.

Macro Definition Documentation

◆ DEBUG_TYPE

#define DEBUG_TYPE   "si-lower"

Definition at line 88 of file SIISelLowering.cpp.

Function Documentation

◆ allocateHSAUserSGPRs()

static void allocateHSAUserSGPRs ( CCState CCInfo,
MachineFunction MF,
const SIRegisterInfo TRI,
SIMachineFunctionInfo Info 
)
static

◆ allocateSGPR32Input()

static ArgDescriptor allocateSGPR32Input ( CCState CCInfo)
static

Definition at line 1150 of file SIISelLowering.cpp.

References allocateSGPR32InputImpl().

Referenced by allocateSpecialInputSGPRs().

◆ allocateSGPR32InputImpl()

static ArgDescriptor allocateSGPR32InputImpl ( CCState CCInfo,
const TargetRegisterClass RC,
unsigned  NumArgRegs 
)
static

◆ allocateSGPR64Input()

static ArgDescriptor allocateSGPR64Input ( CCState CCInfo)
static

Definition at line 1154 of file SIISelLowering.cpp.

References allocateSGPR32InputImpl().

Referenced by allocateSpecialInputSGPRs().

◆ allocateSpecialEntryInputVGPRs()

static void allocateSpecialEntryInputVGPRs ( CCState CCInfo,
MachineFunction MF,
const SIRegisterInfo TRI,
SIMachineFunctionInfo Info 
)
static

◆ allocateSpecialInputSGPRs()

static void allocateSpecialInputSGPRs ( CCState CCInfo,
MachineFunction MF,
const SIRegisterInfo TRI,
SIMachineFunctionInfo Info 
)
static

◆ allocateSpecialInputVGPRs()

static void allocateSpecialInputVGPRs ( CCState CCInfo,
MachineFunction MF,
const SIRegisterInfo TRI,
SIMachineFunctionInfo Info 
)
static

◆ allocateSystemSGPRs()

static void allocateSystemSGPRs ( CCState CCInfo,
MachineFunction MF,
SIMachineFunctionInfo Info,
CallingConv::ID  CallConv,
bool  IsShader 
)
static

◆ allocateVGPR32Input()

static ArgDescriptor allocateVGPR32Input ( CCState CCInfo)
static

◆ bitOpWithConstantIsReducible()

static bool bitOpWithConstantIsReducible ( unsigned  Opc,
uint32_t  Val 
)
static

◆ buildPCRelGlobalAddress()

static SDValue buildPCRelGlobalAddress ( SelectionDAG DAG,
const GlobalValue GV,
const SDLoc DL,
unsigned  Offset,
EVT  PtrVT,
unsigned  GAFlags = SIInstrInfo::MO_NONE 
)
static

◆ buildSMovImm32()

static SDValue buildSMovImm32 ( SelectionDAG DAG,
const SDLoc DL,
uint64_t  Val 
)
static

◆ canFoldOffset()

static bool canFoldOffset ( unsigned  OffsetSize,
unsigned  AS,
const SISubtarget STI 
)
static

◆ canGuaranteeTCO()

static bool canGuaranteeTCO ( CallingConv::ID  CC)
static

◆ computeIndirectRegAndOffset()

static std::pair<unsigned, int> computeIndirectRegAndOffset ( const SIRegisterInfo TRI,
const TargetRegisterClass SuperRC,
unsigned  VecReg,
int  Offset 
)
static

Definition at line 2626 of file SIISelLowering.cpp.

Referenced by emitIndirectDst(), and emitIndirectSrc().

◆ convertBuildVectorCastElt()

static bool convertBuildVectorCastElt ( SelectionDAG DAG,
SDValue Lo,
SDValue Hi 
)
static

Definition at line 5895 of file SIISelLowering.cpp.

References llvm::ISD::ADD, llvm::ISD::ADDCARRY, llvm::AfterLegalizeDAG, llvm::TargetOptions::AllowFPOpFusion, llvm::ISD::ANY_EXTEND, llvm::AMDGPU::HSAMD::Kernel::Key::Args, assert(), llvm::ISD::BITCAST, C, llvm::AMDGPUISD::CVT_F32_UBYTE0, llvm::dyn_cast(), llvm::MVT::f16, llvm::MVT::f32, llvm::MVT::f64, llvm::ISD::FABS, llvm::ISD::FADD, llvm::FPOpFusion::Fast, llvm::ISD::FMA, llvm::ISD::FMAD, llvm::ISD::FNEG, llvm::AMDGPUISD::FP_CLASS, llvm::APInt::getBitsSet(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getConstantFP(), llvm::SDNode::getFlags(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::ISD::getSetCCSwappedOperands(), llvm::SelectionDAG::getTarget(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::ConstantFPSDNode::getValueAPF(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::SelectionDAG::getVTList(), llvm::SDNode::getVTList(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::AMDGPUSubtarget::has16BitInsts(), llvm::AMDGPUSubtarget::hasFP16Denormals(), llvm::AMDGPUSubtarget::hasFP32Denormals(), llvm::SDNodeFlags::hasUnsafeAlgebra(), llvm::MipsISD::Hi, llvm::MVT::i1, llvm::MVT::i32, llvm::ConstantSDNode::isAllOnesValue(), isBoolSGPR(), llvm::SITargetLowering::isFMAFasterThanFMulAndFAdd(), llvm::APFloat::isInfinity(), llvm::APFloat::isNegative(), llvm::ConstantSDNode::isNullValue(), llvm::TargetLoweringBase::isTypeLegal(), llvm::SDValue::isUndef(), llvm::EVT::isVector(), llvm::MipsISD::Lo, llvm::BitmaskEnumDetail::Mask(), llvm::SIInstrFlags::N_INFINITY, llvm::TargetMachine::Options, llvm::SIInstrFlags::P_INFINITY, llvm::ISD::SETEQ, llvm::ISD::SETGE, llvm::ISD::SETGT, llvm::ISD::SETLE, llvm::ISD::SETLT, llvm::ISD::SETNE, llvm::ISD::SETOEQ, llvm::ISD::SETUGE, llvm::ISD::SETUGT, llvm::ISD::SETULE, llvm::ISD::SETULT, llvm::ISD::SIGN_EXTEND, llvm::ISD::SRL, llvm::ISD::SUB, llvm::ISD::SUBCARRY, llvm::AMDGPUTargetLowering::Subtarget, std::swap(), llvm::TargetOptions::UnsafeFPMath, llvm::MVT::v2f16, llvm::MVT::v2i16, llvm::ISD::XOR, and llvm::ISD::ZERO_EXTEND.

◆ emitIndirectDst()

static MachineBasicBlock* emitIndirectDst ( MachineInstr MI,
MachineBasicBlock MBB,
const SISubtarget ST 
)
static

◆ emitIndirectSrc()

static MachineBasicBlock* emitIndirectSrc ( MachineInstr MI,
MachineBasicBlock MBB,
const SISubtarget ST 
)
static

◆ emitLoadM0FromVGPRLoop()

static MachineBasicBlock::iterator emitLoadM0FromVGPRLoop ( const SIInstrInfo TII,
MachineRegisterInfo MRI,
MachineBasicBlock OrigBB,
MachineBasicBlock LoopBB,
const DebugLoc DL,
const MachineOperand IdxReg,
unsigned  InitReg,
unsigned  ResultReg,
unsigned  PhiReg,
unsigned  InitSaveExecReg,
int  Offset,
bool  UseGPRIdxMode 
)
static

◆ emitNonHSAIntrinsicError()

static SDValue emitNonHSAIntrinsicError ( SelectionDAG DAG,
const SDLoc DL,
EVT  VT 
)
static

◆ emitRemovedIntrinsicError()

static SDValue emitRemovedIntrinsicError ( SelectionDAG DAG,
const SDLoc DL,
EVT  VT 
)
static

Definition at line 3844 of file SIISelLowering.cpp.

References llvm::TargetLoweringBase::allowsMemoryAccess(), llvm::AMDGPUTargetLowering::AMDGPUASI, AS, assert(), llvm::AMDGPUISD::ATOMIC_DEC, llvm::AMDGPUISD::ATOMIC_INC, llvm::AMDGPUISD::BFE_I32, llvm::AMDGPUISD::BFE_U32, llvm::ISD::BITCAST, llvm::AMDGPUISD::BUFFER_LOAD, llvm::AMDGPUISD::BUFFER_LOAD_FORMAT, llvm::EVT::changeTypeToInteger(), AMDGPUAS::CONSTANT_ADDRESS, llvm::SITargetLowering::copyToM0(), llvm::AMDGPUISD::COS_HW, llvm::AMDGPUISD::CVT_PKRTZ_F16_F32, llvm::LLVMContext::diagnose(), llvm::AMDGPUFunctionArgInfo::DISPATCH_ID, llvm::AMDGPUFunctionArgInfo::DISPATCH_PTR, llvm::AMDGPUISD::DIV_FIXUP, llvm::AMDGPUISD::DIV_FMAS, llvm::AMDGPUISD::DIV_SCALE, llvm::dyn_cast(), emitNonHSAIntrinsicError(), llvm::TargetLowering::expandUnalignedLoad(), llvm::AMDGPUISD::EXPORT, llvm::AMDGPUISD::EXPORT_DONE, llvm::ISD::EXTLOAD, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::MVT::f16, llvm::MVT::f32, llvm::AMDGPUISD::FFBH_I32, AMDGPUAS::FLAT_ADDRESS, llvm::FloatToBits(), llvm::ISD::FMAXNUM, llvm::AMDGPUISD::FMED3, llvm::ISD::FMINNUM, llvm::ISD::FMUL, llvm::AMDGPUISD::FMUL_LEGACY, llvm::ISD::FNEG, llvm::AMDGPUISD::FP_CLASS, llvm::AMDGPUISD::FRACT, llvm::ISD::FSQRT, llvm::MemSDNode::getAddressSpace(), llvm::MemSDNode::getAlignment(), llvm::LoadSDNode::getBasePtr(), llvm::SIMachineFunctionInfo::getBufferPSV(), llvm::SelectionDAG::getBuildVector(), llvm::MemSDNode::getChain(), llvm::SelectionDAG::getCondCode(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getConstantFP(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::SDLoc::getDebugLoc(), llvm::SelectionDAG::getEntryNode(), llvm::LoadSDNode::getExtensionType(), llvm::SelectionDAG::getExtLoad(), llvm::getFCmpCondCode(), llvm::SDNode::getFlags(), llvm::AMDGPUSubtarget::getFlatWorkGroupSizes(), llvm::Type::getFltSemantics(), llvm::MachineFunction::getFunction(), llvm::AMDGPUSubtarget::getGeneration(), llvm::getICmpCondCode(), llvm::MachineFunction::getInfo(), llvm::APFloat::getLargest(), llvm::SelectionDAG::getMachineFunction(), llvm::MachineFunction::getMachineMemOperand(), llvm::SelectionDAG::getMachineNode(), llvm::AMDGPUSubtarget::getMaxPrivateElementSize(), llvm::SelectionDAG::getMemIntrinsicNode(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getMemoryVT(), llvm::SelectionDAG::getMergeValues(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::AMDGPUSubtarget::getScalarizeGlobalBehavior(), llvm::SelectionDAG::getSelect(), llvm::EVT::getSizeInBits(), llvm::EVT::getStoreSize(), llvm::SITargetLowering::getSubtarget(), llvm::MachineFunction::getSubtarget(), llvm::SelectionDAG::getTarget(), llvm::SelectionDAG::getTargetConstant(), llvm::TargetLoweringBase::getTargetMachine(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::SDNode::getVTList(), llvm::AMDGPUSubtarget::getWavefrontSize(), llvm::ConstantSDNode::getZExtValue(), AMDGPUAS::GLOBAL_ADDRESS, llvm::SI::KernelInputOffsets::GLOBAL_SIZE_X, llvm::SI::KernelInputOffsets::GLOBAL_SIZE_Y, llvm::SI::KernelInputOffsets::GLOBAL_SIZE_Z, llvm::SDNodeFlags::hasAllowReciprocal(), llvm::SIMachineFunctionInfo::hasFlatScratchInit(), llvm::AMDGPUSubtarget::hasFP32Denormals(), llvm::SDNodeFlags::hasUnsafeAlgebra(), llvm::MipsISD::Hi, llvm::MVT::i1, llvm::MVT::i16, llvm::MVT::i32, llvm::MVT::i64, llvm::MVT::i8, llvm::AMDGPUFunctionArgInfo::IMPLICIT_ARG_PTR, llvm::AMDGPUFunctionArgInfo::IMPLICIT_BUFFER_PTR, llvm::AMDGPUISD::INIT_EXEC, llvm::AMDGPUISD::INIT_EXEC_FROM_INPUT, llvm::AMDGPUISD::INTERP_MOV, llvm::AMDGPUISD::INTERP_P1, llvm::AMDGPUISD::INTERP_P2, llvm::ConstantSDNode::isAllOnesValue(), llvm::AMDGPUSubtarget::isAmdCodeObjectV2(), llvm::AMDGPUSubtarget::isAmdHsaOS(), llvm::AMDGPUMachineFunction::isEntryFunction(), llvm::SITargetLowering::isMemOpHasNoClobberedMemOperand(), llvm::SITargetLowering::isMemOpUniform(), llvm::ConstantSDNode::isNullValue(), llvm::ConstantSDNode::isOne(), llvm::TargetLoweringBase::isTypeLegal(), llvm::EVT::isVector(), llvm::MemSDNode::isVolatile(), llvm::AMDGPUFunctionArgInfo::KERNARG_SEGMENT_PTR, llvm::AMDGPUISD::KILL, llvm::AMDGPUISD::LDEXP, llvm_unreachable, llvm::MipsISD::Lo, llvm::SPII::Load, llvm::AMDGPUISD::LOAD_CONSTANT, llvm::AMDGPUTargetLowering::loadInputValue(), AMDGPUAS::LOCAL_ADDRESS, llvm::SI::KernelInputOffsets::LOCAL_SIZE_X, llvm::SI::KernelInputOffsets::LOCAL_SIZE_Y, llvm::SI::KernelInputOffsets::LOCAL_SIZE_Z, llvm::MachineMemOperand::MODereferenceable, llvm::MachineMemOperand::MOInvariant, llvm::MachineMemOperand::MOLoad, llvm::MachineMemOperand::MOStore, llvm::SI::KernelInputOffsets::NGROUPS_X, llvm::SI::KernelInputOffsets::NGROUPS_Y, llvm::SI::KernelInputOffsets::NGROUPS_Z, llvm::ISD::NON_EXTLOAD, llvm::CodeGenOpt::None, llvm::TargetMachine::Options, llvm::MVT::Other, AMDGPUAS::PRIVATE_ADDRESS, llvm::AMDGPUFunctionArgInfo::QUEUE_PTR, llvm::AMDGPUISD::RCP, llvm::AMDGPUISD::RCP_LEGACY, llvm::AMDGPUISD::RSQ, llvm::AMDGPUISD::RSQ_CLAMP, llvm::AMDGPUISD::RSQ_LEGACY, llvm::TargetLowering::scalarizeVectorLoad(), second, llvm::AMDGPUISD::SENDMSG, llvm::AMDGPUISD::SENDMSGHALT, llvm::AMDGPUISD::SETCC, llvm::AMDGPUISD::SIN_HW, llvm::AMDGPUTargetLowering::SplitVectorLoad(), llvm::ARM_MB::ST, llvm::AMDGPUTargetLowering::Subtarget, llvm::AMDGPUISD::TBUFFER_LOAD_FORMAT, llvm::AMDGPUISD::TBUFFER_STORE_FORMAT, llvm::AMDGPUISD::TBUFFER_STORE_FORMAT_X3, llvm::AMDGPUISD::TRIG_PREOP, llvm::ISD::TRUNCATE, llvm::RegState::Undef, llvm::TargetOptions::UnsafeFPMath, llvm::MVT::v2i32, llvm::AMDGPUSubtarget::VOLCANIC_ISLANDS, llvm::AMDGPUFunctionArgInfo::WORKGROUP_ID_X, llvm::AMDGPUFunctionArgInfo::WORKGROUP_ID_Y, llvm::AMDGPUFunctionArgInfo::WORKGROUP_ID_Z, and llvm::SIInstrFlags::WQM.

◆ findFirstFreeSGPR()

static unsigned findFirstFreeSGPR ( CCState CCInfo)
static

◆ findUser()

static SDNode* findUser ( SDValue  Value,
unsigned  Opcode 
)
static

Helper function for LowerBRCOND.

Definition at line 3276 of file SIISelLowering.cpp.

References llvm::ISD::ADD, llvm::AfterLegalizeVectorOps, llvm::AMDGPUTargetLowering::AMDGPUASI, llvm::ISD::AND, llvm::SmallVectorImpl< T >::append(), AS, assert(), llvm::AMDGPUISD::BFI, llvm::AMDGPUISD::BFM, llvm::ISD::BITCAST, llvm::EVT::bitsLE(), llvm::EVT::bitsLT(), llvm::ISD::BR, llvm::ISD::BRCOND, llvm::ISD::BUILD_VECTOR, AMDGPUAS::CONSTANT_ADDRESS, llvm::ISD::CopyToReg, llvm::MachineFrameInfo::CreateFixedObject(), llvm::AMDGPUTargetLowering::CreateLiveInRegister(), llvm::ISD::DEBUGTRAP, llvm::LLVMContext::diagnose(), llvm::DS_Warning, E, llvm::AMDGPUISD::ELSE, llvm::AMDGPUISD::ENDPGM, llvm::MVT::f16, llvm::MVT::f64, AMDGPUAS::FLAT_ADDRESS, llvm::ISD::FP_EXTEND, llvm::ISD::FP_TO_FP16, llvm::ISD::FTRUNC, llvm::PointerType::get(), llvm::UndefValue::get(), llvm::PointerType::getAddressSpace(), llvm::SelectionDAG::getConstant(), llvm::SDNode::getConstantOperandVal(), llvm::Function::getContext(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getCopyToReg(), llvm::SDValue::getDebugLoc(), llvm::SDLoc::getDebugLoc(), llvm::AddrSpaceCastSDNode::getDestAddressSpace(), llvm::MachineFunction::getFrameInfo(), llvm::MachineFunction::getFunction(), llvm::MachineFunction::getInfo(), llvm::Type::getInt8Ty(), llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getMachineNode(), llvm::SelectionDAG::getMergeValues(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNOT(), llvm::AMDGPUTargetMachine::getNullPointerValue(), llvm::SDNode::getNumValues(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::GlobalValue::getParent(), llvm::SIMachineFunctionInfo::getQueuePtrUserSGPR(), llvm::SelectionDAG::getRegister(), llvm::SelectionDAG::getSetCC(), llvm::AddrSpaceCastSDNode::getSrcAddressSpace(), llvm::SelectionDAG::getTargetConstant(), llvm::TargetLoweringBase::getTargetMachine(), llvm::TargetMachine::getTargetTriple(), llvm::AMDGPUSubtarget::getTrapHandlerAbi(), llvm::GlobalValue::getType(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::SelectionDAG::getVTList(), llvm::SDNode::getVTList(), AMDGPUAS::GLOBAL_ADDRESS, llvm::AMDGPUSubtarget::hasApertureRegs(), I, llvm::MVT::i1, llvm::MVT::i16, llvm::MVT::i32, llvm::MVT::i64, llvm::AMDGPU::Hwreg::ID_MEM_BASES, llvm::AMDGPU::Hwreg::ID_SHIFT_, llvm::AMDGPUISD::IF, llvm::ISD::INTRINSIC_VOID, llvm::ISD::INTRINSIC_W_CHAIN, llvm::AMDGPUSubtarget::isTrapHandlerEnabled(), llvm_unreachable, AMDGPUAS::LOCAL_ADDRESS, llvm::AMDGPUISD::LOOP, llvm::MinAlign(), llvm::MachineMemOperand::MODereferenceable, llvm::MachineMemOperand::MOInvariant, llvm::AMDGPU::Hwreg::OFFSET_SHIFT_, llvm::AMDGPU::Hwreg::OFFSET_SRC_PRIVATE_BASE, llvm::AMDGPU::Hwreg::OFFSET_SRC_SHARED_BASE, llvm::SDNode::op_begin(), llvm::SDNode::op_end(), llvm::ISD::OR, llvm::MVT::Other, AMDGPUAS::PRIVATE_ADDRESS, llvm::SmallVectorTemplateBase< T, isPodLike< T >::value >::push_back(), llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), llvm::SelectionDAG::ReplaceAllUsesWith(), llvm::ISD::SELECT, llvm::ISD::SETCC, llvm::SIMachineFunctionInfo::setDebuggerWorkGroupIDStackObjectIndex(), llvm::SIMachineFunctionInfo::setDebuggerWorkItemIDStackObjectIndex(), llvm::ISD::SETNE, llvm::ISD::SHL, llvm::TargetMachine::shouldAssumeDSOLocal(), llvm::AMDGPU::shouldEmitConstantsToTextSection(), llvm::ISD::SRL, llvm::AMDGPUTargetLowering::Subtarget, llvm::AMDGPUISD::TRAP, llvm::AMDGPUSubtarget::TrapHandlerAbiHsa, llvm::AMDGPUSubtarget::TrapIDLLVMDebugTrap, llvm::AMDGPUSubtarget::TrapIDLLVMTrap, llvm::ISD::TRUNCATE, llvm::SDNode::use_begin(), llvm::SDNode::use_end(), llvm::MVT::v2i32, llvm::SDNode::value_begin(), llvm::SDNode::value_end(), llvm::AMDGPU::Hwreg::WIDTH_M1_SHIFT_, llvm::AMDGPU::Hwreg::WIDTH_M1_SRC_PRIVATE_BASE, llvm::AMDGPU::Hwreg::WIDTH_M1_SRC_SHARED_BASE, and llvm::ISD::ZERO_EXTEND.

◆ fp16SrcZerosHighBits()

static bool fp16SrcZerosHighBits ( unsigned  Opc)
static

◆ getFPBinOp()

static SDValue getFPBinOp ( SelectionDAG DAG,
unsigned  Opcode,
const SDLoc SL,
EVT  VT,
SDValue  A,
SDValue  B,
SDValue  GlueChain 
)
static

◆ getFPTernOp()

static SDValue getFPTernOp ( SelectionDAG DAG,
unsigned  Opcode,
const SDLoc SL,
EVT  VT,
SDValue  A,
SDValue  B,
SDValue  C,
SDValue  GlueChain 
)
static

Definition at line 4678 of file SIISelLowering.cpp.

References llvm::TargetLoweringBase::allowsMemoryAccess(), llvm::AMDGPUTargetLowering::AMDGPUASI, AS, assert(), llvm::AMDGPUISD::ATOMIC_CMP_SWAP, B, llvm::ISD::BITCAST, llvm::BitsToFloat(), C, llvm::AMDGPUISD::COS_HW, llvm::AMDGPUISD::CVT_F32_UBYTE0, llvm::AMDGPUISD::DIV_FIXUP, llvm::AMDGPUISD::DIV_FMAS, llvm::AMDGPUISD::DIV_SCALE, llvm::TargetLowering::expandUnalignedStore(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::MVT::f16, llvm::MVT::f32, llvm::MVT::f64, llvm::ISD::FABS, llvm::ISD::FCOS, AMDGPUAS::FLAT_ADDRESS, llvm::ISD::FMA, llvm::AMDGPUISD::FMA_W_CHAIN, llvm::ISD::FMUL, llvm::ISD::FNEG, FP_DENORM_FLUSH_IN_FLUSH_OUT, FP_DENORM_FLUSH_NONE, llvm::ISD::FP_EXTEND, llvm::ISD::FP_ROUND, llvm::AMDGPUISD::FRACT, llvm::ISD::FSIN, llvm::MemSDNode::getAddressSpace(), llvm::MemSDNode::getAlignment(), llvm::StoreSDNode::getBasePtr(), llvm::SelectionDAG::getBuildVector(), llvm::MemSDNode::getChain(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getConstantFP(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getEntryNode(), getFPBinOp(), llvm::AMDGPUSubtarget::getGeneration(), llvm::APInt::getHighBitsSet(), llvm::MachineFunction::getInfo(), llvm::SelectionDAG::getMachineFunction(), llvm::AMDGPUSubtarget::getMaxPrivateElementSize(), llvm::SelectionDAG::getMemIntrinsicNode(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getMemoryVT(), llvm::SelectionDAG::getMergeValues(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDNode::getNumValues(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getRoot(), llvm::EVT::getScalarType(), llvm::SelectionDAG::getSetCC(), llvm::SITargetLowering::getSetCCResultType(), llvm::SelectionDAG::getSExtOrTrunc(), llvm::EVT::getSimpleVT(), llvm::SelectionDAG::getTarget(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getTruncStore(), llvm::SDValue::getValue(), llvm::StoreSDNode::getValue(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::MVT::getVectorVT(), llvm::SelectionDAG::getVTList(), llvm::SDNode::getVTList(), AMDGPUAS::GLOBAL_ADDRESS, llvm::MVT::Glue, llvm::SIMachineFunctionInfo::hasFlatScratchInit(), llvm::AMDGPUSubtarget::hasFP32Denormals(), llvm::MipsISD::Hi, llvm::MVT::i1, llvm::MVT::i16, llvm::MVT::i32, llvm::AMDGPU::Hwreg::ID_MODE, llvm::AtomicSDNode::isCompareAndSwap(), isFlatGlobalAddrSpace(), llvm_unreachable, AMDGPUAS::LOCAL_ADDRESS, llvm::SelectionDAG::MaskedValueIsZero(), N, llvm::AMDGPU::Hwreg::OFFSET_SHIFT_, llvm::TargetMachine::Options, llvm::MVT::Other, AMDGPUAS::PRIVATE_ADDRESS, r0(), r1(), r2(), r3(), llvm::AMDGPUISD::RCP, llvm::TargetLowering::scalarizeVectorStore(), llvm::ISD::SELECT, llvm::ISD::SETEQ, llvm::ISD::SETOGT, llvm::AMDGPUISD::SETREG, llvm::SelectionDAG::setRoot(), llvm::AMDGPUISD::SIN_HW, llvm::AMDGPUSubtarget::SOUTHERN_ISLANDS, llvm::AMDGPUTargetLowering::SplitVectorStore(), llvm::SPII::Store, llvm::AMDGPUTargetLowering::Subtarget, llvm::ISD::TokenFactor, llvm::TargetOptions::UnsafeFPMath, llvm::MVT::v2i32, llvm::AMDGPU::Hwreg::WIDTH_M1_SHIFT_, X, llvm::ISD::XOR, and Y.

◆ getMOVRELDPseudo()

static unsigned getMOVRELDPseudo ( const SIRegisterInfo TRI,
const TargetRegisterClass VecRC 
)
static

Definition at line 2779 of file SIISelLowering.cpp.

References llvm_unreachable.

Referenced by emitIndirectDst().

◆ getSplatConstantFP()

static ConstantFPSDNode* getSplatConstantFP ( SDValue  Op)
static

◆ isBoolSGPR()

static bool isBoolSGPR ( SDValue  V)
static

Definition at line 5189 of file SIISelLowering.cpp.

References llvm::ISD::AND, llvm::ISD::AssertZext, llvm::AMDGPUISD::BFE_U32, llvm::ISD::BITCAST, llvm::tgtok::Bits, llvm::ISD::BUILD_VECTOR, llvm::countPopulation(), llvm::APInt::countTrailingZeros(), llvm::dyn_cast(), llvm::MipsISD::Ext, llvm::ISD::FABS, llvm::AMDGPUISD::FP_CLASS, llvm::ConstantSDNode::getAPIntValue(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::EVT::getIntegerVT(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getSelect(), llvm::SITargetLowering::getSubtarget(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getValueType(), llvm::SDNode::getValueType(), llvm::ConstantSDNode::getZExtValue(), llvm::MVT::i1, llvm::MVT::i32, llvm::MVT::i64, llvm::ConstantFPSDNode::isInfinity(), llvm::ConstantFPSDNode::isNegative(), llvm::isShiftedMask_64(), llvm::BitmaskEnumDetail::Mask(), llvm::SIInstrFlags::N_INFINITY, llvm::SIInstrFlags::N_NORMAL, llvm::SIInstrFlags::N_SUBNORMAL, llvm::SIInstrFlags::N_ZERO, llvm::ISD::OR, llvm::SIInstrFlags::P_INFINITY, llvm::SIInstrFlags::P_NORMAL, llvm::SIInstrFlags::P_SUBNORMAL, llvm::SIInstrFlags::P_ZERO, llvm::SIInstrFlags::Q_NAN, llvm::SIInstrFlags::S_NAN, llvm::ISD::SETCC, llvm::ISD::SETO, llvm::ISD::SETUNE, llvm::ISD::SHL, llvm::ISD::SIGN_EXTEND, Split(), llvm::AMDGPUTargetLowering::split64BitValue(), llvm::ISD::SRL, std::swap(), llvm::MVT::v2i32, X, llvm::ISD::XOR, Y, and llvm::ISD::ZERO_EXTEND.

Referenced by convertBuildVectorCastElt().

◆ isCanonicalized()

static bool isCanonicalized ( SelectionDAG DAG,
SDValue  Op,
const SISubtarget ST,
unsigned  MaxDepth = 5 
)
static

◆ isClampZeroToOne()

static bool isClampZeroToOne ( SDValue  A,
SDValue  B 
)
static

◆ isFlatGlobalAddrSpace()

static bool isFlatGlobalAddrSpace ( unsigned  AS,
AMDGPUAS  AMDGPUASI 
)
static

◆ isFrameIndexOp()

static bool isFrameIndexOp ( SDValue  Op)
static

◆ isKnownNeverSNan()

static bool isKnownNeverSNan ( SelectionDAG DAG,
SDValue  Op 
)
static

◆ loadM0FromVGPR()

static MachineBasicBlock::iterator loadM0FromVGPR ( const SIInstrInfo TII,
MachineBasicBlock MBB,
MachineInstr MI,
unsigned  InitResultReg,
unsigned  PhiReg,
int  Offset,
bool  UseGPRIdxMode 
)
static

◆ mayTailCallThisCC()

static bool mayTailCallThisCC ( CallingConv::ID  CC)
static

Return true if we might ever do TCO for calls with this calling convention.

Definition at line 1990 of file SIISelLowering.cpp.

References llvm::CallingConv::C, and canGuaranteeTCO().

Referenced by llvm::SITargetLowering::isEligibleForTailCallOptimization().

◆ minMaxOpcToMin3Max3Opc()

static unsigned minMaxOpcToMin3Max3Opc ( unsigned  Opc)
static

◆ processShaderInputArgs()

static void processShaderInputArgs ( SmallVectorImpl< ISD::InputArg > &  Splits,
CallingConv::ID  CallConv,
ArrayRef< ISD::InputArg Ins,
BitVector Skipped,
FunctionType FType,
SIMachineFunctionInfo Info 
)
static

◆ reservePrivateMemoryRegs()

static void reservePrivateMemoryRegs ( const TargetMachine TM,
MachineFunction MF,
const SIRegisterInfo TRI,
SIMachineFunctionInfo Info 
)
static

◆ setM0ToIndexFromSGPR()

static bool setM0ToIndexFromSGPR ( const SIInstrInfo TII,
MachineRegisterInfo MRI,
MachineInstr MI,
int  Offset,
bool  UseGPRIdxMode,
bool  IsIndirectSrc 
)
static

◆ STATISTIC()

STATISTIC ( NumTailCalls  ,
"Number of tail calls  
)

◆ SubIdx2Lane()

static unsigned SubIdx2Lane ( unsigned  Idx)
static

Variable Documentation

◆ EnableVGPRIndexMode

cl::opt<bool> EnableVGPRIndexMode("amdgpu-vgpr-index-mode", cl::desc("Use GPR indexing mode instead of movrel for vector indexing"), cl::init(false))
static

Referenced by emitIndirectDst(), and emitIndirectSrc().