LLVM 19.0.0git
|
Custom DAG lowering for SI. More...
#include "SIISelLowering.h"
#include "AMDGPU.h"
#include "AMDGPUInstrInfo.h"
#include "AMDGPUTargetMachine.h"
#include "GCNSubtarget.h"
#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
#include "SIMachineFunctionInfo.h"
#include "SIRegisterInfo.h"
#include "llvm/ADT/APInt.h"
#include "llvm/ADT/FloatingPointMode.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/Analysis/OptimizationRemarkEmitter.h"
#include "llvm/Analysis/UniformityAnalysis.h"
#include "llvm/BinaryFormat/ELF.h"
#include "llvm/CodeGen/Analysis.h"
#include "llvm/CodeGen/ByteProvider.h"
#include "llvm/CodeGen/FunctionLoweringInfo.h"
#include "llvm/CodeGen/GlobalISel/GISelKnownBits.h"
#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineLoopInfo.h"
#include "llvm/IR/DiagnosticInfo.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/IntrinsicsAMDGPU.h"
#include "llvm/IR/IntrinsicsR600.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/KnownBits.h"
#include "llvm/Support/ModRef.h"
#include <optional>
Go to the source code of this file.
Classes | |
struct | DotSrc |
Macros | |
#define | DEBUG_TYPE "si-lower" |
Variables | |
static cl::opt< bool > | DisableLoopAlignment ("amdgpu-disable-loop-alignment", cl::desc("Do not align and prefetch loops"), cl::init(false)) |
static cl::opt< bool > | UseDivergentRegisterIndexing ("amdgpu-use-divergent-register-indexing", cl::Hidden, cl::desc("Use indirect register addressing for divergent indexes"), cl::init(false)) |
Custom DAG lowering for SI.
Definition in file SIISelLowering.cpp.
#define DEBUG_TYPE "si-lower" |
Definition at line 49 of file SIISelLowering.cpp.
Definition at line 13728 of file SIISelLowering.cpp.
References assert(), and llvm::First.
Referenced by placeSources(), and resolveSources().
|
static |
Definition at line 12044 of file SIISelLowering.cpp.
References assert().
Referenced by hasNon16BitAccesses().
|
static |
Definition at line 10197 of file SIISelLowering.cpp.
References Info.
|
static |
Definition at line 5772 of file SIISelLowering.cpp.
References llvm::ISD::BITCAST, llvm::EVT::changeTypeToInteger(), DL, llvm::SelectionDAG::ExtractVectorElements(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getUNDEF(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::EVT::isVector(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), and llvm::ISD::TRUNCATE.
Referenced by constructRetValue().
|
static |
Definition at line 2311 of file SIISelLowering.cpp.
References llvm::MachineFunction::addLiveIn(), llvm::CCState::AllocateReg(), assert(), and llvm::CCState::getMachineFunction().
Referenced by allocateSGPR32Input(), and allocateSGPR64Input().
|
static |
Definition at line 2320 of file SIISelLowering.cpp.
References allocateFixedSGPRInputImpl(), allocateSGPR32InputImpl(), and llvm::ArgDescriptor::getRegister().
Referenced by llvm::SITargetLowering::allocateSpecialInputSGPRs().
|
static |
Definition at line 2291 of file SIISelLowering.cpp.
References llvm::MachineFunction::addLiveIn(), llvm::CCState::AllocateReg(), assert(), llvm::TargetRegisterClass::begin(), llvm::ArgDescriptor::createRegister(), llvm::CCState::getFirstUnallocated(), llvm::CCState::getMachineFunction(), llvm::report_fatal_error(), and llvm::ArrayRef< T >::size().
Referenced by allocateSGPR32Input(), and allocateSGPR64Input().
|
static |
Definition at line 2328 of file SIISelLowering.cpp.
References allocateFixedSGPRInputImpl(), allocateSGPR32InputImpl(), and llvm::ArgDescriptor::getRegister().
Referenced by llvm::SITargetLowering::allocateSpecialInputSGPRs().
|
static |
Definition at line 2267 of file SIISelLowering.cpp.
References llvm::MachineFunction::addLiveIn(), llvm::CCState::AllocateReg(), llvm::CCState::AllocateStack(), assert(), llvm::ArgDescriptor::createArg(), llvm::ArgDescriptor::createRegister(), llvm::ArgDescriptor::createStack(), llvm::CCState::getFirstUnallocated(), llvm::CCState::getMachineFunction(), llvm::MachineFunction::getRegInfo(), llvm::Offset, llvm::LLT::scalar(), llvm::MachineRegisterInfo::setType(), and llvm::ArrayRef< T >::size().
Referenced by llvm::SITargetLowering::allocateSpecialInputVGPRs().
Definition at line 11346 of file SIISelLowering.cpp.
References llvm::ISD::AND, llvm::ISD::OR, and llvm::ISD::XOR.
|
static |
Definition at line 7277 of file SIISelLowering.cpp.
References assert(), DL, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getTargetGlobalAddress(), llvm::SIInstrInfo::MO_NONE, llvm::Offset, and llvm::AMDGPUISD::PC_ADD_REL_OFFSET.
|
static |
Definition at line 15213 of file SIISelLowering.cpp.
References DL, llvm::SelectionDAG::getMachineNode(), and llvm::SelectionDAG::getTargetConstant().
Referenced by llvm::SITargetLowering::buildRSRC(), and llvm::SITargetLowering::wrapAddr64Rsrc().
|
static |
Definition at line 11752 of file SIISelLowering.cpp.
References llvm::ISD::AND, llvm::ISD::ANY_EXTEND, llvm::ISD::AssertSext, llvm::ISD::AssertZext, llvm::BitWidth, llvm::ISD::BSWAP, calculateByteProvider(), calculateSrcByte(), llvm::ISD::CopyFromReg, llvm::Depth, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::ISD::FSHR, llvm::ByteProvider< ISelOp >::getConstantZero(), llvm::SDValue::getValueSizeInBits(), LHS, llvm_unreachable, llvm::ISD::LOAD, llvm::ISD::OR, llvm::AMDGPUISD::PERM, RHS, llvm::ISD::SHL, llvm::ISD::SIGN_EXTEND, llvm::ISD::SIGN_EXTEND_INREG, llvm::ISD::SRA, llvm::ISD::SRL, llvm::ISD::TRUNCATE, llvm::ISD::ZERO_EXTEND, and llvm::ISD::ZEXTLOAD.
|
static |
Definition at line 11687 of file SIISelLowering.cpp.
References calculateSrcByte(), llvm::Depth, llvm::ByteProvider< ISelOp >::getSrc(), llvm::EVT::getStoreSize(), llvm::SDValue::getValueType(), llvm::EVT::isByteSized(), llvm_unreachable, llvm::ISD::SIGN_EXTEND, llvm::ISD::SIGN_EXTEND_INREG, llvm::ISD::SRA, llvm::ISD::SRL, llvm::ISD::TRUNCATE, and llvm::ISD::ZERO_EXTEND.
Referenced by calculateByteProvider(), and calculateSrcByte().
|
static |
Definition at line 3463 of file SIISelLowering.cpp.
References CC, and llvm::CallingConv::Fast.
|
static |
Definition at line 13910 of file SIISelLowering.cpp.
References assert(), llvm::SelectionDAG::computeKnownBits(), llvm::KnownBits::countMinLeadingZeros(), llvm::SDValue::getValueSizeInBits(), and llvm_unreachable.
Definition at line 15432 of file SIISelLowering.cpp.
References llvm::AMDGPU::isInlinableIntLiteral(), and Size.
Referenced by llvm::APInt::APInt(), llvm::APInt::ashrInPlace(), llvm::SITargetLowering::checkAsmConstraintVal(), llvm::APInt::flipAllBits(), llvm::SITargetLowering::LowerAsmOperandForConstraint(), llvm::APInt::operator<<=(), llvm::APInt::operator=(), llvm::APInt::operator^=(), llvm::APInt::operator|=(), and llvm::APInt::setAllBits().
|
static |
Definition at line 4471 of file SIISelLowering.cpp.
References llvm::SIRegisterInfo::getSubRegFromChannel(), llvm::Offset, and TRI.
Referenced by emitIndirectDst(), and emitIndirectSrc().
|
static |
Definition at line 7482 of file SIISelLowering.cpp.
References adjustLoadValueTypeImpl(), llvm::ISD::BITCAST, llvm::EVT::changeTypeToInteger(), llvm::Data, DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::MVT::getVectorVT(), llvm::EVT::isVector(), llvm::MVT::isVector(), padEltsToUndef(), and llvm::ISD::TRUNCATE.
|
static |
Definition at line 64 of file SIISelLowering.cpp.
References llvm::MachineFunction::getInfo(), llvm::DenormalMode::getPreserveSign(), and Info.
Referenced by llvm::SITargetLowering::denormalsEnabledForType(), llvm::SITargetLowering::isFMADLegal(), llvm::SITargetLowering::isFMAFasterThanFMulAndFAdd(), and llvm::SITargetLowering::isFPExtFoldable().
|
static |
Definition at line 69 of file SIISelLowering.cpp.
References llvm::MachineFunction::getInfo(), llvm::DenormalMode::getPreserveSign(), and Info.
Referenced by llvm::SITargetLowering::denormalsEnabledForType(), llvm::SITargetLowering::isFMADLegal(), and llvm::SITargetLowering::isFMAFasterThanFMulAndFAdd().
Definition at line 7084 of file SIISelLowering.cpp.
References assert().
|
static |
Definition at line 16018 of file SIISelLowering.cpp.
References DEBUG_TYPE, llvm::SmallVectorBase< Size_T >::empty(), llvm::Value::getContext(), llvm::AtomicRMWInst::getOperation(), llvm::AtomicRMWInst::getOperationName(), llvm::AtomicRMWInst::getSyncScopeID(), and llvm::LLVMContext::getSyncScopeNames().
Referenced by llvm::SITargetLowering::shouldExpandAtomicRMWInIR().
|
static |
Definition at line 4610 of file SIISelLowering.cpp.
References llvm::MachineInstrBuilder::add(), llvm::MachineInstrBuilder::addImm(), llvm::MachineInstrBuilder::addReg(), assert(), llvm::BuildMI(), computeIndirectRegAndOffset(), DL, getIndirectSGPRIdx(), llvm::MachineBasicBlock::getParent(), llvm::MachineOperand::getReg(), llvm::MachineFunction::getRegInfo(), I, Idx, llvm::MachineOperand::isReg(), loadM0FromVGPR(), MBB, MI, MRI, llvm::Offset, setM0ToIndexFromSGPR(), SubReg, TII, and TRI.
Referenced by llvm::SITargetLowering::EmitInstrWithCustomInserter().
|
static |
Definition at line 4524 of file SIISelLowering.cpp.
References llvm::MachineInstrBuilder::addImm(), llvm::MachineInstrBuilder::addReg(), llvm::BuildMI(), computeIndirectRegAndOffset(), DL, getIndirectSGPRIdx(), llvm::MachineBasicBlock::getParent(), llvm::MachineFunction::getRegInfo(), I, Idx, llvm::RegState::Implicit, loadM0FromVGPR(), MBB, MI, MRI, llvm::Offset, setM0ToIndexFromSGPR(), SubReg, TII, and TRI.
Referenced by llvm::SITargetLowering::EmitInstrWithCustomInserter().
|
static |
Definition at line 4330 of file SIISelLowering.cpp.
References llvm::MachineInstrBuilder::addImm(), llvm::MachineInstrBuilder::addMBB(), llvm::MachineInstrBuilder::addReg(), llvm::MachineBasicBlock::begin(), llvm::BuildMI(), DL, llvm::ilist_node_impl< OptionsT >::getIterator(), llvm::MachineBasicBlock::getParent(), llvm::MachineFunction::getSubtarget(), llvm::getUndefRegState(), I, Idx, llvm::RegState::Kill, MRI, llvm::Offset, TII, and TRI.
Referenced by loadM0FromVGPR().
|
static |
Definition at line 7414 of file SIISelLowering.cpp.
References llvm::LLVMContext::diagnose(), DL, llvm::SelectionDAG::getContext(), llvm::MachineFunction::getFunction(), llvm::SelectionDAG::getMachineFunction(), and llvm::SelectionDAG::getUNDEF().
|
static |
Definition at line 7423 of file SIISelLowering.cpp.
References llvm::LLVMContext::diagnose(), DL, llvm::SelectionDAG::getContext(), llvm::MachineFunction::getFunction(), llvm::SelectionDAG::getMachineFunction(), and llvm::SelectionDAG::getUNDEF().
Definition at line 74 of file SIISelLowering.cpp.
References llvm::CCState::isAllocated(), and llvm_unreachable.
Referenced by llvm::SITargetLowering::allocateSystemSGPRs().
Helper function for LowerBRCOND.
Definition at line 6181 of file SIISelLowering.cpp.
References I, llvm::SDNode::use_begin(), and llvm::SDNode::use_end().
|
static |
Definition at line 13894 of file SIISelLowering.cpp.
|
static |
Definition at line 15550 of file SIISelLowering.cpp.
Referenced by llvm::SITargetLowering::finalizeLowering().
MemSDNode::getBasePtr() does not work for intrinsics, which needs to offset by the chain and intrinsic ID.
Theoretically we would also need to check the specific intrinsic, but they all place the pointer operand first.
Definition at line 11312 of file SIISelLowering.cpp.
References llvm::ISD::INTRINSIC_VOID, llvm::ISD::INTRINSIC_W_CHAIN, N, and llvm::ISD::STORE.
Referenced by llvm::SITargetLowering::hasMemSDNodeUser().
|
static |
Definition at line 7432 of file SIISelLowering.cpp.
References assert(), DL, llvm::ArrayRef< T >::empty(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::MVT::getVectorVT(), and llvm::ArrayRef< T >::size().
Definition at line 11398 of file SIISelLowering.cpp.
References llvm::CallingConv::C.
Referenced by getPermuteMask().
|
static |
Scalar Type
Definition at line 12085 of file SIISelLowering.cpp.
References assert(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::ExtractVectorElements(), llvm::SelectionDAG::getBitcastedAnyExtOrTrunc(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::MVT::getVectorVT(), and llvm::ISD::SRL.
Referenced by matchPERM(), and resolveSources().
|
static |
Definition at line 10480 of file SIISelLowering.cpp.
References A, assert(), B, llvm::ISD::FMUL, llvm::AMDGPUISD::FMUL_W_CHAIN, llvm::SelectionDAG::getNode(), llvm::SDNode::getNumValues(), llvm::SDValue::getValue(), llvm::SelectionDAG::getVTList(), and llvm_unreachable.
|
static |
Definition at line 10502 of file SIISelLowering.cpp.
References A, assert(), B, llvm::CallingConv::C, llvm::ISD::FMA, llvm::AMDGPUISD::FMA_W_CHAIN, llvm::SelectionDAG::getNode(), llvm::SDNode::getNumValues(), llvm::SDValue::getValue(), llvm::SelectionDAG::getVTList(), and llvm_unreachable.
Definition at line 8510 of file SIISelLowering.cpp.
References llvm::isNullConstant().
|
static |
Definition at line 4505 of file SIISelLowering.cpp.
References llvm::MachineInstrBuilder::add(), llvm::MachineInstrBuilder::addImm(), llvm::BuildMI(), DL, I, Idx, MBB, MI, MRI, llvm::Offset, and TII.
Referenced by emitIndirectDst(), and emitIndirectSrc().
|
static |
Definition at line 10108 of file SIISelLowering.cpp.
References llvm::ISD::ANY_EXTEND, llvm::EVT::bitsLT(), llvm::ISD::EXTLOAD, llvm::SelectionDAG::getNode(), llvm_unreachable, llvm::ISD::NON_EXTLOAD, llvm::ISD::SEXTLOAD, llvm::ISD::SIGN_EXTEND, llvm::ISD::TRUNCATE, llvm::ISD::ZERO_EXTEND, and llvm::ISD::ZEXTLOAD.
|
static |
Definition at line 13577 of file SIISelLowering.cpp.
References llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getVTList(), llvm::AMDGPUISD::MAD_I64_I32, llvm::AMDGPUISD::MAD_U64_U32, Signed, and llvm::ISD::TRUNCATE.
Definition at line 11418 of file SIISelLowering.cpp.
References llvm::ISD::AND, assert(), llvm::CallingConv::C, getConstantPermuteMask(), llvm::ConstantSDNode::getZExtValue(), llvm::ISD::OR, llvm::ISD::SHL, and llvm::ISD::SRL.
|
static |
Definition at line 10580 of file SIISelLowering.cpp.
References assert(), llvm::SelectionDAG::getTargetConstant(), and Info.
|
static |
Definition at line 13040 of file SIISelLowering.cpp.
References llvm::CallingConv::C.
|
static |
Definition at line 13716 of file SIISelLowering.cpp.
References calculateByteProvider().
|
static |
Definition at line 16209 of file SIISelLowering.cpp.
References hasCFUser(), llvm::SmallPtrSetImpl< PtrType >::insert(), and IT.
Referenced by hasCFUser(), and llvm::SITargetLowering::requiresUniformRegister().
Definition at line 12063 of file SIISelLowering.cpp.
References addresses16Bits(), isExtendedFrom16Bits(), and llvm::peekThroughBitcasts().
Referenced by matchPERM().
Definition at line 13185 of file SIISelLowering.cpp.
|
static |
Definition at line 15877 of file SIISelLowering.cpp.
References assert(), llvm::ISD::CopyFromReg, llvm::ISD::INLINEASM, llvm::ISD::INLINEASM_BR, and N.
Referenced by llvm::SITargetLowering::isSDNodeSourceOfDivergence().
Definition at line 12018 of file SIISelLowering.cpp.
References llvm::ISD::ANY_EXTEND, llvm::ISD::EXTLOAD, llvm::SDValue::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::isVector(), llvm::ISD::LOAD, llvm::ISD::SEXTLOAD, llvm::ISD::SIGN_EXTEND, llvm::ISD::ZERO_EXTEND, and llvm::ISD::ZEXTLOAD.
Referenced by hasNon16BitAccesses().
Definition at line 14918 of file SIISelLowering.cpp.
References llvm::ISD::AssertZext.
Referenced by llvm::SITargetLowering::legalizeTargetIndependentNode().
Definition at line 15397 of file SIISelLowering.cpp.
References llvm::StringRef::size().
Referenced by llvm::SITargetLowering::getConstraintType(), and llvm::SITargetLowering::LowerAsmOperandForConstraint().
|
static |
Return true if the value is a known valid address, such that a null check is not necessary.
Definition at line 6742 of file SIISelLowering.cpp.
References TM.
Definition at line 13902 of file SIISelLowering.cpp.
References llvm::ISD::MUL, llvm::AMDGPUISD::MUL_I24, and llvm::AMDGPUISD::MUL_U24.
|
static |
Definition at line 15711 of file SIISelLowering.cpp.
References llvm::countl_zero(), llvm::MachineFunction::getFunction(), llvm::GISelKnownBits::getMachineFunction(), llvm::APInt::setHighBits(), and llvm::KnownBits::Zero.
Referenced by llvm::SITargetLowering::computeKnownBitsForTargetInstr().
|
static |
Definition at line 4422 of file SIISelLowering.cpp.
References llvm::MachineInstrBuilder::addReg(), llvm::MachineBasicBlock::addSuccessor(), llvm::MachineBasicBlock::begin(), llvm::BuildMI(), llvm::MachineFunction::CreateMachineBasicBlock(), DL, emitLoadM0FromVGPRLoop(), llvm::First, llvm::MachineBasicBlock::getParent(), llvm::MachineFunction::getRegInfo(), llvm::MachineFunction::getSubtarget(), I, Idx, llvm::MachineFunction::insert(), MBB, MBBI, MI, MRI, llvm::Offset, llvm::MachineBasicBlock::removeSuccessor(), splitBlockForLoop(), TII, and TRI.
Referenced by emitIndirectDst(), and emitIndirectSrc().
|
static |
Definition at line 5959 of file SIISelLowering.cpp.
References llvm::SelectionDAG::getCondCode(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getCopyFromReg(), llvm::SelectionDAG::getEntryNode(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), llvm::SelectionDAG::getZExtOrTrunc(), N, llvm::ISD::SETCC, llvm::AMDGPUISD::SETCC, and llvm::ISD::SETNE.
|
static |
Definition at line 5930 of file SIISelLowering.cpp.
References llvm::EVT::bitsEq(), llvm::ISD::FP_EXTEND, llvm::SelectionDAG::getCondCode(), llvm::SelectionDAG::getContext(), llvm::getFCmpCondCode(), llvm::EVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::SITargetLowering::getSubtarget(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::AMDGPUSubtarget::getWavefrontSize(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::CmpInst::isFPPredicate(), llvm::TargetLoweringBase::isTypeLegal(), N, and llvm::AMDGPUISD::SETCC.
|
static |
Definition at line 5896 of file SIISelLowering.cpp.
References llvm::EVT::bitsEq(), DL, llvm::SelectionDAG::getCondCode(), llvm::SelectionDAG::getContext(), llvm::getICmpCondCode(), llvm::EVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::SITargetLowering::getSubtarget(), llvm::SelectionDAG::getUNDEF(), llvm::AMDGPUSubtarget::getWavefrontSize(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::CmpInst::isIntPredicate(), llvm::CmpInst::isSigned(), llvm::TargetLoweringBase::isTypeLegal(), LHS, N, RHS, llvm::AMDGPUISD::SETCC, llvm::ISD::SIGN_EXTEND, and llvm::ISD::ZERO_EXTEND.
|
static |
Definition at line 4714 of file SIISelLowering.cpp.
References Accumulator, llvm::MachineInstrBuilder::addImm(), llvm::MachineInstrBuilder::addMBB(), llvm::MachineInstrBuilder::addReg(), llvm::BuildMI(), DL, llvm::MachineBasicBlock::end(), llvm::MachineBasicBlock::getParent(), llvm::MachineFunction::getRegInfo(), I, MI, MRI, splitBlockForLoop(), TII, and TRI.
Referenced by llvm::SITargetLowering::EmitInstrWithCustomInserter().
|
static |
Definition at line 12140 of file SIISelLowering.cpp.
References assert(), calculateByteProvider(), llvm::TargetLowering::DAGCombinerInfo::DAG, DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBitcastedAnyExtOrTrunc(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getDataLayout(), getDWordFromOffset(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), hasNon16BitAccesses(), llvm::DataLayout::isBigEndian(), llvm::EVT::isByteSized(), N, P, llvm::AMDGPUISD::PERM, llvm::SmallVectorTemplateBase< T, bool >::push_back(), and llvm::SmallVectorBase< Size_T >::size().
Referenced by llvm::SITargetLowering::PerformDAGCombine().
|
static |
Return true if we might ever do TCO for calls with this calling convention.
Definition at line 3468 of file SIISelLowering.cpp.
References llvm::CallingConv::AMDGPU_Gfx, llvm::CallingConv::C, canGuaranteeTCO(), and CC.
Definition at line 1107 of file SIISelLowering.cpp.
References assert(), llvm::Type::getContext(), llvm::EVT::getEVT(), and llvm::EVT::getVectorVT().
Referenced by llvm::SITargetLowering::getTgtMemIntrinsic(), and memVTFromLoadIntrReturn().
Definition at line 1121 of file SIISelLowering.cpp.
References assert(), and memVTFromLoadIntrData().
Referenced by llvm::SITargetLowering::getTgtMemIntrinsic().
Definition at line 12977 of file SIISelLowering.cpp.
References llvm::AMDGPUISD::FMAX3, llvm::ISD::FMAXIMUM, llvm::AMDGPUISD::FMAXIMUM3, llvm::ISD::FMAXNUM, llvm::ISD::FMAXNUM_IEEE, llvm::AMDGPUISD::FMIN3, llvm::ISD::FMINIMUM, llvm::AMDGPUISD::FMINIMUM3, llvm::ISD::FMINNUM, llvm::ISD::FMINNUM_IEEE, llvm_unreachable, llvm::ISD::SMAX, llvm::AMDGPUISD::SMAX3, llvm::ISD::SMIN, llvm::AMDGPUISD::SMIN3, llvm::ISD::UMAX, llvm::AMDGPUISD::UMAX3, llvm::ISD::UMIN, and llvm::AMDGPUISD::UMIN3.
|
static |
Definition at line 7573 of file SIISelLowering.cpp.
References Addr, llvm::ISD::ANY_EXTEND, DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getNode(), I, and llvm::SmallVectorTemplateBase< T, bool >::push_back().
|
static |
Definition at line 7461 of file SIISelLowering.cpp.
References DL, llvm::SelectionDAG::ExtractVectorElements(), llvm::SelectionDAG::getBuildVector(), llvm::EVT::getScalarType(), llvm::SelectionDAG::getUNDEF(), llvm::EVT::isVector(), and llvm::SmallVectorTemplateBase< T, bool >::push_back().
Referenced by constructRetValue().
|
static |
Definition at line 7555 of file SIISelLowering.cpp.
References DL, llvm::SDValue::getNode(), and llvm::SelectionDAG::getTargetConstant().
|
static |
Definition at line 13748 of file SIISelLowering.cpp.
References addPermMasks(), assert(), llvm::SmallVectorTemplateCommon< T, typename >::end(), llvm::find_if(), I, llvm::Match, llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::ByteProvider< ISelOp >::Src, and llvm::ByteProvider< ISelOp >::SrcOffset.
|
static |
Definition at line 2168 of file SIISelLowering.cpp.
References llvm::CallingConv::AMDGPU_PS, assert(), llvm::ISD::InputArg::Flags, llvm::ISD::InputArg::getOrigArgIndex(), llvm::MVT::getScalarSizeInBits(), I, Info, llvm::ISD::ArgFlagsTy::isInReg(), llvm::ISD::ArgFlagsTy::isSplit(), llvm::ISD::ArgFlagsTy::isSplitEnd(), llvm::MVT::isVector(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::BitVector::set(), llvm::ISD::InputArg::Used, and llvm::ISD::InputArg::VT.
Referenced by llvm::SITargetLowering::LowerFormalArguments().
|
static |
Definition at line 2631 of file SIISelLowering.cpp.
References assert(), llvm::Function::getCallingConv(), llvm::MachineFunction::getFrameInfo(), llvm::MachineFunction::getFunction(), llvm::MachineFunction::getRegInfo(), llvm::MachineFunction::getSubtarget(), llvm::MachineFrameInfo::hasCalls(), llvm::MachineFrameInfo::hasStackObjects(), Info, llvm::AMDGPU::isShader(), MRI, llvm::None, llvm::AMDGPUFunctionArgInfo::PRIVATE_SEGMENT_BUFFER, llvm::report_fatal_error(), TM, and TRI.
Referenced by llvm::SITargetLowering::finalizeLowering().
|
static |
Definition at line 13826 of file SIISelLowering.cpp.
References addPermMasks(), assert(), llvm::SmallVectorTemplateCommon< T, typename >::begin(), llvm::SmallVectorTemplateCommon< T, typename >::end(), llvm::SelectionDAG::getConstant(), getDWordFromOffset(), llvm::SelectionDAG::getNode(), llvm::ISD::OR, llvm::AMDGPUISD::PERM, llvm::SmallVectorTemplateBase< T, bool >::push_back(), and llvm::SmallVectorBase< Size_T >::size().
|
static |
Definition at line 8474 of file SIISelLowering.cpp.
References llvm::SelectionDAG::getRegister(), llvm::GCNSubtarget::hasRestrictedSOffset(), and llvm::isNullConstant().
|
static |
Definition at line 4485 of file SIISelLowering.cpp.
References llvm::MachineInstrBuilder::add(), llvm::MachineInstrBuilder::addImm(), assert(), llvm::BuildMI(), DL, I, Idx, MBB, MI, llvm::Offset, and TII.
Referenced by emitIndirectDst(), and emitIndirectSrc().
|
static |
Definition at line 4228 of file SIISelLowering.cpp.
References llvm::MachineBasicBlock::addSuccessor(), llvm::MachineBasicBlock::begin(), llvm::MachineFunction::CreateMachineBasicBlock(), llvm::MachineBasicBlock::end(), llvm::MachineBasicBlock::getParent(), I, llvm::MachineFunction::insert(), MBB, MBBI, MI, llvm::MachineBasicBlock::splice(), and llvm::MachineBasicBlock::transferSuccessorsAndUpdatePHIs().
Referenced by llvm::SITargetLowering::emitGWSMemViolTestLoop(), loadM0FromVGPR(), and lowerWaveReduce().
STATISTIC | ( | NumTailCalls | , |
"Number of tail calls" | |||
) |
|
static |
Return the source of an fp_extend from f16 to f32, or a converted FP constant.
Definition at line 13453 of file SIISelLowering.cpp.
References llvm::APFloat::convert(), llvm::ISD::FP_EXTEND, llvm::SelectionDAG::getConstantFP(), llvm::APFloatBase::IEEEhalf(), and llvm::APFloatBase::rmNearestTiesToEven.
Helper function for adjustWritemask.
Definition at line 14737 of file SIISelLowering.cpp.
References Idx.
Definition at line 16013 of file SIISelLowering.cpp.
References F.
Referenced by llvm::SITargetLowering::shouldExpandAtomicRMWInIR().
Definition at line 12905 of file SIISelLowering.cpp.
|
static |
Referenced by llvm::SITargetLowering::getPrefLoopAlignment().
|
static |
Referenced by llvm::SITargetLowering::shouldExpandVectorDynExt().