LLVM 20.0.0git
|
Custom DAG lowering for SI. More...
#include "SIISelLowering.h"
#include "AMDGPU.h"
#include "AMDGPUInstrInfo.h"
#include "AMDGPUTargetMachine.h"
#include "GCNSubtarget.h"
#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
#include "SIMachineFunctionInfo.h"
#include "SIRegisterInfo.h"
#include "llvm/ADT/APInt.h"
#include "llvm/ADT/FloatingPointMode.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/Analysis/OptimizationRemarkEmitter.h"
#include "llvm/Analysis/UniformityAnalysis.h"
#include "llvm/BinaryFormat/ELF.h"
#include "llvm/CodeGen/Analysis.h"
#include "llvm/CodeGen/ByteProvider.h"
#include "llvm/CodeGen/FunctionLoweringInfo.h"
#include "llvm/CodeGen/GlobalISel/GISelKnownBits.h"
#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineLoopInfo.h"
#include "llvm/IR/DiagnosticInfo.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/IntrinsicsAMDGPU.h"
#include "llvm/IR/IntrinsicsR600.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/KnownBits.h"
#include "llvm/Support/ModRef.h"
#include <optional>
Go to the source code of this file.
Classes | |
struct | DotSrc |
Macros | |
#define | DEBUG_TYPE "si-lower" |
Variables | |
static cl::opt< bool > | DisableLoopAlignment ("amdgpu-disable-loop-alignment", cl::desc("Do not align and prefetch loops"), cl::init(false)) |
static cl::opt< bool > | UseDivergentRegisterIndexing ("amdgpu-use-divergent-register-indexing", cl::Hidden, cl::desc("Use indirect register addressing for divergent indexes"), cl::init(false)) |
Custom DAG lowering for SI.
Definition in file SIISelLowering.cpp.
#define DEBUG_TYPE "si-lower" |
Definition at line 49 of file SIISelLowering.cpp.
Definition at line 13800 of file SIISelLowering.cpp.
References assert(), and llvm::First.
Referenced by placeSources(), and resolveSources().
|
static |
Definition at line 12091 of file SIISelLowering.cpp.
References assert().
Referenced by hasNon16BitAccesses().
|
static |
Definition at line 10245 of file SIISelLowering.cpp.
References Info.
|
static |
Definition at line 5896 of file SIISelLowering.cpp.
References llvm::ISD::BITCAST, llvm::EVT::changeTypeToInteger(), DL, llvm::SelectionDAG::ExtractVectorElements(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getUNDEF(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::EVT::isVector(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), and llvm::ISD::TRUNCATE.
Referenced by constructRetValue().
|
static |
Definition at line 2337 of file SIISelLowering.cpp.
References llvm::MachineFunction::addLiveIn(), llvm::CCState::AllocateReg(), assert(), and llvm::CCState::getMachineFunction().
Referenced by allocateSGPR32Input(), and allocateSGPR64Input().
|
static |
Definition at line 2346 of file SIISelLowering.cpp.
References allocateFixedSGPRInputImpl(), allocateSGPR32InputImpl(), and llvm::ArgDescriptor::getRegister().
Referenced by llvm::SITargetLowering::allocateSpecialInputSGPRs().
|
static |
Definition at line 2317 of file SIISelLowering.cpp.
References llvm::MachineFunction::addLiveIn(), llvm::CCState::AllocateReg(), assert(), llvm::TargetRegisterClass::begin(), llvm::ArgDescriptor::createRegister(), llvm::CCState::getFirstUnallocated(), llvm::CCState::getMachineFunction(), llvm::report_fatal_error(), and llvm::ArrayRef< T >::size().
Referenced by allocateSGPR32Input(), and allocateSGPR64Input().
|
static |
Definition at line 2354 of file SIISelLowering.cpp.
References allocateFixedSGPRInputImpl(), allocateSGPR32InputImpl(), and llvm::ArgDescriptor::getRegister().
Referenced by llvm::SITargetLowering::allocateSpecialInputSGPRs().
|
static |
Definition at line 2293 of file SIISelLowering.cpp.
References llvm::MachineFunction::addLiveIn(), llvm::CCState::AllocateReg(), llvm::CCState::AllocateStack(), assert(), llvm::ArgDescriptor::createArg(), llvm::ArgDescriptor::createRegister(), llvm::ArgDescriptor::createStack(), llvm::CCState::getFirstUnallocated(), llvm::CCState::getMachineFunction(), llvm::MachineFunction::getRegInfo(), llvm::Offset, llvm::LLT::scalar(), llvm::MachineRegisterInfo::setType(), and llvm::ArrayRef< T >::size().
Referenced by llvm::SITargetLowering::allocateSpecialInputVGPRs().
|
static |
Definition at line 16067 of file SIISelLowering.cpp.
References Flt, llvm::Function::getDenormalMode(), llvm::Type::getFltSemantics(), llvm::Function::getFnAttribute(), llvm::Instruction::getFunction(), llvm::DenormalMode::getPreserveSign(), llvm::Type::getScalarType(), llvm::Value::getType(), llvm::Attribute::getValueAsBool(), and llvm::Instruction::hasMetadata().
Referenced by llvm::SITargetLowering::shouldExpandAtomicRMWInIR().
Definition at line 11395 of file SIISelLowering.cpp.
References llvm::ISD::AND, llvm::ISD::OR, and llvm::ISD::XOR.
|
static |
Definition at line 7574 of file SIISelLowering.cpp.
References assert(), DL, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getTargetGlobalAddress(), llvm::SIInstrInfo::MO_NONE, llvm::Offset, and llvm::AMDGPUISD::PC_ADD_REL_OFFSET.
|
static |
Definition at line 15285 of file SIISelLowering.cpp.
References DL, llvm::SelectionDAG::getMachineNode(), and llvm::SelectionDAG::getTargetConstant().
Referenced by llvm::SITargetLowering::buildRSRC(), and llvm::SITargetLowering::wrapAddr64Rsrc().
|
static |
Definition at line 11801 of file SIISelLowering.cpp.
References llvm::ISD::AND, llvm::ISD::ANY_EXTEND, llvm::ISD::AssertSext, llvm::ISD::AssertZext, llvm::BitWidth, llvm::ISD::BSWAP, calculateByteProvider(), calculateSrcByte(), llvm::ISD::CopyFromReg, llvm::Depth, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::ISD::FSHR, llvm::ByteProvider< ISelOp >::getConstantZero(), llvm::SDValue::getValueSizeInBits(), LHS, llvm_unreachable, llvm::ISD::LOAD, llvm::ISD::OR, llvm::AMDGPUISD::PERM, RHS, llvm::ISD::SHL, llvm::ISD::SIGN_EXTEND, llvm::ISD::SIGN_EXTEND_INREG, llvm::ISD::SRA, llvm::ISD::SRL, llvm::ISD::TRUNCATE, llvm::ISD::ZERO_EXTEND, and llvm::ISD::ZEXTLOAD.
|
static |
Definition at line 11736 of file SIISelLowering.cpp.
References calculateSrcByte(), llvm::Depth, llvm::ByteProvider< ISelOp >::getSrc(), llvm::EVT::getStoreSize(), llvm::SDValue::getValueType(), llvm::EVT::isByteSized(), llvm_unreachable, llvm::ISD::SIGN_EXTEND, llvm::ISD::SIGN_EXTEND_INREG, llvm::ISD::SRA, llvm::ISD::SRL, llvm::ISD::TRUNCATE, and llvm::ISD::ZERO_EXTEND.
Referenced by calculateByteProvider(), and calculateSrcByte().
|
static |
Definition at line 3510 of file SIISelLowering.cpp.
References CC, and llvm::CallingConv::Fast.
|
static |
Definition at line 13982 of file SIISelLowering.cpp.
References assert(), llvm::SelectionDAG::computeKnownBits(), llvm::KnownBits::countMinLeadingZeros(), llvm::SDValue::getValueSizeInBits(), and llvm_unreachable.
Definition at line 15504 of file SIISelLowering.cpp.
References llvm::AMDGPU::isInlinableIntLiteral(), and Size.
Referenced by llvm::APInt::APInt(), llvm::APInt::ashrInPlace(), llvm::SITargetLowering::checkAsmConstraintVal(), llvm::APInt::flipAllBits(), llvm::SITargetLowering::LowerAsmOperandForConstraint(), llvm::APInt::operator<<=(), llvm::APInt::operator=(), llvm::APInt::operator^=(), llvm::APInt::operator|=(), and llvm::APInt::setAllBits().
|
static |
Definition at line 4603 of file SIISelLowering.cpp.
References llvm::SIRegisterInfo::getSubRegFromChannel(), llvm::Offset, and TRI.
Referenced by emitIndirectDst(), and emitIndirectSrc().
|
static |
Definition at line 7779 of file SIISelLowering.cpp.
References adjustLoadValueTypeImpl(), llvm::ISD::BITCAST, llvm::EVT::changeTypeToInteger(), llvm::Data, DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::MVT::getVectorVT(), llvm::EVT::isVector(), llvm::MVT::isVector(), padEltsToUndef(), and llvm::ISD::TRUNCATE.
|
static |
Definition at line 64 of file SIISelLowering.cpp.
References llvm::MachineFunction::getInfo(), llvm::DenormalMode::getPreserveSign(), and Info.
Referenced by llvm::SITargetLowering::denormalsEnabledForType(), llvm::SITargetLowering::isFMADLegal(), llvm::SITargetLowering::isFMAFasterThanFMulAndFAdd(), and llvm::SITargetLowering::isFPExtFoldable().
|
static |
Definition at line 69 of file SIISelLowering.cpp.
References llvm::MachineFunction::getInfo(), llvm::DenormalMode::getPreserveSign(), and Info.
Referenced by llvm::SITargetLowering::denormalsEnabledForType(), llvm::SITargetLowering::isFMADLegal(), and llvm::SITargetLowering::isFMAFasterThanFMulAndFAdd().
Definition at line 7381 of file SIISelLowering.cpp.
References assert().
|
static |
Definition at line 16082 of file SIISelLowering.cpp.
References DEBUG_TYPE, llvm::SmallVectorBase< Size_T >::empty(), llvm::Value::getContext(), llvm::AtomicRMWInst::getOperation(), llvm::AtomicRMWInst::getOperationName(), llvm::AtomicRMWInst::getSyncScopeID(), and llvm::LLVMContext::getSyncScopeNames().
Referenced by llvm::SITargetLowering::shouldExpandAtomicRMWInIR().
|
static |
Definition at line 4742 of file SIISelLowering.cpp.
References llvm::MachineInstrBuilder::add(), llvm::MachineInstrBuilder::addImm(), llvm::MachineInstrBuilder::addReg(), assert(), llvm::BuildMI(), computeIndirectRegAndOffset(), DL, getIndirectSGPRIdx(), llvm::MachineBasicBlock::getParent(), llvm::MachineOperand::getReg(), llvm::MachineFunction::getRegInfo(), I, Idx, llvm::MachineOperand::isReg(), loadM0FromVGPR(), MBB, MI, MRI, llvm::Offset, setM0ToIndexFromSGPR(), SubReg, TII, and TRI.
Referenced by llvm::SITargetLowering::EmitInstrWithCustomInserter().
|
static |
Definition at line 4656 of file SIISelLowering.cpp.
References llvm::MachineInstrBuilder::addImm(), llvm::MachineInstrBuilder::addReg(), llvm::BuildMI(), computeIndirectRegAndOffset(), DL, getIndirectSGPRIdx(), llvm::MachineBasicBlock::getParent(), llvm::MachineFunction::getRegInfo(), I, Idx, llvm::RegState::Implicit, loadM0FromVGPR(), MBB, MI, MRI, llvm::Offset, setM0ToIndexFromSGPR(), SubReg, TII, and TRI.
Referenced by llvm::SITargetLowering::EmitInstrWithCustomInserter().
|
static |
Definition at line 4462 of file SIISelLowering.cpp.
References llvm::MachineInstrBuilder::addImm(), llvm::MachineInstrBuilder::addMBB(), llvm::MachineInstrBuilder::addReg(), llvm::MachineBasicBlock::begin(), llvm::BuildMI(), DL, llvm::ilist_node_impl< OptionsT >::getIterator(), llvm::MachineBasicBlock::getParent(), llvm::MachineFunction::getSubtarget(), llvm::getUndefRegState(), I, Idx, llvm::RegState::Kill, MRI, llvm::Offset, TII, and TRI.
Referenced by loadM0FromVGPR().
|
static |
Definition at line 7711 of file SIISelLowering.cpp.
References llvm::LLVMContext::diagnose(), DL, llvm::SelectionDAG::getContext(), llvm::MachineFunction::getFunction(), llvm::SelectionDAG::getMachineFunction(), and llvm::SelectionDAG::getUNDEF().
|
static |
Definition at line 7720 of file SIISelLowering.cpp.
References llvm::LLVMContext::diagnose(), DL, llvm::SelectionDAG::getContext(), llvm::MachineFunction::getFunction(), llvm::SelectionDAG::getMachineFunction(), and llvm::SelectionDAG::getUNDEF().
Definition at line 74 of file SIISelLowering.cpp.
References llvm::CCState::isAllocated(), and llvm_unreachable.
Referenced by llvm::SITargetLowering::allocateSystemSGPRs().
Helper function for LowerBRCOND.
Definition at line 6478 of file SIISelLowering.cpp.
References I, llvm::SDNode::use_begin(), and llvm::SDNode::use_end().
|
static |
Definition at line 13966 of file SIISelLowering.cpp.
|
static |
Definition at line 15622 of file SIISelLowering.cpp.
Referenced by llvm::SITargetLowering::finalizeLowering().
MemSDNode::getBasePtr() does not work for intrinsics, which needs to offset by the chain and intrinsic ID.
Theoretically we would also need to check the specific intrinsic, but they all place the pointer operand first.
Definition at line 11361 of file SIISelLowering.cpp.
References llvm::ISD::INTRINSIC_VOID, llvm::ISD::INTRINSIC_W_CHAIN, N, and llvm::ISD::STORE.
Referenced by llvm::SITargetLowering::hasMemSDNodeUser().
|
static |
Definition at line 7729 of file SIISelLowering.cpp.
References assert(), DL, llvm::ArrayRef< T >::empty(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::MVT::getVectorVT(), and llvm::ArrayRef< T >::size().
Definition at line 11447 of file SIISelLowering.cpp.
References llvm::CallingConv::C.
Referenced by getPermuteMask().
|
static |
Scalar Type
Definition at line 12132 of file SIISelLowering.cpp.
References assert(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::ExtractVectorElements(), llvm::SelectionDAG::getBitcastedAnyExtOrTrunc(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::MVT::getVectorVT(), and llvm::ISD::SRL.
Referenced by matchPERM(), and resolveSources().
|
static |
Definition at line 10528 of file SIISelLowering.cpp.
References A, assert(), B, llvm::ISD::FMUL, llvm::AMDGPUISD::FMUL_W_CHAIN, llvm::SelectionDAG::getNode(), llvm::SDNode::getNumValues(), llvm::SDValue::getValue(), llvm::SelectionDAG::getVTList(), and llvm_unreachable.
|
static |
Definition at line 10550 of file SIISelLowering.cpp.
References A, assert(), B, llvm::CallingConv::C, llvm::ISD::FMA, llvm::AMDGPUISD::FMA_W_CHAIN, llvm::SelectionDAG::getNode(), llvm::SDNode::getNumValues(), llvm::SDValue::getValue(), llvm::SelectionDAG::getVTList(), and llvm_unreachable.
|
static |
Definition at line 4637 of file SIISelLowering.cpp.
References llvm::MachineInstrBuilder::add(), llvm::MachineInstrBuilder::addImm(), llvm::BuildMI(), DL, I, Idx, MBB, MI, MRI, llvm::Offset, and TII.
Referenced by emitIndirectDst(), and emitIndirectSrc().
|
static |
Definition at line 10156 of file SIISelLowering.cpp.
References llvm::ISD::ANY_EXTEND, llvm::EVT::bitsLT(), llvm::ISD::EXTLOAD, llvm::SelectionDAG::getNode(), llvm_unreachable, llvm::ISD::NON_EXTLOAD, llvm::ISD::SEXTLOAD, llvm::ISD::SIGN_EXTEND, llvm::ISD::TRUNCATE, llvm::ISD::ZERO_EXTEND, and llvm::ISD::ZEXTLOAD.
|
static |
Definition at line 13649 of file SIISelLowering.cpp.
References llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getVTList(), llvm::AMDGPUISD::MAD_I64_I32, llvm::AMDGPUISD::MAD_U64_U32, Signed, and llvm::ISD::TRUNCATE.
Definition at line 11467 of file SIISelLowering.cpp.
References llvm::ISD::AND, assert(), llvm::CallingConv::C, getConstantPermuteMask(), llvm::ConstantSDNode::getZExtValue(), llvm::ISD::OR, llvm::ISD::SHL, and llvm::ISD::SRL.
|
static |
Definition at line 10628 of file SIISelLowering.cpp.
References assert(), llvm::SelectionDAG::getTargetConstant(), and Info.
|
static |
Definition at line 13088 of file SIISelLowering.cpp.
References llvm::CallingConv::C.
|
static |
RMW
, based on the properties of the target memory. Definition at line 16118 of file SIISelLowering.cpp.
References llvm::Function::getFnAttribute(), llvm::Instruction::getFunction(), llvm::Attribute::getValueAsBool(), llvm::Instruction::hasMetadata(), and llvm::GCNSubtarget::supportsAgentScopeFineGrainedRemoteMemoryAtomics().
Referenced by llvm::SITargetLowering::shouldExpandAtomicRMWInIR().
|
static |
Definition at line 13788 of file SIISelLowering.cpp.
References calculateByteProvider().
|
static |
Definition at line 16388 of file SIISelLowering.cpp.
References hasCFUser(), llvm::SmallPtrSetImpl< PtrType >::insert(), and IT.
Referenced by hasCFUser(), and llvm::SITargetLowering::requiresUniformRegister().
Definition at line 12110 of file SIISelLowering.cpp.
References addresses16Bits(), isExtendedFrom16Bits(), and llvm::peekThroughBitcasts().
Referenced by matchPERM().
Definition at line 16111 of file SIISelLowering.cpp.
References llvm::VectorType::getElementType(), llvm::FixedVectorType::getNumElements(), and llvm::Type::isBFloatTy().
Referenced by llvm::SITargetLowering::shouldExpandAtomicRMWInIR().
Definition at line 13257 of file SIISelLowering.cpp.
|
static |
Definition at line 15949 of file SIISelLowering.cpp.
References assert(), llvm::ISD::CopyFromReg, llvm::ISD::INLINEASM, llvm::ISD::INLINEASM_BR, and N.
Referenced by llvm::SITargetLowering::isSDNodeSourceOfDivergence().
Definition at line 12065 of file SIISelLowering.cpp.
References llvm::ISD::ANY_EXTEND, llvm::ISD::EXTLOAD, llvm::SDValue::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::isVector(), llvm::ISD::LOAD, llvm::ISD::SEXTLOAD, llvm::ISD::SIGN_EXTEND, llvm::ISD::ZERO_EXTEND, and llvm::ISD::ZEXTLOAD.
Referenced by hasNon16BitAccesses().
Definition at line 14990 of file SIISelLowering.cpp.
References llvm::ISD::AssertZext.
Referenced by llvm::SITargetLowering::legalizeTargetIndependentNode().
Definition at line 16106 of file SIISelLowering.cpp.
References llvm::VectorType::getElementType(), llvm::FixedVectorType::getNumElements(), and llvm::Type::isHalfTy().
Referenced by llvm::SITargetLowering::shouldExpandAtomicRMWInIR().
Definition at line 16096 of file SIISelLowering.cpp.
References llvm::Type::isBFloatTy(), and llvm::Type::isHalfTy().
Referenced by llvm::SITargetLowering::shouldExpandAtomicRMWInIR().
Definition at line 15469 of file SIISelLowering.cpp.
References llvm::StringRef::size().
Referenced by llvm::SITargetLowering::getConstraintType(), and llvm::SITargetLowering::LowerAsmOperandForConstraint().
|
static |
Return true if the value is a known valid address, such that a null check is not necessary.
Definition at line 7039 of file SIISelLowering.cpp.
References TM.
Definition at line 13974 of file SIISelLowering.cpp.
References llvm::ISD::MUL, llvm::AMDGPUISD::MUL_I24, and llvm::AMDGPUISD::MUL_U24.
|
static |
Definition at line 15783 of file SIISelLowering.cpp.
References llvm::countl_zero(), llvm::MachineFunction::getFunction(), llvm::GISelKnownBits::getMachineFunction(), llvm::APInt::setHighBits(), and llvm::KnownBits::Zero.
Referenced by llvm::SITargetLowering::computeKnownBitsForTargetInstr().
|
static |
Definition at line 4554 of file SIISelLowering.cpp.
References llvm::MachineInstrBuilder::addReg(), llvm::MachineBasicBlock::addSuccessor(), llvm::MachineBasicBlock::begin(), llvm::BuildMI(), llvm::MachineFunction::CreateMachineBasicBlock(), DL, emitLoadM0FromVGPRLoop(), llvm::First, llvm::MachineBasicBlock::getParent(), llvm::MachineFunction::getRegInfo(), llvm::MachineFunction::getSubtarget(), I, Idx, llvm::MachineFunction::insert(), MBB, MBBI, MI, MRI, llvm::Offset, llvm::MachineBasicBlock::removeSuccessor(), splitBlockForLoop(), TII, and TRI.
Referenced by emitIndirectDst(), and emitIndirectSrc().
|
static |
Definition at line 6078 of file SIISelLowering.cpp.
References llvm::SelectionDAG::getCondCode(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getCopyFromReg(), llvm::SelectionDAG::getEntryNode(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), llvm::SelectionDAG::getZExtOrTrunc(), N, llvm::ISD::SETCC, llvm::AMDGPUISD::SETCC, and llvm::ISD::SETNE.
|
static |
Definition at line 6049 of file SIISelLowering.cpp.
References llvm::EVT::bitsEq(), llvm::ISD::FP_EXTEND, llvm::SelectionDAG::getCondCode(), llvm::SelectionDAG::getContext(), llvm::getFCmpCondCode(), llvm::EVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::SITargetLowering::getSubtarget(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::AMDGPUSubtarget::getWavefrontSize(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::CmpInst::isFPPredicate(), llvm::TargetLoweringBase::isTypeLegal(), N, and llvm::AMDGPUISD::SETCC.
|
static |
Definition at line 6015 of file SIISelLowering.cpp.
References llvm::EVT::bitsEq(), DL, llvm::SelectionDAG::getCondCode(), llvm::SelectionDAG::getContext(), llvm::getICmpCondCode(), llvm::EVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::SITargetLowering::getSubtarget(), llvm::SelectionDAG::getUNDEF(), llvm::AMDGPUSubtarget::getWavefrontSize(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::CmpInst::isIntPredicate(), llvm::CmpInst::isSigned(), llvm::TargetLoweringBase::isTypeLegal(), LHS, N, RHS, llvm::AMDGPUISD::SETCC, llvm::ISD::SIGN_EXTEND, and llvm::ISD::ZERO_EXTEND.
|
static |
Definition at line 6115 of file SIISelLowering.cpp.
References assert(), llvm::ISD::CONCAT_VECTORS, llvm::ISD::CONVERGENCECTRL_GLUE, llvm::ISD::EXTRACT_SUBVECTOR, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getAnyExtOrTrunc(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::MVT::getIntegerVT(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::EVT::getSimpleVT(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetConstant(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::SelectionDAG::getVectorIdxConstant(), llvm::EVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::MVT::getVectorVT(), llvm::ISD::INTRINSIC_WO_CHAIN, llvm::EVT::isFloatingPoint(), llvm::EVT::isVector(), llvm_unreachable, N, Operands, llvm::SmallVectorTemplateBase< T, bool >::push_back(), and llvm::MVT::SimpleTy.
|
static |
Definition at line 4846 of file SIISelLowering.cpp.
References Accumulator, llvm::MachineInstrBuilder::addImm(), llvm::MachineInstrBuilder::addMBB(), llvm::MachineInstrBuilder::addReg(), llvm::BuildMI(), DL, llvm::MachineBasicBlock::end(), llvm::MachineBasicBlock::getParent(), llvm::MachineFunction::getRegInfo(), I, MI, MRI, splitBlockForLoop(), TII, and TRI.
Referenced by llvm::SITargetLowering::EmitInstrWithCustomInserter().
|
static |
Definition at line 12187 of file SIISelLowering.cpp.
References assert(), calculateByteProvider(), llvm::TargetLowering::DAGCombinerInfo::DAG, DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBitcastedAnyExtOrTrunc(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getDataLayout(), getDWordFromOffset(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), hasNon16BitAccesses(), llvm::DataLayout::isBigEndian(), llvm::EVT::isByteSized(), N, P, llvm::AMDGPUISD::PERM, llvm::SmallVectorTemplateBase< T, bool >::push_back(), and llvm::SmallVectorBase< Size_T >::size().
Referenced by llvm::SITargetLowering::PerformDAGCombine().
|
static |
Return true if we might ever do TCO for calls with this calling convention.
Definition at line 3515 of file SIISelLowering.cpp.
References llvm::CallingConv::AMDGPU_Gfx, llvm::CallingConv::C, canGuaranteeTCO(), and CC.
|
static |
Definition at line 1117 of file SIISelLowering.cpp.
References assert(), DL, llvm::Type::getContext(), llvm::TargetLoweringBase::getValueType(), and llvm::EVT::getVectorVT().
Referenced by llvm::SITargetLowering::getTgtMemIntrinsic(), and memVTFromLoadIntrReturn().
|
static |
Definition at line 1133 of file SIISelLowering.cpp.
References assert(), DL, and memVTFromLoadIntrData().
Referenced by llvm::SITargetLowering::getTgtMemIntrinsic().
Definition at line 13025 of file SIISelLowering.cpp.
References llvm::AMDGPUISD::FMAX3, llvm::ISD::FMAXIMUM, llvm::AMDGPUISD::FMAXIMUM3, llvm::ISD::FMAXNUM, llvm::ISD::FMAXNUM_IEEE, llvm::AMDGPUISD::FMIN3, llvm::ISD::FMINIMUM, llvm::AMDGPUISD::FMINIMUM3, llvm::ISD::FMINNUM, llvm::ISD::FMINNUM_IEEE, llvm_unreachable, llvm::ISD::SMAX, llvm::AMDGPUISD::SMAX3, llvm::ISD::SMIN, llvm::AMDGPUISD::SMIN3, llvm::ISD::UMAX, llvm::AMDGPUISD::UMAX3, llvm::ISD::UMIN, and llvm::AMDGPUISD::UMIN3.
|
static |
Definition at line 7869 of file SIISelLowering.cpp.
References Addr, llvm::ISD::ANY_EXTEND, DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getNode(), I, and llvm::SmallVectorTemplateBase< T, bool >::push_back().
|
static |
Definition at line 7758 of file SIISelLowering.cpp.
References DL, llvm::SelectionDAG::ExtractVectorElements(), llvm::SelectionDAG::getBuildVector(), llvm::EVT::getScalarType(), llvm::SelectionDAG::getUNDEF(), llvm::EVT::isVector(), and llvm::SmallVectorTemplateBase< T, bool >::push_back().
Referenced by constructRetValue().
|
static |
Definition at line 7851 of file SIISelLowering.cpp.
References DL, llvm::SDValue::getNode(), and llvm::SelectionDAG::getTargetConstant().
|
static |
Definition at line 13820 of file SIISelLowering.cpp.
References addPermMasks(), assert(), llvm::SmallVectorTemplateCommon< T, typename >::end(), llvm::find_if(), I, llvm::Match, llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::ByteProvider< ISelOp >::Src, and llvm::ByteProvider< ISelOp >::SrcOffset.
|
static |
Definition at line 2194 of file SIISelLowering.cpp.
References llvm::CallingConv::AMDGPU_PS, assert(), llvm::ISD::InputArg::Flags, llvm::ISD::InputArg::getOrigArgIndex(), llvm::MVT::getScalarSizeInBits(), I, Info, llvm::ISD::ArgFlagsTy::isInReg(), llvm::ISD::ArgFlagsTy::isSplit(), llvm::ISD::ArgFlagsTy::isSplitEnd(), llvm::MVT::isVector(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::BitVector::set(), llvm::ISD::InputArg::Used, and llvm::ISD::InputArg::VT.
Referenced by llvm::SITargetLowering::LowerFormalArguments().
|
static |
Definition at line 2663 of file SIISelLowering.cpp.
References assert(), llvm::Function::getCallingConv(), llvm::MachineFunction::getFrameInfo(), llvm::MachineFunction::getFunction(), llvm::MachineFunction::getRegInfo(), llvm::MachineFunction::getSubtarget(), llvm::MachineFrameInfo::hasCalls(), llvm::MachineFrameInfo::hasStackObjects(), Info, llvm::AMDGPU::isShader(), MRI, llvm::None, llvm::AMDGPUFunctionArgInfo::PRIVATE_SEGMENT_BUFFER, llvm::report_fatal_error(), TM, and TRI.
Referenced by llvm::SITargetLowering::finalizeLowering().
|
static |
Definition at line 13898 of file SIISelLowering.cpp.
References addPermMasks(), assert(), llvm::SmallVectorTemplateCommon< T, typename >::begin(), llvm::SmallVectorTemplateCommon< T, typename >::end(), llvm::SelectionDAG::getConstant(), getDWordFromOffset(), llvm::SelectionDAG::getNode(), llvm::ISD::OR, llvm::AMDGPUISD::PERM, llvm::SmallVectorTemplateBase< T, bool >::push_back(), and llvm::SmallVectorBase< Size_T >::size().
|
static |
Definition at line 8777 of file SIISelLowering.cpp.
References llvm::SelectionDAG::getRegister(), llvm::GCNSubtarget::hasRestrictedSOffset(), and llvm::isNullConstant().
|
static |
Definition at line 4617 of file SIISelLowering.cpp.
References llvm::MachineInstrBuilder::add(), llvm::MachineInstrBuilder::addImm(), assert(), llvm::BuildMI(), DL, I, Idx, MBB, MI, llvm::Offset, and TII.
Referenced by emitIndirectDst(), and emitIndirectSrc().
|
static |
Definition at line 4360 of file SIISelLowering.cpp.
References llvm::MachineBasicBlock::addSuccessor(), llvm::MachineBasicBlock::begin(), llvm::MachineFunction::CreateMachineBasicBlock(), llvm::MachineBasicBlock::end(), llvm::MachineBasicBlock::getParent(), I, llvm::MachineFunction::insert(), MBB, MBBI, MI, llvm::MachineBasicBlock::splice(), and llvm::MachineBasicBlock::transferSuccessorsAndUpdatePHIs().
Referenced by llvm::SITargetLowering::emitGWSMemViolTestLoop(), loadM0FromVGPR(), and lowerWaveReduce().
STATISTIC | ( | NumTailCalls | , |
"Number of tail calls" | |||
) |
|
static |
Return the source of an fp_extend from f16 to f32, or a converted FP constant.
Definition at line 13525 of file SIISelLowering.cpp.
References llvm::APFloat::convert(), llvm::ISD::FP_EXTEND, llvm::SelectionDAG::getConstantFP(), llvm::APFloatBase::IEEEhalf(), and llvm::APFloatBase::rmNearestTiesToEven.
Helper function for adjustWritemask.
Definition at line 14809 of file SIISelLowering.cpp.
References Idx.
|
static |
Opc
for type VT
. Definition at line 13153 of file SIISelLowering.cpp.
References llvm::AMDGPUISD::FMAX_LEGACY, llvm::ISD::FMAXIMUM, llvm::ISD::FMAXNUM, llvm::ISD::FMAXNUM_IEEE, llvm::AMDGPUISD::FMIN_LEGACY, llvm::ISD::FMINIMUM, llvm::ISD::FMINNUM, llvm::ISD::FMINNUM_IEEE, llvm::GCNSubtarget::hasIEEEMinMax3(), llvm::GCNSubtarget::hasMin3Max3_16(), llvm_unreachable, llvm::ISD::SMAX, llvm::ISD::SMIN, llvm::ISD::UMAX, and llvm::ISD::UMIN.
Definition at line 12953 of file SIISelLowering.cpp.
|
static |
Referenced by llvm::SITargetLowering::getPrefLoopAlignment().
|
static |
Referenced by llvm::SITargetLowering::shouldExpandVectorDynExt().