LLVM 20.0.0git
|
Custom DAG lowering for SI. More...
#include "SIISelLowering.h"
#include "AMDGPU.h"
#include "AMDGPUInstrInfo.h"
#include "AMDGPUTargetMachine.h"
#include "GCNSubtarget.h"
#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
#include "SIMachineFunctionInfo.h"
#include "SIRegisterInfo.h"
#include "llvm/ADT/APInt.h"
#include "llvm/ADT/FloatingPointMode.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/Analysis/OptimizationRemarkEmitter.h"
#include "llvm/Analysis/UniformityAnalysis.h"
#include "llvm/BinaryFormat/ELF.h"
#include "llvm/CodeGen/Analysis.h"
#include "llvm/CodeGen/ByteProvider.h"
#include "llvm/CodeGen/FunctionLoweringInfo.h"
#include "llvm/CodeGen/GlobalISel/GISelKnownBits.h"
#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineLoopInfo.h"
#include "llvm/IR/DiagnosticInfo.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/IntrinsicsAMDGPU.h"
#include "llvm/IR/IntrinsicsR600.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/KnownBits.h"
#include "llvm/Support/ModRef.h"
#include "llvm/Transforms/Utils/LowerAtomic.h"
#include <optional>
Go to the source code of this file.
Classes | |
struct | DotSrc |
Macros | |
#define | DEBUG_TYPE "si-lower" |
Variables | |
static cl::opt< bool > | DisableLoopAlignment ("amdgpu-disable-loop-alignment", cl::desc("Do not align and prefetch loops"), cl::init(false)) |
static cl::opt< bool > | UseDivergentRegisterIndexing ("amdgpu-use-divergent-register-indexing", cl::Hidden, cl::desc("Use indirect register addressing for divergent indexes"), cl::init(false)) |
Custom DAG lowering for SI.
Definition in file SIISelLowering.cpp.
#define DEBUG_TYPE "si-lower" |
Definition at line 50 of file SIISelLowering.cpp.
Definition at line 13800 of file SIISelLowering.cpp.
References assert(), and llvm::First.
Referenced by placeSources(), and resolveSources().
|
static |
Definition at line 12092 of file SIISelLowering.cpp.
References assert().
Referenced by hasNon16BitAccesses().
|
static |
Definition at line 10246 of file SIISelLowering.cpp.
References Info.
|
static |
Definition at line 5897 of file SIISelLowering.cpp.
References llvm::ISD::BITCAST, llvm::EVT::changeTypeToInteger(), DL, llvm::SelectionDAG::ExtractVectorElements(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getUNDEF(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::EVT::isVector(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), and llvm::ISD::TRUNCATE.
Referenced by constructRetValue().
|
static |
Definition at line 2338 of file SIISelLowering.cpp.
References llvm::MachineFunction::addLiveIn(), llvm::CCState::AllocateReg(), assert(), and llvm::CCState::getMachineFunction().
Referenced by allocateSGPR32Input(), and allocateSGPR64Input().
|
static |
Definition at line 2347 of file SIISelLowering.cpp.
References allocateFixedSGPRInputImpl(), allocateSGPR32InputImpl(), and llvm::ArgDescriptor::getRegister().
Referenced by llvm::SITargetLowering::allocateSpecialInputSGPRs().
|
static |
Definition at line 2318 of file SIISelLowering.cpp.
References llvm::MachineFunction::addLiveIn(), llvm::CCState::AllocateReg(), assert(), llvm::TargetRegisterClass::begin(), llvm::ArgDescriptor::createRegister(), llvm::CCState::getFirstUnallocated(), llvm::CCState::getMachineFunction(), llvm::report_fatal_error(), and llvm::ArrayRef< T >::size().
Referenced by allocateSGPR32Input(), and allocateSGPR64Input().
|
static |
Definition at line 2355 of file SIISelLowering.cpp.
References allocateFixedSGPRInputImpl(), allocateSGPR32InputImpl(), and llvm::ArgDescriptor::getRegister().
Referenced by llvm::SITargetLowering::allocateSpecialInputSGPRs().
|
static |
Definition at line 2294 of file SIISelLowering.cpp.
References llvm::MachineFunction::addLiveIn(), llvm::CCState::AllocateReg(), llvm::CCState::AllocateStack(), assert(), llvm::ArgDescriptor::createArg(), llvm::ArgDescriptor::createRegister(), llvm::ArgDescriptor::createStack(), llvm::CCState::getFirstUnallocated(), llvm::CCState::getMachineFunction(), llvm::MachineFunction::getRegInfo(), llvm::Offset, llvm::LLT::scalar(), llvm::MachineRegisterInfo::setType(), and llvm::ArrayRef< T >::size().
Referenced by llvm::SITargetLowering::allocateSpecialInputVGPRs().
|
static |
Definition at line 16067 of file SIISelLowering.cpp.
References Flt, llvm::Function::getDenormalMode(), llvm::Type::getFltSemantics(), llvm::Function::getFnAttribute(), llvm::Instruction::getFunction(), llvm::DenormalMode::getPreserveSign(), llvm::Type::getScalarType(), llvm::Value::getType(), llvm::Attribute::getValueAsBool(), and llvm::Instruction::hasMetadata().
Referenced by llvm::SITargetLowering::shouldExpandAtomicRMWInIR().
|
static |
Definition at line 16172 of file SIISelLowering.cpp.
References llvm::TargetLoweringBase::CmpXChg, llvm::Value::getType(), isAtomicRMWLegalIntTy(), and llvm::TargetLoweringBase::None.
Referenced by llvm::SITargetLowering::shouldExpandAtomicRMWInIR().
Definition at line 11396 of file SIISelLowering.cpp.
References llvm::ISD::AND, llvm::ISD::OR, and llvm::ISD::XOR.
|
static |
Definition at line 7575 of file SIISelLowering.cpp.
References assert(), DL, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getTargetGlobalAddress(), llvm::SIInstrInfo::MO_NONE, llvm::Offset, and llvm::AMDGPUISD::PC_ADD_REL_OFFSET.
|
static |
Definition at line 15285 of file SIISelLowering.cpp.
References DL, llvm::SelectionDAG::getMachineNode(), and llvm::SelectionDAG::getTargetConstant().
Referenced by llvm::SITargetLowering::buildRSRC(), and llvm::SITargetLowering::wrapAddr64Rsrc().
|
static |
Definition at line 11802 of file SIISelLowering.cpp.
References llvm::ISD::AND, llvm::ISD::ANY_EXTEND, llvm::ISD::AssertSext, llvm::ISD::AssertZext, llvm::BitWidth, llvm::ISD::BSWAP, calculateByteProvider(), calculateSrcByte(), llvm::ISD::CopyFromReg, llvm::Depth, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::ISD::FSHR, llvm::ByteProvider< ISelOp >::getConstantZero(), llvm::SDValue::getValueSizeInBits(), LHS, llvm_unreachable, llvm::ISD::LOAD, llvm::ISD::OR, llvm::AMDGPUISD::PERM, RHS, llvm::ISD::SHL, llvm::ISD::SIGN_EXTEND, llvm::ISD::SIGN_EXTEND_INREG, llvm::ISD::SRA, llvm::ISD::SRL, llvm::ISD::TRUNCATE, llvm::ISD::ZERO_EXTEND, and llvm::ISD::ZEXTLOAD.
|
static |
Definition at line 11737 of file SIISelLowering.cpp.
References calculateSrcByte(), llvm::Depth, llvm::ByteProvider< ISelOp >::getSrc(), llvm::EVT::getStoreSize(), llvm::SDValue::getValueType(), llvm::EVT::isByteSized(), llvm_unreachable, llvm::ISD::SIGN_EXTEND, llvm::ISD::SIGN_EXTEND_INREG, llvm::ISD::SRA, llvm::ISD::SRL, llvm::ISD::TRUNCATE, and llvm::ISD::ZERO_EXTEND.
Referenced by calculateByteProvider(), and calculateSrcByte().
|
static |
Definition at line 3511 of file SIISelLowering.cpp.
References CC, and llvm::CallingConv::Fast.
|
static |
Definition at line 13982 of file SIISelLowering.cpp.
References assert(), llvm::SelectionDAG::computeKnownBits(), llvm::KnownBits::countMinLeadingZeros(), llvm::SDValue::getValueSizeInBits(), and llvm_unreachable.
Definition at line 15504 of file SIISelLowering.cpp.
References llvm::AMDGPU::isInlinableIntLiteral(), and Size.
Referenced by llvm::APInt::APInt(), llvm::APInt::ashrInPlace(), llvm::SITargetLowering::checkAsmConstraintVal(), llvm::APInt::flipAllBits(), llvm::SITargetLowering::LowerAsmOperandForConstraint(), llvm::APInt::operator<<=(), llvm::APInt::operator=(), llvm::APInt::operator^=(), llvm::APInt::operator|=(), and llvm::APInt::setAllBits().
|
static |
Definition at line 4604 of file SIISelLowering.cpp.
References llvm::SIRegisterInfo::getSubRegFromChannel(), llvm::Offset, and TRI.
Referenced by emitIndirectDst(), and emitIndirectSrc().
|
static |
Definition at line 7780 of file SIISelLowering.cpp.
References adjustLoadValueTypeImpl(), llvm::ISD::BITCAST, llvm::EVT::changeTypeToInteger(), llvm::Data, DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::MVT::getVectorVT(), llvm::EVT::isVector(), llvm::MVT::isVector(), padEltsToUndef(), and llvm::ISD::TRUNCATE.
|
static |
Definition at line 65 of file SIISelLowering.cpp.
References llvm::MachineFunction::getInfo(), llvm::DenormalMode::getPreserveSign(), and Info.
Referenced by llvm::SITargetLowering::denormalsEnabledForType(), llvm::SITargetLowering::isFMADLegal(), llvm::SITargetLowering::isFMAFasterThanFMulAndFAdd(), and llvm::SITargetLowering::isFPExtFoldable().
|
static |
Definition at line 70 of file SIISelLowering.cpp.
References llvm::MachineFunction::getInfo(), llvm::DenormalMode::getPreserveSign(), and Info.
Referenced by llvm::SITargetLowering::denormalsEnabledForType(), llvm::SITargetLowering::isFMADLegal(), and llvm::SITargetLowering::isFMAFasterThanFMulAndFAdd().
Definition at line 7382 of file SIISelLowering.cpp.
References assert().
|
static |
Definition at line 16082 of file SIISelLowering.cpp.
References DEBUG_TYPE, llvm::SmallVectorBase< Size_T >::empty(), llvm::Value::getContext(), llvm::AtomicRMWInst::getOperation(), llvm::AtomicRMWInst::getOperationName(), llvm::AtomicRMWInst::getSyncScopeID(), and llvm::LLVMContext::getSyncScopeNames().
Referenced by llvm::SITargetLowering::shouldExpandAtomicRMWInIR().
|
static |
Definition at line 4743 of file SIISelLowering.cpp.
References llvm::MachineInstrBuilder::add(), llvm::MachineInstrBuilder::addImm(), llvm::MachineInstrBuilder::addReg(), assert(), llvm::BuildMI(), computeIndirectRegAndOffset(), DL, getIndirectSGPRIdx(), llvm::MachineBasicBlock::getParent(), llvm::MachineOperand::getReg(), llvm::MachineFunction::getRegInfo(), I, Idx, llvm::MachineOperand::isReg(), loadM0FromVGPR(), MBB, MI, MRI, llvm::Offset, setM0ToIndexFromSGPR(), SubReg, TII, and TRI.
Referenced by llvm::SITargetLowering::EmitInstrWithCustomInserter().
|
static |
Definition at line 4657 of file SIISelLowering.cpp.
References llvm::MachineInstrBuilder::addImm(), llvm::MachineInstrBuilder::addReg(), llvm::BuildMI(), computeIndirectRegAndOffset(), DL, getIndirectSGPRIdx(), llvm::MachineBasicBlock::getParent(), llvm::MachineFunction::getRegInfo(), I, Idx, llvm::RegState::Implicit, loadM0FromVGPR(), MBB, MI, MRI, llvm::Offset, setM0ToIndexFromSGPR(), SubReg, TII, and TRI.
Referenced by llvm::SITargetLowering::EmitInstrWithCustomInserter().
|
static |
Definition at line 4463 of file SIISelLowering.cpp.
References llvm::MachineInstrBuilder::addImm(), llvm::MachineInstrBuilder::addMBB(), llvm::MachineInstrBuilder::addReg(), llvm::MachineBasicBlock::begin(), llvm::BuildMI(), DL, llvm::ilist_node_impl< OptionsT >::getIterator(), llvm::MachineBasicBlock::getParent(), llvm::MachineFunction::getSubtarget(), llvm::getUndefRegState(), I, Idx, llvm::RegState::Kill, MRI, llvm::Offset, TII, and TRI.
Referenced by loadM0FromVGPR().
|
static |
Definition at line 7712 of file SIISelLowering.cpp.
References llvm::LLVMContext::diagnose(), DL, llvm::SelectionDAG::getContext(), llvm::MachineFunction::getFunction(), llvm::SelectionDAG::getMachineFunction(), and llvm::SelectionDAG::getUNDEF().
|
static |
Definition at line 7721 of file SIISelLowering.cpp.
References llvm::LLVMContext::diagnose(), DL, llvm::SelectionDAG::getContext(), llvm::MachineFunction::getFunction(), llvm::SelectionDAG::getMachineFunction(), and llvm::SelectionDAG::getUNDEF().
Definition at line 75 of file SIISelLowering.cpp.
References llvm::CCState::isAllocated(), and llvm_unreachable.
Referenced by llvm::SITargetLowering::allocateSystemSGPRs().
Helper function for LowerBRCOND.
Definition at line 6479 of file SIISelLowering.cpp.
References I, llvm::SDNode::use_begin(), and llvm::SDNode::use_end().
|
static |
Definition at line 13966 of file SIISelLowering.cpp.
|
static |
Definition at line 15622 of file SIISelLowering.cpp.
Referenced by llvm::SITargetLowering::finalizeLowering().
MemSDNode::getBasePtr() does not work for intrinsics, which needs to offset by the chain and intrinsic ID.
Theoretically we would also need to check the specific intrinsic, but they all place the pointer operand first.
Definition at line 11362 of file SIISelLowering.cpp.
References llvm::ISD::INTRINSIC_VOID, llvm::ISD::INTRINSIC_W_CHAIN, N, and llvm::ISD::STORE.
Referenced by llvm::SITargetLowering::hasMemSDNodeUser().
|
static |
Definition at line 7730 of file SIISelLowering.cpp.
References assert(), DL, llvm::ArrayRef< T >::empty(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::MVT::getVectorVT(), and llvm::ArrayRef< T >::size().
Definition at line 11448 of file SIISelLowering.cpp.
References llvm::CallingConv::C.
Referenced by getPermuteMask().
|
static |
Scalar Type
Definition at line 12133 of file SIISelLowering.cpp.
References assert(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::ExtractVectorElements(), llvm::SelectionDAG::getBitcastedAnyExtOrTrunc(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::MVT::getVectorVT(), and llvm::ISD::SRL.
Referenced by matchPERM(), and resolveSources().
|
static |
Definition at line 10529 of file SIISelLowering.cpp.
References A, assert(), B, llvm::ISD::FMUL, llvm::AMDGPUISD::FMUL_W_CHAIN, llvm::SelectionDAG::getNode(), llvm::SDNode::getNumValues(), llvm::SDValue::getValue(), llvm::SelectionDAG::getVTList(), and llvm_unreachable.
|
static |
Definition at line 10551 of file SIISelLowering.cpp.
References A, assert(), B, llvm::CallingConv::C, llvm::ISD::FMA, llvm::AMDGPUISD::FMA_W_CHAIN, llvm::SelectionDAG::getNode(), llvm::SDNode::getNumValues(), llvm::SDValue::getValue(), llvm::SelectionDAG::getVTList(), and llvm_unreachable.
|
static |
Definition at line 4638 of file SIISelLowering.cpp.
References llvm::MachineInstrBuilder::add(), llvm::MachineInstrBuilder::addImm(), llvm::BuildMI(), DL, I, Idx, MBB, MI, MRI, llvm::Offset, and TII.
Referenced by emitIndirectDst(), and emitIndirectSrc().
|
static |
Definition at line 10157 of file SIISelLowering.cpp.
References llvm::ISD::ANY_EXTEND, llvm::EVT::bitsLT(), llvm::ISD::EXTLOAD, llvm::SelectionDAG::getNode(), llvm_unreachable, llvm::ISD::NON_EXTLOAD, llvm::ISD::SEXTLOAD, llvm::ISD::SIGN_EXTEND, llvm::ISD::TRUNCATE, llvm::ISD::ZERO_EXTEND, and llvm::ISD::ZEXTLOAD.
|
static |
Definition at line 13649 of file SIISelLowering.cpp.
References llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getVTList(), llvm::AMDGPUISD::MAD_I64_I32, llvm::AMDGPUISD::MAD_U64_U32, Signed, and llvm::ISD::TRUNCATE.
Definition at line 11468 of file SIISelLowering.cpp.
References llvm::ISD::AND, assert(), llvm::CallingConv::C, getConstantPermuteMask(), llvm::ConstantSDNode::getZExtValue(), llvm::ISD::OR, llvm::ISD::SHL, and llvm::ISD::SRL.
|
static |
Definition at line 10629 of file SIISelLowering.cpp.
References assert(), llvm::SelectionDAG::getTargetConstant(), and Info.
|
static |
Definition at line 13088 of file SIISelLowering.cpp.
References llvm::CallingConv::C.
|
static |
RMW
, based on the properties of the target memory. Definition at line 16151 of file SIISelLowering.cpp.
References llvm::Instruction::hasMetadata(), and llvm::GCNSubtarget::supportsAgentScopeFineGrainedRemoteMemoryAtomics().
Referenced by llvm::SITargetLowering::shouldExpandAtomicRMWInIR().
|
static |
Definition at line 13788 of file SIISelLowering.cpp.
References calculateByteProvider().
|
static |
Definition at line 16440 of file SIISelLowering.cpp.
References hasCFUser(), llvm::SmallPtrSetImpl< PtrType >::insert(), and IT.
Referenced by hasCFUser(), and llvm::SITargetLowering::requiresUniformRegister().
Definition at line 12111 of file SIISelLowering.cpp.
References addresses16Bits(), isExtendedFrom16Bits(), and llvm::peekThroughBitcasts().
Referenced by matchPERM().
Definition at line 16117 of file SIISelLowering.cpp.
References IT.
Referenced by atomicSupportedIfLegalIntType(), and isAtomicRMWLegalXChgTy().
|
static |
Definition at line 16127 of file SIISelLowering.cpp.
References DL, llvm::Module::getDataLayout(), llvm::Instruction::getFunction(), llvm::GlobalValue::getParent(), llvm::Value::getType(), isAtomicRMWLegalIntTy(), llvm::Type::isDoubleTy(), and llvm::Type::isFloatTy().
Referenced by llvm::SITargetLowering::shouldExpandAtomicRMWInIR().
Definition at line 16111 of file SIISelLowering.cpp.
References llvm::VectorType::getElementType(), llvm::FixedVectorType::getNumElements(), and llvm::Type::isBFloatTy().
Referenced by llvm::SITargetLowering::shouldExpandAtomicRMWInIR().
Definition at line 13257 of file SIISelLowering.cpp.
|
static |
Definition at line 15949 of file SIISelLowering.cpp.
References assert(), llvm::ISD::CopyFromReg, llvm::ISD::INLINEASM, llvm::ISD::INLINEASM_BR, and N.
Referenced by llvm::SITargetLowering::isSDNodeSourceOfDivergence().
Definition at line 12066 of file SIISelLowering.cpp.
References llvm::ISD::ANY_EXTEND, llvm::ISD::EXTLOAD, llvm::SDValue::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::isVector(), llvm::ISD::LOAD, llvm::ISD::SEXTLOAD, llvm::ISD::SIGN_EXTEND, llvm::ISD::ZERO_EXTEND, and llvm::ISD::ZEXTLOAD.
Referenced by hasNon16BitAccesses().
Definition at line 14990 of file SIISelLowering.cpp.
References llvm::ISD::AssertZext.
Referenced by llvm::SITargetLowering::legalizeTargetIndependentNode().
Definition at line 16106 of file SIISelLowering.cpp.
References llvm::VectorType::getElementType(), llvm::FixedVectorType::getNumElements(), and llvm::Type::isHalfTy().
Referenced by llvm::SITargetLowering::shouldExpandAtomicRMWInIR().
Definition at line 16096 of file SIISelLowering.cpp.
References llvm::Type::isBFloatTy(), and llvm::Type::isHalfTy().
Referenced by llvm::SITargetLowering::shouldExpandAtomicRMWInIR().
Definition at line 15469 of file SIISelLowering.cpp.
References llvm::StringRef::size().
Referenced by llvm::SITargetLowering::getConstraintType(), and llvm::SITargetLowering::LowerAsmOperandForConstraint().
|
static |
Return true if the value is a known valid address, such that a null check is not necessary.
Definition at line 7040 of file SIISelLowering.cpp.
References TM.
Definition at line 13974 of file SIISelLowering.cpp.
References llvm::ISD::MUL, llvm::AMDGPUISD::MUL_I24, and llvm::AMDGPUISD::MUL_U24.
|
static |
Definition at line 15783 of file SIISelLowering.cpp.
References llvm::countl_zero(), llvm::MachineFunction::getFunction(), llvm::GISelKnownBits::getMachineFunction(), llvm::APInt::setHighBits(), and llvm::KnownBits::Zero.
Referenced by llvm::SITargetLowering::computeKnownBitsForTargetInstr().
|
static |
Definition at line 4555 of file SIISelLowering.cpp.
References llvm::MachineInstrBuilder::addReg(), llvm::MachineBasicBlock::addSuccessor(), llvm::MachineBasicBlock::begin(), llvm::BuildMI(), llvm::MachineFunction::CreateMachineBasicBlock(), DL, emitLoadM0FromVGPRLoop(), llvm::First, llvm::MachineBasicBlock::getParent(), llvm::MachineFunction::getRegInfo(), llvm::MachineFunction::getSubtarget(), I, Idx, llvm::MachineFunction::insert(), MBB, MBBI, MI, MRI, llvm::Offset, llvm::MachineBasicBlock::removeSuccessor(), splitBlockForLoop(), TII, and TRI.
Referenced by emitIndirectDst(), and emitIndirectSrc().
|
static |
Definition at line 6079 of file SIISelLowering.cpp.
References llvm::SelectionDAG::getCondCode(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getCopyFromReg(), llvm::SelectionDAG::getEntryNode(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), llvm::SelectionDAG::getZExtOrTrunc(), N, llvm::ISD::SETCC, llvm::AMDGPUISD::SETCC, and llvm::ISD::SETNE.
|
static |
Definition at line 6050 of file SIISelLowering.cpp.
References llvm::EVT::bitsEq(), llvm::ISD::FP_EXTEND, llvm::SelectionDAG::getCondCode(), llvm::SelectionDAG::getContext(), llvm::getFCmpCondCode(), llvm::EVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::SITargetLowering::getSubtarget(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::AMDGPUSubtarget::getWavefrontSize(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::CmpInst::isFPPredicate(), llvm::TargetLoweringBase::isTypeLegal(), N, and llvm::AMDGPUISD::SETCC.
|
static |
Definition at line 6016 of file SIISelLowering.cpp.
References llvm::EVT::bitsEq(), DL, llvm::SelectionDAG::getCondCode(), llvm::SelectionDAG::getContext(), llvm::getICmpCondCode(), llvm::EVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::SITargetLowering::getSubtarget(), llvm::SelectionDAG::getUNDEF(), llvm::AMDGPUSubtarget::getWavefrontSize(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::CmpInst::isIntPredicate(), llvm::CmpInst::isSigned(), llvm::TargetLoweringBase::isTypeLegal(), LHS, N, RHS, llvm::AMDGPUISD::SETCC, llvm::ISD::SIGN_EXTEND, and llvm::ISD::ZERO_EXTEND.
|
static |
Definition at line 6116 of file SIISelLowering.cpp.
References assert(), llvm::ISD::CONCAT_VECTORS, llvm::ISD::CONVERGENCECTRL_GLUE, llvm::ISD::EXTRACT_SUBVECTOR, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getAnyExtOrTrunc(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::MVT::getIntegerVT(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::EVT::getSimpleVT(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetConstant(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::SelectionDAG::getVectorIdxConstant(), llvm::EVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::MVT::getVectorVT(), llvm::ISD::INTRINSIC_WO_CHAIN, llvm::EVT::isFloatingPoint(), llvm::EVT::isVector(), llvm_unreachable, N, Operands, llvm::SmallVectorTemplateBase< T, bool >::push_back(), and llvm::MVT::SimpleTy.
|
static |
Definition at line 4847 of file SIISelLowering.cpp.
References Accumulator, llvm::MachineInstrBuilder::addImm(), llvm::MachineInstrBuilder::addMBB(), llvm::MachineInstrBuilder::addReg(), llvm::BuildMI(), DL, llvm::MachineBasicBlock::end(), llvm::MachineBasicBlock::getParent(), llvm::MachineFunction::getRegInfo(), I, MI, MRI, splitBlockForLoop(), TII, and TRI.
Referenced by llvm::SITargetLowering::EmitInstrWithCustomInserter().
|
static |
Definition at line 12188 of file SIISelLowering.cpp.
References assert(), calculateByteProvider(), llvm::TargetLowering::DAGCombinerInfo::DAG, DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBitcastedAnyExtOrTrunc(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getDataLayout(), getDWordFromOffset(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), hasNon16BitAccesses(), llvm::DataLayout::isBigEndian(), llvm::EVT::isByteSized(), N, P, llvm::AMDGPUISD::PERM, llvm::SmallVectorTemplateBase< T, bool >::push_back(), and llvm::SmallVectorBase< Size_T >::size().
Referenced by llvm::SITargetLowering::PerformDAGCombine().
|
static |
Return true if we might ever do TCO for calls with this calling convention.
Definition at line 3516 of file SIISelLowering.cpp.
References llvm::CallingConv::AMDGPU_Gfx, llvm::CallingConv::C, canGuaranteeTCO(), and CC.
|
static |
Definition at line 1118 of file SIISelLowering.cpp.
References assert(), DL, llvm::Type::getContext(), llvm::TargetLoweringBase::getValueType(), and llvm::EVT::getVectorVT().
Referenced by llvm::SITargetLowering::getTgtMemIntrinsic(), and memVTFromLoadIntrReturn().
|
static |
Definition at line 1134 of file SIISelLowering.cpp.
References assert(), DL, and memVTFromLoadIntrData().
Referenced by llvm::SITargetLowering::getTgtMemIntrinsic().
Definition at line 13025 of file SIISelLowering.cpp.
References llvm::AMDGPUISD::FMAX3, llvm::ISD::FMAXIMUM, llvm::AMDGPUISD::FMAXIMUM3, llvm::ISD::FMAXNUM, llvm::ISD::FMAXNUM_IEEE, llvm::AMDGPUISD::FMIN3, llvm::ISD::FMINIMUM, llvm::AMDGPUISD::FMINIMUM3, llvm::ISD::FMINNUM, llvm::ISD::FMINNUM_IEEE, llvm_unreachable, llvm::ISD::SMAX, llvm::AMDGPUISD::SMAX3, llvm::ISD::SMIN, llvm::AMDGPUISD::SMIN3, llvm::ISD::UMAX, llvm::AMDGPUISD::UMAX3, llvm::ISD::UMIN, and llvm::AMDGPUISD::UMIN3.
|
static |
Definition at line 7870 of file SIISelLowering.cpp.
References Addr, llvm::ISD::ANY_EXTEND, DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getNode(), I, and llvm::SmallVectorTemplateBase< T, bool >::push_back().
|
static |
Definition at line 7759 of file SIISelLowering.cpp.
References DL, llvm::SelectionDAG::ExtractVectorElements(), llvm::SelectionDAG::getBuildVector(), llvm::EVT::getScalarType(), llvm::SelectionDAG::getUNDEF(), llvm::EVT::isVector(), and llvm::SmallVectorTemplateBase< T, bool >::push_back().
Referenced by constructRetValue().
|
static |
Definition at line 7852 of file SIISelLowering.cpp.
References DL, llvm::SDValue::getNode(), and llvm::SelectionDAG::getTargetConstant().
|
static |
Definition at line 13820 of file SIISelLowering.cpp.
References addPermMasks(), assert(), llvm::SmallVectorTemplateCommon< T, typename >::end(), llvm::find_if(), I, llvm::Match, llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::ByteProvider< ISelOp >::Src, and llvm::ByteProvider< ISelOp >::SrcOffset.
|
static |
Definition at line 2195 of file SIISelLowering.cpp.
References llvm::CallingConv::AMDGPU_PS, assert(), llvm::ISD::InputArg::Flags, llvm::ISD::InputArg::getOrigArgIndex(), llvm::MVT::getScalarSizeInBits(), I, Info, llvm::ISD::ArgFlagsTy::isInReg(), llvm::ISD::ArgFlagsTy::isSplit(), llvm::ISD::ArgFlagsTy::isSplitEnd(), llvm::MVT::isVector(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::BitVector::set(), llvm::ISD::InputArg::Used, and llvm::ISD::InputArg::VT.
Referenced by llvm::SITargetLowering::LowerFormalArguments().
|
static |
Definition at line 2664 of file SIISelLowering.cpp.
References assert(), llvm::Function::getCallingConv(), llvm::MachineFunction::getFrameInfo(), llvm::MachineFunction::getFunction(), llvm::MachineFunction::getRegInfo(), llvm::MachineFunction::getSubtarget(), llvm::MachineFrameInfo::hasCalls(), llvm::MachineFrameInfo::hasStackObjects(), Info, llvm::AMDGPU::isShader(), MRI, llvm::None, llvm::AMDGPUFunctionArgInfo::PRIVATE_SEGMENT_BUFFER, llvm::report_fatal_error(), TM, and TRI.
Referenced by llvm::SITargetLowering::finalizeLowering().
|
static |
Definition at line 13898 of file SIISelLowering.cpp.
References addPermMasks(), assert(), llvm::SmallVectorTemplateCommon< T, typename >::begin(), llvm::SmallVectorTemplateCommon< T, typename >::end(), llvm::SelectionDAG::getConstant(), getDWordFromOffset(), llvm::SelectionDAG::getNode(), llvm::ISD::OR, llvm::AMDGPUISD::PERM, llvm::SmallVectorTemplateBase< T, bool >::push_back(), and llvm::SmallVectorBase< Size_T >::size().
|
static |
Definition at line 8778 of file SIISelLowering.cpp.
References llvm::SelectionDAG::getRegister(), llvm::GCNSubtarget::hasRestrictedSOffset(), and llvm::isNullConstant().
|
static |
Definition at line 4618 of file SIISelLowering.cpp.
References llvm::MachineInstrBuilder::add(), llvm::MachineInstrBuilder::addImm(), assert(), llvm::BuildMI(), DL, I, Idx, MBB, MI, llvm::Offset, and TII.
Referenced by emitIndirectDst(), and emitIndirectSrc().
|
static |
Definition at line 4361 of file SIISelLowering.cpp.
References llvm::MachineBasicBlock::addSuccessor(), llvm::MachineBasicBlock::begin(), llvm::MachineFunction::CreateMachineBasicBlock(), llvm::MachineBasicBlock::end(), llvm::MachineBasicBlock::getParent(), I, llvm::MachineFunction::insert(), MBB, MBBI, MI, llvm::MachineBasicBlock::splice(), and llvm::MachineBasicBlock::transferSuccessorsAndUpdatePHIs().
Referenced by llvm::SITargetLowering::emitGWSMemViolTestLoop(), loadM0FromVGPR(), and lowerWaveReduce().
STATISTIC | ( | NumTailCalls | , |
"Number of tail calls" | |||
) |
|
static |
Return the source of an fp_extend from f16 to f32, or a converted FP constant.
Definition at line 13525 of file SIISelLowering.cpp.
References llvm::APFloat::convert(), llvm::ISD::FP_EXTEND, llvm::SelectionDAG::getConstantFP(), llvm::APFloatBase::IEEEhalf(), and llvm::APFloatBase::rmNearestTiesToEven.
Helper function for adjustWritemask.
Definition at line 14809 of file SIISelLowering.cpp.
References Idx.
|
static |
Opc
for type VT
. Definition at line 13153 of file SIISelLowering.cpp.
References llvm::AMDGPUISD::FMAX_LEGACY, llvm::ISD::FMAXIMUM, llvm::ISD::FMAXNUM, llvm::ISD::FMAXNUM_IEEE, llvm::AMDGPUISD::FMIN_LEGACY, llvm::ISD::FMINIMUM, llvm::ISD::FMINNUM, llvm::ISD::FMINNUM_IEEE, llvm::GCNSubtarget::hasIEEEMinMax3(), llvm::GCNSubtarget::hasMin3Max3_16(), llvm_unreachable, llvm::ISD::SMAX, llvm::ISD::SMIN, llvm::ISD::UMAX, and llvm::ISD::UMIN.
Definition at line 12953 of file SIISelLowering.cpp.
|
static |
Referenced by llvm::SITargetLowering::getPrefLoopAlignment().
|
static |
Referenced by llvm::SITargetLowering::shouldExpandVectorDynExt().