LLVM
15.0.0git
|
#include "AMDGPULegalizerInfo.h"
#include "AMDGPU.h"
#include "AMDGPUGlobalISelUtils.h"
#include "AMDGPUInstrInfo.h"
#include "AMDGPUTargetMachine.h"
#include "SIMachineFunctionInfo.h"
#include "Utils/AMDGPUBaseInfo.h"
#include "llvm/ADT/ScopeExit.h"
#include "llvm/BinaryFormat/ELF.h"
#include "llvm/CodeGen/GlobalISel/LegalizerHelper.h"
#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
#include "llvm/IR/DiagnosticInfo.h"
#include "llvm/IR/IntrinsicsAMDGPU.h"
#include "llvm/IR/IntrinsicsR600.h"
Go to the source code of this file.
Macros | |
#define | DEBUG_TYPE "amdgpu-legalinfo" |
Functions | |
static LLT | getPow2VectorType (LLT Ty) |
static LLT | getPow2ScalarType (LLT Ty) |
static LegalityPredicate | isSmallOddVector (unsigned TypeIdx) |
static LegalityPredicate | sizeIsMultipleOf32 (unsigned TypeIdx) |
static LegalityPredicate | isWideVec16 (unsigned TypeIdx) |
static LegalizeMutation | oneMoreElement (unsigned TypeIdx) |
static LegalizeMutation | fewerEltsToSize64Vector (unsigned TypeIdx) |
static LegalizeMutation | moreEltsToNext32Bit (unsigned TypeIdx) |
static LLT | getBitcastRegisterType (const LLT Ty) |
static LegalizeMutation | bitcastToRegisterType (unsigned TypeIdx) |
static LegalizeMutation | bitcastToVectorElement32 (unsigned TypeIdx) |
static LegalityPredicate | vectorSmallerThan (unsigned TypeIdx, unsigned Size) |
static LegalityPredicate | vectorWiderThan (unsigned TypeIdx, unsigned Size) |
static LegalityPredicate | numElementsNotEven (unsigned TypeIdx) |
static bool | isRegisterSize (unsigned Size) |
static bool | isRegisterVectorElementType (LLT EltTy) |
static bool | isRegisterVectorType (LLT Ty) |
static bool | isRegisterType (LLT Ty) |
static LegalityPredicate | isRegisterType (unsigned TypeIdx) |
static LegalityPredicate | elementTypeIsLegal (unsigned TypeIdx) |
static LegalityPredicate | isWideScalarExtLoadTruncStore (unsigned TypeIdx) |
static unsigned | maxSizeForAddrSpace (const GCNSubtarget &ST, unsigned AS, bool IsLoad) |
static bool | isLoadStoreSizeLegal (const GCNSubtarget &ST, const LegalityQuery &Query) |
static bool | loadStoreBitcastWorkaround (const LLT Ty) |
static bool | isLoadStoreLegal (const GCNSubtarget &ST, const LegalityQuery &Query) |
static bool | shouldBitcastLoadStoreType (const GCNSubtarget &ST, const LLT Ty, const LLT MemTy) |
Return true if a load or store of the type should be lowered with a bitcast to a different type. More... | |
static bool | shouldWidenLoad (const GCNSubtarget &ST, LLT MemoryTy, uint64_t AlignInBits, unsigned AddrSpace, unsigned Opcode) |
Return true if we should legalize a load by widening an odd sized memory access up to the alignment. More... | |
static bool | shouldWidenLoad (const GCNSubtarget &ST, const LegalityQuery &Query, unsigned Opcode) |
static bool | isKnownNonNull (Register Val, MachineRegisterInfo &MRI, const AMDGPUTargetMachine &TM, unsigned AddrSpace) |
Return true if the value is a known valid address, such that a null check is not necessary. More... | |
static MachineInstrBuilder | extractF64Exponent (Register Hi, MachineIRBuilder &B) |
static LLT | widenToNextPowerOf2 (LLT Ty) |
static Register | stripAnySourceMods (Register OrigSrc, MachineRegisterInfo &MRI) |
static bool | isNot (const MachineRegisterInfo &MRI, const MachineInstr &MI) |
static MachineInstr * | verifyCFIntrinsic (MachineInstr &MI, MachineRegisterInfo &MRI, MachineInstr *&Br, MachineBasicBlock *&UncondBrTarget, bool &Negated) |
static bool | replaceWithConstant (MachineIRBuilder &B, MachineInstr &MI, int64_t C) |
static std::pair< Register, Register > | emitReciprocalU64 (MachineIRBuilder &B, Register Val) |
static void | toggleSPDenormMode (bool Enable, MachineIRBuilder &B, const GCNSubtarget &ST, AMDGPU::SIModeRegisterDefaults Mode) |
static unsigned | getDSFPAtomicOpcode (Intrinsic::ID IID) |
static unsigned | getBufferAtomicPseudo (Intrinsic::ID IntrID) |
static void | packImage16bitOpsToDwords (MachineIRBuilder &B, MachineInstr &MI, SmallVectorImpl< Register > &PackedAddrs, unsigned ArgOffset, const AMDGPU::ImageDimIntrinsicInfo *Intr, bool IsA16, bool IsG16) |
Turn a set of s16 typed registers in AddrRegs into a dword sized vector with s16 typed elements. More... | |
static void | convertImageAddrToPacked (MachineIRBuilder &B, MachineInstr &MI, int DimIdx, int NumVAddrs) |
Convert from separate vaddr components to a single vector address register, and replace the remaining operands with $noreg. More... | |
Variables | |
static cl::opt< bool > | EnableNewLegality ("amdgpu-global-isel-new-legality", cl::desc("Use GlobalISel desired legality, rather than try to use" "rules compatible with selection patterns"), cl::init(false), cl::ReallyHidden) |
static constexpr unsigned | MaxRegisterSize = 1024 |
This file implements the targeting of the Machinelegalizer class for AMDGPU.
Definition in file AMDGPULegalizerInfo.cpp.
#define DEBUG_TYPE "amdgpu-legalinfo" |
Definition at line 31 of file AMDGPULegalizerInfo.cpp.
|
static |
Definition at line 147 of file AMDGPULegalizerInfo.cpp.
References getBitcastRegisterType().
|
static |
Definition at line 154 of file AMDGPULegalizerInfo.cpp.
References assert(), llvm::LinearPolySize< ElementCount >::getFixed(), llvm::LLT::getSizeInBits(), and llvm::LLT::scalarOrVector().
|
static |
Convert from separate vaddr components to a single vector address register, and replace the remaining operands with $noreg.
Definition at line 4784 of file AMDGPULegalizerInfo.cpp.
References llvm::SmallVectorImpl< T >::append(), assert(), B, llvm::LLT::fixed_vector(), llvm::SrcOp::getReg(), I, llvm::isPowerOf2_32(), MI, llvm::NextPowerOf2(), llvm::LLT::scalar(), and llvm::RegState::Undef.
Referenced by llvm::AMDGPULegalizerInfo::legalizeImageIntrinsic().
|
static |
Definition at line 219 of file AMDGPULegalizerInfo.cpp.
References llvm::LLT::getElementType(), llvm::LLT::getSizeInBits(), llvm::LLT::isVector(), and llvm::LLT::scalar().
Referenced by llvm::AMDGPULegalizerInfo::AMDGPULegalizerInfo().
|
static |
Definition at line 3498 of file AMDGPULegalizerInfo.cpp.
References B, llvm::BitsToFloat(), and llvm::LLT::scalar().
Referenced by llvm::AMDGPULegalizerInfo::legalizeUnsignedDIV_REM64Impl().
|
static |
Definition at line 2121 of file AMDGPULegalizerInfo.cpp.
References B, and llvm::LLT::scalar().
Referenced by llvm::AMDGPULegalizerInfo::legalizeIntrinsicTrunc().
|
static |
Definition at line 104 of file AMDGPULegalizerInfo.cpp.
References llvm::LLT::getElementType(), llvm::LinearPolySize< ElementCount >::getFixed(), llvm::LLT::getNumElements(), llvm::LLT::getSizeInBits(), and llvm::LLT::scalarOrVector().
Referenced by llvm::AMDGPULegalizerInfo::AMDGPULegalizerInfo().
Definition at line 135 of file AMDGPULegalizerInfo.cpp.
References llvm::LinearPolySize< ElementCount >::getFixed(), llvm::LLT::getSizeInBits(), llvm::LLT::scalar(), and llvm::LLT::scalarOrVector().
Referenced by bitcastToRegisterType(), and llvm::AMDGPULegalizerInfo::legalizeSBufferLoad().
|
static |
Definition at line 4595 of file AMDGPULegalizerInfo.cpp.
References llvm_unreachable.
Referenced by llvm::AMDGPULegalizerInfo::legalizeBufferAtomic().
|
static |
Definition at line 4133 of file AMDGPULegalizerInfo.cpp.
References llvm_unreachable.
Referenced by llvm::AMDGPULegalizerInfo::legalizeDSAtomicFPIntrinsic().
Definition at line 57 of file AMDGPULegalizerInfo.cpp.
References llvm::tgtok::Bits, llvm::LLT::getSizeInBits(), llvm::Log2_32_Ceil(), and llvm::LLT::scalar().
Referenced by llvm::AMDGPULegalizerInfo::legalizeSBufferLoad().
Definition at line 50 of file AMDGPULegalizerInfo.cpp.
References llvm::LLT::changeElementCount(), llvm::LinearPolySize< ElementCount >::getFixed(), llvm::LLT::getNumElements(), and llvm::Log2_32_Ceil().
Referenced by llvm::AMDGPULegalizerInfo::legalizeSBufferLoad().
|
static |
Return true if the value is a known valid address, such that a null check is not necessary.
Definition at line 1910 of file AMDGPULegalizerInfo.cpp.
References llvm::tgtok::Def, llvm::ConstantInt::getSExtValue(), llvm::MachineRegisterInfo::getVRegDef(), MRI, and TM.
Referenced by llvm::AMDGPULegalizerInfo::legalizeAddrSpaceCast().
|
static |
Definition at line 355 of file AMDGPULegalizerInfo.cpp.
References isLoadStoreSizeLegal(), isRegisterType(), loadStoreBitcastWorkaround(), llvm::ARM_MB::ST, and llvm::LegalityQuery::Types.
|
static |
Definition at line 267 of file AMDGPULegalizerInfo.cpp.
References llvm::SITargetLowering::allowsMisalignedMemoryAccessesImpl(), assert(), llvm::AMDGPUAS::CONSTANT_ADDRESS_32BIT, llvm::LLT::getSizeInBits(), llvm::LLT::isVector(), llvm::max(), maxSizeForAddrSpace(), llvm::LegalityQuery::MMODescrs, llvm::LegalityQuery::Opcode, RegSize, llvm::ARM_MB::ST, and llvm::LegalityQuery::Types.
Referenced by isLoadStoreLegal().
|
static |
Definition at line 3219 of file AMDGPULegalizerInfo.cpp.
References llvm::getIConstantVRegSExtVal(), MI, and MRI.
Referenced by llvm::MCAsmParserExtension::ParseDirectiveCGProfile(), and verifyCFIntrinsic().
|
static |
Definition at line 185 of file AMDGPULegalizerInfo.cpp.
References MaxRegisterSize.
Referenced by isRegisterType(), and shouldBitcastLoadStoreType().
|
static |
Definition at line 201 of file AMDGPULegalizerInfo.cpp.
References llvm::LLT::getSizeInBits(), isRegisterSize(), isRegisterVectorType(), and llvm::LLT::isVector().
Referenced by llvm::AMDGPULegalizerInfo::AMDGPULegalizerInfo(), isLoadStoreLegal(), isRegisterType(), llvm::AMDGPULegalizerInfo::legalizeLoad(), and shouldBitcastLoadStoreType().
|
static |
Definition at line 213 of file AMDGPULegalizerInfo.cpp.
References isRegisterType().
|
static |
Definition at line 189 of file AMDGPULegalizerInfo.cpp.
References llvm::LLT::getSizeInBits().
Referenced by shouldBitcastLoadStoreType().
|
static |
Definition at line 194 of file AMDGPULegalizerInfo.cpp.
References llvm::LLT::getElementType(), llvm::LLT::getNumElements(), and llvm::LLT::getSizeInBits().
Referenced by isRegisterType().
|
static |
Definition at line 66 of file AMDGPULegalizerInfo.cpp.
References llvm::LLT::getElementType(), llvm::LLT::getNumElements(), llvm::LLT::getSizeInBits(), and llvm::LLT::isVector().
Referenced by llvm::AMDGPULegalizerInfo::AMDGPULegalizerInfo().
|
static |
Definition at line 231 of file AMDGPULegalizerInfo.cpp.
References llvm::LLT::getSizeInBits(), and llvm::LLT::isVector().
|
static |
Definition at line 87 of file AMDGPULegalizerInfo.cpp.
References llvm::LLT::getNumElements(), llvm::LLT::getScalarType(), and llvm::LLT::getSizeInBits().
Definition at line 337 of file AMDGPULegalizerInfo.cpp.
References EnableNewLegality, llvm::LLT::getElementType(), llvm::LLT::getSizeInBits(), llvm::LLT::isPointer(), and llvm::LLT::isVector().
Referenced by isLoadStoreLegal(), and shouldBitcastLoadStoreType().
|
static |
Definition at line 242 of file AMDGPULegalizerInfo.cpp.
References llvm::AMDGPUAS::CONSTANT_ADDRESS, llvm::AMDGPUAS::CONSTANT_ADDRESS_32BIT, llvm::AMDGPUAS::GLOBAL_ADDRESS, llvm::AMDGPUAS::LOCAL_ADDRESS, llvm::AMDGPUAS::PRIVATE_ADDRESS, and llvm::ARM_MB::ST.
Referenced by isLoadStoreSizeLegal(), and shouldWidenLoad().
|
static |
Definition at line 119 of file AMDGPULegalizerInfo.cpp.
References assert(), llvm::LLT::fixed_vector(), llvm::LLT::getElementType(), and llvm::LLT::getSizeInBits().
|
static |
Definition at line 178 of file AMDGPULegalizerInfo.cpp.
References llvm::LLT::getNumElements(), and llvm::LLT::isVector().
|
static |
Definition at line 95 of file AMDGPULegalizerInfo.cpp.
References llvm::LLT::fixed_vector(), llvm::LLT::getElementType(), and llvm::LLT::getNumElements().
Referenced by llvm::AMDGPULegalizerInfo::AMDGPULegalizerInfo().
|
static |
Turn a set of s16 typed registers in AddrRegs
into a dword sized vector with s16 typed elements.
Definition at line 4723 of file AMDGPULegalizerInfo.cpp.
References assert(), B, llvm::LLT::fixed_vector(), llvm::SrcOp::getReg(), I, Intr, MI, and llvm::LLT::scalar().
Referenced by llvm::AMDGPULegalizerInfo::legalizeImageIntrinsic().
|
static |
Definition at line 3340 of file AMDGPULegalizerInfo.cpp.
Referenced by llvm::AMDGPULegalizerInfo::legalizeWorkitemIDIntrinsic().
|
static |
Return true if a load or store of the type should be lowered with a bitcast to a different type.
Definition at line 363 of file AMDGPULegalizerInfo.cpp.
References llvm::LLT::getElementType(), llvm::LLT::getSizeInBits(), isRegisterSize(), isRegisterType(), isRegisterVectorElementType(), llvm::LLT::isVector(), and loadStoreBitcastWorkaround().
Referenced by llvm::AMDGPULegalizerInfo::legalizeSBufferLoad().
|
static |
Definition at line 416 of file AMDGPULegalizerInfo.cpp.
References llvm::LegalityQuery::MMODescrs, llvm::NotAtomic, shouldWidenLoad(), llvm::ARM_MB::ST, and llvm::LegalityQuery::Types.
|
static |
Return true if we should legalize a load by widening an odd sized memory access up to the alignment.
Note this case when the memory access itself changes, not the size of the result register.
Definition at line 382 of file AMDGPULegalizerInfo.cpp.
References Align, llvm::SITargetLowering::allowsMisalignedMemoryAccessesImpl(), llvm::LLT::getSizeInBits(), llvm::isPowerOf2_32(), maxSizeForAddrSpace(), llvm::MachineMemOperand::MOLoad, llvm::NextPowerOf2(), and llvm::ARM_MB::ST.
Referenced by llvm::AMDGPULegalizerInfo::legalizeLoad(), and shouldWidenLoad().
|
static |
Definition at line 80 of file AMDGPULegalizerInfo.cpp.
References llvm::LLT::getSizeInBits().
|
static |
Definition at line 2819 of file AMDGPULegalizerInfo.cpp.
References llvm::getOpcodeDef(), and MRI.
Referenced by llvm::AMDGPULegalizerInfo::legalizeFFloor().
|
static |
Definition at line 3871 of file AMDGPULegalizerInfo.cpp.
References B, Enable, FP_DENORM_FLUSH_NONE, llvm::AMDGPU::Hwreg::ID_MODE, Mode, llvm::AMDGPU::Hwreg::OFFSET_SHIFT_, llvm::ARM_MB::ST, and llvm::AMDGPU::Hwreg::WIDTH_M1_SHIFT_.
Referenced by llvm::AMDGPULegalizerInfo::legalizeFDIV32().
|
static |
Definition at line 164 of file AMDGPULegalizerInfo.cpp.
References llvm::LLT::getSizeInBits(), and llvm::LLT::isVector().
|
static |
Definition at line 171 of file AMDGPULegalizerInfo.cpp.
References llvm::LLT::getSizeInBits(), and llvm::LLT::isVector().
Referenced by llvm::AMDGPULegalizerInfo::AMDGPULegalizerInfo().
|
static |
Definition at line 3228 of file AMDGPULegalizerInfo.cpp.
References llvm::MachineBasicBlock::end(), llvm::MachineFunction::end(), llvm::eraseInstr(), llvm::ilist_node_impl< OptionsT >::getIterator(), llvm::MachineOperand::getMBB(), llvm::MachineInstr::getOpcode(), llvm::MachineInstr::getOperand(), llvm::MachineBasicBlock::getParent(), llvm::MachineInstr::getParent(), llvm::MachineOperand::getReg(), llvm::MachineRegisterInfo::hasOneNonDBGUse(), isNot(), MI, MRI, llvm::MachineRegisterInfo::use_instr_nodbg_begin(), and UseMI.
Referenced by llvm::AMDGPULegalizerInfo::legalizeIntrinsic().
Definition at line 2618 of file AMDGPULegalizerInfo.cpp.
References llvm::LLT::changeElementCount(), llvm::LinearPolySize< ElementCount >::getFixed(), llvm::LLT::getNumElements(), llvm::LLT::getSizeInBits(), llvm::LLT::isVector(), llvm::PowerOf2Ceil(), and llvm::LLT::scalar().
Referenced by llvm::AMDGPULegalizerInfo::legalizeLoad().
|
static |
Referenced by loadStoreBitcastWorkaround().
|
staticconstexpr |
Definition at line 47 of file AMDGPULegalizerInfo.cpp.
Referenced by llvm::AMDGPULegalizerInfo::AMDGPULegalizerInfo(), and isRegisterSize().