|
LLVM 24.0.0git
|
Custom DAG lowering for SI. More...
#include "SIISelLowering.h"#include "AMDGPU.h"#include "AMDGPUIGroupLP.h"#include "AMDGPUInstrInfo.h"#include "AMDGPULaneMaskUtils.h"#include "AMDGPUMachineInstrs.h"#include "AMDGPUMemoryUtils.h"#include "AMDGPUSelectionDAGInfo.h"#include "AMDGPUTargetMachine.h"#include "GCNSubtarget.h"#include "MCTargetDesc/AMDGPUMCTargetDesc.h"#include "SIMachineFunctionInfo.h"#include "SIRegisterInfo.h"#include "llvm/ADT/APFloat.h"#include "llvm/ADT/APInt.h"#include "llvm/ADT/FloatingPointMode.h"#include "llvm/ADT/Statistic.h"#include "llvm/Analysis/OptimizationRemarkEmitter.h"#include "llvm/Analysis/UniformityAnalysis.h"#include "llvm/CodeGen/Analysis.h"#include "llvm/CodeGen/ByteProvider.h"#include "llvm/CodeGen/FunctionLoweringInfo.h"#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h"#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"#include "llvm/CodeGen/GlobalISel/Utils.h"#include "llvm/CodeGen/MachineFrameInfo.h"#include "llvm/CodeGen/MachineFunction.h"#include "llvm/CodeGen/MachineLoopInfo.h"#include "llvm/CodeGen/PseudoSourceValueManager.h"#include "llvm/CodeGen/SDPatternMatch.h"#include "llvm/IR/DiagnosticInfo.h"#include "llvm/IR/IRBuilder.h"#include "llvm/IR/IntrinsicInst.h"#include "llvm/IR/IntrinsicsAMDGPU.h"#include "llvm/IR/IntrinsicsR600.h"#include "llvm/IR/MDBuilder.h"#include "llvm/Support/AMDGPUAddrSpace.h"#include "llvm/Support/KnownBits.h"#include "llvm/Support/ModRef.h"#include "llvm/TargetParser/AtomicScope.h"#include "llvm/Transforms/Utils/LowerAtomic.h"#include <optional>Go to the source code of this file.
Classes | |
| struct | DotSrc |
Macros | |
| #define | DEBUG_TYPE "si-lower" |
Custom DAG lowering for SI.
Definition in file SIISelLowering.cpp.
| #define DEBUG_TYPE "si-lower" |
Definition at line 61 of file SIISelLowering.cpp.
Definition at line 17713 of file SIISelLowering.cpp.
References assert(), and llvm::First.
Referenced by placeSources(), and resolveSources().
|
static |
Definition at line 15557 of file SIISelLowering.cpp.
References assert().
Referenced by hasNon16BitAccesses().
|
static |
Definition at line 13430 of file SIISelLowering.cpp.
|
static |
Definition at line 7973 of file SIISelLowering.cpp.
References llvm::ISD::BITCAST, llvm::EVT::changeTypeToInteger(), DL, llvm::SelectionDAG::ExtractVectorElements(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getPOISON(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::EVT::isVector(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), and llvm::ISD::TRUNCATE.
Referenced by constructRetValue().
|
static |
Definition at line 3087 of file SIISelLowering.cpp.
References llvm::MachineFunction::addLiveIn(), llvm::CCState::AllocateReg(), assert(), llvm::CCState::getMachineFunction(), and Reg.
Referenced by allocateSGPR32Input(), and allocateSGPR64Input().
|
static |
Definition at line 3096 of file SIISelLowering.cpp.
References allocateFixedSGPRInputImpl(), allocateSGPR32InputImpl(), and llvm::ArgDescriptor::getRegister().
Referenced by llvm::SITargetLowering::allocateSpecialInputSGPRs().
|
static |
Definition at line 3067 of file SIISelLowering.cpp.
References llvm::MachineFunction::addLiveIn(), llvm::CCState::AllocateReg(), llvm::ArrayRef(), assert(), llvm::MCRegisterClass::begin(), llvm::ArgDescriptor::createRegister(), llvm::CCState::getFirstUnallocated(), llvm::CCState::getMachineFunction(), Reg, llvm::report_fatal_error(), and llvm::ArrayRef< T >::size().
Referenced by allocateSGPR32Input(), and allocateSGPR64Input().
|
static |
Definition at line 3104 of file SIISelLowering.cpp.
References allocateFixedSGPRInputImpl(), allocateSGPR32InputImpl(), and llvm::ArgDescriptor::getRegister().
Referenced by llvm::SITargetLowering::allocateSpecialInputSGPRs().
|
static |
Definition at line 21138 of file SIISelLowering.cpp.
References llvm::TargetLoweringBase::CmpXChg, llvm::Value::getType(), isAtomicRMWLegalIntTy(), and llvm::TargetLoweringBase::None.
Referenced by llvm::SITargetLowering::shouldExpandAtomicRMWInIR().
Definition at line 14811 of file SIISelLowering.cpp.
References llvm::ISD::AND, Opc, llvm::ISD::OR, and llvm::ISD::XOR.
|
static |
Definition at line 10163 of file SIISelLowering.cpp.
References assert(), DL, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getSubtarget(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getTargetGlobalAddress(), llvm::isInt(), llvm::SIInstrInfo::MO_NONE, and llvm::Offset.
|
static |
Definition at line 20000 of file SIISelLowering.cpp.
References DL, llvm::SelectionDAG::getMachineNode(), llvm::SelectionDAG::getTargetConstant(), SDValue(), and uint64_t.
Referenced by llvm::SITargetLowering::buildRSRC(), and llvm::SITargetLowering::wrapAddr64Rsrc().
|
static |
Definition at line 15268 of file SIISelLowering.cpp.
References llvm::ISD::AND, llvm::ISD::ANY_EXTEND, llvm::ISD::AssertSext, llvm::ISD::AssertZext, llvm::BitWidth, llvm::ISD::BSWAP, calculateByteProvider(), calculateSrcByte(), llvm::cast(), llvm::ISD::CopyFromReg, llvm::Depth, llvm::dyn_cast(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::ISD::FSHR, llvm::ByteProvider< ISelOp >::getConstantZero(), llvm::SDValue::getValueSizeInBits(), LHS, llvm_unreachable, llvm::ISD::LOAD, llvm::ISD::OR, RHS, llvm::ISD::SHL, llvm::ISD::SIGN_EXTEND, llvm::ISD::SIGN_EXTEND_INREG, llvm::ISD::SRA, llvm::ISD::SRL, llvm::ISD::TRUNCATE, uint64_t, llvm::ISD::ZERO_EXTEND, and llvm::ISD::ZEXTLOAD.
|
static |
Definition at line 15199 of file SIISelLowering.cpp.
References llvm::ISD::ANY_EXTEND, calculateSrcByte(), llvm::cast(), llvm::Depth, llvm::dyn_cast(), llvm::ByteProvider< ISelOp >::getSrc(), llvm::EVT::getStoreSize(), llvm::SDValue::getValueType(), llvm::EVT::isByteSized(), llvm_unreachable, llvm::ISD::SIGN_EXTEND, llvm::ISD::SIGN_EXTEND_INREG, llvm::ISD::SRA, llvm::ISD::SRL, llvm::ISD::TRUNCATE, uint64_t, and llvm::ISD::ZERO_EXTEND.
Referenced by calculateByteProvider(), and calculateSrcByte().
|
static |
Definition at line 17893 of file SIISelLowering.cpp.
References assert(), llvm::SelectionDAG::computeKnownBits(), llvm::KnownBits::countMinLeadingZeros(), llvm::SDValue::getValueSizeInBits(), llvm_unreachable, and N.
Definition at line 20270 of file SIISelLowering.cpp.
References llvm::AMDGPU::isInlinableIntLiteral(), llvm::maskTrailingOnes(), llvm::Size, and uint64_t.
Referenced by llvm::APInt::APInt(), llvm::APInt::ashrInPlace(), llvm::SITargetLowering::checkAsmConstraintVal(), llvm::APInt::flipAllBits(), llvm::SITargetLowering::LowerAsmOperandForConstraint(), llvm::APInt::operator<<=(), llvm::APInt::operator=(), llvm::APInt::operator^=(), llvm::APInt::operator|=(), and llvm::APInt::setAllBits().
|
static |
Definition at line 5478 of file SIISelLowering.cpp.
References llvm::SIRegisterInfo::getSubRegFromChannel(), llvm::Offset, and TRI.
Referenced by emitIndirectDst(), and emitIndirectSrc().
|
static |
Definition at line 10410 of file SIISelLowering.cpp.
References adjustLoadValueTypeImpl(), llvm::ISD::BITCAST, llvm::EVT::changeTypeToInteger(), llvm::Data, DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::MVT::getVectorVT(), llvm::EVT::isVector(), llvm::MVT::isVector(), padEltsToUndef(), SDValue(), and llvm::ISD::TRUNCATE.
|
static |
Definition at line 21849 of file SIISelLowering.cpp.
References assert(), llvm::CastInst::CreatePointerCast(), llvm::AMDGPUAS::FLAT_ADDRESS, llvm::PointerType::get(), llvm::Type::getPointerAddressSpace(), llvm::Value::getType(), I, and llvm::AMDGPUAS::PRIVATE_ADDRESS.
Referenced by llvm::SITargetLowering::emitExpandAtomicCmpXchg(), llvm::SITargetLowering::emitExpandAtomicLoad(), llvm::SITargetLowering::emitExpandAtomicRMW(), and llvm::SITargetLowering::emitExpandAtomicStore().
|
static |
Definition at line 69 of file SIISelLowering.cpp.
References llvm::SIModeRegisterDefaults::FP32Denormals, llvm::MachineFunction::getInfo(), llvm::SIMachineFunctionInfo::getMode(), and llvm::DenormalMode::getPreserveSign().
Referenced by llvm::SITargetLowering::denormalsEnabledForType(), llvm::SITargetLowering::denormalsEnabledForType(), llvm::SITargetLowering::isFPExtFoldable(), and llvm::SITargetLowering::isFPExtFoldable().
|
static |
Definition at line 74 of file SIISelLowering.cpp.
References llvm::SIModeRegisterDefaults::FP64FP16Denormals, llvm::MachineFunction::getInfo(), llvm::SIMachineFunctionInfo::getMode(), and llvm::DenormalMode::getPreserveSign().
Referenced by llvm::SITargetLowering::denormalsEnabledForType(), and llvm::SITargetLowering::denormalsEnabledForType().
|
static |
Emit a DiagnosticInfoUnsupported for an unsupported image intrinsic and return poison values of ResultTypes, preserving the chain if present.
Definition at line 10532 of file SIISelLowering.cpp.
References llvm::LLVMContext::diagnose(), DL, llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getErrorMergeValues(), llvm::MachineFunction::getFunction(), llvm::SelectionDAG::getMachineFunction(), and Msg.
Definition at line 9938 of file SIISelLowering.cpp.
References assert().
Definition at line 9943 of file SIISelLowering.cpp.
References assert().
|
static |
Definition at line 21012 of file SIISelLowering.cpp.
References DEBUG_TYPE, llvm::Value::getContext(), llvm::AtomicRMWInst::getOperation(), llvm::AtomicRMWInst::getOperationName(), llvm::AtomicRMWInst::getSyncScopeID(), and llvm_unreachable.
Referenced by llvm::SITargetLowering::shouldExpandAtomicRMWInIR().
|
static |
Definition at line 5621 of file SIISelLowering.cpp.
References llvm::MachineInstrBuilder::add(), llvm::MachineInstrBuilder::addImm(), llvm::MachineInstrBuilder::addReg(), assert(), llvm::BuildMI(), computeIndirectRegAndOffset(), DL, getIndirectSGPRIdx(), llvm::MachineBasicBlock::getParent(), llvm::MachineOperand::getReg(), llvm::MachineFunction::getRegInfo(), I, loadM0FromVGPR(), MBB, MI, MRI, llvm::Offset, setM0ToIndexFromSGPR(), TII, and TRI.
Referenced by llvm::SITargetLowering::EmitInstrWithCustomInserter().
|
static |
Definition at line 5535 of file SIISelLowering.cpp.
References llvm::MachineInstrBuilder::addImm(), llvm::MachineInstrBuilder::addReg(), llvm::BuildMI(), computeIndirectRegAndOffset(), DL, getIndirectSGPRIdx(), llvm::MachineBasicBlock::getParent(), llvm::MachineFunction::getRegInfo(), I, llvm::Implicit, loadM0FromVGPR(), MBB, MI, MRI, llvm::Offset, setM0ToIndexFromSGPR(), TII, and TRI.
Referenced by llvm::SITargetLowering::EmitInstrWithCustomInserter().
|
static |
Definition at line 5332 of file SIISelLowering.cpp.
References llvm::MachineInstrBuilder::addImm(), llvm::MachineInstrBuilder::addMBB(), llvm::MachineInstrBuilder::addReg(), llvm::AMDGPU::LaneMaskConstants::AndSaveExecOpc, llvm::MachineBasicBlock::begin(), llvm::BuildMI(), DL, llvm::AMDGPU::LaneMaskConstants::ExecReg, llvm::AMDGPU::LaneMaskConstants::get(), llvm::ilist_node_impl< OptionsT >::getIterator(), llvm::MachineBasicBlock::getParent(), llvm::MachineFunction::getSubtarget(), llvm::getUndefRegState(), I, MRI, llvm::Offset, llvm::MachineInstrBuilder::setOperandDead(), TII, TRI, and llvm::AMDGPU::LaneMaskConstants::XorTermOpc.
Referenced by loadM0FromVGPR().
|
static |
Definition at line 10344 of file SIISelLowering.cpp.
References llvm::LLVMContext::diagnose(), DL, llvm::SelectionDAG::getContext(), llvm::MachineFunction::getFunction(), llvm::SelectionDAG::getMachineFunction(), and llvm::SelectionDAG::getPOISON().
|
static |
Definition at line 10352 of file SIISelLowering.cpp.
References llvm::LLVMContext::diagnose(), DL, llvm::SelectionDAG::getContext(), llvm::MachineFunction::getFunction(), llvm::SelectionDAG::getMachineFunction(), and llvm::SelectionDAG::getPOISON().
|
static |
Definition at line 5721 of file SIISelLowering.cpp.
References llvm::MachineInstrBuilder::add(), llvm::MachineInstrBuilder::addImm(), llvm::MachineInstrBuilder::addReg(), assert(), llvm::BuildMI(), DL, llvm::MachineBasicBlock::getParent(), llvm::MachineOperand::getReg(), llvm::MachineFunction::getRegInfo(), llvm::MachineFunction::getSubtarget(), llvm::Hi, llvm::MachineOperand::isDead(), llvm::MachineOperand::isDef(), MI, MRI, Opc, TII, and TRI.
Referenced by llvm::SITargetLowering::EmitInstrWithCustomInserter(), and lowerWaveReduce().
|
static |
Definition at line 5780 of file SIISelLowering.cpp.
References llvm::MachineInstrBuilder::add(), llvm::MachineInstrBuilder::addImm(), llvm::MachineInstrBuilder::addReg(), llvm::BuildMI(), DL, llvm::MachineBasicBlock::getParent(), llvm::MachineFunction::getRegInfo(), llvm::MachineFunction::getSubtarget(), MI, MRI, TII, and TRI.
Referenced by llvm::SITargetLowering::EmitInstrWithCustomInserter(), and lowerWaveReduce().
|
static |
Definition at line 5995 of file SIISelLowering.cpp.
References MI, MRI, TII, and TRI.
Referenced by lowerWaveReduce().
Definition at line 79 of file SIISelLowering.cpp.
References llvm::CCState::isAllocated(), llvm_unreachable, and Reg.
Referenced by llvm::SITargetLowering::allocateSystemSGPRs().
Helper function for LowerBRCOND.
Definition at line 8660 of file SIISelLowering.cpp.
References llvm::Value::uses().
|
static |
Definition at line 17877 of file SIISelLowering.cpp.
|
static |
Return if a flat address space atomicrmw can access private memory.
Definition at line 21145 of file SIISelLowering.cpp.
References llvm::AMDGPU::hasValueInRangeLikeMetadata(), I, and llvm::AMDGPUAS::PRIVATE_ADDRESS.
Referenced by llvm::SITargetLowering::shouldExpandAtomicCmpXchgInIR(), and llvm::SITargetLowering::shouldExpandAtomicRMWInIR().
|
static |
Definition at line 20388 of file SIISelLowering.cpp.
Referenced by llvm::SITargetLowering::finalizeLowering().
|
static |
Definition at line 21000 of file SIISelLowering.cpp.
References llvm::Function::getDenormalMode(), llvm::Type::getFltSemantics(), llvm::Instruction::getFunction(), llvm::DenormalMode::getPreserveSign(), llvm::Type::getScalarType(), llvm::Value::getType(), and llvm::Instruction::hasMetadata().
Referenced by llvm::SITargetLowering::shouldExpandAtomicRMWInIR().
MemSDNode::getBasePtr() does not work for intrinsics, which needs to offset by the chain and intrinsic ID.
Theoretically we would also need to check the specific intrinsic, but they all place the pointer operand first.
Definition at line 14778 of file SIISelLowering.cpp.
References llvm::ISD::INTRINSIC_VOID, llvm::ISD::INTRINSIC_W_CHAIN, N, and llvm::ISD::STORE.
Referenced by llvm::SITargetLowering::hasMemSDNodeUser().
|
static |
Definition at line 10360 of file SIISelLowering.cpp.
References assert(), DL, llvm::ArrayRef< T >::empty(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getPOISON(), llvm::SDValue::getValueType(), llvm::MVT::getVectorVT(), and llvm::ArrayRef< T >::size().
|
static |
Definition at line 12037 of file SIISelLowering.cpp.
References llvm::AMDGPUMachineFunctionInfo::allocateBarrierGlobal(), llvm::AMDGPUAS::BARRIER, C(), llvm::cast(), llvm::dyn_cast(), llvm::AMDGPUMachineFunctionInfo::get32BitAbsoluteAddress(), llvm::SelectionDAG::getDataLayout(), llvm::MachineFunction::getInfo(), llvm::SelectionDAG::getMachineFunction(), llvm::AMDGPU::isNamedBarrier(), and llvm::SDNode::isUndef().
Definition at line 14892 of file SIISelLowering.cpp.
References C().
Referenced by getPermuteMask().
|
static |
Definition at line 65 of file SIISelLowering.cpp.
References llvm::MachineFunction::getInfo().
|
static |
Definition at line 5915 of file SIISelLowering.cpp.
References llvm_unreachable, and Opc.
Referenced by lowerWaveReduce().
|
static |
Scalar Type
Definition at line 15598 of file SIISelLowering.cpp.
References assert(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::ExtractVectorElements(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBitcastedAnyExtOrTrunc(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getFixedValue(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::SDValue::getValueType(), llvm::EVT::getVectorVT(), llvm::MVT::getVectorVT(), and llvm::ISD::SRL.
Referenced by matchPERM(), and resolveSources().
Definition at line 9026 of file SIISelLowering.cpp.
References llvm::ISD::ABS, llvm::ISD::ADD, llvm::ISD::AND, llvm::ISD::ANY_EXTEND, CC, llvm::ISD::isSignedIntSetCC(), llvm_unreachable, llvm::ISD::MUL, llvm::ISD::OR, llvm::ISD::SELECT, llvm::ISD::SETCC, llvm::ISD::SHL, llvm::ISD::SIGN_EXTEND, llvm::ISD::SMAX, llvm::ISD::SMIN, llvm::ISD::SRA, llvm::ISD::SRL, llvm::ISD::SUB, llvm::ISD::UADDSAT, llvm::ISD::UMAX, llvm::ISD::UMIN, llvm::ISD::USUBSAT, llvm::ISD::XOR, and llvm::ISD::ZERO_EXTEND.
|
static |
Definition at line 13820 of file SIISelLowering.cpp.
References A(), assert(), B(), llvm::ISD::FMUL, llvm::SelectionDAG::getNode(), llvm::SDNode::getNumValues(), llvm::SDValue::getValue(), llvm::SelectionDAG::getVTList(), and llvm_unreachable.
Referenced by llvm::VPDerivedIVRecipe::computeCost().
|
static |
Definition at line 13843 of file SIISelLowering.cpp.
References A(), assert(), B(), C(), llvm::ISD::FMA, llvm::SelectionDAG::getNode(), llvm::SDNode::getNumValues(), llvm::SDValue::getValue(), llvm::SelectionDAG::getVTList(), and llvm_unreachable.
|
static |
RMW, and why, based on the properties of the target memory. Definition at line 21114 of file SIISelLowering.cpp.
References llvm::Instruction::hasMetadata(), and Subtarget.
Referenced by llvm::SITargetLowering::shouldExpandAtomicRMWInIR().
Definition at line 5844 of file SIISelLowering.cpp.
References llvm_unreachable, Opc, and uint64_t.
Referenced by lowerWaveReduce().
|
static |
Definition at line 5515 of file SIISelLowering.cpp.
References llvm::MachineInstrBuilder::add(), llvm::MachineInstrBuilder::addImm(), llvm::BuildMI(), DL, I, MBB, MI, MRI, llvm::Offset, llvm::MachineInstrBuilder::setOperandDead(), and TII.
Referenced by emitIndirectDst(), and emitIndirectSrc().
Definition at line 1436 of file SIISelLowering.cpp.
References llvm_unreachable.
Referenced by llvm::SITargetLowering::getTgtMemIntrinsic().
|
static |
Definition at line 13342 of file SIISelLowering.cpp.
References llvm::ISD::ANY_EXTEND, llvm::EVT::bitsLT(), llvm::ISD::EXTLOAD, llvm::SelectionDAG::getNode(), llvm_unreachable, llvm::ISD::NON_EXTLOAD, llvm::ISD::SEXTLOAD, llvm::ISD::SIGN_EXTEND, llvm::ISD::TRUNCATE, llvm::ISD::ZERO_EXTEND, and llvm::ISD::ZEXTLOAD.
|
static |
Definition at line 17491 of file SIISelLowering.cpp.
References llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getVTList(), llvm::AMDGPUISD::MAD_I64_I32, llvm::AMDGPUISD::MAD_U64_U32, llvm::Signed, and llvm::ISD::TRUNCATE.
Referenced by tryFoldMADwithSRL().
Definition at line 14916 of file SIISelLowering.cpp.
References llvm::ISD::AND, assert(), C(), llvm::dyn_cast(), getConstantPermuteMask(), llvm::ConstantSDNode::getZExtValue(), llvm::ISD::OR, llvm::ISD::SHL, llvm::ISD::SRL, and V.
|
static |
Definition at line 21152 of file SIISelLowering.cpp.
References llvm::TargetLoweringBase::CustomExpand, and llvm::TargetLoweringBase::NotAtomic.
Referenced by llvm::SITargetLowering::shouldExpandAtomicCmpXchgInIR(), llvm::SITargetLowering::shouldExpandAtomicLoadInIR(), llvm::SITargetLowering::shouldExpandAtomicRMWInIR(), and llvm::SITargetLowering::shouldExpandAtomicStoreInIR().
|
static |
Definition at line 13964 of file SIISelLowering.cpp.
References assert(), llvm::SelectionDAG::getTargetConstant(), and Mode.
|
static |
Definition at line 16769 of file SIISelLowering.cpp.
References C(), and llvm::dyn_cast().
|
static |
Definition at line 17701 of file SIISelLowering.cpp.
References calculateByteProvider().
|
static |
Definition at line 21530 of file SIISelLowering.cpp.
References llvm::dyn_cast(), hasCFUser(), llvm::SmallPtrSetImpl< PtrType >::insert(), llvm::isa(), IT, and V.
Referenced by hasCFUser(), and llvm::SITargetLowering::requiresUniformRegister().
Definition at line 15576 of file SIISelLowering.cpp.
References addresses16Bits(), isExtendedFrom16Bits(), llvm::peekThroughBitcasts(), and uint64_t.
Referenced by matchPERM().
|
static |
Definition at line 12021 of file SIISelLowering.cpp.
References AbstractManglingParser< Derived, Alloc >::Ops, DL, llvm::SelectionDAG::getConstant(), N, llvm::SelectionDAG::UpdateNodeOperands(), and Zero.
Definition at line 5896 of file SIISelLowering.cpp.
References Opc.
Referenced by lowerWaveReduce().
|
static |
Definition at line 12639 of file SIISelLowering.cpp.
References llvm::AMDGPU::lookupBufferIntrinsic().
Definition at line 21079 of file SIISelLowering.cpp.
References llvm::dyn_cast(), and IT.
Referenced by atomicSupportedIfLegalIntType(), llvm::SITargetLowering::emitExpandAtomicRMW(), isAtomicRMWLegalXChgTy(), and llvm::SITargetLowering::shouldExpandAtomicRMWInIR().
|
static |
Definition at line 21089 of file SIISelLowering.cpp.
References DL, llvm::dyn_cast(), llvm::Function::getDataLayout(), llvm::Instruction::getFunction(), llvm::Value::getType(), and isAtomicRMWLegalIntTy().
Referenced by llvm::SITargetLowering::shouldExpandAtomicRMWInIR().
Definition at line 14884 of file SIISelLowering.cpp.
References llvm::ISD::ANY_EXTEND, llvm::isBoolSGPR(), llvm::ISD::SIGN_EXTEND, and V.
Definition at line 17022 of file SIISelLowering.cpp.
References A(), B(), and llvm::dyn_cast().
Definition at line 20854 of file SIISelLowering.cpp.
References assert(), llvm::ISD::CopyFromReg, llvm::ISD::INLINEASM, llvm::ISD::INLINEASM_BR, and N.
Referenced by llvm::SITargetLowering::isSDNodeSourceOfDivergence().
Definition at line 18242 of file SIISelLowering.cpp.
References llvm::ISD::CTLZ, llvm::ISD::CTLZ_ZERO_POISON, and Opc.
Definition at line 15531 of file SIISelLowering.cpp.
References llvm::ISD::ANY_EXTEND, llvm::cast(), llvm::ISD::EXTLOAD, llvm::SDValue::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::isVector(), llvm::ISD::LOAD, llvm::ISD::SEXTLOAD, llvm::ISD::SIGN_EXTEND, llvm::ISD::ZERO_EXTEND, and llvm::ISD::ZEXTLOAD.
Referenced by hasNon16BitAccesses().
Definition at line 5906 of file SIISelLowering.cpp.
References Opc.
Referenced by lowerWaveReduce().
Definition at line 19735 of file SIISelLowering.cpp.
References llvm::ISD::AssertZext, and llvm::isa().
Referenced by llvm::SITargetLowering::legalizeTargetIndependentNode().
Definition at line 19153 of file SIISelLowering.cpp.
References llvm::ISD::FFREXP, llvm::SDPatternMatch::m_IntrinsicWOChain(), llvm::PatternMatch::m_Value(), llvm::SDPatternMatch::sd_match(), and V.
Definition at line 20219 of file SIISelLowering.cpp.
References llvm::StringRef::size().
Referenced by llvm::SITargetLowering::getConstraintType(), and llvm::SITargetLowering::LowerAsmOperandForConstraint().
|
static |
Return true if the value is a known valid address, such that a null check is not necessary.
Definition at line 9534 of file SIISelLowering.cpp.
References llvm::dyn_cast(), llvm::AMDGPU::getNullPointerValue(), and llvm::isa().
Definition at line 17885 of file SIISelLowering.cpp.
References llvm::ISD::MUL.
Definition at line 13084 of file SIISelLowering.cpp.
References llvm::ISD::ADD, llvm::SDNode::getFlags(), llvm::SDNode::getOpcode(), llvm::SDValue::getOpcode(), llvm::SDNodeFlags::hasNoUnsignedWrap(), and llvm::ISD::OR.
Definition at line 21073 of file SIISelLowering.cpp.
References llvm::dyn_cast(), llvm::VectorType::getElementType(), llvm::FixedVectorType::getNumElements(), and llvm::Type::isBFloatTy().
Referenced by llvm::SITargetLowering::shouldExpandAtomicRMWInIR().
Definition at line 21068 of file SIISelLowering.cpp.
References llvm::dyn_cast(), llvm::VectorType::getElementType(), llvm::FixedVectorType::getNumElements(), and llvm::Type::isHalfTy().
Referenced by llvm::SITargetLowering::shouldExpandAtomicRMWInIR().
Definition at line 21058 of file SIISelLowering.cpp.
References llvm::dyn_cast(), llvm::Type::isBFloatTy(), and llvm::Type::isHalfTy().
Referenced by llvm::SITargetLowering::shouldExpandAtomicRMWInIR().
|
static |
Definition at line 20553 of file SIISelLowering.cpp.
References llvm::GISelValueTracking::computeKnownBitsImpl(), llvm::Depth, llvm::getIConstantVRegValWithLookThrough(), llvm::GISelValueTracking::getMachineFunction(), llvm::MachineFunction::getRegInfo(), llvm::Known, llvm::maskTrailingOnes(), MI, MRI, and llvm::Offset.
Referenced by llvm::SITargetLowering::computeKnownBitsForTargetInstr().
|
static |
Definition at line 20545 of file SIISelLowering.cpp.
References llvm::countl_zero(), llvm::MachineFunction::getFunction(), llvm::GISelValueTracking::getMachineFunction(), and llvm::Known.
Referenced by llvm::SITargetLowering::computeKnownBitsForTargetInstr().
|
static |
Definition at line 5428 of file SIISelLowering.cpp.
References llvm::MachineInstrBuilder::addReg(), llvm::MachineBasicBlock::addSuccessor(), llvm::MachineBasicBlock::begin(), llvm::BuildMI(), llvm::MachineFunction::CreateMachineBasicBlock(), DL, emitLoadM0FromVGPRLoop(), llvm::AMDGPU::LaneMaskConstants::ExecReg, llvm::First, llvm::AMDGPU::LaneMaskConstants::get(), llvm::MachineFunction::getRegInfo(), llvm::MachineFunction::getSubtarget(), I, llvm::MachineFunction::insert(), MBB, MBBI, MI, llvm::AMDGPU::LaneMaskConstants::MovOpc, MRI, llvm::Offset, splitBlockForLoop(), TII, and TRI.
Referenced by emitIndirectDst(), and emitIndirectSrc().
|
static |
Definition at line 8123 of file SIISelLowering.cpp.
References llvm::dyn_cast(), llvm::ISD::FP_EXTEND, llvm::SelectionDAG::getCondCode(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getCopyFromReg(), llvm::SelectionDAG::getEntryNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getZExtOrTrunc(), N, SDValue(), llvm::ISD::SETCC, llvm::ISD::SETNE, and TLI.
|
static |
Definition at line 8166 of file SIISelLowering.cpp.
References llvm::LLVMContext::diagnose(), DL, llvm::Intrinsic::getBaseName(), llvm::SelectionDAG::getContext(), llvm::MachineFunction::getFunction(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getPOISON(), llvm::Offset, and llvm::Signed.
|
static |
Definition at line 8190 of file SIISelLowering.cpp.
References assert(), llvm::ISD::CONCAT_VECTORS, llvm::ISD::CONVERGENCECTRL_GLUE, llvm::ISD::EXTRACT_SUBVECTOR, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getAnyExtOrTrunc(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::MVT::getIntegerVT(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::EVT::getSimpleVT(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetConstant(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::SelectionDAG::getVectorIdxConstant(), llvm::EVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::MVT::getVectorVT(), llvm::ISD::INTRINSIC_WO_CHAIN, llvm::EVT::isFloatingPoint(), llvm::AMDGPU::isLegalDPALU_DPPControl(), llvm::EVT::isVector(), llvm_unreachable, N, Operands, llvm::SmallVectorTemplateBase< T, bool >::push_back(), SDValue(), llvm::MVT::SimpleTy, TLI, and llvm::Trunc.
|
static |
Definition at line 6009 of file SIISelLowering.cpp.
References Accumulator, llvm::MachineInstrBuilder::addImm(), llvm::MachineInstrBuilder::addMBB(), llvm::MachineInstrBuilder::addReg(), assert(), llvm::AMDGPU::DPP::BCAST15, llvm::AMDGPU::DPP::BCAST31, llvm::BuildMI(), llvm::MachineOperand::CreateReg(), llvm::Dead, DEFAULT, llvm::Define, DL, llvm::MachineBasicBlock::end(), expand64BitScalarArithmetic(), expand64BitV_CNDMASK(), ExtractSubRegs(), getDPPOpcForWaveReduction(), getIdentityValueForWaveReduction(), llvm::getImm(), llvm::MachineInstr::getOperand(), llvm::MachineBasicBlock::getParent(), llvm::MachineOperand::getReg(), llvm::MachineFunction::getRegInfo(), llvm::AMDGPUSubtarget::GFX12, I, is32bitWaveReduceOperation(), isFloatingPointWaveReduceOperation(), MI, MRI, llvm::SISrcMods::NEG, llvm::SISrcMods::NONE, Opc, Register, llvm::AMDGPU::DPP::ROW_SHR_FIRST, llvm::MachineInstrBuilder::setOperandDead(), splitBlockForLoop(), TII, TRI, and uint64_t.
Referenced by llvm::SITargetLowering::EmitInstrWithCustomInserter().
|
static |
Definition at line 8402 of file SIISelLowering.cpp.
References llvm::ISD::AND, assert(), llvm::SelectionDAG::getAllOnesConstant(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getPOISON(), llvm::SelectionDAG::getSelect(), llvm::SelectionDAG::getSetCC(), llvm::SelectionDAG::getShiftAmountConstant(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetConstant(), llvm::SDNode::getValueType(), llvm::ISD::INTRINSIC_WO_CHAIN, N, Operands, SDValue(), llvm::ISD::SETEQ, llvm::ISD::SHL, TLI, and llvm::ISD::XOR.
|
static |
Definition at line 15661 of file SIISelLowering.cpp.
References assert(), calculateByteProvider(), llvm::TargetLowering::DAGCombinerInfo::DAG, DL, llvm::ISD::FSHR, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBitcastedAnyExtOrTrunc(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getDataLayout(), getDWordFromOffset(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), hasNon16BitAccesses(), llvm::DataLayout::isBigEndian(), llvm::EVT::isByteSized(), N, P, llvm::SmallVectorTemplateBase< T, bool >::push_back(), SDValue(), llvm::SmallVectorTemplateCommon< T, typename >::size(), and uint64_t.
Referenced by llvm::SITargetLowering::PerformDAGCombine().
|
static |
Definition at line 1378 of file SIISelLowering.cpp.
References assert(), DL, llvm::dyn_cast(), llvm::EVT::getVectorVT(), and TLI.
Referenced by llvm::SITargetLowering::getTgtMemIntrinsic(), and memVTFromLoadIntrReturn().
|
static |
Definition at line 1394 of file SIISelLowering.cpp.
References assert(), DL, llvm::dyn_cast(), memVTFromLoadIntrData(), and TLI.
Referenced by llvm::SITargetLowering::getTgtMemIntrinsic().
Definition at line 16704 of file SIISelLowering.cpp.
References llvm::ISD::FMAXIMUM, llvm::ISD::FMAXIMUMNUM, llvm::ISD::FMAXNUM, llvm::ISD::FMAXNUM_IEEE, llvm::ISD::FMINIMUM, llvm::ISD::FMINIMUMNUM, llvm::ISD::FMINNUM, llvm::ISD::FMINNUM_IEEE, llvm_unreachable, Opc, llvm::ISD::SMAX, llvm::ISD::SMIN, llvm::ISD::UMAX, and llvm::ISD::UMIN.
|
static |
Definition at line 10501 of file SIISelLowering.cpp.
References llvm::ISD::ANY_EXTEND, DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getNode(), llvm::SDValue::getValueType(), I, and llvm::SmallVectorTemplateBase< T, bool >::push_back().
|
static |
Definition at line 10389 of file SIISelLowering.cpp.
References DL, llvm::SelectionDAG::ExtractVectorElements(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getPOISON(), llvm::EVT::getScalarType(), llvm::EVT::isVector(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), and llvm::Undef.
Referenced by constructRetValue().
|
static |
Definition at line 1471 of file SIISelLowering.cpp.
References llvm::Acquire, llvm::acquire, llvm::cast(), llvm::CallBase::getArgOperand(), llvm::Monotonic, llvm::Release, llvm::release, llvm::seq_cst, and llvm::SequentiallyConsistent.
Referenced by llvm::SITargetLowering::getTgtMemIntrinsic().
Definition at line 1490 of file SIISelLowering.cpp.
References llvm::cast(), llvm::CallBase::getArgOperand(), llvm::Value::getContext(), llvm::MDNode::getOperand(), and llvm::LLVMContext::getOrInsertSyncScopeID().
Referenced by llvm::SITargetLowering::getTgtMemIntrinsic().
|
static |
Definition at line 10483 of file SIISelLowering.cpp.
References llvm::cast(), DL, llvm::SDValue::getNode(), llvm::SelectionDAG::getTargetConstant(), and uint64_t.
|
static |
Definition at line 17733 of file SIISelLowering.cpp.
References addPermMasks(), assert(), llvm::SmallVectorTemplateCommon< T, typename >::end(), llvm::find_if(), I, and llvm::SmallVectorTemplateBase< T, bool >::push_back().
|
static |
Definition at line 2974 of file SIISelLowering.cpp.
References llvm::CallingConv::AMDGPU_PS, assert(), E(), llvm::ISD::InputArg::Flags, llvm::ISD::InputArg::getOrigArgIndex(), llvm::MVT::getScalarSizeInBits(), I, llvm::ISD::ArgFlagsTy::isInReg(), llvm::ISD::ArgFlagsTy::isSplit(), llvm::ISD::ArgFlagsTy::isSplitEnd(), llvm::MVT::isVector(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::ArrayRef< T >::size(), llvm::ISD::InputArg::Used, and llvm::ISD::InputArg::VT.
Referenced by llvm::SITargetLowering::LowerFormalArguments().
|
static |
Definition at line 3396 of file SIISelLowering.cpp.
References assert(), llvm::Function::getCallingConv(), llvm::MachineFunction::getFrameInfo(), llvm::MachineFunction::getFunction(), llvm::TargetMachine::getOptLevel(), llvm::MachineFunction::getRegInfo(), llvm::MachineFunction::getSubtarget(), llvm::MachineFrameInfo::hasCalls(), llvm::MachineFrameInfo::hasStackObjects(), llvm::AMDGPU::isShader(), MRI, llvm::None, llvm::AMDGPUFunctionArgInfo::PRIVATE_SEGMENT_BUFFER, Reg, llvm::report_fatal_error(), and TRI.
Referenced by llvm::SITargetLowering::finalizeLowering().
|
static |
Definition at line 17809 of file SIISelLowering.cpp.
References addPermMasks(), assert(), llvm::SmallVectorTemplateCommon< T, typename >::begin(), llvm::SmallVectorTemplateCommon< T, typename >::end(), llvm::SelectionDAG::getConstant(), getDWordFromOffset(), llvm::SelectionDAG::getNode(), llvm::ISD::OR, llvm::SmallVectorTemplateBase< T, bool >::push_back(), and llvm::SmallVectorTemplateCommon< T, typename >::size().
|
static |
Definition at line 11898 of file SIISelLowering.cpp.
References llvm::SelectionDAG::getRegister(), llvm::isNullConstant(), and Subtarget.
|
static |
Definition at line 5491 of file SIISelLowering.cpp.
References llvm::MachineInstrBuilder::add(), llvm::MachineInstrBuilder::addImm(), assert(), llvm::BuildMI(), DL, I, MBB, MI, MRI, llvm::Offset, llvm::MachineInstrBuilder::setOperandDead(), and TII.
Referenced by emitIndirectDst(), and emitIndirectSrc().
|
static |
Definition at line 5228 of file SIISelLowering.cpp.
References llvm::MachineBasicBlock::addSuccessor(), llvm::MachineBasicBlock::begin(), llvm::MachineFunction::CreateMachineBasicBlock(), I, llvm::MachineFunction::insert(), MBB, MBBI, MI, llvm::Next, llvm::MachineBasicBlock::splice(), and llvm::MachineBasicBlock::transferSuccessorsAndUpdatePHIs().
Referenced by llvm::SITargetLowering::emitGWSMemViolTestLoop(), loadM0FromVGPR(), and lowerWaveReduce().
|
static |
Definition at line 7953 of file SIISelLowering.cpp.
References llvm::ISD::BITCAST, C(), llvm::EVT::changeTypeToInteger(), llvm::divideCeil(), DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getVectorIdxConstant(), llvm::EVT::getVectorVT(), llvm::EVT::isVector(), and llvm::ISD::TRUNCATE.
| STATISTIC | ( | NumTailCalls | , |
| "Number of tail calls" | ) |
|
static |
Return the source of an fp_extend from f16 to f32, or a converted FP constant.
Definition at line 17370 of file SIISelLowering.cpp.
References llvm::APFloat::convert(), llvm::dyn_cast(), llvm::ISD::FP_EXTEND, llvm::SelectionDAG::getConstantFP(), llvm::APFloatBase::IEEEhalf(), llvm::APFloatBase::rmNearestTiesToEven, and SDValue().
Helper function for adjustWritemask.
Definition at line 19538 of file SIISelLowering.cpp.
|
static |
Opc for type VT. Definition at line 16858 of file SIISelLowering.cpp.
References llvm::ISD::FMAXIMUM, llvm::ISD::FMAXIMUMNUM, llvm::ISD::FMAXNUM, llvm::ISD::FMAXNUM_IEEE, llvm::ISD::FMINIMUM, llvm::ISD::FMINIMUMNUM, llvm::ISD::FMINNUM, llvm::ISD::FMINNUM_IEEE, llvm_unreachable, Opc, llvm::ISD::SMAX, llvm::ISD::SMIN, Subtarget, llvm::ISD::UMAX, and llvm::ISD::UMIN.
|
static |
Definition at line 17505 of file SIISelLowering.cpp.
References llvm::dyn_cast(), llvm::SDNode::getAsZExtVal(), llvm::SelectionDAG::getConstant(), getMad64_32(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getZeroExtendInReg(), llvm::Hi_32(), llvm::Lo_32(), SDValue(), llvm::ISD::SRL, std::swap(), and llvm::ISD::TRUNCATE.
|
static |
Definition at line 18658 of file SIISelLowering.cpp.
References assert(), llvm::KnownFPClass::bitcast(), CC, llvm::SelectionDAG::computeKnownBits(), llvm::KnownBits::eq(), llvm::EVT::getFltSemantics(), llvm::KnownFPClass::isKnownNeverNaN(), llvm::APInt::isSignBitSet(), LHS, RHS, llvm::ISD::SETCC_INVALID, llvm::ISD::SETEQ, llvm::ISD::SETFALSE, llvm::ISD::SETGE, llvm::ISD::SETGT, llvm::ISD::SETLE, llvm::ISD::SETLT, llvm::ISD::SETNE, llvm::ISD::SETOEQ, llvm::ISD::SETOGE, llvm::ISD::SETOGT, llvm::ISD::SETOLE, llvm::ISD::SETOLT, llvm::ISD::SETONE, llvm::ISD::SETTRUE, llvm::ISD::SETUEQ, llvm::ISD::SETUGE, llvm::ISD::SETUGT, llvm::ISD::SETULE, llvm::ISD::SETULT, llvm::ISD::SETUNE, llvm::KnownBits::trunc(), llvm::KnownBits::uge(), llvm::KnownBits::ule(), and llvm::KnownBits::Zero.
Definition at line 16627 of file SIISelLowering.cpp.
References llvm::isa().