23#include "llvm/IR/IntrinsicsSPIRV.h"
25#define DEBUG_TYPE "spirv-lower"
51 if (Ty1->
getOpcode() == SPIRV::OpTypeArray) {
60 return ElemType1 == ElemType2 ||
64 if (Ty1->
getOpcode() == SPIRV::OpTypeStruct) {
70 if (ElemType1 != ElemType2 &&
106 STI.canUseExtension(SPIRV::Extension::SPV_EXT_long_vector)) {
121 unsigned AlignIdx = 0;
122 unsigned OrderingIdx = 0;
126 case Intrinsic::spv_load:
130 case Intrinsic::spv_store:
134 case Intrinsic::spv_atomic_load:
138 case Intrinsic::spv_atomic_store:
148 Info.memVT = MVT::i64;
154 Info.align =
Align(AlignOp->getZExtValue());
172 if (Constraint ==
"m")
177std::pair<unsigned, const TargetRegisterClass *>
183 return std::make_pair(0u, RC);
186 RC = VT.
isVector() ? &SPIRV::vfIDRegClass : &SPIRV::fIDRegClass;
188 RC = VT.
isVector() ? &SPIRV::viIDRegClass : &SPIRV::iIDRegClass;
190 RC = &SPIRV::iIDRegClass;
192 return std::make_pair(0u, RC);
197 return Inst && Inst->
getOpcode() == SPIRV::OpFunctionParameter
214 I.getOperand(OpIdx).setReg(NewReg);
221 SPIRV::StorageClass::StorageClass SC =
222 static_cast<SPIRV::StorageClass::StorageClass
>(
223 OpType->getOperand(1).
getImm());
228 ResTy, MIB, SPIRV::AccessQualifier::ReadWrite,
false);
238 const Type *ResTy =
nullptr) {
241 Register OpReg =
I.getOperand(OpIdx).getReg();
244 if (!ResType || !OpType || OpType->
getOpcode() != SPIRV::OpTypePointer)
247 Register ElemTypeReg = OpType->getOperand(2).getReg();
253 bool IsEqualTypes = IsSameMF ? ElemType == ResType
263 "insert validation bitcast: incompatible result and operand types");
273 constexpr unsigned OpIdx = 2;
275 Register OpReg =
I.getOperand(OpIdx).getReg();
278 if (!OpType || OpType->getOpcode() != SPIRV::OpTypePointer)
282 if (!ElemType || ElemType->
getOpcode() == SPIRV::OpTypeEvent)
296 Register PtrReg =
I.getOperand(0).getReg();
302 if (!PointeeElemType || PointeeElemType->
getOpcode() == SPIRV::OpTypeVoid ||
303 (PointeeElemType->
getOpcode() == SPIRV::OpTypeInt &&
306 I.getOperand(1).setImm(0);
314 Register OpReg =
I.getOperand(OpIdx).getReg();
317 if (!OpType || OpType->getOpcode() != SPIRV::OpTypePointer)
321 if (!ElemType || ElemType->
getOpcode() != SPIRV::OpTypeStruct ||
329 unsigned MemberTypeOp = MemberType->
getOpcode();
330 if (!
isVectorType(MemberType) && MemberTypeOp != SPIRV::OpTypeInt &&
331 MemberTypeOp != SPIRV::OpTypeFloat && MemberTypeOp != SPIRV::OpTypeBool)
335 SPIRV::StorageClass::StorageClass SC =
336 static_cast<SPIRV::StorageClass::StorageClass
>(
337 OpType->getOperand(1).
getImm());
357 if (FunDef->
getOpcode() != SPIRV::OpFunction)
361 FunDef && FunDef->
getOpcode() == SPIRV::OpFunctionParameter &&
367 DefPtrType && DefPtrType->
getOpcode() == SPIRV::OpTypePointer
413 &FunCall->getParent()->getParent()->getRegInfo();
422 if (BaseTypeInst && BaseTypeInst->
getOpcode() == SPIRV::OpTypePointer) {
435 switch (
MI.getOpcode()) {
436 case SPIRV::OpBitwiseAndS:
437 case SPIRV::OpBitwiseOrS:
438 case SPIRV::OpBitwiseXorS:
442 case SPIRV::OpFNegate:
445 case SPIRV::OpIAddCarryS:
448 case SPIRV::OpISubBorrowS:
452 case SPIRV::OpShiftLeftLogicalS:
453 case SPIRV::OpShiftRightArithmeticS:
454 case SPIRV::OpShiftRightLogicalS:
455 case SPIRV::OpStrictFAddS:
456 case SPIRV::OpStrictFDivS:
457 case SPIRV::OpStrictFMulS:
458 case SPIRV::OpStrictFRemS:
459 case SPIRV::OpStrictFSubS:
461 case SPIRV::OpUModS: {
469 MRI->replaceRegWith(
MI.getOperand(0).getReg(), NewResultReg);
475 for (
unsigned I = 2;
I !=
MI.getNumOperands(); ++
I) {
487 "Expected to find Result Type (Vec1)!");
493 case TargetOpcode::COPY: {
508 for (
auto &&U :
MRI->use_instructions(ResVReg))
532 switch (
MI.getOpcode()) {
533 case SPIRV::OpAtomicLoad:
534 case SPIRV::OpAtomicExchange:
535 case SPIRV::OpAtomicCompareExchange:
536 case SPIRV::OpAtomicCompareExchangeWeak:
537 case SPIRV::OpAtomicIIncrement:
538 case SPIRV::OpAtomicIDecrement:
539 case SPIRV::OpAtomicIAdd:
540 case SPIRV::OpAtomicISub:
541 case SPIRV::OpAtomicSMin:
542 case SPIRV::OpAtomicUMin:
543 case SPIRV::OpAtomicSMax:
544 case SPIRV::OpAtomicUMax:
545 case SPIRV::OpAtomicAnd:
546 case SPIRV::OpAtomicOr:
547 case SPIRV::OpAtomicXor:
559 case SPIRV::OpAtomicStore:
570 case SPIRV::OpPtrCastToGeneric:
571 case SPIRV::OpGenericCastToPtr:
572 case SPIRV::OpGenericCastToPtrExplicit:
575 case SPIRV::OpPtrAccessChain:
576 case SPIRV::OpInBoundsPtrAccessChain:
577 if (
MI.getNumOperands() == 4)
581 case SPIRV::OpFunctionCall:
584 if (
MI.getNumOperands() > 3)
588 case SPIRV::OpFunction:
602 MI.setDesc(STI.getInstrInfo()->get(SPIRV::OpLogicalNotEqual));
609 MI.setDesc(STI.getInstrInfo()->get(SPIRV::OpLogicalAnd));
614 case SPIRV::OpBitwiseOrS:
615 case SPIRV::OpBitwiseOrV:
618 MI.setDesc(STI.getInstrInfo()->get(SPIRV::OpLogicalOr));
620 case SPIRV::OpBitwiseAndS:
621 case SPIRV::OpBitwiseAndV:
624 MI.setDesc(STI.getInstrInfo()->get(SPIRV::OpLogicalAnd));
626 case SPIRV::OpBitwiseXorS:
627 case SPIRV::OpBitwiseXorV:
630 MI.setDesc(STI.getInstrInfo()->get(SPIRV::OpLogicalNotEqual));
632 case SPIRV::OpLifetimeStart:
633 case SPIRV::OpLifetimeStop:
634 if (
MI.getOperand(1).getImm() > 0)
637 case SPIRV::OpGroupAsyncCopy:
641 case SPIRV::OpGroupWaitEvents:
645 case SPIRV::OpConstantI: {
647 if (
Type->getOpcode() != SPIRV::OpTypeInt &&
MI.getOperand(2).isImm() &&
648 MI.getOperand(2).getImm() == 0) {
650 MI.setDesc(STI.getInstrInfo()->get(SPIRV::OpConstantNull));
651 for (
unsigned i =
MI.getNumOperands() - 1; i > 1; --i)
655 case SPIRV::OpExtInst: {
657 if (!
MI.getOperand(2).isImm() || !
MI.getOperand(3).isImm() ||
658 MI.getOperand(2).getImm() != SPIRV::InstructionSet::OpenCL_std)
660 switch (
MI.getOperand(3).getImm()) {
661 case SPIRV::OpenCLExtInst::frexp:
662 case SPIRV::OpenCLExtInst::lgamma_r:
663 case SPIRV::OpenCLExtInst::remquo: {
669 assert(RetType &&
"Expected return type");
671 STI,
MRI, GR,
MI,
MI.getNumOperands() - 1,
678 case SPIRV::OpenCLExtInst::fract:
679 case SPIRV::OpenCLExtInst::modf:
680 case SPIRV::OpenCLExtInst::sincos:
683 assert(
MI.getOperand(
MI.getNumOperands() - 2).isReg() &&
686 STI,
MRI, GR,
MI,
MI.getNumOperands() - 1,
688 MI.getOperand(
MI.getNumOperands() - 2).getReg()));
690 case SPIRV::OpenCLExtInst::prefetch:
693 assert(
MI.getOperand(
MI.getNumOperands() - 2).isReg() &&
696 MI.getNumOperands() - 2);
710 MachineInstr &
I,
unsigned int PtrOpIdx,
unsigned int OpIdx)
const {
714 if (PtrType && PtrType->
getOpcode() == SPIRV::OpTypeUntypedPointerKHR)
720 if (PointeeType == OpType)
730 if (
I.getOperand(OpIdx).isDef() &&
757 OldResult.
setReg(NewResultReg);
758 OldType.
setReg(NewTypeReg);
766 *STI.getRegBankInfo());
782 return STI.getTargetTriple().getVendor() ==
Triple::AMD
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
MachineBasicBlock MachineBasicBlock::iterator MBBI
Register const TargetRegisterInfo * TRI
MachineRegisterInfo & MRI
static void validateLifetimeStart(MachineRegisterInfo *MRI, SPIRVGlobalRegistry &GR, MachineInstr &I)
static bool typesLogicallyMatch(const SPIRVTypeInst Ty1, const SPIRVTypeInst Ty2, SPIRVGlobalRegistry &GR)
static void validateVec1Ops(const SPIRVSubtarget &STI, MachineRegisterInfo *MRI, SPIRVGlobalRegistry &GR, MachineInstr &MI)
static void validatePtrTypes(const SPIRVSubtarget &STI, MachineRegisterInfo *MRI, SPIRVGlobalRegistry &GR, MachineInstr &I, unsigned OpIdx, SPIRVTypeInst ResType, const Type *ResTy=nullptr)
static void validateGroupWaitEventsPtr(const SPIRVSubtarget &STI, MachineRegisterInfo *MRI, SPIRVGlobalRegistry &GR, MachineInstr &I)
static void validatePtrUnwrapStructField(const SPIRVSubtarget &STI, MachineRegisterInfo *MRI, SPIRVGlobalRegistry &GR, MachineInstr &I, unsigned OpIdx)
Register getTypeReg(MachineRegisterInfo *MRI, Register OpReg)
void validateAccessChain(const SPIRVSubtarget &STI, MachineRegisterInfo *MRI, SPIRVGlobalRegistry &GR, MachineInstr &I)
void validateFunCallMachineDef(const SPIRVSubtarget &STI, MachineRegisterInfo *DefMRI, MachineRegisterInfo *CallMRI, SPIRVGlobalRegistry &GR, MachineInstr &FunCall, MachineInstr *FunDef)
void validateForwardCalls(const SPIRVSubtarget &STI, MachineRegisterInfo *DefMRI, SPIRVGlobalRegistry &GR, MachineInstr &FunDef)
const Function * validateFunCall(const SPIRVSubtarget &STI, MachineRegisterInfo *CallMRI, SPIRVGlobalRegistry &GR, MachineInstr &FunCall)
static void doInsertBitcast(const SPIRVSubtarget &STI, MachineRegisterInfo *MRI, SPIRVGlobalRegistry &GR, MachineInstr &I, Register OpReg, unsigned OpIdx, SPIRVTypeInst NewPtrType)
static SPIRVTypeInst createNewPtrType(SPIRVGlobalRegistry &GR, MachineInstr &I, SPIRVTypeInst OpType, bool ReuseType, SPIRVTypeInst ResType, const Type *ResTy)
This file describes how to lower LLVM code to machine code.
an instruction that atomically reads a memory location, combines it with another value,...
@ UIncWrap
Increment one up to a maximum value.
@ FMin
*p = minnum(old, v) minnum matches the behavior of llvm.minnum.
@ FMax
*p = maxnum(old, v) maxnum matches the behavior of llvm.maxnum.
@ UDecWrap
Decrement one until a minimum value or zero.
BinOp getOperation() const
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
static constexpr ElementCount getFixed(ScalarTy MinVal)
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
This is an important class for using LLVM in a threaded context.
An instruction for reading from memory.
bool isVector() const
Return true if this is a vector value type.
bool isInteger() const
Return true if this is an integer or a vector integer type.
static MVT getVectorVT(MVT VT, unsigned NumElements)
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
MachineInstrBundleIterator< MachineInstr > iterator
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Function & getFunction()
Return the LLVM function that this machine code represents.
BasicBlockListType::iterator iterator
Helper class to build MachineInstr.
MachineInstrBuilder buildInstr(unsigned Opcode)
Build and insert <empty> = Opcode <empty>.
MachineFunction & getMF()
Getter for the function we currently build.
void constrainAllUses(const TargetInstrInfo &TII, const TargetRegisterInfo &TRI, const RegisterBankInfo &RBI) const
const MachineInstrBuilder & addUse(Register RegNo, RegState Flags={}, unsigned SubReg=0) const
Add a virtual register use operand.
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & addDef(Register RegNo, RegState Flags={}, unsigned SubReg=0) const
Add a virtual register definition operand.
Representation of each machine instruction.
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
const MachineBasicBlock * getParent() const
unsigned getNumOperands() const
Retuns the total number of operands.
const MachineOperand & getOperand(unsigned i) const
Flags
Flags values. These may be or'd together.
MachineOperand class - Representation of each machine instruction operand.
const GlobalValue * getGlobal() const
LLVM_ABI void setReg(Register Reg)
Change the register this operand corresponds to.
Register getReg() const
getReg - Returns the register number.
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
LLVM_ABI LLVM_READONLY MachineInstr * getVRegDef(Register Reg) const
getVRegDef - Return the machine instr that defines the specified virtual register or null if none is ...
bool reservedRegsFrozen() const
reservedRegsFrozen - Returns true after freezeReservedRegs() was called to ensure the set of reserved...
Wrapper class representing virtual and physical registers.
void addForwardCall(const Function *F, MachineInstr *MI)
SPIRVTypeInst getOrCreateSPIRVPointerType(const Type *BaseType, MachineIRBuilder &MIRBuilder, SPIRV::StorageClass::StorageClass SC, bool ForceTyped=false)
SPIRVTypeInst getOrCreateSPIRVIntegerType(unsigned BitWidth, MachineIRBuilder &MIRBuilder)
SPIRVTypeInst getOrCreateSPIRVVectorType(SPIRVTypeInst BaseType, unsigned NumElements, MachineIRBuilder &MIRBuilder, bool EmitIR)
SPIRVTypeInst getResultType(Register VReg, MachineFunction *MF=nullptr)
unsigned getScalarOrVectorComponentCount(Register VReg) const
const Type * getTypeForSPIRVType(SPIRVTypeInst Ty) const
bool isBitcastCompatible(SPIRVTypeInst Type1, SPIRVTypeInst Type2) const
const MachineInstr * getFunctionDefinition(const Function *F)
Register getSPIRVTypeID(SPIRVTypeInst SpirvType) const
SPIRVTypeInst getPointeeType(SPIRVTypeInst PtrType)
SmallPtrSet< MachineInstr *, 8 > * getForwardCalls(const Function *F)
SPIRVTypeInst getOrCreateSPIRVType(const Type *Type, MachineInstr &I, SPIRV::AccessQualifier::AccessQualifier AQ, bool EmitIR)
bool isScalarOrVectorOfType(Register VReg, unsigned TypeOpcode) const
MachineFunction * setCurrentFunc(MachineFunction &MF)
SPIRVTypeInst getSPIRVTypeForVReg(Register VReg, const MachineFunction *MF=nullptr) const
const Function * getFunctionByDefinition(const MachineInstr *MI)
const SPIRVInstrInfo * getInstrInfo() const override
const SPIRVRegisterInfo * getRegisterInfo() const override
const RegisterBankInfo * getRegBankInfo() const override
AtomicExpansionKind shouldCastAtomicRMWIInIR(AtomicRMWInst *RMWI) const override
Returns how the given atomic atomicrmw should be cast by the IR-level AtomicExpand pass.
AtomicExpansionKind shouldCastAtomicLoadInIR(LoadInst *LI) const override
Returns how the given (atomic) load should be cast by the IR-level AtomicExpand pass.
bool enforcePtrTypeCompatibility(MachineInstr &I, unsigned PtrOpIdx, unsigned OpIdx) const
unsigned getNumRegisters(LLVMContext &Context, EVT VT, std::optional< MVT > RegisterVT=std::nullopt) const override
Return the number of registers that this ValueType will eventually require.
unsigned getNumRegistersForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const override
Certain targets require unusual breakdowns of certain types.
MVT getRegisterTypeForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const override
Certain combinations of ABIs, Targets and features require that types are legal for some operations a...
AtomicExpansionKind shouldExpandAtomicRMWInIR(const AtomicRMWInst *RMW) const override
Returns how the IR-level AtomicExpand pass should expand the given AtomicRMW, if at all.
void finalizeLowering(MachineFunction &MF) const override
Execute target specific actions to finalize target lowering.
void getTgtMemIntrinsic(SmallVectorImpl< IntrinsicInfo > &Infos, const CallBase &I, MachineFunction &MF, unsigned Intrinsic) const override
Given an intrinsic, checks if on the target the intrinsic will need to map to a MemIntrinsicNode (tou...
bool insertLogicalCopyOnResult(MachineInstr &I, SPIRVTypeInst NewResultType) const
AtomicExpansionKind shouldCastAtomicStoreInIR(StoreInst *SI) const override
Returns how the given (atomic) store should be cast by the IR-level AtomicExpand pass into.
SPIRVTargetLowering(const TargetMachine &TM, const SPIRVSubtarget &ST)
std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const override
Given a physical register constraint (e.g.
ConstraintType getConstraintType(StringRef Constraint) const override
Given a constraint, return the type of constraint it is for this target.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void push_back(const T &Elt)
An instruction for storing to memory.
Represent a constant reference to a string, i.e.
bool starts_with(StringRef Prefix) const
Check if this string starts with the given Prefix.
static LLVM_ABI TargetExtType * get(LLVMContext &Context, StringRef Name, ArrayRef< Type * > Types={}, ArrayRef< unsigned > Ints={})
Return a target extension type having the specified name and optional type and integer parameters.
virtual void finalizeLowering(MachineFunction &MF) const
Execute target specific actions to finalize target lowering.
virtual AtomicExpansionKind shouldExpandAtomicRMWInIR(const AtomicRMWInst *RMW) const
Returns how the IR-level AtomicExpand pass should expand the given AtomicRMW, if at all.
MVT getRegisterType(LLVMContext &Context, EVT VT) const
Return the type of registers that this ValueType will eventually require.
void setMaxAtomicSizeInBitsSupported(unsigned SizeInBits)
Set the maximum atomic operation size supported by the backend.
void setMinCmpXchgSizeInBits(unsigned SizeInBits)
Sets the minimum cmpxchg or ll/sc size supported by the backend.
AtomicExpansionKind
Enum that specifies what an atomic load/AtomicRMWInst is expanded to, if at all.
TargetLowering(const TargetLowering &)=delete
Primary interface to the complete machine description for the target machine.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
The instances of the Type class are immutable: once they are created, they are never changed.
NodeTy * getNextNode()
Get the next node, or nullptr for the list tail.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
This namespace contains an enum with a value for every intrinsic/builtin function known by LLVM.
This is an optimization pass for GlobalISel generic memory operations.
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
MachineInstr * getDef(const MachineOperand &MO, const MachineRegisterInfo *MRI)
Register createVirtualRegister(SPIRVTypeInst SpvType, SPIRVGlobalRegistry *GR, MachineRegisterInfo *MRI, const MachineFunction &MF)
bool isVectorType(SPIRVTypeInst SPVTy)
MachineInstr * getImm(const MachineOperand &MO, const MachineRegisterInfo *MRI)
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
LLVM_ABI void report_fatal_error(Error Err, bool gen_crash_diag=true)
AtomicOrdering
Atomic ordering for LLVM's memory model.
DWARFExpression::Operation Op
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
MCRegisterClass TargetRegisterClass
This struct is a compact representation of a valid (non-zero power of two) alignment.
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
bool isVector() const
Return true if this is a vector value type.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
bool isInteger() const
Return true if this is an integer or a vector integer type.
MachineMemOperand::Flags flags