20#include "llvm/IR/IntrinsicsSPIRV.h"
24#define DEBUG_TYPE "spirv-builtins"
28#define GET_BuiltinGroup_DECL
29#include "SPIRVGenTables.inc"
33 InstructionSet::InstructionSet
Set;
39#define GET_DemangledBuiltins_DECL
40#define GET_DemangledBuiltins_IMPL
62 InstructionSet::InstructionSet
Set;
66#define GET_NativeBuiltins_DECL
67#define GET_NativeBuiltins_IMPL
85#define GET_GroupBuiltins_DECL
86#define GET_GroupBuiltins_IMPL
95#define GET_IntelSubgroupsBuiltins_DECL
96#define GET_IntelSubgroupsBuiltins_IMPL
103#define GET_AtomicFloatingBuiltins_DECL
104#define GET_AtomicFloatingBuiltins_IMPL
111#define GET_GroupUniformBuiltins_DECL
112#define GET_GroupUniformBuiltins_IMPL
116 InstructionSet::InstructionSet
Set;
120using namespace BuiltIn;
121#define GET_GetBuiltins_DECL
122#define GET_GetBuiltins_IMPL
126 InstructionSet::InstructionSet
Set;
130#define GET_ImageQueryBuiltins_DECL
131#define GET_ImageQueryBuiltins_IMPL
135 InstructionSet::InstructionSet
Set;
145 InstructionSet::InstructionSet
Set;
152using namespace FPRoundingMode;
153#define GET_ConvertBuiltins_DECL
154#define GET_ConvertBuiltins_IMPL
156using namespace InstructionSet;
157#define GET_VectorLoadStoreBuiltins_DECL
158#define GET_VectorLoadStoreBuiltins_IMPL
160#define GET_CLMemoryScope_DECL
161#define GET_CLSamplerAddressingMode_DECL
162#define GET_CLMemoryFenceFlags_DECL
163#define GET_ExtendedBuiltins_DECL
164#include "SPIRVGenTables.inc"
182static std::unique_ptr<const SPIRV::IncomingCall>
184 SPIRV::InstructionSet::InstructionSet Set,
189 std::string BuiltinName =
193 if (BuiltinName.rfind(
"__spirv_ocl_", 0) == 0)
194 BuiltinName = BuiltinName.substr(12);
199 if (BuiltinName.find(
'<') && BuiltinName.back() ==
'>') {
200 BuiltinName = BuiltinName.substr(0, BuiltinName.find(
'<'));
201 BuiltinName = BuiltinName.substr(BuiltinName.find_last_of(
' ') + 1);
209 BuiltinName = BuiltinName.substr(0, BuiltinName.find(
"_R"));
214 DemangledCall.
slice(DemangledCall.
find(
'(') + 1, DemangledCall.
find(
')'));
215 BuiltinArgs.
split(BuiltinArgumentTypes,
',', -1,
false);
220 if ((Builtin = SPIRV::lookupBuiltin(BuiltinName, Set)))
221 return std::make_unique<SPIRV::IncomingCall>(
222 BuiltinName, Builtin, ReturnRegister, ReturnType,
Arguments);
227 if (BuiltinArgumentTypes.
size() >= 1) {
228 char FirstArgumentType = BuiltinArgumentTypes[0][0];
233 switch (FirstArgumentType) {
236 if (Set == SPIRV::InstructionSet::OpenCL_std)
238 else if (Set == SPIRV::InstructionSet::GLSL_std_450)
246 if (Set == SPIRV::InstructionSet::OpenCL_std)
248 else if (Set == SPIRV::InstructionSet::GLSL_std_450)
255 if (Set == SPIRV::InstructionSet::OpenCL_std ||
256 Set == SPIRV::InstructionSet::GLSL_std_450)
262 if (!Prefix.empty() &&
263 (Builtin = SPIRV::lookupBuiltin(Prefix + BuiltinName, Set)))
264 return std::make_unique<SPIRV::IncomingCall>(
265 BuiltinName, Builtin, ReturnRegister, ReturnType,
Arguments);
272 switch (FirstArgumentType) {
293 if (!Suffix.empty() &&
294 (Builtin = SPIRV::lookupBuiltin(BuiltinName + Suffix, Set)))
295 return std::make_unique<SPIRV::IncomingCall>(
296 BuiltinName, Builtin, ReturnRegister, ReturnType,
Arguments);
311 assert(
MI->getOpcode() == TargetOpcode::G_ADDRSPACE_CAST &&
312 MI->getOperand(1).isReg());
313 Register BitcastReg =
MI->getOperand(1).getReg();
344 Register ValueReg =
MI->getOperand(0).getReg();
350 assert(Ty &&
"Type is expected");
362 if (
MI->getOpcode() == TargetOpcode::G_GLOBAL_VALUE)
363 return MI->getOperand(1).getGlobal()->getType();
365 "Blocks in OpenCL C must be traceable to allocation site");
377static std::tuple<Register, SPIRVType *>
383 if (ResultType->
getOpcode() == SPIRV::OpTypeVector) {
398 return std::make_tuple(ResultRegister, BoolType);
409 if (ReturnType->getOpcode() == SPIRV::OpTypeVector) {
418 return MIRBuilder.
buildSelect(ReturnRegister, SourceRegister, TrueConst,
429 if (!DestinationReg.isValid()) {
430 DestinationReg =
MRI->createVirtualRegister(&SPIRV::IDRegClass);
437 MIRBuilder.
buildLoad(DestinationReg, PtrRegister, PtrInfo,
Align());
438 return DestinationReg;
452 VariableType, MIRBuilder, SPIRV::StorageClass::Input);
458 SPIRV::StorageClass::Input,
nullptr, isConst,
459 hasLinkageTy, SPIRV::LinkageType::Import, MIRBuilder,
466 return LoadedRegister;
475 SPIRVGlobalRegistry *GR,
476 MachineIRBuilder &MIB,
477 MachineRegisterInfo &
MRI);
480static SPIRV::MemorySemantics::MemorySemantics
483 case std::memory_order::memory_order_relaxed:
484 return SPIRV::MemorySemantics::None;
485 case std::memory_order::memory_order_acquire:
486 return SPIRV::MemorySemantics::Acquire;
487 case std::memory_order::memory_order_release:
488 return SPIRV::MemorySemantics::Release;
489 case std::memory_order::memory_order_acq_rel:
490 return SPIRV::MemorySemantics::AcquireRelease;
491 case std::memory_order::memory_order_seq_cst:
492 return SPIRV::MemorySemantics::SequentiallyConsistent;
500 case SPIRV::CLMemoryScope::memory_scope_work_item:
501 return SPIRV::Scope::Invocation;
502 case SPIRV::CLMemoryScope::memory_scope_work_group:
503 return SPIRV::Scope::Workgroup;
504 case SPIRV::CLMemoryScope::memory_scope_device:
505 return SPIRV::Scope::Device;
506 case SPIRV::CLMemoryScope::memory_scope_all_svm_devices:
507 return SPIRV::Scope::CrossDevice;
508 case SPIRV::CLMemoryScope::memory_scope_sub_group:
509 return SPIRV::Scope::Subgroup;
522 SPIRV::Scope::Scope Scope,
526 if (CLScopeRegister.
isValid()) {
531 if (CLScope ==
static_cast<unsigned>(Scope)) {
532 MRI->setRegClass(CLScopeRegister, &SPIRV::IDRegClass);
533 return CLScopeRegister;
540 Register PtrRegister,
unsigned &Semantics,
543 if (SemanticsRegister.
isValid()) {
545 std::memory_order Order =
551 if (Order == Semantics) {
552 MRI->setRegClass(SemanticsRegister, &SPIRV::IDRegClass);
553 return SemanticsRegister;
563 MachineRegisterInfo *
MRI = MIRBuilder.
getMRI();
567 unsigned Sz = Call->Arguments.size() - ImmArgs.size();
568 for (
unsigned i = 0; i < Sz; ++i) {
569 Register ArgReg = Call->Arguments[i];
570 if (!
MRI->getRegClassOrNull(ArgReg))
571 MRI->setRegClass(ArgReg, &SPIRV::IDRegClass);
582 if (Call->isSpirvOp())
585 assert(Call->Arguments.size() == 2 &&
586 "Need 2 arguments for atomic init translation");
590 .
addUse(Call->Arguments[0])
591 .
addUse(Call->Arguments[1]);
600 if (Call->isSpirvOp())
603 Register PtrRegister = Call->Arguments[0];
609 if (Call->Arguments.size() > 1) {
610 ScopeRegister = Call->Arguments[1];
616 if (Call->Arguments.size() > 2) {
618 MemSemanticsReg = Call->Arguments[2];
622 SPIRV::MemorySemantics::SequentiallyConsistent |
628 .
addDef(Call->ReturnRegister)
640 if (Call->isSpirvOp())
645 Register PtrRegister = Call->Arguments[0];
648 SPIRV::MemorySemantics::SequentiallyConsistent |
656 .
addUse(Call->Arguments[1]);
664 if (Call->isSpirvOp())
668 bool IsCmpxchg = Call->Builtin->Name.contains(
"cmpxchg");
671 Register ObjectPtr = Call->Arguments[0];
672 Register ExpectedArg = Call->Arguments[1];
673 Register Desired = Call->Arguments[2];
674 MRI->setRegClass(ObjectPtr, &SPIRV::IDRegClass);
675 MRI->setRegClass(ExpectedArg, &SPIRV::IDRegClass);
676 MRI->setRegClass(Desired, &SPIRV::IDRegClass);
678 LLT DesiredLLT =
MRI->getType(Desired);
681 SPIRV::OpTypePointer);
684 assert(IsCmpxchg ? ExpectedType == SPIRV::OpTypeInt
685 : ExpectedType == SPIRV::OpTypePointer);
690 auto StorageClass =
static_cast<SPIRV::StorageClass::StorageClass
>(
698 ? SPIRV::MemorySemantics::None
699 : SPIRV::MemorySemantics::SequentiallyConsistent | MemSemStorage;
702 ? SPIRV::MemorySemantics::None
703 : SPIRV::MemorySemantics::SequentiallyConsistent | MemSemStorage;
704 if (Call->Arguments.size() >= 4) {
705 assert(Call->Arguments.size() >= 5 &&
706 "Need 5+ args for explicit atomic cmpxchg");
713 if (MemOrdEq == MemSemEqual)
714 MemSemEqualReg = Call->Arguments[3];
715 if (MemOrdNeq == MemSemEqual)
716 MemSemUnequalReg = Call->Arguments[4];
717 MRI->setRegClass(Call->Arguments[3], &SPIRV::IDRegClass);
718 MRI->setRegClass(Call->Arguments[4], &SPIRV::IDRegClass);
722 if (!MemSemUnequalReg.
isValid())
726 auto Scope = IsCmpxchg ? SPIRV::Scope::Workgroup : SPIRV::Scope::Device;
727 if (Call->Arguments.size() >= 6) {
728 assert(Call->Arguments.size() == 6 &&
729 "Extra args for explicit atomic cmpxchg");
730 auto ClScope =
static_cast<SPIRV::CLMemoryScope
>(
733 if (ClScope ==
static_cast<unsigned>(Scope))
734 ScopeReg = Call->Arguments[5];
735 MRI->setRegClass(Call->Arguments[5], &SPIRV::IDRegClass);
745 Register Tmp = !IsCmpxchg ?
MRI->createGenericVirtualRegister(DesiredLLT)
746 : Call->ReturnRegister;
747 if (!
MRI->getRegClassOrNull(Tmp))
748 MRI->setRegClass(Tmp, &SPIRV::IDRegClass);
772 if (Call->isSpirvOp())
778 Call->Arguments.size() >= 4 ? Call->Arguments[3] :
Register();
780 assert(Call->Arguments.size() <= 4 &&
781 "Too many args for explicit atomic RMW");
782 ScopeRegister =
buildScopeReg(ScopeRegister, SPIRV::Scope::Workgroup,
783 MIRBuilder, GR,
MRI);
785 Register PtrRegister = Call->Arguments[0];
786 unsigned Semantics = SPIRV::MemorySemantics::None;
787 MRI->setRegClass(PtrRegister, &SPIRV::IDRegClass);
789 Call->Arguments.size() >= 3 ? Call->Arguments[2] :
Register();
791 Semantics, MIRBuilder, GR);
792 MRI->setRegClass(Call->Arguments[1], &SPIRV::IDRegClass);
793 Register ValueReg = Call->Arguments[1];
796 if (Call->ReturnType->getOpcode() == SPIRV::OpTypeFloat) {
797 if (Opcode == SPIRV::OpAtomicIAdd) {
798 Opcode = SPIRV::OpAtomicFAddEXT;
799 }
else if (Opcode == SPIRV::OpAtomicISub) {
802 Opcode = SPIRV::OpAtomicFAddEXT;
804 MRI->createGenericVirtualRegister(
MRI->getType(ValueReg));
805 MRI->setRegClass(NegValueReg, &SPIRV::IDRegClass);
813 ValueReg = NegValueReg;
817 .
addDef(Call->ReturnRegister)
831 assert(Call->Arguments.size() == 4 &&
832 "Wrong number of atomic floating-type builtin");
836 Register PtrReg = Call->Arguments[0];
837 MRI->setRegClass(PtrReg, &SPIRV::IDRegClass);
839 Register ScopeReg = Call->Arguments[1];
840 MRI->setRegClass(ScopeReg, &SPIRV::IDRegClass);
842 Register MemSemanticsReg = Call->Arguments[2];
843 MRI->setRegClass(MemSemanticsReg, &SPIRV::IDRegClass);
845 Register ValueReg = Call->Arguments[3];
846 MRI->setRegClass(ValueReg, &SPIRV::IDRegClass);
849 .
addDef(Call->ReturnRegister)
863 bool IsSet = Opcode == SPIRV::OpAtomicFlagTestAndSet;
865 if (Call->isSpirvOp())
870 Register PtrRegister = Call->Arguments[0];
871 unsigned Semantics = SPIRV::MemorySemantics::SequentiallyConsistent;
873 Call->Arguments.size() >= 2 ? Call->Arguments[1] :
Register();
875 Semantics, MIRBuilder, GR);
877 assert((Opcode != SPIRV::OpAtomicFlagClear ||
878 (Semantics != SPIRV::MemorySemantics::Acquire &&
879 Semantics != SPIRV::MemorySemantics::AcquireRelease)) &&
880 "Invalid memory order argument!");
883 Call->Arguments.size() >= 3 ? Call->Arguments[2] :
Register();
900 if (Call->isSpirvOp())
905 unsigned MemSemantics = SPIRV::MemorySemantics::None;
907 if (MemFlags & SPIRV::CLK_LOCAL_MEM_FENCE)
908 MemSemantics |= SPIRV::MemorySemantics::WorkgroupMemory;
910 if (MemFlags & SPIRV::CLK_GLOBAL_MEM_FENCE)
911 MemSemantics |= SPIRV::MemorySemantics::CrossWorkgroupMemory;
913 if (MemFlags & SPIRV::CLK_IMAGE_MEM_FENCE)
914 MemSemantics |= SPIRV::MemorySemantics::ImageMemory;
916 if (Opcode == SPIRV::OpMemoryBarrier) {
917 std::memory_order MemOrder =
921 MemSemantics |= SPIRV::MemorySemantics::SequentiallyConsistent;
925 if (MemFlags == MemSemantics) {
926 MemSemanticsReg = Call->Arguments[0];
927 MRI->setRegClass(MemSemanticsReg, &SPIRV::IDRegClass);
932 SPIRV::Scope::Scope Scope = SPIRV::Scope::Workgroup;
933 SPIRV::Scope::Scope MemScope = Scope;
934 if (Call->Arguments.size() >= 2) {
936 ((Opcode != SPIRV::OpMemoryBarrier && Call->Arguments.size() == 2) ||
937 (Opcode == SPIRV::OpMemoryBarrier && Call->Arguments.size() == 3)) &&
938 "Extra args for explicitly scoped barrier");
939 Register ScopeArg = (Opcode == SPIRV::OpMemoryBarrier) ? Call->Arguments[2]
940 : Call->Arguments[1];
941 SPIRV::CLMemoryScope CLScope =
944 if (!(MemFlags & SPIRV::CLK_LOCAL_MEM_FENCE) ||
945 (Opcode == SPIRV::OpMemoryBarrier))
948 if (CLScope ==
static_cast<unsigned>(Scope)) {
949 ScopeReg = Call->Arguments[1];
950 MRI->setRegClass(ScopeReg, &SPIRV::IDRegClass);
958 if (Opcode != SPIRV::OpMemoryBarrier)
960 MIB.
addUse(MemSemanticsReg);
966 case SPIRV::Dim::DIM_1D:
967 case SPIRV::Dim::DIM_Buffer:
969 case SPIRV::Dim::DIM_2D:
970 case SPIRV::Dim::DIM_Cube:
971 case SPIRV::Dim::DIM_Rect:
973 case SPIRV::Dim::DIM_3D:
986 return arrayed ? numComps + 1 : numComps;
999 SPIRV::lookupExtendedBuiltin(Builtin->
Name, Builtin->
Set)->Number;
1004 .
addDef(Call->ReturnRegister)
1006 .
addImm(
static_cast<uint32_t>(SPIRV::InstructionSet::OpenCL_std))
1009 for (
auto Argument : Call->Arguments)
1020 SPIRV::lookupNativeBuiltin(Builtin->
Name, Builtin->
Set)->Opcode;
1024 std::tie(CompareRegister, RelationType) =
1032 for (
auto Argument : Call->Arguments)
1036 return buildSelectInst(MIRBuilder, Call->ReturnRegister, CompareRegister,
1037 Call->ReturnType, GR);
1045 SPIRV::lookupGroupBuiltin(Builtin->
Name);
1048 if (Call->isSpirvOp()) {
1054 Register GroupOpReg = Call->Arguments[1];
1056 if (!
MI ||
MI->getOpcode() != TargetOpcode::G_CONSTANT)
1058 "Group Operation parameter must be an integer constant");
1059 uint64_t GrpOp =
MI->getOperand(1).getCImm()->getValue().getZExtValue();
1060 Register ScopeReg = Call->Arguments[0];
1061 if (!
MRI->getRegClassOrNull(ScopeReg))
1062 MRI->setRegClass(ScopeReg, &SPIRV::IDRegClass);
1063 Register ValueReg = Call->Arguments[2];
1064 if (!
MRI->getRegClassOrNull(ValueReg))
1065 MRI->setRegClass(ValueReg, &SPIRV::IDRegClass);
1067 .
addDef(Call->ReturnRegister)
1077 Register ConstRegister = Call->Arguments[0];
1079 (void)ArgInstruction;
1081 assert(ArgInstruction->getOpcode() == TargetOpcode::G_CONSTANT &&
1082 "Only constant bool value args are supported");
1089 Register GroupResultRegister = Call->ReturnRegister;
1090 SPIRVType *GroupResultType = Call->ReturnType;
1094 const bool HasBoolReturnTy =
1099 if (HasBoolReturnTy)
1100 std::tie(GroupResultRegister, GroupResultType) =
1103 auto Scope = Builtin->
Name.
starts_with(
"sub_group") ? SPIRV::Scope::Subgroup
1104 : SPIRV::Scope::Workgroup;
1109 .
addDef(GroupResultRegister)
1115 if (Call->Arguments.size() > 0) {
1117 MRI->setRegClass(Call->Arguments[0], &SPIRV::IDRegClass);
1118 for (
unsigned i = 1; i < Call->Arguments.size(); i++) {
1119 MIB.addUse(Call->Arguments[i]);
1120 MRI->setRegClass(Call->Arguments[i], &SPIRV::IDRegClass);
1125 if (HasBoolReturnTy)
1126 buildSelectInst(MIRBuilder, Call->ReturnRegister, GroupResultRegister,
1127 Call->ReturnType, GR);
1137 if (!ST->canUseExtension(SPIRV::Extension::SPV_INTEL_subgroups)) {
1138 std::string DiagMsg = std::string(Builtin->
Name) +
1139 ": the builtin requires the following SPIR-V "
1140 "extension: SPV_INTEL_subgroups";
1144 SPIRV::lookupIntelSubgroupsBuiltin(Builtin->
Name);
1147 if (Call->isSpirvOp()) {
1148 bool IsSet = OpCode != SPIRV::OpSubgroupBlockWriteINTEL &&
1149 OpCode != SPIRV::OpSubgroupImageBlockWriteINTEL;
1156 if (IntelSubgroups->
IsBlock) {
1159 if (Arg0Type->getOpcode() == SPIRV::OpTypeImage) {
1165 case SPIRV::OpSubgroupBlockReadINTEL:
1166 OpCode = SPIRV::OpSubgroupImageBlockReadINTEL;
1168 case SPIRV::OpSubgroupBlockWriteINTEL:
1169 OpCode = SPIRV::OpSubgroupImageBlockWriteINTEL;
1190 .
addDef(Call->ReturnRegister)
1192 for (
size_t i = 0; i < Call->Arguments.size(); ++i) {
1193 MIB.
addUse(Call->Arguments[i]);
1194 MRI->setRegClass(Call->Arguments[i], &SPIRV::IDRegClass);
1206 if (!ST->canUseExtension(
1207 SPIRV::Extension::SPV_KHR_uniform_group_instructions)) {
1208 std::string DiagMsg = std::string(Builtin->
Name) +
1209 ": the builtin requires the following SPIR-V "
1210 "extension: SPV_KHR_uniform_group_instructions";
1214 SPIRV::lookupGroupUniformBuiltin(Builtin->
Name);
1217 Register GroupResultReg = Call->ReturnRegister;
1218 MRI->setRegClass(GroupResultReg, &SPIRV::IDRegClass);
1221 Register ScopeReg = Call->Arguments[0];
1222 MRI->setRegClass(ScopeReg, &SPIRV::IDRegClass);
1225 Register ConstGroupOpReg = Call->Arguments[1];
1227 if (!Const || Const->getOpcode() != TargetOpcode::G_CONSTANT)
1229 "expect a constant group operation for a uniform group instruction",
1232 if (!ConstOperand.
isCImm())
1238 Register ValueReg = Call->Arguments[2];
1239 MRI->setRegClass(ValueReg, &SPIRV::IDRegClass);
1246 MIB.addUse(ValueReg);
1257 if (!ST->canUseExtension(SPIRV::Extension::SPV_KHR_shader_clock)) {
1258 std::string DiagMsg = std::string(Builtin->
Name) +
1259 ": the builtin requires the following SPIR-V "
1260 "extension: SPV_KHR_shader_clock";
1265 Register ResultReg = Call->ReturnRegister;
1266 MRI->setRegClass(ResultReg, &SPIRV::IDRegClass);
1269 SPIRV::Scope::Scope ScopeArg =
1271 .
EndsWith(
"device", SPIRV::Scope::Scope::Device)
1272 .
EndsWith(
"work_group", SPIRV::Scope::Scope::Workgroup)
1273 .
EndsWith(
"sub_group", SPIRV::Scope::Scope::Subgroup);
1313 SPIRV::BuiltIn::BuiltIn BuiltinValue,
1315 Register IndexRegister = Call->Arguments[0];
1316 const unsigned ResultWidth = Call->ReturnType->getOperand(1).getImm();
1324 Register ToTruncate = Call->ReturnRegister;
1327 bool IsConstantIndex =
1328 IndexInstruction->getOpcode() == TargetOpcode::G_CONSTANT;
1333 Register DefaultReg = Call->ReturnRegister;
1334 if (PointerSize != ResultWidth) {
1335 DefaultReg =
MRI->createGenericVirtualRegister(
LLT::scalar(PointerSize));
1336 MRI->setRegClass(DefaultReg, &SPIRV::IDRegClass);
1338 MIRBuilder.
getMF());
1339 ToTruncate = DefaultReg;
1343 MIRBuilder.
buildCopy(DefaultReg, NewRegister);
1351 Register Extracted = Call->ReturnRegister;
1352 if (!IsConstantIndex || PointerSize != ResultWidth) {
1353 Extracted =
MRI->createGenericVirtualRegister(
LLT::scalar(PointerSize));
1354 MRI->setRegClass(Extracted, &SPIRV::IDRegClass);
1361 ExtractInst.
addUse(LoadedVector).
addUse(IndexRegister);
1364 if (!IsConstantIndex) {
1373 MRI->setRegClass(CompareRegister, &SPIRV::IDRegClass);
1386 Register SelectionResult = Call->ReturnRegister;
1387 if (PointerSize != ResultWidth) {
1390 MRI->setRegClass(SelectionResult, &SPIRV::IDRegClass);
1392 MIRBuilder.
getMF());
1395 MIRBuilder.
buildSelect(SelectionResult, CompareRegister, Extracted,
1397 ToTruncate = SelectionResult;
1399 ToTruncate = Extracted;
1403 if (PointerSize != ResultWidth)
1413 SPIRV::BuiltIn::BuiltIn
Value =
1414 SPIRV::lookupGetBuiltin(Builtin->
Name, Builtin->
Set)->
Value;
1416 if (
Value == SPIRV::BuiltIn::GlobalInvocationId)
1422 if (Call->ReturnType->getOpcode() == SPIRV::OpTypeVector)
1429 LLType, Call->ReturnRegister);
1438 SPIRV::lookupNativeBuiltin(Builtin->
Name, Builtin->
Set)->Opcode;
1441 case SPIRV::OpStore:
1443 case SPIRV::OpAtomicLoad:
1445 case SPIRV::OpAtomicStore:
1447 case SPIRV::OpAtomicCompareExchange:
1448 case SPIRV::OpAtomicCompareExchangeWeak:
1451 case SPIRV::OpAtomicIAdd:
1452 case SPIRV::OpAtomicISub:
1453 case SPIRV::OpAtomicOr:
1454 case SPIRV::OpAtomicXor:
1455 case SPIRV::OpAtomicAnd:
1456 case SPIRV::OpAtomicExchange:
1458 case SPIRV::OpMemoryBarrier:
1460 case SPIRV::OpAtomicFlagTestAndSet:
1461 case SPIRV::OpAtomicFlagClear:
1473 unsigned Opcode = SPIRV::lookupAtomicFloatingBuiltin(Builtin->
Name)->Opcode;
1476 case SPIRV::OpAtomicFAddEXT:
1477 case SPIRV::OpAtomicFMinEXT:
1478 case SPIRV::OpAtomicFMaxEXT:
1491 SPIRV::lookupNativeBuiltin(Builtin->
Name, Builtin->
Set)->Opcode;
1498 MIRBuilder.
buildInstr(TargetOpcode::G_ADDRSPACE_CAST)
1499 .
addDef(Call->ReturnRegister)
1500 .
addUse(Call->Arguments[0]);
1508 bool IsVec = Opcode == SPIRV::OpTypeVector;
1510 MIRBuilder.
buildInstr(IsVec ? SPIRV::OpDot : SPIRV::OpFMulS)
1511 .
addDef(Call->ReturnRegister)
1513 .
addUse(Call->Arguments[0])
1514 .
addUse(Call->Arguments[1]);
1522 SPIRV::BuiltIn::BuiltIn
Value =
1523 SPIRV::lookupGetBuiltin(Builtin->
Name, Builtin->
Set)->
Value;
1526 assert(Call->ReturnType->getOpcode() == SPIRV::OpTypeInt);
1530 MIRBuilder, Call->ReturnType, GR,
Value, LLType, Call->ReturnRegister,
1538 SPIRV::BuiltIn::BuiltIn
Value =
1539 SPIRV::lookupGetBuiltin(Call->Builtin->Name, Call->Builtin->Set)->
Value;
1540 uint64_t IsDefault = (
Value == SPIRV::BuiltIn::GlobalSize ||
1541 Value == SPIRV::BuiltIn::WorkgroupSize ||
1542 Value == SPIRV::BuiltIn::EnqueuedWorkgroupSize);
1552 SPIRV::lookupImageQueryBuiltin(Builtin->
Name, Builtin->
Set)->Component;
1557 unsigned NumExpectedRetComponents =
RetTy->getOpcode() == SPIRV::OpTypeVector
1558 ?
RetTy->getOperand(2).getImm()
1563 Register QueryResult = Call->ReturnRegister;
1564 SPIRVType *QueryResultType = Call->ReturnType;
1565 if (NumExpectedRetComponents != NumActualRetComponents) {
1571 IntTy, NumActualRetComponents, MIRBuilder);
1576 IsDimBuf ? SPIRV::OpImageQuerySize : SPIRV::OpImageQuerySizeLod;
1581 .
addUse(Call->Arguments[0]);
1584 if (NumExpectedRetComponents == NumActualRetComponents)
1586 if (NumExpectedRetComponents == 1) {
1588 unsigned ExtractedComposite =
1589 Component == 3 ? NumActualRetComponents - 1 : Component;
1590 assert(ExtractedComposite < NumActualRetComponents &&
1591 "Invalid composite index!");
1594 if (QueryResultType->
getOpcode() == SPIRV::OpTypeVector) {
1596 if (TypeReg != NewTypeReg &&
1598 TypeReg = NewTypeReg;
1600 MIRBuilder.
buildInstr(SPIRV::OpCompositeExtract)
1601 .
addDef(Call->ReturnRegister)
1604 .
addImm(ExtractedComposite);
1605 if (NewType !=
nullptr)
1610 auto MIB = MIRBuilder.
buildInstr(SPIRV::OpVectorShuffle)
1611 .
addDef(Call->ReturnRegister)
1615 for (
unsigned i = 0; i < NumExpectedRetComponents; ++i)
1616 MIB.
addImm(i < NumActualRetComponents ? i : 0xffffffff);
1624 assert(Call->ReturnType->getOpcode() == SPIRV::OpTypeInt &&
1625 "Image samples query result must be of int type!");
1630 SPIRV::lookupNativeBuiltin(Builtin->
Name, Builtin->
Set)->Opcode;
1632 Register Image = Call->Arguments[0];
1634 SPIRV::Dim::Dim ImageDimensionality =
static_cast<SPIRV::Dim::Dim
>(
1636 (void)ImageDimensionality;
1639 case SPIRV::OpImageQuerySamples:
1640 assert(ImageDimensionality == SPIRV::Dim::DIM_2D &&
1641 "Image must be of 2D dimensionality");
1643 case SPIRV::OpImageQueryLevels:
1644 assert((ImageDimensionality == SPIRV::Dim::DIM_1D ||
1645 ImageDimensionality == SPIRV::Dim::DIM_2D ||
1646 ImageDimensionality == SPIRV::Dim::DIM_3D ||
1647 ImageDimensionality == SPIRV::Dim::DIM_Cube) &&
1648 "Image must be of 1D/2D/3D/Cube dimensionality");
1653 .
addDef(Call->ReturnRegister)
1660static SPIRV::SamplerAddressingMode::SamplerAddressingMode
1662 switch (Bitmask & SPIRV::CLK_ADDRESS_MODE_MASK) {
1663 case SPIRV::CLK_ADDRESS_CLAMP:
1664 return SPIRV::SamplerAddressingMode::Clamp;
1665 case SPIRV::CLK_ADDRESS_CLAMP_TO_EDGE:
1666 return SPIRV::SamplerAddressingMode::ClampToEdge;
1667 case SPIRV::CLK_ADDRESS_REPEAT:
1668 return SPIRV::SamplerAddressingMode::Repeat;
1669 case SPIRV::CLK_ADDRESS_MIRRORED_REPEAT:
1670 return SPIRV::SamplerAddressingMode::RepeatMirrored;
1671 case SPIRV::CLK_ADDRESS_NONE:
1672 return SPIRV::SamplerAddressingMode::None;
1679 return (Bitmask & SPIRV::CLK_NORMALIZED_COORDS_TRUE) ? 1 : 0;
1682static SPIRV::SamplerFilterMode::SamplerFilterMode
1684 if (Bitmask & SPIRV::CLK_FILTER_LINEAR)
1685 return SPIRV::SamplerFilterMode::Linear;
1686 if (Bitmask & SPIRV::CLK_FILTER_NEAREST)
1687 return SPIRV::SamplerFilterMode::Nearest;
1688 return SPIRV::SamplerFilterMode::Nearest;
1695 Register Image = Call->Arguments[0];
1697 MRI->setRegClass(Image, &SPIRV::IDRegClass);
1698 MRI->setRegClass(Call->Arguments[1], &SPIRV::IDRegClass);
1701 if (HasOclSampler || HasMsaa)
1702 MRI->setRegClass(Call->Arguments[2], &SPIRV::IDRegClass);
1703 if (HasOclSampler) {
1704 Register Sampler = Call->Arguments[1];
1718 Register SampledImage =
MRI->createVirtualRegister(&SPIRV::IDRegClass);
1729 bool NeedsExtraction =
false;
1730 if (TempType->
getOpcode() != SPIRV::OpTypeVector) {
1733 NeedsExtraction =
true;
1736 Register TempRegister =
MRI->createGenericVirtualRegister(LLType);
1737 MRI->setRegClass(TempRegister, &SPIRV::IDRegClass);
1740 MIRBuilder.
buildInstr(SPIRV::OpImageSampleExplicitLod)
1741 .
addDef(NeedsExtraction ? TempRegister : Call->ReturnRegister)
1744 .
addUse(Call->Arguments[2])
1745 .
addImm(SPIRV::ImageOperand::Lod)
1748 if (NeedsExtraction)
1749 MIRBuilder.
buildInstr(SPIRV::OpCompositeExtract)
1750 .
addDef(Call->ReturnRegister)
1754 }
else if (HasMsaa) {
1756 .
addDef(Call->ReturnRegister)
1759 .
addUse(Call->Arguments[1])
1760 .
addImm(SPIRV::ImageOperand::Sample)
1761 .
addUse(Call->Arguments[2]);
1764 .
addDef(Call->ReturnRegister)
1767 .
addUse(Call->Arguments[1]);
1779 .
addUse(Call->Arguments[0])
1780 .
addUse(Call->Arguments[1])
1781 .
addUse(Call->Arguments[2]);
1790 if (Call->Builtin->Name.contains_insensitive(
1791 "__translate_sampler_initializer")) {
1798 return Sampler.isValid();
1799 }
else if (Call->Builtin->Name.contains_insensitive(
"__spirv_SampledImage")) {
1801 Register Image = Call->Arguments[0];
1806 Call->ReturnRegister.isValid()
1807 ? Call->ReturnRegister
1808 :
MRI->createVirtualRegister(&SPIRV::IDRegClass);
1813 .
addUse(Call->Arguments[1]);
1815 }
else if (Call->Builtin->Name.contains_insensitive(
1816 "__spirv_ImageSampleExplicitLod")) {
1818 std::string ReturnType = DemangledCall.
str();
1819 if (DemangledCall.
contains(
"_R")) {
1820 ReturnType = ReturnType.substr(ReturnType.find(
"_R") + 2);
1821 ReturnType = ReturnType.substr(0, ReturnType.find(
'('));
1828 std::string DiagMsg =
1829 "Unable to recognize SPIRV type name: " + ReturnType;
1832 MRI->setRegClass(Call->Arguments[0], &SPIRV::IDRegClass);
1833 MRI->setRegClass(Call->Arguments[1], &SPIRV::IDRegClass);
1834 MRI->setRegClass(Call->Arguments[3], &SPIRV::IDRegClass);
1836 MIRBuilder.
buildInstr(SPIRV::OpImageSampleExplicitLod)
1837 .
addDef(Call->ReturnRegister)
1839 .
addUse(Call->Arguments[0])
1840 .
addUse(Call->Arguments[1])
1841 .
addImm(SPIRV::ImageOperand::Lod)
1842 .
addUse(Call->Arguments[3]);
1850 MIRBuilder.
buildSelect(Call->ReturnRegister, Call->Arguments[0],
1851 Call->Arguments[1], Call->Arguments[2]);
1867 SPIRV::lookupNativeBuiltin(Builtin->
Name, Builtin->
Set)->Opcode;
1868 bool IsSet = Opcode != SPIRV::OpCooperativeMatrixStoreKHR;
1869 unsigned ArgSz = Call->Arguments.size();
1870 unsigned LiteralIdx = 0;
1871 if (Opcode == SPIRV::OpCooperativeMatrixLoadKHR && ArgSz > 3)
1873 else if (Opcode == SPIRV::OpCooperativeMatrixStoreKHR && ArgSz > 4)
1880 if (Opcode == SPIRV::OpCooperativeMatrixLengthKHR) {
1885 .
addDef(Call->ReturnRegister)
1891 IsSet ? TypeReg :
Register(0), ImmArgs);
1900 SPIRV::lookupNativeBuiltin(Builtin->
Name, Builtin->
Set)->Opcode;
1904 case SPIRV::OpSpecConstant: {
1908 buildOpDecorate(Call->ReturnRegister, MIRBuilder, SPIRV::Decoration::SpecId,
1911 Register ConstRegister = Call->Arguments[1];
1914 (Const->getOpcode() == TargetOpcode::G_CONSTANT ||
1915 Const->getOpcode() == TargetOpcode::G_FCONSTANT) &&
1916 "Argument should be either an int or floating-point constant");
1919 if (Call->ReturnType->getOpcode() == SPIRV::OpTypeBool) {
1920 assert(ConstOperand.
isCImm() &&
"Int constant operand is expected");
1922 ? SPIRV::OpSpecConstantTrue
1923 : SPIRV::OpSpecConstantFalse;
1926 .
addDef(Call->ReturnRegister)
1929 if (Call->ReturnType->getOpcode() != SPIRV::OpTypeBool) {
1930 if (Const->getOpcode() == TargetOpcode::G_CONSTANT)
1937 case SPIRV::OpSpecConstantComposite: {
1939 .
addDef(Call->ReturnRegister)
1941 for (
unsigned i = 0; i < Call->Arguments.size(); i++)
1942 MIB.
addUse(Call->Arguments[i]);
1954 MRI->setRegClass(Call->Arguments[0], &SPIRV::IDRegClass);
1961 Register TmpReg =
MRI->createVirtualRegister(&SPIRV::IDRegClass);
1965 unsigned NumArgs = Call->Arguments.size();
1967 Register GlobalWorkSize = Call->Arguments[NumArgs < 4 ? 1 : 2];
1968 MRI->setRegClass(GlobalWorkSize, &SPIRV::IDRegClass);
1970 NumArgs == 2 ?
Register(0) : Call->Arguments[NumArgs < 4 ? 2 : 3];
1972 MRI->setRegClass(LocalWorkSize, &SPIRV::IDRegClass);
1973 Register GlobalWorkOffset = NumArgs <= 3 ?
Register(0) : Call->Arguments[1];
1974 if (GlobalWorkOffset.
isValid())
1975 MRI->setRegClass(GlobalWorkOffset, &SPIRV::IDRegClass);
1979 if (SpvTy->
getOpcode() == SPIRV::OpTypePointer) {
1984 if (!
MRI->getRegClassOrNull(GWSPtr))
1985 MRI->setRegClass(GWSPtr, &SPIRV::IDRegClass);
1987 unsigned Size = Call->Builtin->Name ==
"ndrange_3D" ? 3 : 2;
1992 GlobalWorkSize =
MRI->createVirtualRegister(&SPIRV::IDRegClass);
2001 SpvFieldTy, *ST.getInstrInfo());
2006 LocalWorkSize = Const;
2007 if (!GlobalWorkOffset.
isValid())
2008 GlobalWorkOffset = Const;
2016 .
addUse(GlobalWorkOffset);
2018 .
addUse(Call->Arguments[0])
2043 bool IsSpirvOp = Call->isSpirvOp();
2044 bool HasEvents = Call->Builtin->Name.contains(
"events") || IsSpirvOp;
2051 if (Call->Builtin->Name.contains(
"_varargs") || IsSpirvOp) {
2052 const unsigned LocalSizeArrayIdx = HasEvents ? 9 : 6;
2053 Register GepReg = Call->Arguments[LocalSizeArrayIdx];
2060 assert(LocalSizeTy &&
"Local size type is expected");
2062 cast<ArrayType>(LocalSizeTy)->getNumElements();
2066 Int32Ty, MIRBuilder, SPIRV::StorageClass::Function);
2067 for (
unsigned I = 0;
I < LocalSizeNum; ++
I) {
2069 MRI->setType(
Reg, LLType);
2083 auto MIB = MIRBuilder.
buildInstr(SPIRV::OpEnqueueKernel)
2084 .
addDef(Call->ReturnRegister)
2088 const unsigned BlockFIdx = HasEvents ? 6 : 3;
2089 for (
unsigned i = 0; i < BlockFIdx; i++)
2090 MIB.addUse(Call->Arguments[i]);
2097 MIB.addUse(NullPtr);
2098 MIB.addUse(NullPtr);
2106 Register BlockLiteralReg = Call->Arguments[BlockFIdx + 1];
2108 MIB.addUse(BlockLiteralReg);
2118 for (
unsigned i = 0; i < LocalSizes.
size(); i++)
2119 MIB.addUse(LocalSizes[i]);
2129 SPIRV::lookupNativeBuiltin(Builtin->
Name, Builtin->
Set)->Opcode;
2132 case SPIRV::OpRetainEvent:
2133 case SPIRV::OpReleaseEvent:
2136 case SPIRV::OpCreateUserEvent:
2137 case SPIRV::OpGetDefaultQueue:
2139 .
addDef(Call->ReturnRegister)
2141 case SPIRV::OpIsValidEvent:
2144 .
addDef(Call->ReturnRegister)
2146 .
addUse(Call->Arguments[0]);
2147 case SPIRV::OpSetUserEventStatus:
2151 .
addUse(Call->Arguments[0])
2152 .
addUse(Call->Arguments[1]);
2153 case SPIRV::OpCaptureEventProfilingInfo:
2158 .
addUse(Call->Arguments[0])
2159 .
addUse(Call->Arguments[1])
2160 .
addUse(Call->Arguments[2]);
2161 case SPIRV::OpBuildNDRange:
2163 case SPIRV::OpEnqueueKernel:
2176 SPIRV::lookupNativeBuiltin(Builtin->
Name, Builtin->
Set)->Opcode;
2178 bool IsSet = Opcode == SPIRV::OpGroupAsyncCopy;
2180 if (Call->isSpirvOp())
2187 case SPIRV::OpGroupAsyncCopy: {
2189 Call->ReturnType->getOpcode() == SPIRV::OpTypeEvent
2193 unsigned NumArgs = Call->Arguments.size();
2194 Register EventReg = Call->Arguments[NumArgs - 1];
2196 .
addDef(Call->ReturnRegister)
2199 .
addUse(Call->Arguments[0])
2200 .
addUse(Call->Arguments[1])
2201 .
addUse(Call->Arguments[2])
2202 .
addUse(Call->Arguments.size() > 4
2203 ? Call->Arguments[3]
2206 if (NewType !=
nullptr)
2211 case SPIRV::OpGroupWaitEvents:
2214 .
addUse(Call->Arguments[0])
2215 .
addUse(Call->Arguments[1]);
2227 SPIRV::lookupConvertBuiltin(Call->Builtin->Name, Call->Builtin->Set);
2231 SPIRV::Decoration::SaturatedConversion, {});
2234 SPIRV::Decoration::FPRoundingMode,
2235 {(unsigned)Builtin->RoundingMode});
2237 std::string NeedExtMsg;
2238 bool IsRightComponentsNumber =
true;
2239 unsigned Opcode = SPIRV::OpNop;
2246 : SPIRV::OpSatConvertSToU;
2249 : SPIRV::OpSConvert;
2251 SPIRV::OpTypeFloat)) {
2256 if (!ST->canUseExtension(
2257 SPIRV::Extension::SPV_INTEL_bfloat16_conversion))
2258 NeedExtMsg =
"SPV_INTEL_bfloat16_conversion";
2259 IsRightComponentsNumber =
2262 Opcode = SPIRV::OpConvertBF16ToFINTEL;
2264 bool IsSourceSigned =
2266 Opcode = IsSourceSigned ? SPIRV::OpConvertSToF : SPIRV::OpConvertUToF;
2270 SPIRV::OpTypeFloat)) {
2277 if (!ST->canUseExtension(
2278 SPIRV::Extension::SPV_INTEL_bfloat16_conversion))
2279 NeedExtMsg =
"SPV_INTEL_bfloat16_conversion";
2280 IsRightComponentsNumber =
2283 Opcode = SPIRV::OpConvertFToBF16INTEL;
2286 : SPIRV::OpConvertFToU;
2289 SPIRV::OpTypeFloat)) {
2291 Opcode = SPIRV::OpFConvert;
2295 if (!NeedExtMsg.empty()) {
2296 std::string DiagMsg = std::string(Builtin->
Name) +
2297 ": the builtin requires the following SPIR-V "
2302 if (!IsRightComponentsNumber) {
2303 std::string DiagMsg =
2304 std::string(Builtin->
Name) +
2305 ": result and argument must have the same number of components";
2308 assert(Opcode != SPIRV::OpNop &&
2309 "Conversion between the types not implemented!");
2312 .
addDef(Call->ReturnRegister)
2314 .
addUse(Call->Arguments[0]);
2323 SPIRV::lookupVectorLoadStoreBuiltin(Call->Builtin->Name,
2324 Call->Builtin->Set);
2328 .
addDef(Call->ReturnRegister)
2330 .
addImm(
static_cast<uint32_t>(SPIRV::InstructionSet::OpenCL_std))
2332 for (
auto Argument : Call->Arguments)
2350 SPIRV::lookupNativeBuiltin(Builtin->
Name, Builtin->
Set)->Opcode;
2351 bool IsLoad = Opcode == SPIRV::OpLoad;
2355 MIB.
addDef(Call->ReturnRegister);
2359 MIB.
addUse(Call->Arguments[0]);
2361 MRI->setRegClass(Call->Arguments[0], &SPIRV::IDRegClass);
2364 MIB.addUse(Call->Arguments[1]);
2365 MRI->setRegClass(Call->Arguments[1], &SPIRV::IDRegClass);
2368 unsigned NumArgs = Call->Arguments.size();
2369 if ((IsLoad && NumArgs >= 2) || NumArgs >= 3) {
2371 MRI->setRegClass(Call->Arguments[IsLoad ? 1 : 2], &SPIRV::IDRegClass);
2373 if ((IsLoad && NumArgs >= 3) || NumArgs >= 4) {
2375 MRI->setRegClass(Call->Arguments[IsLoad ? 2 : 3], &SPIRV::IDRegClass);
2384 SPIRV::InstructionSet::InstructionSet Set,
2389 LLVM_DEBUG(
dbgs() <<
"Lowering builtin call: " << DemangledCall <<
"\n");
2394 if (OrigRetTy && !OrigRetTy->
isVoidTy()) {
2398 }
else if (OrigRetTy && OrigRetTy->
isVoidTy()) {
2405 std::unique_ptr<const IncomingCall> Call =
2406 lookupBuiltin(DemangledCall, Set, ReturnRegister, ReturnType, Args);
2410 return std::nullopt;
2414 assert(Args.size() >= Call->Builtin->MinNumArgs &&
2415 "Too few arguments to generate the builtin");
2416 if (Call->Builtin->MaxNumArgs && Args.size() > Call->Builtin->MaxNumArgs)
2417 LLVM_DEBUG(
dbgs() <<
"More arguments provided than required!\n");
2420 switch (Call->Builtin->Group) {
2421 case SPIRV::Extended:
2423 case SPIRV::Relational:
2427 case SPIRV::Variable:
2431 case SPIRV::AtomicFloating:
2433 case SPIRV::Barrier:
2435 case SPIRV::CastToPtr:
2441 case SPIRV::GetQuery:
2443 case SPIRV::ImageSizeQuery:
2445 case SPIRV::ImageMiscQuery:
2447 case SPIRV::ReadImage:
2449 case SPIRV::WriteImage:
2451 case SPIRV::SampleImage:
2455 case SPIRV::Construct:
2457 case SPIRV::SpecConstant:
2459 case SPIRV::Enqueue:
2461 case SPIRV::AsyncCopy:
2463 case SPIRV::Convert:
2465 case SPIRV::VectorLoadStore:
2467 case SPIRV::LoadStore:
2469 case SPIRV::IntelSubgroups:
2471 case SPIRV::GroupUniform:
2473 case SPIRV::KernelClock:
2475 case SPIRV::CoopMatr:
2485 DemangledCall.
slice(DemangledCall.
find(
'(') + 1, DemangledCall.
find(
')'));
2486 BuiltinArgs.
split(BuiltinArgsTypeStrs,
',', -1,
false);
2487 if (ArgIdx >= BuiltinArgsTypeStrs.
size())
2489 StringRef TypeStr = BuiltinArgsTypeStrs[ArgIdx].trim();
2495 [[maybe_unused]]
bool IsOCLBuiltinType = TypeStr.
consume_front(
"ocl_");
2496 assert(IsOCLBuiltinType &&
"Invalid OpenCL builtin prefix");
2513 unsigned VecElts = 0;
2527 TypeStr = TypeStr.
substr(0, TypeStr.
find(
']'));
2541#define GET_BuiltinTypes_DECL
2542#define GET_BuiltinTypes_IMPL
2549#define GET_OpenCLTypes_DECL
2550#define GET_OpenCLTypes_IMPL
2552#include "SPIRVGenTables.inc"
2560 if (
Name.starts_with(
"void"))
2562 else if (
Name.starts_with(
"int") ||
Name.starts_with(
"uint"))
2564 else if (
Name.starts_with(
"float"))
2566 else if (
Name.starts_with(
"half"))
2579 unsigned Opcode = TypeRecord->
Opcode;
2594 "Invalid number of parameters for SPIR-V pipe builtin!");
2597 SPIRV::AccessQualifier::AccessQualifier(
2605 "Invalid number of parameters for SPIR-V coop matrices builtin!");
2607 "SPIR-V coop matrices builtin type must have a type parameter!");
2612 MIRBuilder, ExtensionType, ElemType, ExtensionType->
getIntParameter(0),
2619 const SPIRV::AccessQualifier::AccessQualifier Qualifier,
2622 "SPIR-V image builtin type must have sampled type parameter!");
2626 "Invalid number of parameters for SPIR-V image builtin!");
2629 MIRBuilder, SampledType,
2634 Qualifier == SPIRV::AccessQualifier::WriteOnly
2635 ? SPIRV::AccessQualifier::WriteOnly
2636 : SPIRV::AccessQualifier::AccessQualifier(
2644 OpaqueType, SPIRV::AccessQualifier::ReadOnly, MIRBuilder, GR);
2652 StringRef NameWithParameters = TypeName;
2659 SPIRV::lookupOpenCLType(NameWithParameters);
2662 NameWithParameters);
2670 "Unknown builtin opaque type!");
2674 if (!NameWithParameters.
contains(
'_'))
2678 unsigned BaseNameLength = NameWithParameters.
find(
'_') - 1;
2679 SplitString(NameWithParameters.
substr(BaseNameLength + 1), Parameters,
"_");
2682 bool HasTypeParameter = !
isDigit(Parameters[0][0]);
2683 if (HasTypeParameter)
2686 for (
unsigned i = HasTypeParameter ? 1 : 0; i < Parameters.size(); i++) {
2687 unsigned IntParameter = 0;
2688 bool ValidLiteral = !Parameters[i].getAsInteger(10, IntParameter);
2691 "Invalid format of SPIR-V builtin parameter literal!");
2695 NameWithParameters.
substr(0, BaseNameLength),
2696 TypeParameters, IntParameters);
2700 SPIRV::AccessQualifier::AccessQualifier AccessQual,
2731 switch (TypeRecord->
Opcode) {
2732 case SPIRV::OpTypeImage:
2735 case SPIRV::OpTypePipe:
2738 case SPIRV::OpTypeDeviceEvent:
2741 case SPIRV::OpTypeSampler:
2744 case SPIRV::OpTypeSampledImage:
2747 case SPIRV::OpTypeCooperativeMatrixKHR:
unsigned const MachineRegisterInfo * MRI
MachineInstrBuilder MachineInstrBuilder & DefMI
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
AMDGPU Lower Kernel Arguments
static bool isDigit(const char C)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
BaseType
A given derived pointer can have multiple base pointers through phi/selects.
static bool contains(SmallPtrSetImpl< ConstantExpr * > &Cache, ConstantExpr *Expr, Constant *C)
APInt bitcastToAPInt() const
static APFloat getZero(const fltSemantics &Sem, bool Negative=false)
Factory for Positive and Negative Zero.
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
uint64_t getZExtValue() const
Get zero extended value.
This class represents an incoming formal argument to a Function.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
static ArrayType * get(Type *ElementType, uint64_t NumElements)
This static method is the primary way to construct an ArrayType.
@ ICMP_ULT
unsigned less than
const APFloat & getValueAPF() const
const APInt & getValue() const
Return the constant as an APInt value reference.
A parsed version of the target data layout string in and methods for querying it.
Tagged union holding either a T or a Error.
Class to represent fixed width SIMD vectors.
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
static IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
static constexpr LLT vector(ElementCount EC, unsigned ScalarSizeInBits)
Get a low-level vector of some number of elements and element width.
static constexpr LLT scalar(unsigned SizeInBits)
Get a low-level scalar or aggregate "bag of bits".
static constexpr LLT pointer(unsigned AddressSpace, unsigned SizeInBits)
Get a low-level pointer in the given address space.
static constexpr LLT fixed_vector(unsigned NumElements, unsigned ScalarSizeInBits)
Get a low-level fixed-width vector of some number of elements and element width.
This is an important class for using LLVM in a threaded context.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Function & getFunction()
Return the LLVM function that this machine code represents.
Helper class to build MachineInstr.
LLVMContext & getContext() const
MachineInstrBuilder buildSelect(const DstOp &Res, const SrcOp &Tst, const SrcOp &Op0, const SrcOp &Op1, std::optional< unsigned > Flags=std::nullopt)
Build and insert a Res = G_SELECT Tst, Op0, Op1.
MachineInstrBuilder buildICmp(CmpInst::Predicate Pred, const DstOp &Res, const SrcOp &Op0, const SrcOp &Op1)
Build and insert a Res = G_ICMP Pred, Op0, Op1.
MachineBasicBlock::iterator getInsertPt()
Current insertion point for new instructions.
MachineInstrBuilder buildIntrinsic(Intrinsic::ID ID, ArrayRef< Register > Res, bool HasSideEffects, bool isConvergent)
Build and insert a G_INTRINSIC instruction.
MachineInstrBuilder buildLoad(const DstOp &Res, const SrcOp &Addr, MachineMemOperand &MMO)
Build and insert Res = G_LOAD Addr, MMO.
MachineInstrBuilder buildZExtOrTrunc(const DstOp &Res, const SrcOp &Op)
Build and insert Res = G_ZEXT Op, Res = G_TRUNC Op, or Res = COPY Op depending on the differing sizes...
MachineInstrBuilder buildInstr(unsigned Opcode)
Build and insert <empty> = Opcode <empty>.
MachineFunction & getMF()
Getter for the function we currently build.
MachineRegisterInfo * getMRI()
Getter for MRI.
MachineInstrBuilder buildCopy(const DstOp &Res, const SrcOp &Op)
Build and insert Res = COPY Op.
const DataLayout & getDataLayout() const
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & addUse(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register use operand.
const MachineInstrBuilder & addDef(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register definition operand.
Representation of each machine instruction.
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
const MachineOperand & getOperand(unsigned i) const
MachineOperand class - Representation of each machine instruction operand.
const GlobalValue * getGlobal() const
const ConstantInt * getCImm() const
bool isCImm() const
isCImm - Test if this is a MO_CImmediate operand.
bool isReg() const
isReg - Tests if this is a MO_Register operand.
const MDNode * getMetadata() const
Register getReg() const
getReg - Returns the register number.
const ConstantFP * getFPImm() const
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
Register createVirtualRegister(const TargetRegisterClass *RegClass, StringRef Name="")
createVirtualRegister - Create and return a new virtual register in the function with the specified r...
void setType(Register VReg, LLT Ty)
Set the low-level type of VReg to Ty.
void setRegClass(Register Reg, const TargetRegisterClass *RC)
setRegClass - Set the register class of the specified virtual register.
Register createGenericVirtualRegister(LLT Ty, StringRef Name="")
Create and return a new generic virtual register with low-level type Ty.
const TargetRegisterClass * getRegClassOrNull(Register Reg) const
Return the register class of Reg, or null if Reg has not been assigned a register class yet.
unsigned getNumVirtRegs() const
getNumVirtRegs - Return the number of virtual registers created.
static PointerType * get(Type *ElementType, unsigned AddressSpace)
This constructs a pointer to an object of the specified type in a numbered address space.
Wrapper class representing virtual and physical registers.
constexpr bool isValid() const
SPIRVType * getOrCreateOpTypePipe(MachineIRBuilder &MIRBuilder, SPIRV::AccessQualifier::AccessQualifier AccQual)
SPIRVType * getSPIRVTypeForVReg(Register VReg, const MachineFunction *MF=nullptr) const
SPIRVType * getOrCreateSPIRVBoolType(MachineIRBuilder &MIRBuilder)
Register getOrCreateConsIntVector(uint64_t Val, MachineIRBuilder &MIRBuilder, SPIRVType *SpvType, bool EmitIR=true)
const Type * getTypeForSPIRVType(const SPIRVType *Ty) const
Register buildConstantSampler(Register Res, unsigned AddrMode, unsigned Param, unsigned FilerMode, MachineIRBuilder &MIRBuilder, SPIRVType *SpvType)
unsigned getScalarOrVectorComponentCount(Register VReg) const
SPIRVType * getOrCreateOpTypeImage(MachineIRBuilder &MIRBuilder, SPIRVType *SampledType, SPIRV::Dim::Dim Dim, uint32_t Depth, uint32_t Arrayed, uint32_t Multisampled, uint32_t Sampled, SPIRV::ImageFormat::ImageFormat ImageFormat, SPIRV::AccessQualifier::AccessQualifier AccQual)
unsigned getPointerSize() const
SPIRVType * getOrCreateOpTypeByOpcode(const Type *Ty, MachineIRBuilder &MIRBuilder, unsigned Opcode)
Register buildConstantFP(APFloat Val, MachineIRBuilder &MIRBuilder, SPIRVType *SpvType=nullptr)
Register getSPIRVTypeID(const SPIRVType *SpirvType) const
SPIRVType * getOrCreateSPIRVType(const Type *Type, MachineIRBuilder &MIRBuilder, SPIRV::AccessQualifier::AccessQualifier AQ=SPIRV::AccessQualifier::ReadWrite, bool EmitIR=true)
void assignSPIRVTypeToVReg(SPIRVType *Type, Register VReg, MachineFunction &MF)
bool isScalarOfType(Register VReg, unsigned TypeOpcode) const
Register buildGlobalVariable(Register Reg, SPIRVType *BaseType, StringRef Name, const GlobalValue *GV, SPIRV::StorageClass::StorageClass Storage, const MachineInstr *Init, bool IsConst, bool HasLinkageTy, SPIRV::LinkageType::LinkageType LinkageType, MachineIRBuilder &MIRBuilder, bool IsInstSelector)
SPIRVType * getOrCreateOpTypeSampledImage(SPIRVType *ImageType, MachineIRBuilder &MIRBuilder)
SPIRVType * getOrCreateSPIRVTypeByName(StringRef TypeStr, MachineIRBuilder &MIRBuilder, SPIRV::StorageClass::StorageClass SC=SPIRV::StorageClass::Function, SPIRV::AccessQualifier::AccessQualifier AQ=SPIRV::AccessQualifier::ReadWrite)
SPIRVType * assignTypeToVReg(const Type *Type, Register VReg, MachineIRBuilder &MIRBuilder, SPIRV::AccessQualifier::AccessQualifier AQ=SPIRV::AccessQualifier::ReadWrite, bool EmitIR=true)
bool isScalarOrVectorOfType(Register VReg, unsigned TypeOpcode) const
Register getOrCreateConstIntArray(uint64_t Val, size_t Num, MachineInstr &I, SPIRVType *SpvType, const SPIRVInstrInfo &TII)
SPIRVType * getOrCreateOpTypeDeviceEvent(MachineIRBuilder &MIRBuilder)
SPIRVType * getOrCreateSPIRVPointerType(SPIRVType *BaseType, MachineIRBuilder &MIRBuilder, SPIRV::StorageClass::StorageClass SClass=SPIRV::StorageClass::Function)
SPIRVType * getOrCreateOpTypeCoopMatr(MachineIRBuilder &MIRBuilder, const TargetExtType *ExtensionType, const SPIRVType *ElemType, uint32_t Scope, uint32_t Rows, uint32_t Columns, uint32_t Use)
SPIRVType * getOrCreateSPIRVVectorType(SPIRVType *BaseType, unsigned NumElements, MachineIRBuilder &MIRBuilder)
SPIRVType * getOrCreateSPIRVIntegerType(unsigned BitWidth, MachineIRBuilder &MIRBuilder)
SPIRV::StorageClass::StorageClass getPointerStorageClass(Register VReg) const
SPIRVType * getOrCreateOpTypeSampler(MachineIRBuilder &MIRBuilder)
Register buildConstantInt(uint64_t Val, MachineIRBuilder &MIRBuilder, SPIRVType *SpvType=nullptr, bool EmitIR=true)
Register getOrCreateConstNullPtr(MachineIRBuilder &MIRBuilder, SPIRVType *SpvType)
unsigned getScalarOrVectorBitWidth(const SPIRVType *Type) const
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StringRef - Represent a constant reference to a string, i.e.
std::pair< StringRef, StringRef > split(char Separator) const
Split into two substrings around the first occurrence of a separator character.
bool consume_back(StringRef Suffix)
Returns true if this StringRef has the given suffix and removes that suffix.
bool getAsInteger(unsigned Radix, T &Result) const
Parse the current string as an integer of the specified radix.
std::string str() const
str - Get the contents as an std::string.
constexpr StringRef substr(size_t Start, size_t N=npos) const
Return a reference to the substring from [Start, Start + N).
bool starts_with(StringRef Prefix) const
Check if this string starts with the given Prefix.
bool contains_insensitive(StringRef Other) const
Return true if the given string is a substring of *this, and false otherwise.
StringRef slice(size_t Start, size_t End) const
Return a reference to the substring from [Start, End).
bool contains(StringRef Other) const
Return true if the given string is a substring of *this, and false otherwise.
bool consume_front(StringRef Prefix)
Returns true if this StringRef has the given prefix and removes that prefix.
size_t find_first_of(char C, size_t From=0) const
Find the first character in the string that is C, or npos if not found.
size_t find(char C, size_t From=0) const
Search for the first character C in the string.
bool ends_with(StringRef Suffix) const
Check if this string ends with the given Suffix.
A switch()-like statement whose cases are string literals.
StringSwitch & EndsWith(StringLiteral S, T Value)
Class to represent struct types.
static StructType * getTypeByName(LLVMContext &C, StringRef Name)
Return the type with the specified name, or null if there is none by that name.
static StructType * create(LLVMContext &Context, StringRef Name)
This creates an identified struct.
Class to represent target extensions types, which are generally unintrospectable from target-independ...
unsigned getNumIntParameters() const
static TargetExtType * get(LLVMContext &Context, StringRef Name, ArrayRef< Type * > Types=std::nullopt, ArrayRef< unsigned > Ints=std::nullopt)
Return a target extension type having the specified name and optional type and integer parameters.
Type * getTypeParameter(unsigned i) const
unsigned getNumTypeParameters() const
unsigned getIntParameter(unsigned i) const
The instances of the Type class are immutable: once they are created, they are never changed.
static Type * getHalfTy(LLVMContext &C)
StringRef getStructName() const
static Type * getVoidTy(LLVMContext &C)
static IntegerType * getInt8Ty(LLVMContext &C)
static IntegerType * getInt32Ty(LLVMContext &C)
static Type * getFloatTy(LLVMContext &C)
bool isVoidTy() const
Return true if this is 'void'.
LLVM Value Representation.
Value(Type *Ty, unsigned scid)
static VectorType * get(Type *ElementType, ElementCount EC)
This static method is the primary way to construct an VectorType.
NodeTy * getNextNode()
Get the next node, or nullptr for the list tail.
LLVMTypeRef LLVMVectorType(LLVMTypeRef ElementType, unsigned ElementCount)
Create a vector type that contains a defined type and has a specific number of elements.
Type * parseBuiltinCallArgumentBaseType(const StringRef DemangledCall, unsigned ArgIdx, LLVMContext &Ctx)
Parses the provided ArgIdx argument base type in the DemangledCall skeleton.
TargetExtType * parseBuiltinTypeNameToTargetExtType(std::string TypeName, LLVMContext &Context)
Translates a string representing a SPIR-V or OpenCL builtin type to a TargetExtType that can be furth...
std::optional< bool > lowerBuiltin(const StringRef DemangledCall, SPIRV::InstructionSet::InstructionSet Set, MachineIRBuilder &MIRBuilder, const Register OrigRet, const Type *OrigRetTy, const SmallVectorImpl< Register > &Args, SPIRVGlobalRegistry *GR)
SPIRVType * lowerBuiltinType(const Type *OpaqueType, SPIRV::AccessQualifier::AccessQualifier AccessQual, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
This is an optimization pass for GlobalISel generic memory operations.
void buildOpName(Register Target, const StringRef &Name, MachineIRBuilder &MIRBuilder)
unsigned storageClassToAddressSpace(SPIRV::StorageClass::StorageClass SC)
static bool generateGetQueryInst(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
static bool generateLoadStoreInst(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
static Register buildConstantIntReg(uint64_t Val, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR, unsigned BitWidth=32)
static bool generateConstructInst(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
static bool buildAtomicFlagInst(const SPIRV::IncomingCall *Call, unsigned Opcode, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
Helper function for building atomic flag instructions (e.g.
static Register buildBuiltinVariableLoad(MachineIRBuilder &MIRBuilder, SPIRVType *VariableType, SPIRVGlobalRegistry *GR, SPIRV::BuiltIn::BuiltIn BuiltinValue, LLT LLType, Register Reg=Register(0), bool isConst=true, bool hasLinkageTy=true)
Helper function for building a load instruction for loading a builtin global variable of BuiltinValue...
static bool generateImageSizeQueryInst(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
static SPIRV::SamplerFilterMode::SamplerFilterMode getSamplerFilterModeFromBitmask(unsigned Bitmask)
static bool buildAtomicStoreInst(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
Helper function for building an atomic store instruction.
void addNumImm(const APInt &Imm, MachineInstrBuilder &MIB)
static const Type * getBlockStructType(Register ParamReg, MachineRegisterInfo *MRI)
static bool generateGroupInst(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
static unsigned getNumComponentsForDim(SPIRV::Dim::Dim dim)
Register insertAssignInstr(Register Reg, Type *Ty, SPIRVType *SpirvTy, SPIRVGlobalRegistry *GR, MachineIRBuilder &MIB, MachineRegisterInfo &MRI)
Helper external function for inserting ASSIGN_TYPE instuction between Reg and its definition,...
static Register buildScopeReg(Register CLScopeRegister, SPIRV::Scope::Scope Scope, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR, MachineRegisterInfo *MRI)
static std::tuple< Register, SPIRVType * > buildBoolRegister(MachineIRBuilder &MIRBuilder, const SPIRVType *ResultType, SPIRVGlobalRegistry *GR)
Helper function building either a resulting scalar or vector bool register depending on the expected ...
static unsigned getNumSizeComponents(SPIRVType *imgType)
Helper function for obtaining the number of size components.
uint64_t getIConstVal(Register ConstReg, const MachineRegisterInfo *MRI)
static SPIRVType * getSampledImageType(const TargetExtType *OpaqueType, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
SPIRV::MemorySemantics::MemorySemantics getMemSemanticsForStorageClass(SPIRV::StorageClass::StorageClass SC)
static bool generateDotOrFMulInst(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
static bool generateSampleImageInst(const StringRef DemangledCall, const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
static bool generateBarrierInst(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
static SPIRVType * getCoopMatrType(const TargetExtType *ExtensionType, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
static bool generateKernelClockInst(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
static bool generateGroupUniformInst(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
static bool generateWaveInst(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
static SPIRVType * getImageType(const TargetExtType *ExtensionType, const SPIRV::AccessQualifier::AccessQualifier Qualifier, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
void buildOpDecorate(Register Reg, MachineIRBuilder &MIRBuilder, SPIRV::Decoration::Decoration Dec, const std::vector< uint32_t > &DecArgs, StringRef StrImm)
static bool buildBarrierInst(const SPIRV::IncomingCall *Call, unsigned Opcode, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
Helper function for building barriers, i.e., memory/control ordering operations.
static bool generateAsyncCopy(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
static SPIRV::Scope::Scope getSPIRVScope(SPIRV::CLMemoryScope ClScope)
static SPIRVType * getSamplerType(MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
static Register buildLoadInst(SPIRVType *BaseType, Register PtrRegister, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR, LLT LowLevelType, Register DestinationReg=Register(0))
Helper function for building a load instruction loading into the DestinationReg.
static bool generateEnqueueInst(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
static bool buildSelectInst(MachineIRBuilder &MIRBuilder, Register ReturnRegister, Register SourceRegister, const SPIRVType *ReturnType, SPIRVGlobalRegistry *GR)
Helper function for building either a vector or scalar select instruction depending on the expected R...
static const Type * getMachineInstrType(MachineInstr *MI)
static SPIRV::SamplerAddressingMode::SamplerAddressingMode getSamplerAddressingModeFromBitmask(unsigned Bitmask)
static bool generateAtomicInst(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
static bool generateConvertInst(const StringRef DemangledCall, const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
static Register buildMemSemanticsReg(Register SemanticsRegister, Register PtrRegister, unsigned &Semantics, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
static unsigned getConstFromIntrinsic(Register Reg, MachineRegisterInfo *MRI)
static bool generateImageMiscQueryInst(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
static bool generateSelectInst(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder)
static bool buildAtomicLoadInst(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
Helper function for building an atomic load instruction.
static bool generateIntelSubgroupsInst(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
static bool generateSpecConstantInst(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
static SPIRVType * getOrCreateSPIRVDeviceEventPointer(MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
Type * parseBasicTypeName(StringRef &TypeName, LLVMContext &Ctx)
static bool generateVectorLoadStoreInst(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
static bool genWorkgroupQuery(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR, SPIRV::BuiltIn::BuiltIn BuiltinValue, uint64_t DefaultValue)
static bool generateCoopMatrInst(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
static std::unique_ptr< const SPIRV::IncomingCall > lookupBuiltin(StringRef DemangledCall, SPIRV::InstructionSet::InstructionSet Set, Register ReturnRegister, const SPIRVType *ReturnType, const SmallVectorImpl< Register > &Arguments)
Looks up the demangled builtin call in the SPIRVBuiltins.td records using the provided DemangledCall ...
static bool buildAtomicFloatingRMWInst(const SPIRV::IncomingCall *Call, unsigned Opcode, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
Helper function for building an atomic floating-type instruction.
MachineInstr * getDefInstrMaybeConstant(Register &ConstReg, const MachineRegisterInfo *MRI)
constexpr unsigned BitWidth
const MachineInstr SPIRVType
static bool generateReadImageInst(const StringRef DemangledCall, const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
bool hasBuiltinTypePrefix(StringRef Name)
static bool buildEnqueueKernel(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
Type * getMDOperandAsType(const MDNode *N, unsigned I)
static bool buildAtomicRMWInst(const SPIRV::IncomingCall *Call, unsigned Opcode, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
Helper function for building atomic instructions.
static SPIRV::MemorySemantics::MemorySemantics getSPIRVMemSemantics(std::memory_order MemOrder)
static bool generateRelationalInst(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
static bool buildAtomicInitInst(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder)
Helper function for translating atomic init to OpStore.
static bool generateWriteImageInst(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
static SPIRVType * getPipeType(const TargetExtType *ExtensionType, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
static Type * parseTypeString(const StringRef Name, LLVMContext &Context)
bool isSpvIntrinsic(const MachineInstr &MI, Intrinsic::ID IntrinsicID)
static bool generateCastToPtrInst(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder)
static bool generateAtomicFloatingInst(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
static bool generateExtInst(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
static bool buildNDRange(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
static SPIRVType * getNonParameterizedType(const TargetExtType *ExtensionType, const SPIRV::BuiltinType *TypeRecord, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
static MachineInstr * getBlockStructInstr(Register ParamReg, MachineRegisterInfo *MRI)
static bool buildOpFromWrapper(MachineIRBuilder &MIRBuilder, unsigned Opcode, const SPIRV::IncomingCall *Call, Register TypeReg, ArrayRef< uint32_t > ImmArgs={})
static unsigned getSamplerParamFromBitmask(unsigned Bitmask)
static bool buildAtomicCompareExchangeInst(const SPIRV::IncomingCall *Call, const SPIRV::DemangledBuiltin *Builtin, unsigned Opcode, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
Helper function for building an atomic compare-exchange instruction.
std::string getLinkStringForBuiltIn(SPIRV::BuiltIn::BuiltIn BuiltInValue)
static bool generateBuiltinVar(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
static const fltSemantics & IEEEsingle() LLVM_READNONE
This struct is a compact representation of a valid (non-zero power of two) alignment.
This class contains a discriminated union of information about pointers in memory operands,...
FPRoundingMode::FPRoundingMode RoundingMode
InstructionSet::InstructionSet Set
InstructionSet::InstructionSet Set
InstructionSet::InstructionSet Set
InstructionSet::InstructionSet Set
const SmallVectorImpl< Register > & Arguments
const std::string BuiltinName
const SPIRVType * ReturnType
const Register ReturnRegister
const DemangledBuiltin * Builtin
IncomingCall(const std::string BuiltinName, const DemangledBuiltin *Builtin, const Register ReturnRegister, const SPIRVType *ReturnType, const SmallVectorImpl< Register > &Arguments)
InstructionSet::InstructionSet Set
StringRef SpirvTypeLiteral
InstructionSet::InstructionSet Set
FPRoundingMode::FPRoundingMode RoundingMode