37#include "llvm/IR/IntrinsicsSPIRV.h"
43#define DEBUG_TYPE "spirv-isel"
50 std::vector<std::pair<SPIRV::InstructionSet::InstructionSet, uint32_t>>;
55 std::optional<Register> Bias;
56 std::optional<Register>
Offset;
57 std::optional<Register> MinLod;
58 std::optional<Register> GradX;
59 std::optional<Register> GradY;
60 std::optional<Register> Lod;
61 std::optional<Register> Compare;
64llvm::SPIRV::SelectionControl::SelectionControl
65getSelectionOperandForImm(
int Imm) {
67 return SPIRV::SelectionControl::Flatten;
69 return SPIRV::SelectionControl::DontFlatten;
71 return SPIRV::SelectionControl::None;
75#define GET_GLOBALISEL_PREDICATE_BITSET
76#include "SPIRVGenGlobalISel.inc"
77#undef GET_GLOBALISEL_PREDICATE_BITSET
104#define GET_GLOBALISEL_PREDICATES_DECL
105#include "SPIRVGenGlobalISel.inc"
106#undef GET_GLOBALISEL_PREDICATES_DECL
108#define GET_GLOBALISEL_TEMPORARIES_DECL
109#include "SPIRVGenGlobalISel.inc"
110#undef GET_GLOBALISEL_TEMPORARIES_DECL
134 unsigned BitSetOpcode)
const;
138 unsigned BitSetOpcode)
const;
142 unsigned BitSetOpcode,
bool SwapPrimarySide)
const;
149 unsigned Opcode)
const;
152 unsigned Opcode)
const;
173 unsigned NewOpcode,
unsigned NegateOpcode = 0)
const;
182 Register castPtrToPtrToInt(
Register Ptr, SPIRV::StorageClass::StorageClass SC,
186 bool selectAtomicPtrValue(
202 unsigned OpType)
const;
270 unsigned Opcode)
const;
274 unsigned Opcode)
const;
278 unsigned Opcode)
const;
282 unsigned Opcode)
const;
284 template <
bool Signed>
287 template <
bool Signed>
294 template <
typename PickOpcodeFn>
297 PickOpcodeFn &&PickOpcode)
const;
314 template <
typename PickOpcodeFn>
317 PickOpcodeFn &&PickOpcode)
const;
342 bool IsSigned)
const;
344 bool IsSigned,
unsigned Opcode)
const;
346 bool IsSigned)
const;
352 bool IsSigned)
const;
393 GL::GLSLExtInst GLInst,
bool setMIFlags =
true,
394 bool useMISrc =
true,
396 bool selectExtInst(
Register ResVReg, SPIRVTypeInst ResType, MachineInstr &
I,
397 CL::OpenCLExtInst CLInst,
bool setMIFlags =
true,
398 bool useMISrc =
true,
400 bool selectExtInst(
Register ResVReg, SPIRVTypeInst ResType, MachineInstr &
I,
401 CL::OpenCLExtInst CLInst, GL::GLSLExtInst GLInst,
402 bool setMIFlags =
true,
bool useMISrc =
true,
404 bool selectExtInst(
Register ResVReg, SPIRVTypeInst ResType, MachineInstr &
I,
405 const ExtInstList &ExtInsts,
bool setMIFlags =
true,
406 bool useMISrc =
true,
409 bool selectLog10(
Register ResVReg, SPIRVTypeInst ResType,
410 MachineInstr &
I)
const;
412 bool selectFpowi(
Register ResVReg, SPIRVTypeInst ResType,
413 MachineInstr &
I)
const;
415 bool selectSaturate(
Register ResVReg, SPIRVTypeInst ResType,
416 MachineInstr &
I)
const;
418 bool selectWaveOpInst(
Register ResVReg, SPIRVTypeInst ResType,
419 MachineInstr &
I,
unsigned Opcode)
const;
421 bool selectBarrierInst(MachineInstr &
I,
unsigned Scope,
unsigned MemSem,
422 bool WithGroupSync)
const;
424 bool selectWaveActiveCountBits(
Register ResVReg, SPIRVTypeInst ResType,
425 MachineInstr &
I)
const;
427 bool selectWaveActiveAllEqual(
Register ResVReg, SPIRVTypeInst ResType,
428 MachineInstr &
I)
const;
432 bool selectHandleFromBinding(
Register &ResVReg, SPIRVTypeInst ResType,
433 MachineInstr &
I)
const;
435 bool selectCounterHandleFromBinding(
Register &ResVReg, SPIRVTypeInst ResType,
436 MachineInstr &
I)
const;
438 bool selectReadImageIntrinsic(
Register &ResVReg, SPIRVTypeInst ResType,
439 MachineInstr &
I)
const;
440 bool selectGetDimensionsIntrinsic(
Register &ResVReg, SPIRVTypeInst ResType,
441 MachineInstr &
I)
const;
442 bool selectGetDimensionsLevelsIntrinsic(
Register &ResVReg,
443 SPIRVTypeInst ResType,
444 MachineInstr &
I)
const;
445 bool selectGetDimensionsMSIntrinsic(
Register &ResVReg, SPIRVTypeInst ResType,
446 MachineInstr &
I)
const;
449 std::optional<Register> LodReg = std::nullopt)
const;
450 bool selectSampleBasicIntrinsic(
Register &ResVReg, SPIRVTypeInst ResType,
451 MachineInstr &
I)
const;
452 bool selectCalculateLodIntrinsic(
Register &ResVReg, SPIRVTypeInst ResType,
453 MachineInstr &
I)
const;
454 bool selectSampleBiasIntrinsic(
Register &ResVReg, SPIRVTypeInst ResType,
455 MachineInstr &
I)
const;
456 bool selectSampleGradIntrinsic(
Register &ResVReg, SPIRVTypeInst ResType,
457 MachineInstr &
I)
const;
458 bool selectSampleLevelIntrinsic(
Register &ResVReg, SPIRVTypeInst ResType,
459 MachineInstr &
I)
const;
460 bool selectLoadLevelIntrinsic(
Register &ResVReg, SPIRVTypeInst ResType,
461 MachineInstr &
I)
const;
462 bool selectSampleCmpIntrinsic(
Register &ResVReg, SPIRVTypeInst ResType,
463 MachineInstr &
I)
const;
464 bool selectSampleCmpLevelZeroIntrinsic(
Register &ResVReg,
465 SPIRVTypeInst ResType,
466 MachineInstr &
I)
const;
467 bool selectGatherIntrinsic(
Register &ResVReg, SPIRVTypeInst ResType,
468 MachineInstr &
I)
const;
469 bool selectImageWriteIntrinsic(MachineInstr &
I)
const;
470 bool selectResourceGetPointer(
Register &ResVReg, SPIRVTypeInst ResType,
471 MachineInstr &
I)
const;
472 bool selectPushConstantGetPointer(
Register &ResVReg, SPIRVTypeInst ResType,
473 MachineInstr &
I)
const;
474 bool selectResourceNonUniformIndex(
Register &ResVReg, SPIRVTypeInst ResType,
475 MachineInstr &
I)
const;
476 bool selectModf(
Register ResVReg, SPIRVTypeInst ResType,
477 MachineInstr &
I)
const;
478 bool selectUpdateCounter(
Register &ResVReg, SPIRVTypeInst ResType,
479 MachineInstr &
I)
const;
480 bool selectFrexp(
Register ResVReg, SPIRVTypeInst ResType,
481 MachineInstr &
I)
const;
483 bool selectCopySign(
Register ResVReg, SPIRVTypeInst ResType,
484 MachineInstr &
I)
const;
486 bool selectLdexp(
Register ResVReg, SPIRVTypeInst ResType,
487 MachineInstr &
I)
const;
488 bool selectSincos(
Register ResVReg, SPIRVTypeInst ResType,
489 MachineInstr &
I)
const;
490 bool selectExp10(
Register ResVReg, SPIRVTypeInst ResType,
491 MachineInstr &
I)
const;
492 bool selectDerivativeInst(
Register ResVReg, SPIRVTypeInst ResType,
493 MachineInstr &
I,
const unsigned DPdOpCode)
const;
495 Register buildI32ConstantInEntryBlock(uint32_t Val, MachineInstr &
I,
496 SPIRVTypeInst ResType =
nullptr)
const;
498 Register buildZerosVal(SPIRVTypeInst ResType, MachineInstr &
I)
const;
499 bool isScalarOrVectorIntConstantZero(
Register Reg)
const;
500 Register buildZerosValF(SPIRVTypeInst ResType, MachineInstr &
I)
const;
502 MachineInstr &
I)
const;
503 Register buildOnesValF(SPIRVTypeInst ResType, MachineInstr &
I)
const;
505 MachineInstr &
I)
const;
507 bool wrapIntoSpecConstantOp(MachineInstr &
I,
510 SPIRVTypeInst getUcharPtrType(MachineInstr &
I,
511 SPIRV::StorageClass::StorageClass SC)
const;
512 Register getUcharPtrTypeReg(MachineInstr &
I,
513 SPIRV::StorageClass::StorageClass SC)
const;
514 MachineInstrBuilder buildSpecConstantOp(MachineInstr &
I,
Register Dest,
516 uint32_t Opcode)
const;
518 MachineInstrBuilder buildConstGenericPtr(MachineInstr &
I,
Register SrcPtr,
519 SPIRVTypeInst SrcPtrTy)
const;
520 Register buildPointerToResource(SPIRVTypeInst ResType,
521 SPIRV::StorageClass::StorageClass SC,
522 uint32_t Set, uint32_t
Binding,
523 uint32_t ArraySize,
Register IndexReg,
525 MachineIRBuilder MIRBuilder)
const;
526 SPIRVTypeInst widenTypeToVec4(SPIRVTypeInst
Type, MachineInstr &
I)
const;
527 bool extractSubvector(
Register &ResVReg, SPIRVTypeInst ResType,
528 Register &ReadReg, MachineInstr &InsertionPoint)
const;
529 bool generateImageReadOrFetch(
Register &ResVReg, SPIRVTypeInst ResType,
532 const ImageOperands *ImOps =
nullptr)
const;
533 bool generateSampleImage(
Register ResVReg, SPIRVTypeInst ResType,
535 Register CoordinateReg,
const ImageOperands &ImOps,
538 bool loadVec3BuiltinInputID(SPIRV::BuiltIn::BuiltIn BuiltInValue,
539 Register ResVReg, SPIRVTypeInst ResType,
540 MachineInstr &
I)
const;
541 bool loadBuiltinInputID(SPIRV::BuiltIn::BuiltIn BuiltInValue,
542 Register ResVReg, SPIRVTypeInst ResType,
543 MachineInstr &
I)
const;
544 bool loadHandleBeforePosition(
Register &HandleReg, SPIRVTypeInst ResType,
545 GIntrinsic &HandleDef, MachineInstr &Pos)
const;
546 void decorateUsesAsNonUniform(
Register &NonUniformReg)
const;
547 bool errorIfInstrOutsideShader(MachineInstr &
I)
const;
550 handle64BitOverflow(
Register ResVReg, SPIRVTypeInst ResType, MachineInstr &
I,
551 Register SrcReg,
unsigned int Opcode,
552 std::function<
bool(
Register, SPIRVTypeInst,
553 MachineInstr &,
Register,
unsigned)>
557bool sampledTypeIsSignedInteger(
const llvm::Type *HandleType) {
559 if (
TET->getTargetExtName() ==
"spirv.Image") {
562 assert(
TET->getTargetExtName() ==
"spirv.SignedImage");
563 return TET->getTypeParameter(0)->isIntegerTy();
567#define GET_GLOBALISEL_IMPL
568#include "SPIRVGenGlobalISel.inc"
569#undef GET_GLOBALISEL_IMPL
575 TRI(*ST.getRegisterInfo()), RBI(RBI), GR(*ST.getSPIRVGlobalRegistry()),
578#include
"SPIRVGenGlobalISel.inc"
581#include
"SPIRVGenGlobalISel.inc"
593 InstructionSelector::setupMF(MF, VT, CoverageInfo, PSI, BFI);
598 if (HasVRegsReset == &MF)
615 for (
const auto &
MBB : MF) {
616 for (
const auto &
MI :
MBB) {
619 if (
MI.getOpcode() != SPIRV::ASSIGN_TYPE)
626 if (DstType != SrcType)
631 if (DstRC != SrcRC && SrcRC)
643 while (!Stack.empty()) {
648 switch (
MI->getOpcode()) {
649 case TargetOpcode::G_INTRINSIC:
650 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS:
651 case TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS: {
654 if (IntrID != Intrinsic::spv_const_composite &&
655 IntrID != Intrinsic::spv_undef && IntrID != Intrinsic::spv_poison)
659 case TargetOpcode::G_BUILD_VECTOR:
660 case TargetOpcode::G_SPLAT_VECTOR:
662 i < OpDef->getNumOperands(); i++) {
667 Stack.push_back(OpNestedDef);
670 case TargetOpcode::G_CONSTANT:
671 case TargetOpcode::G_FCONSTANT:
672 case TargetOpcode::G_IMPLICIT_DEF:
673 case SPIRV::OpConstantTrue:
674 case SPIRV::OpConstantFalse:
675 case SPIRV::OpConstantI:
676 case SPIRV::OpConstantF:
677 case SPIRV::OpConstantComposite:
678 case SPIRV::OpConstantCompositeContinuedINTEL:
679 case SPIRV::OpConstantSampler:
680 case SPIRV::OpConstantNull:
682 case SPIRV::OpPoisonKHR:
683 case SPIRV::OpConstantFunctionPointerINTEL:
710 case Intrinsic::spv_all:
711 case Intrinsic::spv_alloca:
712 case Intrinsic::spv_any:
713 case Intrinsic::spv_bitcast:
714 case Intrinsic::spv_const_composite:
715 case Intrinsic::spv_degrees:
716 case Intrinsic::spv_distance:
717 case Intrinsic::spv_extractelt:
718 case Intrinsic::spv_extractv:
719 case Intrinsic::spv_faceforward:
720 case Intrinsic::spv_fdot:
721 case Intrinsic::spv_firstbitlow:
722 case Intrinsic::spv_firstbitshigh:
723 case Intrinsic::spv_firstbituhigh:
724 case Intrinsic::spv_frac:
725 case Intrinsic::spv_gep:
726 case Intrinsic::spv_global_offset:
727 case Intrinsic::spv_global_size:
728 case Intrinsic::spv_group_id:
729 case Intrinsic::spv_insertelt:
730 case Intrinsic::spv_insertv:
731 case Intrinsic::spv_isinf:
732 case Intrinsic::spv_isnan:
733 case Intrinsic::spv_isfinite:
734 case Intrinsic::spv_isnormal:
735 case Intrinsic::spv_lerp:
736 case Intrinsic::spv_length:
737 case Intrinsic::spv_normalize:
738 case Intrinsic::spv_num_subgroups:
739 case Intrinsic::spv_num_workgroups:
740 case Intrinsic::spv_ptrcast:
741 case Intrinsic::spv_radians:
742 case Intrinsic::spv_reflect:
743 case Intrinsic::spv_refract:
744 case Intrinsic::spv_resource_getbasepointer:
745 case Intrinsic::spv_resource_getpointer:
746 case Intrinsic::spv_resource_handlefrombinding:
747 case Intrinsic::spv_resource_handlefromimplicitbinding:
748 case Intrinsic::spv_resource_nonuniformindex:
749 case Intrinsic::spv_resource_sample:
750 case Intrinsic::spv_rsqrt:
751 case Intrinsic::spv_saturate:
752 case Intrinsic::spv_sdot:
753 case Intrinsic::spv_sign:
754 case Intrinsic::spv_smoothstep:
755 case Intrinsic::spv_subgroup_id:
756 case Intrinsic::spv_subgroup_local_invocation_id:
757 case Intrinsic::spv_subgroup_max_size:
758 case Intrinsic::spv_subgroup_size:
759 case Intrinsic::spv_thread_id:
760 case Intrinsic::spv_thread_id_in_group:
761 case Intrinsic::spv_udot:
762 case Intrinsic::spv_undef:
763 case Intrinsic::spv_value_md:
764 case Intrinsic::spv_workgroup_size:
776 case SPIRV::OpTypeVoid:
777 case SPIRV::OpTypeBool:
778 case SPIRV::OpTypeInt:
779 case SPIRV::OpTypeFloat:
780 case SPIRV::OpTypeVector:
781 case SPIRV::OpTypeVectorIdEXT:
782 case SPIRV::OpTypeMatrix:
783 case SPIRV::OpTypeImage:
784 case SPIRV::OpTypeSampler:
785 case SPIRV::OpTypeSampledImage:
786 case SPIRV::OpTypeArray:
787 case SPIRV::OpTypeRuntimeArray:
788 case SPIRV::OpTypeStruct:
789 case SPIRV::OpTypeOpaque:
790 case SPIRV::OpTypePointer:
791 case SPIRV::OpTypeFunction:
792 case SPIRV::OpTypeEvent:
793 case SPIRV::OpTypeDeviceEvent:
794 case SPIRV::OpTypeReserveId:
795 case SPIRV::OpTypeQueue:
796 case SPIRV::OpTypePipe:
797 case SPIRV::OpTypeForwardPointer:
798 case SPIRV::OpTypePipeStorage:
799 case SPIRV::OpTypeNamedBarrier:
800 case SPIRV::OpTypeAccelerationStructureNV:
801 case SPIRV::OpTypeCooperativeMatrixNV:
802 case SPIRV::OpTypeCooperativeMatrixKHR:
812 if (
MI.getNumDefs() == 0)
815 for (
const auto &MO :
MI.all_defs()) {
817 if (
Reg.isPhysical()) {
821 for (
const auto &
UseMI :
MRI.use_nodbg_instructions(
Reg)) {
822 if (
UseMI.getOpcode() != SPIRV::OpName) {
829 if (
MI.getOpcode() == TargetOpcode::LOCAL_ESCAPE ||
MI.isFakeUse() ||
830 MI.isLifetimeMarker()) {
833 <<
"Not dead: Opcode is LOCAL_ESCAPE, fake use, or lifetime marker.\n");
844 if (
MI.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS ||
845 MI.getOpcode() == TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS) {
848 LLVM_DEBUG(
dbgs() <<
"Dead: Intrinsic with no real side effects.\n");
853 if (
MI.mayStore() ||
MI.isCall() ||
854 (
MI.mayLoad() &&
MI.hasOrderedMemoryRef()) ||
MI.isPosition() ||
855 MI.isDebugInstr() ||
MI.isTerminator() ||
MI.isJumpTableDebugInfo()) {
856 LLVM_DEBUG(
dbgs() <<
"Not dead: instruction has side effects.\n");
867 LLVM_DEBUG(
dbgs() <<
"Dead: known opcode with no side effects\n");
874void SPIRVInstructionSelector::removeOpNamesForDeadMI(MachineInstr &
MI)
const {
876 for (
const auto &MO :
MI.all_defs()) {
880 SmallVector<MachineInstr *, 4> UselessOpNames;
883 "There is still a use of the dead function.");
886 for (MachineInstr *OpNameMI : UselessOpNames) {
888 OpNameMI->eraseFromParent();
893void SPIRVInstructionSelector::removeDeadInstruction(MachineInstr &
MI)
const {
896 removeOpNamesForDeadMI(
MI);
897 MI.eraseFromParent();
900bool SPIRVInstructionSelector::select(MachineInstr &
I) {
901 resetVRegsType(*
I.getParent()->getParent());
903 assert(
I.getParent() &&
"Instruction should be in a basic block!");
904 assert(
I.getParent()->getParent() &&
"Instruction should be in a function!");
909 removeDeadInstruction(
I);
916 if (Opcode == SPIRV::ASSIGN_TYPE) {
917 Register DstReg =
I.getOperand(0).getReg();
918 Register SrcReg =
I.getOperand(1).getReg();
921 Def->getOpcode() != TargetOpcode::G_CONSTANT &&
922 Def->getOpcode() != TargetOpcode::G_FCONSTANT) {
923 if (
Def->getOpcode() == TargetOpcode::G_SELECT) {
924 Register SelectDstReg =
Def->getOperand(0).getReg();
925 bool SuccessToSelectSelect [[maybe_unused]] = selectSelect(
927 assert(SuccessToSelectSelect);
929 Def->eraseFromParent();
936 bool Res = selectImpl(
I, *CoverageInfo);
938 if (!Res &&
Def->getOpcode() != TargetOpcode::G_CONSTANT) {
939 dbgs() <<
"Unexpected pattern in ASSIGN_TYPE.\nInstruction: ";
943 assert(Res ||
Def->getOpcode() == TargetOpcode::G_CONSTANT);
955 }
else if (
I.getNumDefs() == 1) {
967 removeDeadInstruction(
I);
972 if (
I.getNumOperands() !=
I.getNumExplicitOperands()) {
973 LLVM_DEBUG(
errs() <<
"Generic instr has unexpected implicit operands\n");
979 bool HasDefs =
I.getNumDefs() > 0;
982 assert(!HasDefs || ResType ||
I.getOpcode() == TargetOpcode::G_GLOBAL_VALUE ||
983 I.getOpcode() == TargetOpcode::G_IMPLICIT_DEF);
984 if (spvSelect(ResVReg, ResType,
I)) {
986 for (
unsigned i = 0; i <
I.getNumDefs(); ++i)
997 case TargetOpcode::G_CONSTANT:
998 case TargetOpcode::G_FCONSTANT:
1005 MachineInstr &
I)
const {
1008 if (DstRC != SrcRC && SrcRC)
1010 BuildMI(*
I.getParent(),
I,
I.getDebugLoc(),
TII.get(TargetOpcode::COPY))
1017bool SPIRVInstructionSelector::spvSelect(
Register ResVReg,
1018 SPIRVTypeInst ResType,
1019 MachineInstr &
I)
const {
1020 const unsigned Opcode =
I.getOpcode();
1022 return selectImpl(
I, *CoverageInfo);
1024 case TargetOpcode::G_CONSTANT:
1025 case TargetOpcode::G_FCONSTANT:
1026 return selectConst(ResVReg, ResType,
I);
1027 case TargetOpcode::G_GLOBAL_VALUE:
1028 return selectGlobalValue(ResVReg,
I);
1029 case TargetOpcode::G_IMPLICIT_DEF:
1030 return selectOpUndef(ResVReg, ResType,
I);
1031 case TargetOpcode::G_FREEZE:
1032 return selectFreeze(ResVReg, ResType,
I);
1034 case TargetOpcode::G_INTRINSIC:
1035 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS:
1036 case TargetOpcode::G_INTRINSIC_CONVERGENT:
1037 case TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS:
1038 return selectIntrinsic(ResVReg, ResType,
I);
1039 case TargetOpcode::G_BITREVERSE:
1040 return selectBitreverse(ResVReg, ResType,
I);
1042 case TargetOpcode::G_BUILD_VECTOR:
1043 return selectBuildVector(ResVReg, ResType,
I);
1044 case TargetOpcode::G_SPLAT_VECTOR:
1045 return selectSplatVector(ResVReg, ResType,
I);
1046 case TargetOpcode::G_CONCAT_VECTORS:
1047 return selectConcatVectors(ResVReg, ResType,
I);
1049 case TargetOpcode::G_SHUFFLE_VECTOR: {
1050 MachineBasicBlock &BB = *
I.getParent();
1051 auto MIB =
BuildMI(BB,
I,
I.getDebugLoc(),
TII.get(SPIRV::OpVectorShuffle))
1054 .
addUse(
I.getOperand(1).getReg())
1055 .
addUse(
I.getOperand(2).getReg());
1056 for (
auto V :
I.getOperand(3).getShuffleMask())
1061 case TargetOpcode::G_MEMMOVE:
1062 case TargetOpcode::G_MEMCPY:
1063 case TargetOpcode::G_MEMCPY_INLINE:
1064 case TargetOpcode::G_MEMSET:
1065 case TargetOpcode::G_MEMSET_INLINE:
1066 return selectMemOperation(ResVReg,
I);
1068 case TargetOpcode::G_ICMP:
1069 return selectICmp(ResVReg, ResType,
I);
1070 case TargetOpcode::G_FCMP:
1071 return selectFCmp(ResVReg, ResType,
I);
1073 case TargetOpcode::G_FRAME_INDEX:
1074 return selectFrameIndex(ResVReg, ResType,
I);
1076 case TargetOpcode::G_LOAD:
1077 return selectLoad(ResVReg, ResType,
I);
1078 case TargetOpcode::G_STORE:
1079 return selectStore(
I);
1081 case TargetOpcode::G_BR:
1082 return selectBranch(
I);
1083 case TargetOpcode::G_BRCOND:
1084 return selectBranchCond(
I);
1086 case TargetOpcode::G_PHI:
1087 return selectPhi(ResVReg,
I);
1089 case TargetOpcode::G_FPTOSI:
1090 return selectUnOp(ResVReg, ResType,
I, SPIRV::OpConvertFToS);
1091 case TargetOpcode::G_FPTOUI:
1092 return selectUnOp(ResVReg, ResType,
I, SPIRV::OpConvertFToU);
1094 case TargetOpcode::G_FPTOSI_SAT:
1095 return selectUnOp(ResVReg, ResType,
I, SPIRV::OpConvertFToS);
1096 case TargetOpcode::G_FPTOUI_SAT:
1097 return selectUnOp(ResVReg, ResType,
I, SPIRV::OpConvertFToU);
1099 case TargetOpcode::G_SITOFP:
1100 return selectIToF(ResVReg, ResType,
I,
true, SPIRV::OpConvertSToF);
1101 case TargetOpcode::G_UITOFP:
1102 return selectIToF(ResVReg, ResType,
I,
false, SPIRV::OpConvertUToF);
1104 case TargetOpcode::G_CTPOP:
1105 return selectPopCount(ResVReg, ResType,
I, SPIRV::OpBitCount);
1106 case TargetOpcode::G_SMIN:
1107 return selectExtInst(ResVReg, ResType,
I, CL::s_min, GL::SMin);
1108 case TargetOpcode::G_UMIN:
1109 return selectExtInst(ResVReg, ResType,
I, CL::u_min, GL::UMin);
1111 case TargetOpcode::G_SMAX:
1112 return selectExtInst(ResVReg, ResType,
I, CL::s_max, GL::SMax);
1113 case TargetOpcode::G_UMAX:
1114 return selectExtInst(ResVReg, ResType,
I, CL::u_max, GL::UMax);
1116 case TargetOpcode::G_SCMP:
1117 return selectSUCmp(ResVReg, ResType,
I,
true);
1118 case TargetOpcode::G_UCMP:
1119 return selectSUCmp(ResVReg, ResType,
I,
false);
1120 case TargetOpcode::G_LROUND:
1121 case TargetOpcode::G_LLROUND: {
1126 regForLround, *(
I.getParent()->getParent()));
1128 CL::round, GL::Round,
false);
1130 auto MIB =
BuildMI(BB,
I,
I.getDebugLoc(),
TII.get(SPIRV::OpConvertFToS))
1137 case TargetOpcode::G_STRICT_FMA:
1138 case TargetOpcode::G_FMA: {
1141 auto MIB =
BuildMI(BB,
I,
I.getDebugLoc(),
TII.get(SPIRV::OpFmaKHR))
1144 .
addUse(
I.getOperand(1).getReg())
1145 .
addUse(
I.getOperand(2).getReg())
1146 .
addUse(
I.getOperand(3).getReg())
1151 return selectExtInst(ResVReg, ResType,
I, CL::fma, GL::Fma);
1154 case TargetOpcode::G_FLDEXP:
1155 case TargetOpcode::G_STRICT_FLDEXP:
1156 return selectLdexp(ResVReg, ResType,
I);
1158 case TargetOpcode::G_FPOW:
1159 return selectExtInst(ResVReg, ResType,
I, CL::pow, GL::Pow);
1160 case TargetOpcode::G_FPOWI:
1161 return selectFpowi(ResVReg, ResType,
I);
1163 case TargetOpcode::G_FEXP:
1164 return selectExtInst(ResVReg, ResType,
I, CL::exp, GL::Exp);
1165 case TargetOpcode::G_FEXP2:
1166 return selectExtInst(ResVReg, ResType,
I, CL::exp2, GL::Exp2);
1167 case TargetOpcode::G_FEXP10:
1168 return selectExp10(ResVReg, ResType,
I);
1170 case TargetOpcode::G_FMODF:
1171 return selectModf(ResVReg, ResType,
I);
1172 case TargetOpcode::G_FSINCOS:
1173 return selectSincos(ResVReg, ResType,
I);
1175 case TargetOpcode::G_FLOG:
1176 return selectExtInst(ResVReg, ResType,
I, CL::log, GL::Log);
1177 case TargetOpcode::G_FLOG2:
1178 return selectExtInst(ResVReg, ResType,
I, CL::log2, GL::Log2);
1179 case TargetOpcode::G_FLOG10:
1180 return selectLog10(ResVReg, ResType,
I);
1182 case TargetOpcode::G_FABS:
1183 return selectExtInst(ResVReg, ResType,
I, CL::fabs, GL::FAbs);
1184 case TargetOpcode::G_ABS:
1185 return selectExtInst(ResVReg, ResType,
I, CL::s_abs, GL::SAbs);
1187 case TargetOpcode::G_FMINNUM:
1188 return selectExtInst(ResVReg, ResType,
I, CL::fmin, GL::NMin);
1189 case TargetOpcode::G_FMAXNUM:
1190 return selectExtInst(ResVReg, ResType,
I, CL::fmax, GL::NMax);
1192 case TargetOpcode::G_FCOPYSIGN:
1193 return selectCopySign(ResVReg, ResType,
I);
1195 case TargetOpcode::G_FCEIL:
1196 return selectExtInst(ResVReg, ResType,
I, CL::ceil, GL::Ceil);
1197 case TargetOpcode::G_FFLOOR:
1198 return selectExtInst(ResVReg, ResType,
I, CL::floor, GL::Floor);
1200 case TargetOpcode::G_FCOS:
1201 return selectExtInst(ResVReg, ResType,
I, CL::cos, GL::Cos);
1202 case TargetOpcode::G_FSIN:
1203 return selectExtInst(ResVReg, ResType,
I, CL::sin, GL::Sin);
1204 case TargetOpcode::G_FTAN:
1205 return selectExtInst(ResVReg, ResType,
I, CL::tan, GL::Tan);
1206 case TargetOpcode::G_FACOS:
1207 return selectExtInst(ResVReg, ResType,
I, CL::acos, GL::Acos);
1208 case TargetOpcode::G_FASIN:
1209 return selectExtInst(ResVReg, ResType,
I, CL::asin, GL::Asin);
1210 case TargetOpcode::G_FATAN:
1211 return selectExtInst(ResVReg, ResType,
I, CL::atan, GL::Atan);
1212 case TargetOpcode::G_FATAN2:
1213 return selectExtInst(ResVReg, ResType,
I, CL::atan2, GL::Atan2);
1214 case TargetOpcode::G_FCOSH:
1215 return selectExtInst(ResVReg, ResType,
I, CL::cosh, GL::Cosh);
1216 case TargetOpcode::G_FSINH:
1217 return selectExtInst(ResVReg, ResType,
I, CL::sinh, GL::Sinh);
1218 case TargetOpcode::G_FTANH:
1219 return selectExtInst(ResVReg, ResType,
I, CL::tanh, GL::Tanh);
1221 case TargetOpcode::G_STRICT_FSQRT:
1222 case TargetOpcode::G_FSQRT:
1223 return selectExtInst(ResVReg, ResType,
I, CL::sqrt, GL::Sqrt);
1225 case TargetOpcode::G_CTTZ:
1226 case TargetOpcode::G_CTTZ_ZERO_POISON:
1227 return selectExtInst(ResVReg, ResType,
I, CL::ctz);
1228 case TargetOpcode::G_CTLZ:
1229 case TargetOpcode::G_CTLZ_ZERO_POISON:
1230 return selectExtInst(ResVReg, ResType,
I, CL::clz);
1232 case TargetOpcode::G_INTRINSIC_ROUND:
1233 return selectExtInst(ResVReg, ResType,
I, CL::round, GL::Round);
1234 case TargetOpcode::G_INTRINSIC_ROUNDEVEN:
1235 return selectExtInst(ResVReg, ResType,
I, CL::rint, GL::RoundEven);
1236 case TargetOpcode::G_INTRINSIC_TRUNC:
1237 return selectExtInst(ResVReg, ResType,
I, CL::trunc, GL::Trunc);
1238 case TargetOpcode::G_FRINT:
1239 case TargetOpcode::G_FNEARBYINT:
1240 return selectExtInst(ResVReg, ResType,
I, CL::rint, GL::RoundEven);
1242 case TargetOpcode::G_SMULH:
1243 return selectExtInst(ResVReg, ResType,
I, CL::s_mul_hi);
1244 case TargetOpcode::G_UMULH:
1245 return selectExtInst(ResVReg, ResType,
I, CL::u_mul_hi);
1247 case TargetOpcode::G_SADDSAT:
1248 return selectExtInst(ResVReg, ResType,
I, CL::s_add_sat);
1249 case TargetOpcode::G_UADDSAT:
1250 return selectExtInst(ResVReg, ResType,
I, CL::u_add_sat);
1251 case TargetOpcode::G_SSUBSAT:
1252 return selectExtInst(ResVReg, ResType,
I, CL::s_sub_sat);
1253 case TargetOpcode::G_USUBSAT:
1254 return selectExtInst(ResVReg, ResType,
I, CL::u_sub_sat);
1256 case TargetOpcode::G_FFREXP:
1257 return selectFrexp(ResVReg, ResType,
I);
1259 case TargetOpcode::G_UADDO:
1260 return selectOverflowArith(ResVReg, ResType,
I,
1262 : SPIRV::OpIAddCarryS);
1263 case TargetOpcode::G_USUBO:
1264 return selectOverflowArith(ResVReg, ResType,
I,
1266 : SPIRV::OpISubBorrowS);
1267 case TargetOpcode::G_UMULO:
1268 return selectOverflowArith(ResVReg, ResType,
I, SPIRV::OpUMulExtended);
1269 case TargetOpcode::G_SMULO:
1270 return selectOverflowArith(ResVReg, ResType,
I, SPIRV::OpSMulExtended);
1272 case TargetOpcode::G_SEXT:
1273 return selectExt(ResVReg, ResType,
I,
true);
1274 case TargetOpcode::G_ANYEXT:
1275 case TargetOpcode::G_ZEXT:
1276 return selectExt(ResVReg, ResType,
I,
false);
1277 case TargetOpcode::G_TRUNC:
1278 return selectTrunc(ResVReg, ResType,
I);
1279 case TargetOpcode::G_FPTRUNC:
1280 case TargetOpcode::G_FPEXT:
1281 return selectUnOp(ResVReg, ResType,
I, SPIRV::OpFConvert);
1283 case TargetOpcode::G_PTRTOINT:
1284 return selectUnOp(ResVReg, ResType,
I, SPIRV::OpConvertPtrToU);
1285 case TargetOpcode::G_INTTOPTR:
1286 return selectUnOp(ResVReg, ResType,
I, SPIRV::OpConvertUToPtr);
1287 case TargetOpcode::G_BITCAST:
1288 return selectBitcast(ResVReg, ResType,
I);
1289 case TargetOpcode::G_ADDRSPACE_CAST:
1290 return selectAddrSpaceCast(ResVReg, ResType,
I);
1291 case TargetOpcode::G_PTRMASK:
1292 return selectPtrMask(ResVReg, ResType,
I);
1293 case TargetOpcode::G_PTR_ADD: {
1296 assert(
I.getOperand(1).isReg() &&
I.getOperand(2).isReg());
1299 const bool IsAddrSpaceCast =
1300 (*II).getOpcode() == TargetOpcode::G_ADDRSPACE_CAST;
1301 assert(((*II).getOpcode() == TargetOpcode::G_GLOBAL_VALUE ||
1302 (*II).getOpcode() == TargetOpcode::COPY || IsAddrSpaceCast ||
1303 (*II).getOpcode() == SPIRV::OpVariable ||
1304 (*II).getOpcode() == SPIRV::OpUntypedVariableKHR) &&
1307 bool IsGVInit =
false;
1311 UseIt != UseEnd; UseIt = std::next(UseIt)) {
1312 if ((*UseIt).getOpcode() == TargetOpcode::G_GLOBAL_VALUE ||
1313 (*UseIt).getOpcode() == SPIRV::OpSpecConstantOp ||
1314 (*UseIt).getOpcode() == SPIRV::OpVariable ||
1315 (*UseIt).getOpcode() == SPIRV::OpUntypedVariableKHR) {
1325 if (!
getImm((*II).getOperand(1),
MRI) ||
1327 return diagnoseUnsupported(
1328 I,
"address space cast of a non-null integer used as a G_PTR_ADD "
1330 SPIRVTypeInst NullPtrTy =
1332 MachineBasicBlock &DepMBB =
I.getMF()->front();
1343 const bool UseUntypedPointers =
1344 ResType->
getOpcode() == SPIRV::OpTypeUntypedPointerKHR;
1345 if (UseUntypedPointers) {
1346 BuildMI(BB,
I,
I.getDebugLoc(),
TII.get(SPIRV::OpSpecConstantOp))
1349 .
addImm(
static_cast<uint32_t
>(
1350 SPIRV::Opcode::UntypedInBoundsPtrAccessChainKHR))
1353 .
addUse(
I.getOperand(2).getReg())
1360 if (GVPointeeType && ResPointeeType && GVPointeeType != ResPointeeType) {
1372 return diagnoseUnsupported(
1373 I,
"incompatible result and operand types in a bitcast");
1375 MachineInstrBuilder MIB =
1376 BuildMI(BB,
I,
I.getDebugLoc(),
TII.get(SPIRV::OpBitcast))
1383 : SPIRV::OpInBoundsPtrAccessChain))
1387 .
addUse(
I.getOperand(2).getReg())
1390 BuildMI(BB,
I,
I.getDebugLoc(),
TII.get(SPIRV::OpSpecConstantOp))
1394 static_cast<uint32_t
>(SPIRV::Opcode::InBoundsPtrAccessChain))
1396 .
addUse(
I.getOperand(2).getReg())
1405 Register ChainBase = IsAddrSpaceCast ? GV : bitcastConstPtrToI8(
I, GV);
1410 BuildMI(BB,
I,
I.getDebugLoc(),
TII.get(SPIRV::OpSpecConstantOp))
1413 .
addImm(
static_cast<uint32_t
>(SPIRV::Opcode::InBoundsPtrAccessChain))
1415 .
addUse(
I.getOperand(2).getReg())
1420 case TargetOpcode::G_ATOMICRMW_OR:
1421 return selectAtomicRMW(ResVReg, ResType,
I, SPIRV::OpAtomicOr);
1422 case TargetOpcode::G_ATOMICRMW_ADD:
1423 return selectAtomicRMW(ResVReg, ResType,
I, SPIRV::OpAtomicIAdd);
1424 case TargetOpcode::G_ATOMICRMW_AND:
1425 return selectAtomicRMW(ResVReg, ResType,
I, SPIRV::OpAtomicAnd);
1426 case TargetOpcode::G_ATOMICRMW_MAX:
1427 return selectAtomicRMW(ResVReg, ResType,
I, SPIRV::OpAtomicSMax);
1428 case TargetOpcode::G_ATOMICRMW_MIN:
1429 return selectAtomicRMW(ResVReg, ResType,
I, SPIRV::OpAtomicSMin);
1430 case TargetOpcode::G_ATOMICRMW_SUB:
1431 return selectAtomicRMW(ResVReg, ResType,
I, SPIRV::OpAtomicISub);
1432 case TargetOpcode::G_ATOMICRMW_XOR:
1433 return selectAtomicRMW(ResVReg, ResType,
I, SPIRV::OpAtomicXor);
1434 case TargetOpcode::G_ATOMICRMW_UMAX:
1435 return selectAtomicRMW(ResVReg, ResType,
I, SPIRV::OpAtomicUMax);
1436 case TargetOpcode::G_ATOMICRMW_UMIN:
1437 return selectAtomicRMW(ResVReg, ResType,
I, SPIRV::OpAtomicUMin);
1438 case TargetOpcode::G_ATOMICRMW_XCHG:
1439 return selectAtomicRMW(ResVReg, ResType,
I, SPIRV::OpAtomicExchange);
1441 case TargetOpcode::G_ATOMICRMW_FADD:
1442 return selectAtomicRMW(ResVReg, ResType,
I, SPIRV::OpAtomicFAddEXT);
1443 case TargetOpcode::G_ATOMICRMW_FSUB:
1445 return selectAtomicRMW(ResVReg, ResType,
I, SPIRV::OpAtomicFAddEXT,
1447 : SPIRV::OpFNegate);
1448 case TargetOpcode::G_ATOMICRMW_FMIN:
1449 return selectAtomicRMW(ResVReg, ResType,
I, SPIRV::OpAtomicFMinEXT);
1450 case TargetOpcode::G_ATOMICRMW_FMAX:
1451 return selectAtomicRMW(ResVReg, ResType,
I, SPIRV::OpAtomicFMaxEXT);
1453 case TargetOpcode::G_FENCE:
1454 return selectFence(
I);
1456 case TargetOpcode::G_STACKSAVE:
1457 return selectStackSave(ResVReg, ResType,
I);
1458 case TargetOpcode::G_STACKRESTORE:
1459 return selectStackRestore(
I);
1461 case TargetOpcode::G_UNMERGE_VALUES:
1464 case TargetOpcode::G_TRAP:
1465 case TargetOpcode::G_UBSANTRAP:
1466 return selectTrap(
I);
1471 case TargetOpcode::DBG_LABEL:
1473 case TargetOpcode::G_DEBUGTRAP:
1474 return selectDebugTrap(ResVReg, ResType,
I);
1475 case TargetOpcode::G_PREFETCH:
1476 return selectPrefetch(
I);
1483bool SPIRVInstructionSelector::selectDebugTrap(
Register ResVReg,
1484 SPIRVTypeInst ResType,
1485 MachineInstr &
I)
const {
1486 if (!STI.
canUseExtInstSet(SPIRV::InstructionSet::NonSemantic_DebugBreak)) {
1492 MachineIRBuilder MIRBuilder(
I);
1495 return selectExtInst(ExtReg, VoidTy,
I,
1496 {{SPIRV::InstructionSet::NonSemantic_DebugBreak,
1497 SPIRV::NonSemanticDebugBreakExtInst::DebugBreak}},
1501bool SPIRVInstructionSelector::selectPrefetch(MachineInstr &
I)
const {
1510 MachineIRBuilder MIRBuilder(
I);
1512 const SPIRVTypeInst PointerSizeType =
1520 Register AddrVal =
I.getOperand(0).getReg();
1523 return selectExtInst(ExtReg, GR.
getOpTypeVoid(MIRBuilder),
I, CL::prefetch,
1525 {AddrVal, ConstIntOne});
1530bool SPIRVInstructionSelector::selectExtInst(
Register ResVReg,
1531 SPIRVTypeInst ResType,
1533 GL::GLSLExtInst GLInst,
1534 bool setMIFlags,
bool useMISrc,
1537 SPIRV::InstructionSet::InstructionSet::GLSL_std_450))
1538 return diagnoseUnsupported(
1540 "this instruction is only supported with the GLSL extended instruction "
1542 return selectExtInst(ResVReg, ResType,
I,
1543 {{SPIRV::InstructionSet::GLSL_std_450, GLInst}},
1544 setMIFlags, useMISrc, SrcRegs);
1547bool SPIRVInstructionSelector::selectExtInst(
Register ResVReg,
1548 SPIRVTypeInst ResType,
1550 CL::OpenCLExtInst CLInst,
1551 bool setMIFlags,
bool useMISrc,
1553 return selectExtInst(ResVReg, ResType,
I,
1554 {{SPIRV::InstructionSet::OpenCL_std, CLInst}},
1555 setMIFlags, useMISrc, SrcRegs);
1558bool SPIRVInstructionSelector::selectExtInst(
1559 Register ResVReg, SPIRVTypeInst ResType, MachineInstr &
I,
1560 CL::OpenCLExtInst CLInst, GL::GLSLExtInst GLInst,
bool setMIFlags,
1562 ExtInstList ExtInsts = {{SPIRV::InstructionSet::OpenCL_std, CLInst},
1563 {SPIRV::InstructionSet::GLSL_std_450, GLInst}};
1564 return selectExtInst(ResVReg, ResType,
I, ExtInsts, setMIFlags, useMISrc,
1568bool SPIRVInstructionSelector::selectExtInst(
Register ResVReg,
1569 SPIRVTypeInst ResType,
1572 bool setMIFlags,
bool useMISrc,
1575 for (
const auto &[InstructionSet, Opcode] : Insts) {
1579 auto MIB =
BuildMI(BB,
I,
I.getDebugLoc(),
TII.get(SPIRV::OpExtInst))
1582 .
addImm(
static_cast<uint32_t
>(InstructionSet))
1587 const unsigned NumOps =
I.getNumOperands();
1590 I.getOperand(Index).getType() ==
1591 MachineOperand::MachineOperandType::MO_IntrinsicID)
1594 MIB.
add(
I.getOperand(Index));
1606bool SPIRVInstructionSelector::selectCopySign(
Register ResVReg,
1607 SPIRVTypeInst ResType,
1608 MachineInstr &
I)
const {
1610 return selectExtInst(ResVReg, ResType,
I, CL::copysign);
1615 Register MagnitudeReg =
I.getOperand(1).getReg();
1616 Register SignReg =
I.getOperand(2).getReg();
1624 unsigned AndOpcode, OrOpcode;
1625 if (ComponentCount > 1) {
1629 AndOpcode = SPIRV::OpBitwiseAndV;
1630 OrOpcode = SPIRV::OpBitwiseOrV;
1634 AndOpcode = SPIRV::OpBitwiseAndS;
1635 OrOpcode = SPIRV::OpBitwiseOrS;
1641 return selectOpWithSrcs(ResReg, IntType,
I, SrcRegs, Opcode);
1644 Register MagnitudeInt, SignInt, MagnitudeBits, SignBits, CombinedInt;
1645 if (!EmitBitOp(MagnitudeInt, {MagnitudeReg}, SPIRV::OpBitcast) ||
1646 !EmitBitOp(SignInt, {SignReg}, SPIRV::OpBitcast) ||
1647 !EmitBitOp(MagnitudeBits, {MagnitudeInt, NotSignMask}, AndOpcode) ||
1648 !EmitBitOp(SignBits, {SignInt, SignMask}, AndOpcode) ||
1649 !EmitBitOp(CombinedInt, {MagnitudeBits, SignBits}, OrOpcode))
1652 return selectOpWithSrcs(ResVReg, ResType,
I, {CombinedInt}, SPIRV::OpBitcast);
1655bool SPIRVInstructionSelector::selectFrexp(
Register ResVReg,
1656 SPIRVTypeInst ResType,
1657 MachineInstr &
I)
const {
1658 ExtInstList ExtInsts = {{SPIRV::InstructionSet::OpenCL_std, CL::frexp},
1659 {SPIRV::InstructionSet::GLSL_std_450, GL::Frexp}};
1660 for (
const auto &Ex : ExtInsts) {
1661 SPIRV::InstructionSet::InstructionSet
Set = Ex.first;
1662 uint32_t Opcode = Ex.second;
1666 MachineIRBuilder MIRBuilder(
I);
1669 PointeeTy, MIRBuilder, SPIRV::StorageClass::Function);
1676 const bool IsUntyped =
1677 PointerType->getOpcode() == SPIRV::OpTypeUntypedPointerKHR;
1679 BuildMI(*It->getParent(), It, It->getDebugLoc(),
1680 TII.get(IsUntyped ? SPIRV::OpUntypedVariableKHR
1681 : SPIRV::OpVariable))
1684 .
addImm(
static_cast<uint32_t
>(SPIRV::StorageClass::Function));
1689 BuildMI(*
I.getParent(),
I,
I.getDebugLoc(),
TII.get(SPIRV::OpExtInst))
1692 .
addImm(
static_cast<uint32_t
>(Ex.first))
1694 .
add(
I.getOperand(2))
1698 Register ExpResReg =
I.getOperand(1).getReg();
1700 BuildMI(*
I.getParent(),
I,
I.getDebugLoc(),
TII.get(SPIRV::OpLoad))
1710bool SPIRVInstructionSelector::selectLdexp(
Register ResVReg,
1711 SPIRVTypeInst ResType,
1712 MachineInstr &
I)
const {
1713 Register XReg =
I.getOperand(1).getReg();
1714 Register ExpReg =
I.getOperand(2).getReg();
1720 if (ResType->
getOpcode() == SPIRV::OpTypeVector &&
1721 ExpType->
getOpcode() != SPIRV::OpTypeVector) {
1723 ExpReg = buildVectorSplat(ExpReg, NumElts,
I);
1726 return selectExtInst(ResVReg, ResType,
I, CL::ldexp, GL::Ldexp,
1727 true,
false, {XReg, ExpReg});
1730bool SPIRVInstructionSelector::selectSincos(
Register ResVReg,
1731 SPIRVTypeInst ResType,
1732 MachineInstr &
I)
const {
1733 Register CosResVReg =
I.getOperand(1).getReg();
1734 unsigned SrcIdx =
I.getNumExplicitDefs();
1739 MachineIRBuilder MIRBuilder(
I);
1741 ResType, MIRBuilder, SPIRV::StorageClass::Function);
1748 const bool IsUntyped =
1749 PointerType->getOpcode() == SPIRV::OpTypeUntypedPointerKHR;
1751 BuildMI(*It->getParent(), It, It->getDebugLoc(),
1752 TII.get(IsUntyped ? SPIRV::OpUntypedVariableKHR
1753 : SPIRV::OpVariable))
1756 .
addImm(
static_cast<uint32_t
>(SPIRV::StorageClass::Function));
1760 BuildMI(*
I.getParent(),
I,
I.getDebugLoc(),
TII.get(SPIRV::OpExtInst))
1763 .
addImm(
static_cast<uint32_t
>(SPIRV::InstructionSet::OpenCL_std))
1765 .
add(
I.getOperand(SrcIdx))
1769 BuildMI(*
I.getParent(),
I,
I.getDebugLoc(),
TII.get(SPIRV::OpLoad))
1777 BuildMI(*
I.getParent(),
I,
I.getDebugLoc(),
TII.get(SPIRV::OpExtInst))
1780 .
addImm(
static_cast<uint32_t
>(SPIRV::InstructionSet::GLSL_std_450))
1782 .
add(
I.getOperand(SrcIdx))
1784 BuildMI(*
I.getParent(),
I,
I.getDebugLoc(),
TII.get(SPIRV::OpExtInst))
1787 .
addImm(
static_cast<uint32_t
>(SPIRV::InstructionSet::GLSL_std_450))
1789 .
add(
I.getOperand(SrcIdx))
1796bool SPIRVInstructionSelector::selectOpWithSrcs(
Register ResVReg,
1797 SPIRVTypeInst ResType,
1800 unsigned Opcode)
const {
1801 auto MIB =
BuildMI(*
I.getParent(),
I,
I.getDebugLoc(),
TII.get(Opcode))
1811bool SPIRVInstructionSelector::selectPopCount16(
Register ResVReg,
1812 SPIRVTypeInst ResType,
1815 unsigned Opcode)
const {
1816 MachineIRBuilder MIRBuilder(
I);
1818 Register OpReg =
I.getOperand(1).getReg();
1824 SPIRVTypeInst ExtType =
1831 if (!selectOpWithSrcs(ExtReg, ExtType,
I, {OpReg}, SPIRV::OpUConvert))
1835 if (!selectPopCount32(PopCountReg, ExtType,
I, ExtReg, Opcode))
1838 return selectOpWithSrcs(ResVReg, ResType,
I, {PopCountReg}, ExtOpcode);
1841bool SPIRVInstructionSelector::selectPopCount32(
Register ResVReg,
1842 SPIRVTypeInst ResType,
1845 unsigned Opcode)
const {
1846 return selectOpWithSrcs(ResVReg, ResType,
I, {SrcReg}, Opcode);
1849bool SPIRVInstructionSelector::selectPopCount64(
Register ResVReg,
1850 SPIRVTypeInst ResType,
1853 unsigned Opcode)
const {
1854 MachineIRBuilder MIRBuilder(
I);
1863 SPIRVTypeInst WorkingType =
1870 if (!selectOpWithSrcs(Trunc, WorkingType,
I, {SrcReg}, SPIRV::OpUConvert))
1874 if (!selectOpWithSrcs(LowCount, WorkingType,
I, {Trunc}, SPIRV::OpBitCount))
1882 IsScalar ? SPIRV::OpShiftRightLogicalS : SPIRV::OpShiftRightLogicalV;
1884 if (!selectOpWithSrcs(Shift, SrcType,
I, {SrcReg, ShiftAmount}, ShiftOp))
1888 if (!selectOpWithSrcs(Trunc, WorkingType,
I, {Shift}, SPIRV::OpUConvert))
1892 if (!selectOpWithSrcs(HighCount, WorkingType,
I, {Trunc}, SPIRV::OpBitCount))
1897 if (!selectOpWithSrcs(Sum, WorkingType,
I, {HighCount, LowCount},
1898 IsScalar ? SPIRV::OpIAddS : SPIRV::OpIAddV))
1902 unsigned ConvOp = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert;
1903 return selectOpWithSrcs(ResVReg, ResType,
I, {Sum}, ConvOp);
1906bool SPIRVInstructionSelector::selectPopCount(
Register ResVReg,
1907 SPIRVTypeInst ResType,
1909 unsigned Opcode)
const {
1914 if (!STI.getTargetTriple().isVulkanOS())
1915 return selectUnOp(ResVReg, ResType,
I, Opcode);
1917 Register OpReg =
I.getOperand(1).getReg();
1920 : SPIRV::OpUConvert;
1924 return selectPopCount16(ResVReg, ResType,
I, ExtOpcode, Opcode);
1926 return selectPopCount32(ResVReg, ResType,
I, OpReg, Opcode);
1928 return selectPopCount64(ResVReg, ResType,
I, OpReg, Opcode);
1930 return diagnoseUnsupported(
I,
"unsupported operand bit width for popcount");
1934bool SPIRVInstructionSelector::selectUnOp(
Register ResVReg,
1935 SPIRVTypeInst ResType,
1937 unsigned Opcode)
const {
1939 Register SrcReg =
I.getOperand(1).getReg();
1944 unsigned DefOpCode = DefIt->getOpcode();
1945 if (DefOpCode == SPIRV::ASSIGN_TYPE || DefOpCode == TargetOpcode::COPY) {
1948 if (
auto *VRD =
getVRegDef(*
MRI, DefIt->getOperand(1).getReg()))
1949 DefOpCode = VRD->getOpcode();
1951 if (DefOpCode == TargetOpcode::G_GLOBAL_VALUE ||
1952 DefOpCode == TargetOpcode::G_CONSTANT ||
1953 DefOpCode == SPIRV::OpVariable ||
1954 DefOpCode == SPIRV::OpUntypedVariableKHR ||
1955 DefOpCode == SPIRV::OpConstantI) {
1961 uint32_t SpecOpcode = 0;
1963 case SPIRV::OpConvertPtrToU:
1964 SpecOpcode =
static_cast<uint32_t
>(SPIRV::Opcode::ConvertPtrToU);
1966 case SPIRV::OpConvertUToPtr:
1967 SpecOpcode =
static_cast<uint32_t
>(SPIRV::Opcode::ConvertUToPtr);
1972 TII.get(SPIRV::OpSpecConstantOp))
1982 return selectOpWithSrcs(ResVReg, ResType,
I, {
I.getOperand(1).getReg()},
1986bool SPIRVInstructionSelector::selectBitcast(
Register ResVReg,
1987 SPIRVTypeInst ResType,
1988 MachineInstr &
I)
const {
1989 Register OpReg =
I.getOperand(1).getReg();
1990 SPIRVTypeInst OpType =
1993 return diagnoseUnsupported(
1994 I,
"incompatible result and operand types in a bitcast");
1995 return selectUnOp(ResVReg, ResType,
I, SPIRV::OpBitcast);
2001 std::optional<Align> AlignOverride = std::nullopt) {
2006 SpvMemOp |=
static_cast<uint32_t>(SPIRV::MemoryOperand::Volatile);
2007 if (
MemOp->isNonTemporal())
2008 SpvMemOp |=
static_cast<uint32_t>(SPIRV::MemoryOperand::Nontemporal);
2010 if (!ST->isShader() &&
MemOp->getAlign().value())
2011 SpvMemOp |=
static_cast<uint32_t>(SPIRV::MemoryOperand::Aligned);
2015 if (ST->canUseExtension(SPIRV::Extension::SPV_INTEL_memory_access_aliasing)) {
2016 if (
auto *MD =
MemOp->getAAInfo().Scope) {
2020 static_cast<uint32_t>(SPIRV::MemoryOperand::AliasScopeINTELMask);
2022 if (
auto *MD =
MemOp->getAAInfo().NoAlias) {
2026 static_cast<uint32_t>(SPIRV::MemoryOperand::NoAliasINTELMask);
2030 if (SpvMemOp !=
static_cast<uint32_t>(SPIRV::MemoryOperand::None)) {
2032 if (SpvMemOp &
static_cast<uint32_t>(SPIRV::MemoryOperand::Aligned))
2033 MIB.
addImm(AlignOverride.value_or(
MemOp->getAlign()).value());
2044 SpvMemOp |=
static_cast<uint32_t>(SPIRV::MemoryOperand::Volatile);
2046 SpvMemOp |=
static_cast<uint32_t>(SPIRV::MemoryOperand::Nontemporal);
2048 if (SpvMemOp !=
static_cast<uint32_t>(SPIRV::MemoryOperand::None))
2052bool SPIRVInstructionSelector::selectLoad(
Register ResVReg,
2053 SPIRVTypeInst ResType,
2054 MachineInstr &
I)
const {
2056 Register Ptr =
I.getOperand(1 + OpOffset).getReg();
2061 (IntPtrDef->getIntrinsicID() == Intrinsic::spv_resource_getbasepointer ||
2062 IntPtrDef->getIntrinsicID() == Intrinsic::spv_resource_getpointer)) {
2064 Register HandleReg = IntPtrDef->getOperand(2).getReg();
2066 if (HandleType->
getOpcode() == SPIRV::OpTypeImage) {
2070 if (!loadHandleBeforePosition(NewHandleReg, HandleType, *HandleDef,
I)) {
2074 Register IdxReg = IntPtrDef->getOperand(3).getReg();
2075 return generateImageReadOrFetch(ResVReg, ResType, NewHandleReg, IdxReg,
2076 I.getDebugLoc(),
I);
2080 MachineIRBuilder MIRBuilder(
I);
2082 if (
I.getNumMemOperands()) {
2083 const MachineMemOperand *MemOp = *
I.memoperands_begin();
2084 if (MemOp->isAtomic())
2085 return selectAtomicLoad(ResVReg, ResType,
I);
2088 auto MIB = MIRBuilder.
buildInstr(SPIRV::OpLoad)
2092 if (!
I.getNumMemOperands()) {
2093 assert(
I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS ||
2095 TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS);
2104Register SPIRVInstructionSelector::createPtrSizedIntReg(
2105 MachineIRBuilder &MIRBuilder)
const {
2106 SPIRVTypeInst IntType =
2116SPIRVInstructionSelector::convertPtrToInt(
Register PtrVal,
2117 MachineIRBuilder &MIRBuilder)
const {
2118 SPIRVTypeInst IntType =
2120 Register IntReg = createPtrSizedIntReg(MIRBuilder);
2121 MIRBuilder.
buildInstr(SPIRV::OpConvertPtrToU)
2129Register SPIRVInstructionSelector::castPtrToPtrToInt(
2130 Register Ptr, SPIRV::StorageClass::StorageClass SC,
2131 MachineIRBuilder &MIRBuilder)
const {
2132 SPIRVTypeInst IntType =
2134 SPIRVTypeInst PtrType =
2148bool SPIRVInstructionSelector::selectAtomicPtrValue(
2149 Register ResVReg, SPIRVTypeInst ResType, MachineIRBuilder &MIRBuilder,
2150 function_ref<
Register(SPIRVTypeInst IntType)> EmitAtomic)
const {
2158 Register IntResult = EmitAtomic(IntType);
2160 MIRBuilder.
buildInstr(SPIRV::OpConvertUToPtr)
2168bool SPIRVInstructionSelector::selectAtomicLoad(
Register ResVReg,
2169 SPIRVTypeInst ResType,
2170 MachineInstr &
I)
const {
2171 LLVMContext &
Context =
I.getMF()->getFunction().getContext();
2174 Register Ptr =
I.getOperand(1 + OpOffset).getReg();
2177 return diagnoseUnsupported(
2178 I,
"Lowering to SPIR-V of atomic load is only "
2179 "allowed for integer, floating point or pointer types");
2181 assert(
I.getNumMemOperands());
2182 const MachineMemOperand &MemOp = **
I.memoperands_begin();
2183 assert(MemOp.isAtomic());
2185 uint32_t
Scope =
static_cast<uint32_t
>(
2187 Register ScopeReg = buildI32ConstantInEntryBlock(Scope,
I);
2193 if (MemOp.isVolatile() && STI.getTargetTriple().isVulkanOS())
2194 MemSem |=
static_cast<uint32_t
>(SPIRV::MemorySemantics::Volatile);
2197 Register MemSemReg = buildI32ConstantInEntryBlock(Sem,
I);
2199 MachineIRBuilder MIRBuilder(
I);
2203 return diagnoseUnsupported(
2204 I,
"Lowering to SPIR-V of atomic load is only "
2205 "allowed for pointer types for physical addressing model");
2210 SPIRV::StorageClass::StorageClass SC =
2212 return selectAtomicPtrValue(
2213 ResVReg, ResType, MIRBuilder, [&](SPIRVTypeInst IntType) {
2214 Register CastedPtr = castPtrToPtrToInt(Ptr, SC, MIRBuilder);
2215 Register IntResult = createPtrSizedIntReg(MIRBuilder);
2226 auto AtomicLoad = MIRBuilder.
buildInstr(SPIRV::OpAtomicLoad)
2237bool SPIRVInstructionSelector::selectStore(MachineInstr &
I)
const {
2239 Register StoreVal =
I.getOperand(0 + OpOffset).getReg();
2240 Register Ptr =
I.getOperand(1 + OpOffset).getReg();
2245 (IntPtrDef->getIntrinsicID() == Intrinsic::spv_resource_getbasepointer ||
2246 IntPtrDef->getIntrinsicID() == Intrinsic::spv_resource_getpointer)) {
2248 Register HandleReg = IntPtrDef->getOperand(2).getReg();
2253 if (!loadHandleBeforePosition(NewHandleReg, HandleType, *HandleDef,
I)) {
2257 Register IdxReg = IntPtrDef->getOperand(3).getReg();
2258 if (HandleType->
getOpcode() == SPIRV::OpTypeImage) {
2259 SPIRVTypeInst SampledType =
2261 SPIRVTypeInst StoreValCompType =
2263 if (StoreValCompType && StoreValCompType != SampledType) {
2266 SPIRVTypeInst PackedType = widenTypeToVec4(SampledType,
I);
2269 BuildMI(*
I.getParent(),
I,
I.getDebugLoc(),
TII.get(SPIRV::OpBitcast))
2274 StoreVal = PackedReg;
2277 auto BMI =
BuildMI(*
I.getParent(),
I,
I.getDebugLoc(),
2278 TII.get(SPIRV::OpImageWrite))
2284 if (sampledTypeIsSignedInteger(LLVMHandleType))
2287 BMI.constrainAllUses(
TII,
TRI, RBI);
2294 if (PointeeTy && PointeeTy->
getOpcode() == SPIRV::OpTypeVectorIdEXT &&
2295 StoreTy->
getOpcode() != SPIRV::OpTypeVectorIdEXT &&
2309 if (
I.getNumMemOperands()) {
2310 const MachineMemOperand *MemOp = *
I.memoperands_begin();
2311 if (MemOp->isAtomic())
2312 return selectAtomicStore(
I);
2319 if (PtrSC == SPIRV::StorageClass::UniformConstant ||
2320 PtrSC == SPIRV::StorageClass::Input ||
2321 PtrSC == SPIRV::StorageClass::PushConstant)
2322 return diagnoseUnsupported(
2323 I,
"store into a read-only SPIR-V storage class is not allowed");
2325 MachineIRBuilder MIRBuilder(
I);
2327 if (!
I.getNumMemOperands()) {
2328 assert(
I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS ||
2330 TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS);
2339bool SPIRVInstructionSelector::selectAtomicStore(MachineInstr &
I)
const {
2340 LLVMContext &
Context =
I.getMF()->getFunction().getContext();
2343 Register StoreVal =
I.getOperand(0 + OpOffset).getReg();
2344 Register Ptr =
I.getOperand(1 + OpOffset).getReg();
2349 if (!PointeeType && PtrType &&
2350 PtrType->
getOpcode() == SPIRV::OpTypeUntypedPointerKHR)
2353 return diagnoseUnsupported(
I,
2354 "Lowering to SPIR-V of atomic store is only "
2355 "allowed for integer or floating point types");
2357 assert(
I.getNumMemOperands());
2358 const MachineMemOperand &MemOp = **
I.memoperands_begin();
2359 assert(MemOp.isAtomic());
2361 uint32_t
Scope =
static_cast<uint32_t
>(
2363 Register ScopeReg = buildI32ConstantInEntryBlock(Scope,
I);
2369 if (MemOp.isVolatile() && STI.getTargetTriple().isVulkanOS())
2370 MemSem |=
static_cast<uint32_t
>(SPIRV::MemorySemantics::Volatile);
2373 Register MemSemReg = buildI32ConstantInEntryBlock(Sem,
I);
2374 MachineIRBuilder MIRBuilder(
I);
2378 return diagnoseUnsupported(
2379 I,
"Lowering to SPIR-V of atomic store is only "
2380 "allowed for pointer types for physical addressing model");
2385 SPIRV::StorageClass::StorageClass SC =
2387 return selectAtomicPtrValue(
2388 Register(), SPIRVTypeInst(), MIRBuilder, [&](SPIRVTypeInst IntType) {
2390 Register CastedPtr = castPtrToPtrToInt(Ptr, SC, MIRBuilder);
2403 return diagnoseUnsupported(
I,
2404 "Lowering to SPIR-V of atomic store is only "
2405 "allowed for integer or floating point types");
2407 auto AtomicStore = MIRBuilder.
buildInstr(SPIRV::OpAtomicStore)
2417bool SPIRVInstructionSelector::selectMaskedGather(
Register ResVReg,
2418 SPIRVTypeInst ResType,
2419 MachineInstr &
I)
const {
2420 assert(
I.getNumExplicitDefs() == 1 &&
"Expected single def for gather");
2428 const Register PtrsReg =
I.getOperand(2).getReg();
2429 const uint32_t
Alignment =
I.getOperand(3).getImm();
2430 const Register MaskReg =
I.getOperand(4).getReg();
2431 const Register PassthruReg =
I.getOperand(5).getReg();
2432 const Register AlignmentReg = buildI32ConstantInEntryBlock(Alignment,
I);
2436 BuildMI(BB,
I,
I.getDebugLoc(),
TII.get(SPIRV::OpMaskedGatherINTEL))
2447bool SPIRVInstructionSelector::selectMaskedScatter(MachineInstr &
I)
const {
2448 assert(
I.getNumExplicitDefs() == 0 &&
"Expected no defs for scatter");
2455 const Register ValuesReg =
I.getOperand(1).getReg();
2456 const Register PtrsReg =
I.getOperand(2).getReg();
2457 const uint32_t
Alignment =
I.getOperand(3).getImm();
2458 const Register MaskReg =
I.getOperand(4).getReg();
2459 const Register AlignmentReg = buildI32ConstantInEntryBlock(Alignment,
I);
2463 BuildMI(BB,
I,
I.getDebugLoc(),
TII.get(SPIRV::OpMaskedScatterINTEL))
2472bool SPIRVInstructionSelector::diagnoseUnsupported(
const MachineInstr &
I,
2473 const Twine &
Msg)
const {
2474 const Function &
F =
I.getMF()->getFunction();
2475 F.getContext().diagnose(
2476 DiagnosticInfoUnsupported(
F,
Msg,
I.getDebugLoc(),
DS_Error));
2480bool SPIRVInstructionSelector::selectStackSave(
Register ResVReg,
2481 SPIRVTypeInst ResType,
2482 MachineInstr &
I)
const {
2483 if (!STI.
canUseExtension(SPIRV::Extension::SPV_INTEL_variable_length_array))
2484 return diagnoseUnsupported(
2485 I,
"llvm.stacksave intrinsic: this instruction requires the following "
2486 "SPIR-V extension: SPV_INTEL_variable_length_array");
2488 BuildMI(BB,
I,
I.getDebugLoc(),
TII.get(SPIRV::OpSaveMemoryINTEL))
2495bool SPIRVInstructionSelector::selectStackRestore(MachineInstr &
I)
const {
2496 if (!STI.
canUseExtension(SPIRV::Extension::SPV_INTEL_variable_length_array))
2497 return diagnoseUnsupported(
2499 "llvm.stackrestore intrinsic: this instruction requires the following "
2500 "SPIR-V extension: SPV_INTEL_variable_length_array");
2501 if (!
I.getOperand(0).isReg())
2504 BuildMI(BB,
I,
I.getDebugLoc(),
TII.get(SPIRV::OpRestoreMemoryINTEL))
2505 .
addUse(
I.getOperand(0).getReg())
2511SPIRVInstructionSelector::getOrCreateMemSetGlobal(MachineInstr &
I)
const {
2512 MachineIRBuilder MIRBuilder(
I);
2513 assert(
I.getOperand(1).isReg() &&
I.getOperand(2).isReg());
2520 GlobalVariable *GV =
new GlobalVariable(*CurFunction.
getParent(), LLVMArrTy,
2524 Type *ValTy = Type::getInt8Ty(
I.getMF()->getFunction().getContext());
2525 Type *ArrTy = ArrayType::get(ValTy, Num);
2527 ArrTy, MIRBuilder, SPIRV::StorageClass::UniformConstant);
2530 ArrTy, MIRBuilder, SPIRV::AccessQualifier::None,
false);
2541 const bool IsUntyped = VarTy->
getOpcode() == SPIRV::OpTypeUntypedPointerKHR;
2542 auto MIBVar =
BuildMI(*
I.getParent(),
I,
I.getDebugLoc(),
2543 TII.get(IsUntyped ? SPIRV::OpUntypedVariableKHR
2544 : SPIRV::OpVariable))
2547 .
addImm(SPIRV::StorageClass::UniformConstant);
2560bool SPIRVInstructionSelector::selectCopyMemory(MachineInstr &
I,
2563 Register DstReg =
I.getOperand(0).getReg();
2569 return diagnoseUnsupported(
2570 I,
"OpCopyMemory requires operands to have the same type");
2575 return diagnoseUnsupported(
2576 I,
"Unable to determine pointee type size for OpCopyMemory");
2577 const DataLayout &
DL =
I.getMF()->getFunction().getDataLayout();
2578 if (CopySize !=
DL.getTypeStoreSize(
const_cast<Type *
>(LLVMPointeeTy)))
2579 return diagnoseUnsupported(
2580 I,
"OpCopyMemory requires the size to match the pointee type size");
2583 const unsigned Opcode =
2584 IsLogical ? SPIRV::OpCopyMemory : SPIRV::OpCopyMemorySized;
2585 auto MIB =
BuildMI(BB,
I,
I.getDebugLoc(),
TII.get(Opcode))
2589 MIB.
addUse(
I.getOperand(2).getReg());
2591 if (
I.getNumMemOperands()) {
2592 MachineIRBuilder MIRBuilder(
I);
2593 const MachineMemOperand *DstMemOp = *
I.memoperands_begin();
2595 Align SrcAlign = DstAlign;
2598 if (
I.getNumMemOperands() > 1 && !STI.
isShader())
2599 SrcAlign = (*std::next(
I.memoperands_begin()))->
getAlign();
2609 std::min(DstAlign, SrcAlign));
2616bool SPIRVInstructionSelector::selectMemOperation(
Register ResVReg,
2617 MachineInstr &
I)
const {
2619 Register SizeReg =
I.getOperand(2).getReg();
2621 SizeDef && SizeDef->
getOpcode() == TargetOpcode::G_CONSTANT &&
2625 Register SrcReg =
I.getOperand(1).getReg();
2626 if (
I.getOpcode() == TargetOpcode::G_MEMSET ||
2627 I.getOpcode() == TargetOpcode::G_MEMSET_INLINE) {
2628 Register VarReg = getOrCreateMemSetGlobal(
I);
2631 Type *ValTy = Type::getInt8Ty(
I.getMF()->getFunction().getContext());
2633 ValTy,
I, SPIRV::StorageClass::UniformConstant);
2635 if (!selectOpWithSrcs(SrcReg, SourceTy,
I, {VarReg}, SPIRV::OpBitcast))
2638 if (!selectCopyMemory(
I, SrcReg))
2640 if (ResVReg.
isValid() && ResVReg !=
I.getOperand(0).getReg())
2641 if (!BuildCOPY(ResVReg,
I.getOperand(0).getReg(),
I))
2646bool SPIRVInstructionSelector::selectAtomicRMW(
Register ResVReg,
2647 SPIRVTypeInst ResType,
2650 unsigned NegateOpcode)
const {
2652 const MachineMemOperand *MemOp = *
I.memoperands_begin();
2653 uint32_t
Scope =
static_cast<uint32_t
>(
2655 MemOp->getSyncScopeID()));
2656 Register ScopeReg = buildI32ConstantInEntryBlock(Scope,
I);
2658 Register Ptr =
I.getOperand(1).getReg();
2659 uint32_t ScSem =
static_cast<uint32_t
>(
2663 Register MemSemReg = buildI32ConstantInEntryBlock(
2667 Register ValueReg =
I.getOperand(2).getReg();
2668 if (NegateOpcode != 0) {
2671 if (!selectOpWithSrcs(TmpReg, ResType,
I, {ValueReg}, NegateOpcode))
2677 if (NewOpcode != SPIRV::OpAtomicExchange)
2678 return diagnoseUnsupported(
2679 I,
"Lowering to SPIR-V of this atomic operation is not "
2680 "allowed for pointer types");
2682 return diagnoseUnsupported(
2683 I,
"Lowering to SPIR-V of atomic exchange is only "
2684 "allowed for pointer types for physical addressing model");
2691 MachineIRBuilder MIRBuilder(
I);
2693 return selectAtomicPtrValue(
2694 ResVReg, ResType, MIRBuilder, [&](SPIRVTypeInst IntType) {
2696 Register CastedPtr = castPtrToPtrToInt(Ptr, SC, MIRBuilder);
2697 Register ExchangeResReg = createPtrSizedIntReg(MIRBuilder);
2698 MIRBuilder.
buildInstr(SPIRV::OpAtomicExchange)
2706 return ExchangeResReg;
2710 BuildMI(*
I.getParent(),
I,
I.getDebugLoc(),
TII.get(NewOpcode))
2721bool SPIRVInstructionSelector::selectUnmergeValues(MachineInstr &
I)
const {
2722 unsigned ArgI =
I.getNumOperands() - 1;
2724 I.getOperand(ArgI).isReg() ?
I.getOperand(ArgI).getReg() :
Register(0);
2725 SPIRVTypeInst SrcType =
2729 "cannot select G_UNMERGE_VALUES with a non-vector argument");
2733 unsigned CurrentIndex = 0;
2734 for (
unsigned i = 0; i <
I.getNumDefs(); ++i) {
2735 Register ResVReg =
I.getOperand(i).getReg();
2744 ResType = ScalarType;
2753 BuildMI(BB,
I,
I.getDebugLoc(),
TII.get(SPIRV::OpVectorShuffle))
2759 for (
unsigned j = 0;
j < NumElements; ++
j) {
2760 MIB.
addImm(CurrentIndex + j);
2762 CurrentIndex += NumElements;
2766 BuildMI(BB,
I,
I.getDebugLoc(),
TII.get(SPIRV::OpCompositeExtract))
2778bool SPIRVInstructionSelector::selectFence(MachineInstr &
I)
const {
2783 ? SPIRV::MemorySemantics::UniformMemory |
2784 SPIRV::MemorySemantics::WorkgroupMemory |
2785 SPIRV::MemorySemantics::ImageMemory
2786 : SPIRV::MemorySemantics::WorkgroupMemory |
2787 SPIRV::MemorySemantics::CrossWorkgroupMemory;
2789 STI.getTargetTriple(),
static_cast<uint32_t
>(
getMemSemantics(AO)), ScSem);
2790 Register MemSemReg = buildI32ConstantInEntryBlock(MemSem,
I);
2794 Register ScopeReg = buildI32ConstantInEntryBlock(Scope,
I);
2796 BuildMI(BB,
I,
I.getDebugLoc(),
TII.get(SPIRV::OpMemoryBarrier))
2803bool SPIRVInstructionSelector::selectOverflowArith(
Register ResVReg,
2804 SPIRVTypeInst ResType,
2806 unsigned Opcode)
const {
2807 Type *ResTy =
nullptr;
2810 return diagnoseUnsupported(
2812 "Not enough info to select the arithmetic with overflow instruction");
2814 return diagnoseUnsupported(
I,
2815 "Expect struct type result for the arithmetic "
2816 "with overflow instruction");
2822 MachineIRBuilder MIRBuilder(
I);
2824 ResTy, MIRBuilder, SPIRV::AccessQualifier::ReadWrite,
false);
2825 assert(
I.getNumDefs() > 1 &&
"Not enought operands");
2832 Register ZeroReg = buildZerosVal(ResType,
I);
2837 if (ResName.
size() > 0)
2845 for (
unsigned i =
I.getNumDefs(); i <
I.getNumOperands(); ++i)
2846 MIB.
addUse(
I.getOperand(i).getReg());
2852 for (
unsigned i = 0; i <
I.getNumDefs(); ++i) {
2854 BuildMI(BB,
I,
I.getDebugLoc(),
TII.get(SPIRV::OpCompositeExtract))
2855 .
addDef(i == 1 ? HigherVReg :
I.getOperand(i).getReg())
2862 BuildMI(BB,
I,
I.getDebugLoc(),
TII.get(SPIRV::OpINotEqual))
2863 .
addDef(
I.getOperand(1).getReg())
2871bool SPIRVInstructionSelector::selectAtomicCmpXchg(
Register ResVReg,
2872 SPIRVTypeInst ResType,
2873 MachineInstr &
I)
const {
2875 "selectAtomicCmpXchg only handles the spv_cmpxchg intrinsic");
2876 Register Ptr =
I.getOperand(2).getReg();
2877 Register ScopeReg =
I.getOperand(5).getReg();
2878 Register MemSemEqReg =
I.getOperand(6).getReg();
2879 Register MemSemNeqReg =
I.getOperand(7).getReg();
2881 Register Val =
I.getOperand(4).getReg();
2885 BuildMI(*
I.getParent(),
I,
DL,
TII.get(SPIRV::OpAtomicCompareExchange))
2904 BuildMI(*
I.getParent(),
I,
DL,
TII.get(SPIRV::OpCompositeInsert))
2911 BuildMI(*
I.getParent(),
I,
DL,
TII.get(SPIRV::OpCompositeInsert))
2923 case SPIRV::StorageClass::DeviceOnlyINTEL:
2924 case SPIRV::StorageClass::HostOnlyINTEL:
2934 bool IsGRef =
false;
2935 bool IsAllowedRefs =
2936 llvm::all_of(
MRI->use_instructions(ResVReg), [&IsGRef](
auto const &It) {
2937 unsigned Opcode = It.getOpcode();
2938 if (Opcode == SPIRV::OpConstantComposite ||
2939 Opcode == SPIRV::OpSpecConstantComposite ||
2940 Opcode == SPIRV::OpSpecConstantOp || Opcode == SPIRV::OpVariable ||
2941 Opcode == SPIRV::OpUntypedVariableKHR ||
2942 isSpvIntrinsic(It, Intrinsic::spv_init_global))
2943 return IsGRef = true;
2944 return Opcode == SPIRV::OpName;
2946 return IsAllowedRefs && IsGRef;
2949SPIRVTypeInst SPIRVInstructionSelector::getUcharPtrType(
2950 MachineInstr &
I, SPIRV::StorageClass::StorageClass SC)
const {
2952 Type::getInt8Ty(
I.getMF()->getFunction().getContext()),
I, SC);
2955Register SPIRVInstructionSelector::getUcharPtrTypeReg(
2956 MachineInstr &
I, SPIRV::StorageClass::StorageClass SC)
const {
2961SPIRVInstructionSelector::buildSpecConstantOp(MachineInstr &
I,
Register Dest,
2963 uint32_t Opcode)
const {
2964 return BuildMI(*
I.getParent(),
I,
I.getDebugLoc(),
2965 TII.get(SPIRV::OpSpecConstantOp))
2974Register SPIRVInstructionSelector::bitcastConstPtrToI8(MachineInstr &
I,
2983 MachineInstrBuilder Cast =
2985 static_cast<uint32_t
>(SPIRV::Opcode::Bitcast));
2992SPIRVInstructionSelector::buildConstGenericPtr(MachineInstr &
I,
Register SrcPtr,
2993 SPIRVTypeInst SrcPtrTy)
const {
2994 SPIRVTypeInst GenericPtrTy =
2998 SPIRV::StorageClass::Generic),
3002 MachineInstrBuilder MIB = buildSpecConstantOp(
3004 static_cast<uint32_t
>(SPIRV::Opcode::PtrCastToGeneric));
3014bool SPIRVInstructionSelector::selectAddrSpaceCast(
Register ResVReg,
3015 SPIRVTypeInst ResType,
3016 MachineInstr &
I)
const {
3020 Register SrcPtr =
I.getOperand(1).getReg();
3025 return BuildCOPY(ResVReg, SrcPtr,
I);
3035 unsigned SpecOpcode = [&]() ->
unsigned {
3036 if (SrcSC == SPIRV::StorageClass::CodeSectionINTEL)
3037 return static_cast<uint32_t
>(SPIRV::Opcode::Bitcast);
3039 return static_cast<uint32_t
>(SPIRV::Opcode::PtrCastToGeneric);
3041 return static_cast<uint32_t
>(SPIRV::Opcode::GenericCastToPtr);
3052 if (SrcSC != SPIRV::StorageClass::CodeSectionINTEL &&
3053 (SpecOpcode || ViaGeneric)) {
3054 SrcPtr = bitcastConstPtrToI8(
I, SrcPtr);
3058 buildSpecConstantOp(
I, ResVReg, SrcPtr, getUcharPtrTypeReg(
I, DstSC),
3060 .constrainAllUses(
TII,
TRI, RBI);
3061 }
else if (ViaGeneric) {
3062 MachineInstrBuilder MIB = buildConstGenericPtr(
I, SrcPtr, SrcPtrTy);
3064 buildSpecConstantOp(
3066 static_cast<uint32_t
>(SPIRV::Opcode::GenericCastToPtr))
3067 .constrainAllUses(
TII,
TRI, RBI);
3074 return BuildCOPY(ResVReg, SrcPtr,
I);
3076 if ((SrcSC == SPIRV::StorageClass::Function &&
3077 DstSC == SPIRV::StorageClass::Private) ||
3078 (DstSC == SPIRV::StorageClass::Function &&
3079 SrcSC == SPIRV::StorageClass::Private))
3080 return BuildCOPY(ResVReg, SrcPtr,
I);
3084 return selectUnOp(ResVReg, ResType,
I, SPIRV::OpPtrCastToGeneric);
3087 return selectUnOp(ResVReg, ResType,
I, SPIRV::OpGenericCastToPtr);
3090 SPIRVTypeInst GenericPtrTy =
3109 return selectUnOp(ResVReg, ResType,
I,
3110 SPIRV::OpPtrCastToCrossWorkgroupINTEL);
3112 return selectUnOp(ResVReg, ResType,
I,
3113 SPIRV::OpCrossWorkgroupCastToPtrINTEL);
3115 return selectUnOp(ResVReg, ResType,
I, SPIRV::OpPtrCastToGeneric);
3117 return selectUnOp(ResVReg, ResType,
I, SPIRV::OpGenericCastToPtr);
3127bool SPIRVInstructionSelector::selectPtrMask(
Register ResVReg,
3128 SPIRVTypeInst ResType,
3129 MachineInstr &
I)
const {
3131 return diagnoseUnsupported(
3132 I,
"G_PTRMASK is not supported with logical SPIR-V");
3137 Register PtrReg =
I.getOperand(1).getReg();
3138 Register MaskReg =
I.getOperand(2).getReg();
3157 ? SPIRV::OpBitwiseAndV
3158 : SPIRV::OpBitwiseAndS;
3181 return SPIRV::OpFOrdEqual;
3183 return SPIRV::OpFOrdGreaterThanEqual;
3185 return SPIRV::OpFOrdGreaterThan;
3187 return SPIRV::OpFOrdLessThanEqual;
3189 return SPIRV::OpFOrdLessThan;
3191 return SPIRV::OpFOrdNotEqual;
3193 return SPIRV::OpOrdered;
3195 return SPIRV::OpFUnordEqual;
3197 return SPIRV::OpFUnordGreaterThanEqual;
3199 return SPIRV::OpFUnordGreaterThan;
3201 return SPIRV::OpFUnordLessThanEqual;
3203 return SPIRV::OpFUnordLessThan;
3205 return SPIRV::OpFUnordNotEqual;
3207 return SPIRV::OpUnordered;
3217 return SPIRV::OpIEqual;
3219 return SPIRV::OpINotEqual;
3221 return SPIRV::OpSGreaterThanEqual;
3223 return SPIRV::OpSGreaterThan;
3225 return SPIRV::OpSLessThanEqual;
3227 return SPIRV::OpSLessThan;
3229 return SPIRV::OpUGreaterThanEqual;
3231 return SPIRV::OpUGreaterThan;
3233 return SPIRV::OpULessThanEqual;
3235 return SPIRV::OpULessThan;
3244 return SPIRV::OpPtrEqual;
3246 return SPIRV::OpPtrNotEqual;
3257 return SPIRV::OpLogicalEqual;
3259 return SPIRV::OpLogicalNotEqual;
3297bool SPIRVInstructionSelector::selectAnyOrAll(
Register ResVReg,
3298 SPIRVTypeInst ResType,
3300 unsigned OpAnyOrAll)
const {
3301 assert(
I.getNumOperands() == 3);
3302 assert(
I.getOperand(2).isReg());
3304 Register InputRegister =
I.getOperand(2).getReg();
3307 assert(InputType &&
"VReg has no type assigned");
3311 assert(ResVReg ==
I.getOperand(0).getReg());
3312 return BuildCOPY(ResVReg, InputRegister,
I);
3316 unsigned SpirvNotEqualId =
3317 IsFloatTy ? SPIRV::OpFOrdNotEqual : SPIRV::OpINotEqual;
3319 SPIRVTypeInst SpvBoolTy = SpvBoolScalarTy;
3324 IsBoolTy ? InputRegister
3332 IsFloatTy ? buildZerosValF(InputType,
I) : buildZerosVal(InputType,
I);
3334 BuildMI(BB,
I,
I.getDebugLoc(),
TII.get(SpirvNotEqualId))
3351bool SPIRVInstructionSelector::selectAll(
Register ResVReg,
3352 SPIRVTypeInst ResType,
3353 MachineInstr &
I)
const {
3354 return selectAnyOrAll(ResVReg, ResType,
I, SPIRV::OpAll);
3357bool SPIRVInstructionSelector::selectAny(
Register ResVReg,
3358 SPIRVTypeInst ResType,
3359 MachineInstr &
I)
const {
3360 return selectAnyOrAll(ResVReg, ResType,
I, SPIRV::OpAny);
3364bool SPIRVInstructionSelector::selectFloatDot(
Register ResVReg,
3365 SPIRVTypeInst ResType,
3366 MachineInstr &
I)
const {
3367 assert(
I.getNumOperands() == 4);
3368 assert(
I.getOperand(2).isReg());
3369 assert(
I.getOperand(3).isReg());
3371 [[maybe_unused]] SPIRVTypeInst VecType =
3376 VecType->
getOpcode() == SPIRV::OpTypeVectorIdEXT) &&
3377 "dot product requires either a vector of at least 2 components or"
3378 " the SPV_EXT_long vector extension.");
3380 [[maybe_unused]] SPIRVTypeInst EltType =
3389 .
addUse(
I.getOperand(2).getReg())
3390 .
addUse(
I.getOperand(3).getReg())
3395bool SPIRVInstructionSelector::selectIntegerDot(
Register ResVReg,
3396 SPIRVTypeInst ResType,
3399 assert(
I.getNumOperands() == 4);
3400 assert(
I.getOperand(2).isReg());
3401 assert(
I.getOperand(3).isReg());
3404 auto DotOp =
Signed ? SPIRV::OpSDot : SPIRV::OpUDot;
3408 .
addUse(
I.getOperand(2).getReg())
3409 .
addUse(
I.getOperand(3).getReg())
3416bool SPIRVInstructionSelector::selectIntegerDotExpansion(
3417 Register ResVReg, SPIRVTypeInst ResType, MachineInstr &
I)
const {
3418 assert(
I.getNumOperands() == 4);
3419 assert(
I.getOperand(2).isReg());
3420 assert(
I.getOperand(3).isReg());
3424 Register Vec0 =
I.getOperand(2).getReg();
3425 Register Vec1 =
I.getOperand(3).getReg();
3429 BuildMI(BB,
I,
I.getDebugLoc(),
TII.get(SPIRV::OpIMulV))
3438 VecType->
getOpcode() == SPIRV::OpTypeVectorIdEXT) &&
3439 "dot product requires either a vector of at least 2 components "
3440 "or the SPV_EXT_long_vector extension.");
3443 BuildMI(BB,
I,
I.getDebugLoc(),
TII.get(SPIRV::OpCompositeExtract))
3453 BuildMI(BB,
I,
I.getDebugLoc(),
TII.get(SPIRV::OpCompositeExtract))
3464 BuildMI(BB,
I,
I.getDebugLoc(),
TII.get(SPIRV::OpIAddS))
3476bool SPIRVInstructionSelector::selectOpIsInf(
Register ResVReg,
3477 SPIRVTypeInst ResType,
3478 MachineInstr &
I)
const {
3480 BuildMI(BB,
I,
I.getDebugLoc(),
TII.get(SPIRV::OpIsInf))
3483 .
addUse(
I.getOperand(2).getReg())
3488bool SPIRVInstructionSelector::selectOpIsNan(
Register ResVReg,
3489 SPIRVTypeInst ResType,
3490 MachineInstr &
I)
const {
3492 BuildMI(BB,
I,
I.getDebugLoc(),
TII.get(SPIRV::OpIsNan))
3495 .
addUse(
I.getOperand(2).getReg())
3500bool SPIRVInstructionSelector::selectOpIsFinite(
Register ResVReg,
3501 SPIRVTypeInst ResType,
3502 MachineInstr &
I)
const {
3505 Register Src =
I.getOperand(2).getReg();
3548bool SPIRVInstructionSelector::selectOpIsNormal(
Register ResVReg,
3549 SPIRVTypeInst ResType,
3550 MachineInstr &
I)
const {
3552 BuildMI(BB,
I,
I.getDebugLoc(),
TII.get(SPIRV::OpIsNormal))
3555 .
addUse(
I.getOperand(2).getReg())
3560template <
bool Signed>
3561bool SPIRVInstructionSelector::selectDot4AddPacked(
Register ResVReg,
3562 SPIRVTypeInst ResType,
3563 MachineInstr &
I)
const {
3564 assert(
I.getNumOperands() == 5);
3565 assert(
I.getOperand(2).isReg());
3566 assert(
I.getOperand(3).isReg());
3567 assert(
I.getOperand(4).isReg());
3570 Register Acc =
I.getOperand(2).getReg();
3574 auto DotOp =
Signed ? SPIRV::OpSDot : SPIRV::OpUDot;
3576 auto MIB =
BuildMI(BB,
I,
I.getDebugLoc(),
TII.get(DotOp))
3581 MIB.
addImm(SPIRV::BuiltIn::PackedVectorFormat4x8Bit);
3584 BuildMI(BB,
I,
I.getDebugLoc(),
TII.get(SPIRV::OpIAddS))
3596template <
bool Signed>
3597bool SPIRVInstructionSelector::selectDot4AddPackedExpansion(
3598 Register ResVReg, SPIRVTypeInst ResType, MachineInstr &
I)
const {
3599 assert(
I.getNumOperands() == 5);
3600 assert(
I.getOperand(2).isReg());
3601 assert(
I.getOperand(3).isReg());
3602 assert(
I.getOperand(4).isReg());
3605 Register Acc =
I.getOperand(2).getReg();
3611 Signed ? SPIRV::OpBitFieldSExtract : SPIRV::OpBitFieldUExtract;
3615 for (
unsigned i = 0; i < 4; i++) {
3638 BuildMI(BB,
I,
I.getDebugLoc(),
TII.get(SPIRV::OpIMulS))
3658 BuildMI(BB,
I,
I.getDebugLoc(),
TII.get(SPIRV::OpIAddS))
3673bool SPIRVInstructionSelector::selectSaturate(
Register ResVReg,
3674 SPIRVTypeInst ResType,
3675 MachineInstr &
I)
const {
3676 assert(
I.getNumOperands() == 3);
3677 assert(
I.getOperand(2).isReg());
3679 Register VZero = buildZerosValF(ResType,
I);
3680 Register VOne = buildOnesValF(ResType,
I);
3682 BuildMI(BB,
I,
I.getDebugLoc(),
TII.get(SPIRV::OpExtInst))
3685 .
addImm(
static_cast<uint32_t
>(SPIRV::InstructionSet::GLSL_std_450))
3687 .
addUse(
I.getOperand(2).getReg())
3694bool SPIRVInstructionSelector::selectSign(
Register ResVReg,
3695 SPIRVTypeInst ResType,
3696 MachineInstr &
I)
const {
3697 assert(
I.getNumOperands() == 3);
3698 assert(
I.getOperand(2).isReg());
3700 Register InputRegister =
I.getOperand(2).getReg();
3702 auto &
DL =
I.getDebugLoc();
3705 return diagnoseUnsupported(
I,
"Input Type could not be determined.");
3712 bool NeedsConversion = IsFloatTy || SignBitWidth != ResBitWidth;
3714 auto SignOpcode = IsFloatTy ? GL::FSign : GL::SSign;
3722 .
addImm(
static_cast<uint32_t
>(SPIRV::InstructionSet::GLSL_std_450))
3727 if (NeedsConversion) {
3728 auto ConvertOpcode = IsFloatTy ? SPIRV::OpConvertFToS : SPIRV::OpSConvert;
3739bool SPIRVInstructionSelector::selectWaveOpInst(
Register ResVReg,
3740 SPIRVTypeInst ResType,
3742 unsigned Opcode)
const {
3746 auto BMI =
BuildMI(BB,
I,
I.getDebugLoc(),
TII.get(Opcode))
3752 for (
unsigned J = 2; J <
I.getNumOperands(); J++) {
3753 BMI.addUse(
I.getOperand(J).getReg());
3760bool SPIRVInstructionSelector::selectBarrierInst(MachineInstr &
I,
3763 bool WithGroupSync)
const {
3765 WithGroupSync ? SPIRV::OpControlBarrier : SPIRV::OpMemoryBarrier;
3767 MemSem |= SPIRV::MemorySemantics::AcquireRelease;
3769 assert(((Scope != SPIRV::Scope::Workgroup) ||
3770 ((MemSem & SPIRV::MemorySemantics::WorkgroupMemory) > 0)) &&
3771 "Workgroup Scope must set WorkGroupMemory semantic "
3772 "in Barrier instruction");
3774 assert(((Scope != SPIRV::Scope::Device) ||
3775 ((MemSem & SPIRV::MemorySemantics::UniformMemory) > 0 &&
3776 (MemSem & SPIRV::MemorySemantics::ImageMemory) > 0)) &&
3777 "Device Scope must set UniformMemory and ImageMemory semantic "
3778 "in Barrier instruction");
3784 if (WithGroupSync) {
3785 Register ExecReg = buildI32ConstantInEntryBlock(SPIRV::Scope::Workgroup,
I);
3789 Register ScopeReg = buildI32ConstantInEntryBlock(Scope,
I);
3790 Register MemSemReg = buildI32ConstantInEntryBlock(MemSem,
I);
3792 MI.addUse(ScopeReg).addUse(MemSemReg).constrainAllUses(
TII,
TRI, RBI);
3796bool SPIRVInstructionSelector::selectWaveActiveCountBits(
3797 Register ResVReg, SPIRVTypeInst ResType, MachineInstr &
I)
const {
3802 if (!selectWaveOpInst(BallotReg, BallotType,
I,
3803 SPIRV::OpGroupNonUniformBallot))
3808 TII.get(SPIRV::OpGroupNonUniformBallotBitCount))
3813 .
addImm(SPIRV::GroupOperation::Reduce)
3820bool SPIRVInstructionSelector::selectWaveActiveAllEqual(
Register ResVReg,
3821 SPIRVTypeInst ResType,
3822 MachineInstr &
I)
const {
3827 Register InputReg =
I.getOperand(2).getReg();
3832 bool IsVector = NumElems > 1 ||
3833 (InputType->
getOpcode() == SPIRV::OpTypeVectorIdEXT &&
3847 return selectWaveOpInst(ResVReg, ElemBoolType,
I,
3848 SPIRV::OpGroupNonUniformAllEqual);
3853 ElementResults.
reserve(NumElems);
3855 for (
unsigned Idx = 0;
Idx < NumElems; ++
Idx) {
3868 ElemInput = Extracted;
3874 BuildMI(BB,
I,
DL,
TII.get(SPIRV::OpGroupNonUniformAllEqual))
3885 auto MIB =
BuildMI(BB,
I,
DL,
TII.get(SPIRV::OpCompositeConstruct))
3896bool SPIRVInstructionSelector::selectWavePrefixBitCount(
Register ResVReg,
3897 SPIRVTypeInst ResType,
3898 MachineInstr &
I)
const {
3900 assert(
I.getNumOperands() == 3);
3902 auto Op =
I.getOperand(2);
3912 return diagnoseUnsupported(
I,
"Input Type could not be determined.");
3914 if (InputType->
getOpcode() != SPIRV::OpTypeBool)
3915 return diagnoseUnsupported(
I,
"WavePrefixBitCount requires boolean input");
3936 BuildMI(BB,
I,
DL,
TII.get(SPIRV::OpGroupNonUniformBallotBitCount))
3940 .
addImm(SPIRV::GroupOperation::ExclusiveScan)
3947bool SPIRVInstructionSelector::selectWaveReduceMax(
Register ResVReg,
3948 SPIRVTypeInst ResType,
3950 bool IsUnsigned)
const {
3951 return selectWaveReduce(
3952 ResVReg, ResType,
I, IsUnsigned,
3953 [&](
Register InputRegister,
bool IsUnsigned) {
3954 const bool IsFloatTy =
3956 const auto IntOp = IsUnsigned ? SPIRV::OpGroupNonUniformUMax
3957 : SPIRV::OpGroupNonUniformSMax;
3958 return IsFloatTy ? SPIRV::OpGroupNonUniformFMax : IntOp;
3962bool SPIRVInstructionSelector::selectWaveReduceMin(
Register ResVReg,
3963 SPIRVTypeInst ResType,
3965 bool IsUnsigned)
const {
3966 return selectWaveReduce(
3967 ResVReg, ResType,
I, IsUnsigned,
3968 [&](
Register InputRegister,
bool IsUnsigned) {
3969 const bool IsFloatTy =
3971 const auto IntOp = IsUnsigned ? SPIRV::OpGroupNonUniformUMin
3972 : SPIRV::OpGroupNonUniformSMin;
3973 return IsFloatTy ? SPIRV::OpGroupNonUniformFMin : IntOp;
3977bool SPIRVInstructionSelector::selectWaveReduceSum(
Register ResVReg,
3978 SPIRVTypeInst ResType,
3979 MachineInstr &
I)
const {
3980 return selectWaveReduce(ResVReg, ResType,
I,
false,
3981 [&](
Register InputRegister,
bool IsUnsigned) {
3983 InputRegister, SPIRV::OpTypeFloat);
3984 return IsFloatTy ? SPIRV::OpGroupNonUniformFAdd
3985 : SPIRV::OpGroupNonUniformIAdd;
3989bool SPIRVInstructionSelector::selectWaveReduceProduct(
Register ResVReg,
3990 SPIRVTypeInst ResType,
3991 MachineInstr &
I)
const {
3992 return selectWaveReduce(ResVReg, ResType,
I,
false,
3993 [&](
Register InputRegister,
bool IsUnsigned) {
3995 InputRegister, SPIRV::OpTypeFloat);
3996 return IsFloatTy ? SPIRV::OpGroupNonUniformFMul
3997 : SPIRV::OpGroupNonUniformIMul;
4001template <
typename PickOpcodeFn>
4002bool SPIRVInstructionSelector::selectWaveReduce(
4003 Register ResVReg, SPIRVTypeInst ResType, MachineInstr &
I,
bool IsUnsigned,
4004 PickOpcodeFn &&PickOpcode)
const {
4005 assert(
I.getNumOperands() == 3);
4006 assert(
I.getOperand(2).isReg());
4008 Register InputRegister =
I.getOperand(2).getReg();
4012 return diagnoseUnsupported(
I,
"Input Type could not be determined.");
4015 const unsigned Opcode = PickOpcode(InputRegister, IsUnsigned);
4021 .
addImm(SPIRV::GroupOperation::Reduce)
4022 .
addUse(
I.getOperand(2).getReg())
4027bool SPIRVInstructionSelector::selectWaveReduceOp(
Register ResVReg,
4028 SPIRVTypeInst ResType,
4030 unsigned Opcode)
const {
4031 return selectWaveReduce(
4032 ResVReg, ResType,
I,
false,
4033 [&](
Register InputRegister,
bool IsUnsigned) {
return Opcode; });
4036bool SPIRVInstructionSelector::selectWaveExclusiveScanSum(
4037 Register ResVReg, SPIRVTypeInst ResType, MachineInstr &
I)
const {
4038 return selectWaveExclusiveScan(ResVReg, ResType,
I,
false,
4039 [&](
Register InputRegister,
bool IsUnsigned) {
4041 InputRegister, SPIRV::OpTypeFloat);
4043 ? SPIRV::OpGroupNonUniformFAdd
4044 : SPIRV::OpGroupNonUniformIAdd;
4048bool SPIRVInstructionSelector::selectWaveExclusiveScanProduct(
4049 Register ResVReg, SPIRVTypeInst ResType, MachineInstr &
I)
const {
4050 return selectWaveExclusiveScan(ResVReg, ResType,
I,
false,
4051 [&](
Register InputRegister,
bool IsUnsigned) {
4053 InputRegister, SPIRV::OpTypeFloat);
4055 ? SPIRV::OpGroupNonUniformFMul
4056 : SPIRV::OpGroupNonUniformIMul;
4060template <
typename PickOpcodeFn>
4061bool SPIRVInstructionSelector::selectWaveExclusiveScan(
4062 Register ResVReg, SPIRVTypeInst ResType, MachineInstr &
I,
bool IsUnsigned,
4063 PickOpcodeFn &&PickOpcode)
const {
4064 assert(
I.getNumOperands() == 3);
4065 assert(
I.getOperand(2).isReg());
4067 Register InputRegister =
I.getOperand(2).getReg();
4071 return diagnoseUnsupported(
I,
"Input Type could not be determined.");
4074 const unsigned Opcode = PickOpcode(InputRegister, IsUnsigned);
4080 .
addImm(SPIRV::GroupOperation::ExclusiveScan)
4081 .
addUse(
I.getOperand(2).getReg())
4086bool SPIRVInstructionSelector::selectWaveMultiPrefixOp(
4087 Register ResVReg, SPIRVTypeInst ResType, MachineInstr &
I,
unsigned Opcode,
4088 StringRef BuiltinName)
const {
4090 SPIRV::Extension::SPV_EXT_shader_subgroup_partitioned) &&
4092 SPIRV::Extension::SPV_NV_shader_subgroup_partitioned))
4093 return diagnoseUnsupported(
4094 I, Twine(BuiltinName) +
4095 " requires SPV_EXT_shader_subgroup_partitioned or "
4096 "SPV_NV_shader_subgroup_partitioned");
4098 assert(
I.getNumOperands() == 7);
4103 auto MaskMIB =
BuildMI(*
I.getParent(),
I,
I.getDebugLoc(),
4104 TII.get(SPIRV::OpCompositeConstruct))
4107 for (
unsigned Op = 3;
Op != 7; ++
Op)
4108 MaskMIB.
addUse(
I.getOperand(
Op).getReg());
4111 BuildMI(*
I.getParent(),
I,
I.getDebugLoc(),
TII.get(Opcode))
4116 .
addImm(SPIRV::GroupOperation::PartitionedExclusiveScanNV)
4117 .
addUse(
I.getOperand(2).getReg())
4123bool SPIRVInstructionSelector::selectQuadSwap(
Register ResVReg,
4124 SPIRVTypeInst ResType,
4127 assert(
I.getNumOperands() == 3);
4128 assert(
I.getOperand(2).isReg());
4130 Register InputRegister =
I.getOperand(2).getReg();
4136 BuildMI(BB,
I,
I.getDebugLoc(),
TII.get(SPIRV::OpGroupNonUniformQuadSwap))
4147bool SPIRVInstructionSelector::selectQuadAll(
Register ResVReg,
4148 SPIRVTypeInst ResType,
4149 MachineInstr &
I)
const {
4151 return selectOpWithSrcs(ResVReg, ResType,
I, {
I.getOperand(2).getReg()},
4152 SPIRV::OpGroupNonUniformQuadAllKHR);
4160 if (!selectQuadSwap(QuadValue, ResType,
I,
Direction) ||
4161 !selectOpWithSrcs(
Next, ResType,
I, {
Result, QuadValue},
4162 SPIRV::OpLogicalAnd))
4169bool SPIRVInstructionSelector::selectBitreverseViaI32(
Register ResVReg,
4170 SPIRVTypeInst ResType,
4177 unsigned ShiftOp = SPIRV::OpShiftRightLogicalS;
4182 : SPIRV::OpUConvert;
4184 if (
N > 1 || (ResType->
getOpcode() == SPIRV::OpTypeVectorIdEXT &&
4187 ShiftOp = SPIRV::OpShiftRightLogicalV;
4192 auto MIB =
BuildMI(*
I.getParent(),
I,
I.getDebugLoc(),
4193 TII.get(SPIRV::OpConstantComposite))
4196 for (
unsigned It = 0; It <
N; ++It)
4200 ShiftConst = CompositeReg;
4205 if (!selectOpWithSrcs(ExtReg, Int32Type,
I, {
Op}, ExtendOpcode))
4210 if (!selectBitreverseNative(BitrevReg, Int32Type,
I, ExtReg))
4215 if (!selectOpWithSrcs(ShiftReg, Int32Type,
I, {BitrevReg, ShiftConst},
4220 return selectOpWithSrcs(ResVReg, ResType,
I, {ShiftReg}, ExtendOpcode);
4223bool SPIRVInstructionSelector::handle64BitOverflow(
4225 unsigned int Opcode,
4232 "handle64BitOverflow should only be used for integer types");
4234 assert(ComponentCount < 5 &&
"Vec 5+ will generate invalid SPIR-V ops");
4236 MachineIRBuilder MIRBuilder(
I);
4238 SPIRVTypeInst I64x2Type =
4240 SPIRVTypeInst Vec2ResType =
4243 std::vector<Register> PartialRegs;
4245 unsigned CurrentComponent = 0;
4246 for (; CurrentComponent + 1 < ComponentCount; CurrentComponent += 2) {
4250 auto MIB =
BuildMI(*
I.getParent(),
I,
I.getDebugLoc(),
4251 TII.get(SPIRV::OpVectorShuffle))
4256 .
addImm(CurrentComponent)
4257 .
addImm(CurrentComponent + 1);
4267 PartialRegs.push_back(SubVecReg);
4270 if (CurrentComponent != ComponentCount) {
4276 if (!selectOpWithSrcs(FinalElemReg, I64Type,
I, {SrcReg, ConstIntLastIdx},
4277 SPIRV::OpVectorExtractDynamic))
4286 PartialRegs.push_back(FinalElemResReg);
4290 return selectOpWithSrcs(ResVReg, ResType,
I, PartialRegs,
4291 SPIRV::OpCompositeConstruct);
4294bool SPIRVInstructionSelector::selectBitreverse64(
Register ResVReg,
4295 SPIRVTypeInst ResType,
4299 if (ComponentCount > 2)
4300 return handle64BitOverflow(
4301 ResVReg, ResType,
I, SrcReg, SPIRV::OpBitReverse,
4303 unsigned O) {
return this->selectBitreverse64(R,
T,
I, S); });
4305 MachineIRBuilder MIRBuilder(
I);
4309 I32Type, 2 * ComponentCount, MIRBuilder,
false);
4313 if (!selectOpWithSrcs(Vec32, VecI32Type,
I, {SrcReg}, SPIRV::OpBitcast))
4318 if (!selectBitreverseNative(Reverse32, VecI32Type,
I, Vec32))
4325 auto MIB =
BuildMI(*
I.getParent(),
I,
I.getDebugLoc(),
4326 TII.get(SPIRV::OpVectorShuffle))
4331 for (
unsigned J = 0; J < ComponentCount; ++J) {
4338 return selectOpWithSrcs(ResVReg, ResType,
I, {SwappedVec}, SPIRV::OpBitcast);
4341bool SPIRVInstructionSelector::selectBitreverseNative(
Register ResVReg,
4342 SPIRVTypeInst ResType,
4346 BuildMI(BB,
I,
I.getDebugLoc(),
TII.get(SPIRV::OpBitReverse))
4354bool SPIRVInstructionSelector::selectBitreverse(
Register ResVReg,
4355 SPIRVTypeInst ResType,
4356 MachineInstr &
I)
const {
4357 Register OpReg =
I.getOperand(1).getReg();
4366 return selectBitreverseViaI32(ResVReg, ResType,
I, OpReg);
4368 return selectBitreverseNative(ResVReg, ResType,
I, OpReg);
4370 return selectBitreverse64(ResVReg, ResType,
I, OpReg);
4372 return SPIRVInstructionSelector::diagnoseUnsupported(
4373 I,
"G_BITREVERSE only support 16,32,64 bits.");
4377 return selectBitreverseNative(ResVReg, ResType,
I, OpReg);
4388 unsigned AndOp = SPIRV::OpBitwiseAndS;
4389 unsigned OrOp = SPIRV::OpBitwiseOrS;
4390 unsigned ShlOp = SPIRV::OpShiftLeftLogicalS;
4391 unsigned ShrOp = SPIRV::OpShiftRightLogicalS;
4392 if (
N > 1 || (ResType->
getOpcode() == SPIRV::OpTypeVectorIdEXT &&
4394 AndOp = SPIRV::OpBitwiseAndV;
4395 OrOp = SPIRV::OpBitwiseOrV;
4396 ShlOp = SPIRV::OpShiftLeftLogicalV;
4397 ShrOp = SPIRV::OpShiftRightLogicalV;
4403 const unsigned Shift) ->
Register {
4406 (ResType->
getOpcode() != SPIRV::OpTypeVectorIdEXT ||
4413 Register MaskReg = CreateConst(Mask);
4414 Register ShiftReg = CreateConst(Shift);
4421 if (!selectOpWithSrcs(
T1, ResType,
I, {
Input, ShiftReg}, ShrOp) ||
4422 !selectOpWithSrcs(T2, ResType,
I, {
T1, MaskReg}, AndOp) ||
4423 !selectOpWithSrcs(T3, ResType,
I, {
Input, MaskReg}, AndOp) ||
4424 !selectOpWithSrcs(T4, ResType,
I, {T3, ShiftReg}, ShlOp) ||
4425 !selectOpWithSrcs(Result, ResType,
I, {T2, T4}, OrOp))
4434 while ((Shift >>= 1) > 0) {
4441 return BuildCOPY(ResVReg, Result,
I);
4444bool SPIRVInstructionSelector::selectFreeze(
Register ResVReg,
4445 SPIRVTypeInst ResType,
4446 MachineInstr &
I)
const {
4447 assert(
I.getOperand(0).isReg() &&
I.getOperand(1).isReg() &&
4448 "G_FREEZE must define and use a register");
4449 Register OpReg =
I.getOperand(1).getReg();
4453 BuildMI(*
I.getParent(),
I,
I.getDebugLoc(),
TII.get(SPIRV::OpFreezeKHR))
4467 if (
Def->getOpcode() == TargetOpcode::COPY)
4470 switch (
Def->getOpcode()) {
4471 case SPIRV::ASSIGN_TYPE:
4472 if (MachineInstr *AssignToDef =
4474 if (AssignToDef->getOpcode() == TargetOpcode::G_IMPLICIT_DEF)
4475 Reg =
Def->getOperand(2).getReg();
4478 case SPIRV::OpUndef:
4479 Reg =
Def->getOperand(1).getReg();
4482 unsigned DestOpCode;
4484 DestOpCode = SPIRV::OpConstantNull;
4485 LLVM_DEBUG(
dbgs() <<
"SPV_KHR_poison_freeze is not enabled. freeze of a "
4486 "static undef/poison lowered to OpConstantNull\n");
4488 DestOpCode = TargetOpcode::COPY;
4490 LLVM_DEBUG(
dbgs() <<
"SPV_KHR_poison_freeze is not enabled. freeze "
4491 "skipped, lowered as a copy of the operand\n");
4493 BuildMI(*
I.getParent(),
I,
I.getDebugLoc(),
TII.get(DestOpCode))
4494 .
addDef(
I.getOperand(0).getReg())
4502bool SPIRVInstructionSelector::selectBuildVector(
Register ResVReg,
4503 SPIRVTypeInst ResType,
4504 MachineInstr &
I)
const {
4508 else if (ResType->
getOpcode() == SPIRV::OpTypeArray)
4512 if (
I.getNumExplicitOperands() -
I.getNumExplicitDefs() !=
N)
4517 for (
unsigned i =
I.getNumExplicitDefs();
4518 i <
I.getNumExplicitOperands() && IsConst; ++i)
4523 return diagnoseUnsupported(
4524 I,
"There must be at least two constituent operands in a vector");
4529 for (
unsigned i =
I.getNumExplicitDefs();
4530 i <
I.getNumExplicitOperands() && IsNullVector; ++i) {
4536 BuildMI(*
I.getParent(),
I,
I.getDebugLoc(),
TII.get(SPIRV::OpConstantNull))
4543 auto MIB =
BuildMI(*
I.getParent(),
I,
I.getDebugLoc(),
4544 TII.get(IsConst ? SPIRV::OpConstantComposite
4545 : SPIRV::OpCompositeConstruct))
4548 for (
unsigned i =
I.getNumExplicitDefs(); i <
I.getNumExplicitOperands(); ++i)
4549 MIB.
addUse(
I.getOperand(i).getReg());
4554bool SPIRVInstructionSelector::selectSplatVector(
Register ResVReg,
4555 SPIRVTypeInst ResType,
4556 MachineInstr &
I)
const {
4560 else if (ResType->
getOpcode() == SPIRV::OpTypeArray)
4565 unsigned OpIdx =
I.getNumExplicitDefs();
4566 if (!
I.getOperand(OpIdx).isReg())
4570 Register OpReg =
I.getOperand(OpIdx).getReg();
4574 return diagnoseUnsupported(
4575 I,
"There must be at least two constituent operands in a vector");
4578 auto MIB =
BuildMI(*
I.getParent(),
I,
I.getDebugLoc(),
4579 TII.get(IsConst ? SPIRV::OpConstantComposite
4580 : SPIRV::OpCompositeConstruct))
4583 for (
unsigned i = 0; i <
N; ++i)
4589bool SPIRVInstructionSelector::selectConcatVectors(
Register ResVReg,
4590 SPIRVTypeInst ResType,
4591 MachineInstr &
I)
const {
4597 "Cannot select G_CONCAT_VECTORS with a non-vector result");
4599 auto MIB =
BuildMI(*
I.getParent(),
I,
I.getDebugLoc(),
4600 TII.get(SPIRV::OpCompositeConstruct))
4603 for (
unsigned OpIdx =
I.getNumExplicitDefs();
4604 OpIdx <
I.getNumExplicitOperands(); ++OpIdx)
4605 MIB.
addUse(
I.getOperand(OpIdx).getReg());
4610bool SPIRVInstructionSelector::selectDiscard(
Register ResVReg,
4611 SPIRVTypeInst ResType,
4612 MachineInstr &
I)
const {
4618 SPIRV::Extension::SPV_EXT_demote_to_helper_invocation) ||
4620 Opcode = SPIRV::OpDemoteToHelperInvocation;
4622 Opcode = SPIRV::OpKill;
4627 ToErase.eraseFromParent();
4636bool SPIRVInstructionSelector::selectCmp(
Register ResVReg,
4637 SPIRVTypeInst ResType,
unsigned CmpOpc,
4638 MachineInstr &
I)
const {
4639 Register Cmp0 =
I.getOperand(2).getReg();
4640 Register Cmp1 =
I.getOperand(3).getReg();
4643 "CMP operands should have the same type");
4644 BuildMI(*
I.getParent(),
I,
I.getDebugLoc(),
TII.get(CmpOpc))
4654bool SPIRVInstructionSelector::selectICmp(
Register ResVReg,
4655 SPIRVTypeInst ResType,
4656 MachineInstr &
I)
const {
4660 Register CmpOperand =
I.getOperand(2).getReg();
4662 bool IsPtrCmp = CmpOperandType && CmpOperandType.
isPointer();
4667 Register Op1 =
I.getOperand(3).getReg();
4671 BuildMI(*
I.getParent(),
I,
I.getDebugLoc(),
TII.get(SPIRV::OpBitcast))
4676 I.getOperand(3).setReg(NewOp1);
4682 return selectCmp(ResVReg, ResType, CmpOpc,
I);
4687Register SPIRVInstructionSelector::buildI32ConstantInEntryBlock(
4688 uint32_t Val, MachineInstr &
I, SPIRVTypeInst ResType)
const {
4690 SPIRVTypeInst SpvI32Ty =
4692 auto *ConstInt = ConstantInt::get(LLVMTy, Val);
4697 MachineBasicBlock &EntryBB = *InsertIt->getParent();
4698 MachineInstr *
MI =
nullptr;
4702 MI =
BuildMI(EntryBB, InsertIt, DbgLoc,
TII.get(SPIRV::OpConstantNull))
4706 uint64_t ImmVal = APInt(32, Val).getZExtValue();
4707 MI =
BuildMI(EntryBB, InsertIt, DbgLoc,
TII.get(SPIRV::OpConstantI))
4713 GR.
add(ConstInt,
MI);
4718bool SPIRVInstructionSelector::selectFCmp(
Register ResVReg,
4719 SPIRVTypeInst ResType,
4720 MachineInstr &
I)
const {
4729 for (
unsigned Idx = 0;
Idx < 2; ++
Idx) {
4731 BuildMI(BB,
I,
I.getDebugLoc(),
TII.get(SPIRV::OpIsNan))
4734 .
addUse(
I.getOperand(2 + Idx).getReg())
4741 BuildMI(BB,
I,
I.getDebugLoc(),
TII.get(SPIRV::OpLogicalOr))
4748 BuildMI(BB,
I,
I.getDebugLoc(),
TII.get(SPIRV::OpLogicalNot))
4756 return selectCmp(ResVReg, ResType, CmpOp,
I);
4759bool SPIRVInstructionSelector::selectExp10(
Register ResVReg,
4760 SPIRVTypeInst ResType,
4761 MachineInstr &
I)
const {
4763 return selectExtInst(ResVReg, ResType,
I, CL::exp10);
4773 MachineIRBuilder MIRBuilder(
I);
4780 APFloat ConstVal(3.3219280948873623);
4784 APFloat::rmNearestTiesToEven, &LosesInfo);
4789 isVectorType(ResType) ? SPIRV::OpVectorTimesScalar : SPIRV::OpFMulS;
4791 if (!selectOpWithSrcs(ArgReg, ResType,
I,
4792 {
I.getOperand(1).getReg(), ConstReg}, Opcode))
4794 if (!selectExtInst(ResVReg, ResType,
I,
4795 {{SPIRV::InstructionSet::GLSL_std_450, GL::Exp2}},
false,
4805Register SPIRVInstructionSelector::buildZerosVal(SPIRVTypeInst ResType,
4806 MachineInstr &
I)
const {
4814bool SPIRVInstructionSelector::isScalarOrVectorIntConstantZero(
4820 if (!CompType || CompType->
getOpcode() != SPIRV::OpTypeInt)
4828 if (
Def->getOpcode() == SPIRV::OpConstantNull)
4831 if (
Def->getOpcode() == TargetOpcode::G_CONSTANT ||
4832 Def->getOpcode() == SPIRV::OpConstantI)
4845 if (
Def->getOpcode() == TargetOpcode::G_BUILD_VECTOR ||
4846 (
Def->getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS &&
4848 Intrinsic::spv_const_composite)) {
4849 unsigned StartOp =
Def->getOpcode() == TargetOpcode::G_BUILD_VECTOR ? 1 : 2;
4850 for (
unsigned i = StartOp; i <
Def->getNumOperands(); ++i) {
4851 if (!IsZero(
Def->getOperand(i).getReg()))
4860Register SPIRVInstructionSelector::buildZerosValF(SPIRVTypeInst ResType,
4861 MachineInstr &
I)
const {
4870Register SPIRVInstructionSelector::buildOnesValF(SPIRVTypeInst ResType,
4871 MachineInstr &
I)
const {
4881 SPIRVTypeInst ResType,
4882 MachineInstr &
I)
const {
4893 MachineInstr &
I)
const {
4898 auto MIB =
BuildMI(*
I.getParent(),
I,
I.getDebugLoc(),
4899 TII.get(SPIRV::OpCompositeConstruct))
4902 for (
unsigned J = 0; J < NumElts; ++J)
4908bool SPIRVInstructionSelector::selectSelect(
Register ResVReg,
4909 SPIRVTypeInst ResType,
4910 MachineInstr &
I)
const {
4911 Register SelectFirstArg =
I.getOperand(2).getReg();
4912 Register SelectSecondArg =
I.getOperand(3).getReg();
4920 Register CondReg =
I.getOperand(1).getReg();
4921 bool IsScalarBool = GR.
isScalarOfType(CondReg, SPIRV::OpTypeBool);
4929 CondReg = buildVectorSplat(CondReg, NumElts,
I);
4930 IsScalarBool =
false;
4933 Opcode = IsScalarBool ? SPIRV::OpSelectVFSCond : SPIRV::OpSelectVFVCond;
4934 }
else if (IsPtrTy) {
4935 Opcode = IsScalarBool ? SPIRV::OpSelectVPSCond : SPIRV::OpSelectVPVCond;
4937 Opcode = IsScalarBool ? SPIRV::OpSelectVISCond : SPIRV::OpSelectVIVCond;
4940 assert(IsScalarBool &&
"OpSelect with a scalar result requires a scalar "
4941 "boolean condition");
4943 Opcode = SPIRV::OpSelectSFSCond;
4944 }
else if (IsPtrTy) {
4945 Opcode = SPIRV::OpSelectSPSCond;
4947 Opcode = SPIRV::OpSelectSISCond;
4950 BuildMI(*
I.getParent(),
I,
I.getDebugLoc(),
TII.get(Opcode))
4962bool SPIRVInstructionSelector::selectBoolToInt(
Register ResVReg,
4963 SPIRVTypeInst ResType,
4965 MachineInstr &InsertAt,
4966 bool IsSigned)
const {
4968 Register ZeroReg = buildZerosVal(ResType, InsertAt);
4969 Register OneReg = buildOnesVal(IsSigned, ResType, InsertAt);
4970 bool IsScalarBool = GR.
isScalarOfType(BooleanVReg, SPIRV::OpTypeBool);
4972 IsScalarBool ? SPIRV::OpSelectSISCond : SPIRV::OpSelectVIVCond;
4984bool SPIRVInstructionSelector::selectIToF(
Register ResVReg,
4985 SPIRVTypeInst ResType,
4986 MachineInstr &
I,
bool IsSigned,
4987 unsigned Opcode)
const {
4988 Register SrcReg =
I.getOperand(1).getReg();
4999 selectBoolToInt(SrcReg, TmpType,
I.getOperand(1).getReg(),
I, IsSigned);
5001 return selectOpWithSrcs(ResVReg, ResType,
I, {SrcReg}, Opcode);
5004bool SPIRVInstructionSelector::selectExt(
Register ResVReg,
5005 SPIRVTypeInst ResType, MachineInstr &
I,
5006 bool IsSigned)
const {
5007 Register SrcReg =
I.getOperand(1).getReg();
5009 return selectBoolToInt(ResVReg, ResType,
I.getOperand(1).getReg(),
I,
5013 if (ResType == SrcType)
5014 return BuildCOPY(ResVReg, SrcReg,
I);
5016 unsigned Opcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert;
5017 return selectUnOp(ResVReg, ResType,
I, Opcode);
5020bool SPIRVInstructionSelector::selectSUCmp(
Register ResVReg,
5021 SPIRVTypeInst ResType,
5023 bool IsSigned)
const {
5024 MachineIRBuilder MIRBuilder(
I);
5025 MachineRegisterInfo *
MRI = MIRBuilder.
getMRI();
5030 if (
N > 1 || (ResType->
getOpcode() == SPIRV::OpTypeVectorIdEXT &&
5038 TII.get(IsSigned ? SPIRV::OpSLessThanEqual : SPIRV::OpULessThanEqual))
5041 .
addUse(
I.getOperand(1).getReg())
5042 .
addUse(
I.getOperand(2).getReg())
5047 TII.get(IsSigned ? SPIRV::OpSLessThan : SPIRV::OpULessThan))
5050 .
addUse(
I.getOperand(1).getReg())
5051 .
addUse(
I.getOperand(2).getReg())
5059 unsigned SelectOpcode =
5060 (
N > 1 || (ResType->
getOpcode() == SPIRV::OpTypeVectorIdEXT &&
5062 ? SPIRV::OpSelectVIVCond
5063 : SPIRV::OpSelectSISCond;
5068 .
addUse(buildOnesVal(
true, ResType,
I))
5069 .
addUse(buildZerosVal(ResType,
I))
5076 .
addUse(buildOnesVal(
false, ResType,
I))
5081bool SPIRVInstructionSelector::selectIntToBool(
Register IntReg,
5084 SPIRVTypeInst IntTy,
5085 SPIRVTypeInst BoolTy)
const {
5089 isVectorType(IntTy) ? SPIRV::OpBitwiseAndV : SPIRV::OpBitwiseAndS;
5091 Register One = buildOnesVal(
false, IntTy,
I);
5099 BuildMI(BB,
I,
I.getDebugLoc(),
TII.get(SPIRV::OpINotEqual))
5108bool SPIRVInstructionSelector::selectTrunc(
Register ResVReg,
5109 SPIRVTypeInst ResType,
5110 MachineInstr &
I)
const {
5111 Register IntReg =
I.getOperand(1).getReg();
5114 return selectIntToBool(IntReg, ResVReg,
I, ArgType, ResType);
5115 if (ArgType == ResType)
5116 return BuildCOPY(ResVReg, IntReg,
I);
5118 unsigned Opcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert;
5119 return selectUnOp(ResVReg, ResType,
I, Opcode);
5122bool SPIRVInstructionSelector::selectConst(
Register ResVReg,
5123 SPIRVTypeInst ResType,
5124 MachineInstr &
I)
const {
5125 unsigned Opcode =
I.getOpcode();
5126 unsigned TpOpcode = ResType->
getOpcode();
5128 if (ResType.
isPointer() || TpOpcode == SPIRV::OpTypeEvent) {
5129 assert(Opcode == TargetOpcode::G_CONSTANT &&
5130 I.getOperand(1).getCImm()->isZero());
5131 MachineBasicBlock &DepMBB =
I.getMF()->
front();
5134 }
else if (TpOpcode == SPIRV::OpTypeVectorIdEXT) {
5139 "Expected <1 x T> Vector!");
5140 if (Opcode == TargetOpcode::G_FCONSTANT)
5146 }
else if (Opcode == TargetOpcode::G_FCONSTANT) {
5154 return Reg == ResVReg ?
true : BuildCOPY(ResVReg,
Reg,
I);
5157bool SPIRVInstructionSelector::selectOpUndef(
Register ResVReg,
5158 SPIRVTypeInst ResType,
5159 MachineInstr &
I)
const {
5160 BuildMI(*
I.getParent(),
I,
I.getDebugLoc(),
TII.get(SPIRV::OpUndef))
5167bool SPIRVInstructionSelector::selectInsertVal(
Register ResVReg,
5168 SPIRVTypeInst ResType,
5169 MachineInstr &
I)
const {
5171 auto MIB =
BuildMI(BB,
I,
I.getDebugLoc(),
TII.get(SPIRV::OpCompositeInsert))
5175 .
addUse(
I.getOperand(3).getReg())
5177 .
addUse(
I.getOperand(2).getReg());
5178 for (
unsigned i = 4; i <
I.getNumOperands(); i++)
5184bool SPIRVInstructionSelector::selectExtractVal(
Register ResVReg,
5185 SPIRVTypeInst ResType,
5186 MachineInstr &
I)
const {
5187 Type *MaybeResTy =
nullptr;
5192 "Expected aggregate type for extractv instruction");
5194 SPIRV::AccessQualifier::ReadWrite,
false);
5198 auto MIB =
BuildMI(BB,
I,
I.getDebugLoc(),
TII.get(SPIRV::OpCompositeExtract))
5201 .
addUse(
I.getOperand(2).getReg());
5202 for (
unsigned i = 3; i <
I.getNumOperands(); i++)
5208bool SPIRVInstructionSelector::selectInsertElt(
Register ResVReg,
5209 SPIRVTypeInst ResType,
5210 MachineInstr &
I)
const {
5212 return selectInsertVal(ResVReg, ResType,
I);
5214 BuildMI(BB,
I,
I.getDebugLoc(),
TII.get(SPIRV::OpVectorInsertDynamic))
5217 .
addUse(
I.getOperand(2).getReg())
5218 .
addUse(
I.getOperand(3).getReg())
5219 .
addUse(
I.getOperand(4).getReg())
5224bool SPIRVInstructionSelector::selectExtractElt(
Register ResVReg,
5225 SPIRVTypeInst ResType,
5226 MachineInstr &
I)
const {
5228 return selectExtractVal(ResVReg, ResType,
I);
5230 BuildMI(BB,
I,
I.getDebugLoc(),
TII.get(SPIRV::OpVectorExtractDynamic))
5233 .
addUse(
I.getOperand(2).getReg())
5234 .
addUse(
I.getOperand(3).getReg())
5239bool SPIRVInstructionSelector::selectGEP(
Register ResVReg,
5240 SPIRVTypeInst ResType,
5241 MachineInstr &
I)
const {
5242 const bool IsGEPInBounds =
I.getOperand(2).getImm();
5245 const bool UseUntypedPointers =
5246 ResType->
getOpcode() == SPIRV::OpTypeUntypedPointerKHR;
5251 if (UseUntypedPointers) {
5253 Opcode = IsGEPInBounds ? SPIRV::OpUntypedInBoundsAccessChainKHR
5254 : SPIRV::OpUntypedAccessChainKHR;
5256 Opcode = IsGEPInBounds ? SPIRV::OpUntypedInBoundsPtrAccessChainKHR
5257 : SPIRV::OpUntypedPtrAccessChainKHR;
5266 IsGEPInBounds ? SPIRV::OpInBoundsAccessChain : SPIRV::OpAccessChain;
5268 Opcode = IsGEPInBounds ? SPIRV::OpInBoundsPtrAccessChain
5269 : SPIRV::OpPtrAccessChain;
5274 auto Res =
BuildMI(*
I.getParent(),
I,
I.getDebugLoc(),
TII.get(Opcode))
5279 if (UseUntypedPointers) {
5294 while (Def &&
Def->getOpcode() == TargetOpcode::COPY &&
5295 Def->getOperand(1).isReg())
5297 if (Def &&
Def->getOpcode() == TargetOpcode::G_GLOBAL_VALUE)
5298 if (
const auto *GVar =
5301 SPIRV::AccessQualifier::ReadWrite,
5305 return diagnoseUnsupported(
5306 I,
"could not deduce the base type of an untyped access chain");
5311 Res.addUse(BaseReg);
5313 const bool IsAccessChainOpcode =
5314 (Opcode == SPIRV::OpAccessChain ||
5315 Opcode == SPIRV::OpInBoundsAccessChain ||
5316 Opcode == SPIRV::OpUntypedAccessChainKHR ||
5317 Opcode == SPIRV::OpUntypedInBoundsAccessChainKHR);
5321 "Cannot translate GEP to OpAccessChain.");
5324 const unsigned StartingIndex = IsAccessChainOpcode ? 5 : 4;
5325 for (
unsigned i = StartingIndex; i <
I.getNumExplicitOperands(); ++i)
5326 Res.addUse(
I.getOperand(i).getReg());
5327 Res.constrainAllUses(
TII,
TRI, RBI);
5336 if (Extract.
getOpcode() == SPIRV::OpCompositeExtract) {
5342 BuildMI(*
I.getParent(),
I,
I.getDebugLoc(),
TII.get(SPIRV::OpUndef))
5348 TII.get(SPIRV::OpCompositeInsert))
5362bool SPIRVInstructionSelector::wrapIntoSpecConstantOp(
5364 unsigned Lim =
I.getNumExplicitOperands();
5365 for (
unsigned i =
I.getNumExplicitDefs() + 1; i < Lim; ++i) {
5366 Register OpReg =
I.getOperand(i).getReg();
5370 OpDefine->
getOpcode() == TargetOpcode::G_ADDRSPACE_CAST ||
5371 OpDefine->
getOpcode() == TargetOpcode::G_INTTOPTR ||
5384 SPIRVTypeInst WrapType = OpType;
5385 if (OpType->
getOpcode() == SPIRV::OpTypePointer &&
5387 SPIRV::StorageClass::CodeSectionINTEL) {
5389 SPIRV::StorageClass::Function,
I);
5396 auto MIB =
BuildMI(*
I.getParent(),
I,
I.getDebugLoc(),
5397 TII.get(SPIRV::OpSpecConstantOp))
5400 .
addImm(
static_cast<uint32_t
>(SPIRV::Opcode::Bitcast))
5402 GR.
add(OpDefine, MIB);
5408bool SPIRVInstructionSelector::selectDerivativeInst(
5409 Register ResVReg, SPIRVTypeInst ResType, MachineInstr &
I,
5410 const unsigned DPdOpCode)
const {
5413 if (!errorIfInstrOutsideShader(
I))
5419 Register SrcReg =
I.getOperand(2).getReg();
5424 return BuildMI(*
I.getParent(),
I,
I.getDebugLoc(),
TII.get(DPdOpCode))
5427 .
addUse(
I.getOperand(2).getReg());
5429 MachineIRBuilder MIRBuilder(
I);
5432 if (componentCount != 1)
5440 BuildMI(*
I.getParent(),
I,
I.getDebugLoc(),
TII.get(SPIRV::OpFConvert))
5445 BuildMI(*
I.getParent(),
I,
I.getDebugLoc(),
TII.get(DPdOpCode))
5450 BuildMI(*
I.getParent(),
I,
I.getDebugLoc(),
TII.get(SPIRV::OpFConvert))
5458bool SPIRVInstructionSelector::selectIntrinsic(
Register ResVReg,
5459 SPIRVTypeInst ResType,
5460 MachineInstr &
I)
const {
5464 case Intrinsic::spv_load:
5465 return selectLoad(ResVReg, ResType,
I);
5466 case Intrinsic::spv_atomic_load:
5467 return selectAtomicLoad(ResVReg, ResType,
I);
5468 case Intrinsic::spv_store:
5469 return selectStore(
I);
5470 case Intrinsic::spv_atomic_store:
5471 return selectAtomicStore(
I);
5472 case Intrinsic::spv_extractv:
5473 return selectExtractVal(ResVReg, ResType,
I);
5474 case Intrinsic::spv_insertv:
5475 return selectInsertVal(ResVReg, ResType,
I);
5476 case Intrinsic::spv_extractelt:
5477 return selectExtractElt(ResVReg, ResType,
I);
5478 case Intrinsic::spv_insertelt:
5479 return selectInsertElt(ResVReg, ResType,
I);
5480 case Intrinsic::spv_gep:
5481 return selectGEP(ResVReg, ResType,
I);
5482 case Intrinsic::spv_bitcast: {
5483 Register OpReg =
I.getOperand(2).getReg();
5484 SPIRVTypeInst OpType =
5488 return selectOpWithSrcs(ResVReg, ResType,
I, {OpReg}, SPIRV::OpBitcast);
5490 case Intrinsic::spv_unref_global:
5491 case Intrinsic::spv_init_global: {
5497 Register GVarVReg =
MI->getOperand(0).getReg();
5498 if (!selectGlobalValue(GVarVReg, *
MI, Init))
5503 if (
MI->getOpcode() == TargetOpcode::G_GLOBAL_VALUE) {
5505 MI->eraseFromParent();
5509 case Intrinsic::spv_undef: {
5510 auto MIB =
BuildMI(BB,
I,
I.getDebugLoc(),
TII.get(SPIRV::OpUndef))
5516 case Intrinsic::spv_poison:
5517 BuildMI(BB,
I,
I.getDebugLoc(),
TII.get(SPIRV::OpPoisonKHR))
5522 case Intrinsic::spv_freeze:
5523 BuildMI(BB,
I,
I.getDebugLoc(),
TII.get(SPIRV::OpFreezeKHR))
5526 .
addUse(
I.getOperand(2).getReg())
5529 case Intrinsic::spv_named_boolean_spec_constant: {
5530 auto Opcode =
I.getOperand(3).getImm() ? SPIRV::OpSpecConstantTrue
5531 : SPIRV::OpSpecConstantFalse;
5533 auto MIB =
BuildMI(BB,
I,
I.getDebugLoc(),
TII.get(Opcode))
5534 .
addDef(
I.getOperand(0).getReg())
5537 unsigned SpecId =
I.getOperand(2).getImm();
5539 SPIRV::Decoration::SpecId, {SpecId});
5543 case Intrinsic::spv_const_composite: {
5545 bool IsNull =
I.getNumExplicitDefs() + 1 ==
I.getNumExplicitOperands();
5551 if (!wrapIntoSpecConstantOp(
I, CompositeArgs))
5553 std::function<bool(
Register)> HasSpecConstOperand =
5563 for (
unsigned J =
Def->getNumExplicitDefs() + 1;
5564 J < Def->getNumExplicitOperands(); ++J) {
5565 if (
Def->getOperand(J).isReg() &&
5566 HasSpecConstOperand(
Def->getOperand(J).getReg()))
5572 bool HasSpecConst =
llvm::any_of(CompositeArgs, HasSpecConstOperand);
5573 unsigned CompositeOpc = HasSpecConst ? SPIRV::OpSpecConstantComposite
5574 : SPIRV::OpConstantComposite;
5575 unsigned ContinuedOpc = HasSpecConst
5576 ? SPIRV::OpSpecConstantCompositeContinuedINTEL
5577 : SPIRV::OpConstantCompositeContinuedINTEL;
5578 MachineIRBuilder MIR(
I);
5580 MIR, CompositeOpc, 3, ContinuedOpc, CompositeArgs, ResVReg,
5582 for (
auto *Instr : Instructions) {
5583 Instr->setDebugLoc(
I.getDebugLoc());
5588 auto MIB =
BuildMI(BB,
I,
I.getDebugLoc(),
TII.get(SPIRV::OpConstantNull))
5595 case Intrinsic::spv_assign_name: {
5596 auto MIB =
BuildMI(BB,
I,
I.getDebugLoc(),
TII.get(SPIRV::OpName));
5597 MIB.
addUse(
I.getOperand(
I.getNumExplicitDefs() + 1).getReg());
5598 for (
unsigned i =
I.getNumExplicitDefs() + 2;
5599 i <
I.getNumExplicitOperands(); ++i) {
5600 MIB.
addImm(
I.getOperand(i).getImm());
5605 case Intrinsic::spv_switch: {
5606 auto MIB =
BuildMI(BB,
I,
I.getDebugLoc(),
TII.get(SPIRV::OpSwitch));
5607 for (
unsigned i = 1; i <
I.getNumExplicitOperands(); ++i) {
5608 if (
I.getOperand(i).isReg())
5609 MIB.
addReg(
I.getOperand(i).getReg());
5610 else if (
I.getOperand(i).isCImm())
5611 addNumImm(
I.getOperand(i).getCImm()->getValue(), MIB);
5612 else if (
I.getOperand(i).isMBB())
5613 MIB.
addMBB(
I.getOperand(i).getMBB());
5620 case Intrinsic::spv_loop_merge: {
5621 auto MIB =
BuildMI(BB,
I,
I.getDebugLoc(),
TII.get(SPIRV::OpLoopMerge));
5622 for (
unsigned i = 1; i <
I.getNumExplicitOperands(); ++i) {
5623 if (
I.getOperand(i).isMBB())
5624 MIB.
addMBB(
I.getOperand(i).getMBB());
5631 case Intrinsic::spv_loop_control_intel: {
5633 BuildMI(BB,
I,
I.getDebugLoc(),
TII.get(SPIRV::OpLoopControlINTEL));
5634 for (
unsigned J = 1; J <
I.getNumExplicitOperands(); ++J)
5639 case Intrinsic::spv_selection_merge: {
5641 BuildMI(BB,
I,
I.getDebugLoc(),
TII.get(SPIRV::OpSelectionMerge));
5642 assert(
I.getOperand(1).isMBB() &&
5643 "operand 1 to spv_selection_merge must be a basic block");
5644 MIB.
addMBB(
I.getOperand(1).getMBB());
5645 MIB.
addImm(getSelectionOperandForImm(
I.getOperand(2).getImm()));
5649 case Intrinsic::spv_cmpxchg:
5650 return selectAtomicCmpXchg(ResVReg, ResType,
I);
5651 case Intrinsic::spv_unreachable:
5652 BuildMI(BB,
I,
I.getDebugLoc(),
TII.get(SPIRV::OpUnreachable))
5655 case Intrinsic::spv_abort:
5656 return selectAbort(
I);
5657 case Intrinsic::spv_alloca:
5658 return selectFrameIndex(ResVReg, ResType,
I);
5659 case Intrinsic::spv_alloca_array:
5660 return selectAllocaArray(ResVReg, ResType,
I);
5661 case Intrinsic::spv_assume:
5663 BuildMI(BB,
I,
I.getDebugLoc(),
TII.get(SPIRV::OpAssumeTrueKHR))
5664 .
addUse(
I.getOperand(1).getReg())
5669 case Intrinsic::spv_expect:
5671 BuildMI(BB,
I,
I.getDebugLoc(),
TII.get(SPIRV::OpExpectKHR))
5674 .
addUse(
I.getOperand(2).getReg())
5675 .
addUse(
I.getOperand(3).getReg())
5680 case Intrinsic::arithmetic_fence:
5681 if (STI.
canUseExtension(SPIRV::Extension::SPV_EXT_arithmetic_fence)) {
5682 BuildMI(BB,
I,
I.getDebugLoc(),
TII.get(SPIRV::OpArithmeticFenceEXT))
5685 .
addUse(
I.getOperand(2).getReg())
5689 return BuildCOPY(ResVReg,
I.getOperand(2).getReg(),
I);
5691 case Intrinsic::spv_thread_id:
5697 return loadVec3BuiltinInputID(SPIRV::BuiltIn::GlobalInvocationId, ResVReg,
5699 case Intrinsic::spv_thread_id_in_group:
5705 return loadVec3BuiltinInputID(SPIRV::BuiltIn::LocalInvocationId, ResVReg,
5707 case Intrinsic::spv_group_id:
5713 return loadVec3BuiltinInputID(SPIRV::BuiltIn::WorkgroupId, ResVReg, ResType,
5715 case Intrinsic::spv_flattened_thread_id_in_group:
5722 return loadBuiltinInputID(SPIRV::BuiltIn::LocalInvocationIndex, ResVReg,
5724 case Intrinsic::spv_workgroup_size:
5725 return loadVec3BuiltinInputID(SPIRV::BuiltIn::WorkgroupSize, ResVReg,
5727 case Intrinsic::spv_global_size:
5728 return loadVec3BuiltinInputID(SPIRV::BuiltIn::GlobalSize, ResVReg, ResType,
5730 case Intrinsic::spv_global_offset:
5731 return loadVec3BuiltinInputID(SPIRV::BuiltIn::GlobalOffset, ResVReg,
5733 case Intrinsic::spv_num_workgroups:
5734 return loadVec3BuiltinInputID(SPIRV::BuiltIn::NumWorkgroups, ResVReg,
5736 case Intrinsic::spv_subgroup_size:
5737 return loadBuiltinInputID(SPIRV::BuiltIn::SubgroupSize, ResVReg, ResType,
5739 case Intrinsic::spv_num_subgroups:
5740 return loadBuiltinInputID(SPIRV::BuiltIn::NumSubgroups, ResVReg, ResType,
5742 case Intrinsic::spv_subgroup_id:
5743 return loadBuiltinInputID(SPIRV::BuiltIn::SubgroupId, ResVReg, ResType,
I);
5744 case Intrinsic::spv_subgroup_local_invocation_id:
5745 return loadBuiltinInputID(SPIRV::BuiltIn::SubgroupLocalInvocationId,
5746 ResVReg, ResType,
I);
5747 case Intrinsic::spv_subgroup_max_size:
5748 return loadBuiltinInputID(SPIRV::BuiltIn::SubgroupMaxSize, ResVReg, ResType,
5750 case Intrinsic::spv_fdot:
5751 return selectFloatDot(ResVReg, ResType,
I);
5752 case Intrinsic::spv_udot:
5753 case Intrinsic::spv_sdot:
5754 if (STI.
canUseExtension(SPIRV::Extension::SPV_KHR_integer_dot_product) ||
5756 return selectIntegerDot(ResVReg, ResType,
I,
5757 IID == Intrinsic::spv_sdot);
5758 return selectIntegerDotExpansion(ResVReg, ResType,
I);
5759 case Intrinsic::spv_dot4add_i8packed:
5760 if (STI.
canUseExtension(SPIRV::Extension::SPV_KHR_integer_dot_product) ||
5762 return selectDot4AddPacked<true>(ResVReg, ResType,
I);
5763 return selectDot4AddPackedExpansion<true>(ResVReg, ResType,
I);
5764 case Intrinsic::spv_dot4add_u8packed:
5765 if (STI.
canUseExtension(SPIRV::Extension::SPV_KHR_integer_dot_product) ||
5767 return selectDot4AddPacked<false>(ResVReg, ResType,
I);
5768 return selectDot4AddPackedExpansion<false>(ResVReg, ResType,
I);
5769 case Intrinsic::spv_all:
5770 return selectAll(ResVReg, ResType,
I);
5771 case Intrinsic::spv_any:
5772 return selectAny(ResVReg, ResType,
I);
5773 case Intrinsic::spv_distance:
5774 return selectExtInst(ResVReg, ResType,
I, CL::distance, GL::Distance);
5775 case Intrinsic::spv_lerp:
5776 return selectExtInst(ResVReg, ResType,
I, CL::mix, GL::FMix);
5777 case Intrinsic::spv_length:
5778 return selectExtInst(ResVReg, ResType,
I, CL::length, GL::Length);
5779 case Intrinsic::spv_degrees:
5780 return selectExtInst(ResVReg, ResType,
I, CL::degrees, GL::Degrees);
5781 case Intrinsic::spv_faceforward:
5782 return selectExtInst(ResVReg, ResType,
I, GL::FaceForward);
5783 case Intrinsic::spv_frac:
5784 return selectExtInst(ResVReg, ResType,
I, CL::fract, GL::Fract);
5785 case Intrinsic::spv_isinf:
5786 return selectOpIsInf(ResVReg, ResType,
I);
5787 case Intrinsic::spv_isnan:
5788 return selectOpIsNan(ResVReg, ResType,
I);
5789 case Intrinsic::spv_isfinite:
5790 return selectOpIsFinite(ResVReg, ResType,
I);
5791 case Intrinsic::spv_isnormal:
5792 return selectOpIsNormal(ResVReg, ResType,
I);
5793 case Intrinsic::spv_normalize:
5794 return selectExtInst(ResVReg, ResType,
I, CL::normalize, GL::Normalize);
5795 case Intrinsic::spv_refract:
5796 return selectExtInst(ResVReg, ResType,
I, GL::Refract);
5797 case Intrinsic::spv_reflect:
5798 return selectExtInst(ResVReg, ResType,
I, GL::Reflect);
5799 case Intrinsic::spv_rsqrt:
5800 return selectExtInst(ResVReg, ResType,
I, CL::rsqrt, GL::InverseSqrt);
5801 case Intrinsic::spv_sign:
5802 return selectSign(ResVReg, ResType,
I);
5803 case Intrinsic::spv_smoothstep:
5804 return selectExtInst(ResVReg, ResType,
I, CL::smoothstep, GL::SmoothStep);
5805 case Intrinsic::spv_firstbituhigh:
5806 return selectFirstBitHigh(ResVReg, ResType,
I,
false);
5807 case Intrinsic::spv_firstbitshigh:
5808 return selectFirstBitHigh(ResVReg, ResType,
I,
true);
5809 case Intrinsic::spv_firstbitlow:
5810 return selectFirstBitLow(ResVReg, ResType,
I);
5811 case Intrinsic::spv_all_memory_barrier:
5812 return selectBarrierInst(
I, SPIRV::Scope::Device,
5813 SPIRV::MemorySemantics::UniformMemory |
5814 SPIRV::MemorySemantics::ImageMemory |
5815 SPIRV::MemorySemantics::WorkgroupMemory,
5817 case Intrinsic::spv_all_memory_barrier_with_group_sync:
5818 return selectBarrierInst(
I, SPIRV::Scope::Device,
5819 SPIRV::MemorySemantics::UniformMemory |
5820 SPIRV::MemorySemantics::ImageMemory |
5821 SPIRV::MemorySemantics::WorkgroupMemory,
5823 case Intrinsic::spv_device_memory_barrier:
5824 return selectBarrierInst(
I, SPIRV::Scope::Device,
5825 SPIRV::MemorySemantics::UniformMemory |
5826 SPIRV::MemorySemantics::ImageMemory,
5828 case Intrinsic::spv_device_memory_barrier_with_group_sync:
5829 return selectBarrierInst(
I, SPIRV::Scope::Device,
5830 SPIRV::MemorySemantics::UniformMemory |
5831 SPIRV::MemorySemantics::ImageMemory,
5833 case Intrinsic::spv_group_memory_barrier:
5834 return selectBarrierInst(
I, SPIRV::Scope::Workgroup,
5835 SPIRV::MemorySemantics::WorkgroupMemory,
5837 case Intrinsic::spv_group_memory_barrier_with_group_sync:
5838 return selectBarrierInst(
I, SPIRV::Scope::Workgroup,
5839 SPIRV::MemorySemantics::WorkgroupMemory,
5841 case Intrinsic::spv_generic_cast_to_ptr_explicit: {
5842 Register PtrReg =
I.getOperand(
I.getNumExplicitDefs() + 1).getReg();
5843 SPIRV::StorageClass::StorageClass ResSC =
5846 return diagnoseUnsupported(
I,
"The target storage class is not castable "
5847 "from the Generic storage class");
5848 BuildMI(BB,
I,
I.getDebugLoc(),
TII.get(SPIRV::OpGenericCastToPtrExplicit))
5856 case Intrinsic::spv_lifetime_start:
5857 case Intrinsic::spv_lifetime_end: {
5858 unsigned Op = IID == Intrinsic::spv_lifetime_start ? SPIRV::OpLifetimeStart
5859 : SPIRV::OpLifetimeStop;
5860 int64_t
Size =
I.getOperand(
I.getNumExplicitDefs() + 1).getImm();
5861 Register PtrReg =
I.getOperand(
I.getNumExplicitDefs() + 2).getReg();
5870 case Intrinsic::spv_saturate:
5871 return selectSaturate(ResVReg, ResType,
I);
5872 case Intrinsic::spv_nclamp:
5873 return selectExtInst(ResVReg, ResType,
I, CL::fclamp, GL::NClamp);
5874 case Intrinsic::spv_uclamp:
5875 return selectExtInst(ResVReg, ResType,
I, CL::u_clamp, GL::UClamp);
5876 case Intrinsic::spv_sclamp:
5877 return selectExtInst(ResVReg, ResType,
I, CL::s_clamp, GL::SClamp);
5878 case Intrinsic::spv_subgroup_prefix_bit_count:
5879 return selectWavePrefixBitCount(ResVReg, ResType,
I);
5880 case Intrinsic::spv_wave_active_countbits:
5881 return selectWaveActiveCountBits(ResVReg, ResType,
I);
5882 case Intrinsic::spv_wave_all_equal:
5883 return selectWaveActiveAllEqual(ResVReg, ResType,
I);
5884 case Intrinsic::spv_wave_all:
5885 return selectWaveOpInst(ResVReg, ResType,
I, SPIRV::OpGroupNonUniformAll);
5886 case Intrinsic::spv_wave_any:
5887 return selectWaveOpInst(ResVReg, ResType,
I, SPIRV::OpGroupNonUniformAny);
5888 case Intrinsic::spv_subgroup_ballot:
5889 return selectWaveOpInst(ResVReg, ResType,
I,
5890 SPIRV::OpGroupNonUniformBallot);
5891 case Intrinsic::spv_wave_is_first_lane:
5892 return selectWaveOpInst(ResVReg, ResType,
I, SPIRV::OpGroupNonUniformElect);
5893 case Intrinsic::spv_wave_reduce_or:
5894 return selectWaveReduceOp(ResVReg, ResType,
I,
5895 SPIRV::OpGroupNonUniformBitwiseOr);
5896 case Intrinsic::spv_wave_reduce_xor:
5897 return selectWaveReduceOp(ResVReg, ResType,
I,
5898 SPIRV::OpGroupNonUniformBitwiseXor);
5899 case Intrinsic::spv_wave_reduce_and:
5900 return selectWaveReduceOp(ResVReg, ResType,
I,
5901 SPIRV::OpGroupNonUniformBitwiseAnd);
5902 case Intrinsic::spv_wave_reduce_umax:
5903 return selectWaveReduceMax(ResVReg, ResType,
I,
true);
5904 case Intrinsic::spv_wave_reduce_max:
5905 return selectWaveReduceMax(ResVReg, ResType,
I,
false);
5906 case Intrinsic::spv_wave_reduce_umin:
5907 return selectWaveReduceMin(ResVReg, ResType,
I,
true);
5908 case Intrinsic::spv_wave_reduce_min:
5909 return selectWaveReduceMin(ResVReg, ResType,
I,
false);
5910 case Intrinsic::spv_wave_reduce_sum:
5911 return selectWaveReduceSum(ResVReg, ResType,
I);
5912 case Intrinsic::spv_wave_product:
5913 return selectWaveReduceProduct(ResVReg, ResType,
I);
5914 case Intrinsic::spv_wave_readlane:
5915 return selectWaveOpInst(ResVReg, ResType,
I,
5916 SPIRV::OpGroupNonUniformShuffle);
5917 case Intrinsic::spv_wave_readlane_first:
5918 return selectWaveOpInst(ResVReg, ResType,
I,
5919 SPIRV::OpGroupNonUniformBroadcastFirst);
5920 case Intrinsic::spv_wave_prefix_sum:
5921 return selectWaveExclusiveScanSum(ResVReg, ResType,
I);
5922 case Intrinsic::spv_subgroup_multi_prefix_bit_and:
5923 return selectWaveMultiPrefixOp(ResVReg, ResType,
I,
5924 SPIRV::OpGroupNonUniformBitwiseAnd,
5925 "WaveMultiPrefixBitAnd");
5926 case Intrinsic::spv_subgroup_multi_prefix_bit_or:
5927 return selectWaveMultiPrefixOp(ResVReg, ResType,
I,
5928 SPIRV::OpGroupNonUniformBitwiseOr,
5929 "WaveMultiPrefixBitOr");
5930 case Intrinsic::spv_subgroup_multi_prefix_bit_xor:
5931 return selectWaveMultiPrefixOp(ResVReg, ResType,
I,
5932 SPIRV::OpGroupNonUniformBitwiseXor,
5933 "WaveMultiPrefixBitXor");
5934 case Intrinsic::spv_subgroup_multi_prefix_product: {
5937 return selectWaveMultiPrefixOp(ResVReg, ResType,
I,
5938 IsFloatTy ? SPIRV::OpGroupNonUniformFMul
5939 : SPIRV::OpGroupNonUniformIMul,
5940 "WaveMultiPrefixProduct");
5942 case Intrinsic::spv_subgroup_multi_prefix_sum: {
5945 return selectWaveMultiPrefixOp(ResVReg, ResType,
I,
5946 IsFloatTy ? SPIRV::OpGroupNonUniformFAdd
5947 : SPIRV::OpGroupNonUniformIAdd,
5948 "WaveMultiPrefixSum");
5950 case Intrinsic::spv_wave_prefix_product:
5951 return selectWaveExclusiveScanProduct(ResVReg, ResType,
I);
5952 case Intrinsic::spv_quad_all:
5953 return selectQuadAll(ResVReg, ResType,
I);
5954 case Intrinsic::spv_quad_read_across_x: {
5955 return selectQuadSwap(ResVReg, ResType,
I, 0);
5957 case Intrinsic::spv_quad_read_across_y: {
5958 return selectQuadSwap(ResVReg, ResType,
I, 1);
5960 case Intrinsic::spv_quad_read_across_diagonal: {
5961 return selectQuadSwap(ResVReg, ResType,
I, 2);
5963 case Intrinsic::spv_radians:
5964 return selectExtInst(ResVReg, ResType,
I, CL::radians, GL::Radians);
5968 case Intrinsic::instrprof_increment:
5969 case Intrinsic::instrprof_increment_step:
5970 case Intrinsic::instrprof_value_profile:
5973 case Intrinsic::spv_value_md:
5975 case Intrinsic::spv_resource_handlefrombinding: {
5976 return selectHandleFromBinding(ResVReg, ResType,
I);
5978 case Intrinsic::spv_resource_counterhandlefrombinding:
5979 return selectCounterHandleFromBinding(ResVReg, ResType,
I);
5980 case Intrinsic::spv_resource_updatecounter:
5981 return selectUpdateCounter(ResVReg, ResType,
I);
5982 case Intrinsic::spv_resource_store_typedbuffer: {
5983 return selectImageWriteIntrinsic(
I);
5985 case Intrinsic::spv_resource_load_typedbuffer: {
5986 return selectReadImageIntrinsic(ResVReg, ResType,
I);
5988 case Intrinsic::spv_resource_load_level: {
5989 return selectLoadLevelIntrinsic(ResVReg, ResType,
I);
5991 case Intrinsic::spv_resource_getdimensions_x:
5992 case Intrinsic::spv_resource_getdimensions_xy:
5993 case Intrinsic::spv_resource_getdimensions_xyz: {
5994 return selectGetDimensionsIntrinsic(ResVReg, ResType,
I);
5996 case Intrinsic::spv_resource_getdimensions_levels_x:
5997 case Intrinsic::spv_resource_getdimensions_levels_xy:
5998 case Intrinsic::spv_resource_getdimensions_levels_xyz: {
5999 return selectGetDimensionsLevelsIntrinsic(ResVReg, ResType,
I);
6001 case Intrinsic::spv_resource_getdimensions_ms_xy:
6002 case Intrinsic::spv_resource_getdimensions_ms_xyz: {
6003 return selectGetDimensionsMSIntrinsic(ResVReg, ResType,
I);
6005 case Intrinsic::spv_resource_calculate_lod:
6006 case Intrinsic::spv_resource_calculate_lod_unclamped:
6007 return selectCalculateLodIntrinsic(ResVReg, ResType,
I);
6008 case Intrinsic::spv_resource_sample:
6009 case Intrinsic::spv_resource_sample_clamp:
6010 return selectSampleBasicIntrinsic(ResVReg, ResType,
I);
6011 case Intrinsic::spv_resource_samplebias:
6012 case Intrinsic::spv_resource_samplebias_clamp:
6013 return selectSampleBiasIntrinsic(ResVReg, ResType,
I);
6014 case Intrinsic::spv_resource_samplegrad:
6015 case Intrinsic::spv_resource_samplegrad_clamp:
6016 return selectSampleGradIntrinsic(ResVReg, ResType,
I);
6017 case Intrinsic::spv_resource_samplelevel:
6018 return selectSampleLevelIntrinsic(ResVReg, ResType,
I);
6019 case Intrinsic::spv_resource_samplecmp:
6020 case Intrinsic::spv_resource_samplecmp_clamp:
6021 return selectSampleCmpIntrinsic(ResVReg, ResType,
I);
6022 case Intrinsic::spv_resource_samplecmplevelzero:
6023 return selectSampleCmpLevelZeroIntrinsic(ResVReg, ResType,
I);
6024 case Intrinsic::spv_resource_gather:
6025 case Intrinsic::spv_resource_gather_cmp:
6026 return selectGatherIntrinsic(ResVReg, ResType,
I);
6027 case Intrinsic::spv_resource_getbasepointer:
6028 case Intrinsic::spv_resource_getpointer: {
6029 return selectResourceGetPointer(ResVReg, ResType,
I);
6031 case Intrinsic::spv_pushconstant_getpointer: {
6032 return selectPushConstantGetPointer(ResVReg, ResType,
I);
6034 case Intrinsic::spv_discard: {
6035 return selectDiscard(ResVReg, ResType,
I);
6037 case Intrinsic::spv_resource_nonuniformindex: {
6038 return selectResourceNonUniformIndex(ResVReg, ResType,
I);
6040 case Intrinsic::spv_unpackhalf2x16: {
6041 return selectExtInst(ResVReg, ResType,
I, GL::UnpackHalf2x16);
6043 case Intrinsic::spv_packhalf2x16: {
6044 return selectExtInst(ResVReg, ResType,
I, GL::PackHalf2x16);
6046 case Intrinsic::spv_ddx:
6047 return selectDerivativeInst(ResVReg, ResType,
I, SPIRV::OpDPdx);
6048 case Intrinsic::spv_ddy:
6049 return selectDerivativeInst(ResVReg, ResType,
I, SPIRV::OpDPdy);
6050 case Intrinsic::spv_ddx_coarse:
6051 return selectDerivativeInst(ResVReg, ResType,
I, SPIRV::OpDPdxCoarse);
6052 case Intrinsic::spv_ddy_coarse:
6053 return selectDerivativeInst(ResVReg, ResType,
I, SPIRV::OpDPdyCoarse);
6054 case Intrinsic::spv_ddx_fine:
6055 return selectDerivativeInst(ResVReg, ResType,
I, SPIRV::OpDPdxFine);
6056 case Intrinsic::spv_ddy_fine:
6057 return selectDerivativeInst(ResVReg, ResType,
I, SPIRV::OpDPdyFine);
6058 case Intrinsic::spv_fwidth:
6059 return selectDerivativeInst(ResVReg, ResType,
I, SPIRV::OpFwidth);
6060 case Intrinsic::spv_masked_gather:
6061 if (STI.
canUseExtension(SPIRV::Extension::SPV_INTEL_masked_gather_scatter))
6062 return selectMaskedGather(ResVReg, ResType,
I);
6063 return diagnoseUnsupported(
6064 I,
"llvm.masked.gather requires SPV_INTEL_masked_gather_scatter");
6065 case Intrinsic::spv_masked_scatter:
6066 if (STI.
canUseExtension(SPIRV::Extension::SPV_INTEL_masked_gather_scatter))
6067 return selectMaskedScatter(
I);
6068 return diagnoseUnsupported(
6069 I,
"llvm.masked.scatter requires SPV_INTEL_masked_gather_scatter");
6070 case Intrinsic::returnaddress:
6071 case Intrinsic::frameaddress: {
6073 auto MIB =
BuildMI(BB,
I,
I.getDebugLoc(),
TII.get(SPIRV::OpConstantNull))
6080 return diagnoseUnsupported(
I,
"intrinsic selection not implemented.");
6085bool SPIRVInstructionSelector::selectHandleFromBinding(
Register &ResVReg,
6086 SPIRVTypeInst ResType,
6087 MachineInstr &
I)
const {
6092bool SPIRVInstructionSelector::selectCounterHandleFromBinding(
6093 Register &ResVReg, SPIRVTypeInst ResType, MachineInstr &
I)
const {
6095 assert(Intr.getIntrinsicID() ==
6096 Intrinsic::spv_resource_counterhandlefrombinding);
6099 Register MainHandleReg = Intr.getOperand(2).getReg();
6101 assert(MainHandleDef->getIntrinsicID() ==
6102 Intrinsic::spv_resource_handlefrombinding);
6106 uint32_t ArraySize =
getIConstVal(MainHandleDef->getOperand(4).getReg(),
MRI);
6107 Register IndexReg = MainHandleDef->getOperand(5).getReg();
6108 std::string CounterName =
6113 MachineIRBuilder MIRBuilder(
I);
6115 buildPointerToResource(SPIRVTypeInst(GR.
getPointeeType(ResType)),
6117 ArraySize, IndexReg, CounterName, MIRBuilder);
6119 return BuildCOPY(ResVReg, CounterVarReg,
I);
6122bool SPIRVInstructionSelector::selectUpdateCounter(
Register &ResVReg,
6123 SPIRVTypeInst ResType,
6124 MachineInstr &
I)
const {
6126 assert(Intr.getIntrinsicID() == Intrinsic::spv_resource_updatecounter);
6128 Register CounterHandleReg = Intr.getOperand(2).getReg();
6129 int64_t IncrVal = Intr.getOperand(3).getImm();
6136 SPIRVTypeInst CounterVarPointeeType = GR.
getPointeeType(CounterVarType);
6137 assert(CounterVarPointeeType &&
6138 CounterVarPointeeType->
getOpcode() == SPIRV::OpTypeStruct &&
6139 "Counter variable must be a struct");
6141 SPIRV::StorageClass::StorageBuffer &&
6142 "Counter variable must be in the storage buffer storage class");
6144 "Counter variable must have exactly 1 member in the struct");
6145 const SPIRVTypeInst MemberType =
6148 "Counter variable struct must have a single i32 member");
6152 MachineIRBuilder MIRBuilder(
I);
6154 Type::getInt32Ty(
I.getMF()->getFunction().getContext());
6157 LLVMIntType, MIRBuilder, SPIRV::StorageClass::StorageBuffer);
6163 BuildMI(*
I.getParent(),
I,
I.getDebugLoc(),
TII.get(SPIRV::OpAccessChain))
6166 .
addUse(CounterHandleReg)
6172 Register Scope = buildI32ConstantInEntryBlock(SPIRV::Scope::Device,
I);
6174 buildI32ConstantInEntryBlock(SPIRV::MemorySemantics::None,
I);
6177 buildI32ConstantInEntryBlock(
static_cast<uint32_t
>(IncrVal),
I);
6180 BuildMI(*
I.getParent(),
I,
I.getDebugLoc(),
TII.get(SPIRV::OpAtomicIAdd))
6189 return BuildCOPY(ResVReg, AtomicRes,
I);
6197 BuildMI(*
I.getParent(),
I,
I.getDebugLoc(),
TII.get(SPIRV::OpIAddS))
6205bool SPIRVInstructionSelector::selectReadImageIntrinsic(
Register &ResVReg,
6206 SPIRVTypeInst ResType,
6207 MachineInstr &
I)
const {
6215 Register ImageReg =
I.getOperand(2).getReg();
6223 Register IdxReg =
I.getOperand(3).getReg();
6225 MachineInstr &Pos =
I;
6227 return generateImageReadOrFetch(ResVReg, ResType, NewImageReg, IdxReg, Loc,
6231bool SPIRVInstructionSelector::generateSampleImage(
6234 DebugLoc Loc, MachineInstr &Pos)
const {
6245 if (!loadHandleBeforePosition(NewSamplerReg,
6251 MachineIRBuilder MIRBuilder(Pos);
6264 bool IsExplicitLod = ImOps.GradX.has_value() || ImOps.GradY.has_value() ||
6265 ImOps.Lod.has_value();
6266 unsigned Opcode = IsExplicitLod ? SPIRV::OpImageSampleExplicitLod
6267 : SPIRV::OpImageSampleImplicitLod;
6269 Opcode = IsExplicitLod ? SPIRV::OpImageSampleDrefExplicitLod
6270 : SPIRV::OpImageSampleDrefImplicitLod;
6279 MIB.
addUse(*ImOps.Compare);
6281 uint32_t ImageOperands = 0;
6283 ImageOperands |= SPIRV::ImageOperand::Bias;
6285 ImageOperands |= SPIRV::ImageOperand::Lod;
6286 if (ImOps.GradX && ImOps.GradY)
6287 ImageOperands |= SPIRV::ImageOperand::Grad;
6288 if (ImOps.Offset && !isScalarOrVectorIntConstantZero(*ImOps.Offset)) {
6290 ImageOperands |= SPIRV::ImageOperand::ConstOffset;
6293 "Non-constant offsets are not supported in sample instructions.");
6298 ImageOperands |= SPIRV::ImageOperand::MinLod;
6300 if (ImageOperands != 0) {
6301 MIB.
addImm(ImageOperands);
6302 if (ImageOperands & SPIRV::ImageOperand::Bias)
6304 if (ImageOperands & SPIRV::ImageOperand::Lod)
6306 if (ImageOperands & SPIRV::ImageOperand::Grad) {
6307 MIB.
addUse(*ImOps.GradX);
6308 MIB.
addUse(*ImOps.GradY);
6311 (SPIRV::ImageOperand::ConstOffset | SPIRV::ImageOperand::Offset))
6312 MIB.
addUse(*ImOps.Offset);
6313 if (ImageOperands & SPIRV::ImageOperand::MinLod)
6314 MIB.
addUse(*ImOps.MinLod);
6321bool SPIRVInstructionSelector::selectImageQuerySize(
6323 std::optional<Register> LodReg)
const {
6326 "ImageReg is not an image type.");
6328 auto Dim =
static_cast<SPIRV::Dim::Dim
>(ImageType->
getOperand(2).
getImm());
6330 unsigned NumComponents = 0;
6332 case SPIRV::Dim::DIM_1D:
6333 case SPIRV::Dim::DIM_Buffer:
6334 NumComponents =
IsArray ? 2 : 1;
6336 case SPIRV::Dim::DIM_2D:
6337 case SPIRV::Dim::DIM_Cube:
6338 case SPIRV::Dim::DIM_Rect:
6339 NumComponents =
IsArray ? 3 : 2;
6341 case SPIRV::Dim::DIM_3D:
6345 I.emitGenericError(
"Unsupported image dimension for OpImageQuerySize.");
6355 if (!LodReg && IsSampled && !IsMultisampled &&
6356 Dim != SPIRV::Dim::DIM_Buffer && Dim != SPIRV::Dim::DIM_Rect)
6359 LodReg ? SPIRV::OpImageQuerySizeLod : SPIRV::OpImageQuerySize;
6360 SPIRVTypeInst ResType =
6365 auto MIB =
BuildMI(*
I.getParent(),
I,
I.getDebugLoc(),
TII.get(Opcode))
6375bool SPIRVInstructionSelector::selectGetDimensionsIntrinsic(
6376 Register &ResVReg, SPIRVTypeInst ResType, MachineInstr &
I)
const {
6377 Register ImageReg =
I.getOperand(2).getReg();
6384 return selectImageQuerySize(NewImageReg, ResVReg,
I);
6387bool SPIRVInstructionSelector::selectGetDimensionsLevelsIntrinsic(
6388 Register &ResVReg, SPIRVTypeInst ResType, MachineInstr &
I)
const {
6389 Register ImageReg =
I.getOperand(2).getReg();
6398 Register LodReg =
I.getOperand(3).getReg();
6401 "OpImageQuerySizeLod and OpImageQueryLevels require a sampled image");
6403 if (!selectImageQuerySize(NewImageReg, SizeReg,
I, LodReg)) {
6410 TII.get(SPIRV::OpImageQueryLevels))
6417 TII.get(SPIRV::OpCompositeConstruct))
6427bool SPIRVInstructionSelector::selectGetDimensionsMSIntrinsic(
6428 Register &ResVReg, SPIRVTypeInst ResType, MachineInstr &
I)
const {
6429 Register ImageReg =
I.getOperand(2).getReg();
6440 "OpImageQuerySamples requires a multisampled image");
6442 if (!selectImageQuerySize(NewImageReg, SizeReg,
I)) {
6450 TII.get(SPIRV::OpImageQuerySamples))
6457 TII.get(SPIRV::OpCompositeConstruct))
6467bool SPIRVInstructionSelector::selectCalculateLodIntrinsic(
6468 Register &ResVReg, SPIRVTypeInst ResType, MachineInstr &
I)
const {
6469 Register ImageReg =
I.getOperand(2).getReg();
6470 Register SamplerReg =
I.getOperand(3).getReg();
6471 Register CoordinateReg =
I.getOperand(4).getReg();
6487 if (!loadHandleBeforePosition(
6492 MachineIRBuilder MIRBuilder(
I);
6498 BuildMI(*
I.getParent(),
I,
I.getDebugLoc(),
TII.get(SPIRV::OpSampledImage))
6508 BuildMI(*
I.getParent(),
I,
I.getDebugLoc(),
TII.get(SPIRV::OpImageQueryLod))
6515 unsigned ExtractedIndex =
6517 Intrinsic::spv_resource_calculate_lod_unclamped
6521 MachineInstrBuilder MIB =
BuildMI(*
I.getParent(),
I,
I.getDebugLoc(),
6522 TII.get(SPIRV::OpCompositeExtract))
6532bool SPIRVInstructionSelector::selectSampleBasicIntrinsic(
6533 Register &ResVReg, SPIRVTypeInst ResType, MachineInstr &
I)
const {
6534 Register ImageReg =
I.getOperand(2).getReg();
6535 Register SamplerReg =
I.getOperand(3).getReg();
6536 Register CoordinateReg =
I.getOperand(4).getReg();
6537 ImageOperands ImOps;
6538 if (
I.getNumOperands() > 5)
6539 ImOps.Offset =
I.getOperand(5).getReg();
6540 if (
I.getNumOperands() > 6)
6541 ImOps.MinLod =
I.getOperand(6).getReg();
6542 return generateSampleImage(ResVReg, ResType, ImageReg, SamplerReg,
6543 CoordinateReg, ImOps,
I.getDebugLoc(),
I);
6546bool SPIRVInstructionSelector::selectSampleBiasIntrinsic(
6547 Register &ResVReg, SPIRVTypeInst ResType, MachineInstr &
I)
const {
6548 Register ImageReg =
I.getOperand(2).getReg();
6549 Register SamplerReg =
I.getOperand(3).getReg();
6550 Register CoordinateReg =
I.getOperand(4).getReg();
6551 ImageOperands ImOps;
6552 ImOps.Bias =
I.getOperand(5).getReg();
6553 if (
I.getNumOperands() > 6)
6554 ImOps.Offset =
I.getOperand(6).getReg();
6555 if (
I.getNumOperands() > 7)
6556 ImOps.MinLod =
I.getOperand(7).getReg();
6557 return generateSampleImage(ResVReg, ResType, ImageReg, SamplerReg,
6558 CoordinateReg, ImOps,
I.getDebugLoc(),
I);
6561bool SPIRVInstructionSelector::selectSampleGradIntrinsic(
6562 Register &ResVReg, SPIRVTypeInst ResType, MachineInstr &
I)
const {
6563 Register ImageReg =
I.getOperand(2).getReg();
6564 Register SamplerReg =
I.getOperand(3).getReg();
6565 Register CoordinateReg =
I.getOperand(4).getReg();
6566 ImageOperands ImOps;
6567 ImOps.GradX =
I.getOperand(5).getReg();
6568 ImOps.GradY =
I.getOperand(6).getReg();
6569 if (
I.getNumOperands() > 7)
6570 ImOps.Offset =
I.getOperand(7).getReg();
6571 if (
I.getNumOperands() > 8)
6572 ImOps.MinLod =
I.getOperand(8).getReg();
6573 return generateSampleImage(ResVReg, ResType, ImageReg, SamplerReg,
6574 CoordinateReg, ImOps,
I.getDebugLoc(),
I);
6577bool SPIRVInstructionSelector::selectSampleLevelIntrinsic(
6578 Register &ResVReg, SPIRVTypeInst ResType, MachineInstr &
I)
const {
6579 Register ImageReg =
I.getOperand(2).getReg();
6580 Register SamplerReg =
I.getOperand(3).getReg();
6581 Register CoordinateReg =
I.getOperand(4).getReg();
6582 ImageOperands ImOps;
6583 ImOps.Lod =
I.getOperand(5).getReg();
6584 if (
I.getNumOperands() > 6)
6585 ImOps.Offset =
I.getOperand(6).getReg();
6586 return generateSampleImage(ResVReg, ResType, ImageReg, SamplerReg,
6587 CoordinateReg, ImOps,
I.getDebugLoc(),
I);
6590bool SPIRVInstructionSelector::selectSampleCmpIntrinsic(
Register &ResVReg,
6591 SPIRVTypeInst ResType,
6592 MachineInstr &
I)
const {
6593 Register ImageReg =
I.getOperand(2).getReg();
6594 Register SamplerReg =
I.getOperand(3).getReg();
6595 Register CoordinateReg =
I.getOperand(4).getReg();
6596 ImageOperands ImOps;
6597 ImOps.Compare =
I.getOperand(5).getReg();
6598 if (
I.getNumOperands() > 6)
6599 ImOps.Offset =
I.getOperand(6).getReg();
6600 if (
I.getNumOperands() > 7)
6601 ImOps.MinLod =
I.getOperand(7).getReg();
6602 return generateSampleImage(ResVReg, ResType, ImageReg, SamplerReg,
6603 CoordinateReg, ImOps,
I.getDebugLoc(),
I);
6606bool SPIRVInstructionSelector::selectLoadLevelIntrinsic(
Register &ResVReg,
6607 SPIRVTypeInst ResType,
6608 MachineInstr &
I)
const {
6609 Register ImageReg =
I.getOperand(2).getReg();
6610 Register CoordinateReg =
I.getOperand(3).getReg();
6611 Register LodReg =
I.getOperand(4).getReg();
6613 ImageOperands ImOps;
6615 if (
I.getNumOperands() > 5)
6616 ImOps.Offset =
I.getOperand(5).getReg();
6628 return generateImageReadOrFetch(ResVReg, ResType, NewImageReg, CoordinateReg,
6629 I.getDebugLoc(),
I, &ImOps);
6632bool SPIRVInstructionSelector::selectSampleCmpLevelZeroIntrinsic(
6633 Register &ResVReg, SPIRVTypeInst ResType, MachineInstr &
I)
const {
6634 Register ImageReg =
I.getOperand(2).getReg();
6635 Register SamplerReg =
I.getOperand(3).getReg();
6636 Register CoordinateReg =
I.getOperand(4).getReg();
6637 ImageOperands ImOps;
6638 ImOps.Compare =
I.getOperand(5).getReg();
6639 if (
I.getNumOperands() > 6)
6640 ImOps.Offset =
I.getOperand(6).getReg();
6643 return generateSampleImage(ResVReg, ResType, ImageReg, SamplerReg,
6644 CoordinateReg, ImOps,
I.getDebugLoc(),
I);
6647bool SPIRVInstructionSelector::selectGatherIntrinsic(
Register &ResVReg,
6648 SPIRVTypeInst ResType,
6649 MachineInstr &
I)
const {
6650 Register ImageReg =
I.getOperand(2).getReg();
6651 Register SamplerReg =
I.getOperand(3).getReg();
6652 Register CoordinateReg =
I.getOperand(4).getReg();
6655 "ImageReg is not an image type.");
6660 ComponentOrCompareReg =
I.getOperand(5).getReg();
6661 OffsetReg =
I.getOperand(6).getReg();
6664 if (!loadHandleBeforePosition(NewImageReg, ImageType, *ImageDef,
I)) {
6668 auto Dim =
static_cast<SPIRV::Dim::Dim
>(ImageType->
getOperand(2).
getImm());
6669 if (Dim != SPIRV::Dim::DIM_2D && Dim != SPIRV::Dim::DIM_Cube &&
6670 Dim != SPIRV::Dim::DIM_Rect) {
6672 "Gather operations are only supported for 2D, Cube, and Rect images.");
6679 if (!loadHandleBeforePosition(
6684 MachineIRBuilder MIRBuilder(
I);
6685 SPIRVTypeInst SampledImageType =
6690 BuildMI(*
I.getParent(),
I,
I.getDebugLoc(),
TII.get(SPIRV::OpSampledImage))
6698 bool IsGatherCmp =
IntrId == Intrinsic::spv_resource_gather_cmp;
6700 IsGatherCmp ? SPIRV::OpImageDrefGather : SPIRV::OpImageGather;
6702 auto MIB =
BuildMI(*
I.getParent(),
I,
I.getDebugLoc(),
TII.get(Opcode))
6707 .
addUse(ComponentOrCompareReg);
6709 uint32_t ImageOperands = 0;
6710 if (OffsetReg && !isScalarOrVectorIntConstantZero(OffsetReg)) {
6711 if (Dim == SPIRV::Dim::DIM_Cube) {
6713 "Gather operations with offset are not supported for Cube images.");
6717 ImageOperands |= SPIRV::ImageOperand::ConstOffset;
6719 ImageOperands |= SPIRV::ImageOperand::Offset;
6723 if (ImageOperands != 0) {
6724 MIB.
addImm(ImageOperands);
6726 (SPIRV::ImageOperand::ConstOffset | SPIRV::ImageOperand::Offset))
6734bool SPIRVInstructionSelector::generateImageReadOrFetch(
6737 const ImageOperands *ImOps)
const {
6740 "ImageReg is not an image type.");
6742 bool IsSignedInteger =
6747 bool IsFetch = (SampledOp.getImm() == 1);
6749 auto AddOperands = [&](MachineInstrBuilder &MIB) {
6750 uint32_t ImageOperandsMask = 0;
6751 if (IsSignedInteger)
6752 ImageOperandsMask |= 0x1000;
6754 if (IsFetch && ImOps) {
6756 ImageOperandsMask |= SPIRV::ImageOperand::Lod;
6757 if (ImOps->Offset && !isScalarOrVectorIntConstantZero(*ImOps->Offset)) {
6759 ImageOperandsMask |= SPIRV::ImageOperand::ConstOffset;
6761 ImageOperandsMask |= SPIRV::ImageOperand::Offset;
6765 if (ImageOperandsMask != 0) {
6766 MIB.
addImm(ImageOperandsMask);
6767 if (IsFetch && ImOps) {
6770 if (ImOps->Offset &&
6771 (ImageOperandsMask &
6772 (SPIRV::ImageOperand::Offset | SPIRV::ImageOperand::ConstOffset)))
6773 MIB.
addUse(*ImOps->Offset);
6782 SPIRVTypeInst SampledType =
6785 SPIRVTypeInst ReadType =
6786 widenTypeToVec4(IsPacked ? SampledType : ResType, Pos);
6787 bool ReadTypeMatchesResult = ReadType == ResType;
6789 Register ReadReg = ReadTypeMatchesResult
6795 TII.get(IsFetch ? SPIRV::OpImageFetch : SPIRV::OpImageRead))
6801 BMI.constrainAllUses(
TII,
TRI, RBI);
6803 if (ReadTypeMatchesResult)
6816 if (ResultSize == 1) {
6825 return extractSubvector(ResVReg, ResType, ReadReg, Pos);
6828bool SPIRVInstructionSelector::selectResourceGetPointer(
Register &ResVReg,
6829 SPIRVTypeInst ResType,
6830 MachineInstr &
I)
const {
6831 Register ResourcePtr =
I.getOperand(2).getReg();
6833 if (
RegType->getOpcode() == SPIRV::OpTypeImage) {
6846 BuildMI(*
I.getParent(),
I,
I.getDebugLoc(),
TII.get(SPIRV::OpAccessChain))
6852 if (
I.getNumExplicitOperands() > 3) {
6853 Register IndexReg =
I.getOperand(3).getReg();
6860bool SPIRVInstructionSelector::selectPushConstantGetPointer(
6861 Register &ResVReg, SPIRVTypeInst ResType, MachineInstr &
I)
const {
6866bool SPIRVInstructionSelector::selectResourceNonUniformIndex(
6867 Register &ResVReg, SPIRVTypeInst ResType, MachineInstr &
I)
const {
6868 Register ObjReg =
I.getOperand(2).getReg();
6869 if (!BuildCOPY(ResVReg, ObjReg,
I))
6879 decorateUsesAsNonUniform(ResVReg);
6883void SPIRVInstructionSelector::decorateUsesAsNonUniform(
6886 {NonUniformReg,
nullptr}};
6887 llvm::SmallSet<Register, 8> Visited;
6888 while (WorkList.
size() > 0) {
6891 if (!Visited.
insert(CurrentReg).second)
6894 bool IsDecorated =
false;
6896 if (
Use.getOpcode() == SPIRV::OpDecorate &&
6897 Use.getOperand(1).getImm() == SPIRV::Decoration::NonUniformEXT) {
6903 if (
Use.getOperand(0).isReg() &&
Use.getOperand(0).isDef()) {
6905 if (ResultReg == CurrentReg)
6916 SPIRV::Decoration::NonUniformEXT, {});
6921bool SPIRVInstructionSelector::extractSubvector(
6923 MachineInstr &InsertionPoint)
const {
6925 [[maybe_unused]]
uint64_t InputSize =
6928 [[maybe_unused]]
bool IsLongVectorEXT =
6930 assert((InputSize > 1 || IsLongVectorEXT) &&
"The input must be a vector.");
6931 assert((ResultSize > 1 || IsLongVectorEXT) &&
"The result must be a vector.");
6932 assert(ResultSize < InputSize &&
6933 "Cannot extract more element than there are in the input.");
6940 InsertionPoint.
getDebugLoc(),
TII.get(SPIRV::OpCompositeExtract))
6949 MachineInstrBuilder MIB =
BuildMI(*InsertionPoint.
getParent(), InsertionPoint,
6951 TII.get(SPIRV::OpCompositeConstruct))
6955 for (
Register ComponentReg : ComponentRegisters)
6956 MIB.
addUse(ComponentReg);
6961bool SPIRVInstructionSelector::selectImageWriteIntrinsic(
6962 MachineInstr &
I)
const {
6969 Register ImageReg =
I.getOperand(1).getReg();
6977 Register CoordinateReg =
I.getOperand(2).getReg();
6978 Register DataReg =
I.getOperand(3).getReg();
6981 BuildMI(*
I.getParent(),
I,
I.getDebugLoc(),
TII.get(SPIRV::OpImageWrite))
6989Register SPIRVInstructionSelector::buildPointerToResource(
6990 SPIRVTypeInst SpirvResType, SPIRV::StorageClass::StorageClass SC,
6991 uint32_t Set, uint32_t
Binding, uint32_t ArraySize,
Register IndexReg,
6992 StringRef Name, MachineIRBuilder MIRBuilder)
const {
6994 if (ArraySize == 1) {
6995 SPIRVTypeInst PtrType =
6998 "SpirvResType did not have an explicit layout.");
7003 const Type *VarType = ArrayType::get(
const_cast<Type *
>(ResType), ArraySize);
7004 SPIRVTypeInst VarPointerType =
7007 VarPointerType, Set,
Binding, Name, MIRBuilder);
7009 SPIRVTypeInst ResPointerType =
7022bool SPIRVInstructionSelector::selectFirstBitSet16(
7023 Register ResVReg, SPIRVTypeInst ResType, MachineInstr &
I,
7024 unsigned ExtendOpcode,
unsigned BitSetOpcode)
const {
7026 if (!selectOpWithSrcs(ExtReg, ResType,
I, {
I.getOperand(2).getReg()},
7030 return selectFirstBitSet32(ResVReg, ResType,
I, ExtReg, BitSetOpcode);
7033bool SPIRVInstructionSelector::selectFirstBitSet32(
7035 unsigned BitSetOpcode)
const {
7036 BuildMI(*
I.getParent(),
I,
I.getDebugLoc(),
TII.get(SPIRV::OpExtInst))
7039 .
addImm(
static_cast<uint32_t
>(SPIRV::InstructionSet::GLSL_std_450))
7046bool SPIRVInstructionSelector::selectFirstBitSet64(
7048 unsigned BitSetOpcode,
bool SwapPrimarySide)
const {
7062 if (ComponentCount > 2) {
7063 auto Func = [
this, SwapPrimarySide](
Register ResVReg, SPIRVTypeInst ResType,
7065 unsigned Opcode) ->
bool {
7066 return this->selectFirstBitSet64(ResVReg, ResType,
I, SrcReg, Opcode,
7070 return handle64BitOverflow(ResVReg, ResType,
I, SrcReg, BitSetOpcode, Func);
7074 MachineIRBuilder MIRBuilder(
I);
7076 BaseType, 2 * ComponentCount, MIRBuilder,
false);
7080 if (!selectOpWithSrcs(BitcastReg, PostCastType,
I, {SrcReg},
7086 if (!selectFirstBitSet32(FBSReg, PostCastType,
I, BitcastReg, BitSetOpcode))
7096 if (!selectOpWithSrcs(HighReg, ResType,
I, {FBSReg, ConstIntOne},
7097 SPIRV::OpVectorExtractDynamic))
7099 if (!selectOpWithSrcs(LowReg, ResType,
I, {FBSReg, ConstIntZero},
7100 SPIRV::OpVectorExtractDynamic))
7104 auto MIB =
BuildMI(*
I.getParent(),
I,
I.getDebugLoc(),
7105 TII.get(SPIRV::OpVectorShuffle))
7113 for (
unsigned J = 1; J < ComponentCount * 2; J += 2) {
7119 MIB =
BuildMI(*
I.getParent(),
I,
I.getDebugLoc(),
7120 TII.get(SPIRV::OpVectorShuffle))
7128 for (
unsigned J = 0; J < ComponentCount * 2; J += 2) {
7148 SelectOp = SPIRV::OpSelectSISCond;
7149 AddOp = SPIRV::OpIAddS;
7157 SelectOp = SPIRV::OpSelectVIVCond;
7158 AddOp = SPIRV::OpIAddV;
7164 Register RegSecondaryOffset = Reg0;
7168 if (SwapPrimarySide) {
7169 PrimaryReg = LowReg;
7170 SecondaryReg = HighReg;
7171 RegPrimaryOffset = Reg0;
7172 RegSecondaryOffset = Reg32;
7177 if (!selectOpWithSrcs(RegSecondaryHasVal, BoolType,
I,
7178 {SecondaryReg, NegOneReg}, SPIRV::OpINotEqual))
7183 if (!selectOpWithSrcs(RegPrimaryHasVal, BoolType,
I, {PrimaryReg, NegOneReg},
7184 SPIRV::OpINotEqual))
7191 if (!selectOpWithSrcs(RegReturnBits, ResType,
I,
7192 {RegSecondaryHasVal, SecondaryReg, NegOneReg},
7197 if (SwapPrimarySide) {
7199 if (!selectOpWithSrcs(RegAdd, ResType,
I,
7200 {RegSecondaryHasVal, RegSecondaryOffset, Reg0},
7211 if (!selectOpWithSrcs(RegReturnBits2, ResType,
I,
7212 {RegPrimaryHasVal, PrimaryReg, RegReturnBits},
7217 if (!selectOpWithSrcs(RegAdd2, ResType,
I,
7218 {RegPrimaryHasVal, RegPrimaryOffset, RegAdd}, SelectOp))
7221 return selectOpWithSrcs(ResVReg, ResType,
I, {RegReturnBits2, RegAdd2},
7225bool SPIRVInstructionSelector::selectFirstBitHigh(
Register ResVReg,
7226 SPIRVTypeInst ResType,
7228 bool IsSigned)
const {
7230 Register OpReg =
I.getOperand(2).getReg();
7233 unsigned ExtendOpcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert;
7234 unsigned BitSetOpcode = IsSigned ? GL::FindSMsb : GL::FindUMsb;
7238 return selectFirstBitSet16(ResVReg, ResType,
I, ExtendOpcode, BitSetOpcode);
7240 return selectFirstBitSet32(ResVReg, ResType,
I, OpReg, BitSetOpcode);
7242 return selectFirstBitSet64(ResVReg, ResType,
I, OpReg, BitSetOpcode,
7245 return diagnoseUnsupported(
7247 "spv_firstbituhigh and spv_firstbitshigh only support 16,32,64 bits.");
7251bool SPIRVInstructionSelector::selectFirstBitLow(
Register ResVReg,
7252 SPIRVTypeInst ResType,
7253 MachineInstr &
I)
const {
7255 Register OpReg =
I.getOperand(2).getReg();
7260 unsigned ExtendOpcode = SPIRV::OpUConvert;
7261 unsigned BitSetOpcode = GL::FindILsb;
7265 return selectFirstBitSet16(ResVReg, ResType,
I, ExtendOpcode, BitSetOpcode);
7267 return selectFirstBitSet32(ResVReg, ResType,
I, OpReg, BitSetOpcode);
7269 return selectFirstBitSet64(ResVReg, ResType,
I, OpReg, BitSetOpcode,
7272 return diagnoseUnsupported(
I,
7273 "spv_firstbitlow only supports 16,32,64 bits.");
7277bool SPIRVInstructionSelector::selectAllocaArray(
Register ResVReg,
7278 SPIRVTypeInst ResType,
7279 MachineInstr &
I)
const {
7284 bool UseUntypedPointers =
7285 ResType->
getOpcode() == SPIRV::OpTypeUntypedPointerKHR;
7286 unsigned Opcode = UseUntypedPointers
7287 ? SPIRV::OpUntypedVariableLengthArrayINTEL
7288 : SPIRV::OpVariableLengthArrayINTEL;
7290 auto MIB =
BuildMI(BB,
I,
I.getDebugLoc(),
TII.get(Opcode))
7296 if (UseUntypedPointers) {
7299 "untyped variable length array result must have a recorded element "
7304 MIB.
addUse(
I.getOperand(2).getReg());
7308 unsigned Alignment =
I.getOperand(3).getImm();
7322 while (!Worklist.
empty()) {
7324 switch (
T->getOpcode()) {
7325 case SPIRV::OpTypeInt:
7326 case SPIRV::OpTypeFloat:
7327 case SPIRV::OpTypePointer:
7329 case SPIRV::OpTypeVector:
7330 case SPIRV::OpTypeVectorIdEXT:
7331 case SPIRV::OpTypeMatrix:
7332 case SPIRV::OpTypeArray: {
7333 Register OperandReg =
T->getOperand(1).getReg();
7337 case SPIRV::OpTypeStruct:
7338 for (
unsigned Idx = 1,
E =
T->getNumOperands(); Idx <
E; ++Idx) {
7339 Register OperandReg =
T->getOperand(Idx).getReg();
7354 Register TypeReg = Ty->getOperand(0).getReg();
7355 if (!Visited.
insert(TypeReg).second)
7358 switch (Ty->getOpcode()) {
7359 case SPIRV::OpTypePointer:
7360 if (Ty->getOperand(1).getImm() == SPIRV::StorageClass::StorageBuffer)
7364 case SPIRV::OpTypeArray:
7365 case SPIRV::OpTypeRuntimeArray:
7368 case SPIRV::OpTypeStruct:
7369 for (
unsigned I = 1;
I < Ty->getNumOperands(); ++
I)
7385bool SPIRVInstructionSelector::selectAbort(MachineInstr &
I)
const {
7386 assert(
I.getNumExplicitOperands() == 2);
7388 Register MsgReg =
I.getOperand(1).getReg();
7390 assert(MsgType &&
"Message argument of llvm.spv.abort has no SPIR-V type");
7393 return diagnoseUnsupported(
7395 "llvm.spv.abort message type must be a concrete SPIR-V type (numerical "
7396 "scalar, pointer, vector, matrix, or aggregate of such types)");
7399 BuildMI(BB,
I,
I.getDebugLoc(),
TII.get(SPIRV::OpAbortKHR))
7406bool SPIRVInstructionSelector::selectTrap(MachineInstr &
I)
const {
7417 uint32_t MsgVal = ~0
u;
7418 if (
I.getOpcode() == TargetOpcode::G_UBSANTRAP)
7419 MsgVal =
static_cast<uint32_t
>(
I.getOperand(0).
getImm());
7422 Register MsgReg = buildI32ConstantInEntryBlock(MsgVal,
I, MsgType);
7425 BuildMI(BB,
I,
I.getDebugLoc(),
TII.get(SPIRV::OpAbortKHR))
7432bool SPIRVInstructionSelector::selectFrameIndex(
Register ResVReg,
7433 SPIRVTypeInst ResType,
7434 MachineInstr &
I)
const {
7441 bool UseUntypedPointers =
7442 ResType->
getOpcode() == SPIRV::OpTypeUntypedPointerKHR;
7444 UseUntypedPointers ? SPIRV::OpUntypedVariableKHR : SPIRV::OpVariable;
7447 MachineIRBuilder MIRBuilder(
I);
7450 .
addImm(SPIRV::Extension::SPV_KHR_variable_pointers);
7452 .
addImm(SPIRV::Capability::VariablePointersStorageBuffer);
7455 auto MIB =
BuildMI(*It->getParent(), It, It->getDebugLoc(),
TII.get(Opcode))
7458 .
addImm(
static_cast<uint32_t
>(SPIRV::StorageClass::Function));
7462 if (UseUntypedPointers) {
7466 return diagnoseUnsupported(
7467 I,
"could not deduce the data type of an untyped variable");
7473 unsigned Alignment =
I.getOperand(2).getImm();
7480bool SPIRVInstructionSelector::selectBranch(MachineInstr &
I)
const {
7485 const MachineInstr *PrevI =
I.getPrevNode();
7487 if (PrevI !=
nullptr && PrevI->
getOpcode() == TargetOpcode::G_BRCOND) {
7491 .
addMBB(
I.getOperand(0).getMBB())
7496 .
addMBB(
I.getOperand(0).getMBB())
7501bool SPIRVInstructionSelector::selectBranchCond(MachineInstr &
I)
const {
7512 const MachineInstr *NextI =
I.getNextNode();
7514 if (NextI !=
nullptr && NextI->
getOpcode() == SPIRV::OpBranchConditional)
7520 MachineBasicBlock *NextMBB =
I.getMF()->getBlockNumbered(NextMBBNum);
7522 .
addUse(
I.getOperand(0).getReg())
7523 .
addMBB(
I.getOperand(1).getMBB())
7529bool SPIRVInstructionSelector::selectPhi(
Register ResVReg,
7530 MachineInstr &
I)
const {
7532 BuildMI(*
I.getParent(),
I,
I.getDebugLoc(),
TII.get(TargetOpcode::PHI))
7534 const unsigned NumOps =
I.getNumOperands();
7535 for (
unsigned i = 1; i <
NumOps; i += 2) {
7536 MIB.
addUse(
I.getOperand(i + 0).getReg());
7537 MIB.
addMBB(
I.getOperand(i + 1).getMBB());
7543bool SPIRVInstructionSelector::selectGlobalValue(
7544 Register ResVReg, MachineInstr &
I,
const MachineInstr *Init)
const {
7546 MachineIRBuilder MIRBuilder(
I);
7547 const GlobalValue *GV =
I.
getOperand(1).getGlobal();
7550 std::string GlobalIdent;
7552 unsigned &
ID = UnnamedGlobalIDs[GV];
7554 ID = UnnamedGlobalIDs.
size();
7555 GlobalIdent =
"__unnamed_" + Twine(ID).str();
7581 GVFun ? SPIRV::StorageClass::CodeSectionINTEL
7588 MachineRegisterInfo *
MRI = MIRBuilder.
getMRI();
7593 MachineInstrBuilder MIB1 =
7594 BuildMI(BB,
I,
I.getDebugLoc(),
TII.get(SPIRV::OpUndef))
7597 MachineInstrBuilder MIB2 =
7599 TII.get(SPIRV::OpConstantFunctionPointerINTEL))
7603 GR.
add(ConstVal, MIB2);
7611 MachineInstrBuilder MIB3 =
7612 BuildMI(BB,
I,
I.getDebugLoc(),
TII.get(SPIRV::OpUndef))
7615 GR.
add(ConstVal, MIB3);
7621 assert(NewReg != ResVReg);
7622 return BuildCOPY(ResVReg, NewReg,
I);
7632 const std::optional<SPIRV::LinkageType::LinkageType> LnkType =
7635 if (LnkType && *LnkType == SPIRV::LinkageType::Import)
7641 SPIRVTypeInst ResType =
7645 GlobalVar->isConstant(), LnkType, MIRBuilder,
true);
7650 if (
GlobalVar->isExternallyInitialized() &&
7651 STI.getTargetTriple().getVendor() ==
Triple::AMD) {
7652 constexpr unsigned ReadWriteINTEL = 3u;
7655 MachineInstrBuilder MIB(*MF, --MIRBuilder.
getInsertPt());
7661bool SPIRVInstructionSelector::selectLog10(
Register ResVReg,
7662 SPIRVTypeInst ResType,
7663 MachineInstr &
I)
const {
7665 return selectExtInst(ResVReg, ResType,
I, CL::log10);
7673 MachineIRBuilder MIRBuilder(
I);
7678 BuildMI(BB,
I,
I.getDebugLoc(),
TII.get(SPIRV::OpExtInst))
7681 .
addImm(
static_cast<uint32_t
>(SPIRV::InstructionSet::GLSL_std_450))
7683 .
add(
I.getOperand(1))
7697 APFloat::rmNearestTiesToEven, &LosesInfo);
7702 isVectorType(ResType) ? SPIRV::OpVectorTimesScalar : SPIRV::OpFMulS;
7712bool SPIRVInstructionSelector::selectFpowi(
Register ResVReg,
7713 SPIRVTypeInst ResType,
7714 MachineInstr &
I)
const {
7717 return selectExtInst(ResVReg, ResType,
I, CL::pown);
7723 Register ExpReg =
I.getOperand(2).getReg();
7725 if (!selectOpWithSrcs(FloatExpReg, ResType,
I, {ExpReg},
7726 SPIRV::OpConvertSToF))
7728 return selectExtInst(ResVReg, ResType,
I, GL::Pow,
7735bool SPIRVInstructionSelector::selectModf(
Register ResVReg,
7736 SPIRVTypeInst ResType,
7737 MachineInstr &
I)
const {
7753 MachineIRBuilder MIRBuilder(
I);
7754 SPIRVTypeInst FloatType =
7758 FloatType, MIRBuilder, SPIRV::StorageClass::Function);
7771 MachineBasicBlock &EntryBB =
I.getMF()->
front();
7772 const bool IsUntyped =
7773 PtrType->
getOpcode() == SPIRV::OpTypeUntypedPointerKHR;
7775 BuildMI(EntryBB, VarPos,
I.getDebugLoc(),
7776 TII.get(IsUntyped ? SPIRV::OpUntypedVariableKHR
7777 : SPIRV::OpVariable))
7780 .
addImm(
static_cast<uint32_t
>(SPIRV::StorageClass::Function));
7788 BuildMI(BB,
I,
I.getDebugLoc(),
TII.get(SPIRV::OpExtInst))
7791 .
addImm(
static_cast<uint32_t
>(SPIRV::InstructionSet::OpenCL_std))
7794 .
add(
I.getOperand(
I.getNumExplicitDefs()))
7798 Register IntegralPartReg =
I.getOperand(1).getReg();
7801 auto LoadMIB =
BuildMI(BB,
I,
I.getDebugLoc(),
TII.get(SPIRV::OpLoad))
7811 assert(
false &&
"GLSL::Modf is deprecated.");
7822bool SPIRVInstructionSelector::loadVec3BuiltinInputID(
7823 SPIRV::BuiltIn::BuiltIn BuiltInValue,
Register ResVReg,
7824 SPIRVTypeInst ResType, MachineInstr &
I)
const {
7825 MachineIRBuilder MIRBuilder(
I);
7826 const SPIRVTypeInst Vec3Ty =
7829 Vec3Ty, MIRBuilder, SPIRV::StorageClass::Input);
7841 SPIRV::StorageClass::Input,
nullptr,
true, std::nullopt, MIRBuilder,
7845 MachineRegisterInfo *
MRI = MIRBuilder.
getMRI();
7851 BuildMI(*
I.getParent(),
I,
I.getDebugLoc(),
TII.get(SPIRV::OpLoad))
7858 assert(
I.getOperand(2).isReg());
7859 const uint32_t ThreadId =
foldImm(
I.getOperand(2),
MRI);
7863 auto MIB =
BuildMI(BB,
I,
I.getDebugLoc(),
TII.get(SPIRV::OpCompositeExtract))
7874bool SPIRVInstructionSelector::loadBuiltinInputID(
7875 SPIRV::BuiltIn::BuiltIn BuiltInValue,
Register ResVReg,
7876 SPIRVTypeInst ResType, MachineInstr &
I)
const {
7877 MachineIRBuilder MIRBuilder(
I);
7879 ResType, MIRBuilder, SPIRV::StorageClass::Input);
7894 SPIRV::StorageClass::Input,
nullptr,
true, std::nullopt, MIRBuilder,
7898 auto MIB =
BuildMI(*
I.getParent(),
I,
I.getDebugLoc(),
TII.get(SPIRV::OpLoad))
7907SPIRVTypeInst SPIRVInstructionSelector::widenTypeToVec4(SPIRVTypeInst
Type,
7908 MachineInstr &
I)
const {
7909 MachineIRBuilder MIRBuilder(
I);
7920bool SPIRVInstructionSelector::loadHandleBeforePosition(
7921 Register &HandleReg, SPIRVTypeInst ResType, GIntrinsic &HandleDef,
7922 MachineInstr &Pos)
const {
7925 Intrinsic::spv_resource_handlefrombinding);
7933 bool IsStructuredBuffer = ResType->
getOpcode() == SPIRV::OpTypePointer;
7934 MachineIRBuilder MIRBuilder(HandleDef);
7935 SPIRVTypeInst VarType = ResType;
7936 SPIRV::StorageClass::StorageClass SC = SPIRV::StorageClass::UniformConstant;
7938 if (IsStructuredBuffer) {
7947 .
addImm(SPIRV::Capability::RuntimeDescriptorArrayEXT);
7950 buildPointerToResource(SPIRVTypeInst(VarType), SC, Set,
Binding,
7951 ArraySize, IndexReg, Name, MIRBuilder);
7955 uint32_t LoadOpcode =
7956 IsStructuredBuffer ? SPIRV::OpCopyObject : SPIRV::OpLoad;
7966bool SPIRVInstructionSelector::errorIfInstrOutsideShader(
7967 MachineInstr &
I)
const {
7969 return diagnoseUnsupported(
7970 I,
"this instruction is only supported in shaders.");
7975InstructionSelector *
7979 return new SPIRVInstructionSelector(TM,
Subtarget, RBI);
MachineInstrBuilder & UseMI
MachineInstrBuilder MachineInstrBuilder & DefMI
const AArch64Subtarget & Subtarget
#define GET_GLOBALISEL_PREDICATES_INIT
#define GET_GLOBALISEL_TEMPORARIES_INIT
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
const AArch64InstrInfo MachineBasicBlock::iterator InsertPt
const AArch64InstrInfo & TII
This file declares a class to represent arbitrary precision floating point values and provide a varie...
static bool selectUnmergeValues(MachineInstrBuilder &MIB, const ARMBaseInstrInfo &TII, MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI, const RegisterBankInfo &RBI)
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static uint8_t SwapBits(uint8_t Val)
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
DXIL Resource Implicit Binding
Declares convenience wrapper classes for interpreting MachineInstr instances as specific generic oper...
static MaybeAlign getAlign(Value *Ptr)
LLVMTypeRef LLVMIntType(unsigned NumBits)
const size_t AbstractManglingParser< Derived, Alloc >::NumOps
Loop::LoopBounds::Direction Direction
Register const TargetRegisterInfo * TRI
Promote Memory to Register
uint64_t IntrinsicInst * II
static StringRef getName(Value *V)
static unsigned getFCmpOpcode(CmpInst::Predicate Pred, unsigned Size)
MachineRegisterInfo & MRI
static bool isConcreteSPIRVType(SPIRVTypeInst Ty, const SPIRVGlobalRegistry &GR)
static APFloat getOneFP(const Type *LLVMFloatTy)
static bool isUSMStorageClass(SPIRV::StorageClass::StorageClass SC)
static bool isASCastInGVar(MachineRegisterInfo *MRI, Register ResVReg)
static bool mayApplyGenericSelection(unsigned Opcode)
static APFloat getZeroFP(const Type *LLVMFloatTy)
std::vector< std::pair< SPIRV::InstructionSet::InstructionSet, uint32_t > > ExtInstList
static bool intrinsicHasSideEffects(Intrinsic::ID ID)
static unsigned getBoolCmpOpcode(unsigned PredNum)
static unsigned getICmpOpcode(unsigned PredNum)
static bool isOpcodeWithNoSideEffects(unsigned Opcode)
static bool containsStorageBufferPointer(SPIRVTypeInst Ty, const SPIRVGlobalRegistry &GR, SmallSet< Register, 8 > &Visited)
static bool isConstReg(MachineRegisterInfo *MRI, MachineInstr *OpDef)
static unsigned getPtrCmpOpcode(unsigned Pred)
static void addMemoryOperands(const MachineMemOperand *MemOp, MachineInstrBuilder &MIB, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry &GR, std::optional< Align > AlignOverride=std::nullopt)
bool isDead(const MachineInstr &MI, const MachineRegisterInfo &MRI)
static Register convertPtrToInt(Register Reg, LLT ConvTy, SPIRVTypeInst SpvType, LegalizerHelper &Helper, MachineRegisterInfo &MRI, SPIRVGlobalRegistry *GR)
#define SPIRV_BACKEND_SERVICE_FUN_NAME
BaseType
A given derived pointer can have multiple base pointers through phi/selects.
This file defines the SmallSet class.
static TableGen::Emitter::Opt Y("gen-skeleton-entry", EmitSkeleton, "Generate example skeleton entry")
static ManagedStatic< cl::opt< FnT >, OptCreatorT > CallbackFunction
static const fltSemantics & IEEEsingle()
static const fltSemantics & BFloat()
static const fltSemantics & IEEEdouble()
static const fltSemantics & IEEEhalf()
const fltSemantics & getSemantics() const
static APFloat getOne(const fltSemantics &Sem, bool Negative=false)
Factory for Positive and Negative One.
static APFloat getZero(const fltSemantics &Sem, bool Negative=false)
Factory for Positive and Negative Zero.
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
static APInt getSignMask(unsigned BitWidth)
Get the SignMask for a specific bit width.
Represent a constant reference to an array (0 or more elements consecutively in memory),...
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
@ FCMP_OEQ
0 0 0 1 True if ordered and equal
@ ICMP_SLT
signed less than
@ ICMP_SLE
signed less or equal
@ FCMP_OLT
0 1 0 0 True if ordered and less than
@ FCMP_ULE
1 1 0 1 True if unordered, less than, or equal
@ FCMP_OGT
0 0 1 0 True if ordered and greater than
@ FCMP_OGE
0 0 1 1 True if ordered and greater than or equal
@ ICMP_UGE
unsigned greater or equal
@ ICMP_UGT
unsigned greater than
@ ICMP_SGT
signed greater than
@ FCMP_ULT
1 1 0 0 True if unordered or less than
@ FCMP_ONE
0 1 1 0 True if ordered and operands are unequal
@ FCMP_UEQ
1 0 0 1 True if unordered or equal
@ ICMP_ULT
unsigned less than
@ FCMP_UGT
1 0 1 0 True if unordered or greater than
@ FCMP_OLE
0 1 0 1 True if ordered and less than or equal
@ FCMP_ORD
0 1 1 1 True if ordered (no nans)
@ ICMP_SGE
signed greater or equal
@ FCMP_UNE
1 1 1 0 True if unordered or not equal
@ ICMP_ULE
unsigned less or equal
@ FCMP_UGE
1 0 1 1 True if unordered, greater than, or equal
@ FCMP_UNO
1 0 0 0 True if unordered: isnan(X) | isnan(Y)
static LLVM_ABI Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
static LLVM_ABI std::optional< GFConstant > getConstant(Register Const, const MachineRegisterInfo &MRI)
static LLVM_ABI std::optional< GIConstant > getConstant(Register Const, const MachineRegisterInfo &MRI)
Represents a call to an intrinsic.
Intrinsic::ID getIntrinsicID() const
unsigned getAddressSpace() const
Module * getParent()
Get the module that this global value is contained inside of...
@ InternalLinkage
Rename collisions when linking (static functions).
static LLVM_ABI IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
static constexpr LLT float64()
Get a 64-bit IEEE double value.
static constexpr LLT scalar(unsigned SizeInBits)
Get a low-level scalar or aggregate "bag of bits".
constexpr bool isValid() const
constexpr uint16_t getNumElements() const
Returns the number of elements in a vector LLT.
constexpr bool isVector() const
static constexpr LLT pointer(unsigned AddressSpace, unsigned SizeInBits)
Get a low-level pointer in the given address space.
static constexpr LLT fixed_vector(unsigned NumElements, unsigned ScalarSizeInBits)
Get a low-level fixed-width vector of some number of elements and element width.
int getNumber() const
MachineBasicBlocks are uniquely numbered at the function level, unless they're not in a MachineFuncti...
LLVM_ABI iterator getFirstNonPHI()
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
instr_iterator instr_end()
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
MachineInstrBundleIterator< MachineInstr > iterator
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Function & getFunction()
Return the LLVM function that this machine code represents.
Helper class to build MachineInstr.
MachineBasicBlock::iterator getInsertPt()
Current insertion point for new instructions.
MachineInstrBuilder buildInstr(unsigned Opcode)
Build and insert <empty> = Opcode <empty>.
MachineFunction & getMF()
Getter for the function we currently build.
MachineRegisterInfo * getMRI()
Getter for MRI.
void constrainAllUses(const TargetInstrInfo &TII, const TargetRegisterInfo &TRI, const RegisterBankInfo &RBI) const
const MachineInstrBuilder & addUse(Register RegNo, RegState Flags={}, unsigned SubReg=0) const
Add a virtual register use operand.
const MachineInstrBuilder & addReg(Register RegNo, RegState Flags={}, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & add(const MachineOperand &MO) const
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
const MachineInstrBuilder & addDef(Register RegNo, RegState Flags={}, unsigned SubReg=0) const
Add a virtual register definition operand.
const MachineInstrBuilder & setMIFlags(unsigned Flags) const
MachineInstr * getInstr() const
If conversion operators fail, use this method to get the MachineInstr explicitly.
Representation of each machine instruction.
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
const MachineBasicBlock * getParent() const
unsigned getNumOperands() const
Retuns the total number of operands.
LLVM_ABI unsigned getNumExplicitOperands() const
Returns the number of non-implicit operands.
LLVM_ABI unsigned getNumExplicitDefs() const
Returns the number of non-implicit definitions.
LLVM_ABI void substituteRegister(Register FromReg, Register ToReg, unsigned SubIdx, const TargetRegisterInfo &RegInfo)
Replace all occurrences of FromReg with ToReg:SubIdx, properly composing subreg indices where necessa...
LLVM_ABI void emitGenericError(const Twine &ErrMsg) const
LLVM_ABI const MachineFunction * getMF() const
Return the function that contains the basic block that this instruction belongs to.
const DebugLoc & getDebugLoc() const
Returns the debug location id of this MachineInstr.
const MachineOperand & getOperand(unsigned i) const
A description of a memory reference used in the backend.
@ MOVolatile
The memory access is volatile.
@ MONonTemporal
The memory access is non-temporal.
LLVM_ABI Align getAlign() const
Return the minimum known alignment in bytes of the actual memory reference.
bool isReg() const
isReg - Tests if this is a MO_Register operand.
MachineBasicBlock * getMBB() const
Register getReg() const
getReg - Returns the register number.
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
defusechain_instr_iterator< true, false, false, true > use_instr_iterator
use_instr_iterator/use_instr_begin/use_instr_end - Walk all uses of the specified register,...
const TargetRegisterClass * getRegClass(Register Reg) const
Return the register class of the specified virtual register.
LLVM_ABI LLVM_READONLY MachineInstr * getVRegDef(Register Reg) const
getVRegDef - Return the machine instr that defines the specified virtual register or null if none is ...
use_instr_iterator use_instr_begin(Register RegNo) const
bool use_nodbg_empty(Register RegNo) const
use_nodbg_empty - Return true if there are no non-Debug instructions using the specified register.
static def_instr_iterator def_instr_end()
defusechain_instr_iterator< false, true, false, true > def_instr_iterator
def_instr_iterator/def_instr_begin/def_instr_end - Walk all defs of the specified register,...
LLVM_ABI Register createVirtualRegister(const TargetRegisterClass *RegClass, StringRef Name="")
createVirtualRegister - Create and return a new virtual register in the function with the specified r...
def_instr_iterator def_instr_begin(Register RegNo) const
LLT getType(Register Reg) const
Get the low-level type of Reg or LLT{} if Reg is not a generic (target independent) virtual register.
bool hasOneUse(Register RegNo) const
hasOneUse - Return true if there is exactly one instruction using the specified register.
static use_instr_iterator use_instr_end()
iterator_range< use_instr_nodbg_iterator > use_nodbg_instructions(Register Reg) const
LLVM_ABI void setType(Register VReg, LLT Ty)
Set the low-level type of VReg to Ty.
const MachineFunction & getMF() const
LLVM_ABI void setRegClass(Register Reg, const TargetRegisterClass *RC)
setRegClass - Set the register class of the specified virtual register.
LLVM_ABI Register createGenericVirtualRegister(LLT Ty, StringRef Name="")
Create and return a new generic virtual register with low-level type Ty.
const TargetRegisterClass * getRegClassOrNull(Register Reg) const
Return the register class of Reg, or null if Reg has not been assigned a register class yet.
iterator_range< use_instr_iterator > use_instructions(Register Reg) const
unsigned getNumVirtRegs() const
getNumVirtRegs - Return the number of virtual registers created.
LLVM_ABI void replaceRegWith(Register FromReg, Register ToReg)
replaceRegWith - Replace all instances of FromReg with ToReg in the machine function.
Analysis providing profile information.
Holds all the information related to register banks.
Wrapper class representing virtual and physical registers.
constexpr bool isValid() const
constexpr bool isPhysical() const
Return true if the specified register number is in the physical register namespace.
const SDValue & getOperand(unsigned i) const
bool isScalarOrVectorSigned(SPIRVTypeInst Type) const
SPIRVTypeInst getOrCreateOpTypeSampledImage(SPIRVTypeInst ImageType, MachineIRBuilder &MIRBuilder)
void assignSPIRVTypeToVReg(SPIRVTypeInst Type, Register VReg, const MachineFunction &MF)
SPIRVTypeInst getOrCreateSPIRVPointerType(const Type *BaseType, MachineIRBuilder &MIRBuilder, SPIRV::StorageClass::StorageClass SC, bool ForceTyped=false)
const TargetRegisterClass * getRegClass(SPIRVTypeInst SpvType) const
MachineInstr * getOrAddMemAliasingINTELInst(MachineIRBuilder &MIRBuilder, const MDNode *AliasingListMD)
bool isAggregateType(SPIRVTypeInst Type) const
unsigned getScalarOrVectorBitWidth(SPIRVTypeInst Type) const
SPIRVTypeInst getOrCreateSPIRVIntegerType(unsigned BitWidth, MachineIRBuilder &MIRBuilder)
SPIRVTypeInst getOrCreateSPIRVVectorType(SPIRVTypeInst BaseType, unsigned NumElements, MachineIRBuilder &MIRBuilder, bool EmitIR)
Register buildGlobalVariable(Register Reg, SPIRVTypeInst BaseType, StringRef Name, const GlobalValue *GV, SPIRV::StorageClass::StorageClass Storage, const MachineInstr *Init, bool IsConst, const std::optional< SPIRV::LinkageType::LinkageType > &LinkageType, MachineIRBuilder &MIRBuilder, bool IsInstSelector)
SPIRVTypeInst getResultType(Register VReg, MachineFunction *MF=nullptr)
unsigned getScalarOrVectorComponentCount(Register VReg) const
const Type * getTypeForSPIRVType(SPIRVTypeInst Ty) const
bool isBitcastCompatible(SPIRVTypeInst Type1, SPIRVTypeInst Type2) const
unsigned getPointerSize() const
Register getOrCreateConstFP(APFloat Val, MachineInstr &I, SPIRVTypeInst SpvType, const SPIRVInstrInfo &TII, bool ZeroAsNull=true)
LLT getRegType(SPIRVTypeInst SpvType) const
SPIRVTypeInst getOpTypeVoid(MachineIRBuilder &MIRBuilder)
void invalidateMachineInstr(MachineInstr *MI)
SPIRVTypeInst getOrCreateSPIRVBoolType(MachineIRBuilder &MIRBuilder, bool EmitIR)
bool isScalarOfType(Register VReg, unsigned TypeOpcode) const
Register getSPIRVTypeID(SPIRVTypeInst SpirvType) const
Register getOrCreateConstInt(uint64_t Val, MachineInstr &I, SPIRVTypeInst SpvType, const SPIRVInstrInfo &TII, bool ZeroAsNull=true)
Register getOrCreateConstIntArray(uint64_t Val, size_t Num, MachineInstr &I, SPIRVTypeInst SpvType, const SPIRVInstrInfo &TII)
bool findValueAttrs(const MachineInstr *Key, Type *&Ty, StringRef &Name)
SPIRVTypeInst retrieveScalarOrVectorIntType(SPIRVTypeInst Type) const
Register getOrCreateGlobalVariableWithBinding(SPIRVTypeInst VarType, uint32_t Set, uint32_t Binding, StringRef Name, MachineIRBuilder &MIRBuilder)
SPIRVTypeInst changePointerStorageClass(SPIRVTypeInst PtrType, SPIRV::StorageClass::StorageClass SC, MachineInstr &I)
Register getOrCreateConstVector(uint64_t Val, MachineInstr &I, SPIRVTypeInst SpvType, const SPIRVInstrInfo &TII, bool ZeroAsNull=true)
Register buildConstantFP(APFloat Val, MachineIRBuilder &MIRBuilder, SPIRVTypeInst SpvType=nullptr)
void addGlobalObject(const Value *V, const MachineFunction *MF, Register R)
SPIRVTypeInst getScalarOrVectorComponentType(SPIRVTypeInst Type) const
void recordFunctionPointer(const MachineOperand *MO, const Function *F)
SPIRVTypeInst getOrCreateSPIRVFloatType(unsigned BitWidth, MachineInstr &I, const SPIRVInstrInfo &TII)
SPIRVTypeInst getPointeeType(SPIRVTypeInst PtrType)
SPIRVTypeInst getOrCreateSPIRVType(const Type *Type, MachineInstr &I, SPIRV::AccessQualifier::AccessQualifier AQ, bool EmitIR)
bool isScalarOrVectorOfType(Register VReg, unsigned TypeOpcode) const
MachineFunction * setCurrentFunc(MachineFunction &MF)
Register getOrCreateConstNullPtr(MachineIRBuilder &MIRBuilder, SPIRVTypeInst SpvType)
SPIRVTypeInst getSPIRVTypeForVReg(Register VReg, const MachineFunction *MF=nullptr) const
Type * getDeducedGlobalValueType(const GlobalValue *Global)
Register getOrCreateUndef(MachineInstr &I, SPIRVTypeInst SpvType, const SPIRVInstrInfo &TII)
SPIRV::StorageClass::StorageClass getPointerStorageClass(Register VReg) const
SPIRVTypeInst getUntypedPtrElementType(Register Reg) const
bool erase(const MachineInstr *MI)
bool add(SPIRV::IRHandle Handle, const MachineInstr *MI)
Register find(SPIRV::IRHandle Handle, const MachineFunction *MF)
bool isPhysicalSPIRV() const
bool isAtLeastSPIRVVer(VersionTuple VerToCompareTo) const
bool canUseExtInstSet(SPIRV::InstructionSet::InstructionSet E) const
bool isLogicalSPIRV() const
bool canUseExtension(SPIRV::Extension::Extension E) const
bool isTypeIntOrFloat() const
bool isAnyTypeFloat() const
bool erase(PtrType Ptr)
Remove pointer from the set.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
bool contains(ConstPtrType Ptr) const
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
reference emplace_back(ArgTypes &&... Args)
void reserve(size_type N)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Represent a constant reference to a string, i.e.
constexpr size_t size() const
Get the string size.
static LLVM_ABI StructType * get(LLVMContext &Context, ArrayRef< Type * > Elements, bool isPacked=false)
This static method is the primary way to create a literal StructType.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
The instances of the Type class are immutable: once they are created, they are never changed.
@ HalfTyID
16-bit floating point type
@ FloatTyID
32-bit floating point type
@ BFloatTyID
16-bit floating point type (7-bit significand)
@ DoubleTyID
64-bit floating point type
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
bool isStructTy() const
True if this is an instance of StructType.
bool isAggregateType() const
Return true if the type is an aggregate type.
TypeID getTypeID() const
Return the type id for the type.
Value * getOperand(unsigned i) const
LLVM_ABI StringRef getName() const
Return a constant reference to the value's name.
An efficient, type-erasing, non-owning reference to a callable.
self_iterator getIterator()
NodeTy * getNextNode()
Get the next node, or nullptr for the list tail.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
constexpr char IsConst[]
Key for Kernel::Arg::Metadata::mIsConst.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
ElementType
The element type of an SRV or UAV resource.
constexpr uint64_t PointerSize
aarch64 pointer size.
Scope
Defines the scope in which this symbol should be visible: Default – Visible in the public interface o...
NodeAddr< DefNode * > Def
NodeAddr< InstrNode * > Instr
NodeAddr< UseNode * > Use
NodeAddr< FuncNode * > Func
BaseReg
Stack frame base register. Bit 0 of FREInfo.Info.
unsigned getOpcode(const VPValue *V)
Return the instruction opcode for the recipe defining V or 0 for unsupported recipes and VPValues not...
This is an optimization pass for GlobalISel generic memory operations.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
void addStringImm(StringRef Str, MCInst &Inst)
MachineBasicBlock::iterator getOpVariableMBBIt(MachineFunction &MF)
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
bool isTypeFoldingSupported(unsigned Opcode)
uint32_t getMemSemanticsWithStorageClass(const Triple &TT, uint32_t OrderSem, uint32_t StorageClassSem)
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
MachineInstr * getDef(const MachineOperand &MO, const MachineRegisterInfo *MRI)
void addNumImm(const APInt &Imm, MachineInstrBuilder &MIB)
LLVM_ABI void salvageDebugInfo(const MachineRegisterInfo &MRI, MachineInstr &MI)
Assuming the instruction MI is going to be deleted, attempt to salvage debug users of MI by writing t...
LLVM_ABI void constrainSelectedInstRegOperands(MachineInstr &I, const TargetInstrInfo &TII, const TargetRegisterInfo &TRI, const RegisterBankInfo &RBI)
Mutate the newly-selected instruction I to constrain its (possibly generic) virtual register operands...
bool isPreISelGenericOpcode(unsigned Opcode)
Check whether the given Opcode is a generic opcode that is not supposed to appear after ISel.
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
Register createVirtualRegister(SPIRVTypeInst SpvType, SPIRVGlobalRegistry *GR, MachineRegisterInfo *MRI, const MachineFunction &MF)
unsigned getArrayComponentCount(const MachineRegisterInfo *MRI, const MachineInstr *ResType)
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
void buildOpDecorate(Register Reg, MachineIRBuilder &MIRBuilder, SPIRV::Decoration::Decoration Dec, ArrayRef< uint32_t > DecArgs, StringRef StrImm)
LLVM_ABI bool isNullOrNullSplat(const MachineInstr &MI, const MachineRegisterInfo &MRI, bool AllowUndefs=false)
Return true if the value is a constant 0 integer or a splatted vector of a constant 0 integer (with n...
SPIRV::Scope::Scope getMemScope(const Triple &TT, LLVMContext &Ctx, SyncScope::ID Id)
uint64_t getIConstVal(Register ConstReg, const MachineRegisterInfo *MRI)
SmallVector< MachineInstr *, 4 > createContinuedInstructions(MachineIRBuilder &MIRBuilder, unsigned Opcode, unsigned MinWC, unsigned ContinuedOpcode, ArrayRef< Register > Args, Register ReturnRegister, Register TypeID)
SPIRV::MemorySemantics::MemorySemantics getMemSemanticsForStorageClass(SPIRV::StorageClass::StorageClass SC)
constexpr unsigned storageClassToAddressSpace(SPIRV::StorageClass::StorageClass SC)
bool isVectorType(SPIRVTypeInst SPVTy)
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Value
void buildOpName(Register Target, StringRef Name, MachineIRBuilder &MIRBuilder)
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
MachineInstr * getImm(const MachineOperand &MO, const MachineRegisterInfo *MRI)
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Type * toTypedPointer(Type *Ty)
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
LLVM_ABI void report_fatal_error(Error Err, bool gen_crash_diag=true)
constexpr bool isGenericCastablePtr(SPIRV::StorageClass::StorageClass SC)
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
MachineInstr * passCopy(MachineInstr *Def, const MachineRegisterInfo *MRI)
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
std::optional< SPIRV::LinkageType::LinkageType > getSpirvLinkageTypeFor(const SPIRVSubtarget &ST, const GlobalValue &GV)
LLVM_ABI raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
SPIRV::StorageClass::StorageClass addressSpaceToStorageClass(unsigned AddrSpace, const SPIRVSubtarget &STI)
AtomicOrdering
Atomic ordering for LLVM's memory model.
InstructionSelector * createSPIRVInstructionSelector(const SPIRVTargetMachine &TM, const SPIRVSubtarget &Subtarget, const RegisterBankInfo &RBI)
std::string getStringValueFromReg(Register Reg, MachineRegisterInfo &MRI)
int64_t foldImm(const MachineOperand &MO, const MachineRegisterInfo *MRI)
DWARFExpression::Operation Op
ArrayRef(const T &OneElt) -> ArrayRef< T >
MachineInstr * getDefInstrMaybeConstant(Register &ConstReg, const MachineRegisterInfo *MRI)
constexpr unsigned BitWidth
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
bool hasInitializer(const GlobalVariable *GV)
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Next
bool isSpvIntrinsic(const MachineInstr &MI, Intrinsic::ID IntrinsicID)
MachineInstr * getVRegDef(MachineRegisterInfo &MRI, Register Reg)
SPIRV::MemorySemantics::MemorySemantics getMemSemantics(AtomicOrdering Ord)
std::string getLinkStringForBuiltIn(SPIRV::BuiltIn::BuiltIn BuiltInValue)
LLVM_ABI bool isTriviallyDead(const MachineInstr &MI, const MachineRegisterInfo &MRI)
Check whether an instruction MI is dead: it only defines dead virtual registers, and doesn't have oth...
MCRegisterClass TargetRegisterClass