24#include "llvm/IR/IntrinsicsSPIRV.h"
33#define DEBUG_TYPE "spirv-legalizer"
36 return [IsExtendedInts, TypeIdx](
const LegalityQuery &Query) {
37 const LLT Ty = Query.Types[TypeIdx];
38 return IsExtendedInts && Ty.isValid() && Ty.isScalar();
44 const LLT Ty = Query.Types[TypeIdx];
45 return IsLongVecs && Ty.isValid() && Ty.isVector();
51 ST.canUseExtension(SPIRV::Extension::SPV_EXT_long_vector))
53 unsigned NumElements = Ty.getNumElements();
54 unsigned MaxVectorSize = ST.isShader() ? 4 : 16;
56 NumElements > MaxVectorSize;
102 const unsigned PSize = ST.getPointerSize();
119 auto allPtrsScalarsAndVectors = {
120 p0, p1, p2, p3, p4, p5, p6, p7, p8,
121 p9, p10, p11, p12, p13, s1, s8, s16, s32,
122 s64, s128, v2s1, v2s8, v2s16, v2s32, v2s64, v3s1, v3s8,
123 v3s16, v3s32, v3s64, v4s1, v4s8, v4s16, v4s32, v4s64, v8s1,
124 v8s8, v8s16, v8s32, v8s64, v16s1, v16s8, v16s16, v16s32, v16s64};
126 auto allVectors = {v2s1, v2s8, v2s16, v2s32, v2s64, v3s1, v3s8,
127 v3s16, v3s32, v3s64, v4s1, v4s8, v4s16, v4s32,
128 v4s64, v8s1, v8s8, v8s16, v8s32, v8s64, v16s1,
129 v16s8, v16s16, v16s32, v16s64};
131 auto allShaderVectors = {v2s1, v2s8, v2s16, v2s32, v2s64,
132 v3s1, v3s8, v3s16, v3s32, v3s64,
133 v4s1, v4s8, v4s16, v4s32, v4s64};
135 auto allScalars = {s1, s8, s16, s32, s64};
137 auto allScalarsAndVectors = {
138 s1, s8, s16, s32, s64, s128, v2s1, v2s8,
139 v2s16, v2s32, v2s64, v3s1, v3s8, v3s16, v3s32, v3s64,
140 v4s1, v4s8, v4s16, v4s32, v4s64, v8s1, v8s8, v8s16,
141 v8s32, v8s64, v16s1, v16s8, v16s16, v16s32, v16s64};
143 auto allShaderScalarsAndVectors = {
144 s1, s8, s16, s32, s64, s128, v2s1, v2s8, v2s16, v2s32, v2s64,
145 v3s1, v3s8, v3s16, v3s32, v3s64, v4s1, v4s8, v4s16, v4s32, v4s64};
147 auto &allowedScalarsAndVectors =
148 ST.isShader() ? allShaderScalarsAndVectors : allScalarsAndVectors;
150 auto allIntScalarsAndVectors = {
151 s8, s16, s32, s64, s128, v2s8, v2s16, v2s32, v2s64,
152 v3s8, v3s16, v3s32, v3s64, v4s8, v4s16, v4s32, v4s64, v8s8,
153 v8s16, v8s32, v8s64, v16s8, v16s16, v16s32, v16s64};
155 auto allBoolScalarsAndVectors = {s1, v2s1, v3s1, v4s1, v8s1, v16s1};
156 auto allBoolVectors = {v2s1, v3s1, v4s1, v8s1, v16s1};
158 auto allIntScalars = {s8, s16, s32, s64, s128};
160 auto allShaderIntVectors = {v2s8, v2s16, v2s32, v2s64, v3s8, v3s16,
161 v3s32, v3s64, v4s8, v4s16, v4s32, v4s64};
163 auto allIntVectors = {v2s8, v2s16, v2s32, v2s64, v3s8, v3s16, v3s32,
164 v3s64, v4s8, v4s16, v4s32, v4s64, v8s8, v8s16,
165 v8s32, v8s64, v16s8, v16s16, v16s32, v16s64};
167 auto &allowedIntVectorTypes =
168 ST.isShader() ? allShaderIntVectors : allIntVectors;
170 auto allFloatScalarsAndF16Vector2AndVector4s = {s16, s32, s64, v2s16, v4s16};
172 auto allFloatScalars = {s16, s32, s64};
174 auto allFloatScalarsAndVectors = {
175 s16, s32, s64, v2s16, v2s32, v2s64, v3s16, v3s32, v3s64,
176 v4s16, v4s32, v4s64, v8s16, v8s32, v8s64, v16s16, v16s32, v16s64};
178 auto allShaderFloatVectors = {v2s16, v2s32, v2s64, v3s16, v3s32,
179 v3s64, v4s16, v4s32, v4s64};
181 auto allFloatVectors = {v2s16, v2s32, v2s64, v3s16, v3s32,
182 v3s64, v4s16, v4s32, v4s64, v8s16,
183 v8s32, v8s64, v16s16, v16s32, v16s64};
185 auto &allowedFloatVectorTypes =
186 ST.isShader() ? allShaderFloatVectors : allFloatVectors;
188 auto allFloatAndIntScalarsAndPtrs = {s8, s16, s32, s64, p0, p1,
189 p2, p3, p4, p5, p6, p7,
190 p8, p9, p10, p11, p12, p13};
192 auto allPtrs = {p0, p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13};
194 auto &allowedVectorTypes = ST.isShader() ? allShaderVectors : allVectors;
196 bool HasArbitraryPrecisionInts = ST.canUseExtension(
197 SPIRV::Extension::SPV_ALTERA_arbitrary_precision_integers);
198 bool IsExtendedInts =
199 HasArbitraryPrecisionInts ||
200 ST.canUseExtension(SPIRV::Extension::SPV_KHR_bit_instructions) ||
201 ST.canUseExtension(SPIRV::Extension::SPV_INTEL_int4);
202 bool IsLongVecs = ST.canUseExtension(SPIRV::Extension::SPV_EXT_long_vector);
203 auto ExtendedIntScalarsAndVectors =
205 const LLT Ty = Query.Types[0];
206 return IsExtendedInts && Ty.isValid() &&
207 !Ty.isPointerOrPointerVector() && Ty.getScalarSizeInBits() > 1;
209 auto ExtendedScalarsAndVectorsProduct = [IsExtendedInts](
211 const LLT Ty1 = Query.Types[0], Ty2 = Query.Types[1];
212 return IsExtendedInts && Ty1.
isValid() && Ty2.isValid() &&
215 auto ExtendedPtrsScalarsAndVectors =
217 const LLT Ty = Query.Types[0];
218 return IsExtendedInts && Ty.isValid();
227 uint32_t MaxVectorSize = ST.isShader() ? 4 : 16;
232 case G_EXTRACT_VECTOR_ELT:
255 .customFor(allScalars)
274 .legalFor(allScalars)
351 {G_VECREDUCE_SMIN, G_VECREDUCE_SMAX, G_VECREDUCE_UMIN, G_VECREDUCE_UMAX,
352 G_VECREDUCE_ADD, G_VECREDUCE_MUL, G_VECREDUCE_FMUL, G_VECREDUCE_FMIN,
353 G_VECREDUCE_FMAX, G_VECREDUCE_FMINIMUM, G_VECREDUCE_FMAXIMUM,
354 G_VECREDUCE_OR, G_VECREDUCE_AND, G_VECREDUCE_XOR})
355 .legalFor(allowedVectorTypes)
376 .unsupportedIf(
typeIs(0, p9))
385 .unsupportedIf(
typeIs(1, p9))
393 G_BITREVERSE, G_SADDSAT, G_UADDSAT, G_SSUBSAT,
394 G_USUBSAT, G_SCMP, G_UCMP})
395 .legalFor(allIntScalars)
397 .
legalIf(ExtendedIntScalarsAndVectors)
409 .legalForCartesianProduct(allFloatScalarsAndVectors, allIntScalars);
412 .legalForCartesianProduct(allIntScalarsAndVectors,
413 allFloatScalarsAndVectors)
419 .legalForCartesianProduct(allIntScalarsAndVectors,
420 allFloatScalarsAndVectors);
423 .legalForCartesianProduct(allFloatScalarsAndVectors,
424 allScalarsAndVectors);
428 .
legalIf(ExtendedScalarsAndVectorsProduct)
432 .legalForCartesianProduct(allowedScalarsAndVectors)
433 .
legalIf(ExtendedScalarsAndVectorsProduct)
454 .
legalIf(ExtendedPtrsScalarsAndVectors)
456 return Query.
Types[0].isPointerVector();
458 .moreElementsToNextPow2(0);
462 typeInSet(1, allPtrsScalarsAndVectors)));
465 .legalFor({s1, s128})
466 .legalFor(allFloatAndIntScalarsAndPtrs)
469 return Query.
Types[0].isPointerVector();
487 !SrcTy.isPointer() &&
497 return SrcTy.isPointerVector() && DstTy.
isVector() &&
531 return IsExtendedInts && Ty.isValid() &&
532 !Ty.isPointerOrPointerVector() &&
545 typeInSet(1, allFloatScalarsAndVectors)))
549 G_ATOMICRMW_MAX, G_ATOMICRMW_MIN,
550 G_ATOMICRMW_SUB, G_ATOMICRMW_XOR,
551 G_ATOMICRMW_UMAX, G_ATOMICRMW_UMIN})
552 .legalForCartesianProduct(allIntScalars, allPtrs);
555 {G_ATOMICRMW_FADD, G_ATOMICRMW_FSUB, G_ATOMICRMW_FMIN, G_ATOMICRMW_FMAX})
556 .legalForCartesianProduct(allFloatScalarsAndF16Vector2AndVector4s,
575 {G_SMULFIX, G_UMULFIX, G_SMULFIXSAT, G_UMULFIXSAT});
576 if (!HasArbitraryPrecisionInts)
583 .legalForCartesianProduct(allIntScalarsAndVectors,
584 allFloatScalarsAndVectors);
588 .legalForCartesianProduct(allFloatScalarsAndVectors);
600 allFloatScalarsAndVectors, {s32, v2s32, v3s32, v4s32, v8s32, v16s32});
637 G_INTRINSIC_ROUNDEVEN})
638 .legalFor(allFloatScalars)
648 allFloatScalarsAndVectors)
655 allFloatScalarsAndVectors, allIntScalarsAndVectors);
657 if (ST.canUseExtInstSet(SPIRV::InstructionSet::OpenCL_std)) {
659 {G_CTTZ, G_CTTZ_ZERO_POISON, G_CTLZ, G_CTLZ_ZERO_POISON})
660 .legalForCartesianProduct(allIntScalarsAndVectors,
661 allIntScalarsAndVectors);
670 G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS})
675 verify(*ST.getInstrInfo());
689 MI.eraseFromParent();
705 MI.eraseFromParent();
713 Register ConvReg =
MRI.createGenericVirtualRegister(ConvTy);
728 LLT DstTy =
MRI.getType(DstReg);
741 LLT PtrTy =
MRI.getType(PtrReg);
744 for (
unsigned i = 0; i < NumElts; ++i) {
746 Register EltPtr =
MRI.createGenericVirtualRegister(PtrTy);
756 if (!
MI.memoperands_empty()) {
763 Register EltReg =
MRI.createGenericVirtualRegister(EltTy);
764 MIRBuilder.
buildLoad(EltReg, EltPtr, EltPtrInfo, EltAlign);
769 MI.eraseFromParent();
779 LLT ValTy =
MRI.getType(ValReg);
787 for (
unsigned i = 0; i < NumElts; ++i)
788 SplitRegs.
push_back(
MRI.createGenericVirtualRegister(EltTy));
792 LLT PtrTy =
MRI.getType(PtrReg);
795 for (
unsigned i = 0; i < NumElts; ++i) {
797 Register EltPtr =
MRI.createGenericVirtualRegister(PtrTy);
807 if (!
MI.memoperands_empty()) {
814 MIRBuilder.
buildStore(SplitRegs[i], EltPtr, EltPtrInfo, EltAlign);
817 MI.eraseFromParent();
825 switch (
MI.getOpcode()) {
829 case TargetOpcode::G_BITCAST:
830 return legalizeBitcast(Helper,
MI);
831 case TargetOpcode::G_EXTRACT_VECTOR_ELT:
833 case TargetOpcode::G_INSERT_VECTOR_ELT:
835 case TargetOpcode::G_INTRINSIC:
836 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS:
838 case TargetOpcode::G_IS_FPCLASS:
839 return legalizeIsFPClass(Helper,
MI, LocObserver);
840 case TargetOpcode::G_ICMP: {
841 auto &Op0 =
MI.getOperand(2);
842 auto &Op1 =
MI.getOperand(3);
847 if ((!ST->canDirectlyComparePointers() ||
849 MRI.getType(Reg0).isPointer() &&
MRI.getType(Reg1).isPointer()) {
852 ST->getPointerSize());
854 LLVMTy, Helper.
MIRBuilder, SPIRV::AccessQualifier::ReadWrite,
true);
860 case TargetOpcode::G_LOAD:
862 case TargetOpcode::G_STORE:
882 const Type *LLVMArrTy =
885 LLVMArrTy, MIRBuilder, SPIRV::AccessQualifier::ReadWrite,
true);
887 ArrSpvTy, MIRBuilder, SPIRV::StorageClass::Function);
889 Register StackReg = StackTemp.getReg(0);
905 LLT DstTy =
MRI.getType(DstReg);
906 LLT SrcTy =
MRI.getType(SrcReg);
914 MI.eraseFromParent();
926 LLT DstTy =
MRI.getType(DstReg);
931 LLT SrcTy =
MRI.getType(SrcReg);
936 if (IdxVal < SrcTy.getNumElements()) {
941 for (
unsigned I = 0,
E = SrcTy.getNumElements();
I <
E; ++
I) {
948 Regs[IdxVal] = ValReg;
950 MI.eraseFromParent();
961 MIRBuilder.
buildStore(SrcReg, StackTemp, PtrInfo, VecAlign);
964 LLT PtrTy =
MRI.getType(StackTemp.getReg(0));
965 Register EltPtr =
MRI.createGenericVirtualRegister(PtrTy);
970 .
addUse(StackTemp.getReg(0))
976 MIRBuilder.
buildStore(ValReg, EltPtr, EltPtrInfo, EltAlign);
978 MIRBuilder.
buildLoad(DstReg, StackTemp, PtrInfo, VecAlign);
979 MI.eraseFromParent();
992 LLT SrcTy =
MRI.getType(SrcReg);
1000 if (IdxVal < SrcTy.getNumElements()) {
1001 LLT DstTy =
MRI.getType(DstReg);
1004 for (
unsigned I = 0,
E = SrcTy.getNumElements();
I <
E; ++
I) {
1015 MI.eraseFromParent();
1026 MIRBuilder.
buildStore(SrcReg, StackTemp, PtrInfo, VecAlign);
1029 LLT PtrTy =
MRI.getType(StackTemp.getReg(0));
1030 Register EltPtr =
MRI.createGenericVirtualRegister(PtrTy);
1035 .
addUse(StackTemp.getReg(0))
1041 MIRBuilder.
buildLoad(DstReg, EltPtr, EltPtrInfo, EltAlign);
1043 MI.eraseFromParent();
1056 LLT DstTy =
MRI.getType(DstReg);
1062 if (
MI.getNumOperands() == 2) {
1072 for (
unsigned i = 2; i <
MI.getNumOperands(); ++i) {
1077 MI.eraseFromParent();
1089 unsigned NumElts = Ty.getNumElements();
1091 for (
unsigned I = 0;
I < NumElts; ++
I)
1100 if (Elts.
size() == 1)
1104 Elts.
size(), MIRBuilder,
true);
1122 LLT DstTy =
MRI.getType(DstReg);
1127 unsigned MaxVectorSize = ST.isShader() ? 4 : 16;
1128 unsigned PartSize = NumElts > MaxVectorSize ? MaxVectorSize : 4;
1132 if (!MO.isReg() || !
MRI.getType(MO.getReg()).isVector())
1134 auto [It, Inserted] = OpElts.
try_emplace(MO.getReg());
1143 unsigned Size = std::min(PartSize, NumElts -
Offset);
1145 Size == 1 ? DstEltSpvTy
1151 auto EltsIt = MO.isReg() ? OpElts.
find(MO.getReg()) : OpElts.
end();
1152 if (EltsIt == OpElts.
end()) {
1156 auto [It, Inserted] = PartRegs.
try_emplace(MO.getReg());
1173 MI.eraseFromParent();
1181 switch (IntrinsicID) {
1182 case Intrinsic::spv_bitcast:
1184 case Intrinsic::spv_insertelt:
1186 case Intrinsic::spv_extractelt:
1188 case Intrinsic::spv_const_composite:
1205 MI.eraseFromParent();
1212bool SPIRVLegalizerInfo::legalizeIsFPClass(
1215 auto [DstReg, DstTy, SrcReg, SrcTy] =
MI.getFirst2RegLLTs();
1219 auto &MF = MIRBuilder.
getMF();
1220 MachineRegisterInfo &
MRI = MF.getRegInfo();
1224 if (DstTy.isVector())
1226 SPIRVTypeInst SPIRVDstTy = GR->getOrCreateSPIRVType(
1227 LLVMDstTy, MIRBuilder, SPIRV::AccessQualifier::ReadWrite,
1230 unsigned BitSize = SrcTy.getScalarSizeInBits();
1235 if (SrcTy.isVector()) {
1236 IntTy =
LLT::vector(SrcTy.getElementCount(), IntTy);
1239 SPIRVTypeInst SPIRVIntTy = GR->getOrCreateSPIRVType(
1240 LLVMIntTy, MIRBuilder, SPIRV::AccessQualifier::ReadWrite,
1244 LLT DstTyCopy = DstTy;
1245 const auto assignSPIRVTy = [&](MachineInstrBuilder &&
MI) {
1250 assert((MITy == IntTy || MITy == DstTyCopy) &&
1251 "Unexpected LLT type while lowering G_IS_FPCLASS");
1252 SPIRVTypeInst SPVTy = MITy == IntTy ? SPIRVIntTy : SPIRVDstTy;
1253 GR->assignSPIRVTypeToVReg(SPVTy,
MI.getReg(0), MF);
1258 const auto buildSPIRVConstant = [&](LLT Ty,
auto &&
C) -> MachineInstrBuilder {
1262 assert((Ty == IntTy || Ty == DstTyCopy) &&
1263 "Unexpected LLT type while lowering constant for G_IS_FPCLASS");
1264 SPIRVTypeInst VecEltTy = GR->getOrCreateSPIRVType(
1265 (Ty == IntTy ? LLVMIntTy : LLVMDstTy)->getScalarType(), MIRBuilder,
1266 SPIRV::AccessQualifier::ReadWrite,
1268 GR->assignSPIRVTypeToVReg(VecEltTy, ScalarC.getReg(0), MF);
1273 MIRBuilder.
buildCopy(DstReg, buildSPIRVConstant(DstTy, 0));
1274 MI.eraseFromParent();
1278 MIRBuilder.
buildCopy(DstReg, buildSPIRVConstant(DstTy, 1));
1279 MI.eraseFromParent();
1289 GR->assignSPIRVTypeToVReg(SPIRVIntTy, ResVReg, Helper.
MIRBuilder.
getMF());
1290 auto AsInt = MIRBuilder.
buildInstr(SPIRV::OpBitcast)
1292 .
addUse(GR->getSPIRVTypeID(SPIRVIntTy))
1294 AsInt = assignSPIRVTy(std::move(AsInt));
1300 APInt ExpMask = Inf;
1306 auto SignBitC = buildSPIRVConstant(IntTy, SignBit);
1307 auto ValueMaskC = buildSPIRVConstant(IntTy, ValueMask);
1308 auto InfC = buildSPIRVConstant(IntTy, Inf);
1309 auto ExpMaskC = buildSPIRVConstant(IntTy, ExpMask);
1310 auto ZeroC = buildSPIRVConstant(IntTy, 0);
1312 auto Abs = assignSPIRVTy(MIRBuilder.
buildAnd(IntTy, AsInt, ValueMaskC));
1313 auto Sign = assignSPIRVTy(
1316 MachineInstrBuilder Res;
1318 const auto appendToRes = [&](MachineInstrBuilder &&ToAppend) {
1319 auto Appended = assignSPIRVTy(std::move(ToAppend));
1321 ? assignSPIRVTy(MIRBuilder.
buildOr(DstTyCopy, Res, Appended))
1335 Mask &= ~fcPosFinite;
1339 DstTy, Abs, ExpMaskC));
1341 Mask &= ~fcNegFinite;
1349 auto ExpBits = assignSPIRVTy(MIRBuilder.
buildAnd(IntTy, AsInt, ExpMaskC));
1352 Mask &= ~PartialCheck;
1361 else if (PartialCheck ==
fcZero)
1373 auto OneC = buildSPIRVConstant(IntTy, 1);
1374 auto VMinusOne = MIRBuilder.
buildSub(IntTy,
V, OneC);
1375 auto SubnormalRes = assignSPIRVTy(
1377 buildSPIRVConstant(IntTy, AllOneMantissa)));
1379 SubnormalRes = MIRBuilder.
buildAnd(DstTy, SubnormalRes,
Sign);
1380 appendToRes(std::move(SubnormalRes));
1387 else if (PartialCheck ==
fcInf)
1392 auto NegInfC = buildSPIRVConstant(IntTy, NegInf);
1399 auto InfWithQnanBitC =
1400 buildSPIRVConstant(IntTy, std::move(Inf) | QNaNBitMask);
1401 if (PartialCheck ==
fcNan) {
1405 }
else if (PartialCheck ==
fcQNan) {
1412 auto IsNan = assignSPIRVTy(
1414 auto IsNotQnan = assignSPIRVTy(MIRBuilder.
buildICmp(
1416 appendToRes(MIRBuilder.
buildAnd(DstTy, IsNan, IsNotQnan));
1423 APInt ExpLSB = ExpMask & ~(ExpMask.
shl(1));
1424 auto ExpMinusOne = assignSPIRVTy(
1425 MIRBuilder.
buildSub(IntTy, Abs, buildSPIRVConstant(IntTy, ExpLSB)));
1426 APInt MaxExpMinusOne = std::move(ExpMask) - ExpLSB;
1427 auto NormalRes = assignSPIRVTy(
1429 buildSPIRVConstant(IntTy, MaxExpMinusOne)));
1431 NormalRes = MIRBuilder.
buildAnd(DstTy, NormalRes,
Sign);
1433 auto PosSign = assignSPIRVTy(MIRBuilder.
buildXor(
1434 DstTy,
Sign, buildSPIRVConstant(DstTy, InversionMask)));
1435 NormalRes = MIRBuilder.
buildAnd(DstTy, NormalRes, PosSign);
1437 appendToRes(std::move(NormalRes));
1441 MI.eraseFromParent();
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static void scalarize(Instruction *I, SmallVectorImpl< Instruction * > &Worklist)
Declares convenience wrapper classes for interpreting MachineInstr instances as specific generic oper...
This file declares the MachineIRBuilder class.
Promote Memory to Register
SmallVector< MachineOperand, 4 > Cond
MachineRegisterInfo & MRI
SPIR-V address space definition.
static bool legalizeSpvInsertElt(LegalizerHelper &Helper, MachineInstr &MI, SPIRVGlobalRegistry *GR)
static bool needsVectorLegalization(const LLT &Ty, const SPIRVSubtarget &ST)
static SmallVector< Register, 16 > unmergeToScalars(Register Reg, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
static bool legalizeInsertVectorElt(LegalizerHelper &Helper, MachineInstr &MI)
static MachineInstrBuilder createStackTemporaryForVector(LegalizerHelper &Helper, SPIRVGlobalRegistry *GR, Register SrcReg, LLT SrcTy, MachinePointerInfo &PtrInfo, Align &VecAlign)
static Register convertPtrToInt(Register Reg, LLT ConvTy, SPIRVTypeInst SpvType, LegalizerHelper &Helper, MachineRegisterInfo &MRI, SPIRVGlobalRegistry *GR)
LegalityPredicate typeOfExtendedScalars(unsigned TypeIdx, bool IsExtendedInts)
static bool legalizeStore(LegalizerHelper &Helper, MachineInstr &MI, SPIRVGlobalRegistry *GR)
static Register buildVectorPart(ArrayRef< Register > Elts, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)
static bool legalizeExtractVectorElt(LegalizerHelper &Helper, MachineInstr &MI)
static bool legalizeSpvExtractElt(LegalizerHelper &Helper, MachineInstr &MI, SPIRVGlobalRegistry *GR)
LegalityPredicate typeOfLongVectors(unsigned TypeIdx, bool IsLongVecs)
static bool legalizeSpvBitcast(LegalizerHelper &Helper, MachineInstr &MI, SPIRVGlobalRegistry *GR)
static bool legalizeElementwiseIntrinsic(LegalizerHelper &Helper, GIntrinsic &MI, SPIRVGlobalRegistry *GR)
static bool legalizeSpvConstComposite(LegalizerHelper &Helper, MachineInstr &MI, SPIRVGlobalRegistry *GR)
static bool legalizeLoad(LegalizerHelper &Helper, MachineInstr &MI, SPIRVGlobalRegistry *GR)
APInt bitcastToAPInt() const
static APFloat getLargest(const fltSemantics &Sem, bool Negative=false)
Returns the largest finite number in the given semantics.
static APFloat getInf(const fltSemantics &Sem, bool Negative=false)
Factory for Positive and Negative Infinity.
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
static APInt getSignMask(unsigned BitWidth)
Get the SignMask for a specific bit width.
unsigned getActiveBits() const
Compute the number of active bits in the value.
static APInt getSignedMaxValue(unsigned numBits)
Gets maximum signed value of APInt for a specific bit width.
APInt shl(unsigned shiftAmt) const
Left-shift function.
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
Get the array size.
static LLVM_ABI ArrayType * get(Type *ElementType, uint64_t NumElements)
This static method is the primary way to construct an ArrayType.
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
@ ICMP_UGE
unsigned greater or equal
@ ICMP_UGT
unsigned greater than
@ ICMP_ULT
unsigned less than
iterator find(const_arg_type_t< KeyT > Val)
std::pair< iterator, bool > try_emplace(KeyT &&Key, Ts &&...Args)
static constexpr ElementCount getFixed(ScalarTy MinVal)
Represents a call to an intrinsic.
static LLVM_ABI IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
static constexpr LLT vector(ElementCount EC, unsigned ScalarSizeInBits)
Get a low-level vector of some number of elements and element width.
LLT getScalarType() const
constexpr bool isPointerVector() const
static constexpr LLT scalar(unsigned SizeInBits)
Get a low-level scalar or aggregate "bag of bits".
constexpr bool isValid() const
constexpr uint16_t getNumElements() const
Returns the number of elements in a vector LLT.
constexpr bool isVector() const
static constexpr LLT pointer(unsigned AddressSpace, unsigned SizeInBits)
Get a low-level pointer in the given address space.
constexpr bool isPointer() const
constexpr unsigned getAddressSpace() const
static constexpr LLT fixed_vector(unsigned NumElements, unsigned ScalarSizeInBits)
Get a low-level fixed-width vector of some number of elements and element width.
constexpr bool isPointerOrPointerVector() const
constexpr bool isFixedVector() const
Returns true if the LLT is a fixed vector.
constexpr TypeSize getSizeInBytes() const
Returns the total size of the type in bytes, i.e.
LLT getElementType() const
Returns the vector's element type. Only valid for vector types.
LegalizeRuleSet & minScalar(unsigned TypeIdx, const LLT Ty)
Ensure the scalar is at least as wide as Ty.
LegalizeRuleSet & legalFor(std::initializer_list< LLT > Types)
The instruction is legal when type index 0 is any type in the given list.
LegalizeRuleSet & fewerElementsIf(LegalityPredicate Predicate, LegalizeMutation Mutation)
Remove elements to reach the type selected by the mutation if the predicate is true.
LegalizeRuleSet & unsupportedFor(std::initializer_list< LLT > Types)
LegalizeRuleSet & moreElementsToNextPow2(unsigned TypeIdx)
Add more elements to the vector to reach the next power of two.
LegalizeRuleSet & lower()
The instruction is lowered.
LegalizeRuleSet & scalarizeIf(LegalityPredicate Predicate, unsigned TypeIdx)
LegalizeRuleSet & lowerIf(LegalityPredicate Predicate)
The instruction is lowered if predicate is true.
LegalizeRuleSet & custom()
Unconditionally custom lower.
LegalizeRuleSet & unsupportedIf(LegalityPredicate Predicate)
LegalizeRuleSet & widenScalarIf(LegalityPredicate Predicate, LegalizeMutation Mutation)
Widen the scalar to the one selected by the mutation if the predicate is true.
LegalizeRuleSet & alwaysLegal()
LegalizeRuleSet & customIf(LegalityPredicate Predicate)
LegalizeRuleSet & scalarize(unsigned TypeIdx)
LegalizeRuleSet & legalForCartesianProduct(std::initializer_list< LLT > Types)
The instruction is legal when type indexes 0 and 1 are both in the given list.
LegalizeRuleSet & legalIf(LegalityPredicate Predicate)
The instruction is legal if predicate is true.
LegalizeRuleSet & customFor(std::initializer_list< LLT > Types)
LLVM_ABI MachineInstrBuilder createStackTemporary(TypeSize Bytes, Align Alignment, MachinePointerInfo &PtrInfo)
Create a stack temporary based on the size in bytes and the alignment.
MachineIRBuilder & MIRBuilder
Expose MIRBuilder so clients can set their own RecordInsertInstruction functions.
LLVM_ABI Align getStackTemporaryAlignment(LLT Type, Align MinAlign=Align()) const
Return the alignment to use for a stack temporary object with the given type.
LegalizeRuleSet & getActionDefinitionsBuilder(unsigned Opcode)
Get the action definition builder for the given opcode.
Helper class to build MachineInstr.
LLVMContext & getContext() const
MachineInstrBuilder buildUnmerge(ArrayRef< LLT > Res, const SrcOp &Op)
Build and insert Res0, ... = G_UNMERGE_VALUES Op.
MachineInstrBuilder buildAnd(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1)
Build and insert Res = G_AND Op0, Op1.
MachineInstrBuilder buildICmp(CmpInst::Predicate Pred, const DstOp &Res, const SrcOp &Op0, const SrcOp &Op1, std::optional< unsigned > Flags=std::nullopt)
Build and insert a Res = G_ICMP Pred, Op0, Op1.
MachineInstrBuilder buildSub(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_SUB Op0, Op1.
MachineInstrBuilder buildIntrinsic(Intrinsic::ID ID, ArrayRef< Register > Res, bool HasSideEffects, bool isConvergent)
Build and insert a G_INTRINSIC instruction.
MachineInstrBuilder buildSplatBuildVector(const DstOp &Res, const SrcOp &Src)
Build and insert Res = G_BUILD_VECTOR with Src replicated to fill the number of elements.
MachineInstrBuilder buildBuildVector(const DstOp &Res, ArrayRef< Register > Ops)
Build and insert Res = G_BUILD_VECTOR Op0, ...
MachineInstrBuilder buildLoad(const DstOp &Res, const SrcOp &Addr, MachineMemOperand &MMO)
Build and insert Res = G_LOAD Addr, MMO.
MachineInstrBuilder buildStore(const SrcOp &Val, const SrcOp &Addr, MachineMemOperand &MMO)
Build and insert G_STORE Val, Addr, MMO.
MachineInstrBuilder buildInstr(unsigned Opcode)
Build and insert <empty> = Opcode <empty>.
MachineFunction & getMF()
Getter for the function we currently build.
MachineInstrBuilder buildBitcast(const DstOp &Dst, const SrcOp &Src)
Build and insert Dst = G_BITCAST Src.
MachineRegisterInfo * getMRI()
Getter for MRI.
MachineInstrBuilder buildOr(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_OR Op0, Op1.
MachineInstrBuilder buildCopy(const DstOp &Res, const SrcOp &Op)
Build and insert Res = COPY Op.
MachineInstrBuilder buildXor(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1)
Build and insert Res = G_XOR Op0, Op1.
virtual MachineInstrBuilder buildConstant(const DstOp &Res, const ConstantInt &Val)
Build and insert Res = G_CONSTANT Val.
const MachineInstrBuilder & addUse(Register RegNo, RegState Flags={}, unsigned SubReg=0) const
Add a virtual register use operand.
const MachineInstrBuilder & add(const MachineOperand &MO) const
const MachineInstrBuilder & addDef(Register RegNo, RegState Flags={}, unsigned SubReg=0) const
Add a virtual register definition operand.
MachineInstr * getInstr() const
If conversion operators fail, use this method to get the MachineInstr explicitly.
Representation of each machine instruction.
void setFlags(unsigned flags)
A description of a memory reference used in the backend.
const MachinePointerInfo & getPointerInfo() const
LLVM_ABI Align getAlign() const
Return the minimum known alignment in bytes of the actual memory reference.
MachineOperand class - Representation of each machine instruction operand.
Register getReg() const
getReg - Returns the register number.
static MachineOperand CreateReg(Register Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isEarlyClobber=false, unsigned SubReg=0, bool isDebug=false, bool isInternalRead=false, bool isRenamable=false)
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
LLT getType(Register Reg) const
Get the low-level type of Reg or LLT{} if Reg is not a generic (target independent) virtual register.
LLVM_ABI void setRegClass(Register Reg, const TargetRegisterClass *RC)
setRegClass - Set the register class of the specified virtual register.
LLVM_ABI Register createGenericVirtualRegister(LLT Ty, StringRef Name="")
Create and return a new generic virtual register with low-level type Ty.
Wrapper class representing virtual and physical registers.
void assignSPIRVTypeToVReg(SPIRVTypeInst Type, Register VReg, const MachineFunction &MF)
SPIRVTypeInst getOrCreateSPIRVPointerType(const Type *BaseType, MachineIRBuilder &MIRBuilder, SPIRV::StorageClass::StorageClass SC, bool ForceTyped=false)
const TargetRegisterClass * getRegClass(SPIRVTypeInst SpvType) const
SPIRVTypeInst getOrCreateSPIRVVectorType(SPIRVTypeInst BaseType, unsigned NumElements, MachineIRBuilder &MIRBuilder, bool EmitIR)
const Type * getTypeForSPIRVType(SPIRVTypeInst Ty) const
LLT getRegType(SPIRVTypeInst SpvType) const
SPIRVTypeInst getScalarOrVectorComponentType(SPIRVTypeInst Type) const
SPIRVTypeInst getOrCreateSPIRVType(const Type *Type, MachineInstr &I, SPIRV::AccessQualifier::AccessQualifier AQ, bool EmitIR)
SPIRVTypeInst getSPIRVTypeForVReg(Register VReg, const MachineFunction *MF=nullptr) const
SPIRVLegalizerInfo(const SPIRVSubtarget &ST)
bool legalizeCustom(LegalizerHelper &Helper, MachineInstr &MI, LostDebugLocObserver &LocObserver) const override
Called for instructions with the Custom LegalizationAction.
bool legalizeIntrinsic(LegalizerHelper &Helper, MachineInstr &MI) const override
SPIRVGlobalRegistry * getSPIRVGlobalRegistry() const
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
static constexpr TypeSize getFixed(ScalarTy ExactSize)
The instances of the Type class are immutable: once they are created, they are never changed.
static LLVM_ABI VectorType * get(Type *ElementType, ElementCount EC)
This static method is the primary way to construct an VectorType.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
LLVM_ABI bool isTriviallyScalarizable(ID id)
Returns true if the intrinsic is trivially scalarizable.
LLVM_ABI LegalityPredicate isScalar(unsigned TypeIdx)
True iff the specified type index is a scalar.
LLVM_ABI LegalityPredicate numElementsNotPow2(unsigned TypeIdx)
True iff the specified type index is a vector whose element count is not a power of 2.
LLVM_ABI LegalityPredicate isPointerVector(unsigned TypeIdx)
True iff the specified type index is a vector of pointers (with any address space).
LLVM_ABI LegalityPredicate vectorElementCountIsLessThanOrEqualTo(unsigned TypeIdx, unsigned Size)
True iff the specified type index is a vector with a number of elements that's less than or equal to ...
LLVM_ABI LegalityPredicate typeInSet(unsigned TypeIdx, std::initializer_list< LLT > TypesInit)
True iff the given type index is one of the specified types.
LLVM_ABI LegalityPredicate vectorElementCountIsGreaterThan(unsigned TypeIdx, unsigned Size)
True iff the specified type index is a vector with a number of elements that's greater than the given...
Predicate any(Predicate P0, Predicate P1)
True iff P0 or P1 are true.
LegalityPredicate typeIsNot(unsigned TypeIdx, LLT Type)
True iff the given type index is not the specified type.
Predicate all(Predicate P0, Predicate P1)
True iff P0 and P1 are true.
LLVM_ABI LegalityPredicate typeIs(unsigned TypeIdx, LLT TypesInit)
True iff the given type index is the specified type.
Predicate predNot(Predicate P)
True iff P is false.
LLVM_ABI LegalizeMutation changeElementCountTo(unsigned TypeIdx, unsigned FromTypeIdx)
Keep the same scalar or element type as TypeIdx, but take the number of elements from FromTypeIdx.
LLVM_ABI LegalizeMutation changeElementTo(unsigned TypeIdx, unsigned FromTypeIdx)
Keep the same scalar or element type as the given type index.
LLVM_ABI LegalizeMutation changeElementSizeTo(unsigned TypeIdx, unsigned FromTypeIdx)
Change the scalar size or element size to have the same scalar size as type index FromIndex.
Invariant opcodes: All instruction sets have these as their low opcodes.
This is an optimization pass for GlobalISel generic memory operations.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
LLVM_ABI const llvm::fltSemantics & getFltSemanticForLLT(LLT Ty)
Get the appropriate floating point arithmetic semantic based on the bit size of the given scalar LLT.
Register createVirtualRegister(SPIRVTypeInst SpvType, SPIRVGlobalRegistry *GR, MachineRegisterInfo *MRI, const MachineFunction &MF)
void append_range(Container &C, Range &&R)
Wrapper function to append range R to container C.
std::function< bool(const LegalityQuery &)> LegalityPredicate
MachineInstr * getImm(const MachineOperand &MO, const MachineRegisterInfo *MRI)
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
FPClassTest
Floating-point class tests, supported by 'is_fpclass' intrinsic.
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
const std::set< unsigned > & getTypeFoldingSupportedOpcodes()
int64_t foldImm(const MachineOperand &MO, const MachineRegisterInfo *MRI)
ArrayRef(const T &OneElt) -> ArrayRef< T >
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
This struct is a compact representation of a valid (non-zero power of two) alignment.
The LegalityQuery object bundles together all the information that's needed to decide whether a given...
This class contains a discriminated union of information about pointers in memory operands,...
MachinePointerInfo getWithOffset(int64_t O) const