37#include "llvm/IR/IntrinsicsAMDGPU.h"
38#include "llvm/IR/IntrinsicsR600.h"
41#define DEBUG_TYPE "amdgpu-legalinfo"
60 unsigned Bits = Ty.getSizeInBits();
70 const LLT Ty = Query.Types[TypeIdx];
76 return Ty.getNumElements() % 2 != 0 &&
77 EltSize > 1 && EltSize < 32 &&
78 Ty.getSizeInBits() % 32 != 0;
84 const LLT Ty = Query.Types[TypeIdx];
91 const LLT Ty = Query.Types[TypeIdx];
93 return EltTy.
getSizeInBits() == 16 && Ty.getNumElements() > 2;
99 const LLT Ty = Query.Types[TypeIdx];
101 return std::pair(TypeIdx,
108 const LLT Ty = Query.Types[TypeIdx];
110 unsigned Size = Ty.getSizeInBits();
111 unsigned Pieces = (
Size + 63) / 64;
112 unsigned NewNumElts = (Ty.getNumElements() + 1) / Pieces;
122 const LLT Ty = Query.Types[TypeIdx];
125 const int Size = Ty.getSizeInBits();
127 const int NextMul32 = (
Size + 31) / 32;
131 const int NewNumElts = (32 * NextMul32 + EltSize - 1) / EltSize;
139 unsigned MemSize = Query.MMODescrs[0].MemoryTy.getSizeInBits();
147 const LLT Ty = Query.Types[TypeIdx];
149 const unsigned EltSize = Ty.getElementType().getSizeInBits();
152 assert(EltSize == 32 || EltSize == 64);
157 for (NewNumElts = NumElts; NewNumElts < MaxNumElts; ++NewNumElts) {
161 return std::pair(TypeIdx,
176 const unsigned NumElems = Ty.getElementCount().getFixedValue();
181 const unsigned Size = Ty.getSizeInBits();
194 const LLT Ty = Query.Types[TypeIdx];
201 const LLT Ty = Query.Types[TypeIdx];
202 unsigned Size = Ty.getSizeInBits();
204 return std::pair(TypeIdx,
212 const LLT QueryTy = Query.Types[TypeIdx];
219 const LLT QueryTy = Query.Types[TypeIdx];
226 const LLT QueryTy = Query.Types[TypeIdx];
232 return ((ST.useRealTrue16Insts() &&
Size == 16) ||
Size % 32 == 0) &&
238 return EltSize == 16 || EltSize % 32 == 0;
242 const int EltSize = Ty.getElementType().getSizeInBits();
243 return EltSize == 32 || EltSize == 64 ||
244 (EltSize == 16 && Ty.getNumElements() % 2 == 0) ||
245 EltSize == 128 || EltSize == 256;
274 LLT Ty = Query.Types[TypeIdx];
282 const LLT QueryTy = Query.Types[TypeIdx];
370 if (Ty.isPointerOrPointerVector())
371 Ty = Ty.changeElementType(
LLT::scalar(Ty.getScalarSizeInBits()));
375 (ST.useRealTrue16Insts() && Ty ==
S16) ||
390 const LLT Ty = Query.Types[TypeIdx];
391 return !Ty.
isVector() && Ty.getSizeInBits() > 32 &&
392 Query.MMODescrs[0].MemoryTy.getSizeInBits() < Ty.getSizeInBits();
400 unsigned MemSize = Query.MMODescrs[0].MemoryTy.getSizeInBits();
410 bool IsLoad,
bool IsAtomic) {
414 return ST.hasFlatScratchEnabled() ? 128 : 32;
416 return ST.useDS128() ? 128 : 64;
427 return IsLoad ? 512 : 128;
432 return ST.hasMultiDwordFlatScratchAddressing() || IsAtomic ? 128 : 32;
441 const bool IsLoad = Query.
Opcode != AMDGPU::G_STORE;
443 unsigned RegSize = Ty.getSizeInBits();
446 unsigned AS = Query.
Types[1].getAddressSpace();
457 if (Ty.isVector() && MemSize !=
RegSize)
464 if (IsLoad && MemSize <
Size)
465 MemSize = std::max(MemSize,
Align);
472 if (IsLoad &&
RegSize == 16 && !ST.d16PreservesUnusedBits())
488 if (!ST.hasDwordx3LoadStores())
501 if (AlignBits < MemSize) {
503 if (!
TLI->allowsMisalignedMemoryAccessesImpl(MemSize, AS,
504 Align(AlignBits / 8)))
531 if (ST.getCLOpts().global_isel_new_legality)
534 const unsigned Size = Ty.getSizeInBits();
535 if (Ty.isPointerVector())
545 unsigned EltSize = Ty.getScalarSizeInBits();
546 return EltSize != 32 && EltSize != 64;
566 const unsigned Size = Ty.getSizeInBits();
567 if (
Size != MemSizeInBits)
568 return Size <= 32 && Ty.isVector();
574 return Ty.isVector() && (!MemTy.
isVector() || MemTy == Ty) &&
583 uint64_t AlignInBits,
unsigned AddrSpace,
593 if (SizeInBits == 96 && ST.hasDwordx3LoadStores())
604 if (AlignInBits < RoundedSize)
610 return TLI->allowsMisalignedMemoryAccessesImpl(
611 RoundedSize, AddrSpace,
Align(AlignInBits / 8),
623 Query.
Types[1].getAddressSpace(), Opcode);
646 Register VectorReg =
MRI.createGenericVirtualRegister(VectorTy);
647 std::array<Register, 4> VectorElems;
648 B.setInsertPt(
B.getMBB(), ++
B.getInsertPt());
651 B.buildExtractVectorElementConstant(I32, VectorReg,
I).getReg(0);
652 B.buildMergeValues(MO, VectorElems);
656 Register BitcastReg =
MRI.createGenericVirtualRegister(VectorTy);
657 B.setInsertPt(
B.getMBB(), ++
B.getInsertPt());
658 auto Scalar =
B.buildBitcast(ScalarTy, BitcastReg);
659 B.buildIntToPtr(MO, Scalar);
683 return B.buildBuildVector(VectorTy, PointerParts).getReg(0);
685 Register Scalar =
B.buildPtrToInt(ScalarTy, Pointer).getReg(0);
686 return B.buildBitcast(VectorTy, Scalar).getReg(0);
715 const LLT BufferStridedPtr =
718 const LLT CodePtr = FlatPtr;
720 const std::initializer_list<LLT> AddrSpaces64 = {
721 GlobalPtr, ConstantPtr, FlatPtr
724 const std::initializer_list<LLT> AddrSpaces32 = {
725 LocalPtr, PrivatePtr, Constant32Ptr, RegionPtr
728 const std::initializer_list<LLT> AddrSpaces128 = {RsrcPtr};
730 const std::initializer_list<LLT> FPTypesBase = {
F32,
F64};
731 const std::initializer_list<LLT> FPTypes16 = {
F32,
F64,
F16};
732 const std::initializer_list<LLT> FPTypesPK16 = {
F32,
F64,
F16,
V2F16};
761 if (ST.hasVOP3PInsts() && ST.hasAddNoCarryInsts() && ST.hasIntClamp()) {
763 if (ST.hasAnyPackedU64Ops()) {
766 .clampMaxNumElementsStrict(0,
S16, 2)
772 }
else if (ST.hasScalarAddSub64()) {
775 .clampMaxNumElementsStrict(0,
S16, 2)
783 .clampMaxNumElementsStrict(0,
S16, 2)
790 if (ST.hasScalarSMulU64()) {
793 .clampMaxNumElementsStrict(0,
S16, 2)
801 .clampMaxNumElementsStrict(0,
S16, 2)
811 .minScalarOrElt(0,
S16)
816 }
else if (ST.has16BitInsts()) {
850 .widenScalarToNextMultipleOf(0, 32)
860 if (ST.hasMad64_32())
865 if (ST.hasIntClamp()) {
888 {G_SDIV, G_UDIV, G_SREM, G_UREM, G_SDIVREM, G_UDIVREM})
898 if (ST.hasVOP3PInsts()) {
900 .clampMaxNumElements(0,
S8, 2)
921 {G_UADDO, G_USUBO, G_UADDE, G_SADDE, G_USUBE, G_SSUBE})
937 LocalPtr, ConstantPtr, PrivatePtr, FlatPtr })
959 .customFor({VGPRPtr});
984 auto &FCanonicalizeActions =
986 auto &StrictFPOpActions =
993 if (ST.has16BitInsts()) {
994 if (ST.hasVOP3PInsts()) {
996 FCanonicalizeActions.legalFor({
F16,
V2F16});
997 StrictFPOpActions.legalFor({
F16,
V2F16});
999 FPOpActions.legalFor({
F16});
1000 FCanonicalizeActions.legalFor({
F16});
1001 StrictFPOpActions.legalFor({
F16});
1004 TrigActions.customFor({
F16});
1005 FDIVActions.customFor({
F16});
1008 if (ST.hasBF16PackedInsts()) {
1012 .clampMaxNumElementsStrict(0,
BF16, 2);
1015 .clampMaxNumElementsStrict(0,
BF16, 2);
1018 .clampMaxNumElementsStrict(0,
BF16, 2);
1024 if (ST.hasAnyPackedFP32Ops()) {
1025 FPOpActions.legalFor({
V2F32});
1026 FCanonicalizeActions.legalFor({
V2F32});
1027 StrictFPOpActions.legalFor({
V2F32});
1028 FPOpActions.clampMaxNumElementsStrict(0,
F32, 2);
1029 FCanonicalizeActions.clampMaxNumElementsStrict(0,
F32, 2);
1030 StrictFPOpActions.clampMaxNumElementsStrict(0,
F32, 2);
1033 if (ST.hasAnyPackedFP64Ops()) {
1034 FPOpActions.legalFor({
V2F64});
1035 FCanonicalizeActions.legalFor({
V2F64});
1036 StrictFPOpActions.legalFor({
V2F64});
1037 FPOpActions.clampMaxNumElementsStrict(0,
F64, 2);
1038 FCanonicalizeActions.clampMaxNumElementsStrict(0,
F64, 2);
1039 StrictFPOpActions.clampMaxNumElementsStrict(0,
F64, 2);
1042 auto &MinNumMaxNumIeee =
1045 {G_FMINNUM, G_FMAXNUM, G_FMINIMUMNUM, G_FMAXIMUMNUM});
1047 MinNumMaxNumIeee.
legalFor(!ST.hasIEEEMinimumMaximumInsts(), {F32, F64});
1048 MinNumMaxNum.
legalFor(ST.hasIEEEMinimumMaximumInsts(), {F32, F64})
1051 if (ST.has16BitInsts()) {
1052 MinNumMaxNumIeee.legalFor(!ST.hasIEEEMinimumMaximumInsts(), {F16});
1053 MinNumMaxNum.legalFor(ST.hasIEEEMinimumMaximumInsts(), {F16})
1058 if (ST.hasVOP3PInsts()) {
1059 MinNumMaxNumIeee.legalFor(!ST.hasIEEEMinimumMaximumInsts(), {V2F16})
1062 .clampMaxNumElements(0,
F16, 2);
1063 MinNumMaxNum.legalFor(ST.hasIEEEMinimumMaximumInsts(), {V2F16})
1067 .clampMaxNumElements(0,
F16, 2);
1071 if (ST.hasAnyPackedFP64Ops()) {
1072 MinNumMaxNum.legalFor(ST.hasIEEEMinimumMaximumInsts(), {V2F64})
1076 .clampMaxNumElements(0,
F64, 2);
1080 if (ST.hasBF16PackedInsts()) {
1081 MinNumMaxNumIeee.legalFor(!ST.hasIEEEMinimumMaximumInsts(), {V2BF16})
1082 .clampMaxNumElementsStrict(0,
BF16, 2);
1084 MinNumMaxNum.legalFor(ST.hasIEEEMinimumMaximumInsts(), {V2BF16})
1087 .clampMaxNumElementsStrict(0,
BF16, 2);
1095 if (!ST.has16BitInsts()) {
1096 MinNumMaxNumIeee.minScalar(0,
F32);
1097 MinNumMaxNum.minScalar(0,
F32);
1100 if (ST.hasVOP3PInsts()) {
1101 FPOpActions.clampMaxNumElementsStrict(0,
F16, 2);
1102 FCanonicalizeActions.clampMaxNumElementsStrict(0,
F16, 2);
1103 StrictFPOpActions.clampMaxNumElementsStrict(0,
F16, 2);
1111 if (!ST.has16BitInsts()) {
1122 .legalFor(ST.hasAnyPackedFP32Ops(), {V2F32})
1125 if (ST.hasAnyPackedFP32Ops())
1129 if (ST.has16BitInsts()) {
1132 .legalFor(ST.hasBF16TransInsts(), {BF16})
1142 .legalFor({{
F32, I32}, {
F64, I32}, {
F16, I16}})
1164 if (ST.hasFractBug()) {
1178 .legalFor({{
F32, I32}, {
F64, I32}})
1198 if (ST.hasCvtPkF16F32Inst()) {
1200 .clampMaxNumElements(0,
F16, 2);
1213 if (ST.has16BitInsts()) {
1227 if (ST.hasBF16PackedInsts()) {
1230 .clampMaxNumElementsStrict(0,
BF16, 2);
1235 if (ST.hasAnyPackedFP32Ops())
1243 if (ST.hasMadF16() && ST.hasMadMacF32Insts())
1244 FMad.customFor({
F32,
F16});
1245 else if (ST.hasMadMacF32Insts())
1246 FMad.customFor({
F32});
1247 else if (ST.hasMadF16())
1248 FMad.customFor({
F16});
1253 if (ST.has16BitInsts()) {
1256 FRem.minScalar(0,
F32).customFor({
F32,
F64});
1264 .clampMaxNumElements(0,
S16, 2)
1280 .legalFor({{
F32, I32}, {
F64, I32}})
1284 if (ST.has16BitInsts())
1292 .legalFor({{I32,
F32}, {I32,
F64}})
1293 .customFor({{I64,
F32}, {I64,
F64}})
1296 .widenScalarFor({{I16,
BF16}, {I32,
BF16}, {I64,
BF16}},
1298 if (ST.has16BitInsts())
1307 .legalFor({{I32,
F32}, {I32,
F64}, {I16,
F32}})
1308 .legalFor(ST.has16BitInsts(), {{I16, F16}})
1309 .legalFor(ST.hasVCvtPkIU16F32(), {{V2I16, V2F32}})
1313 if (
ST.has16BitInsts())
1316 if (
ST.hasVCvtPkIU16F32())
1326 getActionDefinitionsBuilder({G_LROUND, G_LLROUND})
1327 .clampScalar(0, I16, I64)
1331 getActionDefinitionsBuilder(G_INTRINSIC_FPTRUNC_ROUND)
1337 getActionDefinitionsBuilder({G_INTRINSIC_ROUND, G_FRINT, G_FNEARBYINT})
1341 getActionDefinitionsBuilder({G_INTRINSIC_LRINT, G_INTRINSIC_LLRINT})
1342 .clampScalar(0, I16, I64)
1346 auto &RoundingActions = getActionDefinitionsBuilder(
1347 {G_INTRINSIC_TRUNC, G_FCEIL, G_INTRINSIC_ROUNDEVEN});
1348 if (
ST.has16BitInsts())
1356 if (!
ST.has16BitInsts())
1359 getActionDefinitionsBuilder(G_PTR_ADD)
1365 getActionDefinitionsBuilder(G_PTRMASK)
1367 .scalarSameSizeAs(1, 0)
1371 getActionDefinitionsBuilder(G_ICMP)
1383 {
S1}, {
S32,
S64, GlobalPtr, LocalPtr, ConstantPtr, PrivatePtr, FlatPtr})
1384 .legalForCartesianProduct(
1385 {
S32}, {
S32,
S64, GlobalPtr, LocalPtr, ConstantPtr, PrivatePtr, FlatPtr});
1386 if (
ST.has16BitInsts()) {
1387 CmpBuilder.legalFor({{
S1,
S16}});
1396 getActionDefinitionsBuilder({G_SCMP, G_UCMP}).
lower();
1399 getActionDefinitionsBuilder(G_FCMP).legalForCartesianProduct(
1400 {
I1},
ST.has16BitInsts() ? FPTypes16 : FPTypesBase);
1402 if (
ST.hasSALUFloatInsts())
1403 FCmpBuilder.legalForCartesianProduct({
I32}, {
F16,
F32});
1406 .widenScalarToNextPow2(1)
1410 getActionDefinitionsBuilder(G_FPOW)
1412 .clampScalar(0,
F32,
F32)
1415 getActionDefinitionsBuilder(G_FPOWI)
1416 .clampScalar(0,
F32,
F32)
1420 getActionDefinitionsBuilder(G_FLOG2)
1421 .legalFor(
ST.has16BitInsts(), {F16})
1422 .legalFor(
ST.hasBF16TransInsts(), {BF16})
1428 getActionDefinitionsBuilder(G_FEXP2)
1429 .legalFor(
ST.has16BitInsts(), {F16})
1430 .legalFor(
ST.hasBF16TransInsts(), {BF16})
1436 getActionDefinitionsBuilder({G_FLOG, G_FLOG10})
1440 getActionDefinitionsBuilder({G_FEXP, G_FEXP10})
1445 getActionDefinitionsBuilder(G_CTPOP)
1447 .clampScalar(0,
S32,
S32)
1448 .widenScalarToNextPow2(1, 32)
1449 .clampScalar(1,
S32,
S64)
1451 .widenScalarToNextPow2(0, 32);
1454 if (
ST.has16BitInsts())
1455 getActionDefinitionsBuilder(G_IS_FPCLASS)
1456 .legalForCartesianProduct({
I1}, FPTypes16)
1457 .widenScalarToNextPow2(1)
1461 getActionDefinitionsBuilder(G_IS_FPCLASS)
1462 .legalForCartesianProduct({
I1}, FPTypesBase)
1463 .lowerFor({
I1,
F16})
1464 .widenScalarToNextPow2(1)
1471 getActionDefinitionsBuilder({G_CTLZ, G_CTTZ})
1473 .clampScalar(0,
S32,
S32)
1474 .clampScalar(1,
S32,
S64)
1475 .widenScalarToNextPow2(0, 32)
1476 .widenScalarToNextPow2(1, 32)
1480 getActionDefinitionsBuilder(G_CTLZ_ZERO_POISON)
1483 .clampScalar(0,
S32,
S32)
1484 .clampScalar(1,
S32,
S64)
1486 .widenScalarToNextPow2(0, 32)
1487 .widenScalarToNextPow2(1, 32);
1489 getActionDefinitionsBuilder(G_CTTZ_ZERO_POISON)
1491 .clampScalar(0,
S32,
S32)
1492 .clampScalar(1,
S32,
S64)
1494 .widenScalarToNextPow2(0, 32)
1495 .widenScalarToNextPow2(1, 32);
1497 getActionDefinitionsBuilder(G_CTLS)
1500 .clampScalar(0,
S32,
S32)
1501 .clampScalar(1,
S32,
S32);
1505 getActionDefinitionsBuilder(G_BITREVERSE)
1507 .clampScalar(0,
S32,
S64)
1509 .widenScalarToNextPow2(0);
1511 if (
ST.has16BitInsts()) {
1512 getActionDefinitionsBuilder(G_BSWAP)
1514 .clampMaxNumElementsStrict(0,
S16, 2)
1517 .widenScalarToNextPow2(0)
1518 .clampScalar(0,
S16,
S32)
1521 if (
ST.hasVOP3PInsts()) {
1522 getActionDefinitionsBuilder(G_ABS)
1524 .clampMaxNumElements(0,
S16, 2)
1526 .widenScalarToNextPow2(0)
1529 if (
ST.useMinMaxI64Insts()) {
1530 getActionDefinitionsBuilder({G_SMIN, G_SMAX, G_UMIN, G_UMAX})
1532 .clampMaxNumElements(0,
S16, 2)
1534 .widenScalarToNextPow2(0)
1538 getActionDefinitionsBuilder({G_SMIN, G_SMAX, G_UMIN, G_UMAX})
1540 .clampMaxNumElements(0,
S16, 2)
1542 .widenScalarToNextPow2(0)
1547 getActionDefinitionsBuilder({G_SMIN, G_SMAX, G_UMIN, G_UMAX, G_ABS})
1549 .widenScalarToNextPow2(0)
1556 getActionDefinitionsBuilder(G_BSWAP)
1561 .widenScalarToNextPow2(0)
1566 getActionDefinitionsBuilder({G_SMIN, G_SMAX, G_UMIN, G_UMAX, G_ABS})
1569 .widenScalarToNextPow2(0)
1574 getActionDefinitionsBuilder(G_INTTOPTR)
1576 .legalForCartesianProduct(AddrSpaces64, {
S64})
1577 .legalForCartesianProduct(AddrSpaces32, {
S32})
1590 getActionDefinitionsBuilder(G_PTRTOINT)
1592 .legalForCartesianProduct(AddrSpaces64, {
S64})
1593 .legalForCartesianProduct(AddrSpaces32, {
S32})
1606 getActionDefinitionsBuilder(G_ADDRSPACE_CAST)
1610 const auto needToSplitMemOp = [=](
const LegalityQuery &Query,
1611 bool IsLoad) ->
bool {
1615 unsigned MemSize = Query.
MMODescrs[0].MemoryTy.getSizeInBits();
1628 !
ST.hasUsableDSOffset() && Query.
MMODescrs[0].AlignInBits == 32)
1633 unsigned NumRegs = (MemSize + 31) / 32;
1635 if (!
ST.hasDwordx3LoadStores())
1646 unsigned GlobalAlign32 =
ST.hasUnalignedBufferAccessEnabled() ? 0 : 32;
1647 unsigned GlobalAlign16 =
ST.hasUnalignedBufferAccessEnabled() ? 0 : 16;
1648 unsigned GlobalAlign8 =
ST.hasUnalignedBufferAccessEnabled() ? 0 : 8;
1649 unsigned LocalAlign64 =
ST.hasUsableDSOffset() ? 32 : 64;
1655 for (
unsigned Op : {G_LOAD, G_STORE}) {
1656 const bool IsStore =
Op == G_STORE;
1658 auto &Actions = getActionDefinitionsBuilder(
Op);
1661 Actions.legalForTypesWithMemDesc(
1662 {{
S32, GlobalPtr,
S32, GlobalAlign32},
1665 {
S64, GlobalPtr,
S64, GlobalAlign32},
1668 {
S32, GlobalPtr,
S8, GlobalAlign8},
1669 {
S32, GlobalPtr,
S16, GlobalAlign16},
1671 {
S32, LocalPtr,
S32, 32},
1672 {
S64, LocalPtr,
S64, LocalAlign64},
1674 {
S32, LocalPtr,
S8, 8},
1675 {
S32, LocalPtr,
S16, 16},
1678 {
S32, PrivatePtr,
S32, 32},
1679 {
S32, PrivatePtr,
S8, 8},
1680 {
S32, PrivatePtr,
S16, 16},
1683 {
S32, ConstantPtr,
S32, GlobalAlign32},
1686 {
S64, ConstantPtr,
S64, GlobalAlign32}});
1688 if (IsStore ||
ST.d16PreservesUnusedBits()) {
1689 Actions.legalForTypesWithMemDesc(
ST.useRealTrue16Insts(),
1690 {{S16, GlobalPtr, S8, GlobalAlign8},
1691 {S16, GlobalPtr, S16, GlobalAlign16},
1692 {S16, LocalPtr, S8, 8},
1693 {S16, LocalPtr, S16, 16},
1694 {S16, PrivatePtr, S8, 8},
1695 {S16, PrivatePtr, S16, 16}});
1706 Actions.unsupportedIf(
1707 typeInSet(1, {BufferFatPtr, BufferStridedPtr, RsrcPtr}));
1723 Actions.customIf(
typeInSet(1, {Constant32Ptr, VGPRPtr}));
1749 return !Query.
Types[0].isVector() &&
1750 needToSplitMemOp(Query,
Op == G_LOAD);
1752 [=](
const LegalityQuery &Query) -> std::pair<unsigned, LLT> {
1757 unsigned MemSize = Query.
MMODescrs[0].MemoryTy.getSizeInBits();
1760 if (DstSize > MemSize)
1766 if (MemSize > MaxSize)
1774 return Query.
Types[0].isVector() &&
1775 needToSplitMemOp(Query,
Op == G_LOAD);
1777 [=](
const LegalityQuery &Query) -> std::pair<unsigned, LLT> {
1791 unsigned MemSize = Query.
MMODescrs[0].MemoryTy.getSizeInBits();
1792 if (MemSize > MaxSize) {
1796 if (MaxSize % EltSize == 0) {
1802 unsigned NumPieces = MemSize / MaxSize;
1806 if (NumPieces == 1 || NumPieces >= NumElts ||
1807 NumElts % NumPieces != 0)
1808 return std::pair(0, EltTy);
1816 return std::pair(0, EltTy);
1831 return std::pair(0, EltTy);
1836 .widenScalarToNextPow2(0)
1843 getActionDefinitionsBuilder({G_SEXTLOAD, G_ZEXTLOAD})
1844 .legalForTypesWithMemDesc({{
S32, GlobalPtr,
S8, 8},
1845 {
S32, GlobalPtr,
S16, 2 * 8},
1846 {
S32, LocalPtr,
S8, 8},
1847 {
S32, LocalPtr,
S16, 16},
1848 {
S32, PrivatePtr,
S8, 8},
1849 {
S32, PrivatePtr,
S16, 16},
1850 {
S32, ConstantPtr,
S8, 8},
1851 {
S32, ConstantPtr,
S16, 2 * 8}})
1852 .legalForTypesWithMemDesc(
ST.useRealTrue16Insts(),
1853 {{S16, GlobalPtr, S8, GlobalAlign8},
1854 {S16, LocalPtr, S8, GlobalAlign8},
1855 {S16, PrivatePtr, S8, GlobalAlign8},
1856 {S16, ConstantPtr, S8, GlobalAlign8}})
1861 if (
ST.hasFlatAddressSpace()) {
1862 ExtLoads.legalForTypesWithMemDesc(
1863 {{
S32, FlatPtr,
S8, 8}, {
S32, FlatPtr,
S16, 16}});
1865 ExtLoads.legalForTypesWithMemDesc(
ST.useRealTrue16Insts(),
1866 {{S16, FlatPtr, S8, GlobalAlign8}});
1876 ExtLoads.customIf(
typeInSet(1, {Constant32Ptr, VGPRPtr}));
1878 ExtLoads.narrowScalarIf(
1885 ExtLoads.clampScalar(0,
S32,
S32)
1886 .widenScalarToNextPow2(0)
1889 auto &Atomics = getActionDefinitionsBuilder(
1890 {G_ATOMICRMW_XCHG, G_ATOMICRMW_ADD, G_ATOMICRMW_SUB,
1891 G_ATOMICRMW_AND, G_ATOMICRMW_OR, G_ATOMICRMW_XOR,
1892 G_ATOMICRMW_MAX, G_ATOMICRMW_MIN, G_ATOMICRMW_UMAX,
1893 G_ATOMICRMW_UMIN, G_ATOMICRMW_UINC_WRAP, G_ATOMICRMW_UDEC_WRAP})
1894 .legalFor({{
S32, GlobalPtr}, {
S32, LocalPtr},
1895 {
S64, GlobalPtr}, {
S64, LocalPtr},
1896 {
S32, RegionPtr}, {
S64, RegionPtr}});
1897 if (
ST.hasFlatAddressSpace()) {
1898 Atomics.legalFor({{
S32, FlatPtr}, {
S64, FlatPtr}});
1902 getActionDefinitionsBuilder({G_ATOMICRMW_USUB_COND, G_ATOMICRMW_USUB_SAT})
1903 .legalFor({{
S32, GlobalPtr}, {
S32, LocalPtr}, {
S32, RegionPtr}});
1904 if (
ST.hasFlatAddressSpace()) {
1905 Atomics32.legalFor({{
S32, FlatPtr}});
1909 auto &
Atomic = getActionDefinitionsBuilder(G_ATOMICRMW_FADD);
1910 if (
ST.hasLDSFPAtomicAddF32()) {
1912 if (
ST.hasLdsAtomicAddF64())
1914 if (
ST.hasAtomicDsPkAdd16Insts())
1917 if (
ST.hasAtomicFaddInsts())
1919 if (
ST.hasFlatAtomicFaddF32Inst())
1922 if (
ST.hasGFX90AInsts() ||
ST.hasGFX1250Insts()) {
1926 Atomic.legalFor({{
F32, GlobalPtr}, {
F64, GlobalPtr}, {
F64, FlatPtr}});
1929 if (
ST.hasAtomicBufferGlobalPkAddF16NoRtnInsts() ||
1930 ST.hasAtomicBufferGlobalPkAddF16Insts())
1932 if (
ST.hasAtomicGlobalPkAddBF16Inst())
1934 if (
ST.hasAtomicFlatPkAdd16Insts())
1940 auto &AtomicFMinFMax =
1941 getActionDefinitionsBuilder({G_ATOMICRMW_FMIN, G_ATOMICRMW_FMAX})
1942 .legalFor({{
F32, LocalPtr}, {
F64, LocalPtr}});
1944 if (
ST.hasAtomicFMinFMaxF32GlobalInsts())
1945 AtomicFMinFMax.legalFor({{
F32, GlobalPtr},{
F32, BufferFatPtr}});
1946 if (
ST.hasAtomicFMinFMaxF64GlobalInsts())
1947 AtomicFMinFMax.legalFor({{
F64, GlobalPtr}, {
F64, BufferFatPtr}});
1948 if (
ST.hasAtomicFMinFMaxF32FlatInsts())
1949 AtomicFMinFMax.legalFor({
F32, FlatPtr});
1950 if (
ST.hasAtomicFMinFMaxF64FlatInsts())
1951 AtomicFMinFMax.legalFor({
F64, FlatPtr});
1955 getActionDefinitionsBuilder(G_ATOMIC_CMPXCHG)
1956 .customFor({{
S32, GlobalPtr}, {
S64, GlobalPtr},
1957 {
S32, FlatPtr}, {
S64, FlatPtr}})
1958 .legalFor({{
S32, LocalPtr}, {
S64, LocalPtr},
1959 {
S32, RegionPtr}, {
S64, RegionPtr}});
1963 getActionDefinitionsBuilder(G_SELECT)
1965 LocalPtr, FlatPtr, PrivatePtr,
1969 .clampScalar(0,
S16,
S64)
1973 .clampMaxNumElements(0,
S32, 2)
1974 .clampMaxNumElements(0, LocalPtr, 2)
1975 .clampMaxNumElements(0, PrivatePtr, 2)
1977 .widenScalarToNextPow2(0)
1982 auto &Shifts = getActionDefinitionsBuilder({G_SHL, G_LSHR, G_ASHR})
1984 if (
ST.has16BitInsts()) {
1985 if (
ST.hasVOP3PInsts()) {
1987 .clampMaxNumElements(0,
S16, 2);
1989 Shifts.legalFor({{
S16,
S16}});
1992 Shifts.widenScalarIf(
1997 const LLT AmountTy = Query.
Types[1];
2003 Shifts.clampScalar(1,
S32,
S32);
2004 Shifts.widenScalarToNextPow2(0, 16);
2005 Shifts.clampScalar(0,
S16,
S64);
2007 getActionDefinitionsBuilder({G_SSHLSAT, G_USHLSAT})
2015 Shifts.clampScalar(1,
S32,
S32);
2016 Shifts.widenScalarToNextPow2(0, 32);
2017 Shifts.clampScalar(0,
S32,
S64);
2019 getActionDefinitionsBuilder({G_SSHLSAT, G_USHLSAT})
2024 Shifts.scalarize(0);
2026 for (
unsigned Op : {G_EXTRACT_VECTOR_ELT, G_INSERT_VECTOR_ELT}) {
2027 unsigned VecTypeIdx =
Op == G_EXTRACT_VECTOR_ELT ? 1 : 0;
2028 unsigned EltTypeIdx =
Op == G_EXTRACT_VECTOR_ELT ? 0 : 1;
2029 unsigned IdxTypeIdx = 2;
2031 getActionDefinitionsBuilder(
Op)
2033 const LLT EltTy = Query.
Types[EltTypeIdx];
2034 const LLT VecTy = Query.
Types[VecTypeIdx];
2035 const LLT IdxTy = Query.
Types[IdxTypeIdx];
2037 const bool isLegalVecType =
2047 return (EltSize == 32 || EltSize == 64) &&
2063 const LLT EltTy = Query.
Types[EltTypeIdx];
2064 const LLT VecTy = Query.
Types[VecTypeIdx];
2068 const unsigned TargetEltSize =
2069 DstEltSize % 64 == 0 ? 64 : 32;
2070 return std::pair(VecTypeIdx,
2074 .clampScalar(EltTypeIdx,
S32,
S64)
2075 .clampScalar(VecTypeIdx,
S32,
S64)
2076 .clampScalar(IdxTypeIdx,
S32,
S32)
2077 .clampMaxNumElements(VecTypeIdx,
S32, 32)
2086 getActionDefinitionsBuilder(G_EXTRACT_VECTOR_ELT)
2088 const LLT &EltTy = Query.
Types[1].getElementType();
2089 return Query.
Types[0] != EltTy;
2092 for (
unsigned Op : {G_EXTRACT, G_INSERT}) {
2093 unsigned BigTyIdx =
Op == G_EXTRACT ? 1 : 0;
2094 unsigned LitTyIdx =
Op == G_EXTRACT ? 0 : 1;
2095 getActionDefinitionsBuilder(
Op)
2098 const LLT BigTy = Query.
Types[BigTyIdx];
2104 const LLT LitTy = Query.
Types[LitTyIdx];
2109 .widenScalarToNextPow2(BigTyIdx, 32)
2117 const LLT BigTy = Query.
Types[BigTyIdx];
2118 const LLT LitTy = Query.
Types[LitTyIdx];
2126 getActionDefinitionsBuilder(G_BUILD_VECTOR)
2136 if (
ST.hasScalarPackInsts()) {
2139 .minScalarOrElt(0,
S16)
2142 getActionDefinitionsBuilder(G_BUILD_VECTOR_TRUNC)
2146 BuildVector.customFor({
V2S16,
S16});
2147 BuildVector.minScalarOrElt(0,
S32);
2149 getActionDefinitionsBuilder(G_BUILD_VECTOR_TRUNC)
2157 getActionDefinitionsBuilder(G_CONCAT_VECTORS)
2159 .clampMaxNumElements(0,
S32, 32)
2160 .clampMaxNumElements(1,
S16, 2)
2161 .clampMaxNumElements(0,
S16, 64);
2163 getActionDefinitionsBuilder(G_SHUFFLE_VECTOR).lower();
2166 for (
unsigned Op : {G_MERGE_VALUES, G_UNMERGE_VALUES}) {
2167 unsigned BigTyIdx =
Op == G_MERGE_VALUES ? 0 : 1;
2168 unsigned LitTyIdx =
Op == G_MERGE_VALUES ? 1 : 0;
2170 auto notValidElt = [=](
const LegalityQuery &Query,
unsigned TypeIdx) {
2171 const LLT Ty = Query.
Types[TypeIdx];
2183 getActionDefinitionsBuilder(
Op)
2187 const LLT BigTy = Query.
Types[BigTyIdx];
2193 .widenScalarToNextPow2(LitTyIdx, 16)
2202 .clampScalar(LitTyIdx,
S32,
S512)
2203 .widenScalarToNextPow2(LitTyIdx, 32)
2207 return notValidElt(Query, LitTyIdx);
2212 return notValidElt(Query, BigTyIdx);
2217 if (
Op == G_MERGE_VALUES) {
2218 Builder.widenScalarIf(
2221 const LLT Ty = Query.
Types[LitTyIdx];
2227 Builder.widenScalarIf(
2229 const LLT Ty = Query.
Types[BigTyIdx];
2235 const LLT &Ty = Query.
Types[BigTyIdx];
2237 if (NewSizeInBits >= 256) {
2239 if (RoundedTo < NewSizeInBits)
2240 NewSizeInBits = RoundedTo;
2242 return std::pair(BigTyIdx,
LLT::scalar(NewSizeInBits));
2251 auto &SextInReg = getActionDefinitionsBuilder(G_SEXT_INREG)
2252 .legalFor({{
S32}, {
S64}})
2253 .clampScalar(0,
S32,
S64);
2255 if (
ST.hasVOP3PInsts()) {
2256 SextInReg.lowerFor({{
V2S16}})
2260 .clampMaxNumElementsStrict(0,
S16, 2);
2261 }
else if (
ST.has16BitInsts()) {
2262 SextInReg.lowerFor({{
S32}, {
S64}, {
S16}});
2266 SextInReg.lowerFor({{
S32}, {
S64}});
2271 .clampScalar(0,
S32,
S64)
2274 getActionDefinitionsBuilder({G_ROTR, G_ROTL})
2278 auto &FSHRActionDefs = getActionDefinitionsBuilder(G_FSHR);
2279 FSHRActionDefs.legalFor({{
S32,
S32}})
2280 .clampMaxNumElementsStrict(0,
S16, 2);
2281 if (
ST.hasVOP3PInsts())
2283 FSHRActionDefs.scalarize(0).lower();
2285 if (
ST.hasVOP3PInsts()) {
2286 getActionDefinitionsBuilder(G_FSHL)
2288 .clampMaxNumElementsStrict(0,
S16, 2)
2292 getActionDefinitionsBuilder(G_FSHL)
2297 getActionDefinitionsBuilder(G_READCYCLECOUNTER)
2300 getActionDefinitionsBuilder(G_READSTEADYCOUNTER).legalFor({
S64});
2302 getActionDefinitionsBuilder(G_FENCE)
2305 getActionDefinitionsBuilder({G_SMULO, G_UMULO})
2310 getActionDefinitionsBuilder({G_SBFX, G_UBFX})
2312 .clampScalar(1,
S32,
S32)
2313 .clampScalar(0,
S32,
S64)
2314 .widenScalarToNextPow2(0)
2317 getActionDefinitionsBuilder(
2321 G_ATOMIC_CMPXCHG_WITH_SUCCESS, G_ATOMICRMW_NAND, G_ATOMICRMW_FSUB,
2327 if (
ST.hasIEEEMinimumMaximumInsts()) {
2328 getActionDefinitionsBuilder({G_FMINIMUM, G_FMAXIMUM})
2329 .legalFor(FPTypesPK16)
2330 .clampMaxNumElements(0,
F16, 2)
2332 }
else if (
ST.hasVOP3PInsts()) {
2333 getActionDefinitionsBuilder({G_FMINIMUM, G_FMAXIMUM})
2335 .clampMaxNumElementsStrict(0,
F16, 2)
2339 getActionDefinitionsBuilder({G_FMINIMUM, G_FMAXIMUM})
2341 .clampScalar(0,
F32,
F64)
2345 getActionDefinitionsBuilder(
2346 {G_MEMCPY, G_MEMCPY_INLINE, G_MEMMOVE, G_MEMSET, G_MEMSET_INLINE})
2349 getActionDefinitionsBuilder({G_TRAP, G_DEBUGTRAP}).custom();
2351 getActionDefinitionsBuilder({G_VASTART, G_VAARG, G_BRJT, G_JUMP_TABLE,
2352 G_INDEXED_LOAD, G_INDEXED_SEXTLOAD,
2353 G_INDEXED_ZEXTLOAD, G_INDEXED_STORE})
2356 getActionDefinitionsBuilder(G_PREFETCH)
2357 .customFor({Constant32Ptr})
2360 getActionDefinitionsBuilder(
2361 {G_VECREDUCE_SMIN, G_VECREDUCE_SMAX, G_VECREDUCE_UMIN, G_VECREDUCE_UMAX,
2362 G_VECREDUCE_ADD, G_VECREDUCE_MUL, G_VECREDUCE_FADD, G_VECREDUCE_FMUL,
2363 G_VECREDUCE_FMIN, G_VECREDUCE_FMAX, G_VECREDUCE_FMINIMUM,
2364 G_VECREDUCE_FMAXIMUM, G_VECREDUCE_OR, G_VECREDUCE_AND, G_VECREDUCE_XOR})
2369 getActionDefinitionsBuilder({G_INTRINSIC, G_INTRINSIC_W_SIDE_EFFECTS,
2370 G_INTRINSIC_CONVERGENT,
2371 G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS})
2383 switch (
MI.getOpcode()) {
2384 case TargetOpcode::G_ADDRSPACE_CAST:
2386 case TargetOpcode::G_FRAME_INDEX:
2388 case TargetOpcode::G_INTRINSIC_ROUNDEVEN:
2390 case TargetOpcode::G_FCEIL:
2392 case TargetOpcode::G_FREM:
2394 case TargetOpcode::G_INTRINSIC_TRUNC:
2396 case TargetOpcode::G_SITOFP:
2398 case TargetOpcode::G_UITOFP:
2400 case TargetOpcode::G_FPTOSI:
2402 case TargetOpcode::G_FPTOUI:
2404 case TargetOpcode::G_FMINNUM:
2405 case TargetOpcode::G_FMAXNUM:
2406 case TargetOpcode::G_FMINIMUMNUM:
2407 case TargetOpcode::G_FMAXIMUMNUM:
2409 case TargetOpcode::G_EXTRACT:
2411 case TargetOpcode::G_INSERT:
2413 case TargetOpcode::G_EXTRACT_VECTOR_ELT:
2415 case TargetOpcode::G_INSERT_VECTOR_ELT:
2417 case TargetOpcode::G_FSIN:
2418 case TargetOpcode::G_FCOS:
2420 case TargetOpcode::G_GLOBAL_VALUE:
2422 case TargetOpcode::G_LOAD:
2423 case TargetOpcode::G_SEXTLOAD:
2424 case TargetOpcode::G_ZEXTLOAD:
2425 case TargetOpcode::G_PREFETCH:
2427 case TargetOpcode::G_STORE:
2429 case TargetOpcode::G_FMAD:
2431 case TargetOpcode::G_FDIV:
2433 case TargetOpcode::G_FFREXP:
2435 case TargetOpcode::G_FSQRT:
2437 case TargetOpcode::G_UDIV:
2438 case TargetOpcode::G_UREM:
2439 case TargetOpcode::G_UDIVREM:
2441 case TargetOpcode::G_SDIV:
2442 case TargetOpcode::G_SREM:
2443 case TargetOpcode::G_SDIVREM:
2445 case TargetOpcode::G_ATOMIC_CMPXCHG:
2447 case TargetOpcode::G_FLOG2:
2449 case TargetOpcode::G_FLOG:
2450 case TargetOpcode::G_FLOG10:
2452 case TargetOpcode::G_FEXP2:
2454 case TargetOpcode::G_FEXP:
2455 case TargetOpcode::G_FEXP10:
2457 case TargetOpcode::G_FPOW:
2459 case TargetOpcode::G_FFLOOR:
2461 case TargetOpcode::G_BUILD_VECTOR:
2462 case TargetOpcode::G_BUILD_VECTOR_TRUNC:
2464 case TargetOpcode::G_MUL:
2466 case TargetOpcode::G_CTLZ:
2467 case TargetOpcode::G_CTTZ:
2469 case TargetOpcode::G_CTLS:
2471 case TargetOpcode::G_CTLZ_ZERO_POISON:
2473 case TargetOpcode::G_STACKSAVE:
2475 case TargetOpcode::G_GET_FPENV:
2477 case TargetOpcode::G_SET_FPENV:
2479 case TargetOpcode::G_TRAP:
2481 case TargetOpcode::G_DEBUGTRAP:
2493 unsigned BaseAS = AS;
2498 Register Aperture = getBaseSegmentAperture(BaseAS,
MRI,
B);
2502 auto Tag =
B.buildConstant(
S32, SANum);
2503 return B.buildOr(
S32, Aperture,
Tag).getReg(0);
2509Register AMDGPULegalizerInfo::getBaseSegmentAperture(
2519 if (ST.hasApertureRegs()) {
2523 const unsigned ApertureRegNo =
2524 IsLDS ? AMDGPU::SRC_SHARED_BASE : AMDGPU::SRC_PRIVATE_BASE;
2525 assert((ApertureRegNo != AMDGPU::SRC_PRIVATE_BASE ||
2526 !ST.hasGloballyAddressableScratch()) &&
2527 "Cannot use src_private_base with globally addressable scratch!");
2528 Register Dst =
MRI.createGenericVirtualRegister(I64);
2529 MRI.setRegClass(Dst, &AMDGPU::SReg_64RegClass);
2530 B.buildCopy({Dst}, {
Register(ApertureRegNo)});
2531 return B.buildUnmerge(I32, Dst).getReg(1);
2546 ST.getTargetLowering()->getImplicitParameterOffset(
B.getMF(), Param);
2562 B.buildObjectPtrOffset(LoadAddr, KernargPtrReg,
2565 return B.buildLoad(I32, LoadAddr, *MMO).getReg(0);
2579 uint32_t StructOffset = IsLDS ? 0x40 : 0x44;
2587 B.buildObjectPtrOffset(
2590 return B.buildLoad(I32, LoadAddr, *MMO).getReg(0);
2598 switch (Def->getOpcode()) {
2599 case AMDGPU::G_FRAME_INDEX:
2600 case AMDGPU::G_GLOBAL_VALUE:
2601 case AMDGPU::G_BLOCK_ADDR:
2603 case AMDGPU::G_CONSTANT: {
2604 const ConstantInt *CI = Def->getOperand(1).getCImm();
2619 assert(
MI.getOpcode() == TargetOpcode::G_ADDRSPACE_CAST);
2625 LLT DstTy =
MRI.getType(Dst);
2626 LLT SrcTy =
MRI.getType(Src);
2628 unsigned SrcAS = SrcTy.getAddressSpace();
2642 MI.setDesc(
B.getTII().get(TargetOpcode::G_BITCAST));
2649 auto castFlatToLocalOrPrivate = [&](
const DstOp &Dst) ->
Register {
2651 ST.hasGloballyAddressableScratch()) {
2654 Register SrcLo =
B.buildExtract(I32, Src, 0).getReg(0);
2656 B.buildInstr(AMDGPU::S_MOV_B32, {I32},
2657 {
Register(AMDGPU::SRC_FLAT_SCRATCH_BASE_LO)})
2659 MRI.setRegClass(FlatScratchBaseLo, &AMDGPU::SReg_32RegClass);
2660 Register Sub =
B.buildSub(I32, SrcLo, FlatScratchBaseLo).getReg(0);
2661 return B.buildIntToPtr(Dst,
Sub).getReg(0);
2664 return B.buildExtract(Dst, Src, 0).getReg(0);
2668 castFlatToLocalOrPrivate(Dst);
2669 MI.eraseFromParent();
2675 auto SegmentNull =
B.buildConstant(DstTy, NullVal);
2676 auto FlatNull =
B.buildConstant(SrcTy, 0);
2679 auto PtrLo32 = castFlatToLocalOrPrivate(DstTy);
2683 B.buildSelect(Dst, CmpRes, PtrLo32, SegmentNull.getReg(0));
2685 MI.eraseFromParent();
2692 auto castLocalOrPrivateToFlat = [&](
const DstOp &Dst) ->
Register {
2695 Register SrcAsInt =
B.buildPtrToInt(I32, Src).getReg(0);
2698 ST.hasGloballyAddressableScratch()) {
2702 Register ThreadID =
B.buildConstant(I32, 0).getReg(0);
2703 ThreadID =
B.buildIntrinsic(Intrinsic::amdgcn_mbcnt_lo, {I32})
2707 if (ST.isWave64()) {
2708 ThreadID =
B.buildIntrinsic(Intrinsic::amdgcn_mbcnt_hi, {I32})
2714 B.buildConstant(I32, 57 - 32 - ST.getWavefrontSizeLog2()).getReg(0);
2715 Register SrcHi =
B.buildShl(I32, ThreadID, ShAmt).getReg(0);
2717 B.buildMergeLikeInstr(DstTy, {SrcAsInt, SrcHi}).
getReg(0);
2721 B.buildInstr(AMDGPU::S_MOV_B64, {I64},
2722 {
Register(AMDGPU::SRC_FLAT_SCRATCH_BASE)})
2724 MRI.setRegClass(FlatScratchBase, &AMDGPU::SReg_64RegClass);
2725 return B.buildPtrAdd(Dst, CvtPtr, FlatScratchBase).getReg(0);
2734 return B.buildMergeLikeInstr(Dst, {SrcAsInt, ApertureReg}).
getReg(0);
2738 castLocalOrPrivateToFlat(Dst);
2739 MI.eraseFromParent();
2743 Register BuildPtr = castLocalOrPrivateToFlat(DstTy);
2750 SegmentNull.getReg(0));
2752 B.buildSelect(Dst, CmpRes, BuildPtr, FlatNull);
2754 MI.eraseFromParent();
2759 SrcTy.getSizeInBits() == 64) {
2761 B.buildExtract(Dst, Src, 0);
2762 MI.eraseFromParent();
2769 uint32_t AddrHiVal = Info->get32BitAddressHighBits();
2770 auto PtrLo =
B.buildPtrToInt(I32, Src);
2771 if (AddrHiVal == 0) {
2772 auto Zext =
B.buildZExt(I64, PtrLo);
2773 B.buildIntToPtr(Dst, Zext);
2775 auto HighAddr =
B.buildConstant(I32, AddrHiVal);
2776 B.buildMergeLikeInstr(Dst, {PtrLo, HighAddr});
2779 MI.eraseFromParent();
2786 MI.eraseFromParent();
2794 LLT Ty =
MRI.getType(Src);
2795 assert(Ty.isScalar() && Ty.getSizeInBits() == 64);
2800 auto C1 =
B.buildFConstant(Ty, C1Val);
2801 auto CopySign =
B.buildFCopysign(Ty,
C1, Src);
2804 auto Tmp1 =
B.buildFAdd(Ty, Src, CopySign);
2805 auto Tmp2 =
B.buildFSub(Ty, Tmp1, CopySign);
2807 auto C2 =
B.buildFConstant(Ty, C2Val);
2808 auto Fabs =
B.buildFAbs(Ty, Src);
2811 B.buildSelect(
MI.getOperand(0).getReg(),
Cond, Src, Tmp2);
2812 MI.eraseFromParent();
2829 auto Trunc =
B.buildIntrinsicTrunc(
F64, Src);
2831 const auto Zero =
B.buildFConstant(
F64, 0.0);
2832 const auto One =
B.buildFConstant(
F64, 1.0);
2835 auto And =
B.buildAnd(
S1, Lt0, NeTrunc);
2839 B.buildFAdd(
MI.getOperand(0).getReg(), Trunc,
Add);
2840 MI.eraseFromParent();
2848 Register Src0Reg =
MI.getOperand(1).getReg();
2849 Register Src1Reg =
MI.getOperand(2).getReg();
2850 auto Flags =
MI.getFlags();
2851 LLT Ty =
MRI.getType(DstReg);
2853 auto Div =
B.buildFDiv(Ty, Src0Reg, Src1Reg, Flags);
2854 auto Trunc =
B.buildIntrinsicTrunc(Ty, Div, Flags);
2855 auto Neg =
B.buildFNeg(Ty, Trunc, Flags);
2856 B.buildFMA(DstReg, Neg, Src1Reg, Src0Reg, Flags);
2857 MI.eraseFromParent();
2863 const unsigned FractBits = 52;
2864 const unsigned ExpBits = 11;
2867 auto Const0 =
B.buildConstant(I32, FractBits - 32);
2868 auto Const1 =
B.buildConstant(I32, ExpBits);
2870 auto ExpPart =
B.buildIntrinsic(Intrinsic::amdgcn_ubfe, {I32})
2872 .addUse(Const0.getReg(0))
2873 .addUse(Const1.getReg(0));
2875 return B.buildSub(I32, ExpPart,
B.buildConstant(I32, 1023));
2888 auto SrcInt =
B.buildBitcast(I64, Src);
2891 auto Unmerge =
B.buildUnmerge({I32, I32}, SrcInt);
2898 const unsigned FractBits = 52;
2901 const auto SignBitMask =
B.buildConstant(I32, UINT32_C(1) << 31);
2902 auto SignBit =
B.buildAnd(I32,
Hi, SignBitMask);
2904 const auto FractMask =
B.buildConstant(I64, (UINT64_C(1) << FractBits) - 1);
2906 const auto Zero32 =
B.buildConstant(I32, 0);
2909 auto SignBit64 =
B.buildMergeLikeInstr(I64, {Zero32, SignBit});
2911 auto Shr =
B.buildAShr(I64, FractMask, Exp);
2912 auto Not =
B.buildNot(I64, Shr);
2913 auto Tmp0 =
B.buildAnd(I64, SrcInt, Not);
2914 auto FiftyOne =
B.buildConstant(I32, FractBits - 1);
2919 auto Tmp1 =
B.buildSelect(I64, ExpLt0, SignBit64, Tmp0);
2920 auto Res =
B.buildSelect(I64, ExpGt51, SrcInt, Tmp1);
2921 B.buildBitcast(
MI.getOperand(0).getReg(), Res);
2922 MI.eraseFromParent();
2938 auto Unmerge =
B.buildUnmerge({I32, I32}, Src);
2939 auto ThirtyTwo =
B.buildConstant(I32, 32);
2941 if (
MRI.getType(Dst) ==
F64) {
2942 auto CvtHi =
Signed ?
B.buildSITOFP(
F64, Unmerge.getReg(1))
2943 :
B.buildUITOFP(
F64, Unmerge.getReg(1));
2945 auto CvtLo =
B.buildUITOFP(
F64, Unmerge.getReg(0));
2946 auto LdExp =
B.buildFLdexp(
F64, CvtHi, ThirtyTwo);
2949 B.buildFAdd(Dst, LdExp, CvtLo);
2950 MI.eraseFromParent();
2956 auto One =
B.buildConstant(I32, 1);
2960 auto ThirtyOne =
B.buildConstant(I32, 31);
2961 auto X =
B.buildXor(I32, Unmerge.getReg(0), Unmerge.getReg(1));
2962 auto OppositeSign =
B.buildAShr(I32,
X, ThirtyOne);
2963 auto MaxShAmt =
B.buildAdd(I32, ThirtyTwo, OppositeSign);
2964 auto LS =
B.buildIntrinsic(Intrinsic::amdgcn_sffbh, {I32})
2965 .addUse(Unmerge.getReg(1));
2966 auto LS2 =
B.buildSub(I32, LS, One);
2967 ShAmt =
B.buildUMin(I32, LS2, MaxShAmt);
2969 ShAmt =
B.buildCTLZ(I32, Unmerge.getReg(1));
2970 auto Norm =
B.buildShl(I64, Src, ShAmt);
2971 auto Unmerge2 =
B.buildUnmerge({I32, I32}, Norm);
2972 auto Adjust =
B.buildUMin(I32, One, Unmerge2.getReg(0));
2973 auto Norm2 =
B.buildOr(I32, Unmerge2.getReg(1), Adjust);
2974 auto FVal =
Signed ?
B.buildSITOFP(
F32, Norm2) :
B.buildUITOFP(
F32, Norm2);
2975 auto Scale =
B.buildSub(I32, ThirtyTwo, ShAmt);
2976 B.buildFLdexp(Dst, FVal, Scale);
2977 MI.eraseFromParent();
2994 const LLT SrcLT =
MRI.getType(Src);
2997 unsigned Flags =
MI.getFlags();
3008 auto Trunc =
B.buildIntrinsicTrunc(SrcLT, Src, Flags);
3016 auto SrcInt =
B.buildBitcast(I32, Src);
3017 Sign =
B.buildAShr(I32, SrcInt,
B.buildConstant(I32, 31));
3018 Trunc =
B.buildFAbs(
F32, Trunc, Flags);
3022 K0 =
B.buildFConstant(
3024 K1 =
B.buildFConstant(
3027 K0 =
B.buildFConstant(
3029 K1 =
B.buildFConstant(
3033 auto Mul =
B.buildFMul(SrcLT, Trunc, K0, Flags);
3034 auto FloorMul =
B.buildFFloor(SrcLT,
Mul, Flags);
3035 auto Fma =
B.buildFMA(SrcLT, FloorMul, K1, Trunc, Flags);
3037 auto Hi = (
Signed && SrcLT ==
F64) ?
B.buildFPTOSI(I32, FloorMul)
3038 :
B.buildFPTOUI(I32, FloorMul);
3039 auto Lo =
B.buildFPTOUI(I32, Fma);
3045 B.buildSub(Dst,
B.buildXor(I64,
B.buildMergeLikeInstr(I64, {Lo, Hi}),
Sign),
3048 B.buildMergeLikeInstr(Dst, {
Lo,
Hi});
3049 MI.eraseFromParent();
3072 uint64_t
Offset =
MI.getOperand(2).getImm();
3080 const LLT DstTy =
MRI.getType(DstReg);
3081 unsigned StartIdx =
Offset / 32;
3085 if (DstCount == 1) {
3087 B.buildIntToPtr(DstReg, Unmerge.getReg(StartIdx));
3090 MRI.replaceRegWith(DstReg, Unmerge.getReg(StartIdx));
3095 for (
unsigned I = 0;
I < DstCount; ++
I)
3096 MergeVec.
push_back(Unmerge.getReg(StartIdx +
I));
3097 B.buildMergeLikeInstr(DstReg, MergeVec);
3100 MI.eraseFromParent();
3110 Register InsertSrc =
MI.getOperand(2).getReg();
3111 uint64_t
Offset =
MI.getOperand(3).getImm();
3113 unsigned DstSize =
MRI.getType(DstReg).getSizeInBits();
3114 const LLT InsertTy =
MRI.getType(InsertSrc);
3119 if (
Offset % 32 != 0 || DstSize % 32 != 0 || InsertSize % 32 != 0)
3123 unsigned DstCount = DstSize / 32;
3124 unsigned InsertCount = InsertSize / 32;
3125 unsigned StartIdx =
Offset / 32;
3127 auto SrcUnmerge =
B.buildUnmerge(I32, SrcReg);
3130 for (
unsigned I = 0;
I < StartIdx; ++
I)
3133 if (InsertCount == 1) {
3137 InsertSrc =
B.buildPtrToInt(I32, InsertSrc).getReg(0);
3140 auto InsertUnmerge =
B.buildUnmerge(I32, InsertSrc);
3141 for (
unsigned I = 0;
I < InsertCount; ++
I)
3145 for (
unsigned I = StartIdx + InsertCount;
I < DstCount; ++
I)
3148 B.buildMergeLikeInstr(DstReg, MergeVec);
3150 MI.eraseFromParent();
3164 LLT VecTy =
MRI.getType(Vec);
3177 auto IntVec =
B.buildPtrToInt(IntVecTy, Vec);
3178 auto IntElt =
B.buildExtractVectorElement(IntTy, IntVec,
MI.getOperand(2));
3179 B.buildIntToPtr(Dst, IntElt);
3181 MI.eraseFromParent();
3188 std::optional<ValueAndVReg> MaybeIdxVal =
3192 const uint64_t IdxVal = MaybeIdxVal->Value.getZExtValue();
3195 auto Unmerge =
B.buildUnmerge(EltTy, Vec);
3196 B.buildCopy(Dst, Unmerge.getReg(IdxVal));
3201 MI.eraseFromParent();
3216 LLT VecTy =
MRI.getType(Vec);
3230 auto IntVecSource =
B.buildPtrToInt(IntVecTy, Vec);
3231 auto IntIns =
B.buildPtrToInt(IntTy, Ins);
3232 auto IntVecDest =
B.buildInsertVectorElement(IntVecTy, IntVecSource, IntIns,
3234 B.buildIntToPtr(Dst, IntVecDest);
3235 MI.eraseFromParent();
3242 std::optional<ValueAndVReg> MaybeIdxVal =
3247 const uint64_t IdxVal = MaybeIdxVal->Value.getZExtValue();
3250 if (IdxVal < NumElts) {
3252 for (
unsigned i = 0; i < NumElts; ++i)
3253 SrcRegs.
push_back(
MRI.createGenericVirtualRegister(EltTy));
3254 B.buildUnmerge(SrcRegs, Vec);
3256 SrcRegs[IdxVal] =
MI.getOperand(2).getReg();
3257 B.buildMergeLikeInstr(Dst, SrcRegs);
3262 MI.eraseFromParent();
3272 LLT Ty =
MRI.getType(DstReg);
3273 unsigned Flags =
MI.getFlags();
3277 if (ST.hasTrigReducedRange()) {
3278 auto MulVal =
B.buildFMul(Ty, SrcReg, OneOver2Pi, Flags);
3279 TrigVal =
B.buildIntrinsic(Intrinsic::amdgcn_fract, {Ty})
3280 .addUse(MulVal.getReg(0))
3284 TrigVal =
B.buildFMul(Ty, SrcReg, OneOver2Pi, Flags).getReg(0);
3287 Intrinsic::amdgcn_sin : Intrinsic::amdgcn_cos;
3291 MI.eraseFromParent();
3299 unsigned GAFlags)
const {
3330 MRI.createVirtualRegister({&AMDGPU::SReg_64RegClass, PCRegTy});
3332 if (ST.has64BitLiterals()) {
3336 B.buildInstr(AMDGPU::SI_PC_ADD_REL_OFFSET64).addDef(PCReg);
3340 B.buildInstr(AMDGPU::SI_PC_ADD_REL_OFFSET).addDef(PCReg);
3351 B.buildExtract(DstReg, PCReg, 0);
3353 B.buildCopy(DstReg, PCReg);
3363 if (RequiresHighHalf && ST.has64BitLiterals()) {
3365 MRI.createVirtualRegister({&AMDGPU::SReg_64RegClass, PtrTy});
3366 B.buildInstr(AMDGPU::S_MOV_B64)
3369 B.buildCopy(DstReg, Addr);
3374 Register AddrLo =
MRI.createVirtualRegister({&AMDGPU::SReg_32RegClass, I32});
3377 B.buildInstr(AMDGPU::S_MOV_B32)
3382 if (RequiresHighHalf) {
3384 "Must provide a 64-bit pointer type!");
3387 MRI.createVirtualRegister({&AMDGPU::SReg_32RegClass, I32});
3389 B.buildInstr(AMDGPU::S_MOV_B32)
3394 MRI.createVirtualRegister({&AMDGPU::SReg_64RegClass,
LLT::integer(64)});
3396 B.buildMergeValues(AddrDst, {AddrLo, AddrHi});
3397 B.buildCast(DstReg, AddrDst);
3399 B.buildCast(DstReg, AddrLo);
3407 LLT Ty =
MRI.getType(DstReg);
3408 unsigned AS = Ty.getAddressSpace();
3419 Fn,
"unsupported use of BARRIER address space",
MI.getDebugLoc(),
3421 B.buildUndef(DstReg);
3422 MI.eraseFromParent();
3426 B.buildConstant(DstReg,
3428 MI.eraseFromParent();
3434 GV->
getName() !=
"llvm.amdgcn.module.lds") {
3437 Fn,
"local memory global used by non-kernel function",
3446 B.buildUndef(DstReg);
3447 MI.eraseFromParent();
3455 if (!
TLI->shouldUseLDSConstAddress(GV)) {
3471 auto Sz =
B.buildIntrinsic(Intrinsic::amdgcn_groupstaticsize, {I32});
3472 B.buildIntToPtr(DstReg, Sz);
3473 MI.eraseFromParent();
3479 MI.eraseFromParent();
3483 if (ST.isAmdPalOS() || ST.isMesa3DOS()) {
3485 MI.eraseFromParent();
3491 if (
TLI->shouldEmitFixup(GV)) {
3493 MI.eraseFromParent();
3497 if (
TLI->shouldEmitPCReloc(GV)) {
3499 MI.eraseFromParent();
3504 Register GOTAddr =
MRI.createGenericVirtualRegister(PtrTy);
3515 if (Ty.getSizeInBits() == 32) {
3517 auto Load =
B.buildLoad(PtrTy, GOTAddr, *GOTMMO);
3518 B.buildExtract(DstReg,
Load, 0);
3520 B.buildLoad(DstReg, GOTAddr, *GOTMMO);
3522 MI.eraseFromParent();
3538 int FI =
MI.getOperand(1).getIndex();
3544 "unsupported alloca in VGPR 'as memory' address space (13)",
3546 B.buildUndef(
MI.getOperand(0));
3547 MI.eraseFromParent();
3555 const unsigned EltSize = Ty.isVector() ? Ty.getScalarSizeInBits() : 32;
3558 if (
Size % EltSize != 0 ||
3562 return Ty.changeElementSize(
Size);
3564 Ty.getElementType());
3566 if (EltSize % 32 == 0)
3567 return Ty.getElementType();
3583 const LLT ValTy =
MRI.getType(ValReg);
3596 Register Loaded =
B.buildLoadFromOffset(LoadTy, PtrReg, MMO, 0).getReg(0);
3599 Loaded = IsSigned ?
B.buildSExtInReg(LoadTy, Loaded, MemSize).getReg(0)
3600 :
B.buildZExtInReg(LoadTy, Loaded, MemSize).getReg(0);
3603 B.buildSExtOrTrunc(ValReg, Loaded);
3605 B.buildZExtOrTrunc(ValReg, Loaded);
3613 if (MemSize != ValSize || MemSize % 32 != 0 || LdSt.
getAlign() <
Align(4)) {
3617 "unsupported access of VGPR 'as memory' address space (13); only "
3618 "dword-aligned whole-dword loads and stores are implemented",
3621 B.buildUndef(ValReg);
3640 B.buildLShr(I32, PtrAsInt,
B.buildConstant(I32, 2));
3650 if (IntTy.getScalarSizeInBits() != 32) {
3651 unsigned NumDwords = ValSize / 32;
3661 B.buildInstr(AMDGPU::G_AMDGPU_REG_STORE, {}, {
Value, Index.getReg(0)})
3662 .addMemOperand(&MMO);
3663 }
else if (RegTy == ValTy) {
3664 B.buildInstr(AMDGPU::G_AMDGPU_REG_LOAD, {ValReg}, {Index.getReg(0)})
3665 .addMemOperand(&MMO);
3668 B.buildInstr(AMDGPU::G_AMDGPU_REG_LOAD, {RegTy}, {Index.getReg(0)})
3669 .addMemOperand(&MMO)
3671 if (IntTy == ValTy) {
3672 B.buildBitcast(ValReg, Result);
3675 Result =
B.buildBitcast(IntTy, Result).getReg(0);
3676 B.buildIntToPtr(ValReg, Result);
3690 unsigned PtrIdx =
MI.getOpcode() == AMDGPU::G_PREFETCH ? 0 : 1;
3691 Register PtrReg =
MI.getOperand(PtrIdx).getReg();
3692 LLT PtrTy =
MRI.getType(PtrReg);
3697 auto Cast =
B.buildAddrSpaceCast(ConstPtr, PtrReg);
3699 MI.getOperand(PtrIdx).setReg(Cast.getReg(0));
3707 if (
MI.getOpcode() != AMDGPU::G_LOAD)
3711 LLT ValTy =
MRI.getType(ValReg);
3728 const uint64_t AlignInBits = 8 * MemAlign.
value();
3736 if (WideMemSize == ValSize) {
3742 MI.setMemRefs(MF, {WideMMO});
3748 if (ValSize > WideMemSize)
3755 WideLoad =
B.buildLoadFromOffset(WideTy, PtrReg, *MMO, 0).getReg(0);
3756 B.buildTrunc(ValReg, WideLoad).getReg(0);
3763 WideLoad =
B.buildLoadFromOffset(WideTy, PtrReg, *MMO, 0).getReg(0);
3764 B.buildExtract(ValReg, WideLoad, 0);
3768 WideLoad =
B.buildLoadFromOffset(WideTy, PtrReg, *MMO, 0).getReg(0);
3769 B.buildDeleteTrailingVectorElements(ValReg, WideLoad);
3773 MI.eraseFromParent();
3786 Register DataReg =
MI.getOperand(0).getReg();
3787 LLT DataTy =
MRI.getType(DataReg);
3796 if (
MRI.getType(
MI.getOperand(1).getReg()).getAddressSpace() ==
3805 LLT Ty =
MRI.getType(
MI.getOperand(0).getReg());
3835 "this should not have been custom lowered");
3837 LLT ValTy =
MRI.getType(CmpVal);
3840 Register PackedVal =
B.buildBuildVector(VecTy, { NewVal, CmpVal }).
getReg(0);
3842 B.buildInstr(AMDGPU::G_AMDGPU_ATOMIC_CMPXCHG)
3846 .setMemRefs(
MI.memoperands());
3848 MI.eraseFromParent();
3856 switch (
DefMI->getOpcode()) {
3857 case TargetOpcode::G_INTRINSIC: {
3859 case Intrinsic::amdgcn_frexp_mant:
3860 case Intrinsic::amdgcn_log:
3861 case Intrinsic::amdgcn_log_clamp:
3862 case Intrinsic::amdgcn_exp2:
3863 case Intrinsic::amdgcn_sqrt:
3871 case TargetOpcode::G_FSQRT:
3873 case TargetOpcode::G_FFREXP: {
3874 if (
DefMI->getOperand(0).getReg() == Src)
3878 case TargetOpcode::G_FPEXT: {
3879 return MRI.getType(
DefMI->getOperand(1).getReg()) ==
F16;
3899std::pair<Register, Register>
3901 unsigned Flags)
const {
3905 auto SmallestNormal =
B.buildFConstant(
3907 auto IsLtSmallestNormal =
3910 auto Scale32 =
B.buildFConstant(
F32, 0x1.0p+32);
3911 auto One =
B.buildFConstant(
F32, 1.0);
3913 B.buildSelect(
F32, IsLtSmallestNormal, Scale32, One, Flags);
3914 auto ScaledInput =
B.buildFMul(
F32, Src, ScaleFactor, Flags);
3916 return {ScaledInput.getReg(0), IsLtSmallestNormal.getReg(0)};
3929 LLT Ty =
B.getMRI()->getType(Dst);
3930 unsigned Flags =
MI.getFlags();
3934 auto Ext =
B.buildFPExt(
F32, Src, Flags);
3935 auto Log2 =
B.buildIntrinsic(Intrinsic::amdgcn_log, {
F32})
3936 .addUse(Ext.getReg(0))
3938 B.buildFPTrunc(Dst,
Log2, Flags);
3939 MI.eraseFromParent();
3947 B.buildIntrinsic(Intrinsic::amdgcn_log, {
MI.getOperand(0)})
3950 MI.eraseFromParent();
3954 auto Log2 =
B.buildIntrinsic(Intrinsic::amdgcn_log, {Ty})
3955 .addUse(ScaledInput)
3958 auto ThirtyTwo =
B.buildFConstant(Ty, 32.0);
3959 auto Zero =
B.buildFConstant(Ty, 0.0);
3961 B.buildSelect(Ty, IsLtSmallestNormal, ThirtyTwo,
Zero, Flags);
3962 B.buildFSub(Dst,
Log2, ResultOffset, Flags);
3964 MI.eraseFromParent();
3970 auto FMul =
B.buildFMul(Ty,
X,
Y, Flags);
3971 return B.buildFAdd(Ty,
FMul, Z, Flags).getReg(0);
3976 const bool IsLog10 =
MI.getOpcode() == TargetOpcode::G_FLOG10;
3977 assert(IsLog10 ||
MI.getOpcode() == TargetOpcode::G_FLOG);
3982 unsigned Flags =
MI.getFlags();
3983 const LLT Ty =
MRI.getType(
X);
3992 auto PromoteSrc =
B.buildFPExt(
F32,
X, Flags);
3994 B.buildFPTrunc(Dst, LogVal, Flags);
3999 MI.eraseFromParent();
4008 B.buildIntrinsic(Intrinsic::amdgcn_log, {Ty}).addUse(
X).setMIFlags(Flags);
4011 if (ST.hasFastFMAF32()) {
4013 const float c_log10 = 0x1.344134p-2f;
4014 const float cc_log10 = 0x1.09f79ep-26f;
4017 const float c_log = 0x1.62e42ep-1f;
4018 const float cc_log = 0x1.efa39ep-25f;
4020 auto C =
B.buildFConstant(Ty, IsLog10 ? c_log10 : c_log);
4021 auto CC =
B.buildFConstant(Ty, IsLog10 ? cc_log10 : cc_log);
4025 R =
B.buildFMul(Ty,
Y,
C, NewFlags).getReg(0);
4026 auto NegR =
B.buildFNeg(Ty, R, NewFlags);
4027 auto FMA0 =
B.buildFMA(Ty,
Y,
C, NegR, NewFlags);
4028 auto FMA1 =
B.buildFMA(Ty,
Y,
CC, FMA0, NewFlags);
4029 R =
B.buildFAdd(Ty, R, FMA1, NewFlags).getReg(0);
4032 const float ch_log10 = 0x1.344000p-2f;
4033 const float ct_log10 = 0x1.3509f6p-18f;
4036 const float ch_log = 0x1.62e000p-1f;
4037 const float ct_log = 0x1.0bfbe8p-15f;
4039 auto CH =
B.buildFConstant(Ty, IsLog10 ? ch_log10 : ch_log);
4040 auto CT =
B.buildFConstant(Ty, IsLog10 ? ct_log10 : ct_log);
4043 auto YInt =
B.buildBitcast(I32,
Y);
4044 auto MaskConst =
B.buildConstant(I32, 0xfffff000);
4045 auto YH =
B.buildBitcast(Ty,
B.buildAnd(I32, YInt, MaskConst));
4046 auto YT =
B.buildFSub(Ty,
Y, YH, Flags);
4050 auto YTCT =
B.buildFMul(Ty, YT, CT, NewFlags);
4053 getMad(
B, Ty, YH.getReg(0), CT.getReg(0), YTCT.getReg(0), NewFlags);
4055 R =
getMad(
B, Ty, YH.getReg(0),
CH.getReg(0), Mad1, NewFlags);
4058 const bool IsFiniteOnly =
4061 if (!IsFiniteOnly) {
4064 auto Fabs =
B.buildFAbs(Ty,
Y);
4067 R =
B.buildSelect(Ty, IsFinite, R,
Y, Flags).getReg(0);
4071 auto Zero =
B.buildFConstant(Ty, 0.0);
4073 B.buildFConstant(Ty, IsLog10 ? 0x1.344136p+3f : 0x1.62e430p+4f);
4074 auto Shift =
B.buildSelect(Ty, IsScaled, ShiftK,
Zero, Flags);
4075 B.buildFSub(Dst, R, Shift, Flags);
4077 B.buildCopy(Dst, R);
4080 MI.eraseFromParent();
4086 unsigned Flags)
const {
4087 const double Log2BaseInverted =
4090 LLT Ty =
B.getMRI()->getType(Dst);
4095 auto LogSrc =
B.buildIntrinsic(Intrinsic::amdgcn_log, {Ty})
4096 .addUse(ScaledInput)
4098 auto ScaledResultOffset =
B.buildFConstant(Ty, -32.0 * Log2BaseInverted);
4099 auto Zero =
B.buildFConstant(Ty, 0.0);
4101 B.buildSelect(Ty, IsScaled, ScaledResultOffset,
Zero, Flags);
4102 auto Log2Inv =
B.buildFConstant(Ty, Log2BaseInverted);
4104 if (ST.hasFastFMAF32())
4105 B.buildFMA(Dst, LogSrc, Log2Inv, ResultOffset, Flags);
4107 auto Mul =
B.buildFMul(Ty, LogSrc, Log2Inv, Flags);
4108 B.buildFAdd(Dst,
Mul, ResultOffset, Flags);
4115 auto Log2Operand = Ty ==
F16 ?
B.buildFLog2(Ty, Src, Flags)
4116 :
B.buildIntrinsic(Intrinsic::amdgcn_log, {Ty})
4119 auto Log2BaseInvertedOperand =
B.buildFConstant(Ty, Log2BaseInverted);
4120 B.buildFMul(Dst, Log2Operand, Log2BaseInvertedOperand, Flags);
4131 unsigned Flags =
MI.getFlags();
4132 LLT Ty =
B.getMRI()->getType(Dst);
4139 auto Ext =
B.buildFPExt(
F32, Src, Flags);
4140 auto Log2 =
B.buildIntrinsic(Intrinsic::amdgcn_exp2, {
F32})
4141 .addUse(Ext.getReg(0))
4143 B.buildFPTrunc(Dst,
Log2, Flags);
4144 MI.eraseFromParent();
4154 MI.eraseFromParent();
4162 auto RangeCheckConst =
B.buildFConstant(Ty, -0x1.f80000p+6f);
4164 RangeCheckConst, Flags);
4166 auto SixtyFour =
B.buildFConstant(Ty, 0x1.0p+6f);
4167 auto Zero =
B.buildFConstant(Ty, 0.0);
4168 auto AddOffset =
B.buildSelect(
F32, NeedsScaling, SixtyFour,
Zero, Flags);
4169 auto AddInput =
B.buildFAdd(
F32, Src, AddOffset, Flags);
4171 auto Exp2 =
B.buildIntrinsic(Intrinsic::amdgcn_exp2, {Ty})
4172 .addUse(AddInput.getReg(0))
4175 auto TwoExpNeg64 =
B.buildFConstant(Ty, 0x1.0p-64f);
4176 auto One =
B.buildFConstant(Ty, 1.0);
4177 auto ResultScale =
B.buildSelect(
F32, NeedsScaling, TwoExpNeg64, One, Flags);
4178 B.buildFMul(Dst, Exp2, ResultScale, Flags);
4179 MI.eraseFromParent();
4184 const SrcOp &Src,
unsigned Flags) {
4185 LLT Ty = Dst.getLLTTy(*
B.getMRI());
4188 return B.buildIntrinsic(Intrinsic::amdgcn_exp2, {Dst})
4189 .addUse(Src.getReg())
4192 return B.buildFExp2(Dst, Src, Flags);
4198 bool IsExp10)
const {
4199 LLT Ty =
B.getMRI()->getType(
X);
4203 auto Const =
B.buildFConstant(Ty, IsExp10 ? 0x1.a934f0p+1f :
numbers::log2e);
4204 auto Mul =
B.buildFMul(Ty,
X, Const, Flags);
4211 LLT Ty =
B.getMRI()->getType(Dst);
4217 auto Threshold =
B.buildFConstant(Ty, -0x1.5d58a0p+6f);
4220 auto ScaleOffset =
B.buildFConstant(Ty, 0x1.0p+6f);
4221 auto ScaledX =
B.buildFAdd(Ty,
X, ScaleOffset, Flags);
4222 auto AdjustedX =
B.buildSelect(Ty, NeedsScaling, ScaledX,
X, Flags);
4225 auto ExpInput =
B.buildFMul(Ty, AdjustedX, Log2E, Flags);
4227 auto Exp2 =
B.buildIntrinsic(Intrinsic::amdgcn_exp2, {Ty})
4228 .addUse(ExpInput.getReg(0))
4231 auto ResultScaleFactor =
B.buildFConstant(Ty, 0x1.969d48p-93f);
4232 auto AdjustedResult =
B.buildFMul(Ty, Exp2, ResultScaleFactor, Flags);
4233 B.buildSelect(Dst, NeedsScaling, AdjustedResult, Exp2, Flags);
4239 unsigned Flags)
const {
4240 LLT Ty =
B.getMRI()->getType(Dst);
4244 auto K0 =
B.buildFConstant(Ty, 0x1.a92000p+1f);
4245 auto K1 =
B.buildFConstant(Ty, 0x1.4f0978p-11f);
4247 auto Mul1 =
B.buildFMul(Ty,
X, K1, Flags);
4248 auto Exp2_1 =
buildExp(
B, Ty, Mul1, Flags);
4249 auto Mul0 =
B.buildFMul(Ty,
X, K0, Flags);
4250 auto Exp2_0 =
buildExp(
B, Ty, Mul0, Flags);
4251 B.buildFMul(Dst, Exp2_0, Exp2_1, Flags);
4261 auto Threshold =
B.buildFConstant(Ty, -0x1.2f7030p+5f);
4265 auto ScaleOffset =
B.buildFConstant(Ty, 0x1.0p+5f);
4266 auto ScaledX =
B.buildFAdd(Ty,
X, ScaleOffset, Flags);
4267 auto AdjustedX =
B.buildSelect(Ty, NeedsScaling, ScaledX,
X);
4269 auto K0 =
B.buildFConstant(Ty, 0x1.a92000p+1f);
4270 auto K1 =
B.buildFConstant(Ty, 0x1.4f0978p-11f);
4272 auto Mul1 =
B.buildFMul(Ty, AdjustedX, K1, Flags);
4273 auto Exp2_1 =
buildExp(
B, Ty, Mul1, Flags);
4274 auto Mul0 =
B.buildFMul(Ty, AdjustedX, K0, Flags);
4275 auto Exp2_0 =
buildExp(
B, Ty, Mul0, Flags);
4277 auto MulExps =
B.buildFMul(Ty, Exp2_0, Exp2_1, Flags);
4278 auto ResultScaleFactor =
B.buildFConstant(Ty, 0x1.9f623ep-107f);
4279 auto AdjustedResult =
B.buildFMul(Ty, MulExps, ResultScaleFactor, Flags);
4281 B.buildSelect(Dst, NeedsScaling, AdjustedResult, MulExps);
4299 if (
MI.getOpcode() == TargetOpcode::G_FEXP2) {
4301 Dn =
B.buildFRint(
F64,
X, Flags).getReg(0);
4303 F =
B.buildFSub(
F64,
X, Dn, Flags).getReg(0);
4305 auto C1 =
B.buildFConstant(
F64,
APFloat(0x1.62e42fefa39efp-1));
4306 auto C2 =
B.buildFConstant(
F64,
APFloat(0x1.abc9e3b39803fp-56));
4307 auto Mul2 =
B.buildFMul(
F64,
F, C2, Flags).getReg(0);
4308 T =
B.buildFMA(
F64,
F,
C1, Mul2, Flags).getReg(0);
4310 }
else if (
MI.getOpcode() == TargetOpcode::G_FEXP10) {
4311 auto C1 =
B.buildFConstant(
F64,
APFloat(0x1.a934f0979a371p+1));
4312 auto Mul =
B.buildFMul(
F64,
X,
C1, Flags).getReg(0);
4313 Dn =
B.buildFRint(
F64,
Mul, Flags).getReg(0);
4315 auto NegDn =
B.buildFNeg(
F64, Dn, Flags).getReg(0);
4316 auto C2 =
B.buildFConstant(
F64,
APFloat(-0x1.9dc1da994fd21p-59));
4317 auto C3 =
B.buildFConstant(
F64,
APFloat(0x1.34413509f79ffp-2));
4318 auto Inner =
B.buildFMA(
F64, NegDn, C3,
X, Flags).getReg(0);
4319 F =
B.buildFMA(
F64, NegDn, C2, Inner, Flags).getReg(0);
4321 auto C4 =
B.buildFConstant(
F64,
APFloat(0x1.26bb1bbb55516p+1));
4322 auto C5 =
B.buildFConstant(
F64,
APFloat(-0x1.f48ad494ea3e9p-53));
4323 auto MulF =
B.buildFMul(
F64,
F, C5, Flags).getReg(0);
4324 T =
B.buildFMA(
F64,
F, C4, MulF, Flags).getReg(0);
4327 auto C1 =
B.buildFConstant(
F64,
APFloat(0x1.71547652b82fep+0));
4328 auto Mul =
B.buildFMul(
F64,
X,
C1, Flags).getReg(0);
4329 Dn =
B.buildFRint(
F64,
Mul, Flags).getReg(0);
4331 auto NegDn =
B.buildFNeg(
F64, Dn, Flags).getReg(0);
4332 auto C2 =
B.buildFConstant(
F64,
APFloat(0x1.abc9e3b39803fp-56));
4333 auto C3 =
B.buildFConstant(
F64,
APFloat(0x1.62e42fefa39efp-1));
4334 auto Inner =
B.buildFMA(
F64, NegDn, C3,
X, Flags).getReg(0);
4335 T =
B.buildFMA(
F64, NegDn, C2, Inner, Flags).getReg(0);
4339 auto P =
B.buildFConstant(
F64, 0x1.ade156a5dcb37p-26);
4340 P =
B.buildFMA(
F64,
T,
P,
B.buildFConstant(
F64, 0x1.28af3fca7ab0cp-22),
4342 P =
B.buildFMA(
F64,
T,
P,
B.buildFConstant(
F64, 0x1.71dee623fde64p-19),
4344 P =
B.buildFMA(
F64,
T,
P,
B.buildFConstant(
F64, 0x1.a01997c89e6b0p-16),
4346 P =
B.buildFMA(
F64,
T,
P,
B.buildFConstant(
F64, 0x1.a01a014761f6ep-13),
4348 P =
B.buildFMA(
F64,
T,
P,
B.buildFConstant(
F64, 0x1.6c16c1852b7b0p-10),
4350 P =
B.buildFMA(
F64,
T,
P,
B.buildFConstant(
F64, 0x1.1111111122322p-7), Flags);
4351 P =
B.buildFMA(
F64,
T,
P,
B.buildFConstant(
F64, 0x1.55555555502a1p-5), Flags);
4352 P =
B.buildFMA(
F64,
T,
P,
B.buildFConstant(
F64, 0x1.5555555555511p-3), Flags);
4353 P =
B.buildFMA(
F64,
T,
P,
B.buildFConstant(
F64, 0x1.000000000000bp-1), Flags);
4355 auto One =
B.buildFConstant(
F64, 1.0);
4356 P =
B.buildFMA(
F64,
T,
P, One, Flags);
4357 P =
B.buildFMA(
F64,
T,
P, One, Flags);
4360 auto DnInt =
B.buildFPTOSI(I32, Dn);
4361 auto Z =
B.buildFLdexp(
F64,
P, DnInt, Flags);
4368 Z =
B.buildSelect(
F64, CondHi, Z, PInf, Flags);
4375 B.buildSelect(
MI.getOperand(0).getReg(), CondLo, Z,
Zero, Flags);
4377 MI.eraseFromParent();
4385 const unsigned Flags =
MI.getFlags();
4388 LLT Ty =
MRI.getType(Dst);
4393 const bool IsExp10 =
MI.getOpcode() == TargetOpcode::G_FEXP10;
4401 MI.eraseFromParent();
4412 auto Ext =
B.buildFPExt(
F32,
X, Flags);
4415 B.buildFPTrunc(Dst, Lowered, Flags);
4416 MI.eraseFromParent();
4427 MI.eraseFromParent();
4455 const unsigned FlagsNoContract = Flags &
~MachineInstr::FmContract;
4458 if (ST.hasFastFMAF32()) {
4460 const float cc_exp = 0x1.4ae0bep-26f;
4461 const float c_exp10 = 0x1.a934f0p+1f;
4462 const float cc_exp10 = 0x1.2f346ep-24f;
4464 auto C =
B.buildFConstant(Ty, IsExp10 ? c_exp10 : c_exp);
4465 PH =
B.buildFMul(Ty,
X,
C, Flags).getReg(0);
4466 auto NegPH =
B.buildFNeg(Ty, PH, Flags);
4467 auto FMA0 =
B.buildFMA(Ty,
X,
C, NegPH, Flags);
4469 auto CC =
B.buildFConstant(Ty, IsExp10 ? cc_exp10 : cc_exp);
4470 PL =
B.buildFMA(Ty,
X,
CC, FMA0, Flags).getReg(0);
4472 const float ch_exp = 0x1.714000p+0f;
4473 const float cl_exp = 0x1.47652ap-12f;
4475 const float ch_exp10 = 0x1.a92000p+1f;
4476 const float cl_exp10 = 0x1.4f0978p-11f;
4479 auto XInt =
B.buildBitcast(I32,
X);
4480 auto MaskConst =
B.buildConstant(I32, 0xfffff000);
4481 auto XH =
B.buildBitcast(Ty,
B.buildAnd(I32, XInt, MaskConst));
4482 auto XL =
B.buildFSub(Ty,
X, XH, Flags);
4484 auto CH =
B.buildFConstant(Ty, IsExp10 ? ch_exp10 : ch_exp);
4485 PH =
B.buildFMul(Ty, XH,
CH, Flags).getReg(0);
4487 auto CL =
B.buildFConstant(Ty, IsExp10 ? cl_exp10 : cl_exp);
4488 auto XLCL =
B.buildFMul(Ty, XL, CL, Flags);
4491 getMad(
B, Ty, XL.getReg(0),
CH.getReg(0), XLCL.getReg(0), Flags);
4492 PL =
getMad(
B, Ty, XH.getReg(0), CL.getReg(0), Mad0, Flags);
4495 auto E =
B.buildIntrinsicRoundeven(Ty, PH, Flags);
4498 auto PHSubE =
B.buildFSub(Ty, PH, E, FlagsNoContract);
4499 auto A =
B.buildFAdd(Ty, PHSubE, PL, Flags);
4501 auto IntE =
B.buildFPTOSI(I32, E);
4503 auto Exp2 =
B.buildIntrinsic(Intrinsic::amdgcn_exp2, {Ty})
4504 .addUse(
A.getReg(0))
4506 auto R =
B.buildFLdexp(Ty, Exp2, IntE, Flags);
4508 auto UnderflowCheckConst =
4509 B.buildFConstant(Ty, IsExp10 ? -0x1.66d3e8p+5f : -0x1.9d1da0p+6f);
4510 auto Zero =
B.buildFConstant(Ty, 0.0);
4514 R =
B.buildSelect(Ty, Underflow,
Zero, R);
4517 auto OverflowCheckConst =
4518 B.buildFConstant(Ty, IsExp10 ? 0x1.344136p+5f : 0x1.62e430p+6f);
4523 R =
B.buildSelect(Ty, Overflow, Inf, R, Flags);
4526 B.buildCopy(Dst, R);
4527 MI.eraseFromParent();
4538 unsigned Flags =
MI.getFlags();
4547 auto Log =
B.buildIntrinsic(Intrinsic::amdgcn_log, {
F32})
4549 .setMIFlags(CoreFlags);
4550 auto Mul =
B.buildIntrinsic(Intrinsic::amdgcn_fmul_legacy, {
F32})
4552 .addUse(Log.getReg(0))
4553 .setMIFlags(CoreFlags);
4555 MI.eraseFromParent();
4559 auto Abs =
B.buildFAbs(
F32,
X, Flags);
4560 auto Log =
B.buildFLog2(
F32, Abs, CoreFlags);
4561 auto Mul =
B.buildIntrinsic(Intrinsic::amdgcn_fmul_legacy, {
F32})
4563 .addUse(Log.getReg(0))
4564 .setMIFlags(CoreFlags);
4569 .signBitIsZeroOrNaN()) {
4570 B.buildFExp2(Dst,
Mul, CoreFlags);
4571 MI.eraseFromParent();
4577 auto YTrunc =
B.buildIntrinsicTrunc(
F32,
Y);
4579 auto YHalf =
B.buildFMul(
F32,
Y,
B.buildFConstant(
F32, 0.5));
4580 auto YHalfTrunc =
B.buildIntrinsicTrunc(
F32, YHalf);
4581 auto YIsOdd =
B.buildAnd(
4585 auto Neg =
B.buildFCopysign(
F32, R,
X);
4587 B.buildSelect(Dst, YIsOdd, Neg, R);
4588 MI.eraseFromParent();
4591 R =
B.buildSelect(
F32, YIsOdd, Neg, R).getReg(0);
4598 auto XNegFinite =
B.buildIsFPClass(
S1,
X, NegFiniteMask);
4600 auto NegNonInt =
B.buildAnd(
S1, XNegFinite,
B.buildNot(
S1, YIsInt));
4602 B.buildSelect(Dst, NegNonInt, NaN, R);
4604 MI.eraseFromParent();
4612 ModSrc = SrcFNeg->getOperand(1).getReg();
4614 ModSrc = SrcFAbs->getOperand(1).getReg();
4616 ModSrc = SrcFAbs->getOperand(1).getReg();
4626 Register OrigSrc =
MI.getOperand(1).getReg();
4627 unsigned Flags =
MI.getFlags();
4629 "this should not have been custom lowered");
4639 auto Fract =
B.buildIntrinsic(Intrinsic::amdgcn_fract, {
F64})
4658 if (MFI->
getMode().
IEEE && !ST.hasIEEEMinimumMaximumInsts())
4659 B.buildFMinNumIEEE(Min, Fract, Const, Flags);
4661 B.buildFMinNum(Min, Fract, Const, Flags);
4666 CorrectedFract =
B.buildSelect(
F64, IsNan, ModSrc, Min, Flags).getReg(0);
4669 auto NegFract =
B.buildFNeg(
F64, CorrectedFract, Flags);
4670 B.buildFAdd(Dst, OrigSrc, NegFract, Flags);
4672 MI.eraseFromParent();
4684 MRI.getType(Dst).getNumElements() == 2 &&
4685 MRI.getType(Dst).getScalarSizeInBits() == 16);
4690 if (
MI.getOpcode() == AMDGPU::G_BUILD_VECTOR_TRUNC) {
4692 Src0 =
B.buildTrunc(I16,
MI.getOperand(1).getReg()).getReg(0);
4693 Src1 =
B.buildTrunc(I16,
MI.getOperand(2).getReg()).getReg(0);
4696 auto Merge =
B.buildMergeLikeInstr(I32, {Src0, Src1});
4697 B.buildBitcast(Dst,
Merge);
4699 MI.eraseFromParent();
4716 bool UsePartialMad64_32,
4717 bool SeparateOddAlignedProducts)
const {
4732 auto getZero32 = [&]() ->
Register {
4734 Zero32 =
B.buildConstant(I32, 0).getReg(0);
4737 auto getZero64 = [&]() ->
Register {
4739 Zero64 =
B.buildConstant(I64, 0).getReg(0);
4744 for (
unsigned i = 0; i < Src0.size(); ++i) {
4755 if (CarryIn.empty())
4758 bool HaveCarryOut =
true;
4760 if (CarryIn.size() == 1) {
4762 LocalAccum =
B.buildZExt(I32, CarryIn[0]).getReg(0);
4766 CarryAccum = getZero32();
4768 CarryAccum =
B.buildZExt(I32, CarryIn[0]).getReg(0);
4769 for (
unsigned i = 1; i + 1 < CarryIn.size(); ++i) {
4771 B.buildUAdde(I32,
S1, CarryAccum, getZero32(), CarryIn[i])
4776 LocalAccum = getZero32();
4777 HaveCarryOut =
false;
4782 B.buildUAdde(I32,
S1, CarryAccum, LocalAccum, CarryIn.back());
4783 LocalAccum =
Add.getReg(0);
4797 auto buildMadChain =
4800 assert((DstIndex + 1 < Accum.
size() && LocalAccum.size() == 2) ||
4801 (DstIndex + 1 >= Accum.
size() && LocalAccum.size() == 1));
4808 if (LocalAccum.size() == 1 &&
4809 (!UsePartialMad64_32 || !CarryIn.empty())) {
4812 unsigned j1 = DstIndex - j0;
4813 if (Src0KnownZeros[j0] || Src1KnownZeros[j1]) {
4817 auto Mul =
B.buildMul(I32, Src0[j0], Src1[j1]);
4819 LocalAccum[0] =
Mul.getReg(0);
4821 if (CarryIn.empty()) {
4822 LocalAccum[0] =
B.buildAdd(I32, LocalAccum[0],
Mul).getReg(0);
4825 B.buildUAdde(I32,
S1, LocalAccum[0],
Mul, CarryIn.back())
4831 }
while (j0 <= DstIndex && (!UsePartialMad64_32 || !CarryIn.empty()));
4835 if (j0 <= DstIndex) {
4836 bool HaveSmallAccum =
false;
4839 if (LocalAccum[0]) {
4840 if (LocalAccum.size() == 1) {
4841 Tmp =
B.buildAnyExt(I64, LocalAccum[0]).getReg(0);
4842 HaveSmallAccum =
true;
4843 }
else if (LocalAccum[1]) {
4844 Tmp =
B.buildMergeLikeInstr(I64, LocalAccum).getReg(0);
4845 HaveSmallAccum =
false;
4847 Tmp =
B.buildZExt(I64, LocalAccum[0]).getReg(0);
4848 HaveSmallAccum =
true;
4851 assert(LocalAccum.size() == 1 || !LocalAccum[1]);
4853 HaveSmallAccum =
true;
4857 unsigned j1 = DstIndex - j0;
4858 if (Src0KnownZeros[j0] || Src1KnownZeros[j1]) {
4862 auto Mad =
B.buildInstr(AMDGPU::G_AMDGPU_MAD_U64_U32, {I64,
S1},
4863 {Src0[j0], Src1[j1], Tmp});
4864 Tmp = Mad.getReg(0);
4865 if (!HaveSmallAccum)
4866 CarryOut.push_back(Mad.getReg(1));
4867 HaveSmallAccum =
false;
4870 }
while (j0 <= DstIndex);
4872 auto Unmerge =
B.buildUnmerge(I32, Tmp);
4873 LocalAccum[0] = Unmerge.getReg(0);
4874 if (LocalAccum.size() > 1)
4875 LocalAccum[1] = Unmerge.getReg(1);
4882 LocalAccum[0] = getZero32();
4886 assert((LocalAccum.size() == 1 || LocalAccum[1]) &&
4887 "Uninitialized accumulator part");
4913 for (
unsigned i = 0; i <= Accum.
size() / 2; ++i) {
4914 Carry OddCarryIn = std::move(OddCarry);
4915 Carry EvenCarryIn = std::move(EvenCarry);
4920 if (2 * i < Accum.
size()) {
4921 auto LocalAccum = Accum.
drop_front(2 * i).take_front(2);
4922 EvenCarry = buildMadChain(LocalAccum, 2 * i, EvenCarryIn);
4927 if (!SeparateOddAlignedProducts) {
4928 auto LocalAccum = Accum.
drop_front(2 * i - 1).take_front(2);
4929 OddCarry = buildMadChain(LocalAccum, 2 * i - 1, OddCarryIn);
4931 bool IsHighest = 2 * i >= Accum.
size();
4934 .take_front(IsHighest ? 1 : 2);
4935 OddCarry = buildMadChain(LocalAccum, 2 * i - 1, OddCarryIn);
4941 Lo =
B.buildUAddo(I32,
S1, Accum[2 * i - 1], SeparateOddOut[0]);
4943 Lo =
B.buildAdd(I32, Accum[2 * i - 1], SeparateOddOut[0]);
4945 Lo =
B.buildUAdde(I32,
S1, Accum[2 * i - 1], SeparateOddOut[0],
4948 Accum[2 * i - 1] =
Lo->getOperand(0).getReg();
4951 auto Hi =
B.buildUAdde(I32,
S1, Accum[2 * i], SeparateOddOut[1],
4952 Lo->getOperand(1).getReg());
4953 Accum[2 * i] =
Hi.getReg(0);
4954 SeparateOddCarry =
Hi.getReg(1);
4961 if (
Register CarryOut = mergeCarry(Accum[2 * i - 1], OddCarryIn))
4962 EvenCarryIn.push_back(CarryOut);
4964 if (2 * i < Accum.
size()) {
4965 if (
Register CarryOut = mergeCarry(Accum[2 * i], EvenCarryIn))
4966 OddCarry.push_back(CarryOut);
4978 assert(ST.hasMad64_32());
4979 assert(
MI.getOpcode() == TargetOpcode::G_MUL);
4988 LLT Ty =
MRI.getType(DstReg);
4991 unsigned Size = Ty.getSizeInBits();
4992 if (ST.useVMulU64Inst() &&
Size == 64)
5007 const bool SeparateOddAlignedProducts = ST.hasFullRate64Ops();
5011 for (
unsigned i = 0; i <
NumParts; ++i) {
5012 Src0Parts.
push_back(
MRI.createGenericVirtualRegister(I32));
5013 Src1Parts.
push_back(
MRI.createGenericVirtualRegister(I32));
5015 B.buildUnmerge(Src0Parts, Src0);
5016 B.buildUnmerge(Src1Parts, Src1);
5019 buildMultiply(Helper, AccumRegs, Src0Parts, Src1Parts, UsePartialMad64_32,
5020 SeparateOddAlignedProducts);
5022 B.buildMergeLikeInstr(DstReg, AccumRegs);
5023 MI.eraseFromParent();
5035 LLT DstTy =
MRI.getType(Dst);
5036 LLT SrcTy =
MRI.getType(Src);
5038 unsigned NewOpc =
MI.getOpcode() == AMDGPU::G_CTLZ
5039 ? AMDGPU::G_AMDGPU_FFBH_U32
5040 : AMDGPU::G_AMDGPU_FFBL_B32;
5041 auto Tmp =
B.buildInstr(NewOpc, {DstTy}, {Src});
5044 MI.eraseFromParent();
5053 LLT SrcTy =
MRI.getType(Src);
5054 TypeSize NumBits = SrcTy.getSizeInBits();
5059 auto ShiftAmt =
B.buildConstant(I32, 32u - NumBits);
5060 auto Extend =
B.buildAnyExt(I32, {Src}).
getReg(0u);
5061 auto Shift =
B.buildShl(I32, Extend, ShiftAmt);
5062 auto Ctlz =
B.buildInstr(AMDGPU::G_AMDGPU_FFBH_U32, {I32}, {Shift});
5063 B.buildTrunc(Dst, Ctlz);
5064 MI.eraseFromParent();
5073 LLT SrcTy =
MRI.getType(Src);
5075 assert(SrcTy == I32 &&
"legalizeCTLS only supports i32");
5076 unsigned BitWidth = SrcTy.getSizeInBits();
5078 auto Sffbh =
B.buildIntrinsic(Intrinsic::amdgcn_sffbh, {I32}).addUse(Src);
5079 auto Clamped =
B.buildUMin(I32, Sffbh,
B.buildConstant(I32,
BitWidth));
5080 B.buildSub(Dst, Clamped,
B.buildConstant(I32, 1));
5081 MI.eraseFromParent();
5087 if (
MI.getOpcode() != TargetOpcode::G_XOR)
5090 return ConstVal == -1;
5097 Register CondDef =
MI.getOperand(0).getReg();
5098 if (!
MRI.hasOneNonDBGUse(CondDef))
5106 if (!
MRI.hasOneNonDBGUse(NegatedCond))
5112 UseMI = &*
MRI.use_instr_nodbg_begin(NegatedCond);
5116 if (
UseMI->getParent() != Parent ||
UseMI->getOpcode() != AMDGPU::G_BRCOND)
5125 UncondBrTarget = &*NextMBB;
5127 if (
Next->getOpcode() != AMDGPU::G_BR)
5146 *ArgRC,
B.getDebugLoc(), ArgTy);
5150 const unsigned Mask = Arg->
getMask();
5158 auto ShiftAmt =
B.buildConstant(I32, Shift);
5159 AndMaskSrc =
B.buildLShr(I32, LiveIn, ShiftAmt).getReg(0);
5162 B.buildAnd(DstReg, AndMaskSrc,
B.buildConstant(I32, Mask >> Shift));
5164 B.buildCopy(DstReg, LiveIn);
5174 if (!ST.hasClusters()) {
5177 MI.eraseFromParent();
5189 Register ClusterIdXYZ =
MRI.createGenericVirtualRegister(I32);
5190 Register ClusterMaxIdXYZ =
MRI.createGenericVirtualRegister(I32);
5191 Register ClusterWorkGroupIdXYZ =
MRI.createGenericVirtualRegister(I32);
5197 auto One =
B.buildConstant(I32, 1);
5198 auto ClusterSizeXYZ =
B.buildAdd(I32, ClusterMaxIdXYZ, One);
5199 auto GlobalIdXYZ =
B.buildAdd(I32, ClusterWorkGroupIdXYZ,
5200 B.buildMul(I32, ClusterIdXYZ, ClusterSizeXYZ));
5207 B.buildCopy(DstReg, GlobalIdXYZ);
5208 MI.eraseFromParent();
5212 B.buildCopy(DstReg, ClusterIdXYZ);
5213 MI.eraseFromParent();
5218 unsigned ClusterIdField = HwregEncoding::encode(ID_IB_STS2, 6, 4);
5219 Register ClusterId =
MRI.createGenericVirtualRegister(I32);
5220 MRI.setRegClass(ClusterId, &AMDGPU::SReg_32RegClass);
5221 B.buildInstr(AMDGPU::S_GETREG_B32_const)
5223 .addImm(ClusterIdField);
5224 auto Zero =
B.buildConstant(I32, 0);
5227 B.buildSelect(DstReg, NoClusters, ClusterIdXYZ, GlobalIdXYZ);
5228 MI.eraseFromParent();
5270 auto LoadConstant = [&](
unsigned N) {
5271 B.buildConstant(DstReg,
N);
5275 if (ST.hasArchitectedSGPRs() &&
5283 Arg = &WorkGroupIDX;
5284 ArgRC = &AMDGPU::SReg_32RegClass;
5288 Arg = &WorkGroupIDY;
5289 ArgRC = &AMDGPU::SReg_32RegClass;
5293 Arg = &WorkGroupIDZ;
5294 ArgRC = &AMDGPU::SReg_32RegClass;
5298 if (HasFixedDims && ClusterDims.
getDims()[0] == 1)
5299 return LoadConstant(0);
5300 Arg = &ClusterWorkGroupIDX;
5301 ArgRC = &AMDGPU::SReg_32RegClass;
5305 if (HasFixedDims && ClusterDims.
getDims()[1] == 1)
5306 return LoadConstant(0);
5307 Arg = &ClusterWorkGroupIDY;
5308 ArgRC = &AMDGPU::SReg_32RegClass;
5312 if (HasFixedDims && ClusterDims.
getDims()[2] == 1)
5313 return LoadConstant(0);
5314 Arg = &ClusterWorkGroupIDZ;
5315 ArgRC = &AMDGPU::SReg_32RegClass;
5320 return LoadConstant(ClusterDims.
getDims()[0] - 1);
5321 Arg = &ClusterWorkGroupMaxIDX;
5322 ArgRC = &AMDGPU::SReg_32RegClass;
5327 return LoadConstant(ClusterDims.
getDims()[1] - 1);
5328 Arg = &ClusterWorkGroupMaxIDY;
5329 ArgRC = &AMDGPU::SReg_32RegClass;
5334 return LoadConstant(ClusterDims.
getDims()[2] - 1);
5335 Arg = &ClusterWorkGroupMaxIDZ;
5336 ArgRC = &AMDGPU::SReg_32RegClass;
5340 Arg = &ClusterWorkGroupMaxFlatID;
5341 ArgRC = &AMDGPU::SReg_32RegClass;
5356 return LoadConstant(0);
5361 B.buildUndef(DstReg);
5365 if (!Arg->isRegister() || !Arg->getRegister().isValid())
5377 MI.eraseFromParent();
5383 B.buildConstant(
MI.getOperand(0).getReg(),
C);
5384 MI.eraseFromParent();
5391 unsigned MaxID = ST.getMaxWorkitemID(
B.getMF().getFunction(), Dim);
5405 B.buildUndef(DstReg);
5406 MI.eraseFromParent();
5410 if (Arg->isMasked()) {
5424 MI.eraseFromParent();
5439 Register KernArgReg =
B.getMRI()->createGenericVirtualRegister(PtrTy);
5448 return B.buildObjectPtrOffset(PtrTy, KernArgReg, COffset).getReg(0);
5456 Align Alignment)
const {
5460 "unexpected kernarg parameter type");
5467 MI.eraseFromParent();
5475 LLT DstTy =
MRI.getType(Dst);
5499 auto FloatY =
B.buildUITOFP(
F32,
Y);
5500 auto RcpIFlag =
B.buildInstr(AMDGPU::G_AMDGPU_RCP_IFLAG, {
F32}, {FloatY});
5502 auto ScaledY =
B.buildFMul(
F32, RcpIFlag, Scale);
5503 auto Z =
B.buildFPTOUI(I32, ScaledY);
5506 auto NegY =
B.buildSub(I32,
B.buildConstant(I32, 0),
Y);
5507 auto NegYZ =
B.buildMul(I32, NegY, Z);
5508 Z =
B.buildAdd(I32, Z,
B.buildUMulH(I32, Z, NegYZ));
5511 auto Q =
B.buildUMulH(I32,
X, Z);
5512 auto R =
B.buildSub(I32,
X,
B.buildMul(I32, Q,
Y));
5515 auto One =
B.buildConstant(I32, 1);
5518 Q =
B.buildSelect(I32,
Cond,
B.buildAdd(I32, Q, One), Q);
5519 R =
B.buildSelect(I32,
Cond,
B.buildSub(I32, R,
Y), R);
5524 B.buildSelect(DstDivReg,
Cond,
B.buildAdd(I32, Q, One), Q);
5527 B.buildSelect(DstRemReg,
Cond,
B.buildSub(I32, R,
Y), R);
5546 auto Unmerge =
B.buildUnmerge(I32, Val);
5548 auto CvtLo =
B.buildUITOFP(
F32, Unmerge.getReg(0));
5549 auto CvtHi =
B.buildUITOFP(
F32, Unmerge.getReg(1));
5551 auto Mad =
B.buildFMAD(
5555 auto Rcp =
B.buildInstr(AMDGPU::G_AMDGPU_RCP_IFLAG, {
F32}, {Mad});
5556 auto Mul1 =
B.buildFMul(
5560 auto Mul2 =
B.buildFMul(
5562 auto Trunc =
B.buildIntrinsicTrunc(
F32, Mul2);
5565 auto Mad2 =
B.buildFMAD(
5569 auto ResultLo =
B.buildFPTOUI(I32, Mad2);
5570 auto ResultHi =
B.buildFPTOUI(I32, Trunc);
5572 return {ResultLo.getReg(0), ResultHi.getReg(0)};
5587 auto Rcp =
B.buildMergeLikeInstr(I64, {RcpLo, RcpHi});
5589 auto Zero64 =
B.buildConstant(I64, 0);
5590 auto NegDenom =
B.buildSub(I64, Zero64, Denom);
5592 auto MulLo1 =
B.buildMul(I64, NegDenom, Rcp);
5593 auto MulHi1 =
B.buildUMulH(I64, Rcp, MulLo1);
5595 auto UnmergeMulHi1 =
B.buildUnmerge(I32, MulHi1);
5596 Register MulHi1_Lo = UnmergeMulHi1.getReg(0);
5597 Register MulHi1_Hi = UnmergeMulHi1.getReg(1);
5599 auto Add1_Lo =
B.buildUAddo(I32,
S1, RcpLo, MulHi1_Lo);
5600 auto Add1_Hi =
B.buildUAdde(I32,
S1, RcpHi, MulHi1_Hi, Add1_Lo.getReg(1));
5601 auto Add1 =
B.buildMergeLikeInstr(I64, {Add1_Lo, Add1_Hi});
5603 auto MulLo2 =
B.buildMul(I64, NegDenom, Add1);
5604 auto MulHi2 =
B.buildUMulH(I64, Add1, MulLo2);
5605 auto UnmergeMulHi2 =
B.buildUnmerge(I32, MulHi2);
5606 Register MulHi2_Lo = UnmergeMulHi2.getReg(0);
5607 Register MulHi2_Hi = UnmergeMulHi2.getReg(1);
5609 auto Zero32 =
B.buildConstant(I32, 0);
5610 auto Add2_Lo =
B.buildUAddo(I32,
S1, Add1_Lo, MulHi2_Lo);
5611 auto Add2_Hi =
B.buildUAdde(I32,
S1, Add1_Hi, MulHi2_Hi, Add2_Lo.getReg(1));
5612 auto Add2 =
B.buildMergeLikeInstr(I64, {Add2_Lo, Add2_Hi});
5614 auto UnmergeNumer =
B.buildUnmerge(I32, Numer);
5615 Register NumerLo = UnmergeNumer.getReg(0);
5616 Register NumerHi = UnmergeNumer.getReg(1);
5618 auto MulHi3 =
B.buildUMulH(I64, Numer, Add2);
5619 auto Mul3 =
B.buildMul(I64, Denom, MulHi3);
5620 auto UnmergeMul3 =
B.buildUnmerge(I32, Mul3);
5621 Register Mul3_Lo = UnmergeMul3.getReg(0);
5622 Register Mul3_Hi = UnmergeMul3.getReg(1);
5623 auto Sub1_Lo =
B.buildUSubo(I32,
S1, NumerLo, Mul3_Lo);
5624 auto Sub1_Hi =
B.buildUSube(I32,
S1, NumerHi, Mul3_Hi, Sub1_Lo.getReg(1));
5625 auto Sub1_Mi =
B.buildSub(I32, NumerHi, Mul3_Hi);
5626 auto Sub1 =
B.buildMergeLikeInstr(I64, {Sub1_Lo, Sub1_Hi});
5628 auto UnmergeDenom =
B.buildUnmerge(I32, Denom);
5629 Register DenomLo = UnmergeDenom.getReg(0);
5630 Register DenomHi = UnmergeDenom.getReg(1);
5633 auto C1 =
B.buildSExt(I32, CmpHi);
5636 auto C2 =
B.buildSExt(I32, CmpLo);
5639 auto C3 =
B.buildSelect(I32, CmpEq, C2,
C1);
5646 auto Sub2_Lo =
B.buildUSubo(I32,
S1, Sub1_Lo, DenomLo);
5647 auto Sub2_Mi =
B.buildUSube(I32,
S1, Sub1_Mi, DenomHi, Sub1_Lo.getReg(1));
5648 auto Sub2_Hi =
B.buildUSube(I32,
S1, Sub2_Mi, Zero32, Sub2_Lo.getReg(1));
5649 auto Sub2 =
B.buildMergeLikeInstr(I64, {Sub2_Lo, Sub2_Hi});
5651 auto One64 =
B.buildConstant(I64, 1);
5652 auto Add3 =
B.buildAdd(I64, MulHi3, One64);
5658 auto C6 =
B.buildSelect(
5662 auto Add4 =
B.buildAdd(I64, Add3, One64);
5663 auto Sub3_Lo =
B.buildUSubo(I32,
S1, Sub2_Lo, DenomLo);
5665 auto Sub3_Mi =
B.buildUSube(I32,
S1, Sub2_Mi, DenomHi, Sub2_Lo.getReg(1));
5666 auto Sub3_Hi =
B.buildUSube(I32,
S1, Sub3_Mi, Zero32, Sub3_Lo.getReg(1));
5667 auto Sub3 =
B.buildMergeLikeInstr(I64, {Sub3_Lo, Sub3_Hi});
5673 auto Sel1 =
B.buildSelect(
5680 auto Sel2 =
B.buildSelect(
5691 switch (
MI.getOpcode()) {
5694 case AMDGPU::G_UDIV: {
5695 DstDivReg =
MI.getOperand(0).getReg();
5698 case AMDGPU::G_UREM: {
5699 DstRemReg =
MI.getOperand(0).getReg();
5702 case AMDGPU::G_UDIVREM: {
5703 DstDivReg =
MI.getOperand(0).getReg();
5704 DstRemReg =
MI.getOperand(1).getReg();
5711 const unsigned FirstSrcOpIdx =
MI.getNumExplicitDefs();
5712 Register Num =
MI.getOperand(FirstSrcOpIdx).getReg();
5713 Register Den =
MI.getOperand(FirstSrcOpIdx + 1).getReg();
5714 LLT Ty =
MRI.getType(
MI.getOperand(0).getReg());
5723 MI.eraseFromParent();
5733 LLT Ty =
MRI.getType(
MI.getOperand(0).getReg());
5734 if (Ty != I32 && Ty != I64)
5737 const unsigned FirstSrcOpIdx =
MI.getNumExplicitDefs();
5738 Register LHS =
MI.getOperand(FirstSrcOpIdx).getReg();
5739 Register RHS =
MI.getOperand(FirstSrcOpIdx + 1).getReg();
5741 auto SignBitOffset =
B.buildConstant(I32, Ty.getSizeInBits() - 1);
5742 auto LHSign =
B.buildAShr(Ty, LHS, SignBitOffset);
5743 auto RHSign =
B.buildAShr(Ty, RHS, SignBitOffset);
5745 LHS =
B.buildAdd(Ty, LHS, LHSign).getReg(0);
5746 RHS =
B.buildAdd(Ty, RHS, RHSign).getReg(0);
5748 LHS =
B.buildXor(Ty, LHS, LHSign).getReg(0);
5749 RHS =
B.buildXor(Ty, RHS, RHSign).getReg(0);
5751 Register DstDivReg, DstRemReg, TmpDivReg, TmpRemReg;
5752 switch (
MI.getOpcode()) {
5755 case AMDGPU::G_SDIV: {
5756 DstDivReg =
MI.getOperand(0).getReg();
5757 TmpDivReg =
MRI.createGenericVirtualRegister(Ty);
5760 case AMDGPU::G_SREM: {
5761 DstRemReg =
MI.getOperand(0).getReg();
5762 TmpRemReg =
MRI.createGenericVirtualRegister(Ty);
5765 case AMDGPU::G_SDIVREM: {
5766 DstDivReg =
MI.getOperand(0).getReg();
5767 DstRemReg =
MI.getOperand(1).getReg();
5768 TmpDivReg =
MRI.createGenericVirtualRegister(Ty);
5769 TmpRemReg =
MRI.createGenericVirtualRegister(Ty);
5780 auto Sign =
B.buildXor(Ty, LHSign, RHSign).getReg(0);
5781 auto SignXor =
B.buildXor(Ty, TmpDivReg,
Sign).getReg(0);
5782 B.buildSub(DstDivReg, SignXor,
Sign);
5786 auto Sign = LHSign.getReg(0);
5787 auto SignXor =
B.buildXor(Ty, TmpRemReg,
Sign).getReg(0);
5788 B.buildSub(DstRemReg, SignXor,
Sign);
5791 MI.eraseFromParent();
5801 uint16_t Flags =
MI.getFlags();
5802 LLT ResTy =
MRI.getType(Res);
5807 if (!AllowInaccurateRcp && ResTy !=
F16)
5818 if (CLHS->isOne()) {
5819 B.buildIntrinsic(Intrinsic::amdgcn_rcp, Res)
5823 MI.eraseFromParent();
5828 if (CLHS->isMinusOne()) {
5829 auto RCP =
B.buildIntrinsic(Intrinsic::amdgcn_rcp, {ResTy})
5832 B.buildFNeg(Res, RCP, Flags);
5834 MI.eraseFromParent();
5841 if (!AllowInaccurateRcp &&
5846 auto RCP =
B.buildIntrinsic(Intrinsic::amdgcn_rcp, {ResTy})
5849 B.buildFMul(Res, LHS, RCP, Flags);
5851 MI.eraseFromParent();
5861 uint16_t Flags =
MI.getFlags();
5862 LLT ResTy =
MRI.getType(Res);
5866 if (!AllowInaccurateRcp)
5874 X =
B.buildFConstant(ResTy, 1.0).getReg(0);
5876 Register NegY = IsNegRcp ?
Y :
B.buildFNeg(ResTy,
Y).getReg(0);
5877 auto One =
B.buildFConstant(ResTy, 1.0);
5879 auto R =
B.buildIntrinsic(Intrinsic::amdgcn_rcp, {ResTy})
5883 R =
B.buildFNeg(ResTy, R);
5885 auto Tmp0 =
B.buildFMA(ResTy, NegY, R, One);
5886 R =
B.buildFMA(ResTy, Tmp0, R, R);
5888 auto Tmp1 =
B.buildFMA(ResTy, NegY, R, One);
5889 R =
B.buildFMA(ResTy, Tmp1, R, R);
5892 if (IsNegRcp || (CLHS && CLHS->
isOne())) {
5893 B.buildCopy(Res, R);
5894 MI.eraseFromParent();
5898 auto Ret =
B.buildFMul(ResTy,
X, R);
5899 auto Tmp2 =
B.buildFMA(ResTy, NegY, Ret,
X);
5901 B.buildFMA(Res, Tmp2, R, Ret);
5902 MI.eraseFromParent();
5916 uint16_t Flags =
MI.getFlags();
5933 auto LHSExt =
B.buildFPExt(
F32, LHS, Flags);
5934 auto RHSExt =
B.buildFPExt(
F32, RHS, Flags);
5935 auto NegRHSExt =
B.buildFNeg(
F32, RHSExt);
5936 auto Rcp =
B.buildIntrinsic(Intrinsic::amdgcn_rcp, {
F32})
5937 .addUse(RHSExt.getReg(0))
5939 auto Quot =
B.buildFMul(
F32, LHSExt, Rcp, Flags);
5941 if (ST.hasMadMacF32Insts()) {
5942 Err =
B.buildFMAD(
F32, NegRHSExt, Quot, LHSExt, Flags);
5943 Quot =
B.buildFMAD(
F32, Err, Rcp, Quot, Flags);
5944 Err =
B.buildFMAD(
F32, NegRHSExt, Quot, LHSExt, Flags);
5946 Err =
B.buildFMA(
F32, NegRHSExt, Quot, LHSExt, Flags);
5947 Quot =
B.buildFMA(
F32, Err, Rcp, Quot, Flags);
5948 Err =
B.buildFMA(
F32, NegRHSExt, Quot, LHSExt, Flags);
5950 auto Tmp =
B.buildFMul(
F32, Err, Rcp, Flags);
5951 auto TmpInt =
B.buildBitcast(I32, Tmp);
5952 auto MaskedInt =
B.buildAnd(I32, TmpInt,
B.buildConstant(I32, 0xff800000));
5953 auto Masked =
B.buildBitcast(
F32, MaskedInt);
5954 Quot =
B.buildFAdd(
F32,
Masked, Quot, Flags);
5955 auto RDst =
B.buildFPTrunc(
F16, Quot, Flags);
5956 B.buildIntrinsic(Intrinsic::amdgcn_div_fixup, Res)
5957 .addUse(RDst.getReg(0))
5962 MI.eraseFromParent();
5975 unsigned SPDenormMode =
5978 if (ST.hasDenormModeInst()) {
5980 uint32_t DPDenormModeDefault =
Mode.fpDenormModeDPValue();
5982 uint32_t NewDenormModeValue = SPDenormMode | (DPDenormModeDefault << 2);
5983 B.buildInstr(AMDGPU::S_DENORM_MODE)
5984 .addImm(NewDenormModeValue);
5987 B.buildInstr(AMDGPU::S_SETREG_IMM32_B32)
5988 .addImm(SPDenormMode)
6005 uint16_t Flags =
MI.getFlags();
6009 auto One =
B.buildFConstant(
F32, 1.0f);
6011 auto DenominatorScaled =
6012 B.buildIntrinsic(Intrinsic::amdgcn_div_scale, {
F32,
S1})
6017 auto NumeratorScaled =
6018 B.buildIntrinsic(Intrinsic::amdgcn_div_scale, {
F32,
S1})
6024 auto ApproxRcp =
B.buildIntrinsic(Intrinsic::amdgcn_rcp, {
F32})
6025 .addUse(DenominatorScaled.getReg(0))
6027 auto NegDivScale0 =
B.buildFNeg(
F32, DenominatorScaled, Flags);
6030 const bool HasDynamicDenormals =
6035 if (!PreservesDenormals) {
6036 if (HasDynamicDenormals) {
6037 SavedSPDenormMode =
MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
6038 B.buildInstr(AMDGPU::S_GETREG_B32)
6039 .addDef(SavedSPDenormMode)
6045 auto Fma0 =
B.buildFMA(
F32, NegDivScale0, ApproxRcp, One, Flags);
6046 auto Fma1 =
B.buildFMA(
F32, Fma0, ApproxRcp, ApproxRcp, Flags);
6047 auto Mul =
B.buildFMul(
F32, NumeratorScaled, Fma1, Flags);
6048 auto Fma2 =
B.buildFMA(
F32, NegDivScale0,
Mul, NumeratorScaled, Flags);
6049 auto Fma3 =
B.buildFMA(
F32, Fma2, Fma1,
Mul, Flags);
6050 auto Fma4 =
B.buildFMA(
F32, NegDivScale0, Fma3, NumeratorScaled, Flags);
6052 if (!PreservesDenormals) {
6053 if (HasDynamicDenormals) {
6054 assert(SavedSPDenormMode);
6055 B.buildInstr(AMDGPU::S_SETREG_B32)
6056 .addReg(SavedSPDenormMode)
6062 auto Fmas =
B.buildIntrinsic(Intrinsic::amdgcn_div_fmas, {
F32})
6063 .addUse(Fma4.getReg(0))
6064 .addUse(Fma1.getReg(0))
6065 .addUse(Fma3.getReg(0))
6066 .addUse(NumeratorScaled.getReg(1))
6069 B.buildIntrinsic(Intrinsic::amdgcn_div_fixup, Res)
6070 .addUse(Fmas.getReg(0))
6075 MI.eraseFromParent();
6089 uint16_t Flags =
MI.getFlags();
6093 auto One =
B.buildFConstant(
F64, 1.0);
6095 auto DivScale0 =
B.buildIntrinsic(Intrinsic::amdgcn_div_scale, {
F64,
S1})
6101 auto NegDivScale0 =
B.buildFNeg(
F64, DivScale0.getReg(0), Flags);
6103 auto Rcp =
B.buildIntrinsic(Intrinsic::amdgcn_rcp, {
F64})
6104 .addUse(DivScale0.getReg(0))
6107 auto Fma0 =
B.buildFMA(
F64, NegDivScale0, Rcp, One, Flags);
6108 auto Fma1 =
B.buildFMA(
F64, Rcp, Fma0, Rcp, Flags);
6109 auto Fma2 =
B.buildFMA(
F64, NegDivScale0, Fma1, One, Flags);
6111 auto DivScale1 =
B.buildIntrinsic(Intrinsic::amdgcn_div_scale, {
F64,
S1})
6117 auto Fma3 =
B.buildFMA(
F64, Fma1, Fma2, Fma1, Flags);
6118 auto Mul =
B.buildFMul(
F64, DivScale1.getReg(0), Fma3, Flags);
6119 auto Fma4 =
B.buildFMA(
F64, NegDivScale0,
Mul, DivScale1.getReg(0), Flags);
6122 if (!ST.hasUsableDivScaleConditionOutput()) {
6129 auto NumUnmerge =
B.buildUnmerge(I32,
B.buildBitcast(I64, LHS));
6130 auto DenUnmerge =
B.buildUnmerge(I32,
B.buildBitcast(I64, RHS));
6131 auto Scale0Unmerge =
B.buildUnmerge(I32,
B.buildBitcast(I64, DivScale0));
6132 auto Scale1Unmerge =
B.buildUnmerge(I32,
B.buildBitcast(I64, DivScale1));
6135 Scale1Unmerge.getReg(1));
6137 Scale0Unmerge.getReg(1));
6138 Scale =
B.buildXor(
S1, CmpNum, CmpDen).getReg(0);
6140 Scale = DivScale1.getReg(1);
6143 auto Fmas =
B.buildIntrinsic(Intrinsic::amdgcn_div_fmas, {
F64})
6144 .addUse(Fma4.getReg(0))
6145 .addUse(Fma3.getReg(0))
6146 .addUse(
Mul.getReg(0))
6150 B.buildIntrinsic(Intrinsic::amdgcn_div_fixup,
ArrayRef(Res))
6151 .addUse(Fmas.getReg(0))
6156 MI.eraseFromParent();
6166 uint16_t Flags =
MI.getFlags();
6168 LLT Ty =
MRI.getType(Res0);
6171 auto Mant =
B.buildIntrinsic(Intrinsic::amdgcn_frexp_mant, {Ty})
6174 auto Exp =
B.buildIntrinsic(Intrinsic::amdgcn_frexp_exp, {InstrExpTy})
6178 if (ST.hasFractBug()) {
6179 auto Fabs =
B.buildFAbs(Ty, Val);
6183 auto Zero =
B.buildConstant(InstrExpTy, 0);
6184 Exp =
B.buildSelect(InstrExpTy, IsFinite, Exp,
Zero);
6185 Mant =
B.buildSelect(Ty, IsFinite, Mant, Val);
6188 B.buildCopy(Res0, Mant);
6189 B.buildSExtOrTrunc(Res1, Exp);
6191 MI.eraseFromParent();
6201 uint16_t Flags =
MI.getFlags();
6205 auto Abs =
B.buildFAbs(
F32, RHS, Flags);
6207 auto C0 =
B.buildFConstant(
F32, 0x1p+96f);
6208 auto C1 =
B.buildFConstant(
F32, 0x1p-32f);
6209 auto C2 =
B.buildFConstant(
F32, 1.0f);
6212 auto Sel =
B.buildSelect(
F32, CmpRes,
C1, C2, Flags);
6214 auto Mul0 =
B.buildFMul(
F32, RHS, Sel, Flags);
6216 auto RCP =
B.buildIntrinsic(Intrinsic::amdgcn_rcp, {
F32})
6217 .addUse(Mul0.getReg(0))
6220 auto Mul1 =
B.buildFMul(
F32, LHS, RCP, Flags);
6222 B.buildFMul(Res, Sel, Mul1, Flags);
6224 MI.eraseFromParent();
6233 unsigned Flags =
MI.getFlags();
6234 assert(!ST.has16BitInsts());
6235 auto Ext =
B.buildFPExt(
F32,
MI.getOperand(1), Flags);
6236 auto Log2 =
B.buildIntrinsic(Intrinsic::amdgcn_sqrt, {
F32})
6237 .addUse(Ext.getReg(0))
6239 B.buildFPTrunc(
MI.getOperand(0),
Log2, Flags);
6240 MI.eraseFromParent();
6250 const unsigned Flags =
MI.getFlags();
6258 MI.eraseFromParent();
6262 auto ScaleThreshold =
B.buildFConstant(
F32, 0x1.0p-96f);
6264 auto ScaleUpFactor =
B.buildFConstant(
F32, 0x1.0p+32f);
6265 auto ScaledX =
B.buildFMul(
F32,
X, ScaleUpFactor, Flags);
6266 auto SqrtX =
B.buildSelect(
F32, NeedScale, ScaledX,
X, Flags);
6271 .addUse(SqrtX.getReg(0))
6274 auto SqrtSInt =
B.buildBitcast(I32, SqrtS);
6275 auto NegOne =
B.buildConstant(I32, -1);
6276 auto SqrtSNextDown =
B.buildBitcast(
F32,
B.buildAdd(I32, SqrtSInt, NegOne));
6278 auto NegSqrtSNextDown =
B.buildFNeg(
F32, SqrtSNextDown, Flags);
6279 auto SqrtVP =
B.buildFMA(
F32, NegSqrtSNextDown, SqrtS, SqrtX, Flags);
6281 auto PosOne =
B.buildConstant(I32, 1);
6282 auto SqrtSNextUp =
B.buildBitcast(
F32,
B.buildAdd(I32, SqrtSInt, PosOne));
6284 auto NegSqrtSNextUp =
B.buildFNeg(
F32, SqrtSNextUp, Flags);
6285 auto SqrtVS =
B.buildFMA(
F32, NegSqrtSNextUp, SqrtS, SqrtX, Flags);
6287 auto Zero =
B.buildFConstant(
F32, 0.0f);
6291 B.buildSelect(
F32, SqrtVPLE0, SqrtSNextDown, SqrtS, Flags).getReg(0);
6295 B.buildSelect(
F32, SqrtVPVSGT0, SqrtSNextUp, SqrtS, Flags).getReg(0);
6298 B.buildIntrinsic(Intrinsic::amdgcn_rsq, {
F32}).addReg(SqrtX.getReg(0));
6299 B.buildFMul(SqrtS, SqrtX, SqrtR, Flags);
6301 auto Half =
B.buildFConstant(
F32, 0.5f);
6302 auto SqrtH =
B.buildFMul(
F32, SqrtR, Half, Flags);
6303 auto NegSqrtH =
B.buildFNeg(
F32, SqrtH, Flags);
6304 auto SqrtE =
B.buildFMA(
F32, NegSqrtH, SqrtS, Half, Flags);
6305 SqrtH =
B.buildFMA(
F32, SqrtH, SqrtE, SqrtH, Flags);
6306 SqrtS =
B.buildFMA(
F32, SqrtS, SqrtE, SqrtS, Flags).getReg(0);
6307 auto NegSqrtS =
B.buildFNeg(
F32, SqrtS, Flags);
6308 auto SqrtD =
B.buildFMA(
F32, NegSqrtS, SqrtS, SqrtX, Flags);
6309 SqrtS =
B.buildFMA(
F32, SqrtD, SqrtH, SqrtS, Flags).getReg(0);
6312 auto ScaleDownFactor =
B.buildFConstant(
F32, 0x1.0p-16f);
6314 auto ScaledDown =
B.buildFMul(
F32, SqrtS, ScaleDownFactor, Flags);
6316 SqrtS =
B.buildSelect(
F32, NeedScale, ScaledDown, SqrtS, Flags).getReg(0);
6319 B.buildSelect(Dst, IsZeroOrInf, SqrtX, SqrtS, Flags);
6321 MI.eraseFromParent();
6352 assert(
MRI.getType(Dst) ==
F64 &&
"only expect to lower f64 sqrt");
6355 unsigned Flags =
MI.getFlags();
6360 auto ScaleConstant =
B.buildFConstant(
F64, 0x1.0p-767);
6362 ZeroInt =
B.buildConstant(I32, 0).getReg(0);
6366 auto ScaleUpFactor =
B.buildConstant(I32, 256);
6367 auto ScaleUp =
B.buildSelect(I32, Scaling, ScaleUpFactor, ZeroInt);
6368 SqrtX =
B.buildFLdexp(
F64,
X, ScaleUp, Flags).getReg(0);
6371 auto SqrtY =
B.buildIntrinsic(Intrinsic::amdgcn_rsq, {
F64}).addReg(SqrtX);
6373 auto Half =
B.buildFConstant(
F64, 0.5);
6374 auto SqrtH0 =
B.buildFMul(
F64, SqrtY, Half);
6375 auto SqrtS0 =
B.buildFMul(
F64, SqrtX, SqrtY);
6377 auto NegSqrtH0 =
B.buildFNeg(
F64, SqrtH0);
6378 auto SqrtR0 =
B.buildFMA(
F64, NegSqrtH0, SqrtS0, Half);
6380 auto SqrtS1 =
B.buildFMA(
F64, SqrtS0, SqrtR0, SqrtS0);
6381 auto SqrtH1 =
B.buildFMA(
F64, SqrtH0, SqrtR0, SqrtH0);
6383 auto NegSqrtS1 =
B.buildFNeg(
F64, SqrtS1);
6384 auto SqrtD0 =
B.buildFMA(
F64, NegSqrtS1, SqrtS1, SqrtX);
6386 auto SqrtS2 =
B.buildFMA(
F64, SqrtD0, SqrtH1, SqrtS1);
6388 Register SqrtRet = SqrtS2.getReg(0);
6390 auto NegSqrtS2 =
B.buildFNeg(
F64, SqrtS2);
6391 auto SqrtD1 =
B.buildFMA(
F64, NegSqrtS2, SqrtS2, SqrtX);
6392 auto SqrtD2 =
B.buildFMA(
F64, SqrtD1, SqrtH1, SqrtS2);
6395 auto ScaleDownFactor =
B.buildConstant(I32, -128);
6396 auto ScaleDown =
B.buildSelect(I32, Scaling, ScaleDownFactor, ZeroInt);
6397 SqrtRet =
B.buildFLdexp(
F64, SqrtD2, ScaleDown, Flags).getReg(0);
6402 auto ZeroFP =
B.buildFConstant(
F64, 0.0);
6405 IsZeroOrInf =
B.buildIsFPClass(I1, SqrtX,
fcZero |
fcPosInf).getReg(0);
6411 B.buildSelect(Dst, IsZeroOrInf, SqrtX, SqrtRet, Flags);
6413 MI.eraseFromParent();
6420 LLT Ty =
MRI.getType(
MI.getOperand(0).getReg());
6444 auto Flags =
MI.getFlags();
6446 LLT Ty =
MRI.getType(Dst);
6456 auto Rsq =
B.buildIntrinsic(Intrinsic::amdgcn_rsq, {Ty})
6466 auto ClampMax = UseIEEE && !ST.hasIEEEMinimumMaximumInsts()
6467 ?
B.buildFMinNumIEEE(Ty, Rsq, MaxFlt, Flags)
6468 :
B.buildFMinNum(Ty, Rsq, MaxFlt, Flags);
6472 if (UseIEEE && !ST.hasIEEEMinimumMaximumInsts())
6473 B.buildFMaxNumIEEE(Dst, ClampMax, MinFlt, Flags);
6475 B.buildFMaxNum(Dst, ClampMax, MinFlt, Flags);
6476 MI.eraseFromParent();
6488 bool IsPermLane16 = IID == Intrinsic::amdgcn_permlane16 ||
6489 IID == Intrinsic::amdgcn_permlanex16;
6490 bool IsSetInactive = IID == Intrinsic::amdgcn_set_inactive ||
6491 IID == Intrinsic::amdgcn_set_inactive_chain_arg;
6492 bool IsPermlaneShuffle = IID == Intrinsic::amdgcn_permlane_bcast ||
6493 IID == Intrinsic::amdgcn_permlane_up ||
6494 IID == Intrinsic::amdgcn_permlane_down ||
6495 IID == Intrinsic::amdgcn_permlane_xor;
6499 auto LaneOp =
B.buildIntrinsic(IID, {VT}).addUse(Src0);
6501 case Intrinsic::amdgcn_readfirstlane:
6502 case Intrinsic::amdgcn_permlane64:
6503 return LaneOp.getReg(0);
6504 case Intrinsic::amdgcn_readlane:
6505 case Intrinsic::amdgcn_set_inactive:
6506 case Intrinsic::amdgcn_set_inactive_chain_arg:
6507 return LaneOp.addUse(Src1).getReg(0);
6508 case Intrinsic::amdgcn_writelane:
6509 case Intrinsic::amdgcn_permlane_bcast:
6510 case Intrinsic::amdgcn_permlane_up:
6511 case Intrinsic::amdgcn_permlane_down:
6512 case Intrinsic::amdgcn_permlane_xor:
6513 return LaneOp.addUse(Src1).addUse(Src2).getReg(0);
6514 case Intrinsic::amdgcn_permlane16:
6515 case Intrinsic::amdgcn_permlanex16: {
6517 int64_t Src4 =
MI.getOperand(6).getImm();
6518 int64_t Src5 =
MI.getOperand(7).getImm();
6519 return LaneOp.addUse(Src1)
6526 case Intrinsic::amdgcn_mov_dpp8:
6527 return LaneOp.addImm(
MI.getOperand(3).getImm()).getReg(0);
6528 case Intrinsic::amdgcn_update_dpp:
6529 return LaneOp.addUse(Src1)
6530 .addImm(
MI.getOperand(4).getImm())
6531 .addImm(
MI.getOperand(5).getImm())
6532 .addImm(
MI.getOperand(6).getImm())
6533 .addImm(
MI.getOperand(7).getImm())
6543 if (IID == Intrinsic::amdgcn_readlane || IID == Intrinsic::amdgcn_writelane ||
6544 IID == Intrinsic::amdgcn_update_dpp || IsSetInactive || IsPermLane16 ||
6545 IsPermlaneShuffle) {
6546 Src1 =
MI.getOperand(3).getReg();
6547 if (IID == Intrinsic::amdgcn_writelane || IsPermLane16 ||
6548 IsPermlaneShuffle) {
6549 Src2 =
MI.getOperand(4).getReg();
6553 LLT Ty =
MRI.getType(DstReg);
6554 unsigned Size = Ty.getSizeInBits();
6556 unsigned SplitSize = 32;
6557 if (IID == Intrinsic::amdgcn_update_dpp && (
Size % 64 == 0) &&
6558 ST.hasDPALU_DPP() &&
6562 if (
Size == SplitSize) {
6569 bool IsFloat = Ty.getScalarType().isFloat();
6573 Src0 =
B.buildBitcast(IntTy, Src0).getReg(0);
6574 if (Src1 &&
MRI.getType(Src1).getScalarType().isFloat())
6575 Src1 =
B.buildBitcast(IntTy, Src1).getReg(0);
6576 if (Src2 &&
MRI.getType(Src2).getScalarType().isFloat())
6577 Src2 =
B.buildBitcast(IntTy, Src2).getReg(0);
6581 Src0 =
B.buildAnyExt(I32, Src0).getReg(0);
6583 if (IID == Intrinsic::amdgcn_update_dpp || IsSetInactive || IsPermLane16)
6584 Src1 =
B.buildAnyExt(I32, Src1).getReg(0);
6586 if (IID == Intrinsic::amdgcn_writelane)
6587 Src2 =
B.buildAnyExt(I32, Src2).getReg(0);
6589 Register LaneOpDst = createLaneOp(Src0, Src1, Src2, I32);
6591 B.buildBitcast(DstReg,
B.buildTrunc(IntTy, LaneOpDst));
6593 B.buildTrunc(DstReg, LaneOpDst);
6594 MI.eraseFromParent();
6598 if (
Size % SplitSize != 0)
6602 bool NeedsBitcast =
false;
6603 if (IntTy.isVector()) {
6606 if (EltSize == SplitSize) {
6607 PartialResTy = EltTy;
6608 }
else if (EltSize == 16 || EltSize == 32) {
6609 unsigned NElem = SplitSize / EltSize;
6612 NeedsBitcast =
true;
6621 if (IID == Intrinsic::amdgcn_update_dpp || IsSetInactive || IsPermLane16)
6622 Src1Parts =
B.buildUnmerge(PartialResTy, Src1);
6624 if (IID == Intrinsic::amdgcn_writelane)
6625 Src2Parts =
B.buildUnmerge(PartialResTy, Src2);
6627 for (
unsigned i = 0; i <
NumParts; ++i) {
6628 Src0 = Src0Parts.
getReg(i);
6630 if (IID == Intrinsic::amdgcn_update_dpp || IsSetInactive || IsPermLane16)
6631 Src1 = Src1Parts.
getReg(i);
6633 if (IID == Intrinsic::amdgcn_writelane)
6634 Src2 = Src2Parts.
getReg(i);
6636 PartialRes.
push_back(createLaneOp(Src0, Src1, Src2, PartialResTy));
6639 if (NeedsBitcast || IsFloat)
6642 B.buildMergeLikeInstr(
LLT::integer(IntTy.getSizeInBits()), PartialRes));
6644 B.buildMergeLikeInstr(DstReg, PartialRes);
6646 MI.eraseFromParent();
6654 ST.getTargetLowering()->getImplicitParameterOffset(
6656 LLT DstTy =
MRI.getType(DstReg);
6659 Register KernargPtrReg =
MRI.createGenericVirtualRegister(DstTy);
6664 B.buildObjectPtrOffset(DstReg, KernargPtrReg,
6665 B.buildConstant(IdxTy,
Offset).getReg(0));
6676 Register Pointer =
MI.getOperand(2).getReg();
6678 Register NumRecords =
MI.getOperand(4).getReg();
6684 B.setInsertPt(
B.getMBB(), ++
B.getInsertPt());
6686 auto ExtStride =
B.buildAnyExt(I32, Stride);
6688 if (ST.getBufferResourceNumRecordsWidth() == 45) {
6689 NumRecords =
B.buildZExtOrTrunc(I64, NumRecords).getReg(0);
6691 B.buildAnd(I64, NumRecords,
B.buildConstant(I64, (1ULL << 45) - 1))
6697 auto PointerInt =
B.buildPtrToInt(PtrIntTy, Pointer);
6698 auto ExtPointer =
B.buildAnyExtOrTrunc(I64, PointerInt);
6699 auto NumRecordsLHS =
B.buildShl(I64, NumRecords,
B.buildConstant(I32, 57));
6700 Register LowHalf =
B.buildOr(I64, ExtPointer, NumRecordsLHS).getReg(0);
6704 auto NumRecordsRHS =
B.buildLShr(I64, NumRecords,
B.buildConstant(I32, 7));
6705 auto ShiftedStride =
B.buildShl(I32, ExtStride,
B.buildConstant(I32, 12));
6706 auto ExtShiftedStride =
6707 B.buildMergeValues(I64, {
Zero, ShiftedStride.getReg(0)});
6708 auto ShiftedFlags =
B.buildShl(I32, Flags,
B.buildConstant(I32, 28));
6709 auto ExtShiftedFlags =
6710 B.buildMergeValues(I64, {
Zero, ShiftedFlags.getReg(0)});
6711 auto CombinedFields =
B.buildOr(I64, NumRecordsRHS, ExtShiftedStride);
6713 B.buildOr(I64, CombinedFields, ExtShiftedFlags).getReg(0);
6714 B.buildMergeValues(Result, {LowHalf, HighHalf});
6716 NumRecords =
B.buildZExtOrTrunc(I32, NumRecords).getReg(0);
6717 auto Unmerge =
B.buildUnmerge(I32, Pointer);
6718 auto LowHalf = Unmerge.getReg(0);
6719 auto HighHalf = Unmerge.getReg(1);
6721 auto AndMask =
B.buildConstant(I32, 0x0000ffff);
6722 auto Masked =
B.buildAnd(I32, HighHalf, AndMask);
6723 auto ShiftConst =
B.buildConstant(I32, 16);
6724 auto ShiftedStride =
B.buildShl(I32, ExtStride, ShiftConst);
6725 auto NewHighHalf =
B.buildOr(I32,
Masked, ShiftedStride);
6726 Register NewHighHalfReg = NewHighHalf.getReg(0);
6727 B.buildMergeValues(Result, {LowHalf, NewHighHalfReg, NumRecords, Flags});
6730 MI.eraseFromParent();
6747 MI.eraseFromParent();
6754 unsigned AddrSpace)
const {
6756 auto Unmerge =
B.buildUnmerge(I32,
MI.getOperand(2).getReg());
6760 ST.hasGloballyAddressableScratch()) {
6762 B.buildInstr(AMDGPU::S_MOV_B32, {I32},
6763 {
Register(AMDGPU::SRC_FLAT_SCRATCH_BASE_HI)})
6765 MRI.setRegClass(FlatScratchBaseHi, &AMDGPU::SReg_32RegClass);
6767 Register XOR =
B.buildXor(I32, Hi32, FlatScratchBaseHi).getReg(0);
6769 B.buildConstant(I32, 1u << 26));
6774 MI.eraseFromParent();
6784std::pair<Register, unsigned>
6796 bool CheckNUW = ST.hasGFX1250Insts();
6798 MRI, OrigOffset,
nullptr, CheckNUW);
6801 if (
MRI.getType(BaseReg).isPointer())
6802 BaseReg =
B.buildPtrToInt(
MRI.getType(OrigOffset), BaseReg).getReg(0);
6812 unsigned Overflow = ImmOffset & ~MaxImm;
6813 ImmOffset -= Overflow;
6814 if ((int32_t)Overflow < 0) {
6815 Overflow += ImmOffset;
6819 if (Overflow != 0) {
6821 BaseReg =
B.buildConstant(I32, Overflow).getReg(0);
6823 auto OverflowVal =
B.buildConstant(I32, Overflow);
6824 BaseReg =
B.buildAdd(I32, BaseReg, OverflowVal).getReg(0);
6829 BaseReg =
B.buildConstant(I32, 0).getReg(0);
6831 return std::pair(BaseReg, ImmOffset);
6838 bool ImageStore)
const {
6841 LLT StoreVT =
MRI.getType(Reg);
6846 StoreVT == I16Vec ? Reg :
B.buildBitcast(I16Vec, Reg).getReg(0);
6848 if (ST.hasUnpackedD16VMem()) {
6849 auto Unmerge =
B.buildUnmerge(I16, RegI16);
6852 for (
int I = 0, E = Unmerge->getNumOperands() - 1;
I != E; ++
I)
6853 WideRegs.
push_back(
B.buildAnyExt(I32, Unmerge.getReg(
I)).getReg(0));
6861 if (ImageStore && ST.hasImageStoreD16Bug()) {
6864 Reg =
B.buildBitcast(I32, RegI16).getReg(0);
6866 PackedRegs.
resize(2,
B.buildUndef(I32).getReg(0));
6873 auto Unmerge =
B.buildUnmerge(I16, RegI16);
6874 for (
int I = 0, E = Unmerge->getNumOperands() - 1;
I != E; ++
I)
6876 PackedRegs.
resize(6,
B.buildUndef(I16).getReg(0));
6884 auto Unmerge =
B.buildUnmerge(I32, Reg);
6885 for (
int I = 0, E = Unmerge->getNumOperands() - 1;
I != E; ++
I)
6887 PackedRegs.
resize(4,
B.buildUndef(I32).getReg(0));
6897 Reg =
B.buildPadVectorWithUndefElements(
6906 bool IsFormat)
const {
6908 LLT Ty =
MRI->getType(VData);
6916 VData =
B.buildBitcast(Ty, VData).getReg(0);
6918 if (Ty.isFloat(16)) {
6920 VData =
B.buildBitcast(Ty, VData).getReg(0);
6927 if (Ty.isVector()) {
6928 if (Ty.getElementType().getSizeInBits() == 16 && Ty.getNumElements() <= 4) {
6943 const bool IsTyped = Info.isTyped();
6944 const bool IsFormat = Info.isFormat();
6946 Register VData =
MI.getOperand(OpIdx(*Info.getVDataArg())).getReg();
6947 LLT Ty =
MRI.getType(VData);
6949 const bool IsD16 = IsFormat && (EltTy.
getSizeInBits() == 16);
6956 if (IsFormat && !IsTyped && !IsD16 && MemTy.
getSizeInBits() < 32) {
6957 const Function &Fn =
B.getMF().getFunction();
6959 Fn,
"unsupported sub-dword format buffer store",
MI.getDebugLoc()));
6960 MI.eraseFromParent();
6967 Register RSrc =
MI.getOperand(OpIdx(Info.RsrcArg)).getReg();
6971 std::optional<unsigned> VIndexArg = Info.getVIndexArg();
6972 const bool HasVIndex = VIndexArg.has_value();
6973 Register VIndex = HasVIndex ?
MI.getOperand(OpIdx(*VIndexArg)).getReg()
6974 :
B.buildConstant(I32, 0).getReg(0);
6976 Register VOffset =
MI.getOperand(OpIdx(*Info.getVOffsetArg())).getReg();
6977 Register SOffset =
MI.getOperand(OpIdx(*Info.getSOffsetArg())).getReg();
6980 if (std::optional<unsigned> FormatArg = Info.getFormatArg())
6981 Format =
MI.getOperand(OpIdx(*FormatArg)).getImm();
6983 unsigned AuxiliaryData =
MI.getOperand(OpIdx(*Info.getAuxArg())).getImm();
6989 Opc = IsD16 ? AMDGPU::G_AMDGPU_TBUFFER_STORE_FORMAT_D16 :
6990 AMDGPU::G_AMDGPU_TBUFFER_STORE_FORMAT;
6991 }
else if (IsFormat) {
6992 Opc = IsD16 ? AMDGPU::G_AMDGPU_BUFFER_STORE_FORMAT_D16 :
6993 AMDGPU::G_AMDGPU_BUFFER_STORE_FORMAT;
6997 Opc = AMDGPU::G_AMDGPU_BUFFER_STORE_BYTE;
7000 Opc = AMDGPU::G_AMDGPU_BUFFER_STORE_SHORT;
7003 Opc = AMDGPU::G_AMDGPU_BUFFER_STORE;
7008 auto MIB =
B.buildInstr(
Opc)
7019 MIB.addImm(AuxiliaryData)
7020 .addImm(HasVIndex ? -1 : 0)
7021 .addMemOperand(MMO);
7023 MI.eraseFromParent();
7029 unsigned ImmOffset,
unsigned Format,
7032 auto MIB =
B.buildInstr(
Opc)
7043 MIB.addImm(AuxiliaryData)
7044 .addImm(HasVIndex ? -1 : 0)
7045 .addMemOperand(MMO);
7051 Register SOffset,
unsigned ImmOffset,
7052 unsigned Format,
unsigned AuxiliaryData,
7056 Register LoadDstReg =
B.getMRI()->createGenericVirtualRegister(LoadTy);
7058 Format, AuxiliaryData, MMO, IsTyped, HasVIndex,
B);
7061 B.buildUnmerge(Unmerge, LoadDstReg);
7071 const bool IsTyped = Info.isTyped();
7072 const bool IsFormat = Info.isFormat();
7082 assert(
MI.getNumExplicitDefs() == 1 ||
MI.getNumExplicitDefs() == 2);
7083 bool IsTFE =
MI.getNumExplicitDefs() == 2;
7085 StatusDst =
MI.getOperand(1).getReg();
7088 Register RSrc =
MI.getOperand(OpIdx(Info.RsrcArg)).getReg();
7090 std::optional<unsigned> VIndexArg = Info.getVIndexArg();
7091 const bool HasVIndex = VIndexArg.has_value();
7092 Register VIndex = HasVIndex ?
MI.getOperand(OpIdx(*VIndexArg)).getReg()
7093 :
B.buildConstant(I32, 0).getReg(0);
7095 Register VOffset =
MI.getOperand(OpIdx(*Info.getVOffsetArg())).getReg();
7096 Register SOffset =
MI.getOperand(OpIdx(*Info.getSOffsetArg())).getReg();
7099 if (std::optional<unsigned> FormatArg = Info.getFormatArg())
7100 Format =
MI.getOperand(OpIdx(*FormatArg)).getImm();
7102 unsigned AuxiliaryData =
MI.getOperand(OpIdx(*Info.getAuxArg())).getImm();
7105 LLT Ty =
MRI.getType(Dst);
7112 Dst =
MI.getOperand(0).getReg();
7113 B.setInsertPt(
B.getMBB(),
MI);
7120 Dst =
MI.getOperand(0).getReg();
7121 B.setInsertPt(
B.getMBB(),
MI);
7125 const bool IsD16 = IsFormat && (EltTy.
getSizeInBits() == 16);
7126 const bool Unpacked = ST.hasUnpackedD16VMem();
7128 if (IsFormat && !IsTyped && !IsD16 && MemTy.
getSizeInBits() < 32) {
7129 const Function &Fn =
B.getMF().getFunction();
7131 Fn,
"unsupported sub-dword format buffer load",
MI.getDebugLoc()));
7134 B.buildUndef(StatusDst);
7135 MI.eraseFromParent();
7139 if (!IsTyped && IsD16 && IsTFE && !ST.hasBufferTFEFormatD16()) {
7140 const Function &Fn =
B.getMF().getFunction();
7142 Fn,
"TFE D16 format buffer load is not supported on this GPU",
7145 B.buildUndef(StatusDst);
7146 MI.eraseFromParent();
7158 Opc = IsD16 ? AMDGPU::G_AMDGPU_TBUFFER_LOAD_FORMAT_D16 :
7159 AMDGPU::G_AMDGPU_TBUFFER_LOAD_FORMAT;
7160 }
else if (IsFormat) {
7162 Opc = IsTFE ? AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT_D16_TFE
7163 : AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT_D16;
7165 Opc = IsTFE ? AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT_TFE
7166 : AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT;
7171 Opc = IsTFE ? AMDGPU::G_AMDGPU_BUFFER_LOAD_UBYTE_TFE
7172 : AMDGPU::G_AMDGPU_BUFFER_LOAD_UBYTE;
7175 Opc = IsTFE ? AMDGPU::G_AMDGPU_BUFFER_LOAD_USHORT_TFE
7176 : AMDGPU::G_AMDGPU_BUFFER_LOAD_USHORT;
7179 Opc = IsTFE ? AMDGPU::G_AMDGPU_BUFFER_LOAD_TFE
7180 : AMDGPU::G_AMDGPU_BUFFER_LOAD;
7185 if (IsTFE && IsD16 && Ty.isVector()) {
7187 const unsigned NumElts = Ty.getNumElements();
7188 const unsigned NumValueDWords = Unpacked ? NumElts :
divideCeil(NumElts, 2);
7191 for (
unsigned I = 0;
I != NumValueDWords; ++
I)
7192 ValueDWords.
push_back(
MRI.createGenericVirtualRegister(I32));
7194 SOffset, ImmOffset,
Format, AuxiliaryData, MMO, IsTyped,
7199 R =
B.buildTrunc(EltTy, R).getReg(0);
7200 B.buildMergeLikeInstr(Dst, ValueDWords);
7209 if (PackedTy == Ty) {
7210 B.buildBitcast(Dst, Merged);
7212 Register Packed =
B.buildBitcast(PackedTy, Merged).getReg(0);
7213 B.buildDeleteTrailingVectorElements(Dst, Packed);
7217 const unsigned NumValueDWords =
divideCeil(Ty.getSizeInBits(), 32);
7219 EltTy.
isFloat() ?
MRI.createGenericVirtualRegister(Ty.changeElementType(
7223 Register ExtDst =
MRI.createGenericVirtualRegister(I32);
7225 ImmOffset,
Format, AuxiliaryData, MMO, IsTyped,
7227 B.buildTrunc(DstInt, ExtDst);
7228 }
else if (NumValueDWords == 1) {
7230 ImmOffset,
Format, AuxiliaryData, MMO, IsTyped,
7234 for (
unsigned I = 0;
I != NumValueDWords; ++
I)
7235 ValueDWords.
push_back(
MRI.createGenericVirtualRegister(I32));
7237 SOffset, ImmOffset,
Format, AuxiliaryData, MMO,
7238 IsTyped, HasVIndex,
B);
7239 B.buildMergeLikeInstr(DstInt, ValueDWords);
7242 B.buildBitcast(Dst, DstInt);
7244 (IsD16 && !Ty.isVector())) {
7245 Register LoadDstReg =
B.getMRI()->createGenericVirtualRegister(I32);
7247 Format, AuxiliaryData, MMO, IsTyped, HasVIndex,
B);
7248 B.setInsertPt(
B.getMBB(), ++
B.getInsertPt());
7249 B.buildTrunc(Dst, LoadDstReg);
7250 }
else if (Unpacked && IsD16 && Ty.isVector()) {
7252 Register LoadDstReg =
B.getMRI()->createGenericVirtualRegister(UnpackedTy);
7254 Format, AuxiliaryData, MMO, IsTyped, HasVIndex,
B);
7255 B.setInsertPt(
B.getMBB(), ++
B.getInsertPt());
7257 auto Unmerge =
B.buildUnmerge(I32, LoadDstReg);
7259 for (
unsigned I = 0,
N = Unmerge->getNumOperands() - 1;
I !=
N; ++
I)
7260 Repack.
push_back(
B.buildTrunc(EltTy, Unmerge.getReg(
I)).getReg(0));
7261 B.buildMergeLikeInstr(Dst, Repack);
7264 AuxiliaryData, MMO, IsTyped, HasVIndex,
B);
7267 MI.eraseFromParent();
7273 case Intrinsic::amdgcn_raw_buffer_atomic_swap:
7274 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_swap:
7275 case Intrinsic::amdgcn_struct_buffer_atomic_swap:
7276 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_swap:
7277 return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SWAP;
7278 case Intrinsic::amdgcn_raw_buffer_atomic_add:
7279 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_add:
7280 case Intrinsic::amdgcn_struct_buffer_atomic_add:
7281 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_add:
7282 return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_ADD;
7283 case Intrinsic::amdgcn_raw_buffer_atomic_sub:
7284 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_sub:
7285 case Intrinsic::amdgcn_struct_buffer_atomic_sub:
7286 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_sub:
7287 return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SUB;
7288 case Intrinsic::amdgcn_raw_buffer_atomic_smin:
7289 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_smin:
7290 case Intrinsic::amdgcn_struct_buffer_atomic_smin:
7291 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_smin:
7292 return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SMIN;
7293 case Intrinsic::amdgcn_raw_buffer_atomic_umin:
7294 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_umin:
7295 case Intrinsic::amdgcn_struct_buffer_atomic_umin:
7296 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_umin:
7297 return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_UMIN;
7298 case Intrinsic::amdgcn_raw_buffer_atomic_smax:
7299 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_smax:
7300 case Intrinsic::amdgcn_struct_buffer_atomic_smax:
7301 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_smax:
7302 return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SMAX;
7303 case Intrinsic::amdgcn_raw_buffer_atomic_umax:
7304 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_umax:
7305 case Intrinsic::amdgcn_struct_buffer_atomic_umax:
7306 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_umax:
7307 return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_UMAX;
7308 case Intrinsic::amdgcn_raw_buffer_atomic_and:
7309 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_and:
7310 case Intrinsic::amdgcn_struct_buffer_atomic_and:
7311 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_and:
7312 return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_AND;
7313 case Intrinsic::amdgcn_raw_buffer_atomic_or:
7314 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_or:
7315 case Intrinsic::amdgcn_struct_buffer_atomic_or:
7316 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_or:
7317 return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_OR;
7318 case Intrinsic::amdgcn_raw_buffer_atomic_xor:
7319 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_xor:
7320 case Intrinsic::amdgcn_struct_buffer_atomic_xor:
7321 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_xor:
7322 return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_XOR;
7323 case Intrinsic::amdgcn_raw_buffer_atomic_inc:
7324 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_inc:
7325 case Intrinsic::amdgcn_struct_buffer_atomic_inc:
7326 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_inc:
7327 return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_INC;
7328 case Intrinsic::amdgcn_raw_buffer_atomic_dec:
7329 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_dec:
7330 case Intrinsic::amdgcn_struct_buffer_atomic_dec:
7331 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_dec:
7332 return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_DEC;
7333 case Intrinsic::amdgcn_raw_buffer_atomic_cmpswap:
7334 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_cmpswap:
7335 case Intrinsic::amdgcn_struct_buffer_atomic_cmpswap:
7336 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_cmpswap:
7337 return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_CMPSWAP;
7338 case Intrinsic::amdgcn_raw_buffer_atomic_fadd:
7339 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_fadd:
7340 case Intrinsic::amdgcn_struct_buffer_atomic_fadd:
7341 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_fadd:
7342 return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_FADD;
7343 case Intrinsic::amdgcn_raw_buffer_atomic_fmin:
7344 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_fmin:
7345 case Intrinsic::amdgcn_struct_buffer_atomic_fmin:
7346 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_fmin:
7347 return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_FMIN;
7348 case Intrinsic::amdgcn_raw_buffer_atomic_fmax:
7349 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_fmax:
7350 case Intrinsic::amdgcn_struct_buffer_atomic_fmax:
7351 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_fmax:
7352 return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_FMAX;
7353 case Intrinsic::amdgcn_raw_buffer_atomic_sub_clamp_u32:
7354 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_sub_clamp_u32:
7355 case Intrinsic::amdgcn_struct_buffer_atomic_sub_clamp_u32:
7356 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_sub_clamp_u32:
7357 return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SUB_CLAMP_U32;
7358 case Intrinsic::amdgcn_raw_buffer_atomic_cond_sub_u32:
7359 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_cond_sub_u32:
7360 case Intrinsic::amdgcn_struct_buffer_atomic_cond_sub_u32:
7361 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_cond_sub_u32:
7362 return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_COND_SUB_U32;
7372 std::optional<unsigned> CmpArg = Info.getCmpArg();
7373 const bool IsCmpSwap = CmpArg.has_value();
7378 Register VData =
MI.getOperand(OpIdx(*Info.getVDataArg())).getReg();
7382 CmpVal =
MI.getOperand(OpIdx(*CmpArg)).getReg();
7385 Register RSrc =
MI.getOperand(OpIdx(Info.RsrcArg)).getReg();
7387 std::optional<unsigned> VIndexArg = Info.getVIndexArg();
7388 const bool HasVIndex = VIndexArg.has_value();
7389 Register VIndex = HasVIndex ?
MI.getOperand(OpIdx(*VIndexArg)).getReg()
7392 Register VOffset =
MI.getOperand(OpIdx(*Info.getVOffsetArg())).getReg();
7393 Register SOffset =
MI.getOperand(OpIdx(*Info.getSOffsetArg())).getReg();
7394 unsigned AuxiliaryData =
MI.getOperand(OpIdx(*Info.getAuxArg())).getImm();
7413 .addImm(AuxiliaryData)
7414 .addImm(HasVIndex ? -1 : 0)
7415 .addMemOperand(MMO);
7417 MI.eraseFromParent();
7427 bool IsA16,
bool IsG16) {
7441 (
B.getMRI()->getType(AddrReg) ==
F16)) {
7446 B.buildBuildVector(
V2F16, {AddrReg, B.buildUndef(F16).getReg(0)})
7450 "Bias needs to be converted to 16 bit in A16 mode");
7452 AddrReg =
B.buildBitcast(
V2F16, AddrReg).getReg(0);
7456 const LLT EltTy =
B.getMRI()->getType(AddrReg);
7460 if (((
I + 1) >= EndIdx) ||
7467 !
MI.getOperand(ArgOffset +
I + 1).isReg()) {
7469 B.buildBuildVector(V2EltTy,
7470 {AddrReg, B.buildUndef(EltTy).getReg(0)})
7475 V2EltTy, {AddrReg, MI.getOperand(ArgOffset + I + 1).getReg()})
7486 int DimIdx,
int NumVAddrs) {
7488 for (
int I = 0;
I != NumVAddrs; ++
I) {
7490 if (
SrcOp.isReg()) {
7493 assert(
B.getMRI()->getType(
Reg).getSizeInBits() == 32);
7494 if (
B.getMRI()->getType(
Reg) != I32)
7495 Reg =
B.buildBitcast(I32,
Reg).getReg(0);
7500 int NumAddrRegs = AddrRegs.
size();
7501 if (NumAddrRegs != 1) {
7502 LLT EltTy =
B.getMRI()->getType(AddrRegs[0]);
7505 MI.getOperand(DimIdx).setReg(VAddr.getReg(0));
7508 for (
int I = 1;
I != NumVAddrs; ++
I) {
7511 MI.getOperand(DimIdx +
I).setReg(AMDGPU::NoRegister);
7533 const unsigned NumDefs =
MI.getNumExplicitDefs();
7534 const unsigned ArgOffset = NumDefs + 1;
7535 bool IsTFE = NumDefs == 2;
7553 VData =
MI.getOperand(NumDefs == 0 ? 1 : 0).getReg();
7554 Ty =
MRI->getType(VData);
7557 const bool IsAtomicPacked16Bit =
7558 (BaseOpcode->
BaseOpcode == AMDGPU::IMAGE_ATOMIC_PK_ADD_F16 ||
7559 BaseOpcode->
BaseOpcode == AMDGPU::IMAGE_ATOMIC_PK_ADD_BF16);
7566 const bool GradTyIs16 = GradTy == I16 || GradTy ==
F16;
7567 const bool AddrTyIs16 = AddrTy == I16 || AddrTy ==
F16;
7568 const bool DataTyIs16 =
7569 Ty.getScalarType() == I16 || Ty.getScalarType() ==
F16;
7571 ST.hasG16() ? (BaseOpcode->
Gradients && GradTyIs16) : GradTyIs16;
7572 const bool IsA16 = AddrTyIs16;
7573 const bool IsD16 = !IsAtomicPacked16Bit && DataTyIs16;
7576 if (!BaseOpcode->
Atomic) {
7577 DMask =
MI.getOperand(ArgOffset + Intr->
DMaskIndex).getImm();
7580 }
else if (DMask != 0) {
7582 }
else if (!IsTFE && !BaseOpcode->
Store) {
7584 B.buildUndef(
MI.getOperand(0));
7585 MI.eraseFromParent();
7593 const unsigned StoreOpcode = IsD16 ? AMDGPU::G_AMDGPU_INTRIN_IMAGE_STORE_D16
7594 : AMDGPU::G_AMDGPU_INTRIN_IMAGE_STORE;
7595 const unsigned LoadOpcode = IsD16 ? AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD_D16
7596 : AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD;
7597 unsigned NewOpcode = LoadOpcode;
7598 if (BaseOpcode->
Store)
7599 NewOpcode = StoreOpcode;
7601 NewOpcode = AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD_NORET;
7604 MI.setDesc(
B.getTII().get(NewOpcode));
7608 if (IsTFE && DMask == 0) {
7611 MI.getOperand(ArgOffset + Intr->
DMaskIndex).setImm(DMask);
7614 if (BaseOpcode->
Atomic) {
7616 LLT Ty =
MRI->getType(VData0);
7619 if (Ty.isVector() && !IsAtomicPacked16Bit)
7626 auto Concat =
B.buildBuildVector(PackedTy, {VData0, VData1});
7627 MI.getOperand(2).setReg(
Concat.getReg(0));
7628 MI.getOperand(3).setReg(AMDGPU::NoRegister);
7632 unsigned CorrectedNumVAddrs = Intr->
NumVAddrs;
7635 if (BaseOpcode->
Gradients && !ST.hasG16() && (IsA16 != IsG16)) {
7641 if (IsA16 && !ST.hasA16()) {
7646 const unsigned NSAMaxSize = ST.getNSAMaxSize(BaseOpcode->
Sampler);
7647 const unsigned HasPartialNSA = ST.hasPartialNSAEncoding();
7649 if (IsA16 || IsG16) {
7657 const bool UseNSA = ST.hasNSAEncoding() &&
7658 PackedRegs.
size() >= ST.getNSAThreshold(MF) &&
7659 (PackedRegs.
size() <= NSAMaxSize || HasPartialNSA);
7660 const bool UsePartialNSA =
7661 UseNSA && HasPartialNSA && PackedRegs.
size() > NSAMaxSize;
7663 if (UsePartialNSA) {
7667 auto Concat =
B.buildConcatVectors(
7668 PackedAddrTy,
ArrayRef(PackedRegs).slice(NSAMaxSize - 1));
7669 PackedRegs[NSAMaxSize - 1] =
Concat.getReg(0);
7670 PackedRegs.
resize(NSAMaxSize);
7671 }
else if (!UseNSA && PackedRegs.
size() > 1) {
7673 auto Concat =
B.buildConcatVectors(PackedAddrTy, PackedRegs);
7674 PackedRegs[0] =
Concat.getReg(0);
7678 const unsigned NumPacked = PackedRegs.
size();
7681 if (!
SrcOp.isReg()) {
7691 SrcOp.setReg(AMDGPU::NoRegister);
7708 const bool UseNSA = ST.hasNSAEncoding() &&
7709 CorrectedNumVAddrs >= ST.getNSAThreshold(MF) &&
7710 (CorrectedNumVAddrs <= NSAMaxSize || HasPartialNSA);
7711 const bool UsePartialNSA =
7712 UseNSA && HasPartialNSA && CorrectedNumVAddrs > NSAMaxSize;
7714 if (UsePartialNSA) {
7716 ArgOffset + Intr->
VAddrStart + NSAMaxSize - 1,
7718 }
else if (!UseNSA && Intr->
NumVAddrs > 1) {
7733 if (!Ty.isVector() || !IsD16)
7737 if (RepackedReg != VData) {
7738 MI.getOperand(1).setReg(RepackedReg);
7746 const int NumElts = Ty.isVector() ? Ty.getNumElements() : 1;
7749 if (NumElts < DMaskLanes)
7752 if (NumElts > 4 || DMaskLanes > 4)
7763 const unsigned AdjustedNumElts = DMaskLanes == 0 ? 1 : DMaskLanes;
7764 const LLT AdjustedTy =
7780 if (IsD16 && ST.hasUnpackedD16VMem()) {
7787 unsigned RoundedElts = (AdjustedTy.
getSizeInBits() + 31) / 32;
7788 unsigned RoundedSize = 32 * RoundedElts;
7792 RegTy = !IsTFE && EltSize == 16 ? V2I16 : I32;
7797 if (!IsTFE && (RoundedTy == Ty || !Ty.
isVector()))
7803 B.setInsertPt(*
MI.getParent(), ++
MI.getIterator());
7807 const LLT LoadResultTy = IsTFE ? TFETy : RoundedTy;
7808 const int ResultNumRegs = LoadResultTy.
getSizeInBits() / 32;
7810 Register NewResultReg =
MRI->createGenericVirtualRegister(LoadResultTy);
7812 MI.getOperand(0).setReg(NewResultReg);
7820 Dst1Reg =
MI.getOperand(1).getReg();
7821 if (
MRI->getType(Dst1Reg) != I32)
7825 MI.removeOperand(1);
7828 if (!Ty.isVector() && Ty.getSizeInBits() == 32) {
7829 auto Unmerge =
B.buildUnmerge({I32, I32}, NewResultReg);
7830 B.buildBitcast(DstReg, Unmerge.getReg(0));
7831 B.buildCopy(Dst1Reg, Unmerge.getReg(1));
7840 const int NumDataRegs = IsTFE ? ResultNumRegs - 1 : ResultNumRegs;
7842 if (ResultNumRegs == 1) {
7844 ResultRegs[0] = NewResultReg;
7847 for (
int I = 0;
I != NumDataRegs; ++
I)
7848 ResultRegs[
I] =
MRI->createGenericVirtualRegister(RegTy);
7849 B.buildUnmerge(ResultRegs, NewResultReg);
7854 ResultRegs.
resize(NumDataRegs);
7859 if (IsD16 && !Ty.isVector()) {
7860 B.buildTrunc(DstReg, ResultRegs[0]);
7865 if ((Ty == V2I16 || Ty ==
V2F16) && NumDataRegs == 1 &&
7866 !ST.hasUnpackedD16VMem()) {
7867 B.buildBitcast(DstReg, ResultRegs[0]);
7879 if (RegTy != V2I16 && !ST.hasUnpackedD16VMem()) {
7881 Reg =
B.buildBitcast(V2I16, Reg).getReg(0);
7882 }
else if (ST.hasUnpackedD16VMem()) {
7884 Reg =
B.buildTrunc(I16, Reg).getReg(0);
7888 auto padWithUndef = [&](
LLT Ty,
int NumElts) {
7892 for (
int I = 0;
I != NumElts; ++
I)
7897 LLT ResTy =
MRI->getType(ResultRegs[0]);
7899 padWithUndef(ResTy, NumElts - ResultRegs.
size());
7900 B.buildBuildVector(DstReg, ResultRegs);
7904 assert(!ST.hasUnpackedD16VMem() && (ResTy == V2I16 || ResTy ==
V2F16));
7905 const int RegsToCover = (Ty.getSizeInBits() + 31) / 32;
7910 if (Ty == V3I16 || Ty == V3F16) {
7912 if (ResultRegs.
size() == 1) {
7913 NewResultReg = ResultRegs[0];
7914 }
else if (ResultRegs.
size() == 2) {
7916 NewResultReg =
B.buildConcatVectors(V4I16, ResultRegs).getReg(0);
7922 LLT DstTy =
MRI->getType(DstReg);
7923 LLT NewResTy =
MRI->getType(NewResultReg);
7927 :
MRI->createGenericVirtualRegister(
7931 B.buildDeleteTrailingVectorElements(ResizeDst, NewResultReg);
7933 B.buildPadVectorWithUndefElements(ResizeDst, NewResultReg);
7935 if (ResizeDst != DstReg)
7936 B.buildBitcast(DstReg, ResizeDst);
7940 padWithUndef(ResTy, RegsToCover - ResultRegs.
size());
7941 B.buildConcatVectors(DstReg, ResultRegs);
7950 Register OrigDst =
MI.getOperand(0).getReg();
7952 LLT Ty =
B.getMRI()->getType(OrigDst);
7953 unsigned Size = Ty.getSizeInBits();
7955 bool HasMMO = !
MI.memoperands_empty();
7960 bool IsSubwordLoad = Ty.isScalar() &&
Size < 32 && ST.hasScalarSubwordLoads();
7961 if (
Size % 32 != 0 && !IsSubwordLoad) {
7964 Fn,
"unsupported s_buffer_load result type",
MI.getDebugLoc()));
7965 B.buildUndef(OrigDst);
7966 MI.eraseFromParent();
7971 if (IsSubwordLoad) {
7973 Opc =
Size == 8 ? AMDGPU::G_AMDGPU_S_BUFFER_LOAD_UBYTE
7974 : AMDGPU::G_AMDGPU_S_BUFFER_LOAD_USHORT;
7977 Dst =
B.getMRI()->createGenericVirtualRegister(
LLT::integer(32));
7979 Opc = AMDGPU::G_AMDGPU_S_BUFFER_LOAD;
7988 B.setInsertPt(
B.getMBB(),
MI);
7993 B.setInsertPt(
B.getMBB(),
MI);
7996 MI.setDesc(
B.getTII().get(
Opc));
7997 MI.removeOperand(1);
8003 const unsigned MemSize = (
Size + 7) / 8;
8004 const Align MemAlign =
B.getDataLayout().getABITypeAlign(
8011 MI.addMemOperand(MF, MMO);
8013 if (Dst != OrigDst) {
8014 MI.getOperand(0).setReg(Dst);
8015 B.setInsertPt(
B.getMBB(), ++
B.getInsertPt());
8016 B.buildTrunc(OrigDst, Dst);
8038 MI.setDesc(
B.getTII().get(AMDGPU::G_AMDGPU_S_BUFFER_PREFETCH));
8039 MI.removeOperand(0);
8050 if (!ST.hasTrapHandler() ||
8069 MI.eraseFromParent();
8081 for (
auto I = SplitPoint, E = BB.
end();
I != E; ++
I) {
8090 BuildMI(*TrapBB, TrapBB->
end(),
DL,
B.getTII().get(AMDGPU::S_ENDPGM))
8092 BuildMI(BB, &
MI,
DL,
B.getTII().get(AMDGPU::S_CBRANCH_EXECNZ))
8096 MI.eraseFromParent();
8105 Register SGPR01(AMDGPU::SGPR0_SGPR1);
8112 ST.getTargetLowering()->getImplicitParameterOffset(
B.getMF(), Param);
8114 Register KernargPtrReg =
MRI.createGenericVirtualRegister(
8130 Register LoadAddr =
MRI.createGenericVirtualRegister(
8132 B.buildObjectPtrOffset(LoadAddr, KernargPtrReg,
8135 Register Temp =
B.buildLoad(I64, LoadAddr, *MMO).getReg(0);
8136 B.buildCopy(SGPR01, Temp);
8137 B.buildInstr(AMDGPU::S_TRAP)
8140 MI.eraseFromParent();
8151 B.buildCopy(SGPR01, LiveIn);
8152 B.buildInstr(AMDGPU::S_TRAP)
8156 MI.eraseFromParent();
8167 if (ST.hasPrivEnabledTrap2NopBug()) {
8179 ST.getInstrInfo()->insertSimulatedTrap(
MRI,
MBB,
MI,
MI.getDebugLoc());
8182 MI.eraseFromParent();
8186 B.buildInstr(AMDGPU::S_TRAP)
8188 MI.eraseFromParent();
8197 if (!ST.hasTrapHandler() ||
8201 Fn,
"debugtrap handler not supported",
MI.getDebugLoc(),
DS_Warning));
8204 B.buildInstr(AMDGPU::S_TRAP)
8208 MI.eraseFromParent();
8222 Register NodePtr =
MI.getOperand(2).getReg();
8223 Register RayExtent =
MI.getOperand(3).getReg();
8224 Register RayOrigin =
MI.getOperand(4).getReg();
8226 Register RayInvDir =
MI.getOperand(6).getReg();
8229 RayExtent =
B.buildBitcast(I32, RayExtent).getReg(0);
8234 const bool IsA16 =
MRI.getType(RayDir).getElementType().getSizeInBits() == 16;
8235 const bool Is64 =
MRI.getType(NodePtr).getSizeInBits() == 64;
8236 const unsigned NumVDataDwords = 4;
8237 const unsigned NumVAddrDwords = IsA16 ? (Is64 ? 9 : 8) : (Is64 ? 12 : 11);
8238 const unsigned NumVAddrs = IsGFX11Plus ? (IsA16 ? 4 : 5) : NumVAddrDwords;
8240 IsGFX12Plus || (ST.hasNSAEncoding() && NumVAddrs <= ST.getNSAMaxSize());
8242 const unsigned BaseOpcodes[2][2] = {
8243 {AMDGPU::IMAGE_BVH_INTERSECT_RAY, AMDGPU::IMAGE_BVH_INTERSECT_RAY_a16},
8244 {AMDGPU::IMAGE_BVH64_INTERSECT_RAY,
8245 AMDGPU::IMAGE_BVH64_INTERSECT_RAY_a16}};
8249 IsGFX12Plus ? AMDGPU::MIMGEncGfx12
8250 : IsGFX11 ? AMDGPU::MIMGEncGfx11NSA
8251 : AMDGPU::MIMGEncGfx10NSA,
8252 NumVDataDwords, NumVAddrDwords);
8256 IsGFX11 ? AMDGPU::MIMGEncGfx11Default
8257 : AMDGPU::MIMGEncGfx10Default,
8258 NumVDataDwords, NumVAddrDwords);
8263 if (UseNSA && IsGFX11Plus) {
8264 auto packLanes = [&
Ops, &I32, &V3I32, &
B](
Register Src) {
8265 auto SrcInt =
B.buildBitcast(V3I32, Src);
8266 auto Unmerge =
B.buildUnmerge({I32, I32, I32}, SrcInt);
8267 auto Merged =
B.buildMergeLikeInstr(
8268 V3I32, {Unmerge.getReg(0), Unmerge.getReg(1), Unmerge.getReg(2)});
8269 Ops.push_back(Merged.getReg(0));
8272 Ops.push_back(NodePtr);
8273 Ops.push_back(RayExtent);
8274 packLanes(RayOrigin);
8277 auto UnmergeRayDir =
8278 B.buildUnmerge({I16, I16, I16},
B.buildBitcast(V3I16, RayDir));
8279 auto UnmergeRayInvDir =
8280 B.buildUnmerge({I16, I16, I16},
B.buildBitcast(V3I16, RayInvDir));
8281 auto MergedDir =
B.buildMergeLikeInstr(
8284 I32,
B.buildMergeLikeInstr(V2I16, {UnmergeRayInvDir.getReg(0),
8285 UnmergeRayDir.getReg(0)}))
8288 I32,
B.buildMergeLikeInstr(V2I16, {UnmergeRayInvDir.getReg(1),
8289 UnmergeRayDir.getReg(1)}))
8292 I32,
B.buildMergeLikeInstr(V2I16, {UnmergeRayInvDir.getReg(2),
8293 UnmergeRayDir.getReg(2)}))
8295 Ops.push_back(MergedDir.getReg(0));
8298 packLanes(RayInvDir);
8302 auto Unmerge =
B.buildUnmerge({I32, I32}, NodePtr);
8303 Ops.push_back(Unmerge.getReg(0));
8304 Ops.push_back(Unmerge.getReg(1));
8306 Ops.push_back(NodePtr);
8308 Ops.push_back(RayExtent);
8310 auto packLanes = [&
Ops, &I32, &V3I32, &
B](
Register Src) {
8311 auto SrcInt =
B.buildBitcast(V3I32, Src);
8312 auto Unmerge =
B.buildUnmerge({I32, I32, I32}, SrcInt);
8313 Ops.push_back(Unmerge.getReg(0));
8314 Ops.push_back(Unmerge.getReg(1));
8315 Ops.push_back(Unmerge.getReg(2));
8318 packLanes(RayOrigin);
8320 auto UnmergeRayDir =
8321 B.buildUnmerge({I16, I16, I16},
B.buildBitcast(V3I16, RayDir));
8322 auto UnmergeRayInvDir =
8323 B.buildUnmerge({I16, I16, I16},
B.buildBitcast(V3I16, RayInvDir));
8324 Register R1 =
MRI.createGenericVirtualRegister(I32);
8326 Register R3 =
MRI.createGenericVirtualRegister(I32);
8327 B.buildMergeLikeInstr(R1,
8328 {UnmergeRayDir.getReg(0), UnmergeRayDir.getReg(1)});
8329 B.buildMergeLikeInstr(
8330 R2, {UnmergeRayDir.getReg(2), UnmergeRayInvDir.getReg(0)});
8331 B.buildMergeLikeInstr(
8332 R3, {UnmergeRayInvDir.getReg(1), UnmergeRayInvDir.getReg(2)});
8338 packLanes(RayInvDir);
8347 Ops.push_back(MergedOps);
8350 auto MIB =
B.buildInstr(AMDGPU::G_AMDGPU_BVH_INTERSECT_RAY)
8359 .addImm(IsA16 ? 1 : 0)
8362 MI.eraseFromParent();
8372 Register DstOrigin =
MI.getOperand(1).getReg();
8374 Register NodePtr =
MI.getOperand(4).getReg();
8375 Register RayExtent =
MI.getOperand(5).getReg();
8376 Register InstanceMask =
MI.getOperand(6).getReg();
8377 Register RayOrigin =
MI.getOperand(7).getReg();
8379 Register Offsets =
MI.getOperand(9).getReg();
8380 Register TDescr =
MI.getOperand(10).getReg();
8383 Intrinsic::amdgcn_image_bvh8_intersect_ray;
8384 const unsigned NumVDataDwords = 10;
8385 const unsigned NumVAddrDwords = IsBVH8 ? 11 : 12;
8387 IsBVH8 ? AMDGPU::IMAGE_BVH8_INTERSECT_RAY
8388 : AMDGPU::IMAGE_BVH_DUAL_INTERSECT_RAY,
8389 AMDGPU::MIMGEncGfx12, NumVDataDwords, NumVAddrDwords);
8392 auto RayExtentInstanceMaskVec =
8393 B.buildMergeLikeInstr(V2I32, {
B.buildBitcast(I32, RayExtent),
8394 B.buildAnyExt(I32, InstanceMask)});
8396 B.buildInstr(IsBVH8 ? AMDGPU::G_AMDGPU_BVH8_INTERSECT_RAY
8397 : AMDGPU::G_AMDGPU_BVH_DUAL_INTERSECT_RAY)
8403 .addUse(RayExtentInstanceMaskVec.getReg(0))
8410 MI.eraseFromParent();
8419 Register StackPtr = Info->getStackPtrOffsetReg();
8421 B.buildInstr(AMDGPU::G_AMDGPU_WAVE_ADDRESS, {DstReg}, {StackPtr});
8422 MI.eraseFromParent();
8429 if (!ST.hasArchitectedSGPRs())
8433 auto TTMP8 =
B.buildCopy(I32,
Register(AMDGPU::TTMP8));
8434 auto LSB =
B.buildConstant(I32, 25);
8435 auto Width =
B.buildConstant(I32, 5);
8436 B.buildUbfx(DstReg, TTMP8, LSB, Width);
8437 MI.eraseFromParent();
8445 unsigned Width)
const {
8449 {&AMDGPU::SReg_32RegClass,
MRI.getType(DstReg)});
8450 B.buildInstr(AMDGPU::S_GETREG_B32_const)
8453 B.buildCopy(DstReg, Result);
8454 MI.eraseFromParent();
8470 if (
MRI.getType(Src) != I64)
8474 B.buildIntrinsic(Intrinsic::amdgcn_s_getreg, {I32},
8478 B.buildIntrinsic(Intrinsic::amdgcn_s_getreg, {I32},
8481 B.buildMergeLikeInstr(Src, {ModeReg, TrapReg});
8482 MI.eraseFromParent();
8492 if (
MRI.getType(Src) != I64)
8495 auto Unmerge =
B.buildUnmerge({I32, I32},
MI.getOperand(0));
8499 .addReg(Unmerge.getReg(0));
8503 .addReg(Unmerge.getReg(1));
8504 MI.eraseFromParent();
8518 switch (BufInfo->Kind) {
8519 case AMDGPU::BufferKindSLoad:
8521 case AMDGPU::BufferKindPrefetch:
8523 case AMDGPU::BufferKindStore:
8525 case AMDGPU::BufferKindLoad:
8526 case AMDGPU::BufferKindAtomicLoad:
8528 case AMDGPU::BufferKindAtomicRMW:
8529 case AMDGPU::BufferKindAtomicCmpSwap:
8531 case AMDGPU::BufferKindLoadToLDS:
8539 case Intrinsic::sponentry:
8544 Register TmpReg =
MRI.createGenericVirtualRegister(I32);
8545 B.buildInstr(AMDGPU::G_AMDGPU_SPONENTRY).addDef(TmpReg);
8548 B.buildIntToPtr(DstReg, TmpReg);
8549 MI.eraseFromParent();
8551 int FI =
B.getMF().getFrameInfo().CreateFixedObject(
8553 B.buildFrameIndex(
MI.getOperand(0), FI);
8554 MI.eraseFromParent();
8557 case Intrinsic::amdgcn_if:
8558 case Intrinsic::amdgcn_else: {
8561 bool Negated =
false;
8571 MRI.createVirtualRegister({WaveMaskRC,
MRI.getType(Def)});
8573 MRI.createVirtualRegister({WaveMaskRC,
MRI.getType(
Use)});
8578 std::swap(CondBrTarget, UncondBrTarget);
8580 B.setInsertPt(
B.getMBB(), BrCond->getIterator());
8581 B.buildCopy(NewUse,
Use);
8582 if (IntrID == Intrinsic::amdgcn_if) {
8583 B.buildInstr(AMDGPU::SI_IF)
8586 .addMBB(UncondBrTarget)
8589 B.buildInstr(AMDGPU::SI_ELSE)
8592 .addMBB(UncondBrTarget)
8602 B.buildBr(*CondBrTarget);
8605 MI.eraseFromParent();
8606 BrCond->eraseFromParent();
8610 MRI.replaceRegWith(Def, NewDef);
8617 case Intrinsic::amdgcn_loop: {
8620 bool Negated =
false;
8629 {
TRI->getWaveMaskRegClass(),
MRI.getType(Reg)});
8632 std::swap(CondBrTarget, UncondBrTarget);
8634 B.setInsertPt(
B.getMBB(), BrCond->getIterator());
8635 B.buildCopy(NewReg, Reg);
8636 B.buildInstr(AMDGPU::SI_LOOP)
8638 .addMBB(UncondBrTarget)
8644 B.buildBr(*CondBrTarget);
8646 MI.eraseFromParent();
8647 BrCond->eraseFromParent();
8653 case Intrinsic::amdgcn_wave_reduce_min:
8654 case Intrinsic::amdgcn_wave_reduce_umin:
8655 case Intrinsic::amdgcn_wave_reduce_fmin:
8656 case Intrinsic::amdgcn_wave_reduce_max:
8657 case Intrinsic::amdgcn_wave_reduce_umax:
8658 case Intrinsic::amdgcn_wave_reduce_fmax:
8659 case Intrinsic::amdgcn_wave_reduce_add:
8660 case Intrinsic::amdgcn_wave_reduce_fadd:
8661 case Intrinsic::amdgcn_wave_reduce_sub:
8662 case Intrinsic::amdgcn_wave_reduce_fsub:
8663 case Intrinsic::amdgcn_wave_reduce_and:
8664 case Intrinsic::amdgcn_wave_reduce_or:
8665 case Intrinsic::amdgcn_wave_reduce_xor: {
8667 if (
MRI.getType(SrcReg).getSizeInBits() != 16)
8670 bool IsFPOp = IntrID == Intrinsic::amdgcn_wave_reduce_fmin ||
8671 IntrID == Intrinsic::amdgcn_wave_reduce_fmax ||
8672 IntrID == Intrinsic::amdgcn_wave_reduce_fadd ||
8673 IntrID == Intrinsic::amdgcn_wave_reduce_fsub;
8674 bool NeedsSignExt = IntrID == Intrinsic::amdgcn_wave_reduce_min ||
8675 IntrID == Intrinsic::amdgcn_wave_reduce_max ||
8676 IntrID == Intrinsic::amdgcn_wave_reduce_add ||
8677 IntrID == Intrinsic::amdgcn_wave_reduce_sub;
8678 auto Ext = IsFPOp ?
B.buildFPExt(
F32, SrcReg)
8685 .addUse(Ext.getReg(0))
8686 .addImm(
MI.getOperand(3).getImm());
8688 B.buildFPTrunc(DstReg, NewDst);
8690 B.buildTrunc(DstReg, NewDst);
8691 MI.eraseFromParent();
8694 case Intrinsic::amdgcn_make_buffer_rsrc:
8696 case Intrinsic::amdgcn_kernarg_segment_ptr:
8699 B.buildConstant(
MI.getOperand(0).getReg(), 0);
8700 MI.eraseFromParent();
8706 case Intrinsic::amdgcn_implicitarg_ptr:
8708 case Intrinsic::amdgcn_workitem_id_x:
8711 case Intrinsic::amdgcn_workitem_id_y:
8714 case Intrinsic::amdgcn_workitem_id_z:
8717 case Intrinsic::amdgcn_workgroup_id_x:
8722 case Intrinsic::amdgcn_workgroup_id_y:
8727 case Intrinsic::amdgcn_workgroup_id_z:
8732 case Intrinsic::amdgcn_cluster_id_x:
8733 return ST.hasClusters() &&
8736 case Intrinsic::amdgcn_cluster_id_y:
8737 return ST.hasClusters() &&
8740 case Intrinsic::amdgcn_cluster_id_z:
8741 return ST.hasClusters() &&
8744 case Intrinsic::amdgcn_cluster_workgroup_id_x:
8745 return ST.hasClusters() &&
8748 case Intrinsic::amdgcn_cluster_workgroup_id_y:
8749 return ST.hasClusters() &&
8752 case Intrinsic::amdgcn_cluster_workgroup_id_z:
8753 return ST.hasClusters() &&
8756 case Intrinsic::amdgcn_cluster_workgroup_flat_id:
8757 return ST.hasClusters() &&
8759 case Intrinsic::amdgcn_cluster_workgroup_max_id_x:
8760 return ST.hasClusters() &&
8763 case Intrinsic::amdgcn_cluster_workgroup_max_id_y:
8764 return ST.hasClusters() &&
8767 case Intrinsic::amdgcn_cluster_workgroup_max_id_z:
8768 return ST.hasClusters() &&
8771 case Intrinsic::amdgcn_cluster_workgroup_max_flat_id:
8772 return ST.hasClusters() &&
8776 case Intrinsic::amdgcn_wave_id:
8778 case Intrinsic::amdgcn_lds_kernel_id:
8781 case Intrinsic::amdgcn_dispatch_ptr:
8784 case Intrinsic::amdgcn_queue_ptr:
8787 case Intrinsic::amdgcn_implicit_buffer_ptr: {
8788 const Function &Fn =
B.getMF().getFunction();
8793 ST.hasArchitectedFlatScratch()
8794 ?
"intrinsic not supported on subtarget"
8795 : !ST.isMesa3DOS() ?
"intrinsic not supported for OS"
8796 :
"intrinsic not supported for calling convention";
8799 B.buildUndef(
MI.getOperand(0).getReg());
8800 MI.eraseFromParent();
8806 case Intrinsic::amdgcn_dispatch_id:
8809 case Intrinsic::r600_read_ngroups_x:
8813 case Intrinsic::r600_read_ngroups_y:
8816 case Intrinsic::r600_read_ngroups_z:
8819 case Intrinsic::r600_read_local_size_x:
8822 case Intrinsic::r600_read_local_size_y:
8826 case Intrinsic::r600_read_local_size_z:
8829 case Intrinsic::amdgcn_fdiv_fast:
8831 case Intrinsic::amdgcn_is_shared:
8833 case Intrinsic::amdgcn_is_private:
8835 case Intrinsic::amdgcn_wavefrontsize: {
8836 B.buildConstant(
MI.getOperand(0), ST.getWavefrontSize());
8837 MI.eraseFromParent();
8840 case Intrinsic::amdgcn_rsq_clamp:
8842 case Intrinsic::amdgcn_image_bvh_intersect_ray:
8844 case Intrinsic::amdgcn_image_bvh_dual_intersect_ray:
8845 case Intrinsic::amdgcn_image_bvh8_intersect_ray:
8847 case Intrinsic::amdgcn_swmmac_f32_16x16x128_fp8_fp8:
8848 case Intrinsic::amdgcn_swmmac_f32_16x16x128_fp8_bf8:
8849 case Intrinsic::amdgcn_swmmac_f32_16x16x128_bf8_fp8:
8850 case Intrinsic::amdgcn_swmmac_f32_16x16x128_bf8_bf8:
8851 case Intrinsic::amdgcn_swmmac_f16_16x16x128_fp8_fp8:
8852 case Intrinsic::amdgcn_swmmac_f16_16x16x128_fp8_bf8:
8853 case Intrinsic::amdgcn_swmmac_f16_16x16x128_bf8_fp8:
8854 case Intrinsic::amdgcn_swmmac_f16_16x16x128_bf8_bf8: {
8857 LLT IndexArgTy =
MRI.getType(Index);
8858 if (IndexArgTy != I64) {
8859 auto NewIndex = IndexArgTy.
isVector() ?
B.buildBitcast(I64, Index)
8860 :
B.buildAnyExt(I64, Index);
8861 MI.getOperand(5).setReg(NewIndex.getReg(0));
8865 case Intrinsic::amdgcn_swmmac_f16_16x16x32_f16:
8866 case Intrinsic::amdgcn_swmmac_bf16_16x16x32_bf16:
8867 case Intrinsic::amdgcn_swmmac_f32_16x16x32_bf16:
8868 case Intrinsic::amdgcn_swmmac_f32_16x16x32_f16:
8869 case Intrinsic::amdgcn_swmmac_f32_16x16x32_fp8_fp8:
8870 case Intrinsic::amdgcn_swmmac_f32_16x16x32_fp8_bf8:
8871 case Intrinsic::amdgcn_swmmac_f32_16x16x32_bf8_fp8:
8872 case Intrinsic::amdgcn_swmmac_f32_16x16x32_bf8_bf8: {
8875 if (
MRI.getType(Index) != I32)
8876 MI.getOperand(5).setReg(
B.buildAnyExt(I32, Index).getReg(0));
8879 case Intrinsic::amdgcn_swmmac_f16_16x16x64_f16:
8880 case Intrinsic::amdgcn_swmmac_bf16_16x16x64_bf16:
8881 case Intrinsic::amdgcn_swmmac_f32_16x16x64_bf16:
8882 case Intrinsic::amdgcn_swmmac_bf16f32_16x16x64_bf16:
8883 case Intrinsic::amdgcn_swmmac_f32_16x16x64_f16:
8884 case Intrinsic::amdgcn_swmmac_i32_16x16x128_iu8:
8885 case Intrinsic::amdgcn_swmmac_i32_16x16x32_iu4:
8886 case Intrinsic::amdgcn_swmmac_i32_16x16x32_iu8:
8887 case Intrinsic::amdgcn_swmmac_i32_16x16x64_iu4: {
8889 LLT IdxTy = IntrID == Intrinsic::amdgcn_swmmac_i32_16x16x128_iu8
8892 LLT IndexArgTy =
MRI.getType(Index);
8893 if (IndexArgTy != IdxTy) {
8894 auto NewIndex = IndexArgTy.
isVector() ?
B.buildBitcast(IdxTy, Index)
8895 :
B.buildAnyExt(IdxTy, Index);
8896 MI.getOperand(7).setReg(NewIndex.getReg(0));
8901 case Intrinsic::amdgcn_fmed3: {
8907 MI.setDesc(
B.getTII().get(AMDGPU::G_AMDGPU_FMED3));
8908 MI.removeOperand(1);
8912 case Intrinsic::amdgcn_readlane:
8913 case Intrinsic::amdgcn_writelane:
8914 case Intrinsic::amdgcn_readfirstlane:
8915 case Intrinsic::amdgcn_permlane16:
8916 case Intrinsic::amdgcn_permlanex16:
8917 case Intrinsic::amdgcn_permlane64:
8918 case Intrinsic::amdgcn_set_inactive:
8919 case Intrinsic::amdgcn_set_inactive_chain_arg:
8920 case Intrinsic::amdgcn_mov_dpp8:
8921 case Intrinsic::amdgcn_update_dpp:
8922 case Intrinsic::amdgcn_permlane_bcast:
8923 case Intrinsic::amdgcn_permlane_up:
8924 case Intrinsic::amdgcn_permlane_down:
8925 case Intrinsic::amdgcn_permlane_xor:
8927 case Intrinsic::amdgcn_dead: {
8931 MI.eraseFromParent();
8934 case Intrinsic::amdgcn_cooperative_atomic_load_32x4B:
8935 case Intrinsic::amdgcn_cooperative_atomic_load_16x8B:
8936 case Intrinsic::amdgcn_cooperative_atomic_load_8x16B:
8937 assert(
MI.hasOneMemOperand() &&
"Expected IRTranslator to set MemOp!");
8938 B.buildLoad(
MI.getOperand(0),
MI.getOperand(2), **
MI.memoperands_begin());
8939 MI.eraseFromParent();
8941 case Intrinsic::amdgcn_cooperative_atomic_store_32x4B:
8942 case Intrinsic::amdgcn_cooperative_atomic_store_16x8B:
8943 case Intrinsic::amdgcn_cooperative_atomic_store_8x16B:
8944 assert(
MI.hasOneMemOperand() &&
"Expected IRTranslator to set MemOp!");
8945 B.buildStore(
MI.getOperand(2),
MI.getOperand(1), **
MI.memoperands_begin());
8946 MI.eraseFromParent();
8948 case Intrinsic::amdgcn_av_load_b8:
8949 case Intrinsic::amdgcn_av_load_b16:
8950 case Intrinsic::amdgcn_av_load_b32:
8951 case Intrinsic::amdgcn_av_load_b64:
8952 case Intrinsic::amdgcn_av_load_b128:
8953 case Intrinsic::amdgcn_av_store_b8:
8954 case Intrinsic::amdgcn_av_store_b16:
8955 case Intrinsic::amdgcn_av_store_b32:
8956 case Intrinsic::amdgcn_av_store_b64:
8957 case Intrinsic::amdgcn_av_store_b128: {
8958 assert(
MI.hasOneMemOperand() &&
"Expected IRTranslator to set MemOp!");
8959 if (
MI.getNumExplicitDefs())
8960 B.buildLoad(
MI.getOperand(0),
MI.getOperand(2), **
MI.memoperands_begin());
8962 B.buildStore(
MI.getOperand(2),
MI.getOperand(1),
8963 **
MI.memoperands_begin());
8964 MI.eraseFromParent();
8967 case Intrinsic::amdgcn_flat_load_monitor_b32:
8968 case Intrinsic::amdgcn_flat_load_monitor_b64:
8969 case Intrinsic::amdgcn_flat_load_monitor_b128:
8970 assert(
MI.hasOneMemOperand() &&
"Expected IRTranslator to set MemOp!");
8971 B.buildInstr(AMDGPU::G_AMDGPU_FLAT_LOAD_MONITOR)
8972 .add(
MI.getOperand(0))
8973 .add(
MI.getOperand(2))
8974 .addMemOperand(*
MI.memoperands_begin());
8975 MI.eraseFromParent();
8977 case Intrinsic::amdgcn_global_load_monitor_b32:
8978 case Intrinsic::amdgcn_global_load_monitor_b64:
8979 case Intrinsic::amdgcn_global_load_monitor_b128:
8980 assert(
MI.hasOneMemOperand() &&
"Expected IRTranslator to set MemOp!");
8981 B.buildInstr(AMDGPU::G_AMDGPU_GLOBAL_LOAD_MONITOR)
8982 .add(
MI.getOperand(0))
8983 .add(
MI.getOperand(2))
8984 .addMemOperand(*
MI.memoperands_begin());
8985 MI.eraseFromParent();
MachineInstrBuilder & UseMI
MachineInstrBuilder MachineInstrBuilder & DefMI
static unsigned getIntrinsicID(const SDNode *N)
const TargetLowering & TLI
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
AMDGPU address space definition.
static SDValue extractF64Exponent(SDValue Hi, const SDLoc &SL, SelectionDAG &DAG)
static SDValue getMad(SelectionDAG &DAG, const SDLoc &SL, EVT VT, SDValue X, SDValue Y, SDValue C, SDNodeFlags Flags=SDNodeFlags())
static bool valueIsKnownNeverF32Denorm(SDValue Src)
Return true if it's known that Src can never be an f32 denormal value.
Contains the definition of a TargetInstrInfo class that is common to all AMD GPUs.
static void packImage16bitOpsToDwords(MachineIRBuilder &B, MachineInstr &MI, SmallVectorImpl< Register > &PackedAddrs, unsigned ArgOffset, const AMDGPU::ImageDimIntrinsicInfo *Intr, bool IsA16, bool IsG16)
Turn a set of f16 typed registers in AddrRegs into a dword sized vector with f16 typed elements.
static unsigned getBufferAtomicPseudo(Intrinsic::ID IntrID)
static LLT getBufferRsrcScalarType(const LLT Ty)
static LegalityPredicate isIllegalRegisterType(const GCNSubtarget &ST, unsigned TypeIdx)
static MachineInstrBuilder buildExp(MachineIRBuilder &B, const DstOp &Dst, const SrcOp &Src, unsigned Flags)
static bool needsDenormHandlingF32(const MachineFunction &MF, Register Src, unsigned Flags)
constexpr std::initializer_list< LLT > AllVectors
static LegalizeMutation bitcastToVectorElement32(unsigned TypeIdx)
static LegalityPredicate isSmallOddVector(unsigned TypeIdx)
static LegalizeMutation oneMoreElement(unsigned TypeIdx)
static LegalityPredicate vectorSmallerThan(unsigned TypeIdx, unsigned Size)
static bool allowApproxFunc(const MachineFunction &MF, unsigned Flags)
static bool shouldBitcastLoadStoreType(const GCNSubtarget &ST, const LLT Ty, const LLT MemTy)
Return true if a load or store of the type should be lowered with a bitcast to a different type.
static constexpr unsigned FPEnvModeBitField
static LegalizeMutation getScalarTypeFromMemDesc(unsigned TypeIdx)
static LegalityPredicate vectorWiderThan(unsigned TypeIdx, unsigned Size)
static bool shouldWidenLoad(const GCNSubtarget &ST, LLT MemoryTy, uint64_t AlignInBits, unsigned AddrSpace, unsigned Opcode)
Return true if we should legalize a load by widening an odd sized memory access up to the alignment.
static bool isRegisterVectorElementType(LLT EltTy)
static LegalizeMutation fewerEltsToSize64Vector(unsigned TypeIdx)
static LegalityPredicate isWideVec16(unsigned TypeIdx)
constexpr std::initializer_list< LLT > AllScalarTypes
static LegalityPredicate isTruncStoreToSizePowerOf2(unsigned TypeIdx)
constexpr std::initializer_list< LLT > AllS32Vectors
static LegalizeMutation moreElementsToNextExistingRegClass(unsigned TypeIdx)
static bool isVGPRLoadStoreSizeSupported(unsigned MemSize, unsigned ValSize)
static Register castBufferRsrcToV4I32(Register Pointer, MachineIRBuilder &B)
Cast a buffer resource (an address space 8 pointer) into a 4xi32, which is the form in which the valu...
static bool isRegisterClassType(const GCNSubtarget &ST, LLT Ty)
static std::pair< Register, Register > emitReciprocalU64(MachineIRBuilder &B, Register Val)
static LLT getBitcastRegisterType(const LLT Ty)
static LLT getBufferRsrcRegisterType(const LLT Ty)
static LegalizeMutation bitcastToRegisterType(unsigned TypeIdx)
static Register stripAnySourceMods(Register OrigSrc, MachineRegisterInfo &MRI)
static LLT castBufferRsrcFromV4I32(MachineInstr &MI, MachineIRBuilder &B, MachineRegisterInfo &MRI, unsigned Idx)
Mutates IR (typicaly a load instruction) to use a <4 x s32> as the initial type of the operand idx an...
static bool replaceWithConstant(MachineIRBuilder &B, MachineInstr &MI, int64_t C)
static constexpr unsigned SPDenormModeBitField
static unsigned maxSizeForAddrSpace(const GCNSubtarget &ST, unsigned AS, bool IsLoad, bool IsAtomic)
static bool isLoadStoreSizeLegal(const GCNSubtarget &ST, const LegalityQuery &Query)
static MachineInstr * verifyCFIntrinsic(MachineInstr &MI, MachineRegisterInfo &MRI, MachineInstr *&Br, MachineBasicBlock *&UncondBrTarget, bool &Negated)
static LegalityPredicate numElementsNotEven(unsigned TypeIdx)
static LLT getVGPRLoadStorePartType(LLT Ty)
constexpr std::initializer_list< LLT > AllS64Vectors
static bool loadStoreBitcastWorkaround(const GCNSubtarget &ST, const LLT Ty)
static void castBufferRsrcArgToV4I32(MachineInstr &MI, MachineIRBuilder &B, unsigned Idx)
static constexpr unsigned FPEnvTrapBitField
static constexpr unsigned MaxRegisterSize
static bool lowerLoadStoreVGPR(LegalizerHelper &Helper, GLoadStore &LdSt)
Lower a whole-dword load or store on AMDGPUAS::VGPR into G_AMDGPU_REG_LOAD / G_AMDGPU_REG_STORE.
static bool isRegisterSize(const GCNSubtarget &ST, unsigned Size)
static LegalityPredicate isWideScalarExtLoadTruncStore(unsigned TypeIdx)
static bool hasBufferRsrcWorkaround(const LLT Ty)
static void toggleSPDenormMode(bool Enable, MachineIRBuilder &B, const GCNSubtarget &ST, SIModeRegisterDefaults Mode)
constexpr std::initializer_list< LLT > AllS16Vectors
static LLT widenToNextPowerOf2(LLT Ty)
static bool isNot(const MachineRegisterInfo &MRI, const MachineInstr &MI)
static void convertImageAddrToPacked(MachineIRBuilder &B, MachineInstr &MI, int DimIdx, int NumVAddrs)
Convert from separate vaddr components to a single vector address register, and replace the remaining...
static bool isLoadStoreLegal(const GCNSubtarget &ST, const LegalityQuery &Query)
static LegalizeMutation moreEltsToNext32Bit(unsigned TypeIdx)
static LLT getPow2VectorType(LLT Ty)
static void buildBufferLoad(unsigned Opc, Register LoadDstReg, Register RSrc, Register VIndex, Register VOffset, Register SOffset, unsigned ImmOffset, unsigned Format, unsigned AuxiliaryData, MachineMemOperand *MMO, bool IsTyped, bool HasVIndex, MachineIRBuilder &B)
static LLT getPow2ScalarType(LLT Ty)
static LegalityPredicate elementTypeIsLegal(unsigned TypeIdx)
static bool isRegisterVectorType(LLT Ty)
static LegalityPredicate sizeIsMultipleOf32(unsigned TypeIdx)
static void buildTFEBufferLoad(unsigned Opc, ArrayRef< Register > ValueDsts, Register StatusDst, Register RSrc, Register VIndex, Register VOffset, Register SOffset, unsigned ImmOffset, unsigned Format, unsigned AuxiliaryData, MachineMemOperand *MMO, bool IsTyped, bool HasVIndex, MachineIRBuilder &B)
static bool isRegisterType(const GCNSubtarget &ST, LLT Ty)
static bool isKnownNonNull(Register Val, MachineRegisterInfo &MRI, const AMDGPUTargetMachine &TM, unsigned AddrSpace)
Return true if the value is a known valid address, such that a null check is not necessary.
This file declares the targeting of the Machinelegalizer class for AMDGPU.
The AMDGPU TargetMachine interface definition for hw codegen targets.
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
Declares convenience wrapper classes for interpreting MachineInstr instances as specific generic oper...
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
Interface for Targets to specify which operations they can successfully select and how the others sho...
Contains matchers for matching SSA Machine Instructions.
This file declares the MachineIRBuilder class.
Register const TargetRegisterInfo * TRI
Promote Memory to Register
static MCRegister getReg(const MCDisassembler *D, unsigned RC, unsigned RegNo)
SmallVector< MachineOperand, 4 > Cond
MachineRegisterInfo & MRI
static cl::opt< RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode > Mode("regalloc-enable-advisor", cl::Hidden, cl::init(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Default), cl::desc("Enable regalloc advisor mode"), cl::values(clEnumValN(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Default, "default", "Default"), clEnumValN(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Release, "release", "precompiled"), clEnumValN(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Development, "development", "for training")))
#define FP_DENORM_FLUSH_NONE
Interface definition for SIInstrInfo.
Interface definition for SIRegisterInfo.
This file defines the scope_exit class, which executes user-defined cleanup logic at scope exit.
static TableGen::Emitter::Opt Y("gen-skeleton-entry", EmitSkeleton, "Generate example skeleton entry")
static constexpr int Concat[]
bool legalizeConstHwRegRead(MachineInstr &MI, MachineIRBuilder &B, AMDGPU::Hwreg::Id HwReg, unsigned LowBit, unsigned Width) const
void buildMultiply(LegalizerHelper &Helper, MutableArrayRef< Register > Accum, ArrayRef< Register > Src0, ArrayRef< Register > Src1, bool UsePartialMad64_32, bool SeparateOddAlignedProducts) const
bool legalizeGlobalValue(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool legalizeFSQRTF16(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool legalizeIntrinsicTrunc(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool legalizeInsert(LegalizerHelper &Helper, MachineInstr &MI) const
bool legalizeTrapHsa(LegalizerHelper &Helper, MachineInstr &MI) const
std::pair< Register, unsigned > splitBufferOffsets(MachineIRBuilder &B, Register OrigOffset) const
bool legalizeBVHIntersectRayIntrinsic(MachineInstr &MI, MachineIRBuilder &B) const
bool legalizeIsAddrSpace(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B, unsigned AddrSpace) const
bool legalizeUnsignedDIV_REM(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool legalizeFSQRTF32(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool legalizeCTLZ_ZERO_POISON(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool legalizeAtomicCmpXChg(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool legalizeMul(LegalizerHelper &Helper, MachineInstr &MI) const
bool legalizeFFREXP(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
Register getSegmentAperture(unsigned AddrSpace, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool legalizeTrapEndpgm(LegalizerHelper &Helper, MachineInstr &MI) const
bool legalizeFDIV64(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool legalizePointerAsRsrcIntrin(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
To create a buffer resource from a 64-bit pointer, mask off the upper 32 bits of the pointer and repl...
bool legalizeFlogCommon(MachineInstr &MI, MachineIRBuilder &B) const
bool legalizeFrameIndexVGPR(MachineInstr &MI, MachineIRBuilder &B) const
bool legalizeFExp2(MachineInstr &MI, MachineIRBuilder &B) const
void legalizeUnsignedDIV_REM32Impl(MachineIRBuilder &B, Register DstDivReg, Register DstRemReg, Register Num, Register Den) const
Register handleD16VData(MachineIRBuilder &B, MachineRegisterInfo &MRI, Register Reg, bool ImageStore=false) const
Handle register layout difference for f16 images for some subtargets.
bool legalizeCTLZ_CTTZ(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool legalizeBuildVector(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool legalizeTrap(LegalizerHelper &Helper, MachineInstr &MI) const
bool legalizeBufferAtomic(MachineInstr &MI, MachineIRBuilder &B, const AMDGPU::BufferIntrinsicInfo &Info) const
bool legalizeFFloor(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
AMDGPULegalizerInfo(const GCNSubtarget &ST, const GCNTargetMachine &TM)
bool legalizeFDIV32(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool legalizeFMad(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool legalizeFDIV(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool legalizeSBufferPrefetch(LegalizerHelper &Helper, MachineInstr &MI) const
bool legalizeFExp10Unsafe(MachineIRBuilder &B, Register Dst, Register Src, unsigned Flags) const
bool legalizeFExp(MachineInstr &MI, MachineIRBuilder &B) const
bool legalizeIntrinsic(LegalizerHelper &Helper, MachineInstr &MI) const override
bool legalizeFrem(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool legalizePreloadedArgIntrin(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B, AMDGPUFunctionArgInfo::PreloadedValue ArgType) const
bool legalizeStore(LegalizerHelper &Helper, MachineInstr &MI) const
bool legalizeCustom(LegalizerHelper &Helper, MachineInstr &MI, LostDebugLocObserver &LocObserver) const override
Called for instructions with the Custom LegalizationAction.
bool buildPCRelGlobalAddress(Register DstReg, LLT PtrTy, MachineIRBuilder &B, const GlobalValue *GV, int64_t Offset, unsigned GAFlags=SIInstrInfo::MO_NONE) const
MachinePointerInfo getKernargSegmentPtrInfo(MachineFunction &MF) const
bool legalizeFDIV16(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool legalizeRsqClampIntrinsic(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool legalizeFExpUnsafeImpl(MachineIRBuilder &B, Register Dst, Register Src, unsigned Flags, bool IsExp10) const
std::pair< Register, Register > getScaledLogInput(MachineIRBuilder &B, Register Src, unsigned Flags) const
bool legalizeFDIVFastIntrin(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool loadInputValue(Register DstReg, MachineIRBuilder &B, AMDGPUFunctionArgInfo::PreloadedValue ArgType) const
bool legalizeBVHDualOrBVH8IntersectRayIntrinsic(MachineInstr &MI, MachineIRBuilder &B) const
bool legalizeInsertVectorElt(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool legalizeFExpUnsafe(MachineIRBuilder &B, Register Dst, Register Src, unsigned Flags) const
bool legalizeFEXPF64(MachineInstr &MI, MachineIRBuilder &B) const
bool legalizeAddrSpaceCast(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool legalizeExtract(LegalizerHelper &Helper, MachineInstr &MI) const
bool legalizeImplicitArgPtr(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool legalizeMinNumMaxNum(LegalizerHelper &Helper, MachineInstr &MI) const
void legalizeUnsignedDIV_REM64Impl(MachineIRBuilder &B, Register DstDivReg, Register DstRemReg, Register Num, Register Den) const
bool legalizeDebugTrap(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool legalizeBufferStore(MachineInstr &MI, LegalizerHelper &Helper, const AMDGPU::BufferIntrinsicInfo &Info) const
bool legalizeFastUnsafeFDIV(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool legalizeSinCos(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool legalizeCTLS(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool legalizeWaveID(MachineInstr &MI, MachineIRBuilder &B) const
bool legalizeFroundeven(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool legalizeWorkGroupId(MachineInstr &MI, MachineIRBuilder &B, AMDGPUFunctionArgInfo::PreloadedValue ClusterIdPV, AMDGPUFunctionArgInfo::PreloadedValue ClusterMaxIdPV, AMDGPUFunctionArgInfo::PreloadedValue ClusterWorkGroupIdPV) const
bool legalizeSignedDIV_REM(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool legalizeBufferLoad(MachineInstr &MI, LegalizerHelper &Helper, const AMDGPU::BufferIntrinsicInfo &Info) const
bool legalizeITOFP(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B, bool Signed) const
bool legalizeFastUnsafeFDIV64(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool legalizeFPTOI(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B, bool Signed) const
bool legalizeStackSave(MachineInstr &MI, MachineIRBuilder &B) const
bool legalizeFlogUnsafe(MachineIRBuilder &B, Register Dst, Register Src, bool IsLog10, unsigned Flags) const
bool legalizeKernargMemParameter(MachineInstr &MI, MachineIRBuilder &B, uint64_t Offset, Align Alignment=Align(4)) const
Legalize a value that's loaded from kernel arguments.
bool legalizeImageIntrinsic(MachineInstr &MI, MachineIRBuilder &B, GISelChangeObserver &Observer, const AMDGPU::ImageDimIntrinsicInfo *ImageDimIntr) const
Rewrite image intrinsics to use register layouts expected by the subtarget.
void buildAbsGlobalAddress(Register DstReg, LLT PtrTy, MachineIRBuilder &B, const GlobalValue *GV, MachineRegisterInfo &MRI) const
bool legalizeGetFPEnv(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool getImplicitArgPtr(Register DstReg, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool legalizeFSQRT(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
Register getKernargParameterPtr(MachineIRBuilder &B, int64_t Offset) const
bool legalizeSBufferLoad(LegalizerHelper &Helper, MachineInstr &MI) const
bool legalizeFPow(LegalizerHelper &Helper, MachineInstr &MI) const
bool legalizeFceil(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool legalizeFSQRTF64(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool legalizeExtractVectorElt(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool legalizeLoad(LegalizerHelper &Helper, MachineInstr &MI) const
Register fixStoreSourceType(MachineIRBuilder &B, Register VData, LLT MemTy, bool IsFormat) const
bool legalizeLaneOp(LegalizerHelper &Helper, MachineInstr &MI, Intrinsic::ID IID) const
bool legalizeSetFPEnv(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool legalizeWorkitemIDIntrinsic(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B, unsigned Dim, AMDGPUFunctionArgInfo::PreloadedValue ArgType) const
void buildLoadInputValue(Register DstReg, MachineIRBuilder &B, const ArgDescriptor *Arg, const TargetRegisterClass *ArgRC, LLT ArgTy) const
bool legalizeTrapHsaQueuePtr(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const
bool legalizeFlog2(MachineInstr &MI, MachineIRBuilder &B) const
static bool isSupportedBitWidth(unsigned Bits)
unsigned allocateBarrierGlobal(const DataLayout &DL, const GlobalVariable &GV)
bool isModuleEntryFunction() const
void setDynLDSAlign(const Function &F, const GlobalVariable &GV)
unsigned allocateLDSGlobal(const DataLayout &DL, const GlobalVariable &GV)
bool isBottomOfStack() const
bool isEntryFunction() const
bool isNoopAddrSpaceCast(const DataLayout &DL, unsigned SrcAS, unsigned DestAS) const override
Returns true if a cast between SrcAS and DestAS is a noop.
const std::array< unsigned, 3 > & getDims() const
static const fltSemantics & IEEEsingle()
static const fltSemantics & IEEEdouble()
static APFloat getQNaN(const fltSemantics &Sem, bool Negative=false, const APInt *payload=nullptr)
Factory for QNaN values.
static APFloat getSmallestNormalized(const fltSemantics &Sem, bool Negative=false)
Returns the smallest (by magnitude) normalized finite number in the given semantics.
static APFloat getLargest(const fltSemantics &Sem, bool Negative=false)
Returns the largest finite number in the given semantics.
static APFloat getInf(const fltSemantics &Sem, bool Negative=false)
Factory for Positive and Negative Infinity.
an instruction to allocate memory on the stack
unsigned getAddressSpace() const
Return the address space for the allocation.
Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
Get the array size.
@ FCMP_OEQ
0 0 0 1 True if ordered and equal
@ ICMP_SLT
signed less than
@ FCMP_OLT
0 1 0 0 True if ordered and less than
@ FCMP_ULE
1 1 0 1 True if unordered, less than, or equal
@ FCMP_OGT
0 0 1 0 True if ordered and greater than
@ ICMP_UGE
unsigned greater or equal
@ ICMP_SGT
signed greater than
@ FCMP_ONE
0 1 1 0 True if ordered and operands are unequal
@ ICMP_ULT
unsigned less than
@ FCMP_OLE
0 1 0 1 True if ordered and less than or equal
@ FCMP_ORD
0 1 1 1 True if ordered (no nans)
@ FCMP_UGE
1 0 1 1 True if unordered, greater than, or equal
ConstantFP - Floating Point Values [float, double].
bool isMinusOne() const
Returns true if this value is exactly -1.0.
bool isOne() const
Returns true if this value is exactly +1.0.
This is the shared class of boolean and integer constants.
int64_t getSExtValue() const
Return the constant as a 64-bit integer value after it has been sign extended as appropriate for the ...
Diagnostic information for unsupported feature in backend.
static constexpr ElementCount getFixed(ScalarTy MinVal)
Lightweight error class with error context and mandatory checking.
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
bool hasImplicitBufferPtr() const
Abstract class that contains various methods for clients to notify about changes.
virtual void changingInstr(MachineInstr &MI)=0
This instruction is about to be mutated in some way.
LLVM_ABI void finishedChangingAllUsesOfReg()
All instructions reported as changing by changingAllUsesOfReg() have finished being changed.
virtual void changedInstr(MachineInstr &MI)=0
This instruction was mutated in some way.
LLVM_ABI void changingAllUsesOfReg(const MachineRegisterInfo &MRI, Register Reg)
All the instructions using the given register are being changed.
Simple wrapper observer that takes several observers, and calls each one for each event.
KnownBits getKnownBits(Register R)
Represents any type of generic load or store.
Register getPointerReg() const
Get the source register of the pointer value.
MachineMemOperand & getMMO() const
Get the MachineMemOperand on this instruction.
Align getAlign() const
Return the minimum known alignment in bytes of the actual memory reference.
Register getReg(unsigned Idx) const
Access the Idx'th operand as a register and return it.
bool hasExternalLinkage() const
Module * getParent()
Get the module that this global value is contained inside of...
LLVM_ABI const DataLayout & getDataLayout() const
Get the data layout of the module this global belongs to.
LLVM_ABI uint64_t getGlobalSize(const DataLayout &DL) const
Get the size of this global variable in bytes.
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
static constexpr LLT float64()
Get a 64-bit IEEE double value.
LLT changeElementCount(ElementCount EC) const
Return a vector or scalar with the same element type and the new element count.
constexpr unsigned getScalarSizeInBits() const
constexpr bool isScalar() const
constexpr LLT changeElementType(LLT NewEltTy) const
If this type is a vector, return a vector with the same number of elements but the new element type.
static constexpr LLT vector(ElementCount EC, unsigned ScalarSizeInBits)
Get a low-level vector of some number of elements and element width.
LLT getScalarType() const
static constexpr LLT scalar(unsigned SizeInBits)
Get a low-level scalar or aggregate "bag of bits".
constexpr bool isValid() const
constexpr uint16_t getNumElements() const
Returns the number of elements in a vector LLT.
constexpr bool isFloat() const
constexpr bool isVector() const
static constexpr LLT pointer(unsigned AddressSpace, unsigned SizeInBits)
Get a low-level pointer in the given address space.
constexpr TypeSize getSizeInBits() const
Returns the total size of the type. Must only be called on sized types.
constexpr bool isPointer() const
static constexpr LLT float16()
Get a 16-bit IEEE half value.
constexpr unsigned getAddressSpace() const
static constexpr LLT fixed_vector(unsigned NumElements, unsigned ScalarSizeInBits)
Get a low-level fixed-width vector of some number of elements and element width.
constexpr bool isPointerOrPointerVector() const
static LLT integer(unsigned SizeInBits)
static constexpr LLT bfloat16()
LLT getElementType() const
Returns the vector's element type. Only valid for vector types.
static constexpr LLT scalarOrVector(ElementCount EC, LLT ScalarTy)
static constexpr LLT float32()
Get a 32-bit IEEE float value.
LLT changeElementSize(unsigned NewEltSize) const
If this type is a vector, return a vector with the same number of elements but the new element size.
LLVM_ABI void diagnose(const DiagnosticInfo &DI)
Report a message to the currently installed diagnostic handler.
LegalizeRuleSet & minScalar(unsigned TypeIdx, const LLT Ty)
Ensure the scalar is at least as wide as Ty.
LegalizeRuleSet & legalFor(std::initializer_list< LLT > Types)
The instruction is legal when type index 0 is any type in the given list.
LegalizeRuleSet & scalarSameSizeAs(unsigned TypeIdx, unsigned SameSizeIdx)
Change the type TypeIdx to have the same scalar size as type SameSizeIdx.
LegalizeRuleSet & fewerElementsIf(LegalityPredicate Predicate, LegalizeMutation Mutation)
Remove elements to reach the type selected by the mutation if the predicate is true.
LegalizeRuleSet & clampScalarOrElt(unsigned TypeIdx, const LLT MinTy, const LLT MaxTy)
Limit the range of scalar sizes to MinTy and MaxTy.
LegalizeRuleSet & maxScalar(unsigned TypeIdx, const LLT Ty)
Ensure the scalar is at most as wide as Ty.
LegalizeRuleSet & minScalarOrElt(unsigned TypeIdx, const LLT Ty)
Ensure the scalar or element is at least as wide as Ty.
LegalizeRuleSet & clampMaxNumElements(unsigned TypeIdx, const LLT EltTy, unsigned MaxElements)
Limit the number of elements in EltTy vectors to at most MaxElements.
LegalizeRuleSet & unsupportedFor(std::initializer_list< LLT > Types)
LegalizeRuleSet & widenScalarFor(std::initializer_list< LLT > Types, LegalizeMutation Mutation)
Widen the scalar, specified in mutation, when type index 0 is any type in the given list.
LegalizeRuleSet & lower()
The instruction is lowered.
LegalizeRuleSet & moreElementsIf(LegalityPredicate Predicate, LegalizeMutation Mutation)
Add more elements to reach the type selected by the mutation if the predicate is true.
LegalizeRuleSet & lowerFor(std::initializer_list< LLT > Types)
The instruction is lowered when type index 0 is any type in the given list.
LegalizeRuleSet & clampScalar(unsigned TypeIdx, const LLT MinTy, const LLT MaxTy)
Limit the range of scalar sizes to MinTy and MaxTy.
LegalizeRuleSet & custom()
Unconditionally custom lower.
LegalizeRuleSet & clampMaxNumElementsStrict(unsigned TypeIdx, const LLT EltTy, unsigned NumElts)
Express EltTy vectors strictly using vectors with NumElts elements (or scalars when NumElts equals 1)...
LegalizeRuleSet & widenScalarIf(LegalityPredicate Predicate, LegalizeMutation Mutation)
Widen the scalar to the one selected by the mutation if the predicate is true.
LegalizeRuleSet & alwaysLegal()
LegalizeRuleSet & maxScalarIf(LegalityPredicate Predicate, unsigned TypeIdx, const LLT Ty)
Conditionally limit the maximum size of the scalar.
LegalizeRuleSet & customIf(LegalityPredicate Predicate)
LegalizeRuleSet & widenScalarToNextPow2(unsigned TypeIdx, unsigned MinSize=0)
Widen the scalar to the next power of two that is at least MinSize.
LegalizeRuleSet & scalarize(unsigned TypeIdx)
LegalizeRuleSet & legalForCartesianProduct(std::initializer_list< LLT > Types)
The instruction is legal when type indexes 0 and 1 are both in the given list.
LegalizeRuleSet & minScalarIf(LegalityPredicate Predicate, unsigned TypeIdx, const LLT Ty)
Ensure the scalar is at least as wide as Ty if condition is met.
LegalizeRuleSet & legalIf(LegalityPredicate Predicate)
The instruction is legal if predicate is true.
LegalizeRuleSet & customFor(std::initializer_list< LLT > Types)
LegalizeRuleSet & widenScalarToNextMultipleOf(unsigned TypeIdx, unsigned Size)
Widen the scalar to the next multiple of Size.
LLVM_ABI LegalizeResult lowerFMinNumMaxNum(MachineInstr &MI)
LLVM_ABI void moreElementsVectorDst(MachineInstr &MI, LLT MoreTy, unsigned OpIdx)
Legalize a single operand OpIdx of the machine instruction MI as a Def by performing it with addition...
LLVM_ABI LegalizeResult bitcast(MachineInstr &MI, unsigned TypeIdx, LLT Ty)
Legalize an instruction by replacing the value type.
LLVM_ABI LegalizeResult lowerInsert(MachineInstr &MI)
LLVM_ABI LegalizeResult lowerExtract(MachineInstr &MI)
GISelValueTracking * getValueTracking() const
LLVM_ABI LegalizeResult reduceLoadStoreWidth(GLoadStore &MI, unsigned TypeIdx, LLT NarrowTy)
@ Legalized
Instruction has been legalized and the MachineFunction changed.
GISelChangeObserver & Observer
To keep track of changes made by the LegalizerHelper.
LLVM_ABI void bitcastDst(MachineInstr &MI, LLT CastTy, unsigned OpIdx)
Legalize a single operand OpIdx of the machine instruction MI as a def by inserting a G_BITCAST from ...
LLVM_ABI LegalizeResult lowerFMad(MachineInstr &MI)
MachineIRBuilder & MIRBuilder
Expose MIRBuilder so clients can set their own RecordInsertInstruction functions.
LLVM_ABI void widenScalarDst(MachineInstr &MI, LLT WideTy, unsigned OpIdx=0, unsigned TruncOpcode=TargetOpcode::G_TRUNC)
Legalize a single operand OpIdx of the machine instruction MI as a Def by extending the operand's typ...
LegalizeRuleSet & getActionDefinitionsBuilder(unsigned Opcode)
Get the action definition builder for the given opcode.
TypeSize getValue() const
Wrapper class representing physical registers. Should be passed by value.
constexpr bool isPhysical() const
Return true if the specified register number is in the physical register namespace.
LLVM_ABI void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
LLVM_ABI MachineBasicBlock * splitAt(MachineInstr &SplitInst, bool UpdateLiveIns=true, LiveIntervals *LIS=nullptr)
Split a basic block into 2 pieces at SplitPoint.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
MachineInstrBundleIterator< MachineInstr > iterator
const AllocaInst * getObjectAllocation(int ObjectIdx) const
Return the underlying Alloca of the specified stack object if it exists.
PseudoSourceValueManager & getPSVManager() const
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
DenormalMode getDenormalMode(const fltSemantics &FPType) const
Returns the denormal handling type for the default rounding mode of the function.
void push_back(MachineBasicBlock *MBB)
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
const DataLayout & getDataLayout() const
Return the DataLayout attached to the Module associated to this MF.
Function & getFunction()
Return the LLVM function that this machine code represents.
BasicBlockListType::iterator iterator
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags F, LLT MemTy, Align BaseAlignment, const MMOMetadata &Metadata=MMOMetadata(), SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *BB=nullptr, std::optional< UniqueBBID > BBID=std::nullopt)
CreateMachineInstr - Allocate a new MachineInstr.
const TargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
Helper class to build MachineInstr.
MachineFunction & getMF()
Getter for the function we currently build.
Register getReg(unsigned Idx) const
Get the register for the operand index.
const MachineInstrBuilder & setOperandDead(unsigned OpIdx) const
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & addGlobalAddress(const GlobalValue *GV, int64_t Offset=0, unsigned TargetFlags=0) const
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
Representation of each machine instruction.
const DebugLoc & getDebugLoc() const
Returns the debug location id of this MachineInstr.
const MachineOperand & getOperand(unsigned i) const
LLVM_ABI MachineInstrBundleIterator< MachineInstr > eraseFromParent()
Unlink 'this' from the containing basic block and delete it.
A description of a memory reference used in the backend.
LocationSize getSize() const
Return the size in bytes of the memory reference.
LLT getMemoryType() const
Return the memory type of the memory reference.
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MOLoad
The memory access reads data.
@ MOInvariant
The memory access always returns the same value (or traps).
LLVM_ABI Align getAlign() const
Return the minimum known alignment in bytes of the actual memory reference.
MachineOperand class - Representation of each machine instruction operand.
MachineBasicBlock * getMBB() const
LLVM_ABI void setReg(Register Reg)
Change the register this operand corresponds to.
void setMBB(MachineBasicBlock *MBB)
static MachineOperand CreateImm(int64_t Val)
Register getReg() const
getReg - Returns the register number.
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
LLVM_ABI Register createGenericVirtualRegister(LLT Ty, StringRef Name="")
Create and return a new generic virtual register with low-level type Ty.
Represent a mutable reference to an array (0 or more elements consecutively in memory),...
MutableArrayRef< T > drop_front(size_t N=1) const
Drop the first N elements of the array.
LLVM_ABI const PseudoSourceValue * getConstantPool()
Return a pseudo source value referencing the constant pool.
Wrapper class representing virtual and physical registers.
constexpr bool isValid() const
constexpr bool isVirtual() const
Return true if the specified register number is in the virtual register namespace.
static unsigned getMaxMUBUFImmOffset(const GCNSubtarget &ST)
This class keeps track of the SPI_SP_INPUT_ADDR config register, which tells the hardware which inter...
GCNUserSGPRUsageInfo & getUserSGPRInfo()
bool hasWorkGroupIDZ() const
AMDGPU::ClusterDimsAttr getClusterDims() const
SIModeRegisterDefaults getMode() const
std::tuple< const ArgDescriptor *, const TargetRegisterClass *, LLT > getPreloadedValue(AMDGPUFunctionArgInfo::PreloadedValue Value) const
static LLVM_READONLY const TargetRegisterClass * getSGPRClassForBitWidth(unsigned BitWidth)
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
A Use represents the edge between a Value definition and its users.
LLVM Value Representation.
LLVM_ABI StringRef getName() const
Return a constant reference to the value's name.
self_iterator getIterator()
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ CONSTANT_ADDRESS_32BIT
Address space for 32-bit constant memory.
@ BUFFER_STRIDED_POINTER
Address space for 192-bit fat buffer pointers with an additional index.
@ BARRIER
Address space for modeling barrier IDs as addresses.
@ REGION_ADDRESS
Address space for region memory. (GDS)
@ LOCAL_ADDRESS
Address space for local memory.
@ VGPR
Address space for VGPRs.
@ CONSTANT_ADDRESS
Address space for constant memory (VTX2).
@ FLAT_ADDRESS
Address space for flat memory.
@ GLOBAL_ADDRESS
Address space for global memory (RAT0, VTX0).
@ BUFFER_FAT_POINTER
Address space for 160-bit buffer fat pointers.
@ PRIVATE_ADDRESS
Address space for private memory.
@ BUFFER_RESOURCE
Address space for 128-bit buffer resources.
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
bool isFlatGlobalAddrSpace(unsigned AS)
bool isGFX12Plus(const MCSubtargetInfo &STI)
constexpr int64_t getNullPointerValue(unsigned AS)
Get the null pointer value for the given address space.
bool isGFX11(const MCSubtargetInfo &STI)
LLVM_READNONE bool isLegalDPALU_DPPControl(const MCSubtargetInfo &ST, unsigned DC)
unsigned getAMDHSACodeObjectVersion(const Module &M)
int getMIMGOpcode(unsigned BaseOpcode, unsigned MIMGEncoding, unsigned VDataDwords, unsigned VAddrDwords, bool IndexedRsrc, bool IndexedSamp)
LLVM_READNONE constexpr bool isKernel(CallingConv::ID CC)
LLVM_READNONE constexpr bool isEntryFunctionCC(CallingConv::ID CC)
LLVM_READNONE constexpr bool isCompute(CallingConv::ID CC)
const BufferIntrinsicInfo * lookupBufferIntrinsic(unsigned Intr)
TargetExtType * isNamedBarrier(const GlobalVariable &GV)
bool isGFX11Plus(const MCSubtargetInfo &STI)
unsigned getMIROperandIdx(const MachineInstr &MI, unsigned Arg)
Convert IR argument index Arg of the intrinsic called by MI, a G_INTRINSIC* instruction,...
LLVM_READONLY const MIMGBaseOpcodeInfo * getMIMGBaseOpcodeInfo(unsigned BaseOpcode)
unsigned getSyntheticApertureNumber(unsigned AS)
LLVM_READNONE constexpr bool isChainCC(CallingConv::ID CC)
std::pair< Register, unsigned > getBaseWithConstantOffset(MachineRegisterInfo &MRI, Register Reg, GISelValueTracking *ValueTracking=nullptr, bool CheckNUW=false)
Returns base register and constant offset.
const ImageDimIntrinsicInfo * getImageDimIntrinsicInfo(unsigned Intr)
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
@ AMDGPU_Gfx
Used for AMD graphics targets.
LLVM_ABI LegalityPredicate scalarOrEltWiderThan(unsigned TypeIdx, unsigned Size)
True iff the specified type index is a scalar or a vector with an element type that's wider than the ...
LLVM_ABI LegalityPredicate isScalar(unsigned TypeIdx)
True iff the specified type index is a scalar.
LLVM_ABI LegalityPredicate isPointer(unsigned TypeIdx)
True iff the specified type index is a pointer (with any address space).
LLVM_ABI LegalityPredicate typeInSet(unsigned TypeIdx, std::initializer_list< LLT > TypesInit)
True iff the given type index is one of the specified types.
LLVM_ABI LegalityPredicate smallerThan(unsigned TypeIdx0, unsigned TypeIdx1)
True iff the first type index has a smaller total bit size than second type index.
LLVM_ABI LegalityPredicate largerThan(unsigned TypeIdx0, unsigned TypeIdx1)
True iff the first type index has a larger total bit size than second type index.
LLVM_ABI LegalityPredicate elementTypeIs(unsigned TypeIdx, LLT EltTy)
True if the type index is a vector with element type EltTy.
LLVM_ABI LegalityPredicate sameSize(unsigned TypeIdx0, unsigned TypeIdx1)
True iff the specified type indices are both the same bit size.
LLVM_ABI LegalityPredicate scalarOrEltNarrowerThan(unsigned TypeIdx, unsigned Size)
True iff the specified type index is a scalar or vector with an element type that's narrower than the...
LegalityPredicate typeIsNot(unsigned TypeIdx, LLT Type)
True iff the given type index is not the specified type.
Predicate all(Predicate P0, Predicate P1)
True iff P0 and P1 are true.
LLVM_ABI LegalityPredicate typeIs(unsigned TypeIdx, LLT TypesInit)
True iff the given type index is the specified type.
LLVM_ABI LegalityPredicate scalarNarrowerThan(unsigned TypeIdx, unsigned Size)
True iff the specified type index is a scalar that's narrower than the given size.
LLVM_ABI LegalizeMutation changeElementCountTo(unsigned TypeIdx, unsigned FromTypeIdx)
Keep the same scalar or element type as TypeIdx, but take the number of elements from FromTypeIdx.
LLVM_ABI LegalizeMutation scalarize(unsigned TypeIdx)
Break up the vector type for the given type index into the element type.
LLVM_ABI LegalizeMutation changeElementTo(unsigned TypeIdx, unsigned FromTypeIdx)
Keep the same scalar or element type as the given type index.
LLVM_ABI LegalizeMutation widenScalarOrEltToNextPow2(unsigned TypeIdx, unsigned Min=0)
Widen the scalar type or vector element type for the given type index to the next power of 2.
LLVM_ABI LegalizeMutation changeTo(unsigned TypeIdx, LLT Ty)
Select this specific type for the given type index.
LLVM_ABI LegalizeMutation changeElementSizeTo(unsigned TypeIdx, unsigned FromTypeIdx)
Change the scalar size or element size to have the same scalar size as type index FromIndex.
Invariant opcodes: All instruction sets have these as their low opcodes.
This is an optimization pass for GlobalISel generic memory operations.
LLVM_ABI Register getFunctionLiveInPhysReg(MachineFunction &MF, const TargetInstrInfo &TII, MCRegister PhysReg, const TargetRegisterClass &RC, const DebugLoc &DL, LLT RegTy=LLT())
Return a virtual register corresponding to the incoming argument register PhysReg.
unsigned Log2_32_Ceil(uint32_t Value)
Return the ceil log base 2 of the specified value, 32 if the value is zero.
LLVM_ABI Type * getTypeForLLT(LLT Ty, LLVMContext &C)
Get the type back from LLT.
LLVM_ABI MachineInstr * getOpcodeDef(unsigned Opcode, Register Reg, const MachineRegisterInfo &MRI)
See if Reg is defined by an single def instruction that is Opcode.
LLVM_ABI const ConstantFP * getConstantFPVRegVal(Register VReg, const MachineRegisterInfo &MRI)
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
constexpr bool isInt(int64_t x)
Checks if an integer fits into the given bit width.
@ Implicit
Not emitted register (e.g. carry, or temporary result).
@ Undef
Value of the register doesn't matter.
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
LLVM_ABI const llvm::fltSemantics & getFltSemanticForLLT(LLT Ty)
Get the appropriate floating point arithmetic semantic based on the bit size of the given scalar LLT.
@ Load
The value being inserted comes from a load (InsertElement only).
std::function< std::pair< unsigned, LLT >(const LegalityQuery &)> LegalizeMutation
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
int bit_width(T Value)
Returns the number of bits needed to represent Value if Value is nonzero.
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
constexpr int popcount(T Value) noexcept
Count the number of set bits in a value.
uint64_t PowerOf2Ceil(uint64_t A)
Returns the power of two which is greater than or equal to the given value.
LLVM_ABI std::optional< int64_t > getIConstantVRegSExtVal(Register VReg, const MachineRegisterInfo &MRI)
If VReg is defined by a G_CONSTANT fits in int64_t returns it.
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
constexpr bool has_single_bit(T Value) noexcept
std::function< bool(const LegalityQuery &)> LegalityPredicate
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
FPClassTest
Floating-point class tests, supported by 'is_fpclass' intrinsic.
constexpr uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
MutableArrayRef(T &OneElt) -> MutableArrayRef< T >
constexpr T divideCeil(U Numerator, V Denominator)
Returns the integer ceil(Numerator / Denominator).
To bit_cast(const From &from) noexcept
@ Mul
Product of integers.
@ Sub
Subtraction of integers.
@ Fast
Assign the register banks as fast as possible (default).
DWARFExpression::Operation Op
ArrayRef(const T &OneElt) -> ArrayRef< T >
constexpr unsigned BitWidth
LLVM_ABI void eraseInstr(MachineInstr &MI, MachineRegisterInfo &MRI, LostDebugLocObserver *LocObserver=nullptr)
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
LLVM_ABI std::optional< ValueAndVReg > getIConstantVRegValWithLookThrough(Register VReg, const MachineRegisterInfo &MRI, bool LookThroughInstrs=true)
If VReg is defined by a statically evaluable chain of instructions rooted on a G_CONSTANT returns its...
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Next
unsigned Log2(Align A)
Returns the log2 of the alignment.
T bit_floor(T Value)
Returns the largest integral power of two no greater than Value if Value is nonzero.
constexpr uint64_t NextPowerOf2(uint64_t A)
Returns the next power of two (in 64-bits) that is strictly greater than A.
MCRegisterClass TargetRegisterClass
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
@ CLUSTER_WORKGROUP_MAX_ID_X
@ CLUSTER_WORKGROUP_MAX_ID_Z
@ CLUSTER_WORKGROUP_MAX_FLAT_ID
@ CLUSTER_WORKGROUP_MAX_ID_Y
The argument layout of a buffer intrinsic.
static constexpr uint64_t encode(Fields... Values)
MIMGBaseOpcode BaseOpcode
This struct is a compact representation of a valid (non-zero power of two) alignment.
constexpr uint64_t value() const
This is a hole in the type system and should not be abused.
MCRegister getRegister() const
static ArgDescriptor createRegister(Register Reg, unsigned Mask=~0u)
DenormalModeKind Input
Denormal treatment kind for floating point instruction inputs in the default floating-point environme...
@ PreserveSign
The sign of a flushed-to-zero number is preserved in the sign of 0.
@ Dynamic
Denormals have unknown treatment.
static constexpr DenormalMode getPreserveSign()
static constexpr DenormalMode getIEEE()
bool isZero() const
Returns true if value is all zero.
The LegalityQuery object bundles together all the information that's needed to decide whether a given...
ArrayRef< MemDesc > MMODescrs
Operations which require memory can use this to place requirements on the memory type for each MMO.
This class contains a discriminated union of information about pointers in memory operands,...
MachinePointerInfo getWithOffset(int64_t O) const
static LLVM_ABI MachinePointerInfo getGOT(MachineFunction &MF)
Return a MachinePointerInfo record that refers to a GOT entry.
DenormalMode FP64FP16Denormals
If this is set, neither input or output denormals are flushed for both f64 and f16/v2f16 instructions...
bool IEEE
Floating point opcodes that support exception flag gathering quiet and propagate signaling NaN inputs...
DenormalMode FP32Denormals
If this is set, neither input or output denormals are flushed for most f32 instructions.