48#include "llvm/IR/IntrinsicsAMDGPU.h"
49#include "llvm/IR/IntrinsicsR600.h"
61#define DEBUG_TYPE "si-lower"
80 unsigned NumSGPRs = AMDGPU::SGPR_32RegClass.getNumRegs();
81 for (
unsigned Reg = 0;
Reg < NumSGPRs; ++
Reg) {
83 return AMDGPU::SGPR0 +
Reg;
99 TRI->getDefaultVectorSuperClassForBitWidth(32);
105 TRI->getDefaultVectorSuperClassForBitWidth(64);
143 TRI->getDefaultVectorSuperClassForBitWidth(320));
147 TRI->getDefaultVectorSuperClassForBitWidth(352));
151 TRI->getDefaultVectorSuperClassForBitWidth(384));
155 TRI->getDefaultVectorSuperClassForBitWidth(512));
162 TRI->getDefaultVectorSuperClassForBitWidth(1024));
164 if (Subtarget->has16BitInsts()) {
165 if (Subtarget->useRealTrue16Insts()) {
195 TRI->getDefaultVectorSuperClassForBitWidth(1024));
211 {MVT::v2i32, MVT::v3i32, MVT::v4i32, MVT::v5i32,
212 MVT::v6i32, MVT::v7i32, MVT::v8i32, MVT::v9i32,
213 MVT::v10i32, MVT::v11i32, MVT::v12i32, MVT::v16i32,
214 MVT::i1, MVT::v32i32},
218 {MVT::v2i32, MVT::v3i32, MVT::v4i32, MVT::v5i32,
219 MVT::v6i32, MVT::v7i32, MVT::v8i32, MVT::v9i32,
220 MVT::v10i32, MVT::v11i32, MVT::v12i32, MVT::v16i32,
221 MVT::i1, MVT::v32i32},
239 if (Subtarget->hasBF16PackedInsts()) {
309 {MVT::f32, MVT::i32, MVT::i64, MVT::f64, MVT::i1},
Expand);
316 {MVT::v2i32, MVT::v3i32, MVT::v4i32, MVT::v5i32,
317 MVT::v6i32, MVT::v7i32, MVT::v8i32, MVT::v9i32,
318 MVT::v10i32, MVT::v11i32, MVT::v12i32, MVT::v16i32},
321 {MVT::v2f32, MVT::v3f32, MVT::v4f32, MVT::v5f32,
322 MVT::v6f32, MVT::v7f32, MVT::v8f32, MVT::v9f32,
323 MVT::v10f32, MVT::v11f32, MVT::v12f32, MVT::v16f32},
327 {MVT::v2i1, MVT::v4i1, MVT::v2i8, MVT::v4i8, MVT::v2i16,
328 MVT::v3i16, MVT::v4i16, MVT::Other},
333 {MVT::i1, MVT::i32, MVT::i64, MVT::f32, MVT::f64},
Expand);
349 {MVT::v8i32, MVT::v8f32, MVT::v9i32, MVT::v9f32, MVT::v10i32,
350 MVT::v10f32, MVT::v11i32, MVT::v11f32, MVT::v12i32, MVT::v12f32,
351 MVT::v16i32, MVT::v16f32, MVT::v2i64, MVT::v2f64, MVT::v4i16,
352 MVT::v4f16, MVT::v4bf16, MVT::v3i64, MVT::v3f64, MVT::v6i32,
353 MVT::v6f32, MVT::v4i64, MVT::v4f64, MVT::v8i64, MVT::v8f64,
354 MVT::v8i16, MVT::v8f16, MVT::v8bf16, MVT::v16i16, MVT::v16f16,
355 MVT::v16bf16, MVT::v16i64, MVT::v16f64, MVT::v32i32, MVT::v32f32,
356 MVT::v32i16, MVT::v32f16, MVT::v32bf16}) {
391 for (
MVT Vec64 : {MVT::v2i64, MVT::v2f64}) {
405 for (
MVT Vec64 : {MVT::v3i64, MVT::v3f64}) {
419 for (
MVT Vec64 : {MVT::v4i64, MVT::v4f64}) {
433 for (
MVT Vec64 : {MVT::v8i64, MVT::v8f64}) {
447 for (
MVT Vec64 : {MVT::v16i64, MVT::v16f64}) {
462 {MVT::v4i32, MVT::v4f32, MVT::v8i32, MVT::v8f32,
463 MVT::v16i32, MVT::v16f32, MVT::v32i32, MVT::v32f32},
466 if (Subtarget->hasPkMovB32()) {
487 {MVT::v2i16, MVT::v2f16, MVT::v2bf16, MVT::v2i8, MVT::v4i8,
488 MVT::v8i8, MVT::v4i16, MVT::v4f16, MVT::v4bf16},
493 {MVT::v3i32, MVT::v3f32, MVT::v4i32, MVT::v4f32},
Custom);
497 {MVT::v5i32, MVT::v5f32, MVT::v6i32, MVT::v6f32,
498 MVT::v7i32, MVT::v7f32, MVT::v8i32, MVT::v8f32,
499 MVT::v9i32, MVT::v9f32, MVT::v10i32, MVT::v10f32,
500 MVT::v11i32, MVT::v11f32, MVT::v12i32, MVT::v12f32},
524 if (Subtarget->hasSMemRealTime() ||
529 if (Subtarget->has16BitInsts()) {
539 if (Subtarget->hasMadMacF32Insts())
557 if (Subtarget->hasIntClamp())
560 if (Subtarget->hasAddNoCarryInsts())
566 if (Subtarget->hasIEEEMinimumMaximumInsts()) {
569 {MVT::f64, MVT::f32},
Legal);
573 {MVT::f64, MVT::f32},
Custom);
578 {MVT::f64, MVT::f32},
Legal);
581 if (Subtarget->haveRoundOpsF64())
611 if (Subtarget->has16BitInsts()) {
659 if (!Subtarget->useRealTrue16Insts() ||
660 !Subtarget->d16PreservesUnusedBits()) {
661 for (
MVT VT : {MVT::i16, MVT::f16, MVT::bf16}) {
667 if (!Subtarget->useRealTrue16Insts()) {
668 for (
MVT VT : {MVT::i16, MVT::f16, MVT::bf16}) {
680 if (Subtarget->hasBF16TransInsts())
696 {MVT::v2i16, MVT::v2f16, MVT::v2bf16, MVT::v4i16, MVT::v4f16,
697 MVT::v4bf16, MVT::v8i16, MVT::v8f16, MVT::v8bf16, MVT::v16i16,
698 MVT::v16f16, MVT::v16bf16, MVT::v32i16, MVT::v32f16}) {
733 if (Subtarget->hasVCvtPkIU16F32())
736 {MVT::v2i16, MVT::v4i16, MVT::v8i16, MVT::v16i16, MVT::v32i16},
743 {MVT::v2i16, MVT::v2f16, MVT::v2bf16},
Legal);
843 {MVT::v2f16, MVT::v2bf16, MVT::v4f16, MVT::v4bf16,
844 MVT::v8f16, MVT::v8bf16, MVT::v16f16, MVT::v16bf16,
845 MVT::v32f16, MVT::v32bf16},
847 if (Subtarget->hasIEEEMinimumMaximumInsts()) {
854 {MVT::v4f16, MVT::v8f16, MVT::v16f16, MVT::v32f16},
Custom);
865 {MVT::v4f16, MVT::v8f16, MVT::v16f16, MVT::v32f16},
869 {MVT::v4f16, MVT::v8f16, MVT::v16f16, MVT::v32f16},
874 {MVT::v8i16, MVT::v8f16, MVT::v8bf16, MVT::v16i16, MVT::v16f16,
875 MVT::v16bf16, MVT::v32i16, MVT::v32f16, MVT::v32bf16}) {
882 if (Subtarget->hasVOP3PInsts()) {
893 {MVT::v2i16, MVT::v2f16, MVT::v2bf16},
Custom);
896 {MVT::v4f16, MVT::v4i16, MVT::v4bf16, MVT::v8f16,
897 MVT::v8i16, MVT::v8bf16, MVT::v16f16, MVT::v16i16,
898 MVT::v16bf16, MVT::v32f16, MVT::v32i16, MVT::v32bf16},
901 for (
MVT VT : {MVT::v4i16, MVT::v8i16, MVT::v16i16, MVT::v32i16})
909 for (
MVT VT : {MVT::v4f16, MVT::v8f16, MVT::v16f16, MVT::v32f16})
915 if (Subtarget->hasIEEEMinimumMaximumInsts()) {
925 {MVT::v2f16, MVT::v4f16},
Custom);
931 if (Subtarget->hasBF16PackedInsts()) {
937 for (
MVT VT : {MVT::v4bf16, MVT::v8bf16, MVT::v16bf16, MVT::v32bf16})
945 if (Subtarget->hasAnyPackedFP32Ops()) {
949 {MVT::v4f32, MVT::v8f32, MVT::v16f32, MVT::v32f32},
952 if (Subtarget->hasAnyPackedFP64Ops()) {
958 {MVT::v4f64, MVT::v8f64, MVT::v16f64, MVT::v32f64},
Custom);
960 if (Subtarget->hasIEEEMinimumMaximumInsts()) {
967 {MVT::v4f64, MVT::v8f64, MVT::v16f64, MVT::v32f64},
Custom);
976 {MVT::v4f64, MVT::v8f64, MVT::v16f64, MVT::v32f64},
981 if (Subtarget->hasAnyPackedU64Ops()) {
985 {MVT::v4i64, MVT::v8i64, MVT::v16i64, MVT::v32i64},
992 if (Subtarget->has16BitInsts()) {
1007 {MVT::v4i16, MVT::v4f16, MVT::v4bf16, MVT::v2i8, MVT::v4i8,
1008 MVT::v8i8, MVT::v8i16, MVT::v8f16, MVT::v8bf16,
1009 MVT::v16i16, MVT::v16f16, MVT::v16bf16, MVT::v32i16,
1010 MVT::v32f16, MVT::v32bf16},
1015 if (Subtarget->useVMulU64Inst())
1017 else if (Subtarget->hasScalarSMulU64())
1020 if (Subtarget->hasMad64_32())
1023 if (Subtarget->hasSafeSmemPrefetch() || Subtarget->hasVmemPrefInsts())
1029 if (Subtarget->hasIEEEMinimumMaximumInsts()) {
1031 {MVT::f16, MVT::f32, MVT::f64, MVT::v2f16},
Legal);
1034 if (Subtarget->hasMinimum3Maximum3F32())
1037 if (Subtarget->hasMinimum3Maximum3PKF16()) {
1041 if (!Subtarget->hasMinimum3Maximum3F16())
1047 if (Subtarget->hasVOP3PInsts()) {
1050 {MVT::v4f16, MVT::v8f16, MVT::v16f16, MVT::v32f16},
1054 if (Subtarget->useMinMaxI64Insts())
1059 {MVT::Other, MVT::f32, MVT::v4f32, MVT::i16, MVT::f16,
1060 MVT::bf16, MVT::v2i16, MVT::v2f16, MVT::v2bf16, MVT::i128,
1065 {MVT::v2f16, MVT::v2i16, MVT::v2bf16, MVT::v3f16,
1066 MVT::v3i16, MVT::v4f16, MVT::v4i16, MVT::v4bf16,
1067 MVT::v8i16, MVT::v8f16, MVT::v8bf16, MVT::Other, MVT::f16,
1068 MVT::i16, MVT::bf16, MVT::i8, MVT::i128},
1083 SBufferLoadDiagnosticVTs.set(VT.SimpleTy);
1088 {MVT::Other, MVT::v2i16, MVT::v2f16, MVT::v2bf16,
1089 MVT::v3i16, MVT::v3f16, MVT::v4f16, MVT::v4i16,
1090 MVT::v4bf16, MVT::v8i16, MVT::v8f16, MVT::v8bf16,
1091 MVT::f16, MVT::i16, MVT::bf16, MVT::i8, MVT::i128},
1106 if (Subtarget->hasBF16ConversionInsts()) {
1108 {MVT::bf16, MVT::v2bf16},
Custom);
1112 if (Subtarget->hasBF16TransInsts()) {
1116 const bool HasE5M3ConversionInsts =
1117 Subtarget->hasFP8ConversionInsts() && Subtarget->hasFP8E5M3Insts();
1118 if (Subtarget->hasOCPFP8ConversionInsts() || HasE5M3ConversionInsts) {
1129 if (Subtarget->hasFP8F16ConversionInsts()) {
1134 if (Subtarget->hasCvtPkF16F32Inst()) {
1136 {MVT::v2f16, MVT::v4f16, MVT::v8f16, MVT::v16f16},
1189 if (Subtarget->has16BitInsts() && !Subtarget->hasMed3_16())
1230 static const MCPhysReg RCRegs[] = {AMDGPU::MODE};
1243 EVT DestVT,
EVT SrcVT)
const {
1245 ((((Opcode ==
ISD::FMAD && Subtarget->hasMadMixInsts()) ||
1246 (Opcode ==
ISD::FMA && Subtarget->hasFmaMixInsts())) &&
1248 (Opcode ==
ISD::FMA && Subtarget->hasFmaMixBF16Insts() &&
1255 LLT DestTy,
LLT SrcTy)
const {
1256 return ((Opcode == TargetOpcode::G_FMAD && Subtarget->hasMadMixInsts()) ||
1257 (Opcode == TargetOpcode::G_FMA && Subtarget->hasFmaMixInsts())) &&
1259 SrcTy.getScalarSizeInBits() == 16 &&
1280 return Subtarget->has16BitInsts()
1286 return Subtarget->has16BitInsts() ? MVT::i16 : MVT::i32;
1290 if (!Subtarget->has16BitInsts() && VT.
getSizeInBits() == 16)
1312 return (NumElts + 1) / 2;
1318 return NumElts * ((
Size + 31) / 32);
1327 unsigned &NumIntermediates,
MVT &RegisterVT)
const {
1336 MVT SimpleIntermediateVT =
1338 IntermediateVT = SimpleIntermediateVT;
1339 RegisterVT = Subtarget->has16BitInsts() ? SimpleIntermediateVT : MVT::i32;
1340 NumIntermediates = (NumElts + 1) / 2;
1341 return (NumElts + 1) / 2;
1346 IntermediateVT = RegisterVT;
1347 NumIntermediates = NumElts;
1348 return NumIntermediates;
1353 RegisterVT = MVT::i16;
1354 IntermediateVT = ScalarVT;
1355 NumIntermediates = NumElts;
1356 return NumIntermediates;
1360 RegisterVT = MVT::i32;
1361 IntermediateVT = ScalarVT;
1362 NumIntermediates = NumElts;
1363 return NumIntermediates;
1367 RegisterVT = MVT::i32;
1368 IntermediateVT = RegisterVT;
1369 NumIntermediates = NumElts * ((
Size + 31) / 32);
1370 return NumIntermediates;
1375 Context,
CC, VT, IntermediateVT, NumIntermediates, RegisterVT);
1380 unsigned MaxNumLanes) {
1381 assert(MaxNumLanes != 0);
1385 unsigned NumElts = std::min(MaxNumLanes, VT->getNumElements());
1390 return TLI.getValueType(
DL, Ty);
1396 unsigned MaxNumLanes) {
1402 assert(ST->getNumContainedTypes() == 2 &&
1403 ST->getContainedType(1)->isIntegerTy(32));
1417 return MVT::amdgpuBufferFatPointer;
1419 DL.getPointerSizeInBits(AS) == 192)
1420 return MVT::amdgpuBufferStridedPointer;
1429 DL.getPointerSizeInBits(AS) == 160) ||
1431 DL.getPointerSizeInBits(AS) == 192))
1438 case Intrinsic::amdgcn_global_load_async_to_lds_b8:
1439 case Intrinsic::amdgcn_cluster_load_async_to_lds_b8:
1440 case Intrinsic::amdgcn_global_store_async_from_lds_b8:
1442 case Intrinsic::amdgcn_global_load_async_to_lds_b32:
1443 case Intrinsic::amdgcn_cluster_load_async_to_lds_b32:
1444 case Intrinsic::amdgcn_global_store_async_from_lds_b32:
1445 case Intrinsic::amdgcn_cooperative_atomic_load_32x4B:
1446 case Intrinsic::amdgcn_cooperative_atomic_store_32x4B:
1447 case Intrinsic::amdgcn_flat_load_monitor_b32:
1448 case Intrinsic::amdgcn_global_load_monitor_b32:
1450 case Intrinsic::amdgcn_global_load_async_to_lds_b64:
1451 case Intrinsic::amdgcn_cluster_load_async_to_lds_b64:
1452 case Intrinsic::amdgcn_global_store_async_from_lds_b64:
1453 case Intrinsic::amdgcn_cooperative_atomic_load_16x8B:
1454 case Intrinsic::amdgcn_cooperative_atomic_store_16x8B:
1455 case Intrinsic::amdgcn_flat_load_monitor_b64:
1456 case Intrinsic::amdgcn_global_load_monitor_b64:
1458 case Intrinsic::amdgcn_global_load_async_to_lds_b128:
1459 case Intrinsic::amdgcn_cluster_load_async_to_lds_b128:
1460 case Intrinsic::amdgcn_global_store_async_from_lds_b128:
1461 case Intrinsic::amdgcn_cooperative_atomic_load_8x16B:
1462 case Intrinsic::amdgcn_cooperative_atomic_store_8x16B:
1463 case Intrinsic::amdgcn_flat_load_monitor_b128:
1464 case Intrinsic::amdgcn_global_load_monitor_b128:
1500 unsigned IntrID)
const {
1502 if (CI.
hasMetadata(LLVMContext::MD_invariant_load))
1518 bool IsSPrefetch = BufInfo && BufInfo->
Kind == AMDGPU::BufferKindPrefetch;
1519 std::optional<unsigned> AuxArg =
1534 if (RsrcIntr->IsImage) {
1549 Info.ptrVal = RsrcArg;
1553 if (RsrcIntr->IsImage) {
1554 unsigned MaxNumLanes = 4;
1569 std::numeric_limits<unsigned>::max());
1579 if (RsrcIntr->IsImage) {
1593 if (BufInfo && BufInfo->
Kind == AMDGPU::BufferKindLoadToLDS) {
1609 CI.
getContext(), Width * 8 * Subtarget->getWavefrontSize());
1619 if (BufInfo && BufInfo->
Kind == AMDGPU::BufferKindAtomicLoad) {
1622 std::numeric_limits<unsigned>::max());
1632 if ((RsrcIntr->IsImage && BaseOpcode->
NoReturn) || IsSPrefetch) {
1634 Info.memVT = MVT::i32;
1647 case Intrinsic::amdgcn_ds_ordered_add:
1648 case Intrinsic::amdgcn_ds_ordered_swap: {
1662 case Intrinsic::amdgcn_ds_add_gs_reg_rtn:
1663 case Intrinsic::amdgcn_ds_sub_gs_reg_rtn: {
1666 Info.ptrVal =
nullptr;
1672 case Intrinsic::amdgcn_ds_append:
1673 case Intrinsic::amdgcn_ds_consume: {
1687 case Intrinsic::amdgcn_ds_atomic_async_barrier_arrive_b64:
1688 case Intrinsic::amdgcn_ds_atomic_barrier_arrive_rtn_b64: {
1689 Info.opc = (IntrID == Intrinsic::amdgcn_ds_atomic_barrier_arrive_rtn_b64)
1694 Info.memVT = MVT::i64;
1702 case Intrinsic::amdgcn_image_bvh_dual_intersect_ray:
1703 case Intrinsic::amdgcn_image_bvh_intersect_ray:
1704 case Intrinsic::amdgcn_image_bvh8_intersect_ray: {
1707 MVT::getVT(IntrID == Intrinsic::amdgcn_image_bvh_intersect_ray
1710 ->getElementType(0));
1719 case Intrinsic::amdgcn_global_atomic_fmin_num:
1720 case Intrinsic::amdgcn_global_atomic_fmax_num:
1721 case Intrinsic::amdgcn_global_atomic_ordered_add_b64:
1722 case Intrinsic::amdgcn_flat_atomic_fmin_num:
1723 case Intrinsic::amdgcn_flat_atomic_fmax_num: {
1734 case Intrinsic::amdgcn_cluster_load_b32:
1735 case Intrinsic::amdgcn_cluster_load_b64:
1736 case Intrinsic::amdgcn_cluster_load_b128:
1737 case Intrinsic::amdgcn_ds_load_tr6_b96:
1738 case Intrinsic::amdgcn_ds_load_tr4_b64:
1739 case Intrinsic::amdgcn_ds_load_tr8_b64:
1740 case Intrinsic::amdgcn_ds_load_tr16_b128:
1741 case Intrinsic::amdgcn_global_load_tr6_b96:
1742 case Intrinsic::amdgcn_global_load_tr4_b64:
1743 case Intrinsic::amdgcn_global_load_tr_b64:
1744 case Intrinsic::amdgcn_global_load_tr_b128:
1745 case Intrinsic::amdgcn_ds_read_tr4_b64:
1746 case Intrinsic::amdgcn_ds_read_tr6_b96:
1747 case Intrinsic::amdgcn_ds_read_tr8_b64:
1748 case Intrinsic::amdgcn_ds_read_tr16_b64: {
1757 case Intrinsic::amdgcn_flat_load_monitor_b32:
1758 case Intrinsic::amdgcn_flat_load_monitor_b64:
1759 case Intrinsic::amdgcn_flat_load_monitor_b128:
1760 case Intrinsic::amdgcn_global_load_monitor_b32:
1761 case Intrinsic::amdgcn_global_load_monitor_b64:
1762 case Intrinsic::amdgcn_global_load_monitor_b128: {
1773 case Intrinsic::amdgcn_cooperative_atomic_load_32x4B:
1774 case Intrinsic::amdgcn_cooperative_atomic_load_16x8B:
1775 case Intrinsic::amdgcn_cooperative_atomic_load_8x16B: {
1786 case Intrinsic::amdgcn_cooperative_atomic_store_32x4B:
1787 case Intrinsic::amdgcn_cooperative_atomic_store_16x8B:
1788 case Intrinsic::amdgcn_cooperative_atomic_store_8x16B: {
1799 case Intrinsic::amdgcn_ds_gws_init:
1800 case Intrinsic::amdgcn_ds_gws_barrier:
1801 case Intrinsic::amdgcn_ds_gws_sema_v:
1802 case Intrinsic::amdgcn_ds_gws_sema_br:
1803 case Intrinsic::amdgcn_ds_gws_sema_p:
1804 case Intrinsic::amdgcn_ds_gws_sema_release_all: {
1814 Info.memVT = MVT::i32;
1816 Info.align =
Align(4);
1818 if (IntrID == Intrinsic::amdgcn_ds_gws_barrier)
1825 case Intrinsic::amdgcn_global_load_async_to_lds_b8:
1826 case Intrinsic::amdgcn_global_load_async_to_lds_b32:
1827 case Intrinsic::amdgcn_global_load_async_to_lds_b64:
1828 case Intrinsic::amdgcn_global_load_async_to_lds_b128:
1829 case Intrinsic::amdgcn_cluster_load_async_to_lds_b8:
1830 case Intrinsic::amdgcn_cluster_load_async_to_lds_b32:
1831 case Intrinsic::amdgcn_cluster_load_async_to_lds_b64:
1832 case Intrinsic::amdgcn_cluster_load_async_to_lds_b128: {
1847 case Intrinsic::amdgcn_global_store_async_from_lds_b8:
1848 case Intrinsic::amdgcn_global_store_async_from_lds_b32:
1849 case Intrinsic::amdgcn_global_store_async_from_lds_b64:
1850 case Intrinsic::amdgcn_global_store_async_from_lds_b128: {
1865 case Intrinsic::amdgcn_av_load_b8:
1866 case Intrinsic::amdgcn_av_load_b16:
1867 case Intrinsic::amdgcn_av_load_b32:
1868 case Intrinsic::amdgcn_av_load_b64:
1869 case Intrinsic::amdgcn_av_load_b128:
1870 case Intrinsic::amdgcn_av_store_b8:
1871 case Intrinsic::amdgcn_av_store_b16:
1872 case Intrinsic::amdgcn_av_store_b32:
1873 case Intrinsic::amdgcn_av_store_b64:
1874 case Intrinsic::amdgcn_av_store_b128: {
1880 Info.align =
Align(Info.memVT.getStoreSize());
1888 unsigned ScopeIdx = CI.
arg_size() - 1;
1892 Info.ssid = Ctx.getOrInsertSyncScopeID(Scope);
1896 case Intrinsic::amdgcn_load_to_lds:
1897 case Intrinsic::amdgcn_load_async_to_lds:
1898 case Intrinsic::amdgcn_global_load_lds:
1899 case Intrinsic::amdgcn_global_load_async_lds: {
1918 Width * 8 * Subtarget->getWavefrontSize());
1924 case Intrinsic::amdgcn_ds_bvh_stack_rtn:
1925 case Intrinsic::amdgcn_ds_bvh_stack_push4_pop1_rtn:
1926 case Intrinsic::amdgcn_ds_bvh_stack_push8_pop1_rtn:
1927 case Intrinsic::amdgcn_ds_bvh_stack_push8_pop2_rtn: {
1937 Info.memVT = MVT::i32;
1939 Info.align =
Align(4);
1945 case Intrinsic::amdgcn_s_prefetch_data:
1946 case Intrinsic::amdgcn_s_prefetch_inst:
1947 case Intrinsic::amdgcn_flat_prefetch:
1948 case Intrinsic::amdgcn_global_prefetch: {
1963 Type *&AccessTy)
const {
1964 Value *Ptr =
nullptr;
1965 switch (
II->getIntrinsicID()) {
1966 case Intrinsic::amdgcn_cluster_load_b128:
1967 case Intrinsic::amdgcn_cluster_load_b64:
1968 case Intrinsic::amdgcn_cluster_load_b32:
1969 case Intrinsic::amdgcn_ds_append:
1970 case Intrinsic::amdgcn_ds_consume:
1971 case Intrinsic::amdgcn_ds_load_tr8_b64:
1972 case Intrinsic::amdgcn_ds_load_tr16_b128:
1973 case Intrinsic::amdgcn_ds_load_tr4_b64:
1974 case Intrinsic::amdgcn_ds_load_tr6_b96:
1975 case Intrinsic::amdgcn_ds_read_tr4_b64:
1976 case Intrinsic::amdgcn_ds_read_tr6_b96:
1977 case Intrinsic::amdgcn_ds_read_tr8_b64:
1978 case Intrinsic::amdgcn_ds_read_tr16_b64:
1979 case Intrinsic::amdgcn_ds_ordered_add:
1980 case Intrinsic::amdgcn_ds_ordered_swap:
1981 case Intrinsic::amdgcn_ds_atomic_async_barrier_arrive_b64:
1982 case Intrinsic::amdgcn_ds_atomic_barrier_arrive_rtn_b64:
1983 case Intrinsic::amdgcn_flat_atomic_fmax_num:
1984 case Intrinsic::amdgcn_flat_atomic_fmin_num:
1985 case Intrinsic::amdgcn_global_atomic_fmax_num:
1986 case Intrinsic::amdgcn_global_atomic_fmin_num:
1987 case Intrinsic::amdgcn_global_atomic_ordered_add_b64:
1988 case Intrinsic::amdgcn_global_load_tr_b64:
1989 case Intrinsic::amdgcn_global_load_tr_b128:
1990 case Intrinsic::amdgcn_global_load_tr4_b64:
1991 case Intrinsic::amdgcn_global_load_tr6_b96:
1992 case Intrinsic::amdgcn_global_store_async_from_lds_b8:
1993 case Intrinsic::amdgcn_global_store_async_from_lds_b32:
1994 case Intrinsic::amdgcn_global_store_async_from_lds_b64:
1995 case Intrinsic::amdgcn_global_store_async_from_lds_b128:
1996 case Intrinsic::amdgcn_av_load_b8:
1997 case Intrinsic::amdgcn_av_load_b16:
1998 case Intrinsic::amdgcn_av_load_b32:
1999 case Intrinsic::amdgcn_av_load_b64:
2000 case Intrinsic::amdgcn_av_load_b128:
2001 case Intrinsic::amdgcn_av_store_b8:
2002 case Intrinsic::amdgcn_av_store_b16:
2003 case Intrinsic::amdgcn_av_store_b32:
2004 case Intrinsic::amdgcn_av_store_b64:
2005 case Intrinsic::amdgcn_av_store_b128:
2006 Ptr =
II->getArgOperand(0);
2008 case Intrinsic::amdgcn_load_to_lds:
2009 case Intrinsic::amdgcn_load_async_to_lds:
2010 case Intrinsic::amdgcn_global_load_lds:
2011 case Intrinsic::amdgcn_global_load_async_lds:
2012 case Intrinsic::amdgcn_global_load_async_to_lds_b8:
2013 case Intrinsic::amdgcn_global_load_async_to_lds_b32:
2014 case Intrinsic::amdgcn_global_load_async_to_lds_b64:
2015 case Intrinsic::amdgcn_global_load_async_to_lds_b128:
2016 case Intrinsic::amdgcn_cluster_load_async_to_lds_b8:
2017 case Intrinsic::amdgcn_cluster_load_async_to_lds_b32:
2018 case Intrinsic::amdgcn_cluster_load_async_to_lds_b64:
2019 case Intrinsic::amdgcn_cluster_load_async_to_lds_b128:
2020 Ptr =
II->getArgOperand(1);
2025 AccessTy =
II->getType();
2031 unsigned AddrSpace)
const {
2032 if (!Subtarget->hasFlatInstOffsets()) {
2039 FlatAddrSpace FlatVariant =
2042 : FlatAddrSpace::FLAT;
2044 return AM.
Scale == 0 &&
2045 (AM.
BaseOffs == 0 || Subtarget->getInstrInfo()->isLegalFLATOffset(
2046 AM.
BaseOffs, AddrSpace, FlatVariant));
2050 if (Subtarget->hasFlatGlobalInsts())
2053 if (!Subtarget->hasAddr64() || Subtarget->useFlatForGlobal()) {
2066 return isLegalMUBUFAddressingMode(AM);
2069bool SITargetLowering::isLegalMUBUFAddressingMode(
const AddrMode &AM)
const {
2080 if (!
TII->isLegalMUBUFImmOffset(AM.BaseOffs))
2092 if (AM.HasBaseReg) {
2124 return isLegalMUBUFAddressingMode(AM);
2126 if (!Subtarget->hasScalarSubwordLoads()) {
2131 if (Ty->isSized() &&
DL.getTypeStoreSize(Ty) < 4)
2179 return Subtarget->hasFlatScratchEnabled()
2181 : isLegalMUBUFAddressingMode(AM);
2228 unsigned Size,
unsigned AddrSpace,
Align Alignment,
2237 if (!Subtarget->hasUnalignedDSAccessEnabled() && Alignment <
Align(4))
2240 Align RequiredAlignment(
2242 if (Subtarget->hasLDSMisalignedBugInWGPMode() &&
Size > 32 &&
2243 Alignment < RequiredAlignment)
2258 if (!Subtarget->hasUsableDSOffset() && Alignment <
Align(8))
2264 RequiredAlignment =
Align(4);
2266 if (Subtarget->hasUnalignedDSAccessEnabled()) {
2282 *IsFast = (Alignment >= RequiredAlignment) ? 64
2283 : (Alignment <
Align(4)) ? 32
2290 if (!Subtarget->hasDS96AndDS128())
2296 if (Subtarget->hasUnalignedDSAccessEnabled()) {
2305 *IsFast = (Alignment >= RequiredAlignment) ? 96
2306 : (Alignment <
Align(4)) ? 32
2313 if (!Subtarget->hasDS96AndDS128() || !Subtarget->useDS128())
2319 RequiredAlignment =
Align(8);
2321 if (Subtarget->hasUnalignedDSAccessEnabled()) {
2330 *IsFast = (Alignment >= RequiredAlignment) ? 128
2331 : (Alignment <
Align(4)) ? 32
2348 *IsFast = (Alignment >= RequiredAlignment) ?
Size : 0;
2350 return Alignment >= RequiredAlignment ||
2351 Subtarget->hasUnalignedDSAccessEnabled();
2359 bool AlignedBy4 = Alignment >=
Align(4);
2360 if (Subtarget->hasUnalignedScratchAccessEnabled()) {
2362 *IsFast = AlignedBy4 ?
Size : 1;
2367 *IsFast = AlignedBy4;
2378 return Alignment >=
Align(4) ||
2379 Subtarget->hasUnalignedBufferAccessEnabled();
2392 if (!Subtarget->hasRelaxedBufferOOBMode() &&
2407 return Size >= 32 && Alignment >=
Align(4);
2412 unsigned *IsFast)
const {
2414 Alignment, Flags, IsFast);
2419 unsigned AddrSpace,
Align Alignment,
2421 unsigned *IsFast)
const {
2427 Alignment, Flags, IsFast);
2438 if (
Op.size() >= 16 &&
2442 if (
Op.size() >= 8 &&
Op.isDstAligned(
Align(4)))
2460 unsigned DestAS)
const {
2463 Subtarget->hasGloballyAddressableScratch()) {
2494 unsigned Index)
const {
2508 unsigned MinAlign = Subtarget->useRealTrue16Insts() ? 16 : 32;
2513 if (Subtarget->has16BitInsts() && VT == MVT::i16) {
2547 if (Subtarget->hasScalarSubwordLoads() &&
N->getOpcode() ==
ISD::LOAD &&
2598 auto [InputPtrReg, RC, ArgTy] =
2608 Chain, SL,
MRI.getLiveInVirtReg(InputPtrReg->getRegister()), PtrVT);
2614 const SDLoc &SL)
const {
2621 const SDLoc &SL)
const {
2624 std::optional<uint32_t> KnownSize =
2626 if (KnownSize.has_value())
2653 Val = getFPExtOrFPRound(DAG, Val, SL, VT);
2668SDValue SITargetLowering::lowerKernargMemParameter(
2673 MachinePointerInfo PtrInfo =
2682 int64_t OffsetDiff =
Offset - AlignDownOffset;
2688 SDValue Ptr = lowerKernArgParameterPtr(DAG, SL, Chain, AlignDownOffset);
2699 ArgVal = convertArgType(DAG, VT, MemVT, SL, ArgVal,
Signed, Arg);
2704 SDValue Ptr = lowerKernArgParameterPtr(DAG, SL, Chain,
Offset);
2718 const SDLoc &SL)
const {
2765 MachinePointerInfo PtrInfo =
2791 SDValue ConvertedVal = convertABITypeToValueType(DAG, ArgValue, VA, SL);
2792 if (ConvertedVal == ArgValue)
2793 return ConvertedVal;
2798SDValue SITargetLowering::lowerWorkGroupId(
2803 if (!Subtarget->hasClusters())
2804 return getPreloadedValue(DAG, MFI, VT, WorkGroupIdPV);
2812 SDValue ClusterIdXYZ = getPreloadedValue(DAG, MFI, VT, WorkGroupIdPV);
2813 SDLoc SL(ClusterIdXYZ);
2814 SDValue ClusterMaxIdXYZ = getPreloadedValue(DAG, MFI, VT, ClusterMaxIdPV);
2817 SDValue ClusterWorkGroupIdXYZ =
2818 getPreloadedValue(DAG, MFI, VT, ClusterWorkGroupIdPV);
2828 return ClusterIdXYZ;
2830 using namespace AMDGPU::Hwreg;
2834 DAG.
getMachineNode(AMDGPU::S_GETREG_B32_const, SL, VT, ClusterIdField);
2845SDValue SITargetLowering::getPreloadedValue(
2848 const ArgDescriptor *
Reg =
nullptr;
2853 const ArgDescriptor WorkGroupIDX =
2861 const ArgDescriptor WorkGroupIDZ =
2863 const ArgDescriptor ClusterWorkGroupIDX =
2865 const ArgDescriptor ClusterWorkGroupIDY =
2867 const ArgDescriptor ClusterWorkGroupIDZ =
2869 const ArgDescriptor ClusterWorkGroupMaxIDX =
2871 const ArgDescriptor ClusterWorkGroupMaxIDY =
2873 const ArgDescriptor ClusterWorkGroupMaxIDZ =
2875 const ArgDescriptor ClusterWorkGroupMaxFlatID =
2878 auto LoadConstant = [&](
unsigned N) {
2882 if (Subtarget->hasArchitectedSGPRs() &&
2890 Reg = &WorkGroupIDX;
2891 RC = &AMDGPU::SReg_32RegClass;
2895 Reg = &WorkGroupIDY;
2896 RC = &AMDGPU::SReg_32RegClass;
2900 Reg = &WorkGroupIDZ;
2901 RC = &AMDGPU::SReg_32RegClass;
2905 if (HasFixedDims && ClusterDims.
getDims()[0] == 1)
2906 return LoadConstant(0);
2907 Reg = &ClusterWorkGroupIDX;
2908 RC = &AMDGPU::SReg_32RegClass;
2912 if (HasFixedDims && ClusterDims.
getDims()[1] == 1)
2913 return LoadConstant(0);
2914 Reg = &ClusterWorkGroupIDY;
2915 RC = &AMDGPU::SReg_32RegClass;
2919 if (HasFixedDims && ClusterDims.
getDims()[2] == 1)
2920 return LoadConstant(0);
2921 Reg = &ClusterWorkGroupIDZ;
2922 RC = &AMDGPU::SReg_32RegClass;
2927 return LoadConstant(ClusterDims.
getDims()[0] - 1);
2928 Reg = &ClusterWorkGroupMaxIDX;
2929 RC = &AMDGPU::SReg_32RegClass;
2934 return LoadConstant(ClusterDims.
getDims()[1] - 1);
2935 Reg = &ClusterWorkGroupMaxIDY;
2936 RC = &AMDGPU::SReg_32RegClass;
2941 return LoadConstant(ClusterDims.
getDims()[2] - 1);
2942 Reg = &ClusterWorkGroupMaxIDZ;
2943 RC = &AMDGPU::SReg_32RegClass;
2947 Reg = &ClusterWorkGroupMaxFlatID;
2948 RC = &AMDGPU::SReg_32RegClass;
2979 for (
unsigned I = 0,
E = Ins.
size(), PSInputNum = 0;
I !=
E; ++
I) {
2983 "vector type argument should have been split");
2988 bool SkipArg = !Arg->
Used && !Info->isPSInputAllocated(PSInputNum);
2996 "unexpected vector split in ps argument type");
3010 Info->markPSInputAllocated(PSInputNum);
3012 Info->markPSInputEnabled(PSInputNum);
3028 if (Info.hasWorkItemIDX()) {
3030 MRI.setType(MF.
addLiveIn(Reg, &AMDGPU::VGPR_32RegClass), I32);
3034 (Subtarget->hasPackedTID() && Info.hasWorkItemIDY()) ? 0x3ff : ~0u;
3038 if (Info.hasWorkItemIDY()) {
3039 assert(Info.hasWorkItemIDX());
3040 if (Subtarget->hasPackedTID()) {
3041 Info.setWorkItemIDY(
3044 unsigned Reg = AMDGPU::VGPR1;
3045 MRI.setType(MF.
addLiveIn(Reg, &AMDGPU::VGPR_32RegClass), I32);
3052 if (Info.hasWorkItemIDZ()) {
3053 assert(Info.hasWorkItemIDX() && Info.hasWorkItemIDY());
3054 if (Subtarget->hasPackedTID()) {
3055 Info.setWorkItemIDZ(
3058 unsigned Reg = AMDGPU::VGPR2;
3059 MRI.setType(MF.
addLiveIn(Reg, &AMDGPU::VGPR_32RegClass), I32);
3069 unsigned NumArgRegs) {
3072 if (RegIdx == ArgSGPRs.
size())
3075 unsigned Reg = ArgSGPRs[RegIdx];
3120 const unsigned Mask = 0x3ff;
3129 auto &
ArgInfo = Info.getArgInfo();
3141 if (Info.hasImplicitArgPtr())
3149 if (Info.hasWorkGroupIDX())
3152 if (Info.hasWorkGroupIDY())
3155 if (Info.hasWorkGroupIDZ())
3158 if (Info.hasLDSKernelId())
3169 Register ImplicitBufferPtrReg = Info.addImplicitBufferPtr(
TRI);
3170 MF.
addLiveIn(ImplicitBufferPtrReg, &AMDGPU::SGPR_64RegClass);
3176 Register PrivateSegmentBufferReg = Info.addPrivateSegmentBuffer(
TRI);
3177 MF.
addLiveIn(PrivateSegmentBufferReg, &AMDGPU::SGPR_128RegClass);
3182 Register DispatchPtrReg = Info.addDispatchPtr(
TRI);
3183 MF.
addLiveIn(DispatchPtrReg, &AMDGPU::SGPR_64RegClass);
3189 MF.
addLiveIn(QueuePtrReg, &AMDGPU::SGPR_64RegClass);
3195 Register InputPtrReg = Info.addKernargSegmentPtr(
TRI);
3204 MF.
addLiveIn(DispatchIDReg, &AMDGPU::SGPR_64RegClass);
3209 Register FlatScratchInitReg = Info.addFlatScratchInit(
TRI);
3210 MF.
addLiveIn(FlatScratchInitReg, &AMDGPU::SGPR_64RegClass);
3215 Register PrivateSegmentSizeReg = Info.addPrivateSegmentSize(
TRI);
3216 MF.
addLiveIn(PrivateSegmentSizeReg, &AMDGPU::SGPR_32RegClass);
3231 unsigned LastExplicitArgOffset = Subtarget->getExplicitKernelArgOffset();
3233 bool InPreloadSequence =
true;
3235 bool AlignedForImplictArgs =
false;
3236 unsigned ImplicitArgOffset = 0;
3237 for (
auto &Arg :
F.args()) {
3238 if (!InPreloadSequence || !Arg.hasInRegAttr())
3241 unsigned ArgIdx = Arg.getArgNo();
3244 if (InIdx < Ins.
size() &&
3245 (!Ins[InIdx].isOrigArg() || Ins[InIdx].getOrigArgIndex() != ArgIdx))
3248 for (; InIdx < Ins.
size() && Ins[InIdx].isOrigArg() &&
3249 Ins[InIdx].getOrigArgIndex() == ArgIdx;
3251 assert(ArgLocs[ArgIdx].isMemLoc());
3252 auto &ArgLoc = ArgLocs[InIdx];
3254 unsigned ArgOffset = ArgLoc.getLocMemOffset();
3256 unsigned NumAllocSGPRs =
3257 alignTo(ArgLoc.getLocVT().getFixedSizeInBits(), 32) / 32;
3260 if (Arg.hasAttribute(
"amdgpu-hidden-argument")) {
3261 if (!AlignedForImplictArgs) {
3263 alignTo(LastExplicitArgOffset,
3264 Subtarget->getAlignmentForImplicitArgPtr()) -
3265 LastExplicitArgOffset;
3266 AlignedForImplictArgs =
true;
3268 ArgOffset += ImplicitArgOffset;
3272 if (ArgLoc.getLocVT().getStoreSize() < 4 && Alignment < 4) {
3273 assert(InIdx >= 1 &&
"No previous SGPR");
3274 Info.getArgInfo().PreloadKernArgs[InIdx].Regs.push_back(
3275 Info.getArgInfo().PreloadKernArgs[InIdx - 1].Regs[0]);
3279 unsigned Padding = ArgOffset - LastExplicitArgOffset;
3280 unsigned PaddingSGPRs =
alignTo(Padding, 4) / 4;
3283 InPreloadSequence =
false;
3289 TRI.getSGPRClassForBitWidth(NumAllocSGPRs * 32);
3291 Info.addPreloadedKernArg(
TRI, RC, NumAllocSGPRs, InIdx, PaddingSGPRs);
3293 if (PreloadRegs->
size() > 1)
3294 RC = &AMDGPU::SGPR_32RegClass;
3295 for (
auto &Reg : *PreloadRegs) {
3301 LastExplicitArgOffset = NumAllocSGPRs * 4 + ArgOffset;
3310 if (Info.hasLDSKernelId()) {
3311 Register Reg = Info.addLDSKernelId();
3312 MF.
addLiveIn(Reg, &AMDGPU::SGPR_32RegClass);
3321 bool IsShader)
const {
3322 bool HasArchitectedSGPRs = Subtarget->hasArchitectedSGPRs();
3323 if (Subtarget->hasUserSGPRInit16BugInWave32() && !IsShader) {
3329 assert(!HasArchitectedSGPRs &&
"Unhandled feature for the subtarget");
3331 unsigned CurrentUserSGPRs = Info.getNumUserSGPRs();
3335 unsigned NumRequiredSystemSGPRs =
3336 Info.hasWorkGroupIDX() + Info.hasWorkGroupIDY() +
3337 Info.hasWorkGroupIDZ() + Info.hasWorkGroupInfo();
3338 for (
unsigned i = NumRequiredSystemSGPRs + CurrentUserSGPRs; i < 16; ++i) {
3339 Register Reg = Info.addReservedUserSGPR();
3340 MF.
addLiveIn(Reg, &AMDGPU::SGPR_32RegClass);
3345 if (!HasArchitectedSGPRs) {
3346 if (Info.hasWorkGroupIDX()) {
3347 Register Reg = Info.addWorkGroupIDX();
3348 MF.
addLiveIn(Reg, &AMDGPU::SGPR_32RegClass);
3352 if (Info.hasWorkGroupIDY()) {
3353 Register Reg = Info.addWorkGroupIDY();
3354 MF.
addLiveIn(Reg, &AMDGPU::SGPR_32RegClass);
3358 if (Info.hasWorkGroupIDZ()) {
3359 Register Reg = Info.addWorkGroupIDZ();
3360 MF.
addLiveIn(Reg, &AMDGPU::SGPR_32RegClass);
3365 if (Info.hasWorkGroupInfo()) {
3366 Register Reg = Info.addWorkGroupInfo();
3367 MF.
addLiveIn(Reg, &AMDGPU::SGPR_32RegClass);
3371 if (Info.hasPrivateSegmentWaveByteOffset()) {
3373 unsigned PrivateSegmentWaveByteOffsetReg;
3376 PrivateSegmentWaveByteOffsetReg =
3377 Info.getPrivateSegmentWaveByteOffsetSystemSGPR();
3381 if (PrivateSegmentWaveByteOffsetReg == AMDGPU::NoRegister) {
3383 Info.setPrivateSegmentWaveByteOffset(PrivateSegmentWaveByteOffsetReg);
3386 PrivateSegmentWaveByteOffsetReg = Info.addPrivateSegmentWaveByteOffset();
3388 MF.
addLiveIn(PrivateSegmentWaveByteOffsetReg, &AMDGPU::SGPR_32RegClass);
3389 CCInfo.
AllocateReg(PrivateSegmentWaveByteOffsetReg);
3392 assert(!Subtarget->hasUserSGPRInit16BugInWave32() || IsShader ||
3393 Info.getNumPreloadedSGPRs() >= 16);
3408 if (HasStackObjects)
3409 Info.setHasNonSpillStackObjects(
true);
3414 HasStackObjects =
true;
3418 bool RequiresStackAccess = HasStackObjects || MFI.
hasCalls();
3420 if (!ST.hasFlatScratchEnabled()) {
3421 if (RequiresStackAccess && ST.isAmdHsaOrMesa(MF.
getFunction())) {
3428 Info.setScratchRSrcReg(PrivateSegmentBufferReg);
3430 unsigned ReservedBufferReg =
TRI.reservedPrivateSegmentBufferReg(MF);
3440 Info.setScratchRSrcReg(ReservedBufferReg);
3459 if (!
MRI.isLiveIn(AMDGPU::SGPR32)) {
3460 Info.setStackPtrOffsetReg(AMDGPU::SGPR32);
3467 for (
unsigned Reg : AMDGPU::SGPR_32RegClass) {
3468 if (!
MRI.isLiveIn(
Reg)) {
3469 Info.setStackPtrOffsetReg(
Reg);
3474 if (Info.getStackPtrOffsetReg() == AMDGPU::SP_REG)
3481 if (ST.getFrameLowering()->hasFP(MF)) {
3482 Info.setFrameOffsetReg(AMDGPU::SGPR33);
3498 const MCPhysReg *IStart =
TRI->getCalleeSavedRegsViaCopy(Entry->getParent());
3507 if (AMDGPU::SReg_64RegClass.
contains(*
I))
3508 RC = &AMDGPU::SGPR_64RegClass;
3509 else if (AMDGPU::SReg_32RegClass.
contains(*
I))
3510 RC = &AMDGPU::SGPR_32RegClass;
3516 Entry->addLiveIn(*
I);
3521 for (
auto *Exit : Exits)
3523 TII->get(TargetOpcode::COPY), *
I)
3538 bool IsError =
false;
3542 Fn,
"unsupported non-compute shaders with HSA",
DL.getDebugLoc()));
3560 !Info->hasLDSKernelId() && !Info->hasWorkItemIDX() &&
3561 !Info->hasWorkItemIDY() && !Info->hasWorkItemIDZ());
3563 if (!Subtarget->hasFlatScratchEnabled())
3568 !Subtarget->hasArchitectedSGPRs())
3569 assert(!Info->hasWorkGroupIDX() && !Info->hasWorkGroupIDY() &&
3570 !Info->hasWorkGroupIDZ());
3573 bool IsWholeWaveFunc = Info->isWholeWaveFunction();
3591 if ((Info->getPSInputAddr() & 0x7F) == 0 ||
3592 ((Info->getPSInputAddr() & 0xF) == 0 && Info->isPSInputAllocated(11))) {
3595 Info->markPSInputAllocated(0);
3596 Info->markPSInputEnabled(0);
3598 if (Subtarget->isAmdPalOS()) {
3607 unsigned PsInputBits = Info->getPSInputAddr() & Info->getPSInputEnable();
3608 if ((PsInputBits & 0x7F) == 0 ||
3609 ((PsInputBits & 0xF) == 0 && (PsInputBits >> 11 & 1)))
3612 }
else if (IsKernel) {
3613 assert(Info->hasWorkGroupIDX() && Info->hasWorkItemIDX());
3625 if (IsKernel && Subtarget->hasKernargPreload())
3629 }
else if (!IsGraphics) {
3634 if (!Subtarget->hasFlatScratchEnabled())
3646 Info->setNumWaveDispatchSGPRs(
3648 Info->setNumWaveDispatchVGPRs(
3650 }
else if (Info->getNumKernargPreloadedSGPRs()) {
3651 Info->setNumWaveDispatchSGPRs(Info->getNumUserSGPRs());
3656 if (IsWholeWaveFunc) {
3658 {MVT::i1, MVT::Other}, Chain);
3670 for (
unsigned i = IsWholeWaveFunc ? 1 : 0, e = Ins.
size(), ArgIdx = 0; i != e;
3681 if (IsEntryFunc && VA.
isMemLoc()) {
3705 if (Arg.
isOrigArg() && Info->getArgInfo().PreloadKernArgs.count(i)) {
3709 int64_t OffsetDiff =
Offset - AlignDownOffset;
3716 Info->getArgInfo().PreloadKernArgs.find(i)->getSecond().Regs[0];
3727 NewArg = convertArgType(DAG, VT, MemVT,
DL, ArgVal,
3728 Ins[i].Flags.isSExt(), &Ins[i]);
3736 Info->getArgInfo().PreloadKernArgs.find(i)->getSecond().Regs;
3739 if (PreloadRegs.
size() == 1) {
3740 Register VReg =
MRI.getLiveInVirtReg(PreloadRegs[0]);
3745 TRI->getRegSizeInBits(*RC)));
3753 for (
auto Reg : PreloadRegs) {
3760 PreloadRegs.size()),
3777 NewArg = convertArgType(DAG, VT, MemVT,
DL, NewArg,
3778 Ins[i].Flags.isSExt(), &Ins[i]);
3790 "hidden argument in kernel signature was not preloaded",
3796 lowerKernargMemParameter(DAG, VT, MemVT,
DL, Chain,
Offset,
3797 Alignment, Ins[i].Flags.isSExt(), &Ins[i]);
3817 if (!IsEntryFunc && VA.
isMemLoc()) {
3818 SDValue Val = lowerStackParameter(DAG, VA,
DL, Chain, Arg);
3829 if (AMDGPU::VGPR_32RegClass.
contains(Reg))
3830 RC = &AMDGPU::VGPR_32RegClass;
3831 else if (AMDGPU::SGPR_32RegClass.
contains(Reg))
3832 RC = &AMDGPU::SGPR_32RegClass;
3838 if (Arg.
Flags.
isInReg() && RC == &AMDGPU::VGPR_32RegClass) {
3844 ReadFirstLane, Val);
3853 Val = convertABITypeToValueType(DAG, Val, VA,
DL);
3862 Info->setBytesInStackArgArea(StackArgSize);
3864 return Chains.
empty() ? Chain
3873 const Type *RetTy)
const {
3881 CCState CCInfo(CallConv, IsVarArg, MF, RVLocs, Context);
3886 unsigned MaxNumVGPRs = Subtarget->getMaxNumVGPRs(MF);
3887 unsigned TotalNumVGPRs = Subtarget->getAddressableNumArchVGPRs();
3888 for (
unsigned i = MaxNumVGPRs; i < TotalNumVGPRs; ++i)
3889 if (CCInfo.
isAllocated(AMDGPU::VGPR_32RegClass.getRegister(i)))
3912 Info->setIfReturnsVoid(Outs.
empty());
3913 bool IsWaveEnd = Info->returnsVoid() && IsShader;
3932 for (
unsigned I = 0, RealRVLocIdx = 0, E = RVLocs.
size();
I != E;
3933 ++
I, ++RealRVLocIdx) {
3937 SDValue Arg = OutVals[RealRVLocIdx];
3960 ReadFirstLane, Arg);
3967 if (!Info->isEntryFunction()) {
3973 if (AMDGPU::SReg_64RegClass.
contains(*
I))
3975 else if (AMDGPU::SReg_32RegClass.
contains(*
I))
3988 unsigned Opc = AMDGPUISD::ENDPGM;
3990 Opc = Info->isWholeWaveFunction() ? AMDGPUISD::WHOLE_WAVE_RETURN
3991 : IsShader ? AMDGPUISD::RETURN_TO_EPILOG
3992 : AMDGPUISD::RET_GLUE;
4097 const auto [OutgoingArg, ArgRC, ArgTy] =
4102 const auto [IncomingArg, IncomingArgRC, Ty] =
4104 assert(IncomingArgRC == ArgRC);
4107 EVT ArgVT =
TRI->getSpillSize(*ArgRC) == 8 ? MVT::i64 : MVT::i32;
4115 InputReg = getImplicitArgPtr(DAG,
DL);
4117 std::optional<uint32_t> Id =
4119 if (Id.has_value()) {
4130 if (OutgoingArg->isRegister()) {
4131 RegsToPass.emplace_back(OutgoingArg->getRegister(), InputReg);
4132 if (!CCInfo.
AllocateReg(OutgoingArg->getRegister()))
4135 unsigned SpecialArgOffset =
4146 auto [OutgoingArg, ArgRC, Ty] =
4149 std::tie(OutgoingArg, ArgRC, Ty) =
4152 std::tie(OutgoingArg, ArgRC, Ty) =
4167 const bool NeedWorkItemIDX = !CLI.
CB->
hasFnAttr(
"amdgpu-no-workitem-id-x");
4168 const bool NeedWorkItemIDY = !CLI.
CB->
hasFnAttr(
"amdgpu-no-workitem-id-y");
4169 const bool NeedWorkItemIDZ = !CLI.
CB->
hasFnAttr(
"amdgpu-no-workitem-id-z");
4174 if (Subtarget->getMaxWorkitemID(
F, 0) != 0) {
4182 NeedWorkItemIDY && Subtarget->getMaxWorkitemID(
F, 1) != 0) {
4192 NeedWorkItemIDZ && Subtarget->getMaxWorkitemID(
F, 2) != 0) {
4201 if (!InputReg && (NeedWorkItemIDX || NeedWorkItemIDY || NeedWorkItemIDZ)) {
4202 if (!IncomingArgX && !IncomingArgY && !IncomingArgZ) {
4213 : IncomingArgY ? *IncomingArgY
4220 if (OutgoingArg->isRegister()) {
4222 RegsToPass.emplace_back(OutgoingArg->getRegister(), InputReg);
4248 if (Callee->isDivergent())
4255 const uint32_t *CallerPreserved =
TRI->getCallPreservedMask(MF, CallerCC);
4259 if (!CallerPreserved)
4262 bool CCMatch = CallerCC == CalleeCC;
4275 if (Arg.hasByValAttr())
4289 const uint32_t *CalleePreserved =
TRI->getCallPreservedMask(MF, CalleeCC);
4290 if (!
TRI->regmaskSubsetEqual(CallerPreserved, CalleePreserved))
4299 CCState CCInfo(CalleeCC, IsVarArg, MF, ArgLocs, Ctx);
4312 for (
const auto &[CCVA, ArgVal] :
zip_equal(ArgLocs, OutVals)) {
4314 if (!CCVA.isRegLoc())
4319 if (ArgVal->
isDivergent() &&
TRI->isSGPRPhysReg(CCVA.getLocReg())) {
4321 dbgs() <<
"Cannot tail call due to divergent outgoing argument in "
4345enum ChainCallArgIdx {
4367 bool UsesDynamicVGPRs =
false;
4368 if (IsChainCallConv) {
4373 auto RequestedExecIt =
4375 return Arg.OrigArgIndex == 2;
4377 assert(RequestedExecIt != CLI.
Outs.end() &&
"No node for EXEC");
4379 size_t SpecialArgsBeginIdx = RequestedExecIt - CLI.
Outs.begin();
4382 CLI.
Outs.erase(RequestedExecIt, CLI.
Outs.end());
4385 "Haven't popped all the special args");
4388 CLI.
Args[ChainCallArgIdx::Exec];
4389 if (!RequestedExecArg.
Ty->
isIntegerTy(Subtarget->getWavefrontSize()))
4397 ArgNode->getAPIntValue(),
DL, ArgNode->getValueType(0)));
4399 ChainCallSpecialArgs.
push_back(Arg.Node);
4402 PushNodeOrTargetConstant(RequestedExecArg);
4408 if (FlagsValue.
isZero()) {
4409 if (CLI.
Args.size() > ChainCallArgIdx::Flags + 1)
4411 "no additional args allowed if flags == 0");
4413 if (CLI.
Args.size() != ChainCallArgIdx::FallbackCallee + 1) {
4417 if (!Subtarget->isWave32()) {
4419 CLI, InVals,
"dynamic VGPR mode is only supported for wave32");
4422 UsesDynamicVGPRs =
true;
4423 std::for_each(CLI.
Args.begin() + ChainCallArgIdx::NumVGPRs,
4424 CLI.
Args.end(), PushNodeOrTargetConstant);
4433 bool IsSibCall =
false;
4447 "unsupported call to variadic function ");
4455 "unsupported required tail call to function ");
4460 Outs, OutVals, Ins, DAG);
4464 "site marked musttail or on llvm.amdgcn.cs.chain");
4471 if (!TailCallOpt && IsTailCall)
4495 if (!Subtarget->hasFlatScratchEnabled()) {
4518 auto *
TRI = Subtarget->getRegisterInfo();
4525 if (!IsSibCall || IsChainCallConv) {
4526 if (!Subtarget->hasFlatScratchEnabled()) {
4532 RegsToPass.emplace_back(
4540 const unsigned NumSpecialInputs = RegsToPass.size();
4542 MVT PtrVT = MVT::i32;
4545 for (
unsigned i = 0, e = ArgLocs.
size(); i != e; ++i) {
4574 RegsToPass.push_back(std::pair(VA.
getLocReg(), Arg));
4582 int32_t
Offset = LocMemOffset;
4589 unsigned OpSize = Flags.isByVal() ? Flags.getByValSize()
4595 ? Flags.getNonZeroByValAlign()
4622 if (Outs[i].Flags.isByVal()) {
4624 DAG.
getConstant(Outs[i].Flags.getByValSize(),
DL, MVT::i32);
4627 Outs[i].Flags.getNonZeroByValAlign(),
4628 Outs[i].Flags.getNonZeroByValAlign(),
4630 nullptr, std::nullopt, DstInfo,
4636 DAG.
getStore(Chain,
DL, Arg, DstAddr, DstInfo, Alignment);
4642 if (!MemOpChains.
empty())
4658 unsigned ArgIdx = 0;
4659 for (
auto [Reg, Val] : RegsToPass) {
4660 if (ArgIdx++ >= NumSpecialInputs &&
4661 (IsChainCallConv || !Val->
isDivergent()) &&
TRI->isSGPRPhysReg(Reg)) {
4687 if (IsTailCall && !IsSibCall) {
4692 std::vector<SDValue>
Ops({Chain});
4698 Ops.push_back(Callee);
4715 Ops.push_back(Callee);
4726 if (IsChainCallConv)
4731 for (
auto &[Reg, Val] : RegsToPass)
4735 const uint32_t *Mask =
TRI->getCallPreservedMask(MF, CallConv);
4736 assert(Mask &&
"Missing call preserved mask for calling convention");
4746 MVT::Glue, GlueOps),
4751 Ops.push_back(InGlue);
4757 unsigned OPC = AMDGPUISD::TC_RETURN;
4760 OPC = AMDGPUISD::TC_RETURN_GFX;
4764 OPC = UsesDynamicVGPRs ? AMDGPUISD::TC_RETURN_CHAIN_DVGPR
4765 : AMDGPUISD::TC_RETURN_CHAIN;
4771 if (Info->isWholeWaveFunction())
4772 OPC = AMDGPUISD::TC_RETURN_GFX_WholeWave;
4782 Chain =
Call.getValue(0);
4783 InGlue =
Call.getValue(1);
4785 uint64_t CalleePopBytes = NumBytes;
4806 EVT VT =
Op.getValueType();
4820 "Stack grows upwards for AMDGPU");
4822 Chain = BaseAddr.getValue(1);
4824 const bool HasFlatScratch = Subtarget->hasFlatScratchEnabled();
4825 const unsigned WavefrontSizeLog2 = Subtarget->getWavefrontSizeLog2();
4828 if (Alignment > StackAlign) {
4829 uint64_t ScaledAlignment = Alignment.value()
4830 << (HasFlatScratch ? 0 : WavefrontSizeLog2);
4831 uint64_t StackAlignMask = ScaledAlignment - 1;
4838 assert(
Size.getValueType() == MVT::i32 &&
"Size must be 32-bit");
4847 DAG.
getConstant(WavefrontSizeLog2, dl, MVT::i32));
4858 if (!HasFlatScratch) {
4861 DAG.
getConstant(WavefrontSizeLog2, dl, MVT::i32));
4878 if (
Op.getValueType() != MVT::i32)
4897 assert(
Op.getValueType() == MVT::i32);
4906 Op.getOperand(0), IntrinID, GetRoundBothImm);
4940 SDValue RoundModeTimesNumBits =
4960 TableEntry, EnumOffset);
4976 static_cast<uint32_t>(ConstMode->getZExtValue()),
4988 if (UseReducedTable) {
4994 SDValue RoundModeTimesNumBits =
5014 SDValue RoundModeTimesNumBits =
5023 NewMode = TruncTable;
5032 ReadFirstLaneID, NewMode);
5045 IntrinID, RoundBothImm, NewMode);
5053 return Op.getOperand(0);
5058 if (!Subtarget->hasSafeSmemPrefetch() && !
Op.getConstantOperandVal(4))
5076 SDValue Src =
Op.getOperand(IsStrict ? 1 : 0);
5077 EVT SrcVT = Src.getValueType();
5086 EVT DstVT =
Op.getValueType();
5095 if (
Op.getValueType() != MVT::i64)
5109 Op.getOperand(0), IntrinID, ModeHwRegImm);
5111 Op.getOperand(0), IntrinID, TrapHwRegImm);
5125 if (
Op.getOperand(1).getValueType() != MVT::i64)
5137 ReadFirstLaneID, NewModeReg);
5139 ReadFirstLaneID, NewTrapReg);
5141 unsigned ModeHwReg =
5144 unsigned TrapHwReg =
5152 IntrinID, ModeHwRegImm, NewModeReg);
5155 IntrinID, TrapHwRegImm, NewTrapReg);
5163 .
Case(
"m0", AMDGPU::M0)
5164 .
Case(
"exec", AMDGPU::EXEC)
5165 .
Case(
"exec_lo", AMDGPU::EXEC_LO)
5166 .
Case(
"exec_hi", AMDGPU::EXEC_HI)
5167 .
Case(
"flat_scratch", AMDGPU::FLAT_SCR)
5168 .
Case(
"flat_scratch_lo", AMDGPU::FLAT_SCR_LO)
5169 .
Case(
"flat_scratch_hi", AMDGPU::FLAT_SCR_HI)
5170 .
Case(
"src_flat_scratch_base", AMDGPU::SRC_FLAT_SCRATCH_BASE)
5171 .
Case(
"src_flat_scratch_base_lo", AMDGPU::SRC_FLAT_SCRATCH_BASE_LO)
5172 .
Case(
"src_flat_scratch_base_hi", AMDGPU::SRC_FLAT_SCRATCH_BASE_HI)
5178 if (!Subtarget->hasFlatScrRegister() &&
5179 TRI->regsOverlap(Reg, AMDGPU::FLAT_SCR))
5182 if (!Subtarget->hasGloballyAddressableScratch() &&
5183 TRI->regsOverlap(Reg, AMDGPU::SRC_FLAT_SCRATCH_BASE))
5188 case AMDGPU::EXEC_LO:
5189 case AMDGPU::EXEC_HI:
5190 case AMDGPU::FLAT_SCR_LO:
5191 case AMDGPU::FLAT_SCR_HI:
5192 case AMDGPU::SRC_FLAT_SCRATCH_BASE_LO:
5193 case AMDGPU::SRC_FLAT_SCRATCH_BASE_HI:
5198 case AMDGPU::FLAT_SCR:
5199 case AMDGPU::SRC_FLAT_SCRATCH_BASE:
5218 MI.setDesc(
TII->getKillTerminatorFromPseudo(
MI.getOpcode()));
5227static std::pair<MachineBasicBlock *, MachineBasicBlock *>
5249 auto Next = std::next(
I);
5260 MBB.addSuccessor(LoopBB);
5262 return std::pair(LoopBB, RemainderBB);
5269 auto I =
MI.getIterator();
5270 auto E = std::next(
I);
5292 Src->setIsKill(
false);
5302 BuildMI(*LoopBB, LoopBB->begin(),
DL,
TII->get(AMDGPU::S_SETREG_IMM32_B32))
5308 Register Reg =
MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
5311 BuildMI(*LoopBB,
I,
DL,
TII->get(AMDGPU::S_GETREG_B32), Reg)
5335 unsigned InitReg,
unsigned ResultReg,
unsigned PhiReg,
5336 unsigned InitSaveExecReg,
int Offset,
bool UseGPRIdxMode,
5346 Register PhiExec =
MRI.createVirtualRegister(BoolRC);
5347 Register NewExec =
MRI.createVirtualRegister(BoolRC);
5349 MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
5350 Register CondReg =
MRI.createVirtualRegister(BoolRC);
5358 BuildMI(LoopBB,
I,
DL,
TII->get(TargetOpcode::PHI), PhiExec)
5365 BuildMI(LoopBB,
I,
DL,
TII->get(AMDGPU::V_READFIRSTLANE_B32), CurrentIdxReg)
5369 BuildMI(LoopBB,
I,
DL,
TII->get(AMDGPU::V_CMP_EQ_U32_e64), CondReg)
5371 .
addReg(Idx.getReg(), {}, Idx.getSubReg());
5378 MRI.setSimpleHint(NewExec, CondReg);
5380 if (UseGPRIdxMode) {
5382 SGPRIdxReg = CurrentIdxReg;
5384 SGPRIdxReg =
MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
5385 BuildMI(LoopBB,
I,
DL,
TII->get(AMDGPU::S_ADD_I32), SGPRIdxReg)
5396 BuildMI(LoopBB,
I,
DL,
TII->get(AMDGPU::S_ADD_I32), AMDGPU::M0)
5429 unsigned InitResultReg,
unsigned PhiReg,
int Offset,
5430 bool UseGPRIdxMode,
Register &SGPRIdxReg) {
5438 const auto *BoolXExecRC =
TRI->getWaveMaskRegClass();
5440 Register SaveExec =
MRI.createVirtualRegister(BoolXExecRC);
5441 Register TmpExec =
MRI.createVirtualRegister(BoolXExecRC);
5457 InitResultReg, DstReg, PhiReg, TmpExec,
5458 Offset, UseGPRIdxMode, SGPRIdxReg);
5464 LoopBB->removeSuccessor(RemainderBB);
5466 LoopBB->addSuccessor(LandingPad);
5477static std::pair<unsigned, int>
5481 int NumElts =
TRI.getRegSizeInBits(*SuperRC) / 32;
5486 return std::pair(AMDGPU::sub0,
Offset);
5500 assert(Idx->getReg() != AMDGPU::NoRegister);
5525 return Idx->getReg();
5527 Register Tmp =
MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
5545 Register SrcReg =
TII->getNamedOperand(
MI, AMDGPU::OpName::src)->getReg();
5546 int Offset =
TII->getNamedOperand(
MI, AMDGPU::OpName::offset)->getImm();
5552 std::tie(SubReg,
Offset) =
5555 const bool UseGPRIdxMode = ST.useVGPRIndexMode();
5558 if (
TII->getRegisterInfo().isSGPRClass(IdxRC)) {
5562 if (UseGPRIdxMode) {
5569 TII->getIndirectGPRIDXPseudo(
TRI.getRegSizeInBits(*VecRC),
true);
5578 .
addReg(SrcReg, {}, SubReg)
5582 MI.eraseFromParent();
5591 Register PhiReg =
MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
5592 Register InitReg =
MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
5598 UseGPRIdxMode, SGPRIdxReg);
5602 if (UseGPRIdxMode) {
5604 TII->getIndirectGPRIDXPseudo(
TRI.getRegSizeInBits(*VecRC),
true);
5606 BuildMI(*LoopBB, InsPt,
DL, GPRIDXDesc, Dst)
5611 BuildMI(*LoopBB, InsPt,
DL,
TII->get(AMDGPU::V_MOVRELS_B32_e32), Dst)
5612 .
addReg(SrcReg, {}, SubReg)
5616 MI.eraseFromParent();
5633 int Offset =
TII->getNamedOperand(
MI, AMDGPU::OpName::offset)->getImm();
5641 std::tie(SubReg,
Offset) =
5643 const bool UseGPRIdxMode = ST.useVGPRIndexMode();
5645 if (Idx->getReg() == AMDGPU::NoRegister) {
5656 MI.eraseFromParent();
5661 if (
TII->getRegisterInfo().isSGPRClass(IdxRC)) {
5665 if (UseGPRIdxMode) {
5669 TII->getIndirectGPRIDXPseudo(
TRI.getRegSizeInBits(*VecRC),
false);
5678 const MCInstrDesc &MovRelDesc =
TII->getIndirectRegWriteMovRelPseudo(
5679 TRI.getRegSizeInBits(*VecRC), 32,
false);
5685 MI.eraseFromParent();
5692 Register PhiReg =
MRI.createVirtualRegister(VecRC);
5696 UseGPRIdxMode, SGPRIdxReg);
5699 if (UseGPRIdxMode) {
5701 TII->getIndirectGPRIDXPseudo(
TRI.getRegSizeInBits(*VecRC),
false);
5703 BuildMI(*LoopBB, InsPt,
DL, GPRIDXDesc, Dst)
5709 const MCInstrDesc &MovRelDesc =
TII->getIndirectRegWriteMovRelPseudo(
5710 TRI.getRegSizeInBits(*VecRC), 32,
false);
5711 BuildMI(*LoopBB, InsPt,
DL, MovRelDesc, Dst)
5717 MI.eraseFromParent();
5733 bool IsAdd = (
MI.getOpcode() == AMDGPU::S_ADD_U64_PSEUDO);
5734 if (ST.hasScalarAddSub64()) {
5736 unsigned Opc = IsAdd ? AMDGPU::S_ADD_U64 : AMDGPU::S_SUB_U64;
5746 Register DestSub0 =
MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
5747 Register DestSub1 =
MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
5750 MI,
MRI, Src0, BoolRC, AMDGPU::sub0, &AMDGPU::SReg_32RegClass);
5752 MI,
MRI, Src0, BoolRC, AMDGPU::sub1, &AMDGPU::SReg_32RegClass);
5755 MI,
MRI, Src1, BoolRC, AMDGPU::sub0, &AMDGPU::SReg_32RegClass);
5757 MI,
MRI, Src1, BoolRC, AMDGPU::sub1, &AMDGPU::SReg_32RegClass);
5762 unsigned LoOpc = IsAdd ? AMDGPU::S_ADD_U32 : AMDGPU::S_SUB_U32;
5763 unsigned HiOpc = IsAdd ? AMDGPU::S_ADDC_U32 : AMDGPU::S_SUBB_U32;
5769 Hi.setOperandDead(3);
5776 MI.eraseFromParent();
5790 Register SrcCond =
MI.getOperand(3).getReg();
5792 Register DstLo =
MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
5793 Register DstHi =
MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
5795 Register SrcCondCopy =
MRI.createVirtualRegister(CondRC);
5798 AMDGPU::getNamedOperandIdx(
MI.getOpcode(), AMDGPU::OpName::src0);
5800 AMDGPU::getNamedOperandIdx(
MI.getOpcode(), AMDGPU::OpName::src1);
5802 TRI->getAllocatableClass(
TII->getRegClass(
MI.getDesc(), Src0Idx));
5804 TRI->getAllocatableClass(
TII->getRegClass(
MI.getDesc(), Src1Idx));
5807 TRI->getSubRegisterClass(Src0RC, AMDGPU::sub0);
5809 TRI->getSubRegisterClass(Src1RC, AMDGPU::sub1);
5812 MI,
MRI, Src0, Src0RC, AMDGPU::sub0, Src0SubRC);
5814 MI,
MRI, Src1, Src1RC, AMDGPU::sub0, Src1SubRC);
5817 MI,
MRI, Src0, Src0RC, AMDGPU::sub1, Src0SubRC);
5819 MI,
MRI, Src1, Src1RC, AMDGPU::sub1, Src1SubRC);
5841 MI.eraseFromParent();
5846 case AMDGPU::S_MIN_U32:
5847 return std::numeric_limits<uint32_t>::max();
5848 case AMDGPU::S_MIN_I32:
5849 return std::numeric_limits<int32_t>::max();
5850 case AMDGPU::S_MAX_U32:
5851 return std::numeric_limits<uint32_t>::min();
5852 case AMDGPU::S_MAX_I32:
5853 return std::numeric_limits<int32_t>::min();
5854 case AMDGPU::V_ADD_F32_e64:
5856 case AMDGPU::V_SUB_F32_e64:
5858 case AMDGPU::S_ADD_I32:
5859 case AMDGPU::S_SUB_I32:
5860 case AMDGPU::S_OR_B32:
5861 case AMDGPU::S_XOR_B32:
5862 return std::numeric_limits<uint32_t>::min();
5863 case AMDGPU::S_AND_B32:
5864 return std::numeric_limits<uint32_t>::max();
5865 case AMDGPU::V_MIN_F32_e64:
5866 case AMDGPU::V_MAX_F32_e64:
5868 case AMDGPU::V_CMP_LT_U64_e64:
5869 return std::numeric_limits<uint64_t>::max();
5870 case AMDGPU::V_CMP_LT_I64_e64:
5871 return std::numeric_limits<int64_t>::max();
5872 case AMDGPU::V_CMP_GT_U64_e64:
5873 return std::numeric_limits<uint64_t>::min();
5874 case AMDGPU::V_CMP_GT_I64_e64:
5875 return std::numeric_limits<int64_t>::min();
5876 case AMDGPU::V_MIN_F64_e64:
5877 case AMDGPU::V_MAX_F64_e64:
5878 case AMDGPU::V_MIN_NUM_F64_e64:
5879 case AMDGPU::V_MAX_NUM_F64_e64:
5880 return 0x7FF8000000000000;
5881 case AMDGPU::S_ADD_U64_PSEUDO:
5882 case AMDGPU::S_SUB_U64_PSEUDO:
5883 case AMDGPU::S_OR_B64:
5884 case AMDGPU::S_XOR_B64:
5885 return std::numeric_limits<uint64_t>::min();
5886 case AMDGPU::S_AND_B64:
5887 return std::numeric_limits<uint64_t>::max();
5888 case AMDGPU::V_ADD_F64_e64:
5889 case AMDGPU::V_ADD_F64_pseudo_e64:
5890 return 0x8000000000000000;
5897 return Opc == AMDGPU::S_MIN_U32 ||
Opc == AMDGPU::S_MIN_I32 ||
5898 Opc == AMDGPU::S_MAX_U32 ||
Opc == AMDGPU::S_MAX_I32 ||
5899 Opc == AMDGPU::S_ADD_I32 ||
Opc == AMDGPU::S_SUB_I32 ||
5900 Opc == AMDGPU::S_AND_B32 ||
Opc == AMDGPU::S_OR_B32 ||
5901 Opc == AMDGPU::S_XOR_B32 ||
Opc == AMDGPU::V_MIN_F32_e64 ||
5902 Opc == AMDGPU::V_MAX_F32_e64 ||
Opc == AMDGPU::V_ADD_F32_e64 ||
5903 Opc == AMDGPU::V_SUB_F32_e64;
5907 return Opc == AMDGPU::V_MIN_F32_e64 ||
Opc == AMDGPU::V_MAX_F32_e64 ||
5908 Opc == AMDGPU::V_ADD_F32_e64 ||
Opc == AMDGPU::V_SUB_F32_e64 ||
5909 Opc == AMDGPU::V_MIN_F64_e64 ||
Opc == AMDGPU::V_MAX_F64_e64 ||
5910 Opc == AMDGPU::V_MIN_NUM_F64_e64 ||
Opc == AMDGPU::V_MAX_NUM_F64_e64 ||
5911 Opc == AMDGPU::V_ADD_F64_e64 ||
Opc == AMDGPU::V_ADD_F64_pseudo_e64;
5914static std::tuple<unsigned, unsigned>
5918 case AMDGPU::S_MIN_U32:
5919 DPPOpc = AMDGPU::V_MIN_U32_dpp;
5921 case AMDGPU::S_MIN_I32:
5922 DPPOpc = AMDGPU::V_MIN_I32_dpp;
5924 case AMDGPU::S_MAX_U32:
5925 DPPOpc = AMDGPU::V_MAX_U32_dpp;
5927 case AMDGPU::S_MAX_I32:
5928 DPPOpc = AMDGPU::V_MAX_I32_dpp;
5930 case AMDGPU::S_ADD_I32:
5931 case AMDGPU::S_SUB_I32:
5932 DPPOpc = ST.hasAddNoCarryInsts() ? AMDGPU::V_ADD_U32_dpp
5933 : AMDGPU::V_ADD_CO_U32_dpp;
5935 case AMDGPU::S_AND_B32:
5936 DPPOpc = AMDGPU::V_AND_B32_dpp;
5938 case AMDGPU::S_OR_B32:
5939 DPPOpc = AMDGPU::V_OR_B32_dpp;
5941 case AMDGPU::S_XOR_B32:
5942 DPPOpc = AMDGPU::V_XOR_B32_dpp;
5944 case AMDGPU::V_ADD_F32_e64:
5945 case AMDGPU::V_SUB_F32_e64:
5946 DPPOpc = AMDGPU::V_ADD_F32_dpp;
5948 case AMDGPU::V_MIN_F32_e64:
5949 DPPOpc = AMDGPU::V_MIN_F32_dpp;
5951 case AMDGPU::V_MAX_F32_e64:
5952 DPPOpc = AMDGPU::V_MAX_F32_dpp;
5954 case AMDGPU::V_CMP_LT_U64_e64:
5955 case AMDGPU::V_CMP_LT_I64_e64:
5956 case AMDGPU::V_CMP_GT_U64_e64:
5957 case AMDGPU::V_CMP_GT_I64_e64:
5958 case AMDGPU::S_ADD_U64_PSEUDO:
5959 case AMDGPU::S_SUB_U64_PSEUDO:
5960 case AMDGPU::S_AND_B64:
5961 case AMDGPU::S_OR_B64:
5962 case AMDGPU::S_XOR_B64:
5963 case AMDGPU::V_MIN_NUM_F64_e64:
5964 case AMDGPU::V_MIN_F64_e64:
5965 case AMDGPU::V_MAX_NUM_F64_e64:
5966 case AMDGPU::V_MAX_F64_e64:
5967 case AMDGPU::V_ADD_F64_pseudo_e64:
5968 case AMDGPU::V_ADD_F64_e64:
5969 DPPOpc = AMDGPU::V_MOV_B64_DPP_PSEUDO;
5974 unsigned ClampOpc =
Opc;
5975 if (
Opc == AMDGPU::V_SUB_F32_e64)
5976 ClampOpc = AMDGPU::V_ADD_F32_e64;
5977 if (!ST.getInstrInfo()->isVALU(
Opc,
false)) {
5978 if (
Opc == AMDGPU::S_SUB_I32)
5979 ClampOpc = AMDGPU::S_ADD_I32;
5980 if (
Opc == AMDGPU::S_ADD_U64_PSEUDO ||
Opc == AMDGPU::S_SUB_U64_PSEUDO)
5981 ClampOpc = AMDGPU::V_ADD_CO_U32_e64;
5982 else if (
Opc == AMDGPU::S_AND_B64)
5983 ClampOpc = AMDGPU::V_AND_B32_e64;
5984 else if (
Opc == AMDGPU::S_OR_B64)
5985 ClampOpc = AMDGPU::V_OR_B32_e64;
5986 else if (
Opc == AMDGPU::S_XOR_B64)
5987 ClampOpc = AMDGPU::V_XOR_B32_e64;
5989 ClampOpc = ST.getInstrInfo()->getVALUOp(ClampOpc);
5991 return {DPPOpc, ClampOpc};
5994static std::pair<Register, Register>
6001 TRI->getSubRegisterClass(SrcRC, AMDGPU::sub0);
6003 TII->buildExtractSubReg(
MI,
MRI,
Op, SrcRC, AMDGPU::sub0, SrcSubRC);
6005 TII->buildExtractSubReg(
MI,
MRI,
Op, SrcRC, AMDGPU::sub1, SrcSubRC);
6006 return {Op1L, Op1H};
6020 bool isSGPR =
TRI->isSGPRClass(
MRI.getRegClass(SrcReg));
6022 unsigned Stratergy =
static_cast<unsigned>(
MI.getOperand(2).
getImm());
6023 enum WAVE_REDUCE_STRATEGY :
unsigned {
DEFAULT = 0, ITERATIVE = 1, DPP = 2 };
6025 unsigned MIOpc =
MI.getOpcode();
6039 case AMDGPU::S_MIN_U32:
6040 case AMDGPU::S_MIN_I32:
6041 case AMDGPU::V_MIN_F32_e64:
6042 case AMDGPU::S_MAX_U32:
6043 case AMDGPU::S_MAX_I32:
6044 case AMDGPU::V_MAX_F32_e64:
6045 case AMDGPU::S_AND_B32:
6046 case AMDGPU::S_OR_B32: {
6052 case AMDGPU::V_CMP_LT_U64_e64:
6053 case AMDGPU::V_CMP_LT_I64_e64:
6054 case AMDGPU::V_CMP_GT_U64_e64:
6055 case AMDGPU::V_CMP_GT_I64_e64:
6056 case AMDGPU::V_MIN_F64_e64:
6057 case AMDGPU::V_MIN_NUM_F64_e64:
6058 case AMDGPU::V_MAX_F64_e64:
6059 case AMDGPU::V_MAX_NUM_F64_e64:
6060 case AMDGPU::S_AND_B64:
6061 case AMDGPU::S_OR_B64: {
6067 case AMDGPU::S_XOR_B32:
6068 case AMDGPU::S_XOR_B64:
6069 case AMDGPU::S_ADD_I32:
6070 case AMDGPU::S_ADD_U64_PSEUDO:
6071 case AMDGPU::V_ADD_F32_e64:
6072 case AMDGPU::V_ADD_F64_e64:
6073 case AMDGPU::V_ADD_F64_pseudo_e64:
6074 case AMDGPU::S_SUB_I32:
6075 case AMDGPU::S_SUB_U64_PSEUDO:
6076 case AMDGPU::V_SUB_F32_e64: {
6079 Register ExecMask =
MRI.createVirtualRegister(WaveMaskRegClass);
6081 MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
6083 bool IsWave32 = ST.isWave32();
6084 unsigned MovOpc = IsWave32 ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64;
6085 MCRegister ExecReg = IsWave32 ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
6086 unsigned BitCountOpc =
6087 IsWave32 ? AMDGPU::S_BCNT1_I32_B32 : AMDGPU::S_BCNT1_I32_B64;
6091 auto NewAccumulator =
6097 case AMDGPU::S_XOR_B32:
6098 case AMDGPU::S_XOR_B64: {
6104 MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
6106 .
addReg(NewAccumulator->getOperand(0).getReg())
6110 if (
TII->getImmOrMaterializedImm(
MRI, SrcReg) == 1) {
6112 if (
Opc == AMDGPU::S_XOR_B32) {
6117 MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
6119 BuildRegSequence(BB,
MI, DstReg, ParityRegister, DstHi);
6123 if (
Opc == AMDGPU::S_XOR_B32) {
6129 MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
6131 MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
6133 MRI.getRegClass(SrcReg), ST,
MRI);
6140 BuildRegSequence(BB,
MI, DstReg, DestSub0, DestSub1);
6144 case AMDGPU::S_SUB_I32: {
6145 Register NegatedVal =
MRI.createVirtualRegister(DstRegClass);
6153 .
addReg(NewAccumulator->getOperand(0).getReg());
6156 case AMDGPU::S_ADD_I32: {
6158 if (
TII->getImmOrMaterializedImm(
MRI, SrcReg) == 1) {
6161 .
addReg(NewAccumulator->getOperand(0).getReg());
6166 .
addReg(NewAccumulator->getOperand(0).getReg());
6169 case AMDGPU::S_ADD_U64_PSEUDO:
6170 case AMDGPU::S_SUB_U64_PSEUDO: {
6171 Register DestSub0 =
MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
6172 Register DestSub1 =
MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
6174 MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
6176 MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
6178 MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
6179 Register AddReg =
MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
6181 MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
6183 MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
6185 MRI.getRegClass(SrcReg), ST,
MRI);
6187 if (
Opc == AMDGPU::S_ADD_U64_PSEUDO &&
6188 TII->getImmOrMaterializedImm(
MRI, SrcReg) == 1) {
6190 Register DstHi =
MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
6192 BuildRegSequence(BB,
MI, DstReg,
6193 NewAccumulator->getOperand(0).getReg(), DstHi);
6196 if (
Opc == AMDGPU::S_SUB_U64_PSEUDO) {
6199 .
addReg(NewAccumulator->getOperand(0).getReg())
6209 Register LowOpcode =
Opc == AMDGPU::S_SUB_U64_PSEUDO
6211 : NewAccumulator->getOperand(0).getReg();
6215 if (ST.hasScalarMulHiInsts()) {
6221 MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
6223 MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
6226 BuildMI(BB,
MI,
DL,
TII->get(AMDGPU::V_MUL_HI_U32_e64), VCarryReg)
6229 BuildMI(BB,
MI,
DL,
TII->get(AMDGPU::V_READFIRSTLANE_B32), CarryReg)
6236 Register HiVal =
Opc == AMDGPU::S_SUB_U64_PSEUDO ? AddReg : DestSub1;
6242 if (
Opc == AMDGPU::S_SUB_U64_PSEUDO) {
6248 BuildRegSequence(BB,
MI, DstReg, DestSub0, DestSub1);
6251 case AMDGPU::V_ADD_F32_e64:
6252 case AMDGPU::V_ADD_F64_e64:
6253 case AMDGPU::V_ADD_F64_pseudo_e64:
6254 case AMDGPU::V_SUB_F32_e64: {
6257 Register ActiveLanesVreg =
MRI.createVirtualRegister(VregRC);
6258 Register DstVreg =
MRI.createVirtualRegister(VregRC);
6261 TII->get(is32BitOpc ? AMDGPU::V_CVT_F32_I32_e64
6262 : AMDGPU::V_CVT_F64_I32_e64),
6264 .
addReg(NewAccumulator->getOperand(0).getReg())
6269 unsigned srcMod = (MIOpc == AMDGPU::WAVE_REDUCE_FSUB_PSEUDO_F32 ||
6270 MIOpc == AMDGPU::WAVE_REDUCE_FSUB_PSEUDO_F64)
6273 unsigned MulOpc = is32BitOpc ? AMDGPU::V_MUL_F32_e64
6275 ? AMDGPU::V_MUL_F64_pseudo_e64
6276 : AMDGPU::V_MUL_F64_e64;
6286 BuildMI(BB,
MI,
DL,
TII->get(AMDGPU::V_READFIRSTLANE_B32), DstReg)
6290 MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
6292 MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
6303 BuildRegSequence(BB,
MI, DstReg, LaneValueLoReg, LaneValueHiReg);
6315 bool NeedsMovDPP = !is32BitOpc;
6320 bool IsWave32 = ST.isWave32();
6321 unsigned MovOpcForExec = IsWave32 ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64;
6322 unsigned ExecReg = IsWave32 ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
6323 if (Stratergy == WAVE_REDUCE_STRATEGY::ITERATIVE ||
6337 Register LoopIterator =
MRI.createVirtualRegister(WaveMaskRegClass);
6338 Register IdentityValReg =
MRI.createVirtualRegister(DstRegClass);
6339 Register AccumulatorReg =
MRI.createVirtualRegister(DstRegClass);
6340 Register ActiveBitsReg =
MRI.createVirtualRegister(WaveMaskRegClass);
6341 Register NewActiveBitsReg =
MRI.createVirtualRegister(WaveMaskRegClass);
6342 Register FF1Reg =
MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
6343 Register LaneValueReg =
MRI.createVirtualRegister(DstRegClass);
6349 MI.getOpcode() == AMDGPU::WAVE_REDUCE_FSUB_PSEUDO_F64
6353 TII->get(is32BitOpc ? AMDGPU::S_MOV_B32
6354 : AMDGPU::S_MOV_B64_IMM_PSEUDO),
6363 I = ComputeLoop->begin();
6365 BuildMI(*ComputeLoop,
I,
DL,
TII->get(AMDGPU::PHI), AccumulatorReg)
6369 BuildMI(*ComputeLoop,
I,
DL,
TII->get(AMDGPU::PHI), ActiveBitsReg)
6373 I = ComputeLoop->end();
6377 IsWave32 ? AMDGPU::S_FF1_I32_B32 : AMDGPU::S_FF1_I32_B64;
6382 bool hasSrc0Modifier = AMDGPU::getNamedOperandIdx(
6383 Opc, AMDGPU::OpName::src0_modifiers) != -1;
6384 bool hasSrc1Modifier = AMDGPU::getNamedOperandIdx(
6385 Opc, AMDGPU::OpName::src1_modifiers) != -1;
6387 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::clamp) != -1;
6389 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel) != -1;
6391 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::omod) != -1;
6392 BuildMI(*ComputeLoop,
I,
DL,
TII->get(AMDGPU::V_READLANE_B32),
6396 if (ST.getInstrInfo()->isVALU(
Opc,
false)) {
6398 Register LaneValVgpr =
MRI.createVirtualRegister(SrcRegClass);
6399 Register VgprResultReg =
MRI.createVirtualRegister(SrcRegClass);
6400 BuildMI(*ComputeLoop,
I,
DL,
TII->get(AMDGPU::COPY), LaneValVgpr)
6402 OpDstReg = VgprResultReg;
6403 LaneValueReg = LaneValVgpr;
6406 if (hasSrc0Modifier)
6408 OpInstr.addReg(AccumulatorReg);
6409 if (hasSrc1Modifier)
6411 OpInstr.addReg(LaneValueReg);
6419 OpInstr.setOperandDead(3);
6420 if (ST.getInstrInfo()->isVALU(
Opc,
false)) {
6421 BuildMI(*ComputeLoop,
I,
DL,
TII->get(AMDGPU::V_READFIRSTLANE_B32),
6427 MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
6429 MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
6431 MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
6433 MRI.getRegClass(SrcReg), ST,
MRI);
6435 BuildMI(*ComputeLoop,
I,
DL,
TII->get(AMDGPU::V_READLANE_B32),
6439 BuildMI(*ComputeLoop,
I,
DL,
TII->get(AMDGPU::V_READLANE_B32),
6443 auto LaneValue = BuildRegSequence(*ComputeLoop,
I, LaneValReg,
6444 LaneValueLoReg, LaneValueHiReg);
6446 case AMDGPU::S_OR_B64:
6447 case AMDGPU::S_AND_B64:
6448 case AMDGPU::S_XOR_B64: {
6451 .
addReg(LaneValue->getOperand(0).getReg())
6455 case AMDGPU::V_CMP_GT_I64_e64:
6456 case AMDGPU::V_CMP_GT_U64_e64:
6457 case AMDGPU::V_CMP_LT_I64_e64:
6458 case AMDGPU::V_CMP_LT_U64_e64: {
6459 Register LaneMaskReg =
MRI.createVirtualRegister(WaveMaskRegClass);
6461 MRI.createVirtualRegister(WaveMaskRegClass);
6463 AMDGPU::getNamedOperandIdx(
MI.getOpcode(), AMDGPU::OpName::src);
6465 TRI->getAllocatableClass(
TII->getRegClass(
MI.getDesc(), SrcIdx));
6466 Register AccumulatorVReg =
MRI.createVirtualRegister(VregClass);
6469 BuildRegSequence(*ComputeLoop,
I, AccumulatorVReg, SrcReg0Sub0,
6472 .
addReg(LaneValue->getOperand(0).getReg())
6473 .
addReg(AccumulatorVReg);
6475 unsigned AndOpc = IsWave32 ? AMDGPU::S_AND_B32 : AMDGPU::S_AND_B64;
6476 BuildMI(*ComputeLoop,
I,
DL,
TII->get(AndOpc), ComparisonResultReg)
6480 NewAccumulator =
BuildMI(*ComputeLoop,
I,
DL,
6481 TII->get(AMDGPU::S_CSELECT_B64), DstReg)
6482 .
addReg(LaneValue->getOperand(0).getReg())
6486 case AMDGPU::V_MIN_F64_e64:
6487 case AMDGPU::V_MIN_NUM_F64_e64:
6488 case AMDGPU::V_MAX_F64_e64:
6489 case AMDGPU::V_MAX_NUM_F64_e64:
6490 case AMDGPU::V_ADD_F64_e64:
6491 case AMDGPU::V_ADD_F64_pseudo_e64: {
6493 AMDGPU::getNamedOperandIdx(
MI.getOpcode(), AMDGPU::OpName::src);
6495 TRI->getAllocatableClass(
TII->getRegClass(
MI.getDesc(), SrcIdx));
6496 Register AccumulatorVReg =
MRI.createVirtualRegister(VregRC);
6497 Register DstVreg =
MRI.createVirtualRegister(VregRC);
6499 MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
6501 MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
6502 BuildMI(*ComputeLoop,
I,
DL,
TII->get(AMDGPU::COPY), AccumulatorVReg)
6505 MI.getOpcode() == AMDGPU::WAVE_REDUCE_FSUB_PSEUDO_F64
6511 .
addReg(LaneValue->getOperand(0).getReg())
6518 TII->get(AMDGPU::V_READFIRSTLANE_B32), LaneValLo);
6521 TII->get(AMDGPU::V_READFIRSTLANE_B32), LaneValHi);
6523 auto [Op1L, Op1H] =
ExtractSubRegs(*Iters, DstVregInst->getOperand(0),
6525 ReadLaneLo.addReg(Op1L);
6526 ReadLaneHi.addReg(Op1H);
6528 BuildRegSequence(*ComputeLoop,
I, DstReg, LaneValLo, LaneValHi);
6531 case AMDGPU::S_ADD_U64_PSEUDO:
6532 case AMDGPU::S_SUB_U64_PSEUDO: {
6535 .
addReg(LaneValue->getOperand(0).getReg())
6544 unsigned BITSETOpc =
6545 IsWave32 ? AMDGPU::S_BITSET0_B32 : AMDGPU::S_BITSET0_B64;
6546 BuildMI(*ComputeLoop,
I,
DL,
TII->get(BITSETOpc), NewActiveBitsReg)
6552 ActiveBits.addReg(NewActiveBitsReg).addMBB(ComputeLoop);
6556 if (!ST.hasScalarCompareEq64()) {
6558 Register LaneMaskReg =
MRI.createVirtualRegister(WaveMaskRegClass);
6559 unsigned CMPOpc = IsWave32 ? AMDGPU::S_OR_B32 : AMDGPU::S_OR_B64;
6561 BuildMI(*ComputeLoop,
I,
DL,
TII->get(CMPOpc), LaneMaskReg);
6564 IsWave32 ? AMDGPU::S_CMP_LG_U32 : AMDGPU::S_CMP_LG_U64;
6565 SetSCCInstr =
BuildMI(*ComputeLoop,
I,
DL,
TII->get(CMPOpc));
6567 SetSCCInstr.
addReg(NewActiveBitsReg);
6568 if (ST.hasScalarCompareEq64())
6571 SetSCCInstr.
addReg(NewActiveBitsReg);
6572 BuildMI(*ComputeLoop,
I,
DL,
TII->get(AMDGPU::S_CBRANCH_SCC1))
6577 assert(ST.hasDPP() &&
"Sub Target does not support DPP Operations");
6579 Register SrcWithIdentity =
MRI.createVirtualRegister(SrcRegClass);
6580 Register IdentityVGPR =
MRI.createVirtualRegister(SrcRegClass);
6581 Register IdentitySGPR =
MRI.createVirtualRegister(DstRegClass);
6582 Register DPPRowShr1 =
MRI.createVirtualRegister(SrcRegClass);
6583 Register DPPRowShr2 =
MRI.createVirtualRegister(SrcRegClass);
6584 Register DPPRowShr4 =
MRI.createVirtualRegister(SrcRegClass);
6585 Register DPPRowShr8 =
MRI.createVirtualRegister(SrcRegClass);
6586 Register RowBcast15 =
MRI.createVirtualRegister(SrcRegClass);
6587 Register ReducedValSGPR =
MRI.createVirtualRegister(DstRegClass);
6588 Register NegatedReducedVal =
MRI.createVirtualRegister(DstRegClass);
6589 Register RowBcast31 =
MRI.createVirtualRegister(SrcRegClass);
6590 Register UndefExec =
MRI.createVirtualRegister(WaveMaskRegClass);
6594 BuildMI(*CurrBB,
MI,
DL,
TII->get(AMDGPU::IMPLICIT_DEF), UndefExec);
6598 TII->get(is32BitOpc ? AMDGPU::S_MOV_B32
6599 : AMDGPU::S_MOV_B64_IMM_PSEUDO),
6602 auto IdentityCopyInstr =
6605 unsigned DPPOpc, ClampOpc;
6621 if (isFPOp && !NeedsMovDPP)
6624 if (isFPOp && !NeedsMovDPP)
6628 if (AMDGPU::getNamedOperandIdx(DPPOpc, AMDGPU::OpName::clamp) >= 0)
6637 bool isAddSub =
false,
6638 bool needsCarryIn =
false,
6640 unsigned InstrOpc = ClampOpc;
6641 Register CarryOutReg =
MRI.createVirtualRegister(WaveMaskRegClass);
6643 InstrOpc = AMDGPU::V_ADDC_U32_e64;
6644 auto ClampInstr =
BuildMI(*CurrBB,
MI,
DL,
TII->get(InstrOpc), Dst);
6649 ClampInstr.addReg(CarryOutReg,
6655 ClampInstr.addReg(Src0);
6658 ClampInstr.addReg(Src1);
6660 ClampInstr.addReg(CarryIn);
6661 if (AMDGPU::getNamedOperandIdx(InstrOpc, AMDGPU::OpName::clamp) >= 0)
6662 ClampInstr.addImm(0);
6664 ClampInstr.addImm(0);
6665 LastBcastInstr = ClampInstr;
6670 Opc == AMDGPU::S_ADD_U64_PSEUDO ||
Opc == AMDGPU::S_SUB_U64_PSEUDO;
6671 bool isBitWiseOpc =
Opc == AMDGPU::S_AND_B64 ||
6672 Opc == AMDGPU::S_OR_B64 ||
Opc == AMDGPU::S_XOR_B64;
6673 Register ReturnReg =
MRI.createVirtualRegister(SrcRegClass);
6674 if (isAddSubOpc || isBitWiseOpc) {
6675 Register ResLo =
MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
6676 Register ResHi =
MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
6681 auto [Src0Lo, Src0Hi] =
6683 auto [Src1Lo, Src1Hi] =
6685 Register CarryReg = BuildClampInstr(
6686 ResLo, Src0Lo, Src1Lo, isAddSubOpc,
false);
6687 BuildClampInstr(ResHi, Src0Hi, Src1Hi, isAddSubOpc,
6688 isAddSubOpc, CarryReg);
6689 BuildRegSequence(*CurrBB,
MI, ReturnReg, ResLo, ResHi);
6700 Register CmpMaskReg =
MRI.createVirtualRegister(WaveMaskRegClass);
6718 SrcWithIdentityInstr =
6719 BuildSetInactiveInstr(SrcWithIdentity, SrcReg, IdentityVGPR);
6722 MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
6724 MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
6726 MI, IdentityCopyInstr->getOperand(0), SrcRegClass, ST,
MRI);
6727 auto [SrcReg0Sub0, SrcReg0Sub1] =
6730 BuildSetInactiveInstr(SrcWithIdentitylo, SrcReg0Sub0, Reg0Sub0);
6732 BuildSetInactiveInstr(SrcWithIdentityhi, SrcReg0Sub1, Reg0Sub1);
6733 SrcWithIdentityInstr =
6734 BuildRegSequence(*CurrBB,
MI, SrcWithIdentity,
6741 BuildDPPMachineInstr(DPPRowShr1, SrcWithIdentityReg,
6744 DPPRowShr1 = BuildPostDPPInstr(SrcWithIdentityReg, DPPRowShr1);
6746 BuildDPPMachineInstr(DPPRowShr2, DPPRowShr1,
6749 DPPRowShr2 = BuildPostDPPInstr(DPPRowShr1, DPPRowShr2);
6751 BuildDPPMachineInstr(DPPRowShr4, DPPRowShr2,
6754 DPPRowShr4 = BuildPostDPPInstr(DPPRowShr2, DPPRowShr4);
6756 BuildDPPMachineInstr(DPPRowShr8, DPPRowShr4,
6759 DPPRowShr8 = BuildPostDPPInstr(DPPRowShr4, DPPRowShr8);
6761 if (ST.hasDPPBroadcasts()) {
6764 RowBcast15 = BuildPostDPPInstr(DPPRowShr8, RowBcast15);
6773 MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
6779 BuildClampInstr(RowBcast15, DPPRowShr8, SwizzledValue);
6782 MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
6784 MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
6785 Register SwizzledValue64 =
MRI.createVirtualRegister(SrcRegClass);
6800 BuildRegSequence(*CurrBB,
MI, SwizzledValue64, SwizzledValuelo,
6803 RowBcast15 = BuildPostDPPInstr(DPPRowShr8, SwizzledValue64);
6805 BuildClampInstr(RowBcast15, DPPRowShr8, SwizzledValue64);
6808 FinalDPPResult = RowBcast15;
6810 if (ST.hasDPPBroadcasts()) {
6813 RowBcast31 = BuildPostDPPInstr(RowBcast15, RowBcast31);
6816 MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
6818 MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
6819 Register PermutedValue =
MRI.createVirtualRegister(SrcRegClass);
6821 MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
6823 MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
6825 MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
6827 MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
6829 BuildMI(*CurrBB,
MI,
DL,
TII->get(AMDGPU::V_MBCNT_LO_U32_B32_e64),
6833 BuildMI(*CurrBB,
MI,
DL,
TII->get(AMDGPU::V_MBCNT_HI_U32_B32_e64),
6839 BuildMI(*CurrBB,
MI,
DL,
TII->get(AMDGPU::S_MOV_B32), Lane32Offset)
6847 BuildMI(*CurrBB,
MI,
DL,
TII->get(AMDGPU::S_MOV_B32), WordSizeConst)
6852 .
addReg(ShiftedThreadID);
6857 .
addReg(PermuteByteOffset)
6862 MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
6864 MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
6867 auto [RowBcast15Lo, RowBcast15Hi] =
6871 .
addReg(PermuteByteOffset)
6876 .
addReg(PermuteByteOffset)
6879 BuildRegSequence(*CurrBB,
MI, PermutedValue, PermutedValuelo,
6883 RowBcast31 = BuildPostDPPInstr(RowBcast15, PermutedValue);
6885 BuildClampInstr(RowBcast31, RowBcast15, PermutedValue);
6887 FinalDPPResult = RowBcast31;
6889 if (MIOpc == AMDGPU::WAVE_REDUCE_FSUB_PSEUDO_F32 ||
6890 MIOpc == AMDGPU::WAVE_REDUCE_FSUB_PSEUDO_F64) {
6891 Register NegatedValVGPR =
MRI.createVirtualRegister(SrcRegClass);
6900 .
addReg(IsWave32 ? RowBcast15 : RowBcast31)
6903 FinalDPPResult = NegatedValVGPR;
6910 .
addImm(ST.getWavefrontSize() - 1);
6913 MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
6915 MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
6925 .
addImm(ST.getWavefrontSize() - 1);
6929 .
addImm(ST.getWavefrontSize() - 1);
6930 BuildRegSequence(*CurrBB,
MI, ReducedValSGPR, LaneValueLoReg,
6933 if (
Opc == AMDGPU::S_SUB_I32) {
6934 BuildMI(*CurrBB,
MI,
DL,
TII->get(AMDGPU::S_SUB_I32), NegatedReducedVal)
6938 }
else if (
Opc == AMDGPU::S_SUB_U64_PSEUDO) {
6939 auto NegatedValInstr =
6948 .
addReg(
Opc == AMDGPU::S_SUB_I32 ||
Opc == AMDGPU::S_SUB_U64_PSEUDO
6954 MI.eraseFromParent();
6969 switch (
MI.getOpcode()) {
6970 case AMDGPU::WAVE_REDUCE_UMIN_PSEUDO_U32:
6972 case AMDGPU::WAVE_REDUCE_UMIN_PSEUDO_U64:
6974 case AMDGPU::WAVE_REDUCE_MIN_PSEUDO_I32:
6976 case AMDGPU::WAVE_REDUCE_MIN_PSEUDO_I64:
6978 case AMDGPU::WAVE_REDUCE_FMIN_PSEUDO_F32:
6980 case AMDGPU::WAVE_REDUCE_FMIN_PSEUDO_F64:
6983 ? AMDGPU::V_MIN_NUM_F64_e64
6984 : AMDGPU::V_MIN_F64_e64);
6985 case AMDGPU::WAVE_REDUCE_UMAX_PSEUDO_U32:
6987 case AMDGPU::WAVE_REDUCE_UMAX_PSEUDO_U64:
6989 case AMDGPU::WAVE_REDUCE_MAX_PSEUDO_I32:
6991 case AMDGPU::WAVE_REDUCE_MAX_PSEUDO_I64:
6993 case AMDGPU::WAVE_REDUCE_FMAX_PSEUDO_F32:
6995 case AMDGPU::WAVE_REDUCE_FMAX_PSEUDO_F64:
6998 ? AMDGPU::V_MAX_NUM_F64_e64
6999 : AMDGPU::V_MAX_F64_e64);
7000 case AMDGPU::WAVE_REDUCE_ADD_PSEUDO_I32:
7002 case AMDGPU::WAVE_REDUCE_ADD_PSEUDO_U64:
7004 case AMDGPU::WAVE_REDUCE_FADD_PSEUDO_F32:
7006 case AMDGPU::WAVE_REDUCE_FADD_PSEUDO_F64:
7009 ? AMDGPU::V_ADD_F64_pseudo_e64
7010 : AMDGPU::V_ADD_F64_e64);
7011 case AMDGPU::WAVE_REDUCE_SUB_PSEUDO_I32:
7013 case AMDGPU::WAVE_REDUCE_SUB_PSEUDO_U64:
7015 case AMDGPU::WAVE_REDUCE_FSUB_PSEUDO_F32:
7017 case AMDGPU::WAVE_REDUCE_FSUB_PSEUDO_F64:
7022 ? AMDGPU::V_ADD_F64_pseudo_e64
7023 : AMDGPU::V_ADD_F64_e64);
7024 case AMDGPU::WAVE_REDUCE_AND_PSEUDO_B32:
7026 case AMDGPU::WAVE_REDUCE_AND_PSEUDO_B64:
7028 case AMDGPU::WAVE_REDUCE_OR_PSEUDO_B32:
7030 case AMDGPU::WAVE_REDUCE_OR_PSEUDO_B64:
7032 case AMDGPU::WAVE_REDUCE_XOR_PSEUDO_B32:
7034 case AMDGPU::WAVE_REDUCE_XOR_PSEUDO_B64:
7036 case AMDGPU::S_UADDO_PSEUDO:
7037 case AMDGPU::S_USUBO_PSEUDO: {
7043 unsigned Opc = (
MI.getOpcode() == AMDGPU::S_UADDO_PSEUDO)
7045 : AMDGPU::S_SUB_U32;
7053 Subtarget->isWave64() ? AMDGPU::S_CSELECT_B64 : AMDGPU::S_CSELECT_B32;
7056 MI.eraseFromParent();
7059 case AMDGPU::S_ADD_U64_PSEUDO:
7060 case AMDGPU::S_SUB_U64_PSEUDO: {
7063 case AMDGPU::V_ADD_U64_PSEUDO:
7064 case AMDGPU::V_SUB_U64_PSEUDO: {
7065 bool IsAdd = (
MI.getOpcode() == AMDGPU::V_ADD_U64_PSEUDO);
7071 if (ST.hasAddSubU64Insts()) {
7073 TII->get(IsAdd ? AMDGPU::V_ADD_U64_e64
7074 : AMDGPU::V_SUB_U64_e64),
7079 TII->legalizeOperands(*
I);
7080 MI.eraseFromParent();
7084 if (IsAdd && ST.hasLshlAddU64Inst()) {
7090 TII->legalizeOperands(*
Add);
7091 MI.eraseFromParent();
7095 const auto *CarryRC =
TRI->getWaveMaskRegClass();
7097 Register DestSub0 =
MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
7098 Register DestSub1 =
MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
7100 Register CarryReg =
MRI.createVirtualRegister(CarryRC);
7101 Register DeadCarryReg =
MRI.createVirtualRegister(CarryRC);
7104 ?
MRI.getRegClass(Src0.getReg())
7105 : &AMDGPU::VReg_64RegClass;
7107 ?
MRI.getRegClass(Src1.getReg())
7108 : &AMDGPU::VReg_64RegClass;
7111 TRI->getSubRegisterClass(Src0RC, AMDGPU::sub0);
7113 TRI->getSubRegisterClass(Src1RC, AMDGPU::sub1);
7116 MI,
MRI, Src0, Src0RC, AMDGPU::sub0, Src0SubRC);
7118 MI,
MRI, Src1, Src1RC, AMDGPU::sub0, Src1SubRC);
7121 MI,
MRI, Src0, Src0RC, AMDGPU::sub1, Src0SubRC);
7123 MI,
MRI, Src1, Src1RC, AMDGPU::sub1, Src1SubRC);
7126 IsAdd ? AMDGPU::V_ADD_CO_U32_e64 : AMDGPU::V_SUB_CO_U32_e64;
7133 unsigned HiOpc = IsAdd ? AMDGPU::V_ADDC_U32_e64 : AMDGPU::V_SUBB_U32_e64;
7147 TII->legalizeOperands(*LoHalf);
7148 TII->legalizeOperands(*HiHalf);
7149 MI.eraseFromParent();
7152 case AMDGPU::S_ADD_CO_PSEUDO:
7153 case AMDGPU::S_SUB_CO_PSEUDO: {
7163 if (Src0.isReg() &&
TRI->isVectorRegister(
MRI, Src0.getReg())) {
7164 Register RegOp0 =
MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
7165 BuildMI(*BB, MII,
DL,
TII->get(AMDGPU::V_READFIRSTLANE_B32), RegOp0)
7167 Src0.setReg(RegOp0);
7169 if (Src1.isReg() &&
TRI->isVectorRegister(
MRI, Src1.getReg())) {
7170 Register RegOp1 =
MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
7171 BuildMI(*BB, MII,
DL,
TII->get(AMDGPU::V_READFIRSTLANE_B32), RegOp1)
7173 Src1.setReg(RegOp1);
7175 Register RegOp2 =
MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
7176 if (
TRI->isVectorRegister(
MRI, Src2.getReg())) {
7177 BuildMI(*BB, MII,
DL,
TII->get(AMDGPU::V_READFIRSTLANE_B32), RegOp2)
7179 Src2.setReg(RegOp2);
7182 if (ST.isWave64()) {
7183 if (ST.hasScalarCompareEq64()) {
7190 TRI->getSubRegisterClass(Src2RC, AMDGPU::sub0);
7192 MII,
MRI, Src2, Src2RC, AMDGPU::sub0, SubRC);
7194 MII,
MRI, Src2, Src2RC, AMDGPU::sub1, SubRC);
7195 Register Src2_32 =
MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
7197 BuildMI(*BB, MII,
DL,
TII->get(AMDGPU::S_OR_B32), Src2_32)
7211 unsigned Opc =
MI.getOpcode() == AMDGPU::S_ADD_CO_PSEUDO
7212 ? AMDGPU::S_ADDC_U32
7213 : AMDGPU::S_SUBB_U32;
7218 ST.isWave64() ? AMDGPU::S_CSELECT_B64 : AMDGPU::S_CSELECT_B32;
7224 MI.eraseFromParent();
7227 case AMDGPU::SI_INIT_M0: {
7230 TII->get(M0Init.
isReg() ? AMDGPU::COPY : AMDGPU::S_MOV_B32),
7233 MI.eraseFromParent();
7236 case AMDGPU::S_BARRIER_SIGNAL_ISFIRST_IMM: {
7239 TII->get(AMDGPU::S_CMP_EQ_U32))
7244 case AMDGPU::GET_GROUPSTATICSIZE: {
7248 .
add(
MI.getOperand(0))
7250 MI.eraseFromParent();
7253 case AMDGPU::GET_SHADERCYCLESHILO: {
7266 Register RegHi1 =
MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
7268 .
addImm(HwregEncoding::encode(ID_SHADER_CYCLES_HI, 0, 32));
7269 Register RegLo1 =
MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
7271 .
addImm(HwregEncoding::encode(ID_SHADER_CYCLES, 0, 32));
7272 Register RegHi2 =
MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
7274 .
addImm(HwregEncoding::encode(ID_SHADER_CYCLES_HI, 0, 32));
7278 Register RegLo =
MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
7283 .
add(
MI.getOperand(0))
7288 MI.eraseFromParent();
7291 case AMDGPU::SI_INDIRECT_SRC_V1:
7292 case AMDGPU::SI_INDIRECT_SRC_V2:
7293 case AMDGPU::SI_INDIRECT_SRC_V3:
7294 case AMDGPU::SI_INDIRECT_SRC_V4:
7295 case AMDGPU::SI_INDIRECT_SRC_V5:
7296 case AMDGPU::SI_INDIRECT_SRC_V6:
7297 case AMDGPU::SI_INDIRECT_SRC_V7:
7298 case AMDGPU::SI_INDIRECT_SRC_V8:
7299 case AMDGPU::SI_INDIRECT_SRC_V9:
7300 case AMDGPU::SI_INDIRECT_SRC_V10:
7301 case AMDGPU::SI_INDIRECT_SRC_V11:
7302 case AMDGPU::SI_INDIRECT_SRC_V12:
7303 case AMDGPU::SI_INDIRECT_SRC_V16:
7304 case AMDGPU::SI_INDIRECT_SRC_V32:
7306 case AMDGPU::SI_INDIRECT_DST_V1:
7307 case AMDGPU::SI_INDIRECT_DST_V2:
7308 case AMDGPU::SI_INDIRECT_DST_V3:
7309 case AMDGPU::SI_INDIRECT_DST_V4:
7310 case AMDGPU::SI_INDIRECT_DST_V5:
7311 case AMDGPU::SI_INDIRECT_DST_V6:
7312 case AMDGPU::SI_INDIRECT_DST_V7:
7313 case AMDGPU::SI_INDIRECT_DST_V8:
7314 case AMDGPU::SI_INDIRECT_DST_V9:
7315 case AMDGPU::SI_INDIRECT_DST_V10:
7316 case AMDGPU::SI_INDIRECT_DST_V11:
7317 case AMDGPU::SI_INDIRECT_DST_V12:
7318 case AMDGPU::SI_INDIRECT_DST_V16:
7319 case AMDGPU::SI_INDIRECT_DST_V32:
7321 case AMDGPU::SI_KILL_F32_COND_IMM_PSEUDO:
7322 case AMDGPU::SI_KILL_I1_PSEUDO:
7324 case AMDGPU::V_CNDMASK_B64_PSEUDO: {
7328 case AMDGPU::SI_BR_UNDEF: {
7330 .
add(
MI.getOperand(0));
7332 MI.eraseFromParent();
7335 case AMDGPU::ADJCALLSTACKUP:
7336 case AMDGPU::ADJCALLSTACKDOWN: {
7343 case AMDGPU::SI_CALL_ISEL: {
7344 unsigned ReturnAddrReg =
TII->getRegisterInfo().getReturnAddressReg(*MF);
7355 MI.eraseFromParent();
7358 case AMDGPU::V_ADDC_U32_e32:
7359 case AMDGPU::V_SUBB_U32_e32:
7360 case AMDGPU::V_SUBBREV_U32_e32:
7363 TII->legalizeOperands(
MI);
7365 case AMDGPU::DS_GWS_INIT:
7366 case AMDGPU::DS_GWS_SEMA_BR:
7367 case AMDGPU::DS_GWS_BARRIER:
7368 case AMDGPU::DS_GWS_SEMA_V:
7369 case AMDGPU::DS_GWS_SEMA_P:
7370 case AMDGPU::DS_GWS_SEMA_RELEASE_ALL:
7378 case AMDGPU::S_SETREG_B32: {
7388 auto [ID,
Offset, Width] =
7394 const unsigned SetMask = WidthMask <<
Offset;
7397 unsigned SetDenormOp = 0;
7398 unsigned SetRoundOp = 0;
7406 SetRoundOp = AMDGPU::S_ROUND_MODE;
7407 SetDenormOp = AMDGPU::S_DENORM_MODE;
7409 SetRoundOp = AMDGPU::S_ROUND_MODE;
7411 SetDenormOp = AMDGPU::S_DENORM_MODE;
7414 if (SetRoundOp || SetDenormOp) {
7416 if (Def && Def->isMoveImmediate() && Def->getOperand(1).isImm()) {
7417 unsigned ImmVal = Def->getOperand(1).getImm();
7431 MI.eraseFromParent();
7440 MI.setDesc(
TII->get(AMDGPU::S_SETREG_B32_mode));
7444 case AMDGPU::S_INVERSE_BALLOT_U32:
7445 case AMDGPU::S_INVERSE_BALLOT_U64:
7448 MI.setDesc(
TII->get(AMDGPU::COPY));
7450 case AMDGPU::ENDPGM_TRAP: {
7452 MI.setDesc(
TII->get(AMDGPU::S_ENDPGM));
7472 MI.eraseFromParent();
7475 case AMDGPU::SIMULATED_TRAP: {
7476 assert(Subtarget->hasPrivEnabledTrap2NopBug());
7478 TII->insertSimulatedTrap(
MRI, *BB,
MI,
MI.getDebugLoc());
7479 MI.eraseFromParent();
7482 case AMDGPU::SI_TCRETURN_GFX_WholeWave:
7483 case AMDGPU::SI_WHOLE_WAVE_FUNC_RETURN: {
7489 assert(Setup &&
"Couldn't find SI_SETUP_WHOLE_WAVE_FUNC");
7490 Register OriginalExec = Setup->getOperand(0).getReg();
7491 MI.getOperand(0).setReg(OriginalExec);
7494 case AMDGPU::V_DOT2_F32_F16:
7495 case AMDGPU::V_DOT2_F32_BF16: {
7499 MRI.setSimpleHint(
MI.getOperand(0).getReg(),
MI.getOperand(6).getReg());
7502 case AMDGPU::SCHED_BARRIER:
7503 case AMDGPU::SCHED_GROUP_BARRIER:
7504 MI.getOperand(0).setImm(
MI.getOperand(0).getImm() &
7541 return (VT == MVT::i16) ? MVT::i16 : MVT::i32;
7545 return (Ty.getScalarSizeInBits() <= 16 && Subtarget->has16BitInsts())
7574 if (!Subtarget->hasMadMacF32Insts())
7575 return Subtarget->hasFastFMAF32();
7581 return Subtarget->hasFastFMAF32() || Subtarget->hasDLInsts();
7584 return Subtarget->hasFastFMAF32() && Subtarget->hasDLInsts();
7590 return Subtarget->has16BitInsts() &&
7608 F.getDenormalFPEnv());
7613 switch (Ty.getScalarSizeInBits()) {
7631 return Subtarget->hasMadMacF32Insts() &&
7634 return Subtarget->hasMadF16() &&
7645 if (Ty.getScalarSizeInBits() == 16)
7647 if (Ty.getScalarSizeInBits() == 32)
7662 F.getDenormalFPEnv());
7673 unsigned Opc =
Op.getOpcode();
7674 EVT VT =
Op.getValueType();
7686 LoOps.
append(TrailingOps.begin(), TrailingOps.end());
7687 HiOps.
append(TrailingOps.begin(), TrailingOps.end());
7700 [[maybe_unused]]
EVT VT =
Op.getValueType();
7702 assert((VT == MVT::v2i32 || VT == MVT::v4i32 || VT == MVT::v8i32 ||
7703 VT == MVT::v16i32) &&
7704 "Unexpected ValueType.");
7713 unsigned Opc =
Op.getOpcode();
7714 EVT VT =
Op.getValueType();
7723 DAG.
getNode(
Opc, SL, Lo0.getValueType(), Lo0, Lo1,
Op->getFlags());
7725 DAG.
getNode(
Opc, SL, Hi0.getValueType(), Hi0, Hi1,
Op->getFlags());
7732 unsigned Opc =
Op.getOpcode();
7733 EVT VT =
Op.getValueType();
7750 DAG.
getNode(
Opc, SL, ResVT.first, Lo0, Lo1, Lo2,
Op->getFlags());
7752 DAG.
getNode(
Opc, SL, ResVT.second, Hi0, Hi1, Hi2,
Op->getFlags());
7758 switch (
Op.getOpcode()) {
7762 return LowerBRCOND(
Op, DAG);
7764 return LowerRETURNADDR(
Op, DAG);
7766 return lowerFrameIndex(
Op, DAG);
7768 return LowerSPONENTRY(
Op, DAG);
7771 assert((!Result.getNode() || Result.getNode()->getNumValues() == 2) &&
7772 "Load should return a value and a chain");
7776 EVT VT =
Op.getValueType();
7778 return lowerFSQRTF32(
Op, DAG);
7780 return lowerFSQRTF64(
Op, DAG);
7785 return LowerTrig(
Op, DAG);
7787 return LowerSELECT(
Op, DAG);
7789 return LowerFDIV(
Op, DAG);
7791 return LowerFFREXP(
Op, DAG);
7793 return LowerATOMIC_CMP_SWAP(
Op, DAG);
7795 return LowerSTORE(
Op, DAG);
7799 return LowerGlobalAddress(MFI,
Op, DAG);
7804 return LowerExternalSymbol(
Op, DAG);
7806 return LowerINTRINSIC_WO_CHAIN(
Op, DAG);
7808 return LowerCONVERT_FROM_ARBITRARY_FP(
Op, DAG);
7810 return LowerCONVERT_TO_ARBITRARY_FP(
Op, DAG);
7812 return LowerINTRINSIC_W_CHAIN(
Op, DAG);
7814 return LowerINTRINSIC_VOID(
Op, DAG);
7816 return lowerADDRSPACECAST(
Op, DAG);
7818 return lowerINSERT_SUBVECTOR(
Op, DAG);
7820 return lowerINSERT_VECTOR_ELT(
Op, DAG);
7822 return lowerEXTRACT_VECTOR_ELT(
Op, DAG);
7824 return lowerVECTOR_SHUFFLE(
Op, DAG);
7826 return lowerSCALAR_TO_VECTOR(
Op, DAG);
7828 return lowerBUILD_VECTOR(
Op, DAG);
7831 return lowerFP_ROUND(
Op, DAG);
7833 return lowerTRAP(
Op, DAG);
7835 return lowerDEBUGTRAP(
Op, DAG);
7844 if (
Op.getValueType().isVector() &&
Op.getValueType() != MVT::v2i16 &&
7845 Op.getOperand(0).getValueType().getScalarType() == MVT::f32)
7849 if (
Op.getValueType() == MVT::bf16) {
7879 return lowerFMINNUM_FMAXNUM(
Op, DAG);
7882 return lowerFMINIMUMNUM_FMAXIMUMNUM(
Op, DAG);
7885 return lowerFLDEXP(
Op, DAG);
7890 if (Subtarget->hasVCvtPkIU16F32() &&
Op.getValueType() == MVT::i16 &&
7891 Op.getOperand(0).getValueType() == MVT::f32) {
7917 return lowerFCOPYSIGN(
Op, DAG);
7919 return lowerMUL(
Op, DAG);
7922 return lowerXMULO(
Op, DAG);
7925 return lowerXMUL_LOHI(
Op, DAG);
7946 return LowerINLINEASM(
Op, DAG);
7952static std::pair<SDValue, SDValue>
7982 EVT FittingLoadVT = LoadVT;
8014SDValue SITargetLowering::adjustLoadValueType(
unsigned Opcode,
MemSDNode *M,
8017 bool IsIntrinsic)
const {
8020 bool IsTFE =
M->getNumValues() == 3;
8021 bool Unpacked = Subtarget->hasUnpackedD16VMem();
8022 EVT LoadVT =
M->getValueType(0);
8024 EVT EquivLoadVT = LoadVT;
8041 SDVTList VTList = DAG.
getVTList(LoadDWordsVT, MVT::Other);
8043 Opcode,
DL, VTList,
Ops,
M->getMemoryVT(),
M->getMemOperand());
8051 SDVTList VTList = DAG.
getVTList(EquivLoadVT, MVT::Other);
8055 M->getMemoryVT(),
M->getMemOperand());
8066 EVT LoadVT =
M->getValueType(0);
8075 "unsupported sub-dword format buffer load",
DL.getDebugLoc()));
8079 assert(
M->getNumValues() == 2 ||
M->getNumValues() == 3);
8080 bool IsTFE =
M->getNumValues() == 3;
8082 if (IsD16 && IsTFE && !Subtarget->hasBufferTFEFormatD16()) {
8085 "TFE D16 format buffer load is not supported on this GPU",
8088 M->getOperand(0),
DL);
8091 unsigned Opc = IsD16 ? (IsTFE ? AMDGPUISD::BUFFER_LOAD_FORMAT_D16_TFE
8092 : AMDGPUISD::BUFFER_LOAD_FORMAT_D16)
8093 : IsFormat ? (IsTFE ? AMDGPUISD::BUFFER_LOAD_FORMAT_TFE
8094 : AMDGPUISD::BUFFER_LOAD_FORMAT)
8095 : IsTFE ? AMDGPUISD::BUFFER_LOAD_TFE
8096 : AMDGPUISD::BUFFER_LOAD;
8099 return adjustLoadValueType(
Opc, M, DAG,
Ops);
8103 return handleByteShortBufferLoads(DAG, LoadVT,
DL,
Ops,
M->getMemOperand(),
8107 return getMemIntrinsicNode(
Opc,
DL,
M->getVTList(),
Ops, IntVT,
8108 M->getMemOperand(), DAG);
8112 SDVTList VTList = IsTFE ? DAG.
getVTList(CastVT, MVT::i32, MVT::Other)
8115 M->getMemOperand(), DAG);
8125 EVT VT =
N->getValueType(0);
8149 Exec = AMDGPU::EXEC_LO;
8151 Exec = AMDGPU::EXEC;
8168 bool Signed = IntrinsicID == Intrinsic::amdgcn_sbfe;
8170 EVT VT =
Op.getValueType();
8175 if (VT != MVT::i32) {
8183 return DAG.
getNode(
Signed ? AMDGPUISD::BFE_I32 : AMDGPUISD::BFE_U32,
DL, VT,
8192 EVT VT =
N->getValueType(0);
8194 unsigned IID =
N->getConstantOperandVal(0);
8195 bool IsPermLane16 = IID == Intrinsic::amdgcn_permlane16 ||
8196 IID == Intrinsic::amdgcn_permlanex16;
8197 bool IsSetInactive = IID == Intrinsic::amdgcn_set_inactive ||
8198 IID == Intrinsic::amdgcn_set_inactive_chain_arg;
8199 bool IsPermlaneShuffle = IID == Intrinsic::amdgcn_permlane_bcast ||
8200 IID == Intrinsic::amdgcn_permlane_up ||
8201 IID == Intrinsic::amdgcn_permlane_down ||
8202 IID == Intrinsic::amdgcn_permlane_xor;
8207 unsigned SplitSize = 32;
8208 if (IID == Intrinsic::amdgcn_update_dpp && (ValSize % 64 == 0) &&
8209 ST->hasDPALU_DPP() &&
8217 case Intrinsic::amdgcn_permlane16:
8218 case Intrinsic::amdgcn_permlanex16:
8219 case Intrinsic::amdgcn_update_dpp:
8224 case Intrinsic::amdgcn_writelane:
8225 case Intrinsic::amdgcn_permlane_bcast:
8226 case Intrinsic::amdgcn_permlane_up:
8227 case Intrinsic::amdgcn_permlane_down:
8228 case Intrinsic::amdgcn_permlane_xor:
8231 case Intrinsic::amdgcn_readlane:
8232 case Intrinsic::amdgcn_set_inactive:
8233 case Intrinsic::amdgcn_set_inactive_chain_arg:
8234 case Intrinsic::amdgcn_mov_dpp8:
8237 case Intrinsic::amdgcn_readfirstlane:
8238 case Intrinsic::amdgcn_permlane64:
8248 if (
SDNode *GL =
N->getGluedNode()) {
8250 GL = GL->getOperand(0).getNode();
8260 if (IID == Intrinsic::amdgcn_readlane || IID == Intrinsic::amdgcn_writelane ||
8261 IID == Intrinsic::amdgcn_mov_dpp8 ||
8262 IID == Intrinsic::amdgcn_update_dpp || IsSetInactive || IsPermLane16 ||
8263 IsPermlaneShuffle) {
8264 Src1 =
N->getOperand(2);
8265 if (IID == Intrinsic::amdgcn_writelane ||
8266 IID == Intrinsic::amdgcn_update_dpp || IsPermLane16 ||
8268 Src2 =
N->getOperand(3);
8271 if (ValSize == SplitSize) {
8281 if (IID == Intrinsic::amdgcn_update_dpp || IsSetInactive || IsPermLane16) {
8286 if (IID == Intrinsic::amdgcn_writelane) {
8291 SDValue LaneOp = createLaneOp(Src0, Src1, Src2, MVT::i32);
8293 return IsFloat ? DAG.
getBitcast(VT, Trunc) : Trunc;
8296 if (ValSize % SplitSize != 0)
8300 EVT VT =
N->getValueType(0);
8304 unsigned NumOperands =
N->getNumOperands();
8306 SDNode *GL =
N->getGluedNode();
8311 for (
unsigned i = 0; i != NE; ++i) {
8312 for (
unsigned j = 0, e = GL ? NumOperands - 1 : NumOperands; j != e;
8314 SDValue Operand =
N->getOperand(j);
8344 if (SplitSize == 32) {
8346 return unrollLaneOp(LaneOp.
getNode());
8352 unsigned SubVecNumElt =
8356 SDValue Src0SubVec, Src1SubVec, Src2SubVec;
8357 for (
unsigned i = 0, EltIdx = 0; i < ValSize / SplitSize; i++) {
8361 if (IID == Intrinsic::amdgcn_update_dpp || IsSetInactive ||
8367 createLaneOp(Src0SubVec, Src1SubVec, Src2, SubVecVT));
8368 }
else if (IID == Intrinsic::amdgcn_writelane) {
8372 createLaneOp(Src0SubVec, Src1, Src2SubVec, SubVecVT));
8374 Pieces.
push_back(createLaneOp(Src0SubVec, Src1, Src2, SubVecVT));
8377 EltIdx += SubVecNumElt;
8391 if (IID == Intrinsic::amdgcn_update_dpp || IsSetInactive || IsPermLane16)
8394 if (IID == Intrinsic::amdgcn_writelane)
8397 SDValue LaneOp = createLaneOp(Src0, Src1, Src2, VecVT);
8404 EVT VT =
N->getValueType(0);
8422 auto MakeIntrinsic = [&DAG, &SL](
unsigned IID,
MVT RetVT,
8431 if (
TLI.getSubtarget()->supportsWaveWideBPermute()) {
8432 SDValue BPermute = MakeIntrinsic(Intrinsic::amdgcn_ds_bpermute, MVT::i32,
8433 {ShiftedIndex, ValueI32});
8443 SDValue WWMValue = MakeIntrinsic(Intrinsic::amdgcn_set_inactive, MVT::i32,
8444 {ValueI32, PoisonVal});
8445 SDValue WWMIndex = MakeIntrinsic(Intrinsic::amdgcn_set_inactive, MVT::i32,
8446 {ShiftedIndex, PoisonVal});
8449 MakeIntrinsic(Intrinsic::amdgcn_permlane64, MVT::i32, {WWMValue});
8452 SDValue BPermSameHalf = MakeIntrinsic(Intrinsic::amdgcn_ds_bpermute, MVT::i32,
8453 {WWMIndex, WWMValue});
8454 SDValue BPermOtherHalf = MakeIntrinsic(Intrinsic::amdgcn_ds_bpermute,
8455 MVT::i32, {WWMIndex, Swapped});
8457 MakeIntrinsic(Intrinsic::amdgcn_wwm, MVT::i32, {BPermOtherHalf});
8465 MakeIntrinsic(Intrinsic::amdgcn_mbcnt_lo, MVT::i32,
8473 DAG.
getSetCC(SL, MVT::i1, SameOrOtherHalf,
8483 switch (
N->getOpcode()) {
8500 unsigned IID =
N->getConstantOperandVal(0);
8502 case Intrinsic::amdgcn_wave_reduce_min:
8503 case Intrinsic::amdgcn_wave_reduce_umin:
8504 case Intrinsic::amdgcn_wave_reduce_max:
8505 case Intrinsic::amdgcn_wave_reduce_umax:
8506 case Intrinsic::amdgcn_wave_reduce_add:
8507 case Intrinsic::amdgcn_wave_reduce_sub:
8508 case Intrinsic::amdgcn_wave_reduce_and:
8509 case Intrinsic::amdgcn_wave_reduce_or:
8510 case Intrinsic::amdgcn_wave_reduce_xor: {
8511 EVT VT =
N->getValueType(0);
8515 bool NeedsSignExt = IID == Intrinsic::amdgcn_wave_reduce_min ||
8516 IID == Intrinsic::amdgcn_wave_reduce_max ||
8517 IID == Intrinsic::amdgcn_wave_reduce_add ||
8518 IID == Intrinsic::amdgcn_wave_reduce_sub;
8522 N->getOperand(0), ExtSrc,
N->getOperand(2));
8526 case Intrinsic::amdgcn_make_buffer_rsrc:
8527 Results.push_back(lowerPointerAsRsrcIntrin(
N, DAG));
8529 case Intrinsic::amdgcn_cvt_pkrtz: {
8534 DAG.
getNode(AMDGPUISD::CVT_PKRTZ_F16_F32, SL, MVT::i32, Src0, Src1);
8538 case Intrinsic::amdgcn_cvt_pknorm_i16:
8539 case Intrinsic::amdgcn_cvt_pknorm_u16:
8540 case Intrinsic::amdgcn_cvt_pk_i16:
8541 case Intrinsic::amdgcn_cvt_pk_u16: {
8547 if (IID == Intrinsic::amdgcn_cvt_pknorm_i16)
8548 Opcode = AMDGPUISD::CVT_PKNORM_I16_F32;
8549 else if (IID == Intrinsic::amdgcn_cvt_pknorm_u16)
8550 Opcode = AMDGPUISD::CVT_PKNORM_U16_F32;
8551 else if (IID == Intrinsic::amdgcn_cvt_pk_i16)
8552 Opcode = AMDGPUISD::CVT_PK_I16_I32;
8554 Opcode = AMDGPUISD::CVT_PK_U16_U32;
8556 EVT VT =
N->getValueType(0);
8565 case Intrinsic::amdgcn_s_buffer_load: {
8567 EVT VT =
Op.getValueType();
8569 Op.getOperand(1),
Op.getOperand(2),
8570 Op.getOperand(3), DAG));
8573 case Intrinsic::amdgcn_dead: {
8574 for (
unsigned I = 0, E =
N->getNumValues();
I < E; ++
I)
8582 if (
N->getConstantOperandVal(1) != Intrinsic::amdgcn_ptr_s_buffer_load &&
8583 N->getValueType(0).isSimple() &&
8584 SBufferLoadDiagnosticVTs[
N->getSimpleValueType(0).SimpleTy])
8589 for (
unsigned I = 0;
I < Res.getNumOperands();
I++) {
8590 Results.push_back(Res.getOperand(
I));
8593 for (
unsigned I = 0;
I <
N->getNumValues(); ++
I)
8594 Results.push_back(Res.getValue(
I));
8603 EVT VT =
N->getValueType(0);
8608 EVT SelectVT = NewVT;
8609 if (NewVT.
bitsLT(MVT::i32)) {
8612 SelectVT = MVT::i32;
8618 if (NewVT != SelectVT)
8624 if (
N->getValueType(0) != MVT::v2f16)
8636 if (
N->getValueType(0) != MVT::v2f16)
8648 if (
N->getValueType(0) != MVT::f16)
8663 if (U.get() !=
Value)
8666 if (U.getUser()->getOpcode() == Opcode)
8672unsigned SITargetLowering::isCFIntrinsic(
const SDNode *Intr)
const {
8675 case Intrinsic::amdgcn_if:
8676 return AMDGPUISD::IF;
8677 case Intrinsic::amdgcn_else:
8678 return AMDGPUISD::ELSE;
8679 case Intrinsic::amdgcn_loop:
8680 return AMDGPUISD::LOOP;
8681 case Intrinsic::amdgcn_end_cf:
8701 if (Subtarget->isAmdPalOS() || Subtarget->isMesa3DOS())
8725 assert(GVar->isDeclaration() &&
8726 "AS 3 & 13 GVs should be declaration here "
8727 "when object linking is enabled");
8742 SDNode *Intr = BRCOND.getOperand(1).getNode();
8759 Intr =
LHS.getNode();
8767 assert(BR &&
"brcond missing unconditional branch user");
8772 unsigned CFNode = isCFIntrinsic(Intr);
8790 Ops.push_back(Target);
8813 for (
unsigned i = 1, e = Intr->
getNumValues() - 1; i != e; ++i) {
8832 MVT VT =
Op.getSimpleValueType();
8835 if (
Op.getConstantOperandVal(0) != 0)
8839 const SIMachineFunctionInfo *
Info = MF.
getInfo<SIMachineFunctionInfo>();
8841 if (
Info->isEntryFunction())
8858 SIMachineFunctionInfo *MFI = MF.
getInfo<SIMachineFunctionInfo>();
8872 return Op.getValueType().bitsLE(VT)
8880 EVT DstVT =
Op.getValueType();
8887 unsigned Opc =
Op.getOpcode();
8898 bool IsStrict =
Op->isStrictFPOpcode();
8899 SDValue Src =
Op.getOperand(IsStrict ? 1 : 0);
8900 EVT SrcVT = Src.getValueType();
8901 EVT DstVT =
Op.getValueType();
8904 assert(Subtarget->hasCvtPkF16F32Inst() &&
"support v_cvt_pk_f16_f32");
8907 return SrcVT == MVT::v2f32 ?
Op : splitFP_ROUNDVectorOp(
Op, DAG);
8914 if (DstVT == MVT::f16) {
8919 if (!Subtarget->has16BitInsts()) {
8924 if (
Op->getFlags().hasApproximateFuncs()) {
8935 "custom lower FP_ROUND for f16 or bf16");
8936 assert(Subtarget->hasBF16ConversionInsts() &&
"f32 -> bf16 is legal");
8953 EVT VT =
Op.getValueType();
8955 const SIMachineFunctionInfo *
Info = MF.
getInfo<SIMachineFunctionInfo>();
8956 bool IsIEEEMode =
Info->getMode().IEEE;
8962 if (IsIEEEMode && !Subtarget->hasIEEEMinimumMaximumInsts())
8965 if (VT == MVT::v4f16 || VT == MVT::v8f16 || VT == MVT::v16f16 ||
8966 VT == MVT::v32f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16 ||
8967 VT == MVT::v16bf16 || VT == MVT::v32bf16 || VT == MVT::v4f64 ||
8968 VT == MVT::v8f64 || VT == MVT::v16f64 || VT == MVT::v32f64)
8974SITargetLowering::lowerFMINIMUMNUM_FMAXIMUMNUM(
SDValue Op,
8976 EVT VT =
Op.getValueType();
8978 const SIMachineFunctionInfo *
Info = MF.
getInfo<SIMachineFunctionInfo>();
8979 bool IsIEEEMode =
Info->getMode().IEEE;
8981 if (IsIEEEMode && !Subtarget->hasIEEEMinimumMaximumInsts())
8984 if (VT == MVT::v4f16 || VT == MVT::v8f16 || VT == MVT::v16f16 ||
8985 VT == MVT::v32f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16 ||
8986 VT == MVT::v16bf16 || VT == MVT::v32bf16 || VT == MVT::v4f64 ||
8987 VT == MVT::v8f64 || VT == MVT::v16f64 || VT == MVT::v32f64)
8994 EVT VT =
Op.getValueType();
8998 EVT ExpVT =
Exp.getValueType();
8999 if (ExpVT == MVT::i16)
9020 {
Op.getOperand(0),
Op.getOperand(1), TruncExp});
9027 switch (
Op->getOpcode()) {
9060SITargetLowering::promoteUniformUnaryOpToI32(
SDValue Op,
9061 DAGCombinerInfo &DCI)
const {
9062 EVT OpTy =
Op.getValueType();
9063 SelectionDAG &DAG = DCI.DAG;
9080 DAGCombinerInfo &DCI)
const {
9081 const unsigned Opc =
Op.getOpcode();
9090 :
Op->getOperand(0).getValueType();
9091 auto &DAG = DCI.DAG;
9094 if (DCI.isBeforeLegalizeOps() ||
9102 LHS =
Op->getOperand(1);
9103 RHS =
Op->getOperand(2);
9105 LHS =
Op->getOperand(0);
9106 RHS =
Op->getOperand(1);
9148 EVT SignVT =
Sign.getValueType();
9150 if (MagVT == SignVT)
9171 EVT VT =
Op.getValueType();
9177 assert(VT == MVT::i64 &&
"The following code is a special for s_mul_u64");
9204 if (
Op->isDivergent())
9217 if (Op0LeadingZeros >= 32 && Op1LeadingZeros >= 32)
9219 DAG.
getMachineNode(AMDGPU::S_MUL_U64_U32_PSEUDO, SL, VT, Op0, Op1), 0);
9222 if (Op0SignBits >= 33 && Op1SignBits >= 33)
9224 DAG.
getMachineNode(AMDGPU::S_MUL_I64_I32_PSEUDO, SL, VT, Op0, Op1), 0);
9230 EVT VT =
Op.getValueType();
9237 const APInt &
C = RHSC->getAPIntValue();
9239 if (
C.isPowerOf2()) {
9241 bool UseArithShift =
isSigned && !
C.isMinSignedValue();
9268 if (
Op->isDivergent()) {
9272 if (Subtarget->hasSMulHi()) {
9283 if (!Subtarget->hasTrapHandler() ||
9285 return lowerTrapEndpgm(
Op, DAG);
9287 return Subtarget->supportsGetDoorbellID() ? lowerTrapHsa(
Op, DAG)
9288 : lowerTrapHsaQueuePtr(
Op, DAG);
9294 return DAG.
getNode(AMDGPUISD::ENDPGM_TRAP, SL, MVT::Other, Chain);
9298SITargetLowering::loadImplicitKernelArgument(
SelectionDAG &DAG,
MVT VT,
9300 ImplicitParameter Param)
const {
9304 MachinePointerInfo PtrInfo =
9321 loadImplicitKernelArgument(DAG, MVT::i64, SL,
Align(8),
QUEUE_PTR);
9324 SIMachineFunctionInfo *
Info = MF.
getInfo<SIMachineFunctionInfo>();
9327 if (UserSGPR == AMDGPU::NoRegister) {
9344 return DAG.
getNode(AMDGPUISD::TRAP, SL, MVT::Other,
Ops);
9353 if (Subtarget->hasPrivEnabledTrap2NopBug())
9354 return DAG.
getNode(AMDGPUISD::SIMULATED_TRAP, SL, MVT::Other, Chain);
9358 return DAG.
getNode(AMDGPUISD::TRAP, SL, MVT::Other,
Ops);
9366 if (!Subtarget->hasTrapHandler() ||
9370 "debugtrap handler not supported",
9378 return DAG.
getNode(AMDGPUISD::TRAP, SL, MVT::Other,
Ops);
9388 const SIRegisterInfo *
TRI = Subtarget->getRegisterInfo();
9389 SmallSet<Register, 8> SGPRInputRegs;
9391 unsigned NumVals = 0;
9394 const InlineAsm::Flag
Flags(
Op.getConstantOperandVal(
I));
9395 NumVals =
Flags.getNumOperandRegisters();
9399 NumVals > 0 &&
Flags.hasRegClassConstraint(RCID) &&
9400 TRI->isSGPRClass(
TRI->getRegClass(RCID));
9402 for (
unsigned J = 0; J < NumVals; ++J) {
9404 if (
const RegisterSDNode *RegNode =
9413 if (SGPRInputRegs.
empty())
9418 SDNode *
N =
Op.getOperand(
NumOps - 1).getNode();
9430 ReadFirstLaneID, SrcVal);
9434 if (
N->getNumOperands() > 3)
9435 Ops.push_back(
N->getOperand(3));
9441 SDNode *
Next =
nullptr;
9442 for (
unsigned I = 0,
E =
N->getNumOperands();
I !=
E; ++
I) {
9443 if (
N->getOperand(
I).getValueType() == MVT::Glue) {
9444 Next =
N->getOperand(
I).getNode();
9454SDValue SITargetLowering::getSegmentAperture(
unsigned AS,
const SDLoc &
DL,
9456 unsigned BaseAS = AS;
9461 SDValue Aperture = getBaseSegmentAperture(BaseAS,
DL, DAG);
9471SDValue SITargetLowering::getBaseSegmentAperture(
unsigned AS,
const SDLoc &
DL,
9475 if (Subtarget->hasApertureRegs()) {
9476 const unsigned ApertureRegNo =
9477 IsLDS ? AMDGPU::SRC_SHARED_BASE : AMDGPU::SRC_PRIVATE_BASE;
9478 assert((ApertureRegNo != AMDGPU::SRC_PRIVATE_BASE ||
9479 !Subtarget->hasGloballyAddressableScratch()) &&
9480 "Cannot use src_private_base with globally addressable scratch!");
9500 return loadImplicitKernelArgument(DAG, MVT::i32,
DL,
Align(4), Param);
9504 SIMachineFunctionInfo *
Info = MF.
getInfo<SIMachineFunctionInfo>();
9506 if (UserSGPR == AMDGPU::NoRegister) {
9517 uint32_t StructOffset = IsLDS ? 0x40 : 0x44;
9551 const AMDGPUTargetMachine &TM =
9555 unsigned SrcAS = ASC->getSrcAddressSpace();
9556 SDValue Src = ASC->getOperand(0);
9557 unsigned DestAS = ASC->getDestAddressSpace();
9558 bool IsNonNull = ASC->getFlags().hasNonNull();
9570 Subtarget->hasGloballyAddressableScratch()) {
9575 AMDGPU::S_MOV_B32, SL, MVT::i32,
9576 DAG.
getRegister(AMDGPU::SRC_FLAT_SCRATCH_BASE_LO, MVT::i32)),
9600 Subtarget->hasGloballyAddressableScratch()) {
9609 if (Subtarget->isWave64())
9615 57 - 32 - Subtarget->getWavefrontSizeLog2(), MVT::i32, SL);
9623 AMDGPU::S_MOV_B64, SL, MVT::i64,
9624 DAG.
getRegister(AMDGPU::SRC_FLAT_SCRATCH_BASE, MVT::i64)),
9626 CvtPtr = DAG.
getNode(
ISD::ADD, SL, MVT::i64, CvtPtr, FlatScratchBase);
9628 SDValue Aperture = getSegmentAperture(SrcAS, SL, DAG);
9649 Op.getValueType() == MVT::i64) {
9650 const SIMachineFunctionInfo *
Info =
9652 if (
Info->get32BitAddressHighBits() == 0)
9661 Src.getValueType() == MVT::i64)
9684 unsigned IdxVal =
Idx->getAsZExtVal();
9689 assert(InsNumElts % 2 == 0 &&
"expect legal vector types");
9694 EVT NewInsVT = InsNumElts == 2 ? MVT::i32
9696 MVT::i32, InsNumElts / 2);
9701 for (
unsigned I = 0;
I != InsNumElts / 2; ++
I) {
9703 if (InsNumElts == 2) {
9716 for (
unsigned I = 0;
I != InsNumElts; ++
I) {
9739 if (NumElts == 4 && EltSize == 16 && KIdx) {
9750 unsigned Idx = KIdx->getZExtValue();
9751 bool InsertLo =
Idx < 2;
9755 DAG.
getConstant(InsertLo ? Idx : (Idx - 2), SL, MVT::i32));
9761 : DAG.getBuildVector(MVT::v2i32, SL, {LoHalf, InsHalf});
9775 unsigned PieceSize = std::min(VecSize, 64u);
9776 unsigned NumPieces =
VecSize / PieceSize;
9777 unsigned EltsPerPiece = PieceSize / EltSize;
9812 InsertIntoPiece(DAG.
getBitcast(IntVT, Vec), BFM));
9824 for (
unsigned I = 0;
I != NumPieces; ++
I) {
9830 Pieces.
push_back(InsertIntoPiece(Piece, Mask));
9840 EVT ResultVT =
Op.getValueType();
9853 if (
SDValue Combined = performExtractVectorEltCombine(
Op.getNode(), DCI))
9856 if (VecSize == 128 || VecSize == 256 || VecSize == 512) {
9860 if (VecSize == 128) {
9868 }
else if (VecSize == 256) {
9871 for (
unsigned P = 0;
P < 4; ++
P) {
9877 Parts[0], Parts[1]));
9879 Parts[2], Parts[3]));
9885 for (
unsigned P = 0;
P < 8; ++
P) {
9892 Parts[0], Parts[1], Parts[2], Parts[3]));
9895 Parts[4], Parts[5], Parts[6], Parts[7]));
9898 EVT IdxVT =
Idx.getValueType();
9915 Src = DAG.
getBitcast(Src.getValueType().changeTypeToInteger(), Src);
9930 if (ResultVT == MVT::f16 || ResultVT == MVT::bf16) {
9940 return Mask[Elt + 1] == Mask[Elt] + 1 && (Mask[Elt] % 2 == 0);
9945 return Mask[Elt] >= 0 && Mask[Elt + 1] >= 0 && (Mask[Elt] & 1) &&
9946 !(Mask[Elt + 1] & 1);
9952 EVT ResultVT =
Op.getValueType();
9955 const int NewSrcNumElts = 2;
9957 int SrcNumElts =
Op.getOperand(0).getValueType().getVectorNumElements();
9973 const bool ShouldUseConsecutiveExtract = EltVT.
getSizeInBits() == 16;
9995 if (ShouldUseConsecutiveExtract &&
9998 int VecIdx =
Idx < SrcNumElts ? 0 : 1;
9999 int EltIdx =
Idx < SrcNumElts ?
Idx :
Idx - SrcNumElts;
10011 if (Idx0 >= SrcNumElts) {
10013 Idx0 -= SrcNumElts;
10016 if (Idx1 >= SrcNumElts) {
10018 Idx1 -= SrcNumElts;
10021 int AlignedIdx0 = Idx0 & ~(NewSrcNumElts - 1);
10022 int AlignedIdx1 = Idx1 & ~(NewSrcNumElts - 1);
10030 int NewMaskIdx0 = Idx0 - AlignedIdx0;
10031 int NewMaskIdx1 = Idx1 - AlignedIdx1;
10036 if (SubVec0 != SubVec1) {
10037 NewMaskIdx1 += NewSrcNumElts;
10044 {NewMaskIdx0, NewMaskIdx1});
10049 int VecIdx0 = Idx0 < SrcNumElts ? 0 : 1;
10050 int VecIdx1 = Idx1 < SrcNumElts ? 0 : 1;
10051 int EltIdx0 = Idx0 < SrcNumElts ? Idx0 : Idx0 - SrcNumElts;
10052 int EltIdx1 = Idx1 < SrcNumElts ? Idx1 : Idx1 - SrcNumElts;
10071 EVT ResultVT =
Op.getValueType();
10087 EVT VT =
Op.getValueType();
10089 if (VT == MVT::v2f16 || VT == MVT::v2i16 || VT == MVT::v2bf16) {
10090 assert(!Subtarget->hasVOP3PInsts() &&
"this should be legal");
10096 if (
Hi.isUndef()) {
10126 PartVT, SL, {
Op.getOperand(
P * 2),
Op.getOperand(
P * 2 + 1)});
10152 if (!Subtarget->isAmdHsaOS())
10195 return DAG.
getNode(AMDGPUISD::PC_ADD_REL_OFFSET64,
DL, PtrVT, Ptr);
10204 return DAG.
getNode(AMDGPUISD::PC_ADD_REL_OFFSET,
DL, PtrVT, PtrLo, PtrHi);
10212 EVT PtrVT =
Op.getValueType();
10214 const GlobalValue *GV = GSD->
getGlobal();
10227 assert(PtrVT == MVT::i32 &&
"32-bit pointer is expected.");
10248 return DAG.
getNode(AMDGPUISD::LDS,
DL, MVT::i32, GA);
10251 if (Subtarget->isAmdPalOS() || Subtarget->isMesa3DOS()) {
10252 if (Subtarget->has64BitLiterals()) {
10261 AddrLo = {DAG.
getMachineNode(AMDGPU::S_MOV_B32,
DL, MVT::i32, AddrLo), 0};
10265 AddrHi = {DAG.
getMachineNode(AMDGPU::S_MOV_B32,
DL, MVT::i32, AddrHi), 0};
10283 MachinePointerInfo PtrInfo =
10303 "unsupported alloca in VGPR 'as memory' address space (13)",
10313 Fn,
"unsupported external symbol",
Op.getDebugLoc()));
10335 unsigned Offset)
const {
10337 SDValue Param = lowerKernargMemParameter(
10348 "non-hsa intrinsic with hsa target",
DL.getDebugLoc()));
10356 "intrinsic not supported on subtarget",
DL.getDebugLoc()));
10364 unsigned NumElts = Elts.
size();
10366 if (NumElts <= 12) {
10370 Type = MVT::v16f32;
10375 for (
unsigned i = 0; i < Elts.
size(); ++i) {
10381 for (
unsigned i = Elts.
size(); i < NumElts; ++i)
10390 SDValue Src,
int ExtraElts) {
10391 EVT SrcVT = Src.getValueType();
10401 while (ExtraElts--)
10412 bool Unpacked,
bool IsD16,
int DMaskPop,
10413 int NumVDataDwords,
bool IsAtomicPacked16Bit,
10417 EVT ReqRetVT = ResultTypes[0];
10419 int NumDataDwords = ((IsD16 && !Unpacked) || IsAtomicPacked16Bit)
10420 ? (ReqRetNumElts + 1) / 2
10423 int MaskPopDwords = (!IsD16 || Unpacked) ? DMaskPop : (DMaskPop + 1) / 2;
10426 NumDataDwords == 1 ? MVT::i32 :
MVT::getVectorVT(MVT::i32, NumDataDwords);
10429 MaskPopDwords == 1 ? MVT::i32 :
MVT::getVectorVT(MVT::i32, MaskPopDwords);
10434 if (DMaskPop > 0 &&
Data.getValueType() != MaskPopVT) {
10438 SDValue(Result, 0), ZeroIdx);
10441 SDValue(Result, 0), ZeroIdx);
10445 if (DataDwordVT.
isVector() && !IsAtomicPacked16Bit)
10447 NumDataDwords - MaskPopDwords);
10452 EVT LegalReqRetVT = ReqRetVT;
10454 if (!
Data.getValueType().isInteger())
10456 Data.getValueType().changeTypeToInteger(),
Data);
10477 if (Result->getNumValues() == 1)
10484 SDValue *LWE,
bool &IsTexFail) {
10504 unsigned DimIdx,
unsigned EndIdx,
10505 unsigned NumGradients) {
10507 for (
unsigned I = DimIdx;
I < EndIdx;
I++) {
10515 if (((
I + 1) >= EndIdx) ||
10516 ((NumGradients / 2) % 2 == 1 && (
I == DimIdx + (NumGradients / 2) - 1 ||
10517 I == DimIdx + NumGradients - 1))) {
10549 !
Op.getNode()->hasAnyUseOfValue(0))
10551 const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode =
10562 ResultTypes.erase(&ResultTypes[0]);
10564 bool IsD16 =
false;
10565 bool IsG16 =
false;
10566 bool IsA16 =
false;
10568 int NumVDataDwords = 0;
10569 bool AdjustRetType =
false;
10570 bool IsAtomicPacked16Bit =
false;
10573 const unsigned ArgOffset = WithChain ? 2 : 1;
10576 unsigned DMaskLanes = 0;
10578 if (BaseOpcode->
Atomic) {
10579 VData =
Op.getOperand(2);
10581 IsAtomicPacked16Bit =
10582 (IntrOpcode == AMDGPU::IMAGE_ATOMIC_PK_ADD_F16 ||
10583 IntrOpcode == AMDGPU::IMAGE_ATOMIC_PK_ADD_F16_NORTN ||
10584 IntrOpcode == AMDGPU::IMAGE_ATOMIC_PK_ADD_BF16 ||
10585 IntrOpcode == AMDGPU::IMAGE_ATOMIC_PK_ADD_BF16_NORTN);
10590 "unsupported image atomic data type");
10602 ResultTypes[0] = Is64Bit ? MVT::v2i64 : MVT::v2i32;
10604 DMask = Is64Bit ? 0xf : 0x3;
10605 NumVDataDwords = Is64Bit ? 4 : 2;
10607 DMask = Is64Bit ? 0x3 : 0x1;
10608 NumVDataDwords = Is64Bit ? 2 : 1;
10611 DMask =
Op->getConstantOperandVal(ArgOffset + Intr->
DMaskIndex);
10614 if (BaseOpcode->
Store) {
10615 VData =
Op.getOperand(2);
10619 if (StoreScalarVT != MVT::f16 && StoreScalarVT.
getSizeInBits() != 32 &&
10622 "unsupported image store data type");
10624 if (StoreScalarVT == MVT::f16) {
10625 if (!Subtarget->hasD16Images() || !BaseOpcode->
HasD16)
10629 VData = handleD16VData(VData, DAG,
true);
10632 NumVDataDwords = (VData.
getValueType().getSizeInBits() + 31) / 32;
10633 }
else if (!BaseOpcode->
NoReturn) {
10638 if (LoadScalarVT != MVT::f16 && LoadScalarVT.
getSizeInBits() != 32 &&
10641 "unsupported image load data type");
10643 if (LoadScalarVT == MVT::f16) {
10644 if (!Subtarget->hasD16Images() || !BaseOpcode->
HasD16)
10652 (!LoadVT.
isVector() && DMaskLanes > 1))
10658 if (IsD16 && !Subtarget->hasUnpackedD16VMem() &&
10659 !(BaseOpcode->
Gather4 && Subtarget->hasImageGather4D16Bug()))
10660 NumVDataDwords = (DMaskLanes + 1) / 2;
10662 NumVDataDwords = DMaskLanes;
10664 AdjustRetType =
true;
10668 unsigned VAddrEnd = ArgOffset + Intr->
VAddrEnd;
10675 MVT GradPackVectorVT = VAddrScalarVT == MVT::f16 ? MVT::v2f16 : MVT::v2i16;
10676 IsG16 = VAddrScalarVT == MVT::f16 || VAddrScalarVT == MVT::i16;
10678 VAddrVT =
Op.getOperand(ArgOffset + Intr->
CoordStart).getSimpleValueType();
10680 MVT AddrPackVectorVT = VAddrScalarVT == MVT::f16 ? MVT::v2f16 : MVT::v2i16;
10681 IsA16 = VAddrScalarVT == MVT::f16 || VAddrScalarVT == MVT::i16;
10685 if (IsA16 && (
Op.getOperand(ArgOffset +
I).getValueType() == MVT::f16)) {
10691 {
Op.getOperand(ArgOffset +
I), DAG.
getPOISON(MVT::f16)});
10695 "Bias needs to be converted to 16 bit in A16 mode");
10700 if (BaseOpcode->
Gradients && !
ST->hasG16() && (IsA16 != IsG16)) {
10704 dbgs() <<
"Failed to lower image intrinsic: 16 bit addresses "
10705 "require 16 bit args for both gradients and addresses");
10710 if (!
ST->hasA16()) {
10711 LLVM_DEBUG(
dbgs() <<
"Failed to lower image intrinsic: Target does not "
10712 "support 16 bit addresses\n");
10722 if (BaseOpcode->
Gradients && IsG16 &&
ST->hasG16()) {
10724 const AMDGPU::MIMGG16MappingInfo *G16MappingInfo =
10726 IntrOpcode = G16MappingInfo->
G16;
10749 for (
unsigned I = ArgOffset + Intr->
CoordStart;
I < VAddrEnd;
I++)
10767 const unsigned NSAMaxSize =
ST->getNSAMaxSize(BaseOpcode->
Sampler);
10768 const bool HasPartialNSAEncoding =
ST->hasPartialNSAEncoding();
10769 const bool UseNSA =
ST->hasNSAEncoding() &&
10770 VAddrs.
size() >=
ST->getNSAThreshold(MF) &&
10771 (VAddrs.
size() <= NSAMaxSize || HasPartialNSAEncoding);
10772 const bool UsePartialNSA =
10773 UseNSA && HasPartialNSAEncoding && VAddrs.
size() > NSAMaxSize;
10776 if (UsePartialNSA) {
10778 ArrayRef(VAddrs).drop_front(NSAMaxSize - 1));
10779 }
else if (!UseNSA) {
10790 Op.getConstantOperandVal(ArgOffset + Intr->
UnormIndex);
10798 bool IsTexFail =
false;
10799 if (!
parseTexFail(TexFail, DAG, &TFE, &LWE, IsTexFail))
10808 NumVDataDwords = 1;
10810 NumVDataDwords += 1;
10811 AdjustRetType =
true;
10816 if (AdjustRetType) {
10819 if (DMaskLanes == 0 && !BaseOpcode->
Store) {
10828 MVT::i32, NumVDataDwords)
10831 ResultTypes[0] = NewVT;
10832 if (ResultTypes.size() == 3) {
10836 ResultTypes.erase(&ResultTypes[1]);
10850 Ops.push_back(VData);
10851 if (UsePartialNSA) {
10853 Ops.push_back(VAddr);
10857 Ops.push_back(VAddr);
10860 bool IndexedRsrc = IsGFX13 && RsrcVT == MVT::i32;
10861 if (RsrcVT != MVT::v4i32 && RsrcVT != MVT::v8i32 && !IndexedRsrc)
10863 Ops.push_back(Rsrc);
10864 bool IsSampIndexed =
false;
10868 IsSampIndexed = IsGFX13 && SampVT == MVT::i32;
10869 if (SampVT != MVT::v4i32 && !IsSampIndexed)
10871 Ops.push_back(Samp);
10876 if (!IsGFX12Plus || BaseOpcode->
Sampler || BaseOpcode->
MSAA)
10877 Ops.push_back(Unorm);
10879 Ops.push_back(IsA16 &&
10880 ST->hasFeature(AMDGPU::FeatureR128A16)
10886 if (!Subtarget->hasGFX90AInsts())
10887 Ops.push_back(TFE);
10891 "TFE is not supported on this GPU",
DL.getDebugLoc()));
10894 if (!IsGFX12Plus || BaseOpcode->
Sampler || BaseOpcode->
MSAA)
10895 Ops.push_back(LWE);
10901 Ops.push_back(
Op.getOperand(0));
10903 int NumVAddrDwords =
10910 NumVAddrDwords, IndexedRsrc, IsSampIndexed);
10911 }
else if (IsGFX12Plus) {
10913 NumVDataDwords, NumVAddrDwords);
10914 }
else if (IsGFX11Plus) {
10916 UseNSA ? AMDGPU::MIMGEncGfx11NSA
10917 : AMDGPU::MIMGEncGfx11Default,
10918 NumVDataDwords, NumVAddrDwords);
10919 }
else if (IsGFX10Plus) {
10921 UseNSA ? AMDGPU::MIMGEncGfx10NSA
10922 : AMDGPU::MIMGEncGfx10Default,
10923 NumVDataDwords, NumVAddrDwords);
10925 if (Subtarget->hasGFX90AInsts()) {
10927 NumVDataDwords, NumVAddrDwords);
10928 if (Opcode == -1) {
10930 DAG,
Op, OrigResultTypes,
DL,
10931 "requested image instruction is not supported on this GPU");
10934 if (Opcode == -1 &&
10937 NumVDataDwords, NumVAddrDwords);
10940 NumVDataDwords, NumVAddrDwords);
10947 MachineMemOperand *MemRef = MemOp->getMemOperand();
10966 Subtarget->hasUnpackedD16VMem(), IsD16, DMaskLanes,
10967 NumVDataDwords, IsAtomicPacked16Bit,
DL);
10976 bool HasChainResult = MMO !=
nullptr;
10980 bool IsSubwordLoad = (MemVT == MVT::i8 || MemVT == MVT::i16) &&
10981 Subtarget->hasScalarSubwordLoads();
10984 MF.
getFunction(),
"unsupported s_buffer_load result type",
10985 DL.getDebugLoc()));
10986 EVT ResultTypes[] = {VT, MVT::Other};
10988 ArrayRef(ResultTypes, HasChainResult ? 2 : 1), Chain,
DL);
10991 if (!HasChainResult) {
11003 if (!
Offset->isDivergent()) {
11011 auto HandleScalarSubwordLoads = [&](
unsigned Opcode) ->
SDValue {
11013 Opcode,
DL, DAG.
getVTList(MVT::i32, MVT::Other),
Ops, MemVT, MMO);
11016 if (HasChainResult)
11020 if (MemVT == MVT::i8 && Subtarget->hasScalarSubwordLoads())
11021 return HandleScalarSubwordLoads(AMDGPUISD::SBUFFER_LOAD_UBYTE);
11023 if (MemVT == MVT::i16 && Subtarget->hasScalarSubwordLoads())
11024 return HandleScalarSubwordLoads(AMDGPUISD::SBUFFER_LOAD_USHORT);
11029 !Subtarget->hasScalarDwordx3Loads()) {
11033 AMDGPUISD::SBUFFER_LOAD,
DL, DAG.
getVTList(WidenedVT, MVT::Other),
11038 if (HasChainResult)
11060 if ((MemVT == MVT::i8 || MemVT == MVT::i16) &&
11061 Subtarget->hasScalarSubwordLoads()) {
11065 if (HasChainResult)
11071 unsigned NumLoads = 1;
11077 if (NumElts == 8 || NumElts == 16) {
11078 NumLoads = NumElts / 4;
11082 SDVTList VTList = DAG.
getVTList({LoadVT, MVT::Other});
11087 NumLoads > 1 ?
Align(16 * NumLoads) :
Align(4));
11091 for (
unsigned i = 0; i < NumLoads; ++i) {
11094 Loads.
push_back(getMemIntrinsicNode(AMDGPUISD::BUFFER_LOAD,
DL, VTList,
Ops,
11095 LoadVT, LoadMMO, DAG));
11098 if (NumElts == 8 || NumElts == 16) {
11100 if (HasChainResult) {
11115 if (!Subtarget->hasArchitectedSGPRs())
11120 return DAG.
getNode(AMDGPUISD::BFE_U32, SL, VT, TTMP8,
11127 unsigned Width)
const {
11129 using namespace AMDGPU::Hwreg;
11131 AMDGPU::S_GETREG_B32_const, SL, MVT::i32,
11171 EVT DstVT =
Op.getValueType();
11173 assert((!IsF16 || Subtarget->hasFP8F16ConversionInsts()) &&
11174 "fp8/bf8 -> f16 conversion requires FP8F16ConversionInsts");
11178 Opc = IsBF8 ? AMDGPUISD::CVT_PK_F16_BF8 : AMDGPUISD::CVT_PK_F16_FP8;
11180 Opc = IsBF8 ? AMDGPUISD::CVT_PK_F32_BF8 : AMDGPUISD::CVT_PK_F32_FP8;
11193SITargetLowering::LowerCONVERT_FROM_ARBITRARY_FP(
SDValue Op,
11202 const bool HasE5M3ConversionInsts =
11203 Subtarget->hasFP8ConversionInsts() && Subtarget->hasFP8E5M3Insts();
11204 const bool IsSupported = IsFP8 || IsBF8 || (IsE5M3 && HasE5M3ConversionInsts);
11208 EVT DstVT =
Op.getValueType();
11212 !Subtarget->hasFP8F16ConversionInsts())
11222 "only the v2f32 vector result is custom lowered");
11228 auto ConvertByte = [&](
unsigned ByteSel) {
11229 return DAG.
getNode(AMDGPUISD::CVT_F32_FP8_E5M3, SL, MVT::f32, Src,
11234 return ConvertByte(0);
11235 return DAG.
getBuildVector(DstVT, SL, {ConvertByte(0), ConvertByte(1)});
11240 if (Src.getValueType() != MVT::i32) {
11250 if (EltVT == MVT::f16 || EltVT == MVT::f32)
11251 return lowerFromFP8(
Op, IsBF8, DAG);
11259 EVT ResVT =
Op.getValueType();
11260 bool IsF16 = Src.getValueType().getScalarType() == MVT::f16;
11261 assert((!IsF16 || Subtarget->hasF16FP8ConversionInsts()) &&
11262 "f16 -> fp8/bf8 conversion requires F16FP8ConversionInsts");
11264 "only the v2i8 vector result is custom lowered");
11268 IsBF8 ? AMDGPUISD::CVT_PK_BF8_F16 : AMDGPUISD::CVT_PK_FP8_F16;
11273 unsigned Opc = IsBF8 ? AMDGPUISD::CVT_PK_BF8_F32
11274 : IsE5M3 ? AMDGPUISD::CVT_PK_FP8_F32_E5M3
11275 : AMDGPUISD::CVT_PK_FP8_F32;
11283 DAG.
getNode(
Opc, SL, MVT::i32, Src, Src, PoisonI32, WordSel);
11295SITargetLowering::LowerCONVERT_TO_ARBITRARY_FP(
SDValue Op,
11304 const bool HasE5M3ConversionInsts =
11305 Subtarget->hasFP8ConversionInsts() && Subtarget->hasFP8E5M3Insts();
11306 const bool IsSupported = IsFP8 || IsBF8 || (IsE5M3 && HasE5M3ConversionInsts);
11316 if (!IsE5M3 &&
Op.getConstantOperandVal(3) != 0)
11319 EVT SrcEltVT =
Op.getOperand(0).getValueType().getScalarType();
11322 if (SrcEltVT == MVT::f32)
11323 return lowerToFP8(
Op, IsBF8, IsE5M3, DAG);
11324 if (!IsE5M3 && SrcEltVT == MVT::f16 &&
11325 Subtarget->hasF16FP8ConversionInsts()) {
11328 if (!
Op.getValueType().isVector())
11330 return lowerToFP8(
Op, IsBF8,
false, DAG);
11338 auto *MFI = MF.
getInfo<SIMachineFunctionInfo>();
11340 EVT VT =
Op.getValueType();
11342 unsigned IntrinsicID =
Op.getConstantOperandVal(0);
11346 switch (IntrinsicID) {
11347 case Intrinsic::amdgcn_wave_reduce_min:
11348 case Intrinsic::amdgcn_wave_reduce_umin:
11349 case Intrinsic::amdgcn_wave_reduce_fmin:
11350 case Intrinsic::amdgcn_wave_reduce_max:
11351 case Intrinsic::amdgcn_wave_reduce_umax:
11352 case Intrinsic::amdgcn_wave_reduce_fmax:
11353 case Intrinsic::amdgcn_wave_reduce_add:
11354 case Intrinsic::amdgcn_wave_reduce_fadd:
11355 case Intrinsic::amdgcn_wave_reduce_sub:
11356 case Intrinsic::amdgcn_wave_reduce_fsub:
11357 case Intrinsic::amdgcn_wave_reduce_and:
11358 case Intrinsic::amdgcn_wave_reduce_or:
11359 case Intrinsic::amdgcn_wave_reduce_xor: {
11360 EVT SrcVT =
Op.getOperand(1).getValueType();
11363 bool NeedsSignExt = IntrinsicID == Intrinsic::amdgcn_wave_reduce_min ||
11364 IntrinsicID == Intrinsic::amdgcn_wave_reduce_max ||
11365 IntrinsicID == Intrinsic::amdgcn_wave_reduce_add ||
11366 IntrinsicID == Intrinsic::amdgcn_wave_reduce_sub;
11370 auto SrcType = IsFPOp ? MVT::f16 : MVT::i16;
11371 auto ExtType = IsFPOp ? MVT::f32 : MVT::i32;
11375 Op.getOperand(0), ExtendedSrc, Strategy);
11384 case Intrinsic::amdgcn_implicit_buffer_ptr: {
11385 if (!MFI->getUserSGPRInfo().hasImplicitBufferPtr()) {
11386 const char *
Error =
11387 Subtarget->hasArchitectedFlatScratch()
11388 ?
"intrinsic not supported on subtarget"
11389 : !Subtarget->isMesa3DOS()
11390 ?
"intrinsic not supported for OS"
11391 :
"intrinsic not supported for calling convention";
11397 return getPreloadedValue(DAG, *MFI, VT,
11400 case Intrinsic::amdgcn_dispatch_ptr:
11401 case Intrinsic::amdgcn_queue_ptr: {
11402 if (!Subtarget->isAmdHsaOrMesa(MF.
getFunction())) {
11404 MF.
getFunction(),
"unsupported hsa intrinsic without hsa target",
11405 DL.getDebugLoc()));
11409 auto RegID = IntrinsicID == Intrinsic::amdgcn_dispatch_ptr
11412 return getPreloadedValue(DAG, *MFI, VT, RegID);
11414 case Intrinsic::amdgcn_implicitarg_ptr: {
11416 return getImplicitArgPtr(DAG,
DL);
11417 return getPreloadedValue(DAG, *MFI, VT,
11420 case Intrinsic::amdgcn_kernarg_segment_ptr: {
11426 return getPreloadedValue(DAG, *MFI, VT,
11429 case Intrinsic::amdgcn_dispatch_id: {
11432 case Intrinsic::amdgcn_rcp:
11433 return DAG.
getNode(AMDGPUISD::RCP,
DL, VT,
Op.getOperand(1));
11434 case Intrinsic::amdgcn_rsq:
11435 return DAG.
getNode(AMDGPUISD::RSQ,
DL, VT,
Op.getOperand(1));
11436 case Intrinsic::amdgcn_rsq_legacy:
11440 case Intrinsic::amdgcn_rcp_legacy:
11443 return DAG.
getNode(AMDGPUISD::RCP_LEGACY,
DL, VT,
Op.getOperand(1));
11444 case Intrinsic::amdgcn_fma_legacy:
11445 case Intrinsic::amdgcn_sudot4:
11446 case Intrinsic::amdgcn_sudot8:
11447 case Intrinsic::amdgcn_tanh:
11449 case Intrinsic::amdgcn_rsq_clamp: {
11451 return DAG.
getNode(AMDGPUISD::RSQ_CLAMP,
DL, VT,
Op.getOperand(1));
11463 case Intrinsic::r600_read_ngroups_x:
11464 if (Subtarget->isAmdHsaOS())
11467 return lowerKernargMemParameter(DAG, VT, VT,
DL, DAG.
getEntryNode(),
11470 case Intrinsic::r600_read_ngroups_y:
11471 if (Subtarget->isAmdHsaOS())
11474 return lowerKernargMemParameter(DAG, VT, VT,
DL, DAG.
getEntryNode(),
11477 case Intrinsic::r600_read_ngroups_z:
11478 if (Subtarget->isAmdHsaOS())
11481 return lowerKernargMemParameter(DAG, VT, VT,
DL, DAG.
getEntryNode(),
11484 case Intrinsic::r600_read_local_size_x:
11485 if (Subtarget->isAmdHsaOS())
11488 return lowerImplicitZextParam(DAG,
Op, MVT::i16,
11490 case Intrinsic::r600_read_local_size_y:
11491 if (Subtarget->isAmdHsaOS())
11494 return lowerImplicitZextParam(DAG,
Op, MVT::i16,
11496 case Intrinsic::r600_read_local_size_z:
11497 if (Subtarget->isAmdHsaOS())
11500 return lowerImplicitZextParam(DAG,
Op, MVT::i16,
11502 case Intrinsic::amdgcn_workgroup_id_x:
11503 return lowerWorkGroupId(DAG, *MFI, VT,
11507 case Intrinsic::amdgcn_workgroup_id_y:
11508 return lowerWorkGroupId(DAG, *MFI, VT,
11512 case Intrinsic::amdgcn_workgroup_id_z:
11513 return lowerWorkGroupId(DAG, *MFI, VT,
11517 case Intrinsic::amdgcn_cluster_id_x:
11518 return Subtarget->hasClusters()
11519 ? getPreloadedValue(DAG, *MFI, VT,
11521 : DAG.getPOISON(VT);
11522 case Intrinsic::amdgcn_cluster_id_y:
11523 return Subtarget->hasClusters()
11524 ? getPreloadedValue(DAG, *MFI, VT,
11527 case Intrinsic::amdgcn_cluster_id_z:
11528 return Subtarget->hasClusters()
11529 ? getPreloadedValue(DAG, *MFI, VT,
11532 case Intrinsic::amdgcn_cluster_workgroup_id_x:
11533 return Subtarget->hasClusters()
11534 ? getPreloadedValue(
11538 case Intrinsic::amdgcn_cluster_workgroup_id_y:
11539 return Subtarget->hasClusters()
11540 ? getPreloadedValue(
11544 case Intrinsic::amdgcn_cluster_workgroup_id_z:
11545 return Subtarget->hasClusters()
11546 ? getPreloadedValue(
11550 case Intrinsic::amdgcn_cluster_workgroup_flat_id:
11551 return Subtarget->hasClusters()
11554 case Intrinsic::amdgcn_cluster_workgroup_max_id_x:
11555 return Subtarget->hasClusters()
11556 ? getPreloadedValue(
11560 case Intrinsic::amdgcn_cluster_workgroup_max_id_y:
11561 return Subtarget->hasClusters()
11562 ? getPreloadedValue(
11566 case Intrinsic::amdgcn_cluster_workgroup_max_id_z:
11567 return Subtarget->hasClusters()
11568 ? getPreloadedValue(
11572 case Intrinsic::amdgcn_cluster_workgroup_max_flat_id:
11573 return Subtarget->hasClusters()
11574 ? getPreloadedValue(
11578 case Intrinsic::amdgcn_wave_id:
11579 return lowerWaveID(DAG,
Op);
11580 case Intrinsic::amdgcn_lds_kernel_id: {
11582 return getLDSKernelId(DAG,
DL);
11583 return getPreloadedValue(DAG, *MFI, VT,
11586 case Intrinsic::amdgcn_workitem_id_x:
11587 return lowerWorkitemID(DAG,
Op, 0, MFI->getArgInfo().WorkItemIDX);
11588 case Intrinsic::amdgcn_workitem_id_y:
11589 return lowerWorkitemID(DAG,
Op, 1, MFI->getArgInfo().WorkItemIDY);
11590 case Intrinsic::amdgcn_workitem_id_z:
11591 return lowerWorkitemID(DAG,
Op, 2, MFI->getArgInfo().WorkItemIDZ);
11592 case Intrinsic::amdgcn_wavefrontsize:
11594 SDLoc(
Op), MVT::i32);
11595 case Intrinsic::amdgcn_s_buffer_load: {
11596 unsigned CPol =
Op.getConstantOperandVal(3);
11604 Op.getOperand(2),
Op.getOperand(3), DAG);
11606 case Intrinsic::amdgcn_fdiv_fast:
11607 return lowerFDIV_FAST(
Op, DAG);
11608 case Intrinsic::amdgcn_sin:
11609 return DAG.
getNode(AMDGPUISD::SIN_HW,
DL, VT,
Op.getOperand(1));
11611 case Intrinsic::amdgcn_cos:
11612 return DAG.
getNode(AMDGPUISD::COS_HW,
DL, VT,
Op.getOperand(1));
11614 case Intrinsic::amdgcn_mul_u24:
11615 return DAG.
getNode(AMDGPUISD::MUL_U24,
DL, VT,
Op.getOperand(1),
11617 case Intrinsic::amdgcn_mul_i24:
11618 return DAG.
getNode(AMDGPUISD::MUL_I24,
DL, VT,
Op.getOperand(1),
11621 case Intrinsic::amdgcn_log_clamp: {
11627 case Intrinsic::amdgcn_fract:
11628 return DAG.
getNode(AMDGPUISD::FRACT,
DL, VT,
Op.getOperand(1));
11630 case Intrinsic::amdgcn_class: {
11632 EVT SrcVT = Src.getValueType();
11633 bool IsLegal = SrcVT == MVT::f32 || SrcVT == MVT::f64 ||
11634 (SrcVT == MVT::f16 && Subtarget->has16BitInsts());
11638 "llvm.amdgcn.class only supports f16, f32, and f64",
11639 DL.getDebugLoc()));
11642 return DAG.
getNode(AMDGPUISD::FP_CLASS,
DL, VT, Src,
Op.getOperand(2));
11644 case Intrinsic::amdgcn_div_fmas:
11645 return DAG.
getNode(AMDGPUISD::DIV_FMAS,
DL, VT,
Op.getOperand(1),
11646 Op.getOperand(2),
Op.getOperand(3),
Op.getOperand(4));
11648 case Intrinsic::amdgcn_div_fixup:
11649 return DAG.
getNode(AMDGPUISD::DIV_FIXUP,
DL, VT,
Op.getOperand(1),
11650 Op.getOperand(2),
Op.getOperand(3));
11652 case Intrinsic::amdgcn_div_scale: {
11658 SDValue Denominator =
Op.getOperand(2);
11667 return DAG.
getNode(AMDGPUISD::DIV_SCALE,
DL,
Op->getVTList(), Src0,
11668 Denominator, Numerator);
11670 case Intrinsic::amdgcn_ballot:
11672 case Intrinsic::amdgcn_fmed3:
11673 return DAG.
getNode(AMDGPUISD::FMED3,
DL, VT,
Op.getOperand(1),
11674 Op.getOperand(2),
Op.getOperand(3),
Op->getFlags());
11675 case Intrinsic::amdgcn_fdot2:
11676 return DAG.
getNode(AMDGPUISD::FDOT2,
DL, VT,
Op.getOperand(1),
11677 Op.getOperand(2),
Op.getOperand(3),
Op.getOperand(4));
11678 case Intrinsic::amdgcn_fmul_legacy:
11679 return DAG.
getNode(AMDGPUISD::FMUL_LEGACY,
DL, VT,
Op.getOperand(1),
11681 case Intrinsic::amdgcn_sbfe:
11682 case Intrinsic::amdgcn_ubfe:
11684 case Intrinsic::amdgcn_cvt_pkrtz:
11685 case Intrinsic::amdgcn_cvt_pknorm_i16:
11686 case Intrinsic::amdgcn_cvt_pknorm_u16:
11687 case Intrinsic::amdgcn_cvt_pk_i16:
11688 case Intrinsic::amdgcn_cvt_pk_u16: {
11690 EVT VT =
Op.getValueType();
11693 if (IntrinsicID == Intrinsic::amdgcn_cvt_pkrtz)
11694 Opcode = AMDGPUISD::CVT_PKRTZ_F16_F32;
11695 else if (IntrinsicID == Intrinsic::amdgcn_cvt_pknorm_i16)
11696 Opcode = AMDGPUISD::CVT_PKNORM_I16_F32;
11697 else if (IntrinsicID == Intrinsic::amdgcn_cvt_pknorm_u16)
11698 Opcode = AMDGPUISD::CVT_PKNORM_U16_F32;
11699 else if (IntrinsicID == Intrinsic::amdgcn_cvt_pk_i16)
11700 Opcode = AMDGPUISD::CVT_PK_I16_I32;
11702 Opcode = AMDGPUISD::CVT_PK_U16_U32;
11705 return DAG.
getNode(Opcode,
DL, VT,
Op.getOperand(1),
Op.getOperand(2));
11708 DAG.
getNode(Opcode,
DL, MVT::i32,
Op.getOperand(1),
Op.getOperand(2));
11711 case Intrinsic::amdgcn_fmad_ftz:
11712 return DAG.
getNode(AMDGPUISD::FMAD_FTZ,
DL, VT,
Op.getOperand(1),
11713 Op.getOperand(2),
Op.getOperand(3));
11715 case Intrinsic::amdgcn_if_break:
11717 Op->getOperand(1),
Op->getOperand(2)),
11720 case Intrinsic::amdgcn_groupstaticsize: {
11726 const GlobalValue *GV =
11732 case Intrinsic::amdgcn_is_shared:
11733 case Intrinsic::amdgcn_is_private: {
11740 unsigned AS = (IntrinsicID == Intrinsic::amdgcn_is_shared)
11744 Subtarget->hasGloballyAddressableScratch()) {
11747 AMDGPU::S_MOV_B32,
DL, MVT::i32,
11748 DAG.
getRegister(AMDGPU::SRC_FLAT_SCRATCH_BASE_HI, MVT::i32)),
11757 SDValue Aperture = getSegmentAperture(AS, SL, DAG);
11760 case Intrinsic::amdgcn_perm:
11761 return DAG.
getNode(AMDGPUISD::PERM,
DL, MVT::i32,
Op.getOperand(1),
11762 Op.getOperand(2),
Op.getOperand(3));
11763 case Intrinsic::amdgcn_reloc_constant: {
11773 case Intrinsic::amdgcn_swmmac_f16_16x16x32_f16:
11774 case Intrinsic::amdgcn_swmmac_bf16_16x16x32_bf16:
11775 case Intrinsic::amdgcn_swmmac_f32_16x16x32_bf16:
11776 case Intrinsic::amdgcn_swmmac_f32_16x16x32_f16:
11777 case Intrinsic::amdgcn_swmmac_f32_16x16x32_fp8_fp8:
11778 case Intrinsic::amdgcn_swmmac_f32_16x16x32_fp8_bf8:
11779 case Intrinsic::amdgcn_swmmac_f32_16x16x32_bf8_fp8:
11780 case Intrinsic::amdgcn_swmmac_f32_16x16x32_bf8_bf8: {
11781 if (
Op.getOperand(4).getValueType() == MVT::i32)
11787 Op.getOperand(0),
Op.getOperand(1),
Op.getOperand(2),
11788 Op.getOperand(3), IndexKeyi32);
11790 case Intrinsic::amdgcn_swmmac_f32_16x16x128_fp8_fp8:
11791 case Intrinsic::amdgcn_swmmac_f32_16x16x128_fp8_bf8:
11792 case Intrinsic::amdgcn_swmmac_f32_16x16x128_bf8_fp8:
11793 case Intrinsic::amdgcn_swmmac_f32_16x16x128_bf8_bf8:
11794 case Intrinsic::amdgcn_swmmac_f16_16x16x128_fp8_fp8:
11795 case Intrinsic::amdgcn_swmmac_f16_16x16x128_fp8_bf8:
11796 case Intrinsic::amdgcn_swmmac_f16_16x16x128_bf8_fp8:
11797 case Intrinsic::amdgcn_swmmac_f16_16x16x128_bf8_bf8: {
11798 if (
Op.getOperand(4).getValueType() == MVT::i64)
11803 Op.getOperand(4).getValueType() == MVT::v2i32
11807 {Op.getOperand(0), Op.getOperand(1), Op.getOperand(2),
11808 Op.getOperand(3), IndexKeyi64, Op.getOperand(5),
11809 Op.getOperand(6)});
11811 case Intrinsic::amdgcn_swmmac_f16_16x16x64_f16:
11812 case Intrinsic::amdgcn_swmmac_bf16_16x16x64_bf16:
11813 case Intrinsic::amdgcn_swmmac_f32_16x16x64_bf16:
11814 case Intrinsic::amdgcn_swmmac_bf16f32_16x16x64_bf16:
11815 case Intrinsic::amdgcn_swmmac_f32_16x16x64_f16:
11816 case Intrinsic::amdgcn_swmmac_i32_16x16x128_iu8: {
11817 EVT IndexKeyTy = IntrinsicID == Intrinsic::amdgcn_swmmac_i32_16x16x128_iu8
11820 if (
Op.getOperand(6).getValueType() == IndexKeyTy)
11825 Op.getOperand(6).getValueType().isVector()
11829 Op.getOperand(0),
Op.getOperand(1),
Op.getOperand(2),
11830 Op.getOperand(3),
Op.getOperand(4),
Op.getOperand(5),
11831 IndexKey,
Op.getOperand(7),
Op.getOperand(8)};
11832 if (IntrinsicID == Intrinsic::amdgcn_swmmac_i32_16x16x128_iu8)
11833 Args.push_back(
Op.getOperand(9));
11836 case Intrinsic::amdgcn_swmmac_i32_16x16x32_iu4:
11837 case Intrinsic::amdgcn_swmmac_i32_16x16x32_iu8:
11838 case Intrinsic::amdgcn_swmmac_i32_16x16x64_iu4: {
11839 if (
Op.getOperand(6).getValueType() == MVT::i32)
11845 {Op.getOperand(0), Op.getOperand(1), Op.getOperand(2),
11846 Op.getOperand(3), Op.getOperand(4), Op.getOperand(5),
11847 IndexKeyi32, Op.getOperand(7)});
11849 case Intrinsic::amdgcn_wmma_scale_f32_16x16x128_f8f6f4:
11850 case Intrinsic::amdgcn_wmma_scale16_f32_16x16x128_f8f6f4: {
11851 unsigned AFmt = (unsigned)
Op.getConstantOperandVal(1);
11852 unsigned BFmt = (unsigned)
Op.getConstantOperandVal(3);
11853 unsigned AScaleFmt = (unsigned)
Op.getConstantOperandVal(8);
11854 unsigned BScaleFmt = (unsigned)
Op.getConstantOperandVal(11);
11858 "invalid matrix and scale format combination in wmma call");
11864 case Intrinsic::amdgcn_readlane:
11865 case Intrinsic::amdgcn_readfirstlane:
11866 case Intrinsic::amdgcn_writelane:
11867 case Intrinsic::amdgcn_permlane16:
11868 case Intrinsic::amdgcn_permlanex16:
11869 case Intrinsic::amdgcn_permlane64:
11870 case Intrinsic::amdgcn_set_inactive:
11871 case Intrinsic::amdgcn_set_inactive_chain_arg:
11872 case Intrinsic::amdgcn_mov_dpp8:
11873 case Intrinsic::amdgcn_update_dpp:
11874 case Intrinsic::amdgcn_permlane_bcast:
11875 case Intrinsic::amdgcn_permlane_up:
11876 case Intrinsic::amdgcn_permlane_down:
11877 case Intrinsic::amdgcn_permlane_xor:
11879 case Intrinsic::amdgcn_dead: {
11881 for (
const EVT ValTy :
Op.getNode()->values())
11885 case Intrinsic::amdgcn_wave_shuffle:
11888 if (
const AMDGPU::ImageDimIntrinsicInfo *ImageDimIntr =
11890 return lowerImage(
Op, ImageDimIntr, DAG,
false);
11901 return DAG.
getRegister(AMDGPU::SGPR_NULL, MVT::i32);
11913 auto GetArg = [&](
unsigned Arg) {
11917 SDValue Rsrc = bufferRsrcPtrToVector(GetArg(
Info.RsrcArg), DAG);
11918 auto [VOffset,
Offset] =
11919 splitBufferOffsets(GetArg(*
Info.getVOffsetArg()), DAG);
11925 Ops.push_back(VData);
11926 if (std::optional<unsigned> CmpArg =
Info.getCmpArg())
11927 Ops.push_back(GetArg(*CmpArg));
11928 Ops.push_back(Rsrc);
11929 std::optional<unsigned> VIndexArg =
Info.getVIndexArg();
11930 Ops.push_back(VIndexArg ? GetArg(*VIndexArg)
11932 Ops.push_back(VOffset);
11933 Ops.push_back(SOffset);
11935 if (std::optional<unsigned> FormatArg =
Info.getFormatArg())
11936 Ops.push_back(GetArg(*FormatArg));
11937 Ops.push_back(GetArg(*
Info.getAuxArg()));
11944 unsigned NewOpcode)
const {
11946 const AMDGPU::BufferIntrinsicInfo &
Info =
11953 "unsupported buffer atomic data type");
11961 M->getMemOperand());
11967SDValue SITargetLowering::lowerBufferIntrinsicWithChain(
11972 auto GetArg = [&](
unsigned Arg) {
11976 switch (
Info.Kind) {
11977 case AMDGPU::BufferKindSLoad: {
11985 return lowerSBuffer(
11986 Op.getValueType(),
M->getMemoryVT(),
DL,
Op.getOperand(0),
11987 bufferRsrcPtrToVector(GetArg(
Info.RsrcArg), DAG),
11988 GetArg(*
Info.getSOffsetArg()), CachePolicy, DAG,
M->getMemOperand());
11990 case AMDGPU::BufferKindLoad:
11991 case AMDGPU::BufferKindAtomicLoad: {
11993 if (!
Info.isTyped())
11994 return lowerIntrinsicLoad(M,
Info.isFormatFromRsrc(), DAG,
Ops);
11996 EVT LoadVT =
Op.getValueType();
11998 return adjustLoadValueType(AMDGPUISD::TBUFFER_LOAD_FORMAT_D16, M, DAG,
12000 return getMemIntrinsicNode(AMDGPUISD::TBUFFER_LOAD_FORMAT,
DL,
12001 Op->getVTList(),
Ops, LoadVT,
M->getMemOperand(),
12004 case AMDGPU::BufferKindAtomicCmpSwap: {
12006 if (Src.getValueSizeInBits() != 32 && Src.getValueSizeInBits() != 64) {
12009 "unsupported buffer atomic data type");
12013 Op->getVTList(),
Ops,
Op.getValueType(),
12014 M->getMemOperand());
12025 unsigned NumOperands =
N->getNumOperands();
12026 if (
N->getOperand(NumOperands - 1) ==
Zero)
12036static std::optional<uint32_t>
12038 std::optional<uint64_t> BarVal;
12040 BarVal =
C->getZExtValue();
12041 }
else if (BarOp->
isUndef()) {
12060 return *BarVal & 0x3F;
12061 return std::nullopt;
12066 unsigned IntrID =
Op.getConstantOperandVal(1);
12069 if (
const AMDGPU::BufferIntrinsicInfo *BufInfo =
12071 BufInfo && BufInfo->
Kind != AMDGPU::BufferKindAtomicRMW)
12072 return lowerBufferIntrinsicWithChain(
Op, *BufInfo, DAG);
12075 case Intrinsic::amdgcn_cluster_load_b32:
12076 case Intrinsic::amdgcn_cluster_load_b64:
12077 case Intrinsic::amdgcn_cluster_load_b128: {
12078 if (Subtarget->hasGFX1250_STRICT())
12082 case Intrinsic::amdgcn_ds_ordered_add:
12083 case Intrinsic::amdgcn_ds_ordered_swap: {
12088 unsigned IndexOperand =
M->getConstantOperandVal(7);
12089 unsigned WaveRelease =
M->getConstantOperandVal(8);
12090 unsigned WaveDone =
M->getConstantOperandVal(9);
12092 unsigned OrderedCountIndex = IndexOperand & 0x3f;
12093 IndexOperand &= ~0x3f;
12094 unsigned CountDw = 0;
12097 CountDw = (IndexOperand >> 24) & 0xf;
12098 IndexOperand &= ~(0xf << 24);
12100 if (CountDw < 1 || CountDw > 4) {
12103 Fn,
"ds_ordered_count: dword count must be between 1 and 4",
12104 DL.getDebugLoc()));
12109 if (IndexOperand) {
12112 Fn,
"ds_ordered_count: bad index operand",
DL.getDebugLoc()));
12115 if (WaveDone && !WaveRelease) {
12119 Fn,
"ds_ordered_count: wave_done requires wave_release",
12120 DL.getDebugLoc()));
12123 unsigned Instruction = IntrID == Intrinsic::amdgcn_ds_ordered_add ? 0 : 1;
12124 unsigned ShaderType =
12126 unsigned Offset0 = OrderedCountIndex << 2;
12127 unsigned Offset1 = WaveRelease | (WaveDone << 1) | (Instruction << 4);
12130 Offset1 |= (CountDw - 1) << 6;
12133 Offset1 |= ShaderType << 2;
12135 unsigned Offset = Offset0 | (Offset1 << 8);
12142 M->getVTList(),
Ops,
M->getMemoryVT(),
12143 M->getMemOperand());
12145 case Intrinsic::amdgcn_raw_buffer_atomic_fadd:
12146 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_fadd:
12147 case Intrinsic::amdgcn_struct_buffer_atomic_fadd:
12148 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_fadd:
12149 return lowerBufferAtomicIntrin(
Op, DAG, AMDGPUISD::BUFFER_ATOMIC_FADD);
12150 case Intrinsic::amdgcn_raw_buffer_atomic_fmin:
12151 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_fmin:
12152 case Intrinsic::amdgcn_struct_buffer_atomic_fmin:
12153 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_fmin:
12154 return lowerBufferAtomicIntrin(
Op, DAG, AMDGPUISD::BUFFER_ATOMIC_FMIN);
12155 case Intrinsic::amdgcn_raw_buffer_atomic_fmax:
12156 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_fmax:
12157 case Intrinsic::amdgcn_struct_buffer_atomic_fmax:
12158 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_fmax:
12159 return lowerBufferAtomicIntrin(
Op, DAG, AMDGPUISD::BUFFER_ATOMIC_FMAX);
12160 case Intrinsic::amdgcn_raw_buffer_atomic_swap:
12161 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_swap:
12162 case Intrinsic::amdgcn_struct_buffer_atomic_swap:
12163 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_swap:
12164 return lowerBufferAtomicIntrin(
Op, DAG, AMDGPUISD::BUFFER_ATOMIC_SWAP);
12165 case Intrinsic::amdgcn_raw_buffer_atomic_add:
12166 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_add:
12167 case Intrinsic::amdgcn_struct_buffer_atomic_add:
12168 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_add:
12169 return lowerBufferAtomicIntrin(
Op, DAG, AMDGPUISD::BUFFER_ATOMIC_ADD);
12170 case Intrinsic::amdgcn_raw_buffer_atomic_sub:
12171 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_sub:
12172 case Intrinsic::amdgcn_struct_buffer_atomic_sub:
12173 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_sub:
12174 return lowerBufferAtomicIntrin(
Op, DAG, AMDGPUISD::BUFFER_ATOMIC_SUB);
12175 case Intrinsic::amdgcn_raw_buffer_atomic_smin:
12176 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_smin:
12177 case Intrinsic::amdgcn_struct_buffer_atomic_smin:
12178 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_smin:
12179 return lowerBufferAtomicIntrin(
Op, DAG, AMDGPUISD::BUFFER_ATOMIC_SMIN);
12180 case Intrinsic::amdgcn_raw_buffer_atomic_umin:
12181 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_umin:
12182 case Intrinsic::amdgcn_struct_buffer_atomic_umin:
12183 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_umin:
12184 return lowerBufferAtomicIntrin(
Op, DAG, AMDGPUISD::BUFFER_ATOMIC_UMIN);
12185 case Intrinsic::amdgcn_raw_buffer_atomic_smax:
12186 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_smax:
12187 case Intrinsic::amdgcn_struct_buffer_atomic_smax:
12188 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_smax:
12189 return lowerBufferAtomicIntrin(
Op, DAG, AMDGPUISD::BUFFER_ATOMIC_SMAX);
12190 case Intrinsic::amdgcn_raw_buffer_atomic_umax:
12191 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_umax:
12192 case Intrinsic::amdgcn_struct_buffer_atomic_umax:
12193 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_umax:
12194 return lowerBufferAtomicIntrin(
Op, DAG, AMDGPUISD::BUFFER_ATOMIC_UMAX);
12195 case Intrinsic::amdgcn_raw_buffer_atomic_and:
12196 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_and:
12197 case Intrinsic::amdgcn_struct_buffer_atomic_and:
12198 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_and:
12199 return lowerBufferAtomicIntrin(
Op, DAG, AMDGPUISD::BUFFER_ATOMIC_AND);
12200 case Intrinsic::amdgcn_raw_buffer_atomic_or:
12201 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_or:
12202 case Intrinsic::amdgcn_struct_buffer_atomic_or:
12203 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_or:
12204 return lowerBufferAtomicIntrin(
Op, DAG, AMDGPUISD::BUFFER_ATOMIC_OR);
12205 case Intrinsic::amdgcn_raw_buffer_atomic_xor:
12206 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_xor:
12207 case Intrinsic::amdgcn_struct_buffer_atomic_xor:
12208 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_xor:
12209 return lowerBufferAtomicIntrin(
Op, DAG, AMDGPUISD::BUFFER_ATOMIC_XOR);
12210 case Intrinsic::amdgcn_raw_buffer_atomic_inc:
12211 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_inc:
12212 case Intrinsic::amdgcn_struct_buffer_atomic_inc:
12213 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_inc:
12214 return lowerBufferAtomicIntrin(
Op, DAG, AMDGPUISD::BUFFER_ATOMIC_INC);
12215 case Intrinsic::amdgcn_raw_buffer_atomic_dec:
12216 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_dec:
12217 case Intrinsic::amdgcn_struct_buffer_atomic_dec:
12218 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_dec:
12219 return lowerBufferAtomicIntrin(
Op, DAG, AMDGPUISD::BUFFER_ATOMIC_DEC);
12220 case Intrinsic::amdgcn_raw_buffer_atomic_sub_clamp_u32:
12221 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_sub_clamp_u32:
12222 case Intrinsic::amdgcn_struct_buffer_atomic_sub_clamp_u32:
12223 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_sub_clamp_u32:
12224 return lowerBufferAtomicIntrin(
Op, DAG, AMDGPUISD::BUFFER_ATOMIC_CSUB);
12225 case Intrinsic::amdgcn_raw_buffer_atomic_cond_sub_u32:
12226 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_cond_sub_u32:
12227 case Intrinsic::amdgcn_struct_buffer_atomic_cond_sub_u32:
12228 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_cond_sub_u32:
12229 return lowerBufferAtomicIntrin(
Op, DAG,
12230 AMDGPUISD::BUFFER_ATOMIC_COND_SUB_U32);
12231 case Intrinsic::amdgcn_image_bvh_dual_intersect_ray:
12232 case Intrinsic::amdgcn_image_bvh8_intersect_ray: {
12234 SDValue NodePtr =
M->getOperand(2);
12235 SDValue RayExtent =
M->getOperand(3);
12236 SDValue InstanceMask =
M->getOperand(4);
12237 SDValue RayOrigin =
M->getOperand(5);
12238 SDValue RayDir =
M->getOperand(6);
12240 SDValue TDescr =
M->getOperand(8);
12245 bool IsBVH8 = IntrID == Intrinsic::amdgcn_image_bvh8_intersect_ray;
12246 const unsigned NumVDataDwords = 10;
12247 const unsigned NumVAddrDwords = IsBVH8 ? 11 : 12;
12249 IsBVH8 ? AMDGPU::IMAGE_BVH8_INTERSECT_RAY
12250 : AMDGPU::IMAGE_BVH_DUAL_INTERSECT_RAY,
12251 AMDGPU::MIMGEncGfx12, NumVDataDwords, NumVAddrDwords);
12255 Ops.push_back(NodePtr);
12258 {DAG.getBitcast(MVT::i32, RayExtent),
12259 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, InstanceMask)}));
12260 Ops.push_back(RayOrigin);
12261 Ops.push_back(RayDir);
12262 Ops.push_back(Offsets);
12263 Ops.push_back(TDescr);
12264 Ops.push_back(
M->getChain());
12267 MachineMemOperand *MemRef =
M->getMemOperand();
12271 case Intrinsic::amdgcn_image_bvh_intersect_ray: {
12273 SDValue NodePtr =
M->getOperand(2);
12274 SDValue RayExtent =
M->getOperand(3);
12275 SDValue RayOrigin =
M->getOperand(4);
12276 SDValue RayDir =
M->getOperand(5);
12277 SDValue RayInvDir =
M->getOperand(6);
12278 SDValue TDescr =
M->getOperand(7);
12290 const unsigned NumVDataDwords = 4;
12291 const unsigned NumVAddrDwords = IsA16 ? (Is64 ? 9 : 8) : (Is64 ? 12 : 11);
12292 const unsigned NumVAddrs = IsGFX11Plus ? (IsA16 ? 4 : 5) : NumVAddrDwords;
12293 const bool UseNSA = (Subtarget->hasNSAEncoding() &&
12296 const unsigned BaseOpcodes[2][2] = {
12297 {AMDGPU::IMAGE_BVH_INTERSECT_RAY, AMDGPU::IMAGE_BVH_INTERSECT_RAY_a16},
12298 {AMDGPU::IMAGE_BVH64_INTERSECT_RAY,
12299 AMDGPU::IMAGE_BVH64_INTERSECT_RAY_a16}};
12303 IsGFX12Plus ? AMDGPU::MIMGEncGfx12
12304 : IsGFX11 ? AMDGPU::MIMGEncGfx11NSA
12305 : AMDGPU::MIMGEncGfx10NSA,
12306 NumVDataDwords, NumVAddrDwords);
12310 IsGFX11 ? AMDGPU::MIMGEncGfx11Default
12311 : AMDGPU::MIMGEncGfx10Default,
12312 NumVDataDwords, NumVAddrDwords);
12318 auto packLanes = [&DAG, &
Ops, &
DL](
SDValue Op,
bool IsAligned) {
12321 if (Lanes[0].getValueSizeInBits() == 32) {
12322 for (
unsigned I = 0;
I < 3; ++
I)
12329 Ops.push_back(Lanes[2]);
12341 if (UseNSA && IsGFX11Plus) {
12342 Ops.push_back(NodePtr);
12344 Ops.push_back(RayOrigin);
12349 for (
unsigned I = 0;
I < 3; ++
I) {
12352 {DirLanes[I], InvDirLanes[I]})));
12356 Ops.push_back(RayDir);
12357 Ops.push_back(RayInvDir);
12364 Ops.push_back(NodePtr);
12367 packLanes(RayOrigin,
true);
12368 packLanes(RayDir,
true);
12369 packLanes(RayInvDir,
false);
12374 if (NumVAddrDwords > 12) {
12382 Ops.push_back(MergedOps);
12385 Ops.push_back(TDescr);
12387 Ops.push_back(
M->getChain());
12390 MachineMemOperand *MemRef =
M->getMemOperand();
12394 case Intrinsic::amdgcn_global_atomic_fmin_num:
12395 case Intrinsic::amdgcn_global_atomic_fmax_num:
12396 case Intrinsic::amdgcn_flat_atomic_fmin_num:
12397 case Intrinsic::amdgcn_flat_atomic_fmax_num: {
12404 unsigned Opcode = 0;
12406 case Intrinsic::amdgcn_global_atomic_fmin_num:
12407 case Intrinsic::amdgcn_flat_atomic_fmin_num: {
12411 case Intrinsic::amdgcn_global_atomic_fmax_num:
12412 case Intrinsic::amdgcn_flat_atomic_fmax_num: {
12419 return DAG.
getAtomic(Opcode, SDLoc(
Op),
M->getMemoryVT(),
M->getVTList(),
12420 Ops,
M->getMemOperand());
12422 case Intrinsic::amdgcn_s_alloc_vgpr: {
12430 ReadFirstLaneID, NumVGPRs);
12433 Op.getOperand(0),
Op.getOperand(1), NumVGPRs);
12435 case Intrinsic::amdgcn_s_get_barrier_state:
12436 case Intrinsic::amdgcn_s_get_named_barrier_state: {
12441 std::optional<uint32_t> BarID;
12442 if (IntrID == Intrinsic::amdgcn_s_get_named_barrier_state)
12445 BarID =
C->getZExtValue();
12448 Opc = AMDGPU::S_GET_BARRIER_STATE_IMM;
12451 Ops.push_back(Chain);
12453 Opc = AMDGPU::S_GET_BARRIER_STATE_M0;
12454 if (IntrID == Intrinsic::amdgcn_s_get_named_barrier_state) {
12465 case Intrinsic::amdgcn_cooperative_atomic_load_32x4B:
12466 case Intrinsic::amdgcn_cooperative_atomic_load_16x8B:
12467 case Intrinsic::amdgcn_cooperative_atomic_load_8x16B: {
12471 EVT VT =
Op->getValueType(0);
12475 case Intrinsic::amdgcn_av_load_b8:
12476 case Intrinsic::amdgcn_av_load_b16:
12477 case Intrinsic::amdgcn_av_load_b32:
12478 case Intrinsic::amdgcn_av_load_b64:
12479 case Intrinsic::amdgcn_av_load_b128: {
12483 EVT VT =
Op->getValueType(0);
12490 case Intrinsic::amdgcn_flat_load_monitor_b32:
12491 case Intrinsic::amdgcn_flat_load_monitor_b64:
12492 case Intrinsic::amdgcn_flat_load_monitor_b128: {
12497 Op->getVTList(), {Chain, Ptr},
12500 case Intrinsic::amdgcn_global_load_monitor_b32:
12501 case Intrinsic::amdgcn_global_load_monitor_b64:
12502 case Intrinsic::amdgcn_global_load_monitor_b128: {
12507 Op->getVTList(), {Chain, Ptr},
12512 if (
const AMDGPU::ImageDimIntrinsicInfo *ImageDimIntr =
12514 return lowerImage(
Op, ImageDimIntr, DAG,
true);
12522SDValue SITargetLowering::getMemIntrinsicNode(
unsigned Opcode,
const SDLoc &
DL,
12529 EVT VT = VTList.
VTs[0];
12532 bool IsTFE = VTList.
NumVTs == 3;
12535 unsigned NumOpDWords = NumValueDWords + 1;
12537 SDVTList OpDWordsVTList = DAG.
getVTList(OpDWordsVT, VTList.
VTs[2]);
12538 MachineMemOperand *OpDWordsMMO =
12540 SDValue Op = getMemIntrinsicNode(Opcode,
DL, OpDWordsVTList,
Ops,
12541 OpDWordsVT, OpDWordsMMO, DAG);
12546 if (!Subtarget->hasDwordx3LoadStores() &&
12547 (VT == MVT::v3i32 || VT == MVT::v3f32)) {
12551 SDVTList WidenedVTList = DAG.
getVTList(WidenedVT, VTList.
VTs[1]);
12553 WidenedMemVT, WidenedMMO);
12563 bool ImageStore)
const {
12573 if (Subtarget->hasUnpackedD16VMem()) {
12587 if (ImageStore && Subtarget->hasImageStoreD16Bug()) {
12598 for (
unsigned I = 0;
I < Elts.
size() / 2;
I += 1) {
12604 if ((NumElements % 2) == 1) {
12606 unsigned I = Elts.
size() / 2;
12622 if (NumElements == 3) {
12642 return BufInfo->IsAsync;
12644 case Intrinsic::amdgcn_load_async_to_lds:
12645 case Intrinsic::amdgcn_global_load_async_lds:
12653SDValue SITargetLowering::lowerBufferIntrinsicVoid(
12659 auto GetArg = [&](
unsigned Arg) {
12663 switch (
Info.Kind) {
12664 case AMDGPU::BufferKindStore: {
12669 if (
Info.isTyped()) {
12672 VData = handleD16VData(VData, DAG);
12674 getBufferIntrinsicOps(
Op, Info, DAG, VData);
12675 unsigned Opc = IsD16 ? AMDGPUISD::TBUFFER_STORE_FORMAT_D16
12676 : AMDGPUISD::TBUFFER_STORE_FORMAT;
12678 M->getMemoryVT(),
M->getMemOperand());
12681 bool IsFormat =
Info.isFormatFromRsrc();
12687 "unsupported sub-dword format buffer store",
DL.getDebugLoc()));
12692 VData = handleD16VData(VData, DAG);
12704 IsFormat ? AMDGPUISD::BUFFER_STORE_FORMAT : AMDGPUISD::BUFFER_STORE;
12705 Opc = IsD16 ? AMDGPUISD::BUFFER_STORE_FORMAT_D16 :
Opc;
12709 return handleByteShortBufferStores(DAG, VDataVT,
DL,
Ops.data(), M);
12712 M->getMemoryVT(),
M->getMemOperand());
12714 case AMDGPU::BufferKindLoadToLDS: {
12716 std::optional<unsigned> VIndexArg =
Info.getVIndexArg();
12717 bool HasVIndex = VIndexArg.has_value();
12720 unsigned Size = GetArg(*
Info.getSizeArg())->getAsZExtVal();
12726 Opc = HasVIndex ? HasVOffset ? AMDGPU::BUFFER_LOAD_UBYTE_LDS_BOTHEN
12727 : AMDGPU::BUFFER_LOAD_UBYTE_LDS_IDXEN
12728 : HasVOffset ? AMDGPU::BUFFER_LOAD_UBYTE_LDS_OFFEN
12729 : AMDGPU::BUFFER_LOAD_UBYTE_LDS_OFFSET;
12732 Opc = HasVIndex ? HasVOffset ? AMDGPU::BUFFER_LOAD_USHORT_LDS_BOTHEN
12733 : AMDGPU::BUFFER_LOAD_USHORT_LDS_IDXEN
12734 : HasVOffset ? AMDGPU::BUFFER_LOAD_USHORT_LDS_OFFEN
12735 : AMDGPU::BUFFER_LOAD_USHORT_LDS_OFFSET;
12738 Opc = HasVIndex ? HasVOffset ? AMDGPU::BUFFER_LOAD_DWORD_LDS_BOTHEN
12739 : AMDGPU::BUFFER_LOAD_DWORD_LDS_IDXEN
12740 : HasVOffset ? AMDGPU::BUFFER_LOAD_DWORD_LDS_OFFEN
12741 : AMDGPU::BUFFER_LOAD_DWORD_LDS_OFFSET;
12744 if (!Subtarget->hasLDSLoadB96_B128())
12746 Opc = HasVIndex ? HasVOffset ? AMDGPU::BUFFER_LOAD_DWORDX3_LDS_BOTHEN
12747 : AMDGPU::BUFFER_LOAD_DWORDX3_LDS_IDXEN
12748 : HasVOffset ? AMDGPU::BUFFER_LOAD_DWORDX3_LDS_OFFEN
12749 : AMDGPU::BUFFER_LOAD_DWORDX3_LDS_OFFSET;
12752 if (!Subtarget->hasLDSLoadB96_B128())
12754 Opc = HasVIndex ? HasVOffset ? AMDGPU::BUFFER_LOAD_DWORDX4_LDS_BOTHEN
12755 : AMDGPU::BUFFER_LOAD_DWORDX4_LDS_IDXEN
12756 : HasVOffset ? AMDGPU::BUFFER_LOAD_DWORDX4_LDS_OFFEN
12757 : AMDGPU::BUFFER_LOAD_DWORDX4_LDS_OFFSET;
12765 if (HasVIndex && HasVOffset)
12768 else if (HasVIndex)
12769 Ops.push_back(GetArg(*VIndexArg));
12770 else if (HasVOffset)
12771 Ops.push_back(VOffset);
12773 SDValue Rsrc = bufferRsrcPtrToVector(GetArg(
Info.RsrcArg), DAG);
12774 Ops.push_back(Rsrc);
12775 Ops.push_back(GetArg(*
Info.getSOffsetArg()));
12776 Ops.push_back(GetArg(*
Info.getImmOffsetArg()));
12778 unsigned Aux = GetArg(*
Info.getAuxArg())->getAsZExtVal();
12796 case AMDGPU::BufferKindPrefetch: {
12799 bufferRsrcPtrToVector(GetArg(
Info.RsrcArg), DAG),
12800 GetArg(*
Info.getSOffsetArg()),
12801 GetArg(*
Info.getSizeArg()),
12805 Op->getVTList(),
Ops,
M->getMemoryVT(),
12806 M->getMemOperand());
12817 unsigned IntrinsicID =
Op.getConstantOperandVal(1);
12819 if (
const AMDGPU::BufferIntrinsicInfo *BufInfo =
12821 return lowerBufferIntrinsicVoid(
Op, *BufInfo, DAG);
12823 switch (IntrinsicID) {
12824 case Intrinsic::amdgcn_cluster_load_async_to_lds_b8:
12825 case Intrinsic::amdgcn_cluster_load_async_to_lds_b32:
12826 case Intrinsic::amdgcn_cluster_load_async_to_lds_b64:
12827 case Intrinsic::amdgcn_cluster_load_async_to_lds_b128: {
12828 if (Subtarget->hasGFX1250_STRICT())
12832 case Intrinsic::amdgcn_exp_compr: {
12853 unsigned Opc =
Done->isZero() ? AMDGPU::EXP : AMDGPU::EXP_DONE;
12860 case Intrinsic::amdgcn_load_to_lds:
12861 case Intrinsic::amdgcn_load_async_to_lds:
12862 case Intrinsic::amdgcn_global_load_lds:
12863 case Intrinsic::amdgcn_global_load_async_lds: {
12864 if (!Subtarget->hasVMemToLDSLoad())
12868 unsigned Size =
Op->getConstantOperandVal(4);
12873 Opc = AMDGPU::GLOBAL_LOAD_LDS_UBYTE;
12876 Opc = AMDGPU::GLOBAL_LOAD_LDS_USHORT;
12879 Opc = AMDGPU::GLOBAL_LOAD_LDS_DWORD;
12882 if (!Subtarget->hasLDSLoadB96_B128())
12884 Opc = AMDGPU::GLOBAL_LOAD_LDS_DWORDX3;
12887 if (!Subtarget->hasLDSLoadB96_B128())
12889 Opc = AMDGPU::GLOBAL_LOAD_LDS_DWORDX4;
12905 if (
LHS->isDivergent())
12909 RHS.getOperand(0).getValueType() == MVT::i32) {
12912 VOffset =
RHS.getOperand(0);
12916 Ops.push_back(Addr);
12924 Ops.push_back(VOffset);
12927 Ops.push_back(
Op.getOperand(5));
12929 unsigned Aux =
Op.getConstantOperandVal(6);
12944 case Intrinsic::amdgcn_end_cf:
12946 Op->getOperand(2), Chain),
12948 case Intrinsic::amdgcn_s_barrier_signal_var: {
12955 if (CntC && CntC->isZero()) {
12960 std::array<SDValue, 2>
Ops = {
12963 Op->getVTList(),
Ops);
12969 case Intrinsic::amdgcn_s_barrier_init: {
12976 unsigned Opc = IntrinsicID == Intrinsic::amdgcn_s_barrier_init
12977 ? AMDGPU::S_BARRIER_INIT_M0
12978 : AMDGPU::S_BARRIER_SIGNAL_M0;
12988 constexpr unsigned ShAmt = 16;
12999 case Intrinsic::amdgcn_s_wakeup_barrier: {
13000 if (!Subtarget->hasSWakeupBarrier())
13004 case Intrinsic::amdgcn_s_barrier_join: {
13012 switch (IntrinsicID) {
13015 case Intrinsic::amdgcn_s_barrier_join:
13016 Opc = AMDGPU::S_BARRIER_JOIN_IMM;
13018 case Intrinsic::amdgcn_s_wakeup_barrier:
13019 Opc = AMDGPU::S_WAKEUP_BARRIER_IMM;
13024 Ops.push_back(Chain);
13026 switch (IntrinsicID) {
13029 case Intrinsic::amdgcn_s_barrier_join:
13030 Opc = AMDGPU::S_BARRIER_JOIN_M0;
13032 case Intrinsic::amdgcn_s_wakeup_barrier:
13033 Opc = AMDGPU::S_WAKEUP_BARRIER_M0;
13045 case Intrinsic::amdgcn_s_prefetch_data:
13046 case Intrinsic::amdgcn_s_prefetch_inst: {
13049 return Op.getOperand(0);
13052 case Intrinsic::amdgcn_cooperative_atomic_store_32x4B:
13053 case Intrinsic::amdgcn_cooperative_atomic_store_16x8B:
13054 case Intrinsic::amdgcn_cooperative_atomic_store_8x16B: {
13062 case Intrinsic::amdgcn_av_store_b8:
13063 case Intrinsic::amdgcn_av_store_b16:
13064 case Intrinsic::amdgcn_av_store_b32:
13065 case Intrinsic::amdgcn_av_store_b64:
13066 case Intrinsic::amdgcn_av_store_b128: {
13074 if (
const AMDGPU::ImageDimIntrinsicInfo *ImageDimIntr =
13076 return lowerImage(
Op, ImageDimIntr, DAG,
true);
13092 return PtrVT == MVT::i64;
13106std::pair<SDValue, SDValue>
13119 bool CheckNUW =
Subtarget->hasGFX1250Insts();
13127 unsigned ImmOffset =
C1->getZExtValue();
13136 unsigned Overflow = ImmOffset & ~MaxImm;
13137 ImmOffset -= Overflow;
13138 if ((int32_t)Overflow < 0) {
13139 Overflow += ImmOffset;
13144 auto OverflowVal = DAG.
getConstant(Overflow,
DL, MVT::i32);
13163void SITargetLowering::setBufferOffsets(
SDValue CombinedOffset,
13165 Align Alignment)
const {
13167 SDLoc
DL(CombinedOffset);
13169 uint32_t
Imm =
C->getZExtValue();
13170 uint32_t SOffset, ImmOffset;
13171 if (
TII->splitMUBUFOffset(
Imm, SOffset, ImmOffset, Alignment)) {
13182 bool CheckNUW = Subtarget->hasGFX1250Insts();
13185 uint32_t SOffset, ImmOffset;
13188 TII->splitMUBUFOffset(
Offset, SOffset, ImmOffset, Alignment)) {
13196 SDValue SOffsetZero = Subtarget->hasRestrictedSOffset()
13205SDValue SITargetLowering::bufferRsrcPtrToVector(
SDValue MaybePointer,
13208 return MaybePointer;
13222 SDValue NumRecords =
Op->getOperand(3);
13228 if (Subtarget->getBufferResourceNumRecordsWidth() == 45) {
13231 DAG.
getConstant((1ULL << 45) - 1, Loc, MVT::i64));
13250 SDValue ExtShiftedStrideVec =
13262 DAG.
getNode(
ISD::OR, Loc, MVT::i64, NumRecordsRHS, ExtShiftedStride);
13264 DAG.
getNode(
ISD::OR, Loc, MVT::i64, CombinedFields, ExtShiftedFlags);
13269 auto [LowHalf, HighHalf] =
13270 DAG.
SplitScalar(Pointer, Loc, MVT::i32, MVT::i32);
13280 NumRecords, Flags);
13292 bool IsTFE)
const {
13297 ? AMDGPUISD::BUFFER_LOAD_UBYTE_TFE
13298 : AMDGPUISD::BUFFER_LOAD_USHORT_TFE;
13301 SDVTList VTs = DAG.
getVTList(MVT::v2i32, MVT::Other);
13313 ? AMDGPUISD::BUFFER_LOAD_UBYTE
13314 : AMDGPUISD::BUFFER_LOAD_USHORT;
13316 SDVTList ResList = DAG.
getVTList(MVT::i32, MVT::Other);
13330 if (VDataType == MVT::f16 || VDataType == MVT::bf16)
13334 Ops[1] = BufferStoreExt;
13335 unsigned Opc = (VDataType == MVT::i8) ? AMDGPUISD::BUFFER_STORE_BYTE
13336 : AMDGPUISD::BUFFER_STORE_SHORT;
13339 M->getMemOperand());
13364 DAGCombinerInfo &DCI)
const {
13365 SelectionDAG &DAG = DCI.DAG;
13366 if (Ld->getAlign() <
Align(4) || Ld->isDivergent())
13370 unsigned AS = Ld->getAddressSpace();
13379 EVT MemVT = Ld->getMemoryVT();
13380 if ((MemVT.
isSimple() && !DCI.isAfterLegalizeDAG()) ||
13387 "unexpected vector extload");
13390 SDValue Ptr = Ld->getBasePtr();
13393 Ld->getOffset(), Ld->getPointerInfo(), MVT::i32, Ld->getAlign(),
13394 Ld->getMemOperand()->getFlags(), Ld->getAAInfo());
13399 "unexpected fp extload");
13414 EVT VT = Ld->getValueType(0);
13417 DCI.AddToWorklist(Cvt.
getNode());
13422 DCI.AddToWorklist(Cvt.
getNode());
13433 if (Info.isEntryFunction())
13434 return Info.getUserSGPRInfo().hasFlatScratchInit();
13444 EVT MemVT = MemOp->getMemoryVT();
13452 "unsupported access of VGPR 'as memory' address space (13); only "
13453 "dword-aligned whole-dword loads and stores are implemented",
13454 DL.getDebugLoc()));
13464 if (MemOp->getAlign() <
Align(4))
13472 if (!Subtarget->getRegisterInfo()->getVGPRClassForBitWidth(
BitWidth) ||
13478 if (MemVT == ViewVT)
13482 Store->getBasePtr(),
Store->getMemOperand());
13487 Load->getMemOperand()->clearRanges();
13489 Load->getBasePtr(),
Load->getMemOperand());
13492 Load->getValueType(0));
13496 SDValue Chain = MemOp->getChain();
13503 unsigned NumDwords =
BitWidth / 32;
13504 RegVT = NumDwords == 1
13506 : EVT::getVectorVT(*DAG.
getContext(), MVT::i32, NumDwords);
13512 AMDGPUISD::REG_STORE,
DL, DAG.
getVTList(MVT::Other),
13513 {Chain, Value, Index}, MemVT,
Store->getMemOperand());
13517 AMDGPUISD::REG_LOAD,
DL, DAG.
getVTList(RegVT, MVT::Other), {Chain, Index},
13518 MemVT,
Load->getMemOperand());
13522 Load->getValueType(0));
13523 if (
Value == NewLoad)
13532 "Unexpected custom extending load!");
13533 EVT MemVT =
Load->getMemoryVT();
13534 MachineMemOperand *MMO =
Load->getMemOperand();
13541 isTypeLegal(MemVT) && Subtarget->hasScalarSubwordLoads() &&
13548 BasePtr, MVT::i16, MMO);
13571 EVT RealMemVT = (MemVT == MVT::i1) ? MVT::i8 : MVT::i16;
13597 Op.getValueType().getVectorElementType() == MVT::i32 &&
13598 "Custom lowering for non-i32 vectors hasn't been implemented.");
13601 unsigned AS =
Load->getAddressSpace();
13602 if (Subtarget->hasLDSMisalignedBugInWGPMode() &&
13609 SIMachineFunctionInfo *MFI = MF.
getInfo<SIMachineFunctionInfo>();
13613 !Subtarget->hasMultiDwordFlatScratchAddressing())
13625 Alignment >=
Align(4) && NumElements < 32) {
13627 (Subtarget->hasScalarDwordx3Loads() && NumElements == 3))
13639 if (NumElements > 4)
13642 if (NumElements == 3 && !Subtarget->hasDwordx3LoadStores())
13652 switch (Subtarget->getMaxPrivateElementSize()) {
13658 if (NumElements > 2)
13663 if (NumElements > 4)
13666 if (NumElements == 3 && !Subtarget->hasDwordx3LoadStores())
13675 auto Flags =
Load->getMemOperand()->getFlags();
13677 Load->getAlign(), Flags, &
Fast) &&
13685 MemVT, *
Load->getMemOperand())) {
13694 EVT VT =
Op.getValueType();
13731 EVT VT =
Op.getValueType();
13732 const SDNodeFlags
Flags =
Op->getFlags();
13734 bool AllowInaccurateRcp =
Flags.hasApproximateFuncs();
13740 if (!AllowInaccurateRcp && VT != MVT::f16 && VT != MVT::bf16)
13743 if (CLHS->isOne()) {
13756 return DAG.
getNode(AMDGPUISD::RCP, SL, VT,
RHS);
13760 if (CLHS->isMinusOne()) {
13769 if (!AllowInaccurateRcp &&
13770 ((VT != MVT::f16 && VT != MVT::bf16) || !
Flags.hasAllowReciprocal()))
13784 EVT VT =
Op.getValueType();
13785 const SDNodeFlags
Flags =
Op->getFlags();
13787 bool AllowInaccurateDiv =
Flags.hasApproximateFuncs();
13788 if (!AllowInaccurateDiv)
13812 if (IsNegRcp || (CLHS && CLHS->
isOne()))
13824 return DAG.
getNode(Opcode, SL, VT,
A,
B, Flags);
13834 Opcode = AMDGPUISD::FMUL_W_CHAIN;
13838 return DAG.
getNode(Opcode, SL, VTList,
13847 return DAG.
getNode(Opcode, SL, VT, {
A,
B,
C}, Flags);
13857 Opcode = AMDGPUISD::FMA_W_CHAIN;
13861 return DAG.
getNode(Opcode, SL, VTList,
13867 if (
SDValue FastLowered = lowerFastUnsafeFDIV(
Op, DAG))
13868 return FastLowered;
13871 EVT VT =
Op.getValueType();
13878 if (VT == MVT::bf16) {
13901 unsigned FMADOpCode =
13905 DAG.
getNode(AMDGPUISD::RCP, SL, MVT::f32, RHSExt,
Op->getFlags());
13908 SDValue Err = DAG.
getNode(FMADOpCode, SL, MVT::f32, NegRHSExt, Quot, LHSExt,
13910 Quot = DAG.
getNode(FMADOpCode, SL, MVT::f32, Err, Rcp, Quot,
Op->getFlags());
13911 Err = DAG.
getNode(FMADOpCode, SL, MVT::f32, NegRHSExt, Quot, LHSExt,
13921 return DAG.
getNode(AMDGPUISD::DIV_FIXUP, SL, MVT::f16, RDst,
RHS,
LHS,
13927 SDNodeFlags
Flags =
Op->getFlags();
13937 const APFloat K0Val(0x1p+96f);
13940 const APFloat K1Val(0x1p-32f);
13967 assert(ST->hasDenormModeInst() &&
"Requires S_DENORM_MODE");
13968 uint32_t DPDenormModeDefault = Info->getMode().fpDenormModeDPValue();
13969 uint32_t Mode = SPDenormMode | (DPDenormModeDefault << 2);
13974 if (
SDValue FastLowered = lowerFastUnsafeFDIV(
Op, DAG))
13975 return FastLowered;
13981 SDNodeFlags
Flags =
Op->getFlags();
13982 Flags.setNoFPExcept(
true);
13990 SDVTList ScaleVT = DAG.
getVTList(MVT::f32, MVT::i1);
13999 DAG.
getNode(AMDGPUISD::RCP, SL, MVT::f32, DenominatorScaled, Flags);
14003 using namespace AMDGPU::Hwreg;
14004 const unsigned Denorm32Reg = HwregEncoding::encode(ID_MODE, 4, 2);
14008 const SIMachineFunctionInfo *
Info = MF.
getInfo<SIMachineFunctionInfo>();
14009 const DenormalMode DenormMode =
Info->getMode().FP32Denormals;
14012 const bool HasDynamicDenormals =
14018 if (!PreservesDenormals) {
14023 SDVTList BindParamVTs = DAG.
getVTList(MVT::Other, MVT::Glue);
14026 if (HasDynamicDenormals) {
14030 SavedDenormMode =
SDValue(GetReg, 0);
14036 SDNode *EnableDenorm;
14037 if (Subtarget->hasDenormModeInst()) {
14038 const SDValue EnableDenormValue =
14041 EnableDenorm = DAG.
getNode(AMDGPUISD::DENORM_MODE, SL, BindParamVTs, Glue,
14045 const SDValue EnableDenormValue =
14047 EnableDenorm = DAG.
getMachineNode(AMDGPU::S_SETREG_B32, SL, BindParamVTs,
14048 {EnableDenormValue,
BitField, Glue});
14058 ApproxRcp, One, NegDivScale0, Flags);
14061 ApproxRcp, Fma0, Flags);
14067 NumeratorScaled,
Mul, Flags);
14073 NumeratorScaled, Fma3, Flags);
14075 if (!PreservesDenormals) {
14076 SDNode *DisableDenorm;
14077 if (!HasDynamicDenormals && Subtarget->hasDenormModeInst()) {
14081 SDVTList BindParamVTs = DAG.
getVTList(MVT::Other, MVT::Glue);
14083 DAG.
getNode(AMDGPUISD::DENORM_MODE, SL, BindParamVTs,
14087 assert(HasDynamicDenormals == (
bool)SavedDenormMode);
14088 const SDValue DisableDenormValue =
14089 HasDynamicDenormals
14094 AMDGPU::S_SETREG_B32, SL, MVT::Other,
14105 {Fma4, Fma1, Fma3, Scale},
Flags);
14107 return DAG.
getNode(AMDGPUISD::DIV_FIXUP, SL, MVT::f32, Fmas,
RHS,
LHS, Flags);
14111 if (
SDValue FastLowered = lowerFastUnsafeFDIV64(
Op, DAG))
14112 return FastLowered;
14120 SDVTList ScaleVT = DAG.
getVTList(MVT::f64, MVT::i1);
14126 SDValue Rcp = DAG.
getNode(AMDGPUISD::RCP, SL, MVT::f64, DivScale0);
14144 if (!Subtarget->hasUsableDivScaleConditionOutput()) {
14174 DAG.
getNode(AMDGPUISD::DIV_FMAS, SL, MVT::f64, Fma4, Fma3,
Mul, Scale);
14176 return DAG.
getNode(AMDGPUISD::DIV_FIXUP, SL, MVT::f64, Fmas,
Y,
X);
14180 EVT VT =
Op.getValueType();
14182 if (VT == MVT::f32)
14183 return LowerFDIV32(
Op, DAG);
14185 if (VT == MVT::f64)
14186 return LowerFDIV64(
Op, DAG);
14188 if (VT == MVT::f16 || VT == MVT::bf16)
14189 return LowerFDIV16(
Op, DAG);
14198 EVT ResultExpVT =
Op->getValueType(1);
14199 EVT InstrExpVT = VT == MVT::f16 ? MVT::i16 : MVT::i32;
14209 if (Subtarget->hasFractBug()) {
14227 assert(!
Store->isTruncatingStore() &&
"Unexpected custom truncating store!");
14228 EVT VT =
Store->getMemoryVT();
14230 if (VT == MVT::i1) {
14234 Store->getBasePtr(), MVT::i1,
Store->getMemOperand());
14238 Store->getValue().getValueType().getScalarType() == MVT::i32);
14240 unsigned AS =
Store->getAddressSpace();
14241 if (Subtarget->hasLDSMisalignedBugInWGPMode() &&
14249 SIMachineFunctionInfo *MFI = MF.
getInfo<SIMachineFunctionInfo>();
14253 !Subtarget->hasMultiDwordFlatScratchAddressing())
14260 if (NumElements > 4)
14263 if (NumElements == 3 && !Subtarget->hasDwordx3LoadStores())
14267 VT, *
Store->getMemOperand()))
14273 switch (Subtarget->getMaxPrivateElementSize()) {
14277 if (NumElements > 2)
14281 if (NumElements > 4 ||
14282 (NumElements == 3 && !Subtarget->hasFlatScratchEnabled()))
14290 auto Flags =
Store->getMemOperand()->getFlags();
14306 assert(!Subtarget->has16BitInsts());
14307 SDNodeFlags
Flags =
Op->getFlags();
14321 SDNodeFlags
Flags =
Op->getFlags();
14322 MVT VT =
Op.getValueType().getSimpleVT();
14430 SDNodeFlags
Flags =
Op->getFlags();
14439 if (!
Flags.hasApproximateFuncs()) {
14471 if (!
Flags.hasApproximateFuncs()) {
14480 ScaleDownFactor, ZeroInt);
14487 if (
Flags.hasNoInfs()) {
14503 EVT VT =
Op.getValueType();
14514 if (!
V.getValueType().isVector())
14522 if (Subtarget->hasTrigReducedRange()) {
14524 TrigVal = UnrollIfVec(DAG.
getNode(AMDGPUISD::FRACT,
DL, VT, MulVal, Flags));
14529 switch (
Op.getOpcode()) {
14531 TrigVal = DAG.
getNode(AMDGPUISD::COS_HW, SDLoc(
Op), VT, TrigVal, Flags);
14534 TrigVal = DAG.
getNode(AMDGPUISD::SIN_HW, SDLoc(
Op), VT, TrigVal, Flags);
14540 return UnrollIfVec(TrigVal);
14560 EVT VT =
Op.getValueType();
14568 Op->getVTList(),
Ops, VT,
14577SITargetLowering::performUCharToFloatCombine(
SDNode *
N,
14578 DAGCombinerInfo &DCI)
const {
14579 EVT VT =
N->getValueType(0);
14581 if (ScalarVT != MVT::f32 && ScalarVT != MVT::f16)
14584 SelectionDAG &DAG = DCI.DAG;
14588 EVT SrcVT = Src.getValueType();
14594 if (DCI.isAfterLegalizeDAG() && SrcVT == MVT::i32) {
14597 DCI.AddToWorklist(Cvt.
getNode());
14600 if (ScalarVT != MVT::f32) {
14612 DAGCombinerInfo &DCI)
const {
14623 SelectionDAG &DAG = DCI.DAG;
14642 for (
unsigned I = 0;
I != NumElts; ++
I) {
14666 if (NewElts.
size() == 1)
14688 for (
unsigned I = 0;
I != NumElts; ++
I) {
14723SDValue SITargetLowering::performSHLPtrCombine(
SDNode *
N,
unsigned AddrSpace,
14725 DAGCombinerInfo &DCI)
const {
14742 SelectionDAG &DAG = DCI.DAG;
14755 AM.BaseOffs =
Offset.getSExtValue();
14760 EVT VT =
N->getValueType(0);
14766 Flags.setNoUnsignedWrap(
14767 N->getFlags().hasNoUnsignedWrap() &&
14779 switch (
N->getOpcode()) {
14790 DAGCombinerInfo &DCI)
const {
14791 SelectionDAG &DAG = DCI.DAG;
14798 SDValue NewPtr = performSHLPtrCombine(Ptr.
getNode(),
N->getAddressSpace(),
14799 N->getMemoryVT(), DCI);
14803 NewOps[PtrIdx] = NewPtr;
14812 return (
Opc ==
ISD::AND && (Val == 0 || Val == 0xffffffff)) ||
14813 (
Opc ==
ISD::OR && (Val == 0xffffffff || Val == 0)) ||
14822SDValue SITargetLowering::splitBinaryBitConstantOp(
14826 uint32_t ValLo =
Lo_32(Val);
14827 uint32_t ValHi =
Hi_32(Val);
14834 if (Subtarget->has64BitLiterals() && CRHS->
hasOneUse() &&
14848 if (V.getValueType() != MVT::i1)
14850 switch (V.getOpcode()) {
14855 case AMDGPUISD::FP_CLASS:
14867 return V.getResNo() == 1;
14869 unsigned IntrinsicID = V.getConstantOperandVal(0);
14870 switch (IntrinsicID) {
14871 case Intrinsic::amdgcn_is_shared:
14872 case Intrinsic::amdgcn_is_private:
14895 if (!(
C & 0x000000ff))
14896 ZeroByteMask |= 0x000000ff;
14897 if (!(
C & 0x0000ff00))
14898 ZeroByteMask |= 0x0000ff00;
14899 if (!(
C & 0x00ff0000))
14900 ZeroByteMask |= 0x00ff0000;
14901 if (!(
C & 0xff000000))
14902 ZeroByteMask |= 0xff000000;
14903 uint32_t NonZeroByteMask = ~ZeroByteMask;
14904 if ((NonZeroByteMask &
C) != NonZeroByteMask)
14917 assert(
V.getValueSizeInBits() == 32);
14919 if (
V.getNumOperands() != 2)
14928 switch (
V.getOpcode()) {
14933 return (0x03020100 & ConstMask) | (0x0c0c0c0c & ~ConstMask);
14938 return (0x03020100 & ~ConstMask) | ConstMask;
14945 return uint32_t((0x030201000c0c0c0cull <<
C) >> 32);
14951 return uint32_t(0x0c0c0c0c03020100ull >>
C);
14958 DAGCombinerInfo &DCI)
const {
14959 if (DCI.isBeforeLegalize())
14962 SelectionDAG &DAG = DCI.DAG;
14963 EVT VT =
N->getValueType(0);
14968 if (VT == MVT::i64 && CRHS) {
14970 splitBinaryBitConstantOp(DCI, SDLoc(
N),
ISD::AND,
LHS, CRHS))
14974 if (CRHS && VT == MVT::i32) {
14984 unsigned Shift = CShift->getZExtValue();
14986 unsigned Offset = NB + Shift;
14987 if ((
Offset & (Bits - 1)) == 0) {
14990 DAG.
getNode(AMDGPUISD::BFE_U32, SL, MVT::i32,
LHS->getOperand(0),
15011 Sel = (
LHS.getConstantOperandVal(2) & Sel) | (~Sel & 0x0c0c0c0c);
15013 return DAG.
getNode(AMDGPUISD::PERM,
DL, MVT::i32,
LHS.getOperand(0),
15023 if (Shift && Shift->getZExtValue() < 16 &&
15042 if (
Y.getOpcode() !=
ISD::FABS ||
Y.getOperand(0) !=
X ||
15047 if (
X !=
LHS.getOperand(1))
15051 const ConstantFPSDNode *
C1 =
15053 if (!
C1 || !
C1->isInfinity() ||
C1->isNegative())
15068 return DAG.
getNode(AMDGPUISD::FP_CLASS,
DL, MVT::i1,
X,
15074 if (
RHS.getOpcode() ==
ISD::SETCC &&
LHS.getOpcode() == AMDGPUISD::FP_CLASS)
15077 if (
LHS.getOpcode() ==
ISD::SETCC &&
RHS.getOpcode() == AMDGPUISD::FP_CLASS &&
15085 (
RHS.getOperand(0) ==
LHS.getOperand(0) &&
15086 LHS.getOperand(0) ==
LHS.getOperand(1))) {
15088 unsigned NewMask = LCC ==
ISD::SETO ?
Mask->getZExtValue() & ~OrdMask
15089 :
Mask->getZExtValue() & OrdMask;
15092 return DAG.
getNode(AMDGPUISD::FP_CLASS,
DL, MVT::i1,
RHS.getOperand(0),
15110 TII->pseudoToMCOpcode(AMDGPU::V_PERM_B32_e64) != -1) {
15113 if (LHSMask != ~0u && RHSMask != ~0u) {
15116 if (LHSMask > RHSMask) {
15123 uint32_t LHSUsedLanes = ~(LHSMask & 0x0c0c0c0c) & 0x0c0c0c0c;
15124 uint32_t RHSUsedLanes = ~(RHSMask & 0x0c0c0c0c) & 0x0c0c0c0c;
15127 if (!(LHSUsedLanes & RHSUsedLanes) &&
15130 !(LHSUsedLanes == 0x0c0c0000 && RHSUsedLanes == 0x00000c0c)) {
15136 uint32_t
Mask = LHSMask & RHSMask;
15137 for (
unsigned I = 0;
I < 32;
I += 8) {
15138 uint32_t ByteSel = 0xff <<
I;
15139 if ((LHSMask & ByteSel) == 0x0c || (RHSMask & ByteSel) == 0x0c)
15140 Mask &= (0x0c <<
I) & 0xffffffff;
15145 uint32_t Sel =
Mask | (LHSUsedLanes & 0x04040404);
15148 return DAG.
getNode(AMDGPUISD::PERM,
DL, MVT::i32,
LHS.getOperand(0),
15198static const std::optional<ByteProvider<SDValue>>
15200 unsigned Depth = 0) {
15203 return std::nullopt;
15205 if (
Op.getValueSizeInBits() < 8)
15206 return std::nullopt;
15208 if (
Op.getValueType().isVector())
15211 switch (
Op->getOpcode()) {
15224 NarrowVT = VTSign->getVT();
15227 return std::nullopt;
15230 if (SrcIndex >= NarrowByteWidth)
15231 return std::nullopt;
15239 return std::nullopt;
15241 uint64_t BitShift = ShiftOp->getZExtValue();
15243 if (BitShift % 8 != 0)
15244 return std::nullopt;
15246 uint64_t NewSrcIndex = SrcIndex + BitShift / 8;
15247 if (NewSrcIndex >=
Op.getScalarValueSizeInBits() / 8)
15248 return std::nullopt;
15267static const std::optional<ByteProvider<SDValue>>
15269 unsigned StartingIndex = 0) {
15273 return std::nullopt;
15275 unsigned BitWidth =
Op.getScalarValueSizeInBits();
15277 return std::nullopt;
15279 return std::nullopt;
15281 bool IsVec =
Op.getValueType().isVector();
15282 switch (
Op.getOpcode()) {
15285 return std::nullopt;
15290 return std::nullopt;
15294 return std::nullopt;
15297 if (!
LHS->isConstantZero() && !
RHS->isConstantZero())
15298 return std::nullopt;
15299 if (!
LHS ||
LHS->isConstantZero())
15301 if (!
RHS ||
RHS->isConstantZero())
15303 return std::nullopt;
15308 return std::nullopt;
15312 return std::nullopt;
15314 uint32_t BitMask = BitMaskOp->getZExtValue();
15316 uint32_t IndexMask = 0xFF << (Index * 8);
15318 if ((IndexMask & BitMask) != IndexMask) {
15321 if (IndexMask & BitMask)
15322 return std::nullopt;
15331 return std::nullopt;
15335 if (!ShiftOp ||
Op.getValueType().isVector())
15336 return std::nullopt;
15338 uint64_t BitsProvided =
Op.getValueSizeInBits();
15339 if (BitsProvided % 8 != 0)
15340 return std::nullopt;
15342 uint64_t BitShift = ShiftOp->getAPIntValue().urem(BitsProvided);
15344 return std::nullopt;
15346 uint64_t ConcatSizeInBytes = BitsProvided / 4;
15347 uint64_t ByteShift = BitShift / 8;
15349 uint64_t NewIndex = (Index + ByteShift) % ConcatSizeInBytes;
15350 uint64_t BytesProvided = BitsProvided / 8;
15351 SDValue NextOp =
Op.getOperand(NewIndex >= BytesProvided ? 0 : 1);
15352 NewIndex %= BytesProvided;
15359 return std::nullopt;
15363 return std::nullopt;
15365 uint64_t BitShift = ShiftOp->getZExtValue();
15367 return std::nullopt;
15369 auto BitsProvided =
Op.getScalarValueSizeInBits();
15370 if (BitsProvided % 8 != 0)
15371 return std::nullopt;
15373 uint64_t BytesProvided = BitsProvided / 8;
15374 uint64_t ByteShift = BitShift / 8;
15375 if (Index + ByteShift < BytesProvided)
15377 Index + ByteShift);
15380 return std::nullopt;
15386 return std::nullopt;
15390 return std::nullopt;
15392 uint64_t BitShift = ShiftOp->getZExtValue();
15393 if (BitShift % 8 != 0)
15394 return std::nullopt;
15395 uint64_t ByteShift = BitShift / 8;
15401 return Index < ByteShift
15404 Depth + 1, StartingIndex);
15413 return std::nullopt;
15421 NarrowBitWidth = VTSign->getVT().getSizeInBits();
15423 if (NarrowBitWidth % 8 != 0)
15424 return std::nullopt;
15425 uint64_t NarrowByteWidth = NarrowBitWidth / 8;
15427 if (Index >= NarrowByteWidth)
15429 ? std::optional<ByteProvider<SDValue>>(
15437 return std::nullopt;
15441 if (NarrowByteWidth >= Index) {
15446 return std::nullopt;
15453 return std::nullopt;
15459 unsigned NarrowBitWidth = L->getMemoryVT().getSizeInBits();
15460 if (NarrowBitWidth % 8 != 0)
15461 return std::nullopt;
15462 uint64_t NarrowByteWidth = NarrowBitWidth / 8;
15467 if (Index >= NarrowByteWidth) {
15469 ? std::optional<ByteProvider<SDValue>>(
15474 if (NarrowByteWidth > Index) {
15478 return std::nullopt;
15483 return std::nullopt;
15486 Depth + 1, StartingIndex);
15492 return std::nullopt;
15493 auto VecIdx = IdxOp->getZExtValue();
15494 auto ScalarSize =
Op.getScalarValueSizeInBits();
15495 if (ScalarSize < 32)
15496 Index = ScalarSize == 8 ? VecIdx : VecIdx * 2 + Index;
15498 StartingIndex, Index);
15501 case AMDGPUISD::PERM: {
15503 return std::nullopt;
15507 return std::nullopt;
15510 (PermMask->getZExtValue() & (0xFF << (Index * 8))) >> (Index * 8);
15511 if (IdxMask > 0x07 && IdxMask != 0x0c)
15512 return std::nullopt;
15514 auto NextOp =
Op.getOperand(IdxMask > 0x03 ? 0 : 1);
15515 auto NextIndex = IdxMask > 0x03 ? IdxMask % 4 : IdxMask;
15517 return IdxMask != 0x0c ?
calculateSrcByte(NextOp, StartingIndex, NextIndex)
15523 return std::nullopt;
15538 return !OpVT.
isVector() && OpVT.getSizeInBits() == 16;
15545 auto MemVT = L->getMemoryVT();
15548 return L->getMemoryVT().getSizeInBits() == 16;
15558 int Low8 = Mask & 0xff;
15559 int Hi8 = (Mask & 0xff00) >> 8;
15561 assert(Low8 < 8 && Hi8 < 8);
15563 bool IsConsecutive = (Hi8 - Low8 == 1);
15568 bool Is16Aligned = !(Low8 % 2);
15570 return IsConsecutive && Is16Aligned;
15578 int Low16 = PermMask & 0xffff;
15579 int Hi16 = (PermMask & 0xffff0000) >> 16;
15589 auto OtherOpIs16Bit = TempOtherOp.getValueSizeInBits() == 16 ||
15591 if (!OtherOpIs16Bit)
15599 unsigned DWordOffset) {
15604 assert(Src.getValueSizeInBits().isKnownMultipleOf(8));
15609 if (Src.getValueType().isVector()) {
15610 auto ScalarTySize = Src.getScalarValueSizeInBits();
15611 auto ScalarTy = Src.getValueType().getScalarType();
15612 if (ScalarTySize == 32) {
15616 if (ScalarTySize > 32) {
15619 DAG.
getConstant(DWordOffset / (ScalarTySize / 32), SL, MVT::i32));
15620 auto ShiftVal = 32 * (DWordOffset % (ScalarTySize / 32));
15627 assert(ScalarTySize < 32);
15636 auto NumElements =
TypeSize / ScalarTySize;
15637 auto Trunc32Elements = (ScalarTySize * NumElements) / 32;
15638 auto NormalizedTrunc = Trunc32Elements * 32 / ScalarTySize;
15639 auto NumElementsIn32 = 32 / ScalarTySize;
15640 auto NumAvailElements = DWordOffset < Trunc32Elements
15642 : NumElements - NormalizedTrunc;
15655 auto ShiftVal = 32 * DWordOffset;
15663 [[maybe_unused]]
EVT VT =
N->getValueType(0);
15668 for (
int i = 0; i < 4; i++) {
15670 std::optional<ByteProvider<SDValue>>
P =
15673 if (!
P ||
P->isConstantZero())
15678 if (PermNodes.
size() != 4)
15681 std::pair<unsigned, unsigned> FirstSrc(0, PermNodes[0].SrcOffset / 4);
15682 std::optional<std::pair<unsigned, unsigned>> SecondSrc;
15684 for (
size_t i = 0; i < PermNodes.
size(); i++) {
15685 auto PermOp = PermNodes[i];
15688 int SrcByteAdjust = 4;
15692 if (!PermOp.hasSameSrc(PermNodes[FirstSrc.first]) ||
15693 ((PermOp.SrcOffset / 4) != FirstSrc.second)) {
15695 if (!PermOp.hasSameSrc(PermNodes[SecondSrc->first]) ||
15696 ((PermOp.SrcOffset / 4) != SecondSrc->second))
15700 SecondSrc = {i, PermNodes[i].SrcOffset / 4};
15701 assert(!(PermNodes[SecondSrc->first].Src->getValueSizeInBits() % 8));
15704 assert((PermOp.SrcOffset % 4) + SrcByteAdjust < 8);
15706 PermMask |= ((PermOp.SrcOffset % 4) + SrcByteAdjust) << (i * 8);
15709 SDValue Op = *PermNodes[FirstSrc.first].Src;
15711 assert(
Op.getValueSizeInBits() == 32);
15715 int Low16 = PermMask & 0xffff;
15716 int Hi16 = (PermMask & 0xffff0000) >> 16;
15718 bool WellFormedLow = (Low16 == 0x0504) || (Low16 == 0x0100);
15719 bool WellFormedHi = (Hi16 == 0x0706) || (Hi16 == 0x0302);
15722 if (WellFormedLow && WellFormedHi)
15726 SDValue OtherOp = SecondSrc ? *PermNodes[SecondSrc->first].Src :
Op;
15735 (
N->getOperand(0) ==
Op ||
N->getOperand(0) == OtherOp) &&
15736 (
N->getOperand(1) ==
Op ||
N->getOperand(1) == OtherOp))
15741 assert(
Op.getValueType().isByteSized() &&
15752 return DAG.
getNode(AMDGPUISD::PERM,
DL, MVT::i32,
Op, OtherOp,
15759 DAGCombinerInfo &DCI)
const {
15760 SelectionDAG &DAG = DCI.DAG;
15764 EVT VT =
N->getValueType(0);
15765 if (VT == MVT::i1) {
15767 if (
LHS.getOpcode() == AMDGPUISD::FP_CLASS &&
15768 RHS.getOpcode() == AMDGPUISD::FP_CLASS) {
15770 if (Src !=
RHS.getOperand(0))
15775 if (!CLHS || !CRHS)
15779 static const uint32_t MaxMask = 0x3ff;
15784 return DAG.
getNode(AMDGPUISD::FP_CLASS,
DL, MVT::i1, Src,
15791 if (VT == MVT::i32 && !DCI.isBeforeLegalize()) {
15792 for (
unsigned I : {1u, 0
u}) {
15801 if (
N->isDivergent() && !
Cond->isDivergent() && !
Cond.hasOneUse())
15815 LHS.getOpcode() == AMDGPUISD::PERM &&
15821 Sel |=
LHS.getConstantOperandVal(2);
15823 return DAG.
getNode(AMDGPUISD::PERM,
DL, MVT::i32,
LHS.getOperand(0),
15830 TII->pseudoToMCOpcode(AMDGPU::V_PERM_B32_e64) != -1) {
15834 auto usesCombinedOperand = [](SDNode *OrUse) {
15837 !OrUse->getValueType(0).isVector())
15841 for (
auto *VUser : OrUse->users()) {
15842 if (!VUser->getValueType(0).isVector())
15849 if (VUser->getOpcode() == VectorwiseOp)
15855 if (!
any_of(
N->users(), usesCombinedOperand))
15861 if (LHSMask != ~0u && RHSMask != ~0u) {
15864 if (LHSMask > RHSMask) {
15871 uint32_t LHSUsedLanes = ~(LHSMask & 0x0c0c0c0c) & 0x0c0c0c0c;
15872 uint32_t RHSUsedLanes = ~(RHSMask & 0x0c0c0c0c) & 0x0c0c0c0c;
15875 if (!(LHSUsedLanes & RHSUsedLanes) &&
15878 !(LHSUsedLanes == 0x0c0c0000 && RHSUsedLanes == 0x00000c0c)) {
15880 LHSMask &= ~RHSUsedLanes;
15881 RHSMask &= ~LHSUsedLanes;
15883 LHSMask |= LHSUsedLanes & 0x04040404;
15885 uint32_t Sel = LHSMask | RHSMask;
15888 return DAG.
getNode(AMDGPUISD::PERM,
DL, MVT::i32,
LHS.getOperand(0),
15893 if (LHSMask == ~0u || RHSMask == ~0u) {
15934 return IdentitySrc;
15940 if (VT != MVT::i64 || DCI.isBeforeLegalizeOps())
15955 if (SrcVT == MVT::i32) {
15960 DCI.AddToWorklist(LowOr.
getNode());
15961 DCI.AddToWorklist(HiBits.getNode());
15972 N->getOperand(0), CRHS))
15980 DAGCombinerInfo &DCI)
const {
15981 if (
SDValue RV = reassociateScalarOps(
N, DCI.DAG))
15988 SelectionDAG &DAG = DCI.DAG;
15990 EVT VT =
N->getValueType(0);
15991 if (CRHS && VT == MVT::i64) {
15993 splitBinaryBitConstantOp(DCI, SDLoc(
N),
ISD::XOR,
LHS, CRHS))
16000 unsigned Opc =
LHS.getOpcode();
16030 LHS->getOperand(0), FNegLHS, FNegRHS);
16039SITargetLowering::performZeroOrAnyExtendCombine(
SDNode *
N,
16040 DAGCombinerInfo &DCI)
const {
16041 if (!Subtarget->has16BitInsts() ||
16045 EVT VT =
N->getValueType(0);
16046 if (VT != MVT::i32)
16050 if (Src.getValueType() != MVT::i16)
16053 if (!Src->hasOneUse())
16056 if (
SDValue Ext = performExtendTruncShiftCombine(
N, DCI))
16063 std::optional<ByteProvider<SDValue>> BP0 =
16065 if (!BP0 || BP0->SrcOffset >= 4 || !BP0->Src)
16069 std::optional<ByteProvider<SDValue>> BP1 =
16071 if (!BP1 || BP1->SrcOffset >= 4 || !BP1->Src)
16079 SelectionDAG &DAG = DCI.DAG;
16081 uint32_t PermMask = 0x0c0c0c0c;
16084 PermMask = (PermMask & ~0xFF) | (BP0->SrcOffset + 4);
16089 PermMask = (PermMask & ~(0xFF << 8)) | (BP1->SrcOffset << 8);
16092 return DAG.
getNode(AMDGPUISD::PERM,
DL, MVT::i32, V0,
V1,
16100SITargetLowering::performExtendTruncShiftCombine(
SDNode *
N,
16101 DAGCombinerInfo &DCI)
const {
16105 const unsigned Opc =
N->getOpcode();
16108 if (
N->getValueType(0) != MVT::i32 || Src.getValueType() != MVT::i16 ||
16109 Src.getOpcode() != (IsSigned ?
ISD::SRA :
ISD::SRL) || !Src.hasOneUse() ||
16115 if (!IsSigned && !Src->isDivergent())
16118 SDValue X = Src.getOperand(0).getOperand(0);
16120 if (
X.getValueType() != MVT::i32 || !Shift || Shift->getZExtValue() == 0 ||
16121 Shift->getZExtValue() >= 16 || (IsSigned && Shift->getZExtValue() != 8))
16124 SelectionDAG &DAG = DCI.DAG;
16126 const unsigned Width = 16 - Shift->getZExtValue();
16139SITargetLowering::performSignExtendInRegCombine(
SDNode *
N,
16140 DAGCombinerInfo &DCI)
const {
16146 if (((Src.getOpcode() == AMDGPUISD::SBUFFER_LOAD_UBYTE &&
16147 VTSign->getVT() == MVT::i8) ||
16148 (Src.getOpcode() == AMDGPUISD::SBUFFER_LOAD_USHORT &&
16149 VTSign->getVT() == MVT::i16))) {
16150 assert(Subtarget->hasScalarSubwordLoads() &&
16151 "s_buffer_load_{u8, i8} are supported "
16152 "in GFX12 (or newer) architectures.");
16153 unsigned Opc = (Src.getOpcode() == AMDGPUISD::SBUFFER_LOAD_UBYTE)
16154 ? AMDGPUISD::SBUFFER_LOAD_BYTE
16155 : AMDGPUISD::SBUFFER_LOAD_SHORT;
16158 DCI.DAG.getVTList(MVT::i32, Src.getOperand(0).getValueType());
16166 SDValue BufferLoad = DCI.DAG.getMemIntrinsicNode(
16167 Opc,
DL, ResList,
Ops,
M->getMemoryVT(),
M->getMemOperand());
16168 return DCI.DAG.getMergeValues({BufferLoad, BufferLoad.
getValue(1)},
DL);
16170 if (((Src.getOpcode() == AMDGPUISD::BUFFER_LOAD_UBYTE &&
16171 VTSign->getVT() == MVT::i8) ||
16172 (Src.getOpcode() == AMDGPUISD::BUFFER_LOAD_USHORT &&
16173 VTSign->getVT() == MVT::i16)) &&
16182 Src.getOperand(6), Src.getOperand(7)};
16185 DCI.DAG.getVTList(MVT::i32, Src.getOperand(0).getValueType());
16186 unsigned Opc = (Src.getOpcode() == AMDGPUISD::BUFFER_LOAD_UBYTE)
16187 ? AMDGPUISD::BUFFER_LOAD_BYTE
16188 : AMDGPUISD::BUFFER_LOAD_SHORT;
16189 SDValue BufferLoadSignExt = DCI.DAG.getMemIntrinsicNode(
16190 Opc, SDLoc(
N), ResList,
Ops,
M->getMemoryVT(),
M->getMemOperand());
16191 return DCI.DAG.getMergeValues(
16192 {BufferLoadSignExt, BufferLoadSignExt.
getValue(1)}, SDLoc(
N));
16198 DAGCombinerInfo &DCI)
const {
16199 SelectionDAG &DAG = DCI.DAG;
16206 if (
N->getOperand(0).isUndef())
16213 DAGCombinerInfo &DCI)
const {
16214 EVT VT =
N->getValueType(0);
16225 return DCI.DAG.getNode(AMDGPUISD::RSQ, SDLoc(
N), VT, N0.
getOperand(0),
16234 unsigned MaxDepth)
const {
16235 EVT VT =
Op.getValueType();
16237 "expected a floating-point value to query canonicality of");
16243 unsigned MaxDepth)
const {
16245 "QueryVT must be a floating-point scalar type");
16246 EVT VT =
Op.getValueType();
16250 unsigned Opcode =
Op.getOpcode();
16255 const auto &
F = CFP->getValueAPF();
16256 if (
F.isNaN() &&
F.isSignaling())
16258 if (!
F.isDenormal())
16290 case AMDGPUISD::FMUL_LEGACY:
16291 case AMDGPUISD::FMAD_FTZ:
16292 case AMDGPUISD::RCP:
16293 case AMDGPUISD::RSQ:
16294 case AMDGPUISD::RSQ_CLAMP:
16295 case AMDGPUISD::RCP_LEGACY:
16296 case AMDGPUISD::RCP_IFLAG:
16297 case AMDGPUISD::LOG:
16298 case AMDGPUISD::EXP:
16299 case AMDGPUISD::DIV_SCALE:
16300 case AMDGPUISD::DIV_FMAS:
16301 case AMDGPUISD::DIV_FIXUP:
16302 case AMDGPUISD::FRACT:
16303 case AMDGPUISD::CVT_PKRTZ_F16_F32:
16304 case AMDGPUISD::CVT_F32_UBYTE0:
16305 case AMDGPUISD::CVT_F32_UBYTE1:
16306 case AMDGPUISD::CVT_F32_UBYTE2:
16307 case AMDGPUISD::CVT_F32_UBYTE3:
16308 case AMDGPUISD::FP_TO_FP16:
16309 case AMDGPUISD::SIN_HW:
16310 case AMDGPUISD::COS_HW:
16322 if (
Op.getValueType() == MVT::i32) {
16328 if (RHS->getZExtValue() == 0xffff0000) {
16339 return Op.getValueType().getScalarType() != MVT::f16;
16349 case AMDGPUISD::CLAMP:
16350 case AMDGPUISD::FMED3:
16351 case AMDGPUISD::FMAX3:
16352 case AMDGPUISD::FMIN3:
16353 case AMDGPUISD::FMAXIMUM3:
16354 case AMDGPUISD::FMINIMUM3: {
16360 if (Subtarget->supportsMinMaxDenormModes() ||
16370 for (
unsigned I = 0, E =
Op.getNumOperands();
I != E; ++
I) {
16385 for (
unsigned i = 0, e =
Op.getNumOperands(); i != e; ++i) {
16419 if (
Op.getValueType() == MVT::i16) {
16431 unsigned IntrinsicID =
Op.getConstantOperandVal(0);
16433 switch (IntrinsicID) {
16434 case Intrinsic::amdgcn_update_dpp: {
16435 enum UpdateDPPOperand {
16442 if (QueryVT == MVT::f32 && (VT == MVT::i32 || VT == MVT::f32) &&
16443 Op.getConstantOperandVal(RowMask) == 0xf &&
16444 Op.getConstantOperandVal(BankMask) == 0xf &&
16445 Op.getConstantOperandVal(BoundCtrl) != 0)
16450 case Intrinsic::amdgcn_cvt_pkrtz:
16451 case Intrinsic::amdgcn_cubeid:
16452 case Intrinsic::amdgcn_frexp_mant:
16453 case Intrinsic::amdgcn_fdot2:
16454 case Intrinsic::amdgcn_rcp:
16455 case Intrinsic::amdgcn_rsq:
16456 case Intrinsic::amdgcn_rsq_clamp:
16457 case Intrinsic::amdgcn_rcp_legacy:
16458 case Intrinsic::amdgcn_rsq_legacy:
16459 case Intrinsic::amdgcn_trig_preop:
16460 case Intrinsic::amdgcn_tanh:
16461 case Intrinsic::amdgcn_log:
16462 case Intrinsic::amdgcn_exp2:
16463 case Intrinsic::amdgcn_sqrt:
16481 unsigned MaxDepth)
const {
16484 unsigned Opcode =
MI->getOpcode();
16486 if (Opcode == AMDGPU::G_FCANONICALIZE)
16489 std::optional<FPValueAndVReg> FCR;
16492 if (FCR->Value.isSignaling())
16494 if (!FCR->Value.isDenormal())
16505 case AMDGPU::G_FADD:
16506 case AMDGPU::G_FSUB:
16507 case AMDGPU::G_FMUL:
16508 case AMDGPU::G_FCEIL:
16509 case AMDGPU::G_FFLOOR:
16510 case AMDGPU::G_FRINT:
16511 case AMDGPU::G_FNEARBYINT:
16512 case AMDGPU::G_INTRINSIC_FPTRUNC_ROUND:
16513 case AMDGPU::G_INTRINSIC_TRUNC:
16514 case AMDGPU::G_INTRINSIC_ROUNDEVEN:
16515 case AMDGPU::G_FMA:
16516 case AMDGPU::G_FMAD:
16517 case AMDGPU::G_FSQRT:
16518 case AMDGPU::G_FDIV:
16519 case AMDGPU::G_FREM:
16520 case AMDGPU::G_FPOW:
16521 case AMDGPU::G_FPEXT:
16522 case AMDGPU::G_FLOG:
16523 case AMDGPU::G_FLOG2:
16524 case AMDGPU::G_FLOG10:
16525 case AMDGPU::G_FPTRUNC:
16526 case AMDGPU::G_AMDGPU_RCP_IFLAG:
16527 case AMDGPU::G_AMDGPU_CVT_F32_UBYTE0:
16528 case AMDGPU::G_AMDGPU_CVT_F32_UBYTE1:
16529 case AMDGPU::G_AMDGPU_CVT_F32_UBYTE2:
16530 case AMDGPU::G_AMDGPU_CVT_F32_UBYTE3:
16532 case AMDGPU::G_FNEG:
16533 case AMDGPU::G_FABS:
16534 case AMDGPU::G_FCOPYSIGN:
16536 case AMDGPU::G_FMINNUM:
16537 case AMDGPU::G_FMAXNUM:
16538 case AMDGPU::G_FMINNUM_IEEE:
16539 case AMDGPU::G_FMAXNUM_IEEE:
16540 case AMDGPU::G_FMINIMUM:
16541 case AMDGPU::G_FMAXIMUM:
16542 case AMDGPU::G_FMINIMUMNUM:
16543 case AMDGPU::G_FMAXIMUMNUM: {
16544 if (Subtarget->supportsMinMaxDenormModes() ||
16551 case AMDGPU::G_BUILD_VECTOR:
16556 case AMDGPU::G_INTRINSIC:
16557 case AMDGPU::G_INTRINSIC_CONVERGENT:
16559 case Intrinsic::amdgcn_fmul_legacy:
16560 case Intrinsic::amdgcn_fmad_ftz:
16561 case Intrinsic::amdgcn_sqrt:
16562 case Intrinsic::amdgcn_fmed3:
16563 case Intrinsic::amdgcn_sin:
16564 case Intrinsic::amdgcn_cos:
16565 case Intrinsic::amdgcn_log:
16566 case Intrinsic::amdgcn_exp2:
16567 case Intrinsic::amdgcn_log_clamp:
16568 case Intrinsic::amdgcn_rcp:
16569 case Intrinsic::amdgcn_rcp_legacy:
16570 case Intrinsic::amdgcn_rsq:
16571 case Intrinsic::amdgcn_rsq_clamp:
16572 case Intrinsic::amdgcn_rsq_legacy:
16573 case Intrinsic::amdgcn_div_scale:
16574 case Intrinsic::amdgcn_div_fmas:
16575 case Intrinsic::amdgcn_div_fixup:
16576 case Intrinsic::amdgcn_fract:
16577 case Intrinsic::amdgcn_cvt_pkrtz:
16578 case Intrinsic::amdgcn_cubeid:
16579 case Intrinsic::amdgcn_cubema:
16580 case Intrinsic::amdgcn_cubesc:
16581 case Intrinsic::amdgcn_cubetc:
16582 case Intrinsic::amdgcn_frexp_mant:
16583 case Intrinsic::amdgcn_fdot2:
16584 case Intrinsic::amdgcn_trig_preop:
16585 case Intrinsic::amdgcn_tanh:
16604 if (
C.isDenormal()) {
16617 if (
C.isSignaling()) {
16632SITargetLowering::performFCanonicalizeCombine(
SDNode *
N,
16633 DAGCombinerInfo &DCI)
const {
16634 SelectionDAG &DAG = DCI.DAG;
16636 EVT VT =
N->getValueType(0);
16645 return getCanonicalConstantFP(DAG, SDLoc(
N), VT, CFP->getValueAPF());
16660 EVT EltVT =
Lo.getValueType();
16669 for (
unsigned I = 0;
I != 2; ++
I) {
16673 getCanonicalConstantFP(DAG, SL, EltVT, CFP->getValueAPF());
16674 }
else if (
Op.isUndef()) {
16709 return AMDGPUISD::FMAX3;
16711 return AMDGPUISD::FMAXIMUM3;
16713 return AMDGPUISD::SMAX3;
16715 return AMDGPUISD::UMAX3;
16719 return AMDGPUISD::FMIN3;
16721 return AMDGPUISD::FMINIMUM3;
16723 return AMDGPUISD::SMIN3;
16725 return AMDGPUISD::UMIN3;
16746 if (!MinK || !MaxK)
16758 unsigned Med3Opc =
Signed ? AMDGPUISD::SMED3 : AMDGPUISD::UMED3;
16759 if (VT == MVT::i32 || (VT == MVT::i16 && Subtarget->hasMed3_16()))
16760 return DAG.
getNode(Med3Opc, SL, VT, Src, MaxVal, MinVal);
16784 bool IsKnownNoNaNs)
const {
16820 const SIMachineFunctionInfo *
Info = MF.
getInfo<SIMachineFunctionInfo>();
16826 if (
Info->getMode().DX10Clamp) {
16835 if (VT == MVT::f32 || (VT == MVT::f16 && Subtarget->hasMed3_16())) {
16867 return (VT == MVT::f32) || (VT == MVT::f16 &&
Subtarget.hasMin3Max3_16()) ||
16868 (VT == MVT::v2f16 &&
Subtarget.hasMin3Max3PKF16());
16871 return (VT == MVT::f32 &&
Subtarget.hasMinimum3Maximum3F32()) ||
16872 (VT == MVT::f16 &&
Subtarget.hasMinimum3Maximum3F16()) ||
16873 (VT == MVT::v2f16 &&
Subtarget.hasMinimum3Maximum3PKF16());
16878 return (VT == MVT::i32) || (VT == MVT::i16 &&
Subtarget.hasMin3Max3_16());
16887 DAGCombinerInfo &DCI)
const {
16888 SelectionDAG &DAG = DCI.DAG;
16899 auto IsTreeWithCombinableChildren = [
Opc](
SDValue Op) {
16900 return (
Op.getOperand(0).getOpcode() ==
Opc &&
16901 Op.getOperand(0).hasOneUse()) ||
16903 Op.getOperand(1).hasOneUse());
16908 bool HasCombinableTreeChild =
16909 CanTreeCombineApply && (IsTreeWithCombinableChildren(Op0) ||
16910 IsTreeWithCombinableChildren(Op1));
16919 if (CanTreeCombineApply && !HasCombinableTreeChild) {
16957 if (
Known.isNonZero() &&
Known.Zero.getBoolValue())
16965 if (
SDValue Med3 = performIntMed3ImmCombine(
16970 if (
SDValue Med3 = performIntMed3ImmCombine(
16976 if (
SDValue Med3 = performIntMed3ImmCombine(
16981 if (
SDValue Med3 = performIntMed3ImmCombine(
16994 (
Opc == AMDGPUISD::FMIN_LEGACY &&
16995 Op0.
getOpcode() == AMDGPUISD::FMAX_LEGACY)) &&
16996 (VT == MVT::f32 || VT == MVT::f64 ||
16997 (VT == MVT::f16 && Subtarget->has16BitInsts()) ||
16998 (VT == MVT::bf16 && Subtarget->hasBF16PackedInsts()) ||
16999 (VT == MVT::v2bf16 && Subtarget->hasBF16PackedInsts()) ||
17000 (VT == MVT::v2f16 && Subtarget->hasVOP3PInsts())) &&
17002 if (
SDValue Res = performFPMed3ImmCombine(DAG, SDLoc(
N), Op0, Op1,
17003 N->getFlags().hasNoNaNs()))
17010 const SDNodeFlags
Flags =
N->getFlags();
17012 !Subtarget->hasIEEEMinimumMaximumInsts() &&
17016 return DAG.
getNode(NewOpc, SDLoc(
N), VT, Op0, Op1, Flags);
17026 return (CA->isPosZero() && CB->isOne()) ||
17027 (CA->isOne() && CB->isPosZero());
17036 DAGCombinerInfo &DCI)
const {
17037 EVT VT =
N->getValueType(0);
17041 SelectionDAG &DAG = DCI.DAG;
17052 return DAG.
getNode(AMDGPUISD::CLAMP, SL, VT, Src2);
17056 const SIMachineFunctionInfo *
Info = MF.
getInfo<SIMachineFunctionInfo>();
17060 if (
Info->getMode().DX10Clamp) {
17073 return DAG.
getNode(AMDGPUISD::CLAMP, SL, VT, Src0);
17080 DAGCombinerInfo &DCI)
const {
17083 if (
Src0.isUndef() &&
Src1.isUndef())
17084 return DCI.DAG.getUNDEF(
N->getValueType(0));
17091SITargetLowering::performExponentOnlyScaleCombine(
SDNode *
N,
17092 DAGCombinerInfo &DCI)
const {
17093 std::optional<unsigned> ScaleArgIdx =
17099 unsigned OpIdx = *ScaleArgIdx + 1;
17100 SelectionDAG &DAG = DCI.DAG;
17107 Ops[OpIdx] = DemandedScale;
17121 bool IsDivergentIdx,
17123 if (Subtarget->getCLOpts().use_divergent_register_indexing)
17126 unsigned VecSize = EltSize * NumElem;
17129 if (VecSize <= 64 && EltSize < 32)
17138 if (IsDivergentIdx)
17142 unsigned NumInsts = NumElem +
17143 ((EltSize + 31) / 32) * NumElem ;
17147 if (Subtarget->useVGPRIndexMode())
17148 return NumInsts <= 16;
17152 if (Subtarget->hasMovrel())
17153 return NumInsts <= 15;
17159 SDValue Idx =
N->getOperand(
N->getNumOperands() - 1);
17170 EltSize, NumElem, Idx->isDivergent(),
getSubtarget());
17174SITargetLowering::performExtractVectorEltCombine(
SDNode *
N,
17175 DAGCombinerInfo &DCI)
const {
17181 EVT ResVT =
N->getValueType(0);
17205 if (!
C ||
C->getZExtValue() != 0x1f)
17221 if (Vec.
hasOneUse() && DCI.isBeforeLegalize() && VecEltVT == ResVT) {
17249 DCI.AddToWorklist(Elt0.
getNode());
17250 DCI.AddToWorklist(Elt1.
getNode());
17281 if (KImm && KImm->getValueType(0).getSizeInBits() == 64) {
17282 uint64_t KImmValue = KImm->getZExtValue();
17284 (KImmValue >> (32 *
Idx->getZExtValue())) & 0xffffffff, SL, MVT::i32);
17287 if (KFPImm && KFPImm->getValueType(0).getSizeInBits() == 64) {
17289 KFPImm->getValueAPF().bitcastToAPInt().getZExtValue();
17290 return DAG.
getConstant((KFPImmValue >> (32 *
Idx->getZExtValue())) &
17296 if (!DCI.isBeforeLegalize())
17303 VecSize > 32 && VecSize % 32 == 0 && Idx) {
17306 unsigned BitIndex =
Idx->getZExtValue() * VecEltSize;
17307 unsigned EltIdx = BitIndex / 32;
17308 unsigned LeftoverBitIdx = BitIndex % 32;
17312 DCI.AddToWorklist(Cast.
getNode());
17316 DCI.AddToWorklist(Elt.
getNode());
17319 DCI.AddToWorklist(Srl.
getNode());
17323 DCI.AddToWorklist(Trunc.
getNode());
17325 if (VecEltVT == ResVT) {
17337SITargetLowering::performInsertVectorEltCombine(
SDNode *
N,
17338 DAGCombinerInfo &DCI)
const {
17352 SelectionDAG &DAG = DCI.DAG;
17355 EVT IdxVT =
Idx.getValueType();
17372 Src.getOperand(0).getValueType() == MVT::f16) {
17373 return Src.getOperand(0);
17377 APFloat Val = CFP->getValueAPF();
17378 bool LosesInfo =
true;
17388 DAGCombinerInfo &DCI)
const {
17389 assert(Subtarget->has16BitInsts() && !Subtarget->hasMed3_16() &&
17390 "combine only useful on gfx8");
17392 SDValue TruncSrc =
N->getOperand(0);
17393 EVT VT =
N->getValueType(0);
17394 if (VT != MVT::f16)
17397 if (TruncSrc.
getOpcode() != AMDGPUISD::FMED3 ||
17401 SelectionDAG &DAG = DCI.DAG;
17432unsigned SITargetLowering::getFusedOpcode(
const SelectionDAG &DAG,
17434 const SDNode *N1)
const {
17439 if (((VT == MVT::f32 &&
17441 (VT == MVT::f16 && Subtarget->hasMadF16() &&
17458 EVT VT =
N->getValueType(0);
17459 if (VT != MVT::i32 && VT != MVT::i64)
17465 unsigned Opc =
N->getOpcode();
17520 if (!Const ||
Hi_32(Const->getZExtValue()) !=
uint32_t(-1))
17539 DAGCombinerInfo &DCI)
const {
17542 SelectionDAG &DAG = DCI.DAG;
17543 EVT VT =
N->getValueType(0);
17553 if (!
N->isDivergent() && Subtarget->hasSMulHi())
17557 if (NumBits <= 32 || NumBits > 64)
17568 if (!Subtarget->hasFullRate64Ops()) {
17569 unsigned NumUsers = 0;
17570 for (SDNode *User :
LHS->
users()) {
17573 if (!
User->isAnyAdd())
17597 bool MulSignedLo =
false;
17598 if (!MulLHSUnsigned32 || !MulRHSUnsigned32) {
17607 if (VT != MVT::i64) {
17630 getMad64_32(DAG, SL, MVT::i64, MulLHSLo, MulRHSLo, AddRHS, MulSignedLo);
17632 if (!MulSignedLo && (!MulLHSUnsigned32 || !MulRHSUnsigned32)) {
17633 auto [AccumLo, AccumHi] = DAG.
SplitScalar(Accum, SL, MVT::i32, MVT::i32);
17635 if (!MulLHSUnsigned32) {
17642 if (!MulRHSUnsigned32) {
17653 if (VT != MVT::i64)
17659SITargetLowering::foldAddSub64WithZeroLowBitsTo32(
SDNode *
N,
17660 DAGCombinerInfo &DCI)
const {
17670 SelectionDAG &DAG = DCI.DAG;
17685 unsigned Opcode =
N->getOpcode();
17689 DAG.
getNode(Opcode, SL, MVT::i32,
Hi, ConstHi32,
N->getFlags());
17700static std::optional<ByteProvider<SDValue>>
17703 if (!Byte0 || Byte0->isConstantZero()) {
17704 return std::nullopt;
17707 if (Byte1 && !Byte1->isConstantZero()) {
17708 return std::nullopt;
17714 unsigned FirstCs =
First & 0x0c0c0c0c;
17715 unsigned SecondCs = Second & 0x0c0c0c0c;
17716 unsigned FirstNoCs =
First & ~0x0c0c0c0c;
17717 unsigned SecondNoCs = Second & ~0x0c0c0c0c;
17719 assert((FirstCs & 0xFF) | (SecondCs & 0xFF));
17720 assert((FirstCs & 0xFF00) | (SecondCs & 0xFF00));
17721 assert((FirstCs & 0xFF0000) | (SecondCs & 0xFF0000));
17722 assert((FirstCs & 0xFF000000) | (SecondCs & 0xFF000000));
17724 return (FirstNoCs | SecondNoCs) | (FirstCs & SecondCs);
17738 assert(Src0.Src.has_value() && Src1.Src.has_value());
17741 Src0s.
push_back({*Src0.Src, ((Src0.SrcOffset % 4) << 24) + 0x0c0c0c,
17742 Src0.SrcOffset / 4});
17743 Src1s.
push_back({*Src1.Src, ((Src1.SrcOffset % 4) << 24) + 0x0c0c0c,
17744 Src1.SrcOffset / 4});
17748 for (
int BPI = 0; BPI < 2; BPI++) {
17751 BPP = {Src1, Src0};
17753 unsigned ZeroMask = 0x0c0c0c0c;
17754 unsigned FMask = 0xFF << (8 * (3 - Step));
17756 unsigned FirstMask =
17757 (BPP.first.SrcOffset % 4) << (8 * (3 - Step)) | (ZeroMask & ~FMask);
17758 unsigned SecondMask =
17759 (BPP.second.SrcOffset % 4) << (8 * (3 - Step)) | (ZeroMask & ~FMask);
17763 int FirstGroup = -1;
17764 for (
int I = 0;
I < 2;
I++) {
17766 auto MatchesFirst = [&BPP](
DotSrc &IterElt) {
17767 return IterElt.SrcOp == *BPP.first.Src &&
17768 (IterElt.DWordOffset == (BPP.first.SrcOffset / 4));
17772 if (Match != Srcs.
end()) {
17773 Match->PermMask =
addPermMasks(FirstMask, Match->PermMask);
17778 if (FirstGroup != -1) {
17780 auto MatchesSecond = [&BPP](
DotSrc &IterElt) {
17781 return IterElt.SrcOp == *BPP.second.Src &&
17782 (IterElt.DWordOffset == (BPP.second.SrcOffset / 4));
17785 if (Match != Srcs.
end()) {
17786 Match->PermMask =
addPermMasks(SecondMask, Match->PermMask);
17788 Srcs.
push_back({*BPP.second.Src, SecondMask, BPP.second.SrcOffset / 4});
17796 unsigned ZeroMask = 0x0c0c0c0c;
17797 unsigned FMask = 0xFF << (8 * (3 - Step));
17801 ((Src0.SrcOffset % 4) << (8 * (3 - Step)) | (ZeroMask & ~FMask)),
17802 Src0.SrcOffset / 4});
17805 ((Src1.SrcOffset % 4) << (8 * (3 - Step)) | (ZeroMask & ~FMask)),
17806 Src1.SrcOffset / 4});
17814 if (Srcs.
size() == 1) {
17815 auto *Elt = Srcs.
begin();
17819 if (Elt->PermMask == 0x3020100)
17822 return DAG.
getNode(AMDGPUISD::PERM, SL, MVT::i32, EltOp, EltOp,
17826 auto *FirstElt = Srcs.
begin();
17827 auto *SecondElt = std::next(FirstElt);
17834 auto FirstMask = FirstElt->PermMask;
17835 auto SecondMask = SecondElt->PermMask;
17837 unsigned FirstCs = FirstMask & 0x0c0c0c0c;
17838 unsigned FirstPlusFour = FirstMask | 0x04040404;
17841 FirstMask = (FirstPlusFour & 0x0F0F0F0F) | FirstCs;
17853 FirstElt = std::next(SecondElt);
17854 if (FirstElt == Srcs.
end())
17857 SecondElt = std::next(FirstElt);
17860 if (SecondElt == Srcs.
end()) {
17865 DAG.
getNode(AMDGPUISD::PERM, SL, MVT::i32, EltOp, EltOp,
17866 DAG.
getConstant(FirstElt->PermMask, SL, MVT::i32)));
17872 return Perms.
size() == 2
17878 for (
auto &[EntryVal, EntryMask, EntryOffset] : Srcs) {
17879 EntryMask = EntryMask >> ((4 - ChainLength) * 8);
17880 auto ZeroMask = ChainLength == 2 ? 0x0c0c0000 : 0x0c000000;
17881 EntryMask += ZeroMask;
17886 auto Opcode =
Op.getOpcode();
17888 return (Opcode ==
ISD::MUL || Opcode == AMDGPUISD::MUL_U24 ||
17889 Opcode == AMDGPUISD::MUL_I24);
17892static std::optional<bool>
17903 bool S0IsSigned = Known0.countMinLeadingOnes() > 0;
17906 bool S1IsSigned = Known1.countMinLeadingOnes() > 0;
17908 assert(!(S0IsUnsigned && S0IsSigned));
17909 assert(!(S1IsUnsigned && S1IsSigned));
17917 if ((S0IsUnsigned && S1IsUnsigned) || (S0IsSigned && S1IsSigned))
17923 if ((S0IsUnsigned && S1IsSigned) || (S0IsSigned && S1IsUnsigned))
17924 return std::nullopt;
17936 if ((S0IsSigned && !(S1IsSigned || S1IsUnsigned)) ||
17937 ((S1IsSigned && !(S0IsSigned || S0IsUnsigned))))
17942 if ((!(S1IsSigned || S1IsUnsigned) && !(S0IsSigned || S0IsUnsigned)))
17948 if ((S0IsUnsigned && !(S1IsSigned || S1IsUnsigned)) ||
17949 ((S1IsUnsigned && !(S0IsSigned || S0IsUnsigned))))
17950 return std::nullopt;
17956 DAGCombinerInfo &DCI)
const {
17957 SelectionDAG &DAG = DCI.DAG;
17958 EVT VT =
N->getValueType(0);
17964 if (Subtarget->hasMad64_32()) {
17970 if (
SDValue V = reassociateScalarOps(
N, DAG)) {
17974 if (VT == MVT::i64) {
17982 (Subtarget->hasDot1Insts() || Subtarget->hasDot8Insts())) {
17984 std::optional<bool> IsSigned;
17990 int ChainLength = 0;
17991 for (
int I = 0;
I < 4;
I++) {
18003 TempNode->getOperand(MulIdx), *Src0, *Src1,
18004 TempNode->getOperand(MulIdx)->getOperand(0),
18005 TempNode->getOperand(MulIdx)->getOperand(1), DAG);
18009 IsSigned = *IterIsSigned;
18010 if (*IterIsSigned != *IsSigned)
18013 auto AddIdx = 1 - MulIdx;
18016 if (
I == 2 &&
isMul(TempNode->getOperand(AddIdx))) {
18017 Src2s.
push_back(TempNode->getOperand(AddIdx));
18027 TempNode->getOperand(AddIdx), *Src0, *Src1,
18028 TempNode->getOperand(AddIdx)->getOperand(0),
18029 TempNode->getOperand(AddIdx)->getOperand(1), DAG);
18033 if (*IterIsSigned != *IsSigned)
18037 ChainLength =
I + 2;
18041 TempNode = TempNode->getOperand(AddIdx);
18043 ChainLength =
I + 1;
18045 if (TempNode.getOpcode() !=
ISD::ADD)
18047 LHS = TempNode->getOperand(0);
18048 RHS = TempNode->getOperand(1);
18051 if (ChainLength < 2)
18057 if (ChainLength < 4) {
18067 bool UseOriginalSrc =
false;
18068 if (ChainLength == 4 && Src0s.
size() == 1 && Src1s.
size() == 1 &&
18069 Src0s.
begin()->PermMask == Src1s.
begin()->PermMask &&
18070 Src0s.
begin()->SrcOp.getValueSizeInBits() >= 32 &&
18071 Src1s.
begin()->SrcOp.getValueSizeInBits() >= 32) {
18072 SmallVector<unsigned, 4> SrcBytes;
18073 auto Src0Mask = Src0s.
begin()->PermMask;
18074 SrcBytes.
push_back((Src0Mask >> 24) & 0xFF);
18075 bool UniqueEntries =
true;
18076 for (
auto I = 1;
I < 4;
I++) {
18077 auto NextByte = (Src0Mask >> ((3 -
I) * 8)) & 0xFF;
18080 UniqueEntries =
false;
18086 if (UniqueEntries) {
18087 UseOriginalSrc =
true;
18089 auto *FirstElt = Src0s.
begin();
18093 auto *SecondElt = Src1s.
begin();
18095 SecondElt->DWordOffset);
18104 if (!UseOriginalSrc) {
18111 DAG.
getExtOrTrunc(*IsSigned, Src2s[ChainLength - 1], SL, MVT::i32);
18114 : Intrinsic::amdgcn_udot4,
18124 if (VT != MVT::i32 || !DCI.isAfterLegalizeDAG())
18129 unsigned Opc =
LHS.getOpcode();
18141 auto Cond =
RHS.getOperand(0);
18146 SDVTList VTList = DAG.
getVTList(MVT::i32, MVT::i1);
18163 DAGCombinerInfo &DCI)
const {
18164 SelectionDAG &DAG = DCI.DAG;
18166 EVT VT =
N->getValueType(0);
18179 SDNodeFlags ShlFlags = N1->
getFlags();
18183 SDNodeFlags NewShlFlags =
18188 DCI.AddToWorklist(Inner.
getNode());
18195 if (Subtarget->hasMad64_32()) {
18205 if (VT == MVT::i64) {
18219 if (!YIsConstant && !ZIsConstant && !
X->isDivergent() &&
18220 Y->isDivergent() !=
Z->isDivergent()) {
18229 if (
Y->isDivergent())
18232 SDNodeFlags ReassocFlags =
18235 DCI.AddToWorklist(UniformInner.
getNode());
18247 DAGCombinerInfo &DCI)
const {
18248 SelectionDAG &DAG = DCI.DAG;
18249 EVT VT =
N->getValueType(0);
18251 if (VT == MVT::i64) {
18256 if (VT != MVT::i32)
18265 unsigned Opc =
RHS.getOpcode();
18272 auto Cond =
RHS.getOperand(0);
18277 SDVTList VTList = DAG.
getVTList(MVT::i32, MVT::i1);
18303 ConstantSDNode *ShiftAmt =
18305 unsigned BitWidth =
X.getValueType().getScalarSizeInBits();
18316 DAGCombinerInfo &DCI)
const {
18320 SelectionDAG &DAG = DCI.DAG;
18321 EVT VT =
N->getValueType(0);
18333 if (
A ==
LHS.getOperand(1)) {
18334 unsigned FusedOp = getFusedOpcode(DAG,
N,
LHS.getNode());
18335 if (FusedOp != 0) {
18337 return DAG.
getNode(FusedOp, SL, VT,
A, Two,
RHS);
18345 if (
A ==
RHS.getOperand(1)) {
18346 unsigned FusedOp = getFusedOpcode(DAG,
N,
RHS.getNode());
18347 if (FusedOp != 0) {
18349 return DAG.
getNode(FusedOp, SL, VT,
A, Two,
LHS);
18358 DAGCombinerInfo &DCI)
const {
18362 SelectionDAG &DAG = DCI.DAG;
18364 EVT VT =
N->getValueType(0);
18377 if (
A ==
LHS.getOperand(1)) {
18378 unsigned FusedOp = getFusedOpcode(DAG,
N,
LHS.getNode());
18379 if (FusedOp != 0) {
18383 return DAG.
getNode(FusedOp, SL, VT,
A, Two, NegRHS);
18392 if (
A ==
RHS.getOperand(1)) {
18393 unsigned FusedOp = getFusedOpcode(DAG,
N,
RHS.getNode());
18394 if (FusedOp != 0) {
18396 return DAG.
getNode(FusedOp, SL, VT,
A, NegTwo,
LHS);
18405 DAGCombinerInfo &DCI)
const {
18406 SelectionDAG &DAG = DCI.DAG;
18408 EVT VT =
N->getValueType(0);
18410 if (VT != MVT::f16 && VT != MVT::bf16)
18416 SDNodeFlags
Flags =
N->getFlags();
18417 SDNodeFlags RHSFlags =
RHS->getFlags();
18423 bool IsNegative =
false;
18424 if (CLHS->
isOne() || (IsNegative = CLHS->isMinusOne())) {
18433 Rsq = DAG.
getNode(AMDGPUISD::RSQ, SL, VT, SqrtOp, Flags);
18434 }
else if (VT == MVT::f16) {
18443 DAG.
getNode(AMDGPUISD::RSQ, SL, MVT::f32, Ext, Flags);
18460 DAGCombinerInfo &DCI)
const {
18461 SelectionDAG &DAG = DCI.DAG;
18462 EVT VT =
N->getValueType(0);
18466 if (!
N->isDivergent() &&
getSubtarget()->hasSALUFloatInsts() &&
18467 (ScalarVT == MVT::f32 || ScalarVT == MVT::f16)) {
18482 if ((ScalarVT == MVT::f64 || ScalarVT == MVT::f32 || ScalarVT == MVT::f16) &&
18487 const ConstantFPSDNode *FalseNode =
18497 if (ScalarVT == MVT::f32 &&
18503 if (TrueNodeExpVal == INT_MIN)
18506 if (FalseNodeExpVal == INT_MIN)
18526 DAGCombinerInfo &DCI)
const {
18527 SelectionDAG &DAG = DCI.DAG;
18528 EVT VT =
N->getValueType(0);
18531 if (!Subtarget->hasDot10Insts() || VT != MVT::f32)
18559 bool AllowInaccuracy =
N->getFlags().hasApproximateFuncs() &&
18560 FMA->getFlags().hasApproximateFuncs();
18561 if (!AllowInaccuracy) {
18564 if (Subtarget->dot2UnconditionalFlush()) {
18576 if (
N->getFlags().hasAllowContract() &&
FMA->getFlags().hasAllowContract()) {
18612 if (Vec1 == Vec2 || Vec3 == Vec4)
18618 if ((Vec1 == Vec3 && Vec2 == Vec4) || (Vec1 == Vec4 && Vec2 == Vec3)) {
18619 return DAG.
getNode(AMDGPUISD::FDOT2, SL, MVT::f32, Vec1, Vec2, FMAAcc,
18662 EVT VT =
LHS.getValueType();
18663 assert(VT == MVT::f64 &&
"Incorrect operand type!");
18710 const std::optional<bool> KnownEq =
18751 const std::optional<bool> KnownUge =
18793 const std::optional<bool> KnownUle =
18816 DAGCombinerInfo &DCI)
const {
18817 SelectionDAG &DAG = DCI.DAG;
18822 EVT VT =
LHS.getValueType();
18851 return LHS.getOperand(0);
18865 const APInt &CT =
LHS.getConstantOperandAPInt(1);
18866 const APInt &CF =
LHS.getConstantOperandAPInt(2);
18871 return DAG.
getNOT(SL,
LHS.getOperand(0), MVT::i1);
18874 return LHS.getOperand(0);
18880 if (VT == MVT::i64) {
18883 const KnownBits LHSKnownLo32 = LHSKnown.
trunc(32);
18884 const KnownBits RHSKnownLo32 = RHSKnown.
trunc(32);
18911 const std::optional<bool> KnownEq =
18919 const std::optional<bool> KnownEq =
18930 const std::optional<bool> KnownUge =
18950 const std::optional<bool> KnownUle =
18974 const KnownBits LHSKnownHi32 = LHSKnown.
extractBits(32, 32);
18975 const KnownBits RHSKnownHi32 = RHSKnown.
extractBits(32, 32);
18976 const std::optional<bool> KnownEq =
18979 if (KnownEq ==
true) {
18985 return DAG.
getSetCC(SL,
N->getValueType(0), LHSLo, RHSLo, NewCC);
19017 DAG.
getVTList(MVT::i32, MVT::i1), {Op0Lo, Op1Lo});
19022 {Op0Hi, Op1Hi, CarryInHi});
19032 DCI.CombineTo(
LHS.getNode(), Result);
19036 if (VT != MVT::f32 && VT != MVT::f64 &&
19037 (!Subtarget->has16BitInsts() || VT != MVT::f16))
19052 const unsigned IsInfMask =
19054 const unsigned IsFiniteMask =
19059 return DAG.
getNode(AMDGPUISD::FP_CLASS, SL, MVT::i1,
LHS.getOperand(0),
19064 if (VT == MVT::f64) {
19075SITargetLowering::performCvtF32UByteNCombine(
SDNode *
N,
19076 DAGCombinerInfo &DCI)
const {
19077 SelectionDAG &DAG = DCI.DAG;
19079 unsigned Offset =
N->getOpcode() - AMDGPUISD::CVT_F32_UBYTE0;
19098 unsigned ShiftOffset = 8 *
Offset;
19100 ShiftOffset -=
C->getZExtValue();
19102 ShiftOffset +=
C->getZExtValue();
19104 if (ShiftOffset < 32 && (ShiftOffset % 8) == 0) {
19105 return DAG.
getNode(AMDGPUISD::CVT_F32_UBYTE0 + ShiftOffset / 8, SL,
19106 MVT::f32, Shifted);
19117 DCI.AddToWorklist(
N);
19124 return DAG.
getNode(
N->getOpcode(), SL, MVT::f32, DemandedSrc);
19130 DAGCombinerInfo &DCI)
const {
19139 (
F.isNaN() && MF.
getInfo<SIMachineFunctionInfo>()->getMode().DX10Clamp)) {
19140 return DCI.DAG.getConstantFP(
Zero, SDLoc(
N),
N->getValueType(0));
19145 return DCI.DAG.getConstantFP(One, SDLoc(
N),
N->getValueType(0));
19147 return getCanonicalConstantFP(DCI.DAG, SDLoc(
N),
N->getValueType(0),
F);
19157 FrexpInput =
V.getOperand(0);
19167SITargetLowering::performFrexpSelectCombine(
SDNode *
N,
19168 DAGCombinerInfo &DCI)
const {
19170 if (Subtarget->hasFractBug())
19182 bool CondSelectsZero;
19190 CondSelectsZero =
true;
19191 }
else if (
isFrexpExp(TrueVal, FrexpInput)) {
19194 CondSelectsZero =
false;
19206 bool IsNonFiniteTest =
false;
19218 bool LHSMatchesFrexp =
19219 (CondLHS == FrexpInput) ||
19220 (LHSIsFabs &&
peekFPSignOps(FAbsInput) == FrexpInputStripped) ||
19222 bool RHSMatchesFrexp = (CondRHS == FrexpInput) ||
19230 SelectionDAG &DAG = DCI.DAG;
19231 if (LHSMatchesFrexp &&
19233 IsNonFiniteTest = CondSelectsZero;
19235 IsNonFiniteTest = CondSelectsZero;
19242 IsNonFiniteTest = CondSelectsZero;
19249 IsNonFiniteTest = !CondSelectsZero;
19255 SelectionDAG &DAG = DCI.DAG;
19256 if (LHSMatchesFrexp &&
19258 IsNonFiniteTest = !CondSelectsZero;
19260 IsNonFiniteTest = !CondSelectsZero;
19264 if (!IsNonFiniteTest)
19272 DAGCombinerInfo &DCI)
const {
19293 bool isFloatingPoint =
LHS.getValueType().isFloatingPoint();
19294 bool isInteger =
LHS.getValueType().isInteger();
19297 if (!isFloatingPoint && !isInteger)
19302 bool isNonEquality =
19304 if (!isEquality && !isNonEquality)
19321 if (isFloatingPoint) {
19323 if (!Val.
isNormal() || Subtarget->getInstrInfo()->isInlineConstant(Val))
19326 const std::optional<int64_t> Val =
19335 if (!(isEquality && TrueVal == ConstVal) &&
19336 !(isNonEquality && FalseVal == ConstVal))
19340 if (isFloatingPoint && isNonEquality && FalseVal == ConstVal &&
19341 !
Cond->getFlags().hasNoNaNs() && !DCI.DAG.isKnownNeverNaN(ArgVal))
19348 SelectLHS, SelectRHS);
19353 switch (
N->getOpcode()) {
19375 if (
auto Res = promoteUniformOpToI32(
SDValue(
N, 0), DCI))
19396 switch (
N->getOpcode()) {
19398 return performAddCombine(
N, DCI);
19400 return performPtrAddCombine(
N, DCI);
19402 return performSubCombine(
N, DCI);
19404 return performFAddCombine(
N, DCI);
19406 return performFSubCombine(
N, DCI);
19408 return performFDivCombine(
N, DCI);
19410 return performFMulCombine(
N, DCI);
19412 return performSetCCCombine(
N, DCI);
19414 if (
auto Res = performFrexpSelectCombine(
N, DCI))
19416 if (
auto Res = performSelectCombine(
N, DCI))
19431 case AMDGPUISD::FMIN_LEGACY:
19432 case AMDGPUISD::FMAX_LEGACY:
19433 return performMinMaxCombine(
N, DCI);
19435 return performFMACombine(
N, DCI);
19437 return performAndCombine(
N, DCI);
19439 return performOrCombine(
N, DCI);
19442 if (
N->getValueType(0) == MVT::i32 &&
N->isDivergent() &&
19443 TII->pseudoToMCOpcode(AMDGPU::V_PERM_B32_e64) != -1) {
19449 return performXorCombine(
N, DCI);
19452 return performZeroOrAnyExtendCombine(
N, DCI);
19454 return performExtendTruncShiftCombine(
N, DCI);
19456 return performSignExtendInRegCombine(
N, DCI);
19457 case AMDGPUISD::FP_CLASS:
19458 return performClassCombine(
N, DCI);
19460 return performFCanonicalizeCombine(
N, DCI);
19461 case AMDGPUISD::RCP:
19462 return performRcpCombine(
N, DCI);
19464 case AMDGPUISD::FRACT:
19465 case AMDGPUISD::RSQ:
19466 case AMDGPUISD::RCP_LEGACY:
19467 case AMDGPUISD::RCP_IFLAG:
19468 case AMDGPUISD::RSQ_CLAMP: {
19477 return performUCharToFloatCombine(
N, DCI);
19479 return performFCopySignCombine(
N, DCI);
19480 case AMDGPUISD::CVT_F32_UBYTE0:
19481 case AMDGPUISD::CVT_F32_UBYTE1:
19482 case AMDGPUISD::CVT_F32_UBYTE2:
19483 case AMDGPUISD::CVT_F32_UBYTE3:
19484 return performCvtF32UByteNCombine(
N, DCI);
19485 case AMDGPUISD::FMED3:
19486 return performFMed3Combine(
N, DCI);
19487 case AMDGPUISD::CVT_PKRTZ_F16_F32:
19488 return performCvtPkRTZCombine(
N, DCI);
19490 if (
SDValue Res = performExponentOnlyScaleCombine(
N, DCI))
19493 case AMDGPUISD::CLAMP:
19494 return performClampCombine(
N, DCI);
19497 EVT VT =
N->getValueType(0);
19500 if (VT == MVT::v2i16 || VT == MVT::v2f16 || VT == MVT::v2bf16) {
19503 EVT EltVT = Src.getValueType();
19504 if (EltVT != MVT::i16)
19514 return performExtractVectorEltCombine(
N, DCI);
19516 return performInsertVectorEltCombine(
N, DCI);
19518 return performFPRoundCombine(
N, DCI);
19527 return performMemSDNodeCombine(MemNode, DCI);
19558 unsigned Opcode =
Node->getMachineOpcode();
19561 int D16Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::d16) - 1;
19562 if (D16Idx >= 0 &&
Node->getConstantOperandVal(D16Idx))
19565 SDNode *
Users[5] = {
nullptr};
19567 unsigned DmaskIdx =
19568 AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::dmask) - 1;
19569 unsigned OldDmask =
Node->getConstantOperandVal(DmaskIdx);
19570 unsigned NewDmask = 0;
19571 unsigned TFEIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::tfe) - 1;
19572 unsigned LWEIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::lwe) - 1;
19573 bool UsesTFC = (int(TFEIdx) >= 0 &&
Node->getConstantOperandVal(TFEIdx)) ||
19574 (
int(LWEIdx) >= 0 &&
Node->getConstantOperandVal(LWEIdx));
19575 unsigned TFCLane = 0;
19576 bool HasChain =
Node->getNumValues() > 1;
19578 if (OldDmask == 0) {
19586 TFCLane = OldBitsSet;
19590 for (SDUse &Use :
Node->uses()) {
19593 if (
Use.getResNo() != 0)
19596 SDNode *
User =
Use.getUser();
19599 if (!
User->isMachineOpcode() ||
19600 User->getMachineOpcode() != TargetOpcode::EXTRACT_SUBREG)
19612 if (UsesTFC && Lane == TFCLane) {
19617 for (
unsigned i = 0, Dmask = OldDmask; (i <= Lane) && (Dmask != 0); i++) {
19619 Dmask &= ~(1 << Comp);
19627 NewDmask |= 1 << Comp;
19632 bool NoChannels = !NewDmask;
19639 if (OldBitsSet == 1)
19645 if (NewDmask == OldDmask)
19654 unsigned NewChannels = BitsSet + UsesTFC;
19658 assert(NewOpcode != -1 &&
19659 NewOpcode !=
static_cast<int>(
Node->getMachineOpcode()) &&
19660 "failed to find equivalent MIMG op");
19668 MVT SVT =
Node->getValueType(0).getVectorElementType().getSimpleVT();
19670 MVT ResultVT = NewChannels == 1
19673 : NewChannels == 5 ? 8
19675 SDVTList NewVTList =
19678 MachineSDNode *NewNode =
19687 if (NewChannels == 1) {
19697 for (
unsigned i = 0, Idx = AMDGPU::sub0; i < 5; ++i) {
19702 if (i || !NoChannels)
19707 if (NewUser != User) {
19717 Idx = AMDGPU::sub1;
19720 Idx = AMDGPU::sub2;
19723 Idx = AMDGPU::sub3;
19726 Idx = AMDGPU::sub4;
19737 Op =
Op.getOperand(0);
19758 MRI.createVirtualRegister(&AMDGPU::VReg_1RegClass), MVT::i1);
19762 Node->getOperand(0), SL, VReg, SrcVal,
19768 return ToResultReg.
getNode();
19773 for (
unsigned i = 0; i <
Node->getNumOperands(); ++i) {
19775 Ops.push_back(
Node->getOperand(i));
19781 Node->getOperand(i).getValueType(),
19782 Node->getOperand(i)),
19794 unsigned Opcode =
Node->getMachineOpcode();
19796 if (
TII->isImage(Opcode) && !
TII->get(Opcode).mayStore() &&
19797 !
TII->isGather4(Opcode) &&
19799 return adjustWritemask(
Node, DAG);
19802 if (Opcode == AMDGPU::INSERT_SUBREG || Opcode == AMDGPU::REG_SEQUENCE) {
19808 case AMDGPU::V_DIV_SCALE_F32_e64:
19809 case AMDGPU::V_DIV_SCALE_F64_e64: {
19817 if ((Src0.isMachineOpcode() &&
19818 Src0.getMachineOpcode() != AMDGPU::IMPLICIT_DEF) &&
19819 (Src0 == Src1 || Src0 == Src2))
19822 MVT VT = Src0.getValueType().getSimpleVT();
19834 if (Src0.isMachineOpcode() &&
19835 Src0.getMachineOpcode() == AMDGPU::IMPLICIT_DEF) {
19836 if (Src1.isMachineOpcode() &&
19837 Src1.getMachineOpcode() != AMDGPU::IMPLICIT_DEF)
19839 else if (Src2.isMachineOpcode() &&
19840 Src2.getMachineOpcode() != AMDGPU::IMPLICIT_DEF)
19843 assert(Src1.getMachineOpcode() == AMDGPU::IMPLICIT_DEF);
19875 AMDGPU::getNamedOperandIdx(
MI.getOpcode(), AMDGPU::OpName::vdata);
19876 unsigned InitIdx = 0;
19878 if (
TII->isImage(
MI)) {
19886 unsigned TFEVal = TFE ? TFE->
getImm() : 0;
19887 unsigned LWEVal = LWE ? LWE->
getImm() : 0;
19888 unsigned D16Val = D16 ? D16->getImm() : 0;
19890 if (!TFEVal && !LWEVal)
19901 assert(MO_Dmask &&
"Expected dmask operand in instruction");
19903 unsigned dmask = MO_Dmask->
getImm();
19908 bool Packed = !Subtarget->hasUnpackedD16VMem();
19910 InitIdx = D16Val && Packed ? ((ActiveLanes + 1) >> 1) + 1 : ActiveLanes + 1;
19917 uint32_t DstSize =
TRI.getRegSizeInBits(*DstRC) / 32;
19918 if (DstSize < InitIdx)
19922 InitIdx =
TRI.getRegSizeInBits(*DstRC) / 32;
19930 Register PrevDst =
MRI.cloneVirtualRegister(
MI.getOperand(DstIdx).getReg());
19931 unsigned NewDst = 0;
19936 unsigned SizeLeft = Subtarget->usePRTStrictNull() ? InitIdx : 1;
19937 unsigned CurrIdx = Subtarget->usePRTStrictNull() ? 0 : (InitIdx - 1);
19940 for (; SizeLeft; SizeLeft--, CurrIdx++) {
19941 NewDst =
MRI.createVirtualRegister(
TII->getOpRegClass(
MI, DstIdx));
19943 Register SubReg =
MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
19961 MI.tieOperands(DstIdx,
MI.getNumOperands() - 1);
19973 if (
TII->isVOP3(
MI.getOpcode())) {
19975 TII->legalizeOperandsVOP3(
MRI,
MI);
19977 if (
TII->isMAI(
MI)) {
19982 int Src0Idx = AMDGPU::getNamedOperandIdx(
MI.getOpcode(),
19983 AMDGPU::OpName::scale_src0);
19984 if (Src0Idx != -1) {
19985 int Src1Idx = AMDGPU::getNamedOperandIdx(
MI.getOpcode(),
19986 AMDGPU::OpName::scale_src1);
19987 if (
TII->usesConstantBus(
MRI,
MI, Src0Idx) &&
19988 TII->usesConstantBus(
MRI,
MI, Src1Idx))
19989 TII->legalizeOpWithMove(
MI, Src1Idx);
19996 if (
TII->isImage(
MI))
19997 TII->enforceOperandRCAlignment(
MI, AMDGPU::OpName::vaddr);
20039 uint64_t RsrcDword2And3)
const {
20069std::pair<unsigned, const TargetRegisterClass *>
20076 if (Constraint.
size() == 1) {
20080 if (VT == MVT::Other)
20083 switch (Constraint[0]) {
20090 RC = &AMDGPU::SReg_32RegClass;
20093 RC = &AMDGPU::SGPR_64RegClass;
20098 return std::pair(0U,
nullptr);
20105 return std::pair(0U,
nullptr);
20107 RC = Subtarget->useRealTrue16Insts() ? &AMDGPU::VGPR_16RegClass
20108 : &AMDGPU::VGPR_32_Lo256RegClass;
20111 RC = Subtarget->hasVGPRMSBs()
20112 ?
TRI->getAlignedLo256VGPRClassForBitWidth(
BitWidth)
20115 return std::pair(0U,
nullptr);
20120 if (!Subtarget->hasMAIInsts())
20124 return std::pair(0U,
nullptr);
20126 RC = &AMDGPU::AGPR_32RegClass;
20131 return std::pair(0U,
nullptr);
20136 }
else if (Constraint ==
"VA" && Subtarget->hasGFX90AInsts()) {
20140 RC = &AMDGPU::AV_32RegClass;
20143 RC =
TRI->getVectorSuperClassForBitWidth(
BitWidth);
20145 return std::pair(0U,
nullptr);
20154 return std::pair(0U, RC);
20157 if (Kind !=
'\0') {
20159 RC = &AMDGPU::VGPR_32_Lo256RegClass;
20160 }
else if (Kind ==
's') {
20161 RC = &AMDGPU::SGPR_32RegClass;
20162 }
else if (Kind ==
'a') {
20163 RC = &AMDGPU::AGPR_32RegClass;
20169 return std::pair(0U,
nullptr);
20175 return std::pair(0U,
nullptr);
20179 RC =
TRI->getVGPRClassForBitWidth(Width);
20181 RC =
TRI->getSGPRClassForBitWidth(Width);
20183 RC =
TRI->getAGPRClassForBitWidth(Width);
20185 Reg =
TRI->getMatchingSuperReg(Reg, AMDGPU::sub0, RC);
20190 return std::pair(0U,
nullptr);
20192 return std::pair(Reg, RC);
20201 return std::pair(0U,
nullptr);
20202 if (RC && Idx < RC->getNumRegs())
20204 return std::pair(0U,
nullptr);
20210 Ret.second =
TRI->getPhysRegBaseClass(Ret.first);
20211 if (!Subtarget->useRealTrue16Insts() &&
20212 TRI->getRegSizeInBits(*Ret.second) == 16)
20213 return std::pair(0U,
nullptr);
20220 if (Constraint.
size() == 1) {
20221 switch (Constraint[0]) {
20231 }
else if (Constraint ==
"DA" || Constraint ==
"DB") {
20239 if (Constraint.
size() == 1) {
20240 switch (Constraint[0]) {
20250 }
else if (Constraint.
size() == 2) {
20251 if (Constraint ==
"VA")
20253 if (Constraint ==
"RF")
20264 if (ConstraintCode ==
"RF")
20279 std::vector<SDValue> &
Ops,
20294 unsigned Size =
Op.getScalarValueSizeInBits();
20298 if (
Size == 16 && !Subtarget->has16BitInsts())
20302 Val =
C->getSExtValue();
20306 Val =
C->getValueAPF().bitcastToAPInt().getSExtValue();
20310 if (
Size != 16 ||
Op.getNumOperands() != 2)
20312 if (
Op.getOperand(0).isUndef() ||
Op.getOperand(1).isUndef())
20315 Val =
C->getSExtValue();
20319 Val =
C->getValueAPF().bitcastToAPInt().getSExtValue();
20328 uint64_t Val)
const {
20329 if (Constraint.
size() == 1) {
20330 switch (Constraint[0]) {
20345 }
else if (Constraint.
size() == 2) {
20346 if (Constraint ==
"DA") {
20347 int64_t HiBits =
static_cast<int32_t
>(Val >> 32);
20348 int64_t LoBits =
static_cast<int32_t
>(Val);
20352 if (Constraint ==
"DB") {
20360 unsigned MaxSize)
const {
20361 unsigned Size = std::min<unsigned>(
Op.getScalarValueSizeInBits(), MaxSize);
20362 bool HasInv2Pi = Subtarget->hasInv2PiInlineImm();
20364 MVT VT =
Op.getSimpleValueType();
20389 switch (UnalignedClassID) {
20390 case AMDGPU::VReg_64RegClassID:
20391 return AMDGPU::VReg_64_Align2RegClassID;
20392 case AMDGPU::VReg_96RegClassID:
20393 return AMDGPU::VReg_96_Align2RegClassID;
20394 case AMDGPU::VReg_128RegClassID:
20395 return AMDGPU::VReg_128_Align2RegClassID;
20396 case AMDGPU::VReg_160RegClassID:
20397 return AMDGPU::VReg_160_Align2RegClassID;
20398 case AMDGPU::VReg_192RegClassID:
20399 return AMDGPU::VReg_192_Align2RegClassID;
20400 case AMDGPU::VReg_224RegClassID:
20401 return AMDGPU::VReg_224_Align2RegClassID;
20402 case AMDGPU::VReg_256RegClassID:
20403 return AMDGPU::VReg_256_Align2RegClassID;
20404 case AMDGPU::VReg_288RegClassID:
20405 return AMDGPU::VReg_288_Align2RegClassID;
20406 case AMDGPU::VReg_320RegClassID:
20407 return AMDGPU::VReg_320_Align2RegClassID;
20408 case AMDGPU::VReg_352RegClassID:
20409 return AMDGPU::VReg_352_Align2RegClassID;
20410 case AMDGPU::VReg_384RegClassID:
20411 return AMDGPU::VReg_384_Align2RegClassID;
20412 case AMDGPU::VReg_512RegClassID:
20413 return AMDGPU::VReg_512_Align2RegClassID;
20414 case AMDGPU::VReg_1024RegClassID:
20415 return AMDGPU::VReg_1024_Align2RegClassID;
20416 case AMDGPU::AReg_64RegClassID:
20417 return AMDGPU::AReg_64_Align2RegClassID;
20418 case AMDGPU::AReg_96RegClassID:
20419 return AMDGPU::AReg_96_Align2RegClassID;
20420 case AMDGPU::AReg_128RegClassID:
20421 return AMDGPU::AReg_128_Align2RegClassID;
20422 case AMDGPU::AReg_160RegClassID:
20423 return AMDGPU::AReg_160_Align2RegClassID;
20424 case AMDGPU::AReg_192RegClassID:
20425 return AMDGPU::AReg_192_Align2RegClassID;
20426 case AMDGPU::AReg_256RegClassID:
20427 return AMDGPU::AReg_256_Align2RegClassID;
20428 case AMDGPU::AReg_512RegClassID:
20429 return AMDGPU::AReg_512_Align2RegClassID;
20430 case AMDGPU::AReg_1024RegClassID:
20431 return AMDGPU::AReg_1024_Align2RegClassID;
20447 if (Info->isEntryFunction()) {
20454 unsigned MaxNumSGPRs = ST.getMaxNumSGPRs(MF);
20456 ? AMDGPU::SGPR_32RegClass.getRegister(MaxNumSGPRs - 1)
20457 :
TRI->getAlignedHighSGPRForRC(MF, 2,
20458 &AMDGPU::SGPR_64RegClass);
20459 Info->setSGPRForEXECCopy(SReg);
20461 assert(!
TRI->isSubRegister(Info->getScratchRSrcReg(),
20462 Info->getStackPtrOffsetReg()));
20463 if (Info->getStackPtrOffsetReg() != AMDGPU::SP_REG)
20464 MRI.replaceRegWith(AMDGPU::SP_REG, Info->getStackPtrOffsetReg());
20468 if (Info->getScratchRSrcReg() != AMDGPU::PRIVATE_RSRC_REG)
20469 MRI.replaceRegWith(AMDGPU::PRIVATE_RSRC_REG, Info->getScratchRSrcReg());
20471 if (Info->getFrameOffsetReg() != AMDGPU::FP_REG)
20472 MRI.replaceRegWith(AMDGPU::FP_REG, Info->getFrameOffsetReg());
20474 Info->limitOccupancy(MF);
20476 if (ST.isWave32() && !MF.
empty()) {
20477 for (
auto &
MBB : MF) {
20478 for (
auto &
MI :
MBB) {
20479 TII->fixImplicitOperands(
MI);
20489 if (ST.needsAlignedVGPRs()) {
20490 for (
unsigned I = 0, E =
MRI.getNumVirtRegs();
I != E; ++
I) {
20496 if (NewClassID != -1)
20497 MRI.setRegClass(Reg,
TRI->getRegClass(NewClassID));
20506 const APInt &DemandedElts,
20508 unsigned Depth)
const {
20510 unsigned Opc =
Op.getOpcode();
20513 unsigned IID =
Op.getConstantOperandVal(0);
20515 case Intrinsic::amdgcn_mbcnt_lo:
20516 case Intrinsic::amdgcn_mbcnt_hi: {
20521 Known.Zero.setBitsFrom(
20522 IID == Intrinsic::amdgcn_mbcnt_lo ? ST.getWavefrontSizeLog2() : 5);
20548 unsigned MaxValue =
20555 unsigned BFEWidth,
bool SExt,
unsigned Depth) {
20559 unsigned Src1Cst = 0;
20560 if (Src1.isImm()) {
20561 Src1Cst = Src1.getImm();
20562 }
else if (Src1.isReg()) {
20566 Src1Cst = Cst->Value.getZExtValue();
20577 if (Width >= BFEWidth)
20594 unsigned Depth)
const {
20597 switch (
MI->getOpcode()) {
20598 case AMDGPU::S_BFE_I32:
20601 case AMDGPU::S_BFE_U32:
20604 case AMDGPU::S_BFE_I64:
20607 case AMDGPU::S_BFE_U64:
20610 case AMDGPU::G_INTRINSIC:
20611 case AMDGPU::G_INTRINSIC_CONVERGENT: {
20614 case Intrinsic::amdgcn_workitem_id_x:
20617 case Intrinsic::amdgcn_workitem_id_y:
20620 case Intrinsic::amdgcn_workitem_id_z:
20623 case Intrinsic::amdgcn_mbcnt_lo:
20624 case Intrinsic::amdgcn_mbcnt_hi: {
20627 Known.Zero.setBitsFrom(IID == Intrinsic::amdgcn_mbcnt_lo
20636 case Intrinsic::amdgcn_groupstaticsize: {
20640 Known.Zero.setHighBits(
20644 case Intrinsic::amdgcn_readfirstlane:
20645 case Intrinsic::amdgcn_readlane: {
20651 case Intrinsic::amdgcn_mul_u24:
20652 case Intrinsic::amdgcn_mul_i24: {
20659 bool SelfMultiply = LHS == RHS;
20665 LHSKnown = RHSKnown;
20673 IID == Intrinsic::amdgcn_mul_i24,
20680 case AMDGPU::G_AMDGPU_BUFFER_LOAD_UBYTE:
20681 Known.Zero.setHighBits(24);
20683 case AMDGPU::G_AMDGPU_BUFFER_LOAD_USHORT:
20684 Known.Zero.setHighBits(16);
20686 case AMDGPU::G_AMDGPU_COPY_SCC_VCC:
20689 Known.Zero.setHighBits(
Known.getBitWidth() - 1);
20691 case AMDGPU::G_AMDGPU_SMED3:
20692 case AMDGPU::G_AMDGPU_UMED3: {
20693 auto [Dst, Src0, Src1, Src2] =
MI->getFirst4Regs();
20720 unsigned Depth)
const {
20729 if (
MaybeAlign RetAlign = Attrs.getRetAlignment())
20750 BlockToAlign =
ML->getHeader();
20754 if (needsFetchWindowAlignment(*BlockToAlign))
20776 if (Header->getAlignment() != PrefAlign)
20777 return Header->getAlignment();
20779 unsigned LoopSize = 0;
20784 LoopSize +=
MBB->getAlignment().value() / 2;
20787 LoopSize +=
TII->getInstSizeInBytes(
MI);
20788 if (LoopSize > 192)
20793 if (LoopSize <= 64)
20796 if (LoopSize <= 128)
20797 return CacheLineAlign;
20803 auto I = Exit->getFirstNonDebugInstr();
20804 if (
I != Exit->end() &&
I->getOpcode() == AMDGPU::S_INST_PREFETCH)
20805 return CacheLineAlign;
20814 if (PreTerm == Pre->
begin() ||
20815 std::prev(PreTerm)->getOpcode() != AMDGPU::S_INST_PREFETCH)
20819 auto ExitHead = Exit->getFirstNonDebugInstr();
20820 if (ExitHead == Exit->end() ||
20821 ExitHead->getOpcode() != AMDGPU::S_INST_PREFETCH)
20826 return CacheLineAlign;
20834 if (needsFetchWindowAlignment(*
MBB))
20839bool SITargetLowering::needsFetchWindowAlignment(
20841 if (!
getSubtarget()->hasLoopHeadInstSplitSensitivity())
20845 if (
MI.isMetaInstruction())
20848 return TII->getInstSizeInBytes(
MI) > 4;
20858 N =
N->getOperand(0).getNode();
20868 switch (
N->getOpcode()) {
20876 if (Reg.isPhysical() ||
MRI.isLiveIn(Reg))
20877 return !
TRI->isSGPRReg(
MRI, Reg);
20883 return !
TRI->isSGPRReg(
MRI, Reg);
20887 unsigned AS = L->getAddressSpace();
20894 case AMDGPUISD::REG_LOAD:
20902 case AMDGPUISD::ATOMIC_CMP_SWAP:
20903 case AMDGPUISD::BUFFER_ATOMIC_SWAP:
20904 case AMDGPUISD::BUFFER_ATOMIC_ADD:
20905 case AMDGPUISD::BUFFER_ATOMIC_SUB:
20906 case AMDGPUISD::BUFFER_ATOMIC_SMIN:
20907 case AMDGPUISD::BUFFER_ATOMIC_UMIN:
20908 case AMDGPUISD::BUFFER_ATOMIC_SMAX:
20909 case AMDGPUISD::BUFFER_ATOMIC_UMAX:
20910 case AMDGPUISD::BUFFER_ATOMIC_AND:
20911 case AMDGPUISD::BUFFER_ATOMIC_OR:
20912 case AMDGPUISD::BUFFER_ATOMIC_XOR:
20913 case AMDGPUISD::BUFFER_ATOMIC_INC:
20914 case AMDGPUISD::BUFFER_ATOMIC_DEC:
20915 case AMDGPUISD::BUFFER_ATOMIC_CMPSWAP:
20916 case AMDGPUISD::BUFFER_ATOMIC_FADD:
20917 case AMDGPUISD::BUFFER_ATOMIC_FMIN:
20918 case AMDGPUISD::BUFFER_ATOMIC_FMAX:
20924 return A->readMem() &&
A->writeMem();
20945 switch (Ty.getScalarSizeInBits()) {
20957 const APInt &DemandedElts,
20960 unsigned Depth)
const {
20961 if (
Op.getOpcode() == AMDGPUISD::CLAMP) {
20965 if (Info->getMode().DX10Clamp)
20978enum class AtomicFlushDenormalReason {
20981 IgnoreDenormalMode,
20982 FunctionFlushesDenormals
20987enum class GlobalFPAtomicLegality {
20989 AgentScopeFineGrainedRemoteMemory,
20990 EmulatedSystemScope,
20992 NoFineGrainedMemory
20999static AtomicFlushDenormalReason
21001 if (RMW->
hasMetadata(LLVMContext::MD_atomic_ignore_denormal_mode))
21002 return AtomicFlushDenormalReason::IgnoreDenormalMode;
21007 ? AtomicFlushDenormalReason::FunctionFlushesDenormals
21008 : AtomicFlushDenormalReason::IEEE;
21013 GlobalFPAtomicLegality MemLegality,
21014 AtomicFlushDenormalReason DenormReason) {
21017 if (MemScope.empty())
21018 MemScope =
"system";
21021 R <<
"hardware instruction generated for atomic "
21023 <<
" at " <<
ore::NV(
"SyncScope", MemScope) <<
" scope since ";
21025 switch (MemLegality) {
21026 case GlobalFPAtomicLegality::AgentScopeFineGrainedRemoteMemory:
21027 R <<
"fine-grained remote memory atomics work below system scope";
21029 case GlobalFPAtomicLegality::EmulatedSystemScope:
21030 R <<
"system scope atomics are emulated in hardware";
21032 case GlobalFPAtomicLegality::NoRemoteMemory:
21033 R <<
"memory is not remote (!amdgpu.no.remote.memory)";
21035 case GlobalFPAtomicLegality::NoFineGrainedMemory:
21036 R <<
"memory is not fine-grained (!amdgpu.no.fine.grained.memory)";
21038 case GlobalFPAtomicLegality::Illegal:
21042 switch (DenormReason) {
21043 case AtomicFlushDenormalReason::Native:
21045 case AtomicFlushDenormalReason::IgnoreDenormalMode:
21046 R <<
", and denormals may be flushed (!atomic.ignore.denormal.mode)";
21048 case AtomicFlushDenormalReason::FunctionFlushesDenormals:
21049 R <<
", and the floating-point environment flushes denormals";
21051 case AtomicFlushDenormalReason::IEEE:
21060 Type *EltTy = VT->getElementType();
21061 return VT->getNumElements() == 2 &&
21081 unsigned BW =
IT->getBitWidth();
21082 return BW == 32 || BW == 64;
21096 unsigned BW =
DL.getPointerSizeInBits(PT->getAddressSpace());
21097 return BW == 32 || BW == 64;
21100 if (Ty->isFloatTy() || Ty->isDoubleTy())
21104 return VT->getNumElements() == 2 &&
21105 VT->getElementType()->getPrimitiveSizeInBits() == 16;
21113static GlobalFPAtomicLegality
21122 if (HasSystemScope) {
21123 if (
Subtarget.hasAgentScopeFineGrainedRemoteMemoryAtomics() &&
21125 return GlobalFPAtomicLegality::NoRemoteMemory;
21126 if (
Subtarget.hasEmulatedSystemScopeAtomics())
21127 return GlobalFPAtomicLegality::EmulatedSystemScope;
21128 }
else if (
Subtarget.hasAgentScopeFineGrainedRemoteMemoryAtomics())
21129 return GlobalFPAtomicLegality::AgentScopeFineGrainedRemoteMemory;
21131 return RMW->
hasMetadata(
"amdgpu.no.fine.grained.memory")
21132 ? GlobalFPAtomicLegality::NoFineGrainedMemory
21133 : GlobalFPAtomicLegality::Illegal;
21146 const MDNode *MD =
I->getMetadata(LLVMContext::MD_noalias_addrspace);
21154 return STI.hasGloballyAddressableScratch()
21177 DL.getTypeSizeInBits(RMW->
getType()) == 64 &&
21181 GlobalFPAtomicLegality MemLegality = GlobalFPAtomicLegality::Illegal;
21182 AtomicFlushDenormalReason DenormReason = AtomicFlushDenormalReason::Native;
21192 bool HasSystemScope =
21224 if (!Subtarget->hasSubClampInsts() ||
21226 !Subtarget->hasAtomicDsCondSubClampInsts()) ||
21228 !Subtarget->hasAtomicCondSubClampFlatInsts()))
21233 if (!
IT ||
IT->getBitWidth() != 32)
21239 if (Subtarget->hasEmulatedSystemScopeAtomics())
21255 if (!HasSystemScope &&
21256 Subtarget->hasAgentScopeFineGrainedRemoteMemoryAtomics())
21268 if (RMW->
hasMetadata(
"amdgpu.no.fine.grained.memory"))
21277 ConstVal && ConstVal->isNullValue() &&
21316 if (Ty->isFloatTy()) {
21321 if (Ty->isDoubleTy()) {
21342 if (Ty->isFloatTy() &&
21343 !Subtarget->hasMemoryAtomicFaddF32DenormalSupport()) {
21345 if (DenormReason == AtomicFlushDenormalReason::IEEE)
21351 if (MemLegality != GlobalFPAtomicLegality::Illegal) {
21358 if (Subtarget->hasAtomicBufferGlobalPkAddF16Insts() &&
isV2F16(Ty))
21362 if (Subtarget->hasAtomicGlobalPkAddBF16Inst() &&
isV2BF16(Ty))
21366 if (Subtarget->hasAtomicBufferGlobalPkAddF16Insts() &&
isV2F16(Ty))
21371 if (Subtarget->hasAtomicBufferPkAddBF16Inst() &&
isV2BF16(Ty))
21376 if (Subtarget->hasFlatBufferGlobalAtomicFaddF64Inst() && Ty->isDoubleTy())
21380 if (Ty->isFloatTy()) {
21383 if (RMW->
use_empty() && Subtarget->hasAtomicFaddNoRtnInsts())
21386 if (!RMW->
use_empty() && Subtarget->hasAtomicFaddRtnInsts())
21391 Subtarget->hasAtomicBufferGlobalPkAddF16NoRtnInsts() &&
21399 if (Subtarget->hasFlatAtomicFaddF32Inst())
21408 if (Subtarget->hasLDSFPAtomicAddF32()) {
21409 if (RMW->
use_empty() && Subtarget->hasAtomicFaddNoRtnInsts())
21411 if (!RMW->
use_empty() && Subtarget->hasAtomicFaddRtnInsts())
21431 if (MemLegality != GlobalFPAtomicLegality::Illegal) {
21441 if (Subtarget->hasAtomicFMinFMaxF32FlatInsts() && Ty->isFloatTy())
21443 if (Subtarget->hasAtomicFMinFMaxF64FlatInsts() && Ty->isDoubleTy())
21447 if (Subtarget->hasAtomicFMinFMaxF32GlobalInsts() && Ty->isFloatTy())
21449 if (Subtarget->hasAtomicFMinFMaxF64GlobalInsts() && Ty->isDoubleTy())
21510 if (RC == &AMDGPU::VReg_1RegClass && !isDivergent)
21511 return Subtarget->isWave64() ? &AMDGPU::SReg_64RegClass
21512 : &AMDGPU::SReg_32RegClass;
21513 if (!
TRI->isSGPRClass(RC) && !isDivergent)
21514 return TRI->getEquivalentSGPRClass(RC);
21515 if (
TRI->isSGPRClass(RC) && isDivergent) {
21516 if (Subtarget->hasGFX90AInsts())
21517 return TRI->getEquivalentAVClass(RC);
21518 return TRI->getEquivalentVGPRClass(RC);
21531 unsigned WaveSize) {
21536 if (!
IT ||
IT->getBitWidth() != WaveSize)
21541 if (!Visited.
insert(
V).second)
21543 bool Result =
false;
21544 for (
const auto *U :
V->users()) {
21546 if (
V == U->getOperand(1)) {
21551 case Intrinsic::amdgcn_if_break:
21552 case Intrinsic::amdgcn_if:
21553 case Intrinsic::amdgcn_else:
21558 if (
V == U->getOperand(0)) {
21563 case Intrinsic::amdgcn_end_cf:
21564 case Intrinsic::amdgcn_loop:
21570 Result =
hasCFUser(U, Visited, WaveSize);
21579 const Value *V)
const {
21581 if (CI->isInlineAsm()) {
21590 for (
auto &TC : TargetConstraints) {
21604 return hasCFUser(V, Visited, Subtarget->getWavefrontSize());
21632 return MRI.hasOneNonDBGUse(N0);
21639 if (
I.getMetadata(
"amdgpu.noclobber"))
21641 if (
I.getMetadata(
"amdgpu.last.use"))
21643 if (
MDNode *
N =
I.getMetadata(
"amdgpu.cfs")) {
21648 "MOCFSB0 and MOCFSB1 are not adjacent bits");
21713 Alignment = RMW->getAlign();
21726 bool FullFlatEmulation =
21728 ((Subtarget->hasAtomicFaddInsts() && RMW->getType()->isFloatTy()) ||
21729 (Subtarget->hasFlatBufferGlobalAtomicFaddF64Inst() &&
21730 RMW->getType()->isDoubleTy()));
21733 bool ReturnValueIsUsed = !AI->
use_empty();
21742 if (FullFlatEmulation) {
21753 std::prev(BB->
end())->eraseFromParent();
21754 Builder.SetInsertPoint(BB);
21756 Value *LoadedShared =
nullptr;
21757 if (FullFlatEmulation) {
21758 Value *IsShared = Builder.CreateIntrinsic(Intrinsic::amdgcn_is_shared,
21759 {Addr},
nullptr,
"is.shared");
21760 Builder.CreateCondBr(IsShared, SharedBB, CheckPrivateBB);
21761 Builder.SetInsertPoint(SharedBB);
21762 Value *CastToLocal = Builder.CreateAddrSpaceCast(
21768 LoadedShared = Clone;
21770 Builder.CreateBr(PhiBB);
21771 Builder.SetInsertPoint(CheckPrivateBB);
21774 Value *IsPrivate = Builder.CreateIntrinsic(Intrinsic::amdgcn_is_private,
21775 {Addr},
nullptr,
"is.private");
21776 Builder.CreateCondBr(IsPrivate, PrivateBB, GlobalBB);
21778 Builder.SetInsertPoint(PrivateBB);
21780 Value *CastToPrivate = Builder.CreateAddrSpaceCast(
21783 Value *LoadedPrivate;
21785 LoadedPrivate = Builder.CreateAlignedLoad(
21786 RMW->getType(), CastToPrivate, RMW->getAlign(), RMW->isVolatile(),
21790 LoadedPrivate, RMW->getValOperand());
21792 Builder.CreateAlignedStore(NewVal, CastToPrivate, RMW->getAlign(),
21793 RMW->isVolatile());
21801 LoadedPrivate = Builder.CreateInsertValue(Insert, Equal, 1);
21804 Builder.CreateBr(PhiBB);
21806 Builder.SetInsertPoint(GlobalBB);
21810 if (FullFlatEmulation) {
21811 Value *CastToGlobal = Builder.CreateAddrSpaceCast(
21820 if (!FullFlatEmulation) {
21825 MDNode *RangeNotPrivate =
21828 LoadedGlobal->
setMetadata(LLVMContext::MD_noalias_addrspace,
21832 Builder.CreateBr(PhiBB);
21834 Builder.SetInsertPoint(PhiBB);
21836 if (ReturnValueIsUsed) {
21839 if (FullFlatEmulation)
21840 Loaded->addIncoming(LoadedShared, SharedBB);
21841 Loaded->addIncoming(LoadedPrivate, PrivateBB);
21842 Loaded->addIncoming(LoadedGlobal, GlobalBB);
21843 Loaded->takeName(AI);
21846 Builder.CreateBr(ExitBB);
21850 unsigned PtrOpIdx) {
21851 Value *PtrOp =
I->getOperand(PtrOpIdx);
21858 I->setOperand(PtrOpIdx, ASCast);
21870 ConstVal && ConstVal->isNullValue() &&
21901 "Expand Atomic Load only handles SCRATCH -> FLAT conversion");
21909 "Expand Atomic Store only handles SCRATCH -> FLAT conversion");
static bool isMul(MachineInstr *MI)
static unsigned getIntrinsicID(const SDNode *N)
const AArch64Subtarget & Subtarget
const TargetLowering & TLI
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
AMDGPU address space definition.
static constexpr std::pair< ImplicitArgumentMask, StringLiteral > ImplicitAttrs[]
static bool allUsesHaveSourceMods(MachineInstr &MI, MachineRegisterInfo &MRI, unsigned CostThreshold=4)
static msgpack::DocNode getNode(msgpack::DocNode DN, msgpack::Type Type, MCValue Val)
static bool isCtlzOpc(unsigned Opc)
Contains the definition of a TargetInstrInfo class that is common to all AMD GPUs.
static bool isNoUnsignedWrap(MachineInstr *Addr)
static bool parseTexFail(uint64_t TexFailCtrl, bool &TFE, bool &LWE, bool &IsTexFail)
static bool isAsyncLDSDMA(Intrinsic::ID Intr)
static std::optional< uint32_t > getConstantNamedBarrierNumber(const MachineOperand &BarOp, const MachineRegisterInfo &MRI)
Return the barrier ID for BarOp if it's statically known, mapping poison to 0 and masking off the hig...
static void packImage16bitOpsToDwords(MachineIRBuilder &B, MachineInstr &MI, SmallVectorImpl< Register > &PackedAddrs, unsigned ArgOffset, const AMDGPU::ImageDimIntrinsicInfo *Intr, bool IsA16, bool IsG16)
Turn a set of f16 typed registers in AddrRegs into a dword sized vector with f16 typed elements.
static bool isKnownNonNull(Register Val, MachineRegisterInfo &MRI, const AMDGPUTargetMachine &TM, unsigned AddrSpace)
Return true if the value is a known valid address, such that a null check is not necessary.
Provides AMDGPU specific target descriptions.
The AMDGPU TargetMachine interface definition for hw codegen targets.
This file declares a class to represent arbitrary precision floating point values and provide a varie...
This file implements a class to represent arbitrary precision integral constant values and operations...
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
static cl::opt< ITMode > IT(cl::desc("IT block support"), cl::Hidden, cl::init(DefaultIT), cl::values(clEnumValN(DefaultIT, "arm-default-it", "Generate any type of IT block"), clEnumValN(RestrictedIT, "arm-restrict-it", "Disallow complex IT blocks")))
Function Alias Analysis Results
static DenormalFPEnv getDenormalFPEnv(AttributeSet FnAttrs)
@ DEFAULT
Default weight is used in cases when there is no dedicated execution weight set.
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static std::optional< SDByteProvider > calculateByteProvider(SDValue Op, unsigned Index, unsigned Depth, std::optional< uint64_t > VectorIndex, unsigned StartingIndex=0, MutableArrayRef< uint8_t > ByteMask={})
static bool isSigned(unsigned Opcode)
Utilities for dealing with flags related to floating point properties and mode controls.
AMD GCN specific subclass of TargetSubtarget.
Provides analysis for querying information about KnownBits during GISel passes.
Declares convenience wrapper classes for interpreting MachineInstr instances as specific generic oper...
const HexagonInstrInfo * TII
iv Induction Variable Users
static constexpr Value * getValue(Ty &ValueOrUse)
const size_t AbstractManglingParser< Derived, Alloc >::NumOps
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
Contains matchers for matching SSA Machine Instructions.
static bool isUndef(const MachineInstr &MI)
Register const TargetRegisterInfo * TRI
Promote Memory to Register
static unsigned getAddressSpace(const Value *V, unsigned MaxLookup)
uint64_t IntrinsicInst * II
static constexpr MCPhysReg SPReg
MachineBasicBlock * Folded
SmallVector< MachineOperand, 4 > Cond
MachineRegisterInfo & MRI
static cl::opt< RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode > Mode("regalloc-enable-advisor", cl::Hidden, cl::init(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Default), cl::desc("Enable regalloc advisor mode"), cl::values(clEnumValN(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Default, "default", "Default"), clEnumValN(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Release, "release", "precompiled"), clEnumValN(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Development, "development", "for training")))
Contains matchers for matching SelectionDAG nodes and values.
static void r0(uint32_t &A, uint32_t &B, uint32_t &C, uint32_t &D, uint32_t &E, int I, uint32_t *Buf)
static void r3(uint32_t &A, uint32_t &B, uint32_t &C, uint32_t &D, uint32_t &E, int I, uint32_t *Buf)
static void r2(uint32_t &A, uint32_t &B, uint32_t &C, uint32_t &D, uint32_t &E, int I, uint32_t *Buf)
static void r1(uint32_t &A, uint32_t &B, uint32_t &C, uint32_t &D, uint32_t &E, int I, uint32_t *Buf)
#define FP_DENORM_FLUSH_NONE
#define FP_DENORM_FLUSH_IN_FLUSH_OUT
static void reservePrivateMemoryRegs(const TargetMachine &TM, MachineFunction &MF, const SIRegisterInfo &TRI, SIMachineFunctionInfo &Info)
static SDValue adjustLoadValueTypeImpl(SDValue Result, EVT LoadVT, const SDLoc &DL, SelectionDAG &DAG, bool Unpacked)
static MachineBasicBlock * emitIndirectSrc(MachineInstr &MI, MachineBasicBlock &MBB, const GCNSubtarget &ST)
static bool denormalModeIsFlushAllF64F16(const MachineFunction &MF)
static bool isAtomicRMWLegalIntTy(Type *Ty)
static void knownBitsForWorkitemID(const GCNSubtarget &ST, GISelValueTracking &VT, KnownBits &Known, unsigned Dim)
static bool flatInstrMayAccessPrivate(const Instruction *I)
Return if a flat address space atomicrmw can access private memory.
static std::pair< unsigned, int > computeIndirectRegAndOffset(const SIRegisterInfo &TRI, const TargetRegisterClass *SuperRC, unsigned VecReg, int Offset)
static bool denormalModeIsFlushAllF32(const MachineFunction &MF)
static bool addresses16Bits(int Mask)
static MachineBasicBlock * expand64BitScalarArithmetic(MachineInstr &MI, MachineBasicBlock *BB)
static bool isClampZeroToOne(SDValue A, SDValue B)
static bool isBoolSGPRExt(SDValue V)
static bool supportsMin3Max3(const GCNSubtarget &Subtarget, unsigned Opc, EVT VT)
static AtomicFlushDenormalReason getAtomicFlushDenormalReason(const AtomicRMWInst *RMW)
static unsigned findFirstFreeSGPR(CCState &CCInfo)
static uint32_t getPermuteMask(SDValue V)
static SDValue lowerLaneOp(const SITargetLowering &TLI, SDNode *N, SelectionDAG &DAG)
static int getAlignedAGPRClassID(unsigned UnalignedClassID)
static void processPSInputArgs(SmallVectorImpl< ISD::InputArg > &Splits, CallingConv::ID CallConv, ArrayRef< ISD::InputArg > Ins, BitVector &Skipped, FunctionType *FType, SIMachineFunctionInfo *Info)
static SDValue selectSOffset(SDValue SOffset, SelectionDAG &DAG, const GCNSubtarget *Subtarget)
static SDValue getLoadExtOrTrunc(SelectionDAG &DAG, ISD::LoadExtType ExtType, SDValue Op, const SDLoc &SL, EVT VT)
static std::tuple< unsigned, unsigned > getDPPOpcForWaveReduction(unsigned Opc, const GCNSubtarget &ST)
static void fixMasks(SmallVectorImpl< DotSrc > &Srcs, unsigned ChainLength)
static bool is32bitWaveReduceOperation(unsigned Opc)
static TargetLowering::AtomicExpansionKind atomicSupportedIfLegalIntType(const AtomicRMWInst *RMW)
static SDValue strictFPExtFromF16(SelectionDAG &DAG, SDValue Src)
Return the source of an fp_extend from f16 to f32, or a converted FP constant.
static bool isAtomicRMWLegalXChgTy(const AtomicRMWInst *RMW)
static bool bitOpWithConstantIsReducible(unsigned Opc, uint32_t Val)
static void convertScratchAtomicToFlatAtomic(Instruction *I, unsigned PtrOpIdx)
static bool isCopyFromRegOfInlineAsm(const SDNode *N)
static bool elementPairIsOddToEven(ArrayRef< int > Mask, int Elt)
static SDValue lowerBFEIntrinsic(SDValue Op, SelectionDAG &DAG, Intrinsic::ID IntrinsicID)
static SDValue getDWordFromOffset(SelectionDAG &DAG, SDLoc SL, SDValue Src, unsigned DWordOffset)
static MachineBasicBlock::iterator loadM0FromVGPR(const SIInstrInfo *TII, MachineBasicBlock &MBB, MachineInstr &MI, unsigned InitResultReg, unsigned PhiReg, int Offset, bool UseGPRIdxMode, Register &SGPRIdxReg)
static bool isFloatingPointWaveReduceOperation(unsigned Opc)
static bool isImmConstraint(StringRef Constraint)
static SDValue padEltsToUndef(SelectionDAG &DAG, const SDLoc &DL, EVT CastVT, SDValue Src, int ExtraElts)
static bool hasCFUser(const Value *V, SmallPtrSet< const Value *, 16 > &Visited, unsigned WaveSize)
static std::pair< Register, Register > ExtractSubRegs(MachineInstr &MI, MachineOperand &Op, const TargetRegisterClass *SrcRC, const GCNSubtarget &ST, MachineRegisterInfo &MRI)
static unsigned SubIdx2Lane(unsigned Idx)
Helper function for adjustWritemask.
static TargetLowering::AtomicExpansionKind getPrivateAtomicExpansionKind(const GCNSubtarget &STI)
static bool addressMayBeAccessedAsPrivate(const MachineMemOperand *MMO, const SIMachineFunctionInfo &Info)
static MachineBasicBlock * lowerWaveReduce(MachineInstr &MI, MachineBasicBlock &BB, const GCNSubtarget &ST, unsigned Opc)
static bool elementPairIsContiguous(ArrayRef< int > Mask, int Elt)
static bool isV2BF16(Type *Ty)
static bool isFrexpExp(SDValue V, SDValue &FrexpInput)
static ArgDescriptor allocateSGPR32InputImpl(CCState &CCInfo, const TargetRegisterClass *RC, unsigned NumArgRegs)
static SDValue getMad64_32(SelectionDAG &DAG, const SDLoc &SL, EVT VT, SDValue N0, SDValue N1, SDValue N2, bool Signed)
static SDValue resolveSources(SelectionDAG &DAG, SDLoc SL, SmallVectorImpl< DotSrc > &Srcs, bool IsSigned, bool IsAny)
static bool hasNon16BitAccesses(uint64_t PermMask, SDValue &Op, SDValue &OtherOp)
static SDValue lowerWaveShuffle(const SITargetLowering &TLI, SDNode *N, SelectionDAG &DAG)
static SDValue diagnoseUnsupportedImage(SelectionDAG &DAG, SDValue Op, ArrayRef< EVT > ResultTypes, const SDLoc &DL, const Twine &Msg)
Emit a DiagnosticInfoUnsupported for an unsupported image intrinsic and return poison values of Resul...
static void placeSources(ByteProvider< SDValue > &Src0, ByteProvider< SDValue > &Src1, SmallVectorImpl< DotSrc > &Src0s, SmallVectorImpl< DotSrc > &Src1s, int Step)
static unsigned parseSyncscopeMDArg(const CallBase &CI, unsigned ArgIdx)
static EVT memVTFromLoadIntrReturn(const SITargetLowering &TLI, const DataLayout &DL, Type *Ty, unsigned MaxNumLanes)
static MachineBasicBlock::iterator emitLoadM0FromVGPRLoop(const SIInstrInfo *TII, MachineRegisterInfo &MRI, MachineBasicBlock &OrigBB, MachineBasicBlock &LoopBB, const DebugLoc &DL, const MachineOperand &Idx, unsigned InitReg, unsigned ResultReg, unsigned PhiReg, unsigned InitSaveExecReg, int Offset, bool UseGPRIdxMode, Register &SGPRIdxReg)
static SDValue matchPERM(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static bool isFrameIndexOp(SDValue Op)
static ConstantFPSDNode * getSplatConstantFP(SDValue Op)
static void allocateSGPR32Input(CCState &CCInfo, ArgDescriptor &Arg)
static void knownBitsForSBFE(const MachineInstr &MI, GISelValueTracking &VT, KnownBits &Known, const APInt &DemandedElts, unsigned BFEWidth, bool SExt, unsigned Depth)
static bool isExtendedFrom16Bits(SDValue &Operand)
static std::optional< bool > checkDot4MulSignedness(const SDValue &N, ByteProvider< SDValue > &Src0, ByteProvider< SDValue > &Src1, const SDValue &S0Op, const SDValue &S1Op, const SelectionDAG &DAG)
static bool vectorEltWillFoldAway(SDValue Op)
static SDValue getSPDenormModeValue(uint32_t SPDenormMode, SelectionDAG &DAG, const SIMachineFunctionInfo *Info, const GCNSubtarget *ST)
static uint32_t getConstantPermuteMask(uint32_t C)
static AtomicOrdering parseAtomicOrderingCABIArg(const CallBase &CI, unsigned ArgIdx)
static MachineBasicBlock * emitIndirectDst(MachineInstr &MI, MachineBasicBlock &MBB, const GCNSubtarget &ST)
static void setM0ToIndexFromSGPR(const SIInstrInfo *TII, MachineRegisterInfo &MRI, MachineInstr &MI, int Offset)
static DenormalFPEnv getDenormalFPEnv(const MachineFunction &MF)
static std::pair< MachineBasicBlock *, MachineBasicBlock * > splitBlockForLoop(MachineInstr &MI, MachineBasicBlock &MBB, bool InstInLoop)
static unsigned getBasePtrIndex(const MemSDNode *N)
MemSDNode::getBasePtr() does not work for intrinsics, which needs to offset by the chain and intrinsi...
static void allocateFixedSGPRInputImpl(CCState &CCInfo, const TargetRegisterClass *RC, MCRegister Reg)
static SDValue constructRetValue(SelectionDAG &DAG, MachineSDNode *Result, ArrayRef< EVT > ResultTypes, bool IsTexFail, bool Unpacked, bool IsD16, int DMaskPop, int NumVDataDwords, bool IsAtomicPacked16Bit, const SDLoc &DL)
static std::pair< SDValue, SDValue > splitTFEValueAndStatus(SDValue Op, EVT VT, const SDLoc &DL, SelectionDAG &DAG)
static std::optional< ByteProvider< SDValue > > handleMulOperand(const SDValue &MulOperand)
static ISD::CondCode tryReduceF64CompareToHiHalf(const ISD::CondCode CC, const SDValue LHS, const SDValue RHS, const SelectionDAG &DAG)
static Register getIndirectSGPRIdx(const SIInstrInfo *TII, MachineRegisterInfo &MRI, MachineInstr &MI, int Offset)
static SDValue emitNonHSAIntrinsicError(SelectionDAG &DAG, const SDLoc &DL, EVT VT)
static EVT memVTFromLoadIntrData(const SITargetLowering &TLI, const DataLayout &DL, Type *Ty, unsigned MaxNumLanes)
static unsigned minMaxOpcToMin3Max3Opc(unsigned Opc)
static unsigned getExtOpcodeForPromotedOp(SDValue Op)
static void expand64BitV_CNDMASK(MachineInstr &MI, MachineBasicBlock *BB)
static GlobalFPAtomicLegality getGlobalMemoryFPAtomicLegality(const GCNSubtarget &Subtarget, const AtomicRMWInst *RMW, bool HasSystemScope)
static SDValue lowerBALLOTIntrinsic(const SITargetLowering &TLI, SDNode *N, SelectionDAG &DAG)
static SDValue buildSMovImm32(SelectionDAG &DAG, const SDLoc &DL, uint64_t Val)
static SDValue tryFoldMADwithSRL(SelectionDAG &DAG, const SDLoc &SL, SDValue MulLHS, SDValue MulRHS, SDValue AddRHS)
static unsigned getIntrMemWidth(unsigned IntrID)
static SDValue getBuildDwordsVector(SelectionDAG &DAG, SDLoc DL, ArrayRef< SDValue > Elts)
static SDNode * findUser(SDValue Value, unsigned Opcode)
Helper function for LowerBRCOND.
static unsigned addPermMasks(unsigned First, unsigned Second)
static uint64_t clearUnusedBits(uint64_t Val, unsigned Size)
static SDValue getFPTernOp(SelectionDAG &DAG, unsigned Opcode, const SDLoc &SL, EVT VT, SDValue A, SDValue B, SDValue C, SDValue GlueChain, SDNodeFlags Flags)
static bool isV2F16OrV2BF16(Type *Ty)
static OptimizationRemark emitAtomicRMWLegalRemark(const AtomicRMWInst *RMW, GlobalFPAtomicLegality MemLegality, AtomicFlushDenormalReason DenormReason)
static SDValue emitRemovedIntrinsicError(SelectionDAG &DAG, const SDLoc &DL, EVT VT)
static SDValue getFPBinOp(SelectionDAG &DAG, unsigned Opcode, const SDLoc &SL, EVT VT, SDValue A, SDValue B, SDValue GlueChain, SDNodeFlags Flags)
static SDValue buildPCRelGlobalAddress(SelectionDAG &DAG, const GlobalValue *GV, const SDLoc &DL, int64_t Offset, EVT PtrVT, unsigned GAFlags=SIInstrInfo::MO_NONE)
static const std::optional< ByteProvider< SDValue > > calculateSrcByte(const SDValue Op, uint64_t DestByte, uint64_t SrcIndex=0, unsigned Depth=0)
static bool isV2F16(Type *Ty)
static void initializeM0ToZeroForClusterLoad(SDValue Op, SelectionDAG &DAG, SDLoc DL)
static void allocateSGPR64Input(CCState &CCInfo, ArgDescriptor &Arg)
static uint64_t getIdentityValueForWaveReduction(unsigned Opc)
SI DAG Lowering interface definition.
Interface definition for SIRegisterInfo.
Func getContext().diagnose(DiagnosticInfoUnsupported(Func
static bool contains(SmallPtrSetImpl< ConstantExpr * > &Cache, ConstantExpr *Expr, Constant *C)
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
static TableGen::Emitter::Opt Y("gen-skeleton-entry", EmitSkeleton, "Generate example skeleton entry")
static constexpr int Concat[]
static bool isSupportedBitWidth(unsigned Bits)
unsigned allocateBarrierGlobal(const DataLayout &DL, const GlobalVariable &GV)
static std::optional< uint32_t > getLDSKernelIdMetadata(const Function &F)
void setDynLDSAlign(const Function &F, const GlobalVariable &GV)
static std::optional< uint32_t > get32BitAbsoluteAddress(const GlobalValue &GV, unsigned AS)
void setUsesDynamicLDS(bool DynLDS)
bool isBottomOfStack() const
uint32_t getLDSSize() const
bool isEntryFunction() const
static unsigned numBitsSigned(SDValue Op, SelectionDAG &DAG)
SDValue SplitVectorLoad(SDValue Op, SelectionDAG &DAG) const
Split a vector load into 2 loads of half the vector.
void analyzeFormalArgumentsCompute(CCState &State, const SmallVectorImpl< ISD::InputArg > &Ins) const
The SelectionDAGBuilder will automatically promote function arguments with illegal types.
SDValue LowerF64ToF16Safe(SDValue Src, const SDLoc &DL, SelectionDAG &DAG) const
SDValue storeStackInputValue(SelectionDAG &DAG, const SDLoc &SL, SDValue Chain, SDValue ArgVal, int64_t Offset) const
void computeKnownBitsForTargetNode(const SDValue Op, KnownBits &Known, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth=0) const override
Determine which of the bits specified in Mask are known to be either zero or one and return them in t...
SDValue splitBinaryBitConstantOpImpl(DAGCombinerInfo &DCI, const SDLoc &SL, unsigned Opc, SDValue LHS, uint32_t ValLo, uint32_t ValHi) const
Split the 64-bit value LHS into two 32-bit components, and perform the binary operation Opc to it wit...
SDValue lowerUnhandledCall(CallLoweringInfo &CLI, SmallVectorImpl< SDValue > &InVals, StringRef Reason) const
virtual SDValue LowerGlobalAddress(AMDGPUMachineFunctionInfo *MFI, SDValue Op, SelectionDAG &DAG) const
SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override
This callback is invoked for operations that are unsupported by the target, which are registered to u...
SDValue addTokenForArgument(SDValue Chain, SelectionDAG &DAG, MachineFrameInfo &MFI, int ClobberedFI) const
bool isKnownNeverNaNForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, bool SNaN=false, unsigned Depth=0) const override
If SNaN is false,.
static bool needsDenormHandlingF32(const SelectionDAG &DAG, SDValue Src, SDNodeFlags Flags)
uint32_t getImplicitParameterOffset(const MachineFunction &MF, const ImplicitParameter Param) const
Helper function that returns the byte offset of the given type of implicit parameter.
SDValue LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG) const
static KnownBits computeKnownBitsForMul24(const KnownBits &LHSKnown, const KnownBits &RHSKnown, unsigned BitWidth, bool IsSigned, bool SelfMultiply)
SDValue loadInputValue(SelectionDAG &DAG, const TargetRegisterClass *RC, EVT VT, const SDLoc &SL, const ArgDescriptor &Arg) const
AMDGPUTargetLowering(const TargetMachine &TM, const TargetSubtargetInfo &STI, const AMDGPUSubtarget &AMDGPUSTI)
static EVT getEquivalentMemType(LLVMContext &Context, EVT VT)
SDValue LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const
SDValue CreateLiveInRegister(SelectionDAG &DAG, const TargetRegisterClass *RC, Register Reg, EVT VT, const SDLoc &SL, bool RawReg=false) const
Helper function that adds Reg to the LiveIn list of the DAG's MachineFunction.
SDValue SplitVectorStore(SDValue Op, SelectionDAG &DAG) const
Split a vector store into 2 stores of half the vector.
std::pair< SDValue, SDValue > split64BitValue(SDValue Op, SelectionDAG &DAG) const
Return 64-bit value Op as two 32-bit integers.
static CCAssignFn * CCAssignFnForReturn(CallingConv::ID CC, bool IsVarArg)
static CCAssignFn * CCAssignFnForCall(CallingConv::ID CC, bool IsVarArg)
Selects the correct CCAssignFn for a given CallingConvention value.
static unsigned numBitsUnsigned(SDValue Op, SelectionDAG &DAG)
static bool allowApproxFunc(const SelectionDAG &DAG, SDNodeFlags Flags)
SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool isVarArg, const SmallVectorImpl< ISD::OutputArg > &Outs, const SmallVectorImpl< SDValue > &OutVals, const SDLoc &DL, SelectionDAG &DAG) const override
This hook must be implemented to lower outgoing return values, described by the Outs array,...
void ReplaceNodeResults(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG) const override
This callback is invoked when a node result type is illegal for the target, and the operation was reg...
SDValue performRcpCombine(SDNode *N, DAGCombinerInfo &DCI) const
SDValue LowerFP_TO_INT_SAT(SDValue Op, SelectionDAG &DAG) const
static bool shouldFoldFNegIntoSrc(SDNode *FNeg, SDValue FNegSrc)
bool isNarrowingProfitable(SDNode *N, EVT SrcVT, EVT DestVT) const override
Return true if it's profitable to narrow operations of type SrcVT to DestVT.
SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override
This method will be invoked for all target nodes and for any target-independent nodes that the target...
SDValue WidenOrSplitVectorLoad(SDValue Op, SelectionDAG &DAG) const
Widen a suitably aligned v3 load.
SDValue getHiHalf64(SDValue Op, SelectionDAG &DAG) const
bool isNoopAddrSpaceCast(const DataLayout &DL, unsigned SrcAS, unsigned DestAS) const override
Returns true if a cast between SrcAS and DestAS is a noop.
static bool EnableObjectLinking
const std::array< unsigned, 3 > & getDims() const
static const LaneMaskConstants & get(const GCNSubtarget &ST)
const unsigned XorTermOpc
const unsigned AndSaveExecOpc
static const fltSemantics & IEEEsingle()
static constexpr roundingMode rmNearestTiesToEven
static const fltSemantics & IEEEhalf()
static APFloat getQNaN(const fltSemantics &Sem, bool Negative=false, const APInt *payload=nullptr)
Factory for QNaN values.
LLVM_ABI opStatus convert(const fltSemantics &ToSemantics, roundingMode RM, bool *losesInfo)
LLVM_READONLY int getExactLog2Abs() const
static APFloat getOne(const fltSemantics &Sem, bool Negative=false)
Factory for Positive and Negative One.
static APFloat getLargest(const fltSemantics &Sem, bool Negative=false)
Returns the largest finite number in the given semantics.
static APFloat getInf(const fltSemantics &Sem, bool Negative=false)
Factory for Positive and Negative Infinity.
static APFloat getZero(const fltSemantics &Sem, bool Negative=false)
Factory for Positive and Negative Zero.
Class for arbitrary precision integers.
LLVM_ABI APInt zext(unsigned width) const
Zero extend to a new width.
static APInt getMaxValue(unsigned numBits)
Gets maximum unsigned value of APInt for specific bit width.
static APInt getBitsSet(unsigned numBits, unsigned loBit, unsigned hiBit)
Get a value with a block of bits set.
bool isZero() const
Determine if this value is zero, i.e. all bits are clear.
bool isSignMask() const
Check if the APInt's value is returned by getSignMask.
unsigned countr_zero() const
Count the number of trailing zero bits.
bool isOneBitSet(unsigned BitNo) const
Determine if this APInt Value only has the specified bit set.
bool isMask(unsigned numBits) const
bool isSignBitSet() const
Determine if sign bit of this APInt is set.
static APInt getHighBitsSet(unsigned numBits, unsigned hiBitsSet)
Constructs an APInt value that has the top hiBitsSet bits set.
bool sge(const APInt &RHS) const
Signed greater or equal comparison.
bool uge(const APInt &RHS) const
Unsigned greater or equal comparison.
unsigned getAddressSpace() const
Return the address space for the allocation.
This class represents an incoming formal argument to a Function.
LLVM_ABI bool hasAttribute(Attribute::AttrKind Kind) const
Check if an argument has a given attribute.
const Function * getParent() const
Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
Get the array size.
bool empty() const
Check if the array is empty.
An instruction that atomically checks whether a specified value is in a memory location,...
Value * getNewValOperand()
bool isVolatile() const
Return true if this is a cmpxchg from a volatile memory location.
unsigned getPointerAddressSpace() const
Returns the address space of the pointer operand.
Value * getCompareOperand()
Align getAlign() const
Return the alignment of the memory that is being allocated by the instruction.
static unsigned getPointerOperandIndex()
an instruction that atomically reads a memory location, combines it with another value,...
static unsigned getPointerOperandIndex()
BinOp
This enumeration lists the possible modifications atomicrmw can make.
@ USubCond
Subtract only if no unsigned overflow.
@ Min
*p = old <signed v ? old : v
@ USubSat
*p = usub.sat(old, v) usub.sat matches the behavior of llvm.usub.sat.
@ UIncWrap
Increment one up to a maximum value.
@ Max
*p = old >signed v ? old : v
@ UMin
*p = old <unsigned v ? old : v
@ FMin
*p = minnum(old, v) minnum matches the behavior of llvm.minnum.
@ UMax
*p = old >unsigned v ? old : v
@ FMax
*p = maxnum(old, v) maxnum matches the behavior of llvm.maxnum.
@ UDecWrap
Decrement one until a minimum value or zero.
void setOperation(BinOp Operation)
BinOp getOperation() const
SyncScope::ID getSyncScopeID() const
Returns the synchronization scope ID of this rmw instruction.
static LLVM_ABI StringRef getOperationName(BinOp Op)
unsigned getPointerAddressSpace() const
Returns the address space of the pointer operand.
bool isCompareAndSwap() const
Returns true if this SDNode represents cmpxchg atomic operation, false otherwise.
This class holds the attributes for a particular argument, parameter, function, or return value.
LLVM_ABI MemoryEffects getMemoryEffects() const
LLVM Basic Block Representation.
LLVM_ABI BasicBlock * splitBasicBlock(iterator I, const Twine &BBName="")
Split the basic block into two basic blocks at the specified instruction.
const Function * getParent() const
Return the enclosing method, or null if none.
static BasicBlock * Create(LLVMContext &Context, const Twine &Name="", Function *Parent=nullptr, BasicBlock *InsertBefore=nullptr)
Creates a new BasicBlock.
A "pseudo-class" with methods for operating on BUILD_VECTORs.
Represents known origin of an individual byte in combine pattern.
static ByteProvider getConstantZero()
static ByteProvider getSrc(std::optional< ISelOp > Val, int64_t ByteOffset, int64_t VectorOffset)
CCState - This class holds information needed while lowering arguments and return values.
MachineFunction & getMachineFunction() const
unsigned getFirstUnallocated(ArrayRef< MCPhysReg > Regs) const
getFirstUnallocated - Return the index of the first unallocated register in the set,...
static LLVM_ABI bool resultsCompatible(CallingConv::ID CalleeCC, CallingConv::ID CallerCC, MachineFunction &MF, LLVMContext &C, const SmallVectorImpl< ISD::InputArg > &Ins, CCAssignFn CalleeFn, CCAssignFn CallerFn)
Returns true if the results of the two calling conventions are compatible.
LLVM_ABI void AnalyzeCallResult(const SmallVectorImpl< ISD::InputArg > &Ins, CCAssignFn Fn)
AnalyzeCallResult - Analyze the return values of a call, incorporating info about the passed values i...
MCRegister AllocateReg(MCPhysReg Reg)
AllocateReg - Attempt to allocate one register.
LLVM_ABI bool CheckReturn(const SmallVectorImpl< ISD::OutputArg > &Outs, CCAssignFn Fn)
CheckReturn - Analyze the return values of a function, returning true if the return can be performed ...
LLVM_ABI void AnalyzeReturn(const SmallVectorImpl< ISD::OutputArg > &Outs, CCAssignFn Fn)
AnalyzeReturn - Analyze the returned values of a return, incorporating info about the result values i...
int64_t AllocateStack(unsigned Size, Align Alignment)
AllocateStack - Allocate a chunk of stack space with the specified size and alignment.
LLVM_ABI void AnalyzeCallOperands(const SmallVectorImpl< ISD::OutputArg > &Outs, CCAssignFn Fn)
AnalyzeCallOperands - Analyze the outgoing arguments to a call, incorporating info about the passed v...
uint64_t getStackSize() const
Returns the size of the currently allocated portion of the stack.
bool isAllocated(MCRegister Reg) const
isAllocated - Return true if the specified register (or an alias) is allocated.
LLVM_ABI void AnalyzeFormalArguments(const SmallVectorImpl< ISD::InputArg > &Ins, CCAssignFn Fn)
AnalyzeFormalArguments - Analyze an array of argument values, incorporating info about the formals in...
CCValAssign - Represent assignment of one arg/retval to a location.
Register getLocReg() const
LocInfo getLocInfo() const
int64_t getLocMemOffset() const
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
bool hasFnAttr(Attribute::AttrKind Kind) const
Determine whether this call has the given attribute.
LLVM_ABI bool isMustTailCall() const
Tests if this call site must be tail call optimized.
Value * getArgOperand(unsigned i) const
unsigned arg_size() const
This class represents a function call, abstracting a target machine's calling convention.
static LLVM_ABI CastInst * CreatePointerCast(Value *S, Type *Ty, const Twine &Name="", InsertPosition InsertBefore=nullptr)
Create a BitCast, AddrSpaceCast or a PtrToInt cast instruction.
const APFloat & getValueAPF() const
bool isPosZero() const
Return true if the value is positive zero.
bool isOne() const
Returns true if this value is exactly +1.0.
bool isMinusOne() const
Returns true if this value is exactly -1.0.
bool isNegative() const
Return true if the value is negative.
This is the shared class of boolean and integer constants.
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
uint64_t getZExtValue() const
const APInt & getAPIntValue() const
This is an important base class in LLVM.
uint64_t getNumOperands() const
A parsed version of the target data layout string in and methods for querying it.
LLVM_ABI Align getABITypeAlign(Type *Ty) const
Returns the minimum ABI-required alignment for the specified type.
Diagnostic information for unsupported feature in backend.
static constexpr ElementCount getFixed(ScalarTy MinVal)
Class to represent fixed width SIMD vectors.
unsigned getNumElements() const
FunctionLoweringInfo - This contains information that is global to a function that is used when lower...
Register DemoteRegister
DemoteRegister - if CanLowerReturn is false, DemoteRegister is a vreg allocated to hold a pointer to ...
LLVM_ABI const Value * getValueFromVirtualReg(Register Vreg)
This method is called from TargetLowerinInfo::isSDNodeSourceOfDivergence to get the Value correspondi...
Class to represent function types.
Type * getParamType(unsigned i) const
Parameter type accessors.
FunctionType * getFunctionType() const
Returns the FunctionType for me.
const DataLayout & getDataLayout() const
Get the data layout of the module this function belongs to.
iterator_range< arg_iterator > args()
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
DenormalMode getDenormalMode(const fltSemantics &FPType) const
Returns the denormal handling type for the default rounding mode of the function.
Argument * getArg(unsigned i) const
const SIInstrInfo * getInstrInfo() const override
unsigned getInstCacheLineSize() const
Instruction cache line size in bytes (64 for pre-GFX11, 128 for GFX11+).
const SIRegisterInfo * getRegisterInfo() const override
unsigned getMaxPrivateElementSize(bool ForBufferRSrc=false) const
bool hasKernargSegmentPtr() const
bool hasDispatchID() const
bool hasPrivateSegmentBuffer() const
unsigned getNumFreeUserSGPRs()
bool hasImplicitBufferPtr() const
bool hasPrivateSegmentSize() const
bool hasDispatchPtr() const
bool hasFlatScratchInit() const
const MachineFunction & getMachineFunction() const
void computeKnownBitsImpl(Register R, KnownBits &Known, const APInt &DemandedElts, unsigned Depth=0)
Evaluate a known-bits query with an explicit worklist instead of recursive descent.
int64_t getOffset() const
LLVM_ABI unsigned getAddressSpace() const
const GlobalValue * getGlobal() const
bool hasExternalLinkage() const
unsigned getAddressSpace() const
Module * getParent()
Get the module that this global value is contained inside of...
LLVM_ABI const DataLayout & getDataLayout() const
Get the data layout of the module this global belongs to.
Type * getValueType() const
LLVM_ABI uint64_t getGlobalSize(const DataLayout &DL) const
Get the size of this global variable in bytes.
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
LLVM_ABI Instruction * clone() const
Create a copy of 'this' instruction that is identical in all ways except the following:
LLVM_ABI void removeFromParent()
This method unlinks 'this' from the containing basic block, but does not delete it.
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
bool hasMetadata() const
Return true if this instruction has any metadata attached to it.
LLVM_ABI const Function * getFunction() const
Return the function this instruction belongs to.
LLVM_ABI void setMetadata(unsigned KindID, MDNode *Node)
Set the metadata of the specified kind to the specified node.
LLVM_ABI const DataLayout & getDataLayout() const
Get the data layout of the module this instruction belongs to.
LLVM_ABI InstListType::iterator insertInto(BasicBlock *ParentBB, InstListType::iterator It)
Inserts an unlinked instruction into ParentBB at position It and returns the iterator of the inserted...
Class to represent integer types.
A wrapper class for inspecting calls to intrinsic functions.
bool isKnownNeverNaN() const
Return true if it's known this can never be a nan.
static LLVM_ABI KnownFPClass bitcast(const fltSemantics &FltSemantics, const KnownBits &Bits)
Report known values for a bitcast into a float with provided semantics.
constexpr unsigned getScalarSizeInBits() const
static constexpr LLT scalar(unsigned SizeInBits)
Get a low-level scalar or aggregate "bag of bits".
static constexpr LLT pointer(unsigned AddressSpace, unsigned SizeInBits)
Get a low-level pointer in the given address space.
constexpr TypeSize getSizeInBits() const
Returns the total size of the type. Must only be called on sized types.
static LLT integer(unsigned SizeInBits)
LLT changeElementSize(unsigned NewEltSize) const
If this type is a vector, return a vector with the same number of elements but the new element size.
This is an important class for using LLVM in a threaded context.
LLVM_ABI void emitError(const Instruction *I, const Twine &ErrorStr)
emitError - Emit an error message to the currently installed error handler with optional location inf...
LLVM_ABI void diagnose(const DiagnosticInfo &DI)
Report a message to the currently installed diagnostic handler.
LLVM_ABI SyncScope::ID getOrInsertSyncScopeID(StringRef SSN)
getOrInsertSyncScopeID - Maps synchronization scope name to synchronization scope ID.
An instruction for reading from memory.
unsigned getPointerAddressSpace() const
Returns the address space of the pointer operand.
static unsigned getPointerOperandIndex()
This class is used to represent ISD::LOAD nodes.
TypeSize getValue() const
Describe properties that are true of each instruction in the target description file.
unsigned getID() const
getID() - Return the register class ID number.
MCRegister getRegister(unsigned i) const
getRegister - Return the specified register in the class.
unsigned getNumRegs() const
getNumRegs - Return the number of registers in this class.
iterator begin() const
begin/end - Return all of the registers in this class.
Wrapper class representing physical registers. Should be passed by value.
LLVM_ABI MDNode * createRange(const APInt &Lo, const APInt &Hi)
Return metadata describing the range [Lo, Hi).
const MDOperand & getOperand(unsigned I) const
Helper class for constructing bundles of MachineInstrs.
MachineBasicBlock::instr_iterator begin() const
Return an iterator to the first bundled instruction.
uint64_t getScalarSizeInBits() const
bool bitsLE(MVT VT) const
Return true if this has no more bits than VT.
unsigned getVectorNumElements() const
bool isVector() const
Return true if this is a vector value type.
bool isScalableVector() const
Return true if this is a vector value type where the runtime length is machine dependent.
MVT changeTypeToInteger()
Return the type converted to an equivalently sized integer or vector with integer element type.
static LLVM_ABI MVT getVT(Type *Ty, bool HandleUnknown=false)
Return the value type corresponding to the specified type.
static auto all_valuetypes()
SimpleValueType Iteration.
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
bool isPow2VectorType() const
Returns true if the given vector is a power of 2.
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
static MVT getVectorVT(MVT VT, unsigned NumElements)
static MVT getIntegerVT(unsigned BitWidth)
MVT getScalarType() const
If this is a vector, return the element type, otherwise return this.
LLVM_ABI void transferSuccessorsAndUpdatePHIs(MachineBasicBlock *FromMBB)
Transfers all the successors, as in transferSuccessors, and update PHI operands in the successor bloc...
LLVM_ABI iterator getFirstTerminator()
Returns an iterator to the first terminator instruction of this basic block.
LLVM_ABI void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
LLVM_ABI MachineBasicBlock * splitAt(MachineInstr &SplitInst, bool UpdateLiveIns=true, LiveIntervals *LIS=nullptr)
Split a basic block into 2 pieces at SplitPoint.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
Align getAlignment() const
Return alignment of the basic block.
MachineInstrBundleIterator< MachineInstr > iterator
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
LLVM_ABI int CreateFixedObject(uint64_t Size, int64_t SPOffset, bool IsImmutable, bool isAliased=false)
Create a new object at a fixed location on the stack.
const AllocaInst * getObjectAllocation(int ObjectIdx) const
Return the underlying Alloca of the specified stack object if it exists.
bool hasCalls() const
Return true if the current function has any function calls.
void setHasTailCall(bool V=true)
void setReturnAddressIsTaken(bool s)
bool hasStackObjects() const
Return true if there are any stack objects in this function.
PseudoSourceValueManager & getPSVManager() const
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
DenormalMode getDenormalMode(const fltSemantics &FPType) const
Returns the denormal handling type for the default rounding mode of the function.
void push_back(MachineBasicBlock *MBB)
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
const DataLayout & getDataLayout() const
Return the DataLayout attached to the Module associated to this MF.
Function & getFunction()
Return the LLVM function that this machine code represents.
BasicBlockListType::iterator iterator
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
Register addLiveIn(MCRegister PReg, const TargetRegisterClass *RC)
addLiveIn - Add the specified physical register as a live-in value and create a corresponding virtual...
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags F, LLT MemTy, Align BaseAlignment, const MMOMetadata &Metadata=MMOMetadata(), SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *BB=nullptr, std::optional< UniqueBBID > BBID=std::nullopt)
CreateMachineInstr - Allocate a new MachineInstr.
void insert(iterator MBBI, MachineBasicBlock *MBB)
const TargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
const MachineInstrBuilder & setOperandDead(unsigned OpIdx) const
const MachineInstrBuilder & addReg(Register RegNo, RegState Flags={}, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & add(const MachineOperand &MO) const
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
const MachineInstrBuilder & addDef(Register RegNo, RegState Flags={}, unsigned SubReg=0) const
Add a virtual register definition operand.
const MachineInstrBuilder & cloneMemRefs(const MachineInstr &OtherMI) const
const MachineInstrBuilder & setMIFlags(unsigned Flags) const
Representation of each machine instruction.
const MachineOperand & getOperand(unsigned i) const
A description of a memory reference used in the backend.
LocationSize getSize() const
Return the size in bytes of the memory reference.
Flags
Flags values. These may be or'd together.
@ MOVolatile
The memory access is volatile.
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MOLoad
The memory access reads data.
@ MONonTemporal
The memory access is non-temporal.
@ MOInvariant
The memory access always returns the same value (or traps).
@ MOStore
The memory access writes data.
Flags getFlags() const
Return the raw flags of the source value,.
MachineOperand class - Representation of each machine instruction operand.
bool isReg() const
isReg - Tests if this is a MO_Register operand.
static MachineOperand CreateImm(int64_t Val)
void setIsUndef(bool Val=true)
Register getReg() const
getReg - Returns the register number.
static MachineOperand CreateReg(Register Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isEarlyClobber=false, unsigned SubReg=0, bool isDebug=false, bool isInternalRead=false, bool isRenamable=false)
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
An SDNode that represents everything that will be needed to construct a MachineInstr.
This is an abstract virtual class for memory operations.
unsigned getAddressSpace() const
Return the address space for the associated pointer.
MachineMemOperand * getMemOperand() const
Return the unique MachineMemOperand object describing the memory reference performed by operation.
EVT getMemoryVT() const
Return the type of the in-memory value.
bool onlyWritesMemory() const
Whether this function only (at most) writes memory.
bool doesNotAccessMemory() const
Whether this function accesses no memory.
bool onlyReadsMemory() const
Whether this function only (at most) reads memory.
const Triple & getTargetTriple() const
Get the target triple which is a string describing the target host.
static LLVM_ABI PointerType * get(LLVMContext &C, unsigned AddressSpace)
This constructs an opaque pointer to an object in a numbered address space.
static LLVM_ABI PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
LLVM_ABI const PseudoSourceValue * getConstantPool()
Return a pseudo source value referencing the constant pool.
Wrapper class representing virtual and physical registers.
static Register index2VirtReg(unsigned Index)
Convert a 0-based index to a virtual register number.
constexpr bool isPhysical() const
Return true if the specified register number is in the physical register namespace.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
bool hasOneUse() const
Return true if there is exactly one use of this node.
value_iterator value_end() const
ISD::CondCode getOperandCondCode(unsigned Num) const
Helper method returns the CondCode of a CondCodeSDNode.
SDNodeFlags getFlags() const
uint64_t getAsZExtVal() const
Helper method returns the zero-extended integer value of a ConstantSDNode.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
const SDValue & getOperand(unsigned Num) const
uint64_t getConstantOperandVal(unsigned Num) const
Helper method returns the integer value of a ConstantSDNode operand.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
bool isUndef() const
Returns true if the node type is UNDEF or POISON.
user_iterator user_begin() const
Provide iteration support to walk over all users of an SDNode.
op_iterator op_end() const
bool isAnyAdd() const
Returns true if the node type is ADD or PTRADD.
value_iterator value_begin() const
op_iterator op_begin() const
Represents a use of a SDNode.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
bool hasOneUse() const
Return true if there is exactly one node using value ResNo of Node, in exactly one operand.
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
const SDValue & getOperand(unsigned i) const
MVT getSimpleValueType() const
Return the simple ValueType of the referenced return value.
unsigned getOpcode() const
unsigned getNumOperands() const
static unsigned getMaxMUBUFImmOffset(const GCNSubtarget &ST)
static unsigned getDSShaderTypeValue(const MachineFunction &MF)
This class keeps track of the SPI_SP_INPUT_ADDR config register, which tells the hardware which inter...
bool isWholeWaveFunction() const
bool hasWorkGroupIDZ() const
AMDGPU::ClusterDimsAttr getClusterDims() const
SIModeRegisterDefaults getMode() const
std::tuple< const ArgDescriptor *, const TargetRegisterClass *, LLT > getPreloadedValue(AMDGPUFunctionArgInfo::PreloadedValue Value) const
unsigned getBytesInStackArgArea() const
const AMDGPUGWSResourcePseudoSourceValue * getGWSPSV(const AMDGPUTargetMachine &TM)
static LLVM_READONLY const TargetRegisterClass * getSGPRClassForBitWidth(unsigned BitWidth)
static unsigned getSubRegFromChannel(uint64_t Channel, uint64_t NumRegs=1)
static bool isVGPRClass(const TargetRegisterClass *RC)
static bool isSGPRClass(const TargetRegisterClass *RC)
static bool isAGPRClass(const TargetRegisterClass *RC)
bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const override
Return true if folding a constant offset with the given GlobalAddress is legal.
bool isTypeDesirableForOp(unsigned Op, EVT VT) const override
Return true if the target has native support for the specified value type and it is 'desirable' to us...
SDNode * PostISelFolding(MachineSDNode *N, SelectionDAG &DAG) const override
Fold the instructions after selecting them.
SDValue splitTernaryVectorOp(SDValue Op, SelectionDAG &DAG) const
MachineSDNode * wrapAddr64Rsrc(SelectionDAG &DAG, const SDLoc &DL, SDValue Ptr) const
bool isFMAFasterThanFMulAndFAdd(const MachineFunction &MF, EVT VT) const override
Return true if an FMA operation is faster than a pair of fmul and fadd instructions.
SDValue lowerGET_ROUNDING(SDValue Op, SelectionDAG &DAG) const
AtomicExpansionKind shouldExpandAtomicRMWInIR(const AtomicRMWInst *) const override
Returns how the IR-level AtomicExpand pass should expand the given AtomicRMW, if at all.
bool requiresUniformRegister(MachineFunction &MF, const Value *V) const override
Allows target to decide about the register class of the specific value that is live outside the defin...
bool isFMADLegal(const SelectionDAG &DAG, const SDNode *N) const override
Returns true if be combined with to form an ISD::FMAD.
AtomicExpansionKind shouldExpandAtomicStoreInIR(StoreInst *SI) const override
Returns how the given (atomic) store should be expanded by the IR-level AtomicExpand pass into.
void bundleInstWithWaitcnt(MachineInstr &MI) const
Insert MI into a BUNDLE with an S_WAITCNT 0 immediately following it.
SDValue lowerROTR(SDValue Op, SelectionDAG &DAG) const
MVT getScalarShiftAmountTy(const DataLayout &, EVT) const override
Return the type to use for a scalar shift opcode, given the shifted amount type.
SDValue LowerCall(CallLoweringInfo &CLI, SmallVectorImpl< SDValue > &InVals) const override
This hook must be implemented to lower calls into the specified DAG.
MVT getPointerTy(const DataLayout &DL, unsigned AS) const override
Map address space 7 to MVT::amdgpuBufferFatPointer because that's its in-memory representation.
bool denormalsEnabledForType(const SelectionDAG &DAG, EVT VT) const
void insertCopiesSplitCSR(MachineBasicBlock *Entry, const SmallVectorImpl< MachineBasicBlock * > &Exits) const override
Insert explicit copies in entry and exit blocks.
EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const override
Return the ValueType of the result of SETCC operations.
SDNode * legalizeTargetIndependentNode(SDNode *Node, SelectionDAG &DAG) const
Legalize target independent instructions (e.g.
bool allowsMisalignedMemoryAccessesImpl(unsigned Size, unsigned AddrSpace, Align Alignment, MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *IsFast=nullptr) const
TargetLoweringBase::LegalizeTypeAction getPreferredVectorAction(MVT VT) const override
Return the preferred vector type legalization action.
SDValue lowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const
const GCNSubtarget * getSubtarget() const
bool enableAggressiveFMAFusion(EVT VT) const override
Return true if target always benefits from combining into FMA for a given value type.
bool shouldEmitGOTReloc(const GlobalValue *GV) const
SDValue splitUnaryVectorOp(SDValue Op, SelectionDAG &DAG) const
SDValue lowerGET_FPENV(SDValue Op, SelectionDAG &DAG) const
bool isCanonicalized(SelectionDAG &DAG, SDValue Op, SDNodeFlags UserFlags={}, unsigned MaxDepth=5) const
void allocateSpecialInputSGPRs(CCState &CCInfo, MachineFunction &MF, const SIRegisterInfo &TRI, SIMachineFunctionInfo &Info) const
void allocateLDSKernelId(CCState &CCInfo, MachineFunction &MF, const SIRegisterInfo &TRI, SIMachineFunctionInfo &Info) const
SDValue LowerSTACKSAVE(SDValue Op, SelectionDAG &DAG) const
bool isReassocProfitable(SelectionDAG &DAG, SDValue N0, SDValue N1) const override
void allocateHSAUserSGPRs(CCState &CCInfo, MachineFunction &MF, const SIRegisterInfo &TRI, SIMachineFunctionInfo &Info) const
ArrayRef< MCPhysReg > getRoundingControlRegisters() const override
Returns a 0 terminated array of rounding control registers that can be attached into strict FP call.
ConstraintType getConstraintType(StringRef Constraint) const override
Given a constraint, return the type of constraint it is for this target.
SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool IsVarArg, const SmallVectorImpl< ISD::OutputArg > &Outs, const SmallVectorImpl< SDValue > &OutVals, const SDLoc &DL, SelectionDAG &DAG) const override
This hook must be implemented to lower outgoing return values, described by the Outs array,...
const TargetRegisterClass * getRegClassFor(MVT VT, bool isDivergent) const override
Return the register class that should be used for the specified value type.
void AddMemOpInit(MachineInstr &MI) const
MachineMemOperand::Flags getTargetMMOFlags(const Instruction &I) const override
This callback is used to inspect load/store instructions and add target-specific MachineMemOperand fl...
bool isLegalGlobalAddressingMode(const AddrMode &AM) const
bool shouldConvertConstantLoadToIntImm(const APInt &Imm, Type *Ty) const override
Return true if it is beneficial to convert a load of a constant to just the constant itself.
InlineAsm::ConstraintCode getInlineAsmMemConstraint(StringRef ConstraintCode) const override
bool allowsMemoryAccess(LLVMContext &Context, const DataLayout &DL, EVT VT, unsigned AddrSpace, Align Alignment, MachineMemOperand::Flags Flags, unsigned *IsFast) const override
Return true if the target supports a memory access of this type for the given address space and align...
std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const override
Given a physical register constraint (e.g.
void emitExpandAtomicStore(StoreInst *SI) const override
Perform a atomic store using a target-specific way.
AtomicExpansionKind shouldExpandAtomicLoadInIR(LoadInst *LI) const override
Returns how the given (atomic) load should be expanded by the IR-level AtomicExpand pass.
Align computeKnownAlignForTargetInstr(GISelValueTracking &Analysis, Register R, const MachineRegisterInfo &MRI, unsigned Depth=0) const override
Determine the known alignment for the pointer value R.
bool getAsmOperandConstVal(SDValue Op, uint64_t &Val) const
bool isShuffleMaskLegal(ArrayRef< int >, EVT) const override
Targets can use this to indicate that they only support some VECTOR_SHUFFLE operations,...
void emitExpandAtomicLoad(LoadInst *LI) const override
Perform a atomic load using a target-specific way.
EVT getOptimalMemOpType(LLVMContext &Context, const MemOp &Op, const AttributeList &FuncAttributes) const override
Returns the target specific optimal type for load and store operations as a result of memset,...
void computeKnownBitsForStackObjectPointer(KnownBits &Known, const MachineFunction &MF, Align Alignment) const override
Determine known bits of a pointer to a known valid stack object.
void ReplaceNodeResults(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG) const override
This callback is invoked when a node result type is illegal for the target, and the operation was reg...
Register getRegisterByName(const char *RegName, LLT VT, const MachineFunction &MF) const override
Return the register ID of the name passed in.
void LowerAsmOperandForConstraint(SDValue Op, StringRef Constraint, std::vector< SDValue > &Ops, SelectionDAG &DAG) const override
Lower the specified operand into the Ops vector.
LLT getPreferredShiftAmountTy(LLT Ty) const override
Return the preferred type to use for a shift opcode, given the shifted amount type is ShiftValueTy.
ExtractSubvectorCost getExtractSubvectorCost(EVT ResVT, EVT SrcVT, unsigned Index) const override
Return the cost of extracting a subvector of type ResVT from a vector of type SrcVT,...
bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty, unsigned AS, Instruction *I=nullptr) const override
Return true if the addressing mode represented by AM is legal for this target, for a load/store of th...
Align getPrefLoopAlignment(MachineLoop *ML, const MachineBasicBlock *BlockToAlign) const override
Return the preferred loop alignment.
SDValue lowerSET_FPENV(SDValue Op, SelectionDAG &DAG) const
bool shouldPreservePtrArith(const Function &F, EVT PtrVT) const override
True if target has some particular form of dealing with pointer arithmetic semantics for pointers wit...
void getTgtMemIntrinsic(SmallVectorImpl< IntrinsicInfo > &, const CallBase &, MachineFunction &MF, unsigned IntrinsicID) const override
Given an intrinsic, checks if on the target the intrinsic will need to map to a MemIntrinsicNode (tou...
void computeKnownBitsForTargetNode(const SDValue Op, KnownBits &Known, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth=0) const override
Determine which of the bits specified in Mask are known to be either zero or one and return them in t...
SDValue lowerSET_ROUNDING(SDValue Op, SelectionDAG &DAG) const
void allocateSpecialInputVGPRsFixed(CCState &CCInfo, MachineFunction &MF, const SIRegisterInfo &TRI, SIMachineFunctionInfo &Info) const
Allocate implicit function VGPR arguments in fixed registers.
MachineBasicBlock * emitGWSMemViolTestLoop(MachineInstr &MI, MachineBasicBlock *BB) const
bool getAddrModeArguments(const IntrinsicInst *I, SmallVectorImpl< Value * > &Ops, Type *&AccessTy) const override
CodeGenPrepare sinks address calculations into the same BB as Load/Store instructions reading the add...
bool checkAsmConstraintValA(SDValue Op, uint64_t Val, unsigned MaxSize=64) const
bool shouldEmitFixup(const GlobalValue *GV) const
MachineBasicBlock * splitKillBlock(MachineInstr &MI, MachineBasicBlock *BB) const
void emitExpandAtomicCmpXchg(AtomicCmpXchgInst *CI) const override
Perform a cmpxchg expansion using a target-specific method.
bool canTransformPtrArithOutOfBounds(const Function &F, EVT PtrVT) const override
True if the target allows transformations of in-bounds pointer arithmetic that cause out-of-bounds in...
bool hasMemSDNodeUser(SDNode *N) const
bool isSDNodeSourceOfDivergence(const SDNode *N, FunctionLoweringInfo *FLI, UniformityInfo *UA) const override
MachineBasicBlock * EmitInstrWithCustomInserter(MachineInstr &MI, MachineBasicBlock *BB) const override
This method should be implemented by targets that mark instructions with the 'usesCustomInserter' fla...
bool isEligibleForTailCallOptimization(SDValue Callee, CallingConv::ID CalleeCC, bool isVarArg, const SmallVectorImpl< ISD::OutputArg > &Outs, const SmallVectorImpl< SDValue > &OutVals, const SmallVectorImpl< ISD::InputArg > &Ins, SelectionDAG &DAG) const
bool isMemOpHasNoClobberedMemOperand(const SDNode *N) const
bool isLegalFlatAddressingMode(const AddrMode &AM, unsigned AddrSpace) const
SDValue LowerCallResult(SDValue Chain, SDValue InGlue, CallingConv::ID CallConv, bool isVarArg, const SmallVectorImpl< ISD::InputArg > &Ins, const SDLoc &DL, SelectionDAG &DAG, SmallVectorImpl< SDValue > &InVals, bool isThisReturn, SDValue ThisVal) const
SDValue LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv, bool isVarArg, const SmallVectorImpl< ISD::InputArg > &Ins, const SDLoc &DL, SelectionDAG &DAG, SmallVectorImpl< SDValue > &InVals) const override
This hook must be implemented to lower the incoming (formal) arguments, described by the Ins array,...
SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override
This method will be invoked for all target nodes and for any target-independent nodes that the target...
bool isFPExtFoldable(const SelectionDAG &DAG, unsigned Opcode, EVT DestVT, EVT SrcVT) const override
Return true if an fpext operation input to an Opcode operation is free (for instance,...
void AdjustInstrPostInstrSelection(MachineInstr &MI, SDNode *Node) const override
Assign the register class depending on the number of bits set in the writemask.
MVT getRegisterTypeForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const override
Certain combinations of ABIs, Targets and features require that types are legal for some operations a...
void finalizeLowering(MachineFunction &MF) const override
Execute target specific actions to finalize target lowering.
static bool isNonGlobalAddrSpace(unsigned AS)
void emitExpandAtomicAddrSpacePredicate(Instruction *AI) const
MachineSDNode * buildRSRC(SelectionDAG &DAG, const SDLoc &DL, SDValue Ptr, uint32_t RsrcDword1, uint64_t RsrcDword2And3) const
Return a resource descriptor with the 'Add TID' bit enabled The TID (Thread ID) is multiplied by the ...
unsigned getNumRegistersForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const override
Certain targets require unusual breakdowns of certain types.
bool mayBeEmittedAsTailCall(const CallInst *) const override
Return true if the target may be able emit the call instruction as a tail call.
void passSpecialInputs(CallLoweringInfo &CLI, CCState &CCInfo, const SIMachineFunctionInfo &Info, SmallVectorImpl< std::pair< unsigned, SDValue > > &RegsToPass, SmallVectorImpl< SDValue > &MemOpChains, SDValue Chain) const
SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override
This callback is invoked for operations that are unsupported by the target, which are registered to u...
bool checkAsmConstraintVal(SDValue Op, StringRef Constraint, uint64_t Val) const
bool isKnownNeverNaNForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, bool SNaN=false, unsigned Depth=0) const override
If SNaN is false,.
void emitExpandAtomicRMW(AtomicRMWInst *AI) const override
Perform a atomicrmw expansion using a target-specific way.
static bool shouldExpandVectorDynExt(unsigned EltSize, unsigned NumElem, bool IsDivergentIdx, const GCNSubtarget *Subtarget)
Check if EXTRACT_VECTOR_ELT/INSERT_VECTOR_ELT (<n x e>, var-idx) should be expanded into a set of cmp...
bool shouldUseLDSConstAddress(const GlobalValue *GV) const
bool supportSplitCSR(MachineFunction *MF) const override
Return true if the target supports that a subset of CSRs for the given machine function is handled ex...
bool isExtractVecEltCheap(EVT VT, unsigned Index) const override
Return true if extraction of a scalar element from the given vector type at the given index is cheap.
SDValue LowerDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG) const
bool allowsMisalignedMemoryAccesses(LLT Ty, unsigned AddrSpace, Align Alignment, MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *IsFast=nullptr) const override
LLT handling variant.
bool canMergeStoresTo(unsigned AS, EVT MemVT, const MachineFunction &MF) const override
Returns if it's reasonable to merge stores to MemVT size.
SDValue lowerPREFETCH(SDValue Op, SelectionDAG &DAG) const
SITargetLowering(const TargetMachine &tm, const GCNSubtarget &STI)
void computeKnownBitsForTargetInstr(GISelValueTracking &Analysis, Register R, KnownBits &Known, const APInt &DemandedElts, const MachineRegisterInfo &MRI, unsigned Depth=0) const override
Determine which of the bits specified in Mask are known to be either zero or one and return them in t...
bool isFreeAddrSpaceCast(const DataLayout &DL, unsigned SrcAS, unsigned DestAS) const override
Returns true if a cast from SrcAS to DestAS is "cheap", such that e.g.
bool shouldEmitPCReloc(const GlobalValue *GV) const
bool isUniformLoad(const LoadSDNode *Load) const
AtomicExpansionKind shouldExpandAtomicCmpXchgInIR(const AtomicCmpXchgInst *AI) const override
Returns how the given atomic cmpxchg should be expanded by the IR-level AtomicExpand pass.
void initializeSplitCSR(MachineBasicBlock *Entry) const override
Perform necessary initialization to handle a subset of CSRs explicitly via copies.
void allocateSpecialEntryInputVGPRs(CCState &CCInfo, MachineFunction &MF, const SIRegisterInfo &TRI, SIMachineFunctionInfo &Info) const
void allocatePreloadKernArgSGPRs(CCState &CCInfo, SmallVectorImpl< CCValAssign > &ArgLocs, const SmallVectorImpl< ISD::InputArg > &Ins, MachineFunction &MF, const SIRegisterInfo &TRI, SIMachineFunctionInfo &Info) const
SDValue copyToM0(SelectionDAG &DAG, SDValue Chain, const SDLoc &DL, SDValue V) const
SDValue splitBinaryVectorOp(SDValue Op, SelectionDAG &DAG) const
MachinePointerInfo getKernargSegmentPtrInfo(MachineFunction &MF) const
unsigned getVectorTypeBreakdownForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT, EVT &IntermediateVT, unsigned &NumIntermediates, MVT &RegisterVT) const override
Certain targets such as MIPS require that some types such as vectors are always broken down into scal...
MVT getPointerMemTy(const DataLayout &DL, unsigned AS) const override
Similarly, the in-memory representation of a p7 is {p8, i32}, aka v8i32 when padding is added.
void allocateSystemSGPRs(CCState &CCInfo, MachineFunction &MF, SIMachineFunctionInfo &Info, CallingConv::ID CallConv, bool IsShader) const
bool CanLowerReturn(CallingConv::ID CallConv, MachineFunction &MF, bool isVarArg, const SmallVectorImpl< ISD::OutputArg > &Outs, LLVMContext &Context, const Type *RetTy) const override
This hook should be implemented to check whether the return values described by the Outs array can fi...
unsigned getMaxPermittedBytesForAlignment(MachineBasicBlock *MBB) const override
Return the maximum amount of bytes allowed to be emitted when padding for alignment.
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
SDValue getTargetGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, unsigned TargetFlags=0)
SDValue getExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT, unsigned Opcode)
Convert Op, which must be of integer type, to the integer type VT, by either any/sign/zero-extending ...
SDValue getExtractVectorElt(const SDLoc &DL, EVT VT, SDValue Vec, unsigned Idx)
Extract element at Idx from Vec.
const SDValue & getRoot() const
Return the root tag of the SelectionDAG.
bool isKnownNeverSNaN(SDValue Op, const APInt &DemandedElts, unsigned Depth=0) const
const TargetSubtargetInfo & getSubtarget() const
SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, Register Reg, SDValue N)
LLVM_ABI SDValue getMergeValues(ArrayRef< SDValue > Ops, const SDLoc &dl)
Create a MERGE_VALUES node from the given operands.
LLVM_ABI SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
LLVM_ABI SDValue getShiftAmountConstant(uint64_t Val, EVT VT, const SDLoc &DL)
LLVM_ABI SDValue getAllOnesConstant(const SDLoc &DL, EVT VT, bool IsTarget=false, bool IsOpaque=false)
LLVM_ABI MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s),...
LLVM_ABI void ExtractVectorElements(SDValue Op, SmallVectorImpl< SDValue > &Args, unsigned Start=0, unsigned Count=0, EVT EltVT=EVT())
Append the extracted elements from Start to Count out of the vector Op in Args.
LLVM_ABI SDValue getAtomicLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT MemVT, EVT VT, SDValue Chain, SDValue Ptr, MachineMemOperand *MMO)
LLVM_ABI SDValue getFreeze(SDValue V)
Return a freeze using the SDLoc of the value operand.
LLVM_ABI bool isConstantIntBuildVectorOrConstantInt(SDValue N, bool AllowOpaques=true) const
Test whether the given value is a constant int or similar node.
LLVM_ABI SDValue UnrollVectorOp(SDNode *N, unsigned ResNE=0)
Utility function used by legalize and lowering to "unroll" a vector operation by splitting out the sc...
LLVM_ABI SDValue getConstantFP(double Val, const SDLoc &DL, EVT VT, bool isTarget=false)
Create a ConstantFPSDNode wrapping a constant value.
LLVM_ABI bool haveNoCommonBitsSet(SDValue A, SDValue B) const
Return true if A and B have no common bits set.
LLVM_ABI SDValue getAddrSpaceCast(const SDLoc &dl, EVT VT, SDValue Ptr, unsigned SrcAS, unsigned DestAS, const SDNodeFlags Flags=SDNodeFlags())
Return an AddrSpaceCastSDNode.
LLVM_ABI SDValue getRegister(Register Reg, EVT VT)
LLVM_ABI bool SignBitIsZeroFP(SDValue Op, unsigned Depth=0) const
Return true if the sign bit of Op is known to be zero, for a floating-point value.
LLVM_ABI SDValue getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef< SDValue > Ops, EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags Flags=MachineMemOperand::MOLoad|MachineMemOperand::MOStore, LocationSize Size=LocationSize::precise(0), const AAMDNodes &AAInfo=AAMDNodes())
Creates a MemIntrinsicNode that may produce a result and takes a list of operands.
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Chain=SDValue(), bool IsSignaling=false, SDNodeFlags Flags={})
Helper function to make it easier to build SetCC's if you just have an ISD::CondCode instead of an SD...
LLVM_ABI SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Val, MachineMemOperand *MMO)
Gets a node for an atomic op, produces result (if relevant) and chain and takes 2 operands.
void addNoMergeSiteInfo(const SDNode *Node, bool NoMerge)
Set NoMergeSiteInfo to be associated with Node if NoMerge is true.
std::pair< SDValue, SDValue > SplitVectorOperand(const SDNode *N, unsigned OpNo)
Split the node's operand with EXTRACT_SUBVECTOR and return the low/high part.
LLVM_ABI SDValue getNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a bitwise NOT operation as (XOR Val, -1).
LLVM_ABI SDValue getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Align DstAlign, Align SrcAlign, bool isVol, bool AlwaysInline, const CallInst *CI, std::optional< bool > OverrideTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo=AAMDNodes(), BatchAAResults *BatchAA=nullptr)
const TargetLowering & getTargetLoweringInfo() const
LLVM_ABI std::pair< EVT, EVT > GetSplitDestVTs(const EVT &VT) const
Compute the VTs needed for the low/hi parts of a type which is split (or expanded) into two not neces...
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
SDValue getCALLSEQ_END(SDValue Chain, SDValue Op1, SDValue Op2, SDValue InGlue, const SDLoc &DL)
Return a new CALLSEQ_END node, which always must have a glue result (to ensure it's not CSE'd).
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
LLVM_ABI SDValue getBitcastedAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by first bitcasting (from potentia...
LLVM_ABI SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, SDValue Offset, MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const MMOMetadata &Metadata=MMOMetadata())
LLVM_ABI SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, Register Reg, EVT VT)
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS, SDNodeFlags Flags=SDNodeFlags())
Helper function to make it easier to build Select's if you just have operands and don't want to check...
LLVM_ABI void setNodeMemRefs(MachineSDNode *N, ArrayRef< MachineMemOperand * > NewMemRefs)
Mutate the specified machine node's memory references to the provided list.
LLVM_ABI SDValue getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return the expression required to zero extend the Op value assuming it was the smaller SrcTy value.
const DataLayout & getDataLayout() const
LLVM_ABI SDValue getTokenFactor(const SDLoc &DL, SmallVectorImpl< SDValue > &Vals)
Creates a new TokenFactor containing Vals.
LLVM_ABI SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const MMOMetadata &Metadata=MMOMetadata())
Helper function to build ISD::STORE nodes.
LLVM_ABI SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
LLVM_ABI SDValue getMemBasePlusOffset(SDValue Base, TypeSize Offset, const SDLoc &DL, const SDNodeFlags Flags=SDNodeFlags())
Returns sum of the base pointer and offset.
SDValue getSignedTargetConstant(int64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
LLVM_ABI void ReplaceAllUsesWith(SDValue From, SDValue To)
Modify anything using 'From' to use 'To' instead.
LLVM_ABI SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const MMOMetadata &Metadata=MMOMetadata())
LLVM_ABI SDValue getSignedConstant(int64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
SDValue getCALLSEQ_START(SDValue Chain, uint64_t InSize, uint64_t OutSize, const SDLoc &DL)
Return a new CALLSEQ_START node, that starts new call frame, in which InSize bytes are set up inside ...
LLVM_ABI void RemoveDeadNode(SDNode *N)
Remove the specified node from the system.
LLVM_ABI SDValue getTargetExtractSubreg(int SRIdx, const SDLoc &DL, EVT VT, SDValue Operand)
A convenience function for creating TargetInstrInfo::EXTRACT_SUBREG nodes.
SDValue getSelectCC(const SDLoc &DL, SDValue LHS, SDValue RHS, SDValue True, SDValue False, ISD::CondCode Cond, SDNodeFlags Flags=SDNodeFlags())
Helper function to make it easier to build SelectCC's if you just have an ISD::CondCode instead of an...
LLVM_ABI SDValue getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either sign-extending or trunca...
LLVM_ABI SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const MMOMetadata &Metadata=MMOMetadata())
Loads are not normal binary operators: their result type is not determined by their operands,...
const TargetMachine & getTarget() const
LLVM_ABI SDValue getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either any-extending or truncat...
LLVM_ABI SDValue getValueType(EVT)
LLVM_ABI SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
LLVM_ABI bool isKnownNeverNaN(SDValue Op, const APInt &DemandedElts, bool SNaN=false, unsigned Depth=0) const
Test whether the given SDValue (or all elements of it, if it is a vector) is known to never be NaN in...
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
LLVM_ABI unsigned ComputeNumSignBits(SDValue Op, unsigned Depth=0) const
Return the number of times the sign bit of the register is replicated into the other bits.
LLVM_ABI bool isBaseWithConstantOffset(SDValue Op) const
Return true if the specified operand is an ISD::ADD with a ConstantSDNode on the right-hand side,...
LLVM_ABI SDValue getVectorIdxConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
LLVM_ABI void ReplaceAllUsesOfValueWith(SDValue From, SDValue To)
Replace any uses of From with To, leaving uses of other values produced by From.getNode() alone.
MachineFunction & getMachineFunction() const
SDValue getPOISON(EVT VT)
Return a POISON node. POISON does not have a useful SDLoc.
SDValue getSplatBuildVector(EVT VT, const SDLoc &DL, SDValue Op)
Return a splat ISD::BUILD_VECTOR node, consisting of Op splatted to all elements.
LLVM_ABI SDValue getErrorMergeValues(ArrayRef< EVT > ResultTypes, SDValue Chain, const SDLoc &dl)
Return poison values for each of ResultTypes, substituting Chain for any result of type MVT::Other,...
LLVM_ABI SDValue getFrameIndex(int FI, EVT VT, bool isTarget=false)
LLVM_ABI KnownBits computeKnownBits(SDValue Op, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in Known.
LLVM_ABI SDValue getRegisterMask(const uint32_t *RegMask)
LLVM_ABI SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
LLVM_ABI SDValue getCondCode(ISD::CondCode Cond)
LLVM_ABI bool MaskedValueIsZero(SDValue Op, const APInt &Mask, unsigned Depth=0) const
Return true if 'Op & Mask' is known to be zero.
SDValue getObjectPtrOffset(const SDLoc &SL, SDValue Ptr, TypeSize Offset)
Create an add instruction with appropriate flags when used for addressing some offset of an object.
LLVMContext * getContext() const
const SDValue & setRoot(SDValue N)
Set the current root tag of the SelectionDAG.
LLVM_ABI SDNode * UpdateNodeOperands(SDNode *N, SDValue Op)
Mutate the specified node in-place to have the specified operands.
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
LLVM_ABI std::pair< SDValue, SDValue > SplitScalar(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the scalar node with EXTRACT_ELEMENT using the provided VTs and return the low/high part.
LLVM_ABI SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
int getMaskElt(unsigned Idx) const
ArrayRef< int > getMask() const
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
size_type count(const T &V) const
count - Return 1 if the element is in the set, 0 otherwise.
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void reserve(size_type N)
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
void push_back(const T &Elt)
LLVM_NO_SANITIZE_CFI_UNRELATED_CAST iterator begin()
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
An instruction for storing to memory.
Represent a constant reference to a string, i.e.
constexpr bool empty() const
Check if the string is empty.
constexpr size_t size() const
Get the string size.
A switch()-like statement whose cases are string literals.
StringSwitch & Case(StringLiteral S, T Value)
Information about stack frame layout on the target.
Align getStackAlign() const
getStackAlignment - This method returns the number of bytes to which the stack pointer must be aligne...
StackDirection getStackGrowthDirection() const
getStackGrowthDirection - Return the direction the stack grows
TargetInstrInfo - Interface to description of machine instruction set.
Type * Ty
Same as OrigTy, or partially legalized for soft float libcalls.
void setBooleanVectorContents(BooleanContent Ty)
Specify how the target extends the result of a vector boolean value from a vector of i1 to a wider ty...
void setOperationAction(unsigned Op, MVT VT, LegalizeAction Action)
Indicate that the specified operation does not work with the specified type and indicate what to do a...
virtual void finalizeLowering(MachineFunction &MF) const
Execute target specific actions to finalize target lowering.
EVT getValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the EVT corresponding to this LLVM type.
virtual const TargetRegisterClass * getRegClassFor(MVT VT, bool isDivergent=false) const
Return the register class that should be used for the specified value type.
virtual Align getPrefLoopAlignment(MachineLoop *ML=nullptr, const MachineBasicBlock *BlockToAlign=nullptr) const
Return the preferred loop alignment.
virtual unsigned getMaxPermittedBytesForAlignment(MachineBasicBlock *MBB) const
Return the maximum amount of bytes allowed to be emitted when padding for alignment.
const TargetMachine & getTargetMachine() const
virtual unsigned getNumRegistersForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const
Certain targets require unusual breakdowns of certain types.
virtual MVT getRegisterTypeForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const
Certain combinations of ABIs, Targets and features require that types are legal for some operations a...
void setOperationPromotedToType(unsigned Opc, MVT OrigVT, MVT DestVT)
Convenience method to set an operation to Promote and specify the type in a single call.
LegalizeTypeAction
This enum indicates whether a types are legal for a target, and if not, what action should be used to...
void setHasExtractBitsInsn(bool hasExtractInsn=true)
Tells the code generator that the target has BitExtract instructions.
virtual TargetLoweringBase::LegalizeTypeAction getPreferredVectorAction(MVT VT) const
Return the preferred vector type legalization action.
virtual unsigned getVectorTypeBreakdownForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT, EVT &IntermediateVT, unsigned &NumIntermediates, MVT &RegisterVT) const
Certain targets such as MIPS require that some types such as vectors are always broken down into scal...
Register getStackPointerRegisterToSaveRestore() const
If a physical register, this specifies the register that llvm.savestack/llvm.restorestack should save...
void setMinFunctionAlignment(Align Alignment)
Set the target's minimum function alignment.
bool isOperationCustom(unsigned Op, EVT VT) const
Return true if the operation uses custom lowering, regardless of whether the type is legal or not.
void setBooleanContents(BooleanContent Ty)
Specify how the target extends the result of integer and floating point boolean values from i1 to a w...
void computeRegisterProperties(const TargetRegisterInfo *TRI)
Once all of the register classes are added, this allows us to compute derived properties we expose.
void addRegisterClass(MVT VT, const TargetRegisterClass *RC)
Add the specified register class as an available regclass for the specified value type.
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
ExtractSubvectorCost
Enum that specifies how expensive lowering an EXTRACT_SUBVECTOR is.
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
void setPrefFunctionAlignment(Align Alignment)
Set the target's preferred function alignment.
bool isOperationLegal(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target.
void setTruncStoreAction(MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified truncating store does not work with the specified type and indicate what ...
@ ZeroOrOneBooleanContent
bool isOperationLegalOrCustom(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
virtual bool allowsMemoryAccess(LLVMContext &Context, const DataLayout &DL, EVT VT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *Fast=nullptr) const
Return true if the target supports a memory access of this type for the given address space and align...
virtual bool isNarrowingProfitable(SDNode *N, EVT SrcVT, EVT DestVT) const
Return true if it's profitable to narrow operations of type SrcVT to DestVT.
void setStackPointerRegisterToSaveRestore(Register R)
If set to a physical register, this specifies the register that llvm.savestack/llvm....
void AddPromotedToType(unsigned Opc, MVT OrigVT, MVT DestVT)
If Opc/OrigVT is specified as being promoted, the promotion code defaults to trying a larger integer/...
AtomicExpansionKind
Enum that specifies what an atomic load/AtomicRMWInst is expanded to, if at all.
void setTargetDAGCombine(ArrayRef< ISD::NodeType > NTs)
Targets should invoke this method for each target independent node that they want to provide a custom...
bool allowsMemoryAccessForAlignment(LLVMContext &Context, const DataLayout &DL, EVT VT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *Fast=nullptr) const
This function returns true if the memory access is aligned or if the target allows this specific unal...
virtual MVT getPointerMemTy(const DataLayout &DL, uint32_t AS=0) const
Return the in-memory pointer type for the given address space, defaults to the pointer type from the ...
void setSchedulingPreference(Sched::Preference Pref)
Specify the target scheduling preference.
LegalizeAction getOperationAction(unsigned Op, EVT VT) const
Return how this operation should be treated: either it is legal, needs to be promoted to a larger siz...
SDValue scalarizeVectorStore(StoreSDNode *ST, SelectionDAG &DAG) const
virtual InlineAsm::ConstraintCode getInlineAsmMemConstraint(StringRef ConstraintCode) const
std::vector< AsmOperandInfo > AsmOperandInfoVector
SDValue SimplifyMultipleUseDemandedBits(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, SelectionDAG &DAG, unsigned Depth=0) const
More limited version of SimplifyDemandedBits that can be used to "lookthrough" ops that don't contrib...
SDValue expandUnalignedStore(StoreSDNode *ST, SelectionDAG &DAG) const
Expands an unaligned store to 2 half-size stores for integer values, and possibly more for vectors.
virtual ConstraintType getConstraintType(StringRef Constraint) const
Given a constraint, return the type of constraint it is for this target.
bool parametersInCSRMatch(const MachineRegisterInfo &MRI, const uint32_t *CallerPreservedMask, const SmallVectorImpl< CCValAssign > &ArgLocs, const SmallVectorImpl< SDValue > &OutVals) const
Check whether parameters to a call that are passed in callee saved registers are the same as from the...
std::pair< SDValue, SDValue > expandUnalignedLoad(LoadSDNode *LD, SelectionDAG &DAG) const
Expands an unaligned load to 2 half-size loads for an integer, and possibly more for vectors.
SDValue expandFMINIMUMNUM_FMAXIMUMNUM(SDNode *N, SelectionDAG &DAG) const
Expand fminimumnum/fmaximumnum into multiple comparison with selects.
virtual bool isTypeDesirableForOp(unsigned, EVT VT) const
Return true if the target has native support for the specified value type and it is 'desirable' to us...
virtual void computeKnownBitsForStackObjectPointer(KnownBits &Known, const MachineFunction &MF, Align Alignment) const
Determine known bits of a pointer to a known valid stack object.
std::pair< SDValue, SDValue > scalarizeVectorLoad(LoadSDNode *LD, SelectionDAG &DAG) const
Turn load of vector type into a load of the individual elements.
virtual std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const
Given a physical register constraint (e.g.
bool SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth=0, bool AssumeSingleUse=false) const
Look at Op.
TargetLowering(const TargetLowering &)=delete
virtual MachineBasicBlock * EmitInstrWithCustomInserter(MachineInstr &MI, MachineBasicBlock *MBB) const
This method should be implemented by targets that mark instructions with the 'usesCustomInserter' fla...
virtual AsmOperandInfoVector ParseConstraints(const DataLayout &DL, const TargetRegisterInfo *TRI, const CallBase &Call) const
Split up the constraint string from the inline assembly value into the specific constraints and their...
SDValue expandRoundInexactToOdd(EVT ResultVT, SDValue Op, const SDLoc &DL, SelectionDAG &DAG) const
Truncate Op to ResultVT.
virtual void ComputeConstraintToUse(AsmOperandInfo &OpInfo, SDValue Op, SelectionDAG *DAG=nullptr) const
Determines the constraint code and constraint type to use for the specific AsmOperandInfo,...
SDValue annotateStackObjectPointer(SDValue Ptr, SelectionDAG &DAG, const SDLoc &DL, Align Alignment) const
Annotate a stack object pointer with known-bits assertions.
virtual void LowerAsmOperandForConstraint(SDValue Op, StringRef Constraint, std::vector< SDValue > &Ops, SelectionDAG &DAG) const
Lower the specified operand into the Ops vector.
SDValue expandFMINNUM_FMAXNUM(SDNode *N, SelectionDAG &DAG) const
Expand fminnum/fmaxnum into fminnum_ieee/fmaxnum_ieee with quieted inputs.
Primary interface to the complete machine description for the target machine.
CodeGenOptLevel getOptLevel() const
Returns the optimization level: None, Less, Default, or Aggressive.
const Triple & getTargetTriple() const
bool shouldAssumeDSOLocal(const GlobalValue *GV) const
unsigned GuaranteedTailCallOpt
GuaranteedTailCallOpt - This flag is enabled when -tailcallopt is specified on the commandline.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
Target - Wrapper for Target specific information.
Triple - Helper class for working with autoconf configuration names.
OSType getOS() const
Get the parsed operating system type of this triple.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
static constexpr TypeSize getFixed(ScalarTy ExactSize)
The instances of the Type class are immutable: once they are created, they are never changed.
static LLVM_ABI IntegerType * getInt32Ty(LLVMContext &C)
bool isBFloatTy() const
Return true if this is 'bfloat', a 16-bit bfloat type.
LLVM_ABI unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
bool isHalfTy() const
Return true if this is 'half', a 16-bit IEEE fp type.
bool isFunctionTy() const
True if this is an instance of FunctionType.
bool isIntegerTy() const
True if this is an instance of IntegerType.
LLVM_ABI const fltSemantics & getFltSemantics() const
bool isVoidTy() const
Return true if this is 'void'.
A Use represents the edge between a Value definition and its users.
LLVM_ABI unsigned getOperandNo() const
Return the operand # of this use in its User.
LLVM_ABI void set(Value *Val)
User * getUser() const
Returns the User that contains this Use.
const Use & getOperandUse(unsigned i) const
Value * getOperand(unsigned i) const
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
bool hasOneUse() const
Return true if there is exactly one use of this value.
LLVM_ABI void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
LLVMContext & getContext() const
All values hold a context through their type.
iterator_range< user_iterator > users()
iterator_range< use_iterator > uses()
Type * getElementType() const
constexpr ScalarTy getFixedValue() const
constexpr bool isKnownEven() const
A return value of true indicates we know at compile time that the number of elements (vscale * Min) i...
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
self_iterator getIterator()
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ CONSTANT_ADDRESS_32BIT
Address space for 32-bit constant memory.
@ BUFFER_STRIDED_POINTER
Address space for 192-bit fat buffer pointers with an additional index.
@ BARRIER
Address space for modeling barrier IDs as addresses.
@ REGION_ADDRESS
Address space for region memory. (GDS)
@ LOCAL_ADDRESS
Address space for local memory.
@ STREAMOUT_REGISTER
Internal address spaces. Can be freely renumbered.
@ VGPR
Address space for VGPRs.
@ CONSTANT_ADDRESS
Address space for constant memory (VTX2).
@ FLAT_ADDRESS
Address space for flat memory.
@ GLOBAL_ADDRESS
Address space for global memory (RAT0, VTX0).
@ BUFFER_FAT_POINTER
Address space for 160-bit buffer fat pointers.
@ PRIVATE_ADDRESS
Address space for private memory.
@ BUFFER_RESOURCE
Address space for 128-bit buffer resources.
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
constexpr char NumVGPRs[]
Key for Kernel::CodeProps::Metadata::mNumVGPRs.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
constexpr char SymbolName[]
Key for Kernel::Metadata::mSymbolName.
APInt getExponentOnlyScaleDemandedBits()
LLVM_READONLY const MIMGG16MappingInfo * getMIMGG16MappingInfo(unsigned G)
bool isInlinableLiteralFP16(int16_t Literal, bool HasInv2Pi)
LLVM_READNONE constexpr bool isShader(CallingConv::ID CC)
bool shouldEmitConstantsToTextSection(const Triple &TT)
bool isFlatGlobalAddrSpace(unsigned AS)
const uint64_t FltRoundToHWConversionTable
bool isGFX12Plus(const MCSubtargetInfo &STI)
unsigned getNSAMaxSize(const MCSubtargetInfo &STI, bool HasSampler)
std::optional< unsigned > getExponentOnlyScaleArgIdx(unsigned IntrID)
constexpr int64_t getNullPointerValue(unsigned AS)
Get the null pointer value for the given address space.
bool isGFX11(const MCSubtargetInfo &STI)
bool isGFX13(const MCSubtargetInfo &STI)
bool hasValueInRangeLikeMetadata(const MDNode &MD, int64_t Val)
Checks if Val is inside MD, a !range-like metadata.
LLVM_ABI IsaVersion getIsaVersion(StringRef GPU)
LLVM_READNONE bool isLegalDPALU_DPPControl(const MCSubtargetInfo &ST, unsigned DC)
LLVM_READNONE constexpr bool mayTailCallThisCC(CallingConv::ID CC)
Return true if we might ever do TCO for calls with this calling convention.
unsigned getAMDHSACodeObjectVersion(const Module &M)
LLVM_READONLY bool hasNamedOperand(uint32_t Opcode, OpName NamedIdx)
int getMIMGOpcode(unsigned BaseOpcode, unsigned MIMGEncoding, unsigned VDataDwords, unsigned VAddrDwords, bool IndexedRsrc, bool IndexedSamp)
LLVM_READNONE constexpr bool isKernel(CallingConv::ID CC)
LLVM_READNONE constexpr bool isEntryFunctionCC(CallingConv::ID CC)
bool isInlinableLiteral32(int32_t Literal, bool HasInv2Pi)
LLVM_READNONE constexpr bool isCompute(CallingConv::ID CC)
const BufferIntrinsicInfo * lookupBufferIntrinsic(unsigned Intr)
bool isIntrinsicSourceOfDivergence(unsigned IntrID)
LLVM_READNONE bool isInlinableIntLiteral(int64_t Literal)
Is this literal inlinable, and not one of the values intended for floating point values.
bool getMUBUFTfe(unsigned Opc)
bool isInlinableLiteralBF16(int16_t Literal, bool HasInv2Pi, bool *IsBF16)
TargetExtType * isNamedBarrier(const GlobalVariable &GV)
LLVM_READONLY int32_t getGlobalSaddrOp(uint32_t Opcode)
bool isGFX11Plus(const MCSubtargetInfo &STI)
std::optional< unsigned > getInlineEncodingV2F16(uint32_t Literal)
std::tuple< char, unsigned, unsigned > parseAsmConstraintPhysReg(StringRef Constraint)
Returns a valid charcode or 0 in the first entry if this is a valid physical register constraint.
bool isGFX10Plus(const MCSubtargetInfo &STI)
bool isValidWMMAScaleFmtCombination(unsigned AFmt, unsigned AScale, unsigned BFmt, unsigned BScale)
@ TowardZeroF32_TowardNegativeF64
bool isUniformMMO(const MachineMemOperand *MMO)
std::optional< unsigned > getInlineEncodingV2I16(uint32_t Literal)
uint32_t decodeFltRoundToHWConversionTable(uint32_t FltRounds)
Read the hardware rounding mode equivalent of a AMDGPUFltRounds value.
bool isExtendedGlobalAddrSpace(unsigned AS)
LLVM_READONLY const MIMGDimInfo * getMIMGDimInfo(unsigned DimEnum)
unsigned getBarrierMemberCountBitMask(const IsaVersion &Version)
std::optional< unsigned > getInlineEncodingV2BF16(uint32_t Literal)
LLVM_READONLY const MIMGBaseOpcodeInfo * getMIMGBaseOpcodeInfo(unsigned BaseOpcode)
unsigned getSyntheticApertureNumber(unsigned AS)
LLVM_READNONE constexpr bool isChainCC(CallingConv::ID CC)
constexpr unsigned getSDOperandIdx(unsigned Arg, bool HasChain=true)
Convert IR argument index Arg of an intrinsic into an operand index of the INTRINSIC_* SelectionDAG n...
int getMaskedMIMGOp(unsigned Opc, unsigned NewChannels)
const ImageDimIntrinsicInfo * getImageDimIntrinsicInfo(unsigned Intr)
bool isInlinableLiteralI16(int32_t Literal, bool HasInv2Pi)
LLVM_READNONE constexpr bool canGuaranteeTCO(CallingConv::ID CC)
LLVM_READNONE constexpr bool isGraphics(CallingConv::ID CC)
bool isInlinableLiteral64(int64_t Literal, bool HasInv2Pi)
Is this literal inlinable.
const RsrcIntrinsic * lookupRsrcIntrinsic(unsigned Intr)
const uint64_t FltRoundConversionTable
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
@ AMDGPU_CS
Used for Mesa/AMDPAL compute shaders.
@ AMDGPU_KERNEL
Used for AMDGPU code object kernels.
@ MaxID
The highest possible ID. Must be some 2^k - 1.
@ AMDGPU_Gfx
Used for AMD graphics targets.
@ AMDGPU_CS_ChainPreserve
Used on AMDGPUs to give the middle-end more control over argument placement.
@ AMDGPU_CS_Chain
Used on AMDGPUs to give the middle-end more control over argument placement.
@ AMDGPU_PS
Used for Mesa/AMDPAL pixel shaders.
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
@ STACKSAVE
STACKSAVE - STACKSAVE has one operand, an input chain.
@ PTRADD
PTRADD represents pointer arithmetic semantics, for targets that opt in using shouldPreservePtrArith(...
@ DELETED_NODE
DELETED_NODE - This is an illegal value that is used to catch errors.
@ POISON
POISON - A poison node.
@ SET_FPENV
Sets the current floating-point environment.
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
@ BSWAP
Byte Swap and Counting operators.
@ ATOMIC_STORE
OUTCHAIN = ATOMIC_STORE(INCHAIN, val, ptr) This corresponds to "store atomic" instruction.
@ FMAD
FMAD - Perform a * b + c, while getting the same result as the separately rounded operations.
@ ADD
Simple integer binary arithmetic operators.
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
@ ATOMIC_CMP_SWAP_WITH_SUCCESS
Val, Success, OUTCHAIN = ATOMIC_CMP_SWAP_WITH_SUCCESS(INCHAIN, ptr, cmp, swap) N.b.
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
@ FADD
Simple binary floating point operators.
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
@ FP16_TO_FP
FP16_TO_FP, FP_TO_FP16 - These operators are used to perform promotions and truncation for half-preci...
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
@ FLDEXP
FLDEXP - ldexp, inspired by libm (op0 * 2**op1).
@ BUILTIN_OP_END
BUILTIN_OP_END - This must be the last enum value in this list.
@ CONVERT_FROM_ARBITRARY_FP
CONVERT_FROM_ARBITRARY_FP - This operator converts from an arbitrary floating-point represented as an...
@ SET_ROUNDING
Set rounding mode.
@ CONVERGENCECTRL_GLUE
This does not correspond to any convergence control intrinsic.
@ SIGN_EXTEND
Conversion operators.
@ SCALAR_TO_VECTOR
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
@ READSTEADYCOUNTER
READSTEADYCOUNTER - This corresponds to the readfixedcounter intrinsic.
@ BR
Control flow instructions. These all have token chains.
@ PREFETCH
PREFETCH - This corresponds to a prefetch intrinsic.
@ FSINCOS
FSINCOS - Compute both fsin and fcos as a single operation.
@ FNEG
Perform various unary floating-point operations inspired by libm.
@ BR_CC
BR_CC - Conditional branch.
@ SSUBO
Same for subtraction.
@ FCANONICALIZE
Returns platform specific canonical encoding of a floating point number.
@ IS_FPCLASS
Performs a check of floating point class property, defined by IEEE-754.
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
@ ATOMIC_LOAD
Val, OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr) This corresponds to "load atomic" instruction.
@ UNDEF
UNDEF - An undefined node.
@ EXTRACT_ELEMENT
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant,...
@ CopyFromReg
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
@ CTLS
Count leading redundant sign bits.
@ GET_ROUNDING
Returns current rounding mode: -1 Undefined 0 Round to 0 1 Round to nearest, ties to even 2 Round to ...
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
@ GET_FPMODE
Reads the current dynamic floating-point control modes.
@ GET_FPENV
Gets the current floating-point environment.
@ SHL
Shift and rotation operations.
@ VECTOR_SHUFFLE
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
@ FMINNUM_IEEE
FMINNUM_IEEE/FMAXNUM_IEEE - Same as FMINNUM/FMAXNUM, except that a signaling NaN operand deterministi...
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ CopyToReg
CopyToReg - This node has three operands: a chain, a register number to set to this value,...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ DEBUGTRAP
DEBUGTRAP - Trap intended to get the attention of a debugger.
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
@ ATOMIC_CMP_SWAP
Val, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap) For double-word atomic operations: ValLo,...
@ FMINNUM
FMINNUM/FMAXNUM - Same semantics as the llvm.minnum/maxnum intrinsics.
@ SMULO
Same for multiplication.
@ DYNAMIC_STACKALLOC
DYNAMIC_STACKALLOC - Allocate some number of bytes on the stack aligned to a specified boundary.
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
@ UADDO_CARRY
Carry-using nodes for multiple precision addition and subtraction.
@ INLINEASM_BR
INLINEASM_BR - Branching version of inline asm. Used by asm-goto.
@ BF16_TO_FP
BF16_TO_FP, FP_TO_BF16 - These operators are used to perform promotions and truncation for bfloat16.
@ STRICT_FP_ROUND
X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision ...
@ FMINIMUM
FMINIMUM/FMAXIMUM - Same semantics as the llvm.minimum/maximum intrinsics.
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ READCYCLECOUNTER
READCYCLECOUNTER - This corresponds to the readcyclecounter intrinsic.
@ STRICT_FP_EXTEND
X = STRICT_FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ TRAP
TRAP - Trapping instruction.
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
@ ATOMIC_SWAP
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN,...
@ CTTZ_ZERO_POISON
Bit counting operators with a poisoned result for zero inputs.
@ FFREXP
FFREXP - frexp, extract fractional and exponent component of a floating-point value.
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
@ SPONENTRY
SPONENTRY - Represents the llvm.sponentry intrinsic.
@ ADDRSPACECAST
ADDRSPACECAST - This operator converts between pointers of different address spaces.
@ INLINEASM
INLINEASM - Represents an inline asm block.
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ BRCOND
BRCOND - Conditional branch.
@ CONVERT_TO_ARBITRARY_FP
CONVERT_TO_ARBITRARY_FP - Converts a native FP value to an arbitrary floating-point format,...
@ SHL_PARTS
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations.
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
@ FMINIMUMNUM
FMINIMUMNUM/FMAXIMUMNUM - Same semantics as the llvm.minimumnum/maximumnum intrinsics.
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
CondCode getUnsignedSetCC(CondCode Code)
Convert a signed integer condition code to unsigned.
LLVM_ABI CondCode getSetCCSwappedOperands(CondCode Operation)
Return the operation corresponding to (Y op X) when given the operation for (X op Y).
bool isSignedIntSetCC(CondCode Code)
Return true if this is a setcc instruction that performs a signed comparison when used with integer o...
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
This namespace contains an enum with a value for every intrinsic/builtin function known by LLVM.
LLVM_ABI Function * getDeclarationIfExists(const Module *M, ID id)
Look up the Function declaration of the intrinsic id in the Module M and return it if it exists.
LLVM_ABI AttributeSet getFnAttributes(LLVMContext &C, ID id)
Return the function attributes for an intrinsic.
LLVM_ABI StringRef getBaseName(ID id)
Return the LLVM name for an intrinsic, without encoded types for overloading, such as "llvm....
LLVM_ABI AttributeList getAttributes(LLVMContext &C, ID id, FunctionType *FT)
Return the attributes for an intrinsic.
LLVM_ABI FunctionType * getType(LLVMContext &Context, ID id, ArrayRef< Type * > OverloadTys={})
Return the function type for an intrinsic.
BinaryOp_match< SpecificConstantMatch, SrcTy, TargetOpcode::G_SUB > m_Neg(const SrcTy &&Src)
Matches a register negated by a G_SUB.
bool mi_match(Reg R, const MachineRegisterInfo &MRI, Pattern &&P)
GFCstOrSplatGFCstMatch m_GFCstOrSplat(std::optional< FPValueAndVReg > &FPValReg)
BinaryOp_match< LHS, RHS, Instruction::Add > m_Add(const LHS &L, const RHS &R)
specificval_ty m_Specific(const Value *V)
Match if we have a specific specified value.
cst_pred_ty< is_one > m_One()
Match an integer 1 or a vector with all elements equal to 1.
specific_fpval m_SpecificFP(double V)
Match a specific floating point value or vector with all elements equal to the value.
auto m_Value()
Match an arbitrary value and ignore it.
auto m_FAbs(const Opnd0 &Op0)
BinaryOp_match< LHS, RHS, Instruction::Shl > m_Shl(const LHS &L, const RHS &R)
BinaryOp_match< LHS, RHS, Instruction::Sub > m_Sub(const LHS &L, const RHS &R)
auto m_IntrinsicWOChain(const OpndPreds &...Opnds)
bool sd_match(SDValue N, Pattern &&P)
ConstantInt_match m_ConstInt()
Match any integer constants or splat of an integer constant.
@ System
Synchronized with respect to all concurrently executing threads.
@ User
could "use" a pointer
DiagnosticInfoOptimizationBase::Argument NV
NodeAddr< UseNode * > Use
NodeAddr< NodeBase * > Node
friend class Instruction
Iterator for Instructions in a `BasicBlock.
unsigned getOpcode(const VPValue *V)
Return the instruction opcode for the recipe defining V or 0 for unsupported recipes and VPValues not...
This is an optimization pass for GlobalISel generic memory operations.
GenericUniformityInfo< SSAContext > UniformityInfo
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
LLVM_ABI void finalizeBundle(MachineBasicBlock &MBB, MachineBasicBlock::instr_iterator FirstMI, MachineBasicBlock::instr_iterator LastMI)
finalizeBundle - Finalize a machine instruction bundle which includes a sequence of instructions star...
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
detail::zippy< detail::zip_first, T, U, Args... > zip_equal(T &&t, U &&u, Args &&...args)
zip iterator that assumes that all iteratees have the same length.
constexpr bool isInt(int64_t x)
Checks if an integer fits into the given bit width.
LLVM_ABI bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
@ Known
Known to have no common set bits.
@ Implicit
Not emitted register (e.g. carry, or temporary result).
@ Undef
Value of the register doesn't matter.
@ Define
Register definition.
LLVM_ABI SDValue peekThroughBitcasts(SDValue V)
Return the non-bitcasted source operand of V if it exists.
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
bool CCAssignFn(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, Type *OrigTy, CCState &State)
CCAssignFn - This function assigns a location for Val, updating State to reflect the change.
@ Load
The value being inserted comes from a load (InsertElement only).
@ Store
The extracted value is stored (ExtractElement only).
constexpr int64_t minIntN(int64_t N)
Gets the minimum value for a N-bit signed integer.
int bit_width(T Value)
Returns the number of bits needed to represent Value if Value is nonzero.
SDValue peekFPSignOps(SDValue Val)
Strip fabs/fneg/fcopysign from a value to get the underlying source.
void append_range(Container &C, Range &&R)
Wrapper function to append range R to container C.
static const MachineMemOperand::Flags MOCFSB1
constexpr T alignDown(U Value, V Align, W Skew=0)
Returns the largest unsigned integer less than or equal to Value and is Skew mod Align.
MemoryEffectsBase< IRMemLocation > MemoryEffects
Summary of how a function affects memory in the program.
static void reportUnsupported(MachineIRBuilder &MIRBuilder, const Twine &Msg)
constexpr int popcount(T Value) noexcept
Count the number of set bits in a value.
LLVM_ABI ConstantFPSDNode * isConstOrConstSplatFP(SDValue N, bool AllowUndefs=false)
Returns the SDNode if it is a constant splat BuildVector or constant float.
LLVM_ABI bool isGuaranteedNotToBeUndef(const Value *V, AssumptionCache *AC=nullptr, const Instruction *CtxI=nullptr, const DominatorTree *DT=nullptr, unsigned Depth=0)
Returns true if V cannot be undef, but may be poison.
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Value
uint64_t PowerOf2Ceil(uint64_t A)
Returns the power of two which is greater than or equal to the given value.
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
constexpr bool isShiftedMask_64(uint64_t Value)
Return true if the argument contains a non-empty sequence of ones with the remainder zero (64 bit ver...
constexpr T MinAlign(U A, V B)
A and B are either alignments or offsets.
static const MachineMemOperand::Flags MONoClobber
Mark the MMO of a uniform load if there are no potentially clobbering stores on any path from the sta...
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
AtomicOrderingCABI
Atomic ordering for C11 / C++11's memory models.
int countl_zero(T Val)
Count number of 0's from the most significant bit to the least stopping at the first 1.
bool isBoolSGPR(SDValue V)
MachineInstr * getImm(const MachineOperand &MO, const MachineRegisterInfo *MRI)
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
constexpr uint32_t Hi_32(uint64_t Value)
Return the high 32 bits of a 64 bit value.
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
LLVM_ABI void report_fatal_error(Error Err, bool gen_crash_diag=true)
constexpr uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
constexpr bool isUInt(uint64_t x)
Checks if an unsigned integer fits into the given bit width.
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
constexpr uint32_t Lo_32(uint64_t Value)
Return the low 32 bits of a 64 bit value.
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
static const MachineMemOperand::Flags MOCooperative
Mark the MMO of cooperative load/store atomics.
LLVM_ABI raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
LLVM_ABI Value * buildAtomicRMWValue(AtomicRMWInst::BinOp Op, IRBuilderBase &Builder, Value *Loaded, Value *Val)
Emit IR to implement the given atomicrmw operation on values in registers, returning the new value.
AtomicOrdering
Atomic ordering for LLVM's memory model.
constexpr T divideCeil(U Numerator, V Denominator)
Returns the integer ceil(Numerator / Denominator).
@ First
Helpers to iterate all locations in the MemoryEffectsBase class.
@ Or
Bitwise or logical OR of integers.
@ Mul
Product of integers.
uint16_t MCPhysReg
An unsigned integer type large enough to represent all physical registers, but not necessarily virtua...
@ Fast
Assign the register banks as fast as possible (default).
DWARFExpression::Operation Op
RoundingMode
Rounding mode.
@ NearestTiesToEven
roundTiesToEven.
unsigned M0(unsigned Val)
ArrayRef(const T &OneElt) -> ArrayRef< T >
LLVM_ABI ConstantSDNode * isConstOrConstSplat(SDValue N, bool AllowUndefs=false, bool AllowTruncation=false)
Returns the SDNode if it is a constant splat BuildVector or constant int.
constexpr int64_t maxIntN(int64_t N)
Gets the maximum value for a N-bit signed integer.
constexpr unsigned BitWidth
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
LLVM_ABI std::pair< Value *, Value * > buildCmpXchgValue(IRBuilderBase &Builder, Value *Ptr, Value *Cmp, Value *Val, Align Alignment, bool IsVolatile=false)
Emit IR to implement the given cmpxchg operation on values in registers, returning the new value.
LLVM_ABI std::optional< ValueAndVReg > getIConstantVRegValWithLookThrough(Register VReg, const MachineRegisterInfo &MRI, bool LookThroughInstrs=true)
If VReg is defined by a statically evaluable chain of instructions rooted on a G_CONSTANT returns its...
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
std::optional< StringRef > getAtomicScopeIRString(const Triple &T, AtomicScope S, bool IsSingleAddressSpace=false)
Returns the LLVM IR syncscope string that T uses to spell S.
LLVM_ABI bool isOneConstant(SDValue V)
Returns true if V is a constant integer one.
static const MachineMemOperand::Flags MOLastUse
Mark the MMO of a load as the last use.
static const MachineMemOperand::Flags MOCFSB0
Use two MMO flags to encode the 2-bit cache fill size field for GFX13+.
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Next
constexpr T maskTrailingOnes(unsigned N)
Create a bitmask with the N right-most bits set to 1, and all other bits set to 0.
constexpr RegState getUndefRegState(bool B)
@ Custom
The result value requires a custom uniformity check.
LLVM_ABI Printable printReg(Register Reg, const TargetRegisterInfo *TRI=nullptr, unsigned SubIdx=0, const MachineRegisterInfo *MRI=nullptr)
Prints virtual and physical registers with or without a TRI instance.
MCRegisterClass TargetRegisterClass
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
@ CLUSTER_WORKGROUP_MAX_ID_X
@ CLUSTER_WORKGROUP_MAX_ID_Z
@ CLUSTER_WORKGROUP_MAX_FLAT_ID
@ CLUSTER_WORKGROUP_MAX_ID_Y
ArgDescriptor WorkItemIDZ
ArgDescriptor WorkItemIDY
std::tuple< const ArgDescriptor *, const TargetRegisterClass *, LLT > getPreloadedValue(PreloadedValue Value) const
static MCRegister getPrivateSegmentBufferReg(CallingConv::ID CC)
Returns the register a non-entry function with calling convention CC expects the private segment buff...
ArgDescriptor WorkItemIDX
static const AMDGPUFunctionArgInfo FixedABIFunctionInfo
The argument layout of a buffer intrinsic.
std::optional< unsigned > getSizeArg() const
The access size of loads to LDS and prefetches.
std::optional< unsigned > getImmOffsetArg() const
The immediate offset of loads to LDS.
std::optional< unsigned > getAuxArg() const
The auxiliary (cache policy) argument.
std::optional< unsigned > getLDSPtrArg() const
The LDS destination of loads to LDS.
static constexpr uint64_t encode(Fields... Values)
static std::tuple< typename Fields::ValueType... > decode(uint64_t Encoded)
unsigned AtomicNoRetBaseOpcode
This struct is a compact representation of a valid (non-zero power of two) alignment.
MCRegister getRegister() const
static ArgDescriptor createArg(const ArgDescriptor &Arg, unsigned Mask)
static ArgDescriptor createRegister(Register Reg, unsigned Mask=~0u)
Helper struct shared between Function Specialization and SCCP Solver.
Represents the full denormal controls for a function, including the default mode and the f32 specific...
Represent subnormal handling kind for floating point instruction inputs and outputs.
@ Dynamic
Denormals have unknown treatment.
static constexpr DenormalMode getPreserveSign()
static constexpr DenormalMode getIEEE()
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
EVT changeTypeToInteger() const
Return the type converted to an equivalently sized integer or vector with integer element type.
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
ElementCount getVectorElementCount() const
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
bool isByteSized() const
Return true if the bit size is a multiple of 8.
uint64_t getScalarSizeInBits() const
bool isPow2VectorType() const
Returns true if the given vector is a power of 2.
TypeSize getStoreSizeInBits() const
Return the number of bits overwritten by a store of the specified value type.
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
bool isVector() const
Return true if this is a vector value type.
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
LLVM_ABI Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
EVT changeElementType(LLVMContext &Context, EVT EltVT) const
Return a VT for a type whose attributes match ourselves with the exception of the element type that i...
bool isVectorOf(EVT EltVT) const
Return true if this is a vector with matching element type.
bool isScalarInteger() const
Return true if this is an integer, but not a vector.
LLVM_ABI const fltSemantics & getFltSemantics() const
Returns an APFloat semantics tag appropriate for the value type.
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
unsigned getPointerAddrSpace() const
unsigned getByValSize() const
Align getNonZeroMemAlign() const
OutputArg - This struct carries flags and a value for a single outgoing (actual) argument or outgoing...
static LLVM_ABI std::optional< bool > eq(const KnownBits &LHS, const KnownBits &RHS)
Determine if these known bits always give the same ICMP_EQ result.
bool isUnknown() const
Returns true if we don't know any bits.
KnownBits trunc(unsigned BitWidth) const
Return known bits for a truncation of the value we're tracking.
static KnownBits add(const KnownBits &LHS, const KnownBits &RHS, bool NSW=false, bool NUW=false, bool SelfAdd=false)
Compute knownbits resulting from addition of LHS and RHS.
KnownBits extractBits(unsigned NumBits, unsigned BitPosition) const
Return a subset of the known bits from [bitPosition,bitPosition+numBits).
unsigned countMinLeadingZeros() const
Returns the minimum number of leading zero bits.
static LLVM_ABI std::optional< bool > ule(const KnownBits &LHS, const KnownBits &RHS)
Determine if these known bits always give the same ICMP_ULE result.
static LLVM_ABI std::optional< bool > uge(const KnownBits &LHS, const KnownBits &RHS)
Determine if these known bits always give the same ICMP_UGE result.
This class contains a discriminated union of information about pointers in memory operands,...
static LLVM_ABI MachinePointerInfo getStack(MachineFunction &MF, int64_t Offset, uint8_t ID=0)
Stack pointer relative access.
MachinePointerInfo getWithOffset(int64_t O) const
static LLVM_ABI MachinePointerInfo getGOT(MachineFunction &MF)
Return a MachinePointerInfo record that refers to a GOT entry.
static LLVM_ABI MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
These are IR-level optimization flags that may be propagated to SDNodes.
bool hasNoUnsignedWrap() const
bool hasAllowContract() const
bool hasNoSignedWrap() const
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
DenormalMode FP64FP16Denormals
If this is set, neither input or output denormals are flushed for both f64 and f16/v2f16 instructions...
DenormalMode FP32Denormals
If this is set, neither input or output denormals are flushed for most f32 instructions.
This represents an addressing mode of: BaseGV + BaseOffs + BaseReg + Scale*ScaleReg + ScalableOffset*...
std::optional< unsigned > fallbackAddressSpace
This structure contains all information that is necessary for lowering calls.
SDValue ConvergenceControlToken
SmallVector< ISD::InputArg, 32 > Ins
SmallVector< ISD::OutputArg, 32 > Outs
SmallVector< SDValue, 32 > OutVals
bool isBeforeLegalize() const