39#include "llvm/IR/IntrinsicsAMDGPU.h"
40#include "llvm/IR/IntrinsicsR600.h"
50#define DEBUG_TYPE "si-lower"
56 cl::desc(
"Do not align and prefetch loops"),
60 "amdgpu-use-divergent-register-indexing",
cl::Hidden,
61 cl::desc(
"Use indirect register addressing for divergent indexes"),
75 unsigned NumSGPRs = AMDGPU::SGPR_32RegClass.getNumRegs();
76 for (
unsigned Reg = 0; Reg < NumSGPRs; ++Reg) {
78 return AMDGPU::SGPR0 + Reg;
193 {MVT::v2i32, MVT::v3i32, MVT::v4i32, MVT::v5i32,
194 MVT::v6i32, MVT::v7i32, MVT::v8i32, MVT::v9i32,
195 MVT::v10i32, MVT::v11i32, MVT::v12i32, MVT::v16i32,
196 MVT::i1, MVT::v32i32},
200 {MVT::v2i32, MVT::v3i32, MVT::v4i32, MVT::v5i32,
201 MVT::v6i32, MVT::v7i32, MVT::v8i32, MVT::v9i32,
202 MVT::v10i32, MVT::v11i32, MVT::v12i32, MVT::v16i32,
203 MVT::i1, MVT::v32i32},
272 {MVT::f32, MVT::i32, MVT::i64, MVT::f64, MVT::i1},
Expand);
279 {MVT::v2i32, MVT::v3i32, MVT::v4i32, MVT::v5i32,
280 MVT::v6i32, MVT::v7i32, MVT::v8i32, MVT::v9i32,
281 MVT::v10i32, MVT::v11i32, MVT::v12i32, MVT::v16i32},
284 {MVT::v2f32, MVT::v3f32, MVT::v4f32, MVT::v5f32,
285 MVT::v6f32, MVT::v7f32, MVT::v8f32, MVT::v9f32,
286 MVT::v10f32, MVT::v11f32, MVT::v12f32, MVT::v16f32},
290 {MVT::v2i1, MVT::v4i1, MVT::v2i8, MVT::v4i8, MVT::v2i16,
291 MVT::v3i16, MVT::v4i16, MVT::Other},
296 {MVT::i1, MVT::i32, MVT::i64, MVT::f32, MVT::f64},
Expand);
312 {MVT::v8i32, MVT::v8f32, MVT::v9i32, MVT::v9f32, MVT::v10i32,
313 MVT::v10f32, MVT::v11i32, MVT::v11f32, MVT::v12i32, MVT::v12f32,
314 MVT::v16i32, MVT::v16f32, MVT::v2i64, MVT::v2f64, MVT::v4i16,
315 MVT::v4f16, MVT::v4bf16, MVT::v3i64, MVT::v3f64, MVT::v6i32,
316 MVT::v6f32, MVT::v4i64, MVT::v4f64, MVT::v8i64, MVT::v8f64,
317 MVT::v8i16, MVT::v8f16, MVT::v8bf16, MVT::v16i16, MVT::v16f16,
318 MVT::v16bf16, MVT::v16i64, MVT::v16f64, MVT::v32i32, MVT::v32f32,
319 MVT::v32i16, MVT::v32f16, MVT::v32bf16}) {
351 for (
MVT Vec64 : {MVT::v2i64, MVT::v2f64}) {
365 for (
MVT Vec64 : {MVT::v3i64, MVT::v3f64}) {
379 for (
MVT Vec64 : {MVT::v4i64, MVT::v4f64}) {
393 for (
MVT Vec64 : {MVT::v8i64, MVT::v8f64}) {
407 for (
MVT Vec64 : {MVT::v16i64, MVT::v16f64}) {
422 {MVT::v4i32, MVT::v4f32, MVT::v8i32, MVT::v8f32,
423 MVT::v16i32, MVT::v16f32, MVT::v32i32, MVT::v32f32},
439 {MVT::v2i16, MVT::v2f16, MVT::v2bf16, MVT::v2i8, MVT::v4i8,
440 MVT::v8i8, MVT::v4i16, MVT::v4f16, MVT::v4bf16},
445 {MVT::v3i32, MVT::v3f32, MVT::v4i32, MVT::v4f32},
Custom);
449 {MVT::v5i32, MVT::v5f32, MVT::v6i32, MVT::v6f32,
450 MVT::v7i32, MVT::v7f32, MVT::v8i32, MVT::v8f32,
451 MVT::v9i32, MVT::v9f32, MVT::v10i32, MVT::v10f32,
452 MVT::v11i32, MVT::v11f32, MVT::v12i32, MVT::v12f32},
533 {MVT::f32, MVT::f64},
Legal);
626 {MVT::v2i16, MVT::v2f16, MVT::v2bf16, MVT::v4i16, MVT::v4f16,
627 MVT::v4bf16, MVT::v8i16, MVT::v8f16, MVT::v8bf16, MVT::v16i16,
628 MVT::v16f16, MVT::v16bf16, MVT::v32i16, MVT::v32f16}) {
764 {MVT::v4f16, MVT::v8f16, MVT::v16f16, MVT::v32f16},
768 {MVT::v4f16, MVT::v8f16, MVT::v16f16, MVT::v32f16},
772 {MVT::v8i16, MVT::v8f16, MVT::v8bf16, MVT::v16i16, MVT::v16f16,
773 MVT::v16bf16, MVT::v32i16, MVT::v32f16, MVT::v32bf16}) {
792 {MVT::v2i16, MVT::v2f16, MVT::v2bf16},
Custom);
795 {MVT::v4f16, MVT::v4i16, MVT::v4bf16, MVT::v8f16,
796 MVT::v8i16, MVT::v8bf16, MVT::v16f16, MVT::v16i16,
797 MVT::v16bf16, MVT::v32f16, MVT::v32i16, MVT::v32bf16},
800 for (
MVT VT : {MVT::v4i16, MVT::v8i16, MVT::v16i16, MVT::v32i16})
808 for (
MVT VT : {MVT::v4f16, MVT::v8f16, MVT::v16f16, MVT::v32f16})
824 {MVT::v4f32, MVT::v8f32, MVT::v16f32, MVT::v32f32},
844 {MVT::v4i16, MVT::v4f16, MVT::v4bf16, MVT::v2i8, MVT::v4i8,
845 MVT::v8i8, MVT::v8i16, MVT::v8f16, MVT::v8bf16,
846 MVT::v16i16, MVT::v16f16, MVT::v16bf16, MVT::v32i16,
847 MVT::v32f16, MVT::v32bf16},
863 {MVT::f16, MVT::f32, MVT::f64, MVT::v2f16},
Legal);
865 {MVT::v4f16, MVT::v8f16, MVT::v16f16, MVT::v32f16},
877 {MVT::Other, MVT::f32, MVT::v4f32, MVT::i16, MVT::f16,
878 MVT::bf16, MVT::v2i16, MVT::v2f16, MVT::v2bf16, MVT::i128,
883 {MVT::v2f16, MVT::v2i16, MVT::v2bf16, MVT::v3f16,
884 MVT::v3i16, MVT::v4f16, MVT::v4i16, MVT::v4bf16,
885 MVT::v8i16, MVT::v8f16, MVT::v8bf16, MVT::Other, MVT::f16,
886 MVT::i16, MVT::bf16, MVT::i8, MVT::i128},
890 {MVT::Other, MVT::v2i16, MVT::v2f16, MVT::v2bf16,
891 MVT::v3i16, MVT::v3f16, MVT::v4f16, MVT::v4i16,
892 MVT::v4bf16, MVT::v8i16, MVT::v8f16, MVT::v8bf16,
893 MVT::f16, MVT::i16, MVT::bf16, MVT::i8, MVT::i128},
1000 static const MCPhysReg RCRegs[] = {AMDGPU::MODE};
1013 EVT DestVT,
EVT SrcVT)
const {
1023 LLT DestTy,
LLT SrcTy)
const {
1024 return ((Opcode == TargetOpcode::G_FMAD && Subtarget->
hasMadMixInsts()) ||
1025 (Opcode == TargetOpcode::G_FMA && Subtarget->
hasFmaMixInsts())) &&
1051 return (ScalarVT == MVT::bf16 ? MVT::i32 : MVT::v2f16);
1053 return VT.
isInteger() ? MVT::i32 : MVT::f32;
1080 return (NumElts + 1) / 2;
1086 return NumElts * ((
Size + 31) / 32);
1095 unsigned &NumIntermediates,
MVT &RegisterVT)
const {
1104 if (ScalarVT == MVT::bf16) {
1105 RegisterVT = MVT::i32;
1106 IntermediateVT = MVT::v2bf16;
1108 RegisterVT = VT.
isInteger() ? MVT::v2i16 : MVT::v2f16;
1109 IntermediateVT = RegisterVT;
1111 NumIntermediates = (NumElts + 1) / 2;
1112 return NumIntermediates;
1117 IntermediateVT = RegisterVT;
1118 NumIntermediates = NumElts;
1119 return NumIntermediates;
1122 if (Size < 16 && Subtarget->has16BitInsts()) {
1124 RegisterVT = MVT::i16;
1125 IntermediateVT = ScalarVT;
1126 NumIntermediates = NumElts;
1127 return NumIntermediates;
1131 RegisterVT = MVT::i32;
1132 IntermediateVT = ScalarVT;
1133 NumIntermediates = NumElts;
1134 return NumIntermediates;
1138 RegisterVT = MVT::i32;
1139 IntermediateVT = RegisterVT;
1140 NumIntermediates = NumElts * ((
Size + 31) / 32);
1141 return NumIntermediates;
1146 Context,
CC, VT, IntermediateVT, NumIntermediates, RegisterVT);
1151 unsigned MaxNumLanes) {
1152 assert(MaxNumLanes != 0);
1155 if (
auto *VT = dyn_cast<FixedVectorType>(Ty)) {
1156 unsigned NumElts = std::min(MaxNumLanes, VT->getNumElements());
1167 unsigned MaxNumLanes) {
1168 auto *ST = dyn_cast<StructType>(Ty);
1173 assert(ST->getNumContainedTypes() == 2 &&
1174 ST->getContainedType(1)->isIntegerTy(32));
1189 DL.getPointerSizeInBits(AS) == 192)
1199 DL.getPointerSizeInBits(AS) == 160) ||
1201 DL.getPointerSizeInBits(AS) == 192))
1209 unsigned IntrID)
const {
1211 if (CI.
hasMetadata(LLVMContext::MD_invariant_load))
1229 if (RsrcIntr->IsImage) {
1237 if (
auto *RsrcPtrTy = dyn_cast<PointerType>(RsrcArg->
getType())) {
1244 Info.ptrVal = RsrcArg;
1247 bool IsSPrefetch = IntrID == Intrinsic::amdgcn_s_buffer_prefetch_data;
1256 if (RsrcIntr->IsImage) {
1257 unsigned MaxNumLanes = 4;
1272 std::numeric_limits<unsigned>::max());
1282 if (RsrcIntr->IsImage) {
1283 unsigned DMask = cast<ConstantInt>(CI.
getArgOperand(1))->getZExtValue();
1303 if ((RsrcIntr->IsImage && BaseOpcode->
NoReturn) || IsSPrefetch) {
1305 Info.memVT = MVT::i32;
1312 case Intrinsic::amdgcn_raw_buffer_load_lds:
1313 case Intrinsic::amdgcn_raw_ptr_buffer_load_lds:
1314 case Intrinsic::amdgcn_struct_buffer_load_lds:
1315 case Intrinsic::amdgcn_struct_ptr_buffer_load_lds: {
1316 unsigned Width = cast<ConstantInt>(CI.
getArgOperand(2))->getZExtValue();
1321 case Intrinsic::amdgcn_raw_atomic_buffer_load:
1322 case Intrinsic::amdgcn_raw_ptr_atomic_buffer_load:
1323 case Intrinsic::amdgcn_struct_atomic_buffer_load:
1324 case Intrinsic::amdgcn_struct_ptr_atomic_buffer_load: {
1327 std::numeric_limits<unsigned>::max());
1337 case Intrinsic::amdgcn_ds_ordered_add:
1338 case Intrinsic::amdgcn_ds_ordered_swap: {
1351 case Intrinsic::amdgcn_ds_add_gs_reg_rtn:
1352 case Intrinsic::amdgcn_ds_sub_gs_reg_rtn: {
1355 Info.ptrVal =
nullptr;
1360 case Intrinsic::amdgcn_ds_append:
1361 case Intrinsic::amdgcn_ds_consume: {
1374 case Intrinsic::amdgcn_global_atomic_csub: {
1383 case Intrinsic::amdgcn_image_bvh_intersect_ray: {
1393 case Intrinsic::amdgcn_global_atomic_fmin_num:
1394 case Intrinsic::amdgcn_global_atomic_fmax_num:
1395 case Intrinsic::amdgcn_global_atomic_ordered_add_b64:
1396 case Intrinsic::amdgcn_flat_atomic_fmin_num:
1397 case Intrinsic::amdgcn_flat_atomic_fmax_num:
1398 case Intrinsic::amdgcn_atomic_cond_sub_u32: {
1408 case Intrinsic::amdgcn_global_load_tr_b64:
1409 case Intrinsic::amdgcn_global_load_tr_b128:
1410 case Intrinsic::amdgcn_ds_read_tr4_b64:
1411 case Intrinsic::amdgcn_ds_read_tr6_b96:
1412 case Intrinsic::amdgcn_ds_read_tr8_b64:
1413 case Intrinsic::amdgcn_ds_read_tr16_b64: {
1421 case Intrinsic::amdgcn_ds_gws_init:
1422 case Intrinsic::amdgcn_ds_gws_barrier:
1423 case Intrinsic::amdgcn_ds_gws_sema_v:
1424 case Intrinsic::amdgcn_ds_gws_sema_br:
1425 case Intrinsic::amdgcn_ds_gws_sema_p:
1426 case Intrinsic::amdgcn_ds_gws_sema_release_all: {
1436 Info.memVT = MVT::i32;
1440 if (IntrID == Intrinsic::amdgcn_ds_gws_barrier)
1446 case Intrinsic::amdgcn_global_load_lds: {
1448 unsigned Width = cast<ConstantInt>(CI.
getArgOperand(2))->getZExtValue();
1454 case Intrinsic::amdgcn_ds_bvh_stack_rtn: {
1464 Info.memVT = MVT::i32;
1471 case Intrinsic::amdgcn_s_prefetch_data: {
1486 case Intrinsic::amdgcn_addrspacecast_nonnull: {
1489 unsigned SrcAS =
I.getOperand(0)->getType()->getPointerAddressSpace();
1490 unsigned DstAS =
I.getType()->getPointerAddressSpace();
1502 Type *&AccessTy)
const {
1504 switch (
II->getIntrinsicID()) {
1505 case Intrinsic::amdgcn_atomic_cond_sub_u32:
1506 case Intrinsic::amdgcn_ds_append:
1507 case Intrinsic::amdgcn_ds_consume:
1508 case Intrinsic::amdgcn_ds_read_tr4_b64:
1509 case Intrinsic::amdgcn_ds_read_tr6_b96:
1510 case Intrinsic::amdgcn_ds_read_tr8_b64:
1511 case Intrinsic::amdgcn_ds_read_tr16_b64:
1512 case Intrinsic::amdgcn_ds_ordered_add:
1513 case Intrinsic::amdgcn_ds_ordered_swap:
1514 case Intrinsic::amdgcn_flat_atomic_fmax_num:
1515 case Intrinsic::amdgcn_flat_atomic_fmin_num:
1516 case Intrinsic::amdgcn_global_atomic_csub:
1517 case Intrinsic::amdgcn_global_atomic_fmax_num:
1518 case Intrinsic::amdgcn_global_atomic_fmin_num:
1519 case Intrinsic::amdgcn_global_atomic_ordered_add_b64:
1520 case Intrinsic::amdgcn_global_load_tr_b64:
1521 case Intrinsic::amdgcn_global_load_tr_b128:
1522 Ptr =
II->getArgOperand(0);
1524 case Intrinsic::amdgcn_global_load_lds:
1525 Ptr =
II->getArgOperand(1);
1530 AccessTy =
II->getType();
1536 unsigned AddrSpace)
const {
1548 return AM.
Scale == 0 &&
1550 AM.
BaseOffs, AddrSpace, FlatVariant));
1570 return isLegalMUBUFAddressingMode(AM);
1573bool SITargetLowering::isLegalMUBUFAddressingMode(
const AddrMode &AM)
const {
1584 if (!
TII->isLegalMUBUFImmOffset(AM.BaseOffs))
1596 if (AM.HasBaseReg) {
1628 return isLegalMUBUFAddressingMode(AM);
1635 if (Ty->
isSized() &&
DL.getTypeStoreSize(Ty) < 4)
1685 : isLegalMUBUFAddressingMode(AM);
1732 unsigned Size,
unsigned AddrSpace,
Align Alignment,
1744 Align RequiredAlignment(
1747 Alignment < RequiredAlignment)
1768 RequiredAlignment =
Align(4);
1786 *IsFast = (Alignment >= RequiredAlignment) ? 64
1787 : (Alignment <
Align(4)) ? 32
1809 *IsFast = (Alignment >= RequiredAlignment) ? 96
1810 : (Alignment <
Align(4)) ? 32
1823 RequiredAlignment =
Align(8);
1834 *IsFast = (Alignment >= RequiredAlignment) ? 128
1835 : (Alignment <
Align(4)) ? 32
1852 *IsFast = (Alignment >= RequiredAlignment) ?
Size : 0;
1854 return Alignment >= RequiredAlignment ||
1863 bool AlignedBy4 = Alignment >=
Align(4);
1865 *IsFast = AlignedBy4;
1876 return Alignment >=
Align(4) ||
1890 return Size >= 32 && Alignment >=
Align(4);
1895 unsigned *IsFast)
const {
1897 Alignment, Flags, IsFast);
1907 if (
Op.size() >= 16 &&
1911 if (
Op.size() >= 8 &&
Op.isDstAligned(
Align(4)))
1919 const MemSDNode *MemNode = cast<MemSDNode>(
N);
1929 unsigned DestAS)
const {
1937 return TM.isNoopAddrSpaceCast(SrcAS, DestAS);
1955 unsigned Index)
const {
1998 auto [InputPtrReg, RC, ArgTy] =
2008 Chain, SL,
MRI.getLiveInVirtReg(InputPtrReg->getRegister()), PtrVT);
2014 const SDLoc &SL)
const {
2021 const SDLoc &SL)
const {
2024 std::optional<uint32_t> KnownSize =
2026 if (KnownSize.has_value())
2052 Val = getFPExtOrFPRound(DAG, Val, SL, VT);
2061SDValue SITargetLowering::lowerKernargMemParameter(
2073 int64_t OffsetDiff =
Offset - AlignDownOffset;
2079 SDValue Ptr = lowerKernArgParameterPtr(DAG, SL, Chain, AlignDownOffset);
2089 ArgVal = convertArgType(DAG, VT, MemVT, SL, ArgVal,
Signed, Arg);
2099 SDValue Val = convertArgType(DAG, VT, MemVT, SL, Load,
Signed, Arg);
2147 ExtType, SL, VA.
getLocVT(), Chain, FIN,
2152SDValue SITargetLowering::getPreloadedValue(
2174 Reg = &WorkGroupIDX;
2175 RC = &AMDGPU::SReg_32RegClass;
2179 Reg = &WorkGroupIDY;
2180 RC = &AMDGPU::SReg_32RegClass;
2184 Reg = &WorkGroupIDZ;
2185 RC = &AMDGPU::SReg_32RegClass;
2216 for (
unsigned I = 0, E = Ins.size(), PSInputNum = 0;
I != E; ++
I) {
2220 "vector type argument should have been split");
2225 bool SkipArg = !Arg->
Used && !
Info->isPSInputAllocated(PSInputNum);
2233 "unexpected vector split in ps argument type");
2247 Info->markPSInputAllocated(PSInputNum);
2249 Info->markPSInputEnabled(PSInputNum);
2265 if (
Info.hasWorkItemIDX()) {
2275 if (
Info.hasWorkItemIDY()) {
2278 Info.setWorkItemIDY(
2281 unsigned Reg = AMDGPU::VGPR1;
2289 if (
Info.hasWorkItemIDZ()) {
2292 Info.setWorkItemIDZ(
2295 unsigned Reg = AMDGPU::VGPR2;
2315 if (RegIdx == ArgVGPRs.
size()) {
2322 unsigned Reg = ArgVGPRs[RegIdx];
2324 assert(Reg != AMDGPU::NoRegister);
2334 unsigned NumArgRegs) {
2337 if (RegIdx == ArgSGPRs.
size())
2340 unsigned Reg = ArgSGPRs[RegIdx];
2342 assert(Reg != AMDGPU::NoRegister);
2356 assert(Reg != AMDGPU::NoRegister);
2382 const unsigned Mask = 0x3ff;
2385 if (
Info.hasWorkItemIDX()) {
2387 Info.setWorkItemIDX(Arg);
2390 if (
Info.hasWorkItemIDY()) {
2392 Info.setWorkItemIDY(Arg);
2395 if (
Info.hasWorkItemIDZ())
2407 const unsigned Mask = 0x3ff;
2428 if (
Info.hasImplicitArgPtr())
2436 if (
Info.hasWorkGroupIDX())
2439 if (
Info.hasWorkGroupIDY())
2442 if (
Info.hasWorkGroupIDZ())
2445 if (
Info.hasLDSKernelId())
2457 MF.
addLiveIn(ImplicitBufferPtrReg, &AMDGPU::SGPR_64RegClass);
2464 MF.
addLiveIn(PrivateSegmentBufferReg, &AMDGPU::SGPR_128RegClass);
2470 MF.
addLiveIn(DispatchPtrReg, &AMDGPU::SGPR_64RegClass);
2476 MF.
addLiveIn(QueuePtrReg, &AMDGPU::SGPR_64RegClass);
2491 MF.
addLiveIn(DispatchIDReg, &AMDGPU::SGPR_64RegClass);
2497 MF.
addLiveIn(FlatScratchInitReg, &AMDGPU::SGPR_64RegClass);
2503 MF.
addLiveIn(PrivateSegmentSizeReg, &AMDGPU::SGPR_32RegClass);
2520 bool InPreloadSequence =
true;
2522 bool AlignedForImplictArgs =
false;
2523 unsigned ImplicitArgOffset = 0;
2524 for (
auto &Arg :
F.args()) {
2525 if (!InPreloadSequence || !Arg.hasInRegAttr())
2528 unsigned ArgIdx = Arg.getArgNo();
2531 if (InIdx < Ins.size() &&
2532 (!Ins[InIdx].isOrigArg() || Ins[InIdx].getOrigArgIndex() != ArgIdx))
2535 for (; InIdx < Ins.size() && Ins[InIdx].isOrigArg() &&
2536 Ins[InIdx].getOrigArgIndex() == ArgIdx;
2538 assert(ArgLocs[ArgIdx].isMemLoc());
2539 auto &ArgLoc = ArgLocs[InIdx];
2541 unsigned ArgOffset = ArgLoc.getLocMemOffset();
2543 unsigned NumAllocSGPRs =
2544 alignTo(ArgLoc.getLocVT().getFixedSizeInBits(), 32) / 32;
2547 if (Arg.hasAttribute(
"amdgpu-hidden-argument")) {
2548 if (!AlignedForImplictArgs) {
2550 alignTo(LastExplicitArgOffset,
2552 LastExplicitArgOffset;
2553 AlignedForImplictArgs =
true;
2555 ArgOffset += ImplicitArgOffset;
2559 if (ArgLoc.getLocVT().getStoreSize() < 4 && Alignment < 4) {
2560 assert(InIdx >= 1 &&
"No previous SGPR");
2561 Info.getArgInfo().PreloadKernArgs[InIdx].Regs.push_back(
2562 Info.getArgInfo().PreloadKernArgs[InIdx - 1].Regs[0]);
2566 unsigned Padding = ArgOffset - LastExplicitArgOffset;
2567 unsigned PaddingSGPRs =
alignTo(Padding, 4) / 4;
2570 InPreloadSequence =
false;
2576 TRI.getSGPRClassForBitWidth(NumAllocSGPRs * 32);
2578 Info.addPreloadedKernArg(
TRI, RC, NumAllocSGPRs, InIdx, PaddingSGPRs);
2580 if (PreloadRegs->
size() > 1)
2581 RC = &AMDGPU::SGPR_32RegClass;
2582 for (
auto &Reg : *PreloadRegs) {
2588 LastExplicitArgOffset = NumAllocSGPRs * 4 + ArgOffset;
2597 if (
Info.hasLDSKernelId()) {
2599 MF.
addLiveIn(Reg, &AMDGPU::SGPR_32RegClass);
2608 bool IsShader)
const {
2616 assert(!HasArchitectedSGPRs &&
"Unhandled feature for the subtarget");
2618 unsigned CurrentUserSGPRs =
Info.getNumUserSGPRs();
2622 unsigned NumRequiredSystemSGPRs =
2623 Info.hasWorkGroupIDX() +
Info.hasWorkGroupIDY() +
2624 Info.hasWorkGroupIDZ() +
Info.hasWorkGroupInfo();
2625 for (
unsigned i = NumRequiredSystemSGPRs + CurrentUserSGPRs; i < 16; ++i) {
2627 MF.
addLiveIn(Reg, &AMDGPU::SGPR_32RegClass);
2632 if (!HasArchitectedSGPRs) {
2633 if (
Info.hasWorkGroupIDX()) {
2635 MF.
addLiveIn(Reg, &AMDGPU::SGPR_32RegClass);
2639 if (
Info.hasWorkGroupIDY()) {
2641 MF.
addLiveIn(Reg, &AMDGPU::SGPR_32RegClass);
2645 if (
Info.hasWorkGroupIDZ()) {
2647 MF.
addLiveIn(Reg, &AMDGPU::SGPR_32RegClass);
2652 if (
Info.hasWorkGroupInfo()) {
2654 MF.
addLiveIn(Reg, &AMDGPU::SGPR_32RegClass);
2658 if (
Info.hasPrivateSegmentWaveByteOffset()) {
2660 unsigned PrivateSegmentWaveByteOffsetReg;
2663 PrivateSegmentWaveByteOffsetReg =
2664 Info.getPrivateSegmentWaveByteOffsetSystemSGPR();
2668 if (PrivateSegmentWaveByteOffsetReg == AMDGPU::NoRegister) {
2670 Info.setPrivateSegmentWaveByteOffset(PrivateSegmentWaveByteOffsetReg);
2673 PrivateSegmentWaveByteOffsetReg =
Info.addPrivateSegmentWaveByteOffset();
2675 MF.
addLiveIn(PrivateSegmentWaveByteOffsetReg, &AMDGPU::SGPR_32RegClass);
2676 CCInfo.
AllocateReg(PrivateSegmentWaveByteOffsetReg);
2680 Info.getNumPreloadedSGPRs() >= 16);
2695 if (HasStackObjects)
2696 Info.setHasNonSpillStackObjects(
true);
2701 HasStackObjects =
true;
2705 bool RequiresStackAccess = HasStackObjects || MFI.
hasCalls();
2707 if (!ST.enableFlatScratch()) {
2708 if (RequiresStackAccess && ST.isAmdHsaOrMesa(MF.
getFunction())) {
2715 Info.setScratchRSrcReg(PrivateSegmentBufferReg);
2717 unsigned ReservedBufferReg =
TRI.reservedPrivateSegmentBufferReg(MF);
2727 Info.setScratchRSrcReg(ReservedBufferReg);
2746 if (!
MRI.isLiveIn(AMDGPU::SGPR32)) {
2747 Info.setStackPtrOffsetReg(AMDGPU::SGPR32);
2754 for (
unsigned Reg : AMDGPU::SGPR_32RegClass) {
2755 if (!
MRI.isLiveIn(Reg)) {
2756 Info.setStackPtrOffsetReg(Reg);
2761 if (
Info.getStackPtrOffsetReg() == AMDGPU::SP_REG)
2768 if (ST.getFrameLowering()->hasFP(MF)) {
2769 Info.setFrameOffsetReg(AMDGPU::SGPR33);
2775 return !
Info->isEntryFunction();
2785 const MCPhysReg *IStart =
TRI->getCalleeSavedRegsViaCopy(Entry->getParent());
2794 if (AMDGPU::SReg_64RegClass.
contains(*
I))
2795 RC = &AMDGPU::SGPR_64RegClass;
2796 else if (AMDGPU::SReg_32RegClass.
contains(*
I))
2797 RC = &AMDGPU::SGPR_32RegClass;
2803 Entry->addLiveIn(*
I);
2808 for (
auto *Exit : Exits)
2810 TII->get(TargetOpcode::COPY), *
I)
2828 Fn,
"unsupported non-compute shaders with HSA",
DL.getDebugLoc());
2847 !
Info->hasLDSKernelId() && !
Info->hasWorkItemIDX() &&
2848 !
Info->hasWorkItemIDY() && !
Info->hasWorkItemIDZ());
2856 !
Info->hasWorkGroupIDZ());
2875 if ((
Info->getPSInputAddr() & 0x7F) == 0 ||
2876 ((
Info->getPSInputAddr() & 0xF) == 0 &&
Info->isPSInputAllocated(11))) {
2879 Info->markPSInputAllocated(0);
2880 Info->markPSInputEnabled(0);
2891 unsigned PsInputBits =
Info->getPSInputAddr() &
Info->getPSInputEnable();
2892 if ((PsInputBits & 0x7F) == 0 ||
2893 ((PsInputBits & 0xF) == 0 && (PsInputBits >> 11 & 1)))
2896 }
else if (IsKernel) {
2899 Splits.
append(Ins.begin(), Ins.end());
2912 }
else if (!IsGraphics) {
2937 for (
unsigned i = 0, e = Ins.size(), ArgIdx = 0; i != e; ++i) {
2947 if (IsEntryFunc && VA.
isMemLoc()) {
2970 if (Arg.
isOrigArg() &&
Info->getArgInfo().PreloadKernArgs.count(i)) {
2974 int64_t OffsetDiff =
Offset - AlignDownOffset;
2981 Info->getArgInfo().PreloadKernArgs.find(i)->getSecond().Regs[0];
2992 NewArg = convertArgType(DAG, VT, MemVT,
DL, ArgVal,
2993 Ins[i].Flags.isSExt(), &Ins[i]);
3001 Info->getArgInfo().PreloadKernArgs.find(i)->getSecond().Regs;
3004 if (PreloadRegs.
size() == 1) {
3005 Register VReg =
MRI.getLiveInVirtReg(PreloadRegs[0]);
3010 TRI->getRegSizeInBits(*RC)));
3018 for (
auto Reg : PreloadRegs) {
3025 PreloadRegs.size()),
3042 NewArg = convertArgType(DAG, VT, MemVT,
DL, NewArg,
3043 Ins[i].Flags.isSExt(), &Ins[i]);
3055 "hidden argument in kernel signature was not preloaded",
3062 lowerKernargMemParameter(DAG, VT, MemVT,
DL, Chain,
Offset,
3063 Alignment, Ins[i].Flags.isSExt(), &Ins[i]);
3068 dyn_cast<PointerType>(FType->
getParamType(Ins[i].getOrigArgIndex()));
3083 if (!IsEntryFunc && VA.
isMemLoc()) {
3084 SDValue Val = lowerStackParameter(DAG, VA,
DL, Chain, Arg);
3095 if (AMDGPU::VGPR_32RegClass.
contains(Reg))
3096 RC = &AMDGPU::VGPR_32RegClass;
3097 else if (AMDGPU::SGPR_32RegClass.
contains(Reg))
3098 RC = &AMDGPU::SGPR_32RegClass;
3158 Info->setBytesInStackArgArea(StackArgSize);
3160 return Chains.
empty() ? Chain
3177 CCState CCInfo(CallConv, IsVarArg, MF, RVLocs, Context);
3183 unsigned TotalNumVGPRs = AMDGPU::VGPR_32RegClass.getNumRegs();
3184 for (
unsigned i = MaxNumVGPRs; i < TotalNumVGPRs; ++i)
3185 if (CCInfo.
isAllocated(AMDGPU::VGPR_32RegClass.getRegister(i)))
3208 bool IsWaveEnd =
Info->returnsVoid() && IsShader;
3226 for (
unsigned I = 0, RealRVLocIdx = 0, E = RVLocs.
size();
I != E;
3227 ++
I, ++RealRVLocIdx) {
3231 SDValue Arg = OutVals[RealRVLocIdx];
3259 if (!
Info->isEntryFunction()) {
3265 if (AMDGPU::SReg_64RegClass.
contains(*
I))
3267 else if (AMDGPU::SReg_32RegClass.
contains(*
I))
3283 return DAG.
getNode(Opc,
DL, MVT::Other, RetOps);
3366 auto &ArgUsageInfo =
3368 CalleeArgInfo = &ArgUsageInfo.lookupFuncArgInfo(*CalleeFunc);
3394 const auto [OutgoingArg, ArgRC, ArgTy] =
3399 const auto [IncomingArg, IncomingArgRC, Ty] =
3401 assert(IncomingArgRC == ArgRC);
3404 EVT ArgVT =
TRI->getSpillSize(*ArgRC) == 8 ? MVT::i64 : MVT::i32;
3412 InputReg = getImplicitArgPtr(DAG,
DL);
3414 std::optional<uint32_t> Id =
3416 if (Id.has_value()) {
3427 if (OutgoingArg->isRegister()) {
3428 RegsToPass.emplace_back(OutgoingArg->getRegister(), InputReg);
3429 if (!CCInfo.
AllocateReg(OutgoingArg->getRegister()))
3432 unsigned SpecialArgOffset =
3443 auto [OutgoingArg, ArgRC, Ty] =
3446 std::tie(OutgoingArg, ArgRC, Ty) =
3449 std::tie(OutgoingArg, ArgRC, Ty) =
3464 const bool NeedWorkItemIDX = !CLI.
CB->
hasFnAttr(
"amdgpu-no-workitem-id-x");
3465 const bool NeedWorkItemIDY = !CLI.
CB->
hasFnAttr(
"amdgpu-no-workitem-id-y");
3466 const bool NeedWorkItemIDZ = !CLI.
CB->
hasFnAttr(
"amdgpu-no-workitem-id-z");
3498 if (!InputReg && (NeedWorkItemIDX || NeedWorkItemIDY || NeedWorkItemIDZ)) {
3499 if (!IncomingArgX && !IncomingArgY && !IncomingArgZ) {
3510 : IncomingArgY ? *IncomingArgY
3517 if (OutgoingArg->isRegister()) {
3519 RegsToPass.emplace_back(OutgoingArg->getRegister(), InputReg);
3560 if (Callee->isDivergent())
3567 const uint32_t *CallerPreserved =
TRI->getCallPreservedMask(MF, CallerCC);
3571 if (!CallerPreserved)
3574 bool CCMatch = CallerCC == CalleeCC;
3587 if (Arg.hasByValAttr())
3601 const uint32_t *CalleePreserved =
TRI->getCallPreservedMask(MF, CalleeCC);
3602 if (!
TRI->regmaskSubsetEqual(CallerPreserved, CalleePreserved))
3611 CCState CCInfo(CalleeCC, IsVarArg, MF, ArgLocs, Ctx);
3624 for (
const auto &[CCVA, ArgVal] :
zip_equal(ArgLocs, OutVals)) {
3626 if (!CCVA.isRegLoc())
3631 if (ArgVal->
isDivergent() &&
TRI->isSGPRPhysReg(CCVA.getLocReg())) {
3633 dbgs() <<
"Cannot tail call due to divergent outgoing argument in "
3662 if (IsChainCallConv) {
3666 RequestedExec = CLI.
Args.back();
3667 assert(RequestedExec.
Node &&
"No node for EXEC");
3672 assert(CLI.
Outs.back().OrigArgIndex == 2 &&
"Unexpected last arg");
3673 CLI.
Outs.pop_back();
3677 assert(CLI.
Outs.back().OrigArgIndex == 2 &&
"Exec wasn't split up");
3678 CLI.
Outs.pop_back();
3683 "Haven't popped all the pieces of the EXEC mask");
3694 bool IsSibCall =
false;
3708 "unsupported call to variadic function ");
3716 "unsupported required tail call to function ");
3721 Outs, OutVals, Ins, DAG);
3725 "site marked musttail or on llvm.amdgcn.cs.chain");
3732 if (!TailCallOpt && IsTailCall)
3778 if (!IsSibCall || IsChainCallConv) {
3785 RegsToPass.emplace_back(IsChainCallConv
3786 ? AMDGPU::SGPR48_SGPR49_SGPR50_SGPR51
3787 : AMDGPU::SGPR0_SGPR1_SGPR2_SGPR3,
3794 const unsigned NumSpecialInputs = RegsToPass.size();
3796 MVT PtrVT = MVT::i32;
3799 for (
unsigned i = 0, e = ArgLocs.
size(); i != e; ++i) {
3827 RegsToPass.push_back(std::pair(VA.
getLocReg(), Arg));
3835 int32_t
Offset = LocMemOffset;
3842 unsigned OpSize = Flags.isByVal() ? Flags.getByValSize()
3848 ? Flags.getNonZeroByValAlign()
3875 if (Outs[i].Flags.isByVal()) {
3877 DAG.
getConstant(Outs[i].Flags.getByValSize(),
DL, MVT::i32);
3880 Outs[i].Flags.getNonZeroByValAlign(),
3882 nullptr, std::nullopt, DstInfo,
3888 DAG.
getStore(Chain,
DL, Arg, DstAddr, DstInfo, Alignment);
3894 if (!MemOpChains.
empty())
3910 unsigned ArgIdx = 0;
3911 for (
auto [Reg, Val] : RegsToPass) {
3912 if (ArgIdx++ >= NumSpecialInputs &&
3913 (IsChainCallConv || !Val->
isDivergent()) &&
TRI->isSGPRPhysReg(Reg)) {
3939 if (IsTailCall && !IsSibCall) {
3944 std::vector<SDValue> Ops({Chain});
3950 Ops.push_back(Callee);
3967 Ops.push_back(Callee);
3978 if (IsChainCallConv)
3979 Ops.push_back(RequestedExec.
Node);
3983 for (
auto &[Reg, Val] : RegsToPass)
3987 const uint32_t *Mask =
TRI->getCallPreservedMask(MF, CallConv);
3988 assert(Mask &&
"Missing call preserved mask for calling convention");
3998 MVT::Glue, GlueOps),
4003 Ops.push_back(InGlue);
4020 return DAG.
getNode(OPC,
DL, MVT::Other, Ops);
4025 Chain = Call.getValue(0);
4026 InGlue = Call.getValue(1);
4028 uint64_t CalleePopBytes = NumBytes;
4049 EVT VT =
Op.getValueType();
4059 Align Alignment = cast<ConstantSDNode>(
Op.getOperand(2))->getAlignValue();
4063 "Stack grows upwards for AMDGPU");
4065 Chain = BaseAddr.getValue(1);
4067 if (Alignment > StackAlign) {
4070 uint64_t StackAlignMask = ScaledAlignment - 1;
4077 assert(
Size.getValueType() == MVT::i32 &&
"Size must be 32-bit");
4079 if (isa<ConstantSDNode>(
Size)) {
4110 if (
Op.getValueType() != MVT::i32)
4129 assert(
Op.getValueType() == MVT::i32);
4138 Op.getOperand(0), IntrinID, GetRoundBothImm);
4172 SDValue RoundModeTimesNumBits =
4192 TableEntry, EnumOffset);
4206 if (
auto *ConstMode = dyn_cast<ConstantSDNode>(NewMode)) {
4208 static_cast<uint32_t>(ConstMode->getZExtValue()),
4220 if (UseReducedTable) {
4226 SDValue RoundModeTimesNumBits =
4246 SDValue RoundModeTimesNumBits =
4255 NewMode = TruncTable;
4264 ReadFirstLaneID, NewMode);
4277 IntrinID, RoundBothImm, NewMode);
4283 if (
Op->isDivergent())
4302 SDValue Src =
Op.getOperand(IsStrict ? 1 : 0);
4303 EVT SrcVT = Src.getValueType();
4312 EVT DstVT =
Op.getValueType();
4321 if (
Op.getValueType() != MVT::i64)
4335 Op.getOperand(0), IntrinID, ModeHwRegImm);
4337 Op.getOperand(0), IntrinID, TrapHwRegImm);
4351 if (
Op.getOperand(1).getValueType() != MVT::i64)
4363 ReadFirstLaneID, NewModeReg);
4365 ReadFirstLaneID, NewTrapReg);
4367 unsigned ModeHwReg =
4370 unsigned TrapHwReg =
4378 IntrinID, ModeHwRegImm, NewModeReg);
4381 IntrinID, TrapHwRegImm, NewTrapReg);
4388 .
Case(
"m0", AMDGPU::M0)
4389 .
Case(
"exec", AMDGPU::EXEC)
4390 .
Case(
"exec_lo", AMDGPU::EXEC_LO)
4391 .
Case(
"exec_hi", AMDGPU::EXEC_HI)
4392 .
Case(
"flat_scratch", AMDGPU::FLAT_SCR)
4393 .
Case(
"flat_scratch_lo", AMDGPU::FLAT_SCR_LO)
4394 .
Case(
"flat_scratch_hi", AMDGPU::FLAT_SCR_HI)
4397 if (Reg == AMDGPU::NoRegister) {
4405 "\" for subtarget."));
4410 case AMDGPU::EXEC_LO:
4411 case AMDGPU::EXEC_HI:
4412 case AMDGPU::FLAT_SCR_LO:
4413 case AMDGPU::FLAT_SCR_HI:
4418 case AMDGPU::FLAT_SCR:
4437 MI.setDesc(
TII->getKillTerminatorFromPseudo(
MI.getOpcode()));
4446static std::pair<MachineBasicBlock *, MachineBasicBlock *>
4468 auto Next = std::next(
I);
4481 return std::pair(LoopBB, RemainderBB);
4488 auto I =
MI.getIterator();
4489 auto E = std::next(
I);
4511 Src->setIsKill(
false);
4521 BuildMI(*LoopBB, LoopBB->begin(),
DL,
TII->get(AMDGPU::S_SETREG_IMM32_B32))
4527 Register Reg =
MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
4530 BuildMI(*LoopBB,
I,
DL,
TII->get(AMDGPU::S_GETREG_B32), Reg)
4554 unsigned InitReg,
unsigned ResultReg,
unsigned PhiReg,
4555 unsigned InitSaveExecReg,
int Offset,
bool UseGPRIdxMode,
4564 Register PhiExec =
MRI.createVirtualRegister(BoolRC);
4565 Register NewExec =
MRI.createVirtualRegister(BoolRC);
4566 Register CurrentIdxReg =
MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
4567 Register CondReg =
MRI.createVirtualRegister(BoolRC);
4575 BuildMI(LoopBB,
I,
DL,
TII->get(TargetOpcode::PHI), PhiExec)
4582 BuildMI(LoopBB,
I,
DL,
TII->get(AMDGPU::V_READFIRSTLANE_B32), CurrentIdxReg)
4586 BuildMI(LoopBB,
I,
DL,
TII->get(AMDGPU::V_CMP_EQ_U32_e64), CondReg)
4592 TII->get(ST.isWave32() ? AMDGPU::S_AND_SAVEEXEC_B32
4593 : AMDGPU::S_AND_SAVEEXEC_B64),
4597 MRI.setSimpleHint(NewExec, CondReg);
4599 if (UseGPRIdxMode) {
4601 SGPRIdxReg = CurrentIdxReg;
4603 SGPRIdxReg =
MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
4604 BuildMI(LoopBB,
I,
DL,
TII->get(AMDGPU::S_ADD_I32), SGPRIdxReg)
4611 BuildMI(LoopBB,
I,
DL,
TII->get(AMDGPU::S_MOV_B32), AMDGPU::M0)
4614 BuildMI(LoopBB,
I,
DL,
TII->get(AMDGPU::S_ADD_I32), AMDGPU::M0)
4621 unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
4624 TII->get(ST.isWave32() ? AMDGPU::S_XOR_B32_term
4625 : AMDGPU::S_XOR_B64_term),
4649 unsigned InitResultReg,
unsigned PhiReg,
int Offset,
4650 bool UseGPRIdxMode,
Register &SGPRIdxReg) {
4658 const auto *BoolXExecRC =
TRI->getWaveMaskRegClass();
4660 Register SaveExec =
MRI.createVirtualRegister(BoolXExecRC);
4661 Register TmpExec =
MRI.createVirtualRegister(BoolXExecRC);
4662 unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
4663 unsigned MovExecOpc = ST.isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64;
4678 InitResultReg, DstReg, PhiReg, TmpExec,
4679 Offset, UseGPRIdxMode, SGPRIdxReg);
4685 LoopBB->removeSuccessor(RemainderBB);
4687 LoopBB->addSuccessor(LandingPad);
4698static std::pair<unsigned, int>
4702 int NumElts =
TRI.getRegSizeInBits(*SuperRC) / 32;
4707 return std::pair(AMDGPU::sub0,
Offset);
4721 assert(
Idx->getReg() != AMDGPU::NoRegister);
4745 return Idx->getReg();
4747 Register Tmp =
MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
4764 Register SrcReg =
TII->getNamedOperand(
MI, AMDGPU::OpName::src)->getReg();
4765 int Offset =
TII->getNamedOperand(
MI, AMDGPU::OpName::offset)->getImm();
4774 const bool UseGPRIdxMode = ST.useVGPRIndexMode();
4777 if (
TII->getRegisterInfo().isSGPRClass(IdxRC)) {
4781 if (UseGPRIdxMode) {
4788 TII->getIndirectGPRIDXPseudo(
TRI.getRegSizeInBits(*VecRC),
true);
4801 MI.eraseFromParent();
4810 Register PhiReg =
MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4811 Register InitReg =
MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4817 UseGPRIdxMode, SGPRIdxReg);
4821 if (UseGPRIdxMode) {
4823 TII->getIndirectGPRIDXPseudo(
TRI.getRegSizeInBits(*VecRC),
true);
4825 BuildMI(*LoopBB, InsPt,
DL, GPRIDXDesc, Dst)
4830 BuildMI(*LoopBB, InsPt,
DL,
TII->get(AMDGPU::V_MOVRELS_B32_e32), Dst)
4835 MI.eraseFromParent();
4852 int Offset =
TII->getNamedOperand(
MI, AMDGPU::OpName::offset)->getImm();
4862 const bool UseGPRIdxMode = ST.useVGPRIndexMode();
4864 if (
Idx->getReg() == AMDGPU::NoRegister) {
4875 MI.eraseFromParent();
4880 if (
TII->getRegisterInfo().isSGPRClass(IdxRC)) {
4884 if (UseGPRIdxMode) {
4888 TII->getIndirectGPRIDXPseudo(
TRI.getRegSizeInBits(*VecRC),
false);
4897 const MCInstrDesc &MovRelDesc =
TII->getIndirectRegWriteMovRelPseudo(
4898 TRI.getRegSizeInBits(*VecRC), 32,
false);
4904 MI.eraseFromParent();
4914 Register PhiReg =
MRI.createVirtualRegister(VecRC);
4918 UseGPRIdxMode, SGPRIdxReg);
4921 if (UseGPRIdxMode) {
4923 TII->getIndirectGPRIDXPseudo(
TRI.getRegSizeInBits(*VecRC),
false);
4925 BuildMI(*LoopBB, InsPt,
DL, GPRIDXDesc, Dst)
4931 const MCInstrDesc &MovRelDesc =
TII->getIndirectRegWriteMovRelPseudo(
4932 TRI.getRegSizeInBits(*VecRC), 32,
false);
4933 BuildMI(*LoopBB, InsPt,
DL, MovRelDesc, Dst)
4939 MI.eraseFromParent();
4954 bool isSGPR =
TRI->isSGPRClass(
MRI.getRegClass(SrcReg));
4985 Register LoopIterator =
MRI.createVirtualRegister(WaveMaskRegClass);
4986 Register InitalValReg =
MRI.createVirtualRegister(DstRegClass);
4988 Register AccumulatorReg =
MRI.createVirtualRegister(DstRegClass);
4989 Register ActiveBitsReg =
MRI.createVirtualRegister(WaveMaskRegClass);
4990 Register NewActiveBitsReg =
MRI.createVirtualRegister(WaveMaskRegClass);
4992 Register FF1Reg =
MRI.createVirtualRegister(DstRegClass);
4993 Register LaneValueReg =
MRI.createVirtualRegister(DstRegClass);
4995 bool IsWave32 = ST.isWave32();
4996 unsigned MovOpc = IsWave32 ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64;
4997 unsigned ExecReg = IsWave32 ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
5002 (Opc == AMDGPU::S_MIN_U32) ? std::numeric_limits<uint32_t>::max() : 0;
5005 BuildMI(BB,
I,
DL,
TII->get(AMDGPU::S_MOV_B32), InitalValReg)
5013 I = ComputeLoop->end();
5015 BuildMI(*ComputeLoop,
I,
DL,
TII->get(AMDGPU::PHI), AccumulatorReg)
5019 BuildMI(*ComputeLoop,
I,
DL,
TII->get(AMDGPU::PHI), ActiveBitsReg)
5020 .
addReg(TmpSReg->getOperand(0).getReg())
5024 unsigned SFFOpc = IsWave32 ? AMDGPU::S_FF1_I32_B32 : AMDGPU::S_FF1_I32_B64;
5025 auto FF1 =
BuildMI(*ComputeLoop,
I,
DL,
TII->get(SFFOpc), FF1Reg)
5026 .
addReg(ActiveBits->getOperand(0).getReg());
5027 auto LaneValue =
BuildMI(*ComputeLoop,
I,
DL,
5028 TII->get(AMDGPU::V_READLANE_B32), LaneValueReg)
5030 .
addReg(FF1->getOperand(0).getReg());
5031 auto NewAccumulator =
BuildMI(*ComputeLoop,
I,
DL,
TII->get(Opc), DstReg)
5033 .
addReg(LaneValue->getOperand(0).getReg());
5036 unsigned BITSETOpc =
5037 IsWave32 ? AMDGPU::S_BITSET0_B32 : AMDGPU::S_BITSET0_B64;
5038 auto NewActiveBits =
5039 BuildMI(*ComputeLoop,
I,
DL,
TII->get(BITSETOpc), NewActiveBitsReg)
5040 .
addReg(FF1->getOperand(0).getReg())
5041 .
addReg(ActiveBits->getOperand(0).getReg());
5044 Accumulator.addReg(NewAccumulator->getOperand(0).getReg())
5045 .addMBB(ComputeLoop);
5046 ActiveBits.addReg(NewActiveBits->getOperand(0).getReg())
5047 .addMBB(ComputeLoop);
5050 unsigned CMPOpc = IsWave32 ? AMDGPU::S_CMP_LG_U32 : AMDGPU::S_CMP_LG_U64;
5052 .
addReg(NewActiveBits->getOperand(0).getReg())
5054 BuildMI(*ComputeLoop,
I,
DL,
TII->get(AMDGPU::S_CBRANCH_SCC1))
5059 MI.eraseFromParent();
5071 switch (
MI.getOpcode()) {
5072 case AMDGPU::WAVE_REDUCE_UMIN_PSEUDO_U32:
5074 case AMDGPU::WAVE_REDUCE_UMAX_PSEUDO_U32:
5076 case AMDGPU::S_UADDO_PSEUDO:
5077 case AMDGPU::S_USUBO_PSEUDO: {
5084 unsigned Opc = (
MI.getOpcode() == AMDGPU::S_UADDO_PSEUDO)
5086 : AMDGPU::S_SUB_I32;
5097 MI.eraseFromParent();
5100 case AMDGPU::S_ADD_U64_PSEUDO:
5101 case AMDGPU::S_SUB_U64_PSEUDO: {
5110 bool IsAdd = (
MI.getOpcode() == AMDGPU::S_ADD_U64_PSEUDO);
5112 unsigned Opc = IsAdd ? AMDGPU::S_ADD_U64 : AMDGPU::S_SUB_U64;
5122 Register DestSub0 =
MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
5123 Register DestSub1 =
MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
5126 MI,
MRI, Src0, BoolRC, AMDGPU::sub0, &AMDGPU::SReg_32RegClass);
5128 MI,
MRI, Src0, BoolRC, AMDGPU::sub1, &AMDGPU::SReg_32RegClass);
5131 MI,
MRI, Src1, BoolRC, AMDGPU::sub0, &AMDGPU::SReg_32RegClass);
5133 MI,
MRI, Src1, BoolRC, AMDGPU::sub1, &AMDGPU::SReg_32RegClass);
5135 unsigned LoOpc = IsAdd ? AMDGPU::S_ADD_U32 : AMDGPU::S_SUB_U32;
5136 unsigned HiOpc = IsAdd ? AMDGPU::S_ADDC_U32 : AMDGPU::S_SUBB_U32;
5149 MI.eraseFromParent();
5152 case AMDGPU::V_ADD_U64_PSEUDO:
5153 case AMDGPU::V_SUB_U64_PSEUDO: {
5159 bool IsAdd = (
MI.getOpcode() == AMDGPU::V_ADD_U64_PSEUDO);
5165 if (IsAdd && ST.hasLshlAddB64()) {
5171 TII->legalizeOperands(*
Add);
5172 MI.eraseFromParent();
5176 const auto *CarryRC =
TRI->getWaveMaskRegClass();
5178 Register DestSub0 =
MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
5179 Register DestSub1 =
MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
5181 Register CarryReg =
MRI.createVirtualRegister(CarryRC);
5182 Register DeadCarryReg =
MRI.createVirtualRegister(CarryRC);
5186 : &AMDGPU::VReg_64RegClass;
5189 : &AMDGPU::VReg_64RegClass;
5192 TRI->getSubRegisterClass(Src0RC, AMDGPU::sub0);
5194 TRI->getSubRegisterClass(Src1RC, AMDGPU::sub1);
5197 MI,
MRI, Src0, Src0RC, AMDGPU::sub0, Src0SubRC);
5199 MI,
MRI, Src1, Src1RC, AMDGPU::sub0, Src1SubRC);
5202 MI,
MRI, Src0, Src0RC, AMDGPU::sub1, Src0SubRC);
5204 MI,
MRI, Src1, Src1RC, AMDGPU::sub1, Src1SubRC);
5207 IsAdd ? AMDGPU::V_ADD_CO_U32_e64 : AMDGPU::V_SUB_CO_U32_e64;
5214 unsigned HiOpc = IsAdd ? AMDGPU::V_ADDC_U32_e64 : AMDGPU::V_SUBB_U32_e64;
5228 TII->legalizeOperands(*LoHalf);
5229 TII->legalizeOperands(*HiHalf);
5230 MI.eraseFromParent();
5233 case AMDGPU::S_ADD_CO_PSEUDO:
5234 case AMDGPU::S_SUB_CO_PSEUDO: {
5248 unsigned Opc = (
MI.getOpcode() == AMDGPU::S_ADD_CO_PSEUDO)
5249 ? AMDGPU::S_ADDC_U32
5250 : AMDGPU::S_SUBB_U32;
5252 Register RegOp0 =
MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
5253 BuildMI(*BB, MII,
DL,
TII->get(AMDGPU::V_READFIRSTLANE_B32), RegOp0)
5258 Register RegOp1 =
MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
5259 BuildMI(*BB, MII,
DL,
TII->get(AMDGPU::V_READFIRSTLANE_B32), RegOp1)
5263 Register RegOp2 =
MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
5265 BuildMI(*BB, MII,
DL,
TII->get(AMDGPU::V_READFIRSTLANE_B32), RegOp2)
5271 unsigned WaveSize =
TRI->getRegSizeInBits(*Src2RC);
5272 assert(WaveSize == 64 || WaveSize == 32);
5274 if (WaveSize == 64) {
5275 if (ST.hasScalarCompareEq64()) {
5281 TRI->getSubRegisterClass(Src2RC, AMDGPU::sub0);
5283 MII,
MRI, Src2, Src2RC, AMDGPU::sub0, SubRC);
5285 MII,
MRI, Src2, Src2RC, AMDGPU::sub1, SubRC);
5286 Register Src2_32 =
MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
5288 BuildMI(*BB, MII,
DL,
TII->get(AMDGPU::S_OR_B32), Src2_32)
5309 (WaveSize == 64) ? AMDGPU::S_CSELECT_B64 : AMDGPU::S_CSELECT_B32;
5315 MI.eraseFromParent();
5318 case AMDGPU::SI_INIT_M0: {
5320 TII->get(AMDGPU::S_MOV_B32), AMDGPU::M0)
5321 .
add(
MI.getOperand(0));
5322 MI.eraseFromParent();
5325 case AMDGPU::GET_GROUPSTATICSIZE: {
5330 .
add(
MI.getOperand(0))
5332 MI.eraseFromParent();
5335 case AMDGPU::GET_SHADERCYCLESHILO: {
5349 using namespace AMDGPU::Hwreg;
5350 Register RegHi1 =
MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
5352 .
addImm(HwregEncoding::encode(ID_SHADER_CYCLES_HI, 0, 32));
5353 Register RegLo1 =
MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
5355 .
addImm(HwregEncoding::encode(ID_SHADER_CYCLES, 0, 32));
5356 Register RegHi2 =
MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
5358 .
addImm(HwregEncoding::encode(ID_SHADER_CYCLES_HI, 0, 32));
5362 Register RegLo =
MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
5367 .
add(
MI.getOperand(0))
5372 MI.eraseFromParent();
5375 case AMDGPU::SI_INDIRECT_SRC_V1:
5376 case AMDGPU::SI_INDIRECT_SRC_V2:
5377 case AMDGPU::SI_INDIRECT_SRC_V4:
5378 case AMDGPU::SI_INDIRECT_SRC_V8:
5379 case AMDGPU::SI_INDIRECT_SRC_V9:
5380 case AMDGPU::SI_INDIRECT_SRC_V10:
5381 case AMDGPU::SI_INDIRECT_SRC_V11:
5382 case AMDGPU::SI_INDIRECT_SRC_V12:
5383 case AMDGPU::SI_INDIRECT_SRC_V16:
5384 case AMDGPU::SI_INDIRECT_SRC_V32:
5386 case AMDGPU::SI_INDIRECT_DST_V1:
5387 case AMDGPU::SI_INDIRECT_DST_V2:
5388 case AMDGPU::SI_INDIRECT_DST_V4:
5389 case AMDGPU::SI_INDIRECT_DST_V8:
5390 case AMDGPU::SI_INDIRECT_DST_V9:
5391 case AMDGPU::SI_INDIRECT_DST_V10:
5392 case AMDGPU::SI_INDIRECT_DST_V11:
5393 case AMDGPU::SI_INDIRECT_DST_V12:
5394 case AMDGPU::SI_INDIRECT_DST_V16:
5395 case AMDGPU::SI_INDIRECT_DST_V32:
5397 case AMDGPU::SI_KILL_F32_COND_IMM_PSEUDO:
5398 case AMDGPU::SI_KILL_I1_PSEUDO:
5400 case AMDGPU::V_CNDMASK_B64_PSEUDO: {
5409 Register SrcCond =
MI.getOperand(3).getReg();
5411 Register DstLo =
MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
5412 Register DstHi =
MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
5413 const auto *CondRC =
TRI->getWaveMaskRegClass();
5414 Register SrcCondCopy =
MRI.createVirtualRegister(CondRC);
5418 : &AMDGPU::VReg_64RegClass;
5421 : &AMDGPU::VReg_64RegClass;
5424 TRI->getSubRegisterClass(Src0RC, AMDGPU::sub0);
5426 TRI->getSubRegisterClass(Src1RC, AMDGPU::sub1);
5429 MI,
MRI, Src0, Src0RC, AMDGPU::sub0, Src0SubRC);
5431 MI,
MRI, Src1, Src1RC, AMDGPU::sub0, Src1SubRC);
5434 MI,
MRI, Src0, Src0RC, AMDGPU::sub1, Src0SubRC);
5436 MI,
MRI, Src1, Src1RC, AMDGPU::sub1, Src1SubRC);
5457 MI.eraseFromParent();
5460 case AMDGPU::SI_BR_UNDEF: {
5464 .
add(
MI.getOperand(0));
5466 MI.eraseFromParent();
5469 case AMDGPU::ADJCALLSTACKUP:
5470 case AMDGPU::ADJCALLSTACKDOWN: {
5477 case AMDGPU::SI_CALL_ISEL: {
5481 unsigned ReturnAddrReg =
TII->getRegisterInfo().getReturnAddressReg(*MF);
5484 MIB =
BuildMI(*BB,
MI,
DL,
TII->get(AMDGPU::SI_CALL), ReturnAddrReg);
5490 MI.eraseFromParent();
5493 case AMDGPU::V_ADD_CO_U32_e32:
5494 case AMDGPU::V_SUB_CO_U32_e32:
5495 case AMDGPU::V_SUBREV_CO_U32_e32: {
5498 unsigned Opc =
MI.getOpcode();
5500 bool NeedClampOperand =
false;
5501 if (
TII->pseudoToMCOpcode(Opc) == -1) {
5503 NeedClampOperand =
true;
5507 if (
TII->isVOP3(*
I)) {
5512 I.add(
MI.getOperand(1)).add(
MI.getOperand(2));
5513 if (NeedClampOperand)
5516 TII->legalizeOperands(*
I);
5518 MI.eraseFromParent();
5521 case AMDGPU::V_ADDC_U32_e32:
5522 case AMDGPU::V_SUBB_U32_e32:
5523 case AMDGPU::V_SUBBREV_U32_e32:
5526 TII->legalizeOperands(
MI);
5528 case AMDGPU::DS_GWS_INIT:
5529 case AMDGPU::DS_GWS_SEMA_BR:
5530 case AMDGPU::DS_GWS_BARRIER:
5531 TII->enforceOperandRCAlignment(
MI, AMDGPU::OpName::data0);
5533 case AMDGPU::DS_GWS_SEMA_V:
5534 case AMDGPU::DS_GWS_SEMA_P:
5535 case AMDGPU::DS_GWS_SEMA_RELEASE_ALL:
5543 case AMDGPU::S_SETREG_B32: {
5558 const unsigned WidthMask = maskTrailingOnes<unsigned>(Width);
5559 const unsigned SetMask = WidthMask <<
Offset;
5562 unsigned SetDenormOp = 0;
5563 unsigned SetRoundOp = 0;
5571 SetRoundOp = AMDGPU::S_ROUND_MODE;
5572 SetDenormOp = AMDGPU::S_DENORM_MODE;
5574 SetRoundOp = AMDGPU::S_ROUND_MODE;
5576 SetDenormOp = AMDGPU::S_DENORM_MODE;
5579 if (SetRoundOp || SetDenormOp) {
5582 if (Def && Def->isMoveImmediate() && Def->getOperand(1).isImm()) {
5583 unsigned ImmVal = Def->getOperand(1).getImm();
5597 MI.eraseFromParent();
5606 MI.setDesc(
TII->get(AMDGPU::S_SETREG_B32_mode));
5610 case AMDGPU::S_INVERSE_BALLOT_U32:
5611 case AMDGPU::S_INVERSE_BALLOT_U64:
5614 MI.setDesc(
TII->get(AMDGPU::COPY));
5616 case AMDGPU::ENDPGM_TRAP: {
5619 MI.setDesc(
TII->get(AMDGPU::S_ENDPGM));
5639 MI.eraseFromParent();
5642 case AMDGPU::SIMULATED_TRAP: {
5646 TII->insertSimulatedTrap(
MRI, *BB,
MI,
MI.getDebugLoc());
5647 MI.eraseFromParent();
5684 return (VT == MVT::i16) ? MVT::i16 : MVT::i32;
5800 EVT VT =
N->getValueType(0);
5804 if (VT == MVT::f16) {
5820 unsigned Opc =
Op.getOpcode();
5821 EVT VT =
Op.getValueType();
5822 assert(VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4f32 ||
5823 VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v16i16 ||
5824 VT == MVT::v16f16 || VT == MVT::v8f32 || VT == MVT::v16f32 ||
5825 VT == MVT::v32f32 || VT == MVT::v32i16 || VT == MVT::v32f16);
5840 unsigned Opc =
Op.getOpcode();
5841 EVT VT =
Op.getValueType();
5842 assert(VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4f32 ||
5843 VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v16i16 ||
5844 VT == MVT::v16f16 || VT == MVT::v8f32 || VT == MVT::v16f32 ||
5845 VT == MVT::v32f32 || VT == MVT::v32i16 || VT == MVT::v32f16);
5853 DAG.
getNode(Opc, SL, Lo0.getValueType(), Lo0, Lo1,
Op->getFlags());
5855 DAG.
getNode(Opc, SL, Hi0.getValueType(), Hi0, Hi1,
Op->getFlags());
5862 unsigned Opc =
Op.getOpcode();
5863 EVT VT =
Op.getValueType();
5864 assert(VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v8i16 ||
5865 VT == MVT::v8f16 || VT == MVT::v4f32 || VT == MVT::v16i16 ||
5866 VT == MVT::v16f16 || VT == MVT::v8f32 || VT == MVT::v16f32 ||
5867 VT == MVT::v32f32 || VT == MVT::v32f16 || VT == MVT::v32i16 ||
5868 VT == MVT::v4bf16 || VT == MVT::v8bf16 || VT == MVT::v16bf16 ||
5869 VT == MVT::v32bf16);
5874 : std::pair(Op0, Op0);
5883 DAG.
getNode(Opc, SL, ResVT.first, Lo0, Lo1, Lo2,
Op->getFlags());
5885 DAG.
getNode(Opc, SL, ResVT.second, Hi0, Hi1, Hi2,
Op->getFlags());
5891 switch (
Op.getOpcode()) {
5895 return LowerBRCOND(
Op, DAG);
5897 return LowerRETURNADDR(
Op, DAG);
5900 assert((!Result.getNode() || Result.getNode()->getNumValues() == 2) &&
5901 "Load should return a value and a chain");
5905 EVT VT =
Op.getValueType();
5907 return lowerFSQRTF32(
Op, DAG);
5909 return lowerFSQRTF64(
Op, DAG);
5914 return LowerTrig(
Op, DAG);
5916 return LowerSELECT(
Op, DAG);
5918 return LowerFDIV(
Op, DAG);
5920 return LowerFFREXP(
Op, DAG);
5922 return LowerATOMIC_CMP_SWAP(
Op, DAG);
5924 return LowerSTORE(
Op, DAG);
5928 return LowerGlobalAddress(MFI,
Op, DAG);
5931 return LowerINTRINSIC_WO_CHAIN(
Op, DAG);
5933 return LowerINTRINSIC_W_CHAIN(
Op, DAG);
5935 return LowerINTRINSIC_VOID(
Op, DAG);
5937 return lowerADDRSPACECAST(
Op, DAG);
5939 return lowerINSERT_SUBVECTOR(
Op, DAG);
5941 return lowerINSERT_VECTOR_ELT(
Op, DAG);
5943 return lowerEXTRACT_VECTOR_ELT(
Op, DAG);
5945 return lowerVECTOR_SHUFFLE(
Op, DAG);
5947 return lowerSCALAR_TO_VECTOR(
Op, DAG);
5949 return lowerBUILD_VECTOR(
Op, DAG);
5952 return lowerFP_ROUND(
Op, DAG);
5954 return lowerTRAP(
Op, DAG);
5956 return lowerDEBUGTRAP(
Op, DAG);
5965 return lowerFMINNUM_FMAXNUM(
Op, DAG);
5968 return lowerFLDEXP(
Op, DAG);
5997 return lowerMUL(
Op, DAG);
6000 return lowerXMULO(
Op, DAG);
6003 return lowerXMUL_LOHI(
Op, DAG);
6036 EVT FittingLoadVT = LoadVT;
6068SDValue SITargetLowering::adjustLoadValueType(
unsigned Opcode,
MemSDNode *M,
6071 bool IsIntrinsic)
const {
6075 EVT LoadVT =
M->getValueType(0);
6077 EVT EquivLoadVT = LoadVT;
6095 M->getMemoryVT(),
M->getMemOperand());
6106 EVT LoadVT =
M->getValueType(0);
6112 assert(
M->getNumValues() == 2 ||
M->getNumValues() == 3);
6113 bool IsTFE =
M->getNumValues() == 3;
6126 return handleByteShortBufferLoads(DAG, LoadVT,
DL, Ops,
M->getMemOperand(),
6130 return getMemIntrinsicNode(Opc,
DL,
M->getVTList(), Ops, IntVT,
6131 M->getMemOperand(), DAG);
6136 SDValue MemNode = getMemIntrinsicNode(Opc,
DL, VTList, Ops, CastVT,
6137 M->getMemOperand(), DAG);
6145 EVT VT =
N->getValueType(0);
6146 unsigned CondCode =
N->getConstantOperandVal(3);
6157 EVT CmpVT =
LHS.getValueType();
6158 if (CmpVT == MVT::i16 && !TLI.
isTypeLegal(MVT::i16)) {
6159 unsigned PromoteOp =
6179 EVT VT =
N->getValueType(0);
6181 unsigned CondCode =
N->getConstantOperandVal(3);
6190 if (CmpVT == MVT::f16 && !TLI.
isTypeLegal(CmpVT)) {
6208 EVT VT =
N->getValueType(0);
6215 Src.getOperand(1), Src.getOperand(2));
6226 Exec = AMDGPU::EXEC_LO;
6228 Exec = AMDGPU::EXEC;
6245 EVT VT =
N->getValueType(0);
6247 unsigned IID =
N->getConstantOperandVal(0);
6248 bool IsPermLane16 = IID == Intrinsic::amdgcn_permlane16 ||
6249 IID == Intrinsic::amdgcn_permlanex16;
6250 bool IsSetInactive = IID == Intrinsic::amdgcn_set_inactive ||
6251 IID == Intrinsic::amdgcn_set_inactive_chain_arg;
6255 unsigned SplitSize = 32;
6256 if (IID == Intrinsic::amdgcn_update_dpp && (ValSize % 64 == 0) &&
6257 ST->hasDPALU_DPP() &&
6265 case Intrinsic::amdgcn_permlane16:
6266 case Intrinsic::amdgcn_permlanex16:
6267 case Intrinsic::amdgcn_update_dpp:
6272 case Intrinsic::amdgcn_writelane:
6275 case Intrinsic::amdgcn_readlane:
6276 case Intrinsic::amdgcn_set_inactive:
6277 case Intrinsic::amdgcn_set_inactive_chain_arg:
6278 case Intrinsic::amdgcn_mov_dpp8:
6281 case Intrinsic::amdgcn_readfirstlane:
6282 case Intrinsic::amdgcn_permlane64:
6292 if (
SDNode *GL =
N->getGluedNode()) {
6294 GL = GL->getOperand(0).getNode();
6304 if (IID == Intrinsic::amdgcn_readlane || IID == Intrinsic::amdgcn_writelane ||
6305 IID == Intrinsic::amdgcn_mov_dpp8 ||
6306 IID == Intrinsic::amdgcn_update_dpp || IsSetInactive || IsPermLane16) {
6307 Src1 =
N->getOperand(2);
6308 if (IID == Intrinsic::amdgcn_writelane ||
6309 IID == Intrinsic::amdgcn_update_dpp || IsPermLane16)
6310 Src2 =
N->getOperand(3);
6313 if (ValSize == SplitSize) {
6323 if (IID == Intrinsic::amdgcn_update_dpp || IsSetInactive || IsPermLane16) {
6328 if (IID == Intrinsic::amdgcn_writelane) {
6333 SDValue LaneOp = createLaneOp(Src0, Src1, Src2, MVT::i32);
6335 return IsFloat ? DAG.
getBitcast(VT, Trunc) : Trunc;
6338 if (ValSize % SplitSize != 0)
6342 EVT VT =
N->getValueType(0);
6346 unsigned NumOperands =
N->getNumOperands();
6348 SDNode *GL =
N->getGluedNode();
6353 for (
unsigned i = 0; i != NE; ++i) {
6354 for (
unsigned j = 0, e = GL ? NumOperands - 1 : NumOperands; j != e;
6356 SDValue Operand =
N->getOperand(j);
6386 if (SplitSize == 32) {
6388 return unrollLaneOp(LaneOp.
getNode());
6394 unsigned SubVecNumElt =
6398 SDValue Src0SubVec, Src1SubVec, Src2SubVec;
6399 for (
unsigned i = 0, EltIdx = 0; i < ValSize / SplitSize; i++) {
6403 if (IID == Intrinsic::amdgcn_update_dpp || IsSetInactive ||
6408 if (IID == Intrinsic::amdgcn_writelane)
6413 IID == Intrinsic::amdgcn_update_dpp || IsSetInactive || IsPermLane16
6414 ? createLaneOp(Src0SubVec, Src1SubVec, Src2, SubVecVT)
6415 : createLaneOp(Src0SubVec, Src1, Src2SubVec, SubVecVT));
6416 EltIdx += SubVecNumElt;
6430 if (IID == Intrinsic::amdgcn_update_dpp || IsSetInactive || IsPermLane16)
6433 if (IID == Intrinsic::amdgcn_writelane)
6436 SDValue LaneOp = createLaneOp(Src0, Src1, Src2, VecVT);
6444 switch (
N->getOpcode()) {
6456 unsigned IID =
N->getConstantOperandVal(0);
6458 case Intrinsic::amdgcn_make_buffer_rsrc:
6459 Results.push_back(lowerPointerAsRsrcIntrin(
N, DAG));
6461 case Intrinsic::amdgcn_cvt_pkrtz: {
6470 case Intrinsic::amdgcn_cvt_pknorm_i16:
6471 case Intrinsic::amdgcn_cvt_pknorm_u16:
6472 case Intrinsic::amdgcn_cvt_pk_i16:
6473 case Intrinsic::amdgcn_cvt_pk_u16: {
6479 if (IID == Intrinsic::amdgcn_cvt_pknorm_i16)
6481 else if (IID == Intrinsic::amdgcn_cvt_pknorm_u16)
6483 else if (IID == Intrinsic::amdgcn_cvt_pk_i16)
6488 EVT VT =
N->getValueType(0);
6497 case Intrinsic::amdgcn_s_buffer_load: {
6509 EVT VT =
Op.getValueType();
6510 assert(VT == MVT::i8 &&
"Expected 8-bit s_buffer_load intrinsics.\n");
6522 if (!
Offset->isDivergent()) {
6541 LoadVal = handleByteShortBufferLoads(DAG, VT,
DL, Ops, MMO);
6553 for (
unsigned I = 0;
I < Res.getNumOperands();
I++) {
6554 Results.push_back(Res.getOperand(
I));
6558 Results.push_back(Res.getValue(1));
6567 EVT VT =
N->getValueType(0);
6572 EVT SelectVT = NewVT;
6573 if (NewVT.
bitsLT(MVT::i32)) {
6576 SelectVT = MVT::i32;
6582 if (NewVT != SelectVT)
6588 if (
N->getValueType(0) != MVT::v2f16)
6600 if (
N->getValueType(0) != MVT::v2f16)
6612 if (
N->getValueType(0) != MVT::f16)
6627 if (U.get() !=
Value)
6630 if (U.getUser()->getOpcode() == Opcode)
6636unsigned SITargetLowering::isCFIntrinsic(
const SDNode *
Intr)
const {
6638 switch (
Intr->getConstantOperandVal(1)) {
6639 case Intrinsic::amdgcn_if:
6641 case Intrinsic::amdgcn_else:
6643 case Intrinsic::amdgcn_loop:
6645 case Intrinsic::amdgcn_end_cf:
6692 SDNode *
Intr = BRCOND.getOperand(1).getNode();
6705 assert(BR &&
"brcond missing unconditional branch user");
6706 Target = BR->getOperand(1);
6709 unsigned CFNode = isCFIntrinsic(
Intr);
6728 Ops.
append(
Intr->op_begin() + (HaveChain ? 2 : 1),
Intr->op_end());
6752 for (
unsigned i = 1, e =
Intr->getNumValues() - 1; i != e; ++i) {
6765 Intr->getOperand(0));
6771 MVT VT =
Op.getSimpleValueType();
6774 if (
Op.getConstantOperandVal(0) != 0)
6780 if (
Info->isEntryFunction())
6797 return Op.getValueType().bitsLE(VT)
6804 assert(
Op.getValueType() == MVT::f16 &&
6805 "Do not know how to custom lower FP_ROUND for non-f16 type");
6808 EVT SrcVT = Src.getValueType();
6809 if (SrcVT != MVT::f64)
6825 EVT VT =
Op.getValueType();
6828 bool IsIEEEMode =
Info->getMode().IEEE;
6837 if (VT == MVT::v4f16 || VT == MVT::v8f16 || VT == MVT::v16f16 ||
6845 EVT VT =
Op.getValueType();
6849 EVT ExpVT =
Exp.getValueType();
6850 if (ExpVT == MVT::i16)
6871 {
Op.getOperand(0),
Op.getOperand(1), TruncExp});
6878 switch (
Op->getOpcode()) {
6908 DAGCombinerInfo &DCI)
const {
6909 const unsigned Opc =
Op.getOpcode();
6917 :
Op->getOperand(0).getValueType();
6920 if (DCI.isBeforeLegalizeOps() ||
6924 auto &DAG = DCI.DAG;
6930 LHS =
Op->getOperand(1);
6931 RHS =
Op->getOperand(2);
6933 LHS =
Op->getOperand(0);
6934 RHS =
Op->getOperand(1);
6965 EVT VT =
Op.getValueType();
6971 assert(VT == MVT::i64 &&
"The following code is a special for s_mul_u64");
6998 if (
Op->isDivergent())
7011 if (Op0LeadingZeros >= 32 && Op1LeadingZeros >= 32)
7013 DAG.
getMachineNode(AMDGPU::S_MUL_U64_U32_PSEUDO, SL, VT, Op0, Op1), 0);
7016 if (Op0SignBits >= 33 && Op1SignBits >= 33)
7018 DAG.
getMachineNode(AMDGPU::S_MUL_I64_I32_PSEUDO, SL, VT, Op0, Op1), 0);
7024 EVT VT =
Op.getValueType();
7031 const APInt &
C = RHSC->getAPIntValue();
7033 if (
C.isPowerOf2()) {
7035 bool UseArithShift =
isSigned && !
C.isMinSignedValue();
7062 if (
Op->isDivergent()) {
7079 return lowerTrapEndpgm(
Op, DAG);
7082 : lowerTrapHsaQueuePtr(
Op, DAG);
7092SITargetLowering::loadImplicitKernelArgument(
SelectionDAG &DAG,
MVT VT,
7094 ImplicitParameter Param)
const {
7114 loadImplicitKernelArgument(DAG, MVT::i64, SL,
Align(8),
QUEUE_PTR);
7120 if (UserSGPR == AMDGPU::NoRegister) {
7162 "debugtrap handler not supported",
7175SDValue SITargetLowering::getSegmentAperture(
unsigned AS,
const SDLoc &
DL,
7179 ? AMDGPU::SRC_SHARED_BASE
7180 : AMDGPU::SRC_PRIVATE_BASE;
7203 {SDValue(Mov, 0), DAG.getConstant(32, DL, MVT::i64)}));
7212 return loadImplicitKernelArgument(DAG, MVT::i32,
DL,
Align(4), Param);
7218 if (UserSGPR == AMDGPU::NoRegister) {
7248 if (isa<FrameIndexSDNode>(Val) || isa<GlobalAddressSDNode>(Val) ||
7249 isa<BasicBlockSDNode>(Val))
7252 if (
auto *ConstVal = dyn_cast<ConstantSDNode>(Val))
7253 return ConstVal->getSExtValue() != TM.getNullPointerValue(AddrSpace);
7267 unsigned DestAS, SrcAS;
7269 bool IsNonNull =
false;
7270 if (
const auto *ASC = dyn_cast<AddrSpaceCastSDNode>(
Op)) {
7271 SrcAS = ASC->getSrcAddressSpace();
7272 Src = ASC->getOperand(0);
7273 DestAS = ASC->getDestAddressSpace();
7276 Op.getConstantOperandVal(0) ==
7277 Intrinsic::amdgcn_addrspacecast_nonnull);
7278 Src =
Op->getOperand(1);
7279 SrcAS =
Op->getConstantOperandVal(2);
7280 DestAS =
Op->getConstantOperandVal(3);
7295 unsigned NullVal =
TM.getNullPointerValue(DestAS);
7309 SDValue Aperture = getSegmentAperture(SrcAS, SL, DAG);
7317 unsigned NullVal =
TM.getNullPointerValue(SrcAS);
7329 Op.getValueType() == MVT::i64) {
7338 Src.getValueType() == MVT::i64)
7362 EVT InsVT =
Ins.getValueType();
7365 unsigned IdxVal =
Idx->getAsZExtVal();
7370 assert(InsNumElts % 2 == 0 &&
"expect legal vector types");
7375 EVT NewInsVT = InsNumElts == 2 ? MVT::i32
7377 MVT::i32, InsNumElts / 2);
7382 for (
unsigned I = 0;
I != InsNumElts / 2; ++
I) {
7384 if (InsNumElts == 2) {
7397 for (
unsigned I = 0;
I != InsNumElts; ++
I) {
7419 auto *KIdx = dyn_cast<ConstantSDNode>(
Idx);
7420 if (NumElts == 4 && EltSize == 16 && KIdx) {
7431 unsigned Idx = KIdx->getZExtValue();
7432 bool InsertLo =
Idx < 2;
7449 if (isa<ConstantSDNode>(
Idx))
7455 assert(VecSize <= 64 &&
"Expected target vector size to be <= 64 bits");
7461 const auto EltMask = maskTrailingOnes<uint64_t>(EltSize);
7490 EVT ResultVT =
Op.getValueType();
7503 if (
SDValue Combined = performExtractVectorEltCombine(
Op.getNode(), DCI))
7506 if (VecSize == 128 || VecSize == 256 || VecSize == 512) {
7510 if (VecSize == 128) {
7518 }
else if (VecSize == 256) {
7521 for (
unsigned P = 0;
P < 4; ++
P) {
7527 Parts[0], Parts[1]));
7529 Parts[2], Parts[3]));
7535 for (
unsigned P = 0;
P < 8; ++
P) {
7542 Parts[0], Parts[1], Parts[2], Parts[3]));
7545 Parts[4], Parts[5], Parts[6], Parts[7]));
7548 EVT IdxVT =
Idx.getValueType();
7565 Src = DAG.
getBitcast(Src.getValueType().changeTypeToInteger(), Src);
7580 if (ResultVT == MVT::f16 || ResultVT == MVT::bf16) {
7590 return Mask[Elt + 1] == Mask[Elt] + 1 && (Mask[Elt] % 2 == 0);
7595 return Mask[Elt] >= 0 && Mask[Elt + 1] >= 0 && (Mask[Elt] & 1) &&
7596 !(Mask[Elt + 1] & 1);
7602 EVT ResultVT =
Op.getValueType();
7605 const int NewSrcNumElts = 2;
7607 int SrcNumElts =
Op.getOperand(0).getValueType().getVectorNumElements();
7623 const bool ShouldUseConsecutiveExtract = EltVT.
getSizeInBits() == 16;
7645 if (ShouldUseConsecutiveExtract &&
7648 int VecIdx =
Idx < SrcNumElts ? 0 : 1;
7649 int EltIdx =
Idx < SrcNumElts ?
Idx :
Idx - SrcNumElts;
7661 if (Idx0 >= SrcNumElts) {
7666 if (Idx1 >= SrcNumElts) {
7671 int AlignedIdx0 = Idx0 & ~(NewSrcNumElts - 1);
7672 int AlignedIdx1 = Idx1 & ~(NewSrcNumElts - 1);
7680 int NewMaskIdx0 = Idx0 - AlignedIdx0;
7681 int NewMaskIdx1 = Idx1 - AlignedIdx1;
7686 if (SubVec0 != SubVec1) {
7687 NewMaskIdx1 += NewSrcNumElts;
7694 {NewMaskIdx0, NewMaskIdx1});
7699 int VecIdx0 = Idx0 < SrcNumElts ? 0 : 1;
7700 int VecIdx1 = Idx1 < SrcNumElts ? 0 : 1;
7701 int EltIdx0 = Idx0 < SrcNumElts ? Idx0 : Idx0 - SrcNumElts;
7702 int EltIdx1 = Idx1 < SrcNumElts ? Idx1 : Idx1 - SrcNumElts;
7721 EVT ResultVT =
Op.getValueType();
7737 EVT VT =
Op.getValueType();
7739 if (VT == MVT::v2f16 || VT == MVT::v2i16 || VT == MVT::v2bf16) {
7774 for (
unsigned P = 0;
P < NumParts; ++
P) {
7776 PartVT, SL, {
Op.getOperand(
P * 2),
Op.getOperand(
P * 2 + 1)});
7809 assert(isInt<32>(
Offset + 4) &&
"32-bit offset is expected!");
7847 EVT PtrVT =
Op.getValueType();
7863 assert(PtrVT == MVT::i32 &&
"32-bit pointer is expected.");
7934 SDValue Param = lowerKernargMemParameter(
7944 "non-hsa intrinsic with hsa target",
7953 "intrinsic not supported on subtarget",
7963 unsigned NumElts = Elts.
size();
7965 if (NumElts <= 12) {
7974 for (
unsigned i = 0; i < Elts.
size(); ++i) {
7980 for (
unsigned i = Elts.
size(); i < NumElts; ++i)
7981 VecElts[i] = DAG.
getUNDEF(MVT::f32);
7990 EVT SrcVT = Src.getValueType();
8011 bool Unpacked,
bool IsD16,
int DMaskPop,
8012 int NumVDataDwords,
bool IsAtomicPacked16Bit,
8016 EVT ReqRetVT = ResultTypes[0];
8018 int NumDataDwords = ((IsD16 && !Unpacked) || IsAtomicPacked16Bit)
8019 ? (ReqRetNumElts + 1) / 2
8022 int MaskPopDwords = (!IsD16 || Unpacked) ? DMaskPop : (DMaskPop + 1) / 2;
8033 if (DMaskPop > 0 &&
Data.getValueType() != MaskPopVT) {
8044 if (DataDwordVT.
isVector() && !IsAtomicPacked16Bit)
8046 NumDataDwords - MaskPopDwords);
8051 EVT LegalReqRetVT = ReqRetVT;
8053 if (!
Data.getValueType().isInteger())
8055 Data.getValueType().changeTypeToInteger(),
Data);
8076 if (Result->getNumValues() == 1)
8083 SDValue *LWE,
bool &IsTexFail) {
8084 auto *TexFailCtrlConst = cast<ConstantSDNode>(TexFailCtrl.
getNode());
8103 unsigned DimIdx,
unsigned EndIdx,
8104 unsigned NumGradients) {
8106 for (
unsigned I = DimIdx;
I < EndIdx;
I++) {
8114 if (((
I + 1) >= EndIdx) ||
8115 ((NumGradients / 2) % 2 == 1 && (
I == DimIdx + (NumGradients / 2) - 1 ||
8116 I == DimIdx + NumGradients - 1))) {
8117 if (
Addr.getValueType() != MVT::i16)
8138 unsigned IntrOpcode =
Intr->BaseOpcode;
8149 int NumVDataDwords = 0;
8150 bool AdjustRetType =
false;
8151 bool IsAtomicPacked16Bit =
false;
8154 const unsigned ArgOffset = WithChain ? 2 : 1;
8157 unsigned DMaskLanes = 0;
8159 if (BaseOpcode->Atomic) {
8160 VData =
Op.getOperand(2);
8162 IsAtomicPacked16Bit =
8163 (
Intr->BaseOpcode == AMDGPU::IMAGE_ATOMIC_PK_ADD_F16 ||
8164 Intr->BaseOpcode == AMDGPU::IMAGE_ATOMIC_PK_ADD_BF16);
8167 if (BaseOpcode->AtomicX2) {
8174 ResultTypes[0] = Is64Bit ? MVT::v2i64 : MVT::v2i32;
8175 DMask = Is64Bit ? 0xf : 0x3;
8176 NumVDataDwords = Is64Bit ? 4 : 2;
8178 DMask = Is64Bit ? 0x3 : 0x1;
8179 NumVDataDwords = Is64Bit ? 2 : 1;
8182 DMask =
Op->getConstantOperandVal(ArgOffset +
Intr->DMaskIndex);
8185 if (BaseOpcode->Store) {
8186 VData =
Op.getOperand(2);
8194 VData = handleD16VData(VData, DAG,
true);
8198 }
else if (!BaseOpcode->NoReturn) {
8211 (!LoadVT.
isVector() && DMaskLanes > 1))
8219 NumVDataDwords = (DMaskLanes + 1) / 2;
8221 NumVDataDwords = DMaskLanes;
8223 AdjustRetType =
true;
8227 unsigned VAddrEnd = ArgOffset +
Intr->VAddrEnd;
8232 Op.getOperand(ArgOffset +
Intr->GradientStart).getSimpleValueType();
8234 MVT GradPackVectorVT = VAddrScalarVT == MVT::f16 ? MVT::v2f16 : MVT::v2i16;
8235 IsG16 = VAddrScalarVT == MVT::f16 || VAddrScalarVT == MVT::i16;
8237 VAddrVT =
Op.getOperand(ArgOffset +
Intr->CoordStart).getSimpleValueType();
8239 MVT AddrPackVectorVT = VAddrScalarVT == MVT::f16 ? MVT::v2f16 : MVT::v2i16;
8240 IsA16 = VAddrScalarVT == MVT::f16 || VAddrScalarVT == MVT::i16;
8243 for (
unsigned I =
Intr->VAddrStart; I < Intr->GradientStart;
I++) {
8244 if (IsA16 && (
Op.getOperand(ArgOffset +
I).getValueType() == MVT::f16)) {
8245 assert(
I ==
Intr->BiasIndex &&
"Got unexpected 16-bit extra argument");
8250 {
Op.getOperand(ArgOffset +
I), DAG.
getUNDEF(MVT::f16)});
8254 "Bias needs to be converted to 16 bit in A16 mode");
8259 if (BaseOpcode->Gradients && !
ST->hasG16() && (IsA16 != IsG16)) {
8263 dbgs() <<
"Failed to lower image intrinsic: 16 bit addresses "
8264 "require 16 bit args for both gradients and addresses");
8269 if (!
ST->hasA16()) {
8270 LLVM_DEBUG(
dbgs() <<
"Failed to lower image intrinsic: Target does not "
8271 "support 16 bit addresses\n");
8281 if (BaseOpcode->Gradients && IsG16 &&
ST->hasG16()) {
8285 IntrOpcode = G16MappingInfo->
G16;
8293 ArgOffset +
Intr->GradientStart,
8294 ArgOffset +
Intr->CoordStart,
Intr->NumGradients);
8296 for (
unsigned I = ArgOffset +
Intr->GradientStart;
8297 I < ArgOffset + Intr->CoordStart;
I++)
8304 ArgOffset +
Intr->CoordStart, VAddrEnd,
8308 for (
unsigned I = ArgOffset +
Intr->CoordStart;
I < VAddrEnd;
I++)
8326 const unsigned NSAMaxSize =
ST->getNSAMaxSize(BaseOpcode->Sampler);
8327 const bool HasPartialNSAEncoding =
ST->hasPartialNSAEncoding();
8328 const bool UseNSA =
ST->hasNSAEncoding() &&
8329 VAddrs.
size() >=
ST->getNSAThreshold(MF) &&
8330 (VAddrs.
size() <= NSAMaxSize || HasPartialNSAEncoding);
8331 const bool UsePartialNSA =
8332 UseNSA && HasPartialNSAEncoding && VAddrs.
size() > NSAMaxSize;
8335 if (UsePartialNSA) {
8337 ArrayRef(VAddrs).drop_front(NSAMaxSize - 1));
8338 }
else if (!UseNSA) {
8345 if (!BaseOpcode->Sampler) {
8349 Op.getConstantOperandVal(ArgOffset +
Intr->UnormIndex);
8351 Unorm = UnormConst ? True : False;
8356 SDValue TexFail =
Op.getOperand(ArgOffset +
Intr->TexFailCtrlIndex);
8357 bool IsTexFail =
false;
8358 if (!
parseTexFail(TexFail, DAG, &TFE, &LWE, IsTexFail))
8369 NumVDataDwords += 1;
8370 AdjustRetType =
true;
8375 if (AdjustRetType) {
8378 if (DMaskLanes == 0 && !BaseOpcode->Store) {
8381 if (isa<MemSDNode>(
Op))
8387 MVT::i32, NumVDataDwords)
8390 ResultTypes[0] = NewVT;
8391 if (ResultTypes.size() == 3) {
8395 ResultTypes.erase(&ResultTypes[1]);
8399 unsigned CPol =
Op.getConstantOperandVal(ArgOffset +
Intr->CachePolicyIndex);
8400 if (BaseOpcode->Atomic)
8407 if (BaseOpcode->Store || BaseOpcode->Atomic)
8409 if (UsePartialNSA) {
8418 if (RsrcVT != MVT::v4i32 && RsrcVT != MVT::v8i32)
8421 if (BaseOpcode->Sampler) {
8430 if (!IsGFX12Plus || BaseOpcode->Sampler || BaseOpcode->MSAA)
8434 ST->hasFeature(AMDGPU::FeatureR128A16)
8444 if (!IsGFX12Plus || BaseOpcode->Sampler || BaseOpcode->MSAA)
8448 if (BaseOpcode->HasD16)
8450 if (isa<MemSDNode>(
Op))
8453 int NumVAddrDwords =
8459 NumVDataDwords, NumVAddrDwords);
8460 }
else if (IsGFX11Plus) {
8462 UseNSA ? AMDGPU::MIMGEncGfx11NSA
8463 : AMDGPU::MIMGEncGfx11Default,
8464 NumVDataDwords, NumVAddrDwords);
8465 }
else if (IsGFX10Plus) {
8467 UseNSA ? AMDGPU::MIMGEncGfx10NSA
8468 : AMDGPU::MIMGEncGfx10Default,
8469 NumVDataDwords, NumVAddrDwords);
8473 NumVDataDwords, NumVAddrDwords);
8476 "requested image instruction is not supported on this GPU");
8481 NumVDataDwords, NumVAddrDwords);
8484 NumVDataDwords, NumVAddrDwords);
8490 if (
auto *
MemOp = dyn_cast<MemSDNode>(
Op)) {
8495 if (BaseOpcode->AtomicX2) {
8500 if (BaseOpcode->NoReturn)
8504 NumVDataDwords, IsAtomicPacked16Bit,
DL);
8522 if (!
Offset->isDivergent()) {
8567 return handleByteShortBufferLoads(DAG, VT,
DL, Ops, MMO);
8571 unsigned NumLoads = 1;
8577 if (NumElts == 8 || NumElts == 16) {
8578 NumLoads = NumElts / 4;
8586 setBufferOffsets(
Offset, DAG, &Ops[3],
8587 NumLoads > 1 ?
Align(16 * NumLoads) :
Align(4));
8590 for (
unsigned i = 0; i < NumLoads; ++i) {
8596 if (NumElts == 8 || NumElts == 16)
8643 EVT VT =
Op.getValueType();
8645 unsigned IntrinsicID =
Op.getConstantOperandVal(0);
8649 switch (IntrinsicID) {
8650 case Intrinsic::amdgcn_implicit_buffer_ptr: {
8653 return getPreloadedValue(DAG, *MFI, VT,
8656 case Intrinsic::amdgcn_dispatch_ptr:
8657 case Intrinsic::amdgcn_queue_ptr: {
8660 MF.
getFunction(),
"unsupported hsa intrinsic without hsa target",
8666 auto RegID = IntrinsicID == Intrinsic::amdgcn_dispatch_ptr
8669 return getPreloadedValue(DAG, *MFI, VT, RegID);
8671 case Intrinsic::amdgcn_implicitarg_ptr: {
8673 return getImplicitArgPtr(DAG,
DL);
8674 return getPreloadedValue(DAG, *MFI, VT,
8677 case Intrinsic::amdgcn_kernarg_segment_ptr: {
8683 return getPreloadedValue(DAG, *MFI, VT,
8686 case Intrinsic::amdgcn_dispatch_id: {
8689 case Intrinsic::amdgcn_rcp:
8691 case Intrinsic::amdgcn_rsq:
8693 case Intrinsic::amdgcn_rsq_legacy:
8697 case Intrinsic::amdgcn_rcp_legacy:
8701 case Intrinsic::amdgcn_rsq_clamp: {
8715 case Intrinsic::r600_read_ngroups_x:
8719 return lowerKernargMemParameter(DAG, VT, VT,
DL, DAG.
getEntryNode(),
8722 case Intrinsic::r600_read_ngroups_y:
8726 return lowerKernargMemParameter(DAG, VT, VT,
DL, DAG.
getEntryNode(),
8729 case Intrinsic::r600_read_ngroups_z:
8733 return lowerKernargMemParameter(DAG, VT, VT,
DL, DAG.
getEntryNode(),
8736 case Intrinsic::r600_read_global_size_x:
8740 return lowerKernargMemParameter(DAG, VT, VT,
DL, DAG.
getEntryNode(),
8743 case Intrinsic::r600_read_global_size_y:
8747 return lowerKernargMemParameter(DAG, VT, VT,
DL, DAG.
getEntryNode(),
8750 case Intrinsic::r600_read_global_size_z:
8754 return lowerKernargMemParameter(DAG, VT, VT,
DL, DAG.
getEntryNode(),
8757 case Intrinsic::r600_read_local_size_x:
8761 return lowerImplicitZextParam(DAG,
Op, MVT::i16,
8763 case Intrinsic::r600_read_local_size_y:
8767 return lowerImplicitZextParam(DAG,
Op, MVT::i16,
8769 case Intrinsic::r600_read_local_size_z:
8773 return lowerImplicitZextParam(DAG,
Op, MVT::i16,
8775 case Intrinsic::amdgcn_workgroup_id_x:
8776 return getPreloadedValue(DAG, *MFI, VT,
8778 case Intrinsic::amdgcn_workgroup_id_y:
8779 return getPreloadedValue(DAG, *MFI, VT,
8781 case Intrinsic::amdgcn_workgroup_id_z:
8782 return getPreloadedValue(DAG, *MFI, VT,
8784 case Intrinsic::amdgcn_wave_id:
8785 return lowerWaveID(DAG,
Op);
8786 case Intrinsic::amdgcn_lds_kernel_id: {
8788 return getLDSKernelId(DAG,
DL);
8789 return getPreloadedValue(DAG, *MFI, VT,
8792 case Intrinsic::amdgcn_workitem_id_x:
8793 return lowerWorkitemID(DAG,
Op, 0, MFI->getArgInfo().WorkItemIDX);
8794 case Intrinsic::amdgcn_workitem_id_y:
8795 return lowerWorkitemID(DAG,
Op, 1, MFI->getArgInfo().WorkItemIDY);
8796 case Intrinsic::amdgcn_workitem_id_z:
8797 return lowerWorkitemID(DAG,
Op, 2, MFI->getArgInfo().WorkItemIDZ);
8798 case Intrinsic::amdgcn_wavefrontsize:
8801 case Intrinsic::amdgcn_s_buffer_load: {
8802 unsigned CPol =
Op.getConstantOperandVal(3);
8809 return lowerSBuffer(VT,
DL,
Op.getOperand(1),
Op.getOperand(2),
8810 Op.getOperand(3), DAG);
8812 case Intrinsic::amdgcn_fdiv_fast:
8813 return lowerFDIV_FAST(
Op, DAG);
8814 case Intrinsic::amdgcn_sin:
8817 case Intrinsic::amdgcn_cos:
8820 case Intrinsic::amdgcn_mul_u24:
8823 case Intrinsic::amdgcn_mul_i24:
8827 case Intrinsic::amdgcn_log_clamp: {
8833 case Intrinsic::amdgcn_fract:
8836 case Intrinsic::amdgcn_class:
8839 case Intrinsic::amdgcn_div_fmas:
8841 Op.getOperand(2),
Op.getOperand(3),
Op.getOperand(4));
8843 case Intrinsic::amdgcn_div_fixup:
8845 Op.getOperand(2),
Op.getOperand(3));
8847 case Intrinsic::amdgcn_div_scale: {
8860 SDValue Src0 =
Param->isAllOnes() ? Numerator : Denominator;
8863 Denominator, Numerator);
8865 case Intrinsic::amdgcn_icmp: {
8867 if (
Op.getOperand(1).getValueType() == MVT::i1 &&
8868 Op.getConstantOperandVal(2) == 0 &&
8873 case Intrinsic::amdgcn_fcmp: {
8876 case Intrinsic::amdgcn_ballot:
8878 case Intrinsic::amdgcn_fmed3:
8880 Op.getOperand(2),
Op.getOperand(3));
8881 case Intrinsic::amdgcn_fdot2:
8883 Op.getOperand(2),
Op.getOperand(3),
Op.getOperand(4));
8884 case Intrinsic::amdgcn_fmul_legacy:
8887 case Intrinsic::amdgcn_sffbh:
8889 case Intrinsic::amdgcn_sbfe:
8891 Op.getOperand(2),
Op.getOperand(3));
8892 case Intrinsic::amdgcn_ubfe:
8894 Op.getOperand(2),
Op.getOperand(3));
8895 case Intrinsic::amdgcn_cvt_pkrtz:
8896 case Intrinsic::amdgcn_cvt_pknorm_i16:
8897 case Intrinsic::amdgcn_cvt_pknorm_u16:
8898 case Intrinsic::amdgcn_cvt_pk_i16:
8899 case Intrinsic::amdgcn_cvt_pk_u16: {
8901 EVT VT =
Op.getValueType();
8904 if (IntrinsicID == Intrinsic::amdgcn_cvt_pkrtz)
8906 else if (IntrinsicID == Intrinsic::amdgcn_cvt_pknorm_i16)
8908 else if (IntrinsicID == Intrinsic::amdgcn_cvt_pknorm_u16)
8910 else if (IntrinsicID == Intrinsic::amdgcn_cvt_pk_i16)
8916 return DAG.
getNode(Opcode,
DL, VT,
Op.getOperand(1),
Op.getOperand(2));
8919 DAG.
getNode(Opcode,
DL, MVT::i32,
Op.getOperand(1),
Op.getOperand(2));
8922 case Intrinsic::amdgcn_fmad_ftz:
8924 Op.getOperand(2),
Op.getOperand(3));
8926 case Intrinsic::amdgcn_if_break:
8928 Op->getOperand(1),
Op->getOperand(2)),
8931 case Intrinsic::amdgcn_groupstaticsize: {
8943 case Intrinsic::amdgcn_is_shared:
8944 case Intrinsic::amdgcn_is_private: {
8946 unsigned AS = (IntrinsicID == Intrinsic::amdgcn_is_shared)
8949 SDValue Aperture = getSegmentAperture(AS, SL, DAG);
8957 case Intrinsic::amdgcn_perm:
8959 Op.getOperand(2),
Op.getOperand(3));
8960 case Intrinsic::amdgcn_reloc_constant: {
8964 auto *RelocSymbol = cast<GlobalVariable>(
8970 case Intrinsic::amdgcn_swmmac_f16_16x16x32_f16:
8971 case Intrinsic::amdgcn_swmmac_bf16_16x16x32_bf16:
8972 case Intrinsic::amdgcn_swmmac_f32_16x16x32_bf16:
8973 case Intrinsic::amdgcn_swmmac_f32_16x16x32_f16:
8974 case Intrinsic::amdgcn_swmmac_f32_16x16x32_fp8_fp8:
8975 case Intrinsic::amdgcn_swmmac_f32_16x16x32_fp8_bf8:
8976 case Intrinsic::amdgcn_swmmac_f32_16x16x32_bf8_fp8:
8977 case Intrinsic::amdgcn_swmmac_f32_16x16x32_bf8_bf8: {
8978 if (
Op.getOperand(4).getValueType() == MVT::i32)
8984 Op.getOperand(0),
Op.getOperand(1),
Op.getOperand(2),
8985 Op.getOperand(3), IndexKeyi32);
8987 case Intrinsic::amdgcn_swmmac_i32_16x16x32_iu4:
8988 case Intrinsic::amdgcn_swmmac_i32_16x16x32_iu8:
8989 case Intrinsic::amdgcn_swmmac_i32_16x16x64_iu4: {
8990 if (
Op.getOperand(6).getValueType() == MVT::i32)
8996 {Op.getOperand(0), Op.getOperand(1), Op.getOperand(2),
8997 Op.getOperand(3), Op.getOperand(4), Op.getOperand(5),
8998 IndexKeyi32, Op.getOperand(7)});
9000 case Intrinsic::amdgcn_addrspacecast_nonnull:
9001 return lowerADDRSPACECAST(
Op, DAG);
9002 case Intrinsic::amdgcn_readlane:
9003 case Intrinsic::amdgcn_readfirstlane:
9004 case Intrinsic::amdgcn_writelane:
9005 case Intrinsic::amdgcn_permlane16:
9006 case Intrinsic::amdgcn_permlanex16:
9007 case Intrinsic::amdgcn_permlane64:
9008 case Intrinsic::amdgcn_set_inactive:
9009 case Intrinsic::amdgcn_set_inactive_chain_arg:
9010 case Intrinsic::amdgcn_mov_dpp8:
9011 case Intrinsic::amdgcn_update_dpp:
9016 return lowerImage(
Op, ImageDimIntr, DAG,
false);
9027 return DAG.
getRegister(AMDGPU::SGPR_NULL, MVT::i32);
9033 unsigned NewOpcode)
const {
9037 SDValue Rsrc = bufferRsrcPtrToVector(
Op.getOperand(3), DAG);
9038 auto [VOffset,
Offset] = splitBufferOffsets(
Op.getOperand(4), DAG);
9052 auto *
M = cast<MemSDNode>(
Op);
9056 M->getMemOperand());
9061 unsigned NewOpcode)
const {
9065 SDValue Rsrc = bufferRsrcPtrToVector(
Op.getOperand(3), DAG);
9066 auto [VOffset,
Offset] = splitBufferOffsets(
Op.getOperand(5), DAG);
9080 auto *
M = cast<MemSDNode>(
Op);
9084 M->getMemOperand());
9089 unsigned IntrID =
Op.getConstantOperandVal(1);
9093 case Intrinsic::amdgcn_ds_ordered_add:
9094 case Intrinsic::amdgcn_ds_ordered_swap: {
9099 unsigned IndexOperand =
M->getConstantOperandVal(7);
9100 unsigned WaveRelease =
M->getConstantOperandVal(8);
9101 unsigned WaveDone =
M->getConstantOperandVal(9);
9103 unsigned OrderedCountIndex = IndexOperand & 0x3f;
9104 IndexOperand &= ~0x3f;
9105 unsigned CountDw = 0;
9108 CountDw = (IndexOperand >> 24) & 0xf;
9109 IndexOperand &= ~(0xf << 24);
9111 if (CountDw < 1 || CountDw > 4) {
9113 "ds_ordered_count: dword count must be between 1 and 4");
9120 if (WaveDone && !WaveRelease)
9123 unsigned Instruction = IntrID == Intrinsic::amdgcn_ds_ordered_add ? 0 : 1;
9124 unsigned ShaderType =
9126 unsigned Offset0 = OrderedCountIndex << 2;
9127 unsigned Offset1 = WaveRelease | (WaveDone << 1) | (
Instruction << 4);
9130 Offset1 |= (CountDw - 1) << 6;
9133 Offset1 |= ShaderType << 2;
9135 unsigned Offset = Offset0 | (Offset1 << 8);
9142 M->getVTList(), Ops,
M->getMemoryVT(),
9143 M->getMemOperand());
9145 case Intrinsic::amdgcn_raw_buffer_load:
9146 case Intrinsic::amdgcn_raw_ptr_buffer_load:
9147 case Intrinsic::amdgcn_raw_atomic_buffer_load:
9148 case Intrinsic::amdgcn_raw_ptr_atomic_buffer_load:
9149 case Intrinsic::amdgcn_raw_buffer_load_format:
9150 case Intrinsic::amdgcn_raw_ptr_buffer_load_format: {
9151 const bool IsFormat =
9152 IntrID == Intrinsic::amdgcn_raw_buffer_load_format ||
9153 IntrID == Intrinsic::amdgcn_raw_ptr_buffer_load_format;
9155 SDValue Rsrc = bufferRsrcPtrToVector(
Op.getOperand(2), DAG);
9156 auto [VOffset,
Offset] = splitBufferOffsets(
Op.getOperand(3), DAG);
9169 auto *
M = cast<MemSDNode>(
Op);
9170 return lowerIntrinsicLoad(M, IsFormat, DAG, Ops);
9172 case Intrinsic::amdgcn_struct_buffer_load:
9173 case Intrinsic::amdgcn_struct_ptr_buffer_load:
9174 case Intrinsic::amdgcn_struct_buffer_load_format:
9175 case Intrinsic::amdgcn_struct_ptr_buffer_load_format:
9176 case Intrinsic::amdgcn_struct_atomic_buffer_load:
9177 case Intrinsic::amdgcn_struct_ptr_atomic_buffer_load: {
9178 const bool IsFormat =
9179 IntrID == Intrinsic::amdgcn_struct_buffer_load_format ||
9180 IntrID == Intrinsic::amdgcn_struct_ptr_buffer_load_format;
9182 SDValue Rsrc = bufferRsrcPtrToVector(
Op.getOperand(2), DAG);
9183 auto [VOffset,
Offset] = splitBufferOffsets(
Op.getOperand(4), DAG);
9196 return lowerIntrinsicLoad(cast<MemSDNode>(
Op), IsFormat, DAG, Ops);
9198 case Intrinsic::amdgcn_raw_tbuffer_load:
9199 case Intrinsic::amdgcn_raw_ptr_tbuffer_load: {
9201 EVT LoadVT =
Op.getValueType();
9202 SDValue Rsrc = bufferRsrcPtrToVector(
Op.getOperand(2), DAG);
9203 auto [VOffset,
Offset] = splitBufferOffsets(
Op.getOperand(3), DAG);
9222 Op->getVTList(), Ops, LoadVT,
M->getMemOperand(),
9225 case Intrinsic::amdgcn_struct_tbuffer_load:
9226 case Intrinsic::amdgcn_struct_ptr_tbuffer_load: {
9228 EVT LoadVT =
Op.getValueType();
9229 SDValue Rsrc = bufferRsrcPtrToVector(
Op.getOperand(2), DAG);
9230 auto [VOffset,
Offset] = splitBufferOffsets(
Op.getOperand(4), DAG);
9249 Op->getVTList(), Ops, LoadVT,
M->getMemOperand(),
9252 case Intrinsic::amdgcn_raw_buffer_atomic_fadd:
9253 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_fadd:
9255 case Intrinsic::amdgcn_struct_buffer_atomic_fadd:
9256 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_fadd:
9257 return lowerStructBufferAtomicIntrin(
Op, DAG,
9259 case Intrinsic::amdgcn_raw_buffer_atomic_fmin:
9260 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_fmin:
9262 case Intrinsic::amdgcn_struct_buffer_atomic_fmin:
9263 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_fmin:
9264 return lowerStructBufferAtomicIntrin(
Op, DAG,
9266 case Intrinsic::amdgcn_raw_buffer_atomic_fmax:
9267 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_fmax:
9269 case Intrinsic::amdgcn_struct_buffer_atomic_fmax:
9270 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_fmax:
9271 return lowerStructBufferAtomicIntrin(
Op, DAG,
9273 case Intrinsic::amdgcn_raw_buffer_atomic_swap:
9274 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_swap:
9276 case Intrinsic::amdgcn_raw_buffer_atomic_add:
9277 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_add:
9279 case Intrinsic::amdgcn_raw_buffer_atomic_sub:
9280 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_sub:
9282 case Intrinsic::amdgcn_raw_buffer_atomic_smin:
9283 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_smin:
9285 case Intrinsic::amdgcn_raw_buffer_atomic_umin:
9286 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_umin:
9288 case Intrinsic::amdgcn_raw_buffer_atomic_smax:
9289 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_smax:
9291 case Intrinsic::amdgcn_raw_buffer_atomic_umax:
9292 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_umax:
9294 case Intrinsic::amdgcn_raw_buffer_atomic_and:
9295 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_and:
9297 case Intrinsic::amdgcn_raw_buffer_atomic_or:
9298 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_or:
9300 case Intrinsic::amdgcn_raw_buffer_atomic_xor:
9301 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_xor:
9303 case Intrinsic::amdgcn_raw_buffer_atomic_inc:
9304 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_inc:
9306 case Intrinsic::amdgcn_raw_buffer_atomic_dec:
9307 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_dec:
9309 case Intrinsic::amdgcn_raw_buffer_atomic_cond_sub_u32:
9310 return lowerRawBufferAtomicIntrin(
Op, DAG,
9312 case Intrinsic::amdgcn_struct_buffer_atomic_swap:
9313 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_swap:
9314 return lowerStructBufferAtomicIntrin(
Op, DAG,
9316 case Intrinsic::amdgcn_struct_buffer_atomic_add:
9317 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_add:
9319 case Intrinsic::amdgcn_struct_buffer_atomic_sub:
9320 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_sub:
9322 case Intrinsic::amdgcn_struct_buffer_atomic_smin:
9323 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_smin:
9324 return lowerStructBufferAtomicIntrin(
Op, DAG,
9326 case Intrinsic::amdgcn_struct_buffer_atomic_umin:
9327 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_umin:
9328 return lowerStructBufferAtomicIntrin(
Op, DAG,
9330 case Intrinsic::amdgcn_struct_buffer_atomic_smax:
9331 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_smax:
9332 return lowerStructBufferAtomicIntrin(
Op, DAG,
9334 case Intrinsic::amdgcn_struct_buffer_atomic_umax:
9335 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_umax:
9336 return lowerStructBufferAtomicIntrin(
Op, DAG,
9338 case Intrinsic::amdgcn_struct_buffer_atomic_and:
9339 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_and:
9341 case Intrinsic::amdgcn_struct_buffer_atomic_or:
9342 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_or:
9344 case Intrinsic::amdgcn_struct_buffer_atomic_xor:
9345 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_xor:
9347 case Intrinsic::amdgcn_struct_buffer_atomic_inc:
9348 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_inc:
9350 case Intrinsic::amdgcn_struct_buffer_atomic_dec:
9351 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_dec:
9353 case Intrinsic::amdgcn_struct_buffer_atomic_cond_sub_u32:
9354 return lowerStructBufferAtomicIntrin(
Op, DAG,
9357 case Intrinsic::amdgcn_raw_buffer_atomic_cmpswap:
9358 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_cmpswap: {
9359 SDValue Rsrc = bufferRsrcPtrToVector(
Op.getOperand(4), DAG);
9360 auto [VOffset,
Offset] = splitBufferOffsets(
Op.getOperand(5), DAG);
9374 EVT VT =
Op.getValueType();
9375 auto *
M = cast<MemSDNode>(
Op);
9378 Op->getVTList(), Ops, VT,
9379 M->getMemOperand());
9381 case Intrinsic::amdgcn_struct_buffer_atomic_cmpswap:
9382 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_cmpswap: {
9383 SDValue Rsrc = bufferRsrcPtrToVector(
Op->getOperand(4), DAG);
9384 auto [VOffset,
Offset] = splitBufferOffsets(
Op.getOperand(6), DAG);
9398 EVT VT =
Op.getValueType();
9399 auto *
M = cast<MemSDNode>(
Op);
9402 Op->getVTList(), Ops, VT,
9403 M->getMemOperand());
9405 case Intrinsic::amdgcn_image_bvh_intersect_ray: {
9407 SDValue NodePtr =
M->getOperand(2);
9408 SDValue RayExtent =
M->getOperand(3);
9409 SDValue RayOrigin =
M->getOperand(4);
9411 SDValue RayInvDir =
M->getOperand(6);
9429 const unsigned NumVDataDwords = 4;
9430 const unsigned NumVAddrDwords = IsA16 ? (Is64 ? 9 : 8) : (Is64 ? 12 : 11);
9431 const unsigned NumVAddrs = IsGFX11Plus ? (IsA16 ? 4 : 5) : NumVAddrDwords;
9435 const unsigned BaseOpcodes[2][2] = {
9436 {AMDGPU::IMAGE_BVH_INTERSECT_RAY, AMDGPU::IMAGE_BVH_INTERSECT_RAY_a16},
9437 {AMDGPU::IMAGE_BVH64_INTERSECT_RAY,
9438 AMDGPU::IMAGE_BVH64_INTERSECT_RAY_a16}};
9442 IsGFX12Plus ? AMDGPU::MIMGEncGfx12
9443 : IsGFX11 ? AMDGPU::MIMGEncGfx11NSA
9444 : AMDGPU::MIMGEncGfx10NSA,
9445 NumVDataDwords, NumVAddrDwords);
9449 IsGFX11 ? AMDGPU::MIMGEncGfx11Default
9450 : AMDGPU::MIMGEncGfx10Default,
9451 NumVDataDwords, NumVAddrDwords);
9457 auto packLanes = [&DAG, &Ops, &
DL](
SDValue Op,
bool IsAligned) {
9460 if (Lanes[0].getValueSizeInBits() == 32) {
9461 for (
unsigned I = 0;
I < 3; ++
I)
9480 if (UseNSA && IsGFX11Plus) {
9488 for (
unsigned I = 0;
I < 3; ++
I) {
9491 {DirLanes[I], InvDirLanes[I]})));
9506 packLanes(RayOrigin,
true);
9507 packLanes(RayDir,
true);
9508 packLanes(RayInvDir,
false);
9513 if (NumVAddrDwords > 12) {
9533 case Intrinsic::amdgcn_global_atomic_fmin_num:
9534 case Intrinsic::amdgcn_global_atomic_fmax_num:
9535 case Intrinsic::amdgcn_flat_atomic_fmin_num:
9536 case Intrinsic::amdgcn_flat_atomic_fmax_num: {
9543 unsigned Opcode = 0;
9545 case Intrinsic::amdgcn_global_atomic_fmin_num:
9546 case Intrinsic::amdgcn_flat_atomic_fmin_num: {
9550 case Intrinsic::amdgcn_global_atomic_fmax_num:
9551 case Intrinsic::amdgcn_flat_atomic_fmax_num: {
9559 Ops,
M->getMemOperand());
9561 case Intrinsic::amdgcn_s_get_barrier_state:
9562 case Intrinsic::amdgcn_s_get_named_barrier_state: {
9567 if (isa<ConstantSDNode>(
Op->getOperand(2))) {
9568 uint64_t BarID = cast<ConstantSDNode>(
Op->getOperand(2))->getZExtValue();
9569 if (IntrID == Intrinsic::amdgcn_s_get_named_barrier_state)
9570 BarID = (BarID >> 4) & 0x3F;
9571 Opc = AMDGPU::S_GET_BARRIER_STATE_IMM;
9576 Opc = AMDGPU::S_GET_BARRIER_STATE_M0;
9577 if (IntrID == Intrinsic::amdgcn_s_get_named_barrier_state) {
9597 return lowerImage(
Op, ImageDimIntr, DAG,
true);
9605SDValue SITargetLowering::getMemIntrinsicNode(
unsigned Opcode,
const SDLoc &
DL,
9615 bool IsTFE = VTList.
NumVTs == 3;
9618 unsigned NumOpDWords = NumValueDWords + 1;
9623 SDValue Op = getMemIntrinsicNode(Opcode,
DL, OpDWordsVTList, Ops,
9624 OpDWordsVT, OpDWordsMMO, DAG);
9639 (VT == MVT::v3i32 || VT == MVT::v3f32)) {
9645 WidenedMemVT, WidenedMMO);
9655 bool ImageStore)
const {
9690 for (
unsigned I = 0;
I < Elts.
size() / 2;
I += 1) {
9696 if ((NumElements % 2) == 1) {
9698 unsigned I = Elts.
size() / 2;
9714 if (NumElements == 3) {
9735 unsigned IntrinsicID =
Op.getConstantOperandVal(1);
9738 switch (IntrinsicID) {
9739 case Intrinsic::amdgcn_exp_compr: {
9743 "intrinsic not supported on subtarget",
DL.getDebugLoc());
9766 unsigned Opc =
Done->isZero() ? AMDGPU::EXP : AMDGPU::EXP_DONE;
9769 case Intrinsic::amdgcn_s_barrier:
9770 case Intrinsic::amdgcn_s_barrier_signal:
9771 case Intrinsic::amdgcn_s_barrier_wait: {
9774 unsigned WGSize =
ST.getFlatWorkGroupSizes(MF.
getFunction()).second;
9775 if (WGSize <=
ST.getWavefrontSize()) {
9778 if (IntrinsicID == Intrinsic::amdgcn_s_barrier_signal)
9779 return Op.getOperand(0);
9782 MVT::Other,
Op.getOperand(0)),
9787 if (
ST.hasSplitBarriers() && IntrinsicID == Intrinsic::amdgcn_s_barrier) {
9793 MVT::Other, K,
Op.getOperand(0)),
9805 case Intrinsic::amdgcn_struct_tbuffer_store:
9806 case Intrinsic::amdgcn_struct_ptr_tbuffer_store: {
9810 VData = handleD16VData(VData, DAG);
9811 SDValue Rsrc = bufferRsrcPtrToVector(
Op.getOperand(3), DAG);
9812 auto [VOffset,
Offset] = splitBufferOffsets(
Op.getOperand(5), DAG);
9830 M->getMemoryVT(),
M->getMemOperand());
9833 case Intrinsic::amdgcn_raw_tbuffer_store:
9834 case Intrinsic::amdgcn_raw_ptr_tbuffer_store: {
9838 VData = handleD16VData(VData, DAG);
9839 SDValue Rsrc = bufferRsrcPtrToVector(
Op.getOperand(3), DAG);
9840 auto [VOffset,
Offset] = splitBufferOffsets(
Op.getOperand(4), DAG);
9858 M->getMemoryVT(),
M->getMemOperand());
9861 case Intrinsic::amdgcn_raw_buffer_store:
9862 case Intrinsic::amdgcn_raw_ptr_buffer_store:
9863 case Intrinsic::amdgcn_raw_buffer_store_format:
9864 case Intrinsic::amdgcn_raw_ptr_buffer_store_format: {
9865 const bool IsFormat =
9866 IntrinsicID == Intrinsic::amdgcn_raw_buffer_store_format ||
9867 IntrinsicID == Intrinsic::amdgcn_raw_ptr_buffer_store_format;
9874 VData = handleD16VData(VData, DAG);
9884 SDValue Rsrc = bufferRsrcPtrToVector(
Op.getOperand(3), DAG);
9885 auto [VOffset,
Offset] = splitBufferOffsets(
Op.getOperand(4), DAG);
9905 return handleByteShortBufferStores(DAG, VDataVT,
DL, Ops, M);
9908 M->getMemoryVT(),
M->getMemOperand());
9911 case Intrinsic::amdgcn_struct_buffer_store:
9912 case Intrinsic::amdgcn_struct_ptr_buffer_store:
9913 case Intrinsic::amdgcn_struct_buffer_store_format:
9914 case Intrinsic::amdgcn_struct_ptr_buffer_store_format: {
9915 const bool IsFormat =
9916 IntrinsicID == Intrinsic::amdgcn_struct_buffer_store_format ||
9917 IntrinsicID == Intrinsic::amdgcn_struct_ptr_buffer_store_format;
9925 VData = handleD16VData(VData, DAG);
9935 auto Rsrc = bufferRsrcPtrToVector(
Op.getOperand(3), DAG);
9936 auto [VOffset,
Offset] = splitBufferOffsets(
Op.getOperand(5), DAG);
9957 return handleByteShortBufferStores(DAG, VDataType,
DL, Ops, M);
9960 M->getMemoryVT(),
M->getMemOperand());
9962 case Intrinsic::amdgcn_raw_buffer_load_lds:
9963 case Intrinsic::amdgcn_raw_ptr_buffer_load_lds:
9964 case Intrinsic::amdgcn_struct_buffer_load_lds:
9965 case Intrinsic::amdgcn_struct_ptr_buffer_load_lds: {
9969 IntrinsicID == Intrinsic::amdgcn_struct_buffer_load_lds ||
9970 IntrinsicID == Intrinsic::amdgcn_struct_ptr_buffer_load_lds;
9971 unsigned OpOffset = HasVIndex ? 1 : 0;
9972 SDValue VOffset =
Op.getOperand(5 + OpOffset);
9974 unsigned Size =
Op->getConstantOperandVal(4);
9980 Opc = HasVIndex ? HasVOffset ? AMDGPU::BUFFER_LOAD_UBYTE_LDS_BOTHEN
9981 : AMDGPU::BUFFER_LOAD_UBYTE_LDS_IDXEN
9982 : HasVOffset ? AMDGPU::BUFFER_LOAD_UBYTE_LDS_OFFEN
9983 : AMDGPU::BUFFER_LOAD_UBYTE_LDS_OFFSET;
9986 Opc = HasVIndex ? HasVOffset ? AMDGPU::BUFFER_LOAD_USHORT_LDS_BOTHEN
9987 : AMDGPU::BUFFER_LOAD_USHORT_LDS_IDXEN
9988 : HasVOffset ? AMDGPU::BUFFER_LOAD_USHORT_LDS_OFFEN
9989 : AMDGPU::BUFFER_LOAD_USHORT_LDS_OFFSET;
9992 Opc = HasVIndex ? HasVOffset ? AMDGPU::BUFFER_LOAD_DWORD_LDS_BOTHEN
9993 : AMDGPU::BUFFER_LOAD_DWORD_LDS_IDXEN
9994 : HasVOffset ? AMDGPU::BUFFER_LOAD_DWORD_LDS_OFFEN
9995 : AMDGPU::BUFFER_LOAD_DWORD_LDS_OFFSET;
10000 Opc = HasVIndex ? HasVOffset ? AMDGPU::BUFFER_LOAD_DWORDX3_LDS_BOTHEN
10001 : AMDGPU::BUFFER_LOAD_DWORDX3_LDS_IDXEN
10002 : HasVOffset ? AMDGPU::BUFFER_LOAD_DWORDX3_LDS_OFFEN
10003 : AMDGPU::BUFFER_LOAD_DWORDX3_LDS_OFFSET;
10008 Opc = HasVIndex ? HasVOffset ? AMDGPU::BUFFER_LOAD_DWORDX4_LDS_BOTHEN
10009 : AMDGPU::BUFFER_LOAD_DWORDX4_LDS_IDXEN
10010 : HasVOffset ? AMDGPU::BUFFER_LOAD_DWORDX4_LDS_OFFEN
10011 : AMDGPU::BUFFER_LOAD_DWORDX4_LDS_OFFSET;
10019 if (HasVIndex && HasVOffset)
10023 else if (HasVIndex)
10025 else if (HasVOffset)
10028 SDValue Rsrc = bufferRsrcPtrToVector(
Op.getOperand(2), DAG);
10033 unsigned Aux =
Op.getConstantOperandVal(8 + OpOffset);
10045 auto *
M = cast<MemSDNode>(
Op);
10072 case Intrinsic::amdgcn_global_load_lds: {
10074 unsigned Size =
Op->getConstantOperandVal(4);
10079 Opc = AMDGPU::GLOBAL_LOAD_LDS_UBYTE;
10082 Opc = AMDGPU::GLOBAL_LOAD_LDS_USHORT;
10085 Opc = AMDGPU::GLOBAL_LOAD_LDS_DWORD;
10090 Opc = AMDGPU::GLOBAL_LOAD_LDS_DWORDX3;
10095 Opc = AMDGPU::GLOBAL_LOAD_LDS_DWORDX4;
10099 auto *
M = cast<MemSDNode>(
Op);
10112 if (
LHS->isDivergent())
10116 RHS.getOperand(0).getValueType() == MVT::i32) {
10119 VOffset =
RHS.getOperand(0);
10124 if (!
Addr->isDivergent()) {
10141 LoadPtrI.
Offset =
Op->getConstantOperandVal(5);
10161 case Intrinsic::amdgcn_end_cf:
10163 Op->getOperand(2), Chain),
10165 case Intrinsic::amdgcn_s_barrier_init:
10166 case Intrinsic::amdgcn_s_barrier_signal_var: {
10173 unsigned Opc = IntrinsicID == Intrinsic::amdgcn_s_barrier_init
10174 ? AMDGPU::S_BARRIER_INIT_M0
10175 : AMDGPU::S_BARRIER_SIGNAL_M0;
10190 constexpr unsigned ShAmt = 16;
10202 case Intrinsic::amdgcn_s_barrier_join: {
10209 if (isa<ConstantSDNode>(BarOp)) {
10210 uint64_t BarVal = cast<ConstantSDNode>(BarOp)->getZExtValue();
10211 Opc = AMDGPU::S_BARRIER_JOIN_IMM;
10214 unsigned BarID = (BarVal >> 4) & 0x3F;
10219 Opc = AMDGPU::S_BARRIER_JOIN_M0;
10235 case Intrinsic::amdgcn_s_prefetch_data: {
10238 return Op.getOperand(0);
10241 case Intrinsic::amdgcn_s_buffer_prefetch_data: {
10243 Chain, bufferRsrcPtrToVector(
Op.getOperand(2), DAG),
10250 Op->getVTList(), Ops,
M->getMemoryVT(),
10251 M->getMemOperand());
10256 return lowerImage(
Op, ImageDimIntr, DAG,
true);
10269std::pair<SDValue, SDValue>
10276 if ((C1 = dyn_cast<ConstantSDNode>(N0)))
10279 C1 = cast<ConstantSDNode>(N0.
getOperand(1));
10293 unsigned Overflow = ImmOffset & ~MaxImm;
10294 ImmOffset -= Overflow;
10295 if ((int32_t)Overflow < 0) {
10296 Overflow += ImmOffset;
10301 auto OverflowVal = DAG.
getConstant(Overflow,
DL, MVT::i32);
10305 SDValue Ops[] = {N0, OverflowVal};
10320void SITargetLowering::setBufferOffsets(
SDValue CombinedOffset,
10322 Align Alignment)
const {
10325 if (
auto *
C = dyn_cast<ConstantSDNode>(CombinedOffset)) {
10328 if (
TII->splitMUBUFOffset(Imm, SOffset, ImmOffset, Alignment)) {
10339 int Offset = cast<ConstantSDNode>(N1)->getSExtValue();
10341 TII->splitMUBUFOffset(
Offset, SOffset, ImmOffset, Alignment)) {
10358SDValue SITargetLowering::bufferRsrcPtrToVector(
SDValue MaybePointer,
10361 return MaybePointer;
10375 SDValue NumRecords =
Op->getOperand(3);
10378 auto [LowHalf, HighHalf] = DAG.
SplitScalar(Pointer, Loc, MVT::i32, MVT::i32);
10381 std::optional<uint32_t> ConstStride = std::nullopt;
10382 if (
auto *ConstNode = dyn_cast<ConstantSDNode>(Stride))
10383 ConstStride = ConstNode->getZExtValue();
10386 if (!ConstStride || *ConstStride != 0) {
10389 ShiftedStride = DAG.
getConstant(*ConstStride << 16, Loc, MVT::i32);
10400 NewHighHalf, NumRecords, Flags);
10410 bool IsTFE)
const {
10420 SDValue Op = getMemIntrinsicNode(Opc,
DL, VTs, Ops, MVT::v2i32, OpMMO, DAG);
10448 if (VDataType == MVT::f16 || VDataType == MVT::bf16)
10452 Ops[1] = BufferStoreExt;
10457 M->getMemOperand());
10482 DAGCombinerInfo &DCI)
const {
10498 if ((MemVT.
isSimple() && !DCI.isAfterLegalizeDAG()) ||
10505 "unexpected vector extload");
10518 "unexpected fp extload");
10536 DCI.AddToWorklist(Cvt.
getNode());
10541 DCI.AddToWorklist(Cvt.
getNode());
10552 if (
Info.isEntryFunction())
10553 return Info.getUserSGPRInfo().hasFlatScratchInit();
10561 EVT MemVT =
Load->getMemoryVT();
10574 EVT RealMemVT = (MemVT == MVT::i1) ? MVT::i8 : MVT::i16;
10602 assert(
Op.getValueType().getVectorElementType() == MVT::i32 &&
10603 "Custom lowering for non-i32 vectors hasn't been implemented.");
10606 unsigned AS =
Load->getAddressSpace();
10630 Alignment >=
Align(4) && NumElements < 32) {
10644 if (NumElements > 4)
10663 if (NumElements > 2)
10668 if (NumElements > 4)
10680 auto Flags =
Load->getMemOperand()->getFlags();
10682 Load->getAlign(), Flags, &
Fast) &&
10691 MemVT, *
Load->getMemOperand())) {
10700 EVT VT =
Op.getValueType();
10737 EVT VT =
Op.getValueType();
10740 bool AllowInaccurateRcp =
10747 if (!AllowInaccurateRcp && VT != MVT::f16)
10750 if (CLHS->isExactlyValue(1.0)) {
10767 if (CLHS->isExactlyValue(-1.0)) {
10776 if (!AllowInaccurateRcp && (VT != MVT::f16 || !
Flags.hasAllowReciprocal()))
10790 EVT VT =
Op.getValueType();
10793 bool AllowInaccurateDiv =
10795 if (!AllowInaccurateDiv)
10816 return DAG.
getNode(Opcode, SL, VT,
A,
B, Flags);
10830 return DAG.
getNode(Opcode, SL, VTList,
10839 return DAG.
getNode(Opcode, SL, VT, {
A,
B,
C}, Flags);
10853 return DAG.
getNode(Opcode, SL, VTList,
10859 if (
SDValue FastLowered = lowerFastUnsafeFDIV(
Op, DAG))
10860 return FastLowered;
10880 unsigned FMADOpCode =
10890 SDValue Err = DAG.
getNode(FMADOpCode, SL, MVT::f32, NegRHSExt, Quot, LHSExt,
10892 Quot = DAG.
getNode(FMADOpCode, SL, MVT::f32, Err, Rcp, Quot,
Op->getFlags());
10893 Err = DAG.
getNode(FMADOpCode, SL, MVT::f32, NegRHSExt, Quot, LHSExt,
10916 const APFloat K0Val(0x1p+96f);
10919 const APFloat K1Val(0x1p-32f);
10946 assert(ST->hasDenormModeInst() &&
"Requires S_DENORM_MODE");
10947 uint32_t DPDenormModeDefault =
Info->getMode().fpDenormModeDPValue();
10948 uint32_t Mode = SPDenormMode | (DPDenormModeDefault << 2);
10953 if (
SDValue FastLowered = lowerFastUnsafeFDIV(
Op, DAG))
10954 return FastLowered;
10961 Flags.setNoFPExcept(
true);
10982 using namespace AMDGPU::Hwreg;
10983 const unsigned Denorm32Reg = HwregEncoding::encode(ID_MODE, 4, 2);
10991 const bool HasDynamicDenormals =
10997 if (!PreservesDenormals) {
11005 if (HasDynamicDenormals) {
11009 SavedDenormMode =
SDValue(GetReg, 0);
11017 const SDValue EnableDenormValue =
11024 const SDValue EnableDenormValue =
11026 EnableDenorm = DAG.
getMachineNode(AMDGPU::S_SETREG_B32, SL, BindParamVTs,
11027 {EnableDenormValue,
BitField, Glue});
11037 ApproxRcp, One, NegDivScale0, Flags);
11040 ApproxRcp, Fma0, Flags);
11046 NumeratorScaled,
Mul, Flags);
11052 NumeratorScaled, Fma3, Flags);
11054 if (!PreservesDenormals) {
11062 DisableDenormValue, Fma4.
getValue(2))
11065 assert(HasDynamicDenormals == (
bool)SavedDenormMode);
11066 const SDValue DisableDenormValue =
11067 HasDynamicDenormals
11072 AMDGPU::S_SETREG_B32, SL, MVT::Other,
11083 {Fma4, Fma1, Fma3, Scale},
Flags);
11089 if (
SDValue FastLowered = lowerFastUnsafeFDIV64(
Op, DAG))
11090 return FastLowered;
11158 EVT VT =
Op.getValueType();
11160 if (VT == MVT::f32)
11161 return LowerFDIV32(
Op, DAG);
11163 if (VT == MVT::f64)
11164 return LowerFDIV64(
Op, DAG);
11166 if (VT == MVT::f16)
11167 return LowerFDIV16(
Op, DAG);
11176 EVT ResultExpVT =
Op->getValueType(1);
11177 EVT InstrExpVT = VT == MVT::f16 ? MVT::i16 : MVT::i32;
11207 if (VT == MVT::i1) {
11211 Store->getBasePtr(), MVT::i1,
Store->getMemOperand());
11215 Store->getValue().getValueType().getScalarType() == MVT::i32);
11217 unsigned AS =
Store->getAddressSpace();
11236 if (NumElements > 4)
11243 VT, *
Store->getMemOperand()))
11253 if (NumElements > 2)
11257 if (NumElements > 4 ||
11266 auto Flags =
Store->getMemOperand()->getFlags();
11301 MVT VT =
Op.getValueType().getSimpleVT();
11472 EVT VT =
Op.getValueType();
11489 switch (
Op.getOpcode()) {
11516 EVT VT =
Op.getValueType();
11524 Op->getVTList(), Ops, VT,
11533SITargetLowering::performUCharToFloatCombine(
SDNode *
N,
11534 DAGCombinerInfo &DCI)
const {
11535 EVT VT =
N->getValueType(0);
11537 if (ScalarVT != MVT::f32 && ScalarVT != MVT::f16)
11544 EVT SrcVT = Src.getValueType();
11550 if (DCI.isAfterLegalizeDAG() && SrcVT == MVT::i32) {
11553 DCI.AddToWorklist(Cvt.
getNode());
11556 if (ScalarVT != MVT::f32) {
11568 DAGCombinerInfo &DCI)
const {
11569 SDValue MagnitudeOp =
N->getOperand(0);
11570 SDValue SignOp =
N->getOperand(1);
11626SDValue SITargetLowering::performSHLPtrCombine(
SDNode *
N,
unsigned AddrSpace,
11628 DAGCombinerInfo &DCI)
const {
11658 AM.HasBaseReg =
true;
11659 AM.BaseOffs =
Offset.getSExtValue();
11664 EVT VT =
N->getValueType(0);
11670 Flags.setNoUnsignedWrap(
11671 N->getFlags().hasNoUnsignedWrap() &&
11681 switch (
N->getOpcode()) {
11692 DAGCombinerInfo &DCI)
const {
11701 SDValue NewPtr = performSHLPtrCombine(
Ptr.getNode(),
N->getAddressSpace(),
11702 N->getMemoryVT(), DCI);
11706 NewOps[PtrIdx] = NewPtr;
11715 return (Opc ==
ISD::AND && (Val == 0 || Val == 0xffffffff)) ||
11716 (Opc ==
ISD::OR && (Val == 0xffffffff || Val == 0)) ||
11725SDValue SITargetLowering::splitBinaryBitConstantOp(
11726 DAGCombinerInfo &DCI,
const SDLoc &SL,
unsigned Opc,
SDValue LHS,
11746 if (V.getValueType() != MVT::i1)
11748 switch (V.getOpcode()) {
11767 if (!(
C & 0x000000ff))
11768 ZeroByteMask |= 0x000000ff;
11769 if (!(
C & 0x0000ff00))
11770 ZeroByteMask |= 0x0000ff00;
11771 if (!(
C & 0x00ff0000))
11772 ZeroByteMask |= 0x00ff0000;
11773 if (!(
C & 0xff000000))
11774 ZeroByteMask |= 0xff000000;
11775 uint32_t NonZeroByteMask = ~ZeroByteMask;
11776 if ((NonZeroByteMask &
C) != NonZeroByteMask)
11789 assert(V.getValueSizeInBits() == 32);
11791 if (V.getNumOperands() != 2)
11800 switch (V.getOpcode()) {
11805 return (0x03020100 & ConstMask) | (0x0c0c0c0c & ~ConstMask);
11810 return (0x03020100 & ~ConstMask) | ConstMask;
11817 return uint32_t((0x030201000c0c0c0cull <<
C) >> 32);
11823 return uint32_t(0x0c0c0c0c03020100ull >>
C);
11830 DAGCombinerInfo &DCI)
const {
11831 if (DCI.isBeforeLegalize())
11835 EVT VT =
N->getValueType(0);
11840 if (VT == MVT::i64 && CRHS) {
11846 if (CRHS && VT == MVT::i32) {
11855 if (
auto *CShift = dyn_cast<ConstantSDNode>(
LHS->getOperand(1))) {
11856 unsigned Shift = CShift->getZExtValue();
11858 unsigned Offset = NB + Shift;
11859 if ((
Offset & (Bits - 1)) == 0) {
11877 isa<ConstantSDNode>(
LHS.getOperand(2))) {
11883 Sel = (
LHS.getConstantOperandVal(2) & Sel) | (~Sel & 0x0c0c0c0c);
11898 if (
Y.getOpcode() !=
ISD::FABS ||
Y.getOperand(0) !=
X ||
11903 if (
X !=
LHS.getOperand(1))
11908 dyn_cast<ConstantFPSDNode>(
RHS.getOperand(1));
11941 (
RHS.getOperand(0) ==
LHS.getOperand(0) &&
11942 LHS.getOperand(0) ==
LHS.getOperand(1))) {
11944 unsigned NewMask = LCC ==
ISD::SETO ?
Mask->getZExtValue() & ~OrdMask
11945 :
Mask->getZExtValue() & OrdMask;
11966 N->isDivergent() &&
TII->pseudoToMCOpcode(AMDGPU::V_PERM_B32_e64) != -1) {
11969 if (LHSMask != ~0u && RHSMask != ~0u) {
11972 if (LHSMask > RHSMask) {
11979 uint32_t LHSUsedLanes = ~(LHSMask & 0x0c0c0c0c) & 0x0c0c0c0c;
11980 uint32_t RHSUsedLanes = ~(RHSMask & 0x0c0c0c0c) & 0x0c0c0c0c;
11983 if (!(LHSUsedLanes & RHSUsedLanes) &&
11986 !(LHSUsedLanes == 0x0c0c0000 && RHSUsedLanes == 0x00000c0c)) {
11993 for (
unsigned I = 0;
I < 32;
I += 8) {
11995 if ((LHSMask & ByteSel) == 0x0c || (RHSMask & ByteSel) == 0x0c)
11996 Mask &= (0x0c <<
I) & 0xffffffff;
12054static const std::optional<ByteProvider<SDValue>>
12056 unsigned Depth = 0) {
12059 return std::nullopt;
12061 if (
Op.getValueSizeInBits() < 8)
12062 return std::nullopt;
12064 if (
Op.getValueType().isVector())
12067 switch (
Op->getOpcode()) {
12078 auto *VTSign = cast<VTSDNode>(
Op->getOperand(1));
12079 NarrowVT = VTSign->getVT();
12082 return std::nullopt;
12085 if (SrcIndex >= NarrowByteWidth)
12086 return std::nullopt;
12092 auto *ShiftOp = dyn_cast<ConstantSDNode>(
Op->getOperand(1));
12094 return std::nullopt;
12096 uint64_t BitShift = ShiftOp->getZExtValue();
12098 if (BitShift % 8 != 0)
12099 return std::nullopt;
12101 SrcIndex += BitShift / 8;
12119static const std::optional<ByteProvider<SDValue>>
12121 unsigned StartingIndex = 0) {
12125 return std::nullopt;
12127 unsigned BitWidth =
Op.getScalarValueSizeInBits();
12129 return std::nullopt;
12131 return std::nullopt;
12133 bool IsVec =
Op.getValueType().isVector();
12134 switch (
Op.getOpcode()) {
12137 return std::nullopt;
12142 return std::nullopt;
12146 return std::nullopt;
12149 if (!
LHS->isConstantZero() && !
RHS->isConstantZero())
12150 return std::nullopt;
12151 if (!
LHS ||
LHS->isConstantZero())
12153 if (!
RHS ||
RHS->isConstantZero())
12155 return std::nullopt;
12160 return std::nullopt;
12162 auto *BitMaskOp = dyn_cast<ConstantSDNode>(
Op->getOperand(1));
12164 return std::nullopt;
12166 uint32_t BitMask = BitMaskOp->getZExtValue();
12168 uint32_t IndexMask = 0xFF << (Index * 8);
12170 if ((IndexMask & BitMask) != IndexMask) {
12173 if (IndexMask & BitMask)
12174 return std::nullopt;
12183 return std::nullopt;
12186 auto *ShiftOp = dyn_cast<ConstantSDNode>(
Op->getOperand(2));
12187 if (!ShiftOp ||
Op.getValueType().isVector())
12188 return std::nullopt;
12190 uint64_t BitsProvided =
Op.getValueSizeInBits();
12191 if (BitsProvided % 8 != 0)
12192 return std::nullopt;
12194 uint64_t BitShift = ShiftOp->getAPIntValue().urem(BitsProvided);
12196 return std::nullopt;
12198 uint64_t ConcatSizeInBytes = BitsProvided / 4;
12199 uint64_t ByteShift = BitShift / 8;
12201 uint64_t NewIndex = (Index + ByteShift) % ConcatSizeInBytes;
12202 uint64_t BytesProvided = BitsProvided / 8;
12203 SDValue NextOp =
Op.getOperand(NewIndex >= BytesProvided ? 0 : 1);
12204 NewIndex %= BytesProvided;
12211 return std::nullopt;
12213 auto *ShiftOp = dyn_cast<ConstantSDNode>(
Op->getOperand(1));
12215 return std::nullopt;
12217 uint64_t BitShift = ShiftOp->getZExtValue();
12219 return std::nullopt;
12221 auto BitsProvided =
Op.getScalarValueSizeInBits();
12222 if (BitsProvided % 8 != 0)
12223 return std::nullopt;
12225 uint64_t BytesProvided = BitsProvided / 8;
12226 uint64_t ByteShift = BitShift / 8;
12231 return BytesProvided - ByteShift > Index
12239 return std::nullopt;
12241 auto *ShiftOp = dyn_cast<ConstantSDNode>(
Op->getOperand(1));
12243 return std::nullopt;
12245 uint64_t BitShift = ShiftOp->getZExtValue();
12246 if (BitShift % 8 != 0)
12247 return std::nullopt;
12248 uint64_t ByteShift = BitShift / 8;
12254 return Index < ByteShift
12257 Depth + 1, StartingIndex);
12266 return std::nullopt;
12273 auto *VTSign = cast<VTSDNode>(
Op->getOperand(1));
12274 NarrowBitWidth = VTSign->getVT().getSizeInBits();
12276 if (NarrowBitWidth % 8 != 0)
12277 return std::nullopt;
12278 uint64_t NarrowByteWidth = NarrowBitWidth / 8;
12280 if (Index >= NarrowByteWidth)
12282 ? std::optional<ByteProvider<SDValue>>(
12290 return std::nullopt;
12294 if (NarrowByteWidth >= Index) {
12299 return std::nullopt;
12306 return std::nullopt;
12310 auto *L = cast<LoadSDNode>(
Op.getNode());
12312 unsigned NarrowBitWidth = L->getMemoryVT().getSizeInBits();
12313 if (NarrowBitWidth % 8 != 0)
12314 return std::nullopt;
12315 uint64_t NarrowByteWidth = NarrowBitWidth / 8;
12320 if (Index >= NarrowByteWidth) {
12322 ? std::optional<ByteProvider<SDValue>>(
12327 if (NarrowByteWidth > Index) {
12331 return std::nullopt;
12336 return std::nullopt;
12339 Depth + 1, StartingIndex);
12343 auto *IdxOp = dyn_cast<ConstantSDNode>(
Op->getOperand(1));
12345 return std::nullopt;
12346 auto VecIdx = IdxOp->getZExtValue();
12347 auto ScalarSize =
Op.getScalarValueSizeInBits();
12348 if (ScalarSize < 32)
12349 Index = ScalarSize == 8 ? VecIdx : VecIdx * 2 + Index;
12351 StartingIndex, Index);
12356 return std::nullopt;
12358 auto *PermMask = dyn_cast<ConstantSDNode>(
Op->getOperand(2));
12360 return std::nullopt;
12363 (PermMask->getZExtValue() & (0xFF << (Index * 8))) >> (Index * 8);
12364 if (IdxMask > 0x07 && IdxMask != 0x0c)
12365 return std::nullopt;
12367 auto NextOp =
Op.getOperand(IdxMask > 0x03 ? 0 : 1);
12368 auto NextIndex = IdxMask > 0x03 ? IdxMask % 4 : IdxMask;
12370 return IdxMask != 0x0c ?
calculateSrcByte(NextOp, StartingIndex, NextIndex)
12376 return std::nullopt;
12391 return !OpVT.
isVector() && OpVT.getSizeInBits() == 16;
12395 auto ExtType = cast<LoadSDNode>(L)->getExtensionType();
12398 auto MemVT = L->getMemoryVT();
12401 return L->getMemoryVT().getSizeInBits() == 16;
12411 int Low8 = Mask & 0xff;
12412 int Hi8 = (Mask & 0xff00) >> 8;
12414 assert(Low8 < 8 && Hi8 < 8);
12416 bool IsConsecutive = (Hi8 - Low8 == 1);
12421 bool Is16Aligned = !(Low8 % 2);
12423 return IsConsecutive && Is16Aligned;
12431 int Low16 = PermMask & 0xffff;
12432 int Hi16 = (PermMask & 0xffff0000) >> 16;
12442 auto OtherOpIs16Bit = TempOtherOp.getValueSizeInBits() == 16 ||
12444 if (!OtherOpIs16Bit)
12452 unsigned DWordOffset) {
12455 auto TypeSize = Src.getValueSizeInBits().getFixedValue();
12457 assert(Src.getValueSizeInBits().isKnownMultipleOf(8));
12462 if (Src.getValueType().isVector()) {
12463 auto ScalarTySize = Src.getScalarValueSizeInBits();
12464 auto ScalarTy = Src.getValueType().getScalarType();
12465 if (ScalarTySize == 32) {
12469 if (ScalarTySize > 32) {
12472 DAG.
getConstant(DWordOffset / (ScalarTySize / 32), SL, MVT::i32));
12473 auto ShiftVal = 32 * (DWordOffset % (ScalarTySize / 32));
12480 assert(ScalarTySize < 32);
12481 auto NumElements =
TypeSize / ScalarTySize;
12482 auto Trunc32Elements = (ScalarTySize * NumElements) / 32;
12483 auto NormalizedTrunc = Trunc32Elements * 32 / ScalarTySize;
12484 auto NumElementsIn32 = 32 / ScalarTySize;
12485 auto NumAvailElements = DWordOffset < Trunc32Elements
12487 : NumElements - NormalizedTrunc;
12500 auto ShiftVal = 32 * DWordOffset;
12508 [[maybe_unused]]
EVT VT =
N->getValueType(0);
12513 for (
int i = 0; i < 4; i++) {
12515 std::optional<ByteProvider<SDValue>>
P =
12518 if (!
P ||
P->isConstantZero())
12523 if (PermNodes.
size() != 4)
12526 std::pair<unsigned, unsigned> FirstSrc(0, PermNodes[0].SrcOffset / 4);
12527 std::optional<std::pair<unsigned, unsigned>> SecondSrc;
12529 for (
size_t i = 0; i < PermNodes.
size(); i++) {
12530 auto PermOp = PermNodes[i];
12533 int SrcByteAdjust = 4;
12537 if (!PermOp.hasSameSrc(PermNodes[FirstSrc.first]) ||
12538 ((PermOp.SrcOffset / 4) != FirstSrc.second)) {
12540 if (!PermOp.hasSameSrc(PermNodes[SecondSrc->first]) ||
12541 ((PermOp.SrcOffset / 4) != SecondSrc->second))
12545 SecondSrc = {i, PermNodes[i].SrcOffset / 4};
12546 assert(!(PermNodes[SecondSrc->first].Src->getValueSizeInBits() % 8));
12549 assert((PermOp.SrcOffset % 4) + SrcByteAdjust < 8);
12551 PermMask |= ((PermOp.SrcOffset % 4) + SrcByteAdjust) << (i * 8);
12554 SDValue Op = *PermNodes[FirstSrc.first].Src;
12556 assert(
Op.getValueSizeInBits() == 32);
12560 int Low16 = PermMask & 0xffff;
12561 int Hi16 = (PermMask & 0xffff0000) >> 16;
12563 bool WellFormedLow = (Low16 == 0x0504) || (Low16 == 0x0100);
12564 bool WellFormedHi = (Hi16 == 0x0706) || (Hi16 == 0x0302);
12567 if (WellFormedLow && WellFormedHi)
12571 SDValue OtherOp = SecondSrc ? *PermNodes[SecondSrc->first].Src :
Op;
12580 assert(
Op.getValueType().isByteSized() &&
12598 DAGCombinerInfo &DCI)
const {
12603 EVT VT =
N->getValueType(0);
12604 if (VT == MVT::i1) {
12609 if (Src !=
RHS.getOperand(0))
12614 if (!CLHS || !CRHS)
12618 static const uint32_t MaxMask = 0x3ff;
12633 isa<ConstantSDNode>(
LHS.getOperand(2))) {
12638 Sel |=
LHS.getConstantOperandVal(2);
12647 N->isDivergent() &&
TII->pseudoToMCOpcode(AMDGPU::V_PERM_B32_e64) != -1) {
12651 auto usesCombinedOperand = [](
SDNode *OrUse) {
12654 !OrUse->getValueType(0).isVector())
12658 for (
auto *VUser : OrUse->users()) {
12659 if (!VUser->getValueType(0).isVector())
12666 if (VUser->getOpcode() == VectorwiseOp)
12672 if (!
any_of(
N->users(), usesCombinedOperand))
12678 if (LHSMask != ~0u && RHSMask != ~0u) {
12681 if (LHSMask > RHSMask) {
12688 uint32_t LHSUsedLanes = ~(LHSMask & 0x0c0c0c0c) & 0x0c0c0c0c;
12689 uint32_t RHSUsedLanes = ~(RHSMask & 0x0c0c0c0c) & 0x0c0c0c0c;
12692 if (!(LHSUsedLanes & RHSUsedLanes) &&
12695 !(LHSUsedLanes == 0x0c0c0000 && RHSUsedLanes == 0x00000c0c)) {
12697 LHSMask &= ~RHSUsedLanes;
12698 RHSMask &= ~LHSUsedLanes;
12700 LHSMask |= LHSUsedLanes & 0x04040404;
12710 if (LHSMask == ~0u || RHSMask == ~0u) {
12716 if (VT != MVT::i64 || DCI.isBeforeLegalizeOps())
12731 if (SrcVT == MVT::i32) {
12736 DCI.AddToWorklist(LowOr.
getNode());
12737 DCI.AddToWorklist(HiBits.getNode());
12745 const ConstantSDNode *CRHS = dyn_cast<ConstantSDNode>(
N->getOperand(1));
12748 N->getOperand(0), CRHS))
12756 DAGCombinerInfo &DCI)
const {
12757 if (
SDValue RV = reassociateScalarOps(
N, DCI.DAG))
12766 EVT VT =
N->getValueType(0);
12767 if (CRHS && VT == MVT::i64) {
12789 LHS->getOperand(0), FNegLHS, FNegRHS);
12798 DAGCombinerInfo &DCI)
const {
12803 EVT VT =
N->getValueType(0);
12804 if (VT != MVT::i32)
12808 if (Src.getValueType() != MVT::i16)
12815SITargetLowering::performSignExtendInRegCombine(
SDNode *
N,
12816 DAGCombinerInfo &DCI)
const {
12818 auto *VTSign = cast<VTSDNode>(
N->getOperand(1));
12823 VTSign->getVT() == MVT::i8) ||
12825 VTSign->getVT() == MVT::i16))) {
12827 "s_buffer_load_{u8, i8} are supported "
12828 "in GFX12 (or newer) architectures.");
12829 EVT VT = Src.getValueType();
12834 SDVTList ResList = DCI.DAG.getVTList(MVT::i32);
12840 auto *
M = cast<MemSDNode>(Src);
12841 SDValue BufferLoad = DCI.DAG.getMemIntrinsicNode(
12842 Opc,
DL, ResList, Ops,
M->getMemoryVT(),
M->getMemOperand());
12847 VTSign->getVT() == MVT::i8) ||
12849 VTSign->getVT() == MVT::i16)) &&
12851 auto *
M = cast<MemSDNode>(Src);
12852 SDValue Ops[] = {Src.getOperand(0),
12858 Src.getOperand(6), Src.getOperand(7)};
12861 DCI.DAG.getVTList(MVT::i32, Src.getOperand(0).getValueType());
12865 SDValue BufferLoadSignExt = DCI.DAG.getMemIntrinsicNode(
12866 Opc,
SDLoc(
N), ResList, Ops,
M->getMemoryVT(),
M->getMemOperand());
12867 return DCI.DAG.getMergeValues(
12874 DAGCombinerInfo &DCI)
const {
12882 if (
N->getOperand(0).isUndef())
12889 DAGCombinerInfo &DCI)
const {
12890 EVT VT =
N->getValueType(0);
12915 unsigned MaxDepth)
const {
12916 unsigned Opcode =
Op.getOpcode();
12920 if (
auto *CFP = dyn_cast<ConstantFPSDNode>(
Op)) {
12921 const auto &
F = CFP->getValueAPF();
12922 if (
F.isNaN() &&
F.isSignaling())
12924 if (!
F.isDenormal())
12987 if (
Op.getValueType() == MVT::i32) {
12992 if (
auto *
RHS = dyn_cast<ConstantSDNode>(
Op.getOperand(1))) {
12993 if (
RHS->getZExtValue() == 0xffff0000) {
13003 return Op.getValueType().getScalarType() != MVT::f16;
13071 if (
Op.getValueType() == MVT::i16) {
13082 unsigned IntrinsicID =
Op.getConstantOperandVal(0);
13084 switch (IntrinsicID) {
13085 case Intrinsic::amdgcn_cvt_pkrtz:
13086 case Intrinsic::amdgcn_cubeid:
13087 case Intrinsic::amdgcn_frexp_mant:
13088 case Intrinsic::amdgcn_fdot2:
13089 case Intrinsic::amdgcn_rcp:
13090 case Intrinsic::amdgcn_rsq:
13091 case Intrinsic::amdgcn_rsq_clamp:
13092 case Intrinsic::amdgcn_rcp_legacy:
13093 case Intrinsic::amdgcn_rsq_legacy:
13094 case Intrinsic::amdgcn_trig_preop:
13095 case Intrinsic::amdgcn_log:
13096 case Intrinsic::amdgcn_exp2:
13097 case Intrinsic::amdgcn_sqrt:
13115 unsigned MaxDepth)
const {
13118 unsigned Opcode =
MI->getOpcode();
13120 if (Opcode == AMDGPU::G_FCANONICALIZE)
13123 std::optional<FPValueAndVReg> FCR;
13126 if (FCR->Value.isSignaling())
13128 if (!FCR->Value.isDenormal())
13139 case AMDGPU::G_FADD:
13140 case AMDGPU::G_FSUB:
13141 case AMDGPU::G_FMUL:
13142 case AMDGPU::G_FCEIL:
13143 case AMDGPU::G_FFLOOR:
13144 case AMDGPU::G_FRINT:
13145 case AMDGPU::G_FNEARBYINT:
13146 case AMDGPU::G_INTRINSIC_FPTRUNC_ROUND:
13147 case AMDGPU::G_INTRINSIC_TRUNC:
13148 case AMDGPU::G_INTRINSIC_ROUNDEVEN:
13149 case AMDGPU::G_FMA:
13150 case AMDGPU::G_FMAD:
13151 case AMDGPU::G_FSQRT:
13152 case AMDGPU::G_FDIV:
13153 case AMDGPU::G_FREM:
13154 case AMDGPU::G_FPOW:
13155 case AMDGPU::G_FPEXT:
13156 case AMDGPU::G_FLOG:
13157 case AMDGPU::G_FLOG2:
13158 case AMDGPU::G_FLOG10:
13159 case AMDGPU::G_FPTRUNC:
13160 case AMDGPU::G_AMDGPU_RCP_IFLAG:
13161 case AMDGPU::G_AMDGPU_CVT_F32_UBYTE0:
13162 case AMDGPU::G_AMDGPU_CVT_F32_UBYTE1:
13163 case AMDGPU::G_AMDGPU_CVT_F32_UBYTE2:
13164 case AMDGPU::G_AMDGPU_CVT_F32_UBYTE3:
13166 case AMDGPU::G_FNEG:
13167 case AMDGPU::G_FABS:
13168 case AMDGPU::G_FCOPYSIGN:
13170 case AMDGPU::G_FMINNUM:
13171 case AMDGPU::G_FMAXNUM:
13172 case AMDGPU::G_FMINNUM_IEEE:
13173 case AMDGPU::G_FMAXNUM_IEEE:
13174 case AMDGPU::G_FMINIMUM:
13175 case AMDGPU::G_FMAXIMUM: {
13183 case AMDGPU::G_BUILD_VECTOR:
13188 case AMDGPU::G_INTRINSIC:
13189 case AMDGPU::G_INTRINSIC_CONVERGENT:
13191 case Intrinsic::amdgcn_fmul_legacy:
13192 case Intrinsic::amdgcn_fmad_ftz:
13193 case Intrinsic::amdgcn_sqrt:
13194 case Intrinsic::amdgcn_fmed3:
13195 case Intrinsic::amdgcn_sin:
13196 case Intrinsic::amdgcn_cos:
13197 case Intrinsic::amdgcn_log:
13198 case Intrinsic::amdgcn_exp2:
13199 case Intrinsic::amdgcn_log_clamp:
13200 case Intrinsic::amdgcn_rcp:
13201 case Intrinsic::amdgcn_rcp_legacy:
13202 case Intrinsic::amdgcn_rsq:
13203 case Intrinsic::amdgcn_rsq_clamp:
13204 case Intrinsic::amdgcn_rsq_legacy:
13205 case Intrinsic::amdgcn_div_scale:
13206 case Intrinsic::amdgcn_div_fmas:
13207 case Intrinsic::amdgcn_div_fixup:
13208 case Intrinsic::amdgcn_fract:
13209 case Intrinsic::amdgcn_cvt_pkrtz:
13210 case Intrinsic::amdgcn_cubeid:
13211 case Intrinsic::amdgcn_cubema:
13212 case Intrinsic::amdgcn_cubesc:
13213 case Intrinsic::amdgcn_cubetc:
13214 case Intrinsic::amdgcn_frexp_mant:
13215 case Intrinsic::amdgcn_fdot2:
13216 case Intrinsic::amdgcn_trig_preop:
13235 if (
C.isDenormal()) {
13249 if (
C.isSignaling()) {
13268 return Op.isUndef() || isa<ConstantFPSDNode>(
Op);
13272SITargetLowering::performFCanonicalizeCombine(
SDNode *
N,
13273 DAGCombinerInfo &DCI)
const {
13276 EVT VT =
N->getValueType(0);
13285 EVT VT =
N->getValueType(0);
13286 return getCanonicalConstantFP(DAG,
SDLoc(
N), VT, CFP->getValueAPF());
13302 EVT EltVT =
Lo.getValueType();
13305 for (
unsigned I = 0;
I != 2; ++
I) {
13309 getCanonicalConstantFP(DAG, SL, EltVT, CFP->getValueAPF());
13310 }
else if (
Op.isUndef()) {
13322 if (isa<ConstantFPSDNode>(NewElts[1]))
13323 NewElts[0] = isa<ConstantFPSDNode>(NewElts[1])
13329 NewElts[1] = isa<ConstantFPSDNode>(NewElts[0])
13381 if (!MinK || !MaxK)
13394 if (VT == MVT::i32 || (VT == MVT::i16 && Subtarget->
hasMed3_16()))
13395 return DAG.
getNode(Med3Opc, SL, VT, Src, MaxVal, MinVal);
13436 if (
Info->getMode().DX10Clamp) {
13445 if (VT == MVT::f32 || (VT == MVT::f16 && Subtarget->
hasMed3_16())) {
13477 return (VT == MVT::f32) || (VT == MVT::f16 && Subtarget.
hasMin3Max3_16());
13486 return (VT == MVT::i32) || (VT == MVT::i16 && Subtarget.
hasMin3Max3_16());
13495 DAGCombinerInfo &DCI)
const {
13498 EVT VT =
N->getValueType(0);
13499 unsigned Opc =
N->getOpcode();
13528 if (
SDValue Med3 = performIntMed3ImmCombine(
13533 if (
SDValue Med3 = performIntMed3ImmCombine(
13539 if (
SDValue Med3 = performIntMed3ImmCombine(
13544 if (
SDValue Med3 = performIntMed3ImmCombine(
13554 (VT == MVT::f32 || VT == MVT::f64 ||
13558 if (
SDValue Res = performFPMed3ImmCombine(DAG,
SDLoc(
N), Op0, Op1))
13569 return (CA->isExactlyValue(0.0) && CB->isExactlyValue(1.0)) ||
13570 (CA->isExactlyValue(1.0) && CB->isExactlyValue(0.0));
13579 DAGCombinerInfo &DCI)
const {
13580 EVT VT =
N->getValueType(0);
13603 if (
Info->getMode().DX10Clamp) {
13606 if (isa<ConstantFPSDNode>(Src0) && !isa<ConstantFPSDNode>(Src1))
13609 if (isa<ConstantFPSDNode>(Src1) && !isa<ConstantFPSDNode>(Src2))
13612 if (isa<ConstantFPSDNode>(Src0) && !isa<ConstantFPSDNode>(Src1))
13623 DAGCombinerInfo &DCI)
const {
13627 return DCI.DAG.getUNDEF(
N->getValueType(0));
13635 bool IsDivergentIdx,
13640 unsigned VecSize = EltSize * NumElem;
13643 if (VecSize <= 64 && EltSize < 32)
13652 if (IsDivergentIdx)
13656 unsigned NumInsts = NumElem +
13657 ((EltSize + 31) / 32) * NumElem ;
13662 return NumInsts <= 16;
13667 return NumInsts <= 15;
13674 if (isa<ConstantSDNode>(
Idx))
13688SITargetLowering::performExtractVectorEltCombine(
SDNode *
N,
13689 DAGCombinerInfo &DCI)
const {
13695 EVT ResVT =
N->getValueType(0);
13714 if (Vec.
hasOneUse() && DCI.isBeforeLegalize() && VecEltVT == ResVT) {
13742 DCI.AddToWorklist(Elt0.
getNode());
13743 DCI.AddToWorklist(Elt1.
getNode());
13765 if (!DCI.isBeforeLegalize())
13771 auto *
Idx = dyn_cast<ConstantSDNode>(
N->getOperand(1));
13772 if (isa<MemSDNode>(Vec) && VecEltSize <= 16 && VecEltVT.
isByteSized() &&
13773 VecSize > 32 && VecSize % 32 == 0 &&
Idx) {
13776 unsigned BitIndex =
Idx->getZExtValue() * VecEltSize;
13777 unsigned EltIdx = BitIndex / 32;
13778 unsigned LeftoverBitIdx = BitIndex % 32;
13782 DCI.AddToWorklist(Cast.
getNode());
13786 DCI.AddToWorklist(Elt.
getNode());
13789 DCI.AddToWorklist(Srl.
getNode());
13793 DCI.AddToWorklist(Trunc.
getNode());
13795 if (VecEltVT == ResVT) {
13807SITargetLowering::performInsertVectorEltCombine(
SDNode *
N,
13808 DAGCombinerInfo &DCI)
const {
13822 EVT IdxVT =
Idx.getValueType();
13839 Src.getOperand(0).getValueType() == MVT::f16) {
13840 return Src.getOperand(0);
13843 if (
auto *CFP = dyn_cast<ConstantFPSDNode>(Src)) {
13844 APFloat Val = CFP->getValueAPF();
13845 bool LosesInfo =
true;
13855 DAGCombinerInfo &DCI)
const {
13857 "combine only useful on gfx8");
13859 SDValue TruncSrc =
N->getOperand(0);
13860 EVT VT =
N->getValueType(0);
13861 if (VT != MVT::f16)
13899unsigned SITargetLowering::getFusedOpcode(
const SelectionDAG &DAG,
13901 const SDNode *N1)
const {
13906 if (((VT == MVT::f32 &&
13908 (VT == MVT::f16 && Subtarget->
hasMadF16() &&
13928 EVT VT =
N->getValueType(0);
13929 if (VT != MVT::i32 && VT != MVT::i64)
13935 unsigned Opc =
N->getOpcode();
13958 return DAG.
getNode(Opc, SL, VT, Add1, Op2);
13990 if (!Const ||
Hi_32(Const->getZExtValue()) !=
uint32_t(-1))
14009 DAGCombinerInfo &DCI)
const {
14013 EVT VT =
N->getValueType(0);
14023 if (!
N->isDivergent() && Subtarget->
hasSMulHi())
14027 if (NumBits <= 32 || NumBits > 64)
14039 unsigned NumUsers = 0;
14067 bool MulSignedLo =
false;
14068 if (!MulLHSUnsigned32 || !MulRHSUnsigned32) {
14077 if (VT != MVT::i64) {
14100 getMad64_32(DAG, SL, MVT::i64, MulLHSLo, MulRHSLo, AddRHS, MulSignedLo);
14102 if (!MulSignedLo && (!MulLHSUnsigned32 || !MulRHSUnsigned32)) {
14103 auto [AccumLo, AccumHi] = DAG.
SplitScalar(Accum, SL, MVT::i32, MVT::i32);
14105 if (!MulLHSUnsigned32) {
14112 if (!MulRHSUnsigned32) {
14123 if (VT != MVT::i64)
14129SITargetLowering::foldAddSub64WithZeroLowBitsTo32(
SDNode *
N,
14130 DAGCombinerInfo &DCI)
const {
14132 auto *CRHS = dyn_cast<ConstantSDNode>(RHS);
14156 DAG.
getNode(
N->getOpcode(), SL, MVT::i32,
Hi, ConstHi32,
N->getFlags());
14167static std::optional<ByteProvider<SDValue>>
14170 if (!Byte0 || Byte0->isConstantZero()) {
14171 return std::nullopt;
14174 if (Byte1 && !Byte1->isConstantZero()) {
14175 return std::nullopt;
14181 unsigned FirstCs =
First & 0x0c0c0c0c;
14182 unsigned SecondCs = Second & 0x0c0c0c0c;
14183 unsigned FirstNoCs =
First & ~0x0c0c0c0c;
14184 unsigned SecondNoCs = Second & ~0x0c0c0c0c;
14186 assert((FirstCs & 0xFF) | (SecondCs & 0xFF));
14187 assert((FirstCs & 0xFF00) | (SecondCs & 0xFF00));
14188 assert((FirstCs & 0xFF0000) | (SecondCs & 0xFF0000));
14189 assert((FirstCs & 0xFF000000) | (SecondCs & 0xFF000000));
14191 return (FirstNoCs | SecondNoCs) | (FirstCs & SecondCs);
14215 for (
int BPI = 0; BPI < 2; BPI++) {
14218 BPP = {Src1, Src0};
14220 unsigned ZeroMask = 0x0c0c0c0c;
14221 unsigned FMask = 0xFF << (8 * (3 - Step));
14223 unsigned FirstMask =
14224 (BPP.first.SrcOffset % 4) << (8 * (3 - Step)) | (ZeroMask & ~FMask);
14225 unsigned SecondMask =
14226 (BPP.second.SrcOffset % 4) << (8 * (3 - Step)) | (ZeroMask & ~FMask);
14230 int FirstGroup = -1;
14231 for (
int I = 0;
I < 2;
I++) {
14233 auto MatchesFirst = [&BPP](
DotSrc &IterElt) {
14234 return IterElt.SrcOp == *BPP.first.Src &&
14235 (IterElt.DWordOffset == (BPP.first.SrcOffset / 4));
14245 if (FirstGroup != -1) {
14247 auto MatchesSecond = [&BPP](
DotSrc &IterElt) {
14248 return IterElt.SrcOp == *BPP.second.Src &&
14249 (IterElt.DWordOffset == (BPP.second.SrcOffset / 4));
14255 Srcs.
push_back({*BPP.second.Src, SecondMask, BPP.second.SrcOffset / 4});
14263 unsigned ZeroMask = 0x0c0c0c0c;
14264 unsigned FMask = 0xFF << (8 * (3 - Step));
14268 ((Src0.
SrcOffset % 4) << (8 * (3 - Step)) | (ZeroMask & ~FMask)),
14272 ((Src1.
SrcOffset % 4) << (8 * (3 - Step)) | (ZeroMask & ~FMask)),
14281 if (Srcs.
size() == 1) {
14282 auto *Elt = Srcs.
begin();
14286 if (Elt->PermMask == 0x3020100)
14293 auto *FirstElt = Srcs.
begin();
14294 auto *SecondElt = std::next(FirstElt);
14301 auto FirstMask = FirstElt->PermMask;
14302 auto SecondMask = SecondElt->PermMask;
14304 unsigned FirstCs = FirstMask & 0x0c0c0c0c;
14305 unsigned FirstPlusFour = FirstMask | 0x04040404;
14308 FirstMask = (FirstPlusFour & 0x0F0F0F0F) | FirstCs;
14320 FirstElt = std::next(SecondElt);
14321 if (FirstElt == Srcs.
end())
14324 SecondElt = std::next(FirstElt);
14327 if (SecondElt == Srcs.
end()) {
14333 DAG.
getConstant(FirstElt->PermMask, SL, MVT::i32)));
14339 return Perms.
size() == 2
14345 for (
auto &[EntryVal, EntryMask, EntryOffset] : Srcs) {
14346 EntryMask = EntryMask >> ((4 - ChainLength) * 8);
14347 auto ZeroMask = ChainLength == 2 ? 0x0c0c0000 : 0x0c000000;
14348 EntryMask += ZeroMask;
14353 auto Opcode =
Op.getOpcode();
14359static std::optional<bool>
14370 bool S0IsSigned = Known0.countMinLeadingOnes() > 0;
14373 bool S1IsSigned = Known1.countMinLeadingOnes() > 0;
14375 assert(!(S0IsUnsigned && S0IsSigned));
14376 assert(!(S1IsUnsigned && S1IsSigned));
14384 if ((S0IsUnsigned && S1IsUnsigned) || (S0IsSigned && S1IsSigned))
14390 if ((S0IsUnsigned && S1IsSigned) || (S0IsSigned && S1IsUnsigned))
14391 return std::nullopt;
14403 if ((S0IsSigned && !(S1IsSigned || S1IsUnsigned)) ||
14404 ((S1IsSigned && !(S0IsSigned || S0IsUnsigned))))
14409 if ((!(S1IsSigned || S1IsUnsigned) && !(S0IsSigned || S0IsUnsigned)))
14415 if ((S0IsUnsigned && !(S1IsSigned || S1IsUnsigned)) ||
14416 ((S1IsUnsigned && !(S0IsSigned || S0IsUnsigned))))
14417 return std::nullopt;
14423 DAGCombinerInfo &DCI)
const {
14425 EVT VT =
N->getValueType(0);
14432 if (
SDValue Folded = tryFoldToMad64_32(
N, DCI))
14437 if (
SDValue V = reassociateScalarOps(
N, DAG)) {
14441 if (VT == MVT::i64) {
14442 if (
SDValue Folded = foldAddSub64WithZeroLowBitsTo32(
N, DCI))
14449 std::optional<bool> IsSigned;
14455 int ChainLength = 0;
14456 for (
int I = 0;
I < 4;
I++) {
14457 auto MulIdx =
isMul(LHS) ? 0 :
isMul(RHS) ? 1 : -1;
14460 auto Src0 =
handleMulOperand(TempNode->getOperand(MulIdx)->getOperand(0));
14463 auto Src1 =
handleMulOperand(TempNode->getOperand(MulIdx)->getOperand(1));
14468 TempNode->getOperand(MulIdx), *Src0, *Src1,
14469 TempNode->getOperand(MulIdx)->getOperand(0),
14470 TempNode->getOperand(MulIdx)->getOperand(1), DAG);
14474 IsSigned = *IterIsSigned;
14475 if (*IterIsSigned != *IsSigned)
14478 auto AddIdx = 1 - MulIdx;
14481 if (
I == 2 &&
isMul(TempNode->getOperand(AddIdx))) {
14482 Src2s.
push_back(TempNode->getOperand(AddIdx));
14492 TempNode->getOperand(AddIdx), *Src0, *Src1,
14493 TempNode->getOperand(AddIdx)->getOperand(0),
14494 TempNode->getOperand(AddIdx)->getOperand(1), DAG);
14498 if (*IterIsSigned != *IsSigned)
14502 ChainLength =
I + 2;
14506 TempNode = TempNode->getOperand(AddIdx);
14508 ChainLength =
I + 1;
14509 if (TempNode->getNumOperands() < 2)
14511 LHS = TempNode->getOperand(0);
14512 RHS = TempNode->getOperand(1);
14515 if (ChainLength < 2)
14521 if (ChainLength < 4) {
14531 bool UseOriginalSrc =
false;
14532 if (ChainLength == 4 && Src0s.
size() == 1 && Src1s.
size() == 1 &&
14533 Src0s.
begin()->PermMask == Src1s.
begin()->PermMask &&
14534 Src0s.
begin()->SrcOp.getValueSizeInBits() >= 32 &&
14535 Src1s.
begin()->SrcOp.getValueSizeInBits() >= 32) {
14537 auto Src0Mask = Src0s.
begin()->PermMask;
14538 SrcBytes.
push_back(Src0Mask & 0xFF000000);
14539 bool UniqueEntries =
true;
14540 for (
auto I = 1;
I < 4;
I++) {
14541 auto NextByte = Src0Mask & (0xFF << ((3 -
I) * 8));
14544 UniqueEntries =
false;
14550 if (UniqueEntries) {
14551 UseOriginalSrc =
true;
14553 auto *FirstElt = Src0s.
begin();
14557 auto *SecondElt = Src1s.
begin();
14559 SecondElt->DWordOffset);
14568 if (!UseOriginalSrc) {
14575 DAG.
getExtOrTrunc(*IsSigned, Src2s[ChainLength - 1], SL, MVT::i32);
14578 : Intrinsic::amdgcn_udot4,
14588 if (VT != MVT::i32 || !DCI.isAfterLegalizeDAG())
14593 unsigned Opc =
LHS.getOpcode();
14598 Opc =
RHS.getOpcode();
14605 auto Cond =
RHS.getOperand(0);
14613 return DAG.
getNode(Opc, SL, VTList, Args);
14627 DAGCombinerInfo &DCI)
const {
14629 EVT VT =
N->getValueType(0);
14631 if (VT == MVT::i64) {
14632 if (
SDValue Folded = foldAddSub64WithZeroLowBitsTo32(
N, DCI))
14636 if (VT != MVT::i32)
14645 unsigned Opc =
RHS.getOpcode();
14652 auto Cond =
RHS.getOperand(0);
14660 return DAG.
getNode(Opc, SL, VTList, Args);
14675SITargetLowering::performAddCarrySubCarryCombine(
SDNode *
N,
14676 DAGCombinerInfo &DCI)
const {
14678 if (
N->getValueType(0) != MVT::i32)
14689 unsigned LHSOpc =
LHS.getOpcode();
14690 unsigned Opc =
N->getOpcode();
14700 DAGCombinerInfo &DCI)
const {
14705 EVT VT =
N->getValueType(0);
14717 if (
A ==
LHS.getOperand(1)) {
14718 unsigned FusedOp = getFusedOpcode(DAG,
N,
LHS.getNode());
14719 if (FusedOp != 0) {
14721 return DAG.
getNode(FusedOp, SL, VT,
A, Two, RHS);
14729 if (
A ==
RHS.getOperand(1)) {
14730 unsigned FusedOp = getFusedOpcode(DAG,
N,
RHS.getNode());
14731 if (FusedOp != 0) {
14733 return DAG.
getNode(FusedOp, SL, VT,
A, Two, LHS);
14742 DAGCombinerInfo &DCI)
const {
14748 EVT VT =
N->getValueType(0);
14761 if (
A ==
LHS.getOperand(1)) {
14762 unsigned FusedOp = getFusedOpcode(DAG,
N,
LHS.getNode());
14763 if (FusedOp != 0) {
14767 return DAG.
getNode(FusedOp, SL, VT,
A, Two, NegRHS);
14776 if (
A ==
RHS.getOperand(1)) {
14777 unsigned FusedOp = getFusedOpcode(DAG,
N,
RHS.getNode());
14778 if (FusedOp != 0) {
14780 return DAG.
getNode(FusedOp, SL, VT,
A, NegTwo, LHS);
14789 DAGCombinerInfo &DCI)
const {
14792 EVT VT =
N->getValueType(0);
14806 bool IsNegative =
false;
14807 if (CLHS->isExactlyValue(1.0) ||
14808 (IsNegative = CLHS->isExactlyValue(-1.0))) {
14824 DAGCombinerInfo &DCI)
const {
14826 EVT VT =
N->getValueType(0);
14840 if ((ScalarVT == MVT::f64 || ScalarVT == MVT::f32 || ScalarVT == MVT::f16) &&
14855 if (ScalarVT == MVT::f32 &&
14861 if (TrueNodeExpVal == INT_MIN)
14864 if (FalseNodeExpVal == INT_MIN)
14884 DAGCombinerInfo &DCI)
const {
14886 EVT VT =
N->getValueType(0);
14907 (
N->getFlags().hasAllowContract() &&
14908 FMA->getFlags().hasAllowContract())) {
14942 if (Vec1 == Vec2 || Vec3 == Vec4)
14948 if ((Vec1 == Vec3 && Vec2 == Vec4) || (Vec1 == Vec4 && Vec2 == Vec3)) {
14957 DAGCombinerInfo &DCI)
const {
14963 EVT VT =
LHS.getValueType();
14966 auto *CRHS = dyn_cast<ConstantSDNode>(RHS);
14968 CRHS = dyn_cast<ConstantSDNode>(LHS);
14992 return LHS.getOperand(0);
14998 isa<ConstantSDNode>(
LHS.getOperand(1)) &&
14999 isa<ConstantSDNode>(
LHS.getOperand(2)) &&
15000 LHS.getConstantOperandVal(1) !=
LHS.getConstantOperandVal(2) &&
15007 const APInt &CT =
LHS.getConstantOperandAPInt(1);
15008 const APInt &CF =
LHS.getConstantOperandAPInt(2);
15016 return LHS.getOperand(0);
15020 if (VT != MVT::f32 && VT != MVT::f64 &&
15036 const unsigned IsInfMask =
15038 const unsigned IsFiniteMask =
15052SITargetLowering::performCvtF32UByteNCombine(
SDNode *
N,
15053 DAGCombinerInfo &DCI)
const {
15071 if (
auto *
C = dyn_cast<ConstantSDNode>(Shift.
getOperand(1))) {
15075 unsigned ShiftOffset = 8 *
Offset;
15077 ShiftOffset -=
C->getZExtValue();
15079 ShiftOffset +=
C->getZExtValue();
15081 if (ShiftOffset < 32 && (ShiftOffset % 8) == 0) {
15083 MVT::f32, Shifted);
15094 DCI.AddToWorklist(
N);
15101 return DAG.
getNode(
N->getOpcode(), SL, MVT::f32, DemandedSrc);
15107 DAGCombinerInfo &DCI)
const {
15117 return DCI.DAG.getConstantFP(Zero,
SDLoc(
N),
N->getValueType(0));
15120 APFloat One(
F.getSemantics(),
"1.0");
15122 return DCI.DAG.getConstantFP(One,
SDLoc(
N),
N->getValueType(0));
15129 switch (
N->getOpcode()) {
15145 if (
auto Res = promoteUniformOpToI32(
SDValue(
N, 0), DCI))
15155 switch (
N->getOpcode()) {
15157 return performAddCombine(
N, DCI);
15159 return performSubCombine(
N, DCI);
15162 return performAddCarrySubCarryCombine(
N, DCI);
15164 return performFAddCombine(
N, DCI);
15166 return performFSubCombine(
N, DCI);
15168 return performFDivCombine(
N, DCI);
15170 return performFMulCombine(
N, DCI);
15172 return performSetCCCombine(
N, DCI);
15185 return performMinMaxCombine(
N, DCI);
15187 return performFMACombine(
N, DCI);
15189 return performAndCombine(
N, DCI);
15191 return performOrCombine(
N, DCI);
15194 if (
N->getValueType(0) == MVT::i32 &&
N->isDivergent() &&
15195 TII->pseudoToMCOpcode(AMDGPU::V_PERM_B32_e64) != -1) {
15201 return performXorCombine(
N, DCI);
15203 return performZeroExtendCombine(
N, DCI);
15205 return performSignExtendInRegCombine(
N, DCI);
15207 return performClassCombine(
N, DCI);
15209 return performFCanonicalizeCombine(
N, DCI);
15211 return performRcpCombine(
N, DCI);
15226 return performUCharToFloatCombine(
N, DCI);
15228 return performFCopySignCombine(
N, DCI);
15233 return performCvtF32UByteNCombine(
N, DCI);
15235 return performFMed3Combine(
N, DCI);
15237 return performCvtPkRTZCombine(
N, DCI);
15239 return performClampCombine(
N, DCI);
15242 EVT VT =
N->getValueType(0);
15245 if (VT == MVT::v2i16 || VT == MVT::v2f16 || VT == MVT::v2bf16) {
15248 EVT EltVT = Src.getValueType();
15249 if (EltVT != MVT::i16)
15259 return performExtractVectorEltCombine(
N, DCI);
15261 return performInsertVectorEltCombine(
N, DCI);
15263 return performFPRoundCombine(
N, DCI);
15265 if (
SDValue Widened = widenLoad(cast<LoadSDNode>(
N), DCI))
15271 if (
MemSDNode *MemNode = dyn_cast<MemSDNode>(
N))
15272 return performMemSDNodeCombine(MemNode, DCI);
15303 unsigned Opcode =
Node->getMachineOpcode();
15307 if (D16Idx >= 0 &&
Node->getConstantOperandVal(D16Idx))
15312 unsigned DmaskIdx =
15314 unsigned OldDmask =
Node->getConstantOperandVal(DmaskIdx);
15315 unsigned NewDmask = 0;
15318 bool UsesTFC = ((int(TFEIdx) >= 0 &&
Node->getConstantOperandVal(TFEIdx)) ||
15319 (
int(LWEIdx) >= 0 &&
Node->getConstantOperandVal(LWEIdx)))
15322 unsigned TFCLane = 0;
15323 bool HasChain =
Node->getNumValues() > 1;
15325 if (OldDmask == 0) {
15333 TFCLane = OldBitsSet;
15340 if (
Use.getResNo() != 0)
15346 if (!
User->isMachineOpcode() ||
15347 User->getMachineOpcode() != TargetOpcode::EXTRACT_SUBREG)
15359 if (UsesTFC && Lane == TFCLane) {
15364 for (
unsigned i = 0, Dmask = OldDmask; (i <= Lane) && (Dmask != 0); i++) {
15366 Dmask &= ~(1 << Comp);
15374 NewDmask |= 1 << Comp;
15379 bool NoChannels = !NewDmask;
15386 if (OldBitsSet == 1)
15392 if (NewDmask == OldDmask)
15401 unsigned NewChannels = BitsSet + UsesTFC;
15405 assert(NewOpcode != -1 &&
15406 NewOpcode !=
static_cast<int>(
Node->getMachineOpcode()) &&
15407 "failed to find equivalent MIMG op");
15415 MVT SVT =
Node->getValueType(0).getVectorElementType().getSimpleVT();
15417 MVT ResultVT = NewChannels == 1
15420 : NewChannels == 5 ? 8
15434 if (NewChannels == 1) {
15444 for (
unsigned i = 0,
Idx = AMDGPU::sub0; i < 5; ++i) {
15449 if (i || !NoChannels)
15454 if (NewUser !=
User) {
15464 Idx = AMDGPU::sub1;
15467 Idx = AMDGPU::sub2;
15470 Idx = AMDGPU::sub3;
15473 Idx = AMDGPU::sub4;
15484 Op =
Op.getOperand(0);
15486 return isa<FrameIndexSDNode>(
Op);
15496 RegisterSDNode *DestReg = cast<RegisterSDNode>(Node->getOperand(1));
15497 SDValue SrcVal = Node->getOperand(2);
15505 MRI.createVirtualRegister(&AMDGPU::VReg_1RegClass), MVT::i1);
15507 SDNode *Glued = Node->getGluedNode();
15509 Node->getOperand(0), SL, VReg, SrcVal,
15515 return ToResultReg.
getNode();
15520 for (
unsigned i = 0; i < Node->getNumOperands(); ++i) {
15528 Node->getOperand(i).getValueType(),
15529 Node->getOperand(i)),
15541 unsigned Opcode = Node->getMachineOpcode();
15543 if (
TII->isImage(Opcode) && !
TII->get(Opcode).mayStore() &&
15544 !
TII->isGather4(Opcode) &&
15546 return adjustWritemask(Node, DAG);
15549 if (Opcode == AMDGPU::INSERT_SUBREG || Opcode == AMDGPU::REG_SEQUENCE) {
15555 case AMDGPU::V_DIV_SCALE_F32_e64:
15556 case AMDGPU::V_DIV_SCALE_F64_e64: {
15560 SDValue Src0 = Node->getOperand(1);
15561 SDValue Src1 = Node->getOperand(3);
15562 SDValue Src2 = Node->getOperand(5);
15566 (Src0 == Src1 || Src0 == Src2))
15623 unsigned InitIdx = 0;
15625 if (
TII->isImage(
MI)) {
15633 unsigned TFEVal = TFE ? TFE->
getImm() : 0;
15634 unsigned LWEVal = LWE ? LWE->
getImm() : 0;
15635 unsigned D16Val = D16 ? D16->getImm() : 0;
15637 if (!TFEVal && !LWEVal)
15648 assert(MO_Dmask &&
"Expected dmask operand in instruction");
15650 unsigned dmask = MO_Dmask->
getImm();
15657 InitIdx = D16Val && Packed ? ((ActiveLanes + 1) >> 1) + 1 : ActiveLanes + 1;
15663 TRI.getRegSizeInBits(*
TII->getOpRegClass(
MI, DstIdx)) / 32;
15664 if (DstSize < InitIdx)
15667 InitIdx =
TRI.getRegSizeInBits(*
TII->getOpRegClass(
MI, DstIdx)) / 32;
15675 Register PrevDst =
MRI.cloneVirtualRegister(
MI.getOperand(DstIdx).getReg());
15676 unsigned NewDst = 0;
15685 for (; SizeLeft; SizeLeft--, CurrIdx++) {
15686 NewDst =
MRI.createVirtualRegister(
TII->getOpRegClass(
MI, DstIdx));
15706 MI.tieOperands(DstIdx,
MI.getNumOperands() - 1);
15719 if (
TII->isVOP3(
MI.getOpcode())) {
15721 TII->legalizeOperandsVOP3(
MRI,
MI);
15726 if (!
MI.getDesc().operands().empty()) {
15727 unsigned Opc =
MI.getOpcode();
15728 bool HasAGPRs =
Info->mayNeedAGPRs();
15736 if ((
I == Src2Idx) && (HasAGPRs))
15739 if (!
Op.isReg() || !
Op.getReg().isVirtual())
15741 auto *RC =
TRI->getRegClassForReg(
MRI,
Op.getReg());
15742 if (!
TRI->hasAGPRs(RC))
15744 auto *Src =
MRI.getUniqueVRegDef(
Op.getReg());
15745 if (!Src || !Src->isCopy() ||
15746 !
TRI->isSGPRReg(
MRI, Src->getOperand(1).getReg()))
15748 auto *NewRC =
TRI->getEquivalentVGPRClass(RC);
15752 MRI.setRegClass(
Op.getReg(), NewRC);
15755 if (
TII->isMAI(
MI)) {
15761 AMDGPU::OpName::scale_src0);
15762 if (Src0Idx != -1) {
15764 AMDGPU::OpName::scale_src1);
15765 if (
TII->usesConstantBus(
MRI,
MI, Src0Idx) &&
15766 TII->usesConstantBus(
MRI,
MI, Src1Idx))
15767 TII->legalizeOpWithMove(
MI, Src1Idx);
15775 if (
auto *Src2 =
TII->getNamedOperand(
MI, AMDGPU::OpName::src2)) {
15776 if (Src2->isReg() && Src2->getReg().isVirtual()) {
15777 auto *RC =
TRI->getRegClassForReg(
MRI, Src2->getReg());
15778 if (
TRI->isVectorSuperClass(RC)) {
15779 auto *NewRC =
TRI->getEquivalentAGPRClass(RC);
15780 MRI.setRegClass(Src2->getReg(), NewRC);
15781 if (Src2->isTied())
15782 MRI.setRegClass(
MI.getOperand(0).getReg(), NewRC);
15791 if (
TII->isImage(
MI))
15792 TII->enforceOperandRCAlignment(
MI, AMDGPU::OpName::vaddr);
15866std::pair<unsigned, const TargetRegisterClass *>
15873 if (Constraint.
size() == 1) {
15875 switch (Constraint[0]) {
15882 RC = &AMDGPU::SReg_32RegClass;
15885 RC = &AMDGPU::SGPR_64RegClass;
15890 return std::pair(0U,
nullptr);
15897 RC = &AMDGPU::VGPR_32RegClass;
15902 return std::pair(0U,
nullptr);
15911 RC = &AMDGPU::AGPR_32RegClass;
15916 return std::pair(0U,
nullptr);
15925 return std::pair(0U, RC);
15930 if (
RegName.consume_front(
"v")) {
15931 RC = &AMDGPU::VGPR_32RegClass;
15932 }
else if (
RegName.consume_front(
"s")) {
15933 RC = &AMDGPU::SGPR_32RegClass;
15934 }
else if (
RegName.consume_front(
"a")) {
15935 RC = &AMDGPU::AGPR_32RegClass;
15940 if (
RegName.consume_front(
"[")) {
15951 return std::pair(0U,
nullptr);
15954 RC =
TRI->getVGPRClassForBitWidth(Width);
15956 RC =
TRI->getSGPRClassForBitWidth(Width);
15958 RC =
TRI->getAGPRClassForBitWidth(Width);
15960 Reg =
TRI->getMatchingSuperReg(Reg, AMDGPU::sub0, RC);
15965 return std::pair(0U,
nullptr);
15967 return std::pair(Reg, RC);
15973 return std::pair(0U,
nullptr);
15975 if (!
Failed && Idx < RC->getNumRegs())
15983 Ret.second =
TRI->getPhysRegBaseClass(Ret.first);
15989 if (Constraint.
size() == 1) {
15990 switch (Constraint[0]) {
16000 }
else if (Constraint ==
"DA" || Constraint ==
"DB") {
16008 if (Constraint.
size() == 1) {
16009 switch (Constraint[0]) {
16026 Val = Val & maskTrailingOnes<uint64_t>(
Size);
16033 std::vector<SDValue> &Ops,
16048 unsigned Size =
Op.getScalarValueSizeInBits();
16056 Val =
C->getSExtValue();
16060 Val =
C->getValueAPF().bitcastToAPInt().getSExtValue();
16066 if (
Op.getOperand(0).isUndef() ||
Op.getOperand(1).isUndef())
16069 Val =
C->getSExtValue();
16073 Val =
C->getValueAPF().bitcastToAPInt().getSExtValue();
16083 if (Constraint.
size() == 1) {
16084 switch (Constraint[0]) {
16088 return isInt<16>(Val);
16092 return isInt<32>(Val);
16099 }
else if (Constraint.
size() == 2) {
16100 if (Constraint ==
"DA") {
16101 int64_t HiBits =
static_cast<int32_t
>(Val >> 32);
16102 int64_t LoBits =
static_cast<int32_t
>(Val);
16106 if (Constraint ==
"DB") {
16114 unsigned MaxSize)
const {
16115 unsigned Size = std::min<unsigned>(
Op.getScalarValueSizeInBits(), MaxSize);
16118 MVT VT =
Op.getSimpleValueType();
16143 switch (UnalignedClassID) {
16144 case AMDGPU::VReg_64RegClassID:
16145 return AMDGPU::VReg_64_Align2RegClassID;
16146 case AMDGPU::VReg_96RegClassID:
16147 return AMDGPU::VReg_96_Align2RegClassID;
16148 case AMDGPU::VReg_128RegClassID:
16149 return AMDGPU::VReg_128_Align2RegClassID;
16150 case AMDGPU::VReg_160RegClassID:
16151 return AMDGPU::VReg_160_Align2RegClassID;
16152 case AMDGPU::VReg_192RegClassID:
16153 return AMDGPU::VReg_192_Align2RegClassID;
16154 case AMDGPU::VReg_224RegClassID:
16155 return AMDGPU::VReg_224_Align2RegClassID;
16156 case AMDGPU::VReg_256RegClassID:
16157 return AMDGPU::VReg_256_Align2RegClassID;
16158 case AMDGPU::VReg_288RegClassID:
16159 return AMDGPU::VReg_288_Align2RegClassID;
16160 case AMDGPU::VReg_320RegClassID:
16161 return AMDGPU::VReg_320_Align2RegClassID;
16162 case AMDGPU::VReg_352RegClassID:
16163 return AMDGPU::VReg_352_Align2RegClassID;
16164 case AMDGPU::VReg_384RegClassID:
16165 return AMDGPU::VReg_384_Align2RegClassID;
16166 case AMDGPU::VReg_512RegClassID:
16167 return AMDGPU::VReg_512_Align2RegClassID;
16168 case AMDGPU::VReg_1024RegClassID:
16169 return AMDGPU::VReg_1024_Align2RegClassID;
16170 case AMDGPU::AReg_64RegClassID:
16171 return AMDGPU::AReg_64_Align2RegClassID;
16172 case AMDGPU::AReg_96RegClassID:
16173 return AMDGPU::AReg_96_Align2RegClassID;
16174 case AMDGPU::AReg_128RegClassID:
16175 return AMDGPU::AReg_128_Align2RegClassID;
16176 case AMDGPU::AReg_160RegClassID:
16177 return AMDGPU::AReg_160_Align2RegClassID;
16178 case AMDGPU::AReg_192RegClassID:
16179 return AMDGPU::AReg_192_Align2RegClassID;
16180 case AMDGPU::AReg_256RegClassID:
16181 return AMDGPU::AReg_256_Align2RegClassID;
16182 case AMDGPU::AReg_512RegClassID:
16183 return AMDGPU::AReg_512_Align2RegClassID;
16184 case AMDGPU::AReg_1024RegClassID:
16185 return AMDGPU::AReg_1024_Align2RegClassID;
16201 if (
Info->isEntryFunction()) {
16208 unsigned MaxNumSGPRs = ST.getMaxNumSGPRs(MF);
16210 ? AMDGPU::SGPR_32RegClass.getRegister(MaxNumSGPRs - 1)
16211 :
TRI->getAlignedHighSGPRForRC(MF, 2,
16212 &AMDGPU::SGPR_64RegClass);
16213 Info->setSGPRForEXECCopy(SReg);
16216 Info->getStackPtrOffsetReg()));
16217 if (
Info->getStackPtrOffsetReg() != AMDGPU::SP_REG)
16218 MRI.replaceRegWith(AMDGPU::SP_REG,
Info->getStackPtrOffsetReg());
16222 if (
Info->getScratchRSrcReg() != AMDGPU::PRIVATE_RSRC_REG)
16223 MRI.replaceRegWith(AMDGPU::PRIVATE_RSRC_REG,
Info->getScratchRSrcReg());
16225 if (
Info->getFrameOffsetReg() != AMDGPU::FP_REG)
16226 MRI.replaceRegWith(AMDGPU::FP_REG,
Info->getFrameOffsetReg());
16228 Info->limitOccupancy(MF);
16230 if (ST.isWave32() && !MF.
empty()) {
16231 for (
auto &
MBB : MF) {
16232 for (
auto &
MI :
MBB) {
16233 TII->fixImplicitOperands(
MI);
16243 if (ST.needsAlignedVGPRs()) {
16244 for (
unsigned I = 0, E =
MRI.getNumVirtRegs();
I != E; ++
I) {
16250 if (NewClassID != -1)
16251 MRI.setRegClass(Reg,
TRI->getRegClass(NewClassID));
16260 const APInt &DemandedElts,
16262 unsigned Depth)
const {
16264 unsigned Opc =
Op.getOpcode();
16267 unsigned IID =
Op.getConstantOperandVal(0);
16269 case Intrinsic::amdgcn_mbcnt_lo:
16270 case Intrinsic::amdgcn_mbcnt_hi: {
16276 IID == Intrinsic::amdgcn_mbcnt_lo ? ST.getWavefrontSizeLog2() : 5);
16286 Op, Known, DemandedElts, DAG,
Depth);
16301 unsigned MaxValue =
16310 switch (
MI->getOpcode()) {
16311 case AMDGPU::G_INTRINSIC:
16312 case AMDGPU::G_INTRINSIC_CONVERGENT: {
16315 case Intrinsic::amdgcn_workitem_id_x:
16318 case Intrinsic::amdgcn_workitem_id_y:
16321 case Intrinsic::amdgcn_workitem_id_z:
16324 case Intrinsic::amdgcn_mbcnt_lo:
16325 case Intrinsic::amdgcn_mbcnt_hi: {
16337 case Intrinsic::amdgcn_groupstaticsize: {
16348 case AMDGPU::G_AMDGPU_BUFFER_LOAD_UBYTE:
16351 case AMDGPU::G_AMDGPU_BUFFER_LOAD_USHORT:
16354 case AMDGPU::G_AMDGPU_SMED3:
16355 case AMDGPU::G_AMDGPU_UMED3: {
16356 auto [Dst, Src0, Src1, Src2] =
MI->getFirst4Regs();
16383 unsigned Depth)
const {
16385 if (
auto *GI = dyn_cast<GIntrinsic>(
MI)) {
16391 if (
MaybeAlign RetAlign = Attrs.getRetAlignment())
16418 if (Header->getAlignment() != PrefAlign)
16419 return Header->getAlignment();
16421 unsigned LoopSize = 0;
16429 LoopSize +=
TII->getInstSizeInBytes(
MI);
16430 if (LoopSize > 192)
16435 if (LoopSize <= 64)
16438 if (LoopSize <= 128)
16439 return CacheLineAlign;
16445 auto I = Exit->getFirstNonDebugInstr();
16446 if (
I != Exit->end() &&
I->getOpcode() == AMDGPU::S_INST_PREFETCH)
16447 return CacheLineAlign;
16456 if (PreTerm == Pre->
begin() ||
16457 std::prev(PreTerm)->getOpcode() != AMDGPU::S_INST_PREFETCH)
16461 auto ExitHead = Exit->getFirstNonDebugInstr();
16462 if (ExitHead == Exit->end() ||
16463 ExitHead->getOpcode() != AMDGPU::S_INST_PREFETCH)
16468 return CacheLineAlign;
16476 N =
N->getOperand(0).getNode();
16486 switch (
N->getOpcode()) {
16494 if (Reg.isPhysical() ||
MRI.isLiveIn(Reg))
16495 return !
TRI->isSGPRReg(
MRI, Reg);
16497 if (
const Value *V = FLI->getValueFromVirtualReg(R->getReg()))
16501 return !
TRI->isSGPRReg(
MRI, Reg);
16505 unsigned AS = L->getAddressSpace();
16536 if (
auto *
A = dyn_cast<AtomicSDNode>(
N)) {
16538 return A->readMem() &&
A->writeMem();
16573 unsigned Depth)
const {
16578 if (
Info->getMode().DX10Clamp)
16590 if (RMW->
hasMetadata(
"amdgpu.ignore.denormal.mode"))
16610 <<
"Hardware instruction generated for atomic "
16612 <<
" operation at memory scope " << MemScope;
16616 if (
auto *VT = dyn_cast<FixedVectorType>(Ty)) {
16617 Type *EltTy = VT->getElementType();
16618 return VT->getNumElements() == 2 &&
16637 if (
auto *
IT = dyn_cast<IntegerType>(Ty)) {
16638 unsigned BW =
IT->getBitWidth();
16639 return BW == 32 || BW == 64;
16651 if (
PointerType *PT = dyn_cast<PointerType>(Ty)) {
16653 unsigned BW =
DL.getPointerSizeInBits(PT->getAddressSpace());
16654 return BW == 32 || BW == 64;
16661 return VT->getNumElements() == 2 &&
16662 VT->getElementType()->getPrimitiveSizeInBits() == 16;
16672 bool HasSystemScope) {
16679 if (HasSystemScope) {
16686 return RMW->
hasMetadata(
"amdgpu.no.fine.grained.memory");
16699 const MDNode *NoaliasAddrSpaceMD =
16700 I->getMetadata(LLVMContext::MD_noalias_addrspace);
16701 if (!NoaliasAddrSpaceMD)
16704 for (
unsigned I = 0, E = NoaliasAddrSpaceMD->
getNumOperands() / 2;
I != E;
16706 auto *
Low = mdconst::extract<ConstantInt>(
16709 auto *
High = mdconst::extract<ConstantInt>(
16731 DL.getTypeSizeInBits(RMW->
getType()) == 64 &&
16744 bool HasSystemScope =
16931 if (HasSystemScope)
16983 if (RC == &AMDGPU::VReg_1RegClass && !isDivergent)
16984 return Subtarget->
isWave64() ? &AMDGPU::SReg_64RegClass
16985 : &AMDGPU::SReg_32RegClass;
16986 if (!
TRI->isSGPRClass(RC) && !isDivergent)
16987 return TRI->getEquivalentSGPRClass(RC);
16988 if (
TRI->isSGPRClass(RC) && isDivergent)
16989 return TRI->getEquivalentVGPRClass(RC);
17001 unsigned WaveSize) {
17006 if (!
IT ||
IT->getBitWidth() != WaveSize)
17009 if (!isa<Instruction>(V))
17011 if (!Visited.
insert(V).second)
17013 bool Result =
false;
17014 for (
const auto *U : V->users()) {
17015 if (
const IntrinsicInst *Intrinsic = dyn_cast<IntrinsicInst>(U)) {
17016 if (V == U->getOperand(1)) {
17017 switch (Intrinsic->getIntrinsicID()) {
17021 case Intrinsic::amdgcn_if_break:
17022 case Intrinsic::amdgcn_if:
17023 case Intrinsic::amdgcn_else:
17028 if (V == U->getOperand(0)) {
17029 switch (Intrinsic->getIntrinsicID()) {
17033 case Intrinsic::amdgcn_end_cf:
17034 case Intrinsic::amdgcn_loop:
17040 Result =
hasCFUser(U, Visited, WaveSize);
17049 const Value *V)
const {
17050 if (
const CallInst *CI = dyn_cast<CallInst>(V)) {
17051 if (CI->isInlineAsm()) {
17060 for (
auto &TC : TargetConstraints) {
17102 return MRI.hasOneNonDBGUse(N0);
17109 if (
I.getMetadata(
"amdgpu.noclobber"))
17111 if (
I.getMetadata(
"amdgpu.last.use"))
17121 if (!Def->isMachineOpcode())
17131 if (
II.isCompare() &&
II.hasImplicitDefOfPhysReg(AMDGPU::SCC)) {
17132 PhysReg = AMDGPU::SCC;
17134 TRI->getMinimalPhysRegClass(PhysReg, Def->getSimpleValueType(ResNo));
17143 if (!
I->hasOneUse())
17149 switch (
I->getOpcode()) {
17150 case Instruction::FMul: {
17151 if (
User->getOpcode() != Instruction::FSub &&
17152 User->getOpcode() != Instruction::FAdd)
17157 return ((!
I->hasAllowContract() || !
User->hasAllowContract()) &&
17216 auto *RMW = dyn_cast<AtomicRMWInst>(AI);
17227 Alignment = RMW->getAlign();
17242 RMW->getType()->isFloatTy();
17245 bool ReturnValueIsUsed = !AI->
use_empty();
17254 if (FullFlatEmulation) {
17265 std::prev(BB->
end())->eraseFromParent();
17268 Value *LoadedShared =
nullptr;
17269 if (FullFlatEmulation) {
17271 Intrinsic::amdgcn_is_shared, {}, {
Addr},
nullptr,
"is.shared");
17272 Builder.
CreateCondBr(IsShared, SharedBB, CheckPrivateBB);
17280 LoadedShared = Clone;
17287 Intrinsic::amdgcn_is_private, {}, {
Addr},
nullptr,
"is.private");
17295 Value *LoadedPrivate;
17298 RMW->getType(), CastToPrivate, RMW->getAlign(),
"loaded.private");
17301 LoadedPrivate, RMW->getValOperand());
17305 auto [ResultLoad, Equal] =
17320 if (FullFlatEmulation) {
17330 if (!FullFlatEmulation) {
17335 MDNode *RangeNotPrivate =
17338 LoadedGlobal->
setMetadata(LLVMContext::MD_noalias_addrspace,
17346 if (ReturnValueIsUsed) {
17349 if (FullFlatEmulation)
17364 if (
const auto *ConstVal = dyn_cast<Constant>(AI->
getValOperand());
17365 ConstVal && ConstVal->isNullValue()) {
static bool isMul(MachineInstr *MI)
unsigned const MachineRegisterInfo * MRI
static unsigned getIntrinsicID(const SDNode *N)
static bool canGuaranteeTCO(CallingConv::ID CC, bool GuaranteeTailCalls)
Return true if the calling convention is one that we can guarantee TCO for.
static bool mayTailCallThisCC(CallingConv::ID CC)
Return true if we might ever do TCO for calls with this calling convention.
static constexpr std::pair< ImplicitArgumentMask, StringLiteral > ImplicitAttrs[]
Contains the definition of a TargetInstrInfo class that is common to all AMD GPUs.
static bool parseTexFail(uint64_t TexFailCtrl, bool &TFE, bool &LWE, bool &IsTexFail)
static void packImage16bitOpsToDwords(MachineIRBuilder &B, MachineInstr &MI, SmallVectorImpl< Register > &PackedAddrs, unsigned ArgOffset, const AMDGPU::ImageDimIntrinsicInfo *Intr, bool IsA16, bool IsG16)
Turn a set of s16 typed registers in AddrRegs into a dword sized vector with s16 typed elements.
static bool isKnownNonNull(Register Val, MachineRegisterInfo &MRI, const AMDGPUTargetMachine &TM, unsigned AddrSpace)
Return true if the value is a known valid address, such that a null check is not necessary.
Provides AMDGPU specific target descriptions.
The AMDGPU TargetMachine interface definition for hw codegen targets.
This file implements a class to represent arbitrary precision integral constant values and operations...
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
static cl::opt< ITMode > IT(cl::desc("IT block support"), cl::Hidden, cl::init(DefaultIT), cl::values(clEnumValN(DefaultIT, "arm-default-it", "Generate any type of IT block"), clEnumValN(RestrictedIT, "arm-restrict-it", "Disallow complex IT blocks")))
Function Alias Analysis Results
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
Analysis containing CSE Info
#define LLVM_ATTRIBUTE_UNUSED
static std::optional< SDByteProvider > calculateByteProvider(SDValue Op, unsigned Index, unsigned Depth, std::optional< uint64_t > VectorIndex, unsigned StartingIndex=0)
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
static bool isSigned(unsigned int Opcode)
Utilities for dealing with flags related to floating point properties and mode controls.
AMD GCN specific subclass of TargetSubtarget.
Provides analysis for querying information about KnownBits during GISel passes.
Declares convenience wrapper classes for interpreting MachineInstr instances as specific generic oper...
const HexagonInstrInfo * TII
iv Induction Variable Users
Contains matchers for matching SSA Machine Instructions.
mir Rename Register Operands
static bool isUndef(const MachineInstr &MI)
unsigned const TargetRegisterInfo * TRI
static unsigned getAddressSpace(const Value *V, unsigned MaxLookup)
uint64_t IntrinsicInst * II
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
static constexpr Register SPReg
const SmallVectorImpl< MachineOperand > & Cond
static void r0(uint32_t &A, uint32_t &B, uint32_t &C, uint32_t &D, uint32_t &E, int I, uint32_t *Buf)
static void r3(uint32_t &A, uint32_t &B, uint32_t &C, uint32_t &D, uint32_t &E, int I, uint32_t *Buf)
static void r2(uint32_t &A, uint32_t &B, uint32_t &C, uint32_t &D, uint32_t &E, int I, uint32_t *Buf)
static void r1(uint32_t &A, uint32_t &B, uint32_t &C, uint32_t &D, uint32_t &E, int I, uint32_t *Buf)
#define FP_DENORM_FLUSH_NONE
#define FP_DENORM_FLUSH_IN_FLUSH_OUT
static void reservePrivateMemoryRegs(const TargetMachine &TM, MachineFunction &MF, const SIRegisterInfo &TRI, SIMachineFunctionInfo &Info)
static SDValue adjustLoadValueTypeImpl(SDValue Result, EVT LoadVT, const SDLoc &DL, SelectionDAG &DAG, bool Unpacked)
static MachineBasicBlock * emitIndirectSrc(MachineInstr &MI, MachineBasicBlock &MBB, const GCNSubtarget &ST)
static bool denormalModeIsFlushAllF64F16(const MachineFunction &MF)
static bool isAtomicRMWLegalIntTy(Type *Ty)
static bool flatInstrMayAccessPrivate(const Instruction *I)
Return if a flat address space atomicrmw can access private memory.
static std::pair< unsigned, int > computeIndirectRegAndOffset(const SIRegisterInfo &TRI, const TargetRegisterClass *SuperRC, unsigned VecReg, int Offset)
static bool denormalModeIsFlushAllF32(const MachineFunction &MF)
static bool addresses16Bits(int Mask)
static bool isClampZeroToOne(SDValue A, SDValue B)
static bool supportsMin3Max3(const GCNSubtarget &Subtarget, unsigned Opc, EVT VT)
static unsigned findFirstFreeSGPR(CCState &CCInfo)
static uint32_t getPermuteMask(SDValue V)
static SDValue lowerLaneOp(const SITargetLowering &TLI, SDNode *N, SelectionDAG &DAG)
static int getAlignedAGPRClassID(unsigned UnalignedClassID)
static void processPSInputArgs(SmallVectorImpl< ISD::InputArg > &Splits, CallingConv::ID CallConv, ArrayRef< ISD::InputArg > Ins, BitVector &Skipped, FunctionType *FType, SIMachineFunctionInfo *Info)
static SDValue selectSOffset(SDValue SOffset, SelectionDAG &DAG, const GCNSubtarget *Subtarget)
static SDValue getLoadExtOrTrunc(SelectionDAG &DAG, ISD::LoadExtType ExtType, SDValue Op, const SDLoc &SL, EVT VT)
static bool globalMemoryFPAtomicIsLegal(const GCNSubtarget &Subtarget, const AtomicRMWInst *RMW, bool HasSystemScope)
static void fixMasks(SmallVectorImpl< DotSrc > &Srcs, unsigned ChainLength)
static TargetLowering::AtomicExpansionKind atomicSupportedIfLegalIntType(const AtomicRMWInst *RMW)
static SDValue strictFPExtFromF16(SelectionDAG &DAG, SDValue Src)
Return the source of an fp_extend from f16 to f32, or a converted FP constant.
static bool isAtomicRMWLegalXChgTy(const AtomicRMWInst *RMW)
static bool bitOpWithConstantIsReducible(unsigned Opc, uint32_t Val)
static bool elementPairIsOddToEven(ArrayRef< int > Mask, int Elt)
static cl::opt< bool > DisableLoopAlignment("amdgpu-disable-loop-alignment", cl::desc("Do not align and prefetch loops"), cl::init(false))
static SDValue getDWordFromOffset(SelectionDAG &DAG, SDLoc SL, SDValue Src, unsigned DWordOffset)
static MachineBasicBlock::iterator loadM0FromVGPR(const SIInstrInfo *TII, MachineBasicBlock &MBB, MachineInstr &MI, unsigned InitResultReg, unsigned PhiReg, int Offset, bool UseGPRIdxMode, Register &SGPRIdxReg)
static bool isImmConstraint(StringRef Constraint)
static SDValue padEltsToUndef(SelectionDAG &DAG, const SDLoc &DL, EVT CastVT, SDValue Src, int ExtraElts)
static SDValue lowerICMPIntrinsic(const SITargetLowering &TLI, SDNode *N, SelectionDAG &DAG)
static bool hasCFUser(const Value *V, SmallPtrSet< const Value *, 16 > &Visited, unsigned WaveSize)
static OptimizationRemark emitAtomicRMWLegalRemark(const AtomicRMWInst *RMW)
static unsigned SubIdx2Lane(unsigned Idx)
Helper function for adjustWritemask.
static bool addressMayBeAccessedAsPrivate(const MachineMemOperand *MMO, const SIMachineFunctionInfo &Info)
static MachineBasicBlock * lowerWaveReduce(MachineInstr &MI, MachineBasicBlock &BB, const GCNSubtarget &ST, unsigned Opc)
static bool elementPairIsContiguous(ArrayRef< int > Mask, int Elt)
static bool isV2BF16(Type *Ty)
static ArgDescriptor allocateSGPR32InputImpl(CCState &CCInfo, const TargetRegisterClass *RC, unsigned NumArgRegs)
static SDValue getMad64_32(SelectionDAG &DAG, const SDLoc &SL, EVT VT, SDValue N0, SDValue N1, SDValue N2, bool Signed)
static SDValue resolveSources(SelectionDAG &DAG, SDLoc SL, SmallVectorImpl< DotSrc > &Srcs, bool IsSigned, bool IsAny)
static bool hasNon16BitAccesses(uint64_t PermMask, SDValue &Op, SDValue &OtherOp)
static void placeSources(ByteProvider< SDValue > &Src0, ByteProvider< SDValue > &Src1, SmallVectorImpl< DotSrc > &Src0s, SmallVectorImpl< DotSrc > &Src1s, int Step)
static EVT memVTFromLoadIntrReturn(const SITargetLowering &TLI, const DataLayout &DL, Type *Ty, unsigned MaxNumLanes)
static MachineBasicBlock::iterator emitLoadM0FromVGPRLoop(const SIInstrInfo *TII, MachineRegisterInfo &MRI, MachineBasicBlock &OrigBB, MachineBasicBlock &LoopBB, const DebugLoc &DL, const MachineOperand &Idx, unsigned InitReg, unsigned ResultReg, unsigned PhiReg, unsigned InitSaveExecReg, int Offset, bool UseGPRIdxMode, Register &SGPRIdxReg)
static SDValue matchPERM(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static bool isFrameIndexOp(SDValue Op)
static ConstantFPSDNode * getSplatConstantFP(SDValue Op)
static void allocateSGPR32Input(CCState &CCInfo, ArgDescriptor &Arg)
static bool isExtendedFrom16Bits(SDValue &Operand)
static std::optional< bool > checkDot4MulSignedness(const SDValue &N, ByteProvider< SDValue > &Src0, ByteProvider< SDValue > &Src1, const SDValue &S0Op, const SDValue &S1Op, const SelectionDAG &DAG)
static bool vectorEltWillFoldAway(SDValue Op)
static SDValue getSPDenormModeValue(uint32_t SPDenormMode, SelectionDAG &DAG, const SIMachineFunctionInfo *Info, const GCNSubtarget *ST)
static uint32_t getConstantPermuteMask(uint32_t C)
static MachineBasicBlock * emitIndirectDst(MachineInstr &MI, MachineBasicBlock &MBB, const GCNSubtarget &ST)
static void setM0ToIndexFromSGPR(const SIInstrInfo *TII, MachineRegisterInfo &MRI, MachineInstr &MI, int Offset)
static ArgDescriptor allocateVGPR32Input(CCState &CCInfo, unsigned Mask=~0u, ArgDescriptor Arg=ArgDescriptor())
static std::pair< MachineBasicBlock *, MachineBasicBlock * > splitBlockForLoop(MachineInstr &MI, MachineBasicBlock &MBB, bool InstInLoop)
static unsigned getBasePtrIndex(const MemSDNode *N)
MemSDNode::getBasePtr() does not work for intrinsics, which needs to offset by the chain and intrinsi...
static void knownBitsForWorkitemID(const GCNSubtarget &ST, GISelKnownBits &KB, KnownBits &Known, unsigned Dim)
static LLVM_ATTRIBUTE_UNUSED bool isCopyFromRegOfInlineAsm(const SDNode *N)
static void allocateFixedSGPRInputImpl(CCState &CCInfo, const TargetRegisterClass *RC, MCRegister Reg)
static SDValue constructRetValue(SelectionDAG &DAG, MachineSDNode *Result, ArrayRef< EVT > ResultTypes, bool IsTexFail, bool Unpacked, bool IsD16, int DMaskPop, int NumVDataDwords, bool IsAtomicPacked16Bit, const SDLoc &DL)
static std::optional< ByteProvider< SDValue > > handleMulOperand(const SDValue &MulOperand)
static SDValue lowerFCMPIntrinsic(const SITargetLowering &TLI, SDNode *N, SelectionDAG &DAG)
static Register getIndirectSGPRIdx(const SIInstrInfo *TII, MachineRegisterInfo &MRI, MachineInstr &MI, int Offset)
static SDValue emitNonHSAIntrinsicError(SelectionDAG &DAG, const SDLoc &DL, EVT VT)
static EVT memVTFromLoadIntrData(const SITargetLowering &TLI, const DataLayout &DL, Type *Ty, unsigned MaxNumLanes)
static unsigned minMaxOpcToMin3Max3Opc(unsigned Opc)
static unsigned getExtOpcodeForPromotedOp(SDValue Op)
static SDValue lowerBALLOTIntrinsic(const SITargetLowering &TLI, SDNode *N, SelectionDAG &DAG)
static SDValue buildSMovImm32(SelectionDAG &DAG, const SDLoc &DL, uint64_t Val)
static SDValue tryFoldMADwithSRL(SelectionDAG &DAG, const SDLoc &SL, SDValue MulLHS, SDValue MulRHS, SDValue AddRHS)
static SDValue getBuildDwordsVector(SelectionDAG &DAG, SDLoc DL, ArrayRef< SDValue > Elts)
static SDNode * findUser(SDValue Value, unsigned Opcode)
Helper function for LowerBRCOND.
static unsigned addPermMasks(unsigned First, unsigned Second)
static uint64_t clearUnusedBits(uint64_t Val, unsigned Size)
static SDValue getFPTernOp(SelectionDAG &DAG, unsigned Opcode, const SDLoc &SL, EVT VT, SDValue A, SDValue B, SDValue C, SDValue GlueChain, SDNodeFlags Flags)
static bool isV2F16OrV2BF16(Type *Ty)
static bool atomicIgnoresDenormalModeOrFPModeIsFTZ(const AtomicRMWInst *RMW)
static SDValue emitRemovedIntrinsicError(SelectionDAG &DAG, const SDLoc &DL, EVT VT)
static SDValue getFPBinOp(SelectionDAG &DAG, unsigned Opcode, const SDLoc &SL, EVT VT, SDValue A, SDValue B, SDValue GlueChain, SDNodeFlags Flags)
static SDValue buildPCRelGlobalAddress(SelectionDAG &DAG, const GlobalValue *GV, const SDLoc &DL, int64_t Offset, EVT PtrVT, unsigned GAFlags=SIInstrInfo::MO_NONE)
static cl::opt< bool > UseDivergentRegisterIndexing("amdgpu-use-divergent-register-indexing", cl::Hidden, cl::desc("Use indirect register addressing for divergent indexes"), cl::init(false))
static const std::optional< ByteProvider< SDValue > > calculateSrcByte(const SDValue Op, uint64_t DestByte, uint64_t SrcIndex=0, unsigned Depth=0)
static bool isV2F16(Type *Ty)
static void allocateSGPR64Input(CCState &CCInfo, ArgDescriptor &Arg)
SI DAG Lowering interface definition.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
Interface definition for SIRegisterInfo.
static bool contains(SmallPtrSetImpl< ConstantExpr * > &Cache, ConstantExpr *Expr, Constant *C)
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
static constexpr int Concat[]
static const AMDGPUFunctionArgInfo FixedABIFunctionInfo
void setFuncArgInfo(const Function &F, const AMDGPUFunctionArgInfo &ArgInfo)
static bool isUniformMMO(const MachineMemOperand *MMO)
static std::optional< uint32_t > getLDSKernelIdMetadata(const Function &F)
uint32_t getLDSSize() const
void setUsesDynamicLDS(bool DynLDS)
void setDynLDSAlign(const Function &F, const GlobalVariable &GV)
bool isEntryFunction() const
Align getAlignmentForImplicitArgPtr() const
bool hasMadMacF32Insts() const
bool hasCvtPkF16F32Inst() const
bool useRealTrue16Insts() const
Return true if real (non-fake) variants of True16 instructions using 16-bit registers should be code-...
bool hasBF16ConversionInsts() const
unsigned getMaxWorkitemID(const Function &Kernel, unsigned Dimension) const
Return the maximum workitem ID value in the function, for the given (0, 1, 2) dimension.
bool hasMadMixInsts() const
unsigned getWavefrontSizeLog2() const
bool has16BitInsts() const
bool isAmdHsaOrMesa(const Function &F) const
bool hasFastFMAF32() const
bool hasTrigReducedRange() const
unsigned getExplicitKernelArgOffset() const
Returns the offset in bytes from the start of the input buffer of the first explicit kernel argument.
unsigned getWavefrontSize() const
bool hasInv2PiInlineImm() const
bool hasVOP3PInsts() const
static unsigned numBitsSigned(SDValue Op, SelectionDAG &DAG)
SDValue SplitVectorLoad(SDValue Op, SelectionDAG &DAG) const
Split a vector load into 2 loads of half the vector.
void analyzeFormalArgumentsCompute(CCState &State, const SmallVectorImpl< ISD::InputArg > &Ins) const
The SelectionDAGBuilder will automatically promote function arguments with illegal types.
SDValue storeStackInputValue(SelectionDAG &DAG, const SDLoc &SL, SDValue Chain, SDValue ArgVal, int64_t Offset) const
void computeKnownBitsForTargetNode(const SDValue Op, KnownBits &Known, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth=0) const override
Determine which of the bits specified in Mask are known to be either zero or one and return them in t...
SDValue splitBinaryBitConstantOpImpl(DAGCombinerInfo &DCI, const SDLoc &SL, unsigned Opc, SDValue LHS, uint32_t ValLo, uint32_t ValHi) const
Split the 64-bit value LHS into two 32-bit components, and perform the binary operation Opc to it wit...
SDValue lowerUnhandledCall(CallLoweringInfo &CLI, SmallVectorImpl< SDValue > &InVals, StringRef Reason) const
SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override
This callback is invoked for operations that are unsupported by the target, which are registered to u...
SDValue addTokenForArgument(SDValue Chain, SelectionDAG &DAG, MachineFrameInfo &MFI, int ClobberedFI) const
static bool needsDenormHandlingF32(const SelectionDAG &DAG, SDValue Src, SDNodeFlags Flags)
uint32_t getImplicitParameterOffset(const MachineFunction &MF, const ImplicitParameter Param) const
Helper function that returns the byte offset of the given type of implicit parameter.
SDValue LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG) const
virtual SDValue LowerGlobalAddress(AMDGPUMachineFunction *MFI, SDValue Op, SelectionDAG &DAG) const
SDValue loadInputValue(SelectionDAG &DAG, const TargetRegisterClass *RC, EVT VT, const SDLoc &SL, const ArgDescriptor &Arg) const
static EVT getEquivalentMemType(LLVMContext &Context, EVT VT)
SDValue CreateLiveInRegister(SelectionDAG &DAG, const TargetRegisterClass *RC, Register Reg, EVT VT, const SDLoc &SL, bool RawReg=false) const
Helper function that adds Reg to the LiveIn list of the DAG's MachineFunction.
SDValue SplitVectorStore(SDValue Op, SelectionDAG &DAG) const
Split a vector store into 2 stores of half the vector.
std::pair< SDValue, SDValue > split64BitValue(SDValue Op, SelectionDAG &DAG) const
Return 64-bit value Op as two 32-bit integers.
static CCAssignFn * CCAssignFnForReturn(CallingConv::ID CC, bool IsVarArg)
static CCAssignFn * CCAssignFnForCall(CallingConv::ID CC, bool IsVarArg)
Selects the correct CCAssignFn for a given CallingConvention value.
static bool allUsesHaveSourceMods(const SDNode *N, unsigned CostThreshold=4)
bool isKnownNeverNaNForTargetNode(SDValue Op, const SelectionDAG &DAG, bool SNaN=false, unsigned Depth=0) const override
If SNaN is false,.
static unsigned numBitsUnsigned(SDValue Op, SelectionDAG &DAG)
static bool allowApproxFunc(const SelectionDAG &DAG, SDNodeFlags Flags)
SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool isVarArg, const SmallVectorImpl< ISD::OutputArg > &Outs, const SmallVectorImpl< SDValue > &OutVals, const SDLoc &DL, SelectionDAG &DAG) const override
This hook must be implemented to lower outgoing return values, described by the Outs array,...
void ReplaceNodeResults(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG) const override
This callback is invoked when a node result type is illegal for the target, and the operation was reg...
SDValue performRcpCombine(SDNode *N, DAGCombinerInfo &DCI) const
static bool shouldFoldFNegIntoSrc(SDNode *FNeg, SDValue FNegSrc)
bool isNarrowingProfitable(SDNode *N, EVT SrcVT, EVT DestVT) const override
Return true if it's profitable to narrow operations of type SrcVT to DestVT.
SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override
This method will be invoked for all target nodes and for any target-independent nodes that the target...
SDValue WidenOrSplitVectorLoad(SDValue Op, SelectionDAG &DAG) const
Widen a suitably aligned v3 load.
SDValue getHiHalf64(SDValue Op, SelectionDAG &DAG) const
static APFloat getQNaN(const fltSemantics &Sem, bool Negative=false, const APInt *payload=nullptr)
Factory for QNaN values.
opStatus convert(const fltSemantics &ToSemantics, roundingMode RM, bool *losesInfo)
LLVM_READONLY int getExactLog2Abs() const
APInt bitcastToAPInt() const
static APFloat getLargest(const fltSemantics &Sem, bool Negative=false)
Returns the largest finite number in the given semantics.
static APFloat getInf(const fltSemantics &Sem, bool Negative=false)
Factory for Positive and Negative Infinity.
static APFloat getZero(const fltSemantics &Sem, bool Negative=false)
Factory for Positive and Negative Zero.
Class for arbitrary precision integers.
void setHighBits(unsigned hiBits)
Set the top hiBits bits.
void setBitsFrom(unsigned loBit)
Set the top bits starting from loBit.
static APInt getBitsSet(unsigned numBits, unsigned loBit, unsigned hiBit)
Get a value with a block of bits set.
bool isSignMask() const
Check if the APInt's value is returned by getSignMask.
unsigned countr_zero() const
Count the number of trailing zero bits.
static APInt getHighBitsSet(unsigned numBits, unsigned hiBitsSet)
Constructs an APInt value that has the top hiBitsSet bits set.
bool sge(const APInt &RHS) const
Signed greater or equal comparison.
bool uge(const APInt &RHS) const
Unsigned greater or equal comparison.
This class represents an incoming formal argument to a Function.
bool hasAttribute(Attribute::AttrKind Kind) const
Check if an argument has a given attribute.
const Function * getParent() const
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
size - Get the array size.
bool empty() const
empty - Check if the array is empty.
An instruction that atomically checks whether a specified value is in a memory location,...
Value * getNewValOperand()
unsigned getPointerAddressSpace() const
Returns the address space of the pointer operand.
Value * getCompareOperand()
Align getAlign() const
Return the alignment of the memory that is being allocated by the instruction.
static unsigned getPointerOperandIndex()
an instruction that atomically reads a memory location, combines it with another value,...
Align getAlign() const
Return the alignment of the memory that is being allocated by the instruction.
static unsigned getPointerOperandIndex()
BinOp
This enumeration lists the possible modifications atomicrmw can make.
@ Min
*p = old <signed v ? old : v
@ UIncWrap
Increment one up to a maximum value.
@ Max
*p = old >signed v ? old : v
@ UMin
*p = old <unsigned v ? old : v
@ FMin
*p = minnum(old, v) minnum matches the behavior of llvm.minnum.
@ UMax
*p = old >unsigned v ? old : v
@ FMax
*p = maxnum(old, v) maxnum matches the behavior of llvm.maxnum.
@ UDecWrap
Decrement one until a minimum value or zero.
Value * getPointerOperand()
void setOperation(BinOp Operation)
BinOp getOperation() const
SyncScope::ID getSyncScopeID() const
Returns the synchronization scope ID of this rmw instruction.
static StringRef getOperationName(BinOp Op)
AtomicOrdering getOrdering() const
Returns the ordering constraint of this rmw instruction.
unsigned getPointerAddressSpace() const
Returns the address space of the pointer operand.
This is an SDNode representing atomic operations.
bool isCompareAndSwap() const
Returns true if this SDNode represents cmpxchg atomic operation, false otherwise.
MemoryEffects getMemoryEffects() const
Returns memory effects of the function.
bool getValueAsBool() const
Return the attribute's value as a boolean.
LLVM Basic Block Representation.
static BasicBlock * Create(LLVMContext &Context, const Twine &Name="", Function *Parent=nullptr, BasicBlock *InsertBefore=nullptr)
Creates a new BasicBlock.
BasicBlock * splitBasicBlock(iterator I, const Twine &BBName="", bool Before=false)
Split the basic block into two basic blocks at the specified instruction.
const Function * getParent() const
Return the enclosing method, or null if none.
A "pseudo-class" with methods for operating on BUILD_VECTORs.
Represents known origin of an individual byte in combine pattern.
static ByteProvider getConstantZero()
static ByteProvider getSrc(std::optional< ISelOp > Val, int64_t ByteOffset, int64_t VectorOffset)
std::optional< ISelOp > Src
CCState - This class holds information needed while lowering arguments and return values.
MachineFunction & getMachineFunction() const
unsigned getFirstUnallocated(ArrayRef< MCPhysReg > Regs) const
getFirstUnallocated - Return the index of the first unallocated register in the set,...
static bool resultsCompatible(CallingConv::ID CalleeCC, CallingConv::ID CallerCC, MachineFunction &MF, LLVMContext &C, const SmallVectorImpl< ISD::InputArg > &Ins, CCAssignFn CalleeFn, CCAssignFn CallerFn)
Returns true if the results of the two calling conventions are compatible.
void AnalyzeCallResult(const SmallVectorImpl< ISD::InputArg > &Ins, CCAssignFn Fn)
AnalyzeCallResult - Analyze the return values of a call, incorporating info about the passed values i...
MCRegister AllocateReg(MCPhysReg Reg)
AllocateReg - Attempt to allocate one register.
bool CheckReturn(const SmallVectorImpl< ISD::OutputArg > &Outs, CCAssignFn Fn)
CheckReturn - Analyze the return values of a function, returning true if the return can be performed ...
void AnalyzeReturn(const SmallVectorImpl< ISD::OutputArg > &Outs, CCAssignFn Fn)
AnalyzeReturn - Analyze the returned values of a return, incorporating info about the result values i...
int64_t AllocateStack(unsigned Size, Align Alignment)
AllocateStack - Allocate a chunk of stack space with the specified size and alignment.
void AnalyzeCallOperands(const SmallVectorImpl< ISD::OutputArg > &Outs, CCAssignFn Fn)
AnalyzeCallOperands - Analyze the outgoing arguments to a call, incorporating info about the passed v...
uint64_t getStackSize() const
Returns the size of the currently allocated portion of the stack.
bool isAllocated(MCRegister Reg) const
isAllocated - Return true if the specified register (or an alias) is allocated.
void AnalyzeFormalArguments(const SmallVectorImpl< ISD::InputArg > &Ins, CCAssignFn Fn)
AnalyzeFormalArguments - Analyze an array of argument values, incorporating info about the formals in...
CCValAssign - Represent assignment of one arg/retval to a location.
Register getLocReg() const
LocInfo getLocInfo() const
int64_t getLocMemOffset() const
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
bool hasFnAttr(Attribute::AttrKind Kind) const
Determine whether this call has the given attribute.
bool isMustTailCall() const
Tests if this call site must be tail call optimized.
Value * getArgOperand(unsigned i) const
unsigned arg_size() const
This class represents a function call, abstracting a target machine's calling convention.
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
bool isFPPredicate() const
bool isIntPredicate() const
const APFloat & getValueAPF() const
bool isExactlyValue(double V) const
We don't rely on operator== working on double values, as it returns true for things that are clearly ...
bool isNegative() const
Return true if the value is negative.
bool isInfinity() const
Return true if the value is an infinity.
This is the shared class of boolean and integer constants.
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
uint64_t getZExtValue() const
const APInt & getAPIntValue() const
This is an important base class in LLVM.
bool isNullValue() const
Return true if this is the value that would be returned by getNullValue.
This class represents an Operation in the Expression.
uint64_t getNumOperands() const
A parsed version of the target data layout string in and methods for querying it.
Align getABITypeAlign(Type *Ty) const
Returns the minimum ABI-required alignment for the specified type.
TypeSize getTypeAllocSize(Type *Ty) const
Returns the offset in bytes between successive objects of the specified type, including alignment pad...
Diagnostic information for unsupported feature in backend.
Class to represent fixed width SIMD vectors.
unsigned getNumElements() const
FunctionLoweringInfo - This contains information that is global to a function that is used when lower...
Class to represent function types.
Type * getParamType(unsigned i) const
Parameter type accessors.
FunctionType * getFunctionType() const
Returns the FunctionType for me.
const DataLayout & getDataLayout() const
Get the data layout of the module this function belongs to.
iterator_range< arg_iterator > args()
Attribute getFnAttribute(Attribute::AttrKind Kind) const
Return the attribute for the given attribute kind.
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
DenormalMode getDenormalMode(const fltSemantics &FPType) const
Returns the denormal handling type for the default rounding mode of the function.
Argument * getArg(unsigned i) const
bool hasMemoryAtomicFaddF32DenormalSupport() const
bool hasD16Images() const
bool hasMinimum3Maximum3F32() const
bool useVGPRIndexMode() const
bool hasAtomicDsPkAdd16Insts() const
bool hasImageStoreD16Bug() const
bool hasUsableDivScaleConditionOutput() const
Condition output from div_scale is usable.
bool hasUsableDSOffset() const
True if the offset field of DS instructions works as expected.
bool hasAtomicFMinFMaxF64FlatInsts() const
bool hasDot7Insts() const
bool hasApertureRegs() const
bool hasFlatInstOffsets() const
bool hasAtomicFMinFMaxF32FlatInsts() const
bool hasCompressedExport() const
Return true if the target's EXP instruction has the COMPR flag, which affects the meaning of the EN (...
bool hasGFX90AInsts() const
bool hasBCNT(unsigned Size) const
bool hasLDSLoadB96_B128() const
Returns true if the target supports global_load_lds_dwordx3/global_load_lds_dwordx4 or buffer_load_dw...
bool supportsAgentScopeFineGrainedRemoteMemoryAtomics() const
bool hasMultiDwordFlatScratchAddressing() const
bool hasArchitectedSGPRs() const
bool hasDenormModeInst() const
bool hasPrivEnabledTrap2NopBug() const
bool hasUnalignedDSAccessEnabled() const
const SIInstrInfo * getInstrInfo() const override
bool hasDot1Insts() const
bool hasAtomicFaddRtnInsts() const
Align getStackAlignment() const
bool hasScalarSubwordLoads() const
bool enableFlatScratch() const
bool hasDwordx3LoadStores() const
bool hasFlatScrRegister() const
bool supportsGetDoorbellID() const
bool hasFlatAtomicFaddF32Inst() const
bool hasKernargPreload() const
const SIRegisterInfo * getRegisterInfo() const override
unsigned getMaxNumVGPRs(unsigned WavesPerEU) const
bool hasMinimum3Maximum3PKF16() const
bool hasLDSMisalignedBug() const
bool hasUserSGPRInit16Bug() const
TrapHandlerAbi getTrapHandlerAbi() const
const SIFrameLowering * getFrameLowering() const override
bool hasMinimum3Maximum3F16() const
bool hasAtomicFMinFMaxF32GlobalInsts() const
bool hasRestrictedSOffset() const
bool hasMin3Max3_16() const
bool hasGFX10_AEncoding() const
bool hasPackedFP32Ops() const
bool hasFullRate64Ops() const
bool isTrapHandlerEnabled() const
bool hasLDSFPAtomicAddF64() const
bool hasFlatGlobalInsts() const
bool getScalarizeGlobalBehavior() const
bool hasScalarSMulU64() const
unsigned getKnownHighZeroBitsForFrameIndex() const
Return the number of high bits known to be zero for a frame index.
bool hasShaderCyclesHiLoRegisters() const
bool hasNSAEncoding() const
bool hasSMemRealTime() const
bool usePRTStrictNull() const
bool hasAtomicFMinFMaxF64GlobalInsts() const
bool hasUnalignedScratchAccessEnabled() const
bool hasAtomicFlatPkAdd16Insts() const
bool hasUnalignedBufferAccessEnabled() const
unsigned getMaxPrivateElementSize(bool ForBufferRSrc=false) const
bool hasImageGather4D16Bug() const
bool hasDot10Insts() const
bool supportsMinMaxDenormModes() const
bool hasAtomicFaddInsts() const
unsigned getNSAMaxSize(bool HasSampler=false) const
bool hasAtomicBufferGlobalPkAddF16NoRtnInsts() const
bool hasAtomicBufferPkAddBF16Inst() const
bool hasAtomicFaddNoRtnInsts() const
bool hasFlatBufferGlobalAtomicFaddF64Inst() const
bool hasScalarDwordx3Loads() const
bool hasLDSFPAtomicAddF32() const
bool haveRoundOpsF64() const
Have v_trunc_f64, v_ceil_f64, v_rndne_f64.
bool hasDot8Insts() const
bool hasDS96AndDS128() const
bool useFlatForGlobal() const
Generation getGeneration() const
bool hasAtomicBufferGlobalPkAddF16Insts() const
bool hasScalarAddSub64() const
bool hasUnpackedD16VMem() const
bool hasAtomicGlobalPkAddBF16Inst() const
bool hasIEEEMinMax() const
bool hasFmaMixInsts() const
bool hasPackedTID() const
bool hasAddNoCarry() const
bool hasGWSAutoReplay() const
bool hasKernargSegmentPtr() const
bool hasDispatchID() const
bool hasPrivateSegmentBuffer() const
unsigned getNumFreeUserSGPRs()
bool hasImplicitBufferPtr() const
bool hasPrivateSegmentSize() const
bool hasDispatchPtr() const
bool hasFlatScratchInit() const
virtual void computeKnownBitsImpl(Register R, KnownBits &Known, const APInt &DemandedElts, unsigned Depth=0)
const MachineFunction & getMachineFunction() const
int64_t getOffset() const
unsigned getAddressSpace() const
const GlobalValue * getGlobal() const
bool hasExternalLinkage() const
unsigned getAddressSpace() const
Module * getParent()
Get the module that this global value is contained inside of...
Type * getValueType() const
Value * CreateInsertValue(Value *Agg, Value *Val, ArrayRef< unsigned > Idxs, const Twine &Name="")
LoadInst * CreateAlignedLoad(Type *Ty, Value *Ptr, MaybeAlign Align, const char *Name)
BasicBlock::iterator GetInsertPoint() const
BasicBlock * GetInsertBlock() const
CallInst * CreateIntrinsic(Intrinsic::ID ID, ArrayRef< Type * > Types, ArrayRef< Value * > Args, FMFSource FMFSource={}, const Twine &Name="")
Create a call to intrinsic ID with Args, mangled using Types.
PHINode * CreatePHI(Type *Ty, unsigned NumReservedValues, const Twine &Name="")
BranchInst * CreateCondBr(Value *Cond, BasicBlock *True, BasicBlock *False, MDNode *BranchWeights=nullptr, MDNode *Unpredictable=nullptr)
Create a conditional 'br Cond, TrueDest, FalseDest' instruction.
LLVMContext & getContext() const
BranchInst * CreateBr(BasicBlock *Dest)
Create an unconditional 'br label X' instruction.
void SetInsertPoint(BasicBlock *TheBB)
This specifies that created instructions should be appended to the end of the specified block.
StoreInst * CreateAlignedStore(Value *Val, Value *Ptr, MaybeAlign Align, bool isVolatile=false)
Value * CreateAddrSpaceCast(Value *V, Type *DestTy, const Twine &Name="")
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Instruction * clone() const
Create a copy of 'this' instruction that is identical in all ways except the following:
void removeFromParent()
This method unlinks 'this' from the containing basic block, but does not delete it.
bool hasMetadata() const
Return true if this instruction has any metadata attached to it.
InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
const Function * getFunction() const
Return the function this instruction belongs to.
void setMetadata(unsigned KindID, MDNode *Node)
Set the metadata of the specified kind to the specified node.
void copyMetadata(const Instruction &SrcInst, ArrayRef< unsigned > WL=ArrayRef< unsigned >())
Copy metadata from SrcInst to this instruction.
const DataLayout & getDataLayout() const
Get the data layout of the module this instruction belongs to.
InstListType::iterator insertInto(BasicBlock *ParentBB, InstListType::iterator It)
Inserts an unlinked instruction into ParentBB at position It and returns the iterator of the inserted...
Class to represent integer types.
A wrapper class for inspecting calls to intrinsic functions.
constexpr unsigned getScalarSizeInBits() const
constexpr bool isScalar() const
static constexpr LLT scalar(unsigned SizeInBits)
Get a low-level scalar or aggregate "bag of bits".
static constexpr LLT pointer(unsigned AddressSpace, unsigned SizeInBits)
Get a low-level pointer in the given address space.
constexpr TypeSize getSizeInBits() const
Returns the total size of the type. Must only be called on sized types.
constexpr LLT changeElementSize(unsigned NewEltSize) const
If this type is a vector, return a vector with the same number of elements but the new element size.
This is an important class for using LLVM in a threaded context.
std::optional< StringRef > getSyncScopeName(SyncScope::ID Id) const
getSyncScopeName - Returns the name of a SyncScope::ID registered with LLVMContext,...
void diagnose(const DiagnosticInfo &DI)
Report a message to the currently installed diagnostic handler.
SyncScope::ID getOrInsertSyncScopeID(StringRef SSN)
getOrInsertSyncScopeID - Maps synchronization scope name to synchronization scope ID.
An instruction for reading from memory.
unsigned getPointerAddressSpace() const
Returns the address space of the pointer operand.
void setAtomic(AtomicOrdering Ordering, SyncScope::ID SSID=SyncScope::System)
Sets the ordering constraint and the synchronization scope ID of this load instruction.
This class is used to represent ISD::LOAD nodes.
const SDValue & getBasePtr() const
const SDValue & getOffset() const
ISD::LoadExtType getExtensionType() const
Return whether this is a plain node, or one of the varieties of value-extending loads.
Describe properties that are true of each instruction in the target description file.
Wrapper class representing physical registers. Should be passed by value.
MDNode * createRange(const APInt &Lo, const APInt &Hi)
Return metadata describing the range [Lo, Hi).
const MDOperand & getOperand(unsigned I) const
unsigned getNumOperands() const
Return number of MDNode operands.
Helper class for constructing bundles of MachineInstrs.
MachineBasicBlock::instr_iterator begin() const
Return an iterator to the first bundled instruction.
uint64_t getScalarSizeInBits() const
bool bitsLE(MVT VT) const
Return true if this has no more bits than VT.
unsigned getVectorNumElements() const
bool isVector() const
Return true if this is a vector value type.
bool isScalableVector() const
Return true if this is a vector value type where the runtime length is machine dependent.
static MVT getVT(Type *Ty, bool HandleUnknown=false)
Return the value type corresponding to the specified type.
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
bool isPow2VectorType() const
Returns true if the given vector is a power of 2.
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
static MVT getVectorVT(MVT VT, unsigned NumElements)
static MVT getIntegerVT(unsigned BitWidth)
MVT getScalarType() const
If this is a vector, return the element type, otherwise return this.
void transferSuccessorsAndUpdatePHIs(MachineBasicBlock *FromMBB)
Transfers all the successors, as in transferSuccessors, and update PHI operands in the successor bloc...
iterator getFirstTerminator()
Returns an iterator to the first terminator instruction of this basic block.
void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
MachineBasicBlock * splitAt(MachineInstr &SplitInst, bool UpdateLiveIns=true, LiveIntervals *LIS=nullptr)
Split a basic block into 2 pieces at SplitPoint.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
Align getAlignment() const
Return alignment of the basic block.
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
int CreateFixedObject(uint64_t Size, int64_t SPOffset, bool IsImmutable, bool isAliased=false)
Create a new object at a fixed location on the stack.
bool hasCalls() const
Return true if the current function has any function calls.
void setHasTailCall(bool V=true)
void setReturnAddressIsTaken(bool s)
bool hasStackObjects() const
Return true if there are any stack objects in this function.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
DenormalMode getDenormalMode(const fltSemantics &FPType) const
Returns the denormal handling type for the default rounding mode of the function.
void push_back(MachineBasicBlock *MBB)
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
const DataLayout & getDataLayout() const
Return the DataLayout attached to the Module associated to this MF.
Function & getFunction()
Return the LLVM function that this machine code represents.
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
Register addLiveIn(MCRegister PReg, const TargetRegisterClass *RC)
addLiveIn - Add the specified physical register as a live-in value and create a corresponding virtual...
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *BB=nullptr, std::optional< UniqueBBID > BBID=std::nullopt)
CreateMachineBasicBlock - Allocate a new MachineBasicBlock.
void insert(iterator MBBI, MachineBasicBlock *MBB)
const TargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & add(const MachineOperand &MO) const
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
const MachineInstrBuilder & cloneMemRefs(const MachineInstr &OtherMI) const
Representation of each machine instruction.
const MachineOperand & getOperand(unsigned i) const
A description of a memory reference used in the backend.
Flags
Flags values. These may be or'd together.
@ MOVolatile
The memory access is volatile.
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MOLoad
The memory access reads data.
@ MONonTemporal
The memory access is non-temporal.
@ MOInvariant
The memory access always returns the same value (or traps).
@ MOStore
The memory access writes data.
const MachinePointerInfo & getPointerInfo() const
Flags getFlags() const
Return the raw flags of the source value,.
AAMDNodes getAAInfo() const
Return the AA tags for the memory reference.
Align getBaseAlign() const
Return the minimum known alignment in bytes of the base address, without the offset.
MachineOperand class - Representation of each machine instruction operand.
bool isReg() const
isReg - Tests if this is a MO_Register operand.
void setReg(Register Reg)
Change the register this operand corresponds to.
static MachineOperand CreateImm(int64_t Val)
void setIsUndef(bool Val=true)
Register getReg() const
getReg - Returns the register number.
static MachineOperand CreateReg(Register Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isEarlyClobber=false, unsigned SubReg=0, bool isDebug=false, bool isInternalRead=false, bool isRenamable=false)
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
void setType(Register VReg, LLT Ty)
Set the low-level type of VReg to Ty.
An SDNode that represents everything that will be needed to construct a MachineInstr.
This is an abstract virtual class for memory operations.
unsigned getAddressSpace() const
Return the address space for the associated pointer.
AAMDNodes getAAInfo() const
Returns the AA info that describes the dereference.
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
const MachinePointerInfo & getPointerInfo() const
const SDValue & getChain() const
EVT getMemoryVT() const
Return the type of the in-memory value.
bool onlyWritesMemory() const
Whether this function only (at most) writes memory.
bool doesNotAccessMemory() const
Whether this function accesses no memory.
bool onlyReadsMemory() const
Whether this function only (at most) reads memory.
A Module instance is used to store all the information related to an LLVM module.
const DataLayout & getDataLayout() const
Get the data layout for the module's target platform.
void addIncoming(Value *V, BasicBlock *BB)
Add an incoming value to the end of the PHI list.
AnalysisType & getAnalysis() const
getAnalysis<AnalysisType>() - This function is used by subclasses to get to the analysis information ...
static PointerType * get(Type *ElementType, unsigned AddressSpace)
This constructs a pointer to an object of the specified type in a numbered address space.
static PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
Wrapper class representing virtual and physical registers.
static Register index2VirtReg(unsigned Index)
Convert a 0-based index to a virtual register number.
constexpr bool isPhysical() const
Return true if the specified register number is in the physical register namespace.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
const DebugLoc & getDebugLoc() const
Represents one node in the SelectionDAG.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
bool hasOneUse() const
Return true if there is exactly one use of this node.
SDNodeFlags getFlags() const
uint64_t getAsZExtVal() const
Helper method returns the zero-extended integer value of a ConstantSDNode.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
unsigned getMachineOpcode() const
This may only be called if isMachineOpcode returns true.
const SDValue & getOperand(unsigned Num) const
uint64_t getConstantOperandVal(unsigned Num) const
Helper method returns the integer value of a ConstantSDNode operand.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
user_iterator user_begin() const
Provide iteration support to walk over all users of an SDNode.
Represents a use of a SDNode.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
bool hasOneUse() const
Return true if there is exactly one node using value ResNo of Node.
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
bool isMachineOpcode() const
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
const SDValue & getOperand(unsigned i) const
MVT getSimpleValueType() const
Return the simple ValueType of the referenced return value.
unsigned getMachineOpcode() const
unsigned getOpcode() const
static unsigned getMaxMUBUFImmOffset(const GCNSubtarget &ST)
static unsigned getDSShaderTypeValue(const MachineFunction &MF)
bool isLegalFLATOffset(int64_t Offset, unsigned AddrSpace, uint64_t FlatVariant) const
Returns if Offset is legal for the subtarget as the offset to a FLAT encoded instruction.
This class keeps track of the SPI_SP_INPUT_ADDR config register, which tells the hardware which inter...
bool hasWorkGroupIDZ() const
SIModeRegisterDefaults getMode() const
std::tuple< const ArgDescriptor *, const TargetRegisterClass *, LLT > getPreloadedValue(AMDGPUFunctionArgInfo::PreloadedValue Value) const
unsigned getBytesInStackArgArea() const
const AMDGPUGWSResourcePseudoSourceValue * getGWSPSV(const AMDGPUTargetMachine &TM)
static unsigned getSubRegFromChannel(unsigned Channel, unsigned NumRegs=1)
static LLVM_READONLY const TargetRegisterClass * getSGPRClassForBitWidth(unsigned BitWidth)
static bool isVGPRClass(const TargetRegisterClass *RC)
static bool isSGPRClass(const TargetRegisterClass *RC)
static bool isAGPRClass(const TargetRegisterClass *RC)
bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const override
Return true if folding a constant offset with the given GlobalAddress is legal.
bool isTypeDesirableForOp(unsigned Op, EVT VT) const override
Return true if the target has native support for the specified value type and it is 'desirable' to us...
SDNode * PostISelFolding(MachineSDNode *N, SelectionDAG &DAG) const override
Fold the instructions after selecting them.
SDValue splitTernaryVectorOp(SDValue Op, SelectionDAG &DAG) const
MachineSDNode * wrapAddr64Rsrc(SelectionDAG &DAG, const SDLoc &DL, SDValue Ptr) const
bool isFMAFasterThanFMulAndFAdd(const MachineFunction &MF, EVT VT) const override
Return true if an FMA operation is faster than a pair of fmul and fadd instructions.
SDValue lowerGET_ROUNDING(SDValue Op, SelectionDAG &DAG) const
bool checkForPhysRegDependency(SDNode *Def, SDNode *User, unsigned Op, const TargetRegisterInfo *TRI, const TargetInstrInfo *TII, unsigned &PhysReg, int &Cost) const override
Allows the target to handle physreg-carried dependency in target-specific way.
EVT getOptimalMemOpType(const MemOp &Op, const AttributeList &FuncAttributes) const override
Returns the target specific optimal type for load and store operations as a result of memset,...
bool requiresUniformRegister(MachineFunction &MF, const Value *V) const override
Allows target to decide about the register class of the specific value that is live outside the defin...
bool isFMADLegal(const SelectionDAG &DAG, const SDNode *N) const override
Returns true if be combined with to form an ISD::FMAD.
AtomicExpansionKind shouldExpandAtomicStoreInIR(StoreInst *SI) const override
Returns how the given (atomic) store should be expanded by the IR-level AtomicExpand pass into.
void bundleInstWithWaitcnt(MachineInstr &MI) const
Insert MI into a BUNDLE with an S_WAITCNT 0 immediately following it.
MVT getScalarShiftAmountTy(const DataLayout &, EVT) const override
Return the type to use for a scalar shift opcode, given the shifted amount type.
SDValue LowerCall(CallLoweringInfo &CLI, SmallVectorImpl< SDValue > &InVals) const override
This hook must be implemented to lower calls into the specified DAG.
MVT getPointerTy(const DataLayout &DL, unsigned AS) const override
Map address space 7 to MVT::v5i32 because that's its in-memory representation.
bool denormalsEnabledForType(const SelectionDAG &DAG, EVT VT) const
void insertCopiesSplitCSR(MachineBasicBlock *Entry, const SmallVectorImpl< MachineBasicBlock * > &Exits) const override
Insert explicit copies in entry and exit blocks.
EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const override
Return the ValueType of the result of SETCC operations.
SDNode * legalizeTargetIndependentNode(SDNode *Node, SelectionDAG &DAG) const
Legalize target independent instructions (e.g.
bool allowsMisalignedMemoryAccessesImpl(unsigned Size, unsigned AddrSpace, Align Alignment, MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *IsFast=nullptr) const
TargetLoweringBase::LegalizeTypeAction getPreferredVectorAction(MVT VT) const override
Return the preferred vector type legalization action.
SDValue lowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const
const GCNSubtarget * getSubtarget() const
bool enableAggressiveFMAFusion(EVT VT) const override
Return true if target always benefits from combining into FMA for a given value type.
bool shouldEmitGOTReloc(const GlobalValue *GV) const
bool isCanonicalized(SelectionDAG &DAG, SDValue Op, unsigned MaxDepth=5) const
void CollectTargetIntrinsicOperands(const CallInst &I, SmallVectorImpl< SDValue > &Ops, SelectionDAG &DAG) const override
SDValue splitUnaryVectorOp(SDValue Op, SelectionDAG &DAG) const
SDValue lowerGET_FPENV(SDValue Op, SelectionDAG &DAG) const
void allocateSpecialInputSGPRs(CCState &CCInfo, MachineFunction &MF, const SIRegisterInfo &TRI, SIMachineFunctionInfo &Info) const
void allocateLDSKernelId(CCState &CCInfo, MachineFunction &MF, const SIRegisterInfo &TRI, SIMachineFunctionInfo &Info) const
SDValue LowerSTACKSAVE(SDValue Op, SelectionDAG &DAG) const
bool isReassocProfitable(SelectionDAG &DAG, SDValue N0, SDValue N1) const override
void allocateHSAUserSGPRs(CCState &CCInfo, MachineFunction &MF, const SIRegisterInfo &TRI, SIMachineFunctionInfo &Info) const
ArrayRef< MCPhysReg > getRoundingControlRegisters() const override
Returns a 0 terminated array of rounding control registers that can be attached into strict FP call.
ConstraintType getConstraintType(StringRef Constraint) const override
Given a constraint, return the type of constraint it is for this target.
SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool IsVarArg, const SmallVectorImpl< ISD::OutputArg > &Outs, const SmallVectorImpl< SDValue > &OutVals, const SDLoc &DL, SelectionDAG &DAG) const override
This hook must be implemented to lower outgoing return values, described by the Outs array,...
const TargetRegisterClass * getRegClassFor(MVT VT, bool isDivergent) const override
Return the register class that should be used for the specified value type.
void AddMemOpInit(MachineInstr &MI) const
MachineMemOperand::Flags getTargetMMOFlags(const Instruction &I) const override
This callback is used to inspect load/store instructions and add target-specific MachineMemOperand fl...
bool isLegalGlobalAddressingMode(const AddrMode &AM) const
void computeKnownBitsForFrameIndex(int FrameIdx, KnownBits &Known, const MachineFunction &MF) const override
Determine which of the bits of FrameIndex FIOp are known to be 0.
bool shouldConvertConstantLoadToIntImm(const APInt &Imm, Type *Ty) const override
Return true if it is beneficial to convert a load of a constant to just the constant itself.
Align getPrefLoopAlignment(MachineLoop *ML) const override
Return the preferred loop alignment.
std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const override
Given a physical register constraint (e.g.
AtomicExpansionKind shouldExpandAtomicLoadInIR(LoadInst *LI) const override
Returns how the given (atomic) load should be expanded by the IR-level AtomicExpand pass.
bool getAsmOperandConstVal(SDValue Op, uint64_t &Val) const
bool isShuffleMaskLegal(ArrayRef< int >, EVT) const override
Targets can use this to indicate that they only support some VECTOR_SHUFFLE operations,...
void ReplaceNodeResults(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG) const override
This callback is invoked when a node result type is illegal for the target, and the operation was reg...
Register getRegisterByName(const char *RegName, LLT VT, const MachineFunction &MF) const override
Return the register ID of the name passed in.
void LowerAsmOperandForConstraint(SDValue Op, StringRef Constraint, std::vector< SDValue > &Ops, SelectionDAG &DAG) const override
Lower the specified operand into the Ops vector.
LLT getPreferredShiftAmountTy(LLT Ty) const override
Return the preferred type to use for a shift opcode, given the shifted amount type is ShiftValueTy.
bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty, unsigned AS, Instruction *I=nullptr) const override
Return true if the addressing mode represented by AM is legal for this target, for a load/store of th...
SDValue lowerSET_FPENV(SDValue Op, SelectionDAG &DAG) const
void computeKnownBitsForTargetNode(const SDValue Op, KnownBits &Known, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth=0) const override
Determine which of the bits specified in Mask are known to be either zero or one and return them in t...
SDValue lowerSET_ROUNDING(SDValue Op, SelectionDAG &DAG) const
void allocateSpecialInputVGPRsFixed(CCState &CCInfo, MachineFunction &MF, const SIRegisterInfo &TRI, SIMachineFunctionInfo &Info) const
Allocate implicit function VGPR arguments in fixed registers.
LoadInst * lowerIdempotentRMWIntoFencedLoad(AtomicRMWInst *AI) const override
On some platforms, an AtomicRMW that never actually modifies the value (such as fetch_add of 0) can b...
MachineBasicBlock * emitGWSMemViolTestLoop(MachineInstr &MI, MachineBasicBlock *BB) const
bool getAddrModeArguments(const IntrinsicInst *I, SmallVectorImpl< Value * > &Ops, Type *&AccessTy) const override
CodeGenPrepare sinks address calculations into the same BB as Load/Store instructions reading the add...
bool checkAsmConstraintValA(SDValue Op, uint64_t Val, unsigned MaxSize=64) const
AtomicExpansionKind shouldExpandAtomicRMWInIR(AtomicRMWInst *) const override
Returns how the IR-level AtomicExpand pass should expand the given AtomicRMW, if at all.
bool shouldEmitFixup(const GlobalValue *GV) const
MachineBasicBlock * splitKillBlock(MachineInstr &MI, MachineBasicBlock *BB) const
void emitExpandAtomicCmpXchg(AtomicCmpXchgInst *CI) const override
Perform a cmpxchg expansion using a target-specific method.
bool hasMemSDNodeUser(SDNode *N) const
bool isSDNodeSourceOfDivergence(const SDNode *N, FunctionLoweringInfo *FLI, UniformityInfo *UA) const override
MachineBasicBlock * EmitInstrWithCustomInserter(MachineInstr &MI, MachineBasicBlock *BB) const override
This method should be implemented by targets that mark instructions with the 'usesCustomInserter' fla...
bool isEligibleForTailCallOptimization(SDValue Callee, CallingConv::ID CalleeCC, bool isVarArg, const SmallVectorImpl< ISD::OutputArg > &Outs, const SmallVectorImpl< SDValue > &OutVals, const SmallVectorImpl< ISD::InputArg > &Ins, SelectionDAG &DAG) const
bool isMemOpHasNoClobberedMemOperand(const SDNode *N) const
bool isLegalFlatAddressingMode(const AddrMode &AM, unsigned AddrSpace) const
SDValue LowerCallResult(SDValue Chain, SDValue InGlue, CallingConv::ID CallConv, bool isVarArg, const SmallVectorImpl< ISD::InputArg > &Ins, const SDLoc &DL, SelectionDAG &DAG, SmallVectorImpl< SDValue > &InVals, bool isThisReturn, SDValue ThisVal) const
SDValue LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv, bool isVarArg, const SmallVectorImpl< ISD::InputArg > &Ins, const SDLoc &DL, SelectionDAG &DAG, SmallVectorImpl< SDValue > &InVals) const override
This hook must be implemented to lower the incoming (formal) arguments, described by the Ins array,...
SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override
This method will be invoked for all target nodes and for any target-independent nodes that the target...
bool isKnownNeverNaNForTargetNode(SDValue Op, const SelectionDAG &DAG, bool SNaN=false, unsigned Depth=0) const override
If SNaN is false,.
AtomicExpansionKind shouldExpandAtomicCmpXchgInIR(AtomicCmpXchgInst *AI) const override
Returns how the given atomic cmpxchg should be expanded by the IR-level AtomicExpand pass.
bool isFPExtFoldable(const SelectionDAG &DAG, unsigned Opcode, EVT DestVT, EVT SrcVT) const override
Return true if an fpext operation input to an Opcode operation is free (for instance,...
void AdjustInstrPostInstrSelection(MachineInstr &MI, SDNode *Node) const override
Assign the register class depending on the number of bits set in the writemask.
MVT getRegisterTypeForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const override
Certain combinations of ABIs, Targets and features require that types are legal for some operations a...
void allocateSpecialInputVGPRs(CCState &CCInfo, MachineFunction &MF, const SIRegisterInfo &TRI, SIMachineFunctionInfo &Info) const
Allocate implicit function VGPR arguments at the end of allocated user arguments.
void finalizeLowering(MachineFunction &MF) const override
Execute target specific actions to finalize target lowering.
static bool isNonGlobalAddrSpace(unsigned AS)
void emitExpandAtomicAddrSpacePredicate(Instruction *AI) const
MachineSDNode * buildRSRC(SelectionDAG &DAG, const SDLoc &DL, SDValue Ptr, uint32_t RsrcDword1, uint64_t RsrcDword2And3) const
Return a resource descriptor with the 'Add TID' bit enabled The TID (Thread ID) is multiplied by the ...
unsigned getNumRegistersForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const override
Certain targets require unusual breakdowns of certain types.
bool mayBeEmittedAsTailCall(const CallInst *) const override
Return true if the target may be able emit the call instruction as a tail call.
void passSpecialInputs(CallLoweringInfo &CLI, CCState &CCInfo, const SIMachineFunctionInfo &Info, SmallVectorImpl< std::pair< unsigned, SDValue > > &RegsToPass, SmallVectorImpl< SDValue > &MemOpChains, SDValue Chain) const
SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override
This callback is invoked for operations that are unsupported by the target, which are registered to u...
bool checkAsmConstraintVal(SDValue Op, StringRef Constraint, uint64_t Val) const
void emitExpandAtomicRMW(AtomicRMWInst *AI) const override
Perform a atomicrmw expansion using a target-specific way.
static bool shouldExpandVectorDynExt(unsigned EltSize, unsigned NumElem, bool IsDivergentIdx, const GCNSubtarget *Subtarget)
Check if EXTRACT_VECTOR_ELT/INSERT_VECTOR_ELT (<n x e>, var-idx) should be expanded into a set of cmp...
bool shouldUseLDSConstAddress(const GlobalValue *GV) const
bool supportSplitCSR(MachineFunction *MF) const override
Return true if the target supports that a subset of CSRs for the given machine function is handled ex...
bool isExtractVecEltCheap(EVT VT, unsigned Index) const override
Return true if extraction of a scalar element from the given vector type at the given index is cheap.
SDValue LowerDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG) const
bool allowsMisalignedMemoryAccesses(LLT Ty, unsigned AddrSpace, Align Alignment, MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *IsFast=nullptr) const override
LLT handling variant.
bool isExtractSubvectorCheap(EVT ResVT, EVT SrcVT, unsigned Index) const override
Return true if EXTRACT_SUBVECTOR is cheap for extracting this result type from this source type with ...
void computeKnownBitsForTargetInstr(GISelKnownBits &Analysis, Register R, KnownBits &Known, const APInt &DemandedElts, const MachineRegisterInfo &MRI, unsigned Depth=0) const override
Determine which of the bits specified in Mask are known to be either zero or one and return them in t...
bool canMergeStoresTo(unsigned AS, EVT MemVT, const MachineFunction &MF) const override
Returns if it's reasonable to merge stores to MemVT size.
SDValue lowerPREFETCH(SDValue Op, SelectionDAG &DAG) const
SITargetLowering(const TargetMachine &tm, const GCNSubtarget &STI)
bool isFreeAddrSpaceCast(unsigned SrcAS, unsigned DestAS) const override
Returns true if a cast from SrcAS to DestAS is "cheap", such that e.g.
bool shouldEmitPCReloc(const GlobalValue *GV) const
void initializeSplitCSR(MachineBasicBlock *Entry) const override
Perform necessary initialization to handle a subset of CSRs explicitly via copies.
void allocateSpecialEntryInputVGPRs(CCState &CCInfo, MachineFunction &MF, const SIRegisterInfo &TRI, SIMachineFunctionInfo &Info) const
void allocatePreloadKernArgSGPRs(CCState &CCInfo, SmallVectorImpl< CCValAssign > &ArgLocs, const SmallVectorImpl< ISD::InputArg > &Ins, MachineFunction &MF, const SIRegisterInfo &TRI, SIMachineFunctionInfo &Info) const
bool isProfitableToHoist(Instruction *I) const override
Check if it is profitable to hoist instruction in then/else to if.
SDValue copyToM0(SelectionDAG &DAG, SDValue Chain, const SDLoc &DL, SDValue V) const
bool getTgtMemIntrinsic(IntrinsicInfo &, const CallInst &, MachineFunction &MF, unsigned IntrinsicID) const override
Given an intrinsic, checks if on the target the intrinsic will need to map to a MemIntrinsicNode (tou...
SDValue splitBinaryVectorOp(SDValue Op, SelectionDAG &DAG) const
unsigned getVectorTypeBreakdownForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT, EVT &IntermediateVT, unsigned &NumIntermediates, MVT &RegisterVT) const override
Certain targets such as MIPS require that some types such as vectors are always broken down into scal...
Align computeKnownAlignForTargetInstr(GISelKnownBits &Analysis, Register R, const MachineRegisterInfo &MRI, unsigned Depth=0) const override
Determine the known alignment for the pointer value R.
MVT getPointerMemTy(const DataLayout &DL, unsigned AS) const override
Similarly, the in-memory representation of a p7 is {p8, i32}, aka v8i32 when padding is added.
void allocateSystemSGPRs(CCState &CCInfo, MachineFunction &MF, SIMachineFunctionInfo &Info, CallingConv::ID CallConv, bool IsShader) const
bool CanLowerReturn(CallingConv::ID CallConv, MachineFunction &MF, bool isVarArg, const SmallVectorImpl< ISD::OutputArg > &Outs, LLVMContext &Context, const Type *RetTy) const override
This hook should be implemented to check whether the return values described by the Outs array can fi...
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getTargetGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, unsigned TargetFlags=0)
SDValue getExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT, unsigned Opcode)
Convert Op, which must be of integer type, to the integer type VT, by either any/sign/zero-extending ...
const SDValue & getRoot() const
Return the root tag of the SelectionDAG.
bool isKnownNeverSNaN(SDValue Op, unsigned Depth=0) const
SDValue getAddrSpaceCast(const SDLoc &dl, EVT VT, SDValue Ptr, unsigned SrcAS, unsigned DestAS)
Return an AddrSpaceCastSDNode.
SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, Register Reg, SDValue N)
const Pass * getPass() const
SDValue getMergeValues(ArrayRef< SDValue > Ops, const SDLoc &dl)
Create a MERGE_VALUES node from the given operands.
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
SDValue getShiftAmountConstant(uint64_t Val, EVT VT, const SDLoc &DL)
SDValue getAllOnesConstant(const SDLoc &DL, EVT VT, bool IsTarget=false, bool IsOpaque=false)
MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s),...
void ExtractVectorElements(SDValue Op, SmallVectorImpl< SDValue > &Args, unsigned Start=0, unsigned Count=0, EVT EltVT=EVT())
Append the extracted elements from Start to Count out of the vector Op in Args.
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Chain=SDValue(), bool IsSignaling=false)
Helper function to make it easier to build SetCC's if you just have an ISD::CondCode instead of an SD...
SDValue UnrollVectorOp(SDNode *N, unsigned ResNE=0)
Utility function used by legalize and lowering to "unroll" a vector operation by splitting out the sc...
SDValue getConstantFP(double Val, const SDLoc &DL, EVT VT, bool isTarget=false)
Create a ConstantFPSDNode wrapping a constant value.
bool haveNoCommonBitsSet(SDValue A, SDValue B) const
Return true if A and B have no common bits set.
SDValue getRegister(Register Reg, EVT VT)
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands,...
SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Val, MachineMemOperand *MMO)
Gets a node for an atomic op, produces result (if relevant) and chain and takes 2 operands.
SDValue getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Align Alignment, bool isVol, bool AlwaysInline, const CallInst *CI, std::optional< bool > OverrideTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo=AAMDNodes(), BatchAAResults *BatchAA=nullptr)
std::pair< SDValue, SDValue > SplitVectorOperand(const SDNode *N, unsigned OpNo)
Split the node's operand with EXTRACT_SUBVECTOR and return the low/high part.
SDValue getNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a bitwise NOT operation as (XOR Val, -1).
const TargetLowering & getTargetLoweringInfo() const
std::pair< EVT, EVT > GetSplitDestVTs(const EVT &VT) const
Compute the VTs needed for the low/hi parts of a type which is split (or expanded) into two not neces...
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
SDValue getCALLSEQ_END(SDValue Chain, SDValue Op1, SDValue Op2, SDValue InGlue, const SDLoc &DL)
Return a new CALLSEQ_END node, which always must have a glue result (to ensure it's not CSE'd).
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
SDValue getBitcastedAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by first bitcasting (from potentia...
SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, Register Reg, EVT VT)
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS, SDNodeFlags Flags=SDNodeFlags())
Helper function to make it easier to build Select's if you just have operands and don't want to check...
void setNodeMemRefs(MachineSDNode *N, ArrayRef< MachineMemOperand * > NewMemRefs)
Mutate the specified machine node's memory references to the provided list.
SDValue getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return the expression required to zero extend the Op value assuming it was the smaller SrcTy value.
const DataLayout & getDataLayout() const
SDValue getTokenFactor(const SDLoc &DL, SmallVectorImpl< SDValue > &Vals)
Creates a new TokenFactor containing Vals.
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
SDValue getSignedTargetConstant(int64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
void ReplaceAllUsesWith(SDValue From, SDValue To)
Modify anything using 'From' to use 'To' instead.
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
SDValue getSignedConstant(int64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
SDValue getCALLSEQ_START(SDValue Chain, uint64_t InSize, uint64_t OutSize, const SDLoc &DL)
Return a new CALLSEQ_START node, that starts new call frame, in which InSize bytes are set up inside ...
void RemoveDeadNode(SDNode *N)
Remove the specified node from the system.
SDValue getTargetExtractSubreg(int SRIdx, const SDLoc &DL, EVT VT, SDValue Operand)
A convenience function for creating TargetInstrInfo::EXTRACT_SUBREG nodes.
SDValue getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either sign-extending or trunca...
const TargetMachine & getTarget() const
SDValue getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either any-extending or truncat...
SDValue getSelectCC(const SDLoc &DL, SDValue LHS, SDValue RHS, SDValue True, SDValue False, ISD::CondCode Cond)
Helper function to make it easier to build SelectCC's if you just have an ISD::CondCode instead of an...
SDValue getValueType(EVT)
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
bool isKnownNeverNaN(SDValue Op, bool SNaN=false, unsigned Depth=0) const
Test whether the given SDValue (or all elements of it, if it is a vector) is known to never be NaN.
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
unsigned ComputeNumSignBits(SDValue Op, unsigned Depth=0) const
Return the number of times the sign bit of the register is replicated into the other bits.
bool isBaseWithConstantOffset(SDValue Op) const
Return true if the specified operand is an ISD::ADD with a ConstantSDNode on the right-hand side,...
SDValue getVectorIdxConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
void ReplaceAllUsesOfValueWith(SDValue From, SDValue To)
Replace any uses of From with To, leaving uses of other values produced by From.getNode() alone.
MachineFunction & getMachineFunction() const
SDValue getSplatBuildVector(EVT VT, const SDLoc &DL, SDValue Op)
Return a splat ISD::BUILD_VECTOR node, consisting of Op splatted to all elements.
SDValue getFrameIndex(int FI, EVT VT, bool isTarget=false)
KnownBits computeKnownBits(SDValue Op, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in Known.
SDValue getRegisterMask(const uint32_t *RegMask)
SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
SDValue getCondCode(ISD::CondCode Cond)
bool MaskedValueIsZero(SDValue Op, const APInt &Mask, unsigned Depth=0) const
Return true if 'Op & Mask' is known to be zero.
SDValue getObjectPtrOffset(const SDLoc &SL, SDValue Ptr, TypeSize Offset)
Create an add instruction with appropriate flags when used for addressing some offset of an object.
LLVMContext * getContext() const
const SDValue & setRoot(SDValue N)
Set the current root tag of the SelectionDAG.
SDValue getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef< SDValue > Ops, EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags Flags=MachineMemOperand::MOLoad|MachineMemOperand::MOStore, LocationSize Size=0, const AAMDNodes &AAInfo=AAMDNodes())
Creates a MemIntrinsicNode that may produce a result and takes a list of operands.
SDNode * UpdateNodeOperands(SDNode *N, SDValue Op)
Mutate the specified node in-place to have the specified operands.
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
std::pair< SDValue, SDValue > SplitScalar(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the scalar node with EXTRACT_ELEMENT using the provided VTs and return the low/high part.
SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
This SDNode is used to implement the code generator support for the llvm IR shufflevector instruction...
int getMaskElt(unsigned Idx) const
ArrayRef< int > getMask() const
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
iterator insert(iterator I, T &&Elt)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
An instruction for storing to memory.
This class is used to represent ISD::STORE nodes.
A wrapper around a string literal that serves as a proxy for constructing global tables of StringRefs...
StringRef - Represent a constant reference to a string, i.e.
bool starts_with(StringRef Prefix) const
Check if this string starts with the given Prefix.
constexpr size_t size() const
size - Get the string size.
constexpr const char * data() const
data - Get a pointer to the start of the string (which may not be null terminated).
bool ends_with(StringRef Suffix) const
Check if this string ends with the given Suffix.
A switch()-like statement whose cases are string literals.
StringSwitch & Case(StringLiteral S, T Value)
Information about stack frame layout on the target.
Align getStackAlign() const
getStackAlignment - This method returns the number of bytes to which the stack pointer must be aligne...
StackDirection getStackGrowthDirection() const
getStackGrowthDirection - Return the direction the stack grows
TargetInstrInfo - Interface to description of machine instruction set.
void setBooleanVectorContents(BooleanContent Ty)
Specify how the target extends the result of a vector boolean value from a vector of i1 to a wider ty...
void setOperationAction(unsigned Op, MVT VT, LegalizeAction Action)
Indicate that the specified operation does not work with the specified type and indicate what to do a...
virtual void finalizeLowering(MachineFunction &MF) const
Execute target specific actions to finalize target lowering.
EVT getValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the EVT corresponding to this LLVM type.
virtual const TargetRegisterClass * getRegClassFor(MVT VT, bool isDivergent=false) const
Return the register class that should be used for the specified value type.
const TargetMachine & getTargetMachine() const
virtual unsigned getNumRegistersForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const
Certain targets require unusual breakdowns of certain types.
virtual MVT getRegisterTypeForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const
Certain combinations of ABIs, Targets and features require that types are legal for some operations a...
LegalizeTypeAction
This enum indicates whether a types are legal for a target, and if not, what action should be used to...
void setHasExtractBitsInsn(bool hasExtractInsn=true)
Tells the code generator that the target has BitExtract instructions.
virtual TargetLoweringBase::LegalizeTypeAction getPreferredVectorAction(MVT VT) const
Return the preferred vector type legalization action.
virtual unsigned getVectorTypeBreakdownForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT, EVT &IntermediateVT, unsigned &NumIntermediates, MVT &RegisterVT) const
Certain targets such as MIPS require that some types such as vectors are always broken down into scal...
Register getStackPointerRegisterToSaveRestore() const
If a physical register, this specifies the register that llvm.savestack/llvm.restorestack should save...
void setBooleanContents(BooleanContent Ty)
Specify how the target extends the result of integer and floating point boolean values from i1 to a w...
virtual Align getPrefLoopAlignment(MachineLoop *ML=nullptr) const
Return the preferred loop alignment.
void computeRegisterProperties(const TargetRegisterInfo *TRI)
Once all of the register classes are added, this allows us to compute derived properties we expose.
void addRegisterClass(MVT VT, const TargetRegisterClass *RC)
Add the specified register class as an available regclass for the specified value type.
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
bool isOperationLegal(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target.
void setTruncStoreAction(MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified truncating store does not work with the specified type and indicate what ...
@ ZeroOrOneBooleanContent
bool isOperationLegalOrCustom(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
void setStackPointerRegisterToSaveRestore(Register R)
If set to a physical register, this specifies the register that llvm.savestack/llvm....
void AddPromotedToType(unsigned Opc, MVT OrigVT, MVT DestVT)
If Opc/OrigVT is specified as being promoted, the promotion code defaults to trying a larger integer/...
AtomicExpansionKind
Enum that specifies what an atomic load/AtomicRMWInst is expanded to, if at all.
void setTargetDAGCombine(ArrayRef< ISD::NodeType > NTs)
Targets should invoke this method for each target independent node that they want to provide a custom...
bool allowsMemoryAccessForAlignment(LLVMContext &Context, const DataLayout &DL, EVT VT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *Fast=nullptr) const
This function returns true if the memory access is aligned or if the target allows this specific unal...
virtual MVT getPointerMemTy(const DataLayout &DL, uint32_t AS=0) const
Return the in-memory pointer type for the given address space, defaults to the pointer type from the ...
void setSchedulingPreference(Sched::Preference Pref)
Specify the target scheduling preference.
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
virtual void computeKnownBitsForFrameIndex(int FIOp, KnownBits &Known, const MachineFunction &MF) const
Determine which of the bits of FrameIndex FIOp are known to be 0.
SDValue scalarizeVectorStore(StoreSDNode *ST, SelectionDAG &DAG) const
std::vector< AsmOperandInfo > AsmOperandInfoVector
SDValue SimplifyMultipleUseDemandedBits(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, SelectionDAG &DAG, unsigned Depth=0) const
More limited version of SimplifyDemandedBits that can be used to "look through" ops that don't contri...
SDValue expandUnalignedStore(StoreSDNode *ST, SelectionDAG &DAG) const
Expands an unaligned store to 2 half-size stores for integer values, and possibly more for vectors.
virtual ConstraintType getConstraintType(StringRef Constraint) const
Given a constraint, return the type of constraint it is for this target.
bool parametersInCSRMatch(const MachineRegisterInfo &MRI, const uint32_t *CallerPreservedMask, const SmallVectorImpl< CCValAssign > &ArgLocs, const SmallVectorImpl< SDValue > &OutVals) const
Check whether parameters to a call that are passed in callee saved registers are the same as from the...
std::pair< SDValue, SDValue > expandUnalignedLoad(LoadSDNode *LD, SelectionDAG &DAG) const
Expands an unaligned load to 2 half-size loads for an integer, and possibly more for vectors.
virtual bool isTypeDesirableForOp(unsigned, EVT VT) const
Return true if the target has native support for the specified value type and it is 'desirable' to us...
std::pair< SDValue, SDValue > scalarizeVectorLoad(LoadSDNode *LD, SelectionDAG &DAG) const
Turn load of vector type into a load of the individual elements.
virtual std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const
Given a physical register constraint (e.g.
bool SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth=0, bool AssumeSingleUse=false) const
Look at Op.
virtual MachineBasicBlock * EmitInstrWithCustomInserter(MachineInstr &MI, MachineBasicBlock *MBB) const
This method should be implemented by targets that mark instructions with the 'usesCustomInserter' fla...
virtual AsmOperandInfoVector ParseConstraints(const DataLayout &DL, const TargetRegisterInfo *TRI, const CallBase &Call) const
Split up the constraint string from the inline assembly value into the specific constraints and their...
virtual void ComputeConstraintToUse(AsmOperandInfo &OpInfo, SDValue Op, SelectionDAG *DAG=nullptr) const
Determines the constraint code and constraint type to use for the specific AsmOperandInfo,...
virtual void LowerAsmOperandForConstraint(SDValue Op, StringRef Constraint, std::vector< SDValue > &Ops, SelectionDAG &DAG) const
Lower the specified operand into the Ops vector.
SDValue expandFMINNUM_FMAXNUM(SDNode *N, SelectionDAG &DAG) const
Expand fminnum/fmaxnum into fminnum_ieee/fmaxnum_ieee with quieted inputs.
Primary interface to the complete machine description for the target machine.
CodeGenOptLevel getOptLevel() const
Returns the optimization level: None, Less, Default, or Aggressive.
const Triple & getTargetTriple() const
bool shouldAssumeDSOLocal(const GlobalValue *GV) const
unsigned UnsafeFPMath
UnsafeFPMath - This flag is enabled when the -enable-unsafe-fp-math flag is specified on the command ...
unsigned GuaranteedTailCallOpt
GuaranteedTailCallOpt - This flag is enabled when -tailcallopt is specified on the commandline.
unsigned getID() const
Return the register class ID number.
MCRegister getRegister(unsigned i) const
Return the specified register in the class.
int getCopyCost() const
Return the cost of copying a value between two registers in this class.
iterator begin() const
begin/end - Return all of the registers in this class.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
Target - Wrapper for Target specific information.
Triple - Helper class for working with autoconf configuration names.
OSType getOS() const
Get the parsed operating system type of this triple.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
static constexpr TypeSize getFixed(ScalarTy ExactSize)
The instances of the Type class are immutable: once they are created, they are never changed.
const fltSemantics & getFltSemantics() const
bool isFloatTy() const
Return true if this is 'float', a 32-bit IEEE fp type.
bool isBFloatTy() const
Return true if this is 'bfloat', a 16-bit bfloat type.
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
bool isSized(SmallPtrSetImpl< Type * > *Visited=nullptr) const
Return true if it makes sense to take the size of this type.
bool isHalfTy() const
Return true if this is 'half', a 16-bit IEEE fp type.
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
bool isDoubleTy() const
Return true if this is 'double', a 64-bit IEEE fp type.
bool isFunctionTy() const
True if this is an instance of FunctionType.
static IntegerType * getInt32Ty(LLVMContext &C)
bool isIntegerTy() const
True if this is an instance of IntegerType.
bool isVoidTy() const
Return true if this is 'void'.
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
A Use represents the edge between a Value definition and its users.
User * getUser() const
Returns the User that contains this Use.
unsigned getOperandNo() const
Return the operand # of this use in its User.
const Use & getOperandUse(unsigned i) const
Value * getOperand(unsigned i) const
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
bool hasOneUse() const
Return true if there is exactly one use of this value.
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
iterator_range< user_iterator > users()
LLVMContext & getContext() const
All values hold a context through their type.
iterator_range< use_iterator > uses()
void takeName(Value *V)
Transfer the name from V to this value.
Type * getElementType() const
constexpr bool isZero() const
const ParentTy * getParent() const
self_iterator getIterator()
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ CONSTANT_ADDRESS_32BIT
Address space for 32-bit constant memory.
@ BUFFER_STRIDED_POINTER
Address space for 192-bit fat buffer pointers with an additional index.
@ REGION_ADDRESS
Address space for region memory. (GDS)
@ LOCAL_ADDRESS
Address space for local memory.
@ STREAMOUT_REGISTER
Internal address spaces. Can be freely renumbered.
@ CONSTANT_ADDRESS
Address space for constant memory (VTX2).
@ FLAT_ADDRESS
Address space for flat memory.
@ GLOBAL_ADDRESS
Address space for global memory (RAT0, VTX0).
@ BUFFER_FAT_POINTER
Address space for 160-bit buffer fat pointers.
@ PRIVATE_ADDRESS
Address space for private memory.
@ BUFFER_RESOURCE
Address space for 128-bit buffer resources.
@ BUFFER_ATOMIC_COND_SUB_U32
@ TBUFFER_LOAD_FORMAT_D16
@ TBUFFER_STORE_FORMAT_D16
@ BUFFER_STORE_FORMAT_D16
@ CLAMP
CLAMP value between 0.0 and 1.0.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
constexpr char SymbolName[]
Key for Kernel::Metadata::mSymbolName.
bool isInlinableLiteralBF16(int16_t Literal, bool HasInv2Pi)
LLVM_READONLY const MIMGG16MappingInfo * getMIMGG16MappingInfo(unsigned G)
LLVM_READONLY int getGlobalSaddrOp(uint16_t Opcode)
bool isInlinableLiteralFP16(int16_t Literal, bool HasInv2Pi)
int getMIMGOpcode(unsigned BaseOpcode, unsigned MIMGEncoding, unsigned VDataDwords, unsigned VAddrDwords)
LLVM_READNONE bool isLegalDPALU_DPPControl(unsigned DC)
bool shouldEmitConstantsToTextSection(const Triple &TT)
bool isFlatGlobalAddrSpace(unsigned AS)
LLVM_READONLY int16_t getNamedOperandIdx(uint16_t Opcode, uint16_t NamedIdx)
const uint64_t FltRoundToHWConversionTable
bool isGFX12Plus(const MCSubtargetInfo &STI)
bool isEntryFunctionCC(CallingConv::ID CC)
LLVM_READNONE bool isKernel(CallingConv::ID CC)
bool isGFX11(const MCSubtargetInfo &STI)
bool isCompute(CallingConv::ID cc)
unsigned getAMDHSACodeObjectVersion(const Module &M)
bool isChainCC(CallingConv::ID CC)
bool isInlinableLiteral32(int32_t Literal, bool HasInv2Pi)
bool isIntrinsicSourceOfDivergence(unsigned IntrID)
LLVM_READONLY bool hasNamedOperand(uint64_t Opcode, uint64_t NamedIdx)
LLVM_READNONE bool isInlinableIntLiteral(int64_t Literal)
Is this literal inlinable, and not one of the values intended for floating point values.
bool getMUBUFTfe(unsigned Opc)
bool isGFX11Plus(const MCSubtargetInfo &STI)
std::optional< unsigned > getInlineEncodingV2F16(uint32_t Literal)
bool isShader(CallingConv::ID cc)
bool isGFX10Plus(const MCSubtargetInfo &STI)
LLVM_READONLY int getVOPe64(uint16_t Opcode)
@ TowardZeroF32_TowardNegativeF64
std::optional< unsigned > getInlineEncodingV2I16(uint32_t Literal)
uint32_t decodeFltRoundToHWConversionTable(uint32_t FltRounds)
Read the hardware rounding mode equivalent of a AMDGPUFltRounds value.
bool isExtendedGlobalAddrSpace(unsigned AS)
LLVM_READONLY const MIMGDimInfo * getMIMGDimInfo(unsigned DimEnum)
std::optional< unsigned > getInlineEncodingV2BF16(uint32_t Literal)
LLVM_READONLY const MIMGBaseOpcodeInfo * getMIMGBaseOpcodeInfo(unsigned BaseOpcode)
int getMaskedMIMGOp(unsigned Opc, unsigned NewChannels)
const ImageDimIntrinsicInfo * getImageDimIntrinsicInfo(unsigned Intr)
bool isInlinableLiteralI16(int32_t Literal, bool HasInv2Pi)
bool isInlinableLiteral64(int64_t Literal, bool HasInv2Pi)
Is this literal inlinable.
const RsrcIntrinsic * lookupRsrcIntrinsic(unsigned Intr)
bool isGraphics(CallingConv::ID cc)
const uint64_t FltRoundConversionTable
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ AMDGPU_CS
Used for Mesa/AMDPAL compute shaders.
@ AMDGPU_KERNEL
Used for AMDGPU code object kernels.
@ MaxID
The highest possible ID. Must be some 2^k - 1.
@ AMDGPU_Gfx
Used for AMD graphics targets.
@ AMDGPU_CS_ChainPreserve
Used on AMDGPUs to give the middle-end more control over argument placement.
@ AMDGPU_CS_Chain
Used on AMDGPUs to give the middle-end more control over argument placement.
@ AMDGPU_PS
Used for Mesa/AMDPAL pixel shaders.
@ Fast
Attempts to make calls as fast as possible (e.g.
@ C
The default llvm calling convention, compatible with C.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
@ STACKSAVE
STACKSAVE - STACKSAVE has one operand, an input chain.
@ DELETED_NODE
DELETED_NODE - This is an illegal value that is used to catch errors.
@ SET_FPENV
Sets the current floating-point environment.
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
@ BSWAP
Byte Swap and Counting operators.
@ ATOMIC_STORE
OUTCHAIN = ATOMIC_STORE(INCHAIN, val, ptr) This corresponds to "store atomic" instruction.
@ FMAD
FMAD - Perform a * b + c, while getting the same result as the separately rounded operations.
@ ADD
Simple integer binary arithmetic operators.
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
@ ATOMIC_CMP_SWAP_WITH_SUCCESS
Val, Success, OUTCHAIN = ATOMIC_CMP_SWAP_WITH_SUCCESS(INCHAIN, ptr, cmp, swap) N.b.
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
@ FADD
Simple binary floating point operators.
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
@ FP16_TO_FP
FP16_TO_FP, FP_TO_FP16 - These operators are used to perform promotions and truncation for half-preci...
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
@ FLDEXP
FLDEXP - ldexp, inspired by libm (op0 * 2**op1).
@ BUILTIN_OP_END
BUILTIN_OP_END - This must be the last enum value in this list.
@ SET_ROUNDING
Set rounding mode.
@ SIGN_EXTEND
Conversion operators.
@ SCALAR_TO_VECTOR
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
@ READSTEADYCOUNTER
READSTEADYCOUNTER - This corresponds to the readfixedcounter intrinsic.
@ BR
Control flow instructions. These all have token chains.
@ CTTZ_ZERO_UNDEF
Bit counting operators with an undefined result for zero inputs.
@ PREFETCH
PREFETCH - This corresponds to a prefetch intrinsic.
@ FSINCOS
FSINCOS - Compute both fsin and fcos as a single operation.
@ FNEG
Perform various unary floating-point operations inspired by libm.
@ BR_CC
BR_CC - Conditional branch.
@ FCANONICALIZE
Returns platform specific canonical encoding of a floating point number.
@ IS_FPCLASS
Performs a check of floating point class property, defined by IEEE-754.
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
@ ATOMIC_LOAD
Val, OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr) This corresponds to "load atomic" instruction.
@ UNDEF
UNDEF - An undefined node.
@ EXTRACT_ELEMENT
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant,...
@ CopyFromReg
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
@ GET_ROUNDING
Returns current rounding mode: -1 Undefined 0 Round to 0 1 Round to nearest, ties to even 2 Round to ...
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
@ GET_FPMODE
Reads the current dynamic floating-point control modes.
@ GET_FPENV
Gets the current floating-point environment.
@ SHL
Shift and rotation operations.
@ VECTOR_SHUFFLE
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
@ FMINNUM_IEEE
FMINNUM_IEEE/FMAXNUM_IEEE - Perform floating-point minimumNumber or maximumNumber on two values,...
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ CopyToReg
CopyToReg - This node has three operands: a chain, a register number to set to this value,...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ DEBUGTRAP
DEBUGTRAP - Trap intended to get the attention of a debugger.
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
@ ATOMIC_CMP_SWAP
Val, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap) For double-word atomic operations: ValLo,...
@ FMINNUM
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
@ SMULO
Same for multiplication.
@ DYNAMIC_STACKALLOC
DYNAMIC_STACKALLOC - Allocate some number of bytes on the stack aligned to a specified boundary.
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ UADDO_CARRY
Carry-using nodes for multiple precision addition and subtraction.
@ INLINEASM_BR
INLINEASM_BR - Branching version of inline asm. Used by asm-goto.
@ BF16_TO_FP
BF16_TO_FP, FP_TO_BF16 - These operators are used to perform promotions and truncation for bfloat16.
@ STRICT_FP_ROUND
X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision ...
@ FMINIMUM
FMINIMUM/FMAXIMUM - NaN-propagating minimum/maximum that also treat -0.0 as less than 0....
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ READCYCLECOUNTER
READCYCLECOUNTER - This corresponds to the readcyclecounter intrinsic.
@ STRICT_FP_EXTEND
X = STRICT_FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ TRAP
TRAP - Trapping instruction.
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
@ ATOMIC_SWAP
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN,...
@ FFREXP
FFREXP - frexp, extract fractional and exponent component of a floating-point value.
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
@ ADDRSPACECAST
ADDRSPACECAST - This operator converts between pointers of different address spaces.
@ INLINEASM
INLINEASM - Represents an inline asm block.
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ BRCOND
BRCOND - Conditional branch.
@ SHL_PARTS
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations.
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
@ FMINIMUMNUM
FMINIMUMNUM/FMAXIMUMNUM - minimumnum/maximumnum that is same with FMINNUM_IEEE and FMAXNUM_IEEE besid...
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
CondCode getSetCCSwappedOperands(CondCode Operation)
Return the operation corresponding to (Y op X) when given the operation for (X op Y).
bool isSignedIntSetCC(CondCode Code)
Return true if this is a setcc instruction that performs a signed comparison when used with integer o...
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
AttributeList getAttributes(LLVMContext &C, ID id)
Return the attributes for an intrinsic.
Function * getDeclarationIfExists(Module *M, ID id, ArrayRef< Type * > Tys, FunctionType *FT=nullptr)
This version supports overloaded intrinsics.
GFCstOrSplatGFCstMatch m_GFCstOrSplat(std::optional< FPValueAndVReg > &FPValReg)
@ Implicit
Not emitted register (e.g. carry, or temporary result).
@ Define
Register definition.
@ Kill
The last use of a register.
@ Undef
Value of the register doesn't matter.
@ System
Synchronized with respect to all concurrently executing threads.
Reg
All possible values of the reg field in the ModR/M byte.
initializer< Ty > init(const Ty &Val)
This is an optimization pass for GlobalISel generic memory operations.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
@ Low
Lower the current thread's priority such that it does not affect foreground tasks significantly.
ISD::CondCode getICmpCondCode(ICmpInst::Predicate Pred)
getICmpCondCode - Return the ISD condition code corresponding to the given LLVM IR integer condition ...
void finalizeBundle(MachineBasicBlock &MBB, MachineBasicBlock::instr_iterator FirstMI, MachineBasicBlock::instr_iterator LastMI)
finalizeBundle - Finalize a machine instruction bundle which includes a sequence of instructions star...
int64_t maxIntN(int64_t N)
Gets the maximum value for a N-bit signed integer.
int popcount(T Value) noexcept
Count the number of set bits in a value.
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
detail::zippy< detail::zip_first, T, U, Args... > zip_equal(T &&t, U &&u, Args &&...args)
zip iterator that assumes that all iteratees have the same length.
bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
std::pair< Value *, Value * > buildCmpXchgValue(IRBuilderBase &Builder, Value *Ptr, Value *Cmp, Value *Val, Align Alignment)
Emit IR to implement the given cmpxchg operation on values in registers, returning the new value.
SDValue peekThroughBitcasts(SDValue V)
Return the non-bitcasted source operand of V if it exists.
testing::Matcher< const detail::ErrorHolder & > Failed()
int bit_width(T Value)
Returns the number of bits needed to represent Value if Value is nonzero.
void append_range(Container &C, Range &&R)
Wrapper function to append range R to container C.
constexpr T alignDown(U Value, V Align, W Skew=0)
Returns the largest unsigned integer less than or equal to Value and is Skew mod Align.
ConstantFPSDNode * isConstOrConstSplatFP(SDValue N, bool AllowUndefs=false)
Returns the SDNode if it is a constant splat BuildVector or constant float.
uint64_t PowerOf2Ceil(uint64_t A)
Returns the power of two which is greater than or equal to the given value.
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
constexpr bool isShiftedMask_64(uint64_t Value)
Return true if the argument contains a non-empty sequence of ones with the remainder zero (64 bit ver...
bool isReleaseOrStronger(AtomicOrdering AO)
static const MachineMemOperand::Flags MONoClobber
Mark the MMO of a uniform load if there are no potentially clobbering stores on any path from the sta...
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
int countl_zero(T Val)
Count number of 0's from the most significant bit to the least stopping at the first 1.
bool isBoolSGPR(SDValue V)
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
constexpr uint32_t Hi_32(uint64_t Value)
Return the high 32 bits of a 64 bit value.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
ISD::CondCode getFCmpCondCode(FCmpInst::Predicate Pred)
getFCmpCondCode - Return the ISD condition code corresponding to the given LLVM IR floating-point con...
constexpr uint32_t Lo_32(uint64_t Value)
Return the low 32 bits of a 64 bit value.
Value * buildAtomicRMWValue(AtomicRMWInst::BinOp Op, IRBuilderBase &Builder, Value *Loaded, Value *Val)
Emit IR to implement the given atomicrmw operation on values in registers, returning the new value.
constexpr T divideCeil(U Numerator, V Denominator)
Returns the integer ceil(Numerator / Denominator).
@ First
Helpers to iterate all locations in the MemoryEffectsBase class.
unsigned getUndefRegState(bool B)
bool CCAssignFn(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
CCAssignFn - This function assigns a location for Val, updating State to reflect the change.
@ Or
Bitwise or logical OR of integers.
@ Mul
Product of integers.
uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
DWARFExpression::Operation Op
unsigned M0(unsigned Val)
int64_t minIntN(int64_t N)
Gets the minimum value for a N-bit signed integer.
ConstantSDNode * isConstOrConstSplat(SDValue N, bool AllowUndefs=false, bool AllowTruncation=false)
Returns the SDNode if it is a constant splat BuildVector or constant int.
constexpr unsigned BitWidth
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
static const MachineMemOperand::Flags MOLastUse
Mark the MMO of a load as the last use.
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
Printable printReg(Register Reg, const TargetRegisterInfo *TRI=nullptr, unsigned SubIdx=0, const MachineRegisterInfo *MRI=nullptr)
Prints virtual and physical registers with or without a TRI instance.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
ArgDescriptor WorkItemIDZ
ArgDescriptor WorkItemIDY
std::tuple< const ArgDescriptor *, const TargetRegisterClass *, LLT > getPreloadedValue(PreloadedValue Value) const
ArgDescriptor WorkItemIDX
static constexpr uint64_t encode(Fields... Values)
static std::tuple< typename Fields::ValueType... > decode(uint64_t Encoded)
static constexpr roundingMode rmNearestTiesToEven
static const fltSemantics & IEEEhalf() LLVM_READNONE
This struct is a compact representation of a valid (non-zero power of two) alignment.
uint64_t value() const
This is a hole in the type system and should not be abused.
static ArgDescriptor createStack(unsigned Offset, unsigned Mask=~0u)
MCRegister getRegister() const
static ArgDescriptor createArg(const ArgDescriptor &Arg, unsigned Mask)
static ArgDescriptor createRegister(Register Reg, unsigned Mask=~0u)
Helper struct shared between Function Specialization and SCCP Solver.
Represent subnormal handling kind for floating point instruction inputs and outputs.
@ Dynamic
Denormals have unknown treatment.
static constexpr DenormalMode getPreserveSign()
static constexpr DenormalMode getIEEE()
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
EVT changeTypeToInteger() const
Return the type converted to an equivalently sized integer or vector with integer element type.
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
bool isByteSized() const
Return true if the bit size is a multiple of 8.
EVT changeElementType(EVT EltVT) const
Return a VT for a type whose attributes match ourselves with the exception of the element type that i...
uint64_t getScalarSizeInBits() const
bool isPow2VectorType() const
Returns true if the given vector is a power of 2.
TypeSize getStoreSizeInBits() const
Return the number of bits overwritten by a store of the specified value type.
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
bool isVector() const
Return true if this is a vector value type.
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
bool bitsEq(EVT VT) const
Return true if this has the same number of bits as VT.
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
bool isScalarInteger() const
Return true if this is an integer, but not a vector.
const fltSemantics & getFltSemantics() const
Returns an APFloat semantics tag appropriate for the value type.
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
bool isInteger() const
Return true if this is an integer or a vector integer type.
unsigned getPointerAddrSpace() const
unsigned getByValSize() const
bool isUnknown() const
Returns true if we don't know any bits.
void resetAll()
Resets the known state of all bits.
static KnownBits add(const KnownBits &LHS, const KnownBits &RHS, bool NSW=false, bool NUW=false)
Compute knownbits resulting from addition of LHS and RHS.
unsigned countMinLeadingZeros() const
Returns the minimum number of leading zero bits.
This class contains a discriminated union of information about pointers in memory operands,...
static MachinePointerInfo getStack(MachineFunction &MF, int64_t Offset, uint8_t ID=0)
Stack pointer relative access.
int64_t Offset
Offset - This is an offset from the base Value*.
PointerUnion< const Value *, const PseudoSourceValue * > V
This is the IR pointer value for the access, or it is null if unknown.
static MachinePointerInfo getGOT(MachineFunction &MF)
Return a MachinePointerInfo record that refers to a GOT entry.
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
These are IR-level optimization flags that may be propagated to SDNodes.
bool hasNoUnsignedWrap() const
bool hasAllowContract() const
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
bool DX10Clamp
Used by the vector ALU to force DX10-style treatment of NaNs: when set, clamp NaN to zero; otherwise,...
This represents an addressing mode of: BaseGV + BaseOffs + BaseReg + Scale*ScaleReg + ScalableOffset*...
This structure contains all information that is necessary for lowering calls.
SDValue ConvergenceControlToken
SmallVector< ISD::InputArg, 32 > Ins
SmallVector< ISD::OutputArg, 32 > Outs
SmallVector< SDValue, 32 > OutVals
bool isBeforeLegalize() const