LLVM 24.0.0git
AMDGPUBaseInfo.cpp
Go to the documentation of this file.
1//===- AMDGPUBaseInfo.cpp - AMDGPU Base encoding information --------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "AMDGPUBaseInfo.h"
10#include "AMDGPU.h"
11#include "AMDGPUAsmUtils.h"
12#include "AMDKernelCodeT.h"
17#include "llvm/IR/Attributes.h"
18#include "llvm/IR/Constants.h"
19#include "llvm/IR/Function.h"
20#include "llvm/IR/GlobalValue.h"
22#include "llvm/IR/IntrinsicsAMDGPU.h"
23#include "llvm/IR/IntrinsicsR600.h"
24#include "llvm/IR/LLVMContext.h"
25#include "llvm/IR/Metadata.h"
26#include "llvm/MC/MCInstrInfo.h"
31#include <optional>
32
33#define GET_INSTRINFO_NAMED_OPS
34#define GET_INSTRMAP_INFO
35#include "AMDGPUGenInstrInfo.inc"
36
38 "amdhsa-code-object-version", llvm::cl::Hidden,
40 llvm::cl::desc("Set default AMDHSA Code Object Version (module flag "
41 "or asm directive still take priority if present)"));
42
43namespace {
44
45/// \returns Bit mask for given bit \p Shift and bit \p Width.
46unsigned getBitMask(unsigned Shift, unsigned Width) {
47 return ((1 << Width) - 1) << Shift;
48}
49
50/// Packs \p Src into \p Dst for given bit \p Shift and bit \p Width.
51///
52/// \returns Packed \p Dst.
53unsigned packBits(unsigned Src, unsigned Dst, unsigned Shift, unsigned Width) {
54 unsigned Mask = getBitMask(Shift, Width);
55 return ((Src << Shift) & Mask) | (Dst & ~Mask);
56}
57
58/// Unpacks bits from \p Src for given bit \p Shift and bit \p Width.
59///
60/// \returns Unpacked bits.
61unsigned unpackBits(unsigned Src, unsigned Shift, unsigned Width) {
62 return (Src & getBitMask(Shift, Width)) >> Shift;
63}
64
65/// \returns Vmcnt bit shift (lower bits).
66unsigned getVmcntBitShiftLo(unsigned VersionMajor) {
67 return VersionMajor >= 11 ? 10 : 0;
68}
69
70/// \returns Vmcnt bit width (lower bits).
71unsigned getVmcntBitWidthLo(unsigned VersionMajor) {
72 return VersionMajor >= 11 ? 6 : 4;
73}
74
75/// \returns Expcnt bit shift.
76unsigned getExpcntBitShift(unsigned VersionMajor) {
77 return VersionMajor >= 11 ? 0 : 4;
78}
79
80/// \returns Expcnt bit width.
81unsigned getExpcntBitWidth(unsigned VersionMajor) { return 3; }
82
83/// \returns Lgkmcnt bit shift.
84unsigned getLgkmcntBitShift(unsigned VersionMajor) {
85 return VersionMajor >= 11 ? 4 : 8;
86}
87
88/// \returns Lgkmcnt bit width.
89unsigned getLgkmcntBitWidth(unsigned VersionMajor) {
90 return VersionMajor >= 10 ? 6 : 4;
91}
92
93/// \returns Vmcnt bit shift (higher bits).
94unsigned getVmcntBitShiftHi(unsigned VersionMajor) { return 14; }
95
96/// \returns Vmcnt bit width (higher bits).
97unsigned getVmcntBitWidthHi(unsigned VersionMajor) {
98 return (VersionMajor == 9 || VersionMajor == 10) ? 2 : 0;
99}
100
101/// \returns Loadcnt bit width
102unsigned getLoadcntBitWidth(unsigned VersionMajor) {
103 return VersionMajor >= 12 ? 6 : 0;
104}
105
106/// \returns Samplecnt bit width.
107unsigned getSamplecntBitWidth(unsigned VersionMajor) {
108 return VersionMajor >= 12 ? 6 : 0;
109}
110
111/// \returns Bvhcnt bit width.
112unsigned getBvhcntBitWidth(unsigned VersionMajor) {
113 return VersionMajor >= 12 ? 3 : 0;
114}
115
116/// \returns Dscnt bit width.
117unsigned getDscntBitWidth(unsigned VersionMajor) {
118 return VersionMajor >= 12 ? 6 : 0;
119}
120
121/// \returns Dscnt bit shift in combined S_WAIT instructions.
122unsigned getDscntBitShift(unsigned VersionMajor) { return 0; }
123
124/// \returns Storecnt or Vscnt bit width, depending on VersionMajor.
125unsigned getStorecntBitWidth(unsigned VersionMajor) {
126 return VersionMajor >= 10 ? 6 : 0;
127}
128
129/// \returns Kmcnt bit width.
130unsigned getKmcntBitWidth(unsigned VersionMajor) {
131 return VersionMajor >= 12 ? 5 : 0;
132}
133
134/// \returns Xcnt bit width.
135unsigned getXcntBitWidth(unsigned VersionMajor, unsigned VersionMinor) {
136 return VersionMajor == 12 && VersionMinor == 5 ? 6 : 0;
137}
138
139/// \returns Asynccnt bit width.
140unsigned getAsynccntBitWidth(unsigned VersionMajor, unsigned VersionMinor) {
141 return VersionMajor == 12 && VersionMinor == 5 ? 6 : 0;
142}
143
144/// \returns shift for Loadcnt/Storecnt in combined S_WAIT instructions.
145unsigned getLoadcntStorecntBitShift(unsigned VersionMajor) {
146 return VersionMajor >= 12 ? 8 : 0;
147}
148
149/// \returns barrier member count bit width in M0.
150unsigned getBarrierMemberCountBitWidth(unsigned VersionMajor) {
151 return VersionMajor >= 13 ? 8 : 6;
152}
153
154/// \returns VaSdst bit width
155inline unsigned getVaSdstBitWidth() { return 3; }
156
157/// \returns VaSdst bit shift
158inline unsigned getVaSdstBitShift() { return 9; }
159
160/// \returns VmVsrc bit width
161inline unsigned getVmVsrcBitWidth() { return 3; }
162
163/// \returns VmVsrc bit shift
164inline unsigned getVmVsrcBitShift() { return 2; }
165
166/// \returns VaVdst bit width
167inline unsigned getVaVdstBitWidth() { return 4; }
168
169/// \returns VaVdst bit shift
170inline unsigned getVaVdstBitShift() { return 12; }
171
172/// \returns VaVcc bit width
173inline unsigned getVaVccBitWidth() { return 1; }
174
175/// \returns VaVcc bit shift
176inline unsigned getVaVccBitShift() { return 1; }
177
178/// \returns SaSdst bit width
179inline unsigned getSaSdstBitWidth() { return 1; }
180
181/// \returns SaSdst bit shift
182inline unsigned getSaSdstBitShift() { return 0; }
183
184/// \returns VaSsrc width
185inline unsigned getVaSsrcBitWidth() { return 1; }
186
187/// \returns VaSsrc bit shift
188inline unsigned getVaSsrcBitShift() { return 8; }
189
190/// \returns HoldCnt bit shift
191inline unsigned getHoldCntWidth(unsigned VersionMajor, unsigned VersionMinor) {
192 static constexpr const unsigned MinMajor = 10;
193 static constexpr const unsigned MinMinor = 3;
194 return std::tie(VersionMajor, VersionMinor) >= std::tie(MinMajor, MinMinor)
195 ? 1
196 : 0;
197}
198
199/// \returns HoldCnt bit shift
200inline unsigned getHoldCntBitShift() { return 7; }
201
202} // end anonymous namespace
203
204namespace llvm {
205
206namespace AMDGPU {
207
208/// \returns true if the target supports signed immediate offset for SMRD
209/// instructions.
211 return isGFX9Plus(ST);
212}
213
214/// \returns True if \p STI is AMDHSA.
215bool isHsaAbi(const MCSubtargetInfo &STI) {
216 return STI.getTargetTriple().getOS() == Triple::AMDHSA;
217}
218
221 M.getModuleFlag("amdhsa_code_object_version"))) {
222 return (unsigned)Ver->getZExtValue() / 100;
223 }
224
226}
227
231
232unsigned getAMDHSACodeObjectVersion(unsigned ABIVersion) {
233 switch (ABIVersion) {
235 return 4;
237 return 5;
239 return 6;
240 default:
242 }
243}
244
245uint8_t getELFABIVersion(const Triple &T, unsigned CodeObjectVersion) {
246 if (T.getOS() != Triple::AMDHSA)
247 return 0;
248
249 switch (CodeObjectVersion) {
250 case 4:
252 case 5:
254 case 6:
256 default:
257 report_fatal_error("Unsupported AMDHSA Code Object Version " +
258 Twine(CodeObjectVersion));
259 }
260}
261
262unsigned getMultigridSyncArgImplicitArgPosition(unsigned CodeObjectVersion) {
263 switch (CodeObjectVersion) {
264 case AMDHSA_COV4:
265 return 48;
266 case AMDHSA_COV5:
267 case AMDHSA_COV6:
268 default:
270 }
271}
272
273// FIXME: All such magic numbers about the ABI should be in a
274// central TD file.
275unsigned getHostcallImplicitArgPosition(unsigned CodeObjectVersion) {
276 switch (CodeObjectVersion) {
277 case AMDHSA_COV4:
278 return 24;
279 case AMDHSA_COV5:
280 case AMDHSA_COV6:
281 default:
283 }
284}
285
286unsigned getDefaultQueueImplicitArgPosition(unsigned CodeObjectVersion) {
287 switch (CodeObjectVersion) {
288 case AMDHSA_COV4:
289 return 32;
290 case AMDHSA_COV5:
291 case AMDHSA_COV6:
292 default:
294 }
295}
296
297unsigned getCompletionActionImplicitArgPosition(unsigned CodeObjectVersion) {
298 switch (CodeObjectVersion) {
299 case AMDHSA_COV4:
300 return 40;
301 case AMDHSA_COV5:
302 case AMDHSA_COV6:
303 default:
305 }
306}
307
308#define GET_MIMGBaseOpcodesTable_IMPL
309#define GET_MIMGDimInfoTable_IMPL
310#define GET_MIMGInfoTable_IMPL
311#define GET_MIMGLZMappingTable_IMPL
312#define GET_MIMGMIPMappingTable_IMPL
313#define GET_MIMGBiasMappingTable_IMPL
314#define GET_MIMGOffsetMappingTable_IMPL
315#define GET_MIMGG16MappingTable_IMPL
316#define GET_MAIInstInfoTable_IMPL
317#define GET_WMMAInstInfoTable_IMPL
318#include "AMDGPUGenSearchableTables.inc"
319
320int getMIMGOpcode(unsigned BaseOpcode, unsigned MIMGEncoding,
321 unsigned VDataDwords, unsigned VAddrDwords, bool IndexedRsrc,
322 bool IndexedSamp) {
323 const MIMGInfo *Info =
324 getMIMGOpcodeHelper(BaseOpcode, MIMGEncoding, VDataDwords, VAddrDwords,
325 IndexedRsrc, IndexedSamp);
326 return Info ? Info->Opcode : -1;
327}
328
330 const MIMGInfo *Info = getMIMGInfo(Opc);
331 return Info ? getMIMGBaseOpcodeInfo(Info->BaseOpcode) : nullptr;
332}
333
334int getMaskedMIMGOp(unsigned Opc, unsigned NewChannels) {
335 const MIMGInfo *OrigInfo = getMIMGInfo(Opc);
336 const MIMGInfo *NewInfo = getMIMGOpcodeHelper(
337 OrigInfo->BaseOpcode, OrigInfo->MIMGEncoding, NewChannels,
338 OrigInfo->VAddrDwords, OrigInfo->IndexedRsrc, OrigInfo->IndexedSamp);
339 return NewInfo ? NewInfo->Opcode : -1;
340}
341
342unsigned getAddrSizeMIMGOp(const MIMGBaseOpcodeInfo *BaseOpcode,
343 const MIMGDimInfo *Dim, bool IsA16,
344 bool IsG16Supported) {
345 unsigned AddrWords = BaseOpcode->NumExtraArgs;
346 unsigned AddrComponents = (BaseOpcode->Coordinates ? Dim->NumCoords : 0) +
347 (BaseOpcode->LodOrClampOrMip ? 1 : 0);
348 if (IsA16)
349 AddrWords += divideCeil(AddrComponents, 2);
350 else
351 AddrWords += AddrComponents;
352
353 // Note: For subtargets that support A16 but not G16, enabling A16 also
354 // enables 16 bit gradients.
355 // For subtargets that support A16 (operand) and G16 (done with a different
356 // instruction encoding), they are independent.
357
358 if (BaseOpcode->Gradients) {
359 if ((IsA16 && !IsG16Supported) || BaseOpcode->G16)
360 // There are two gradients per coordinate, we pack them separately.
361 // For the 3d case,
362 // we get (dy/du, dx/du) (-, dz/du) (dy/dv, dx/dv) (-, dz/dv)
363 AddrWords += alignTo<2>(Dim->NumGradients / 2);
364 else
365 AddrWords += Dim->NumGradients;
366 }
367 return AddrWords;
368}
369
380
389
390struct SMInfo {
392};
393
397
401
405
410
418
422
425 bool IsX;
426 bool IsY;
427};
428
429#define GET_FP4FP8DstByteSelTable_DECL
430#define GET_FP4FP8DstByteSelTable_IMPL
431
435
441
445
446#define GET_DPMACCInstructionTable_DECL
447#define GET_DPMACCInstructionTable_IMPL
448#define GET_MTBUFInfoTable_DECL
449#define GET_MTBUFInfoTable_IMPL
450#define GET_MUBUFInfoTable_DECL
451#define GET_MUBUFInfoTable_IMPL
452#define GET_SMInfoTable_DECL
453#define GET_SMInfoTable_IMPL
454#define GET_VOP1InfoTable_DECL
455#define GET_VOP1InfoTable_IMPL
456#define GET_VOP2InfoTable_DECL
457#define GET_VOP2InfoTable_IMPL
458#define GET_VOP3InfoTable_DECL
459#define GET_VOP3InfoTable_IMPL
460#define GET_VOPC64DPPTable_DECL
461#define GET_VOPC64DPPTable_IMPL
462#define GET_VOPC64DPP8Table_DECL
463#define GET_VOPC64DPP8Table_IMPL
464#define GET_VOPCAsmOnlyInfoTable_DECL
465#define GET_VOPCAsmOnlyInfoTable_IMPL
466#define GET_VOP3CAsmOnlyInfoTable_DECL
467#define GET_VOP3CAsmOnlyInfoTable_IMPL
468#define GET_VOPDComponentTable_DECL
469#define GET_VOPDComponentTable_IMPL
470#define GET_VOPDPairs_DECL
471#define GET_VOPDPairs_IMPL
472#define GET_VOPDXYTable_DECL
473#define GET_VOPDXYTable_IMPL
474#define GET_VOPTrue16Table_DECL
475#define GET_VOPTrue16Table_IMPL
476#define GET_VOP3PHasNoOpSelHiTable_DECL
477#define GET_VOP3PHasNoOpSelHiTable_IMPL
478#define GET_True16D16Table_IMPL
479#define GET_WMMAOpcode2AddrMappingTable_DECL
480#define GET_WMMAOpcode2AddrMappingTable_IMPL
481#define GET_WMMAOpcode3AddrMappingTable_DECL
482#define GET_WMMAOpcode3AddrMappingTable_IMPL
483#define GET_getMFMA_F8F6F4_WithSize_DECL
484#define GET_getMFMA_F8F6F4_WithSize_IMPL
485#define GET_isMFMA_F8F6F4Table_IMPL
486#define GET_isCvtScaleF32_F32F16ToF8F4Table_IMPL
487#define GET_VLdStIdxOpcodeInfoTable_IMPL
488
489#include "AMDGPUGenSearchableTables.inc"
490
491const VLdStIdxOpcodeInfo *
492getVLdStIdxOpcodeInfoByKey(unsigned BitWidth, bool IsStore, bool IsGPRIdx) {
493 assert(isUInt<16>(BitWidth) && "the table's key is 16 bits");
494 return getVLdStIdxOpcodeInfoByKeyImpl(BitWidth, IsStore, IsGPRIdx);
495}
496
497int getMTBUFBaseOpcode(unsigned Opc) {
498 const MTBUFInfo *Info = getMTBUFInfoFromOpcode(Opc);
499 return Info ? Info->BaseOpcode : -1;
500}
501
502int getMTBUFOpcode(unsigned BaseOpc, unsigned Elements) {
503 const MTBUFInfo *Info =
504 getMTBUFInfoFromBaseOpcodeAndElements(BaseOpc, Elements);
505 return Info ? Info->Opcode : -1;
506}
507
508int getMTBUFElements(unsigned Opc) {
509 const MTBUFInfo *Info = getMTBUFOpcodeHelper(Opc);
510 return Info ? Info->elements : 0;
511}
512
513bool getMTBUFHasVAddr(unsigned Opc) {
514 const MTBUFInfo *Info = getMTBUFOpcodeHelper(Opc);
515 return Info && Info->has_vaddr;
516}
517
518bool getMTBUFHasSrsrc(unsigned Opc) {
519 const MTBUFInfo *Info = getMTBUFOpcodeHelper(Opc);
520 return Info && Info->has_srsrc;
521}
522
523bool getMTBUFHasSoffset(unsigned Opc) {
524 const MTBUFInfo *Info = getMTBUFOpcodeHelper(Opc);
525 return Info && Info->has_soffset;
526}
527
528int getMUBUFBaseOpcode(unsigned Opc) {
529 const MUBUFInfo *Info = getMUBUFInfoFromOpcode(Opc);
530 return Info ? Info->BaseOpcode : -1;
531}
532
533int getMUBUFOpcode(unsigned BaseOpc, unsigned Elements) {
534 const MUBUFInfo *Info =
535 getMUBUFInfoFromBaseOpcodeAndElements(BaseOpc, Elements);
536 return Info ? Info->Opcode : -1;
537}
538
539int getMUBUFElements(unsigned Opc) {
540 const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opc);
541 return Info ? Info->elements : 0;
542}
543
544bool getMUBUFHasVAddr(unsigned Opc) {
545 const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opc);
546 return Info && Info->has_vaddr;
547}
548
549bool getMUBUFHasSrsrc(unsigned Opc) {
550 const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opc);
551 return Info && Info->has_srsrc;
552}
553
554bool getMUBUFHasSoffset(unsigned Opc) {
555 const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opc);
556 return Info && Info->has_soffset;
557}
558
559bool getMUBUFIsBufferInv(unsigned Opc) {
560 const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opc);
561 return Info && Info->IsBufferInv;
562}
563
564bool getMUBUFTfe(unsigned Opc) {
565 const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opc);
566 return Info && Info->tfe;
567}
568
569bool getSMEMIsBuffer(unsigned Opc) {
570 return isSMEMOpcodeHelper(Opc) != nullptr;
571}
572
573bool getVOP1IsSingle(unsigned Opc) {
574 return isVOP1SingleOpcodeHelper(Opc) != nullptr;
575}
576
577bool getVOP2IsSingle(unsigned Opc) {
578 return isVOP2SingleOpcodeHelper(Opc) != nullptr;
579}
580
581bool getVOP3IsSingle(unsigned Opc) {
582 return isVOP3SingleOpcodeHelper(Opc) != nullptr;
583}
584
585bool isVOPC64DPP(unsigned Opc) {
586 return isVOPC64DPPOpcodeHelper(Opc) || isVOPC64DPP8OpcodeHelper(Opc);
587}
588
589bool isVOPCAsmOnly(unsigned Opc) { return isVOPCAsmOnlyOpcodeHelper(Opc); }
590
591bool getMAIIsDGEMM(unsigned Opc) {
592 const MAIInstInfo *Info = getMAIInstInfoHelper(Opc);
593 return Info && Info->is_dgemm;
594}
595
596bool getMAIIsGFX940XDL(unsigned Opc) {
597 const MAIInstInfo *Info = getMAIInstInfoHelper(Opc);
598 return Info && Info->is_gfx940_xdl;
599}
600
601bool getWMMAIsXDL(unsigned Opc) {
602 const WMMAInstInfo *Info = getWMMAInstInfoHelper(Opc);
603 return Info ? Info->is_wmma_xdl : false;
604}
605
606bool getHasMatrixScale(unsigned Opc) {
607 const WMMAInstInfo *Info = getWMMAInstInfoHelper(Opc);
608 return Info && Info->HasMatrixScale;
609}
610
612 switch (EncodingVal) {
615 return 6;
617 return 4;
620 default:
621 return 8;
622 }
623
624 llvm_unreachable("covered switch over mfma scale formats");
625}
626
628 unsigned BLGP,
629 unsigned F8F8Opcode) {
630 uint8_t SrcANumRegs = mfmaScaleF8F6F4FormatToNumRegs(CBSZ);
631 uint8_t SrcBNumRegs = mfmaScaleF8F6F4FormatToNumRegs(BLGP);
632 return getMFMA_F8F6F4_InstWithNumRegs(SrcANumRegs, SrcBNumRegs, F8F8Opcode);
633}
634
636 switch (Fmt) {
639 return 16;
642 return 12;
644 return 8;
645 }
646
647 llvm_unreachable("covered switch over wmma scale formats");
648}
649
651 unsigned FmtB,
652 unsigned F8F8Opcode) {
653 uint8_t SrcANumRegs = wmmaScaleF8F6F4FormatToNumRegs(FmtA);
654 uint8_t SrcBNumRegs = wmmaScaleF8F6F4FormatToNumRegs(FmtB);
655 return getMFMA_F8F6F4_InstWithNumRegs(SrcANumRegs, SrcBNumRegs, F8F8Opcode);
656}
657
658bool isValidWMMAScaleFmtCombination(unsigned AFmt, unsigned AScale,
659 unsigned BFmt, unsigned BScale) {
660 auto isValid = [](unsigned Fmt, unsigned Scale) -> bool {
661 switch (Fmt) {
666 if (Scale != WMMA::MATRIX_SCALE_FMT_E8)
667 return false;
668 break;
670 if (Scale != WMMA::MATRIX_SCALE_FMT_E8 &&
673 return false;
674 break;
675 }
676 return true;
677 };
678
679 if (!isValid(AFmt, AScale) || !isValid(BFmt, BScale))
680 return false;
681
682 if (AFmt == WMMA::MATRIX_FMT_FP4 && BFmt == WMMA::MATRIX_FMT_FP4 &&
683 AScale != BScale)
684 return false;
685
686 return true;
687}
688
690 if (ST.hasFeature(AMDGPU::FeatureGFX13Insts))
692 if (ST.hasFeature(AMDGPU::FeatureGFX1250Insts))
694 if (ST.hasFeature(AMDGPU::FeatureGFX12Insts))
696 if (ST.hasFeature(AMDGPU::FeatureGFX11_7Insts))
698 if (ST.hasFeature(AMDGPU::FeatureGFX11Insts))
700 llvm_unreachable("Subtarget generation does not support VOPD!");
701}
702
703CanBeVOPD getCanBeVOPD(unsigned Opc, unsigned EncodingFamily, bool VOPD3) {
704 bool IsConvertibleToBitOp = VOPD3 ? getBitOp2(Opc) : 0;
705 Opc = IsConvertibleToBitOp ? (unsigned)AMDGPU::V_BITOP3_B32_e64 : Opc;
706 // Normalize through VOPDComponentTable so that e32 and e64 variants
707 // of the same logical opcode all share a single entry.
708 const VOPDComponentInfo *Info = getVOPDComponentHelper(Opc);
709 if (!Info)
710 return {false, false};
711 unsigned Key =
712 (Info->VOPDOp << 5) | (EncodingFamily << 1) | (VOPD3 ? 1u : 0u);
713 const VOPDXYInfo *XYInfo = getVOPDXYInfo(static_cast<uint16_t>(Key));
714 if (!XYInfo)
715 return {false, false};
716 return {XYInfo->IsX, XYInfo->IsY};
717}
718
719unsigned getVOPDOpcode(unsigned Opc, bool VOPD3) {
720 bool IsConvertibleToBitOp = VOPD3 ? getBitOp2(Opc) : 0;
721 Opc = IsConvertibleToBitOp ? (unsigned)AMDGPU::V_BITOP3_B32_e64 : Opc;
722 const VOPDComponentInfo *Info = getVOPDComponentHelper(Opc);
723 return Info ? Info->VOPDOp : ~0u;
724}
725
726bool isVOPD(unsigned Opc) {
727 return AMDGPU::hasNamedOperand(Opc, AMDGPU::OpName::src0X);
728}
729
730bool isMAC(unsigned Opc) {
731 return Opc == AMDGPU::V_MAC_F32_e64_gfx6_gfx7 ||
732 Opc == AMDGPU::V_MAC_F32_e64_gfx10 ||
733 Opc == AMDGPU::V_MAC_F32_e64_vi ||
734 Opc == AMDGPU::V_MAC_LEGACY_F32_e64_gfx6_gfx7 ||
735 Opc == AMDGPU::V_MAC_LEGACY_F32_e64_gfx10 ||
736 Opc == AMDGPU::V_MAC_F16_e64_vi ||
737 Opc == AMDGPU::V_FMAC_F64_e64_gfx90a ||
738 Opc == AMDGPU::V_FMAC_F64_e64_gfx12 ||
739 Opc == AMDGPU::V_FMAC_F64_e64_gfx13 ||
740 Opc == AMDGPU::V_FMAC_F32_e64_gfx10 ||
741 Opc == AMDGPU::V_FMAC_F32_e64_gfx11 ||
742 Opc == AMDGPU::V_FMAC_F32_e64_gfx12 ||
743 Opc == AMDGPU::V_FMAC_F32_e64_gfx13 ||
744 Opc == AMDGPU::V_FMAC_F32_e64_vi ||
745 Opc == AMDGPU::V_FMAC_LEGACY_F32_e64_gfx10 ||
746 Opc == AMDGPU::V_FMAC_DX9_ZERO_F32_e64_gfx11 ||
747 Opc == AMDGPU::V_FMAC_F16_e64_gfx10 ||
748 Opc == AMDGPU::V_FMAC_F16_t16_e64_gfx11 ||
749 Opc == AMDGPU::V_FMAC_F16_fake16_e64_gfx11 ||
750 Opc == AMDGPU::V_FMAC_F16_t16_e64_gfx12 ||
751 Opc == AMDGPU::V_FMAC_F16_fake16_e64_gfx12 ||
752 Opc == AMDGPU::V_FMAC_F16_t16_e64_gfx13 ||
753 Opc == AMDGPU::V_FMAC_F16_fake16_e64_gfx13 ||
754 Opc == AMDGPU::V_DOT2C_F32_F16_e64_vi ||
755 Opc == AMDGPU::V_DOT2C_F32_BF16_e64_vi ||
756 Opc == AMDGPU::V_DOT2C_I32_I16_e64_vi ||
757 Opc == AMDGPU::V_DOT4C_I32_I8_e64_vi ||
758 Opc == AMDGPU::V_DOT8C_I32_I4_e64_vi;
759}
760
761bool isPermlane16(unsigned Opc) {
762 return Opc == AMDGPU::V_PERMLANE16_B32_gfx10 ||
763 Opc == AMDGPU::V_PERMLANEX16_B32_gfx10 ||
764 Opc == AMDGPU::V_PERMLANE16_B32_e64_gfx11 ||
765 Opc == AMDGPU::V_PERMLANEX16_B32_e64_gfx11 ||
766 Opc == AMDGPU::V_PERMLANE16_B32_e64_gfx12 ||
767 Opc == AMDGPU::V_PERMLANE16_B32_e64_gfx13 ||
768 Opc == AMDGPU::V_PERMLANEX16_B32_e64_gfx12 ||
769 Opc == AMDGPU::V_PERMLANEX16_B32_e64_gfx13 ||
770 Opc == AMDGPU::V_PERMLANE16_VAR_B32_e64_gfx12 ||
771 Opc == AMDGPU::V_PERMLANE16_VAR_B32_e64_gfx13 ||
772 Opc == AMDGPU::V_PERMLANEX16_VAR_B32_e64_gfx12 ||
773 Opc == AMDGPU::V_PERMLANEX16_VAR_B32_e64_gfx13;
774}
775
777 return Opc == AMDGPU::V_CVT_F32_BF8_e64_gfx12 ||
778 Opc == AMDGPU::V_CVT_F32_FP8_e64_gfx12 ||
779 Opc == AMDGPU::V_CVT_F32_BF8_e64_dpp_gfx12 ||
780 Opc == AMDGPU::V_CVT_F32_FP8_e64_dpp_gfx12 ||
781 Opc == AMDGPU::V_CVT_F32_BF8_e64_dpp8_gfx12 ||
782 Opc == AMDGPU::V_CVT_F32_FP8_e64_dpp8_gfx12 ||
783 Opc == AMDGPU::V_CVT_PK_F32_BF8_fake16_e64_gfx12 ||
784 Opc == AMDGPU::V_CVT_PK_F32_FP8_fake16_e64_gfx12 ||
785 Opc == AMDGPU::V_CVT_PK_F32_BF8_t16_e64_gfx12 ||
786 Opc == AMDGPU::V_CVT_PK_F32_FP8_t16_e64_gfx12;
787}
788
789bool isGenericAtomic(unsigned Opc) {
790 return Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SWAP ||
791 Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_ADD ||
792 Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SUB ||
793 Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SMIN ||
794 Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_UMIN ||
795 Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SMAX ||
796 Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_UMAX ||
797 Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_AND ||
798 Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_OR ||
799 Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_XOR ||
800 Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_INC ||
801 Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_DEC ||
802 Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_FADD ||
803 Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_FMIN ||
804 Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_FMAX ||
805 Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_CMPSWAP ||
806 Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SUB_CLAMP_U32 ||
807 Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_COND_SUB_U32 ||
808 Opc == AMDGPU::G_AMDGPU_ATOMIC_CMPXCHG;
809}
810
811bool isAsyncStore(unsigned Opc) {
812 return Opc == GLOBAL_STORE_ASYNC_FROM_LDS_B8_gfx1250 ||
813 Opc == GLOBAL_STORE_ASYNC_FROM_LDS_B32_gfx1250 ||
814 Opc == GLOBAL_STORE_ASYNC_FROM_LDS_B64_gfx1250 ||
815 Opc == GLOBAL_STORE_ASYNC_FROM_LDS_B128_gfx1250 ||
816 Opc == GLOBAL_STORE_ASYNC_FROM_LDS_B8_SADDR_gfx1250 ||
817 Opc == GLOBAL_STORE_ASYNC_FROM_LDS_B32_SADDR_gfx1250 ||
818 Opc == GLOBAL_STORE_ASYNC_FROM_LDS_B64_SADDR_gfx1250 ||
819 Opc == GLOBAL_STORE_ASYNC_FROM_LDS_B128_SADDR_gfx1250;
820}
821
822bool isTensorStore(unsigned Opc) {
823 return Opc == TENSOR_STORE_FROM_LDS_d2_gfx1250 ||
824 Opc == TENSOR_STORE_FROM_LDS_d4_gfx1250;
825}
826
827unsigned getTemporalHintType(const MCInstrDesc TID) {
828 if (SIInstrFlags::isAtomic(TID))
830 unsigned Opc = TID.getOpcode();
831 // Async and Tensor store should have the temporal hint type of TH_TYPE_STORE
832 if (TID.mayStore() &&
833 (isAsyncStore(Opc) || isTensorStore(Opc) || !TID.mayLoad()))
834 return CPol::TH_TYPE_STORE;
835
836 // This will default to returning TH_TYPE_LOAD when neither MayStore nor
837 // MayLoad flag is present which is the case with instructions like
838 // image_get_resinfo.
839 return CPol::TH_TYPE_LOAD;
840}
841
842bool isTrue16Inst(unsigned Opc) { return isTrue16Opcode(Opc) != nullptr; }
843
844bool hasNoOpSelHi(unsigned Opc) {
845 return getVOP3PHasNoOpSelHiHelper(Opc) != nullptr;
846}
847
849 const FP4FP8DstByteSelInfo *Info = getFP4FP8DstByteSelHelper(Opc);
850 if (!Info)
851 return FPType::None;
852 if (Info->HasFP8DstByteSel)
853 return FPType::FP8;
854 if (Info->HasFP4DstByteSel)
855 return FPType::FP4;
856
857 return FPType::None;
858}
859
860bool isDPMACCInstruction(unsigned Opc) {
861 return isDPMACCInstructionHelper(Opc) != nullptr;
862}
863
864unsigned mapWMMA2AddrTo3AddrOpcode(unsigned Opc) {
865 const WMMAOpcodeMappingInfo *Info = getWMMAMappingInfoFrom2AddrOpcode(Opc);
866 return Info ? Info->Opcode3Addr : ~0u;
867}
868
869// Wrapper for Tablegen'd function. enum Subtarget is not defined in any
870// header files, so we need to wrap it in a function that takes unsigned
871// instead.
872int32_t getMCOpcode(uint32_t Opcode, unsigned Gen) {
873 return getMCOpcodeGen(Opcode, static_cast<Subtarget>(Gen));
874}
875
876unsigned getBitOp2(unsigned Opc) {
877 switch (Opc) {
878 default:
879 return 0;
880 case AMDGPU::V_AND_B32_e32:
881 return 0x40;
882 case AMDGPU::V_OR_B32_e32:
883 return 0x54;
884 case AMDGPU::V_XOR_B32_e32:
885 return 0x14;
886 case AMDGPU::V_XNOR_B32_e32:
887 return 0x41;
888 }
889}
890
891int getVOPDFull(unsigned OpX, unsigned OpY, unsigned EncodingFamily,
892 bool VOPD3) {
893 bool IsConvertibleToBitOp = VOPD3 ? getBitOp2(OpY) : 0;
894 OpY = IsConvertibleToBitOp ? (unsigned)AMDGPU::V_BITOP3_B32_e64 : OpY;
895 const VOPDInfo *Info = getVOPDInfoFromComponentOpcodes(
896 static_cast<uint8_t>(OpX), static_cast<uint8_t>(OpY),
897 static_cast<uint8_t>(EncodingFamily), VOPD3);
898 return Info ? Info->Opcode : -1;
899}
900
901std::pair<unsigned, unsigned> getVOPDComponents(unsigned VOPDOpcode) {
902 const VOPDInfo *Info = getVOPDOpcodeHelper(VOPDOpcode);
903 assert(Info);
904 const auto *OpX = getVOPDBaseFromComponent(Info->OpX);
905 const auto *OpY = getVOPDBaseFromComponent(Info->OpY);
906 assert(OpX && OpY);
907 return {OpX->BaseVOP, OpY->BaseVOP};
908}
909
910namespace VOPD {
911
912ComponentProps::ComponentProps(const MCInstrDesc &OpDesc, bool VOP3Layout) {
914
917 auto TiedIdx = OpDesc.getOperandConstraint(Component::SRC2, MCOI::TIED_TO);
918 assert(TiedIdx == -1 || TiedIdx == Component::DST);
919 HasSrc2Acc = TiedIdx != -1;
920 Opcode = OpDesc.getOpcode();
921
922 IsVOP3 = VOP3Layout || SIInstrFlags::isVOP3(OpDesc);
923 SrcOperandsNum = AMDGPU::hasNamedOperand(Opcode, AMDGPU::OpName::src2) ? 3
924 : AMDGPU::hasNamedOperand(Opcode, AMDGPU::OpName::imm) ? 3
925 : AMDGPU::hasNamedOperand(Opcode, AMDGPU::OpName::src1) ? 2
926 : 1;
927 assert(SrcOperandsNum <= Component::MAX_SRC_NUM);
928
929 if (Opcode == AMDGPU::V_CNDMASK_B32_e32 ||
930 Opcode == AMDGPU::V_CNDMASK_B32_e64) {
931 // CNDMASK is an awkward exception, it has FP modifiers, but not FP
932 // operands.
933 NumVOPD3Mods = 2;
934 if (IsVOP3)
935 SrcOperandsNum = 3;
936 } else if (Opcode == AMDGPU::V_DOT2_F32_F16 ||
937 Opcode == AMDGPU::V_DOT2_F32_BF16) {
938 // VOP3P opcodes that have VOPD but don't have VOP2 version. Using VOPD3
939 // path in getIndexOfSrcInMCOperands to get correct src operand indexes,
940 // but generating VOPD, not VOPD3.
941 NumVOPD3Mods = SrcOperandsNum;
942 } else if (isSISrcFPOperand(OpDesc,
943 getNamedOperandIdx(Opcode, OpName::src0))) {
944 // All FP VOPD instructions have Neg modifiers for all operands except
945 // for tied src2.
946 NumVOPD3Mods = SrcOperandsNum;
947 if (HasSrc2Acc)
948 --NumVOPD3Mods;
949 }
950
951 if (SIInstrFlags::isVOP3(OpDesc))
952 return;
953
954 auto OperandsNum = OpDesc.getNumOperands();
955 unsigned CompOprIdx;
956 for (CompOprIdx = Component::SRC1; CompOprIdx < OperandsNum; ++CompOprIdx) {
957 if (OpDesc.operands()[CompOprIdx].OperandType == AMDGPU::OPERAND_KIMM32) {
958 MandatoryLiteralIdx = CompOprIdx;
959 break;
960 }
961 }
962}
963
965 return getNamedOperandIdx(Opcode, OpName::bitop3);
966}
967
968unsigned ComponentInfo::getIndexInParsedOperands(unsigned CompOprIdx) const {
969 assert(CompOprIdx < Component::MAX_OPR_NUM);
970
971 if (CompOprIdx == Component::DST)
973
974 auto CompSrcIdx = CompOprIdx - Component::DST_NUM;
975 if (CompSrcIdx < getCompParsedSrcOperandsNum())
976 return getIndexOfSrcInParsedOperands(CompSrcIdx);
977
978 // The specified operand does not exist.
979 return 0;
980}
981
983 std::function<MCRegister(unsigned, unsigned)> GetRegIdx,
984 const MCRegisterInfo &MRI, bool SkipSrc, bool AllowSameVGPR, bool VOPD3,
985 bool HasGFX11InterlockHazard) const {
986
987 auto OpXRegs = getRegIndices(ComponentIndex::X, GetRegIdx,
988 CompInfo[ComponentIndex::X].isVOP3());
989 auto OpYRegs = getRegIndices(ComponentIndex::Y, GetRegIdx,
990 CompInfo[ComponentIndex::Y].isVOP3());
991
992 const auto banksOverlap = [&MRI](MCRegister X, MCRegister Y,
993 unsigned BanksMask) -> bool {
994 MCRegister BaseX = MRI.getSubReg(X, AMDGPU::sub0);
995 MCRegister BaseY = MRI.getSubReg(Y, AMDGPU::sub0);
996 if (!BaseX)
997 BaseX = X;
998 if (!BaseY)
999 BaseY = Y;
1000 if ((BaseX.id() & BanksMask) == (BaseY.id() & BanksMask))
1001 return true;
1002 if (BaseX != X /* This is 64-bit register */ &&
1003 ((BaseX.id() + 1) & BanksMask) == (BaseY.id() & BanksMask))
1004 return true;
1005 if (BaseY != Y &&
1006 (BaseX.id() & BanksMask) == ((BaseY.id() + 1) & BanksMask))
1007 return true;
1008
1009 // If both are 64-bit bank conflict will be detected yet while checking
1010 // the first subreg.
1011 return false;
1012 };
1013
1014 unsigned CompOprIdx;
1015 for (CompOprIdx = 0; CompOprIdx < Component::MAX_OPR_NUM; ++CompOprIdx) {
1016 unsigned BanksMasks = VOPD3 ? VOPD3_VGPR_BANK_MASKS[CompOprIdx]
1017 : HasGFX11InterlockHazard
1018 ? VOPD_GFX11_VGPR_BANK_MASKS[CompOprIdx]
1019 : VOPD_VGPR_BANK_MASKS[CompOprIdx];
1020 if (!OpXRegs[CompOprIdx] || !OpYRegs[CompOprIdx])
1021 continue;
1022
1023 if (getVGPREncodingMSBs(OpXRegs[CompOprIdx], MRI) !=
1024 getVGPREncodingMSBs(OpYRegs[CompOprIdx], MRI))
1025 return CompOprIdx;
1026
1027 if (SkipSrc && CompOprIdx >= Component::DST_NUM)
1028 continue;
1029
1030 if (CompOprIdx < Component::DST_NUM) {
1031 // Even if we do not check vdst parity, vdst operands still shall not
1032 // overlap.
1033 if (MRI.regsOverlap(OpXRegs[CompOprIdx], OpYRegs[CompOprIdx]))
1034 return CompOprIdx;
1035 if (VOPD3) // No need to check dst parity.
1036 continue;
1037 }
1038
1039 if (banksOverlap(OpXRegs[CompOprIdx], OpYRegs[CompOprIdx], BanksMasks) &&
1040 (!AllowSameVGPR || CompOprIdx < Component::DST_NUM ||
1041 OpXRegs[CompOprIdx] != OpYRegs[CompOprIdx]))
1042 return CompOprIdx;
1043 }
1044
1045 return {};
1046}
1047
1048// Return an array of VGPR registers [DST,SRC0,SRC1,SRC2] used
1049// by the specified component. If an operand is unused
1050// or is not a VGPR, the corresponding value is 0.
1051//
1052// GetRegIdx(Component, MCOperandIdx) must return a VGPR register index
1053// for the specified component and MC operand. The callback must return 0
1054// if the operand is not a register or not a VGPR.
1056InstInfo::getRegIndices(unsigned CompIdx,
1057 std::function<MCRegister(unsigned, unsigned)> GetRegIdx,
1058 bool VOPD3) const {
1059 assert(CompIdx < COMPONENTS_NUM);
1060
1061 const auto &Comp = CompInfo[CompIdx];
1063
1064 RegIndices[DST] = GetRegIdx(CompIdx, Comp.getIndexOfDstInMCOperands());
1065
1066 for (unsigned CompOprIdx : {SRC0, SRC1, SRC2}) {
1067 unsigned CompSrcIdx = CompOprIdx - DST_NUM;
1068 RegIndices[CompOprIdx] =
1069 Comp.hasRegSrcOperand(CompSrcIdx)
1070 ? GetRegIdx(CompIdx,
1071 Comp.getIndexOfSrcInMCOperands(CompSrcIdx, VOPD3))
1072 : MCRegister();
1073 }
1074 return RegIndices;
1075}
1076
1077} // namespace VOPD
1078
1080 return VOPD::InstInfo(OpX, OpY);
1081}
1082
1084 const MCInstrInfo *InstrInfo) {
1085 auto [OpX, OpY] = getVOPDComponents(VOPDOpcode);
1086 const auto &OpXDesc = InstrInfo->get(OpX);
1087 const auto &OpYDesc = InstrInfo->get(OpY);
1088 bool VOPD3 = SIInstrFlags::isVOPD3(*InstrInfo, VOPDOpcode);
1090 VOPD::ComponentInfo OpYInfo(OpYDesc, OpXInfo, VOPD3);
1091 return VOPD::InstInfo(OpXInfo, OpYInfo);
1092}
1093
1095 StringRef FeatureString) {
1096 // In codegen the mode comes from module flags and FeatureString is empty, so
1097 // the processor defaults apply. The assembler has no target directive, so it
1098 // pins the mode via the +xnack/-xnack/+sramecc/-sramecc feature string.
1100 STI.getCPU(), FeatureString);
1101}
1102
1103namespace IsaInfo {
1104
1106 if (STI.getFeatureBits().test(FeatureInstCacheLineSize128))
1107 return 128;
1108 if (STI.getFeatureBits().test(FeatureInstCacheLineSize64))
1109 return 64;
1110 return 64;
1111}
1112
1113unsigned getWavefrontSize(const MCSubtargetInfo &STI) {
1114 if (STI.getFeatureBits().test(FeatureWavefrontSize16))
1115 return 16;
1116 if (STI.getFeatureBits().test(FeatureWavefrontSize32))
1117 return 32;
1118
1119 return 64;
1120}
1121
1122// Maximum LDS a single work-group can address. This is a fixed HW cap. It does
1123// not depend on how many SIMDs a work-group runs on.
1125 if (STI.getFeatureBits().test(FeatureAddressableLocalMemorySize32768))
1126 return 32768;
1127 if (STI.getFeatureBits().test(FeatureAddressableLocalMemorySize65536))
1128 return 65536;
1129 if (STI.getFeatureBits().test(FeatureAddressableLocalMemorySize163840))
1130 return 163840;
1131 if (STI.getFeatureBits().test(FeatureAddressableLocalMemorySize196608))
1132 return 196608;
1133 if (STI.getFeatureBits().test(FeatureAddressableLocalMemorySize327680))
1134 return 327680;
1135 return 32768;
1136}
1137
1138// Total physical size of LDS on the block, in bytes. On targets with
1139// FeatureHalfAddressablePhysicalLocalMemory the physical block is twice the
1140// addressable size (gfx6: 64 KiB physical and 32 KiB addressable;
1141// gfx10/11/12: 128 KiB physical and 64 KiB addressable). On other targets it is
1142// equal to the addressable size.
1143static unsigned getPhysicalLocalMemorySize(const MCSubtargetInfo &STI) {
1144 unsigned Addressable = getMaxHWAddressableLocalMemorySize(STI);
1145 if (STI.getFeatureBits().test(FeatureHalfAddressablePhysicalLocalMemory))
1146 return 2 * Addressable;
1147 return Addressable;
1148}
1149
1150// Sizes in use, by generation (addressable / physical block):
1151// gfx6 : 32 KiB addressable, 64 KiB physical block
1152// gfx7 / gfx8 / gfx9: 64 KiB
1153// gfx9.5 (gfx950) : 160 KiB
1154// gfx10 / 11 / 12 : 64 KiB addressable, 128 KiB physical block
1155// gfx12.5 (gfx1250) : 320 KiB (always runs on four SIMDs)
1156// gfx13 : 192 KiB on four SIMDs, 96 KiB on two
1157// Total available in the current mode. The physical size is halved when a
1158// work-group runs on two SIMDs.
1160 unsigned Size = getPhysicalLocalMemorySize(STI);
1161 if (!isFullSIMDMode(STI))
1162 Size /= 2;
1163 return Size;
1164}
1165
1166// What one work-group can allocate in the current mode. This is the HW
1167// addressable cap, but never more than the total available in the current mode.
1169 return std::min(getMaxHWAddressableLocalMemorySize(STI),
1170 getLocalMemorySize(STI));
1171}
1172
1174 unsigned FlatWorkGroupSize) {
1175 assert(FlatWorkGroupSize != 0);
1176 if (!STI.getTargetTriple().isAMDGCN())
1177 return 8;
1178 GPUKind Kind = parseArchAMDGCN(STI.getCPU());
1179 unsigned MaxWaves =
1181 unsigned N = getWavesPerWorkGroup(STI, FlatWorkGroupSize);
1182 if (N == 1) {
1183 // Single-wave workgroups don't consume barrier resources.
1184 return MaxWaves;
1185 }
1186
1187 unsigned MaxBarriers = 16;
1188 if (isGFX10Plus(STI) && !STI.getFeatureBits().test(FeatureCuMode))
1189 MaxBarriers = 32;
1190
1191 return std::min(MaxWaves / N, MaxBarriers);
1192}
1193
1195 unsigned FlatWorkGroupSize) {
1196 return divideCeil(getWavesPerWorkGroup(STI, FlatWorkGroupSize),
1198}
1199
1201 unsigned FlatWorkGroupSize) {
1202 return divideCeil(FlatWorkGroupSize, getWavefrontSize(STI));
1203}
1204
1205unsigned getSGPREncodingGranule(const MCSubtargetInfo &STI) { return 8; }
1206
1207// Per-wave SGPRs reserved for the trap handler when enabled.
1208static unsigned getSGPRTrapHandlerReserve(const MCSubtargetInfo &STI) {
1209 return STI.getFeatureBits().test(FeatureTrapHandler) ? TRAP_NUM_SGPRS : 0;
1210}
1211
1212// Per-wave SGPR budget (before the addressable clamp): take off the trap
1213// reserve, round down to \p Granule. Shared by getMinNumSGPRs() and
1214// getMaxNumSGPRs(); getOccupancyWithNumSGPRs() is the closed-form algebraic
1215// inverse of this same budget (it does not call this helper), so the two encode
1216// one model.
1217static unsigned getSGPRBudgetPerWave(unsigned TotalNumSGPRs,
1218 unsigned WavesPerEU, unsigned TrapReserve,
1219 unsigned Granule) {
1220 assert(WavesPerEU != 0 && Granule != 0);
1221 unsigned Budget = TotalNumSGPRs / WavesPerEU;
1222 Budget -= std::min(Budget, TrapReserve);
1223 return alignDown(Budget, Granule);
1224}
1225
1226unsigned getMinNumSGPRs(const MCSubtargetInfo &STI, unsigned WavesPerEU) {
1227 assert(WavesPerEU != 0);
1228
1230 if (Version.Major >= 10)
1231 return 0;
1232
1233 GPUKind Kind = parseArchAMDGCN(STI.getCPU());
1234 if (WavesPerEU >= getMaxWavesPerEU(Kind))
1235 return 0;
1236
1237 unsigned MinNumSGPRs =
1238 getSGPRBudgetPerWave(getTotalNumSGPRs(Kind), WavesPerEU + 1,
1240 getSGPRAllocGranule(Kind)) +
1241 1;
1242 return std::min(MinNumSGPRs, getAddressableNumSGPRs(Kind));
1243}
1244
1245unsigned getMaxNumSGPRs(const MCSubtargetInfo &STI, unsigned WavesPerEU,
1246 bool Addressable) {
1247 assert(WavesPerEU != 0);
1248
1249 GPUKind Kind = parseArchAMDGCN(STI.getCPU());
1250 unsigned AddressableNumSGPRs = getAddressableNumSGPRs(Kind);
1252 if (Version.Major >= 10)
1253 return Addressable ? AddressableNumSGPRs : 108;
1254 if (Version.Major >= 8 && !Addressable)
1255 AddressableNumSGPRs = 112;
1256 unsigned MaxNumSGPRs = getSGPRBudgetPerWave(
1257 getTotalNumSGPRs(Kind), WavesPerEU, getSGPRTrapHandlerReserve(STI),
1258 getSGPRAllocGranule(Kind));
1259 return std::min(MaxNumSGPRs, AddressableNumSGPRs);
1260}
1261
1263 // From GFX10 on the SGPR file is large enough that SGPRs never limit
1264 // occupancy. Kept as one capability so callers don't each test the version.
1265 return getIsaVersion(STI.getCPU()).Major < 10;
1266}
1267
1268unsigned getNumExtraSGPRs(const MCSubtargetInfo &STI, bool VCCUsed,
1269 bool FlatScrUsed, bool XNACKUsed) {
1270 unsigned ExtraSGPRs = 0;
1271 if (VCCUsed)
1272 ExtraSGPRs = 2;
1273
1275 if (Version.Major >= 10)
1276 return ExtraSGPRs;
1277
1278 if (Version.Major < 8) {
1279 if (FlatScrUsed)
1280 ExtraSGPRs = 4;
1281 } else {
1282 if (XNACKUsed)
1283 ExtraSGPRs = 4;
1284
1285 if (FlatScrUsed ||
1286 STI.getFeatureBits().test(AMDGPU::FeatureArchitectedFlatScratch))
1287 ExtraSGPRs = 6;
1288 }
1289
1290 return ExtraSGPRs;
1291}
1292
1293static unsigned getGranulatedNumRegisterBlocks(unsigned NumRegs,
1294 unsigned Granule) {
1295 return divideCeil(std::max(1u, NumRegs), Granule);
1296}
1297
1298unsigned getNumSGPRBlocks(const MCSubtargetInfo &STI, unsigned NumSGPRs) {
1299 // SGPRBlocks is actual number of SGPR blocks minus 1.
1301 1;
1302}
1303
1305 unsigned DynamicVGPRBlockSize,
1306 std::optional<bool> EnableWavefrontSize32) {
1307 if (STI.getFeatureBits().test(FeatureGFX90AInsts))
1308 return 8;
1309
1310 if (DynamicVGPRBlockSize != 0)
1311 return DynamicVGPRBlockSize;
1312
1313 bool IsWave32 = EnableWavefrontSize32
1314 ? *EnableWavefrontSize32
1315 : STI.getFeatureBits().test(FeatureWavefrontSize32);
1316
1317 if (STI.getFeatureBits().test(Feature1536VGPRs))
1318 return IsWave32 ? 24 : 12;
1319
1320 if (hasGFX10_3Insts(STI))
1321 return IsWave32 ? 16 : 8;
1322
1323 return IsWave32 ? 8 : 4;
1324}
1325
1327 std::optional<bool> EnableWavefrontSize32) {
1328 if (STI.getFeatureBits().test(FeatureGFX90AInsts))
1329 return 8;
1330
1331 bool IsWave32 = EnableWavefrontSize32
1332 ? *EnableWavefrontSize32
1333 : STI.getFeatureBits().test(FeatureWavefrontSize32);
1334
1335 if (STI.getFeatureBits().test(FeatureVGPRMSBs))
1336 return IsWave32 ? 16 : 8;
1337
1338 return IsWave32 ? 8 : 4;
1339}
1340
1341unsigned getArchVGPRAllocGranule() { return 4; }
1342
1344 const auto &Features = STI.getFeatureBits();
1345 if (Features.test(FeatureVGPRMSBs))
1346 return Features.test(FeatureWavefrontSize32) ? 1024 : 512;
1347 return 256;
1348}
1349
1351 unsigned DynamicVGPRBlockSize) {
1352 const auto &Features = STI.getFeatureBits();
1353 if (Features.test(FeatureGFX90AInsts))
1354 return 512;
1355
1356 if (DynamicVGPRBlockSize != 0) {
1357 // On GFX12 we can allocate at most MaxDynamicVGPRBlocks blocks of VGPRs.
1358 return MaxDynamicVGPRBlocks *
1359 getVGPRAllocGranule(STI, DynamicVGPRBlockSize);
1360 }
1361 return getAddressableNumArchVGPRs(STI);
1362}
1363
1365 unsigned NumVGPRs,
1366 unsigned DynamicVGPRBlockSize) {
1367 GPUKind Kind = parseArchAMDGCN(STI.getCPU());
1368 bool IsWave32 = STI.getFeatureBits().test(FeatureWavefrontSize32);
1370 NumVGPRs, getVGPRAllocGranule(STI, DynamicVGPRBlockSize),
1371 getMaxWavesPerEU(Kind), AMDGPU::getTotalNumVGPRs(Kind, IsWave32));
1372}
1373
1374unsigned getNumWavesPerEUWithNumVGPRs(unsigned NumVGPRs, unsigned Granule,
1375 unsigned MaxWaves,
1376 unsigned TotalNumVGPRs) {
1377 if (NumVGPRs < Granule)
1378 return MaxWaves;
1379 unsigned RoundedRegs = alignTo(NumVGPRs, Granule);
1380 return std::min(std::max(TotalNumVGPRs / RoundedRegs, 1u), MaxWaves);
1381}
1382
1383unsigned getOccupancyWithNumSGPRs(unsigned SGPRs, unsigned MaxWaves,
1384 unsigned TotalNumSGPRs, unsigned Granule,
1385 unsigned TrapReserve) {
1386 // Closed-form inverse of getMaxNumSGPRs(): the budget condition
1387 // SGPRs <= alignDown(TotalNumSGPRs / W - TrapReserve, Granule)
1388 // solves to W <= TotalNumSGPRs / (alignTo(SGPRs, Granule) + TrapReserve).
1389 unsigned PerWave = alignTo(SGPRs, Granule) + TrapReserve;
1390 return PerWave ? std::clamp(TotalNumSGPRs / PerWave, 1u, MaxWaves) : MaxWaves;
1391}
1392
1393unsigned getOccupancyWithNumSGPRs(const MCSubtargetInfo &STI, unsigned SGPRs) {
1394 GPUKind Kind = parseArchAMDGCN(STI.getCPU());
1395 unsigned MaxWaves = getMaxWavesPerEU(Kind);
1396
1397 if (!isSGPROccupancyLimited(STI))
1398 return MaxWaves;
1399
1400 return getOccupancyWithNumSGPRs(SGPRs, MaxWaves, getTotalNumSGPRs(Kind),
1401 getSGPRAllocGranule(Kind),
1403}
1404
1405unsigned getMinNumVGPRs(const MCSubtargetInfo &STI, unsigned WavesPerEU,
1406 unsigned DynamicVGPRBlockSize) {
1407 assert(WavesPerEU != 0);
1408
1409 // In dynamic VGPR mode, (static) occupancy does not depend on VGPR usage,
1410 // so getMaxNumVGPRs does not depend on WavesPerEU, and thus we need to return
1411 // zero because there is no nonzero VGPR usage N where going below N
1412 // achieves higher (static) occupancy.
1413 bool DynamicVGPREnabled = (DynamicVGPRBlockSize != 0);
1414 if (DynamicVGPREnabled)
1415 return 0;
1416
1417 GPUKind Kind = parseArchAMDGCN(STI.getCPU());
1418 unsigned MaxWavesPerEU = getMaxWavesPerEU(Kind);
1419 if (WavesPerEU >= MaxWavesPerEU)
1420 return 0;
1421
1422 unsigned TotNumVGPRs = AMDGPU::getTotalNumVGPRs(
1423 Kind, STI.getFeatureBits().test(FeatureWavefrontSize32));
1424 unsigned AddrsableNumVGPRs =
1425 getAddressableNumVGPRs(STI, DynamicVGPRBlockSize);
1426 unsigned Granule = getVGPRAllocGranule(STI, DynamicVGPRBlockSize);
1427 unsigned MaxNumVGPRs = alignDown(TotNumVGPRs / WavesPerEU, Granule);
1428
1429 if (MaxNumVGPRs == alignDown(TotNumVGPRs / MaxWavesPerEU, Granule))
1430 return 0;
1431
1432 unsigned MinWavesPerEU = getNumWavesPerEUWithNumVGPRs(STI, AddrsableNumVGPRs,
1433 DynamicVGPRBlockSize);
1434 if (WavesPerEU < MinWavesPerEU)
1435 return getMinNumVGPRs(STI, MinWavesPerEU, DynamicVGPRBlockSize);
1436
1437 unsigned MaxNumVGPRsNext = alignDown(TotNumVGPRs / (WavesPerEU + 1), Granule);
1438 unsigned MinNumVGPRs = 1 + std::min(MaxNumVGPRs - Granule, MaxNumVGPRsNext);
1439 return std::min(MinNumVGPRs, AddrsableNumVGPRs);
1440}
1441
1442unsigned getMaxNumVGPRs(const MCSubtargetInfo &STI, unsigned WavesPerEU,
1443 unsigned DynamicVGPRBlockSize) {
1444 assert(WavesPerEU != 0);
1445
1446 unsigned TotNumVGPRs = AMDGPU::getTotalNumVGPRs(
1447 parseArchAMDGCN(STI.getCPU()),
1448 STI.getFeatureBits().test(FeatureWavefrontSize32));
1449
1450 // In dynamic VGPR mode, WavesPerEU does not imply a VGPR limit.
1451 bool DynamicVGPREnabled = (DynamicVGPRBlockSize != 0);
1452 unsigned MaxNumVGPRs =
1453 DynamicVGPREnabled
1454 ? TotNumVGPRs
1455 : alignDown(TotNumVGPRs / WavesPerEU,
1456 getVGPRAllocGranule(STI, DynamicVGPRBlockSize));
1457 unsigned AddressableNumVGPRs =
1458 getAddressableNumVGPRs(STI, DynamicVGPRBlockSize);
1459 return std::min(MaxNumVGPRs, AddressableNumVGPRs);
1460}
1461
1463 unsigned NumVGPRs,
1464 unsigned DynamicVGPRBlockSize,
1465 std::optional<bool> EnableWavefrontSize32) {
1467 NumVGPRs,
1468 getVGPRAllocGranule(STI, DynamicVGPRBlockSize, EnableWavefrontSize32));
1469}
1470} // end namespace IsaInfo
1471
1473 const MCSubtargetInfo &STI) {
1475 KernelCode.amd_kernel_code_version_major = 1;
1476 KernelCode.amd_kernel_code_version_minor = 2;
1477 KernelCode.amd_machine_kind = 1; // AMD_MACHINE_KIND_AMDGPU
1478 KernelCode.amd_machine_version_major = Version.Major;
1479 KernelCode.amd_machine_version_minor = Version.Minor;
1480 KernelCode.amd_machine_version_stepping = Version.Stepping;
1482 if (STI.getFeatureBits().test(FeatureWavefrontSize32)) {
1483 KernelCode.wavefront_size = 5;
1485 } else {
1486 KernelCode.wavefront_size = 6;
1487 }
1488
1489 // If the code object does not support indirect functions, then the value must
1490 // be 0xffffffff.
1491 KernelCode.call_convention = -1;
1492
1493 // These alignment values are specified in powers of two, so alignment =
1494 // 2^n. The minimum alignment is 2^4 = 16.
1495 KernelCode.kernarg_segment_alignment = 4;
1496 KernelCode.group_segment_alignment = 4;
1497 KernelCode.private_segment_alignment = 4;
1498
1499 if (Version.Major >= 10) {
1500 KernelCode.compute_pgm_resource_registers |=
1501 S_00B848_WGP_MODE(STI.getFeatureBits().test(FeatureCuMode) ? 0 : 1) |
1503 }
1504}
1505
1507 unsigned AS = GV->getAddressSpace();
1508 return AS == AMDGPUAS::CONSTANT_ADDRESS ||
1510}
1511
1513 return TT.getArch() == Triple::r600;
1514}
1515
1516static bool isValidRegPrefix(char C) {
1517 return C == 'v' || C == 's' || C == 'a';
1518}
1519
1520std::tuple<char, unsigned, unsigned> parseAsmPhysRegName(StringRef RegName) {
1521 if (RegName.empty())
1522 return {};
1523
1524 char Kind = RegName.front();
1525 if (!isValidRegPrefix(Kind))
1526 return {};
1527
1528 RegName = RegName.drop_front();
1529 if (RegName.consume_front("[")) {
1530 unsigned Idx, End;
1531 bool Failed = RegName.consumeInteger(10, Idx);
1532 Failed |= !RegName.consume_front(":");
1533 Failed |= RegName.consumeInteger(10, End);
1534 Failed |= !RegName.consume_back("]");
1535 if (!Failed) {
1536 unsigned NumRegs = End - Idx + 1;
1537 if (NumRegs > 1)
1538 return {Kind, Idx, NumRegs};
1539 }
1540 } else {
1541 unsigned Idx;
1542 bool Failed = RegName.getAsInteger(10, Idx);
1543 if (!Failed)
1544 return {Kind, Idx, 1};
1545 }
1546
1547 return {};
1548}
1549
1550std::tuple<char, unsigned, unsigned>
1552 StringRef RegName = Constraint;
1553 if (!RegName.consume_front("{") || !RegName.consume_back("}"))
1554 return {};
1556}
1557
1558std::pair<unsigned, unsigned>
1560 std::pair<unsigned, unsigned> Default,
1561 bool OnlyFirstRequired) {
1562 if (auto Attr = getIntegerPairAttribute(F, Name, OnlyFirstRequired))
1563 return {Attr->first, Attr->second.value_or(Default.second)};
1564 return Default;
1565}
1566
1567std::optional<std::pair<unsigned, std::optional<unsigned>>>
1569 bool OnlyFirstRequired) {
1570 Attribute A = F.getFnAttribute(Name);
1571 if (!A.isStringAttribute())
1572 return std::nullopt;
1573
1574 LLVMContext &Ctx = F.getContext();
1575 std::pair<unsigned, std::optional<unsigned>> Ints;
1576 std::pair<StringRef, StringRef> Strs = A.getValueAsString().split(',');
1577 if (Strs.first.trim().getAsInteger(0, Ints.first)) {
1578 Ctx.emitError("can't parse first integer attribute " + Name);
1579 return std::nullopt;
1580 }
1581 unsigned Second = 0;
1582 if (Strs.second.trim().getAsInteger(0, Second)) {
1583 if (!OnlyFirstRequired || !Strs.second.trim().empty()) {
1584 Ctx.emitError("can't parse second integer attribute " + Name);
1585 return std::nullopt;
1586 }
1587 } else {
1588 Ints.second = Second;
1589 }
1590
1591 return Ints;
1592}
1593
1595 unsigned Size,
1596 unsigned DefaultVal) {
1597 std::optional<SmallVector<unsigned>> R =
1599 return R.has_value() ? *R : SmallVector<unsigned>(Size, DefaultVal);
1600}
1601
1602std::optional<SmallVector<unsigned>>
1604 assert(Size > 2);
1605 LLVMContext &Ctx = F.getContext();
1606
1607 Attribute A = F.getFnAttribute(Name);
1608 if (!A.isValid())
1609 return std::nullopt;
1610 if (!A.isStringAttribute()) {
1611 Ctx.emitError(Name + " is not a string attribute");
1612 return std::nullopt;
1613 }
1614
1616
1617 StringRef S = A.getValueAsString();
1618 unsigned i = 0;
1619 for (; !S.empty() && i < Size; i++) {
1620 std::pair<StringRef, StringRef> Strs = S.split(',');
1621 unsigned IntVal;
1622 if (Strs.first.trim().getAsInteger(0, IntVal)) {
1623 Ctx.emitError("can't parse integer attribute " + Strs.first + " in " +
1624 Name);
1625 return std::nullopt;
1626 }
1627 Vals[i] = IntVal;
1628 S = Strs.second;
1629 }
1630
1631 if (!S.empty() || i < Size) {
1632 Ctx.emitError("attribute " + Name +
1633 " has incorrect number of integers; expected " +
1635 return std::nullopt;
1636 }
1637 return Vals;
1638}
1639
1641 return getIntegerVecAttribute(F, "amdgpu-max-num-workgroups", 3,
1642 std::numeric_limits<uint32_t>::max());
1643}
1644
1645bool hasValueInRangeLikeMetadata(const MDNode &MD, int64_t Val) {
1646 assert((MD.getNumOperands() % 2 == 0) && "invalid number of operands!");
1647 for (unsigned I = 0, E = MD.getNumOperands() / 2; I != E; ++I) {
1648 auto Low =
1649 mdconst::extract<ConstantInt>(MD.getOperand(2 * I + 0))->getValue();
1650 auto High =
1651 mdconst::extract<ConstantInt>(MD.getOperand(2 * I + 1))->getValue();
1652 // There are two types of [A; B) ranges:
1653 // A < B, e.g. [4; 5) which is a range that only includes 4.
1654 // A > B, e.g. [5; 4) which is a range that wraps around and includes
1655 // everything except 4.
1656 if (Low.ult(High)) {
1657 if (Low.ule(Val) && High.ugt(Val))
1658 return true;
1659 } else {
1660 if (Low.uge(Val) && High.ult(Val))
1661 return true;
1662 }
1663 }
1664
1665 return false;
1666}
1667
1669 return (1 << (getVmcntBitWidthLo(Version.Major) +
1670 getVmcntBitWidthHi(Version.Major))) -
1671 1;
1672}
1673
1675 return (1 << getLoadcntBitWidth(Version.Major)) - 1;
1676}
1677
1679 return (1 << getSamplecntBitWidth(Version.Major)) - 1;
1680}
1681
1683 return (1 << getBvhcntBitWidth(Version.Major)) - 1;
1684}
1685
1687 return (1 << getExpcntBitWidth(Version.Major)) - 1;
1688}
1689
1691 return (1 << getLgkmcntBitWidth(Version.Major)) - 1;
1692}
1693
1695 return (1 << getDscntBitWidth(Version.Major)) - 1;
1696}
1697
1699 return (1 << getKmcntBitWidth(Version.Major)) - 1;
1700}
1701
1703 return (1 << getXcntBitWidth(Version.Major, Version.Minor)) - 1;
1704}
1705
1707 return (1 << getAsynccntBitWidth(Version.Major, Version.Minor)) - 1;
1708}
1709
1711 return (1 << getStorecntBitWidth(Version.Major)) - 1;
1712}
1713
1715 return (1 << getBarrierMemberCountBitWidth(Version.Major)) - 1;
1716}
1717
1719 unsigned VmcntLo = getBitMask(getVmcntBitShiftLo(Version.Major),
1720 getVmcntBitWidthLo(Version.Major));
1721 unsigned Expcnt = getBitMask(getExpcntBitShift(Version.Major),
1722 getExpcntBitWidth(Version.Major));
1723 unsigned Lgkmcnt = getBitMask(getLgkmcntBitShift(Version.Major),
1724 getLgkmcntBitWidth(Version.Major));
1725 unsigned VmcntHi = getBitMask(getVmcntBitShiftHi(Version.Major),
1726 getVmcntBitWidthHi(Version.Major));
1727 return VmcntLo | Expcnt | Lgkmcnt | VmcntHi;
1728}
1729
1730unsigned decodeVmcnt(const IsaVersion &Version, unsigned Waitcnt) {
1731 unsigned VmcntLo = unpackBits(Waitcnt, getVmcntBitShiftLo(Version.Major),
1732 getVmcntBitWidthLo(Version.Major));
1733 unsigned VmcntHi = unpackBits(Waitcnt, getVmcntBitShiftHi(Version.Major),
1734 getVmcntBitWidthHi(Version.Major));
1735 return VmcntLo | VmcntHi << getVmcntBitWidthLo(Version.Major);
1736}
1737
1738unsigned decodeExpcnt(const IsaVersion &Version, unsigned Waitcnt) {
1739 return unpackBits(Waitcnt, getExpcntBitShift(Version.Major),
1740 getExpcntBitWidth(Version.Major));
1741}
1742
1743unsigned decodeLgkmcnt(const IsaVersion &Version, unsigned Waitcnt) {
1744 return unpackBits(Waitcnt, getLgkmcntBitShift(Version.Major),
1745 getLgkmcntBitWidth(Version.Major));
1746}
1747
1748unsigned decodeLoadcnt(const IsaVersion &Version, unsigned Waitcnt) {
1749 return unpackBits(Waitcnt, getLoadcntStorecntBitShift(Version.Major),
1750 getLoadcntBitWidth(Version.Major));
1751}
1752
1753unsigned decodeStorecnt(const IsaVersion &Version, unsigned Waitcnt) {
1754 return unpackBits(Waitcnt, getLoadcntStorecntBitShift(Version.Major),
1755 getStorecntBitWidth(Version.Major));
1756}
1757
1758unsigned decodeDscnt(const IsaVersion &Version, unsigned Waitcnt) {
1759 return unpackBits(Waitcnt, getDscntBitShift(Version.Major),
1760 getDscntBitWidth(Version.Major));
1761}
1762
1763void decodeWaitcnt(const IsaVersion &Version, unsigned Waitcnt, unsigned &Vmcnt,
1764 unsigned &Expcnt, unsigned &Lgkmcnt) {
1765 Vmcnt = decodeVmcnt(Version, Waitcnt);
1766 Expcnt = decodeExpcnt(Version, Waitcnt);
1767 Lgkmcnt = decodeLgkmcnt(Version, Waitcnt);
1768}
1769
1770unsigned encodeVmcnt(const IsaVersion &Version, unsigned Waitcnt,
1771 unsigned Vmcnt) {
1772 Waitcnt = packBits(Vmcnt, Waitcnt, getVmcntBitShiftLo(Version.Major),
1773 getVmcntBitWidthLo(Version.Major));
1774 return packBits(Vmcnt >> getVmcntBitWidthLo(Version.Major), Waitcnt,
1775 getVmcntBitShiftHi(Version.Major),
1776 getVmcntBitWidthHi(Version.Major));
1777}
1778
1779unsigned encodeExpcnt(const IsaVersion &Version, unsigned Waitcnt,
1780 unsigned Expcnt) {
1781 return packBits(Expcnt, Waitcnt, getExpcntBitShift(Version.Major),
1782 getExpcntBitWidth(Version.Major));
1783}
1784
1785unsigned encodeLgkmcnt(const IsaVersion &Version, unsigned Waitcnt,
1786 unsigned Lgkmcnt) {
1787 return packBits(Lgkmcnt, Waitcnt, getLgkmcntBitShift(Version.Major),
1788 getLgkmcntBitWidth(Version.Major));
1789}
1790
1791unsigned encodeWaitcnt(const IsaVersion &Version, unsigned Vmcnt,
1792 unsigned Expcnt, unsigned Lgkmcnt) {
1793 unsigned Waitcnt = getWaitcntBitMask(Version);
1795 Waitcnt = encodeExpcnt(Version, Waitcnt, Expcnt);
1796 Waitcnt = encodeLgkmcnt(Version, Waitcnt, Lgkmcnt);
1797 return Waitcnt;
1798}
1799
1801 bool IsStore) {
1802 unsigned Dscnt = getBitMask(getDscntBitShift(Version.Major),
1803 getDscntBitWidth(Version.Major));
1804 if (IsStore) {
1805 unsigned Storecnt = getBitMask(getLoadcntStorecntBitShift(Version.Major),
1806 getStorecntBitWidth(Version.Major));
1807 return Dscnt | Storecnt;
1808 }
1809 unsigned Loadcnt = getBitMask(getLoadcntStorecntBitShift(Version.Major),
1810 getLoadcntBitWidth(Version.Major));
1811 return Dscnt | Loadcnt;
1812}
1813
1814static unsigned encodeLoadcnt(const IsaVersion &Version, unsigned Waitcnt,
1815 unsigned Loadcnt) {
1816 return packBits(Loadcnt, Waitcnt, getLoadcntStorecntBitShift(Version.Major),
1817 getLoadcntBitWidth(Version.Major));
1818}
1819
1820static unsigned encodeStorecnt(const IsaVersion &Version, unsigned Waitcnt,
1821 unsigned Storecnt) {
1822 return packBits(Storecnt, Waitcnt, getLoadcntStorecntBitShift(Version.Major),
1823 getStorecntBitWidth(Version.Major));
1824}
1825
1826static unsigned encodeDscnt(const IsaVersion &Version, unsigned Waitcnt,
1827 unsigned Dscnt) {
1828 return packBits(Dscnt, Waitcnt, getDscntBitShift(Version.Major),
1829 getDscntBitWidth(Version.Major));
1830}
1831
1832unsigned encodeLoadcntDscnt(const IsaVersion &Version, unsigned Loadcnt,
1833 unsigned Dscnt) {
1834 unsigned Waitcnt = getCombinedCountBitMask(Version, false);
1835 Waitcnt = encodeLoadcnt(Version, Waitcnt, Loadcnt);
1837 return Waitcnt;
1838}
1839
1840unsigned encodeStorecntDscnt(const IsaVersion &Version, unsigned Storecnt,
1841 unsigned Dscnt) {
1842 unsigned Waitcnt = getCombinedCountBitMask(Version, true);
1843 Waitcnt = encodeStorecnt(Version, Waitcnt, Storecnt);
1845 return Waitcnt;
1846}
1847
1848//===----------------------------------------------------------------------===//
1849// Custom Operand Values
1850//===----------------------------------------------------------------------===//
1851
1853 int Size, unsigned Code,
1854 bool &HasNonDefaultVal,
1855 const MCSubtargetInfo &STI) {
1856 unsigned UsedOprMask = 0;
1857 HasNonDefaultVal = false;
1858 for (int Idx = 0; Idx < Size; ++Idx) {
1859 const auto &Op = Opr[Idx];
1860 if (!Op.isSupported(STI))
1861 continue;
1862 UsedOprMask |= Op.getMask();
1863 unsigned Val = Op.decode(Code);
1864 if (!Op.isValid(Val))
1865 return false;
1866 HasNonDefaultVal |= (Val != Op.Default);
1867 }
1868 return (Code & ~UsedOprMask) == 0;
1869}
1870
1871static bool decodeCustomOperand(const CustomOperandVal *Opr, int Size,
1872 unsigned Code, int &Idx, StringRef &Name,
1873 unsigned &Val, bool &IsDefault,
1874 const MCSubtargetInfo &STI) {
1875 while (Idx < Size) {
1876 const auto &Op = Opr[Idx++];
1877 if (Op.isSupported(STI)) {
1878 Name = Op.Name;
1879 Val = Op.decode(Code);
1880 IsDefault = (Val == Op.Default);
1881 return true;
1882 }
1883 }
1884
1885 return false;
1886}
1887
1889 int64_t InputVal) {
1890 if (InputVal < 0 || InputVal > Op.Max)
1891 return OPR_VAL_INVALID;
1892 return Op.encode(static_cast<unsigned>(InputVal));
1893}
1894
1895static int encodeCustomOperand(const CustomOperandVal *Opr, int Size,
1896 const StringRef Name, int64_t InputVal,
1897 unsigned &UsedOprMask,
1898 const MCSubtargetInfo &STI) {
1899 int InvalidId = OPR_ID_UNKNOWN;
1900 for (int Idx = 0; Idx < Size; ++Idx) {
1901 const auto &Op = Opr[Idx];
1902 if (Op.Name == Name) {
1903 if (!Op.isSupported(STI)) {
1904 InvalidId = OPR_ID_UNSUPPORTED;
1905 continue;
1906 }
1907 auto OprMask = Op.getMask();
1908 if (OprMask & UsedOprMask)
1909 return OPR_ID_DUPLICATE;
1910 UsedOprMask |= OprMask;
1911 return encodeCustomOperandVal(Op, InputVal);
1912 }
1913 }
1914 return InvalidId;
1915}
1916
1917//===----------------------------------------------------------------------===//
1918// DepCtr
1919//===----------------------------------------------------------------------===//
1920
1921namespace DepCtr {
1922
1924 unsigned Enc = 0;
1925 for (int Idx = 0; Idx < DEP_CTR_SIZE; ++Idx) {
1926 const auto &Op = DepCtrInfo[Idx];
1927 if (Op.isSupported(STI))
1928 Enc |= Op.encode(Op.Default);
1929 }
1930 return Enc;
1931}
1932
1933bool isSymbolicDepCtrEncoding(unsigned Code, bool &HasNonDefaultVal,
1934 const MCSubtargetInfo &STI) {
1936 HasNonDefaultVal, STI);
1937}
1938
1939bool decodeDepCtr(unsigned Code, int &Id, StringRef &Name, unsigned &Val,
1940 bool &IsDefault, const MCSubtargetInfo &STI) {
1941 return decodeCustomOperand(DepCtrInfo, DEP_CTR_SIZE, Code, Id, Name, Val,
1942 IsDefault, STI);
1943}
1944
1945int encodeDepCtr(const StringRef Name, int64_t Val, unsigned &UsedOprMask,
1946 const MCSubtargetInfo &STI) {
1947 return encodeCustomOperand(DepCtrInfo, DEP_CTR_SIZE, Name, Val, UsedOprMask,
1948 STI);
1949}
1950
1951unsigned getVaVdstBitMask() { return (1 << getVaVdstBitWidth()) - 1; }
1952
1953unsigned getVaSdstBitMask() { return (1 << getVaSdstBitWidth()) - 1; }
1954
1955unsigned getVaSsrcBitMask() { return (1 << getVaSsrcBitWidth()) - 1; }
1956
1958 return (1 << getHoldCntWidth(Version.Major, Version.Minor)) - 1;
1959}
1960
1961unsigned getVmVsrcBitMask() { return (1 << getVmVsrcBitWidth()) - 1; }
1962
1963unsigned getVaVccBitMask() { return (1 << getVaVccBitWidth()) - 1; }
1964
1965unsigned getSaSdstBitMask() { return (1 << getSaSdstBitWidth()) - 1; }
1966
1967unsigned decodeFieldVmVsrc(unsigned Encoded) {
1968 return unpackBits(Encoded, getVmVsrcBitShift(), getVmVsrcBitWidth());
1969}
1970
1971unsigned decodeFieldVaVdst(unsigned Encoded) {
1972 return unpackBits(Encoded, getVaVdstBitShift(), getVaVdstBitWidth());
1973}
1974
1975unsigned decodeFieldSaSdst(unsigned Encoded) {
1976 return unpackBits(Encoded, getSaSdstBitShift(), getSaSdstBitWidth());
1977}
1978
1979unsigned decodeFieldVaSdst(unsigned Encoded) {
1980 return unpackBits(Encoded, getVaSdstBitShift(), getVaSdstBitWidth());
1981}
1982
1983unsigned decodeFieldVaVcc(unsigned Encoded) {
1984 return unpackBits(Encoded, getVaVccBitShift(), getVaVccBitWidth());
1985}
1986
1987unsigned decodeFieldVaSsrc(unsigned Encoded) {
1988 return unpackBits(Encoded, getVaSsrcBitShift(), getVaSsrcBitWidth());
1989}
1990
1991unsigned decodeFieldHoldCnt(unsigned Encoded, const IsaVersion &Version) {
1992 return unpackBits(Encoded, getHoldCntBitShift(),
1993 getHoldCntWidth(Version.Major, Version.Minor));
1994}
1995
1996unsigned encodeFieldVmVsrc(unsigned Encoded, unsigned VmVsrc) {
1997 return packBits(VmVsrc, Encoded, getVmVsrcBitShift(), getVmVsrcBitWidth());
1998}
1999
2000unsigned encodeFieldVmVsrc(unsigned VmVsrc, const MCSubtargetInfo &STI) {
2001 unsigned Encoded = getDefaultDepCtrEncoding(STI);
2002 return encodeFieldVmVsrc(Encoded, VmVsrc);
2003}
2004
2005unsigned encodeFieldVaVdst(unsigned Encoded, unsigned VaVdst) {
2006 return packBits(VaVdst, Encoded, getVaVdstBitShift(), getVaVdstBitWidth());
2007}
2008
2009unsigned encodeFieldVaVdst(unsigned VaVdst, const MCSubtargetInfo &STI) {
2010 unsigned Encoded = getDefaultDepCtrEncoding(STI);
2011 return encodeFieldVaVdst(Encoded, VaVdst);
2012}
2013
2014unsigned encodeFieldSaSdst(unsigned Encoded, unsigned SaSdst) {
2015 return packBits(SaSdst, Encoded, getSaSdstBitShift(), getSaSdstBitWidth());
2016}
2017
2018unsigned encodeFieldSaSdst(unsigned SaSdst, const MCSubtargetInfo &STI) {
2019 unsigned Encoded = getDefaultDepCtrEncoding(STI);
2020 return encodeFieldSaSdst(Encoded, SaSdst);
2021}
2022
2023unsigned encodeFieldVaSdst(unsigned Encoded, unsigned VaSdst) {
2024 return packBits(VaSdst, Encoded, getVaSdstBitShift(), getVaSdstBitWidth());
2025}
2026
2027unsigned encodeFieldVaSdst(unsigned VaSdst, const MCSubtargetInfo &STI) {
2028 unsigned Encoded = getDefaultDepCtrEncoding(STI);
2029 return encodeFieldVaSdst(Encoded, VaSdst);
2030}
2031
2032unsigned encodeFieldVaVcc(unsigned Encoded, unsigned VaVcc) {
2033 return packBits(VaVcc, Encoded, getVaVccBitShift(), getVaVccBitWidth());
2034}
2035
2036unsigned encodeFieldVaVcc(unsigned VaVcc, const MCSubtargetInfo &STI) {
2037 unsigned Encoded = getDefaultDepCtrEncoding(STI);
2038 return encodeFieldVaVcc(Encoded, VaVcc);
2039}
2040
2041unsigned encodeFieldVaSsrc(unsigned Encoded, unsigned VaSsrc) {
2042 return packBits(VaSsrc, Encoded, getVaSsrcBitShift(), getVaSsrcBitWidth());
2043}
2044
2045unsigned encodeFieldVaSsrc(unsigned VaSsrc, const MCSubtargetInfo &STI) {
2046 unsigned Encoded = getDefaultDepCtrEncoding(STI);
2047 return encodeFieldVaSsrc(Encoded, VaSsrc);
2048}
2049
2050unsigned encodeFieldHoldCnt(unsigned Encoded, unsigned HoldCnt,
2051 const IsaVersion &Version) {
2052 return packBits(HoldCnt, Encoded, getHoldCntBitShift(),
2053 getHoldCntWidth(Version.Major, Version.Minor));
2054}
2055
2056unsigned encodeFieldHoldCnt(unsigned HoldCnt, const MCSubtargetInfo &STI) {
2057 unsigned Encoded = getDefaultDepCtrEncoding(STI);
2058 return encodeFieldHoldCnt(Encoded, HoldCnt, getIsaVersion(STI.getCPU()));
2059}
2060
2061} // namespace DepCtr
2062
2063//===----------------------------------------------------------------------===//
2064// exp tgt
2065//===----------------------------------------------------------------------===//
2066
2067namespace Exp {
2068
2069struct ExpTgt {
2071 unsigned Tgt;
2072 unsigned MaxIndex;
2073};
2074
2075// clang-format off
2076static constexpr ExpTgt ExpTgtInfo[] = {
2077 {{"null"}, ET_NULL, ET_NULL_MAX_IDX},
2078 {{"mrtz"}, ET_MRTZ, ET_MRTZ_MAX_IDX},
2079 {{"prim"}, ET_PRIM, ET_PRIM_MAX_IDX},
2080 {{"mrt"}, ET_MRT0, ET_MRT_MAX_IDX},
2081 {{"pos"}, ET_POS0, ET_POS_MAX_IDX},
2082 {{"dual_src_blend"},ET_DUAL_SRC_BLEND0, ET_DUAL_SRC_BLEND_MAX_IDX},
2083 {{"param"}, ET_PARAM0, ET_PARAM_MAX_IDX},
2084};
2085// clang-format on
2086
2087bool getTgtName(unsigned Id, StringRef &Name, int &Index) {
2088 for (const ExpTgt &Val : ExpTgtInfo) {
2089 if (Val.Tgt <= Id && Id <= Val.Tgt + Val.MaxIndex) {
2090 Index = (Val.MaxIndex == 0) ? -1 : (Id - Val.Tgt);
2091 Name = Val.Name;
2092 return true;
2093 }
2094 }
2095 return false;
2096}
2097
2098unsigned getTgtId(const StringRef Name) {
2099
2100 for (const ExpTgt &Val : ExpTgtInfo) {
2101 if (Val.MaxIndex == 0 && Name == Val.Name)
2102 return Val.Tgt;
2103
2104 if (Val.MaxIndex > 0 && Name.starts_with(Val.Name)) {
2105 StringRef Suffix = Name.drop_front(Val.Name.size());
2106
2107 unsigned Id;
2108 if (Suffix.getAsInteger(10, Id) || Id > Val.MaxIndex)
2109 return ET_INVALID;
2110
2111 // Disable leading zeroes
2112 if (Suffix.size() > 1 && Suffix[0] == '0')
2113 return ET_INVALID;
2114
2115 return Val.Tgt + Id;
2116 }
2117 }
2118 return ET_INVALID;
2119}
2120
2121bool isSupportedTgtId(unsigned Id, const MCSubtargetInfo &STI) {
2122 switch (Id) {
2123 case ET_NULL:
2124 return !isGFX11Plus(STI);
2125 case ET_POS4:
2126 case ET_PRIM:
2127 return isGFX10Plus(STI);
2128 case ET_DUAL_SRC_BLEND0:
2129 case ET_DUAL_SRC_BLEND1:
2130 return isGFX11Plus(STI);
2131 default:
2132 if (Id >= ET_PARAM0 && Id <= ET_PARAM31)
2133 return !isGFX11Plus(STI) || isGFX13Plus(STI);
2134 return true;
2135 }
2136}
2137
2138} // namespace Exp
2139
2140//===----------------------------------------------------------------------===//
2141// MTBUF Format
2142//===----------------------------------------------------------------------===//
2143
2144namespace MTBUFFormat {
2145
2146int64_t getDfmt(const StringRef Name) {
2147 for (int Id = DFMT_MIN; Id <= DFMT_MAX; ++Id) {
2148 if (Name == DfmtSymbolic[Id])
2149 return Id;
2150 }
2151 return DFMT_UNDEF;
2152}
2153
2155 assert(Id <= DFMT_MAX);
2156 return DfmtSymbolic[Id];
2157}
2158
2160 if (isSI(STI) || isCI(STI))
2161 return NfmtSymbolicSICI;
2162 if (isVI(STI) || isGFX9(STI))
2163 return NfmtSymbolicVI;
2164 return NfmtSymbolicGFX10;
2165}
2166
2167int64_t getNfmt(const StringRef Name, const MCSubtargetInfo &STI) {
2168 const auto *lookupTable = getNfmtLookupTable(STI);
2169 for (int Id = NFMT_MIN; Id <= NFMT_MAX; ++Id) {
2170 if (Name == lookupTable[Id])
2171 return Id;
2172 }
2173 return NFMT_UNDEF;
2174}
2175
2176StringRef getNfmtName(unsigned Id, const MCSubtargetInfo &STI) {
2177 assert(Id <= NFMT_MAX);
2178 return getNfmtLookupTable(STI)[Id];
2179}
2180
2181bool isValidDfmtNfmt(unsigned Id, const MCSubtargetInfo &STI) {
2182 unsigned Dfmt;
2183 unsigned Nfmt;
2184 decodeDfmtNfmt(Id, Dfmt, Nfmt);
2185 return isValidNfmt(Nfmt, STI);
2186}
2187
2188bool isValidNfmt(unsigned Id, const MCSubtargetInfo &STI) {
2189 return !getNfmtName(Id, STI).empty();
2190}
2191
2192int64_t encodeDfmtNfmt(unsigned Dfmt, unsigned Nfmt) {
2193 return (Dfmt << DFMT_SHIFT) | (Nfmt << NFMT_SHIFT);
2194}
2195
2196void decodeDfmtNfmt(unsigned Format, unsigned &Dfmt, unsigned &Nfmt) {
2197 Dfmt = (Format >> DFMT_SHIFT) & DFMT_MASK;
2198 Nfmt = (Format >> NFMT_SHIFT) & NFMT_MASK;
2199}
2200
2201int64_t getUnifiedFormat(const StringRef Name, const MCSubtargetInfo &STI) {
2202 if (isGFX11Plus(STI)) {
2203 for (int Id = UfmtGFX11::UFMT_FIRST; Id <= UfmtGFX11::UFMT_LAST; ++Id) {
2204 if (Name == UfmtSymbolicGFX11[Id])
2205 return Id;
2206 }
2207 } else {
2208 for (int Id = UfmtGFX10::UFMT_FIRST; Id <= UfmtGFX10::UFMT_LAST; ++Id) {
2209 if (Name == UfmtSymbolicGFX10[Id])
2210 return Id;
2211 }
2212 }
2213 return UFMT_UNDEF;
2214}
2215
2217 if (isValidUnifiedFormat(Id, STI))
2218 return isGFX10(STI) ? UfmtSymbolicGFX10[Id] : UfmtSymbolicGFX11[Id];
2219 return "";
2220}
2221
2222bool isValidUnifiedFormat(unsigned Id, const MCSubtargetInfo &STI) {
2223 return isGFX10(STI) ? Id <= UfmtGFX10::UFMT_LAST : Id <= UfmtGFX11::UFMT_LAST;
2224}
2225
2226int64_t convertDfmtNfmt2Ufmt(unsigned Dfmt, unsigned Nfmt,
2227 const MCSubtargetInfo &STI) {
2228 int64_t Fmt = encodeDfmtNfmt(Dfmt, Nfmt);
2229 if (isGFX11Plus(STI)) {
2230 for (int Id = UfmtGFX11::UFMT_FIRST; Id <= UfmtGFX11::UFMT_LAST; ++Id) {
2231 if (Fmt == DfmtNfmt2UFmtGFX11[Id])
2232 return Id;
2233 }
2234 } else {
2235 for (int Id = UfmtGFX10::UFMT_FIRST; Id <= UfmtGFX10::UFMT_LAST; ++Id) {
2236 if (Fmt == DfmtNfmt2UFmtGFX10[Id])
2237 return Id;
2238 }
2239 }
2240 return UFMT_UNDEF;
2241}
2242
2243bool isValidFormatEncoding(unsigned Val, const MCSubtargetInfo &STI) {
2244 return isGFX10Plus(STI) ? (Val <= UFMT_MAX) : (Val <= DFMT_NFMT_MAX);
2245}
2246
2248 if (isGFX10Plus(STI))
2249 return UFMT_DEFAULT;
2250 return DFMT_NFMT_DEFAULT;
2251}
2252
2253} // namespace MTBUFFormat
2254
2255//===----------------------------------------------------------------------===//
2256// SendMsg
2257//===----------------------------------------------------------------------===//
2258
2259namespace SendMsg {
2260
2264
2265bool isValidMsgId(int64_t MsgId, const MCSubtargetInfo &STI) {
2266 return (MsgId & ~(getMsgIdMask(STI))) == 0;
2267}
2268
2269bool isValidMsgOp(int64_t MsgId, int64_t OpId, const MCSubtargetInfo &STI,
2270 bool Strict) {
2271 assert(isValidMsgId(MsgId, STI));
2272
2273 if (!Strict)
2274 return 0 <= OpId && isUInt<OP_WIDTH_>(OpId);
2275
2276 if (msgRequiresOp(MsgId, STI)) {
2277 if (MsgId == ID_GS_PreGFX11 && OpId == OP_GS_NOP)
2278 return false;
2279
2280 return !getMsgOpName(MsgId, OpId, STI).empty();
2281 }
2282
2283 return OpId == OP_NONE_;
2284}
2285
2286bool isValidMsgStream(int64_t MsgId, int64_t OpId, int64_t StreamId,
2287 const MCSubtargetInfo &STI, bool Strict) {
2288 assert(isValidMsgOp(MsgId, OpId, STI, Strict));
2289
2290 if (!Strict)
2292
2293 if (!isGFX11Plus(STI)) {
2294 switch (MsgId) {
2295 case ID_GS_PreGFX11:
2298 return (OpId == OP_GS_NOP)
2301 }
2302 }
2303 return StreamId == STREAM_ID_NONE_;
2304}
2305
2306bool msgRequiresOp(int64_t MsgId, const MCSubtargetInfo &STI) {
2307 return MsgId == ID_SYSMSG ||
2308 (!isGFX11Plus(STI) &&
2309 (MsgId == ID_GS_PreGFX11 || MsgId == ID_GS_DONE_PreGFX11));
2310}
2311
2312bool msgSupportsStream(int64_t MsgId, int64_t OpId,
2313 const MCSubtargetInfo &STI) {
2314 return !isGFX11Plus(STI) &&
2315 (MsgId == ID_GS_PreGFX11 || MsgId == ID_GS_DONE_PreGFX11) &&
2316 OpId != OP_GS_NOP;
2317}
2318
2319void decodeMsg(unsigned Val, uint16_t &MsgId, uint16_t &OpId,
2320 uint16_t &StreamId, const MCSubtargetInfo &STI) {
2321 MsgId = static_cast<uint16_t>(Val & getMsgIdMask(STI));
2322 if (isGFX11Plus(STI)) {
2323 OpId = 0;
2324 StreamId = 0;
2325 } else {
2326 OpId = (Val & OP_MASK_) >> OP_SHIFT_;
2328 }
2329}
2330
2332 return MsgId | (OpId << OP_SHIFT_) | (StreamId << STREAM_ID_SHIFT_);
2333}
2334
2335bool msgDoesNotUseM0(int64_t MsgId, const MCSubtargetInfo &STI) {
2336 // Explicitly list message types that are known to not use m0.
2337 // This is safer than excluding only GS_ALLOC_REQ, in case new message
2338 // types are added in the future that do use m0.
2339 if (isGFX11Plus(STI)) {
2340 switch (MsgId) {
2342 return true;
2343 default:
2344 break;
2345 }
2346 }
2347 switch (MsgId) {
2348 case ID_SAVEWAVE:
2349 case ID_STALL_WAVE_GEN:
2350 case ID_HALT_WAVES:
2351 case ID_ORDERED_PS_DONE:
2353 case ID_GET_DOORBELL:
2354 case ID_GET_DDID:
2355 case ID_SYSMSG:
2356 return true;
2357 default:
2358 return false;
2359 }
2360}
2361
2362} // namespace SendMsg
2363
2364//===----------------------------------------------------------------------===//
2365//
2366//===----------------------------------------------------------------------===//
2367
2369 return static_cast<unsigned>(
2370 F.getFnAttributeAsParsedInteger("InitialPSInputAddr", 0));
2371}
2372
2374 // As a safe default always respond as if PS has color exports.
2375 return F.getFnAttributeAsParsedInteger(
2376 "amdgpu-color-export",
2377 F.getCallingConv() == CallingConv::AMDGPU_PS ? 1 : 0) != 0;
2378}
2379
2381 return F.getFnAttributeAsParsedInteger("amdgpu-depth-export", 0) != 0;
2382}
2383
2385 unsigned BlockSize = static_cast<unsigned>(
2386 F.getFnAttributeAsParsedInteger("amdgpu-dynamic-vgpr-block-size", 0));
2387
2388 if (BlockSize == 16 || BlockSize == 32)
2389 return BlockSize;
2390
2391 return 0;
2392}
2393
2395 return STI.hasFeature(AMDGPU::FeatureMIMG_R128) &&
2396 !STI.hasFeature(AMDGPU::FeatureR128A16);
2397}
2398
2399bool hasA16(const MCSubtargetInfo &STI) {
2400 return STI.hasFeature(AMDGPU::FeatureA16);
2401}
2402
2403bool hasG16(const MCSubtargetInfo &STI) {
2404 return STI.hasFeature(AMDGPU::FeatureG16);
2405}
2406
2408 return !STI.hasFeature(AMDGPU::FeatureUnpackedD16VMem) && !isCI(STI) &&
2409 !isSI(STI);
2410}
2411
2412bool hasGDS(const MCSubtargetInfo &STI) {
2413 return STI.hasFeature(AMDGPU::FeatureGDS);
2414}
2415
2416unsigned getNSAMaxSize(const MCSubtargetInfo &STI, bool HasSampler) {
2417 auto Version = getIsaVersion(STI.getCPU());
2418 if (Version.Major == 10)
2419 return Version.Minor >= 3 ? 13 : 5;
2420 if (Version.Major == 11)
2421 return 5;
2422 if (Version.Major >= 12)
2423 return HasSampler ? 4 : 5;
2424 return 0;
2425}
2426
2428 if (isGFX1250Plus(STI))
2429 return 32;
2430 return 16;
2431}
2432
2433bool isSI(const MCSubtargetInfo &STI) {
2434 return STI.hasFeature(AMDGPU::FeatureSouthernIslands);
2435}
2436
2437bool isCI(const MCSubtargetInfo &STI) {
2438 return STI.hasFeature(AMDGPU::FeatureSeaIslands);
2439}
2440
2441bool isVI(const MCSubtargetInfo &STI) {
2442 return STI.hasFeature(AMDGPU::FeatureVolcanicIslands);
2443}
2444
2445bool isGFX9(const MCSubtargetInfo &STI) {
2446 return STI.hasFeature(AMDGPU::FeatureGFX9);
2447}
2448
2450 return isGFX9(STI) || isGFX10(STI);
2451}
2452
2454 return isGFX9(STI) || isGFX10(STI) || isGFX11(STI);
2455}
2456
2458 return isVI(STI) || isGFX9(STI) || isGFX10(STI);
2459}
2460
2461bool isGFX8Plus(const MCSubtargetInfo &STI) {
2462 return isVI(STI) || isGFX9Plus(STI);
2463}
2464
2465bool isGFX9Plus(const MCSubtargetInfo &STI) {
2466 return isGFX9(STI) || isGFX10Plus(STI);
2467}
2468
2469bool isNotGFX9Plus(const MCSubtargetInfo &STI) { return !isGFX9Plus(STI); }
2470
2472 return STI.hasFeature(AMDGPU::FeaturePopsExitingWaveID);
2473}
2474
2476 return STI.hasFeature(AMDGPU::FeatureApertureRegs) &&
2477 !STI.hasFeature(AMDGPU::FeatureGloballyAddressableScratch);
2478}
2479
2480bool isGFX10(const MCSubtargetInfo &STI) {
2481 return STI.hasFeature(AMDGPU::FeatureGFX10);
2482}
2483
2485 return isGFX10(STI) || isGFX11(STI);
2486}
2487
2489 return isGFX10(STI) || isGFX11Plus(STI);
2490}
2491
2492bool isGFX11(const MCSubtargetInfo &STI) {
2493 return STI.hasFeature(AMDGPU::FeatureGFX11);
2494}
2495
2497 return isGFX11(STI) || isGFX12Plus(STI);
2498}
2499
2500bool isGFX12(const MCSubtargetInfo &STI) {
2501 return STI.getFeatureBits()[AMDGPU::FeatureGFX12];
2502}
2503
2505 return isGFX12(STI) || isGFX13Plus(STI);
2506}
2507
2508bool isNotGFX12Plus(const MCSubtargetInfo &STI) { return !isGFX12Plus(STI); }
2509
2510bool isGFX1250(const MCSubtargetInfo &STI) {
2511 return STI.getFeatureBits()[AMDGPU::FeatureGFX1250Insts] && !isGFX13(STI);
2512}
2513
2515 return isGFX1250(STI) || !STI.getFeatureBits().test(FeatureCuMode);
2516}
2517
2519 return STI.getFeatureBits()[AMDGPU::FeatureGFX1250Insts];
2520}
2521
2522bool isGFX13(const MCSubtargetInfo &STI) {
2523 return STI.getFeatureBits()[AMDGPU::FeatureGFX13];
2524}
2525
2526bool isGFX13Plus(const MCSubtargetInfo &STI) { return isGFX13(STI); }
2527
2529 if (isGFX1250(STI))
2530 return false;
2531 return isGFX10Plus(STI);
2532}
2533
2534bool isNotGFX11Plus(const MCSubtargetInfo &STI) { return !isGFX11Plus(STI); }
2535
2537 return isSI(STI) || isCI(STI) || isVI(STI) || isGFX9(STI);
2538}
2539
2541 return isGFX10(STI) && !AMDGPU::isGFX10_BEncoding(STI);
2542}
2543
2545 return STI.hasFeature(AMDGPU::FeatureGCN3Encoding);
2546}
2547
2549 return STI.hasFeature(AMDGPU::FeatureGFX10_BEncoding);
2550}
2551
2553 return STI.hasFeature(AMDGPU::FeatureGFX10_3Insts);
2554}
2555
2557 return isGFX10_BEncoding(STI) && !isGFX12Plus(STI);
2558}
2559
2560bool isGFX90A(const MCSubtargetInfo &STI) {
2561 return STI.hasFeature(AMDGPU::FeatureGFX90AInsts);
2562}
2563
2564bool isGFX940(const MCSubtargetInfo &STI) {
2565 return STI.hasFeature(AMDGPU::FeatureGFX940Insts);
2566}
2567
2569 return STI.hasFeature(AMDGPU::FeatureArchitectedFlatScratch);
2570}
2571
2573 return STI.hasFeature(AMDGPU::FeatureMAIInsts);
2574}
2575
2576bool hasVOPD(const MCSubtargetInfo &STI) {
2577 return STI.hasFeature(AMDGPU::FeatureVOPDInsts);
2578}
2579
2581 return STI.hasFeature(AMDGPU::FeatureDPPSrc1SGPR);
2582}
2583
2585 return STI.hasFeature(AMDGPU::FeatureKernargPreload);
2586}
2587
2588uint32_t getTotalNumVGPRs(bool has90AInsts, uint32_t ArgNumAGPR,
2589 uint32_t ArgNumVGPR) {
2590 if (has90AInsts && ArgNumAGPR)
2591 return alignTo(ArgNumVGPR, 4) + ArgNumAGPR;
2592 return std::max(ArgNumVGPR, ArgNumAGPR);
2593}
2594
2596 const MCRegisterClass &SGPRClass =
2597 TRI->getRegClass(AMDGPU::SReg_32RegClassID);
2598 const MCRegister FirstSubReg = TRI->getSubReg(Reg, AMDGPU::sub0);
2599 return SGPRClass.contains(FirstSubReg != 0 ? FirstSubReg : Reg) ||
2600 Reg == AMDGPU::SCC;
2601}
2602
2604 return MRI.getRegClass(AMDGPU::RsrcReg32RegClassID).contains(Reg);
2605}
2606
2608 return MRI.getEncodingValue(Reg) & AMDGPU::HWEncoding::IS_HI16;
2609}
2610
2611#define MAP_REG2REG \
2612 using namespace AMDGPU; \
2613 switch (Reg.id()) { \
2614 default: \
2615 return Reg; \
2616 CASE_CI_VI(FLAT_SCR) \
2617 CASE_CI_VI(FLAT_SCR_LO) \
2618 CASE_CI_VI(FLAT_SCR_HI) \
2619 CASE_VI_GFX9PLUS(TTMP0) \
2620 CASE_VI_GFX9PLUS(TTMP1) \
2621 CASE_VI_GFX9PLUS(TTMP2) \
2622 CASE_VI_GFX9PLUS(TTMP3) \
2623 CASE_VI_GFX9PLUS(TTMP4) \
2624 CASE_VI_GFX9PLUS(TTMP5) \
2625 CASE_VI_GFX9PLUS(TTMP6) \
2626 CASE_VI_GFX9PLUS(TTMP7) \
2627 CASE_VI_GFX9PLUS(TTMP8) \
2628 CASE_VI_GFX9PLUS(TTMP9) \
2629 CASE_VI_GFX9PLUS(TTMP10) \
2630 CASE_VI_GFX9PLUS(TTMP11) \
2631 CASE_VI_GFX9PLUS(TTMP12) \
2632 CASE_VI_GFX9PLUS(TTMP13) \
2633 CASE_VI_GFX9PLUS(TTMP14) \
2634 CASE_VI_GFX9PLUS(TTMP15) \
2635 CASE_VI_GFX9PLUS(TTMP0_TTMP1) \
2636 CASE_VI_GFX9PLUS(TTMP2_TTMP3) \
2637 CASE_VI_GFX9PLUS(TTMP4_TTMP5) \
2638 CASE_VI_GFX9PLUS(TTMP6_TTMP7) \
2639 CASE_VI_GFX9PLUS(TTMP8_TTMP9) \
2640 CASE_VI_GFX9PLUS(TTMP10_TTMP11) \
2641 CASE_VI_GFX9PLUS(TTMP12_TTMP13) \
2642 CASE_VI_GFX9PLUS(TTMP14_TTMP15) \
2643 CASE_VI_GFX9PLUS(TTMP0_TTMP1_TTMP2_TTMP3) \
2644 CASE_VI_GFX9PLUS(TTMP4_TTMP5_TTMP6_TTMP7) \
2645 CASE_VI_GFX9PLUS(TTMP8_TTMP9_TTMP10_TTMP11) \
2646 CASE_VI_GFX9PLUS(TTMP12_TTMP13_TTMP14_TTMP15) \
2647 CASE_VI_GFX9PLUS(TTMP0_TTMP1_TTMP2_TTMP3_TTMP4_TTMP5_TTMP6_TTMP7) \
2648 CASE_VI_GFX9PLUS(TTMP4_TTMP5_TTMP6_TTMP7_TTMP8_TTMP9_TTMP10_TTMP11) \
2649 CASE_VI_GFX9PLUS(TTMP8_TTMP9_TTMP10_TTMP11_TTMP12_TTMP13_TTMP14_TTMP15) \
2650 CASE_VI_GFX9PLUS( \
2651 TTMP0_TTMP1_TTMP2_TTMP3_TTMP4_TTMP5_TTMP6_TTMP7_TTMP8_TTMP9_TTMP10_TTMP11_TTMP12_TTMP13_TTMP14_TTMP15) \
2652 CASE_GFXPRE11_GFX11PLUS(M0) \
2653 CASE_GFXPRE11_GFX11PLUS(SGPR_NULL) \
2654 CASE_GFXPRE11_GFX11PLUS_TO(SGPR_NULL64, SGPR_NULL) \
2655 }
2656
2657#define CASE_CI_VI(node) \
2658 assert(!isSI(STI)); \
2659 case node: \
2660 return isCI(STI) ? node##_ci : node##_vi;
2661
2662#define CASE_VI_GFX9PLUS(node) \
2663 case node: \
2664 return isGFX9Plus(STI) ? node##_gfx9plus : node##_vi;
2665
2666#define CASE_GFXPRE11_GFX11PLUS(node) \
2667 case node: \
2668 return isGFX11Plus(STI) ? node##_gfx11plus : node##_gfxpre11;
2669
2670#define CASE_GFXPRE11_GFX11PLUS_TO(node, result) \
2671 case node: \
2672 return isGFX11Plus(STI) ? result##_gfx11plus : result##_gfxpre11;
2673
2675 if (STI.getTargetTriple().getArch() == Triple::r600)
2676 return Reg;
2678}
2679
2680#undef CASE_CI_VI
2681#undef CASE_VI_GFX9PLUS
2682#undef CASE_GFXPRE11_GFX11PLUS
2683#undef CASE_GFXPRE11_GFX11PLUS_TO
2684
2685#define CASE_CI_VI(node) \
2686 case node##_ci: \
2687 case node##_vi: \
2688 return node;
2689#define CASE_VI_GFX9PLUS(node) \
2690 case node##_vi: \
2691 case node##_gfx9plus: \
2692 return node;
2693#define CASE_GFXPRE11_GFX11PLUS(node) \
2694 case node##_gfx11plus: \
2695 case node##_gfxpre11: \
2696 return node;
2697#define CASE_GFXPRE11_GFX11PLUS_TO(node, result)
2698
2700
2702 switch (Reg.id()) {
2703 case AMDGPU::SRC_SHARED_BASE_LO:
2704 case AMDGPU::SRC_SHARED_BASE:
2705 case AMDGPU::SRC_SHARED_LIMIT_LO:
2706 case AMDGPU::SRC_SHARED_LIMIT:
2707 case AMDGPU::SRC_PRIVATE_BASE_LO:
2708 case AMDGPU::SRC_PRIVATE_BASE:
2709 case AMDGPU::SRC_PRIVATE_LIMIT_LO:
2710 case AMDGPU::SRC_PRIVATE_LIMIT:
2711 case AMDGPU::SRC_FLAT_SCRATCH_BASE_LO:
2712 case AMDGPU::SRC_FLAT_SCRATCH_BASE_HI:
2713 case AMDGPU::SRC_POPS_EXITING_WAVE_ID:
2714 return true;
2715 case AMDGPU::SRC_VCCZ:
2716 case AMDGPU::SRC_EXECZ:
2717 case AMDGPU::SRC_SCC:
2718 return true;
2719 case AMDGPU::SGPR_NULL:
2720 return true;
2721 default:
2722 return false;
2723 }
2724}
2725
2726#undef CASE_CI_VI
2727#undef CASE_VI_GFX9PLUS
2728#undef CASE_GFXPRE11_GFX11PLUS
2729#undef CASE_GFXPRE11_GFX11PLUS_TO
2730#undef MAP_REG2REG
2731
2732bool isKImmOperand(const MCInstrDesc &Desc, unsigned OpNo) {
2733 assert(OpNo < Desc.NumOperands);
2734 unsigned OpType = Desc.operands()[OpNo].OperandType;
2735 return OpType >= AMDGPU::OPERAND_KIMM_FIRST &&
2736 OpType <= AMDGPU::OPERAND_KIMM_LAST;
2737}
2738
2739bool isSISrcFPOperand(const MCInstrDesc &Desc, unsigned OpNo) {
2740 assert(OpNo < Desc.NumOperands);
2741 unsigned OpType = Desc.operands()[OpNo].OperandType;
2742 switch (OpType) {
2758 return true;
2759 default:
2760 return false;
2761 }
2762}
2763
2764// Avoid using MCRegisterClass::getSize, since that function will go away
2765// (move from MC* level to Target* level). Return size in bits.
2766unsigned getRegBitWidth(unsigned RCID) {
2767 switch (RCID) {
2768 case AMDGPU::VGPR_16RegClassID:
2769 case AMDGPU::VGPR_16_Lo128RegClassID:
2770 case AMDGPU::SGPR_LO16RegClassID:
2771 case AMDGPU::AGPR_LO16RegClassID:
2772 return 16;
2773 case AMDGPU::SGPR_32RegClassID:
2774 case AMDGPU::VGPR_32RegClassID:
2775 case AMDGPU::VGPR_32_Lo256RegClassID:
2776 case AMDGPU::VRegOrLds_32RegClassID:
2777 case AMDGPU::AGPR_32RegClassID:
2778 case AMDGPU::VS_32RegClassID:
2779 case AMDGPU::AV_32RegClassID:
2780 case AMDGPU::SReg_32RegClassID:
2781 case AMDGPU::SReg_32_XM0RegClassID:
2782 case AMDGPU::SRegOrLds_32RegClassID:
2783 return 32;
2784 case AMDGPU::SGPR_64RegClassID:
2785 case AMDGPU::VS_64RegClassID:
2786 case AMDGPU::SReg_64RegClassID:
2787 case AMDGPU::VReg_64RegClassID:
2788 case AMDGPU::AReg_64RegClassID:
2789 case AMDGPU::SReg_64_XEXECRegClassID:
2790 case AMDGPU::VReg_64_Align2RegClassID:
2791 case AMDGPU::AReg_64_Align2RegClassID:
2792 case AMDGPU::AV_64RegClassID:
2793 case AMDGPU::AV_64_Align2RegClassID:
2794 case AMDGPU::VReg_64_Lo256_Align2RegClassID:
2795 case AMDGPU::VS_64_Lo256RegClassID:
2796 return 64;
2797 case AMDGPU::SGPR_96RegClassID:
2798 case AMDGPU::SReg_96RegClassID:
2799 case AMDGPU::VReg_96RegClassID:
2800 case AMDGPU::AReg_96RegClassID:
2801 case AMDGPU::VReg_96_Align2RegClassID:
2802 case AMDGPU::AReg_96_Align2RegClassID:
2803 case AMDGPU::AV_96RegClassID:
2804 case AMDGPU::AV_96_Align2RegClassID:
2805 case AMDGPU::VReg_96_Lo256_Align2RegClassID:
2806 return 96;
2807 case AMDGPU::SGPR_128RegClassID:
2808 case AMDGPU::SReg_128RegClassID:
2809 case AMDGPU::VReg_128RegClassID:
2810 case AMDGPU::AReg_128RegClassID:
2811 case AMDGPU::VReg_128_Align2RegClassID:
2812 case AMDGPU::AReg_128_Align2RegClassID:
2813 case AMDGPU::AV_128RegClassID:
2814 case AMDGPU::AV_128_Align2RegClassID:
2815 case AMDGPU::SReg_128_XNULLRegClassID:
2816 case AMDGPU::VReg_128_Lo256_Align2RegClassID:
2817 return 128;
2818 case AMDGPU::SGPR_160RegClassID:
2819 case AMDGPU::SReg_160RegClassID:
2820 case AMDGPU::VReg_160RegClassID:
2821 case AMDGPU::AReg_160RegClassID:
2822 case AMDGPU::VReg_160_Align2RegClassID:
2823 case AMDGPU::AReg_160_Align2RegClassID:
2824 case AMDGPU::AV_160RegClassID:
2825 case AMDGPU::AV_160_Align2RegClassID:
2826 case AMDGPU::VReg_160_Lo256_Align2RegClassID:
2827 return 160;
2828 case AMDGPU::SGPR_192RegClassID:
2829 case AMDGPU::SReg_192RegClassID:
2830 case AMDGPU::VReg_192RegClassID:
2831 case AMDGPU::AReg_192RegClassID:
2832 case AMDGPU::VReg_192_Align2RegClassID:
2833 case AMDGPU::AReg_192_Align2RegClassID:
2834 case AMDGPU::AV_192RegClassID:
2835 case AMDGPU::AV_192_Align2RegClassID:
2836 case AMDGPU::VReg_192_Lo256_Align2RegClassID:
2837 return 192;
2838 case AMDGPU::SGPR_224RegClassID:
2839 case AMDGPU::SReg_224RegClassID:
2840 case AMDGPU::VReg_224RegClassID:
2841 case AMDGPU::AReg_224RegClassID:
2842 case AMDGPU::VReg_224_Align2RegClassID:
2843 case AMDGPU::AReg_224_Align2RegClassID:
2844 case AMDGPU::AV_224RegClassID:
2845 case AMDGPU::AV_224_Align2RegClassID:
2846 case AMDGPU::VReg_224_Lo256_Align2RegClassID:
2847 return 224;
2848 case AMDGPU::SGPR_256RegClassID:
2849 case AMDGPU::SReg_256RegClassID:
2850 case AMDGPU::VReg_256RegClassID:
2851 case AMDGPU::AReg_256RegClassID:
2852 case AMDGPU::VReg_256_Align2RegClassID:
2853 case AMDGPU::AReg_256_Align2RegClassID:
2854 case AMDGPU::AV_256RegClassID:
2855 case AMDGPU::AV_256_Align2RegClassID:
2856 case AMDGPU::SReg_256_XNULLRegClassID:
2857 case AMDGPU::VReg_256_Lo256_Align2RegClassID:
2858 return 256;
2859 case AMDGPU::SGPR_288RegClassID:
2860 case AMDGPU::SReg_288RegClassID:
2861 case AMDGPU::VReg_288RegClassID:
2862 case AMDGPU::AReg_288RegClassID:
2863 case AMDGPU::VReg_288_Align2RegClassID:
2864 case AMDGPU::AReg_288_Align2RegClassID:
2865 case AMDGPU::AV_288RegClassID:
2866 case AMDGPU::AV_288_Align2RegClassID:
2867 case AMDGPU::VReg_288_Lo256_Align2RegClassID:
2868 return 288;
2869 case AMDGPU::SGPR_320RegClassID:
2870 case AMDGPU::SReg_320RegClassID:
2871 case AMDGPU::VReg_320RegClassID:
2872 case AMDGPU::AReg_320RegClassID:
2873 case AMDGPU::VReg_320_Align2RegClassID:
2874 case AMDGPU::AReg_320_Align2RegClassID:
2875 case AMDGPU::AV_320RegClassID:
2876 case AMDGPU::AV_320_Align2RegClassID:
2877 case AMDGPU::VReg_320_Lo256_Align2RegClassID:
2878 return 320;
2879 case AMDGPU::SGPR_352RegClassID:
2880 case AMDGPU::SReg_352RegClassID:
2881 case AMDGPU::VReg_352RegClassID:
2882 case AMDGPU::AReg_352RegClassID:
2883 case AMDGPU::VReg_352_Align2RegClassID:
2884 case AMDGPU::AReg_352_Align2RegClassID:
2885 case AMDGPU::AV_352RegClassID:
2886 case AMDGPU::AV_352_Align2RegClassID:
2887 case AMDGPU::VReg_352_Lo256_Align2RegClassID:
2888 return 352;
2889 case AMDGPU::SGPR_384RegClassID:
2890 case AMDGPU::SReg_384RegClassID:
2891 case AMDGPU::VReg_384RegClassID:
2892 case AMDGPU::AReg_384RegClassID:
2893 case AMDGPU::VReg_384_Align2RegClassID:
2894 case AMDGPU::AReg_384_Align2RegClassID:
2895 case AMDGPU::AV_384RegClassID:
2896 case AMDGPU::AV_384_Align2RegClassID:
2897 case AMDGPU::VReg_384_Lo256_Align2RegClassID:
2898 return 384;
2899 case AMDGPU::SGPR_512RegClassID:
2900 case AMDGPU::SReg_512RegClassID:
2901 case AMDGPU::VReg_512RegClassID:
2902 case AMDGPU::AReg_512RegClassID:
2903 case AMDGPU::VReg_512_Align2RegClassID:
2904 case AMDGPU::AReg_512_Align2RegClassID:
2905 case AMDGPU::AV_512RegClassID:
2906 case AMDGPU::AV_512_Align2RegClassID:
2907 case AMDGPU::VReg_512_Lo256_Align2RegClassID:
2908 return 512;
2909 case AMDGPU::SGPR_1024RegClassID:
2910 case AMDGPU::SReg_1024RegClassID:
2911 case AMDGPU::VReg_1024RegClassID:
2912 case AMDGPU::AReg_1024RegClassID:
2913 case AMDGPU::VReg_1024_Align2RegClassID:
2914 case AMDGPU::AReg_1024_Align2RegClassID:
2915 case AMDGPU::AV_1024RegClassID:
2916 case AMDGPU::AV_1024_Align2RegClassID:
2917 case AMDGPU::VReg_1024_Lo256_Align2RegClassID:
2918 return 1024;
2919 default:
2920 llvm_unreachable("Unexpected register class");
2921 }
2922}
2923
2924unsigned getRegBitWidth(const MCRegisterClass &RC) {
2925 return getRegBitWidth(RC.getID());
2926}
2927
2928bool isInlinableLiteral64(int64_t Literal, bool HasInv2Pi) {
2930 return true;
2931
2932 uint64_t Val = static_cast<uint64_t>(Literal);
2933 return (Val == llvm::bit_cast<uint64_t>(0.0)) ||
2934 (Val == llvm::bit_cast<uint64_t>(1.0)) ||
2935 (Val == llvm::bit_cast<uint64_t>(-1.0)) ||
2936 (Val == llvm::bit_cast<uint64_t>(0.5)) ||
2937 (Val == llvm::bit_cast<uint64_t>(-0.5)) ||
2938 (Val == llvm::bit_cast<uint64_t>(2.0)) ||
2939 (Val == llvm::bit_cast<uint64_t>(-2.0)) ||
2940 (Val == llvm::bit_cast<uint64_t>(4.0)) ||
2941 (Val == llvm::bit_cast<uint64_t>(-4.0)) ||
2942 (Val == 0x3fc45f306dc9c882 && HasInv2Pi);
2943}
2944
2945bool isInlinableLiteral32(int32_t Literal, bool HasInv2Pi) {
2947 return true;
2948
2949 // The actual type of the operand does not seem to matter as long
2950 // as the bits match one of the inline immediate values. For example:
2951 //
2952 // -nan has the hexadecimal encoding of 0xfffffffe which is -2 in decimal,
2953 // so it is a legal inline immediate.
2954 //
2955 // 1065353216 has the hexadecimal encoding 0x3f800000 which is 1.0f in
2956 // floating-point, so it is a legal inline immediate.
2957
2958 uint32_t Val = static_cast<uint32_t>(Literal);
2959 return (Val == llvm::bit_cast<uint32_t>(0.0f)) ||
2960 (Val == llvm::bit_cast<uint32_t>(1.0f)) ||
2961 (Val == llvm::bit_cast<uint32_t>(-1.0f)) ||
2962 (Val == llvm::bit_cast<uint32_t>(0.5f)) ||
2963 (Val == llvm::bit_cast<uint32_t>(-0.5f)) ||
2964 (Val == llvm::bit_cast<uint32_t>(2.0f)) ||
2965 (Val == llvm::bit_cast<uint32_t>(-2.0f)) ||
2966 (Val == llvm::bit_cast<uint32_t>(4.0f)) ||
2967 (Val == llvm::bit_cast<uint32_t>(-4.0f)) ||
2968 (Val == 0x3e22f983 && HasInv2Pi);
2969}
2970
2971bool isInlinableLiteralBF16(int16_t Literal, bool HasInv2Pi, bool *IsBF16) {
2972 if (!HasInv2Pi)
2973 return false;
2974 if (IsBF16)
2975 *IsBF16 = false;
2977 return true;
2978 uint16_t Val = static_cast<uint16_t>(Literal);
2979 if (IsBF16)
2980 *IsBF16 = true;
2981 return Val == 0x3F00 || // 0.5
2982 Val == 0xBF00 || // -0.5
2983 Val == 0x3F80 || // 1.0
2984 Val == 0xBF80 || // -1.0
2985 Val == 0x4000 || // 2.0
2986 Val == 0xC000 || // -2.0
2987 Val == 0x4080 || // 4.0
2988 Val == 0xC080 || // -4.0
2989 Val == 0x3E22; // 1.0 / (2.0 * pi)
2990}
2991
2992bool isInlinableLiteralI16(int32_t Literal, bool HasInv2Pi) {
2993 return isInlinableLiteral32(Literal, HasInv2Pi);
2994}
2995
2996bool isInlinableLiteralFP16(int16_t Literal, bool HasInv2Pi) {
2997 if (!HasInv2Pi)
2998 return false;
3000 return true;
3001 uint16_t Val = static_cast<uint16_t>(Literal);
3002 return Val == 0x3C00 || // 1.0
3003 Val == 0xBC00 || // -1.0
3004 Val == 0x3800 || // 0.5
3005 Val == 0xB800 || // -0.5
3006 Val == 0x4000 || // 2.0
3007 Val == 0xC000 || // -2.0
3008 Val == 0x4400 || // 4.0
3009 Val == 0xC400 || // -4.0
3010 Val == 0x3118; // 1/2pi
3011}
3012
3013std::optional<unsigned> getInlineEncodingV216(bool IsFloat, uint32_t Literal) {
3014 // Unfortunately, the Instruction Set Architecture Reference Guide is
3015 // misleading about how the inline operands work for (packed) 16-bit
3016 // instructions. In a nutshell, the actual HW behavior is:
3017 //
3018 // - integer encodings (-16 .. 64) are always produced as sign-extended
3019 // 32-bit values
3020 // - float encodings are produced as:
3021 // - for F16 instructions: corresponding half-precision float values in
3022 // the LSBs, 0 in the MSBs
3023 // - for UI16 instructions: corresponding single-precision float value
3024 int32_t Signed = static_cast<int32_t>(Literal);
3025 if (Signed >= 0 && Signed <= 64)
3026 return 128 + Signed;
3027
3028 if (Signed >= -16 && Signed <= -1)
3029 return 192 + std::abs(Signed);
3030
3031 if (IsFloat) {
3032 // clang-format off
3033 switch (Literal) {
3034 case 0x3800: return 240; // 0.5
3035 case 0xB800: return 241; // -0.5
3036 case 0x3C00: return 242; // 1.0
3037 case 0xBC00: return 243; // -1.0
3038 case 0x4000: return 244; // 2.0
3039 case 0xC000: return 245; // -2.0
3040 case 0x4400: return 246; // 4.0
3041 case 0xC400: return 247; // -4.0
3042 case 0x3118: return 248; // 1.0 / (2.0 * pi)
3043 default: break;
3044 }
3045 // clang-format on
3046 } else {
3047 // clang-format off
3048 switch (Literal) {
3049 case 0x3F000000: return 240; // 0.5
3050 case 0xBF000000: return 241; // -0.5
3051 case 0x3F800000: return 242; // 1.0
3052 case 0xBF800000: return 243; // -1.0
3053 case 0x40000000: return 244; // 2.0
3054 case 0xC0000000: return 245; // -2.0
3055 case 0x40800000: return 246; // 4.0
3056 case 0xC0800000: return 247; // -4.0
3057 case 0x3E22F983: return 248; // 1.0 / (2.0 * pi)
3058 default: break;
3059 }
3060 // clang-format on
3061 }
3062
3063 return {};
3064}
3065
3066// Encoding of the literal as an inline constant for a V_PK_*_IU16 instruction
3067// or nullopt.
3068std::optional<unsigned> getInlineEncodingV2I16(uint32_t Literal) {
3069 return getInlineEncodingV216(false, Literal);
3070}
3071
3072// Encoding of the literal as an inline constant for a V_PK_*_BF16 instruction
3073// or nullopt.
3074std::optional<unsigned> getInlineEncodingV2BF16(uint32_t Literal) {
3075 int32_t Signed = static_cast<int32_t>(Literal);
3076 if (Signed >= 0 && Signed <= 64)
3077 return 128 + Signed;
3078
3079 if (Signed >= -16 && Signed <= -1)
3080 return 192 + std::abs(Signed);
3081
3082 // clang-format off
3083 switch (Literal) {
3084 case 0x3F00: return 240; // 0.5
3085 case 0xBF00: return 241; // -0.5
3086 case 0x3F80: return 242; // 1.0
3087 case 0xBF80: return 243; // -1.0
3088 case 0x4000: return 244; // 2.0
3089 case 0xC000: return 245; // -2.0
3090 case 0x4080: return 246; // 4.0
3091 case 0xC080: return 247; // -4.0
3092 case 0x3E22: return 248; // 1.0 / (2.0 * pi)
3093 default: break;
3094 }
3095 // clang-format on
3096
3097 return std::nullopt;
3098}
3099
3100// Encoding of the literal as an inline constant for a V_PK_*_F16 instruction
3101// or nullopt.
3102std::optional<unsigned> getInlineEncodingV2F16(uint32_t Literal) {
3103 return getInlineEncodingV216(true, Literal);
3104}
3105
3106// Encoding of the literal as an inline constant for V_PK_FMAC_F16 instruction
3107// or nullopt. This accounts for different inline constant behavior:
3108// - Pre-GFX11: fp16 inline constants have the value in low 16 bits, 0 in high
3109// - GFX11+: fp16 inline constants are duplicated into both halves
3111 bool IsGFX11Plus) {
3112 // Pre-GFX11 behavior: f16 in low bits, 0 in high bits
3113 if (!IsGFX11Plus)
3114 return getInlineEncodingV216(/*IsFloat=*/true, Literal);
3115
3116 // GFX11+ behavior: f16 duplicated in both halves
3117 // First, check for sign-extended integer inline constants (-16 to 64)
3118 // These work the same across all generations
3119 int32_t Signed = static_cast<int32_t>(Literal);
3120 if (Signed >= 0 && Signed <= 64)
3121 return 128 + Signed;
3122
3123 if (Signed >= -16 && Signed <= -1)
3124 return 192 + std::abs(Signed);
3125
3126 // For float inline constants on GFX11+, both halves must be equal
3127 uint16_t Lo = static_cast<uint16_t>(Literal);
3128 uint16_t Hi = static_cast<uint16_t>(Literal >> 16);
3129 if (Lo != Hi)
3130 return std::nullopt;
3131 return getInlineEncodingV216(/*IsFloat=*/true, Lo);
3132}
3133
3134// Whether the given literal can be inlined for a V_PK_* instruction.
3136 switch (OpType) {
3139 return getInlineEncodingV216(false, Literal).has_value();
3142 return getInlineEncodingV216(true, Literal).has_value();
3144 llvm_unreachable("OPERAND_REG_IMM_V2FP16_SPLAT is not supported");
3149 return false;
3150 default:
3151 llvm_unreachable("bad packed operand type");
3152 }
3153}
3154
3155// Whether the given literal can be inlined for a V_PK_*_IU16 instruction.
3159
3160// Whether the given literal can be inlined for a V_PK_*_BF16 instruction.
3164
3165// Whether the given literal can be inlined for a V_PK_*_F16 instruction.
3169
3170// Whether the given literal can be inlined for V_PK_FMAC_F16 instruction.
3172 return getPKFMACF16InlineEncoding(Literal, IsGFX11Plus).has_value();
3173}
3174
3175bool isValid32BitLiteral(uint64_t Val, bool IsFP64) {
3176 if (IsFP64)
3177 return !Lo_32(Val);
3178
3179 return isUInt<32>(Val) || isInt<32>(Val);
3180}
3181
3182int64_t encode32BitLiteral(int64_t Imm, OperandType Type, bool IsLit) {
3183 switch (Type) {
3184 default:
3185 break;
3191 return Imm & 0xffff;
3205 return Lo_32(Imm);
3208 return IsLit ? Imm : Hi_32(Imm);
3209 }
3210 return Imm;
3211}
3212
3214 const Function *F = A->getParent();
3215
3216 // Arguments to compute shaders are never a source of divergence.
3217 CallingConv::ID CC = F->getCallingConv();
3218 switch (CC) {
3221 return true;
3232 // For non-compute shaders, SGPR inputs are marked with either inreg or
3233 // byval. Everything else is in VGPRs.
3234 return A->hasAttribute(Attribute::InReg) ||
3235 A->hasAttribute(Attribute::ByVal);
3236 default:
3237 // TODO: treat i1 as divergent?
3238 return A->hasAttribute(Attribute::InReg);
3239 }
3240}
3241
3242bool isArgPassedInSGPR(const CallBase *CB, unsigned ArgNo) {
3243 // Arguments to compute shaders are never a source of divergence.
3245 switch (CC) {
3248 return true;
3259 // For non-compute shaders, SGPR inputs are marked with either inreg or
3260 // byval. Everything else is in VGPRs.
3261 return CB->paramHasAttr(ArgNo, Attribute::InReg) ||
3262 CB->isByValArgument(ArgNo);
3263 default:
3264 return CB->paramHasAttr(ArgNo, Attribute::InReg);
3265 }
3266}
3267
3268static bool hasSMEMByteOffset(const MCSubtargetInfo &ST) {
3269 return isGCN3Encoding(ST) || isGFX10Plus(ST);
3270}
3271
3273 int64_t EncodedOffset) {
3274 if (isGFX12Plus(ST))
3275 return isUInt<23>(EncodedOffset);
3276
3277 return hasSMEMByteOffset(ST) ? isUInt<20>(EncodedOffset)
3278 : isUInt<8>(EncodedOffset);
3279}
3280
3282 int64_t EncodedOffset, bool IsBuffer) {
3283 if (isGFX12Plus(ST)) {
3284 if (IsBuffer && EncodedOffset < 0)
3285 return false;
3286 return isInt<24>(EncodedOffset);
3287 }
3288
3289 return !IsBuffer && hasSMRDSignedImmOffset(ST) && isInt<21>(EncodedOffset);
3290}
3291
3292static bool isDwordAligned(uint64_t ByteOffset) {
3293 return (ByteOffset & 3) == 0;
3294}
3295
3297 uint64_t ByteOffset) {
3298 if (hasSMEMByteOffset(ST))
3299 return ByteOffset;
3300
3301 assert(isDwordAligned(ByteOffset));
3302 return ByteOffset >> 2;
3303}
3304
3305std::optional<int64_t> getSMRDEncodedOffset(const MCSubtargetInfo &ST,
3306 int64_t ByteOffset, bool IsBuffer,
3307 bool HasSOffset) {
3308 // For unbuffered smem loads, it is illegal for the Immediate Offset to be
3309 // negative if the resulting (Offset + (M0 or SOffset or zero) is negative.
3310 // Handle case where SOffset is not present.
3311 if (!IsBuffer && !HasSOffset && ByteOffset < 0 && hasSMRDSignedImmOffset(ST))
3312 return std::nullopt;
3313
3314 if (isGFX12Plus(ST)) // 24 bit signed offsets
3315 return isInt<24>(ByteOffset) ? std::optional<int64_t>(ByteOffset)
3316 : std::nullopt;
3317
3318 // The signed version is always a byte offset.
3319 if (!IsBuffer && hasSMRDSignedImmOffset(ST)) {
3321 return isInt<20>(ByteOffset) ? std::optional<int64_t>(ByteOffset)
3322 : std::nullopt;
3323 }
3324
3325 if (!isDwordAligned(ByteOffset) && !hasSMEMByteOffset(ST))
3326 return std::nullopt;
3327
3328 int64_t EncodedOffset = convertSMRDOffsetUnits(ST, ByteOffset);
3329 return isLegalSMRDEncodedUnsignedOffset(ST, EncodedOffset)
3330 ? std::optional<int64_t>(EncodedOffset)
3331 : std::nullopt;
3332}
3333
3334std::optional<int64_t> getSMRDEncodedLiteralOffset32(const MCSubtargetInfo &ST,
3335 int64_t ByteOffset) {
3336 if (!isCI(ST) || !isDwordAligned(ByteOffset))
3337 return std::nullopt;
3338
3339 int64_t EncodedOffset = convertSMRDOffsetUnits(ST, ByteOffset);
3340 return isUInt<32>(EncodedOffset) ? std::optional<int64_t>(EncodedOffset)
3341 : std::nullopt;
3342}
3343
3345 if (ST.getFeatureBits().test(FeatureFlatOffsetBits12))
3346 return 12;
3347 if (ST.getFeatureBits().test(FeatureFlatOffsetBits24))
3348 return 24;
3349 return 13;
3350}
3351
3352namespace {
3353
3354struct SourceOfDivergence {
3355 unsigned Intr;
3356};
3357const SourceOfDivergence *lookupSourceOfDivergence(unsigned Intr);
3358
3359struct AlwaysUniform {
3360 unsigned Intr;
3361};
3362const AlwaysUniform *lookupAlwaysUniform(unsigned Intr);
3363
3364struct LDSDMAIntrinsic {
3365 unsigned Intr;
3366};
3367const LDSDMAIntrinsic *lookupLDSDMAIntrinsic(unsigned Intr);
3368
3369struct ExponentOnlyScaleIntrinsic {
3370 unsigned Intr;
3371 uint8_t ScaleArg;
3372};
3373const ExponentOnlyScaleIntrinsic *
3374lookupExponentOnlyScaleIntrinsic(unsigned Intr);
3375
3376#define GET_SourcesOfDivergence_IMPL
3377#define GET_UniformIntrinsics_IMPL
3378#define GET_LDSDMAIntrinsicTable_IMPL
3379#define GET_ExponentOnlyScaleIntrinsics_IMPL
3380#define GET_Gfx9BufferFormat_IMPL
3381#define GET_Gfx10BufferFormat_IMPL
3382#define GET_Gfx11PlusBufferFormat_IMPL
3383
3384#include "AMDGPUGenSearchableTables.inc"
3385
3386} // end anonymous namespace
3387
3388bool isIntrinsicSourceOfDivergence(unsigned IntrID) {
3389 return lookupSourceOfDivergence(IntrID);
3390}
3391
3392bool isIntrinsicAlwaysUniform(unsigned IntrID) {
3393 return lookupAlwaysUniform(IntrID);
3394}
3395
3396bool isTriviallyUniform(const Value *V, const Instruction *CtxI) {
3397 if (isa<Constant>(V))
3398 return true;
3399 if (const auto *A = dyn_cast<Argument>(V))
3400 return isArgPassedInSGPR(A);
3401 if (const auto *II = dyn_cast<IntrinsicInst>(V)) {
3402 if (!isIntrinsicAlwaysUniform(II->getIntrinsicID()))
3403 return false;
3404 // If II and CtxI are in different blocks then there is a possibility of a
3405 // temporal divergence.
3406 return II->getParent() == CtxI->getParent();
3407 }
3408 return false;
3409}
3410
3411bool isTriviallyUniform(const Use &U) {
3412 return isTriviallyUniform(U.get(), cast<Instruction>(U.getUser()));
3413}
3414
3415bool isLDSDMAIntrinsic(unsigned IntrID) {
3416 return lookupLDSDMAIntrinsic(IntrID);
3417}
3418
3419std::optional<unsigned> getExponentOnlyScaleArgIdx(unsigned IntrID) {
3420 if (const ExponentOnlyScaleIntrinsic *Info =
3421 lookupExponentOnlyScaleIntrinsic(IntrID))
3422 return Info->ScaleArg;
3423 return std::nullopt;
3424}
3425
3427 uint8_t NumComponents,
3428 uint8_t NumFormat,
3429 const MCSubtargetInfo &STI) {
3430 return isGFX11Plus(STI) ? getGfx11PlusBufferFormatInfo(
3431 BitsPerComp, NumComponents, NumFormat)
3432 : isGFX10(STI)
3433 ? getGfx10BufferFormatInfo(BitsPerComp, NumComponents, NumFormat)
3434 : getGfx9BufferFormatInfo(BitsPerComp, NumComponents, NumFormat);
3435}
3436
3438 const MCSubtargetInfo &STI) {
3439 return isGFX11Plus(STI) ? getGfx11PlusBufferFormatInfo(Format)
3440 : isGFX10(STI) ? getGfx10BufferFormatInfo(Format)
3441 : getGfx9BufferFormatInfo(Format);
3442}
3443
3445 const MCRegisterInfo &MRI) {
3446 const unsigned VGPRClasses[] = {
3447 AMDGPU::VGPR_16RegClassID, AMDGPU::VGPR_32RegClassID,
3448 AMDGPU::VReg_64RegClassID, AMDGPU::VReg_96RegClassID,
3449 AMDGPU::VReg_128RegClassID, AMDGPU::VReg_160RegClassID,
3450 AMDGPU::VReg_192RegClassID, AMDGPU::VReg_224RegClassID,
3451 AMDGPU::VReg_256RegClassID, AMDGPU::VReg_288RegClassID,
3452 AMDGPU::VReg_320RegClassID, AMDGPU::VReg_352RegClassID,
3453 AMDGPU::VReg_384RegClassID, AMDGPU::VReg_512RegClassID,
3454 AMDGPU::VReg_1024RegClassID};
3455
3456 for (unsigned RCID : VGPRClasses) {
3457 const MCRegisterClass &RC = MRI.getRegClass(RCID);
3458 if (RC.contains(Reg))
3459 return &RC;
3460 }
3461
3462 return nullptr;
3463}
3464
3466 unsigned Enc = MRI.getEncodingValue(Reg);
3467 unsigned Idx = Enc & AMDGPU::HWEncoding::REG_IDX_MASK;
3468 return Idx >> 8;
3469}
3470
3472 const MCRegisterInfo &MRI) {
3473 unsigned Enc = MRI.getEncodingValue(Reg);
3474 unsigned Idx = Enc & AMDGPU::HWEncoding::REG_IDX_MASK;
3475 if (Idx >= 0x100)
3476 return MCRegister();
3477
3479 if (!RC)
3480 return MCRegister();
3481
3482 Idx |= MSBs << 8;
3483 if (RC->getID() == AMDGPU::VGPR_16RegClassID) {
3484 // This class has 2048 registers with interleaved lo16 and hi16.
3485 Idx *= 2;
3487 ++Idx;
3488 }
3489
3490 return RC->getRegister(Idx);
3491}
3492
3493static std::optional<unsigned>
3495 bool HasSetregVGPRMSBFixup) {
3496 constexpr unsigned VGPRMSBShift =
3498
3499 auto [HwRegId, Offset, Size] = Hwreg::HwregEncoding::decode(Simm16);
3500 if (HwRegId != Hwreg::ID_MODE ||
3501 (!HasSetregVGPRMSBFixup && (Offset + Size) < VGPRMSBShift))
3502 return {};
3503 // If there is SetregVGPRMSBFixup then Offset is ignored.
3504 if (!HasSetregVGPRMSBFixup)
3505 Imm <<= Offset;
3506 Imm = (Imm & Hwreg::VGPR_MSB_MASK) >> VGPRMSBShift;
3507 if (!HasSetregVGPRMSBFixup)
3509 return llvm::rotr<uint8_t>(static_cast<uint8_t>(Imm), /*R=*/2);
3510}
3511
3512std::optional<unsigned> convertSetRegImmToVgprMSBs(const MachineInstr &MI,
3513 bool HasSetregVGPRMSBFixup) {
3514 assert(MI.getOpcode() == AMDGPU::S_SETREG_IMM32_B32);
3515 return convertSetRegImmToVgprMSBs(MI.getOperand(0).getImm(),
3516 MI.getOperand(1).getImm(),
3517 HasSetregVGPRMSBFixup);
3518}
3519
3520std::optional<unsigned> convertSetRegImmToVgprMSBs(const MCInst &MI,
3521 bool HasSetregVGPRMSBFixup) {
3522 assert(MI.getOpcode() == AMDGPU::S_SETREG_IMM32_B32_gfx12);
3523 return convertSetRegImmToVgprMSBs(MI.getOperand(0).getImm(),
3524 MI.getOperand(1).getImm(),
3525 HasSetregVGPRMSBFixup);
3526}
3527
3528std::pair<const AMDGPU::OpName *, const AMDGPU::OpName *>
3530 static const AMDGPU::OpName VOPOps[4] = {
3531 AMDGPU::OpName::src0, AMDGPU::OpName::src1, AMDGPU::OpName::src2,
3532 AMDGPU::OpName::vdst};
3533 static const AMDGPU::OpName VDSOps[4] = {
3534 AMDGPU::OpName::addr, AMDGPU::OpName::data0, AMDGPU::OpName::data1,
3535 AMDGPU::OpName::vdst};
3536 static const AMDGPU::OpName FLATOps[4] = {
3537 AMDGPU::OpName::vaddr, AMDGPU::OpName::vdata,
3538 AMDGPU::OpName::NUM_OPERAND_NAMES, AMDGPU::OpName::vdst};
3539 static const AMDGPU::OpName BUFOps[4] = {
3540 AMDGPU::OpName::vaddr, AMDGPU::OpName::NUM_OPERAND_NAMES,
3541 AMDGPU::OpName::NUM_OPERAND_NAMES, AMDGPU::OpName::vdata};
3542 static const AMDGPU::OpName VIMGOps[4] = {
3543 AMDGPU::OpName::vaddr0, AMDGPU::OpName::vaddr1, AMDGPU::OpName::vaddr2,
3544 AMDGPU::OpName::vdata};
3545
3546 // For VOPD instructions MSB of a corresponding Y component operand VGPR
3547 // address is supposed to match X operand, otherwise VOPD shall not be
3548 // combined.
3549 static const AMDGPU::OpName VOPDOpsX[4] = {
3550 AMDGPU::OpName::src0X, AMDGPU::OpName::vsrc1X, AMDGPU::OpName::vsrc2X,
3551 AMDGPU::OpName::vdstX};
3552 static const AMDGPU::OpName VOPDOpsY[4] = {
3553 AMDGPU::OpName::src0Y, AMDGPU::OpName::vsrc1Y, AMDGPU::OpName::vsrc2Y,
3554 AMDGPU::OpName::vdstY};
3555
3556 // VOP2 MADMK instructions use src0, imm, src1 scheme.
3557 static const AMDGPU::OpName VOP2MADMKOps[4] = {
3558 AMDGPU::OpName::src0, AMDGPU::OpName::NUM_OPERAND_NAMES,
3559 AMDGPU::OpName::src1, AMDGPU::OpName::vdst};
3560 static const AMDGPU::OpName VOPDFMAMKOpsX[4] = {
3561 AMDGPU::OpName::src0X, AMDGPU::OpName::NUM_OPERAND_NAMES,
3562 AMDGPU::OpName::vsrc1X, AMDGPU::OpName::vdstX};
3563 static const AMDGPU::OpName VOPDFMAMKOpsY[4] = {
3564 AMDGPU::OpName::src0Y, AMDGPU::OpName::NUM_OPERAND_NAMES,
3565 AMDGPU::OpName::vsrc1Y, AMDGPU::OpName::vdstY};
3566
3570 switch (Desc.getOpcode()) {
3571 // LD_SCALE operands ignore MSB.
3572 case AMDGPU::V_WMMA_LD_SCALE_PAIRED_B32:
3573 case AMDGPU::V_WMMA_LD_SCALE_PAIRED_B32_gfx1250:
3574 case AMDGPU::V_WMMA_LD_SCALE16_PAIRED_B64:
3575 case AMDGPU::V_WMMA_LD_SCALE16_PAIRED_B64_gfx1250:
3576 return {};
3577 case AMDGPU::V_FMAMK_F16:
3578 case AMDGPU::V_FMAMK_F16_t16:
3579 case AMDGPU::V_FMAMK_F16_t16_gfx12:
3580 case AMDGPU::V_FMAMK_F16_fake16:
3581 case AMDGPU::V_FMAMK_F16_fake16_gfx12:
3582 case AMDGPU::V_FMAMK_F32:
3583 case AMDGPU::V_FMAMK_F32_gfx12:
3584 case AMDGPU::V_FMAMK_F64:
3585 case AMDGPU::V_FMAMK_F64_gfx1250:
3586 return {VOP2MADMKOps, nullptr};
3587 default:
3588 break;
3589 }
3590 return {VOPOps, nullptr};
3591 }
3592
3594 return {VDSOps, nullptr};
3595
3597 return {FLATOps, nullptr};
3598
3600 return {BUFOps, nullptr};
3601
3603 return {VIMGOps, nullptr};
3604
3605 if (AMDGPU::isVOPD(Desc.getOpcode())) {
3606 auto [OpX, OpY] = getVOPDComponents(Desc.getOpcode());
3607 return {(OpX == AMDGPU::V_FMAMK_F32) ? VOPDFMAMKOpsX : VOPDOpsX,
3608 (OpY == AMDGPU::V_FMAMK_F32) ? VOPDFMAMKOpsY : VOPDOpsY};
3609 }
3610
3612
3614 llvm_unreachable("Sample and export VGPR lowering is not implemented and"
3615 " these instructions are not expected on gfx1250");
3616
3617 return {};
3618}
3619
3620bool supportsScaleOffset(const MCInstrInfo &MII, unsigned Opcode) {
3621 const MCInstrDesc &Desc = MII.get(Opcode);
3623 return Desc.mayLoad() && !Desc.mayStore() && !getSMEMIsBuffer(Opcode);
3625 return false;
3626
3627 // Only SV and SVS modes are supported.
3628 if (SIInstrFlags::isFlatScratch(MII, Opcode))
3629 return hasNamedOperand(Opcode, OpName::vaddr);
3630
3631 // Only GVS mode is supported.
3632 return hasNamedOperand(Opcode, OpName::vaddr) &&
3633 hasNamedOperand(Opcode, OpName::saddr);
3634
3635 return false;
3636}
3637
3638static bool hasAny64BitVGPROperands(const MCInstrDesc &OpDesc,
3639 const MCInstrInfo &MII,
3640 const MCSubtargetInfo &ST) {
3641 for (auto OpName : {OpName::vdst, OpName::src0, OpName::src1, OpName::src2}) {
3642 int Idx = getNamedOperandIdx(OpDesc.getOpcode(), OpName);
3643 if (Idx == -1)
3644 continue;
3645
3646 const MCOperandInfo &OpInfo = OpDesc.operands()[Idx];
3647 int16_t RegClass = MII.getOpRegClassID(
3648 OpInfo, ST.getHwMode(MCSubtargetInfo::HwMode_RegInfo));
3649 if (RegClass == AMDGPU::VReg_64RegClassID ||
3650 RegClass == AMDGPU::VReg_64_Align2RegClassID)
3651 return true;
3652 }
3653
3654 return false;
3655}
3656
3657bool isDPALU_DPP32BitOpc(unsigned Opc) {
3658 switch (Opc) {
3659 case AMDGPU::V_MUL_LO_U32_e64:
3660 case AMDGPU::V_MUL_LO_U32_e64_dpp:
3661 case AMDGPU::V_MUL_LO_U32_e64_dpp_gfx1250:
3662 case AMDGPU::V_MUL_HI_U32_e64:
3663 case AMDGPU::V_MUL_HI_U32_e64_dpp:
3664 case AMDGPU::V_MUL_HI_U32_e64_dpp_gfx1250:
3665 case AMDGPU::V_MUL_HI_I32_e64:
3666 case AMDGPU::V_MUL_HI_I32_e64_dpp:
3667 case AMDGPU::V_MUL_HI_I32_e64_dpp_gfx1250:
3668 case AMDGPU::V_MAD_U32_e64:
3669 case AMDGPU::V_MAD_U32_e64_dpp:
3670 case AMDGPU::V_MAD_U32_e64_dpp_gfx1250:
3671 return true;
3672 default:
3673 return false;
3674 }
3675}
3676
3677bool isDPALU_DPP(const MCInstrDesc &OpDesc, const MCInstrInfo &MII,
3678 const MCSubtargetInfo &ST) {
3679 if (isDPALU_DPP32BitOpc(OpDesc.getOpcode()))
3680 return true;
3681
3682 return hasAny64BitVGPROperands(OpDesc, MII, ST);
3683}
3684
3686 switch (Opc) {
3687 case AMDGPU::V_PK_ADD_F32_gfx1250:
3688 case AMDGPU::V_PK_ADD_F32_gfx1250_gfx12:
3689 case AMDGPU::V_PK_MUL_F32_gfx1250:
3690 case AMDGPU::V_PK_MUL_F32_gfx1250_gfx12:
3691 case AMDGPU::V_PK_FMA_F32_gfx1250:
3692 case AMDGPU::V_PK_FMA_F32_gfx1250_gfx12:
3693 return true;
3694 default:
3695 return false;
3696 }
3697}
3698
3699// NOTE: This function is currently only used before pseudo-expansion.
3701 switch (Opc) {
3702 case AMDGPU::V_PK_ADD_F64:
3703 case AMDGPU::V_PK_MUL_F64:
3704 case AMDGPU::V_PK_FMA_F64:
3705 case AMDGPU::V_PK_MAX_NUM_F64:
3706 case AMDGPU::V_PK_MIN_NUM_F64:
3707 case AMDGPU::V_PK_ADD_NC_U64:
3708 case AMDGPU::V_PK_SUB_NC_U64:
3709 case AMDGPU::V_PK_LSHL_ADD_U64:
3710 return true;
3711 default:
3712 return false;
3713 }
3714}
3715
3719
3720const std::array<unsigned, 3> &ClusterDimsAttr::getDims() const {
3721 assert(isFixedDims() && "expect kind to be FixedDims");
3722 return Dims;
3723}
3724
3725std::string ClusterDimsAttr::to_string() const {
3726 SmallString<10> Buffer;
3727 raw_svector_ostream OS(Buffer);
3728
3729 switch (getKind()) {
3730 case Kind::Unknown:
3731 return "";
3732 case Kind::NoCluster: {
3733 OS << EncoNoCluster << ',' << EncoNoCluster << ',' << EncoNoCluster;
3734 return Buffer.c_str();
3735 }
3736 case Kind::VariableDims: {
3737 OS << EncoVariableDims << ',' << EncoVariableDims << ','
3738 << EncoVariableDims;
3739 return Buffer.c_str();
3740 }
3741 case Kind::FixedDims: {
3742 OS << Dims[0] << ',' << Dims[1] << ',' << Dims[2];
3743 return Buffer.c_str();
3744 }
3745 }
3746 llvm_unreachable("Unknown ClusterDimsAttr kind");
3747}
3748
3750 std::optional<SmallVector<unsigned>> Attr =
3751 getIntegerVecAttribute(F, "amdgpu-cluster-dims", /*Size=*/3);
3753
3754 if (!Attr.has_value())
3755 AttrKind = Kind::Unknown;
3756 else if (all_of(*Attr, equal_to(EncoNoCluster)))
3757 AttrKind = Kind::NoCluster;
3758 else if (all_of(*Attr, equal_to(EncoVariableDims)))
3759 AttrKind = Kind::VariableDims;
3760
3761 ClusterDimsAttr A(AttrKind);
3762 if (AttrKind == Kind::FixedDims)
3763 A.Dims = {(*Attr)[0], (*Attr)[1], (*Attr)[2]};
3764
3765 return A;
3766}
3767
3768std::optional<APFloat> evaluateRcp(const APFloat &Val) {
3769 const fltSemantics &Sem = Val.getSemantics();
3770
3771 // v_rcp_f16/bf16 are correctly rounded.
3772 if (&Sem == &APFloat::IEEEhalf() || &Sem == &APFloat::BFloat())
3773 return APFloat::getOne(Sem) / Val;
3774
3775 // v_rcp_f32/f64 always flush a denormal input to zero (preserving sign)
3776 // before reciprocating.
3777 APFloat Arg = Val;
3778 if (Arg.isDenormal())
3779 Arg = APFloat::getZero(Sem, Arg.isNegative());
3780
3781 APFloat Result = APFloat::getOne(Sem) / Arg;
3782
3783 // v_rcp_f32/f64 always flush a denormal result to zero (preserving sign).
3784 if (Result.isDenormal())
3785 Result = APFloat::getZero(Sem, Result.isNegative());
3786
3787 // v_rcp_f32/f64 only approximate the reciprocal, except for these special
3788 // cases where the result is exact.
3789 if (!Result.isZero() && !Result.isInfinity() && !Result.isNaN() &&
3790 !Result.isOne() && !Result.isMinusOne())
3791 return std::nullopt;
3792
3793 return Result;
3794}
3795
3796} // namespace AMDGPU
3797
3799 switch (S) {
3800 case (AMDGPU::TargetIDSetting::Unsupported):
3801 OS << "Unsupported";
3802 break;
3803 case (AMDGPU::TargetIDSetting::Any):
3804 OS << "Any";
3805 break;
3806 case (AMDGPU::TargetIDSetting::Off):
3807 OS << "Off";
3808 break;
3809 case (AMDGPU::TargetIDSetting::On):
3810 OS << "On";
3811 break;
3812 }
3813 return OS;
3814}
3815
3816} // namespace llvm
const AArch64Subtarget & Subtarget
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
static llvm::cl::opt< unsigned > DefaultAMDHSACodeObjectVersion("amdhsa-code-object-version", llvm::cl::Hidden, llvm::cl::init(llvm::AMDGPU::AMDHSA_COV6), llvm::cl::desc("Set default AMDHSA Code Object Version (module flag " "or asm directive still take priority if present)"))
#define MAP_REG2REG
uint64_t uint64_t
Provides AMDGPU specific target descriptions.
MC layer struct for AMDGPUMCKernelCodeT, provides MCExpr functionality where required.
@ AMD_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32
This file contains the simple types necessary to represent the attributes associated with functions a...
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
This file contains the declarations for the subclasses of Constant, which represent the different fla...
IRTranslator LLVM IR MI
#define RegName(no)
#define F(x, y, z)
Definition MD5.cpp:54
Register Reg
Register const TargetRegisterInfo * TRI
This file contains the declarations for metadata subclasses.
#define T
uint64_t High
uint64_t IntrinsicInst * II
RISCVCC::CondCode CC
MachineRegisterInfo & MRI
static bool isValid(const char C)
Returns true if C is a valid mangled character: <0-9a-zA-Z_>.
#define S_00B848_MEM_ORDERED(x)
Definition SIDefines.h:1502
#define S_00B848_WGP_MODE(x)
Definition SIDefines.h:1499
#define S_00B848_FWD_PROGRESS(x)
Definition SIDefines.h:1505
This file contains some functions that are useful when dealing with strings.
static const int BlockSize
Definition TarWriter.cpp:33
static ClusterDimsAttr get(const Function &F)
const std::array< unsigned, 3 > & getDims() const
static TargetID createFromSubtargetFeatures(const Triple &TT, StringRef CPU, StringRef FeatureString)
Construct a TargetID for triple TT and processor CPU, taking the xnack/sramecc modes from the subtarg...
unsigned getIndexInParsedOperands(unsigned CompOprIdx) const
unsigned getIndexOfSrcInParsedOperands(unsigned CompSrcIdx) const
std::optional< unsigned > getInvalidCompOperandIndex(std::function< MCRegister(unsigned, unsigned)> GetRegIdx, const MCRegisterInfo &MRI, bool SkipSrc=false, bool AllowSameVGPR=false, bool VOPD3=false, bool HasGFX11InterlockHazard=false) const
std::array< MCRegister, Component::MAX_OPR_NUM > RegIndices
Represents the counter values to wait for in an s_waitcnt instruction.
static const fltSemantics & BFloat()
Definition APFloat.h:303
static const fltSemantics & IEEEhalf()
Definition APFloat.h:302
bool isNegative() const
Definition APFloat.h:1583
bool isDenormal() const
Definition APFloat.h:1584
const fltSemantics & getSemantics() const
Definition APFloat.h:1591
static APFloat getOne(const fltSemantics &Sem, bool Negative=false)
Factory for Positive and Negative One.
Definition APFloat.h:1192
static APFloat getZero(const fltSemantics &Sem, bool Negative=false)
Factory for Positive and Negative Zero.
Definition APFloat.h:1183
This class represents an incoming formal argument to a Function.
Definition Argument.h:32
Functions, function parameters, and return types can have attributes to indicate how they should be t...
Definition Attributes.h:107
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
CallingConv::ID getCallingConv() const
LLVM_ABI bool paramHasAttr(unsigned ArgNo, Attribute::AttrKind Kind) const
Determine whether the argument or parameter has the given attribute.
bool isByValArgument(unsigned ArgNo) const
Determine whether this argument is passed by value.
constexpr bool test(unsigned I) const
unsigned getAddressSpace() const
This is an important class for using LLVM in a threaded context.
Definition LLVMContext.h:76
Instances of this class represent a single low-level machine instruction.
Definition MCInst.h:188
Describe properties that are true of each instruction in the target description file.
unsigned getNumOperands() const
Return the number of declared MachineOperands for this MachineInstruction.
ArrayRef< MCOperandInfo > operands() const
bool mayStore() const
Return true if this instruction could possibly modify memory.
bool mayLoad() const
Return true if this instruction could possibly read memory.
unsigned getNumDefs() const
Return the number of MachineOperands that are register definitions.
int getOperandConstraint(unsigned OpNum, MCOI::OperandConstraint Constraint) const
Returns the value of the specified operand constraint if it is present.
unsigned getOpcode() const
Return the opcode number for this descriptor.
Interface to description of machine instruction set.
Definition MCInstrInfo.h:27
const MCInstrDesc & get(unsigned Opcode) const
Return the machine instruction descriptor that corresponds to the specified instruction opcode.
Definition MCInstrInfo.h:89
int16_t getOpRegClassID(const MCOperandInfo &OpInfo, unsigned HwModeId) const
Return the ID of the register class to use for OpInfo, for the active HwMode HwModeId.
Definition MCInstrInfo.h:79
This holds information about one operand of a machine instruction, indicating the register class for ...
Definition MCInstrDesc.h:88
MCRegisterClass - Base class of TargetRegisterClass.
unsigned getID() const
getID() - Return the register class ID number.
MCRegister getRegister(unsigned i) const
getRegister - Return the specified register in the class.
bool contains(MCRegister Reg) const
contains - Return true if the specified register is included in this register class.
MCRegisterInfo base class - We assume that the target defines a static array of MCRegisterDesc object...
Wrapper class representing physical registers. Should be passed by value.
Definition MCRegister.h:41
constexpr unsigned id() const
Definition MCRegister.h:82
Generic base class for all target subtargets.
bool hasFeature(unsigned Feature) const
const Triple & getTargetTriple() const
const FeatureBitset & getFeatureBits() const
StringRef getCPU() const
Metadata node.
Definition Metadata.h:1081
const MDOperand & getOperand(unsigned I) const
Definition Metadata.h:1437
unsigned getNumOperands() const
Return number of MDNode operands.
Definition Metadata.h:1443
Representation of each machine instruction.
A Module instance is used to store all the information related to an LLVM module.
Definition Module.h:68
SmallString - A SmallString is just a SmallVector with methods and accessors that make it work better...
Definition SmallString.h:28
const char * c_str()
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
A wrapper around a string literal that serves as a proxy for constructing global tables of StringRefs...
Definition StringRef.h:888
Represent a constant reference to a string, i.e.
Definition StringRef.h:56
std::pair< StringRef, StringRef > split(char Separator) const
Split into two substrings around the first occurrence of a separator character.
Definition StringRef.h:736
bool getAsInteger(unsigned Radix, T &Result) const
Parse the current string as an integer of the specified radix.
Definition StringRef.h:490
constexpr bool empty() const
Check if the string is empty.
Definition StringRef.h:141
constexpr size_t size() const
Get the string size.
Definition StringRef.h:144
Triple - Helper class for working with autoconf configuration names.
Definition Triple.h:48
OSType getOS() const
Get the parsed operating system type of this triple.
Definition Triple.h:526
ArchType getArch() const
Get the parsed architecture type of this triple.
Definition Triple.h:517
bool isAMDGCN() const
Tests whether the target is AMDGCN.
Definition Triple.h:998
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition Twine.h:82
The instances of the Type class are immutable: once they are created, they are never changed.
Definition Type.h:46
A Use represents the edge between a Value definition and its users.
Definition Use.h:35
LLVM Value Representation.
Definition Value.h:75
const ParentTy * getParent() const
Definition ilist_node.h:34
This class implements an extremely fast bulk output stream that can only output to a stream.
Definition raw_ostream.h:53
A raw_ostream that writes to an SmallVector or SmallString.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ CONSTANT_ADDRESS_32BIT
Address space for 32-bit constant memory.
@ CONSTANT_ADDRESS
Address space for constant memory (VTX2).
unsigned decodeFieldVaVcc(unsigned Encoded)
unsigned encodeFieldVaVcc(unsigned Encoded, unsigned VaVcc)
unsigned decodeFieldHoldCnt(unsigned Encoded, const IsaVersion &Version)
bool decodeDepCtr(unsigned Code, int &Id, StringRef &Name, unsigned &Val, bool &IsDefault, const MCSubtargetInfo &STI)
unsigned encodeFieldHoldCnt(unsigned Encoded, unsigned HoldCnt, const IsaVersion &Version)
unsigned encodeFieldVaSsrc(unsigned Encoded, unsigned VaSsrc)
unsigned encodeFieldVaVdst(unsigned Encoded, unsigned VaVdst)
unsigned decodeFieldSaSdst(unsigned Encoded)
unsigned getHoldCntBitMask(const IsaVersion &Version)
unsigned decodeFieldVaSdst(unsigned Encoded)
unsigned encodeFieldVmVsrc(unsigned Encoded, unsigned VmVsrc)
unsigned decodeFieldVaSsrc(unsigned Encoded)
int encodeDepCtr(const StringRef Name, int64_t Val, unsigned &UsedOprMask, const MCSubtargetInfo &STI)
unsigned encodeFieldSaSdst(unsigned Encoded, unsigned SaSdst)
const CustomOperandVal DepCtrInfo[]
bool isSymbolicDepCtrEncoding(unsigned Code, bool &HasNonDefaultVal, const MCSubtargetInfo &STI)
unsigned decodeFieldVaVdst(unsigned Encoded)
int getDefaultDepCtrEncoding(const MCSubtargetInfo &STI)
unsigned decodeFieldVmVsrc(unsigned Encoded)
unsigned encodeFieldVaSdst(unsigned Encoded, unsigned VaSdst)
bool isSupportedTgtId(unsigned Id, const MCSubtargetInfo &STI)
static constexpr ExpTgt ExpTgtInfo[]
bool getTgtName(unsigned Id, StringRef &Name, int &Index)
unsigned getTgtId(const StringRef Name)
constexpr uint32_t VersionMinor
HSA metadata minor version.
constexpr uint32_t VersionMajor
HSA metadata major version.
unsigned getNumWavesPerEUWithNumVGPRs(const MCSubtargetInfo &STI, unsigned NumVGPRs, unsigned DynamicVGPRBlockSize)
static unsigned getMaxHWAddressableLocalMemorySize(const MCSubtargetInfo &STI)
unsigned getAddressableNumArchVGPRs(const MCSubtargetInfo &STI)
bool isSGPROccupancyLimited(const MCSubtargetInfo &STI)
unsigned getArchVGPRAllocGranule()
For subtargets with a unified VGPR file and mixed ArchVGPR/AGPR usage, returns the allocation granule...
static unsigned getPhysicalLocalMemorySize(const MCSubtargetInfo &STI)
static unsigned getSGPRTrapHandlerReserve(const MCSubtargetInfo &STI)
unsigned getAddressableLocalMemorySize(const MCSubtargetInfo &STI)
unsigned getVGPREncodingGranule(const MCSubtargetInfo &STI, std::optional< bool > EnableWavefrontSize32)
unsigned getMaxWorkGroupsPerCU(const MCSubtargetInfo &STI, unsigned FlatWorkGroupSize)
unsigned getMinNumSGPRs(const MCSubtargetInfo &STI, unsigned WavesPerEU)
unsigned getMaxNumSGPRs(const MCSubtargetInfo &STI, unsigned WavesPerEU, bool Addressable)
unsigned getWavefrontSize(const MCSubtargetInfo &STI)
unsigned getWavesPerEUForWorkGroup(const MCSubtargetInfo &STI, unsigned FlatWorkGroupSize)
unsigned getInstCacheLineSize(const MCSubtargetInfo &STI)
static constexpr unsigned MaxDynamicVGPRBlocks
Maximum number of VGPR blocks that can be allocated in dynamic VGPR mode.
unsigned getSGPREncodingGranule(const MCSubtargetInfo &STI)
static unsigned getSGPRBudgetPerWave(unsigned TotalNumSGPRs, unsigned WavesPerEU, unsigned TrapReserve, unsigned Granule)
unsigned getMinNumVGPRs(const MCSubtargetInfo &STI, unsigned WavesPerEU, unsigned DynamicVGPRBlockSize)
unsigned getAddressableNumVGPRs(const MCSubtargetInfo &STI, unsigned DynamicVGPRBlockSize)
unsigned getWavesPerWorkGroup(const MCSubtargetInfo &STI, unsigned FlatWorkGroupSize)
unsigned getAllocatedNumVGPRBlocks(const MCSubtargetInfo &STI, unsigned NumVGPRs, unsigned DynamicVGPRBlockSize, std::optional< bool > EnableWavefrontSize32)
unsigned getOccupancyWithNumSGPRs(unsigned SGPRs, unsigned MaxWaves, unsigned TotalNumSGPRs, unsigned Granule, unsigned TrapReserve)
unsigned getNumSGPRBlocks(const MCSubtargetInfo &STI, unsigned NumSGPRs)
unsigned getNumExtraSGPRs(const MCSubtargetInfo &STI, bool VCCUsed, bool FlatScrUsed, bool XNACKUsed)
unsigned getLocalMemorySize(const MCSubtargetInfo &STI)
unsigned getMaxNumVGPRs(const MCSubtargetInfo &STI, unsigned WavesPerEU, unsigned DynamicVGPRBlockSize)
static unsigned getGranulatedNumRegisterBlocks(unsigned NumRegs, unsigned Granule)
unsigned getVGPRAllocGranule(const MCSubtargetInfo &STI, unsigned DynamicVGPRBlockSize, std::optional< bool > EnableWavefrontSize32)
StringLiteral const UfmtSymbolicGFX11[]
bool isValidUnifiedFormat(unsigned Id, const MCSubtargetInfo &STI)
unsigned getDefaultFormatEncoding(const MCSubtargetInfo &STI)
StringRef getUnifiedFormatName(unsigned Id, const MCSubtargetInfo &STI)
unsigned const DfmtNfmt2UFmtGFX10[]
StringLiteral const DfmtSymbolic[]
static StringLiteral const * getNfmtLookupTable(const MCSubtargetInfo &STI)
bool isValidNfmt(unsigned Id, const MCSubtargetInfo &STI)
StringLiteral const NfmtSymbolicGFX10[]
bool isValidDfmtNfmt(unsigned Id, const MCSubtargetInfo &STI)
int64_t convertDfmtNfmt2Ufmt(unsigned Dfmt, unsigned Nfmt, const MCSubtargetInfo &STI)
StringRef getDfmtName(unsigned Id)
int64_t encodeDfmtNfmt(unsigned Dfmt, unsigned Nfmt)
int64_t getUnifiedFormat(const StringRef Name, const MCSubtargetInfo &STI)
bool isValidFormatEncoding(unsigned Val, const MCSubtargetInfo &STI)
StringRef getNfmtName(unsigned Id, const MCSubtargetInfo &STI)
unsigned const DfmtNfmt2UFmtGFX11[]
StringLiteral const NfmtSymbolicVI[]
StringLiteral const NfmtSymbolicSICI[]
int64_t getNfmt(const StringRef Name, const MCSubtargetInfo &STI)
int64_t getDfmt(const StringRef Name)
StringLiteral const UfmtSymbolicGFX10[]
void decodeDfmtNfmt(unsigned Format, unsigned &Dfmt, unsigned &Nfmt)
uint64_t encodeMsg(uint64_t MsgId, uint64_t OpId, uint64_t StreamId)
bool msgSupportsStream(int64_t MsgId, int64_t OpId, const MCSubtargetInfo &STI)
void decodeMsg(unsigned Val, uint16_t &MsgId, uint16_t &OpId, uint16_t &StreamId, const MCSubtargetInfo &STI)
bool isValidMsgId(int64_t MsgId, const MCSubtargetInfo &STI)
bool isValidMsgStream(int64_t MsgId, int64_t OpId, int64_t StreamId, const MCSubtargetInfo &STI, bool Strict)
bool msgDoesNotUseM0(int64_t MsgId, const MCSubtargetInfo &STI)
Returns true if the message does not use the m0 operand.
StringRef getMsgOpName(int64_t MsgId, uint64_t Encoding, const MCSubtargetInfo &STI)
Map from an encoding to the symbolic name for a sendmsg operation.
static uint64_t getMsgIdMask(const MCSubtargetInfo &STI)
bool msgRequiresOp(int64_t MsgId, const MCSubtargetInfo &STI)
bool isValidMsgOp(int64_t MsgId, int64_t OpId, const MCSubtargetInfo &STI, bool Strict)
constexpr unsigned VOPD_VGPR_BANK_MASKS[]
constexpr unsigned COMPONENTS_NUM
constexpr unsigned VOPD3_VGPR_BANK_MASKS[]
constexpr unsigned VOPD_GFX11_VGPR_BANK_MASKS[]
bool isGCN3Encoding(const MCSubtargetInfo &STI)
bool isGFX10_BEncoding(const MCSubtargetInfo &STI)
bool isInlineValue(MCRegister Reg)
bool isGFX10_GFX11(const MCSubtargetInfo &STI)
bool isInlinableLiteralV216(uint32_t Literal, uint8_t OpType)
bool isPKFMACF16InlineConstant(uint32_t Literal, bool IsGFX11Plus)
LLVM_READONLY const MIMGInfo * getMIMGInfo(unsigned Opc)
bool isInlinableLiteralFP16(int16_t Literal, bool HasInv2Pi)
bool isSGPR(MCRegister Reg, const MCRegisterInfo *TRI)
Is Reg - scalar register.
uint64_t convertSMRDOffsetUnits(const MCSubtargetInfo &ST, uint64_t ByteOffset)
Convert ByteOffset to dwords if the subtarget uses dword SMRD immediate offsets.
static unsigned encodeStorecnt(const IsaVersion &Version, unsigned Waitcnt, unsigned Storecnt)
MCRegister getMCReg(MCRegister Reg, const MCSubtargetInfo &STI)
If Reg is a pseudo reg, return the correct hardware register given STI otherwise return Reg.
static bool hasSMEMByteOffset(const MCSubtargetInfo &ST)
LLVM_ABI unsigned getMaxWavesPerEU(GPUKind AK)
bool isVOPCAsmOnly(unsigned Opc)
bool getMTBUFHasSrsrc(unsigned Opc)
std::optional< int64_t > getSMRDEncodedLiteralOffset32(const MCSubtargetInfo &ST, int64_t ByteOffset)
bool getWMMAIsXDL(unsigned Opc)
uint8_t wmmaScaleF8F6F4FormatToNumRegs(unsigned Fmt)
static bool isSymbolicCustomOperandEncoding(const CustomOperandVal *Opr, int Size, unsigned Code, bool &HasNonDefaultVal, const MCSubtargetInfo &STI)
bool isGFX10Before1030(const MCSubtargetInfo &STI)
unsigned mapWMMA2AddrTo3AddrOpcode(unsigned Opc)
const int OPR_ID_UNSUPPORTED
void initDefaultAMDKernelCodeT(AMDGPUMCKernelCodeT &KernelCode, const MCSubtargetInfo &STI)
bool shouldEmitConstantsToTextSection(const Triple &TT)
bool isInlinableLiteralV2I16(uint32_t Literal)
bool isDPMACCInstruction(unsigned Opc)
int getMTBUFElements(unsigned Opc)
constexpr unsigned getNumWorkGroupSIMDs(bool FullSIMDMode)
bool isHi16Reg(MCRegister Reg, const MCRegisterInfo &MRI)
static int encodeCustomOperandVal(const CustomOperandVal &Op, int64_t InputVal)
unsigned getTemporalHintType(const MCInstrDesc TID)
bool isGFX10(const MCSubtargetInfo &STI)
bool isInlinableLiteralV2BF16(uint32_t Literal)
unsigned getMaxNumUserSGPRs(const MCSubtargetInfo &STI)
std::optional< unsigned > getInlineEncodingV216(bool IsFloat, uint32_t Literal)
FPType getFPDstSelType(unsigned Opc)
unsigned getNumFlatOffsetBits(const MCSubtargetInfo &ST)
For pre-GFX12 FLAT instructions the offset must be positive; MSB is ignored and forced to zero.
bool hasA16(const MCSubtargetInfo &STI)
bool isLegalSMRDEncodedSignedOffset(const MCSubtargetInfo &ST, int64_t EncodedOffset, bool IsBuffer)
bool isGFX12Plus(const MCSubtargetInfo &STI)
unsigned getNSAMaxSize(const MCSubtargetInfo &STI, bool HasSampler)
const MCRegisterClass * getVGPRPhysRegClass(MCRegister Reg, const MCRegisterInfo &MRI)
unsigned encodeLoadcntDscnt(const IsaVersion &Version, const Waitcnt &Decoded)
bool getHasMatrixScale(unsigned Opc)
bool hasPackedD16(const MCSubtargetInfo &STI)
unsigned getStorecntBitMask(const IsaVersion &Version)
bool isFullSIMDMode(const MCSubtargetInfo &STI)
bool isGFX940(const MCSubtargetInfo &STI)
std::optional< unsigned > getExponentOnlyScaleArgIdx(unsigned IntrID)
bool isInlinableLiteralV2F16(uint32_t Literal)
bool isHsaAbi(const MCSubtargetInfo &STI)
bool isGFX11(const MCSubtargetInfo &STI)
static std::optional< unsigned > convertSetRegImmToVgprMSBs(uint64_t Imm, uint64_t Simm16, bool HasSetregVGPRMSBFixup)
const int OPR_VAL_INVALID
bool getSMEMIsBuffer(unsigned Opc)
bool isPackedSingleSGPRFP32Inst(unsigned Opc)
The opcode is a packed fp32 instruction which only reads low 32 bits of a scalar operand and propagat...
bool isGFX10_3_GFX11(const MCSubtargetInfo &STI)
bool isGFX13(const MCSubtargetInfo &STI)
unsigned getAsynccntBitMask(const IsaVersion &Version)
bool hasValueInRangeLikeMetadata(const MDNode &MD, int64_t Val)
Checks if Val is inside MD, a !range-like metadata.
LLVM_ABI unsigned getAddressableNumSGPRs(GPUKind AK)
TargetID createAMDGPUTargetID(const MCSubtargetInfo &STI, StringRef FeatureString)
Construct TargetID from MCSubtargetInfo.
uint8_t mfmaScaleF8F6F4FormatToNumRegs(unsigned EncodingVal)
unsigned getVOPDOpcode(unsigned Opc, bool VOPD3)
LLVM_ABI IsaVersion getIsaVersion(StringRef GPU)
bool getMTBUFHasSoffset(unsigned Opc)
unsigned getRegBitWidth(unsigned RCID)
Get the size in bits of a register from the register class RC.
bool isValid32BitLiteral(uint64_t Val, bool IsFP64)
static unsigned getCombinedCountBitMask(const IsaVersion &Version, bool IsStore)
LLVM_ABI unsigned getTotalNumVGPRs(GPUKind AK, bool IsWave32)
CanBeVOPD getCanBeVOPD(unsigned Opc, unsigned EncodingFamily, bool VOPD3)
bool isVOPC64DPP(unsigned Opc)
int getMUBUFOpcode(unsigned BaseOpc, unsigned Elements)
bool getMAIIsGFX940XDL(unsigned Opc)
bool isSI(const MCSubtargetInfo &STI)
unsigned getDefaultAMDHSACodeObjectVersion()
LLVM_ABI unsigned getTotalNumSGPRs(GPUKind AK)
bool hasPrivateApertureRegs(const MCSubtargetInfo &STI)
bool isReadOnlySegment(const GlobalValue *GV)
Waitcnt decodeWaitcnt(const IsaVersion &Version, unsigned Encoded)
bool isArgPassedInSGPR(const Argument *A)
bool isIntrinsicAlwaysUniform(unsigned IntrID)
int getMUBUFBaseOpcode(unsigned Opc)
unsigned encodeWaitcnt(const IsaVersion &Version, const Waitcnt &Decoded)
unsigned getAMDHSACodeObjectVersion(const Module &M)
unsigned decodeLgkmcnt(const IsaVersion &Version, unsigned Waitcnt)
unsigned getWaitcntBitMask(const IsaVersion &Version)
bool getVOP3IsSingle(unsigned Opc)
bool isPackedSingleSGPR64BitInst(unsigned Opc)
The opcode is a packed 64-bit instruction which only reads low 64 bits of a scalar operand and propag...
bool isGFX9(const MCSubtargetInfo &STI)
bool isDPALU_DPP32BitOpc(unsigned Opc)
bool getVOP1IsSingle(unsigned Opc)
static bool isDwordAligned(uint64_t ByteOffset)
unsigned getVOPDEncodingFamily(const MCSubtargetInfo &ST)
LLVM_READONLY bool hasNamedOperand(uint32_t Opcode, OpName NamedIdx)
bool isKImmOperand(const MCInstrDesc &Desc, unsigned OpNo)
Is this a KImm operand?
bool getHasColorExport(const Function &F)
GPUKind
GPU kinds supported by the AMDGPU target.
int getMTBUFBaseOpcode(unsigned Opc)
bool isGFX90A(const MCSubtargetInfo &STI)
unsigned getSamplecntBitMask(const IsaVersion &Version)
unsigned getDefaultQueueImplicitArgPosition(unsigned CodeObjectVersion)
int getMIMGOpcode(unsigned BaseOpcode, unsigned MIMGEncoding, unsigned VDataDwords, unsigned VAddrDwords, bool IndexedRsrc, bool IndexedSamp)
std::tuple< char, unsigned, unsigned > parseAsmPhysRegName(StringRef RegName)
Returns a valid charcode or 0 in the first entry if this is a valid physical register name.
bool isLDSDMAIntrinsic(unsigned IntrID)
bool getHasDepthExport(const Function &F)
bool isGFX8_GFX9_GFX10(const MCSubtargetInfo &STI)
bool getMUBUFHasVAddr(unsigned Opc)
bool hasNoOpSelHi(unsigned Opc)
Returns true if this VOP3P opcode has no op_sel_hi field, in which case those bits are encoded as zer...
bool isTrue16Inst(unsigned Opc)
LLVM_ABI unsigned getSGPRAllocGranule(GPUKind AK)
unsigned getVGPREncodingMSBs(MCRegister Reg, const MCRegisterInfo &MRI)
std::pair< unsigned, unsigned > getVOPDComponents(unsigned VOPDOpcode)
bool isInlinableLiteral32(int32_t Literal, bool HasInv2Pi)
bool isGFX12(const MCSubtargetInfo &STI)
unsigned getInitialPSInputAddr(const Function &F)
unsigned encodeExpcnt(const IsaVersion &Version, unsigned Waitcnt, unsigned Expcnt)
bool isAsyncStore(unsigned Opc)
unsigned getDynamicVGPRBlockSize(const Function &F)
unsigned getKmcntBitMask(const IsaVersion &Version)
bool isTriviallyUniform(const Value *V, const Instruction *CtxI)
Return true if we can easily prove that V is uniform at CtxI.
MCRegister getVGPRWithMSBs(MCRegister Reg, unsigned MSBs, const MCRegisterInfo &MRI)
If Reg is a low VGPR return a corresponding high VGPR with MSBs set.
unsigned getVmcntBitMask(const IsaVersion &Version)
bool isNotGFX10Plus(const MCSubtargetInfo &STI)
bool hasMAIInsts(const MCSubtargetInfo &STI)
unsigned getBitOp2(unsigned Opc)
bool isIntrinsicSourceOfDivergence(unsigned IntrID)
unsigned getXcntBitMask(const IsaVersion &Version)
bool isGenericAtomic(unsigned Opc)
const MFMA_F8F6F4_Info * getWMMA_F8F6F4_WithFormatArgs(unsigned FmtA, unsigned FmtB, unsigned F8F8Opcode)
bool isGFX8Plus(const MCSubtargetInfo &STI)
const VLdStIdxOpcodeInfo * getVLdStIdxOpcodeInfoByKey(unsigned BitWidth, bool IsStore, bool IsGPRIdx)
LLVM_READNONE bool isInlinableIntLiteral(int64_t Literal)
Is this literal inlinable, and not one of the values intended for floating point values.
unsigned getLgkmcntBitMask(const IsaVersion &Version)
bool getMUBUFTfe(unsigned Opc)
unsigned getBvhcntBitMask(const IsaVersion &Version)
bool isInlinableLiteralBF16(int16_t Literal, bool HasInv2Pi, bool *IsBF16)
bool hasSMRDSignedImmOffset(const MCSubtargetInfo &ST)
bool hasMIMG_R128(const MCSubtargetInfo &STI)
LLVM_ABI GPUKind parseArchAMDGCN(StringRef CPU)
bool hasGFX10_3Insts(const MCSubtargetInfo &STI)
unsigned decodeDscnt(const IsaVersion &Version, unsigned Waitcnt)
std::pair< const AMDGPU::OpName *, const AMDGPU::OpName * > getVGPRLoweringOperandTables(const MCInstrDesc &Desc)
bool hasG16(const MCSubtargetInfo &STI)
unsigned getAddrSizeMIMGOp(const MIMGBaseOpcodeInfo *BaseOpcode, const MIMGDimInfo *Dim, bool IsA16, bool IsG16Supported)
int getMTBUFOpcode(unsigned BaseOpc, unsigned Elements)
bool isGFX13Plus(const MCSubtargetInfo &STI)
unsigned getExpcntBitMask(const IsaVersion &Version)
bool hasArchitectedFlatScratch(const MCSubtargetInfo &STI)
int32_t getMCOpcode(uint32_t Opcode, unsigned Gen)
bool getMUBUFHasSoffset(unsigned Opc)
bool isNotGFX11Plus(const MCSubtargetInfo &STI)
bool isGFX11Plus(const MCSubtargetInfo &STI)
std::optional< unsigned > getInlineEncodingV2F16(uint32_t Literal)
bool isSISrcFPOperand(const MCInstrDesc &Desc, unsigned OpNo)
Is this floating-point operand?
static bool hasAny64BitVGPROperands(const MCInstrDesc &OpDesc, const MCInstrInfo &MII, const MCSubtargetInfo &ST)
std::optional< APFloat > evaluateRcp(const APFloat &Val)
Evaluate the constant-folded result of v_rcp for Val, accounting for the hardware's denormal flushing...
std::tuple< char, unsigned, unsigned > parseAsmConstraintPhysReg(StringRef Constraint)
Returns a valid charcode or 0 in the first entry if this is a valid physical register constraint.
unsigned getHostcallImplicitArgPosition(unsigned CodeObjectVersion)
static unsigned encodeLoadcnt(const IsaVersion &Version, unsigned Waitcnt, unsigned Loadcnt)
bool isGFX10Plus(const MCSubtargetInfo &STI)
static bool decodeCustomOperand(const CustomOperandVal *Opr, int Size, unsigned Code, int &Idx, StringRef &Name, unsigned &Val, bool &IsDefault, const MCSubtargetInfo &STI)
static bool isValidRegPrefix(char C)
std::optional< int64_t > getSMRDEncodedOffset(const MCSubtargetInfo &ST, int64_t ByteOffset, bool IsBuffer, bool HasSOffset)
SmallVector< unsigned > getMaxNumWorkGroups(const Function &F)
int64_t encode32BitLiteral(int64_t Imm, OperandType Type, bool IsLit)
bool isValidWMMAScaleFmtCombination(unsigned AFmt, unsigned AScale, unsigned BFmt, unsigned BScale)
@ OPERAND_REG_IMM_V2FP64
Definition SIDefines.h:447
@ OPERAND_KIMM32
Operand with 32-bit immediate that uses the constant bus.
Definition SIDefines.h:465
@ OPERAND_REG_IMM_V2FP16
Definition SIDefines.h:440
@ OPERAND_REG_INLINE_C_FP64
Definition SIDefines.h:456
@ OPERAND_REG_IMM_NOINLINE_FP16
Definition SIDefines.h:438
@ OPERAND_REG_INLINE_C_BF16
Definition SIDefines.h:453
@ OPERAND_REG_INLINE_C_V2BF16
Definition SIDefines.h:458
@ OPERAND_REG_IMM_V2INT16
Definition SIDefines.h:442
@ OPERAND_REG_IMM_BF16
Definition SIDefines.h:436
@ OPERAND_REG_IMM_INT32
Operands with register, 32-bit, or 64-bit immediate.
Definition SIDefines.h:431
@ OPERAND_REG_IMM_V2BF16
Definition SIDefines.h:439
@ OPERAND_REG_IMM_FP16
Definition SIDefines.h:437
@ OPERAND_REG_IMM_V2FP16_SPLAT
Definition SIDefines.h:441
@ OPERAND_REG_IMM_NOINLINE_V2FP16
Definition SIDefines.h:444
@ OPERAND_REG_IMM_FP64
Definition SIDefines.h:435
@ OPERAND_REG_INLINE_C_V2FP16
Definition SIDefines.h:459
@ OPERAND_REG_INLINE_AC_INT32
Operands with an AccVGPR register or inline constant.
Definition SIDefines.h:470
@ OPERAND_REG_INLINE_AC_FP32
Definition SIDefines.h:471
@ OPERAND_REG_IMM_V2INT32
Definition SIDefines.h:445
@ OPERAND_REG_IMM_FP32
Definition SIDefines.h:434
@ OPERAND_REG_INLINE_C_FP32
Definition SIDefines.h:455
@ OPERAND_REG_INLINE_C_INT32
Definition SIDefines.h:451
@ OPERAND_REG_INLINE_C_V2INT16
Definition SIDefines.h:457
@ OPERAND_REG_IMM_V2FP32
Definition SIDefines.h:446
@ OPERAND_REG_INLINE_AC_FP64
Definition SIDefines.h:472
@ OPERAND_REG_INLINE_C_FP16
Definition SIDefines.h:454
@ OPERAND_INLINE_SPLIT_BARRIER_INT32
Definition SIDefines.h:462
std::optional< unsigned > getPKFMACF16InlineEncoding(uint32_t Literal, bool IsGFX11Plus)
bool isNotGFX9Plus(const MCSubtargetInfo &STI)
bool isDPALU_DPP(const MCInstrDesc &OpDesc, const MCInstrInfo &MII, const MCSubtargetInfo &ST)
bool hasGDS(const MCSubtargetInfo &STI)
bool isLegalSMRDEncodedUnsignedOffset(const MCSubtargetInfo &ST, int64_t EncodedOffset)
bool isGFX9Plus(const MCSubtargetInfo &STI)
bool hasDPPSrc1SGPR(const MCSubtargetInfo &STI)
const int OPR_ID_DUPLICATE
bool isVOPD(unsigned Opc)
VOPD::InstInfo getVOPDInstInfo(const MCInstrDesc &OpX, const MCInstrDesc &OpY)
unsigned encodeVmcnt(const IsaVersion &Version, unsigned Waitcnt, unsigned Vmcnt)
unsigned decodeExpcnt(const IsaVersion &Version, unsigned Waitcnt)
bool isRsrcIndexReg(MCRegister Reg, const MCRegisterInfo &MRI)
bool isCvt_F32_Fp8_Bf8_e64(unsigned Opc)
std::optional< unsigned > getInlineEncodingV2I16(uint32_t Literal)
unsigned encodeStorecntDscnt(const IsaVersion &Version, const Waitcnt &Decoded)
bool isGFX1250(const MCSubtargetInfo &STI)
const MIMGBaseOpcodeInfo * getMIMGBaseOpcode(unsigned Opc)
bool isVI(const MCSubtargetInfo &STI)
bool isSingleSGPRReadInst(unsigned Opc)
Packed instructions that read a single SGPR for SGPR operands, except for 64-bit elements which read ...
bool isTensorStore(unsigned Opc)
bool getMUBUFIsBufferInv(unsigned Opc)
bool supportsScaleOffset(const MCInstrInfo &MII, unsigned Opcode)
unsigned getBarrierMemberCountBitMask(const IsaVersion &Version)
MCRegister mc2PseudoReg(MCRegister Reg)
Convert hardware register Reg to a pseudo register.
std::optional< unsigned > getInlineEncodingV2BF16(uint32_t Literal)
static int encodeCustomOperand(const CustomOperandVal *Opr, int Size, const StringRef Name, int64_t InputVal, unsigned &UsedOprMask, const MCSubtargetInfo &STI)
unsigned hasKernargPreload(const MCSubtargetInfo &STI)
bool supportsWGP(const MCSubtargetInfo &STI)
bool isMAC(unsigned Opc)
bool isCI(const MCSubtargetInfo &STI)
unsigned encodeLgkmcnt(const IsaVersion &Version, unsigned Waitcnt, unsigned Lgkmcnt)
bool getVOP2IsSingle(unsigned Opc)
bool getMAIIsDGEMM(unsigned Opc)
Returns true if MAI operation is a double precision GEMM.
LLVM_READONLY const MIMGBaseOpcodeInfo * getMIMGBaseOpcodeInfo(unsigned BaseOpcode)
const int OPR_ID_UNKNOWN
unsigned getCompletionActionImplicitArgPosition(unsigned CodeObjectVersion)
SmallVector< unsigned > getIntegerVecAttribute(const Function &F, StringRef Name, unsigned Size, unsigned DefaultVal)
unsigned decodeStorecnt(const IsaVersion &Version, unsigned Waitcnt)
bool isGFX1250Plus(const MCSubtargetInfo &STI)
int getMaskedMIMGOp(unsigned Opc, unsigned NewChannels)
bool isNotGFX12Plus(const MCSubtargetInfo &STI)
bool getMTBUFHasVAddr(unsigned Opc)
bool hasPopsExitingWaveID(const MCSubtargetInfo &STI)
unsigned decodeVmcnt(const IsaVersion &Version, unsigned Waitcnt)
uint8_t getELFABIVersion(const Triple &T, unsigned CodeObjectVersion)
std::pair< unsigned, unsigned > getIntegerPairAttribute(const Function &F, StringRef Name, std::pair< unsigned, unsigned > Default, bool OnlyFirstRequired)
unsigned getLoadcntBitMask(const IsaVersion &Version)
bool isInlinableLiteralI16(int32_t Literal, bool HasInv2Pi)
bool hasVOPD(const MCSubtargetInfo &STI)
int getVOPDFull(unsigned OpX, unsigned OpY, unsigned EncodingFamily, bool VOPD3)
static unsigned encodeDscnt(const IsaVersion &Version, unsigned Waitcnt, unsigned Dscnt)
bool isInlinableLiteral64(int64_t Literal, bool HasInv2Pi)
Is this literal inlinable.
const MFMA_F8F6F4_Info * getMFMA_F8F6F4_WithFormatArgs(unsigned CBSZ, unsigned BLGP, unsigned F8F8Opcode)
unsigned decodeLoadcnt(const IsaVersion &Version, unsigned Waitcnt)
unsigned getMultigridSyncArgImplicitArgPosition(unsigned CodeObjectVersion)
bool isGFX9_GFX10_GFX11(const MCSubtargetInfo &STI)
bool isGFX9_GFX10(const MCSubtargetInfo &STI)
int getMUBUFElements(unsigned Opc)
const GcnBufferFormatInfo * getGcnBufferFormatInfo(uint8_t BitsPerComp, uint8_t NumComponents, uint8_t NumFormat, const MCSubtargetInfo &STI)
bool isPermlane16(unsigned Opc)
bool getMUBUFHasSrsrc(unsigned Opc)
unsigned getDscntBitMask(const IsaVersion &Version)
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition CallingConv.h:24
@ AMDGPU_CS
Used for Mesa/AMDPAL compute shaders.
@ AMDGPU_VS
Used for Mesa vertex shaders, or AMDPAL last shader stage before rasterization (vertex shader if tess...
@ AMDGPU_KERNEL
Used for AMDGPU code object kernels.
@ AMDGPU_Gfx
Used for AMD graphics targets.
@ AMDGPU_CS_ChainPreserve
Used on AMDGPUs to give the middle-end more control over argument placement.
@ AMDGPU_HS
Used for Mesa/AMDPAL hull shaders (= tessellation control shaders).
@ AMDGPU_GS
Used for Mesa/AMDPAL geometry shaders.
@ AMDGPU_CS_Chain
Used on AMDGPUs to give the middle-end more control over argument placement.
@ AMDGPU_PS
Used for Mesa/AMDPAL pixel shaders.
@ SPIR_KERNEL
Used for SPIR kernel functions.
@ AMDGPU_ES
Used for AMDPAL shader stage before geometry shader if geometry is in use.
@ AMDGPU_LS
Used for AMDPAL vertex shader if tessellation is in use.
@ ELFABIVERSION_AMDGPU_HSA_V4
Definition ELF.h:192
@ ELFABIVERSION_AMDGPU_HSA_V5
Definition ELF.h:193
@ ELFABIVERSION_AMDGPU_HSA_V6
Definition ELF.h:194
constexpr bool isVOPC(const T &...O)
Definition SIDefines.h:236
constexpr bool isVOP3(const T &...O)
Definition SIDefines.h:239
constexpr bool isVOP1(const T &...O)
Definition SIDefines.h:230
constexpr bool isVOP2(const T &...O)
Definition SIDefines.h:233
constexpr bool isFLAT(const T &...O)
Definition SIDefines.h:286
constexpr bool isBuffer(const T &...O)
Definition SIDefines.h:267
constexpr bool isVIMAGE(const T &...O)
Definition SIDefines.h:277
constexpr bool isSMRD(const T &...O)
Definition SIDefines.h:271
constexpr bool isVOP3Like(const T &...O)
Definition SIDefines.h:245
constexpr bool isFlatScratch(const T &...O)
Definition SIDefines.h:361
constexpr bool isMIMG(const T &...O)
Definition SIDefines.h:274
constexpr bool isVOPD3(const T &...O)
Definition SIDefines.h:385
constexpr bool isEXP(const T &...O)
Definition SIDefines.h:283
constexpr bool isVSAMPLE(const T &...O)
Definition SIDefines.h:280
constexpr bool isDS(const T &...O)
Definition SIDefines.h:289
constexpr bool isAtomic(const T &...O)
Definition SIDefines.h:396
constexpr bool isDPP(const T &...O)
Definition SIDefines.h:255
initializer< Ty > init(const Ty &Val)
std::enable_if_t< detail::IsValidPointer< X, Y >::value, X * > extract_or_null(Y &&MD)
Extract a Value from Metadata, allowing null.
Definition Metadata.h:694
std::enable_if_t< detail::IsValidPointer< X, Y >::value, X * > extract(Y &&MD)
Extract a Value from Metadata.
Definition Metadata.h:679
This is an optimization pass for GlobalISel generic memory operations.
@ Low
Lower the current thread's priority such that it does not affect foreground tasks significantly.
Definition Threading.h:280
@ Offset
Definition DWP.cpp:577
constexpr T rotr(T V, int R)
Definition bit.h:399
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1755
constexpr bool isInt(int64_t x)
Checks if an integer fits into the given bit width.
Definition MathExtras.h:166
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:643
testing::Matcher< const detail::ErrorHolder & > Failed()
Definition Error.h:198
constexpr T alignDown(U Value, V Align, W Skew=0)
Returns the largest unsigned integer less than or equal to Value and is Skew mod Align.
Definition MathExtras.h:546
std::string utostr(uint64_t X, bool isNeg=false)
constexpr auto equal_to(T &&Arg)
Functor variant of std::equal_to that can be used as a UnaryPredicate in functional algorithms like a...
Definition STLExtras.h:2189
Op::Description Desc
constexpr uint32_t Hi_32(uint64_t Value)
Return the high 32 bits of a 64 bit value.
Definition MathExtras.h:151
LLVM_ABI void report_fatal_error(Error Err, bool gen_crash_diag=true)
Definition Error.cpp:163
constexpr uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
Definition Alignment.h:149
constexpr bool isUInt(uint64_t x)
Checks if an unsigned integer fits into the given bit width.
Definition MathExtras.h:190
constexpr uint32_t Lo_32(uint64_t Value)
Return the low 32 bits of a 64 bit value.
Definition MathExtras.h:156
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
Definition Casting.h:547
LLVM_ATTRIBUTE_VISIBILITY_DEFAULT AnalysisKey InnerAnalysisManagerProxy< AnalysisManagerT, IRUnitT, ExtraArgTs... >::Key
constexpr T divideCeil(U Numerator, V Denominator)
Returns the integer ceil(Numerator / Denominator).
Definition MathExtras.h:389
To bit_cast(const From &from) noexcept
Definition bit.h:90
DWARFExpression::Operation Op
raw_ostream & operator<<(raw_ostream &OS, const APFixedPoint &FX)
constexpr unsigned BitWidth
constexpr int countr_zero_constexpr(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
Definition bit.h:190
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:559
constexpr T maskTrailingOnes(unsigned N)
Create a bitmask with the N right-most bits set to 1, and all other bits set to 0.
Definition MathExtras.h:78
@ End
Run the machine verifier once, at the end of the pipeline.
Definition CodeGen.h:320
@ AlwaysUniform
The result value is always uniform.
Definition Uniformity.h:23
#define N
AMD Kernel Code Object (amd_kernel_code_t).
static std::tuple< typename Fields::ValueType... > decode(uint64_t Encoded)
Instruction set architecture version.