LLVM 24.0.0git
AMDGPUInstructionSelector.h
Go to the documentation of this file.
1//===- AMDGPUInstructionSelector --------------------------------*- C++ -*-==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file declares the targeting of the InstructionSelector class for
10/// AMDGPU.
11//===----------------------------------------------------------------------===//
12
13#ifndef LLVM_LIB_TARGET_AMDGPU_AMDGPUINSTRUCTIONSELECTOR_H
14#define LLVM_LIB_TARGET_AMDGPU_AMDGPUINSTRUCTIONSELECTOR_H
15
16#include "SIDefines.h"
18#include "llvm/IR/InstrTypes.h"
19
20namespace {
21#define GET_GLOBALISEL_PREDICATE_BITSET
22#define AMDGPUSubtarget GCNSubtarget
23#include "AMDGPUGenGlobalISel.inc"
24#undef GET_GLOBALISEL_PREDICATE_BITSET
25#undef AMDGPUSubtarget
26}
27
28namespace llvm {
29
30namespace AMDGPU {
33}
34
39class GCNSubtarget;
40class MachineInstr;
42class MachineOperand;
44class RegisterBank;
45class SIInstrInfo;
46class SIRegisterInfo;
47class MCRegisterClass;
49
51private:
53 const GCNSubtarget *Subtarget;
54
55public:
57 const AMDGPURegisterBankInfo &RBI);
58
59 bool select(MachineInstr &I) override;
60 static const char *getName();
61
64 BlockFrequencyInfo *BFI) override;
65
66private:
67 struct GEPInfo {
70 int64_t Imm = 0;
71 };
72
73 bool isSGPR(Register Reg) const;
74
75 bool isInstrUniform(const MachineInstr &MI) const;
76 bool isVCC(Register Reg, const MachineRegisterInfo &MRI) const;
77
78 const RegisterBank *getArtifactRegBank(
80 const TargetRegisterInfo &TRI) const;
81
82 /// tblgen-erated 'select' implementation.
83 bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const;
84
85 MachineOperand getSubOperand64(MachineOperand &MO,
86 const TargetRegisterClass &SubRC,
87 unsigned SubIdx) const;
88
89 bool constrainCopyLikeIntrin(MachineInstr &MI, unsigned NewOpc) const;
90 bool selectCOPY(MachineInstr &I) const;
91 bool selectCOPY_SCC_VCC(MachineInstr &I) const;
92 bool selectCOPY_VCC_SCC(MachineInstr &I) const;
93 bool selectReadAnyLane(MachineInstr &I) const;
94 bool selectPHI(MachineInstr &I) const;
95 bool selectG_TRUNC(MachineInstr &I) const;
96 bool selectG_SZA_EXT(MachineInstr &I) const;
97 bool selectG_FPEXT(MachineInstr &I) const;
98 bool selectG_FNEG(MachineInstr &I) const;
99 bool selectG_FABS(MachineInstr &I) const;
100 bool selectG_AND_OR_XOR(MachineInstr &I) const;
101 bool selectG_ADD_SUB(MachineInstr &I) const;
102 bool selectG_UADDO_USUBO_UADDE_USUBE(MachineInstr &I) const;
103 bool selectG_AMDGPU_MAD_64_32(MachineInstr &I) const;
104 bool selectG_EXTRACT(MachineInstr &I) const;
105 bool selectS16MergeToS32(MachineInstr &MI) const;
106 bool selectS16MergeToWide(MachineInstr &MI) const;
107 bool selectG_MERGE_VALUES(MachineInstr &I) const;
108 bool selectG_UNMERGE_VALUES(MachineInstr &I) const;
109 bool selectG_BUILD_VECTOR(MachineInstr &I) const;
110 bool selectG_IMPLICIT_DEF(MachineInstr &I) const;
111 bool selectG_INSERT(MachineInstr &I) const;
112 bool selectG_SBFX_UBFX(MachineInstr &I) const;
113
114 bool selectInterpP1F16(MachineInstr &MI) const;
115 bool selectWritelane(MachineInstr &MI) const;
116 bool selectDivScale(MachineInstr &MI) const;
117 bool selectBallot(MachineInstr &I) const;
118 bool selectRelocConstant(MachineInstr &I) const;
119 bool selectGroupStaticSize(MachineInstr &I) const;
120 bool selectReturnAddress(MachineInstr &I) const;
121 bool selectG_INTRINSIC(MachineInstr &I) const;
122
123 bool selectEndCfIntrinsic(MachineInstr &MI) const;
124 bool selectDSOrderedIntrinsic(MachineInstr &MI, Intrinsic::ID IID) const;
125 bool selectDSGWSIntrinsic(MachineInstr &MI, Intrinsic::ID IID) const;
126 bool selectDSAppendConsume(MachineInstr &MI, bool IsAppend) const;
127 bool selectInitWholeWave(MachineInstr &MI) const;
128 bool selectDSBvhStackIntrinsic(MachineInstr &MI) const;
129
130 bool selectImageIntrinsic(MachineInstr &MI,
131 const AMDGPU::ImageDimIntrinsicInfo *Intr) const;
132 bool selectG_INTRINSIC_W_SIDE_EFFECTS(MachineInstr &I) const;
133 int getS_CMPOpcode(CmpInst::Predicate P, uint64_t Size) const;
134 bool selectG_ICMP_or_FCMP(MachineInstr &I) const;
135 bool hasVgprParts(ArrayRef<GEPInfo> AddrInfo) const;
136 void getAddrModeInfo(const MachineInstr &Load, const MachineRegisterInfo &MRI,
137 SmallVectorImpl<GEPInfo> &AddrInfo) const;
138
139 void initM0(MachineInstr &I) const;
140 bool selectG_LOAD_STORE_ATOMICRMW(MachineInstr &I) const;
141 bool selectG_SELECT(MachineInstr &I) const;
142 bool selectG_BRCOND(MachineInstr &I) const;
143 bool selectG_GLOBAL_VALUE(MachineInstr &I) const;
144 bool selectG_PTRMASK(MachineInstr &I) const;
145 bool selectG_EXTRACT_VECTOR_ELT(MachineInstr &I) const;
146 bool selectG_INSERT_VECTOR_ELT(MachineInstr &I) const;
147 bool selectBufferLoadLds(MachineInstr &MI,
148 const AMDGPU::BufferIntrinsicInfo &Info) const;
149 bool selectGlobalLoadLds(MachineInstr &MI) const;
150 bool selectTensorLoadStore(MachineInstr &MI, Intrinsic::ID IID) const;
151 bool selectBVHIntersectRayIntrinsic(MachineInstr &I) const;
152 bool selectSMFMACIntrin(MachineInstr &I) const;
153 bool selectPermlaneSwapIntrin(MachineInstr &I, Intrinsic::ID IntrID) const;
154 bool selectWaveAddress(MachineInstr &I) const;
155 bool selectBITOP3(MachineInstr &I) const;
156 bool selectWriteRegister(MachineInstr &MI) const;
157 bool selectStackRestore(MachineInstr &MI) const;
158 bool selectNamedBarrierInit(MachineInstr &I, Intrinsic::ID IID) const;
159 bool selectNamedBarrierInst(MachineInstr &I, Intrinsic::ID IID) const;
160 bool selectSBarrierSignalIsfirst(MachineInstr &I, Intrinsic::ID IID) const;
161 bool selectSGetBarrierState(MachineInstr &I, Intrinsic::ID IID) const;
162 bool selectWaveShuffleIntrin(MachineInstr &I) const;
163
164 std::pair<Register, unsigned> selectVOP3ModsImpl(Register Src,
165 bool IsCanonicalizing = true,
166 bool AllowAbs = true,
167 bool OpSel = false) const;
168 std::pair<Register, unsigned> selectVOP3PModsF32Impl(Register Src) const;
169
170 Register copyToVGPRIfSrcFolded(Register Src, unsigned Mods,
171 MachineOperand Root, MachineInstr *InsertPt,
172 bool ForceVGPR = false) const;
173
174 Register widenSrcIfVGPR16(Register Src, MachineInstr *InsertPt) const;
175
177 selectVCSRC(MachineOperand &Root) const;
178
180 selectVSRC0(MachineOperand &Root) const;
181
183 selectVOP3Mods0(MachineOperand &Root) const;
185 selectVOP3BMods0(MachineOperand &Root) const;
187 selectVOP3OMods(MachineOperand &Root) const;
189 selectVOP3Mods(MachineOperand &Root) const;
191 selectVOP3ModsNonCanonicalizing(MachineOperand &Root) const;
193 selectVOP3BMods(MachineOperand &Root) const;
194
195 ComplexRendererFns selectVOP3NoMods(MachineOperand &Root) const;
196 ComplexRendererFns selectVOP3NoOpSelMods(MachineOperand &Root) const;
197
198 std::pair<Register, unsigned>
199 selectVOP3PModsImpl(Register RootReg, const MachineRegisterInfo &MRI,
200 bool IsDOT = false, bool NoOpSel0 = false) const;
202 selectVOP3PRetHelper(MachineOperand &Root, bool IsDOT = false,
203 bool NoOpSel0 = false) const;
204
206 selectVOP3PMods(MachineOperand &Root) const;
208 selectVOP3PModsPkFmaOpSel0(MachineOperand &Root) const;
210 selectVOP3PModsPkAddMulOpSel0(MachineOperand &Root) const;
211
213 selectVOP3PModsDOT(MachineOperand &Root) const;
215 selectVOP3PNoModsDOT(MachineOperand &Root) const;
217 selectVOP3PModsF32(MachineOperand &Root) const;
219 selectVOP3PNoModsF32(MachineOperand &Root) const;
220
222 selectWMMAOpSelVOP3PMods(MachineOperand &Root) const;
223
225 selectWMMAModsF32NegAbs(MachineOperand &Root) const;
227 selectWMMAModsF16Neg(MachineOperand &Root) const;
229 selectWMMAModsF16NegAbs(MachineOperand &Root) const;
231 selectWMMAVISrc(MachineOperand &Root) const;
233 selectSWMMACIndex8(MachineOperand &Root) const;
235 selectSWMMACIndex16(MachineOperand &Root) const;
237 selectSWMMACIndex32(MachineOperand &Root) const;
238
240 selectVOP3OpSelMods(MachineOperand &Root) const;
241
243 selectVINTERPMods(MachineOperand &Root) const;
245 selectVINTERPModsHi(MachineOperand &Root) const;
246
247 bool selectScaleOffset(MachineOperand &Root, Register &Offset, bool IsSigned,
248 bool IsSMEM = false) const;
249 bool selectSmrdOffset(MachineOperand &Root, Register &Base, Register *SOffset,
250 int64_t *Offset, bool *ScaleOffset) const;
252 selectSmrdImm(MachineOperand &Root) const;
254 selectSmrdImm32(MachineOperand &Root) const;
256 selectSmrdSgpr(MachineOperand &Root) const;
258 selectSmrdSgprImm(MachineOperand &Root) const;
259
260 std::pair<Register, int64_t>
261 selectFlatOffsetImpl(MachineOperand &Root,
262 AMDGPU::FlatAddrSpace FlatVariant) const;
263
265 selectFlatOffset(MachineOperand &Root) const;
267 selectGlobalOffset(MachineOperand &Root) const;
269 selectScratchOffset(MachineOperand &Root) const;
270
272 selectGlobalSAddr(MachineOperand &Root, unsigned CPolBits,
273 bool NeedIOffset = true) const;
275 selectGlobalSAddr(MachineOperand &Root) const;
277 selectGlobalSAddrCPol(MachineOperand &Root) const;
279 selectGlobalSAddrCPolM0(MachineOperand &Root) const;
281 selectGlobalSAddrGLC(MachineOperand &Root) const;
283 selectGlobalSAddrNoIOffset(MachineOperand &Root) const;
285 selectGlobalSAddrNoIOffsetM0(MachineOperand &Root) const;
286
288 selectScratchSAddr(MachineOperand &Root) const;
289 bool checkFlatScratchSVSSwizzleBug(Register VAddr, Register SAddr,
290 uint64_t ImmOffset) const;
292 selectScratchSVAddr(MachineOperand &Root) const;
293
295 selectMUBUFScratchOffen(MachineOperand &Root) const;
297 selectMUBUFScratchOffset(MachineOperand &Root) const;
298
299 bool isDSOffsetLegal(Register Base, int64_t Offset) const;
300 bool isDSOffset2Legal(Register Base, int64_t Offset0, int64_t Offset1,
301 unsigned Size) const;
302 bool isFlatScratchBaseLegal(Register Addr) const;
303 bool isFlatScratchBaseLegalSV(Register Addr) const;
304 bool isFlatScratchBaseLegalSVImm(Register Addr) const;
305
306 std::pair<Register, int64_t>
307 selectDS1Addr1OffsetImpl(MachineOperand &Root) const;
309 selectDS1Addr1Offset(MachineOperand &Root) const;
310
312 selectDS64Bit4ByteAligned(MachineOperand &Root) const;
313
315 selectDS128Bit8ByteAligned(MachineOperand &Root) const;
316
317 std::pair<Register, int64_t> selectDSReadWrite2Impl(MachineOperand &Root,
318 unsigned size) const;
320 selectDSReadWrite2(MachineOperand &Root, unsigned size) const;
321
322 std::tuple<Register, int64_t, bool>
323 getPtrBaseWithConstantOffset(Register Root,
324 const MachineRegisterInfo &MRI) const;
325
326 // Parse out a chain of up to two g_ptr_add instructions.
327 // g_ptr_add (n0, _)
328 // g_ptr_add (n0, (n1 = g_ptr_add n2, n3))
329 struct MUBUFAddressData {
330 Register N0, N2, N3;
331 int64_t Offset = 0;
332 };
333
334 bool shouldUseAddr64(MUBUFAddressData AddrData) const;
335
336 void splitIllegalMUBUFOffset(MachineIRBuilder &B,
337 Register &SOffset, int64_t &ImmOffset) const;
338
339 MUBUFAddressData parseMUBUFAddress(Register Src) const;
340
341 bool selectMUBUFAddr64Impl(MachineOperand &Root, Register &VAddr,
342 Register &RSrcReg, Register &SOffset,
343 int64_t &Offset) const;
344
345 bool selectMUBUFOffsetImpl(MachineOperand &Root, Register &RSrcReg,
346 Register &SOffset, int64_t &Offset) const;
347
349 selectBUFSOffset(MachineOperand &Root) const;
350
352 selectMUBUFAddr64(MachineOperand &Root) const;
353
355 selectMUBUFOffset(MachineOperand &Root) const;
356
357 ComplexRendererFns selectSMRDBufferImm(MachineOperand &Root) const;
358 ComplexRendererFns selectSMRDBufferImm32(MachineOperand &Root) const;
359 ComplexRendererFns selectSMRDBufferSgprImm(MachineOperand &Root) const;
360
361 std::pair<Register, unsigned> selectVOP3PMadMixModsImpl(MachineOperand &Root,
362 bool &Matched) const;
363 ComplexRendererFns selectVOP3PMadMixModsExt(MachineOperand &Root) const;
364 ComplexRendererFns selectVOP3PMadMixMods(MachineOperand &Root) const;
365 ComplexRendererFns selectVOP3PMadMixModsExtNeg(MachineOperand &Root) const;
366 ComplexRendererFns selectVOP3PMadMixModsNeg(MachineOperand &Root) const;
367
368 void renderTruncTImm(MachineInstrBuilder &MIB, const MachineInstr &MI,
369 int OpIdx) const;
370 void renderPrefetchLen(MachineInstrBuilder &MIB, const MachineInstr &MI,
371 int OpIdx) const;
372 void renderZextBoolTImm(MachineInstrBuilder &MIB, const MachineInstr &MI,
373 int OpIdx) const;
374
375 void renderOpSelTImm(MachineInstrBuilder &MIB, const MachineInstr &MI,
376 int OpIdx) const;
377
378 void renderSrcAndDstSelToOpSelXForm_0_0(MachineInstrBuilder &MIB,
379 const MachineInstr &MI,
380 int OpIdx) const;
381
382 void renderSrcAndDstSelToOpSelXForm_0_1(MachineInstrBuilder &MIB,
383 const MachineInstr &MI,
384 int OpIdx) const;
385
386 void renderSrcAndDstSelToOpSelXForm_1_0(MachineInstrBuilder &MIB,
387 const MachineInstr &MI,
388 int OpIdx) const;
389
390 void renderSrcAndDstSelToOpSelXForm_1_1(MachineInstrBuilder &MIB,
391 const MachineInstr &MI,
392 int OpIdx) const;
393
394 void renderDstSelToOpSelXForm(MachineInstrBuilder &MIB,
395 const MachineInstr &MI, int OpIdx) const;
396
397 void renderSrcSelToOpSelXForm(MachineInstrBuilder &MIB,
398 const MachineInstr &MI, int OpIdx) const;
399
400 void renderSrcAndDstSelToOpSelXForm_2_0(MachineInstrBuilder &MIB,
401 const MachineInstr &MI,
402 int OpIdx) const;
403
404 void renderDstSelToOpSel3XFormXForm(MachineInstrBuilder &MIB,
405 const MachineInstr &MI, int OpIdx) const;
406
407 void renderNegateImm(MachineInstrBuilder &MIB, const MachineInstr &MI,
408 int OpIdx) const;
409
410 void renderBitcastFPImm(MachineInstrBuilder &MIB, const MachineInstr &MI,
411 int OpIdx) const;
412
413 void renderBitcastFPImm32(MachineInstrBuilder &MIB, const MachineInstr &MI,
414 int OpIdx) const {
415 renderBitcastFPImm(MIB, MI, OpIdx);
416 }
417 void renderBitcastFPImm64(MachineInstrBuilder &MIB, const MachineInstr &MI,
418 int OpIdx) const {
419 renderBitcastFPImm(MIB, MI, OpIdx);
420 }
421
422 void renderCountTrailingOnesImm(MachineInstrBuilder &MIB,
423 const MachineInstr &MI, int OpIdx) const;
424 void renderExtractCPol(MachineInstrBuilder &MIB, const MachineInstr &MI,
425 int OpIdx) const;
426 void renderExtractSWZ(MachineInstrBuilder &MIB, const MachineInstr &MI,
427 int OpIdx) const;
428 void renderExtractCpolSetGLC(MachineInstrBuilder &MIB, const MachineInstr &MI,
429 int OpIdx) const;
430
431 void renderFPPow2ToExponent(MachineInstrBuilder &MIB, const MachineInstr &MI,
432 int OpIdx) const;
433
434 void renderRoundMode(MachineInstrBuilder &MIB, const MachineInstr &MI,
435 int OpIdx) const;
436
437 void renderVOP3PModsNeg(MachineInstrBuilder &MIB, const MachineInstr &MI,
438 int OpIdx) const;
439 void renderVOP3PModsNegs(MachineInstrBuilder &MIB, const MachineInstr &MI,
440 int OpIdx) const;
441 void renderVOP3PModsNegAbs(MachineInstrBuilder &MIB, const MachineInstr &MI,
442 int OpIdx) const;
443
444 void renderPrefetchLoc(MachineInstrBuilder &MIB, const MachineInstr &MI,
445 int OpIdx) const;
446
447 void renderScaledMAIIntrinsicOperand(MachineInstrBuilder &MIB,
448 const MachineInstr &MI, int OpIdx) const;
449
450 bool isInlineImmediate(const APInt &Imm) const;
451 bool isInlineImmediate(const APFloat &Imm) const;
452
453 // Returns true if TargetOpcode::G_AND MachineInstr `MI`'s masking of the
454 // shift amount operand's `ShAmtBits` bits is unneeded.
455 bool isUnneededShiftMask(const MachineInstr &MI, unsigned ShAmtBits) const;
456
457 /// Match a zero extend from a 32-bit value to 64-bits.
458 Register matchZeroExtendFromS32(Register Reg) const;
459 /// Match a sign extend from a 32-bit value to 64-bits.
460 Register matchSignExtendFromS32(Register Reg) const;
461 /// Match a zero extend from a 32-bit value to 64-bits, or \p Reg itself if it
462 /// is 32-bit.
463 Register matchZeroExtendFromS32OrS32(Register Reg) const;
464 /// Match a sign extend from a 32-bit value to 64-bits, or \p Reg itself if it
465 /// is 32-bit.
466 Register matchSignExtendFromS32OrS32(Register Reg) const;
467 /// Match either sign or zero extend depending on the \p IsSigned from a
468 /// 32-bit value to 64-bits, or \p Reg itself if it is 32-bit.
469 Register matchExtendFromS32OrS32(Register Reg, bool IsSigned) const;
470 /// Match an any extend from a 32-bit value to 64-bit.
471 Register matchAnyExtendFromS32(Register Reg) const;
472
473 const SIInstrInfo &TII;
474 const SIRegisterInfo &TRI;
475 const AMDGPURegisterBankInfo &RBI;
476 const GCNSubtarget &STI;
477#define GET_GLOBALISEL_PREDICATES_DECL
478#define AMDGPUSubtarget GCNSubtarget
479#include "AMDGPUGenGlobalISel.inc"
480#undef GET_GLOBALISEL_PREDICATES_DECL
481#undef AMDGPUSubtarget
482
483#define GET_GLOBALISEL_TEMPORARIES_DECL
484#include "AMDGPUGenGlobalISel.inc"
485#undef GET_GLOBALISEL_TEMPORARIES_DECL
486};
487
488} // End llvm namespace.
489#endif
uint64_t uint64_t
uint64_t Imm
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
IRTranslator LLVM IR MI
#define I(x, y, z)
Definition MD5.cpp:57
Register Reg
Register const TargetRegisterInfo * TRI
Promote Memory to Register
Definition Mem2Reg.cpp:110
#define P(N)
MachineRegisterInfo & MRI
AMDGPUInstructionSelector(const GCNSubtarget &STI, const AMDGPURegisterBankInfo &RBI)
static const char * getName()
bool select(MachineInstr &I) override
Select the (possibly generic) instruction I to only use target-specific opcodes.
void setupMF(MachineFunction &MF, GISelValueTracking *VT, CodeGenCoverage *CoverageInfo, ProfileSummaryInfo *PSI, BlockFrequencyInfo *BFI) override
Setup per-MF executor state.
Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition ArrayRef.h:40
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition InstrTypes.h:740
std::optional< SmallVector< std::function< void(MachineInstrBuilder &)>, 4 > > ComplexRendererFns
MCRegisterClass - Base class of TargetRegisterClass.
Helper class to build MachineInstr.
Representation of each machine instruction.
MachineOperand class - Representation of each machine instruction operand.
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
Analysis providing profile information.
This class implements the register bank concept.
Wrapper class representing virtual and physical registers.
Definition Register.h:20
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
This is an optimization pass for GlobalISel generic memory operations.
@ Offset
Definition DWP.cpp:577
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
Definition STLExtras.h:1685
@ Load
The value being inserted comes from a load (InsertElement only).
MCRegisterClass TargetRegisterClass
Definition FastISel.h:58
The argument layout of a buffer intrinsic.