LLVM 24.0.0git
SILoadStoreOptimizer.cpp
Go to the documentation of this file.
1//===- SILoadStoreOptimizer.cpp -------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass tries to fuse DS instructions with close by immediate offsets.
10// This will fuse operations such as
11// ds_read_b32 v0, v2 offset:16
12// ds_read_b32 v1, v2 offset:32
13// ==>
14// ds_read2_b32 v[0:1], v2, offset0:4 offset1:8
15//
16// The same is done for certain SMEM and VMEM opcodes, e.g.:
17// s_buffer_load_dword s4, s[0:3], 4
18// s_buffer_load_dword s5, s[0:3], 8
19// ==>
20// s_buffer_load_dwordx2 s[4:5], s[0:3], 4
21//
22// This pass also tries to promote constant offset to the immediate by
23// adjusting the base. It tries to use a base from the nearby instructions that
24// allows it to have a 13bit constant offset and then promotes the 13bit offset
25// to the immediate.
26// E.g.
27// s_movk_i32 s0, 0x1800
28// v_add_co_u32_e32 v0, vcc, s0, v2
29// v_addc_co_u32_e32 v1, vcc, 0, v6, vcc
30//
31// s_movk_i32 s0, 0x1000
32// v_add_co_u32_e32 v5, vcc, s0, v2
33// v_addc_co_u32_e32 v6, vcc, 0, v6, vcc
34// global_load_dwordx2 v[5:6], v[5:6], off
35// global_load_dwordx2 v[0:1], v[0:1], off
36// =>
37// s_movk_i32 s0, 0x1000
38// v_add_co_u32_e32 v5, vcc, s0, v2
39// v_addc_co_u32_e32 v6, vcc, 0, v6, vcc
40// global_load_dwordx2 v[5:6], v[5:6], off
41// global_load_dwordx2 v[0:1], v[5:6], off offset:2048
42//
43// Future improvements:
44//
45// - This is currently missing stores of constants because loading
46// the constant into the data register is placed between the stores, although
47// this is arguably a scheduling problem.
48//
49// - Live interval recomputing seems inefficient. This currently only matches
50// one pair, and recomputes live intervals and moves on to the next pair. It
51// would be better to compute a list of all merges that need to occur.
52//
53// - With a list of instructions to process, we can also merge more. If a
54// cluster of loads have offsets that are too large to fit in the 8-bit
55// offsets, but are close enough to fit in the 8 bits, we can add to the base
56// pointer and use the new reduced offsets.
57//
58//===----------------------------------------------------------------------===//
59
60#include "SILoadStoreOptimizer.h"
61#include "AMDGPU.h"
62#include "GCNSubtarget.h"
63#include "SIDefines.h"
64#include "llvm/Analysis/AliasAnalysis.h"
65#include "llvm/CodeGen/MachineFunctionPass.h"
66#include "llvm/InitializePasses.h"
67
68using namespace llvm;
69
70#define DEBUG_TYPE "si-load-store-opt"
71
72namespace {
73enum InstClassEnum {
74 UNKNOWN,
75 DS_READ,
76 DS_WRITE,
77 S_BUFFER_LOAD_IMM,
78 S_BUFFER_LOAD_SGPR_IMM,
79 S_LOAD_IMM,
80 BUFFER_LOAD,
81 BUFFER_STORE,
82 MIMG,
83 TBUFFER_LOAD,
84 TBUFFER_STORE,
85 GLOBAL_LOAD_SADDR,
86 GLOBAL_STORE_SADDR,
87 FLAT_LOAD,
88 FLAT_STORE,
89 FLAT_LOAD_SADDR,
90 FLAT_STORE_SADDR,
91 GLOBAL_LOAD, // GLOBAL_LOAD/GLOBAL_STORE are never used as the InstClass of
92 GLOBAL_STORE // any CombineInfo, they are only ever returned by
93 // getCommonInstClass.
94};
95
96struct AddressRegs {
97 unsigned char NumVAddrs = 0;
98 bool SBase = false;
99 bool SRsrc = false;
100 bool SOffset = false;
101 bool SAddr = false;
102 bool VAddr = false;
103 bool Addr = false;
104 bool SSamp = false;
105};
106
107// GFX10 image_sample instructions can have 12 vaddrs + srsrc + ssamp.
108const unsigned MaxAddressRegs = 12 + 1 + 1;
109
110class SILoadStoreOptimizer {
111 struct CombineInfo {
112 MachineBasicBlock::iterator I;
113 unsigned EltSize;
114 unsigned Offset;
115 unsigned Width;
116 unsigned Format;
117 unsigned BaseOff;
118 unsigned DMask;
119 InstClassEnum InstClass;
120 unsigned CPol = 0;
121 bool GDS = false;
122 const TargetRegisterClass *DataRC;
123 bool UseST64;
124 int AddrIdx[MaxAddressRegs];
125 const MachineOperand *AddrReg[MaxAddressRegs];
126 unsigned NumAddresses;
127 unsigned Order;
128
129 bool hasSameBaseAddress(const CombineInfo &CI) {
130 if (NumAddresses != CI.NumAddresses)
131 return false;
132
133 const MachineInstr &MI = *CI.I;
134 for (unsigned i = 0; i < NumAddresses; i++) {
135 const MachineOperand &AddrRegNext = MI.getOperand(AddrIdx[i]);
136
137 if (AddrReg[i]->isImm() || AddrRegNext.isImm()) {
138 if (AddrReg[i]->isImm() != AddrRegNext.isImm() ||
139 AddrReg[i]->getImm() != AddrRegNext.getImm()) {
140 return false;
141 }
142 continue;
143 }
144
145 // Check same base pointer. Be careful of subregisters, which can occur
146 // with vectors of pointers.
147 if (AddrReg[i]->getReg() != AddrRegNext.getReg() ||
148 AddrReg[i]->getSubReg() != AddrRegNext.getSubReg()) {
149 return false;
150 }
151 }
152 return true;
153 }
154
155 bool hasMergeableAddress(const MachineRegisterInfo &MRI) {
156 for (unsigned i = 0; i < NumAddresses; ++i) {
157 const MachineOperand *AddrOp = AddrReg[i];
158 // Immediates are always OK.
159 if (AddrOp->isImm())
160 continue;
161
162 // Don't try to merge addresses that aren't either immediates or registers.
163 // TODO: Should be possible to merge FrameIndexes and maybe some other
164 // non-register
165 if (!AddrOp->isReg())
166 return false;
167
168 // TODO: We should be able to merge instructions with other physical reg
169 // addresses too.
170 if (AddrOp->getReg().isPhysical() &&
171 AddrOp->getReg() != AMDGPU::SGPR_NULL)
172 return false;
173
174 // If an address has only one use then there will be no other
175 // instructions with the same address, so we can't merge this one.
176 if (MRI.hasOneNonDBGUse(AddrOp->getReg()))
177 return false;
178 }
179 return true;
180 }
181
182 void setMI(MachineBasicBlock::iterator MI, const SILoadStoreOptimizer &LSO);
183
184 // Compare by pointer order.
185 bool operator<(const CombineInfo& Other) const {
186 return (InstClass == MIMG) ? DMask < Other.DMask : Offset < Other.Offset;
187 }
188 };
189
190 struct BaseRegisters {
191 Register LoReg;
192 Register HiReg;
193
194 unsigned LoSubReg = 0;
195 unsigned HiSubReg = 0;
196 // True when using V_ADD_U64_e64 pattern
197 bool UseV64Pattern = false;
198 // True when using V_ADD_U32_e64 pattern (32-bit FLAT_SCRATCH vaddr)
199 bool UseScratchVGPRPattern = false;
200 };
201
202 struct MemAddress {
203 BaseRegisters Base;
204 int64_t Offset = 0;
205 };
206
207 using MemInfoMap = DenseMap<MachineInstr *, MemAddress>;
208
209private:
210 MachineFunction *MF = nullptr;
211 const GCNSubtarget *STM = nullptr;
212 const SIInstrInfo *TII = nullptr;
213 const SIRegisterInfo *TRI = nullptr;
214 MachineRegisterInfo *MRI = nullptr;
215 AliasAnalysis *AA = nullptr;
216 bool OptimizeAgain;
217
218 bool canSwapInstructions(const DenseSet<Register> &ARegDefs,
219 const DenseSet<Register> &ARegUses,
220 const MachineInstr &A, const MachineInstr &B) const;
221 static bool dmasksCanBeCombined(const CombineInfo &CI,
222 const SIInstrInfo &TII,
223 const CombineInfo &Paired);
224 static bool offsetsCanBeCombined(CombineInfo &CI, const GCNSubtarget &STI,
225 CombineInfo &Paired, bool Modify = false);
226 static bool widthsFit(const GCNSubtarget &STI, const CombineInfo &CI,
227 const CombineInfo &Paired);
228 unsigned getNewOpcode(const CombineInfo &CI, const CombineInfo &Paired);
229 static std::pair<unsigned, unsigned> getSubRegIdxs(const CombineInfo &CI,
230 const CombineInfo &Paired);
231 const TargetRegisterClass *
232 getTargetRegisterClass(const CombineInfo &CI,
233 const CombineInfo &Paired) const;
234 const TargetRegisterClass *getDataRegClass(const MachineInstr &MI) const;
235
236 CombineInfo *checkAndPrepareMerge(CombineInfo &CI, CombineInfo &Paired);
237
238 void copyToDestRegs(CombineInfo &CI, CombineInfo &Paired,
239 MachineBasicBlock::iterator InsertBefore,
240 const DebugLoc &DL, AMDGPU::OpName OpName,
241 Register DestReg) const;
242 Register copyFromSrcRegs(CombineInfo &CI, CombineInfo &Paired,
243 MachineBasicBlock::iterator InsertBefore,
244 const DebugLoc &DL, AMDGPU::OpName OpName) const;
245
246 unsigned read2Opcode(unsigned EltSize, bool GDS) const;
247 unsigned read2ST64Opcode(unsigned EltSize, bool GDS) const;
248 MachineBasicBlock::iterator
249 mergeRead2Pair(CombineInfo &CI, CombineInfo &Paired,
250 MachineBasicBlock::iterator InsertBefore);
251
252 unsigned write2Opcode(unsigned EltSize, bool GDS) const;
253 unsigned write2ST64Opcode(unsigned EltSize, bool GDS) const;
254 unsigned getWrite2Opcode(const CombineInfo &CI) const;
255
256 MachineBasicBlock::iterator
257 mergeWrite2Pair(CombineInfo &CI, CombineInfo &Paired,
258 MachineBasicBlock::iterator InsertBefore);
259 MachineBasicBlock::iterator
260 mergeImagePair(CombineInfo &CI, CombineInfo &Paired,
261 MachineBasicBlock::iterator InsertBefore);
262 MachineBasicBlock::iterator
263 mergeSMemLoadImmPair(CombineInfo &CI, CombineInfo &Paired,
264 MachineBasicBlock::iterator InsertBefore);
265 MachineBasicBlock::iterator
266 mergeBufferLoadPair(CombineInfo &CI, CombineInfo &Paired,
267 MachineBasicBlock::iterator InsertBefore);
268 MachineBasicBlock::iterator
269 mergeBufferStorePair(CombineInfo &CI, CombineInfo &Paired,
270 MachineBasicBlock::iterator InsertBefore);
271 MachineBasicBlock::iterator
272 mergeTBufferLoadPair(CombineInfo &CI, CombineInfo &Paired,
273 MachineBasicBlock::iterator InsertBefore);
274 MachineBasicBlock::iterator
275 mergeTBufferStorePair(CombineInfo &CI, CombineInfo &Paired,
276 MachineBasicBlock::iterator InsertBefore);
277 MachineBasicBlock::iterator
278 mergeFlatLoadPair(CombineInfo &CI, CombineInfo &Paired,
279 MachineBasicBlock::iterator InsertBefore);
280 MachineBasicBlock::iterator
281 mergeFlatStorePair(CombineInfo &CI, CombineInfo &Paired,
282 MachineBasicBlock::iterator InsertBefore);
283
284 void updateBaseAndOffset(MachineInstr &I, Register NewBase,
285 int32_t NewOffset) const;
286 void updateAsyncLDSAddress(MachineInstr &MI, int32_t OffsetDiff) const;
287 Register computeBase(MachineInstr &MI, const MemAddress &Addr) const;
288 MachineOperand createRegOrImm(int32_t Val, MachineInstr &MI) const;
289 bool processBaseWithConstOffset64(MachineInstr *AddDef,
290 const MachineOperand &Base,
291 MemAddress &Addr) const;
292 bool processBaseWithConstOffset32(MachineInstr *AddDef,
293 MemAddress &Addr) const;
294 void processBaseWithConstOffset(const MachineOperand &Base, MemAddress &Addr) const;
295 /// Promotes constant offset to the immediate by adjusting the base. It
296 /// tries to use a base from the nearby instructions that allows it to have
297 /// a 13bit constant offset which gets promoted to the immediate.
298 bool promoteConstantOffsetToImm(MachineInstr &CI,
299 MemInfoMap &Visited,
300 SmallPtrSet<MachineInstr *, 4> &Promoted) const;
301 void addInstToMergeableList(const CombineInfo &CI,
302 std::list<std::list<CombineInfo> > &MergeableInsts) const;
303
304 std::pair<MachineBasicBlock::iterator, bool> collectMergeableInsts(
305 MachineBasicBlock::iterator Begin, MachineBasicBlock::iterator End,
306 MemInfoMap &Visited, SmallPtrSet<MachineInstr *, 4> &AnchorList,
307 std::list<std::list<CombineInfo>> &MergeableInsts) const;
308
309 static MachineMemOperand *combineKnownAdjacentMMOs(const CombineInfo &CI,
310 const CombineInfo &Paired);
311
312 static InstClassEnum getCommonInstClass(const CombineInfo &CI,
313 const CombineInfo &Paired);
314
315 bool optimizeInstsWithSameBaseAddr(std::list<CombineInfo> &MergeList,
316 bool &OptimizeListAgain);
317 bool optimizeBlock(std::list<std::list<CombineInfo> > &MergeableInsts);
318
319public:
320 SILoadStoreOptimizer(AliasAnalysis *AA) : AA(AA) {}
321 bool run(MachineFunction &MF);
322};
323
324class SILoadStoreOptimizerLegacy : public MachineFunctionPass {
325public:
326 static char ID;
327
328 SILoadStoreOptimizerLegacy() : MachineFunctionPass(ID) {}
329
330 bool runOnMachineFunction(MachineFunction &MF) override;
331
332 StringRef getPassName() const override { return "SI Load Store Optimizer"; }
333
334 void getAnalysisUsage(AnalysisUsage &AU) const override {
335 AU.setPreservesCFG();
336 AU.addRequired<AAResultsWrapperPass>();
337
338 MachineFunctionPass::getAnalysisUsage(AU);
339 }
340
341 MachineFunctionProperties getRequiredProperties() const override {
342 return MachineFunctionProperties().setIsSSA();
343 }
344};
345
346static unsigned getOpcodeWidth(const MachineInstr &MI, const SIInstrInfo &TII) {
347 const unsigned Opc = MI.getOpcode();
348
349 if (TII.isMUBUF(Opc)) {
350 // FIXME: Handle d16 correctly
351 return AMDGPU::getMUBUFElements(Opc);
352 }
353 if (TII.isImage(MI)) {
354 uint64_t DMaskImm =
355 TII.getNamedOperand(MI, AMDGPU::OpName::dmask)->getImm();
356 return llvm::popcount(DMaskImm);
357 }
358 if (TII.isMTBUF(Opc)) {
359 return AMDGPU::getMTBUFElements(Opc);
360 }
361
362 switch (Opc) {
363 case AMDGPU::S_BUFFER_LOAD_DWORD_IMM:
364 case AMDGPU::S_BUFFER_LOAD_DWORD_SGPR_IMM:
365 case AMDGPU::S_LOAD_DWORD_IMM:
366 case AMDGPU::GLOBAL_LOAD_DWORD:
367 case AMDGPU::GLOBAL_LOAD_DWORD_SADDR:
368 case AMDGPU::GLOBAL_STORE_DWORD:
369 case AMDGPU::GLOBAL_STORE_DWORD_SADDR:
370 case AMDGPU::FLAT_LOAD_DWORD:
371 case AMDGPU::FLAT_STORE_DWORD:
372 case AMDGPU::FLAT_LOAD_DWORD_SADDR:
373 case AMDGPU::FLAT_STORE_DWORD_SADDR:
374 return 1;
375 case AMDGPU::S_BUFFER_LOAD_DWORDX2_IMM:
376 case AMDGPU::S_BUFFER_LOAD_DWORDX2_SGPR_IMM:
377 case AMDGPU::S_BUFFER_LOAD_DWORDX2_IMM_ec:
378 case AMDGPU::S_BUFFER_LOAD_DWORDX2_SGPR_IMM_ec:
379 case AMDGPU::S_LOAD_DWORDX2_IMM:
380 case AMDGPU::S_LOAD_DWORDX2_IMM_ec:
381 case AMDGPU::GLOBAL_LOAD_DWORDX2:
382 case AMDGPU::GLOBAL_LOAD_DWORDX2_SADDR:
383 case AMDGPU::GLOBAL_STORE_DWORDX2:
384 case AMDGPU::GLOBAL_STORE_DWORDX2_SADDR:
385 case AMDGPU::FLAT_LOAD_DWORDX2:
386 case AMDGPU::FLAT_STORE_DWORDX2:
387 case AMDGPU::FLAT_LOAD_DWORDX2_SADDR:
388 case AMDGPU::FLAT_STORE_DWORDX2_SADDR:
389 return 2;
390 case AMDGPU::S_BUFFER_LOAD_DWORDX3_IMM:
391 case AMDGPU::S_BUFFER_LOAD_DWORDX3_SGPR_IMM:
392 case AMDGPU::S_BUFFER_LOAD_DWORDX3_IMM_ec:
393 case AMDGPU::S_BUFFER_LOAD_DWORDX3_SGPR_IMM_ec:
394 case AMDGPU::S_LOAD_DWORDX3_IMM:
395 case AMDGPU::S_LOAD_DWORDX3_IMM_ec:
396 case AMDGPU::GLOBAL_LOAD_DWORDX3:
397 case AMDGPU::GLOBAL_LOAD_DWORDX3_SADDR:
398 case AMDGPU::GLOBAL_STORE_DWORDX3:
399 case AMDGPU::GLOBAL_STORE_DWORDX3_SADDR:
400 case AMDGPU::FLAT_LOAD_DWORDX3:
401 case AMDGPU::FLAT_STORE_DWORDX3:
402 case AMDGPU::FLAT_LOAD_DWORDX3_SADDR:
403 case AMDGPU::FLAT_STORE_DWORDX3_SADDR:
404 return 3;
405 case AMDGPU::S_BUFFER_LOAD_DWORDX4_IMM:
406 case AMDGPU::S_BUFFER_LOAD_DWORDX4_SGPR_IMM:
407 case AMDGPU::S_BUFFER_LOAD_DWORDX4_IMM_ec:
408 case AMDGPU::S_BUFFER_LOAD_DWORDX4_SGPR_IMM_ec:
409 case AMDGPU::S_LOAD_DWORDX4_IMM:
410 case AMDGPU::S_LOAD_DWORDX4_IMM_ec:
411 case AMDGPU::GLOBAL_LOAD_DWORDX4:
412 case AMDGPU::GLOBAL_LOAD_DWORDX4_SADDR:
413 case AMDGPU::GLOBAL_STORE_DWORDX4:
414 case AMDGPU::GLOBAL_STORE_DWORDX4_SADDR:
415 case AMDGPU::FLAT_LOAD_DWORDX4:
416 case AMDGPU::FLAT_STORE_DWORDX4:
417 case AMDGPU::FLAT_LOAD_DWORDX4_SADDR:
418 case AMDGPU::FLAT_STORE_DWORDX4_SADDR:
419 return 4;
420 case AMDGPU::S_BUFFER_LOAD_DWORDX8_IMM:
421 case AMDGPU::S_BUFFER_LOAD_DWORDX8_SGPR_IMM:
422 case AMDGPU::S_BUFFER_LOAD_DWORDX8_IMM_ec:
423 case AMDGPU::S_BUFFER_LOAD_DWORDX8_SGPR_IMM_ec:
424 case AMDGPU::S_LOAD_DWORDX8_IMM:
425 case AMDGPU::S_LOAD_DWORDX8_IMM_ec:
426 return 8;
427 case AMDGPU::DS_READ_B32:
428 case AMDGPU::DS_READ_B32_gfx9:
429 case AMDGPU::DS_WRITE_B32:
430 case AMDGPU::DS_WRITE_B32_gfx9:
431 return 1;
432 case AMDGPU::DS_READ_B64:
433 case AMDGPU::DS_READ_B64_gfx9:
434 case AMDGPU::DS_WRITE_B64:
435 case AMDGPU::DS_WRITE_B64_gfx9:
436 return 2;
437 default:
438 return 0;
439 }
440}
441
442/// Maps instruction opcode to enum InstClassEnum.
443static InstClassEnum getInstClass(unsigned Opc, const SIInstrInfo &TII) {
444 switch (Opc) {
445 default:
446 if (TII.isMUBUF(Opc)) {
447 switch (AMDGPU::getMUBUFBaseOpcode(Opc)) {
448 default:
449 return UNKNOWN;
450 case AMDGPU::BUFFER_LOAD_DWORD_BOTHEN:
451 case AMDGPU::BUFFER_LOAD_DWORD_BOTHEN_exact:
452 case AMDGPU::BUFFER_LOAD_DWORD_IDXEN:
453 case AMDGPU::BUFFER_LOAD_DWORD_IDXEN_exact:
454 case AMDGPU::BUFFER_LOAD_DWORD_OFFEN:
455 case AMDGPU::BUFFER_LOAD_DWORD_OFFEN_exact:
456 case AMDGPU::BUFFER_LOAD_DWORD_OFFSET:
457 case AMDGPU::BUFFER_LOAD_DWORD_OFFSET_exact:
458 case AMDGPU::BUFFER_LOAD_DWORD_VBUFFER_BOTHEN:
459 case AMDGPU::BUFFER_LOAD_DWORD_VBUFFER_BOTHEN_exact:
460 case AMDGPU::BUFFER_LOAD_DWORD_VBUFFER_IDXEN:
461 case AMDGPU::BUFFER_LOAD_DWORD_VBUFFER_IDXEN_exact:
462 case AMDGPU::BUFFER_LOAD_DWORD_VBUFFER_OFFEN:
463 case AMDGPU::BUFFER_LOAD_DWORD_VBUFFER_OFFEN_exact:
464 case AMDGPU::BUFFER_LOAD_DWORD_VBUFFER_OFFSET:
465 case AMDGPU::BUFFER_LOAD_DWORD_VBUFFER_OFFSET_exact:
466 return BUFFER_LOAD;
467 case AMDGPU::BUFFER_STORE_DWORD_BOTHEN:
468 case AMDGPU::BUFFER_STORE_DWORD_BOTHEN_exact:
469 case AMDGPU::BUFFER_STORE_DWORD_IDXEN:
470 case AMDGPU::BUFFER_STORE_DWORD_IDXEN_exact:
471 case AMDGPU::BUFFER_STORE_DWORD_OFFEN:
472 case AMDGPU::BUFFER_STORE_DWORD_OFFEN_exact:
473 case AMDGPU::BUFFER_STORE_DWORD_OFFSET:
474 case AMDGPU::BUFFER_STORE_DWORD_OFFSET_exact:
475 case AMDGPU::BUFFER_STORE_DWORD_VBUFFER_BOTHEN:
476 case AMDGPU::BUFFER_STORE_DWORD_VBUFFER_BOTHEN_exact:
477 case AMDGPU::BUFFER_STORE_DWORD_VBUFFER_IDXEN:
478 case AMDGPU::BUFFER_STORE_DWORD_VBUFFER_IDXEN_exact:
479 case AMDGPU::BUFFER_STORE_DWORD_VBUFFER_OFFEN:
480 case AMDGPU::BUFFER_STORE_DWORD_VBUFFER_OFFEN_exact:
481 case AMDGPU::BUFFER_STORE_DWORD_VBUFFER_OFFSET:
482 case AMDGPU::BUFFER_STORE_DWORD_VBUFFER_OFFSET_exact:
483 return BUFFER_STORE;
484 }
485 }
486 if (TII.isImage(Opc)) {
487 // Ignore instructions encoded without vaddr.
488 if (!AMDGPU::hasNamedOperand(Opc, AMDGPU::OpName::vaddr) &&
489 !AMDGPU::hasNamedOperand(Opc, AMDGPU::OpName::vaddr0))
490 return UNKNOWN;
491 // Ignore BVH instructions
492 if (AMDGPU::getMIMGBaseOpcode(Opc)->BVH)
493 return UNKNOWN;
494 // TODO: Support IMAGE_GET_RESINFO and IMAGE_GET_LOD.
495 if (TII.get(Opc).mayStore() || !TII.get(Opc).mayLoad() ||
496 TII.isGather4(Opc))
497 return UNKNOWN;
498 return MIMG;
499 }
500 if (TII.isMTBUF(Opc)) {
501 switch (AMDGPU::getMTBUFBaseOpcode(Opc)) {
502 default:
503 return UNKNOWN;
504 case AMDGPU::TBUFFER_LOAD_FORMAT_X_BOTHEN:
505 case AMDGPU::TBUFFER_LOAD_FORMAT_X_BOTHEN_exact:
506 case AMDGPU::TBUFFER_LOAD_FORMAT_X_IDXEN:
507 case AMDGPU::TBUFFER_LOAD_FORMAT_X_IDXEN_exact:
508 case AMDGPU::TBUFFER_LOAD_FORMAT_X_OFFEN:
509 case AMDGPU::TBUFFER_LOAD_FORMAT_X_OFFEN_exact:
510 case AMDGPU::TBUFFER_LOAD_FORMAT_X_OFFSET:
511 case AMDGPU::TBUFFER_LOAD_FORMAT_X_OFFSET_exact:
512 case AMDGPU::TBUFFER_LOAD_FORMAT_X_VBUFFER_BOTHEN:
513 case AMDGPU::TBUFFER_LOAD_FORMAT_X_VBUFFER_BOTHEN_exact:
514 case AMDGPU::TBUFFER_LOAD_FORMAT_X_VBUFFER_IDXEN:
515 case AMDGPU::TBUFFER_LOAD_FORMAT_X_VBUFFER_IDXEN_exact:
516 case AMDGPU::TBUFFER_LOAD_FORMAT_X_VBUFFER_OFFEN:
517 case AMDGPU::TBUFFER_LOAD_FORMAT_X_VBUFFER_OFFEN_exact:
518 case AMDGPU::TBUFFER_LOAD_FORMAT_X_VBUFFER_OFFSET:
519 case AMDGPU::TBUFFER_LOAD_FORMAT_X_VBUFFER_OFFSET_exact:
520 return TBUFFER_LOAD;
521 case AMDGPU::TBUFFER_STORE_FORMAT_X_OFFEN:
522 case AMDGPU::TBUFFER_STORE_FORMAT_X_OFFEN_exact:
523 case AMDGPU::TBUFFER_STORE_FORMAT_X_OFFSET:
524 case AMDGPU::TBUFFER_STORE_FORMAT_X_OFFSET_exact:
525 case AMDGPU::TBUFFER_STORE_FORMAT_X_VBUFFER_OFFEN:
526 case AMDGPU::TBUFFER_STORE_FORMAT_X_VBUFFER_OFFEN_exact:
527 case AMDGPU::TBUFFER_STORE_FORMAT_X_VBUFFER_OFFSET:
528 case AMDGPU::TBUFFER_STORE_FORMAT_X_VBUFFER_OFFSET_exact:
529 return TBUFFER_STORE;
530 }
531 }
532 return UNKNOWN;
533 case AMDGPU::S_BUFFER_LOAD_DWORD_IMM:
534 case AMDGPU::S_BUFFER_LOAD_DWORDX2_IMM:
535 case AMDGPU::S_BUFFER_LOAD_DWORDX3_IMM:
536 case AMDGPU::S_BUFFER_LOAD_DWORDX4_IMM:
537 case AMDGPU::S_BUFFER_LOAD_DWORDX8_IMM:
538 case AMDGPU::S_BUFFER_LOAD_DWORDX2_IMM_ec:
539 case AMDGPU::S_BUFFER_LOAD_DWORDX3_IMM_ec:
540 case AMDGPU::S_BUFFER_LOAD_DWORDX4_IMM_ec:
541 case AMDGPU::S_BUFFER_LOAD_DWORDX8_IMM_ec:
542 return S_BUFFER_LOAD_IMM;
543 case AMDGPU::S_BUFFER_LOAD_DWORD_SGPR_IMM:
544 case AMDGPU::S_BUFFER_LOAD_DWORDX2_SGPR_IMM:
545 case AMDGPU::S_BUFFER_LOAD_DWORDX3_SGPR_IMM:
546 case AMDGPU::S_BUFFER_LOAD_DWORDX4_SGPR_IMM:
547 case AMDGPU::S_BUFFER_LOAD_DWORDX8_SGPR_IMM:
548 case AMDGPU::S_BUFFER_LOAD_DWORDX2_SGPR_IMM_ec:
549 case AMDGPU::S_BUFFER_LOAD_DWORDX3_SGPR_IMM_ec:
550 case AMDGPU::S_BUFFER_LOAD_DWORDX4_SGPR_IMM_ec:
551 case AMDGPU::S_BUFFER_LOAD_DWORDX8_SGPR_IMM_ec:
552 return S_BUFFER_LOAD_SGPR_IMM;
553 case AMDGPU::S_LOAD_DWORD_IMM:
554 case AMDGPU::S_LOAD_DWORDX2_IMM:
555 case AMDGPU::S_LOAD_DWORDX3_IMM:
556 case AMDGPU::S_LOAD_DWORDX4_IMM:
557 case AMDGPU::S_LOAD_DWORDX8_IMM:
558 case AMDGPU::S_LOAD_DWORDX2_IMM_ec:
559 case AMDGPU::S_LOAD_DWORDX3_IMM_ec:
560 case AMDGPU::S_LOAD_DWORDX4_IMM_ec:
561 case AMDGPU::S_LOAD_DWORDX8_IMM_ec:
562 return S_LOAD_IMM;
563 case AMDGPU::DS_READ_B32:
564 case AMDGPU::DS_READ_B32_gfx9:
565 case AMDGPU::DS_READ_B64:
566 case AMDGPU::DS_READ_B64_gfx9:
567 return DS_READ;
568 case AMDGPU::DS_WRITE_B32:
569 case AMDGPU::DS_WRITE_B32_gfx9:
570 case AMDGPU::DS_WRITE_B64:
571 case AMDGPU::DS_WRITE_B64_gfx9:
572 return DS_WRITE;
573 case AMDGPU::GLOBAL_LOAD_DWORD:
574 case AMDGPU::GLOBAL_LOAD_DWORDX2:
575 case AMDGPU::GLOBAL_LOAD_DWORDX3:
576 case AMDGPU::GLOBAL_LOAD_DWORDX4:
577 case AMDGPU::FLAT_LOAD_DWORD:
578 case AMDGPU::FLAT_LOAD_DWORDX2:
579 case AMDGPU::FLAT_LOAD_DWORDX3:
580 case AMDGPU::FLAT_LOAD_DWORDX4:
581 return FLAT_LOAD;
582 case AMDGPU::GLOBAL_LOAD_DWORD_SADDR:
583 case AMDGPU::GLOBAL_LOAD_DWORDX2_SADDR:
584 case AMDGPU::GLOBAL_LOAD_DWORDX3_SADDR:
585 case AMDGPU::GLOBAL_LOAD_DWORDX4_SADDR:
586 return GLOBAL_LOAD_SADDR;
587 case AMDGPU::GLOBAL_STORE_DWORD:
588 case AMDGPU::GLOBAL_STORE_DWORDX2:
589 case AMDGPU::GLOBAL_STORE_DWORDX3:
590 case AMDGPU::GLOBAL_STORE_DWORDX4:
591 case AMDGPU::FLAT_STORE_DWORD:
592 case AMDGPU::FLAT_STORE_DWORDX2:
593 case AMDGPU::FLAT_STORE_DWORDX3:
594 case AMDGPU::FLAT_STORE_DWORDX4:
595 return FLAT_STORE;
596 case AMDGPU::GLOBAL_STORE_DWORD_SADDR:
597 case AMDGPU::GLOBAL_STORE_DWORDX2_SADDR:
598 case AMDGPU::GLOBAL_STORE_DWORDX3_SADDR:
599 case AMDGPU::GLOBAL_STORE_DWORDX4_SADDR:
600 return GLOBAL_STORE_SADDR;
601 case AMDGPU::FLAT_LOAD_DWORD_SADDR:
602 case AMDGPU::FLAT_LOAD_DWORDX2_SADDR:
603 case AMDGPU::FLAT_LOAD_DWORDX3_SADDR:
604 case AMDGPU::FLAT_LOAD_DWORDX4_SADDR:
605 return FLAT_LOAD_SADDR;
606 case AMDGPU::FLAT_STORE_DWORD_SADDR:
607 case AMDGPU::FLAT_STORE_DWORDX2_SADDR:
608 case AMDGPU::FLAT_STORE_DWORDX3_SADDR:
609 case AMDGPU::FLAT_STORE_DWORDX4_SADDR:
610 return FLAT_STORE_SADDR;
611 }
612}
613
614/// Determines instruction subclass from opcode. Only instructions
615/// of the same subclass can be merged together. The merged instruction may have
616/// a different subclass but must have the same class.
617static unsigned getInstSubclass(unsigned Opc, const SIInstrInfo &TII) {
618 switch (Opc) {
619 default:
620 if (TII.isMUBUF(Opc))
621 return AMDGPU::getMUBUFBaseOpcode(Opc);
622 if (TII.isImage(Opc)) {
623 const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(Opc);
624 assert(Info);
625 return Info->BaseOpcode;
626 }
627 if (TII.isMTBUF(Opc))
628 return AMDGPU::getMTBUFBaseOpcode(Opc);
629 return -1;
630 case AMDGPU::DS_READ_B32:
631 case AMDGPU::DS_READ_B32_gfx9:
632 case AMDGPU::DS_READ_B64:
633 case AMDGPU::DS_READ_B64_gfx9:
634 case AMDGPU::DS_WRITE_B32:
635 case AMDGPU::DS_WRITE_B32_gfx9:
636 case AMDGPU::DS_WRITE_B64:
637 case AMDGPU::DS_WRITE_B64_gfx9:
638 return Opc;
639 case AMDGPU::S_BUFFER_LOAD_DWORD_IMM:
640 case AMDGPU::S_BUFFER_LOAD_DWORDX2_IMM:
641 case AMDGPU::S_BUFFER_LOAD_DWORDX3_IMM:
642 case AMDGPU::S_BUFFER_LOAD_DWORDX4_IMM:
643 case AMDGPU::S_BUFFER_LOAD_DWORDX8_IMM:
644 case AMDGPU::S_BUFFER_LOAD_DWORDX2_IMM_ec:
645 case AMDGPU::S_BUFFER_LOAD_DWORDX3_IMM_ec:
646 case AMDGPU::S_BUFFER_LOAD_DWORDX4_IMM_ec:
647 case AMDGPU::S_BUFFER_LOAD_DWORDX8_IMM_ec:
648 return AMDGPU::S_BUFFER_LOAD_DWORD_IMM;
649 case AMDGPU::S_BUFFER_LOAD_DWORD_SGPR_IMM:
650 case AMDGPU::S_BUFFER_LOAD_DWORDX2_SGPR_IMM:
651 case AMDGPU::S_BUFFER_LOAD_DWORDX3_SGPR_IMM:
652 case AMDGPU::S_BUFFER_LOAD_DWORDX4_SGPR_IMM:
653 case AMDGPU::S_BUFFER_LOAD_DWORDX8_SGPR_IMM:
654 case AMDGPU::S_BUFFER_LOAD_DWORDX2_SGPR_IMM_ec:
655 case AMDGPU::S_BUFFER_LOAD_DWORDX3_SGPR_IMM_ec:
656 case AMDGPU::S_BUFFER_LOAD_DWORDX4_SGPR_IMM_ec:
657 case AMDGPU::S_BUFFER_LOAD_DWORDX8_SGPR_IMM_ec:
658 return AMDGPU::S_BUFFER_LOAD_DWORD_SGPR_IMM;
659 case AMDGPU::S_LOAD_DWORD_IMM:
660 case AMDGPU::S_LOAD_DWORDX2_IMM:
661 case AMDGPU::S_LOAD_DWORDX3_IMM:
662 case AMDGPU::S_LOAD_DWORDX4_IMM:
663 case AMDGPU::S_LOAD_DWORDX8_IMM:
664 case AMDGPU::S_LOAD_DWORDX2_IMM_ec:
665 case AMDGPU::S_LOAD_DWORDX3_IMM_ec:
666 case AMDGPU::S_LOAD_DWORDX4_IMM_ec:
667 case AMDGPU::S_LOAD_DWORDX8_IMM_ec:
668 return AMDGPU::S_LOAD_DWORD_IMM;
669 case AMDGPU::GLOBAL_LOAD_DWORD:
670 case AMDGPU::GLOBAL_LOAD_DWORDX2:
671 case AMDGPU::GLOBAL_LOAD_DWORDX3:
672 case AMDGPU::GLOBAL_LOAD_DWORDX4:
673 case AMDGPU::FLAT_LOAD_DWORD:
674 case AMDGPU::FLAT_LOAD_DWORDX2:
675 case AMDGPU::FLAT_LOAD_DWORDX3:
676 case AMDGPU::FLAT_LOAD_DWORDX4:
677 return AMDGPU::FLAT_LOAD_DWORD;
678 case AMDGPU::GLOBAL_LOAD_DWORD_SADDR:
679 case AMDGPU::GLOBAL_LOAD_DWORDX2_SADDR:
680 case AMDGPU::GLOBAL_LOAD_DWORDX3_SADDR:
681 case AMDGPU::GLOBAL_LOAD_DWORDX4_SADDR:
682 return AMDGPU::GLOBAL_LOAD_DWORD_SADDR;
683 case AMDGPU::GLOBAL_STORE_DWORD:
684 case AMDGPU::GLOBAL_STORE_DWORDX2:
685 case AMDGPU::GLOBAL_STORE_DWORDX3:
686 case AMDGPU::GLOBAL_STORE_DWORDX4:
687 case AMDGPU::FLAT_STORE_DWORD:
688 case AMDGPU::FLAT_STORE_DWORDX2:
689 case AMDGPU::FLAT_STORE_DWORDX3:
690 case AMDGPU::FLAT_STORE_DWORDX4:
691 return AMDGPU::FLAT_STORE_DWORD;
692 case AMDGPU::GLOBAL_STORE_DWORD_SADDR:
693 case AMDGPU::GLOBAL_STORE_DWORDX2_SADDR:
694 case AMDGPU::GLOBAL_STORE_DWORDX3_SADDR:
695 case AMDGPU::GLOBAL_STORE_DWORDX4_SADDR:
696 return AMDGPU::GLOBAL_STORE_DWORD_SADDR;
697 case AMDGPU::FLAT_LOAD_DWORD_SADDR:
698 case AMDGPU::FLAT_LOAD_DWORDX2_SADDR:
699 case AMDGPU::FLAT_LOAD_DWORDX3_SADDR:
700 case AMDGPU::FLAT_LOAD_DWORDX4_SADDR:
701 return AMDGPU::FLAT_LOAD_DWORD_SADDR;
702 case AMDGPU::FLAT_STORE_DWORD_SADDR:
703 case AMDGPU::FLAT_STORE_DWORDX2_SADDR:
704 case AMDGPU::FLAT_STORE_DWORDX3_SADDR:
705 case AMDGPU::FLAT_STORE_DWORDX4_SADDR:
706 return AMDGPU::FLAT_STORE_DWORD_SADDR;
707 }
708}
709
710// GLOBAL loads and stores are classified as FLAT initially. If both combined
711// instructions are FLAT GLOBAL adjust the class to GLOBAL_LOAD or GLOBAL_STORE.
712// If either or both instructions are non segment specific FLAT the resulting
713// combined operation will be FLAT, potentially promoting one of the GLOBAL
714// operations to FLAT.
715// For other instructions return the original unmodified class.
716InstClassEnum
717SILoadStoreOptimizer::getCommonInstClass(const CombineInfo &CI,
718 const CombineInfo &Paired) {
719 assert(CI.InstClass == Paired.InstClass);
720
721 if ((CI.InstClass == FLAT_LOAD || CI.InstClass == FLAT_STORE) &&
722 SIInstrInfo::isFLATGlobal(*CI.I) && SIInstrInfo::isFLATGlobal(*Paired.I))
723 return (CI.InstClass == FLAT_STORE) ? GLOBAL_STORE : GLOBAL_LOAD;
724
725 return CI.InstClass;
726}
727
728static AddressRegs getRegs(unsigned Opc, const SIInstrInfo &TII) {
729 AddressRegs Result;
730
731 if (TII.isMUBUF(Opc)) {
732 if (AMDGPU::getMUBUFHasVAddr(Opc))
733 Result.VAddr = true;
734 if (AMDGPU::getMUBUFHasSrsrc(Opc))
735 Result.SRsrc = true;
736 if (AMDGPU::getMUBUFHasSoffset(Opc))
737 Result.SOffset = true;
738
739 return Result;
740 }
741
742 if (TII.isImage(Opc)) {
743 int VAddr0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vaddr0);
744 if (VAddr0Idx >= 0) {
745 AMDGPU::OpName RsrcName =
746 TII.isMIMG(Opc) ? AMDGPU::OpName::srsrc : AMDGPU::OpName::rsrc;
747 int RsrcIdx = AMDGPU::getNamedOperandIdx(Opc, RsrcName);
748 Result.NumVAddrs = RsrcIdx - VAddr0Idx;
749 } else {
750 Result.VAddr = true;
751 }
752 Result.SRsrc = true;
753 const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(Opc);
754 if (Info && AMDGPU::getMIMGBaseOpcodeInfo(Info->BaseOpcode)->Sampler)
755 Result.SSamp = true;
756
757 return Result;
758 }
759 if (TII.isMTBUF(Opc)) {
760 if (AMDGPU::getMTBUFHasVAddr(Opc))
761 Result.VAddr = true;
762 if (AMDGPU::getMTBUFHasSrsrc(Opc))
763 Result.SRsrc = true;
764 if (AMDGPU::getMTBUFHasSoffset(Opc))
765 Result.SOffset = true;
766
767 return Result;
768 }
769
770 switch (Opc) {
771 default:
772 return Result;
773 case AMDGPU::S_BUFFER_LOAD_DWORD_SGPR_IMM:
774 case AMDGPU::S_BUFFER_LOAD_DWORDX2_SGPR_IMM:
775 case AMDGPU::S_BUFFER_LOAD_DWORDX3_SGPR_IMM:
776 case AMDGPU::S_BUFFER_LOAD_DWORDX4_SGPR_IMM:
777 case AMDGPU::S_BUFFER_LOAD_DWORDX8_SGPR_IMM:
778 case AMDGPU::S_BUFFER_LOAD_DWORDX2_SGPR_IMM_ec:
779 case AMDGPU::S_BUFFER_LOAD_DWORDX3_SGPR_IMM_ec:
780 case AMDGPU::S_BUFFER_LOAD_DWORDX4_SGPR_IMM_ec:
781 case AMDGPU::S_BUFFER_LOAD_DWORDX8_SGPR_IMM_ec:
782 Result.SOffset = true;
783 [[fallthrough]];
784 case AMDGPU::S_BUFFER_LOAD_DWORD_IMM:
785 case AMDGPU::S_BUFFER_LOAD_DWORDX2_IMM:
786 case AMDGPU::S_BUFFER_LOAD_DWORDX3_IMM:
787 case AMDGPU::S_BUFFER_LOAD_DWORDX4_IMM:
788 case AMDGPU::S_BUFFER_LOAD_DWORDX8_IMM:
789 case AMDGPU::S_BUFFER_LOAD_DWORDX2_IMM_ec:
790 case AMDGPU::S_BUFFER_LOAD_DWORDX3_IMM_ec:
791 case AMDGPU::S_BUFFER_LOAD_DWORDX4_IMM_ec:
792 case AMDGPU::S_BUFFER_LOAD_DWORDX8_IMM_ec:
793 case AMDGPU::S_LOAD_DWORD_IMM:
794 case AMDGPU::S_LOAD_DWORDX2_IMM:
795 case AMDGPU::S_LOAD_DWORDX3_IMM:
796 case AMDGPU::S_LOAD_DWORDX4_IMM:
797 case AMDGPU::S_LOAD_DWORDX8_IMM:
798 case AMDGPU::S_LOAD_DWORDX2_IMM_ec:
799 case AMDGPU::S_LOAD_DWORDX3_IMM_ec:
800 case AMDGPU::S_LOAD_DWORDX4_IMM_ec:
801 case AMDGPU::S_LOAD_DWORDX8_IMM_ec:
802 Result.SBase = true;
803 return Result;
804 case AMDGPU::DS_READ_B32:
805 case AMDGPU::DS_READ_B64:
806 case AMDGPU::DS_READ_B32_gfx9:
807 case AMDGPU::DS_READ_B64_gfx9:
808 case AMDGPU::DS_WRITE_B32:
809 case AMDGPU::DS_WRITE_B64:
810 case AMDGPU::DS_WRITE_B32_gfx9:
811 case AMDGPU::DS_WRITE_B64_gfx9:
812 Result.Addr = true;
813 return Result;
814 case AMDGPU::GLOBAL_LOAD_DWORD_SADDR:
815 case AMDGPU::GLOBAL_LOAD_DWORDX2_SADDR:
816 case AMDGPU::GLOBAL_LOAD_DWORDX3_SADDR:
817 case AMDGPU::GLOBAL_LOAD_DWORDX4_SADDR:
818 case AMDGPU::GLOBAL_STORE_DWORD_SADDR:
819 case AMDGPU::GLOBAL_STORE_DWORDX2_SADDR:
820 case AMDGPU::GLOBAL_STORE_DWORDX3_SADDR:
821 case AMDGPU::GLOBAL_STORE_DWORDX4_SADDR:
822 case AMDGPU::FLAT_LOAD_DWORD_SADDR:
823 case AMDGPU::FLAT_LOAD_DWORDX2_SADDR:
824 case AMDGPU::FLAT_LOAD_DWORDX3_SADDR:
825 case AMDGPU::FLAT_LOAD_DWORDX4_SADDR:
826 case AMDGPU::FLAT_STORE_DWORD_SADDR:
827 case AMDGPU::FLAT_STORE_DWORDX2_SADDR:
828 case AMDGPU::FLAT_STORE_DWORDX3_SADDR:
829 case AMDGPU::FLAT_STORE_DWORDX4_SADDR:
830 Result.SAddr = true;
831 [[fallthrough]];
832 case AMDGPU::GLOBAL_LOAD_DWORD:
833 case AMDGPU::GLOBAL_LOAD_DWORDX2:
834 case AMDGPU::GLOBAL_LOAD_DWORDX3:
835 case AMDGPU::GLOBAL_LOAD_DWORDX4:
836 case AMDGPU::GLOBAL_STORE_DWORD:
837 case AMDGPU::GLOBAL_STORE_DWORDX2:
838 case AMDGPU::GLOBAL_STORE_DWORDX3:
839 case AMDGPU::GLOBAL_STORE_DWORDX4:
840 case AMDGPU::FLAT_LOAD_DWORD:
841 case AMDGPU::FLAT_LOAD_DWORDX2:
842 case AMDGPU::FLAT_LOAD_DWORDX3:
843 case AMDGPU::FLAT_LOAD_DWORDX4:
844 case AMDGPU::FLAT_STORE_DWORD:
845 case AMDGPU::FLAT_STORE_DWORDX2:
846 case AMDGPU::FLAT_STORE_DWORDX3:
847 case AMDGPU::FLAT_STORE_DWORDX4:
848 Result.VAddr = true;
849 return Result;
850 }
851}
852
853void SILoadStoreOptimizer::CombineInfo::setMI(MachineBasicBlock::iterator MI,
854 const SILoadStoreOptimizer &LSO) {
855 I = MI;
856 unsigned Opc = MI->getOpcode();
857 InstClass = getInstClass(Opc, *LSO.TII);
858
859 if (InstClass == UNKNOWN)
860 return;
861
862 DataRC = LSO.getDataRegClass(*MI);
863
864 switch (InstClass) {
865 case DS_READ:
866 EltSize =
867 (Opc == AMDGPU::DS_READ_B64 || Opc == AMDGPU::DS_READ_B64_gfx9) ? 8
868 : 4;
869 break;
870 case DS_WRITE:
871 EltSize =
872 (Opc == AMDGPU::DS_WRITE_B64 || Opc == AMDGPU::DS_WRITE_B64_gfx9) ? 8
873 : 4;
874 break;
875 case S_BUFFER_LOAD_IMM:
876 case S_BUFFER_LOAD_SGPR_IMM:
877 case S_LOAD_IMM:
878 EltSize = AMDGPU::convertSMRDOffsetUnits(*LSO.STM, 4);
879 break;
880 default:
881 EltSize = 4;
882 break;
883 }
884
885 if (InstClass == MIMG) {
886 DMask = LSO.TII->getNamedOperand(*I, AMDGPU::OpName::dmask)->getImm();
887 // Offset is not considered for MIMG instructions.
888 Offset = 0;
889 } else {
890 int OffsetIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::offset);
891 Offset = I->getOperand(OffsetIdx).getImm();
892 }
893
894 if (InstClass == TBUFFER_LOAD || InstClass == TBUFFER_STORE) {
895 Format = LSO.TII->getNamedOperand(*I, AMDGPU::OpName::format)->getImm();
896 const AMDGPU::GcnBufferFormatInfo *Info =
897 AMDGPU::getGcnBufferFormatInfo(Format, *LSO.STM);
898 EltSize = Info->BitsPerComp / 8;
899 }
900
901 Width = getOpcodeWidth(*I, *LSO.TII);
902
903 if ((InstClass == DS_READ) || (InstClass == DS_WRITE)) {
904 Offset &= 0xffff;
905 GDS = LSO.TII->getNamedOperand(*I, AMDGPU::OpName::gds)->getImm();
906 } else if (InstClass != MIMG) {
907 CPol = LSO.TII->getNamedOperand(*I, AMDGPU::OpName::cpol)->getImm();
908 }
909
910 AddressRegs Regs = getRegs(Opc, *LSO.TII);
911 bool isVIMAGEorVSAMPLE = LSO.TII->isVIMAGE(*I) || LSO.TII->isVSAMPLE(*I);
912
913 NumAddresses = 0;
914 for (unsigned J = 0; J < Regs.NumVAddrs; J++)
915 AddrIdx[NumAddresses++] =
916 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vaddr0) + J;
917 if (Regs.Addr)
918 AddrIdx[NumAddresses++] =
919 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::addr);
920 if (Regs.SBase)
921 AddrIdx[NumAddresses++] =
922 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::sbase);
923 if (Regs.SRsrc)
924 AddrIdx[NumAddresses++] = AMDGPU::getNamedOperandIdx(
925 Opc, isVIMAGEorVSAMPLE ? AMDGPU::OpName::rsrc : AMDGPU::OpName::srsrc);
926 if (Regs.SOffset)
927 AddrIdx[NumAddresses++] =
928 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::soffset);
929 if (Regs.SAddr)
930 AddrIdx[NumAddresses++] =
931 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::saddr);
932 if (Regs.VAddr)
933 AddrIdx[NumAddresses++] =
934 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vaddr);
935 if (Regs.SSamp)
936 AddrIdx[NumAddresses++] = AMDGPU::getNamedOperandIdx(
937 Opc, isVIMAGEorVSAMPLE ? AMDGPU::OpName::samp : AMDGPU::OpName::ssamp);
938 assert(NumAddresses <= MaxAddressRegs);
939
940 for (unsigned J = 0; J < NumAddresses; J++)
941 AddrReg[J] = &I->getOperand(AddrIdx[J]);
942}
943
944} // end anonymous namespace.
945
946INITIALIZE_PASS_BEGIN(SILoadStoreOptimizerLegacy, DEBUG_TYPE,
947 "SI Load Store Optimizer", false, false)
948INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass)
949INITIALIZE_PASS_END(SILoadStoreOptimizerLegacy, DEBUG_TYPE,
950 "SI Load Store Optimizer", false, false)
951
952char SILoadStoreOptimizerLegacy::ID = 0;
953
954char &llvm::SILoadStoreOptimizerLegacyID = SILoadStoreOptimizerLegacy::ID;
955
956static void addDefsUsesToList(const MachineInstr &MI,
957 DenseSet<Register> &RegDefs,
958 DenseSet<Register> &RegUses) {
959 for (const auto &Op : MI.operands()) {
960 if (!Op.isReg())
961 continue;
962 if (Op.isDef())
963 RegDefs.insert(Op.getReg());
964 if (Op.readsReg())
965 RegUses.insert(Op.getReg());
966 }
967}
968
969bool SILoadStoreOptimizer::canSwapInstructions(
970 const DenseSet<Register> &ARegDefs, const DenseSet<Register> &ARegUses,
971 const MachineInstr &A, const MachineInstr &B) const {
972 if (A.mayLoadOrStore() && B.mayLoadOrStore() &&
973 (A.mayStore() || B.mayStore()) && A.mayAlias(AA, B, true))
974 return false;
975 for (const auto &BOp : B.operands()) {
976 if (!BOp.isReg())
977 continue;
978 if ((BOp.isDef() || BOp.readsReg()) && ARegDefs.contains(BOp.getReg()))
979 return false;
980 if (BOp.isDef() && ARegUses.contains(BOp.getReg()))
981 return false;
982 }
983 return true;
984}
985
986// Given that \p CI and \p Paired are adjacent memory operations produce a new
987// MMO for the combined operation with a new access size.
988MachineMemOperand *
989SILoadStoreOptimizer::combineKnownAdjacentMMOs(const CombineInfo &CI,
990 const CombineInfo &Paired) {
991 const MachineMemOperand *MMOa = *CI.I->memoperands_begin();
992 const MachineMemOperand *MMOb = *Paired.I->memoperands_begin();
993
994 unsigned Size = MMOa->getSize().getValue() + MMOb->getSize().getValue();
995
996 // A base pointer for the combined operation is the same as the leading
997 // operation's pointer.
998 if (Paired < CI)
999 std::swap(MMOa, MMOb);
1000
1001 MachinePointerInfo PtrInfo(MMOa->getPointerInfo());
1002 // If merging FLAT and GLOBAL set address space to FLAT.
1003 if (MMOb->getAddrSpace() == AMDGPUAS::FLAT_ADDRESS)
1004 PtrInfo.AddrSpace = AMDGPUAS::FLAT_ADDRESS;
1005
1006 MachineFunction *MF = CI.I->getMF();
1007 return MF->getMachineMemOperand(MMOa, PtrInfo, Size);
1008}
1009
1010bool SILoadStoreOptimizer::dmasksCanBeCombined(const CombineInfo &CI,
1011 const SIInstrInfo &TII,
1012 const CombineInfo &Paired) {
1013 assert(CI.InstClass == MIMG);
1014
1015 // Ignore instructions with tfe/lwe set.
1016 const auto *TFEOp = TII.getNamedOperand(*CI.I, AMDGPU::OpName::tfe);
1017 const auto *LWEOp = TII.getNamedOperand(*CI.I, AMDGPU::OpName::lwe);
1018
1019 if ((TFEOp && TFEOp->getImm()) || (LWEOp && LWEOp->getImm()))
1020 return false;
1021
1022 // Check other optional immediate operands for equality.
1023 AMDGPU::OpName OperandsToMatch[] = {
1024 AMDGPU::OpName::cpol, AMDGPU::OpName::d16, AMDGPU::OpName::unorm,
1025 AMDGPU::OpName::da, AMDGPU::OpName::r128, AMDGPU::OpName::a16,
1026 AMDGPU::OpName::dim};
1027
1028 for (AMDGPU::OpName op : OperandsToMatch) {
1029 int Idx = AMDGPU::getNamedOperandIdx(CI.I->getOpcode(), op);
1030 if (AMDGPU::getNamedOperandIdx(Paired.I->getOpcode(), op) != Idx)
1031 return false;
1032 if (Idx != -1 &&
1033 CI.I->getOperand(Idx).getImm() != Paired.I->getOperand(Idx).getImm())
1034 return false;
1035 }
1036
1037 // Check DMask for overlaps.
1038 unsigned MaxMask = std::max(CI.DMask, Paired.DMask);
1039 unsigned MinMask = std::min(CI.DMask, Paired.DMask);
1040
1041 if (!MaxMask)
1042 return false;
1043
1044 unsigned AllowedBitsForMin = llvm::countr_zero(MaxMask);
1045 if ((1u << AllowedBitsForMin) <= MinMask)
1046 return false;
1047
1048 return true;
1049}
1050
1051static unsigned getBufferFormatWithCompCount(unsigned OldFormat,
1052 unsigned ComponentCount,
1053 const GCNSubtarget &STI) {
1054 if (ComponentCount > 4)
1055 return 0;
1056
1057 const llvm::AMDGPU::GcnBufferFormatInfo *OldFormatInfo =
1058 llvm::AMDGPU::getGcnBufferFormatInfo(OldFormat, STI);
1059 if (!OldFormatInfo)
1060 return 0;
1061
1062 const llvm::AMDGPU::GcnBufferFormatInfo *NewFormatInfo =
1063 llvm::AMDGPU::getGcnBufferFormatInfo(OldFormatInfo->BitsPerComp,
1064 ComponentCount,
1065 OldFormatInfo->NumFormat, STI);
1066
1067 if (!NewFormatInfo)
1068 return 0;
1069
1070 assert(NewFormatInfo->NumFormat == OldFormatInfo->NumFormat &&
1071 NewFormatInfo->BitsPerComp == OldFormatInfo->BitsPerComp);
1072
1073 return NewFormatInfo->Format;
1074}
1075
1076// Return the value in the inclusive range [Lo,Hi] that is aligned to the
1077// highest power of two. Note that the result is well defined for all inputs
1078// including corner cases like:
1079// - if Lo == Hi, return that value
1080// - if Lo == 0, return 0 (even though the "- 1" below underflows
1081// - if Lo > Hi, return 0 (as if the range wrapped around)
1085
1086bool SILoadStoreOptimizer::offsetsCanBeCombined(CombineInfo &CI,
1087 const GCNSubtarget &STI,
1088 CombineInfo &Paired,
1089 bool Modify) {
1090 assert(CI.InstClass != MIMG);
1091
1092 // XXX - Would the same offset be OK? Is there any reason this would happen or
1093 // be useful?
1094 if (CI.Offset == Paired.Offset)
1095 return false;
1096
1097 if (CI.GDS != Paired.GDS)
1098 return false;
1099
1100 // This won't be valid if the offset isn't aligned.
1101 if ((CI.Offset % CI.EltSize != 0) || (Paired.Offset % CI.EltSize != 0))
1102 return false;
1103
1104 if (CI.InstClass == TBUFFER_LOAD || CI.InstClass == TBUFFER_STORE) {
1105
1106 const llvm::AMDGPU::GcnBufferFormatInfo *Info0 =
1107 llvm::AMDGPU::getGcnBufferFormatInfo(CI.Format, STI);
1108 const llvm::AMDGPU::GcnBufferFormatInfo *Info1 =
1109 llvm::AMDGPU::getGcnBufferFormatInfo(Paired.Format, STI);
1110
1111 if (Info0->BitsPerComp != Info1->BitsPerComp ||
1112 Info0->NumFormat != Info1->NumFormat)
1113 return false;
1114
1115 // For 8-bit or 16-bit formats there is no 3-component variant.
1116 // If NumCombinedComponents is 3, try the 4-component format and use XYZ.
1117 // Example:
1118 // tbuffer_load_format_x + tbuffer_load_format_x + tbuffer_load_format_x
1119 // ==> tbuffer_load_format_xyz with format:[BUF_FMT_16_16_16_16_SNORM]
1120 unsigned NumCombinedComponents = CI.Width + Paired.Width;
1121 if (NumCombinedComponents == 3 && CI.EltSize <= 2)
1122 NumCombinedComponents = 4;
1123
1124 if (getBufferFormatWithCompCount(CI.Format, NumCombinedComponents, STI) ==
1125 0)
1126 return false;
1127
1128 // Merge only when the two access ranges are strictly back-to-back,
1129 // any gap or overlap can over-write data or leave holes.
1130 unsigned ElemIndex0 = CI.Offset / CI.EltSize;
1131 unsigned ElemIndex1 = Paired.Offset / Paired.EltSize;
1132 if (ElemIndex0 + CI.Width != ElemIndex1 &&
1133 ElemIndex1 + Paired.Width != ElemIndex0)
1134 return false;
1135
1136 // 1-byte formats require 1-byte alignment.
1137 // 2-byte formats require 2-byte alignment.
1138 // 4-byte and larger formats require 4-byte alignment.
1139 unsigned MergedBytes = CI.EltSize * NumCombinedComponents;
1140 unsigned RequiredAlign = std::min(MergedBytes, 4u);
1141 unsigned MinOff = std::min(CI.Offset, Paired.Offset);
1142 if (MinOff % RequiredAlign != 0)
1143 return false;
1144
1145 return true;
1146 }
1147
1148 uint32_t EltOffset0 = CI.Offset / CI.EltSize;
1149 uint32_t EltOffset1 = Paired.Offset / CI.EltSize;
1150 CI.UseST64 = false;
1151 CI.BaseOff = 0;
1152
1153 // Handle all non-DS instructions.
1154 if ((CI.InstClass != DS_READ) && (CI.InstClass != DS_WRITE)) {
1155 if (EltOffset0 + CI.Width != EltOffset1 &&
1156 EltOffset1 + Paired.Width != EltOffset0)
1157 return false;
1158 // Instructions with scale_offset modifier cannot be combined unless we
1159 // also generate a code to scale the offset and reset that bit.
1160 if (CI.CPol != Paired.CPol || (CI.CPol & AMDGPU::CPol::SCAL))
1161 return false;
1162 if (CI.InstClass == S_LOAD_IMM || CI.InstClass == S_BUFFER_LOAD_IMM ||
1163 CI.InstClass == S_BUFFER_LOAD_SGPR_IMM) {
1164 // Reject cases like:
1165 // dword + dwordx2 -> dwordx3
1166 // dword + dwordx3 -> dwordx4
1167 // If we tried to combine these cases, we would fail to extract a subreg
1168 // for the result of the second load due to SGPR alignment requirements.
1169 if (CI.Width != Paired.Width &&
1170 (CI.Width < Paired.Width) == (CI.Offset < Paired.Offset))
1171 return false;
1172 }
1173 return true;
1174 }
1175
1176 // If the offset in elements doesn't fit in 8-bits, we might be able to use
1177 // the stride 64 versions.
1178 if ((EltOffset0 % 64 == 0) && (EltOffset1 % 64) == 0 &&
1179 isUInt<8>(EltOffset0 / 64) && isUInt<8>(EltOffset1 / 64)) {
1180 if (Modify) {
1181 CI.Offset = EltOffset0 / 64;
1182 Paired.Offset = EltOffset1 / 64;
1183 CI.UseST64 = true;
1184 }
1185 return true;
1186 }
1187
1188 // Check if the new offsets fit in the reduced 8-bit range.
1189 if (isUInt<8>(EltOffset0) && isUInt<8>(EltOffset1)) {
1190 if (Modify) {
1191 CI.Offset = EltOffset0;
1192 Paired.Offset = EltOffset1;
1193 }
1194 return true;
1195 }
1196
1197 // Try to shift base address to decrease offsets.
1198 uint32_t Min = std::min(EltOffset0, EltOffset1);
1199 uint32_t Max = std::max(EltOffset0, EltOffset1);
1200
1201 const uint32_t Mask = maskTrailingOnes<uint32_t>(8) * 64;
1202 if (((Max - Min) & ~Mask) == 0) {
1203 if (Modify) {
1204 // From the range of values we could use for BaseOff, choose the one that
1205 // is aligned to the highest power of two, to maximise the chance that
1206 // the same offset can be reused for other load/store pairs.
1207 uint32_t BaseOff = mostAlignedValueInRange(Max - 0xff * 64, Min);
1208 // Copy the low bits of the offsets, so that when we adjust them by
1209 // subtracting BaseOff they will be multiples of 64.
1210 BaseOff |= Min & maskTrailingOnes<uint32_t>(6);
1211 CI.BaseOff = BaseOff * CI.EltSize;
1212 CI.Offset = (EltOffset0 - BaseOff) / 64;
1213 Paired.Offset = (EltOffset1 - BaseOff) / 64;
1214 CI.UseST64 = true;
1215 }
1216 return true;
1217 }
1218
1219 if (isUInt<8>(Max - Min)) {
1220 if (Modify) {
1221 // From the range of values we could use for BaseOff, choose the one that
1222 // is aligned to the highest power of two, to maximise the chance that
1223 // the same offset can be reused for other load/store pairs.
1224 uint32_t BaseOff = mostAlignedValueInRange(Max - 0xff, Min);
1225 CI.BaseOff = BaseOff * CI.EltSize;
1226 CI.Offset = EltOffset0 - BaseOff;
1227 Paired.Offset = EltOffset1 - BaseOff;
1228 }
1229 return true;
1230 }
1231
1232 return false;
1233}
1234
1235bool SILoadStoreOptimizer::widthsFit(const GCNSubtarget &STM,
1236 const CombineInfo &CI,
1237 const CombineInfo &Paired) {
1238 const unsigned Width = (CI.Width + Paired.Width);
1239 switch (CI.InstClass) {
1240 default:
1241 return (Width <= 4) && (STM.hasDwordx3LoadStores() || (Width != 3));
1242 case S_BUFFER_LOAD_IMM:
1243 case S_BUFFER_LOAD_SGPR_IMM:
1244 case S_LOAD_IMM:
1245 switch (Width) {
1246 default:
1247 return false;
1248 case 2:
1249 case 4:
1250 case 8:
1251 return true;
1252 case 3:
1253 return STM.hasScalarDwordx3Loads();
1254 }
1255 }
1256}
1257
1258const TargetRegisterClass *
1259SILoadStoreOptimizer::getDataRegClass(const MachineInstr &MI) const {
1260 if (const auto *Dst = TII->getNamedOperand(MI, AMDGPU::OpName::vdst)) {
1261 return TRI->getRegClassForReg(*MRI, Dst->getReg());
1262 }
1263 if (const auto *Src = TII->getNamedOperand(MI, AMDGPU::OpName::vdata)) {
1264 return TRI->getRegClassForReg(*MRI, Src->getReg());
1265 }
1266 if (const auto *Src = TII->getNamedOperand(MI, AMDGPU::OpName::data0)) {
1267 return TRI->getRegClassForReg(*MRI, Src->getReg());
1268 }
1269 if (const auto *Dst = TII->getNamedOperand(MI, AMDGPU::OpName::sdst)) {
1270 return TRI->getRegClassForReg(*MRI, Dst->getReg());
1271 }
1272 if (const auto *Src = TII->getNamedOperand(MI, AMDGPU::OpName::sdata)) {
1273 return TRI->getRegClassForReg(*MRI, Src->getReg());
1274 }
1275 return nullptr;
1276}
1277
1278/// This function assumes that CI comes before Paired in a basic block. Return
1279/// an insertion point for the merged instruction or nullptr on failure.
1280SILoadStoreOptimizer::CombineInfo *
1281SILoadStoreOptimizer::checkAndPrepareMerge(CombineInfo &CI,
1282 CombineInfo &Paired) {
1283 // If another instruction has already been merged into CI, it may now be a
1284 // type that we can't do any further merging into.
1285 if (CI.InstClass == UNKNOWN || Paired.InstClass == UNKNOWN)
1286 return nullptr;
1287 assert(CI.InstClass == Paired.InstClass);
1288
1289 if (getInstSubclass(CI.I->getOpcode(), *TII) !=
1290 getInstSubclass(Paired.I->getOpcode(), *TII))
1291 return nullptr;
1292
1293 // Check both offsets (or masks for MIMG) can be combined and fit in the
1294 // reduced range.
1295 if (CI.InstClass == MIMG) {
1296 if (!dmasksCanBeCombined(CI, *TII, Paired))
1297 return nullptr;
1298 } else {
1299 if (!widthsFit(*STM, CI, Paired) || !offsetsCanBeCombined(CI, *STM, Paired))
1300 return nullptr;
1301 }
1302
1303 DenseSet<Register> RegDefs;
1304 DenseSet<Register> RegUses;
1305 CombineInfo *Where;
1306 if (CI.I->mayLoad()) {
1307 // Try to hoist Paired up to CI.
1308 addDefsUsesToList(*Paired.I, RegDefs, RegUses);
1309 for (MachineBasicBlock::iterator MBBI = Paired.I; --MBBI != CI.I;) {
1310 if (!canSwapInstructions(RegDefs, RegUses, *Paired.I, *MBBI))
1311 return nullptr;
1312 }
1313 Where = &CI;
1314 } else {
1315 // Try to sink CI down to Paired.
1316 addDefsUsesToList(*CI.I, RegDefs, RegUses);
1317 for (MachineBasicBlock::iterator MBBI = CI.I; ++MBBI != Paired.I;) {
1318 if (!canSwapInstructions(RegDefs, RegUses, *CI.I, *MBBI))
1319 return nullptr;
1320 }
1321 Where = &Paired;
1322 }
1323
1324 // Call offsetsCanBeCombined with modify = true so that the offsets are
1325 // correct for the new instruction. This should return true, because
1326 // this function should only be called on CombineInfo objects that
1327 // have already been confirmed to be mergeable.
1328 if (CI.InstClass == DS_READ || CI.InstClass == DS_WRITE) {
1329 if (STM->hasNeedsAligned2addrDS() &&
1330 (CI.I->memoperands_empty() ||
1331 (*CI.I->memoperands_begin())->getAlign().value() < CI.Width * 4))
1332 return nullptr;
1333 offsetsCanBeCombined(CI, *STM, Paired, true);
1334 }
1335
1336 if (CI.InstClass == DS_WRITE) {
1337 // Both data operands must be AGPR or VGPR, so the data registers needs to
1338 // be constrained to one or the other. We expect to only emit the VGPR form
1339 // here for now.
1340 //
1341 // FIXME: There is currently a hack in getRegClass to report that the write2
1342 // operands are VGPRs. In the future we should have separate agpr
1343 // instruction definitions.
1344 const MachineOperand *Data0 =
1345 TII->getNamedOperand(*CI.I, AMDGPU::OpName::data0);
1346 const MachineOperand *Data1 =
1347 TII->getNamedOperand(*Paired.I, AMDGPU::OpName::data0);
1348
1349 const MCInstrDesc &Write2Opc = TII->get(getWrite2Opcode(CI));
1350 int Data0Idx = AMDGPU::getNamedOperandIdx(Write2Opc.getOpcode(),
1351 AMDGPU::OpName::data0);
1352 int Data1Idx = AMDGPU::getNamedOperandIdx(Write2Opc.getOpcode(),
1353 AMDGPU::OpName::data1);
1354
1355 const TargetRegisterClass *DataRC0 = TII->getRegClass(Write2Opc, Data0Idx);
1356
1357 const TargetRegisterClass *DataRC1 = TII->getRegClass(Write2Opc, Data1Idx);
1358
1359 if (unsigned SubReg = Data0->getSubReg()) {
1360 DataRC0 = TRI->getMatchingSuperRegClass(MRI->getRegClass(Data0->getReg()),
1361 DataRC0, SubReg);
1362 }
1363
1364 if (unsigned SubReg = Data1->getSubReg()) {
1365 DataRC1 = TRI->getMatchingSuperRegClass(MRI->getRegClass(Data1->getReg()),
1366 DataRC1, SubReg);
1367 }
1368
1369 if (!MRI->constrainRegClass(Data0->getReg(), DataRC0) ||
1370 !MRI->constrainRegClass(Data1->getReg(), DataRC1))
1371 return nullptr;
1372
1373 // TODO: If one register can be constrained, and not the other, insert a
1374 // copy.
1375 }
1376
1377 return Where;
1378}
1379
1380// Copy the merged load result from DestReg to the original dest regs of CI and
1381// Paired.
1382void SILoadStoreOptimizer::copyToDestRegs(
1383 CombineInfo &CI, CombineInfo &Paired,
1384 MachineBasicBlock::iterator InsertBefore, const DebugLoc &DL,
1385 AMDGPU::OpName OpName, Register DestReg) const {
1386 MachineBasicBlock *MBB = CI.I->getParent();
1387
1388 auto [SubRegIdx0, SubRegIdx1] = getSubRegIdxs(CI, Paired);
1389
1390 // Copy to the old destination registers.
1391 const MCInstrDesc &CopyDesc = TII->get(TargetOpcode::COPY);
1392 auto *Dest0 = TII->getNamedOperand(*CI.I, OpName);
1393 auto *Dest1 = TII->getNamedOperand(*Paired.I, OpName);
1394
1395 // The constrained sload instructions in S_LOAD_IMM class will have
1396 // `early-clobber` flag in the dst operand. Remove the flag before using the
1397 // MOs in copies.
1398 Dest0->setIsEarlyClobber(false);
1399 Dest1->setIsEarlyClobber(false);
1400
1401 BuildMI(*MBB, InsertBefore, DL, CopyDesc)
1402 .add(*Dest0) // Copy to same destination including flags and sub reg.
1403 .addReg(DestReg, {}, SubRegIdx0);
1404 BuildMI(*MBB, InsertBefore, DL, CopyDesc)
1405 .add(*Dest1)
1406 .addReg(DestReg, {}, SubRegIdx1);
1407}
1408
1409// Return a register for the source of the merged store after copying the
1410// original source regs of CI and Paired into it.
1411Register
1412SILoadStoreOptimizer::copyFromSrcRegs(CombineInfo &CI, CombineInfo &Paired,
1413 MachineBasicBlock::iterator InsertBefore,
1414 const DebugLoc &DL,
1415 AMDGPU::OpName OpName) const {
1416 MachineBasicBlock *MBB = CI.I->getParent();
1417
1418 auto [SubRegIdx0, SubRegIdx1] = getSubRegIdxs(CI, Paired);
1419
1420 // Copy to the new source register.
1421 const TargetRegisterClass *SuperRC = getTargetRegisterClass(CI, Paired);
1422 Register SrcReg = MRI->createVirtualRegister(SuperRC);
1423
1424 const auto *Src0 = TII->getNamedOperand(*CI.I, OpName);
1425 const auto *Src1 = TII->getNamedOperand(*Paired.I, OpName);
1426
1427 BuildMI(*MBB, InsertBefore, DL, TII->get(AMDGPU::REG_SEQUENCE), SrcReg)
1428 .add(*Src0)
1429 .addImm(SubRegIdx0)
1430 .add(*Src1)
1431 .addImm(SubRegIdx1);
1432
1433 return SrcReg;
1434}
1435
1436unsigned SILoadStoreOptimizer::read2Opcode(unsigned EltSize, bool GDS) const {
1437 if (GDS || STM->ldsRequiresM0Init())
1438 return (EltSize == 4) ? AMDGPU::DS_READ2_B32 : AMDGPU::DS_READ2_B64;
1439 return (EltSize == 4) ? AMDGPU::DS_READ2_B32_gfx9 : AMDGPU::DS_READ2_B64_gfx9;
1440}
1441
1442unsigned SILoadStoreOptimizer::read2ST64Opcode(unsigned EltSize,
1443 bool GDS) const {
1444 if (GDS || STM->ldsRequiresM0Init())
1445 return (EltSize == 4) ? AMDGPU::DS_READ2ST64_B32 : AMDGPU::DS_READ2ST64_B64;
1446
1447 return (EltSize == 4) ? AMDGPU::DS_READ2ST64_B32_gfx9
1448 : AMDGPU::DS_READ2ST64_B64_gfx9;
1449}
1450
1451MachineBasicBlock::iterator
1452SILoadStoreOptimizer::mergeRead2Pair(CombineInfo &CI, CombineInfo &Paired,
1453 MachineBasicBlock::iterator InsertBefore) {
1454 MachineBasicBlock *MBB = CI.I->getParent();
1455
1456 // Be careful, since the addresses could be subregisters themselves in weird
1457 // cases, like vectors of pointers.
1458 const auto *AddrReg = TII->getNamedOperand(*CI.I, AMDGPU::OpName::addr);
1459
1460 unsigned NewOffset0 = std::min(CI.Offset, Paired.Offset);
1461 unsigned NewOffset1 = std::max(CI.Offset, Paired.Offset);
1462 unsigned Opc = CI.UseST64 ? read2ST64Opcode(CI.EltSize, CI.GDS)
1463 : read2Opcode(CI.EltSize, CI.GDS);
1464
1465 assert((isUInt<8>(NewOffset0) && isUInt<8>(NewOffset1)) &&
1466 (NewOffset0 != NewOffset1) && "Computed offset doesn't fit");
1467
1468 const MCInstrDesc &Read2Desc = TII->get(Opc);
1469
1470 const TargetRegisterClass *SuperRC = getTargetRegisterClass(CI, Paired);
1471 Register DestReg = MRI->createVirtualRegister(SuperRC);
1472
1473 DebugLoc DL =
1474 DebugLoc::getMergedLocation(CI.I->getDebugLoc(), Paired.I->getDebugLoc());
1475
1476 Register BaseReg = AddrReg->getReg();
1477 unsigned BaseSubReg = AddrReg->getSubReg();
1478 if (CI.BaseOff) {
1479 Register ImmReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass);
1480 BuildMI(*MBB, InsertBefore, DL, TII->get(AMDGPU::S_MOV_B32), ImmReg)
1481 .addImm(CI.BaseOff);
1482
1483 BaseReg = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
1484
1485 TII->getAddNoCarry(*MBB, InsertBefore, DL, BaseReg)
1486 .addReg(ImmReg)
1487 .addReg(AddrReg->getReg(), {}, BaseSubReg)
1488 .addImm(0); // clamp bit
1489 BaseSubReg = 0;
1490 }
1491
1492 MachineInstrBuilder Read2 =
1493 BuildMI(*MBB, InsertBefore, DL, Read2Desc, DestReg)
1494 .addReg(BaseReg, {}, BaseSubReg) // addr
1495 .addImm(NewOffset0) // offset0
1496 .addImm(NewOffset1) // offset1
1497 .addImm(CI.GDS) // gds
1498 .cloneMergedMemRefs({&*CI.I, &*Paired.I});
1499
1500 copyToDestRegs(CI, Paired, InsertBefore, DL, AMDGPU::OpName::vdst, DestReg);
1501
1502 CI.I->eraseFromParent();
1503 Paired.I->eraseFromParent();
1504
1505 LLVM_DEBUG(dbgs() << "Inserted read2: " << *Read2 << '\n');
1506 return Read2;
1507}
1508
1509unsigned SILoadStoreOptimizer::write2Opcode(unsigned EltSize, bool GDS) const {
1510 if (GDS || STM->ldsRequiresM0Init())
1511 return (EltSize == 4) ? AMDGPU::DS_WRITE2_B32 : AMDGPU::DS_WRITE2_B64;
1512 return (EltSize == 4) ? AMDGPU::DS_WRITE2_B32_gfx9
1513 : AMDGPU::DS_WRITE2_B64_gfx9;
1514}
1515
1516unsigned SILoadStoreOptimizer::write2ST64Opcode(unsigned EltSize,
1517 bool GDS) const {
1518 if (GDS || STM->ldsRequiresM0Init())
1519 return (EltSize == 4) ? AMDGPU::DS_WRITE2ST64_B32
1520 : AMDGPU::DS_WRITE2ST64_B64;
1521
1522 return (EltSize == 4) ? AMDGPU::DS_WRITE2ST64_B32_gfx9
1523 : AMDGPU::DS_WRITE2ST64_B64_gfx9;
1524}
1525
1526unsigned SILoadStoreOptimizer::getWrite2Opcode(const CombineInfo &CI) const {
1527 return CI.UseST64 ? write2ST64Opcode(CI.EltSize, CI.GDS)
1528 : write2Opcode(CI.EltSize, CI.GDS);
1529}
1530
1531MachineBasicBlock::iterator SILoadStoreOptimizer::mergeWrite2Pair(
1532 CombineInfo &CI, CombineInfo &Paired,
1533 MachineBasicBlock::iterator InsertBefore) {
1534 MachineBasicBlock *MBB = CI.I->getParent();
1535
1536 // Be sure to use .addOperand(), and not .addReg() with these. We want to be
1537 // sure we preserve the subregister index and any register flags set on them.
1538 const MachineOperand *AddrReg =
1539 TII->getNamedOperand(*CI.I, AMDGPU::OpName::addr);
1540 const MachineOperand *Data0 =
1541 TII->getNamedOperand(*CI.I, AMDGPU::OpName::data0);
1542 const MachineOperand *Data1 =
1543 TII->getNamedOperand(*Paired.I, AMDGPU::OpName::data0);
1544
1545 unsigned NewOffset0 = CI.Offset;
1546 unsigned NewOffset1 = Paired.Offset;
1547 unsigned Opc = getWrite2Opcode(CI);
1548
1549 if (NewOffset0 > NewOffset1) {
1550 // Canonicalize the merged instruction so the smaller offset comes first.
1551 std::swap(NewOffset0, NewOffset1);
1552 std::swap(Data0, Data1);
1553 }
1554
1555 assert((isUInt<8>(NewOffset0) && isUInt<8>(NewOffset1)) &&
1556 (NewOffset0 != NewOffset1) && "Computed offset doesn't fit");
1557
1558 const MCInstrDesc &Write2Desc = TII->get(Opc);
1559 DebugLoc DL =
1560 DebugLoc::getMergedLocation(CI.I->getDebugLoc(), Paired.I->getDebugLoc());
1561
1562 Register BaseReg = AddrReg->getReg();
1563 unsigned BaseSubReg = AddrReg->getSubReg();
1564 if (CI.BaseOff) {
1565 Register ImmReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass);
1566 BuildMI(*MBB, InsertBefore, DL, TII->get(AMDGPU::S_MOV_B32), ImmReg)
1567 .addImm(CI.BaseOff);
1568
1569 BaseReg = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
1570
1571 TII->getAddNoCarry(*MBB, InsertBefore, DL, BaseReg)
1572 .addReg(ImmReg)
1573 .addReg(AddrReg->getReg(), {}, BaseSubReg)
1574 .addImm(0); // clamp bit
1575 BaseSubReg = 0;
1576 }
1577
1578 MachineInstrBuilder Write2 = BuildMI(*MBB, InsertBefore, DL, Write2Desc)
1579 .addReg(BaseReg, {}, BaseSubReg) // addr
1580 .add(*Data0) // data0
1581 .add(*Data1) // data1
1582 .addImm(NewOffset0) // offset0
1583 .addImm(NewOffset1) // offset1
1584 .addImm(CI.GDS) // gds
1585 .cloneMergedMemRefs({&*CI.I, &*Paired.I});
1586
1587 CI.I->eraseFromParent();
1588 Paired.I->eraseFromParent();
1589
1590 LLVM_DEBUG(dbgs() << "Inserted write2 inst: " << *Write2 << '\n');
1591 return Write2;
1592}
1593
1594MachineBasicBlock::iterator
1595SILoadStoreOptimizer::mergeImagePair(CombineInfo &CI, CombineInfo &Paired,
1596 MachineBasicBlock::iterator InsertBefore) {
1597 MachineBasicBlock *MBB = CI.I->getParent();
1598 DebugLoc DL =
1599 DebugLoc::getMergedLocation(CI.I->getDebugLoc(), Paired.I->getDebugLoc());
1600
1601 const unsigned Opcode = getNewOpcode(CI, Paired);
1602
1603 const TargetRegisterClass *SuperRC = getTargetRegisterClass(CI, Paired);
1604
1605 Register DestReg = MRI->createVirtualRegister(SuperRC);
1606 unsigned MergedDMask = CI.DMask | Paired.DMask;
1607 unsigned DMaskIdx =
1608 AMDGPU::getNamedOperandIdx(CI.I->getOpcode(), AMDGPU::OpName::dmask);
1609
1610 auto MIB = BuildMI(*MBB, InsertBefore, DL, TII->get(Opcode), DestReg);
1611 for (unsigned I = 1, E = (*CI.I).getNumOperands(); I != E; ++I) {
1612 if (I == DMaskIdx)
1613 MIB.addImm(MergedDMask);
1614 else
1615 MIB.add((*CI.I).getOperand(I));
1616 }
1617
1618 // It shouldn't be possible to get this far if the two instructions
1619 // don't have a single memoperand, because MachineInstr::mayAlias()
1620 // will return true if this is the case.
1621 assert(CI.I->hasOneMemOperand() && Paired.I->hasOneMemOperand());
1622
1623 MachineInstr *New = MIB.addMemOperand(combineKnownAdjacentMMOs(CI, Paired));
1624
1625 copyToDestRegs(CI, Paired, InsertBefore, DL, AMDGPU::OpName::vdata, DestReg);
1626
1627 CI.I->eraseFromParent();
1628 Paired.I->eraseFromParent();
1629 return New;
1630}
1631
1632MachineBasicBlock::iterator SILoadStoreOptimizer::mergeSMemLoadImmPair(
1633 CombineInfo &CI, CombineInfo &Paired,
1634 MachineBasicBlock::iterator InsertBefore) {
1635 MachineBasicBlock *MBB = CI.I->getParent();
1636 DebugLoc DL =
1637 DebugLoc::getMergedLocation(CI.I->getDebugLoc(), Paired.I->getDebugLoc());
1638
1639 const unsigned Opcode = getNewOpcode(CI, Paired);
1640
1641 const TargetRegisterClass *SuperRC = getTargetRegisterClass(CI, Paired);
1642
1643 Register DestReg = MRI->createVirtualRegister(SuperRC);
1644 unsigned MergedOffset = std::min(CI.Offset, Paired.Offset);
1645
1646 // It shouldn't be possible to get this far if the two instructions
1647 // don't have a single memoperand, because MachineInstr::mayAlias()
1648 // will return true if this is the case.
1649 assert(CI.I->hasOneMemOperand() && Paired.I->hasOneMemOperand());
1650
1651 MachineInstrBuilder New =
1652 BuildMI(*MBB, InsertBefore, DL, TII->get(Opcode), DestReg)
1653 .add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::sbase));
1654 if (CI.InstClass == S_BUFFER_LOAD_SGPR_IMM)
1655 New.add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::soffset));
1656 New.addImm(MergedOffset);
1657 New.addImm(CI.CPol).addMemOperand(combineKnownAdjacentMMOs(CI, Paired));
1658
1659 copyToDestRegs(CI, Paired, InsertBefore, DL, AMDGPU::OpName::sdst, DestReg);
1660
1661 CI.I->eraseFromParent();
1662 Paired.I->eraseFromParent();
1663 return New;
1664}
1665
1666MachineBasicBlock::iterator SILoadStoreOptimizer::mergeBufferLoadPair(
1667 CombineInfo &CI, CombineInfo &Paired,
1668 MachineBasicBlock::iterator InsertBefore) {
1669 MachineBasicBlock *MBB = CI.I->getParent();
1670
1671 DebugLoc DL =
1672 DebugLoc::getMergedLocation(CI.I->getDebugLoc(), Paired.I->getDebugLoc());
1673
1674 const unsigned Opcode = getNewOpcode(CI, Paired);
1675
1676 const TargetRegisterClass *SuperRC = getTargetRegisterClass(CI, Paired);
1677
1678 // Copy to the new source register.
1679 Register DestReg = MRI->createVirtualRegister(SuperRC);
1680 unsigned MergedOffset = std::min(CI.Offset, Paired.Offset);
1681
1682 auto MIB = BuildMI(*MBB, InsertBefore, DL, TII->get(Opcode), DestReg);
1683
1684 AddressRegs Regs = getRegs(Opcode, *TII);
1685
1686 if (Regs.VAddr)
1687 MIB.add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::vaddr));
1688
1689 // It shouldn't be possible to get this far if the two instructions
1690 // don't have a single memoperand, because MachineInstr::mayAlias()
1691 // will return true if this is the case.
1692 assert(CI.I->hasOneMemOperand() && Paired.I->hasOneMemOperand());
1693
1694 MachineInstr *New =
1695 MIB.add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::srsrc))
1696 .add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::soffset))
1697 .addImm(MergedOffset) // offset
1698 .addImm(CI.CPol) // cpol
1699 .addImm(0) // swz
1700 .addMemOperand(combineKnownAdjacentMMOs(CI, Paired));
1701
1702 copyToDestRegs(CI, Paired, InsertBefore, DL, AMDGPU::OpName::vdata, DestReg);
1703
1704 CI.I->eraseFromParent();
1705 Paired.I->eraseFromParent();
1706 return New;
1707}
1708
1709MachineBasicBlock::iterator SILoadStoreOptimizer::mergeTBufferLoadPair(
1710 CombineInfo &CI, CombineInfo &Paired,
1711 MachineBasicBlock::iterator InsertBefore) {
1712 MachineBasicBlock *MBB = CI.I->getParent();
1713
1714 DebugLoc DL =
1715 DebugLoc::getMergedLocation(CI.I->getDebugLoc(), Paired.I->getDebugLoc());
1716
1717 const unsigned Opcode = getNewOpcode(CI, Paired);
1718
1719 const TargetRegisterClass *SuperRC = getTargetRegisterClass(CI, Paired);
1720
1721 // Copy to the new source register.
1722 Register DestReg = MRI->createVirtualRegister(SuperRC);
1723 unsigned MergedOffset = std::min(CI.Offset, Paired.Offset);
1724
1725 auto MIB = BuildMI(*MBB, InsertBefore, DL, TII->get(Opcode), DestReg);
1726
1727 AddressRegs Regs = getRegs(Opcode, *TII);
1728
1729 if (Regs.VAddr)
1730 MIB.add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::vaddr));
1731
1732 // For 8-bit or 16-bit tbuffer formats there is no 3-component encoding.
1733 // If the combined count is 3 (e.g. X+X+X or XY+X), promote to 4 components
1734 // and use XYZ of XYZW to enable the merge.
1735 unsigned NumCombinedComponents = CI.Width + Paired.Width;
1736 if (NumCombinedComponents == 3 && CI.EltSize <= 2)
1737 NumCombinedComponents = 4;
1738 unsigned JoinedFormat =
1739 getBufferFormatWithCompCount(CI.Format, NumCombinedComponents, *STM);
1740
1741 // It shouldn't be possible to get this far if the two instructions
1742 // don't have a single memoperand, because MachineInstr::mayAlias()
1743 // will return true if this is the case.
1744 assert(CI.I->hasOneMemOperand() && Paired.I->hasOneMemOperand());
1745
1746 MachineInstr *New =
1747 MIB.add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::srsrc))
1748 .add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::soffset))
1749 .addImm(MergedOffset) // offset
1750 .addImm(JoinedFormat) // format
1751 .addImm(CI.CPol) // cpol
1752 .addImm(0) // swz
1753 .addMemOperand(combineKnownAdjacentMMOs(CI, Paired));
1754
1755 copyToDestRegs(CI, Paired, InsertBefore, DL, AMDGPU::OpName::vdata, DestReg);
1756
1757 CI.I->eraseFromParent();
1758 Paired.I->eraseFromParent();
1759 return New;
1760}
1761
1762MachineBasicBlock::iterator SILoadStoreOptimizer::mergeTBufferStorePair(
1763 CombineInfo &CI, CombineInfo &Paired,
1764 MachineBasicBlock::iterator InsertBefore) {
1765 MachineBasicBlock *MBB = CI.I->getParent();
1766 DebugLoc DL =
1767 DebugLoc::getMergedLocation(CI.I->getDebugLoc(), Paired.I->getDebugLoc());
1768
1769 const unsigned Opcode = getNewOpcode(CI, Paired);
1770
1771 Register SrcReg =
1772 copyFromSrcRegs(CI, Paired, InsertBefore, DL, AMDGPU::OpName::vdata);
1773
1774 auto MIB = BuildMI(*MBB, InsertBefore, DL, TII->get(Opcode)).addReg(SrcReg);
1775
1776 AddressRegs Regs = getRegs(Opcode, *TII);
1777
1778 if (Regs.VAddr)
1779 MIB.add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::vaddr));
1780
1781 // For 8-bit or 16-bit tbuffer formats there is no 3-component encoding.
1782 // If the combined count is 3 (e.g. X+X+X or XY+X), promote to 4 components
1783 // and use XYZ of XYZW to enable the merge.
1784 unsigned NumCombinedComponents = CI.Width + Paired.Width;
1785 if (NumCombinedComponents == 3 && CI.EltSize <= 2)
1786 NumCombinedComponents = 4;
1787 unsigned JoinedFormat =
1788 getBufferFormatWithCompCount(CI.Format, NumCombinedComponents, *STM);
1789
1790 // It shouldn't be possible to get this far if the two instructions
1791 // don't have a single memoperand, because MachineInstr::mayAlias()
1792 // will return true if this is the case.
1793 assert(CI.I->hasOneMemOperand() && Paired.I->hasOneMemOperand());
1794
1795 MachineInstr *New =
1796 MIB.add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::srsrc))
1797 .add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::soffset))
1798 .addImm(std::min(CI.Offset, Paired.Offset)) // offset
1799 .addImm(JoinedFormat) // format
1800 .addImm(CI.CPol) // cpol
1801 .addImm(0) // swz
1802 .addMemOperand(combineKnownAdjacentMMOs(CI, Paired));
1803
1804 CI.I->eraseFromParent();
1805 Paired.I->eraseFromParent();
1806 return New;
1807}
1808
1809MachineBasicBlock::iterator SILoadStoreOptimizer::mergeFlatLoadPair(
1810 CombineInfo &CI, CombineInfo &Paired,
1811 MachineBasicBlock::iterator InsertBefore) {
1812 MachineBasicBlock *MBB = CI.I->getParent();
1813
1814 DebugLoc DL =
1815 DebugLoc::getMergedLocation(CI.I->getDebugLoc(), Paired.I->getDebugLoc());
1816
1817 const unsigned Opcode = getNewOpcode(CI, Paired);
1818
1819 const TargetRegisterClass *SuperRC = getTargetRegisterClass(CI, Paired);
1820 Register DestReg = MRI->createVirtualRegister(SuperRC);
1821
1822 auto MIB = BuildMI(*MBB, InsertBefore, DL, TII->get(Opcode), DestReg);
1823
1824 if (auto *SAddr = TII->getNamedOperand(*CI.I, AMDGPU::OpName::saddr))
1825 MIB.add(*SAddr);
1826
1827 MachineInstr *New =
1828 MIB.add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::vaddr))
1829 .addImm(std::min(CI.Offset, Paired.Offset))
1830 .addImm(CI.CPol)
1831 .addMemOperand(combineKnownAdjacentMMOs(CI, Paired));
1832
1833 copyToDestRegs(CI, Paired, InsertBefore, DL, AMDGPU::OpName::vdst, DestReg);
1834
1835 CI.I->eraseFromParent();
1836 Paired.I->eraseFromParent();
1837 return New;
1838}
1839
1840MachineBasicBlock::iterator SILoadStoreOptimizer::mergeFlatStorePair(
1841 CombineInfo &CI, CombineInfo &Paired,
1842 MachineBasicBlock::iterator InsertBefore) {
1843 MachineBasicBlock *MBB = CI.I->getParent();
1844
1845 DebugLoc DL =
1846 DebugLoc::getMergedLocation(CI.I->getDebugLoc(), Paired.I->getDebugLoc());
1847
1848 const unsigned Opcode = getNewOpcode(CI, Paired);
1849
1850 Register SrcReg =
1851 copyFromSrcRegs(CI, Paired, InsertBefore, DL, AMDGPU::OpName::vdata);
1852
1853 auto MIB = BuildMI(*MBB, InsertBefore, DL, TII->get(Opcode))
1854 .add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::vaddr))
1855 .addReg(SrcReg);
1856
1857 if (auto *SAddr = TII->getNamedOperand(*CI.I, AMDGPU::OpName::saddr))
1858 MIB.add(*SAddr);
1859
1860 MachineInstr *New =
1861 MIB.addImm(std::min(CI.Offset, Paired.Offset))
1862 .addImm(CI.CPol)
1863 .addMemOperand(combineKnownAdjacentMMOs(CI, Paired));
1864
1865 CI.I->eraseFromParent();
1866 Paired.I->eraseFromParent();
1867 return New;
1868}
1869
1870static bool needsConstrainedOpcode(const GCNSubtarget &STM,
1871 ArrayRef<MachineMemOperand *> MMOs,
1872 unsigned Width) {
1873 // Conservatively returns true if not found the MMO.
1874 return STM.isXNACKEnabled() &&
1875 (MMOs.size() != 1 || MMOs[0]->getAlign().value() < Width * 4);
1876}
1877
1878unsigned SILoadStoreOptimizer::getNewOpcode(const CombineInfo &CI,
1879 const CombineInfo &Paired) {
1880 const unsigned Width = CI.Width + Paired.Width;
1881 const CombineInfo &Leading = Paired < CI ? Paired : CI;
1882 // If XNACK is enabled, use the constrained opcodes when the first load is
1883 // under-aligned.
1884 const bool NeedsConstrainedOpc =
1885 needsConstrainedOpcode(*STM, Leading.I->memoperands(), Width);
1886
1887 switch (getCommonInstClass(CI, Paired)) {
1888 default:
1889 assert(CI.InstClass == BUFFER_LOAD || CI.InstClass == BUFFER_STORE);
1890 // FIXME: Handle d16 correctly
1891 return AMDGPU::getMUBUFOpcode(AMDGPU::getMUBUFBaseOpcode(CI.I->getOpcode()),
1892 Width);
1893 case TBUFFER_LOAD:
1894 case TBUFFER_STORE:
1895 return AMDGPU::getMTBUFOpcode(AMDGPU::getMTBUFBaseOpcode(CI.I->getOpcode()),
1896 Width);
1897
1898 case UNKNOWN:
1899 llvm_unreachable("Unknown instruction class");
1900 case S_BUFFER_LOAD_IMM: {
1901 switch (Width) {
1902 default:
1903 return 0;
1904 case 2:
1905 return NeedsConstrainedOpc ? AMDGPU::S_BUFFER_LOAD_DWORDX2_IMM_ec
1906 : AMDGPU::S_BUFFER_LOAD_DWORDX2_IMM;
1907 case 3:
1908 return NeedsConstrainedOpc ? AMDGPU::S_BUFFER_LOAD_DWORDX3_IMM_ec
1909 : AMDGPU::S_BUFFER_LOAD_DWORDX3_IMM;
1910 case 4:
1911 return NeedsConstrainedOpc ? AMDGPU::S_BUFFER_LOAD_DWORDX4_IMM_ec
1912 : AMDGPU::S_BUFFER_LOAD_DWORDX4_IMM;
1913 case 8:
1914 return NeedsConstrainedOpc ? AMDGPU::S_BUFFER_LOAD_DWORDX8_IMM_ec
1915 : AMDGPU::S_BUFFER_LOAD_DWORDX8_IMM;
1916 }
1917 }
1918 case S_BUFFER_LOAD_SGPR_IMM: {
1919 switch (Width) {
1920 default:
1921 return 0;
1922 case 2:
1923 return NeedsConstrainedOpc ? AMDGPU::S_BUFFER_LOAD_DWORDX2_SGPR_IMM_ec
1924 : AMDGPU::S_BUFFER_LOAD_DWORDX2_SGPR_IMM;
1925 case 3:
1926 return NeedsConstrainedOpc ? AMDGPU::S_BUFFER_LOAD_DWORDX3_SGPR_IMM_ec
1927 : AMDGPU::S_BUFFER_LOAD_DWORDX3_SGPR_IMM;
1928 case 4:
1929 return NeedsConstrainedOpc ? AMDGPU::S_BUFFER_LOAD_DWORDX4_SGPR_IMM_ec
1930 : AMDGPU::S_BUFFER_LOAD_DWORDX4_SGPR_IMM;
1931 case 8:
1932 return NeedsConstrainedOpc ? AMDGPU::S_BUFFER_LOAD_DWORDX8_SGPR_IMM_ec
1933 : AMDGPU::S_BUFFER_LOAD_DWORDX8_SGPR_IMM;
1934 }
1935 }
1936 case S_LOAD_IMM: {
1937 switch (Width) {
1938 default:
1939 return 0;
1940 case 2:
1941 return NeedsConstrainedOpc ? AMDGPU::S_LOAD_DWORDX2_IMM_ec
1942 : AMDGPU::S_LOAD_DWORDX2_IMM;
1943 case 3:
1944 return NeedsConstrainedOpc ? AMDGPU::S_LOAD_DWORDX3_IMM_ec
1945 : AMDGPU::S_LOAD_DWORDX3_IMM;
1946 case 4:
1947 return NeedsConstrainedOpc ? AMDGPU::S_LOAD_DWORDX4_IMM_ec
1948 : AMDGPU::S_LOAD_DWORDX4_IMM;
1949 case 8:
1950 return NeedsConstrainedOpc ? AMDGPU::S_LOAD_DWORDX8_IMM_ec
1951 : AMDGPU::S_LOAD_DWORDX8_IMM;
1952 }
1953 }
1954 case GLOBAL_LOAD:
1955 switch (Width) {
1956 default:
1957 return 0;
1958 case 2:
1959 return AMDGPU::GLOBAL_LOAD_DWORDX2;
1960 case 3:
1961 return AMDGPU::GLOBAL_LOAD_DWORDX3;
1962 case 4:
1963 return AMDGPU::GLOBAL_LOAD_DWORDX4;
1964 }
1965 case GLOBAL_LOAD_SADDR:
1966 switch (Width) {
1967 default:
1968 return 0;
1969 case 2:
1970 return AMDGPU::GLOBAL_LOAD_DWORDX2_SADDR;
1971 case 3:
1972 return AMDGPU::GLOBAL_LOAD_DWORDX3_SADDR;
1973 case 4:
1974 return AMDGPU::GLOBAL_LOAD_DWORDX4_SADDR;
1975 }
1976 case GLOBAL_STORE:
1977 switch (Width) {
1978 default:
1979 return 0;
1980 case 2:
1981 return AMDGPU::GLOBAL_STORE_DWORDX2;
1982 case 3:
1983 return AMDGPU::GLOBAL_STORE_DWORDX3;
1984 case 4:
1985 return AMDGPU::GLOBAL_STORE_DWORDX4;
1986 }
1987 case GLOBAL_STORE_SADDR:
1988 switch (Width) {
1989 default:
1990 return 0;
1991 case 2:
1992 return AMDGPU::GLOBAL_STORE_DWORDX2_SADDR;
1993 case 3:
1994 return AMDGPU::GLOBAL_STORE_DWORDX3_SADDR;
1995 case 4:
1996 return AMDGPU::GLOBAL_STORE_DWORDX4_SADDR;
1997 }
1998 case FLAT_LOAD:
1999 switch (Width) {
2000 default:
2001 return 0;
2002 case 2:
2003 return AMDGPU::FLAT_LOAD_DWORDX2;
2004 case 3:
2005 return AMDGPU::FLAT_LOAD_DWORDX3;
2006 case 4:
2007 return AMDGPU::FLAT_LOAD_DWORDX4;
2008 }
2009 case FLAT_STORE:
2010 switch (Width) {
2011 default:
2012 return 0;
2013 case 2:
2014 return AMDGPU::FLAT_STORE_DWORDX2;
2015 case 3:
2016 return AMDGPU::FLAT_STORE_DWORDX3;
2017 case 4:
2018 return AMDGPU::FLAT_STORE_DWORDX4;
2019 }
2020 case FLAT_LOAD_SADDR:
2021 switch (Width) {
2022 default:
2023 return 0;
2024 case 2:
2025 return AMDGPU::FLAT_LOAD_DWORDX2_SADDR;
2026 case 3:
2027 return AMDGPU::FLAT_LOAD_DWORDX3_SADDR;
2028 case 4:
2029 return AMDGPU::FLAT_LOAD_DWORDX4_SADDR;
2030 }
2031 case FLAT_STORE_SADDR:
2032 switch (Width) {
2033 default:
2034 return 0;
2035 case 2:
2036 return AMDGPU::FLAT_STORE_DWORDX2_SADDR;
2037 case 3:
2038 return AMDGPU::FLAT_STORE_DWORDX3_SADDR;
2039 case 4:
2040 return AMDGPU::FLAT_STORE_DWORDX4_SADDR;
2041 }
2042 case MIMG:
2043 assert(((unsigned)llvm::popcount(CI.DMask | Paired.DMask) == Width) &&
2044 "No overlaps");
2045 return AMDGPU::getMaskedMIMGOp(CI.I->getOpcode(), Width);
2046 }
2047}
2048
2049std::pair<unsigned, unsigned>
2050SILoadStoreOptimizer::getSubRegIdxs(const CombineInfo &CI,
2051 const CombineInfo &Paired) {
2052 assert((CI.InstClass != MIMG ||
2053 ((unsigned)llvm::popcount(CI.DMask | Paired.DMask) ==
2054 CI.Width + Paired.Width)) &&
2055 "No overlaps");
2056
2057 unsigned Idx0;
2058 unsigned Idx1;
2059
2060 static const unsigned Idxs[5][4] = {
2061 {AMDGPU::sub0, AMDGPU::sub0_sub1, AMDGPU::sub0_sub1_sub2, AMDGPU::sub0_sub1_sub2_sub3},
2062 {AMDGPU::sub1, AMDGPU::sub1_sub2, AMDGPU::sub1_sub2_sub3, AMDGPU::sub1_sub2_sub3_sub4},
2063 {AMDGPU::sub2, AMDGPU::sub2_sub3, AMDGPU::sub2_sub3_sub4, AMDGPU::sub2_sub3_sub4_sub5},
2064 {AMDGPU::sub3, AMDGPU::sub3_sub4, AMDGPU::sub3_sub4_sub5, AMDGPU::sub3_sub4_sub5_sub6},
2065 {AMDGPU::sub4, AMDGPU::sub4_sub5, AMDGPU::sub4_sub5_sub6, AMDGPU::sub4_sub5_sub6_sub7},
2066 };
2067
2068 assert(CI.Width >= 1 && CI.Width <= 4);
2069 assert(Paired.Width >= 1 && Paired.Width <= 4);
2070
2071 if (Paired < CI) {
2072 Idx1 = Idxs[0][Paired.Width - 1];
2073 Idx0 = Idxs[Paired.Width][CI.Width - 1];
2074 } else {
2075 Idx0 = Idxs[0][CI.Width - 1];
2076 Idx1 = Idxs[CI.Width][Paired.Width - 1];
2077 }
2078
2079 return {Idx0, Idx1};
2080}
2081
2082const TargetRegisterClass *
2083SILoadStoreOptimizer::getTargetRegisterClass(const CombineInfo &CI,
2084 const CombineInfo &Paired) const {
2085 if (CI.InstClass == S_BUFFER_LOAD_IMM ||
2086 CI.InstClass == S_BUFFER_LOAD_SGPR_IMM || CI.InstClass == S_LOAD_IMM) {
2087 switch (CI.Width + Paired.Width) {
2088 default:
2089 return nullptr;
2090 case 2:
2091 return &AMDGPU::SReg_64_XEXECRegClass;
2092 case 3:
2093 return &AMDGPU::SGPR_96RegClass;
2094 case 4:
2095 return &AMDGPU::SGPR_128RegClass;
2096 case 8:
2097 return &AMDGPU::SGPR_256RegClass;
2098 case 16:
2099 return &AMDGPU::SGPR_512RegClass;
2100 }
2101 }
2102
2103 // FIXME: This should compute the instruction to use, and then use the result
2104 // of TII->getRegClass.
2105 unsigned BitWidth = 32 * (CI.Width + Paired.Width);
2106 return TRI->isAGPRClass(getDataRegClass(*CI.I))
2107 ? TRI->getAGPRClassForBitWidth(BitWidth)
2108 : TRI->getVGPRClassForBitWidth(BitWidth);
2109}
2110
2111MachineBasicBlock::iterator SILoadStoreOptimizer::mergeBufferStorePair(
2112 CombineInfo &CI, CombineInfo &Paired,
2113 MachineBasicBlock::iterator InsertBefore) {
2114 MachineBasicBlock *MBB = CI.I->getParent();
2115 DebugLoc DL =
2116 DebugLoc::getMergedLocation(CI.I->getDebugLoc(), Paired.I->getDebugLoc());
2117
2118 const unsigned Opcode = getNewOpcode(CI, Paired);
2119
2120 Register SrcReg =
2121 copyFromSrcRegs(CI, Paired, InsertBefore, DL, AMDGPU::OpName::vdata);
2122
2123 auto MIB = BuildMI(*MBB, InsertBefore, DL, TII->get(Opcode)).addReg(SrcReg);
2124
2125 AddressRegs Regs = getRegs(Opcode, *TII);
2126
2127 if (Regs.VAddr)
2128 MIB.add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::vaddr));
2129
2130
2131 // It shouldn't be possible to get this far if the two instructions
2132 // don't have a single memoperand, because MachineInstr::mayAlias()
2133 // will return true if this is the case.
2134 assert(CI.I->hasOneMemOperand() && Paired.I->hasOneMemOperand());
2135
2136 MachineInstr *New =
2137 MIB.add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::srsrc))
2138 .add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::soffset))
2139 .addImm(std::min(CI.Offset, Paired.Offset)) // offset
2140 .addImm(CI.CPol) // cpol
2141 .addImm(0) // swz
2142 .addMemOperand(combineKnownAdjacentMMOs(CI, Paired));
2143
2144 CI.I->eraseFromParent();
2145 Paired.I->eraseFromParent();
2146 return New;
2147}
2148
2149MachineOperand
2150SILoadStoreOptimizer::createRegOrImm(int32_t Val, MachineInstr &MI) const {
2151 APInt V(32, Val, true);
2152 if (TII->isInlineConstant(V))
2153 return MachineOperand::CreateImm(Val);
2154
2155 Register Reg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass);
2156 MachineInstr *Mov =
2157 BuildMI(*MI.getParent(), MI.getIterator(), MI.getDebugLoc(),
2158 TII->get(AMDGPU::S_MOV_B32), Reg)
2159 .addImm(Val);
2160 (void)Mov;
2161 LLVM_DEBUG(dbgs() << " "; Mov->dump());
2162 return MachineOperand::CreateReg(Reg, false);
2163}
2164
2165// Compute base address using Addr and return the final register.
2166Register SILoadStoreOptimizer::computeBase(MachineInstr &MI,
2167 const MemAddress &Addr) const {
2168 MachineBasicBlock *MBB = MI.getParent();
2169 MachineBasicBlock::iterator MBBI = MI.getIterator();
2170 const DebugLoc &DL = MI.getDebugLoc();
2171
2172 LLVM_DEBUG(dbgs() << " Re-Computed Anchor-Base:\n");
2173
2174 // Use V_ADD_U64_e64 when the original pattern used it (gfx1250+)
2175 if (Addr.Base.UseV64Pattern) {
2176 Register FullDestReg = MRI->createVirtualRegister(
2177 TII->getRegClass(TII->get(AMDGPU::V_ADD_U64_e64), 0));
2178
2179 // Load the 64-bit offset into an SGPR pair if needed
2180 Register OffsetReg = MRI->createVirtualRegister(&AMDGPU::SReg_64RegClass);
2181 MachineInstr *MovOffset =
2182 BuildMI(*MBB, MBBI, DL, TII->get(AMDGPU::S_MOV_B64_IMM_PSEUDO),
2183 OffsetReg)
2184 .addImm(Addr.Offset);
2185 MachineInstr *Add64 =
2186 BuildMI(*MBB, MBBI, DL, TII->get(AMDGPU::V_ADD_U64_e64), FullDestReg)
2187 .addReg(Addr.Base.LoReg)
2188 .addReg(OffsetReg)
2189 .addImm(0);
2190 (void)MovOffset;
2191 (void)Add64;
2192 LLVM_DEBUG(dbgs() << " " << *MovOffset << "\n";
2193 dbgs() << " " << *Add64 << "\n\n";);
2194
2195 return FullDestReg;
2196 }
2197
2198 // Use V_ADD_U32_e64 for the 32-bit FLAT_SCRATCH vaddr base.
2199 if (Addr.Base.UseScratchVGPRPattern) {
2200 MachineOperand OffsetOp =
2201 createRegOrImm(static_cast<int32_t>(Addr.Offset), MI);
2202
2203 Register FullDestReg = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
2204 MachineInstr *Add32 =
2205 BuildMI(*MBB, MBBI, DL, TII->get(AMDGPU::V_ADD_U32_e64), FullDestReg)
2206 .addReg(Addr.Base.LoReg, {}, Addr.Base.LoSubReg)
2207 .add(OffsetOp)
2208 .addImm(0); // clamp bit
2209 (void)Add32;
2210 LLVM_DEBUG(dbgs() << " " << *Add32 << "\n\n";);
2211
2212 return FullDestReg;
2213 }
2214
2215 // Original carry-chain pattern (V_ADD_CO_U32 + V_ADDC_U32)
2216 assert((TRI->getRegSizeInBits(Addr.Base.LoReg, *MRI) == 32 ||
2217 Addr.Base.LoSubReg) &&
2218 "Expected 32-bit Base-Register-Low!!");
2219
2220 assert((TRI->getRegSizeInBits(Addr.Base.HiReg, *MRI) == 32 ||
2221 Addr.Base.HiSubReg) &&
2222 "Expected 32-bit Base-Register-Hi!!");
2223
2224 MachineOperand OffsetLo = createRegOrImm(static_cast<int32_t>(Addr.Offset), MI);
2225 MachineOperand OffsetHi =
2226 createRegOrImm(static_cast<int32_t>(Addr.Offset >> 32), MI);
2227
2228 const auto *CarryRC = TRI->getWaveMaskRegClass();
2229 Register CarryReg = MRI->createVirtualRegister(CarryRC);
2230 Register DeadCarryReg = MRI->createVirtualRegister(CarryRC);
2231
2232 Register DestSub0 = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
2233 Register DestSub1 = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
2234 MachineInstr *LoHalf =
2235 BuildMI(*MBB, MBBI, DL, TII->get(AMDGPU::V_ADD_CO_U32_e64), DestSub0)
2236 .addReg(CarryReg, RegState::Define)
2237 .addReg(Addr.Base.LoReg, {}, Addr.Base.LoSubReg)
2238 .add(OffsetLo)
2239 .addImm(0); // clamp bit
2240
2241 MachineInstr *HiHalf =
2242 BuildMI(*MBB, MBBI, DL, TII->get(AMDGPU::V_ADDC_U32_e64), DestSub1)
2243 .addReg(DeadCarryReg, RegState::Define | RegState::Dead)
2244 .addReg(Addr.Base.HiReg, {}, Addr.Base.HiSubReg)
2245 .add(OffsetHi)
2246 .addReg(CarryReg)
2247 .addImm(0); // clamp bit
2248
2249 Register FullDestReg = MRI->createVirtualRegister(TRI->getVGPR64Class());
2250 MachineInstr *FullBase =
2251 BuildMI(*MBB, MBBI, DL, TII->get(TargetOpcode::REG_SEQUENCE), FullDestReg)
2252 .addReg(DestSub0)
2253 .addImm(AMDGPU::sub0)
2254 .addReg(DestSub1)
2255 .addImm(AMDGPU::sub1);
2256
2257 (void)LoHalf;
2258 (void)HiHalf;
2259 (void)FullBase;
2260 LLVM_DEBUG(dbgs() << " " << *LoHalf << "\n";
2261 dbgs() << " " << *HiHalf << "\n";
2262 dbgs() << " " << *FullBase << "\n\n";);
2263
2264 return FullDestReg;
2265}
2266
2267// Update base and offset with the NewBase and NewOffset in MI.
2268void SILoadStoreOptimizer::updateBaseAndOffset(MachineInstr &MI,
2269 Register NewBase,
2270 int32_t NewOffset) const {
2271 auto *Base = TII->getNamedOperand(MI, AMDGPU::OpName::vaddr);
2272 Base->setReg(NewBase);
2273 Base->setIsKill(false);
2274 TII->getNamedOperand(MI, AMDGPU::OpName::offset)->setImm(NewOffset);
2275}
2276
2277// Helper to extract a 64-bit constant offset from a V_ADD_U64_e64 instruction.
2278// Returns true if successful, populating Addr with base register info and
2279// offset.
2280bool SILoadStoreOptimizer::processBaseWithConstOffset64(
2281 MachineInstr *AddDef, const MachineOperand &Base, MemAddress &Addr) const {
2282 if (!Base.isReg())
2283 return false;
2284
2285 MachineOperand *Src0 = TII->getNamedOperand(*AddDef, AMDGPU::OpName::src0);
2286 MachineOperand *Src1 = TII->getNamedOperand(*AddDef, AMDGPU::OpName::src1);
2287
2288 const MachineOperand *BaseOp = nullptr;
2289
2290 auto Offset = TII->getImmOrMaterializedImm(*MRI, *Src1);
2291
2292 if (Offset) {
2293 BaseOp = Src0;
2294 Addr.Offset = *Offset;
2295 } else {
2296 // Both or neither are constants - can't handle this pattern
2297 return false;
2298 }
2299
2300 // Now extract the base register (which should be a 64-bit VGPR).
2301 Addr.Base.LoReg = BaseOp->getReg();
2302 Addr.Base.UseV64Pattern = true;
2303 return true;
2304}
2305
2306// Helper to extract a 32-bit constant offset from a V_ADD_U32_e64 instruction.
2307// This is the pattern used for the 32-bit vaddr of FLAT_SCRATCH instructions.
2308// Returns true if successful, populating Addr with base register info and
2309// offset.
2310bool SILoadStoreOptimizer::processBaseWithConstOffset32(
2311 MachineInstr *AddDef, MemAddress &Addr) const {
2312 // A clamped add saturates, so its constant cannot be split off.
2313 if (TII->getNamedOperand(*AddDef, AMDGPU::OpName::clamp)->getImm())
2314 return false;
2315
2316 MachineOperand *Src0 = TII->getNamedOperand(*AddDef, AMDGPU::OpName::src0);
2317 MachineOperand *Src1 = TII->getNamedOperand(*AddDef, AMDGPU::OpName::src1);
2318
2319 const MachineOperand *BaseOp = nullptr;
2320 std::optional<int64_t> Offset;
2321
2322 if ((Offset = TII->getImmOrMaterializedImm(*MRI, *Src1))) {
2323 BaseOp = Src0;
2324 } else if ((Offset = TII->getImmOrMaterializedImm(*MRI, *Src0))) {
2325 BaseOp = Src1;
2326 } else {
2327 // Both or neither are constants - can't handle this pattern
2328 return false;
2329 }
2330
2331 // An SGPR base could make the rebuilt add read two SGPRs.
2332 if (!BaseOp->isReg() || !TRI->isVGPR(*MRI, BaseOp->getReg()))
2333 return false;
2334
2335 Addr.Base.LoReg = BaseOp->getReg();
2336 Addr.Base.LoSubReg = BaseOp->getSubReg();
2337 Addr.Base.UseScratchVGPRPattern = true;
2338 Addr.Offset = *Offset;
2339 return true;
2340}
2341
2342// Analyze Base and extracts:
2343// - 32bit base registers, subregisters
2344// - 64bit constant offset
2345// Expecting base computation as:
2346// %OFFSET0:sgpr_32 = S_MOV_B32 8000
2347// %LO:vgpr_32, %c:sreg_64_xexec =
2348// V_ADD_CO_U32_e64 %BASE_LO:vgpr_32, %103:sgpr_32,
2349// %HI:vgpr_32, = V_ADDC_U32_e64 %BASE_HI:vgpr_32, 0, killed %c:sreg_64_xexec
2350// %Base:vreg_64 =
2351// REG_SEQUENCE %LO:vgpr_32, %subreg.sub0, %HI:vgpr_32, %subreg.sub1
2352//
2353// Also handles V_ADD_U64_e64 pattern (gfx1250+):
2354// %OFFSET:sreg_64 = S_MOV_B64_IMM_PSEUDO 256
2355// %Base:vreg_64 = V_ADD_U64_e64 %BASE:vreg_64, %OFFSET:sreg_64, 0
2356//
2357// Also handles the 32-bit VGPR pattern used for FLAT_SCRATCH vaddr:
2358// %OFFSET:sreg_32 = S_MOV_B32 4096
2359// %Base:vgpr_32 = V_ADD_U32_e64 %BASE:vgpr_32, %OFFSET:sreg_32, 0
2360void SILoadStoreOptimizer::processBaseWithConstOffset(const MachineOperand &Base,
2361 MemAddress &Addr) const {
2362 if (!Base.isReg())
2363 return;
2364
2365 MachineInstr *Def = MRI->getUniqueVRegDef(Base.getReg());
2366 if (!Def)
2367 return;
2368
2369 // Try V_ADD_U64_e64 pattern first (simpler, used on gfx1250+)
2370 if (Def->getOpcode() == AMDGPU::V_ADD_U64_e64) {
2371 if (processBaseWithConstOffset64(Def, Base, Addr))
2372 return;
2373 }
2374
2375 // Try V_ADD_U32_e64 pattern (32-bit base, used for FLAT_SCRATCH vaddr)
2376 // Not GLOBAL saddr: it zero-extends vaddr, so a 32-bit wrap cannot be split.
2377 if (Def->getOpcode() == AMDGPU::V_ADD_U32_e64 &&
2378 SIInstrInfo::isFLATScratch(*Base.getParent())) {
2379 if (processBaseWithConstOffset32(Def, Addr))
2380 return;
2381 }
2382
2383 // Fall through to REG_SEQUENCE + V_ADD_CO_U32 + V_ADDC_U32 pattern
2384 if (Def->getOpcode() != AMDGPU::REG_SEQUENCE || Def->getNumOperands() != 5)
2385 return;
2386
2387 MachineOperand BaseLo = Def->getOperand(1);
2388 MachineOperand BaseHi = Def->getOperand(3);
2389 if (!BaseLo.isReg() || !BaseHi.isReg())
2390 return;
2391
2392 MachineInstr *BaseLoDef = MRI->getUniqueVRegDef(BaseLo.getReg());
2393 MachineInstr *BaseHiDef = MRI->getUniqueVRegDef(BaseHi.getReg());
2394
2395 if (!BaseLoDef || BaseLoDef->getOpcode() != AMDGPU::V_ADD_CO_U32_e64 ||
2396 !BaseHiDef || BaseHiDef->getOpcode() != AMDGPU::V_ADDC_U32_e64)
2397 return;
2398
2399 MachineOperand *Src0 = TII->getNamedOperand(*BaseLoDef, AMDGPU::OpName::src0);
2400 MachineOperand *Src1 = TII->getNamedOperand(*BaseLoDef, AMDGPU::OpName::src1);
2401
2402 auto Offset0P = TII->getImmOrMaterializedImm(*MRI, *Src0);
2403 if (Offset0P)
2404 BaseLo = *Src1;
2405 else {
2406 if (!(Offset0P = TII->getImmOrMaterializedImm(*MRI, *Src1)))
2407 return;
2408 BaseLo = *Src0;
2409 }
2410
2411 if (!BaseLo.isReg())
2412 return;
2413
2414 Src0 = TII->getNamedOperand(*BaseHiDef, AMDGPU::OpName::src0);
2415 Src1 = TII->getNamedOperand(*BaseHiDef, AMDGPU::OpName::src1);
2416
2417 if (Src0->isImm())
2418 std::swap(Src0, Src1);
2419
2420 if (!Src1->isImm() || Src0->isImm())
2421 return;
2422
2423 uint64_t Offset1 = Src1->getImm();
2424 BaseHi = *Src0;
2425
2426 if (!BaseHi.isReg())
2427 return;
2428
2429 Addr.Base.LoReg = BaseLo.getReg();
2430 Addr.Base.HiReg = BaseHi.getReg();
2431 Addr.Base.LoSubReg = BaseLo.getSubReg();
2432 Addr.Base.HiSubReg = BaseHi.getSubReg();
2433 Addr.Offset = (*Offset0P & 0x00000000ffffffff) | (Offset1 << 32);
2434}
2435
2436// Maintain the correct LDS address for async loads and stores.
2437// It becomes incorrect when promoteConstantOffsetToImm adds an offset only
2438// meant for the global address operand. For async loads the LDS address is in
2439// vdst. For async stores, the LDS address is in vdata.
2440void SILoadStoreOptimizer::updateAsyncLDSAddress(MachineInstr &MI,
2441 int32_t OffsetDiff) const {
2442 if (!TII->usesASYNC_CNT(MI) || OffsetDiff == 0)
2443 return;
2444
2445 MachineOperand *LDSAddr = TII->getNamedOperand(MI, AMDGPU::OpName::vdst);
2446 if (!LDSAddr)
2447 LDSAddr = TII->getNamedOperand(MI, AMDGPU::OpName::vdata);
2448 assert(LDSAddr);
2449
2450 Register OldReg = LDSAddr->getReg();
2451 Register NewReg = MRI->createVirtualRegister(MRI->getRegClass(OldReg));
2452 MachineBasicBlock &MBB = *MI.getParent();
2453 const DebugLoc &DL = MI.getDebugLoc();
2454 BuildMI(MBB, MI, DL, TII->get(AMDGPU::V_ADD_U32_e64), NewReg)
2455 .addReg(OldReg)
2456 .addImm(-OffsetDiff)
2457 .addImm(0);
2458
2459 LDSAddr->setReg(NewReg);
2460}
2461
2462bool SILoadStoreOptimizer::promoteConstantOffsetToImm(
2463 MachineInstr &MI,
2464 MemInfoMap &Visited,
2465 SmallPtrSet<MachineInstr *, 4> &AnchorList) const {
2466
2467 if (!STM->hasFlatInstOffsets() || !SIInstrInfo::isFLAT(MI))
2468 return false;
2469
2470 if (SIInstrInfo::isFLATScratch(MI) &&
2471 !TII->getNamedOperand(MI, AMDGPU::OpName::vaddr))
2472 return false;
2473
2474 // Moving part of vaddr into the offset can trigger the SVS swizzle bug.
2475 if (STM->hasFlatScratchSVSSwizzleBug() && SIInstrInfo::isFLATScratch(MI) &&
2476 TII->getNamedOperand(MI, AMDGPU::OpName::saddr))
2477 return false;
2478
2479 unsigned AS = SIInstrInfo::isFLATGlobal(MI) ? AMDGPUAS::GLOBAL_ADDRESS
2480 : SIInstrInfo::isFLATScratch(MI) ? AMDGPUAS::PRIVATE_ADDRESS
2481 : AMDGPUAS::FLAT_ADDRESS;
2482
2483 AMDGPU::FlatAddrSpace FlatVariant =
2484 AS == AMDGPUAS::GLOBAL_ADDRESS ? AMDGPU::FlatAddrSpace::FlatGlobal
2485 : AS == AMDGPUAS::PRIVATE_ADDRESS ? AMDGPU::FlatAddrSpace::FlatScratch
2486 : AMDGPU::FlatAddrSpace::FLAT;
2487 bool AllowNegativeOffset =
2488 TII->allowNegativeFlatOffset(FlatVariant) && !TII->usesASYNC_CNT(MI);
2489 // The async global instructions use i24 offset for global address but u16
2490 // offset for LDS address. In this case, we just only promote when the offset
2491 // is u16.
2492 bool IsOffsetU16 = TII->usesASYNC_CNT(MI);
2493
2494 if (AnchorList.count(&MI))
2495 return false;
2496
2497 LLVM_DEBUG(dbgs() << "\nTryToPromoteConstantOffsetToImmFor "; MI.dump());
2498
2499 if (TII->getNamedOperand(MI, AMDGPU::OpName::offset)->getImm()) {
2500 LLVM_DEBUG(dbgs() << " Const-offset is already promoted.\n";);
2501 return false;
2502 }
2503
2504 // Step1: Find the base-registers and a 64bit constant offset.
2505 MachineOperand &Base = *TII->getNamedOperand(MI, AMDGPU::OpName::vaddr);
2506 auto [It, Inserted] = Visited.try_emplace(&MI);
2507 MemAddress MAddr;
2508 if (Inserted) {
2509 processBaseWithConstOffset(Base, MAddr);
2510 It->second = MAddr;
2511 } else
2512 MAddr = It->second;
2513
2514 if (MAddr.Offset == 0) {
2515 LLVM_DEBUG(dbgs() << " Failed to extract constant-offset or there are no"
2516 " constant offsets that can be promoted.\n";);
2517 return false;
2518 }
2519
2520 LLVM_DEBUG(dbgs() << " BASE: {" << printReg(MAddr.Base.HiReg, TRI) << ", "
2521 << printReg(MAddr.Base.LoReg, TRI)
2522 << "} Offset: " << MAddr.Offset << "\n\n";);
2523
2524 // Step2: Traverse through MI's basic block and find an anchor(that has the
2525 // same base-registers) with the highest 13bit distance from MI's offset.
2526 // E.g. (64bit loads)
2527 // bb:
2528 // addr1 = &a + 4096; load1 = load(addr1, 0)
2529 // addr2 = &a + 6144; load2 = load(addr2, 0)
2530 // addr3 = &a + 8192; load3 = load(addr3, 0)
2531 // addr4 = &a + 10240; load4 = load(addr4, 0)
2532 // addr5 = &a + 12288; load5 = load(addr5, 0)
2533 //
2534 // Starting from the first load, the optimization will try to find a new base
2535 // from which (&a + 4096) has 13 bit distance. Both &a + 6144 and &a + 8192
2536 // has 13bit distance from &a + 4096. The heuristic considers &a + 8192
2537 // as the new-base(anchor) because of the maximum distance which can
2538 // accommodate more intermediate bases presumably.
2539 //
2540 // Step3: move (&a + 8192) above load1. Compute and promote offsets from
2541 // (&a + 8192) for load1, load2, load4.
2542 // addr = &a + 8192
2543 // load1 = load(addr, -4096)
2544 // load2 = load(addr, -2048)
2545 // load3 = load(addr, 0)
2546 // load4 = load(addr, 2048)
2547 // addr5 = &a + 12288; load5 = load(addr5, 0)
2548 //
2549 MachineInstr *AnchorInst = nullptr;
2550 MemAddress AnchorAddr;
2551 uint32_t MaxDist = std::numeric_limits<uint32_t>::min();
2552 SmallVector<std::pair<MachineInstr *, int64_t>, 4> InstsWCommonBase;
2553 bool MIIsAnchor = false;
2554
2555 MachineBasicBlock *MBB = MI.getParent();
2556 MachineBasicBlock::iterator E = MBB->end();
2557 MachineBasicBlock::iterator MBBI = MI.getIterator();
2558 ++MBBI;
2559 const SITargetLowering *TLI = STM->getTargetLowering();
2560
2561 for ( ; MBBI != E; ++MBBI) {
2562 MachineInstr &MINext = *MBBI;
2563 // TODO: Support finding an anchor(with same base) from store addresses or
2564 // any other load addresses where the opcodes are different.
2565 if (MINext.getOpcode() != MI.getOpcode() ||
2566 TII->getNamedOperand(MINext, AMDGPU::OpName::offset)->getImm())
2567 continue;
2568
2569 const MachineOperand &BaseNext =
2570 *TII->getNamedOperand(MINext, AMDGPU::OpName::vaddr);
2571 MemAddress MAddrNext;
2572 auto [It, Inserted] = Visited.try_emplace(&MINext);
2573 if (Inserted) {
2574 processBaseWithConstOffset(BaseNext, MAddrNext);
2575 It->second = MAddrNext;
2576 } else
2577 MAddrNext = It->second;
2578
2579 if (MAddrNext.Base.LoReg != MAddr.Base.LoReg ||
2580 MAddrNext.Base.HiReg != MAddr.Base.HiReg ||
2581 MAddrNext.Base.LoSubReg != MAddr.Base.LoSubReg ||
2582 MAddrNext.Base.HiSubReg != MAddr.Base.HiSubReg)
2583 continue;
2584
2585 InstsWCommonBase.emplace_back(&MINext, MAddrNext.Offset);
2586
2587 if (AllowNegativeOffset) {
2588 int64_t Dist = MAddr.Offset - MAddrNext.Offset;
2589 TargetLoweringBase::AddrMode AM;
2590 AM.HasBaseReg = true;
2591 AM.BaseOffs = Dist;
2592 if (TLI->isLegalFlatAddressingMode(AM, AS) &&
2593 (uint32_t)std::abs(Dist) > MaxDist) {
2594 MaxDist = std::abs(Dist);
2595
2596 AnchorAddr = MAddrNext;
2597 AnchorInst = &MINext;
2598 }
2599 }
2600 }
2601
2602 // When negative offsets are not allowed, pick the candidate with the smallest
2603 // offset as anchor so all promoted offsets are non-negative. If MI itself has
2604 // the smallest offset, MI becomes the reference point (MIIsAnchor).
2605 if (!AllowNegativeOffset && !InstsWCommonBase.empty()) {
2606 for (auto &[Inst, Offset] : InstsWCommonBase) {
2607 int64_t Dist = MAddr.Offset - Offset;
2608 TargetLoweringBase::AddrMode AM;
2609 AM.HasBaseReg = true;
2610 AM.BaseOffs = Dist;
2611 if (Dist >= 0 && TLI->isLegalFlatAddressingMode(AM, AS) &&
2612 (!IsOffsetU16 || isUInt<16>(Dist)) &&
2613 (!AnchorInst || Offset < AnchorAddr.Offset)) {
2614 AnchorAddr = Visited[Inst];
2615 AnchorInst = Inst;
2616 }
2617 }
2618 if (!AnchorInst)
2619 MIIsAnchor = true;
2620 }
2621
2622 if (AnchorInst) {
2623 LLVM_DEBUG(dbgs() << " Anchor-Inst(with max-distance from Offset): ";
2624 AnchorInst->dump());
2625 LLVM_DEBUG(dbgs() << " Anchor-Offset from BASE: "
2626 << AnchorAddr.Offset << "\n\n");
2627
2628 // Instead of moving up, just re-compute anchor-instruction's base address.
2629 Register Base = computeBase(MI, AnchorAddr);
2630
2631 int32_t OffsetDiff = MAddr.Offset - AnchorAddr.Offset;
2632 updateBaseAndOffset(MI, Base, OffsetDiff);
2633 updateAsyncLDSAddress(MI, OffsetDiff);
2634 LLVM_DEBUG(dbgs() << " After promotion: "; MI.dump(););
2635
2636 for (auto [OtherMI, OtherOffset] : InstsWCommonBase) {
2637 TargetLoweringBase::AddrMode AM;
2638 AM.HasBaseReg = true;
2639 AM.BaseOffs = OtherOffset - AnchorAddr.Offset;
2640
2641 if (TLI->isLegalFlatAddressingMode(AM, AS) &&
2642 (AllowNegativeOffset || AM.BaseOffs >= 0) &&
2643 (!IsOffsetU16 || isUInt<16>(AM.BaseOffs))) {
2644 LLVM_DEBUG(dbgs() << " Promote Offset(" << OtherOffset; dbgs() << ")";
2645 OtherMI->dump());
2646 int32_t OtherOffsetDiff = OtherOffset - AnchorAddr.Offset;
2647 updateBaseAndOffset(*OtherMI, Base, OtherOffsetDiff);
2648 updateAsyncLDSAddress(*OtherMI, OtherOffsetDiff);
2649 LLVM_DEBUG(dbgs() << " After promotion: "; OtherMI->dump());
2650 }
2651 }
2652 AnchorList.insert(AnchorInst);
2653 return true;
2654 }
2655
2656 if (MIIsAnchor) {
2657 LLVM_DEBUG(dbgs() << " MI is anchor (smallest offset); promoting "
2658 "candidates relative to MI's base.\n");
2659
2660 Register Base = TII->getNamedOperand(MI, AMDGPU::OpName::vaddr)->getReg();
2661 bool AnyPromoted = false;
2662
2663 for (auto [OtherMI, OtherOffset] : InstsWCommonBase) {
2664 int64_t Dist = OtherOffset - MAddr.Offset;
2665 TargetLoweringBase::AddrMode AM;
2666 AM.HasBaseReg = true;
2667 AM.BaseOffs = Dist;
2668 if (Dist >= 0 && TLI->isLegalFlatAddressingMode(AM, AS) &&
2669 (!IsOffsetU16 || isUInt<16>(Dist))) {
2670 LLVM_DEBUG(dbgs() << " Promote Offset(" << OtherOffset << ")";
2671 OtherMI->dump());
2672 updateBaseAndOffset(*OtherMI, Base, Dist);
2673 updateAsyncLDSAddress(*OtherMI, Dist);
2674 LLVM_DEBUG(dbgs() << " After promotion: "; OtherMI->dump());
2675 AnyPromoted = true;
2676 }
2677 }
2678
2679 if (AnyPromoted) {
2680 TII->getNamedOperand(MI, AMDGPU::OpName::vaddr)->setIsKill(false);
2681 AnchorList.insert(&MI);
2682 return true;
2683 }
2684 }
2685
2686 return false;
2687}
2688
2689void SILoadStoreOptimizer::addInstToMergeableList(const CombineInfo &CI,
2690 std::list<std::list<CombineInfo> > &MergeableInsts) const {
2691 for (std::list<CombineInfo> &AddrList : MergeableInsts) {
2692 if (AddrList.front().InstClass == CI.InstClass &&
2693 AddrList.front().hasSameBaseAddress(CI)) {
2694 AddrList.emplace_back(CI);
2695 return;
2696 }
2697 }
2698
2699 // Base address not found, so add a new list.
2700 MergeableInsts.emplace_back(1, CI);
2701}
2702
2703std::pair<MachineBasicBlock::iterator, bool>
2704SILoadStoreOptimizer::collectMergeableInsts(
2705 MachineBasicBlock::iterator Begin, MachineBasicBlock::iterator End,
2706 MemInfoMap &Visited, SmallPtrSet<MachineInstr *, 4> &AnchorList,
2707 std::list<std::list<CombineInfo>> &MergeableInsts) const {
2708 bool Modified = false;
2709
2710 // Sort potential mergeable instructions into lists. One list per base address.
2711 unsigned Order = 0;
2712 MachineBasicBlock::iterator BlockI = Begin;
2713 for (; BlockI != End; ++BlockI) {
2714 MachineInstr &MI = *BlockI;
2715
2716 // We run this before checking if an address is mergeable, because it can produce
2717 // better code even if the instructions aren't mergeable.
2718 if (promoteConstantOffsetToImm(MI, Visited, AnchorList))
2719 Modified = true;
2720
2721 // Treat volatile accesses, ordered accesses and unmodeled side effects as
2722 // barriers. We can look after this barrier for separate merges.
2723 if (MI.hasOrderedMemoryRef() || MI.hasUnmodeledSideEffects()) {
2724 LLVM_DEBUG(dbgs() << "Breaking search on barrier: " << MI);
2725
2726 // Search will resume after this instruction in a separate merge list.
2727 ++BlockI;
2728 break;
2729 }
2730
2731 const InstClassEnum InstClass = getInstClass(MI.getOpcode(), *TII);
2732 if (InstClass == UNKNOWN)
2733 continue;
2734
2735 // Do not merge VMEM buffer instructions with "swizzled" bit set.
2736 int Swizzled =
2737 AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::swz);
2738 if (Swizzled != -1 && MI.getOperand(Swizzled).getImm())
2739 continue;
2740
2741 if (InstClass == TBUFFER_LOAD || InstClass == TBUFFER_STORE) {
2742 if (!STM->hasRelaxedTBufferOOBMode()) {
2743 LLVM_DEBUG(
2744 dbgs() << "Skip tbuffer combine: relaxed OOB mode not enabled\n");
2745 continue;
2746 }
2747
2748 const MachineOperand *Fmt =
2749 TII->getNamedOperand(MI, AMDGPU::OpName::format);
2750 if (!AMDGPU::getGcnBufferFormatInfo(Fmt->getImm(), *STM)) {
2751 LLVM_DEBUG(dbgs() << "Skip tbuffer with unknown format: " << MI);
2752 continue;
2753 }
2754 } else if (InstClass == MIMG) {
2755 // Do not merge MIMG instructions with tfe or lwe enabled.
2756 // TFE/LWE add a status result that the image merge path does not model.
2757 const auto *TFEOp = TII->getNamedOperand(MI, AMDGPU::OpName::tfe);
2758 if (TFEOp && TFEOp->getImm())
2759 continue;
2760
2761 const auto *LWEOp = TII->getNamedOperand(MI, AMDGPU::OpName::lwe);
2762 if (LWEOp && LWEOp->getImm())
2763 continue;
2764 }
2765
2766 CombineInfo CI;
2767 CI.setMI(MI, *this);
2768 CI.Order = Order++;
2769
2770 if (!CI.hasMergeableAddress(*MRI))
2771 continue;
2772
2773 LLVM_DEBUG(dbgs() << "Mergeable: " << MI);
2774
2775 addInstToMergeableList(CI, MergeableInsts);
2776 }
2777
2778 // At this point we have lists of Mergeable instructions.
2779 //
2780 // Part 2: Sort lists by offset and then for each CombineInfo object in the
2781 // list try to find an instruction that can be merged with I. If an instruction
2782 // is found, it is stored in the Paired field. If no instructions are found, then
2783 // the CombineInfo object is deleted from the list.
2784
2785 for (std::list<std::list<CombineInfo>>::iterator I = MergeableInsts.begin(),
2786 E = MergeableInsts.end(); I != E;) {
2787
2788 std::list<CombineInfo> &MergeList = *I;
2789 if (MergeList.size() <= 1) {
2790 // This means we have found only one instruction with a given address
2791 // that can be merged, and we need at least 2 instructions to do a merge,
2792 // so this list can be discarded.
2793 I = MergeableInsts.erase(I);
2794 continue;
2795 }
2796
2797 // Sort the lists by offsets, this way mergeable instructions will be
2798 // adjacent to each other in the list, which will make it easier to find
2799 // matches.
2800 MergeList.sort(
2801 [] (const CombineInfo &A, const CombineInfo &B) {
2802 return A.Offset < B.Offset;
2803 });
2804 ++I;
2805 }
2806
2807 return {BlockI, Modified};
2808}
2809
2810// Scan through looking for adjacent LDS operations with constant offsets from
2811// the same base register. We rely on the scheduler to do the hard work of
2812// clustering nearby loads, and assume these are all adjacent.
2813bool SILoadStoreOptimizer::optimizeBlock(
2814 std::list<std::list<CombineInfo> > &MergeableInsts) {
2815 bool Modified = false;
2816
2817 for (std::list<std::list<CombineInfo>>::iterator I = MergeableInsts.begin(),
2818 E = MergeableInsts.end(); I != E;) {
2819 std::list<CombineInfo> &MergeList = *I;
2820
2821 bool OptimizeListAgain = false;
2822 if (!optimizeInstsWithSameBaseAddr(MergeList, OptimizeListAgain)) {
2823 // We weren't able to make any changes, so delete the list so we don't
2824 // process the same instructions the next time we try to optimize this
2825 // block.
2826 I = MergeableInsts.erase(I);
2827 continue;
2828 }
2829
2830 Modified = true;
2831
2832 // We made changes, but also determined that there were no more optimization
2833 // opportunities, so we don't need to reprocess the list
2834 if (!OptimizeListAgain) {
2835 I = MergeableInsts.erase(I);
2836 continue;
2837 }
2838 OptimizeAgain = true;
2839 }
2840 return Modified;
2841}
2842
2843bool
2844SILoadStoreOptimizer::optimizeInstsWithSameBaseAddr(
2845 std::list<CombineInfo> &MergeList,
2846 bool &OptimizeListAgain) {
2847 if (MergeList.empty())
2848 return false;
2849
2850 bool Modified = false;
2851
2852 for (auto I = MergeList.begin(), Next = std::next(I); Next != MergeList.end();
2853 Next = std::next(I)) {
2854
2855 auto First = I;
2856 auto Second = Next;
2857
2858 if ((*First).Order > (*Second).Order)
2859 std::swap(First, Second);
2860 CombineInfo &CI = *First;
2861 CombineInfo &Paired = *Second;
2862
2863 CombineInfo *Where = checkAndPrepareMerge(CI, Paired);
2864 if (!Where) {
2865 ++I;
2866 continue;
2867 }
2868
2869 Modified = true;
2870
2871 LLVM_DEBUG(dbgs() << "Merging: " << *CI.I << " with: " << *Paired.I);
2872
2873 MachineBasicBlock::iterator NewMI;
2874 switch (CI.InstClass) {
2875 default:
2876 llvm_unreachable("unknown InstClass");
2877 break;
2878 case DS_READ:
2879 NewMI = mergeRead2Pair(CI, Paired, Where->I);
2880 break;
2881 case DS_WRITE:
2882 NewMI = mergeWrite2Pair(CI, Paired, Where->I);
2883 break;
2884 case S_BUFFER_LOAD_IMM:
2885 case S_BUFFER_LOAD_SGPR_IMM:
2886 case S_LOAD_IMM:
2887 NewMI = mergeSMemLoadImmPair(CI, Paired, Where->I);
2888 OptimizeListAgain |= CI.Width + Paired.Width < 8;
2889 break;
2890 case BUFFER_LOAD:
2891 NewMI = mergeBufferLoadPair(CI, Paired, Where->I);
2892 OptimizeListAgain |= CI.Width + Paired.Width < 4;
2893 break;
2894 case BUFFER_STORE:
2895 NewMI = mergeBufferStorePair(CI, Paired, Where->I);
2896 OptimizeListAgain |= CI.Width + Paired.Width < 4;
2897 break;
2898 case MIMG:
2899 NewMI = mergeImagePair(CI, Paired, Where->I);
2900 OptimizeListAgain |= CI.Width + Paired.Width < 4;
2901 break;
2902 case TBUFFER_LOAD:
2903 NewMI = mergeTBufferLoadPair(CI, Paired, Where->I);
2904 OptimizeListAgain |= CI.Width + Paired.Width < 4;
2905 break;
2906 case TBUFFER_STORE:
2907 NewMI = mergeTBufferStorePair(CI, Paired, Where->I);
2908 OptimizeListAgain |= CI.Width + Paired.Width < 4;
2909 break;
2910 case FLAT_LOAD:
2911 case FLAT_LOAD_SADDR:
2912 case GLOBAL_LOAD:
2913 case GLOBAL_LOAD_SADDR:
2914 NewMI = mergeFlatLoadPair(CI, Paired, Where->I);
2915 OptimizeListAgain |= CI.Width + Paired.Width < 4;
2916 break;
2917 case FLAT_STORE:
2918 case FLAT_STORE_SADDR:
2919 case GLOBAL_STORE:
2920 case GLOBAL_STORE_SADDR:
2921 NewMI = mergeFlatStorePair(CI, Paired, Where->I);
2922 OptimizeListAgain |= CI.Width + Paired.Width < 4;
2923 break;
2924 }
2925 CI.setMI(NewMI, *this);
2926 CI.Order = Where->Order;
2927 if (I == Second)
2928 I = Next;
2929
2930 MergeList.erase(Second);
2931 }
2932
2933 return Modified;
2934}
2935
2936bool SILoadStoreOptimizerLegacy::runOnMachineFunction(MachineFunction &MF) {
2937 if (skipFunction(MF.getFunction()))
2938 return false;
2939 return SILoadStoreOptimizer(
2940 &getAnalysis<AAResultsWrapperPass>().getAAResults())
2941 .run(MF);
2942}
2943
2944bool SILoadStoreOptimizer::run(MachineFunction &MF) {
2945 this->MF = &MF;
2946 STM = &MF.getSubtarget<GCNSubtarget>();
2947 if (!STM->loadStoreOptEnabled())
2948 return false;
2949
2950 TII = STM->getInstrInfo();
2951 TRI = &TII->getRegisterInfo();
2952
2953 MRI = &MF.getRegInfo();
2954
2955 LLVM_DEBUG(dbgs() << "Running SILoadStoreOptimizer\n");
2956
2957 bool Modified = false;
2958
2959 // Contains the list of instructions for which constant offsets are being
2960 // promoted to the IMM. This is tracked for an entire block at time.
2961 SmallPtrSet<MachineInstr *, 4> AnchorList;
2962 MemInfoMap Visited;
2963
2964 for (MachineBasicBlock &MBB : MF) {
2965 MachineBasicBlock::iterator SectionEnd;
2966 for (MachineBasicBlock::iterator I = MBB.begin(), E = MBB.end(); I != E;
2967 I = SectionEnd) {
2968 bool CollectModified;
2969 std::list<std::list<CombineInfo>> MergeableInsts;
2970
2971 // First pass: Collect list of all instructions we know how to merge in a
2972 // subset of the block.
2973 std::tie(SectionEnd, CollectModified) =
2974 collectMergeableInsts(I, E, Visited, AnchorList, MergeableInsts);
2975
2976 Modified |= CollectModified;
2977
2978 do {
2979 OptimizeAgain = false;
2980 Modified |= optimizeBlock(MergeableInsts);
2981 } while (OptimizeAgain);
2982 }
2983
2984 Visited.clear();
2985 AnchorList.clear();
2986 }
2987
2988 return Modified;
2989}
2990
2991PreservedAnalyses
2992SILoadStoreOptimizerPass::run(MachineFunction &MF,
2993 MachineFunctionAnalysisManager &MFAM) {
2994 MFPropsModifier _(*this, MF);
2995
2996 if (MF.getFunction().hasOptNone())
2997 return PreservedAnalyses::all();
2998
2999 auto &FAM = MFAM.getResult<FunctionAnalysisManagerMachineFunctionProxy>(MF)
3000 .getManager();
3001 AAResults &AA = FAM.getResult<AAManager>(MF.getFunction());
3002
3003 bool Changed = SILoadStoreOptimizer(&AA).run(MF);
3004 if (!Changed)
3005 return PreservedAnalyses::all();
3006
3007 PreservedAnalyses PA = getMachineFunctionPassPreservedAnalyses();
3008 PA.preserveSet<CFGAnalyses>();
3009 return PA;
3010}
TLI
const TargetLowering & TLI
Definition AArch64ISelLowering.cpp:28479
assert
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
const
aarch64 promote const
Definition AArch64PromoteConstant.cpp:223
uint64_t
uint64_t uint64_t
Definition AMDGPUDisassembler.cpp:553
addInstToMergeableList
INITIALIZE_PASS(AMDGPUImageIntrinsicOptimizer, DEBUG_TYPE, "AMDGPU Image Intrinsic Optimizer", false, false) char AMDGPUImageIntrinsicOptimizer void addInstToMergeableList(IntrinsicInst *II, SmallVector< SmallVector< IntrinsicInst *, 4 > > &MergeableInsts, const AMDGPU::ImageDimIntrinsicInfo *ImageDimIntr)
Definition AMDGPUImageIntrinsicOptimizer.cpp:98
collectMergeableInsts
BasicBlock::iterator collectMergeableInsts(BasicBlock::iterator I, BasicBlock::iterator E, SmallVector< SmallVector< IntrinsicInst *, 4 > > &MergeableInsts)
Definition AMDGPUImageIntrinsicOptimizer.cpp:142
MBB
MachineBasicBlock & MBB
Definition ARMSLSHardening.cpp:71
DL
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Definition ARMSLSHardening.cpp:73
MBBI
MachineBasicBlock MachineBasicBlock::iterator MBBI
Definition ARMSLSHardening.cpp:72
A
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
E
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
B
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
GCNSubtarget.h
AMD GCN specific subclass of TargetSubtarget.
DEBUG_TYPE
#define DEBUG_TYPE
Definition GenericCycleImpl.h:34
op
#define op(i)
TII
const HexagonInstrInfo * TII
Definition HexagonCopyToCombine.cpp:118
_
#define _
Definition HexagonMCCodeEmitter.cpp:46
getAlign
static MaybeAlign getAlign(Value *Ptr)
Definition IRBuilder.cpp:543
MI
IRTranslator LLVM IR MI
Definition IRTranslator.cpp:870
OffsetOp
std::pair< Instruction::BinaryOps, Value * > OffsetOp
Find all possible pairs (BinOp, RHS) that BinOp V, RHS can be simplified.
Definition InstCombineCompares.cpp:6221
I
#define I(x, y, z)
Definition MD5.cpp:57
Reg
Register Reg
Definition MachineSink.cpp:2127
TRI
Register const TargetRegisterInfo * TRI
Definition MachineSink.cpp:2128
Register
Promote Memory to Register
Definition Mem2Reg.cpp:110
getReg
static MCRegister getReg(const MCDisassembler *D, unsigned RC, unsigned RegNo)
Definition MipsDisassembler.cpp:106
FAM
FunctionAnalysisManager FAM
Definition PassBuilderBindings.cpp:61
INITIALIZE_PASS_DEPENDENCY
#define INITIALIZE_PASS_DEPENDENCY(depName)
Definition PassSupport.h:42
INITIALIZE_PASS_END
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
Definition PassSupport.h:44
INITIALIZE_PASS_BEGIN
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
Definition PassSupport.h:39
MRI
MachineRegisterInfo & MRI
Definition RISCVLateBranchOpt.cpp:60
Lo
MachineInstr *& Lo
Definition RISCVMergeBaseOffset.cpp:78
mostAlignedValueInRange
static uint32_t mostAlignedValueInRange(uint32_t Lo, uint32_t Hi)
Definition SILoadStoreOptimizer.cpp:1082
needsConstrainedOpcode
static bool needsConstrainedOpcode(const GCNSubtarget &STM, ArrayRef< MachineMemOperand * > MMOs, unsigned Width)
Definition SILoadStoreOptimizer.cpp:1870
addDefsUsesToList
static void addDefsUsesToList(const MachineInstr &MI, DenseSet< Register > &RegDefs, DenseSet< Register > &RegUses)
Definition SILoadStoreOptimizer.cpp:956
getBufferFormatWithCompCount
static unsigned getBufferFormatWithCompCount(unsigned OldFormat, unsigned ComponentCount, const GCNSubtarget &STI)
Definition SILoadStoreOptimizer.cpp:1051
optimizeBlock
static bool optimizeBlock(BasicBlock &BB, bool &ModifiedDT, const TargetTransformInfo &TTI, const DataLayout &DL, bool HasBranchDivergence, DomTreeUpdater *DTU)
Definition ScalarizeMaskedMemIntrin.cpp:1228
LLVM_DEBUG
#define LLVM_DEBUG(...)
Definition Debug.h:119
llvm::AAManager
A manager for alias analyses.
Definition AliasAnalysis.h:967
llvm::AAResultsWrapperPass
A wrapper pass to provide the legacy pass manager access to a suitably prepared AAResults object.
Definition AliasAnalysis.h:1015
llvm::AnalysisManager::getResult
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
Definition PassManager.h:436
llvm::AnalysisUsage
Represent the analysis usage information of a pass.
Definition PassAnalysisSupport.h:48
llvm::AnalysisUsage::addRequired
AnalysisUsage & addRequired()
Definition PassAnalysisSupport.h:76
llvm::AnalysisUsage::setPreservesCFG
LLVM_ABI void setPreservesCFG()
This function should be called by the pass, iff they do not:
Definition Pass.cpp:280
llvm::ArrayRef
Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition ArrayRef.h:40
llvm::ArrayRef::size
size_t size() const
Get the array size.
Definition ArrayRef.h:141
llvm::CFGAnalyses
Represents analyses that only rely on functions' control flow.
Definition Analysis.h:73
llvm::DebugLoc
A debug info location.
Definition DebugLoc.h:126
llvm::DebugLoc::getMergedLocation
static LLVM_ABI DebugLoc getMergedLocation(DebugLoc LocA, DebugLoc LocB)
When two instructions are combined into a single instruction we also need to combine the original loc...
Definition DebugLoc.cpp:186
llvm::DenseSet
Implements a dense probed hash-table based set.
Definition DenseSet.h:281
llvm::Function::hasOptNone
bool hasOptNone() const
Do not optimize this function (-O0).
Definition Function.h:686
llvm::GCNSubtarget::loadStoreOptEnabled
bool loadStoreOptEnabled() const
Definition GCNSubtarget.h:544
llvm::GCNSubtarget::getInstrInfo
const SIInstrInfo * getInstrInfo() const override
Definition GCNSubtarget.h:135
llvm::GCNSubtarget::hasDwordx3LoadStores
bool hasDwordx3LoadStores() const
Definition GCNSubtarget.h:555
llvm::GCNSubtarget::getTargetLowering
const SITargetLowering * getTargetLowering() const override
Definition GCNSubtarget.h:141
llvm::GCNSubtarget::hasRelaxedTBufferOOBMode
bool hasRelaxedTBufferOOBMode() const
Definition GCNSubtarget.h:349
llvm::GCNSubtarget::ldsRequiresM0Init
bool ldsRequiresM0Init() const
Return if most LDS instructions have an m0 use that require m0 to be initialized.
Definition GCNSubtarget.h:408
llvm::GCNSubtarget::hasFlatScratchSVSSwizzleBug
bool hasFlatScratchSVSSwizzleBug() const
Definition GCNSubtarget.h:658
llvm::GCNSubtarget::isXNACKEnabled
bool isXNACKEnabled() const
Definition GCNSubtarget.h:346
llvm::HexagonInstrInfo::getRegisterInfo
const HexagonRegisterInfo & getRegisterInfo() const
Definition HexagonInstrInfo.h:53
llvm::LocationSize::getValue
TypeSize getValue() const
Definition MemoryLocation.h:158
llvm::MCInstrDesc::getOpcode
unsigned getOpcode() const
Return the opcode number for this descriptor.
Definition MCInstrDesc.h:233
llvm::MFPropsModifier
An RAII based helper class to modify MachineFunctionProperties when running pass.
Definition MachinePassManager.h:40
llvm::MachineBasicBlock::getParent
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
Definition MachineBasicBlock.h:322
llvm::MachineBasicBlock::iterator
MachineInstrBundleIterator< MachineInstr > iterator
Definition MachineBasicBlock.h:340
llvm::MachineFunctionPass
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
Definition MachineFunctionPass.h:31
llvm::MachineFunctionPass::getAnalysisUsage
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
Definition MachineFunctionPass.cpp:148
llvm::MachineFunctionProperties
Properties which a MachineFunction may have at a given point in time.
Definition MachineFunction.h:138
llvm::MachineFunction::getSubtarget
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
Definition MachineFunction.h:778
llvm::MachineFunction::getRegInfo
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Definition MachineFunction.h:788
llvm::MachineFunction::getFunction
Function & getFunction()
Return the LLVM function that this machine code represents.
Definition MachineFunction.h:739
llvm::MachineFunction::getMachineMemOperand
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags F, LLT MemTy, Align BaseAlignment, const MMOMetadata &Metadata=MMOMetadata(), SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
Definition MachineFunction.cpp:583
llvm::MachineInstrBuilder::cloneMergedMemRefs
const MachineInstrBuilder & cloneMergedMemRefs(ArrayRef< const MachineInstr * > OtherMIs) const
Definition MachineInstrBuilder.h:321
llvm::MachineInstrBuilder::addReg
const MachineInstrBuilder & addReg(Register RegNo, RegState Flags={}, unsigned SubReg=0) const
Add a new virtual register operand.
Definition MachineInstrBuilder.h:199
llvm::MachineInstrBuilder::addImm
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
Definition MachineInstrBuilder.h:233
llvm::MachineInstrBuilder::add
const MachineInstrBuilder & add(const MachineOperand &MO) const
Definition MachineInstrBuilder.h:326
llvm::MachineInstr
Representation of each machine instruction.
Definition MachineInstr.h:74
llvm::MachineInstr::getOpcode
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
Definition MachineInstr.h:609
llvm::MachineInstr::dump
LLVM_ABI void dump() const
Definition MachineInstr.cpp:1782
llvm::MachineMemOperand
A description of a memory reference used in the backend.
Definition MachineMemOperand.h:141
llvm::MachineMemOperand::getSize
LocationSize getSize() const
Return the size in bytes of the memory reference.
Definition MachineMemOperand.h:255
llvm::MachineMemOperand::getAddrSpace
unsigned getAddrSpace() const
Definition MachineMemOperand.h:248
llvm::MachineMemOperand::getPointerInfo
const MachinePointerInfo & getPointerInfo() const
Definition MachineMemOperand.h:219
llvm::MachineOperand
MachineOperand class - Representation of each machine instruction operand.
Definition MachineOperand.h:49
llvm::MachineOperand::getSubReg
unsigned getSubReg() const
Definition MachineOperand.h:380
llvm::MachineOperand::getImm
int64_t getImm() const
Definition MachineOperand.h:563
llvm::MachineOperand::isReg
bool isReg() const
isReg - Tests if this is a MO_Register operand.
Definition MachineOperand.h:334
llvm::MachineOperand::setReg
LLVM_ABI void setReg(Register Reg)
Change the register this operand corresponds to.
Definition MachineOperand.cpp:60
llvm::MachineOperand::isImm
bool isImm() const
isImm - Tests if this is a MO_Immediate operand.
Definition MachineOperand.h:336
llvm::MachineOperand::CreateImm
static MachineOperand CreateImm(int64_t Val)
Definition MachineOperand.h:836
llvm::MachineOperand::getReg
Register getReg() const
getReg - Returns the register number.
Definition MachineOperand.h:375
llvm::MachineOperand::CreateReg
static MachineOperand CreateReg(Register Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isEarlyClobber=false, unsigned SubReg=0, bool isDebug=false, bool isInternalRead=false, bool isRenamable=false)
Definition MachineOperand.h:854
llvm::MachineRegisterInfo
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
Definition MachineRegisterInfo.h:55
llvm::MachineRegisterInfo::getRegClass
const TargetRegisterClass * getRegClass(Register Reg) const
Return the register class of the specified virtual register.
Definition MachineRegisterInfo.h:684
llvm::MachineRegisterInfo::createVirtualRegister
LLVM_ABI Register createVirtualRegister(const TargetRegisterClass *RegClass, StringRef Name="")
createVirtualRegister - Create and return a new virtual register in the function with the specified r...
Definition MachineRegisterInfo.cpp:156
llvm::MachineRegisterInfo::constrainRegClass
LLVM_ABI const TargetRegisterClass * constrainRegClass(Register Reg, const TargetRegisterClass *RC, unsigned MinNumRegs=0)
constrainRegClass - Constrain the register class of the specified virtual register to be a common sub...
Definition MachineRegisterInfo.cpp:86
llvm::MachineRegisterInfo::getUniqueVRegDef
LLVM_ABI LLVM_READONLY MachineInstr * getUniqueVRegDef(Register Reg) const
getUniqueVRegDef - Return the unique machine instr that defines the specified virtual register or nul...
Definition MachineRegisterInfo.cpp:419
llvm::PreservedAnalyses
A set of analyses that are preserved following a run of a transformation pass.
Definition Analysis.h:112
llvm::PreservedAnalyses::all
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Definition Analysis.h:118
llvm::PreservedAnalyses::preserveSet
PreservedAnalyses & preserveSet()
Mark an analysis set as preserved.
Definition Analysis.h:151
llvm::Register
Wrapper class representing virtual and physical registers.
Definition Register.h:20
llvm::Register::isPhysical
constexpr bool isPhysical() const
Return true if the specified register number is in the physical register namespace.
Definition Register.h:83
llvm::SIInstrInfo::isFLATScratch
static bool isFLATScratch(const MachineInstr &MI)
Definition SIInstrInfo.h:748
llvm::SIInstrInfo::isVIMAGE
static bool isVIMAGE(const MachineInstr &MI)
Definition SIInstrInfo.h:702
llvm::SIInstrInfo::isFLATGlobal
static bool isFLATGlobal(const MachineInstr &MI)
Definition SIInstrInfo.h:740
llvm::SIInstrInfo::isVSAMPLE
static bool isVSAMPLE(const MachineInstr &MI)
Definition SIInstrInfo.h:710
llvm::SIInstrInfo::isFLAT
static bool isFLAT(const MachineInstr &MI)
Definition SIInstrInfo.h:726
llvm::SIInstrInfo::getNamedOperand
LLVM_READONLY MachineOperand * getNamedOperand(MachineInstr &MI, AMDGPU::OpName OperandName) const
Returns the operand named Op.
Definition SIInstrInfo.cpp:10237
llvm::SILoadStoreOptimizerPass::run
PreservedAnalyses run(MachineFunction &MF, MachineFunctionAnalysisManager &MFAM)
Definition SILoadStoreOptimizer.cpp:2992
llvm::SmallPtrSet
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
Definition SmallPtrSet.h:519
llvm::SmallVectorImpl::emplace_back
reference emplace_back(ArgTypes &&... Args)
Definition SmallVector.h:995
llvm::StringRef
Represent a constant reference to a string, i.e.
Definition StringRef.h:56
llvm::detail::DenseSetImpl::contains
bool contains(const_arg_type_t< ValueT > V) const
Check if the set contains the given element.
Definition DenseSet.h:182
llvm::ilist_node_impl::getIterator
self_iterator getIterator()
Definition ilist_node.h:123
uint32_t
llvm_unreachable
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition ErrorHandling.h:164
llvm::AA
Abstract Attribute helper functions.
Definition Attributor.h:164
llvm::AMDGPUAS::FLAT_ADDRESS
@ FLAT_ADDRESS
Address space for flat memory.
Definition AMDGPUAddrSpace.h:31
llvm::AMDGPUAS::GLOBAL_ADDRESS
@ GLOBAL_ADDRESS
Address space for global memory (RAT0, VTX0).
Definition AMDGPUAddrSpace.h:32
llvm::AMDGPUAS::PRIVATE_ADDRESS
@ PRIVATE_ADDRESS
Address space for private memory.
Definition AMDGPUAddrSpace.h:37
llvm::AMDGPU::getMIMGInfo
LLVM_READONLY const MIMGInfo * getMIMGInfo(unsigned Opc)
llvm::AMDGPU::convertSMRDOffsetUnits
uint64_t convertSMRDOffsetUnits(const MCSubtargetInfo &ST, uint64_t ByteOffset)
Convert ByteOffset to dwords if the subtarget uses dword SMRD immediate offsets.
Definition AMDGPUBaseInfo.cpp:3288
llvm::AMDGPU::getMTBUFHasSrsrc
bool getMTBUFHasSrsrc(unsigned Opc)
Definition AMDGPUBaseInfo.cpp:510
llvm::AMDGPU::getMTBUFElements
int getMTBUFElements(unsigned Opc)
Definition AMDGPUBaseInfo.cpp:500
llvm::AMDGPU::getMTBUFHasSoffset
bool getMTBUFHasSoffset(unsigned Opc)
Definition AMDGPUBaseInfo.cpp:515
llvm::AMDGPU::getMUBUFOpcode
int getMUBUFOpcode(unsigned BaseOpc, unsigned Elements)
Definition AMDGPUBaseInfo.cpp:525
llvm::AMDGPU::getMUBUFBaseOpcode
int getMUBUFBaseOpcode(unsigned Opc)
Definition AMDGPUBaseInfo.cpp:520
llvm::AMDGPU::hasNamedOperand
LLVM_READONLY bool hasNamedOperand(uint32_t Opcode, OpName NamedIdx)
Definition AMDGPUBaseInfo.h:349
llvm::AMDGPU::getMTBUFBaseOpcode
int getMTBUFBaseOpcode(unsigned Opc)
Definition AMDGPUBaseInfo.cpp:489
llvm::AMDGPU::getMUBUFHasVAddr
bool getMUBUFHasVAddr(unsigned Opc)
Definition AMDGPUBaseInfo.cpp:536
llvm::AMDGPU::getMTBUFOpcode
int getMTBUFOpcode(unsigned BaseOpc, unsigned Elements)
Definition AMDGPUBaseInfo.cpp:494
llvm::AMDGPU::getMUBUFHasSoffset
bool getMUBUFHasSoffset(unsigned Opc)
Definition AMDGPUBaseInfo.cpp:546
llvm::AMDGPU::getMIMGBaseOpcode
const MIMGBaseOpcodeInfo * getMIMGBaseOpcode(unsigned Opc)
Definition AMDGPUBaseInfo.cpp:328
llvm::AMDGPU::getMIMGBaseOpcodeInfo
LLVM_READONLY const MIMGBaseOpcodeInfo * getMIMGBaseOpcodeInfo(unsigned BaseOpcode)
llvm::AMDGPU::getMaskedMIMGOp
int getMaskedMIMGOp(unsigned Opc, unsigned NewChannels)
Definition AMDGPUBaseInfo.cpp:333
llvm::AMDGPU::getMTBUFHasVAddr
bool getMTBUFHasVAddr(unsigned Opc)
Definition AMDGPUBaseInfo.cpp:505
llvm::AMDGPU::getMUBUFElements
int getMUBUFElements(unsigned Opc)
Definition AMDGPUBaseInfo.cpp:531
llvm::AMDGPU::getGcnBufferFormatInfo
const GcnBufferFormatInfo * getGcnBufferFormatInfo(uint8_t BitsPerComp, uint8_t NumComponents, uint8_t NumFormat, const MCSubtargetInfo &STI)
Definition AMDGPUBaseInfo.cpp:3399
llvm::AMDGPU::getMUBUFHasSrsrc
bool getMUBUFHasSrsrc(unsigned Opc)
Definition AMDGPUBaseInfo.cpp:541
llvm::ARM_AM::add
@ add
Definition ARMAddressingModes.h:39
llvm::BitmaskEnumDetail::Mask
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
Definition BitmaskEnum.h:126
llvm::dwarf_linker::DebugSectionKind::DebugLoc
@ DebugLoc
Definition DWARFLinkerBase.h:35
llvm::ms_demangle::IntrinsicFunctionKind::New
@ New
Definition MicrosoftDemangleNodes.h:123
llvm::rdf::Def
NodeAddr< DefNode * > Def
Definition RDFGraph.h:384
llvm::sframe::BaseReg
BaseReg
Stack frame base register. Bit 0 of FREInfo.Info.
Definition SFrame.h:77
llvm
This is an optimization pass for GlobalISel generic memory operations.
Definition DefaultTargetInfo.h:21
llvm::Offset
@ Offset
Definition DWP.cpp:577
llvm::operator<
bool operator<(int64_t V1, const APSInt &V2)
Definition APSInt.h:360
llvm::BuildMI
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
Definition MachineInstrBuilder.h:449
llvm::maskLeadingOnes
constexpr T maskLeadingOnes(unsigned N)
Create a bitmask with the N left-most bits set to 1, and all other bits set to 0.
Definition MathExtras.h:89
llvm::MachineFunctionAnalysisManager
AnalysisManager< MachineFunction > MachineFunctionAnalysisManager
Definition MachineFunctionAnalysisManager.h:24
llvm::SILoadStoreOptimizerLegacyID
char & SILoadStoreOptimizerLegacyID
Definition SILoadStoreOptimizer.cpp:954
llvm::popcount
constexpr int popcount(T Value) noexcept
Count the number of set bits in a value.
Definition bit.h:156
llvm::countr_zero
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
Definition bit.h:204
llvm::getMachineFunctionPassPreservedAnalyses
LLVM_ABI PreservedAnalyses getMachineFunctionPassPreservedAnalyses()
Returns the minimum set of Analyses that all machine function passes must preserve.
Definition MachinePassManager.cpp:162
llvm::countl_zero
int countl_zero(T Val)
Count number of 0's from the most significant bit to the least stopping at the first 1.
Definition bit.h:263
llvm::dbgs
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition Debug.cpp:209
llvm::isUInt
constexpr bool isUInt(uint64_t x)
Checks if an unsigned integer fits into the given bit width.
Definition MathExtras.h:190
llvm::SmallVector
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
Definition SmallVector.h:1187
llvm::IRMemLocation::Other
@ Other
Any other memory.
Definition ModRef.h:68
llvm::IRMemLocation::First
@ First
Helpers to iterate all locations in the MemoryEffectsBase class.
Definition ModRef.h:74
llvm::Op
DWARFExpression::Operation Op
Definition DWARFExpressionPrinter.cpp:25
llvm::AnchorList
std::vector< std::pair< LineLocation, FunctionId > > AnchorList
Definition SampleProfileMatcher.h:25
llvm::BitWidth
constexpr unsigned BitWidth
Definition BitmaskEnum.h:219
llvm::Next
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Next
Definition InstrProf.h:147
llvm::maskTrailingOnes
constexpr T maskTrailingOnes(unsigned N)
Create a bitmask with the N right-most bits set to 1, and all other bits set to 0.
Definition MathExtras.h:78
llvm::MachineVerifierMode::End
@ End
Run the machine verifier once, at the end of the pipeline.
Definition CodeGen.h:320
llvm::AliasAnalysis
AAResults AliasAnalysis
Temporary typedef for legacy code that uses a generic AliasAnalysis pointer or reference.
Definition AliasAnalysis.h:692
llvm::printReg
LLVM_ABI Printable printReg(Register Reg, const TargetRegisterInfo *TRI=nullptr, unsigned SubIdx=0, const MachineRegisterInfo *MRI=nullptr)
Prints virtual and physical registers with or without a TRI instance.
Definition TargetRegisterInfo.cpp:106
llvm::TargetRegisterClass
MCRegisterClass TargetRegisterClass
Definition FastISel.h:58
std::swap
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition BitVector.h:880
Generated on for LLVM by doxygen 1.14.0