Bug Summary

File:llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
Warning:line 2010, column 15
Called C++ object pointer is uninitialized

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name SIInstrInfo.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mframe-pointer=none -fmath-errno -fno-rounding-math -mconstructor-aliases -munwind-tables -target-cpu x86-64 -tune-cpu generic -debugger-tuning=gdb -ffunction-sections -fdata-sections -fcoverage-compilation-dir=/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/build-llvm/lib/Target/AMDGPU -resource-dir /usr/lib/llvm-13/lib/clang/13.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/build-llvm/lib/Target/AMDGPU -I /build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU -I /build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/build-llvm/include -I /build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/include -D NDEBUG -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/x86_64-linux-gnu/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10/backward -internal-isystem /usr/lib/llvm-13/lib/clang/13.0.0/include -internal-isystem /usr/local/include -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../x86_64-linux-gnu/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-class-memaccess -Wno-redundant-move -Wno-pessimizing-move -Wno-noexcept-type -Wno-comment -std=c++14 -fdeprecated-macro -fdebug-compilation-dir=/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/build-llvm/lib/Target/AMDGPU -fdebug-prefix-map=/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b=. -ferror-limit 19 -fvisibility hidden -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -D__GCC_HAVE_DWARF2_CFI_ASM=1 -o /tmp/scan-build-2021-06-17-010711-25934-1 -x c++ /build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp

/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp

1//===- SIInstrInfo.cpp - SI Instruction Information ----------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// SI Implementation of TargetInstrInfo.
11//
12//===----------------------------------------------------------------------===//
13
14#include "SIInstrInfo.h"
15#include "AMDGPU.h"
16#include "AMDGPUInstrInfo.h"
17#include "GCNHazardRecognizer.h"
18#include "GCNSubtarget.h"
19#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
20#include "SIMachineFunctionInfo.h"
21#include "llvm/Analysis/ValueTracking.h"
22#include "llvm/CodeGen/LiveVariables.h"
23#include "llvm/CodeGen/MachineDominators.h"
24#include "llvm/CodeGen/RegisterScavenging.h"
25#include "llvm/CodeGen/ScheduleDAG.h"
26#include "llvm/IR/DiagnosticInfo.h"
27#include "llvm/IR/IntrinsicsAMDGPU.h"
28#include "llvm/Support/CommandLine.h"
29#include "llvm/Target/TargetMachine.h"
30
31using namespace llvm;
32
33#define DEBUG_TYPE"si-instr-info" "si-instr-info"
34
35#define GET_INSTRINFO_CTOR_DTOR
36#include "AMDGPUGenInstrInfo.inc"
37
38namespace llvm {
39
40class AAResults;
41
42namespace AMDGPU {
43#define GET_D16ImageDimIntrinsics_IMPL
44#define GET_ImageDimIntrinsicTable_IMPL
45#define GET_RsrcIntrinsics_IMPL
46#include "AMDGPUGenSearchableTables.inc"
47}
48}
49
50
51// Must be at least 4 to be able to branch over minimum unconditional branch
52// code. This is only for making it possible to write reasonably small tests for
53// long branches.
54static cl::opt<unsigned>
55BranchOffsetBits("amdgpu-s-branch-bits", cl::ReallyHidden, cl::init(16),
56 cl::desc("Restrict range of branch instructions (DEBUG)"));
57
58static cl::opt<bool> Fix16BitCopies(
59 "amdgpu-fix-16-bit-physreg-copies",
60 cl::desc("Fix copies between 32 and 16 bit registers by extending to 32 bit"),
61 cl::init(true),
62 cl::ReallyHidden);
63
64SIInstrInfo::SIInstrInfo(const GCNSubtarget &ST)
65 : AMDGPUGenInstrInfo(AMDGPU::ADJCALLSTACKUP, AMDGPU::ADJCALLSTACKDOWN),
66 RI(ST), ST(ST) {
67 SchedModel.init(&ST);
68}
69
70//===----------------------------------------------------------------------===//
71// TargetInstrInfo callbacks
72//===----------------------------------------------------------------------===//
73
74static unsigned getNumOperandsNoGlue(SDNode *Node) {
75 unsigned N = Node->getNumOperands();
76 while (N && Node->getOperand(N - 1).getValueType() == MVT::Glue)
77 --N;
78 return N;
79}
80
81/// Returns true if both nodes have the same value for the given
82/// operand \p Op, or if both nodes do not have this operand.
83static bool nodesHaveSameOperandValue(SDNode *N0, SDNode* N1, unsigned OpName) {
84 unsigned Opc0 = N0->getMachineOpcode();
85 unsigned Opc1 = N1->getMachineOpcode();
86
87 int Op0Idx = AMDGPU::getNamedOperandIdx(Opc0, OpName);
88 int Op1Idx = AMDGPU::getNamedOperandIdx(Opc1, OpName);
89
90 if (Op0Idx == -1 && Op1Idx == -1)
91 return true;
92
93
94 if ((Op0Idx == -1 && Op1Idx != -1) ||
95 (Op1Idx == -1 && Op0Idx != -1))
96 return false;
97
98 // getNamedOperandIdx returns the index for the MachineInstr's operands,
99 // which includes the result as the first operand. We are indexing into the
100 // MachineSDNode's operands, so we need to skip the result operand to get
101 // the real index.
102 --Op0Idx;
103 --Op1Idx;
104
105 return N0->getOperand(Op0Idx) == N1->getOperand(Op1Idx);
106}
107
108bool SIInstrInfo::isReallyTriviallyReMaterializable(const MachineInstr &MI,
109 AAResults *AA) const {
110 // TODO: The generic check fails for VALU instructions that should be
111 // rematerializable due to implicit reads of exec. We really want all of the
112 // generic logic for this except for this.
113 switch (MI.getOpcode()) {
114 case AMDGPU::V_MOV_B32_e32:
115 case AMDGPU::V_MOV_B32_e64:
116 case AMDGPU::V_MOV_B64_PSEUDO:
117 case AMDGPU::V_ACCVGPR_READ_B32_e64:
118 case AMDGPU::V_ACCVGPR_WRITE_B32_e64:
119 // No non-standard implicit operands.
120 assert(MI.getDesc().getNumOperands() == 2)(static_cast <bool> (MI.getDesc().getNumOperands() == 2
) ? void (0) : __assert_fail ("MI.getDesc().getNumOperands() == 2"
, "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 120, __extension__ __PRETTY_FUNCTION__))
;
121 assert(MI.getDesc().getNumImplicitDefs() == 0)(static_cast <bool> (MI.getDesc().getNumImplicitDefs() ==
0) ? void (0) : __assert_fail ("MI.getDesc().getNumImplicitDefs() == 0"
, "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 121, __extension__ __PRETTY_FUNCTION__))
;
122 assert(MI.getDesc().getNumImplicitUses() == 1)(static_cast <bool> (MI.getDesc().getNumImplicitUses() ==
1) ? void (0) : __assert_fail ("MI.getDesc().getNumImplicitUses() == 1"
, "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 122, __extension__ __PRETTY_FUNCTION__))
;
123 return MI.getNumOperands() == 3;
124 default:
125 return false;
126 }
127}
128
129bool SIInstrInfo::areLoadsFromSameBasePtr(SDNode *Load0, SDNode *Load1,
130 int64_t &Offset0,
131 int64_t &Offset1) const {
132 if (!Load0->isMachineOpcode() || !Load1->isMachineOpcode())
133 return false;
134
135 unsigned Opc0 = Load0->getMachineOpcode();
136 unsigned Opc1 = Load1->getMachineOpcode();
137
138 // Make sure both are actually loads.
139 if (!get(Opc0).mayLoad() || !get(Opc1).mayLoad())
140 return false;
141
142 if (isDS(Opc0) && isDS(Opc1)) {
143
144 // FIXME: Handle this case:
145 if (getNumOperandsNoGlue(Load0) != getNumOperandsNoGlue(Load1))
146 return false;
147
148 // Check base reg.
149 if (Load0->getOperand(0) != Load1->getOperand(0))
150 return false;
151
152 // Skip read2 / write2 variants for simplicity.
153 // TODO: We should report true if the used offsets are adjacent (excluded
154 // st64 versions).
155 int Offset0Idx = AMDGPU::getNamedOperandIdx(Opc0, AMDGPU::OpName::offset);
156 int Offset1Idx = AMDGPU::getNamedOperandIdx(Opc1, AMDGPU::OpName::offset);
157 if (Offset0Idx == -1 || Offset1Idx == -1)
158 return false;
159
160 // XXX - be careful of datalesss loads
161 // getNamedOperandIdx returns the index for MachineInstrs. Since they
162 // include the output in the operand list, but SDNodes don't, we need to
163 // subtract the index by one.
164 Offset0Idx -= get(Opc0).NumDefs;
165 Offset1Idx -= get(Opc1).NumDefs;
166 Offset0 = cast<ConstantSDNode>(Load0->getOperand(Offset0Idx))->getZExtValue();
167 Offset1 = cast<ConstantSDNode>(Load1->getOperand(Offset1Idx))->getZExtValue();
168 return true;
169 }
170
171 if (isSMRD(Opc0) && isSMRD(Opc1)) {
172 // Skip time and cache invalidation instructions.
173 if (AMDGPU::getNamedOperandIdx(Opc0, AMDGPU::OpName::sbase) == -1 ||
174 AMDGPU::getNamedOperandIdx(Opc1, AMDGPU::OpName::sbase) == -1)
175 return false;
176
177 assert(getNumOperandsNoGlue(Load0) == getNumOperandsNoGlue(Load1))(static_cast <bool> (getNumOperandsNoGlue(Load0) == getNumOperandsNoGlue
(Load1)) ? void (0) : __assert_fail ("getNumOperandsNoGlue(Load0) == getNumOperandsNoGlue(Load1)"
, "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 177, __extension__ __PRETTY_FUNCTION__))
;
178
179 // Check base reg.
180 if (Load0->getOperand(0) != Load1->getOperand(0))
181 return false;
182
183 const ConstantSDNode *Load0Offset =
184 dyn_cast<ConstantSDNode>(Load0->getOperand(1));
185 const ConstantSDNode *Load1Offset =
186 dyn_cast<ConstantSDNode>(Load1->getOperand(1));
187
188 if (!Load0Offset || !Load1Offset)
189 return false;
190
191 Offset0 = Load0Offset->getZExtValue();
192 Offset1 = Load1Offset->getZExtValue();
193 return true;
194 }
195
196 // MUBUF and MTBUF can access the same addresses.
197 if ((isMUBUF(Opc0) || isMTBUF(Opc0)) && (isMUBUF(Opc1) || isMTBUF(Opc1))) {
198
199 // MUBUF and MTBUF have vaddr at different indices.
200 if (!nodesHaveSameOperandValue(Load0, Load1, AMDGPU::OpName::soffset) ||
201 !nodesHaveSameOperandValue(Load0, Load1, AMDGPU::OpName::vaddr) ||
202 !nodesHaveSameOperandValue(Load0, Load1, AMDGPU::OpName::srsrc))
203 return false;
204
205 int OffIdx0 = AMDGPU::getNamedOperandIdx(Opc0, AMDGPU::OpName::offset);
206 int OffIdx1 = AMDGPU::getNamedOperandIdx(Opc1, AMDGPU::OpName::offset);
207
208 if (OffIdx0 == -1 || OffIdx1 == -1)
209 return false;
210
211 // getNamedOperandIdx returns the index for MachineInstrs. Since they
212 // include the output in the operand list, but SDNodes don't, we need to
213 // subtract the index by one.
214 OffIdx0 -= get(Opc0).NumDefs;
215 OffIdx1 -= get(Opc1).NumDefs;
216
217 SDValue Off0 = Load0->getOperand(OffIdx0);
218 SDValue Off1 = Load1->getOperand(OffIdx1);
219
220 // The offset might be a FrameIndexSDNode.
221 if (!isa<ConstantSDNode>(Off0) || !isa<ConstantSDNode>(Off1))
222 return false;
223
224 Offset0 = cast<ConstantSDNode>(Off0)->getZExtValue();
225 Offset1 = cast<ConstantSDNode>(Off1)->getZExtValue();
226 return true;
227 }
228
229 return false;
230}
231
232static bool isStride64(unsigned Opc) {
233 switch (Opc) {
234 case AMDGPU::DS_READ2ST64_B32:
235 case AMDGPU::DS_READ2ST64_B64:
236 case AMDGPU::DS_WRITE2ST64_B32:
237 case AMDGPU::DS_WRITE2ST64_B64:
238 return true;
239 default:
240 return false;
241 }
242}
243
244bool SIInstrInfo::getMemOperandsWithOffsetWidth(
245 const MachineInstr &LdSt, SmallVectorImpl<const MachineOperand *> &BaseOps,
246 int64_t &Offset, bool &OffsetIsScalable, unsigned &Width,
247 const TargetRegisterInfo *TRI) const {
248 if (!LdSt.mayLoadOrStore())
249 return false;
250
251 unsigned Opc = LdSt.getOpcode();
252 OffsetIsScalable = false;
253 const MachineOperand *BaseOp, *OffsetOp;
254 int DataOpIdx;
255
256 if (isDS(LdSt)) {
257 BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::addr);
258 OffsetOp = getNamedOperand(LdSt, AMDGPU::OpName::offset);
259 if (OffsetOp) {
260 // Normal, single offset LDS instruction.
261 if (!BaseOp) {
262 // DS_CONSUME/DS_APPEND use M0 for the base address.
263 // TODO: find the implicit use operand for M0 and use that as BaseOp?
264 return false;
265 }
266 BaseOps.push_back(BaseOp);
267 Offset = OffsetOp->getImm();
268 // Get appropriate operand, and compute width accordingly.
269 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst);
270 if (DataOpIdx == -1)
271 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data0);
272 Width = getOpSize(LdSt, DataOpIdx);
273 } else {
274 // The 2 offset instructions use offset0 and offset1 instead. We can treat
275 // these as a load with a single offset if the 2 offsets are consecutive.
276 // We will use this for some partially aligned loads.
277 const MachineOperand *Offset0Op =
278 getNamedOperand(LdSt, AMDGPU::OpName::offset0);
279 const MachineOperand *Offset1Op =
280 getNamedOperand(LdSt, AMDGPU::OpName::offset1);
281
282 unsigned Offset0 = Offset0Op->getImm();
283 unsigned Offset1 = Offset1Op->getImm();
284 if (Offset0 + 1 != Offset1)
285 return false;
286
287 // Each of these offsets is in element sized units, so we need to convert
288 // to bytes of the individual reads.
289
290 unsigned EltSize;
291 if (LdSt.mayLoad())
292 EltSize = TRI->getRegSizeInBits(*getOpRegClass(LdSt, 0)) / 16;
293 else {
294 assert(LdSt.mayStore())(static_cast <bool> (LdSt.mayStore()) ? void (0) : __assert_fail
("LdSt.mayStore()", "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 294, __extension__ __PRETTY_FUNCTION__))
;
295 int Data0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data0);
296 EltSize = TRI->getRegSizeInBits(*getOpRegClass(LdSt, Data0Idx)) / 8;
297 }
298
299 if (isStride64(Opc))
300 EltSize *= 64;
301
302 BaseOps.push_back(BaseOp);
303 Offset = EltSize * Offset0;
304 // Get appropriate operand(s), and compute width accordingly.
305 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst);
306 if (DataOpIdx == -1) {
307 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data0);
308 Width = getOpSize(LdSt, DataOpIdx);
309 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data1);
310 Width += getOpSize(LdSt, DataOpIdx);
311 } else {
312 Width = getOpSize(LdSt, DataOpIdx);
313 }
314 }
315 return true;
316 }
317
318 if (isMUBUF(LdSt) || isMTBUF(LdSt)) {
319 const MachineOperand *RSrc = getNamedOperand(LdSt, AMDGPU::OpName::srsrc);
320 if (!RSrc) // e.g. BUFFER_WBINVL1_VOL
321 return false;
322 BaseOps.push_back(RSrc);
323 BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::vaddr);
324 if (BaseOp && !BaseOp->isFI())
325 BaseOps.push_back(BaseOp);
326 const MachineOperand *OffsetImm =
327 getNamedOperand(LdSt, AMDGPU::OpName::offset);
328 Offset = OffsetImm->getImm();
329 const MachineOperand *SOffset =
330 getNamedOperand(LdSt, AMDGPU::OpName::soffset);
331 if (SOffset) {
332 if (SOffset->isReg())
333 BaseOps.push_back(SOffset);
334 else
335 Offset += SOffset->getImm();
336 }
337 // Get appropriate operand, and compute width accordingly.
338 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst);
339 if (DataOpIdx == -1)
340 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdata);
341 Width = getOpSize(LdSt, DataOpIdx);
342 return true;
343 }
344
345 if (isMIMG(LdSt)) {
346 int SRsrcIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::srsrc);
347 BaseOps.push_back(&LdSt.getOperand(SRsrcIdx));
348 int VAddr0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vaddr0);
349 if (VAddr0Idx >= 0) {
350 // GFX10 possible NSA encoding.
351 for (int I = VAddr0Idx; I < SRsrcIdx; ++I)
352 BaseOps.push_back(&LdSt.getOperand(I));
353 } else {
354 BaseOps.push_back(getNamedOperand(LdSt, AMDGPU::OpName::vaddr));
355 }
356 Offset = 0;
357 // Get appropriate operand, and compute width accordingly.
358 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdata);
359 Width = getOpSize(LdSt, DataOpIdx);
360 return true;
361 }
362
363 if (isSMRD(LdSt)) {
364 BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::sbase);
365 if (!BaseOp) // e.g. S_MEMTIME
366 return false;
367 BaseOps.push_back(BaseOp);
368 OffsetOp = getNamedOperand(LdSt, AMDGPU::OpName::offset);
369 Offset = OffsetOp ? OffsetOp->getImm() : 0;
370 // Get appropriate operand, and compute width accordingly.
371 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::sdst);
372 Width = getOpSize(LdSt, DataOpIdx);
373 return true;
374 }
375
376 if (isFLAT(LdSt)) {
377 // Instructions have either vaddr or saddr or both or none.
378 BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::vaddr);
379 if (BaseOp)
380 BaseOps.push_back(BaseOp);
381 BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::saddr);
382 if (BaseOp)
383 BaseOps.push_back(BaseOp);
384 Offset = getNamedOperand(LdSt, AMDGPU::OpName::offset)->getImm();
385 // Get appropriate operand, and compute width accordingly.
386 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst);
387 if (DataOpIdx == -1)
388 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdata);
389 Width = getOpSize(LdSt, DataOpIdx);
390 return true;
391 }
392
393 return false;
394}
395
396static bool memOpsHaveSameBasePtr(const MachineInstr &MI1,
397 ArrayRef<const MachineOperand *> BaseOps1,
398 const MachineInstr &MI2,
399 ArrayRef<const MachineOperand *> BaseOps2) {
400 // Only examine the first "base" operand of each instruction, on the
401 // assumption that it represents the real base address of the memory access.
402 // Other operands are typically offsets or indices from this base address.
403 if (BaseOps1.front()->isIdenticalTo(*BaseOps2.front()))
404 return true;
405
406 if (!MI1.hasOneMemOperand() || !MI2.hasOneMemOperand())
407 return false;
408
409 auto MO1 = *MI1.memoperands_begin();
410 auto MO2 = *MI2.memoperands_begin();
411 if (MO1->getAddrSpace() != MO2->getAddrSpace())
412 return false;
413
414 auto Base1 = MO1->getValue();
415 auto Base2 = MO2->getValue();
416 if (!Base1 || !Base2)
417 return false;
418 Base1 = getUnderlyingObject(Base1);
419 Base2 = getUnderlyingObject(Base2);
420
421 if (isa<UndefValue>(Base1) || isa<UndefValue>(Base2))
422 return false;
423
424 return Base1 == Base2;
425}
426
427bool SIInstrInfo::shouldClusterMemOps(ArrayRef<const MachineOperand *> BaseOps1,
428 ArrayRef<const MachineOperand *> BaseOps2,
429 unsigned NumLoads,
430 unsigned NumBytes) const {
431 // If the mem ops (to be clustered) do not have the same base ptr, then they
432 // should not be clustered
433 if (!BaseOps1.empty() && !BaseOps2.empty()) {
434 const MachineInstr &FirstLdSt = *BaseOps1.front()->getParent();
435 const MachineInstr &SecondLdSt = *BaseOps2.front()->getParent();
436 if (!memOpsHaveSameBasePtr(FirstLdSt, BaseOps1, SecondLdSt, BaseOps2))
437 return false;
438 } else if (!BaseOps1.empty() || !BaseOps2.empty()) {
439 // If only one base op is empty, they do not have the same base ptr
440 return false;
441 }
442
443 // In order to avoid regester pressure, on an average, the number of DWORDS
444 // loaded together by all clustered mem ops should not exceed 8. This is an
445 // empirical value based on certain observations and performance related
446 // experiments.
447 // The good thing about this heuristic is - it avoids clustering of too many
448 // sub-word loads, and also avoids clustering of wide loads. Below is the
449 // brief summary of how the heuristic behaves for various `LoadSize`.
450 // (1) 1 <= LoadSize <= 4: cluster at max 8 mem ops
451 // (2) 5 <= LoadSize <= 8: cluster at max 4 mem ops
452 // (3) 9 <= LoadSize <= 12: cluster at max 2 mem ops
453 // (4) 13 <= LoadSize <= 16: cluster at max 2 mem ops
454 // (5) LoadSize >= 17: do not cluster
455 const unsigned LoadSize = NumBytes / NumLoads;
456 const unsigned NumDWORDs = ((LoadSize + 3) / 4) * NumLoads;
457 return NumDWORDs <= 8;
458}
459
460// FIXME: This behaves strangely. If, for example, you have 32 load + stores,
461// the first 16 loads will be interleaved with the stores, and the next 16 will
462// be clustered as expected. It should really split into 2 16 store batches.
463//
464// Loads are clustered until this returns false, rather than trying to schedule
465// groups of stores. This also means we have to deal with saying different
466// address space loads should be clustered, and ones which might cause bank
467// conflicts.
468//
469// This might be deprecated so it might not be worth that much effort to fix.
470bool SIInstrInfo::shouldScheduleLoadsNear(SDNode *Load0, SDNode *Load1,
471 int64_t Offset0, int64_t Offset1,
472 unsigned NumLoads) const {
473 assert(Offset1 > Offset0 &&(static_cast <bool> (Offset1 > Offset0 && "Second offset should be larger than first offset!"
) ? void (0) : __assert_fail ("Offset1 > Offset0 && \"Second offset should be larger than first offset!\""
, "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 474, __extension__ __PRETTY_FUNCTION__))
474 "Second offset should be larger than first offset!")(static_cast <bool> (Offset1 > Offset0 && "Second offset should be larger than first offset!"
) ? void (0) : __assert_fail ("Offset1 > Offset0 && \"Second offset should be larger than first offset!\""
, "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 474, __extension__ __PRETTY_FUNCTION__))
;
475 // If we have less than 16 loads in a row, and the offsets are within 64
476 // bytes, then schedule together.
477
478 // A cacheline is 64 bytes (for global memory).
479 return (NumLoads <= 16 && (Offset1 - Offset0) < 64);
480}
481
482static void reportIllegalCopy(const SIInstrInfo *TII, MachineBasicBlock &MBB,
483 MachineBasicBlock::iterator MI,
484 const DebugLoc &DL, MCRegister DestReg,
485 MCRegister SrcReg, bool KillSrc,
486 const char *Msg = "illegal SGPR to VGPR copy") {
487 MachineFunction *MF = MBB.getParent();
488 DiagnosticInfoUnsupported IllegalCopy(MF->getFunction(), Msg, DL, DS_Error);
489 LLVMContext &C = MF->getFunction().getContext();
490 C.diagnose(IllegalCopy);
491
492 BuildMI(MBB, MI, DL, TII->get(AMDGPU::SI_ILLEGAL_COPY), DestReg)
493 .addReg(SrcReg, getKillRegState(KillSrc));
494}
495
496/// Handle copying from SGPR to AGPR, or from AGPR to AGPR. It is not possible
497/// to directly copy, so an intermediate VGPR needs to be used.
498static void indirectCopyToAGPR(const SIInstrInfo &TII,
499 MachineBasicBlock &MBB,
500 MachineBasicBlock::iterator MI,
501 const DebugLoc &DL, MCRegister DestReg,
502 MCRegister SrcReg, bool KillSrc,
503 RegScavenger &RS,
504 Register ImpDefSuperReg = Register(),
505 Register ImpUseSuperReg = Register()) {
506 const SIRegisterInfo &RI = TII.getRegisterInfo();
507
508 assert(AMDGPU::SReg_32RegClass.contains(SrcReg) ||(static_cast <bool> (AMDGPU::SReg_32RegClass.contains(SrcReg
) || AMDGPU::AGPR_32RegClass.contains(SrcReg)) ? void (0) : __assert_fail
("AMDGPU::SReg_32RegClass.contains(SrcReg) || AMDGPU::AGPR_32RegClass.contains(SrcReg)"
, "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 509, __extension__ __PRETTY_FUNCTION__))
509 AMDGPU::AGPR_32RegClass.contains(SrcReg))(static_cast <bool> (AMDGPU::SReg_32RegClass.contains(SrcReg
) || AMDGPU::AGPR_32RegClass.contains(SrcReg)) ? void (0) : __assert_fail
("AMDGPU::SReg_32RegClass.contains(SrcReg) || AMDGPU::AGPR_32RegClass.contains(SrcReg)"
, "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 509, __extension__ __PRETTY_FUNCTION__))
;
510
511 // First try to find defining accvgpr_write to avoid temporary registers.
512 for (auto Def = MI, E = MBB.begin(); Def != E; ) {
513 --Def;
514 if (!Def->definesRegister(SrcReg, &RI))
515 continue;
516 if (Def->getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64)
517 break;
518
519 MachineOperand &DefOp = Def->getOperand(1);
520 assert(DefOp.isReg() || DefOp.isImm())(static_cast <bool> (DefOp.isReg() || DefOp.isImm()) ? void
(0) : __assert_fail ("DefOp.isReg() || DefOp.isImm()", "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 520, __extension__ __PRETTY_FUNCTION__))
;
521
522 if (DefOp.isReg()) {
523 // Check that register source operand if not clobbered before MI.
524 // Immediate operands are always safe to propagate.
525 bool SafeToPropagate = true;
526 for (auto I = Def; I != MI && SafeToPropagate; ++I)
527 if (I->modifiesRegister(DefOp.getReg(), &RI))
528 SafeToPropagate = false;
529
530 if (!SafeToPropagate)
531 break;
532
533 DefOp.setIsKill(false);
534 }
535
536 MachineInstrBuilder Builder =
537 BuildMI(MBB, MI, DL, TII.get(AMDGPU::V_ACCVGPR_WRITE_B32_e64), DestReg)
538 .add(DefOp);
539 if (ImpDefSuperReg)
540 Builder.addReg(ImpDefSuperReg, RegState::Define | RegState::Implicit);
541
542 if (ImpUseSuperReg) {
543 Builder.addReg(ImpUseSuperReg,
544 getKillRegState(KillSrc) | RegState::Implicit);
545 }
546
547 return;
548 }
549
550 RS.enterBasicBlock(MBB);
551 RS.forward(MI);
552
553 // Ideally we want to have three registers for a long reg_sequence copy
554 // to hide 2 waitstates between v_mov_b32 and accvgpr_write.
555 unsigned MaxVGPRs = RI.getRegPressureLimit(&AMDGPU::VGPR_32RegClass,
556 *MBB.getParent());
557
558 // Registers in the sequence are allocated contiguously so we can just
559 // use register number to pick one of three round-robin temps.
560 unsigned RegNo = DestReg % 3;
561 Register Tmp = RS.scavengeRegister(&AMDGPU::VGPR_32RegClass, 0);
562 if (!Tmp)
563 report_fatal_error("Cannot scavenge VGPR to copy to AGPR");
564 RS.setRegUsed(Tmp);
565
566 if (!TII.getSubtarget().hasGFX90AInsts()) {
567 // Only loop through if there are any free registers left, otherwise
568 // scavenger may report a fatal error without emergency spill slot
569 // or spill with the slot.
570 while (RegNo-- && RS.FindUnusedReg(&AMDGPU::VGPR_32RegClass)) {
571 Register Tmp2 = RS.scavengeRegister(&AMDGPU::VGPR_32RegClass, 0);
572 if (!Tmp2 || RI.getHWRegIndex(Tmp2) >= MaxVGPRs)
573 break;
574 Tmp = Tmp2;
575 RS.setRegUsed(Tmp);
576 }
577 }
578
579 // Insert copy to temporary VGPR.
580 unsigned TmpCopyOp = AMDGPU::V_MOV_B32_e32;
581 if (AMDGPU::AGPR_32RegClass.contains(SrcReg)) {
582 TmpCopyOp = AMDGPU::V_ACCVGPR_READ_B32_e64;
583 } else {
584 assert(AMDGPU::SReg_32RegClass.contains(SrcReg))(static_cast <bool> (AMDGPU::SReg_32RegClass.contains(SrcReg
)) ? void (0) : __assert_fail ("AMDGPU::SReg_32RegClass.contains(SrcReg)"
, "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 584, __extension__ __PRETTY_FUNCTION__))
;
585 }
586
587 MachineInstrBuilder UseBuilder = BuildMI(MBB, MI, DL, TII.get(TmpCopyOp), Tmp)
588 .addReg(SrcReg, getKillRegState(KillSrc));
589 if (ImpUseSuperReg) {
590 UseBuilder.addReg(ImpUseSuperReg,
591 getKillRegState(KillSrc) | RegState::Implicit);
592 }
593
594 MachineInstrBuilder DefBuilder
595 = BuildMI(MBB, MI, DL, TII.get(AMDGPU::V_ACCVGPR_WRITE_B32_e64), DestReg)
596 .addReg(Tmp, RegState::Kill);
597
598 if (ImpDefSuperReg)
599 DefBuilder.addReg(ImpDefSuperReg, RegState::Define | RegState::Implicit);
600}
601
602static void expandSGPRCopy(const SIInstrInfo &TII, MachineBasicBlock &MBB,
603 MachineBasicBlock::iterator MI, const DebugLoc &DL,
604 MCRegister DestReg, MCRegister SrcReg, bool KillSrc,
605 const TargetRegisterClass *RC, bool Forward) {
606 const SIRegisterInfo &RI = TII.getRegisterInfo();
607 ArrayRef<int16_t> BaseIndices = RI.getRegSplitParts(RC, 4);
608 MachineBasicBlock::iterator I = MI;
609 MachineInstr *FirstMI = nullptr, *LastMI = nullptr;
610
611 for (unsigned Idx = 0; Idx < BaseIndices.size(); ++Idx) {
612 int16_t SubIdx = BaseIndices[Idx];
613 Register Reg = RI.getSubReg(DestReg, SubIdx);
614 unsigned Opcode = AMDGPU::S_MOV_B32;
615
616 // Is SGPR aligned? If so try to combine with next.
617 Register Src = RI.getSubReg(SrcReg, SubIdx);
618 bool AlignedDest = ((Reg - AMDGPU::SGPR0) % 2) == 0;
619 bool AlignedSrc = ((Src - AMDGPU::SGPR0) % 2) == 0;
620 if (AlignedDest && AlignedSrc && (Idx + 1 < BaseIndices.size())) {
621 // Can use SGPR64 copy
622 unsigned Channel = RI.getChannelFromSubReg(SubIdx);
623 SubIdx = RI.getSubRegFromChannel(Channel, 2);
624 Opcode = AMDGPU::S_MOV_B64;
625 Idx++;
626 }
627
628 LastMI = BuildMI(MBB, I, DL, TII.get(Opcode), RI.getSubReg(DestReg, SubIdx))
629 .addReg(RI.getSubReg(SrcReg, SubIdx))
630 .addReg(SrcReg, RegState::Implicit);
631
632 if (!FirstMI)
633 FirstMI = LastMI;
634
635 if (!Forward)
636 I--;
637 }
638
639 assert(FirstMI && LastMI)(static_cast <bool> (FirstMI && LastMI) ? void (
0) : __assert_fail ("FirstMI && LastMI", "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 639, __extension__ __PRETTY_FUNCTION__))
;
640 if (!Forward)
641 std::swap(FirstMI, LastMI);
642
643 FirstMI->addOperand(
644 MachineOperand::CreateReg(DestReg, true /*IsDef*/, true /*IsImp*/));
645
646 if (KillSrc)
647 LastMI->addRegisterKilled(SrcReg, &RI);
648}
649
650void SIInstrInfo::copyPhysReg(MachineBasicBlock &MBB,
651 MachineBasicBlock::iterator MI,
652 const DebugLoc &DL, MCRegister DestReg,
653 MCRegister SrcReg, bool KillSrc) const {
654 const TargetRegisterClass *RC = RI.getPhysRegClass(DestReg);
655
656 // FIXME: This is hack to resolve copies between 16 bit and 32 bit
657 // registers until all patterns are fixed.
658 if (Fix16BitCopies &&
659 ((RI.getRegSizeInBits(*RC) == 16) ^
660 (RI.getRegSizeInBits(*RI.getPhysRegClass(SrcReg)) == 16))) {
661 MCRegister &RegToFix = (RI.getRegSizeInBits(*RC) == 16) ? DestReg : SrcReg;
662 MCRegister Super = RI.get32BitRegister(RegToFix);
663 assert(RI.getSubReg(Super, AMDGPU::lo16) == RegToFix)(static_cast <bool> (RI.getSubReg(Super, AMDGPU::lo16) ==
RegToFix) ? void (0) : __assert_fail ("RI.getSubReg(Super, AMDGPU::lo16) == RegToFix"
, "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 663, __extension__ __PRETTY_FUNCTION__))
;
664 RegToFix = Super;
665
666 if (DestReg == SrcReg) {
667 // Insert empty bundle since ExpandPostRA expects an instruction here.
668 BuildMI(MBB, MI, DL, get(AMDGPU::BUNDLE));
669 return;
670 }
671
672 RC = RI.getPhysRegClass(DestReg);
673 }
674
675 if (RC == &AMDGPU::VGPR_32RegClass) {
676 assert(AMDGPU::VGPR_32RegClass.contains(SrcReg) ||(static_cast <bool> (AMDGPU::VGPR_32RegClass.contains(SrcReg
) || AMDGPU::SReg_32RegClass.contains(SrcReg) || AMDGPU::AGPR_32RegClass
.contains(SrcReg)) ? void (0) : __assert_fail ("AMDGPU::VGPR_32RegClass.contains(SrcReg) || AMDGPU::SReg_32RegClass.contains(SrcReg) || AMDGPU::AGPR_32RegClass.contains(SrcReg)"
, "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 678, __extension__ __PRETTY_FUNCTION__))
677 AMDGPU::SReg_32RegClass.contains(SrcReg) ||(static_cast <bool> (AMDGPU::VGPR_32RegClass.contains(SrcReg
) || AMDGPU::SReg_32RegClass.contains(SrcReg) || AMDGPU::AGPR_32RegClass
.contains(SrcReg)) ? void (0) : __assert_fail ("AMDGPU::VGPR_32RegClass.contains(SrcReg) || AMDGPU::SReg_32RegClass.contains(SrcReg) || AMDGPU::AGPR_32RegClass.contains(SrcReg)"
, "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 678, __extension__ __PRETTY_FUNCTION__))
678 AMDGPU::AGPR_32RegClass.contains(SrcReg))(static_cast <bool> (AMDGPU::VGPR_32RegClass.contains(SrcReg
) || AMDGPU::SReg_32RegClass.contains(SrcReg) || AMDGPU::AGPR_32RegClass
.contains(SrcReg)) ? void (0) : __assert_fail ("AMDGPU::VGPR_32RegClass.contains(SrcReg) || AMDGPU::SReg_32RegClass.contains(SrcReg) || AMDGPU::AGPR_32RegClass.contains(SrcReg)"
, "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 678, __extension__ __PRETTY_FUNCTION__))
;
679 unsigned Opc = AMDGPU::AGPR_32RegClass.contains(SrcReg) ?
680 AMDGPU::V_ACCVGPR_READ_B32_e64 : AMDGPU::V_MOV_B32_e32;
681 BuildMI(MBB, MI, DL, get(Opc), DestReg)
682 .addReg(SrcReg, getKillRegState(KillSrc));
683 return;
684 }
685
686 if (RC == &AMDGPU::SReg_32_XM0RegClass ||
687 RC == &AMDGPU::SReg_32RegClass) {
688 if (SrcReg == AMDGPU::SCC) {
689 BuildMI(MBB, MI, DL, get(AMDGPU::S_CSELECT_B32), DestReg)
690 .addImm(1)
691 .addImm(0);
692 return;
693 }
694
695 if (DestReg == AMDGPU::VCC_LO) {
696 if (AMDGPU::SReg_32RegClass.contains(SrcReg)) {
697 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), AMDGPU::VCC_LO)
698 .addReg(SrcReg, getKillRegState(KillSrc));
699 } else {
700 // FIXME: Hack until VReg_1 removed.
701 assert(AMDGPU::VGPR_32RegClass.contains(SrcReg))(static_cast <bool> (AMDGPU::VGPR_32RegClass.contains(SrcReg
)) ? void (0) : __assert_fail ("AMDGPU::VGPR_32RegClass.contains(SrcReg)"
, "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 701, __extension__ __PRETTY_FUNCTION__))
;
702 BuildMI(MBB, MI, DL, get(AMDGPU::V_CMP_NE_U32_e32))
703 .addImm(0)
704 .addReg(SrcReg, getKillRegState(KillSrc));
705 }
706
707 return;
708 }
709
710 if (!AMDGPU::SReg_32RegClass.contains(SrcReg)) {
711 reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc);
712 return;
713 }
714
715 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), DestReg)
716 .addReg(SrcReg, getKillRegState(KillSrc));
717 return;
718 }
719
720 if (RC == &AMDGPU::SReg_64RegClass) {
721 if (SrcReg == AMDGPU::SCC) {
722 BuildMI(MBB, MI, DL, get(AMDGPU::S_CSELECT_B64), DestReg)
723 .addImm(1)
724 .addImm(0);
725 return;
726 }
727
728 if (DestReg == AMDGPU::VCC) {
729 if (AMDGPU::SReg_64RegClass.contains(SrcReg)) {
730 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B64), AMDGPU::VCC)
731 .addReg(SrcReg, getKillRegState(KillSrc));
732 } else {
733 // FIXME: Hack until VReg_1 removed.
734 assert(AMDGPU::VGPR_32RegClass.contains(SrcReg))(static_cast <bool> (AMDGPU::VGPR_32RegClass.contains(SrcReg
)) ? void (0) : __assert_fail ("AMDGPU::VGPR_32RegClass.contains(SrcReg)"
, "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 734, __extension__ __PRETTY_FUNCTION__))
;
735 BuildMI(MBB, MI, DL, get(AMDGPU::V_CMP_NE_U32_e32))
736 .addImm(0)
737 .addReg(SrcReg, getKillRegState(KillSrc));
738 }
739
740 return;
741 }
742
743 if (!AMDGPU::SReg_64RegClass.contains(SrcReg)) {
744 reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc);
745 return;
746 }
747
748 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B64), DestReg)
749 .addReg(SrcReg, getKillRegState(KillSrc));
750 return;
751 }
752
753 if (DestReg == AMDGPU::SCC) {
754 // Copying 64-bit or 32-bit sources to SCC barely makes sense,
755 // but SelectionDAG emits such copies for i1 sources.
756 if (AMDGPU::SReg_64RegClass.contains(SrcReg)) {
757 // This copy can only be produced by patterns
758 // with explicit SCC, which are known to be enabled
759 // only for subtargets with S_CMP_LG_U64 present.
760 assert(ST.hasScalarCompareEq64())(static_cast <bool> (ST.hasScalarCompareEq64()) ? void (
0) : __assert_fail ("ST.hasScalarCompareEq64()", "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 760, __extension__ __PRETTY_FUNCTION__))
;
761 BuildMI(MBB, MI, DL, get(AMDGPU::S_CMP_LG_U64))
762 .addReg(SrcReg, getKillRegState(KillSrc))
763 .addImm(0);
764 } else {
765 assert(AMDGPU::SReg_32RegClass.contains(SrcReg))(static_cast <bool> (AMDGPU::SReg_32RegClass.contains(SrcReg
)) ? void (0) : __assert_fail ("AMDGPU::SReg_32RegClass.contains(SrcReg)"
, "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 765, __extension__ __PRETTY_FUNCTION__))
;
766 BuildMI(MBB, MI, DL, get(AMDGPU::S_CMP_LG_U32))
767 .addReg(SrcReg, getKillRegState(KillSrc))
768 .addImm(0);
769 }
770
771 return;
772 }
773
774 if (RC == &AMDGPU::AGPR_32RegClass) {
775 if (AMDGPU::VGPR_32RegClass.contains(SrcReg)) {
776 BuildMI(MBB, MI, DL, get(AMDGPU::V_ACCVGPR_WRITE_B32_e64), DestReg)
777 .addReg(SrcReg, getKillRegState(KillSrc));
778 return;
779 }
780
781 if (AMDGPU::AGPR_32RegClass.contains(SrcReg) && ST.hasGFX90AInsts()) {
782 BuildMI(MBB, MI, DL, get(AMDGPU::V_ACCVGPR_MOV_B32), DestReg)
783 .addReg(SrcReg, getKillRegState(KillSrc));
784 return;
785 }
786
787 // FIXME: Pass should maintain scavenger to avoid scan through the block on
788 // every AGPR spill.
789 RegScavenger RS;
790 indirectCopyToAGPR(*this, MBB, MI, DL, DestReg, SrcReg, KillSrc, RS);
791 return;
792 }
793
794 const unsigned Size = RI.getRegSizeInBits(*RC);
795 if (Size == 16) {
796 assert(AMDGPU::VGPR_LO16RegClass.contains(SrcReg) ||(static_cast <bool> (AMDGPU::VGPR_LO16RegClass.contains
(SrcReg) || AMDGPU::VGPR_HI16RegClass.contains(SrcReg) || AMDGPU
::SReg_LO16RegClass.contains(SrcReg) || AMDGPU::AGPR_LO16RegClass
.contains(SrcReg)) ? void (0) : __assert_fail ("AMDGPU::VGPR_LO16RegClass.contains(SrcReg) || AMDGPU::VGPR_HI16RegClass.contains(SrcReg) || AMDGPU::SReg_LO16RegClass.contains(SrcReg) || AMDGPU::AGPR_LO16RegClass.contains(SrcReg)"
, "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 799, __extension__ __PRETTY_FUNCTION__))
797 AMDGPU::VGPR_HI16RegClass.contains(SrcReg) ||(static_cast <bool> (AMDGPU::VGPR_LO16RegClass.contains
(SrcReg) || AMDGPU::VGPR_HI16RegClass.contains(SrcReg) || AMDGPU
::SReg_LO16RegClass.contains(SrcReg) || AMDGPU::AGPR_LO16RegClass
.contains(SrcReg)) ? void (0) : __assert_fail ("AMDGPU::VGPR_LO16RegClass.contains(SrcReg) || AMDGPU::VGPR_HI16RegClass.contains(SrcReg) || AMDGPU::SReg_LO16RegClass.contains(SrcReg) || AMDGPU::AGPR_LO16RegClass.contains(SrcReg)"
, "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 799, __extension__ __PRETTY_FUNCTION__))
798 AMDGPU::SReg_LO16RegClass.contains(SrcReg) ||(static_cast <bool> (AMDGPU::VGPR_LO16RegClass.contains
(SrcReg) || AMDGPU::VGPR_HI16RegClass.contains(SrcReg) || AMDGPU
::SReg_LO16RegClass.contains(SrcReg) || AMDGPU::AGPR_LO16RegClass
.contains(SrcReg)) ? void (0) : __assert_fail ("AMDGPU::VGPR_LO16RegClass.contains(SrcReg) || AMDGPU::VGPR_HI16RegClass.contains(SrcReg) || AMDGPU::SReg_LO16RegClass.contains(SrcReg) || AMDGPU::AGPR_LO16RegClass.contains(SrcReg)"
, "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 799, __extension__ __PRETTY_FUNCTION__))
799 AMDGPU::AGPR_LO16RegClass.contains(SrcReg))(static_cast <bool> (AMDGPU::VGPR_LO16RegClass.contains
(SrcReg) || AMDGPU::VGPR_HI16RegClass.contains(SrcReg) || AMDGPU
::SReg_LO16RegClass.contains(SrcReg) || AMDGPU::AGPR_LO16RegClass
.contains(SrcReg)) ? void (0) : __assert_fail ("AMDGPU::VGPR_LO16RegClass.contains(SrcReg) || AMDGPU::VGPR_HI16RegClass.contains(SrcReg) || AMDGPU::SReg_LO16RegClass.contains(SrcReg) || AMDGPU::AGPR_LO16RegClass.contains(SrcReg)"
, "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 799, __extension__ __PRETTY_FUNCTION__))
;
800
801 bool IsSGPRDst = AMDGPU::SReg_LO16RegClass.contains(DestReg);
802 bool IsSGPRSrc = AMDGPU::SReg_LO16RegClass.contains(SrcReg);
803 bool IsAGPRDst = AMDGPU::AGPR_LO16RegClass.contains(DestReg);
804 bool IsAGPRSrc = AMDGPU::AGPR_LO16RegClass.contains(SrcReg);
805 bool DstLow = AMDGPU::VGPR_LO16RegClass.contains(DestReg) ||
806 AMDGPU::SReg_LO16RegClass.contains(DestReg) ||
807 AMDGPU::AGPR_LO16RegClass.contains(DestReg);
808 bool SrcLow = AMDGPU::VGPR_LO16RegClass.contains(SrcReg) ||
809 AMDGPU::SReg_LO16RegClass.contains(SrcReg) ||
810 AMDGPU::AGPR_LO16RegClass.contains(SrcReg);
811 MCRegister NewDestReg = RI.get32BitRegister(DestReg);
812 MCRegister NewSrcReg = RI.get32BitRegister(SrcReg);
813
814 if (IsSGPRDst) {
815 if (!IsSGPRSrc) {
816 reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc);
817 return;
818 }
819
820 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), NewDestReg)
821 .addReg(NewSrcReg, getKillRegState(KillSrc));
822 return;
823 }
824
825 if (IsAGPRDst || IsAGPRSrc) {
826 if (!DstLow || !SrcLow) {
827 reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc,
828 "Cannot use hi16 subreg with an AGPR!");
829 }
830
831 copyPhysReg(MBB, MI, DL, NewDestReg, NewSrcReg, KillSrc);
832 return;
833 }
834
835 if (IsSGPRSrc && !ST.hasSDWAScalar()) {
836 if (!DstLow || !SrcLow) {
837 reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc,
838 "Cannot use hi16 subreg on VI!");
839 }
840
841 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), NewDestReg)
842 .addReg(NewSrcReg, getKillRegState(KillSrc));
843 return;
844 }
845
846 auto MIB = BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_sdwa), NewDestReg)
847 .addImm(0) // src0_modifiers
848 .addReg(NewSrcReg)
849 .addImm(0) // clamp
850 .addImm(DstLow ? AMDGPU::SDWA::SdwaSel::WORD_0
851 : AMDGPU::SDWA::SdwaSel::WORD_1)
852 .addImm(AMDGPU::SDWA::DstUnused::UNUSED_PRESERVE)
853 .addImm(SrcLow ? AMDGPU::SDWA::SdwaSel::WORD_0
854 : AMDGPU::SDWA::SdwaSel::WORD_1)
855 .addReg(NewDestReg, RegState::Implicit | RegState::Undef);
856 // First implicit operand is $exec.
857 MIB->tieOperands(0, MIB->getNumOperands() - 1);
858 return;
859 }
860
861 const TargetRegisterClass *SrcRC = RI.getPhysRegClass(SrcReg);
862 if (RC == RI.getVGPR64Class() && (SrcRC == RC || RI.isSGPRClass(SrcRC))) {
863 if (ST.hasPackedFP32Ops()) {
864 BuildMI(MBB, MI, DL, get(AMDGPU::V_PK_MOV_B32), DestReg)
865 .addImm(SISrcMods::OP_SEL_1)
866 .addReg(SrcReg)
867 .addImm(SISrcMods::OP_SEL_0 | SISrcMods::OP_SEL_1)
868 .addReg(SrcReg)
869 .addImm(0) // op_sel_lo
870 .addImm(0) // op_sel_hi
871 .addImm(0) // neg_lo
872 .addImm(0) // neg_hi
873 .addImm(0) // clamp
874 .addReg(SrcReg, getKillRegState(KillSrc) | RegState::Implicit);
875 return;
876 }
877 }
878
879 const bool Forward = RI.getHWRegIndex(DestReg) <= RI.getHWRegIndex(SrcReg);
880 if (RI.isSGPRClass(RC)) {
881 if (!RI.isSGPRClass(SrcRC)) {
882 reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc);
883 return;
884 }
885 expandSGPRCopy(*this, MBB, MI, DL, DestReg, SrcReg, KillSrc, RC, Forward);
886 return;
887 }
888
889 unsigned EltSize = 4;
890 unsigned Opcode = AMDGPU::V_MOV_B32_e32;
891 if (RI.hasAGPRs(RC)) {
892 Opcode = (RI.hasVGPRs(SrcRC)) ?
893 AMDGPU::V_ACCVGPR_WRITE_B32_e64 : AMDGPU::INSTRUCTION_LIST_END;
894 } else if (RI.hasVGPRs(RC) && RI.hasAGPRs(SrcRC)) {
895 Opcode = AMDGPU::V_ACCVGPR_READ_B32_e64;
896 } else if ((Size % 64 == 0) && RI.hasVGPRs(RC) &&
897 (RI.isProperlyAlignedRC(*RC) &&
898 (SrcRC == RC || RI.isSGPRClass(SrcRC)))) {
899 // TODO: In 96-bit case, could do a 64-bit mov and then a 32-bit mov.
900 if (ST.hasPackedFP32Ops()) {
901 Opcode = AMDGPU::V_PK_MOV_B32;
902 EltSize = 8;
903 }
904 }
905
906 // For the cases where we need an intermediate instruction/temporary register
907 // (destination is an AGPR), we need a scavenger.
908 //
909 // FIXME: The pass should maintain this for us so we don't have to re-scan the
910 // whole block for every handled copy.
911 std::unique_ptr<RegScavenger> RS;
912 if (Opcode == AMDGPU::INSTRUCTION_LIST_END)
913 RS.reset(new RegScavenger());
914
915 ArrayRef<int16_t> SubIndices = RI.getRegSplitParts(RC, EltSize);
916
917 // If there is an overlap, we can't kill the super-register on the last
918 // instruction, since it will also kill the components made live by this def.
919 const bool CanKillSuperReg = KillSrc && !RI.regsOverlap(SrcReg, DestReg);
920
921 for (unsigned Idx = 0; Idx < SubIndices.size(); ++Idx) {
922 unsigned SubIdx;
923 if (Forward)
924 SubIdx = SubIndices[Idx];
925 else
926 SubIdx = SubIndices[SubIndices.size() - Idx - 1];
927
928 bool UseKill = CanKillSuperReg && Idx == SubIndices.size() - 1;
929
930 if (Opcode == AMDGPU::INSTRUCTION_LIST_END) {
931 Register ImpDefSuper = Idx == 0 ? Register(DestReg) : Register();
932 Register ImpUseSuper = SrcReg;
933 indirectCopyToAGPR(*this, MBB, MI, DL, RI.getSubReg(DestReg, SubIdx),
934 RI.getSubReg(SrcReg, SubIdx), UseKill, *RS,
935 ImpDefSuper, ImpUseSuper);
936 } else if (Opcode == AMDGPU::V_PK_MOV_B32) {
937 Register DstSubReg = RI.getSubReg(DestReg, SubIdx);
938 Register SrcSubReg = RI.getSubReg(SrcReg, SubIdx);
939 MachineInstrBuilder MIB =
940 BuildMI(MBB, MI, DL, get(AMDGPU::V_PK_MOV_B32), DstSubReg)
941 .addImm(SISrcMods::OP_SEL_1)
942 .addReg(SrcSubReg)
943 .addImm(SISrcMods::OP_SEL_0 | SISrcMods::OP_SEL_1)
944 .addReg(SrcSubReg)
945 .addImm(0) // op_sel_lo
946 .addImm(0) // op_sel_hi
947 .addImm(0) // neg_lo
948 .addImm(0) // neg_hi
949 .addImm(0) // clamp
950 .addReg(SrcReg, getKillRegState(UseKill) | RegState::Implicit);
951 if (Idx == 0)
952 MIB.addReg(DestReg, RegState::Define | RegState::Implicit);
953 } else {
954 MachineInstrBuilder Builder =
955 BuildMI(MBB, MI, DL, get(Opcode), RI.getSubReg(DestReg, SubIdx))
956 .addReg(RI.getSubReg(SrcReg, SubIdx));
957 if (Idx == 0)
958 Builder.addReg(DestReg, RegState::Define | RegState::Implicit);
959
960 Builder.addReg(SrcReg, getKillRegState(UseKill) | RegState::Implicit);
961 }
962 }
963}
964
965int SIInstrInfo::commuteOpcode(unsigned Opcode) const {
966 int NewOpc;
967
968 // Try to map original to commuted opcode
969 NewOpc = AMDGPU::getCommuteRev(Opcode);
970 if (NewOpc != -1)
971 // Check if the commuted (REV) opcode exists on the target.
972 return pseudoToMCOpcode(NewOpc) != -1 ? NewOpc : -1;
973
974 // Try to map commuted to original opcode
975 NewOpc = AMDGPU::getCommuteOrig(Opcode);
976 if (NewOpc != -1)
977 // Check if the original (non-REV) opcode exists on the target.
978 return pseudoToMCOpcode(NewOpc) != -1 ? NewOpc : -1;
979
980 return Opcode;
981}
982
983void SIInstrInfo::materializeImmediate(MachineBasicBlock &MBB,
984 MachineBasicBlock::iterator MI,
985 const DebugLoc &DL, unsigned DestReg,
986 int64_t Value) const {
987 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
988 const TargetRegisterClass *RegClass = MRI.getRegClass(DestReg);
989 if (RegClass == &AMDGPU::SReg_32RegClass ||
990 RegClass == &AMDGPU::SGPR_32RegClass ||
991 RegClass == &AMDGPU::SReg_32_XM0RegClass ||
992 RegClass == &AMDGPU::SReg_32_XM0_XEXECRegClass) {
993 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), DestReg)
994 .addImm(Value);
995 return;
996 }
997
998 if (RegClass == &AMDGPU::SReg_64RegClass ||
999 RegClass == &AMDGPU::SGPR_64RegClass ||
1000 RegClass == &AMDGPU::SReg_64_XEXECRegClass) {
1001 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B64), DestReg)
1002 .addImm(Value);
1003 return;
1004 }
1005
1006 if (RegClass == &AMDGPU::VGPR_32RegClass) {
1007 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DestReg)
1008 .addImm(Value);
1009 return;
1010 }
1011 if (RegClass->hasSuperClassEq(&AMDGPU::VReg_64RegClass)) {
1012 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B64_PSEUDO), DestReg)
1013 .addImm(Value);
1014 return;
1015 }
1016
1017 unsigned EltSize = 4;
1018 unsigned Opcode = AMDGPU::V_MOV_B32_e32;
1019 if (RI.isSGPRClass(RegClass)) {
1020 if (RI.getRegSizeInBits(*RegClass) > 32) {
1021 Opcode = AMDGPU::S_MOV_B64;
1022 EltSize = 8;
1023 } else {
1024 Opcode = AMDGPU::S_MOV_B32;
1025 EltSize = 4;
1026 }
1027 }
1028
1029 ArrayRef<int16_t> SubIndices = RI.getRegSplitParts(RegClass, EltSize);
1030 for (unsigned Idx = 0; Idx < SubIndices.size(); ++Idx) {
1031 int64_t IdxValue = Idx == 0 ? Value : 0;
1032
1033 MachineInstrBuilder Builder = BuildMI(MBB, MI, DL,
1034 get(Opcode), RI.getSubReg(DestReg, SubIndices[Idx]));
1035 Builder.addImm(IdxValue);
1036 }
1037}
1038
1039const TargetRegisterClass *
1040SIInstrInfo::getPreferredSelectRegClass(unsigned Size) const {
1041 return &AMDGPU::VGPR_32RegClass;
1042}
1043
1044void SIInstrInfo::insertVectorSelect(MachineBasicBlock &MBB,
1045 MachineBasicBlock::iterator I,
1046 const DebugLoc &DL, Register DstReg,
1047 ArrayRef<MachineOperand> Cond,
1048 Register TrueReg,
1049 Register FalseReg) const {
1050 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
1051 const TargetRegisterClass *BoolXExecRC =
1052 RI.getRegClass(AMDGPU::SReg_1_XEXECRegClassID);
1053 assert(MRI.getRegClass(DstReg) == &AMDGPU::VGPR_32RegClass &&(static_cast <bool> (MRI.getRegClass(DstReg) == &AMDGPU
::VGPR_32RegClass && "Not a VGPR32 reg") ? void (0) :
__assert_fail ("MRI.getRegClass(DstReg) == &AMDGPU::VGPR_32RegClass && \"Not a VGPR32 reg\""
, "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 1054, __extension__ __PRETTY_FUNCTION__))
1054 "Not a VGPR32 reg")(static_cast <bool> (MRI.getRegClass(DstReg) == &AMDGPU
::VGPR_32RegClass && "Not a VGPR32 reg") ? void (0) :
__assert_fail ("MRI.getRegClass(DstReg) == &AMDGPU::VGPR_32RegClass && \"Not a VGPR32 reg\""
, "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 1054, __extension__ __PRETTY_FUNCTION__))
;
1055
1056 if (Cond.size() == 1) {
1057 Register SReg = MRI.createVirtualRegister(BoolXExecRC);
1058 BuildMI(MBB, I, DL, get(AMDGPU::COPY), SReg)
1059 .add(Cond[0]);
1060 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
1061 .addImm(0)
1062 .addReg(FalseReg)
1063 .addImm(0)
1064 .addReg(TrueReg)
1065 .addReg(SReg);
1066 } else if (Cond.size() == 2) {
1067 assert(Cond[0].isImm() && "Cond[0] is not an immediate")(static_cast <bool> (Cond[0].isImm() && "Cond[0] is not an immediate"
) ? void (0) : __assert_fail ("Cond[0].isImm() && \"Cond[0] is not an immediate\""
, "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 1067, __extension__ __PRETTY_FUNCTION__))
;
1068 switch (Cond[0].getImm()) {
1069 case SIInstrInfo::SCC_TRUE: {
1070 Register SReg = MRI.createVirtualRegister(BoolXExecRC);
1071 BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_CSELECT_B32
1072 : AMDGPU::S_CSELECT_B64), SReg)
1073 .addImm(1)
1074 .addImm(0);
1075 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
1076 .addImm(0)
1077 .addReg(FalseReg)
1078 .addImm(0)
1079 .addReg(TrueReg)
1080 .addReg(SReg);
1081 break;
1082 }
1083 case SIInstrInfo::SCC_FALSE: {
1084 Register SReg = MRI.createVirtualRegister(BoolXExecRC);
1085 BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_CSELECT_B32
1086 : AMDGPU::S_CSELECT_B64), SReg)
1087 .addImm(0)
1088 .addImm(1);
1089 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
1090 .addImm(0)
1091 .addReg(FalseReg)
1092 .addImm(0)
1093 .addReg(TrueReg)
1094 .addReg(SReg);
1095 break;
1096 }
1097 case SIInstrInfo::VCCNZ: {
1098 MachineOperand RegOp = Cond[1];
1099 RegOp.setImplicit(false);
1100 Register SReg = MRI.createVirtualRegister(BoolXExecRC);
1101 BuildMI(MBB, I, DL, get(AMDGPU::COPY), SReg)
1102 .add(RegOp);
1103 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
1104 .addImm(0)
1105 .addReg(FalseReg)
1106 .addImm(0)
1107 .addReg(TrueReg)
1108 .addReg(SReg);
1109 break;
1110 }
1111 case SIInstrInfo::VCCZ: {
1112 MachineOperand RegOp = Cond[1];
1113 RegOp.setImplicit(false);
1114 Register SReg = MRI.createVirtualRegister(BoolXExecRC);
1115 BuildMI(MBB, I, DL, get(AMDGPU::COPY), SReg)
1116 .add(RegOp);
1117 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
1118 .addImm(0)
1119 .addReg(TrueReg)
1120 .addImm(0)
1121 .addReg(FalseReg)
1122 .addReg(SReg);
1123 break;
1124 }
1125 case SIInstrInfo::EXECNZ: {
1126 Register SReg = MRI.createVirtualRegister(BoolXExecRC);
1127 Register SReg2 = MRI.createVirtualRegister(RI.getBoolRC());
1128 BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_OR_SAVEEXEC_B32
1129 : AMDGPU::S_OR_SAVEEXEC_B64), SReg2)
1130 .addImm(0);
1131 BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_CSELECT_B32
1132 : AMDGPU::S_CSELECT_B64), SReg)
1133 .addImm(1)
1134 .addImm(0);
1135 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
1136 .addImm(0)
1137 .addReg(FalseReg)
1138 .addImm(0)
1139 .addReg(TrueReg)
1140 .addReg(SReg);
1141 break;
1142 }
1143 case SIInstrInfo::EXECZ: {
1144 Register SReg = MRI.createVirtualRegister(BoolXExecRC);
1145 Register SReg2 = MRI.createVirtualRegister(RI.getBoolRC());
1146 BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_OR_SAVEEXEC_B32
1147 : AMDGPU::S_OR_SAVEEXEC_B64), SReg2)
1148 .addImm(0);
1149 BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_CSELECT_B32
1150 : AMDGPU::S_CSELECT_B64), SReg)
1151 .addImm(0)
1152 .addImm(1);
1153 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
1154 .addImm(0)
1155 .addReg(FalseReg)
1156 .addImm(0)
1157 .addReg(TrueReg)
1158 .addReg(SReg);
1159 llvm_unreachable("Unhandled branch predicate EXECZ")::llvm::llvm_unreachable_internal("Unhandled branch predicate EXECZ"
, "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 1159)
;
1160 break;
1161 }
1162 default:
1163 llvm_unreachable("invalid branch predicate")::llvm::llvm_unreachable_internal("invalid branch predicate",
"/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 1163)
;
1164 }
1165 } else {
1166 llvm_unreachable("Can only handle Cond size 1 or 2")::llvm::llvm_unreachable_internal("Can only handle Cond size 1 or 2"
, "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 1166)
;
1167 }
1168}
1169
1170Register SIInstrInfo::insertEQ(MachineBasicBlock *MBB,
1171 MachineBasicBlock::iterator I,
1172 const DebugLoc &DL,
1173 Register SrcReg, int Value) const {
1174 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
1175 Register Reg = MRI.createVirtualRegister(RI.getBoolRC());
1176 BuildMI(*MBB, I, DL, get(AMDGPU::V_CMP_EQ_I32_e64), Reg)
1177 .addImm(Value)
1178 .addReg(SrcReg);
1179
1180 return Reg;
1181}
1182
1183Register SIInstrInfo::insertNE(MachineBasicBlock *MBB,
1184 MachineBasicBlock::iterator I,
1185 const DebugLoc &DL,
1186 Register SrcReg, int Value) const {
1187 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
1188 Register Reg = MRI.createVirtualRegister(RI.getBoolRC());
1189 BuildMI(*MBB, I, DL, get(AMDGPU::V_CMP_NE_I32_e64), Reg)
1190 .addImm(Value)
1191 .addReg(SrcReg);
1192
1193 return Reg;
1194}
1195
1196unsigned SIInstrInfo::getMovOpcode(const TargetRegisterClass *DstRC) const {
1197
1198 if (RI.hasAGPRs(DstRC))
1199 return AMDGPU::COPY;
1200 if (RI.getRegSizeInBits(*DstRC) == 32) {
1201 return RI.isSGPRClass(DstRC) ? AMDGPU::S_MOV_B32 : AMDGPU::V_MOV_B32_e32;
1202 } else if (RI.getRegSizeInBits(*DstRC) == 64 && RI.isSGPRClass(DstRC)) {
1203 return AMDGPU::S_MOV_B64;
1204 } else if (RI.getRegSizeInBits(*DstRC) == 64 && !RI.isSGPRClass(DstRC)) {
1205 return AMDGPU::V_MOV_B64_PSEUDO;
1206 }
1207 return AMDGPU::COPY;
1208}
1209
1210const MCInstrDesc &
1211SIInstrInfo::getIndirectGPRIDXPseudo(unsigned VecSize,
1212 bool IsIndirectSrc) const {
1213 if (IsIndirectSrc) {
1214 if (VecSize <= 32) // 4 bytes
1215 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V1);
1216 if (VecSize <= 64) // 8 bytes
1217 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V2);
1218 if (VecSize <= 96) // 12 bytes
1219 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V3);
1220 if (VecSize <= 128) // 16 bytes
1221 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V4);
1222 if (VecSize <= 160) // 20 bytes
1223 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V5);
1224 if (VecSize <= 256) // 32 bytes
1225 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V8);
1226 if (VecSize <= 512) // 64 bytes
1227 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V16);
1228 if (VecSize <= 1024) // 128 bytes
1229 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V32);
1230
1231 llvm_unreachable("unsupported size for IndirectRegReadGPRIDX pseudos")::llvm::llvm_unreachable_internal("unsupported size for IndirectRegReadGPRIDX pseudos"
, "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 1231)
;
1232 }
1233
1234 if (VecSize <= 32) // 4 bytes
1235 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V1);
1236 if (VecSize <= 64) // 8 bytes
1237 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V2);
1238 if (VecSize <= 96) // 12 bytes
1239 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V3);
1240 if (VecSize <= 128) // 16 bytes
1241 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V4);
1242 if (VecSize <= 160) // 20 bytes
1243 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V5);
1244 if (VecSize <= 256) // 32 bytes
1245 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V8);
1246 if (VecSize <= 512) // 64 bytes
1247 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V16);
1248 if (VecSize <= 1024) // 128 bytes
1249 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V32);
1250
1251 llvm_unreachable("unsupported size for IndirectRegWriteGPRIDX pseudos")::llvm::llvm_unreachable_internal("unsupported size for IndirectRegWriteGPRIDX pseudos"
, "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 1251)
;
1252}
1253
1254static unsigned getIndirectVGPRWriteMovRelPseudoOpc(unsigned VecSize) {
1255 if (VecSize <= 32) // 4 bytes
1256 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V1;
1257 if (VecSize <= 64) // 8 bytes
1258 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V2;
1259 if (VecSize <= 96) // 12 bytes
1260 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V3;
1261 if (VecSize <= 128) // 16 bytes
1262 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V4;
1263 if (VecSize <= 160) // 20 bytes
1264 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V5;
1265 if (VecSize <= 256) // 32 bytes
1266 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V8;
1267 if (VecSize <= 512) // 64 bytes
1268 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V16;
1269 if (VecSize <= 1024) // 128 bytes
1270 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V32;
1271
1272 llvm_unreachable("unsupported size for IndirectRegWrite pseudos")::llvm::llvm_unreachable_internal("unsupported size for IndirectRegWrite pseudos"
, "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 1272)
;
1273}
1274
1275static unsigned getIndirectSGPRWriteMovRelPseudo32(unsigned VecSize) {
1276 if (VecSize <= 32) // 4 bytes
1277 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V1;
1278 if (VecSize <= 64) // 8 bytes
1279 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V2;
1280 if (VecSize <= 96) // 12 bytes
1281 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V3;
1282 if (VecSize <= 128) // 16 bytes
1283 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V4;
1284 if (VecSize <= 160) // 20 bytes
1285 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V5;
1286 if (VecSize <= 256) // 32 bytes
1287 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V8;
1288 if (VecSize <= 512) // 64 bytes
1289 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V16;
1290 if (VecSize <= 1024) // 128 bytes
1291 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V32;
1292
1293 llvm_unreachable("unsupported size for IndirectRegWrite pseudos")::llvm::llvm_unreachable_internal("unsupported size for IndirectRegWrite pseudos"
, "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 1293)
;
1294}
1295
1296static unsigned getIndirectSGPRWriteMovRelPseudo64(unsigned VecSize) {
1297 if (VecSize <= 64) // 8 bytes
1298 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V1;
1299 if (VecSize <= 128) // 16 bytes
1300 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V2;
1301 if (VecSize <= 256) // 32 bytes
1302 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V4;
1303 if (VecSize <= 512) // 64 bytes
1304 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V8;
1305 if (VecSize <= 1024) // 128 bytes
1306 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V16;
1307
1308 llvm_unreachable("unsupported size for IndirectRegWrite pseudos")::llvm::llvm_unreachable_internal("unsupported size for IndirectRegWrite pseudos"
, "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 1308)
;
1309}
1310
1311const MCInstrDesc &
1312SIInstrInfo::getIndirectRegWriteMovRelPseudo(unsigned VecSize, unsigned EltSize,
1313 bool IsSGPR) const {
1314 if (IsSGPR) {
1315 switch (EltSize) {
1316 case 32:
1317 return get(getIndirectSGPRWriteMovRelPseudo32(VecSize));
1318 case 64:
1319 return get(getIndirectSGPRWriteMovRelPseudo64(VecSize));
1320 default:
1321 llvm_unreachable("invalid reg indexing elt size")::llvm::llvm_unreachable_internal("invalid reg indexing elt size"
, "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 1321)
;
1322 }
1323 }
1324
1325 assert(EltSize == 32 && "invalid reg indexing elt size")(static_cast <bool> (EltSize == 32 && "invalid reg indexing elt size"
) ? void (0) : __assert_fail ("EltSize == 32 && \"invalid reg indexing elt size\""
, "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 1325, __extension__ __PRETTY_FUNCTION__))
;
1326 return get(getIndirectVGPRWriteMovRelPseudoOpc(VecSize));
1327}
1328
1329static unsigned getSGPRSpillSaveOpcode(unsigned Size) {
1330 switch (Size) {
1331 case 4:
1332 return AMDGPU::SI_SPILL_S32_SAVE;
1333 case 8:
1334 return AMDGPU::SI_SPILL_S64_SAVE;
1335 case 12:
1336 return AMDGPU::SI_SPILL_S96_SAVE;
1337 case 16:
1338 return AMDGPU::SI_SPILL_S128_SAVE;
1339 case 20:
1340 return AMDGPU::SI_SPILL_S160_SAVE;
1341 case 24:
1342 return AMDGPU::SI_SPILL_S192_SAVE;
1343 case 32:
1344 return AMDGPU::SI_SPILL_S256_SAVE;
1345 case 64:
1346 return AMDGPU::SI_SPILL_S512_SAVE;
1347 case 128:
1348 return AMDGPU::SI_SPILL_S1024_SAVE;
1349 default:
1350 llvm_unreachable("unknown register size")::llvm::llvm_unreachable_internal("unknown register size", "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 1350)
;
1351 }
1352}
1353
1354static unsigned getVGPRSpillSaveOpcode(unsigned Size) {
1355 switch (Size) {
1356 case 4:
1357 return AMDGPU::SI_SPILL_V32_SAVE;
1358 case 8:
1359 return AMDGPU::SI_SPILL_V64_SAVE;
1360 case 12:
1361 return AMDGPU::SI_SPILL_V96_SAVE;
1362 case 16:
1363 return AMDGPU::SI_SPILL_V128_SAVE;
1364 case 20:
1365 return AMDGPU::SI_SPILL_V160_SAVE;
1366 case 24:
1367 return AMDGPU::SI_SPILL_V192_SAVE;
1368 case 32:
1369 return AMDGPU::SI_SPILL_V256_SAVE;
1370 case 64:
1371 return AMDGPU::SI_SPILL_V512_SAVE;
1372 case 128:
1373 return AMDGPU::SI_SPILL_V1024_SAVE;
1374 default:
1375 llvm_unreachable("unknown register size")::llvm::llvm_unreachable_internal("unknown register size", "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 1375)
;
1376 }
1377}
1378
1379static unsigned getAGPRSpillSaveOpcode(unsigned Size) {
1380 switch (Size) {
1381 case 4:
1382 return AMDGPU::SI_SPILL_A32_SAVE;
1383 case 8:
1384 return AMDGPU::SI_SPILL_A64_SAVE;
1385 case 12:
1386 return AMDGPU::SI_SPILL_A96_SAVE;
1387 case 16:
1388 return AMDGPU::SI_SPILL_A128_SAVE;
1389 case 20:
1390 return AMDGPU::SI_SPILL_A160_SAVE;
1391 case 24:
1392 return AMDGPU::SI_SPILL_A192_SAVE;
1393 case 32:
1394 return AMDGPU::SI_SPILL_A256_SAVE;
1395 case 64:
1396 return AMDGPU::SI_SPILL_A512_SAVE;
1397 case 128:
1398 return AMDGPU::SI_SPILL_A1024_SAVE;
1399 default:
1400 llvm_unreachable("unknown register size")::llvm::llvm_unreachable_internal("unknown register size", "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 1400)
;
1401 }
1402}
1403
1404void SIInstrInfo::storeRegToStackSlot(MachineBasicBlock &MBB,
1405 MachineBasicBlock::iterator MI,
1406 Register SrcReg, bool isKill,
1407 int FrameIndex,
1408 const TargetRegisterClass *RC,
1409 const TargetRegisterInfo *TRI) const {
1410 MachineFunction *MF = MBB.getParent();
1411 SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>();
1412 MachineFrameInfo &FrameInfo = MF->getFrameInfo();
1413 const DebugLoc &DL = MBB.findDebugLoc(MI);
1414
1415 MachinePointerInfo PtrInfo
1416 = MachinePointerInfo::getFixedStack(*MF, FrameIndex);
1417 MachineMemOperand *MMO = MF->getMachineMemOperand(
1418 PtrInfo, MachineMemOperand::MOStore, FrameInfo.getObjectSize(FrameIndex),
1419 FrameInfo.getObjectAlign(FrameIndex));
1420 unsigned SpillSize = TRI->getSpillSize(*RC);
1421
1422 if (RI.isSGPRClass(RC)) {
1423 MFI->setHasSpilledSGPRs();
1424 assert(SrcReg != AMDGPU::M0 && "m0 should not be spilled")(static_cast <bool> (SrcReg != AMDGPU::M0 && "m0 should not be spilled"
) ? void (0) : __assert_fail ("SrcReg != AMDGPU::M0 && \"m0 should not be spilled\""
, "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 1424, __extension__ __PRETTY_FUNCTION__))
;
1425 assert(SrcReg != AMDGPU::EXEC_LO && SrcReg != AMDGPU::EXEC_HI &&(static_cast <bool> (SrcReg != AMDGPU::EXEC_LO &&
SrcReg != AMDGPU::EXEC_HI && SrcReg != AMDGPU::EXEC &&
"exec should not be spilled") ? void (0) : __assert_fail ("SrcReg != AMDGPU::EXEC_LO && SrcReg != AMDGPU::EXEC_HI && SrcReg != AMDGPU::EXEC && \"exec should not be spilled\""
, "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 1426, __extension__ __PRETTY_FUNCTION__))
1426 SrcReg != AMDGPU::EXEC && "exec should not be spilled")(static_cast <bool> (SrcReg != AMDGPU::EXEC_LO &&
SrcReg != AMDGPU::EXEC_HI && SrcReg != AMDGPU::EXEC &&
"exec should not be spilled") ? void (0) : __assert_fail ("SrcReg != AMDGPU::EXEC_LO && SrcReg != AMDGPU::EXEC_HI && SrcReg != AMDGPU::EXEC && \"exec should not be spilled\""
, "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 1426, __extension__ __PRETTY_FUNCTION__))
;
1427
1428 // We are only allowed to create one new instruction when spilling
1429 // registers, so we need to use pseudo instruction for spilling SGPRs.
1430 const MCInstrDesc &OpDesc = get(getSGPRSpillSaveOpcode(SpillSize));
1431
1432 // The SGPR spill/restore instructions only work on number sgprs, so we need
1433 // to make sure we are using the correct register class.
1434 if (SrcReg.isVirtual() && SpillSize == 4) {
1435 MachineRegisterInfo &MRI = MF->getRegInfo();
1436 MRI.constrainRegClass(SrcReg, &AMDGPU::SReg_32_XM0_XEXECRegClass);
1437 }
1438
1439 BuildMI(MBB, MI, DL, OpDesc)
1440 .addReg(SrcReg, getKillRegState(isKill)) // data
1441 .addFrameIndex(FrameIndex) // addr
1442 .addMemOperand(MMO)
1443 .addReg(MFI->getStackPtrOffsetReg(), RegState::Implicit);
1444
1445 if (RI.spillSGPRToVGPR())
1446 FrameInfo.setStackID(FrameIndex, TargetStackID::SGPRSpill);
1447 return;
1448 }
1449
1450 unsigned Opcode = RI.hasAGPRs(RC) ? getAGPRSpillSaveOpcode(SpillSize)
1451 : getVGPRSpillSaveOpcode(SpillSize);
1452 MFI->setHasSpilledVGPRs();
1453
1454 BuildMI(MBB, MI, DL, get(Opcode))
1455 .addReg(SrcReg, getKillRegState(isKill)) // data
1456 .addFrameIndex(FrameIndex) // addr
1457 .addReg(MFI->getStackPtrOffsetReg()) // scratch_offset
1458 .addImm(0) // offset
1459 .addMemOperand(MMO);
1460}
1461
1462static unsigned getSGPRSpillRestoreOpcode(unsigned Size) {
1463 switch (Size) {
1464 case 4:
1465 return AMDGPU::SI_SPILL_S32_RESTORE;
1466 case 8:
1467 return AMDGPU::SI_SPILL_S64_RESTORE;
1468 case 12:
1469 return AMDGPU::SI_SPILL_S96_RESTORE;
1470 case 16:
1471 return AMDGPU::SI_SPILL_S128_RESTORE;
1472 case 20:
1473 return AMDGPU::SI_SPILL_S160_RESTORE;
1474 case 24:
1475 return AMDGPU::SI_SPILL_S192_RESTORE;
1476 case 32:
1477 return AMDGPU::SI_SPILL_S256_RESTORE;
1478 case 64:
1479 return AMDGPU::SI_SPILL_S512_RESTORE;
1480 case 128:
1481 return AMDGPU::SI_SPILL_S1024_RESTORE;
1482 default:
1483 llvm_unreachable("unknown register size")::llvm::llvm_unreachable_internal("unknown register size", "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 1483)
;
1484 }
1485}
1486
1487static unsigned getVGPRSpillRestoreOpcode(unsigned Size) {
1488 switch (Size) {
1489 case 4:
1490 return AMDGPU::SI_SPILL_V32_RESTORE;
1491 case 8:
1492 return AMDGPU::SI_SPILL_V64_RESTORE;
1493 case 12:
1494 return AMDGPU::SI_SPILL_V96_RESTORE;
1495 case 16:
1496 return AMDGPU::SI_SPILL_V128_RESTORE;
1497 case 20:
1498 return AMDGPU::SI_SPILL_V160_RESTORE;
1499 case 24:
1500 return AMDGPU::SI_SPILL_V192_RESTORE;
1501 case 32:
1502 return AMDGPU::SI_SPILL_V256_RESTORE;
1503 case 64:
1504 return AMDGPU::SI_SPILL_V512_RESTORE;
1505 case 128:
1506 return AMDGPU::SI_SPILL_V1024_RESTORE;
1507 default:
1508 llvm_unreachable("unknown register size")::llvm::llvm_unreachable_internal("unknown register size", "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 1508)
;
1509 }
1510}
1511
1512static unsigned getAGPRSpillRestoreOpcode(unsigned Size) {
1513 switch (Size) {
1514 case 4:
1515 return AMDGPU::SI_SPILL_A32_RESTORE;
1516 case 8:
1517 return AMDGPU::SI_SPILL_A64_RESTORE;
1518 case 12:
1519 return AMDGPU::SI_SPILL_A96_RESTORE;
1520 case 16:
1521 return AMDGPU::SI_SPILL_A128_RESTORE;
1522 case 20:
1523 return AMDGPU::SI_SPILL_A160_RESTORE;
1524 case 24:
1525 return AMDGPU::SI_SPILL_A192_RESTORE;
1526 case 32:
1527 return AMDGPU::SI_SPILL_A256_RESTORE;
1528 case 64:
1529 return AMDGPU::SI_SPILL_A512_RESTORE;
1530 case 128:
1531 return AMDGPU::SI_SPILL_A1024_RESTORE;
1532 default:
1533 llvm_unreachable("unknown register size")::llvm::llvm_unreachable_internal("unknown register size", "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 1533)
;
1534 }
1535}
1536
1537void SIInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
1538 MachineBasicBlock::iterator MI,
1539 Register DestReg, int FrameIndex,
1540 const TargetRegisterClass *RC,
1541 const TargetRegisterInfo *TRI) const {
1542 MachineFunction *MF = MBB.getParent();
1543 SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>();
1544 MachineFrameInfo &FrameInfo = MF->getFrameInfo();
1545 const DebugLoc &DL = MBB.findDebugLoc(MI);
1546 unsigned SpillSize = TRI->getSpillSize(*RC);
1547
1548 MachinePointerInfo PtrInfo
1549 = MachinePointerInfo::getFixedStack(*MF, FrameIndex);
1550
1551 MachineMemOperand *MMO = MF->getMachineMemOperand(
1552 PtrInfo, MachineMemOperand::MOLoad, FrameInfo.getObjectSize(FrameIndex),
1553 FrameInfo.getObjectAlign(FrameIndex));
1554
1555 if (RI.isSGPRClass(RC)) {
1556 MFI->setHasSpilledSGPRs();
1557 assert(DestReg != AMDGPU::M0 && "m0 should not be reloaded into")(static_cast <bool> (DestReg != AMDGPU::M0 && "m0 should not be reloaded into"
) ? void (0) : __assert_fail ("DestReg != AMDGPU::M0 && \"m0 should not be reloaded into\""
, "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 1557, __extension__ __PRETTY_FUNCTION__))
;
1558 assert(DestReg != AMDGPU::EXEC_LO && DestReg != AMDGPU::EXEC_HI &&(static_cast <bool> (DestReg != AMDGPU::EXEC_LO &&
DestReg != AMDGPU::EXEC_HI && DestReg != AMDGPU::EXEC
&& "exec should not be spilled") ? void (0) : __assert_fail
("DestReg != AMDGPU::EXEC_LO && DestReg != AMDGPU::EXEC_HI && DestReg != AMDGPU::EXEC && \"exec should not be spilled\""
, "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 1559, __extension__ __PRETTY_FUNCTION__))
1559 DestReg != AMDGPU::EXEC && "exec should not be spilled")(static_cast <bool> (DestReg != AMDGPU::EXEC_LO &&
DestReg != AMDGPU::EXEC_HI && DestReg != AMDGPU::EXEC
&& "exec should not be spilled") ? void (0) : __assert_fail
("DestReg != AMDGPU::EXEC_LO && DestReg != AMDGPU::EXEC_HI && DestReg != AMDGPU::EXEC && \"exec should not be spilled\""
, "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 1559, __extension__ __PRETTY_FUNCTION__))
;
1560
1561 // FIXME: Maybe this should not include a memoperand because it will be
1562 // lowered to non-memory instructions.
1563 const MCInstrDesc &OpDesc = get(getSGPRSpillRestoreOpcode(SpillSize));
1564 if (DestReg.isVirtual() && SpillSize == 4) {
1565 MachineRegisterInfo &MRI = MF->getRegInfo();
1566 MRI.constrainRegClass(DestReg, &AMDGPU::SReg_32_XM0_XEXECRegClass);
1567 }
1568
1569 if (RI.spillSGPRToVGPR())
1570 FrameInfo.setStackID(FrameIndex, TargetStackID::SGPRSpill);
1571 BuildMI(MBB, MI, DL, OpDesc, DestReg)
1572 .addFrameIndex(FrameIndex) // addr
1573 .addMemOperand(MMO)
1574 .addReg(MFI->getStackPtrOffsetReg(), RegState::Implicit);
1575
1576 return;
1577 }
1578
1579 unsigned Opcode = RI.hasAGPRs(RC) ? getAGPRSpillRestoreOpcode(SpillSize)
1580 : getVGPRSpillRestoreOpcode(SpillSize);
1581 BuildMI(MBB, MI, DL, get(Opcode), DestReg)
1582 .addFrameIndex(FrameIndex) // vaddr
1583 .addReg(MFI->getStackPtrOffsetReg()) // scratch_offset
1584 .addImm(0) // offset
1585 .addMemOperand(MMO);
1586}
1587
1588void SIInstrInfo::insertNoop(MachineBasicBlock &MBB,
1589 MachineBasicBlock::iterator MI) const {
1590 insertNoops(MBB, MI, 1);
1591}
1592
1593void SIInstrInfo::insertNoops(MachineBasicBlock &MBB,
1594 MachineBasicBlock::iterator MI,
1595 unsigned Quantity) const {
1596 DebugLoc DL = MBB.findDebugLoc(MI);
1597 while (Quantity > 0) {
1598 unsigned Arg = std::min(Quantity, 8u);
1599 Quantity -= Arg;
1600 BuildMI(MBB, MI, DL, get(AMDGPU::S_NOP)).addImm(Arg - 1);
1601 }
1602}
1603
1604void SIInstrInfo::insertReturn(MachineBasicBlock &MBB) const {
1605 auto MF = MBB.getParent();
1606 SIMachineFunctionInfo *Info = MF->getInfo<SIMachineFunctionInfo>();
1607
1608 assert(Info->isEntryFunction())(static_cast <bool> (Info->isEntryFunction()) ? void
(0) : __assert_fail ("Info->isEntryFunction()", "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 1608, __extension__ __PRETTY_FUNCTION__))
;
1609
1610 if (MBB.succ_empty()) {
1611 bool HasNoTerminator = MBB.getFirstTerminator() == MBB.end();
1612 if (HasNoTerminator) {
1613 if (Info->returnsVoid()) {
1614 BuildMI(MBB, MBB.end(), DebugLoc(), get(AMDGPU::S_ENDPGM)).addImm(0);
1615 } else {
1616 BuildMI(MBB, MBB.end(), DebugLoc(), get(AMDGPU::SI_RETURN_TO_EPILOG));
1617 }
1618 }
1619 }
1620}
1621
1622unsigned SIInstrInfo::getNumWaitStates(const MachineInstr &MI) {
1623 switch (MI.getOpcode()) {
1624 default: return 1; // FIXME: Do wait states equal cycles?
1625
1626 case AMDGPU::S_NOP:
1627 return MI.getOperand(0).getImm() + 1;
1628 }
1629}
1630
1631bool SIInstrInfo::expandPostRAPseudo(MachineInstr &MI) const {
1632 const SIRegisterInfo *TRI = ST.getRegisterInfo();
1633 MachineBasicBlock &MBB = *MI.getParent();
1634 DebugLoc DL = MBB.findDebugLoc(MI);
1635 switch (MI.getOpcode()) {
1
Control jumps to 'case V_MOV_B64_DPP_PSEUDO:' at line 1752
1636 default: return TargetInstrInfo::expandPostRAPseudo(MI);
1637 case AMDGPU::S_MOV_B64_term:
1638 // This is only a terminator to get the correct spill code placement during
1639 // register allocation.
1640 MI.setDesc(get(AMDGPU::S_MOV_B64));
1641 break;
1642
1643 case AMDGPU::S_MOV_B32_term:
1644 // This is only a terminator to get the correct spill code placement during
1645 // register allocation.
1646 MI.setDesc(get(AMDGPU::S_MOV_B32));
1647 break;
1648
1649 case AMDGPU::S_XOR_B64_term:
1650 // This is only a terminator to get the correct spill code placement during
1651 // register allocation.
1652 MI.setDesc(get(AMDGPU::S_XOR_B64));
1653 break;
1654
1655 case AMDGPU::S_XOR_B32_term:
1656 // This is only a terminator to get the correct spill code placement during
1657 // register allocation.
1658 MI.setDesc(get(AMDGPU::S_XOR_B32));
1659 break;
1660 case AMDGPU::S_OR_B64_term:
1661 // This is only a terminator to get the correct spill code placement during
1662 // register allocation.
1663 MI.setDesc(get(AMDGPU::S_OR_B64));
1664 break;
1665 case AMDGPU::S_OR_B32_term:
1666 // This is only a terminator to get the correct spill code placement during
1667 // register allocation.
1668 MI.setDesc(get(AMDGPU::S_OR_B32));
1669 break;
1670
1671 case AMDGPU::S_ANDN2_B64_term:
1672 // This is only a terminator to get the correct spill code placement during
1673 // register allocation.
1674 MI.setDesc(get(AMDGPU::S_ANDN2_B64));
1675 break;
1676
1677 case AMDGPU::S_ANDN2_B32_term:
1678 // This is only a terminator to get the correct spill code placement during
1679 // register allocation.
1680 MI.setDesc(get(AMDGPU::S_ANDN2_B32));
1681 break;
1682
1683 case AMDGPU::S_AND_B64_term:
1684 // This is only a terminator to get the correct spill code placement during
1685 // register allocation.
1686 MI.setDesc(get(AMDGPU::S_AND_B64));
1687 break;
1688
1689 case AMDGPU::S_AND_B32_term:
1690 // This is only a terminator to get the correct spill code placement during
1691 // register allocation.
1692 MI.setDesc(get(AMDGPU::S_AND_B32));
1693 break;
1694
1695 case AMDGPU::V_MOV_B64_PSEUDO: {
1696 Register Dst = MI.getOperand(0).getReg();
1697 Register DstLo = RI.getSubReg(Dst, AMDGPU::sub0);
1698 Register DstHi = RI.getSubReg(Dst, AMDGPU::sub1);
1699
1700 const MachineOperand &SrcOp = MI.getOperand(1);
1701 // FIXME: Will this work for 64-bit floating point immediates?
1702 assert(!SrcOp.isFPImm())(static_cast <bool> (!SrcOp.isFPImm()) ? void (0) : __assert_fail
("!SrcOp.isFPImm()", "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 1702, __extension__ __PRETTY_FUNCTION__))
;
1703 if (SrcOp.isImm()) {
1704 APInt Imm(64, SrcOp.getImm());
1705 APInt Lo(32, Imm.getLoBits(32).getZExtValue());
1706 APInt Hi(32, Imm.getHiBits(32).getZExtValue());
1707 if (ST.hasPackedFP32Ops() && Lo == Hi && isInlineConstant(Lo)) {
1708 BuildMI(MBB, MI, DL, get(AMDGPU::V_PK_MOV_B32), Dst)
1709 .addImm(SISrcMods::OP_SEL_1)
1710 .addImm(Lo.getSExtValue())
1711 .addImm(SISrcMods::OP_SEL_1)
1712 .addImm(Lo.getSExtValue())
1713 .addImm(0) // op_sel_lo
1714 .addImm(0) // op_sel_hi
1715 .addImm(0) // neg_lo
1716 .addImm(0) // neg_hi
1717 .addImm(0); // clamp
1718 } else {
1719 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstLo)
1720 .addImm(Lo.getZExtValue())
1721 .addReg(Dst, RegState::Implicit | RegState::Define);
1722 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstHi)
1723 .addImm(Hi.getZExtValue())
1724 .addReg(Dst, RegState::Implicit | RegState::Define);
1725 }
1726 } else {
1727 assert(SrcOp.isReg())(static_cast <bool> (SrcOp.isReg()) ? void (0) : __assert_fail
("SrcOp.isReg()", "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 1727, __extension__ __PRETTY_FUNCTION__))
;
1728 if (ST.hasPackedFP32Ops() &&
1729 !RI.isAGPR(MBB.getParent()->getRegInfo(), SrcOp.getReg())) {
1730 BuildMI(MBB, MI, DL, get(AMDGPU::V_PK_MOV_B32), Dst)
1731 .addImm(SISrcMods::OP_SEL_1) // src0_mod
1732 .addReg(SrcOp.getReg())
1733 .addImm(SISrcMods::OP_SEL_0 | SISrcMods::OP_SEL_1) // src1_mod
1734 .addReg(SrcOp.getReg())
1735 .addImm(0) // op_sel_lo
1736 .addImm(0) // op_sel_hi
1737 .addImm(0) // neg_lo
1738 .addImm(0) // neg_hi
1739 .addImm(0); // clamp
1740 } else {
1741 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstLo)
1742 .addReg(RI.getSubReg(SrcOp.getReg(), AMDGPU::sub0))
1743 .addReg(Dst, RegState::Implicit | RegState::Define);
1744 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstHi)
1745 .addReg(RI.getSubReg(SrcOp.getReg(), AMDGPU::sub1))
1746 .addReg(Dst, RegState::Implicit | RegState::Define);
1747 }
1748 }
1749 MI.eraseFromParent();
1750 break;
1751 }
1752 case AMDGPU::V_MOV_B64_DPP_PSEUDO: {
1753 expandMovDPP64(MI);
2
Calling 'SIInstrInfo::expandMovDPP64'
1754 break;
1755 }
1756 case AMDGPU::V_SET_INACTIVE_B32: {
1757 unsigned NotOpc = ST.isWave32() ? AMDGPU::S_NOT_B32 : AMDGPU::S_NOT_B64;
1758 unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
1759 auto FirstNot = BuildMI(MBB, MI, DL, get(NotOpc), Exec).addReg(Exec);
1760 FirstNot->addRegisterDead(AMDGPU::SCC, TRI); // SCC is overwritten
1761 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), MI.getOperand(0).getReg())
1762 .add(MI.getOperand(2));
1763 BuildMI(MBB, MI, DL, get(NotOpc), Exec)
1764 .addReg(Exec);
1765 MI.eraseFromParent();
1766 break;
1767 }
1768 case AMDGPU::V_SET_INACTIVE_B64: {
1769 unsigned NotOpc = ST.isWave32() ? AMDGPU::S_NOT_B32 : AMDGPU::S_NOT_B64;
1770 unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
1771 auto FirstNot = BuildMI(MBB, MI, DL, get(NotOpc), Exec).addReg(Exec);
1772 FirstNot->addRegisterDead(AMDGPU::SCC, TRI); // SCC is overwritten
1773 MachineInstr *Copy = BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B64_PSEUDO),
1774 MI.getOperand(0).getReg())
1775 .add(MI.getOperand(2));
1776 expandPostRAPseudo(*Copy);
1777 BuildMI(MBB, MI, DL, get(NotOpc), Exec)
1778 .addReg(Exec);
1779 MI.eraseFromParent();
1780 break;
1781 }
1782 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V1:
1783 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V2:
1784 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V3:
1785 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V4:
1786 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V5:
1787 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V8:
1788 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V16:
1789 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V32:
1790 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V1:
1791 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V2:
1792 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V3:
1793 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V4:
1794 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V5:
1795 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V8:
1796 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V16:
1797 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V32:
1798 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V1:
1799 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V2:
1800 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V4:
1801 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V8:
1802 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V16: {
1803 const TargetRegisterClass *EltRC = getOpRegClass(MI, 2);
1804
1805 unsigned Opc;
1806 if (RI.hasVGPRs(EltRC)) {
1807 Opc = AMDGPU::V_MOVRELD_B32_e32;
1808 } else {
1809 Opc = RI.getRegSizeInBits(*EltRC) == 64 ? AMDGPU::S_MOVRELD_B64
1810 : AMDGPU::S_MOVRELD_B32;
1811 }
1812
1813 const MCInstrDesc &OpDesc = get(Opc);
1814 Register VecReg = MI.getOperand(0).getReg();
1815 bool IsUndef = MI.getOperand(1).isUndef();
1816 unsigned SubReg = MI.getOperand(3).getImm();
1817 assert(VecReg == MI.getOperand(1).getReg())(static_cast <bool> (VecReg == MI.getOperand(1).getReg(
)) ? void (0) : __assert_fail ("VecReg == MI.getOperand(1).getReg()"
, "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 1817, __extension__ __PRETTY_FUNCTION__))
;
1818
1819 MachineInstrBuilder MIB =
1820 BuildMI(MBB, MI, DL, OpDesc)
1821 .addReg(RI.getSubReg(VecReg, SubReg), RegState::Undef)
1822 .add(MI.getOperand(2))
1823 .addReg(VecReg, RegState::ImplicitDefine)
1824 .addReg(VecReg, RegState::Implicit | (IsUndef ? RegState::Undef : 0));
1825
1826 const int ImpDefIdx =
1827 OpDesc.getNumOperands() + OpDesc.getNumImplicitUses();
1828 const int ImpUseIdx = ImpDefIdx + 1;
1829 MIB->tieOperands(ImpDefIdx, ImpUseIdx);
1830 MI.eraseFromParent();
1831 break;
1832 }
1833 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V1:
1834 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V2:
1835 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V3:
1836 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V4:
1837 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V5:
1838 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V8:
1839 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V16:
1840 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V32: {
1841 assert(ST.useVGPRIndexMode())(static_cast <bool> (ST.useVGPRIndexMode()) ? void (0) :
__assert_fail ("ST.useVGPRIndexMode()", "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 1841, __extension__ __PRETTY_FUNCTION__))
;
1842 Register VecReg = MI.getOperand(0).getReg();
1843 bool IsUndef = MI.getOperand(1).isUndef();
1844 Register Idx = MI.getOperand(3).getReg();
1845 Register SubReg = MI.getOperand(4).getImm();
1846
1847 MachineInstr *SetOn = BuildMI(MBB, MI, DL, get(AMDGPU::S_SET_GPR_IDX_ON))
1848 .addReg(Idx)
1849 .addImm(AMDGPU::VGPRIndexMode::DST_ENABLE);
1850 SetOn->getOperand(3).setIsUndef();
1851
1852 const MCInstrDesc &OpDesc = get(AMDGPU::V_MOV_B32_indirect);
1853 MachineInstrBuilder MIB =
1854 BuildMI(MBB, MI, DL, OpDesc)
1855 .addReg(RI.getSubReg(VecReg, SubReg), RegState::Undef)
1856 .add(MI.getOperand(2))
1857 .addReg(VecReg, RegState::ImplicitDefine)
1858 .addReg(VecReg,
1859 RegState::Implicit | (IsUndef ? RegState::Undef : 0));
1860
1861 const int ImpDefIdx = OpDesc.getNumOperands() + OpDesc.getNumImplicitUses();
1862 const int ImpUseIdx = ImpDefIdx + 1;
1863 MIB->tieOperands(ImpDefIdx, ImpUseIdx);
1864
1865 MachineInstr *SetOff = BuildMI(MBB, MI, DL, get(AMDGPU::S_SET_GPR_IDX_OFF));
1866
1867 finalizeBundle(MBB, SetOn->getIterator(), std::next(SetOff->getIterator()));
1868
1869 MI.eraseFromParent();
1870 break;
1871 }
1872 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V1:
1873 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V2:
1874 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V3:
1875 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V4:
1876 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V5:
1877 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V8:
1878 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V16:
1879 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V32: {
1880 assert(ST.useVGPRIndexMode())(static_cast <bool> (ST.useVGPRIndexMode()) ? void (0) :
__assert_fail ("ST.useVGPRIndexMode()", "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 1880, __extension__ __PRETTY_FUNCTION__))
;
1881 Register Dst = MI.getOperand(0).getReg();
1882 Register VecReg = MI.getOperand(1).getReg();
1883 bool IsUndef = MI.getOperand(1).isUndef();
1884 Register Idx = MI.getOperand(2).getReg();
1885 Register SubReg = MI.getOperand(3).getImm();
1886
1887 MachineInstr *SetOn = BuildMI(MBB, MI, DL, get(AMDGPU::S_SET_GPR_IDX_ON))
1888 .addReg(Idx)
1889 .addImm(AMDGPU::VGPRIndexMode::SRC0_ENABLE);
1890 SetOn->getOperand(3).setIsUndef();
1891
1892 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32))
1893 .addDef(Dst)
1894 .addReg(RI.getSubReg(VecReg, SubReg), RegState::Undef)
1895 .addReg(VecReg, RegState::Implicit | (IsUndef ? RegState::Undef : 0))
1896 .addReg(AMDGPU::M0, RegState::Implicit);
1897
1898 MachineInstr *SetOff = BuildMI(MBB, MI, DL, get(AMDGPU::S_SET_GPR_IDX_OFF));
1899
1900 finalizeBundle(MBB, SetOn->getIterator(), std::next(SetOff->getIterator()));
1901
1902 MI.eraseFromParent();
1903 break;
1904 }
1905 case AMDGPU::SI_PC_ADD_REL_OFFSET: {
1906 MachineFunction &MF = *MBB.getParent();
1907 Register Reg = MI.getOperand(0).getReg();
1908 Register RegLo = RI.getSubReg(Reg, AMDGPU::sub0);
1909 Register RegHi = RI.getSubReg(Reg, AMDGPU::sub1);
1910
1911 // Create a bundle so these instructions won't be re-ordered by the
1912 // post-RA scheduler.
1913 MIBundleBuilder Bundler(MBB, MI);
1914 Bundler.append(BuildMI(MF, DL, get(AMDGPU::S_GETPC_B64), Reg));
1915
1916 // Add 32-bit offset from this instruction to the start of the
1917 // constant data.
1918 Bundler.append(BuildMI(MF, DL, get(AMDGPU::S_ADD_U32), RegLo)
1919 .addReg(RegLo)
1920 .add(MI.getOperand(1)));
1921
1922 MachineInstrBuilder MIB = BuildMI(MF, DL, get(AMDGPU::S_ADDC_U32), RegHi)
1923 .addReg(RegHi);
1924 MIB.add(MI.getOperand(2));
1925
1926 Bundler.append(MIB);
1927 finalizeBundle(MBB, Bundler.begin());
1928
1929 MI.eraseFromParent();
1930 break;
1931 }
1932 case AMDGPU::ENTER_STRICT_WWM: {
1933 // This only gets its own opcode so that SIPreAllocateWWMRegs can tell when
1934 // Whole Wave Mode is entered.
1935 MI.setDesc(get(ST.isWave32() ? AMDGPU::S_OR_SAVEEXEC_B32
1936 : AMDGPU::S_OR_SAVEEXEC_B64));
1937 break;
1938 }
1939 case AMDGPU::ENTER_STRICT_WQM: {
1940 // This only gets its own opcode so that SIPreAllocateWWMRegs can tell when
1941 // STRICT_WQM is entered.
1942 const unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
1943 const unsigned WQMOp = ST.isWave32() ? AMDGPU::S_WQM_B32 : AMDGPU::S_WQM_B64;
1944 const unsigned MovOp = ST.isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64;
1945 BuildMI(MBB, MI, DL, get(MovOp), MI.getOperand(0).getReg()).addReg(Exec);
1946 BuildMI(MBB, MI, DL, get(WQMOp), Exec).addReg(Exec);
1947
1948 MI.eraseFromParent();
1949 break;
1950 }
1951 case AMDGPU::EXIT_STRICT_WWM:
1952 case AMDGPU::EXIT_STRICT_WQM: {
1953 // This only gets its own opcode so that SIPreAllocateWWMRegs can tell when
1954 // WWM/STICT_WQM is exited.
1955 MI.setDesc(get(ST.isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64));
1956 break;
1957 }
1958 }
1959 return true;
1960}
1961
1962std::pair<MachineInstr*, MachineInstr*>
1963SIInstrInfo::expandMovDPP64(MachineInstr &MI) const {
1964 assert (MI.getOpcode() == AMDGPU::V_MOV_B64_DPP_PSEUDO)(static_cast <bool> (MI.getOpcode() == AMDGPU::V_MOV_B64_DPP_PSEUDO
) ? void (0) : __assert_fail ("MI.getOpcode() == AMDGPU::V_MOV_B64_DPP_PSEUDO"
, "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 1964, __extension__ __PRETTY_FUNCTION__))
;
3
'?' condition is true
1965
1966 MachineBasicBlock &MBB = *MI.getParent();
1967 DebugLoc DL = MBB.findDebugLoc(MI);
1968 MachineFunction *MF = MBB.getParent();
1969 MachineRegisterInfo &MRI = MF->getRegInfo();
1970 Register Dst = MI.getOperand(0).getReg();
1971 unsigned Part = 0;
1972 MachineInstr *Split[2];
1973
1974 for (auto Sub : { AMDGPU::sub0, AMDGPU::sub1 }) {
4
Assuming '__begin1' is equal to '__end1'
1975 auto MovDPP = BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_dpp));
1976 if (Dst.isPhysical()) {
1977 MovDPP.addDef(RI.getSubReg(Dst, Sub));
1978 } else {
1979 assert(MRI.isSSA())(static_cast <bool> (MRI.isSSA()) ? void (0) : __assert_fail
("MRI.isSSA()", "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 1979, __extension__ __PRETTY_FUNCTION__))
;
1980 auto Tmp = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
1981 MovDPP.addDef(Tmp);
1982 }
1983
1984 for (unsigned I = 1; I <= 2; ++I) { // old and src operands.
1985 const MachineOperand &SrcOp = MI.getOperand(I);
1986 assert(!SrcOp.isFPImm())(static_cast <bool> (!SrcOp.isFPImm()) ? void (0) : __assert_fail
("!SrcOp.isFPImm()", "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 1986, __extension__ __PRETTY_FUNCTION__))
;
1987 if (SrcOp.isImm()) {
1988 APInt Imm(64, SrcOp.getImm());
1989 Imm.ashrInPlace(Part * 32);
1990 MovDPP.addImm(Imm.getLoBits(32).getZExtValue());
1991 } else {
1992 assert(SrcOp.isReg())(static_cast <bool> (SrcOp.isReg()) ? void (0) : __assert_fail
("SrcOp.isReg()", "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 1992, __extension__ __PRETTY_FUNCTION__))
;
1993 Register Src = SrcOp.getReg();
1994 if (Src.isPhysical())
1995 MovDPP.addReg(RI.getSubReg(Src, Sub));
1996 else
1997 MovDPP.addReg(Src, SrcOp.isUndef() ? RegState::Undef : 0, Sub);
1998 }
1999 }
2000
2001 for (unsigned I = 3; I < MI.getNumExplicitOperands(); ++I)
2002 MovDPP.addImm(MI.getOperand(I).getImm());
2003
2004 Split[Part] = MovDPP;
2005 ++Part;
2006 }
2007
2008 if (Dst.isVirtual())
5
Calling 'Register::isVirtual'
11
Returning from 'Register::isVirtual'
12
Taking true branch
2009 BuildMI(MBB, MI, DL, get(AMDGPU::REG_SEQUENCE), Dst)
2010 .addReg(Split[0]->getOperand(0).getReg())
13
Called C++ object pointer is uninitialized
2011 .addImm(AMDGPU::sub0)
2012 .addReg(Split[1]->getOperand(0).getReg())
2013 .addImm(AMDGPU::sub1);
2014
2015 MI.eraseFromParent();
2016 return std::make_pair(Split[0], Split[1]);
2017}
2018
2019bool SIInstrInfo::swapSourceModifiers(MachineInstr &MI,
2020 MachineOperand &Src0,
2021 unsigned Src0OpName,
2022 MachineOperand &Src1,
2023 unsigned Src1OpName) const {
2024 MachineOperand *Src0Mods = getNamedOperand(MI, Src0OpName);
2025 if (!Src0Mods)
2026 return false;
2027
2028 MachineOperand *Src1Mods = getNamedOperand(MI, Src1OpName);
2029 assert(Src1Mods &&(static_cast <bool> (Src1Mods && "All commutable instructions have both src0 and src1 modifiers"
) ? void (0) : __assert_fail ("Src1Mods && \"All commutable instructions have both src0 and src1 modifiers\""
, "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 2030, __extension__ __PRETTY_FUNCTION__))
2030 "All commutable instructions have both src0 and src1 modifiers")(static_cast <bool> (Src1Mods && "All commutable instructions have both src0 and src1 modifiers"
) ? void (0) : __assert_fail ("Src1Mods && \"All commutable instructions have both src0 and src1 modifiers\""
, "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 2030, __extension__ __PRETTY_FUNCTION__))
;
2031
2032 int Src0ModsVal = Src0Mods->getImm();
2033 int Src1ModsVal = Src1Mods->getImm();
2034
2035 Src1Mods->setImm(Src0ModsVal);
2036 Src0Mods->setImm(Src1ModsVal);
2037 return true;
2038}
2039
2040static MachineInstr *swapRegAndNonRegOperand(MachineInstr &MI,
2041 MachineOperand &RegOp,
2042 MachineOperand &NonRegOp) {
2043 Register Reg = RegOp.getReg();
2044 unsigned SubReg = RegOp.getSubReg();
2045 bool IsKill = RegOp.isKill();
2046 bool IsDead = RegOp.isDead();
2047 bool IsUndef = RegOp.isUndef();
2048 bool IsDebug = RegOp.isDebug();
2049
2050 if (NonRegOp.isImm())
2051 RegOp.ChangeToImmediate(NonRegOp.getImm());
2052 else if (NonRegOp.isFI())
2053 RegOp.ChangeToFrameIndex(NonRegOp.getIndex());
2054 else if (NonRegOp.isGlobal()) {
2055 RegOp.ChangeToGA(NonRegOp.getGlobal(), NonRegOp.getOffset(),
2056 NonRegOp.getTargetFlags());
2057 } else
2058 return nullptr;
2059
2060 // Make sure we don't reinterpret a subreg index in the target flags.
2061 RegOp.setTargetFlags(NonRegOp.getTargetFlags());
2062
2063 NonRegOp.ChangeToRegister(Reg, false, false, IsKill, IsDead, IsUndef, IsDebug);
2064 NonRegOp.setSubReg(SubReg);
2065
2066 return &MI;
2067}
2068
2069MachineInstr *SIInstrInfo::commuteInstructionImpl(MachineInstr &MI, bool NewMI,
2070 unsigned Src0Idx,
2071 unsigned Src1Idx) const {
2072 assert(!NewMI && "this should never be used")(static_cast <bool> (!NewMI && "this should never be used"
) ? void (0) : __assert_fail ("!NewMI && \"this should never be used\""
, "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 2072, __extension__ __PRETTY_FUNCTION__))
;
2073
2074 unsigned Opc = MI.getOpcode();
2075 int CommutedOpcode = commuteOpcode(Opc);
2076 if (CommutedOpcode == -1)
2077 return nullptr;
2078
2079 assert(AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0) ==(static_cast <bool> (AMDGPU::getNamedOperandIdx(Opc, AMDGPU
::OpName::src0) == static_cast<int>(Src0Idx) &&
AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1) == static_cast
<int>(Src1Idx) && "inconsistency with findCommutedOpIndices"
) ? void (0) : __assert_fail ("AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0) == static_cast<int>(Src0Idx) && AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1) == static_cast<int>(Src1Idx) && \"inconsistency with findCommutedOpIndices\""
, "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 2083, __extension__ __PRETTY_FUNCTION__))
2080 static_cast<int>(Src0Idx) &&(static_cast <bool> (AMDGPU::getNamedOperandIdx(Opc, AMDGPU
::OpName::src0) == static_cast<int>(Src0Idx) &&
AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1) == static_cast
<int>(Src1Idx) && "inconsistency with findCommutedOpIndices"
) ? void (0) : __assert_fail ("AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0) == static_cast<int>(Src0Idx) && AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1) == static_cast<int>(Src1Idx) && \"inconsistency with findCommutedOpIndices\""
, "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 2083, __extension__ __PRETTY_FUNCTION__))
2081 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1) ==(static_cast <bool> (AMDGPU::getNamedOperandIdx(Opc, AMDGPU
::OpName::src0) == static_cast<int>(Src0Idx) &&
AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1) == static_cast
<int>(Src1Idx) && "inconsistency with findCommutedOpIndices"
) ? void (0) : __assert_fail ("AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0) == static_cast<int>(Src0Idx) && AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1) == static_cast<int>(Src1Idx) && \"inconsistency with findCommutedOpIndices\""
, "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 2083, __extension__ __PRETTY_FUNCTION__))
2082 static_cast<int>(Src1Idx) &&(static_cast <bool> (AMDGPU::getNamedOperandIdx(Opc, AMDGPU
::OpName::src0) == static_cast<int>(Src0Idx) &&
AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1) == static_cast
<int>(Src1Idx) && "inconsistency with findCommutedOpIndices"
) ? void (0) : __assert_fail ("AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0) == static_cast<int>(Src0Idx) && AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1) == static_cast<int>(Src1Idx) && \"inconsistency with findCommutedOpIndices\""
, "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 2083, __extension__ __PRETTY_FUNCTION__))
2083 "inconsistency with findCommutedOpIndices")(static_cast <bool> (AMDGPU::getNamedOperandIdx(Opc, AMDGPU
::OpName::src0) == static_cast<int>(Src0Idx) &&
AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1) == static_cast
<int>(Src1Idx) && "inconsistency with findCommutedOpIndices"
) ? void (0) : __assert_fail ("AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0) == static_cast<int>(Src0Idx) && AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1) == static_cast<int>(Src1Idx) && \"inconsistency with findCommutedOpIndices\""
, "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 2083, __extension__ __PRETTY_FUNCTION__))
;
2084
2085 MachineOperand &Src0 = MI.getOperand(Src0Idx);
2086 MachineOperand &Src1 = MI.getOperand(Src1Idx);
2087
2088 MachineInstr *CommutedMI = nullptr;
2089 if (Src0.isReg() && Src1.isReg()) {
2090 if (isOperandLegal(MI, Src1Idx, &Src0)) {
2091 // Be sure to copy the source modifiers to the right place.
2092 CommutedMI
2093 = TargetInstrInfo::commuteInstructionImpl(MI, NewMI, Src0Idx, Src1Idx);
2094 }
2095
2096 } else if (Src0.isReg() && !Src1.isReg()) {
2097 // src0 should always be able to support any operand type, so no need to
2098 // check operand legality.
2099 CommutedMI = swapRegAndNonRegOperand(MI, Src0, Src1);
2100 } else if (!Src0.isReg() && Src1.isReg()) {
2101 if (isOperandLegal(MI, Src1Idx, &Src0))
2102 CommutedMI = swapRegAndNonRegOperand(MI, Src1, Src0);
2103 } else {
2104 // FIXME: Found two non registers to commute. This does happen.
2105 return nullptr;
2106 }
2107
2108 if (CommutedMI) {
2109 swapSourceModifiers(MI, Src0, AMDGPU::OpName::src0_modifiers,
2110 Src1, AMDGPU::OpName::src1_modifiers);
2111
2112 CommutedMI->setDesc(get(CommutedOpcode));
2113 }
2114
2115 return CommutedMI;
2116}
2117
2118// This needs to be implemented because the source modifiers may be inserted
2119// between the true commutable operands, and the base
2120// TargetInstrInfo::commuteInstruction uses it.
2121bool SIInstrInfo::findCommutedOpIndices(const MachineInstr &MI,
2122 unsigned &SrcOpIdx0,
2123 unsigned &SrcOpIdx1) const {
2124 return findCommutedOpIndices(MI.getDesc(), SrcOpIdx0, SrcOpIdx1);
2125}
2126
2127bool SIInstrInfo::findCommutedOpIndices(MCInstrDesc Desc, unsigned &SrcOpIdx0,
2128 unsigned &SrcOpIdx1) const {
2129 if (!Desc.isCommutable())
2130 return false;
2131
2132 unsigned Opc = Desc.getOpcode();
2133 int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0);
2134 if (Src0Idx == -1)
2135 return false;
2136
2137 int Src1Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1);
2138 if (Src1Idx == -1)
2139 return false;
2140
2141 return fixCommutedOpIndices(SrcOpIdx0, SrcOpIdx1, Src0Idx, Src1Idx);
2142}
2143
2144bool SIInstrInfo::isBranchOffsetInRange(unsigned BranchOp,
2145 int64_t BrOffset) const {
2146 // BranchRelaxation should never have to check s_setpc_b64 because its dest
2147 // block is unanalyzable.
2148 assert(BranchOp != AMDGPU::S_SETPC_B64)(static_cast <bool> (BranchOp != AMDGPU::S_SETPC_B64) ?
void (0) : __assert_fail ("BranchOp != AMDGPU::S_SETPC_B64",
"/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 2148, __extension__ __PRETTY_FUNCTION__))
;
2149
2150 // Convert to dwords.
2151 BrOffset /= 4;
2152
2153 // The branch instructions do PC += signext(SIMM16 * 4) + 4, so the offset is
2154 // from the next instruction.
2155 BrOffset -= 1;
2156
2157 return isIntN(BranchOffsetBits, BrOffset);
2158}
2159
2160MachineBasicBlock *SIInstrInfo::getBranchDestBlock(
2161 const MachineInstr &MI) const {
2162 if (MI.getOpcode() == AMDGPU::S_SETPC_B64) {
2163 // This would be a difficult analysis to perform, but can always be legal so
2164 // there's no need to analyze it.
2165 return nullptr;
2166 }
2167
2168 return MI.getOperand(0).getMBB();
2169}
2170
2171unsigned SIInstrInfo::insertIndirectBranch(MachineBasicBlock &MBB,
2172 MachineBasicBlock &DestBB,
2173 const DebugLoc &DL,
2174 int64_t BrOffset,
2175 RegScavenger *RS) const {
2176 assert(RS && "RegScavenger required for long branching")(static_cast <bool> (RS && "RegScavenger required for long branching"
) ? void (0) : __assert_fail ("RS && \"RegScavenger required for long branching\""
, "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 2176, __extension__ __PRETTY_FUNCTION__))
;
2177 assert(MBB.empty() &&(static_cast <bool> (MBB.empty() && "new block should be inserted for expanding unconditional branch"
) ? void (0) : __assert_fail ("MBB.empty() && \"new block should be inserted for expanding unconditional branch\""
, "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 2178, __extension__ __PRETTY_FUNCTION__))
2178 "new block should be inserted for expanding unconditional branch")(static_cast <bool> (MBB.empty() && "new block should be inserted for expanding unconditional branch"
) ? void (0) : __assert_fail ("MBB.empty() && \"new block should be inserted for expanding unconditional branch\""
, "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 2178, __extension__ __PRETTY_FUNCTION__))
;
2179 assert(MBB.pred_size() == 1)(static_cast <bool> (MBB.pred_size() == 1) ? void (0) :
__assert_fail ("MBB.pred_size() == 1", "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 2179, __extension__ __PRETTY_FUNCTION__))
;
2180
2181 MachineFunction *MF = MBB.getParent();
2182 MachineRegisterInfo &MRI = MF->getRegInfo();
2183
2184 // FIXME: Virtual register workaround for RegScavenger not working with empty
2185 // blocks.
2186 Register PCReg = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
2187
2188 auto I = MBB.end();
2189
2190 // We need to compute the offset relative to the instruction immediately after
2191 // s_getpc_b64. Insert pc arithmetic code before last terminator.
2192 MachineInstr *GetPC = BuildMI(MBB, I, DL, get(AMDGPU::S_GETPC_B64), PCReg);
2193
2194 // TODO: Handle > 32-bit block address.
2195 if (BrOffset >= 0) {
2196 BuildMI(MBB, I, DL, get(AMDGPU::S_ADD_U32))
2197 .addReg(PCReg, RegState::Define, AMDGPU::sub0)
2198 .addReg(PCReg, 0, AMDGPU::sub0)
2199 .addMBB(&DestBB, MO_LONG_BRANCH_FORWARD);
2200 BuildMI(MBB, I, DL, get(AMDGPU::S_ADDC_U32))
2201 .addReg(PCReg, RegState::Define, AMDGPU::sub1)
2202 .addReg(PCReg, 0, AMDGPU::sub1)
2203 .addImm(0);
2204 } else {
2205 // Backwards branch.
2206 BuildMI(MBB, I, DL, get(AMDGPU::S_SUB_U32))
2207 .addReg(PCReg, RegState::Define, AMDGPU::sub0)
2208 .addReg(PCReg, 0, AMDGPU::sub0)
2209 .addMBB(&DestBB, MO_LONG_BRANCH_BACKWARD);
2210 BuildMI(MBB, I, DL, get(AMDGPU::S_SUBB_U32))
2211 .addReg(PCReg, RegState::Define, AMDGPU::sub1)
2212 .addReg(PCReg, 0, AMDGPU::sub1)
2213 .addImm(0);
2214 }
2215
2216 // Insert the indirect branch after the other terminator.
2217 BuildMI(&MBB, DL, get(AMDGPU::S_SETPC_B64))
2218 .addReg(PCReg);
2219
2220 // FIXME: If spilling is necessary, this will fail because this scavenger has
2221 // no emergency stack slots. It is non-trivial to spill in this situation,
2222 // because the restore code needs to be specially placed after the
2223 // jump. BranchRelaxation then needs to be made aware of the newly inserted
2224 // block.
2225 //
2226 // If a spill is needed for the pc register pair, we need to insert a spill
2227 // restore block right before the destination block, and insert a short branch
2228 // into the old destination block's fallthrough predecessor.
2229 // e.g.:
2230 //
2231 // s_cbranch_scc0 skip_long_branch:
2232 //
2233 // long_branch_bb:
2234 // spill s[8:9]
2235 // s_getpc_b64 s[8:9]
2236 // s_add_u32 s8, s8, restore_bb
2237 // s_addc_u32 s9, s9, 0
2238 // s_setpc_b64 s[8:9]
2239 //
2240 // skip_long_branch:
2241 // foo;
2242 //
2243 // .....
2244 //
2245 // dest_bb_fallthrough_predecessor:
2246 // bar;
2247 // s_branch dest_bb
2248 //
2249 // restore_bb:
2250 // restore s[8:9]
2251 // fallthrough dest_bb
2252 ///
2253 // dest_bb:
2254 // buzz;
2255
2256 RS->enterBasicBlockEnd(MBB);
2257 Register Scav = RS->scavengeRegisterBackwards(
2258 AMDGPU::SReg_64RegClass,
2259 MachineBasicBlock::iterator(GetPC), false, 0);
2260 MRI.replaceRegWith(PCReg, Scav);
2261 MRI.clearVirtRegs();
2262 RS->setRegUsed(Scav);
2263
2264 return 4 + 8 + 4 + 4;
2265}
2266
2267unsigned SIInstrInfo::getBranchOpcode(SIInstrInfo::BranchPredicate Cond) {
2268 switch (Cond) {
2269 case SIInstrInfo::SCC_TRUE:
2270 return AMDGPU::S_CBRANCH_SCC1;
2271 case SIInstrInfo::SCC_FALSE:
2272 return AMDGPU::S_CBRANCH_SCC0;
2273 case SIInstrInfo::VCCNZ:
2274 return AMDGPU::S_CBRANCH_VCCNZ;
2275 case SIInstrInfo::VCCZ:
2276 return AMDGPU::S_CBRANCH_VCCZ;
2277 case SIInstrInfo::EXECNZ:
2278 return AMDGPU::S_CBRANCH_EXECNZ;
2279 case SIInstrInfo::EXECZ:
2280 return AMDGPU::S_CBRANCH_EXECZ;
2281 default:
2282 llvm_unreachable("invalid branch predicate")::llvm::llvm_unreachable_internal("invalid branch predicate",
"/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 2282)
;
2283 }
2284}
2285
2286SIInstrInfo::BranchPredicate SIInstrInfo::getBranchPredicate(unsigned Opcode) {
2287 switch (Opcode) {
2288 case AMDGPU::S_CBRANCH_SCC0:
2289 return SCC_FALSE;
2290 case AMDGPU::S_CBRANCH_SCC1:
2291 return SCC_TRUE;
2292 case AMDGPU::S_CBRANCH_VCCNZ:
2293 return VCCNZ;
2294 case AMDGPU::S_CBRANCH_VCCZ:
2295 return VCCZ;
2296 case AMDGPU::S_CBRANCH_EXECNZ:
2297 return EXECNZ;
2298 case AMDGPU::S_CBRANCH_EXECZ:
2299 return EXECZ;
2300 default:
2301 return INVALID_BR;
2302 }
2303}
2304
2305bool SIInstrInfo::analyzeBranchImpl(MachineBasicBlock &MBB,
2306 MachineBasicBlock::iterator I,
2307 MachineBasicBlock *&TBB,
2308 MachineBasicBlock *&FBB,
2309 SmallVectorImpl<MachineOperand> &Cond,
2310 bool AllowModify) const {
2311 if (I->getOpcode() == AMDGPU::S_BRANCH) {
2312 // Unconditional Branch
2313 TBB = I->getOperand(0).getMBB();
2314 return false;
2315 }
2316
2317 MachineBasicBlock *CondBB = nullptr;
2318
2319 if (I->getOpcode() == AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO) {
2320 CondBB = I->getOperand(1).getMBB();
2321 Cond.push_back(I->getOperand(0));
2322 } else {
2323 BranchPredicate Pred = getBranchPredicate(I->getOpcode());
2324 if (Pred == INVALID_BR)
2325 return true;
2326
2327 CondBB = I->getOperand(0).getMBB();
2328 Cond.push_back(MachineOperand::CreateImm(Pred));
2329 Cond.push_back(I->getOperand(1)); // Save the branch register.
2330 }
2331 ++I;
2332
2333 if (I == MBB.end()) {
2334 // Conditional branch followed by fall-through.
2335 TBB = CondBB;
2336 return false;
2337 }
2338
2339 if (I->getOpcode() == AMDGPU::S_BRANCH) {
2340 TBB = CondBB;
2341 FBB = I->getOperand(0).getMBB();
2342 return false;
2343 }
2344
2345 return true;
2346}
2347
2348bool SIInstrInfo::analyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB,
2349 MachineBasicBlock *&FBB,
2350 SmallVectorImpl<MachineOperand> &Cond,
2351 bool AllowModify) const {
2352 MachineBasicBlock::iterator I = MBB.getFirstTerminator();
2353 auto E = MBB.end();
2354 if (I == E)
2355 return false;
2356
2357 // Skip over the instructions that are artificially terminators for special
2358 // exec management.
2359 while (I != E && !I->isBranch() && !I->isReturn()) {
2360 switch (I->getOpcode()) {
2361 case AMDGPU::S_MOV_B64_term:
2362 case AMDGPU::S_XOR_B64_term:
2363 case AMDGPU::S_OR_B64_term:
2364 case AMDGPU::S_ANDN2_B64_term:
2365 case AMDGPU::S_AND_B64_term:
2366 case AMDGPU::S_MOV_B32_term:
2367 case AMDGPU::S_XOR_B32_term:
2368 case AMDGPU::S_OR_B32_term:
2369 case AMDGPU::S_ANDN2_B32_term:
2370 case AMDGPU::S_AND_B32_term:
2371 break;
2372 case AMDGPU::SI_IF:
2373 case AMDGPU::SI_ELSE:
2374 case AMDGPU::SI_KILL_I1_TERMINATOR:
2375 case AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR:
2376 // FIXME: It's messy that these need to be considered here at all.
2377 return true;
2378 default:
2379 llvm_unreachable("unexpected non-branch terminator inst")::llvm::llvm_unreachable_internal("unexpected non-branch terminator inst"
, "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 2379)
;
2380 }
2381
2382 ++I;
2383 }
2384
2385 if (I == E)
2386 return false;
2387
2388 return analyzeBranchImpl(MBB, I, TBB, FBB, Cond, AllowModify);
2389}
2390
2391unsigned SIInstrInfo::removeBranch(MachineBasicBlock &MBB,
2392 int *BytesRemoved) const {
2393 MachineBasicBlock::iterator I = MBB.getFirstTerminator();
2394
2395 unsigned Count = 0;
2396 unsigned RemovedSize = 0;
2397 while (I != MBB.end()) {
2398 MachineBasicBlock::iterator Next = std::next(I);
2399 RemovedSize += getInstSizeInBytes(*I);
2400 I->eraseFromParent();
2401 ++Count;
2402 I = Next;
2403 }
2404
2405 if (BytesRemoved)
2406 *BytesRemoved = RemovedSize;
2407
2408 return Count;
2409}
2410
2411// Copy the flags onto the implicit condition register operand.
2412static void preserveCondRegFlags(MachineOperand &CondReg,
2413 const MachineOperand &OrigCond) {
2414 CondReg.setIsUndef(OrigCond.isUndef());
2415 CondReg.setIsKill(OrigCond.isKill());
2416}
2417
2418unsigned SIInstrInfo::insertBranch(MachineBasicBlock &MBB,
2419 MachineBasicBlock *TBB,
2420 MachineBasicBlock *FBB,
2421 ArrayRef<MachineOperand> Cond,
2422 const DebugLoc &DL,
2423 int *BytesAdded) const {
2424 if (!FBB && Cond.empty()) {
2425 BuildMI(&MBB, DL, get(AMDGPU::S_BRANCH))
2426 .addMBB(TBB);
2427 if (BytesAdded)
2428 *BytesAdded = ST.hasOffset3fBug() ? 8 : 4;
2429 return 1;
2430 }
2431
2432 if(Cond.size() == 1 && Cond[0].isReg()) {
2433 BuildMI(&MBB, DL, get(AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO))
2434 .add(Cond[0])
2435 .addMBB(TBB);
2436 return 1;
2437 }
2438
2439 assert(TBB && Cond[0].isImm())(static_cast <bool> (TBB && Cond[0].isImm()) ? void
(0) : __assert_fail ("TBB && Cond[0].isImm()", "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 2439, __extension__ __PRETTY_FUNCTION__))
;
2440
2441 unsigned Opcode
2442 = getBranchOpcode(static_cast<BranchPredicate>(Cond[0].getImm()));
2443
2444 if (!FBB) {
2445 Cond[1].isUndef();
2446 MachineInstr *CondBr =
2447 BuildMI(&MBB, DL, get(Opcode))
2448 .addMBB(TBB);
2449
2450 // Copy the flags onto the implicit condition register operand.
2451 preserveCondRegFlags(CondBr->getOperand(1), Cond[1]);
2452 fixImplicitOperands(*CondBr);
2453
2454 if (BytesAdded)
2455 *BytesAdded = ST.hasOffset3fBug() ? 8 : 4;
2456 return 1;
2457 }
2458
2459 assert(TBB && FBB)(static_cast <bool> (TBB && FBB) ? void (0) : __assert_fail
("TBB && FBB", "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 2459, __extension__ __PRETTY_FUNCTION__))
;
2460
2461 MachineInstr *CondBr =
2462 BuildMI(&MBB, DL, get(Opcode))
2463 .addMBB(TBB);
2464 fixImplicitOperands(*CondBr);
2465 BuildMI(&MBB, DL, get(AMDGPU::S_BRANCH))
2466 .addMBB(FBB);
2467
2468 MachineOperand &CondReg = CondBr->getOperand(1);
2469 CondReg.setIsUndef(Cond[1].isUndef());
2470 CondReg.setIsKill(Cond[1].isKill());
2471
2472 if (BytesAdded)
2473 *BytesAdded = ST.hasOffset3fBug() ? 16 : 8;
2474
2475 return 2;
2476}
2477
2478bool SIInstrInfo::reverseBranchCondition(
2479 SmallVectorImpl<MachineOperand> &Cond) const {
2480 if (Cond.size() != 2) {
2481 return true;
2482 }
2483
2484 if (Cond[0].isImm()) {
2485 Cond[0].setImm(-Cond[0].getImm());
2486 return false;
2487 }
2488
2489 return true;
2490}
2491
2492bool SIInstrInfo::canInsertSelect(const MachineBasicBlock &MBB,
2493 ArrayRef<MachineOperand> Cond,
2494 Register DstReg, Register TrueReg,
2495 Register FalseReg, int &CondCycles,
2496 int &TrueCycles, int &FalseCycles) const {
2497 switch (Cond[0].getImm()) {
2498 case VCCNZ:
2499 case VCCZ: {
2500 const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
2501 const TargetRegisterClass *RC = MRI.getRegClass(TrueReg);
2502 if (MRI.getRegClass(FalseReg) != RC)
2503 return false;
2504
2505 int NumInsts = AMDGPU::getRegBitWidth(RC->getID()) / 32;
2506 CondCycles = TrueCycles = FalseCycles = NumInsts; // ???
2507
2508 // Limit to equal cost for branch vs. N v_cndmask_b32s.
2509 return RI.hasVGPRs(RC) && NumInsts <= 6;
2510 }
2511 case SCC_TRUE:
2512 case SCC_FALSE: {
2513 // FIXME: We could insert for VGPRs if we could replace the original compare
2514 // with a vector one.
2515 const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
2516 const TargetRegisterClass *RC = MRI.getRegClass(TrueReg);
2517 if (MRI.getRegClass(FalseReg) != RC)
2518 return false;
2519
2520 int NumInsts = AMDGPU::getRegBitWidth(RC->getID()) / 32;
2521
2522 // Multiples of 8 can do s_cselect_b64
2523 if (NumInsts % 2 == 0)
2524 NumInsts /= 2;
2525
2526 CondCycles = TrueCycles = FalseCycles = NumInsts; // ???
2527 return RI.isSGPRClass(RC);
2528 }
2529 default:
2530 return false;
2531 }
2532}
2533
2534void SIInstrInfo::insertSelect(MachineBasicBlock &MBB,
2535 MachineBasicBlock::iterator I, const DebugLoc &DL,
2536 Register DstReg, ArrayRef<MachineOperand> Cond,
2537 Register TrueReg, Register FalseReg) const {
2538 BranchPredicate Pred = static_cast<BranchPredicate>(Cond[0].getImm());
2539 if (Pred == VCCZ || Pred == SCC_FALSE) {
2540 Pred = static_cast<BranchPredicate>(-Pred);
2541 std::swap(TrueReg, FalseReg);
2542 }
2543
2544 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
2545 const TargetRegisterClass *DstRC = MRI.getRegClass(DstReg);
2546 unsigned DstSize = RI.getRegSizeInBits(*DstRC);
2547
2548 if (DstSize == 32) {
2549 MachineInstr *Select;
2550 if (Pred == SCC_TRUE) {
2551 Select = BuildMI(MBB, I, DL, get(AMDGPU::S_CSELECT_B32), DstReg)
2552 .addReg(TrueReg)
2553 .addReg(FalseReg);
2554 } else {
2555 // Instruction's operands are backwards from what is expected.
2556 Select = BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e32), DstReg)
2557 .addReg(FalseReg)
2558 .addReg(TrueReg);
2559 }
2560
2561 preserveCondRegFlags(Select->getOperand(3), Cond[1]);
2562 return;
2563 }
2564
2565 if (DstSize == 64 && Pred == SCC_TRUE) {
2566 MachineInstr *Select =
2567 BuildMI(MBB, I, DL, get(AMDGPU::S_CSELECT_B64), DstReg)
2568 .addReg(TrueReg)
2569 .addReg(FalseReg);
2570
2571 preserveCondRegFlags(Select->getOperand(3), Cond[1]);
2572 return;
2573 }
2574
2575 static const int16_t Sub0_15[] = {
2576 AMDGPU::sub0, AMDGPU::sub1, AMDGPU::sub2, AMDGPU::sub3,
2577 AMDGPU::sub4, AMDGPU::sub5, AMDGPU::sub6, AMDGPU::sub7,
2578 AMDGPU::sub8, AMDGPU::sub9, AMDGPU::sub10, AMDGPU::sub11,
2579 AMDGPU::sub12, AMDGPU::sub13, AMDGPU::sub14, AMDGPU::sub15,
2580 };
2581
2582 static const int16_t Sub0_15_64[] = {
2583 AMDGPU::sub0_sub1, AMDGPU::sub2_sub3,
2584 AMDGPU::sub4_sub5, AMDGPU::sub6_sub7,
2585 AMDGPU::sub8_sub9, AMDGPU::sub10_sub11,
2586 AMDGPU::sub12_sub13, AMDGPU::sub14_sub15,
2587 };
2588
2589 unsigned SelOp = AMDGPU::V_CNDMASK_B32_e32;
2590 const TargetRegisterClass *EltRC = &AMDGPU::VGPR_32RegClass;
2591 const int16_t *SubIndices = Sub0_15;
2592 int NElts = DstSize / 32;
2593
2594 // 64-bit select is only available for SALU.
2595 // TODO: Split 96-bit into 64-bit and 32-bit, not 3x 32-bit.
2596 if (Pred == SCC_TRUE) {
2597 if (NElts % 2) {
2598 SelOp = AMDGPU::S_CSELECT_B32;
2599 EltRC = &AMDGPU::SGPR_32RegClass;
2600 } else {
2601 SelOp = AMDGPU::S_CSELECT_B64;
2602 EltRC = &AMDGPU::SGPR_64RegClass;
2603 SubIndices = Sub0_15_64;
2604 NElts /= 2;
2605 }
2606 }
2607
2608 MachineInstrBuilder MIB = BuildMI(
2609 MBB, I, DL, get(AMDGPU::REG_SEQUENCE), DstReg);
2610
2611 I = MIB->getIterator();
2612
2613 SmallVector<Register, 8> Regs;
2614 for (int Idx = 0; Idx != NElts; ++Idx) {
2615 Register DstElt = MRI.createVirtualRegister(EltRC);
2616 Regs.push_back(DstElt);
2617
2618 unsigned SubIdx = SubIndices[Idx];
2619
2620 MachineInstr *Select;
2621 if (SelOp == AMDGPU::V_CNDMASK_B32_e32) {
2622 Select =
2623 BuildMI(MBB, I, DL, get(SelOp), DstElt)
2624 .addReg(FalseReg, 0, SubIdx)
2625 .addReg(TrueReg, 0, SubIdx);
2626 } else {
2627 Select =
2628 BuildMI(MBB, I, DL, get(SelOp), DstElt)
2629 .addReg(TrueReg, 0, SubIdx)
2630 .addReg(FalseReg, 0, SubIdx);
2631 }
2632
2633 preserveCondRegFlags(Select->getOperand(3), Cond[1]);
2634 fixImplicitOperands(*Select);
2635
2636 MIB.addReg(DstElt)
2637 .addImm(SubIdx);
2638 }
2639}
2640
2641bool SIInstrInfo::isFoldableCopy(const MachineInstr &MI) const {
2642 switch (MI.getOpcode()) {
2643 case AMDGPU::V_MOV_B32_e32:
2644 case AMDGPU::V_MOV_B32_e64:
2645 case AMDGPU::V_MOV_B64_PSEUDO: {
2646 // If there are additional implicit register operands, this may be used for
2647 // register indexing so the source register operand isn't simply copied.
2648 unsigned NumOps = MI.getDesc().getNumOperands() +
2649 MI.getDesc().getNumImplicitUses();
2650
2651 return MI.getNumOperands() == NumOps;
2652 }
2653 case AMDGPU::S_MOV_B32:
2654 case AMDGPU::S_MOV_B64:
2655 case AMDGPU::COPY:
2656 case AMDGPU::V_ACCVGPR_WRITE_B32_e64:
2657 case AMDGPU::V_ACCVGPR_READ_B32_e64:
2658 case AMDGPU::V_ACCVGPR_MOV_B32:
2659 return true;
2660 default:
2661 return false;
2662 }
2663}
2664
2665unsigned SIInstrInfo::getAddressSpaceForPseudoSourceKind(
2666 unsigned Kind) const {
2667 switch(Kind) {
2668 case PseudoSourceValue::Stack:
2669 case PseudoSourceValue::FixedStack:
2670 return AMDGPUAS::PRIVATE_ADDRESS;
2671 case PseudoSourceValue::ConstantPool:
2672 case PseudoSourceValue::GOT:
2673 case PseudoSourceValue::JumpTable:
2674 case PseudoSourceValue::GlobalValueCallEntry:
2675 case PseudoSourceValue::ExternalSymbolCallEntry:
2676 case PseudoSourceValue::TargetCustom:
2677 return AMDGPUAS::CONSTANT_ADDRESS;
2678 }
2679 return AMDGPUAS::FLAT_ADDRESS;
2680}
2681
2682static void removeModOperands(MachineInstr &MI) {
2683 unsigned Opc = MI.getOpcode();
2684 int Src0ModIdx = AMDGPU::getNamedOperandIdx(Opc,
2685 AMDGPU::OpName::src0_modifiers);
2686 int Src1ModIdx = AMDGPU::getNamedOperandIdx(Opc,
2687 AMDGPU::OpName::src1_modifiers);
2688 int Src2ModIdx = AMDGPU::getNamedOperandIdx(Opc,
2689 AMDGPU::OpName::src2_modifiers);
2690
2691 MI.RemoveOperand(Src2ModIdx);
2692 MI.RemoveOperand(Src1ModIdx);
2693 MI.RemoveOperand(Src0ModIdx);
2694}
2695
2696bool SIInstrInfo::FoldImmediate(MachineInstr &UseMI, MachineInstr &DefMI,
2697 Register Reg, MachineRegisterInfo *MRI) const {
2698 if (!MRI->hasOneNonDBGUse(Reg))
2699 return false;
2700
2701 switch (DefMI.getOpcode()) {
2702 default:
2703 return false;
2704 case AMDGPU::S_MOV_B64:
2705 // TODO: We could fold 64-bit immediates, but this get compilicated
2706 // when there are sub-registers.
2707 return false;
2708
2709 case AMDGPU::V_MOV_B32_e32:
2710 case AMDGPU::S_MOV_B32:
2711 case AMDGPU::V_ACCVGPR_WRITE_B32_e64:
2712 break;
2713 }
2714
2715 const MachineOperand *ImmOp = getNamedOperand(DefMI, AMDGPU::OpName::src0);
2716 assert(ImmOp)(static_cast <bool> (ImmOp) ? void (0) : __assert_fail (
"ImmOp", "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 2716, __extension__ __PRETTY_FUNCTION__))
;
2717 // FIXME: We could handle FrameIndex values here.
2718 if (!ImmOp->isImm())
2719 return false;
2720
2721 unsigned Opc = UseMI.getOpcode();
2722 if (Opc == AMDGPU::COPY) {
2723 Register DstReg = UseMI.getOperand(0).getReg();
2724 bool Is16Bit = getOpSize(UseMI, 0) == 2;
2725 bool isVGPRCopy = RI.isVGPR(*MRI, DstReg);
2726 unsigned NewOpc = isVGPRCopy ? AMDGPU::V_MOV_B32_e32 : AMDGPU::S_MOV_B32;
2727 APInt Imm(32, ImmOp->getImm());
2728
2729 if (UseMI.getOperand(1).getSubReg() == AMDGPU::hi16)
2730 Imm = Imm.ashr(16);
2731
2732 if (RI.isAGPR(*MRI, DstReg)) {
2733 if (!isInlineConstant(Imm))
2734 return false;
2735 NewOpc = AMDGPU::V_ACCVGPR_WRITE_B32_e64;
2736 }
2737
2738 if (Is16Bit) {
2739 if (isVGPRCopy)
2740 return false; // Do not clobber vgpr_hi16
2741
2742 if (DstReg.isVirtual() &&
2743 UseMI.getOperand(0).getSubReg() != AMDGPU::lo16)
2744 return false;
2745
2746 UseMI.getOperand(0).setSubReg(0);
2747 if (DstReg.isPhysical()) {
2748 DstReg = RI.get32BitRegister(DstReg);
2749 UseMI.getOperand(0).setReg(DstReg);
2750 }
2751 assert(UseMI.getOperand(1).getReg().isVirtual())(static_cast <bool> (UseMI.getOperand(1).getReg().isVirtual
()) ? void (0) : __assert_fail ("UseMI.getOperand(1).getReg().isVirtual()"
, "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 2751, __extension__ __PRETTY_FUNCTION__))
;
2752 }
2753
2754 UseMI.setDesc(get(NewOpc));
2755 UseMI.getOperand(1).ChangeToImmediate(Imm.getSExtValue());
2756 UseMI.addImplicitDefUseOperands(*UseMI.getParent()->getParent());
2757 return true;
2758 }
2759
2760 if (Opc == AMDGPU::V_MAD_F32_e64 || Opc == AMDGPU::V_MAC_F32_e64 ||
2761 Opc == AMDGPU::V_MAD_F16_e64 || Opc == AMDGPU::V_MAC_F16_e64 ||
2762 Opc == AMDGPU::V_FMA_F32_e64 || Opc == AMDGPU::V_FMAC_F32_e64 ||
2763 Opc == AMDGPU::V_FMA_F16_e64 || Opc == AMDGPU::V_FMAC_F16_e64) {
2764 // Don't fold if we are using source or output modifiers. The new VOP2
2765 // instructions don't have them.
2766 if (hasAnyModifiersSet(UseMI))
2767 return false;
2768
2769 // If this is a free constant, there's no reason to do this.
2770 // TODO: We could fold this here instead of letting SIFoldOperands do it
2771 // later.
2772 MachineOperand *Src0 = getNamedOperand(UseMI, AMDGPU::OpName::src0);
2773
2774 // Any src operand can be used for the legality check.
2775 if (isInlineConstant(UseMI, *Src0, *ImmOp))
2776 return false;
2777
2778 bool IsF32 = Opc == AMDGPU::V_MAD_F32_e64 || Opc == AMDGPU::V_MAC_F32_e64 ||
2779 Opc == AMDGPU::V_FMA_F32_e64 || Opc == AMDGPU::V_FMAC_F32_e64;
2780 bool IsFMA = Opc == AMDGPU::V_FMA_F32_e64 || Opc == AMDGPU::V_FMAC_F32_e64 ||
2781 Opc == AMDGPU::V_FMA_F16_e64 || Opc == AMDGPU::V_FMAC_F16_e64;
2782 MachineOperand *Src1 = getNamedOperand(UseMI, AMDGPU::OpName::src1);
2783 MachineOperand *Src2 = getNamedOperand(UseMI, AMDGPU::OpName::src2);
2784
2785 // Multiplied part is the constant: Use v_madmk_{f16, f32}.
2786 // We should only expect these to be on src0 due to canonicalizations.
2787 if (Src0->isReg() && Src0->getReg() == Reg) {
2788 if (!Src1->isReg() || RI.isSGPRClass(MRI->getRegClass(Src1->getReg())))
2789 return false;
2790
2791 if (!Src2->isReg() || RI.isSGPRClass(MRI->getRegClass(Src2->getReg())))
2792 return false;
2793
2794 unsigned NewOpc =
2795 IsFMA ? (IsF32 ? AMDGPU::V_FMAMK_F32 : AMDGPU::V_FMAMK_F16)
2796 : (IsF32 ? AMDGPU::V_MADMK_F32 : AMDGPU::V_MADMK_F16);
2797 if (pseudoToMCOpcode(NewOpc) == -1)
2798 return false;
2799
2800 // We need to swap operands 0 and 1 since madmk constant is at operand 1.
2801
2802 const int64_t Imm = ImmOp->getImm();
2803
2804 // FIXME: This would be a lot easier if we could return a new instruction
2805 // instead of having to modify in place.
2806
2807 // Remove these first since they are at the end.
2808 UseMI.RemoveOperand(
2809 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::omod));
2810 UseMI.RemoveOperand(
2811 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::clamp));
2812
2813 Register Src1Reg = Src1->getReg();
2814 unsigned Src1SubReg = Src1->getSubReg();
2815 Src0->setReg(Src1Reg);
2816 Src0->setSubReg(Src1SubReg);
2817 Src0->setIsKill(Src1->isKill());
2818
2819 if (Opc == AMDGPU::V_MAC_F32_e64 ||
2820 Opc == AMDGPU::V_MAC_F16_e64 ||
2821 Opc == AMDGPU::V_FMAC_F32_e64 ||
2822 Opc == AMDGPU::V_FMAC_F16_e64)
2823 UseMI.untieRegOperand(
2824 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2));
2825
2826 Src1->ChangeToImmediate(Imm);
2827
2828 removeModOperands(UseMI);
2829 UseMI.setDesc(get(NewOpc));
2830
2831 bool DeleteDef = MRI->hasOneNonDBGUse(Reg);
2832 if (DeleteDef)
2833 DefMI.eraseFromParent();
2834
2835 return true;
2836 }
2837
2838 // Added part is the constant: Use v_madak_{f16, f32}.
2839 if (Src2->isReg() && Src2->getReg() == Reg) {
2840 // Not allowed to use constant bus for another operand.
2841 // We can however allow an inline immediate as src0.
2842 bool Src0Inlined = false;
2843 if (Src0->isReg()) {
2844 // Try to inline constant if possible.
2845 // If the Def moves immediate and the use is single
2846 // We are saving VGPR here.
2847 MachineInstr *Def = MRI->getUniqueVRegDef(Src0->getReg());
2848 if (Def && Def->isMoveImmediate() &&
2849 isInlineConstant(Def->getOperand(1)) &&
2850 MRI->hasOneUse(Src0->getReg())) {
2851 Src0->ChangeToImmediate(Def->getOperand(1).getImm());
2852 Src0Inlined = true;
2853 } else if ((Src0->getReg().isPhysical() &&
2854 (ST.getConstantBusLimit(Opc) <= 1 &&
2855 RI.isSGPRClass(RI.getPhysRegClass(Src0->getReg())))) ||
2856 (Src0->getReg().isVirtual() &&
2857 (ST.getConstantBusLimit(Opc) <= 1 &&
2858 RI.isSGPRClass(MRI->getRegClass(Src0->getReg())))))
2859 return false;
2860 // VGPR is okay as Src0 - fallthrough
2861 }
2862
2863 if (Src1->isReg() && !Src0Inlined ) {
2864 // We have one slot for inlinable constant so far - try to fill it
2865 MachineInstr *Def = MRI->getUniqueVRegDef(Src1->getReg());
2866 if (Def && Def->isMoveImmediate() &&
2867 isInlineConstant(Def->getOperand(1)) &&
2868 MRI->hasOneUse(Src1->getReg()) &&
2869 commuteInstruction(UseMI)) {
2870 Src0->ChangeToImmediate(Def->getOperand(1).getImm());
2871 } else if ((Src1->getReg().isPhysical() &&
2872 RI.isSGPRClass(RI.getPhysRegClass(Src1->getReg()))) ||
2873 (Src1->getReg().isVirtual() &&
2874 RI.isSGPRClass(MRI->getRegClass(Src1->getReg()))))
2875 return false;
2876 // VGPR is okay as Src1 - fallthrough
2877 }
2878
2879 unsigned NewOpc =
2880 IsFMA ? (IsF32 ? AMDGPU::V_FMAAK_F32 : AMDGPU::V_FMAAK_F16)
2881 : (IsF32 ? AMDGPU::V_MADAK_F32 : AMDGPU::V_MADAK_F16);
2882 if (pseudoToMCOpcode(NewOpc) == -1)
2883 return false;
2884
2885 const int64_t Imm = ImmOp->getImm();
2886
2887 // FIXME: This would be a lot easier if we could return a new instruction
2888 // instead of having to modify in place.
2889
2890 // Remove these first since they are at the end.
2891 UseMI.RemoveOperand(
2892 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::omod));
2893 UseMI.RemoveOperand(
2894 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::clamp));
2895
2896 if (Opc == AMDGPU::V_MAC_F32_e64 ||
2897 Opc == AMDGPU::V_MAC_F16_e64 ||
2898 Opc == AMDGPU::V_FMAC_F32_e64 ||
2899 Opc == AMDGPU::V_FMAC_F16_e64)
2900 UseMI.untieRegOperand(
2901 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2));
2902
2903 // ChangingToImmediate adds Src2 back to the instruction.
2904 Src2->ChangeToImmediate(Imm);
2905
2906 // These come before src2.
2907 removeModOperands(UseMI);
2908 UseMI.setDesc(get(NewOpc));
2909 // It might happen that UseMI was commuted
2910 // and we now have SGPR as SRC1. If so 2 inlined
2911 // constant and SGPR are illegal.
2912 legalizeOperands(UseMI);
2913
2914 bool DeleteDef = MRI->hasOneNonDBGUse(Reg);
2915 if (DeleteDef)
2916 DefMI.eraseFromParent();
2917
2918 return true;
2919 }
2920 }
2921
2922 return false;
2923}
2924
2925static bool
2926memOpsHaveSameBaseOperands(ArrayRef<const MachineOperand *> BaseOps1,
2927 ArrayRef<const MachineOperand *> BaseOps2) {
2928 if (BaseOps1.size() != BaseOps2.size())
2929 return false;
2930 for (size_t I = 0, E = BaseOps1.size(); I < E; ++I) {
2931 if (!BaseOps1[I]->isIdenticalTo(*BaseOps2[I]))
2932 return false;
2933 }
2934 return true;
2935}
2936
2937static bool offsetsDoNotOverlap(int WidthA, int OffsetA,
2938 int WidthB, int OffsetB) {
2939 int LowOffset = OffsetA < OffsetB ? OffsetA : OffsetB;
2940 int HighOffset = OffsetA < OffsetB ? OffsetB : OffsetA;
2941 int LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB;
2942 return LowOffset + LowWidth <= HighOffset;
2943}
2944
2945bool SIInstrInfo::checkInstOffsetsDoNotOverlap(const MachineInstr &MIa,
2946 const MachineInstr &MIb) const {
2947 SmallVector<const MachineOperand *, 4> BaseOps0, BaseOps1;
2948 int64_t Offset0, Offset1;
2949 unsigned Dummy0, Dummy1;
2950 bool Offset0IsScalable, Offset1IsScalable;
2951 if (!getMemOperandsWithOffsetWidth(MIa, BaseOps0, Offset0, Offset0IsScalable,
2952 Dummy0, &RI) ||
2953 !getMemOperandsWithOffsetWidth(MIb, BaseOps1, Offset1, Offset1IsScalable,
2954 Dummy1, &RI))
2955 return false;
2956
2957 if (!memOpsHaveSameBaseOperands(BaseOps0, BaseOps1))
2958 return false;
2959
2960 if (!MIa.hasOneMemOperand() || !MIb.hasOneMemOperand()) {
2961 // FIXME: Handle ds_read2 / ds_write2.
2962 return false;
2963 }
2964 unsigned Width0 = MIa.memoperands().front()->getSize();
2965 unsigned Width1 = MIb.memoperands().front()->getSize();
2966 return offsetsDoNotOverlap(Width0, Offset0, Width1, Offset1);
2967}
2968
2969bool SIInstrInfo::areMemAccessesTriviallyDisjoint(const MachineInstr &MIa,
2970 const MachineInstr &MIb) const {
2971 assert(MIa.mayLoadOrStore() &&(static_cast <bool> (MIa.mayLoadOrStore() && "MIa must load from or modify a memory location"
) ? void (0) : __assert_fail ("MIa.mayLoadOrStore() && \"MIa must load from or modify a memory location\""
, "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 2972, __extension__ __PRETTY_FUNCTION__))
2972 "MIa must load from or modify a memory location")(static_cast <bool> (MIa.mayLoadOrStore() && "MIa must load from or modify a memory location"
) ? void (0) : __assert_fail ("MIa.mayLoadOrStore() && \"MIa must load from or modify a memory location\""
, "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 2972, __extension__ __PRETTY_FUNCTION__))
;
2973 assert(MIb.mayLoadOrStore() &&(static_cast <bool> (MIb.mayLoadOrStore() && "MIb must load from or modify a memory location"
) ? void (0) : __assert_fail ("MIb.mayLoadOrStore() && \"MIb must load from or modify a memory location\""
, "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 2974, __extension__ __PRETTY_FUNCTION__))
2974 "MIb must load from or modify a memory location")(static_cast <bool> (MIb.mayLoadOrStore() && "MIb must load from or modify a memory location"
) ? void (0) : __assert_fail ("MIb.mayLoadOrStore() && \"MIb must load from or modify a memory location\""
, "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 2974, __extension__ __PRETTY_FUNCTION__))
;
2975
2976 if (MIa.hasUnmodeledSideEffects() || MIb.hasUnmodeledSideEffects())
2977 return false;
2978
2979 // XXX - Can we relax this between address spaces?
2980 if (MIa.hasOrderedMemoryRef() || MIb.hasOrderedMemoryRef())
2981 return false;
2982
2983 // TODO: Should we check the address space from the MachineMemOperand? That
2984 // would allow us to distinguish objects we know don't alias based on the
2985 // underlying address space, even if it was lowered to a different one,
2986 // e.g. private accesses lowered to use MUBUF instructions on a scratch
2987 // buffer.
2988 if (isDS(MIa)) {
2989 if (isDS(MIb))
2990 return checkInstOffsetsDoNotOverlap(MIa, MIb);
2991
2992 return !isFLAT(MIb) || isSegmentSpecificFLAT(MIb);
2993 }
2994
2995 if (isMUBUF(MIa) || isMTBUF(MIa)) {
2996 if (isMUBUF(MIb) || isMTBUF(MIb))
2997 return checkInstOffsetsDoNotOverlap(MIa, MIb);
2998
2999 return !isFLAT(MIb) && !isSMRD(MIb);
3000 }
3001
3002 if (isSMRD(MIa)) {
3003 if (isSMRD(MIb))
3004 return checkInstOffsetsDoNotOverlap(MIa, MIb);
3005
3006 return !isFLAT(MIb) && !isMUBUF(MIb) && !isMTBUF(MIb);
3007 }
3008
3009 if (isFLAT(MIa)) {
3010 if (isFLAT(MIb))
3011 return checkInstOffsetsDoNotOverlap(MIa, MIb);
3012
3013 return false;
3014 }
3015
3016 return false;
3017}
3018
3019static int64_t getFoldableImm(const MachineOperand* MO) {
3020 if (!MO->isReg())
3021 return false;
3022 const MachineFunction *MF = MO->getParent()->getParent()->getParent();
3023 const MachineRegisterInfo &MRI = MF->getRegInfo();
3024 auto Def = MRI.getUniqueVRegDef(MO->getReg());
3025 if (Def && Def->getOpcode() == AMDGPU::V_MOV_B32_e32 &&
3026 Def->getOperand(1).isImm())
3027 return Def->getOperand(1).getImm();
3028 return AMDGPU::NoRegister;
3029}
3030
3031static void updateLiveVariables(LiveVariables *LV, MachineInstr &MI,
3032 MachineInstr &NewMI) {
3033 if (LV) {
3034 unsigned NumOps = MI.getNumOperands();
3035 for (unsigned I = 1; I < NumOps; ++I) {
3036 MachineOperand &Op = MI.getOperand(I);
3037 if (Op.isReg() && Op.isKill())
3038 LV->replaceKillInstruction(Op.getReg(), MI, NewMI);
3039 }
3040 }
3041}
3042
3043MachineInstr *SIInstrInfo::convertToThreeAddress(MachineFunction::iterator &MBB,
3044 MachineInstr &MI,
3045 LiveVariables *LV) const {
3046 unsigned Opc = MI.getOpcode();
3047 bool IsF16 = false;
3048 bool IsFMA = Opc == AMDGPU::V_FMAC_F32_e32 || Opc == AMDGPU::V_FMAC_F32_e64 ||
3049 Opc == AMDGPU::V_FMAC_F16_e32 || Opc == AMDGPU::V_FMAC_F16_e64 ||
3050 Opc == AMDGPU::V_FMAC_F64_e32 || Opc == AMDGPU::V_FMAC_F64_e64;
3051 bool IsF64 = Opc == AMDGPU::V_FMAC_F64_e32 || Opc == AMDGPU::V_FMAC_F64_e64;
3052
3053 switch (Opc) {
3054 default:
3055 return nullptr;
3056 case AMDGPU::V_MAC_F16_e64:
3057 case AMDGPU::V_FMAC_F16_e64:
3058 IsF16 = true;
3059 LLVM_FALLTHROUGH[[gnu::fallthrough]];
3060 case AMDGPU::V_MAC_F32_e64:
3061 case AMDGPU::V_FMAC_F32_e64:
3062 case AMDGPU::V_FMAC_F64_e64:
3063 break;
3064 case AMDGPU::V_MAC_F16_e32:
3065 case AMDGPU::V_FMAC_F16_e32:
3066 IsF16 = true;
3067 LLVM_FALLTHROUGH[[gnu::fallthrough]];
3068 case AMDGPU::V_MAC_F32_e32:
3069 case AMDGPU::V_FMAC_F32_e32:
3070 case AMDGPU::V_FMAC_F64_e32: {
3071 int Src0Idx = AMDGPU::getNamedOperandIdx(MI.getOpcode(),
3072 AMDGPU::OpName::src0);
3073 const MachineOperand *Src0 = &MI.getOperand(Src0Idx);
3074 if (!Src0->isReg() && !Src0->isImm())
3075 return nullptr;
3076
3077 if (Src0->isImm() && !isInlineConstant(MI, Src0Idx, *Src0))
3078 return nullptr;
3079
3080 break;
3081 }
3082 }
3083
3084 const MachineOperand *Dst = getNamedOperand(MI, AMDGPU::OpName::vdst);
3085 const MachineOperand *Src0 = getNamedOperand(MI, AMDGPU::OpName::src0);
3086 const MachineOperand *Src0Mods =
3087 getNamedOperand(MI, AMDGPU::OpName::src0_modifiers);
3088 const MachineOperand *Src1 = getNamedOperand(MI, AMDGPU::OpName::src1);
3089 const MachineOperand *Src1Mods =
3090 getNamedOperand(MI, AMDGPU::OpName::src1_modifiers);
3091 const MachineOperand *Src2 = getNamedOperand(MI, AMDGPU::OpName::src2);
3092 const MachineOperand *Clamp = getNamedOperand(MI, AMDGPU::OpName::clamp);
3093 const MachineOperand *Omod = getNamedOperand(MI, AMDGPU::OpName::omod);
3094 MachineInstrBuilder MIB;
3095
3096 if (!Src0Mods && !Src1Mods && !Clamp && !Omod && !IsF64 &&
3097 // If we have an SGPR input, we will violate the constant bus restriction.
3098 (ST.getConstantBusLimit(Opc) > 1 || !Src0->isReg() ||
3099 !RI.isSGPRReg(MBB->getParent()->getRegInfo(), Src0->getReg()))) {
3100 if (auto Imm = getFoldableImm(Src2)) {
3101 unsigned NewOpc =
3102 IsFMA ? (IsF16 ? AMDGPU::V_FMAAK_F16 : AMDGPU::V_FMAAK_F32)
3103 : (IsF16 ? AMDGPU::V_MADAK_F16 : AMDGPU::V_MADAK_F32);
3104 if (pseudoToMCOpcode(NewOpc) != -1) {
3105 MIB = BuildMI(*MBB, MI, MI.getDebugLoc(), get(NewOpc))
3106 .add(*Dst)
3107 .add(*Src0)
3108 .add(*Src1)
3109 .addImm(Imm);
3110 updateLiveVariables(LV, MI, *MIB);
3111 return MIB;
3112 }
3113 }
3114 unsigned NewOpc = IsFMA
3115 ? (IsF16 ? AMDGPU::V_FMAMK_F16 : AMDGPU::V_FMAMK_F32)
3116 : (IsF16 ? AMDGPU::V_MADMK_F16 : AMDGPU::V_MADMK_F32);
3117 if (auto Imm = getFoldableImm(Src1)) {
3118 if (pseudoToMCOpcode(NewOpc) != -1) {
3119 MIB = BuildMI(*MBB, MI, MI.getDebugLoc(), get(NewOpc))
3120 .add(*Dst)
3121 .add(*Src0)
3122 .addImm(Imm)
3123 .add(*Src2);
3124 updateLiveVariables(LV, MI, *MIB);
3125 return MIB;
3126 }
3127 }
3128 if (auto Imm = getFoldableImm(Src0)) {
3129 if (pseudoToMCOpcode(NewOpc) != -1 &&
3130 isOperandLegal(
3131 MI, AMDGPU::getNamedOperandIdx(NewOpc, AMDGPU::OpName::src0),
3132 Src1)) {
3133 MIB = BuildMI(*MBB, MI, MI.getDebugLoc(), get(NewOpc))
3134 .add(*Dst)
3135 .add(*Src1)
3136 .addImm(Imm)
3137 .add(*Src2);
3138 updateLiveVariables(LV, MI, *MIB);
3139 return MIB;
3140 }
3141 }
3142 }
3143
3144 unsigned NewOpc = IsFMA ? (IsF16 ? AMDGPU::V_FMA_F16_e64
3145 : IsF64 ? AMDGPU::V_FMA_F64_e64
3146 : AMDGPU::V_FMA_F32_e64)
3147 : (IsF16 ? AMDGPU::V_MAD_F16_e64 : AMDGPU::V_MAD_F32_e64);
3148 if (pseudoToMCOpcode(NewOpc) == -1)
3149 return nullptr;
3150
3151 MIB = BuildMI(*MBB, MI, MI.getDebugLoc(), get(NewOpc))
3152 .add(*Dst)
3153 .addImm(Src0Mods ? Src0Mods->getImm() : 0)
3154 .add(*Src0)
3155 .addImm(Src1Mods ? Src1Mods->getImm() : 0)
3156 .add(*Src1)
3157 .addImm(0) // Src mods
3158 .add(*Src2)
3159 .addImm(Clamp ? Clamp->getImm() : 0)
3160 .addImm(Omod ? Omod->getImm() : 0);
3161 updateLiveVariables(LV, MI, *MIB);
3162 return MIB;
3163}
3164
3165// It's not generally safe to move VALU instructions across these since it will
3166// start using the register as a base index rather than directly.
3167// XXX - Why isn't hasSideEffects sufficient for these?
3168static bool changesVGPRIndexingMode(const MachineInstr &MI) {
3169 switch (MI.getOpcode()) {
3170 case AMDGPU::S_SET_GPR_IDX_ON:
3171 case AMDGPU::S_SET_GPR_IDX_MODE:
3172 case AMDGPU::S_SET_GPR_IDX_OFF:
3173 return true;
3174 default:
3175 return false;
3176 }
3177}
3178
3179bool SIInstrInfo::isSchedulingBoundary(const MachineInstr &MI,
3180 const MachineBasicBlock *MBB,
3181 const MachineFunction &MF) const {
3182 // Skipping the check for SP writes in the base implementation. The reason it
3183 // was added was apparently due to compile time concerns.
3184 //
3185 // TODO: Do we really want this barrier? It triggers unnecessary hazard nops
3186 // but is probably avoidable.
3187
3188 // Copied from base implementation.
3189 // Terminators and labels can't be scheduled around.
3190 if (MI.isTerminator() || MI.isPosition())
3191 return true;
3192
3193 // INLINEASM_BR can jump to another block
3194 if (MI.getOpcode() == TargetOpcode::INLINEASM_BR)
3195 return true;
3196
3197 // Target-independent instructions do not have an implicit-use of EXEC, even
3198 // when they operate on VGPRs. Treating EXEC modifications as scheduling
3199 // boundaries prevents incorrect movements of such instructions.
3200 return MI.modifiesRegister(AMDGPU::EXEC, &RI) ||
3201 MI.getOpcode() == AMDGPU::S_SETREG_IMM32_B32 ||
3202 MI.getOpcode() == AMDGPU::S_SETREG_B32 ||
3203 changesVGPRIndexingMode(MI);
3204}
3205
3206bool SIInstrInfo::isAlwaysGDS(uint16_t Opcode) const {
3207 return Opcode == AMDGPU::DS_ORDERED_COUNT ||
3208 Opcode == AMDGPU::DS_GWS_INIT ||
3209 Opcode == AMDGPU::DS_GWS_SEMA_V ||
3210 Opcode == AMDGPU::DS_GWS_SEMA_BR ||
3211 Opcode == AMDGPU::DS_GWS_SEMA_P ||
3212 Opcode == AMDGPU::DS_GWS_SEMA_RELEASE_ALL ||
3213 Opcode == AMDGPU::DS_GWS_BARRIER;
3214}
3215
3216bool SIInstrInfo::modifiesModeRegister(const MachineInstr &MI) {
3217 // Skip the full operand and register alias search modifiesRegister
3218 // does. There's only a handful of instructions that touch this, it's only an
3219 // implicit def, and doesn't alias any other registers.
3220 if (const MCPhysReg *ImpDef = MI.getDesc().getImplicitDefs()) {
3221 for (; ImpDef && *ImpDef; ++ImpDef) {
3222 if (*ImpDef == AMDGPU::MODE)
3223 return true;
3224 }
3225 }
3226
3227 return false;
3228}
3229
3230bool SIInstrInfo::hasUnwantedEffectsWhenEXECEmpty(const MachineInstr &MI) const {
3231 unsigned Opcode = MI.getOpcode();
3232
3233 if (MI.mayStore() && isSMRD(MI))
3234 return true; // scalar store or atomic
3235
3236 // This will terminate the function when other lanes may need to continue.
3237 if (MI.isReturn())
3238 return true;
3239
3240 // These instructions cause shader I/O that may cause hardware lockups
3241 // when executed with an empty EXEC mask.
3242 //
3243 // Note: exp with VM = DONE = 0 is automatically skipped by hardware when
3244 // EXEC = 0, but checking for that case here seems not worth it
3245 // given the typical code patterns.
3246 if (Opcode == AMDGPU::S_SENDMSG || Opcode == AMDGPU::S_SENDMSGHALT ||
3247 isEXP(Opcode) ||
3248 Opcode == AMDGPU::DS_ORDERED_COUNT || Opcode == AMDGPU::S_TRAP ||
3249 Opcode == AMDGPU::DS_GWS_INIT || Opcode == AMDGPU::DS_GWS_BARRIER)
3250 return true;
3251
3252 if (MI.isCall() || MI.isInlineAsm())
3253 return true; // conservative assumption
3254
3255 // A mode change is a scalar operation that influences vector instructions.
3256 if (modifiesModeRegister(MI))
3257 return true;
3258
3259 // These are like SALU instructions in terms of effects, so it's questionable
3260 // whether we should return true for those.
3261 //
3262 // However, executing them with EXEC = 0 causes them to operate on undefined
3263 // data, which we avoid by returning true here.
3264 if (Opcode == AMDGPU::V_READFIRSTLANE_B32 ||
3265 Opcode == AMDGPU::V_READLANE_B32 || Opcode == AMDGPU::V_WRITELANE_B32)
3266 return true;
3267
3268 return false;
3269}
3270
3271bool SIInstrInfo::mayReadEXEC(const MachineRegisterInfo &MRI,
3272 const MachineInstr &MI) const {
3273 if (MI.isMetaInstruction())
3274 return false;
3275
3276 // This won't read exec if this is an SGPR->SGPR copy.
3277 if (MI.isCopyLike()) {
3278 if (!RI.isSGPRReg(MRI, MI.getOperand(0).getReg()))
3279 return true;
3280
3281 // Make sure this isn't copying exec as a normal operand
3282 return MI.readsRegister(AMDGPU::EXEC, &RI);
3283 }
3284
3285 // Make a conservative assumption about the callee.
3286 if (MI.isCall())
3287 return true;
3288
3289 // Be conservative with any unhandled generic opcodes.
3290 if (!isTargetSpecificOpcode(MI.getOpcode()))
3291 return true;
3292
3293 return !isSALU(MI) || MI.readsRegister(AMDGPU::EXEC, &RI);
3294}
3295
3296bool SIInstrInfo::isInlineConstant(const APInt &Imm) const {
3297 switch (Imm.getBitWidth()) {
3298 case 1: // This likely will be a condition code mask.
3299 return true;
3300
3301 case 32:
3302 return AMDGPU::isInlinableLiteral32(Imm.getSExtValue(),
3303 ST.hasInv2PiInlineImm());
3304 case 64:
3305 return AMDGPU::isInlinableLiteral64(Imm.getSExtValue(),
3306 ST.hasInv2PiInlineImm());
3307 case 16:
3308 return ST.has16BitInsts() &&
3309 AMDGPU::isInlinableLiteral16(Imm.getSExtValue(),
3310 ST.hasInv2PiInlineImm());
3311 default:
3312 llvm_unreachable("invalid bitwidth")::llvm::llvm_unreachable_internal("invalid bitwidth", "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 3312)
;
3313 }
3314}
3315
3316bool SIInstrInfo::isInlineConstant(const MachineOperand &MO,
3317 uint8_t OperandType) const {
3318 if (!MO.isImm() ||
3319 OperandType < AMDGPU::OPERAND_SRC_FIRST ||
3320 OperandType > AMDGPU::OPERAND_SRC_LAST)
3321 return false;
3322
3323 // MachineOperand provides no way to tell the true operand size, since it only
3324 // records a 64-bit value. We need to know the size to determine if a 32-bit
3325 // floating point immediate bit pattern is legal for an integer immediate. It
3326 // would be for any 32-bit integer operand, but would not be for a 64-bit one.
3327
3328 int64_t Imm = MO.getImm();
3329 switch (OperandType) {
3330 case AMDGPU::OPERAND_REG_IMM_INT32:
3331 case AMDGPU::OPERAND_REG_IMM_FP32:
3332 case AMDGPU::OPERAND_REG_INLINE_C_INT32:
3333 case AMDGPU::OPERAND_REG_INLINE_C_FP32:
3334 case AMDGPU::OPERAND_REG_IMM_V2FP32:
3335 case AMDGPU::OPERAND_REG_INLINE_C_V2FP32:
3336 case AMDGPU::OPERAND_REG_IMM_V2INT32:
3337 case AMDGPU::OPERAND_REG_INLINE_C_V2INT32:
3338 case AMDGPU::OPERAND_REG_INLINE_AC_INT32:
3339 case AMDGPU::OPERAND_REG_INLINE_AC_FP32: {
3340 int32_t Trunc = static_cast<int32_t>(Imm);
3341 return AMDGPU::isInlinableLiteral32(Trunc, ST.hasInv2PiInlineImm());
3342 }
3343 case AMDGPU::OPERAND_REG_IMM_INT64:
3344 case AMDGPU::OPERAND_REG_IMM_FP64:
3345 case AMDGPU::OPERAND_REG_INLINE_C_INT64:
3346 case AMDGPU::OPERAND_REG_INLINE_C_FP64:
3347 case AMDGPU::OPERAND_REG_INLINE_AC_FP64:
3348 return AMDGPU::isInlinableLiteral64(MO.getImm(),
3349 ST.hasInv2PiInlineImm());
3350 case AMDGPU::OPERAND_REG_IMM_INT16:
3351 case AMDGPU::OPERAND_REG_INLINE_C_INT16:
3352 case AMDGPU::OPERAND_REG_INLINE_AC_INT16:
3353 // We would expect inline immediates to not be concerned with an integer/fp
3354 // distinction. However, in the case of 16-bit integer operations, the
3355 // "floating point" values appear to not work. It seems read the low 16-bits
3356 // of 32-bit immediates, which happens to always work for the integer
3357 // values.
3358 //
3359 // See llvm bugzilla 46302.
3360 //
3361 // TODO: Theoretically we could use op-sel to use the high bits of the
3362 // 32-bit FP values.
3363 return AMDGPU::isInlinableIntLiteral(Imm);
3364 case AMDGPU::OPERAND_REG_IMM_V2INT16:
3365 case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
3366 case AMDGPU::OPERAND_REG_INLINE_AC_V2INT16:
3367 // This suffers the same problem as the scalar 16-bit cases.
3368 return AMDGPU::isInlinableIntLiteralV216(Imm);
3369 case AMDGPU::OPERAND_REG_IMM_FP16:
3370 case AMDGPU::OPERAND_REG_INLINE_C_FP16:
3371 case AMDGPU::OPERAND_REG_INLINE_AC_FP16: {
3372 if (isInt<16>(Imm) || isUInt<16>(Imm)) {
3373 // A few special case instructions have 16-bit operands on subtargets
3374 // where 16-bit instructions are not legal.
3375 // TODO: Do the 32-bit immediates work? We shouldn't really need to handle
3376 // constants in these cases
3377 int16_t Trunc = static_cast<int16_t>(Imm);
3378 return ST.has16BitInsts() &&
3379 AMDGPU::isInlinableLiteral16(Trunc, ST.hasInv2PiInlineImm());
3380 }
3381
3382 return false;
3383 }
3384 case AMDGPU::OPERAND_REG_IMM_V2FP16:
3385 case AMDGPU::OPERAND_REG_INLINE_C_V2FP16:
3386 case AMDGPU::OPERAND_REG_INLINE_AC_V2FP16: {
3387 uint32_t Trunc = static_cast<uint32_t>(Imm);
3388 return AMDGPU::isInlinableLiteralV216(Trunc, ST.hasInv2PiInlineImm());
3389 }
3390 default:
3391 llvm_unreachable("invalid bitwidth")::llvm::llvm_unreachable_internal("invalid bitwidth", "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 3391)
;
3392 }
3393}
3394
3395bool SIInstrInfo::isLiteralConstantLike(const MachineOperand &MO,
3396 const MCOperandInfo &OpInfo) const {
3397 switch (MO.getType()) {
3398 case MachineOperand::MO_Register:
3399 return false;
3400 case MachineOperand::MO_Immediate:
3401 return !isInlineConstant(MO, OpInfo);
3402 case MachineOperand::MO_FrameIndex:
3403 case MachineOperand::MO_MachineBasicBlock:
3404 case MachineOperand::MO_ExternalSymbol:
3405 case MachineOperand::MO_GlobalAddress:
3406 case MachineOperand::MO_MCSymbol:
3407 return true;
3408 default:
3409 llvm_unreachable("unexpected operand type")::llvm::llvm_unreachable_internal("unexpected operand type", "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 3409)
;
3410 }
3411}
3412
3413static bool compareMachineOp(const MachineOperand &Op0,
3414 const MachineOperand &Op1) {
3415 if (Op0.getType() != Op1.getType())
3416 return false;
3417
3418 switch (Op0.getType()) {
3419 case MachineOperand::MO_Register:
3420 return Op0.getReg() == Op1.getReg();
3421 case MachineOperand::MO_Immediate:
3422 return Op0.getImm() == Op1.getImm();
3423 default:
3424 llvm_unreachable("Didn't expect to be comparing these operand types")::llvm::llvm_unreachable_internal("Didn't expect to be comparing these operand types"
, "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 3424)
;
3425 }
3426}
3427
3428bool SIInstrInfo::isImmOperandLegal(const MachineInstr &MI, unsigned OpNo,
3429 const MachineOperand &MO) const {
3430 const MCInstrDesc &InstDesc = MI.getDesc();
3431 const MCOperandInfo &OpInfo = InstDesc.OpInfo[OpNo];
3432
3433 assert(MO.isImm() || MO.isTargetIndex() || MO.isFI() || MO.isGlobal())(static_cast <bool> (MO.isImm() || MO.isTargetIndex() ||
MO.isFI() || MO.isGlobal()) ? void (0) : __assert_fail ("MO.isImm() || MO.isTargetIndex() || MO.isFI() || MO.isGlobal()"
, "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 3433, __extension__ __PRETTY_FUNCTION__))
;
3434
3435 if (OpInfo.OperandType == MCOI::OPERAND_IMMEDIATE)
3436 return true;
3437
3438 if (OpInfo.RegClass < 0)
3439 return false;
3440
3441 if (MO.isImm() && isInlineConstant(MO, OpInfo)) {
3442 if (isMAI(MI) && ST.hasMFMAInlineLiteralBug() &&
3443 OpNo ==(unsigned)AMDGPU::getNamedOperandIdx(MI.getOpcode(),
3444 AMDGPU::OpName::src2))
3445 return false;
3446 return RI.opCanUseInlineConstant(OpInfo.OperandType);
3447 }
3448
3449 if (!RI.opCanUseLiteralConstant(OpInfo.OperandType))
3450 return false;
3451
3452 if (!isVOP3(MI) || !AMDGPU::isSISrcOperand(InstDesc, OpNo))
3453 return true;
3454
3455 return ST.hasVOP3Literal();
3456}
3457
3458bool SIInstrInfo::hasVALU32BitEncoding(unsigned Opcode) const {
3459 // GFX90A does not have V_MUL_LEGACY_F32_e32.
3460 if (Opcode == AMDGPU::V_MUL_LEGACY_F32_e64 && ST.hasGFX90AInsts())
3461 return false;
3462
3463 int Op32 = AMDGPU::getVOPe32(Opcode);
3464 if (Op32 == -1)
3465 return false;
3466
3467 return pseudoToMCOpcode(Op32) != -1;
3468}
3469
3470bool SIInstrInfo::hasModifiers(unsigned Opcode) const {
3471 // The src0_modifier operand is present on all instructions
3472 // that have modifiers.
3473
3474 return AMDGPU::getNamedOperandIdx(Opcode,
3475 AMDGPU::OpName::src0_modifiers) != -1;
3476}
3477
3478bool SIInstrInfo::hasModifiersSet(const MachineInstr &MI,
3479 unsigned OpName) const {
3480 const MachineOperand *Mods = getNamedOperand(MI, OpName);
3481 return Mods && Mods->getImm();
3482}
3483
3484bool SIInstrInfo::hasAnyModifiersSet(const MachineInstr &MI) const {
3485 return hasModifiersSet(MI, AMDGPU::OpName::src0_modifiers) ||
3486 hasModifiersSet(MI, AMDGPU::OpName::src1_modifiers) ||
3487 hasModifiersSet(MI, AMDGPU::OpName::src2_modifiers) ||
3488 hasModifiersSet(MI, AMDGPU::OpName::clamp) ||
3489 hasModifiersSet(MI, AMDGPU::OpName::omod);
3490}
3491
3492bool SIInstrInfo::canShrink(const MachineInstr &MI,
3493 const MachineRegisterInfo &MRI) const {
3494 const MachineOperand *Src2 = getNamedOperand(MI, AMDGPU::OpName::src2);
3495 // Can't shrink instruction with three operands.
3496 // FIXME: v_cndmask_b32 has 3 operands and is shrinkable, but we need to add
3497 // a special case for it. It can only be shrunk if the third operand
3498 // is vcc, and src0_modifiers and src1_modifiers are not set.
3499 // We should handle this the same way we handle vopc, by addding
3500 // a register allocation hint pre-regalloc and then do the shrinking
3501 // post-regalloc.
3502 if (Src2) {
3503 switch (MI.getOpcode()) {
3504 default: return false;
3505
3506 case AMDGPU::V_ADDC_U32_e64:
3507 case AMDGPU::V_SUBB_U32_e64:
3508 case AMDGPU::V_SUBBREV_U32_e64: {
3509 const MachineOperand *Src1
3510 = getNamedOperand(MI, AMDGPU::OpName::src1);
3511 if (!Src1->isReg() || !RI.isVGPR(MRI, Src1->getReg()))
3512 return false;
3513 // Additional verification is needed for sdst/src2.
3514 return true;
3515 }
3516 case AMDGPU::V_MAC_F32_e64:
3517 case AMDGPU::V_MAC_F16_e64:
3518 case AMDGPU::V_FMAC_F32_e64:
3519 case AMDGPU::V_FMAC_F16_e64:
3520 case AMDGPU::V_FMAC_F64_e64:
3521 if (!Src2->isReg() || !RI.isVGPR(MRI, Src2->getReg()) ||
3522 hasModifiersSet(MI, AMDGPU::OpName::src2_modifiers))
3523 return false;
3524 break;
3525
3526 case AMDGPU::V_CNDMASK_B32_e64:
3527 break;
3528 }
3529 }
3530
3531 const MachineOperand *Src1 = getNamedOperand(MI, AMDGPU::OpName::src1);
3532 if (Src1 && (!Src1->isReg() || !RI.isVGPR(MRI, Src1->getReg()) ||
3533 hasModifiersSet(MI, AMDGPU::OpName::src1_modifiers)))
3534 return false;
3535
3536 // We don't need to check src0, all input types are legal, so just make sure
3537 // src0 isn't using any modifiers.
3538 if (hasModifiersSet(MI, AMDGPU::OpName::src0_modifiers))
3539 return false;
3540
3541 // Can it be shrunk to a valid 32 bit opcode?
3542 if (!hasVALU32BitEncoding(MI.getOpcode()))
3543 return false;
3544
3545 // Check output modifiers
3546 return !hasModifiersSet(MI, AMDGPU::OpName::omod) &&
3547 !hasModifiersSet(MI, AMDGPU::OpName::clamp);
3548}
3549
3550// Set VCC operand with all flags from \p Orig, except for setting it as
3551// implicit.
3552static void copyFlagsToImplicitVCC(MachineInstr &MI,
3553 const MachineOperand &Orig) {
3554
3555 for (MachineOperand &Use : MI.implicit_operands()) {
3556 if (Use.isUse() &&
3557 (Use.getReg() == AMDGPU::VCC || Use.getReg() == AMDGPU::VCC_LO)) {
3558 Use.setIsUndef(Orig.isUndef());
3559 Use.setIsKill(Orig.isKill());
3560 return;
3561 }
3562 }
3563}
3564
3565MachineInstr *SIInstrInfo::buildShrunkInst(MachineInstr &MI,
3566 unsigned Op32) const {
3567 MachineBasicBlock *MBB = MI.getParent();;
3568 MachineInstrBuilder Inst32 =
3569 BuildMI(*MBB, MI, MI.getDebugLoc(), get(Op32))
3570 .setMIFlags(MI.getFlags());
3571
3572 // Add the dst operand if the 32-bit encoding also has an explicit $vdst.
3573 // For VOPC instructions, this is replaced by an implicit def of vcc.
3574 int Op32DstIdx = AMDGPU::getNamedOperandIdx(Op32, AMDGPU::OpName::vdst);
3575 if (Op32DstIdx != -1) {
3576 // dst
3577 Inst32.add(MI.getOperand(0));
3578 } else {
3579 assert(((MI.getOperand(0).getReg() == AMDGPU::VCC) ||(static_cast <bool> (((MI.getOperand(0).getReg() == AMDGPU
::VCC) || (MI.getOperand(0).getReg() == AMDGPU::VCC_LO)) &&
"Unexpected case") ? void (0) : __assert_fail ("((MI.getOperand(0).getReg() == AMDGPU::VCC) || (MI.getOperand(0).getReg() == AMDGPU::VCC_LO)) && \"Unexpected case\""
, "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 3581, __extension__ __PRETTY_FUNCTION__))
3580 (MI.getOperand(0).getReg() == AMDGPU::VCC_LO)) &&(static_cast <bool> (((MI.getOperand(0).getReg() == AMDGPU
::VCC) || (MI.getOperand(0).getReg() == AMDGPU::VCC_LO)) &&
"Unexpected case") ? void (0) : __assert_fail ("((MI.getOperand(0).getReg() == AMDGPU::VCC) || (MI.getOperand(0).getReg() == AMDGPU::VCC_LO)) && \"Unexpected case\""
, "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 3581, __extension__ __PRETTY_FUNCTION__))
3581 "Unexpected case")(static_cast <bool> (((MI.getOperand(0).getReg() == AMDGPU
::VCC) || (MI.getOperand(0).getReg() == AMDGPU::VCC_LO)) &&
"Unexpected case") ? void (0) : __assert_fail ("((MI.getOperand(0).getReg() == AMDGPU::VCC) || (MI.getOperand(0).getReg() == AMDGPU::VCC_LO)) && \"Unexpected case\""
, "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 3581, __extension__ __PRETTY_FUNCTION__))
;
3582 }
3583
3584 Inst32.add(*getNamedOperand(MI, AMDGPU::OpName::src0));
3585
3586 const MachineOperand *Src1 = getNamedOperand(MI, AMDGPU::OpName::src1);
3587 if (Src1)
3588 Inst32.add(*Src1);
3589
3590 const MachineOperand *Src2 = getNamedOperand(MI, AMDGPU::OpName::src2);
3591
3592 if (Src2) {
3593 int Op32Src2Idx = AMDGPU::getNamedOperandIdx(Op32, AMDGPU::OpName::src2);
3594 if (Op32Src2Idx != -1) {
3595 Inst32.add(*Src2);
3596 } else {
3597 // In the case of V_CNDMASK_B32_e32, the explicit operand src2 is
3598 // replaced with an implicit read of vcc or vcc_lo. The implicit read
3599 // of vcc was already added during the initial BuildMI, but we
3600 // 1) may need to change vcc to vcc_lo to preserve the original register
3601 // 2) have to preserve the original flags.
3602 fixImplicitOperands(*Inst32);
3603 copyFlagsToImplicitVCC(*Inst32, *Src2);
3604 }
3605 }
3606
3607 return Inst32;
3608}
3609
3610bool SIInstrInfo::usesConstantBus(const MachineRegisterInfo &MRI,
3611 const MachineOperand &MO,
3612 const MCOperandInfo &OpInfo) const {
3613 // Literal constants use the constant bus.
3614 //if (isLiteralConstantLike(MO, OpInfo))
3615 // return true;
3616 if (MO.isImm())
3617 return !isInlineConstant(MO, OpInfo);
3618
3619 if (!MO.isReg())
3620 return true; // Misc other operands like FrameIndex
3621
3622 if (!MO.isUse())
3623 return false;
3624
3625 if (MO.getReg().isVirtual())
3626 return RI.isSGPRClass(MRI.getRegClass(MO.getReg()));
3627
3628 // Null is free
3629 if (MO.getReg() == AMDGPU::SGPR_NULL)
3630 return false;
3631
3632 // SGPRs use the constant bus
3633 if (MO.isImplicit()) {
3634 return MO.getReg() == AMDGPU::M0 ||
3635 MO.getReg() == AMDGPU::VCC ||
3636 MO.getReg() == AMDGPU::VCC_LO;
3637 } else {
3638 return AMDGPU::SReg_32RegClass.contains(MO.getReg()) ||
3639 AMDGPU::SReg_64RegClass.contains(MO.getReg());
3640 }
3641}
3642
3643static Register findImplicitSGPRRead(const MachineInstr &MI) {
3644 for (const MachineOperand &MO : MI.implicit_operands()) {
3645 // We only care about reads.
3646 if (MO.isDef())
3647 continue;
3648
3649 switch (MO.getReg()) {
3650 case AMDGPU::VCC:
3651 case AMDGPU::VCC_LO:
3652 case AMDGPU::VCC_HI:
3653 case AMDGPU::M0:
3654 case AMDGPU::FLAT_SCR:
3655 return MO.getReg();
3656
3657 default:
3658 break;
3659 }
3660 }
3661
3662 return AMDGPU::NoRegister;
3663}
3664
3665static bool shouldReadExec(const MachineInstr &MI) {
3666 if (SIInstrInfo::isVALU(MI)) {
3667 switch (MI.getOpcode()) {
3668 case AMDGPU::V_READLANE_B32:
3669 case AMDGPU::V_WRITELANE_B32:
3670 return false;
3671 }
3672
3673 return true;
3674 }
3675
3676 if (MI.isPreISelOpcode() ||
3677 SIInstrInfo::isGenericOpcode(MI.getOpcode()) ||
3678 SIInstrInfo::isSALU(MI) ||
3679 SIInstrInfo::isSMRD(MI))
3680 return false;
3681
3682 return true;
3683}
3684
3685static bool isSubRegOf(const SIRegisterInfo &TRI,
3686 const MachineOperand &SuperVec,
3687 const MachineOperand &SubReg) {
3688 if (SubReg.getReg().isPhysical())
3689 return TRI.isSubRegister(SuperVec.getReg(), SubReg.getReg());
3690
3691 return SubReg.getSubReg() != AMDGPU::NoSubRegister &&
3692 SubReg.getReg() == SuperVec.getReg();
3693}
3694
3695bool SIInstrInfo::verifyInstruction(const MachineInstr &MI,
3696 StringRef &ErrInfo) const {
3697 uint16_t Opcode = MI.getOpcode();
3698 if (SIInstrInfo::isGenericOpcode(MI.getOpcode()))
3699 return true;
3700
3701 const MachineFunction *MF = MI.getParent()->getParent();
3702 const MachineRegisterInfo &MRI = MF->getRegInfo();
3703
3704 int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0);
3705 int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src1);
3706 int Src2Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src2);
3707
3708 // Make sure the number of operands is correct.
3709 const MCInstrDesc &Desc = get(Opcode);
3710 if (!Desc.isVariadic() &&
3711 Desc.getNumOperands() != MI.getNumExplicitOperands()) {
3712 ErrInfo = "Instruction has wrong number of operands.";
3713 return false;
3714 }
3715
3716 if (MI.isInlineAsm()) {
3717 // Verify register classes for inlineasm constraints.
3718 for (unsigned I = InlineAsm::MIOp_FirstOperand, E = MI.getNumOperands();
3719 I != E; ++I) {
3720 const TargetRegisterClass *RC = MI.getRegClassConstraint(I, this, &RI);
3721 if (!RC)
3722 continue;
3723
3724 const MachineOperand &Op = MI.getOperand(I);
3725 if (!Op.isReg())
3726 continue;
3727
3728 Register Reg = Op.getReg();
3729 if (!Reg.isVirtual() && !RC->contains(Reg)) {
3730 ErrInfo = "inlineasm operand has incorrect register class.";
3731 return false;
3732 }
3733 }
3734
3735 return true;
3736 }
3737
3738 if (isMIMG(MI) && MI.memoperands_empty() && MI.mayLoadOrStore()) {
3739 ErrInfo = "missing memory operand from MIMG instruction.";
3740 return false;
3741 }
3742
3743 // Make sure the register classes are correct.
3744 for (int i = 0, e = Desc.getNumOperands(); i != e; ++i) {
3745 const MachineOperand &MO = MI.getOperand(i);
3746 if (MO.isFPImm()) {
3747 ErrInfo = "FPImm Machine Operands are not supported. ISel should bitcast "
3748 "all fp values to integers.";
3749 return false;
3750 }
3751
3752 int RegClass = Desc.OpInfo[i].RegClass;
3753
3754 switch (Desc.OpInfo[i].OperandType) {
3755 case MCOI::OPERAND_REGISTER:
3756 if (MI.getOperand(i).isImm() || MI.getOperand(i).isGlobal()) {
3757 ErrInfo = "Illegal immediate value for operand.";
3758 return false;
3759 }
3760 break;
3761 case AMDGPU::OPERAND_REG_IMM_INT32:
3762 case AMDGPU::OPERAND_REG_IMM_FP32:
3763 break;
3764 case AMDGPU::OPERAND_REG_INLINE_C_INT32:
3765 case AMDGPU::OPERAND_REG_INLINE_C_FP32:
3766 case AMDGPU::OPERAND_REG_INLINE_C_INT64:
3767 case AMDGPU::OPERAND_REG_INLINE_C_FP64:
3768 case AMDGPU::OPERAND_REG_INLINE_C_INT16:
3769 case AMDGPU::OPERAND_REG_INLINE_C_FP16:
3770 case AMDGPU::OPERAND_REG_INLINE_AC_INT32:
3771 case AMDGPU::OPERAND_REG_INLINE_AC_FP32:
3772 case AMDGPU::OPERAND_REG_INLINE_AC_INT16:
3773 case AMDGPU::OPERAND_REG_INLINE_AC_FP16:
3774 case AMDGPU::OPERAND_REG_INLINE_AC_FP64: {
3775 if (!MO.isReg() && (!MO.isImm() || !isInlineConstant(MI, i))) {
3776 ErrInfo = "Illegal immediate value for operand.";
3777 return false;
3778 }
3779 break;
3780 }
3781 case MCOI::OPERAND_IMMEDIATE:
3782 case AMDGPU::OPERAND_KIMM32:
3783 // Check if this operand is an immediate.
3784 // FrameIndex operands will be replaced by immediates, so they are
3785 // allowed.
3786 if (!MI.getOperand(i).isImm() && !MI.getOperand(i).isFI()) {
3787 ErrInfo = "Expected immediate, but got non-immediate";
3788 return false;
3789 }
3790 LLVM_FALLTHROUGH[[gnu::fallthrough]];
3791 default:
3792 continue;
3793 }
3794
3795 if (!MO.isReg())
3796 continue;
3797 Register Reg = MO.getReg();
3798 if (!Reg)
3799 continue;
3800
3801 // FIXME: Ideally we would have separate instruction definitions with the
3802 // aligned register constraint.
3803 // FIXME: We do not verify inline asm operands, but custom inline asm
3804 // verification is broken anyway
3805 if (ST.needsAlignedVGPRs()) {
3806 const TargetRegisterClass *RC = RI.getRegClassForReg(MRI, Reg);
3807 const bool IsVGPR = RI.hasVGPRs(RC);
3808 const bool IsAGPR = !IsVGPR && RI.hasAGPRs(RC);
3809 if ((IsVGPR || IsAGPR) && MO.getSubReg()) {
3810 const TargetRegisterClass *SubRC =
3811 RI.getSubRegClass(RC, MO.getSubReg());
3812 RC = RI.getCompatibleSubRegClass(RC, SubRC, MO.getSubReg());
3813 if (RC)
3814 RC = SubRC;
3815 }
3816
3817 // Check that this is the aligned version of the class.
3818 if (!RC || !RI.isProperlyAlignedRC(*RC)) {
3819 ErrInfo = "Subtarget requires even aligned vector registers";
3820 return false;
3821 }
3822 }
3823
3824 if (RegClass != -1) {
3825 if (Reg.isVirtual())
3826 continue;
3827
3828 const TargetRegisterClass *RC = RI.getRegClass(RegClass);
3829 if (!RC->contains(Reg)) {
3830 ErrInfo = "Operand has incorrect register class.";
3831 return false;
3832 }
3833 }
3834 }
3835
3836 // Verify SDWA
3837 if (isSDWA(MI)) {
3838 if (!ST.hasSDWA()) {
3839 ErrInfo = "SDWA is not supported on this target";
3840 return false;
3841 }
3842
3843 int DstIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdst);
3844
3845 const int OpIndicies[] = { DstIdx, Src0Idx, Src1Idx, Src2Idx };
3846
3847 for (int OpIdx: OpIndicies) {
3848 if (OpIdx == -1)
3849 continue;
3850 const MachineOperand &MO = MI.getOperand(OpIdx);
3851
3852 if (!ST.hasSDWAScalar()) {
3853 // Only VGPRS on VI
3854 if (!MO.isReg() || !RI.hasVGPRs(RI.getRegClassForReg(MRI, MO.getReg()))) {
3855 ErrInfo = "Only VGPRs allowed as operands in SDWA instructions on VI";
3856 return false;
3857 }
3858 } else {
3859 // No immediates on GFX9
3860 if (!MO.isReg()) {
3861 ErrInfo =
3862 "Only reg allowed as operands in SDWA instructions on GFX9+";
3863 return false;
3864 }
3865 }
3866 }
3867
3868 if (!ST.hasSDWAOmod()) {
3869 // No omod allowed on VI
3870 const MachineOperand *OMod = getNamedOperand(MI, AMDGPU::OpName::omod);
3871 if (OMod != nullptr &&
3872 (!OMod->isImm() || OMod->getImm() != 0)) {
3873 ErrInfo = "OMod not allowed in SDWA instructions on VI";
3874 return false;
3875 }
3876 }
3877
3878 uint16_t BasicOpcode = AMDGPU::getBasicFromSDWAOp(Opcode);
3879 if (isVOPC(BasicOpcode)) {
3880 if (!ST.hasSDWASdst() && DstIdx != -1) {
3881 // Only vcc allowed as dst on VI for VOPC
3882 const MachineOperand &Dst = MI.getOperand(DstIdx);
3883 if (!Dst.isReg() || Dst.getReg() != AMDGPU::VCC) {
3884 ErrInfo = "Only VCC allowed as dst in SDWA instructions on VI";
3885 return false;
3886 }
3887 } else if (!ST.hasSDWAOutModsVOPC()) {
3888 // No clamp allowed on GFX9 for VOPC
3889 const MachineOperand *Clamp = getNamedOperand(MI, AMDGPU::OpName::clamp);
3890 if (Clamp && (!Clamp->isImm() || Clamp->getImm() != 0)) {
3891 ErrInfo = "Clamp not allowed in VOPC SDWA instructions on VI";
3892 return false;
3893 }
3894
3895 // No omod allowed on GFX9 for VOPC
3896 const MachineOperand *OMod = getNamedOperand(MI, AMDGPU::OpName::omod);
3897 if (OMod && (!OMod->isImm() || OMod->getImm() != 0)) {
3898 ErrInfo = "OMod not allowed in VOPC SDWA instructions on VI";
3899 return false;
3900 }
3901 }
3902 }
3903
3904 const MachineOperand *DstUnused = getNamedOperand(MI, AMDGPU::OpName::dst_unused);
3905 if (DstUnused && DstUnused->isImm() &&
3906 DstUnused->getImm() == AMDGPU::SDWA::UNUSED_PRESERVE) {
3907 const MachineOperand &Dst = MI.getOperand(DstIdx);
3908 if (!Dst.isReg() || !Dst.isTied()) {
3909 ErrInfo = "Dst register should have tied register";
3910 return false;
3911 }
3912
3913 const MachineOperand &TiedMO =
3914 MI.getOperand(MI.findTiedOperandIdx(DstIdx));
3915 if (!TiedMO.isReg() || !TiedMO.isImplicit() || !TiedMO.isUse()) {
3916 ErrInfo =
3917 "Dst register should be tied to implicit use of preserved register";
3918 return false;
3919 } else if (TiedMO.getReg().isPhysical() &&
3920 Dst.getReg() != TiedMO.getReg()) {
3921 ErrInfo = "Dst register should use same physical register as preserved";
3922 return false;
3923 }
3924 }
3925 }
3926
3927 // Verify MIMG
3928 if (isMIMG(MI.getOpcode()) && !MI.mayStore()) {
3929 // Ensure that the return type used is large enough for all the options
3930 // being used TFE/LWE require an extra result register.
3931 const MachineOperand *DMask = getNamedOperand(MI, AMDGPU::OpName::dmask);
3932 if (DMask) {
3933 uint64_t DMaskImm = DMask->getImm();
3934 uint32_t RegCount =
3935 isGather4(MI.getOpcode()) ? 4 : countPopulation(DMaskImm);
3936 const MachineOperand *TFE = getNamedOperand(MI, AMDGPU::OpName::tfe);
3937 const MachineOperand *LWE = getNamedOperand(MI, AMDGPU::OpName::lwe);
3938 const MachineOperand *D16 = getNamedOperand(MI, AMDGPU::OpName::d16);
3939
3940 // Adjust for packed 16 bit values
3941 if (D16 && D16->getImm() && !ST.hasUnpackedD16VMem())
3942 RegCount >>= 1;
3943
3944 // Adjust if using LWE or TFE
3945 if ((LWE && LWE->getImm()) || (TFE && TFE->getImm()))
3946 RegCount += 1;
3947
3948 const uint32_t DstIdx =
3949 AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::vdata);
3950 const MachineOperand &Dst = MI.getOperand(DstIdx);
3951 if (Dst.isReg()) {
3952 const TargetRegisterClass *DstRC = getOpRegClass(MI, DstIdx);
3953 uint32_t DstSize = RI.getRegSizeInBits(*DstRC) / 32;
3954 if (RegCount > DstSize) {
3955 ErrInfo = "MIMG instruction returns too many registers for dst "
3956 "register class";
3957 return false;
3958 }
3959 }
3960 }
3961 }
3962
3963 // Verify VOP*. Ignore multiple sgpr operands on writelane.
3964 if (Desc.getOpcode() != AMDGPU::V_WRITELANE_B32
3965 && (isVOP1(MI) || isVOP2(MI) || isVOP3(MI) || isVOPC(MI) || isSDWA(MI))) {
3966 // Only look at the true operands. Only a real operand can use the constant
3967 // bus, and we don't want to check pseudo-operands like the source modifier
3968 // flags.
3969 const int OpIndices[] = { Src0Idx, Src1Idx, Src2Idx };
3970
3971 unsigned ConstantBusCount = 0;
3972 bool UsesLiteral = false;
3973 const MachineOperand *LiteralVal = nullptr;
3974
3975 if (AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::imm) != -1)
3976 ++ConstantBusCount;
3977
3978 SmallVector<Register, 2> SGPRsUsed;
3979 Register SGPRUsed;
3980
3981 for (int OpIdx : OpIndices) {
3982 if (OpIdx == -1)
3983 break;
3984 const MachineOperand &MO = MI.getOperand(OpIdx);
3985 if (usesConstantBus(MRI, MO, MI.getDesc().OpInfo[OpIdx])) {
3986 if (MO.isReg()) {
3987 SGPRUsed = MO.getReg();
3988 if (llvm::all_of(SGPRsUsed, [SGPRUsed](unsigned SGPR) {
3989 return SGPRUsed != SGPR;
3990 })) {
3991 ++ConstantBusCount;
3992 SGPRsUsed.push_back(SGPRUsed);
3993 }
3994 } else {
3995 if (!UsesLiteral) {
3996 ++ConstantBusCount;
3997 UsesLiteral = true;
3998 LiteralVal = &MO;
3999 } else if (!MO.isIdenticalTo(*LiteralVal)) {
4000 assert(isVOP3(MI))(static_cast <bool> (isVOP3(MI)) ? void (0) : __assert_fail
("isVOP3(MI)", "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 4000, __extension__ __PRETTY_FUNCTION__))
;
4001 ErrInfo = "VOP3 instruction uses more than one literal";
4002 return false;
4003 }
4004 }
4005 }
4006 }
4007
4008 SGPRUsed = findImplicitSGPRRead(MI);
4009 if (SGPRUsed != AMDGPU::NoRegister) {
4010 // Implicit uses may safely overlap true overands
4011 if (llvm::all_of(SGPRsUsed, [this, SGPRUsed](unsigned SGPR) {
4012 return !RI.regsOverlap(SGPRUsed, SGPR);
4013 })) {
4014 ++ConstantBusCount;
4015 SGPRsUsed.push_back(SGPRUsed);
4016 }
4017 }
4018
4019 // v_writelane_b32 is an exception from constant bus restriction:
4020 // vsrc0 can be sgpr, const or m0 and lane select sgpr, m0 or inline-const
4021 if (ConstantBusCount > ST.getConstantBusLimit(Opcode) &&
4022 Opcode != AMDGPU::V_WRITELANE_B32) {
4023 ErrInfo = "VOP* instruction violates constant bus restriction";
4024 return false;
4025 }
4026
4027 if (isVOP3(MI) && UsesLiteral && !ST.hasVOP3Literal()) {
4028 ErrInfo = "VOP3 instruction uses literal";
4029 return false;
4030 }
4031 }
4032
4033 // Special case for writelane - this can break the multiple constant bus rule,
4034 // but still can't use more than one SGPR register
4035 if (Desc.getOpcode() == AMDGPU::V_WRITELANE_B32) {
4036 unsigned SGPRCount = 0;
4037 Register SGPRUsed = AMDGPU::NoRegister;
4038
4039 for (int OpIdx : {Src0Idx, Src1Idx, Src2Idx}) {
4040 if (OpIdx == -1)
4041 break;
4042
4043 const MachineOperand &MO = MI.getOperand(OpIdx);
4044
4045 if (usesConstantBus(MRI, MO, MI.getDesc().OpInfo[OpIdx])) {
4046 if (MO.isReg() && MO.getReg() != AMDGPU::M0) {
4047 if (MO.getReg() != SGPRUsed)
4048 ++SGPRCount;
4049 SGPRUsed = MO.getReg();
4050 }
4051 }
4052 if (SGPRCount > ST.getConstantBusLimit(Opcode)) {
4053 ErrInfo = "WRITELANE instruction violates constant bus restriction";
4054 return false;
4055 }
4056 }
4057 }
4058
4059 // Verify misc. restrictions on specific instructions.
4060 if (Desc.getOpcode() == AMDGPU::V_DIV_SCALE_F32_e64 ||
4061 Desc.getOpcode() == AMDGPU::V_DIV_SCALE_F64_e64) {
4062 const MachineOperand &Src0 = MI.getOperand(Src0Idx);
4063 const MachineOperand &Src1 = MI.getOperand(Src1Idx);
4064 const MachineOperand &Src2 = MI.getOperand(Src2Idx);
4065 if (Src0.isReg() && Src1.isReg() && Src2.isReg()) {
4066 if (!compareMachineOp(Src0, Src1) &&
4067 !compareMachineOp(Src0, Src2)) {
4068 ErrInfo = "v_div_scale_{f32|f64} require src0 = src1 or src2";
4069 return false;
4070 }
4071 }
4072 if ((getNamedOperand(MI, AMDGPU::OpName::src0_modifiers)->getImm() &
4073 SISrcMods::ABS) ||
4074 (getNamedOperand(MI, AMDGPU::OpName::src1_modifiers)->getImm() &
4075 SISrcMods::ABS) ||
4076 (getNamedOperand(MI, AMDGPU::OpName::src2_modifiers)->getImm() &
4077 SISrcMods::ABS)) {
4078 ErrInfo = "ABS not allowed in VOP3B instructions";
4079 return false;
4080 }
4081 }
4082
4083 if (isSOP2(MI) || isSOPC(MI)) {
4084 const MachineOperand &Src0 = MI.getOperand(Src0Idx);
4085 const MachineOperand &Src1 = MI.getOperand(Src1Idx);
4086 unsigned Immediates = 0;
4087
4088 if (!Src0.isReg() &&
4089 !isInlineConstant(Src0, Desc.OpInfo[Src0Idx].OperandType))
4090 Immediates++;
4091 if (!Src1.isReg() &&
4092 !isInlineConstant(Src1, Desc.OpInfo[Src1Idx].OperandType))
4093 Immediates++;
4094
4095 if (Immediates > 1) {
4096 ErrInfo = "SOP2/SOPC instruction requires too many immediate constants";
4097 return false;
4098 }
4099 }
4100
4101 if (isSOPK(MI)) {
4102 auto Op = getNamedOperand(MI, AMDGPU::OpName::simm16);
4103 if (Desc.isBranch()) {
4104 if (!Op->isMBB()) {
4105 ErrInfo = "invalid branch target for SOPK instruction";
4106 return false;
4107 }
4108 } else {
4109 uint64_t Imm = Op->getImm();
4110 if (sopkIsZext(MI)) {
4111 if (!isUInt<16>(Imm)) {
4112 ErrInfo = "invalid immediate for SOPK instruction";
4113 return false;
4114 }
4115 } else {
4116 if (!isInt<16>(Imm)) {
4117 ErrInfo = "invalid immediate for SOPK instruction";
4118 return false;
4119 }
4120 }
4121 }
4122 }
4123
4124 if (Desc.getOpcode() == AMDGPU::V_MOVRELS_B32_e32 ||
4125 Desc.getOpcode() == AMDGPU::V_MOVRELS_B32_e64 ||
4126 Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e32 ||
4127 Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e64) {
4128 const bool IsDst = Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e32 ||
4129 Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e64;
4130
4131 const unsigned StaticNumOps = Desc.getNumOperands() +
4132 Desc.getNumImplicitUses();
4133 const unsigned NumImplicitOps = IsDst ? 2 : 1;
4134
4135 // Allow additional implicit operands. This allows a fixup done by the post
4136 // RA scheduler where the main implicit operand is killed and implicit-defs
4137 // are added for sub-registers that remain live after this instruction.
4138 if (MI.getNumOperands() < StaticNumOps + NumImplicitOps) {
4139 ErrInfo = "missing implicit register operands";
4140 return false;
4141 }
4142
4143 const MachineOperand *Dst = getNamedOperand(MI, AMDGPU::OpName::vdst);
4144 if (IsDst) {
4145 if (!Dst->isUse()) {
4146 ErrInfo = "v_movreld_b32 vdst should be a use operand";
4147 return false;
4148 }
4149
4150 unsigned UseOpIdx;
4151 if (!MI.isRegTiedToUseOperand(StaticNumOps, &UseOpIdx) ||
4152 UseOpIdx != StaticNumOps + 1) {
4153 ErrInfo = "movrel implicit operands should be tied";
4154 return false;
4155 }
4156 }
4157
4158 const MachineOperand &Src0 = MI.getOperand(Src0Idx);
4159 const MachineOperand &ImpUse
4160 = MI.getOperand(StaticNumOps + NumImplicitOps - 1);
4161 if (!ImpUse.isReg() || !ImpUse.isUse() ||
4162 !isSubRegOf(RI, ImpUse, IsDst ? *Dst : Src0)) {
4163 ErrInfo = "src0 should be subreg of implicit vector use";
4164 return false;
4165 }
4166 }
4167
4168 // Make sure we aren't losing exec uses in the td files. This mostly requires
4169 // being careful when using let Uses to try to add other use registers.
4170 if (shouldReadExec(MI)) {
4171 if (!MI.hasRegisterImplicitUseOperand(AMDGPU::EXEC)) {
4172 ErrInfo = "VALU instruction does not implicitly read exec mask";
4173 return false;
4174 }
4175 }
4176
4177 if (isSMRD(MI)) {
4178 if (MI.mayStore()) {
4179 // The register offset form of scalar stores may only use m0 as the
4180 // soffset register.
4181 const MachineOperand *Soff = getNamedOperand(MI, AMDGPU::OpName::soff);
4182 if (Soff && Soff->getReg() != AMDGPU::M0) {
4183 ErrInfo = "scalar stores must use m0 as offset register";
4184 return false;
4185 }
4186 }
4187 }
4188
4189 if (isFLAT(MI) && !ST.hasFlatInstOffsets()) {
4190 const MachineOperand *Offset = getNamedOperand(MI, AMDGPU::OpName::offset);
4191 if (Offset->getImm() != 0) {
4192 ErrInfo = "subtarget does not support offsets in flat instructions";
4193 return false;
4194 }
4195 }
4196
4197 if (isMIMG(MI)) {
4198 const MachineOperand *DimOp = getNamedOperand(MI, AMDGPU::OpName::dim);
4199 if (DimOp) {
4200 int VAddr0Idx = AMDGPU::getNamedOperandIdx(Opcode,
4201 AMDGPU::OpName::vaddr0);
4202 int SRsrcIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::srsrc);
4203 const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(Opcode);
4204 const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode =
4205 AMDGPU::getMIMGBaseOpcodeInfo(Info->BaseOpcode);
4206 const AMDGPU::MIMGDimInfo *Dim =
4207 AMDGPU::getMIMGDimInfoByEncoding(DimOp->getImm());
4208
4209 if (!Dim) {
4210 ErrInfo = "dim is out of range";
4211 return false;
4212 }
4213
4214 bool IsA16 = false;
4215 if (ST.hasR128A16()) {
4216 const MachineOperand *R128A16 = getNamedOperand(MI, AMDGPU::OpName::r128);
4217 IsA16 = R128A16->getImm() != 0;
4218 } else if (ST.hasGFX10A16()) {
4219 const MachineOperand *A16 = getNamedOperand(MI, AMDGPU::OpName::a16);
4220 IsA16 = A16->getImm() != 0;
4221 }
4222
4223 bool IsNSA = SRsrcIdx - VAddr0Idx > 1;
4224
4225 unsigned AddrWords =
4226 AMDGPU::getAddrSizeMIMGOp(BaseOpcode, Dim, IsA16, ST.hasG16());
4227
4228 unsigned VAddrWords;
4229 if (IsNSA) {
4230 VAddrWords = SRsrcIdx - VAddr0Idx;
4231 } else {
4232 const TargetRegisterClass *RC = getOpRegClass(MI, VAddr0Idx);
4233 VAddrWords = MRI.getTargetRegisterInfo()->getRegSizeInBits(*RC) / 32;
4234 if (AddrWords > 8)
4235 AddrWords = 16;
4236 else if (AddrWords > 5)
4237 AddrWords = 8;
4238 }
4239
4240 if (VAddrWords != AddrWords) {
4241 LLVM_DEBUG(dbgs() << "bad vaddr size, expected " << AddrWordsdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("si-instr-info")) { dbgs() << "bad vaddr size, expected "
<< AddrWords << " but got " << VAddrWords <<
"\n"; } } while (false)
4242 << " but got " << VAddrWords << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("si-instr-info")) { dbgs() << "bad vaddr size, expected "
<< AddrWords << " but got " << VAddrWords <<
"\n"; } } while (false)
;
4243 ErrInfo = "bad vaddr size";
4244 return false;
4245 }
4246 }
4247 }
4248
4249 const MachineOperand *DppCt = getNamedOperand(MI, AMDGPU::OpName::dpp_ctrl);
4250 if (DppCt) {
4251 using namespace AMDGPU::DPP;
4252
4253 unsigned DC = DppCt->getImm();
4254 if (DC == DppCtrl::DPP_UNUSED1 || DC == DppCtrl::DPP_UNUSED2 ||
4255 DC == DppCtrl::DPP_UNUSED3 || DC > DppCtrl::DPP_LAST ||
4256 (DC >= DppCtrl::DPP_UNUSED4_FIRST && DC <= DppCtrl::DPP_UNUSED4_LAST) ||
4257 (DC >= DppCtrl::DPP_UNUSED5_FIRST && DC <= DppCtrl::DPP_UNUSED5_LAST) ||
4258 (DC >= DppCtrl::DPP_UNUSED6_FIRST && DC <= DppCtrl::DPP_UNUSED6_LAST) ||
4259 (DC >= DppCtrl::DPP_UNUSED7_FIRST && DC <= DppCtrl::DPP_UNUSED7_LAST) ||
4260 (DC >= DppCtrl::DPP_UNUSED8_FIRST && DC <= DppCtrl::DPP_UNUSED8_LAST)) {
4261 ErrInfo = "Invalid dpp_ctrl value";
4262 return false;
4263 }
4264 if (DC >= DppCtrl::WAVE_SHL1 && DC <= DppCtrl::WAVE_ROR1 &&
4265 ST.getGeneration() >= AMDGPUSubtarget::GFX10) {
4266 ErrInfo = "Invalid dpp_ctrl value: "
4267 "wavefront shifts are not supported on GFX10+";
4268 return false;
4269 }
4270 if (DC >= DppCtrl::BCAST15 && DC <= DppCtrl::BCAST31 &&
4271 ST.getGeneration() >= AMDGPUSubtarget::GFX10) {
4272 ErrInfo = "Invalid dpp_ctrl value: "
4273 "broadcasts are not supported on GFX10+";
4274 return false;
4275 }
4276 if (DC >= DppCtrl::ROW_SHARE_FIRST && DC <= DppCtrl::ROW_XMASK_LAST &&
4277 ST.getGeneration() < AMDGPUSubtarget::GFX10) {
4278 if (DC >= DppCtrl::ROW_NEWBCAST_FIRST &&
4279 DC <= DppCtrl::ROW_NEWBCAST_LAST &&
4280 !ST.hasGFX90AInsts()) {
4281 ErrInfo = "Invalid dpp_ctrl value: "
4282 "row_newbroadcast/row_share is not supported before "
4283 "GFX90A/GFX10";
4284 return false;
4285 } else if (DC > DppCtrl::ROW_NEWBCAST_LAST || !ST.hasGFX90AInsts()) {
4286 ErrInfo = "Invalid dpp_ctrl value: "
4287 "row_share and row_xmask are not supported before GFX10";
4288 return false;
4289 }
4290 }
4291
4292 int DstIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdst);
4293 int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0);
4294
4295 if (Opcode != AMDGPU::V_MOV_B64_DPP_PSEUDO &&
4296 ((DstIdx >= 0 &&
4297 (Desc.OpInfo[DstIdx].RegClass == AMDGPU::VReg_64RegClassID ||
4298 Desc.OpInfo[DstIdx].RegClass == AMDGPU::VReg_64_Align2RegClassID)) ||
4299 ((Src0Idx >= 0 &&
4300 (Desc.OpInfo[Src0Idx].RegClass == AMDGPU::VReg_64RegClassID ||
4301 Desc.OpInfo[Src0Idx].RegClass ==
4302 AMDGPU::VReg_64_Align2RegClassID)))) &&
4303 !AMDGPU::isLegal64BitDPPControl(DC)) {
4304 ErrInfo = "Invalid dpp_ctrl value: "
4305 "64 bit dpp only support row_newbcast";
4306 return false;
4307 }
4308 }
4309
4310 if ((MI.mayStore() || MI.mayLoad()) && !isVGPRSpill(MI)) {
4311 const MachineOperand *Dst = getNamedOperand(MI, AMDGPU::OpName::vdst);
4312 uint16_t DataNameIdx = isDS(Opcode) ? AMDGPU::OpName::data0
4313 : AMDGPU::OpName::vdata;
4314 const MachineOperand *Data = getNamedOperand(MI, DataNameIdx);
4315 const MachineOperand *Data2 = getNamedOperand(MI, AMDGPU::OpName::data1);
4316 if (Data && !Data->isReg())
4317 Data = nullptr;
4318
4319 if (ST.hasGFX90AInsts()) {
4320 if (Dst && Data &&
4321 (RI.isAGPR(MRI, Dst->getReg()) != RI.isAGPR(MRI, Data->getReg()))) {
4322 ErrInfo = "Invalid register class: "
4323 "vdata and vdst should be both VGPR or AGPR";
4324 return false;
4325 }
4326 if (Data && Data2 &&
4327 (RI.isAGPR(MRI, Data->getReg()) != RI.isAGPR(MRI, Data2->getReg()))) {
4328 ErrInfo = "Invalid register class: "
4329 "both data operands should be VGPR or AGPR";
4330 return false;
4331 }
4332 } else {
4333 if ((Dst && RI.isAGPR(MRI, Dst->getReg())) ||
4334 (Data && RI.isAGPR(MRI, Data->getReg())) ||
4335 (Data2 && RI.isAGPR(MRI, Data2->getReg()))) {
4336 ErrInfo = "Invalid register class: "
4337 "agpr loads and stores not supported on this GPU";
4338 return false;
4339 }
4340 }
4341 }
4342
4343 if (ST.needsAlignedVGPRs() &&
4344 (MI.getOpcode() == AMDGPU::DS_GWS_INIT ||
4345 MI.getOpcode() == AMDGPU::DS_GWS_SEMA_BR ||
4346 MI.getOpcode() == AMDGPU::DS_GWS_BARRIER)) {
4347 const MachineOperand *Op = getNamedOperand(MI, AMDGPU::OpName::data0);
4348 Register Reg = Op->getReg();
4349 bool Aligned = true;
4350 if (Reg.isPhysical()) {
4351 Aligned = !(RI.getHWRegIndex(Reg) & 1);
4352 } else {
4353 const TargetRegisterClass &RC = *MRI.getRegClass(Reg);
4354 Aligned = RI.getRegSizeInBits(RC) > 32 && RI.isProperlyAlignedRC(RC) &&
4355 !(RI.getChannelFromSubReg(Op->getSubReg()) & 1);
4356 }
4357
4358 if (!Aligned) {
4359 ErrInfo = "Subtarget requires even aligned vector registers "
4360 "for DS_GWS instructions";
4361 return false;
4362 }
4363 }
4364
4365 return true;
4366}
4367
4368unsigned SIInstrInfo::getVALUOp(const MachineInstr &MI) const {
4369 switch (MI.getOpcode()) {
4370 default: return AMDGPU::INSTRUCTION_LIST_END;
4371 case AMDGPU::REG_SEQUENCE: return AMDGPU::REG_SEQUENCE;
4372 case AMDGPU::COPY: return AMDGPU::COPY;
4373 case AMDGPU::PHI: return AMDGPU::PHI;
4374 case AMDGPU::INSERT_SUBREG: return AMDGPU::INSERT_SUBREG;
4375 case AMDGPU::WQM: return AMDGPU::WQM;
4376 case AMDGPU::SOFT_WQM: return AMDGPU::SOFT_WQM;
4377 case AMDGPU::STRICT_WWM: return AMDGPU::STRICT_WWM;
4378 case AMDGPU::STRICT_WQM: return AMDGPU::STRICT_WQM;
4379 case AMDGPU::S_MOV_B32: {
4380 const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
4381 return MI.getOperand(1).isReg() ||
4382 RI.isAGPR(MRI, MI.getOperand(0).getReg()) ?
4383 AMDGPU::COPY : AMDGPU::V_MOV_B32_e32;
4384 }
4385 case AMDGPU::S_ADD_I32:
4386 return ST.hasAddNoCarry() ? AMDGPU::V_ADD_U32_e64 : AMDGPU::V_ADD_CO_U32_e32;
4387 case AMDGPU::S_ADDC_U32:
4388 return AMDGPU::V_ADDC_U32_e32;
4389 case AMDGPU::S_SUB_I32:
4390 return ST.hasAddNoCarry() ? AMDGPU::V_SUB_U32_e64 : AMDGPU::V_SUB_CO_U32_e32;
4391 // FIXME: These are not consistently handled, and selected when the carry is
4392 // used.
4393 case AMDGPU::S_ADD_U32:
4394 return AMDGPU::V_ADD_CO_U32_e32;
4395 case AMDGPU::S_SUB_U32:
4396 return AMDGPU::V_SUB_CO_U32_e32;
4397 case AMDGPU::S_SUBB_U32: return AMDGPU::V_SUBB_U32_e32;
4398 case AMDGPU::S_MUL_I32: return AMDGPU::V_MUL_LO_U32_e64;
4399 case AMDGPU::S_MUL_HI_U32: return AMDGPU::V_MUL_HI_U32_e64;
4400 case AMDGPU::S_MUL_HI_I32: return AMDGPU::V_MUL_HI_I32_e64;
4401 case AMDGPU::S_AND_B32: return AMDGPU::V_AND_B32_e64;
4402 case AMDGPU::S_OR_B32: return AMDGPU::V_OR_B32_e64;
4403 case AMDGPU::S_XOR_B32: return AMDGPU::V_XOR_B32_e64;
4404 case AMDGPU::S_XNOR_B32:
4405 return ST.hasDLInsts() ? AMDGPU::V_XNOR_B32_e64 : AMDGPU::INSTRUCTION_LIST_END;
4406 case AMDGPU::S_MIN_I32: return AMDGPU::V_MIN_I32_e64;
4407 case AMDGPU::S_MIN_U32: return AMDGPU::V_MIN_U32_e64;
4408 case AMDGPU::S_MAX_I32: return AMDGPU::V_MAX_I32_e64;
4409 case AMDGPU::S_MAX_U32: return AMDGPU::V_MAX_U32_e64;
4410 case AMDGPU::S_ASHR_I32: return AMDGPU::V_ASHR_I32_e32;
4411 case AMDGPU::S_ASHR_I64: return AMDGPU::V_ASHR_I64_e64;
4412 case AMDGPU::S_LSHL_B32: return AMDGPU::V_LSHL_B32_e32;
4413 case AMDGPU::S_LSHL_B64: return AMDGPU::V_LSHL_B64_e64;
4414 case AMDGPU::S_LSHR_B32: return AMDGPU::V_LSHR_B32_e32;
4415 case AMDGPU::S_LSHR_B64: return AMDGPU::V_LSHR_B64_e64;
4416 case AMDGPU::S_SEXT_I32_I8: return AMDGPU::V_BFE_I32_e64;
4417 case AMDGPU::S_SEXT_I32_I16: return AMDGPU::V_BFE_I32_e64;
4418 case AMDGPU::S_BFE_U32: return AMDGPU::V_BFE_U32_e64;
4419 case AMDGPU::S_BFE_I32: return AMDGPU::V_BFE_I32_e64;
4420 case AMDGPU::S_BFM_B32: return AMDGPU::V_BFM_B32_e64;
4421 case AMDGPU::S_BREV_B32: return AMDGPU::V_BFREV_B32_e32;
4422 case AMDGPU::S_NOT_B32: return AMDGPU::V_NOT_B32_e32;
4423 case AMDGPU::S_NOT_B64: return AMDGPU::V_NOT_B32_e32;
4424 case AMDGPU::S_CMP_EQ_I32: return AMDGPU::V_CMP_EQ_I32_e32;
4425 case AMDGPU::S_CMP_LG_I32: return AMDGPU::V_CMP_NE_I32_e32;
4426 case AMDGPU::S_CMP_GT_I32: return AMDGPU::V_CMP_GT_I32_e32;
4427 case AMDGPU::S_CMP_GE_I32: return AMDGPU::V_CMP_GE_I32_e32;
4428 case AMDGPU::S_CMP_LT_I32: return AMDGPU::V_CMP_LT_I32_e32;
4429 case AMDGPU::S_CMP_LE_I32: return AMDGPU::V_CMP_LE_I32_e32;
4430 case AMDGPU::S_CMP_EQ_U32: return AMDGPU::V_CMP_EQ_U32_e32;
4431 case AMDGPU::S_CMP_LG_U32: return AMDGPU::V_CMP_NE_U32_e32;
4432 case AMDGPU::S_CMP_GT_U32: return AMDGPU::V_CMP_GT_U32_e32;
4433 case AMDGPU::S_CMP_GE_U32: return AMDGPU::V_CMP_GE_U32_e32;
4434 case AMDGPU::S_CMP_LT_U32: return AMDGPU::V_CMP_LT_U32_e32;
4435 case AMDGPU::S_CMP_LE_U32: return AMDGPU::V_CMP_LE_U32_e32;
4436 case AMDGPU::S_CMP_EQ_U64: return AMDGPU::V_CMP_EQ_U64_e32;
4437 case AMDGPU::S_CMP_LG_U64: return AMDGPU::V_CMP_NE_U64_e32;
4438 case AMDGPU::S_BCNT1_I32_B32: return AMDGPU::V_BCNT_U32_B32_e64;
4439 case AMDGPU::S_FF1_I32_B32: return AMDGPU::V_FFBL_B32_e32;
4440 case AMDGPU::S_FLBIT_I32_B32: return AMDGPU::V_FFBH_U32_e32;
4441 case AMDGPU::S_FLBIT_I32: return AMDGPU::V_FFBH_I32_e64;
4442 case AMDGPU::S_CBRANCH_SCC0: return AMDGPU::S_CBRANCH_VCCZ;
4443 case AMDGPU::S_CBRANCH_SCC1: return AMDGPU::S_CBRANCH_VCCNZ;
4444 }
4445 llvm_unreachable(::llvm::llvm_unreachable_internal("Unexpected scalar opcode without corresponding vector one!"
, "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 4446)
4446 "Unexpected scalar opcode without corresponding vector one!")::llvm::llvm_unreachable_internal("Unexpected scalar opcode without corresponding vector one!"
, "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 4446)
;
4447}
4448
4449static unsigned adjustAllocatableRegClass(const GCNSubtarget &ST,
4450 const MachineRegisterInfo &MRI,
4451 const MCInstrDesc &TID,
4452 unsigned RCID,
4453 bool IsAllocatable) {
4454 if ((IsAllocatable || !ST.hasGFX90AInsts() || !MRI.reservedRegsFrozen()) &&
4455 (TID.mayLoad() || TID.mayStore() ||
4456 (TID.TSFlags & (SIInstrFlags::DS | SIInstrFlags::MIMG)))) {
4457 switch (RCID) {
4458 case AMDGPU::AV_32RegClassID: return AMDGPU::VGPR_32RegClassID;
4459 case AMDGPU::AV_64RegClassID: return AMDGPU::VReg_64RegClassID;
4460 case AMDGPU::AV_96RegClassID: return AMDGPU::VReg_96RegClassID;
4461 case AMDGPU::AV_128RegClassID: return AMDGPU::VReg_128RegClassID;
4462 case AMDGPU::AV_160RegClassID: return AMDGPU::VReg_160RegClassID;
4463 default:
4464 break;
4465 }
4466 }
4467 return RCID;
4468}
4469
4470const TargetRegisterClass *SIInstrInfo::getRegClass(const MCInstrDesc &TID,
4471 unsigned OpNum, const TargetRegisterInfo *TRI,
4472 const MachineFunction &MF)
4473 const {
4474 if (OpNum >= TID.getNumOperands())
4475 return nullptr;
4476 auto RegClass = TID.OpInfo[OpNum].RegClass;
4477 bool IsAllocatable = false;
4478 if (TID.TSFlags & (SIInstrFlags::DS | SIInstrFlags::FLAT)) {
4479 // vdst and vdata should be both VGPR or AGPR, same for the DS instructions
4480 // with two data operands. Request register class constainted to VGPR only
4481 // of both operands present as Machine Copy Propagation can not check this
4482 // constraint and possibly other passes too.
4483 //
4484 // The check is limited to FLAT and DS because atomics in non-flat encoding
4485 // have their vdst and vdata tied to be the same register.
4486 const int VDstIdx = AMDGPU::getNamedOperandIdx(TID.Opcode,
4487 AMDGPU::OpName::vdst);
4488 const int DataIdx = AMDGPU::getNamedOperandIdx(TID.Opcode,
4489 (TID.TSFlags & SIInstrFlags::DS) ? AMDGPU::OpName::data0
4490 : AMDGPU::OpName::vdata);
4491 if (DataIdx != -1) {
4492 IsAllocatable = VDstIdx != -1 ||
4493 AMDGPU::getNamedOperandIdx(TID.Opcode,
4494 AMDGPU::OpName::data1) != -1;
4495 }
4496 }
4497 RegClass = adjustAllocatableRegClass(ST, MF.getRegInfo(), TID, RegClass,
4498 IsAllocatable);
4499 return RI.getRegClass(RegClass);
4500}
4501
4502const TargetRegisterClass *SIInstrInfo::getOpRegClass(const MachineInstr &MI,
4503 unsigned OpNo) const {
4504 const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
4505 const MCInstrDesc &Desc = get(MI.getOpcode());
4506 if (MI.isVariadic() || OpNo >= Desc.getNumOperands() ||
4507 Desc.OpInfo[OpNo].RegClass == -1) {
4508 Register Reg = MI.getOperand(OpNo).getReg();
4509
4510 if (Reg.isVirtual())
4511 return MRI.getRegClass(Reg);
4512 return RI.getPhysRegClass(Reg);
4513 }
4514
4515 unsigned RCID = Desc.OpInfo[OpNo].RegClass;
4516 RCID = adjustAllocatableRegClass(ST, MRI, Desc, RCID, true);
4517 return RI.getRegClass(RCID);
4518}
4519
4520void SIInstrInfo::legalizeOpWithMove(MachineInstr &MI, unsigned OpIdx) const {
4521 MachineBasicBlock::iterator I = MI;
4522 MachineBasicBlock *MBB = MI.getParent();
4523 MachineOperand &MO = MI.getOperand(OpIdx);
4524 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
4525 unsigned RCID = get(MI.getOpcode()).OpInfo[OpIdx].RegClass;
4526 const TargetRegisterClass *RC = RI.getRegClass(RCID);
4527 unsigned Size = RI.getRegSizeInBits(*RC);
4528 unsigned Opcode = (Size == 64) ? AMDGPU::V_MOV_B64_PSEUDO : AMDGPU::V_MOV_B32_e32;
4529 if (MO.isReg())
4530 Opcode = AMDGPU::COPY;
4531 else if (RI.isSGPRClass(RC))
4532 Opcode = (Size == 64) ? AMDGPU::S_MOV_B64 : AMDGPU::S_MOV_B32;
4533
4534 const TargetRegisterClass *VRC = RI.getEquivalentVGPRClass(RC);
4535 const TargetRegisterClass *VRC64 = RI.getVGPR64Class();
4536 if (RI.getCommonSubClass(VRC64, VRC))
4537 VRC = VRC64;
4538 else
4539 VRC = &AMDGPU::VGPR_32RegClass;
4540
4541 Register Reg = MRI.createVirtualRegister(VRC);
4542 DebugLoc DL = MBB->findDebugLoc(I);
4543 BuildMI(*MI.getParent(), I, DL, get(Opcode), Reg).add(MO);
4544 MO.ChangeToRegister(Reg, false);
4545}
4546
4547unsigned SIInstrInfo::buildExtractSubReg(MachineBasicBlock::iterator MI,
4548 MachineRegisterInfo &MRI,
4549 MachineOperand &SuperReg,
4550 const TargetRegisterClass *SuperRC,
4551 unsigned SubIdx,
4552 const TargetRegisterClass *SubRC)
4553 const {
4554 MachineBasicBlock *MBB = MI->getParent();
4555 DebugLoc DL = MI->getDebugLoc();
4556 Register SubReg = MRI.createVirtualRegister(SubRC);
4557
4558 if (SuperReg.getSubReg() == AMDGPU::NoSubRegister) {
4559 BuildMI(*MBB, MI, DL, get(TargetOpcode::COPY), SubReg)
4560 .addReg(SuperReg.getReg(), 0, SubIdx);
4561 return SubReg;
4562 }
4563
4564 // Just in case the super register is itself a sub-register, copy it to a new
4565 // value so we don't need to worry about merging its subreg index with the
4566 // SubIdx passed to this function. The register coalescer should be able to
4567 // eliminate this extra copy.
4568 Register NewSuperReg = MRI.createVirtualRegister(SuperRC);
4569
4570 BuildMI(*MBB, MI, DL, get(TargetOpcode::COPY), NewSuperReg)
4571 .addReg(SuperReg.getReg(), 0, SuperReg.getSubReg());
4572
4573 BuildMI(*MBB, MI, DL, get(TargetOpcode::COPY), SubReg)
4574 .addReg(NewSuperReg, 0, SubIdx);
4575
4576 return SubReg;
4577}
4578
4579MachineOperand SIInstrInfo::buildExtractSubRegOrImm(
4580 MachineBasicBlock::iterator MII,
4581 MachineRegisterInfo &MRI,
4582 MachineOperand &Op,
4583 const TargetRegisterClass *SuperRC,
4584 unsigned SubIdx,
4585 const TargetRegisterClass *SubRC) const {
4586 if (Op.isImm()) {
4587 if (SubIdx == AMDGPU::sub0)
4588 return MachineOperand::CreateImm(static_cast<int32_t>(Op.getImm()));
4589 if (SubIdx == AMDGPU::sub1)
4590 return MachineOperand::CreateImm(static_cast<int32_t>(Op.getImm() >> 32));
4591
4592 llvm_unreachable("Unhandled register index for immediate")::llvm::llvm_unreachable_internal("Unhandled register index for immediate"
, "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 4592)
;
4593 }
4594
4595 unsigned SubReg = buildExtractSubReg(MII, MRI, Op, SuperRC,
4596 SubIdx, SubRC);
4597 return MachineOperand::CreateReg(SubReg, false);
4598}
4599
4600// Change the order of operands from (0, 1, 2) to (0, 2, 1)
4601void SIInstrInfo::swapOperands(MachineInstr &Inst) const {
4602 assert(Inst.getNumExplicitOperands() == 3)(static_cast <bool> (Inst.getNumExplicitOperands() == 3
) ? void (0) : __assert_fail ("Inst.getNumExplicitOperands() == 3"
, "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 4602, __extension__ __PRETTY_FUNCTION__))
;
4603 MachineOperand Op1 = Inst.getOperand(1);
4604 Inst.RemoveOperand(1);
4605 Inst.addOperand(Op1);
4606}
4607
4608bool SIInstrInfo::isLegalRegOperand(const MachineRegisterInfo &MRI,
4609 const MCOperandInfo &OpInfo,
4610 const MachineOperand &MO) const {
4611 if (!MO.isReg())
4612 return false;
4613
4614 Register Reg = MO.getReg();
4615
4616 const TargetRegisterClass *DRC = RI.getRegClass(OpInfo.RegClass);
4617 if (Reg.isPhysical())
4618 return DRC->contains(Reg);
4619
4620 const TargetRegisterClass *RC = MRI.getRegClass(Reg);
4621
4622 if (MO.getSubReg()) {
4623 const MachineFunction *MF = MO.getParent()->getParent()->getParent();
4624 const TargetRegisterClass *SuperRC = RI.getLargestLegalSuperClass(RC, *MF);
4625 if (!SuperRC)
4626 return false;
4627
4628 DRC = RI.getMatchingSuperRegClass(SuperRC, DRC, MO.getSubReg());
4629 if (!DRC)
4630 return false;
4631 }
4632 return RC->hasSuperClassEq(DRC);
4633}
4634
4635bool SIInstrInfo::isLegalVSrcOperand(const MachineRegisterInfo &MRI,
4636 const MCOperandInfo &OpInfo,
4637 const MachineOperand &MO) const {
4638 if (MO.isReg())
4639 return isLegalRegOperand(MRI, OpInfo, MO);
4640
4641 // Handle non-register types that are treated like immediates.
4642 assert(MO.isImm() || MO.isTargetIndex() || MO.isFI() || MO.isGlobal())(static_cast <bool> (MO.isImm() || MO.isTargetIndex() ||
MO.isFI() || MO.isGlobal()) ? void (0) : __assert_fail ("MO.isImm() || MO.isTargetIndex() || MO.isFI() || MO.isGlobal()"
, "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 4642, __extension__ __PRETTY_FUNCTION__))
;
4643 return true;
4644}
4645
4646bool SIInstrInfo::isOperandLegal(const MachineInstr &MI, unsigned OpIdx,
4647 const MachineOperand *MO) const {
4648 const MachineFunction &MF = *MI.getParent()->getParent();
4649 const MachineRegisterInfo &MRI = MF.getRegInfo();
4650 const MCInstrDesc &InstDesc = MI.getDesc();
4651 const MCOperandInfo &OpInfo = InstDesc.OpInfo[OpIdx];
4652 const TargetRegisterClass *DefinedRC =
4653 OpInfo.RegClass != -1 ? RI.getRegClass(OpInfo.RegClass) : nullptr;
4654 if (!MO)
4655 MO = &MI.getOperand(OpIdx);
4656
4657 int ConstantBusLimit = ST.getConstantBusLimit(MI.getOpcode());
4658 int VOP3LiteralLimit = ST.hasVOP3Literal() ? 1 : 0;
4659 if (isVALU(MI) && usesConstantBus(MRI, *MO, OpInfo)) {
4660 if (isVOP3(MI) && isLiteralConstantLike(*MO, OpInfo) && !VOP3LiteralLimit--)
4661 return false;
4662
4663 SmallDenseSet<RegSubRegPair> SGPRsUsed;
4664 if (MO->isReg())
4665 SGPRsUsed.insert(RegSubRegPair(MO->getReg(), MO->getSubReg()));
4666
4667 for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
4668 if (i == OpIdx)
4669 continue;
4670 const MachineOperand &Op = MI.getOperand(i);
4671 if (Op.isReg()) {
4672 RegSubRegPair SGPR(Op.getReg(), Op.getSubReg());
4673 if (!SGPRsUsed.count(SGPR) &&
4674 usesConstantBus(MRI, Op, InstDesc.OpInfo[i])) {
4675 if (--ConstantBusLimit <= 0)
4676 return false;
4677 SGPRsUsed.insert(SGPR);
4678 }
4679 } else if (InstDesc.OpInfo[i].OperandType == AMDGPU::OPERAND_KIMM32) {
4680 if (--ConstantBusLimit <= 0)
4681 return false;
4682 } else if (isVOP3(MI) && AMDGPU::isSISrcOperand(InstDesc, i) &&
4683 isLiteralConstantLike(Op, InstDesc.OpInfo[i])) {
4684 if (!VOP3LiteralLimit--)
4685 return false;
4686 if (--ConstantBusLimit <= 0)
4687 return false;
4688 }
4689 }
4690 }
4691
4692 if (MO->isReg()) {
4693 assert(DefinedRC)(static_cast <bool> (DefinedRC) ? void (0) : __assert_fail
("DefinedRC", "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 4693, __extension__ __PRETTY_FUNCTION__))
;
4694 if (!isLegalRegOperand(MRI, OpInfo, *MO))
4695 return false;
4696 bool IsAGPR = RI.isAGPR(MRI, MO->getReg());
4697 if (IsAGPR && !ST.hasMAIInsts())
4698 return false;
4699 unsigned Opc = MI.getOpcode();
4700 if (IsAGPR &&
4701 (!ST.hasGFX90AInsts() || !MRI.reservedRegsFrozen()) &&
4702 (MI.mayLoad() || MI.mayStore() || isDS(Opc) || isMIMG(Opc)))
4703 return false;
4704 // Atomics should have both vdst and vdata either vgpr or agpr.
4705 const int VDstIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst);
4706 const int DataIdx = AMDGPU::getNamedOperandIdx(Opc,
4707 isDS(Opc) ? AMDGPU::OpName::data0 : AMDGPU::OpName::vdata);
4708 if ((int)OpIdx == VDstIdx && DataIdx != -1 &&
4709 MI.getOperand(DataIdx).isReg() &&
4710 RI.isAGPR(MRI, MI.getOperand(DataIdx).getReg()) != IsAGPR)
4711 return false;
4712 if ((int)OpIdx == DataIdx) {
4713 if (VDstIdx != -1 &&
4714 RI.isAGPR(MRI, MI.getOperand(VDstIdx).getReg()) != IsAGPR)
4715 return false;
4716 // DS instructions with 2 src operands also must have tied RC.
4717 const int Data1Idx = AMDGPU::getNamedOperandIdx(Opc,
4718 AMDGPU::OpName::data1);
4719 if (Data1Idx != -1 && MI.getOperand(Data1Idx).isReg() &&
4720 RI.isAGPR(MRI, MI.getOperand(Data1Idx).getReg()) != IsAGPR)
4721 return false;
4722 }
4723 if (Opc == AMDGPU::V_ACCVGPR_WRITE_B32_e64 &&
4724 (int)OpIdx == AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0) &&
4725 RI.isSGPRReg(MRI, MO->getReg()))
4726 return false;
4727 return true;
4728 }
4729
4730 // Handle non-register types that are treated like immediates.
4731 assert(MO->isImm() || MO->isTargetIndex() || MO->isFI() || MO->isGlobal())(static_cast <bool> (MO->isImm() || MO->isTargetIndex
() || MO->isFI() || MO->isGlobal()) ? void (0) : __assert_fail
("MO->isImm() || MO->isTargetIndex() || MO->isFI() || MO->isGlobal()"
, "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 4731, __extension__ __PRETTY_FUNCTION__))
;
4732
4733 if (!DefinedRC) {
4734 // This operand expects an immediate.
4735 return true;
4736 }
4737
4738 return isImmOperandLegal(MI, OpIdx, *MO);
4739}
4740
4741void SIInstrInfo::legalizeOperandsVOP2(MachineRegisterInfo &MRI,
4742 MachineInstr &MI) const {
4743 unsigned Opc = MI.getOpcode();
4744 const MCInstrDesc &InstrDesc = get(Opc);
4745
4746 int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0);
4747 MachineOperand &Src0 = MI.getOperand(Src0Idx);
4748
4749 int Src1Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1);
4750 MachineOperand &Src1 = MI.getOperand(Src1Idx);
4751
4752 // If there is an implicit SGPR use such as VCC use for v_addc_u32/v_subb_u32
4753 // we need to only have one constant bus use before GFX10.
4754 bool HasImplicitSGPR = findImplicitSGPRRead(MI) != AMDGPU::NoRegister;
4755 if (HasImplicitSGPR && ST.getConstantBusLimit(Opc) <= 1 &&
4756 Src0.isReg() && (RI.isSGPRReg(MRI, Src0.getReg()) ||
4757 isLiteralConstantLike(Src0, InstrDesc.OpInfo[Src0Idx])))
4758 legalizeOpWithMove(MI, Src0Idx);
4759
4760 // Special case: V_WRITELANE_B32 accepts only immediate or SGPR operands for
4761 // both the value to write (src0) and lane select (src1). Fix up non-SGPR
4762 // src0/src1 with V_READFIRSTLANE.
4763 if (Opc == AMDGPU::V_WRITELANE_B32) {
4764 const DebugLoc &DL = MI.getDebugLoc();
4765 if (Src0.isReg() && RI.isVGPR(MRI, Src0.getReg())) {
4766 Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
4767 BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg)
4768 .add(Src0);
4769 Src0.ChangeToRegister(Reg, false);
4770 }
4771 if (Src1.isReg() && RI.isVGPR(MRI, Src1.getReg())) {
4772 Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
4773 const DebugLoc &DL = MI.getDebugLoc();
4774 BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg)
4775 .add(Src1);
4776 Src1.ChangeToRegister(Reg, false);
4777 }
4778 return;
4779 }
4780
4781 // No VOP2 instructions support AGPRs.
4782 if (Src0.isReg() && RI.isAGPR(MRI, Src0.getReg()))
4783 legalizeOpWithMove(MI, Src0Idx);
4784
4785 if (Src1.isReg() && RI.isAGPR(MRI, Src1.getReg()))
4786 legalizeOpWithMove(MI, Src1Idx);
4787
4788 // VOP2 src0 instructions support all operand types, so we don't need to check
4789 // their legality. If src1 is already legal, we don't need to do anything.
4790 if (isLegalRegOperand(MRI, InstrDesc.OpInfo[Src1Idx], Src1))
4791 return;
4792
4793 // Special case: V_READLANE_B32 accepts only immediate or SGPR operands for
4794 // lane select. Fix up using V_READFIRSTLANE, since we assume that the lane
4795 // select is uniform.
4796 if (Opc == AMDGPU::V_READLANE_B32 && Src1.isReg() &&
4797 RI.isVGPR(MRI, Src1.getReg())) {
4798 Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
4799 const DebugLoc &DL = MI.getDebugLoc();
4800 BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg)
4801 .add(Src1);
4802 Src1.ChangeToRegister(Reg, false);
4803 return;
4804 }
4805
4806 // We do not use commuteInstruction here because it is too aggressive and will
4807 // commute if it is possible. We only want to commute here if it improves
4808 // legality. This can be called a fairly large number of times so don't waste
4809 // compile time pointlessly swapping and checking legality again.
4810 if (HasImplicitSGPR || !MI.isCommutable()) {
4811 legalizeOpWithMove(MI, Src1Idx);
4812 return;
4813 }
4814
4815 // If src0 can be used as src1, commuting will make the operands legal.
4816 // Otherwise we have to give up and insert a move.
4817 //
4818 // TODO: Other immediate-like operand kinds could be commuted if there was a
4819 // MachineOperand::ChangeTo* for them.
4820 if ((!Src1.isImm() && !Src1.isReg()) ||
4821 !isLegalRegOperand(MRI, InstrDesc.OpInfo[Src1Idx], Src0)) {
4822 legalizeOpWithMove(MI, Src1Idx);
4823 return;
4824 }
4825
4826 int CommutedOpc = commuteOpcode(MI);
4827 if (CommutedOpc == -1) {
4828 legalizeOpWithMove(MI, Src1Idx);
4829 return;
4830 }
4831
4832 MI.setDesc(get(CommutedOpc));
4833
4834 Register Src0Reg = Src0.getReg();
4835 unsigned Src0SubReg = Src0.getSubReg();
4836 bool Src0Kill = Src0.isKill();
4837
4838 if (Src1.isImm())
4839 Src0.ChangeToImmediate(Src1.getImm());
4840 else if (Src1.isReg()) {
4841 Src0.ChangeToRegister(Src1.getReg(), false, false, Src1.isKill());
4842 Src0.setSubReg(Src1.getSubReg());
4843 } else
4844 llvm_unreachable("Should only have register or immediate operands")::llvm::llvm_unreachable_internal("Should only have register or immediate operands"
, "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 4844)
;
4845
4846 Src1.ChangeToRegister(Src0Reg, false, false, Src0Kill);
4847 Src1.setSubReg(Src0SubReg);
4848 fixImplicitOperands(MI);
4849}
4850
4851// Legalize VOP3 operands. All operand types are supported for any operand
4852// but only one literal constant and only starting from GFX10.
4853void SIInstrInfo::legalizeOperandsVOP3(MachineRegisterInfo &MRI,
4854 MachineInstr &MI) const {
4855 unsigned Opc = MI.getOpcode();
4856
4857 int VOP3Idx[3] = {
4858 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0),
4859 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1),
4860 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2)
4861 };
4862
4863 if (Opc == AMDGPU::V_PERMLANE16_B32_e64 ||
4864 Opc == AMDGPU::V_PERMLANEX16_B32_e64) {
4865 // src1 and src2 must be scalar
4866 MachineOperand &Src1 = MI.getOperand(VOP3Idx[1]);
4867 MachineOperand &Src2 = MI.getOperand(VOP3Idx[2]);
4868 const DebugLoc &DL = MI.getDebugLoc();
4869 if (Src1.isReg() && !RI.isSGPRClass(MRI.getRegClass(Src1.getReg()))) {
4870 Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
4871 BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg)
4872 .add(Src1);
4873 Src1.ChangeToRegister(Reg, false);
4874 }
4875 if (Src2.isReg() && !RI.isSGPRClass(MRI.getRegClass(Src2.getReg()))) {
4876 Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
4877 BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg)
4878 .add(Src2);
4879 Src2.ChangeToRegister(Reg, false);
4880 }
4881 }
4882
4883 // Find the one SGPR operand we are allowed to use.
4884 int ConstantBusLimit = ST.getConstantBusLimit(Opc);
4885 int LiteralLimit = ST.hasVOP3Literal() ? 1 : 0;
4886 SmallDenseSet<unsigned> SGPRsUsed;
4887 Register SGPRReg = findUsedSGPR(MI, VOP3Idx);
4888 if (SGPRReg != AMDGPU::NoRegister) {
4889 SGPRsUsed.insert(SGPRReg);
4890 --ConstantBusLimit;
4891 }
4892
4893 for (unsigned i = 0; i < 3; ++i) {
4894 int Idx = VOP3Idx[i];
4895 if (Idx == -1)
4896 break;
4897 MachineOperand &MO = MI.getOperand(Idx);
4898
4899 if (!MO.isReg()) {
4900 if (!isLiteralConstantLike(MO, get(Opc).OpInfo[Idx]))
4901 continue;
4902
4903 if (LiteralLimit > 0 && ConstantBusLimit > 0) {
4904 --LiteralLimit;
4905 --ConstantBusLimit;
4906 continue;
4907 }
4908
4909 --LiteralLimit;
4910 --ConstantBusLimit;
4911 legalizeOpWithMove(MI, Idx);
4912 continue;
4913 }
4914
4915 if (RI.hasAGPRs(MRI.getRegClass(MO.getReg())) &&
4916 !isOperandLegal(MI, Idx, &MO)) {
4917 legalizeOpWithMove(MI, Idx);
4918 continue;
4919 }
4920
4921 if (!RI.isSGPRClass(MRI.getRegClass(MO.getReg())))
4922 continue; // VGPRs are legal
4923
4924 // We can use one SGPR in each VOP3 instruction prior to GFX10
4925 // and two starting from GFX10.
4926 if (SGPRsUsed.count(MO.getReg()))
4927 continue;
4928 if (ConstantBusLimit > 0) {
4929 SGPRsUsed.insert(MO.getReg());
4930 --ConstantBusLimit;
4931 continue;
4932 }
4933
4934 // If we make it this far, then the operand is not legal and we must
4935 // legalize it.
4936 legalizeOpWithMove(MI, Idx);
4937 }
4938}
4939
4940Register SIInstrInfo::readlaneVGPRToSGPR(Register SrcReg, MachineInstr &UseMI,
4941 MachineRegisterInfo &MRI) const {
4942 const TargetRegisterClass *VRC = MRI.getRegClass(SrcReg);
4943 const TargetRegisterClass *SRC = RI.getEquivalentSGPRClass(VRC);
4944 Register DstReg = MRI.createVirtualRegister(SRC);
4945 unsigned SubRegs = RI.getRegSizeInBits(*VRC) / 32;
4946
4947 if (RI.hasAGPRs(VRC)) {
4948 VRC = RI.getEquivalentVGPRClass(VRC);
4949 Register NewSrcReg = MRI.createVirtualRegister(VRC);
4950 BuildMI(*UseMI.getParent(), UseMI, UseMI.getDebugLoc(),
4951 get(TargetOpcode::COPY), NewSrcReg)
4952 .addReg(SrcReg);
4953 SrcReg = NewSrcReg;
4954 }
4955
4956 if (SubRegs == 1) {
4957 BuildMI(*UseMI.getParent(), UseMI, UseMI.getDebugLoc(),
4958 get(AMDGPU::V_READFIRSTLANE_B32), DstReg)
4959 .addReg(SrcReg);
4960 return DstReg;
4961 }
4962
4963 SmallVector<unsigned, 8> SRegs;
4964 for (unsigned i = 0; i < SubRegs; ++i) {
4965 Register SGPR = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
4966 BuildMI(*UseMI.getParent(), UseMI, UseMI.getDebugLoc(),
4967 get(AMDGPU::V_READFIRSTLANE_B32), SGPR)
4968 .addReg(SrcReg, 0, RI.getSubRegFromChannel(i));
4969 SRegs.push_back(SGPR);
4970 }
4971
4972 MachineInstrBuilder MIB =
4973 BuildMI(*UseMI.getParent(), UseMI, UseMI.getDebugLoc(),
4974 get(AMDGPU::REG_SEQUENCE), DstReg);
4975 for (unsigned i = 0; i < SubRegs; ++i) {
4976 MIB.addReg(SRegs[i]);
4977 MIB.addImm(RI.getSubRegFromChannel(i));
4978 }
4979 return DstReg;
4980}
4981
4982void SIInstrInfo::legalizeOperandsSMRD(MachineRegisterInfo &MRI,
4983 MachineInstr &MI) const {
4984
4985 // If the pointer is store in VGPRs, then we need to move them to
4986 // SGPRs using v_readfirstlane. This is safe because we only select
4987 // loads with uniform pointers to SMRD instruction so we know the
4988 // pointer value is uniform.
4989 MachineOperand *SBase = getNamedOperand(MI, AMDGPU::OpName::sbase);
4990 if (SBase && !RI.isSGPRClass(MRI.getRegClass(SBase->getReg()))) {
4991 Register SGPR = readlaneVGPRToSGPR(SBase->getReg(), MI, MRI);
4992 SBase->setReg(SGPR);
4993 }
4994 MachineOperand *SOff = getNamedOperand(MI, AMDGPU::OpName::soff);
4995 if (SOff && !RI.isSGPRClass(MRI.getRegClass(SOff->getReg()))) {
4996 Register SGPR = readlaneVGPRToSGPR(SOff->getReg(), MI, MRI);
4997 SOff->setReg(SGPR);
4998 }
4999}
5000
5001bool SIInstrInfo::moveFlatAddrToVGPR(MachineInstr &Inst) const {
5002 unsigned Opc = Inst.getOpcode();
5003 int OldSAddrIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::saddr);
5004 if (OldSAddrIdx < 0)
5005 return false;
5006
5007 assert(isSegmentSpecificFLAT(Inst))(static_cast <bool> (isSegmentSpecificFLAT(Inst)) ? void
(0) : __assert_fail ("isSegmentSpecificFLAT(Inst)", "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 5007, __extension__ __PRETTY_FUNCTION__))
;
5008
5009 int NewOpc = AMDGPU::getGlobalVaddrOp(Opc);
5010 if (NewOpc < 0)
5011 NewOpc = AMDGPU::getFlatScratchInstSVfromSS(Opc);
5012 if (NewOpc < 0)
5013 return false;
5014
5015 MachineRegisterInfo &MRI = Inst.getMF()->getRegInfo();
5016 MachineOperand &SAddr = Inst.getOperand(OldSAddrIdx);
5017 if (RI.isSGPRReg(MRI, SAddr.getReg()))
5018 return false;
5019
5020 int NewVAddrIdx = AMDGPU::getNamedOperandIdx(NewOpc, AMDGPU::OpName::vaddr);
5021 if (NewVAddrIdx < 0)
5022 return false;
5023
5024 int OldVAddrIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vaddr);
5025
5026 // Check vaddr, it shall be zero or absent.
5027 MachineInstr *VAddrDef = nullptr;
5028 if (OldVAddrIdx >= 0) {
5029 MachineOperand &VAddr = Inst.getOperand(OldVAddrIdx);
5030 VAddrDef = MRI.getUniqueVRegDef(VAddr.getReg());
5031 if (!VAddrDef || VAddrDef->getOpcode() != AMDGPU::V_MOV_B32_e32 ||
5032 !VAddrDef->getOperand(1).isImm() ||
5033 VAddrDef->getOperand(1).getImm() != 0)
5034 return false;
5035 }
5036
5037 const MCInstrDesc &NewDesc = get(NewOpc);
5038 Inst.setDesc(NewDesc);
5039
5040 // Callers expect interator to be valid after this call, so modify the
5041 // instruction in place.
5042 if (OldVAddrIdx == NewVAddrIdx) {
5043 MachineOperand &NewVAddr = Inst.getOperand(NewVAddrIdx);
5044 // Clear use list from the old vaddr holding a zero register.
5045 MRI.removeRegOperandFromUseList(&NewVAddr);
5046 MRI.moveOperands(&NewVAddr, &SAddr, 1);
5047 Inst.RemoveOperand(OldSAddrIdx);
5048 // Update the use list with the pointer we have just moved from vaddr to
5049 // saddr poisition. Otherwise new vaddr will be missing from the use list.
5050 MRI.removeRegOperandFromUseList(&NewVAddr);
5051 MRI.addRegOperandToUseList(&NewVAddr);
5052 } else {
5053 assert(OldSAddrIdx == NewVAddrIdx)(static_cast <bool> (OldSAddrIdx == NewVAddrIdx) ? void
(0) : __assert_fail ("OldSAddrIdx == NewVAddrIdx", "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 5053, __extension__ __PRETTY_FUNCTION__))
;
5054
5055 if (OldVAddrIdx >= 0) {
5056 int NewVDstIn = AMDGPU::getNamedOperandIdx(NewOpc,
5057 AMDGPU::OpName::vdst_in);
5058
5059 // RemoveOperand doesn't try to fixup tied operand indexes at it goes, so
5060 // it asserts. Untie the operands for now and retie them afterwards.
5061 if (NewVDstIn != -1) {
5062 int OldVDstIn = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst_in);
5063 Inst.untieRegOperand(OldVDstIn);
5064 }
5065
5066 Inst.RemoveOperand(OldVAddrIdx);
5067
5068 if (NewVDstIn != -1) {
5069 int NewVDst = AMDGPU::getNamedOperandIdx(NewOpc, AMDGPU::OpName::vdst);
5070 Inst.tieOperands(NewVDst, NewVDstIn);
5071 }
5072 }
5073 }
5074
5075 if (VAddrDef && MRI.use_nodbg_empty(VAddrDef->getOperand(0).getReg()))
5076 VAddrDef->eraseFromParent();
5077
5078 return true;
5079}
5080
5081// FIXME: Remove this when SelectionDAG is obsoleted.
5082void SIInstrInfo::legalizeOperandsFLAT(MachineRegisterInfo &MRI,
5083 MachineInstr &MI) const {
5084 if (!isSegmentSpecificFLAT(MI))
5085 return;
5086
5087 // Fixup SGPR operands in VGPRs. We only select these when the DAG divergence
5088 // thinks they are uniform, so a readfirstlane should be valid.
5089 MachineOperand *SAddr = getNamedOperand(MI, AMDGPU::OpName::saddr);
5090 if (!SAddr || RI.isSGPRClass(MRI.getRegClass(SAddr->getReg())))
5091 return;
5092
5093 if (moveFlatAddrToVGPR(MI))
5094 return;
5095
5096 Register ToSGPR = readlaneVGPRToSGPR(SAddr->getReg(), MI, MRI);
5097 SAddr->setReg(ToSGPR);
5098}
5099
5100void SIInstrInfo::legalizeGenericOperand(MachineBasicBlock &InsertMBB,
5101 MachineBasicBlock::iterator I,
5102 const TargetRegisterClass *DstRC,
5103 MachineOperand &Op,
5104 MachineRegisterInfo &MRI,
5105 const DebugLoc &DL) const {
5106 Register OpReg = Op.getReg();
5107 unsigned OpSubReg = Op.getSubReg();
5108
5109 const TargetRegisterClass *OpRC = RI.getSubClassWithSubReg(
5110 RI.getRegClassForReg(MRI, OpReg), OpSubReg);
5111
5112 // Check if operand is already the correct register class.
5113 if (DstRC == OpRC)
5114 return;
5115
5116 Register DstReg = MRI.createVirtualRegister(DstRC);
5117 MachineInstr *Copy =
5118 BuildMI(InsertMBB, I, DL, get(AMDGPU::COPY), DstReg).add(Op);
5119
5120 Op.setReg(DstReg);
5121 Op.setSubReg(0);
5122
5123 MachineInstr *Def = MRI.getVRegDef(OpReg);
5124 if (!Def)
5125 return;
5126
5127 // Try to eliminate the copy if it is copying an immediate value.
5128 if (Def->isMoveImmediate() && DstRC != &AMDGPU::VReg_1RegClass)
5129 FoldImmediate(*Copy, *Def, OpReg, &MRI);
5130
5131 bool ImpDef = Def->isImplicitDef();
5132 while (!ImpDef && Def && Def->isCopy()) {
5133 if (Def->getOperand(1).getReg().isPhysical())
5134 break;
5135 Def = MRI.getUniqueVRegDef(Def->getOperand(1).getReg());
5136 ImpDef = Def && Def->isImplicitDef();
5137 }
5138 if (!RI.isSGPRClass(DstRC) && !Copy->readsRegister(AMDGPU::EXEC, &RI) &&
5139 !ImpDef)
5140 Copy->addOperand(MachineOperand::CreateReg(AMDGPU::EXEC, false, true));
5141}
5142
5143// Emit the actual waterfall loop, executing the wrapped instruction for each
5144// unique value of \p Rsrc across all lanes. In the best case we execute 1
5145// iteration, in the worst case we execute 64 (once per lane).
5146static void
5147emitLoadSRsrcFromVGPRLoop(const SIInstrInfo &TII, MachineRegisterInfo &MRI,
5148 MachineBasicBlock &OrigBB, MachineBasicBlock &LoopBB,
5149 const DebugLoc &DL, MachineOperand &Rsrc) {
5150 MachineFunction &MF = *OrigBB.getParent();
5151 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
5152 const SIRegisterInfo *TRI = ST.getRegisterInfo();
5153 unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
5154 unsigned SaveExecOpc =
5155 ST.isWave32() ? AMDGPU::S_AND_SAVEEXEC_B32 : AMDGPU::S_AND_SAVEEXEC_B64;
5156 unsigned XorTermOpc =
5157 ST.isWave32() ? AMDGPU::S_XOR_B32_term : AMDGPU::S_XOR_B64_term;
5158 unsigned AndOpc =
5159 ST.isWave32() ? AMDGPU::S_AND_B32 : AMDGPU::S_AND_B64;
5160 const auto *BoolXExecRC = TRI->getRegClass(AMDGPU::SReg_1_XEXECRegClassID);
5161
5162 MachineBasicBlock::iterator I = LoopBB.begin();
5163
5164 SmallVector<Register, 8> ReadlanePieces;
5165 Register CondReg = AMDGPU::NoRegister;
5166
5167 Register VRsrc = Rsrc.getReg();
5168 unsigned VRsrcUndef = getUndefRegState(Rsrc.isUndef());
5169
5170 unsigned RegSize = TRI->getRegSizeInBits(Rsrc.getReg(), MRI);
5171 unsigned NumSubRegs = RegSize / 32;
5172 assert(NumSubRegs % 2 == 0 && NumSubRegs <= 32 && "Unhandled register size")(static_cast <bool> (NumSubRegs % 2 == 0 && NumSubRegs
<= 32 && "Unhandled register size") ? void (0) : __assert_fail
("NumSubRegs % 2 == 0 && NumSubRegs <= 32 && \"Unhandled register size\""
, "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 5172, __extension__ __PRETTY_FUNCTION__))
;
5173
5174 for (unsigned Idx = 0; Idx < NumSubRegs; Idx += 2) {
5175
5176 Register CurRegLo = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
5177 Register CurRegHi = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
5178
5179 // Read the next variant <- also loop target.
5180 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), CurRegLo)
5181 .addReg(VRsrc, VRsrcUndef, TRI->getSubRegFromChannel(Idx));
5182
5183 // Read the next variant <- also loop target.
5184 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), CurRegHi)
5185 .addReg(VRsrc, VRsrcUndef, TRI->getSubRegFromChannel(Idx + 1));
5186
5187 ReadlanePieces.push_back(CurRegLo);
5188 ReadlanePieces.push_back(CurRegHi);
5189
5190 // Comparison is to be done as 64-bit.
5191 Register CurReg = MRI.createVirtualRegister(&AMDGPU::SGPR_64RegClass);
5192 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::REG_SEQUENCE), CurReg)
5193 .addReg(CurRegLo)
5194 .addImm(AMDGPU::sub0)
5195 .addReg(CurRegHi)
5196 .addImm(AMDGPU::sub1);
5197
5198 Register NewCondReg = MRI.createVirtualRegister(BoolXExecRC);
5199 auto Cmp =
5200 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_CMP_EQ_U64_e64), NewCondReg)
5201 .addReg(CurReg);
5202 if (NumSubRegs <= 2)
5203 Cmp.addReg(VRsrc);
5204 else
5205 Cmp.addReg(VRsrc, VRsrcUndef, TRI->getSubRegFromChannel(Idx, 2));
5206
5207 // Combine the comparision results with AND.
5208 if (CondReg == AMDGPU::NoRegister) // First.
5209 CondReg = NewCondReg;
5210 else { // If not the first, we create an AND.
5211 Register AndReg = MRI.createVirtualRegister(BoolXExecRC);
5212 BuildMI(LoopBB, I, DL, TII.get(AndOpc), AndReg)
5213 .addReg(CondReg)
5214 .addReg(NewCondReg);
5215 CondReg = AndReg;
5216 }
5217 } // End for loop.
5218
5219 auto SRsrcRC = TRI->getEquivalentSGPRClass(MRI.getRegClass(VRsrc));
5220 Register SRsrc = MRI.createVirtualRegister(SRsrcRC);
5221
5222 // Build scalar Rsrc.
5223 auto Merge = BuildMI(LoopBB, I, DL, TII.get(AMDGPU::REG_SEQUENCE), SRsrc);
5224 unsigned Channel = 0;
5225 for (Register Piece : ReadlanePieces) {
5226 Merge.addReg(Piece)
5227 .addImm(TRI->getSubRegFromChannel(Channel++));
5228 }
5229
5230 // Update Rsrc operand to use the SGPR Rsrc.
5231 Rsrc.setReg(SRsrc);
5232 Rsrc.setIsKill(true);
5233
5234 Register SaveExec = MRI.createVirtualRegister(BoolXExecRC);
5235 MRI.setSimpleHint(SaveExec, CondReg);
5236
5237 // Update EXEC to matching lanes, saving original to SaveExec.
5238 BuildMI(LoopBB, I, DL, TII.get(SaveExecOpc), SaveExec)
5239 .addReg(CondReg, RegState::Kill);
5240
5241 // The original instruction is here; we insert the terminators after it.
5242 I = LoopBB.end();
5243
5244 // Update EXEC, switch all done bits to 0 and all todo bits to 1.
5245 BuildMI(LoopBB, I, DL, TII.get(XorTermOpc), Exec)
5246 .addReg(Exec)
5247 .addReg(SaveExec);
5248
5249 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::S_CBRANCH_EXECNZ)).addMBB(&LoopBB);
5250}
5251
5252// Build a waterfall loop around \p MI, replacing the VGPR \p Rsrc register
5253// with SGPRs by iterating over all unique values across all lanes.
5254// Returns the loop basic block that now contains \p MI.
5255static MachineBasicBlock *
5256loadSRsrcFromVGPR(const SIInstrInfo &TII, MachineInstr &MI,
5257 MachineOperand &Rsrc, MachineDominatorTree *MDT,
5258 MachineBasicBlock::iterator Begin = nullptr,
5259 MachineBasicBlock::iterator End = nullptr) {
5260 MachineBasicBlock &MBB = *MI.getParent();
5261 MachineFunction &MF = *MBB.getParent();
5262 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
5263 const SIRegisterInfo *TRI = ST.getRegisterInfo();
5264 MachineRegisterInfo &MRI = MF.getRegInfo();
5265 if (!Begin.isValid())
5266 Begin = &MI;
5267 if (!End.isValid()) {
5268 End = &MI;
5269 ++End;
5270 }
5271 const DebugLoc &DL = MI.getDebugLoc();
5272 unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
5273 unsigned MovExecOpc = ST.isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64;
5274 const auto *BoolXExecRC = TRI->getRegClass(AMDGPU::SReg_1_XEXECRegClassID);
5275
5276 Register SaveExec = MRI.createVirtualRegister(BoolXExecRC);
5277
5278 // Save the EXEC mask
5279 BuildMI(MBB, Begin, DL, TII.get(MovExecOpc), SaveExec).addReg(Exec);
5280
5281 // Killed uses in the instruction we are waterfalling around will be
5282 // incorrect due to the added control-flow.
5283 MachineBasicBlock::iterator AfterMI = MI;
5284 ++AfterMI;
5285 for (auto I = Begin; I != AfterMI; I++) {
5286 for (auto &MO : I->uses()) {
5287 if (MO.isReg() && MO.isUse()) {
5288 MRI.clearKillFlags(MO.getReg());
5289 }
5290 }
5291 }
5292
5293 // To insert the loop we need to split the block. Move everything after this
5294 // point to a new block, and insert a new empty block between the two.
5295 MachineBasicBlock *LoopBB = MF.CreateMachineBasicBlock();
5296 MachineBasicBlock *RemainderBB = MF.CreateMachineBasicBlock();
5297 MachineFunction::iterator MBBI(MBB);
5298 ++MBBI;
5299
5300 MF.insert(MBBI, LoopBB);
5301 MF.insert(MBBI, RemainderBB);
5302
5303 LoopBB->addSuccessor(LoopBB);
5304 LoopBB->addSuccessor(RemainderBB);
5305
5306 // Move Begin to MI to the LoopBB, and the remainder of the block to
5307 // RemainderBB.
5308 RemainderBB->transferSuccessorsAndUpdatePHIs(&MBB);
5309 RemainderBB->splice(RemainderBB->begin(), &MBB, End, MBB.end());
5310 LoopBB->splice(LoopBB->begin(), &MBB, Begin, MBB.end());
5311
5312 MBB.addSuccessor(LoopBB);
5313
5314 // Update dominators. We know that MBB immediately dominates LoopBB, that
5315 // LoopBB immediately dominates RemainderBB, and that RemainderBB immediately
5316 // dominates all of the successors transferred to it from MBB that MBB used
5317 // to properly dominate.
5318 if (MDT) {
5319 MDT->addNewBlock(LoopBB, &MBB);
5320 MDT->addNewBlock(RemainderBB, LoopBB);
5321 for (auto &Succ : RemainderBB->successors()) {
5322 if (MDT->properlyDominates(&MBB, Succ)) {
5323 MDT->changeImmediateDominator(Succ, RemainderBB);
5324 }
5325 }
5326 }
5327
5328 emitLoadSRsrcFromVGPRLoop(TII, MRI, MBB, *LoopBB, DL, Rsrc);
5329
5330 // Restore the EXEC mask
5331 MachineBasicBlock::iterator First = RemainderBB->begin();
5332 BuildMI(*RemainderBB, First, DL, TII.get(MovExecOpc), Exec).addReg(SaveExec);
5333 return LoopBB;
5334}
5335
5336// Extract pointer from Rsrc and return a zero-value Rsrc replacement.
5337static std::tuple<unsigned, unsigned>
5338extractRsrcPtr(const SIInstrInfo &TII, MachineInstr &MI, MachineOperand &Rsrc) {
5339 MachineBasicBlock &MBB = *MI.getParent();
5340 MachineFunction &MF = *MBB.getParent();
5341 MachineRegisterInfo &MRI = MF.getRegInfo();
5342
5343 // Extract the ptr from the resource descriptor.
5344 unsigned RsrcPtr =
5345 TII.buildExtractSubReg(MI, MRI, Rsrc, &AMDGPU::VReg_128RegClass,
5346 AMDGPU::sub0_sub1, &AMDGPU::VReg_64RegClass);
5347
5348 // Create an empty resource descriptor
5349 Register Zero64 = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
5350 Register SRsrcFormatLo = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
5351 Register SRsrcFormatHi = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
5352 Register NewSRsrc = MRI.createVirtualRegister(&AMDGPU::SGPR_128RegClass);
5353 uint64_t RsrcDataFormat = TII.getDefaultRsrcDataFormat();
5354
5355 // Zero64 = 0
5356 BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::S_MOV_B64), Zero64)
5357 .addImm(0);
5358
5359 // SRsrcFormatLo = RSRC_DATA_FORMAT{31-0}
5360 BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::S_MOV_B32), SRsrcFormatLo)
5361 .addImm(RsrcDataFormat & 0xFFFFFFFF);
5362
5363 // SRsrcFormatHi = RSRC_DATA_FORMAT{63-32}
5364 BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::S_MOV_B32), SRsrcFormatHi)
5365 .addImm(RsrcDataFormat >> 32);
5366
5367 // NewSRsrc = {Zero64, SRsrcFormat}
5368 BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::REG_SEQUENCE), NewSRsrc)
5369 .addReg(Zero64)
5370 .addImm(AMDGPU::sub0_sub1)
5371 .addReg(SRsrcFormatLo)
5372 .addImm(AMDGPU::sub2)
5373 .addReg(SRsrcFormatHi)
5374 .addImm(AMDGPU::sub3);
5375
5376 return std::make_tuple(RsrcPtr, NewSRsrc);
5377}
5378
5379MachineBasicBlock *
5380SIInstrInfo::legalizeOperands(MachineInstr &MI,
5381 MachineDominatorTree *MDT) const {
5382 MachineFunction &MF = *MI.getParent()->getParent();
5383 MachineRegisterInfo &MRI = MF.getRegInfo();
5384 MachineBasicBlock *CreatedBB = nullptr;
5385
5386 // Legalize VOP2
5387 if (isVOP2(MI) || isVOPC(MI)) {
5388 legalizeOperandsVOP2(MRI, MI);
5389 return CreatedBB;
5390 }
5391
5392 // Legalize VOP3
5393 if (isVOP3(MI)) {
5394 legalizeOperandsVOP3(MRI, MI);
5395 return CreatedBB;
5396 }
5397
5398 // Legalize SMRD
5399 if (isSMRD(MI)) {
5400 legalizeOperandsSMRD(MRI, MI);
5401 return CreatedBB;
5402 }
5403
5404 // Legalize FLAT
5405 if (isFLAT(MI)) {
5406 legalizeOperandsFLAT(MRI, MI);
5407 return CreatedBB;
5408 }
5409
5410 // Legalize REG_SEQUENCE and PHI
5411 // The register class of the operands much be the same type as the register
5412 // class of the output.
5413 if (MI.getOpcode() == AMDGPU::PHI) {
5414 const TargetRegisterClass *RC = nullptr, *SRC = nullptr, *VRC = nullptr;
5415 for (unsigned i = 1, e = MI.getNumOperands(); i != e; i += 2) {
5416 if (!MI.getOperand(i).isReg() || !MI.getOperand(i).getReg().isVirtual())
5417 continue;
5418 const TargetRegisterClass *OpRC =
5419 MRI.getRegClass(MI.getOperand(i).getReg());
5420 if (RI.hasVectorRegisters(OpRC)) {
5421 VRC = OpRC;
5422 } else {
5423 SRC = OpRC;
5424 }
5425 }
5426
5427 // If any of the operands are VGPR registers, then they all most be
5428 // otherwise we will create illegal VGPR->SGPR copies when legalizing
5429 // them.
5430 if (VRC || !RI.isSGPRClass(getOpRegClass(MI, 0))) {
5431 if (!VRC) {
5432 assert(SRC)(static_cast <bool> (SRC) ? void (0) : __assert_fail ("SRC"
, "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 5432, __extension__ __PRETTY_FUNCTION__))
;
5433 if (getOpRegClass(MI, 0) == &AMDGPU::VReg_1RegClass) {
5434 VRC = &AMDGPU::VReg_1RegClass;
5435 } else
5436 VRC = RI.hasAGPRs(getOpRegClass(MI, 0))
5437 ? RI.getEquivalentAGPRClass(SRC)
5438 : RI.getEquivalentVGPRClass(SRC);
5439 } else {
5440 VRC = RI.hasAGPRs(getOpRegClass(MI, 0))
5441 ? RI.getEquivalentAGPRClass(VRC)
5442 : RI.getEquivalentVGPRClass(VRC);
5443 }
5444 RC = VRC;
5445 } else {
5446 RC = SRC;
5447 }
5448
5449 // Update all the operands so they have the same type.
5450 for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) {
5451 MachineOperand &Op = MI.getOperand(I);
5452 if (!Op.isReg() || !Op.getReg().isVirtual())
5453 continue;
5454
5455 // MI is a PHI instruction.
5456 MachineBasicBlock *InsertBB = MI.getOperand(I + 1).getMBB();
5457 MachineBasicBlock::iterator Insert = InsertBB->getFirstTerminator();
5458
5459 // Avoid creating no-op copies with the same src and dst reg class. These
5460 // confuse some of the machine passes.
5461 legalizeGenericOperand(*InsertBB, Insert, RC, Op, MRI, MI.getDebugLoc());
5462 }
5463 }
5464
5465 // REG_SEQUENCE doesn't really require operand legalization, but if one has a
5466 // VGPR dest type and SGPR sources, insert copies so all operands are
5467 // VGPRs. This seems to help operand folding / the register coalescer.
5468 if (MI.getOpcode() == AMDGPU::REG_SEQUENCE) {
5469 MachineBasicBlock *MBB = MI.getParent();
5470 const TargetRegisterClass *DstRC = getOpRegClass(MI, 0);
5471 if (RI.hasVGPRs(DstRC)) {
5472 // Update all the operands so they are VGPR register classes. These may
5473 // not be the same register class because REG_SEQUENCE supports mixing
5474 // subregister index types e.g. sub0_sub1 + sub2 + sub3
5475 for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) {
5476 MachineOperand &Op = MI.getOperand(I);
5477 if (!Op.isReg() || !Op.getReg().isVirtual())
5478 continue;
5479
5480 const TargetRegisterClass *OpRC = MRI.getRegClass(Op.getReg());
5481 const TargetRegisterClass *VRC = RI.getEquivalentVGPRClass(OpRC);
5482 if (VRC == OpRC)
5483 continue;
5484
5485 legalizeGenericOperand(*MBB, MI, VRC, Op, MRI, MI.getDebugLoc());
5486 Op.setIsKill();
5487 }
5488 }
5489
5490 return CreatedBB;
5491 }
5492
5493 // Legalize INSERT_SUBREG
5494 // src0 must have the same register class as dst
5495 if (MI.getOpcode() == AMDGPU::INSERT_SUBREG) {
5496 Register Dst = MI.getOperand(0).getReg();
5497 Register Src0 = MI.getOperand(1).getReg();
5498 const TargetRegisterClass *DstRC = MRI.getRegClass(Dst);
5499 const TargetRegisterClass *Src0RC = MRI.getRegClass(Src0);
5500 if (DstRC != Src0RC) {
5501 MachineBasicBlock *MBB = MI.getParent();
5502 MachineOperand &Op = MI.getOperand(1);
5503 legalizeGenericOperand(*MBB, MI, DstRC, Op, MRI, MI.getDebugLoc());
5504 }
5505 return CreatedBB;
5506 }
5507
5508 // Legalize SI_INIT_M0
5509 if (MI.getOpcode() == AMDGPU::SI_INIT_M0) {
5510 MachineOperand &Src = MI.getOperand(0);
5511 if (Src.isReg() && RI.hasVectorRegisters(MRI.getRegClass(Src.getReg())))
5512 Src.setReg(readlaneVGPRToSGPR(Src.getReg(), MI, MRI));
5513 return CreatedBB;
5514 }
5515
5516 // Legalize MIMG and MUBUF/MTBUF for shaders.
5517 //
5518 // Shaders only generate MUBUF/MTBUF instructions via intrinsics or via
5519 // scratch memory access. In both cases, the legalization never involves
5520 // conversion to the addr64 form.
5521 if (isMIMG(MI) || (AMDGPU::isGraphics(MF.getFunction().getCallingConv()) &&
5522 (isMUBUF(MI) || isMTBUF(MI)))) {
5523 MachineOperand *SRsrc = getNamedOperand(MI, AMDGPU::OpName::srsrc);
5524 if (SRsrc && !RI.isSGPRClass(MRI.getRegClass(SRsrc->getReg())))
5525 CreatedBB = loadSRsrcFromVGPR(*this, MI, *SRsrc, MDT);
5526
5527 MachineOperand *SSamp = getNamedOperand(MI, AMDGPU::OpName::ssamp);
5528 if (SSamp && !RI.isSGPRClass(MRI.getRegClass(SSamp->getReg())))
5529 CreatedBB = loadSRsrcFromVGPR(*this, MI, *SSamp, MDT);
5530
5531 return CreatedBB;
5532 }
5533
5534 // Legalize SI_CALL
5535 if (MI.getOpcode() == AMDGPU::SI_CALL_ISEL) {
5536 MachineOperand *Dest = &MI.getOperand(0);
5537 if (!RI.isSGPRClass(MRI.getRegClass(Dest->getReg()))) {
5538 // Move everything between ADJCALLSTACKUP and ADJCALLSTACKDOWN and
5539 // following copies, we also need to move copies from and to physical
5540 // registers into the loop block.
5541 unsigned FrameSetupOpcode = getCallFrameSetupOpcode();
5542 unsigned FrameDestroyOpcode = getCallFrameDestroyOpcode();
5543
5544 // Also move the copies to physical registers into the loop block
5545 MachineBasicBlock &MBB = *MI.getParent();
5546 MachineBasicBlock::iterator Start(&MI);
5547 while (Start->getOpcode() != FrameSetupOpcode)
5548 --Start;
5549 MachineBasicBlock::iterator End(&MI);
5550 while (End->getOpcode() != FrameDestroyOpcode)
5551 ++End;
5552 // Also include following copies of the return value
5553 ++End;
5554 while (End != MBB.end() && End->isCopy() && End->getOperand(1).isReg() &&
5555 MI.definesRegister(End->getOperand(1).getReg()))
5556 ++End;
5557 CreatedBB = loadSRsrcFromVGPR(*this, MI, *Dest, MDT, Start, End);
5558 }
5559 }
5560
5561 // Legalize MUBUF* instructions.
5562 int RsrcIdx =
5563 AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::srsrc);
5564 if (RsrcIdx != -1) {
5565 // We have an MUBUF instruction
5566 MachineOperand *Rsrc = &MI.getOperand(RsrcIdx);
5567 unsigned RsrcRC = get(MI.getOpcode()).OpInfo[RsrcIdx].RegClass;
5568 if (RI.getCommonSubClass(MRI.getRegClass(Rsrc->getReg()),
5569 RI.getRegClass(RsrcRC))) {
5570 // The operands are legal.
5571 // FIXME: We may need to legalize operands besided srsrc.
5572 return CreatedBB;
5573 }
5574
5575 // Legalize a VGPR Rsrc.
5576 //
5577 // If the instruction is _ADDR64, we can avoid a waterfall by extracting
5578 // the base pointer from the VGPR Rsrc, adding it to the VAddr, then using
5579 // a zero-value SRsrc.
5580 //
5581 // If the instruction is _OFFSET (both idxen and offen disabled), and we
5582 // support ADDR64 instructions, we can convert to ADDR64 and do the same as
5583 // above.
5584 //
5585 // Otherwise we are on non-ADDR64 hardware, and/or we have
5586 // idxen/offen/bothen and we fall back to a waterfall loop.
5587
5588 MachineBasicBlock &MBB = *MI.getParent();
5589
5590 MachineOperand *VAddr = getNamedOperand(MI, AMDGPU::OpName::vaddr);
5591 if (VAddr && AMDGPU::getIfAddr64Inst(MI.getOpcode()) != -1) {
5592 // This is already an ADDR64 instruction so we need to add the pointer
5593 // extracted from the resource descriptor to the current value of VAddr.
5594 Register NewVAddrLo = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
5595 Register NewVAddrHi = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
5596 Register NewVAddr = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass);
5597
5598 const auto *BoolXExecRC = RI.getRegClass(AMDGPU::SReg_1_XEXECRegClassID);
5599 Register CondReg0 = MRI.createVirtualRegister(BoolXExecRC);
5600 Register CondReg1 = MRI.createVirtualRegister(BoolXExecRC);
5601
5602 unsigned RsrcPtr, NewSRsrc;
5603 std::tie(RsrcPtr, NewSRsrc) = extractRsrcPtr(*this, MI, *Rsrc);
5604
5605 // NewVaddrLo = RsrcPtr:sub0 + VAddr:sub0
5606 const DebugLoc &DL = MI.getDebugLoc();
5607 BuildMI(MBB, MI, DL, get(AMDGPU::V_ADD_CO_U32_e64), NewVAddrLo)
5608 .addDef(CondReg0)
5609 .addReg(RsrcPtr, 0, AMDGPU::sub0)
5610 .addReg(VAddr->getReg(), 0, AMDGPU::sub0)
5611 .addImm(0);
5612
5613 // NewVaddrHi = RsrcPtr:sub1 + VAddr:sub1
5614 BuildMI(MBB, MI, DL, get(AMDGPU::V_ADDC_U32_e64), NewVAddrHi)
5615 .addDef(CondReg1, RegState::Dead)
5616 .addReg(RsrcPtr, 0, AMDGPU::sub1)
5617 .addReg(VAddr->getReg(), 0, AMDGPU::sub1)
5618 .addReg(CondReg0, RegState::Kill)
5619 .addImm(0);
5620
5621 // NewVaddr = {NewVaddrHi, NewVaddrLo}
5622 BuildMI(MBB, MI, MI.getDebugLoc(), get(AMDGPU::REG_SEQUENCE), NewVAddr)
5623 .addReg(NewVAddrLo)
5624 .addImm(AMDGPU::sub0)
5625 .addReg(NewVAddrHi)
5626 .addImm(AMDGPU::sub1);
5627
5628 VAddr->setReg(NewVAddr);
5629 Rsrc->setReg(NewSRsrc);
5630 } else if (!VAddr && ST.hasAddr64()) {
5631 // This instructions is the _OFFSET variant, so we need to convert it to
5632 // ADDR64.
5633 assert(ST.getGeneration() < AMDGPUSubtarget::VOLCANIC_ISLANDS &&(static_cast <bool> (ST.getGeneration() < AMDGPUSubtarget
::VOLCANIC_ISLANDS && "FIXME: Need to emit flat atomics here"
) ? void (0) : __assert_fail ("ST.getGeneration() < AMDGPUSubtarget::VOLCANIC_ISLANDS && \"FIXME: Need to emit flat atomics here\""
, "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 5634, __extension__ __PRETTY_FUNCTION__))
5634 "FIXME: Need to emit flat atomics here")(static_cast <bool> (ST.getGeneration() < AMDGPUSubtarget
::VOLCANIC_ISLANDS && "FIXME: Need to emit flat atomics here"
) ? void (0) : __assert_fail ("ST.getGeneration() < AMDGPUSubtarget::VOLCANIC_ISLANDS && \"FIXME: Need to emit flat atomics here\""
, "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 5634, __extension__ __PRETTY_FUNCTION__))
;
5635
5636 unsigned RsrcPtr, NewSRsrc;
5637 std::tie(RsrcPtr, NewSRsrc) = extractRsrcPtr(*this, MI, *Rsrc);
5638
5639 Register NewVAddr = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass);
5640 MachineOperand *VData = getNamedOperand(MI, AMDGPU::OpName::vdata);
5641 MachineOperand *Offset = getNamedOperand(MI, AMDGPU::OpName::offset);
5642 MachineOperand *SOffset = getNamedOperand(MI, AMDGPU::OpName::soffset);
5643 unsigned Addr64Opcode = AMDGPU::getAddr64Inst(MI.getOpcode());
5644
5645 // Atomics rith return have have an additional tied operand and are
5646 // missing some of the special bits.
5647 MachineOperand *VDataIn = getNamedOperand(MI, AMDGPU::OpName::vdata_in);
5648 MachineInstr *Addr64;
5649
5650 if (!VDataIn) {
5651 // Regular buffer load / store.
5652 MachineInstrBuilder MIB =
5653 BuildMI(MBB, MI, MI.getDebugLoc(), get(Addr64Opcode))
5654 .add(*VData)
5655 .addReg(NewVAddr)
5656 .addReg(NewSRsrc)
5657 .add(*SOffset)
5658 .add(*Offset);
5659
5660 if (const MachineOperand *CPol =
5661 getNamedOperand(MI, AMDGPU::OpName::cpol)) {
5662 MIB.addImm(CPol->getImm());
5663 }
5664
5665 if (const MachineOperand *TFE =
5666 getNamedOperand(MI, AMDGPU::OpName::tfe)) {
5667 MIB.addImm(TFE->getImm());
5668 }
5669
5670 MIB.addImm(getNamedImmOperand(MI, AMDGPU::OpName::swz));
5671
5672 MIB.cloneMemRefs(MI);
5673 Addr64 = MIB;
5674 } else {
5675 // Atomics with return.
5676 Addr64 = BuildMI(MBB, MI, MI.getDebugLoc(), get(Addr64Opcode))
5677 .add(*VData)
5678 .add(*VDataIn)
5679 .addReg(NewVAddr)
5680 .addReg(NewSRsrc)
5681 .add(*SOffset)
5682 .add(*Offset)
5683 .addImm(getNamedImmOperand(MI, AMDGPU::OpName::cpol))
5684 .cloneMemRefs(MI);
5685 }
5686
5687 MI.removeFromParent();
5688
5689 // NewVaddr = {NewVaddrHi, NewVaddrLo}
5690 BuildMI(MBB, Addr64, Addr64->getDebugLoc(), get(AMDGPU::REG_SEQUENCE),
5691 NewVAddr)
5692 .addReg(RsrcPtr, 0, AMDGPU::sub0)
5693 .addImm(AMDGPU::sub0)
5694 .addReg(RsrcPtr, 0, AMDGPU::sub1)
5695 .addImm(AMDGPU::sub1);
5696 } else {
5697 // This is another variant; legalize Rsrc with waterfall loop from VGPRs
5698 // to SGPRs.
5699 CreatedBB = loadSRsrcFromVGPR(*this, MI, *Rsrc, MDT);
5700 return CreatedBB;
5701 }
5702 }
5703 return CreatedBB;
5704}
5705
5706MachineBasicBlock *SIInstrInfo::moveToVALU(MachineInstr &TopInst,
5707 MachineDominatorTree *MDT) const {
5708 SetVectorType Worklist;
5709 Worklist.insert(&TopInst);
5710 MachineBasicBlock *CreatedBB = nullptr;
5711 MachineBasicBlock *CreatedBBTmp = nullptr;
5712
5713 while (!Worklist.empty()) {
5714 MachineInstr &Inst = *Worklist.pop_back_val();
5715 MachineBasicBlock *MBB = Inst.getParent();
5716 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
5717
5718 unsigned Opcode = Inst.getOpcode();
5719 unsigned NewOpcode = getVALUOp(Inst);
5720
5721 // Handle some special cases
5722 switch (Opcode) {
5723 default:
5724 break;
5725 case AMDGPU::S_ADD_U64_PSEUDO:
5726 case AMDGPU::S_SUB_U64_PSEUDO:
5727 splitScalar64BitAddSub(Worklist, Inst, MDT);
5728 Inst.eraseFromParent();
5729 continue;
5730 case AMDGPU::S_ADD_I32:
5731 case AMDGPU::S_SUB_I32: {
5732 // FIXME: The u32 versions currently selected use the carry.
5733 bool Changed;
5734 std::tie(Changed, CreatedBBTmp) = moveScalarAddSub(Worklist, Inst, MDT);
5735 if (CreatedBBTmp && TopInst.getParent() == CreatedBBTmp)
5736 CreatedBB = CreatedBBTmp;
5737 if (Changed)
5738 continue;
5739
5740 // Default handling
5741 break;
5742 }
5743 case AMDGPU::S_AND_B64:
5744 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_AND_B32, MDT);
5745 Inst.eraseFromParent();
5746 continue;
5747
5748 case AMDGPU::S_OR_B64:
5749 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_OR_B32, MDT);
5750 Inst.eraseFromParent();
5751 continue;
5752
5753 case AMDGPU::S_XOR_B64:
5754 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_XOR_B32, MDT);
5755 Inst.eraseFromParent();
5756 continue;
5757
5758 case AMDGPU::S_NAND_B64:
5759 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_NAND_B32, MDT);
5760 Inst.eraseFromParent();
5761 continue;
5762
5763 case AMDGPU::S_NOR_B64:
5764 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_NOR_B32, MDT);
5765 Inst.eraseFromParent();
5766 continue;
5767
5768 case AMDGPU::S_XNOR_B64:
5769 if (ST.hasDLInsts())
5770 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_XNOR_B32, MDT);
5771 else
5772 splitScalar64BitXnor(Worklist, Inst, MDT);
5773 Inst.eraseFromParent();
5774 continue;
5775
5776 case AMDGPU::S_ANDN2_B64:
5777 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_ANDN2_B32, MDT);
5778 Inst.eraseFromParent();
5779 continue;
5780
5781 case AMDGPU::S_ORN2_B64:
5782 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_ORN2_B32, MDT);
5783 Inst.eraseFromParent();
5784 continue;
5785
5786 case AMDGPU::S_BREV_B64:
5787 splitScalar64BitUnaryOp(Worklist, Inst, AMDGPU::S_BREV_B32, true);
5788 Inst.eraseFromParent();
5789 continue;
5790
5791 case AMDGPU::S_NOT_B64:
5792 splitScalar64BitUnaryOp(Worklist, Inst, AMDGPU::S_NOT_B32);
5793 Inst.eraseFromParent();
5794 continue;
5795
5796 case AMDGPU::S_BCNT1_I32_B64:
5797 splitScalar64BitBCNT(Worklist, Inst);
5798 Inst.eraseFromParent();
5799 continue;
5800
5801 case AMDGPU::S_BFE_I64:
5802 splitScalar64BitBFE(Worklist, Inst);
5803 Inst.eraseFromParent();
5804 continue;
5805
5806 case AMDGPU::S_LSHL_B32:
5807 if (ST.hasOnlyRevVALUShifts()) {
5808 NewOpcode = AMDGPU::V_LSHLREV_B32_e64;
5809 swapOperands(Inst);
5810 }
5811 break;
5812 case AMDGPU::S_ASHR_I32:
5813 if (ST.hasOnlyRevVALUShifts()) {
5814 NewOpcode = AMDGPU::V_ASHRREV_I32_e64;
5815 swapOperands(Inst);
5816 }
5817 break;
5818 case AMDGPU::S_LSHR_B32:
5819 if (ST.hasOnlyRevVALUShifts()) {
5820 NewOpcode = AMDGPU::V_LSHRREV_B32_e64;
5821 swapOperands(Inst);
5822 }
5823 break;
5824 case AMDGPU::S_LSHL_B64:
5825 if (ST.hasOnlyRevVALUShifts()) {
5826 NewOpcode = AMDGPU::V_LSHLREV_B64_e64;
5827 swapOperands(Inst);
5828 }
5829 break;
5830 case AMDGPU::S_ASHR_I64:
5831 if (ST.hasOnlyRevVALUShifts()) {
5832 NewOpcode = AMDGPU::V_ASHRREV_I64_e64;
5833 swapOperands(Inst);
5834 }
5835 break;
5836 case AMDGPU::S_LSHR_B64:
5837 if (ST.hasOnlyRevVALUShifts()) {
5838 NewOpcode = AMDGPU::V_LSHRREV_B64_e64;
5839 swapOperands(Inst);
5840 }
5841 break;
5842
5843 case AMDGPU::S_ABS_I32:
5844 lowerScalarAbs(Worklist, Inst);
5845 Inst.eraseFromParent();
5846 continue;
5847
5848 case AMDGPU::S_CBRANCH_SCC0:
5849 case AMDGPU::S_CBRANCH_SCC1:
5850 // Clear unused bits of vcc
5851 if (ST.isWave32())
5852 BuildMI(*MBB, Inst, Inst.getDebugLoc(), get(AMDGPU::S_AND_B32),
5853 AMDGPU::VCC_LO)
5854 .addReg(AMDGPU::EXEC_LO)
5855 .addReg(AMDGPU::VCC_LO);
5856 else
5857 BuildMI(*MBB, Inst, Inst.getDebugLoc(), get(AMDGPU::S_AND_B64),
5858 AMDGPU::VCC)
5859 .addReg(AMDGPU::EXEC)
5860 .addReg(AMDGPU::VCC);
5861 break;
5862
5863 case AMDGPU::S_BFE_U64:
5864 case AMDGPU::S_BFM_B64:
5865 llvm_unreachable("Moving this op to VALU not implemented")::llvm::llvm_unreachable_internal("Moving this op to VALU not implemented"
, "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 5865)
;
5866
5867 case AMDGPU::S_PACK_LL_B32_B16:
5868 case AMDGPU::S_PACK_LH_B32_B16:
5869 case AMDGPU::S_PACK_HH_B32_B16:
5870 movePackToVALU(Worklist, MRI, Inst);
5871 Inst.eraseFromParent();
5872 continue;
5873
5874 case AMDGPU::S_XNOR_B32:
5875 lowerScalarXnor(Worklist, Inst);
5876 Inst.eraseFromParent();
5877 continue;
5878
5879 case AMDGPU::S_NAND_B32:
5880 splitScalarNotBinop(Worklist, Inst, AMDGPU::S_AND_B32);
5881 Inst.eraseFromParent();
5882 continue;
5883
5884 case AMDGPU::S_NOR_B32:
5885 splitScalarNotBinop(Worklist, Inst, AMDGPU::S_OR_B32);
5886 Inst.eraseFromParent();
5887 continue;
5888
5889 case AMDGPU::S_ANDN2_B32:
5890 splitScalarBinOpN2(Worklist, Inst, AMDGPU::S_AND_B32);
5891 Inst.eraseFromParent();
5892 continue;
5893
5894 case AMDGPU::S_ORN2_B32:
5895 splitScalarBinOpN2(Worklist, Inst, AMDGPU::S_OR_B32);
5896 Inst.eraseFromParent();
5897 continue;
5898
5899 // TODO: remove as soon as everything is ready
5900 // to replace VGPR to SGPR copy with V_READFIRSTLANEs.
5901 // S_ADD/SUB_CO_PSEUDO as well as S_UADDO/USUBO_PSEUDO
5902 // can only be selected from the uniform SDNode.
5903 case AMDGPU::S_ADD_CO_PSEUDO:
5904 case AMDGPU::S_SUB_CO_PSEUDO: {
5905 unsigned Opc = (Inst.getOpcode() == AMDGPU::S_ADD_CO_PSEUDO)
5906 ? AMDGPU::V_ADDC_U32_e64
5907 : AMDGPU::V_SUBB_U32_e64;
5908 const auto *CarryRC = RI.getRegClass(AMDGPU::SReg_1_XEXECRegClassID);
5909
5910 Register CarryInReg = Inst.getOperand(4).getReg();
5911 if (!MRI.constrainRegClass(CarryInReg, CarryRC)) {
5912 Register NewCarryReg = MRI.createVirtualRegister(CarryRC);
5913 BuildMI(*MBB, &Inst, Inst.getDebugLoc(), get(AMDGPU::COPY), NewCarryReg)
5914 .addReg(CarryInReg);
5915 }
5916
5917 Register CarryOutReg = Inst.getOperand(1).getReg();
5918
5919 Register DestReg = MRI.createVirtualRegister(RI.getEquivalentVGPRClass(
5920 MRI.getRegClass(Inst.getOperand(0).getReg())));
5921 MachineInstr *CarryOp =
5922 BuildMI(*MBB, &Inst, Inst.getDebugLoc(), get(Opc), DestReg)
5923 .addReg(CarryOutReg, RegState::Define)
5924 .add(Inst.getOperand(2))
5925 .add(Inst.getOperand(3))
5926 .addReg(CarryInReg)
5927 .addImm(0);
5928 CreatedBBTmp = legalizeOperands(*CarryOp);
5929 if (CreatedBBTmp && TopInst.getParent() == CreatedBBTmp)
5930 CreatedBB = CreatedBBTmp;
5931 MRI.replaceRegWith(Inst.getOperand(0).getReg(), DestReg);
5932 addUsersToMoveToVALUWorklist(DestReg, MRI, Worklist);
5933 Inst.eraseFromParent();
5934 }
5935 continue;
5936 case AMDGPU::S_UADDO_PSEUDO:
5937 case AMDGPU::S_USUBO_PSEUDO: {
5938 const DebugLoc &DL = Inst.getDebugLoc();
5939 MachineOperand &Dest0 = Inst.getOperand(0);
5940 MachineOperand &Dest1 = Inst.getOperand(1);
5941 MachineOperand &Src0 = Inst.getOperand(2);
5942 MachineOperand &Src1 = Inst.getOperand(3);
5943
5944 unsigned Opc = (Inst.getOpcode() == AMDGPU::S_UADDO_PSEUDO)
5945 ? AMDGPU::V_ADD_CO_U32_e64
5946 : AMDGPU::V_SUB_CO_U32_e64;
5947 const TargetRegisterClass *NewRC =
5948 RI.getEquivalentVGPRClass(MRI.getRegClass(Dest0.getReg()));
5949 Register DestReg = MRI.createVirtualRegister(NewRC);
5950 MachineInstr *NewInstr = BuildMI(*MBB, &Inst, DL, get(Opc), DestReg)
5951 .addReg(Dest1.getReg(), RegState::Define)
5952 .add(Src0)
5953 .add(Src1)
5954 .addImm(0); // clamp bit
5955
5956 CreatedBBTmp = legalizeOperands(*NewInstr, MDT);
5957 if (CreatedBBTmp && TopInst.getParent() == CreatedBBTmp)
5958 CreatedBB = CreatedBBTmp;
5959
5960 MRI.replaceRegWith(Dest0.getReg(), DestReg);
5961 addUsersToMoveToVALUWorklist(NewInstr->getOperand(0).getReg(), MRI,
5962 Worklist);
5963 Inst.eraseFromParent();
5964 }
5965 continue;
5966
5967 case AMDGPU::S_CSELECT_B32:
5968 case AMDGPU::S_CSELECT_B64:
5969 lowerSelect(Worklist, Inst, MDT);
5970 Inst.eraseFromParent();
5971 continue;
5972 }
5973
5974 if (NewOpcode == AMDGPU::INSTRUCTION_LIST_END) {
5975 // We cannot move this instruction to the VALU, so we should try to
5976 // legalize its operands instead.
5977 CreatedBBTmp = legalizeOperands(Inst, MDT);
5978 if (CreatedBBTmp && TopInst.getParent() == CreatedBBTmp)
5979 CreatedBB = CreatedBBTmp;
5980 continue;
5981 }
5982
5983 // Use the new VALU Opcode.
5984 const MCInstrDesc &NewDesc = get(NewOpcode);
5985 Inst.setDesc(NewDesc);
5986
5987 // Remove any references to SCC. Vector instructions can't read from it, and
5988 // We're just about to add the implicit use / defs of VCC, and we don't want
5989 // both.
5990 for (unsigned i = Inst.getNumOperands() - 1; i > 0; --i) {
5991 MachineOperand &Op = Inst.getOperand(i);
5992 if (Op.isReg() && Op.getReg() == AMDGPU::SCC) {
5993 // Only propagate through live-def of SCC.
5994 if (Op.isDef() && !Op.isDead())
5995 addSCCDefUsersToVALUWorklist(Op, Inst, Worklist);
5996 if (Op.isUse())
5997 addSCCDefsToVALUWorklist(Op, Worklist);
5998 Inst.RemoveOperand(i);
5999 }
6000 }
6001
6002 if (Opcode == AMDGPU::S_SEXT_I32_I8 || Opcode == AMDGPU::S_SEXT_I32_I16) {
6003 // We are converting these to a BFE, so we need to add the missing
6004 // operands for the size and offset.
6005 unsigned Size = (Opcode == AMDGPU::S_SEXT_I32_I8) ? 8 : 16;
6006 Inst.addOperand(MachineOperand::CreateImm(0));
6007 Inst.addOperand(MachineOperand::CreateImm(Size));
6008
6009 } else if (Opcode == AMDGPU::S_BCNT1_I32_B32) {
6010 // The VALU version adds the second operand to the result, so insert an
6011 // extra 0 operand.
6012 Inst.addOperand(MachineOperand::CreateImm(0));
6013 }
6014
6015 Inst.addImplicitDefUseOperands(*Inst.getParent()->getParent());
6016 fixImplicitOperands(Inst);
6017
6018 if (Opcode == AMDGPU::S_BFE_I32 || Opcode == AMDGPU::S_BFE_U32) {
6019 const MachineOperand &OffsetWidthOp = Inst.getOperand(2);
6020 // If we need to move this to VGPRs, we need to unpack the second operand
6021 // back into the 2 separate ones for bit offset and width.
6022 assert(OffsetWidthOp.isImm() &&(static_cast <bool> (OffsetWidthOp.isImm() && "Scalar BFE is only implemented for constant width and offset"
) ? void (0) : __assert_fail ("OffsetWidthOp.isImm() && \"Scalar BFE is only implemented for constant width and offset\""
, "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 6023, __extension__ __PRETTY_FUNCTION__))
6023 "Scalar BFE is only implemented for constant width and offset")(static_cast <bool> (OffsetWidthOp.isImm() && "Scalar BFE is only implemented for constant width and offset"
) ? void (0) : __assert_fail ("OffsetWidthOp.isImm() && \"Scalar BFE is only implemented for constant width and offset\""
, "/build/llvm-toolchain-snapshot-13~++20210616111117+5c1639fe064b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp"
, 6023, __extension__ __PRETTY_FUNCTION__))
;
6024 uint32_t Imm = OffsetWidthOp.getImm();
6025
6026 uint32_t Offset = Imm & 0x3f; // Extract bits [5:0].
6027 uint32_t BitWidth = (Imm & 0x7f0000) >> 16; // Extract bits [22:16].
6028 Inst.RemoveOperand(2); // Remove old immediate.
6029 Inst.addOperand(MachineOperand::CreateImm(Offset));
6030 Inst.addOperand(MachineOperand::CreateImm(BitWidth));
6031 }
6032
6033 bool HasDst = Inst.getOperand(0).isReg() && Inst.getOperand(0).isDef();
6034 unsigned NewDstReg = AMDGPU::NoRegister;
6035 if (HasDst) {
6036 Register DstReg = Inst.getOperand(0).getReg();
6037 if (DstReg.isPhysical())
6038 continue;
6039
6040 // Update the destination register class.
6041 const TargetRegisterClass *NewDstRC = getDestEquivalentVGPRClass(Inst);
6042 if (!NewDstRC)
6043 continue;
6044
6045 if (Inst.isCopy() && Inst.getOperand(1).getReg().isVirtual() &&
6046 NewDstRC == RI.getRegClassForReg(MRI, Inst.getOperand(1).getReg())) {
6047 // Instead of creating a copy where src and dst are the same register
6048 // class, we just replace all uses of dst with src. These kinds of
6049 // copies interfere with the heuristics MachineSink uses to decide
6050 // whether or not to split a critical edge. Since the pass assumes
6051 // that copies will end up as machine instructions and not be
6052 // eliminated.
6053 addUsersToMoveToVALUWorklist(DstReg, MRI, Worklist);
6054 MRI.replaceRegWith(DstReg, Inst.getOperand(1).getReg());
6055 MRI.clearKillFlags(Inst.getOperand(1).getReg());
6056 Inst.getOperand(0).setReg(DstReg);
6057
6058 // Make sure we don't leave around a dead VGPR->SGPR copy. Normally
6059 // these are deleted later, but at -O0 it would leave a suspicious
6060 // looking illegal copy of an undef register.
6061 for (unsigned I = Inst.getNumOperands() - 1; I != 0; --I)
6062 Inst.RemoveOperand(I);
6063 Inst.setDesc(get(AMDGPU::IMPLICIT_DEF));
6064 continue;
6065 }
6066
6067 NewDstReg = MRI.createVirtualRegister(NewDstRC);
6068 MRI.replaceRegWith(DstReg, NewDstReg);
6069 }
6070
6071 // Legalize the operands
6072 CreatedBBTmp = legalizeOperands(Inst, MDT);
6073 if (CreatedBBTmp && TopInst.getParent() == CreatedBBTmp)
6074 CreatedBB = CreatedBBTmp;
6075