LLVM  15.0.0git
SIInstrInfo.cpp
Go to the documentation of this file.
1 //===- SIInstrInfo.cpp - SI Instruction Information ----------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 /// \file
10 /// SI Implementation of TargetInstrInfo.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "SIInstrInfo.h"
15 #include "AMDGPU.h"
16 #include "AMDGPUInstrInfo.h"
17 #include "GCNHazardRecognizer.h"
18 #include "GCNSubtarget.h"
19 #include "SIMachineFunctionInfo.h"
28 #include "llvm/IR/DiagnosticInfo.h"
29 #include "llvm/IR/IntrinsicsAMDGPU.h"
30 #include "llvm/MC/MCContext.h"
33 
34 using namespace llvm;
35 
36 #define DEBUG_TYPE "si-instr-info"
37 
38 #define GET_INSTRINFO_CTOR_DTOR
39 #include "AMDGPUGenInstrInfo.inc"
40 
41 namespace llvm {
42 
43 class AAResults;
44 
45 namespace AMDGPU {
46 #define GET_D16ImageDimIntrinsics_IMPL
47 #define GET_ImageDimIntrinsicTable_IMPL
48 #define GET_RsrcIntrinsics_IMPL
49 #include "AMDGPUGenSearchableTables.inc"
50 }
51 }
52 
53 
54 // Must be at least 4 to be able to branch over minimum unconditional branch
55 // code. This is only for making it possible to write reasonably small tests for
56 // long branches.
57 static cl::opt<unsigned>
58 BranchOffsetBits("amdgpu-s-branch-bits", cl::ReallyHidden, cl::init(16),
59  cl::desc("Restrict range of branch instructions (DEBUG)"));
60 
62  "amdgpu-fix-16-bit-physreg-copies",
63  cl::desc("Fix copies between 32 and 16 bit registers by extending to 32 bit"),
64  cl::init(true),
66 
68  : AMDGPUGenInstrInfo(AMDGPU::ADJCALLSTACKUP, AMDGPU::ADJCALLSTACKDOWN),
69  RI(ST), ST(ST) {
70  SchedModel.init(&ST);
71 }
72 
73 //===----------------------------------------------------------------------===//
74 // TargetInstrInfo callbacks
75 //===----------------------------------------------------------------------===//
76 
77 static unsigned getNumOperandsNoGlue(SDNode *Node) {
78  unsigned N = Node->getNumOperands();
79  while (N && Node->getOperand(N - 1).getValueType() == MVT::Glue)
80  --N;
81  return N;
82 }
83 
84 /// Returns true if both nodes have the same value for the given
85 /// operand \p Op, or if both nodes do not have this operand.
86 static bool nodesHaveSameOperandValue(SDNode *N0, SDNode* N1, unsigned OpName) {
87  unsigned Opc0 = N0->getMachineOpcode();
88  unsigned Opc1 = N1->getMachineOpcode();
89 
90  int Op0Idx = AMDGPU::getNamedOperandIdx(Opc0, OpName);
91  int Op1Idx = AMDGPU::getNamedOperandIdx(Opc1, OpName);
92 
93  if (Op0Idx == -1 && Op1Idx == -1)
94  return true;
95 
96 
97  if ((Op0Idx == -1 && Op1Idx != -1) ||
98  (Op1Idx == -1 && Op0Idx != -1))
99  return false;
100 
101  // getNamedOperandIdx returns the index for the MachineInstr's operands,
102  // which includes the result as the first operand. We are indexing into the
103  // MachineSDNode's operands, so we need to skip the result operand to get
104  // the real index.
105  --Op0Idx;
106  --Op1Idx;
107 
108  return N0->getOperand(Op0Idx) == N1->getOperand(Op1Idx);
109 }
110 
112  const MachineInstr &MI) const {
113  if (isVOP1(MI) || isVOP2(MI) || isVOP3(MI) || isSDWA(MI) || isSALU(MI)) {
114  // Normally VALU use of exec would block the rematerialization, but that
115  // is OK in this case to have an implicit exec read as all VALU do.
116  // We really want all of the generic logic for this except for this.
117 
118  // Another potential implicit use is mode register. The core logic of
119  // the RA will not attempt rematerialization if mode is set anywhere
120  // in the function, otherwise it is safe since mode is not changed.
121 
122  // There is difference to generic method which does not allow
123  // rematerialization if there are virtual register uses. We allow this,
124  // therefore this method includes SOP instructions as well.
125  return !MI.hasImplicitDef() &&
126  MI.getNumImplicitOperands() == MI.getDesc().getNumImplicitUses() &&
127  !MI.mayRaiseFPException();
128  }
129 
130  return false;
131 }
132 
133 // Returns true if the scalar result of a VALU instruction depends on exec.
134 static bool resultDependsOnExec(const MachineInstr &MI) {
135  // Ignore comparisons which are only used masked with exec.
136  // This allows some hoisting/sinking of VALU comparisons.
137  if (MI.isCompare()) {
138  const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
139  Register DstReg = MI.getOperand(0).getReg();
140  if (!DstReg.isVirtual())
141  return true;
142  for (MachineInstr &Use : MRI.use_nodbg_instructions(DstReg)) {
143  switch (Use.getOpcode()) {
144  case AMDGPU::S_AND_SAVEEXEC_B32:
145  case AMDGPU::S_AND_SAVEEXEC_B64:
146  break;
147  case AMDGPU::S_AND_B32:
148  case AMDGPU::S_AND_B64:
149  if (!Use.readsRegister(AMDGPU::EXEC))
150  return true;
151  break;
152  default:
153  return true;
154  }
155  }
156  return false;
157  }
158 
159  switch (MI.getOpcode()) {
160  default:
161  break;
162  case AMDGPU::V_READFIRSTLANE_B32:
163  return true;
164  }
165 
166  return false;
167 }
168 
170  // Any implicit use of exec by VALU is not a real register read.
171  return MO.getReg() == AMDGPU::EXEC && MO.isImplicit() &&
172  isVALU(*MO.getParent()) && !resultDependsOnExec(*MO.getParent());
173 }
174 
176  int64_t &Offset0,
177  int64_t &Offset1) const {
178  if (!Load0->isMachineOpcode() || !Load1->isMachineOpcode())
179  return false;
180 
181  unsigned Opc0 = Load0->getMachineOpcode();
182  unsigned Opc1 = Load1->getMachineOpcode();
183 
184  // Make sure both are actually loads.
185  if (!get(Opc0).mayLoad() || !get(Opc1).mayLoad())
186  return false;
187 
188  if (isDS(Opc0) && isDS(Opc1)) {
189 
190  // FIXME: Handle this case:
191  if (getNumOperandsNoGlue(Load0) != getNumOperandsNoGlue(Load1))
192  return false;
193 
194  // Check base reg.
195  if (Load0->getOperand(0) != Load1->getOperand(0))
196  return false;
197 
198  // Skip read2 / write2 variants for simplicity.
199  // TODO: We should report true if the used offsets are adjacent (excluded
200  // st64 versions).
201  int Offset0Idx = AMDGPU::getNamedOperandIdx(Opc0, AMDGPU::OpName::offset);
202  int Offset1Idx = AMDGPU::getNamedOperandIdx(Opc1, AMDGPU::OpName::offset);
203  if (Offset0Idx == -1 || Offset1Idx == -1)
204  return false;
205 
206  // XXX - be careful of dataless loads
207  // getNamedOperandIdx returns the index for MachineInstrs. Since they
208  // include the output in the operand list, but SDNodes don't, we need to
209  // subtract the index by one.
210  Offset0Idx -= get(Opc0).NumDefs;
211  Offset1Idx -= get(Opc1).NumDefs;
212  Offset0 = cast<ConstantSDNode>(Load0->getOperand(Offset0Idx))->getZExtValue();
213  Offset1 = cast<ConstantSDNode>(Load1->getOperand(Offset1Idx))->getZExtValue();
214  return true;
215  }
216 
217  if (isSMRD(Opc0) && isSMRD(Opc1)) {
218  // Skip time and cache invalidation instructions.
219  if (AMDGPU::getNamedOperandIdx(Opc0, AMDGPU::OpName::sbase) == -1 ||
220  AMDGPU::getNamedOperandIdx(Opc1, AMDGPU::OpName::sbase) == -1)
221  return false;
222 
223  unsigned NumOps = getNumOperandsNoGlue(Load0);
224  if (NumOps != getNumOperandsNoGlue(Load1))
225  return false;
226 
227  // Check base reg.
228  if (Load0->getOperand(0) != Load1->getOperand(0))
229  return false;
230 
231  // Match register offsets, if both register and immediate offsets present.
232  assert(NumOps == 4 || NumOps == 5);
233  if (NumOps == 5 && Load0->getOperand(1) != Load1->getOperand(1))
234  return false;
235 
236  const ConstantSDNode *Load0Offset =
237  dyn_cast<ConstantSDNode>(Load0->getOperand(NumOps - 3));
238  const ConstantSDNode *Load1Offset =
239  dyn_cast<ConstantSDNode>(Load1->getOperand(NumOps - 3));
240 
241  if (!Load0Offset || !Load1Offset)
242  return false;
243 
244  Offset0 = Load0Offset->getZExtValue();
245  Offset1 = Load1Offset->getZExtValue();
246  return true;
247  }
248 
249  // MUBUF and MTBUF can access the same addresses.
250  if ((isMUBUF(Opc0) || isMTBUF(Opc0)) && (isMUBUF(Opc1) || isMTBUF(Opc1))) {
251 
252  // MUBUF and MTBUF have vaddr at different indices.
253  if (!nodesHaveSameOperandValue(Load0, Load1, AMDGPU::OpName::soffset) ||
254  !nodesHaveSameOperandValue(Load0, Load1, AMDGPU::OpName::vaddr) ||
255  !nodesHaveSameOperandValue(Load0, Load1, AMDGPU::OpName::srsrc))
256  return false;
257 
258  int OffIdx0 = AMDGPU::getNamedOperandIdx(Opc0, AMDGPU::OpName::offset);
259  int OffIdx1 = AMDGPU::getNamedOperandIdx(Opc1, AMDGPU::OpName::offset);
260 
261  if (OffIdx0 == -1 || OffIdx1 == -1)
262  return false;
263 
264  // getNamedOperandIdx returns the index for MachineInstrs. Since they
265  // include the output in the operand list, but SDNodes don't, we need to
266  // subtract the index by one.
267  OffIdx0 -= get(Opc0).NumDefs;
268  OffIdx1 -= get(Opc1).NumDefs;
269 
270  SDValue Off0 = Load0->getOperand(OffIdx0);
271  SDValue Off1 = Load1->getOperand(OffIdx1);
272 
273  // The offset might be a FrameIndexSDNode.
274  if (!isa<ConstantSDNode>(Off0) || !isa<ConstantSDNode>(Off1))
275  return false;
276 
277  Offset0 = cast<ConstantSDNode>(Off0)->getZExtValue();
278  Offset1 = cast<ConstantSDNode>(Off1)->getZExtValue();
279  return true;
280  }
281 
282  return false;
283 }
284 
285 static bool isStride64(unsigned Opc) {
286  switch (Opc) {
287  case AMDGPU::DS_READ2ST64_B32:
288  case AMDGPU::DS_READ2ST64_B64:
289  case AMDGPU::DS_WRITE2ST64_B32:
290  case AMDGPU::DS_WRITE2ST64_B64:
291  return true;
292  default:
293  return false;
294  }
295 }
296 
299  int64_t &Offset, bool &OffsetIsScalable, unsigned &Width,
300  const TargetRegisterInfo *TRI) const {
301  if (!LdSt.mayLoadOrStore())
302  return false;
303 
304  unsigned Opc = LdSt.getOpcode();
305  OffsetIsScalable = false;
306  const MachineOperand *BaseOp, *OffsetOp;
307  int DataOpIdx;
308 
309  if (isDS(LdSt)) {
310  BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::addr);
311  OffsetOp = getNamedOperand(LdSt, AMDGPU::OpName::offset);
312  if (OffsetOp) {
313  // Normal, single offset LDS instruction.
314  if (!BaseOp) {
315  // DS_CONSUME/DS_APPEND use M0 for the base address.
316  // TODO: find the implicit use operand for M0 and use that as BaseOp?
317  return false;
318  }
319  BaseOps.push_back(BaseOp);
320  Offset = OffsetOp->getImm();
321  // Get appropriate operand, and compute width accordingly.
322  DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst);
323  if (DataOpIdx == -1)
324  DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data0);
325  Width = getOpSize(LdSt, DataOpIdx);
326  } else {
327  // The 2 offset instructions use offset0 and offset1 instead. We can treat
328  // these as a load with a single offset if the 2 offsets are consecutive.
329  // We will use this for some partially aligned loads.
330  const MachineOperand *Offset0Op =
331  getNamedOperand(LdSt, AMDGPU::OpName::offset0);
332  const MachineOperand *Offset1Op =
333  getNamedOperand(LdSt, AMDGPU::OpName::offset1);
334 
335  unsigned Offset0 = Offset0Op->getImm();
336  unsigned Offset1 = Offset1Op->getImm();
337  if (Offset0 + 1 != Offset1)
338  return false;
339 
340  // Each of these offsets is in element sized units, so we need to convert
341  // to bytes of the individual reads.
342 
343  unsigned EltSize;
344  if (LdSt.mayLoad())
345  EltSize = TRI->getRegSizeInBits(*getOpRegClass(LdSt, 0)) / 16;
346  else {
347  assert(LdSt.mayStore());
348  int Data0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data0);
349  EltSize = TRI->getRegSizeInBits(*getOpRegClass(LdSt, Data0Idx)) / 8;
350  }
351 
352  if (isStride64(Opc))
353  EltSize *= 64;
354 
355  BaseOps.push_back(BaseOp);
356  Offset = EltSize * Offset0;
357  // Get appropriate operand(s), and compute width accordingly.
358  DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst);
359  if (DataOpIdx == -1) {
360  DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data0);
361  Width = getOpSize(LdSt, DataOpIdx);
362  DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data1);
363  Width += getOpSize(LdSt, DataOpIdx);
364  } else {
365  Width = getOpSize(LdSt, DataOpIdx);
366  }
367  }
368  return true;
369  }
370 
371  if (isMUBUF(LdSt) || isMTBUF(LdSt)) {
372  const MachineOperand *RSrc = getNamedOperand(LdSt, AMDGPU::OpName::srsrc);
373  if (!RSrc) // e.g. BUFFER_WBINVL1_VOL
374  return false;
375  BaseOps.push_back(RSrc);
376  BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::vaddr);
377  if (BaseOp && !BaseOp->isFI())
378  BaseOps.push_back(BaseOp);
379  const MachineOperand *OffsetImm =
380  getNamedOperand(LdSt, AMDGPU::OpName::offset);
381  Offset = OffsetImm->getImm();
382  const MachineOperand *SOffset =
383  getNamedOperand(LdSt, AMDGPU::OpName::soffset);
384  if (SOffset) {
385  if (SOffset->isReg())
386  BaseOps.push_back(SOffset);
387  else
388  Offset += SOffset->getImm();
389  }
390  // Get appropriate operand, and compute width accordingly.
391  DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst);
392  if (DataOpIdx == -1)
393  DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdata);
394  if (DataOpIdx == -1) // LDS DMA
395  return false;
396  Width = getOpSize(LdSt, DataOpIdx);
397  return true;
398  }
399 
400  if (isMIMG(LdSt)) {
401  int SRsrcIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::srsrc);
402  BaseOps.push_back(&LdSt.getOperand(SRsrcIdx));
403  int VAddr0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vaddr0);
404  if (VAddr0Idx >= 0) {
405  // GFX10 possible NSA encoding.
406  for (int I = VAddr0Idx; I < SRsrcIdx; ++I)
407  BaseOps.push_back(&LdSt.getOperand(I));
408  } else {
409  BaseOps.push_back(getNamedOperand(LdSt, AMDGPU::OpName::vaddr));
410  }
411  Offset = 0;
412  // Get appropriate operand, and compute width accordingly.
413  DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdata);
414  Width = getOpSize(LdSt, DataOpIdx);
415  return true;
416  }
417 
418  if (isSMRD(LdSt)) {
419  BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::sbase);
420  if (!BaseOp) // e.g. S_MEMTIME
421  return false;
422  BaseOps.push_back(BaseOp);
423  OffsetOp = getNamedOperand(LdSt, AMDGPU::OpName::offset);
424  Offset = OffsetOp ? OffsetOp->getImm() : 0;
425  // Get appropriate operand, and compute width accordingly.
426  DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::sdst);
427  Width = getOpSize(LdSt, DataOpIdx);
428  return true;
429  }
430 
431  if (isFLAT(LdSt)) {
432  // Instructions have either vaddr or saddr or both or none.
433  BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::vaddr);
434  if (BaseOp)
435  BaseOps.push_back(BaseOp);
436  BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::saddr);
437  if (BaseOp)
438  BaseOps.push_back(BaseOp);
439  Offset = getNamedOperand(LdSt, AMDGPU::OpName::offset)->getImm();
440  // Get appropriate operand, and compute width accordingly.
441  DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst);
442  if (DataOpIdx == -1)
443  DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdata);
444  if (DataOpIdx == -1) // LDS DMA
445  return false;
446  Width = getOpSize(LdSt, DataOpIdx);
447  return true;
448  }
449 
450  return false;
451 }
452 
453 static bool memOpsHaveSameBasePtr(const MachineInstr &MI1,
455  const MachineInstr &MI2,
457  // Only examine the first "base" operand of each instruction, on the
458  // assumption that it represents the real base address of the memory access.
459  // Other operands are typically offsets or indices from this base address.
460  if (BaseOps1.front()->isIdenticalTo(*BaseOps2.front()))
461  return true;
462 
463  if (!MI1.hasOneMemOperand() || !MI2.hasOneMemOperand())
464  return false;
465 
466  auto MO1 = *MI1.memoperands_begin();
467  auto MO2 = *MI2.memoperands_begin();
468  if (MO1->getAddrSpace() != MO2->getAddrSpace())
469  return false;
470 
471  auto Base1 = MO1->getValue();
472  auto Base2 = MO2->getValue();
473  if (!Base1 || !Base2)
474  return false;
475  Base1 = getUnderlyingObject(Base1);
476  Base2 = getUnderlyingObject(Base2);
477 
478  if (isa<UndefValue>(Base1) || isa<UndefValue>(Base2))
479  return false;
480 
481  return Base1 == Base2;
482 }
483 
486  unsigned NumLoads,
487  unsigned NumBytes) const {
488  // If the mem ops (to be clustered) do not have the same base ptr, then they
489  // should not be clustered
490  if (!BaseOps1.empty() && !BaseOps2.empty()) {
491  const MachineInstr &FirstLdSt = *BaseOps1.front()->getParent();
492  const MachineInstr &SecondLdSt = *BaseOps2.front()->getParent();
493  if (!memOpsHaveSameBasePtr(FirstLdSt, BaseOps1, SecondLdSt, BaseOps2))
494  return false;
495  } else if (!BaseOps1.empty() || !BaseOps2.empty()) {
496  // If only one base op is empty, they do not have the same base ptr
497  return false;
498  }
499 
500  // In order to avoid register pressure, on an average, the number of DWORDS
501  // loaded together by all clustered mem ops should not exceed 8. This is an
502  // empirical value based on certain observations and performance related
503  // experiments.
504  // The good thing about this heuristic is - it avoids clustering of too many
505  // sub-word loads, and also avoids clustering of wide loads. Below is the
506  // brief summary of how the heuristic behaves for various `LoadSize`.
507  // (1) 1 <= LoadSize <= 4: cluster at max 8 mem ops
508  // (2) 5 <= LoadSize <= 8: cluster at max 4 mem ops
509  // (3) 9 <= LoadSize <= 12: cluster at max 2 mem ops
510  // (4) 13 <= LoadSize <= 16: cluster at max 2 mem ops
511  // (5) LoadSize >= 17: do not cluster
512  const unsigned LoadSize = NumBytes / NumLoads;
513  const unsigned NumDWORDs = ((LoadSize + 3) / 4) * NumLoads;
514  return NumDWORDs <= 8;
515 }
516 
517 // FIXME: This behaves strangely. If, for example, you have 32 load + stores,
518 // the first 16 loads will be interleaved with the stores, and the next 16 will
519 // be clustered as expected. It should really split into 2 16 store batches.
520 //
521 // Loads are clustered until this returns false, rather than trying to schedule
522 // groups of stores. This also means we have to deal with saying different
523 // address space loads should be clustered, and ones which might cause bank
524 // conflicts.
525 //
526 // This might be deprecated so it might not be worth that much effort to fix.
528  int64_t Offset0, int64_t Offset1,
529  unsigned NumLoads) const {
530  assert(Offset1 > Offset0 &&
531  "Second offset should be larger than first offset!");
532  // If we have less than 16 loads in a row, and the offsets are within 64
533  // bytes, then schedule together.
534 
535  // A cacheline is 64 bytes (for global memory).
536  return (NumLoads <= 16 && (Offset1 - Offset0) < 64);
537 }
538 
541  const DebugLoc &DL, MCRegister DestReg,
542  MCRegister SrcReg, bool KillSrc,
543  const char *Msg = "illegal SGPR to VGPR copy") {
544  MachineFunction *MF = MBB.getParent();
545  DiagnosticInfoUnsupported IllegalCopy(MF->getFunction(), Msg, DL, DS_Error);
546  LLVMContext &C = MF->getFunction().getContext();
547  C.diagnose(IllegalCopy);
548 
549  BuildMI(MBB, MI, DL, TII->get(AMDGPU::SI_ILLEGAL_COPY), DestReg)
550  .addReg(SrcReg, getKillRegState(KillSrc));
551 }
552 
553 /// Handle copying from SGPR to AGPR, or from AGPR to AGPR on GFX908. It is not
554 /// possible to have a direct copy in these cases on GFX908, so an intermediate
555 /// VGPR copy is required.
556 static void indirectCopyToAGPR(const SIInstrInfo &TII,
559  const DebugLoc &DL, MCRegister DestReg,
560  MCRegister SrcReg, bool KillSrc,
561  RegScavenger &RS,
562  Register ImpDefSuperReg = Register(),
563  Register ImpUseSuperReg = Register()) {
564  assert((TII.getSubtarget().hasMAIInsts() &&
565  !TII.getSubtarget().hasGFX90AInsts()) &&
566  "Expected GFX908 subtarget.");
567 
568  assert((AMDGPU::SReg_32RegClass.contains(SrcReg) ||
569  AMDGPU::AGPR_32RegClass.contains(SrcReg)) &&
570  "Source register of the copy should be either an SGPR or an AGPR.");
571 
572  assert(AMDGPU::AGPR_32RegClass.contains(DestReg) &&
573  "Destination register of the copy should be an AGPR.");
574 
575  const SIRegisterInfo &RI = TII.getRegisterInfo();
576 
577  // First try to find defining accvgpr_write to avoid temporary registers.
578  for (auto Def = MI, E = MBB.begin(); Def != E; ) {
579  --Def;
580  if (!Def->definesRegister(SrcReg, &RI))
581  continue;
582  if (Def->getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64)
583  break;
584 
585  MachineOperand &DefOp = Def->getOperand(1);
586  assert(DefOp.isReg() || DefOp.isImm());
587 
588  if (DefOp.isReg()) {
589  // Check that register source operand if not clobbered before MI.
590  // Immediate operands are always safe to propagate.
591  bool SafeToPropagate = true;
592  for (auto I = Def; I != MI && SafeToPropagate; ++I)
593  if (I->modifiesRegister(DefOp.getReg(), &RI))
594  SafeToPropagate = false;
595 
596  if (!SafeToPropagate)
597  break;
598 
599  DefOp.setIsKill(false);
600  }
601 
603  BuildMI(MBB, MI, DL, TII.get(AMDGPU::V_ACCVGPR_WRITE_B32_e64), DestReg)
604  .add(DefOp);
605  if (ImpDefSuperReg)
606  Builder.addReg(ImpDefSuperReg, RegState::Define | RegState::Implicit);
607 
608  if (ImpUseSuperReg) {
609  Builder.addReg(ImpUseSuperReg,
611  }
612 
613  return;
614  }
615 
616  RS.enterBasicBlock(MBB);
617  RS.forward(MI);
618 
619  // Ideally we want to have three registers for a long reg_sequence copy
620  // to hide 2 waitstates between v_mov_b32 and accvgpr_write.
621  unsigned MaxVGPRs = RI.getRegPressureLimit(&AMDGPU::VGPR_32RegClass,
622  *MBB.getParent());
623 
624  // Registers in the sequence are allocated contiguously so we can just
625  // use register number to pick one of three round-robin temps.
626  unsigned RegNo = (DestReg - AMDGPU::AGPR0) % 3;
627  Register Tmp =
628  MBB.getParent()->getInfo<SIMachineFunctionInfo>()->getVGPRForAGPRCopy();
630  "VGPR used for an intermediate copy should have been reserved.");
631 
632  // Only loop through if there are any free registers left, otherwise
633  // scavenger may report a fatal error without emergency spill slot
634  // or spill with the slot.
635  while (RegNo-- && RS.FindUnusedReg(&AMDGPU::VGPR_32RegClass)) {
636  Register Tmp2 = RS.scavengeRegister(&AMDGPU::VGPR_32RegClass, 0);
637  if (!Tmp2 || RI.getHWRegIndex(Tmp2) >= MaxVGPRs)
638  break;
639  Tmp = Tmp2;
640  RS.setRegUsed(Tmp);
641  }
642 
643  // Insert copy to temporary VGPR.
644  unsigned TmpCopyOp = AMDGPU::V_MOV_B32_e32;
645  if (AMDGPU::AGPR_32RegClass.contains(SrcReg)) {
646  TmpCopyOp = AMDGPU::V_ACCVGPR_READ_B32_e64;
647  } else {
648  assert(AMDGPU::SReg_32RegClass.contains(SrcReg));
649  }
650 
651  MachineInstrBuilder UseBuilder = BuildMI(MBB, MI, DL, TII.get(TmpCopyOp), Tmp)
652  .addReg(SrcReg, getKillRegState(KillSrc));
653  if (ImpUseSuperReg) {
654  UseBuilder.addReg(ImpUseSuperReg,
656  }
657 
658  MachineInstrBuilder DefBuilder
659  = BuildMI(MBB, MI, DL, TII.get(AMDGPU::V_ACCVGPR_WRITE_B32_e64), DestReg)
660  .addReg(Tmp, RegState::Kill);
661 
662  if (ImpDefSuperReg)
663  DefBuilder.addReg(ImpDefSuperReg, RegState::Define | RegState::Implicit);
664 }
665 
668  MCRegister DestReg, MCRegister SrcReg, bool KillSrc,
669  const TargetRegisterClass *RC, bool Forward) {
670  const SIRegisterInfo &RI = TII.getRegisterInfo();
671  ArrayRef<int16_t> BaseIndices = RI.getRegSplitParts(RC, 4);
673  MachineInstr *FirstMI = nullptr, *LastMI = nullptr;
674 
675  for (unsigned Idx = 0; Idx < BaseIndices.size(); ++Idx) {
676  int16_t SubIdx = BaseIndices[Idx];
677  Register Reg = RI.getSubReg(DestReg, SubIdx);
678  unsigned Opcode = AMDGPU::S_MOV_B32;
679 
680  // Is SGPR aligned? If so try to combine with next.
681  Register Src = RI.getSubReg(SrcReg, SubIdx);
682  bool AlignedDest = ((Reg - AMDGPU::SGPR0) % 2) == 0;
683  bool AlignedSrc = ((Src - AMDGPU::SGPR0) % 2) == 0;
684  if (AlignedDest && AlignedSrc && (Idx + 1 < BaseIndices.size())) {
685  // Can use SGPR64 copy
686  unsigned Channel = RI.getChannelFromSubReg(SubIdx);
687  SubIdx = RI.getSubRegFromChannel(Channel, 2);
688  Opcode = AMDGPU::S_MOV_B64;
689  Idx++;
690  }
691 
692  LastMI = BuildMI(MBB, I, DL, TII.get(Opcode), RI.getSubReg(DestReg, SubIdx))
693  .addReg(RI.getSubReg(SrcReg, SubIdx))
694  .addReg(SrcReg, RegState::Implicit);
695 
696  if (!FirstMI)
697  FirstMI = LastMI;
698 
699  if (!Forward)
700  I--;
701  }
702 
703  assert(FirstMI && LastMI);
704  if (!Forward)
705  std::swap(FirstMI, LastMI);
706 
707  FirstMI->addOperand(
708  MachineOperand::CreateReg(DestReg, true /*IsDef*/, true /*IsImp*/));
709 
710  if (KillSrc)
711  LastMI->addRegisterKilled(SrcReg, &RI);
712 }
713 
716  const DebugLoc &DL, MCRegister DestReg,
717  MCRegister SrcReg, bool KillSrc) const {
718  const TargetRegisterClass *RC = RI.getPhysRegClass(DestReg);
719 
720  // FIXME: This is hack to resolve copies between 16 bit and 32 bit
721  // registers until all patterns are fixed.
722  if (Fix16BitCopies &&
723  ((RI.getRegSizeInBits(*RC) == 16) ^
724  (RI.getRegSizeInBits(*RI.getPhysRegClass(SrcReg)) == 16))) {
725  MCRegister &RegToFix = (RI.getRegSizeInBits(*RC) == 16) ? DestReg : SrcReg;
726  MCRegister Super = RI.get32BitRegister(RegToFix);
727  assert(RI.getSubReg(Super, AMDGPU::lo16) == RegToFix);
728  RegToFix = Super;
729 
730  if (DestReg == SrcReg) {
731  // Insert empty bundle since ExpandPostRA expects an instruction here.
732  BuildMI(MBB, MI, DL, get(AMDGPU::BUNDLE));
733  return;
734  }
735 
736  RC = RI.getPhysRegClass(DestReg);
737  }
738 
739  if (RC == &AMDGPU::VGPR_32RegClass) {
740  assert(AMDGPU::VGPR_32RegClass.contains(SrcReg) ||
741  AMDGPU::SReg_32RegClass.contains(SrcReg) ||
742  AMDGPU::AGPR_32RegClass.contains(SrcReg));
743  unsigned Opc = AMDGPU::AGPR_32RegClass.contains(SrcReg) ?
744  AMDGPU::V_ACCVGPR_READ_B32_e64 : AMDGPU::V_MOV_B32_e32;
745  BuildMI(MBB, MI, DL, get(Opc), DestReg)
746  .addReg(SrcReg, getKillRegState(KillSrc));
747  return;
748  }
749 
750  if (RC == &AMDGPU::SReg_32_XM0RegClass ||
751  RC == &AMDGPU::SReg_32RegClass) {
752  if (SrcReg == AMDGPU::SCC) {
753  BuildMI(MBB, MI, DL, get(AMDGPU::S_CSELECT_B32), DestReg)
754  .addImm(1)
755  .addImm(0);
756  return;
757  }
758 
759  if (DestReg == AMDGPU::VCC_LO) {
760  if (AMDGPU::SReg_32RegClass.contains(SrcReg)) {
761  BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), AMDGPU::VCC_LO)
762  .addReg(SrcReg, getKillRegState(KillSrc));
763  } else {
764  // FIXME: Hack until VReg_1 removed.
765  assert(AMDGPU::VGPR_32RegClass.contains(SrcReg));
766  BuildMI(MBB, MI, DL, get(AMDGPU::V_CMP_NE_U32_e32))
767  .addImm(0)
768  .addReg(SrcReg, getKillRegState(KillSrc));
769  }
770 
771  return;
772  }
773 
774  if (!AMDGPU::SReg_32RegClass.contains(SrcReg)) {
775  reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc);
776  return;
777  }
778 
779  BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), DestReg)
780  .addReg(SrcReg, getKillRegState(KillSrc));
781  return;
782  }
783 
784  if (RC == &AMDGPU::SReg_64RegClass) {
785  if (SrcReg == AMDGPU::SCC) {
786  BuildMI(MBB, MI, DL, get(AMDGPU::S_CSELECT_B64), DestReg)
787  .addImm(1)
788  .addImm(0);
789  return;
790  }
791 
792  if (DestReg == AMDGPU::VCC) {
793  if (AMDGPU::SReg_64RegClass.contains(SrcReg)) {
794  BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B64), AMDGPU::VCC)
795  .addReg(SrcReg, getKillRegState(KillSrc));
796  } else {
797  // FIXME: Hack until VReg_1 removed.
798  assert(AMDGPU::VGPR_32RegClass.contains(SrcReg));
799  BuildMI(MBB, MI, DL, get(AMDGPU::V_CMP_NE_U32_e32))
800  .addImm(0)
801  .addReg(SrcReg, getKillRegState(KillSrc));
802  }
803 
804  return;
805  }
806 
807  if (!AMDGPU::SReg_64RegClass.contains(SrcReg)) {
808  reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc);
809  return;
810  }
811 
812  BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B64), DestReg)
813  .addReg(SrcReg, getKillRegState(KillSrc));
814  return;
815  }
816 
817  if (DestReg == AMDGPU::SCC) {
818  // Copying 64-bit or 32-bit sources to SCC barely makes sense,
819  // but SelectionDAG emits such copies for i1 sources.
820  if (AMDGPU::SReg_64RegClass.contains(SrcReg)) {
821  // This copy can only be produced by patterns
822  // with explicit SCC, which are known to be enabled
823  // only for subtargets with S_CMP_LG_U64 present.
825  BuildMI(MBB, MI, DL, get(AMDGPU::S_CMP_LG_U64))
826  .addReg(SrcReg, getKillRegState(KillSrc))
827  .addImm(0);
828  } else {
829  assert(AMDGPU::SReg_32RegClass.contains(SrcReg));
830  BuildMI(MBB, MI, DL, get(AMDGPU::S_CMP_LG_U32))
831  .addReg(SrcReg, getKillRegState(KillSrc))
832  .addImm(0);
833  }
834 
835  return;
836  }
837 
838  if (RC == &AMDGPU::AGPR_32RegClass) {
839  if (AMDGPU::VGPR_32RegClass.contains(SrcReg) ||
840  (ST.hasGFX90AInsts() && AMDGPU::SReg_32RegClass.contains(SrcReg))) {
841  BuildMI(MBB, MI, DL, get(AMDGPU::V_ACCVGPR_WRITE_B32_e64), DestReg)
842  .addReg(SrcReg, getKillRegState(KillSrc));
843  return;
844  }
845 
846  if (AMDGPU::AGPR_32RegClass.contains(SrcReg) && ST.hasGFX90AInsts()) {
847  BuildMI(MBB, MI, DL, get(AMDGPU::V_ACCVGPR_MOV_B32), DestReg)
848  .addReg(SrcReg, getKillRegState(KillSrc));
849  return;
850  }
851 
852  // FIXME: Pass should maintain scavenger to avoid scan through the block on
853  // every AGPR spill.
854  RegScavenger RS;
855  indirectCopyToAGPR(*this, MBB, MI, DL, DestReg, SrcReg, KillSrc, RS);
856  return;
857  }
858 
859  const unsigned Size = RI.getRegSizeInBits(*RC);
860  if (Size == 16) {
861  assert(AMDGPU::VGPR_LO16RegClass.contains(SrcReg) ||
862  AMDGPU::VGPR_HI16RegClass.contains(SrcReg) ||
863  AMDGPU::SReg_LO16RegClass.contains(SrcReg) ||
864  AMDGPU::AGPR_LO16RegClass.contains(SrcReg));
865 
866  bool IsSGPRDst = AMDGPU::SReg_LO16RegClass.contains(DestReg);
867  bool IsSGPRSrc = AMDGPU::SReg_LO16RegClass.contains(SrcReg);
868  bool IsAGPRDst = AMDGPU::AGPR_LO16RegClass.contains(DestReg);
869  bool IsAGPRSrc = AMDGPU::AGPR_LO16RegClass.contains(SrcReg);
870  bool DstLow = AMDGPU::VGPR_LO16RegClass.contains(DestReg) ||
871  AMDGPU::SReg_LO16RegClass.contains(DestReg) ||
872  AMDGPU::AGPR_LO16RegClass.contains(DestReg);
873  bool SrcLow = AMDGPU::VGPR_LO16RegClass.contains(SrcReg) ||
874  AMDGPU::SReg_LO16RegClass.contains(SrcReg) ||
875  AMDGPU::AGPR_LO16RegClass.contains(SrcReg);
876  MCRegister NewDestReg = RI.get32BitRegister(DestReg);
877  MCRegister NewSrcReg = RI.get32BitRegister(SrcReg);
878 
879  if (IsSGPRDst) {
880  if (!IsSGPRSrc) {
881  reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc);
882  return;
883  }
884 
885  BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), NewDestReg)
886  .addReg(NewSrcReg, getKillRegState(KillSrc));
887  return;
888  }
889 
890  if (IsAGPRDst || IsAGPRSrc) {
891  if (!DstLow || !SrcLow) {
892  reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc,
893  "Cannot use hi16 subreg with an AGPR!");
894  }
895 
896  copyPhysReg(MBB, MI, DL, NewDestReg, NewSrcReg, KillSrc);
897  return;
898  }
899 
900  if (IsSGPRSrc && !ST.hasSDWAScalar()) {
901  if (!DstLow || !SrcLow) {
902  reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc,
903  "Cannot use hi16 subreg on VI!");
904  }
905 
906  BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), NewDestReg)
907  .addReg(NewSrcReg, getKillRegState(KillSrc));
908  return;
909  }
910 
911  auto MIB = BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_sdwa), NewDestReg)
912  .addImm(0) // src0_modifiers
913  .addReg(NewSrcReg)
914  .addImm(0) // clamp
920  .addReg(NewDestReg, RegState::Implicit | RegState::Undef);
921  // First implicit operand is $exec.
922  MIB->tieOperands(0, MIB->getNumOperands() - 1);
923  return;
924  }
925 
926  const TargetRegisterClass *SrcRC = RI.getPhysRegClass(SrcReg);
927  if (RC == RI.getVGPR64Class() && (SrcRC == RC || RI.isSGPRClass(SrcRC))) {
928  if (ST.hasMovB64()) {
929  BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B64_e32), DestReg)
930  .addReg(SrcReg, getKillRegState(KillSrc));
931  return;
932  }
933  if (ST.hasPackedFP32Ops()) {
934  BuildMI(MBB, MI, DL, get(AMDGPU::V_PK_MOV_B32), DestReg)
936  .addReg(SrcReg)
938  .addReg(SrcReg)
939  .addImm(0) // op_sel_lo
940  .addImm(0) // op_sel_hi
941  .addImm(0) // neg_lo
942  .addImm(0) // neg_hi
943  .addImm(0) // clamp
944  .addReg(SrcReg, getKillRegState(KillSrc) | RegState::Implicit);
945  return;
946  }
947  }
948 
949  const bool Forward = RI.getHWRegIndex(DestReg) <= RI.getHWRegIndex(SrcReg);
950  if (RI.isSGPRClass(RC)) {
951  if (!RI.isSGPRClass(SrcRC)) {
952  reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc);
953  return;
954  }
955  const bool CanKillSuperReg = KillSrc && !RI.regsOverlap(SrcReg, DestReg);
956  expandSGPRCopy(*this, MBB, MI, DL, DestReg, SrcReg, CanKillSuperReg, RC,
957  Forward);
958  return;
959  }
960 
961  unsigned EltSize = 4;
962  unsigned Opcode = AMDGPU::V_MOV_B32_e32;
963  if (RI.isAGPRClass(RC)) {
964  if (ST.hasGFX90AInsts() && RI.isAGPRClass(SrcRC))
965  Opcode = AMDGPU::V_ACCVGPR_MOV_B32;
966  else if (RI.hasVGPRs(SrcRC) ||
967  (ST.hasGFX90AInsts() && RI.isSGPRClass(SrcRC)))
968  Opcode = AMDGPU::V_ACCVGPR_WRITE_B32_e64;
969  else
970  Opcode = AMDGPU::INSTRUCTION_LIST_END;
971  } else if (RI.hasVGPRs(RC) && RI.isAGPRClass(SrcRC)) {
972  Opcode = AMDGPU::V_ACCVGPR_READ_B32_e64;
973  } else if ((Size % 64 == 0) && RI.hasVGPRs(RC) &&
974  (RI.isProperlyAlignedRC(*RC) &&
975  (SrcRC == RC || RI.isSGPRClass(SrcRC)))) {
976  // TODO: In 96-bit case, could do a 64-bit mov and then a 32-bit mov.
977  if (ST.hasMovB64()) {
978  Opcode = AMDGPU::V_MOV_B64_e32;
979  EltSize = 8;
980  } else if (ST.hasPackedFP32Ops()) {
981  Opcode = AMDGPU::V_PK_MOV_B32;
982  EltSize = 8;
983  }
984  }
985 
986  // For the cases where we need an intermediate instruction/temporary register
987  // (destination is an AGPR), we need a scavenger.
988  //
989  // FIXME: The pass should maintain this for us so we don't have to re-scan the
990  // whole block for every handled copy.
991  std::unique_ptr<RegScavenger> RS;
992  if (Opcode == AMDGPU::INSTRUCTION_LIST_END)
993  RS.reset(new RegScavenger());
994 
995  ArrayRef<int16_t> SubIndices = RI.getRegSplitParts(RC, EltSize);
996 
997  // If there is an overlap, we can't kill the super-register on the last
998  // instruction, since it will also kill the components made live by this def.
999  const bool CanKillSuperReg = KillSrc && !RI.regsOverlap(SrcReg, DestReg);
1000 
1001  for (unsigned Idx = 0; Idx < SubIndices.size(); ++Idx) {
1002  unsigned SubIdx;
1003  if (Forward)
1004  SubIdx = SubIndices[Idx];
1005  else
1006  SubIdx = SubIndices[SubIndices.size() - Idx - 1];
1007 
1008  bool UseKill = CanKillSuperReg && Idx == SubIndices.size() - 1;
1009 
1010  if (Opcode == AMDGPU::INSTRUCTION_LIST_END) {
1011  Register ImpDefSuper = Idx == 0 ? Register(DestReg) : Register();
1012  Register ImpUseSuper = SrcReg;
1013  indirectCopyToAGPR(*this, MBB, MI, DL, RI.getSubReg(DestReg, SubIdx),
1014  RI.getSubReg(SrcReg, SubIdx), UseKill, *RS,
1015  ImpDefSuper, ImpUseSuper);
1016  } else if (Opcode == AMDGPU::V_PK_MOV_B32) {
1017  Register DstSubReg = RI.getSubReg(DestReg, SubIdx);
1018  Register SrcSubReg = RI.getSubReg(SrcReg, SubIdx);
1019  MachineInstrBuilder MIB =
1020  BuildMI(MBB, MI, DL, get(AMDGPU::V_PK_MOV_B32), DstSubReg)
1022  .addReg(SrcSubReg)
1024  .addReg(SrcSubReg)
1025  .addImm(0) // op_sel_lo
1026  .addImm(0) // op_sel_hi
1027  .addImm(0) // neg_lo
1028  .addImm(0) // neg_hi
1029  .addImm(0) // clamp
1030  .addReg(SrcReg, getKillRegState(UseKill) | RegState::Implicit);
1031  if (Idx == 0)
1032  MIB.addReg(DestReg, RegState::Define | RegState::Implicit);
1033  } else {
1035  BuildMI(MBB, MI, DL, get(Opcode), RI.getSubReg(DestReg, SubIdx))
1036  .addReg(RI.getSubReg(SrcReg, SubIdx));
1037  if (Idx == 0)
1038  Builder.addReg(DestReg, RegState::Define | RegState::Implicit);
1039 
1040  Builder.addReg(SrcReg, getKillRegState(UseKill) | RegState::Implicit);
1041  }
1042  }
1043 }
1044 
1045 int SIInstrInfo::commuteOpcode(unsigned Opcode) const {
1046  int NewOpc;
1047 
1048  // Try to map original to commuted opcode
1049  NewOpc = AMDGPU::getCommuteRev(Opcode);
1050  if (NewOpc != -1)
1051  // Check if the commuted (REV) opcode exists on the target.
1052  return pseudoToMCOpcode(NewOpc) != -1 ? NewOpc : -1;
1053 
1054  // Try to map commuted to original opcode
1055  NewOpc = AMDGPU::getCommuteOrig(Opcode);
1056  if (NewOpc != -1)
1057  // Check if the original (non-REV) opcode exists on the target.
1058  return pseudoToMCOpcode(NewOpc) != -1 ? NewOpc : -1;
1059 
1060  return Opcode;
1061 }
1062 
1065  const DebugLoc &DL, unsigned DestReg,
1066  int64_t Value) const {
1068  const TargetRegisterClass *RegClass = MRI.getRegClass(DestReg);
1069  if (RegClass == &AMDGPU::SReg_32RegClass ||
1070  RegClass == &AMDGPU::SGPR_32RegClass ||
1071  RegClass == &AMDGPU::SReg_32_XM0RegClass ||
1072  RegClass == &AMDGPU::SReg_32_XM0_XEXECRegClass) {
1073  BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), DestReg)
1074  .addImm(Value);
1075  return;
1076  }
1077 
1078  if (RegClass == &AMDGPU::SReg_64RegClass ||
1079  RegClass == &AMDGPU::SGPR_64RegClass ||
1080  RegClass == &AMDGPU::SReg_64_XEXECRegClass) {
1081  BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B64), DestReg)
1082  .addImm(Value);
1083  return;
1084  }
1085 
1086  if (RegClass == &AMDGPU::VGPR_32RegClass) {
1087  BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DestReg)
1088  .addImm(Value);
1089  return;
1090  }
1091  if (RegClass->hasSuperClassEq(&AMDGPU::VReg_64RegClass)) {
1092  BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B64_PSEUDO), DestReg)
1093  .addImm(Value);
1094  return;
1095  }
1096 
1097  unsigned EltSize = 4;
1098  unsigned Opcode = AMDGPU::V_MOV_B32_e32;
1099  if (RI.isSGPRClass(RegClass)) {
1100  if (RI.getRegSizeInBits(*RegClass) > 32) {
1101  Opcode = AMDGPU::S_MOV_B64;
1102  EltSize = 8;
1103  } else {
1104  Opcode = AMDGPU::S_MOV_B32;
1105  EltSize = 4;
1106  }
1107  }
1108 
1109  ArrayRef<int16_t> SubIndices = RI.getRegSplitParts(RegClass, EltSize);
1110  for (unsigned Idx = 0; Idx < SubIndices.size(); ++Idx) {
1111  int64_t IdxValue = Idx == 0 ? Value : 0;
1112 
1114  get(Opcode), RI.getSubReg(DestReg, SubIndices[Idx]));
1115  Builder.addImm(IdxValue);
1116  }
1117 }
1118 
1119 const TargetRegisterClass *
1121  return &AMDGPU::VGPR_32RegClass;
1122 }
1123 
1126  const DebugLoc &DL, Register DstReg,
1128  Register TrueReg,
1129  Register FalseReg) const {
1131  const TargetRegisterClass *BoolXExecRC =
1132  RI.getRegClass(AMDGPU::SReg_1_XEXECRegClassID);
1133  assert(MRI.getRegClass(DstReg) == &AMDGPU::VGPR_32RegClass &&
1134  "Not a VGPR32 reg");
1135 
1136  if (Cond.size() == 1) {
1137  Register SReg = MRI.createVirtualRegister(BoolXExecRC);
1138  BuildMI(MBB, I, DL, get(AMDGPU::COPY), SReg)
1139  .add(Cond[0]);
1140  BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
1141  .addImm(0)
1142  .addReg(FalseReg)
1143  .addImm(0)
1144  .addReg(TrueReg)
1145  .addReg(SReg);
1146  } else if (Cond.size() == 2) {
1147  assert(Cond[0].isImm() && "Cond[0] is not an immediate");
1148  switch (Cond[0].getImm()) {
1149  case SIInstrInfo::SCC_TRUE: {
1150  Register SReg = MRI.createVirtualRegister(BoolXExecRC);
1151  BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_CSELECT_B32
1152  : AMDGPU::S_CSELECT_B64), SReg)
1153  .addImm(1)
1154  .addImm(0);
1155  BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
1156  .addImm(0)
1157  .addReg(FalseReg)
1158  .addImm(0)
1159  .addReg(TrueReg)
1160  .addReg(SReg);
1161  break;
1162  }
1163  case SIInstrInfo::SCC_FALSE: {
1164  Register SReg = MRI.createVirtualRegister(BoolXExecRC);
1165  BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_CSELECT_B32
1166  : AMDGPU::S_CSELECT_B64), SReg)
1167  .addImm(0)
1168  .addImm(1);
1169  BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
1170  .addImm(0)
1171  .addReg(FalseReg)
1172  .addImm(0)
1173  .addReg(TrueReg)
1174  .addReg(SReg);
1175  break;
1176  }
1177  case SIInstrInfo::VCCNZ: {
1178  MachineOperand RegOp = Cond[1];
1179  RegOp.setImplicit(false);
1180  Register SReg = MRI.createVirtualRegister(BoolXExecRC);
1181  BuildMI(MBB, I, DL, get(AMDGPU::COPY), SReg)
1182  .add(RegOp);
1183  BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
1184  .addImm(0)
1185  .addReg(FalseReg)
1186  .addImm(0)
1187  .addReg(TrueReg)
1188  .addReg(SReg);
1189  break;
1190  }
1191  case SIInstrInfo::VCCZ: {
1192  MachineOperand RegOp = Cond[1];
1193  RegOp.setImplicit(false);
1194  Register SReg = MRI.createVirtualRegister(BoolXExecRC);
1195  BuildMI(MBB, I, DL, get(AMDGPU::COPY), SReg)
1196  .add(RegOp);
1197  BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
1198  .addImm(0)
1199  .addReg(TrueReg)
1200  .addImm(0)
1201  .addReg(FalseReg)
1202  .addReg(SReg);
1203  break;
1204  }
1205  case SIInstrInfo::EXECNZ: {
1206  Register SReg = MRI.createVirtualRegister(BoolXExecRC);
1208  BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_OR_SAVEEXEC_B32
1209  : AMDGPU::S_OR_SAVEEXEC_B64), SReg2)
1210  .addImm(0);
1211  BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_CSELECT_B32
1212  : AMDGPU::S_CSELECT_B64), SReg)
1213  .addImm(1)
1214  .addImm(0);
1215  BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
1216  .addImm(0)
1217  .addReg(FalseReg)
1218  .addImm(0)
1219  .addReg(TrueReg)
1220  .addReg(SReg);
1221  break;
1222  }
1223  case SIInstrInfo::EXECZ: {
1224  Register SReg = MRI.createVirtualRegister(BoolXExecRC);
1226  BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_OR_SAVEEXEC_B32
1227  : AMDGPU::S_OR_SAVEEXEC_B64), SReg2)
1228  .addImm(0);
1229  BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_CSELECT_B32
1230  : AMDGPU::S_CSELECT_B64), SReg)
1231  .addImm(0)
1232  .addImm(1);
1233  BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
1234  .addImm(0)
1235  .addReg(FalseReg)
1236  .addImm(0)
1237  .addReg(TrueReg)
1238  .addReg(SReg);
1239  llvm_unreachable("Unhandled branch predicate EXECZ");
1240  break;
1241  }
1242  default:
1243  llvm_unreachable("invalid branch predicate");
1244  }
1245  } else {
1246  llvm_unreachable("Can only handle Cond size 1 or 2");
1247  }
1248 }
1249 
1252  const DebugLoc &DL,
1253  Register SrcReg, int Value) const {
1256  BuildMI(*MBB, I, DL, get(AMDGPU::V_CMP_EQ_I32_e64), Reg)
1257  .addImm(Value)
1258  .addReg(SrcReg);
1259 
1260  return Reg;
1261 }
1262 
1265  const DebugLoc &DL,
1266  Register SrcReg, int Value) const {
1269  BuildMI(*MBB, I, DL, get(AMDGPU::V_CMP_NE_I32_e64), Reg)
1270  .addImm(Value)
1271  .addReg(SrcReg);
1272 
1273  return Reg;
1274 }
1275 
1276 unsigned SIInstrInfo::getMovOpcode(const TargetRegisterClass *DstRC) const {
1277 
1278  if (RI.isAGPRClass(DstRC))
1279  return AMDGPU::COPY;
1280  if (RI.getRegSizeInBits(*DstRC) == 32) {
1281  return RI.isSGPRClass(DstRC) ? AMDGPU::S_MOV_B32 : AMDGPU::V_MOV_B32_e32;
1282  } else if (RI.getRegSizeInBits(*DstRC) == 64 && RI.isSGPRClass(DstRC)) {
1283  return AMDGPU::S_MOV_B64;
1284  } else if (RI.getRegSizeInBits(*DstRC) == 64 && !RI.isSGPRClass(DstRC)) {
1285  return AMDGPU::V_MOV_B64_PSEUDO;
1286  }
1287  return AMDGPU::COPY;
1288 }
1289 
1290 const MCInstrDesc &
1292  bool IsIndirectSrc) const {
1293  if (IsIndirectSrc) {
1294  if (VecSize <= 32) // 4 bytes
1295  return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V1);
1296  if (VecSize <= 64) // 8 bytes
1297  return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V2);
1298  if (VecSize <= 96) // 12 bytes
1299  return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V3);
1300  if (VecSize <= 128) // 16 bytes
1301  return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V4);
1302  if (VecSize <= 160) // 20 bytes
1303  return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V5);
1304  if (VecSize <= 256) // 32 bytes
1305  return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V8);
1306  if (VecSize <= 512) // 64 bytes
1307  return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V16);
1308  if (VecSize <= 1024) // 128 bytes
1309  return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V32);
1310 
1311  llvm_unreachable("unsupported size for IndirectRegReadGPRIDX pseudos");
1312  }
1313 
1314  if (VecSize <= 32) // 4 bytes
1315  return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V1);
1316  if (VecSize <= 64) // 8 bytes
1317  return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V2);
1318  if (VecSize <= 96) // 12 bytes
1319  return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V3);
1320  if (VecSize <= 128) // 16 bytes
1321  return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V4);
1322  if (VecSize <= 160) // 20 bytes
1323  return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V5);
1324  if (VecSize <= 256) // 32 bytes
1325  return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V8);
1326  if (VecSize <= 512) // 64 bytes
1327  return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V16);
1328  if (VecSize <= 1024) // 128 bytes
1329  return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V32);
1330 
1331  llvm_unreachable("unsupported size for IndirectRegWriteGPRIDX pseudos");
1332 }
1333 
1335  if (VecSize <= 32) // 4 bytes
1336  return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V1;
1337  if (VecSize <= 64) // 8 bytes
1338  return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V2;
1339  if (VecSize <= 96) // 12 bytes
1340  return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V3;
1341  if (VecSize <= 128) // 16 bytes
1342  return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V4;
1343  if (VecSize <= 160) // 20 bytes
1344  return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V5;
1345  if (VecSize <= 256) // 32 bytes
1346  return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V8;
1347  if (VecSize <= 512) // 64 bytes
1348  return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V16;
1349  if (VecSize <= 1024) // 128 bytes
1350  return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V32;
1351 
1352  llvm_unreachable("unsupported size for IndirectRegWrite pseudos");
1353 }
1354 
1355 static unsigned getIndirectSGPRWriteMovRelPseudo32(unsigned VecSize) {
1356  if (VecSize <= 32) // 4 bytes
1357  return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V1;
1358  if (VecSize <= 64) // 8 bytes
1359  return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V2;
1360  if (VecSize <= 96) // 12 bytes
1361  return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V3;
1362  if (VecSize <= 128) // 16 bytes
1363  return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V4;
1364  if (VecSize <= 160) // 20 bytes
1365  return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V5;
1366  if (VecSize <= 256) // 32 bytes
1367  return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V8;
1368  if (VecSize <= 512) // 64 bytes
1369  return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V16;
1370  if (VecSize <= 1024) // 128 bytes
1371  return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V32;
1372 
1373  llvm_unreachable("unsupported size for IndirectRegWrite pseudos");
1374 }
1375 
1376 static unsigned getIndirectSGPRWriteMovRelPseudo64(unsigned VecSize) {
1377  if (VecSize <= 64) // 8 bytes
1378  return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V1;
1379  if (VecSize <= 128) // 16 bytes
1380  return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V2;
1381  if (VecSize <= 256) // 32 bytes
1382  return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V4;
1383  if (VecSize <= 512) // 64 bytes
1384  return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V8;
1385  if (VecSize <= 1024) // 128 bytes
1386  return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V16;
1387 
1388  llvm_unreachable("unsupported size for IndirectRegWrite pseudos");
1389 }
1390 
1391 const MCInstrDesc &
1393  bool IsSGPR) const {
1394  if (IsSGPR) {
1395  switch (EltSize) {
1396  case 32:
1398  case 64:
1400  default:
1401  llvm_unreachable("invalid reg indexing elt size");
1402  }
1403  }
1404 
1405  assert(EltSize == 32 && "invalid reg indexing elt size");
1407 }
1408 
1409 static unsigned getSGPRSpillSaveOpcode(unsigned Size) {
1410  switch (Size) {
1411  case 4:
1412  return AMDGPU::SI_SPILL_S32_SAVE;
1413  case 8:
1414  return AMDGPU::SI_SPILL_S64_SAVE;
1415  case 12:
1416  return AMDGPU::SI_SPILL_S96_SAVE;
1417  case 16:
1418  return AMDGPU::SI_SPILL_S128_SAVE;
1419  case 20:
1420  return AMDGPU::SI_SPILL_S160_SAVE;
1421  case 24:
1422  return AMDGPU::SI_SPILL_S192_SAVE;
1423  case 28:
1424  return AMDGPU::SI_SPILL_S224_SAVE;
1425  case 32:
1426  return AMDGPU::SI_SPILL_S256_SAVE;
1427  case 64:
1428  return AMDGPU::SI_SPILL_S512_SAVE;
1429  case 128:
1430  return AMDGPU::SI_SPILL_S1024_SAVE;
1431  default:
1432  llvm_unreachable("unknown register size");
1433  }
1434 }
1435 
1436 static unsigned getVGPRSpillSaveOpcode(unsigned Size) {
1437  switch (Size) {
1438  case 4:
1439  return AMDGPU::SI_SPILL_V32_SAVE;
1440  case 8:
1441  return AMDGPU::SI_SPILL_V64_SAVE;
1442  case 12:
1443  return AMDGPU::SI_SPILL_V96_SAVE;
1444  case 16:
1445  return AMDGPU::SI_SPILL_V128_SAVE;
1446  case 20:
1447  return AMDGPU::SI_SPILL_V160_SAVE;
1448  case 24:
1449  return AMDGPU::SI_SPILL_V192_SAVE;
1450  case 28:
1451  return AMDGPU::SI_SPILL_V224_SAVE;
1452  case 32:
1453  return AMDGPU::SI_SPILL_V256_SAVE;
1454  case 64:
1455  return AMDGPU::SI_SPILL_V512_SAVE;
1456  case 128:
1457  return AMDGPU::SI_SPILL_V1024_SAVE;
1458  default:
1459  llvm_unreachable("unknown register size");
1460  }
1461 }
1462 
1463 static unsigned getAGPRSpillSaveOpcode(unsigned Size) {
1464  switch (Size) {
1465  case 4:
1466  return AMDGPU::SI_SPILL_A32_SAVE;
1467  case 8:
1468  return AMDGPU::SI_SPILL_A64_SAVE;
1469  case 12:
1470  return AMDGPU::SI_SPILL_A96_SAVE;
1471  case 16:
1472  return AMDGPU::SI_SPILL_A128_SAVE;
1473  case 20:
1474  return AMDGPU::SI_SPILL_A160_SAVE;
1475  case 24:
1476  return AMDGPU::SI_SPILL_A192_SAVE;
1477  case 28:
1478  return AMDGPU::SI_SPILL_A224_SAVE;
1479  case 32:
1480  return AMDGPU::SI_SPILL_A256_SAVE;
1481  case 64:
1482  return AMDGPU::SI_SPILL_A512_SAVE;
1483  case 128:
1484  return AMDGPU::SI_SPILL_A1024_SAVE;
1485  default:
1486  llvm_unreachable("unknown register size");
1487  }
1488 }
1489 
1490 static unsigned getAVSpillSaveOpcode(unsigned Size) {
1491  switch (Size) {
1492  case 4:
1493  return AMDGPU::SI_SPILL_AV32_SAVE;
1494  case 8:
1495  return AMDGPU::SI_SPILL_AV64_SAVE;
1496  case 12:
1497  return AMDGPU::SI_SPILL_AV96_SAVE;
1498  case 16:
1499  return AMDGPU::SI_SPILL_AV128_SAVE;
1500  case 20:
1501  return AMDGPU::SI_SPILL_AV160_SAVE;
1502  case 24:
1503  return AMDGPU::SI_SPILL_AV192_SAVE;
1504  case 28:
1505  return AMDGPU::SI_SPILL_AV224_SAVE;
1506  case 32:
1507  return AMDGPU::SI_SPILL_AV256_SAVE;
1508  case 64:
1509  return AMDGPU::SI_SPILL_AV512_SAVE;
1510  case 128:
1511  return AMDGPU::SI_SPILL_AV1024_SAVE;
1512  default:
1513  llvm_unreachable("unknown register size");
1514  }
1515 }
1516 
1519  Register SrcReg, bool isKill,
1520  int FrameIndex,
1521  const TargetRegisterClass *RC,
1522  const TargetRegisterInfo *TRI) const {
1523  MachineFunction *MF = MBB.getParent();
1525  MachineFrameInfo &FrameInfo = MF->getFrameInfo();
1526  const DebugLoc &DL = MBB.findDebugLoc(MI);
1527 
1528  MachinePointerInfo PtrInfo
1531  PtrInfo, MachineMemOperand::MOStore, FrameInfo.getObjectSize(FrameIndex),
1532  FrameInfo.getObjectAlign(FrameIndex));
1533  unsigned SpillSize = TRI->getSpillSize(*RC);
1534 
1536  if (RI.isSGPRClass(RC)) {
1537  MFI->setHasSpilledSGPRs();
1538  assert(SrcReg != AMDGPU::M0 && "m0 should not be spilled");
1539  assert(SrcReg != AMDGPU::EXEC_LO && SrcReg != AMDGPU::EXEC_HI &&
1540  SrcReg != AMDGPU::EXEC && "exec should not be spilled");
1541 
1542  // We are only allowed to create one new instruction when spilling
1543  // registers, so we need to use pseudo instruction for spilling SGPRs.
1544  const MCInstrDesc &OpDesc = get(getSGPRSpillSaveOpcode(SpillSize));
1545 
1546  // The SGPR spill/restore instructions only work on number sgprs, so we need
1547  // to make sure we are using the correct register class.
1548  if (SrcReg.isVirtual() && SpillSize == 4) {
1549  MRI.constrainRegClass(SrcReg, &AMDGPU::SReg_32_XM0_XEXECRegClass);
1550  }
1551 
1552  BuildMI(MBB, MI, DL, OpDesc)
1553  .addReg(SrcReg, getKillRegState(isKill)) // data
1554  .addFrameIndex(FrameIndex) // addr
1555  .addMemOperand(MMO)
1557 
1558  if (RI.spillSGPRToVGPR())
1559  FrameInfo.setStackID(FrameIndex, TargetStackID::SGPRSpill);
1560  return;
1561  }
1562 
1563  unsigned Opcode = RI.isVectorSuperClass(RC) ? getAVSpillSaveOpcode(SpillSize)
1564  : RI.isAGPRClass(RC) ? getAGPRSpillSaveOpcode(SpillSize)
1565  : getVGPRSpillSaveOpcode(SpillSize);
1566  MFI->setHasSpilledVGPRs();
1567 
1568  BuildMI(MBB, MI, DL, get(Opcode))
1569  .addReg(SrcReg, getKillRegState(isKill)) // data
1570  .addFrameIndex(FrameIndex) // addr
1571  .addReg(MFI->getStackPtrOffsetReg()) // scratch_offset
1572  .addImm(0) // offset
1573  .addMemOperand(MMO);
1574 }
1575 
1576 static unsigned getSGPRSpillRestoreOpcode(unsigned Size) {
1577  switch (Size) {
1578  case 4:
1579  return AMDGPU::SI_SPILL_S32_RESTORE;
1580  case 8:
1581  return AMDGPU::SI_SPILL_S64_RESTORE;
1582  case 12:
1583  return AMDGPU::SI_SPILL_S96_RESTORE;
1584  case 16:
1585  return AMDGPU::SI_SPILL_S128_RESTORE;
1586  case 20:
1587  return AMDGPU::SI_SPILL_S160_RESTORE;
1588  case 24:
1589  return AMDGPU::SI_SPILL_S192_RESTORE;
1590  case 28:
1591  return AMDGPU::SI_SPILL_S224_RESTORE;
1592  case 32:
1593  return AMDGPU::SI_SPILL_S256_RESTORE;
1594  case 64:
1595  return AMDGPU::SI_SPILL_S512_RESTORE;
1596  case 128:
1597  return AMDGPU::SI_SPILL_S1024_RESTORE;
1598  default:
1599  llvm_unreachable("unknown register size");
1600  }
1601 }
1602 
1603 static unsigned getVGPRSpillRestoreOpcode(unsigned Size) {
1604  switch (Size) {
1605  case 4:
1606  return AMDGPU::SI_SPILL_V32_RESTORE;
1607  case 8:
1608  return AMDGPU::SI_SPILL_V64_RESTORE;
1609  case 12:
1610  return AMDGPU::SI_SPILL_V96_RESTORE;
1611  case 16:
1612  return AMDGPU::SI_SPILL_V128_RESTORE;
1613  case 20:
1614  return AMDGPU::SI_SPILL_V160_RESTORE;
1615  case 24:
1616  return AMDGPU::SI_SPILL_V192_RESTORE;
1617  case 28:
1618  return AMDGPU::SI_SPILL_V224_RESTORE;
1619  case 32:
1620  return AMDGPU::SI_SPILL_V256_RESTORE;
1621  case 64:
1622  return AMDGPU::SI_SPILL_V512_RESTORE;
1623  case 128:
1624  return AMDGPU::SI_SPILL_V1024_RESTORE;
1625  default:
1626  llvm_unreachable("unknown register size");
1627  }
1628 }
1629 
1630 static unsigned getAGPRSpillRestoreOpcode(unsigned Size) {
1631  switch (Size) {
1632  case 4:
1633  return AMDGPU::SI_SPILL_A32_RESTORE;
1634  case 8:
1635  return AMDGPU::SI_SPILL_A64_RESTORE;
1636  case 12:
1637  return AMDGPU::SI_SPILL_A96_RESTORE;
1638  case 16:
1639  return AMDGPU::SI_SPILL_A128_RESTORE;
1640  case 20:
1641  return AMDGPU::SI_SPILL_A160_RESTORE;
1642  case 24:
1643  return AMDGPU::SI_SPILL_A192_RESTORE;
1644  case 28:
1645  return AMDGPU::SI_SPILL_A224_RESTORE;
1646  case 32:
1647  return AMDGPU::SI_SPILL_A256_RESTORE;
1648  case 64:
1649  return AMDGPU::SI_SPILL_A512_RESTORE;
1650  case 128:
1651  return AMDGPU::SI_SPILL_A1024_RESTORE;
1652  default:
1653  llvm_unreachable("unknown register size");
1654  }
1655 }
1656 
1657 static unsigned getAVSpillRestoreOpcode(unsigned Size) {
1658  switch (Size) {
1659  case 4:
1660  return AMDGPU::SI_SPILL_AV32_RESTORE;
1661  case 8:
1662  return AMDGPU::SI_SPILL_AV64_RESTORE;
1663  case 12:
1664  return AMDGPU::SI_SPILL_AV96_RESTORE;
1665  case 16:
1666  return AMDGPU::SI_SPILL_AV128_RESTORE;
1667  case 20:
1668  return AMDGPU::SI_SPILL_AV160_RESTORE;
1669  case 24:
1670  return AMDGPU::SI_SPILL_AV192_RESTORE;
1671  case 28:
1672  return AMDGPU::SI_SPILL_AV224_RESTORE;
1673  case 32:
1674  return AMDGPU::SI_SPILL_AV256_RESTORE;
1675  case 64:
1676  return AMDGPU::SI_SPILL_AV512_RESTORE;
1677  case 128:
1678  return AMDGPU::SI_SPILL_AV1024_RESTORE;
1679  default:
1680  llvm_unreachable("unknown register size");
1681  }
1682 }
1683 
1686  Register DestReg, int FrameIndex,
1687  const TargetRegisterClass *RC,
1688  const TargetRegisterInfo *TRI) const {
1689  MachineFunction *MF = MBB.getParent();
1691  MachineFrameInfo &FrameInfo = MF->getFrameInfo();
1692  const DebugLoc &DL = MBB.findDebugLoc(MI);
1693  unsigned SpillSize = TRI->getSpillSize(*RC);
1694 
1695  MachinePointerInfo PtrInfo
1697 
1699  PtrInfo, MachineMemOperand::MOLoad, FrameInfo.getObjectSize(FrameIndex),
1700  FrameInfo.getObjectAlign(FrameIndex));
1701 
1702  if (RI.isSGPRClass(RC)) {
1703  MFI->setHasSpilledSGPRs();
1704  assert(DestReg != AMDGPU::M0 && "m0 should not be reloaded into");
1705  assert(DestReg != AMDGPU::EXEC_LO && DestReg != AMDGPU::EXEC_HI &&
1706  DestReg != AMDGPU::EXEC && "exec should not be spilled");
1707 
1708  // FIXME: Maybe this should not include a memoperand because it will be
1709  // lowered to non-memory instructions.
1710  const MCInstrDesc &OpDesc = get(getSGPRSpillRestoreOpcode(SpillSize));
1711  if (DestReg.isVirtual() && SpillSize == 4) {
1713  MRI.constrainRegClass(DestReg, &AMDGPU::SReg_32_XM0_XEXECRegClass);
1714  }
1715 
1716  if (RI.spillSGPRToVGPR())
1717  FrameInfo.setStackID(FrameIndex, TargetStackID::SGPRSpill);
1718  BuildMI(MBB, MI, DL, OpDesc, DestReg)
1719  .addFrameIndex(FrameIndex) // addr
1720  .addMemOperand(MMO)
1722 
1723  return;
1724  }
1725 
1726  unsigned Opcode = RI.isVectorSuperClass(RC)
1727  ? getAVSpillRestoreOpcode(SpillSize)
1728  : RI.isAGPRClass(RC) ? getAGPRSpillRestoreOpcode(SpillSize)
1729  : getVGPRSpillRestoreOpcode(SpillSize);
1730  BuildMI(MBB, MI, DL, get(Opcode), DestReg)
1731  .addFrameIndex(FrameIndex) // vaddr
1732  .addReg(MFI->getStackPtrOffsetReg()) // scratch_offset
1733  .addImm(0) // offset
1734  .addMemOperand(MMO);
1735 }
1736 
1739  insertNoops(MBB, MI, 1);
1740 }
1741 
1744  unsigned Quantity) const {
1746  while (Quantity > 0) {
1747  unsigned Arg = std::min(Quantity, 8u);
1748  Quantity -= Arg;
1749  BuildMI(MBB, MI, DL, get(AMDGPU::S_NOP)).addImm(Arg - 1);
1750  }
1751 }
1752 
1754  auto MF = MBB.getParent();
1756 
1757  assert(Info->isEntryFunction());
1758 
1759  if (MBB.succ_empty()) {
1760  bool HasNoTerminator = MBB.getFirstTerminator() == MBB.end();
1761  if (HasNoTerminator) {
1762  if (Info->returnsVoid()) {
1763  BuildMI(MBB, MBB.end(), DebugLoc(), get(AMDGPU::S_ENDPGM)).addImm(0);
1764  } else {
1765  BuildMI(MBB, MBB.end(), DebugLoc(), get(AMDGPU::SI_RETURN_TO_EPILOG));
1766  }
1767  }
1768  }
1769 }
1770 
1772  switch (MI.getOpcode()) {
1773  default:
1774  if (MI.isMetaInstruction())
1775  return 0;
1776  return 1; // FIXME: Do wait states equal cycles?
1777 
1778  case AMDGPU::S_NOP:
1779  return MI.getOperand(0).getImm() + 1;
1780  // SI_RETURN_TO_EPILOG is a fallthrough to code outside of the function. The
1781  // hazard, even if one exist, won't really be visible. Should we handle it?
1782  }
1783 }
1784 
1786  const SIRegisterInfo *TRI = ST.getRegisterInfo();
1787  MachineBasicBlock &MBB = *MI.getParent();
1789  switch (MI.getOpcode()) {
1790  default: return TargetInstrInfo::expandPostRAPseudo(MI);
1791  case AMDGPU::S_MOV_B64_term:
1792  // This is only a terminator to get the correct spill code placement during
1793  // register allocation.
1794  MI.setDesc(get(AMDGPU::S_MOV_B64));
1795  break;
1796 
1797  case AMDGPU::S_MOV_B32_term:
1798  // This is only a terminator to get the correct spill code placement during
1799  // register allocation.
1800  MI.setDesc(get(AMDGPU::S_MOV_B32));
1801  break;
1802 
1803  case AMDGPU::S_XOR_B64_term:
1804  // This is only a terminator to get the correct spill code placement during
1805  // register allocation.
1806  MI.setDesc(get(AMDGPU::S_XOR_B64));
1807  break;
1808 
1809  case AMDGPU::S_XOR_B32_term:
1810  // This is only a terminator to get the correct spill code placement during
1811  // register allocation.
1812  MI.setDesc(get(AMDGPU::S_XOR_B32));
1813  break;
1814  case AMDGPU::S_OR_B64_term:
1815  // This is only a terminator to get the correct spill code placement during
1816  // register allocation.
1817  MI.setDesc(get(AMDGPU::S_OR_B64));
1818  break;
1819  case AMDGPU::S_OR_B32_term:
1820  // This is only a terminator to get the correct spill code placement during
1821  // register allocation.
1822  MI.setDesc(get(AMDGPU::S_OR_B32));
1823  break;
1824 
1825  case AMDGPU::S_ANDN2_B64_term:
1826  // This is only a terminator to get the correct spill code placement during
1827  // register allocation.
1828  MI.setDesc(get(AMDGPU::S_ANDN2_B64));
1829  break;
1830 
1831  case AMDGPU::S_ANDN2_B32_term:
1832  // This is only a terminator to get the correct spill code placement during
1833  // register allocation.
1834  MI.setDesc(get(AMDGPU::S_ANDN2_B32));
1835  break;
1836 
1837  case AMDGPU::S_AND_B64_term:
1838  // This is only a terminator to get the correct spill code placement during
1839  // register allocation.
1840  MI.setDesc(get(AMDGPU::S_AND_B64));
1841  break;
1842 
1843  case AMDGPU::S_AND_B32_term:
1844  // This is only a terminator to get the correct spill code placement during
1845  // register allocation.
1846  MI.setDesc(get(AMDGPU::S_AND_B32));
1847  break;
1848 
1849  case AMDGPU::V_MOV_B64_PSEUDO: {
1850  Register Dst = MI.getOperand(0).getReg();
1851  Register DstLo = RI.getSubReg(Dst, AMDGPU::sub0);
1852  Register DstHi = RI.getSubReg(Dst, AMDGPU::sub1);
1853 
1854  const MachineOperand &SrcOp = MI.getOperand(1);
1855  // FIXME: Will this work for 64-bit floating point immediates?
1856  assert(!SrcOp.isFPImm());
1857  if (ST.hasMovB64()) {
1858  MI.setDesc(get(AMDGPU::V_MOV_B64_e32));
1859  if (!isLiteralConstant(MI, 1) || isUInt<32>(SrcOp.getImm()))
1860  break;
1861  }
1862  if (SrcOp.isImm()) {
1863  APInt Imm(64, SrcOp.getImm());
1864  APInt Lo(32, Imm.getLoBits(32).getZExtValue());
1865  APInt Hi(32, Imm.getHiBits(32).getZExtValue());
1866  if (ST.hasPackedFP32Ops() && Lo == Hi && isInlineConstant(Lo)) {
1867  BuildMI(MBB, MI, DL, get(AMDGPU::V_PK_MOV_B32), Dst)
1869  .addImm(Lo.getSExtValue())
1871  .addImm(Lo.getSExtValue())
1872  .addImm(0) // op_sel_lo
1873  .addImm(0) // op_sel_hi
1874  .addImm(0) // neg_lo
1875  .addImm(0) // neg_hi
1876  .addImm(0); // clamp
1877  } else {
1878  BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstLo)
1879  .addImm(Lo.getSExtValue())
1881  BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstHi)
1882  .addImm(Hi.getSExtValue())
1884  }
1885  } else {
1886  assert(SrcOp.isReg());
1887  if (ST.hasPackedFP32Ops() &&
1888  !RI.isAGPR(MBB.getParent()->getRegInfo(), SrcOp.getReg())) {
1889  BuildMI(MBB, MI, DL, get(AMDGPU::V_PK_MOV_B32), Dst)
1890  .addImm(SISrcMods::OP_SEL_1) // src0_mod
1891  .addReg(SrcOp.getReg())
1893  .addReg(SrcOp.getReg())
1894  .addImm(0) // op_sel_lo
1895  .addImm(0) // op_sel_hi
1896  .addImm(0) // neg_lo
1897  .addImm(0) // neg_hi
1898  .addImm(0); // clamp
1899  } else {
1900  BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstLo)
1901  .addReg(RI.getSubReg(SrcOp.getReg(), AMDGPU::sub0))
1903  BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstHi)
1904  .addReg(RI.getSubReg(SrcOp.getReg(), AMDGPU::sub1))
1906  }
1907  }
1908  MI.eraseFromParent();
1909  break;
1910  }
1911  case AMDGPU::V_MOV_B64_DPP_PSEUDO: {
1912  expandMovDPP64(MI);
1913  break;
1914  }
1915  case AMDGPU::S_MOV_B64_IMM_PSEUDO: {
1916  const MachineOperand &SrcOp = MI.getOperand(1);
1917  assert(!SrcOp.isFPImm());
1918  APInt Imm(64, SrcOp.getImm());
1919  if (Imm.isIntN(32) || isInlineConstant(Imm)) {
1920  MI.setDesc(get(AMDGPU::S_MOV_B64));
1921  break;
1922  }
1923 
1924  Register Dst = MI.getOperand(0).getReg();
1925  Register DstLo = RI.getSubReg(Dst, AMDGPU::sub0);
1926  Register DstHi = RI.getSubReg(Dst, AMDGPU::sub1);
1927 
1928  APInt Lo(32, Imm.getLoBits(32).getZExtValue());
1929  APInt Hi(32, Imm.getHiBits(32).getZExtValue());
1930  BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), DstLo)
1931  .addImm(Lo.getSExtValue())
1933  BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), DstHi)
1934  .addImm(Hi.getSExtValue())
1936  MI.eraseFromParent();
1937  break;
1938  }
1939  case AMDGPU::V_SET_INACTIVE_B32: {
1940  unsigned NotOpc = ST.isWave32() ? AMDGPU::S_NOT_B32 : AMDGPU::S_NOT_B64;
1941  unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
1942  // FIXME: We may possibly optimize the COPY once we find ways to make LLVM
1943  // optimizations (mainly Register Coalescer) aware of WWM register liveness.
1944  BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), MI.getOperand(0).getReg())
1945  .add(MI.getOperand(1));
1946  auto FirstNot = BuildMI(MBB, MI, DL, get(NotOpc), Exec).addReg(Exec);
1947  FirstNot->addRegisterDead(AMDGPU::SCC, TRI); // SCC is overwritten
1948  BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), MI.getOperand(0).getReg())
1949  .add(MI.getOperand(2));
1950  BuildMI(MBB, MI, DL, get(NotOpc), Exec)
1951  .addReg(Exec);
1952  MI.eraseFromParent();
1953  break;
1954  }
1955  case AMDGPU::V_SET_INACTIVE_B64: {
1956  unsigned NotOpc = ST.isWave32() ? AMDGPU::S_NOT_B32 : AMDGPU::S_NOT_B64;
1957  unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
1958  MachineInstr *Copy = BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B64_PSEUDO),
1959  MI.getOperand(0).getReg())
1960  .add(MI.getOperand(1));
1961  expandPostRAPseudo(*Copy);
1962  auto FirstNot = BuildMI(MBB, MI, DL, get(NotOpc), Exec).addReg(Exec);
1963  FirstNot->addRegisterDead(AMDGPU::SCC, TRI); // SCC is overwritten
1964  Copy = BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B64_PSEUDO),
1965  MI.getOperand(0).getReg())
1966  .add(MI.getOperand(2));
1967  expandPostRAPseudo(*Copy);
1968  BuildMI(MBB, MI, DL, get(NotOpc), Exec)
1969  .addReg(Exec);
1970  MI.eraseFromParent();
1971  break;
1972  }
1973  case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V1:
1974  case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V2:
1975  case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V3:
1976  case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V4:
1977  case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V5:
1978  case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V8:
1979  case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V16:
1980  case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V32:
1981  case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V1:
1982  case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V2:
1983  case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V3:
1984  case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V4:
1985  case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V5:
1986  case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V8:
1987  case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V16:
1988  case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V32:
1989  case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V1:
1990  case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V2:
1991  case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V4:
1992  case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V8:
1993  case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V16: {
1994  const TargetRegisterClass *EltRC = getOpRegClass(MI, 2);
1995 
1996  unsigned Opc;
1997  if (RI.hasVGPRs(EltRC)) {
1998  Opc = AMDGPU::V_MOVRELD_B32_e32;
1999  } else {
2000  Opc = RI.getRegSizeInBits(*EltRC) == 64 ? AMDGPU::S_MOVRELD_B64
2001  : AMDGPU::S_MOVRELD_B32;
2002  }
2003 
2004  const MCInstrDesc &OpDesc = get(Opc);
2005  Register VecReg = MI.getOperand(0).getReg();
2006  bool IsUndef = MI.getOperand(1).isUndef();
2007  unsigned SubReg = MI.getOperand(3).getImm();
2008  assert(VecReg == MI.getOperand(1).getReg());
2009 
2010  MachineInstrBuilder MIB =
2011  BuildMI(MBB, MI, DL, OpDesc)
2012  .addReg(RI.getSubReg(VecReg, SubReg), RegState::Undef)
2013  .add(MI.getOperand(2))
2015  .addReg(VecReg, RegState::Implicit | (IsUndef ? RegState::Undef : 0));
2016 
2017  const int ImpDefIdx =
2018  OpDesc.getNumOperands() + OpDesc.getNumImplicitUses();
2019  const int ImpUseIdx = ImpDefIdx + 1;
2020  MIB->tieOperands(ImpDefIdx, ImpUseIdx);
2021  MI.eraseFromParent();
2022  break;
2023  }
2024  case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V1:
2025  case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V2:
2026  case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V3:
2027  case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V4:
2028  case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V5:
2029  case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V8:
2030  case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V16:
2031  case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V32: {
2032  assert(ST.useVGPRIndexMode());
2033  Register VecReg = MI.getOperand(0).getReg();
2034  bool IsUndef = MI.getOperand(1).isUndef();
2035  Register Idx = MI.getOperand(3).getReg();
2036  Register SubReg = MI.getOperand(4).getImm();
2037 
2038  MachineInstr *SetOn = BuildMI(MBB, MI, DL, get(AMDGPU::S_SET_GPR_IDX_ON))
2039  .addReg(Idx)
2041  SetOn->getOperand(3).setIsUndef();
2042 
2043  const MCInstrDesc &OpDesc = get(AMDGPU::V_MOV_B32_indirect_write);
2044  MachineInstrBuilder MIB =
2045  BuildMI(MBB, MI, DL, OpDesc)
2046  .addReg(RI.getSubReg(VecReg, SubReg), RegState::Undef)
2047  .add(MI.getOperand(2))
2049  .addReg(VecReg,
2050  RegState::Implicit | (IsUndef ? RegState::Undef : 0));
2051 
2052  const int ImpDefIdx = OpDesc.getNumOperands() + OpDesc.getNumImplicitUses();
2053  const int ImpUseIdx = ImpDefIdx + 1;
2054  MIB->tieOperands(ImpDefIdx, ImpUseIdx);
2055 
2056  MachineInstr *SetOff = BuildMI(MBB, MI, DL, get(AMDGPU::S_SET_GPR_IDX_OFF));
2057 
2058  finalizeBundle(MBB, SetOn->getIterator(), std::next(SetOff->getIterator()));
2059 
2060  MI.eraseFromParent();
2061  break;
2062  }
2063  case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V1:
2064  case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V2:
2065  case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V3:
2066  case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V4:
2067  case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V5:
2068  case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V8:
2069  case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V16:
2070  case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V32: {
2071  assert(ST.useVGPRIndexMode());
2072  Register Dst = MI.getOperand(0).getReg();
2073  Register VecReg = MI.getOperand(1).getReg();
2074  bool IsUndef = MI.getOperand(1).isUndef();
2075  Register Idx = MI.getOperand(2).getReg();
2076  Register SubReg = MI.getOperand(3).getImm();
2077 
2078  MachineInstr *SetOn = BuildMI(MBB, MI, DL, get(AMDGPU::S_SET_GPR_IDX_ON))
2079  .addReg(Idx)
2081  SetOn->getOperand(3).setIsUndef();
2082 
2083  BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_indirect_read))
2084  .addDef(Dst)
2085  .addReg(RI.getSubReg(VecReg, SubReg), RegState::Undef)
2086  .addReg(VecReg, RegState::Implicit | (IsUndef ? RegState::Undef : 0));
2087 
2088  MachineInstr *SetOff = BuildMI(MBB, MI, DL, get(AMDGPU::S_SET_GPR_IDX_OFF));
2089 
2090  finalizeBundle(MBB, SetOn->getIterator(), std::next(SetOff->getIterator()));
2091 
2092  MI.eraseFromParent();
2093  break;
2094  }
2095  case AMDGPU::SI_PC_ADD_REL_OFFSET: {
2096  MachineFunction &MF = *MBB.getParent();
2097  Register Reg = MI.getOperand(0).getReg();
2098  Register RegLo = RI.getSubReg(Reg, AMDGPU::sub0);
2099  Register RegHi = RI.getSubReg(Reg, AMDGPU::sub1);
2100 
2101  // Create a bundle so these instructions won't be re-ordered by the
2102  // post-RA scheduler.
2103  MIBundleBuilder Bundler(MBB, MI);
2104  Bundler.append(BuildMI(MF, DL, get(AMDGPU::S_GETPC_B64), Reg));
2105 
2106  // Add 32-bit offset from this instruction to the start of the
2107  // constant data.
2108  Bundler.append(BuildMI(MF, DL, get(AMDGPU::S_ADD_U32), RegLo)
2109  .addReg(RegLo)
2110  .add(MI.getOperand(1)));
2111 
2112  MachineInstrBuilder MIB = BuildMI(MF, DL, get(AMDGPU::S_ADDC_U32), RegHi)
2113  .addReg(RegHi);
2114  MIB.add(MI.getOperand(2));
2115 
2116  Bundler.append(MIB);
2117  finalizeBundle(MBB, Bundler.begin());
2118 
2119  MI.eraseFromParent();
2120  break;
2121  }
2122  case AMDGPU::ENTER_STRICT_WWM: {
2123  // This only gets its own opcode so that SIPreAllocateWWMRegs can tell when
2124  // Whole Wave Mode is entered.
2125  MI.setDesc(get(ST.isWave32() ? AMDGPU::S_OR_SAVEEXEC_B32
2126  : AMDGPU::S_OR_SAVEEXEC_B64));
2127  break;
2128  }
2129  case AMDGPU::ENTER_STRICT_WQM: {
2130  // This only gets its own opcode so that SIPreAllocateWWMRegs can tell when
2131  // STRICT_WQM is entered.
2132  const unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
2133  const unsigned WQMOp = ST.isWave32() ? AMDGPU::S_WQM_B32 : AMDGPU::S_WQM_B64;
2134  const unsigned MovOp = ST.isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64;
2135  BuildMI(MBB, MI, DL, get(MovOp), MI.getOperand(0).getReg()).addReg(Exec);
2136  BuildMI(MBB, MI, DL, get(WQMOp), Exec).addReg(Exec);
2137 
2138  MI.eraseFromParent();
2139  break;
2140  }
2141  case AMDGPU::EXIT_STRICT_WWM:
2142  case AMDGPU::EXIT_STRICT_WQM: {
2143  // This only gets its own opcode so that SIPreAllocateWWMRegs can tell when
2144  // WWM/STICT_WQM is exited.
2145  MI.setDesc(get(ST.isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64));
2146  break;
2147  }
2148  case AMDGPU::SI_RETURN: {
2149  const MachineFunction *MF = MBB.getParent();
2150  const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
2151  const SIRegisterInfo *TRI = ST.getRegisterInfo();
2152  // Hiding the return address use with SI_RETURN may lead to extra kills in
2153  // the function and missing live-ins. We are fine in practice because callee
2154  // saved register handling ensures the register value is restored before
2155  // RET, but we need the undef flag here to appease the MachineVerifier
2156  // liveness checks.
2157  MachineInstrBuilder MIB =
2158  BuildMI(MBB, MI, DL, get(AMDGPU::S_SETPC_B64_return))
2159  .addReg(TRI->getReturnAddressReg(*MF), RegState::Undef);
2160 
2161  MIB.copyImplicitOps(MI);
2162  MI.eraseFromParent();
2163  break;
2164  }
2165  }
2166  return true;
2167 }
2168 
2169 std::pair<MachineInstr*, MachineInstr*>
2171  assert (MI.getOpcode() == AMDGPU::V_MOV_B64_DPP_PSEUDO);
2172 
2173  if (ST.hasMovB64() &&
2175  getNamedOperand(MI, AMDGPU::OpName::dpp_ctrl)->getImm())) {
2176  MI.setDesc(get(AMDGPU::V_MOV_B64_dpp));
2177  return std::make_pair(&MI, nullptr);
2178  }
2179 
2180  MachineBasicBlock &MBB = *MI.getParent();
2182  MachineFunction *MF = MBB.getParent();
2184  Register Dst = MI.getOperand(0).getReg();
2185  unsigned Part = 0;
2186  MachineInstr *Split[2];
2187 
2188  for (auto Sub : { AMDGPU::sub0, AMDGPU::sub1 }) {
2189  auto MovDPP = BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_dpp));
2190  if (Dst.isPhysical()) {
2191  MovDPP.addDef(RI.getSubReg(Dst, Sub));
2192  } else {
2193  assert(MRI.isSSA());
2194  auto Tmp = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
2195  MovDPP.addDef(Tmp);
2196  }
2197 
2198  for (unsigned I = 1; I <= 2; ++I) { // old and src operands.
2199  const MachineOperand &SrcOp = MI.getOperand(I);
2200  assert(!SrcOp.isFPImm());
2201  if (SrcOp.isImm()) {
2202  APInt Imm(64, SrcOp.getImm());
2203  Imm.ashrInPlace(Part * 32);
2204  MovDPP.addImm(Imm.getLoBits(32).getZExtValue());
2205  } else {
2206  assert(SrcOp.isReg());
2207  Register Src = SrcOp.getReg();
2208  if (Src.isPhysical())
2209  MovDPP.addReg(RI.getSubReg(Src, Sub));
2210  else
2211  MovDPP.addReg(Src, SrcOp.isUndef() ? RegState::Undef : 0, Sub);
2212  }
2213  }
2214 
2215  for (unsigned I = 3; I < MI.getNumExplicitOperands(); ++I)
2216  MovDPP.addImm(MI.getOperand(I).getImm());
2217 
2218  Split[Part] = MovDPP;
2219  ++Part;
2220  }
2221 
2222  if (Dst.isVirtual())
2223  BuildMI(MBB, MI, DL, get(AMDGPU::REG_SEQUENCE), Dst)
2224  .addReg(Split[0]->getOperand(0).getReg())
2225  .addImm(AMDGPU::sub0)
2226  .addReg(Split[1]->getOperand(0).getReg())
2227  .addImm(AMDGPU::sub1);
2228 
2229  MI.eraseFromParent();
2230  return std::make_pair(Split[0], Split[1]);
2231 }
2232 
2234  MachineOperand &Src0,
2235  unsigned Src0OpName,
2236  MachineOperand &Src1,
2237  unsigned Src1OpName) const {
2238  MachineOperand *Src0Mods = getNamedOperand(MI, Src0OpName);
2239  if (!Src0Mods)
2240  return false;
2241 
2242  MachineOperand *Src1Mods = getNamedOperand(MI, Src1OpName);
2243  assert(Src1Mods &&
2244  "All commutable instructions have both src0 and src1 modifiers");
2245 
2246  int Src0ModsVal = Src0Mods->getImm();
2247  int Src1ModsVal = Src1Mods->getImm();
2248 
2249  Src1Mods->setImm(Src0ModsVal);
2250  Src0Mods->setImm(Src1ModsVal);
2251  return true;
2252 }
2253 
2255  MachineOperand &RegOp,
2256  MachineOperand &NonRegOp) {
2257  Register Reg = RegOp.getReg();
2258  unsigned SubReg = RegOp.getSubReg();
2259  bool IsKill = RegOp.isKill();
2260  bool IsDead = RegOp.isDead();
2261  bool IsUndef = RegOp.isUndef();
2262  bool IsDebug = RegOp.isDebug();
2263 
2264  if (NonRegOp.isImm())
2265  RegOp.ChangeToImmediate(NonRegOp.getImm());
2266  else if (NonRegOp.isFI())
2267  RegOp.ChangeToFrameIndex(NonRegOp.getIndex());
2268  else if (NonRegOp.isGlobal()) {
2269  RegOp.ChangeToGA(NonRegOp.getGlobal(), NonRegOp.getOffset(),
2270  NonRegOp.getTargetFlags());
2271  } else
2272  return nullptr;
2273 
2274  // Make sure we don't reinterpret a subreg index in the target flags.
2275  RegOp.setTargetFlags(NonRegOp.getTargetFlags());
2276 
2277  NonRegOp.ChangeToRegister(Reg, false, false, IsKill, IsDead, IsUndef, IsDebug);
2278  NonRegOp.setSubReg(SubReg);
2279 
2280  return &MI;
2281 }
2282 
2284  unsigned Src0Idx,
2285  unsigned Src1Idx) const {
2286  assert(!NewMI && "this should never be used");
2287 
2288  unsigned Opc = MI.getOpcode();
2289  int CommutedOpcode = commuteOpcode(Opc);
2290  if (CommutedOpcode == -1)
2291  return nullptr;
2292 
2293  assert(AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0) ==
2294  static_cast<int>(Src0Idx) &&
2295  AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1) ==
2296  static_cast<int>(Src1Idx) &&
2297  "inconsistency with findCommutedOpIndices");
2298 
2299  MachineOperand &Src0 = MI.getOperand(Src0Idx);
2300  MachineOperand &Src1 = MI.getOperand(Src1Idx);
2301 
2302  MachineInstr *CommutedMI = nullptr;
2303  if (Src0.isReg() && Src1.isReg()) {
2304  if (isOperandLegal(MI, Src1Idx, &Src0)) {
2305  // Be sure to copy the source modifiers to the right place.
2306  CommutedMI
2307  = TargetInstrInfo::commuteInstructionImpl(MI, NewMI, Src0Idx, Src1Idx);
2308  }
2309 
2310  } else if (Src0.isReg() && !Src1.isReg()) {
2311  // src0 should always be able to support any operand type, so no need to
2312  // check operand legality.
2313  CommutedMI = swapRegAndNonRegOperand(MI, Src0, Src1);
2314  } else if (!Src0.isReg() && Src1.isReg()) {
2315  if (isOperandLegal(MI, Src1Idx, &Src0))
2316  CommutedMI = swapRegAndNonRegOperand(MI, Src1, Src0);
2317  } else {
2318  // FIXME: Found two non registers to commute. This does happen.
2319  return nullptr;
2320  }
2321 
2322  if (CommutedMI) {
2323  swapSourceModifiers(MI, Src0, AMDGPU::OpName::src0_modifiers,
2324  Src1, AMDGPU::OpName::src1_modifiers);
2325 
2326  CommutedMI->setDesc(get(CommutedOpcode));
2327  }
2328 
2329  return CommutedMI;
2330 }
2331 
2332 // This needs to be implemented because the source modifiers may be inserted
2333 // between the true commutable operands, and the base
2334 // TargetInstrInfo::commuteInstruction uses it.
2336  unsigned &SrcOpIdx0,
2337  unsigned &SrcOpIdx1) const {
2338  return findCommutedOpIndices(MI.getDesc(), SrcOpIdx0, SrcOpIdx1);
2339 }
2340 
2341 bool SIInstrInfo::findCommutedOpIndices(MCInstrDesc Desc, unsigned &SrcOpIdx0,
2342  unsigned &SrcOpIdx1) const {
2343  if (!Desc.isCommutable())
2344  return false;
2345 
2346  unsigned Opc = Desc.getOpcode();
2347  int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0);
2348  if (Src0Idx == -1)
2349  return false;
2350 
2351  int Src1Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1);
2352  if (Src1Idx == -1)
2353  return false;
2354 
2355  return fixCommutedOpIndices(SrcOpIdx0, SrcOpIdx1, Src0Idx, Src1Idx);
2356 }
2357 
2358 bool SIInstrInfo::isBranchOffsetInRange(unsigned BranchOp,
2359  int64_t BrOffset) const {
2360  // BranchRelaxation should never have to check s_setpc_b64 because its dest
2361  // block is unanalyzable.
2362  assert(BranchOp != AMDGPU::S_SETPC_B64);
2363 
2364  // Convert to dwords.
2365  BrOffset /= 4;
2366 
2367  // The branch instructions do PC += signext(SIMM16 * 4) + 4, so the offset is
2368  // from the next instruction.
2369  BrOffset -= 1;
2370 
2371  return isIntN(BranchOffsetBits, BrOffset);
2372 }
2373 
2375  const MachineInstr &MI) const {
2376  if (MI.getOpcode() == AMDGPU::S_SETPC_B64) {
2377  // This would be a difficult analysis to perform, but can always be legal so
2378  // there's no need to analyze it.
2379  return nullptr;
2380  }
2381 
2382  return MI.getOperand(0).getMBB();
2383 }
2384 
2386  MachineBasicBlock &DestBB,
2387  MachineBasicBlock &RestoreBB,
2388  const DebugLoc &DL, int64_t BrOffset,
2389  RegScavenger *RS) const {
2390  assert(RS && "RegScavenger required for long branching");
2391  assert(MBB.empty() &&
2392  "new block should be inserted for expanding unconditional branch");
2393  assert(MBB.pred_size() == 1);
2394  assert(RestoreBB.empty() &&
2395  "restore block should be inserted for restoring clobbered registers");
2396 
2397  MachineFunction *MF = MBB.getParent();
2399 
2400  // FIXME: Virtual register workaround for RegScavenger not working with empty
2401  // blocks.
2402  Register PCReg = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
2403 
2404  auto I = MBB.end();
2405 
2406  // We need to compute the offset relative to the instruction immediately after
2407  // s_getpc_b64. Insert pc arithmetic code before last terminator.
2408  MachineInstr *GetPC = BuildMI(MBB, I, DL, get(AMDGPU::S_GETPC_B64), PCReg);
2409 
2410  auto &MCCtx = MF->getContext();
2411  MCSymbol *PostGetPCLabel =
2412  MCCtx.createTempSymbol("post_getpc", /*AlwaysAddSuffix=*/true);
2413  GetPC->setPostInstrSymbol(*MF, PostGetPCLabel);
2414 
2415  MCSymbol *OffsetLo =
2416  MCCtx.createTempSymbol("offset_lo", /*AlwaysAddSuffix=*/true);
2417  MCSymbol *OffsetHi =
2418  MCCtx.createTempSymbol("offset_hi", /*AlwaysAddSuffix=*/true);
2419  BuildMI(MBB, I, DL, get(AMDGPU::S_ADD_U32))
2420  .addReg(PCReg, RegState::Define, AMDGPU::sub0)
2421  .addReg(PCReg, 0, AMDGPU::sub0)
2422  .addSym(OffsetLo, MO_FAR_BRANCH_OFFSET);
2423  BuildMI(MBB, I, DL, get(AMDGPU::S_ADDC_U32))
2424  .addReg(PCReg, RegState::Define, AMDGPU::sub1)
2425  .addReg(PCReg, 0, AMDGPU::sub1)
2426  .addSym(OffsetHi, MO_FAR_BRANCH_OFFSET);
2427 
2428  // Insert the indirect branch after the other terminator.
2429  BuildMI(&MBB, DL, get(AMDGPU::S_SETPC_B64))
2430  .addReg(PCReg);
2431 
2432  // FIXME: If spilling is necessary, this will fail because this scavenger has
2433  // no emergency stack slots. It is non-trivial to spill in this situation,
2434  // because the restore code needs to be specially placed after the
2435  // jump. BranchRelaxation then needs to be made aware of the newly inserted
2436  // block.
2437  //
2438  // If a spill is needed for the pc register pair, we need to insert a spill
2439  // restore block right before the destination block, and insert a short branch
2440  // into the old destination block's fallthrough predecessor.
2441  // e.g.:
2442  //
2443  // s_cbranch_scc0 skip_long_branch:
2444  //
2445  // long_branch_bb:
2446  // spill s[8:9]
2447  // s_getpc_b64 s[8:9]
2448  // s_add_u32 s8, s8, restore_bb
2449  // s_addc_u32 s9, s9, 0
2450  // s_setpc_b64 s[8:9]
2451  //
2452  // skip_long_branch:
2453  // foo;
2454  //
2455  // .....
2456  //
2457  // dest_bb_fallthrough_predecessor:
2458  // bar;
2459  // s_branch dest_bb
2460  //
2461  // restore_bb:
2462  // restore s[8:9]
2463  // fallthrough dest_bb
2464  ///
2465  // dest_bb:
2466  // buzz;
2467 
2468  RS->enterBasicBlockEnd(MBB);
2470  AMDGPU::SReg_64RegClass, MachineBasicBlock::iterator(GetPC),
2471  /* RestoreAfter */ false, 0, /* AllowSpill */ false);
2472  if (Scav) {
2473  RS->setRegUsed(Scav);
2474  MRI.replaceRegWith(PCReg, Scav);
2475  MRI.clearVirtRegs();
2476  } else {
2477  // As SGPR needs VGPR to be spilled, we reuse the slot of temporary VGPR for
2478  // SGPR spill.
2479  const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
2480  const SIRegisterInfo *TRI = ST.getRegisterInfo();
2481  TRI->spillEmergencySGPR(GetPC, RestoreBB, AMDGPU::SGPR0_SGPR1, RS);
2482  MRI.replaceRegWith(PCReg, AMDGPU::SGPR0_SGPR1);
2483  MRI.clearVirtRegs();
2484  }
2485 
2486  MCSymbol *DestLabel = Scav ? DestBB.getSymbol() : RestoreBB.getSymbol();
2487  // Now, the distance could be defined.
2488  auto *Offset = MCBinaryExpr::createSub(
2489  MCSymbolRefExpr::create(DestLabel, MCCtx),
2490  MCSymbolRefExpr::create(PostGetPCLabel, MCCtx), MCCtx);
2491  // Add offset assignments.
2492  auto *Mask = MCConstantExpr::create(0xFFFFFFFFULL, MCCtx);
2493  OffsetLo->setVariableValue(MCBinaryExpr::createAnd(Offset, Mask, MCCtx));
2494  auto *ShAmt = MCConstantExpr::create(32, MCCtx);
2495  OffsetHi->setVariableValue(MCBinaryExpr::createAShr(Offset, ShAmt, MCCtx));
2496 }
2497 
2498 unsigned SIInstrInfo::getBranchOpcode(SIInstrInfo::BranchPredicate Cond) {
2499  switch (Cond) {
2500  case SIInstrInfo::SCC_TRUE:
2501  return AMDGPU::S_CBRANCH_SCC1;
2502  case SIInstrInfo::SCC_FALSE:
2503  return AMDGPU::S_CBRANCH_SCC0;
2504  case SIInstrInfo::VCCNZ:
2505  return AMDGPU::S_CBRANCH_VCCNZ;
2506  case SIInstrInfo::VCCZ:
2507  return AMDGPU::S_CBRANCH_VCCZ;
2508  case SIInstrInfo::EXECNZ:
2509  return AMDGPU::S_CBRANCH_EXECNZ;
2510  case SIInstrInfo::EXECZ:
2511  return AMDGPU::S_CBRANCH_EXECZ;
2512  default:
2513  llvm_unreachable("invalid branch predicate");
2514  }
2515 }
2516 
2517 SIInstrInfo::BranchPredicate SIInstrInfo::getBranchPredicate(unsigned Opcode) {
2518  switch (Opcode) {
2519  case AMDGPU::S_CBRANCH_SCC0:
2520  return SCC_FALSE;
2521  case AMDGPU::S_CBRANCH_SCC1:
2522  return SCC_TRUE;
2523  case AMDGPU::S_CBRANCH_VCCNZ:
2524  return VCCNZ;
2525  case AMDGPU::S_CBRANCH_VCCZ:
2526  return VCCZ;
2527  case AMDGPU::S_CBRANCH_EXECNZ:
2528  return EXECNZ;
2529  case AMDGPU::S_CBRANCH_EXECZ:
2530  return EXECZ;
2531  default:
2532  return INVALID_BR;
2533  }
2534 }
2535 
2538  MachineBasicBlock *&TBB,
2539  MachineBasicBlock *&FBB,
2541  bool AllowModify) const {
2542  if (I->getOpcode() == AMDGPU::S_BRANCH) {
2543  // Unconditional Branch
2544  TBB = I->getOperand(0).getMBB();
2545  return false;
2546  }
2547 
2548  MachineBasicBlock *CondBB = nullptr;
2549 
2550  if (I->getOpcode() == AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO) {
2551  CondBB = I->getOperand(1).getMBB();
2552  Cond.push_back(I->getOperand(0));
2553  } else {
2554  BranchPredicate Pred = getBranchPredicate(I->getOpcode());
2555  if (Pred == INVALID_BR)
2556  return true;
2557 
2558  CondBB = I->getOperand(0).getMBB();
2559  Cond.push_back(MachineOperand::CreateImm(Pred));
2560  Cond.push_back(I->getOperand(1)); // Save the branch register.
2561  }
2562  ++I;
2563 
2564  if (I == MBB.end()) {
2565  // Conditional branch followed by fall-through.
2566  TBB = CondBB;
2567  return false;
2568  }
2569 
2570  if (I->getOpcode() == AMDGPU::S_BRANCH) {
2571  TBB = CondBB;
2572  FBB = I->getOperand(0).getMBB();
2573  return false;
2574  }
2575 
2576  return true;
2577 }
2578 
2580  MachineBasicBlock *&FBB,
2582  bool AllowModify) const {
2584  auto E = MBB.end();
2585  if (I == E)
2586  return false;
2587 
2588  // Skip over the instructions that are artificially terminators for special
2589  // exec management.
2590  while (I != E && !I->isBranch() && !I->isReturn()) {
2591  switch (I->getOpcode()) {
2592  case AMDGPU::S_MOV_B64_term:
2593  case AMDGPU::S_XOR_B64_term:
2594  case AMDGPU::S_OR_B64_term:
2595  case AMDGPU::S_ANDN2_B64_term:
2596  case AMDGPU::S_AND_B64_term:
2597  case AMDGPU::S_MOV_B32_term:
2598  case AMDGPU::S_XOR_B32_term:
2599  case AMDGPU::S_OR_B32_term:
2600  case AMDGPU::S_ANDN2_B32_term:
2601  case AMDGPU::S_AND_B32_term:
2602  break;
2603  case AMDGPU::SI_IF:
2604  case AMDGPU::SI_ELSE:
2605  case AMDGPU::SI_KILL_I1_TERMINATOR:
2606  case AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR:
2607  // FIXME: It's messy that these need to be considered here at all.
2608  return true;
2609  default:
2610  llvm_unreachable("unexpected non-branch terminator inst");
2611  }
2612 
2613  ++I;
2614  }
2615 
2616  if (I == E)
2617  return false;
2618 
2619  return analyzeBranchImpl(MBB, I, TBB, FBB, Cond, AllowModify);
2620 }
2621 
2623  int *BytesRemoved) const {
2624  unsigned Count = 0;
2625  unsigned RemovedSize = 0;
2627  // Skip over artificial terminators when removing instructions.
2628  if (MI.isBranch() || MI.isReturn()) {
2629  RemovedSize += getInstSizeInBytes(MI);
2630  MI.eraseFromParent();
2631  ++Count;
2632  }
2633  }
2634 
2635  if (BytesRemoved)
2636  *BytesRemoved = RemovedSize;
2637 
2638  return Count;
2639 }
2640 
2641 // Copy the flags onto the implicit condition register operand.
2643  const MachineOperand &OrigCond) {
2644  CondReg.setIsUndef(OrigCond.isUndef());
2645  CondReg.setIsKill(OrigCond.isKill());
2646 }
2647 
2649  MachineBasicBlock *TBB,
2650  MachineBasicBlock *FBB,
2652  const DebugLoc &DL,
2653  int *BytesAdded) const {
2654  if (!FBB && Cond.empty()) {
2655  BuildMI(&MBB, DL, get(AMDGPU::S_BRANCH))
2656  .addMBB(TBB);
2657  if (BytesAdded)
2658  *BytesAdded = ST.hasOffset3fBug() ? 8 : 4;
2659  return 1;
2660  }
2661 
2662  if(Cond.size() == 1 && Cond[0].isReg()) {
2663  BuildMI(&MBB, DL, get(AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO))
2664  .add(Cond[0])
2665  .addMBB(TBB);
2666  return 1;
2667  }
2668 
2669  assert(TBB && Cond[0].isImm());
2670 
2671  unsigned Opcode
2672  = getBranchOpcode(static_cast<BranchPredicate>(Cond[0].getImm()));
2673 
2674  if (!FBB) {
2675  Cond[1].isUndef();
2676  MachineInstr *CondBr =
2677  BuildMI(&MBB, DL, get(Opcode))
2678  .addMBB(TBB);
2679 
2680  // Copy the flags onto the implicit condition register operand.
2681  preserveCondRegFlags(CondBr->getOperand(1), Cond[1]);
2682  fixImplicitOperands(*CondBr);
2683 
2684  if (BytesAdded)
2685  *BytesAdded = ST.hasOffset3fBug() ? 8 : 4;
2686  return 1;
2687  }
2688 
2689  assert(TBB && FBB);
2690 
2691  MachineInstr *CondBr =
2692  BuildMI(&MBB, DL, get(Opcode))
2693  .addMBB(TBB);
2694  fixImplicitOperands(*CondBr);
2695  BuildMI(&MBB, DL, get(AMDGPU::S_BRANCH))
2696  .addMBB(FBB);
2697 
2698  MachineOperand &CondReg = CondBr->getOperand(1);
2699  CondReg.setIsUndef(Cond[1].isUndef());
2700  CondReg.setIsKill(Cond[1].isKill());
2701 
2702  if (BytesAdded)
2703  *BytesAdded = ST.hasOffset3fBug() ? 16 : 8;
2704 
2705  return 2;
2706 }
2707 
2710  if (Cond.size() != 2) {
2711  return true;
2712  }
2713 
2714  if (Cond[0].isImm()) {
2715  Cond[0].setImm(-Cond[0].getImm());
2716  return false;
2717  }
2718 
2719  return true;
2720 }
2721 
2724  Register DstReg, Register TrueReg,
2725  Register FalseReg, int &CondCycles,
2726  int &TrueCycles, int &FalseCycles) const {
2727  switch (Cond[0].getImm()) {
2728  case VCCNZ:
2729  case VCCZ: {
2731  const TargetRegisterClass *RC = MRI.getRegClass(TrueReg);
2732  if (MRI.getRegClass(FalseReg) != RC)
2733  return false;
2734 
2735  int NumInsts = AMDGPU::getRegBitWidth(RC->getID()) / 32;
2736  CondCycles = TrueCycles = FalseCycles = NumInsts; // ???
2737 
2738  // Limit to equal cost for branch vs. N v_cndmask_b32s.
2739  return RI.hasVGPRs(RC) && NumInsts <= 6;
2740  }
2741  case SCC_TRUE:
2742  case SCC_FALSE: {
2743  // FIXME: We could insert for VGPRs if we could replace the original compare
2744  // with a vector one.
2746  const TargetRegisterClass *RC = MRI.getRegClass(TrueReg);
2747  if (MRI.getRegClass(FalseReg) != RC)
2748  return false;
2749 
2750  int NumInsts = AMDGPU::getRegBitWidth(RC->getID()) / 32;
2751 
2752  // Multiples of 8 can do s_cselect_b64
2753  if (NumInsts % 2 == 0)
2754  NumInsts /= 2;
2755 
2756  CondCycles = TrueCycles = FalseCycles = NumInsts; // ???
2757  return RI.isSGPRClass(RC);
2758  }
2759  default:
2760  return false;
2761  }
2762 }
2763 
2767  Register TrueReg, Register FalseReg) const {
2768  BranchPredicate Pred = static_cast<BranchPredicate>(Cond[0].getImm());
2769  if (Pred == VCCZ || Pred == SCC_FALSE) {
2770  Pred = static_cast<BranchPredicate>(-Pred);
2771  std::swap(TrueReg, FalseReg);
2772  }
2773 
2775  const TargetRegisterClass *DstRC = MRI.getRegClass(DstReg);
2776  unsigned DstSize = RI.getRegSizeInBits(*DstRC);
2777 
2778  if (DstSize == 32) {
2780  if (Pred == SCC_TRUE) {
2781  Select = BuildMI(MBB, I, DL, get(AMDGPU::S_CSELECT_B32), DstReg)
2782  .addReg(TrueReg)
2783  .addReg(FalseReg);
2784  } else {
2785  // Instruction's operands are backwards from what is expected.
2786  Select = BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e32), DstReg)
2787  .addReg(FalseReg)
2788  .addReg(TrueReg);
2789  }
2790 
2791  preserveCondRegFlags(Select->getOperand(3), Cond[1]);
2792  return;
2793  }
2794 
2795  if (DstSize == 64 && Pred == SCC_TRUE) {
2796  MachineInstr *Select =
2797  BuildMI(MBB, I, DL, get(AMDGPU::S_CSELECT_B64), DstReg)
2798  .addReg(TrueReg)
2799  .addReg(FalseReg);
2800 
2801  preserveCondRegFlags(Select->getOperand(3), Cond[1]);
2802  return;
2803  }
2804 
2805  static const int16_t Sub0_15[] = {
2806  AMDGPU::sub0, AMDGPU::sub1, AMDGPU::sub2, AMDGPU::sub3,
2807  AMDGPU::sub4, AMDGPU::sub5, AMDGPU::sub6, AMDGPU::sub7,
2808  AMDGPU::sub8, AMDGPU::sub9, AMDGPU::sub10, AMDGPU::sub11,
2809  AMDGPU::sub12, AMDGPU::sub13, AMDGPU::sub14, AMDGPU::sub15,
2810  };
2811 
2812  static const int16_t Sub0_15_64[] = {
2813  AMDGPU::sub0_sub1, AMDGPU::sub2_sub3,
2814  AMDGPU::sub4_sub5, AMDGPU::sub6_sub7,
2815  AMDGPU::sub8_sub9, AMDGPU::sub10_sub11,
2816  AMDGPU::sub12_sub13, AMDGPU::sub14_sub15,
2817  };
2818 
2819  unsigned SelOp = AMDGPU::V_CNDMASK_B32_e32;
2820  const TargetRegisterClass *EltRC = &AMDGPU::VGPR_32RegClass;
2821  const int16_t *SubIndices = Sub0_15;
2822  int NElts = DstSize / 32;
2823 
2824  // 64-bit select is only available for SALU.
2825  // TODO: Split 96-bit into 64-bit and 32-bit, not 3x 32-bit.
2826  if (Pred == SCC_TRUE) {
2827  if (NElts % 2) {
2828  SelOp = AMDGPU::S_CSELECT_B32;
2829  EltRC = &AMDGPU::SGPR_32RegClass;
2830  } else {
2831  SelOp = AMDGPU::S_CSELECT_B64;
2832  EltRC = &AMDGPU::SGPR_64RegClass;
2833  SubIndices = Sub0_15_64;
2834  NElts /= 2;
2835  }
2836  }
2837 
2839  MBB, I, DL, get(AMDGPU::REG_SEQUENCE), DstReg);
2840 
2841  I = MIB->getIterator();
2842 
2844  for (int Idx = 0; Idx != NElts; ++Idx) {
2845  Register DstElt = MRI.createVirtualRegister(EltRC);
2846  Regs.push_back(DstElt);
2847 
2848  unsigned SubIdx = SubIndices[Idx];
2849 
2851  if (SelOp == AMDGPU::V_CNDMASK_B32_e32) {
2852  Select =
2853  BuildMI(MBB, I, DL, get(SelOp), DstElt)
2854  .addReg(FalseReg, 0, SubIdx)
2855  .addReg(TrueReg, 0, SubIdx);
2856  } else {
2857  Select =
2858  BuildMI(MBB, I, DL, get(SelOp), DstElt)
2859  .addReg(TrueReg, 0, SubIdx)
2860  .addReg(FalseReg, 0, SubIdx);
2861  }
2862 
2863  preserveCondRegFlags(Select->getOperand(3), Cond[1]);
2865 
2866  MIB.addReg(DstElt)
2867  .addImm(SubIdx);
2868  }
2869 }
2870 
2872  switch (MI.getOpcode()) {
2873  case AMDGPU::V_MOV_B32_e32:
2874  case AMDGPU::V_MOV_B32_e64:
2875  case AMDGPU::V_MOV_B64_PSEUDO:
2876  case AMDGPU::V_MOV_B64_e32:
2877  case AMDGPU::V_MOV_B64_e64:
2878  case AMDGPU::S_MOV_B32:
2879  case AMDGPU::S_MOV_B64:
2880  case AMDGPU::COPY:
2881  case AMDGPU::V_ACCVGPR_WRITE_B32_e64:
2882  case AMDGPU::V_ACCVGPR_READ_B32_e64:
2883  case AMDGPU::V_ACCVGPR_MOV_B32:
2884  return true;
2885  default:
2886  return false;
2887  }
2888 }
2889 
2890 static constexpr unsigned ModifierOpNames[] = {
2891  AMDGPU::OpName::src0_modifiers, AMDGPU::OpName::src1_modifiers,
2892  AMDGPU::OpName::src2_modifiers, AMDGPU::OpName::clamp,
2893  AMDGPU::OpName::omod};
2894 
2896  unsigned Opc = MI.getOpcode();
2897  for (unsigned Name : reverse(ModifierOpNames))
2898  MI.removeOperand(AMDGPU::getNamedOperandIdx(Opc, Name));
2899 }
2900 
2902  Register Reg, MachineRegisterInfo *MRI) const {
2903  if (!MRI->hasOneNonDBGUse(Reg))
2904  return false;
2905 
2906  switch (DefMI.getOpcode()) {
2907  default:
2908  return false;
2909  case AMDGPU::S_MOV_B64:
2910  // TODO: We could fold 64-bit immediates, but this get complicated
2911  // when there are sub-registers.
2912  return false;
2913 
2914  case AMDGPU::V_MOV_B32_e32:
2915  case AMDGPU::S_MOV_B32:
2916  case AMDGPU::V_ACCVGPR_WRITE_B32_e64:
2917  break;
2918  }
2919 
2920  const MachineOperand *ImmOp = getNamedOperand(DefMI, AMDGPU::OpName::src0);
2921  assert(ImmOp);
2922  // FIXME: We could handle FrameIndex values here.
2923  if (!ImmOp->isImm())
2924  return false;
2925 
2926  unsigned Opc = UseMI.getOpcode();
2927  if (Opc == AMDGPU::COPY) {
2928  Register DstReg = UseMI.getOperand(0).getReg();
2929  bool Is16Bit = getOpSize(UseMI, 0) == 2;
2930  bool isVGPRCopy = RI.isVGPR(*MRI, DstReg);
2931  unsigned NewOpc = isVGPRCopy ? AMDGPU::V_MOV_B32_e32 : AMDGPU::S_MOV_B32;
2932  APInt Imm(32, ImmOp->getImm());
2933 
2934  if (UseMI.getOperand(1).getSubReg() == AMDGPU::hi16)
2935  Imm = Imm.ashr(16);
2936 
2937  if (RI.isAGPR(*MRI, DstReg)) {
2938  if (!isInlineConstant(Imm))
2939  return false;
2940  NewOpc = AMDGPU::V_ACCVGPR_WRITE_B32_e64;
2941  }
2942 
2943  if (Is16Bit) {
2944  if (isVGPRCopy)
2945  return false; // Do not clobber vgpr_hi16
2946 
2947  if (DstReg.isVirtual() && UseMI.getOperand(0).getSubReg() != AMDGPU::lo16)
2948  return false;
2949 
2950  UseMI.getOperand(0).setSubReg(0);
2951  if (DstReg.isPhysical()) {
2952  DstReg = RI.get32BitRegister(DstReg);
2953  UseMI.getOperand(0).setReg(DstReg);
2954  }
2955  assert(UseMI.getOperand(1).getReg().isVirtual());
2956  }
2957 
2958  UseMI.setDesc(get(NewOpc));
2959  UseMI.getOperand(1).ChangeToImmediate(Imm.getSExtValue());
2960  UseMI.addImplicitDefUseOperands(*UseMI.getParent()->getParent());
2961  return true;
2962  }
2963 
2964  if (Opc == AMDGPU::V_MAD_F32_e64 || Opc == AMDGPU::V_MAC_F32_e64 ||
2965  Opc == AMDGPU::V_MAD_F16_e64 || Opc == AMDGPU::V_MAC_F16_e64 ||
2966  Opc == AMDGPU::V_FMA_F32_e64 || Opc == AMDGPU::V_FMAC_F32_e64 ||
2967  Opc == AMDGPU::V_FMA_F16_e64 || Opc == AMDGPU::V_FMAC_F16_e64) {
2968  // Don't fold if we are using source or output modifiers. The new VOP2
2969  // instructions don't have them.
2971  return false;
2972 
2973  // If this is a free constant, there's no reason to do this.
2974  // TODO: We could fold this here instead of letting SIFoldOperands do it
2975  // later.
2976  MachineOperand *Src0 = getNamedOperand(UseMI, AMDGPU::OpName::src0);
2977 
2978  // Any src operand can be used for the legality check.
2979  if (isInlineConstant(UseMI, *Src0, *ImmOp))
2980  return false;
2981 
2982  bool IsF32 = Opc == AMDGPU::V_MAD_F32_e64 || Opc == AMDGPU::V_MAC_F32_e64 ||
2983  Opc == AMDGPU::V_FMA_F32_e64 || Opc == AMDGPU::V_FMAC_F32_e64;
2984  bool IsFMA = Opc == AMDGPU::V_FMA_F32_e64 || Opc == AMDGPU::V_FMAC_F32_e64 ||
2985  Opc == AMDGPU::V_FMA_F16_e64 || Opc == AMDGPU::V_FMAC_F16_e64;
2986  MachineOperand *Src1 = getNamedOperand(UseMI, AMDGPU::OpName::src1);
2987  MachineOperand *Src2 = getNamedOperand(UseMI, AMDGPU::OpName::src2);
2988 
2989  // Multiplied part is the constant: Use v_madmk_{f16, f32}.
2990  // We should only expect these to be on src0 due to canonicalization.
2991  if (Src0->isReg() && Src0->getReg() == Reg) {
2992  if (!Src1->isReg() || RI.isSGPRClass(MRI->getRegClass(Src1->getReg())))
2993  return false;
2994 
2995  if (!Src2->isReg() || RI.isSGPRClass(MRI->getRegClass(Src2->getReg())))
2996  return false;
2997 
2998  unsigned NewOpc =
2999  IsFMA ? (IsF32 ? AMDGPU::V_FMAMK_F32 : AMDGPU::V_FMAMK_F16)
3000  : (IsF32 ? AMDGPU::V_MADMK_F32 : AMDGPU::V_MADMK_F16);
3001  if (pseudoToMCOpcode(NewOpc) == -1)
3002  return false;
3003 
3004  // We need to swap operands 0 and 1 since madmk constant is at operand 1.
3005 
3006  const int64_t Imm = ImmOp->getImm();
3007 
3008  // FIXME: This would be a lot easier if we could return a new instruction
3009  // instead of having to modify in place.
3010 
3011  Register Src1Reg = Src1->getReg();
3012  unsigned Src1SubReg = Src1->getSubReg();
3013  Src0->setReg(Src1Reg);
3014  Src0->setSubReg(Src1SubReg);
3015  Src0->setIsKill(Src1->isKill());
3016 
3017  if (Opc == AMDGPU::V_MAC_F32_e64 ||
3018  Opc == AMDGPU::V_MAC_F16_e64 ||
3019  Opc == AMDGPU::V_FMAC_F32_e64 ||
3020  Opc == AMDGPU::V_FMAC_F16_e64)
3021  UseMI.untieRegOperand(
3022  AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2));
3023 
3024  Src1->ChangeToImmediate(Imm);
3025 
3027  UseMI.setDesc(get(NewOpc));
3028 
3029  bool DeleteDef = MRI->use_nodbg_empty(Reg);
3030  if (DeleteDef)
3031  DefMI.eraseFromParent();
3032 
3033  return true;
3034  }
3035 
3036  // Added part is the constant: Use v_madak_{f16, f32}.
3037  if (Src2->isReg() && Src2->getReg() == Reg) {
3038  // Not allowed to use constant bus for another operand.
3039  // We can however allow an inline immediate as src0.
3040  bool Src0Inlined = false;
3041  if (Src0->isReg()) {
3042  // Try to inline constant if possible.
3043  // If the Def moves immediate and the use is single
3044  // We are saving VGPR here.
3046  if (Def && Def->isMoveImmediate() &&
3047  isInlineConstant(Def->getOperand(1)) &&
3048  MRI->hasOneUse(Src0->getReg())) {
3049  Src0->ChangeToImmediate(Def->getOperand(1).getImm());
3050  Src0Inlined = true;
3051  } else if ((Src0->getReg().isPhysical() &&
3052  (ST.getConstantBusLimit(Opc) <= 1 &&
3053  RI.isSGPRClass(RI.getPhysRegClass(Src0->getReg())))) ||
3054  (Src0->getReg().isVirtual() &&
3055  (ST.getConstantBusLimit(Opc) <= 1 &&
3056  RI.isSGPRClass(MRI->getRegClass(Src0->getReg())))))
3057  return false;
3058  // VGPR is okay as Src0 - fallthrough
3059  }
3060 
3061  if (Src1->isReg() && !Src0Inlined ) {
3062  // We have one slot for inlinable constant so far - try to fill it
3064  if (Def && Def->isMoveImmediate() &&
3065  isInlineConstant(Def->getOperand(1)) &&
3066  MRI->hasOneUse(Src1->getReg()) &&
3067  commuteInstruction(UseMI)) {
3068  Src0->ChangeToImmediate(Def->getOperand(1).getImm());
3069  } else if ((Src1->getReg().isPhysical() &&
3070  RI.isSGPRClass(RI.getPhysRegClass(Src1->getReg()))) ||
3071  (Src1->getReg().isVirtual() &&
3072  RI.isSGPRClass(MRI->getRegClass(Src1->getReg()))))
3073  return false;
3074  // VGPR is okay as Src1 - fallthrough
3075  }
3076 
3077  unsigned NewOpc =
3078  IsFMA ? (IsF32 ? AMDGPU::V_FMAAK_F32 : AMDGPU::V_FMAAK_F16)
3079  : (IsF32 ? AMDGPU::V_MADAK_F32 : AMDGPU::V_MADAK_F16);
3080  if (pseudoToMCOpcode(NewOpc) == -1)
3081  return false;
3082 
3083  const int64_t Imm = ImmOp->getImm();
3084 
3085  // FIXME: This would be a lot easier if we could return a new instruction
3086  // instead of having to modify in place.
3087 
3088  if (Opc == AMDGPU::V_MAC_F32_e64 ||
3089  Opc == AMDGPU::V_MAC_F16_e64 ||
3090  Opc == AMDGPU::V_FMAC_F32_e64 ||
3091  Opc == AMDGPU::V_FMAC_F16_e64)
3092  UseMI.untieRegOperand(
3093  AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2));
3094 
3095  // ChangingToImmediate adds Src2 back to the instruction.
3096  Src2->ChangeToImmediate(Imm);
3097 
3098  // These come before src2.
3100  UseMI.setDesc(get(NewOpc));
3101  // It might happen that UseMI was commuted
3102  // and we now have SGPR as SRC1. If so 2 inlined
3103  // constant and SGPR are illegal.
3105 
3106  bool DeleteDef = MRI->use_nodbg_empty(Reg);
3107  if (DeleteDef)
3108  DefMI.eraseFromParent();
3109 
3110  return true;
3111  }
3112  }
3113 
3114  return false;
3115 }
3116 
3117 static bool
3120  if (BaseOps1.size() != BaseOps2.size())
3121  return false;
3122  for (size_t I = 0, E = BaseOps1.size(); I < E; ++I) {
3123  if (!BaseOps1[I]->isIdenticalTo(*BaseOps2[I]))
3124  return false;
3125  }
3126  return true;
3127 }
3128 
3129 static bool offsetsDoNotOverlap(int WidthA, int OffsetA,
3130  int WidthB, int OffsetB) {
3131  int LowOffset = OffsetA < OffsetB ? OffsetA : OffsetB;
3132  int HighOffset = OffsetA < OffsetB ? OffsetB : OffsetA;
3133  int LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB;
3134  return LowOffset + LowWidth <= HighOffset;
3135 }
3136 
3137 bool SIInstrInfo::checkInstOffsetsDoNotOverlap(const MachineInstr &MIa,
3138  const MachineInstr &MIb) const {
3139  SmallVector<const MachineOperand *, 4> BaseOps0, BaseOps1;
3140  int64_t Offset0, Offset1;
3141  unsigned Dummy0, Dummy1;
3142  bool Offset0IsScalable, Offset1IsScalable;
3143  if (!getMemOperandsWithOffsetWidth(MIa, BaseOps0, Offset0, Offset0IsScalable,
3144  Dummy0, &RI) ||
3145  !getMemOperandsWithOffsetWidth(MIb, BaseOps1, Offset1, Offset1IsScalable,
3146  Dummy1, &RI))
3147  return false;
3148 
3149  if (!memOpsHaveSameBaseOperands(BaseOps0, BaseOps1))
3150  return false;
3151 
3152  if (!MIa.hasOneMemOperand() || !MIb.hasOneMemOperand()) {
3153  // FIXME: Handle ds_read2 / ds_write2.
3154  return false;
3155  }
3156  unsigned Width0 = MIa.memoperands().front()->getSize();
3157  unsigned Width1 = MIb.memoperands().front()->getSize();
3158  return offsetsDoNotOverlap(Width0, Offset0, Width1, Offset1);
3159 }
3160 
3162  const MachineInstr &MIb) const {
3163  assert(MIa.mayLoadOrStore() &&
3164  "MIa must load from or modify a memory location");
3165  assert(MIb.mayLoadOrStore() &&
3166  "MIb must load from or modify a memory location");
3167 
3169  return false;
3170 
3171  // XXX - Can we relax this between address spaces?
3172  if (MIa.hasOrderedMemoryRef() || MIb.hasOrderedMemoryRef())
3173  return false;
3174 
3175  // TODO: Should we check the address space from the MachineMemOperand? That
3176  // would allow us to distinguish objects we know don't alias based on the
3177  // underlying address space, even if it was lowered to a different one,
3178  // e.g. private accesses lowered to use MUBUF instructions on a scratch
3179  // buffer.
3180  if (isDS(MIa)) {
3181  if (isDS(MIb))
3182  return checkInstOffsetsDoNotOverlap(MIa, MIb);
3183 
3184  return !isFLAT(MIb) || isSegmentSpecificFLAT(MIb);
3185  }
3186 
3187  if (isMUBUF(MIa) || isMTBUF(MIa)) {
3188  if (isMUBUF(MIb) || isMTBUF(MIb))
3189  return checkInstOffsetsDoNotOverlap(MIa, MIb);
3190 
3191  return !isFLAT(MIb) && !isSMRD(MIb);
3192  }
3193 
3194  if (isSMRD(MIa)) {
3195  if (isSMRD(MIb))
3196  return checkInstOffsetsDoNotOverlap(MIa, MIb);
3197 
3198  return !isFLAT(MIb) && !isMUBUF(MIb) && !isMTBUF(MIb);
3199  }
3200 
3201  if (isFLAT(MIa)) {
3202  if (isFLAT(MIb))
3203  return checkInstOffsetsDoNotOverlap(MIa, MIb);
3204 
3205  return false;
3206  }
3207 
3208  return false;
3209 }
3210 
3212  int64_t &Imm, MachineInstr **DefMI = nullptr) {
3213  if (Reg.isPhysical())
3214  return false;
3215  auto *Def = MRI.getUniqueVRegDef(Reg);
3216  if (Def && SIInstrInfo::isFoldableCopy(*Def) && Def->getOperand(1).isImm()) {
3217  Imm = Def->getOperand(1).getImm();
3218  if (DefMI)
3219  *DefMI = Def;
3220  return true;
3221  }
3222  return false;
3223 }
3224 
3225 static bool getFoldableImm(const MachineOperand *MO, int64_t &Imm,
3226  MachineInstr **DefMI = nullptr) {
3227  if (!MO->isReg())
3228  return false;
3229  const MachineFunction *MF = MO->getParent()->getParent()->getParent();
3230  const MachineRegisterInfo &MRI = MF->getRegInfo();
3231  return getFoldableImm(MO->getReg(), MRI, Imm, DefMI);
3232 }
3233 
3235  MachineInstr &NewMI) {
3236  if (LV) {
3237  unsigned NumOps = MI.getNumOperands();
3238  for (unsigned I = 1; I < NumOps; ++I) {
3239  MachineOperand &Op = MI.getOperand(I);
3240  if (Op.isReg() && Op.isKill())
3241  LV->replaceKillInstruction(Op.getReg(), MI, NewMI);
3242  }
3243  }
3244 }
3245 
3247  LiveVariables *LV,
3248  LiveIntervals *LIS) const {
3249  MachineBasicBlock &MBB = *MI.getParent();
3250  unsigned Opc = MI.getOpcode();
3251 
3252  // Handle MFMA.
3253  int NewMFMAOpc = AMDGPU::getMFMAEarlyClobberOp(Opc);
3254  if (NewMFMAOpc != -1) {
3255  MachineInstrBuilder MIB =
3256  BuildMI(MBB, MI, MI.getDebugLoc(), get(NewMFMAOpc));
3257  for (unsigned I = 0, E = MI.getNumOperands(); I != E; ++I)
3258  MIB.add(MI.getOperand(I));
3259  updateLiveVariables(LV, MI, *MIB);
3260  if (LIS)
3261  LIS->ReplaceMachineInstrInMaps(MI, *MIB);
3262  return MIB;
3263  }
3264 
3265  if (SIInstrInfo::isWMMA(MI)) {
3266  unsigned NewOpc = AMDGPU::mapWMMA2AddrTo3AddrOpcode(MI.getOpcode());
3267  MachineInstrBuilder MIB = BuildMI(MBB, MI, MI.getDebugLoc(), get(NewOpc))
3268  .setMIFlags(MI.getFlags());
3269  for (unsigned I = 0, E = MI.getNumOperands(); I != E; ++I)
3270  MIB->addOperand(MI.getOperand(I));
3271 
3272  updateLiveVariables(LV, MI, *MIB);
3273  if (LIS)
3274  LIS->ReplaceMachineInstrInMaps(MI, *MIB);
3275 
3276  return MIB;
3277  }
3278 
3279  // Handle MAC/FMAC.
3280  bool IsF16 = Opc == AMDGPU::V_MAC_F16_e32 || Opc == AMDGPU::V_MAC_F16_e64 ||
3281  Opc == AMDGPU::V_FMAC_F16_e32 || Opc == AMDGPU::V_FMAC_F16_e64;
3282  bool IsFMA = Opc == AMDGPU::V_FMAC_F32_e32 || Opc == AMDGPU::V_FMAC_F32_e64 ||
3283  Opc == AMDGPU::V_FMAC_LEGACY_F32_e32 ||
3284  Opc == AMDGPU::V_FMAC_LEGACY_F32_e64 ||
3285  Opc == AMDGPU::V_FMAC_F16_e32 || Opc == AMDGPU::V_FMAC_F16_e64 ||
3286  Opc == AMDGPU::V_FMAC_F64_e32 || Opc == AMDGPU::V_FMAC_F64_e64;
3287  bool IsF64 = Opc == AMDGPU::V_FMAC_F64_e32 || Opc == AMDGPU::V_FMAC_F64_e64;
3288  bool IsLegacy = Opc == AMDGPU::V_MAC_LEGACY_F32_e32 ||
3289  Opc == AMDGPU::V_MAC_LEGACY_F32_e64 ||
3290  Opc == AMDGPU::V_FMAC_LEGACY_F32_e32 ||
3291  Opc == AMDGPU::V_FMAC_LEGACY_F32_e64;
3292  bool Src0Literal = false;
3293 
3294  switch (Opc) {
3295  default:
3296  return nullptr;
3297  case AMDGPU::V_MAC_F16_e64:
3298  case AMDGPU::V_FMAC_F16_e64:
3299  case AMDGPU::V_MAC_F32_e64:
3300  case AMDGPU::V_MAC_LEGACY_F32_e64:
3301  case AMDGPU::V_FMAC_F32_e64:
3302  case AMDGPU::V_FMAC_LEGACY_F32_e64:
3303  case AMDGPU::V_FMAC_F64_e64:
3304  break;
3305  case AMDGPU::V_MAC_F16_e32:
3306  case AMDGPU::V_FMAC_F16_e32:
3307  case AMDGPU::V_MAC_F32_e32:
3308  case AMDGPU::V_MAC_LEGACY_F32_e32:
3309  case AMDGPU::V_FMAC_F32_e32:
3310  case AMDGPU::V_FMAC_LEGACY_F32_e32:
3311  case AMDGPU::V_FMAC_F64_e32: {
3312  int Src0Idx = AMDGPU::getNamedOperandIdx(MI.getOpcode(),
3313  AMDGPU::OpName::src0);
3314  const MachineOperand *Src0 = &MI.getOperand(Src0Idx);
3315  if (!Src0->isReg() && !Src0->isImm())
3316  return nullptr;
3317 
3318  if (Src0->isImm() && !isInlineConstant(MI, Src0Idx, *Src0))
3319  Src0Literal = true;
3320 
3321  break;
3322  }
3323  }
3324 
3325  MachineInstrBuilder MIB;
3326  const MachineOperand *Dst = getNamedOperand(MI, AMDGPU::OpName::vdst);
3327  const MachineOperand *Src0 = getNamedOperand(MI, AMDGPU::OpName::src0);
3328  const MachineOperand *Src0Mods =
3329  getNamedOperand(MI, AMDGPU::OpName::src0_modifiers);
3330  const MachineOperand *Src1 = getNamedOperand(MI, AMDGPU::OpName::src1);
3331  const MachineOperand *Src1Mods =
3332  getNamedOperand(MI, AMDGPU::OpName::src1_modifiers);
3333  const MachineOperand *Src2 = getNamedOperand(MI, AMDGPU::OpName::src2);
3334  const MachineOperand *Src2Mods =
3335  getNamedOperand(MI, AMDGPU::OpName::src2_modifiers);
3337  const MachineOperand *Omod = getNamedOperand(MI, AMDGPU::OpName::omod);
3338 
3339  if (!Src0Mods && !Src1Mods && !Src2Mods && !Clamp && !Omod && !IsF64 &&
3340  !IsLegacy &&
3341  // If we have an SGPR input, we will violate the constant bus restriction.
3342  (ST.getConstantBusLimit(Opc) > 1 || !Src0->isReg() ||
3343  !RI.isSGPRReg(MBB.getParent()->getRegInfo(), Src0->getReg()))) {
3345  const auto killDef = [&]() -> void {
3347  // The only user is the instruction which will be killed.
3348  Register DefReg = DefMI->getOperand(0).getReg();
3349  if (!MRI.hasOneNonDBGUse(DefReg))
3350  return;
3351  // We cannot just remove the DefMI here, calling pass will crash.
3352  DefMI->setDesc(get(AMDGPU::IMPLICIT_DEF));
3353  for (unsigned I = DefMI->getNumOperands() - 1; I != 0; --I)
3354  DefMI->removeOperand(I);
3355  if (LV)
3356  LV->getVarInfo(DefReg).AliveBlocks.clear();
3357  };
3358 
3359  int64_t Imm;
3360  if (!Src0Literal && getFoldableImm(Src2, Imm, &DefMI)) {
3361  unsigned NewOpc =
3362  IsFMA ? (IsF16 ? AMDGPU::V_FMAAK_F16 : AMDGPU::V_FMAAK_F32)
3363  : (IsF16 ? AMDGPU::V_MADAK_F16 : AMDGPU::V_MADAK_F32);
3364  if (pseudoToMCOpcode(NewOpc) != -1) {
3365  MIB = BuildMI(MBB, MI, MI.getDebugLoc(), get(NewOpc))
3366  .add(*Dst)
3367  .add(*Src0)
3368  .add(*Src1)
3369  .addImm(Imm);
3370  updateLiveVariables(LV, MI, *MIB);
3371  if (LIS)
3372  LIS->ReplaceMachineInstrInMaps(MI, *MIB);
3373  killDef();
3374  return MIB;
3375  }
3376  }
3377  unsigned NewOpc = IsFMA
3378  ? (IsF16 ? AMDGPU::V_FMAMK_F16 : AMDGPU::V_FMAMK_F32)
3379  : (IsF16 ? AMDGPU::V_MADMK_F16 : AMDGPU::V_MADMK_F32);
3380  if (!Src0Literal && getFoldableImm(Src1, Imm, &DefMI)) {
3381  if (pseudoToMCOpcode(NewOpc) != -1) {
3382  MIB = BuildMI(MBB, MI, MI.getDebugLoc(), get(NewOpc))
3383  .add(*Dst)
3384  .add(*Src0)
3385  .addImm(Imm)
3386  .add(*Src2);
3387  updateLiveVariables(LV, MI, *MIB);
3388  if (LIS)
3389  LIS->ReplaceMachineInstrInMaps(MI, *MIB);
3390  killDef();
3391  return MIB;
3392  }
3393  }
3394  if (Src0Literal || getFoldableImm(Src0, Imm, &DefMI)) {
3395  if (Src0Literal) {
3396  Imm = Src0->getImm();
3397  DefMI = nullptr;
3398  }
3399  if (pseudoToMCOpcode(NewOpc) != -1 &&
3401  MI, AMDGPU::getNamedOperandIdx(NewOpc, AMDGPU::OpName::src0),
3402  Src1)) {
3403  MIB = BuildMI(MBB, MI, MI.getDebugLoc(), get(NewOpc))
3404  .add(*Dst)
3405  .add(*Src1)
3406  .addImm(Imm)
3407  .add(*Src2);
3408  updateLiveVariables(LV, MI, *MIB);
3409  if (LIS)
3410  LIS->ReplaceMachineInstrInMaps(MI, *MIB);
3411  if (DefMI)
3412  killDef();
3413  return MIB;
3414  }
3415  }
3416  }
3417 
3418  // VOP2 mac/fmac with a literal operand cannot be converted to VOP3 mad/fma
3419  // because VOP3 does not allow a literal operand.
3420  // TODO: Remove this restriction for GFX10.
3421  if (Src0Literal)
3422  return nullptr;
3423 
3424  unsigned NewOpc = IsFMA ? IsF16 ? AMDGPU::V_FMA_F16_gfx9_e64
3425  : IsF64 ? AMDGPU::V_FMA_F64_e64
3426  : IsLegacy
3427  ? AMDGPU::V_FMA_LEGACY_F32_e64
3428  : AMDGPU::V_FMA_F32_e64
3429  : IsF16 ? AMDGPU::V_MAD_F16_e64
3430  : IsLegacy ? AMDGPU::V_MAD_LEGACY_F32_e64
3431  : AMDGPU::V_MAD_F32_e64;
3432  if (pseudoToMCOpcode(NewOpc) == -1)
3433  return nullptr;
3434 
3435  MIB = BuildMI(MBB, MI, MI.getDebugLoc(), get(NewOpc))
3436  .add(*Dst)
3437  .addImm(Src0Mods ? Src0Mods->getImm() : 0)
3438  .add(*Src0)
3439  .addImm(Src1Mods ? Src1Mods->getImm() : 0)
3440  .add(*Src1)
3441  .addImm(Src2Mods ? Src2Mods->getImm() : 0)
3442  .add(*Src2)
3443  .addImm(Clamp ? Clamp->getImm() : 0)
3444  .addImm(Omod ? Omod->getImm() : 0);
3445  updateLiveVariables(LV, MI, *MIB);
3446  if (LIS)
3447  LIS->ReplaceMachineInstrInMaps(MI, *MIB);
3448  return MIB;
3449 }
3450 
3451 // It's not generally safe to move VALU instructions across these since it will
3452 // start using the register as a base index rather than directly.
3453 // XXX - Why isn't hasSideEffects sufficient for these?
3455  switch (MI.getOpcode()) {
3456  case AMDGPU::S_SET_GPR_IDX_ON:
3457  case AMDGPU::S_SET_GPR_IDX_MODE:
3458  case AMDGPU::S_SET_GPR_IDX_OFF:
3459  return true;
3460  default:
3461  return false;
3462  }
3463 }
3464 
3466  const MachineBasicBlock *MBB,
3467  const MachineFunction &MF) const {
3468  // Skipping the check for SP writes in the base implementation. The reason it
3469  // was added was apparently due to compile time concerns.
3470  //
3471  // TODO: Do we really want this barrier? It triggers unnecessary hazard nops
3472  // but is probably avoidable.
3473 
3474  // Copied from base implementation.
3475  // Terminators and labels can't be scheduled around.
3476  if (MI.isTerminator() || MI.isPosition())
3477  return true;
3478 
3479  // INLINEASM_BR can jump to another block
3480  if (MI.getOpcode() == TargetOpcode::INLINEASM_BR)
3481  return true;
3482 
3483  if (MI.getOpcode() == AMDGPU::SCHED_BARRIER && MI.getOperand(0).getImm() == 0)
3484  return true;
3485 
3486  // Target-independent instructions do not have an implicit-use of EXEC, even
3487  // when they operate on VGPRs. Treating EXEC modifications as scheduling
3488  // boundaries prevents incorrect movements of such instructions.
3489  return MI.modifiesRegister(AMDGPU::EXEC, &RI) ||
3490  MI.getOpcode() == AMDGPU::S_SETREG_IMM32_B32 ||
3491  MI.getOpcode() == AMDGPU::S_SETREG_B32 ||
3493 }
3494 
3496  return Opcode == AMDGPU::DS_ORDERED_COUNT ||
3497  Opcode == AMDGPU::DS_GWS_INIT ||
3498  Opcode == AMDGPU::DS_GWS_SEMA_V ||
3499  Opcode == AMDGPU::DS_GWS_SEMA_BR ||
3500  Opcode == AMDGPU::DS_GWS_SEMA_P ||
3501  Opcode == AMDGPU::DS_GWS_SEMA_RELEASE_ALL ||
3502  Opcode == AMDGPU::DS_GWS_BARRIER;
3503 }
3504 
3506  // Skip the full operand and register alias search modifiesRegister
3507  // does. There's only a handful of instructions that touch this, it's only an
3508  // implicit def, and doesn't alias any other registers.
3509  if (const MCPhysReg *ImpDef = MI.getDesc().getImplicitDefs()) {
3510  for (; ImpDef && *ImpDef; ++ImpDef) {
3511  if (*ImpDef == AMDGPU::MODE)
3512  return true;
3513  }
3514  }
3515 
3516  return false;
3517 }
3518 
3520  unsigned Opcode = MI.getOpcode();
3521 
3522  if (MI.mayStore() && isSMRD(MI))
3523  return true; // scalar store or atomic
3524 
3525  // This will terminate the function when other lanes may need to continue.
3526  if (MI.isReturn())
3527  return true;
3528 
3529  // These instructions cause shader I/O that may cause hardware lockups
3530  // when executed with an empty EXEC mask.
3531  //
3532  // Note: exp with VM = DONE = 0 is automatically skipped by hardware when
3533  // EXEC = 0, but checking for that case here seems not worth it
3534  // given the typical code patterns.
3535  if (Opcode == AMDGPU::S_SENDMSG || Opcode == AMDGPU::S_SENDMSGHALT ||
3536  isEXP(Opcode) ||
3537  Opcode == AMDGPU::DS_ORDERED_COUNT || Opcode == AMDGPU::S_TRAP ||
3538  Opcode == AMDGPU::DS_GWS_INIT || Opcode == AMDGPU::DS_GWS_BARRIER)
3539  return true;
3540 
3541  if (MI.isCall() || MI.isInlineAsm())
3542  return true; // conservative assumption
3543 
3544  // A mode change is a scalar operation that influences vector instructions.
3545  if (modifiesModeRegister(MI))
3546  return true;
3547 
3548  // These are like SALU instructions in terms of effects, so it's questionable
3549  // whether we should return true for those.
3550  //
3551  // However, executing them with EXEC = 0 causes them to operate on undefined
3552  // data, which we avoid by returning true here.
3553  if (Opcode == AMDGPU::V_READFIRSTLANE_B32 ||
3554  Opcode == AMDGPU::V_READLANE_B32 || Opcode == AMDGPU::V_WRITELANE_B32)
3555  return true;
3556 
3557  return false;
3558 }
3559 
3561  const MachineInstr &MI) const {
3562  if (MI.isMetaInstruction())
3563  return false;
3564 
3565  // This won't read exec if this is an SGPR->SGPR copy.
3566  if (MI.isCopyLike()) {
3567  if (!RI.isSGPRReg(MRI, MI.getOperand(0).getReg()))
3568  return true;
3569 
3570  // Make sure this isn't copying exec as a normal operand
3571  return MI.readsRegister(AMDGPU::EXEC, &RI);
3572  }
3573 
3574  // Make a conservative assumption about the callee.
3575  if (MI.isCall())
3576  return true;
3577 
3578  // Be conservative with any unhandled generic opcodes.
3579  if (!isTargetSpecificOpcode(MI.getOpcode()))
3580  return true;
3581 
3582  return !isSALU(MI) || MI.readsRegister(AMDGPU::EXEC, &RI);
3583 }
3584 
3586  switch (Imm.getBitWidth()) {
3587  case 1: // This likely will be a condition code mask.
3588  return true;
3589 
3590  case 32:
3591  return AMDGPU::isInlinableLiteral32(Imm.getSExtValue(),
3592  ST.hasInv2PiInlineImm());
3593  case 64:
3594  return AMDGPU::isInlinableLiteral64(Imm.getSExtValue(),
3595  ST.hasInv2PiInlineImm());
3596  case 16:
3597  return ST.has16BitInsts() &&
3598  AMDGPU::isInlinableLiteral16(Imm.getSExtValue(),
3599  ST.hasInv2PiInlineImm());
3600  default:
3601  llvm_unreachable("invalid bitwidth");
3602  }
3603 }
3604 
3606  uint8_t OperandType) const {
3607  if (!MO.isImm() ||
3610  return false;
3611 
3612  // MachineOperand provides no way to tell the true operand size, since it only
3613  // records a 64-bit value. We need to know the size to determine if a 32-bit
3614  // floating point immediate bit pattern is legal for an integer immediate. It
3615  // would be for any 32-bit integer operand, but would not be for a 64-bit one.
3616 
3617  int64_t Imm = MO.getImm();
3618  switch (OperandType) {
3630  int32_t Trunc = static_cast<int32_t>(Imm);
3632  }
3639  ST.hasInv2PiInlineImm());
3643  // We would expect inline immediates to not be concerned with an integer/fp
3644  // distinction. However, in the case of 16-bit integer operations, the
3645  // "floating point" values appear to not work. It seems read the low 16-bits
3646  // of 32-bit immediates, which happens to always work for the integer
3647  // values.
3648  //
3649  // See llvm bugzilla 46302.
3650  //
3651  // TODO: Theoretically we could use op-sel to use the high bits of the
3652  // 32-bit FP values.
3657  // This suffers the same problem as the scalar 16-bit cases.
3663  if (isInt<16>(Imm) || isUInt<16>(Imm)) {
3664  // A few special case instructions have 16-bit operands on subtargets
3665  // where 16-bit instructions are not legal.
3666  // TODO: Do the 32-bit immediates work? We shouldn't really need to handle
3667  // constants in these cases
3668  int16_t Trunc = static_cast<int16_t>(Imm);
3669  return ST.has16BitInsts() &&
3671  }
3672 
3673  return false;
3674  }
3678  uint32_t Trunc = static_cast<uint32_t>(Imm);
3680  }
3683  return false;
3684  default:
3685  llvm_unreachable("invalid bitwidth");
3686  }
3687 }
3688 
3690  const MCOperandInfo &OpInfo) const {
3691  switch (MO.getType()) {
3693  return false;
3695  return !isInlineConstant(MO, OpInfo);
3701  return true;
3702  default:
3703  llvm_unreachable("unexpected operand type");
3704  }
3705 }
3706 
3707 static bool compareMachineOp(const MachineOperand &Op0,
3708  const MachineOperand &Op1) {
3709  if (Op0.getType() != Op1.getType())
3710  return false;
3711 
3712  switch (Op0.getType()) {
3714  return Op0.getReg() == Op1.getReg();
3716  return Op0.getImm() == Op1.getImm();
3717  default:
3718  llvm_unreachable("Didn't expect to be comparing these operand types");
3719  }
3720 }
3721 
3723  const MachineOperand &MO) const {
3724  const MCInstrDesc &InstDesc = MI.getDesc();
3725  const MCOperandInfo &OpInfo = InstDesc.OpInfo[OpNo];
3726 
3727  assert(MO.isImm() || MO.isTargetIndex() || MO.isFI() || MO.isGlobal());
3728 
3729  if (OpInfo.OperandType == MCOI::OPERAND_IMMEDIATE)
3730  return true;
3731 
3732  if (OpInfo.RegClass < 0)
3733  return false;
3734 
3735  if (MO.isImm() && isInlineConstant(MO, OpInfo)) {
3736  if (isMAI(MI) && ST.hasMFMAInlineLiteralBug() &&
3737  OpNo ==(unsigned)AMDGPU::getNamedOperandIdx(MI.getOpcode(),
3738  AMDGPU::OpName::src2))
3739  return false;
3740  return RI.opCanUseInlineConstant(OpInfo.OperandType);
3741  }
3742 
3743  if (!RI.opCanUseLiteralConstant(OpInfo.OperandType))
3744  return false;
3745 
3746  if (!isVOP3(MI) || !AMDGPU::isSISrcOperand(InstDesc, OpNo))
3747  return true;
3748 
3749  return ST.hasVOP3Literal();
3750 }
3751 
3752 bool SIInstrInfo::hasVALU32BitEncoding(unsigned Opcode) const {
3753  // GFX90A does not have V_MUL_LEGACY_F32_e32.
3754  if (Opcode == AMDGPU::V_MUL_LEGACY_F32_e64 && ST.hasGFX90AInsts())
3755  return false;
3756 
3757  int Op32 = AMDGPU::getVOPe32(Opcode);
3758  if (Op32 == -1)
3759  return false;
3760 
3761  return pseudoToMCOpcode(Op32) != -1;
3762 }
3763 
3764 bool SIInstrInfo::hasModifiers(unsigned Opcode) const {
3765  // The src0_modifier operand is present on all instructions
3766  // that have modifiers.
3767 
3768  return AMDGPU::getNamedOperandIdx(Opcode,
3769  AMDGPU::OpName::src0_modifiers) != -1;
3770 }
3771 
3773  unsigned OpName) const {
3774  const MachineOperand *Mods = getNamedOperand(MI, OpName);
3775  return Mods && Mods->getImm();
3776 }
3777 
3779  return any_of(ModifierOpNames,
3780  [&](unsigned Name) { return hasModifiersSet(MI, Name); });
3781 }
3782 
3784  const MachineRegisterInfo &MRI) const {
3785  const MachineOperand *Src2 = getNamedOperand(MI, AMDGPU::OpName::src2);
3786  // Can't shrink instruction with three operands.
3787  if (Src2) {
3788  switch (MI.getOpcode()) {
3789  default: return false;
3790 
3791  case AMDGPU::V_ADDC_U32_e64:
3792  case AMDGPU::V_SUBB_U32_e64:
3793  case AMDGPU::V_SUBBREV_U32_e64: {
3794  const MachineOperand *Src1
3795  = getNamedOperand(MI, AMDGPU::OpName::src1);
3796  if (!Src1->isReg() || !RI.isVGPR(MRI, Src1->getReg()))
3797  return false;
3798  // Additional verification is needed for sdst/src2.
3799  return true;
3800  }
3801  case AMDGPU::V_MAC_F16_e64:
3802  case AMDGPU::V_MAC_F32_e64:
3803  case AMDGPU::V_MAC_LEGACY_F32_e64:
3804  case AMDGPU::V_FMAC_F16_e64:
3805  case AMDGPU::V_FMAC_F32_e64:
3806  case AMDGPU::V_FMAC_F64_e64:
3807  case AMDGPU::V_FMAC_LEGACY_F32_e64:
3808  if (!Src2->isReg() || !RI.isVGPR(MRI, Src2->getReg()) ||
3809  hasModifiersSet(MI, AMDGPU::OpName::src2_modifiers))
3810  return false;
3811  break;
3812 
3813  case AMDGPU::V_CNDMASK_B32_e64:
3814  break;
3815  }
3816  }
3817 
3818  const MachineOperand *Src1 = getNamedOperand(MI, AMDGPU::OpName::src1);
3819  if (Src1 && (!Src1->isReg() || !RI.isVGPR(MRI, Src1->getReg()) ||
3820  hasModifiersSet(MI, AMDGPU::OpName::src1_modifiers)))
3821  return false;
3822 
3823  // We don't need to check src0, all input types are legal, so just make sure
3824  // src0 isn't using any modifiers.
3825  if (hasModifiersSet(MI, AMDGPU::OpName::src0_modifiers))
3826  return false;
3827 
3828  // Can it be shrunk to a valid 32 bit opcode?
3829  if (!hasVALU32BitEncoding(MI.getOpcode()))
3830  return false;
3831 
3832  // Check output modifiers
3833  return !hasModifiersSet(MI, AMDGPU::OpName::omod) &&
3835 }
3836 
3837 // Set VCC operand with all flags from \p Orig, except for setting it as
3838 // implicit.
3840  const MachineOperand &Orig) {
3841 
3842  for (MachineOperand &Use : MI.implicit_operands()) {
3843  if (Use.isUse() &&
3844  (Use.getReg() == AMDGPU::VCC || Use.getReg() == AMDGPU::VCC_LO)) {
3845  Use.setIsUndef(Orig.isUndef());
3846  Use.setIsKill(Orig.isKill());
3847  return;
3848  }
3849  }
3850 }
3851 
3853  unsigned Op32) const {
3854  MachineBasicBlock *MBB = MI.getParent();
3855  MachineInstrBuilder Inst32 =
3856  BuildMI(*MBB, MI, MI.getDebugLoc(), get(Op32))
3857  .setMIFlags(MI.getFlags());
3858 
3859  // Add the dst operand if the 32-bit encoding also has an explicit $vdst.
3860  // For VOPC instructions, this is replaced by an implicit def of vcc.
3861  if (AMDGPU::getNamedOperandIdx(Op32, AMDGPU::OpName::vdst) != -1) {
3862  // dst
3863  Inst32.add(MI.getOperand(0));
3864  } else if (AMDGPU::getNamedOperandIdx(Op32, AMDGPU::OpName::sdst) != -1) {
3865  // VOPCX instructions won't be writing to an explicit dst, so this should
3866  // not fail for these instructions.
3867  assert(((MI.getOperand(0).getReg() == AMDGPU::VCC) ||
3868  (MI.getOperand(0).getReg() == AMDGPU::VCC_LO)) &&
3869  "Unexpected case");
3870  }
3871 
3872  Inst32.add(*getNamedOperand(MI, AMDGPU::OpName::src0));
3873 
3874  const MachineOperand *Src1 = getNamedOperand(MI, AMDGPU::OpName::src1);
3875  if (Src1)
3876  Inst32.add(*Src1);
3877 
3878  const MachineOperand *Src2 = getNamedOperand(MI, AMDGPU::OpName::src2);
3879 
3880  if (Src2) {
3881  int Op32Src2Idx = AMDGPU::getNamedOperandIdx(Op32, AMDGPU::OpName::src2);
3882  if (Op32Src2Idx != -1) {
3883  Inst32.add(*Src2);
3884  } else {
3885  // In the case of V_CNDMASK_B32_e32, the explicit operand src2 is
3886  // replaced with an implicit read of vcc or vcc_lo. The implicit read
3887  // of vcc was already added during the initial BuildMI, but we
3888  // 1) may need to change vcc to vcc_lo to preserve the original register
3889  // 2) have to preserve the original flags.
3890  fixImplicitOperands(*Inst32);
3891  copyFlagsToImplicitVCC(*Inst32, *Src2);
3892  }
3893  }
3894 
3895  return Inst32;
3896 }
3897 
3899  const MachineOperand &MO,
3900  const MCOperandInfo &OpInfo) const {
3901  // Literal constants use the constant bus.
3902  //if (isLiteralConstantLike(MO, OpInfo))
3903  // return true;
3904  if (MO.isImm())
3905  return !isInlineConstant(MO, OpInfo);
3906 
3907  if (!MO.isReg())
3908  return true; // Misc other operands like FrameIndex
3909 
3910  if (!MO.isUse())
3911  return false;
3912 
3913  if (MO.getReg().isVirtual())
3914  return RI.isSGPRClass(MRI.getRegClass(MO.getReg()));
3915 
3916  // Null is free
3917  if (MO.getReg() == AMDGPU::SGPR_NULL || MO.getReg() == AMDGPU::SGPR_NULL64)
3918  return false;
3919 
3920  // SGPRs use the constant bus
3921  if (MO.isImplicit()) {
3922  return MO.getReg() == AMDGPU::M0 ||
3923  MO.getReg() == AMDGPU::VCC ||
3924  MO.getReg() == AMDGPU::VCC_LO;
3925  } else {
3926  return AMDGPU::SReg_32RegClass.contains(MO.getReg()) ||
3927  AMDGPU::SReg_64RegClass.contains(MO.getReg());
3928  }
3929 }
3930 
3932  for (const MachineOperand &MO : MI.implicit_operands()) {
3933  // We only care about reads.
3934  if (MO.isDef())
3935  continue;
3936 
3937  switch (MO.getReg()) {
3938  case AMDGPU::VCC:
3939  case AMDGPU::VCC_LO:
3940  case AMDGPU::VCC_HI:
3941  case AMDGPU::M0:
3942  case AMDGPU::FLAT_SCR:
3943  return MO.getReg();
3944 
3945  default:
3946  break;
3947  }
3948  }
3949 
3950  return AMDGPU::NoRegister;
3951 }
3952 
3953 static bool shouldReadExec(const MachineInstr &MI) {
3954  if (SIInstrInfo::isVALU(MI)) {
3955  switch (MI.getOpcode()) {
3956  case AMDGPU::V_READLANE_B32:
3957  case AMDGPU::V_WRITELANE_B32:
3958  return false;
3959  }
3960 
3961  return true;
3962  }
3963 
3964  if (MI.isPreISelOpcode() ||
3965  SIInstrInfo::isGenericOpcode(MI.getOpcode()) ||
3968  return false;
3969 
3970  return true;
3971 }
3972 
3973 static bool isSubRegOf(const SIRegisterInfo &TRI,
3974  const MachineOperand &SuperVec,
3975  const MachineOperand &SubReg) {
3976  if (SubReg.getReg().isPhysical())
3977  return TRI.isSubRegister(SuperVec.getReg(), SubReg.getReg());
3978 
3979  return SubReg.getSubReg() != AMDGPU::NoSubRegister &&
3980  SubReg.getReg() == SuperVec.getReg();
3981 }
3982 
3984  StringRef &ErrInfo) const {
3985  uint16_t Opcode = MI.getOpcode();
3986  if (SIInstrInfo::isGenericOpcode(MI.getOpcode()))
3987  return true;
3988 
3989  const MachineFunction *MF = MI.getParent()->getParent();
3990  const MachineRegisterInfo &MRI = MF->getRegInfo();
3991 
3992  int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0);
3993  int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src1);
3994  int Src2Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src2);
3995  int Src3Idx = -1;
3996  if (Src0Idx == -1) {
3997  // VOPD V_DUAL_* instructions use different operand names.
3998  Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0X);
3999  Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vsrc1X);
4000  Src2Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0Y);
4001  Src3Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vsrc1Y);
4002  }
4003 
4004  // Make sure the number of operands is correct.
4005  const MCInstrDesc &Desc = get(Opcode);
4006  if (!Desc.isVariadic() &&
4007  Desc.getNumOperands() != MI.getNumExplicitOperands()) {
4008  ErrInfo = "Instruction has wrong number of operands.";
4009  return false;
4010  }
4011 
4012  if (MI.isInlineAsm()) {
4013  // Verify register classes for inlineasm constraints.
4014  for (unsigned I = InlineAsm::MIOp_FirstOperand, E = MI.getNumOperands();
4015  I != E; ++I) {
4016  const TargetRegisterClass *RC = MI.getRegClassConstraint(I, this, &RI);
4017  if (!RC)
4018  continue;
4019 
4020  const MachineOperand &Op = MI.getOperand(I);
4021  if (!Op.isReg())
4022  continue;
4023 
4024  Register Reg = Op.getReg();
4025  if (!Reg.isVirtual() && !RC->contains(Reg)) {
4026  ErrInfo = "inlineasm operand has incorrect register class.";
4027  return false;
4028  }
4029  }
4030 
4031  return true;
4032  }
4033 
4034  if (isMIMG(MI) && MI.memoperands_empty() && MI.mayLoadOrStore()) {
4035  ErrInfo = "missing memory operand from MIMG instruction.";
4036  return false;
4037  }
4038 
4039  // Make sure the register classes are correct.
4040  for (int i = 0, e = Desc.getNumOperands(); i != e; ++i) {
4041  const MachineOperand &MO = MI.getOperand(i);
4042  if (MO.isFPImm()) {
4043  ErrInfo = "FPImm Machine Operands are not supported. ISel should bitcast "
4044  "all fp values to integers.";
4045  return false;
4046  }
4047 
4048  int RegClass = Desc.OpInfo[i].RegClass;
4049 
4050  switch (Desc.OpInfo[i].OperandType) {
4052  if (MI.getOperand(i).isImm() || MI.getOperand(i).isGlobal()) {
4053  ErrInfo = "Illegal immediate value for operand.";
4054  return false;
4055  }
4056  break;
4061  break;
4073  if (!MO.isReg() && (!MO.isImm() || !isInlineConstant(MI, i))) {
4074  ErrInfo = "Illegal immediate value for operand.";
4075  return false;
4076  }
4077  break;
4078  }
4081  // Check if this operand is an immediate.
4082  // FrameIndex operands will be replaced by immediates, so they are
4083  // allowed.
4084  if (!MI.getOperand(i).isImm() && !MI.getOperand(i).isFI()) {
4085  ErrInfo = "Expected immediate, but got non-immediate";
4086  return false;
4087  }
4089  default:
4090  continue;
4091  }
4092 
4093  if (!MO.isReg())
4094  continue;
4095  Register Reg = MO.getReg();
4096  if (!Reg)
4097  continue;
4098 
4099  // FIXME: Ideally we would have separate instruction definitions with the
4100  // aligned register constraint.
4101  // FIXME: We do not verify inline asm operands, but custom inline asm
4102  // verification is broken anyway
4103  if (ST.needsAlignedVGPRs()) {
4104  const TargetRegisterClass *RC = RI.getRegClassForReg(MRI, Reg);
4105  if (RI.hasVectorRegisters(RC) && MO.getSubReg()) {
4106  const TargetRegisterClass *SubRC =
4107  RI.getSubRegClass(RC, MO.getSubReg());
4108  RC = RI.getCompatibleSubRegClass(RC, SubRC, MO.getSubReg());
4109  if (RC)
4110  RC = SubRC;
4111  }
4112 
4113  // Check that this is the aligned version of the class.
4114  if (!RC || !RI.isProperlyAlignedRC(*RC)) {
4115  ErrInfo = "Subtarget requires even aligned vector registers";
4116  return false;
4117  }
4118  }
4119 
4120  if (RegClass != -1) {
4121  if (Reg.isVirtual())
4122  continue;
4123 
4124  const TargetRegisterClass *RC = RI.getRegClass(RegClass);
4125  if (!RC->contains(Reg)) {
4126  ErrInfo = "Operand has incorrect register class.";
4127  return false;
4128  }
4129  }
4130  }
4131 
4132  // Verify SDWA
4133  if (isSDWA(MI)) {
4134  if (!ST.hasSDWA()) {
4135  ErrInfo = "SDWA is not supported on this target";
4136  return false;
4137  }
4138 
4139  int DstIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdst);
4140 
4141  for (int OpIdx : {DstIdx, Src0Idx, Src1Idx, Src2Idx}) {
4142  if (OpIdx == -1)
4143  continue;
4144  const MachineOperand &MO = MI.getOperand(OpIdx);
4145 
4146  if (!ST.hasSDWAScalar()) {
4147  // Only VGPRS on VI
4148  if (!MO.isReg() || !RI.hasVGPRs(RI.getRegClassForReg(MRI, MO.getReg()))) {
4149  ErrInfo = "Only VGPRs allowed as operands in SDWA instructions on VI";
4150  return false;
4151  }
4152  } else {
4153  // No immediates on GFX9
4154  if (!MO.isReg()) {
4155  ErrInfo =
4156  "Only reg allowed as operands in SDWA instructions on GFX9+";
4157  return false;
4158  }
4159  }
4160  }
4161 
4162  if (!ST.hasSDWAOmod()) {
4163  // No omod allowed on VI
4164  const MachineOperand *OMod = getNamedOperand(MI, AMDGPU::OpName::omod);
4165  if (OMod != nullptr &&
4166  (!OMod->isImm() || OMod->getImm() != 0)) {
4167  ErrInfo = "OMod not allowed in SDWA instructions on VI";
4168  return false;
4169  }
4170  }
4171 
4172  uint16_t BasicOpcode = AMDGPU::getBasicFromSDWAOp(Opcode);
4173  if (isVOPC(BasicOpcode)) {
4174  if (!ST.hasSDWASdst() && DstIdx != -1) {
4175  // Only vcc allowed as dst on VI for VOPC
4176  const MachineOperand &Dst = MI.getOperand(DstIdx);
4177  if (!Dst.isReg() || Dst.getReg() != AMDGPU::VCC) {
4178  ErrInfo = "Only VCC allowed as dst in SDWA instructions on VI";
4179  return false;
4180  }
4181  } else if (!ST.hasSDWAOutModsVOPC()) {
4182  // No clamp allowed on GFX9 for VOPC
4184  if (Clamp && (!Clamp->isImm() || Clamp->getImm() != 0)) {
4185  ErrInfo = "Clamp not allowed in VOPC SDWA instructions on VI";
4186  return false;
4187  }
4188 
4189  // No omod allowed on GFX9 for VOPC
4190  const MachineOperand *OMod = getNamedOperand(MI, AMDGPU::OpName::omod);
4191  if (OMod && (!OMod->isImm() || OMod->getImm() != 0)) {
4192  ErrInfo = "OMod not allowed in VOPC SDWA instructions on VI";
4193  return false;
4194  }
4195  }
4196  }
4197 
4198  const MachineOperand *DstUnused = getNamedOperand(MI, AMDGPU::OpName::dst_unused);
4199  if (DstUnused && DstUnused->isImm() &&
4201  const MachineOperand &Dst = MI.getOperand(DstIdx);
4202  if (!Dst.isReg() || !Dst.isTied()) {
4203  ErrInfo = "Dst register should have tied register";
4204  return false;
4205  }
4206 
4207  const MachineOperand &TiedMO =
4208  MI.getOperand(MI.findTiedOperandIdx(DstIdx));
4209  if (!TiedMO.isReg() || !TiedMO.isImplicit() || !TiedMO.isUse()) {
4210  ErrInfo =
4211  "Dst register should be tied to implicit use of preserved register";
4212  return false;
4213  } else if (TiedMO.getReg().isPhysical() &&
4214  Dst.getReg() != TiedMO.getReg()) {
4215  ErrInfo = "Dst register should use same physical register as preserved";
4216  return false;
4217  }
4218  }
4219  }
4220 
4221  // Verify MIMG
4222  if (isMIMG(MI.getOpcode()) && !MI.mayStore()) {
4223  // Ensure that the return type used is large enough for all the options
4224  // being used TFE/LWE require an extra result register.
4225  const MachineOperand *DMask = getNamedOperand(MI, AMDGPU::OpName::dmask);
4226  if (DMask) {
4227  uint64_t DMaskImm = DMask->getImm();
4228  uint32_t RegCount =
4229  isGather4(MI.getOpcode()) ? 4 : countPopulation(DMaskImm);
4230  const MachineOperand *TFE = getNamedOperand(MI, AMDGPU::OpName::tfe);
4231  const MachineOperand *LWE = getNamedOperand(MI, AMDGPU::OpName::lwe);
4232  const MachineOperand *D16 = getNamedOperand(MI, AMDGPU::OpName::d16);
4233 
4234  // Adjust for packed 16 bit values
4235  if (D16 && D16->getImm() && !ST.hasUnpackedD16VMem())
4236  RegCount >>= 1;
4237 
4238  // Adjust if using LWE or TFE
4239  if ((LWE && LWE->getImm()) || (TFE && TFE->getImm()))
4240  RegCount += 1;
4241 
4242  const uint32_t DstIdx =
4243  AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::vdata);
4244  const MachineOperand &Dst = MI.getOperand(DstIdx);
4245  if (Dst.isReg()) {
4246  const TargetRegisterClass *DstRC = getOpRegClass(MI, DstIdx);
4247  uint32_t DstSize = RI.getRegSizeInBits(*DstRC) / 32;
4248  if (RegCount > DstSize) {
4249  ErrInfo = "MIMG instruction returns too many registers for dst "
4250  "register class";
4251  return false;
4252  }
4253  }
4254  }
4255  }
4256 
4257  // Verify VOP*. Ignore multiple sgpr operands on writelane.
4258  if (isVALU(MI) && Desc.getOpcode() != AMDGPU::V_WRITELANE_B32) {
4259  unsigned ConstantBusCount = 0;
4260  bool UsesLiteral = false;
4261  const MachineOperand *LiteralVal = nullptr;
4262 
4263  int ImmIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::imm);
4264  if (ImmIdx != -1) {
4265  ++ConstantBusCount;
4266  UsesLiteral = true;
4267  LiteralVal = &MI.getOperand(ImmIdx);
4268  }
4269 
4270  SmallVector<Register, 2> SGPRsUsed;
4271  Register SGPRUsed;
4272 
4273  // Only look at the true operands. Only a real operand can use the constant
4274  // bus, and we don't want to check pseudo-operands like the source modifier
4275  // flags.
4276  for (int OpIdx : {Src0Idx, Src1Idx, Src2Idx, Src3Idx}) {
4277  if (OpIdx == -1)
4278  continue;
4279  const MachineOperand &MO = MI.getOperand(OpIdx);
4280  if (usesConstantBus(MRI, MO, MI.getDesc().OpInfo[OpIdx])) {
4281  if (MO.isReg()) {
4282  SGPRUsed = MO.getReg();
4283  if (llvm::all_of(SGPRsUsed, [SGPRUsed](unsigned SGPR) {
4284  return SGPRUsed != SGPR;
4285  })) {
4286  ++ConstantBusCount;
4287  SGPRsUsed.push_back(SGPRUsed);
4288  }
4289  } else {
4290  if (!UsesLiteral) {
4291  ++ConstantBusCount;
4292  UsesLiteral = true;
4293  LiteralVal = &MO;
4294  } else if (!MO.isIdenticalTo(*LiteralVal)) {
4295  assert(isVOP2(MI) || isVOP3(MI));
4296  ErrInfo = "VOP2/VOP3 instruction uses more than one literal";
4297  return false;
4298  }
4299  }
4300  }
4301  }
4302 
4303  SGPRUsed = findImplicitSGPRRead(MI);
4304  if (SGPRUsed != AMDGPU::NoRegister) {
4305  // Implicit uses may safely overlap true operands
4306  if (llvm::all_of(SGPRsUsed, [this, SGPRUsed](unsigned SGPR) {
4307  return !RI.regsOverlap(SGPRUsed, SGPR);
4308  })) {
4309  ++ConstantBusCount;
4310  SGPRsUsed.push_back(SGPRUsed);
4311  }
4312  }
4313 
4314  // v_writelane_b32 is an exception from constant bus restriction:
4315  // vsrc0 can be sgpr, const or m0 and lane select sgpr, m0 or inline-const
4316  if (ConstantBusCount > ST.getConstantBusLimit(Opcode) &&
4317  Opcode != AMDGPU::V_WRITELANE_B32) {
4318  ErrInfo = "VOP* instruction violates constant bus restriction";
4319  return false;
4320  }
4321 
4322  if (isVOP3(MI) && UsesLiteral && !ST.hasVOP3Literal()) {
4323  ErrInfo = "VOP3 instruction uses literal";
4324  return false;
4325  }
4326  }
4327 
4328  // Special case for writelane - this can break the multiple constant bus rule,
4329  // but still can't use more than one SGPR register
4330  if (Desc.getOpcode() == AMDGPU::V_WRITELANE_B32) {
4331  unsigned SGPRCount = 0;
4332  Register SGPRUsed = AMDGPU::NoRegister;
4333 
4334  for (int OpIdx : {Src0Idx, Src1Idx}) {
4335  if (OpIdx == -1)
4336  break;
4337 
4338  const MachineOperand &MO = MI.getOperand(OpIdx);
4339 
4340  if (usesConstantBus(MRI, MO, MI.getDesc().OpInfo[OpIdx])) {
4341  if (MO.isReg() && MO.getReg() != AMDGPU::M0) {
4342  if (MO.getReg() != SGPRUsed)
4343  ++SGPRCount;
4344  SGPRUsed = MO.getReg();
4345  }
4346  }
4347  if (SGPRCount > ST.getConstantBusLimit(Opcode)) {
4348  ErrInfo = "WRITELANE instruction violates constant bus restriction";
4349  return false;
4350  }
4351  }
4352  }
4353 
4354  // Verify misc. restrictions on specific instructions.
4355  if (Desc.getOpcode() == AMDGPU::V_DIV_SCALE_F32_e64 ||
4356  Desc.getOpcode() == AMDGPU::V_DIV_SCALE_F64_e64) {
4357  const MachineOperand &Src0 = MI.getOperand(Src0Idx);
4358  const MachineOperand &Src1 = MI.getOperand(Src1Idx);
4359  const MachineOperand &Src2 = MI.getOperand(Src2Idx);
4360  if (Src0.isReg() && Src1.isReg() && Src2.isReg()) {
4361  if (!compareMachineOp(Src0, Src1) &&
4362  !compareMachineOp(Src0, Src2)) {
4363  ErrInfo = "v_div_scale_{f32|f64} require src0 = src1 or src2";
4364  return false;
4365  }
4366  }
4367  if ((getNamedOperand(MI, AMDGPU::OpName::src0_modifiers)->getImm() &
4368  SISrcMods::ABS) ||
4369  (getNamedOperand(MI, AMDGPU::OpName::src1_modifiers)->getImm() &
4370  SISrcMods::ABS) ||
4371  (getNamedOperand(MI, AMDGPU::OpName::src2_modifiers)->getImm() &
4372  SISrcMods::ABS)) {
4373  ErrInfo = "ABS not allowed in VOP3B instructions";
4374  return false;
4375  }
4376  }
4377 
4378  if (isSOP2(MI) || isSOPC(MI)) {
4379  const MachineOperand &Src0 = MI.getOperand(Src0Idx);
4380  const MachineOperand &Src1 = MI.getOperand(Src1Idx);
4381 
4382  if (!Src0.isReg() && !Src1.isReg() &&
4383  !isInlineConstant(Src0, Desc.OpInfo[Src0Idx].OperandType) &&
4384  !isInlineConstant(Src1, Desc.OpInfo[Src1Idx].OperandType) &&
4385  !Src0.isIdenticalTo(Src1)) {
4386  ErrInfo = "SOP2/SOPC instruction requires too many immediate constants";
4387  return false;
4388  }
4389  }
4390 
4391  if (isSOPK(MI)) {
4392  auto Op = getNamedOperand(MI, AMDGPU::OpName::simm16);
4393  if (Desc.isBranch()) {
4394  if (!Op->isMBB()) {
4395  ErrInfo = "invalid branch target for SOPK instruction";
4396  return false;
4397  }
4398  } else {
4399  uint64_t Imm = Op->getImm();
4400  if (sopkIsZext(MI)) {
4401  if (!isUInt<16>(Imm)) {
4402  ErrInfo = "invalid immediate for SOPK instruction";
4403  return false;
4404  }
4405  } else {
4406  if (!isInt<16>(Imm)) {
4407  ErrInfo = "invalid immediate for SOPK instruction";
4408  return false;
4409  }
4410  }
4411  }
4412  }
4413 
4414  if (Desc.getOpcode() == AMDGPU::V_MOVRELS_B32_e32 ||
4415  Desc.getOpcode() == AMDGPU::V_MOVRELS_B32_e64 ||
4416  Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e32 ||
4417  Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e64) {
4418  const bool IsDst = Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e32 ||
4419  Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e64;
4420 
4421  const unsigned StaticNumOps = Desc.getNumOperands() +
4422  Desc.getNumImplicitUses();
4423  const unsigned NumImplicitOps = IsDst ? 2 : 1;
4424 
4425  // Allow additional implicit operands. This allows a fixup done by the post
4426  // RA scheduler where the main implicit operand is killed and implicit-defs
4427  // are added for sub-registers that remain live after this instruction.
4428  if (MI.getNumOperands() < StaticNumOps + NumImplicitOps) {
4429  ErrInfo = "missing implicit register operands";
4430  return false;
4431  }
4432 
4433  const MachineOperand *Dst = getNamedOperand(MI, AMDGPU::OpName::vdst);
4434  if (IsDst) {
4435  if (!Dst->isUse()) {
4436  ErrInfo = "v_movreld_b32 vdst should be a use operand";
4437  return false;
4438  }
4439 
4440  unsigned UseOpIdx;
4441  if (!MI.isRegTiedToUseOperand(StaticNumOps, &UseOpIdx) ||
4442  UseOpIdx != StaticNumOps + 1) {
4443  ErrInfo = "movrel implicit operands should be tied";
4444  return false;
4445  }
4446  }
4447 
4448  const MachineOperand &Src0 = MI.getOperand(Src0Idx);
4449  const MachineOperand &ImpUse
4450  = MI.getOperand(StaticNumOps + NumImplicitOps - 1);
4451  if (!ImpUse.isReg() || !ImpUse.isUse() ||
4452  !isSubRegOf(RI, ImpUse, IsDst ? *Dst : Src0)) {
4453  ErrInfo = "src0 should be subreg of implicit vector use";
4454  return false;
4455  }
4456  }
4457 
4458  // Make sure we aren't losing exec uses in the td files. This mostly requires
4459  // being careful when using let Uses to try to add other use registers.
4460  if (shouldReadExec(MI)) {
4461  if (!MI.hasRegisterImplicitUseOperand(AMDGPU::EXEC)) {
4462  ErrInfo = "VALU instruction does not implicitly read exec mask";
4463  return false;
4464  }
4465  }
4466 
4467  if (isSMRD(MI)) {
4468  if (MI.mayStore() &&
4470  // The register offset form of scalar stores may only use m0 as the
4471  // soffset register.
4472  const MachineOperand *Soff = getNamedOperand(MI, AMDGPU::OpName::soffset);
4473  if (Soff && Soff->getReg() != AMDGPU::M0) {
4474  ErrInfo = "scalar stores must use m0 as offset register";
4475  return false;
4476  }
4477  }
4478  }
4479 
4480  if (isFLAT(MI) && !ST.hasFlatInstOffsets()) {
4481  const MachineOperand *Offset = getNamedOperand(MI, AMDGPU::OpName::offset);
4482  if (Offset->getImm() != 0) {
4483  ErrInfo = "subtarget does not support offsets in flat instructions";
4484  return false;
4485  }
4486  }
4487 
4488  if (isMIMG(MI)) {
4489  const MachineOperand *DimOp = getNamedOperand(MI, AMDGPU::OpName::dim);
4490  if (DimOp) {
4491  int VAddr0Idx = AMDGPU::getNamedOperandIdx(Opcode,
4492  AMDGPU::OpName::vaddr0);
4493  int SRsrcIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::srsrc);
4494  const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(Opcode);
4495  const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode =
4496  AMDGPU::getMIMGBaseOpcodeInfo(Info->BaseOpcode);
4497  const AMDGPU::MIMGDimInfo *Dim =
4499 
4500  if (!Dim) {
4501  ErrInfo = "dim is out of range";
4502  return false;
4503  }
4504 
4505  bool IsA16 = false;
4506  if (ST.hasR128A16()) {
4507  const MachineOperand *R128A16 = getNamedOperand(MI, AMDGPU::OpName::r128);
4508  IsA16 = R128A16->getImm() != 0;
4509  } else if (ST.hasGFX10A16()) {
4510  const MachineOperand *A16 = getNamedOperand(MI, AMDGPU::OpName::a16);
4511  IsA16 = A16->getImm() != 0;
4512  }
4513 
4514  bool IsNSA = SRsrcIdx - VAddr0Idx > 1;
4515 
4516  unsigned AddrWords =
4517  AMDGPU::getAddrSizeMIMGOp(BaseOpcode, Dim, IsA16, ST.hasG16());
4518 
4519  unsigned VAddrWords;
4520  if (IsNSA) {
4521  VAddrWords = SRsrcIdx - VAddr0Idx;
4522  } else {
4523  const TargetRegisterClass *RC = getOpRegClass(MI, VAddr0Idx);
4524  VAddrWords = MRI.getTargetRegisterInfo()->getRegSizeInBits(*RC) / 32;
4525  if (AddrWords > 8)
4526  AddrWords = 16;
4527  }
4528 
4529  if (VAddrWords != AddrWords) {
4530  LLVM_DEBUG(dbgs() << "bad vaddr size, expected " << AddrWords
4531  << " but got " << VAddrWords << "\n");
4532  ErrInfo = "bad vaddr size";
4533  return false;
4534  }
4535  }
4536  }
4537 
4538  const MachineOperand *DppCt = getNamedOperand(MI, AMDGPU::OpName::dpp_ctrl);
4539  if (DppCt) {
4540  using namespace AMDGPU::DPP;
4541 
4542  unsigned DC = DppCt->getImm();
4550  ErrInfo = "Invalid dpp_ctrl value";
4551  return false;
4552  }
4553  if (DC >= DppCtrl::WAVE_SHL1 && DC <= DppCtrl::WAVE_ROR1 &&
4554  ST.getGeneration() >= AMDGPUSubtarget::GFX10) {
4555  ErrInfo = "Invalid dpp_ctrl value: "
4556  "wavefront shifts are not supported on GFX10+";
4557  return false;
4558  }
4559  if (DC >= DppCtrl::BCAST15 && DC <= DppCtrl::BCAST31 &&
4560  ST.getGeneration() >= AMDGPUSubtarget::GFX10) {
4561  ErrInfo = "Invalid dpp_ctrl value: "
4562  "broadcasts are not supported on GFX10+";
4563  return false;
4564  }
4566  ST.getGeneration() < AMDGPUSubtarget::GFX10) {
4569  !ST.hasGFX90AInsts()) {
4570  ErrInfo = "Invalid dpp_ctrl value: "
4571  "row_newbroadcast/row_share is not supported before "
4572  "GFX90A/GFX10";
4573  return false;
4574  } else if (DC > DppCtrl::ROW_NEWBCAST_LAST || !ST.hasGFX90AInsts()) {
4575  ErrInfo = "Invalid dpp_ctrl value: "
4576  "row_share and row_xmask are not supported before GFX10";
4577  return false;
4578  }
4579  }
4580 
4581  int DstIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdst);
4582 
4583  if (Opcode != AMDGPU::V_MOV_B64_DPP_PSEUDO &&
4584  ((DstIdx >= 0 &&
4585  (Desc.OpInfo[DstIdx].RegClass == AMDGPU::VReg_64RegClassID ||
4586  Desc.OpInfo[DstIdx].RegClass == AMDGPU::VReg_64_Align2RegClassID)) ||
4587  ((Src0Idx >= 0 &&
4588  (Desc.OpInfo[Src0Idx].RegClass == AMDGPU::VReg_64RegClassID ||
4589  Desc.OpInfo[Src0Idx].RegClass ==
4590  AMDGPU::VReg_64_Align2RegClassID)))) &&
4592  ErrInfo = "Invalid dpp_ctrl value: "
4593  "64 bit dpp only support row_newbcast";
4594  return false;
4595  }
4596  }
4597 
4598  if ((MI.mayStore() || MI.mayLoad()) && !isVGPRSpill(MI)) {
4599  const MachineOperand *Dst = getNamedOperand(MI, AMDGPU::OpName::vdst);
4600  uint16_t DataNameIdx = isDS(Opcode) ? AMDGPU::OpName::data0
4601  : AMDGPU::OpName::vdata;
4602  const MachineOperand *Data = getNamedOperand(MI, DataNameIdx);
4603  const MachineOperand *Data2 = getNamedOperand(MI, AMDGPU::OpName::data1);
4604  if (Data && !Data->isReg())
4605  Data = nullptr;
4606 
4607  if (ST.hasGFX90AInsts()) {
4608  if (Dst && Data &&
4609  (RI.isAGPR(MRI, Dst->getReg()) != RI.isAGPR(MRI, Data->getReg()))) {
4610  ErrInfo = "Invalid register class: "
4611  "vdata and vdst should be both VGPR or AGPR";
4612  return false;
4613  }
4614  if (Data && Data2 &&
4615  (RI.isAGPR(MRI, Data->getReg()) != RI.isAGPR(MRI, Data2->getReg()))) {
4616  ErrInfo = "Invalid register class: "
4617  "both data operands should be VGPR or AGPR";
4618  return false;
4619  }
4620  } else {
4621  if ((Dst && RI.isAGPR(MRI, Dst->getReg())) ||
4622  (Data && RI.isAGPR(MRI, Data->getReg())) ||
4623  (Data2 && RI.isAGPR(MRI, Data2->getReg()))) {
4624  ErrInfo = "Invalid register class: "
4625  "agpr loads and stores not supported on this GPU";
4626  return false;
4627  }
4628  }
4629  }
4630 
4631  if (ST.needsAlignedVGPRs()) {
4632  const auto isAlignedReg = [&MI, &MRI, this](unsigned OpName) -> bool {
4634  if (!Op)
4635  return true;
4636  Register Reg = Op->getReg();
4637  if (Reg.isPhysical())
4638  return !(RI.getHWRegIndex(Reg) & 1);
4639  const TargetRegisterClass &RC = *MRI.getRegClass(Reg);
4640  return RI.getRegSizeInBits(RC) > 32 && RI.isProperlyAlignedRC(RC) &&
4641  !(RI.getChannelFromSubReg(Op->getSubReg()) & 1);
4642  };
4643 
4644  if (MI.getOpcode() == AMDGPU::DS_GWS_INIT ||
4645  MI.getOpcode() == AMDGPU::DS_GWS_SEMA_BR ||
4646  MI.getOpcode() == AMDGPU::DS_GWS_BARRIER) {
4647 
4648  if (!isAlignedReg(AMDGPU::OpName::data0)) {
4649  ErrInfo = "Subtarget requires even aligned vector registers "
4650  "for DS_GWS instructions";
4651  return false;
4652  }
4653  }
4654 
4655  if (isMIMG(MI)) {
4656  if (!isAlignedReg(AMDGPU::OpName::vaddr)) {
4657  ErrInfo = "Subtarget requires even aligned vector registers "
4658  "for vaddr operand of image instructions";
4659  return false;
4660  }
4661  }
4662  }
4663 
4664  if (MI.getOpcode() == AMDGPU::V_ACCVGPR_WRITE_B32_e64 &&
4665  !ST.hasGFX90AInsts()) {
4666  const MachineOperand *Src = getNamedOperand(MI, AMDGPU::OpName::src0);
4667  if (Src->isReg() && RI.isSGPRReg(MRI, Src->getReg())) {
4668  ErrInfo = "Invalid register class: "
4669  "v_accvgpr_write with an SGPR is not supported on this GPU";
4670  return false;
4671  }
4672  }
4673 
4674  if (Desc.getOpcode() == AMDGPU::G_AMDGPU_WAVE_ADDRESS) {
4675  const MachineOperand &SrcOp = MI.getOperand(1);
4676  if (!SrcOp.isReg() || SrcOp.getReg().isVirtual()) {
4677  ErrInfo = "pseudo expects only physical SGPRs";
4678  return false;
4679  }
4680  }
4681 
4682  return true;
4683 }
4684 
4685 unsigned SIInstrInfo::getVALUOp(const MachineInstr &MI) const {
4686  switch (MI.getOpcode()) {
4687  default: return AMDGPU::INSTRUCTION_LIST_END;
4688  case AMDGPU::REG_SEQUENCE: return AMDGPU::REG_SEQUENCE;
4689  case AMDGPU::COPY: return AMDGPU::COPY;
4690  case AMDGPU::PHI: return AMDGPU::PHI;
4691  case AMDGPU::INSERT_SUBREG: return AMDGPU::INSERT_SUBREG;
4692  case AMDGPU::WQM: return AMDGPU::WQM;
4693  case AMDGPU::SOFT_WQM: return AMDGPU::SOFT_WQM;
4694  case AMDGPU::STRICT_WWM: return AMDGPU::STRICT_WWM;
4695  case AMDGPU::STRICT_WQM: return AMDGPU::STRICT_WQM;
4696  case AMDGPU::S_MOV_B32: {
4697  const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
4698  return MI.getOperand(1).isReg() ||
4699  RI.isAGPR(MRI, MI.getOperand(0).getReg()) ?
4700  AMDGPU::COPY : AMDGPU::V_MOV_B32_e32;
4701  }
4702  case AMDGPU::S_ADD_I32:
4703  return ST.hasAddNoCarry() ? AMDGPU::V_ADD_U32_e64 : AMDGPU::V_ADD_CO_U32_e32;
4704  case AMDGPU::S_ADDC_U32:
4705  return AMDGPU::V_ADDC_U32_e32;
4706  case AMDGPU::S_SUB_I32:
4707  return ST.hasAddNoCarry() ? AMDGPU::V_SUB_U32_e64 : AMDGPU::V_SUB_CO_U32_e32;
4708  // FIXME: These are not consistently handled, and selected when the carry is
4709  // used.
4710  case AMDGPU::S_ADD_U32:
4711  return AMDGPU::V_ADD_CO_U32_e32;
4712  case AMDGPU::S_SUB_U32:
4713  return AMDGPU::V_SUB_CO_U32_e32;
4714  case AMDGPU::S_SUBB_U32: return AMDGPU::V_SUBB_U32_e32;
4715  case AMDGPU::S_MUL_I32: return AMDGPU::V_MUL_LO_U32_e64;
4716  case AMDGPU::S_MUL_HI_U32: return AMDGPU::V_MUL_HI_U32_e64;
4717  case AMDGPU::S_MUL_HI_I32: return AMDGPU::V_MUL_HI_I32_e64;
4718  case AMDGPU::S_AND_B32: return AMDGPU::V_AND_B32_e64;
4719  case AMDGPU::S_OR_B32: return AMDGPU::V_OR_B32_e64;
4720  case AMDGPU::S_XOR_B32: return AMDGPU::V_XOR_B32_e64;
4721  case AMDGPU::S_XNOR_B32:
4722  return ST.hasDLInsts() ? AMDGPU::V_XNOR_B32_e64 : AMDGPU::INSTRUCTION_LIST_END;
4723  case AMDGPU::S_MIN_I32: return AMDGPU::V_MIN_I32_e64;
4724  case AMDGPU::S_MIN_U32: return AMDGPU::V_MIN_U32_e64;
4725  case AMDGPU::S_MAX_I32: return AMDGPU::V_MAX_I32_e64;
4726  case AMDGPU::S_MAX_U32: return AMDGPU::V_MAX_U32_e64;
4727  case AMDGPU::S_ASHR_I32: return AMDGPU::V_ASHR_I32_e32;
4728  case AMDGPU::S_ASHR_I64: return AMDGPU::V_ASHR_I64_e64;
4729  case AMDGPU::S_LSHL_B32: return AMDGPU::V_LSHL_B32_e32;
4730  case AMDGPU::S_LSHL_B64: return AMDGPU::V_LSHL_B64_e64;
4731  case AMDGPU::S_LSHR_B32: return AMDGPU::V_LSHR_B32_e32;
4732  case AMDGPU::S_LSHR_B64: return AMDGPU::V_LSHR_B64_e64;
4733  case AMDGPU::S_SEXT_I32_I8: return AMDGPU::V_BFE_I32_e64;
4734  case AMDGPU::S_SEXT_I32_I16: return AMDGPU::V_BFE_I32_e64;
4735  case AMDGPU::S_BFE_U32: return AMDGPU::V_BFE_U32_e64;
4736  case AMDGPU::S_BFE_I32: return AMDGPU::V_BFE_I32_e64;
4737  case AMDGPU::S_BFM_B32: return AMDGPU::V_BFM_B32_e64;
4738  case AMDGPU::S_BREV_B32: return AMDGPU::V_BFREV_B32_e32;
4739  case AMDGPU::S_NOT_B32: return AMDGPU::V_NOT_B32_e32;
4740  case AMDGPU::S_NOT_B64: return AMDGPU::V_NOT_B32_e32;
4741  case AMDGPU::S_CMP_EQ_I32: return AMDGPU::V_CMP_EQ_I32_e64;
4742  case AMDGPU::S_CMP_LG_I32: return AMDGPU::V_CMP_NE_I32_e64;
4743  case AMDGPU::S_CMP_GT_I32: return AMDGPU::V_CMP_GT_I32_e64;
4744  case AMDGPU::S_CMP_GE_I32: return AMDGPU::V_CMP_GE_I32_e64;
4745  case AMDGPU::S_CMP_LT_I32: return AMDGPU::V_CMP_LT_I32_e64;
4746  case AMDGPU::S_CMP_LE_I32: return AMDGPU::V_CMP_LE_I32_e64;
4747  case AMDGPU::S_CMP_EQ_U32: return AMDGPU::V_CMP_EQ_U32_e64;
4748  case AMDGPU::S_CMP_LG_U32: return AMDGPU::V_CMP_NE_U32_e64;
4749  case AMDGPU::S_CMP_GT_U32: return AMDGPU::V_CMP_GT_U32_e64;
4750  case AMDGPU::S_CMP_GE_U32: return AMDGPU::V_CMP_GE_U32_e64;
4751  case AMDGPU::S_CMP_LT_U32: return AMDGPU::V_CMP_LT_U32_e64;
4752  case AMDGPU::S_CMP_LE_U32: return AMDGPU::V_CMP_LE_U32_e64;
4753  case AMDGPU::S_CMP_EQ_U64: return AMDGPU::V_CMP_EQ_U64_e64;
4754  case AMDGPU::S_CMP_LG_U64: return AMDGPU::V_CMP_NE_U64_e64;
4755  case AMDGPU::S_BCNT1_I32_B32: return AMDGPU::V_BCNT_U32_B32_e64;
4756  case AMDGPU::S_FF1_I32_B32: return AMDGPU::V_FFBL_B32_e32;
4757  case AMDGPU::S_FLBIT_I32_B32: return AMDGPU::V_FFBH_U32_e32;
4758  case AMDGPU::S_FLBIT_I32: return AMDGPU::V_FFBH_I32_e64;
4759  case AMDGPU::S_CBRANCH_SCC0: return AMDGPU::S_CBRANCH_VCCZ;
4760  case AMDGPU::S_CBRANCH_SCC1: return AMDGPU::S_CBRANCH_VCCNZ;
4761  }
4763  "Unexpected scalar opcode without corresponding vector one!");
4764 }
4765 
4766 static const TargetRegisterClass *
4768  const MachineRegisterInfo &MRI,
4769  const MCInstrDesc &TID, unsigned RCID,
4770  bool IsAllocatable) {
4771  if ((IsAllocatable || !ST.hasGFX90AInsts() || !MRI.reservedRegsFrozen()) &&
4772  (((TID.mayLoad() || TID.mayStore()) &&
4773  !(TID.TSFlags & SIInstrFlags::VGPRSpill)) ||
4775  switch (RCID) {
4776  case AMDGPU::AV_32RegClassID:
4777  RCID = AMDGPU::VGPR_32RegClassID;
4778  break;
4779  case AMDGPU::AV_64RegClassID:
4780  RCID = AMDGPU::VReg_64RegClassID;
4781  break;
4782  case AMDGPU::AV_96RegClassID:
4783  RCID = AMDGPU::VReg_96RegClassID;
4784  break;
4785  case AMDGPU::AV_128RegClassID:
4786  RCID = AMDGPU::VReg_128RegClassID;
4787  break;
4788  case AMDGPU::AV_160RegClassID:
4789  RCID = AMDGPU::VReg_160RegClassID;
4790  break;
4791  case AMDGPU::AV_512RegClassID:
4792  RCID = AMDGPU::VReg_512RegClassID;
4793  break;
4794  default:
4795  break;
4796  }
4797  }
4798 
4799  return RI.getProperlyAlignedRC(RI.getRegClass(RCID));
4800 }
4801 
4803  unsigned OpNum, const TargetRegisterInfo *TRI,
4804  const MachineFunction &MF)
4805  const {
4806  if (OpNum >= TID.getNumOperands())
4807  return nullptr;
4808  auto RegClass = TID.OpInfo[OpNum].RegClass;
4809  bool IsAllocatable = false;
4811  // vdst and vdata should be both VGPR or AGPR, same for the DS instructions
4812  // with two data operands. Request register class constrained to VGPR only
4813  // of both operands present as Machine Copy Propagation can not check this
4814  // constraint and possibly other passes too.
4815  //
4816  // The check is limited to FLAT and DS because atomics in non-flat encoding
4817  // have their vdst and vdata tied to be the same register.
4818  const int VDstIdx = AMDGPU::getNamedOperandIdx(TID.Opcode,
4819  AMDGPU::OpName::vdst);
4820  const int DataIdx = AMDGPU::getNamedOperandIdx(TID.Opcode,
4821  (TID.TSFlags & SIInstrFlags::DS) ? AMDGPU::OpName::data0
4822  : AMDGPU::OpName::vdata);
4823  if (DataIdx != -1) {
4824  IsAllocatable = VDstIdx != -1 ||
4825