LLVM  16.0.0git
SIInstrInfo.cpp
Go to the documentation of this file.
1 //===- SIInstrInfo.cpp - SI Instruction Information ----------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 /// \file
10 /// SI Implementation of TargetInstrInfo.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "SIInstrInfo.h"
15 #include "AMDGPU.h"
16 #include "AMDGPUInstrInfo.h"
17 #include "GCNHazardRecognizer.h"
18 #include "GCNSubtarget.h"
19 #include "SIMachineFunctionInfo.h"
28 #include "llvm/IR/DiagnosticInfo.h"
29 #include "llvm/IR/IntrinsicsAMDGPU.h"
30 #include "llvm/MC/MCContext.h"
33 
34 using namespace llvm;
35 
36 #define DEBUG_TYPE "si-instr-info"
37 
38 #define GET_INSTRINFO_CTOR_DTOR
39 #include "AMDGPUGenInstrInfo.inc"
40 
41 namespace llvm {
42 namespace AMDGPU {
43 #define GET_D16ImageDimIntrinsics_IMPL
44 #define GET_ImageDimIntrinsicTable_IMPL
45 #define GET_RsrcIntrinsics_IMPL
46 #include "AMDGPUGenSearchableTables.inc"
47 }
48 }
49 
50 
51 // Must be at least 4 to be able to branch over minimum unconditional branch
52 // code. This is only for making it possible to write reasonably small tests for
53 // long branches.
54 static cl::opt<unsigned>
55 BranchOffsetBits("amdgpu-s-branch-bits", cl::ReallyHidden, cl::init(16),
56  cl::desc("Restrict range of branch instructions (DEBUG)"));
57 
59  "amdgpu-fix-16-bit-physreg-copies",
60  cl::desc("Fix copies between 32 and 16 bit registers by extending to 32 bit"),
61  cl::init(true),
63 
65  : AMDGPUGenInstrInfo(AMDGPU::ADJCALLSTACKUP, AMDGPU::ADJCALLSTACKDOWN),
66  RI(ST), ST(ST) {
67  SchedModel.init(&ST);
68 }
69 
70 //===----------------------------------------------------------------------===//
71 // TargetInstrInfo callbacks
72 //===----------------------------------------------------------------------===//
73 
74 static unsigned getNumOperandsNoGlue(SDNode *Node) {
75  unsigned N = Node->getNumOperands();
76  while (N && Node->getOperand(N - 1).getValueType() == MVT::Glue)
77  --N;
78  return N;
79 }
80 
81 /// Returns true if both nodes have the same value for the given
82 /// operand \p Op, or if both nodes do not have this operand.
83 static bool nodesHaveSameOperandValue(SDNode *N0, SDNode* N1, unsigned OpName) {
84  unsigned Opc0 = N0->getMachineOpcode();
85  unsigned Opc1 = N1->getMachineOpcode();
86 
87  int Op0Idx = AMDGPU::getNamedOperandIdx(Opc0, OpName);
88  int Op1Idx = AMDGPU::getNamedOperandIdx(Opc1, OpName);
89 
90  if (Op0Idx == -1 && Op1Idx == -1)
91  return true;
92 
93 
94  if ((Op0Idx == -1 && Op1Idx != -1) ||
95  (Op1Idx == -1 && Op0Idx != -1))
96  return false;
97 
98  // getNamedOperandIdx returns the index for the MachineInstr's operands,
99  // which includes the result as the first operand. We are indexing into the
100  // MachineSDNode's operands, so we need to skip the result operand to get
101  // the real index.
102  --Op0Idx;
103  --Op1Idx;
104 
105  return N0->getOperand(Op0Idx) == N1->getOperand(Op1Idx);
106 }
107 
109  const MachineInstr &MI) const {
110  if (isVOP1(MI) || isVOP2(MI) || isVOP3(MI) || isSDWA(MI) || isSALU(MI)) {
111  // Normally VALU use of exec would block the rematerialization, but that
112  // is OK in this case to have an implicit exec read as all VALU do.
113  // We really want all of the generic logic for this except for this.
114 
115  // Another potential implicit use is mode register. The core logic of
116  // the RA will not attempt rematerialization if mode is set anywhere
117  // in the function, otherwise it is safe since mode is not changed.
118 
119  // There is difference to generic method which does not allow
120  // rematerialization if there are virtual register uses. We allow this,
121  // therefore this method includes SOP instructions as well.
122  return !MI.hasImplicitDef() &&
123  MI.getNumImplicitOperands() == MI.getDesc().getNumImplicitUses() &&
124  !MI.mayRaiseFPException();
125  }
126 
127  return false;
128 }
129 
130 // Returns true if the scalar result of a VALU instruction depends on exec.
131 static bool resultDependsOnExec(const MachineInstr &MI) {
132  // Ignore comparisons which are only used masked with exec.
133  // This allows some hoisting/sinking of VALU comparisons.
134  if (MI.isCompare()) {
135  const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
136  Register DstReg = MI.getOperand(0).getReg();
137  if (!DstReg.isVirtual())
138  return true;
139  for (MachineInstr &Use : MRI.use_nodbg_instructions(DstReg)) {
140  switch (Use.getOpcode()) {
141  case AMDGPU::S_AND_SAVEEXEC_B32:
142  case AMDGPU::S_AND_SAVEEXEC_B64:
143  break;
144  case AMDGPU::S_AND_B32:
145  case AMDGPU::S_AND_B64:
146  if (!Use.readsRegister(AMDGPU::EXEC))
147  return true;
148  break;
149  default:
150  return true;
151  }
152  }
153  return false;
154  }
155 
156  switch (MI.getOpcode()) {
157  default:
158  break;
159  case AMDGPU::V_READFIRSTLANE_B32:
160  return true;
161  }
162 
163  return false;
164 }
165 
167  // Any implicit use of exec by VALU is not a real register read.
168  return MO.getReg() == AMDGPU::EXEC && MO.isImplicit() &&
169  isVALU(*MO.getParent()) && !resultDependsOnExec(*MO.getParent());
170 }
171 
173  int64_t &Offset0,
174  int64_t &Offset1) const {
175  if (!Load0->isMachineOpcode() || !Load1->isMachineOpcode())
176  return false;
177 
178  unsigned Opc0 = Load0->getMachineOpcode();
179  unsigned Opc1 = Load1->getMachineOpcode();
180 
181  // Make sure both are actually loads.
182  if (!get(Opc0).mayLoad() || !get(Opc1).mayLoad())
183  return false;
184 
185  if (isDS(Opc0) && isDS(Opc1)) {
186 
187  // FIXME: Handle this case:
188  if (getNumOperandsNoGlue(Load0) != getNumOperandsNoGlue(Load1))
189  return false;
190 
191  // Check base reg.
192  if (Load0->getOperand(0) != Load1->getOperand(0))
193  return false;
194 
195  // Skip read2 / write2 variants for simplicity.
196  // TODO: We should report true if the used offsets are adjacent (excluded
197  // st64 versions).
198  int Offset0Idx = AMDGPU::getNamedOperandIdx(Opc0, AMDGPU::OpName::offset);
199  int Offset1Idx = AMDGPU::getNamedOperandIdx(Opc1, AMDGPU::OpName::offset);
200  if (Offset0Idx == -1 || Offset1Idx == -1)
201  return false;
202 
203  // XXX - be careful of dataless loads
204  // getNamedOperandIdx returns the index for MachineInstrs. Since they
205  // include the output in the operand list, but SDNodes don't, we need to
206  // subtract the index by one.
207  Offset0Idx -= get(Opc0).NumDefs;
208  Offset1Idx -= get(Opc1).NumDefs;
209  Offset0 = cast<ConstantSDNode>(Load0->getOperand(Offset0Idx))->getZExtValue();
210  Offset1 = cast<ConstantSDNode>(Load1->getOperand(Offset1Idx))->getZExtValue();
211  return true;
212  }
213 
214  if (isSMRD(Opc0) && isSMRD(Opc1)) {
215  // Skip time and cache invalidation instructions.
216  if (!AMDGPU::hasNamedOperand(Opc0, AMDGPU::OpName::sbase) ||
217  !AMDGPU::hasNamedOperand(Opc1, AMDGPU::OpName::sbase))
218  return false;
219 
220  unsigned NumOps = getNumOperandsNoGlue(Load0);
221  if (NumOps != getNumOperandsNoGlue(Load1))
222  return false;
223 
224  // Check base reg.
225  if (Load0->getOperand(0) != Load1->getOperand(0))
226  return false;
227 
228  // Match register offsets, if both register and immediate offsets present.
229  assert(NumOps == 4 || NumOps == 5);
230  if (NumOps == 5 && Load0->getOperand(1) != Load1->getOperand(1))
231  return false;
232 
233  const ConstantSDNode *Load0Offset =
234  dyn_cast<ConstantSDNode>(Load0->getOperand(NumOps - 3));
235  const ConstantSDNode *Load1Offset =
236  dyn_cast<ConstantSDNode>(Load1->getOperand(NumOps - 3));
237 
238  if (!Load0Offset || !Load1Offset)
239  return false;
240 
241  Offset0 = Load0Offset->getZExtValue();
242  Offset1 = Load1Offset->getZExtValue();
243  return true;
244  }
245 
246  // MUBUF and MTBUF can access the same addresses.
247  if ((isMUBUF(Opc0) || isMTBUF(Opc0)) && (isMUBUF(Opc1) || isMTBUF(Opc1))) {
248 
249  // MUBUF and MTBUF have vaddr at different indices.
250  if (!nodesHaveSameOperandValue(Load0, Load1, AMDGPU::OpName::soffset) ||
251  !nodesHaveSameOperandValue(Load0, Load1, AMDGPU::OpName::vaddr) ||
252  !nodesHaveSameOperandValue(Load0, Load1, AMDGPU::OpName::srsrc))
253  return false;
254 
255  int OffIdx0 = AMDGPU::getNamedOperandIdx(Opc0, AMDGPU::OpName::offset);
256  int OffIdx1 = AMDGPU::getNamedOperandIdx(Opc1, AMDGPU::OpName::offset);
257 
258  if (OffIdx0 == -1 || OffIdx1 == -1)
259  return false;
260 
261  // getNamedOperandIdx returns the index for MachineInstrs. Since they
262  // include the output in the operand list, but SDNodes don't, we need to
263  // subtract the index by one.
264  OffIdx0 -= get(Opc0).NumDefs;
265  OffIdx1 -= get(Opc1).NumDefs;
266 
267  SDValue Off0 = Load0->getOperand(OffIdx0);
268  SDValue Off1 = Load1->getOperand(OffIdx1);
269 
270  // The offset might be a FrameIndexSDNode.
271  if (!isa<ConstantSDNode>(Off0) || !isa<ConstantSDNode>(Off1))
272  return false;
273 
274  Offset0 = cast<ConstantSDNode>(Off0)->getZExtValue();
275  Offset1 = cast<ConstantSDNode>(Off1)->getZExtValue();
276  return true;
277  }
278 
279  return false;
280 }
281 
282 static bool isStride64(unsigned Opc) {
283  switch (Opc) {
284  case AMDGPU::DS_READ2ST64_B32:
285  case AMDGPU::DS_READ2ST64_B64:
286  case AMDGPU::DS_WRITE2ST64_B32:
287  case AMDGPU::DS_WRITE2ST64_B64:
288  return true;
289  default:
290  return false;
291  }
292 }
293 
296  int64_t &Offset, bool &OffsetIsScalable, unsigned &Width,
297  const TargetRegisterInfo *TRI) const {
298  if (!LdSt.mayLoadOrStore())
299  return false;
300 
301  unsigned Opc = LdSt.getOpcode();
302  OffsetIsScalable = false;
303  const MachineOperand *BaseOp, *OffsetOp;
304  int DataOpIdx;
305 
306  if (isDS(LdSt)) {
307  BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::addr);
308  OffsetOp = getNamedOperand(LdSt, AMDGPU::OpName::offset);
309  if (OffsetOp) {
310  // Normal, single offset LDS instruction.
311  if (!BaseOp) {
312  // DS_CONSUME/DS_APPEND use M0 for the base address.
313  // TODO: find the implicit use operand for M0 and use that as BaseOp?
314  return false;
315  }
316  BaseOps.push_back(BaseOp);
317  Offset = OffsetOp->getImm();
318  // Get appropriate operand, and compute width accordingly.
319  DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst);
320  if (DataOpIdx == -1)
321  DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data0);
322  Width = getOpSize(LdSt, DataOpIdx);
323  } else {
324  // The 2 offset instructions use offset0 and offset1 instead. We can treat
325  // these as a load with a single offset if the 2 offsets are consecutive.
326  // We will use this for some partially aligned loads.
327  const MachineOperand *Offset0Op =
328  getNamedOperand(LdSt, AMDGPU::OpName::offset0);
329  const MachineOperand *Offset1Op =
330  getNamedOperand(LdSt, AMDGPU::OpName::offset1);
331 
332  unsigned Offset0 = Offset0Op->getImm();
333  unsigned Offset1 = Offset1Op->getImm();
334  if (Offset0 + 1 != Offset1)
335  return false;
336 
337  // Each of these offsets is in element sized units, so we need to convert
338  // to bytes of the individual reads.
339 
340  unsigned EltSize;
341  if (LdSt.mayLoad())
342  EltSize = TRI->getRegSizeInBits(*getOpRegClass(LdSt, 0)) / 16;
343  else {
344  assert(LdSt.mayStore());
345  int Data0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data0);
346  EltSize = TRI->getRegSizeInBits(*getOpRegClass(LdSt, Data0Idx)) / 8;
347  }
348 
349  if (isStride64(Opc))
350  EltSize *= 64;
351 
352  BaseOps.push_back(BaseOp);
353  Offset = EltSize * Offset0;
354  // Get appropriate operand(s), and compute width accordingly.
355  DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst);
356  if (DataOpIdx == -1) {
357  DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data0);
358  Width = getOpSize(LdSt, DataOpIdx);
359  DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data1);
360  Width += getOpSize(LdSt, DataOpIdx);
361  } else {
362  Width = getOpSize(LdSt, DataOpIdx);
363  }
364  }
365  return true;
366  }
367 
368  if (isMUBUF(LdSt) || isMTBUF(LdSt)) {
369  const MachineOperand *RSrc = getNamedOperand(LdSt, AMDGPU::OpName::srsrc);
370  if (!RSrc) // e.g. BUFFER_WBINVL1_VOL
371  return false;
372  BaseOps.push_back(RSrc);
373  BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::vaddr);
374  if (BaseOp && !BaseOp->isFI())
375  BaseOps.push_back(BaseOp);
376  const MachineOperand *OffsetImm =
377  getNamedOperand(LdSt, AMDGPU::OpName::offset);
378  Offset = OffsetImm->getImm();
379  const MachineOperand *SOffset =
380  getNamedOperand(LdSt, AMDGPU::OpName::soffset);
381  if (SOffset) {
382  if (SOffset->isReg())
383  BaseOps.push_back(SOffset);
384  else
385  Offset += SOffset->getImm();
386  }
387  // Get appropriate operand, and compute width accordingly.
388  DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst);
389  if (DataOpIdx == -1)
390  DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdata);
391  if (DataOpIdx == -1) // LDS DMA
392  return false;
393  Width = getOpSize(LdSt, DataOpIdx);
394  return true;
395  }
396 
397  if (isMIMG(LdSt)) {
398  int SRsrcIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::srsrc);
399  BaseOps.push_back(&LdSt.getOperand(SRsrcIdx));
400  int VAddr0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vaddr0);
401  if (VAddr0Idx >= 0) {
402  // GFX10 possible NSA encoding.
403  for (int I = VAddr0Idx; I < SRsrcIdx; ++I)
404  BaseOps.push_back(&LdSt.getOperand(I));
405  } else {
406  BaseOps.push_back(getNamedOperand(LdSt, AMDGPU::OpName::vaddr));
407  }
408  Offset = 0;
409  // Get appropriate operand, and compute width accordingly.
410  DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdata);
411  Width = getOpSize(LdSt, DataOpIdx);
412  return true;
413  }
414 
415  if (isSMRD(LdSt)) {
416  BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::sbase);
417  if (!BaseOp) // e.g. S_MEMTIME
418  return false;
419  BaseOps.push_back(BaseOp);
420  OffsetOp = getNamedOperand(LdSt, AMDGPU::OpName::offset);
421  Offset = OffsetOp ? OffsetOp->getImm() : 0;
422  // Get appropriate operand, and compute width accordingly.
423  DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::sdst);
424  Width = getOpSize(LdSt, DataOpIdx);
425  return true;
426  }
427 
428  if (isFLAT(LdSt)) {
429  // Instructions have either vaddr or saddr or both or none.
430  BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::vaddr);
431  if (BaseOp)
432  BaseOps.push_back(BaseOp);
433  BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::saddr);
434  if (BaseOp)
435  BaseOps.push_back(BaseOp);
436  Offset = getNamedOperand(LdSt, AMDGPU::OpName::offset)->getImm();
437  // Get appropriate operand, and compute width accordingly.
438  DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst);
439  if (DataOpIdx == -1)
440  DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdata);
441  if (DataOpIdx == -1) // LDS DMA
442  return false;
443  Width = getOpSize(LdSt, DataOpIdx);
444  return true;
445  }
446 
447  return false;
448 }
449 
450 static bool memOpsHaveSameBasePtr(const MachineInstr &MI1,
452  const MachineInstr &MI2,
454  // Only examine the first "base" operand of each instruction, on the
455  // assumption that it represents the real base address of the memory access.
456  // Other operands are typically offsets or indices from this base address.
457  if (BaseOps1.front()->isIdenticalTo(*BaseOps2.front()))
458  return true;
459 
460  if (!MI1.hasOneMemOperand() || !MI2.hasOneMemOperand())
461  return false;
462 
463  auto MO1 = *MI1.memoperands_begin();
464  auto MO2 = *MI2.memoperands_begin();
465  if (MO1->getAddrSpace() != MO2->getAddrSpace())
466  return false;
467 
468  auto Base1 = MO1->getValue();
469  auto Base2 = MO2->getValue();
470  if (!Base1 || !Base2)
471  return false;
472  Base1 = getUnderlyingObject(Base1);
473  Base2 = getUnderlyingObject(Base2);
474 
475  if (isa<UndefValue>(Base1) || isa<UndefValue>(Base2))
476  return false;
477 
478  return Base1 == Base2;
479 }
480 
483  unsigned NumLoads,
484  unsigned NumBytes) const {
485  // If the mem ops (to be clustered) do not have the same base ptr, then they
486  // should not be clustered
487  if (!BaseOps1.empty() && !BaseOps2.empty()) {
488  const MachineInstr &FirstLdSt = *BaseOps1.front()->getParent();
489  const MachineInstr &SecondLdSt = *BaseOps2.front()->getParent();
490  if (!memOpsHaveSameBasePtr(FirstLdSt, BaseOps1, SecondLdSt, BaseOps2))
491  return false;
492  } else if (!BaseOps1.empty() || !BaseOps2.empty()) {
493  // If only one base op is empty, they do not have the same base ptr
494  return false;
495  }
496 
497  // In order to avoid register pressure, on an average, the number of DWORDS
498  // loaded together by all clustered mem ops should not exceed 8. This is an
499  // empirical value based on certain observations and performance related
500  // experiments.
501  // The good thing about this heuristic is - it avoids clustering of too many
502  // sub-word loads, and also avoids clustering of wide loads. Below is the
503  // brief summary of how the heuristic behaves for various `LoadSize`.
504  // (1) 1 <= LoadSize <= 4: cluster at max 8 mem ops
505  // (2) 5 <= LoadSize <= 8: cluster at max 4 mem ops
506  // (3) 9 <= LoadSize <= 12: cluster at max 2 mem ops
507  // (4) 13 <= LoadSize <= 16: cluster at max 2 mem ops
508  // (5) LoadSize >= 17: do not cluster
509  const unsigned LoadSize = NumBytes / NumLoads;
510  const unsigned NumDWORDs = ((LoadSize + 3) / 4) * NumLoads;
511  return NumDWORDs <= 8;
512 }
513 
514 // FIXME: This behaves strangely. If, for example, you have 32 load + stores,
515 // the first 16 loads will be interleaved with the stores, and the next 16 will
516 // be clustered as expected. It should really split into 2 16 store batches.
517 //
518 // Loads are clustered until this returns false, rather than trying to schedule
519 // groups of stores. This also means we have to deal with saying different
520 // address space loads should be clustered, and ones which might cause bank
521 // conflicts.
522 //
523 // This might be deprecated so it might not be worth that much effort to fix.
525  int64_t Offset0, int64_t Offset1,
526  unsigned NumLoads) const {
527  assert(Offset1 > Offset0 &&
528  "Second offset should be larger than first offset!");
529  // If we have less than 16 loads in a row, and the offsets are within 64
530  // bytes, then schedule together.
531 
532  // A cacheline is 64 bytes (for global memory).
533  return (NumLoads <= 16 && (Offset1 - Offset0) < 64);
534 }
535 
538  const DebugLoc &DL, MCRegister DestReg,
539  MCRegister SrcReg, bool KillSrc,
540  const char *Msg = "illegal SGPR to VGPR copy") {
541  MachineFunction *MF = MBB.getParent();
542  DiagnosticInfoUnsupported IllegalCopy(MF->getFunction(), Msg, DL, DS_Error);
543  LLVMContext &C = MF->getFunction().getContext();
544  C.diagnose(IllegalCopy);
545 
546  BuildMI(MBB, MI, DL, TII->get(AMDGPU::SI_ILLEGAL_COPY), DestReg)
547  .addReg(SrcReg, getKillRegState(KillSrc));
548 }
549 
550 /// Handle copying from SGPR to AGPR, or from AGPR to AGPR on GFX908. It is not
551 /// possible to have a direct copy in these cases on GFX908, so an intermediate
552 /// VGPR copy is required.
553 static void indirectCopyToAGPR(const SIInstrInfo &TII,
556  const DebugLoc &DL, MCRegister DestReg,
557  MCRegister SrcReg, bool KillSrc,
558  RegScavenger &RS,
559  Register ImpDefSuperReg = Register(),
560  Register ImpUseSuperReg = Register()) {
561  assert((TII.getSubtarget().hasMAIInsts() &&
562  !TII.getSubtarget().hasGFX90AInsts()) &&
563  "Expected GFX908 subtarget.");
564 
565  assert((AMDGPU::SReg_32RegClass.contains(SrcReg) ||
566  AMDGPU::AGPR_32RegClass.contains(SrcReg)) &&
567  "Source register of the copy should be either an SGPR or an AGPR.");
568 
569  assert(AMDGPU::AGPR_32RegClass.contains(DestReg) &&
570  "Destination register of the copy should be an AGPR.");
571 
572  const SIRegisterInfo &RI = TII.getRegisterInfo();
573 
574  // First try to find defining accvgpr_write to avoid temporary registers.
575  for (auto Def = MI, E = MBB.begin(); Def != E; ) {
576  --Def;
577  if (!Def->definesRegister(SrcReg, &RI))
578  continue;
579  if (Def->getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64)
580  break;
581 
582  MachineOperand &DefOp = Def->getOperand(1);
583  assert(DefOp.isReg() || DefOp.isImm());
584 
585  if (DefOp.isReg()) {
586  // Check that register source operand if not clobbered before MI.
587  // Immediate operands are always safe to propagate.
588  bool SafeToPropagate = true;
589  for (auto I = Def; I != MI && SafeToPropagate; ++I)
590  if (I->modifiesRegister(DefOp.getReg(), &RI))
591  SafeToPropagate = false;
592 
593  if (!SafeToPropagate)
594  break;
595 
596  DefOp.setIsKill(false);
597  }
598 
600  BuildMI(MBB, MI, DL, TII.get(AMDGPU::V_ACCVGPR_WRITE_B32_e64), DestReg)
601  .add(DefOp);
602  if (ImpDefSuperReg)
603  Builder.addReg(ImpDefSuperReg, RegState::Define | RegState::Implicit);
604 
605  if (ImpUseSuperReg) {
606  Builder.addReg(ImpUseSuperReg,
608  }
609 
610  return;
611  }
612 
613  RS.enterBasicBlock(MBB);
614  RS.forward(MI);
615 
616  // Ideally we want to have three registers for a long reg_sequence copy
617  // to hide 2 waitstates between v_mov_b32 and accvgpr_write.
618  unsigned MaxVGPRs = RI.getRegPressureLimit(&AMDGPU::VGPR_32RegClass,
619  *MBB.getParent());
620 
621  // Registers in the sequence are allocated contiguously so we can just
622  // use register number to pick one of three round-robin temps.
623  unsigned RegNo = (DestReg - AMDGPU::AGPR0) % 3;
624  Register Tmp =
625  MBB.getParent()->getInfo<SIMachineFunctionInfo>()->getVGPRForAGPRCopy();
627  "VGPR used for an intermediate copy should have been reserved.");
628 
629  // Only loop through if there are any free registers left, otherwise
630  // scavenger may report a fatal error without emergency spill slot
631  // or spill with the slot.
632  while (RegNo-- && RS.FindUnusedReg(&AMDGPU::VGPR_32RegClass)) {
633  Register Tmp2 = RS.scavengeRegister(&AMDGPU::VGPR_32RegClass, 0);
634  if (!Tmp2 || RI.getHWRegIndex(Tmp2) >= MaxVGPRs)
635  break;
636  Tmp = Tmp2;
637  RS.setRegUsed(Tmp);
638  }
639 
640  // Insert copy to temporary VGPR.
641  unsigned TmpCopyOp = AMDGPU::V_MOV_B32_e32;
642  if (AMDGPU::AGPR_32RegClass.contains(SrcReg)) {
643  TmpCopyOp = AMDGPU::V_ACCVGPR_READ_B32_e64;
644  } else {
645  assert(AMDGPU::SReg_32RegClass.contains(SrcReg));
646  }
647 
648  MachineInstrBuilder UseBuilder = BuildMI(MBB, MI, DL, TII.get(TmpCopyOp), Tmp)
649  .addReg(SrcReg, getKillRegState(KillSrc));
650  if (ImpUseSuperReg) {
651  UseBuilder.addReg(ImpUseSuperReg,
653  }
654 
655  MachineInstrBuilder DefBuilder
656  = BuildMI(MBB, MI, DL, TII.get(AMDGPU::V_ACCVGPR_WRITE_B32_e64), DestReg)
657  .addReg(Tmp, RegState::Kill);
658 
659  if (ImpDefSuperReg)
660  DefBuilder.addReg(ImpDefSuperReg, RegState::Define | RegState::Implicit);
661 }
662 
665  MCRegister DestReg, MCRegister SrcReg, bool KillSrc,
666  const TargetRegisterClass *RC, bool Forward) {
667  const SIRegisterInfo &RI = TII.getRegisterInfo();
668  ArrayRef<int16_t> BaseIndices = RI.getRegSplitParts(RC, 4);
670  MachineInstr *FirstMI = nullptr, *LastMI = nullptr;
671 
672  for (unsigned Idx = 0; Idx < BaseIndices.size(); ++Idx) {
673  int16_t SubIdx = BaseIndices[Idx];
674  Register Reg = RI.getSubReg(DestReg, SubIdx);
675  unsigned Opcode = AMDGPU::S_MOV_B32;
676 
677  // Is SGPR aligned? If so try to combine with next.
678  Register Src = RI.getSubReg(SrcReg, SubIdx);
679  bool AlignedDest = ((Reg - AMDGPU::SGPR0) % 2) == 0;
680  bool AlignedSrc = ((Src - AMDGPU::SGPR0) % 2) == 0;
681  if (AlignedDest && AlignedSrc && (Idx + 1 < BaseIndices.size())) {
682  // Can use SGPR64 copy
683  unsigned Channel = RI.getChannelFromSubReg(SubIdx);
684  SubIdx = RI.getSubRegFromChannel(Channel, 2);
685  Opcode = AMDGPU::S_MOV_B64;
686  Idx++;
687  }
688 
689  LastMI = BuildMI(MBB, I, DL, TII.get(Opcode), RI.getSubReg(DestReg, SubIdx))
690  .addReg(RI.getSubReg(SrcReg, SubIdx))
691  .addReg(SrcReg, RegState::Implicit);
692 
693  if (!FirstMI)
694  FirstMI = LastMI;
695 
696  if (!Forward)
697  I--;
698  }
699 
700  assert(FirstMI && LastMI);
701  if (!Forward)
702  std::swap(FirstMI, LastMI);
703 
704  FirstMI->addOperand(
705  MachineOperand::CreateReg(DestReg, true /*IsDef*/, true /*IsImp*/));
706 
707  if (KillSrc)
708  LastMI->addRegisterKilled(SrcReg, &RI);
709 }
710 
713  const DebugLoc &DL, MCRegister DestReg,
714  MCRegister SrcReg, bool KillSrc) const {
715  const TargetRegisterClass *RC = RI.getPhysRegClass(DestReg);
716 
717  // FIXME: This is hack to resolve copies between 16 bit and 32 bit
718  // registers until all patterns are fixed.
719  if (Fix16BitCopies &&
720  ((RI.getRegSizeInBits(*RC) == 16) ^
721  (RI.getRegSizeInBits(*RI.getPhysRegClass(SrcReg)) == 16))) {
722  MCRegister &RegToFix = (RI.getRegSizeInBits(*RC) == 16) ? DestReg : SrcReg;
723  MCRegister Super = RI.get32BitRegister(RegToFix);
724  assert(RI.getSubReg(Super, AMDGPU::lo16) == RegToFix);
725  RegToFix = Super;
726 
727  if (DestReg == SrcReg) {
728  // Insert empty bundle since ExpandPostRA expects an instruction here.
729  BuildMI(MBB, MI, DL, get(AMDGPU::BUNDLE));
730  return;
731  }
732 
733  RC = RI.getPhysRegClass(DestReg);
734  }
735 
736  if (RC == &AMDGPU::VGPR_32RegClass) {
737  assert(AMDGPU::VGPR_32RegClass.contains(SrcReg) ||
738  AMDGPU::SReg_32RegClass.contains(SrcReg) ||
739  AMDGPU::AGPR_32RegClass.contains(SrcReg));
740  unsigned Opc = AMDGPU::AGPR_32RegClass.contains(SrcReg) ?
741  AMDGPU::V_ACCVGPR_READ_B32_e64 : AMDGPU::V_MOV_B32_e32;
742  BuildMI(MBB, MI, DL, get(Opc), DestReg)
743  .addReg(SrcReg, getKillRegState(KillSrc));
744  return;
745  }
746 
747  if (RC == &AMDGPU::SReg_32_XM0RegClass ||
748  RC == &AMDGPU::SReg_32RegClass) {
749  if (SrcReg == AMDGPU::SCC) {
750  BuildMI(MBB, MI, DL, get(AMDGPU::S_CSELECT_B32), DestReg)
751  .addImm(1)
752  .addImm(0);
753  return;
754  }
755 
756  if (DestReg == AMDGPU::VCC_LO) {
757  if (AMDGPU::SReg_32RegClass.contains(SrcReg)) {
758  BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), AMDGPU::VCC_LO)
759  .addReg(SrcReg, getKillRegState(KillSrc));
760  } else {
761  // FIXME: Hack until VReg_1 removed.
762  assert(AMDGPU::VGPR_32RegClass.contains(SrcReg));
763  BuildMI(MBB, MI, DL, get(AMDGPU::V_CMP_NE_U32_e32))
764  .addImm(0)
765  .addReg(SrcReg, getKillRegState(KillSrc));
766  }
767 
768  return;
769  }
770 
771  if (!AMDGPU::SReg_32RegClass.contains(SrcReg)) {
772  reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc);
773  return;
774  }
775 
776  BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), DestReg)
777  .addReg(SrcReg, getKillRegState(KillSrc));
778  return;
779  }
780 
781  if (RC == &AMDGPU::SReg_64RegClass) {
782  if (SrcReg == AMDGPU::SCC) {
783  BuildMI(MBB, MI, DL, get(AMDGPU::S_CSELECT_B64), DestReg)
784  .addImm(1)
785  .addImm(0);
786  return;
787  }
788 
789  if (DestReg == AMDGPU::VCC) {
790  if (AMDGPU::SReg_64RegClass.contains(SrcReg)) {
791  BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B64), AMDGPU::VCC)
792  .addReg(SrcReg, getKillRegState(KillSrc));
793  } else {
794  // FIXME: Hack until VReg_1 removed.
795  assert(AMDGPU::VGPR_32RegClass.contains(SrcReg));
796  BuildMI(MBB, MI, DL, get(AMDGPU::V_CMP_NE_U32_e32))
797  .addImm(0)
798  .addReg(SrcReg, getKillRegState(KillSrc));
799  }
800 
801  return;
802  }
803 
804  if (!AMDGPU::SReg_64RegClass.contains(SrcReg)) {
805  reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc);
806  return;
807  }
808 
809  BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B64), DestReg)
810  .addReg(SrcReg, getKillRegState(KillSrc));
811  return;
812  }
813 
814  if (DestReg == AMDGPU::SCC) {
815  // Copying 64-bit or 32-bit sources to SCC barely makes sense,
816  // but SelectionDAG emits such copies for i1 sources.
817  if (AMDGPU::SReg_64RegClass.contains(SrcReg)) {
818  // This copy can only be produced by patterns
819  // with explicit SCC, which are known to be enabled
820  // only for subtargets with S_CMP_LG_U64 present.
822  BuildMI(MBB, MI, DL, get(AMDGPU::S_CMP_LG_U64))
823  .addReg(SrcReg, getKillRegState(KillSrc))
824  .addImm(0);
825  } else {
826  assert(AMDGPU::SReg_32RegClass.contains(SrcReg));
827  BuildMI(MBB, MI, DL, get(AMDGPU::S_CMP_LG_U32))
828  .addReg(SrcReg, getKillRegState(KillSrc))
829  .addImm(0);
830  }
831 
832  return;
833  }
834 
835  if (RC == &AMDGPU::AGPR_32RegClass) {
836  if (AMDGPU::VGPR_32RegClass.contains(SrcReg) ||
837  (ST.hasGFX90AInsts() && AMDGPU::SReg_32RegClass.contains(SrcReg))) {
838  BuildMI(MBB, MI, DL, get(AMDGPU::V_ACCVGPR_WRITE_B32_e64), DestReg)
839  .addReg(SrcReg, getKillRegState(KillSrc));
840  return;
841  }
842 
843  if (AMDGPU::AGPR_32RegClass.contains(SrcReg) && ST.hasGFX90AInsts()) {
844  BuildMI(MBB, MI, DL, get(AMDGPU::V_ACCVGPR_MOV_B32), DestReg)
845  .addReg(SrcReg, getKillRegState(KillSrc));
846  return;
847  }
848 
849  // FIXME: Pass should maintain scavenger to avoid scan through the block on
850  // every AGPR spill.
851  RegScavenger RS;
852  indirectCopyToAGPR(*this, MBB, MI, DL, DestReg, SrcReg, KillSrc, RS);
853  return;
854  }
855 
856  const unsigned Size = RI.getRegSizeInBits(*RC);
857  if (Size == 16) {
858  assert(AMDGPU::VGPR_LO16RegClass.contains(SrcReg) ||
859  AMDGPU::VGPR_HI16RegClass.contains(SrcReg) ||
860  AMDGPU::SReg_LO16RegClass.contains(SrcReg) ||
861  AMDGPU::AGPR_LO16RegClass.contains(SrcReg));
862 
863  bool IsSGPRDst = AMDGPU::SReg_LO16RegClass.contains(DestReg);
864  bool IsSGPRSrc = AMDGPU::SReg_LO16RegClass.contains(SrcReg);
865  bool IsAGPRDst = AMDGPU::AGPR_LO16RegClass.contains(DestReg);
866  bool IsAGPRSrc = AMDGPU::AGPR_LO16RegClass.contains(SrcReg);
867  bool DstLow = AMDGPU::VGPR_LO16RegClass.contains(DestReg) ||
868  AMDGPU::SReg_LO16RegClass.contains(DestReg) ||
869  AMDGPU::AGPR_LO16RegClass.contains(DestReg);
870  bool SrcLow = AMDGPU::VGPR_LO16RegClass.contains(SrcReg) ||
871  AMDGPU::SReg_LO16RegClass.contains(SrcReg) ||
872  AMDGPU::AGPR_LO16RegClass.contains(SrcReg);
873  MCRegister NewDestReg = RI.get32BitRegister(DestReg);
874  MCRegister NewSrcReg = RI.get32BitRegister(SrcReg);
875 
876  if (IsSGPRDst) {
877  if (!IsSGPRSrc) {
878  reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc);
879  return;
880  }
881 
882  BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), NewDestReg)
883  .addReg(NewSrcReg, getKillRegState(KillSrc));
884  return;
885  }
886 
887  if (IsAGPRDst || IsAGPRSrc) {
888  if (!DstLow || !SrcLow) {
889  reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc,
890  "Cannot use hi16 subreg with an AGPR!");
891  }
892 
893  copyPhysReg(MBB, MI, DL, NewDestReg, NewSrcReg, KillSrc);
894  return;
895  }
896 
897  if (IsSGPRSrc && !ST.hasSDWAScalar()) {
898  if (!DstLow || !SrcLow) {
899  reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc,
900  "Cannot use hi16 subreg on VI!");
901  }
902 
903  BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), NewDestReg)
904  .addReg(NewSrcReg, getKillRegState(KillSrc));
905  return;
906  }
907 
908  auto MIB = BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_sdwa), NewDestReg)
909  .addImm(0) // src0_modifiers
910  .addReg(NewSrcReg)
911  .addImm(0) // clamp
917  .addReg(NewDestReg, RegState::Implicit | RegState::Undef);
918  // First implicit operand is $exec.
919  MIB->tieOperands(0, MIB->getNumOperands() - 1);
920  return;
921  }
922 
923  const TargetRegisterClass *SrcRC = RI.getPhysRegClass(SrcReg);
924  if (RC == RI.getVGPR64Class() && (SrcRC == RC || RI.isSGPRClass(SrcRC))) {
925  if (ST.hasMovB64()) {
926  BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B64_e32), DestReg)
927  .addReg(SrcReg, getKillRegState(KillSrc));
928  return;
929  }
930  if (ST.hasPackedFP32Ops()) {
931  BuildMI(MBB, MI, DL, get(AMDGPU::V_PK_MOV_B32), DestReg)
933  .addReg(SrcReg)
935  .addReg(SrcReg)
936  .addImm(0) // op_sel_lo
937  .addImm(0) // op_sel_hi
938  .addImm(0) // neg_lo
939  .addImm(0) // neg_hi
940  .addImm(0) // clamp
941  .addReg(SrcReg, getKillRegState(KillSrc) | RegState::Implicit);
942  return;
943  }
944  }
945 
946  const bool Forward = RI.getHWRegIndex(DestReg) <= RI.getHWRegIndex(SrcReg);
947  if (RI.isSGPRClass(RC)) {
948  if (!RI.isSGPRClass(SrcRC)) {
949  reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc);
950  return;
951  }
952  const bool CanKillSuperReg = KillSrc && !RI.regsOverlap(SrcReg, DestReg);
953  expandSGPRCopy(*this, MBB, MI, DL, DestReg, SrcReg, CanKillSuperReg, RC,
954  Forward);
955  return;
956  }
957 
958  unsigned EltSize = 4;
959  unsigned Opcode = AMDGPU::V_MOV_B32_e32;
960  if (RI.isAGPRClass(RC)) {
961  if (ST.hasGFX90AInsts() && RI.isAGPRClass(SrcRC))
962  Opcode = AMDGPU::V_ACCVGPR_MOV_B32;
963  else if (RI.hasVGPRs(SrcRC) ||
964  (ST.hasGFX90AInsts() && RI.isSGPRClass(SrcRC)))
965  Opcode = AMDGPU::V_ACCVGPR_WRITE_B32_e64;
966  else
967  Opcode = AMDGPU::INSTRUCTION_LIST_END;
968  } else if (RI.hasVGPRs(RC) && RI.isAGPRClass(SrcRC)) {
969  Opcode = AMDGPU::V_ACCVGPR_READ_B32_e64;
970  } else if ((Size % 64 == 0) && RI.hasVGPRs(RC) &&
971  (RI.isProperlyAlignedRC(*RC) &&
972  (SrcRC == RC || RI.isSGPRClass(SrcRC)))) {
973  // TODO: In 96-bit case, could do a 64-bit mov and then a 32-bit mov.
974  if (ST.hasMovB64()) {
975  Opcode = AMDGPU::V_MOV_B64_e32;
976  EltSize = 8;
977  } else if (ST.hasPackedFP32Ops()) {
978  Opcode = AMDGPU::V_PK_MOV_B32;
979  EltSize = 8;
980  }
981  }
982 
983  // For the cases where we need an intermediate instruction/temporary register
984  // (destination is an AGPR), we need a scavenger.
985  //
986  // FIXME: The pass should maintain this for us so we don't have to re-scan the
987  // whole block for every handled copy.
988  std::unique_ptr<RegScavenger> RS;
989  if (Opcode == AMDGPU::INSTRUCTION_LIST_END)
990  RS.reset(new RegScavenger());
991 
992  ArrayRef<int16_t> SubIndices = RI.getRegSplitParts(RC, EltSize);
993 
994  // If there is an overlap, we can't kill the super-register on the last
995  // instruction, since it will also kill the components made live by this def.
996  const bool CanKillSuperReg = KillSrc && !RI.regsOverlap(SrcReg, DestReg);
997 
998  for (unsigned Idx = 0; Idx < SubIndices.size(); ++Idx) {
999  unsigned SubIdx;
1000  if (Forward)
1001  SubIdx = SubIndices[Idx];
1002  else
1003  SubIdx = SubIndices[SubIndices.size() - Idx - 1];
1004 
1005  bool UseKill = CanKillSuperReg && Idx == SubIndices.size() - 1;
1006 
1007  if (Opcode == AMDGPU::INSTRUCTION_LIST_END) {
1008  Register ImpDefSuper = Idx == 0 ? Register(DestReg) : Register();
1009  Register ImpUseSuper = SrcReg;
1010  indirectCopyToAGPR(*this, MBB, MI, DL, RI.getSubReg(DestReg, SubIdx),
1011  RI.getSubReg(SrcReg, SubIdx), UseKill, *RS,
1012  ImpDefSuper, ImpUseSuper);
1013  } else if (Opcode == AMDGPU::V_PK_MOV_B32) {
1014  Register DstSubReg = RI.getSubReg(DestReg, SubIdx);
1015  Register SrcSubReg = RI.getSubReg(SrcReg, SubIdx);
1016  MachineInstrBuilder MIB =
1017  BuildMI(MBB, MI, DL, get(AMDGPU::V_PK_MOV_B32), DstSubReg)
1019  .addReg(SrcSubReg)
1021  .addReg(SrcSubReg)
1022  .addImm(0) // op_sel_lo
1023  .addImm(0) // op_sel_hi
1024  .addImm(0) // neg_lo
1025  .addImm(0) // neg_hi
1026  .addImm(0) // clamp
1027  .addReg(SrcReg, getKillRegState(UseKill) | RegState::Implicit);
1028  if (Idx == 0)
1029  MIB.addReg(DestReg, RegState::Define | RegState::Implicit);
1030  } else {
1032  BuildMI(MBB, MI, DL, get(Opcode), RI.getSubReg(DestReg, SubIdx))
1033  .addReg(RI.getSubReg(SrcReg, SubIdx));
1034  if (Idx == 0)
1035  Builder.addReg(DestReg, RegState::Define | RegState::Implicit);
1036 
1037  Builder.addReg(SrcReg, getKillRegState(UseKill) | RegState::Implicit);
1038  }
1039  }
1040 }
1041 
1042 int SIInstrInfo::commuteOpcode(unsigned Opcode) const {
1043  int NewOpc;
1044 
1045  // Try to map original to commuted opcode
1046  NewOpc = AMDGPU::getCommuteRev(Opcode);
1047  if (NewOpc != -1)
1048  // Check if the commuted (REV) opcode exists on the target.
1049  return pseudoToMCOpcode(NewOpc) != -1 ? NewOpc : -1;
1050 
1051  // Try to map commuted to original opcode
1052  NewOpc = AMDGPU::getCommuteOrig(Opcode);
1053  if (NewOpc != -1)
1054  // Check if the original (non-REV) opcode exists on the target.
1055  return pseudoToMCOpcode(NewOpc) != -1 ? NewOpc : -1;
1056 
1057  return Opcode;
1058 }
1059 
1062  const DebugLoc &DL, Register DestReg,
1063  int64_t Value) const {
1065  const TargetRegisterClass *RegClass = MRI.getRegClass(DestReg);
1066  if (RegClass == &AMDGPU::SReg_32RegClass ||
1067  RegClass == &AMDGPU::SGPR_32RegClass ||
1068  RegClass == &AMDGPU::SReg_32_XM0RegClass ||
1069  RegClass == &AMDGPU::SReg_32_XM0_XEXECRegClass) {
1070  BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), DestReg)
1071  .addImm(Value);
1072  return;
1073  }
1074 
1075  if (RegClass == &AMDGPU::SReg_64RegClass ||
1076  RegClass == &AMDGPU::SGPR_64RegClass ||
1077  RegClass == &AMDGPU::SReg_64_XEXECRegClass) {
1078  BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B64), DestReg)
1079  .addImm(Value);
1080  return;
1081  }
1082 
1083  if (RegClass == &AMDGPU::VGPR_32RegClass) {
1084  BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DestReg)
1085  .addImm(Value);
1086  return;
1087  }
1088  if (RegClass->hasSuperClassEq(&AMDGPU::VReg_64RegClass)) {
1089  BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B64_PSEUDO), DestReg)
1090  .addImm(Value);
1091  return;
1092  }
1093 
1094  unsigned EltSize = 4;
1095  unsigned Opcode = AMDGPU::V_MOV_B32_e32;
1096  if (RI.isSGPRClass(RegClass)) {
1097  if (RI.getRegSizeInBits(*RegClass) > 32) {
1098  Opcode = AMDGPU::S_MOV_B64;
1099  EltSize = 8;
1100  } else {
1101  Opcode = AMDGPU::S_MOV_B32;
1102  EltSize = 4;
1103  }
1104  }
1105 
1106  ArrayRef<int16_t> SubIndices = RI.getRegSplitParts(RegClass, EltSize);
1107  for (unsigned Idx = 0; Idx < SubIndices.size(); ++Idx) {
1108  int64_t IdxValue = Idx == 0 ? Value : 0;
1109 
1111  get(Opcode), RI.getSubReg(DestReg, SubIndices[Idx]));
1112  Builder.addImm(IdxValue);
1113  }
1114 }
1115 
1116 const TargetRegisterClass *
1118  return &AMDGPU::VGPR_32RegClass;
1119 }
1120 
1123  const DebugLoc &DL, Register DstReg,
1125  Register TrueReg,
1126  Register FalseReg) const {
1128  const TargetRegisterClass *BoolXExecRC =
1129  RI.getRegClass(AMDGPU::SReg_1_XEXECRegClassID);
1130  assert(MRI.getRegClass(DstReg) == &AMDGPU::VGPR_32RegClass &&
1131  "Not a VGPR32 reg");
1132 
1133  if (Cond.size() == 1) {
1134  Register SReg = MRI.createVirtualRegister(BoolXExecRC);
1135  BuildMI(MBB, I, DL, get(AMDGPU::COPY), SReg)
1136  .add(Cond[0]);
1137  BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
1138  .addImm(0)
1139  .addReg(FalseReg)
1140  .addImm(0)
1141  .addReg(TrueReg)
1142  .addReg(SReg);
1143  } else if (Cond.size() == 2) {
1144  assert(Cond[0].isImm() && "Cond[0] is not an immediate");
1145  switch (Cond[0].getImm()) {
1146  case SIInstrInfo::SCC_TRUE: {
1147  Register SReg = MRI.createVirtualRegister(BoolXExecRC);
1148  BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_CSELECT_B32
1149  : AMDGPU::S_CSELECT_B64), SReg)
1150  .addImm(1)
1151  .addImm(0);
1152  BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
1153  .addImm(0)
1154  .addReg(FalseReg)
1155  .addImm(0)
1156  .addReg(TrueReg)
1157  .addReg(SReg);
1158  break;
1159  }
1160  case SIInstrInfo::SCC_FALSE: {
1161  Register SReg = MRI.createVirtualRegister(BoolXExecRC);
1162  BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_CSELECT_B32
1163  : AMDGPU::S_CSELECT_B64), SReg)
1164  .addImm(0)
1165  .addImm(1);
1166  BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
1167  .addImm(0)
1168  .addReg(FalseReg)
1169  .addImm(0)
1170  .addReg(TrueReg)
1171  .addReg(SReg);
1172  break;
1173  }
1174  case SIInstrInfo::VCCNZ: {
1175  MachineOperand RegOp = Cond[1];
1176  RegOp.setImplicit(false);
1177  Register SReg = MRI.createVirtualRegister(BoolXExecRC);
1178  BuildMI(MBB, I, DL, get(AMDGPU::COPY), SReg)
1179  .add(RegOp);
1180  BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
1181  .addImm(0)
1182  .addReg(FalseReg)
1183  .addImm(0)
1184  .addReg(TrueReg)
1185  .addReg(SReg);
1186  break;
1187  }
1188  case SIInstrInfo::VCCZ: {
1189  MachineOperand RegOp = Cond[1];
1190  RegOp.setImplicit(false);
1191  Register SReg = MRI.createVirtualRegister(BoolXExecRC);
1192  BuildMI(MBB, I, DL, get(AMDGPU::COPY), SReg)
1193  .add(RegOp);
1194  BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
1195  .addImm(0)
1196  .addReg(TrueReg)
1197  .addImm(0)
1198  .addReg(FalseReg)
1199  .addReg(SReg);
1200  break;
1201  }
1202  case SIInstrInfo::EXECNZ: {
1203  Register SReg = MRI.createVirtualRegister(BoolXExecRC);
1205  BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_OR_SAVEEXEC_B32
1206  : AMDGPU::S_OR_SAVEEXEC_B64), SReg2)
1207  .addImm(0);
1208  BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_CSELECT_B32
1209  : AMDGPU::S_CSELECT_B64), SReg)
1210  .addImm(1)
1211  .addImm(0);
1212  BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
1213  .addImm(0)
1214  .addReg(FalseReg)
1215  .addImm(0)
1216  .addReg(TrueReg)
1217  .addReg(SReg);
1218  break;
1219  }
1220  case SIInstrInfo::EXECZ: {
1221  Register SReg = MRI.createVirtualRegister(BoolXExecRC);
1223  BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_OR_SAVEEXEC_B32
1224  : AMDGPU::S_OR_SAVEEXEC_B64), SReg2)
1225  .addImm(0);
1226  BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_CSELECT_B32
1227  : AMDGPU::S_CSELECT_B64), SReg)
1228  .addImm(0)
1229  .addImm(1);
1230  BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
1231  .addImm(0)
1232  .addReg(FalseReg)
1233  .addImm(0)
1234  .addReg(TrueReg)
1235  .addReg(SReg);
1236  llvm_unreachable("Unhandled branch predicate EXECZ");
1237  break;
1238  }
1239  default:
1240  llvm_unreachable("invalid branch predicate");
1241  }
1242  } else {
1243  llvm_unreachable("Can only handle Cond size 1 or 2");
1244  }
1245 }
1246 
1249  const DebugLoc &DL,
1250  Register SrcReg, int Value) const {
1253  BuildMI(*MBB, I, DL, get(AMDGPU::V_CMP_EQ_I32_e64), Reg)
1254  .addImm(Value)
1255  .addReg(SrcReg);
1256 
1257  return Reg;
1258 }
1259 
1262  const DebugLoc &DL,
1263  Register SrcReg, int Value) const {
1266  BuildMI(*MBB, I, DL, get(AMDGPU::V_CMP_NE_I32_e64), Reg)
1267  .addImm(Value)
1268  .addReg(SrcReg);
1269 
1270  return Reg;
1271 }
1272 
1273 unsigned SIInstrInfo::getMovOpcode(const TargetRegisterClass *DstRC) const {
1274 
1275  if (RI.isAGPRClass(DstRC))
1276  return AMDGPU::COPY;
1277  if (RI.getRegSizeInBits(*DstRC) == 32) {
1278  return RI.isSGPRClass(DstRC) ? AMDGPU::S_MOV_B32 : AMDGPU::V_MOV_B32_e32;
1279  } else if (RI.getRegSizeInBits(*DstRC) == 64 && RI.isSGPRClass(DstRC)) {
1280  return AMDGPU::S_MOV_B64;
1281  } else if (RI.getRegSizeInBits(*DstRC) == 64 && !RI.isSGPRClass(DstRC)) {
1282  return AMDGPU::V_MOV_B64_PSEUDO;
1283  }
1284  return AMDGPU::COPY;
1285 }
1286 
1287 const MCInstrDesc &
1289  bool IsIndirectSrc) const {
1290  if (IsIndirectSrc) {
1291  if (VecSize <= 32) // 4 bytes
1292  return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V1);
1293  if (VecSize <= 64) // 8 bytes
1294  return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V2);
1295  if (VecSize <= 96) // 12 bytes
1296  return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V3);
1297  if (VecSize <= 128) // 16 bytes
1298  return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V4);
1299  if (VecSize <= 160) // 20 bytes
1300  return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V5);
1301  if (VecSize <= 256) // 32 bytes
1302  return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V8);
1303  if (VecSize <= 512) // 64 bytes
1304  return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V16);
1305  if (VecSize <= 1024) // 128 bytes
1306  return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V32);
1307 
1308  llvm_unreachable("unsupported size for IndirectRegReadGPRIDX pseudos");
1309  }
1310 
1311  if (VecSize <= 32) // 4 bytes
1312  return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V1);
1313  if (VecSize <= 64) // 8 bytes
1314  return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V2);
1315  if (VecSize <= 96) // 12 bytes
1316  return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V3);
1317  if (VecSize <= 128) // 16 bytes
1318  return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V4);
1319  if (VecSize <= 160) // 20 bytes
1320  return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V5);
1321  if (VecSize <= 256) // 32 bytes
1322  return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V8);
1323  if (VecSize <= 512) // 64 bytes
1324  return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V16);
1325  if (VecSize <= 1024) // 128 bytes
1326  return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V32);
1327 
1328  llvm_unreachable("unsupported size for IndirectRegWriteGPRIDX pseudos");
1329 }
1330 
1332  if (VecSize <= 32) // 4 bytes
1333  return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V1;
1334  if (VecSize <= 64) // 8 bytes
1335  return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V2;
1336  if (VecSize <= 96) // 12 bytes
1337  return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V3;
1338  if (VecSize <= 128) // 16 bytes
1339  return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V4;
1340  if (VecSize <= 160) // 20 bytes
1341  return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V5;
1342  if (VecSize <= 256) // 32 bytes
1343  return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V8;
1344  if (VecSize <= 512) // 64 bytes
1345  return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V16;
1346  if (VecSize <= 1024) // 128 bytes
1347  return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V32;
1348 
1349  llvm_unreachable("unsupported size for IndirectRegWrite pseudos");
1350 }
1351 
1352 static unsigned getIndirectSGPRWriteMovRelPseudo32(unsigned VecSize) {
1353  if (VecSize <= 32) // 4 bytes
1354  return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V1;
1355  if (VecSize <= 64) // 8 bytes
1356  return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V2;
1357  if (VecSize <= 96) // 12 bytes
1358  return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V3;
1359  if (VecSize <= 128) // 16 bytes
1360  return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V4;
1361  if (VecSize <= 160) // 20 bytes
1362  return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V5;
1363  if (VecSize <= 256) // 32 bytes
1364  return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V8;
1365  if (VecSize <= 512) // 64 bytes
1366  return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V16;
1367  if (VecSize <= 1024) // 128 bytes
1368  return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V32;
1369 
1370  llvm_unreachable("unsupported size for IndirectRegWrite pseudos");
1371 }
1372 
1373 static unsigned getIndirectSGPRWriteMovRelPseudo64(unsigned VecSize) {
1374  if (VecSize <= 64) // 8 bytes
1375  return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V1;
1376  if (VecSize <= 128) // 16 bytes
1377  return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V2;
1378  if (VecSize <= 256) // 32 bytes
1379  return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V4;
1380  if (VecSize <= 512) // 64 bytes
1381  return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V8;
1382  if (VecSize <= 1024) // 128 bytes
1383  return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V16;
1384 
1385  llvm_unreachable("unsupported size for IndirectRegWrite pseudos");
1386 }
1387 
1388 const MCInstrDesc &
1390  bool IsSGPR) const {
1391  if (IsSGPR) {
1392  switch (EltSize) {
1393  case 32:
1395  case 64:
1397  default:
1398  llvm_unreachable("invalid reg indexing elt size");
1399  }
1400  }
1401 
1402  assert(EltSize == 32 && "invalid reg indexing elt size");
1404 }
1405 
1406 static unsigned getSGPRSpillSaveOpcode(unsigned Size) {
1407  switch (Size) {
1408  case 4:
1409  return AMDGPU::SI_SPILL_S32_SAVE;
1410  case 8:
1411  return AMDGPU::SI_SPILL_S64_SAVE;
1412  case 12:
1413  return AMDGPU::SI_SPILL_S96_SAVE;
1414  case 16:
1415  return AMDGPU::SI_SPILL_S128_SAVE;
1416  case 20:
1417  return AMDGPU::SI_SPILL_S160_SAVE;
1418  case 24:
1419  return AMDGPU::SI_SPILL_S192_SAVE;
1420  case 28:
1421  return AMDGPU::SI_SPILL_S224_SAVE;
1422  case 32:
1423  return AMDGPU::SI_SPILL_S256_SAVE;
1424  case 64:
1425  return AMDGPU::SI_SPILL_S512_SAVE;
1426  case 128:
1427  return AMDGPU::SI_SPILL_S1024_SAVE;
1428  default:
1429  llvm_unreachable("unknown register size");
1430  }
1431 }
1432 
1433 static unsigned getVGPRSpillSaveOpcode(unsigned Size) {
1434  switch (Size) {
1435  case 4:
1436  return AMDGPU::SI_SPILL_V32_SAVE;
1437  case 8:
1438  return AMDGPU::SI_SPILL_V64_SAVE;
1439  case 12:
1440  return AMDGPU::SI_SPILL_V96_SAVE;
1441  case 16:
1442  return AMDGPU::SI_SPILL_V128_SAVE;
1443  case 20:
1444  return AMDGPU::SI_SPILL_V160_SAVE;
1445  case 24:
1446  return AMDGPU::SI_SPILL_V192_SAVE;
1447  case 28:
1448  return AMDGPU::SI_SPILL_V224_SAVE;
1449  case 32:
1450  return AMDGPU::SI_SPILL_V256_SAVE;
1451  case 64:
1452  return AMDGPU::SI_SPILL_V512_SAVE;
1453  case 128:
1454  return AMDGPU::SI_SPILL_V1024_SAVE;
1455  default:
1456  llvm_unreachable("unknown register size");
1457  }
1458 }
1459 
1460 static unsigned getAGPRSpillSaveOpcode(unsigned Size) {
1461  switch (Size) {
1462  case 4:
1463  return AMDGPU::SI_SPILL_A32_SAVE;
1464  case 8:
1465  return AMDGPU::SI_SPILL_A64_SAVE;
1466  case 12:
1467  return AMDGPU::SI_SPILL_A96_SAVE;
1468  case 16:
1469  return AMDGPU::SI_SPILL_A128_SAVE;
1470  case 20:
1471  return AMDGPU::SI_SPILL_A160_SAVE;
1472  case 24:
1473  return AMDGPU::SI_SPILL_A192_SAVE;
1474  case 28:
1475  return AMDGPU::SI_SPILL_A224_SAVE;
1476  case 32:
1477  return AMDGPU::SI_SPILL_A256_SAVE;
1478  case 64:
1479  return AMDGPU::SI_SPILL_A512_SAVE;
1480  case 128:
1481  return AMDGPU::SI_SPILL_A1024_SAVE;
1482  default:
1483  llvm_unreachable("unknown register size");
1484  }
1485 }
1486 
1487 static unsigned getAVSpillSaveOpcode(unsigned Size) {
1488  switch (Size) {
1489  case 4:
1490  return AMDGPU::SI_SPILL_AV32_SAVE;
1491  case 8:
1492  return AMDGPU::SI_SPILL_AV64_SAVE;
1493  case 12:
1494  return AMDGPU::SI_SPILL_AV96_SAVE;
1495  case 16:
1496  return AMDGPU::SI_SPILL_AV128_SAVE;
1497  case 20:
1498  return AMDGPU::SI_SPILL_AV160_SAVE;
1499  case 24:
1500  return AMDGPU::SI_SPILL_AV192_SAVE;
1501  case 28:
1502  return AMDGPU::SI_SPILL_AV224_SAVE;
1503  case 32:
1504  return AMDGPU::SI_SPILL_AV256_SAVE;
1505  case 64:
1506  return AMDGPU::SI_SPILL_AV512_SAVE;
1507  case 128:
1508  return AMDGPU::SI_SPILL_AV1024_SAVE;
1509  default:
1510  llvm_unreachable("unknown register size");
1511  }
1512 }
1513 
1516  Register SrcReg, bool isKill,
1517  int FrameIndex,
1518  const TargetRegisterClass *RC,
1519  const TargetRegisterInfo *TRI) const {
1520  MachineFunction *MF = MBB.getParent();
1522  MachineFrameInfo &FrameInfo = MF->getFrameInfo();
1523  const DebugLoc &DL = MBB.findDebugLoc(MI);
1524 
1525  MachinePointerInfo PtrInfo
1528  PtrInfo, MachineMemOperand::MOStore, FrameInfo.getObjectSize(FrameIndex),
1529  FrameInfo.getObjectAlign(FrameIndex));
1530  unsigned SpillSize = TRI->getSpillSize(*RC);
1531 
1533  if (RI.isSGPRClass(RC)) {
1534  MFI->setHasSpilledSGPRs();
1535  assert(SrcReg != AMDGPU::M0 && "m0 should not be spilled");
1536  assert(SrcReg != AMDGPU::EXEC_LO && SrcReg != AMDGPU::EXEC_HI &&
1537  SrcReg != AMDGPU::EXEC && "exec should not be spilled");
1538 
1539  // We are only allowed to create one new instruction when spilling
1540  // registers, so we need to use pseudo instruction for spilling SGPRs.
1541  const MCInstrDesc &OpDesc = get(getSGPRSpillSaveOpcode(SpillSize));
1542 
1543  // The SGPR spill/restore instructions only work on number sgprs, so we need
1544  // to make sure we are using the correct register class.
1545  if (SrcReg.isVirtual() && SpillSize == 4) {
1546  MRI.constrainRegClass(SrcReg, &AMDGPU::SReg_32_XM0_XEXECRegClass);
1547  }
1548 
1549  BuildMI(MBB, MI, DL, OpDesc)
1550  .addReg(SrcReg, getKillRegState(isKill)) // data
1551  .addFrameIndex(FrameIndex) // addr
1552  .addMemOperand(MMO)
1554 
1555  if (RI.spillSGPRToVGPR())
1556  FrameInfo.setStackID(FrameIndex, TargetStackID::SGPRSpill);
1557  return;
1558  }
1559 
1560  unsigned Opcode = RI.isVectorSuperClass(RC) ? getAVSpillSaveOpcode(SpillSize)
1561  : RI.isAGPRClass(RC) ? getAGPRSpillSaveOpcode(SpillSize)
1562  : getVGPRSpillSaveOpcode(SpillSize);
1563  MFI->setHasSpilledVGPRs();
1564 
1565  BuildMI(MBB, MI, DL, get(Opcode))
1566  .addReg(SrcReg, getKillRegState(isKill)) // data
1567  .addFrameIndex(FrameIndex) // addr
1568  .addReg(MFI->getStackPtrOffsetReg()) // scratch_offset
1569  .addImm(0) // offset
1570  .addMemOperand(MMO);
1571 }
1572 
1573 static unsigned getSGPRSpillRestoreOpcode(unsigned Size) {
1574  switch (Size) {
1575  case 4:
1576  return AMDGPU::SI_SPILL_S32_RESTORE;
1577  case 8:
1578  return AMDGPU::SI_SPILL_S64_RESTORE;
1579  case 12:
1580  return AMDGPU::SI_SPILL_S96_RESTORE;
1581  case 16:
1582  return AMDGPU::SI_SPILL_S128_RESTORE;
1583  case 20:
1584  return AMDGPU::SI_SPILL_S160_RESTORE;
1585  case 24:
1586  return AMDGPU::SI_SPILL_S192_RESTORE;
1587  case 28:
1588  return AMDGPU::SI_SPILL_S224_RESTORE;
1589  case 32:
1590  return AMDGPU::SI_SPILL_S256_RESTORE;
1591  case 64:
1592  return AMDGPU::SI_SPILL_S512_RESTORE;
1593  case 128:
1594  return AMDGPU::SI_SPILL_S1024_RESTORE;
1595  default:
1596  llvm_unreachable("unknown register size");
1597  }
1598 }
1599 
1600 static unsigned getVGPRSpillRestoreOpcode(unsigned Size) {
1601  switch (Size) {
1602  case 4:
1603  return AMDGPU::SI_SPILL_V32_RESTORE;
1604  case 8:
1605  return AMDGPU::SI_SPILL_V64_RESTORE;
1606  case 12:
1607  return AMDGPU::SI_SPILL_V96_RESTORE;
1608  case 16:
1609  return AMDGPU::SI_SPILL_V128_RESTORE;
1610  case 20:
1611  return AMDGPU::SI_SPILL_V160_RESTORE;
1612  case 24:
1613  return AMDGPU::SI_SPILL_V192_RESTORE;
1614  case 28:
1615  return AMDGPU::SI_SPILL_V224_RESTORE;
1616  case 32:
1617  return AMDGPU::SI_SPILL_V256_RESTORE;
1618  case 64:
1619  return AMDGPU::SI_SPILL_V512_RESTORE;
1620  case 128:
1621  return AMDGPU::SI_SPILL_V1024_RESTORE;
1622  default:
1623  llvm_unreachable("unknown register size");
1624  }
1625 }
1626 
1627 static unsigned getAGPRSpillRestoreOpcode(unsigned Size) {
1628  switch (Size) {
1629  case 4:
1630  return AMDGPU::SI_SPILL_A32_RESTORE;
1631  case 8:
1632  return AMDGPU::SI_SPILL_A64_RESTORE;
1633  case 12:
1634  return AMDGPU::SI_SPILL_A96_RESTORE;
1635  case 16:
1636  return AMDGPU::SI_SPILL_A128_RESTORE;
1637  case 20:
1638  return AMDGPU::SI_SPILL_A160_RESTORE;
1639  case 24:
1640  return AMDGPU::SI_SPILL_A192_RESTORE;
1641  case 28:
1642  return AMDGPU::SI_SPILL_A224_RESTORE;
1643  case 32:
1644  return AMDGPU::SI_SPILL_A256_RESTORE;
1645  case 64:
1646  return AMDGPU::SI_SPILL_A512_RESTORE;
1647  case 128:
1648  return AMDGPU::SI_SPILL_A1024_RESTORE;
1649  default:
1650  llvm_unreachable("unknown register size");
1651  }
1652 }
1653 
1654 static unsigned getAVSpillRestoreOpcode(unsigned Size) {
1655  switch (Size) {
1656  case 4:
1657  return AMDGPU::SI_SPILL_AV32_RESTORE;
1658  case 8:
1659  return AMDGPU::SI_SPILL_AV64_RESTORE;
1660  case 12:
1661  return AMDGPU::SI_SPILL_AV96_RESTORE;
1662  case 16:
1663  return AMDGPU::SI_SPILL_AV128_RESTORE;
1664  case 20:
1665  return AMDGPU::SI_SPILL_AV160_RESTORE;
1666  case 24:
1667  return AMDGPU::SI_SPILL_AV192_RESTORE;
1668  case 28:
1669  return AMDGPU::SI_SPILL_AV224_RESTORE;
1670  case 32:
1671  return AMDGPU::SI_SPILL_AV256_RESTORE;
1672  case 64:
1673  return AMDGPU::SI_SPILL_AV512_RESTORE;
1674  case 128:
1675  return AMDGPU::SI_SPILL_AV1024_RESTORE;
1676  default:
1677  llvm_unreachable("unknown register size");
1678  }
1679 }
1680 
1683  Register DestReg, int FrameIndex,
1684  const TargetRegisterClass *RC,
1685  const TargetRegisterInfo *TRI) const {
1686  MachineFunction *MF = MBB.getParent();
1688  MachineFrameInfo &FrameInfo = MF->getFrameInfo();
1689  const DebugLoc &DL = MBB.findDebugLoc(MI);
1690  unsigned SpillSize = TRI->getSpillSize(*RC);
1691 
1692  MachinePointerInfo PtrInfo
1694 
1696  PtrInfo, MachineMemOperand::MOLoad, FrameInfo.getObjectSize(FrameIndex),
1697  FrameInfo.getObjectAlign(FrameIndex));
1698 
1699  if (RI.isSGPRClass(RC)) {
1700  MFI->setHasSpilledSGPRs();
1701  assert(DestReg != AMDGPU::M0 && "m0 should not be reloaded into");
1702  assert(DestReg != AMDGPU::EXEC_LO && DestReg != AMDGPU::EXEC_HI &&
1703  DestReg != AMDGPU::EXEC && "exec should not be spilled");
1704 
1705  // FIXME: Maybe this should not include a memoperand because it will be
1706  // lowered to non-memory instructions.
1707  const MCInstrDesc &OpDesc = get(getSGPRSpillRestoreOpcode(SpillSize));
1708  if (DestReg.isVirtual() && SpillSize == 4) {
1710  MRI.constrainRegClass(DestReg, &AMDGPU::SReg_32_XM0_XEXECRegClass);
1711  }
1712 
1713  if (RI.spillSGPRToVGPR())
1714  FrameInfo.setStackID(FrameIndex, TargetStackID::SGPRSpill);
1715  BuildMI(MBB, MI, DL, OpDesc, DestReg)
1716  .addFrameIndex(FrameIndex) // addr
1717  .addMemOperand(MMO)
1719 
1720  return;
1721  }
1722 
1723  unsigned Opcode = RI.isVectorSuperClass(RC)
1724  ? getAVSpillRestoreOpcode(SpillSize)
1725  : RI.isAGPRClass(RC) ? getAGPRSpillRestoreOpcode(SpillSize)
1726  : getVGPRSpillRestoreOpcode(SpillSize);
1727  BuildMI(MBB, MI, DL, get(Opcode), DestReg)
1728  .addFrameIndex(FrameIndex) // vaddr
1729  .addReg(MFI->getStackPtrOffsetReg()) // scratch_offset
1730  .addImm(0) // offset
1731  .addMemOperand(MMO);
1732 }
1733 
1736  insertNoops(MBB, MI, 1);
1737 }
1738 
1741  unsigned Quantity) const {
1743  while (Quantity > 0) {
1744  unsigned Arg = std::min(Quantity, 8u);
1745  Quantity -= Arg;
1746  BuildMI(MBB, MI, DL, get(AMDGPU::S_NOP)).addImm(Arg - 1);
1747  }
1748 }
1749 
1751  auto MF = MBB.getParent();
1753 
1754  assert(Info->isEntryFunction());
1755 
1756  if (MBB.succ_empty()) {
1757  bool HasNoTerminator = MBB.getFirstTerminator() == MBB.end();
1758  if (HasNoTerminator) {
1759  if (Info->returnsVoid()) {
1760  BuildMI(MBB, MBB.end(), DebugLoc(), get(AMDGPU::S_ENDPGM)).addImm(0);
1761  } else {
1762  BuildMI(MBB, MBB.end(), DebugLoc(), get(AMDGPU::SI_RETURN_TO_EPILOG));
1763  }
1764  }
1765  }
1766 }
1767 
1769  switch (MI.getOpcode()) {
1770  default:
1771  if (MI.isMetaInstruction())
1772  return 0;
1773  return 1; // FIXME: Do wait states equal cycles?
1774 
1775  case AMDGPU::S_NOP:
1776  return MI.getOperand(0).getImm() + 1;
1777  // SI_RETURN_TO_EPILOG is a fallthrough to code outside of the function. The
1778  // hazard, even if one exist, won't really be visible. Should we handle it?
1779  }
1780 }
1781 
1783  const SIRegisterInfo *TRI = ST.getRegisterInfo();
1784  MachineBasicBlock &MBB = *MI.getParent();
1786  switch (MI.getOpcode()) {
1787  default: return TargetInstrInfo::expandPostRAPseudo(MI);
1788  case AMDGPU::S_MOV_B64_term:
1789  // This is only a terminator to get the correct spill code placement during
1790  // register allocation.
1791  MI.setDesc(get(AMDGPU::S_MOV_B64));
1792  break;
1793 
1794  case AMDGPU::S_MOV_B32_term:
1795  // This is only a terminator to get the correct spill code placement during
1796  // register allocation.
1797  MI.setDesc(get(AMDGPU::S_MOV_B32));
1798  break;
1799 
1800  case AMDGPU::S_XOR_B64_term:
1801  // This is only a terminator to get the correct spill code placement during
1802  // register allocation.
1803  MI.setDesc(get(AMDGPU::S_XOR_B64));
1804  break;
1805 
1806  case AMDGPU::S_XOR_B32_term:
1807  // This is only a terminator to get the correct spill code placement during
1808  // register allocation.
1809  MI.setDesc(get(AMDGPU::S_XOR_B32));
1810  break;
1811  case AMDGPU::S_OR_B64_term:
1812  // This is only a terminator to get the correct spill code placement during
1813  // register allocation.
1814  MI.setDesc(get(AMDGPU::S_OR_B64));
1815  break;
1816  case AMDGPU::S_OR_B32_term:
1817  // This is only a terminator to get the correct spill code placement during
1818  // register allocation.
1819  MI.setDesc(get(AMDGPU::S_OR_B32));
1820  break;
1821 
1822  case AMDGPU::S_ANDN2_B64_term:
1823  // This is only a terminator to get the correct spill code placement during
1824  // register allocation.
1825  MI.setDesc(get(AMDGPU::S_ANDN2_B64));
1826  break;
1827 
1828  case AMDGPU::S_ANDN2_B32_term:
1829  // This is only a terminator to get the correct spill code placement during
1830  // register allocation.
1831  MI.setDesc(get(AMDGPU::S_ANDN2_B32));
1832  break;
1833 
1834  case AMDGPU::S_AND_B64_term:
1835  // This is only a terminator to get the correct spill code placement during
1836  // register allocation.
1837  MI.setDesc(get(AMDGPU::S_AND_B64));
1838  break;
1839 
1840  case AMDGPU::S_AND_B32_term:
1841  // This is only a terminator to get the correct spill code placement during
1842  // register allocation.
1843  MI.setDesc(get(AMDGPU::S_AND_B32));
1844  break;
1845 
1846  case AMDGPU::V_MOV_B64_PSEUDO: {
1847  Register Dst = MI.getOperand(0).getReg();
1848  Register DstLo = RI.getSubReg(Dst, AMDGPU::sub0);
1849  Register DstHi = RI.getSubReg(Dst, AMDGPU::sub1);
1850 
1851  const MachineOperand &SrcOp = MI.getOperand(1);
1852  // FIXME: Will this work for 64-bit floating point immediates?
1853  assert(!SrcOp.isFPImm());
1854  if (ST.hasMovB64()) {
1855  MI.setDesc(get(AMDGPU::V_MOV_B64_e32));
1856  if (SrcOp.isReg() || isInlineConstant(MI, 1) ||
1857  isUInt<32>(SrcOp.getImm()))
1858  break;
1859  }
1860  if (SrcOp.isImm()) {
1861  APInt Imm(64, SrcOp.getImm());
1862  APInt Lo(32, Imm.getLoBits(32).getZExtValue());
1863  APInt Hi(32, Imm.getHiBits(32).getZExtValue());
1864  if (ST.hasPackedFP32Ops() && Lo == Hi && isInlineConstant(Lo)) {
1865  BuildMI(MBB, MI, DL, get(AMDGPU::V_PK_MOV_B32), Dst)
1867  .addImm(Lo.getSExtValue())
1869  .addImm(Lo.getSExtValue())
1870  .addImm(0) // op_sel_lo
1871  .addImm(0) // op_sel_hi
1872  .addImm(0) // neg_lo
1873  .addImm(0) // neg_hi
1874  .addImm(0); // clamp
1875  } else {
1876  BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstLo)
1877  .addImm(Lo.getSExtValue())
1879  BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstHi)
1880  .addImm(Hi.getSExtValue())
1882  }
1883  } else {
1884  assert(SrcOp.isReg());
1885  if (ST.hasPackedFP32Ops() &&
1886  !RI.isAGPR(MBB.getParent()->getRegInfo(), SrcOp.getReg())) {
1887  BuildMI(MBB, MI, DL, get(AMDGPU::V_PK_MOV_B32), Dst)
1888  .addImm(SISrcMods::OP_SEL_1) // src0_mod
1889  .addReg(SrcOp.getReg())
1891  .addReg(SrcOp.getReg())
1892  .addImm(0) // op_sel_lo
1893  .addImm(0) // op_sel_hi
1894  .addImm(0) // neg_lo
1895  .addImm(0) // neg_hi
1896  .addImm(0); // clamp
1897  } else {
1898  BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstLo)
1899  .addReg(RI.getSubReg(SrcOp.getReg(), AMDGPU::sub0))
1901  BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstHi)
1902  .addReg(RI.getSubReg(SrcOp.getReg(), AMDGPU::sub1))
1904  }
1905  }
1906  MI.eraseFromParent();
1907  break;
1908  }
1909  case AMDGPU::V_MOV_B64_DPP_PSEUDO: {
1910  expandMovDPP64(MI);
1911  break;
1912  }
1913  case AMDGPU::S_MOV_B64_IMM_PSEUDO: {
1914  const MachineOperand &SrcOp = MI.getOperand(1);
1915  assert(!SrcOp.isFPImm());
1916  APInt Imm(64, SrcOp.getImm());
1917  if (Imm.isIntN(32) || isInlineConstant(Imm)) {
1918  MI.setDesc(get(AMDGPU::S_MOV_B64));
1919  break;
1920  }
1921 
1922  Register Dst = MI.getOperand(0).getReg();
1923  Register DstLo = RI.getSubReg(Dst, AMDGPU::sub0);
1924  Register DstHi = RI.getSubReg(Dst, AMDGPU::sub1);
1925 
1926  APInt Lo(32, Imm.getLoBits(32).getZExtValue());
1927  APInt Hi(32, Imm.getHiBits(32).getZExtValue());
1928  BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), DstLo)
1929  .addImm(Lo.getSExtValue())
1931  BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), DstHi)
1932  .addImm(Hi.getSExtValue())
1934  MI.eraseFromParent();
1935  break;
1936  }
1937  case AMDGPU::V_SET_INACTIVE_B32: {
1938  unsigned NotOpc = ST.isWave32() ? AMDGPU::S_NOT_B32 : AMDGPU::S_NOT_B64;
1939  unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
1940  // FIXME: We may possibly optimize the COPY once we find ways to make LLVM
1941  // optimizations (mainly Register Coalescer) aware of WWM register liveness.
1942  BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), MI.getOperand(0).getReg())
1943  .add(MI.getOperand(1));
1944  auto FirstNot = BuildMI(MBB, MI, DL, get(NotOpc), Exec).addReg(Exec);
1945  FirstNot->addRegisterDead(AMDGPU::SCC, TRI); // SCC is overwritten
1946  BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), MI.getOperand(0).getReg())
1947  .add(MI.getOperand(2));
1948  BuildMI(MBB, MI, DL, get(NotOpc), Exec)
1949  .addReg(Exec);
1950  MI.eraseFromParent();
1951  break;
1952  }
1953  case AMDGPU::V_SET_INACTIVE_B64: {
1954  unsigned NotOpc = ST.isWave32() ? AMDGPU::S_NOT_B32 : AMDGPU::S_NOT_B64;
1955  unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
1956  MachineInstr *Copy = BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B64_PSEUDO),
1957  MI.getOperand(0).getReg())
1958  .add(MI.getOperand(1));
1959  expandPostRAPseudo(*Copy);
1960  auto FirstNot = BuildMI(MBB, MI, DL, get(NotOpc), Exec).addReg(Exec);
1961  FirstNot->addRegisterDead(AMDGPU::SCC, TRI); // SCC is overwritten
1962  Copy = BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B64_PSEUDO),
1963  MI.getOperand(0).getReg())
1964  .add(MI.getOperand(2));
1965  expandPostRAPseudo(*Copy);
1966  BuildMI(MBB, MI, DL, get(NotOpc), Exec)
1967  .addReg(Exec);
1968  MI.eraseFromParent();
1969  break;
1970  }
1971  case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V1:
1972  case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V2:
1973  case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V3:
1974  case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V4:
1975  case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V5:
1976  case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V8:
1977  case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V16:
1978  case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V32:
1979  case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V1:
1980  case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V2:
1981  case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V3:
1982  case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V4:
1983  case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V5:
1984  case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V8:
1985  case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V16:
1986  case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V32:
1987  case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V1:
1988  case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V2:
1989  case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V4:
1990  case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V8:
1991  case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V16: {
1992  const TargetRegisterClass *EltRC = getOpRegClass(MI, 2);
1993 
1994  unsigned Opc;
1995  if (RI.hasVGPRs(EltRC)) {
1996  Opc = AMDGPU::V_MOVRELD_B32_e32;
1997  } else {
1998  Opc = RI.getRegSizeInBits(*EltRC) == 64 ? AMDGPU::S_MOVRELD_B64
1999  : AMDGPU::S_MOVRELD_B32;
2000  }
2001 
2002  const MCInstrDesc &OpDesc = get(Opc);
2003  Register VecReg = MI.getOperand(0).getReg();
2004  bool IsUndef = MI.getOperand(1).isUndef();
2005  unsigned SubReg = MI.getOperand(3).getImm();
2006  assert(VecReg == MI.getOperand(1).getReg());
2007 
2008  MachineInstrBuilder MIB =
2009  BuildMI(MBB, MI, DL, OpDesc)
2010  .addReg(RI.getSubReg(VecReg, SubReg), RegState::Undef)
2011  .add(MI.getOperand(2))
2013  .addReg(VecReg, RegState::Implicit | (IsUndef ? RegState::Undef : 0));
2014 
2015  const int ImpDefIdx =
2016  OpDesc.getNumOperands() + OpDesc.getNumImplicitUses();
2017  const int ImpUseIdx = ImpDefIdx + 1;
2018  MIB->tieOperands(ImpDefIdx, ImpUseIdx);
2019  MI.eraseFromParent();
2020  break;
2021  }
2022  case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V1:
2023  case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V2:
2024  case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V3:
2025  case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V4:
2026  case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V5:
2027  case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V8:
2028  case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V16:
2029  case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V32: {
2030  assert(ST.useVGPRIndexMode());
2031  Register VecReg = MI.getOperand(0).getReg();
2032  bool IsUndef = MI.getOperand(1).isUndef();
2033  Register Idx = MI.getOperand(3).getReg();
2034  Register SubReg = MI.getOperand(4).getImm();
2035 
2036  MachineInstr *SetOn = BuildMI(MBB, MI, DL, get(AMDGPU::S_SET_GPR_IDX_ON))
2037  .addReg(Idx)
2039  SetOn->getOperand(3).setIsUndef();
2040 
2041  const MCInstrDesc &OpDesc = get(AMDGPU::V_MOV_B32_indirect_write);
2042  MachineInstrBuilder MIB =
2043  BuildMI(MBB, MI, DL, OpDesc)
2044  .addReg(RI.getSubReg(VecReg, SubReg), RegState::Undef)
2045  .add(MI.getOperand(2))
2047  .addReg(VecReg,
2048  RegState::Implicit | (IsUndef ? RegState::Undef : 0));
2049 
2050  const int ImpDefIdx = OpDesc.getNumOperands() + OpDesc.getNumImplicitUses();
2051  const int ImpUseIdx = ImpDefIdx + 1;
2052  MIB->tieOperands(ImpDefIdx, ImpUseIdx);
2053 
2054  MachineInstr *SetOff = BuildMI(MBB, MI, DL, get(AMDGPU::S_SET_GPR_IDX_OFF));
2055 
2056  finalizeBundle(MBB, SetOn->getIterator(), std::next(SetOff->getIterator()));
2057 
2058  MI.eraseFromParent();
2059  break;
2060  }
2061  case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V1:
2062  case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V2:
2063  case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V3:
2064  case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V4:
2065  case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V5:
2066  case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V8:
2067  case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V16:
2068  case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V32: {
2069  assert(ST.useVGPRIndexMode());
2070  Register Dst = MI.getOperand(0).getReg();
2071  Register VecReg = MI.getOperand(1).getReg();
2072  bool IsUndef = MI.getOperand(1).isUndef();
2073  Register Idx = MI.getOperand(2).getReg();
2074  Register SubReg = MI.getOperand(3).getImm();
2075 
2076  MachineInstr *SetOn = BuildMI(MBB, MI, DL, get(AMDGPU::S_SET_GPR_IDX_ON))
2077  .addReg(Idx)
2079  SetOn->getOperand(3).setIsUndef();
2080 
2081  BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_indirect_read))
2082  .addDef(Dst)
2083  .addReg(RI.getSubReg(VecReg, SubReg), RegState::Undef)
2084  .addReg(VecReg, RegState::Implicit | (IsUndef ? RegState::Undef : 0));
2085 
2086  MachineInstr *SetOff = BuildMI(MBB, MI, DL, get(AMDGPU::S_SET_GPR_IDX_OFF));
2087 
2088  finalizeBundle(MBB, SetOn->getIterator(), std::next(SetOff->getIterator()));
2089 
2090  MI.eraseFromParent();
2091  break;
2092  }
2093  case AMDGPU::SI_PC_ADD_REL_OFFSET: {
2094  MachineFunction &MF = *MBB.getParent();
2095  Register Reg = MI.getOperand(0).getReg();
2096  Register RegLo = RI.getSubReg(Reg, AMDGPU::sub0);
2097  Register RegHi = RI.getSubReg(Reg, AMDGPU::sub1);
2098 
2099  // Create a bundle so these instructions won't be re-ordered by the
2100  // post-RA scheduler.
2101  MIBundleBuilder Bundler(MBB, MI);
2102  Bundler.append(BuildMI(MF, DL, get(AMDGPU::S_GETPC_B64), Reg));
2103 
2104  // Add 32-bit offset from this instruction to the start of the
2105  // constant data.
2106  Bundler.append(BuildMI(MF, DL, get(AMDGPU::S_ADD_U32), RegLo)
2107  .addReg(RegLo)
2108  .add(MI.getOperand(1)));
2109 
2110  MachineInstrBuilder MIB = BuildMI(MF, DL, get(AMDGPU::S_ADDC_U32), RegHi)
2111  .addReg(RegHi);
2112  MIB.add(MI.getOperand(2));
2113 
2114  Bundler.append(MIB);
2115  finalizeBundle(MBB, Bundler.begin());
2116 
2117  MI.eraseFromParent();
2118  break;
2119  }
2120  case AMDGPU::ENTER_STRICT_WWM: {
2121  // This only gets its own opcode so that SIPreAllocateWWMRegs can tell when
2122  // Whole Wave Mode is entered.
2123  MI.setDesc(get(ST.isWave32() ? AMDGPU::S_OR_SAVEEXEC_B32
2124  : AMDGPU::S_OR_SAVEEXEC_B64));
2125  break;
2126  }
2127  case AMDGPU::ENTER_STRICT_WQM: {
2128  // This only gets its own opcode so that SIPreAllocateWWMRegs can tell when
2129  // STRICT_WQM is entered.
2130  const unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
2131  const unsigned WQMOp = ST.isWave32() ? AMDGPU::S_WQM_B32 : AMDGPU::S_WQM_B64;
2132  const unsigned MovOp = ST.isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64;
2133  BuildMI(MBB, MI, DL, get(MovOp), MI.getOperand(0).getReg()).addReg(Exec);
2134  BuildMI(MBB, MI, DL, get(WQMOp), Exec).addReg(Exec);
2135 
2136  MI.eraseFromParent();
2137  break;
2138  }
2139  case AMDGPU::EXIT_STRICT_WWM:
2140  case AMDGPU::EXIT_STRICT_WQM: {
2141  // This only gets its own opcode so that SIPreAllocateWWMRegs can tell when
2142  // WWM/STICT_WQM is exited.
2143  MI.setDesc(get(ST.isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64));
2144  break;
2145  }
2146  case AMDGPU::ENTER_PSEUDO_WM:
2147  case AMDGPU::EXIT_PSEUDO_WM: {
2148  // These do nothing.
2149  MI.eraseFromParent();
2150  break;
2151  }
2152  case AMDGPU::SI_RETURN: {
2153  const MachineFunction *MF = MBB.getParent();
2154  const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
2155  const SIRegisterInfo *TRI = ST.getRegisterInfo();
2156  // Hiding the return address use with SI_RETURN may lead to extra kills in
2157  // the function and missing live-ins. We are fine in practice because callee
2158  // saved register handling ensures the register value is restored before
2159  // RET, but we need the undef flag here to appease the MachineVerifier
2160  // liveness checks.
2161  MachineInstrBuilder MIB =
2162  BuildMI(MBB, MI, DL, get(AMDGPU::S_SETPC_B64_return))
2163  .addReg(TRI->getReturnAddressReg(*MF), RegState::Undef);
2164 
2165  MIB.copyImplicitOps(MI);
2166  MI.eraseFromParent();
2167  break;
2168  }
2169  }
2170  return true;
2171 }
2172 
2173 std::pair<MachineInstr*, MachineInstr*>
2175  assert (MI.getOpcode() == AMDGPU::V_MOV_B64_DPP_PSEUDO);
2176 
2177  if (ST.hasMovB64() &&
2179  getNamedOperand(MI, AMDGPU::OpName::dpp_ctrl)->getImm())) {
2180  MI.setDesc(get(AMDGPU::V_MOV_B64_dpp));
2181  return std::make_pair(&MI, nullptr);
2182  }
2183 
2184  MachineBasicBlock &MBB = *MI.getParent();
2186  MachineFunction *MF = MBB.getParent();
2188  Register Dst = MI.getOperand(0).getReg();
2189  unsigned Part = 0;
2190  MachineInstr *Split[2];
2191 
2192  for (auto Sub : { AMDGPU::sub0, AMDGPU::sub1 }) {
2193  auto MovDPP = BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_dpp));
2194  if (Dst.isPhysical()) {
2195  MovDPP.addDef(RI.getSubReg(Dst, Sub));
2196  } else {
2197  assert(MRI.isSSA());
2198  auto Tmp = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
2199  MovDPP.addDef(Tmp);
2200  }
2201 
2202  for (unsigned I = 1; I <= 2; ++I) { // old and src operands.
2203  const MachineOperand &SrcOp = MI.getOperand(I);
2204  assert(!SrcOp.isFPImm());
2205  if (SrcOp.isImm()) {
2206  APInt Imm(64, SrcOp.getImm());
2207  Imm.ashrInPlace(Part * 32);
2208  MovDPP.addImm(Imm.getLoBits(32).getZExtValue());
2209  } else {
2210  assert(SrcOp.isReg());
2211  Register Src = SrcOp.getReg();
2212  if (Src.isPhysical())
2213  MovDPP.addReg(RI.getSubReg(Src, Sub));
2214  else
2215  MovDPP.addReg(Src, SrcOp.isUndef() ? RegState::Undef : 0, Sub);
2216  }
2217  }
2218 
2219  for (const MachineOperand &MO : llvm::drop_begin(MI.explicit_operands(), 3))
2220  MovDPP.addImm(MO.getImm());
2221 
2222  Split[Part] = MovDPP;
2223  ++Part;
2224  }
2225 
2226  if (Dst.isVirtual())
2227  BuildMI(MBB, MI, DL, get(AMDGPU::REG_SEQUENCE), Dst)
2228  .addReg(Split[0]->getOperand(0).getReg())
2229  .addImm(AMDGPU::sub0)
2230  .addReg(Split[1]->getOperand(0).getReg())
2231  .addImm(AMDGPU::sub1);
2232 
2233  MI.eraseFromParent();
2234  return std::make_pair(Split[0], Split[1]);
2235 }
2236 
2238  MachineOperand &Src0,
2239  unsigned Src0OpName,
2240  MachineOperand &Src1,
2241  unsigned Src1OpName) const {
2242  MachineOperand *Src0Mods = getNamedOperand(MI, Src0OpName);
2243  if (!Src0Mods)
2244  return false;
2245 
2246  MachineOperand *Src1Mods = getNamedOperand(MI, Src1OpName);
2247  assert(Src1Mods &&
2248  "All commutable instructions have both src0 and src1 modifiers");
2249 
2250  int Src0ModsVal = Src0Mods->getImm();
2251  int Src1ModsVal = Src1Mods->getImm();
2252 
2253  Src1Mods->setImm(Src0ModsVal);
2254  Src0Mods->setImm(Src1ModsVal);
2255  return true;
2256 }
2257 
2259  MachineOperand &RegOp,
2260  MachineOperand &NonRegOp) {
2261  Register Reg = RegOp.getReg();
2262  unsigned SubReg = RegOp.getSubReg();
2263  bool IsKill = RegOp.isKill();
2264  bool IsDead = RegOp.isDead();
2265  bool IsUndef = RegOp.isUndef();
2266  bool IsDebug = RegOp.isDebug();
2267 
2268  if (NonRegOp.isImm())
2269  RegOp.ChangeToImmediate(NonRegOp.getImm());
2270  else if (NonRegOp.isFI())
2271  RegOp.ChangeToFrameIndex(NonRegOp.getIndex());
2272  else if (NonRegOp.isGlobal()) {
2273  RegOp.ChangeToGA(NonRegOp.getGlobal(), NonRegOp.getOffset(),
2274  NonRegOp.getTargetFlags());
2275  } else
2276  return nullptr;
2277 
2278  // Make sure we don't reinterpret a subreg index in the target flags.
2279  RegOp.setTargetFlags(NonRegOp.getTargetFlags());
2280 
2281  NonRegOp.ChangeToRegister(Reg, false, false, IsKill, IsDead, IsUndef, IsDebug);
2282  NonRegOp.setSubReg(SubReg);
2283 
2284  return &MI;
2285 }
2286 
2288  unsigned Src0Idx,
2289  unsigned Src1Idx) const {
2290  assert(!NewMI && "this should never be used");
2291 
2292  unsigned Opc = MI.getOpcode();
2293  int CommutedOpcode = commuteOpcode(Opc);
2294  if (CommutedOpcode == -1)
2295  return nullptr;
2296 
2297  assert(AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0) ==
2298  static_cast<int>(Src0Idx) &&
2299  AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1) ==
2300  static_cast<int>(Src1Idx) &&
2301  "inconsistency with findCommutedOpIndices");
2302 
2303  MachineOperand &Src0 = MI.getOperand(Src0Idx);
2304  MachineOperand &Src1 = MI.getOperand(Src1Idx);
2305 
2306  MachineInstr *CommutedMI = nullptr;
2307  if (Src0.isReg() && Src1.isReg()) {
2308  if (isOperandLegal(MI, Src1Idx, &Src0)) {
2309  // Be sure to copy the source modifiers to the right place.
2310  CommutedMI
2311  = TargetInstrInfo::commuteInstructionImpl(MI, NewMI, Src0Idx, Src1Idx);
2312  }
2313 
2314  } else if (Src0.isReg() && !Src1.isReg()) {
2315  // src0 should always be able to support any operand type, so no need to
2316  // check operand legality.
2317  CommutedMI = swapRegAndNonRegOperand(MI, Src0, Src1);
2318  } else if (!Src0.isReg() && Src1.isReg()) {
2319  if (isOperandLegal(MI, Src1Idx, &Src0))
2320  CommutedMI = swapRegAndNonRegOperand(MI, Src1, Src0);
2321  } else {
2322  // FIXME: Found two non registers to commute. This does happen.
2323  return nullptr;
2324  }
2325 
2326  if (CommutedMI) {
2327  swapSourceModifiers(MI, Src0, AMDGPU::OpName::src0_modifiers,
2328  Src1, AMDGPU::OpName::src1_modifiers);
2329 
2330  CommutedMI->setDesc(get(CommutedOpcode));
2331  }
2332 
2333  return CommutedMI;
2334 }
2335 
2336 // This needs to be implemented because the source modifiers may be inserted
2337 // between the true commutable operands, and the base
2338 // TargetInstrInfo::commuteInstruction uses it.
2340  unsigned &SrcOpIdx0,
2341  unsigned &SrcOpIdx1) const {
2342  return findCommutedOpIndices(MI.getDesc(), SrcOpIdx0, SrcOpIdx1);
2343 }
2344 
2345 bool SIInstrInfo::findCommutedOpIndices(MCInstrDesc Desc, unsigned &SrcOpIdx0,
2346  unsigned &SrcOpIdx1) const {
2347  if (!Desc.isCommutable())
2348  return false;
2349 
2350  unsigned Opc = Desc.getOpcode();
2351  int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0);
2352  if (Src0Idx == -1)
2353  return false;
2354 
2355  int Src1Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1);
2356  if (Src1Idx == -1)
2357  return false;
2358 
2359  return fixCommutedOpIndices(SrcOpIdx0, SrcOpIdx1, Src0Idx, Src1Idx);
2360 }
2361 
2362 bool SIInstrInfo::isBranchOffsetInRange(unsigned BranchOp,
2363  int64_t BrOffset) const {
2364  // BranchRelaxation should never have to check s_setpc_b64 because its dest
2365  // block is unanalyzable.
2366  assert(BranchOp != AMDGPU::S_SETPC_B64);
2367 
2368  // Convert to dwords.
2369  BrOffset /= 4;
2370 
2371  // The branch instructions do PC += signext(SIMM16 * 4) + 4, so the offset is
2372  // from the next instruction.
2373  BrOffset -= 1;
2374 
2375  return isIntN(BranchOffsetBits, BrOffset);
2376 }
2377 
2379  const MachineInstr &MI) const {
2380  if (MI.getOpcode() == AMDGPU::S_SETPC_B64) {
2381  // This would be a difficult analysis to perform, but can always be legal so
2382  // there's no need to analyze it.
2383  return nullptr;
2384  }
2385 
2386  return MI.getOperand(0).getMBB();
2387 }
2388 
2390  for (const MachineInstr &MI : MBB->terminators()) {
2391  if (MI.getOpcode() == AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO ||
2392  MI.getOpcode() == AMDGPU::SI_IF || MI.getOpcode() == AMDGPU::SI_ELSE ||
2393  MI.getOpcode() == AMDGPU::SI_LOOP)
2394  return true;
2395  }
2396  return false;
2397 }
2398 
2400  MachineBasicBlock &DestBB,
2401  MachineBasicBlock &RestoreBB,
2402  const DebugLoc &DL, int64_t BrOffset,
2403  RegScavenger *RS) const {
2404  assert(RS && "RegScavenger required for long branching");
2405  assert(MBB.empty() &&
2406  "new block should be inserted for expanding unconditional branch");
2407  assert(MBB.pred_size() == 1);
2408  assert(RestoreBB.empty() &&
2409  "restore block should be inserted for restoring clobbered registers");
2410 
2411  MachineFunction *MF = MBB.getParent();
2413 
2414  // FIXME: Virtual register workaround for RegScavenger not working with empty
2415  // blocks.
2416  Register PCReg = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
2417 
2418  auto I = MBB.end();
2419 
2420  // We need to compute the offset relative to the instruction immediately after
2421  // s_getpc_b64. Insert pc arithmetic code before last terminator.
2422  MachineInstr *GetPC = BuildMI(MBB, I, DL, get(AMDGPU::S_GETPC_B64), PCReg);
2423 
2424  auto &MCCtx = MF->getContext();
2425  MCSymbol *PostGetPCLabel =
2426  MCCtx.createTempSymbol("post_getpc", /*AlwaysAddSuffix=*/true);
2427  GetPC->setPostInstrSymbol(*MF, PostGetPCLabel);
2428 
2429  MCSymbol *OffsetLo =
2430  MCCtx.createTempSymbol("offset_lo", /*AlwaysAddSuffix=*/true);
2431  MCSymbol *OffsetHi =
2432  MCCtx.createTempSymbol("offset_hi", /*AlwaysAddSuffix=*/true);
2433  BuildMI(MBB, I, DL, get(AMDGPU::S_ADD_U32))
2434  .addReg(PCReg, RegState::Define, AMDGPU::sub0)
2435  .addReg(PCReg, 0, AMDGPU::sub0)
2436  .addSym(OffsetLo, MO_FAR_BRANCH_OFFSET);
2437  BuildMI(MBB, I, DL, get(AMDGPU::S_ADDC_U32))
2438  .addReg(PCReg, RegState::Define, AMDGPU::sub1)
2439  .addReg(PCReg, 0, AMDGPU::sub1)
2440  .addSym(OffsetHi, MO_FAR_BRANCH_OFFSET);
2441 
2442  // Insert the indirect branch after the other terminator.
2443  BuildMI(&MBB, DL, get(AMDGPU::S_SETPC_B64))
2444  .addReg(PCReg);
2445 
2446  // FIXME: If spilling is necessary, this will fail because this scavenger has
2447  // no emergency stack slots. It is non-trivial to spill in this situation,
2448  // because the restore code needs to be specially placed after the
2449  // jump. BranchRelaxation then needs to be made aware of the newly inserted
2450  // block.
2451  //
2452  // If a spill is needed for the pc register pair, we need to insert a spill
2453  // restore block right before the destination block, and insert a short branch
2454  // into the old destination block's fallthrough predecessor.
2455  // e.g.:
2456  //
2457  // s_cbranch_scc0 skip_long_branch:
2458  //
2459  // long_branch_bb:
2460  // spill s[8:9]
2461  // s_getpc_b64 s[8:9]
2462  // s_add_u32 s8, s8, restore_bb
2463  // s_addc_u32 s9, s9, 0
2464  // s_setpc_b64 s[8:9]
2465  //
2466  // skip_long_branch:
2467  // foo;
2468  //
2469  // .....
2470  //
2471  // dest_bb_fallthrough_predecessor:
2472  // bar;
2473  // s_branch dest_bb
2474  //
2475  // restore_bb:
2476  // restore s[8:9]
2477  // fallthrough dest_bb
2478  ///
2479  // dest_bb:
2480  // buzz;
2481 
2482  RS->enterBasicBlockEnd(MBB);
2484  AMDGPU::SReg_64RegClass, MachineBasicBlock::iterator(GetPC),
2485  /* RestoreAfter */ false, 0, /* AllowSpill */ false);
2486  if (Scav) {
2487  RS->setRegUsed(Scav);
2488  MRI.replaceRegWith(PCReg, Scav);
2489  MRI.clearVirtRegs();
2490  } else {
2491  // As SGPR needs VGPR to be spilled, we reuse the slot of temporary VGPR for
2492  // SGPR spill.
2493  const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
2494  const SIRegisterInfo *TRI = ST.getRegisterInfo();
2495  TRI->spillEmergencySGPR(GetPC, RestoreBB, AMDGPU::SGPR0_SGPR1, RS);
2496  MRI.replaceRegWith(PCReg, AMDGPU::SGPR0_SGPR1);
2497  MRI.clearVirtRegs();
2498  }
2499 
2500  MCSymbol *DestLabel = Scav ? DestBB.getSymbol() : RestoreBB.getSymbol();
2501  // Now, the distance could be defined.
2502  auto *Offset = MCBinaryExpr::createSub(
2503  MCSymbolRefExpr::create(DestLabel, MCCtx),
2504  MCSymbolRefExpr::create(PostGetPCLabel, MCCtx), MCCtx);
2505  // Add offset assignments.
2506  auto *Mask = MCConstantExpr::create(0xFFFFFFFFULL, MCCtx);
2507  OffsetLo->setVariableValue(MCBinaryExpr::createAnd(Offset, Mask, MCCtx));
2508  auto *ShAmt = MCConstantExpr::create(32, MCCtx);
2509  OffsetHi->setVariableValue(MCBinaryExpr::createAShr(Offset, ShAmt, MCCtx));
2510 }
2511 
2512 unsigned SIInstrInfo::getBranchOpcode(SIInstrInfo::BranchPredicate Cond) {
2513  switch (Cond) {
2514  case SIInstrInfo::SCC_TRUE:
2515  return AMDGPU::S_CBRANCH_SCC1;
2516  case SIInstrInfo::SCC_FALSE:
2517  return AMDGPU::S_CBRANCH_SCC0;
2518  case SIInstrInfo::VCCNZ:
2519  return AMDGPU::S_CBRANCH_VCCNZ;
2520  case SIInstrInfo::VCCZ:
2521  return AMDGPU::S_CBRANCH_VCCZ;
2522  case SIInstrInfo::EXECNZ:
2523  return AMDGPU::S_CBRANCH_EXECNZ;
2524  case SIInstrInfo::EXECZ:
2525  return AMDGPU::S_CBRANCH_EXECZ;
2526  default:
2527  llvm_unreachable("invalid branch predicate");
2528  }
2529 }
2530 
2531 SIInstrInfo::BranchPredicate SIInstrInfo::getBranchPredicate(unsigned Opcode) {
2532  switch (Opcode) {
2533  case AMDGPU::S_CBRANCH_SCC0:
2534  return SCC_FALSE;
2535  case AMDGPU::S_CBRANCH_SCC1:
2536  return SCC_TRUE;
2537  case AMDGPU::S_CBRANCH_VCCNZ:
2538  return VCCNZ;
2539  case AMDGPU::S_CBRANCH_VCCZ:
2540  return VCCZ;
2541  case AMDGPU::S_CBRANCH_EXECNZ:
2542  return EXECNZ;
2543  case AMDGPU::S_CBRANCH_EXECZ:
2544  return EXECZ;
2545  default:
2546  return INVALID_BR;
2547  }
2548 }
2549 
2553  MachineBasicBlock *&FBB,
2555  bool AllowModify) const {
2556  if (I->getOpcode() == AMDGPU::S_BRANCH) {
2557  // Unconditional Branch
2558  TBB = I->getOperand(0).getMBB();
2559  return false;
2560  }
2561 
2562  MachineBasicBlock *CondBB = nullptr;
2563 
2564  if (I->getOpcode() == AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO) {
2565  CondBB = I->getOperand(1).getMBB();
2566  Cond.push_back(I->getOperand(0));
2567  } else {
2568  BranchPredicate Pred = getBranchPredicate(I->getOpcode());
2569  if (Pred == INVALID_BR)
2570  return true;
2571 
2572  CondBB = I->getOperand(0).getMBB();
2573  Cond.push_back(MachineOperand::CreateImm(Pred));
2574  Cond.push_back(I->getOperand(1)); // Save the branch register.
2575  }
2576  ++I;
2577 
2578  if (I == MBB.end()) {
2579  // Conditional branch followed by fall-through.
2580  TBB = CondBB;
2581  return false;
2582  }
2583 
2584  if (I->getOpcode() == AMDGPU::S_BRANCH) {
2585  TBB = CondBB;
2586  FBB = I->getOperand(0).getMBB();
2587  return false;
2588  }
2589 
2590  return true;
2591 }
2592 
2594  MachineBasicBlock *&FBB,
2596  bool AllowModify) const {
2598  auto E = MBB.end();
2599  if (I == E)
2600  return false;
2601 
2602  // Skip over the instructions that are artificially terminators for special
2603  // exec management.
2604  while (I != E && !I->isBranch() && !I->isReturn()) {
2605  switch (I->getOpcode()) {
2606  case AMDGPU::S_MOV_B64_term:
2607  case AMDGPU::S_XOR_B64_term:
2608  case AMDGPU::S_OR_B64_term:
2609  case AMDGPU::S_ANDN2_B64_term:
2610  case AMDGPU::S_AND_B64_term:
2611  case AMDGPU::S_MOV_B32_term:
2612  case AMDGPU::S_XOR_B32_term:
2613  case AMDGPU::S_OR_B32_term:
2614  case AMDGPU::S_ANDN2_B32_term:
2615  case AMDGPU::S_AND_B32_term:
2616  break;
2617  case AMDGPU::SI_IF:
2618  case AMDGPU::SI_ELSE:
2619  case AMDGPU::SI_KILL_I1_TERMINATOR:
2620  case AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR:
2621  // FIXME: It's messy that these need to be considered here at all.
2622  return true;
2623  default:
2624  llvm_unreachable("unexpected non-branch terminator inst");
2625  }
2626 
2627  ++I;
2628  }
2629 
2630  if (I == E)
2631  return false;
2632 
2633  return analyzeBranchImpl(MBB, I, TBB, FBB, Cond, AllowModify);
2634 }
2635 
2637  int *BytesRemoved) const {
2638  unsigned Count = 0;
2639  unsigned RemovedSize = 0;
2641  // Skip over artificial terminators when removing instructions.
2642  if (MI.isBranch() || MI.isReturn()) {
2643  RemovedSize += getInstSizeInBytes(MI);
2644  MI.eraseFromParent();
2645  ++Count;
2646  }
2647  }
2648 
2649  if (BytesRemoved)
2650  *BytesRemoved = RemovedSize;
2651 
2652  return Count;
2653 }
2654 
2655 // Copy the flags onto the implicit condition register operand.
2657  const MachineOperand &OrigCond) {
2658  CondReg.setIsUndef(OrigCond.isUndef());
2659  CondReg.setIsKill(OrigCond.isKill());
2660 }
2661 
2664  MachineBasicBlock *FBB,
2666  const DebugLoc &DL,
2667  int *BytesAdded) const {
2668  if (!FBB && Cond.empty()) {
2669  BuildMI(&MBB, DL, get(AMDGPU::S_BRANCH))
2670  .addMBB(TBB);
2671  if (BytesAdded)
2672  *BytesAdded = ST.hasOffset3fBug() ? 8 : 4;
2673  return 1;
2674  }
2675 
2676  if(Cond.size() == 1 && Cond[0].isReg()) {
2677  BuildMI(&MBB, DL, get(AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO))
2678  .add(Cond[0])
2679  .addMBB(TBB);
2680  return 1;
2681  }
2682 
2683  assert(TBB && Cond[0].isImm());
2684 
2685  unsigned Opcode
2686  = getBranchOpcode(static_cast<BranchPredicate>(Cond[0].getImm()));
2687 
2688  if (!FBB) {
2689  Cond[1].isUndef();
2690  MachineInstr *CondBr =
2691  BuildMI(&MBB, DL, get(Opcode))
2692  .addMBB(TBB);
2693 
2694  // Copy the flags onto the implicit condition register operand.
2695  preserveCondRegFlags(CondBr->getOperand(1), Cond[1]);
2696  fixImplicitOperands(*CondBr);
2697 
2698  if (BytesAdded)
2699  *BytesAdded = ST.hasOffset3fBug() ? 8 : 4;
2700  return 1;
2701  }
2702 
2703  assert(TBB && FBB);
2704 
2705  MachineInstr *CondBr =
2706  BuildMI(&MBB, DL, get(Opcode))
2707  .addMBB(TBB);
2708  fixImplicitOperands(*CondBr);
2709  BuildMI(&MBB, DL, get(AMDGPU::S_BRANCH))
2710  .addMBB(FBB);
2711 
2712  MachineOperand &CondReg = CondBr->getOperand(1);
2713  CondReg.setIsUndef(Cond[1].isUndef());
2714  CondReg.setIsKill(Cond[1].isKill());
2715 
2716  if (BytesAdded)
2717  *BytesAdded = ST.hasOffset3fBug() ? 16 : 8;
2718 
2719  return 2;
2720 }
2721 
2724  if (Cond.size() != 2) {
2725  return true;
2726  }
2727 
2728  if (Cond[0].isImm()) {
2729  Cond[0].setImm(-Cond[0].getImm());
2730  return false;
2731  }
2732 
2733  return true;
2734 }
2735 
2738  Register DstReg, Register TrueReg,
2739  Register FalseReg, int &CondCycles,
2740  int &TrueCycles, int &FalseCycles) const {
2741  switch (Cond[0].getImm()) {
2742  case VCCNZ:
2743  case VCCZ: {
2745  const TargetRegisterClass *RC = MRI.getRegClass(TrueReg);
2746  if (MRI.getRegClass(FalseReg) != RC)
2747  return false;
2748 
2749  int NumInsts = AMDGPU::getRegBitWidth(RC->getID()) / 32;
2750  CondCycles = TrueCycles = FalseCycles = NumInsts; // ???
2751 
2752  // Limit to equal cost for branch vs. N v_cndmask_b32s.
2753  return RI.hasVGPRs(RC) && NumInsts <= 6;
2754  }
2755  case SCC_TRUE:
2756  case SCC_FALSE: {
2757  // FIXME: We could insert for VGPRs if we could replace the original compare
2758  // with a vector one.
2760  const TargetRegisterClass *RC = MRI.getRegClass(TrueReg);
2761  if (MRI.getRegClass(FalseReg) != RC)
2762  return false;
2763 
2764  int NumInsts = AMDGPU::getRegBitWidth(RC->getID()) / 32;
2765 
2766  // Multiples of 8 can do s_cselect_b64
2767  if (NumInsts % 2 == 0)
2768  NumInsts /= 2;
2769 
2770  CondCycles = TrueCycles = FalseCycles = NumInsts; // ???
2771  return RI.isSGPRClass(RC);
2772  }
2773  default:
2774  return false;
2775  }
2776 }
2777 
2781  Register TrueReg, Register FalseReg) const {
2782  BranchPredicate Pred = static_cast<BranchPredicate>(Cond[0].getImm());
2783  if (Pred == VCCZ || Pred == SCC_FALSE) {
2784  Pred = static_cast<BranchPredicate>(-Pred);
2785  std::swap(TrueReg, FalseReg);
2786  }
2787 
2789  const TargetRegisterClass *DstRC = MRI.getRegClass(DstReg);
2790  unsigned DstSize = RI.getRegSizeInBits(*DstRC);
2791 
2792  if (DstSize == 32) {
2794  if (Pred == SCC_TRUE) {
2795  Select = BuildMI(MBB, I, DL, get(AMDGPU::S_CSELECT_B32), DstReg)
2796  .addReg(TrueReg)
2797  .addReg(FalseReg);
2798  } else {
2799  // Instruction's operands are backwards from what is expected.
2800  Select = BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e32), DstReg)
2801  .addReg(FalseReg)
2802  .addReg(TrueReg);
2803  }
2804 
2805  preserveCondRegFlags(Select->getOperand(3), Cond[1]);
2806  return;
2807  }
2808 
2809  if (DstSize == 64 && Pred == SCC_TRUE) {
2810  MachineInstr *Select =
2811  BuildMI(MBB, I, DL, get(AMDGPU::S_CSELECT_B64), DstReg)
2812  .addReg(TrueReg)
2813  .addReg(FalseReg);
2814 
2815  preserveCondRegFlags(Select->getOperand(3), Cond[1]);
2816  return;
2817  }
2818 
2819  static const int16_t Sub0_15[] = {
2820  AMDGPU::sub0, AMDGPU::sub1, AMDGPU::sub2, AMDGPU::sub3,
2821  AMDGPU::sub4, AMDGPU::sub5, AMDGPU::sub6, AMDGPU::sub7,
2822  AMDGPU::sub8, AMDGPU::sub9, AMDGPU::sub10, AMDGPU::sub11,
2823  AMDGPU::sub12, AMDGPU::sub13, AMDGPU::sub14, AMDGPU::sub15,
2824  };
2825 
2826  static const int16_t Sub0_15_64[] = {
2827  AMDGPU::sub0_sub1, AMDGPU::sub2_sub3,
2828  AMDGPU::sub4_sub5, AMDGPU::sub6_sub7,
2829  AMDGPU::sub8_sub9, AMDGPU::sub10_sub11,
2830  AMDGPU::sub12_sub13, AMDGPU::sub14_sub15,
2831  };
2832 
2833  unsigned SelOp = AMDGPU::V_CNDMASK_B32_e32;
2834  const TargetRegisterClass *EltRC = &AMDGPU::VGPR_32RegClass;
2835  const int16_t *SubIndices = Sub0_15;
2836  int NElts = DstSize / 32;
2837 
2838  // 64-bit select is only available for SALU.
2839  // TODO: Split 96-bit into 64-bit and 32-bit, not 3x 32-bit.
2840  if (Pred == SCC_TRUE) {
2841  if (NElts % 2) {
2842  SelOp = AMDGPU::S_CSELECT_B32;
2843  EltRC = &AMDGPU::SGPR_32RegClass;
2844  } else {
2845  SelOp = AMDGPU::S_CSELECT_B64;
2846  EltRC = &AMDGPU::SGPR_64RegClass;
2847  SubIndices = Sub0_15_64;
2848  NElts /= 2;
2849  }
2850  }
2851 
2853  MBB, I, DL, get(AMDGPU::REG_SEQUENCE), DstReg);
2854 
2855  I = MIB->getIterator();
2856 
2858  for (int Idx = 0; Idx != NElts; ++Idx) {
2859  Register DstElt = MRI.createVirtualRegister(EltRC);
2860  Regs.push_back(DstElt);
2861 
2862  unsigned SubIdx = SubIndices[Idx];
2863 
2865  if (SelOp == AMDGPU::V_CNDMASK_B32_e32) {
2866  Select =
2867  BuildMI(MBB, I, DL, get(SelOp), DstElt)
2868  .addReg(FalseReg, 0, SubIdx)
2869  .addReg(TrueReg, 0, SubIdx);
2870  } else {
2871  Select =
2872  BuildMI(MBB, I, DL, get(SelOp), DstElt)
2873  .addReg(TrueReg, 0, SubIdx)
2874  .addReg(FalseReg, 0, SubIdx);
2875  }
2876 
2877  preserveCondRegFlags(Select->getOperand(3), Cond[1]);
2879 
2880  MIB.addReg(DstElt)
2881  .addImm(SubIdx);
2882  }
2883 }
2884 
2886  switch (MI.getOpcode()) {
2887  case AMDGPU::V_MOV_B32_e32:
2888  case AMDGPU::V_MOV_B32_e64:
2889  case AMDGPU::V_MOV_B64_PSEUDO:
2890  case AMDGPU::V_MOV_B64_e32:
2891  case AMDGPU::V_MOV_B64_e64:
2892  case AMDGPU::S_MOV_B32:
2893  case AMDGPU::S_MOV_B64:
2894  case AMDGPU::COPY:
2895  case AMDGPU::V_ACCVGPR_WRITE_B32_e64:
2896  case AMDGPU::V_ACCVGPR_READ_B32_e64:
2897  case AMDGPU::V_ACCVGPR_MOV_B32:
2898  return true;
2899  default:
2900  return false;
2901  }
2902 }
2903 
2904 static constexpr unsigned ModifierOpNames[] = {
2905  AMDGPU::OpName::src0_modifiers, AMDGPU::OpName::src1_modifiers,
2906  AMDGPU::OpName::src2_modifiers, AMDGPU::OpName::clamp,
2907  AMDGPU::OpName::omod, AMDGPU::OpName::op_sel};
2908 
2910  unsigned Opc = MI.getOpcode();
2911  for (unsigned Name : reverse(ModifierOpNames)) {
2912  int Idx = AMDGPU::getNamedOperandIdx(Opc, Name);
2913  if (Idx >= 0)
2914  MI.removeOperand(Idx);
2915  }
2916 }
2917 
2919  Register Reg, MachineRegisterInfo *MRI) const {
2920  if (!MRI->hasOneNonDBGUse(Reg))
2921  return false;
2922 
2923  switch (DefMI.getOpcode()) {
2924  default:
2925  return false;
2926  case AMDGPU::S_MOV_B64:
2927  // TODO: We could fold 64-bit immediates, but this get complicated
2928  // when there are sub-registers.
2929  return false;
2930 
2931  case AMDGPU::V_MOV_B32_e32:
2932  case AMDGPU::S_MOV_B32:
2933  case AMDGPU::V_ACCVGPR_WRITE_B32_e64:
2934  break;
2935  }
2936 
2937  const MachineOperand *ImmOp = getNamedOperand(DefMI, AMDGPU::OpName::src0);
2938  assert(ImmOp);
2939  // FIXME: We could handle FrameIndex values here.
2940  if (!ImmOp->isImm())
2941  return false;
2942 
2943  unsigned Opc = UseMI.getOpcode();
2944  if (Opc == AMDGPU::COPY) {
2945  Register DstReg = UseMI.getOperand(0).getReg();
2946  bool Is16Bit = getOpSize(UseMI, 0) == 2;
2947  bool isVGPRCopy = RI.isVGPR(*MRI, DstReg);
2948  unsigned NewOpc = isVGPRCopy ? AMDGPU::V_MOV_B32_e32 : AMDGPU::S_MOV_B32;
2949  APInt Imm(32, ImmOp->getImm());
2950 
2951  if (UseMI.getOperand(1).getSubReg() == AMDGPU::hi16)
2952  Imm = Imm.ashr(16);
2953 
2954  if (RI.isAGPR(*MRI, DstReg)) {
2955  if (!isInlineConstant(Imm))
2956  return false;
2957  NewOpc = AMDGPU::V_ACCVGPR_WRITE_B32_e64;
2958  }
2959 
2960  if (Is16Bit) {
2961  if (isVGPRCopy)
2962  return false; // Do not clobber vgpr_hi16
2963 
2964  if (DstReg.isVirtual() && UseMI.getOperand(0).getSubReg() != AMDGPU::lo16)
2965  return false;
2966 
2967  UseMI.getOperand(0).setSubReg(0);
2968  if (DstReg.isPhysical()) {
2969  DstReg = RI.get32BitRegister(DstReg);
2970  UseMI.getOperand(0).setReg(DstReg);
2971  }
2972  assert(UseMI.getOperand(1).getReg().isVirtual());
2973  }
2974 
2975  UseMI.setDesc(get(NewOpc));
2976  UseMI.getOperand(1).ChangeToImmediate(Imm.getSExtValue());
2977  UseMI.addImplicitDefUseOperands(*UseMI.getParent()->getParent());
2978  return true;
2979  }
2980 
2981  if (Opc == AMDGPU::V_MAD_F32_e64 || Opc == AMDGPU::V_MAC_F32_e64 ||
2982  Opc == AMDGPU::V_MAD_F16_e64 || Opc == AMDGPU::V_MAC_F16_e64 ||
2983  Opc == AMDGPU::V_FMA_F32_e64 || Opc == AMDGPU::V_FMAC_F32_e64 ||
2984  Opc == AMDGPU::V_FMA_F16_e64 || Opc == AMDGPU::V_FMAC_F16_e64 ||
2985  Opc == AMDGPU::V_FMAC_F16_t16_e64) {
2986  // Don't fold if we are using source or output modifiers. The new VOP2
2987  // instructions don't have them.
2989  return false;
2990 
2991  // If this is a free constant, there's no reason to do this.
2992  // TODO: We could fold this here instead of letting SIFoldOperands do it
2993  // later.
2994  MachineOperand *Src0 = getNamedOperand(UseMI, AMDGPU::OpName::src0);
2995 
2996  // Any src operand can be used for the legality check.
2997  if (isInlineConstant(UseMI, *Src0, *ImmOp))
2998  return false;
2999 
3000  bool IsF32 = Opc == AMDGPU::V_MAD_F32_e64 || Opc == AMDGPU::V_MAC_F32_e64 ||
3001  Opc == AMDGPU::V_FMA_F32_e64 || Opc == AMDGPU::V_FMAC_F32_e64;
3002  bool IsFMA =
3003  Opc == AMDGPU::V_FMA_F32_e64 || Opc == AMDGPU::V_FMAC_F32_e64 ||
3004  Opc == AMDGPU::V_FMA_F16_e64 || Opc == AMDGPU::V_FMAC_F16_e64 ||
3005  Opc == AMDGPU::V_FMAC_F16_t16_e64;
3006  MachineOperand *Src1 = getNamedOperand(UseMI, AMDGPU::OpName::src1);
3007  MachineOperand *Src2 = getNamedOperand(UseMI, AMDGPU::OpName::src2);
3008 
3009  // Multiplied part is the constant: Use v_madmk_{f16, f32}.
3010  // We should only expect these to be on src0 due to canonicalization.
3011  if (Src0->isReg() && Src0->getReg() == Reg) {
3012  if (!Src1->isReg() || RI.isSGPRClass(MRI->getRegClass(Src1->getReg())))
3013  return false;
3014 
3015  if (!Src2->isReg() || RI.isSGPRClass(MRI->getRegClass(Src2->getReg())))
3016  return false;
3017 
3018  unsigned NewOpc =
3019  IsFMA ? (IsF32 ? AMDGPU::V_FMAMK_F32
3020  : ST.hasTrue16BitInsts() ? AMDGPU::V_FMAMK_F16_t16
3021  : AMDGPU::V_FMAMK_F16)
3022  : (IsF32 ? AMDGPU::V_MADMK_F32 : AMDGPU::V_MADMK_F16);
3023  if (pseudoToMCOpcode(NewOpc) == -1)
3024  return false;
3025 
3026  // We need to swap operands 0 and 1 since madmk constant is at operand 1.
3027 
3028  const int64_t Imm = ImmOp->getImm();
3029 
3030  // FIXME: This would be a lot easier if we could return a new instruction
3031  // instead of having to modify in place.
3032 
3033  Register Src1Reg = Src1->getReg();
3034  unsigned Src1SubReg = Src1->getSubReg();
3035  Src0->setReg(Src1Reg);
3036  Src0->setSubReg(Src1SubReg);
3037  Src0->setIsKill(Src1->isKill());
3038 
3039  if (Opc == AMDGPU::V_MAC_F32_e64 || Opc == AMDGPU::V_MAC_F16_e64 ||
3040  Opc == AMDGPU::V_FMAC_F32_e64 || Opc == AMDGPU::V_FMAC_F16_t16_e64 ||
3041  Opc == AMDGPU::V_FMAC_F16_e64)
3042  UseMI.untieRegOperand(
3043  AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2));
3044 
3045  Src1->ChangeToImmediate(Imm);
3046 
3048  UseMI.setDesc(get(NewOpc));
3049 
3050  bool DeleteDef = MRI->use_nodbg_empty(Reg);
3051  if (DeleteDef)
3052  DefMI.eraseFromParent();
3053 
3054  return true;
3055  }
3056 
3057  // Added part is the constant: Use v_madak_{f16, f32}.
3058  if (Src2->isReg() && Src2->getReg() == Reg) {
3059  // Not allowed to use constant bus for another operand.
3060  // We can however allow an inline immediate as src0.
3061  bool Src0Inlined = false;
3062  if (Src0->isReg()) {
3063  // Try to inline constant if possible.
3064  // If the Def moves immediate and the use is single
3065  // We are saving VGPR here.
3067  if (Def && Def->isMoveImmediate() &&
3068  isInlineConstant(Def->getOperand(1)) &&
3069  MRI->hasOneUse(Src0->getReg())) {
3070  Src0->ChangeToImmediate(Def->getOperand(1).getImm());
3071  Src0Inlined = true;
3072  } else if ((Src0->getReg().isPhysical() &&
3073  (ST.getConstantBusLimit(Opc) <= 1 &&
3074  RI.isSGPRClass(RI.getPhysRegClass(Src0->getReg())))) ||
3075  (Src0->getReg().isVirtual() &&
3076  (ST.getConstantBusLimit(Opc) <= 1 &&
3077  RI.isSGPRClass(MRI->getRegClass(Src0->getReg())))))
3078  return false;
3079  // VGPR is okay as Src0 - fallthrough
3080  }
3081 
3082  if (Src1->isReg() && !Src0Inlined ) {
3083  // We have one slot for inlinable constant so far - try to fill it
3085  if (Def && Def->isMoveImmediate() &&
3086  isInlineConstant(Def->getOperand(1)) &&
3087  MRI->hasOneUse(Src1->getReg()) &&
3088  commuteInstruction(UseMI)) {
3089  Src0->ChangeToImmediate(Def->getOperand(1).getImm());
3090  } else if ((Src1->getReg().isPhysical() &&
3091  RI.isSGPRClass(RI.getPhysRegClass(Src1->getReg()))) ||
3092  (Src1->getReg().isVirtual() &&
3093  RI.isSGPRClass(MRI->getRegClass(Src1->getReg()))))
3094  return false;
3095  // VGPR is okay as Src1 - fallthrough
3096  }
3097 
3098  unsigned NewOpc =
3099  IsFMA ? (IsF32 ? AMDGPU::V_FMAAK_F32
3100  : ST.hasTrue16BitInsts() ? AMDGPU::V_FMAAK_F16_t16
3101  : AMDGPU::V_FMAAK_F16)
3102  : (IsF32 ? AMDGPU::V_MADAK_F32 : AMDGPU::V_MADAK_F16);
3103  if (pseudoToMCOpcode(NewOpc) == -1)
3104  return false;
3105 
3106  const int64_t Imm = ImmOp->getImm();
3107 
3108  // FIXME: This would be a lot easier if we could return a new instruction
3109  // instead of having to modify in place.
3110 
3111  if (Opc == AMDGPU::V_MAC_F32_e64 || Opc == AMDGPU::V_MAC_F16_e64 ||
3112  Opc == AMDGPU::V_FMAC_F32_e64 || Opc == AMDGPU::V_FMAC_F16_t16_e64 ||
3113  Opc == AMDGPU::V_FMAC_F16_e64)
3114  UseMI.untieRegOperand(
3115  AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2));
3116 
3117  // ChangingToImmediate adds Src2 back to the instruction.
3118  Src2->ChangeToImmediate(Imm);
3119 
3120  // These come before src2.
3122  UseMI.setDesc(get(NewOpc));
3123  // It might happen that UseMI was commuted
3124  // and we now have SGPR as SRC1. If so 2 inlined
3125  // constant and SGPR are illegal.
3127 
3128  bool DeleteDef = MRI->use_nodbg_empty(Reg);
3129  if (DeleteDef)
3130  DefMI.eraseFromParent();
3131 
3132  return true;
3133  }
3134  }
3135 
3136  return false;
3137 }
3138 
3139 static bool
3142  if (BaseOps1.size() != BaseOps2.size())
3143  return false;
3144  for (size_t I = 0, E = BaseOps1.size(); I < E; ++I) {
3145  if (!BaseOps1[I]->isIdenticalTo(*BaseOps2[I]))
3146  return false;
3147  }
3148  return true;
3149 }
3150 
3151 static bool offsetsDoNotOverlap(int WidthA, int OffsetA,
3152  int WidthB, int OffsetB) {
3153  int LowOffset = OffsetA < OffsetB ? OffsetA : OffsetB;
3154  int HighOffset = OffsetA < OffsetB ? OffsetB : OffsetA;
3155  int LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB;
3156  return LowOffset + LowWidth <= HighOffset;
3157 }
3158 
3159 bool SIInstrInfo::checkInstOffsetsDoNotOverlap(const MachineInstr &MIa,
3160  const MachineInstr &MIb) const {
3161  SmallVector<const MachineOperand *, 4> BaseOps0, BaseOps1;
3162  int64_t Offset0, Offset1;
3163  unsigned Dummy0, Dummy1;
3164  bool Offset0IsScalable, Offset1IsScalable;
3165  if (!getMemOperandsWithOffsetWidth(MIa, BaseOps0, Offset0, Offset0IsScalable,
3166  Dummy0, &RI) ||
3167  !getMemOperandsWithOffsetWidth(MIb, BaseOps1, Offset1, Offset1IsScalable,
3168  Dummy1, &RI))
3169  return false;
3170 
3171  if (!memOpsHaveSameBaseOperands(BaseOps0, BaseOps1))
3172  return false;
3173 
3174  if (!MIa.hasOneMemOperand() || !MIb.hasOneMemOperand()) {
3175  // FIXME: Handle ds_read2 / ds_write2.
3176  return false;
3177  }
3178  unsigned Width0 = MIa.memoperands().front()->getSize();
3179  unsigned Width1 = MIb.memoperands().front()->getSize();
3180  return offsetsDoNotOverlap(Width0, Offset0, Width1, Offset1);
3181 }
3182 
3184  const MachineInstr &MIb) const {
3185  assert(MIa.mayLoadOrStore() &&
3186  "MIa must load from or modify a memory location");
3187  assert(MIb.mayLoadOrStore() &&
3188  "MIb must load from or modify a memory location");
3189 
3191  return false;
3192 
3193  // XXX - Can we relax this between address spaces?
3194  if (MIa.hasOrderedMemoryRef() || MIb.hasOrderedMemoryRef())
3195  return false;
3196 
3197  // TODO: Should we check the address space from the MachineMemOperand? That
3198  // would allow us to distinguish objects we know don't alias based on the
3199  // underlying address space, even if it was lowered to a different one,
3200  // e.g. private accesses lowered to use MUBUF instructions on a scratch
3201  // buffer.
3202  if (isDS(MIa)) {
3203  if (isDS(MIb))
3204  return checkInstOffsetsDoNotOverlap(MIa, MIb);
3205 
3206  return !isFLAT(MIb) || isSegmentSpecificFLAT(MIb);
3207  }
3208 
3209  if (isMUBUF(MIa) || isMTBUF(MIa)) {
3210  if (isMUBUF(MIb) || isMTBUF(MIb))
3211  return checkInstOffsetsDoNotOverlap(MIa, MIb);
3212 
3213  return !isFLAT(MIb) && !isSMRD(MIb);
3214  }
3215 
3216  if (isSMRD(MIa)) {
3217  if (isSMRD(MIb))
3218  return checkInstOffsetsDoNotOverlap(MIa, MIb);
3219 
3220  return !isFLAT(MIb) && !isMUBUF(MIb) && !isMTBUF(MIb);
3221  }
3222 
3223  if (isFLAT(MIa)) {
3224  if (isFLAT(MIb))
3225  return checkInstOffsetsDoNotOverlap(MIa, MIb);
3226 
3227  return false;
3228  }
3229 
3230  return false;
3231 }
3232 
3234  int64_t &Imm, MachineInstr **DefMI = nullptr) {
3235  if (Reg.isPhysical())
3236  return false;
3237  auto *Def = MRI.getUniqueVRegDef(Reg);
3238  if (Def && SIInstrInfo::isFoldableCopy(*Def) && Def->getOperand(1).isImm()) {
3239  Imm = Def->getOperand(1).getImm();
3240  if (DefMI)
3241  *DefMI = Def;
3242  return true;
3243  }
3244  return false;
3245 }
3246 
3247 static bool getFoldableImm(const MachineOperand *MO, int64_t &Imm,
3248  MachineInstr **DefMI = nullptr) {
3249  if (!MO->isReg())
3250  return false;
3251  const MachineFunction *MF = MO->getParent()->getParent()->getParent();
3252  const MachineRegisterInfo &MRI = MF->getRegInfo();
3253  return getFoldableImm(MO->getReg(), MRI, Imm, DefMI);
3254 }
3255 
3257  MachineInstr &NewMI) {
3258  if (LV) {
3259  unsigned NumOps = MI.getNumOperands();
3260  for (unsigned I = 1; I < NumOps; ++I) {
3261  MachineOperand &Op = MI.getOperand(I);
3262  if (Op.isReg() && Op.isKill())
3263  LV->replaceKillInstruction(Op.getReg(), MI, NewMI);
3264  }
3265  }
3266 }
3267 
3269  LiveVariables *LV,
3270  LiveIntervals *LIS) const {
3271  MachineBasicBlock &MBB = *MI.getParent();
3272  unsigned Opc = MI.getOpcode();
3273 
3274  // Handle MFMA.
3275  int NewMFMAOpc = AMDGPU::getMFMAEarlyClobberOp(Opc);
3276  if (NewMFMAOpc != -1) {
3277  MachineInstrBuilder MIB =
3278  BuildMI(MBB, MI, MI.getDebugLoc(), get(NewMFMAOpc));
3279  for (unsigned I = 0, E = MI.getNumOperands(); I != E; ++I)
3280  MIB.add(MI.getOperand(I));
3281  updateLiveVariables(LV, MI, *MIB);
3282  if (LIS)
3283  LIS->ReplaceMachineInstrInMaps(MI, *MIB);
3284  return MIB;
3285  }
3286 
3287  if (SIInstrInfo::isWMMA(MI)) {
3288  unsigned NewOpc = AMDGPU::mapWMMA2AddrTo3AddrOpcode(MI.getOpcode());
3289  MachineInstrBuilder MIB = BuildMI(MBB, MI, MI.getDebugLoc(), get(NewOpc))
3290  .setMIFlags(MI.getFlags());
3291  for (unsigned I = 0, E = MI.getNumOperands(); I != E; ++I)
3292  MIB->addOperand(MI.getOperand(I));
3293 
3294  updateLiveVariables(LV, MI, *MIB);
3295  if (LIS)
3296  LIS->ReplaceMachineInstrInMaps(MI, *MIB);
3297 
3298  return MIB;
3299  }
3300 
3301  assert(Opc != AMDGPU::V_FMAC_F16_t16_e32 &&
3302  "V_FMAC_F16_t16_e32 is not supported and not expected to be present "
3303  "pre-RA");
3304 
3305  // Handle MAC/FMAC.
3306  bool IsF16 = Opc == AMDGPU::V_MAC_F16_e32 || Opc == AMDGPU::V_MAC_F16_e64 ||
3307  Opc == AMDGPU::V_FMAC_F16_e32 || Opc == AMDGPU::V_FMAC_F16_e64 ||
3308  Opc == AMDGPU::V_FMAC_F16_t16_e64;
3309  bool IsFMA = Opc == AMDGPU::V_FMAC_F32_e32 || Opc == AMDGPU::V_FMAC_F32_e64 ||
3310  Opc == AMDGPU::V_FMAC_LEGACY_F32_e32 ||
3311  Opc == AMDGPU::V_FMAC_LEGACY_F32_e64 ||
3312  Opc == AMDGPU::V_FMAC_F16_e32 || Opc == AMDGPU::V_FMAC_F16_e64 ||
3313  Opc == AMDGPU::V_FMAC_F16_t16_e64 ||
3314  Opc == AMDGPU::V_FMAC_F64_e32 || Opc == AMDGPU::V_FMAC_F64_e64;
3315  bool IsF64 = Opc == AMDGPU::V_FMAC_F64_e32 || Opc == AMDGPU::V_FMAC_F64_e64;
3316  bool IsLegacy = Opc == AMDGPU::V_MAC_LEGACY_F32_e32 ||
3317  Opc == AMDGPU::V_MAC_LEGACY_F32_e64 ||
3318  Opc == AMDGPU::V_FMAC_LEGACY_F32_e32 ||
3319  Opc == AMDGPU::V_FMAC_LEGACY_F32_e64;
3320  bool Src0Literal = false;
3321 
3322  switch (Opc) {
3323  default:
3324  return nullptr;
3325  case AMDGPU::V_MAC_F16_e64:
3326  case AMDGPU::V_FMAC_F16_e64:
3327  case AMDGPU::V_FMAC_F16_t16_e64:
3328  case AMDGPU::V_MAC_F32_e64:
3329  case AMDGPU::V_MAC_LEGACY_F32_e64:
3330  case AMDGPU::V_FMAC_F32_e64:
3331  case AMDGPU::V_FMAC_LEGACY_F32_e64:
3332  case AMDGPU::V_FMAC_F64_e64:
3333  break;
3334  case AMDGPU::V_MAC_F16_e32:
3335  case AMDGPU::V_FMAC_F16_e32:
3336  case AMDGPU::V_MAC_F32_e32:
3337  case AMDGPU::V_MAC_LEGACY_F32_e32:
3338  case AMDGPU::V_FMAC_F32_e32:
3339  case AMDGPU::V_FMAC_LEGACY_F32_e32:
3340  case AMDGPU::V_FMAC_F64_e32: {
3341  int Src0Idx = AMDGPU::getNamedOperandIdx(MI.getOpcode(),
3342  AMDGPU::OpName::src0);
3343  const MachineOperand *Src0 = &MI.getOperand(Src0Idx);
3344  if (!Src0->isReg() && !Src0->isImm())
3345  return nullptr;
3346 
3347  if (Src0->isImm() && !isInlineConstant(MI, Src0Idx, *Src0))
3348  Src0Literal = true;
3349 
3350  break;
3351  }
3352  }
3353 
3354  MachineInstrBuilder MIB;
3355  const MachineOperand *Dst = getNamedOperand(MI, AMDGPU::OpName::vdst);
3356  const MachineOperand *Src0 = getNamedOperand(MI, AMDGPU::OpName::src0);
3357  const MachineOperand *Src0Mods =
3358  getNamedOperand(MI, AMDGPU::OpName::src0_modifiers);
3359  const MachineOperand *Src1 = getNamedOperand(MI, AMDGPU::OpName::src1);
3360  const MachineOperand *Src1Mods =
3361  getNamedOperand(MI, AMDGPU::OpName::src1_modifiers);
3362  const MachineOperand *Src2 = getNamedOperand(MI, AMDGPU::OpName::src2);
3363  const MachineOperand *Src2Mods =
3364  getNamedOperand(MI, AMDGPU::OpName::src2_modifiers);
3365  const MachineOperand *Clamp = getNamedOperand(MI, AMDGPU::OpName::clamp);
3366  const MachineOperand *Omod = getNamedOperand(MI, AMDGPU::OpName::omod);
3367 
3368  if (!Src0Mods && !Src1Mods && !Src2Mods && !Clamp && !Omod && !IsF64 &&
3369  !IsLegacy &&
3370  // If we have an SGPR input, we will violate the constant bus restriction.
3371  (ST.getConstantBusLimit(Opc) > 1 || !Src0->isReg() ||
3372  !RI.isSGPRReg(MBB.getParent()->getRegInfo(), Src0->getReg()))) {
3374  const auto killDef = [&]() -> void {
3376  // The only user is the instruction which will be killed.
3377  Register DefReg = DefMI->getOperand(0).getReg();
3378  if (!MRI.hasOneNonDBGUse(DefReg))
3379  return;
3380  // We cannot just remove the DefMI here, calling pass will crash.
3381  DefMI->setDesc(get(AMDGPU::IMPLICIT_DEF));
3382  for (unsigned I = DefMI->getNumOperands() - 1; I != 0; --I)
3383  DefMI->removeOperand(I);
3384  if (LV)
3385  LV->getVarInfo(DefReg).AliveBlocks.clear();
3386  };
3387 
3388  int64_t Imm;
3389  if (!Src0Literal && getFoldableImm(Src2, Imm, &DefMI)) {
3390  unsigned NewOpc =
3391  IsFMA ? (IsF16 ? (ST.hasTrue16BitInsts() ? AMDGPU::V_FMAAK_F16_t16
3392  : AMDGPU::V_FMAAK_F16)
3393  : AMDGPU::V_FMAAK_F32)
3394  : (IsF16 ? AMDGPU::V_MADAK_F16 : AMDGPU::V_MADAK_F32);
3395  if (pseudoToMCOpcode(NewOpc) != -1) {
3396  MIB = BuildMI(MBB, MI, MI.getDebugLoc(), get(NewOpc))
3397  .add(*Dst)
3398  .add(*Src0)
3399  .add(*Src1)
3400  .addImm(Imm);
3401  updateLiveVariables(LV, MI, *MIB);
3402  if (LIS)
3403  LIS->ReplaceMachineInstrInMaps(MI, *MIB);
3404  killDef();
3405  return MIB;
3406  }
3407  }
3408  unsigned NewOpc =
3409  IsFMA ? (IsF16 ? (ST.hasTrue16BitInsts() ? AMDGPU::V_FMAMK_F16_t16
3410  : AMDGPU::V_FMAMK_F16)
3411  : AMDGPU::V_FMAMK_F32)
3412  : (IsF16 ? AMDGPU::V_MADMK_F16 : AMDGPU::V_MADMK_F32);
3413  if (!Src0Literal && getFoldableImm(Src1, Imm, &DefMI)) {
3414  if (pseudoToMCOpcode(NewOpc) != -1) {
3415  MIB = BuildMI(MBB, MI, MI.getDebugLoc(), get(NewOpc))
3416  .add(*Dst)
3417  .add(*Src0)
3418  .addImm(Imm)
3419  .add(*Src2);
3420  updateLiveVariables(LV, MI, *MIB);
3421  if (LIS)
3422  LIS->ReplaceMachineInstrInMaps(MI, *MIB);
3423  killDef();
3424  return MIB;
3425  }
3426  }
3427  if (Src0Literal || getFoldableImm(Src0, Imm, &DefMI)) {
3428  if (Src0Literal) {
3429  Imm = Src0->getImm();
3430  DefMI = nullptr;
3431  }
3432  if (pseudoToMCOpcode(NewOpc) != -1 &&
3434  MI, AMDGPU::getNamedOperandIdx(NewOpc, AMDGPU::OpName::src0),
3435  Src1)) {
3436  MIB = BuildMI(MBB, MI, MI.getDebugLoc(), get(NewOpc))
3437  .add(*Dst)
3438  .add(*Src1)
3439  .addImm(Imm)
3440  .add(*Src2);
3441  updateLiveVariables(LV, MI, *MIB);
3442  if (LIS)
3443  LIS->ReplaceMachineInstrInMaps(MI, *MIB);
3444  if (DefMI)
3445  killDef();
3446  return MIB;
3447  }
3448  }
3449  }
3450 
3451  // VOP2 mac/fmac with a literal operand cannot be converted to VOP3 mad/fma
3452  // if VOP3 does not allow a literal operand.
3453  if (Src0Literal && !ST.hasVOP3Literal())
3454  return nullptr;
3455 
3456  unsigned NewOpc = IsFMA ? IsF16 ? AMDGPU::V_FMA_F16_gfx9_e64
3457  : IsF64 ? AMDGPU::V_FMA_F64_e64
3458  : IsLegacy
3459  ? AMDGPU::V_FMA_LEGACY_F32_e64
3460  : AMDGPU::V_FMA_F32_e64
3461  : IsF16 ? AMDGPU::V_MAD_F16_e64
3462  : IsLegacy ? AMDGPU::V_MAD_LEGACY_F32_e64
3463  : AMDGPU::V_MAD_F32_e64;
3464  if (pseudoToMCOpcode(NewOpc) == -1)
3465  return nullptr;
3466 
3467  MIB = BuildMI(MBB, MI, MI.getDebugLoc(), get(NewOpc))
3468  .add(*Dst)
3469  .addImm(Src0Mods ? Src0Mods->getImm() : 0)
3470  .add(*Src0)
3471  .addImm(Src1Mods ? Src1Mods->getImm() : 0)
3472  .add(*Src1)
3473  .addImm(Src2Mods ? Src2Mods->getImm() : 0)
3474  .add(*Src2)
3475  .addImm(Clamp ? Clamp->getImm() : 0)
3476  .addImm(Omod ? Omod->getImm() : 0);
3477  updateLiveVariables(LV, MI, *MIB);
3478  if (LIS)
3479  LIS->ReplaceMachineInstrInMaps(MI, *MIB);
3480  return MIB;
3481 }
3482 
3483 // It's not generally safe to move VALU instructions across these since it will
3484 // start using the register as a base index rather than directly.
3485 // XXX - Why isn't hasSideEffects sufficient for these?
3487  switch (MI.getOpcode()) {
3488  case AMDGPU::S_SET_GPR_IDX_ON:
3489  case AMDGPU::S_SET_GPR_IDX_MODE:
3490  case AMDGPU::S_SET_GPR_IDX_OFF:
3491  return true;
3492  default:
3493  return false;
3494  }
3495 }
3496 
3498  const MachineBasicBlock *MBB,
3499  const MachineFunction &MF) const {
3500  // Skipping the check for SP writes in the base implementation. The reason it
3501  // was added was apparently due to compile time concerns.
3502  //
3503  // TODO: Do we really want this barrier? It triggers unnecessary hazard nops
3504  // but is probably avoidable.
3505 
3506  // Copied from base implementation.
3507  // Terminators and labels can't be scheduled around.
3508  if (MI.isTerminator() || MI.isPosition())
3509  return true;
3510 
3511  // INLINEASM_BR can jump to another block
3512  if (MI.getOpcode() == TargetOpcode::INLINEASM_BR)
3513  return true;
3514 
3515  if (MI.getOpcode() == AMDGPU::SCHED_BARRIER && MI.getOperand(0).getImm() == 0)
3516  return true;
3517 
3518  // Target-independent instructions do not have an implicit-use of EXEC, even
3519  // when they operate on VGPRs. Treating EXEC modifications as scheduling
3520  // boundaries prevents incorrect movements of such instructions.
3521  return MI.modifiesRegister(AMDGPU::EXEC, &RI) ||
3522  MI.getOpcode() == AMDGPU::S_SETREG_IMM32_B32 ||
3523  MI.getOpcode() == AMDGPU::S_SETREG_B32 ||
3524  MI.getOpcode() == AMDGPU::S_SETPRIO ||
3526 }
3527 
3529  return Opcode == AMDGPU::DS_ORDERED_COUNT ||
3530  Opcode == AMDGPU::DS_GWS_INIT ||
3531  Opcode == AMDGPU::DS_GWS_SEMA_V ||
3532  Opcode == AMDGPU::DS_GWS_SEMA_BR ||
3533  Opcode == AMDGPU::DS_GWS_SEMA_P ||
3534  Opcode == AMDGPU::DS_GWS_SEMA_RELEASE_ALL ||
3535  Opcode == AMDGPU::DS_GWS_BARRIER;
3536 }
3537 
3539  // Skip the full operand and register alias search modifiesRegister
3540  // does. There's only a handful of instructions that touch this, it's only an
3541  // implicit def, and doesn't alias any other registers.
3542  if (const MCPhysReg *ImpDef = MI.getDesc().getImplicitDefs()) {
3543  for (; ImpDef && *ImpDef; ++ImpDef) {
3544  if (*ImpDef == AMDGPU::MODE)
3545  return true;
3546  }
3547  }
3548 
3549  return false;
3550 }
3551 
3553  unsigned Opcode = MI.getOpcode();
3554 
3555  if (MI.mayStore() && isSMRD(MI))
3556  return true; // scalar store or atomic
3557 
3558  // This will terminate the function when other lanes may need to continue.
3559  if (MI.isReturn())
3560  return true;
3561 
3562  // These instructions cause shader I/O that may cause hardware lockups
3563  // when executed with an empty EXEC mask.
3564  //
3565  // Note: exp with VM = DONE = 0 is automatically skipped by hardware when
3566  // EXEC = 0, but checking for that case here seems not worth it
3567  // given the typical code patterns.
3568  if (Opcode == AMDGPU::S_SENDMSG || Opcode == AMDGPU::S_SENDMSGHALT ||
3569  isEXP(Opcode) ||
3570  Opcode == AMDGPU::DS_ORDERED_COUNT || Opcode == AMDGPU::S_TRAP ||
3571  Opcode == AMDGPU::DS_GWS_INIT || Opcode == AMDGPU::DS_GWS_BARRIER)
3572  return true;
3573 
3574  if (MI.isCall() || MI.isInlineAsm())
3575  return true; // conservative assumption
3576 
3577  // A mode change is a scalar operation that influences vector instructions.
3578  if (modifiesModeRegister(MI))
3579  return true;
3580 
3581  // These are like SALU instructions in terms of effects, so it's questionable
3582  // whether we should return true for those.
3583  //
3584  // However, executing them with EXEC = 0 causes them to operate on undefined
3585  // data, which we avoid by returning true here.
3586  if (Opcode == AMDGPU::V_READFIRSTLANE_B32 ||
3587  Opcode == AMDGPU::V_READLANE_B32 || Opcode == AMDGPU::V_WRITELANE_B32)
3588  return true;
3589 
3590  return false;
3591 }
3592 
3594  const MachineInstr &MI) const {
3595  if (MI.isMetaInstruction())
3596  return false;
3597 
3598  // This won't read exec if this is an SGPR->SGPR copy.
3599  if (MI.isCopyLike()) {
3600  if (!RI.isSGPRReg(MRI, MI.getOperand(0).getReg()))
3601  return true;
3602 
3603  // Make sure this isn't copying exec as a normal operand
3604  return MI.readsRegister(AMDGPU::EXEC, &RI);
3605  }
3606 
3607  // Make a conservative assumption about the callee.
3608  if (MI.isCall())
3609  return true;
3610 
3611  // Be conservative with any unhandled generic opcodes.
3612  if (!isTargetSpecificOpcode(MI.getOpcode()))
3613  return true;
3614 
3615  return !isSALU(MI) || MI.readsRegister(AMDGPU::EXEC, &RI);
3616 }
3617 
3619  switch (Imm.getBitWidth()) {
3620  case 1: // This likely will be a condition code mask.
3621  return true;
3622 
3623  case 32:
3624  return AMDGPU::isInlinableLiteral32(Imm.getSExtValue(),
3625  ST.hasInv2PiInlineImm());
3626  case 64:
3627  return AMDGPU::isInlinableLiteral64(Imm.getSExtValue(),
3628  ST.hasInv2PiInlineImm());
3629  case 16:
3630  return ST.has16BitInsts() &&
3631  AMDGPU::isInlinableLiteral16(Imm.getSExtValue(),
3632  ST.hasInv2PiInlineImm());
3633  default:
3634  llvm_unreachable("invalid bitwidth");
3635  }
3636 }
3637 
3639  uint8_t OperandType) const {
3640  assert(!MO.isReg() && "isInlineConstant called on register operand!");
3641  if (!MO.isImm() ||
3644  return false;
3645 
3646  // MachineOperand provides no way to tell the true operand size, since it only
3647  // records a 64-bit value. We need to know the size to determine if a 32-bit
3648  // floating point immediate bit pattern is legal for an integer immediate. It
3649  // would be for any 32-bit integer operand, but would not be for a 64-bit one.
3650 
3651  int64_t Imm = MO.getImm();
3652  switch (OperandType) {
3664  int32_t Trunc = static_cast<int32_t>(Imm);
3666  }
3673  ST.hasInv2PiInlineImm());
3677  // We would expect inline immediates to not be concerned with an integer/fp
3678  // distinction. However, in the case of 16-bit integer operations, the
3679  // "floating point" values appear to not work. It seems read the low 16-bits
3680  // of 32-bit immediates, which happens to always work for the integer
3681  // values.
3682  //
3683  // See llvm bugzilla 46302.
3684  //
3685  // TODO: Theoretically we could use op-sel to use the high bits of the
3686  // 32-bit FP values.
3691  // This suffers the same problem as the scalar 16-bit cases.
3697  if (isInt<16>(Imm) || isUInt<16>(Imm)) {
3698  // A few special case instructions have 16-bit operands on subtargets
3699  // where 16-bit instructions are not legal.
3700  // TODO: Do the 32-bit immediates work? We shouldn't really need to handle
3701  // constants in these cases
3702  int16_t Trunc = static_cast<int16_t>(Imm);
3703  return ST.has16BitInsts() &&
3705  }
3706 
3707  return false;
3708  }
3712  uint32_t Trunc = static_cast<uint32_t>(Imm);
3714  }
3717  return false;
3718  default:
3719  llvm_unreachable("invalid bitwidth");
3720  }
3721 }
3722 
3723 static bool compareMachineOp(const MachineOperand &Op0,
3724  const MachineOperand &Op1) {
3725  if (Op0.getType() != Op1.getType())
3726  return false;
3727 
3728  switch (Op0.getType()) {
3730  return Op0.getReg() == Op1.getReg();
3732  return Op0.getImm() == Op1.getImm();
3733  default:
3734  llvm_unreachable("Didn't expect to be comparing these operand types");
3735  }
3736 }
3737 
3739  const MachineOperand &MO) const {
3740  const MCInstrDesc &InstDesc = MI.getDesc();
3741  const MCOperandInfo &OpInfo = InstDesc.OpInfo[OpNo];
3742 
3743  assert(MO.isImm() || MO.isTargetIndex() || MO.isFI() || MO.isGlobal());
3744 
3745  if (OpInfo.OperandType == MCOI::OPERAND_IMMEDIATE)
3746  return true;
3747 
3748  if (OpInfo.RegClass < 0)
3749  return false;
3750 
3751  if (MO.isImm() && isInlineConstant(MO, OpInfo)) {
3752  if (isMAI(MI) && ST.hasMFMAInlineLiteralBug() &&
3753  OpNo ==(unsigned)AMDGPU::getNamedOperandIdx(MI.getOpcode(),
3754  AMDGPU::OpName::src2))
3755  return false;
3756  return RI.opCanUseInlineConstant(OpInfo.OperandType);
3757  }
3758 
3759  if (!RI.opCanUseLiteralConstant(OpInfo.OperandType))
3760  return false;
3761 
3762  if (!isVOP3(MI) || !AMDGPU::isSISrcOperand(InstDesc, OpNo))
3763  return true;
3764 
3765  return ST.hasVOP3Literal();
3766 }
3767 
3768 bool SIInstrInfo::hasVALU32BitEncoding(unsigned Opcode) const {
3769  // GFX90A does not have V_MUL_LEGACY_F32_e32.
3770  if (Opcode == AMDGPU::V_MUL_LEGACY_F32_e64 && ST.hasGFX90AInsts())
3771  return false;
3772 
3773  int Op32 = AMDGPU::getVOPe32(Opcode);
3774  if (Op32 == -1)
3775  return false;
3776 
3777  return pseudoToMCOpcode(Op32) != -1;
3778 }
3779 
3780 bool SIInstrInfo::hasModifiers(unsigned Opcode) const {
3781  // The src0_modifier operand is present on all instructions
3782  // that have modifiers.
3783 
3784  return AMDGPU::hasNamedOperand(Opcode, AMDGPU::OpName::src0_modifiers);
3785 }
3786 
3788  unsigned OpName) const {
3789  const MachineOperand *Mods = getNamedOperand(MI, OpName);
3790  return Mods && Mods->getImm();
3791 }
3792 
3794  return any_of(ModifierOpNames,
3795  [&](unsigned Name) { return hasModifiersSet(MI, Name); });
3796 }
3797 
3799  const MachineRegisterInfo &MRI) const {
3800  const MachineOperand *Src2 = getNamedOperand(MI, AMDGPU::OpName::src2);
3801  // Can't shrink instruction with three operands.
3802  if (Src2) {
3803  switch (MI.getOpcode()) {
3804  default: return false;
3805 
3806  case AMDGPU::V_ADDC_U32_e64:
3807  case AMDGPU::V_SUBB_U32_e64:
3808  case AMDGPU::V_SUBBREV_U32_e64: {
3809  const MachineOperand *Src1
3810  = getNamedOperand(MI, AMDGPU::OpName::src1);
3811  if (!Src1->isReg() || !RI.isVGPR(MRI, Src1->getReg()))
3812  return false;
3813  // Additional verification is needed for sdst/src2.
3814  return true;
3815  }
3816  case AMDGPU::V_MAC_F16_e64:
3817  case AMDGPU::V_MAC_F32_e64:
3818  case AMDGPU::V_MAC_LEGACY_F32_e64:
3819  case AMDGPU::V_FMAC_F16_e64:
3820  case AMDGPU::V_FMAC_F16_t16_e64:
3821  case AMDGPU::V_FMAC_F32_e64:
3822  case AMDGPU::V_FMAC_F64_e64:
3823  case AMDGPU::V_FMAC_LEGACY_F32_e64:
3824  if (!Src2->isReg() || !RI.isVGPR(MRI, Src2->getReg()) ||
3825  hasModifiersSet(MI, AMDGPU::OpName::src2_modifiers))
3826  return false;
3827  break;
3828 
3829  case AMDGPU::V_CNDMASK_B32_e64:
3830  break;
3831  }
3832  }
3833 
3834  const MachineOperand *Src1 = getNamedOperand(MI, AMDGPU::OpName::src1);
3835  if (Src1 && (!Src1->isReg() || !RI.isVGPR(MRI, Src1->getReg()) ||
3836  hasModifiersSet(MI, AMDGPU::OpName::src1_modifiers)))
3837  return false;
3838 
3839  // We don't need to check src0, all input types are legal, so just make sure
3840  // src0 isn't using any modifiers.
3841  if (hasModifiersSet(MI, AMDGPU::OpName::src0_modifiers))
3842  return false;
3843 
3844  // Can it be shrunk to a valid 32 bit opcode?
3845  if (!hasVALU32BitEncoding(MI.getOpcode()))
3846  return false;
3847 
3848  // Check output modifiers
3849  return !hasModifiersSet(MI, AMDGPU::OpName::omod) &&
3850  !hasModifiersSet(MI, AMDGPU::OpName::clamp);
3851 }
3852 
3853 // Set VCC operand with all flags from \p Orig, except for setting it as
3854 // implicit.
3856  const MachineOperand &Orig) {
3857 
3858  for (MachineOperand &Use : MI.implicit_operands()) {
3859  if (Use.isUse() &&
3860  (Use.getReg() == AMDGPU::VCC || Use.getReg() == AMDGPU::VCC_LO)) {
3861  Use.setIsUndef(Orig.isUndef());
3862  Use.setIsKill(Orig.isKill());
3863  return;
3864  }
3865  }
3866 }
3867 
3869  unsigned Op32) const {
3870  MachineBasicBlock *MBB = MI.getParent();
3871  MachineInstrBuilder Inst32 =
3872  BuildMI(*MBB, MI, MI.getDebugLoc(), get(Op32))
3873  .setMIFlags(MI.getFlags());
3874 
3875  // Add the dst operand if the 32-bit encoding also has an explicit $vdst.
3876  // For VOPC instructions, this is replaced by an implicit def of vcc.
3877  if (AMDGPU::hasNamedOperand(Op32, AMDGPU::OpName::vdst)) {
3878  // dst
3879  Inst32.add(MI.getOperand(0));
3880  } else if (AMDGPU::hasNamedOperand(Op32, AMDGPU::OpName::sdst)) {
3881  // VOPCX instructions won't be writing to an explicit dst, so this should
3882  // not fail for these instructions.
3883  assert(((MI.getOperand(0).getReg() == AMDGPU::VCC) ||
3884  (MI.getOperand(0).getReg() == AMDGPU::VCC_LO)) &&
3885  "Unexpected case");
3886  }
3887 
3888  Inst32.add(*getNamedOperand(MI, AMDGPU::OpName::src0));
3889 
3890  const MachineOperand *Src1 = getNamedOperand(MI, AMDGPU::OpName::src1);
3891  if (Src1)
3892  Inst32.add(*Src1);
3893 
3894  const MachineOperand *Src2 = getNamedOperand(MI, AMDGPU::OpName::src2);
3895 
3896  if (Src2) {
3897  int Op32Src2Idx = AMDGPU::getNamedOperandIdx(Op32, AMDGPU::OpName::src2);
3898  if (Op32Src2Idx != -1) {
3899  Inst32.add(*Src2);
3900  } else {
3901  // In the case of V_CNDMASK_B32_e32, the explicit operand src2 is
3902  // replaced with an implicit read of vcc or vcc_lo. The implicit read
3903  // of vcc was already added during the initial BuildMI, but we
3904  // 1) may need to change vcc to vcc_lo to preserve the original register
3905  // 2) have to preserve the original flags.
3906  fixImplicitOperands(*Inst32);
3907  copyFlagsToImplicitVCC(*Inst32, *Src2);
3908  }
3909  }
3910 
3911  return Inst32;
3912 }
3913 
3915  const MachineOperand &MO,
3916  const MCOperandInfo &OpInfo) const {
3917  // Literal constants use the constant bus.
3918  if (!MO.isReg())
3919  return !isInlineConstant(MO, OpInfo);
3920 
3921  if (!MO.isUse())
3922  return false;
3923 
3924  if (MO.getReg().isVirtual())
3925  return RI.isSGPRClass(MRI.getRegClass(MO.getReg()));
3926 
3927  // Null is free
3928  if (MO.getReg() == AMDGPU::SGPR_NULL || MO.getReg() == AMDGPU::SGPR_NULL64)
3929  return false;
3930 
3931  // SGPRs use the constant bus
3932  if (MO.isImplicit()) {
3933  return MO.getReg() == AMDGPU::M0 ||
3934  MO.getReg() == AMDGPU::VCC ||
3935  MO.getReg() == AMDGPU::VCC_LO;
3936  } else {
3937  return AMDGPU::SReg_32RegClass.contains(MO.getReg()) ||
3938  AMDGPU::SReg_64RegClass.contains(MO.getReg());
3939  }
3940 }
3941 
3943  for (const MachineOperand &MO : MI.implicit_operands()) {
3944  // We only care about reads.
3945  if (MO.isDef())
3946  continue;
3947 
3948  switch (MO.getReg()) {
3949  case AMDGPU::VCC:
3950  case AMDGPU::VCC_LO:
3951  case AMDGPU::VCC_HI:
3952  case AMDGPU::M0:
3953  case AMDGPU::FLAT_SCR:
3954  return MO.getReg();
3955 
3956  default:
3957  break;
3958  }
3959  }
3960 
3961  return Register();
3962 }
3963 
3964 static bool shouldReadExec(const MachineInstr &MI) {
3965  if (SIInstrInfo::isVALU(MI)) {
3966  switch (MI.getOpcode()) {
3967  case AMDGPU::V_READLANE_B32:
3968  case AMDGPU::V_WRITELANE_B32:
3969  return false;
3970  }
3971 
3972  return true;
3973  }
3974 
3975  if (MI.isPreISelOpcode() ||
3976  SIInstrInfo::isGenericOpcode(MI.getOpcode()) ||
3979  return false;
3980 
3981  return true;
3982 }
3983 
3984 static bool isSubRegOf(const SIRegisterInfo &TRI,
3985  const MachineOperand &SuperVec,
3986  const MachineOperand &SubReg) {
3987  if (SubReg.getReg().isPhysical())
3988  return TRI.isSubRegister(SuperVec.getReg(), SubReg.getReg());
3989 
3990  return SubReg.getSubReg() != AMDGPU::NoSubRegister &&
3991  SubReg.getReg() == SuperVec.getReg();
3992 }
3993 
3995  StringRef &ErrInfo) const {
3996  uint16_t Opcode = MI.getOpcode();
3997  if (SIInstrInfo::isGenericOpcode(MI.getOpcode()))
3998  return true;
3999 
4000  const MachineFunction *MF = MI.getParent()->getParent();
4001  const MachineRegisterInfo &MRI = MF->getRegInfo();
4002 
4003  int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0);
4004  int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src1);
4005  int Src2Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src2);
4006  int Src3Idx = -1;
4007  if (Src0Idx == -1) {
4008  // VOPD V_DUAL_* instructions use different operand names.
4009  Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0X);
4010  Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vsrc1X);
4011  Src2Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0Y);
4012  Src3Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vsrc1Y);
4013  }
4014 
4015  // Make sure the number of operands is correct.
4016  const MCInstrDesc &Desc = get(Opcode);
4017  if (!Desc.isVariadic() &&
4018  Desc.getNumOperands() != MI.getNumExplicitOperands()) {
4019  ErrInfo = "Instruction has wrong number of operands.";
4020  return false;
4021  }
4022 
4023  if (MI.isInlineAsm()) {
4024  // Verify register classes for inlineasm constraints.
4025  for (unsigned I = InlineAsm::MIOp_FirstOperand, E = MI.getNumOperands();
4026  I != E; ++I) {
4027  const TargetRegisterClass *RC = MI.getRegClassConstraint(I, this, &RI);
4028  if (!RC)
4029  continue;
4030 
4031  const MachineOperand &Op = MI.getOperand(I);
4032  if (!Op.isReg())
4033  continue;
4034 
4035  Register Reg = Op.getReg();
4036  if (!Reg.isVirtual() && !RC->contains(Reg)) {
4037  ErrInfo = "inlineasm operand has incorrect register class.";
4038  return false;
4039  }
4040  }
4041 
4042  return true;
4043  }
4044 
4045  if (isMIMG(MI) && MI.memoperands_empty() && MI.mayLoadOrStore()) {
4046  ErrInfo = "missing memory operand from MIMG instruction.";
4047  return false;
4048  }
4049 
4050  // Make sure the register classes are correct.
4051  for (int i = 0, e = Desc.getNumOperands(); i != e; ++i) {
4052  const MachineOperand &MO = MI.getOperand(i);
4053  if (MO.isFPImm()) {
4054  ErrInfo = "FPImm Machine Operands are not supported. ISel should bitcast "
4055  "all fp values to integers.";
4056  return false;
4057  }
4058 
4059  int RegClass = Desc.OpInfo[i].RegClass;
4060 
4061  switch (Desc.OpInfo[i].OperandType) {
4063  if (MI.getOperand(i).isImm() || MI.getOperand(i).isGlobal()) {
4064  ErrInfo = "Illegal immediate value for operand.";
4065  return false;
4066  }
4067  break;
4072  break;
4084  if (!MO.isReg() && (!MO.isImm() || !isInlineConstant(MI, i))) {
4085  ErrInfo = "Illegal immediate value for operand.";
4086  return false;
4087  }
4088  break;
4089  }
4092  // Check if this operand is an immediate.
4093  // FrameIndex operands will be replaced by immediates, so they are
4094  // allowed.
4095  if (!MI.getOperand(i).isImm() && !MI.getOperand(i).isFI()) {
4096  ErrInfo = "Expected immediate, but got non-immediate";
4097  return false;
4098  }
4099  [[fallthrough]];
4100  default:
4101  continue;
4102  }
4103 
4104  if (!MO.isReg())
4105  continue;
4106  Register Reg = MO.getReg();
4107  if (!Reg)
4108  continue;
4109 
4110  // FIXME: Ideally we would have separate instruction definitions with the
4111  // aligned register constraint.
4112  // FIXME: We do not verify inline asm operands, but custom inline asm
4113  // verification is broken anyway
4114  if (ST.needsAlignedVGPRs()) {
4115  const TargetRegisterClass *RC = RI.getRegClassForReg(MRI, Reg);
4116  if (RI.hasVectorRegisters(RC) && MO.getSubReg()) {
4117  const TargetRegisterClass *SubRC =
4118  RI.getSubRegisterClass(RC, MO.getSubReg());
4119  RC = RI.getCompatibleSubRegClass(RC, SubRC, MO.getSubReg());
4120  if (RC)
4121  RC = SubRC;
4122  }
4123 
4124  // Check that this is the aligned version of the class.
4125  if (!RC || !RI.isProperlyAlignedRC(*RC)) {
4126  ErrInfo = "Subtarget requires even aligned vector registers";
4127  return false;
4128  }
4129  }
4130 
4131  if (RegClass != -1) {
4132  if (Reg.isVirtual())
4133  continue;
4134 
4135  const TargetRegisterClass *RC = RI.getRegClass(RegClass);
4136  if (!RC->contains(Reg)) {
4137  ErrInfo = "Operand has incorrect register class.";
4138  return false;
4139  }
4140  }
4141  }
4142 
4143  // Verify SDWA
4144  if (isSDWA(MI)) {
4145  if (!ST.hasSDWA()) {
4146  ErrInfo = "SDWA is not supported on this target";
4147  return false;
4148  }
4149 
4150  int DstIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdst);
4151 
4152  for (int OpIdx : {DstIdx, Src0Idx, Src1Idx, Src2Idx}) {
4153  if (OpIdx == -1)
4154  continue;
4155  const MachineOperand &MO = MI.getOperand(OpIdx);
4156 
4157  if (!ST.hasSDWAScalar()) {
4158  // Only VGPRS on VI
4159  if (!MO.isReg() || !RI.hasVGPRs(RI.getRegClassForReg(MRI, MO.getReg()))) {
4160  ErrInfo = "Only VGPRs allowed as operands in SDWA instructions on VI";
4161  return false;
4162  }
4163  } else {
4164  // No immediates on GFX9
4165  if (!MO.isReg()) {
4166  ErrInfo =
4167  "Only reg allowed as operands in SDWA instructions on GFX9+";
4168  return false;
4169  }
4170  }
4171  }
4172 
4173  if (!ST.hasSDWAOmod()) {
4174  // No omod allowed on VI
4175  const MachineOperand *OMod = getNamedOperand(MI, AMDGPU::OpName::omod);
4176  if (OMod != nullptr &&
4177  (!OMod->isImm() || OMod->getImm() != 0)) {
4178  ErrInfo = "OMod not allowed in SDWA instructions on VI";
4179  return false;
4180  }
4181  }
4182 
4183  uint16_t BasicOpcode = AMDGPU::getBasicFromSDWAOp(Opcode);
4184  if (isVOPC(BasicOpcode)) {
4185  if (!ST.hasSDWASdst() && DstIdx != -1) {
4186  // Only vcc allowed as dst on VI for VOPC
4187  const MachineOperand &Dst = MI.getOperand(DstIdx);
4188  if (!Dst.isReg() || Dst.getReg() != AMDGPU::VCC) {
4189  ErrInfo = "Only VCC allowed as dst in SDWA instructions on VI";
4190  return false;
4191  }
4192  } else if (!ST.hasSDWAOutModsVOPC()) {
4193  // No clamp allowed on GFX9 for VOPC
4194  const MachineOperand *Clamp = getNamedOperand(MI, AMDGPU::OpName::clamp);
4195  if (Clamp && (!Clamp->isImm() || Clamp->getImm() != 0)) {
4196  ErrInfo = "Clamp not allowed in VOPC SDWA instructions on VI";
4197  return false;
4198  }
4199 
4200  // No omod allowed on GFX9 for VOPC
4201  const MachineOperand *OMod = getNamedOperand(MI, AMDGPU::OpName::omod);
4202  if (OMod && (!OMod->isImm() || OMod->getImm() != 0)) {
4203  ErrInfo = "OMod not allowed in VOPC SDWA instructions on VI";
4204  return false;
4205  }
4206  }
4207  }
4208 
4209  const MachineOperand *DstUnused = getNamedOperand(MI, AMDGPU::OpName::dst_unused);
4210  if (DstUnused && DstUnused->isImm() &&
4212  const MachineOperand &Dst = MI.getOperand(DstIdx);
4213  if (!Dst.isReg() || !Dst.isTied()) {
4214  ErrInfo = "Dst register should have tied register";
4215  return false;
4216  }
4217 
4218  const MachineOperand &TiedMO =
4219  MI.getOperand(MI.findTiedOperandIdx(DstIdx));
4220  if (!TiedMO.isReg() || !TiedMO.isImplicit() || !TiedMO.isUse()) {
4221  ErrInfo =
4222  "Dst register should be tied to implicit use of preserved register";
4223  return false;
4224  } else if (TiedMO.getReg().isPhysical() &&
4225  Dst.getReg() != TiedMO.getReg()) {
4226  ErrInfo = "Dst register should use same physical register as preserved";
4227  return false;
4228  }
4229  }
4230  }
4231 
4232  // Verify MIMG
4233  if (isMIMG(MI.getOpcode()) && !MI.mayStore()) {
4234  // Ensure that the return type used is large enough for all the options
4235  // being used TFE/LWE require an extra result register.
4236  const MachineOperand *DMask = getNamedOperand(MI, AMDGPU::OpName::dmask);
4237  if (DMask) {
4238  uint64_t DMaskImm = DMask->getImm();
4239  uint32_t RegCount =
4240  isGather4(MI.getOpcode()) ? 4 : countPopulation(DMaskImm);
4241  const MachineOperand *TFE = getNamedOperand(MI, AMDGPU::OpName::tfe);
4242  const MachineOperand *LWE = getNamedOperand(MI, AMDGPU::OpName::lwe);
4243  const MachineOperand *D16 = getNamedOperand(MI, AMDGPU::OpName::d16);
4244 
4245  // Adjust for packed 16 bit values
4246  if (D16 && D16->getImm() && !ST.hasUnpackedD16VMem())
4247  RegCount >>= 1;
4248 
4249  // Adjust if using LWE or TFE
4250  if ((LWE && LWE->getImm()) || (TFE && TFE->getImm()))
4251  RegCount += 1;
4252 
4253  const uint32_t DstIdx =
4254  AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::vdata);
4255  const MachineOperand &Dst = MI.getOperand(DstIdx);
4256  if (Dst.isReg()) {
4257  const TargetRegisterClass *DstRC = getOpRegClass(MI, DstIdx);
4258  uint32_t DstSize = RI.getRegSizeInBits(*DstRC) / 32;
4259  if (RegCount > DstSize) {
4260  ErrInfo = "MIMG instruction returns too many registers for dst "
4261  "register class";
4262  return false;
4263  }
4264  }
4265  }
4266  }
4267 
4268  // Verify VOP*. Ignore multiple sgpr operands on writelane.
4269  if (isVALU(MI) && Desc.getOpcode() != AMDGPU::V_WRITELANE_B32) {
4270  unsigned ConstantBusCount = 0;
4271  bool UsesLiteral = false;
4272  const MachineOperand *LiteralVal = nullptr;
4273 
4274  int ImmIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::imm);
4275  if (ImmIdx != -1) {
4276  ++ConstantBusCount;
4277  UsesLiteral = true;
4278  LiteralVal = &MI.getOperand(ImmIdx);
4279  }
4280 
4281  SmallVector<Register, 2> SGPRsUsed;
4282  Register SGPRUsed;
4283 
4284  // Only look at the true operands. Only a real operand can use the constant
4285  // bus, and we don't want to check pseudo-operands like the source modifier
4286  // flags.
4287  for (int OpIdx : {Src0Idx, Src1Idx, Src2Idx, Src3Idx}) {
4288  if (OpIdx == -1)
4289  continue;
4290  const MachineOperand &MO = MI.getOperand(OpIdx);
4291  if (usesConstantBus(MRI, MO, MI.getDesc().OpInfo[OpIdx])) {
4292  if (MO.isReg()) {
4293  SGPRUsed = MO.getReg();
4294  if (!llvm::is_contained(SGPRsUsed, SGPRUsed)) {
4295  ++ConstantBusCount;
4296  SGPRsUsed.push_back(SGPRUsed);
4297  }
4298  } else {
4299  if (!UsesLiteral) {
4300  ++ConstantBusCount;
4301  UsesLiteral = true;
4302  LiteralVal = &MO;
4303  } else if (!MO.isIdenticalTo(*LiteralVal)) {
4304  assert(isVOP2(MI) || isVOP3(MI));
4305  ErrInfo = "VOP2/VOP3 instruction uses more than one literal";
4306  return false;
4307  }
4308  }
4309  }
4310  }
4311 
4312  SGPRUsed = findImplicitSGPRRead(MI);
4313  if (SGPRUsed) {
4314  // Implicit uses may safely overlap true operands
4315  if (llvm::all_of(SGPRsUsed, [this, SGPRUsed](unsigned SGPR) {
4316  return !RI.regsOverlap(SGPRUsed, SGPR);
4317  })) {
4318  ++ConstantBusCount;
4319  SGPRsUsed.push_back(SGPRUsed);
4320  }
4321  }
4322 
4323  // v_writelane_b32 is an exception from constant bus restriction:
4324  // vsrc0 can be sgpr, const or m0 and lane select sgpr, m0 or inline-const
4325  if (ConstantBusCount > ST.getConstantBusLimit(Opcode) &&
4326  Opcode != AMDGPU::V_WRITELANE_B32) {
4327  ErrInfo = "VOP* instruction violates constant bus restriction";
4328  return false;
4329  }
4330 
4331  if (isVOP3(MI) && UsesLiteral && !ST.hasVOP3Literal()) {
4332  ErrInfo = "VOP3 instruction uses literal";
4333  return false;
4334  }
4335  }
4336 
4337  // Special case for writelane - this can break the multiple constant bus rule,
4338  // but still can't use more than one SGPR register
4339  if (Desc.getOpcode() == AMDGPU::V_WRITELANE_B32) {
4340  unsigned SGPRCount = 0;
4341  Register SGPRUsed;
4342 
4343  for (int OpIdx : {Src0Idx, Src1Idx}) {
4344  if (OpIdx == -1)
4345  break;
4346 
4347  const MachineOperand &MO = MI.getOperand(OpIdx);
4348 
4349  if (usesConstantBus(MRI, MO, MI.getDesc().OpInfo[OpIdx])) {
4350  if (MO.isReg() && MO.getReg() != AMDGPU::M0) {
4351  if (MO.getReg() != SGPRUsed)
4352  ++SGPRCount;
4353  SGPRUsed = MO.getReg();
4354  }
4355  }
4356  if (SGPRCount > ST.getConstantBusLimit(Opcode)) {
4357  ErrInfo = "WRITELANE instruction violates constant bus restriction";
4358  return false;
4359  }
4360  }
4361  }
4362 
4363  // Verify misc. restrictions on specific instructions.
4364  if (Desc.getOpcode() == AMDGPU::V_DIV_SCALE_F32_e64 ||
4365  Desc.getOpcode() == AMDGPU::V_DIV_SCALE_F64_e64) {
4366  const MachineOperand &Src0 = MI.getOperand(Src0Idx);
4367  const MachineOperand &Src1 = MI.getOperand(Src1Idx);
4368  const MachineOperand &Src2 = MI.getOperand(Src2Idx);
4369  if (Src0.isReg() && Src1.isReg() && Src2.isReg()) {
4370  if (!compareMachineOp(Src0, Src1) &&
4371  !compareMachineOp(Src0, Src2)) {
4372  ErrInfo = "v_div_scale_{f32|f64} require src0 = src1 or src2";
4373  return false;
4374  }
4375  }
4376  if ((getNamedOperand(MI, AMDGPU::OpName::src0_modifiers)->getImm() &
4377  SISrcMods::ABS) ||
4378  (getNamedOperand(MI, AMDGPU::OpName::src1_modifiers)->getImm() &
4379  SISrcMods::ABS) ||
4380  (getNamedOperand(MI, AMDGPU::OpName::src2_modifiers)->getImm() &
4381  SISrcMods::ABS)) {
4382  ErrInfo = "ABS not allowed in VOP3B instructions";
4383  return false;
4384  }
4385  }
4386 
4387  if (isSOP2(MI) || isSOPC(MI)) {
4388  const MachineOperand &Src0 = MI.getOperand(Src0Idx);
4389  const MachineOperand &Src1 = MI.getOperand(Src1Idx);
4390 
4391  if (!Src0.isReg() && !Src1.isReg() &&
4392  !isInlineConstant(Src0, Desc.OpInfo[Src0Idx]) &&
4393  !isInlineConstant(Src1, Desc.OpInfo[Src1Idx]) &&
4394  !Src0.isIdenticalTo(Src1)) {
4395  ErrInfo = "SOP2/SOPC instruction requires too many immediate constants";
4396  return false;
4397  }
4398  }
4399 
4400  if (isSOPK(MI)) {
4401  auto Op = getNamedOperand(MI, AMDGPU::OpName::simm16);
4402  if (Desc.isBranch()) {
4403  if (!Op->isMBB()) {
4404  ErrInfo = "invalid branch target for SOPK instruction";
4405  return false;
4406  }
4407  } else {
4408  uint64_t Imm = Op->getImm();
4409  if (sopkIsZext(MI)) {
4410  if (!isUInt<16>(Imm)) {
4411  ErrInfo = "invalid immediate for SOPK instruction";
4412  return false;
4413  }
4414  } else {
4415  if (!isInt<16>(Imm)) {
4416  ErrInfo = "invalid immediate for SOPK instruction";
4417  return false;
4418  }
4419  }
4420  }
4421  }
4422 
4423  if (Desc.getOpcode() == AMDGPU::V_MOVRELS_B32_e32 ||
4424  Desc.getOpcode() == AMDGPU::V_MOVRELS_B32_e64 ||
4425  Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e32 ||
4426  Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e64) {
4427  const bool IsDst = Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e32 ||
4428  Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e64;
4429 
4430  const unsigned StaticNumOps = Desc.getNumOperands() +
4431  Desc.getNumImplicitUses();
4432  const unsigned NumImplicitOps = IsDst ? 2 : 1;
4433 
4434  // Allow additional implicit operands. This allows a fixup done by the post
4435  // RA scheduler where the main implicit operand is killed and implicit-defs
4436  // are added for sub-registers that remain live after this instruction.
4437  if (MI.getNumOperands() < StaticNumOps + NumImplicitOps) {
4438  ErrInfo = "missing implicit register operands";
4439  return false;
4440  }
4441 
4442  const MachineOperand *Dst = getNamedOperand(MI, AMDGPU::OpName::vdst);
4443  if (IsDst) {
4444  if (!Dst->isUse()) {
4445  ErrInfo = "v_movreld_b32 vdst should be a use operand";
4446  return false;
4447  }
4448 
4449  unsigned UseOpIdx;
4450  if (!MI.isRegTiedToUseOperand(StaticNumOps, &UseOpIdx) ||
4451  UseOpIdx != StaticNumOps + 1) {
4452  ErrInfo = "movrel implicit operands should be tied";
4453  return false;
4454  }
4455  }
4456 
4457  const MachineOperand &Src0 = MI.getOperand(Src0Idx);
4458  const MachineOperand &ImpUse
4459  = MI.getOperand(StaticNumOps + NumImplicitOps - 1);
4460  if (!ImpUse.isReg() || !ImpUse.isUse() ||
4461  !isSubRegOf(RI, ImpUse, IsDst ? *Dst : Src0)) {
4462  ErrInfo = "src0 should be subreg of implicit vector use";
4463  return false;
4464  }
4465  }
4466 
4467  // Make sure we aren't losing exec uses in the td files. This mostly requires
4468  // being careful when using let Uses to try to add other use registers.
4469  if (shouldReadExec(MI)) {
4470  if (!MI.hasRegisterImplicitUseOperand(AMDGPU::EXEC)) {
4471  ErrInfo = "VALU instruction does not implicitly read exec mask";
4472  return false;
4473  }
4474  }
4475 
4476  if (isSMRD(MI)) {
4477  if (MI.mayStore() &&
4479  // The register offset form of scalar stores may only use m0 as the
4480  // soffset register.
4481  const MachineOperand *Soff = getNamedOperand(MI, AMDGPU::OpName::soffset);
4482  if (Soff && Soff->getReg() != AMDGPU::M0) {
4483  ErrInfo = "scalar stores must use m0 as offset register";
4484  return false;
4485  }
4486  }
4487  }
4488 
4489  if (isFLAT(MI) && !ST.hasFlatInstOffsets()) {
4490  const MachineOperand *Offset = getNamedOperand(MI, AMDGPU::OpName::offset);
4491  if (Offset->getImm() != 0) {
4492  ErrInfo = "subtarget does not support offsets in flat instructions";
4493  return false;
4494  }
4495  }
4496 
4497  if (isMIMG(MI)) {
4498  const MachineOperand *DimOp = getNamedOperand(MI, AMDGPU::OpName::dim);
4499  if (DimOp) {
4500  int VAddr0Idx = AMDGPU::getNamedOperandIdx(Opcode,
4501  AMDGPU::OpName::vaddr0);
4502  int SRsrcIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::srsrc);
4503  const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(Opcode);
4504  const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode =
4505  AMDGPU::getMIMGBaseOpcodeInfo(Info->BaseOpcode);
4506  const AMDGPU::MIMGDimInfo *Dim =
4508 
4509  if (!Dim) {
4510  ErrInfo = "dim is out of range";
4511  return false;
4512  }
4513 
4514  bool IsA16 = false;
4515  if (ST.hasR128A16()) {
4516  const MachineOperand *R128A16 = getNamedOperand(MI, AMDGPU::OpName::r128);
4517  IsA16 = R128A16->getImm() != 0;
4518  } else if (ST.hasGFX10A16()) {
4519  const MachineOperand *A16 = getNamedOperand(MI, AMDGPU::OpName::a16);
4520  IsA16 = A16->getImm() != 0;
4521  }
4522 
4523  bool IsNSA = SRsrcIdx - VAddr0Idx > 1;
4524 
4525  unsigned AddrWords =
4526  AMDGPU::getAddrSizeMIMGOp(BaseOpcode, Dim, IsA16, ST.hasG16());
4527 
4528  unsigned VAddrWords;
4529  if (IsNSA) {
4530  VAddrWords = SRsrcIdx - VAddr0Idx;
4531  } else {
4532  const TargetRegisterClass *RC = getOpRegClass(MI, VAddr0Idx);
4533  VAddrWords = MRI.getTargetRegisterInfo()->getRegSizeInBits(*RC) / 32;
4534  if (AddrWords > 8)
4535  AddrWords = 16;
4536  }
4537 
4538  if (VAddrWords != AddrWords) {
4539  LLVM_DEBUG(dbgs() << "bad vaddr size, expected " << AddrWords
4540  << " but got " << VAddrWords << "\n");
4541  ErrInfo = "bad vaddr size";
4542  return false;
4543  }
4544  }
4545  }
4546 
4547  const MachineOperand *DppCt = getNamedOperand(MI, AMDGPU::OpName::dpp_ctrl);
4548  if (DppCt) {
4549  using namespace AMDGPU::DPP;
4550 
4551  unsigned DC = DppCt->getImm();
4552  if (DC == DppCtrl::DPP_UNUSED1 || DC == DppCtrl::DPP_UNUSED2 ||
4553  DC == DppCtrl::DPP_UNUSED3 || DC > DppCtrl::DPP_LAST ||
4559  ErrInfo = "Invalid dpp_ctrl value";
4560  return false;
4561  }
4562  if (DC >= DppCtrl::WAVE_SHL1 && DC <= DppCtrl::WAVE_ROR1 &&
4563  ST.getGeneration() >= AMDGPUSubtarget::GFX10) {
4564  ErrInfo = "Invalid dpp_ctrl value: "
4565  "wavefront shifts are not supported on GFX10+";
4566  return false;
4567  }
4568  if (DC >= DppCtrl::BCAST15 && DC <= DppCtrl::BCAST31 &&
4569  ST.getGeneration() >= AMDGPUSubtarget::GFX10) {
4570  ErrInfo = "Invalid dpp_ctrl value: "
4571  "broadcasts are not supported on GFX10+";
4572  return false;
4573  }
4575  ST.getGeneration() < AMDGPUSubtarget::GFX10) {
4576  if (DC >= DppCtrl::ROW_NEWBCAST_FIRST &&
4578  !ST.hasGFX90AInsts()) {
4579  ErrInfo = "Invalid dpp_ctrl value: "
4580  "row_newbroadcast/row_share is not supported before "
4581  "GFX90A/GFX10";
4582  return false;
4583  } else if (DC > DppCtrl::ROW_NEWBCAST_LAST || !ST.hasGFX90AInsts()) {
4584  ErrInfo = "Invalid dpp_ctrl value: "
4585  "row_share and row_xmask are not supported before GFX10";
4586  return false;
4587  }
4588  }
4589 
4590  int DstIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdst);
4591 
4592  if (Opcode != AMDGPU::V_MOV_B64_DPP_PSEUDO &&
4593  ((DstIdx >= 0 &&
4594  (Desc.OpInfo[DstIdx].RegClass == AMDGPU::VReg_64RegClassID ||
4595  Desc.OpInfo[DstIdx].RegClass == AMDGPU::VReg_64_Align2RegClassID)) ||
4596  ((Src0Idx >= 0 &&
4597  (Desc.OpInfo[Src0Idx].RegClass == AMDGPU::VReg_64RegClassID ||
4598  Desc.OpInfo[Src0Idx].RegClass ==
4599  AMDGPU::VReg_64_Align2RegClassID)))) &&
4601  ErrInfo = "Invalid dpp_ctrl value: "
4602  "64 bit dpp only support row_newbcast";
4603  return false;
4604  }
4605  }
4606 
4607  if ((MI.mayStore() || MI.mayLoad()) && !isVGPRSpill(MI)) {
4608  const MachineOperand *Dst = getNamedOperand(MI, AMDGPU::OpName::vdst);
4609  uint16_t DataNameIdx = isDS(Opcode) ? AMDGPU::OpName::data0
4610  : AMDGPU::OpName::vdata;
4611  const MachineOperand *Data = getNamedOperand(MI, DataNameIdx);
4612  const MachineOperand *Data2 = getNamedOperand(MI, AMDGPU::OpName::data1);
4613  if (Data && !Data->isReg())
4614  Data = nullptr;
4615 
4616  if (ST.hasGFX90AInsts()) {
4617  if (Dst && Data &&
4618  (RI.isAGPR(MRI, Dst->getReg()) != RI.isAGPR(MRI, Data->getReg()))) {
4619  ErrInfo = "Invalid register class: "
4620  "vdata and vdst should be both VGPR or AGPR";
4621  return false;
4622  }
4623  if (Data && Data2 &&
4624  (RI.isAGPR(MRI, Data->getReg()) != RI.isAGPR(MRI, Data2->getReg()))) {
4625  ErrInfo = "Invalid register class: "
4626  "both data operands should be VGPR or AGPR";
4627  return false;
4628  }
4629  } else {
4630  if ((Dst && RI.isAGPR(MRI, Dst->getReg())) ||
4631  (Data && RI.isAGPR(MRI, Data->getReg())) ||
4632  (Data2 && RI.isAGPR(MRI, Data2->getReg()))) {
4633  ErrInfo = "Invalid register class: "
4634  "agpr loads and stores not supported on this GPU";
4635  return false;
4636  }
4637  }
4638  }
4639 
4640  if (ST.needsAlignedVGPRs()) {
4641  const auto isAlignedReg = [&MI, &MRI, this](unsigned OpName) -> bool {
4643  if (!Op)
4644  return true;
4645  Register Reg = Op->getReg();
4646  if (Reg.isPhysical())
4647  return !(RI.getHWRegIndex(Reg) & 1);
4648  const TargetRegisterClass &RC = *MRI.getRegClass(Reg);
4649  return RI.getRegSizeInBits(RC) > 32 && RI.isProperlyAlignedRC(RC) &&
4650  !(RI.getChannelFromSubReg(Op->getSubReg()) & 1);
4651  };
4652 
4653  if (MI.getOpcode() == AMDGPU::DS_GWS_INIT ||
4654  MI.getOpcode() == AMDGPU::DS_GWS_SEMA_BR ||
4655  MI.getOpcode() == AMDGPU::DS_GWS_BARRIER) {
4656 
4657  if (!isAlignedReg(AMDGPU::OpName::data0)) {
4658  ErrInfo = "Subtarget requires even aligned vector registers "
4659  "for DS_GWS instructions";
4660  return false;
4661  }
4662  }
4663 
4664  if (isMIMG(MI)) {
4665  if (!isAlignedReg(AMDGPU::OpName::vaddr)) {
4666  ErrInfo = "Subtarget requires even aligned vector registers "
4667  "for vaddr operand of image instructions";
4668  return false;
4669  }
4670  }
4671  }
4672 
4673  if (MI.getOpcode() == AMDGPU::V_ACCVGPR_WRITE_B32_e64 &&
4674  !ST.hasGFX90AInsts()) {
4675  const MachineOperand *Src = getNamedOperand(MI, AMDGPU::OpName::src0);
4676  if (Src->isReg() && RI.isSGPRReg(MRI, Src->getReg())) {
4677  ErrInfo = "Invalid register class: "
4678  "v_accvgpr_write with an SGPR is not supported on this GPU";
4679  return false;
4680  }
4681  }
4682 
4683  if (Desc.getOpcode() == AMDGPU::G_AMDGPU_WAVE_ADDRESS) {
4684  const MachineOperand &SrcOp = MI.getOperand(1);
4685  if (!SrcOp.isReg() || SrcOp.getReg().isVirtual()) {
4686  ErrInfo = "pseudo expects only physical SGPRs";
4687  return false;
4688  }
4689  }
4690 
4691  return true;
4692 }
4693 
4694 unsigned SIInstrInfo::getVALUOp(const MachineInstr &MI) const {
4695  switch (MI.getOpcode()) {
4696  default: return AMDGPU::INSTRUCTION_LIST_END;
4697  case AMDGPU::REG_SEQUENCE: return AMDGPU::REG_SEQUENCE;
4698  case AMDGPU::COPY: return AMDGPU::COPY;
4699  case AMDGPU::PHI: return AMDGPU::PHI;
4700  case AMDGPU::INSERT_SUBREG: return AMDGPU::INSERT_SUBREG;
4701  case AMDGPU::WQM: return AMDGPU::WQM;
4702  case AMDGPU::SOFT_WQM: return AMDGPU::SOFT_WQM;
4703  case AMDGPU::STRICT_WWM: return AMDGPU::STRICT_WWM;
4704  case AMDGPU::STRICT_WQM: return AMDGPU::STRICT_WQM;
4705  case AMDGPU::S_MOV_B32: {
4706  const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
4707  return MI.getOperand(1).isReg() ||
4708  RI.isAGPR(MRI, MI.getOperand(0).getReg()) ?
4709  AMDGPU::COPY : AMDGPU::V_MOV_B32_e32;
4710  }
4711  case AMDGPU::S_ADD_I32:
4712  return ST.hasAddNoCarry() ? AMDGPU::V_ADD_U32_e64 : AMDGPU::V_ADD_CO_U32_e32;
4713  case AMDGPU::S_ADDC_U32:
4714  return AMDGPU::V_ADDC_U32_e32;
4715  case AMDGPU::S_SUB_I32:
4716  return ST.hasAddNoCarry() ? AMDGPU::V_SUB_U32_e64 : AMDGPU::V_SUB_CO_U32_e32;
4717  // FIXME: These are not consistently handled, and selected when the carry is
4718  // used.
4719  case AMDGPU::S_ADD_U32:
4720  return AMDGPU::V_ADD_CO_U32_e32;
4721  case AMDGPU::S_SUB_U32:
4722  return AMDGPU::V_SUB_CO_U32_e32;
4723  case AMDGPU::S_SUBB_U32: return AMDGPU::V_SUBB_U32_e32;
4724  case AMDGPU::S_MUL_I32: return AMDGPU::V_MUL_LO_U32_e64;
4725  case AMDGPU::S_MUL_HI_U32: return AMDGPU::V_MUL_HI_U32_e64;
4726  case AMDGPU::S_MUL_HI_I32: return AMDGPU::V_MUL_HI_I32_e64;
4727  case AMDGPU::S_AND_B32: return AMDGPU::V_AND_B32_e64;
4728  case AMDGPU::S_OR_B32: return AMDGPU::V_OR_B32_e64;
4729  case AMDGPU::S_XOR_B32: return AMDGPU::V_XOR_B32_e64;
4730  case AMDGPU::S_XNOR_B32:
4731  return ST.hasDLInsts() ? AMDGPU::V_XNOR_B32_e64 : AMDGPU::INSTRUCTION_LIST_END;
4732  case AMDGPU::S_MIN_I32: return AMDGPU::V_MIN_I32_e64;
4733  case AMDGPU::S_MIN_U32: return AMDGPU::V_MIN_U32_e64;
4734  case AMDGPU::S_MAX_I32: return AMDGPU::V_MAX_I32_e64;
4735  case AMDGPU::S_MAX_U32: return AMDGPU::V_MAX_U32_e64;
4736  case AMDGPU::S_ASHR_I32: return AMDGPU::V_ASHR_I32_e32;
4737  case AMDGPU::S_ASHR_I64: return AMDGPU::V_ASHR_I64_e64;
4738  case AMDGPU::S_LSHL_B32: return AMDGPU::V_LSHL_B32_e32;
4739  case AMDGPU::S_LSHL_B64: return AMDGPU::V_LSHL_B64_e64;
4740  case AMDGPU::S_LSHR_B32: return AMDGPU::V_LSHR_B32_e32;
4741  case AMDGPU::S_LSHR_B64: return AMDGPU::V_LSHR_B64_e64;
4742  case AMDGPU::S_SEXT_I32_I8: return AMDGPU::V_BFE_I32_e64;
4743  case AMDGPU::S_SEXT_I32_I16: return AMDGPU::V_BFE_I32_e64;
4744  case AMDGPU::S_BFE_U32: return AMDGPU::V_BFE_U32_e64;
4745  case AMDGPU::S_BFE_I32: return AMDGPU::V_BFE_I32_e64;
4746  case AMDGPU::S_BFM_B32: return AMDGPU::V_BFM_B32_e64;
4747  case AMDGPU::S_BREV_B32: return AMDGPU::V_BFREV_B32_e32;
4748  case AMDGPU::S_NOT_B32: return AMDGPU::V_NOT_B32_e32;
4749  case AMDGPU::S_NOT_B64: return AMDGPU::V_NOT_B32_e32;
4750  case AMDGPU::S_CMP_EQ_I32: return AMDGPU::V_CMP_EQ_I32_e64;
4751  case AMDGPU::S_CMP_LG_I32: return AMDGPU::V_CMP_NE_I32_e64;
4752  case AMDGPU::S_CMP_GT_I32: return AMDGPU::V_CMP_GT_I32_e64;
4753  case AMDGPU::S_CMP_GE_I32: return AMDGPU::V_CMP_GE_I32_e64;
4754  case AMDGPU::S_CMP_LT_I32: return AMDGPU::V_CMP_LT_I32_e64;
4755  case AMDGPU::S_CMP_LE_I32: return AMDGPU::V_CMP_LE_I32_e64;
4756  case AMDGPU::S_CMP_EQ_U32: return AMDGPU::V_CMP_EQ_U32_e64;
4757  case AMDGPU::S_CMP_LG_U32: return AMDGPU::V_CMP_NE_U32_e64;
4758  case AMDGPU::S_CMP_GT_U32: return AMDGPU::V_CMP_GT_U32_e64;
4759  case AMDGPU::S_CMP_GE_U32: return AMDGPU::V_CMP_GE_U32_e64;
4760  case AMDGPU::S_CMP_LT_U32: return AMDGPU::V_CMP_LT_U32_e64;
4761  case AMDGPU::S_CMP_LE_U32: return AMDGPU::V_CMP_LE_U32_e64;
4762  case AMDGPU::S_CMP_EQ_U64: return AMDGPU::V_CMP_EQ_U64_e64;
4763  case AMDGPU::S_CMP_LG_U64: return AMDGPU::V_CMP_NE_U64_e64;
4764  case AMDGPU::S_BCNT1_I32_B32: return AMDGPU::V_BCNT_U32_B32_e64;
4765  case AMDGPU::S_FF1_I32_B32: return AMDGPU::V_FFBL_B32_e32;
4766  case AMDGPU::S_FLBIT_I32_B32: return AMDGPU::V_FFBH_U32_e32;
4767  case AMDGPU::S_FLBIT_I32: return AMDGPU::V_FFBH_I32_e64;
4768  case AMDGPU::S_CBRANCH_SCC0: return AMDGPU::S_CBRANCH_VCCZ;
4769  case AMDGPU::S_CBRANCH_SCC1: return AMDGPU::S_CBRANCH_VCCNZ;
4770  }
4772  "Unexpected scalar opcode without corresponding vector one!");
4773 }
4774 
4775 static const TargetRegisterClass *
4777  const MachineRegisterInfo &MRI,
4778  const MCInstrDesc &TID, unsigned RCID,
4779  bool IsAllocatable) {
4780  if ((IsAllocatable || !ST.hasGFX90AInsts() || !MRI.reservedRegsFrozen()) &&
4781  (((TID.mayLoad() || TID.mayStore()) &&
4782  !(TID.TSFlags & SIInstrFlags::VGPRSpill)) ||
4784  switch (RCID) {
4785  case AMDGPU::AV_32RegClassID:
4786  RCID = AMDGPU::VGPR_32RegClassID;
4787  break;
4788  case AMDGPU::AV_64RegClassID:
4789  RCID = AMDGPU::VReg_64RegClassID;
4790  break;
4791  case AMDGPU::AV_96RegClassID:
4792  RCID = AMDGPU::VReg_96RegClassID;
4793  break;
4794  case AMDGPU::AV_128RegClassID:
4795  RCID = AMDGPU::VReg_128RegClassID;
4796  break;
4797  case AMDGPU::AV_160RegClassID:
4798  RCID = AMDGPU::VReg_160RegClassID;
4799  break;
4800  case AMDGPU::AV_512RegClassID:
4801  RCID = AMDGPU::VReg_512RegClassID;
4802  break;
4803  default:
4804  break;
4805  }
4806  }
4807 
4808  return RI.getProperlyAlignedRC(RI.getRegClass(RCID));
4809 }
4810 
4812  unsigned OpNum, const TargetRegisterInfo *TRI,
4813  const MachineFunction &MF)
4814  const {
4815  if (OpNum >= TID.getNumOperands())
4816  return nullptr;
4817  auto RegClass = TID.OpInfo[OpNum].RegClass;
4818  bool IsAllocatable = false;
4820  // vdst and vdata should be both VGPR or AGPR, same for the DS instructions
4821  // with two data operands. Request register class constrained to VGPR only
4822  // of both operands present as Machine Copy Propagation can not check this
4823  // constraint and possibly other passes too.
4824  //
4825  // The check is limited to FLAT and DS because atomics in non-flat encoding
4826  // have their vdst and vdata tied to be the same register.
4827  const int VDstIdx = AMDGPU::getNamedOperandIdx(TID.Opcode,
4828  AMDGPU::OpName::vdst);
4829  const int DataIdx = AMDGPU::getNamedOperandIdx(TID.Opcode,
4830  (TID.TSFlags & SIInstrFlags::DS) ? AMDGPU::OpName::data0
4831  : AMDGPU::OpName::vdata);
4832  if (DataIdx != -1) {
4833  IsAllocatable = VDstIdx != -1 || AMDGPU::hasNamedOperand(
4834  TID.Opcode, AMDGPU::OpName::data1);
4835  }
4836  }
<