LLVM  10.0.0svn
SIInstrInfo.cpp
Go to the documentation of this file.
1 //===- SIInstrInfo.cpp - SI Instruction Information ----------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 /// \file
10 /// SI Implementation of TargetInstrInfo.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "SIInstrInfo.h"
15 #include "AMDGPU.h"
16 #include "AMDGPUSubtarget.h"
17 #include "GCNHazardRecognizer.h"
18 #include "SIDefines.h"
19 #include "SIMachineFunctionInfo.h"
20 #include "SIRegisterInfo.h"
22 #include "Utils/AMDGPUBaseInfo.h"
23 #include "llvm/ADT/APInt.h"
24 #include "llvm/ADT/ArrayRef.h"
25 #include "llvm/ADT/SmallVector.h"
26 #include "llvm/ADT/StringRef.h"
46 #include "llvm/IR/DebugLoc.h"
47 #include "llvm/IR/DiagnosticInfo.h"
48 #include "llvm/IR/Function.h"
49 #include "llvm/IR/InlineAsm.h"
50 #include "llvm/IR/LLVMContext.h"
51 #include "llvm/MC/MCInstrDesc.h"
52 #include "llvm/Support/Casting.h"
54 #include "llvm/Support/Compiler.h"
59 #include <cassert>
60 #include <cstdint>
61 #include <iterator>
62 #include <utility>
63 
64 using namespace llvm;
65 
66 #define GET_INSTRINFO_CTOR_DTOR
67 #include "AMDGPUGenInstrInfo.inc"
68 
69 namespace llvm {
70 namespace AMDGPU {
71 #define GET_D16ImageDimIntrinsics_IMPL
72 #define GET_ImageDimIntrinsicTable_IMPL
73 #define GET_RsrcIntrinsics_IMPL
74 #include "AMDGPUGenSearchableTables.inc"
75 }
76 }
77 
78 
79 // Must be at least 4 to be able to branch over minimum unconditional branch
80 // code. This is only for making it possible to write reasonably small tests for
81 // long branches.
82 static cl::opt<unsigned>
83 BranchOffsetBits("amdgpu-s-branch-bits", cl::ReallyHidden, cl::init(16),
84  cl::desc("Restrict range of branch instructions (DEBUG)"));
85 
87  : AMDGPUGenInstrInfo(AMDGPU::ADJCALLSTACKUP, AMDGPU::ADJCALLSTACKDOWN),
88  RI(ST), ST(ST) {}
89 
90 //===----------------------------------------------------------------------===//
91 // TargetInstrInfo callbacks
92 //===----------------------------------------------------------------------===//
93 
94 static unsigned getNumOperandsNoGlue(SDNode *Node) {
95  unsigned N = Node->getNumOperands();
96  while (N && Node->getOperand(N - 1).getValueType() == MVT::Glue)
97  --N;
98  return N;
99 }
100 
101 /// Returns true if both nodes have the same value for the given
102 /// operand \p Op, or if both nodes do not have this operand.
103 static bool nodesHaveSameOperandValue(SDNode *N0, SDNode* N1, unsigned OpName) {
104  unsigned Opc0 = N0->getMachineOpcode();
105  unsigned Opc1 = N1->getMachineOpcode();
106 
107  int Op0Idx = AMDGPU::getNamedOperandIdx(Opc0, OpName);
108  int Op1Idx = AMDGPU::getNamedOperandIdx(Opc1, OpName);
109 
110  if (Op0Idx == -1 && Op1Idx == -1)
111  return true;
112 
113 
114  if ((Op0Idx == -1 && Op1Idx != -1) ||
115  (Op1Idx == -1 && Op0Idx != -1))
116  return false;
117 
118  // getNamedOperandIdx returns the index for the MachineInstr's operands,
119  // which includes the result as the first operand. We are indexing into the
120  // MachineSDNode's operands, so we need to skip the result operand to get
121  // the real index.
122  --Op0Idx;
123  --Op1Idx;
124 
125  return N0->getOperand(Op0Idx) == N1->getOperand(Op1Idx);
126 }
127 
129  AliasAnalysis *AA) const {
130  // TODO: The generic check fails for VALU instructions that should be
131  // rematerializable due to implicit reads of exec. We really want all of the
132  // generic logic for this except for this.
133  switch (MI.getOpcode()) {
134  case AMDGPU::V_MOV_B32_e32:
135  case AMDGPU::V_MOV_B32_e64:
136  case AMDGPU::V_MOV_B64_PSEUDO:
137  // No implicit operands.
138  return MI.getNumOperands() == MI.getDesc().getNumOperands();
139  default:
140  return false;
141  }
142 }
143 
145  int64_t &Offset0,
146  int64_t &Offset1) const {
147  if (!Load0->isMachineOpcode() || !Load1->isMachineOpcode())
148  return false;
149 
150  unsigned Opc0 = Load0->getMachineOpcode();
151  unsigned Opc1 = Load1->getMachineOpcode();
152 
153  // Make sure both are actually loads.
154  if (!get(Opc0).mayLoad() || !get(Opc1).mayLoad())
155  return false;
156 
157  if (isDS(Opc0) && isDS(Opc1)) {
158 
159  // FIXME: Handle this case:
160  if (getNumOperandsNoGlue(Load0) != getNumOperandsNoGlue(Load1))
161  return false;
162 
163  // Check base reg.
164  if (Load0->getOperand(0) != Load1->getOperand(0))
165  return false;
166 
167  // Skip read2 / write2 variants for simplicity.
168  // TODO: We should report true if the used offsets are adjacent (excluded
169  // st64 versions).
170  int Offset0Idx = AMDGPU::getNamedOperandIdx(Opc0, AMDGPU::OpName::offset);
171  int Offset1Idx = AMDGPU::getNamedOperandIdx(Opc1, AMDGPU::OpName::offset);
172  if (Offset0Idx == -1 || Offset1Idx == -1)
173  return false;
174 
175  // XXX - be careful of datalesss loads
176  // getNamedOperandIdx returns the index for MachineInstrs. Since they
177  // include the output in the operand list, but SDNodes don't, we need to
178  // subtract the index by one.
179  Offset0Idx -= get(Opc0).NumDefs;
180  Offset1Idx -= get(Opc1).NumDefs;
181  Offset0 = cast<ConstantSDNode>(Load0->getOperand(Offset0Idx))->getZExtValue();
182  Offset1 = cast<ConstantSDNode>(Load1->getOperand(Offset1Idx))->getZExtValue();
183  return true;
184  }
185 
186  if (isSMRD(Opc0) && isSMRD(Opc1)) {
187  // Skip time and cache invalidation instructions.
188  if (AMDGPU::getNamedOperandIdx(Opc0, AMDGPU::OpName::sbase) == -1 ||
189  AMDGPU::getNamedOperandIdx(Opc1, AMDGPU::OpName::sbase) == -1)
190  return false;
191 
193 
194  // Check base reg.
195  if (Load0->getOperand(0) != Load1->getOperand(0))
196  return false;
197 
198  const ConstantSDNode *Load0Offset =
199  dyn_cast<ConstantSDNode>(Load0->getOperand(1));
200  const ConstantSDNode *Load1Offset =
201  dyn_cast<ConstantSDNode>(Load1->getOperand(1));
202 
203  if (!Load0Offset || !Load1Offset)
204  return false;
205 
206  Offset0 = Load0Offset->getZExtValue();
207  Offset1 = Load1Offset->getZExtValue();
208  return true;
209  }
210 
211  // MUBUF and MTBUF can access the same addresses.
212  if ((isMUBUF(Opc0) || isMTBUF(Opc0)) && (isMUBUF(Opc1) || isMTBUF(Opc1))) {
213 
214  // MUBUF and MTBUF have vaddr at different indices.
215  if (!nodesHaveSameOperandValue(Load0, Load1, AMDGPU::OpName::soffset) ||
216  !nodesHaveSameOperandValue(Load0, Load1, AMDGPU::OpName::vaddr) ||
217  !nodesHaveSameOperandValue(Load0, Load1, AMDGPU::OpName::srsrc))
218  return false;
219 
220  int OffIdx0 = AMDGPU::getNamedOperandIdx(Opc0, AMDGPU::OpName::offset);
221  int OffIdx1 = AMDGPU::getNamedOperandIdx(Opc1, AMDGPU::OpName::offset);
222 
223  if (OffIdx0 == -1 || OffIdx1 == -1)
224  return false;
225 
226  // getNamedOperandIdx returns the index for MachineInstrs. Since they
227  // include the output in the operand list, but SDNodes don't, we need to
228  // subtract the index by one.
229  OffIdx0 -= get(Opc0).NumDefs;
230  OffIdx1 -= get(Opc1).NumDefs;
231 
232  SDValue Off0 = Load0->getOperand(OffIdx0);
233  SDValue Off1 = Load1->getOperand(OffIdx1);
234 
235  // The offset might be a FrameIndexSDNode.
236  if (!isa<ConstantSDNode>(Off0) || !isa<ConstantSDNode>(Off1))
237  return false;
238 
239  Offset0 = cast<ConstantSDNode>(Off0)->getZExtValue();
240  Offset1 = cast<ConstantSDNode>(Off1)->getZExtValue();
241  return true;
242  }
243 
244  return false;
245 }
246 
247 static bool isStride64(unsigned Opc) {
248  switch (Opc) {
249  case AMDGPU::DS_READ2ST64_B32:
250  case AMDGPU::DS_READ2ST64_B64:
251  case AMDGPU::DS_WRITE2ST64_B32:
252  case AMDGPU::DS_WRITE2ST64_B64:
253  return true;
254  default:
255  return false;
256  }
257 }
258 
260  const MachineOperand *&BaseOp,
261  int64_t &Offset,
262  const TargetRegisterInfo *TRI) const {
263  unsigned Opc = LdSt.getOpcode();
264 
265  if (isDS(LdSt)) {
266  const MachineOperand *OffsetImm =
267  getNamedOperand(LdSt, AMDGPU::OpName::offset);
268  if (OffsetImm) {
269  // Normal, single offset LDS instruction.
270  BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::addr);
271  // TODO: ds_consume/ds_append use M0 for the base address. Is it safe to
272  // report that here?
273  if (!BaseOp)
274  return false;
275 
276  Offset = OffsetImm->getImm();
277  assert(BaseOp->isReg() && "getMemOperandWithOffset only supports base "
278  "operands of type register.");
279  return true;
280  }
281 
282  // The 2 offset instructions use offset0 and offset1 instead. We can treat
283  // these as a load with a single offset if the 2 offsets are consecutive. We
284  // will use this for some partially aligned loads.
285  const MachineOperand *Offset0Imm =
286  getNamedOperand(LdSt, AMDGPU::OpName::offset0);
287  const MachineOperand *Offset1Imm =
288  getNamedOperand(LdSt, AMDGPU::OpName::offset1);
289 
290  uint8_t Offset0 = Offset0Imm->getImm();
291  uint8_t Offset1 = Offset1Imm->getImm();
292 
293  if (Offset1 > Offset0 && Offset1 - Offset0 == 1) {
294  // Each of these offsets is in element sized units, so we need to convert
295  // to bytes of the individual reads.
296 
297  unsigned EltSize;
298  if (LdSt.mayLoad())
299  EltSize = TRI->getRegSizeInBits(*getOpRegClass(LdSt, 0)) / 16;
300  else {
301  assert(LdSt.mayStore());
302  int Data0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data0);
303  EltSize = TRI->getRegSizeInBits(*getOpRegClass(LdSt, Data0Idx)) / 8;
304  }
305 
306  if (isStride64(Opc))
307  EltSize *= 64;
308 
309  BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::addr);
310  Offset = EltSize * Offset0;
311  assert(BaseOp->isReg() && "getMemOperandWithOffset only supports base "
312  "operands of type register.");
313  return true;
314  }
315 
316  return false;
317  }
318 
319  if (isMUBUF(LdSt) || isMTBUF(LdSt)) {
320  const MachineOperand *SOffset = getNamedOperand(LdSt, AMDGPU::OpName::soffset);
321  if (SOffset && SOffset->isReg()) {
322  // We can only handle this if it's a stack access, as any other resource
323  // would require reporting multiple base registers.
324  const MachineOperand *AddrReg = getNamedOperand(LdSt, AMDGPU::OpName::vaddr);
325  if (AddrReg && !AddrReg->isFI())
326  return false;
327 
328  const MachineOperand *RSrc = getNamedOperand(LdSt, AMDGPU::OpName::srsrc);
329  const SIMachineFunctionInfo *MFI
331  if (RSrc->getReg() != MFI->getScratchRSrcReg())
332  return false;
333 
334  const MachineOperand *OffsetImm =
335  getNamedOperand(LdSt, AMDGPU::OpName::offset);
336  BaseOp = SOffset;
337  Offset = OffsetImm->getImm();
338  return true;
339  }
340 
341  const MachineOperand *AddrReg = getNamedOperand(LdSt, AMDGPU::OpName::vaddr);
342  if (!AddrReg)
343  return false;
344 
345  const MachineOperand *OffsetImm =
346  getNamedOperand(LdSt, AMDGPU::OpName::offset);
347  BaseOp = AddrReg;
348  Offset = OffsetImm->getImm();
349 
350  if (SOffset) // soffset can be an inline immediate.
351  Offset += SOffset->getImm();
352 
353  assert(BaseOp->isReg() && "getMemOperandWithOffset only supports base "
354  "operands of type register.");
355  return true;
356  }
357 
358  if (isSMRD(LdSt)) {
359  const MachineOperand *OffsetImm =
360  getNamedOperand(LdSt, AMDGPU::OpName::offset);
361  if (!OffsetImm)
362  return false;
363 
364  const MachineOperand *SBaseReg = getNamedOperand(LdSt, AMDGPU::OpName::sbase);
365  BaseOp = SBaseReg;
366  Offset = OffsetImm->getImm();
367  assert(BaseOp->isReg() && "getMemOperandWithOffset only supports base "
368  "operands of type register.");
369  return true;
370  }
371 
372  if (isFLAT(LdSt)) {
373  const MachineOperand *VAddr = getNamedOperand(LdSt, AMDGPU::OpName::vaddr);
374  if (VAddr) {
375  // Can't analyze 2 offsets.
376  if (getNamedOperand(LdSt, AMDGPU::OpName::saddr))
377  return false;
378 
379  BaseOp = VAddr;
380  } else {
381  // scratch instructions have either vaddr or saddr.
382  BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::saddr);
383  }
384 
385  Offset = getNamedOperand(LdSt, AMDGPU::OpName::offset)->getImm();
386  assert(BaseOp->isReg() && "getMemOperandWithOffset only supports base "
387  "operands of type register.");
388  return true;
389  }
390 
391  return false;
392 }
393 
394 static bool memOpsHaveSameBasePtr(const MachineInstr &MI1,
395  const MachineOperand &BaseOp1,
396  const MachineInstr &MI2,
397  const MachineOperand &BaseOp2) {
398  // Support only base operands with base registers.
399  // Note: this could be extended to support FI operands.
400  if (!BaseOp1.isReg() || !BaseOp2.isReg())
401  return false;
402 
403  if (BaseOp1.isIdenticalTo(BaseOp2))
404  return true;
405 
406  if (!MI1.hasOneMemOperand() || !MI2.hasOneMemOperand())
407  return false;
408 
409  auto MO1 = *MI1.memoperands_begin();
410  auto MO2 = *MI2.memoperands_begin();
411  if (MO1->getAddrSpace() != MO2->getAddrSpace())
412  return false;
413 
414  auto Base1 = MO1->getValue();
415  auto Base2 = MO2->getValue();
416  if (!Base1 || !Base2)
417  return false;
418  const MachineFunction &MF = *MI1.getParent()->getParent();
419  const DataLayout &DL = MF.getFunction().getParent()->getDataLayout();
420  Base1 = GetUnderlyingObject(Base1, DL);
421  Base2 = GetUnderlyingObject(Base1, DL);
422 
423  if (isa<UndefValue>(Base1) || isa<UndefValue>(Base2))
424  return false;
425 
426  return Base1 == Base2;
427 }
428 
430  const MachineOperand &BaseOp2,
431  unsigned NumLoads) const {
432  const MachineInstr &FirstLdSt = *BaseOp1.getParent();
433  const MachineInstr &SecondLdSt = *BaseOp2.getParent();
434 
435  if (!memOpsHaveSameBasePtr(FirstLdSt, BaseOp1, SecondLdSt, BaseOp2))
436  return false;
437 
438  const MachineOperand *FirstDst = nullptr;
439  const MachineOperand *SecondDst = nullptr;
440 
441  if ((isMUBUF(FirstLdSt) && isMUBUF(SecondLdSt)) ||
442  (isMTBUF(FirstLdSt) && isMTBUF(SecondLdSt)) ||
443  (isFLAT(FirstLdSt) && isFLAT(SecondLdSt))) {
444  const unsigned MaxGlobalLoadCluster = 6;
445  if (NumLoads > MaxGlobalLoadCluster)
446  return false;
447 
448  FirstDst = getNamedOperand(FirstLdSt, AMDGPU::OpName::vdata);
449  if (!FirstDst)
450  FirstDst = getNamedOperand(FirstLdSt, AMDGPU::OpName::vdst);
451  SecondDst = getNamedOperand(SecondLdSt, AMDGPU::OpName::vdata);
452  if (!SecondDst)
453  SecondDst = getNamedOperand(SecondLdSt, AMDGPU::OpName::vdst);
454  } else if (isSMRD(FirstLdSt) && isSMRD(SecondLdSt)) {
455  FirstDst = getNamedOperand(FirstLdSt, AMDGPU::OpName::sdst);
456  SecondDst = getNamedOperand(SecondLdSt, AMDGPU::OpName::sdst);
457  } else if (isDS(FirstLdSt) && isDS(SecondLdSt)) {
458  FirstDst = getNamedOperand(FirstLdSt, AMDGPU::OpName::vdst);
459  SecondDst = getNamedOperand(SecondLdSt, AMDGPU::OpName::vdst);
460  }
461 
462  if (!FirstDst || !SecondDst)
463  return false;
464 
465  // Try to limit clustering based on the total number of bytes loaded
466  // rather than the number of instructions. This is done to help reduce
467  // register pressure. The method used is somewhat inexact, though,
468  // because it assumes that all loads in the cluster will load the
469  // same number of bytes as FirstLdSt.
470 
471  // The unit of this value is bytes.
472  // FIXME: This needs finer tuning.
473  unsigned LoadClusterThreshold = 16;
474 
475  const MachineRegisterInfo &MRI =
476  FirstLdSt.getParent()->getParent()->getRegInfo();
477 
478  const Register Reg = FirstDst->getReg();
479 
481  ? MRI.getRegClass(Reg)
482  : RI.getPhysRegClass(Reg);
483 
484  return (NumLoads * (RI.getRegSizeInBits(*DstRC) / 8)) <= LoadClusterThreshold;
485 }
486 
487 // FIXME: This behaves strangely. If, for example, you have 32 load + stores,
488 // the first 16 loads will be interleaved with the stores, and the next 16 will
489 // be clustered as expected. It should really split into 2 16 store batches.
490 //
491 // Loads are clustered until this returns false, rather than trying to schedule
492 // groups of stores. This also means we have to deal with saying different
493 // address space loads should be clustered, and ones which might cause bank
494 // conflicts.
495 //
496 // This might be deprecated so it might not be worth that much effort to fix.
498  int64_t Offset0, int64_t Offset1,
499  unsigned NumLoads) const {
500  assert(Offset1 > Offset0 &&
501  "Second offset should be larger than first offset!");
502  // If we have less than 16 loads in a row, and the offsets are within 64
503  // bytes, then schedule together.
504 
505  // A cacheline is 64 bytes (for global memory).
506  return (NumLoads <= 16 && (Offset1 - Offset0) < 64);
507 }
508 
511  const DebugLoc &DL, unsigned DestReg,
512  unsigned SrcReg, bool KillSrc) {
513  MachineFunction *MF = MBB.getParent();
514  DiagnosticInfoUnsupported IllegalCopy(MF->getFunction(),
515  "illegal SGPR to VGPR copy",
516  DL, DS_Error);
517  LLVMContext &C = MF->getFunction().getContext();
518  C.diagnose(IllegalCopy);
519 
520  BuildMI(MBB, MI, DL, TII->get(AMDGPU::SI_ILLEGAL_COPY), DestReg)
521  .addReg(SrcReg, getKillRegState(KillSrc));
522 }
523 
526  const DebugLoc &DL, unsigned DestReg,
527  unsigned SrcReg, bool KillSrc) const {
528  const TargetRegisterClass *RC = RI.getPhysRegClass(DestReg);
529 
530  if (RC == &AMDGPU::VGPR_32RegClass) {
531  assert(AMDGPU::VGPR_32RegClass.contains(SrcReg) ||
532  AMDGPU::SReg_32RegClass.contains(SrcReg) ||
533  AMDGPU::AGPR_32RegClass.contains(SrcReg));
534  unsigned Opc = AMDGPU::AGPR_32RegClass.contains(SrcReg) ?
535  AMDGPU::V_ACCVGPR_READ_B32 : AMDGPU::V_MOV_B32_e32;
536  BuildMI(MBB, MI, DL, get(Opc), DestReg)
537  .addReg(SrcReg, getKillRegState(KillSrc));
538  return;
539  }
540 
541  if (RC == &AMDGPU::SReg_32_XM0RegClass ||
542  RC == &AMDGPU::SReg_32RegClass) {
543  if (SrcReg == AMDGPU::SCC) {
544  BuildMI(MBB, MI, DL, get(AMDGPU::S_CSELECT_B32), DestReg)
545  .addImm(-1)
546  .addImm(0);
547  return;
548  }
549 
550  if (DestReg == AMDGPU::VCC_LO) {
551  if (AMDGPU::SReg_32RegClass.contains(SrcReg)) {
552  BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), AMDGPU::VCC_LO)
553  .addReg(SrcReg, getKillRegState(KillSrc));
554  } else {
555  // FIXME: Hack until VReg_1 removed.
556  assert(AMDGPU::VGPR_32RegClass.contains(SrcReg));
557  BuildMI(MBB, MI, DL, get(AMDGPU::V_CMP_NE_U32_e32))
558  .addImm(0)
559  .addReg(SrcReg, getKillRegState(KillSrc));
560  }
561 
562  return;
563  }
564 
565  if (!AMDGPU::SReg_32RegClass.contains(SrcReg)) {
566  reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc);
567  return;
568  }
569 
570  BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), DestReg)
571  .addReg(SrcReg, getKillRegState(KillSrc));
572  return;
573  }
574 
575  if (RC == &AMDGPU::SReg_64RegClass) {
576  if (DestReg == AMDGPU::VCC) {
577  if (AMDGPU::SReg_64RegClass.contains(SrcReg)) {
578  BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B64), AMDGPU::VCC)
579  .addReg(SrcReg, getKillRegState(KillSrc));
580  } else {
581  // FIXME: Hack until VReg_1 removed.
582  assert(AMDGPU::VGPR_32RegClass.contains(SrcReg));
583  BuildMI(MBB, MI, DL, get(AMDGPU::V_CMP_NE_U32_e32))
584  .addImm(0)
585  .addReg(SrcReg, getKillRegState(KillSrc));
586  }
587 
588  return;
589  }
590 
591  if (!AMDGPU::SReg_64RegClass.contains(SrcReg)) {
592  reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc);
593  return;
594  }
595 
596  BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B64), DestReg)
597  .addReg(SrcReg, getKillRegState(KillSrc));
598  return;
599  }
600 
601  if (DestReg == AMDGPU::SCC) {
602  assert(AMDGPU::SReg_32RegClass.contains(SrcReg));
603  BuildMI(MBB, MI, DL, get(AMDGPU::S_CMP_LG_U32))
604  .addReg(SrcReg, getKillRegState(KillSrc))
605  .addImm(0);
606  return;
607  }
608 
609  if (RC == &AMDGPU::AGPR_32RegClass) {
610  assert(AMDGPU::VGPR_32RegClass.contains(SrcReg) ||
611  AMDGPU::SReg_32RegClass.contains(SrcReg) ||
612  AMDGPU::AGPR_32RegClass.contains(SrcReg));
613  if (!AMDGPU::VGPR_32RegClass.contains(SrcReg)) {
614  // First try to find defining accvgpr_write to avoid temporary registers.
615  for (auto Def = MI, E = MBB.begin(); Def != E; ) {
616  --Def;
617  if (!Def->definesRegister(SrcReg, &RI))
618  continue;
619  if (Def->getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32)
620  break;
621 
622  MachineOperand &DefOp = Def->getOperand(1);
623  assert(DefOp.isReg() || DefOp.isImm());
624 
625  if (DefOp.isReg()) {
626  // Check that register source operand if not clobbered before MI.
627  // Immediate operands are always safe to propagate.
628  bool SafeToPropagate = true;
629  for (auto I = Def; I != MI && SafeToPropagate; ++I)
630  if (I->modifiesRegister(DefOp.getReg(), &RI))
631  SafeToPropagate = false;
632 
633  if (!SafeToPropagate)
634  break;
635 
636  DefOp.setIsKill(false);
637  }
638 
639  BuildMI(MBB, MI, DL, get(AMDGPU::V_ACCVGPR_WRITE_B32), DestReg)
640  .add(DefOp);
641  return;
642  }
643 
644  RegScavenger RS;
645  RS.enterBasicBlock(MBB);
646  RS.forward(MI);
647 
648  // Ideally we want to have three registers for a long reg_sequence copy
649  // to hide 2 waitstates between v_mov_b32 and accvgpr_write.
650  unsigned MaxVGPRs = RI.getRegPressureLimit(&AMDGPU::VGPR_32RegClass,
651  *MBB.getParent());
652 
653  // Registers in the sequence are allocated contiguously so we can just
654  // use register number to pick one of three round-robin temps.
655  unsigned RegNo = DestReg % 3;
656  unsigned Tmp = RS.scavengeRegister(&AMDGPU::VGPR_32RegClass, 0);
657  if (!Tmp)
658  report_fatal_error("Cannot scavenge VGPR to copy to AGPR");
659  RS.setRegUsed(Tmp);
660  // Only loop through if there are any free registers left, otherwise
661  // scavenger may report a fatal error without emergency spill slot
662  // or spill with the slot.
663  while (RegNo-- && RS.FindUnusedReg(&AMDGPU::VGPR_32RegClass)) {
664  unsigned Tmp2 = RS.scavengeRegister(&AMDGPU::VGPR_32RegClass, 0);
665  if (!Tmp2 || RI.getHWRegIndex(Tmp2) >= MaxVGPRs)
666  break;
667  Tmp = Tmp2;
668  RS.setRegUsed(Tmp);
669  }
670  copyPhysReg(MBB, MI, DL, Tmp, SrcReg, KillSrc);
671  BuildMI(MBB, MI, DL, get(AMDGPU::V_ACCVGPR_WRITE_B32), DestReg)
672  .addReg(Tmp, RegState::Kill);
673  return;
674  }
675 
676  BuildMI(MBB, MI, DL, get(AMDGPU::V_ACCVGPR_WRITE_B32), DestReg)
677  .addReg(SrcReg, getKillRegState(KillSrc));
678  return;
679  }
680 
681  unsigned EltSize = 4;
682  unsigned Opcode = AMDGPU::V_MOV_B32_e32;
683  if (RI.isSGPRClass(RC)) {
684  // TODO: Copy vec3/vec5 with s_mov_b64s then final s_mov_b32.
685  if (!(RI.getRegSizeInBits(*RC) % 64)) {
686  Opcode = AMDGPU::S_MOV_B64;
687  EltSize = 8;
688  } else {
689  Opcode = AMDGPU::S_MOV_B32;
690  EltSize = 4;
691  }
692 
693  if (!RI.isSGPRClass(RI.getPhysRegClass(SrcReg))) {
694  reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc);
695  return;
696  }
697  } else if (RI.hasAGPRs(RC)) {
698  Opcode = RI.hasVGPRs(RI.getPhysRegClass(SrcReg)) ?
699  AMDGPU::V_ACCVGPR_WRITE_B32 : AMDGPU::COPY;
700  } else if (RI.hasVGPRs(RC) && RI.hasAGPRs(RI.getPhysRegClass(SrcReg))) {
701  Opcode = AMDGPU::V_ACCVGPR_READ_B32;
702  }
703 
704  ArrayRef<int16_t> SubIndices = RI.getRegSplitParts(RC, EltSize);
705  bool Forward = RI.getHWRegIndex(DestReg) <= RI.getHWRegIndex(SrcReg);
706 
707  for (unsigned Idx = 0; Idx < SubIndices.size(); ++Idx) {
708  unsigned SubIdx;
709  if (Forward)
710  SubIdx = SubIndices[Idx];
711  else
712  SubIdx = SubIndices[SubIndices.size() - Idx - 1];
713 
714  if (Opcode == TargetOpcode::COPY) {
715  copyPhysReg(MBB, MI, DL, RI.getSubReg(DestReg, SubIdx),
716  RI.getSubReg(SrcReg, SubIdx), KillSrc);
717  continue;
718  }
719 
720  MachineInstrBuilder Builder = BuildMI(MBB, MI, DL,
721  get(Opcode), RI.getSubReg(DestReg, SubIdx));
722 
723  Builder.addReg(RI.getSubReg(SrcReg, SubIdx));
724 
725  if (Idx == 0)
726  Builder.addReg(DestReg, RegState::Define | RegState::Implicit);
727 
728  bool UseKill = KillSrc && Idx == SubIndices.size() - 1;
729  Builder.addReg(SrcReg, getKillRegState(UseKill) | RegState::Implicit);
730  }
731 }
732 
733 int SIInstrInfo::commuteOpcode(unsigned Opcode) const {
734  int NewOpc;
735 
736  // Try to map original to commuted opcode
737  NewOpc = AMDGPU::getCommuteRev(Opcode);
738  if (NewOpc != -1)
739  // Check if the commuted (REV) opcode exists on the target.
740  return pseudoToMCOpcode(NewOpc) != -1 ? NewOpc : -1;
741 
742  // Try to map commuted to original opcode
743  NewOpc = AMDGPU::getCommuteOrig(Opcode);
744  if (NewOpc != -1)
745  // Check if the original (non-REV) opcode exists on the target.
746  return pseudoToMCOpcode(NewOpc) != -1 ? NewOpc : -1;
747 
748  return Opcode;
749 }
750 
753  const DebugLoc &DL, unsigned DestReg,
754  int64_t Value) const {
756  const TargetRegisterClass *RegClass = MRI.getRegClass(DestReg);
757  if (RegClass == &AMDGPU::SReg_32RegClass ||
758  RegClass == &AMDGPU::SGPR_32RegClass ||
759  RegClass == &AMDGPU::SReg_32_XM0RegClass ||
760  RegClass == &AMDGPU::SReg_32_XM0_XEXECRegClass) {
761  BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), DestReg)
762  .addImm(Value);
763  return;
764  }
765 
766  if (RegClass == &AMDGPU::SReg_64RegClass ||
767  RegClass == &AMDGPU::SGPR_64RegClass ||
768  RegClass == &AMDGPU::SReg_64_XEXECRegClass) {
769  BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B64), DestReg)
770  .addImm(Value);
771  return;
772  }
773 
774  if (RegClass == &AMDGPU::VGPR_32RegClass) {
775  BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DestReg)
776  .addImm(Value);
777  return;
778  }
779  if (RegClass == &AMDGPU::VReg_64RegClass) {
780  BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B64_PSEUDO), DestReg)
781  .addImm(Value);
782  return;
783  }
784 
785  unsigned EltSize = 4;
786  unsigned Opcode = AMDGPU::V_MOV_B32_e32;
787  if (RI.isSGPRClass(RegClass)) {
788  if (RI.getRegSizeInBits(*RegClass) > 32) {
789  Opcode = AMDGPU::S_MOV_B64;
790  EltSize = 8;
791  } else {
792  Opcode = AMDGPU::S_MOV_B32;
793  EltSize = 4;
794  }
795  }
796 
797  ArrayRef<int16_t> SubIndices = RI.getRegSplitParts(RegClass, EltSize);
798  for (unsigned Idx = 0; Idx < SubIndices.size(); ++Idx) {
799  int64_t IdxValue = Idx == 0 ? Value : 0;
800 
801  MachineInstrBuilder Builder = BuildMI(MBB, MI, DL,
802  get(Opcode), RI.getSubReg(DestReg, Idx));
803  Builder.addImm(IdxValue);
804  }
805 }
806 
807 const TargetRegisterClass *
809  return &AMDGPU::VGPR_32RegClass;
810 }
811 
814  const DebugLoc &DL, unsigned DstReg,
816  unsigned TrueReg,
817  unsigned FalseReg) const {
819  MachineFunction *MF = MBB.getParent();
820  const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
821  const TargetRegisterClass *BoolXExecRC =
822  RI.getRegClass(AMDGPU::SReg_1_XEXECRegClassID);
823  assert(MRI.getRegClass(DstReg) == &AMDGPU::VGPR_32RegClass &&
824  "Not a VGPR32 reg");
825 
826  if (Cond.size() == 1) {
827  Register SReg = MRI.createVirtualRegister(BoolXExecRC);
828  BuildMI(MBB, I, DL, get(AMDGPU::COPY), SReg)
829  .add(Cond[0]);
830  BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
831  .addImm(0)
832  .addReg(FalseReg)
833  .addImm(0)
834  .addReg(TrueReg)
835  .addReg(SReg);
836  } else if (Cond.size() == 2) {
837  assert(Cond[0].isImm() && "Cond[0] is not an immediate");
838  switch (Cond[0].getImm()) {
839  case SIInstrInfo::SCC_TRUE: {
840  Register SReg = MRI.createVirtualRegister(BoolXExecRC);
841  BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_CSELECT_B32
842  : AMDGPU::S_CSELECT_B64), SReg)
843  .addImm(-1)
844  .addImm(0);
845  BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
846  .addImm(0)
847  .addReg(FalseReg)
848  .addImm(0)
849  .addReg(TrueReg)
850  .addReg(SReg);
851  break;
852  }
853  case SIInstrInfo::SCC_FALSE: {
854  Register SReg = MRI.createVirtualRegister(BoolXExecRC);
855  BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_CSELECT_B32
856  : AMDGPU::S_CSELECT_B64), SReg)
857  .addImm(0)
858  .addImm(-1);
859  BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
860  .addImm(0)
861  .addReg(FalseReg)
862  .addImm(0)
863  .addReg(TrueReg)
864  .addReg(SReg);
865  break;
866  }
867  case SIInstrInfo::VCCNZ: {
868  MachineOperand RegOp = Cond[1];
869  RegOp.setImplicit(false);
870  Register SReg = MRI.createVirtualRegister(BoolXExecRC);
871  BuildMI(MBB, I, DL, get(AMDGPU::COPY), SReg)
872  .add(RegOp);
873  BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
874  .addImm(0)
875  .addReg(FalseReg)
876  .addImm(0)
877  .addReg(TrueReg)
878  .addReg(SReg);
879  break;
880  }
881  case SIInstrInfo::VCCZ: {
882  MachineOperand RegOp = Cond[1];
883  RegOp.setImplicit(false);
884  Register SReg = MRI.createVirtualRegister(BoolXExecRC);
885  BuildMI(MBB, I, DL, get(AMDGPU::COPY), SReg)
886  .add(RegOp);
887  BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
888  .addImm(0)
889  .addReg(TrueReg)
890  .addImm(0)
891  .addReg(FalseReg)
892  .addReg(SReg);
893  break;
894  }
895  case SIInstrInfo::EXECNZ: {
896  Register SReg = MRI.createVirtualRegister(BoolXExecRC);
897  Register SReg2 = MRI.createVirtualRegister(RI.getBoolRC());
898  BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_OR_SAVEEXEC_B32
899  : AMDGPU::S_OR_SAVEEXEC_B64), SReg2)
900  .addImm(0);
901  BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_CSELECT_B32
902  : AMDGPU::S_CSELECT_B64), SReg)
903  .addImm(-1)
904  .addImm(0);
905  BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
906  .addImm(0)
907  .addReg(FalseReg)
908  .addImm(0)
909  .addReg(TrueReg)
910  .addReg(SReg);
911  break;
912  }
913  case SIInstrInfo::EXECZ: {
914  Register SReg = MRI.createVirtualRegister(BoolXExecRC);
915  Register SReg2 = MRI.createVirtualRegister(RI.getBoolRC());
916  BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_OR_SAVEEXEC_B32
917  : AMDGPU::S_OR_SAVEEXEC_B64), SReg2)
918  .addImm(0);
919  BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_CSELECT_B32
920  : AMDGPU::S_CSELECT_B64), SReg)
921  .addImm(0)
922  .addImm(-1);
923  BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
924  .addImm(0)
925  .addReg(FalseReg)
926  .addImm(0)
927  .addReg(TrueReg)
928  .addReg(SReg);
929  llvm_unreachable("Unhandled branch predicate EXECZ");
930  break;
931  }
932  default:
933  llvm_unreachable("invalid branch predicate");
934  }
935  } else {
936  llvm_unreachable("Can only handle Cond size 1 or 2");
937  }
938 }
939 
942  const DebugLoc &DL,
943  unsigned SrcReg, int Value) const {
946  BuildMI(*MBB, I, DL, get(AMDGPU::V_CMP_EQ_I32_e64), Reg)
947  .addImm(Value)
948  .addReg(SrcReg);
949 
950  return Reg;
951 }
952 
955  const DebugLoc &DL,
956  unsigned SrcReg, int Value) const {
959  BuildMI(*MBB, I, DL, get(AMDGPU::V_CMP_NE_I32_e64), Reg)
960  .addImm(Value)
961  .addReg(SrcReg);
962 
963  return Reg;
964 }
965 
966 unsigned SIInstrInfo::getMovOpcode(const TargetRegisterClass *DstRC) const {
967 
968  if (RI.hasAGPRs(DstRC))
969  return AMDGPU::COPY;
970  if (RI.getRegSizeInBits(*DstRC) == 32) {
971  return RI.isSGPRClass(DstRC) ? AMDGPU::S_MOV_B32 : AMDGPU::V_MOV_B32_e32;
972  } else if (RI.getRegSizeInBits(*DstRC) == 64 && RI.isSGPRClass(DstRC)) {
973  return AMDGPU::S_MOV_B64;
974  } else if (RI.getRegSizeInBits(*DstRC) == 64 && !RI.isSGPRClass(DstRC)) {
975  return AMDGPU::V_MOV_B64_PSEUDO;
976  }
977  return AMDGPU::COPY;
978 }
979 
980 static unsigned getSGPRSpillSaveOpcode(unsigned Size) {
981  switch (Size) {
982  case 4:
983  return AMDGPU::SI_SPILL_S32_SAVE;
984  case 8:
985  return AMDGPU::SI_SPILL_S64_SAVE;
986  case 12:
987  return AMDGPU::SI_SPILL_S96_SAVE;
988  case 16:
989  return AMDGPU::SI_SPILL_S128_SAVE;
990  case 20:
991  return AMDGPU::SI_SPILL_S160_SAVE;
992  case 32:
993  return AMDGPU::SI_SPILL_S256_SAVE;
994  case 64:
995  return AMDGPU::SI_SPILL_S512_SAVE;
996  case 128:
997  return AMDGPU::SI_SPILL_S1024_SAVE;
998  default:
999  llvm_unreachable("unknown register size");
1000  }
1001 }
1002 
1003 static unsigned getVGPRSpillSaveOpcode(unsigned Size) {
1004  switch (Size) {
1005  case 4:
1006  return AMDGPU::SI_SPILL_V32_SAVE;
1007  case 8:
1008  return AMDGPU::SI_SPILL_V64_SAVE;
1009  case 12:
1010  return AMDGPU::SI_SPILL_V96_SAVE;
1011  case 16:
1012  return AMDGPU::SI_SPILL_V128_SAVE;
1013  case 20:
1014  return AMDGPU::SI_SPILL_V160_SAVE;
1015  case 32:
1016  return AMDGPU::SI_SPILL_V256_SAVE;
1017  case 64:
1018  return AMDGPU::SI_SPILL_V512_SAVE;
1019  case 128:
1020  return AMDGPU::SI_SPILL_V1024_SAVE;
1021  default:
1022  llvm_unreachable("unknown register size");
1023  }
1024 }
1025 
1026 static unsigned getAGPRSpillSaveOpcode(unsigned Size) {
1027  switch (Size) {
1028  case 4:
1029  return AMDGPU::SI_SPILL_A32_SAVE;
1030  case 8:
1031  return AMDGPU::SI_SPILL_A64_SAVE;
1032  case 16:
1033  return AMDGPU::SI_SPILL_A128_SAVE;
1034  case 64:
1035  return AMDGPU::SI_SPILL_A512_SAVE;
1036  case 128:
1037  return AMDGPU::SI_SPILL_A1024_SAVE;
1038  default:
1039  llvm_unreachable("unknown register size");
1040  }
1041 }
1042 
1045  unsigned SrcReg, bool isKill,
1046  int FrameIndex,
1047  const TargetRegisterClass *RC,
1048  const TargetRegisterInfo *TRI) const {
1049  MachineFunction *MF = MBB.getParent();
1051  MachineFrameInfo &FrameInfo = MF->getFrameInfo();
1052  const DebugLoc &DL = MBB.findDebugLoc(MI);
1053 
1054  unsigned Size = FrameInfo.getObjectSize(FrameIndex);
1055  unsigned Align = FrameInfo.getObjectAlignment(FrameIndex);
1056  MachinePointerInfo PtrInfo
1057  = MachinePointerInfo::getFixedStack(*MF, FrameIndex);
1058  MachineMemOperand *MMO
1060  Size, Align);
1061  unsigned SpillSize = TRI->getSpillSize(*RC);
1062 
1063  if (RI.isSGPRClass(RC)) {
1064  MFI->setHasSpilledSGPRs();
1065 
1066  // We are only allowed to create one new instruction when spilling
1067  // registers, so we need to use pseudo instruction for spilling SGPRs.
1068  const MCInstrDesc &OpDesc = get(getSGPRSpillSaveOpcode(SpillSize));
1069 
1070  // The SGPR spill/restore instructions only work on number sgprs, so we need
1071  // to make sure we are using the correct register class.
1072  if (Register::isVirtualRegister(SrcReg) && SpillSize == 4) {
1074  MRI.constrainRegClass(SrcReg, &AMDGPU::SReg_32_XM0RegClass);
1075  }
1076 
1077  MachineInstrBuilder Spill = BuildMI(MBB, MI, DL, OpDesc)
1078  .addReg(SrcReg, getKillRegState(isKill)) // data
1079  .addFrameIndex(FrameIndex) // addr
1080  .addMemOperand(MMO)
1082  .addReg(MFI->getStackPtrOffsetReg(), RegState::Implicit);
1083  // Add the scratch resource registers as implicit uses because we may end up
1084  // needing them, and need to ensure that the reserved registers are
1085  // correctly handled.
1086  if (RI.spillSGPRToVGPR())
1087  FrameInfo.setStackID(FrameIndex, TargetStackID::SGPRSpill);
1088  if (ST.hasScalarStores()) {
1089  // m0 is used for offset to scalar stores if used to spill.
1090  Spill.addReg(AMDGPU::M0, RegState::ImplicitDefine | RegState::Dead);
1091  }
1092 
1093  return;
1094  }
1095 
1096  unsigned Opcode = RI.hasAGPRs(RC) ? getAGPRSpillSaveOpcode(SpillSize)
1097  : getVGPRSpillSaveOpcode(SpillSize);
1098  MFI->setHasSpilledVGPRs();
1099 
1100  auto MIB = BuildMI(MBB, MI, DL, get(Opcode));
1101  if (RI.hasAGPRs(RC)) {
1103  Register Tmp = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
1104  MIB.addReg(Tmp, RegState::Define);
1105  }
1106  MIB.addReg(SrcReg, getKillRegState(isKill)) // data
1107  .addFrameIndex(FrameIndex) // addr
1108  .addReg(MFI->getScratchRSrcReg()) // scratch_rsrc
1109  .addReg(MFI->getStackPtrOffsetReg()) // scratch_offset
1110  .addImm(0) // offset
1111  .addMemOperand(MMO);
1112 }
1113 
1114 static unsigned getSGPRSpillRestoreOpcode(unsigned Size) {
1115  switch (Size) {
1116  case 4:
1117  return AMDGPU::SI_SPILL_S32_RESTORE;
1118  case 8:
1119  return AMDGPU::SI_SPILL_S64_RESTORE;
1120  case 12:
1121  return AMDGPU::SI_SPILL_S96_RESTORE;
1122  case 16:
1123  return AMDGPU::SI_SPILL_S128_RESTORE;
1124  case 20:
1125  return AMDGPU::SI_SPILL_S160_RESTORE;
1126  case 32:
1127  return AMDGPU::SI_SPILL_S256_RESTORE;
1128  case 64:
1129  return AMDGPU::SI_SPILL_S512_RESTORE;
1130  case 128:
1131  return AMDGPU::SI_SPILL_S1024_RESTORE;
1132  default:
1133  llvm_unreachable("unknown register size");
1134  }
1135 }
1136 
1137 static unsigned getVGPRSpillRestoreOpcode(unsigned Size) {
1138  switch (Size) {
1139  case 4:
1140  return AMDGPU::SI_SPILL_V32_RESTORE;
1141  case 8:
1142  return AMDGPU::SI_SPILL_V64_RESTORE;
1143  case 12:
1144  return AMDGPU::SI_SPILL_V96_RESTORE;
1145  case 16:
1146  return AMDGPU::SI_SPILL_V128_RESTORE;
1147  case 20:
1148  return AMDGPU::SI_SPILL_V160_RESTORE;
1149  case 32:
1150  return AMDGPU::SI_SPILL_V256_RESTORE;
1151  case 64:
1152  return AMDGPU::SI_SPILL_V512_RESTORE;
1153  case 128:
1154  return AMDGPU::SI_SPILL_V1024_RESTORE;
1155  default:
1156  llvm_unreachable("unknown register size");
1157  }
1158 }
1159 
1160 static unsigned getAGPRSpillRestoreOpcode(unsigned Size) {
1161  switch (Size) {
1162  case 4:
1163  return AMDGPU::SI_SPILL_A32_RESTORE;
1164  case 8:
1165  return AMDGPU::SI_SPILL_A64_RESTORE;
1166  case 16:
1167  return AMDGPU::SI_SPILL_A128_RESTORE;
1168  case 64:
1169  return AMDGPU::SI_SPILL_A512_RESTORE;
1170  case 128:
1171  return AMDGPU::SI_SPILL_A1024_RESTORE;
1172  default:
1173  llvm_unreachable("unknown register size");
1174  }
1175 }
1176 
1179  unsigned DestReg, int FrameIndex,
1180  const TargetRegisterClass *RC,
1181  const TargetRegisterInfo *TRI) const {
1182  MachineFunction *MF = MBB.getParent();
1184  MachineFrameInfo &FrameInfo = MF->getFrameInfo();
1185  const DebugLoc &DL = MBB.findDebugLoc(MI);
1186  unsigned Align = FrameInfo.getObjectAlignment(FrameIndex);
1187  unsigned Size = FrameInfo.getObjectSize(FrameIndex);
1188  unsigned SpillSize = TRI->getSpillSize(*RC);
1189 
1190  MachinePointerInfo PtrInfo
1191  = MachinePointerInfo::getFixedStack(*MF, FrameIndex);
1192 
1194  PtrInfo, MachineMemOperand::MOLoad, Size, Align);
1195 
1196  if (RI.isSGPRClass(RC)) {
1197  MFI->setHasSpilledSGPRs();
1198 
1199  // FIXME: Maybe this should not include a memoperand because it will be
1200  // lowered to non-memory instructions.
1201  const MCInstrDesc &OpDesc = get(getSGPRSpillRestoreOpcode(SpillSize));
1202  if (Register::isVirtualRegister(DestReg) && SpillSize == 4) {
1204  MRI.constrainRegClass(DestReg, &AMDGPU::SReg_32_XM0RegClass);
1205  }
1206 
1207  if (RI.spillSGPRToVGPR())
1208  FrameInfo.setStackID(FrameIndex, TargetStackID::SGPRSpill);
1209  MachineInstrBuilder Spill = BuildMI(MBB, MI, DL, OpDesc, DestReg)
1210  .addFrameIndex(FrameIndex) // addr
1211  .addMemOperand(MMO)
1213  .addReg(MFI->getStackPtrOffsetReg(), RegState::Implicit);
1214 
1215  if (ST.hasScalarStores()) {
1216  // m0 is used for offset to scalar stores if used to spill.
1217  Spill.addReg(AMDGPU::M0, RegState::ImplicitDefine | RegState::Dead);
1218  }
1219 
1220  return;
1221  }
1222 
1223  unsigned Opcode = RI.hasAGPRs(RC) ? getAGPRSpillRestoreOpcode(SpillSize)
1224  : getVGPRSpillRestoreOpcode(SpillSize);
1225  auto MIB = BuildMI(MBB, MI, DL, get(Opcode), DestReg);
1226  if (RI.hasAGPRs(RC)) {
1228  Register Tmp = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
1229  MIB.addReg(Tmp, RegState::Define);
1230  }
1231  MIB.addFrameIndex(FrameIndex) // vaddr
1232  .addReg(MFI->getScratchRSrcReg()) // scratch_rsrc
1233  .addReg(MFI->getStackPtrOffsetReg()) // scratch_offset
1234  .addImm(0) // offset
1235  .addMemOperand(MMO);
1236 }
1237 
1238 /// \param @Offset Offset in bytes of the FrameIndex being spilled
1240  MachineBasicBlock &MBB, MachineInstr &MI, RegScavenger *RS, unsigned TmpReg,
1241  unsigned FrameOffset, unsigned Size) const {
1242  MachineFunction *MF = MBB.getParent();
1244  const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
1245  const DebugLoc &DL = MBB.findDebugLoc(MI);
1246  unsigned WorkGroupSize = MFI->getMaxFlatWorkGroupSize();
1247  unsigned WavefrontSize = ST.getWavefrontSize();
1248 
1249  unsigned TIDReg = MFI->getTIDReg();
1250  if (!MFI->hasCalculatedTID()) {
1251  MachineBasicBlock &Entry = MBB.getParent()->front();
1252  MachineBasicBlock::iterator Insert = Entry.front();
1253  const DebugLoc &DL = Insert->getDebugLoc();
1254 
1255  TIDReg = RI.findUnusedRegister(MF->getRegInfo(), &AMDGPU::VGPR_32RegClass,
1256  *MF);
1257  if (TIDReg == AMDGPU::NoRegister)
1258  return TIDReg;
1259 
1261  WorkGroupSize > WavefrontSize) {
1262  Register TIDIGXReg =
1264  Register TIDIGYReg =
1266  Register TIDIGZReg =
1268  Register InputPtrReg =
1270  for (unsigned Reg : {TIDIGXReg, TIDIGYReg, TIDIGZReg}) {
1271  if (!Entry.isLiveIn(Reg))
1272  Entry.addLiveIn(Reg);
1273  }
1274 
1275  RS->enterBasicBlock(Entry);
1276  // FIXME: Can we scavenge an SReg_64 and access the subregs?
1277  unsigned STmp0 = RS->scavengeRegister(&AMDGPU::SGPR_32RegClass, 0);
1278  unsigned STmp1 = RS->scavengeRegister(&AMDGPU::SGPR_32RegClass, 0);
1279  BuildMI(Entry, Insert, DL, get(AMDGPU::S_LOAD_DWORD_IMM), STmp0)
1280  .addReg(InputPtrReg)
1282  BuildMI(Entry, Insert, DL, get(AMDGPU::S_LOAD_DWORD_IMM), STmp1)
1283  .addReg(InputPtrReg)
1285 
1286  // NGROUPS.X * NGROUPS.Y
1287  BuildMI(Entry, Insert, DL, get(AMDGPU::S_MUL_I32), STmp1)
1288  .addReg(STmp1)
1289  .addReg(STmp0);
1290  // (NGROUPS.X * NGROUPS.Y) * TIDIG.X
1291  BuildMI(Entry, Insert, DL, get(AMDGPU::V_MUL_U32_U24_e32), TIDReg)
1292  .addReg(STmp1)
1293  .addReg(TIDIGXReg);
1294  // NGROUPS.Z * TIDIG.Y + (NGROUPS.X * NGROPUS.Y * TIDIG.X)
1295  BuildMI(Entry, Insert, DL, get(AMDGPU::V_MAD_U32_U24), TIDReg)
1296  .addReg(STmp0)
1297  .addReg(TIDIGYReg)
1298  .addReg(TIDReg);
1299  // (NGROUPS.Z * TIDIG.Y + (NGROUPS.X * NGROPUS.Y * TIDIG.X)) + TIDIG.Z
1300  getAddNoCarry(Entry, Insert, DL, TIDReg)
1301  .addReg(TIDReg)
1302  .addReg(TIDIGZReg)
1303  .addImm(0); // clamp bit
1304  } else {
1305  // Get the wave id
1306  BuildMI(Entry, Insert, DL, get(AMDGPU::V_MBCNT_LO_U32_B32_e64),
1307  TIDReg)
1308  .addImm(-1)
1309  .addImm(0);
1310 
1311  BuildMI(Entry, Insert, DL, get(AMDGPU::V_MBCNT_HI_U32_B32_e64),
1312  TIDReg)
1313  .addImm(-1)
1314  .addReg(TIDReg);
1315  }
1316 
1317  BuildMI(Entry, Insert, DL, get(AMDGPU::V_LSHLREV_B32_e32),
1318  TIDReg)
1319  .addImm(2)
1320  .addReg(TIDReg);
1321  MFI->setTIDReg(TIDReg);
1322  }
1323 
1324  // Add FrameIndex to LDS offset
1325  unsigned LDSOffset = MFI->getLDSSize() + (FrameOffset * WorkGroupSize);
1326  getAddNoCarry(MBB, MI, DL, TmpReg)
1327  .addImm(LDSOffset)
1328  .addReg(TIDReg)
1329  .addImm(0); // clamp bit
1330 
1331  return TmpReg;
1332 }
1333 
1336  int Count) const {
1337  DebugLoc DL = MBB.findDebugLoc(MI);
1338  while (Count > 0) {
1339  int Arg;
1340  if (Count >= 8)
1341  Arg = 7;
1342  else
1343  Arg = Count - 1;
1344  Count -= 8;
1345  BuildMI(MBB, MI, DL, get(AMDGPU::S_NOP))
1346  .addImm(Arg);
1347  }
1348 }
1349 
1352  insertWaitStates(MBB, MI, 1);
1353 }
1354 
1356  auto MF = MBB.getParent();
1358 
1359  assert(Info->isEntryFunction());
1360 
1361  if (MBB.succ_empty()) {
1362  bool HasNoTerminator = MBB.getFirstTerminator() == MBB.end();
1363  if (HasNoTerminator) {
1364  if (Info->returnsVoid()) {
1365  BuildMI(MBB, MBB.end(), DebugLoc(), get(AMDGPU::S_ENDPGM)).addImm(0);
1366  } else {
1367  BuildMI(MBB, MBB.end(), DebugLoc(), get(AMDGPU::SI_RETURN_TO_EPILOG));
1368  }
1369  }
1370  }
1371 }
1372 
1374  switch (MI.getOpcode()) {
1375  default: return 1; // FIXME: Do wait states equal cycles?
1376 
1377  case AMDGPU::S_NOP:
1378  return MI.getOperand(0).getImm() + 1;
1379  }
1380 }
1381 
1383  MachineBasicBlock &MBB = *MI.getParent();
1384  DebugLoc DL = MBB.findDebugLoc(MI);
1385  switch (MI.getOpcode()) {
1386  default: return TargetInstrInfo::expandPostRAPseudo(MI);
1387  case AMDGPU::S_MOV_B64_term:
1388  // This is only a terminator to get the correct spill code placement during
1389  // register allocation.
1390  MI.setDesc(get(AMDGPU::S_MOV_B64));
1391  break;
1392 
1393  case AMDGPU::S_MOV_B32_term:
1394  // This is only a terminator to get the correct spill code placement during
1395  // register allocation.
1396  MI.setDesc(get(AMDGPU::S_MOV_B32));
1397  break;
1398 
1399  case AMDGPU::S_XOR_B64_term:
1400  // This is only a terminator to get the correct spill code placement during
1401  // register allocation.
1402  MI.setDesc(get(AMDGPU::S_XOR_B64));
1403  break;
1404 
1405  case AMDGPU::S_XOR_B32_term:
1406  // This is only a terminator to get the correct spill code placement during
1407  // register allocation.
1408  MI.setDesc(get(AMDGPU::S_XOR_B32));
1409  break;
1410 
1411  case AMDGPU::S_OR_B32_term:
1412  // This is only a terminator to get the correct spill code placement during
1413  // register allocation.
1414  MI.setDesc(get(AMDGPU::S_OR_B32));
1415  break;
1416 
1417  case AMDGPU::S_ANDN2_B64_term:
1418  // This is only a terminator to get the correct spill code placement during
1419  // register allocation.
1420  MI.setDesc(get(AMDGPU::S_ANDN2_B64));
1421  break;
1422 
1423  case AMDGPU::S_ANDN2_B32_term:
1424  // This is only a terminator to get the correct spill code placement during
1425  // register allocation.
1426  MI.setDesc(get(AMDGPU::S_ANDN2_B32));
1427  break;
1428 
1429  case AMDGPU::V_MOV_B64_PSEUDO: {
1430  Register Dst = MI.getOperand(0).getReg();
1431  Register DstLo = RI.getSubReg(Dst, AMDGPU::sub0);
1432  Register DstHi = RI.getSubReg(Dst, AMDGPU::sub1);
1433 
1434  const MachineOperand &SrcOp = MI.getOperand(1);
1435  // FIXME: Will this work for 64-bit floating point immediates?
1436  assert(!SrcOp.isFPImm());
1437  if (SrcOp.isImm()) {
1438  APInt Imm(64, SrcOp.getImm());
1439  BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstLo)
1440  .addImm(Imm.getLoBits(32).getZExtValue())
1441  .addReg(Dst, RegState::Implicit | RegState::Define);
1442  BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstHi)
1443  .addImm(Imm.getHiBits(32).getZExtValue())
1444  .addReg(Dst, RegState::Implicit | RegState::Define);
1445  } else {
1446  assert(SrcOp.isReg());
1447  BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstLo)
1448  .addReg(RI.getSubReg(SrcOp.getReg(), AMDGPU::sub0))
1450  BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstHi)
1451  .addReg(RI.getSubReg(SrcOp.getReg(), AMDGPU::sub1))
1453  }
1454  MI.eraseFromParent();
1455  break;
1456  }
1457  case AMDGPU::V_SET_INACTIVE_B32: {
1458  unsigned NotOpc = ST.isWave32() ? AMDGPU::S_NOT_B32 : AMDGPU::S_NOT_B64;
1459  unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
1460  BuildMI(MBB, MI, DL, get(NotOpc), Exec)
1461  .addReg(Exec);
1462  BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), MI.getOperand(0).getReg())
1463  .add(MI.getOperand(2));
1464  BuildMI(MBB, MI, DL, get(NotOpc), Exec)
1465  .addReg(Exec);
1466  MI.eraseFromParent();
1467  break;
1468  }
1469  case AMDGPU::V_SET_INACTIVE_B64: {
1470  unsigned NotOpc = ST.isWave32() ? AMDGPU::S_NOT_B32 : AMDGPU::S_NOT_B64;
1471  unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
1472  BuildMI(MBB, MI, DL, get(NotOpc), Exec)
1473  .addReg(Exec);
1474  MachineInstr *Copy = BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B64_PSEUDO),
1475  MI.getOperand(0).getReg())
1476  .add(MI.getOperand(2));
1477  expandPostRAPseudo(*Copy);
1478  BuildMI(MBB, MI, DL, get(NotOpc), Exec)
1479  .addReg(Exec);
1480  MI.eraseFromParent();
1481  break;
1482  }
1483  case AMDGPU::V_MOVRELD_B32_V1:
1484  case AMDGPU::V_MOVRELD_B32_V2:
1485  case AMDGPU::V_MOVRELD_B32_V4:
1486  case AMDGPU::V_MOVRELD_B32_V8:
1487  case AMDGPU::V_MOVRELD_B32_V16: {
1488  const MCInstrDesc &MovRelDesc = get(AMDGPU::V_MOVRELD_B32_e32);
1489  Register VecReg = MI.getOperand(0).getReg();
1490  bool IsUndef = MI.getOperand(1).isUndef();
1491  unsigned SubReg = AMDGPU::sub0 + MI.getOperand(3).getImm();
1492  assert(VecReg == MI.getOperand(1).getReg());
1493 
1494  MachineInstr *MovRel =
1495  BuildMI(MBB, MI, DL, MovRelDesc)
1496  .addReg(RI.getSubReg(VecReg, SubReg), RegState::Undef)
1497  .add(MI.getOperand(2))
1498  .addReg(VecReg, RegState::ImplicitDefine)
1499  .addReg(VecReg,
1500  RegState::Implicit | (IsUndef ? RegState::Undef : 0));
1501 
1502  const int ImpDefIdx =
1503  MovRelDesc.getNumOperands() + MovRelDesc.getNumImplicitUses();
1504  const int ImpUseIdx = ImpDefIdx + 1;
1505  MovRel->tieOperands(ImpDefIdx, ImpUseIdx);
1506 
1507  MI.eraseFromParent();
1508  break;
1509  }
1510  case AMDGPU::SI_PC_ADD_REL_OFFSET: {
1511  MachineFunction &MF = *MBB.getParent();
1512  Register Reg = MI.getOperand(0).getReg();
1513  Register RegLo = RI.getSubReg(Reg, AMDGPU::sub0);
1514  Register RegHi = RI.getSubReg(Reg, AMDGPU::sub1);
1515 
1516  // Create a bundle so these instructions won't be re-ordered by the
1517  // post-RA scheduler.
1518  MIBundleBuilder Bundler(MBB, MI);
1519  Bundler.append(BuildMI(MF, DL, get(AMDGPU::S_GETPC_B64), Reg));
1520 
1521  // Add 32-bit offset from this instruction to the start of the
1522  // constant data.
1523  Bundler.append(BuildMI(MF, DL, get(AMDGPU::S_ADD_U32), RegLo)
1524  .addReg(RegLo)
1525  .add(MI.getOperand(1)));
1526 
1527  MachineInstrBuilder MIB = BuildMI(MF, DL, get(AMDGPU::S_ADDC_U32), RegHi)
1528  .addReg(RegHi);
1529  MIB.add(MI.getOperand(2));
1530 
1531  Bundler.append(MIB);
1532  finalizeBundle(MBB, Bundler.begin());
1533 
1534  MI.eraseFromParent();
1535  break;
1536  }
1537  case AMDGPU::ENTER_WWM: {
1538  // This only gets its own opcode so that SIPreAllocateWWMRegs can tell when
1539  // WWM is entered.
1540  MI.setDesc(get(ST.isWave32() ? AMDGPU::S_OR_SAVEEXEC_B32
1541  : AMDGPU::S_OR_SAVEEXEC_B64));
1542  break;
1543  }
1544  case AMDGPU::EXIT_WWM: {
1545  // This only gets its own opcode so that SIPreAllocateWWMRegs can tell when
1546  // WWM is exited.
1547  MI.setDesc(get(ST.isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64));
1548  break;
1549  }
1550  case TargetOpcode::BUNDLE: {
1551  if (!MI.mayLoad() || MI.hasUnmodeledSideEffects())
1552  return false;
1553 
1554  // If it is a load it must be a memory clause
1556  I->isBundledWithSucc(); ++I) {
1557  I->unbundleFromSucc();
1558  for (MachineOperand &MO : I->operands())
1559  if (MO.isReg())
1560  MO.setIsInternalRead(false);
1561  }
1562 
1563  MI.eraseFromParent();
1564  break;
1565  }
1566  }
1567  return true;
1568 }
1569 
1571  MachineOperand &Src0,
1572  unsigned Src0OpName,
1573  MachineOperand &Src1,
1574  unsigned Src1OpName) const {
1575  MachineOperand *Src0Mods = getNamedOperand(MI, Src0OpName);
1576  if (!Src0Mods)
1577  return false;
1578 
1579  MachineOperand *Src1Mods = getNamedOperand(MI, Src1OpName);
1580  assert(Src1Mods &&
1581  "All commutable instructions have both src0 and src1 modifiers");
1582 
1583  int Src0ModsVal = Src0Mods->getImm();
1584  int Src1ModsVal = Src1Mods->getImm();
1585 
1586  Src1Mods->setImm(Src0ModsVal);
1587  Src0Mods->setImm(Src1ModsVal);
1588  return true;
1589 }
1590 
1592  MachineOperand &RegOp,
1593  MachineOperand &NonRegOp) {
1594  Register Reg = RegOp.getReg();
1595  unsigned SubReg = RegOp.getSubReg();
1596  bool IsKill = RegOp.isKill();
1597  bool IsDead = RegOp.isDead();
1598  bool IsUndef = RegOp.isUndef();
1599  bool IsDebug = RegOp.isDebug();
1600 
1601  if (NonRegOp.isImm())
1602  RegOp.ChangeToImmediate(NonRegOp.getImm());
1603  else if (NonRegOp.isFI())
1604  RegOp.ChangeToFrameIndex(NonRegOp.getIndex());
1605  else
1606  return nullptr;
1607 
1608  NonRegOp.ChangeToRegister(Reg, false, false, IsKill, IsDead, IsUndef, IsDebug);
1609  NonRegOp.setSubReg(SubReg);
1610 
1611  return &MI;
1612 }
1613 
1615  unsigned Src0Idx,
1616  unsigned Src1Idx) const {
1617  assert(!NewMI && "this should never be used");
1618 
1619  unsigned Opc = MI.getOpcode();
1620  int CommutedOpcode = commuteOpcode(Opc);
1621  if (CommutedOpcode == -1)
1622  return nullptr;
1623 
1624  assert(AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0) ==
1625  static_cast<int>(Src0Idx) &&
1626  AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1) ==
1627  static_cast<int>(Src1Idx) &&
1628  "inconsistency with findCommutedOpIndices");
1629 
1630  MachineOperand &Src0 = MI.getOperand(Src0Idx);
1631  MachineOperand &Src1 = MI.getOperand(Src1Idx);
1632 
1633  MachineInstr *CommutedMI = nullptr;
1634  if (Src0.isReg() && Src1.isReg()) {
1635  if (isOperandLegal(MI, Src1Idx, &Src0)) {
1636  // Be sure to copy the source modifiers to the right place.
1637  CommutedMI
1638  = TargetInstrInfo::commuteInstructionImpl(MI, NewMI, Src0Idx, Src1Idx);
1639  }
1640 
1641  } else if (Src0.isReg() && !Src1.isReg()) {
1642  // src0 should always be able to support any operand type, so no need to
1643  // check operand legality.
1644  CommutedMI = swapRegAndNonRegOperand(MI, Src0, Src1);
1645  } else if (!Src0.isReg() && Src1.isReg()) {
1646  if (isOperandLegal(MI, Src1Idx, &Src0))
1647  CommutedMI = swapRegAndNonRegOperand(MI, Src1, Src0);
1648  } else {
1649  // FIXME: Found two non registers to commute. This does happen.
1650  return nullptr;
1651  }
1652 
1653  if (CommutedMI) {
1654  swapSourceModifiers(MI, Src0, AMDGPU::OpName::src0_modifiers,
1655  Src1, AMDGPU::OpName::src1_modifiers);
1656 
1657  CommutedMI->setDesc(get(CommutedOpcode));
1658  }
1659 
1660  return CommutedMI;
1661 }
1662 
1663 // This needs to be implemented because the source modifiers may be inserted
1664 // between the true commutable operands, and the base
1665 // TargetInstrInfo::commuteInstruction uses it.
1667  unsigned &SrcOpIdx0,
1668  unsigned &SrcOpIdx1) const {
1669  return findCommutedOpIndices(MI.getDesc(), SrcOpIdx0, SrcOpIdx1);
1670 }
1671 
1672 bool SIInstrInfo::findCommutedOpIndices(MCInstrDesc Desc, unsigned &SrcOpIdx0,
1673  unsigned &SrcOpIdx1) const {
1674  if (!Desc.isCommutable())
1675  return false;
1676 
1677  unsigned Opc = Desc.getOpcode();
1678  int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0);
1679  if (Src0Idx == -1)
1680  return false;
1681 
1682  int Src1Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1);
1683  if (Src1Idx == -1)
1684  return false;
1685 
1686  return fixCommutedOpIndices(SrcOpIdx0, SrcOpIdx1, Src0Idx, Src1Idx);
1687 }
1688 
1689 bool SIInstrInfo::isBranchOffsetInRange(unsigned BranchOp,
1690  int64_t BrOffset) const {
1691  // BranchRelaxation should never have to check s_setpc_b64 because its dest
1692  // block is unanalyzable.
1693  assert(BranchOp != AMDGPU::S_SETPC_B64);
1694 
1695  // Convert to dwords.
1696  BrOffset /= 4;
1697 
1698  // The branch instructions do PC += signext(SIMM16 * 4) + 4, so the offset is
1699  // from the next instruction.
1700  BrOffset -= 1;
1701 
1702  return isIntN(BranchOffsetBits, BrOffset);
1703 }
1704 
1706  const MachineInstr &MI) const {
1707  if (MI.getOpcode() == AMDGPU::S_SETPC_B64) {
1708  // This would be a difficult analysis to perform, but can always be legal so
1709  // there's no need to analyze it.
1710  return nullptr;
1711  }
1712 
1713  return MI.getOperand(0).getMBB();
1714 }
1715 
1717  MachineBasicBlock &DestBB,
1718  const DebugLoc &DL,
1719  int64_t BrOffset,
1720  RegScavenger *RS) const {
1721  assert(RS && "RegScavenger required for long branching");
1722  assert(MBB.empty() &&
1723  "new block should be inserted for expanding unconditional branch");
1724  assert(MBB.pred_size() == 1);
1725 
1726  MachineFunction *MF = MBB.getParent();
1727  MachineRegisterInfo &MRI = MF->getRegInfo();
1728 
1729  // FIXME: Virtual register workaround for RegScavenger not working with empty
1730  // blocks.
1731  Register PCReg = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
1732 
1733  auto I = MBB.end();
1734 
1735  // We need to compute the offset relative to the instruction immediately after
1736  // s_getpc_b64. Insert pc arithmetic code before last terminator.
1737  MachineInstr *GetPC = BuildMI(MBB, I, DL, get(AMDGPU::S_GETPC_B64), PCReg);
1738 
1739  // TODO: Handle > 32-bit block address.
1740  if (BrOffset >= 0) {
1741  BuildMI(MBB, I, DL, get(AMDGPU::S_ADD_U32))
1742  .addReg(PCReg, RegState::Define, AMDGPU::sub0)
1743  .addReg(PCReg, 0, AMDGPU::sub0)
1744  .addMBB(&DestBB, MO_LONG_BRANCH_FORWARD);
1745  BuildMI(MBB, I, DL, get(AMDGPU::S_ADDC_U32))
1746  .addReg(PCReg, RegState::Define, AMDGPU::sub1)
1747  .addReg(PCReg, 0, AMDGPU::sub1)
1748  .addImm(0);
1749  } else {
1750  // Backwards branch.
1751  BuildMI(MBB, I, DL, get(AMDGPU::S_SUB_U32))
1752  .addReg(PCReg, RegState::Define, AMDGPU::sub0)
1753  .addReg(PCReg, 0, AMDGPU::sub0)
1754  .addMBB(&DestBB, MO_LONG_BRANCH_BACKWARD);
1755  BuildMI(MBB, I, DL, get(AMDGPU::S_SUBB_U32))
1756  .addReg(PCReg, RegState::Define, AMDGPU::sub1)
1757  .addReg(PCReg, 0, AMDGPU::sub1)
1758  .addImm(0);
1759  }
1760 
1761  // Insert the indirect branch after the other terminator.
1762  BuildMI(&MBB, DL, get(AMDGPU::S_SETPC_B64))
1763  .addReg(PCReg);
1764 
1765  // FIXME: If spilling is necessary, this will fail because this scavenger has
1766  // no emergency stack slots. It is non-trivial to spill in this situation,
1767  // because the restore code needs to be specially placed after the
1768  // jump. BranchRelaxation then needs to be made aware of the newly inserted
1769  // block.
1770  //
1771  // If a spill is needed for the pc register pair, we need to insert a spill
1772  // restore block right before the destination block, and insert a short branch
1773  // into the old destination block's fallthrough predecessor.
1774  // e.g.:
1775  //
1776  // s_cbranch_scc0 skip_long_branch:
1777  //
1778  // long_branch_bb:
1779  // spill s[8:9]
1780  // s_getpc_b64 s[8:9]
1781  // s_add_u32 s8, s8, restore_bb
1782  // s_addc_u32 s9, s9, 0
1783  // s_setpc_b64 s[8:9]
1784  //
1785  // skip_long_branch:
1786  // foo;
1787  //
1788  // .....
1789  //
1790  // dest_bb_fallthrough_predecessor:
1791  // bar;
1792  // s_branch dest_bb
1793  //
1794  // restore_bb:
1795  // restore s[8:9]
1796  // fallthrough dest_bb
1797  ///
1798  // dest_bb:
1799  // buzz;
1800 
1801  RS->enterBasicBlockEnd(MBB);
1802  unsigned Scav = RS->scavengeRegisterBackwards(
1803  AMDGPU::SReg_64RegClass,
1804  MachineBasicBlock::iterator(GetPC), false, 0);
1805  MRI.replaceRegWith(PCReg, Scav);
1806  MRI.clearVirtRegs();
1807  RS->setRegUsed(Scav);
1808 
1809  return 4 + 8 + 4 + 4;
1810 }
1811 
1812 unsigned SIInstrInfo::getBranchOpcode(SIInstrInfo::BranchPredicate Cond) {
1813  switch (Cond) {
1814  case SIInstrInfo::SCC_TRUE:
1815  return AMDGPU::S_CBRANCH_SCC1;
1816  case SIInstrInfo::SCC_FALSE:
1817  return AMDGPU::S_CBRANCH_SCC0;
1818  case SIInstrInfo::VCCNZ:
1819  return AMDGPU::S_CBRANCH_VCCNZ;
1820  case SIInstrInfo::VCCZ:
1821  return AMDGPU::S_CBRANCH_VCCZ;
1822  case SIInstrInfo::EXECNZ:
1823  return AMDGPU::S_CBRANCH_EXECNZ;
1824  case SIInstrInfo::EXECZ:
1825  return AMDGPU::S_CBRANCH_EXECZ;
1826  default:
1827  llvm_unreachable("invalid branch predicate");
1828  }
1829 }
1830 
1831 SIInstrInfo::BranchPredicate SIInstrInfo::getBranchPredicate(unsigned Opcode) {
1832  switch (Opcode) {
1833  case AMDGPU::S_CBRANCH_SCC0:
1834  return SCC_FALSE;
1835  case AMDGPU::S_CBRANCH_SCC1:
1836  return SCC_TRUE;
1837  case AMDGPU::S_CBRANCH_VCCNZ:
1838  return VCCNZ;
1839  case AMDGPU::S_CBRANCH_VCCZ:
1840  return VCCZ;
1841  case AMDGPU::S_CBRANCH_EXECNZ:
1842  return EXECNZ;
1843  case AMDGPU::S_CBRANCH_EXECZ:
1844  return EXECZ;
1845  default:
1846  return INVALID_BR;
1847  }
1848 }
1849 
1852  MachineBasicBlock *&TBB,
1853  MachineBasicBlock *&FBB,
1855  bool AllowModify) const {
1856  if (I->getOpcode() == AMDGPU::S_BRANCH) {
1857  // Unconditional Branch
1858  TBB = I->getOperand(0).getMBB();
1859  return false;
1860  }
1861 
1862  MachineBasicBlock *CondBB = nullptr;
1863 
1864  if (I->getOpcode() == AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO) {
1865  CondBB = I->getOperand(1).getMBB();
1866  Cond.push_back(I->getOperand(0));
1867  } else {
1868  BranchPredicate Pred = getBranchPredicate(I->getOpcode());
1869  if (Pred == INVALID_BR)
1870  return true;
1871 
1872  CondBB = I->getOperand(0).getMBB();
1874  Cond.push_back(I->getOperand(1)); // Save the branch register.
1875  }
1876  ++I;
1877 
1878  if (I == MBB.end()) {
1879  // Conditional branch followed by fall-through.
1880  TBB = CondBB;
1881  return false;
1882  }
1883 
1884  if (I->getOpcode() == AMDGPU::S_BRANCH) {
1885  TBB = CondBB;
1886  FBB = I->getOperand(0).getMBB();
1887  return false;
1888  }
1889 
1890  return true;
1891 }
1892 
1894  MachineBasicBlock *&FBB,
1896  bool AllowModify) const {
1898  auto E = MBB.end();
1899  if (I == E)
1900  return false;
1901 
1902  // Skip over the instructions that are artificially terminators for special
1903  // exec management.
1904  while (I != E && !I->isBranch() && !I->isReturn() &&
1905  I->getOpcode() != AMDGPU::SI_MASK_BRANCH) {
1906  switch (I->getOpcode()) {
1907  case AMDGPU::SI_MASK_BRANCH:
1908  case AMDGPU::S_MOV_B64_term:
1909  case AMDGPU::S_XOR_B64_term:
1910  case AMDGPU::S_ANDN2_B64_term:
1911  case AMDGPU::S_MOV_B32_term:
1912  case AMDGPU::S_XOR_B32_term:
1913  case AMDGPU::S_OR_B32_term:
1914  case AMDGPU::S_ANDN2_B32_term:
1915  break;
1916  case AMDGPU::SI_IF:
1917  case AMDGPU::SI_ELSE:
1918  case AMDGPU::SI_KILL_I1_TERMINATOR:
1919  case AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR:
1920  // FIXME: It's messy that these need to be considered here at all.
1921  return true;
1922  default:
1923  llvm_unreachable("unexpected non-branch terminator inst");
1924  }
1925 
1926  ++I;
1927  }
1928 
1929  if (I == E)
1930  return false;
1931 
1932  if (I->getOpcode() != AMDGPU::SI_MASK_BRANCH)
1933  return analyzeBranchImpl(MBB, I, TBB, FBB, Cond, AllowModify);
1934 
1935  ++I;
1936 
1937  // TODO: Should be able to treat as fallthrough?
1938  if (I == MBB.end())
1939  return true;
1940 
1941  if (analyzeBranchImpl(MBB, I, TBB, FBB, Cond, AllowModify))
1942  return true;
1943 
1944  MachineBasicBlock *MaskBrDest = I->getOperand(0).getMBB();
1945 
1946  // Specifically handle the case where the conditional branch is to the same
1947  // destination as the mask branch. e.g.
1948  //
1949  // si_mask_branch BB8
1950  // s_cbranch_execz BB8
1951  // s_cbranch BB9
1952  //
1953  // This is required to understand divergent loops which may need the branches
1954  // to be relaxed.
1955  if (TBB != MaskBrDest || Cond.empty())
1956  return true;
1957 
1958  auto Pred = Cond[0].getImm();
1959  return (Pred != EXECZ && Pred != EXECNZ);
1960 }
1961 
1963  int *BytesRemoved) const {
1965 
1966  unsigned Count = 0;
1967  unsigned RemovedSize = 0;
1968  while (I != MBB.end()) {
1969  MachineBasicBlock::iterator Next = std::next(I);
1970  if (I->getOpcode() == AMDGPU::SI_MASK_BRANCH) {
1971  I = Next;
1972  continue;
1973  }
1974 
1975  RemovedSize += getInstSizeInBytes(*I);
1976  I->eraseFromParent();
1977  ++Count;
1978  I = Next;
1979  }
1980 
1981  if (BytesRemoved)
1982  *BytesRemoved = RemovedSize;
1983 
1984  return Count;
1985 }
1986 
1987 // Copy the flags onto the implicit condition register operand.
1989  const MachineOperand &OrigCond) {
1990  CondReg.setIsUndef(OrigCond.isUndef());
1991  CondReg.setIsKill(OrigCond.isKill());
1992 }
1993 
1995  MachineBasicBlock *TBB,
1996  MachineBasicBlock *FBB,
1998  const DebugLoc &DL,
1999  int *BytesAdded) const {
2000  if (!FBB && Cond.empty()) {
2001  BuildMI(&MBB, DL, get(AMDGPU::S_BRANCH))
2002  .addMBB(TBB);
2003  if (BytesAdded)
2004  *BytesAdded = 4;
2005  return 1;
2006  }
2007 
2008  if(Cond.size() == 1 && Cond[0].isReg()) {
2009  BuildMI(&MBB, DL, get(AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO))
2010  .add(Cond[0])
2011  .addMBB(TBB);
2012  return 1;
2013  }
2014 
2015  assert(TBB && Cond[0].isImm());
2016 
2017  unsigned Opcode
2018  = getBranchOpcode(static_cast<BranchPredicate>(Cond[0].getImm()));
2019 
2020  if (!FBB) {
2021  Cond[1].isUndef();
2022  MachineInstr *CondBr =
2023  BuildMI(&MBB, DL, get(Opcode))
2024  .addMBB(TBB);
2025 
2026  // Copy the flags onto the implicit condition register operand.
2027  preserveCondRegFlags(CondBr->getOperand(1), Cond[1]);
2028 
2029  if (BytesAdded)
2030  *BytesAdded = 4;
2031  return 1;
2032  }
2033 
2034  assert(TBB && FBB);
2035 
2036  MachineInstr *CondBr =
2037  BuildMI(&MBB, DL, get(Opcode))
2038  .addMBB(TBB);
2039  BuildMI(&MBB, DL, get(AMDGPU::S_BRANCH))
2040  .addMBB(FBB);
2041 
2042  MachineOperand &CondReg = CondBr->getOperand(1);
2043  CondReg.setIsUndef(Cond[1].isUndef());
2044  CondReg.setIsKill(Cond[1].isKill());
2045 
2046  if (BytesAdded)
2047  *BytesAdded = 8;
2048 
2049  return 2;
2050 }
2051 
2053  SmallVectorImpl<MachineOperand> &Cond) const {
2054  if (Cond.size() != 2) {
2055  return true;
2056  }
2057 
2058  if (Cond[0].isImm()) {
2059  Cond[0].setImm(-Cond[0].getImm());
2060  return false;
2061  }
2062 
2063  return true;
2064 }
2065 
2068  unsigned TrueReg, unsigned FalseReg,
2069  int &CondCycles,
2070  int &TrueCycles, int &FalseCycles) const {
2071  switch (Cond[0].getImm()) {
2072  case VCCNZ:
2073  case VCCZ: {
2074  const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
2075  const TargetRegisterClass *RC = MRI.getRegClass(TrueReg);
2076  assert(MRI.getRegClass(FalseReg) == RC);
2077 
2078  int NumInsts = AMDGPU::getRegBitWidth(RC->getID()) / 32;
2079  CondCycles = TrueCycles = FalseCycles = NumInsts; // ???
2080 
2081  // Limit to equal cost for branch vs. N v_cndmask_b32s.
2082  return RI.hasVGPRs(RC) && NumInsts <= 6;
2083  }
2084  case SCC_TRUE:
2085  case SCC_FALSE: {
2086  // FIXME: We could insert for VGPRs if we could replace the original compare
2087  // with a vector one.
2088  const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
2089  const TargetRegisterClass *RC = MRI.getRegClass(TrueReg);
2090  assert(MRI.getRegClass(FalseReg) == RC);
2091 
2092  int NumInsts = AMDGPU::getRegBitWidth(RC->getID()) / 32;
2093 
2094  // Multiples of 8 can do s_cselect_b64
2095  if (NumInsts % 2 == 0)
2096  NumInsts /= 2;
2097 
2098  CondCycles = TrueCycles = FalseCycles = NumInsts; // ???
2099  return RI.isSGPRClass(RC);
2100  }
2101  default:
2102  return false;
2103  }
2104 }
2105 
2108  unsigned DstReg, ArrayRef<MachineOperand> Cond,
2109  unsigned TrueReg, unsigned FalseReg) const {
2110  BranchPredicate Pred = static_cast<BranchPredicate>(Cond[0].getImm());
2111  if (Pred == VCCZ || Pred == SCC_FALSE) {
2112  Pred = static_cast<BranchPredicate>(-Pred);
2113  std::swap(TrueReg, FalseReg);
2114  }
2115 
2117  const TargetRegisterClass *DstRC = MRI.getRegClass(DstReg);
2118  unsigned DstSize = RI.getRegSizeInBits(*DstRC);
2119 
2120  if (DstSize == 32) {
2121  unsigned SelOp = Pred == SCC_TRUE ?
2122  AMDGPU::S_CSELECT_B32 : AMDGPU::V_CNDMASK_B32_e32;
2123 
2124  // Instruction's operands are backwards from what is expected.
2125  MachineInstr *Select =
2126  BuildMI(MBB, I, DL, get(SelOp), DstReg)
2127  .addReg(FalseReg)
2128  .addReg(TrueReg);
2129 
2130  preserveCondRegFlags(Select->getOperand(3), Cond[1]);
2131  return;
2132  }
2133 
2134  if (DstSize == 64 && Pred == SCC_TRUE) {
2135  MachineInstr *Select =
2136  BuildMI(MBB, I, DL, get(AMDGPU::S_CSELECT_B64), DstReg)
2137  .addReg(FalseReg)
2138  .addReg(TrueReg);
2139 
2140  preserveCondRegFlags(Select->getOperand(3), Cond[1]);
2141  return;
2142  }
2143 
2144  static const int16_t Sub0_15[] = {
2145  AMDGPU::sub0, AMDGPU::sub1, AMDGPU::sub2, AMDGPU::sub3,
2146  AMDGPU::sub4, AMDGPU::sub5, AMDGPU::sub6, AMDGPU::sub7,
2147  AMDGPU::sub8, AMDGPU::sub9, AMDGPU::sub10, AMDGPU::sub11,
2148  AMDGPU::sub12, AMDGPU::sub13, AMDGPU::sub14, AMDGPU::sub15,
2149  };
2150 
2151  static const int16_t Sub0_15_64[] = {
2152  AMDGPU::sub0_sub1, AMDGPU::sub2_sub3,
2153  AMDGPU::sub4_sub5, AMDGPU::sub6_sub7,
2154  AMDGPU::sub8_sub9, AMDGPU::sub10_sub11,
2155  AMDGPU::sub12_sub13, AMDGPU::sub14_sub15,
2156  };
2157 
2158  unsigned SelOp = AMDGPU::V_CNDMASK_B32_e32;
2159  const TargetRegisterClass *EltRC = &AMDGPU::VGPR_32RegClass;
2160  const int16_t *SubIndices = Sub0_15;
2161  int NElts = DstSize / 32;
2162 
2163  // 64-bit select is only available for SALU.
2164  // TODO: Split 96-bit into 64-bit and 32-bit, not 3x 32-bit.
2165  if (Pred == SCC_TRUE) {
2166  if (NElts % 2) {
2167  SelOp = AMDGPU::S_CSELECT_B32;
2168  EltRC = &AMDGPU::SGPR_32RegClass;
2169  } else {
2170  SelOp = AMDGPU::S_CSELECT_B64;
2171  EltRC = &AMDGPU::SGPR_64RegClass;
2172  SubIndices = Sub0_15_64;
2173  NElts /= 2;
2174  }
2175  }
2176 
2178  MBB, I, DL, get(AMDGPU::REG_SEQUENCE), DstReg);
2179 
2180  I = MIB->getIterator();
2181 
2183  for (int Idx = 0; Idx != NElts; ++Idx) {
2184  Register DstElt = MRI.createVirtualRegister(EltRC);
2185  Regs.push_back(DstElt);
2186 
2187  unsigned SubIdx = SubIndices[Idx];
2188 
2189  MachineInstr *Select =
2190  BuildMI(MBB, I, DL, get(SelOp), DstElt)
2191  .addReg(FalseReg, 0, SubIdx)
2192  .addReg(TrueReg, 0, SubIdx);
2193  preserveCondRegFlags(Select->getOperand(3), Cond[1]);
2194  fixImplicitOperands(*Select);
2195 
2196  MIB.addReg(DstElt)
2197  .addImm(SubIdx);
2198  }
2199 }
2200 
2202  switch (MI.getOpcode()) {
2203  case AMDGPU::V_MOV_B32_e32:
2204  case AMDGPU::V_MOV_B32_e64:
2205  case AMDGPU::V_MOV_B64_PSEUDO: {
2206  // If there are additional implicit register operands, this may be used for
2207  // register indexing so the source register operand isn't simply copied.
2208  unsigned NumOps = MI.getDesc().getNumOperands() +
2209  MI.getDesc().getNumImplicitUses();
2210 
2211  return MI.getNumOperands() == NumOps;
2212  }
2213  case AMDGPU::S_MOV_B32:
2214  case AMDGPU::S_MOV_B64:
2215  case AMDGPU::COPY:
2216  case AMDGPU::V_ACCVGPR_WRITE_B32:
2217  case AMDGPU::V_ACCVGPR_READ_B32:
2218  return true;
2219  default:
2220  return false;
2221  }
2222 }
2223 
2225  unsigned Kind) const {
2226  switch(Kind) {
2237  }
2238  return AMDGPUAS::FLAT_ADDRESS;
2239 }
2240 
2242  unsigned Opc = MI.getOpcode();
2243  int Src0ModIdx = AMDGPU::getNamedOperandIdx(Opc,
2244  AMDGPU::OpName::src0_modifiers);
2245  int Src1ModIdx = AMDGPU::getNamedOperandIdx(Opc,
2246  AMDGPU::OpName::src1_modifiers);
2247  int Src2ModIdx = AMDGPU::getNamedOperandIdx(Opc,
2248  AMDGPU::OpName::src2_modifiers);
2249 
2250  MI.RemoveOperand(Src2ModIdx);
2251  MI.RemoveOperand(Src1ModIdx);
2252  MI.RemoveOperand(Src0ModIdx);
2253 }
2254 
2256  unsigned Reg, MachineRegisterInfo *MRI) const {
2257  if (!MRI->hasOneNonDBGUse(Reg))
2258  return false;
2259 
2260  switch (DefMI.getOpcode()) {
2261  default:
2262  return false;
2263  case AMDGPU::S_MOV_B64:
2264  // TODO: We could fold 64-bit immediates, but this get compilicated
2265  // when there are sub-registers.
2266  return false;
2267 
2268  case AMDGPU::V_MOV_B32_e32:
2269  case AMDGPU::S_MOV_B32:
2270  case AMDGPU::V_ACCVGPR_WRITE_B32:
2271  break;
2272  }
2273 
2274  const MachineOperand *ImmOp = getNamedOperand(DefMI, AMDGPU::OpName::src0);
2275  assert(ImmOp);
2276  // FIXME: We could handle FrameIndex values here.
2277  if (!ImmOp->isImm())
2278  return false;
2279 
2280  unsigned Opc = UseMI.getOpcode();
2281  if (Opc == AMDGPU::COPY) {
2282  bool isVGPRCopy = RI.isVGPR(*MRI, UseMI.getOperand(0).getReg());
2283  unsigned NewOpc = isVGPRCopy ? AMDGPU::V_MOV_B32_e32 : AMDGPU::S_MOV_B32;
2284  if (RI.isAGPR(*MRI, UseMI.getOperand(0).getReg())) {
2286  return false;
2287  NewOpc = AMDGPU::V_ACCVGPR_WRITE_B32;
2288  }
2289  UseMI.setDesc(get(NewOpc));
2290  UseMI.getOperand(1).ChangeToImmediate(ImmOp->getImm());
2291  UseMI.addImplicitDefUseOperands(*UseMI.getParent()->getParent());
2292  return true;
2293  }
2294 
2295  if (Opc == AMDGPU::V_MAD_F32 || Opc == AMDGPU::V_MAC_F32_e64 ||
2296  Opc == AMDGPU::V_MAD_F16 || Opc == AMDGPU::V_MAC_F16_e64 ||
2297  Opc == AMDGPU::V_FMA_F32 || Opc == AMDGPU::V_FMAC_F32_e64 ||
2298  Opc == AMDGPU::V_FMA_F16 || Opc == AMDGPU::V_FMAC_F16_e64) {
2299  // Don't fold if we are using source or output modifiers. The new VOP2
2300  // instructions don't have them.
2301  if (hasAnyModifiersSet(UseMI))
2302  return false;
2303 
2304  // If this is a free constant, there's no reason to do this.
2305  // TODO: We could fold this here instead of letting SIFoldOperands do it
2306  // later.
2307  MachineOperand *Src0 = getNamedOperand(UseMI, AMDGPU::OpName::src0);
2308 
2309  // Any src operand can be used for the legality check.
2310  if (isInlineConstant(UseMI, *Src0, *ImmOp))
2311  return false;
2312 
2313  bool IsF32 = Opc == AMDGPU::V_MAD_F32 || Opc == AMDGPU::V_MAC_F32_e64 ||
2314  Opc == AMDGPU::V_FMA_F32 || Opc == AMDGPU::V_FMAC_F32_e64;
2315  bool IsFMA = Opc == AMDGPU::V_FMA_F32 || Opc == AMDGPU::V_FMAC_F32_e64 ||
2316  Opc == AMDGPU::V_FMA_F16 || Opc == AMDGPU::V_FMAC_F16_e64;
2317  MachineOperand *Src1 = getNamedOperand(UseMI, AMDGPU::OpName::src1);
2318  MachineOperand *Src2 = getNamedOperand(UseMI, AMDGPU::OpName::src2);
2319 
2320  // Multiplied part is the constant: Use v_madmk_{f16, f32}.
2321  // We should only expect these to be on src0 due to canonicalizations.
2322  if (Src0->isReg() && Src0->getReg() == Reg) {
2323  if (!Src1->isReg() || RI.isSGPRClass(MRI->getRegClass(Src1->getReg())))
2324  return false;
2325 
2326  if (!Src2->isReg() || RI.isSGPRClass(MRI->getRegClass(Src2->getReg())))
2327  return false;
2328 
2329  unsigned NewOpc =
2330  IsFMA ? (IsF32 ? AMDGPU::V_FMAMK_F32 : AMDGPU::V_FMAMK_F16)
2331  : (IsF32 ? AMDGPU::V_MADMK_F32 : AMDGPU::V_MADMK_F16);
2332  if (pseudoToMCOpcode(NewOpc) == -1)
2333  return false;
2334 
2335  // We need to swap operands 0 and 1 since madmk constant is at operand 1.
2336 
2337  const int64_t Imm = ImmOp->getImm();
2338 
2339  // FIXME: This would be a lot easier if we could return a new instruction
2340  // instead of having to modify in place.
2341 
2342  // Remove these first since they are at the end.
2343  UseMI.RemoveOperand(
2344  AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::omod));
2345  UseMI.RemoveOperand(
2346  AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::clamp));
2347 
2348  Register Src1Reg = Src1->getReg();
2349  unsigned Src1SubReg = Src1->getSubReg();
2350  Src0->setReg(Src1Reg);
2351  Src0->setSubReg(Src1SubReg);
2352  Src0->setIsKill(Src1->isKill());
2353 
2354  if (Opc == AMDGPU::V_MAC_F32_e64 ||
2355  Opc == AMDGPU::V_MAC_F16_e64 ||
2356  Opc == AMDGPU::V_FMAC_F32_e64 ||
2357  Opc == AMDGPU::V_FMAC_F16_e64)
2358  UseMI.untieRegOperand(
2359  AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2));
2360 
2361  Src1->ChangeToImmediate(Imm);
2362 
2363  removeModOperands(UseMI);
2364  UseMI.setDesc(get(NewOpc));
2365 
2366  bool DeleteDef = MRI->hasOneNonDBGUse(Reg);
2367  if (DeleteDef)
2368  DefMI.eraseFromParent();
2369 
2370  return true;
2371  }
2372 
2373  // Added part is the constant: Use v_madak_{f16, f32}.
2374  if (Src2->isReg() && Src2->getReg() == Reg) {
2375  // Not allowed to use constant bus for another operand.
2376  // We can however allow an inline immediate as src0.
2377  bool Src0Inlined = false;
2378  if (Src0->isReg()) {
2379  // Try to inline constant if possible.
2380  // If the Def moves immediate and the use is single
2381  // We are saving VGPR here.
2382  MachineInstr *Def = MRI->getUniqueVRegDef(Src0->getReg());
2383  if (Def && Def->isMoveImmediate() &&
2384  isInlineConstant(Def->getOperand(1)) &&
2385  MRI->hasOneUse(Src0->getReg())) {
2386  Src0->ChangeToImmediate(Def->getOperand(1).getImm());
2387  Src0Inlined = true;
2388  } else if ((Register::isPhysicalRegister(Src0->getReg()) &&
2389  (ST.getConstantBusLimit(Opc) <= 1 &&
2390  RI.isSGPRClass(RI.getPhysRegClass(Src0->getReg())))) ||
2391  (Register::isVirtualRegister(Src0->getReg()) &&
2392  (ST.getConstantBusLimit(Opc) <= 1 &&
2393  RI.isSGPRClass(MRI->getRegClass(Src0->getReg())))))
2394  return false;
2395  // VGPR is okay as Src0 - fallthrough
2396  }
2397 
2398  if (Src1->isReg() && !Src0Inlined ) {
2399  // We have one slot for inlinable constant so far - try to fill it
2400  MachineInstr *Def = MRI->getUniqueVRegDef(Src1->getReg());
2401  if (Def && Def->isMoveImmediate() &&
2402  isInlineConstant(Def->getOperand(1)) &&
2403  MRI->hasOneUse(Src1->getReg()) &&
2404  commuteInstruction(UseMI)) {
2405  Src0->ChangeToImmediate(Def->getOperand(1).getImm());
2406  } else if ((Register::isPhysicalRegister(Src1->getReg()) &&
2407  RI.isSGPRClass(RI.getPhysRegClass(Src1->getReg()))) ||
2408  (Register::isVirtualRegister(Src1->getReg()) &&
2409  RI.isSGPRClass(MRI->getRegClass(Src1->getReg()))))
2410  return false;
2411  // VGPR is okay as Src1 - fallthrough
2412  }
2413 
2414  unsigned NewOpc =
2415  IsFMA ? (IsF32 ? AMDGPU::V_FMAAK_F32 : AMDGPU::V_FMAAK_F16)
2416  : (IsF32 ? AMDGPU::V_MADAK_F32 : AMDGPU::V_MADAK_F16);
2417  if (pseudoToMCOpcode(NewOpc) == -1)
2418  return false;
2419 
2420  const int64_t Imm = ImmOp->getImm();
2421 
2422  // FIXME: This would be a lot easier if we could return a new instruction
2423  // instead of having to modify in place.
2424 
2425  // Remove these first since they are at the end.
2426  UseMI.RemoveOperand(
2427  AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::omod));
2428  UseMI.RemoveOperand(
2429  AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::clamp));
2430 
2431  if (Opc == AMDGPU::V_MAC_F32_e64 ||
2432  Opc == AMDGPU::V_MAC_F16_e64 ||
2433  Opc == AMDGPU::V_FMAC_F32_e64 ||
2434  Opc == AMDGPU::V_FMAC_F16_e64)
2435  UseMI.untieRegOperand(
2436  AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2));
2437 
2438  // ChangingToImmediate adds Src2 back to the instruction.
2439  Src2->ChangeToImmediate(Imm);
2440 
2441  // These come before src2.
2442  removeModOperands(UseMI);
2443  UseMI.setDesc(get(NewOpc));
2444  // It might happen that UseMI was commuted
2445  // and we now have SGPR as SRC1. If so 2 inlined
2446  // constant and SGPR are illegal.
2447  legalizeOperands(UseMI);
2448 
2449  bool DeleteDef = MRI->hasOneNonDBGUse(Reg);
2450  if (DeleteDef)
2451  DefMI.eraseFromParent();
2452 
2453  return true;
2454  }
2455  }
2456 
2457  return false;
2458 }
2459 
2460 static bool offsetsDoNotOverlap(int WidthA, int OffsetA,
2461  int WidthB, int OffsetB) {
2462  int LowOffset = OffsetA < OffsetB ? OffsetA : OffsetB;
2463  int HighOffset = OffsetA < OffsetB ? OffsetB : OffsetA;
2464  int LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB;
2465  return LowOffset + LowWidth <= HighOffset;
2466 }
2467 
2468 bool SIInstrInfo::checkInstOffsetsDoNotOverlap(const MachineInstr &MIa,
2469  const MachineInstr &MIb) const {
2470  const MachineOperand *BaseOp0, *BaseOp1;
2471  int64_t Offset0, Offset1;
2472 
2473  if (getMemOperandWithOffset(MIa, BaseOp0, Offset0, &RI) &&
2474  getMemOperandWithOffset(MIb, BaseOp1, Offset1, &RI)) {
2475  if (!BaseOp0->isIdenticalTo(*BaseOp1))
2476  return false;
2477 
2478  if (!MIa.hasOneMemOperand() || !MIb.hasOneMemOperand()) {
2479  // FIXME: Handle ds_read2 / ds_write2.
2480  return false;
2481  }
2482  unsigned Width0 = (*MIa.memoperands_begin())->getSize();
2483  unsigned Width1 = (*MIb.memoperands_begin())->getSize();
2484  if (offsetsDoNotOverlap(Width0, Offset0, Width1, Offset1)) {
2485  return true;
2486  }
2487  }
2488 
2489  return false;
2490 }
2491 
2493  const MachineInstr &MIb) const {
2494  assert((MIa.mayLoad() || MIa.mayStore()) &&
2495  "MIa must load from or modify a memory location");
2496  assert((MIb.mayLoad() || MIb.mayStore()) &&
2497  "MIb must load from or modify a memory location");
2498 
2500  return false;
2501 
2502  // XXX - Can we relax this between address spaces?
2503  if (MIa.hasOrderedMemoryRef() || MIb.hasOrderedMemoryRef())
2504  return false;
2505 
2506  // TODO: Should we check the address space from the MachineMemOperand? That
2507  // would allow us to distinguish objects we know don't alias based on the
2508  // underlying address space, even if it was lowered to a different one,
2509  // e.g. private accesses lowered to use MUBUF instructions on a scratch
2510  // buffer.
2511  if (isDS(MIa)) {
2512  if (isDS(MIb))
2513  return checkInstOffsetsDoNotOverlap(MIa, MIb);
2514 
2515  return !isFLAT(MIb) || isSegmentSpecificFLAT(MIb);
2516  }
2517 
2518  if (isMUBUF(MIa) || isMTBUF(MIa)) {
2519  if (isMUBUF(MIb) || isMTBUF(MIb))
2520  return checkInstOffsetsDoNotOverlap(MIa, MIb);
2521 
2522  return !isFLAT(MIb) && !isSMRD(MIb);
2523  }
2524 
2525  if (isSMRD(MIa)) {
2526  if (isSMRD(MIb))
2527  return checkInstOffsetsDoNotOverlap(MIa, MIb);
2528 
2529  return !isFLAT(MIb) && !isMUBUF(MIa) && !isMTBUF(MIa);
2530  }
2531 
2532  if (isFLAT(MIa)) {
2533  if (isFLAT(MIb))
2534  return checkInstOffsetsDoNotOverlap(MIa, MIb);
2535 
2536  return false;
2537  }
2538 
2539  return false;
2540 }
2541 
2542 static int64_t getFoldableImm(const MachineOperand* MO) {
2543  if (!MO->isReg())
2544  return false;
2545  const MachineFunction *MF = MO->getParent()->getParent()->getParent();
2546  const MachineRegisterInfo &MRI = MF->getRegInfo();
2547  auto Def = MRI.getUniqueVRegDef(MO->getReg());
2548  if (Def && Def->getOpcode() == AMDGPU::V_MOV_B32_e32 &&
2549  Def->getOperand(1).isImm())
2550  return Def->getOperand(1).getImm();
2551  return AMDGPU::NoRegister;
2552 }
2553 
2555  MachineInstr &MI,
2556  LiveVariables *LV) const {
2557  unsigned Opc = MI.getOpcode();
2558  bool IsF16 = false;
2559  bool IsFMA = Opc == AMDGPU::V_FMAC_F32_e32 || Opc == AMDGPU::V_FMAC_F32_e64 ||
2560  Opc == AMDGPU::V_FMAC_F16_e32 || Opc == AMDGPU::V_FMAC_F16_e64;
2561 
2562  switch (Opc) {
2563  default:
2564  return nullptr;
2565  case AMDGPU::V_MAC_F16_e64:
2566  case AMDGPU::V_FMAC_F16_e64:
2567  IsF16 = true;
2569  case AMDGPU::V_MAC_F32_e64:
2570  case AMDGPU::V_FMAC_F32_e64:
2571  break;
2572  case AMDGPU::V_MAC_F16_e32:
2573  case AMDGPU::V_FMAC_F16_e32:
2574  IsF16 = true;
2576  case AMDGPU::V_MAC_F32_e32:
2577  case AMDGPU::V_FMAC_F32_e32: {
2578  int Src0Idx = AMDGPU::getNamedOperandIdx(MI.getOpcode(),
2579  AMDGPU::OpName::src0);
2580  const MachineOperand *Src0 = &MI.getOperand(Src0Idx);
2581  if (!Src0->isReg() && !Src0->isImm())
2582  return nullptr;
2583 
2584  if (Src0->isImm() && !isInlineConstant(MI, Src0Idx, *Src0))
2585  return nullptr;
2586 
2587  break;
2588  }
2589  }
2590 
2591  const MachineOperand *Dst = getNamedOperand(MI, AMDGPU::OpName::vdst);
2592  const MachineOperand *Src0 = getNamedOperand(MI, AMDGPU::OpName::src0);
2593  const MachineOperand *Src0Mods =
2594  getNamedOperand(MI, AMDGPU::OpName::src0_modifiers);
2595  const MachineOperand *Src1 = getNamedOperand(MI, AMDGPU::OpName::src1);
2596  const MachineOperand *Src1Mods =
2597  getNamedOperand(MI, AMDGPU::OpName::src1_modifiers);
2598  const MachineOperand *Src2 = getNamedOperand(MI, AMDGPU::OpName::src2);
2599  const MachineOperand *Clamp = getNamedOperand(MI, AMDGPU::OpName::clamp);
2600  const MachineOperand *Omod = getNamedOperand(MI, AMDGPU::OpName::omod);
2601 
2602  if (!Src0Mods && !Src1Mods && !Clamp && !Omod &&
2603  // If we have an SGPR input, we will violate the constant bus restriction.
2604  (ST.getConstantBusLimit(Opc) > 1 ||
2605  !Src0->isReg() ||
2606  !RI.isSGPRReg(MBB->getParent()->getRegInfo(), Src0->getReg()))) {
2607  if (auto Imm = getFoldableImm(Src2)) {
2608  unsigned NewOpc =
2609  IsFMA ? (IsF16 ? AMDGPU::V_FMAAK_F16 : AMDGPU::V_FMAAK_F32)
2610  : (IsF16 ? AMDGPU::V_MADAK_F16 : AMDGPU::V_MADAK_F32);
2611  if (pseudoToMCOpcode(NewOpc) != -1)
2612  return BuildMI(*MBB, MI, MI.getDebugLoc(), get(NewOpc))
2613  .add(*Dst)
2614  .add(*Src0)
2615  .add(*Src1)
2616  .addImm(Imm);
2617  }
2618  unsigned NewOpc =
2619  IsFMA ? (IsF16 ? AMDGPU::V_FMAMK_F16 : AMDGPU::V_FMAMK_F32)
2620  : (IsF16 ? AMDGPU::V_MADMK_F16 : AMDGPU::V_MADMK_F32);
2621  if (auto Imm = getFoldableImm(Src1)) {
2622  if (pseudoToMCOpcode(NewOpc) != -1)
2623  return BuildMI(*MBB, MI, MI.getDebugLoc(), get(NewOpc))
2624  .add(*Dst)
2625  .add(*Src0)
2626  .addImm(Imm)
2627  .add(*Src2);
2628  }
2629  if (auto Imm = getFoldableImm(Src0)) {
2630  if (pseudoToMCOpcode(NewOpc) != -1 &&
2632  AMDGPU::OpName::src0), Src1))
2633  return BuildMI(*MBB, MI, MI.getDebugLoc(), get(NewOpc))
2634  .add(*Dst)
2635  .add(*Src1)
2636  .addImm(Imm)
2637  .add(*Src2);
2638  }
2639  }
2640 
2641  unsigned NewOpc = IsFMA ? (IsF16 ? AMDGPU::V_FMA_F16 : AMDGPU::V_FMA_F32)
2642  : (IsF16 ? AMDGPU::V_MAD_F16 : AMDGPU::V_MAD_F32);
2643  if (pseudoToMCOpcode(NewOpc) == -1)
2644  return nullptr;
2645 
2646  return BuildMI(*MBB, MI, MI.getDebugLoc(), get(NewOpc))
2647  .add(*Dst)
2648  .addImm(Src0Mods ? Src0Mods->getImm() : 0)
2649  .add(*Src0)
2650  .addImm(Src1Mods ? Src1Mods->getImm() : 0)
2651  .add(*Src1)
2652  .addImm(0) // Src mods
2653  .add(*Src2)
2654  .addImm(Clamp ? Clamp->getImm() : 0)
2655  .addImm(Omod ? Omod->getImm() : 0);
2656 }
2657 
2658 // It's not generally safe to move VALU instructions across these since it will
2659 // start using the register as a base index rather than directly.
2660 // XXX - Why isn't hasSideEffects sufficient for these?
2662  switch (MI.getOpcode()) {
2663  case AMDGPU::S_SET_GPR_IDX_ON:
2664  case AMDGPU::S_SET_GPR_IDX_MODE:
2665  case AMDGPU::S_SET_GPR_IDX_OFF:
2666  return true;
2667  default:
2668  return false;
2669  }
2670 }
2671 
2673  const MachineBasicBlock *MBB,
2674  const MachineFunction &MF) const {
2675  // XXX - Do we want the SP check in the base implementation?
2676 
2677  // Target-independent instructions do not have an implicit-use of EXEC, even
2678  // when they operate on VGPRs. Treating EXEC modifications as scheduling
2679  // boundaries prevents incorrect movements of such instructions.
2680  return TargetInstrInfo::isSchedulingBoundary(MI, MBB, MF) ||
2681  MI.modifiesRegister(AMDGPU::EXEC, &RI) ||
2682  MI.getOpcode() == AMDGPU::S_SETREG_IMM32_B32 ||
2683  MI.getOpcode() == AMDGPU::S_SETREG_B32 ||
2684  MI.getOpcode() == AMDGPU::S_DENORM_MODE ||
2686 }
2687 
2688 bool SIInstrInfo::isAlwaysGDS(uint16_t Opcode) const {
2689  return Opcode == AMDGPU::DS_ORDERED_COUNT ||
2690  Opcode == AMDGPU::DS_GWS_INIT ||
2691  Opcode == AMDGPU::DS_GWS_SEMA_V ||
2692  Opcode == AMDGPU::DS_GWS_SEMA_BR ||
2693  Opcode == AMDGPU::DS_GWS_SEMA_P ||
2694  Opcode == AMDGPU::DS_GWS_SEMA_RELEASE_ALL ||
2695  Opcode == AMDGPU::DS_GWS_BARRIER;
2696 }
2697 
2699  unsigned Opcode = MI.getOpcode();
2700 
2701  if (MI.mayStore() && isSMRD(MI))
2702  return true; // scalar store or atomic
2703 
2704  // This will terminate the function when other lanes may need to continue.
2705  if (MI.isReturn())
2706  return true;
2707 
2708  // These instructions cause shader I/O that may cause hardware lockups
2709  // when executed with an empty EXEC mask.
2710  //
2711  // Note: exp with VM = DONE = 0 is automatically skipped by hardware when
2712  // EXEC = 0, but checking for that case here seems not worth it
2713  // given the typical code patterns.
2714  if (Opcode == AMDGPU::S_SENDMSG || Opcode == AMDGPU::S_SENDMSGHALT ||
2715  Opcode == AMDGPU::EXP || Opcode == AMDGPU::EXP_DONE ||
2716  Opcode == AMDGPU::DS_ORDERED_COUNT || Opcode == AMDGPU::S_TRAP ||
2717  Opcode == AMDGPU::DS_GWS_INIT || Opcode == AMDGPU::DS_GWS_BARRIER)
2718  return true;
2719 
2720  if (MI.isCall() || MI.isInlineAsm())
2721  return true; // conservative assumption
2722 
2723  // These are like SALU instructions in terms of effects, so it's questionable
2724  // whether we should return true for those.
2725  //
2726  // However, executing them with EXEC = 0 causes them to operate on undefined
2727  // data, which we avoid by returning true here.
2728  if (Opcode == AMDGPU::V_READFIRSTLANE_B32 || Opcode == AMDGPU::V_READLANE_B32)
2729  return true;
2730 
2731  return false;
2732 }
2733 
2735  const MachineInstr &MI) const {
2736  if (MI.isMetaInstruction())
2737  return false;
2738 
2739  // This won't read exec if this is an SGPR->SGPR copy.
2740  if (MI.isCopyLike()) {
2741  if (!RI.isSGPRReg(MRI, MI.getOperand(0).getReg()))
2742  return true;
2743 
2744  // Make sure this isn't copying exec as a normal operand
2745  return MI.readsRegister(AMDGPU::EXEC, &RI);
2746  }
2747 
2748  // Make a conservative assumption about the callee.
2749  if (MI.isCall())
2750  return true;
2751 
2752  // Be conservative with any unhandled generic opcodes.
2753  if (!isTargetSpecificOpcode(MI.getOpcode()))
2754  return true;
2755 
2756  return !isSALU(MI) || MI.readsRegister(AMDGPU::EXEC, &RI);
2757 }
2758 
2759 bool SIInstrInfo::isInlineConstant(const APInt &Imm) const {
2760  switch (Imm.getBitWidth()) {
2761  case 1: // This likely will be a condition code mask.
2762  return true;
2763 
2764  case 32:
2766  ST.hasInv2PiInlineImm());
2767  case 64:
2769  ST.hasInv2PiInlineImm());
2770  case 16:
2771  return ST.has16BitInsts() &&
2773  ST.hasInv2PiInlineImm());
2774  default:
2775  llvm_unreachable("invalid bitwidth");
2776  }
2777 }
2778 
2780  uint8_t OperandType) const {
2781  if (!MO.isImm() ||
2782  OperandType < AMDGPU::OPERAND_SRC_FIRST ||
2783  OperandType > AMDGPU::OPERAND_SRC_LAST)
2784  return false;
2785 
2786  // MachineOperand provides no way to tell the true operand size, since it only
2787  // records a 64-bit value. We need to know the size to determine if a 32-bit
2788  // floating point immediate bit pattern is legal for an integer immediate. It
2789  // would be for any 32-bit integer operand, but would not be for a 64-bit one.
2790 
2791  int64_t Imm = MO.getImm();
2792  switch (OperandType) {
2799  int32_t Trunc = static_cast<int32_t>(Imm);
2801  }
2807  ST.hasInv2PiInlineImm());
2814  if (isInt<16>(Imm) || isUInt<16>(Imm)) {
2815  // A few special case instructions have 16-bit operands on subtargets
2816  // where 16-bit instructions are not legal.
2817  // TODO: Do the 32-bit immediates work? We shouldn't really need to handle
2818  // constants in these cases
2819  int16_t Trunc = static_cast<int16_t>(Imm);
2820  return ST.has16BitInsts() &&
2822  }
2823 
2824  return false;
2825  }
2832  uint32_t Trunc = static_cast<uint32_t>(Imm);
2834  }
2835  default:
2836  llvm_unreachable("invalid bitwidth");
2837  }
2838 }
2839 
2841  const MCOperandInfo &OpInfo) const {
2842  switch (MO.getType()) {
2844  return false;
2846  return !isInlineConstant(MO, OpInfo);
2852  return true;
2853  default:
2854  llvm_unreachable("unexpected operand type");
2855  }
2856 }
2857 
2858 static bool compareMachineOp(const MachineOperand &Op0,
2859  const MachineOperand &Op1) {
2860  if (Op0.getType() != Op1.getType())
2861  return false;
2862 
2863  switch (Op0.getType()) {
2865  return Op0.getReg() == Op1.getReg();
2867  return Op0.getImm() == Op1.getImm();
2868  default:
2869  llvm_unreachable("Didn't expect to be comparing these operand types");
2870  }
2871 }
2872 
2874  const MachineOperand &MO) const {
2875  const MCInstrDesc &InstDesc = MI.getDesc();
2876  const MCOperandInfo &OpInfo = InstDesc.OpInfo[OpNo];
2877 
2878  assert(MO.isImm() || MO.isTargetIndex() || MO.isFI() || MO.isGlobal());
2879 
2880  if (OpInfo.OperandType == MCOI::OPERAND_IMMEDIATE)
2881  return true;
2882 
2883  if (OpInfo.RegClass < 0)
2884  return false;
2885 
2886  const MachineFunction *MF = MI.getParent()->getParent();
2887  const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
2888 
2889  if (MO.isImm() && isInlineConstant(MO, OpInfo)) {
2890  if (isMAI(MI) && ST.hasMFMAInlineLiteralBug() &&
2892  AMDGPU::OpName::src2))
2893  return false;
2894  return RI.opCanUseInlineConstant(OpInfo.OperandType);
2895  }
2896 
2897  if (!RI.opCanUseLiteralConstant(OpInfo.OperandType))
2898  return false;
2899 
2900  if (!isVOP3(MI) || !AMDGPU::isSISrcOperand(InstDesc, OpNo))
2901  return true;
2902 
2903  return ST.hasVOP3Literal();
2904 }
2905 
2906 bool SIInstrInfo::hasVALU32BitEncoding(unsigned Opcode) const {
2907  int Op32 = AMDGPU::getVOPe32(Opcode);
2908  if (Op32 == -1)
2909  return false;
2910 
2911  return pseudoToMCOpcode(Op32) != -1;
2912 }
2913 
2914 bool SIInstrInfo::hasModifiers(unsigned Opcode) const {
2915  // The src0_modifier operand is present on all instructions
2916  // that have modifiers.
2917 
2918  return AMDGPU::getNamedOperandIdx(Opcode,
2919  AMDGPU::OpName::src0_modifiers) != -1;
2920 }
2921 
2923  unsigned OpName) const {
2924  const MachineOperand *Mods = getNamedOperand(MI, OpName);
2925  return Mods && Mods->getImm();
2926 }
2927 
2929  return hasModifiersSet(MI, AMDGPU::OpName::src0_modifiers) ||
2930  hasModifiersSet(MI, AMDGPU::OpName::src1_modifiers) ||
2931  hasModifiersSet(MI, AMDGPU::OpName::src2_modifiers) ||
2932  hasModifiersSet(MI, AMDGPU::OpName::clamp) ||
2933  hasModifiersSet(MI, AMDGPU::OpName::omod);
2934 }
2935 
2937  const MachineRegisterInfo &MRI) const {
2938  const MachineOperand *Src2 = getNamedOperand(MI, AMDGPU::OpName::src2);
2939  // Can't shrink instruction with three operands.
2940  // FIXME: v_cndmask_b32 has 3 operands and is shrinkable, but we need to add
2941  // a special case for it. It can only be shrunk if the third operand
2942  // is vcc, and src0_modifiers and src1_modifiers are not set.
2943  // We should handle this the same way we handle vopc, by addding
2944  // a register allocation hint pre-regalloc and then do the shrinking
2945  // post-regalloc.
2946  if (Src2) {
2947  switch (MI.getOpcode()) {
2948  default: return false;
2949 
2950  case AMDGPU::V_ADDC_U32_e64:
2951  case AMDGPU::V_SUBB_U32_e64:
2952  case AMDGPU::V_SUBBREV_U32_e64: {
2953  const MachineOperand *Src1
2954  = getNamedOperand(MI, AMDGPU::OpName::src1);
2955  if (!Src1->isReg() || !RI.isVGPR(MRI, Src1->getReg()))
2956  return false;
2957  // Additional verification is needed for sdst/src2.
2958  return true;
2959  }
2960  case AMDGPU::V_MAC_F32_e64:
2961  case AMDGPU::V_MAC_F16_e64:
2962  case AMDGPU::V_FMAC_F32_e64:
2963  case AMDGPU::V_FMAC_F16_e64:
2964  if (!Src2->isReg() || !RI.isVGPR(MRI, Src2->getReg()) ||
2965  hasModifiersSet(MI, AMDGPU::OpName::src2_modifiers))
2966  return false;
2967  break;
2968 
2969  case AMDGPU::V_CNDMASK_B32_e64:
2970  break;
2971  }
2972  }
2973 
2974  const MachineOperand *Src1 = getNamedOperand(MI, AMDGPU::OpName::src1);
2975  if (Src1 && (!Src1->isReg() || !RI.isVGPR(MRI, Src1->getReg()) ||
2976  hasModifiersSet(MI, AMDGPU::OpName::src1_modifiers)))
2977  return false;
2978 
2979  // We don't need to check src0, all input types are legal, so just make sure
2980  // src0 isn't using any modifiers.
2981  if (hasModifiersSet(MI, AMDGPU::OpName::src0_modifiers))
2982  return false;
2983 
2984  // Can it be shrunk to a valid 32 bit opcode?
2985  if (!hasVALU32BitEncoding(MI.getOpcode()))
2986  return false;
2987 
2988  // Check output modifiers
2989  return !hasModifiersSet(MI, AMDGPU::OpName::omod) &&
2990  !hasModifiersSet(MI, AMDGPU::OpName::clamp);
2991 }
2992 
2993 // Set VCC operand with all flags from \p Orig, except for setting it as
2994 // implicit.
2996  const MachineOperand &Orig) {
2997 
2998  for (MachineOperand &Use : MI.implicit_operands()) {
2999  if (Use.isUse() && Use.getReg() == AMDGPU::VCC) {
3000  Use.setIsUndef(Orig.isUndef());
3001  Use.setIsKill(Orig.isKill());
3002  return;
3003  }
3004  }
3005 }
3006 
3008  unsigned Op32) const {
3009  MachineBasicBlock *MBB = MI.getParent();;
3010  MachineInstrBuilder Inst32 =
3011  BuildMI(*MBB, MI, MI.getDebugLoc(), get(Op32));
3012 
3013  // Add the dst operand if the 32-bit encoding also has an explicit $vdst.
3014  // For VOPC instructions, this is replaced by an implicit def of vcc.
3015  int Op32DstIdx = AMDGPU::getNamedOperandIdx(Op32, AMDGPU::OpName::vdst);
3016  if (Op32DstIdx != -1) {
3017  // dst
3018  Inst32.add(MI.getOperand(0));
3019  } else {
3020  assert(((MI.getOperand(0).getReg() == AMDGPU::VCC) ||
3021  (MI.getOperand(0).getReg() == AMDGPU::VCC_LO)) &&
3022  "Unexpected case");
3023  }
3024 
3025  Inst32.add(*getNamedOperand(MI, AMDGPU::OpName::src0));
3026 
3027  const MachineOperand *Src1 = getNamedOperand(MI, AMDGPU::OpName::src1);
3028  if (Src1)
3029  Inst32.add(*Src1);
3030 
3031  const MachineOperand *Src2 = getNamedOperand(MI, AMDGPU::OpName::src2);
3032 
3033  if (Src2) {
3034  int Op32Src2Idx = AMDGPU::getNamedOperandIdx(Op32, AMDGPU::OpName::src2);
3035  if (Op32Src2Idx != -1) {
3036  Inst32.add(*Src2);
3037  } else {
3038  // In the case of V_CNDMASK_B32_e32, the explicit operand src2 is
3039  // replaced with an implicit read of vcc. This was already added
3040  // during the initial BuildMI, so find it to preserve the flags.
3041  copyFlagsToImplicitVCC(*Inst32, *Src2);
3042  }
3043  }
3044 
3045  return Inst32;
3046 }
3047 
3049  const MachineOperand &MO,
3050  const MCOperandInfo &OpInfo) const {
3051  // Literal constants use the constant bus.
3052  //if (isLiteralConstantLike(MO, OpInfo))
3053  // return true;
3054  if (MO.isImm())
3055  return !isInlineConstant(MO, OpInfo);
3056 
3057  if (!MO.isReg())
3058  return true; // Misc other operands like FrameIndex
3059 
3060  if (!MO.isUse())
3061  return false;
3062 
3064  return RI.isSGPRClass(MRI.getRegClass(MO.getReg()));
3065 
3066  // Null is free
3067  if (MO.getReg() == AMDGPU::SGPR_NULL)
3068  return false;
3069 
3070  // SGPRs use the constant bus
3071  if (MO.isImplicit()) {
3072  return MO.getReg() == AMDGPU::M0 ||
3073  MO.getReg() == AMDGPU::VCC ||
3074  MO.getReg() == AMDGPU::VCC_LO;
3075  } else {
3076  return AMDGPU::SReg_32RegClass.contains(MO.getReg()) ||
3077  AMDGPU::SReg_64RegClass.contains(MO.getReg());
3078  }
3079 }
3080 
3081 static unsigned findImplicitSGPRRead(const MachineInstr &MI) {
3082  for (const MachineOperand &MO : MI.implicit_operands()) {
3083  // We only care about reads.
3084  if (MO.isDef())
3085  continue;
3086 
3087  switch (MO.getReg()) {
3088  case AMDGPU::VCC:
3089  case AMDGPU::VCC_LO:
3090  case AMDGPU::VCC_HI:
3091  case AMDGPU::M0:
3092  case AMDGPU::FLAT_SCR:
3093  return MO.getReg();
3094 
3095  default:
3096  break;
3097  }
3098  }
3099 
3100  return AMDGPU::NoRegister;
3101 }
3102 
3103 static bool shouldReadExec(const MachineInstr &MI) {
3104  if (SIInstrInfo::isVALU(MI)) {
3105  switch (MI.getOpcode()) {
3106  case AMDGPU::V_READLANE_B32:
3107  case AMDGPU::V_READLANE_B32_gfx6_gfx7:
3108  case AMDGPU::V_READLANE_B32_gfx10:
3109  case AMDGPU::V_READLANE_B32_vi:
3110  case AMDGPU::V_WRITELANE_B32:
3111  case AMDGPU::V_WRITELANE_B32_gfx6_gfx7:
3112  case AMDGPU::V_WRITELANE_B32_gfx10:
3113  case AMDGPU::V_WRITELANE_B32_vi:
3114  return false;
3115  }
3116 
3117  return true;
3118  }
3119 
3120  if (MI.isPreISelOpcode() ||
3121  SIInstrInfo::isGenericOpcode(MI.getOpcode()) ||
3122  SIInstrInfo::isSALU(MI) ||
3123  SIInstrInfo::isSMRD(MI))
3124  return false;
3125 
3126  return true;
3127 }
3128 
3129 static bool isSubRegOf(const SIRegisterInfo &TRI,
3130  const MachineOperand &SuperVec,
3131  const MachineOperand &SubReg) {
3132  if (Register::isPhysicalRegister(SubReg.getReg()))
3133  return TRI.isSubRegister(SuperVec.getReg(), SubReg.getReg());
3134 
3135  return SubReg.getSubReg() != AMDGPU::NoSubRegister &&
3136  SubReg.getReg() == SuperVec.getReg();
3137 }
3138 
3140  StringRef &ErrInfo) const {
3141  uint16_t Opcode = MI.getOpcode();
3142  if (SIInstrInfo::isGenericOpcode(MI.getOpcode()))
3143  return true;
3144 
3145  const MachineFunction *MF = MI.getParent()->getParent();
3146  const MachineRegisterInfo &MRI = MF->getRegInfo();
3147 
3148  int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0);
3149  int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src1);
3150  int Src2Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src2);
3151 
3152  // Make sure the number of operands is correct.
3153  const MCInstrDesc &Desc = get(Opcode);
3154  if (!Desc.isVariadic() &&
3155  Desc.getNumOperands() != MI.getNumExplicitOperands()) {
3156  ErrInfo = "Instruction has wrong number of operands.";
3157  return false;
3158  }
3159 
3160  if (MI.isInlineAsm()) {
3161  // Verify register classes for inlineasm constraints.
3162  for (unsigned I = InlineAsm::MIOp_FirstOperand, E = MI.getNumOperands();
3163  I != E; ++I) {
3164  const TargetRegisterClass *RC = MI.getRegClassConstraint(I, this, &RI);
3165  if (!RC)
3166  continue;
3167 
3168  const MachineOperand &Op = MI.getOperand(I);
3169  if (!Op.isReg())
3170  continue;
3171 
3172  Register Reg = Op.getReg();
3173  if (!Register::isVirtualRegister(Reg) && !RC->contains(Reg)) {
3174  ErrInfo = "inlineasm operand has incorrect register class.";
3175  return false;
3176  }
3177  }
3178 
3179  return true;
3180  }
3181 
3182  // Make sure the register classes are correct.
3183  for (int i = 0, e = Desc.getNumOperands(); i != e; ++i) {
3184  if (MI.getOperand(i).isFPImm()) {
3185  ErrInfo = "FPImm Machine Operands are not supported. ISel should bitcast "
3186  "all fp values to integers.";
3187  return false;
3188  }
3189 
3190  int RegClass = Desc.OpInfo[i].RegClass;
3191 
3192  switch (Desc.OpInfo[i].OperandType) {
3194  if (MI.getOperand(i).isImm() || MI.getOperand(i).isGlobal()) {
3195  ErrInfo = "Illegal immediate value for operand.";
3196  return false;
3197  }
3198  break;
3201  break;
3212  const MachineOperand &MO = MI.getOperand(i);
3213  if (!MO.isReg() && (!MO.isImm() || !isInlineConstant(MI, i))) {
3214  ErrInfo = "Illegal immediate value for operand.";
3215  return false;
3216  }
3217  break;
3218  }
3221  // Check if this operand is an immediate.
3222  // FrameIndex operands will be replaced by immediates, so they are
3223  // allowed.
3224  if (!MI.getOperand(i).isImm() && !MI.getOperand(i).isFI()) {
3225  ErrInfo = "Expected immediate, but got non-immediate";
3226  return false;
3227  }
3229  default:
3230  continue;
3231  }
3232 
3233  if (!MI.getOperand(i).isReg())
3234  continue;
3235 
3236  if (RegClass != -1) {
3237  Register Reg = MI.getOperand(i).getReg();
3238  if (Reg == AMDGPU::NoRegister || Register::isVirtualRegister(Reg))
3239  continue;
3240 
3241  const TargetRegisterClass *RC = RI.getRegClass(RegClass);
3242  if (!RC->contains(Reg)) {
3243  ErrInfo = "Operand has incorrect register class.";
3244  return false;
3245  }
3246  }
3247  }
3248 
3249  // Verify SDWA
3250  if (isSDWA(MI)) {
3251  if (!ST.hasSDWA()) {
3252  ErrInfo = "SDWA is not supported on this target";
3253  return false;
3254  }
3255 
3256  int DstIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdst);
3257 
3258  const int OpIndicies[] = { DstIdx, Src0Idx, Src1Idx, Src2Idx };
3259 
3260  for (int OpIdx: OpIndicies) {
3261  if (OpIdx == -1)
3262  continue;
3263  const MachineOperand &MO = MI.getOperand(OpIdx);
3264 
3265  if (!ST.hasSDWAScalar()) {
3266  // Only VGPRS on VI
3267  if (!MO.isReg() || !RI.hasVGPRs(RI.getRegClassForReg(MRI, MO.getReg()))) {
3268  ErrInfo = "Only VGPRs allowed as operands in SDWA instructions on VI";
3269  return false;
3270  }
3271  } else {
3272  // No immediates on GFX9
3273  if (!MO.isReg()) {
3274  ErrInfo = "Only reg allowed as operands in SDWA instructions on GFX9";
3275  return false;
3276  }
3277  }
3278  }
3279 
3280  if (!ST.hasSDWAOmod()) {
3281  // No omod allowed on VI
3282  const MachineOperand *OMod = getNamedOperand(MI, AMDGPU::OpName::omod);
3283  if (OMod != nullptr &&
3284  (!OMod->isImm() || OMod->getImm() != 0)) {
3285  ErrInfo = "OMod not allowed in SDWA instructions on VI";
3286  return false;
3287  }
3288  }
3289 
3290  uint16_t BasicOpcode = AMDGPU::getBasicFromSDWAOp(Opcode);
3291  if (isVOPC(BasicOpcode)) {
3292  if (!ST.hasSDWASdst() && DstIdx != -1) {
3293  // Only vcc allowed as dst on VI for VOPC
3294  const MachineOperand &Dst = MI.getOperand(DstIdx);
3295  if (!Dst.isReg() || Dst.getReg() != AMDGPU::VCC) {
3296  ErrInfo = "Only VCC allowed as dst in SDWA instructions on VI";
3297  return false;
3298  }
3299  } else if (!ST.hasSDWAOutModsVOPC()) {
3300  // No clamp allowed on GFX9 for VOPC
3301  const MachineOperand *Clamp = getNamedOperand(MI, AMDGPU::OpName::clamp);
3302  if (Clamp && (!Clamp->isImm() || Clamp->getImm() != 0)) {
3303  ErrInfo = "Clamp not allowed in VOPC SDWA instructions on VI";
3304  return false;
3305  }
3306 
3307  // No omod allowed on GFX9 for VOPC
3308  const MachineOperand *OMod = getNamedOperand(MI, AMDGPU::OpName::omod);
3309  if (OMod && (!OMod->isImm() || OMod->getImm() != 0)) {
3310  ErrInfo = "OMod not allowed in VOPC SDWA instructions on VI";
3311  return false;
3312  }
3313  }
3314  }
3315 
3316  const MachineOperand *DstUnused = getNamedOperand(MI, AMDGPU::OpName::dst_unused);
3317  if (DstUnused && DstUnused->isImm() &&
3318  DstUnused->getImm() == AMDGPU::SDWA::UNUSED_PRESERVE) {
3319  const MachineOperand &Dst = MI.getOperand(DstIdx);
3320  if (!Dst.isReg() || !Dst.isTied()) {
3321  ErrInfo = "Dst register should have tied register";
3322  return false;
3323  }
3324 
3325  const MachineOperand &TiedMO =
3326  MI.getOperand(MI.findTiedOperandIdx(DstIdx));
3327  if (!TiedMO.isReg() || !TiedMO.isImplicit() || !TiedMO.isUse()) {
3328  ErrInfo =
3329  "Dst register should be tied to implicit use of preserved register";
3330  return false;
3331  } else if (Register::isPhysicalRegister(TiedMO.getReg()) &&
3332  Dst.getReg() != TiedMO.getReg()) {
3333  ErrInfo = "Dst register should use same physical register as preserved";
3334  return false;
3335  }
3336  }
3337  }
3338 
3339  // Verify MIMG
3340  if (isMIMG(MI.getOpcode()) && !MI.mayStore()) {
3341  // Ensure that the return type used is large enough for all the options
3342  // being used TFE/LWE require an extra result register.
3343  const MachineOperand *DMask = getNamedOperand(MI, AMDGPU::OpName::dmask);
3344  if (DMask) {
3345  uint64_t DMaskImm = DMask->getImm();
3346  uint32_t RegCount =
3347  isGather4(MI.getOpcode()) ? 4 : countPopulation(DMaskImm);
3348  const MachineOperand *TFE = getNamedOperand(MI, AMDGPU::OpName::tfe);
3349  const MachineOperand *LWE = getNamedOperand(MI, AMDGPU::OpName::lwe);
3350  const MachineOperand *D16 = getNamedOperand(MI, AMDGPU::OpName::d16);
3351 
3352  // Adjust for packed 16 bit values
3353  if (D16 && D16->getImm() && !ST.hasUnpackedD16VMem())
3354  RegCount >>= 1;
3355 
3356  // Adjust if using LWE or TFE
3357  if ((LWE && LWE->getImm()) || (TFE && TFE->getImm()))
3358  RegCount += 1;
3359 
3360  const uint32_t DstIdx =
3361  AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::vdata);
3362  const MachineOperand &Dst = MI.getOperand(DstIdx);
3363  if (Dst.isReg()) {
3364  const TargetRegisterClass *DstRC = getOpRegClass(MI, DstIdx);
3365  uint32_t DstSize = RI.getRegSizeInBits(*DstRC) / 32;
3366  if (RegCount > DstSize) {
3367  ErrInfo = "MIMG instruction returns too many registers for dst "
3368  "register class";
3369  return false;
3370  }
3371  }
3372  }
3373  }
3374 
3375  // Verify VOP*. Ignore multiple sgpr operands on writelane.
3376  if (Desc.getOpcode() != AMDGPU::V_WRITELANE_B32
3377  && (isVOP1(MI) || isVOP2(MI) || isVOP3(MI) || isVOPC(MI) || isSDWA(MI))) {
3378  // Only look at the true operands. Only a real operand can use the constant
3379  // bus, and we don't want to check pseudo-operands like the source modifier
3380  // flags.
3381  const int OpIndices[] = { Src0Idx, Src1Idx, Src2Idx };
3382 
3383  unsigned ConstantBusCount = 0;
3384  unsigned LiteralCount = 0;
3385 
3386  if (AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::imm) != -1)
3387  ++ConstantBusCount;
3388 
3389  SmallVector<unsigned, 2> SGPRsUsed;
3390  unsigned SGPRUsed = findImplicitSGPRRead(MI);
3391  if (SGPRUsed != AMDGPU::NoRegister) {
3392  ++ConstantBusCount;
3393  SGPRsUsed.push_back(SGPRUsed);
3394  }
3395 
3396  for (int OpIdx : OpIndices) {
3397  if (OpIdx == -1)
3398  break;
3399  const MachineOperand &MO = MI.getOperand(OpIdx);
3400  if (usesConstantBus(MRI, MO, MI.getDesc().OpInfo[OpIdx])) {
3401  if (MO.isReg()) {
3402  SGPRUsed = MO.getReg();
3403  if (llvm::all_of(SGPRsUsed, [this, SGPRUsed](unsigned SGPR) {
3404  return !RI.regsOverlap(SGPRUsed, SGPR);
3405  })) {
3406  ++ConstantBusCount;
3407  SGPRsUsed.push_back(SGPRUsed);
3408  }
3409  } else {
3410  ++ConstantBusCount;
3411  ++LiteralCount;
3412  }
3413  }
3414  }
3415  const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
3416  // v_writelane_b32 is an exception from constant bus restriction:
3417  // vsrc0 can be sgpr, const or m0 and lane select sgpr, m0 or inline-const
3418  if (ConstantBusCount > ST.getConstantBusLimit(Opcode) &&
3419  Opcode != AMDGPU::V_WRITELANE_B32) {
3420  ErrInfo = "VOP* instruction violates constant bus restriction";
3421  return false;
3422  }
3423 
3424  if (isVOP3(MI) && LiteralCount) {
3425  if (LiteralCount && !ST.hasVOP3Literal()) {
3426  ErrInfo = "VOP3 instruction uses literal";
3427  return false;
3428  }
3429  if (LiteralCount > 1) {
3430  ErrInfo = "VOP3 instruction uses more than one literal";
3431  return false;
3432  }
3433  }
3434  }
3435 
3436  // Verify misc. restrictions on specific instructions.
3437  if (Desc.getOpcode() == AMDGPU::V_DIV_SCALE_F32 ||
3438  Desc.getOpcode() == AMDGPU::V_DIV_SCALE_F64) {
3439  const MachineOperand &Src0 = MI.getOperand(Src0Idx);
3440  const MachineOperand &Src1 = MI.getOperand(Src1Idx);
3441  const MachineOperand &Src2 = MI.getOperand(Src2Idx);
3442  if (Src0.isReg() && Src1.isReg() && Src2.isReg()) {
3443  if (!compareMachineOp(Src0, Src1) &&
3444  !compareMachineOp(Src0, Src2)) {
3445  ErrInfo = "v_div_scale_{f32|f64} require src0 = src1 or src2";
3446  return false;
3447  }
3448  }
3449  }
3450 
3451  if (isSOP2(MI) || isSOPC(MI)) {
3452  const MachineOperand &Src0 = MI.getOperand(Src0Idx);
3453  const MachineOperand &Src1 = MI.getOperand(Src1Idx);
3454  unsigned Immediates = 0;
3455 
3456  if (!Src0.isReg() &&
3457  !isInlineConstant(Src0, Desc.OpInfo[Src0Idx].OperandType))
3458  Immediates++;
3459  if (!Src1.isReg() &&
3460  !isInlineConstant(Src1, Desc.OpInfo[Src1Idx].OperandType))
3461  Immediates++;
3462 
3463  if (Immediates > 1) {
3464  ErrInfo = "SOP2/SOPC instruction requires too many immediate constants";
3465  return false;
3466  }
3467  }
3468 
3469  if (isSOPK(MI)) {
3470  auto Op = getNamedOperand(MI, AMDGPU::OpName::simm16);
3471  if (Desc.isBranch()) {
3472  if (!Op->isMBB()) {
3473  ErrInfo = "invalid branch target for SOPK instruction";
3474  return false;
3475  }
3476  } else {
3477  uint64_t Imm = Op->getImm();
3478  if (sopkIsZext(MI)) {
3479  if (!isUInt<16>(Imm)) {
3480  ErrInfo = "invalid immediate for SOPK instruction";
3481  return false;
3482  }
3483  } else {
3484  if (!isInt<16>(Imm)) {
3485  ErrInfo = "invalid immediate for SOPK instruction";
3486  return false;
3487  }
3488  }
3489  }
3490  }
3491 
3492  if (Desc.getOpcode() == AMDGPU::V_MOVRELS_B32_e32 ||
3493  Desc.getOpcode() == AMDGPU::V_MOVRELS_B32_e64 ||
3494  Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e32 ||
3495  Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e64) {
3496  const bool IsDst = Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e32 ||
3497  Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e64;
3498 
3499  const unsigned StaticNumOps = Desc.getNumOperands() +
3500  Desc.getNumImplicitUses();
3501  const unsigned NumImplicitOps = IsDst ? 2 : 1;
3502 
3503  // Allow additional implicit operands. This allows a fixup done by the post
3504  // RA scheduler where the main implicit operand is killed and implicit-defs
3505  // are added for sub-registers that remain live after this instruction.
3506  if (MI.getNumOperands() < StaticNumOps + NumImplicitOps) {
3507  ErrInfo = "missing implicit register operands";
3508  return false;
3509  }
3510 
3511  const MachineOperand *Dst = getNamedOperand(MI, AMDGPU::OpName::vdst);
3512  if (IsDst) {
3513  if (!Dst->isUse()) {
3514  ErrInfo = "v_movreld_b32 vdst should be a use operand";
3515  return false;
3516  }
3517 
3518  unsigned UseOpIdx;
3519  if (!MI.isRegTiedToUseOperand(StaticNumOps, &UseOpIdx) ||
3520  UseOpIdx != StaticNumOps + 1) {
3521  ErrInfo = "movrel implicit operands should be tied";
3522  return false;
3523  }
3524  }
3525 
3526  const MachineOperand &Src0 = MI.getOperand(Src0Idx);
3527  const MachineOperand &ImpUse
3528  = MI.getOperand(StaticNumOps + NumImplicitOps - 1);
3529  if (!ImpUse.isReg() || !ImpUse.isUse() ||
3530  !isSubRegOf(RI, ImpUse, IsDst ? *Dst : Src0)) {
3531  ErrInfo = "src0 should be subreg of implicit vector use";
3532  return false;
3533  }
3534  }
3535 
3536  // Make sure we aren't losing exec uses in the td files. This mostly requires
3537  // being careful when using let Uses to try to add other use registers.
3538  if (shouldReadExec(MI)) {
3539  if (!MI.hasRegisterImplicitUseOperand(AMDGPU::EXEC)) {
3540  ErrInfo = "VALU instruction does not implicitly read exec mask";
3541  return false;
3542  }
3543  }
3544 
3545  if (isSMRD(MI)) {
3546  if (MI.mayStore()) {
3547  // The register offset form of scalar stores may only use m0 as the
3548  // soffset register.
3549  const MachineOperand *Soff = getNamedOperand(MI, AMDGPU::OpName::soff);
3550  if (Soff && Soff->getReg() != AMDGPU::M0) {
3551  ErrInfo = "scalar stores must use m0 as offset register";
3552  return false;
3553  }
3554  }
3555  }
3556 
3557  if (isFLAT(MI) && !MF->getSubtarget<GCNSubtarget>().hasFlatInstOffsets()) {
3558  const MachineOperand *Offset = getNamedOperand(MI, AMDGPU::OpName::offset);
3559  if (Offset->getImm() != 0) {
3560  ErrInfo = "subtarget does not support offsets in flat instructions";
3561  return false;
3562  }
3563  }
3564 
3565  if (isMIMG(MI)) {
3566  const MachineOperand *DimOp = getNamedOperand(MI, AMDGPU::OpName::dim);
3567  if (DimOp) {
3568  int VAddr0Idx = AMDGPU::getNamedOperandIdx(Opcode,
3569  AMDGPU::OpName::vaddr0);
3570  int SRsrcIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::srsrc);
3571  const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(Opcode);
3572  const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode =
3574  const AMDGPU::MIMGDimInfo *Dim =
3576 
3577  if (!Dim) {
3578  ErrInfo = "dim is out of range";
3579  return false;
3580  }
3581 
3582  bool IsNSA = SRsrcIdx - VAddr0Idx > 1;
3583  unsigned AddrWords = BaseOpcode->NumExtraArgs +
3584  (BaseOpcode->Gradients ? Dim->NumGradients : 0) +
3585  (BaseOpcode->Coordinates ? Dim->NumCoords : 0) +
3586  (BaseOpcode->LodOrClampOrMip ? 1 : 0);
3587 
3588  unsigned VAddrWords;
3589  if (IsNSA) {
3590  VAddrWords = SRsrcIdx - VAddr0Idx;
3591  } else {
3592  const TargetRegisterClass *RC = getOpRegClass(MI, VAddr0Idx);
3593  VAddrWords = MRI.getTargetRegisterInfo()->getRegSizeInBits(*RC) / 32;
3594  if (AddrWords > 8)
3595  AddrWords = 16;
3596  else if (AddrWords > 4)
3597  AddrWords = 8;
3598  else if (AddrWords == 3 && VAddrWords == 4) {
3599  // CodeGen uses the V4 variant of instructions for three addresses,
3600  // because the selection DAG does not support non-power-of-two types.
3601  AddrWords = 4;
3602  }
3603  }
3604 
3605  if (VAddrWords != AddrWords) {
3606  ErrInfo = "bad vaddr size";
3607  return false;
3608  }
3609  }
3610  }
3611 
3612  const MachineOperand *DppCt = getNamedOperand(MI, AMDGPU::OpName::dpp_ctrl);
3613  if (DppCt) {
3614  using namespace AMDGPU::DPP;
3615 
3616  unsigned DC = DppCt->getImm();
3617  if (DC == DppCtrl::DPP_UNUSED1 || DC == DppCtrl::DPP_UNUSED2 ||
3618  DC == DppCtrl::DPP_UNUSED3 || DC > DppCtrl::DPP_LAST ||
3624  ErrInfo = "Invalid dpp_ctrl value";
3625  return false;
3626  }
3627  if (DC >= DppCtrl::WAVE_SHL1 && DC <= DppCtrl::WAVE_ROR1 &&
3628  ST.getGeneration() >= AMDGPUSubtarget::GFX10) {
3629  ErrInfo = "Invalid dpp_ctrl value: "
3630  "wavefront shifts are not supported on GFX10+";
3631  return false;
3632  }
3633  if (DC >= DppCtrl::BCAST15 && DC <= DppCtrl::BCAST31 &&
3634  ST.getGeneration() >= AMDGPUSubtarget::GFX10) {
3635  ErrInfo = "Invalid dpp_ctrl value: "
3636  "broadcasts are not supported on GFX10+";
3637  return false;
3638  }
3640  ST.getGeneration() < AMDGPUSubtarget::GFX10) {
3641  ErrInfo = "Invalid dpp_ctrl value: "
3642  "row_share and row_xmask are not supported before GFX10";
3643  return false;
3644  }
3645  }
3646 
3647  return true;
3648 }
3649 
3650 unsigned SIInstrInfo::getVALUOp(const MachineInstr &MI) const {
3651  switch (MI.getOpcode()) {
3652  default: return AMDGPU::INSTRUCTION_LIST_END;
3653  case AMDGPU::REG_SEQUENCE: return AMDGPU::REG_SEQUENCE;
3654  case AMDGPU::COPY: return AMDGPU::COPY;
3655  case AMDGPU::PHI: return AMDGPU::PHI;
3656  case AMDGPU::INSERT_SUBREG: return AMDGPU::INSERT_SUBREG;
3657  case AMDGPU::WQM: return AMDGPU::WQM;
3658  case AMDGPU::SOFT_WQM: return AMDGPU::SOFT_WQM;
3659  case AMDGPU::WWM: return AMDGPU::WWM;
3660  case AMDGPU::S_MOV_B32: {
3662  return MI.getOperand(1).isReg() ||
3663  RI.isAGPR(MRI, MI.getOperand(0).getReg()) ?
3664  AMDGPU::COPY : AMDGPU::V_MOV_B32_e32;
3665  }
3666  case AMDGPU::S_ADD_I32:
3667  return ST.hasAddNoCarry() ? AMDGPU::V_ADD_U32_e64 : AMDGPU::V_ADD_I32_e32;
3668  case AMDGPU::S_ADDC_U32:
3669  return AMDGPU::V_ADDC_U32_e32;
3670  case AMDGPU::S_SUB_I32:
3671  return ST.hasAddNoCarry() ? AMDGPU::V_SUB_U32_e64 : AMDGPU::V_SUB_I32_e32;
3672  // FIXME: These are not consistently handled, and selected when the carry is
3673  // used.
3674  case AMDGPU::S_ADD_U32:
3675  return AMDGPU::V_ADD_I32_e32;
3676  case AMDGPU::S_SUB_U32:
3677  return AMDGPU::V_SUB_I32_e32;
3678  case AMDGPU::S_SUBB_U32: return AMDGPU::V_SUBB_U32_e32;
3679  case AMDGPU::S_MUL_I32: return AMDGPU::V_MUL_LO_U32;
3680  case AMDGPU::S_MUL_HI_U32: return AMDGPU::V_MUL_HI_U32;
3681  case AMDGPU::S_MUL_HI_I32: return AMDGPU::V_MUL_HI_I32;
3682  case AMDGPU::S_AND_B32: return AMDGPU::V_AND_B32_e64;
3683  case AMDGPU::S_OR_B32: return AMDGPU::V_OR_B32_e64;
3684  case AMDGPU::S_XOR_B32: return AMDGPU::V_XOR_B32_e64;
3685  case AMDGPU::S_XNOR_B32:
3686  return ST.hasDLInsts() ? AMDGPU::V_XNOR_B32_e64 : AMDGPU::INSTRUCTION_LIST_END;
3687  case AMDGPU::S_MIN_I32: return AMDGPU::V_MIN_I32_e64;
3688  case AMDGPU::S_MIN_U32: return AMDGPU::V_MIN_U32_e64;
3689  case AMDGPU::S_MAX_I32: return AMDGPU::V_MAX_I32_e64;
3690  case AMDGPU::S_MAX_U32: return AMDGPU::V_MAX_U32_e64;
3691  case AMDGPU::S_ASHR_I32: return AMDGPU::V_ASHR_I32_e32;
3692  case AMDGPU::S_ASHR_I64: return AMDGPU::V_ASHR_I64;
3693  case AMDGPU::S_LSHL_B32: return AMDGPU::V_LSHL_B32_e32;
3694  case AMDGPU::S_LSHL_B64: return AMDGPU::V_LSHL_B64;
3695  case AMDGPU::S_LSHR_B32: return AMDGPU::V_LSHR_B32_e32;
3696  case AMDGPU::S_LSHR_B64: return AMDGPU::V_LSHR_B64;
3697  case AMDGPU::S_SEXT_I32_I8: return AMDGPU::V_BFE_I32;
3698  case AMDGPU::S_SEXT_I32_I16: return AMDGPU::V_BFE_I32;
3699  case AMDGPU::S_BFE_U32: return AMDGPU::V_BFE_U32;
3700  case AMDGPU::S_BFE_I32: return AMDGPU::V_BFE_I32;
3701  case AMDGPU::S_BFM_B32: return AMDGPU::V_BFM_B32_e64;
3702  case AMDGPU::S_BREV_B32: return AMDGPU::V_BFREV_B32_e32;
3703  case AMDGPU::S_NOT_B32: return AMDGPU::V_NOT_B32_e32;
3704  case AMDGPU::S_NOT_B64: return AMDGPU::V_NOT_B32_e32;
3705  case AMDGPU::S_CMP_EQ_I32: return AMDGPU::V_CMP_EQ_I32_e32;
3706  case AMDGPU::S_CMP_LG_I32: return AMDGPU::V_CMP_NE_I32_e32;
3707  case AMDGPU::S_CMP_GT_I32: return AMDGPU::V_CMP_GT_I32_e32;
3708  case AMDGPU::S_CMP_GE_I32: return AMDGPU::V_CMP_GE_I32_e32;
3709  case AMDGPU::S_CMP_LT_I32: return AMDGPU::V_CMP_LT_I32_e32;
3710  case AMDGPU::S_CMP_LE_I32: return AMDGPU::V_CMP_LE_I32_e32;
3711  case AMDGPU::S_CMP_EQ_U32: return AMDGPU::V_CMP_EQ_U32_e32;
3712  case AMDGPU::S_CMP_LG_U32: return AMDGPU::V_CMP_NE_U32_e32;
3713  case AMDGPU::S_CMP_GT_U32: return AMDGPU::V_CMP_GT_U32_e32;
3714  case AMDGPU::S_CMP_GE_U32: return AMDGPU::V_CMP_GE_U32_e32;
3715  case AMDGPU::S_CMP_LT_U32: return AMDGPU::V_CMP_LT_U32_e32;
3716  case AMDGPU::S_CMP_LE_U32: return AMDGPU::V_CMP_LE_U32_e32;
3717  case AMDGPU::S_CMP_EQ_U64: return AMDGPU::V_CMP_EQ_U64_e32;
3718  case AMDGPU::S_CMP_LG_U64: return AMDGPU::V_CMP_NE_U64_e32;
3719  case AMDGPU::S_BCNT1_I32_B32: return AMDGPU::V_BCNT_U32_B32_e64;
3720  case AMDGPU::S_FF1_I32_B32: return AMDGPU::V_FFBL_B32_e32;
3721  case AMDGPU::S_FLBIT_I32_B32: return AMDGPU::V_FFBH_U32_e32;
3722  case AMDGPU::S_FLBIT_I32: return AMDGPU::V_FFBH_I32_e64;
3723  case AMDGPU::S_CBRANCH_SCC0: return AMDGPU::S_CBRANCH_VCCZ;
3724  case AMDGPU::S_CBRANCH_SCC1: return AMDGPU::S_CBRANCH_VCCNZ;
3725  }
3727  "Unexpected scalar opcode without corresponding vector one!");
3728 }
3729 
3731  unsigned OpNo) const {
3733  const MCInstrDesc &Desc = get(MI.getOpcode());
3734  if (MI.isVariadic() || OpNo >= Desc.getNumOperands() ||
3735  Desc.OpInfo[OpNo].RegClass == -1) {
3736  Register Reg = MI.getOperand(OpNo).getReg();
3737 
3738  if (Register::isVirtualRegister(Reg))
3739  return MRI.getRegClass(Reg);
3740  return RI.getPhysRegClass(Reg);
3741  }
3742 
3743  unsigned RCID = Desc.OpInfo[OpNo].RegClass;
3744  return RI.getRegClass(RCID);
3745 }
3746 
3747 void SIInstrInfo::legalizeOpWithMove(MachineInstr &MI, unsigned OpIdx) const {
3749  MachineBasicBlock *MBB = MI.getParent();
3750  MachineOperand &MO = MI.getOperand(OpIdx);
3752  const SIRegisterInfo *TRI =
3753  static_cast<const SIRegisterInfo*>(MRI.getTargetRegisterInfo());
3754  unsigned RCID = get(MI.getOpcode()).OpInfo[OpIdx].RegClass;
3755  const TargetRegisterClass *RC = RI.getRegClass(RCID);
3756  unsigned Size = TRI->getRegSizeInBits(*RC);
3757  unsigned Opcode = (Size == 64) ? AMDGPU::V_MOV_B64_PSEUDO : AMDGPU::V_MOV_B32_e32;
3758  if (MO.isReg())
3759  Opcode = AMDGPU::COPY;
3760  else if (RI.isSGPRClass(RC))
3761  Opcode = (Size == 64) ? AMDGPU::S_MOV_B64 : AMDGPU::S_MOV_B32;
3762 
3763  const TargetRegisterClass *VRC = RI.getEquivalentVGPRClass(RC);
3764  if (RI.getCommonSubClass(&AMDGPU::VReg_64RegClass, VRC))
3765  VRC = &AMDGPU::VReg_64RegClass;
3766  else
3767  VRC = &AMDGPU::VGPR_32RegClass;
3768 
3769  Register Reg = MRI.createVirtualRegister(VRC);
3770  DebugLoc DL = MBB->findDebugLoc(I);
3771  BuildMI(*MI.getParent(), I, DL, get(Opcode), Reg).add(MO);
3772  MO.ChangeToRegister(Reg, false);
3773 }
3774 
3777  MachineOperand &SuperReg,
3778  const TargetRegisterClass *SuperRC,
3779  unsigned SubIdx,
3780  const TargetRegisterClass *SubRC)
3781  const {
3782  MachineBasicBlock *MBB = MI->getParent();
3783  DebugLoc DL = MI->getDebugLoc();
3784  Register SubReg = MRI.createVirtualRegister(SubRC);
3785 
3786  if (SuperReg.getSubReg() == AMDGPU::NoSubRegister) {
3787  BuildMI(*MBB, MI, DL, get(TargetOpcode::COPY), SubReg)
3788  .addReg(SuperReg.getReg(), 0, SubIdx);
3789  return SubReg;
3790  }
3791 
3792  // Just in case the super register is itself a sub-register, copy it to a new
3793  // value so we don't need to worry about merging its subreg index with the
3794  // SubIdx passed to this function. The register coalescer should be able to
3795  // eliminate this extra copy.
3796  Register NewSuperReg = MRI.createVirtualRegister(SuperRC);
3797 
3798  BuildMI(*MBB, MI, DL, get(TargetOpcode::COPY), NewSuperReg)
3799  .addReg(SuperReg.getReg(), 0, SuperReg.getSubReg());
3800 
3801  BuildMI(*MBB, MI, DL, get(TargetOpcode::COPY), SubReg)
3802  .addReg(NewSuperReg, 0, SubIdx);
3803 
3804  return SubReg;
3805 }
3806 
3810  MachineOperand &Op,
3811  const TargetRegisterClass *SuperRC,
3812  unsigned SubIdx,
3813  const TargetRegisterClass *SubRC) const {
3814  if (Op.isImm()) {
3815  if (SubIdx == AMDGPU::sub0)
3816  return MachineOperand::CreateImm(static_cast<int32_t>(Op.getImm()));
3817  if (SubIdx == AMDGPU::sub1)
3818  return MachineOperand::CreateImm(static_cast<int32_t>(Op.getImm() >> 32));
3819 
3820  llvm_unreachable("Unhandled register index for immediate");
3821  }
3822 
3823  unsigned SubReg = buildExtractSubReg(MII, MRI, Op, SuperRC,
3824  SubIdx, SubRC);
3825  return MachineOperand::CreateReg(SubReg, false);
3826 }
3827 
3828 // Change the order of operands from (0, 1, 2) to (0, 2, 1)
3829 void SIInstrInfo::swapOperands(MachineInstr &Inst) const {
3830  assert(Inst.getNumExplicitOperands() == 3);
3831  MachineOperand Op1 = Inst.getOperand(1);
3832  Inst.RemoveOperand(1);
3833  Inst.addOperand(Op1);
3834 }
3835 
3837  const MCOperandInfo &OpInfo,
3838  const MachineOperand &MO) const {
3839  if (!MO.isReg())
3840  return false;
3841 
3842  Register Reg = MO.getReg();
3844  ? MRI.getRegClass(Reg)
3845  : RI.getPhysRegClass(Reg);
3846 
3847  const SIRegisterInfo *TRI =
3848  static_cast<const SIRegisterInfo*>(MRI.getTargetRegisterInfo());
3849  RC = TRI->getSubRegClass(RC, MO.getSubReg());
3850 
3851  // In order to be legal, the common sub-class must be equal to the
3852  // class of the current operand. For example:
3853  //
3854  // v_mov_b32 s0 ; Operand defined as vsrc_b32
3855  // ; RI.getCommonSubClass(s0,vsrc_b32) = sgpr ; LEGAL
3856  //
3857  // s_sendmsg 0, s0 ; Operand defined as m0reg
3858  // ; RI.getCommonSubClass(s0,m0reg) = m0reg ; NOT LEGAL
3859 
3860  return RI.getCommonSubClass(RC, RI.getRegClass(OpInfo.RegClass)) == RC;
3861 }
3862 
3864  const MCOperandInfo &OpInfo,
3865  const MachineOperand &MO) const {
3866  if (MO.isReg())
3867  return isLegalRegOperand(MRI, OpInfo, MO);
3868 
3869  // Handle non-register types that are treated like immediates.
3870  assert(MO.isImm() || MO.isTargetIndex() || MO.isFI() || MO.isGlobal());
3871  return true;
3872 }
3873 
3874 bool SIInstrInfo::isOperandLegal(const MachineInstr &MI, unsigned OpIdx,
3875  const MachineOperand *MO) const {
3876  const MachineFunction &MF = *MI.getParent()->getParent();
3877  const MachineRegisterInfo &MRI = MF.getRegInfo();
3878  const MCInstrDesc &InstDesc = MI.getDesc();
3879  const MCOperandInfo &OpInfo = InstDesc.OpInfo[OpIdx];
3880  const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
3881  const TargetRegisterClass *DefinedRC =
3882  OpInfo.RegClass != -1 ? RI.getRegClass(OpInfo.RegClass) : nullptr;
3883  if (!MO)
3884  MO = &MI.getOperand(OpIdx);
3885 
3886  int ConstantBusLimit = ST.getConstantBusLimit(MI.getOpcode());
3887  int VOP3LiteralLimit = ST.hasVOP3Literal() ? 1 : 0;
3888  if (isVALU(MI) && usesConstantBus(MRI, *MO, OpInfo)) {
3889  if (isVOP3(MI) && isLiteralConstantLike(*MO, OpInfo) && !VOP3LiteralLimit--)
3890  return false;
3891 
3892  SmallDenseSet<RegSubRegPair> SGPRsUsed;
3893  if (MO->isReg())
3894  SGPRsUsed.insert(RegSubRegPair(MO->getReg(), MO->getSubReg()));
3895 
3896  for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
3897  if (i == OpIdx)
3898  continue;
3899  const MachineOperand &Op = MI.getOperand(i);
3900  if (Op.isReg()) {
3901  RegSubRegPair SGPR(Op.getReg(), Op.getSubReg());
3902  if (!SGPRsUsed.count(SGPR) &&
3903  usesConstantBus(MRI, Op, InstDesc.OpInfo[i])) {
3904  if (--ConstantBusLimit <= 0)
3905  return false;
3906  SGPRsUsed.insert(SGPR);
3907  }
3908  } else if (InstDesc.OpInfo[i].OperandType == AMDGPU::OPERAND_KIMM32) {
3909  if (--ConstantBusLimit <= 0)
3910  return false;
3911  } else if (isVOP3(MI) && AMDGPU::isSISrcOperand(InstDesc, i) &&
3912  isLiteralConstantLike(Op, InstDesc.OpInfo[i])) {
3913  if (!VOP3LiteralLimit--)
3914  return false;
3915  if (--ConstantBusLimit <= 0)
3916  return false;
3917  }
3918  }
3919  }
3920 
3921  if (MO->isReg()) {
3922  assert(DefinedRC);
3923  return isLegalRegOperand(MRI, OpInfo, *MO);
3924  }
3925 
3926  // Handle non-register types that are treated like immediates.
3927  assert(MO->isImm() || MO->isTargetIndex() || MO->isFI() || MO->isGlobal());
3928 
3929  if (!DefinedRC) {
3930  // This operand expects an immediate.
3931  return true;
3932  }
3933 
3934  return isImmOperandLegal(MI, OpIdx, *MO);
3935 }
3936 
3938  MachineInstr &MI) const {
3939  unsigned Opc = MI.getOpcode();
3940  const MCInstrDesc &InstrDesc = get(Opc);
3941 
3942  int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0);
3943  MachineOperand &Src0 = MI.getOperand(Src0Idx);
3944 
3945  int Src1Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1);
3946  MachineOperand &Src1 = MI.getOperand(Src1Idx);
3947 
3948  // If there is an implicit SGPR use such as VCC use for v_addc_u32/v_subb_u32
3949  // we need to only have one constant bus use before GFX10.
3950  bool HasImplicitSGPR = findImplicitSGPRRead(MI) != AMDGPU::NoRegister;
3951  if (HasImplicitSGPR && ST.getConstantBusLimit(Opc) <= 1 &&
3952  Src0.isReg() && (RI.isSGPRReg(MRI, Src0.getReg()) ||
3953  isLiteralConstantLike(Src0, InstrDesc.OpInfo[Src0Idx])))
3954  legalizeOpWithMove(MI, Src0Idx);
3955 
3956  // Special case: V_WRITELANE_B32 accepts only immediate or SGPR operands for
3957  // both the value to write (src0) and lane select (src1). Fix up non-SGPR
3958  // src0/src1 with V_READFIRSTLANE.
3959  if (Opc == AMDGPU::V_WRITELANE_B32) {
3960  const DebugLoc &DL = MI.getDebugLoc();
3961  if (Src0.isReg() && RI.isVGPR(MRI, Src0.getReg())) {
3962  Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
3963  BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg)
3964  .add(Src0);
3965  Src0.ChangeToRegister(Reg, false);
3966  }
3967  if (Src1.isReg() && RI.isVGPR(MRI, Src1.getReg())) {
3968  Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
3969  const DebugLoc &DL = MI.getDebugLoc();
3970  BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg)
3971  .add(Src1);
3972  Src1.ChangeToRegister(Reg, false);
3973  }
3974  return;
3975  }
3976 
3977  // No VOP2 instructions support AGPRs.
3978  if (Src0.isReg() && RI.isAGPR(MRI, Src0.getReg()))
3979  legalizeOpWithMove(MI, Src0Idx);
3980 
3981  if (Src1.isReg() && RI.isAGPR(MRI, Src1.getReg()))
3982  legalizeOpWithMove(MI, Src1Idx);
3983 
3984  // VOP2 src0 instructions support all operand types, so we don't need to check
3985  // their legality. If src1 is already legal, we don't need to do anything.
3986  if (isLegalRegOperand(MRI, InstrDesc.OpInfo[Src1Idx], Src1))
3987  return;
3988 
3989  // Special case: V_READLANE_B32 accepts only immediate or SGPR operands for
3990  // lane select. Fix up using V_READFIRSTLANE, since we assume that the lane
3991  // select is uniform.
3992  if (Opc == AMDGPU::V_READLANE_B32 && Src1.isReg() &&
3993  RI.isVGPR(MRI, Src1.getReg())) {
3994  Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
3995  const DebugLoc &DL = MI.getDebugLoc();
3996  BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg)
3997  .add(Src1);
3998  Src1.ChangeToRegister(Reg, false);
3999  return;
4000  }
4001 
4002  // We do not use commuteInstruction here because it is too aggressive and will
4003  // commute if it is possible. We only want to commute here if it improves
4004  // legality. This can be called a fairly large number of times so don't waste
4005  // compile time pointlessly swapping and checking legality again.
4006  if (HasImplicitSGPR || !MI.isCommutable()) {
4007  legalizeOpWithMove(MI, Src1Idx);
4008  return;
4009  }
4010 
4011  // If src0 can be used as src1, commuting will make the operands legal.
4012  // Otherwise we have to give up and insert a move.
4013  //
4014  // TODO: Other immediate-like operand kinds could be commuted if there was a
4015  // MachineOperand::ChangeTo* for them.
4016  if ((!Src1.isImm() && !Src1.isReg()) ||
4017  !isLegalRegOperand(MRI, InstrDesc.OpInfo[Src1Idx], Src0)) {
4018  legalizeOpWithMove(MI, Src1Idx);
4019  return;
4020  }
4021 
4022  int CommutedOpc = commuteOpcode(MI);
4023  if (CommutedOpc == -1) {
4024  legalizeOpWithMove(MI, Src1Idx);
4025  return;
4026  }
4027 
4028  MI.setDesc(get(CommutedOpc));
4029 
4030  Register Src0Reg = Src0.getReg();
4031  unsigned Src0SubReg = Src0.getSubReg();
4032  bool Src0Kill = Src0.isKill();
4033 
4034  if (Src1.isImm())
4035  Src0.ChangeToImmediate(Src1.getImm());
4036  else if (Src1.isReg()) {
4037  Src0.ChangeToRegister(Src1.getReg(), false, false, Src1.isKill());
4038  Src0.setSubReg(Src1.getSubReg());
4039  } else
4040  llvm_unreachable("Should only have register or immediate operands");
4041 
4042  Src1.ChangeToRegister(Src0Reg, false, false, Src0Kill);
4043  Src1.setSubReg(Src0SubReg);
4044  fixImplicitOperands(MI);
4045 }
4046 
4047 // Legalize VOP3 operands. All operand types are supported for any operand
4048 // but only one literal constant and only starting from GFX10.
4050  MachineInstr &MI) const {
4051  unsigned Opc = MI.getOpcode();
4052 
4053  int VOP3Idx[3] = {
4054  AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0),
4055  AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1),
4056  AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2)
4057  };
4058 
4059  if (Opc == AMDGPU::V_PERMLANE16_B32 ||
4060  Opc == AMDGPU::V_PERMLANEX16_B32) {
4061  // src1 and src2 must be scalar
4062  MachineOperand &Src1 = MI.getOperand(VOP3Idx[1]);
4063  MachineOperand &Src2 = MI.getOperand(VOP3Idx[2]);
4064  const DebugLoc &DL = MI.getDebugLoc();
4065  if (Src1.isReg() && !RI.isSGPRClass(MRI.getRegClass(Src1.getReg()))) {
4066  Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
4067  BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg)
4068  .add(Src1);
4069  Src1.ChangeToRegister(Reg, false);
4070  }
4071  if (Src2.isReg() && !RI.isSGPRClass(MRI.getRegClass(Src2.getReg()))) {
4072  Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
4073  BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg)
4074  .add(Src2);
4075  Src2.ChangeToRegister(Reg, false);
4076  }
4077  }
4078 
4079  // Find the one SGPR operand we are allowed to use.
4080  int ConstantBusLimit = ST.getConstantBusLimit(Opc);
4081  int LiteralLimit = ST.hasVOP3Literal() ? 1 : 0;
4082  SmallDenseSet<unsigned> SGPRsUsed;
4083  unsigned SGPRReg = findUsedSGPR(MI, VOP3Idx);
4084  if (SGPRReg != AMDGPU::NoRegister) {
4085  SGPRsUsed.insert(SGPRReg);
4086  --ConstantBusLimit;
4087  }
4088 
4089  for (unsigned i = 0; i < 3; ++i) {
4090  int Idx = VOP3Idx[i];
4091  if (Idx == -1)
4092  break;
4093  MachineOperand &MO = MI.getOperand(Idx);
4094 
4095  if (!MO.isReg()) {
4096  if (!isLiteralConstantLike(MO, get(Opc).OpInfo[Idx]))
4097  continue;
4098 
4099  if (LiteralLimit > 0 && ConstantBusLimit > 0) {
4100  --LiteralLimit;
4101  --ConstantBusLimit;
4102  continue;
4103  }
4104 
4105  --LiteralLimit;
4106  --ConstantBusLimit;
4107  legalizeOpWithMove(MI, Idx);
4108  continue;
4109  }
4110 
4111  if (RI.hasAGPRs(MRI.getRegClass(MO.getReg())) &&
4112  !isOperandLegal(MI, Idx, &MO)) {
4113  legalizeOpWithMove(MI, Idx);
4114  continue;
4115  }
4116 
4117  if (!RI.isSGPRClass(MRI.getRegClass(MO.getReg())))
4118  continue; // VGPRs are legal
4119 
4120  // We can use one SGPR in each VOP3 instruction prior to GFX10
4121  // and two starting from GFX10.
4122  if (SGPRsUsed.count(MO.getReg()))
4123  continue;
4124  if (ConstantBusLimit > 0) {
4125  SGPRsUsed.insert(MO.getReg());
4126  --ConstantBusLimit;
4127  continue;
4128  }
4129 
4130  // If we make it this far, then the operand is not legal and we must
4131  // legalize it.
4132  legalizeOpWithMove(MI, Idx);
4133  }
4134 }
4135 
4137  MachineRegisterInfo &MRI) const {
4138  const TargetRegisterClass *VRC = MRI.getRegClass(SrcReg);
4139  const TargetRegisterClass *SRC = RI.getEquivalentSGPRClass(VRC);
4140  Register DstReg = MRI.createVirtualRegister(SRC);
4141  unsigned SubRegs = RI.getRegSizeInBits(*VRC) / 32;
4142 
4143  if (RI.hasAGPRs(VRC)) {
4144  VRC = RI.getEquivalentVGPRClass(VRC);
4145  Register NewSrcReg = MRI.createVirtualRegister(VRC);
4146  BuildMI(*UseMI.getParent(), UseMI, UseMI.getDebugLoc(),
4147  get(TargetOpcode::COPY), NewSrcReg)
4148  .addReg(SrcReg);
4149  SrcReg = NewSrcReg;
4150  }
4151 
4152  if (SubRegs == 1) {
4153  BuildMI(*UseMI.getParent(), UseMI, UseMI.getDebugLoc(),
4154  get(AMDGPU::V_READFIRSTLANE_B32), DstReg)
4155  .addReg(SrcReg);
4156  return DstReg;
4157  }
4158 
4160  for (unsigned i = 0; i < SubRegs; ++i) {
4161  Register SGPR = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
4162  BuildMI(*UseMI.getParent(), UseMI, UseMI.getDebugLoc(),
4163  get(AMDGPU::V_READFIRSTLANE_B32), SGPR)
4164  .addReg(SrcReg, 0, RI.getSubRegFromChannel(i));
4165  SRegs.push_back(SGPR);
4166  }
4167 
4168  MachineInstrBuilder MIB =
4169  BuildMI(*UseMI.getParent(), UseMI, UseMI.getDebugLoc(),
4170  get(AMDGPU::REG_SEQUENCE), DstReg);
4171  for (unsigned i = 0; i < SubRegs; ++i) {
4172  MIB.addReg(SRegs[i]);
4173  MIB.addImm(RI.getSubRegFromChannel(i));
4174  }
4175  return DstReg;
4176 }
4177 
4179  MachineInstr &MI) const {
4180 
4181  // If the pointer is store in VGPRs, then we need to move them to
4182  // SGPRs using v_readfirstlane. This is safe because we only select
4183  // loads with uniform pointers to SMRD instruction so we know the
4184  // pointer value is uniform.
4185  MachineOperand *SBase = getNamedOperand(MI, AMDGPU::OpName::sbase);
4186  if (SBase && !RI.isSGPRClass(MRI.getRegClass(SBase->getReg()))) {
4187  unsigned SGPR = readlaneVGPRToSGPR(SBase->getReg(), MI, MRI);
4188  SBase->setReg(SGPR);
4189  }
4190  MachineOperand *SOff = getNamedOperand(MI, AMDGPU::OpName::soff);
4191  if (SOff && !RI.isSGPRClass(MRI.getRegClass(SOff->getReg()))) {
4192  unsigned SGPR = readlaneVGPRToSGPR(SOff->getReg(), MI, MRI);
4193  SOff->setReg(SGPR);
4194  }
4195 }
4196 
4199  const TargetRegisterClass *DstRC,
4200  MachineOperand &Op,
4202  const DebugLoc &DL) const {
4203  Register OpReg = Op.getReg();
4204  unsigned OpSubReg = Op.getSubReg();
4205 
4206  const TargetRegisterClass *OpRC = RI.getSubClassWithSubReg(
4207  RI.getRegClassForReg(MRI, OpReg), OpSubReg);
4208 
4209  // Check if operand is already the correct register class.
4210  if (DstRC == OpRC)
4211  return;
4212 
4213  Register DstReg = MRI.createVirtualRegister(DstRC);
4214  MachineInstr *Copy =
4215  BuildMI(InsertMBB, I, DL, get(AMDGPU::COPY), DstReg).add(Op);
4216 
4217  Op.setReg(DstReg);
4218  Op.setSubReg(0);
4219 
4220  MachineInstr *Def = MRI.getVRegDef(OpReg);
4221  if (!Def)
4222  return;
4223 
4224  // Try to eliminate the copy if it is copying an immediate value.
4225  if (Def->isMoveImmediate())
4226  FoldImmediate(*Copy, *Def, OpReg, &MRI);
4227 
4228  bool ImpDef = Def->isImplicitDef();
4229  while (!ImpDef && Def && Def->isCopy()) {
4230  Def = MRI.getUniqueVRegDef(Def->getOperand(1).getReg());
4231  ImpDef = Def && Def->isImplicitDef();
4232  }
4233  if (!RI.isSGPRClass(DstRC) && !Copy->readsRegister(AMDGPU::EXEC, &RI) &&
4234  !ImpDef)
4235  Copy->addOperand(MachineOperand::CreateReg(AMDGPU::EXEC, false, true));
4236 }
4237 
4238 // Emit the actual waterfall loop, executing the wrapped instruction for each
4239 // unique value of \p Rsrc across all lanes. In the best case we execute 1
4240 // iteration, in the worst case we execute 64 (once per lane).
4241 static void
4243  MachineBasicBlock &OrigBB, MachineBasicBlock &LoopBB,
4244  const DebugLoc &DL, MachineOperand &Rsrc) {
4245  MachineFunction &MF = *OrigBB.getParent();
4246  const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
4247  const SIRegisterInfo *TRI = ST.getRegisterInfo();
4248  unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
4249  unsigned SaveExecOpc =
4250  ST.isWave32() ? AMDGPU::S_AND_SAVEEXEC_B32 : AMDGPU::S_AND_SAVEEXEC_B64;
4251  unsigned XorTermOpc =
4252  ST.isWave32() ? AMDGPU::S_XOR_B32_term : AMDGPU::S_XOR_B64_term;
4253  unsigned AndOpc =
4254  ST.isWave32() ? AMDGPU::S_AND_B32 : AMDGPU::S_AND_B64;
4255  const auto *BoolXExecRC = TRI->getRegClass(AMDGPU::SReg_1_XEXECRegClassID);
4256 
4257  MachineBasicBlock::iterator I = LoopBB.begin();
4258 
4259  Register VRsrc = Rsrc.getReg();
4260  unsigned VRsrcUndef = getUndefRegState(Rsrc.isUndef());
4261 
4262  Register SaveExec = MRI.createVirtualRegister(BoolXExecRC);
4263  Register CondReg0 = MRI.createVirtualRegister(BoolXExecRC);
4264  Register CondReg1 = MRI.createVirtualRegister(BoolXExecRC);
4265  Register AndCond = MRI.createVirtualRegister(BoolXExecRC);
4266  Register SRsrcSub0 = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
4267  Register SRsrcSub1 = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
4268  Register SRsrcSub2 = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
4269  Register SRsrcSub3 = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
4270  Register SRsrc = MRI.createVirtualRegister(&AMDGPU::SGPR_128RegClass);
4271 
4272  // Beginning of the loop, read the next Rsrc variant.
4273  BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), SRsrcSub0)
4274  .addReg(VRsrc, VRsrcUndef, AMDGPU::sub0);
4275  BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), SRsrcSub1)
4276  .addReg(VRsrc, VRsrcUndef, AMDGPU::sub1);
4277  BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), SRsrcSub2)
4278  .addReg(VRsrc, VRsrcUndef, AMDGPU::sub2);
4279  BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), SRsrcSub3)
4280  .addReg(VRsrc, VRsrcUndef, AMDGPU::sub3);
4281 
4282  BuildMI(LoopBB, I, DL, TII.get(AMDGPU::REG_SEQUENCE), SRsrc)
4283  .addReg(SRsrcSub0)
4284  .addImm(AMDGPU::sub0)
4285  .addReg(SRsrcSub1)
4286  .addImm(AMDGPU::sub1)
4287  .addReg(SRsrcSub2)
4288  .addImm(AMDGPU::sub2)
4289  .addReg(SRsrcSub3)
4290  .addImm(AMDGPU::sub3);
4291 
4292  // Update Rsrc operand to use the SGPR Rsrc.
4293  Rsrc.setReg(SRsrc);
4294  Rsrc.setIsKill(true);
4295 
4296  // Identify all lanes with identical Rsrc operands in their VGPRs.
4297  BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_CMP_EQ_U64_e64), CondReg0)
4298  .addReg(SRsrc, 0, AMDGPU::sub0_sub1)
4299  .addReg(VRsrc, 0, AMDGPU::sub0_sub1);
4300  BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_CMP_EQ_U64_e64), CondReg1)
4301  .addReg(SRsrc, 0, AMDGPU::sub2_sub3)
4302  .addReg(VRsrc, 0, AMDGPU::sub2_sub3);
4303  BuildMI(LoopBB, I, DL, TII.get(AndOpc), AndCond)
4304  .addReg(CondReg0)
4305  .addReg(CondReg1);
4306 
4307  MRI.setSimpleHint(SaveExec, AndCond);
4308 
4309  // Update EXEC to matching lanes, saving original to SaveExec.
4310  BuildMI(LoopBB, I, DL, TII.get(SaveExecOpc), SaveExec)
4311  .addReg(AndCond, RegState::Kill);
4312 
4313  // The original instruction is here; we insert the terminators after it.
4314  I = LoopBB.end();
4315 
4316  // Update EXEC, switch all done bits to 0 and all todo bits to 1.
4317  BuildMI(LoopBB, I, DL, TII.get(XorTermOpc), Exec)
4318  .addReg(Exec)
4319  .addReg(SaveExec);
4320  BuildMI(LoopBB, I, DL, TII.get(AMDGPU::S_CBRANCH_EXECNZ)).addMBB(&LoopBB);
4321 }
4322 
4323 // Build a waterfall loop around \p MI, replacing the VGPR \p Rsrc register
4324 // with SGPRs by iterating over all unique values across all lanes.
4326  MachineOperand &Rsrc, MachineDominatorTree *MDT) {
4327  MachineBasicBlock &MBB = *MI.getParent();
4328  MachineFunction &MF = *MBB.getParent();
4329  const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
4330  const SIRegisterInfo *TRI = ST.getRegisterInfo();
4333  const DebugLoc &DL = MI.getDebugLoc();
4334  unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
4335  unsigned MovExecOpc = ST.isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64;
4336  const auto *BoolXExecRC = TRI->getRegClass(AMDGPU::SReg_1_XEXECRegClassID);
4337 
4338  Register SaveExec = MRI.createVirtualRegister(BoolXExecRC);
4339 
4340  // Save the EXEC mask
4341  BuildMI(MBB, I, DL, TII.get(MovExecOpc), SaveExec).addReg(Exec);
4342 
4343  // Killed uses in the instruction we are waterfalling around will be
4344  // incorrect due to the added control-flow.
4345  for (auto &MO : MI.uses()) {
4346  if (MO.isReg() && MO.isUse()) {
4347  MRI.clearKillFlags(MO.getReg());
4348  }
4349  }
4350 
4351  // To insert the loop we need to split the block. Move everything after this
4352  // point to a new block, and insert a new empty block between the two.
4354  MachineBasicBlock *RemainderBB = MF.CreateMachineBasicBlock();
4355  MachineFunction::iterator MBBI(MBB);
4356  ++MBBI;
4357 
4358  MF.insert(MBBI, LoopBB);
4359  MF.insert(MBBI, RemainderBB);
4360 
4361  LoopBB->addSuccessor(LoopBB);
4362  LoopBB->addSuccessor(RemainderBB);
4363 
4364  // Move MI to the LoopBB, and the remainder of the block to RemainderBB.
4366  RemainderBB->transferSuccessorsAndUpdatePHIs(&MBB);
4367  RemainderBB->splice(RemainderBB->begin(), &MBB, I, MBB.end());
4368  LoopBB->splice(LoopBB->begin(), &MBB, J);
4369 
4370  MBB.addSuccessor(LoopBB);
4371 
4372  // Update dominators. We know that MBB immediately dominates LoopBB, that
4373  // LoopBB immediately dominates RemainderBB, and that RemainderBB immediately
4374  // dominates all of the successors transferred to it from MBB that MBB used
4375  // to dominate.
4376  if (MDT) {
4377  MDT->addNewBlock(LoopBB, &MBB);
4378  MDT->addNewBlock(RemainderBB, LoopBB);
4379  for (auto &Succ : RemainderBB->successors()) {
4380  if (MDT->dominates(&MBB, Succ)) {
4381  MDT->changeImmediateDominator(Succ, RemainderBB);
4382  }
4383  }
4384  }
4385 
4386  emitLoadSRsrcFromVGPRLoop(TII, MRI, MBB, *LoopBB, DL, Rsrc);
4387 
4388  // Restore the EXEC mask
4389  MachineBasicBlock::iterator First = RemainderBB->begin();
4390  BuildMI(*RemainderBB, First, DL, TII.get(MovExecOpc), Exec).addReg(SaveExec);
4391 }
4392 
4393 // Extract pointer from Rsrc and return a zero-value Rsrc replacement.
4394 static std::tuple<unsigned, unsigned>
4396  MachineBasicBlock &MBB = *MI.getParent();
4397  MachineFunction &MF = *MBB.getParent();
4399 
4400  // Extract the ptr from the resource descriptor.
4401  unsigned RsrcPtr =
4402  TII.buildExtractSubReg(MI, MRI, Rsrc, &AMDGPU::VReg_128RegClass,
4403  AMDGPU::sub0_sub1, &AMDGPU::VReg_64RegClass);
4404 
4405  // Create an empty resource descriptor
4406  Register Zero64 = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
4407  Register SRsrcFormatLo = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
4408  Register SRsrcFormatHi = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
4409  Register NewSRsrc = MRI.createVirtualRegister(&AMDGPU::SGPR_128RegClass);
4410  uint64_t RsrcDataFormat = TII.getDefaultRsrcDataFormat();
4411 
4412  // Zero64 = 0
4413  BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::S_MOV_B64), Zero64)
4414  .addImm(0);
4415 
4416  // SRsrcFormatLo = RSRC_DATA_FORMAT{31-0}
4417  BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::S_MOV_B32), SRsrcFormatLo)
4418  .addImm(RsrcDataFormat & 0xFFFFFFFF);
4419 
4420  // SRsrcFormatHi = RSRC_DATA_FORMAT{63-32}
4421  BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::S_MOV_B32), SRsrcFormatHi)
4422  .addImm(RsrcDataFormat >> 32);
4423 
4424  // NewSRsrc = {Zero64, SRsrcFormat}
4425  BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::REG_SEQUENCE), NewSRsrc)
4426  .addReg(Zero64)
4427  .addImm(AMDGPU::sub0_sub1)
4428  .addReg(SRsrcFormatLo)
4429  .addImm(AMDGPU::sub2)
4430  .addReg(SRsrcFormatHi)
4431  .addImm(AMDGPU::sub3);
4432 
4433  return std::make_tuple(RsrcPtr, NewSRsrc);
4434 }
4435 
4437  MachineDominatorTree *MDT) const {
4438  MachineFunction &MF = *MI.getParent()->getParent();
4440 
4441  // Legalize VOP2
4442  if (isVOP2(MI) || isVOPC(MI)) {
4443  legalizeOperandsVOP2(MRI, MI);
4444  return;
4445  }
4446 
4447  // Legalize VOP3
4448  if (isVOP3(MI)) {
4449  legalizeOperandsVOP3(MRI, MI);
4450  return;
4451  }
4452 
4453  // Legalize SMRD
4454  if (isSMRD(MI)) {
4455  legalizeOperandsSMRD(MRI, MI);
4456  return;
4457  }
4458 
4459  // Legalize REG_SEQUENCE and PHI
4460  // The register class of the operands much be the same type as the register
4461  // class of the output.
4462  if (MI.getOpcode() == AMDGPU::PHI) {
4463  const TargetRegisterClass *RC = nullptr, *SRC = nullptr, *VRC = nullptr;
4464  for (unsigned i = 1, e = MI.getNumOperands(); i != e; i += 2) {
4465  if (!MI.getOperand(i).isReg() ||
4467  continue;
4468  const TargetRegisterClass *OpRC =
4469  MRI.getRegClass(MI.getOperand(i).getReg());
4470  if (RI.hasVectorRegisters(OpRC)) {
4471  VRC = OpRC;
4472  } else {
4473  SRC = OpRC;
4474  }
4475  }
4476 
4477  // If any of the operands are VGPR registers, then they all most be
4478  // otherwise we will create illegal VGPR->SGPR copies when legalizing
4479  // them.
4480  if (VRC || !RI.isSGPRClass(getOpRegClass(MI, 0))) {
4481  if (!VRC) {
4482  assert(SRC);
4483  VRC = RI.hasAGPRs(getOpRegClass(MI, 0)) ? RI.getEquivalentAGPRClass(SRC)
4484  : RI.getEquivalentVGPRClass(SRC);
4485  }
4486  RC = VRC;
4487  } else {
4488  RC = SRC;
4489  }
4490 
4491  // Update all the operands so they have the same type.
4492  for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) {
4493  MachineOperand &Op = MI.getOperand(I);
4494  if (!Op.isReg() || !Register::isVirtualRegister(Op.getReg()))
4495  continue;
4496 
4497  // MI is a PHI instruction.
4498  MachineBasicBlock *InsertBB = MI.getOperand(I + 1).getMBB();
4499  MachineBasicBlock::iterator Insert = InsertBB->getFirstTerminator();
4500 
4501  // Avoid creating no-op copies with the same src and dst reg class. These
4502  // confuse some of the machine passes.
4503  legalizeGenericOperand(*InsertBB, Insert, RC, Op, MRI, MI.getDebugLoc());
4504  }
4505  }
4506 
4507  // REG_SEQUENCE doesn't really require operand legalization, but if one has a
4508  // VGPR dest type and SGPR sources, insert copies so all operands are
4509  // VGPRs. This seems to help operand folding / the register coalescer.
4510  if (MI.getOpcode() == AMDGPU::REG_SEQUENCE) {
4511  MachineBasicBlock *MBB = MI.getParent();
4512  const TargetRegisterClass *DstRC = getOpRegClass(MI, 0);
4513  if (RI.hasVGPRs(DstRC)) {
4514  // Update all the operands so they are VGPR register classes. These may
4515  // not be the same register class because REG_SEQUENCE supports mixing
4516  // subregister index types e.g. sub0_sub1 + sub2 + sub3
4517  for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) {
4518  MachineOperand &Op = MI.getOperand(I);
4519  if (!Op.isReg() || !Register::isVirtualRegister(Op.getReg()))
4520  continue;
4521 
4522  const TargetRegisterClass *OpRC = MRI.getRegClass(Op.getReg());
4523  const TargetRegisterClass *VRC = RI.getEquivalentVGPRClass(OpRC);
4524  if (VRC == OpRC)
4525  continue;
4526 
4527  legalizeGenericOperand(*MBB, MI, VRC, Op, MRI, MI.getDebugLoc());
4528  Op.setIsKill();
4529  }
4530  }
4531 
4532  return;
4533  }
4534 
4535  // Legalize INSERT_SUBREG
4536  // src0 must have the same register class as dst
4537  if (MI.getOpcode() == AMDGPU::INSERT_SUBREG) {
4538  Register Dst = MI.getOperand(0).getReg();
4539  Register Src0 = MI.getOperand(1).getReg();
4540  const TargetRegisterClass *DstRC = MRI.getRegClass(Dst);
4541  const TargetRegisterClass *Src0RC = MRI.getRegClass(Src0);
4542  if (DstRC != Src0RC) {
4543  MachineBasicBlock *MBB = MI.getParent();
4544  MachineOperand &Op = MI.getOperand(1);
4545  legalizeGenericOperand(*MBB, MI, DstRC, Op, MRI, MI.getDebugLoc());
4546  }
4547  return;
4548  }
4549 
4550  // Legalize SI_INIT_M0
4551  if (MI.getOpcode() == AMDGPU::SI_INIT_M0) {
4552  MachineOperand &Src = MI.getOperand(0);
4553  if (Src.isReg() && RI.hasVectorRegisters(MRI.getRegClass(Src.getReg())))
4554  Src.setReg(readlaneVGPRToSGPR(Src.getReg(), MI, MRI));
4555  return;
4556  }
4557 
4558  // Legalize MIMG and MUBUF/MTBUF for shaders.
4559  //
4560  // Shaders only generate MUBUF/MTBUF instructions via intrinsics or via
4561  // scratch memory access. In both cases, the legalization never involves
4562  // conversion to the addr64 form.
4563  if (isMIMG(MI) ||
4565  (isMUBUF(MI) || isMTBUF(MI)))) {
4566  MachineOperand *SRsrc = getNamedOperand(MI, AMDGPU::OpName::srsrc);
4567  if (SRsrc && !RI.isSGPRClass(MRI.getRegClass(SRsrc->getReg()))) {
4568  unsigned SGPR = readlaneVGPRToSGPR(SRsrc->getReg(), MI, MRI);
4569  SRsrc->setReg(SGPR);
4570  }
4571 
4572  MachineOperand *SSamp = getNamedOperand(MI, AMDGPU::OpName::ssamp);
4573  if (SSamp && !RI.isSGPRClass(MRI.getRegClass(SSamp->getReg()))) {
4574  unsigned SGPR = readlaneVGPRToSGPR(SSamp->getReg(), MI, MRI);
4575  SSamp->setReg(SGPR);
4576  }
4577  return;
4578  }
4579 
4580  // Legalize MUBUF* instructions.
4581  int RsrcIdx =
4582  AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::srsrc);
4583  if (RsrcIdx != -1) {
4584  // We have an MUBUF instruction
4585  MachineOperand *Rsrc = &MI.getOperand(RsrcIdx);
4586  unsigned RsrcRC = get(MI.getOpcode()).OpInfo[RsrcIdx].RegClass;
4587  if (RI.getCommonSubClass(MRI.getRegClass(Rsrc->getReg()),
4588  RI.getRegClass(RsrcRC))) {
4589  // The operands are legal.
4590  // FIXME: We may need to legalize operands besided srsrc.
4591  return;
4592  }
4593 
4594  // Legalize a VGPR Rsrc.
4595  //
4596  // If the instruction is _ADDR64, we can avoid a waterfall by extracting
4597  // the base pointer from the VGPR Rsrc, adding it to the VAddr, then using
4598  // a zero-value SRsrc.
4599  //
4600  // If the instruction is _OFFSET (both idxen and offen disabled), and we
4601  // support ADDR64 instructions, we can convert to ADDR64 and do the same as
4602  // above.
4603  //
4604  // Otherwise we are on non-ADDR64 hardware, and/or we have
4605  // idxen/offen/bothen and we fall back to a waterfall loop.
4606 
4607  MachineBasicBlock &MBB = *MI.getParent();
4608 
4609  MachineOperand *VAddr = getNamedOperand(MI, AMDGPU::OpName::vaddr);
4610  if (VAddr && AMDGPU::getIfAddr64Inst(MI.getOpcode()) != -1) {
4611  // This is already an ADDR64 instruction so we need to add the pointer
4612  // extracted from the resource descriptor to the current value of VAddr.
4613  Register NewVAddrLo = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4614  Register NewVAddrHi = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4615  Register NewVAddr = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass);
4616 
4617  const auto *BoolXExecRC = RI.getRegClass(AMDGPU::SReg_1_XEXECRegClassID);
4618  Register CondReg0 = MRI.createVirtualRegister(BoolXExecRC);
4619  Register CondReg1 = MRI.createVirtualRegister(BoolXExecRC);
4620 
4621  unsigned RsrcPtr, NewSRsrc;
4622  std::tie(RsrcPtr, NewSRsrc) = extractRsrcPtr(*this, MI, *Rsrc);
4623 
4624  // NewVaddrLo = RsrcPtr:sub0 + VAddr:sub0
4625  const DebugLoc &DL = MI.getDebugLoc();
4626  BuildMI(MBB, MI, DL, get(AMDGPU::V_ADD_I32_e64), NewVAddrLo)
4627  .addDef(CondReg0)
4628  .addReg(RsrcPtr, 0, AMDGPU::sub0)
4629  .addReg(VAddr->getReg(), 0, AMDGPU::sub0)
4630  .addImm(0);
4631 
4632  // NewVaddrHi = RsrcPtr:sub1 + VAddr:sub1
4633  BuildMI(MBB, MI, DL, get(AMDGPU::V_ADDC_U32_e64), NewVAddrHi)
4634  .addDef(CondReg1, RegState::Dead)
4635  .addReg(RsrcPtr, 0, AMDGPU::sub1)
4636  .addReg(VAddr->getReg(), 0, AMDGPU::sub1)
4637  .addReg(CondReg0, RegState::Kill)
4638  .addImm(0);
4639 
4640  // NewVaddr = {NewVaddrHi, NewVaddrLo}
4641  BuildMI(MBB, MI, MI.getDebugLoc(), get(AMDGPU::REG_SEQUENCE), NewVAddr)
4642  .addReg(NewVAddrLo)
4643  .addImm(AMDGPU::sub0)
4644  .addReg(NewVAddrHi)
4645  .addImm(AMDGPU::sub1);
4646 
4647  VAddr->setReg(NewVAddr);
4648  Rsrc->setReg(NewSRsrc);
4649  } else if (!VAddr && ST.hasAddr64()) {
4650  // This instructions is the _OFFSET variant, so we need to convert it to
4651  // ADDR64.
4652  assert(MBB.getParent()->getSubtarget<GCNSubtarget>().getGeneration()
4654  "FIXME: Need to emit flat atomics here");
4655 
4656  unsigned RsrcPtr, NewSRsrc;
4657  std::tie(RsrcPtr, NewSRsrc) = extractRsrcPtr(*this, MI, *Rsrc);
4658 
4659  Register NewVAddr = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass);
4660  MachineOperand *VData = getNamedOperand(MI, AMDGPU::OpName::vdata);
4661  MachineOperand *Offset = getNamedOperand(MI, AMDGPU::OpName::offset);
4662  MachineOperand *SOffset = getNamedOperand(MI, AMDGPU::OpName::soffset);
4663  unsigned Addr64Opcode = AMDGPU::getAddr64Inst(MI.getOpcode());
4664 
4665  // Atomics rith return have have an additional tied operand and are
4666  // missing some of the special bits.
4667  MachineOperand *VDataIn = getNamedOperand(MI, AMDGPU::OpName::vdata_in);
4668  MachineInstr *Addr64;
4669 
4670  if (!VDataIn) {
4671  // Regular buffer load / store.
4672  MachineInstrBuilder MIB =
4673  BuildMI(MBB, MI, MI.getDebugLoc(), get(Addr64Opcode))
4674  .add(*VData)
4675  .addReg(NewVAddr)
4676  .addReg(NewSRsrc)
4677  .add(*SOffset)
4678  .add(*Offset);
4679 
4680  // Atomics do not have this operand.
4681  if (const MachineOperand *GLC =
4682  getNamedOperand(MI, AMDGPU::OpName::glc)) {
4683  MIB.addImm(GLC->getImm());
4684  }
4685  if (const MachineOperand *DLC =
4686  getNamedOperand(MI, AMDGPU::OpName::dlc)) {
4687  MIB.addImm(DLC->getImm());
4688  }
4689 
4690  MIB.addImm(getNamedImmOperand(MI, AMDGPU::OpName::slc));
4691 
4692  if (const MachineOperand *TFE =
4693  getNamedOperand(MI, AMDGPU::OpName::tfe)) {
4694  MIB.addImm(TFE->getImm());
4695  }
4696 
4697  MIB.addImm(getNamedImmOperand(MI, AMDGPU::OpName::swz));
4698 
4699  MIB.cloneMemRefs(MI);
4700  Addr64 = MIB;
4701  } else {
4702  // Atomics with return.
4703  Addr64 = BuildMI(MBB, MI, MI.getDebugLoc(), get(Addr64Opcode))
4704  .add(*VData)
4705  .add(*VDataIn)
4706  .addReg(NewVAddr)
4707  .addReg(NewSRsrc)
4708  .add(*SOffset)
4709  .add(*Offset)
4710  .addImm(getNamedImmOperand(MI, AMDGPU::OpName::slc))
4711  .cloneMemRefs(MI);
4712  }
4713 
4714  MI.removeFromParent();
4715 
4716  // NewVaddr = {NewVaddrHi, NewVaddrLo}
4717  BuildMI(MBB, Addr64, Addr64->getDebugLoc(), get(AMDGPU::REG_SEQUENCE),
4718  NewVAddr)
4719  .addReg(RsrcPtr, 0, AMDGPU::sub0)
4720  .addImm(AMDGPU::sub0)
4721  .addReg(RsrcPtr, 0, AMDGPU::sub1)
4722  .addImm(AMDGPU::sub1);
4723  } else {
4724  // This is another variant; legalize Rsrc with waterfall loop from VGPRs
4725  // to SGPRs.
4726  loadSRsrcFromVGPR(*this, MI, *Rsrc, MDT);
4727  }
4728  }
4729 }
4730 
4732  MachineDominatorTree *MDT) const {
4733  SetVectorType Worklist;
4734  Worklist.insert(&TopInst);
4735 
4736  while (!Worklist.empty()) {
4737  MachineInstr &Inst = *Worklist.pop_back_val();
4738  MachineBasicBlock *MBB = Inst.getParent();
4740 
4741  unsigned Opcode = Inst.getOpcode();
4742  unsigned NewOpcode = getVALUOp(Inst);
4743 
4744  // Handle some special cases
4745  switch (Opcode) {
4746  default:
4747  break;
4748  case AMDGPU::S_ADD_U64_PSEUDO:
4749  case AMDGPU::S_SUB_U64_PSEUDO:
4750  splitScalar64BitAddSub(Worklist, Inst, MDT);
4751  Inst.eraseFromParent();
4752  continue;
4753  case AMDGPU::S_ADD_I32:
4754  case AMDGPU::S_SUB_I32:
4755  // FIXME: The u32 versions currently selected use the carry.
4756  if (moveScalarAddSub(Worklist, Inst, MDT))
4757  continue;
4758 
4759  // Default handling
4760  break;
4761  case AMDGPU::S_AND_B64:
4762  splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_AND_B32, MDT);
4763  Inst.eraseFromParent();
4764  continue;
4765 
4766  case AMDGPU::S_OR_B64:
4767  splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_OR_B32, MDT);
4768  Inst.eraseFromParent();
4769  continue;
4770 
4771  case AMDGPU::S_XOR_B64:
4772  splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_XOR_B32, MDT);
4773  Inst.eraseFromParent();
4774  continue;
4775 
4776  case AMDGPU::S_NAND_B64:
4777  splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_NAND_B32, MDT);
4778  Inst.eraseFromParent();
4779  continue;
4780 
4781  case AMDGPU::S_NOR_B64:
4782  splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_NOR_B32, MDT);
4783  Inst.eraseFromParent();
4784  continue;
4785 
4786  case AMDGPU::S_XNOR_B64:
4787  if (ST.hasDLInsts())
4788  splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_XNOR_B32, MDT);
4789  else
4790  splitScalar64BitXnor(Worklist, Inst, MDT);
4791  Inst.eraseFromParent();
4792  continue;
4793 
4794  case AMDGPU::S_ANDN2_B64:
4795  splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_ANDN2_B32, MDT);
4796  Inst.eraseFromParent();
4797  continue;
4798 
4799  case AMDGPU::S_ORN2_B64:
4800  splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_ORN2_B32, MDT);
4801  Inst.eraseFromParent();
4802  continue;
4803 
4804  case AMDGPU::S_NOT_B64:
4805  splitScalar64BitUnaryOp(Worklist, Inst, AMDGPU::S_NOT_B32);
4806  Inst.eraseFromParent();
4807  continue;
4808 
4809  case AMDGPU::S_BCNT1_I32_B64:
4810  splitScalar64BitBCNT(Worklist, Inst);
4811  Inst.eraseFromParent();
4812  continue;
4813 
4814  case AMDGPU::S_BFE_I64:
4815  splitScalar64BitBFE(Worklist, Inst);
4816  Inst.eraseFromParent();
4817  continue;
4818 
4819  case AMDGPU::S_LSHL_B32:
4820  if (ST.hasOnlyRevVALUShifts()) {
4821  NewOpcode = AMDGPU::V_LSHLREV_B32_e64;
4822  swapOperands(Inst);
4823  }
4824  break;
4825  case AMDGPU::S_ASHR_I32:
4826  if (ST.hasOnlyRevVALUShifts()) {
4827  NewOpcode = AMDGPU::V_ASHRREV_I32_e64;
4828  swapOperands(Inst);
4829  }
4830  break;
4831  case AMDGPU::S_LSHR_B32:
4832  if (ST.hasOnlyRevVALUShifts()) {
4833  NewOpcode = AMDGPU::V_LSHRREV_B32_e64;
4834  swapOperands(Inst);
4835  }
4836  break;
4837  case AMDGPU::S_LSHL_B64:
4838  if (ST.hasOnlyRevVALUShifts()) {
4839  NewOpcode = AMDGPU::V_LSHLREV_B64;
4840  swapOperands(Inst);
4841  }
4842  break;
4843  case AMDGPU::S_ASHR_I64:
4844  if (ST.hasOnlyRevVALUShifts()) {
4845  NewOpcode = AMDGPU::V_ASHRREV_I64;
4846  swapOperands(Inst);
4847  }
4848  break;
4849  case AMDGPU::S_LSHR_B64:
4850  if (ST.hasOnlyRevVALUShifts()) {
4851  NewOpcode = AMDGPU::V_LSHRREV_B64;
4852  swapOperands(Inst);
4853  }
4854  break;
4855 
4856  case AMDGPU::S_ABS_I32:
4857  lowerScalarAbs(Worklist, Inst);
4858  Inst.eraseFromParent();
4859  continue;
4860 
4861  case AMDGPU::S_CBRANCH_SCC0:
4862  case AMDGPU::S_CBRANCH_SCC1:
4863  // Clear unused bits of vcc
4864  if (ST.isWave32())
4865  BuildMI(*MBB, Inst, Inst.getDebugLoc(), get(AMDGPU::S_AND_B32),
4866  AMDGPU::VCC_LO)
4867  .addReg(AMDGPU::EXEC_LO)
4868  .addReg(AMDGPU::VCC_LO);
4869  else
4870  BuildMI(*MBB, Inst, Inst.getDebugLoc(), get(AMDGPU::S_AND_B64),
4871  AMDGPU::VCC)
4872  .addReg(AMDGPU::EXEC)
4873  .addReg(AMDGPU::VCC);
4874  break;
4875 
4876  case AMDGPU::S_BFE_U64:
4877  case AMDGPU::S_BFM_B64:
4878  llvm_unreachable("Moving this op to VALU not implemented");
4879 
4880  case AMDGPU::S_PACK_LL_B32_B16:
4881  case AMDGPU::S_PACK_LH_B32_B16:
4882  case AMDGPU::S_PACK_HH_B32_B16:
4883  movePackToVALU(Worklist, MRI, Inst);
4884  Inst.eraseFromParent();
4885  continue;
4886 
4887  case AMDGPU::S_XNOR_B32:
4888  lowerScalarXnor(Worklist, Inst);
4889  Inst.eraseFromParent();
4890  continue;
4891 
4892  case AMDGPU::S_NAND_B32:
4893  splitScalarNotBinop(Worklist, Inst, AMDGPU::S_AND_B32);
4894  Inst.eraseFromParent();
4895  continue;
4896 
4897  case AMDGPU::S_NOR_B32:
4898  splitScalarNotBinop(Worklist, Inst, AMDGPU::S_OR_B32);
4899  Inst.eraseFromParent();
4900  continue;
4901 
4902  case AMDGPU::S_ANDN2_B32:
4903  splitScalarBinOpN2(Worklist, Inst, AMDGPU::S_AND_B32);
4904  Inst.eraseFromParent();
4905  continue;
4906 
4907  case AMDGPU::S_ORN2_B32:
4908  splitScalarBinOpN2(Worklist, Inst, AMDGPU::S_OR_B32);
4909  Inst.eraseFromParent();
4910  continue;
4911  }
4912 
4913  if (NewOpcode == AMDGPU::INSTRUCTION_LIST_END) {
4914  // We cannot move this instruction to the VALU, so we should try to
4915  // legalize its operands instead.
4916  legalizeOperands(Inst, MDT);
4917  continue;
4918  }
4919 
4920  // Use the new VALU Opcode.
4921  const MCInstrDesc &NewDesc = get(NewOpcode);
4922  Inst.setDesc(NewDesc);
4923 
4924  // Remove any references to SCC. Vector instructions can't read from it, and
4925  // We're just about to add the implicit use / defs of VCC, and we don't want
4926  // both.
4927  for (unsigned i = Inst.getNumOperands() - 1; i > 0; --i) {
4928  MachineOperand &Op = Inst.getOperand(i);
4929  if (Op.isReg() && Op.getReg() == AMDGPU::SCC) {
4930  // Only propagate through live-def of SCC.
4931  if (Op.isDef() && !Op.isDead())
4932  addSCCDefUsersToVALUWorklist(Op, Inst, Worklist);
4933  Inst.RemoveOperand(i);
4934  }
4935  }
4936 
4937  if (Opcode == AMDGPU::S_SEXT_I32_I8 || Opcode == AMDGPU::S_SEXT_I32_I16) {
4938  // We are converting these to a BFE, so we need to add the missing
4939  // operands for the size and offset.
4940  unsigned Size = (Opcode == AMDGPU::S_SEXT_I32_I8) ? 8 : 16;
4943 
4944  } else if (Opcode == AMDGPU::S_BCNT1_I32_B32) {
4945  // The VALU version adds the second operand to the result, so insert an
4946  // extra 0 operand.
4948  }
4949 
4951  fixImplicitOperands(Inst);
4952 
4953  if (Opcode == AMDGPU::S_BFE_I32 || Opcode == AMDGPU::S_BFE_U32) {
4954  const MachineOperand &OffsetWidthOp = Inst.getOperand(2);
4955  // If we need to move this to VGPRs, we need to unpack the second operand
4956  // back into the 2 separate ones for bit offset and width.
4957  assert(OffsetWidthOp.isImm() &&
4958  "Scalar BFE is only implemented for constant width and offset");
4959  uint32_t Imm = OffsetWidthOp.getImm();
4960 
4961  uint32_t Offset = Imm & 0x3f; // Extract bits [5:0].
4962  uint32_t BitWidth = (Imm & 0x7f0000) >> 16; // Extract bits [22:16].
4963  Inst.RemoveOperand(2); // Remove old immediate.
4964  Inst.addOperand(MachineOperand::CreateImm(Offset));
4965  Inst.addOperand(MachineOperand::CreateImm(BitWidth));
4966  }
4967 
4968  bool HasDst = Inst.getOperand(0).isReg() && Inst.getOperand(0).isDef();
4969  unsigned NewDstReg = AMDGPU::NoRegister;
4970  if (HasDst) {
4971  Register DstReg = Inst.getOperand(0).getReg();
4972  if (Register::isPhysicalRegister(DstReg))
4973  continue;
4974 
4975  // Update the destination register class.
4976  const TargetRegisterClass *NewDstRC = getDestEquivalentVGPRClass(Inst);
4977  if (!NewDstRC)
4978  continue;
4979 
4980  if (Inst.isCopy() &&
4982  NewDstRC == RI.getRegClassForReg(MRI, Inst.getOperand(1).getReg())) {
4983  // Instead of creating a copy where src and dst are the same register
4984  // class, we just replace all uses of dst with src. These kinds of
4985  // copies interfere with the heuristics MachineSink uses to decide
4986  // whether or not to split a critical edge. Since the pass assumes
4987  // that copies will end up as machine instructions and not be
4988  // eliminated.
4989  addUsersToMoveToVALUWorklist(DstReg, MRI, Worklist);
4990  MRI.replaceRegWith(DstReg, Inst.getOperand(1).getReg());
4991  MRI.clearKillFlags(Inst.getOperand(1).getReg());
4992  Inst.getOperand(0).setReg(DstReg);
4993 
4994  // Make sure we don't leave around a dead VGPR->SGPR copy. Normally
4995  // these are deleted later, but at -O0 it would leave a suspicious
4996  // looking illegal copy of an undef register.
4997  for (unsigned I = Inst.getNumOperands() - 1; I != 0; --I)
4998  Inst.RemoveOperand(I);
4999  Inst.setDesc(get(AMDGPU::IMPLICIT_DEF));
5000  continue;
5001  }
5002 
5003  NewDstReg = MRI.createVirtualRegister(NewDstRC);
5004  MRI.replaceRegWith(DstReg, NewDstReg);
5005  }
5006 
5007  // Legalize the operands
5008  legalizeOperands(Inst, MDT);
5009 
5010  if (HasDst)
5011  addUsersToMoveToVALUWorklist(NewDstReg, MRI, Worklist);
5012  }
5013 }
5014 
5015 // Add/sub require special handling to deal with carry outs.
5016 bool SIInstrInfo::moveScalarAddSub(SetVectorType &Worklist, MachineInstr &Inst,
5017  MachineDominatorTree *MDT) const {
5018  if (ST.hasAddNoCarry()) {
5019  // Assume there is no user of scc since we don't select this in that case.
5020  // Since scc isn't used, it doesn't really matter if the i32 or u32 variant
5021  // is used.
5022 
5023  MachineBasicBlock &MBB = *Inst.getParent();
5025 
5026  Register OldDstReg = Inst.getOperand(0).getReg();
5027  Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
5028 
5029  unsigned Opc = Inst.getOpcode();
5030  assert(Opc == AMDGPU::S_ADD_I32 || Opc == AMDGPU::S_SUB_I32);
5031 
5032  unsigned NewOpc = Opc == AMDGPU::S_ADD_I32 ?
5033  AMDGPU::V_ADD_U32_e64 : AMDGPU::V_SUB_U32_e64;
5034 
5035  assert(Inst.getOperand(3).getReg() == AMDGPU::SCC);
5036  Inst.RemoveOperand(3);
5037 
5038  Inst.setDesc(get(NewOpc));
5039  Inst.addOperand(MachineOperand::CreateImm(0)); // clamp bit
5040  Inst.addImplicitDefUseOperands(*MBB.getParent());
5041  MRI.replaceRegWith(OldDstReg, ResultReg);
5042  legalizeOperands(Inst, MDT);
5043 
5044  addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist);
5045  return true;
5046  }
5047 
5048  return false;
5049 }
5050 
5051 void SIInstrInfo::lowerScalarAbs(SetVectorType &Worklist,
5052  MachineInstr &Inst) const {
5053  MachineBasicBlock &MBB = *Inst.getParent();
5055  MachineBasicBlock::iterator MII = Inst;
5056  DebugLoc DL = Inst.getDebugLoc();
5057 
5058  MachineOperand &Dest = Inst.getOperand(0);
5059  MachineOperand &Src = Inst.getOperand(1);
5060  Register TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
5061  Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
5062 
5063  unsigned SubOp = ST.hasAddNoCarry() ?
5064  AMDGPU::V_SUB_U32_e32 : AMDGPU::V_SUB_I32_e32;
5065 
5066  BuildMI(MBB, MII, DL, get(SubOp), TmpReg)
5067  .addImm(0)
5068  .addReg(Src.getReg());
5069 
5070  BuildMI(MBB, MII, DL, get(AMDGPU::V_MAX_I32_e64), ResultReg)
5071  .addReg(Src.getReg())
5072  .addReg(TmpReg);
5073 
5074  MRI.replaceRegWith(Dest.getReg(), ResultReg);
5075  addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist);
5076 }
5077 
5078 void SIInstrInfo::lowerScalarXnor(SetVectorType &Worklist,
5079  MachineInstr &Inst) const {
5080  MachineBasicBlock &MBB = *Inst.getParent();
5082  MachineBasicBlock::iterator MII = Inst;
5083  const DebugLoc &DL = Inst.getDebugLoc();
5084 
5085  MachineOperand &Dest = Inst.getOperand(0);
5086  MachineOperand &Src0 = Inst.getOperand(1);
5087  MachineOperand &Src1 = Inst.getOperand(2);
5088 
5089  if (ST.hasDLInsts()) {
5090  Register NewDest = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
5091  legalizeGenericOperand(MBB, MII, &AMDGPU::VGPR_32RegClass, Src0, MRI, DL);
5092  legalizeGenericOperand(MBB, MII, &AMDGPU::VGPR_32RegClass, Src1, MRI, DL);
5093 
5094  BuildMI(MBB, MII, DL, get(AMDGPU::V_XNOR_B32_e64), NewDest)
5095  .add(Src0)
5096  .add(Src1);
5097 
5098  MRI.replaceRegWith(Dest.getReg(), NewDest);
5099  addUsersToMoveToVALUWorklist(NewDest, MRI, Worklist);
5100  } else {
5101  // Using the identity !(x ^ y) == (!x ^ y) == (x ^ !y), we can
5102  // invert either source and then perform the XOR. If either source is a
5103  // scalar register, then we can leave the inversion on the scalar unit to
5104  // acheive a better distrubution of scalar and vector instructions.
5105  bool Src0IsSGPR = Src0.isReg() &&
5106  RI.isSGPRClass(MRI.getRegClass(Src0.getReg()));
5107  bool Src1IsSGPR = Src1.isReg() &&
5108  RI.isSGPRClass(MRI.getRegClass(Src1.getReg()));
5109  MachineInstr *Xor;
5110  Register Temp = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
5111  Register NewDest = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
5112 
5113  // Build a pair of scalar instructions and add them to the work list.
5114  // The next iteration over the work list will lower these to the vector
5115  // unit as necessary.
5116  if (Src0IsSGPR) {
5117  BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), Temp).add(Src0);
5118  Xor = BuildMI(MBB, MII, DL, get(AMDGPU::S_XOR_B32), NewDest)
5119  .addReg(Temp)
5120  .add(Src1);
5121  } else if (Src1IsSGPR) {
5122  BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), Temp).add(Src1);
5123  Xor = BuildMI(MBB, MII, DL, get(AMDGPU::S_XOR_B32), NewDest)
5124  .add(Src0)
5125  .addReg(Temp);
5126  } else {
5127  Xor = BuildMI(MBB, MII, DL, get(AMDGPU::S_XOR_B32), Temp)
5128  .add(Src0)
5129  .add(Src1);
5130  MachineInstr *Not =
5131  BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), NewDest).addReg(Temp);
5132  Worklist.insert(Not);
5133  }
5134 
5135  MRI.replaceRegWith(Dest.getReg(), NewDest);
5136 
5137  Worklist.insert(Xor);
5138 
5139  addUsersToMoveToVALUWorklist(NewDest, MRI, Worklist);
5140  }
5141 }
5142 
5143 void SIInstrInfo::splitScalarNotBinop(SetVectorType &Worklist,
5144  MachineInstr &Inst,
5145  unsigned Opcode) const {
5146  MachineBasicBlock &MBB = *Inst.getParent();
5148  MachineBasicBlock::iterator MII = Inst;
5149  const DebugLoc &DL = Inst.getDebugLoc();
5150 
5151  MachineOperand &Dest = Inst.getOperand(0);
5152  MachineOperand &Src0 = Inst.getOperand(1);
5153  MachineOperand &Src1 = Inst.getOperand(2);
5154 
5155  Register NewDest = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
5156  Register Interm = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
5157 
5158  MachineInstr &Op = *BuildMI(MBB, MII, DL, get(Opcode), Interm)
5159  .add(Src0)
5160  .add(Src1);
5161 
5162  MachineInstr &Not = *BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), NewDest)
5163  .addReg(Interm);
5164 
5165  Worklist.insert(&Op);
5166  Worklist.insert(&Not);
5167 
5168  MRI.replaceRegWith(Dest.getReg(), NewDest);
5169  addUsersToMoveToVALUWorklist(NewDest, MRI, Worklist);
5170 }
5171 
5172 void SIInstrInfo::splitScalarBinOpN2(SetVectorType& Worklist,
5173  MachineInstr &Inst,
5174  unsigned Opcode) const {
5175  MachineBasicBlock &MBB = *Inst.getParent();
5177  MachineBasicBlock::iterator MII = Inst;
5178  const DebugLoc &DL = Inst.getDebugLoc();
5179 
5180  MachineOperand &Dest = Inst.getOperand(0);
5181  MachineOperand &Src0 = Inst.getOperand(1);
5182  MachineOperand &Src1 = Inst.getOperand(2);
5183 
5184  Register NewDest = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
5185  Register Interm = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
5186 
5187  MachineInstr &Not = *BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), Interm)
5188  .add(Src1);
5189 
5190  MachineInstr &Op = *BuildMI(MBB, MII, DL, get(Opcode), NewDest)
5191  .add(Src0)
5192  .addReg(Interm);
5193 
5194  Worklist.insert(&Not);
5195  Worklist.insert(&Op);
5196 
5197  MRI.replaceRegWith(Dest.getReg(), NewDest);
5198  addUsersToMoveToVALUWorklist(NewDest, MRI, Worklist);
5199 }
5200 
5201 void SIInstrInfo::splitScalar64BitUnaryOp(
5202  SetVectorType &Worklist, MachineInstr &Inst,
5203  unsigned Opcode) const {
5204  MachineBasicBlock &MBB = *Inst.getParent();
5206 
5207  MachineOperand &Dest = Inst.getOperand(0);
5208  MachineOperand &Src0 = Inst.getOperand(1);
5209  DebugLoc DL = Inst.getDebugLoc();
5210 
5211  MachineBasicBlock::iterator MII = Inst;
5212 
5213  const MCInstrDesc &InstDesc = get(Opcode);
5214  const TargetRegisterClass *Src0RC = Src0.isReg() ?
5215  MRI.getRegClass(Src0.getReg()) :
5216  &AMDGPU::SGPR_32RegClass;
5217 
5218  const TargetRegisterClass *Src0SubRC = RI.getSubRegClass(Src0RC, AMDGPU::sub0);
5219 
5220  MachineOperand SrcReg0Sub0 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC,
5221  AMDGPU::sub0, Src0SubRC);
5222 
5223  const TargetRegisterClass *DestRC = MRI.getRegClass(Dest.getReg());
5224  const TargetRegisterClass *NewDestRC = RI.getEquivalentVGPRClass(DestRC);
5225  const TargetRegisterClass *NewDestSubRC = RI.getSubRegClass(NewDestRC, AMDGPU::sub0);
5226 
5227  Register DestSub0 = MRI.createVirtualRegister(NewDestSubRC);
5228  MachineInstr &LoHalf = *BuildMI(MBB, MII, DL, InstDesc, DestSub0).add(SrcReg0Sub0);
5229 
5230  MachineOperand SrcReg0Sub1 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC,
5231  AMDGPU::sub1, Src0SubRC);
5232 
5233  Register DestSub1 = MRI.createVirtualRegister(NewDestSubRC);
5234  MachineInstr &HiHalf = *BuildMI(MBB, MII, DL, InstDesc, DestSub1).add(SrcReg0Sub1);
5235 
5236  Register FullDestReg = MRI.createVirtualRegister(NewDestRC);
5237  BuildMI(MBB, MII, DL, get(TargetOpcode::REG_SEQUENCE), FullDestReg)
5238  .addReg(DestSub0)
5239  .addImm(AMDGPU::sub0)
5240  .addReg(DestSub1)
5241  .addImm(AMDGPU::sub1);
5242 
5243  MRI.replaceRegWith(Dest.getReg(), FullDestReg);
5244 
5245  Worklist.insert(&LoHalf);
5246  Worklist.insert(&HiHalf);
5247 
5248  // We don't need to legalizeOperands here because for a single operand, src0
5249  // will support any kind of input.
5250 
5251  // Move all users of this moved value.
5252  addUsersToMoveToVALUWorklist(FullDestReg, MRI, Worklist);
5253 }
5254 
5255 void SIInstrInfo::splitScalar64BitAddSub(SetVectorType &Worklist,
5256  MachineInstr &Inst,
5257  MachineDominatorTree *MDT) const {
5258  bool IsAdd = (Inst.getOpcode() == AMDGPU::S_ADD_U64_PSEUDO);
5259 
5260  MachineBasicBlock &MBB = *Inst.getParent();
5262  const auto *CarryRC = RI.getRegClass(AMDGPU::SReg_1_XEXECRegClassID);
5263 
5264  Register FullDestReg = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass);
5265  Register DestSub0 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
5266  Register DestSub1 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
5267 
5268  Register CarryReg = MRI.createVirtualRegister(CarryRC);
5269  Register DeadCarryReg = MRI.createVirtualRegister(CarryRC);
5270 
5271  MachineOperand &Dest = Inst.getOperand(0);
5272  MachineOperand &Src0 = Inst.getOperand(1);
5273  MachineOperand &Src1 = Inst.getOperand(2);
5274  const DebugLoc &DL = Inst.getDebugLoc();
5275  MachineBasicBlock::iterator MII = Inst;
5276 
5277  const TargetRegisterClass *Src0RC = MRI.getRegClass(Src0.getReg());
5278  const TargetRegisterClass *Src1RC = MRI.