LLVM  8.0.0svn
SIInstrInfo.cpp
Go to the documentation of this file.
1 //===- SIInstrInfo.cpp - SI Instruction Information ----------------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 /// \file
11 /// SI Implementation of TargetInstrInfo.
12 //
13 //===----------------------------------------------------------------------===//
14 
15 #include "SIInstrInfo.h"
16 #include "AMDGPU.h"
17 #include "AMDGPUIntrinsicInfo.h"
18 #include "AMDGPUSubtarget.h"
19 #include "GCNHazardRecognizer.h"
20 #include "SIDefines.h"
21 #include "SIMachineFunctionInfo.h"
22 #include "SIRegisterInfo.h"
24 #include "Utils/AMDGPUBaseInfo.h"
25 #include "llvm/ADT/APInt.h"
26 #include "llvm/ADT/ArrayRef.h"
27 #include "llvm/ADT/SmallVector.h"
28 #include "llvm/ADT/StringRef.h"
48 #include "llvm/IR/DebugLoc.h"
49 #include "llvm/IR/DiagnosticInfo.h"
50 #include "llvm/IR/Function.h"
51 #include "llvm/IR/InlineAsm.h"
52 #include "llvm/IR/LLVMContext.h"
53 #include "llvm/MC/MCInstrDesc.h"
54 #include "llvm/Support/Casting.h"
56 #include "llvm/Support/Compiler.h"
61 #include <cassert>
62 #include <cstdint>
63 #include <iterator>
64 #include <utility>
65 
66 using namespace llvm;
67 
68 #define GET_INSTRINFO_CTOR_DTOR
69 #include "AMDGPUGenInstrInfo.inc"
70 
71 namespace llvm {
72 namespace AMDGPU {
73 #define GET_D16ImageDimIntrinsics_IMPL
74 #define GET_ImageDimIntrinsicTable_IMPL
75 #define GET_RsrcIntrinsics_IMPL
76 #include "AMDGPUGenSearchableTables.inc"
77 }
78 }
79 
80 
81 // Must be at least 4 to be able to branch over minimum unconditional branch
82 // code. This is only for making it possible to write reasonably small tests for
83 // long branches.
84 static cl::opt<unsigned>
85 BranchOffsetBits("amdgpu-s-branch-bits", cl::ReallyHidden, cl::init(16),
86  cl::desc("Restrict range of branch instructions (DEBUG)"));
87 
89  : AMDGPUGenInstrInfo(AMDGPU::ADJCALLSTACKUP, AMDGPU::ADJCALLSTACKDOWN),
90  RI(ST), ST(ST) {}
91 
92 //===----------------------------------------------------------------------===//
93 // TargetInstrInfo callbacks
94 //===----------------------------------------------------------------------===//
95 
96 static unsigned getNumOperandsNoGlue(SDNode *Node) {
97  unsigned N = Node->getNumOperands();
98  while (N && Node->getOperand(N - 1).getValueType() == MVT::Glue)
99  --N;
100  return N;
101 }
102 
104  SDValue LastOp = Load->getOperand(getNumOperandsNoGlue(Load) - 1);
105  assert(LastOp.getValueType() == MVT::Other && "Chain missing from load node");
106  return LastOp;
107 }
108 
109 /// Returns true if both nodes have the same value for the given
110 /// operand \p Op, or if both nodes do not have this operand.
111 static bool nodesHaveSameOperandValue(SDNode *N0, SDNode* N1, unsigned OpName) {
112  unsigned Opc0 = N0->getMachineOpcode();
113  unsigned Opc1 = N1->getMachineOpcode();
114 
115  int Op0Idx = AMDGPU::getNamedOperandIdx(Opc0, OpName);
116  int Op1Idx = AMDGPU::getNamedOperandIdx(Opc1, OpName);
117 
118  if (Op0Idx == -1 && Op1Idx == -1)
119  return true;
120 
121 
122  if ((Op0Idx == -1 && Op1Idx != -1) ||
123  (Op1Idx == -1 && Op0Idx != -1))
124  return false;
125 
126  // getNamedOperandIdx returns the index for the MachineInstr's operands,
127  // which includes the result as the first operand. We are indexing into the
128  // MachineSDNode's operands, so we need to skip the result operand to get
129  // the real index.
130  --Op0Idx;
131  --Op1Idx;
132 
133  return N0->getOperand(Op0Idx) == N1->getOperand(Op1Idx);
134 }
135 
137  AliasAnalysis *AA) const {
138  // TODO: The generic check fails for VALU instructions that should be
139  // rematerializable due to implicit reads of exec. We really want all of the
140  // generic logic for this except for this.
141  switch (MI.getOpcode()) {
142  case AMDGPU::V_MOV_B32_e32:
143  case AMDGPU::V_MOV_B32_e64:
144  case AMDGPU::V_MOV_B64_PSEUDO:
145  return true;
146  default:
147  return false;
148  }
149 }
150 
152  int64_t &Offset0,
153  int64_t &Offset1) const {
154  if (!Load0->isMachineOpcode() || !Load1->isMachineOpcode())
155  return false;
156 
157  unsigned Opc0 = Load0->getMachineOpcode();
158  unsigned Opc1 = Load1->getMachineOpcode();
159 
160  // Make sure both are actually loads.
161  if (!get(Opc0).mayLoad() || !get(Opc1).mayLoad())
162  return false;
163 
164  if (isDS(Opc0) && isDS(Opc1)) {
165 
166  // FIXME: Handle this case:
167  if (getNumOperandsNoGlue(Load0) != getNumOperandsNoGlue(Load1))
168  return false;
169 
170  // Check base reg.
171  if (Load0->getOperand(1) != Load1->getOperand(1))
172  return false;
173 
174  // Check chain.
175  if (findChainOperand(Load0) != findChainOperand(Load1))
176  return false;
177 
178  // Skip read2 / write2 variants for simplicity.
179  // TODO: We should report true if the used offsets are adjacent (excluded
180  // st64 versions).
181  if (AMDGPU::getNamedOperandIdx(Opc0, AMDGPU::OpName::data1) != -1 ||
182  AMDGPU::getNamedOperandIdx(Opc1, AMDGPU::OpName::data1) != -1)
183  return false;
184 
185  Offset0 = cast<ConstantSDNode>(Load0->getOperand(2))->getZExtValue();
186  Offset1 = cast<ConstantSDNode>(Load1->getOperand(2))->getZExtValue();
187  return true;
188  }
189 
190  if (isSMRD(Opc0) && isSMRD(Opc1)) {
191  // Skip time and cache invalidation instructions.
192  if (AMDGPU::getNamedOperandIdx(Opc0, AMDGPU::OpName::sbase) == -1 ||
193  AMDGPU::getNamedOperandIdx(Opc1, AMDGPU::OpName::sbase) == -1)
194  return false;
195 
197 
198  // Check base reg.
199  if (Load0->getOperand(0) != Load1->getOperand(0))
200  return false;
201 
202  const ConstantSDNode *Load0Offset =
203  dyn_cast<ConstantSDNode>(Load0->getOperand(1));
204  const ConstantSDNode *Load1Offset =
205  dyn_cast<ConstantSDNode>(Load1->getOperand(1));
206 
207  if (!Load0Offset || !Load1Offset)
208  return false;
209 
210  // Check chain.
211  if (findChainOperand(Load0) != findChainOperand(Load1))
212  return false;
213 
214  Offset0 = Load0Offset->getZExtValue();
215  Offset1 = Load1Offset->getZExtValue();
216  return true;
217  }
218 
219  // MUBUF and MTBUF can access the same addresses.
220  if ((isMUBUF(Opc0) || isMTBUF(Opc0)) && (isMUBUF(Opc1) || isMTBUF(Opc1))) {
221 
222  // MUBUF and MTBUF have vaddr at different indices.
223  if (!nodesHaveSameOperandValue(Load0, Load1, AMDGPU::OpName::soffset) ||
224  findChainOperand(Load0) != findChainOperand(Load1) ||
225  !nodesHaveSameOperandValue(Load0, Load1, AMDGPU::OpName::vaddr) ||
226  !nodesHaveSameOperandValue(Load0, Load1, AMDGPU::OpName::srsrc))
227  return false;
228 
229  int OffIdx0 = AMDGPU::getNamedOperandIdx(Opc0, AMDGPU::OpName::offset);
230  int OffIdx1 = AMDGPU::getNamedOperandIdx(Opc1, AMDGPU::OpName::offset);
231 
232  if (OffIdx0 == -1 || OffIdx1 == -1)
233  return false;
234 
235  // getNamedOperandIdx returns the index for MachineInstrs. Since they
236  // inlcude the output in the operand list, but SDNodes don't, we need to
237  // subtract the index by one.
238  --OffIdx0;
239  --OffIdx1;
240 
241  SDValue Off0 = Load0->getOperand(OffIdx0);
242  SDValue Off1 = Load1->getOperand(OffIdx1);
243 
244  // The offset might be a FrameIndexSDNode.
245  if (!isa<ConstantSDNode>(Off0) || !isa<ConstantSDNode>(Off1))
246  return false;
247 
248  Offset0 = cast<ConstantSDNode>(Off0)->getZExtValue();
249  Offset1 = cast<ConstantSDNode>(Off1)->getZExtValue();
250  return true;
251  }
252 
253  return false;
254 }
255 
256 static bool isStride64(unsigned Opc) {
257  switch (Opc) {
258  case AMDGPU::DS_READ2ST64_B32:
259  case AMDGPU::DS_READ2ST64_B64:
260  case AMDGPU::DS_WRITE2ST64_B32:
261  case AMDGPU::DS_WRITE2ST64_B64:
262  return true;
263  default:
264  return false;
265  }
266 }
267 
269  MachineOperand *&BaseOp,
270  int64_t &Offset,
271  const TargetRegisterInfo *TRI) const {
272  unsigned Opc = LdSt.getOpcode();
273 
274  if (isDS(LdSt)) {
275  const MachineOperand *OffsetImm =
276  getNamedOperand(LdSt, AMDGPU::OpName::offset);
277  if (OffsetImm) {
278  // Normal, single offset LDS instruction.
279  BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::addr);
280  Offset = OffsetImm->getImm();
281  assert(BaseOp->isReg() && "getMemOperandWithOffset only supports base "
282  "operands of type register.");
283  return true;
284  }
285 
286  // The 2 offset instructions use offset0 and offset1 instead. We can treat
287  // these as a load with a single offset if the 2 offsets are consecutive. We
288  // will use this for some partially aligned loads.
289  const MachineOperand *Offset0Imm =
290  getNamedOperand(LdSt, AMDGPU::OpName::offset0);
291  const MachineOperand *Offset1Imm =
292  getNamedOperand(LdSt, AMDGPU::OpName::offset1);
293 
294  uint8_t Offset0 = Offset0Imm->getImm();
295  uint8_t Offset1 = Offset1Imm->getImm();
296 
297  if (Offset1 > Offset0 && Offset1 - Offset0 == 1) {
298  // Each of these offsets is in element sized units, so we need to convert
299  // to bytes of the individual reads.
300 
301  unsigned EltSize;
302  if (LdSt.mayLoad())
303  EltSize = TRI->getRegSizeInBits(*getOpRegClass(LdSt, 0)) / 16;
304  else {
305  assert(LdSt.mayStore());
306  int Data0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data0);
307  EltSize = TRI->getRegSizeInBits(*getOpRegClass(LdSt, Data0Idx)) / 8;
308  }
309 
310  if (isStride64(Opc))
311  EltSize *= 64;
312 
313  BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::addr);
314  Offset = EltSize * Offset0;
315  assert(BaseOp->isReg() && "getMemOperandWithOffset only supports base "
316  "operands of type register.");
317  return true;
318  }
319 
320  return false;
321  }
322 
323  if (isMUBUF(LdSt) || isMTBUF(LdSt)) {
324  const MachineOperand *SOffset = getNamedOperand(LdSt, AMDGPU::OpName::soffset);
325  if (SOffset && SOffset->isReg())
326  return false;
327 
328  MachineOperand *AddrReg = getNamedOperand(LdSt, AMDGPU::OpName::vaddr);
329  if (!AddrReg)
330  return false;
331 
332  const MachineOperand *OffsetImm =
333  getNamedOperand(LdSt, AMDGPU::OpName::offset);
334  BaseOp = AddrReg;
335  Offset = OffsetImm->getImm();
336 
337  if (SOffset) // soffset can be an inline immediate.
338  Offset += SOffset->getImm();
339 
340  assert(BaseOp->isReg() && "getMemOperandWithOffset only supports base "
341  "operands of type register.");
342  return true;
343  }
344 
345  if (isSMRD(LdSt)) {
346  const MachineOperand *OffsetImm =
347  getNamedOperand(LdSt, AMDGPU::OpName::offset);
348  if (!OffsetImm)
349  return false;
350 
351  MachineOperand *SBaseReg = getNamedOperand(LdSt, AMDGPU::OpName::sbase);
352  BaseOp = SBaseReg;
353  Offset = OffsetImm->getImm();
354  assert(BaseOp->isReg() && "getMemOperandWithOffset only supports base "
355  "operands of type register.");
356  return true;
357  }
358 
359  if (isFLAT(LdSt)) {
360  MachineOperand *VAddr = getNamedOperand(LdSt, AMDGPU::OpName::vaddr);
361  if (VAddr) {
362  // Can't analyze 2 offsets.
363  if (getNamedOperand(LdSt, AMDGPU::OpName::saddr))
364  return false;
365 
366  BaseOp = VAddr;
367  } else {
368  // scratch instructions have either vaddr or saddr.
369  BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::saddr);
370  }
371 
372  Offset = getNamedOperand(LdSt, AMDGPU::OpName::offset)->getImm();
373  assert(BaseOp->isReg() && "getMemOperandWithOffset only supports base "
374  "operands of type register.");
375  return true;
376  }
377 
378  return false;
379 }
380 
381 static bool memOpsHaveSameBasePtr(const MachineInstr &MI1,
382  const MachineOperand &BaseOp1,
383  const MachineInstr &MI2,
384  const MachineOperand &BaseOp2) {
385  // Support only base operands with base registers.
386  // Note: this could be extended to support FI operands.
387  if (!BaseOp1.isReg() || !BaseOp2.isReg())
388  return false;
389 
390  if (BaseOp1.isIdenticalTo(BaseOp2))
391  return true;
392 
393  if (!MI1.hasOneMemOperand() || !MI2.hasOneMemOperand())
394  return false;
395 
396  auto MO1 = *MI1.memoperands_begin();
397  auto MO2 = *MI2.memoperands_begin();
398  if (MO1->getAddrSpace() != MO2->getAddrSpace())
399  return false;
400 
401  auto Base1 = MO1->getValue();
402  auto Base2 = MO2->getValue();
403  if (!Base1 || !Base2)
404  return false;
405  const MachineFunction &MF = *MI1.getParent()->getParent();
406  const DataLayout &DL = MF.getFunction().getParent()->getDataLayout();
407  Base1 = GetUnderlyingObject(Base1, DL);
408  Base2 = GetUnderlyingObject(Base1, DL);
409 
410  if (isa<UndefValue>(Base1) || isa<UndefValue>(Base2))
411  return false;
412 
413  return Base1 == Base2;
414 }
415 
417  MachineOperand &BaseOp2,
418  unsigned NumLoads) const {
419  MachineInstr &FirstLdSt = *BaseOp1.getParent();
420  MachineInstr &SecondLdSt = *BaseOp2.getParent();
421 
422  if (!memOpsHaveSameBasePtr(FirstLdSt, BaseOp1, SecondLdSt, BaseOp2))
423  return false;
424 
425  const MachineOperand *FirstDst = nullptr;
426  const MachineOperand *SecondDst = nullptr;
427 
428  if ((isMUBUF(FirstLdSt) && isMUBUF(SecondLdSt)) ||
429  (isMTBUF(FirstLdSt) && isMTBUF(SecondLdSt)) ||
430  (isFLAT(FirstLdSt) && isFLAT(SecondLdSt))) {
431  const unsigned MaxGlobalLoadCluster = 6;
432  if (NumLoads > MaxGlobalLoadCluster)
433  return false;
434 
435  FirstDst = getNamedOperand(FirstLdSt, AMDGPU::OpName::vdata);
436  if (!FirstDst)
437  FirstDst = getNamedOperand(FirstLdSt, AMDGPU::OpName::vdst);
438  SecondDst = getNamedOperand(SecondLdSt, AMDGPU::OpName::vdata);
439  if (!SecondDst)
440  SecondDst = getNamedOperand(SecondLdSt, AMDGPU::OpName::vdst);
441  } else if (isSMRD(FirstLdSt) && isSMRD(SecondLdSt)) {
442  FirstDst = getNamedOperand(FirstLdSt, AMDGPU::OpName::sdst);
443  SecondDst = getNamedOperand(SecondLdSt, AMDGPU::OpName::sdst);
444  } else if (isDS(FirstLdSt) && isDS(SecondLdSt)) {
445  FirstDst = getNamedOperand(FirstLdSt, AMDGPU::OpName::vdst);
446  SecondDst = getNamedOperand(SecondLdSt, AMDGPU::OpName::vdst);
447  }
448 
449  if (!FirstDst || !SecondDst)
450  return false;
451 
452  // Try to limit clustering based on the total number of bytes loaded
453  // rather than the number of instructions. This is done to help reduce
454  // register pressure. The method used is somewhat inexact, though,
455  // because it assumes that all loads in the cluster will load the
456  // same number of bytes as FirstLdSt.
457 
458  // The unit of this value is bytes.
459  // FIXME: This needs finer tuning.
460  unsigned LoadClusterThreshold = 16;
461 
462  const MachineRegisterInfo &MRI =
463  FirstLdSt.getParent()->getParent()->getRegInfo();
464  const TargetRegisterClass *DstRC = MRI.getRegClass(FirstDst->getReg());
465 
466  return (NumLoads * (RI.getRegSizeInBits(*DstRC) / 8)) <= LoadClusterThreshold;
467 }
468 
469 // FIXME: This behaves strangely. If, for example, you have 32 load + stores,
470 // the first 16 loads will be interleaved with the stores, and the next 16 will
471 // be clustered as expected. It should really split into 2 16 store batches.
472 //
473 // Loads are clustered until this returns false, rather than trying to schedule
474 // groups of stores. This also means we have to deal with saying different
475 // address space loads should be clustered, and ones which might cause bank
476 // conflicts.
477 //
478 // This might be deprecated so it might not be worth that much effort to fix.
480  int64_t Offset0, int64_t Offset1,
481  unsigned NumLoads) const {
482  assert(Offset1 > Offset0 &&
483  "Second offset should be larger than first offset!");
484  // If we have less than 16 loads in a row, and the offsets are within 64
485  // bytes, then schedule together.
486 
487  // A cacheline is 64 bytes (for global memory).
488  return (NumLoads <= 16 && (Offset1 - Offset0) < 64);
489 }
490 
493  const DebugLoc &DL, unsigned DestReg,
494  unsigned SrcReg, bool KillSrc) {
495  MachineFunction *MF = MBB.getParent();
496  DiagnosticInfoUnsupported IllegalCopy(MF->getFunction(),
497  "illegal SGPR to VGPR copy",
498  DL, DS_Error);
499  LLVMContext &C = MF->getFunction().getContext();
500  C.diagnose(IllegalCopy);
501 
502  BuildMI(MBB, MI, DL, TII->get(AMDGPU::SI_ILLEGAL_COPY), DestReg)
503  .addReg(SrcReg, getKillRegState(KillSrc));
504 }
505 
508  const DebugLoc &DL, unsigned DestReg,
509  unsigned SrcReg, bool KillSrc) const {
510  const TargetRegisterClass *RC = RI.getPhysRegClass(DestReg);
511 
512  if (RC == &AMDGPU::VGPR_32RegClass) {
513  assert(AMDGPU::VGPR_32RegClass.contains(SrcReg) ||
514  AMDGPU::SReg_32RegClass.contains(SrcReg));
515  BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DestReg)
516  .addReg(SrcReg, getKillRegState(KillSrc));
517  return;
518  }
519 
520  if (RC == &AMDGPU::SReg_32_XM0RegClass ||
521  RC == &AMDGPU::SReg_32RegClass) {
522  if (SrcReg == AMDGPU::SCC) {
523  BuildMI(MBB, MI, DL, get(AMDGPU::S_CSELECT_B32), DestReg)
524  .addImm(-1)
525  .addImm(0);
526  return;
527  }
528 
529  if (!AMDGPU::SReg_32RegClass.contains(SrcReg)) {
530  reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc);
531  return;
532  }
533 
534  BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), DestReg)
535  .addReg(SrcReg, getKillRegState(KillSrc));
536  return;
537  }
538 
539  if (RC == &AMDGPU::SReg_64RegClass) {
540  if (DestReg == AMDGPU::VCC) {
541  if (AMDGPU::SReg_64RegClass.contains(SrcReg)) {
542  BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B64), AMDGPU::VCC)
543  .addReg(SrcReg, getKillRegState(KillSrc));
544  } else {
545  // FIXME: Hack until VReg_1 removed.
546  assert(AMDGPU::VGPR_32RegClass.contains(SrcReg));
547  BuildMI(MBB, MI, DL, get(AMDGPU::V_CMP_NE_U32_e32))
548  .addImm(0)
549  .addReg(SrcReg, getKillRegState(KillSrc));
550  }
551 
552  return;
553  }
554 
555  if (!AMDGPU::SReg_64RegClass.contains(SrcReg)) {
556  reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc);
557  return;
558  }
559 
560  BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B64), DestReg)
561  .addReg(SrcReg, getKillRegState(KillSrc));
562  return;
563  }
564 
565  if (DestReg == AMDGPU::SCC) {
566  assert(AMDGPU::SReg_32RegClass.contains(SrcReg));
567  BuildMI(MBB, MI, DL, get(AMDGPU::S_CMP_LG_U32))
568  .addReg(SrcReg, getKillRegState(KillSrc))
569  .addImm(0);
570  return;
571  }
572 
573  unsigned EltSize = 4;
574  unsigned Opcode = AMDGPU::V_MOV_B32_e32;
575  if (RI.isSGPRClass(RC)) {
576  if (RI.getRegSizeInBits(*RC) > 32) {
577  Opcode = AMDGPU::S_MOV_B64;
578  EltSize = 8;
579  } else {
580  Opcode = AMDGPU::S_MOV_B32;
581  EltSize = 4;
582  }
583 
584  if (!RI.isSGPRClass(RI.getPhysRegClass(SrcReg))) {
585  reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc);
586  return;
587  }
588  }
589 
590  ArrayRef<int16_t> SubIndices = RI.getRegSplitParts(RC, EltSize);
591  bool Forward = RI.getHWRegIndex(DestReg) <= RI.getHWRegIndex(SrcReg);
592 
593  for (unsigned Idx = 0; Idx < SubIndices.size(); ++Idx) {
594  unsigned SubIdx;
595  if (Forward)
596  SubIdx = SubIndices[Idx];
597  else
598  SubIdx = SubIndices[SubIndices.size() - Idx - 1];
599 
600  MachineInstrBuilder Builder = BuildMI(MBB, MI, DL,
601  get(Opcode), RI.getSubReg(DestReg, SubIdx));
602 
603  Builder.addReg(RI.getSubReg(SrcReg, SubIdx));
604 
605  if (Idx == 0)
606  Builder.addReg(DestReg, RegState::Define | RegState::Implicit);
607 
608  bool UseKill = KillSrc && Idx == SubIndices.size() - 1;
609  Builder.addReg(SrcReg, getKillRegState(UseKill) | RegState::Implicit);
610  }
611 }
612 
613 int SIInstrInfo::commuteOpcode(unsigned Opcode) const {
614  int NewOpc;
615 
616  // Try to map original to commuted opcode
617  NewOpc = AMDGPU::getCommuteRev(Opcode);
618  if (NewOpc != -1)
619  // Check if the commuted (REV) opcode exists on the target.
620  return pseudoToMCOpcode(NewOpc) != -1 ? NewOpc : -1;
621 
622  // Try to map commuted to original opcode
623  NewOpc = AMDGPU::getCommuteOrig(Opcode);
624  if (NewOpc != -1)
625  // Check if the original (non-REV) opcode exists on the target.
626  return pseudoToMCOpcode(NewOpc) != -1 ? NewOpc : -1;
627 
628  return Opcode;
629 }
630 
633  const DebugLoc &DL, unsigned DestReg,
634  int64_t Value) const {
636  const TargetRegisterClass *RegClass = MRI.getRegClass(DestReg);
637  if (RegClass == &AMDGPU::SReg_32RegClass ||
638  RegClass == &AMDGPU::SGPR_32RegClass ||
639  RegClass == &AMDGPU::SReg_32_XM0RegClass ||
640  RegClass == &AMDGPU::SReg_32_XM0_XEXECRegClass) {
641  BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), DestReg)
642  .addImm(Value);
643  return;
644  }
645 
646  if (RegClass == &AMDGPU::SReg_64RegClass ||
647  RegClass == &AMDGPU::SGPR_64RegClass ||
648  RegClass == &AMDGPU::SReg_64_XEXECRegClass) {
649  BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B64), DestReg)
650  .addImm(Value);
651  return;
652  }
653 
654  if (RegClass == &AMDGPU::VGPR_32RegClass) {
655  BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DestReg)
656  .addImm(Value);
657  return;
658  }
659  if (RegClass == &AMDGPU::VReg_64RegClass) {
660  BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B64_PSEUDO), DestReg)
661  .addImm(Value);
662  return;
663  }
664 
665  unsigned EltSize = 4;
666  unsigned Opcode = AMDGPU::V_MOV_B32_e32;
667  if (RI.isSGPRClass(RegClass)) {
668  if (RI.getRegSizeInBits(*RegClass) > 32) {
669  Opcode = AMDGPU::S_MOV_B64;
670  EltSize = 8;
671  } else {
672  Opcode = AMDGPU::S_MOV_B32;
673  EltSize = 4;
674  }
675  }
676 
677  ArrayRef<int16_t> SubIndices = RI.getRegSplitParts(RegClass, EltSize);
678  for (unsigned Idx = 0; Idx < SubIndices.size(); ++Idx) {
679  int64_t IdxValue = Idx == 0 ? Value : 0;
680 
681  MachineInstrBuilder Builder = BuildMI(MBB, MI, DL,
682  get(Opcode), RI.getSubReg(DestReg, Idx));
683  Builder.addImm(IdxValue);
684  }
685 }
686 
687 const TargetRegisterClass *
689  return &AMDGPU::VGPR_32RegClass;
690 }
691 
694  const DebugLoc &DL, unsigned DstReg,
696  unsigned TrueReg,
697  unsigned FalseReg) const {
699  assert(MRI.getRegClass(DstReg) == &AMDGPU::VGPR_32RegClass &&
700  "Not a VGPR32 reg");
701 
702  if (Cond.size() == 1) {
703  unsigned SReg = MRI.createVirtualRegister(&AMDGPU::SReg_64_XEXECRegClass);
704  BuildMI(MBB, I, DL, get(AMDGPU::COPY), SReg)
705  .add(Cond[0]);
706  BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
707  .addReg(FalseReg)
708  .addReg(TrueReg)
709  .addReg(SReg);
710  } else if (Cond.size() == 2) {
711  assert(Cond[0].isImm() && "Cond[0] is not an immediate");
712  switch (Cond[0].getImm()) {
713  case SIInstrInfo::SCC_TRUE: {
714  unsigned SReg = MRI.createVirtualRegister(&AMDGPU::SReg_64_XEXECRegClass);
715  BuildMI(MBB, I, DL, get(AMDGPU::S_CSELECT_B64), SReg)
716  .addImm(-1)
717  .addImm(0);
718  BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
719  .addReg(FalseReg)
720  .addReg(TrueReg)
721  .addReg(SReg);
722  break;
723  }
724  case SIInstrInfo::SCC_FALSE: {
725  unsigned SReg = MRI.createVirtualRegister(&AMDGPU::SReg_64_XEXECRegClass);
726  BuildMI(MBB, I, DL, get(AMDGPU::S_CSELECT_B64), SReg)
727  .addImm(0)
728  .addImm(-1);
729  BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
730  .addReg(FalseReg)
731  .addReg(TrueReg)
732  .addReg(SReg);
733  break;
734  }
735  case SIInstrInfo::VCCNZ: {
736  MachineOperand RegOp = Cond[1];
737  RegOp.setImplicit(false);
738  unsigned SReg = MRI.createVirtualRegister(&AMDGPU::SReg_64_XEXECRegClass);
739  BuildMI(MBB, I, DL, get(AMDGPU::COPY), SReg)
740  .add(RegOp);
741  BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
742  .addReg(FalseReg)
743  .addReg(TrueReg)
744  .addReg(SReg);
745  break;
746  }
747  case SIInstrInfo::VCCZ: {
748  MachineOperand RegOp = Cond[1];
749  RegOp.setImplicit(false);
750  unsigned SReg = MRI.createVirtualRegister(&AMDGPU::SReg_64_XEXECRegClass);
751  BuildMI(MBB, I, DL, get(AMDGPU::COPY), SReg)
752  .add(RegOp);
753  BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
754  .addReg(TrueReg)
755  .addReg(FalseReg)
756  .addReg(SReg);
757  break;
758  }
759  case SIInstrInfo::EXECNZ: {
760  unsigned SReg = MRI.createVirtualRegister(&AMDGPU::SReg_64_XEXECRegClass);
761  unsigned SReg2 = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
762  BuildMI(MBB, I, DL, get(AMDGPU::S_OR_SAVEEXEC_B64), SReg2)
763  .addImm(0);
764  BuildMI(MBB, I, DL, get(AMDGPU::S_CSELECT_B64), SReg)
765  .addImm(-1)
766  .addImm(0);
767  BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
768  .addReg(FalseReg)
769  .addReg(TrueReg)
770  .addReg(SReg);
771  break;
772  }
773  case SIInstrInfo::EXECZ: {
774  unsigned SReg = MRI.createVirtualRegister(&AMDGPU::SReg_64_XEXECRegClass);
775  unsigned SReg2 = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
776  BuildMI(MBB, I, DL, get(AMDGPU::S_OR_SAVEEXEC_B64), SReg2)
777  .addImm(0);
778  BuildMI(MBB, I, DL, get(AMDGPU::S_CSELECT_B64), SReg)
779  .addImm(0)
780  .addImm(-1);
781  BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
782  .addReg(FalseReg)
783  .addReg(TrueReg)
784  .addReg(SReg);
785  llvm_unreachable("Unhandled branch predicate EXECZ");
786  break;
787  }
788  default:
789  llvm_unreachable("invalid branch predicate");
790  }
791  } else {
792  llvm_unreachable("Can only handle Cond size 1 or 2");
793  }
794 }
795 
798  const DebugLoc &DL,
799  unsigned SrcReg, int Value) const {
801  unsigned Reg = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
802  BuildMI(*MBB, I, DL, get(AMDGPU::V_CMP_EQ_I32_e64), Reg)
803  .addImm(Value)
804  .addReg(SrcReg);
805 
806  return Reg;
807 }
808 
811  const DebugLoc &DL,
812  unsigned SrcReg, int Value) const {
814  unsigned Reg = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
815  BuildMI(*MBB, I, DL, get(AMDGPU::V_CMP_NE_I32_e64), Reg)
816  .addImm(Value)
817  .addReg(SrcReg);
818 
819  return Reg;
820 }
821 
822 unsigned SIInstrInfo::getMovOpcode(const TargetRegisterClass *DstRC) const {
823 
824  if (RI.getRegSizeInBits(*DstRC) == 32) {
825  return RI.isSGPRClass(DstRC) ? AMDGPU::S_MOV_B32 : AMDGPU::V_MOV_B32_e32;
826  } else if (RI.getRegSizeInBits(*DstRC) == 64 && RI.isSGPRClass(DstRC)) {
827  return AMDGPU::S_MOV_B64;
828  } else if (RI.getRegSizeInBits(*DstRC) == 64 && !RI.isSGPRClass(DstRC)) {
829  return AMDGPU::V_MOV_B64_PSEUDO;
830  }
831  return AMDGPU::COPY;
832 }
833 
834 static unsigned getSGPRSpillSaveOpcode(unsigned Size) {
835  switch (Size) {
836  case 4:
837  return AMDGPU::SI_SPILL_S32_SAVE;
838  case 8:
839  return AMDGPU::SI_SPILL_S64_SAVE;
840  case 16:
841  return AMDGPU::SI_SPILL_S128_SAVE;
842  case 32:
843  return AMDGPU::SI_SPILL_S256_SAVE;
844  case 64:
845  return AMDGPU::SI_SPILL_S512_SAVE;
846  default:
847  llvm_unreachable("unknown register size");
848  }
849 }
850 
851 static unsigned getVGPRSpillSaveOpcode(unsigned Size) {
852  switch (Size) {
853  case 4:
854  return AMDGPU::SI_SPILL_V32_SAVE;
855  case 8:
856  return AMDGPU::SI_SPILL_V64_SAVE;
857  case 12:
858  return AMDGPU::SI_SPILL_V96_SAVE;
859  case 16:
860  return AMDGPU::SI_SPILL_V128_SAVE;
861  case 32:
862  return AMDGPU::SI_SPILL_V256_SAVE;
863  case 64:
864  return AMDGPU::SI_SPILL_V512_SAVE;
865  default:
866  llvm_unreachable("unknown register size");
867  }
868 }
869 
872  unsigned SrcReg, bool isKill,
873  int FrameIndex,
874  const TargetRegisterClass *RC,
875  const TargetRegisterInfo *TRI) const {
876  MachineFunction *MF = MBB.getParent();
878  MachineFrameInfo &FrameInfo = MF->getFrameInfo();
879  const DebugLoc &DL = MBB.findDebugLoc(MI);
880 
881  unsigned Size = FrameInfo.getObjectSize(FrameIndex);
882  unsigned Align = FrameInfo.getObjectAlignment(FrameIndex);
883  MachinePointerInfo PtrInfo
884  = MachinePointerInfo::getFixedStack(*MF, FrameIndex);
885  MachineMemOperand *MMO
887  Size, Align);
888  unsigned SpillSize = TRI->getSpillSize(*RC);
889 
890  if (RI.isSGPRClass(RC)) {
891  MFI->setHasSpilledSGPRs();
892 
893  // We are only allowed to create one new instruction when spilling
894  // registers, so we need to use pseudo instruction for spilling SGPRs.
895  const MCInstrDesc &OpDesc = get(getSGPRSpillSaveOpcode(SpillSize));
896 
897  // The SGPR spill/restore instructions only work on number sgprs, so we need
898  // to make sure we are using the correct register class.
899  if (TargetRegisterInfo::isVirtualRegister(SrcReg) && SpillSize == 4) {
901  MRI.constrainRegClass(SrcReg, &AMDGPU::SReg_32_XM0RegClass);
902  }
903 
904  MachineInstrBuilder Spill = BuildMI(MBB, MI, DL, OpDesc)
905  .addReg(SrcReg, getKillRegState(isKill)) // data
906  .addFrameIndex(FrameIndex) // addr
907  .addMemOperand(MMO)
909  .addReg(MFI->getFrameOffsetReg(), RegState::Implicit);
910  // Add the scratch resource registers as implicit uses because we may end up
911  // needing them, and need to ensure that the reserved registers are
912  // correctly handled.
913 
914  FrameInfo.setStackID(FrameIndex, SIStackID::SGPR_SPILL);
915  if (ST.hasScalarStores()) {
916  // m0 is used for offset to scalar stores if used to spill.
917  Spill.addReg(AMDGPU::M0, RegState::ImplicitDefine | RegState::Dead);
918  }
919 
920  return;
921  }
922 
923  assert(RI.hasVGPRs(RC) && "Only VGPR spilling expected");
924 
925  unsigned Opcode = getVGPRSpillSaveOpcode(SpillSize);
926  MFI->setHasSpilledVGPRs();
927  BuildMI(MBB, MI, DL, get(Opcode))
928  .addReg(SrcReg, getKillRegState(isKill)) // data
929  .addFrameIndex(FrameIndex) // addr
930  .addReg(MFI->getScratchRSrcReg()) // scratch_rsrc
931  .addReg(MFI->getFrameOffsetReg()) // scratch_offset
932  .addImm(0) // offset
933  .addMemOperand(MMO);
934 }
935 
936 static unsigned getSGPRSpillRestoreOpcode(unsigned Size) {
937  switch (Size) {
938  case 4:
939  return AMDGPU::SI_SPILL_S32_RESTORE;
940  case 8:
941  return AMDGPU::SI_SPILL_S64_RESTORE;
942  case 16:
943  return AMDGPU::SI_SPILL_S128_RESTORE;
944  case 32:
945  return AMDGPU::SI_SPILL_S256_RESTORE;
946  case 64:
947  return AMDGPU::SI_SPILL_S512_RESTORE;
948  default:
949  llvm_unreachable("unknown register size");
950  }
951 }
952 
953 static unsigned getVGPRSpillRestoreOpcode(unsigned Size) {
954  switch (Size) {
955  case 4:
956  return AMDGPU::SI_SPILL_V32_RESTORE;
957  case 8:
958  return AMDGPU::SI_SPILL_V64_RESTORE;
959  case 12:
960  return AMDGPU::SI_SPILL_V96_RESTORE;
961  case 16:
962  return AMDGPU::SI_SPILL_V128_RESTORE;
963  case 32:
964  return AMDGPU::SI_SPILL_V256_RESTORE;
965  case 64:
966  return AMDGPU::SI_SPILL_V512_RESTORE;
967  default:
968  llvm_unreachable("unknown register size");
969  }
970 }
971 
974  unsigned DestReg, int FrameIndex,
975  const TargetRegisterClass *RC,
976  const TargetRegisterInfo *TRI) const {
977  MachineFunction *MF = MBB.getParent();
979  MachineFrameInfo &FrameInfo = MF->getFrameInfo();
980  const DebugLoc &DL = MBB.findDebugLoc(MI);
981  unsigned Align = FrameInfo.getObjectAlignment(FrameIndex);
982  unsigned Size = FrameInfo.getObjectSize(FrameIndex);
983  unsigned SpillSize = TRI->getSpillSize(*RC);
984 
985  MachinePointerInfo PtrInfo
986  = MachinePointerInfo::getFixedStack(*MF, FrameIndex);
987 
989  PtrInfo, MachineMemOperand::MOLoad, Size, Align);
990 
991  if (RI.isSGPRClass(RC)) {
992  MFI->setHasSpilledSGPRs();
993 
994  // FIXME: Maybe this should not include a memoperand because it will be
995  // lowered to non-memory instructions.
996  const MCInstrDesc &OpDesc = get(getSGPRSpillRestoreOpcode(SpillSize));
997  if (TargetRegisterInfo::isVirtualRegister(DestReg) && SpillSize == 4) {
999  MRI.constrainRegClass(DestReg, &AMDGPU::SReg_32_XM0RegClass);
1000  }
1001 
1002  FrameInfo.setStackID(FrameIndex, SIStackID::SGPR_SPILL);
1003  MachineInstrBuilder Spill = BuildMI(MBB, MI, DL, OpDesc, DestReg)
1004  .addFrameIndex(FrameIndex) // addr
1005  .addMemOperand(MMO)
1007  .addReg(MFI->getFrameOffsetReg(), RegState::Implicit);
1008 
1009  if (ST.hasScalarStores()) {
1010  // m0 is used for offset to scalar stores if used to spill.
1011  Spill.addReg(AMDGPU::M0, RegState::ImplicitDefine | RegState::Dead);
1012  }
1013 
1014  return;
1015  }
1016 
1017  assert(RI.hasVGPRs(RC) && "Only VGPR spilling expected");
1018 
1019  unsigned Opcode = getVGPRSpillRestoreOpcode(SpillSize);
1020  BuildMI(MBB, MI, DL, get(Opcode), DestReg)
1021  .addFrameIndex(FrameIndex) // vaddr
1022  .addReg(MFI->getScratchRSrcReg()) // scratch_rsrc
1023  .addReg(MFI->getFrameOffsetReg()) // scratch_offset
1024  .addImm(0) // offset
1025  .addMemOperand(MMO);
1026 }
1027 
1028 /// \param @Offset Offset in bytes of the FrameIndex being spilled
1030  MachineBasicBlock &MBB, MachineInstr &MI, RegScavenger *RS, unsigned TmpReg,
1031  unsigned FrameOffset, unsigned Size) const {
1032  MachineFunction *MF = MBB.getParent();
1034  const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
1035  const DebugLoc &DL = MBB.findDebugLoc(MI);
1036  unsigned WorkGroupSize = MFI->getMaxFlatWorkGroupSize();
1037  unsigned WavefrontSize = ST.getWavefrontSize();
1038 
1039  unsigned TIDReg = MFI->getTIDReg();
1040  if (!MFI->hasCalculatedTID()) {
1041  MachineBasicBlock &Entry = MBB.getParent()->front();
1042  MachineBasicBlock::iterator Insert = Entry.front();
1043  const DebugLoc &DL = Insert->getDebugLoc();
1044 
1045  TIDReg = RI.findUnusedRegister(MF->getRegInfo(), &AMDGPU::VGPR_32RegClass,
1046  *MF);
1047  if (TIDReg == AMDGPU::NoRegister)
1048  return TIDReg;
1049 
1051  WorkGroupSize > WavefrontSize) {
1052  unsigned TIDIGXReg
1054  unsigned TIDIGYReg
1056  unsigned TIDIGZReg
1058  unsigned InputPtrReg =
1060  for (unsigned Reg : {TIDIGXReg, TIDIGYReg, TIDIGZReg}) {
1061  if (!Entry.isLiveIn(Reg))
1062  Entry.addLiveIn(Reg);
1063  }
1064 
1065  RS->enterBasicBlock(Entry);
1066  // FIXME: Can we scavenge an SReg_64 and access the subregs?
1067  unsigned STmp0 = RS->scavengeRegister(&AMDGPU::SGPR_32RegClass, 0);
1068  unsigned STmp1 = RS->scavengeRegister(&AMDGPU::SGPR_32RegClass, 0);
1069  BuildMI(Entry, Insert, DL, get(AMDGPU::S_LOAD_DWORD_IMM), STmp0)
1070  .addReg(InputPtrReg)
1072  BuildMI(Entry, Insert, DL, get(AMDGPU::S_LOAD_DWORD_IMM), STmp1)
1073  .addReg(InputPtrReg)
1075 
1076  // NGROUPS.X * NGROUPS.Y
1077  BuildMI(Entry, Insert, DL, get(AMDGPU::S_MUL_I32), STmp1)
1078  .addReg(STmp1)
1079  .addReg(STmp0);
1080  // (NGROUPS.X * NGROUPS.Y) * TIDIG.X
1081  BuildMI(Entry, Insert, DL, get(AMDGPU::V_MUL_U32_U24_e32), TIDReg)
1082  .addReg(STmp1)
1083  .addReg(TIDIGXReg);
1084  // NGROUPS.Z * TIDIG.Y + (NGROUPS.X * NGROPUS.Y * TIDIG.X)
1085  BuildMI(Entry, Insert, DL, get(AMDGPU::V_MAD_U32_U24), TIDReg)
1086  .addReg(STmp0)
1087  .addReg(TIDIGYReg)
1088  .addReg(TIDReg);
1089  // (NGROUPS.Z * TIDIG.Y + (NGROUPS.X * NGROPUS.Y * TIDIG.X)) + TIDIG.Z
1090  getAddNoCarry(Entry, Insert, DL, TIDReg)
1091  .addReg(TIDReg)
1092  .addReg(TIDIGZReg);
1093  } else {
1094  // Get the wave id
1095  BuildMI(Entry, Insert, DL, get(AMDGPU::V_MBCNT_LO_U32_B32_e64),
1096  TIDReg)
1097  .addImm(-1)
1098  .addImm(0);
1099 
1100  BuildMI(Entry, Insert, DL, get(AMDGPU::V_MBCNT_HI_U32_B32_e64),
1101  TIDReg)
1102  .addImm(-1)
1103  .addReg(TIDReg);
1104  }
1105 
1106  BuildMI(Entry, Insert, DL, get(AMDGPU::V_LSHLREV_B32_e32),
1107  TIDReg)
1108  .addImm(2)
1109  .addReg(TIDReg);
1110  MFI->setTIDReg(TIDReg);
1111  }
1112 
1113  // Add FrameIndex to LDS offset
1114  unsigned LDSOffset = MFI->getLDSSize() + (FrameOffset * WorkGroupSize);
1115  getAddNoCarry(MBB, MI, DL, TmpReg)
1116  .addImm(LDSOffset)
1117  .addReg(TIDReg);
1118 
1119  return TmpReg;
1120 }
1121 
1124  int Count) const {
1125  DebugLoc DL = MBB.findDebugLoc(MI);
1126  while (Count > 0) {
1127  int Arg;
1128  if (Count >= 8)
1129  Arg = 7;
1130  else
1131  Arg = Count - 1;
1132  Count -= 8;
1133  BuildMI(MBB, MI, DL, get(AMDGPU::S_NOP))
1134  .addImm(Arg);
1135  }
1136 }
1137 
1140  insertWaitStates(MBB, MI, 1);
1141 }
1142 
1144  auto MF = MBB.getParent();
1145  SIMachineFunctionInfo *Info = MF->getInfo<SIMachineFunctionInfo>();
1146 
1147  assert(Info->isEntryFunction());
1148 
1149  if (MBB.succ_empty()) {
1150  bool HasNoTerminator = MBB.getFirstTerminator() == MBB.end();
1151  if (HasNoTerminator)
1152  BuildMI(MBB, MBB.end(), DebugLoc(),
1153  get(Info->returnsVoid() ? AMDGPU::S_ENDPGM : AMDGPU::SI_RETURN_TO_EPILOG));
1154  }
1155 }
1156 
1158  switch (MI.getOpcode()) {
1159  default: return 1; // FIXME: Do wait states equal cycles?
1160 
1161  case AMDGPU::S_NOP:
1162  return MI.getOperand(0).getImm() + 1;
1163  }
1164 }
1165 
1167  MachineBasicBlock &MBB = *MI.getParent();
1168  DebugLoc DL = MBB.findDebugLoc(MI);
1169  switch (MI.getOpcode()) {
1170  default: return TargetInstrInfo::expandPostRAPseudo(MI);
1171  case AMDGPU::S_MOV_B64_term:
1172  // This is only a terminator to get the correct spill code placement during
1173  // register allocation.
1174  MI.setDesc(get(AMDGPU::S_MOV_B64));
1175  break;
1176 
1177  case AMDGPU::S_XOR_B64_term:
1178  // This is only a terminator to get the correct spill code placement during
1179  // register allocation.
1180  MI.setDesc(get(AMDGPU::S_XOR_B64));
1181  break;
1182 
1183  case AMDGPU::S_ANDN2_B64_term:
1184  // This is only a terminator to get the correct spill code placement during
1185  // register allocation.
1186  MI.setDesc(get(AMDGPU::S_ANDN2_B64));
1187  break;
1188 
1189  case AMDGPU::V_MOV_B64_PSEUDO: {
1190  unsigned Dst = MI.getOperand(0).getReg();
1191  unsigned DstLo = RI.getSubReg(Dst, AMDGPU::sub0);
1192  unsigned DstHi = RI.getSubReg(Dst, AMDGPU::sub1);
1193 
1194  const MachineOperand &SrcOp = MI.getOperand(1);
1195  // FIXME: Will this work for 64-bit floating point immediates?
1196  assert(!SrcOp.isFPImm());
1197  if (SrcOp.isImm()) {
1198  APInt Imm(64, SrcOp.getImm());
1199  BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstLo)
1200  .addImm(Imm.getLoBits(32).getZExtValue())
1201  .addReg(Dst, RegState::Implicit | RegState::Define);
1202  BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstHi)
1203  .addImm(Imm.getHiBits(32).getZExtValue())
1204  .addReg(Dst, RegState::Implicit | RegState::Define);
1205  } else {
1206  assert(SrcOp.isReg());
1207  BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstLo)
1208  .addReg(RI.getSubReg(SrcOp.getReg(), AMDGPU::sub0))
1210  BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstHi)
1211  .addReg(RI.getSubReg(SrcOp.getReg(), AMDGPU::sub1))
1213  }
1214  MI.eraseFromParent();
1215  break;
1216  }
1217  case AMDGPU::V_SET_INACTIVE_B32: {
1218  BuildMI(MBB, MI, DL, get(AMDGPU::S_NOT_B64), AMDGPU::EXEC)
1219  .addReg(AMDGPU::EXEC);
1220  BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), MI.getOperand(0).getReg())
1221  .add(MI.getOperand(2));
1222  BuildMI(MBB, MI, DL, get(AMDGPU::S_NOT_B64), AMDGPU::EXEC)
1223  .addReg(AMDGPU::EXEC);
1224  MI.eraseFromParent();
1225  break;
1226  }
1227  case AMDGPU::V_SET_INACTIVE_B64: {
1228  BuildMI(MBB, MI, DL, get(AMDGPU::S_NOT_B64), AMDGPU::EXEC)
1229  .addReg(AMDGPU::EXEC);
1230  MachineInstr *Copy = BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B64_PSEUDO),
1231  MI.getOperand(0).getReg())
1232  .add(MI.getOperand(2));
1233  expandPostRAPseudo(*Copy);
1234  BuildMI(MBB, MI, DL, get(AMDGPU::S_NOT_B64), AMDGPU::EXEC)
1235  .addReg(AMDGPU::EXEC);
1236  MI.eraseFromParent();
1237  break;
1238  }
1239  case AMDGPU::V_MOVRELD_B32_V1:
1240  case AMDGPU::V_MOVRELD_B32_V2:
1241  case AMDGPU::V_MOVRELD_B32_V4:
1242  case AMDGPU::V_MOVRELD_B32_V8:
1243  case AMDGPU::V_MOVRELD_B32_V16: {
1244  const MCInstrDesc &MovRelDesc = get(AMDGPU::V_MOVRELD_B32_e32);
1245  unsigned VecReg = MI.getOperand(0).getReg();
1246  bool IsUndef = MI.getOperand(1).isUndef();
1247  unsigned SubReg = AMDGPU::sub0 + MI.getOperand(3).getImm();
1248  assert(VecReg == MI.getOperand(1).getReg());
1249 
1250  MachineInstr *MovRel =
1251  BuildMI(MBB, MI, DL, MovRelDesc)
1252  .addReg(RI.getSubReg(VecReg, SubReg), RegState::Undef)
1253  .add(MI.getOperand(2))
1254  .addReg(VecReg, RegState::ImplicitDefine)
1255  .addReg(VecReg,
1256  RegState::Implicit | (IsUndef ? RegState::Undef : 0));
1257 
1258  const int ImpDefIdx =
1259  MovRelDesc.getNumOperands() + MovRelDesc.getNumImplicitUses();
1260  const int ImpUseIdx = ImpDefIdx + 1;
1261  MovRel->tieOperands(ImpDefIdx, ImpUseIdx);
1262 
1263  MI.eraseFromParent();
1264  break;
1265  }
1266  case AMDGPU::SI_PC_ADD_REL_OFFSET: {
1267  MachineFunction &MF = *MBB.getParent();
1268  unsigned Reg = MI.getOperand(0).getReg();
1269  unsigned RegLo = RI.getSubReg(Reg, AMDGPU::sub0);
1270  unsigned RegHi = RI.getSubReg(Reg, AMDGPU::sub1);
1271 
1272  // Create a bundle so these instructions won't be re-ordered by the
1273  // post-RA scheduler.
1274  MIBundleBuilder Bundler(MBB, MI);
1275  Bundler.append(BuildMI(MF, DL, get(AMDGPU::S_GETPC_B64), Reg));
1276 
1277  // Add 32-bit offset from this instruction to the start of the
1278  // constant data.
1279  Bundler.append(BuildMI(MF, DL, get(AMDGPU::S_ADD_U32), RegLo)
1280  .addReg(RegLo)
1281  .add(MI.getOperand(1)));
1282 
1283  MachineInstrBuilder MIB = BuildMI(MF, DL, get(AMDGPU::S_ADDC_U32), RegHi)
1284  .addReg(RegHi);
1286  MIB.addImm(0);
1287  else
1288  MIB.add(MI.getOperand(2));
1289 
1290  Bundler.append(MIB);
1291  finalizeBundle(MBB, Bundler.begin());
1292 
1293  MI.eraseFromParent();
1294  break;
1295  }
1296  case AMDGPU::EXIT_WWM: {
1297  // This only gets its own opcode so that SIFixWWMLiveness can tell when WWM
1298  // is exited.
1299  MI.setDesc(get(AMDGPU::S_MOV_B64));
1300  break;
1301  }
1302  case TargetOpcode::BUNDLE: {
1303  if (!MI.mayLoad())
1304  return false;
1305 
1306  // If it is a load it must be a memory clause
1308  I->isBundledWithSucc(); ++I) {
1309  I->unbundleFromSucc();
1310  for (MachineOperand &MO : I->operands())
1311  if (MO.isReg())
1312  MO.setIsInternalRead(false);
1313  }
1314 
1315  MI.eraseFromParent();
1316  break;
1317  }
1318  }
1319  return true;
1320 }
1321 
1323  MachineOperand &Src0,
1324  unsigned Src0OpName,
1325  MachineOperand &Src1,
1326  unsigned Src1OpName) const {
1327  MachineOperand *Src0Mods = getNamedOperand(MI, Src0OpName);
1328  if (!Src0Mods)
1329  return false;
1330 
1331  MachineOperand *Src1Mods = getNamedOperand(MI, Src1OpName);
1332  assert(Src1Mods &&
1333  "All commutable instructions have both src0 and src1 modifiers");
1334 
1335  int Src0ModsVal = Src0Mods->getImm();
1336  int Src1ModsVal = Src1Mods->getImm();
1337 
1338  Src1Mods->setImm(Src0ModsVal);
1339  Src0Mods->setImm(Src1ModsVal);
1340  return true;
1341 }
1342 
1344  MachineOperand &RegOp,
1345  MachineOperand &NonRegOp) {
1346  unsigned Reg = RegOp.getReg();
1347  unsigned SubReg = RegOp.getSubReg();
1348  bool IsKill = RegOp.isKill();
1349  bool IsDead = RegOp.isDead();
1350  bool IsUndef = RegOp.isUndef();
1351  bool IsDebug = RegOp.isDebug();
1352 
1353  if (NonRegOp.isImm())
1354  RegOp.ChangeToImmediate(NonRegOp.getImm());
1355  else if (NonRegOp.isFI())
1356  RegOp.ChangeToFrameIndex(NonRegOp.getIndex());
1357  else
1358  return nullptr;
1359 
1360  NonRegOp.ChangeToRegister(Reg, false, false, IsKill, IsDead, IsUndef, IsDebug);
1361  NonRegOp.setSubReg(SubReg);
1362 
1363  return &MI;
1364 }
1365 
1367  unsigned Src0Idx,
1368  unsigned Src1Idx) const {
1369  assert(!NewMI && "this should never be used");
1370 
1371  unsigned Opc = MI.getOpcode();
1372  int CommutedOpcode = commuteOpcode(Opc);
1373  if (CommutedOpcode == -1)
1374  return nullptr;
1375 
1376  assert(AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0) ==
1377  static_cast<int>(Src0Idx) &&
1378  AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1) ==
1379  static_cast<int>(Src1Idx) &&
1380  "inconsistency with findCommutedOpIndices");
1381 
1382  MachineOperand &Src0 = MI.getOperand(Src0Idx);
1383  MachineOperand &Src1 = MI.getOperand(Src1Idx);
1384 
1385  MachineInstr *CommutedMI = nullptr;
1386  if (Src0.isReg() && Src1.isReg()) {
1387  if (isOperandLegal(MI, Src1Idx, &Src0)) {
1388  // Be sure to copy the source modifiers to the right place.
1389  CommutedMI
1390  = TargetInstrInfo::commuteInstructionImpl(MI, NewMI, Src0Idx, Src1Idx);
1391  }
1392 
1393  } else if (Src0.isReg() && !Src1.isReg()) {
1394  // src0 should always be able to support any operand type, so no need to
1395  // check operand legality.
1396  CommutedMI = swapRegAndNonRegOperand(MI, Src0, Src1);
1397  } else if (!Src0.isReg() && Src1.isReg()) {
1398  if (isOperandLegal(MI, Src1Idx, &Src0))
1399  CommutedMI = swapRegAndNonRegOperand(MI, Src1, Src0);
1400  } else {
1401  // FIXME: Found two non registers to commute. This does happen.
1402  return nullptr;
1403  }
1404 
1405  if (CommutedMI) {
1406  swapSourceModifiers(MI, Src0, AMDGPU::OpName::src0_modifiers,
1407  Src1, AMDGPU::OpName::src1_modifiers);
1408 
1409  CommutedMI->setDesc(get(CommutedOpcode));
1410  }
1411 
1412  return CommutedMI;
1413 }
1414 
1415 // This needs to be implemented because the source modifiers may be inserted
1416 // between the true commutable operands, and the base
1417 // TargetInstrInfo::commuteInstruction uses it.
1419  unsigned &SrcOpIdx1) const {
1420  return findCommutedOpIndices(MI.getDesc(), SrcOpIdx0, SrcOpIdx1);
1421 }
1422 
1423 bool SIInstrInfo::findCommutedOpIndices(MCInstrDesc Desc, unsigned &SrcOpIdx0,
1424  unsigned &SrcOpIdx1) const {
1425  if (!Desc.isCommutable())
1426  return false;
1427 
1428  unsigned Opc = Desc.getOpcode();
1429  int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0);
1430  if (Src0Idx == -1)
1431  return false;
1432 
1433  int Src1Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1);
1434  if (Src1Idx == -1)
1435  return false;
1436 
1437  return fixCommutedOpIndices(SrcOpIdx0, SrcOpIdx1, Src0Idx, Src1Idx);
1438 }
1439 
1440 bool SIInstrInfo::isBranchOffsetInRange(unsigned BranchOp,
1441  int64_t BrOffset) const {
1442  // BranchRelaxation should never have to check s_setpc_b64 because its dest
1443  // block is unanalyzable.
1444  assert(BranchOp != AMDGPU::S_SETPC_B64);
1445 
1446  // Convert to dwords.
1447  BrOffset /= 4;
1448 
1449  // The branch instructions do PC += signext(SIMM16 * 4) + 4, so the offset is
1450  // from the next instruction.
1451  BrOffset -= 1;
1452 
1453  return isIntN(BranchOffsetBits, BrOffset);
1454 }
1455 
1457  const MachineInstr &MI) const {
1458  if (MI.getOpcode() == AMDGPU::S_SETPC_B64) {
1459  // This would be a difficult analysis to perform, but can always be legal so
1460  // there's no need to analyze it.
1461  return nullptr;
1462  }
1463 
1464  return MI.getOperand(0).getMBB();
1465 }
1466 
1468  MachineBasicBlock &DestBB,
1469  const DebugLoc &DL,
1470  int64_t BrOffset,
1471  RegScavenger *RS) const {
1472  assert(RS && "RegScavenger required for long branching");
1473  assert(MBB.empty() &&
1474  "new block should be inserted for expanding unconditional branch");
1475  assert(MBB.pred_size() == 1);
1476 
1477  MachineFunction *MF = MBB.getParent();
1478  MachineRegisterInfo &MRI = MF->getRegInfo();
1479 
1480  // FIXME: Virtual register workaround for RegScavenger not working with empty
1481  // blocks.
1482  unsigned PCReg = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
1483 
1484  auto I = MBB.end();
1485 
1486  // We need to compute the offset relative to the instruction immediately after
1487  // s_getpc_b64. Insert pc arithmetic code before last terminator.
1488  MachineInstr *GetPC = BuildMI(MBB, I, DL, get(AMDGPU::S_GETPC_B64), PCReg);
1489 
1490  // TODO: Handle > 32-bit block address.
1491  if (BrOffset >= 0) {
1492  BuildMI(MBB, I, DL, get(AMDGPU::S_ADD_U32))
1493  .addReg(PCReg, RegState::Define, AMDGPU::sub0)
1494  .addReg(PCReg, 0, AMDGPU::sub0)
1496  BuildMI(MBB, I, DL, get(AMDGPU::S_ADDC_U32))
1497  .addReg(PCReg, RegState::Define, AMDGPU::sub1)
1498  .addReg(PCReg, 0, AMDGPU::sub1)
1499  .addImm(0);
1500  } else {
1501  // Backwards branch.
1502  BuildMI(MBB, I, DL, get(AMDGPU::S_SUB_U32))
1503  .addReg(PCReg, RegState::Define, AMDGPU::sub0)
1504  .addReg(PCReg, 0, AMDGPU::sub0)
1506  BuildMI(MBB, I, DL, get(AMDGPU::S_SUBB_U32))
1507  .addReg(PCReg, RegState::Define, AMDGPU::sub1)
1508  .addReg(PCReg, 0, AMDGPU::sub1)
1509  .addImm(0);
1510  }
1511 
1512  // Insert the indirect branch after the other terminator.
1513  BuildMI(&MBB, DL, get(AMDGPU::S_SETPC_B64))
1514  .addReg(PCReg);
1515 
1516  // FIXME: If spilling is necessary, this will fail because this scavenger has
1517  // no emergency stack slots. It is non-trivial to spill in this situation,
1518  // because the restore code needs to be specially placed after the
1519  // jump. BranchRelaxation then needs to be made aware of the newly inserted
1520  // block.
1521  //
1522  // If a spill is needed for the pc register pair, we need to insert a spill
1523  // restore block right before the destination block, and insert a short branch
1524  // into the old destination block's fallthrough predecessor.
1525  // e.g.:
1526  //
1527  // s_cbranch_scc0 skip_long_branch:
1528  //
1529  // long_branch_bb:
1530  // spill s[8:9]
1531  // s_getpc_b64 s[8:9]
1532  // s_add_u32 s8, s8, restore_bb
1533  // s_addc_u32 s9, s9, 0
1534  // s_setpc_b64 s[8:9]
1535  //
1536  // skip_long_branch:
1537  // foo;
1538  //
1539  // .....
1540  //
1541  // dest_bb_fallthrough_predecessor:
1542  // bar;
1543  // s_branch dest_bb
1544  //
1545  // restore_bb:
1546  // restore s[8:9]
1547  // fallthrough dest_bb
1548  ///
1549  // dest_bb:
1550  // buzz;
1551 
1552  RS->enterBasicBlockEnd(MBB);
1553  unsigned Scav = RS->scavengeRegisterBackwards(
1554  AMDGPU::SReg_64RegClass,
1555  MachineBasicBlock::iterator(GetPC), false, 0);
1556  MRI.replaceRegWith(PCReg, Scav);
1557  MRI.clearVirtRegs();
1558  RS->setRegUsed(Scav);
1559 
1560  return 4 + 8 + 4 + 4;
1561 }
1562 
1563 unsigned SIInstrInfo::getBranchOpcode(SIInstrInfo::BranchPredicate Cond) {
1564  switch (Cond) {
1565  case SIInstrInfo::SCC_TRUE:
1566  return AMDGPU::S_CBRANCH_SCC1;
1567  case SIInstrInfo::SCC_FALSE:
1568  return AMDGPU::S_CBRANCH_SCC0;
1569  case SIInstrInfo::VCCNZ:
1570  return AMDGPU::S_CBRANCH_VCCNZ;
1571  case SIInstrInfo::VCCZ:
1572  return AMDGPU::S_CBRANCH_VCCZ;
1573  case SIInstrInfo::EXECNZ:
1574  return AMDGPU::S_CBRANCH_EXECNZ;
1575  case SIInstrInfo::EXECZ:
1576  return AMDGPU::S_CBRANCH_EXECZ;
1577  default:
1578  llvm_unreachable("invalid branch predicate");
1579  }
1580 }
1581 
1582 SIInstrInfo::BranchPredicate SIInstrInfo::getBranchPredicate(unsigned Opcode) {
1583  switch (Opcode) {
1584  case AMDGPU::S_CBRANCH_SCC0:
1585  return SCC_FALSE;
1586  case AMDGPU::S_CBRANCH_SCC1:
1587  return SCC_TRUE;
1588  case AMDGPU::S_CBRANCH_VCCNZ:
1589  return VCCNZ;
1590  case AMDGPU::S_CBRANCH_VCCZ:
1591  return VCCZ;
1592  case AMDGPU::S_CBRANCH_EXECNZ:
1593  return EXECNZ;
1594  case AMDGPU::S_CBRANCH_EXECZ:
1595  return EXECZ;
1596  default:
1597  return INVALID_BR;
1598  }
1599 }
1600 
1603  MachineBasicBlock *&TBB,
1604  MachineBasicBlock *&FBB,
1606  bool AllowModify) const {
1607  if (I->getOpcode() == AMDGPU::S_BRANCH) {
1608  // Unconditional Branch
1609  TBB = I->getOperand(0).getMBB();
1610  return false;
1611  }
1612 
1613  MachineBasicBlock *CondBB = nullptr;
1614 
1615  if (I->getOpcode() == AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO) {
1616  CondBB = I->getOperand(1).getMBB();
1617  Cond.push_back(I->getOperand(0));
1618  } else {
1619  BranchPredicate Pred = getBranchPredicate(I->getOpcode());
1620  if (Pred == INVALID_BR)
1621  return true;
1622 
1623  CondBB = I->getOperand(0).getMBB();
1625  Cond.push_back(I->getOperand(1)); // Save the branch register.
1626  }
1627  ++I;
1628 
1629  if (I == MBB.end()) {
1630  // Conditional branch followed by fall-through.
1631  TBB = CondBB;
1632  return false;
1633  }
1634 
1635  if (I->getOpcode() == AMDGPU::S_BRANCH) {
1636  TBB = CondBB;
1637  FBB = I->getOperand(0).getMBB();
1638  return false;
1639  }
1640 
1641  return true;
1642 }
1643 
1645  MachineBasicBlock *&FBB,
1647  bool AllowModify) const {
1649  auto E = MBB.end();
1650  if (I == E)
1651  return false;
1652 
1653  // Skip over the instructions that are artificially terminators for special
1654  // exec management.
1655  while (I != E && !I->isBranch() && !I->isReturn() &&
1656  I->getOpcode() != AMDGPU::SI_MASK_BRANCH) {
1657  switch (I->getOpcode()) {
1658  case AMDGPU::SI_MASK_BRANCH:
1659  case AMDGPU::S_MOV_B64_term:
1660  case AMDGPU::S_XOR_B64_term:
1661  case AMDGPU::S_ANDN2_B64_term:
1662  break;
1663  case AMDGPU::SI_IF:
1664  case AMDGPU::SI_ELSE:
1665  case AMDGPU::SI_KILL_I1_TERMINATOR:
1666  case AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR:
1667  // FIXME: It's messy that these need to be considered here at all.
1668  return true;
1669  default:
1670  llvm_unreachable("unexpected non-branch terminator inst");
1671  }
1672 
1673  ++I;
1674  }
1675 
1676  if (I == E)
1677  return false;
1678 
1679  if (I->getOpcode() != AMDGPU::SI_MASK_BRANCH)
1680  return analyzeBranchImpl(MBB, I, TBB, FBB, Cond, AllowModify);
1681 
1682  ++I;
1683 
1684  // TODO: Should be able to treat as fallthrough?
1685  if (I == MBB.end())
1686  return true;
1687 
1688  if (analyzeBranchImpl(MBB, I, TBB, FBB, Cond, AllowModify))
1689  return true;
1690 
1691  MachineBasicBlock *MaskBrDest = I->getOperand(0).getMBB();
1692 
1693  // Specifically handle the case where the conditional branch is to the same
1694  // destination as the mask branch. e.g.
1695  //
1696  // si_mask_branch BB8
1697  // s_cbranch_execz BB8
1698  // s_cbranch BB9
1699  //
1700  // This is required to understand divergent loops which may need the branches
1701  // to be relaxed.
1702  if (TBB != MaskBrDest || Cond.empty())
1703  return true;
1704 
1705  auto Pred = Cond[0].getImm();
1706  return (Pred != EXECZ && Pred != EXECNZ);
1707 }
1708 
1710  int *BytesRemoved) const {
1712 
1713  unsigned Count = 0;
1714  unsigned RemovedSize = 0;
1715  while (I != MBB.end()) {
1716  MachineBasicBlock::iterator Next = std::next(I);
1717  if (I->getOpcode() == AMDGPU::SI_MASK_BRANCH) {
1718  I = Next;
1719  continue;
1720  }
1721 
1722  RemovedSize += getInstSizeInBytes(*I);
1723  I->eraseFromParent();
1724  ++Count;
1725  I = Next;
1726  }
1727 
1728  if (BytesRemoved)
1729  *BytesRemoved = RemovedSize;
1730 
1731  return Count;
1732 }
1733 
1734 // Copy the flags onto the implicit condition register operand.
1736  const MachineOperand &OrigCond) {
1737  CondReg.setIsUndef(OrigCond.isUndef());
1738  CondReg.setIsKill(OrigCond.isKill());
1739 }
1740 
1742  MachineBasicBlock *TBB,
1743  MachineBasicBlock *FBB,
1745  const DebugLoc &DL,
1746  int *BytesAdded) const {
1747  if (!FBB && Cond.empty()) {
1748  BuildMI(&MBB, DL, get(AMDGPU::S_BRANCH))
1749  .addMBB(TBB);
1750  if (BytesAdded)
1751  *BytesAdded = 4;
1752  return 1;
1753  }
1754 
1755  if(Cond.size() == 1 && Cond[0].isReg()) {
1756  BuildMI(&MBB, DL, get(AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO))
1757  .add(Cond[0])
1758  .addMBB(TBB);
1759  return 1;
1760  }
1761 
1762  assert(TBB && Cond[0].isImm());
1763 
1764  unsigned Opcode
1765  = getBranchOpcode(static_cast<BranchPredicate>(Cond[0].getImm()));
1766 
1767  if (!FBB) {
1768  Cond[1].isUndef();
1769  MachineInstr *CondBr =
1770  BuildMI(&MBB, DL, get(Opcode))
1771  .addMBB(TBB);
1772 
1773  // Copy the flags onto the implicit condition register operand.
1774  preserveCondRegFlags(CondBr->getOperand(1), Cond[1]);
1775 
1776  if (BytesAdded)
1777  *BytesAdded = 4;
1778  return 1;
1779  }
1780 
1781  assert(TBB && FBB);
1782 
1783  MachineInstr *CondBr =
1784  BuildMI(&MBB, DL, get(Opcode))
1785  .addMBB(TBB);
1786  BuildMI(&MBB, DL, get(AMDGPU::S_BRANCH))
1787  .addMBB(FBB);
1788 
1789  MachineOperand &CondReg = CondBr->getOperand(1);
1790  CondReg.setIsUndef(Cond[1].isUndef());
1791  CondReg.setIsKill(Cond[1].isKill());
1792 
1793  if (BytesAdded)
1794  *BytesAdded = 8;
1795 
1796  return 2;
1797 }
1798 
1800  SmallVectorImpl<MachineOperand> &Cond) const {
1801  if (Cond.size() != 2) {
1802  return true;
1803  }
1804 
1805  if (Cond[0].isImm()) {
1806  Cond[0].setImm(-Cond[0].getImm());
1807  return false;
1808  }
1809 
1810  return true;
1811 }
1812 
1815  unsigned TrueReg, unsigned FalseReg,
1816  int &CondCycles,
1817  int &TrueCycles, int &FalseCycles) const {
1818  switch (Cond[0].getImm()) {
1819  case VCCNZ:
1820  case VCCZ: {
1821  const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
1822  const TargetRegisterClass *RC = MRI.getRegClass(TrueReg);
1823  assert(MRI.getRegClass(FalseReg) == RC);
1824 
1825  int NumInsts = AMDGPU::getRegBitWidth(RC->getID()) / 32;
1826  CondCycles = TrueCycles = FalseCycles = NumInsts; // ???
1827 
1828  // Limit to equal cost for branch vs. N v_cndmask_b32s.
1829  return !RI.isSGPRClass(RC) && NumInsts <= 6;
1830  }
1831  case SCC_TRUE:
1832  case SCC_FALSE: {
1833  // FIXME: We could insert for VGPRs if we could replace the original compare
1834  // with a vector one.
1835  const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
1836  const TargetRegisterClass *RC = MRI.getRegClass(TrueReg);
1837  assert(MRI.getRegClass(FalseReg) == RC);
1838 
1839  int NumInsts = AMDGPU::getRegBitWidth(RC->getID()) / 32;
1840 
1841  // Multiples of 8 can do s_cselect_b64
1842  if (NumInsts % 2 == 0)
1843  NumInsts /= 2;
1844 
1845  CondCycles = TrueCycles = FalseCycles = NumInsts; // ???
1846  return RI.isSGPRClass(RC);
1847  }
1848  default:
1849  return false;
1850  }
1851 }
1852 
1855  unsigned DstReg, ArrayRef<MachineOperand> Cond,
1856  unsigned TrueReg, unsigned FalseReg) const {
1857  BranchPredicate Pred = static_cast<BranchPredicate>(Cond[0].getImm());
1858  if (Pred == VCCZ || Pred == SCC_FALSE) {
1859  Pred = static_cast<BranchPredicate>(-Pred);
1860  std::swap(TrueReg, FalseReg);
1861  }
1862 
1864  const TargetRegisterClass *DstRC = MRI.getRegClass(DstReg);
1865  unsigned DstSize = RI.getRegSizeInBits(*DstRC);
1866 
1867  if (DstSize == 32) {
1868  unsigned SelOp = Pred == SCC_TRUE ?
1869  AMDGPU::S_CSELECT_B32 : AMDGPU::V_CNDMASK_B32_e32;
1870 
1871  // Instruction's operands are backwards from what is expected.
1872  MachineInstr *Select =
1873  BuildMI(MBB, I, DL, get(SelOp), DstReg)
1874  .addReg(FalseReg)
1875  .addReg(TrueReg);
1876 
1877  preserveCondRegFlags(Select->getOperand(3), Cond[1]);
1878  return;
1879  }
1880 
1881  if (DstSize == 64 && Pred == SCC_TRUE) {
1882  MachineInstr *Select =
1883  BuildMI(MBB, I, DL, get(AMDGPU::S_CSELECT_B64), DstReg)
1884  .addReg(FalseReg)
1885  .addReg(TrueReg);
1886 
1887  preserveCondRegFlags(Select->getOperand(3), Cond[1]);
1888  return;
1889  }
1890 
1891  static const int16_t Sub0_15[] = {
1892  AMDGPU::sub0, AMDGPU::sub1, AMDGPU::sub2, AMDGPU::sub3,
1893  AMDGPU::sub4, AMDGPU::sub5, AMDGPU::sub6, AMDGPU::sub7,
1894  AMDGPU::sub8, AMDGPU::sub9, AMDGPU::sub10, AMDGPU::sub11,
1895  AMDGPU::sub12, AMDGPU::sub13, AMDGPU::sub14, AMDGPU::sub15,
1896  };
1897 
1898  static const int16_t Sub0_15_64[] = {
1899  AMDGPU::sub0_sub1, AMDGPU::sub2_sub3,
1900  AMDGPU::sub4_sub5, AMDGPU::sub6_sub7,
1901  AMDGPU::sub8_sub9, AMDGPU::sub10_sub11,
1902  AMDGPU::sub12_sub13, AMDGPU::sub14_sub15,
1903  };
1904 
1905  unsigned SelOp = AMDGPU::V_CNDMASK_B32_e32;
1906  const TargetRegisterClass *EltRC = &AMDGPU::VGPR_32RegClass;
1907  const int16_t *SubIndices = Sub0_15;
1908  int NElts = DstSize / 32;
1909 
1910  // 64-bit select is only avaialble for SALU.
1911  if (Pred == SCC_TRUE) {
1912  SelOp = AMDGPU::S_CSELECT_B64;
1913  EltRC = &AMDGPU::SGPR_64RegClass;
1914  SubIndices = Sub0_15_64;
1915 
1916  assert(NElts % 2 == 0);
1917  NElts /= 2;
1918  }
1919 
1921  MBB, I, DL, get(AMDGPU::REG_SEQUENCE), DstReg);
1922 
1923  I = MIB->getIterator();
1924 
1926  for (int Idx = 0; Idx != NElts; ++Idx) {
1927  unsigned DstElt = MRI.createVirtualRegister(EltRC);
1928  Regs.push_back(DstElt);
1929 
1930  unsigned SubIdx = SubIndices[Idx];
1931 
1932  MachineInstr *Select =
1933  BuildMI(MBB, I, DL, get(SelOp), DstElt)
1934  .addReg(FalseReg, 0, SubIdx)
1935  .addReg(TrueReg, 0, SubIdx);
1936  preserveCondRegFlags(Select->getOperand(3), Cond[1]);
1937 
1938  MIB.addReg(DstElt)
1939  .addImm(SubIdx);
1940  }
1941 }
1942 
1944  switch (MI.getOpcode()) {
1945  case AMDGPU::V_MOV_B32_e32:
1946  case AMDGPU::V_MOV_B32_e64:
1947  case AMDGPU::V_MOV_B64_PSEUDO: {
1948  // If there are additional implicit register operands, this may be used for
1949  // register indexing so the source register operand isn't simply copied.
1950  unsigned NumOps = MI.getDesc().getNumOperands() +
1951  MI.getDesc().getNumImplicitUses();
1952 
1953  return MI.getNumOperands() == NumOps;
1954  }
1955  case AMDGPU::S_MOV_B32:
1956  case AMDGPU::S_MOV_B64:
1957  case AMDGPU::COPY:
1958  return true;
1959  default:
1960  return false;
1961  }
1962 }
1963 
1965  unsigned Kind) const {
1966  switch(Kind) {
1977  }
1978  return AMDGPUAS::FLAT_ADDRESS;
1979 }
1980 
1982  unsigned Opc = MI.getOpcode();
1983  int Src0ModIdx = AMDGPU::getNamedOperandIdx(Opc,
1984  AMDGPU::OpName::src0_modifiers);
1985  int Src1ModIdx = AMDGPU::getNamedOperandIdx(Opc,
1986  AMDGPU::OpName::src1_modifiers);
1987  int Src2ModIdx = AMDGPU::getNamedOperandIdx(Opc,
1988  AMDGPU::OpName::src2_modifiers);
1989 
1990  MI.RemoveOperand(Src2ModIdx);
1991  MI.RemoveOperand(Src1ModIdx);
1992  MI.RemoveOperand(Src0ModIdx);
1993 }
1994 
1996  unsigned Reg, MachineRegisterInfo *MRI) const {
1997  if (!MRI->hasOneNonDBGUse(Reg))
1998  return false;
1999 
2000  switch (DefMI.getOpcode()) {
2001  default:
2002  return false;
2003  case AMDGPU::S_MOV_B64:
2004  // TODO: We could fold 64-bit immediates, but this get compilicated
2005  // when there are sub-registers.
2006  return false;
2007 
2008  case AMDGPU::V_MOV_B32_e32:
2009  case AMDGPU::S_MOV_B32:
2010  break;
2011  }
2012 
2013  const MachineOperand *ImmOp = getNamedOperand(DefMI, AMDGPU::OpName::src0);
2014  assert(ImmOp);
2015  // FIXME: We could handle FrameIndex values here.
2016  if (!ImmOp->isImm())
2017  return false;
2018 
2019  unsigned Opc = UseMI.getOpcode();
2020  if (Opc == AMDGPU::COPY) {
2021  bool isVGPRCopy = RI.isVGPR(*MRI, UseMI.getOperand(0).getReg());
2022  unsigned NewOpc = isVGPRCopy ? AMDGPU::V_MOV_B32_e32 : AMDGPU::S_MOV_B32;
2023  UseMI.setDesc(get(NewOpc));
2024  UseMI.getOperand(1).ChangeToImmediate(ImmOp->getImm());
2025  UseMI.addImplicitDefUseOperands(*UseMI.getParent()->getParent());
2026  return true;
2027  }
2028 
2029  if (Opc == AMDGPU::V_MAD_F32 || Opc == AMDGPU::V_MAC_F32_e64 ||
2030  Opc == AMDGPU::V_MAD_F16 || Opc == AMDGPU::V_MAC_F16_e64) {
2031  // Don't fold if we are using source or output modifiers. The new VOP2
2032  // instructions don't have them.
2033  if (hasAnyModifiersSet(UseMI))
2034  return false;
2035 
2036  // If this is a free constant, there's no reason to do this.
2037  // TODO: We could fold this here instead of letting SIFoldOperands do it
2038  // later.
2039  MachineOperand *Src0 = getNamedOperand(UseMI, AMDGPU::OpName::src0);
2040 
2041  // Any src operand can be used for the legality check.
2042  if (isInlineConstant(UseMI, *Src0, *ImmOp))
2043  return false;
2044 
2045  bool IsF32 = Opc == AMDGPU::V_MAD_F32 || Opc == AMDGPU::V_MAC_F32_e64;
2046  MachineOperand *Src1 = getNamedOperand(UseMI, AMDGPU::OpName::src1);
2047  MachineOperand *Src2 = getNamedOperand(UseMI, AMDGPU::OpName::src2);
2048 
2049  // Multiplied part is the constant: Use v_madmk_{f16, f32}.
2050  // We should only expect these to be on src0 due to canonicalizations.
2051  if (Src0->isReg() && Src0->getReg() == Reg) {
2052  if (!Src1->isReg() || RI.isSGPRClass(MRI->getRegClass(Src1->getReg())))
2053  return false;
2054 
2055  if (!Src2->isReg() || RI.isSGPRClass(MRI->getRegClass(Src2->getReg())))
2056  return false;
2057 
2058  // We need to swap operands 0 and 1 since madmk constant is at operand 1.
2059 
2060  const int64_t Imm = ImmOp->getImm();
2061 
2062  // FIXME: This would be a lot easier if we could return a new instruction
2063  // instead of having to modify in place.
2064 
2065  // Remove these first since they are at the end.
2066  UseMI.RemoveOperand(
2067  AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::omod));
2068  UseMI.RemoveOperand(
2069  AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::clamp));
2070 
2071  unsigned Src1Reg = Src1->getReg();
2072  unsigned Src1SubReg = Src1->getSubReg();
2073  Src0->setReg(Src1Reg);
2074  Src0->setSubReg(Src1SubReg);
2075  Src0->setIsKill(Src1->isKill());
2076 
2077  if (Opc == AMDGPU::V_MAC_F32_e64 ||
2078  Opc == AMDGPU::V_MAC_F16_e64)
2079  UseMI.untieRegOperand(
2080  AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2));
2081 
2082  Src1->ChangeToImmediate(Imm);
2083 
2084  removeModOperands(UseMI);
2085  UseMI.setDesc(get(IsF32 ? AMDGPU::V_MADMK_F32 : AMDGPU::V_MADMK_F16));
2086 
2087  bool DeleteDef = MRI->hasOneNonDBGUse(Reg);
2088  if (DeleteDef)
2089  DefMI.eraseFromParent();
2090 
2091  return true;
2092  }
2093 
2094  // Added part is the constant: Use v_madak_{f16, f32}.
2095  if (Src2->isReg() && Src2->getReg() == Reg) {
2096  // Not allowed to use constant bus for another operand.
2097  // We can however allow an inline immediate as src0.
2098  bool Src0Inlined = false;
2099  if (Src0->isReg()) {
2100  // Try to inline constant if possible.
2101  // If the Def moves immediate and the use is single
2102  // We are saving VGPR here.
2103  MachineInstr *Def = MRI->getUniqueVRegDef(Src0->getReg());
2104  if (Def && Def->isMoveImmediate() &&
2105  isInlineConstant(Def->getOperand(1)) &&
2106  MRI->hasOneUse(Src0->getReg())) {
2107  Src0->ChangeToImmediate(Def->getOperand(1).getImm());
2108  Src0Inlined = true;
2109  } else if ((RI.isPhysicalRegister(Src0->getReg()) &&
2110  RI.isSGPRClass(RI.getPhysRegClass(Src0->getReg()))) ||
2111  (RI.isVirtualRegister(Src0->getReg()) &&
2112  RI.isSGPRClass(MRI->getRegClass(Src0->getReg()))))
2113  return false;
2114  // VGPR is okay as Src0 - fallthrough
2115  }
2116 
2117  if (Src1->isReg() && !Src0Inlined ) {
2118  // We have one slot for inlinable constant so far - try to fill it
2119  MachineInstr *Def = MRI->getUniqueVRegDef(Src1->getReg());
2120  if (Def && Def->isMoveImmediate() &&
2121  isInlineConstant(Def->getOperand(1)) &&
2122  MRI->hasOneUse(Src1->getReg()) &&
2123  commuteInstruction(UseMI)) {
2124  Src0->ChangeToImmediate(Def->getOperand(1).getImm());
2125  } else if ((RI.isPhysicalRegister(Src1->getReg()) &&
2126  RI.isSGPRClass(RI.getPhysRegClass(Src1->getReg()))) ||
2127  (RI.isVirtualRegister(Src1->getReg()) &&
2128  RI.isSGPRClass(MRI->getRegClass(Src1->getReg()))))
2129  return false;
2130  // VGPR is okay as Src1 - fallthrough
2131  }
2132 
2133  const int64_t Imm = ImmOp->getImm();
2134 
2135  // FIXME: This would be a lot easier if we could return a new instruction
2136  // instead of having to modify in place.
2137 
2138  // Remove these first since they are at the end.
2139  UseMI.RemoveOperand(
2140  AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::omod));
2141  UseMI.RemoveOperand(
2142  AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::clamp));
2143 
2144  if (Opc == AMDGPU::V_MAC_F32_e64 ||
2145  Opc == AMDGPU::V_MAC_F16_e64)
2146  UseMI.untieRegOperand(
2147  AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2));
2148 
2149  // ChangingToImmediate adds Src2 back to the instruction.
2150  Src2->ChangeToImmediate(Imm);
2151 
2152  // These come before src2.
2153  removeModOperands(UseMI);
2154  UseMI.setDesc(get(IsF32 ? AMDGPU::V_MADAK_F32 : AMDGPU::V_MADAK_F16));
2155 
2156  bool DeleteDef = MRI->hasOneNonDBGUse(Reg);
2157  if (DeleteDef)
2158  DefMI.eraseFromParent();
2159 
2160  return true;
2161  }
2162  }
2163 
2164  return false;
2165 }
2166 
2167 static bool offsetsDoNotOverlap(int WidthA, int OffsetA,
2168  int WidthB, int OffsetB) {
2169  int LowOffset = OffsetA < OffsetB ? OffsetA : OffsetB;
2170  int HighOffset = OffsetA < OffsetB ? OffsetB : OffsetA;
2171  int LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB;
2172  return LowOffset + LowWidth <= HighOffset;
2173 }
2174 
2175 bool SIInstrInfo::checkInstOffsetsDoNotOverlap(MachineInstr &MIa,
2176  MachineInstr &MIb) const {
2177  MachineOperand *BaseOp0, *BaseOp1;
2178  int64_t Offset0, Offset1;
2179 
2180  if (getMemOperandWithOffset(MIa, BaseOp0, Offset0, &RI) &&
2181  getMemOperandWithOffset(MIb, BaseOp1, Offset1, &RI)) {
2182  if (!BaseOp0->isIdenticalTo(*BaseOp1))
2183  return false;
2184 
2185  if (!MIa.hasOneMemOperand() || !MIb.hasOneMemOperand()) {
2186  // FIXME: Handle ds_read2 / ds_write2.
2187  return false;
2188  }
2189  unsigned Width0 = (*MIa.memoperands_begin())->getSize();
2190  unsigned Width1 = (*MIb.memoperands_begin())->getSize();
2191  if (offsetsDoNotOverlap(Width0, Offset0, Width1, Offset1)) {
2192  return true;
2193  }
2194  }
2195 
2196  return false;
2197 }
2198 
2200  MachineInstr &MIb,
2201  AliasAnalysis *AA) const {
2202  assert((MIa.mayLoad() || MIa.mayStore()) &&
2203  "MIa must load from or modify a memory location");
2204  assert((MIb.mayLoad() || MIb.mayStore()) &&
2205  "MIb must load from or modify a memory location");
2206 
2208  return false;
2209 
2210  // XXX - Can we relax this between address spaces?
2211  if (MIa.hasOrderedMemoryRef() || MIb.hasOrderedMemoryRef())
2212  return false;
2213 
2214  if (AA && MIa.hasOneMemOperand() && MIb.hasOneMemOperand()) {
2215  const MachineMemOperand *MMOa = *MIa.memoperands_begin();
2216  const MachineMemOperand *MMOb = *MIb.memoperands_begin();
2217  if (MMOa->getValue() && MMOb->getValue()) {
2218  MemoryLocation LocA(MMOa->getValue(), MMOa->getSize(), MMOa->getAAInfo());
2219  MemoryLocation LocB(MMOb->getValue(), MMOb->getSize(), MMOb->getAAInfo());
2220  if (!AA->alias(LocA, LocB))
2221  return true;
2222  }
2223  }
2224 
2225  // TODO: Should we check the address space from the MachineMemOperand? That
2226  // would allow us to distinguish objects we know don't alias based on the
2227  // underlying address space, even if it was lowered to a different one,
2228  // e.g. private accesses lowered to use MUBUF instructions on a scratch
2229  // buffer.
2230  if (isDS(MIa)) {
2231  if (isDS(MIb))
2232  return checkInstOffsetsDoNotOverlap(MIa, MIb);
2233 
2234  return !isFLAT(MIb) || isSegmentSpecificFLAT(MIb);
2235  }
2236 
2237  if (isMUBUF(MIa) || isMTBUF(MIa)) {
2238  if (isMUBUF(MIb) || isMTBUF(MIb))
2239  return checkInstOffsetsDoNotOverlap(MIa, MIb);
2240 
2241  return !isFLAT(MIb) && !isSMRD(MIb);
2242  }
2243 
2244  if (isSMRD(MIa)) {
2245  if (isSMRD(MIb))
2246  return checkInstOffsetsDoNotOverlap(MIa, MIb);
2247 
2248  return !isFLAT(MIb) && !isMUBUF(MIa) && !isMTBUF(MIa);
2249  }
2250 
2251  if (isFLAT(MIa)) {
2252  if (isFLAT(MIb))
2253  return checkInstOffsetsDoNotOverlap(MIa, MIb);
2254 
2255  return false;
2256  }
2257 
2258  return false;
2259 }
2260 
2261 static int64_t getFoldableImm(const MachineOperand* MO) {
2262  if (!MO->isReg())
2263  return false;
2264  const MachineFunction *MF = MO->getParent()->getParent()->getParent();
2265  const MachineRegisterInfo &MRI = MF->getRegInfo();
2266  auto Def = MRI.getUniqueVRegDef(MO->getReg());
2267  if (Def && Def->getOpcode() == AMDGPU::V_MOV_B32_e32 &&
2268  Def->getOperand(1).isImm())
2269  return Def->getOperand(1).getImm();
2270  return AMDGPU::NoRegister;
2271 }
2272 
2274  MachineInstr &MI,
2275  LiveVariables *LV) const {
2276  unsigned Opc = MI.getOpcode();
2277  bool IsF16 = false;
2278  bool IsFMA = Opc == AMDGPU::V_FMAC_F32_e32 || Opc == AMDGPU::V_FMAC_F32_e64;
2279 
2280  switch (Opc) {
2281  default:
2282  return nullptr;
2283  case AMDGPU::V_MAC_F16_e64:
2284  IsF16 = true;
2286  case AMDGPU::V_MAC_F32_e64:
2287  case AMDGPU::V_FMAC_F32_e64:
2288  break;
2289  case AMDGPU::V_MAC_F16_e32:
2290  IsF16 = true;
2292  case AMDGPU::V_MAC_F32_e32:
2293  case AMDGPU::V_FMAC_F32_e32: {
2294  int Src0Idx = AMDGPU::getNamedOperandIdx(MI.getOpcode(),
2295  AMDGPU::OpName::src0);
2296  const MachineOperand *Src0 = &MI.getOperand(Src0Idx);
2297  if (!Src0->isReg() && !Src0->isImm())
2298  return nullptr;
2299 
2300  if (Src0->isImm() && !isInlineConstant(MI, Src0Idx, *Src0))
2301  return nullptr;
2302 
2303  break;
2304  }
2305  }
2306 
2307  const MachineOperand *Dst = getNamedOperand(MI, AMDGPU::OpName::vdst);
2308  const MachineOperand *Src0 = getNamedOperand(MI, AMDGPU::OpName::src0);
2309  const MachineOperand *Src0Mods =
2310  getNamedOperand(MI, AMDGPU::OpName::src0_modifiers);
2311  const MachineOperand *Src1 = getNamedOperand(MI, AMDGPU::OpName::src1);
2312  const MachineOperand *Src1Mods =
2313  getNamedOperand(MI, AMDGPU::OpName::src1_modifiers);
2314  const MachineOperand *Src2 = getNamedOperand(MI, AMDGPU::OpName::src2);
2315  const MachineOperand *Clamp = getNamedOperand(MI, AMDGPU::OpName::clamp);
2316  const MachineOperand *Omod = getNamedOperand(MI, AMDGPU::OpName::omod);
2317 
2318  if (!IsFMA && !Src0Mods && !Src1Mods && !Clamp && !Omod &&
2319  // If we have an SGPR input, we will violate the constant bus restriction.
2320  (!Src0->isReg() || !RI.isSGPRReg(MBB->getParent()->getRegInfo(), Src0->getReg()))) {
2321  if (auto Imm = getFoldableImm(Src2)) {
2322  return BuildMI(*MBB, MI, MI.getDebugLoc(),
2323  get(IsF16 ? AMDGPU::V_MADAK_F16 : AMDGPU::V_MADAK_F32))
2324  .add(*Dst)
2325  .add(*Src0)
2326  .add(*Src1)
2327  .addImm(Imm);
2328  }
2329  if (auto Imm = getFoldableImm(Src1)) {
2330  return BuildMI(*MBB, MI, MI.getDebugLoc(),
2331  get(IsF16 ? AMDGPU::V_MADMK_F16 : AMDGPU::V_MADMK_F32))
2332  .add(*Dst)
2333  .add(*Src0)
2334  .addImm(Imm)
2335  .add(*Src2);
2336  }
2337  if (auto Imm = getFoldableImm(Src0)) {
2338  if (isOperandLegal(MI, AMDGPU::getNamedOperandIdx(AMDGPU::V_MADMK_F32,
2339  AMDGPU::OpName::src0), Src1))
2340  return BuildMI(*MBB, MI, MI.getDebugLoc(),
2341  get(IsF16 ? AMDGPU::V_MADMK_F16 : AMDGPU::V_MADMK_F32))
2342  .add(*Dst)
2343  .add(*Src1)
2344  .addImm(Imm)
2345  .add(*Src2);
2346  }
2347  }
2348 
2349  assert((!IsFMA || !IsF16) && "fmac only expected with f32");
2350  unsigned NewOpc = IsFMA ? AMDGPU::V_FMA_F32 :
2351  (IsF16 ? AMDGPU::V_MAD_F16 : AMDGPU::V_MAD_F32);
2352  return BuildMI(*MBB, MI, MI.getDebugLoc(), get(NewOpc))
2353  .add(*Dst)
2354  .addImm(Src0Mods ? Src0Mods->getImm() : 0)
2355  .add(*Src0)
2356  .addImm(Src1Mods ? Src1Mods->getImm() : 0)
2357  .add(*Src1)
2358  .addImm(0) // Src mods
2359  .add(*Src2)
2360  .addImm(Clamp ? Clamp->getImm() : 0)
2361  .addImm(Omod ? Omod->getImm() : 0);
2362 }
2363 
2364 // It's not generally safe to move VALU instructions across these since it will
2365 // start using the register as a base index rather than directly.
2366 // XXX - Why isn't hasSideEffects sufficient for these?
2368  switch (MI.getOpcode()) {
2369  case AMDGPU::S_SET_GPR_IDX_ON:
2370  case AMDGPU::S_SET_GPR_IDX_MODE:
2371  case AMDGPU::S_SET_GPR_IDX_OFF:
2372  return true;
2373  default:
2374  return false;
2375  }
2376 }
2377 
2379  const MachineBasicBlock *MBB,
2380  const MachineFunction &MF) const {
2381  // XXX - Do we want the SP check in the base implementation?
2382 
2383  // Target-independent instructions do not have an implicit-use of EXEC, even
2384  // when they operate on VGPRs. Treating EXEC modifications as scheduling
2385  // boundaries prevents incorrect movements of such instructions.
2386  return TargetInstrInfo::isSchedulingBoundary(MI, MBB, MF) ||
2387  MI.modifiesRegister(AMDGPU::EXEC, &RI) ||
2388  MI.getOpcode() == AMDGPU::S_SETREG_IMM32_B32 ||
2389  MI.getOpcode() == AMDGPU::S_SETREG_B32 ||
2391 }
2392 
2394  unsigned Opcode = MI.getOpcode();
2395 
2396  if (MI.mayStore() && isSMRD(MI))
2397  return true; // scalar store or atomic
2398 
2399  // These instructions cause shader I/O that may cause hardware lockups
2400  // when executed with an empty EXEC mask.
2401  //
2402  // Note: exp with VM = DONE = 0 is automatically skipped by hardware when
2403  // EXEC = 0, but checking for that case here seems not worth it
2404  // given the typical code patterns.
2405  if (Opcode == AMDGPU::S_SENDMSG || Opcode == AMDGPU::S_SENDMSGHALT ||
2406  Opcode == AMDGPU::EXP || Opcode == AMDGPU::EXP_DONE)
2407  return true;
2408 
2409  if (MI.isInlineAsm())
2410  return true; // conservative assumption
2411 
2412  // These are like SALU instructions in terms of effects, so it's questionable
2413  // whether we should return true for those.
2414  //
2415  // However, executing them with EXEC = 0 causes them to operate on undefined
2416  // data, which we avoid by returning true here.
2417  if (Opcode == AMDGPU::V_READFIRSTLANE_B32 || Opcode == AMDGPU::V_READLANE_B32)
2418  return true;
2419 
2420  return false;
2421 }
2422 
2423 bool SIInstrInfo::isInlineConstant(const APInt &Imm) const {
2424  switch (Imm.getBitWidth()) {
2425  case 32:
2427  ST.hasInv2PiInlineImm());
2428  case 64:
2430  ST.hasInv2PiInlineImm());
2431  case 16:
2432  return ST.has16BitInsts() &&
2434  ST.hasInv2PiInlineImm());
2435  default:
2436  llvm_unreachable("invalid bitwidth");
2437  }
2438 }
2439 
2441  uint8_t OperandType) const {
2442  if (!MO.isImm() ||
2443  OperandType < AMDGPU::OPERAND_SRC_FIRST ||
2444  OperandType > AMDGPU::OPERAND_SRC_LAST)
2445  return false;
2446 
2447  // MachineOperand provides no way to tell the true operand size, since it only
2448  // records a 64-bit value. We need to know the size to determine if a 32-bit
2449  // floating point immediate bit pattern is legal for an integer immediate. It
2450  // would be for any 32-bit integer operand, but would not be for a 64-bit one.
2451 
2452  int64_t Imm = MO.getImm();
2453  switch (OperandType) {
2458  int32_t Trunc = static_cast<int32_t>(Imm);
2460  }
2466  ST.hasInv2PiInlineImm());
2471  if (isInt<16>(Imm) || isUInt<16>(Imm)) {
2472  // A few special case instructions have 16-bit operands on subtargets
2473  // where 16-bit instructions are not legal.
2474  // TODO: Do the 32-bit immediates work? We shouldn't really need to handle
2475  // constants in these cases
2476  int16_t Trunc = static_cast<int16_t>(Imm);
2477  return ST.has16BitInsts() &&
2479  }
2480 
2481  return false;
2482  }
2485  if (isUInt<16>(Imm)) {
2486  int16_t Trunc = static_cast<int16_t>(Imm);
2487  return ST.has16BitInsts() &&
2489  }
2490  if (!(Imm & 0xffff)) {
2491  return ST.has16BitInsts() &&
2493  }
2494  uint32_t Trunc = static_cast<uint32_t>(Imm);
2496  }
2497  default:
2498  llvm_unreachable("invalid bitwidth");
2499  }
2500 }
2501 
2503  const MCOperandInfo &OpInfo) const {
2504  switch (MO.getType()) {
2506  return false;
2508  return !isInlineConstant(MO, OpInfo);
2514  return true;
2515  default:
2516  llvm_unreachable("unexpected operand type");
2517  }
2518 }
2519 
2520 static bool compareMachineOp(const MachineOperand &Op0,
2521  const MachineOperand &Op1) {
2522  if (Op0.getType() != Op1.getType())
2523  return false;
2524 
2525  switch (Op0.getType()) {
2527  return Op0.getReg() == Op1.getReg();
2529  return Op0.getImm() == Op1.getImm();
2530  default:
2531  llvm_unreachable("Didn't expect to be comparing these operand types");
2532  }
2533 }
2534 
2536  const MachineOperand &MO) const {
2537  const MCOperandInfo &OpInfo = get(MI.getOpcode()).OpInfo[OpNo];
2538 
2539  assert(MO.isImm() || MO.isTargetIndex() || MO.isFI());
2540 
2541  if (OpInfo.OperandType == MCOI::OPERAND_IMMEDIATE)
2542  return true;
2543 
2544  if (OpInfo.RegClass < 0)
2545  return false;
2546 
2547  if (MO.isImm() && isInlineConstant(MO, OpInfo))
2548  return RI.opCanUseInlineConstant(OpInfo.OperandType);
2549 
2550  return RI.opCanUseLiteralConstant(OpInfo.OperandType);
2551 }
2552 
2553 bool SIInstrInfo::hasVALU32BitEncoding(unsigned Opcode) const {
2554  int Op32 = AMDGPU::getVOPe32(Opcode);
2555  if (Op32 == -1)
2556  return false;
2557 
2558  return pseudoToMCOpcode(Op32) != -1;
2559 }
2560 
2561 bool SIInstrInfo::hasModifiers(unsigned Opcode) const {
2562  // The src0_modifier operand is present on all instructions
2563  // that have modifiers.
2564 
2565  return AMDGPU::getNamedOperandIdx(Opcode,
2566  AMDGPU::OpName::src0_modifiers) != -1;
2567 }
2568 
2570  unsigned OpName) const {
2571  const MachineOperand *Mods = getNamedOperand(MI, OpName);
2572  return Mods && Mods->getImm();
2573 }
2574 
2576  return hasModifiersSet(MI, AMDGPU::OpName::src0_modifiers) ||
2577  hasModifiersSet(MI, AMDGPU::OpName::src1_modifiers) ||
2578  hasModifiersSet(MI, AMDGPU::OpName::src2_modifiers) ||
2579  hasModifiersSet(MI, AMDGPU::OpName::clamp) ||
2580  hasModifiersSet(MI, AMDGPU::OpName::omod);
2581 }
2582 
2584  const MachineRegisterInfo &MRI) const {
2585  const MachineOperand *Src2 = getNamedOperand(MI, AMDGPU::OpName::src2);
2586  // Can't shrink instruction with three operands.
2587  // FIXME: v_cndmask_b32 has 3 operands and is shrinkable, but we need to add
2588  // a special case for it. It can only be shrunk if the third operand
2589  // is vcc. We should handle this the same way we handle vopc, by addding
2590  // a register allocation hint pre-regalloc and then do the shrinking
2591  // post-regalloc.
2592  if (Src2) {
2593  switch (MI.getOpcode()) {
2594  default: return false;
2595 
2596  case AMDGPU::V_ADDC_U32_e64:
2597  case AMDGPU::V_SUBB_U32_e64:
2598  case AMDGPU::V_SUBBREV_U32_e64: {
2599  const MachineOperand *Src1
2600  = getNamedOperand(MI, AMDGPU::OpName::src1);
2601  if (!Src1->isReg() || !RI.isVGPR(MRI, Src1->getReg()))
2602  return false;
2603  // Additional verification is needed for sdst/src2.
2604  return true;
2605  }
2606  case AMDGPU::V_MAC_F32_e64:
2607  case AMDGPU::V_MAC_F16_e64:
2608  case AMDGPU::V_FMAC_F32_e64:
2609  if (!Src2->isReg() || !RI.isVGPR(MRI, Src2->getReg()) ||
2610  hasModifiersSet(MI, AMDGPU::OpName::src2_modifiers))
2611  return false;
2612  break;
2613 
2614  case AMDGPU::V_CNDMASK_B32_e64:
2615  break;
2616  }
2617  }
2618 
2619  const MachineOperand *Src1 = getNamedOperand(MI, AMDGPU::OpName::src1);
2620  if (Src1 && (!Src1->isReg() || !RI.isVGPR(MRI, Src1->getReg()) ||
2621  hasModifiersSet(MI, AMDGPU::OpName::src1_modifiers)))
2622  return false;
2623 
2624  // We don't need to check src0, all input types are legal, so just make sure
2625  // src0 isn't using any modifiers.
2626  if (hasModifiersSet(MI, AMDGPU::OpName::src0_modifiers))
2627  return false;
2628 
2629  // Can it be shrunk to a valid 32 bit opcode?
2630  if (!hasVALU32BitEncoding(MI.getOpcode()))
2631  return false;
2632 
2633  // Check output modifiers
2634  return !hasModifiersSet(MI, AMDGPU::OpName::omod) &&
2635  !hasModifiersSet(MI, AMDGPU::OpName::clamp);
2636 }
2637 
2638 // Set VCC operand with all flags from \p Orig, except for setting it as
2639 // implicit.
2641  const MachineOperand &Orig) {
2642 
2643  for (MachineOperand &Use : MI.implicit_operands()) {
2644  if (Use.isUse() && Use.getReg() == AMDGPU::VCC) {
2645  Use.setIsUndef(Orig.isUndef());
2646  Use.setIsKill(Orig.isKill());
2647  return;
2648  }
2649  }
2650 }
2651 
2653  unsigned Op32) const {
2654  MachineBasicBlock *MBB = MI.getParent();;
2655  MachineInstrBuilder Inst32 =
2656  BuildMI(*MBB, MI, MI.getDebugLoc(), get(Op32));
2657 
2658  // Add the dst operand if the 32-bit encoding also has an explicit $vdst.
2659  // For VOPC instructions, this is replaced by an implicit def of vcc.
2660  int Op32DstIdx = AMDGPU::getNamedOperandIdx(Op32, AMDGPU::OpName::vdst);
2661  if (Op32DstIdx != -1) {
2662  // dst
2663  Inst32.add(MI.getOperand(0));
2664  } else {
2665  assert(MI.getOperand(0).getReg() == AMDGPU::VCC &&
2666  "Unexpected case");
2667  }
2668 
2669  Inst32.add(*getNamedOperand(MI, AMDGPU::OpName::src0));
2670 
2671  const MachineOperand *Src1 = getNamedOperand(MI, AMDGPU::OpName::src1);
2672  if (Src1)
2673  Inst32.add(*Src1);
2674 
2675  const MachineOperand *Src2 = getNamedOperand(MI, AMDGPU::OpName::src2);
2676 
2677  if (Src2) {
2678  int Op32Src2Idx = AMDGPU::getNamedOperandIdx(Op32, AMDGPU::OpName::src2);
2679  if (Op32Src2Idx != -1) {
2680  Inst32.add(*Src2);
2681  } else {
2682  // In the case of V_CNDMASK_B32_e32, the explicit operand src2 is
2683  // replaced with an implicit read of vcc. This was already added
2684  // during the initial BuildMI, so find it to preserve the flags.
2685  copyFlagsToImplicitVCC(*Inst32, *Src2);
2686  }
2687  }
2688 
2689  return Inst32;
2690 }
2691 
2693  const MachineOperand &MO,
2694  const MCOperandInfo &OpInfo) const {
2695  // Literal constants use the constant bus.
2696  //if (isLiteralConstantLike(MO, OpInfo))
2697  // return true;
2698  if (MO.isImm())
2699  return !isInlineConstant(MO, OpInfo);
2700 
2701  if (!MO.isReg())
2702  return true; // Misc other operands like FrameIndex
2703 
2704  if (!MO.isUse())
2705  return false;
2706 
2708  return RI.isSGPRClass(MRI.getRegClass(MO.getReg()));
2709 
2710  // FLAT_SCR is just an SGPR pair.
2711  if (!MO.isImplicit() && (MO.getReg() == AMDGPU::FLAT_SCR))
2712  return true;
2713 
2714  // EXEC register uses the constant bus.
2715  if (!MO.isImplicit() && MO.getReg() == AMDGPU::EXEC)
2716  return true;
2717 
2718  // SGPRs use the constant bus
2719  return (MO.getReg() == AMDGPU::VCC || MO.getReg() == AMDGPU::M0 ||
2720  (!MO.isImplicit() &&
2721  (AMDGPU::SGPR_32RegClass.contains(MO.getReg()) ||
2722  AMDGPU::SGPR_64RegClass.contains(MO.getReg()))));
2723 }
2724 
2725 static unsigned findImplicitSGPRRead(const MachineInstr &MI) {
2726  for (const MachineOperand &MO : MI.implicit_operands()) {
2727  // We only care about reads.
2728  if (MO.isDef())
2729  continue;
2730 
2731  switch (MO.getReg()) {
2732  case AMDGPU::VCC:
2733  case AMDGPU::M0:
2734  case AMDGPU::FLAT_SCR:
2735  return MO.getReg();
2736 
2737  default:
2738  break;
2739  }
2740  }
2741 
2742  return AMDGPU::NoRegister;
2743 }
2744 
2745 static bool shouldReadExec(const MachineInstr &MI) {
2746  if (SIInstrInfo::isVALU(MI)) {
2747  switch (MI.getOpcode()) {
2748  case AMDGPU::V_READLANE_B32:
2749  case AMDGPU::V_READLANE_B32_si:
2750  case AMDGPU::V_READLANE_B32_vi:
2751  case AMDGPU::V_WRITELANE_B32:
2752  case AMDGPU::V_WRITELANE_B32_si:
2753  case AMDGPU::V_WRITELANE_B32_vi:
2754  return false;
2755  }
2756 
2757  return true;
2758  }
2759 
2760  if (SIInstrInfo::isGenericOpcode(MI.getOpcode()) ||
2761  SIInstrInfo::isSALU(MI) ||
2762  SIInstrInfo::isSMRD(MI))
2763  return false;
2764 
2765  return true;
2766 }
2767 
2768 static bool isSubRegOf(const SIRegisterInfo &TRI,
2769  const MachineOperand &SuperVec,
2770  const MachineOperand &SubReg) {
2772  return TRI.isSubRegister(SuperVec.getReg(), SubReg.getReg());
2773 
2774  return SubReg.getSubReg() != AMDGPU::NoSubRegister &&
2775  SubReg.getReg() == SuperVec.getReg();
2776 }
2777 
2779  StringRef &ErrInfo) const {
2780  uint16_t Opcode = MI.getOpcode();
2781  if (SIInstrInfo::isGenericOpcode(MI.getOpcode()))
2782  return true;
2783 
2784  const MachineFunction *MF = MI.getParent()->getParent();
2785  const MachineRegisterInfo &MRI = MF->getRegInfo();
2786 
2787  int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0);
2788  int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src1);
2789  int Src2Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src2);
2790 
2791  // Make sure the number of operands is correct.
2792  const MCInstrDesc &Desc = get(Opcode);
2793  if (!Desc.isVariadic() &&
2794  Desc.getNumOperands() != MI.getNumExplicitOperands()) {
2795  ErrInfo = "Instruction has wrong number of operands.";
2796  return false;
2797  }
2798 
2799  if (MI.isInlineAsm()) {
2800  // Verify register classes for inlineasm constraints.
2801  for (unsigned I = InlineAsm::MIOp_FirstOperand, E = MI.getNumOperands();
2802  I != E; ++I) {
2803  const TargetRegisterClass *RC = MI.getRegClassConstraint(I, this, &RI);
2804  if (!RC)
2805  continue;
2806 
2807  const MachineOperand &Op = MI.getOperand(I);
2808  if (!Op.isReg())
2809  continue;
2810 
2811  unsigned Reg = Op.getReg();
2812  if (!TargetRegisterInfo::isVirtualRegister(Reg) && !RC->contains(Reg)) {
2813  ErrInfo = "inlineasm operand has incorrect register class.";
2814  return false;
2815  }
2816  }
2817 
2818  return true;
2819  }
2820 
2821  // Make sure the register classes are correct.
2822  for (int i = 0, e = Desc.getNumOperands(); i != e; ++i) {
2823  if (MI.getOperand(i).isFPImm()) {
2824  ErrInfo = "FPImm Machine Operands are not supported. ISel should bitcast "
2825  "all fp values to integers.";
2826  return false;
2827  }
2828 
2829  int RegClass = Desc.OpInfo[i].RegClass;
2830 
2831  switch (Desc.OpInfo[i].OperandType) {
2833  if (MI.getOperand(i).isImm()) {
2834  ErrInfo = "Illegal immediate value for operand.";
2835  return false;
2836  }
2837  break;
2840  break;
2847  const MachineOperand &MO = MI.getOperand(i);
2848  if (!MO.isReg() && (!MO.isImm() || !isInlineConstant(MI, i))) {
2849  ErrInfo = "Illegal immediate value for operand.";
2850  return false;
2851  }
2852  break;
2853  }
2856  // Check if this operand is an immediate.
2857  // FrameIndex operands will be replaced by immediates, so they are
2858  // allowed.
2859  if (!MI.getOperand(i).isImm() && !MI.getOperand(i).isFI()) {
2860  ErrInfo = "Expected immediate, but got non-immediate";
2861  return false;
2862  }
2864  default:
2865  continue;
2866  }
2867 
2868  if (!MI.getOperand(i).isReg())
2869  continue;
2870 
2871  if (RegClass != -1) {
2872  unsigned Reg = MI.getOperand(i).getReg();
2873  if (Reg == AMDGPU::NoRegister ||
2875  continue;
2876 
2877  const TargetRegisterClass *RC = RI.getRegClass(RegClass);
2878  if (!RC->contains(Reg)) {
2879  ErrInfo = "Operand has incorrect register class.";
2880  return false;
2881  }
2882  }
2883  }
2884 
2885  // Verify SDWA
2886  if (isSDWA(MI)) {
2887  if (!ST.hasSDWA()) {
2888  ErrInfo = "SDWA is not supported on this target";
2889  return false;
2890  }
2891 
2892  int DstIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdst);
2893 
2894  const int OpIndicies[] = { DstIdx, Src0Idx, Src1Idx, Src2Idx };
2895 
2896  for (int OpIdx: OpIndicies) {
2897  if (OpIdx == -1)
2898  continue;
2899  const MachineOperand &MO = MI.getOperand(OpIdx);
2900 
2901  if (!ST.hasSDWAScalar()) {
2902  // Only VGPRS on VI
2903  if (!MO.isReg() || !RI.hasVGPRs(RI.getRegClassForReg(MRI, MO.getReg()))) {
2904  ErrInfo = "Only VGPRs allowed as operands in SDWA instructions on VI";
2905  return false;
2906  }
2907  } else {
2908  // No immediates on GFX9
2909  if (!MO.isReg()) {
2910  ErrInfo = "Only reg allowed as operands in SDWA instructions on GFX9";
2911  return false;
2912  }
2913  }
2914  }
2915 
2916  if (!ST.hasSDWAOmod()) {
2917  // No omod allowed on VI
2918  const MachineOperand *OMod = getNamedOperand(MI, AMDGPU::OpName::omod);
2919  if (OMod != nullptr &&
2920  (!OMod->isImm() || OMod->getImm() != 0)) {
2921  ErrInfo = "OMod not allowed in SDWA instructions on VI";
2922  return false;
2923  }
2924  }
2925 
2926  uint16_t BasicOpcode = AMDGPU::getBasicFromSDWAOp(Opcode);
2927  if (isVOPC(BasicOpcode)) {
2928  if (!ST.hasSDWASdst() && DstIdx != -1) {
2929  // Only vcc allowed as dst on VI for VOPC
2930  const MachineOperand &Dst = MI.getOperand(DstIdx);
2931  if (!Dst.isReg() || Dst.getReg() != AMDGPU::VCC) {
2932  ErrInfo = "Only VCC allowed as dst in SDWA instructions on VI";
2933  return false;
2934  }
2935  } else if (!ST.hasSDWAOutModsVOPC()) {
2936  // No clamp allowed on GFX9 for VOPC
2937  const MachineOperand *Clamp = getNamedOperand(MI, AMDGPU::OpName::clamp);
2938  if (Clamp && (!Clamp->isImm() || Clamp->getImm() != 0)) {
2939  ErrInfo = "Clamp not allowed in VOPC SDWA instructions on VI";
2940  return false;
2941  }
2942 
2943  // No omod allowed on GFX9 for VOPC
2944  const MachineOperand *OMod = getNamedOperand(MI, AMDGPU::OpName::omod);
2945  if (OMod && (!OMod->isImm() || OMod->getImm() != 0)) {
2946  ErrInfo = "OMod not allowed in VOPC SDWA instructions on VI";
2947  return false;
2948  }
2949  }
2950  }
2951 
2952  const MachineOperand *DstUnused = getNamedOperand(MI, AMDGPU::OpName::dst_unused);
2953  if (DstUnused && DstUnused->isImm() &&
2954  DstUnused->getImm() == AMDGPU::SDWA::UNUSED_PRESERVE) {
2955  const MachineOperand &Dst = MI.getOperand(DstIdx);
2956  if (!Dst.isReg() || !Dst.isTied()) {
2957  ErrInfo = "Dst register should have tied register";
2958  return false;
2959  }
2960 
2961  const MachineOperand &TiedMO =
2962  MI.getOperand(MI.findTiedOperandIdx(DstIdx));
2963  if (!TiedMO.isReg() || !TiedMO.isImplicit() || !TiedMO.isUse()) {
2964  ErrInfo =
2965  "Dst register should be tied to implicit use of preserved register";
2966  return false;
2967  } else if (TargetRegisterInfo::isPhysicalRegister(TiedMO.getReg()) &&
2968  Dst.getReg() != TiedMO.getReg()) {
2969  ErrInfo = "Dst register should use same physical register as preserved";
2970  return false;
2971  }
2972  }
2973  }
2974 
2975  // Verify VOP*. Ignore multiple sgpr operands on writelane.
2976  if (Desc.getOpcode() != AMDGPU::V_WRITELANE_B32
2977  && (isVOP1(MI) || isVOP2(MI) || isVOP3(MI) || isVOPC(MI) || isSDWA(MI))) {
2978  // Only look at the true operands. Only a real operand can use the constant
2979  // bus, and we don't want to check pseudo-operands like the source modifier
2980  // flags.
2981  const int OpIndices[] = { Src0Idx, Src1Idx, Src2Idx };
2982 
2983  unsigned ConstantBusCount = 0;
2984  unsigned LiteralCount = 0;
2985 
2986  if (AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::imm) != -1)
2987  ++ConstantBusCount;
2988 
2989  unsigned SGPRUsed = findImplicitSGPRRead(MI);
2990  if (SGPRUsed != AMDGPU::NoRegister)
2991  ++ConstantBusCount;
2992 
2993  for (int OpIdx : OpIndices) {
2994  if (OpIdx == -1)
2995  break;
2996  const MachineOperand &MO = MI.getOperand(OpIdx);
2997  if (usesConstantBus(MRI, MO, MI.getDesc().OpInfo[OpIdx])) {
2998  if (MO.isReg()) {
2999  if (MO.getReg() != SGPRUsed)
3000  ++ConstantBusCount;
3001  SGPRUsed = MO.getReg();
3002  } else {
3003  ++ConstantBusCount;
3004  ++LiteralCount;
3005  }
3006  }
3007  }
3008  if (ConstantBusCount > 1) {
3009  ErrInfo = "VOP* instruction uses the constant bus more than once";
3010  return false;
3011  }
3012 
3013  if (isVOP3(MI) && LiteralCount) {
3014  ErrInfo = "VOP3 instruction uses literal";
3015  return false;
3016  }
3017  }
3018 
3019  // Verify misc. restrictions on specific instructions.
3020  if (Desc.getOpcode() == AMDGPU::V_DIV_SCALE_F32 ||
3021  Desc.getOpcode() == AMDGPU::V_DIV_SCALE_F64) {
3022  const MachineOperand &Src0 = MI.getOperand(Src0Idx);
3023  const MachineOperand &Src1 = MI.getOperand(Src1Idx);
3024  const MachineOperand &Src2 = MI.getOperand(Src2Idx);
3025  if (Src0.isReg() && Src1.isReg() && Src2.isReg()) {
3026  if (!compareMachineOp(Src0, Src1) &&
3027  !compareMachineOp(Src0, Src2)) {
3028  ErrInfo = "v_div_scale_{f32|f64} require src0 = src1 or src2";
3029  return false;
3030  }
3031  }
3032  }
3033 
3034  if (isSOPK(MI)) {
3035  int64_t Imm = getNamedOperand(MI, AMDGPU::OpName::simm16)->getImm();
3036  if (sopkIsZext(MI)) {
3037  if (!isUInt<16>(Imm)) {
3038  ErrInfo = "invalid immediate for SOPK instruction";
3039  return false;
3040  }
3041  } else {
3042  if (!isInt<16>(Imm)) {
3043  ErrInfo = "invalid immediate for SOPK instruction";
3044  return false;
3045  }
3046  }
3047  }
3048 
3049  if (Desc.getOpcode() == AMDGPU::V_MOVRELS_B32_e32 ||
3050  Desc.getOpcode() == AMDGPU::V_MOVRELS_B32_e64 ||
3051  Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e32 ||
3052  Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e64) {
3053  const bool IsDst = Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e32 ||
3054  Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e64;
3055 
3056  const unsigned StaticNumOps = Desc.getNumOperands() +
3057  Desc.getNumImplicitUses();
3058  const unsigned NumImplicitOps = IsDst ? 2 : 1;
3059 
3060  // Allow additional implicit operands. This allows a fixup done by the post
3061  // RA scheduler where the main implicit operand is killed and implicit-defs
3062  // are added for sub-registers that remain live after this instruction.
3063  if (MI.getNumOperands() < StaticNumOps + NumImplicitOps) {
3064  ErrInfo = "missing implicit register operands";
3065  return false;
3066  }
3067 
3068  const MachineOperand *Dst = getNamedOperand(MI, AMDGPU::OpName::vdst);
3069  if (IsDst) {
3070  if (!Dst->isUse()) {
3071  ErrInfo = "v_movreld_b32 vdst should be a use operand";
3072  return false;
3073  }
3074 
3075  unsigned UseOpIdx;
3076  if (!MI.isRegTiedToUseOperand(StaticNumOps, &UseOpIdx) ||
3077  UseOpIdx != StaticNumOps + 1) {
3078  ErrInfo = "movrel implicit operands should be tied";
3079  return false;
3080  }
3081  }
3082 
3083  const MachineOperand &Src0 = MI.getOperand(Src0Idx);
3084  const MachineOperand &ImpUse
3085  = MI.getOperand(StaticNumOps + NumImplicitOps - 1);
3086  if (!ImpUse.isReg() || !ImpUse.isUse() ||
3087  !isSubRegOf(RI, ImpUse, IsDst ? *Dst : Src0)) {
3088  ErrInfo = "src0 should be subreg of implicit vector use";
3089  return false;
3090  }
3091  }
3092 
3093  // Make sure we aren't losing exec uses in the td files. This mostly requires
3094  // being careful when using let Uses to try to add other use registers.
3095  if (shouldReadExec(MI)) {
3096  if (!MI.hasRegisterImplicitUseOperand(AMDGPU::EXEC)) {
3097  ErrInfo = "VALU instruction does not implicitly read exec mask";
3098  return false;
3099  }
3100  }
3101 
3102  if (isSMRD(MI)) {
3103  if (MI.mayStore()) {
3104  // The register offset form of scalar stores may only use m0 as the
3105  // soffset register.
3106  const MachineOperand *Soff = getNamedOperand(MI, AMDGPU::OpName::soff);
3107  if (Soff && Soff->getReg() != AMDGPU::M0) {
3108  ErrInfo = "scalar stores must use m0 as offset register";
3109  return false;
3110  }
3111  }
3112  }
3113 
3114  if (isFLAT(MI) && !MF->getSubtarget<GCNSubtarget>().hasFlatInstOffsets()) {
3115  const MachineOperand *Offset = getNamedOperand(MI, AMDGPU::OpName::offset);
3116  if (Offset->getImm() != 0) {
3117  ErrInfo = "subtarget does not support offsets in flat instructions";
3118  return false;
3119  }
3120  }
3121 
3122  const MachineOperand *DppCt = getNamedOperand(MI, AMDGPU::OpName::dpp_ctrl);
3123  if (DppCt) {
3124  using namespace AMDGPU::DPP;
3125 
3126  unsigned DC = DppCt->getImm();
3127  if (DC == DppCtrl::DPP_UNUSED1 || DC == DppCtrl::DPP_UNUSED2 ||
3128  DC == DppCtrl::DPP_UNUSED3 || DC > DppCtrl::DPP_LAST ||
3133  ErrInfo = "Invalid dpp_ctrl value";
3134  return false;
3135  }
3136  }
3137 
3138  return true;
3139 }
3140 
3141 unsigned SIInstrInfo::getVALUOp(const MachineInstr &MI) const {
3142  switch (MI.getOpcode()) {
3143  default: return AMDGPU::INSTRUCTION_LIST_END;
3144  case AMDGPU::REG_SEQUENCE: return AMDGPU::REG_SEQUENCE;
3145  case AMDGPU::COPY: return AMDGPU::COPY;
3146  case AMDGPU::PHI: return AMDGPU::PHI;
3147  case AMDGPU::INSERT_SUBREG: return AMDGPU::INSERT_SUBREG;
3148  case AMDGPU::WQM: return AMDGPU::WQM;
3149  case AMDGPU::WWM: return AMDGPU::WWM;
3150  case AMDGPU::S_MOV_B32:
3151  return MI.getOperand(1).isReg() ?
3152  AMDGPU::COPY : AMDGPU::V_MOV_B32_e32;
3153  case AMDGPU::S_ADD_I32:
3154  return ST.hasAddNoCarry() ? AMDGPU::V_ADD_U32_e64 : AMDGPU::V_ADD_I32_e32;
3155  case AMDGPU::S_ADDC_U32:
3156  return AMDGPU::V_ADDC_U32_e32;
3157  case AMDGPU::S_SUB_I32:
3158  return ST.hasAddNoCarry() ? AMDGPU::V_SUB_U32_e64 : AMDGPU::V_SUB_I32_e32;
3159  // FIXME: These are not consistently handled, and selected when the carry is
3160  // used.
3161  case AMDGPU::S_ADD_U32:
3162  return AMDGPU::V_ADD_I32_e32;
3163  case AMDGPU::S_SUB_U32:
3164  return AMDGPU::V_SUB_I32_e32;
3165  case AMDGPU::S_SUBB_U32: return AMDGPU::V_SUBB_U32_e32;
3166  case AMDGPU::S_MUL_I32: return AMDGPU::V_MUL_LO_I32;
3167  case AMDGPU::S_AND_B32: return AMDGPU::V_AND_B32_e64;
3168  case AMDGPU::S_OR_B32: return AMDGPU::V_OR_B32_e64;
3169  case AMDGPU::S_XOR_B32: return AMDGPU::V_XOR_B32_e64;
3170  case AMDGPU::S_XNOR_B32:
3171  return ST.hasDLInsts() ? AMDGPU::V_XNOR_B32_e64 : AMDGPU::INSTRUCTION_LIST_END;
3172  case AMDGPU::S_MIN_I32: return AMDGPU::V_MIN_I32_e64;
3173  case AMDGPU::S_MIN_U32: return AMDGPU::V_MIN_U32_e64;
3174  case AMDGPU::S_MAX_I32: return AMDGPU::V_MAX_I32_e64;
3175  case AMDGPU::S_MAX_U32: return AMDGPU::V_MAX_U32_e64;
3176  case AMDGPU::S_ASHR_I32: return AMDGPU::V_ASHR_I32_e32;
3177  case AMDGPU::S_ASHR_I64: return AMDGPU::V_ASHR_I64;
3178  case AMDGPU::S_LSHL_B32: return AMDGPU::V_LSHL_B32_e32;
3179  case AMDGPU::S_LSHL_B64: return AMDGPU::V_LSHL_B64;
3180  case AMDGPU::S_LSHR_B32: return AMDGPU::V_LSHR_B32_e32;
3181  case AMDGPU::S_LSHR_B64: return AMDGPU::V_LSHR_B64;
3182  case AMDGPU::S_SEXT_I32_I8: return AMDGPU::V_BFE_I32;
3183  case AMDGPU::S_SEXT_I32_I16: return AMDGPU::V_BFE_I32;
3184  case AMDGPU::S_BFE_U32: return AMDGPU::V_BFE_U32;
3185  case AMDGPU::S_BFE_I32: return AMDGPU::V_BFE_I32;
3186  case AMDGPU::S_BFM_B32: return AMDGPU::V_BFM_B32_e64;
3187  case AMDGPU::S_BREV_B32: return AMDGPU::V_BFREV_B32_e32;
3188  case AMDGPU::S_NOT_B32: return AMDGPU::V_NOT_B32_e32;
3189  case AMDGPU::S_NOT_B64: return AMDGPU::V_NOT_B32_e32;
3190  case AMDGPU::S_CMP_EQ_I32: return AMDGPU::V_CMP_EQ_I32_e32;
3191  case AMDGPU::S_CMP_LG_I32: return AMDGPU::V_CMP_NE_I32_e32;
3192  case AMDGPU::S_CMP_GT_I32: return AMDGPU::V_CMP_GT_I32_e32;
3193  case AMDGPU::S_CMP_GE_I32: return AMDGPU::V_CMP_GE_I32_e32;
3194  case AMDGPU::S_CMP_LT_I32: return AMDGPU::V_CMP_LT_I32_e32;
3195  case AMDGPU::S_CMP_LE_I32: return AMDGPU::V_CMP_LE_I32_e32;
3196  case AMDGPU::S_CMP_EQ_U32: return AMDGPU::V_CMP_EQ_U32_e32;
3197  case AMDGPU::S_CMP_LG_U32: return AMDGPU::V_CMP_NE_U32_e32;
3198  case AMDGPU::S_CMP_GT_U32: return AMDGPU::V_CMP_GT_U32_e32;
3199  case AMDGPU::S_CMP_GE_U32: return AMDGPU::V_CMP_GE_U32_e32;
3200  case AMDGPU::S_CMP_LT_U32: return AMDGPU::V_CMP_LT_U32_e32;
3201  case AMDGPU::S_CMP_LE_U32: return AMDGPU::V_CMP_LE_U32_e32;
3202  case AMDGPU::S_CMP_EQ_U64: return AMDGPU::V_CMP_EQ_U64_e32;
3203  case AMDGPU::S_CMP_LG_U64: return AMDGPU::V_CMP_NE_U64_e32;
3204  case AMDGPU::S_BCNT1_I32_B32: return AMDGPU::V_BCNT_U32_B32_e64;
3205  case AMDGPU::S_FF1_I32_B32: return AMDGPU::V_FFBL_B32_e32;
3206  case AMDGPU::S_FLBIT_I32_B32: return AMDGPU::V_FFBH_U32_e32;
3207  case AMDGPU::S_FLBIT_I32: return AMDGPU::V_FFBH_I32_e64;
3208  case AMDGPU::S_CBRANCH_SCC0: return AMDGPU::S_CBRANCH_VCCZ;
3209  case AMDGPU::S_CBRANCH_SCC1: return AMDGPU::S_CBRANCH_VCCNZ;
3210  }
3211 }
3212 
3214  unsigned OpNo) const {
3216  const MCInstrDesc &Desc = get(MI.getOpcode());
3217  if (MI.isVariadic() || OpNo >= Desc.getNumOperands() ||
3218  Desc.OpInfo[OpNo].RegClass == -1) {
3219  unsigned Reg = MI.getOperand(OpNo).getReg();
3220 
3222  return MRI.getRegClass(Reg);
3223  return RI.getPhysRegClass(Reg);
3224  }
3225 
3226  unsigned RCID = Desc.OpInfo[OpNo].RegClass;
3227  return RI.getRegClass(RCID);
3228 }
3229 
3230 bool SIInstrInfo::canReadVGPR(const MachineInstr &MI, unsigned OpNo) const {
3231  switch (MI.getOpcode()) {
3232  case AMDGPU::COPY:
3233  case AMDGPU::REG_SEQUENCE:
3234  case AMDGPU::PHI:
3235  case AMDGPU::INSERT_SUBREG:
3236  return RI.hasVGPRs(getOpRegClass(MI, 0));
3237  default:
3238  return RI.hasVGPRs(getOpRegClass(MI, OpNo));
3239  }
3240 }
3241 
3242 void SIInstrInfo::legalizeOpWithMove(MachineInstr &MI, unsigned OpIdx) const {
3244  MachineBasicBlock *MBB = MI.getParent();
3245  MachineOperand &MO = MI.getOperand(OpIdx);
3247  unsigned RCID = get(MI.getOpcode()).OpInfo[OpIdx].RegClass;
3248  const TargetRegisterClass *RC = RI.getRegClass(RCID);
3249  unsigned Opcode = AMDGPU::V_MOV_B32_e32;
3250  if (MO.isReg())
3251  Opcode = AMDGPU::COPY;
3252  else if (RI.isSGPRClass(RC))
3253  Opcode = AMDGPU::S_MOV_B32;
3254 
3255  const TargetRegisterClass *VRC = RI.getEquivalentVGPRClass(RC);
3256  if (RI.getCommonSubClass(&AMDGPU::VReg_64RegClass, VRC))
3257  VRC = &AMDGPU::VReg_64RegClass;
3258  else
3259  VRC = &AMDGPU::VGPR_32RegClass;
3260 
3261  unsigned Reg = MRI.createVirtualRegister(VRC);
3262  DebugLoc DL = MBB->findDebugLoc(I);
3263  BuildMI(*MI.getParent(), I, DL, get(Opcode), Reg).add(MO);
3264  MO.ChangeToRegister(Reg, false);
3265 }
3266 
3269  MachineOperand &SuperReg,
3270  const TargetRegisterClass *SuperRC,
3271  unsigned SubIdx,
3272  const TargetRegisterClass *SubRC)
3273  const {
3274  MachineBasicBlock *MBB = MI->getParent();
3275  DebugLoc DL = MI->getDebugLoc();
3276  unsigned SubReg = MRI.createVirtualRegister(SubRC);
3277 
3278  if (SuperReg.getSubReg() == AMDGPU::NoSubRegister) {
3279  BuildMI(*MBB, MI, DL, get(TargetOpcode::COPY), SubReg)
3280  .addReg(SuperReg.getReg(), 0, SubIdx);
3281  return SubReg;
3282  }
3283 
3284  // Just in case the super register is itself a sub-register, copy it to a new
3285  // value so we don't need to worry about merging its subreg index with the
3286  // SubIdx passed to this function. The register coalescer should be able to
3287  // eliminate this extra copy.
3288  unsigned NewSuperReg = MRI.createVirtualRegister(SuperRC);
3289 
3290  BuildMI(*MBB, MI, DL, get(TargetOpcode::COPY), NewSuperReg)
3291  .addReg(SuperReg.getReg(), 0, SuperReg.getSubReg());
3292 
3293  BuildMI(*MBB, MI, DL, get(TargetOpcode::COPY), SubReg)
3294  .addReg(NewSuperReg, 0, SubIdx);
3295 
3296  return SubReg;
3297 }
3298 
3302  MachineOperand &Op,
3303  const TargetRegisterClass *SuperRC,
3304  unsigned SubIdx,
3305  const TargetRegisterClass *SubRC) const {
3306  if (Op.isImm()) {
3307  if (SubIdx == AMDGPU::sub0)
3308  return MachineOperand::CreateImm(static_cast<int32_t>(Op.getImm()));
3309  if (SubIdx == AMDGPU::sub1)
3310  return MachineOperand::CreateImm(static_cast<int32_t>(Op.getImm() >> 32));
3311 
3312  llvm_unreachable("Unhandled register index for immediate");
3313  }
3314 
3315  unsigned SubReg = buildExtractSubReg(MII, MRI, Op, SuperRC,
3316  SubIdx, SubRC);
3317  return MachineOperand::CreateReg(SubReg, false);
3318 }
3319 
3320 // Change the order of operands from (0, 1, 2) to (0, 2, 1)
3321 void SIInstrInfo::swapOperands(MachineInstr &Inst) const {
3322  assert(Inst.getNumExplicitOperands() == 3);
3323  MachineOperand Op1 = Inst.getOperand(1);
3324  Inst.RemoveOperand(1);
3325  Inst.addOperand(Op1);
3326 }
3327 
3329  const MCOperandInfo &OpInfo,
3330  const MachineOperand &MO) const {
3331  if (!MO.isReg())
3332  return false;
3333 
3334  unsigned Reg = MO.getReg();
3335  const TargetRegisterClass *RC =
3337  MRI.getRegClass(Reg) :
3338  RI.getPhysRegClass(Reg);
3339 
3340  const SIRegisterInfo *TRI =
3341  static_cast<const SIRegisterInfo*>(MRI.getTargetRegisterInfo());
3342  RC = TRI->getSubRegClass(RC, MO.getSubReg());
3343 
3344  // In order to be legal, the common sub-class must be equal to the
3345  // class of the current operand. For example:
3346  //
3347  // v_mov_b32 s0 ; Operand defined as vsrc_b32
3348  // ; RI.getCommonSubClass(s0,vsrc_b32) = sgpr ; LEGAL
3349  //
3350  // s_sendmsg 0, s0 ; Operand defined as m0reg
3351  // ; RI.getCommonSubClass(s0,m0reg) = m0reg ; NOT LEGAL
3352 
3353  return RI.getCommonSubClass(RC, RI.getRegClass(OpInfo.RegClass)) == RC;
3354 }
3355 
3357  const MCOperandInfo &OpInfo,
3358  const MachineOperand &MO) const {
3359  if (MO.isReg())
3360  return isLegalRegOperand(MRI, OpInfo, MO);
3361 
3362  // Handle non-register types that are treated like immediates.
3363  assert(MO.isImm() || MO.isTargetIndex() || MO.isFI());
3364  return true;
3365 }
3366 
3367 bool SIInstrInfo::isOperandLegal(const MachineInstr &MI, unsigned OpIdx,
3368  const MachineOperand *MO) const {
3370  const MCInstrDesc &InstDesc = MI.getDesc();
3371  const MCOperandInfo &OpInfo = InstDesc.OpInfo[OpIdx];
3372  const TargetRegisterClass *DefinedRC =
3373  OpInfo.RegClass != -1 ? RI.getRegClass(OpInfo.RegClass) : nullptr;
3374  if (!MO)
3375  MO = &MI.getOperand(OpIdx);
3376 
3377  if (isVALU(MI) && usesConstantBus(MRI, *MO, OpInfo)) {
3378 
3379  RegSubRegPair SGPRUsed;
3380  if (MO->isReg())
3381  SGPRUsed = RegSubRegPair(MO->getReg(), MO->getSubReg());
3382 
3383  for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
3384  if (i == OpIdx)
3385  continue;
3386  const MachineOperand &Op = MI.getOperand(i);
3387  if (Op.isReg()) {
3388  if ((Op.getReg() != SGPRUsed.Reg || Op.getSubReg() != SGPRUsed.SubReg) &&
3389  usesConstantBus(MRI, Op, InstDesc.OpInfo[i])) {
3390  return false;
3391  }
3392  } else if (InstDesc.OpInfo[i].OperandType == AMDGPU::OPERAND_KIMM32) {
3393  return false;
3394  }
3395  }
3396  }
3397 
3398  if (MO->isReg()) {
3399  assert(DefinedRC);
3400  return isLegalRegOperand(MRI, OpInfo, *MO);
3401  }
3402 
3403  // Handle non-register types that are treated like immediates.
3404  assert(MO->isImm() || MO->isTargetIndex() || MO->isFI());
3405 
3406  if (!DefinedRC) {
3407  // This operand expects an immediate.
3408  return true;
3409  }
3410 
3411  return isImmOperandLegal(MI, OpIdx, *MO);
3412 }
3413 
3415  MachineInstr &MI) const {
3416  unsigned Opc = MI.getOpcode();
3417  const MCInstrDesc &InstrDesc = get(Opc);
3418 
3419  int Src1Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1);
3420  MachineOperand &Src1 = MI.getOperand(Src1Idx);
3421 
3422  // If there is an implicit SGPR use such as VCC use for v_addc_u32/v_subb_u32
3423  // we need to only have one constant bus use.
3424  //
3425  // Note we do not need to worry about literal constants here. They are
3426  // disabled for the operand type for instructions because they will always
3427  // violate the one constant bus use rule.
3428  bool HasImplicitSGPR = findImplicitSGPRRead(MI) != AMDGPU::NoRegister;
3429  if (HasImplicitSGPR) {
3430  int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0);
3431  MachineOperand &Src0 = MI.getOperand(Src0Idx);
3432 
3433  if (Src0.isReg() && RI.isSGPRReg(MRI, Src0.getReg()))
3434  legalizeOpWithMove(MI, Src0Idx);
3435  }
3436 
3437  // Special case: V_WRITELANE_B32 accepts only immediate or SGPR operands for
3438  // both the value to write (src0) and lane select (src1). Fix up non-SGPR
3439  // src0/src1 with V_READFIRSTLANE.
3440  if (Opc == AMDGPU::V_WRITELANE_B32) {
3441  int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0);
3442  MachineOperand &Src0 = MI.getOperand(Src0Idx);
3443  const DebugLoc &DL = MI.getDebugLoc();
3444  if (Src0.isReg() && RI.isVGPR(MRI, Src0.getReg())) {
3445  unsigned Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
3446  BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg)
3447  .add(Src0);
3448  Src0.ChangeToRegister(Reg, false);
3449  }
3450  if (Src1.isReg() && RI.isVGPR(MRI, Src1.getReg())) {
3451  unsigned Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
3452  const DebugLoc &DL = MI.getDebugLoc();
3453  BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg)
3454  .add(Src1);
3455  Src1.ChangeToRegister(Reg, false);
3456  }
3457  return;
3458  }
3459 
3460  // VOP2 src0 instructions support all operand types, so we don't need to check
3461  // their legality. If src1 is already legal, we don't need to do anything.
3462  if (isLegalRegOperand(MRI, InstrDesc.OpInfo[Src1Idx], Src1))
3463  return;
3464 
3465  // Special case: V_READLANE_B32 accepts only immediate or SGPR operands for
3466  // lane select. Fix up using V_READFIRSTLANE, since we assume that the lane
3467  // select is uniform.
3468  if (Opc == AMDGPU::V_READLANE_B32 && Src1.isReg() &&
3469  RI.isVGPR(MRI, Src1.getReg())) {
3470  unsigned Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
3471  const DebugLoc &DL = MI.getDebugLoc();
3472  BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg)
3473  .add(Src1);
3474  Src1.ChangeToRegister(Reg, false);
3475  return;
3476  }
3477 
3478  // We do not use commuteInstruction here because it is too aggressive and will
3479  // commute if it is possible. We only want to commute here if it improves
3480  // legality. This can be called a fairly large number of times so don't waste
3481  // compile time pointlessly swapping and checking legality again.
3482  if (HasImplicitSGPR || !MI.isCommutable()) {
3483  legalizeOpWithMove(MI, Src1Idx);
3484  return;
3485  }
3486 
3487  int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0);
3488  MachineOperand &Src0 = MI.getOperand(Src0Idx);
3489 
3490  // If src0 can be used as src1, commuting will make the operands legal.
3491  // Otherwise we have to give up and insert a move.
3492  //
3493  // TODO: Other immediate-like operand kinds could be commuted if there was a
3494  // MachineOperand::ChangeTo* for them.
3495  if ((!Src1.isImm() && !Src1.isReg()) ||
3496  !isLegalRegOperand(MRI, InstrDesc.OpInfo[Src1Idx], Src0)) {
3497  legalizeOpWithMove(MI, Src1Idx);
3498  return;
3499  }
3500 
3501  int CommutedOpc = commuteOpcode(MI);
3502  if (CommutedOpc == -1) {
3503  legalizeOpWithMove(MI, Src1Idx);
3504  return;
3505  }
3506 
3507  MI.setDesc(get(CommutedOpc));
3508 
3509  unsigned Src0Reg = Src0.getReg();
3510  unsigned Src0SubReg = Src0.getSubReg();
3511  bool Src0Kill = Src0.isKill();
3512 
3513  if (Src1.isImm())
3514  Src0.ChangeToImmediate(Src1.getImm());
3515  else if (Src1.isReg()) {
3516  Src0.ChangeToRegister(Src1.getReg(), false, false, Src1.isKill());
3517  Src0.setSubReg(Src1.getSubReg());
3518  } else
3519  llvm_unreachable("Should only have register or immediate operands");
3520 
3521  Src1.ChangeToRegister(Src0Reg, false, false, Src0Kill);
3522  Src1.setSubReg(Src0SubReg);
3523 }
3524 
3525 // Legalize VOP3 operands. Because all operand types are supported for any
3526 // operand, and since literal constants are not allowed and should never be
3527 // seen, we only need to worry about inserting copies if we use multiple SGPR
3528 // operands.
3530  MachineInstr &MI) const {
3531  unsigned Opc = MI.getOpcode();
3532 
3533  int VOP3Idx[3] = {
3534  AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0),
3535  AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1),
3536  AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2)
3537  };
3538 
3539  // Find the one SGPR operand we are allowed to use.
3540  unsigned SGPRReg = findUsedSGPR(MI, VOP3Idx);
3541 
3542  for (unsigned i = 0; i < 3; ++i) {
3543  int Idx = VOP3Idx[i];
3544  if (Idx == -1)
3545  break;
3546  MachineOperand &MO = MI.getOperand(Idx);
3547 
3548  // We should never see a VOP3 instruction with an illegal immediate operand.
3549  if (!MO.isReg())
3550  continue;
3551 
3552  if (!RI.isSGPRClass(MRI.getRegClass(MO.getReg())))
3553  continue; // VGPRs are legal
3554 
3555  if (SGPRReg == AMDGPU::NoRegister || SGPRReg == MO.getReg()) {
3556  SGPRReg = MO.getReg();
3557  // We can use one SGPR in each VOP3 instruction.
3558  continue;
3559  }
3560 
3561  // If we make it this far, then the operand is not legal and we must
3562  // legalize it.
3563  legalizeOpWithMove(MI, Idx);
3564  }
3565 }
3566 
3568  MachineRegisterInfo &MRI) const {
3569  const TargetRegisterClass *VRC = MRI.getRegClass(SrcReg);
3570  const TargetRegisterClass *SRC = RI.getEquivalentSGPRClass(VRC);
3571  unsigned DstReg = MRI.createVirtualRegister(SRC);
3572  unsigned SubRegs = RI.getRegSizeInBits(*VRC) / 32;
3573 
3574  if (SubRegs == 1) {
3575  BuildMI(*UseMI.getParent(), UseMI, UseMI.getDebugLoc(),
3576  get(AMDGPU::V_READFIRSTLANE_B32), DstReg)
3577  .addReg(SrcReg);
3578  return DstReg;
3579  }
3580 
3582  for (unsigned i = 0; i < SubRegs; ++i) {
3583  unsigned SGPR = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
3584  BuildMI(*UseMI.getParent(), UseMI, UseMI.getDebugLoc(),
3585  get(AMDGPU::V_READFIRSTLANE_B32), SGPR)
3586  .addReg(SrcReg, 0, RI.getSubRegFromChannel(i));
3587  SRegs.push_back(SGPR);
3588  }
3589 
3590  MachineInstrBuilder MIB =
3591  BuildMI(*UseMI.getParent(), UseMI, UseMI.getDebugLoc(),
3592  get(AMDGPU::REG_SEQUENCE), DstReg);
3593  for (unsigned i = 0; i < SubRegs; ++i) {
3594  MIB.addReg(SRegs[i]);
3595  MIB.addImm(RI.getSubRegFromChannel(i));
3596  }
3597  return DstReg;
3598 }
3599 
3601  MachineInstr &MI) const {
3602 
3603  // If the pointer is store in VGPRs, then we need to move them to
3604  // SGPRs using v_readfirstlane. This is safe because we only select
3605  // loads with uniform pointers to SMRD instruction so we know the
3606  // pointer value is uniform.
3607  MachineOperand *SBase = getNamedOperand(MI, AMDGPU::OpName::sbase);
3608  if (SBase && !RI.isSGPRClass(MRI.getRegClass(SBase->getReg()))) {
3609  unsigned SGPR = readlaneVGPRToSGPR(SBase->getReg(), MI, MRI);
3610  SBase->setReg(SGPR);
3611  }
3612  MachineOperand *SOff = getNamedOperand(MI, AMDGPU::OpName::soff);
3613  if (SOff && !RI.isSGPRClass(MRI.getRegClass(SOff->getReg()))) {
3614  unsigned SGPR = readlaneVGPRToSGPR(SOff->getReg(), MI, MRI);
3615  SOff->setReg(SGPR);
3616  }
3617 }
3618 
3621  const TargetRegisterClass *DstRC,
3622  MachineOperand &Op,
3624  const DebugLoc &DL) const {
3625  unsigned OpReg = Op.getReg();
3626  unsigned OpSubReg = Op.getSubReg();
3627 
3628  const TargetRegisterClass *OpRC = RI.getSubClassWithSubReg(
3629  RI.getRegClassForReg(MRI, OpReg), OpSubReg);
3630 
3631  // Check if operand is already the correct register class.
3632  if (DstRC == OpRC)
3633  return;
3634 
3635  unsigned DstReg = MRI.createVirtualRegister(DstRC);
3636  MachineInstr *Copy =
3637  BuildMI(InsertMBB, I, DL, get(AMDGPU::COPY), DstReg).add(Op);
3638 
3639  Op.setReg(DstReg);
3640  Op.setSubReg(0);
3641 
3642  MachineInstr *Def = MRI.getVRegDef(OpReg);
3643  if (!Def)
3644  return;
3645 
3646  // Try to eliminate the copy if it is copying an immediate value.
3647  if (Def->isMoveImmediate())
3648  FoldImmediate(*Copy, *Def, OpReg, &MRI);
3649 }
3650 
3651 // Emit the actual waterfall loop, executing the wrapped instruction for each
3652 // unique value of \p Rsrc across all lanes. In the best case we execute 1
3653 // iteration, in the worst case we execute 64 (once per lane).
3654 static void
3656  MachineBasicBlock &OrigBB, MachineBasicBlock &LoopBB,
3657  const DebugLoc &DL, MachineOperand &Rsrc) {
3658  MachineBasicBlock::iterator I = LoopBB.begin();
3659 
3660  unsigned VRsrc = Rsrc.getReg();
3661  unsigned VRsrcUndef = getUndefRegState(Rsrc.isUndef());
3662 
3663  unsigned SaveExec = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
3664  unsigned CondReg0 = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
3665  unsigned CondReg1 = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
3666  unsigned AndCond = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
3667  unsigned SRsrcSub0 = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
3668  unsigned SRsrcSub1 = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
3669  unsigned SRsrcSub2 = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
3670  unsigned SRsrcSub3 = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
3671  unsigned SRsrc = MRI.createVirtualRegister(&AMDGPU::SReg_128RegClass);
3672 
3673  // Beginning of the loop, read the next Rsrc variant.
3674  BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), SRsrcSub0)
3675  .addReg(VRsrc, VRsrcUndef, AMDGPU::sub0);
3676  BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), SRsrcSub1)
3677  .addReg(VRsrc, VRsrcUndef, AMDGPU::sub1);
3678  BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), SRsrcSub2)
3679  .addReg(VRsrc, VRsrcUndef, AMDGPU::sub2);
3680  BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), SRsrcSub3)
3681  .addReg(VRsrc, VRsrcUndef, AMDGPU::sub3);
3682 
3683  BuildMI(LoopBB, I, DL, TII.get(AMDGPU::REG_SEQUENCE), SRsrc)
3684  .addReg(SRsrcSub0)
3685  .addImm(AMDGPU::sub0)
3686  .addReg(SRsrcSub1)
3687  .addImm(AMDGPU::sub1)
3688  .addReg(SRsrcSub2)
3689  .addImm(AMDGPU::sub2)
3690  .addReg(SRsrcSub3)
3691  .addImm(AMDGPU::sub3);
3692 
3693  // Update Rsrc operand to use the SGPR Rsrc.
3694  Rsrc.setReg(SRsrc);
3695  Rsrc.setIsKill(true);
3696 
3697  // Identify all lanes with identical Rsrc operands in their VGPRs.
3698  BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_CMP_EQ_U64_e64), CondReg0)
3699  .addReg(SRsrc, 0, AMDGPU::sub0_sub1)
3700  .addReg(VRsrc, 0, AMDGPU::sub0_sub1);
3701  BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_CMP_EQ_U64_e64), CondReg1)
3702  .addReg(SRsrc, 0, AMDGPU::sub2_sub3)
3703  .addReg(VRsrc, 0, AMDGPU::sub2_sub3);
3704  BuildMI(LoopBB, I, DL, TII.get(AMDGPU::S_AND_B64), AndCond)
3705  .addReg(CondReg0)
3706  .addReg(CondReg1);
3707 
3708  MRI.setSimpleHint(SaveExec, AndCond);
3709 
3710  // Update EXEC to matching lanes, saving original to SaveExec.
3711  BuildMI(LoopBB, I, DL, TII.get(AMDGPU::S_AND_SAVEEXEC_B64), SaveExec)
3712  .addReg(AndCond, RegState::Kill);
3713 
3714  // The original instruction is here; we insert the terminators after it.
3715  I = LoopBB.end();
3716 
3717  // Update EXEC, switch all done bits to 0 and all todo bits to 1.
3718  BuildMI(LoopBB, I, DL, TII.get(AMDGPU::S_XOR_B64_term), AMDGPU::EXEC)
3719  .addReg(AMDGPU::EXEC)
3720  .addReg(SaveExec);
3721  BuildMI(LoopBB, I, DL, TII.get(AMDGPU::S_CBRANCH_EXECNZ)).addMBB(&LoopBB);
3722 }
3723 
3724 // Build a waterfall loop around \p MI, replacing the VGPR \p Rsrc register
3725 // with SGPRs by iterating over all unique values across all lanes.
3727  MachineOperand &Rsrc, MachineDominatorTree *MDT) {
3728  MachineBasicBlock &MBB = *MI.getParent();
3729  MachineFunction &MF = *MBB.getParent();
3732  const DebugLoc &DL = MI.getDebugLoc();
3733 
3734  unsigned SaveExec = MRI.createVirtualRegister(&AMDGPU::SReg_64_XEXECRegClass);
3735 
3736  // Save the EXEC mask
3737  BuildMI(MBB, I, DL, TII.get(AMDGPU::S_MOV_B64), SaveExec)
3738  .addReg(AMDGPU::EXEC);
3739 
3740  // Killed uses in the instruction we are waterfalling around will be
3741  // incorrect due to the added control-flow.
3742  for (auto &MO : MI.uses()) {
3743  if (MO.isReg() && MO.isUse()) {
3744  MRI.clearKillFlags(MO.getReg());
3745  }
3746  }
3747 
3748  // To insert the loop we need to split the block. Move everything after this
3749  // point to a new block, and insert a new empty block between the two.
3751  MachineBasicBlock *RemainderBB = MF.CreateMachineBasicBlock();
3752  MachineFunction::iterator MBBI(MBB);
3753  ++MBBI;
3754 
3755  MF.insert(MBBI, LoopBB);
3756  MF.insert(MBBI, RemainderBB);
3757 
3758  LoopBB->addSuccessor(LoopBB);
3759  LoopBB->addSuccessor(RemainderBB);
3760 
3761  // Move MI to the LoopBB, and the remainder of the block to RemainderBB.
3763  RemainderBB->transferSuccessorsAndUpdatePHIs(&MBB);
3764  RemainderBB->splice(RemainderBB->begin(), &MBB, I, MBB.end());
3765  LoopBB->splice(LoopBB->begin(), &MBB, J);
3766 
3767  MBB.addSuccessor(LoopBB);
3768 
3769  // Update dominators. We know that MBB immediately dominates LoopBB, that
3770  // LoopBB immediately dominates RemainderBB, and that RemainderBB immediately
3771  // dominates all of the successors transferred to it from MBB that MBB used
3772  // to dominate.
3773  if (MDT) {
3774  MDT->addNewBlock(LoopBB, &MBB);
3775  MDT->addNewBlock(RemainderBB, LoopBB);
3776  for (auto &Succ : RemainderBB->successors()) {
3777  if (MDT->dominates(&MBB, Succ)) {
3778  MDT->changeImmediateDominator(Succ, RemainderBB);
3779  }
3780  }
3781  }
3782 
3783  emitLoadSRsrcFromVGPRLoop(TII, MRI, MBB, *LoopBB, DL, Rsrc);
3784 
3785  // Restore the EXEC mask
3786  MachineBasicBlock::iterator First = RemainderBB->begin();
3787  BuildMI(*RemainderBB, First, DL, TII.get(AMDGPU::S_MOV_B64), AMDGPU::EXEC)
3788  .addReg(SaveExec);
3789 }
3790 
3791 // Extract pointer from Rsrc and return a zero-value Rsrc replacement.
3792 static std::tuple<unsigned, unsigned>
3794  MachineBasicBlock &MBB = *MI.getParent();
3795  MachineFunction &MF = *MBB.getParent();
3797 
3798  // Extract the ptr from the resource descriptor.
3799  unsigned RsrcPtr =
3800  TII.buildExtractSubReg(MI, MRI, Rsrc, &AMDGPU::VReg_128RegClass,
3801  AMDGPU::sub0_sub1, &AMDGPU::VReg_64RegClass);
3802 
3803  // Create an empty resource descriptor
3804  unsigned Zero64 = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
3805  unsigned SRsrcFormatLo = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
3806  unsigned SRsrcFormatHi = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
3807  unsigned NewSRsrc = MRI.createVirtualRegister(&AMDGPU::SReg_128RegClass);
3808  uint64_t RsrcDataFormat = TII.getDefaultRsrcDataFormat();
3809 
3810  // Zero64 = 0
3811  BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::S_MOV_B64), Zero64)
3812  .addImm(0);
3813 
3814  // SRsrcFormatLo = RSRC_DATA_FORMAT{31-0}
3815  BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::S_MOV_B32), SRsrcFormatLo)
3816  .addImm(RsrcDataFormat & 0xFFFFFFFF);
3817 
3818  // SRsrcFormatHi = RSRC_DATA_FORMAT{63-32}
3819  BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::S_MOV_B32), SRsrcFormatHi)
3820  .addImm(RsrcDataFormat >> 32);
3821 
3822  // NewSRsrc = {Zero64, SRsrcFormat}
3823  BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::REG_SEQUENCE), NewSRsrc)
3824  .addReg(Zero64)
3825  .addImm(AMDGPU::sub0_sub1)
3826  .addReg(SRsrcFormatLo)
3827  .addImm(AMDGPU::sub2)
3828  .addReg(SRsrcFormatHi)
3829  .addImm(AMDGPU::sub3);
3830 
3831  return std::make_tuple(RsrcPtr, NewSRsrc);
3832 }
3833 
3835  MachineDominatorTree *MDT) const {
3836  MachineFunction &MF = *MI.getParent()->getParent();
3838 
3839  // Legalize VOP2
3840  if (isVOP2(MI) || isVOPC(MI)) {
3841  legalizeOperandsVOP2(MRI, MI);
3842  return;
3843  }
3844 
3845  // Legalize VOP3
3846  if (isVOP3(MI)) {
3847  legalizeOperandsVOP3(MRI, MI);
3848  return;
3849  }
3850 
3851  // Legalize SMRD
3852  if (isSMRD(MI)) {
3853  legalizeOperandsSMRD(MRI, MI);
3854  return;
3855  }
3856 
3857  // Legalize REG_SEQUENCE and PHI
3858  // The register class of the operands much be the same type as the register
3859  // class of the output.
3860  if (MI.getOpcode() == AMDGPU::PHI) {
3861  const TargetRegisterClass *RC = nullptr, *SRC = nullptr, *VRC = nullptr;
3862  for (unsigned i = 1, e = MI.getNumOperands(); i != e; i += 2) {
3863  if (!MI.getOperand(i).isReg() ||
3865  continue;
3866  const TargetRegisterClass *OpRC =
3867  MRI.getRegClass(MI.getOperand(i).getReg());
3868  if (RI.hasVGPRs(OpRC)) {
3869  VRC = OpRC;
3870  } else {
3871  SRC = OpRC;
3872  }
3873  }
3874 
3875  // If any of the operands are VGPR registers, then they all most be
3876  // otherwise we will create illegal VGPR->SGPR copies when legalizing
3877  // them.
3878  if (VRC || !RI.isSGPRClass(getOpRegClass(MI, 0))) {
3879  if (!VRC) {
3880  assert(SRC);
3881  VRC = RI.getEquivalentVGPRClass(SRC);
3882  }
3883  RC = VRC;
3884  } else {
3885  RC = SRC;
3886  }
3887 
3888  // Update all the operands so they have the same type.
3889  for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) {
3890  MachineOperand &Op = MI.getOperand(I);
3892  continue;
3893 
3894  // MI is a PHI instruction.
3895  MachineBasicBlock *InsertBB = MI.getOperand(I + 1).getMBB();
3896  MachineBasicBlock::iterator Insert = InsertBB->getFirstTerminator();
3897 
3898  // Avoid creating no-op copies with the same src and dst reg class. These
3899  // confuse some of the machine passes.
3900  legalizeGenericOperand(*InsertBB, Insert, RC, Op, MRI, MI.getDebugLoc());
3901  }
3902  }
3903 
3904  // REG_SEQUENCE doesn't really require operand legalization, but if one has a
3905  // VGPR dest type and SGPR sources, insert copies so all operands are
3906  // VGPRs. This seems to help operand folding / the register coalescer.
3907  if (MI.getOpcode() == AMDGPU::REG_SEQUENCE) {
3908  MachineBasicBlock *MBB = MI.getParent();
3909  const TargetRegisterClass *DstRC = getOpRegClass(MI, 0);
3910  if (RI.hasVGPRs(DstRC)) {
3911  // Update all the operands so they are VGPR register classes. These may
3912  // not be the same register class because REG_SEQUENCE supports mixing
3913  // subregister index types e.g. sub0_sub1 + sub2 + sub3
3914  for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) {
3915  MachineOperand &Op = MI.getOperand(I);
3917  continue;
3918 
3919  const TargetRegisterClass *OpRC = MRI.getRegClass(Op.getReg());
3920  const TargetRegisterClass *VRC = RI.getEquivalentVGPRClass(OpRC);
3921  if (VRC == OpRC)
3922  continue;
3923 
3924  legalizeGenericOperand(*MBB, MI, VRC, Op, MRI, MI.getDebugLoc());
3925  Op.setIsKill();
3926  }
3927  }
3928 
3929  return;
3930  }
3931 
3932  // Legalize INSERT_SUBREG
3933  // src0 must have the same register class as dst
3934  if (MI.getOpcode() == AMDGPU::INSERT_SUBREG) {
3935  unsigned Dst = MI.getOperand(0).getReg();
3936  unsigned Src0 = MI.getOperand(1).getReg();
3937  const TargetRegisterClass *DstRC = MRI.getRegClass(Dst);
3938  const TargetRegisterClass *Src0RC = MRI.getRegClass(Src0);
3939  if (DstRC != Src0RC) {
3940  MachineBasicBlock *MBB = MI.getParent();
3941  MachineOperand &Op = MI.getOperand(1);
3942  legalizeGenericOperand(*MBB, MI, DstRC, Op, MRI, MI.getDebugLoc());
3943  }
3944  return;
3945  }
3946 
3947  // Legalize SI_INIT_M0
3948  if (MI.getOpcode() == AMDGPU::SI_INIT_M0) {
3949  MachineOperand &Src = MI.getOperand(0);
3950  if (Src.isReg() && RI.hasVGPRs(MRI.getRegClass(Src.getReg())))
3951  Src.setReg(readlaneVGPRToSGPR(Src.getReg(), MI, MRI));
3952  return;
3953  }
3954 
3955  // Legalize MIMG and MUBUF/MTBUF for shaders.
3956  //
3957  // Shaders only generate MUBUF/MTBUF instructions via intrinsics or via
3958  // scratch memory access. In both cases, the legalization never involves
3959  // conversion to the addr64 form.
3960  if (isMIMG(MI) ||
3962  (isMUBUF(MI) || isMTBUF(MI)))) {
3963  MachineOperand *SRsrc = getNamedOperand(MI, AMDGPU::OpName::srsrc);
3964  if (SRsrc && !RI.isSGPRClass(MRI.getRegClass(SRsrc->getReg()))) {
3965  unsigned SGPR = readlaneVGPRToSGPR(SRsrc->getReg(), MI, MRI);
3966  SRsrc->setReg(SGPR);
3967  }
3968 
3969  MachineOperand *SSamp = getNamedOperand(MI, AMDGPU::OpName::ssamp);
3970  if (SSamp && !RI.isSGPRClass(MRI.getRegClass(SSamp->getReg()))) {
3971  unsigned SGPR = readlaneVGPRToSGPR(SSamp->getReg(), MI, MRI);
3972  SSamp->setReg(SGPR);
3973  }
3974  return;
3975  }
3976 
3977  // Legalize MUBUF* instructions.
3978  int RsrcIdx =
3979  AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::srsrc);
3980  if (RsrcIdx != -1) {
3981  // We have an MUBUF instruction
3982  MachineOperand *Rsrc = &MI.getOperand(RsrcIdx);
3983  unsigned RsrcRC = get(MI.getOpcode()).OpInfo[RsrcIdx].RegClass;
3984  if (RI.getCommonSubClass(MRI.getRegClass(Rsrc->getReg()),
3985  RI.getRegClass(RsrcRC))) {
3986  // The operands are legal.
3987  // FIXME: We may need to legalize operands besided srsrc.
3988  return;
3989  }
3990 
3991  // Legalize a VGPR Rsrc.
3992  //
3993  // If the instruction is _ADDR64, we can avoid a waterfall by extracting
3994  // the base pointer from the VGPR Rsrc, adding it to the VAddr, then using
3995  // a zero-value SRsrc.
3996  //
3997  // If the instruction is _OFFSET (both idxen and offen disabled), and we
3998  // support ADDR64 instructions, we can convert to ADDR64 and do the same as
3999  // above.
4000  //
4001  // Otherwise we are on non-ADDR64 hardware, and/or we have
4002  // idxen/offen/bothen and we fall back to a waterfall loop.
4003 
4004  MachineBasicBlock &MBB = *MI.getParent();
4005 
4006  MachineOperand *VAddr = getNamedOperand(MI, AMDGPU::OpName::vaddr);
4007  if (VAddr && AMDGPU::getIfAddr64Inst(MI.getOpcode()) != -1) {
4008  // This is already an ADDR64 instruction so we need to add the pointer
4009  // extracted from the resource descriptor to the current value of VAddr.
4010  unsigned NewVAddrLo = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4011  unsigned NewVAddrHi = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4012  unsigned NewVAddr = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass);
4013 
4014  unsigned RsrcPtr, NewSRsrc;
4015  std::tie(RsrcPtr, NewSRsrc) = extractRsrcPtr(*this, MI, *Rsrc);
4016 
4017  // NewVaddrLo = RsrcPtr:sub0 + VAddr:sub0
4018  DebugLoc DL = MI.getDebugLoc();
4019  BuildMI(MBB, MI, DL, get(AMDGPU::V_ADD_I32_e32), NewVAddrLo)
4020  .addReg(RsrcPtr, 0, AMDGPU::sub0)
4021  .addReg(VAddr->getReg(), 0, AMDGPU::sub0);
4022 
4023  // NewVaddrHi = RsrcPtr:sub1 + VAddr:sub1
4024  BuildMI(MBB, MI, DL, get(AMDGPU::V_ADDC_U32_e32), NewVAddrHi)
4025  .addReg(RsrcPtr, 0, AMDGPU::sub1)
4026  .addReg(VAddr->getReg(), 0, AMDGPU::sub1);
4027 
4028  // NewVaddr = {NewVaddrHi, NewVaddrLo}
4029  BuildMI(MBB, MI, MI.getDebugLoc(), get(AMDGPU::REG_SEQUENCE), NewVAddr)
4030  .addReg(NewVAddrLo)
4031  .addImm(AMDGPU::sub0)
4032  .addReg(NewVAddrHi)
4033  .addImm(AMDGPU::sub1);
4034 
4035  VAddr->setReg(NewVAddr);
4036  Rsrc->setReg(NewSRsrc);
4037  } else if (!VAddr && ST.hasAddr64()) {
4038  // This instructions is the _OFFSET variant, so we need to convert it to
4039  // ADDR64.
4040  assert(MBB.getParent()->getSubtarget<GCNSubtarget>().getGeneration()
4042  "FIXME: Need to emit flat atomics here");
4043 
4044  unsigned RsrcPtr, NewSRsrc;
4045  std::tie(RsrcPtr, NewSRsrc) = extractRsrcPtr(*this, MI, *Rsrc);
4046 
4047  unsigned NewVAddr = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass);
4048  MachineOperand *VData = getNamedOperand(MI, AMDGPU::OpName::vdata);
4049  MachineOperand *Offset = getNamedOperand(MI, AMDGPU::OpName::offset);
4050  MachineOperand *SOffset = getNamedOperand(MI, AMDGPU::OpName::soffset);
4051  unsigned Addr64Opcode = AMDGPU::getAddr64Inst(MI.getOpcode());
4052 
4053  // Atomics rith return have have an additional tied operand and are
4054  // missing some of the special bits.
4055  MachineOperand *VDataIn = getNamedOperand(MI, AMDGPU::OpName::vdata_in);
4056  MachineInstr *Addr64;
4057 
4058  if (!VDataIn) {
4059  // Regular buffer load / store.
4060  MachineInstrBuilder MIB =
4061  BuildMI(MBB, MI, MI.getDebugLoc(), get(Addr64Opcode))
4062  .add(*VData)
4063  .addReg(NewVAddr)
4064  .addReg(NewSRsrc)
4065  .add(*SOffset)
4066  .add(*Offset);
4067 
4068  // Atomics do not have this operand.
4069  if (const MachineOperand *GLC =
4070  getNamedOperand(MI, AMDGPU::OpName::glc)) {
4071  MIB.addImm(GLC->getImm());
4072  }
4073 
4074  MIB.addImm(getNamedImmOperand(MI, AMDGPU::OpName::slc));
4075 
4076  if (const MachineOperand *TFE =
4077  getNamedOperand(MI, AMDGPU::OpName::tfe)) {
4078  MIB.addImm(TFE->getImm());
4079  }
4080 
4081  MIB.cloneMemRefs(MI);
4082  Addr64 = MIB;
4083  } else {
4084  // Atomics with return.
4085  Addr64 = BuildMI(MBB, MI, MI.getDebugLoc(), get(Addr64Opcode))
4086  .add(*VData)
4087  .add(*VDataIn)
4088  .addReg(NewVAddr)
4089  .addReg(NewSRsrc)
4090  .add(*SOffset)
4091  .add(*Offset)
4092  .addImm(getNamedImmOperand(MI, AMDGPU::OpName::slc))
4093  .cloneMemRefs(MI);
4094  }
4095 
4096  MI.removeFromParent();
4097 
4098  // NewVaddr = {NewVaddrHi, NewVaddrLo}
4099  BuildMI(MBB, Addr64, Addr64->getDebugLoc(), get(AMDGPU::REG_SEQUENCE),
4100  NewVAddr)
4101  .addReg(RsrcPtr, 0, AMDGPU::sub0)
4102  .addImm(AMDGPU::sub0)
4103  .addReg(RsrcPtr, 0, AMDGPU::sub1)
4104  .addImm(AMDGPU::sub1);
4105  } else {
4106  // This is another variant; legalize Rsrc with waterfall loop from VGPRs
4107  // to SGPRs.
4108  loadSRsrcFromVGPR(*this, MI, *Rsrc, MDT);
4109  }
4110  }
4111 }
4112 
4114  MachineDominatorTree *MDT) const {
4115  SetVectorType Worklist;
4116  Worklist.insert(&TopInst);
4117 
4118  while (!Worklist.empty()) {
4119  MachineInstr &Inst = *Worklist.pop_back_val();
4120  MachineBasicBlock *MBB = Inst.getParent();
4122 
4123  unsigned Opcode = Inst.getOpcode();
4124  unsigned NewOpcode = getVALUOp(Inst);
4125 
4126  // Handle some special cases
4127  switch (Opcode) {
4128  default:
4129  break;
4130  case AMDGPU::S_ADD_U64_PSEUDO:
4131  case AMDGPU::S_SUB_U64_PSEUDO:
4132  splitScalar64BitAddSub(Worklist, Inst, MDT);
4133  Inst.eraseFromParent();
4134  continue;
4135  case AMDGPU::S_ADD_I32:
4136  case AMDGPU::S_SUB_I32:
4137  // FIXME: The u32 versions currently selected use the carry.
4138  if (moveScalarAddSub(Worklist, Inst, MDT))
4139  continue;
4140 
4141  // Default handling
4142  break;
4143  case AMDGPU::S_AND_B64:
4144  splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_AND_B32, MDT);
4145  Inst.eraseFromParent();
4146  continue;
4147 
4148  case AMDGPU::S_OR_B64:
4149  splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_OR_B32, MDT);
4150  Inst.eraseFromParent();
4151  continue;
4152 
4153  case AMDGPU::S_XOR_B64:
4154  splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_XOR_B32, MDT);
4155  Inst.eraseFromParent();
4156  continue;
4157 
4158  case AMDGPU::S_NAND_B64:
4159  splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_NAND_B32, MDT);
4160  Inst.eraseFromParent();
4161  continue;
4162 
4163  case AMDGPU::S_NOR_B64:
4164  splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_NOR_B32, MDT);
4165  Inst.eraseFromParent();
4166  continue;
4167 
4168  case AMDGPU::S_XNOR_B64:
4169  if (ST.hasDLInsts())
4170  splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_XNOR_B32, MDT);
4171  else
4172  splitScalar64BitXnor(Worklist, Inst, MDT);
4173  Inst.eraseFromParent();
4174  continue;
4175 
4176  case AMDGPU::S_ANDN2_B64:
4177  splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_ANDN2_B32, MDT);
4178  Inst.eraseFromParent();
4179  continue;
4180 
4181  case AMDGPU::S_ORN2_B64:
4182  splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_ORN2_B32, MDT);
4183  Inst.eraseFromParent();
4184  continue;
4185 
4186  case AMDGPU::S_NOT_B64:
4187  splitScalar64BitUnaryOp(Worklist, Inst, AMDGPU::S_NOT_B32);
4188  Inst.eraseFromParent();
4189  continue;
4190 
4191  case AMDGPU::S_BCNT1_I32_B64:
4192  splitScalar64BitBCNT(Worklist, Inst);
4193  Inst.eraseFromParent();
4194  continue;
4195 
4196  case AMDGPU::S_BFE_I64:
4197  splitScalar64BitBFE(Worklist, Inst);
4198  Inst.eraseFromParent();
4199  continue;
4200 
4201  case AMDGPU::S_LSHL_B32:
4202  if (ST.getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS) {
4203  NewOpcode = AMDGPU::V_LSHLREV_B32_e64;
4204  swapOperands(Inst);
4205  }
4206  break;
4207  case AMDGPU::S_ASHR_I32:
4208  if (ST.getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS) {
4209  NewOpcode = AMDGPU::V_ASHRREV_I32_e64;
4210  swapOperands(Inst);
4211  }
4212  break;
4213  case AMDGPU::S_LSHR_B32:
4214  if (ST.getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS) {
4215  NewOpcode = AMDGPU::V_LSHRREV_B32_e64;
4216  swapOperands(Inst);
4217  }
4218  break;
4219  case AMDGPU::S_LSHL_B64:
4220  if (ST.getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS) {
4221  NewOpcode = AMDGPU::V_LSHLREV_B64;
4222  swapOperands(Inst);
4223  }
4224  break;
4225  case AMDGPU::S_ASHR_I64:
4226  if (ST.getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS) {
4227  NewOpcode = AMDGPU::V_ASHRREV_I64;
4228  swapOperands(Inst);
4229  }
4230  break;
4231  case AMDGPU::S_LSHR_B64:
4232  if (ST.getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS) {
4233  NewOpcode = AMDGPU::V_LSHRREV_B64;
4234  swapOperands(Inst);
4235  }
4236  break;
4237 
4238  case AMDGPU::S_ABS_I32:
4239  lowerScalarAbs(Worklist, Inst);
4240  Inst.eraseFromParent();
4241  continue;
4242 
4243  case AMDGPU::S_CBRANCH_SCC0:
4244  case AMDGPU::S_CBRANCH_SCC1:
4245  // Clear unused bits of vcc
4246  BuildMI(*MBB, Inst, Inst.getDebugLoc(), get(AMDGPU::S_AND_B64),
4247  AMDGPU::VCC)
4248  .addReg(AMDGPU::EXEC)
4249  .addReg(AMDGPU::VCC);
4250  break;
4251 
4252  case AMDGPU::S_BFE_U64:
4253  case AMDGPU::S_BFM_B64:
4254  llvm_unreachable("Moving this op to VALU not implemented");
4255 
4256  case AMDGPU::S_PACK_LL_B32_B16:
4257  case AMDGPU::S_PACK_LH_B32_B16:
4258  case AMDGPU::S_PACK_HH_B32_B16:
4259  movePackToVALU(Worklist, MRI, Inst);
4260  Inst.eraseFromParent();
4261  continue;
4262 
4263  case AMDGPU::S_XNOR_B32:
4264  lowerScalarXnor(Worklist, Inst);
4265  Inst.eraseFromParent();
4266  continue;
4267 
4268  case AMDGPU::S_NAND_B32:
4269  splitScalarNotBinop(Worklist, Inst, AMDGPU::S_AND_B32);
4270  Inst.eraseFromParent();
4271  continue;
4272 
4273  case AMDGPU::S_NOR_B32:
4274  splitScalarNotBinop(Worklist, Inst, AMDGPU::S_OR_B32);
4275  Inst.eraseFromParent();
4276  continue;
4277 
4278  case AMDGPU::S_ANDN2_B32:
4279  splitScalarBinOpN2(Worklist, Inst, AMDGPU::S_AND_B32);
4280  Inst.eraseFromParent();
4281  continue;
4282 
4283  case AMDGPU::S_ORN2_B32:
4284  splitScalarBinOpN2(Worklist, Inst, AMDGPU::S_OR_B32);
4285  Inst.eraseFromParent();
4286  continue;
4287  }
4288 
4289  if (NewOpcode == AMDGPU::INSTRUCTION_LIST_END) {
4290  // We cannot move this instruction to the VALU, so we should try to
4291  // legalize its operands instead.
4292  legalizeOperands(Inst, MDT);
4293  continue;
4294  }
4295 
4296  // Use the new VALU Opcode.
4297  const MCInstrDesc &NewDesc = get(NewOpcode);
4298  Inst.setDesc(NewDesc);
4299 
4300  // Remove any references to SCC. Vector instructions can't read from it, and
4301  // We're just about to add the implicit use / defs of VCC, and we don't want
4302  // both.
4303  for (unsigned i = Inst.getNumOperands() - 1; i > 0; --i) {
4304  MachineOperand &Op = Inst.getOperand(i);
4305  if (Op.isReg() && Op.getReg() == AMDGPU::SCC) {
4306  Inst.RemoveOperand(i);
4307  addSCCDefUsersToVALUWorklist(Inst, Worklist);
4308  }
4309  }
4310 
4311  if (Opcode == AMDGPU::S_SEXT_I32_I8 || Opcode == AMDGPU::S_SEXT_I32_I16) {
4312  // We are converting these to a BFE, so we need to add the missing
4313  // operands for the size and offset.
4314  unsigned Size = (Opcode == AMDGPU::S_SEXT_I32_I8) ? 8 : 16;
4317 
4318  } else if (Opcode == AMDGPU::S_BCNT1_I32_B32) {
4319  // The VALU version adds the second operand to the result, so insert an
4320  // extra 0 operand.
4322  }
4323 
4325 
4326  if (Opcode == AMDGPU::S_BFE_I32 || Opcode == AMDGPU::S_BFE_U32) {
4327  const MachineOperand &OffsetWidthOp = Inst.getOperand(2);
4328  // If we need to move this to VGPRs, we need to unpack the second operand
4329  // back into the 2 separate ones for bit offset and width.
4330  assert(OffsetWidthOp.isImm() &&
4331  "Scalar BFE is only implemented for constant width and offset");
4332  uint32_t Imm = OffsetWidthOp.getImm();
4333 
4334  uint32_t Offset = Imm & 0x3f; // Extract bits [5:0].
4335  uint32_t BitWidth = (Imm & 0x7f0000) >> 16; // Extract bits [22:16].
4336  Inst.RemoveOperand(2); // Remove old immediate.
4337  Inst.addOperand(MachineOperand::CreateImm(Offset));
4338  Inst.addOperand(MachineOperand::CreateImm(BitWidth));
4339  }
4340 
4341  bool HasDst = Inst.getOperand(0).isReg() && Inst.getOperand(0).isDef();
4342  unsigned NewDstReg = AMDGPU::NoRegister;
4343  if (HasDst) {
4344  unsigned DstReg = Inst.getOperand(0).getReg();
4346  continue;
4347 
4348  // Update the destination register class.
4349  const TargetRegisterClass *NewDstRC = getDestEquivalentVGPRClass(Inst);
4350  if (!NewDstRC)
4351  continue;
4352 
4353  if (Inst.isCopy() &&
4355  NewDstRC == RI.getRegClassForReg(MRI, Inst.getOperand(1).getReg())) {
4356  // Instead of creating a copy where src and dst are the same register
4357  // class, we just replace all uses of dst with src. These kinds of
4358  // copies interfere with the heuristics MachineSink uses to decide
4359  // whether or not to split a critical edge. Since the pass assumes
4360  // that copies will end up as machine instructions and not be
4361  // eliminated.
4362  addUsersToMoveToVALUWorklist(DstReg, MRI, Worklist);
4363  MRI.replaceRegWith(DstReg, Inst.getOperand(1).getReg());
4364  MRI.clearKillFlags(Inst.getOperand(1).getReg());
4365  Inst.getOperand(0).setReg(DstReg);
4366 
4367  // Make sure we don't leave around a dead VGPR->SGPR copy. Normally
4368  // these are deleted later, but at -O0 it would leave a suspicious
4369  // looking illegal copy of an undef register.
4370  for (unsigned I = Inst.getNumOperands() - 1; I != 0; --I)
4371  Inst.RemoveOperand(I);
4372  Inst.setDesc(get(AMDGPU::IMPLICIT_DEF));
4373  continue;
4374  }
4375 
4376  NewDstReg = MRI.createVirtualRegister(NewDstRC);
4377  MRI.replaceRegWith(DstReg, NewDstReg);
4378  }
4379 
4380  // Legalize the operands
4381  legalizeOperands(Inst, MDT);
4382 
4383  if (HasDst)
4384  addUsersToMoveToVALUWorklist(NewDstReg, MRI, Worklist);
4385  }
4386 }
4387 
4388 // Add/sub require special handling to deal with carry outs.
4389 bool SIInstrInfo::moveScalarAddSub(SetVectorType &Worklist, MachineInstr &Inst,
4390  MachineDominatorTree *MDT) const {
4391  if (ST.hasAddNoCarry()) {
4392  // Assume there is no user of scc since we don't select this in that case.
4393  // Since scc isn't used, it doesn't really matter if the i32 or u32 variant
4394  // is used.
4395 
4396  MachineBasicBlock &MBB = *Inst.getParent();
4398 
4399  unsigned OldDstReg = Inst.getOperand(0).getReg();
4400  unsigned ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4401 
4402  unsigned Opc = Inst.getOpcode();
4403  assert(Opc == AMDGPU::S_ADD_I32 || Opc == AMDGPU::S_SUB_I32);
4404 
4405  unsigned NewOpc = Opc == AMDGPU::S_ADD_I32 ?
4406  AMDGPU::V_ADD_U32_e64 : AMDGPU::V_SUB_U32_e64;
4407 
4408  assert(Inst.getOperand(3).getReg() == AMDGPU::SCC);
4409  Inst.RemoveOperand(3);
4410 
4411  Inst.setDesc(get(NewOpc));
4412  Inst.addImplicitDefUseOperands(*MBB.getParent());
4413  MRI.replaceRegWith(OldDstReg, ResultReg);
4414  legalizeOperands(Inst, MDT);
4415 
4416  addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist);
4417  return true;
4418  }
4419 
4420  return false;
4421 }
4422 
4423 void SIInstrInfo::lowerScalarAbs(SetVectorType &Worklist,
4424  MachineInstr &Inst) const {
4425  MachineBasicBlock &MBB = *Inst.getParent();
4427  MachineBasicBlock::iterator MII = Inst;
4428  DebugLoc DL = Inst.getDebugLoc();
4429 
4430  MachineOperand &Dest = Inst.getOperand(0);
4431  MachineOperand &Src = Inst.getOperand(1);
4432  unsigned TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4433  unsigned ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4434 
4435  unsigned SubOp = ST.hasAddNoCarry() ?
4436  AMDGPU::V_SUB_U32_e32 : AMDGPU::V_SUB_I32_e32;
4437 
4438  BuildMI(MBB, MII, DL, get(SubOp), TmpReg)
4439  .addImm(0)
4440  .addReg(Src.getReg());
4441 
4442  BuildMI(MBB, MII, DL, get(AMDGPU::V_MAX_I32_e64), ResultReg)
4443  .addReg(Src.getReg())
4444  .addReg(TmpReg);
4445 
4446  MRI.replaceRegWith(Dest.getReg(), ResultReg);
4447  addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist);
4448 }
4449 
4450 void SIInstrInfo::lowerScalarXnor(SetVectorType &Worklist,
4451  MachineInstr &Inst) const {
4452  MachineBasicBlock &MBB = *Inst.getParent();
4454  MachineBasicBlock::iterator MII = Inst;
4455  const DebugLoc &DL = Inst.getDebugLoc();
4456 
4457  MachineOperand &Dest = Inst.getOperand(0);
4458  MachineOperand &Src0 = Inst.getOperand(1);
4459  MachineOperand &Src1 = Inst.getOperand(2);
4460 
4461  if (ST.hasDLInsts()) {
4462  unsigned NewDest = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4463  legalizeGenericOperand(MBB, MII, &AMDGPU::VGPR_32RegClass, Src0, MRI, DL);
4464  legalizeGenericOperand(MBB, MII, &AMDGPU::VGPR_32RegClass, Src1, MRI, DL);
4465 
4466  BuildMI(MBB, MII, DL, get(AMDGPU::V_XNOR_B32_e64), NewDest)
4467  .add(Src0)
4468  .add(Src1);
4469 
4470  MRI.replaceRegWith(Dest.getReg(), NewDest);
4471  addUsersToMoveToVALUWorklist(NewDest, MRI, Worklist);
4472  } else {
4473  // Using the identity !(x ^ y) == (!x ^ y) == (x ^ !y), we can
4474  // invert either source and then perform the XOR. If either source is a
4475  // scalar register, then we can leave the inversion on the scalar unit to
4476  // acheive a better distrubution of scalar and vector instructions.
4477  bool Src0IsSGPR = Src0.isReg() &&
4478  RI.isSGPRClass(MRI.getRegClass(Src0.getReg()));
4479  bool Src1IsSGPR = Src1.isReg() &&
4480  RI.isSGPRClass(MRI.getRegClass(Src1.getReg()));
4481  MachineInstr *Not = nullptr;
4482  MachineInstr *Xor = nullptr;
4483  unsigned Temp = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
4484  unsigned NewDest = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
4485 
4486  // Build a pair of scalar instructions and add them to the work list.
4487  // The next iteration over the work list will lower these to the vector
4488  // unit as necessary.
4489  if (Src0IsSGPR) {
4490  Not = BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), Temp)
4491  .add(Src0);
4492  Xor = BuildMI(MBB, MII, DL, get(AMDGPU::S_XOR_B32), NewDest)
4493  .addReg(Temp)
4494  .add(Src1);
4495  } else if (Src1IsSGPR) {
4496  Not = BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), Temp)
4497  .add(Src1);
4498  Xor = BuildMI(MBB, MII, DL, get(AMDGPU::S_XOR_B32), NewDest)
4499  .add(Src0)
4500  .addReg(Temp);
4501  } else {
4502  Xor = BuildMI(MBB, MII, DL, get(AMDGPU::S_XOR_B32), Temp)
4503  .add(Src0)
4504  .add(Src1);
4505  Not = BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), NewDest)
4506  .addReg(Temp);
4507  Worklist.insert(Not);
4508  }
4509 
4510  MRI.replaceRegWith(Dest.getReg(), NewDest);
4511 
4512  Worklist.insert(Xor);
4513 
4514  addUsersToMoveToVALUWorklist(NewDest, MRI, Worklist);
4515  }
4516 }
4517 
4518 void SIInstrInfo::splitScalarNotBinop(SetVectorType &Worklist,
4519  MachineInstr &Inst,
4520  unsigned Opcode) const {
4521  MachineBasicBlock &MBB = *Inst.getParent();
4523  MachineBasicBlock::iterator MII = Inst;
4524  const DebugLoc &DL = Inst.getDebugLoc();
4525 
4526  MachineOperand &Dest = Inst.getOperand(0);
4527  MachineOperand &Src0 = Inst.getOperand(1);
4528  MachineOperand &Src1 = Inst.getOperand(2);
4529 
4530  unsigned NewDest = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
4531  unsigned Interm = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
4532 
4533  MachineInstr &Op = *BuildMI(MBB, MII, DL, get(Opcode), Interm)
4534  .add(Src0)
4535  .add(Src1);
4536 
4537  MachineInstr &Not = *BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), NewDest)
4538  .addReg(Interm);
4539 
4540  Worklist.insert(&Op);
4541  Worklist.insert(&Not);
4542 
4543  MRI.replaceRegWith(Dest.getReg(), NewDest);
4544  addUsersToMoveToVALUWorklist(NewDest, MRI, Worklist);
4545 }
4546 
4547 void SIInstrInfo::splitScalarBinOpN2(SetVectorType& Worklist,
4548  MachineInstr &Inst,
4549  unsigned Opcode) const {
4550  MachineBasicBlock &MBB = *Inst.getParent();
4552  MachineBasicBlock::iterator MII = Inst;
4553  const DebugLoc &DL = Inst.getDebugLoc();
4554 
4555  MachineOperand &Dest = Inst.getOperand(0);
4556  MachineOperand &Src0 = Inst.getOperand(1);
4557  MachineOperand &Src1 = Inst.getOperand(2);
4558 
4559  unsigned NewDest = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
4560  unsigned Interm = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
4561 
4562  MachineInstr &Not = *BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), Interm)
4563  .add(Src1);
4564 
4565  MachineInstr &Op = *BuildMI(MBB, MII, DL, get(Opcode), NewDest)
4566  .add(Src0)
4567  .addReg(Interm);
4568 
4569  Worklist.insert(&Not);
4570  Worklist.insert(&Op);
4571 
4572  MRI.replaceRegWith(Dest.getReg(), NewDest);
4573  addUsersToMoveToVALUWorklist(NewDest, MRI, Worklist);
4574 }
4575 
4576 void SIInstrInfo::splitScalar64BitUnaryOp(
4577  SetVectorType &Worklist, MachineInstr &Inst,
4578  unsigned Opcode) const {
4579  MachineBasicBlock &MBB = *Inst.getParent();
4581 
4582  MachineOperand &Dest = Inst.getOperand(0);
4583  MachineOperand &Src0 = Inst.getOperand(1);
4584  DebugLoc DL = Inst.getDebugLoc();
4585 
4586  MachineBasicBlock::iterator MII = Inst;
4587 
4588  const MCInstrDesc &InstDesc = get(Opcode);
4589  const TargetRegisterClass *Src0RC = Src0.isReg() ?
4590  MRI.getRegClass(Src0.getReg()) :
4591  &AMDGPU::SGPR_32RegClass;
4592 
4593  const TargetRegisterClass *Src0SubRC = RI.getSubRegClass(Src0RC, AMDGPU::sub0);
4594 
4595  MachineOperand SrcReg0Sub0 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC,
4596  AMDGPU::sub0, Src0SubRC);
4597 
4598  const TargetRegisterClass *DestRC = MRI.getRegClass(Dest.getReg());
4599  const TargetRegisterClass *NewDestRC = RI.getEquivalentVGPRClass(DestRC);
4600  const TargetRegisterClass *NewDestSubRC = RI.getSubRegClass(NewDestRC, AMDGPU::sub0);
4601 
4602  unsigned DestSub0 = MRI.createVirtualRegister(NewDestSubRC);
4603  MachineInstr &LoHalf = *BuildMI(MBB, MII, DL, InstDesc, DestSub0).add(SrcReg0Sub0);
4604 
4605  MachineOperand SrcReg0Sub1 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC,
4606  AMDGPU::sub1, Src0SubRC);
4607 
4608  unsigned DestSub1 = MRI.createVirtualRegister(NewDestSubRC);
4609  MachineInstr &HiHalf = *BuildMI(MBB, MII, DL, InstDesc, DestSub1).add(SrcReg0Sub1);
4610 
4611  unsigned FullDestReg = MRI.createVirtualRegister(NewDestRC);
4612  BuildMI(MBB, MII, DL, get(TargetOpcode::REG_SEQUENCE), FullDestReg)
4613  .addReg(DestSub0)
4614  .addImm(AMDGPU::sub0)
4615  .addReg(DestSub1)
4616  .addImm(AMDGPU::sub1);
4617 
4618  MRI.replaceRegWith(Dest.getReg(), FullDestReg);
4619 
4620  Worklist.insert(&LoHalf);
4621  Worklist.insert(&HiHalf);
4622 
4623  // We don't need to legalizeOperands here because for a single operand, src0
4624  // will support any kind of input.
4625 
4626  // Move all users of this moved value.
4627  addUsersToMoveToVALUWorklist(FullDestReg, MRI, Worklist);
4628 }
4629 
4630 void SIInstrInfo::splitScalar64BitAddSub(SetVectorType &Worklist,
4631  MachineInstr &Inst,
4632  MachineDominatorTree *MDT) const {
4633  bool IsAdd = (Inst.getOpcode() == AMDGPU::S_ADD_U64_PSEUDO);
4634 
4635  MachineBasicBlock &MBB = *Inst.getParent();
4637 
4638  unsigned FullDestReg = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass);
4639  unsigned DestSub0 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4640  unsigned DestSub1 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4641 
4642  unsigned CarryReg = MRI.createVirtualRegister(&AMDGPU::SReg_64_XEXECRegClass);
4643  unsigned DeadCarryReg = MRI.createVirtualRegister(&AMDGPU::SReg_64_XEXECRegClass);
4644 
4645  MachineOperand &Dest = Inst.getOperand(0);
4646  MachineOperand &Src0 = Inst.getOperand(1);
4647  MachineOperand &Src1 = Inst.getOperand(2);
4648  const DebugLoc &DL = Inst.getDebugLoc();
4649  MachineBasicBlock::iterator MII = Inst;
4650 
4651  const TargetRegisterClass *Src0RC = MRI.getRegClass(Src0.getReg());
4652  const TargetRegisterClass *Src1RC = MRI.getRegClass(Src1.getReg());
4653  const TargetRegisterClass *Src0SubRC = RI.getSubRegClass(Src0RC, AMDGPU::sub0);
4654  const TargetRegisterClass *Src1SubRC = RI.getSubRegClass(Src1RC, AMDGPU::sub0);
4655 
4656  MachineOperand SrcReg0Sub0 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC,
4657  AMDGPU::sub0, Src0SubRC);
4658  MachineOperand SrcReg1Sub0 = buildExtractSubRegOrImm(MII, MRI, Src1, Src1RC,
4659  AMDGPU::sub0, Src1SubRC);
4660 
4661 
4662  MachineOperand SrcReg0Sub1 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC,
4663  AMDGPU::sub1, Src0SubRC);
4664  MachineOperand SrcReg1Sub1 = buildExtractSubRegOrImm(MII, MRI, Src1, Src1RC,
4665  AMDGPU::sub1, Src1SubRC);
4666 
4667  unsigned LoOpc = IsAdd ? AMDGPU::V_ADD_I32_e64 : AMDGPU::V_SUB_I32_e64;
4668  MachineInstr *LoHalf =
4669  BuildMI(MBB, MII, DL, get(LoOpc), DestSub0)
4670  .addReg(CarryReg, RegState::Define)
4671  .add(SrcReg0Sub0)
4672  .add(SrcReg1Sub0);
4673 
4674  unsigned HiOpc = IsAdd ? AMDGPU::V_ADDC_U32_e64 : AMDGPU::V_SUBB_U32_e64;
4675  MachineInstr *HiHalf =
4676  BuildMI(MBB, MII, DL, get(HiOpc), DestSub1)
4677  .addReg(DeadCarryReg, RegState::Define | RegState::Dead)
4678  .add(SrcReg0Sub1)
4679  .add(SrcReg1Sub1)
4680  .addReg(CarryReg, RegState::Kill);
4681 
4682  BuildMI(MBB, MII, DL, get(TargetOpcode::REG_SEQUENCE), FullDestReg)
4683  .addReg(DestSub0)
4684  .addImm(AMDGPU::sub0)
4685  .addReg(DestSub1)
4686  .addImm(AMDGPU::sub1);
4687 
4688  MRI.replaceRegWith(Dest.getReg(), FullDestReg);
4689 
4690  // Try to legalize the operands in case we need to swap the order to keep it
4691  // valid.
4692  legalizeOperands(*LoHalf, MDT);
4693  legalizeOperands(*HiHalf, MDT);
4694 
4695  // Move all users of this moved vlaue.
4696  addUsersToMoveToVALUWorklist(FullDestReg, MRI, Worklist);
4697 }
4698 
4699 void SIInstrInfo::splitScalar64BitBinaryOp(SetVectorType &Worklist,
4700  MachineInstr &Inst, unsigned Opcode,
4701  MachineDominatorTree *MDT) const {
4702  MachineBasicBlock &MBB = *Inst.getParent();
4704 
4705  MachineOperand &Dest = Inst.getOperand(0);
4706  MachineOperand &Src0 = Inst.getOperand(1);
4707  MachineOperand &Src1 = Inst.getOperand(2);
4708  DebugLoc DL = Inst.getDebugLoc();
4709 
4710  MachineBasicBlock::iterator MII = Inst;
4711 
4712  const MCInstrDesc &InstDesc = get(Opcode);
4713  const TargetRegisterClass *Src0RC = Src0.isReg() ?
4714  MRI.getRegClass(Src0.getReg()) :
4715  &AMDGPU::SGPR_32RegClass;
4716 
4717  const TargetRegisterClass *Src0SubRC = RI.getSubRegClass(Src0RC, AMDGPU::sub0);
4718  const TargetRegisterClass *Src1RC = Src1.isReg() ?
4719  MRI.getRegClass(Src1.getReg()) :
4720  &AMDGPU::SGPR_32RegClass;
4721 
4722  const TargetRegisterClass *Src1SubRC = RI.getSubRegClass(Src1RC, AMDGPU::sub0);
4723 
4724  MachineOperand SrcReg0Sub0 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC,
4725  AMDGPU::sub0, Src0SubRC);
4726  MachineOperand SrcReg1Sub0 = buildExtractSubRegOrImm(MII, MRI, Src1, Src1RC,
4727  AMDGPU::sub0, Src1SubRC);
4728  MachineOperand SrcReg0Sub1 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC,
4729  AMDGPU::sub1, Src0SubRC);
4730  MachineOperand SrcReg1Sub1 = buildExtractSubRegOrImm(MII, MRI, Src1, Src1RC,
4731  AMDGPU::sub1, Src1SubRC);
4732 
4733  const TargetRegisterClass *DestRC = MRI.getRegClass(Dest.getReg());
4734  const TargetRegisterClass *NewDestRC = RI.getEquivalentVGPRClass(DestRC);
4735  const TargetRegisterClass *NewDestSubRC = RI.getSubRegClass(NewDestRC, AMDGPU::sub0);
4736 
4737  unsigned DestSub0 = MRI.createVirtualRegister(NewDestSubRC);
4738  MachineInstr &LoHalf = *BuildMI(MBB, MII, DL, InstDesc, DestSub0)
4739  .add(SrcReg0Sub0)
4740  .add(SrcReg1Sub0);
4741 
4742  unsigned DestSub1 = MRI.createVirtualRegister(NewDestSubRC);
4743  MachineInstr &HiHalf = *BuildMI(MBB, MII, DL, InstDesc, DestSub1)
4744  .add(SrcReg0Sub1)
4745  .add(SrcReg1Sub1);
4746 
4747  unsigned FullDestReg = MRI.createVirtualRegister(NewDestRC);
4748  BuildMI(MBB, MII, DL, get(TargetOpcode::REG_SEQUENCE), FullDestReg)
4749  .addReg(DestSub0)
4750  .addImm(AMDGPU::sub0)
4751  .addReg(DestSub1)
4752  .addImm(AMDGPU::sub1);
4753 
4754  MRI.replaceRegWith(Dest.getReg(), FullDestReg);
4755 
4756  Worklist.insert(&LoHalf);
4757  Worklist.insert(&HiHalf);
4758 
4759  // Move all users of this moved vlaue.
4760  addUsersToMoveToVALUWorklist(FullDestReg, MRI, Worklist);
4761 }
4762 
4763 void SIInstrInfo::splitScalar64BitXnor(SetVectorType &Worklist,
4764  MachineInstr &Inst,
4765  MachineDominatorTree *MDT) const {
4766  MachineBasicBlock &MBB = *Inst.getParent();
4768 
4769  MachineOperand &Dest = Inst.getOperand(0);
4770  MachineOperand &Src0 = Inst.getOperand(1);
4771  MachineOperand &Src1 = Inst.getOperand(2);
4772  const DebugLoc &DL = Inst.getDebugLoc();
4773 
4774  MachineBasicBlock::iterator MII = Inst;
4775 
4776  const TargetRegisterClass *DestRC = MRI.getRegClass(Dest.getReg());
4777 
4778  unsigned Interm = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
4779 
4780  MachineOperand* Op0;
4781  MachineOperand* Op1;
4782 
4783  if (Src0.isReg() && RI.isSGPRReg(MRI, Src0.getReg())) {
4784  Op0 = &Src0;
4785  Op1 = &Src1;
4786  } else {
4787  Op0 = &Src1;
4788  Op1 = &Src0;
4789  }
4790 
4791  BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B64), Interm)
4792  .add(*Op0);
4793 
4794  unsigned NewDest = MRI.createVirtualRegister(DestRC);
4795 
4796  MachineInstr &Xor = *BuildMI(MBB, MII, DL, get(AMDGPU::S_XOR_B64), NewDest)
4797  .addReg(Interm)
4798  .add(*Op1);
4799 
4800  MRI.replaceRegWith(Dest.getReg(), NewDest);
4801 
4802  Worklist.insert(&Xor);
4803 }
4804 
4805 void SIInstrInfo::splitScalar64BitBCNT(
4806  SetVectorType &Worklist, MachineInstr &Inst) const {
4807  MachineBasicBlock &MBB = *Inst.getParent();
4809 
4810  MachineBasicBlock::iterator MII = Inst;
4811  const DebugLoc &DL = Inst.getDebugLoc();
4812 
4813  MachineOperand &Dest = Inst.getOperand(0);
4814  MachineOperand &Src = Inst.getOperand(1);
4815 
4816  const MCInstrDesc &InstDesc = get(AMDGPU::V_BCNT_U32_B32_e64);
4817  const TargetRegisterClass *SrcRC = Src.isReg() ?
4818  MRI.getRegClass(Src.getReg()) :
4819  &AMDGPU::SGPR_32RegClass;
4820 
4821  unsigned MidReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4822  unsigned ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4823 
4824  const TargetRegisterClass *SrcSubRC = RI.getSubRegClass(SrcRC, AMDGPU::sub0);
4825 
4826  MachineOperand SrcRegSub0 = buildExtractSubRegOrImm(MII, MRI, Src, SrcRC,
4827  AMDGPU::sub0, SrcSubRC);
4828  MachineOperand SrcRegSub1 = buildExtractSubRegOrImm(MII, MRI, Src, SrcRC,
4829  AMDGPU::sub1, SrcSubRC);
4830 
4831  BuildMI(MBB, MII, DL, InstDesc, MidReg).add(SrcRegSub0).addImm(0);
4832 
4833  BuildMI(MBB, MII, DL, InstDesc, ResultReg).add(SrcRegSub1).addReg(MidReg);
4834 
4835  MRI.replaceRegWith(Dest.getReg(), ResultReg);
4836 
4837  // We don't need to legalize operands here. src0 for etiher instruction can be
4838  // an SGPR, and the second input is unused or determined here.
4839  addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist);
4840 }
4841 
4842 void SIInstrInfo::splitScalar64BitBFE(SetVectorType &Worklist,
4843  MachineInstr &Inst) const {
4844  MachineBasicBlock &MBB = *Inst.getParent();
4846  MachineBasicBlock::iterator MII = Inst;
4847  const DebugLoc &DL = Inst.getDebugLoc();
4848 
4849  MachineOperand &Dest = Inst.getOperand(0);
4850  uint32_t Imm = Inst.getOperand(2).getImm();
4851  uint32_t Offset = Imm & 0x3f; // Extract bits [5:0].
4852  uint32_t BitWidth = (Imm & 0x7f0000) >> 16; // Extract bits [22:16].
4853 
4854  (void) Offset;
4855 
4856  // Only sext_inreg cases handled.
4857  assert(Inst.getOpcode() == AMDGPU::S_BFE_I64 && BitWidth <= 32 &&
4858  Offset == 0 && "Not implemented");
4859 
4860  if (BitWidth < 32) {
4861  unsigned MidRegLo = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4862  unsigned MidRegHi = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4863  unsigned ResultReg = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass);
4864 
4865  BuildMI(MBB, MII, DL, get(AMDGPU::V_BFE_I32), MidRegLo)
4866  .addReg(Inst.getOperand(1).getReg(), 0, AMDGPU::sub0)
4867  .addImm(0)
4868  .addImm(BitWidth);
4869 
4870  BuildMI(MBB, MII, DL, get(AMDGPU::V_ASHRREV_I32_e32), MidRegHi)
4871  .addImm(31)
4872  .addReg(MidRegLo);
4873 
4874  BuildMI(MBB, MII, DL, get(TargetOpcode::REG_SEQUENCE), ResultReg)
4875  .addReg(MidRegLo)
4876  .addImm(AMDGPU::sub0)
4877  .addReg(MidRegHi)
4878  .addImm(AMDGPU::sub1);
4879 
4880  MRI.replaceRegWith(Dest.getReg(), ResultReg);
4881  addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist);
4882  return;
4883  }
4884 
4885  MachineOperand &Src = Inst.getOperand(1);
4886  unsigned TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4887  unsigned ResultReg = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass);
4888 
4889  BuildMI(MBB, MII, DL, get(AMDGPU::V_ASHRREV_I32_e64), TmpReg)
4890  .addImm(31)
4891  .addReg(Src.getReg(), 0, AMDGPU::sub0);
4892 
4893  BuildMI(MBB, MII, DL, get(TargetOpcode::REG_SEQUENCE), ResultReg)
4894  .addReg(Src.getReg(), 0, AMDGPU::sub0)
4895  .addImm(AMDGPU::sub0)
4896  .addReg(TmpReg)
4897  .addImm(AMDGPU::sub1);
4898 
4899  MRI.replaceRegWith(Dest.getReg(), ResultReg);
4900  addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist);
4901 }
4902 
4903 void SIInstrInfo::addUsersToMoveToVALUWorklist(
4904  unsigned DstReg,
4906  SetVectorType &Worklist) const {
4907  for (MachineRegisterInfo::use_iterator I = MRI.use_begin(DstReg),
4908  E = MRI.use_end(); I != E;) {
4909  MachineInstr &UseMI = *I->getParent();
4910  if (!canReadVGPR(UseMI, I.getOperandNo())) {
4911  Worklist.insert(&UseMI);
4912 
4913  do {
4914  ++I;
4915  } while (I != E && I->getParent() == &UseMI);
4916  } else {
4917  ++I;
4918  }
4919  }
4920 }
4921 
4922 void SIInstrInfo::movePackToVALU(SetVectorType &Worklist,
4923  MachineRegisterInfo &MRI,
4924  MachineInstr &Inst) const {
4925  unsigned ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4926  MachineBasicBlock *MBB = Inst.getParent();
4927  MachineOperand &Src0 = Inst.getOperand(1);
4928  MachineOperand &Src1 = Inst.getOperand(2);
4929  const DebugLoc &DL = Inst.getDebugLoc();
4930 
4931  switch (Inst.getOpcode()) {
4932  case AMDGPU::S_PACK_LL_B32_B16: {
4933  unsigned ImmReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4934  unsigned TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4935 
4936  // FIXME: Can do a lot better if we know the high bits of src0 or src1 are
4937  // 0.
4938  BuildMI(*MBB, Inst, DL, get(AMDGPU::V_MOV_B32_e32), ImmReg)
4939  .addImm(0xffff);
4940 
4941  BuildMI(*MBB, Inst, DL, get(AMDGPU::V_AND_B32_e64), TmpReg)
4942  .addReg(ImmReg, RegState::Kill)
4943  .add(Src0);
4944 
4945  BuildMI(*MBB, Inst, DL, get(AMDGPU::V_LSHL_OR_B32), ResultReg)
4946  .add(Src1)
4947  .addImm(16)
4948  .addReg(TmpReg, RegState::Kill);
4949  break;
4950  }
4951  case AMDGPU::S_PACK_LH_B32_B16: {
4952  unsigned ImmReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4953  BuildMI(*MBB, Inst, DL, get(AMDGPU::V_MOV_B32_e32), ImmReg)
4954  .addImm(0xffff);
4955  BuildMI(*MBB, Inst, DL, get(AMDGPU::V_BFI_B32), ResultReg)
4956  .addReg(ImmReg, RegState::Kill)
4957  .add(Src0)
4958  .add(Src1);
4959  break;
4960  }
4961  case AMDGPU::S_PACK_HH_B32_B16: {
4962  unsigned ImmReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4963  unsigned TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4964  BuildMI(*MBB, Inst, DL, get(AMDGPU::V_LSHRREV_B32_e64), TmpReg)
4965  .addImm(16)
4966  .add(Src0);
4967  BuildMI(*MBB, Inst, DL, get(AMDGPU::V_MOV_B32_e32), ImmReg)
4968  .addImm(0xffff0000);
4969  BuildMI(*MBB, Inst, DL, get(AMDGPU::V_AND_OR_B32), ResultReg)
4970  .add(Src1)
4971  .addReg(ImmReg, RegState::Kill)
4972  .addReg(TmpReg, RegState::Kill);
4973  break;
4974  }
4975  default:
4976  llvm_unreachable("unhandled s_pack_* instruction");
4977  }
4978 
4979  MachineOperand &Dest = Inst.getOperand(0);
4980  MRI.replaceRegWith(Dest.getReg(), ResultReg);
4981  addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist);
4982 }
4983 
4984 void SIInstrInfo::addSCCDefUsersToVALUWorklist(
4985  MachineInstr &SCCDefInst, SetVectorType &Worklist) const {
4986  // This assumes that all the users of SCC are in the same block
4987  // as the SCC def.
4988  for (MachineInstr &MI :
4990  SCCDefInst.getParent()->end())) {
4991  // Exit if we find another SCC def.
4992  if (MI.findRegisterDefOperandIdx(AMDGPU::SCC, false, false, &RI) != -1)
4993  return;
4994 
4995  if (MI.findRegisterUseOperandIdx(AMDGPU::SCC, false, &RI) != -1)
4996  Worklist.insert(&MI);
4997  }
4998 }
4999 
5000 const TargetRegisterClass *SIInstrInfo::getDestEquivalentVGPRClass(
5001  const MachineInstr &Inst) const {
5002  const TargetRegisterClass *NewDstRC = getOpRegClass(Inst, 0);
5003 
5004  switch (Inst.getOpcode()) {
5005  // For target instructions, getOpRegClass just returns the virtual register
5006  // class associated with the operand, so we need to find an equivalent VGPR
5007  // register class in order to move the instruction to the VALU.
5008  case AMDGPU::COPY:
5009  case AMDGPU::PHI:
5010  case AMDGPU::REG_SEQUENCE:
5011  case AMDGPU::INSERT_SUBREG:
5012  case AMDGPU::WQM:
5013  case AMDGPU::WWM:
5014  if (RI.hasVGPRs(NewDstRC))
5015  return nullptr;
5016 
5017  NewDstRC = RI.getEquivalentVGPRClass(NewDstRC);
5018  if (!NewDstRC)
5019  return nullptr;
5020  return NewDstRC;
5021  default:
5022  return NewDstRC;
5023  }
5024 }
5025 
5026 // Find the one SGPR operand we are allowed to use.
5027 unsigned SIInstrInfo::findUsedSGPR(const MachineInstr &MI,
5028  int OpIndices[3]) const {
5029  const MCInstrDesc &Desc = MI.getDesc();
5030 
5031  // Find the one SGPR operand we are allowed to use.
5032  //
5033  // First we need to consider the instruction's operand requirements before
5034  // legalizing. Some operands are required to be SGPRs, such as implicit uses
5035  // of VCC, but we are still bound by the constant bus requirement to only use
5036  // one.
5037  //
5038  // If the operand's class is an SGPR, we can never move it.
5039 
5040  unsigned SGPRReg = findImplicitSGPRRead(MI);
5041  if (SGPRReg != AMDGPU::NoRegister)
5042  return SGPRReg;
5043 
5044  unsigned UsedSGPRs[3] = { AMDGPU::NoRegister };
5045  const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
5046 
5047  for (unsigned i = 0; i < 3; ++i) {
5048  int Idx = OpIndices[i];
5049  if (Idx == -1)
5050  break;
5051 
5052  const MachineOperand &MO = MI.getOperand(Idx);
5053  if (!MO.isReg())
5054  continue;
5055 
5056  // Is this operand statically required to be an SGPR based on the operand
5057  // constraints?
5058  const TargetRegisterClass *OpRC = RI.getRegClass(Desc.OpInfo[Idx].RegClass);
5059  bool IsRequiredSGPR = RI.isSGPRClass(OpRC);
5060  if (IsRequiredSGPR)
5061  return MO.getReg();
5062 
5063  // If this could be a VGPR or an SGPR, Check the dynamic register class.
5064  unsigned Reg = MO.getReg();
5065  const TargetRegisterClass *RegRC = MRI.getRegClass(Reg);
5066  if (RI.isSGPRClass(RegRC))
5067  UsedSGPRs[i] = Reg;
5068  }
5069 
5070  // We don't have a required SGPR operand, so we have a bit more freedom in
5071  // selecting operands to move.
5072 
5073  // Try to select the most used SGPR. If an SGPR is equal to one of the
5074  // others, we choose that.
5075  //
5076  // e.g.
5077  // V_FMA_F32 v0, s0, s0, s0 -> No moves
5078  // V_FMA_F32 v0, s0, s1, s0 -> Move s1
5079 
5080  // TODO: If some of the operands are 64-bit SGPRs and some 32, we should
5081  // prefer those.
5082 
5083  if (UsedSGPRs[0] != AMDGPU::NoRegister) {
5084  if (UsedSGPRs[0] == UsedSGPRs[1] || UsedSGPRs[0] == UsedSGPRs[2])
5085  SGPRReg = UsedSGPRs[0];
5086  }
5087 
5088  if (SGPRReg == AMDGPU::NoRegister && UsedSGPRs[1] != AMDGPU::NoRegister) {
5089  if (UsedSGPRs[1] == UsedSGPRs[2])
5090  SGPRReg = UsedSGPRs[1];
5091  }
5092 
5093  return SGPRReg;
5094 }
5095 
5097  unsigned OperandName) const {
5098  int Idx = AMDGPU::getNamedOperandIdx(MI.getOpcode(), OperandName);
5099  if (Idx == -1)
5100  return nullptr;
5101 
5102  return &MI.getOperand(Idx);
5103 }
5104 
5106  uint64_t RsrcDataFormat = AMDGPU::RSRC_DATA_FORMAT;
5107  if (ST.isAmdHsaOS()) {
5108  // Set ATC = 1. GFX9 doesn't have this bit.
5109  if (ST.getGeneration() <= AMDGPUSubtarget::VOLCANIC_ISLANDS)
5110  RsrcDataFormat |= (1ULL << 56);
5111 
5112  // Set MTYPE = 2 (MTYPE_UC = uncached). GFX9 doesn't have this.
5113  // BTW, it disables TC L2 and therefore decreases performance.
5114  if (ST.getGeneration() == AMDGPUSubtarget::VOLCANIC_ISLANDS)
5115  RsrcDataFormat |= (2ULL << 59);
5116  }
5117 
5118  return RsrcDataFormat;
5119 }
5120 
5122  uint64_t Rsrc23 = getDefaultRsrcDataFormat() |
5124  0xffffffff; // Size;
5125 
5126  // GFX9 doesn't have ELEMENT_SIZE.
5127  if (ST.getGeneration() <= AMDGPUSubtarget::VOLCANIC_ISLANDS) {
5128  uint64_t EltSizeValue = Log2_32(ST.getMaxPrivateElementSize()) - 1;
5129  Rsrc23 |= EltSizeValue << AMDGPU::RSRC_ELEMENT_SIZE_SHIFT;
5130  }
5131 
5132  // IndexStride = 64.
5133  Rsrc23 |= UINT64_C(3) << AMDGPU::RSRC_INDEX_STRIDE_SHIFT;
5134 
5135  // If TID_ENABLE is set, DATA_FORMAT specifies stride bits [14:17].
5136  // Clear them unless we want a huge stride.
5137  if (ST.getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS)
5138  Rsrc23 &= ~AMDGPU::RSRC_DATA_FORMAT;
5139 
5140  return Rsrc23;
5141 }
5142 
5144  unsigned Opc = MI.getOpcode();
5145 
5146  return isSMRD(Opc);
5147 }
5148 
5150  unsigned Opc = MI.getOpcode();
5151 
5152  return isMUBUF(Opc) || isMTBUF(Opc) || isMIMG(Opc);
5153 }
5154 
5156  int &FrameIndex) const {
5157  const MachineOperand *Addr = getNamedOperand(MI, AMDGPU::OpName::vaddr);
5158  if (!Addr || !Addr->isFI())
5159  return AMDGPU::NoRegister;
5160 
5161  assert(!MI.memoperands_empty() &&
5163 
5164  FrameIndex = Addr->getIndex();
5165  return getNamedOperand(MI, AMDGPU::OpName::vdata)->getReg();
5166 }
5167 
5169  int &FrameIndex) const {
5170  const MachineOperand *Addr = getNamedOperand(MI, AMDGPU::OpName::addr);
5171  assert(Addr && Addr->isFI());
5172  FrameIndex = Addr->getIndex();
5173  return getNamedOperand(MI, AMDGPU::OpName::data)->getReg();
5174 }
5175 
5177  int &FrameIndex) const {
5178  if (!MI.mayLoad())
5179  return AMDGPU::NoRegister;
5180 
5181  if (isMUBUF(MI) || isVGPRSpill(MI))
5182  return isStackAccess(MI, FrameIndex);
5183 
5184  if (isSGPRSpill(MI))
5185  return isSGPRStackAccess(MI, FrameIndex);
5186 
5187  return AMDGPU::NoRegister;
5188 }
5189 
5191  int &FrameIndex) const {
5192  if (!MI.mayStore())
5193  return AMDGPU::NoRegister;
5194 
5195  if (isMUBUF(MI) || isVGPRSpill(MI))
5196  return isStackAccess(MI, FrameIndex);
5197 
5198  if (isSGPRSpill(MI))
5199  return isSGPRStackAccess(MI, FrameIndex);
5200 
5201  return AMDGPU::NoRegister;
5202 }
5203 
5205  unsigned Size = 0;
5208  while (++I != E && I->isInsideBundle()) {
5209  assert(!I->isBundle() && "No nested bundle!");
5210  Size += getInstSizeInBytes(*I);
5211  }
5212 
5213  return Size;
5214 }
5215 
5217  unsigned Opc = MI.getOpcode();
5218  const MCInstrDesc &Desc = getMCOpcodeFromPseudo(Opc);
5219  unsigned DescSize = Desc.getSize();
5220 
5221  // If we have a definitive size, we can use it. Otherwise we need to inspect
5222  // the operands to know the size.
5223  if (isFixedSize(MI))
5224  return DescSize;
5225 
5226  // 4-byte instructions may have a 32-bit literal encoded after them. Check
5227  // operands that coud ever be literals.
5228  if (isVALU(MI) || isSALU(MI)) {
5229  int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0);
5230  if (Src0Idx == -1)
5231  return DescSize; // No operands.
5232 
5233  if (isLiteralConstantLike(MI.getOperand(Src0Idx), Desc.OpInfo[Src0Idx]))
5234  return DescSize + 4;
5235 
5236  int Src1Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1);
5237  if (Src1Idx == -1)
5238  return DescSize;
5239 
5240  if (isLiteralConstantLike(MI.getOperand(Src1Idx), Desc.OpInfo[Src1Idx]))
5241  return DescSize + 4;
5242 
5243  int Src2Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2);
5244  if (Src2Idx == -1)
5245  return DescSize;
5246 
5247  if (isLiteralConstantLike(MI.getOperand(Src2Idx), Desc.OpInfo[Src2Idx]))
5248  return DescSize + 4;
5249 
5250  return DescSize;
5251  }
5252 
5253  switch (Opc) {
5254  case TargetOpcode::IMPLICIT_DEF:
5255  case TargetOpcode::KILL:
5256  case TargetOpcode::DBG_VALUE:
5258  return 0;
5259  case TargetOpcode::BUNDLE:
5260  return getInstBundleSize(MI);
5261  case TargetOpcode::INLINEASM: {
5262  const MachineFunction *MF = MI.getParent()->getParent();
5263  const char *AsmStr = MI.getOperand(0).getSymbolName();
5264  return getInlineAsmLength(AsmStr, *MF->getTarget().getMCAsmInfo());
5265  }
5266  default:
5267  return DescSize;
5268  }
5269 }
5270 
5272  if (!isFLAT(MI))
5273  return false;
5274 
5275  if (MI.memoperands_empty())
5276  return true;
5277 
5278  for (const MachineMemOperand *MMO : MI.memoperands()) {
5279  if (MMO->getAddrSpace() == AMDGPUAS::FLAT_ADDRESS)
5280  return true;
5281  }
5282  return false;
5283 }
5284 
5286  return Branch.getOpcode() == AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO;
5287 }
5288