LLVM  8.0.0svn
SIInstrInfo.cpp
Go to the documentation of this file.
1 //===- SIInstrInfo.cpp - SI Instruction Information ----------------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 /// \file
11 /// SI Implementation of TargetInstrInfo.
12 //
13 //===----------------------------------------------------------------------===//
14 
15 #include "SIInstrInfo.h"
16 #include "AMDGPU.h"
17 #include "AMDGPUIntrinsicInfo.h"
18 #include "AMDGPUSubtarget.h"
19 #include "GCNHazardRecognizer.h"
20 #include "SIDefines.h"
21 #include "SIMachineFunctionInfo.h"
22 #include "SIRegisterInfo.h"
24 #include "Utils/AMDGPUBaseInfo.h"
25 #include "llvm/ADT/APInt.h"
26 #include "llvm/ADT/ArrayRef.h"
27 #include "llvm/ADT/SmallVector.h"
28 #include "llvm/ADT/StringRef.h"
48 #include "llvm/IR/DebugLoc.h"
49 #include "llvm/IR/DiagnosticInfo.h"
50 #include "llvm/IR/Function.h"
51 #include "llvm/IR/InlineAsm.h"
52 #include "llvm/IR/LLVMContext.h"
53 #include "llvm/MC/MCInstrDesc.h"
54 #include "llvm/Support/Casting.h"
56 #include "llvm/Support/Compiler.h"
61 #include <cassert>
62 #include <cstdint>
63 #include <iterator>
64 #include <utility>
65 
66 using namespace llvm;
67 
68 #define GET_INSTRINFO_CTOR_DTOR
69 #include "AMDGPUGenInstrInfo.inc"
70 
71 namespace llvm {
72 namespace AMDGPU {
73 #define GET_D16ImageDimIntrinsics_IMPL
74 #define GET_ImageDimIntrinsicTable_IMPL
75 #define GET_RsrcIntrinsics_IMPL
76 #include "AMDGPUGenSearchableTables.inc"
77 }
78 }
79 
80 
81 // Must be at least 4 to be able to branch over minimum unconditional branch
82 // code. This is only for making it possible to write reasonably small tests for
83 // long branches.
84 static cl::opt<unsigned>
85 BranchOffsetBits("amdgpu-s-branch-bits", cl::ReallyHidden, cl::init(16),
86  cl::desc("Restrict range of branch instructions (DEBUG)"));
87 
89  : AMDGPUGenInstrInfo(AMDGPU::ADJCALLSTACKUP, AMDGPU::ADJCALLSTACKDOWN),
90  RI(ST), ST(ST) {}
91 
92 //===----------------------------------------------------------------------===//
93 // TargetInstrInfo callbacks
94 //===----------------------------------------------------------------------===//
95 
96 static unsigned getNumOperandsNoGlue(SDNode *Node) {
97  unsigned N = Node->getNumOperands();
98  while (N && Node->getOperand(N - 1).getValueType() == MVT::Glue)
99  --N;
100  return N;
101 }
102 
104  SDValue LastOp = Load->getOperand(getNumOperandsNoGlue(Load) - 1);
105  assert(LastOp.getValueType() == MVT::Other && "Chain missing from load node");
106  return LastOp;
107 }
108 
109 /// Returns true if both nodes have the same value for the given
110 /// operand \p Op, or if both nodes do not have this operand.
111 static bool nodesHaveSameOperandValue(SDNode *N0, SDNode* N1, unsigned OpName) {
112  unsigned Opc0 = N0->getMachineOpcode();
113  unsigned Opc1 = N1->getMachineOpcode();
114 
115  int Op0Idx = AMDGPU::getNamedOperandIdx(Opc0, OpName);
116  int Op1Idx = AMDGPU::getNamedOperandIdx(Opc1, OpName);
117 
118  if (Op0Idx == -1 && Op1Idx == -1)
119  return true;
120 
121 
122  if ((Op0Idx == -1 && Op1Idx != -1) ||
123  (Op1Idx == -1 && Op0Idx != -1))
124  return false;
125 
126  // getNamedOperandIdx returns the index for the MachineInstr's operands,
127  // which includes the result as the first operand. We are indexing into the
128  // MachineSDNode's operands, so we need to skip the result operand to get
129  // the real index.
130  --Op0Idx;
131  --Op1Idx;
132 
133  return N0->getOperand(Op0Idx) == N1->getOperand(Op1Idx);
134 }
135 
137  AliasAnalysis *AA) const {
138  // TODO: The generic check fails for VALU instructions that should be
139  // rematerializable due to implicit reads of exec. We really want all of the
140  // generic logic for this except for this.
141  switch (MI.getOpcode()) {
142  case AMDGPU::V_MOV_B32_e32:
143  case AMDGPU::V_MOV_B32_e64:
144  case AMDGPU::V_MOV_B64_PSEUDO:
145  return true;
146  default:
147  return false;
148  }
149 }
150 
152  int64_t &Offset0,
153  int64_t &Offset1) const {
154  if (!Load0->isMachineOpcode() || !Load1->isMachineOpcode())
155  return false;
156 
157  unsigned Opc0 = Load0->getMachineOpcode();
158  unsigned Opc1 = Load1->getMachineOpcode();
159 
160  // Make sure both are actually loads.
161  if (!get(Opc0).mayLoad() || !get(Opc1).mayLoad())
162  return false;
163 
164  if (isDS(Opc0) && isDS(Opc1)) {
165 
166  // FIXME: Handle this case:
167  if (getNumOperandsNoGlue(Load0) != getNumOperandsNoGlue(Load1))
168  return false;
169 
170  // Check base reg.
171  if (Load0->getOperand(1) != Load1->getOperand(1))
172  return false;
173 
174  // Check chain.
175  if (findChainOperand(Load0) != findChainOperand(Load1))
176  return false;
177 
178  // Skip read2 / write2 variants for simplicity.
179  // TODO: We should report true if the used offsets are adjacent (excluded
180  // st64 versions).
181  if (AMDGPU::getNamedOperandIdx(Opc0, AMDGPU::OpName::data1) != -1 ||
182  AMDGPU::getNamedOperandIdx(Opc1, AMDGPU::OpName::data1) != -1)
183  return false;
184 
185  Offset0 = cast<ConstantSDNode>(Load0->getOperand(2))->getZExtValue();
186  Offset1 = cast<ConstantSDNode>(Load1->getOperand(2))->getZExtValue();
187  return true;
188  }
189 
190  if (isSMRD(Opc0) && isSMRD(Opc1)) {
191  // Skip time and cache invalidation instructions.
192  if (AMDGPU::getNamedOperandIdx(Opc0, AMDGPU::OpName::sbase) == -1 ||
193  AMDGPU::getNamedOperandIdx(Opc1, AMDGPU::OpName::sbase) == -1)
194  return false;
195 
197 
198  // Check base reg.
199  if (Load0->getOperand(0) != Load1->getOperand(0))
200  return false;
201 
202  const ConstantSDNode *Load0Offset =
203  dyn_cast<ConstantSDNode>(Load0->getOperand(1));
204  const ConstantSDNode *Load1Offset =
205  dyn_cast<ConstantSDNode>(Load1->getOperand(1));
206 
207  if (!Load0Offset || !Load1Offset)
208  return false;
209 
210  // Check chain.
211  if (findChainOperand(Load0) != findChainOperand(Load1))
212  return false;
213 
214  Offset0 = Load0Offset->getZExtValue();
215  Offset1 = Load1Offset->getZExtValue();
216  return true;
217  }
218 
219  // MUBUF and MTBUF can access the same addresses.
220  if ((isMUBUF(Opc0) || isMTBUF(Opc0)) && (isMUBUF(Opc1) || isMTBUF(Opc1))) {
221 
222  // MUBUF and MTBUF have vaddr at different indices.
223  if (!nodesHaveSameOperandValue(Load0, Load1, AMDGPU::OpName::soffset) ||
224  findChainOperand(Load0) != findChainOperand(Load1) ||
225  !nodesHaveSameOperandValue(Load0, Load1, AMDGPU::OpName::vaddr) ||
226  !nodesHaveSameOperandValue(Load0, Load1, AMDGPU::OpName::srsrc))
227  return false;
228 
229  int OffIdx0 = AMDGPU::getNamedOperandIdx(Opc0, AMDGPU::OpName::offset);
230  int OffIdx1 = AMDGPU::getNamedOperandIdx(Opc1, AMDGPU::OpName::offset);
231 
232  if (OffIdx0 == -1 || OffIdx1 == -1)
233  return false;
234 
235  // getNamedOperandIdx returns the index for MachineInstrs. Since they
236  // inlcude the output in the operand list, but SDNodes don't, we need to
237  // subtract the index by one.
238  --OffIdx0;
239  --OffIdx1;
240 
241  SDValue Off0 = Load0->getOperand(OffIdx0);
242  SDValue Off1 = Load1->getOperand(OffIdx1);
243 
244  // The offset might be a FrameIndexSDNode.
245  if (!isa<ConstantSDNode>(Off0) || !isa<ConstantSDNode>(Off1))
246  return false;
247 
248  Offset0 = cast<ConstantSDNode>(Off0)->getZExtValue();
249  Offset1 = cast<ConstantSDNode>(Off1)->getZExtValue();
250  return true;
251  }
252 
253  return false;
254 }
255 
256 static bool isStride64(unsigned Opc) {
257  switch (Opc) {
258  case AMDGPU::DS_READ2ST64_B32:
259  case AMDGPU::DS_READ2ST64_B64:
260  case AMDGPU::DS_WRITE2ST64_B32:
261  case AMDGPU::DS_WRITE2ST64_B64:
262  return true;
263  default:
264  return false;
265  }
266 }
267 
268 bool SIInstrInfo::getMemOpBaseRegImmOfs(MachineInstr &LdSt, unsigned &BaseReg,
269  int64_t &Offset,
270  const TargetRegisterInfo *TRI) const {
271  unsigned Opc = LdSt.getOpcode();
272 
273  if (isDS(LdSt)) {
274  const MachineOperand *OffsetImm =
275  getNamedOperand(LdSt, AMDGPU::OpName::offset);
276  if (OffsetImm) {
277  // Normal, single offset LDS instruction.
278  const MachineOperand *AddrReg =
279  getNamedOperand(LdSt, AMDGPU::OpName::addr);
280 
281  BaseReg = AddrReg->getReg();
282  Offset = OffsetImm->getImm();
283  return true;
284  }
285 
286  // The 2 offset instructions use offset0 and offset1 instead. We can treat
287  // these as a load with a single offset if the 2 offsets are consecutive. We
288  // will use this for some partially aligned loads.
289  const MachineOperand *Offset0Imm =
290  getNamedOperand(LdSt, AMDGPU::OpName::offset0);
291  const MachineOperand *Offset1Imm =
292  getNamedOperand(LdSt, AMDGPU::OpName::offset1);
293 
294  uint8_t Offset0 = Offset0Imm->getImm();
295  uint8_t Offset1 = Offset1Imm->getImm();
296 
297  if (Offset1 > Offset0 && Offset1 - Offset0 == 1) {
298  // Each of these offsets is in element sized units, so we need to convert
299  // to bytes of the individual reads.
300 
301  unsigned EltSize;
302  if (LdSt.mayLoad())
303  EltSize = TRI->getRegSizeInBits(*getOpRegClass(LdSt, 0)) / 16;
304  else {
305  assert(LdSt.mayStore());
306  int Data0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data0);
307  EltSize = TRI->getRegSizeInBits(*getOpRegClass(LdSt, Data0Idx)) / 8;
308  }
309 
310  if (isStride64(Opc))
311  EltSize *= 64;
312 
313  const MachineOperand *AddrReg =
314  getNamedOperand(LdSt, AMDGPU::OpName::addr);
315  BaseReg = AddrReg->getReg();
316  Offset = EltSize * Offset0;
317  return true;
318  }
319 
320  return false;
321  }
322 
323  if (isMUBUF(LdSt) || isMTBUF(LdSt)) {
324  const MachineOperand *SOffset = getNamedOperand(LdSt, AMDGPU::OpName::soffset);
325  if (SOffset && SOffset->isReg())
326  return false;
327 
328  const MachineOperand *AddrReg =
329  getNamedOperand(LdSt, AMDGPU::OpName::vaddr);
330  if (!AddrReg)
331  return false;
332 
333  const MachineOperand *OffsetImm =
334  getNamedOperand(LdSt, AMDGPU::OpName::offset);
335  BaseReg = AddrReg->getReg();
336  Offset = OffsetImm->getImm();
337 
338  if (SOffset) // soffset can be an inline immediate.
339  Offset += SOffset->getImm();
340 
341  return true;
342  }
343 
344  if (isSMRD(LdSt)) {
345  const MachineOperand *OffsetImm =
346  getNamedOperand(LdSt, AMDGPU::OpName::offset);
347  if (!OffsetImm)
348  return false;
349 
350  const MachineOperand *SBaseReg =
351  getNamedOperand(LdSt, AMDGPU::OpName::sbase);
352  BaseReg = SBaseReg->getReg();
353  Offset = OffsetImm->getImm();
354  return true;
355  }
356 
357  if (isFLAT(LdSt)) {
358  const MachineOperand *VAddr = getNamedOperand(LdSt, AMDGPU::OpName::vaddr);
359  if (VAddr) {
360  // Can't analyze 2 offsets.
361  if (getNamedOperand(LdSt, AMDGPU::OpName::saddr))
362  return false;
363 
364  BaseReg = VAddr->getReg();
365  } else {
366  // scratch instructions have either vaddr or saddr.
367  BaseReg = getNamedOperand(LdSt, AMDGPU::OpName::saddr)->getReg();
368  }
369 
370  Offset = getNamedOperand(LdSt, AMDGPU::OpName::offset)->getImm();
371  return true;
372  }
373 
374  return false;
375 }
376 
377 static bool memOpsHaveSameBasePtr(const MachineInstr &MI1, unsigned BaseReg1,
378  const MachineInstr &MI2, unsigned BaseReg2) {
379  if (BaseReg1 == BaseReg2)
380  return true;
381 
382  if (!MI1.hasOneMemOperand() || !MI2.hasOneMemOperand())
383  return false;
384 
385  auto MO1 = *MI1.memoperands_begin();
386  auto MO2 = *MI2.memoperands_begin();
387  if (MO1->getAddrSpace() != MO2->getAddrSpace())
388  return false;
389 
390  auto Base1 = MO1->getValue();
391  auto Base2 = MO2->getValue();
392  if (!Base1 || !Base2)
393  return false;
394  const MachineFunction &MF = *MI1.getParent()->getParent();
395  const DataLayout &DL = MF.getFunction().getParent()->getDataLayout();
396  Base1 = GetUnderlyingObject(Base1, DL);
397  Base2 = GetUnderlyingObject(Base1, DL);
398 
399  if (isa<UndefValue>(Base1) || isa<UndefValue>(Base2))
400  return false;
401 
402  return Base1 == Base2;
403 }
404 
406  unsigned BaseReg1,
407  MachineInstr &SecondLdSt,
408  unsigned BaseReg2,
409  unsigned NumLoads) const {
410  if (!memOpsHaveSameBasePtr(FirstLdSt, BaseReg1, SecondLdSt, BaseReg2))
411  return false;
412 
413  const MachineOperand *FirstDst = nullptr;
414  const MachineOperand *SecondDst = nullptr;
415 
416  if ((isMUBUF(FirstLdSt) && isMUBUF(SecondLdSt)) ||
417  (isMTBUF(FirstLdSt) && isMTBUF(SecondLdSt)) ||
418  (isFLAT(FirstLdSt) && isFLAT(SecondLdSt))) {
419  const unsigned MaxGlobalLoadCluster = 6;
420  if (NumLoads > MaxGlobalLoadCluster)
421  return false;
422 
423  FirstDst = getNamedOperand(FirstLdSt, AMDGPU::OpName::vdata);
424  if (!FirstDst)
425  FirstDst = getNamedOperand(FirstLdSt, AMDGPU::OpName::vdst);
426  SecondDst = getNamedOperand(SecondLdSt, AMDGPU::OpName::vdata);
427  if (!SecondDst)
428  SecondDst = getNamedOperand(SecondLdSt, AMDGPU::OpName::vdst);
429  } else if (isSMRD(FirstLdSt) && isSMRD(SecondLdSt)) {
430  FirstDst = getNamedOperand(FirstLdSt, AMDGPU::OpName::sdst);
431  SecondDst = getNamedOperand(SecondLdSt, AMDGPU::OpName::sdst);
432  } else if (isDS(FirstLdSt) && isDS(SecondLdSt)) {
433  FirstDst = getNamedOperand(FirstLdSt, AMDGPU::OpName::vdst);
434  SecondDst = getNamedOperand(SecondLdSt, AMDGPU::OpName::vdst);
435  }
436 
437  if (!FirstDst || !SecondDst)
438  return false;
439 
440  // Try to limit clustering based on the total number of bytes loaded
441  // rather than the number of instructions. This is done to help reduce
442  // register pressure. The method used is somewhat inexact, though,
443  // because it assumes that all loads in the cluster will load the
444  // same number of bytes as FirstLdSt.
445 
446  // The unit of this value is bytes.
447  // FIXME: This needs finer tuning.
448  unsigned LoadClusterThreshold = 16;
449 
450  const MachineRegisterInfo &MRI =
451  FirstLdSt.getParent()->getParent()->getRegInfo();
452  const TargetRegisterClass *DstRC = MRI.getRegClass(FirstDst->getReg());
453 
454  return (NumLoads * (RI.getRegSizeInBits(*DstRC) / 8)) <= LoadClusterThreshold;
455 }
456 
457 // FIXME: This behaves strangely. If, for example, you have 32 load + stores,
458 // the first 16 loads will be interleaved with the stores, and the next 16 will
459 // be clustered as expected. It should really split into 2 16 store batches.
460 //
461 // Loads are clustered until this returns false, rather than trying to schedule
462 // groups of stores. This also means we have to deal with saying different
463 // address space loads should be clustered, and ones which might cause bank
464 // conflicts.
465 //
466 // This might be deprecated so it might not be worth that much effort to fix.
468  int64_t Offset0, int64_t Offset1,
469  unsigned NumLoads) const {
470  assert(Offset1 > Offset0 &&
471  "Second offset should be larger than first offset!");
472  // If we have less than 16 loads in a row, and the offsets are within 64
473  // bytes, then schedule together.
474 
475  // A cacheline is 64 bytes (for global memory).
476  return (NumLoads <= 16 && (Offset1 - Offset0) < 64);
477 }
478 
481  const DebugLoc &DL, unsigned DestReg,
482  unsigned SrcReg, bool KillSrc) {
483  MachineFunction *MF = MBB.getParent();
484  DiagnosticInfoUnsupported IllegalCopy(MF->getFunction(),
485  "illegal SGPR to VGPR copy",
486  DL, DS_Error);
487  LLVMContext &C = MF->getFunction().getContext();
488  C.diagnose(IllegalCopy);
489 
490  BuildMI(MBB, MI, DL, TII->get(AMDGPU::SI_ILLEGAL_COPY), DestReg)
491  .addReg(SrcReg, getKillRegState(KillSrc));
492 }
493 
496  const DebugLoc &DL, unsigned DestReg,
497  unsigned SrcReg, bool KillSrc) const {
498  const TargetRegisterClass *RC = RI.getPhysRegClass(DestReg);
499 
500  if (RC == &AMDGPU::VGPR_32RegClass) {
501  assert(AMDGPU::VGPR_32RegClass.contains(SrcReg) ||
502  AMDGPU::SReg_32RegClass.contains(SrcReg));
503  BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DestReg)
504  .addReg(SrcReg, getKillRegState(KillSrc));
505  return;
506  }
507 
508  if (RC == &AMDGPU::SReg_32_XM0RegClass ||
509  RC == &AMDGPU::SReg_32RegClass) {
510  if (SrcReg == AMDGPU::SCC) {
511  BuildMI(MBB, MI, DL, get(AMDGPU::S_CSELECT_B32), DestReg)
512  .addImm(-1)
513  .addImm(0);
514  return;
515  }
516 
517  if (!AMDGPU::SReg_32RegClass.contains(SrcReg)) {
518  reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc);
519  return;
520  }
521 
522  BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), DestReg)
523  .addReg(SrcReg, getKillRegState(KillSrc));
524  return;
525  }
526 
527  if (RC == &AMDGPU::SReg_64RegClass) {
528  if (DestReg == AMDGPU::VCC) {
529  if (AMDGPU::SReg_64RegClass.contains(SrcReg)) {
530  BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B64), AMDGPU::VCC)
531  .addReg(SrcReg, getKillRegState(KillSrc));
532  } else {
533  // FIXME: Hack until VReg_1 removed.
534  assert(AMDGPU::VGPR_32RegClass.contains(SrcReg));
535  BuildMI(MBB, MI, DL, get(AMDGPU::V_CMP_NE_U32_e32))
536  .addImm(0)
537  .addReg(SrcReg, getKillRegState(KillSrc));
538  }
539 
540  return;
541  }
542 
543  if (!AMDGPU::SReg_64RegClass.contains(SrcReg)) {
544  reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc);
545  return;
546  }
547 
548  BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B64), DestReg)
549  .addReg(SrcReg, getKillRegState(KillSrc));
550  return;
551  }
552 
553  if (DestReg == AMDGPU::SCC) {
554  assert(AMDGPU::SReg_32RegClass.contains(SrcReg));
555  BuildMI(MBB, MI, DL, get(AMDGPU::S_CMP_LG_U32))
556  .addReg(SrcReg, getKillRegState(KillSrc))
557  .addImm(0);
558  return;
559  }
560 
561  unsigned EltSize = 4;
562  unsigned Opcode = AMDGPU::V_MOV_B32_e32;
563  if (RI.isSGPRClass(RC)) {
564  if (RI.getRegSizeInBits(*RC) > 32) {
565  Opcode = AMDGPU::S_MOV_B64;
566  EltSize = 8;
567  } else {
568  Opcode = AMDGPU::S_MOV_B32;
569  EltSize = 4;
570  }
571 
572  if (!RI.isSGPRClass(RI.getPhysRegClass(SrcReg))) {
573  reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc);
574  return;
575  }
576  }
577 
578  ArrayRef<int16_t> SubIndices = RI.getRegSplitParts(RC, EltSize);
579  bool Forward = RI.getHWRegIndex(DestReg) <= RI.getHWRegIndex(SrcReg);
580 
581  for (unsigned Idx = 0; Idx < SubIndices.size(); ++Idx) {
582  unsigned SubIdx;
583  if (Forward)
584  SubIdx = SubIndices[Idx];
585  else
586  SubIdx = SubIndices[SubIndices.size() - Idx - 1];
587 
588  MachineInstrBuilder Builder = BuildMI(MBB, MI, DL,
589  get(Opcode), RI.getSubReg(DestReg, SubIdx));
590 
591  Builder.addReg(RI.getSubReg(SrcReg, SubIdx));
592 
593  if (Idx == 0)
594  Builder.addReg(DestReg, RegState::Define | RegState::Implicit);
595 
596  bool UseKill = KillSrc && Idx == SubIndices.size() - 1;
597  Builder.addReg(SrcReg, getKillRegState(UseKill) | RegState::Implicit);
598  }
599 }
600 
601 int SIInstrInfo::commuteOpcode(unsigned Opcode) const {
602  int NewOpc;
603 
604  // Try to map original to commuted opcode
605  NewOpc = AMDGPU::getCommuteRev(Opcode);
606  if (NewOpc != -1)
607  // Check if the commuted (REV) opcode exists on the target.
608  return pseudoToMCOpcode(NewOpc) != -1 ? NewOpc : -1;
609 
610  // Try to map commuted to original opcode
611  NewOpc = AMDGPU::getCommuteOrig(Opcode);
612  if (NewOpc != -1)
613  // Check if the original (non-REV) opcode exists on the target.
614  return pseudoToMCOpcode(NewOpc) != -1 ? NewOpc : -1;
615 
616  return Opcode;
617 }
618 
621  const DebugLoc &DL, unsigned DestReg,
622  int64_t Value) const {
624  const TargetRegisterClass *RegClass = MRI.getRegClass(DestReg);
625  if (RegClass == &AMDGPU::SReg_32RegClass ||
626  RegClass == &AMDGPU::SGPR_32RegClass ||
627  RegClass == &AMDGPU::SReg_32_XM0RegClass ||
628  RegClass == &AMDGPU::SReg_32_XM0_XEXECRegClass) {
629  BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), DestReg)
630  .addImm(Value);
631  return;
632  }
633 
634  if (RegClass == &AMDGPU::SReg_64RegClass ||
635  RegClass == &AMDGPU::SGPR_64RegClass ||
636  RegClass == &AMDGPU::SReg_64_XEXECRegClass) {
637  BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B64), DestReg)
638  .addImm(Value);
639  return;
640  }
641 
642  if (RegClass == &AMDGPU::VGPR_32RegClass) {
643  BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DestReg)
644  .addImm(Value);
645  return;
646  }
647  if (RegClass == &AMDGPU::VReg_64RegClass) {
648  BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B64_PSEUDO), DestReg)
649  .addImm(Value);
650  return;
651  }
652 
653  unsigned EltSize = 4;
654  unsigned Opcode = AMDGPU::V_MOV_B32_e32;
655  if (RI.isSGPRClass(RegClass)) {
656  if (RI.getRegSizeInBits(*RegClass) > 32) {
657  Opcode = AMDGPU::S_MOV_B64;
658  EltSize = 8;
659  } else {
660  Opcode = AMDGPU::S_MOV_B32;
661  EltSize = 4;
662  }
663  }
664 
665  ArrayRef<int16_t> SubIndices = RI.getRegSplitParts(RegClass, EltSize);
666  for (unsigned Idx = 0; Idx < SubIndices.size(); ++Idx) {
667  int64_t IdxValue = Idx == 0 ? Value : 0;
668 
669  MachineInstrBuilder Builder = BuildMI(MBB, MI, DL,
670  get(Opcode), RI.getSubReg(DestReg, Idx));
671  Builder.addImm(IdxValue);
672  }
673 }
674 
675 const TargetRegisterClass *
677  return &AMDGPU::VGPR_32RegClass;
678 }
679 
682  const DebugLoc &DL, unsigned DstReg,
684  unsigned TrueReg,
685  unsigned FalseReg) const {
687  assert(MRI.getRegClass(DstReg) == &AMDGPU::VGPR_32RegClass &&
688  "Not a VGPR32 reg");
689 
690  if (Cond.size() == 1) {
691  unsigned SReg = MRI.createVirtualRegister(&AMDGPU::SReg_64_XEXECRegClass);
692  BuildMI(MBB, I, DL, get(AMDGPU::COPY), SReg)
693  .add(Cond[0]);
694  BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
695  .addReg(FalseReg)
696  .addReg(TrueReg)
697  .addReg(SReg);
698  } else if (Cond.size() == 2) {
699  assert(Cond[0].isImm() && "Cond[0] is not an immediate");
700  switch (Cond[0].getImm()) {
701  case SIInstrInfo::SCC_TRUE: {
702  unsigned SReg = MRI.createVirtualRegister(&AMDGPU::SReg_64_XEXECRegClass);
703  BuildMI(MBB, I, DL, get(AMDGPU::S_CSELECT_B64), SReg)
704  .addImm(-1)
705  .addImm(0);
706  BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
707  .addReg(FalseReg)
708  .addReg(TrueReg)
709  .addReg(SReg);
710  break;
711  }
712  case SIInstrInfo::SCC_FALSE: {
713  unsigned SReg = MRI.createVirtualRegister(&AMDGPU::SReg_64_XEXECRegClass);
714  BuildMI(MBB, I, DL, get(AMDGPU::S_CSELECT_B64), SReg)
715  .addImm(0)
716  .addImm(-1);
717  BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
718  .addReg(FalseReg)
719  .addReg(TrueReg)
720  .addReg(SReg);
721  break;
722  }
723  case SIInstrInfo::VCCNZ: {
724  MachineOperand RegOp = Cond[1];
725  RegOp.setImplicit(false);
726  unsigned SReg = MRI.createVirtualRegister(&AMDGPU::SReg_64_XEXECRegClass);
727  BuildMI(MBB, I, DL, get(AMDGPU::COPY), SReg)
728  .add(RegOp);
729  BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
730  .addReg(FalseReg)
731  .addReg(TrueReg)
732  .addReg(SReg);
733  break;
734  }
735  case SIInstrInfo::VCCZ: {
736  MachineOperand RegOp = Cond[1];
737  RegOp.setImplicit(false);
738  unsigned SReg = MRI.createVirtualRegister(&AMDGPU::SReg_64_XEXECRegClass);
739  BuildMI(MBB, I, DL, get(AMDGPU::COPY), SReg)
740  .add(RegOp);
741  BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
742  .addReg(TrueReg)
743  .addReg(FalseReg)
744  .addReg(SReg);
745  break;
746  }
747  case SIInstrInfo::EXECNZ: {
748  unsigned SReg = MRI.createVirtualRegister(&AMDGPU::SReg_64_XEXECRegClass);
749  unsigned SReg2 = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
750  BuildMI(MBB, I, DL, get(AMDGPU::S_OR_SAVEEXEC_B64), SReg2)
751  .addImm(0);
752  BuildMI(MBB, I, DL, get(AMDGPU::S_CSELECT_B64), SReg)
753  .addImm(-1)
754  .addImm(0);
755  BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
756  .addReg(FalseReg)
757  .addReg(TrueReg)
758  .addReg(SReg);
759  break;
760  }
761  case SIInstrInfo::EXECZ: {
762  unsigned SReg = MRI.createVirtualRegister(&AMDGPU::SReg_64_XEXECRegClass);
763  unsigned SReg2 = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
764  BuildMI(MBB, I, DL, get(AMDGPU::S_OR_SAVEEXEC_B64), SReg2)
765  .addImm(0);
766  BuildMI(MBB, I, DL, get(AMDGPU::S_CSELECT_B64), SReg)
767  .addImm(0)
768  .addImm(-1);
769  BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
770  .addReg(FalseReg)
771  .addReg(TrueReg)
772  .addReg(SReg);
773  llvm_unreachable("Unhandled branch predicate EXECZ");
774  break;
775  }
776  default:
777  llvm_unreachable("invalid branch predicate");
778  }
779  } else {
780  llvm_unreachable("Can only handle Cond size 1 or 2");
781  }
782 }
783 
786  const DebugLoc &DL,
787  unsigned SrcReg, int Value) const {
789  unsigned Reg = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
790  BuildMI(*MBB, I, DL, get(AMDGPU::V_CMP_EQ_I32_e64), Reg)
791  .addImm(Value)
792  .addReg(SrcReg);
793 
794  return Reg;
795 }
796 
799  const DebugLoc &DL,
800  unsigned SrcReg, int Value) const {
802  unsigned Reg = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
803  BuildMI(*MBB, I, DL, get(AMDGPU::V_CMP_NE_I32_e64), Reg)
804  .addImm(Value)
805  .addReg(SrcReg);
806 
807  return Reg;
808 }
809 
810 unsigned SIInstrInfo::getMovOpcode(const TargetRegisterClass *DstRC) const {
811 
812  if (RI.getRegSizeInBits(*DstRC) == 32) {
813  return RI.isSGPRClass(DstRC) ? AMDGPU::S_MOV_B32 : AMDGPU::V_MOV_B32_e32;
814  } else if (RI.getRegSizeInBits(*DstRC) == 64 && RI.isSGPRClass(DstRC)) {
815  return AMDGPU::S_MOV_B64;
816  } else if (RI.getRegSizeInBits(*DstRC) == 64 && !RI.isSGPRClass(DstRC)) {
817  return AMDGPU::V_MOV_B64_PSEUDO;
818  }
819  return AMDGPU::COPY;
820 }
821 
822 static unsigned getSGPRSpillSaveOpcode(unsigned Size) {
823  switch (Size) {
824  case 4:
825  return AMDGPU::SI_SPILL_S32_SAVE;
826  case 8:
827  return AMDGPU::SI_SPILL_S64_SAVE;
828  case 16:
829  return AMDGPU::SI_SPILL_S128_SAVE;
830  case 32:
831  return AMDGPU::SI_SPILL_S256_SAVE;
832  case 64:
833  return AMDGPU::SI_SPILL_S512_SAVE;
834  default:
835  llvm_unreachable("unknown register size");
836  }
837 }
838 
839 static unsigned getVGPRSpillSaveOpcode(unsigned Size) {
840  switch (Size) {
841  case 4:
842  return AMDGPU::SI_SPILL_V32_SAVE;
843  case 8:
844  return AMDGPU::SI_SPILL_V64_SAVE;
845  case 12:
846  return AMDGPU::SI_SPILL_V96_SAVE;
847  case 16:
848  return AMDGPU::SI_SPILL_V128_SAVE;
849  case 32:
850  return AMDGPU::SI_SPILL_V256_SAVE;
851  case 64:
852  return AMDGPU::SI_SPILL_V512_SAVE;
853  default:
854  llvm_unreachable("unknown register size");
855  }
856 }
857 
860  unsigned SrcReg, bool isKill,
861  int FrameIndex,
862  const TargetRegisterClass *RC,
863  const TargetRegisterInfo *TRI) const {
864  MachineFunction *MF = MBB.getParent();
866  MachineFrameInfo &FrameInfo = MF->getFrameInfo();
867  DebugLoc DL = MBB.findDebugLoc(MI);
868 
869  unsigned Size = FrameInfo.getObjectSize(FrameIndex);
870  unsigned Align = FrameInfo.getObjectAlignment(FrameIndex);
871  MachinePointerInfo PtrInfo
872  = MachinePointerInfo::getFixedStack(*MF, FrameIndex);
873  MachineMemOperand *MMO
875  Size, Align);
876  unsigned SpillSize = TRI->getSpillSize(*RC);
877 
878  if (RI.isSGPRClass(RC)) {
879  MFI->setHasSpilledSGPRs();
880 
881  // We are only allowed to create one new instruction when spilling
882  // registers, so we need to use pseudo instruction for spilling SGPRs.
883  const MCInstrDesc &OpDesc = get(getSGPRSpillSaveOpcode(SpillSize));
884 
885  // The SGPR spill/restore instructions only work on number sgprs, so we need
886  // to make sure we are using the correct register class.
887  if (TargetRegisterInfo::isVirtualRegister(SrcReg) && SpillSize == 4) {
889  MRI.constrainRegClass(SrcReg, &AMDGPU::SReg_32_XM0RegClass);
890  }
891 
892  MachineInstrBuilder Spill = BuildMI(MBB, MI, DL, OpDesc)
893  .addReg(SrcReg, getKillRegState(isKill)) // data
894  .addFrameIndex(FrameIndex) // addr
895  .addMemOperand(MMO)
897  .addReg(MFI->getFrameOffsetReg(), RegState::Implicit);
898  // Add the scratch resource registers as implicit uses because we may end up
899  // needing them, and need to ensure that the reserved registers are
900  // correctly handled.
901 
902  FrameInfo.setStackID(FrameIndex, SIStackID::SGPR_SPILL);
903  if (ST.hasScalarStores()) {
904  // m0 is used for offset to scalar stores if used to spill.
905  Spill.addReg(AMDGPU::M0, RegState::ImplicitDefine | RegState::Dead);
906  }
907 
908  return;
909  }
910 
911  if (!ST.isVGPRSpillingEnabled(MF->getFunction())) {
912  LLVMContext &Ctx = MF->getFunction().getContext();
913  Ctx.emitError("SIInstrInfo::storeRegToStackSlot - Do not know how to"
914  " spill register");
915  BuildMI(MBB, MI, DL, get(AMDGPU::KILL))
916  .addReg(SrcReg);
917 
918  return;
919  }
920 
921  assert(RI.hasVGPRs(RC) && "Only VGPR spilling expected");
922 
923  unsigned Opcode = getVGPRSpillSaveOpcode(SpillSize);
924  MFI->setHasSpilledVGPRs();
925  BuildMI(MBB, MI, DL, get(Opcode))
926  .addReg(SrcReg, getKillRegState(isKill)) // data
927  .addFrameIndex(FrameIndex) // addr
928  .addReg(MFI->getScratchRSrcReg()) // scratch_rsrc
929  .addReg(MFI->getFrameOffsetReg()) // scratch_offset
930  .addImm(0) // offset
931  .addMemOperand(MMO);
932 }
933 
934 static unsigned getSGPRSpillRestoreOpcode(unsigned Size) {
935  switch (Size) {
936  case 4:
937  return AMDGPU::SI_SPILL_S32_RESTORE;
938  case 8:
939  return AMDGPU::SI_SPILL_S64_RESTORE;
940  case 16:
941  return AMDGPU::SI_SPILL_S128_RESTORE;
942  case 32:
943  return AMDGPU::SI_SPILL_S256_RESTORE;
944  case 64:
945  return AMDGPU::SI_SPILL_S512_RESTORE;
946  default:
947  llvm_unreachable("unknown register size");
948  }
949 }
950 
951 static unsigned getVGPRSpillRestoreOpcode(unsigned Size) {
952  switch (Size) {
953  case 4:
954  return AMDGPU::SI_SPILL_V32_RESTORE;
955  case 8:
956  return AMDGPU::SI_SPILL_V64_RESTORE;
957  case 12:
958  return AMDGPU::SI_SPILL_V96_RESTORE;
959  case 16:
960  return AMDGPU::SI_SPILL_V128_RESTORE;
961  case 32:
962  return AMDGPU::SI_SPILL_V256_RESTORE;
963  case 64:
964  return AMDGPU::SI_SPILL_V512_RESTORE;
965  default:
966  llvm_unreachable("unknown register size");
967  }
968 }
969 
972  unsigned DestReg, int FrameIndex,
973  const TargetRegisterClass *RC,
974  const TargetRegisterInfo *TRI) const {
975  MachineFunction *MF = MBB.getParent();
977  MachineFrameInfo &FrameInfo = MF->getFrameInfo();
978  DebugLoc DL = MBB.findDebugLoc(MI);
979  unsigned Align = FrameInfo.getObjectAlignment(FrameIndex);
980  unsigned Size = FrameInfo.getObjectSize(FrameIndex);
981  unsigned SpillSize = TRI->getSpillSize(*RC);
982 
983  MachinePointerInfo PtrInfo
984  = MachinePointerInfo::getFixedStack(*MF, FrameIndex);
985 
987  PtrInfo, MachineMemOperand::MOLoad, Size, Align);
988 
989  if (RI.isSGPRClass(RC)) {
990  // FIXME: Maybe this should not include a memoperand because it will be
991  // lowered to non-memory instructions.
992  const MCInstrDesc &OpDesc = get(getSGPRSpillRestoreOpcode(SpillSize));
993  if (TargetRegisterInfo::isVirtualRegister(DestReg) && SpillSize == 4) {
995  MRI.constrainRegClass(DestReg, &AMDGPU::SReg_32_XM0RegClass);
996  }
997 
998  FrameInfo.setStackID(FrameIndex, SIStackID::SGPR_SPILL);
999  MachineInstrBuilder Spill = BuildMI(MBB, MI, DL, OpDesc, DestReg)
1000  .addFrameIndex(FrameIndex) // addr
1001  .addMemOperand(MMO)
1003  .addReg(MFI->getFrameOffsetReg(), RegState::Implicit);
1004 
1005  if (ST.hasScalarStores()) {
1006  // m0 is used for offset to scalar stores if used to spill.
1007  Spill.addReg(AMDGPU::M0, RegState::ImplicitDefine | RegState::Dead);
1008  }
1009 
1010  return;
1011  }
1012 
1013  if (!ST.isVGPRSpillingEnabled(MF->getFunction())) {
1014  LLVMContext &Ctx = MF->getFunction().getContext();
1015  Ctx.emitError("SIInstrInfo::loadRegFromStackSlot - Do not know how to"
1016  " restore register");
1017  BuildMI(MBB, MI, DL, get(AMDGPU::IMPLICIT_DEF), DestReg);
1018 
1019  return;
1020  }
1021 
1022  assert(RI.hasVGPRs(RC) && "Only VGPR spilling expected");
1023 
1024  unsigned Opcode = getVGPRSpillRestoreOpcode(SpillSize);
1025  BuildMI(MBB, MI, DL, get(Opcode), DestReg)
1026  .addFrameIndex(FrameIndex) // vaddr
1027  .addReg(MFI->getScratchRSrcReg()) // scratch_rsrc
1028  .addReg(MFI->getFrameOffsetReg()) // scratch_offset
1029  .addImm(0) // offset
1030  .addMemOperand(MMO);
1031 }
1032 
1033 /// \param @Offset Offset in bytes of the FrameIndex being spilled
1035  MachineBasicBlock &MBB, MachineInstr &MI, RegScavenger *RS, unsigned TmpReg,
1036  unsigned FrameOffset, unsigned Size) const {
1037  MachineFunction *MF = MBB.getParent();
1039  const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
1040  DebugLoc DL = MBB.findDebugLoc(MI);
1041  unsigned WorkGroupSize = MFI->getMaxFlatWorkGroupSize();
1042  unsigned WavefrontSize = ST.getWavefrontSize();
1043 
1044  unsigned TIDReg = MFI->getTIDReg();
1045  if (!MFI->hasCalculatedTID()) {
1046  MachineBasicBlock &Entry = MBB.getParent()->front();
1047  MachineBasicBlock::iterator Insert = Entry.front();
1048  DebugLoc DL = Insert->getDebugLoc();
1049 
1050  TIDReg = RI.findUnusedRegister(MF->getRegInfo(), &AMDGPU::VGPR_32RegClass,
1051  *MF);
1052  if (TIDReg == AMDGPU::NoRegister)
1053  return TIDReg;
1054 
1056  WorkGroupSize > WavefrontSize) {
1057  unsigned TIDIGXReg
1059  unsigned TIDIGYReg
1061  unsigned TIDIGZReg
1063  unsigned InputPtrReg =
1065  for (unsigned Reg : {TIDIGXReg, TIDIGYReg, TIDIGZReg}) {
1066  if (!Entry.isLiveIn(Reg))
1067  Entry.addLiveIn(Reg);
1068  }
1069 
1070  RS->enterBasicBlock(Entry);
1071  // FIXME: Can we scavenge an SReg_64 and access the subregs?
1072  unsigned STmp0 = RS->scavengeRegister(&AMDGPU::SGPR_32RegClass, 0);
1073  unsigned STmp1 = RS->scavengeRegister(&AMDGPU::SGPR_32RegClass, 0);
1074  BuildMI(Entry, Insert, DL, get(AMDGPU::S_LOAD_DWORD_IMM), STmp0)
1075  .addReg(InputPtrReg)
1077  BuildMI(Entry, Insert, DL, get(AMDGPU::S_LOAD_DWORD_IMM), STmp1)
1078  .addReg(InputPtrReg)
1080 
1081  // NGROUPS.X * NGROUPS.Y
1082  BuildMI(Entry, Insert, DL, get(AMDGPU::S_MUL_I32), STmp1)
1083  .addReg(STmp1)
1084  .addReg(STmp0);
1085  // (NGROUPS.X * NGROUPS.Y) * TIDIG.X
1086  BuildMI(Entry, Insert, DL, get(AMDGPU::V_MUL_U32_U24_e32), TIDReg)
1087  .addReg(STmp1)
1088  .addReg(TIDIGXReg);
1089  // NGROUPS.Z * TIDIG.Y + (NGROUPS.X * NGROPUS.Y * TIDIG.X)
1090  BuildMI(Entry, Insert, DL, get(AMDGPU::V_MAD_U32_U24), TIDReg)
1091  .addReg(STmp0)
1092  .addReg(TIDIGYReg)
1093  .addReg(TIDReg);
1094  // (NGROUPS.Z * TIDIG.Y + (NGROUPS.X * NGROPUS.Y * TIDIG.X)) + TIDIG.Z
1095  getAddNoCarry(Entry, Insert, DL, TIDReg)
1096  .addReg(TIDReg)
1097  .addReg(TIDIGZReg);
1098  } else {
1099  // Get the wave id
1100  BuildMI(Entry, Insert, DL, get(AMDGPU::V_MBCNT_LO_U32_B32_e64),
1101  TIDReg)
1102  .addImm(-1)
1103  .addImm(0);
1104 
1105  BuildMI(Entry, Insert, DL, get(AMDGPU::V_MBCNT_HI_U32_B32_e64),
1106  TIDReg)
1107  .addImm(-1)
1108  .addReg(TIDReg);
1109  }
1110 
1111  BuildMI(Entry, Insert, DL, get(AMDGPU::V_LSHLREV_B32_e32),
1112  TIDReg)
1113  .addImm(2)
1114  .addReg(TIDReg);
1115  MFI->setTIDReg(TIDReg);
1116  }
1117 
1118  // Add FrameIndex to LDS offset
1119  unsigned LDSOffset = MFI->getLDSSize() + (FrameOffset * WorkGroupSize);
1120  getAddNoCarry(MBB, MI, DL, TmpReg)
1121  .addImm(LDSOffset)
1122  .addReg(TIDReg);
1123 
1124  return TmpReg;
1125 }
1126 
1129  int Count) const {
1130  DebugLoc DL = MBB.findDebugLoc(MI);
1131  while (Count > 0) {
1132  int Arg;
1133  if (Count >= 8)
1134  Arg = 7;
1135  else
1136  Arg = Count - 1;
1137  Count -= 8;
1138  BuildMI(MBB, MI, DL, get(AMDGPU::S_NOP))
1139  .addImm(Arg);
1140  }
1141 }
1142 
1145  insertWaitStates(MBB, MI, 1);
1146 }
1147 
1149  auto MF = MBB.getParent();
1150  SIMachineFunctionInfo *Info = MF->getInfo<SIMachineFunctionInfo>();
1151 
1152  assert(Info->isEntryFunction());
1153 
1154  if (MBB.succ_empty()) {
1155  bool HasNoTerminator = MBB.getFirstTerminator() == MBB.end();
1156  if (HasNoTerminator)
1157  BuildMI(MBB, MBB.end(), DebugLoc(),
1158  get(Info->returnsVoid() ? AMDGPU::S_ENDPGM : AMDGPU::SI_RETURN_TO_EPILOG));
1159  }
1160 }
1161 
1163  switch (MI.getOpcode()) {
1164  default: return 1; // FIXME: Do wait states equal cycles?
1165 
1166  case AMDGPU::S_NOP:
1167  return MI.getOperand(0).getImm() + 1;
1168  }
1169 }
1170 
1172  MachineBasicBlock &MBB = *MI.getParent();
1173  DebugLoc DL = MBB.findDebugLoc(MI);
1174  switch (MI.getOpcode()) {
1175  default: return TargetInstrInfo::expandPostRAPseudo(MI);
1176  case AMDGPU::S_MOV_B64_term:
1177  // This is only a terminator to get the correct spill code placement during
1178  // register allocation.
1179  MI.setDesc(get(AMDGPU::S_MOV_B64));
1180  break;
1181 
1182  case AMDGPU::S_XOR_B64_term:
1183  // This is only a terminator to get the correct spill code placement during
1184  // register allocation.
1185  MI.setDesc(get(AMDGPU::S_XOR_B64));
1186  break;
1187 
1188  case AMDGPU::S_ANDN2_B64_term:
1189  // This is only a terminator to get the correct spill code placement during
1190  // register allocation.
1191  MI.setDesc(get(AMDGPU::S_ANDN2_B64));
1192  break;
1193 
1194  case AMDGPU::V_MOV_B64_PSEUDO: {
1195  unsigned Dst = MI.getOperand(0).getReg();
1196  unsigned DstLo = RI.getSubReg(Dst, AMDGPU::sub0);
1197  unsigned DstHi = RI.getSubReg(Dst, AMDGPU::sub1);
1198 
1199  const MachineOperand &SrcOp = MI.getOperand(1);
1200  // FIXME: Will this work for 64-bit floating point immediates?
1201  assert(!SrcOp.isFPImm());
1202  if (SrcOp.isImm()) {
1203  APInt Imm(64, SrcOp.getImm());
1204  BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstLo)
1205  .addImm(Imm.getLoBits(32).getZExtValue())
1206  .addReg(Dst, RegState::Implicit | RegState::Define);
1207  BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstHi)
1208  .addImm(Imm.getHiBits(32).getZExtValue())
1209  .addReg(Dst, RegState::Implicit | RegState::Define);
1210  } else {
1211  assert(SrcOp.isReg());
1212  BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstLo)
1213  .addReg(RI.getSubReg(SrcOp.getReg(), AMDGPU::sub0))
1215  BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstHi)
1216  .addReg(RI.getSubReg(SrcOp.getReg(), AMDGPU::sub1))
1218  }
1219  MI.eraseFromParent();
1220  break;
1221  }
1222  case AMDGPU::V_SET_INACTIVE_B32: {
1223  BuildMI(MBB, MI, DL, get(AMDGPU::S_NOT_B64), AMDGPU::EXEC)
1224  .addReg(AMDGPU::EXEC);
1225  BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), MI.getOperand(0).getReg())
1226  .add(MI.getOperand(2));
1227  BuildMI(MBB, MI, DL, get(AMDGPU::S_NOT_B64), AMDGPU::EXEC)
1228  .addReg(AMDGPU::EXEC);
1229  MI.eraseFromParent();
1230  break;
1231  }
1232  case AMDGPU::V_SET_INACTIVE_B64: {
1233  BuildMI(MBB, MI, DL, get(AMDGPU::S_NOT_B64), AMDGPU::EXEC)
1234  .addReg(AMDGPU::EXEC);
1235  MachineInstr *Copy = BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B64_PSEUDO),
1236  MI.getOperand(0).getReg())
1237  .add(MI.getOperand(2));
1238  expandPostRAPseudo(*Copy);
1239  BuildMI(MBB, MI, DL, get(AMDGPU::S_NOT_B64), AMDGPU::EXEC)
1240  .addReg(AMDGPU::EXEC);
1241  MI.eraseFromParent();
1242  break;
1243  }
1244  case AMDGPU::V_MOVRELD_B32_V1:
1245  case AMDGPU::V_MOVRELD_B32_V2:
1246  case AMDGPU::V_MOVRELD_B32_V4:
1247  case AMDGPU::V_MOVRELD_B32_V8:
1248  case AMDGPU::V_MOVRELD_B32_V16: {
1249  const MCInstrDesc &MovRelDesc = get(AMDGPU::V_MOVRELD_B32_e32);
1250  unsigned VecReg = MI.getOperand(0).getReg();
1251  bool IsUndef = MI.getOperand(1).isUndef();
1252  unsigned SubReg = AMDGPU::sub0 + MI.getOperand(3).getImm();
1253  assert(VecReg == MI.getOperand(1).getReg());
1254 
1255  MachineInstr *MovRel =
1256  BuildMI(MBB, MI, DL, MovRelDesc)
1257  .addReg(RI.getSubReg(VecReg, SubReg), RegState::Undef)
1258  .add(MI.getOperand(2))
1259  .addReg(VecReg, RegState::ImplicitDefine)
1260  .addReg(VecReg,
1261  RegState::Implicit | (IsUndef ? RegState::Undef : 0));
1262 
1263  const int ImpDefIdx =
1264  MovRelDesc.getNumOperands() + MovRelDesc.getNumImplicitUses();
1265  const int ImpUseIdx = ImpDefIdx + 1;
1266  MovRel->tieOperands(ImpDefIdx, ImpUseIdx);
1267 
1268  MI.eraseFromParent();
1269  break;
1270  }
1271  case AMDGPU::SI_PC_ADD_REL_OFFSET: {
1272  MachineFunction &MF = *MBB.getParent();
1273  unsigned Reg = MI.getOperand(0).getReg();
1274  unsigned RegLo = RI.getSubReg(Reg, AMDGPU::sub0);
1275  unsigned RegHi = RI.getSubReg(Reg, AMDGPU::sub1);
1276 
1277  // Create a bundle so these instructions won't be re-ordered by the
1278  // post-RA scheduler.
1279  MIBundleBuilder Bundler(MBB, MI);
1280  Bundler.append(BuildMI(MF, DL, get(AMDGPU::S_GETPC_B64), Reg));
1281 
1282  // Add 32-bit offset from this instruction to the start of the
1283  // constant data.
1284  Bundler.append(BuildMI(MF, DL, get(AMDGPU::S_ADD_U32), RegLo)
1285  .addReg(RegLo)
1286  .add(MI.getOperand(1)));
1287 
1288  MachineInstrBuilder MIB = BuildMI(MF, DL, get(AMDGPU::S_ADDC_U32), RegHi)
1289  .addReg(RegHi);
1291  MIB.addImm(0);
1292  else
1293  MIB.add(MI.getOperand(2));
1294 
1295  Bundler.append(MIB);
1296  finalizeBundle(MBB, Bundler.begin());
1297 
1298  MI.eraseFromParent();
1299  break;
1300  }
1301  case AMDGPU::EXIT_WWM: {
1302  // This only gets its own opcode so that SIFixWWMLiveness can tell when WWM
1303  // is exited.
1304  MI.setDesc(get(AMDGPU::S_MOV_B64));
1305  break;
1306  }
1307  case TargetOpcode::BUNDLE: {
1308  if (!MI.mayLoad())
1309  return false;
1310 
1311  // If it is a load it must be a memory clause
1313  I->isBundledWithSucc(); ++I) {
1314  I->unbundleFromSucc();
1315  for (MachineOperand &MO : I->operands())
1316  if (MO.isReg())
1317  MO.setIsInternalRead(false);
1318  }
1319 
1320  MI.eraseFromParent();
1321  break;
1322  }
1323  }
1324  return true;
1325 }
1326 
1328  MachineOperand &Src0,
1329  unsigned Src0OpName,
1330  MachineOperand &Src1,
1331  unsigned Src1OpName) const {
1332  MachineOperand *Src0Mods = getNamedOperand(MI, Src0OpName);
1333  if (!Src0Mods)
1334  return false;
1335 
1336  MachineOperand *Src1Mods = getNamedOperand(MI, Src1OpName);
1337  assert(Src1Mods &&
1338  "All commutable instructions have both src0 and src1 modifiers");
1339 
1340  int Src0ModsVal = Src0Mods->getImm();
1341  int Src1ModsVal = Src1Mods->getImm();
1342 
1343  Src1Mods->setImm(Src0ModsVal);
1344  Src0Mods->setImm(Src1ModsVal);
1345  return true;
1346 }
1347 
1349  MachineOperand &RegOp,
1350  MachineOperand &NonRegOp) {
1351  unsigned Reg = RegOp.getReg();
1352  unsigned SubReg = RegOp.getSubReg();
1353  bool IsKill = RegOp.isKill();
1354  bool IsDead = RegOp.isDead();
1355  bool IsUndef = RegOp.isUndef();
1356  bool IsDebug = RegOp.isDebug();
1357 
1358  if (NonRegOp.isImm())
1359  RegOp.ChangeToImmediate(NonRegOp.getImm());
1360  else if (NonRegOp.isFI())
1361  RegOp.ChangeToFrameIndex(NonRegOp.getIndex());
1362  else
1363  return nullptr;
1364 
1365  NonRegOp.ChangeToRegister(Reg, false, false, IsKill, IsDead, IsUndef, IsDebug);
1366  NonRegOp.setSubReg(SubReg);
1367 
1368  return &MI;
1369 }
1370 
1372  unsigned Src0Idx,
1373  unsigned Src1Idx) const {
1374  assert(!NewMI && "this should never be used");
1375 
1376  unsigned Opc = MI.getOpcode();
1377  int CommutedOpcode = commuteOpcode(Opc);
1378  if (CommutedOpcode == -1)
1379  return nullptr;
1380 
1381  assert(AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0) ==
1382  static_cast<int>(Src0Idx) &&
1383  AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1) ==
1384  static_cast<int>(Src1Idx) &&
1385  "inconsistency with findCommutedOpIndices");
1386 
1387  MachineOperand &Src0 = MI.getOperand(Src0Idx);
1388  MachineOperand &Src1 = MI.getOperand(Src1Idx);
1389 
1390  MachineInstr *CommutedMI = nullptr;
1391  if (Src0.isReg() && Src1.isReg()) {
1392  if (isOperandLegal(MI, Src1Idx, &Src0)) {
1393  // Be sure to copy the source modifiers to the right place.
1394  CommutedMI
1395  = TargetInstrInfo::commuteInstructionImpl(MI, NewMI, Src0Idx, Src1Idx);
1396  }
1397 
1398  } else if (Src0.isReg() && !Src1.isReg()) {
1399  // src0 should always be able to support any operand type, so no need to
1400  // check operand legality.
1401  CommutedMI = swapRegAndNonRegOperand(MI, Src0, Src1);
1402  } else if (!Src0.isReg() && Src1.isReg()) {
1403  if (isOperandLegal(MI, Src1Idx, &Src0))
1404  CommutedMI = swapRegAndNonRegOperand(MI, Src1, Src0);
1405  } else {
1406  // FIXME: Found two non registers to commute. This does happen.
1407  return nullptr;
1408  }
1409 
1410  if (CommutedMI) {
1411  swapSourceModifiers(MI, Src0, AMDGPU::OpName::src0_modifiers,
1412  Src1, AMDGPU::OpName::src1_modifiers);
1413 
1414  CommutedMI->setDesc(get(CommutedOpcode));
1415  }
1416 
1417  return CommutedMI;
1418 }
1419 
1420 // This needs to be implemented because the source modifiers may be inserted
1421 // between the true commutable operands, and the base
1422 // TargetInstrInfo::commuteInstruction uses it.
1424  unsigned &SrcOpIdx1) const {
1425  return findCommutedOpIndices(MI.getDesc(), SrcOpIdx0, SrcOpIdx1);
1426 }
1427 
1428 bool SIInstrInfo::findCommutedOpIndices(MCInstrDesc Desc, unsigned &SrcOpIdx0,
1429  unsigned &SrcOpIdx1) const {
1430  if (!Desc.isCommutable())
1431  return false;
1432 
1433  unsigned Opc = Desc.getOpcode();
1434  int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0);
1435  if (Src0Idx == -1)
1436  return false;
1437 
1438  int Src1Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1);
1439  if (Src1Idx == -1)
1440  return false;
1441 
1442  return fixCommutedOpIndices(SrcOpIdx0, SrcOpIdx1, Src0Idx, Src1Idx);
1443 }
1444 
1445 bool SIInstrInfo::isBranchOffsetInRange(unsigned BranchOp,
1446  int64_t BrOffset) const {
1447  // BranchRelaxation should never have to check s_setpc_b64 because its dest
1448  // block is unanalyzable.
1449  assert(BranchOp != AMDGPU::S_SETPC_B64);
1450 
1451  // Convert to dwords.
1452  BrOffset /= 4;
1453 
1454  // The branch instructions do PC += signext(SIMM16 * 4) + 4, so the offset is
1455  // from the next instruction.
1456  BrOffset -= 1;
1457 
1458  return isIntN(BranchOffsetBits, BrOffset);
1459 }
1460 
1462  const MachineInstr &MI) const {
1463  if (MI.getOpcode() == AMDGPU::S_SETPC_B64) {
1464  // This would be a difficult analysis to perform, but can always be legal so
1465  // there's no need to analyze it.
1466  return nullptr;
1467  }
1468 
1469  return MI.getOperand(0).getMBB();
1470 }
1471 
1473  MachineBasicBlock &DestBB,
1474  const DebugLoc &DL,
1475  int64_t BrOffset,
1476  RegScavenger *RS) const {
1477  assert(RS && "RegScavenger required for long branching");
1478  assert(MBB.empty() &&
1479  "new block should be inserted for expanding unconditional branch");
1480  assert(MBB.pred_size() == 1);
1481 
1482  MachineFunction *MF = MBB.getParent();
1483  MachineRegisterInfo &MRI = MF->getRegInfo();
1484 
1485  // FIXME: Virtual register workaround for RegScavenger not working with empty
1486  // blocks.
1487  unsigned PCReg = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
1488 
1489  auto I = MBB.end();
1490 
1491  // We need to compute the offset relative to the instruction immediately after
1492  // s_getpc_b64. Insert pc arithmetic code before last terminator.
1493  MachineInstr *GetPC = BuildMI(MBB, I, DL, get(AMDGPU::S_GETPC_B64), PCReg);
1494 
1495  // TODO: Handle > 32-bit block address.
1496  if (BrOffset >= 0) {
1497  BuildMI(MBB, I, DL, get(AMDGPU::S_ADD_U32))
1498  .addReg(PCReg, RegState::Define, AMDGPU::sub0)
1499  .addReg(PCReg, 0, AMDGPU::sub0)
1501  BuildMI(MBB, I, DL, get(AMDGPU::S_ADDC_U32))
1502  .addReg(PCReg, RegState::Define, AMDGPU::sub1)
1503  .addReg(PCReg, 0, AMDGPU::sub1)
1504  .addImm(0);
1505  } else {
1506  // Backwards branch.
1507  BuildMI(MBB, I, DL, get(AMDGPU::S_SUB_U32))
1508  .addReg(PCReg, RegState::Define, AMDGPU::sub0)
1509  .addReg(PCReg, 0, AMDGPU::sub0)
1511  BuildMI(MBB, I, DL, get(AMDGPU::S_SUBB_U32))
1512  .addReg(PCReg, RegState::Define, AMDGPU::sub1)
1513  .addReg(PCReg, 0, AMDGPU::sub1)
1514  .addImm(0);
1515  }
1516 
1517  // Insert the indirect branch after the other terminator.
1518  BuildMI(&MBB, DL, get(AMDGPU::S_SETPC_B64))
1519  .addReg(PCReg);
1520 
1521  // FIXME: If spilling is necessary, this will fail because this scavenger has
1522  // no emergency stack slots. It is non-trivial to spill in this situation,
1523  // because the restore code needs to be specially placed after the
1524  // jump. BranchRelaxation then needs to be made aware of the newly inserted
1525  // block.
1526  //
1527  // If a spill is needed for the pc register pair, we need to insert a spill
1528  // restore block right before the destination block, and insert a short branch
1529  // into the old destination block's fallthrough predecessor.
1530  // e.g.:
1531  //
1532  // s_cbranch_scc0 skip_long_branch:
1533  //
1534  // long_branch_bb:
1535  // spill s[8:9]
1536  // s_getpc_b64 s[8:9]
1537  // s_add_u32 s8, s8, restore_bb
1538  // s_addc_u32 s9, s9, 0
1539  // s_setpc_b64 s[8:9]
1540  //
1541  // skip_long_branch:
1542  // foo;
1543  //
1544  // .....
1545  //
1546  // dest_bb_fallthrough_predecessor:
1547  // bar;
1548  // s_branch dest_bb
1549  //
1550  // restore_bb:
1551  // restore s[8:9]
1552  // fallthrough dest_bb
1553  ///
1554  // dest_bb:
1555  // buzz;
1556 
1557  RS->enterBasicBlockEnd(MBB);
1558  unsigned Scav = RS->scavengeRegister(&AMDGPU::SReg_64RegClass,
1559  MachineBasicBlock::iterator(GetPC), 0);
1560  MRI.replaceRegWith(PCReg, Scav);
1561  MRI.clearVirtRegs();
1562  RS->setRegUsed(Scav);
1563 
1564  return 4 + 8 + 4 + 4;
1565 }
1566 
1567 unsigned SIInstrInfo::getBranchOpcode(SIInstrInfo::BranchPredicate Cond) {
1568  switch (Cond) {
1569  case SIInstrInfo::SCC_TRUE:
1570  return AMDGPU::S_CBRANCH_SCC1;
1571  case SIInstrInfo::SCC_FALSE:
1572  return AMDGPU::S_CBRANCH_SCC0;
1573  case SIInstrInfo::VCCNZ:
1574  return AMDGPU::S_CBRANCH_VCCNZ;
1575  case SIInstrInfo::VCCZ:
1576  return AMDGPU::S_CBRANCH_VCCZ;
1577  case SIInstrInfo::EXECNZ:
1578  return AMDGPU::S_CBRANCH_EXECNZ;
1579  case SIInstrInfo::EXECZ:
1580  return AMDGPU::S_CBRANCH_EXECZ;
1581  default:
1582  llvm_unreachable("invalid branch predicate");
1583  }
1584 }
1585 
1586 SIInstrInfo::BranchPredicate SIInstrInfo::getBranchPredicate(unsigned Opcode) {
1587  switch (Opcode) {
1588  case AMDGPU::S_CBRANCH_SCC0:
1589  return SCC_FALSE;
1590  case AMDGPU::S_CBRANCH_SCC1:
1591  return SCC_TRUE;
1592  case AMDGPU::S_CBRANCH_VCCNZ:
1593  return VCCNZ;
1594  case AMDGPU::S_CBRANCH_VCCZ:
1595  return VCCZ;
1596  case AMDGPU::S_CBRANCH_EXECNZ:
1597  return EXECNZ;
1598  case AMDGPU::S_CBRANCH_EXECZ:
1599  return EXECZ;
1600  default:
1601  return INVALID_BR;
1602  }
1603 }
1604 
1607  MachineBasicBlock *&TBB,
1608  MachineBasicBlock *&FBB,
1610  bool AllowModify) const {
1611  if (I->getOpcode() == AMDGPU::S_BRANCH) {
1612  // Unconditional Branch
1613  TBB = I->getOperand(0).getMBB();
1614  return false;
1615  }
1616 
1617  MachineBasicBlock *CondBB = nullptr;
1618 
1619  if (I->getOpcode() == AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO) {
1620  CondBB = I->getOperand(1).getMBB();
1621  Cond.push_back(I->getOperand(0));
1622  } else {
1623  BranchPredicate Pred = getBranchPredicate(I->getOpcode());
1624  if (Pred == INVALID_BR)
1625  return true;
1626 
1627  CondBB = I->getOperand(0).getMBB();
1629  Cond.push_back(I->getOperand(1)); // Save the branch register.
1630  }
1631  ++I;
1632 
1633  if (I == MBB.end()) {
1634  // Conditional branch followed by fall-through.
1635  TBB = CondBB;
1636  return false;
1637  }
1638 
1639  if (I->getOpcode() == AMDGPU::S_BRANCH) {
1640  TBB = CondBB;
1641  FBB = I->getOperand(0).getMBB();
1642  return false;
1643  }
1644 
1645  return true;
1646 }
1647 
1649  MachineBasicBlock *&FBB,
1651  bool AllowModify) const {
1653  if (I == MBB.end())
1654  return false;
1655 
1656  if (I->getOpcode() != AMDGPU::SI_MASK_BRANCH)
1657  return analyzeBranchImpl(MBB, I, TBB, FBB, Cond, AllowModify);
1658 
1659  ++I;
1660 
1661  // TODO: Should be able to treat as fallthrough?
1662  if (I == MBB.end())
1663  return true;
1664 
1665  if (analyzeBranchImpl(MBB, I, TBB, FBB, Cond, AllowModify))
1666  return true;
1667 
1668  MachineBasicBlock *MaskBrDest = I->getOperand(0).getMBB();
1669 
1670  // Specifically handle the case where the conditional branch is to the same
1671  // destination as the mask branch. e.g.
1672  //
1673  // si_mask_branch BB8
1674  // s_cbranch_execz BB8
1675  // s_cbranch BB9
1676  //
1677  // This is required to understand divergent loops which may need the branches
1678  // to be relaxed.
1679  if (TBB != MaskBrDest || Cond.empty())
1680  return true;
1681 
1682  auto Pred = Cond[0].getImm();
1683  return (Pred != EXECZ && Pred != EXECNZ);
1684 }
1685 
1687  int *BytesRemoved) const {
1689 
1690  unsigned Count = 0;
1691  unsigned RemovedSize = 0;
1692  while (I != MBB.end()) {
1693  MachineBasicBlock::iterator Next = std::next(I);
1694  if (I->getOpcode() == AMDGPU::SI_MASK_BRANCH) {
1695  I = Next;
1696  continue;
1697  }
1698 
1699  RemovedSize += getInstSizeInBytes(*I);
1700  I->eraseFromParent();
1701  ++Count;
1702  I = Next;
1703  }
1704 
1705  if (BytesRemoved)
1706  *BytesRemoved = RemovedSize;
1707 
1708  return Count;
1709 }
1710 
1711 // Copy the flags onto the implicit condition register operand.
1713  const MachineOperand &OrigCond) {
1714  CondReg.setIsUndef(OrigCond.isUndef());
1715  CondReg.setIsKill(OrigCond.isKill());
1716 }
1717 
1719  MachineBasicBlock *TBB,
1720  MachineBasicBlock *FBB,
1722  const DebugLoc &DL,
1723  int *BytesAdded) const {
1724  if (!FBB && Cond.empty()) {
1725  BuildMI(&MBB, DL, get(AMDGPU::S_BRANCH))
1726  .addMBB(TBB);
1727  if (BytesAdded)
1728  *BytesAdded = 4;
1729  return 1;
1730  }
1731 
1732  if(Cond.size() == 1 && Cond[0].isReg()) {
1733  BuildMI(&MBB, DL, get(AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO))
1734  .add(Cond[0])
1735  .addMBB(TBB);
1736  return 1;
1737  }
1738 
1739  assert(TBB && Cond[0].isImm());
1740 
1741  unsigned Opcode
1742  = getBranchOpcode(static_cast<BranchPredicate>(Cond[0].getImm()));
1743 
1744  if (!FBB) {
1745  Cond[1].isUndef();
1746  MachineInstr *CondBr =
1747  BuildMI(&MBB, DL, get(Opcode))
1748  .addMBB(TBB);
1749 
1750  // Copy the flags onto the implicit condition register operand.
1751  preserveCondRegFlags(CondBr->getOperand(1), Cond[1]);
1752 
1753  if (BytesAdded)
1754  *BytesAdded = 4;
1755  return 1;
1756  }
1757 
1758  assert(TBB && FBB);
1759 
1760  MachineInstr *CondBr =
1761  BuildMI(&MBB, DL, get(Opcode))
1762  .addMBB(TBB);
1763  BuildMI(&MBB, DL, get(AMDGPU::S_BRANCH))
1764  .addMBB(FBB);
1765 
1766  MachineOperand &CondReg = CondBr->getOperand(1);
1767  CondReg.setIsUndef(Cond[1].isUndef());
1768  CondReg.setIsKill(Cond[1].isKill());
1769 
1770  if (BytesAdded)
1771  *BytesAdded = 8;
1772 
1773  return 2;
1774 }
1775 
1777  SmallVectorImpl<MachineOperand> &Cond) const {
1778  if (Cond.size() != 2) {
1779  return true;
1780  }
1781 
1782  if (Cond[0].isImm()) {
1783  Cond[0].setImm(-Cond[0].getImm());
1784  return false;
1785  }
1786 
1787  return true;
1788 }
1789 
1792  unsigned TrueReg, unsigned FalseReg,
1793  int &CondCycles,
1794  int &TrueCycles, int &FalseCycles) const {
1795  switch (Cond[0].getImm()) {
1796  case VCCNZ:
1797  case VCCZ: {
1798  const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
1799  const TargetRegisterClass *RC = MRI.getRegClass(TrueReg);
1800  assert(MRI.getRegClass(FalseReg) == RC);
1801 
1802  int NumInsts = AMDGPU::getRegBitWidth(RC->getID()) / 32;
1803  CondCycles = TrueCycles = FalseCycles = NumInsts; // ???
1804 
1805  // Limit to equal cost for branch vs. N v_cndmask_b32s.
1806  return !RI.isSGPRClass(RC) && NumInsts <= 6;
1807  }
1808  case SCC_TRUE:
1809  case SCC_FALSE: {
1810  // FIXME: We could insert for VGPRs if we could replace the original compare
1811  // with a vector one.
1812  const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
1813  const TargetRegisterClass *RC = MRI.getRegClass(TrueReg);
1814  assert(MRI.getRegClass(FalseReg) == RC);
1815 
1816  int NumInsts = AMDGPU::getRegBitWidth(RC->getID()) / 32;
1817 
1818  // Multiples of 8 can do s_cselect_b64
1819  if (NumInsts % 2 == 0)
1820  NumInsts /= 2;
1821 
1822  CondCycles = TrueCycles = FalseCycles = NumInsts; // ???
1823  return RI.isSGPRClass(RC);
1824  }
1825  default:
1826  return false;
1827  }
1828 }
1829 
1832  unsigned DstReg, ArrayRef<MachineOperand> Cond,
1833  unsigned TrueReg, unsigned FalseReg) const {
1834  BranchPredicate Pred = static_cast<BranchPredicate>(Cond[0].getImm());
1835  if (Pred == VCCZ || Pred == SCC_FALSE) {
1836  Pred = static_cast<BranchPredicate>(-Pred);
1837  std::swap(TrueReg, FalseReg);
1838  }
1839 
1841  const TargetRegisterClass *DstRC = MRI.getRegClass(DstReg);
1842  unsigned DstSize = RI.getRegSizeInBits(*DstRC);
1843 
1844  if (DstSize == 32) {
1845  unsigned SelOp = Pred == SCC_TRUE ?
1846  AMDGPU::S_CSELECT_B32 : AMDGPU::V_CNDMASK_B32_e32;
1847 
1848  // Instruction's operands are backwards from what is expected.
1849  MachineInstr *Select =
1850  BuildMI(MBB, I, DL, get(SelOp), DstReg)
1851  .addReg(FalseReg)
1852  .addReg(TrueReg);
1853 
1854  preserveCondRegFlags(Select->getOperand(3), Cond[1]);
1855  return;
1856  }
1857 
1858  if (DstSize == 64 && Pred == SCC_TRUE) {
1859  MachineInstr *Select =
1860  BuildMI(MBB, I, DL, get(AMDGPU::S_CSELECT_B64), DstReg)
1861  .addReg(FalseReg)
1862  .addReg(TrueReg);
1863 
1864  preserveCondRegFlags(Select->getOperand(3), Cond[1]);
1865  return;
1866  }
1867 
1868  static const int16_t Sub0_15[] = {
1869  AMDGPU::sub0, AMDGPU::sub1, AMDGPU::sub2, AMDGPU::sub3,
1870  AMDGPU::sub4, AMDGPU::sub5, AMDGPU::sub6, AMDGPU::sub7,
1871  AMDGPU::sub8, AMDGPU::sub9, AMDGPU::sub10, AMDGPU::sub11,
1872  AMDGPU::sub12, AMDGPU::sub13, AMDGPU::sub14, AMDGPU::sub15,
1873  };
1874 
1875  static const int16_t Sub0_15_64[] = {
1876  AMDGPU::sub0_sub1, AMDGPU::sub2_sub3,
1877  AMDGPU::sub4_sub5, AMDGPU::sub6_sub7,
1878  AMDGPU::sub8_sub9, AMDGPU::sub10_sub11,
1879  AMDGPU::sub12_sub13, AMDGPU::sub14_sub15,
1880  };
1881 
1882  unsigned SelOp = AMDGPU::V_CNDMASK_B32_e32;
1883  const TargetRegisterClass *EltRC = &AMDGPU::VGPR_32RegClass;
1884  const int16_t *SubIndices = Sub0_15;
1885  int NElts = DstSize / 32;
1886 
1887  // 64-bit select is only avaialble for SALU.
1888  if (Pred == SCC_TRUE) {
1889  SelOp = AMDGPU::S_CSELECT_B64;
1890  EltRC = &AMDGPU::SGPR_64RegClass;
1891  SubIndices = Sub0_15_64;
1892 
1893  assert(NElts % 2 == 0);
1894  NElts /= 2;
1895  }
1896 
1898  MBB, I, DL, get(AMDGPU::REG_SEQUENCE), DstReg);
1899 
1900  I = MIB->getIterator();
1901 
1903  for (int Idx = 0; Idx != NElts; ++Idx) {
1904  unsigned DstElt = MRI.createVirtualRegister(EltRC);
1905  Regs.push_back(DstElt);
1906 
1907  unsigned SubIdx = SubIndices[Idx];
1908 
1909  MachineInstr *Select =
1910  BuildMI(MBB, I, DL, get(SelOp), DstElt)
1911  .addReg(FalseReg, 0, SubIdx)
1912  .addReg(TrueReg, 0, SubIdx);
1913  preserveCondRegFlags(Select->getOperand(3), Cond[1]);
1914 
1915  MIB.addReg(DstElt)
1916  .addImm(SubIdx);
1917  }
1918 }
1919 
1921  switch (MI.getOpcode()) {
1922  case AMDGPU::V_MOV_B32_e32:
1923  case AMDGPU::V_MOV_B32_e64:
1924  case AMDGPU::V_MOV_B64_PSEUDO: {
1925  // If there are additional implicit register operands, this may be used for
1926  // register indexing so the source register operand isn't simply copied.
1927  unsigned NumOps = MI.getDesc().getNumOperands() +
1928  MI.getDesc().getNumImplicitUses();
1929 
1930  return MI.getNumOperands() == NumOps;
1931  }
1932  case AMDGPU::S_MOV_B32:
1933  case AMDGPU::S_MOV_B64:
1934  case AMDGPU::COPY:
1935  return true;
1936  default:
1937  return false;
1938  }
1939 }
1940 
1942  unsigned Kind) const {
1943  switch(Kind) {
1954  }
1955  return AMDGPUAS::FLAT_ADDRESS;
1956 }
1957 
1959  unsigned Opc = MI.getOpcode();
1960  int Src0ModIdx = AMDGPU::getNamedOperandIdx(Opc,
1961  AMDGPU::OpName::src0_modifiers);
1962  int Src1ModIdx = AMDGPU::getNamedOperandIdx(Opc,
1963  AMDGPU::OpName::src1_modifiers);
1964  int Src2ModIdx = AMDGPU::getNamedOperandIdx(Opc,
1965  AMDGPU::OpName::src2_modifiers);
1966 
1967  MI.RemoveOperand(Src2ModIdx);
1968  MI.RemoveOperand(Src1ModIdx);
1969  MI.RemoveOperand(Src0ModIdx);
1970 }
1971 
1973  unsigned Reg, MachineRegisterInfo *MRI) const {
1974  if (!MRI->hasOneNonDBGUse(Reg))
1975  return false;
1976 
1977  switch (DefMI.getOpcode()) {
1978  default:
1979  return false;
1980  case AMDGPU::S_MOV_B64:
1981  // TODO: We could fold 64-bit immediates, but this get compilicated
1982  // when there are sub-registers.
1983  return false;
1984 
1985  case AMDGPU::V_MOV_B32_e32:
1986  case AMDGPU::S_MOV_B32:
1987  break;
1988  }
1989 
1990  const MachineOperand *ImmOp = getNamedOperand(DefMI, AMDGPU::OpName::src0);
1991  assert(ImmOp);
1992  // FIXME: We could handle FrameIndex values here.
1993  if (!ImmOp->isImm())
1994  return false;
1995 
1996  unsigned Opc = UseMI.getOpcode();
1997  if (Opc == AMDGPU::COPY) {
1998  bool isVGPRCopy = RI.isVGPR(*MRI, UseMI.getOperand(0).getReg());
1999  unsigned NewOpc = isVGPRCopy ? AMDGPU::V_MOV_B32_e32 : AMDGPU::S_MOV_B32;
2000  UseMI.setDesc(get(NewOpc));
2001  UseMI.getOperand(1).ChangeToImmediate(ImmOp->getImm());
2002  UseMI.addImplicitDefUseOperands(*UseMI.getParent()->getParent());
2003  return true;
2004  }
2005 
2006  if (Opc == AMDGPU::V_MAD_F32 || Opc == AMDGPU::V_MAC_F32_e64 ||
2007  Opc == AMDGPU::V_MAD_F16 || Opc == AMDGPU::V_MAC_F16_e64) {
2008  // Don't fold if we are using source or output modifiers. The new VOP2
2009  // instructions don't have them.
2010  if (hasAnyModifiersSet(UseMI))
2011  return false;
2012 
2013  // If this is a free constant, there's no reason to do this.
2014  // TODO: We could fold this here instead of letting SIFoldOperands do it
2015  // later.
2016  MachineOperand *Src0 = getNamedOperand(UseMI, AMDGPU::OpName::src0);
2017 
2018  // Any src operand can be used for the legality check.
2019  if (isInlineConstant(UseMI, *Src0, *ImmOp))
2020  return false;
2021 
2022  bool IsF32 = Opc == AMDGPU::V_MAD_F32 || Opc == AMDGPU::V_MAC_F32_e64;
2023  MachineOperand *Src1 = getNamedOperand(UseMI, AMDGPU::OpName::src1);
2024  MachineOperand *Src2 = getNamedOperand(UseMI, AMDGPU::OpName::src2);
2025 
2026  // Multiplied part is the constant: Use v_madmk_{f16, f32}.
2027  // We should only expect these to be on src0 due to canonicalizations.
2028  if (Src0->isReg() && Src0->getReg() == Reg) {
2029  if (!Src1->isReg() || RI.isSGPRClass(MRI->getRegClass(Src1->getReg())))
2030  return false;
2031 
2032  if (!Src2->isReg() || RI.isSGPRClass(MRI->getRegClass(Src2->getReg())))
2033  return false;
2034 
2035  // We need to swap operands 0 and 1 since madmk constant is at operand 1.
2036 
2037  const int64_t Imm = ImmOp->getImm();
2038 
2039  // FIXME: This would be a lot easier if we could return a new instruction
2040  // instead of having to modify in place.
2041 
2042  // Remove these first since they are at the end.
2043  UseMI.RemoveOperand(
2044  AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::omod));
2045  UseMI.RemoveOperand(
2046  AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::clamp));
2047 
2048  unsigned Src1Reg = Src1->getReg();
2049  unsigned Src1SubReg = Src1->getSubReg();
2050  Src0->setReg(Src1Reg);
2051  Src0->setSubReg(Src1SubReg);
2052  Src0->setIsKill(Src1->isKill());
2053 
2054  if (Opc == AMDGPU::V_MAC_F32_e64 ||
2055  Opc == AMDGPU::V_MAC_F16_e64)
2056  UseMI.untieRegOperand(
2057  AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2));
2058 
2059  Src1->ChangeToImmediate(Imm);
2060 
2061  removeModOperands(UseMI);
2062  UseMI.setDesc(get(IsF32 ? AMDGPU::V_MADMK_F32 : AMDGPU::V_MADMK_F16));
2063 
2064  bool DeleteDef = MRI->hasOneNonDBGUse(Reg);
2065  if (DeleteDef)
2066  DefMI.eraseFromParent();
2067 
2068  return true;
2069  }
2070 
2071  // Added part is the constant: Use v_madak_{f16, f32}.
2072  if (Src2->isReg() && Src2->getReg() == Reg) {
2073  // Not allowed to use constant bus for another operand.
2074  // We can however allow an inline immediate as src0.
2075  bool Src0Inlined = false;
2076  if (Src0->isReg()) {
2077  // Try to inline constant if possible.
2078  // If the Def moves immediate and the use is single
2079  // We are saving VGPR here.
2080  MachineInstr *Def = MRI->getUniqueVRegDef(Src0->getReg());
2081  if (Def && Def->isMoveImmediate() &&
2082  isInlineConstant(Def->getOperand(1)) &&
2083  MRI->hasOneUse(Src0->getReg())) {
2084  Src0->ChangeToImmediate(Def->getOperand(1).getImm());
2085  Src0Inlined = true;
2086  } else if ((RI.isPhysicalRegister(Src0->getReg()) &&
2087  RI.isSGPRClass(RI.getPhysRegClass(Src0->getReg()))) ||
2088  (RI.isVirtualRegister(Src0->getReg()) &&
2089  RI.isSGPRClass(MRI->getRegClass(Src0->getReg()))))
2090  return false;
2091  // VGPR is okay as Src0 - fallthrough
2092  }
2093 
2094  if (Src1->isReg() && !Src0Inlined ) {
2095  // We have one slot for inlinable constant so far - try to fill it
2096  MachineInstr *Def = MRI->getUniqueVRegDef(Src1->getReg());
2097  if (Def && Def->isMoveImmediate() &&
2098  isInlineConstant(Def->getOperand(1)) &&
2099  MRI->hasOneUse(Src1->getReg()) &&
2100  commuteInstruction(UseMI)) {
2101  Src0->ChangeToImmediate(Def->getOperand(1).getImm());
2102  } else if ((RI.isPhysicalRegister(Src1->getReg()) &&
2103  RI.isSGPRClass(RI.getPhysRegClass(Src1->getReg()))) ||
2104  (RI.isVirtualRegister(Src1->getReg()) &&
2105  RI.isSGPRClass(MRI->getRegClass(Src1->getReg()))))
2106  return false;
2107  // VGPR is okay as Src1 - fallthrough
2108  }
2109 
2110  const int64_t Imm = ImmOp->getImm();
2111 
2112  // FIXME: This would be a lot easier if we could return a new instruction
2113  // instead of having to modify in place.
2114 
2115  // Remove these first since they are at the end.
2116  UseMI.RemoveOperand(
2117  AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::omod));
2118  UseMI.RemoveOperand(
2119  AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::clamp));
2120 
2121  if (Opc == AMDGPU::V_MAC_F32_e64 ||
2122  Opc == AMDGPU::V_MAC_F16_e64)
2123  UseMI.untieRegOperand(
2124  AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2));
2125 
2126  // ChangingToImmediate adds Src2 back to the instruction.
2127  Src2->ChangeToImmediate(Imm);
2128 
2129  // These come before src2.
2130  removeModOperands(UseMI);
2131  UseMI.setDesc(get(IsF32 ? AMDGPU::V_MADAK_F32 : AMDGPU::V_MADAK_F16));
2132 
2133  bool DeleteDef = MRI->hasOneNonDBGUse(Reg);
2134  if (DeleteDef)
2135  DefMI.eraseFromParent();
2136 
2137  return true;
2138  }
2139  }
2140 
2141  return false;
2142 }
2143 
2144 static bool offsetsDoNotOverlap(int WidthA, int OffsetA,
2145  int WidthB, int OffsetB) {
2146  int LowOffset = OffsetA < OffsetB ? OffsetA : OffsetB;
2147  int HighOffset = OffsetA < OffsetB ? OffsetB : OffsetA;
2148  int LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB;
2149  return LowOffset + LowWidth <= HighOffset;
2150 }
2151 
2152 bool SIInstrInfo::checkInstOffsetsDoNotOverlap(MachineInstr &MIa,
2153  MachineInstr &MIb) const {
2154  unsigned BaseReg0, BaseReg1;
2155  int64_t Offset0, Offset1;
2156 
2157  if (getMemOpBaseRegImmOfs(MIa, BaseReg0, Offset0, &RI) &&
2158  getMemOpBaseRegImmOfs(MIb, BaseReg1, Offset1, &RI)) {
2159 
2160  if (!MIa.hasOneMemOperand() || !MIb.hasOneMemOperand()) {
2161  // FIXME: Handle ds_read2 / ds_write2.
2162  return false;
2163  }
2164  unsigned Width0 = (*MIa.memoperands_begin())->getSize();
2165  unsigned Width1 = (*MIb.memoperands_begin())->getSize();
2166  if (BaseReg0 == BaseReg1 &&
2167  offsetsDoNotOverlap(Width0, Offset0, Width1, Offset1)) {
2168  return true;
2169  }
2170  }
2171 
2172  return false;
2173 }
2174 
2176  MachineInstr &MIb,
2177  AliasAnalysis *AA) const {
2178  assert((MIa.mayLoad() || MIa.mayStore()) &&
2179  "MIa must load from or modify a memory location");
2180  assert((MIb.mayLoad() || MIb.mayStore()) &&
2181  "MIb must load from or modify a memory location");
2182 
2184  return false;
2185 
2186  // XXX - Can we relax this between address spaces?
2187  if (MIa.hasOrderedMemoryRef() || MIb.hasOrderedMemoryRef())
2188  return false;
2189 
2190  if (AA && MIa.hasOneMemOperand() && MIb.hasOneMemOperand()) {
2191  const MachineMemOperand *MMOa = *MIa.memoperands_begin();
2192  const MachineMemOperand *MMOb = *MIb.memoperands_begin();
2193  if (MMOa->getValue() && MMOb->getValue()) {
2194  MemoryLocation LocA(MMOa->getValue(), MMOa->getSize(), MMOa->getAAInfo());
2195  MemoryLocation LocB(MMOb->getValue(), MMOb->getSize(), MMOb->getAAInfo());
2196  if (!AA->alias(LocA, LocB))
2197  return true;
2198  }
2199  }
2200 
2201  // TODO: Should we check the address space from the MachineMemOperand? That
2202  // would allow us to distinguish objects we know don't alias based on the
2203  // underlying address space, even if it was lowered to a different one,
2204  // e.g. private accesses lowered to use MUBUF instructions on a scratch
2205  // buffer.
2206  if (isDS(MIa)) {
2207  if (isDS(MIb))
2208  return checkInstOffsetsDoNotOverlap(MIa, MIb);
2209 
2210  return !isFLAT(MIb) || isSegmentSpecificFLAT(MIb);
2211  }
2212 
2213  if (isMUBUF(MIa) || isMTBUF(MIa)) {
2214  if (isMUBUF(MIb) || isMTBUF(MIb))
2215  return checkInstOffsetsDoNotOverlap(MIa, MIb);
2216 
2217  return !isFLAT(MIb) && !isSMRD(MIb);
2218  }
2219 
2220  if (isSMRD(MIa)) {
2221  if (isSMRD(MIb))
2222  return checkInstOffsetsDoNotOverlap(MIa, MIb);
2223 
2224  return !isFLAT(MIb) && !isMUBUF(MIa) && !isMTBUF(MIa);
2225  }
2226 
2227  if (isFLAT(MIa)) {
2228  if (isFLAT(MIb))
2229  return checkInstOffsetsDoNotOverlap(MIa, MIb);
2230 
2231  return false;
2232  }
2233 
2234  return false;
2235 }
2236 
2237 static int64_t getFoldableImm(const MachineOperand* MO) {
2238  if (!MO->isReg())
2239  return false;
2240  const MachineFunction *MF = MO->getParent()->getParent()->getParent();
2241  const MachineRegisterInfo &MRI = MF->getRegInfo();
2242  auto Def = MRI.getUniqueVRegDef(MO->getReg());
2243  if (Def && Def->getOpcode() == AMDGPU::V_MOV_B32_e32 &&
2244  Def->getOperand(1).isImm())
2245  return Def->getOperand(1).getImm();
2246  return AMDGPU::NoRegister;
2247 }
2248 
2250  MachineInstr &MI,
2251  LiveVariables *LV) const {
2252  unsigned Opc = MI.getOpcode();
2253  bool IsF16 = false;
2254  bool IsFMA = Opc == AMDGPU::V_FMAC_F32_e32 || Opc == AMDGPU::V_FMAC_F32_e64;
2255 
2256  switch (Opc) {
2257  default:
2258  return nullptr;
2259  case AMDGPU::V_MAC_F16_e64:
2260  IsF16 = true;
2262  case AMDGPU::V_MAC_F32_e64:
2263  case AMDGPU::V_FMAC_F32_e64:
2264  break;
2265  case AMDGPU::V_MAC_F16_e32:
2266  IsF16 = true;
2268  case AMDGPU::V_MAC_F32_e32:
2269  case AMDGPU::V_FMAC_F32_e32: {
2270  int Src0Idx = AMDGPU::getNamedOperandIdx(MI.getOpcode(),
2271  AMDGPU::OpName::src0);
2272  const MachineOperand *Src0 = &MI.getOperand(Src0Idx);
2273  if (!Src0->isReg() && !Src0->isImm())
2274  return nullptr;
2275 
2276  if (Src0->isImm() && !isInlineConstant(MI, Src0Idx, *Src0))
2277  return nullptr;
2278 
2279  break;
2280  }
2281  }
2282 
2283  const MachineOperand *Dst = getNamedOperand(MI, AMDGPU::OpName::vdst);
2284  const MachineOperand *Src0 = getNamedOperand(MI, AMDGPU::OpName::src0);
2285  const MachineOperand *Src0Mods =
2286  getNamedOperand(MI, AMDGPU::OpName::src0_modifiers);
2287  const MachineOperand *Src1 = getNamedOperand(MI, AMDGPU::OpName::src1);
2288  const MachineOperand *Src1Mods =
2289  getNamedOperand(MI, AMDGPU::OpName::src1_modifiers);
2290  const MachineOperand *Src2 = getNamedOperand(MI, AMDGPU::OpName::src2);
2291  const MachineOperand *Clamp = getNamedOperand(MI, AMDGPU::OpName::clamp);
2292  const MachineOperand *Omod = getNamedOperand(MI, AMDGPU::OpName::omod);
2293 
2294  if (!IsFMA && !Src0Mods && !Src1Mods && !Clamp && !Omod &&
2295  // If we have an SGPR input, we will violate the constant bus restriction.
2296  (!Src0->isReg() || !RI.isSGPRReg(MBB->getParent()->getRegInfo(), Src0->getReg()))) {
2297  if (auto Imm = getFoldableImm(Src2)) {
2298  return BuildMI(*MBB, MI, MI.getDebugLoc(),
2299  get(IsF16 ? AMDGPU::V_MADAK_F16 : AMDGPU::V_MADAK_F32))
2300  .add(*Dst)
2301  .add(*Src0)
2302  .add(*Src1)
2303  .addImm(Imm);
2304  }
2305  if (auto Imm = getFoldableImm(Src1)) {
2306  return BuildMI(*MBB, MI, MI.getDebugLoc(),
2307  get(IsF16 ? AMDGPU::V_MADMK_F16 : AMDGPU::V_MADMK_F32))
2308  .add(*Dst)
2309  .add(*Src0)
2310  .addImm(Imm)
2311  .add(*Src2);
2312  }
2313  if (auto Imm = getFoldableImm(Src0)) {
2314  if (isOperandLegal(MI, AMDGPU::getNamedOperandIdx(AMDGPU::V_MADMK_F32,
2315  AMDGPU::OpName::src0), Src1))
2316  return BuildMI(*MBB, MI, MI.getDebugLoc(),
2317  get(IsF16 ? AMDGPU::V_MADMK_F16 : AMDGPU::V_MADMK_F32))
2318  .add(*Dst)
2319  .add(*Src1)
2320  .addImm(Imm)
2321  .add(*Src2);
2322  }
2323  }
2324 
2325  assert((!IsFMA || !IsF16) && "fmac only expected with f32");
2326  unsigned NewOpc = IsFMA ? AMDGPU::V_FMA_F32 :
2327  (IsF16 ? AMDGPU::V_MAD_F16 : AMDGPU::V_MAD_F32);
2328  return BuildMI(*MBB, MI, MI.getDebugLoc(), get(NewOpc))
2329  .add(*Dst)
2330  .addImm(Src0Mods ? Src0Mods->getImm() : 0)
2331  .add(*Src0)
2332  .addImm(Src1Mods ? Src1Mods->getImm() : 0)
2333  .add(*Src1)
2334  .addImm(0) // Src mods
2335  .add(*Src2)
2336  .addImm(Clamp ? Clamp->getImm() : 0)
2337  .addImm(Omod ? Omod->getImm() : 0);
2338 }
2339 
2340 // It's not generally safe to move VALU instructions across these since it will
2341 // start using the register as a base index rather than directly.
2342 // XXX - Why isn't hasSideEffects sufficient for these?
2344  switch (MI.getOpcode()) {
2345  case AMDGPU::S_SET_GPR_IDX_ON:
2346  case AMDGPU::S_SET_GPR_IDX_MODE:
2347  case AMDGPU::S_SET_GPR_IDX_OFF:
2348  return true;
2349  default:
2350  return false;
2351  }
2352 }
2353 
2355  const MachineBasicBlock *MBB,
2356  const MachineFunction &MF) const {
2357  // XXX - Do we want the SP check in the base implementation?
2358 
2359  // Target-independent instructions do not have an implicit-use of EXEC, even
2360  // when they operate on VGPRs. Treating EXEC modifications as scheduling
2361  // boundaries prevents incorrect movements of such instructions.
2362  return TargetInstrInfo::isSchedulingBoundary(MI, MBB, MF) ||
2363  MI.modifiesRegister(AMDGPU::EXEC, &RI) ||
2364  MI.getOpcode() == AMDGPU::S_SETREG_IMM32_B32 ||
2365  MI.getOpcode() == AMDGPU::S_SETREG_B32 ||
2367 }
2368 
2370  unsigned Opcode = MI.getOpcode();
2371 
2372  if (MI.mayStore() && isSMRD(MI))
2373  return true; // scalar store or atomic
2374 
2375  // These instructions cause shader I/O that may cause hardware lockups
2376  // when executed with an empty EXEC mask.
2377  //
2378  // Note: exp with VM = DONE = 0 is automatically skipped by hardware when
2379  // EXEC = 0, but checking for that case here seems not worth it
2380  // given the typical code patterns.
2381  if (Opcode == AMDGPU::S_SENDMSG || Opcode == AMDGPU::S_SENDMSGHALT ||
2382  Opcode == AMDGPU::EXP || Opcode == AMDGPU::EXP_DONE)
2383  return true;
2384 
2385  if (MI.isInlineAsm())
2386  return true; // conservative assumption
2387 
2388  // These are like SALU instructions in terms of effects, so it's questionable
2389  // whether we should return true for those.
2390  //
2391  // However, executing them with EXEC = 0 causes them to operate on undefined
2392  // data, which we avoid by returning true here.
2393  if (Opcode == AMDGPU::V_READFIRSTLANE_B32 || Opcode == AMDGPU::V_READLANE_B32)
2394  return true;
2395 
2396  return false;
2397 }
2398 
2399 bool SIInstrInfo::isInlineConstant(const APInt &Imm) const {
2400  switch (Imm.getBitWidth()) {
2401  case 32:
2403  ST.hasInv2PiInlineImm());
2404  case 64:
2406  ST.hasInv2PiInlineImm());
2407  case 16:
2408  return ST.has16BitInsts() &&
2410  ST.hasInv2PiInlineImm());
2411  default:
2412  llvm_unreachable("invalid bitwidth");
2413  }
2414 }
2415 
2417  uint8_t OperandType) const {
2418  if (!MO.isImm() ||
2419  OperandType < AMDGPU::OPERAND_SRC_FIRST ||
2420  OperandType > AMDGPU::OPERAND_SRC_LAST)
2421  return false;
2422 
2423  // MachineOperand provides no way to tell the true operand size, since it only
2424  // records a 64-bit value. We need to know the size to determine if a 32-bit
2425  // floating point immediate bit pattern is legal for an integer immediate. It
2426  // would be for any 32-bit integer operand, but would not be for a 64-bit one.
2427 
2428  int64_t Imm = MO.getImm();
2429  switch (OperandType) {
2434  int32_t Trunc = static_cast<int32_t>(Imm);
2436  }
2442  ST.hasInv2PiInlineImm());
2447  if (isInt<16>(Imm) || isUInt<16>(Imm)) {
2448  // A few special case instructions have 16-bit operands on subtargets
2449  // where 16-bit instructions are not legal.
2450  // TODO: Do the 32-bit immediates work? We shouldn't really need to handle
2451  // constants in these cases
2452  int16_t Trunc = static_cast<int16_t>(Imm);
2453  return ST.has16BitInsts() &&
2455  }
2456 
2457  return false;
2458  }
2461  if (isUInt<16>(Imm)) {
2462  int16_t Trunc = static_cast<int16_t>(Imm);
2463  return ST.has16BitInsts() &&
2465  }
2466  if (!(Imm & 0xffff)) {
2467  return ST.has16BitInsts() &&
2469  }
2470  uint32_t Trunc = static_cast<uint32_t>(Imm);
2472  }
2473  default:
2474  llvm_unreachable("invalid bitwidth");
2475  }
2476 }
2477 
2479  const MCOperandInfo &OpInfo) const {
2480  switch (MO.getType()) {
2482  return false;
2484  return !isInlineConstant(MO, OpInfo);
2490  return true;
2491  default:
2492  llvm_unreachable("unexpected operand type");
2493  }
2494 }
2495 
2496 static bool compareMachineOp(const MachineOperand &Op0,
2497  const MachineOperand &Op1) {
2498  if (Op0.getType() != Op1.getType())
2499  return false;
2500 
2501  switch (Op0.getType()) {
2503  return Op0.getReg() == Op1.getReg();
2505  return Op0.getImm() == Op1.getImm();
2506  default:
2507  llvm_unreachable("Didn't expect to be comparing these operand types");
2508  }
2509 }
2510 
2512  const MachineOperand &MO) const {
2513  const MCOperandInfo &OpInfo = get(MI.getOpcode()).OpInfo[OpNo];
2514 
2515  assert(MO.isImm() || MO.isTargetIndex() || MO.isFI());
2516 
2517  if (OpInfo.OperandType == MCOI::OPERAND_IMMEDIATE)
2518  return true;
2519 
2520  if (OpInfo.RegClass < 0)
2521  return false;
2522 
2523  if (MO.isImm() && isInlineConstant(MO, OpInfo))
2524  return RI.opCanUseInlineConstant(OpInfo.OperandType);
2525 
2526  return RI.opCanUseLiteralConstant(OpInfo.OperandType);
2527 }
2528 
2529 bool SIInstrInfo::hasVALU32BitEncoding(unsigned Opcode) const {
2530  int Op32 = AMDGPU::getVOPe32(Opcode);
2531  if (Op32 == -1)
2532  return false;
2533 
2534  return pseudoToMCOpcode(Op32) != -1;
2535 }
2536 
2537 bool SIInstrInfo::hasModifiers(unsigned Opcode) const {
2538  // The src0_modifier operand is present on all instructions
2539  // that have modifiers.
2540 
2541  return AMDGPU::getNamedOperandIdx(Opcode,
2542  AMDGPU::OpName::src0_modifiers) != -1;
2543 }
2544 
2546  unsigned OpName) const {
2547  const MachineOperand *Mods = getNamedOperand(MI, OpName);
2548  return Mods && Mods->getImm();
2549 }
2550 
2552  return hasModifiersSet(MI, AMDGPU::OpName::src0_modifiers) ||
2553  hasModifiersSet(MI, AMDGPU::OpName::src1_modifiers) ||
2554  hasModifiersSet(MI, AMDGPU::OpName::src2_modifiers) ||
2555  hasModifiersSet(MI, AMDGPU::OpName::clamp) ||
2556  hasModifiersSet(MI, AMDGPU::OpName::omod);
2557 }
2558 
2560  const MachineRegisterInfo &MRI) const {
2561  const MachineOperand *Src2 = getNamedOperand(MI, AMDGPU::OpName::src2);
2562  // Can't shrink instruction with three operands.
2563  // FIXME: v_cndmask_b32 has 3 operands and is shrinkable, but we need to add
2564  // a special case for it. It can only be shrunk if the third operand
2565  // is vcc. We should handle this the same way we handle vopc, by addding
2566  // a register allocation hint pre-regalloc and then do the shrinking
2567  // post-regalloc.
2568  if (Src2) {
2569  switch (MI.getOpcode()) {
2570  default: return false;
2571 
2572  case AMDGPU::V_ADDC_U32_e64:
2573  case AMDGPU::V_SUBB_U32_e64:
2574  case AMDGPU::V_SUBBREV_U32_e64: {
2575  const MachineOperand *Src1
2576  = getNamedOperand(MI, AMDGPU::OpName::src1);
2577  if (!Src1->isReg() || !RI.isVGPR(MRI, Src1->getReg()))
2578  return false;
2579  // Additional verification is needed for sdst/src2.
2580  return true;
2581  }
2582  case AMDGPU::V_MAC_F32_e64:
2583  case AMDGPU::V_MAC_F16_e64:
2584  case AMDGPU::V_FMAC_F32_e64:
2585  if (!Src2->isReg() || !RI.isVGPR(MRI, Src2->getReg()) ||
2586  hasModifiersSet(MI, AMDGPU::OpName::src2_modifiers))
2587  return false;
2588  break;
2589 
2590  case AMDGPU::V_CNDMASK_B32_e64:
2591  break;
2592  }
2593  }
2594 
2595  const MachineOperand *Src1 = getNamedOperand(MI, AMDGPU::OpName::src1);
2596  if (Src1 && (!Src1->isReg() || !RI.isVGPR(MRI, Src1->getReg()) ||
2597  hasModifiersSet(MI, AMDGPU::OpName::src1_modifiers)))
2598  return false;
2599 
2600  // We don't need to check src0, all input types are legal, so just make sure
2601  // src0 isn't using any modifiers.
2602  if (hasModifiersSet(MI, AMDGPU::OpName::src0_modifiers))
2603  return false;
2604 
2605  // Check output modifiers
2606  return !hasModifiersSet(MI, AMDGPU::OpName::omod) &&
2607  !hasModifiersSet(MI, AMDGPU::OpName::clamp);
2608 }
2609 
2610 // Set VCC operand with all flags from \p Orig, except for setting it as
2611 // implicit.
2613  const MachineOperand &Orig) {
2614 
2615  for (MachineOperand &Use : MI.implicit_operands()) {
2616  if (Use.isUse() && Use.getReg() == AMDGPU::VCC) {
2617  Use.setIsUndef(Orig.isUndef());
2618  Use.setIsKill(Orig.isKill());
2619  return;
2620  }
2621  }
2622 }
2623 
2625  unsigned Op32) const {
2626  MachineBasicBlock *MBB = MI.getParent();;
2627  MachineInstrBuilder Inst32 =
2628  BuildMI(*MBB, MI, MI.getDebugLoc(), get(Op32));
2629 
2630  // Add the dst operand if the 32-bit encoding also has an explicit $vdst.
2631  // For VOPC instructions, this is replaced by an implicit def of vcc.
2632  int Op32DstIdx = AMDGPU::getNamedOperandIdx(Op32, AMDGPU::OpName::vdst);
2633  if (Op32DstIdx != -1) {
2634  // dst
2635  Inst32.add(MI.getOperand(0));
2636  } else {
2637  assert(MI.getOperand(0).getReg() == AMDGPU::VCC &&
2638  "Unexpected case");
2639  }
2640 
2641  Inst32.add(*getNamedOperand(MI, AMDGPU::OpName::src0));
2642 
2643  const MachineOperand *Src1 = getNamedOperand(MI, AMDGPU::OpName::src1);
2644  if (Src1)
2645  Inst32.add(*Src1);
2646 
2647  const MachineOperand *Src2 = getNamedOperand(MI, AMDGPU::OpName::src2);
2648 
2649  if (Src2) {
2650  int Op32Src2Idx = AMDGPU::getNamedOperandIdx(Op32, AMDGPU::OpName::src2);
2651  if (Op32Src2Idx != -1) {
2652  Inst32.add(*Src2);
2653  } else {
2654  // In the case of V_CNDMASK_B32_e32, the explicit operand src2 is
2655  // replaced with an implicit read of vcc. This was already added
2656  // during the initial BuildMI, so find it to preserve the flags.
2657  copyFlagsToImplicitVCC(*Inst32, *Src2);
2658  }
2659  }
2660 
2661  return Inst32;
2662 }
2663 
2665  const MachineOperand &MO,
2666  const MCOperandInfo &OpInfo) const {
2667  // Literal constants use the constant bus.
2668  //if (isLiteralConstantLike(MO, OpInfo))
2669  // return true;
2670  if (MO.isImm())
2671  return !isInlineConstant(MO, OpInfo);
2672 
2673  if (!MO.isReg())
2674  return true; // Misc other operands like FrameIndex
2675 
2676  if (!MO.isUse())
2677  return false;
2678 
2680  return RI.isSGPRClass(MRI.getRegClass(MO.getReg()));
2681 
2682  // FLAT_SCR is just an SGPR pair.
2683  if (!MO.isImplicit() && (MO.getReg() == AMDGPU::FLAT_SCR))
2684  return true;
2685 
2686  // EXEC register uses the constant bus.
2687  if (!MO.isImplicit() && MO.getReg() == AMDGPU::EXEC)
2688  return true;
2689 
2690  // SGPRs use the constant bus
2691  return (MO.getReg() == AMDGPU::VCC || MO.getReg() == AMDGPU::M0 ||
2692  (!MO.isImplicit() &&
2693  (AMDGPU::SGPR_32RegClass.contains(MO.getReg()) ||
2694  AMDGPU::SGPR_64RegClass.contains(MO.getReg()))));
2695 }
2696 
2697 static unsigned findImplicitSGPRRead(const MachineInstr &MI) {
2698  for (const MachineOperand &MO : MI.implicit_operands()) {
2699  // We only care about reads.
2700  if (MO.isDef())
2701  continue;
2702 
2703  switch (MO.getReg()) {
2704  case AMDGPU::VCC:
2705  case AMDGPU::M0:
2706  case AMDGPU::FLAT_SCR:
2707  return MO.getReg();
2708 
2709  default:
2710  break;
2711  }
2712  }
2713 
2714  return AMDGPU::NoRegister;
2715 }
2716 
2717 static bool shouldReadExec(const MachineInstr &MI) {
2718  if (SIInstrInfo::isVALU(MI)) {
2719  switch (MI.getOpcode()) {
2720  case AMDGPU::V_READLANE_B32:
2721  case AMDGPU::V_READLANE_B32_si:
2722  case AMDGPU::V_READLANE_B32_vi:
2723  case AMDGPU::V_WRITELANE_B32:
2724  case AMDGPU::V_WRITELANE_B32_si:
2725  case AMDGPU::V_WRITELANE_B32_vi:
2726  return false;
2727  }
2728 
2729  return true;
2730  }
2731 
2732  if (SIInstrInfo::isGenericOpcode(MI.getOpcode()) ||
2733  SIInstrInfo::isSALU(MI) ||
2734  SIInstrInfo::isSMRD(MI))
2735  return false;
2736 
2737  return true;
2738 }
2739 
2740 static bool isSubRegOf(const SIRegisterInfo &TRI,
2741  const MachineOperand &SuperVec,
2742  const MachineOperand &SubReg) {
2744  return TRI.isSubRegister(SuperVec.getReg(), SubReg.getReg());
2745 
2746  return SubReg.getSubReg() != AMDGPU::NoSubRegister &&
2747  SubReg.getReg() == SuperVec.getReg();
2748 }
2749 
2751  StringRef &ErrInfo) const {
2752  uint16_t Opcode = MI.getOpcode();
2753  if (SIInstrInfo::isGenericOpcode(MI.getOpcode()))
2754  return true;
2755 
2756  const MachineFunction *MF = MI.getParent()->getParent();
2757  const MachineRegisterInfo &MRI = MF->getRegInfo();
2758 
2759  int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0);
2760  int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src1);
2761  int Src2Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src2);
2762 
2763  // Make sure the number of operands is correct.
2764  const MCInstrDesc &Desc = get(Opcode);
2765  if (!Desc.isVariadic() &&
2766  Desc.getNumOperands() != MI.getNumExplicitOperands()) {
2767  ErrInfo = "Instruction has wrong number of operands.";
2768  return false;
2769  }
2770 
2771  if (MI.isInlineAsm()) {
2772  // Verify register classes for inlineasm constraints.
2773  for (unsigned I = InlineAsm::MIOp_FirstOperand, E = MI.getNumOperands();
2774  I != E; ++I) {
2775  const TargetRegisterClass *RC = MI.getRegClassConstraint(I, this, &RI);
2776  if (!RC)
2777  continue;
2778 
2779  const MachineOperand &Op = MI.getOperand(I);
2780  if (!Op.isReg())
2781  continue;
2782 
2783  unsigned Reg = Op.getReg();
2784  if (!TargetRegisterInfo::isVirtualRegister(Reg) && !RC->contains(Reg)) {
2785  ErrInfo = "inlineasm operand has incorrect register class.";
2786  return false;
2787  }
2788  }
2789 
2790  return true;
2791  }
2792 
2793  // Make sure the register classes are correct.
2794  for (int i = 0, e = Desc.getNumOperands(); i != e; ++i) {
2795  if (MI.getOperand(i).isFPImm()) {
2796  ErrInfo = "FPImm Machine Operands are not supported. ISel should bitcast "
2797  "all fp values to integers.";
2798  return false;
2799  }
2800 
2801  int RegClass = Desc.OpInfo[i].RegClass;
2802 
2803  switch (Desc.OpInfo[i].OperandType) {
2805  if (MI.getOperand(i).isImm()) {
2806  ErrInfo = "Illegal immediate value for operand.";
2807  return false;
2808  }
2809  break;
2812  break;
2819  const MachineOperand &MO = MI.getOperand(i);
2820  if (!MO.isReg() && (!MO.isImm() || !isInlineConstant(MI, i))) {
2821  ErrInfo = "Illegal immediate value for operand.";
2822  return false;
2823  }
2824  break;
2825  }
2828  // Check if this operand is an immediate.
2829  // FrameIndex operands will be replaced by immediates, so they are
2830  // allowed.
2831  if (!MI.getOperand(i).isImm() && !MI.getOperand(i).isFI()) {
2832  ErrInfo = "Expected immediate, but got non-immediate";
2833  return false;
2834  }
2836  default:
2837  continue;
2838  }
2839 
2840  if (!MI.getOperand(i).isReg())
2841  continue;
2842 
2843  if (RegClass != -1) {
2844  unsigned Reg = MI.getOperand(i).getReg();
2845  if (Reg == AMDGPU::NoRegister ||
2847  continue;
2848 
2849  const TargetRegisterClass *RC = RI.getRegClass(RegClass);
2850  if (!RC->contains(Reg)) {
2851  ErrInfo = "Operand has incorrect register class.";
2852  return false;
2853  }
2854  }
2855  }
2856 
2857  // Verify SDWA
2858  if (isSDWA(MI)) {
2859  if (!ST.hasSDWA()) {
2860  ErrInfo = "SDWA is not supported on this target";
2861  return false;
2862  }
2863 
2864  int DstIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdst);
2865 
2866  const int OpIndicies[] = { DstIdx, Src0Idx, Src1Idx, Src2Idx };
2867 
2868  for (int OpIdx: OpIndicies) {
2869  if (OpIdx == -1)
2870  continue;
2871  const MachineOperand &MO = MI.getOperand(OpIdx);
2872 
2873  if (!ST.hasSDWAScalar()) {
2874  // Only VGPRS on VI
2875  if (!MO.isReg() || !RI.hasVGPRs(RI.getRegClassForReg(MRI, MO.getReg()))) {
2876  ErrInfo = "Only VGPRs allowed as operands in SDWA instructions on VI";
2877  return false;
2878  }
2879  } else {
2880  // No immediates on GFX9
2881  if (!MO.isReg()) {
2882  ErrInfo = "Only reg allowed as operands in SDWA instructions on GFX9";
2883  return false;
2884  }
2885  }
2886  }
2887 
2888  if (!ST.hasSDWAOmod()) {
2889  // No omod allowed on VI
2890  const MachineOperand *OMod = getNamedOperand(MI, AMDGPU::OpName::omod);
2891  if (OMod != nullptr &&
2892  (!OMod->isImm() || OMod->getImm() != 0)) {
2893  ErrInfo = "OMod not allowed in SDWA instructions on VI";
2894  return false;
2895  }
2896  }
2897 
2898  uint16_t BasicOpcode = AMDGPU::getBasicFromSDWAOp(Opcode);
2899  if (isVOPC(BasicOpcode)) {
2900  if (!ST.hasSDWASdst() && DstIdx != -1) {
2901  // Only vcc allowed as dst on VI for VOPC
2902  const MachineOperand &Dst = MI.getOperand(DstIdx);
2903  if (!Dst.isReg() || Dst.getReg() != AMDGPU::VCC) {
2904  ErrInfo = "Only VCC allowed as dst in SDWA instructions on VI";
2905  return false;
2906  }
2907  } else if (!ST.hasSDWAOutModsVOPC()) {
2908  // No clamp allowed on GFX9 for VOPC
2909  const MachineOperand *Clamp = getNamedOperand(MI, AMDGPU::OpName::clamp);
2910  if (Clamp && (!Clamp->isImm() || Clamp->getImm() != 0)) {
2911  ErrInfo = "Clamp not allowed in VOPC SDWA instructions on VI";
2912  return false;
2913  }
2914 
2915  // No omod allowed on GFX9 for VOPC
2916  const MachineOperand *OMod = getNamedOperand(MI, AMDGPU::OpName::omod);
2917  if (OMod && (!OMod->isImm() || OMod->getImm() != 0)) {
2918  ErrInfo = "OMod not allowed in VOPC SDWA instructions on VI";
2919  return false;
2920  }
2921  }
2922  }
2923 
2924  const MachineOperand *DstUnused = getNamedOperand(MI, AMDGPU::OpName::dst_unused);
2925  if (DstUnused && DstUnused->isImm() &&
2926  DstUnused->getImm() == AMDGPU::SDWA::UNUSED_PRESERVE) {
2927  const MachineOperand &Dst = MI.getOperand(DstIdx);
2928  if (!Dst.isReg() || !Dst.isTied()) {
2929  ErrInfo = "Dst register should have tied register";
2930  return false;
2931  }
2932 
2933  const MachineOperand &TiedMO =
2934  MI.getOperand(MI.findTiedOperandIdx(DstIdx));
2935  if (!TiedMO.isReg() || !TiedMO.isImplicit() || !TiedMO.isUse()) {
2936  ErrInfo =
2937  "Dst register should be tied to implicit use of preserved register";
2938  return false;
2939  } else if (TargetRegisterInfo::isPhysicalRegister(TiedMO.getReg()) &&
2940  Dst.getReg() != TiedMO.getReg()) {
2941  ErrInfo = "Dst register should use same physical register as preserved";
2942  return false;
2943  }
2944  }
2945  }
2946 
2947  // Verify VOP*. Ignore multiple sgpr operands on writelane.
2948  if (Desc.getOpcode() != AMDGPU::V_WRITELANE_B32
2949  && (isVOP1(MI) || isVOP2(MI) || isVOP3(MI) || isVOPC(MI) || isSDWA(MI))) {
2950  // Only look at the true operands. Only a real operand can use the constant
2951  // bus, and we don't want to check pseudo-operands like the source modifier
2952  // flags.
2953  const int OpIndices[] = { Src0Idx, Src1Idx, Src2Idx };
2954 
2955  unsigned ConstantBusCount = 0;
2956  unsigned LiteralCount = 0;
2957 
2958  if (AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::imm) != -1)
2959  ++ConstantBusCount;
2960 
2961  unsigned SGPRUsed = findImplicitSGPRRead(MI);
2962  if (SGPRUsed != AMDGPU::NoRegister)
2963  ++ConstantBusCount;
2964 
2965  for (int OpIdx : OpIndices) {
2966  if (OpIdx == -1)
2967  break;
2968  const MachineOperand &MO = MI.getOperand(OpIdx);
2969  if (usesConstantBus(MRI, MO, MI.getDesc().OpInfo[OpIdx])) {
2970  if (MO.isReg()) {
2971  if (MO.getReg() != SGPRUsed)
2972  ++ConstantBusCount;
2973  SGPRUsed = MO.getReg();
2974  } else {
2975  ++ConstantBusCount;
2976  ++LiteralCount;
2977  }
2978  }
2979  }
2980  if (ConstantBusCount > 1) {
2981  ErrInfo = "VOP* instruction uses the constant bus more than once";
2982  return false;
2983  }
2984 
2985  if (isVOP3(MI) && LiteralCount) {
2986  ErrInfo = "VOP3 instruction uses literal";
2987  return false;
2988  }
2989  }
2990 
2991  // Verify misc. restrictions on specific instructions.
2992  if (Desc.getOpcode() == AMDGPU::V_DIV_SCALE_F32 ||
2993  Desc.getOpcode() == AMDGPU::V_DIV_SCALE_F64) {
2994  const MachineOperand &Src0 = MI.getOperand(Src0Idx);
2995  const MachineOperand &Src1 = MI.getOperand(Src1Idx);
2996  const MachineOperand &Src2 = MI.getOperand(Src2Idx);
2997  if (Src0.isReg() && Src1.isReg() && Src2.isReg()) {
2998  if (!compareMachineOp(Src0, Src1) &&
2999  !compareMachineOp(Src0, Src2)) {
3000  ErrInfo = "v_div_scale_{f32|f64} require src0 = src1 or src2";
3001  return false;
3002  }
3003  }
3004  }
3005 
3006  if (isSOPK(MI)) {
3007  int64_t Imm = getNamedOperand(MI, AMDGPU::OpName::simm16)->getImm();
3008  if (sopkIsZext(MI)) {
3009  if (!isUInt<16>(Imm)) {
3010  ErrInfo = "invalid immediate for SOPK instruction";
3011  return false;
3012  }
3013  } else {
3014  if (!isInt<16>(Imm)) {
3015  ErrInfo = "invalid immediate for SOPK instruction";
3016  return false;
3017  }
3018  }
3019  }
3020 
3021  if (Desc.getOpcode() == AMDGPU::V_MOVRELS_B32_e32 ||
3022  Desc.getOpcode() == AMDGPU::V_MOVRELS_B32_e64 ||
3023  Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e32 ||
3024  Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e64) {
3025  const bool IsDst = Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e32 ||
3026  Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e64;
3027 
3028  const unsigned StaticNumOps = Desc.getNumOperands() +
3029  Desc.getNumImplicitUses();
3030  const unsigned NumImplicitOps = IsDst ? 2 : 1;
3031 
3032  // Allow additional implicit operands. This allows a fixup done by the post
3033  // RA scheduler where the main implicit operand is killed and implicit-defs
3034  // are added for sub-registers that remain live after this instruction.
3035  if (MI.getNumOperands() < StaticNumOps + NumImplicitOps) {
3036  ErrInfo = "missing implicit register operands";
3037  return false;
3038  }
3039 
3040  const MachineOperand *Dst = getNamedOperand(MI, AMDGPU::OpName::vdst);
3041  if (IsDst) {
3042  if (!Dst->isUse()) {
3043  ErrInfo = "v_movreld_b32 vdst should be a use operand";
3044  return false;
3045  }
3046 
3047  unsigned UseOpIdx;
3048  if (!MI.isRegTiedToUseOperand(StaticNumOps, &UseOpIdx) ||
3049  UseOpIdx != StaticNumOps + 1) {
3050  ErrInfo = "movrel implicit operands should be tied";
3051  return false;
3052  }
3053  }
3054 
3055  const MachineOperand &Src0 = MI.getOperand(Src0Idx);
3056  const MachineOperand &ImpUse
3057  = MI.getOperand(StaticNumOps + NumImplicitOps - 1);
3058  if (!ImpUse.isReg() || !ImpUse.isUse() ||
3059  !isSubRegOf(RI, ImpUse, IsDst ? *Dst : Src0)) {
3060  ErrInfo = "src0 should be subreg of implicit vector use";
3061  return false;
3062  }
3063  }
3064 
3065  // Make sure we aren't losing exec uses in the td files. This mostly requires
3066  // being careful when using let Uses to try to add other use registers.
3067  if (shouldReadExec(MI)) {
3068  if (!MI.hasRegisterImplicitUseOperand(AMDGPU::EXEC)) {
3069  ErrInfo = "VALU instruction does not implicitly read exec mask";
3070  return false;
3071  }
3072  }
3073 
3074  if (isSMRD(MI)) {
3075  if (MI.mayStore()) {
3076  // The register offset form of scalar stores may only use m0 as the
3077  // soffset register.
3078  const MachineOperand *Soff = getNamedOperand(MI, AMDGPU::OpName::soff);
3079  if (Soff && Soff->getReg() != AMDGPU::M0) {
3080  ErrInfo = "scalar stores must use m0 as offset register";
3081  return false;
3082  }
3083  }
3084  }
3085 
3086  if (isFLAT(MI) && !MF->getSubtarget<GCNSubtarget>().hasFlatInstOffsets()) {
3087  const MachineOperand *Offset = getNamedOperand(MI, AMDGPU::OpName::offset);
3088  if (Offset->getImm() != 0) {
3089  ErrInfo = "subtarget does not support offsets in flat instructions";
3090  return false;
3091  }
3092  }
3093 
3094  const MachineOperand *DppCt = getNamedOperand(MI, AMDGPU::OpName::dpp_ctrl);
3095  if (DppCt) {
3096  using namespace AMDGPU::DPP;
3097 
3098  unsigned DC = DppCt->getImm();
3099  if (DC == DppCtrl::DPP_UNUSED1 || DC == DppCtrl::DPP_UNUSED2 ||
3100  DC == DppCtrl::DPP_UNUSED3 || DC > DppCtrl::DPP_LAST ||
3105  ErrInfo = "Invalid dpp_ctrl value";
3106  return false;
3107  }
3108  }
3109 
3110  return true;
3111 }
3112 
3113 unsigned SIInstrInfo::getVALUOp(const MachineInstr &MI) const {
3114  switch (MI.getOpcode()) {
3115  default: return AMDGPU::INSTRUCTION_LIST_END;
3116  case AMDGPU::REG_SEQUENCE: return AMDGPU::REG_SEQUENCE;
3117  case AMDGPU::COPY: return AMDGPU::COPY;
3118  case AMDGPU::PHI: return AMDGPU::PHI;
3119  case AMDGPU::INSERT_SUBREG: return AMDGPU::INSERT_SUBREG;
3120  case AMDGPU::WQM: return AMDGPU::WQM;
3121  case AMDGPU::WWM: return AMDGPU::WWM;
3122  case AMDGPU::S_MOV_B32:
3123  return MI.getOperand(1).isReg() ?
3124  AMDGPU::COPY : AMDGPU::V_MOV_B32_e32;
3125  case AMDGPU::S_ADD_I32:
3126  return ST.hasAddNoCarry() ? AMDGPU::V_ADD_U32_e64 : AMDGPU::V_ADD_I32_e32;
3127  case AMDGPU::S_ADDC_U32:
3128  return AMDGPU::V_ADDC_U32_e32;
3129  case AMDGPU::S_SUB_I32:
3130  return ST.hasAddNoCarry() ? AMDGPU::V_SUB_U32_e64 : AMDGPU::V_SUB_I32_e32;
3131  // FIXME: These are not consistently handled, and selected when the carry is
3132  // used.
3133  case AMDGPU::S_ADD_U32:
3134  return AMDGPU::V_ADD_I32_e32;
3135  case AMDGPU::S_SUB_U32:
3136  return AMDGPU::V_SUB_I32_e32;
3137  case AMDGPU::S_SUBB_U32: return AMDGPU::V_SUBB_U32_e32;
3138  case AMDGPU::S_MUL_I32: return AMDGPU::V_MUL_LO_I32;
3139  case AMDGPU::S_AND_B32: return AMDGPU::V_AND_B32_e64;
3140  case AMDGPU::S_OR_B32: return AMDGPU::V_OR_B32_e64;
3141  case AMDGPU::S_XOR_B32: return AMDGPU::V_XOR_B32_e64;
3142  case AMDGPU::S_MIN_I32: return AMDGPU::V_MIN_I32_e64;
3143  case AMDGPU::S_MIN_U32: return AMDGPU::V_MIN_U32_e64;
3144  case AMDGPU::S_MAX_I32: return AMDGPU::V_MAX_I32_e64;
3145  case AMDGPU::S_MAX_U32: return AMDGPU::V_MAX_U32_e64;
3146  case AMDGPU::S_ASHR_I32: return AMDGPU::V_ASHR_I32_e32;
3147  case AMDGPU::S_ASHR_I64: return AMDGPU::V_ASHR_I64;
3148  case AMDGPU::S_LSHL_B32: return AMDGPU::V_LSHL_B32_e32;
3149  case AMDGPU::S_LSHL_B64: return AMDGPU::V_LSHL_B64;
3150  case AMDGPU::S_LSHR_B32: return AMDGPU::V_LSHR_B32_e32;
3151  case AMDGPU::S_LSHR_B64: return AMDGPU::V_LSHR_B64;
3152  case AMDGPU::S_SEXT_I32_I8: return AMDGPU::V_BFE_I32;
3153  case AMDGPU::S_SEXT_I32_I16: return AMDGPU::V_BFE_I32;
3154  case AMDGPU::S_BFE_U32: return AMDGPU::V_BFE_U32;
3155  case AMDGPU::S_BFE_I32: return AMDGPU::V_BFE_I32;
3156  case AMDGPU::S_BFM_B32: return AMDGPU::V_BFM_B32_e64;
3157  case AMDGPU::S_BREV_B32: return AMDGPU::V_BFREV_B32_e32;
3158  case AMDGPU::S_NOT_B32: return AMDGPU::V_NOT_B32_e32;
3159  case AMDGPU::S_NOT_B64: return AMDGPU::V_NOT_B32_e32;
3160  case AMDGPU::S_CMP_EQ_I32: return AMDGPU::V_CMP_EQ_I32_e32;
3161  case AMDGPU::S_CMP_LG_I32: return AMDGPU::V_CMP_NE_I32_e32;
3162  case AMDGPU::S_CMP_GT_I32: return AMDGPU::V_CMP_GT_I32_e32;
3163  case AMDGPU::S_CMP_GE_I32: return AMDGPU::V_CMP_GE_I32_e32;
3164  case AMDGPU::S_CMP_LT_I32: return AMDGPU::V_CMP_LT_I32_e32;
3165  case AMDGPU::S_CMP_LE_I32: return AMDGPU::V_CMP_LE_I32_e32;
3166  case AMDGPU::S_CMP_EQ_U32: return AMDGPU::V_CMP_EQ_U32_e32;
3167  case AMDGPU::S_CMP_LG_U32: return AMDGPU::V_CMP_NE_U32_e32;
3168  case AMDGPU::S_CMP_GT_U32: return AMDGPU::V_CMP_GT_U32_e32;
3169  case AMDGPU::S_CMP_GE_U32: return AMDGPU::V_CMP_GE_U32_e32;
3170  case AMDGPU::S_CMP_LT_U32: return AMDGPU::V_CMP_LT_U32_e32;
3171  case AMDGPU::S_CMP_LE_U32: return AMDGPU::V_CMP_LE_U32_e32;
3172  case AMDGPU::S_CMP_EQ_U64: return AMDGPU::V_CMP_EQ_U64_e32;
3173  case AMDGPU::S_CMP_LG_U64: return AMDGPU::V_CMP_NE_U64_e32;
3174  case AMDGPU::S_BCNT1_I32_B32: return AMDGPU::V_BCNT_U32_B32_e64;
3175  case AMDGPU::S_FF1_I32_B32: return AMDGPU::V_FFBL_B32_e32;
3176  case AMDGPU::S_FLBIT_I32_B32: return AMDGPU::V_FFBH_U32_e32;
3177  case AMDGPU::S_FLBIT_I32: return AMDGPU::V_FFBH_I32_e64;
3178  case AMDGPU::S_CBRANCH_SCC0: return AMDGPU::S_CBRANCH_VCCZ;
3179  case AMDGPU::S_CBRANCH_SCC1: return AMDGPU::S_CBRANCH_VCCNZ;
3180  }
3181 }
3182 
3184  unsigned OpNo) const {
3186  const MCInstrDesc &Desc = get(MI.getOpcode());
3187  if (MI.isVariadic() || OpNo >= Desc.getNumOperands() ||
3188  Desc.OpInfo[OpNo].RegClass == -1) {
3189  unsigned Reg = MI.getOperand(OpNo).getReg();
3190 
3192  return MRI.getRegClass(Reg);
3193  return RI.getPhysRegClass(Reg);
3194  }
3195 
3196  unsigned RCID = Desc.OpInfo[OpNo].RegClass;
3197  return RI.getRegClass(RCID);
3198 }
3199 
3200 bool SIInstrInfo::canReadVGPR(const MachineInstr &MI, unsigned OpNo) const {
3201  switch (MI.getOpcode()) {
3202  case AMDGPU::COPY:
3203  case AMDGPU::REG_SEQUENCE:
3204  case AMDGPU::PHI:
3205  case AMDGPU::INSERT_SUBREG:
3206  return RI.hasVGPRs(getOpRegClass(MI, 0));
3207  default:
3208  return RI.hasVGPRs(getOpRegClass(MI, OpNo));
3209  }
3210 }
3211 
3212 void SIInstrInfo::legalizeOpWithMove(MachineInstr &MI, unsigned OpIdx) const {
3214  MachineBasicBlock *MBB = MI.getParent();
3215  MachineOperand &MO = MI.getOperand(OpIdx);
3217  unsigned RCID = get(MI.getOpcode()).OpInfo[OpIdx].RegClass;
3218  const TargetRegisterClass *RC = RI.getRegClass(RCID);
3219  unsigned Opcode = AMDGPU::V_MOV_B32_e32;
3220  if (MO.isReg())
3221  Opcode = AMDGPU::COPY;
3222  else if (RI.isSGPRClass(RC))
3223  Opcode = AMDGPU::S_MOV_B32;
3224 
3225  const TargetRegisterClass *VRC = RI.getEquivalentVGPRClass(RC);
3226  if (RI.getCommonSubClass(&AMDGPU::VReg_64RegClass, VRC))
3227  VRC = &AMDGPU::VReg_64RegClass;
3228  else
3229  VRC = &AMDGPU::VGPR_32RegClass;
3230 
3231  unsigned Reg = MRI.createVirtualRegister(VRC);
3232  DebugLoc DL = MBB->findDebugLoc(I);
3233  BuildMI(*MI.getParent(), I, DL, get(Opcode), Reg).add(MO);
3234  MO.ChangeToRegister(Reg, false);
3235 }
3236 
3239  MachineOperand &SuperReg,
3240  const TargetRegisterClass *SuperRC,
3241  unsigned SubIdx,
3242  const TargetRegisterClass *SubRC)
3243  const {
3244  MachineBasicBlock *MBB = MI->getParent();
3245  DebugLoc DL = MI->getDebugLoc();
3246  unsigned SubReg = MRI.createVirtualRegister(SubRC);
3247 
3248  if (SuperReg.getSubReg() == AMDGPU::NoSubRegister) {
3249  BuildMI(*MBB, MI, DL, get(TargetOpcode::COPY), SubReg)
3250  .addReg(SuperReg.getReg(), 0, SubIdx);
3251  return SubReg;
3252  }
3253 
3254  // Just in case the super register is itself a sub-register, copy it to a new
3255  // value so we don't need to worry about merging its subreg index with the
3256  // SubIdx passed to this function. The register coalescer should be able to
3257  // eliminate this extra copy.
3258  unsigned NewSuperReg = MRI.createVirtualRegister(SuperRC);
3259 
3260  BuildMI(*MBB, MI, DL, get(TargetOpcode::COPY), NewSuperReg)
3261  .addReg(SuperReg.getReg(), 0, SuperReg.getSubReg());
3262 
3263  BuildMI(*MBB, MI, DL, get(TargetOpcode::COPY), SubReg)
3264  .addReg(NewSuperReg, 0, SubIdx);
3265 
3266  return SubReg;
3267 }
3268 
3272  MachineOperand &Op,
3273  const TargetRegisterClass *SuperRC,
3274  unsigned SubIdx,
3275  const TargetRegisterClass *SubRC) const {
3276  if (Op.isImm()) {
3277  if (SubIdx == AMDGPU::sub0)
3278  return MachineOperand::CreateImm(static_cast<int32_t>(Op.getImm()));
3279  if (SubIdx == AMDGPU::sub1)
3280  return MachineOperand::CreateImm(static_cast<int32_t>(Op.getImm() >> 32));
3281 
3282  llvm_unreachable("Unhandled register index for immediate");
3283  }
3284 
3285  unsigned SubReg = buildExtractSubReg(MII, MRI, Op, SuperRC,
3286  SubIdx, SubRC);
3287  return MachineOperand::CreateReg(SubReg, false);
3288 }
3289 
3290 // Change the order of operands from (0, 1, 2) to (0, 2, 1)
3291 void SIInstrInfo::swapOperands(MachineInstr &Inst) const {
3292  assert(Inst.getNumExplicitOperands() == 3);
3293  MachineOperand Op1 = Inst.getOperand(1);
3294  Inst.RemoveOperand(1);
3295  Inst.addOperand(Op1);
3296 }
3297 
3299  const MCOperandInfo &OpInfo,
3300  const MachineOperand &MO) const {
3301  if (!MO.isReg())
3302  return false;
3303 
3304  unsigned Reg = MO.getReg();
3305  const TargetRegisterClass *RC =
3307  MRI.getRegClass(Reg) :
3308  RI.getPhysRegClass(Reg);
3309 
3310  const SIRegisterInfo *TRI =
3311  static_cast<const SIRegisterInfo*>(MRI.getTargetRegisterInfo());
3312  RC = TRI->getSubRegClass(RC, MO.getSubReg());
3313 
3314  // In order to be legal, the common sub-class must be equal to the
3315  // class of the current operand. For example:
3316  //
3317  // v_mov_b32 s0 ; Operand defined as vsrc_b32
3318  // ; RI.getCommonSubClass(s0,vsrc_b32) = sgpr ; LEGAL
3319  //
3320  // s_sendmsg 0, s0 ; Operand defined as m0reg
3321  // ; RI.getCommonSubClass(s0,m0reg) = m0reg ; NOT LEGAL
3322 
3323  return RI.getCommonSubClass(RC, RI.getRegClass(OpInfo.RegClass)) == RC;
3324 }
3325 
3327  const MCOperandInfo &OpInfo,
3328  const MachineOperand &MO) const {
3329  if (MO.isReg())
3330  return isLegalRegOperand(MRI, OpInfo, MO);
3331 
3332  // Handle non-register types that are treated like immediates.
3333  assert(MO.isImm() || MO.isTargetIndex() || MO.isFI());
3334  return true;
3335 }
3336 
3337 bool SIInstrInfo::isOperandLegal(const MachineInstr &MI, unsigned OpIdx,
3338  const MachineOperand *MO) const {
3340  const MCInstrDesc &InstDesc = MI.getDesc();
3341  const MCOperandInfo &OpInfo = InstDesc.OpInfo[OpIdx];
3342  const TargetRegisterClass *DefinedRC =
3343  OpInfo.RegClass != -1 ? RI.getRegClass(OpInfo.RegClass) : nullptr;
3344  if (!MO)
3345  MO = &MI.getOperand(OpIdx);
3346 
3347  if (isVALU(MI) && usesConstantBus(MRI, *MO, OpInfo)) {
3348 
3349  RegSubRegPair SGPRUsed;
3350  if (MO->isReg())
3351  SGPRUsed = RegSubRegPair(MO->getReg(), MO->getSubReg());
3352 
3353  for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
3354  if (i == OpIdx)
3355  continue;
3356  const MachineOperand &Op = MI.getOperand(i);
3357  if (Op.isReg()) {
3358  if ((Op.getReg() != SGPRUsed.Reg || Op.getSubReg() != SGPRUsed.SubReg) &&
3359  usesConstantBus(MRI, Op, InstDesc.OpInfo[i])) {
3360  return false;
3361  }
3362  } else if (InstDesc.OpInfo[i].OperandType == AMDGPU::OPERAND_KIMM32) {
3363  return false;
3364  }
3365  }
3366  }
3367 
3368  if (MO->isReg()) {
3369  assert(DefinedRC);
3370  return isLegalRegOperand(MRI, OpInfo, *MO);
3371  }
3372 
3373  // Handle non-register types that are treated like immediates.
3374  assert(MO->isImm() || MO->isTargetIndex() || MO->isFI());
3375 
3376  if (!DefinedRC) {
3377  // This operand expects an immediate.
3378  return true;
3379  }
3380 
3381  return isImmOperandLegal(MI, OpIdx, *MO);
3382 }
3383 
3385  MachineInstr &MI) const {
3386  unsigned Opc = MI.getOpcode();
3387  const MCInstrDesc &InstrDesc = get(Opc);
3388 
3389  int Src1Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1);
3390  MachineOperand &Src1 = MI.getOperand(Src1Idx);
3391 
3392  // If there is an implicit SGPR use such as VCC use for v_addc_u32/v_subb_u32
3393  // we need to only have one constant bus use.
3394  //
3395  // Note we do not need to worry about literal constants here. They are
3396  // disabled for the operand type for instructions because they will always
3397  // violate the one constant bus use rule.
3398  bool HasImplicitSGPR = findImplicitSGPRRead(MI) != AMDGPU::NoRegister;
3399  if (HasImplicitSGPR) {
3400  int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0);
3401  MachineOperand &Src0 = MI.getOperand(Src0Idx);
3402 
3403  if (Src0.isReg() && RI.isSGPRReg(MRI, Src0.getReg()))
3404  legalizeOpWithMove(MI, Src0Idx);
3405  }
3406 
3407  // Special case: V_WRITELANE_B32 accepts only immediate or SGPR operands for
3408  // both the value to write (src0) and lane select (src1). Fix up non-SGPR
3409  // src0/src1 with V_READFIRSTLANE.
3410  if (Opc == AMDGPU::V_WRITELANE_B32) {
3411  int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0);
3412  MachineOperand &Src0 = MI.getOperand(Src0Idx);
3413  const DebugLoc &DL = MI.getDebugLoc();
3414  if (Src0.isReg() && RI.isVGPR(MRI, Src0.getReg())) {
3415  unsigned Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
3416  BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg)
3417  .add(Src0);
3418  Src0.ChangeToRegister(Reg, false);
3419  }
3420  if (Src1.isReg() && RI.isVGPR(MRI, Src1.getReg())) {
3421  unsigned Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
3422  const DebugLoc &DL = MI.getDebugLoc();
3423  BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg)
3424  .add(Src1);
3425  Src1.ChangeToRegister(Reg, false);
3426  }
3427  return;
3428  }
3429 
3430  // VOP2 src0 instructions support all operand types, so we don't need to check
3431  // their legality. If src1 is already legal, we don't need to do anything.
3432  if (isLegalRegOperand(MRI, InstrDesc.OpInfo[Src1Idx], Src1))
3433  return;
3434 
3435  // Special case: V_READLANE_B32 accepts only immediate or SGPR operands for
3436  // lane select. Fix up using V_READFIRSTLANE, since we assume that the lane
3437  // select is uniform.
3438  if (Opc == AMDGPU::V_READLANE_B32 && Src1.isReg() &&
3439  RI.isVGPR(MRI, Src1.getReg())) {
3440  unsigned Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
3441  const DebugLoc &DL = MI.getDebugLoc();
3442  BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg)
3443  .add(Src1);
3444  Src1.ChangeToRegister(Reg, false);
3445  return;
3446  }
3447 
3448  // We do not use commuteInstruction here because it is too aggressive and will
3449  // commute if it is possible. We only want to commute here if it improves
3450  // legality. This can be called a fairly large number of times so don't waste
3451  // compile time pointlessly swapping and checking legality again.
3452  if (HasImplicitSGPR || !MI.isCommutable()) {
3453  legalizeOpWithMove(MI, Src1Idx);
3454  return;
3455  }
3456 
3457  int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0);
3458  MachineOperand &Src0 = MI.getOperand(Src0Idx);
3459 
3460  // If src0 can be used as src1, commuting will make the operands legal.
3461  // Otherwise we have to give up and insert a move.
3462  //
3463  // TODO: Other immediate-like operand kinds could be commuted if there was a
3464  // MachineOperand::ChangeTo* for them.
3465  if ((!Src1.isImm() && !Src1.isReg()) ||
3466  !isLegalRegOperand(MRI, InstrDesc.OpInfo[Src1Idx], Src0)) {
3467  legalizeOpWithMove(MI, Src1Idx);
3468  return;
3469  }
3470 
3471  int CommutedOpc = commuteOpcode(MI);
3472  if (CommutedOpc == -1) {
3473  legalizeOpWithMove(MI, Src1Idx);
3474  return;
3475  }
3476 
3477  MI.setDesc(get(CommutedOpc));
3478 
3479  unsigned Src0Reg = Src0.getReg();
3480  unsigned Src0SubReg = Src0.getSubReg();
3481  bool Src0Kill = Src0.isKill();
3482 
3483  if (Src1.isImm())
3484  Src0.ChangeToImmediate(Src1.getImm());
3485  else if (Src1.isReg()) {
3486  Src0.ChangeToRegister(Src1.getReg(), false, false, Src1.isKill());
3487  Src0.setSubReg(Src1.getSubReg());
3488  } else
3489  llvm_unreachable("Should only have register or immediate operands");
3490 
3491  Src1.ChangeToRegister(Src0Reg, false, false, Src0Kill);
3492  Src1.setSubReg(Src0SubReg);
3493 }
3494 
3495 // Legalize VOP3 operands. Because all operand types are supported for any
3496 // operand, and since literal constants are not allowed and should never be
3497 // seen, we only need to worry about inserting copies if we use multiple SGPR
3498 // operands.
3500  MachineInstr &MI) const {
3501  unsigned Opc = MI.getOpcode();
3502 
3503  int VOP3Idx[3] = {
3504  AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0),
3505  AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1),
3506  AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2)
3507  };
3508 
3509  // Find the one SGPR operand we are allowed to use.
3510  unsigned SGPRReg = findUsedSGPR(MI, VOP3Idx);
3511 
3512  for (unsigned i = 0; i < 3; ++i) {
3513  int Idx = VOP3Idx[i];
3514  if (Idx == -1)
3515  break;
3516  MachineOperand &MO = MI.getOperand(Idx);
3517 
3518  // We should never see a VOP3 instruction with an illegal immediate operand.
3519  if (!MO.isReg())
3520  continue;
3521 
3522  if (!RI.isSGPRClass(MRI.getRegClass(MO.getReg())))
3523  continue; // VGPRs are legal
3524 
3525  if (SGPRReg == AMDGPU::NoRegister || SGPRReg == MO.getReg()) {
3526  SGPRReg = MO.getReg();
3527  // We can use one SGPR in each VOP3 instruction.
3528  continue;
3529  }
3530 
3531  // If we make it this far, then the operand is not legal and we must
3532  // legalize it.
3533  legalizeOpWithMove(MI, Idx);
3534  }
3535 }
3536 
3538  MachineRegisterInfo &MRI) const {
3539  const TargetRegisterClass *VRC = MRI.getRegClass(SrcReg);
3540  const TargetRegisterClass *SRC = RI.getEquivalentSGPRClass(VRC);
3541  unsigned DstReg = MRI.createVirtualRegister(SRC);
3542  unsigned SubRegs = RI.getRegSizeInBits(*VRC) / 32;
3543 
3544  if (SubRegs == 1) {
3545  BuildMI(*UseMI.getParent(), UseMI, UseMI.getDebugLoc(),
3546  get(AMDGPU::V_READFIRSTLANE_B32), DstReg)
3547  .addReg(SrcReg);
3548  return DstReg;
3549  }
3550 
3552  for (unsigned i = 0; i < SubRegs; ++i) {
3553  unsigned SGPR = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
3554  BuildMI(*UseMI.getParent(), UseMI, UseMI.getDebugLoc(),
3555  get(AMDGPU::V_READFIRSTLANE_B32), SGPR)
3556  .addReg(SrcReg, 0, RI.getSubRegFromChannel(i));
3557  SRegs.push_back(SGPR);
3558  }
3559 
3560  MachineInstrBuilder MIB =
3561  BuildMI(*UseMI.getParent(), UseMI, UseMI.getDebugLoc(),
3562  get(AMDGPU::REG_SEQUENCE), DstReg);
3563  for (unsigned i = 0; i < SubRegs; ++i) {
3564  MIB.addReg(SRegs[i]);
3565  MIB.addImm(RI.getSubRegFromChannel(i));
3566  }
3567  return DstReg;
3568 }
3569 
3571  MachineInstr &MI) const {
3572 
3573  // If the pointer is store in VGPRs, then we need to move them to
3574  // SGPRs using v_readfirstlane. This is safe because we only select
3575  // loads with uniform pointers to SMRD instruction so we know the
3576  // pointer value is uniform.
3577  MachineOperand *SBase = getNamedOperand(MI, AMDGPU::OpName::sbase);
3578  if (SBase && !RI.isSGPRClass(MRI.getRegClass(SBase->getReg()))) {
3579  unsigned SGPR = readlaneVGPRToSGPR(SBase->getReg(), MI, MRI);
3580  SBase->setReg(SGPR);
3581  }
3582  MachineOperand *SOff = getNamedOperand(MI, AMDGPU::OpName::soff);
3583  if (SOff && !RI.isSGPRClass(MRI.getRegClass(SOff->getReg()))) {
3584  unsigned SGPR = readlaneVGPRToSGPR(SOff->getReg(), MI, MRI);
3585  SOff->setReg(SGPR);
3586  }
3587 }
3588 
3591  const TargetRegisterClass *DstRC,
3592  MachineOperand &Op,
3594  const DebugLoc &DL) const {
3595  unsigned OpReg = Op.getReg();
3596  unsigned OpSubReg = Op.getSubReg();
3597 
3598  const TargetRegisterClass *OpRC = RI.getSubClassWithSubReg(
3599  RI.getRegClassForReg(MRI, OpReg), OpSubReg);
3600 
3601  // Check if operand is already the correct register class.
3602  if (DstRC == OpRC)
3603  return;
3604 
3605  unsigned DstReg = MRI.createVirtualRegister(DstRC);
3606  MachineInstr *Copy =
3607  BuildMI(InsertMBB, I, DL, get(AMDGPU::COPY), DstReg).add(Op);
3608 
3609  Op.setReg(DstReg);
3610  Op.setSubReg(0);
3611 
3612  MachineInstr *Def = MRI.getVRegDef(OpReg);
3613  if (!Def)
3614  return;
3615 
3616  // Try to eliminate the copy if it is copying an immediate value.
3617  if (Def->isMoveImmediate())
3618  FoldImmediate(*Copy, *Def, OpReg, &MRI);
3619 }
3620 
3621 // Emit the actual waterfall loop, executing the wrapped instruction for each
3622 // unique value of \p Rsrc across all lanes. In the best case we execute 1
3623 // iteration, in the worst case we execute 64 (once per lane).
3624 static void
3626  MachineBasicBlock &OrigBB, MachineBasicBlock &LoopBB,
3627  const DebugLoc &DL, MachineOperand &Rsrc) {
3628  MachineBasicBlock::iterator I = LoopBB.begin();
3629 
3630  unsigned VRsrc = Rsrc.getReg();
3631  unsigned VRsrcUndef = getUndefRegState(Rsrc.isUndef());
3632 
3633  unsigned SaveExec = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
3634  unsigned CondReg0 = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
3635  unsigned CondReg1 = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
3636  unsigned AndCond = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
3637  unsigned SRsrcSub0 = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
3638  unsigned SRsrcSub1 = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
3639  unsigned SRsrcSub2 = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
3640  unsigned SRsrcSub3 = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
3641  unsigned SRsrc = MRI.createVirtualRegister(&AMDGPU::SReg_128RegClass);
3642 
3643  // Beginning of the loop, read the next Rsrc variant.
3644  BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), SRsrcSub0)
3645  .addReg(VRsrc, VRsrcUndef, AMDGPU::sub0);
3646  BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), SRsrcSub1)
3647  .addReg(VRsrc, VRsrcUndef, AMDGPU::sub1);
3648  BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), SRsrcSub2)
3649  .addReg(VRsrc, VRsrcUndef, AMDGPU::sub2);
3650  BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), SRsrcSub3)
3651  .addReg(VRsrc, VRsrcUndef, AMDGPU::sub3);
3652 
3653  BuildMI(LoopBB, I, DL, TII.get(AMDGPU::REG_SEQUENCE), SRsrc)
3654  .addReg(SRsrcSub0)
3655  .addImm(AMDGPU::sub0)
3656  .addReg(SRsrcSub1)
3657  .addImm(AMDGPU::sub1)
3658  .addReg(SRsrcSub2)
3659  .addImm(AMDGPU::sub2)
3660  .addReg(SRsrcSub3)
3661  .addImm(AMDGPU::sub3);
3662 
3663  // Update Rsrc operand to use the SGPR Rsrc.
3664  Rsrc.setReg(SRsrc);
3665  Rsrc.setIsKill(true);
3666 
3667  // Identify all lanes with identical Rsrc operands in their VGPRs.
3668  BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_CMP_EQ_U64_e64), CondReg0)
3669  .addReg(SRsrc, 0, AMDGPU::sub0_sub1)
3670  .addReg(VRsrc, 0, AMDGPU::sub0_sub1);
3671  BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_CMP_EQ_U64_e64), CondReg1)
3672  .addReg(SRsrc, 0, AMDGPU::sub2_sub3)
3673  .addReg(VRsrc, 0, AMDGPU::sub2_sub3);
3674  BuildMI(LoopBB, I, DL, TII.get(AMDGPU::S_AND_B64), AndCond)
3675  .addReg(CondReg0)
3676  .addReg(CondReg1);
3677 
3678  MRI.setSimpleHint(SaveExec, AndCond);
3679 
3680  // Update EXEC to matching lanes, saving original to SaveExec.
3681  BuildMI(LoopBB, I, DL, TII.get(AMDGPU::S_AND_SAVEEXEC_B64), SaveExec)
3682  .addReg(AndCond, RegState::Kill);
3683 
3684  // The original instruction is here; we insert the terminators after it.
3685  I = LoopBB.end();
3686 
3687  // Update EXEC, switch all done bits to 0 and all todo bits to 1.
3688  BuildMI(LoopBB, I, DL, TII.get(AMDGPU::S_XOR_B64_term), AMDGPU::EXEC)
3689  .addReg(AMDGPU::EXEC)
3690  .addReg(SaveExec);
3691  BuildMI(LoopBB, I, DL, TII.get(AMDGPU::S_CBRANCH_EXECNZ)).addMBB(&LoopBB);
3692 }
3693 
3694 // Build a waterfall loop around \p MI, replacing the VGPR \p Rsrc register
3695 // with SGPRs by iterating over all unique values across all lanes.
3697  MachineOperand &Rsrc, MachineDominatorTree *MDT) {
3698  MachineBasicBlock &MBB = *MI.getParent();
3699  MachineFunction &MF = *MBB.getParent();
3702  const DebugLoc &DL = MI.getDebugLoc();
3703 
3704  unsigned SaveExec = MRI.createVirtualRegister(&AMDGPU::SReg_64_XEXECRegClass);
3705 
3706  // Save the EXEC mask
3707  BuildMI(MBB, I, DL, TII.get(AMDGPU::S_MOV_B64), SaveExec)
3708  .addReg(AMDGPU::EXEC);
3709 
3710  // Killed uses in the instruction we are waterfalling around will be
3711  // incorrect due to the added control-flow.
3712  for (auto &MO : MI.uses()) {
3713  if (MO.isReg() && MO.isUse()) {
3714  MRI.clearKillFlags(MO.getReg());
3715  }
3716  }
3717 
3718  // To insert the loop we need to split the block. Move everything after this
3719  // point to a new block, and insert a new empty block between the two.
3721  MachineBasicBlock *RemainderBB = MF.CreateMachineBasicBlock();
3722  MachineFunction::iterator MBBI(MBB);
3723  ++MBBI;
3724 
3725  MF.insert(MBBI, LoopBB);
3726  MF.insert(MBBI, RemainderBB);
3727 
3728  LoopBB->addSuccessor(LoopBB);
3729  LoopBB->addSuccessor(RemainderBB);
3730 
3731  // Move MI to the LoopBB, and the remainder of the block to RemainderBB.
3733  RemainderBB->transferSuccessorsAndUpdatePHIs(&MBB);
3734  RemainderBB->splice(RemainderBB->begin(), &MBB, I, MBB.end());
3735  LoopBB->splice(LoopBB->begin(), &MBB, J);
3736 
3737  MBB.addSuccessor(LoopBB);
3738 
3739  // Update dominators. We know that MBB immediately dominates LoopBB, that
3740  // LoopBB immediately dominates RemainderBB, and that RemainderBB immediately
3741  // dominates all of the successors transferred to it from MBB that MBB used
3742  // to dominate.
3743  if (MDT) {
3744  MDT->addNewBlock(LoopBB, &MBB);
3745  MDT->addNewBlock(RemainderBB, LoopBB);
3746  for (auto &Succ : RemainderBB->successors()) {
3747  if (MDT->dominates(&MBB, Succ)) {
3748  MDT->changeImmediateDominator(Succ, RemainderBB);
3749  }
3750  }
3751  }
3752 
3753  emitLoadSRsrcFromVGPRLoop(TII, MRI, MBB, *LoopBB, DL, Rsrc);
3754 
3755  // Restore the EXEC mask
3756  MachineBasicBlock::iterator First = RemainderBB->begin();
3757  BuildMI(*RemainderBB, First, DL, TII.get(AMDGPU::S_MOV_B64), AMDGPU::EXEC)
3758  .addReg(SaveExec);
3759 }
3760 
3761 // Extract pointer from Rsrc and return a zero-value Rsrc replacement.
3762 static std::tuple<unsigned, unsigned>
3764  MachineBasicBlock &MBB = *MI.getParent();
3765  MachineFunction &MF = *MBB.getParent();
3767 
3768  // Extract the ptr from the resource descriptor.
3769  unsigned RsrcPtr =
3770  TII.buildExtractSubReg(MI, MRI, Rsrc, &AMDGPU::VReg_128RegClass,
3771  AMDGPU::sub0_sub1, &AMDGPU::VReg_64RegClass);
3772 
3773  // Create an empty resource descriptor
3774  unsigned Zero64 = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
3775  unsigned SRsrcFormatLo = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
3776  unsigned SRsrcFormatHi = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
3777  unsigned NewSRsrc = MRI.createVirtualRegister(&AMDGPU::SReg_128RegClass);
3778  uint64_t RsrcDataFormat = TII.getDefaultRsrcDataFormat();
3779 
3780  // Zero64 = 0
3781  BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::S_MOV_B64), Zero64)
3782  .addImm(0);
3783 
3784  // SRsrcFormatLo = RSRC_DATA_FORMAT{31-0}
3785  BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::S_MOV_B32), SRsrcFormatLo)
3786  .addImm(RsrcDataFormat & 0xFFFFFFFF);
3787 
3788  // SRsrcFormatHi = RSRC_DATA_FORMAT{63-32}
3789  BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::S_MOV_B32), SRsrcFormatHi)
3790  .addImm(RsrcDataFormat >> 32);
3791 
3792  // NewSRsrc = {Zero64, SRsrcFormat}
3793  BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::REG_SEQUENCE), NewSRsrc)
3794  .addReg(Zero64)
3795  .addImm(AMDGPU::sub0_sub1)
3796  .addReg(SRsrcFormatLo)
3797  .addImm(AMDGPU::sub2)
3798  .addReg(SRsrcFormatHi)
3799  .addImm(AMDGPU::sub3);
3800 
3801  return std::make_tuple(RsrcPtr, NewSRsrc);
3802 }
3803 
3805  MachineDominatorTree *MDT) const {
3806  MachineFunction &MF = *MI.getParent()->getParent();
3808 
3809  // Legalize VOP2
3810  if (isVOP2(MI) || isVOPC(MI)) {
3811  legalizeOperandsVOP2(MRI, MI);
3812  return;
3813  }
3814 
3815  // Legalize VOP3
3816  if (isVOP3(MI)) {
3817  legalizeOperandsVOP3(MRI, MI);
3818  return;
3819  }
3820 
3821  // Legalize SMRD
3822  if (isSMRD(MI)) {
3823  legalizeOperandsSMRD(MRI, MI);
3824  return;
3825  }
3826 
3827  // Legalize REG_SEQUENCE and PHI
3828  // The register class of the operands much be the same type as the register
3829  // class of the output.
3830  if (MI.getOpcode() == AMDGPU::PHI) {
3831  const TargetRegisterClass *RC = nullptr, *SRC = nullptr, *VRC = nullptr;
3832  for (unsigned i = 1, e = MI.getNumOperands(); i != e; i += 2) {
3833  if (!MI.getOperand(i).isReg() ||
3835  continue;
3836  const TargetRegisterClass *OpRC =
3837  MRI.getRegClass(MI.getOperand(i).getReg());
3838  if (RI.hasVGPRs(OpRC)) {
3839  VRC = OpRC;
3840  } else {
3841  SRC = OpRC;
3842  }
3843  }
3844 
3845  // If any of the operands are VGPR registers, then they all most be
3846  // otherwise we will create illegal VGPR->SGPR copies when legalizing
3847  // them.
3848  if (VRC || !RI.isSGPRClass(getOpRegClass(MI, 0))) {
3849  if (!VRC) {
3850  assert(SRC);
3851  VRC = RI.getEquivalentVGPRClass(SRC);
3852  }
3853  RC = VRC;
3854  } else {
3855  RC = SRC;
3856  }
3857 
3858  // Update all the operands so they have the same type.
3859  for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) {
3860  MachineOperand &Op = MI.getOperand(I);
3862  continue;
3863 
3864  // MI is a PHI instruction.
3865  MachineBasicBlock *InsertBB = MI.getOperand(I + 1).getMBB();
3866  MachineBasicBlock::iterator Insert = InsertBB->getFirstTerminator();
3867 
3868  // Avoid creating no-op copies with the same src and dst reg class. These
3869  // confuse some of the machine passes.
3870  legalizeGenericOperand(*InsertBB, Insert, RC, Op, MRI, MI.getDebugLoc());
3871  }
3872  }
3873 
3874  // REG_SEQUENCE doesn't really require operand legalization, but if one has a
3875  // VGPR dest type and SGPR sources, insert copies so all operands are
3876  // VGPRs. This seems to help operand folding / the register coalescer.
3877  if (MI.getOpcode() == AMDGPU::REG_SEQUENCE) {
3878  MachineBasicBlock *MBB = MI.getParent();
3879  const TargetRegisterClass *DstRC = getOpRegClass(MI, 0);
3880  if (RI.hasVGPRs(DstRC)) {
3881  // Update all the operands so they are VGPR register classes. These may
3882  // not be the same register class because REG_SEQUENCE supports mixing
3883  // subregister index types e.g. sub0_sub1 + sub2 + sub3
3884  for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) {
3885  MachineOperand &Op = MI.getOperand(I);
3887  continue;
3888 
3889  const TargetRegisterClass *OpRC = MRI.getRegClass(Op.getReg());
3890  const TargetRegisterClass *VRC = RI.getEquivalentVGPRClass(OpRC);
3891  if (VRC == OpRC)
3892  continue;
3893 
3894  legalizeGenericOperand(*MBB, MI, VRC, Op, MRI, MI.getDebugLoc());
3895  Op.setIsKill();
3896  }
3897  }
3898 
3899  return;
3900  }
3901 
3902  // Legalize INSERT_SUBREG
3903  // src0 must have the same register class as dst
3904  if (MI.getOpcode() == AMDGPU::INSERT_SUBREG) {
3905  unsigned Dst = MI.getOperand(0).getReg();
3906  unsigned Src0 = MI.getOperand(1).getReg();
3907  const TargetRegisterClass *DstRC = MRI.getRegClass(Dst);
3908  const TargetRegisterClass *Src0RC = MRI.getRegClass(Src0);
3909  if (DstRC != Src0RC) {
3910  MachineBasicBlock *MBB = MI.getParent();
3911  MachineOperand &Op = MI.getOperand(1);
3912  legalizeGenericOperand(*MBB, MI, DstRC, Op, MRI, MI.getDebugLoc());
3913  }
3914  return;
3915  }
3916 
3917  // Legalize SI_INIT_M0
3918  if (MI.getOpcode() == AMDGPU::SI_INIT_M0) {
3919  MachineOperand &Src = MI.getOperand(0);
3920  if (Src.isReg() && RI.hasVGPRs(MRI.getRegClass(Src.getReg())))
3921  Src.setReg(readlaneVGPRToSGPR(Src.getReg(), MI, MRI));
3922  return;
3923  }
3924 
3925  // Legalize MIMG and MUBUF/MTBUF for shaders.
3926  //
3927  // Shaders only generate MUBUF/MTBUF instructions via intrinsics or via
3928  // scratch memory access. In both cases, the legalization never involves
3929  // conversion to the addr64 form.
3930  if (isMIMG(MI) ||
3932  (isMUBUF(MI) || isMTBUF(MI)))) {
3933  MachineOperand *SRsrc = getNamedOperand(MI, AMDGPU::OpName::srsrc);
3934  if (SRsrc && !RI.isSGPRClass(MRI.getRegClass(SRsrc->getReg()))) {
3935  unsigned SGPR = readlaneVGPRToSGPR(SRsrc->getReg(), MI, MRI);
3936  SRsrc->setReg(SGPR);
3937  }
3938 
3939  MachineOperand *SSamp = getNamedOperand(MI, AMDGPU::OpName::ssamp);
3940  if (SSamp && !RI.isSGPRClass(MRI.getRegClass(SSamp->getReg()))) {
3941  unsigned SGPR = readlaneVGPRToSGPR(SSamp->getReg(), MI, MRI);
3942  SSamp->setReg(SGPR);
3943  }
3944  return;
3945  }
3946 
3947  // Legalize MUBUF* instructions.
3948  int RsrcIdx =
3949  AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::srsrc);
3950  if (RsrcIdx != -1) {
3951  // We have an MUBUF instruction
3952  MachineOperand *Rsrc = &MI.getOperand(RsrcIdx);
3953  unsigned RsrcRC = get(MI.getOpcode()).OpInfo[RsrcIdx].RegClass;
3954  if (RI.getCommonSubClass(MRI.getRegClass(Rsrc->getReg()),
3955  RI.getRegClass(RsrcRC))) {
3956  // The operands are legal.
3957  // FIXME: We may need to legalize operands besided srsrc.
3958  return;
3959  }
3960 
3961  // Legalize a VGPR Rsrc.
3962  //
3963  // If the instruction is _ADDR64, we can avoid a waterfall by extracting
3964  // the base pointer from the VGPR Rsrc, adding it to the VAddr, then using
3965  // a zero-value SRsrc.
3966  //
3967  // If the instruction is _OFFSET (both idxen and offen disabled), and we
3968  // support ADDR64 instructions, we can convert to ADDR64 and do the same as
3969  // above.
3970  //
3971  // Otherwise we are on non-ADDR64 hardware, and/or we have
3972  // idxen/offen/bothen and we fall back to a waterfall loop.
3973 
3974  MachineBasicBlock &MBB = *MI.getParent();
3975 
3976  MachineOperand *VAddr = getNamedOperand(MI, AMDGPU::OpName::vaddr);
3977  if (VAddr && AMDGPU::getIfAddr64Inst(MI.getOpcode()) != -1) {
3978  // This is already an ADDR64 instruction so we need to add the pointer
3979  // extracted from the resource descriptor to the current value of VAddr.
3980  unsigned NewVAddrLo = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
3981  unsigned NewVAddrHi = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
3982  unsigned NewVAddr = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass);
3983 
3984  unsigned RsrcPtr, NewSRsrc;
3985  std::tie(RsrcPtr, NewSRsrc) = extractRsrcPtr(*this, MI, *Rsrc);
3986 
3987  // NewVaddrLo = RsrcPtr:sub0 + VAddr:sub0
3988  DebugLoc DL = MI.getDebugLoc();
3989  BuildMI(MBB, MI, DL, get(AMDGPU::V_ADD_I32_e32), NewVAddrLo)
3990  .addReg(RsrcPtr, 0, AMDGPU::sub0)
3991  .addReg(VAddr->getReg(), 0, AMDGPU::sub0);
3992 
3993  // NewVaddrHi = RsrcPtr:sub1 + VAddr:sub1
3994  BuildMI(MBB, MI, DL, get(AMDGPU::V_ADDC_U32_e32), NewVAddrHi)
3995  .addReg(RsrcPtr, 0, AMDGPU::sub1)
3996  .addReg(VAddr->getReg(), 0, AMDGPU::sub1);
3997 
3998  // NewVaddr = {NewVaddrHi, NewVaddrLo}
3999  BuildMI(MBB, MI, MI.getDebugLoc(), get(AMDGPU::REG_SEQUENCE), NewVAddr)
4000  .addReg(NewVAddrLo)
4001  .addImm(AMDGPU::sub0)
4002  .addReg(NewVAddrHi)
4003  .addImm(AMDGPU::sub1);
4004 
4005  VAddr->setReg(NewVAddr);
4006  Rsrc->setReg(NewSRsrc);
4007  } else if (!VAddr && ST.hasAddr64()) {
4008  // This instructions is the _OFFSET variant, so we need to convert it to
4009  // ADDR64.
4010  assert(MBB.getParent()->getSubtarget<GCNSubtarget>().getGeneration()
4012  "FIXME: Need to emit flat atomics here");
4013 
4014  unsigned RsrcPtr, NewSRsrc;
4015  std::tie(RsrcPtr, NewSRsrc) = extractRsrcPtr(*this, MI, *Rsrc);
4016 
4017  unsigned NewVAddr = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass);
4018  MachineOperand *VData = getNamedOperand(MI, AMDGPU::OpName::vdata);
4019  MachineOperand *Offset = getNamedOperand(MI, AMDGPU::OpName::offset);
4020  MachineOperand *SOffset = getNamedOperand(MI, AMDGPU::OpName::soffset);
4021  unsigned Addr64Opcode = AMDGPU::getAddr64Inst(MI.getOpcode());
4022 
4023  // Atomics rith return have have an additional tied operand and are
4024  // missing some of the special bits.
4025  MachineOperand *VDataIn = getNamedOperand(MI, AMDGPU::OpName::vdata_in);
4026  MachineInstr *Addr64;
4027 
4028  if (!VDataIn) {
4029  // Regular buffer load / store.
4030  MachineInstrBuilder MIB =
4031  BuildMI(MBB, MI, MI.getDebugLoc(), get(Addr64Opcode))
4032  .add(*VData)
4033  .addReg(NewVAddr)
4034  .addReg(NewSRsrc)
4035  .add(*SOffset)
4036  .add(*Offset);
4037 
4038  // Atomics do not have this operand.
4039  if (const MachineOperand *GLC =
4040  getNamedOperand(MI, AMDGPU::OpName::glc)) {
4041  MIB.addImm(GLC->getImm());
4042  }
4043 
4044  MIB.addImm(getNamedImmOperand(MI, AMDGPU::OpName::slc));
4045 
4046  if (const MachineOperand *TFE =
4047  getNamedOperand(MI, AMDGPU::OpName::tfe)) {
4048  MIB.addImm(TFE->getImm());
4049  }
4050 
4051  MIB.cloneMemRefs(MI);
4052  Addr64 = MIB;
4053  } else {
4054  // Atomics with return.
4055  Addr64 = BuildMI(MBB, MI, MI.getDebugLoc(), get(Addr64Opcode))
4056  .add(*VData)
4057  .add(*VDataIn)
4058  .addReg(NewVAddr)
4059  .addReg(NewSRsrc)
4060  .add(*SOffset)
4061  .add(*Offset)
4062  .addImm(getNamedImmOperand(MI, AMDGPU::OpName::slc))
4063  .cloneMemRefs(MI);
4064  }
4065 
4066  MI.removeFromParent();
4067 
4068  // NewVaddr = {NewVaddrHi, NewVaddrLo}
4069  BuildMI(MBB, Addr64, Addr64->getDebugLoc(), get(AMDGPU::REG_SEQUENCE),
4070  NewVAddr)
4071  .addReg(RsrcPtr, 0, AMDGPU::sub0)
4072  .addImm(AMDGPU::sub0)
4073  .addReg(RsrcPtr, 0, AMDGPU::sub1)
4074  .addImm(AMDGPU::sub1);
4075  } else {
4076  // This is another variant; legalize Rsrc with waterfall loop from VGPRs
4077  // to SGPRs.
4078  loadSRsrcFromVGPR(*this, MI, *Rsrc, MDT);
4079  }
4080  }
4081 }
4082 
4084  MachineDominatorTree *MDT) const {
4085  SetVectorType Worklist;
4086  Worklist.insert(&TopInst);
4087 
4088  while (!Worklist.empty()) {
4089  MachineInstr &Inst = *Worklist.pop_back_val();
4090  MachineBasicBlock *MBB = Inst.getParent();
4092 
4093  unsigned Opcode = Inst.getOpcode();
4094  unsigned NewOpcode = getVALUOp(Inst);
4095 
4096  // Handle some special cases
4097  switch (Opcode) {
4098  default:
4099  break;
4100  case AMDGPU::S_ADD_U64_PSEUDO:
4101  case AMDGPU::S_SUB_U64_PSEUDO:
4102  splitScalar64BitAddSub(Worklist, Inst, MDT);
4103  Inst.eraseFromParent();
4104  continue;
4105  case AMDGPU::S_ADD_I32:
4106  case AMDGPU::S_SUB_I32:
4107  // FIXME: The u32 versions currently selected use the carry.
4108  if (moveScalarAddSub(Worklist, Inst, MDT))
4109  continue;
4110 
4111  // Default handling
4112  break;
4113  case AMDGPU::S_AND_B64:
4114  splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::V_AND_B32_e64, MDT);
4115  Inst.eraseFromParent();
4116  continue;
4117 
4118  case AMDGPU::S_OR_B64:
4119  splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::V_OR_B32_e64, MDT);
4120  Inst.eraseFromParent();
4121  continue;
4122 
4123  case AMDGPU::S_XOR_B64:
4124  splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::V_XOR_B32_e64, MDT);
4125  Inst.eraseFromParent();
4126  continue;
4127 
4128  case AMDGPU::S_NOT_B64:
4129  splitScalar64BitUnaryOp(Worklist, Inst, AMDGPU::V_NOT_B32_e32);
4130  Inst.eraseFromParent();
4131  continue;
4132 
4133  case AMDGPU::S_BCNT1_I32_B64:
4134  splitScalar64BitBCNT(Worklist, Inst);
4135  Inst.eraseFromParent();
4136  continue;
4137 
4138  case AMDGPU::S_BFE_I64:
4139  splitScalar64BitBFE(Worklist, Inst);
4140  Inst.eraseFromParent();
4141  continue;
4142 
4143  case AMDGPU::S_LSHL_B32:
4144  if (ST.getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS) {
4145  NewOpcode = AMDGPU::V_LSHLREV_B32_e64;
4146  swapOperands(Inst);
4147  }
4148  break;
4149  case AMDGPU::S_ASHR_I32:
4150  if (ST.getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS) {
4151  NewOpcode = AMDGPU::V_ASHRREV_I32_e64;
4152  swapOperands(Inst);
4153  }
4154  break;
4155  case AMDGPU::S_LSHR_B32:
4156  if (ST.getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS) {
4157  NewOpcode = AMDGPU::V_LSHRREV_B32_e64;
4158  swapOperands(Inst);
4159  }
4160  break;
4161  case AMDGPU::S_LSHL_B64:
4162  if (ST.getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS) {
4163  NewOpcode = AMDGPU::V_LSHLREV_B64;
4164  swapOperands(Inst);
4165  }
4166  break;
4167  case AMDGPU::S_ASHR_I64:
4168  if (ST.getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS) {
4169  NewOpcode = AMDGPU::V_ASHRREV_I64;
4170  swapOperands(Inst);
4171  }
4172  break;
4173  case AMDGPU::S_LSHR_B64:
4174  if (ST.getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS) {
4175  NewOpcode = AMDGPU::V_LSHRREV_B64;
4176  swapOperands(Inst);
4177  }
4178  break;
4179 
4180  case AMDGPU::S_ABS_I32:
4181  lowerScalarAbs(Worklist, Inst);
4182  Inst.eraseFromParent();
4183  continue;
4184 
4185  case AMDGPU::S_CBRANCH_SCC0:
4186  case AMDGPU::S_CBRANCH_SCC1:
4187  // Clear unused bits of vcc
4188  BuildMI(*MBB, Inst, Inst.getDebugLoc(), get(AMDGPU::S_AND_B64),
4189  AMDGPU::VCC)
4190  .addReg(AMDGPU::EXEC)
4191  .addReg(AMDGPU::VCC);
4192  break;
4193 
4194  case AMDGPU::S_BFE_U64:
4195  case AMDGPU::S_BFM_B64:
4196  llvm_unreachable("Moving this op to VALU not implemented");
4197 
4198  case AMDGPU::S_PACK_LL_B32_B16:
4199  case AMDGPU::S_PACK_LH_B32_B16:
4200  case AMDGPU::S_PACK_HH_B32_B16:
4201  movePackToVALU(Worklist, MRI, Inst);
4202  Inst.eraseFromParent();
4203  continue;
4204 
4205  case AMDGPU::S_XNOR_B32:
4206  lowerScalarXnor(Worklist, Inst);
4207  Inst.eraseFromParent();
4208  continue;
4209 
4210  case AMDGPU::S_XNOR_B64:
4211  splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_XNOR_B32, MDT);
4212  Inst.eraseFromParent();
4213  continue;
4214  }
4215 
4216  if (NewOpcode == AMDGPU::INSTRUCTION_LIST_END) {
4217  // We cannot move this instruction to the VALU, so we should try to
4218  // legalize its operands instead.
4219  legalizeOperands(Inst, MDT);
4220  continue;
4221  }
4222 
4223  // Use the new VALU Opcode.
4224  const MCInstrDesc &NewDesc = get(NewOpcode);
4225  Inst.setDesc(NewDesc);
4226 
4227  // Remove any references to SCC. Vector instructions can't read from it, and
4228  // We're just about to add the implicit use / defs of VCC, and we don't want
4229  // both.
4230  for (unsigned i = Inst.getNumOperands() - 1; i > 0; --i) {
4231  MachineOperand &Op = Inst.getOperand(i);
4232  if (Op.isReg() && Op.getReg() == AMDGPU::SCC) {
4233  Inst.RemoveOperand(i);
4234  addSCCDefUsersToVALUWorklist(Inst, Worklist);
4235  }
4236  }
4237 
4238  if (Opcode == AMDGPU::S_SEXT_I32_I8 || Opcode == AMDGPU::S_SEXT_I32_I16) {
4239  // We are converting these to a BFE, so we need to add the missing
4240  // operands for the size and offset.
4241  unsigned Size = (Opcode == AMDGPU::S_SEXT_I32_I8) ? 8 : 16;
4244 
4245  } else if (Opcode == AMDGPU::S_BCNT1_I32_B32) {
4246  // The VALU version adds the second operand to the result, so insert an
4247  // extra 0 operand.
4249  }
4250 
4252 
4253  if (Opcode == AMDGPU::S_BFE_I32 || Opcode == AMDGPU::S_BFE_U32) {
4254  const MachineOperand &OffsetWidthOp = Inst.getOperand(2);
4255  // If we need to move this to VGPRs, we need to unpack the second operand
4256  // back into the 2 separate ones for bit offset and width.
4257  assert(OffsetWidthOp.isImm() &&
4258  "Scalar BFE is only implemented for constant width and offset");
4259  uint32_t Imm = OffsetWidthOp.getImm();
4260 
4261  uint32_t Offset = Imm & 0x3f; // Extract bits [5:0].
4262  uint32_t BitWidth = (Imm & 0x7f0000) >> 16; // Extract bits [22:16].
4263  Inst.RemoveOperand(2); // Remove old immediate.
4264  Inst.addOperand(MachineOperand::CreateImm(Offset));
4265  Inst.addOperand(MachineOperand::CreateImm(BitWidth));
4266  }
4267 
4268  bool HasDst = Inst.getOperand(0).isReg() && Inst.getOperand(0).isDef();
4269  unsigned NewDstReg = AMDGPU::NoRegister;
4270  if (HasDst) {
4271  unsigned DstReg = Inst.getOperand(0).getReg();
4273  continue;
4274 
4275  // Update the destination register class.
4276  const TargetRegisterClass *NewDstRC = getDestEquivalentVGPRClass(Inst);
4277  if (!NewDstRC)
4278  continue;
4279 
4280  if (Inst.isCopy() &&
4282  NewDstRC == RI.getRegClassForReg(MRI, Inst.getOperand(1).getReg())) {
4283  // Instead of creating a copy where src and dst are the same register
4284  // class, we just replace all uses of dst with src. These kinds of
4285  // copies interfere with the heuristics MachineSink uses to decide
4286  // whether or not to split a critical edge. Since the pass assumes
4287  // that copies will end up as machine instructions and not be
4288  // eliminated.
4289  addUsersToMoveToVALUWorklist(DstReg, MRI, Worklist);
4290  MRI.replaceRegWith(DstReg, Inst.getOperand(1).getReg());
4291  MRI.clearKillFlags(Inst.getOperand(1).getReg());
4292  Inst.getOperand(0).setReg(DstReg);
4293 
4294  // Make sure we don't leave around a dead VGPR->SGPR copy. Normally
4295  // these are deleted later, but at -O0 it would leave a suspicious
4296  // looking illegal copy of an undef register.
4297  for (unsigned I = Inst.getNumOperands() - 1; I != 0; --I)
4298  Inst.RemoveOperand(I);
4299  Inst.setDesc(get(AMDGPU::IMPLICIT_DEF));
4300  continue;
4301  }
4302 
4303  NewDstReg = MRI.createVirtualRegister(NewDstRC);
4304  MRI.replaceRegWith(DstReg, NewDstReg);
4305  }
4306 
4307  // Legalize the operands
4308  legalizeOperands(Inst, MDT);
4309 
4310  if (HasDst)
4311  addUsersToMoveToVALUWorklist(NewDstReg, MRI, Worklist);
4312  }
4313 }
4314 
4315 // Add/sub require special handling to deal with carry outs.
4316 bool SIInstrInfo::moveScalarAddSub(SetVectorType &Worklist, MachineInstr &Inst,
4317  MachineDominatorTree *MDT) const {
4318  if (ST.hasAddNoCarry()) {
4319  // Assume there is no user of scc since we don't select this in that case.
4320  // Since scc isn't used, it doesn't really matter if the i32 or u32 variant
4321  // is used.
4322 
4323  MachineBasicBlock &MBB = *Inst.getParent();
4325 
4326  unsigned OldDstReg = Inst.getOperand(0).getReg();
4327  unsigned ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4328 
4329  unsigned Opc = Inst.getOpcode();
4330  assert(Opc == AMDGPU::S_ADD_I32 || Opc == AMDGPU::S_SUB_I32);
4331 
4332  unsigned NewOpc = Opc == AMDGPU::S_ADD_I32 ?
4333  AMDGPU::V_ADD_U32_e64 : AMDGPU::V_SUB_U32_e64;
4334 
4335  assert(Inst.getOperand(3).getReg() == AMDGPU::SCC);
4336  Inst.RemoveOperand(3);
4337 
4338  Inst.setDesc(get(NewOpc));
4339  Inst.addImplicitDefUseOperands(*MBB.getParent());
4340  MRI.replaceRegWith(OldDstReg, ResultReg);
4341  legalizeOperands(Inst, MDT);
4342 
4343  addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist);
4344  return true;
4345  }
4346 
4347  return false;
4348 }
4349 
4350 void SIInstrInfo::lowerScalarAbs(SetVectorType &Worklist,
4351  MachineInstr &Inst) const {
4352  MachineBasicBlock &MBB = *Inst.getParent();
4354  MachineBasicBlock::iterator MII = Inst;
4355  DebugLoc DL = Inst.getDebugLoc();
4356 
4357  MachineOperand &Dest = Inst.getOperand(0);
4358  MachineOperand &Src = Inst.getOperand(1);
4359  unsigned TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4360  unsigned ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4361 
4362  unsigned SubOp = ST.hasAddNoCarry() ?
4363  AMDGPU::V_SUB_U32_e32 : AMDGPU::V_SUB_I32_e32;
4364 
4365  BuildMI(MBB, MII, DL, get(SubOp), TmpReg)
4366  .addImm(0)
4367  .addReg(Src.getReg());
4368 
4369  BuildMI(MBB, MII, DL, get(AMDGPU::V_MAX_I32_e64), ResultReg)
4370  .addReg(Src.getReg())
4371  .addReg(TmpReg);
4372 
4373  MRI.replaceRegWith(Dest.getReg(), ResultReg);
4374  addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist);
4375 }
4376 
4377 void SIInstrInfo::lowerScalarXnor(SetVectorType &Worklist,
4378  MachineInstr &Inst) const {
4379  MachineBasicBlock &MBB = *Inst.getParent();
4381  MachineBasicBlock::iterator MII = Inst;
4382  const DebugLoc &DL = Inst.getDebugLoc();
4383 
4384  MachineOperand &Dest = Inst.getOperand(0);
4385  MachineOperand &Src0 = Inst.getOperand(1);
4386  MachineOperand &Src1 = Inst.getOperand(2);
4387 
4388  legalizeGenericOperand(MBB, MII, &AMDGPU::VGPR_32RegClass, Src0, MRI, DL);
4389  legalizeGenericOperand(MBB, MII, &AMDGPU::VGPR_32RegClass, Src1, MRI, DL);
4390 
4391  unsigned NewDest = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4392  if (ST.hasDLInsts()) {
4393  BuildMI(MBB, MII, DL, get(AMDGPU::V_XNOR_B32_e64), NewDest)
4394  .add(Src0)
4395  .add(Src1);
4396  } else {
4397  unsigned Xor = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4398  BuildMI(MBB, MII, DL, get(AMDGPU::V_XOR_B32_e64), Xor)
4399  .add(Src0)
4400  .add(Src1);
4401 
4402  BuildMI(MBB, MII, DL, get(AMDGPU::V_NOT_B32_e64), NewDest)
4403  .addReg(Xor);
4404  }
4405 
4406  MRI.replaceRegWith(Dest.getReg(), NewDest);
4407  addUsersToMoveToVALUWorklist(NewDest, MRI, Worklist);
4408 }
4409 
4410 void SIInstrInfo::splitScalar64BitUnaryOp(
4411  SetVectorType &Worklist, MachineInstr &Inst,
4412  unsigned Opcode) const {
4413  MachineBasicBlock &MBB = *Inst.getParent();
4415 
4416  MachineOperand &Dest = Inst.getOperand(0);
4417  MachineOperand &Src0 = Inst.getOperand(1);
4418  DebugLoc DL = Inst.getDebugLoc();
4419 
4420  MachineBasicBlock::iterator MII = Inst;
4421 
4422  const MCInstrDesc &InstDesc = get(Opcode);
4423  const TargetRegisterClass *Src0RC = Src0.isReg() ?
4424  MRI.getRegClass(Src0.getReg()) :
4425  &AMDGPU::SGPR_32RegClass;
4426 
4427  const TargetRegisterClass *Src0SubRC = RI.getSubRegClass(Src0RC, AMDGPU::sub0);
4428 
4429  MachineOperand SrcReg0Sub0 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC,
4430  AMDGPU::sub0, Src0SubRC);
4431 
4432  const TargetRegisterClass *DestRC = MRI.getRegClass(Dest.getReg());
4433  const TargetRegisterClass *NewDestRC = RI.getEquivalentVGPRClass(DestRC);
4434  const TargetRegisterClass *NewDestSubRC = RI.getSubRegClass(NewDestRC, AMDGPU::sub0);
4435 
4436  unsigned DestSub0 = MRI.createVirtualRegister(NewDestSubRC);
4437  BuildMI(MBB, MII, DL, InstDesc, DestSub0).add(SrcReg0Sub0);
4438 
4439  MachineOperand SrcReg0Sub1 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC,
4440  AMDGPU::sub1, Src0SubRC);
4441 
4442  unsigned DestSub1 = MRI.createVirtualRegister(NewDestSubRC);
4443  BuildMI(MBB, MII, DL, InstDesc, DestSub1).add(SrcReg0Sub1);
4444 
4445  unsigned FullDestReg = MRI.createVirtualRegister(NewDestRC);
4446  BuildMI(MBB, MII, DL, get(TargetOpcode::REG_SEQUENCE), FullDestReg)
4447  .addReg(DestSub0)
4448  .addImm(AMDGPU::sub0)
4449  .addReg(DestSub1)
4450  .addImm(AMDGPU::sub1);
4451 
4452  MRI.replaceRegWith(Dest.getReg(), FullDestReg);
4453 
4454  // We don't need to legalizeOperands here because for a single operand, src0
4455  // will support any kind of input.
4456 
4457  // Move all users of this moved value.
4458  addUsersToMoveToVALUWorklist(FullDestReg, MRI, Worklist);
4459 }
4460 
4461 void SIInstrInfo::splitScalar64BitAddSub(SetVectorType &Worklist,
4462  MachineInstr &Inst,
4463  MachineDominatorTree *MDT) const {
4464  bool IsAdd = (Inst.getOpcode() == AMDGPU::S_ADD_U64_PSEUDO);
4465 
4466  MachineBasicBlock &MBB = *Inst.getParent();
4468 
4469  unsigned FullDestReg = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass);
4470  unsigned DestSub0 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4471  unsigned DestSub1 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4472 
4473  unsigned CarryReg = MRI.createVirtualRegister(&AMDGPU::SReg_64_XEXECRegClass);
4474  unsigned DeadCarryReg = MRI.createVirtualRegister(&AMDGPU::SReg_64_XEXECRegClass);
4475 
4476  MachineOperand &Dest = Inst.getOperand(0);
4477  MachineOperand &Src0 = Inst.getOperand(1);
4478  MachineOperand &Src1 = Inst.getOperand(2);
4479  const DebugLoc &DL = Inst.getDebugLoc();
4480  MachineBasicBlock::iterator MII = Inst;
4481 
4482  const TargetRegisterClass *Src0RC = MRI.getRegClass(Src0.getReg());
4483  const TargetRegisterClass *Src1RC = MRI.getRegClass(Src1.getReg());
4484  const TargetRegisterClass *Src0SubRC = RI.getSubRegClass(Src0RC, AMDGPU::sub0);
4485  const TargetRegisterClass *Src1SubRC = RI.getSubRegClass(Src1RC, AMDGPU::sub0);
4486 
4487  MachineOperand SrcReg0Sub0 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC,
4488  AMDGPU::sub0, Src0SubRC);
4489  MachineOperand SrcReg1Sub0 = buildExtractSubRegOrImm(MII, MRI, Src1, Src1RC,
4490  AMDGPU::sub0, Src1SubRC);
4491 
4492 
4493  MachineOperand SrcReg0Sub1 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC,
4494  AMDGPU::sub1, Src0SubRC);
4495  MachineOperand SrcReg1Sub1 = buildExtractSubRegOrImm(MII, MRI, Src1, Src1RC,
4496  AMDGPU::sub1, Src1SubRC);
4497 
4498  unsigned LoOpc = IsAdd ? AMDGPU::V_ADD_I32_e64 : AMDGPU::V_SUB_I32_e64;
4499  MachineInstr *LoHalf =
4500  BuildMI(MBB, MII, DL, get(LoOpc), DestSub0)
4501  .addReg(CarryReg, RegState::Define)
4502  .add(SrcReg0Sub0)
4503  .add(SrcReg1Sub0);
4504 
4505  unsigned HiOpc = IsAdd ? AMDGPU::V_ADDC_U32_e64 : AMDGPU::V_SUBB_U32_e64;
4506  MachineInstr *HiHalf =
4507  BuildMI(MBB, MII, DL, get(HiOpc), DestSub1)
4508  .addReg(DeadCarryReg, RegState::Define | RegState::Dead)
4509  .add(SrcReg0Sub1)
4510  .add(SrcReg1Sub1)
4511  .addReg(CarryReg, RegState::Kill);
4512 
4513  BuildMI(MBB, MII, DL, get(TargetOpcode::REG_SEQUENCE), FullDestReg)
4514  .addReg(DestSub0)
4515  .addImm(AMDGPU::sub0)
4516  .addReg(DestSub1)
4517  .addImm(AMDGPU::sub1);
4518 
4519  MRI.replaceRegWith(Dest.getReg(), FullDestReg);
4520 
4521  // Try to legalize the operands in case we need to swap the order to keep it
4522  // valid.
4523  legalizeOperands(*LoHalf, MDT);
4524  legalizeOperands(*HiHalf, MDT);
4525 
4526  // Move all users of this moved vlaue.
4527  addUsersToMoveToVALUWorklist(FullDestReg, MRI, Worklist);
4528 }
4529 
4530 void SIInstrInfo::splitScalar64BitBinaryOp(SetVectorType &Worklist,
4531  MachineInstr &Inst, unsigned Opcode,
4532  MachineDominatorTree *MDT) const {
4533  MachineBasicBlock &MBB = *Inst.getParent();
4535 
4536  MachineOperand &Dest = Inst.getOperand(0);
4537  MachineOperand &Src0 = Inst.getOperand(1);
4538  MachineOperand &Src1 = Inst.getOperand(2);
4539  DebugLoc DL = Inst.getDebugLoc();
4540 
4541  MachineBasicBlock::iterator MII = Inst;
4542 
4543  const MCInstrDesc &InstDesc = get(Opcode);
4544  const TargetRegisterClass *Src0RC = Src0.isReg() ?
4545  MRI.getRegClass(Src0.getReg()) :
4546  &AMDGPU::SGPR_32RegClass;
4547 
4548  const TargetRegisterClass *Src0SubRC = RI.getSubRegClass(Src0RC, AMDGPU::sub0);
4549  const TargetRegisterClass *Src1RC = Src1.isReg() ?
4550  MRI.getRegClass(Src1.getReg()) :
4551  &AMDGPU::SGPR_32RegClass;
4552 
4553  const TargetRegisterClass *Src1SubRC = RI.getSubRegClass(Src1RC, AMDGPU::sub0);
4554 
4555  MachineOperand SrcReg0Sub0 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC,
4556  AMDGPU::sub0, Src0SubRC);
4557  MachineOperand SrcReg1Sub0 = buildExtractSubRegOrImm(MII, MRI, Src1, Src1RC,
4558  AMDGPU::sub0, Src1SubRC);
4559 
4560  const TargetRegisterClass *DestRC = MRI.getRegClass(Dest.getReg());
4561  const TargetRegisterClass *NewDestRC = RI.getEquivalentVGPRClass(DestRC);
4562  const TargetRegisterClass *NewDestSubRC = RI.getSubRegClass(NewDestRC, AMDGPU::sub0);
4563 
4564  unsigned DestSub0 = MRI.createVirtualRegister(NewDestSubRC);
4565  MachineInstr &LoHalf = *BuildMI(MBB, MII, DL, InstDesc, DestSub0)
4566  .add(SrcReg0Sub0)
4567  .add(SrcReg1Sub0);
4568 
4569  MachineOperand SrcReg0Sub1 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC,
4570  AMDGPU::sub1, Src0SubRC);
4571  MachineOperand SrcReg1Sub1 = buildExtractSubRegOrImm(MII, MRI, Src1, Src1RC,
4572  AMDGPU::sub1, Src1SubRC);
4573 
4574  unsigned DestSub1 = MRI.createVirtualRegister(NewDestSubRC);
4575  MachineInstr &HiHalf = *BuildMI(MBB, MII, DL, InstDesc, DestSub1)
4576  .add(SrcReg0Sub1)
4577  .add(SrcReg1Sub1);
4578 
4579  unsigned FullDestReg = MRI.createVirtualRegister(NewDestRC);
4580  BuildMI(MBB, MII, DL, get(TargetOpcode::REG_SEQUENCE), FullDestReg)
4581  .addReg(DestSub0)
4582  .addImm(AMDGPU::sub0)
4583  .addReg(DestSub1)
4584  .addImm(AMDGPU::sub1);
4585 
4586  MRI.replaceRegWith(Dest.getReg(), FullDestReg);
4587 
4588  // Try to legalize the operands in case we need to swap the order to keep it
4589  // valid.
4590  legalizeOperands(LoHalf, MDT);
4591  legalizeOperands(HiHalf, MDT);
4592 
4593  // Move all users of this moved vlaue.
4594  addUsersToMoveToVALUWorklist(FullDestReg, MRI, Worklist);
4595 }
4596 
4597 void SIInstrInfo::splitScalar64BitBCNT(
4598  SetVectorType &Worklist, MachineInstr &Inst) const {
4599  MachineBasicBlock &MBB = *Inst.getParent();
4601 
4602  MachineBasicBlock::iterator MII = Inst;
4603  DebugLoc DL = Inst.getDebugLoc();
4604 
4605  MachineOperand &Dest = Inst.getOperand(0);
4606  MachineOperand &Src = Inst.getOperand(1);
4607 
4608  const MCInstrDesc &InstDesc = get(AMDGPU::V_BCNT_U32_B32_e64);
4609  const TargetRegisterClass *SrcRC = Src.isReg() ?
4610  MRI.getRegClass(Src.getReg()) :
4611  &AMDGPU::SGPR_32RegClass;
4612 
4613  unsigned MidReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4614  unsigned ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4615 
4616  const TargetRegisterClass *SrcSubRC = RI.getSubRegClass(SrcRC, AMDGPU::sub0);
4617 
4618  MachineOperand SrcRegSub0 = buildExtractSubRegOrImm(MII, MRI, Src, SrcRC,
4619  AMDGPU::sub0, SrcSubRC);
4620  MachineOperand SrcRegSub1 = buildExtractSubRegOrImm(MII, MRI, Src, SrcRC,
4621  AMDGPU::sub1, SrcSubRC);
4622 
4623  BuildMI(MBB, MII, DL, InstDesc, MidReg).add(SrcRegSub0).addImm(0);
4624 
4625  BuildMI(MBB, MII, DL, InstDesc, ResultReg).add(SrcRegSub1).addReg(MidReg);
4626 
4627  MRI.replaceRegWith(Dest.getReg(), ResultReg);
4628 
4629  // We don't need to legalize operands here. src0 for etiher instruction can be
4630  // an SGPR, and the second input is unused or determined here.
4631  addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist);
4632 }
4633 
4634 void SIInstrInfo::splitScalar64BitBFE(SetVectorType &Worklist,
4635  MachineInstr &Inst) const {
4636  MachineBasicBlock &MBB = *Inst.getParent();
4638  MachineBasicBlock::iterator MII = Inst;
4639  DebugLoc DL = Inst.getDebugLoc();
4640 
4641  MachineOperand &Dest = Inst.getOperand(0);
4642  uint32_t Imm = Inst.getOperand(2).getImm();
4643  uint32_t Offset = Imm & 0x3f; // Extract bits [5:0].
4644  uint32_t BitWidth = (Imm & 0x7f0000) >> 16; // Extract bits [22:16].
4645 
4646  (void) Offset;
4647 
4648  // Only sext_inreg cases handled.
4649  assert(Inst.getOpcode() == AMDGPU::S_BFE_I64 && BitWidth <= 32 &&
4650  Offset == 0 && "Not implemented");
4651 
4652  if (BitWidth < 32) {
4653  unsigned MidRegLo = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4654  unsigned MidRegHi = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4655  unsigned ResultReg = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass);
4656 
4657  BuildMI(MBB, MII, DL, get(AMDGPU::V_BFE_I32), MidRegLo)
4658  .addReg(Inst.getOperand(1).getReg(), 0, AMDGPU::sub0)
4659  .addImm(0)
4660  .addImm(BitWidth);
4661 
4662  BuildMI(MBB, MII, DL, get(AMDGPU::V_ASHRREV_I32_e32), MidRegHi)
4663  .addImm(31)
4664  .addReg(MidRegLo);
4665 
4666  BuildMI(MBB, MII, DL, get(TargetOpcode::REG_SEQUENCE), ResultReg)
4667  .addReg(MidRegLo)
4668  .addImm(AMDGPU::sub0)
4669  .addReg(MidRegHi)
4670  .addImm(AMDGPU::sub1);
4671 
4672  MRI.replaceRegWith(Dest.getReg(), ResultReg);
4673  addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist);
4674  return;
4675  }
4676 
4677  MachineOperand &Src = Inst.getOperand(1);
4678  unsigned TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4679  unsigned ResultReg = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass);
4680 
4681  BuildMI(MBB, MII, DL, get(AMDGPU::V_ASHRREV_I32_e64), TmpReg)
4682  .addImm(31)
4683  .addReg(Src.getReg(), 0, AMDGPU::sub0);
4684 
4685  BuildMI(MBB, MII, DL, get(TargetOpcode::REG_SEQUENCE), ResultReg)
4686  .addReg(Src.getReg(), 0, AMDGPU::sub0)
4687  .addImm(AMDGPU::sub0)
4688  .addReg(TmpReg)
4689  .addImm(AMDGPU::sub1);
4690 
4691  MRI.replaceRegWith(Dest.getReg(), ResultReg);
4692  addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist);
4693 }
4694 
4695 void SIInstrInfo::addUsersToMoveToVALUWorklist(
4696  unsigned DstReg,
4698  SetVectorType &Worklist) const {
4699  for (MachineRegisterInfo::use_iterator I = MRI.use_begin(DstReg),
4700  E = MRI.use_end(); I != E;) {
4701  MachineInstr &UseMI = *I->getParent();
4702  if (!canReadVGPR(UseMI, I.getOperandNo())) {
4703  Worklist.insert(&UseMI);
4704 
4705  do {
4706  ++I;
4707  } while (I != E && I->getParent() == &UseMI);
4708  } else {
4709  ++I;
4710  }
4711  }
4712 }
4713 
4714 void SIInstrInfo::movePackToVALU(SetVectorType &Worklist,
4715  MachineRegisterInfo &MRI,
4716  MachineInstr &Inst) const {
4717  unsigned ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4718  MachineBasicBlock *MBB = Inst.getParent();
4719  MachineOperand &Src0 = Inst.getOperand(1);
4720  MachineOperand &Src1 = Inst.getOperand(2);
4721  const DebugLoc &DL = Inst.getDebugLoc();
4722 
4723  switch (Inst.getOpcode()) {
4724  case AMDGPU::S_PACK_LL_B32_B16: {
4725  unsigned ImmReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4726  unsigned TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4727 
4728  // FIXME: Can do a lot better if we know the high bits of src0 or src1 are
4729  // 0.
4730  BuildMI(*MBB, Inst, DL, get(AMDGPU::V_MOV_B32_e32), ImmReg)
4731  .addImm(0xffff);
4732 
4733  BuildMI(*MBB, Inst, DL, get(AMDGPU::V_AND_B32_e64), TmpReg)
4734  .addReg(ImmReg, RegState::Kill)
4735  .add(Src0);
4736 
4737  BuildMI(*MBB, Inst, DL, get(AMDGPU::V_LSHL_OR_B32), ResultReg)
4738  .add(Src1)
4739  .addImm(16)
4740  .addReg(TmpReg, RegState::Kill);
4741  break;
4742  }
4743  case AMDGPU::S_PACK_LH_B32_B16: {
4744  unsigned ImmReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4745  BuildMI(*MBB, Inst, DL, get(AMDGPU::V_MOV_B32_e32), ImmReg)
4746  .addImm(0xffff);
4747  BuildMI(*MBB, Inst, DL, get(AMDGPU::V_BFI_B32), ResultReg)
4748  .addReg(ImmReg, RegState::Kill)
4749  .add(Src0)
4750  .add(Src1);
4751  break;
4752  }
4753  case AMDGPU::S_PACK_HH_B32_B16: {
4754  unsigned ImmReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4755  unsigned TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4756  BuildMI(*MBB, Inst, DL, get(AMDGPU::V_LSHRREV_B32_e64), TmpReg)
4757  .addImm(16)
4758  .add(Src0);
4759  BuildMI(*MBB, Inst, DL, get(AMDGPU::V_MOV_B32_e32), ImmReg)
4760  .addImm(0xffff0000);
4761  BuildMI(*MBB, Inst, DL, get(AMDGPU::V_AND_OR_B32), ResultReg)
4762  .add(Src1)
4763  .addReg(ImmReg, RegState::Kill)
4764  .addReg(TmpReg, RegState::Kill);
4765  break;
4766  }
4767  default:
4768  llvm_unreachable("unhandled s_pack_* instruction");
4769  }
4770 
4771  MachineOperand &Dest = Inst.getOperand(0);
4772  MRI.replaceRegWith(Dest.getReg(), ResultReg);
4773  addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist);
4774 }
4775 
4776 void SIInstrInfo::addSCCDefUsersToVALUWorklist(
4777  MachineInstr &SCCDefInst, SetVectorType &Worklist) const {
4778  // This assumes that all the users of SCC are in the same block
4779  // as the SCC def.
4780  for (MachineInstr &MI :
4782  SCCDefInst.getParent()->end())) {
4783  // Exit if we find another SCC def.
4784  if (MI.findRegisterDefOperandIdx(AMDGPU::SCC) != -1)
4785  return;
4786 
4787  if (MI.findRegisterUseOperandIdx(AMDGPU::SCC) != -1)
4788  Worklist.insert(&MI);
4789  }
4790 }
4791 
4792 const TargetRegisterClass *SIInstrInfo::getDestEquivalentVGPRClass(
4793  const MachineInstr &Inst) const {
4794  const TargetRegisterClass *NewDstRC = getOpRegClass(Inst, 0);
4795 
4796  switch (Inst.getOpcode()) {
4797  // For target instructions, getOpRegClass just returns the virtual register
4798  // class associated with the operand, so we need to find an equivalent VGPR
4799  // register class in order to move the instruction to the VALU.
4800  case AMDGPU::COPY:
4801  case AMDGPU::PHI:
4802  case AMDGPU::REG_SEQUENCE:
4803  case AMDGPU::INSERT_SUBREG:
4804  case AMDGPU::WQM:
4805  case AMDGPU::WWM:
4806  if (RI.hasVGPRs(NewDstRC))
4807  return nullptr;
4808 
4809  NewDstRC = RI.getEquivalentVGPRClass(NewDstRC);
4810  if (!NewDstRC)
4811  return nullptr;
4812  return NewDstRC;
4813  default:
4814  return NewDstRC;
4815  }
4816 }
4817 
4818 // Find the one SGPR operand we are allowed to use.
4819 unsigned SIInstrInfo::findUsedSGPR(const MachineInstr &MI,
4820  int OpIndices[3]) const {
4821  const MCInstrDesc &Desc = MI.getDesc();
4822 
4823  // Find the one SGPR operand we are allowed to use.
4824  //
4825  // First we need to consider the instruction's operand requirements before
4826  // legalizing. Some operands are required to be SGPRs, such as implicit uses
4827  // of VCC, but we are still bound by the constant bus requirement to only use
4828  // one.
4829  //
4830  // If the operand's class is an SGPR, we can never move it.
4831 
4832  unsigned SGPRReg = findImplicitSGPRRead(MI);
4833  if (SGPRReg != AMDGPU::NoRegister)
4834  return SGPRReg;
4835 
4836  unsigned UsedSGPRs[3] = { AMDGPU::NoRegister };
4837  const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
4838 
4839  for (unsigned i = 0; i < 3; ++i) {
4840  int Idx = OpIndices[i];
4841  if (Idx == -1)
4842  break;
4843 
4844  const MachineOperand &MO = MI.getOperand(Idx);
4845  if (!MO.isReg())
4846  continue;
4847 
4848  // Is this operand statically required to be an SGPR based on the operand
4849  // constraints?
4850  const TargetRegisterClass *OpRC = RI.getRegClass(Desc.OpInfo[Idx].RegClass);
4851  bool IsRequiredSGPR = RI.isSGPRClass(OpRC);
4852  if (IsRequiredSGPR)
4853  return MO.getReg();
4854 
4855  // If this could be a VGPR or an SGPR, Check the dynamic register class.
4856  unsigned Reg = MO.getReg();
4857  const TargetRegisterClass *RegRC = MRI.getRegClass(Reg);
4858  if (RI.isSGPRClass(RegRC))
4859  UsedSGPRs[i] = Reg;
4860  }
4861 
4862  // We don't have a required SGPR operand, so we have a bit more freedom in
4863  // selecting operands to move.
4864 
4865  // Try to select the most used SGPR. If an SGPR is equal to one of the
4866  // others, we choose that.
4867  //
4868  // e.g.
4869  // V_FMA_F32 v0, s0, s0, s0 -> No moves
4870  // V_FMA_F32 v0, s0, s1, s0 -> Move s1
4871 
4872  // TODO: If some of the operands are 64-bit SGPRs and some 32, we should
4873  // prefer those.
4874 
4875  if (UsedSGPRs[0] != AMDGPU::NoRegister) {
4876  if (UsedSGPRs[0] == UsedSGPRs[1] || UsedSGPRs[0] == UsedSGPRs[2])
4877  SGPRReg = UsedSGPRs[0];
4878  }
4879 
4880  if (SGPRReg == AMDGPU::NoRegister && UsedSGPRs[1] != AMDGPU::NoRegister) {
4881  if (UsedSGPRs[1] == UsedSGPRs[2])
4882  SGPRReg = UsedSGPRs[1];
4883  }
4884 
4885  return SGPRReg;
4886 }
4887 
4889  unsigned OperandName) const {
4890  int Idx = AMDGPU::getNamedOperandIdx(MI.getOpcode(), OperandName);
4891  if (Idx == -1)
4892  return nullptr;
4893 
4894  return &MI.getOperand(Idx);
4895 }
4896 
4898  uint64_t RsrcDataFormat = AMDGPU::RSRC_DATA_FORMAT;
4899  if (ST.isAmdHsaOS()) {
4900  // Set ATC = 1. GFX9 doesn't have this bit.
4901  if (ST.getGeneration() <= AMDGPUSubtarget::VOLCANIC_ISLANDS)
4902  RsrcDataFormat |= (1ULL << 56);
4903 
4904  // Set MTYPE = 2 (MTYPE_UC = uncached). GFX9 doesn't have this.
4905  // BTW, it disables TC L2 and therefore decreases performance.
4906  if (ST.getGeneration() == AMDGPUSubtarget::VOLCANIC_ISLANDS)
4907  RsrcDataFormat |= (2ULL << 59);
4908  }
4909 
4910  return RsrcDataFormat;
4911 }
4912 
4914  uint64_t Rsrc23 = getDefaultRsrcDataFormat() |
4916  0xffffffff; // Size;
4917 
4918  // GFX9 doesn't have ELEMENT_SIZE.
4919  if (ST.getGeneration() <= AMDGPUSubtarget::VOLCANIC_ISLANDS) {
4920  uint64_t EltSizeValue = Log2_32(ST.getMaxPrivateElementSize()) - 1;
4921  Rsrc23 |= EltSizeValue << AMDGPU::RSRC_ELEMENT_SIZE_SHIFT;
4922  }
4923 
4924  // IndexStride = 64.
4925  Rsrc23 |= UINT64_C(3) << AMDGPU::RSRC_INDEX_STRIDE_SHIFT;
4926 
4927  // If TID_ENABLE is set, DATA_FORMAT specifies stride bits [14:17].
4928  // Clear them unless we want a huge stride.
4929  if (ST.getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS)
4930  Rsrc23 &= ~AMDGPU::RSRC_DATA_FORMAT;
4931 
4932  return Rsrc23;
4933 }
4934 
4936  unsigned Opc = MI.getOpcode();
4937 
4938  return isSMRD(Opc);
4939 }
4940 
4942  unsigned Opc = MI.getOpcode();
4943 
4944  return isMUBUF(Opc) || isMTBUF(Opc) || isMIMG(Opc);
4945 }
4946 
4948  int &FrameIndex) const {
4949  const MachineOperand *Addr = getNamedOperand(MI, AMDGPU::OpName::vaddr);
4950  if (!Addr || !Addr->isFI())
4951  return AMDGPU::NoRegister;
4952 
4953  assert(!MI.memoperands_empty() &&
4955 
4956  FrameIndex = Addr->getIndex();
4957  return getNamedOperand(MI, AMDGPU::OpName::vdata)->getReg();
4958 }
4959 
4961  int &FrameIndex) const {
4962  const MachineOperand *Addr = getNamedOperand(MI, AMDGPU::OpName::addr);
4963  assert(Addr && Addr->isFI());
4964  FrameIndex = Addr->getIndex();
4965  return getNamedOperand(MI, AMDGPU::OpName::data)->getReg();
4966 }
4967 
4969  int &FrameIndex) const {
4970  if (!MI.mayLoad())
4971  return AMDGPU::NoRegister;
4972 
4973  if (isMUBUF(MI) || isVGPRSpill(MI))
4974  return isStackAccess(MI, FrameIndex);
4975 
4976  if (isSGPRSpill(MI))
4977  return isSGPRStackAccess(MI, FrameIndex);
4978 
4979  return AMDGPU::NoRegister;
4980 }
4981 
4983  int &FrameIndex) const {
4984  if (!MI.mayStore())
4985  return AMDGPU::NoRegister;
4986 
4987  if (isMUBUF(MI) || isVGPRSpill(MI))
4988  return isStackAccess(MI, FrameIndex);
4989 
4990  if (isSGPRSpill(MI))
4991  return isSGPRStackAccess(MI, FrameIndex);
4992 
4993  return AMDGPU::NoRegister;
4994 }
4995 
4997  unsigned Size = 0;
5000  while (++I != E && I->isInsideBundle()) {
5001  assert(!I->isBundle() && "No nested bundle!");
5002  Size += getInstSizeInBytes(*I);
5003  }
5004 
5005  return Size;
5006 }
5007 
5009  unsigned Opc = MI.getOpcode();
5010  const MCInstrDesc &Desc = getMCOpcodeFromPseudo(Opc);
5011  unsigned DescSize = Desc.getSize();
5012 
5013  // If we have a definitive size, we can use it. Otherwise we need to inspect
5014  // the operands to know the size.
5015  if (isFixedSize(MI))
5016  return DescSize;
5017 
5018  // 4-byte instructions may have a 32-bit literal encoded after them. Check
5019  // operands that coud ever be literals.
5020  if (isVALU(MI) || isSALU(MI)) {
5021  int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0);
5022  if (Src0Idx == -1)
5023  return DescSize; // No operands.
5024 
5025  if (isLiteralConstantLike(MI.getOperand(Src0Idx), Desc.OpInfo[Src0Idx]))
5026  return DescSize + 4;
5027 
5028  int Src1Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1);
5029  if (Src1Idx == -1)
5030  return DescSize;
5031 
5032  if (isLiteralConstantLike(MI.getOperand(Src1Idx), Desc.OpInfo[Src1Idx]))
5033  return DescSize + 4;
5034 
5035  int Src2Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2);
5036  if (Src2Idx == -1)
5037  return DescSize;
5038 
5039  if (isLiteralConstantLike(MI.getOperand(Src2Idx), Desc.OpInfo[Src2Idx]))
5040  return DescSize + 4;
5041 
5042  return DescSize;
5043  }
5044 
5045  switch (Opc) {
5046  case TargetOpcode::IMPLICIT_DEF:
5047  case TargetOpcode::KILL:
5048  case TargetOpcode::DBG_VALUE:
5050  return 0;
5051  case TargetOpcode::BUNDLE:
5052  return getInstBundleSize(MI);
5053  case TargetOpcode::INLINEASM: {
5054  const MachineFunction *MF = MI.getParent()->getParent();
5055  const char *AsmStr = MI.getOperand(0).getSymbolName();
5056  return getInlineAsmLength(AsmStr, *MF->getTarget().getMCAsmInfo());
5057  }
5058  default:
5059  return DescSize;
5060  }
5061 }
5062 
5064  if (!isFLAT(MI))
5065  return false;
5066 
5067  if (MI.memoperands_empty())
5068  return true;
5069 
5070  for (const MachineMemOperand *MMO : MI.memoperands()) {
5071  if (MMO->getAddrSpace() == AMDGPUAS::FLAT_ADDRESS)
5072  return true;
5073  }
5074  return false;
5075 }
5076 
5078  return Branch.getOpcode() == AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO;
5079 }
5080 
5082  MachineBasicBlock *IfEnd) const {
5084  assert(TI != IfEntry->end());
5085 
5086  MachineInstr *Branch = &(*TI);
5087  MachineFunction *MF = IfEntry->getParent();
5088  MachineRegisterInfo &MRI = IfEntry->getParent()->getRegInfo();
5089 
5090  if (Branch->getOpcode() == AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO) {
5091  unsigned DstReg = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
5092  MachineInstr *SIIF =
5093  BuildMI(*MF, Branch->getDebugLoc(), get(AMDGPU::SI_IF), DstReg)
5094  .add(Branch->getOperand(0))
5095  .add(Branch->getOperand(1));
5096  MachineInstr *SIEND =
5097  BuildMI(*MF, Branch->getDebugLoc(), get(AMDGPU::SI_END_CF))
5098  .addReg(DstReg);
5099 
5100  IfEntry->erase(TI);
5101  IfEntry->insert(IfEntry->end(), SIIF);
5102  IfEnd->insert(IfEnd->getFirstNonPHI(), SIEND);
5103  }
5104 }
5105 
5107  MachineBasicBlock *LoopEntry, MachineBasicBlock *LoopEnd) const {
5109  // We expect 2 terminators, one conditional and one unconditional.
5110  assert(TI != LoopEnd->end());
5111 
5112  MachineInstr *Branch = &(*TI);
5113  MachineFunction *MF = LoopEnd->getParent();
5114  MachineRegisterInfo &MRI = LoopEnd->getParent()->getRegInfo();
5115 
5116  if (Branch->getOpcode() == AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO) {
5117 
5118  unsigned DstReg = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
5119  unsigned BackEdgeReg = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
5120  MachineInstrBuilder HeaderPHIBuilder =
5121  BuildMI(*(MF), Branch->getDebugLoc(), get(TargetOpcode::PHI), DstReg);
5122  for (MachineBasicBlock::pred_iterator PI = LoopEntry->pred_begin(),
5123  E = LoopEntry->pred_end();
5124  PI != E; ++PI) {
5125  if (*PI == LoopEnd) {
5126  HeaderPHIBuilder.addReg(BackEdgeReg);
5127  } else {
5128  MachineBasicBlock *PMBB = *PI;
5129  unsigned ZeroReg = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
5131  ZeroReg, 0);
5132  HeaderPHIBuilder.addReg(ZeroReg);
5133  }
5134  HeaderPHIBuilder.addMBB(*PI);
5135  }
5136  MachineInstr *HeaderPhi = HeaderPHIBuilder;
5137  MachineInstr *SIIFBREAK = BuildMI(*(MF), Branch->getDebugLoc(),
5138  get(AMDGPU::SI_IF_BREAK), BackEdgeReg)
5139  .addReg(DstReg)
5140  .add(Branch->getOperand(0));
5141  MachineInstr *SILOOP =
5142  BuildMI(*(MF), Branch->getDebugLoc(), get(AMDGPU::SI_LOOP))
5143  .addReg(BackEdgeReg)
5144  .addMBB(LoopEntry);
5145 
5146  LoopEntry->insert(LoopEntry->begin(), HeaderPhi);
5147  LoopEnd->erase(TI);
5148  LoopEnd->insert(LoopEnd->end(), SIIFBREAK);
5149  LoopEnd->insert(LoopEnd->end(), SILOOP);
5150  }
5151 }
5152 
5155  static const std::pair<int, const char *> TargetIndices[] = {
5156  {AMDGPU::TI_CONSTDATA_START, "amdgpu-constdata-start"},
5157  {AMDGPU::TI_SCRATCH_RSRC_DWORD0, "amdgpu-scratch-rsrc-dword0"},
5158  {AMDGPU::TI_SCRATCH_RSRC_DWORD1, "amdgpu-scratch-rsrc-dword1"},
5159  {AMDGPU::TI_SCRATCH_RSRC_DWORD2, "amdgpu-scratch-rsrc-dword2"},
5160  {AMDGPU::TI_SCRATCH_RSRC_DWORD3, "amdgpu-scratch-rsrc-dword3"}};
5161  return makeArrayRef(TargetIndices);
5162 }
5163 
5164 /// This is used by the post-RA scheduler (SchedulePostRAList.cpp). The
5165 /// post-RA version of misched uses CreateTargetMIHazardRecognizer.
5168  const ScheduleDAG *DAG) const {
5169  return new GCNHazardRecognizer(DAG->MF);
5170 }
5171 
5172 /// This is the hazard recognizer used at -O0 by the PostRAHazardRecognizer
5173 /// pass.
5176  return new GCNHazardRecognizer(MF);
5177 }
5178 
5179 std::pair<unsigned, unsigned>
5181  return std::make_pair(TF & MO_MASK, TF & ~MO_MASK);
5182 }
5183 
5186  static const std::pair<unsigned, const char *> TargetFlags[] = {
5187  { MO_GOTPCREL, "amdgpu-gotprel" },
5188  { MO_GOTPCREL32_LO, "amdgpu-gotprel32-lo" },
5189  { MO_GOTPCREL32_HI, "amdgpu-gotprel32-hi" },
5190  { MO_REL32_LO, "amdgpu-rel32-lo" },
5191  { MO_REL32_HI, "amdgpu-rel32-hi" }
5192  };
5193 
5194  return makeArrayRef(TargetFlags);
5195 }
5196 
5198  return !MI.isTerminator() && MI.getOpcode() != AMDGPU::COPY &&
5199  MI.modifiesRegister(AMDGPU::EXEC, &RI);
5200 }
5201 
5205  const DebugLoc &DL,
5206  unsigned DestReg) const {
5207  if (ST.hasAddNoCarry())
5208  return BuildMI(MBB, I, DL, get(AMDGPU::V_ADD_U32_e64), DestReg);
5209 
5210  MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
5211  unsigned UnusedCarry = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
5212  MRI.setRegAllocationHint(UnusedCarry, 0, AMDGPU::VCC);
5213 
5214  return BuildMI(MBB, I, DL, get(AMDGPU::V_ADD_I32_e64), DestReg)
5215  .addReg(UnusedCarry, RegState::Define | RegState::Dead);
5216 }
5217 
5218 bool SIInstrInfo::isKillTerminator(unsigned Opcode) {
5219  switch (Opcode) {
5220  case AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR:
5221  case AMDGPU::SI_KILL_I1_TERMINATOR:
5222  return true;
5223  default:
5224  return false;
5225  }
5226 }
5227 
5229  switch (Opcode) {
5230  case AMDGPU::SI_KILL_F32_COND_IMM_PSEUDO:
5231  return get(AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR);
5232  case AMDGPU::SI_KILL_I1_PSEUDO:
5233  return get(AMDGPU::SI_KILL_I1_TERMINATOR);
5234  default:
5235  llvm_unreachable("invalid opcode, expected SI_KILL_*_PSEUDO");
5236  }
5237 }
5238 
5240  if (!isSMRD(MI))
5241  return false;
5242 
5243  // Check that it is using a buffer resource.
5244  int Idx = AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::sbase);
5245  if (Idx == -1) // e.g. s_memtime
5246  return false;
5247 
5248  const auto RCID = MI.getDesc().OpInfo[Idx].RegClass;
5249  return RCID == AMDGPU::SReg_128RegClassID;
5250 }
5251 
5252 // This must be kept in sync with the SIEncodingFamily class in SIInstrInfo.td
5254  SI = 0,
5255  VI = 1,
5256  SDWA = 2,
5257  SDWA9 = 3,
5258  GFX80 = 4,