LLVM  9.0.0svn
SIInstrInfo.cpp
Go to the documentation of this file.
1 //===- SIInstrInfo.cpp - SI Instruction Information ----------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 /// \file
10 /// SI Implementation of TargetInstrInfo.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "SIInstrInfo.h"
15 #include "AMDGPU.h"
16 #include "AMDGPUSubtarget.h"
17 #include "GCNHazardRecognizer.h"
18 #include "SIDefines.h"
19 #include "SIMachineFunctionInfo.h"
20 #include "SIRegisterInfo.h"
22 #include "Utils/AMDGPUBaseInfo.h"
23 #include "llvm/ADT/APInt.h"
24 #include "llvm/ADT/ArrayRef.h"
25 #include "llvm/ADT/SmallVector.h"
26 #include "llvm/ADT/StringRef.h"
46 #include "llvm/IR/DebugLoc.h"
47 #include "llvm/IR/DiagnosticInfo.h"
48 #include "llvm/IR/Function.h"
49 #include "llvm/IR/InlineAsm.h"
50 #include "llvm/IR/LLVMContext.h"
51 #include "llvm/MC/MCInstrDesc.h"
52 #include "llvm/Support/Casting.h"
54 #include "llvm/Support/Compiler.h"
59 #include <cassert>
60 #include <cstdint>
61 #include <iterator>
62 #include <utility>
63 
64 using namespace llvm;
65 
66 #define GET_INSTRINFO_CTOR_DTOR
67 #include "AMDGPUGenInstrInfo.inc"
68 
69 namespace llvm {
70 namespace AMDGPU {
71 #define GET_D16ImageDimIntrinsics_IMPL
72 #define GET_ImageDimIntrinsicTable_IMPL
73 #define GET_RsrcIntrinsics_IMPL
74 #include "AMDGPUGenSearchableTables.inc"
75 }
76 }
77 
78 
79 // Must be at least 4 to be able to branch over minimum unconditional branch
80 // code. This is only for making it possible to write reasonably small tests for
81 // long branches.
82 static cl::opt<unsigned>
83 BranchOffsetBits("amdgpu-s-branch-bits", cl::ReallyHidden, cl::init(16),
84  cl::desc("Restrict range of branch instructions (DEBUG)"));
85 
87  : AMDGPUGenInstrInfo(AMDGPU::ADJCALLSTACKUP, AMDGPU::ADJCALLSTACKDOWN),
88  RI(ST), ST(ST) {}
89 
90 //===----------------------------------------------------------------------===//
91 // TargetInstrInfo callbacks
92 //===----------------------------------------------------------------------===//
93 
94 static unsigned getNumOperandsNoGlue(SDNode *Node) {
95  unsigned N = Node->getNumOperands();
96  while (N && Node->getOperand(N - 1).getValueType() == MVT::Glue)
97  --N;
98  return N;
99 }
100 
101 /// Returns true if both nodes have the same value for the given
102 /// operand \p Op, or if both nodes do not have this operand.
103 static bool nodesHaveSameOperandValue(SDNode *N0, SDNode* N1, unsigned OpName) {
104  unsigned Opc0 = N0->getMachineOpcode();
105  unsigned Opc1 = N1->getMachineOpcode();
106 
107  int Op0Idx = AMDGPU::getNamedOperandIdx(Opc0, OpName);
108  int Op1Idx = AMDGPU::getNamedOperandIdx(Opc1, OpName);
109 
110  if (Op0Idx == -1 && Op1Idx == -1)
111  return true;
112 
113 
114  if ((Op0Idx == -1 && Op1Idx != -1) ||
115  (Op1Idx == -1 && Op0Idx != -1))
116  return false;
117 
118  // getNamedOperandIdx returns the index for the MachineInstr's operands,
119  // which includes the result as the first operand. We are indexing into the
120  // MachineSDNode's operands, so we need to skip the result operand to get
121  // the real index.
122  --Op0Idx;
123  --Op1Idx;
124 
125  return N0->getOperand(Op0Idx) == N1->getOperand(Op1Idx);
126 }
127 
129  AliasAnalysis *AA) const {
130  // TODO: The generic check fails for VALU instructions that should be
131  // rematerializable due to implicit reads of exec. We really want all of the
132  // generic logic for this except for this.
133  switch (MI.getOpcode()) {
134  case AMDGPU::V_MOV_B32_e32:
135  case AMDGPU::V_MOV_B32_e64:
136  case AMDGPU::V_MOV_B64_PSEUDO:
137  // No implicit operands.
138  return MI.getNumOperands() == MI.getDesc().getNumOperands();
139  default:
140  return false;
141  }
142 }
143 
145  int64_t &Offset0,
146  int64_t &Offset1) const {
147  if (!Load0->isMachineOpcode() || !Load1->isMachineOpcode())
148  return false;
149 
150  unsigned Opc0 = Load0->getMachineOpcode();
151  unsigned Opc1 = Load1->getMachineOpcode();
152 
153  // Make sure both are actually loads.
154  if (!get(Opc0).mayLoad() || !get(Opc1).mayLoad())
155  return false;
156 
157  if (isDS(Opc0) && isDS(Opc1)) {
158 
159  // FIXME: Handle this case:
160  if (getNumOperandsNoGlue(Load0) != getNumOperandsNoGlue(Load1))
161  return false;
162 
163  // Check base reg.
164  if (Load0->getOperand(0) != Load1->getOperand(0))
165  return false;
166 
167  // Skip read2 / write2 variants for simplicity.
168  // TODO: We should report true if the used offsets are adjacent (excluded
169  // st64 versions).
170  int Offset0Idx = AMDGPU::getNamedOperandIdx(Opc0, AMDGPU::OpName::offset);
171  int Offset1Idx = AMDGPU::getNamedOperandIdx(Opc1, AMDGPU::OpName::offset);
172  if (Offset0Idx == -1 || Offset1Idx == -1)
173  return false;
174 
175  // XXX - be careful of datalesss loads
176  // getNamedOperandIdx returns the index for MachineInstrs. Since they
177  // include the output in the operand list, but SDNodes don't, we need to
178  // subtract the index by one.
179  Offset0Idx -= get(Opc0).NumDefs;
180  Offset1Idx -= get(Opc1).NumDefs;
181  Offset0 = cast<ConstantSDNode>(Load0->getOperand(Offset0Idx))->getZExtValue();
182  Offset1 = cast<ConstantSDNode>(Load1->getOperand(Offset1Idx))->getZExtValue();
183  return true;
184  }
185 
186  if (isSMRD(Opc0) && isSMRD(Opc1)) {
187  // Skip time and cache invalidation instructions.
188  if (AMDGPU::getNamedOperandIdx(Opc0, AMDGPU::OpName::sbase) == -1 ||
189  AMDGPU::getNamedOperandIdx(Opc1, AMDGPU::OpName::sbase) == -1)
190  return false;
191 
193 
194  // Check base reg.
195  if (Load0->getOperand(0) != Load1->getOperand(0))
196  return false;
197 
198  const ConstantSDNode *Load0Offset =
199  dyn_cast<ConstantSDNode>(Load0->getOperand(1));
200  const ConstantSDNode *Load1Offset =
201  dyn_cast<ConstantSDNode>(Load1->getOperand(1));
202 
203  if (!Load0Offset || !Load1Offset)
204  return false;
205 
206  Offset0 = Load0Offset->getZExtValue();
207  Offset1 = Load1Offset->getZExtValue();
208  return true;
209  }
210 
211  // MUBUF and MTBUF can access the same addresses.
212  if ((isMUBUF(Opc0) || isMTBUF(Opc0)) && (isMUBUF(Opc1) || isMTBUF(Opc1))) {
213 
214  // MUBUF and MTBUF have vaddr at different indices.
215  if (!nodesHaveSameOperandValue(Load0, Load1, AMDGPU::OpName::soffset) ||
216  !nodesHaveSameOperandValue(Load0, Load1, AMDGPU::OpName::vaddr) ||
217  !nodesHaveSameOperandValue(Load0, Load1, AMDGPU::OpName::srsrc))
218  return false;
219 
220  int OffIdx0 = AMDGPU::getNamedOperandIdx(Opc0, AMDGPU::OpName::offset);
221  int OffIdx1 = AMDGPU::getNamedOperandIdx(Opc1, AMDGPU::OpName::offset);
222 
223  if (OffIdx0 == -1 || OffIdx1 == -1)
224  return false;
225 
226  // getNamedOperandIdx returns the index for MachineInstrs. Since they
227  // include the output in the operand list, but SDNodes don't, we need to
228  // subtract the index by one.
229  OffIdx0 -= get(Opc0).NumDefs;
230  OffIdx1 -= get(Opc1).NumDefs;
231 
232  SDValue Off0 = Load0->getOperand(OffIdx0);
233  SDValue Off1 = Load1->getOperand(OffIdx1);
234 
235  // The offset might be a FrameIndexSDNode.
236  if (!isa<ConstantSDNode>(Off0) || !isa<ConstantSDNode>(Off1))
237  return false;
238 
239  Offset0 = cast<ConstantSDNode>(Off0)->getZExtValue();
240  Offset1 = cast<ConstantSDNode>(Off1)->getZExtValue();
241  return true;
242  }
243 
244  return false;
245 }
246 
247 static bool isStride64(unsigned Opc) {
248  switch (Opc) {
249  case AMDGPU::DS_READ2ST64_B32:
250  case AMDGPU::DS_READ2ST64_B64:
251  case AMDGPU::DS_WRITE2ST64_B32:
252  case AMDGPU::DS_WRITE2ST64_B64:
253  return true;
254  default:
255  return false;
256  }
257 }
258 
260  const MachineOperand *&BaseOp,
261  int64_t &Offset,
262  const TargetRegisterInfo *TRI) const {
263  unsigned Opc = LdSt.getOpcode();
264 
265  if (isDS(LdSt)) {
266  const MachineOperand *OffsetImm =
267  getNamedOperand(LdSt, AMDGPU::OpName::offset);
268  if (OffsetImm) {
269  // Normal, single offset LDS instruction.
270  BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::addr);
271  // TODO: ds_consume/ds_append use M0 for the base address. Is it safe to
272  // report that here?
273  if (!BaseOp)
274  return false;
275 
276  Offset = OffsetImm->getImm();
277  assert(BaseOp->isReg() && "getMemOperandWithOffset only supports base "
278  "operands of type register.");
279  return true;
280  }
281 
282  // The 2 offset instructions use offset0 and offset1 instead. We can treat
283  // these as a load with a single offset if the 2 offsets are consecutive. We
284  // will use this for some partially aligned loads.
285  const MachineOperand *Offset0Imm =
286  getNamedOperand(LdSt, AMDGPU::OpName::offset0);
287  const MachineOperand *Offset1Imm =
288  getNamedOperand(LdSt, AMDGPU::OpName::offset1);
289 
290  uint8_t Offset0 = Offset0Imm->getImm();
291  uint8_t Offset1 = Offset1Imm->getImm();
292 
293  if (Offset1 > Offset0 && Offset1 - Offset0 == 1) {
294  // Each of these offsets is in element sized units, so we need to convert
295  // to bytes of the individual reads.
296 
297  unsigned EltSize;
298  if (LdSt.mayLoad())
299  EltSize = TRI->getRegSizeInBits(*getOpRegClass(LdSt, 0)) / 16;
300  else {
301  assert(LdSt.mayStore());
302  int Data0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data0);
303  EltSize = TRI->getRegSizeInBits(*getOpRegClass(LdSt, Data0Idx)) / 8;
304  }
305 
306  if (isStride64(Opc))
307  EltSize *= 64;
308 
309  BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::addr);
310  Offset = EltSize * Offset0;
311  assert(BaseOp->isReg() && "getMemOperandWithOffset only supports base "
312  "operands of type register.");
313  return true;
314  }
315 
316  return false;
317  }
318 
319  if (isMUBUF(LdSt) || isMTBUF(LdSt)) {
320  const MachineOperand *SOffset = getNamedOperand(LdSt, AMDGPU::OpName::soffset);
321  if (SOffset && SOffset->isReg())
322  return false;
323 
324  const MachineOperand *AddrReg = getNamedOperand(LdSt, AMDGPU::OpName::vaddr);
325  if (!AddrReg)
326  return false;
327 
328  const MachineOperand *OffsetImm =
329  getNamedOperand(LdSt, AMDGPU::OpName::offset);
330  BaseOp = AddrReg;
331  Offset = OffsetImm->getImm();
332 
333  if (SOffset) // soffset can be an inline immediate.
334  Offset += SOffset->getImm();
335 
336  assert(BaseOp->isReg() && "getMemOperandWithOffset only supports base "
337  "operands of type register.");
338  return true;
339  }
340 
341  if (isSMRD(LdSt)) {
342  const MachineOperand *OffsetImm =
343  getNamedOperand(LdSt, AMDGPU::OpName::offset);
344  if (!OffsetImm)
345  return false;
346 
347  const MachineOperand *SBaseReg = getNamedOperand(LdSt, AMDGPU::OpName::sbase);
348  BaseOp = SBaseReg;
349  Offset = OffsetImm->getImm();
350  assert(BaseOp->isReg() && "getMemOperandWithOffset only supports base "
351  "operands of type register.");
352  return true;
353  }
354 
355  if (isFLAT(LdSt)) {
356  const MachineOperand *VAddr = getNamedOperand(LdSt, AMDGPU::OpName::vaddr);
357  if (VAddr) {
358  // Can't analyze 2 offsets.
359  if (getNamedOperand(LdSt, AMDGPU::OpName::saddr))
360  return false;
361 
362  BaseOp = VAddr;
363  } else {
364  // scratch instructions have either vaddr or saddr.
365  BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::saddr);
366  }
367 
368  Offset = getNamedOperand(LdSt, AMDGPU::OpName::offset)->getImm();
369  assert(BaseOp->isReg() && "getMemOperandWithOffset only supports base "
370  "operands of type register.");
371  return true;
372  }
373 
374  return false;
375 }
376 
377 static bool memOpsHaveSameBasePtr(const MachineInstr &MI1,
378  const MachineOperand &BaseOp1,
379  const MachineInstr &MI2,
380  const MachineOperand &BaseOp2) {
381  // Support only base operands with base registers.
382  // Note: this could be extended to support FI operands.
383  if (!BaseOp1.isReg() || !BaseOp2.isReg())
384  return false;
385 
386  if (BaseOp1.isIdenticalTo(BaseOp2))
387  return true;
388 
389  if (!MI1.hasOneMemOperand() || !MI2.hasOneMemOperand())
390  return false;
391 
392  auto MO1 = *MI1.memoperands_begin();
393  auto MO2 = *MI2.memoperands_begin();
394  if (MO1->getAddrSpace() != MO2->getAddrSpace())
395  return false;
396 
397  auto Base1 = MO1->getValue();
398  auto Base2 = MO2->getValue();
399  if (!Base1 || !Base2)
400  return false;
401  const MachineFunction &MF = *MI1.getParent()->getParent();
402  const DataLayout &DL = MF.getFunction().getParent()->getDataLayout();
403  Base1 = GetUnderlyingObject(Base1, DL);
404  Base2 = GetUnderlyingObject(Base1, DL);
405 
406  if (isa<UndefValue>(Base1) || isa<UndefValue>(Base2))
407  return false;
408 
409  return Base1 == Base2;
410 }
411 
413  const MachineOperand &BaseOp2,
414  unsigned NumLoads) const {
415  const MachineInstr &FirstLdSt = *BaseOp1.getParent();
416  const MachineInstr &SecondLdSt = *BaseOp2.getParent();
417 
418  if (!memOpsHaveSameBasePtr(FirstLdSt, BaseOp1, SecondLdSt, BaseOp2))
419  return false;
420 
421  const MachineOperand *FirstDst = nullptr;
422  const MachineOperand *SecondDst = nullptr;
423 
424  if ((isMUBUF(FirstLdSt) && isMUBUF(SecondLdSt)) ||
425  (isMTBUF(FirstLdSt) && isMTBUF(SecondLdSt)) ||
426  (isFLAT(FirstLdSt) && isFLAT(SecondLdSt))) {
427  const unsigned MaxGlobalLoadCluster = 6;
428  if (NumLoads > MaxGlobalLoadCluster)
429  return false;
430 
431  FirstDst = getNamedOperand(FirstLdSt, AMDGPU::OpName::vdata);
432  if (!FirstDst)
433  FirstDst = getNamedOperand(FirstLdSt, AMDGPU::OpName::vdst);
434  SecondDst = getNamedOperand(SecondLdSt, AMDGPU::OpName::vdata);
435  if (!SecondDst)
436  SecondDst = getNamedOperand(SecondLdSt, AMDGPU::OpName::vdst);
437  } else if (isSMRD(FirstLdSt) && isSMRD(SecondLdSt)) {
438  FirstDst = getNamedOperand(FirstLdSt, AMDGPU::OpName::sdst);
439  SecondDst = getNamedOperand(SecondLdSt, AMDGPU::OpName::sdst);
440  } else if (isDS(FirstLdSt) && isDS(SecondLdSt)) {
441  FirstDst = getNamedOperand(FirstLdSt, AMDGPU::OpName::vdst);
442  SecondDst = getNamedOperand(SecondLdSt, AMDGPU::OpName::vdst);
443  }
444 
445  if (!FirstDst || !SecondDst)
446  return false;
447 
448  // Try to limit clustering based on the total number of bytes loaded
449  // rather than the number of instructions. This is done to help reduce
450  // register pressure. The method used is somewhat inexact, though,
451  // because it assumes that all loads in the cluster will load the
452  // same number of bytes as FirstLdSt.
453 
454  // The unit of this value is bytes.
455  // FIXME: This needs finer tuning.
456  unsigned LoadClusterThreshold = 16;
457 
458  const MachineRegisterInfo &MRI =
459  FirstLdSt.getParent()->getParent()->getRegInfo();
460 
461  const unsigned Reg = FirstDst->getReg();
462 
464  ? MRI.getRegClass(Reg)
465  : RI.getPhysRegClass(Reg);
466 
467  return (NumLoads * (RI.getRegSizeInBits(*DstRC) / 8)) <= LoadClusterThreshold;
468 }
469 
470 // FIXME: This behaves strangely. If, for example, you have 32 load + stores,
471 // the first 16 loads will be interleaved with the stores, and the next 16 will
472 // be clustered as expected. It should really split into 2 16 store batches.
473 //
474 // Loads are clustered until this returns false, rather than trying to schedule
475 // groups of stores. This also means we have to deal with saying different
476 // address space loads should be clustered, and ones which might cause bank
477 // conflicts.
478 //
479 // This might be deprecated so it might not be worth that much effort to fix.
481  int64_t Offset0, int64_t Offset1,
482  unsigned NumLoads) const {
483  assert(Offset1 > Offset0 &&
484  "Second offset should be larger than first offset!");
485  // If we have less than 16 loads in a row, and the offsets are within 64
486  // bytes, then schedule together.
487 
488  // A cacheline is 64 bytes (for global memory).
489  return (NumLoads <= 16 && (Offset1 - Offset0) < 64);
490 }
491 
494  const DebugLoc &DL, unsigned DestReg,
495  unsigned SrcReg, bool KillSrc) {
496  MachineFunction *MF = MBB.getParent();
497  DiagnosticInfoUnsupported IllegalCopy(MF->getFunction(),
498  "illegal SGPR to VGPR copy",
499  DL, DS_Error);
500  LLVMContext &C = MF->getFunction().getContext();
501  C.diagnose(IllegalCopy);
502 
503  BuildMI(MBB, MI, DL, TII->get(AMDGPU::SI_ILLEGAL_COPY), DestReg)
504  .addReg(SrcReg, getKillRegState(KillSrc));
505 }
506 
509  const DebugLoc &DL, unsigned DestReg,
510  unsigned SrcReg, bool KillSrc) const {
511  const TargetRegisterClass *RC = RI.getPhysRegClass(DestReg);
512 
513  if (RC == &AMDGPU::VGPR_32RegClass) {
514  assert(AMDGPU::VGPR_32RegClass.contains(SrcReg) ||
515  AMDGPU::SReg_32RegClass.contains(SrcReg));
516  BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DestReg)
517  .addReg(SrcReg, getKillRegState(KillSrc));
518  return;
519  }
520 
521  if (RC == &AMDGPU::SReg_32_XM0RegClass ||
522  RC == &AMDGPU::SReg_32RegClass) {
523  if (SrcReg == AMDGPU::SCC) {
524  BuildMI(MBB, MI, DL, get(AMDGPU::S_CSELECT_B32), DestReg)
525  .addImm(-1)
526  .addImm(0);
527  return;
528  }
529 
530  if (DestReg == AMDGPU::VCC_LO) {
531  if (AMDGPU::SReg_32RegClass.contains(SrcReg)) {
532  BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), AMDGPU::VCC_LO)
533  .addReg(SrcReg, getKillRegState(KillSrc));
534  } else {
535  // FIXME: Hack until VReg_1 removed.
536  assert(AMDGPU::VGPR_32RegClass.contains(SrcReg));
537  BuildMI(MBB, MI, DL, get(AMDGPU::V_CMP_NE_U32_e32))
538  .addImm(0)
539  .addReg(SrcReg, getKillRegState(KillSrc));
540  }
541 
542  return;
543  }
544 
545  if (!AMDGPU::SReg_32RegClass.contains(SrcReg)) {
546  reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc);
547  return;
548  }
549 
550  BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), DestReg)
551  .addReg(SrcReg, getKillRegState(KillSrc));
552  return;
553  }
554 
555  if (RC == &AMDGPU::SReg_64RegClass) {
556  if (DestReg == AMDGPU::VCC) {
557  if (AMDGPU::SReg_64RegClass.contains(SrcReg)) {
558  BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B64), AMDGPU::VCC)
559  .addReg(SrcReg, getKillRegState(KillSrc));
560  } else {
561  // FIXME: Hack until VReg_1 removed.
562  assert(AMDGPU::VGPR_32RegClass.contains(SrcReg));
563  BuildMI(MBB, MI, DL, get(AMDGPU::V_CMP_NE_U32_e32))
564  .addImm(0)
565  .addReg(SrcReg, getKillRegState(KillSrc));
566  }
567 
568  return;
569  }
570 
571  if (!AMDGPU::SReg_64RegClass.contains(SrcReg)) {
572  reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc);
573  return;
574  }
575 
576  BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B64), DestReg)
577  .addReg(SrcReg, getKillRegState(KillSrc));
578  return;
579  }
580 
581  if (DestReg == AMDGPU::SCC) {
582  assert(AMDGPU::SReg_32RegClass.contains(SrcReg));
583  BuildMI(MBB, MI, DL, get(AMDGPU::S_CMP_LG_U32))
584  .addReg(SrcReg, getKillRegState(KillSrc))
585  .addImm(0);
586  return;
587  }
588 
589  unsigned EltSize = 4;
590  unsigned Opcode = AMDGPU::V_MOV_B32_e32;
591  if (RI.isSGPRClass(RC)) {
592  // TODO: Copy vec3/vec5 with s_mov_b64s then final s_mov_b32.
593  if (!(RI.getRegSizeInBits(*RC) % 64)) {
594  Opcode = AMDGPU::S_MOV_B64;
595  EltSize = 8;
596  } else {
597  Opcode = AMDGPU::S_MOV_B32;
598  EltSize = 4;
599  }
600 
601  if (!RI.isSGPRClass(RI.getPhysRegClass(SrcReg))) {
602  reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc);
603  return;
604  }
605  }
606 
607  ArrayRef<int16_t> SubIndices = RI.getRegSplitParts(RC, EltSize);
608  bool Forward = RI.getHWRegIndex(DestReg) <= RI.getHWRegIndex(SrcReg);
609 
610  for (unsigned Idx = 0; Idx < SubIndices.size(); ++Idx) {
611  unsigned SubIdx;
612  if (Forward)
613  SubIdx = SubIndices[Idx];
614  else
615  SubIdx = SubIndices[SubIndices.size() - Idx - 1];
616 
617  MachineInstrBuilder Builder = BuildMI(MBB, MI, DL,
618  get(Opcode), RI.getSubReg(DestReg, SubIdx));
619 
620  Builder.addReg(RI.getSubReg(SrcReg, SubIdx));
621 
622  if (Idx == 0)
623  Builder.addReg(DestReg, RegState::Define | RegState::Implicit);
624 
625  bool UseKill = KillSrc && Idx == SubIndices.size() - 1;
626  Builder.addReg(SrcReg, getKillRegState(UseKill) | RegState::Implicit);
627  }
628 }
629 
630 int SIInstrInfo::commuteOpcode(unsigned Opcode) const {
631  int NewOpc;
632 
633  // Try to map original to commuted opcode
634  NewOpc = AMDGPU::getCommuteRev(Opcode);
635  if (NewOpc != -1)
636  // Check if the commuted (REV) opcode exists on the target.
637  return pseudoToMCOpcode(NewOpc) != -1 ? NewOpc : -1;
638 
639  // Try to map commuted to original opcode
640  NewOpc = AMDGPU::getCommuteOrig(Opcode);
641  if (NewOpc != -1)
642  // Check if the original (non-REV) opcode exists on the target.
643  return pseudoToMCOpcode(NewOpc) != -1 ? NewOpc : -1;
644 
645  return Opcode;
646 }
647 
650  const DebugLoc &DL, unsigned DestReg,
651  int64_t Value) const {
653  const TargetRegisterClass *RegClass = MRI.getRegClass(DestReg);
654  if (RegClass == &AMDGPU::SReg_32RegClass ||
655  RegClass == &AMDGPU::SGPR_32RegClass ||
656  RegClass == &AMDGPU::SReg_32_XM0RegClass ||
657  RegClass == &AMDGPU::SReg_32_XM0_XEXECRegClass) {
658  BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), DestReg)
659  .addImm(Value);
660  return;
661  }
662 
663  if (RegClass == &AMDGPU::SReg_64RegClass ||
664  RegClass == &AMDGPU::SGPR_64RegClass ||
665  RegClass == &AMDGPU::SReg_64_XEXECRegClass) {
666  BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B64), DestReg)
667  .addImm(Value);
668  return;
669  }
670 
671  if (RegClass == &AMDGPU::VGPR_32RegClass) {
672  BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DestReg)
673  .addImm(Value);
674  return;
675  }
676  if (RegClass == &AMDGPU::VReg_64RegClass) {
677  BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B64_PSEUDO), DestReg)
678  .addImm(Value);
679  return;
680  }
681 
682  unsigned EltSize = 4;
683  unsigned Opcode = AMDGPU::V_MOV_B32_e32;
684  if (RI.isSGPRClass(RegClass)) {
685  if (RI.getRegSizeInBits(*RegClass) > 32) {
686  Opcode = AMDGPU::S_MOV_B64;
687  EltSize = 8;
688  } else {
689  Opcode = AMDGPU::S_MOV_B32;
690  EltSize = 4;
691  }
692  }
693 
694  ArrayRef<int16_t> SubIndices = RI.getRegSplitParts(RegClass, EltSize);
695  for (unsigned Idx = 0; Idx < SubIndices.size(); ++Idx) {
696  int64_t IdxValue = Idx == 0 ? Value : 0;
697 
698  MachineInstrBuilder Builder = BuildMI(MBB, MI, DL,
699  get(Opcode), RI.getSubReg(DestReg, Idx));
700  Builder.addImm(IdxValue);
701  }
702 }
703 
704 const TargetRegisterClass *
706  return &AMDGPU::VGPR_32RegClass;
707 }
708 
711  const DebugLoc &DL, unsigned DstReg,
713  unsigned TrueReg,
714  unsigned FalseReg) const {
716  MachineFunction *MF = MBB.getParent();
717  const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
718  const TargetRegisterClass *BoolXExecRC =
719  RI.getRegClass(AMDGPU::SReg_1_XEXECRegClassID);
720  assert(MRI.getRegClass(DstReg) == &AMDGPU::VGPR_32RegClass &&
721  "Not a VGPR32 reg");
722 
723  if (Cond.size() == 1) {
724  unsigned SReg = MRI.createVirtualRegister(BoolXExecRC);
725  BuildMI(MBB, I, DL, get(AMDGPU::COPY), SReg)
726  .add(Cond[0]);
727  BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
728  .addImm(0)
729  .addReg(FalseReg)
730  .addImm(0)
731  .addReg(TrueReg)
732  .addReg(SReg);
733  } else if (Cond.size() == 2) {
734  assert(Cond[0].isImm() && "Cond[0] is not an immediate");
735  switch (Cond[0].getImm()) {
736  case SIInstrInfo::SCC_TRUE: {
737  unsigned SReg = MRI.createVirtualRegister(BoolXExecRC);
738  BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_CSELECT_B32
739  : AMDGPU::S_CSELECT_B64), SReg)
740  .addImm(-1)
741  .addImm(0);
742  BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
743  .addImm(0)
744  .addReg(FalseReg)
745  .addImm(0)
746  .addReg(TrueReg)
747  .addReg(SReg);
748  break;
749  }
750  case SIInstrInfo::SCC_FALSE: {
751  unsigned SReg = MRI.createVirtualRegister(BoolXExecRC);
752  BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_CSELECT_B32
753  : AMDGPU::S_CSELECT_B64), SReg)
754  .addImm(0)
755  .addImm(-1);
756  BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
757  .addImm(0)
758  .addReg(FalseReg)
759  .addImm(0)
760  .addReg(TrueReg)
761  .addReg(SReg);
762  break;
763  }
764  case SIInstrInfo::VCCNZ: {
765  MachineOperand RegOp = Cond[1];
766  RegOp.setImplicit(false);
767  unsigned SReg = MRI.createVirtualRegister(BoolXExecRC);
768  BuildMI(MBB, I, DL, get(AMDGPU::COPY), SReg)
769  .add(RegOp);
770  BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
771  .addImm(0)
772  .addReg(FalseReg)
773  .addImm(0)
774  .addReg(TrueReg)
775  .addReg(SReg);
776  break;
777  }
778  case SIInstrInfo::VCCZ: {
779  MachineOperand RegOp = Cond[1];
780  RegOp.setImplicit(false);
781  unsigned SReg = MRI.createVirtualRegister(BoolXExecRC);
782  BuildMI(MBB, I, DL, get(AMDGPU::COPY), SReg)
783  .add(RegOp);
784  BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
785  .addImm(0)
786  .addReg(TrueReg)
787  .addImm(0)
788  .addReg(FalseReg)
789  .addReg(SReg);
790  break;
791  }
792  case SIInstrInfo::EXECNZ: {
793  unsigned SReg = MRI.createVirtualRegister(BoolXExecRC);
794  unsigned SReg2 = MRI.createVirtualRegister(RI.getBoolRC());
795  BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_OR_SAVEEXEC_B32
796  : AMDGPU::S_OR_SAVEEXEC_B64), SReg2)
797  .addImm(0);
798  BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_CSELECT_B32
799  : AMDGPU::S_CSELECT_B64), SReg)
800  .addImm(-1)
801  .addImm(0);
802  BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
803  .addImm(0)
804  .addReg(FalseReg)
805  .addImm(0)
806  .addReg(TrueReg)
807  .addReg(SReg);
808  break;
809  }
810  case SIInstrInfo::EXECZ: {
811  unsigned SReg = MRI.createVirtualRegister(BoolXExecRC);
812  unsigned SReg2 = MRI.createVirtualRegister(RI.getBoolRC());
813  BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_OR_SAVEEXEC_B32
814  : AMDGPU::S_OR_SAVEEXEC_B64), SReg2)
815  .addImm(0);
816  BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_CSELECT_B32
817  : AMDGPU::S_CSELECT_B64), SReg)
818  .addImm(0)
819  .addImm(-1);
820  BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
821  .addImm(0)
822  .addReg(FalseReg)
823  .addImm(0)
824  .addReg(TrueReg)
825  .addReg(SReg);
826  llvm_unreachable("Unhandled branch predicate EXECZ");
827  break;
828  }
829  default:
830  llvm_unreachable("invalid branch predicate");
831  }
832  } else {
833  llvm_unreachable("Can only handle Cond size 1 or 2");
834  }
835 }
836 
839  const DebugLoc &DL,
840  unsigned SrcReg, int Value) const {
842  unsigned Reg = MRI.createVirtualRegister(RI.getBoolRC());
843  BuildMI(*MBB, I, DL, get(AMDGPU::V_CMP_EQ_I32_e64), Reg)
844  .addImm(Value)
845  .addReg(SrcReg);
846 
847  return Reg;
848 }
849 
852  const DebugLoc &DL,
853  unsigned SrcReg, int Value) const {
855  unsigned Reg = MRI.createVirtualRegister(RI.getBoolRC());
856  BuildMI(*MBB, I, DL, get(AMDGPU::V_CMP_NE_I32_e64), Reg)
857  .addImm(Value)
858  .addReg(SrcReg);
859 
860  return Reg;
861 }
862 
863 unsigned SIInstrInfo::getMovOpcode(const TargetRegisterClass *DstRC) const {
864 
865  if (RI.getRegSizeInBits(*DstRC) == 32) {
866  return RI.isSGPRClass(DstRC) ? AMDGPU::S_MOV_B32 : AMDGPU::V_MOV_B32_e32;
867  } else if (RI.getRegSizeInBits(*DstRC) == 64 && RI.isSGPRClass(DstRC)) {
868  return AMDGPU::S_MOV_B64;
869  } else if (RI.getRegSizeInBits(*DstRC) == 64 && !RI.isSGPRClass(DstRC)) {
870  return AMDGPU::V_MOV_B64_PSEUDO;
871  }
872  return AMDGPU::COPY;
873 }
874 
875 static unsigned getSGPRSpillSaveOpcode(unsigned Size) {
876  switch (Size) {
877  case 4:
878  return AMDGPU::SI_SPILL_S32_SAVE;
879  case 8:
880  return AMDGPU::SI_SPILL_S64_SAVE;
881  case 12:
882  return AMDGPU::SI_SPILL_S96_SAVE;
883  case 16:
884  return AMDGPU::SI_SPILL_S128_SAVE;
885  case 20:
886  return AMDGPU::SI_SPILL_S160_SAVE;
887  case 32:
888  return AMDGPU::SI_SPILL_S256_SAVE;
889  case 64:
890  return AMDGPU::SI_SPILL_S512_SAVE;
891  default:
892  llvm_unreachable("unknown register size");
893  }
894 }
895 
896 static unsigned getVGPRSpillSaveOpcode(unsigned Size) {
897  switch (Size) {
898  case 4:
899  return AMDGPU::SI_SPILL_V32_SAVE;
900  case 8:
901  return AMDGPU::SI_SPILL_V64_SAVE;
902  case 12:
903  return AMDGPU::SI_SPILL_V96_SAVE;
904  case 16:
905  return AMDGPU::SI_SPILL_V128_SAVE;
906  case 20:
907  return AMDGPU::SI_SPILL_V160_SAVE;
908  case 32:
909  return AMDGPU::SI_SPILL_V256_SAVE;
910  case 64:
911  return AMDGPU::SI_SPILL_V512_SAVE;
912  default:
913  llvm_unreachable("unknown register size");
914  }
915 }
916 
919  unsigned SrcReg, bool isKill,
920  int FrameIndex,
921  const TargetRegisterClass *RC,
922  const TargetRegisterInfo *TRI) const {
923  MachineFunction *MF = MBB.getParent();
925  MachineFrameInfo &FrameInfo = MF->getFrameInfo();
926  const DebugLoc &DL = MBB.findDebugLoc(MI);
927 
928  unsigned Size = FrameInfo.getObjectSize(FrameIndex);
929  unsigned Align = FrameInfo.getObjectAlignment(FrameIndex);
930  MachinePointerInfo PtrInfo
931  = MachinePointerInfo::getFixedStack(*MF, FrameIndex);
932  MachineMemOperand *MMO
934  Size, Align);
935  unsigned SpillSize = TRI->getSpillSize(*RC);
936 
937  if (RI.isSGPRClass(RC)) {
938  MFI->setHasSpilledSGPRs();
939 
940  // We are only allowed to create one new instruction when spilling
941  // registers, so we need to use pseudo instruction for spilling SGPRs.
942  const MCInstrDesc &OpDesc = get(getSGPRSpillSaveOpcode(SpillSize));
943 
944  // The SGPR spill/restore instructions only work on number sgprs, so we need
945  // to make sure we are using the correct register class.
946  if (TargetRegisterInfo::isVirtualRegister(SrcReg) && SpillSize == 4) {
948  MRI.constrainRegClass(SrcReg, &AMDGPU::SReg_32_XM0RegClass);
949  }
950 
951  MachineInstrBuilder Spill = BuildMI(MBB, MI, DL, OpDesc)
952  .addReg(SrcReg, getKillRegState(isKill)) // data
953  .addFrameIndex(FrameIndex) // addr
954  .addMemOperand(MMO)
956  .addReg(MFI->getStackPtrOffsetReg(), RegState::Implicit);
957  // Add the scratch resource registers as implicit uses because we may end up
958  // needing them, and need to ensure that the reserved registers are
959  // correctly handled.
960 
961  FrameInfo.setStackID(FrameIndex, TargetStackID::SGPRSpill);
962  if (ST.hasScalarStores()) {
963  // m0 is used for offset to scalar stores if used to spill.
964  Spill.addReg(AMDGPU::M0, RegState::ImplicitDefine | RegState::Dead);
965  }
966 
967  return;
968  }
969 
970  assert(RI.hasVGPRs(RC) && "Only VGPR spilling expected");
971 
972  unsigned Opcode = getVGPRSpillSaveOpcode(SpillSize);
973  MFI->setHasSpilledVGPRs();
974  BuildMI(MBB, MI, DL, get(Opcode))
975  .addReg(SrcReg, getKillRegState(isKill)) // data
976  .addFrameIndex(FrameIndex) // addr
977  .addReg(MFI->getScratchRSrcReg()) // scratch_rsrc
978  .addReg(MFI->getStackPtrOffsetReg()) // scratch_offset
979  .addImm(0) // offset
980  .addMemOperand(MMO);
981 }
982 
983 static unsigned getSGPRSpillRestoreOpcode(unsigned Size) {
984  switch (Size) {
985  case 4:
986  return AMDGPU::SI_SPILL_S32_RESTORE;
987  case 8:
988  return AMDGPU::SI_SPILL_S64_RESTORE;
989  case 12:
990  return AMDGPU::SI_SPILL_S96_RESTORE;
991  case 16:
992  return AMDGPU::SI_SPILL_S128_RESTORE;
993  case 20:
994  return AMDGPU::SI_SPILL_S160_RESTORE;
995  case 32:
996  return AMDGPU::SI_SPILL_S256_RESTORE;
997  case 64:
998  return AMDGPU::SI_SPILL_S512_RESTORE;
999  default:
1000  llvm_unreachable("unknown register size");
1001  }
1002 }
1003 
1004 static unsigned getVGPRSpillRestoreOpcode(unsigned Size) {
1005  switch (Size) {
1006  case 4:
1007  return AMDGPU::SI_SPILL_V32_RESTORE;
1008  case 8:
1009  return AMDGPU::SI_SPILL_V64_RESTORE;
1010  case 12:
1011  return AMDGPU::SI_SPILL_V96_RESTORE;
1012  case 16:
1013  return AMDGPU::SI_SPILL_V128_RESTORE;
1014  case 20:
1015  return AMDGPU::SI_SPILL_V160_RESTORE;
1016  case 32:
1017  return AMDGPU::SI_SPILL_V256_RESTORE;
1018  case 64:
1019  return AMDGPU::SI_SPILL_V512_RESTORE;
1020  default:
1021  llvm_unreachable("unknown register size");
1022  }
1023 }
1024 
1027  unsigned DestReg, int FrameIndex,
1028  const TargetRegisterClass *RC,
1029  const TargetRegisterInfo *TRI) const {
1030  MachineFunction *MF = MBB.getParent();
1032  MachineFrameInfo &FrameInfo = MF->getFrameInfo();
1033  const DebugLoc &DL = MBB.findDebugLoc(MI);
1034  unsigned Align = FrameInfo.getObjectAlignment(FrameIndex);
1035  unsigned Size = FrameInfo.getObjectSize(FrameIndex);
1036  unsigned SpillSize = TRI->getSpillSize(*RC);
1037 
1038  MachinePointerInfo PtrInfo
1039  = MachinePointerInfo::getFixedStack(*MF, FrameIndex);
1040 
1042  PtrInfo, MachineMemOperand::MOLoad, Size, Align);
1043 
1044  if (RI.isSGPRClass(RC)) {
1045  MFI->setHasSpilledSGPRs();
1046 
1047  // FIXME: Maybe this should not include a memoperand because it will be
1048  // lowered to non-memory instructions.
1049  const MCInstrDesc &OpDesc = get(getSGPRSpillRestoreOpcode(SpillSize));
1050  if (TargetRegisterInfo::isVirtualRegister(DestReg) && SpillSize == 4) {
1052  MRI.constrainRegClass(DestReg, &AMDGPU::SReg_32_XM0RegClass);
1053  }
1054 
1055  FrameInfo.setStackID(FrameIndex, TargetStackID::SGPRSpill);
1056  MachineInstrBuilder Spill = BuildMI(MBB, MI, DL, OpDesc, DestReg)
1057  .addFrameIndex(FrameIndex) // addr
1058  .addMemOperand(MMO)
1060  .addReg(MFI->getStackPtrOffsetReg(), RegState::Implicit);
1061 
1062  if (ST.hasScalarStores()) {
1063  // m0 is used for offset to scalar stores if used to spill.
1064  Spill.addReg(AMDGPU::M0, RegState::ImplicitDefine | RegState::Dead);
1065  }
1066 
1067  return;
1068  }
1069 
1070  assert(RI.hasVGPRs(RC) && "Only VGPR spilling expected");
1071 
1072  unsigned Opcode = getVGPRSpillRestoreOpcode(SpillSize);
1073  BuildMI(MBB, MI, DL, get(Opcode), DestReg)
1074  .addFrameIndex(FrameIndex) // vaddr
1075  .addReg(MFI->getScratchRSrcReg()) // scratch_rsrc
1076  .addReg(MFI->getStackPtrOffsetReg()) // scratch_offset
1077  .addImm(0) // offset
1078  .addMemOperand(MMO);
1079 }
1080 
1081 /// \param @Offset Offset in bytes of the FrameIndex being spilled
1083  MachineBasicBlock &MBB, MachineInstr &MI, RegScavenger *RS, unsigned TmpReg,
1084  unsigned FrameOffset, unsigned Size) const {
1085  MachineFunction *MF = MBB.getParent();
1087  const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
1088  const DebugLoc &DL = MBB.findDebugLoc(MI);
1089  unsigned WorkGroupSize = MFI->getMaxFlatWorkGroupSize();
1090  unsigned WavefrontSize = ST.getWavefrontSize();
1091 
1092  unsigned TIDReg = MFI->getTIDReg();
1093  if (!MFI->hasCalculatedTID()) {
1094  MachineBasicBlock &Entry = MBB.getParent()->front();
1095  MachineBasicBlock::iterator Insert = Entry.front();
1096  const DebugLoc &DL = Insert->getDebugLoc();
1097 
1098  TIDReg = RI.findUnusedRegister(MF->getRegInfo(), &AMDGPU::VGPR_32RegClass,
1099  *MF);
1100  if (TIDReg == AMDGPU::NoRegister)
1101  return TIDReg;
1102 
1104  WorkGroupSize > WavefrontSize) {
1105  unsigned TIDIGXReg
1107  unsigned TIDIGYReg
1109  unsigned TIDIGZReg
1111  unsigned InputPtrReg =
1113  for (unsigned Reg : {TIDIGXReg, TIDIGYReg, TIDIGZReg}) {
1114  if (!Entry.isLiveIn(Reg))
1115  Entry.addLiveIn(Reg);
1116  }
1117 
1118  RS->enterBasicBlock(Entry);
1119  // FIXME: Can we scavenge an SReg_64 and access the subregs?
1120  unsigned STmp0 = RS->scavengeRegister(&AMDGPU::SGPR_32RegClass, 0);
1121  unsigned STmp1 = RS->scavengeRegister(&AMDGPU::SGPR_32RegClass, 0);
1122  BuildMI(Entry, Insert, DL, get(AMDGPU::S_LOAD_DWORD_IMM), STmp0)
1123  .addReg(InputPtrReg)
1125  BuildMI(Entry, Insert, DL, get(AMDGPU::S_LOAD_DWORD_IMM), STmp1)
1126  .addReg(InputPtrReg)
1128 
1129  // NGROUPS.X * NGROUPS.Y
1130  BuildMI(Entry, Insert, DL, get(AMDGPU::S_MUL_I32), STmp1)
1131  .addReg(STmp1)
1132  .addReg(STmp0);
1133  // (NGROUPS.X * NGROUPS.Y) * TIDIG.X
1134  BuildMI(Entry, Insert, DL, get(AMDGPU::V_MUL_U32_U24_e32), TIDReg)
1135  .addReg(STmp1)
1136  .addReg(TIDIGXReg);
1137  // NGROUPS.Z * TIDIG.Y + (NGROUPS.X * NGROPUS.Y * TIDIG.X)
1138  BuildMI(Entry, Insert, DL, get(AMDGPU::V_MAD_U32_U24), TIDReg)
1139  .addReg(STmp0)
1140  .addReg(TIDIGYReg)
1141  .addReg(TIDReg);
1142  // (NGROUPS.Z * TIDIG.Y + (NGROUPS.X * NGROPUS.Y * TIDIG.X)) + TIDIG.Z
1143  getAddNoCarry(Entry, Insert, DL, TIDReg)
1144  .addReg(TIDReg)
1145  .addReg(TIDIGZReg)
1146  .addImm(0); // clamp bit
1147  } else {
1148  // Get the wave id
1149  BuildMI(Entry, Insert, DL, get(AMDGPU::V_MBCNT_LO_U32_B32_e64),
1150  TIDReg)
1151  .addImm(-1)
1152  .addImm(0);
1153 
1154  BuildMI(Entry, Insert, DL, get(AMDGPU::V_MBCNT_HI_U32_B32_e64),
1155  TIDReg)
1156  .addImm(-1)
1157  .addReg(TIDReg);
1158  }
1159 
1160  BuildMI(Entry, Insert, DL, get(AMDGPU::V_LSHLREV_B32_e32),
1161  TIDReg)
1162  .addImm(2)
1163  .addReg(TIDReg);
1164  MFI->setTIDReg(TIDReg);
1165  }
1166 
1167  // Add FrameIndex to LDS offset
1168  unsigned LDSOffset = MFI->getLDSSize() + (FrameOffset * WorkGroupSize);
1169  getAddNoCarry(MBB, MI, DL, TmpReg)
1170  .addImm(LDSOffset)
1171  .addReg(TIDReg)
1172  .addImm(0); // clamp bit
1173 
1174  return TmpReg;
1175 }
1176 
1179  int Count) const {
1180  DebugLoc DL = MBB.findDebugLoc(MI);
1181  while (Count > 0) {
1182  int Arg;
1183  if (Count >= 8)
1184  Arg = 7;
1185  else
1186  Arg = Count - 1;
1187  Count -= 8;
1188  BuildMI(MBB, MI, DL, get(AMDGPU::S_NOP))
1189  .addImm(Arg);
1190  }
1191 }
1192 
1195  insertWaitStates(MBB, MI, 1);
1196 }
1197 
1199  auto MF = MBB.getParent();
1201 
1202  assert(Info->isEntryFunction());
1203 
1204  if (MBB.succ_empty()) {
1205  bool HasNoTerminator = MBB.getFirstTerminator() == MBB.end();
1206  if (HasNoTerminator) {
1207  if (Info->returnsVoid()) {
1208  BuildMI(MBB, MBB.end(), DebugLoc(), get(AMDGPU::S_ENDPGM)).addImm(0);
1209  } else {
1210  BuildMI(MBB, MBB.end(), DebugLoc(), get(AMDGPU::SI_RETURN_TO_EPILOG));
1211  }
1212  }
1213  }
1214 }
1215 
1217  switch (MI.getOpcode()) {
1218  default: return 1; // FIXME: Do wait states equal cycles?
1219 
1220  case AMDGPU::S_NOP:
1221  return MI.getOperand(0).getImm() + 1;
1222  }
1223 }
1224 
1226  MachineBasicBlock &MBB = *MI.getParent();
1227  DebugLoc DL = MBB.findDebugLoc(MI);
1228  switch (MI.getOpcode()) {
1229  default: return TargetInstrInfo::expandPostRAPseudo(MI);
1230  case AMDGPU::S_MOV_B64_term:
1231  // This is only a terminator to get the correct spill code placement during
1232  // register allocation.
1233  MI.setDesc(get(AMDGPU::S_MOV_B64));
1234  break;
1235 
1236  case AMDGPU::S_MOV_B32_term:
1237  // This is only a terminator to get the correct spill code placement during
1238  // register allocation.
1239  MI.setDesc(get(AMDGPU::S_MOV_B32));
1240  break;
1241 
1242  case AMDGPU::S_XOR_B64_term:
1243  // This is only a terminator to get the correct spill code placement during
1244  // register allocation.
1245  MI.setDesc(get(AMDGPU::S_XOR_B64));
1246  break;
1247 
1248  case AMDGPU::S_XOR_B32_term:
1249  // This is only a terminator to get the correct spill code placement during
1250  // register allocation.
1251  MI.setDesc(get(AMDGPU::S_XOR_B32));
1252  break;
1253 
1254  case AMDGPU::S_OR_B32_term:
1255  // This is only a terminator to get the correct spill code placement during
1256  // register allocation.
1257  MI.setDesc(get(AMDGPU::S_OR_B32));
1258  break;
1259 
1260  case AMDGPU::S_ANDN2_B64_term:
1261  // This is only a terminator to get the correct spill code placement during
1262  // register allocation.
1263  MI.setDesc(get(AMDGPU::S_ANDN2_B64));
1264  break;
1265 
1266  case AMDGPU::S_ANDN2_B32_term:
1267  // This is only a terminator to get the correct spill code placement during
1268  // register allocation.
1269  MI.setDesc(get(AMDGPU::S_ANDN2_B32));
1270  break;
1271 
1272  case AMDGPU::V_MOV_B64_PSEUDO: {
1273  unsigned Dst = MI.getOperand(0).getReg();
1274  unsigned DstLo = RI.getSubReg(Dst, AMDGPU::sub0);
1275  unsigned DstHi = RI.getSubReg(Dst, AMDGPU::sub1);
1276 
1277  const MachineOperand &SrcOp = MI.getOperand(1);
1278  // FIXME: Will this work for 64-bit floating point immediates?
1279  assert(!SrcOp.isFPImm());
1280  if (SrcOp.isImm()) {
1281  APInt Imm(64, SrcOp.getImm());
1282  BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstLo)
1283  .addImm(Imm.getLoBits(32).getZExtValue())
1284  .addReg(Dst, RegState::Implicit | RegState::Define);
1285  BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstHi)
1286  .addImm(Imm.getHiBits(32).getZExtValue())
1287  .addReg(Dst, RegState::Implicit | RegState::Define);
1288  } else {
1289  assert(SrcOp.isReg());
1290  BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstLo)
1291  .addReg(RI.getSubReg(SrcOp.getReg(), AMDGPU::sub0))
1293  BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstHi)
1294  .addReg(RI.getSubReg(SrcOp.getReg(), AMDGPU::sub1))
1296  }
1297  MI.eraseFromParent();
1298  break;
1299  }
1300  case AMDGPU::V_SET_INACTIVE_B32: {
1301  unsigned NotOpc = ST.isWave32() ? AMDGPU::S_NOT_B32 : AMDGPU::S_NOT_B64;
1302  unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
1303  BuildMI(MBB, MI, DL, get(NotOpc), Exec)
1304  .addReg(Exec);
1305  BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), MI.getOperand(0).getReg())
1306  .add(MI.getOperand(2));
1307  BuildMI(MBB, MI, DL, get(NotOpc), Exec)
1308  .addReg(Exec);
1309  MI.eraseFromParent();
1310  break;
1311  }
1312  case AMDGPU::V_SET_INACTIVE_B64: {
1313  unsigned NotOpc = ST.isWave32() ? AMDGPU::S_NOT_B32 : AMDGPU::S_NOT_B64;
1314  unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
1315  BuildMI(MBB, MI, DL, get(NotOpc), Exec)
1316  .addReg(Exec);
1317  MachineInstr *Copy = BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B64_PSEUDO),
1318  MI.getOperand(0).getReg())
1319  .add(MI.getOperand(2));
1320  expandPostRAPseudo(*Copy);
1321  BuildMI(MBB, MI, DL, get(NotOpc), Exec)
1322  .addReg(Exec);
1323  MI.eraseFromParent();
1324  break;
1325  }
1326  case AMDGPU::V_MOVRELD_B32_V1:
1327  case AMDGPU::V_MOVRELD_B32_V2:
1328  case AMDGPU::V_MOVRELD_B32_V4:
1329  case AMDGPU::V_MOVRELD_B32_V8:
1330  case AMDGPU::V_MOVRELD_B32_V16: {
1331  const MCInstrDesc &MovRelDesc = get(AMDGPU::V_MOVRELD_B32_e32);
1332  unsigned VecReg = MI.getOperand(0).getReg();
1333  bool IsUndef = MI.getOperand(1).isUndef();
1334  unsigned SubReg = AMDGPU::sub0 + MI.getOperand(3).getImm();
1335  assert(VecReg == MI.getOperand(1).getReg());
1336 
1337  MachineInstr *MovRel =
1338  BuildMI(MBB, MI, DL, MovRelDesc)
1339  .addReg(RI.getSubReg(VecReg, SubReg), RegState::Undef)
1340  .add(MI.getOperand(2))
1341  .addReg(VecReg, RegState::ImplicitDefine)
1342  .addReg(VecReg,
1343  RegState::Implicit | (IsUndef ? RegState::Undef : 0));
1344 
1345  const int ImpDefIdx =
1346  MovRelDesc.getNumOperands() + MovRelDesc.getNumImplicitUses();
1347  const int ImpUseIdx = ImpDefIdx + 1;
1348  MovRel->tieOperands(ImpDefIdx, ImpUseIdx);
1349 
1350  MI.eraseFromParent();
1351  break;
1352  }
1353  case AMDGPU::SI_PC_ADD_REL_OFFSET: {
1354  MachineFunction &MF = *MBB.getParent();
1355  unsigned Reg = MI.getOperand(0).getReg();
1356  unsigned RegLo = RI.getSubReg(Reg, AMDGPU::sub0);
1357  unsigned RegHi = RI.getSubReg(Reg, AMDGPU::sub1);
1358 
1359  // Create a bundle so these instructions won't be re-ordered by the
1360  // post-RA scheduler.
1361  MIBundleBuilder Bundler(MBB, MI);
1362  Bundler.append(BuildMI(MF, DL, get(AMDGPU::S_GETPC_B64), Reg));
1363 
1364  // Add 32-bit offset from this instruction to the start of the
1365  // constant data.
1366  Bundler.append(BuildMI(MF, DL, get(AMDGPU::S_ADD_U32), RegLo)
1367  .addReg(RegLo)
1368  .add(MI.getOperand(1)));
1369 
1370  MachineInstrBuilder MIB = BuildMI(MF, DL, get(AMDGPU::S_ADDC_U32), RegHi)
1371  .addReg(RegHi);
1372  MIB.add(MI.getOperand(2));
1373 
1374  Bundler.append(MIB);
1375  finalizeBundle(MBB, Bundler.begin());
1376 
1377  MI.eraseFromParent();
1378  break;
1379  }
1380  case AMDGPU::ENTER_WWM: {
1381  // This only gets its own opcode so that SIPreAllocateWWMRegs can tell when
1382  // WWM is entered.
1383  MI.setDesc(get(ST.isWave32() ? AMDGPU::S_OR_SAVEEXEC_B32
1384  : AMDGPU::S_OR_SAVEEXEC_B64));
1385  break;
1386  }
1387  case AMDGPU::EXIT_WWM: {
1388  // This only gets its own opcode so that SIPreAllocateWWMRegs can tell when
1389  // WWM is exited.
1390  MI.setDesc(get(ST.isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64));
1391  break;
1392  }
1393  case TargetOpcode::BUNDLE: {
1394  if (!MI.mayLoad())
1395  return false;
1396 
1397  // If it is a load it must be a memory clause
1399  I->isBundledWithSucc(); ++I) {
1400  I->unbundleFromSucc();
1401  for (MachineOperand &MO : I->operands())
1402  if (MO.isReg())
1403  MO.setIsInternalRead(false);
1404  }
1405 
1406  MI.eraseFromParent();
1407  break;
1408  }
1409  }
1410  return true;
1411 }
1412 
1414  MachineOperand &Src0,
1415  unsigned Src0OpName,
1416  MachineOperand &Src1,
1417  unsigned Src1OpName) const {
1418  MachineOperand *Src0Mods = getNamedOperand(MI, Src0OpName);
1419  if (!Src0Mods)
1420  return false;
1421 
1422  MachineOperand *Src1Mods = getNamedOperand(MI, Src1OpName);
1423  assert(Src1Mods &&
1424  "All commutable instructions have both src0 and src1 modifiers");
1425 
1426  int Src0ModsVal = Src0Mods->getImm();
1427  int Src1ModsVal = Src1Mods->getImm();
1428 
1429  Src1Mods->setImm(Src0ModsVal);
1430  Src0Mods->setImm(Src1ModsVal);
1431  return true;
1432 }
1433 
1435  MachineOperand &RegOp,
1436  MachineOperand &NonRegOp) {
1437  unsigned Reg = RegOp.getReg();
1438  unsigned SubReg = RegOp.getSubReg();
1439  bool IsKill = RegOp.isKill();
1440  bool IsDead = RegOp.isDead();
1441  bool IsUndef = RegOp.isUndef();
1442  bool IsDebug = RegOp.isDebug();
1443 
1444  if (NonRegOp.isImm())
1445  RegOp.ChangeToImmediate(NonRegOp.getImm());
1446  else if (NonRegOp.isFI())
1447  RegOp.ChangeToFrameIndex(NonRegOp.getIndex());
1448  else
1449  return nullptr;
1450 
1451  NonRegOp.ChangeToRegister(Reg, false, false, IsKill, IsDead, IsUndef, IsDebug);
1452  NonRegOp.setSubReg(SubReg);
1453 
1454  return &MI;
1455 }
1456 
1458  unsigned Src0Idx,
1459  unsigned Src1Idx) const {
1460  assert(!NewMI && "this should never be used");
1461 
1462  unsigned Opc = MI.getOpcode();
1463  int CommutedOpcode = commuteOpcode(Opc);
1464  if (CommutedOpcode == -1)
1465  return nullptr;
1466 
1467  assert(AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0) ==
1468  static_cast<int>(Src0Idx) &&
1469  AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1) ==
1470  static_cast<int>(Src1Idx) &&
1471  "inconsistency with findCommutedOpIndices");
1472 
1473  MachineOperand &Src0 = MI.getOperand(Src0Idx);
1474  MachineOperand &Src1 = MI.getOperand(Src1Idx);
1475 
1476  MachineInstr *CommutedMI = nullptr;
1477  if (Src0.isReg() && Src1.isReg()) {
1478  if (isOperandLegal(MI, Src1Idx, &Src0)) {
1479  // Be sure to copy the source modifiers to the right place.
1480  CommutedMI
1481  = TargetInstrInfo::commuteInstructionImpl(MI, NewMI, Src0Idx, Src1Idx);
1482  }
1483 
1484  } else if (Src0.isReg() && !Src1.isReg()) {
1485  // src0 should always be able to support any operand type, so no need to
1486  // check operand legality.
1487  CommutedMI = swapRegAndNonRegOperand(MI, Src0, Src1);
1488  } else if (!Src0.isReg() && Src1.isReg()) {
1489  if (isOperandLegal(MI, Src1Idx, &Src0))
1490  CommutedMI = swapRegAndNonRegOperand(MI, Src1, Src0);
1491  } else {
1492  // FIXME: Found two non registers to commute. This does happen.
1493  return nullptr;
1494  }
1495 
1496  if (CommutedMI) {
1497  swapSourceModifiers(MI, Src0, AMDGPU::OpName::src0_modifiers,
1498  Src1, AMDGPU::OpName::src1_modifiers);
1499 
1500  CommutedMI->setDesc(get(CommutedOpcode));
1501  }
1502 
1503  return CommutedMI;
1504 }
1505 
1506 // This needs to be implemented because the source modifiers may be inserted
1507 // between the true commutable operands, and the base
1508 // TargetInstrInfo::commuteInstruction uses it.
1510  unsigned &SrcOpIdx1) const {
1511  return findCommutedOpIndices(MI.getDesc(), SrcOpIdx0, SrcOpIdx1);
1512 }
1513 
1514 bool SIInstrInfo::findCommutedOpIndices(MCInstrDesc Desc, unsigned &SrcOpIdx0,
1515  unsigned &SrcOpIdx1) const {
1516  if (!Desc.isCommutable())
1517  return false;
1518 
1519  unsigned Opc = Desc.getOpcode();
1520  int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0);
1521  if (Src0Idx == -1)
1522  return false;
1523 
1524  int Src1Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1);
1525  if (Src1Idx == -1)
1526  return false;
1527 
1528  return fixCommutedOpIndices(SrcOpIdx0, SrcOpIdx1, Src0Idx, Src1Idx);
1529 }
1530 
1531 bool SIInstrInfo::isBranchOffsetInRange(unsigned BranchOp,
1532  int64_t BrOffset) const {
1533  // BranchRelaxation should never have to check s_setpc_b64 because its dest
1534  // block is unanalyzable.
1535  assert(BranchOp != AMDGPU::S_SETPC_B64);
1536 
1537  // Convert to dwords.
1538  BrOffset /= 4;
1539 
1540  // The branch instructions do PC += signext(SIMM16 * 4) + 4, so the offset is
1541  // from the next instruction.
1542  BrOffset -= 1;
1543 
1544  return isIntN(BranchOffsetBits, BrOffset);
1545 }
1546 
1548  const MachineInstr &MI) const {
1549  if (MI.getOpcode() == AMDGPU::S_SETPC_B64) {
1550  // This would be a difficult analysis to perform, but can always be legal so
1551  // there's no need to analyze it.
1552  return nullptr;
1553  }
1554 
1555  return MI.getOperand(0).getMBB();
1556 }
1557 
1559  MachineBasicBlock &DestBB,
1560  const DebugLoc &DL,
1561  int64_t BrOffset,
1562  RegScavenger *RS) const {
1563  assert(RS && "RegScavenger required for long branching");
1564  assert(MBB.empty() &&
1565  "new block should be inserted for expanding unconditional branch");
1566  assert(MBB.pred_size() == 1);
1567 
1568  MachineFunction *MF = MBB.getParent();
1569  MachineRegisterInfo &MRI = MF->getRegInfo();
1570 
1571  // FIXME: Virtual register workaround for RegScavenger not working with empty
1572  // blocks.
1573  unsigned PCReg = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
1574 
1575  auto I = MBB.end();
1576 
1577  // We need to compute the offset relative to the instruction immediately after
1578  // s_getpc_b64. Insert pc arithmetic code before last terminator.
1579  MachineInstr *GetPC = BuildMI(MBB, I, DL, get(AMDGPU::S_GETPC_B64), PCReg);
1580 
1581  // TODO: Handle > 32-bit block address.
1582  if (BrOffset >= 0) {
1583  BuildMI(MBB, I, DL, get(AMDGPU::S_ADD_U32))
1584  .addReg(PCReg, RegState::Define, AMDGPU::sub0)
1585  .addReg(PCReg, 0, AMDGPU::sub0)
1586  .addMBB(&DestBB, MO_LONG_BRANCH_FORWARD);
1587  BuildMI(MBB, I, DL, get(AMDGPU::S_ADDC_U32))
1588  .addReg(PCReg, RegState::Define, AMDGPU::sub1)
1589  .addReg(PCReg, 0, AMDGPU::sub1)
1590  .addImm(0);
1591  } else {
1592  // Backwards branch.
1593  BuildMI(MBB, I, DL, get(AMDGPU::S_SUB_U32))
1594  .addReg(PCReg, RegState::Define, AMDGPU::sub0)
1595  .addReg(PCReg, 0, AMDGPU::sub0)
1596  .addMBB(&DestBB, MO_LONG_BRANCH_BACKWARD);
1597  BuildMI(MBB, I, DL, get(AMDGPU::S_SUBB_U32))
1598  .addReg(PCReg, RegState::Define, AMDGPU::sub1)
1599  .addReg(PCReg, 0, AMDGPU::sub1)
1600  .addImm(0);
1601  }
1602 
1603  // Insert the indirect branch after the other terminator.
1604  BuildMI(&MBB, DL, get(AMDGPU::S_SETPC_B64))
1605  .addReg(PCReg);
1606 
1607  // FIXME: If spilling is necessary, this will fail because this scavenger has
1608  // no emergency stack slots. It is non-trivial to spill in this situation,
1609  // because the restore code needs to be specially placed after the
1610  // jump. BranchRelaxation then needs to be made aware of the newly inserted
1611  // block.
1612  //
1613  // If a spill is needed for the pc register pair, we need to insert a spill
1614  // restore block right before the destination block, and insert a short branch
1615  // into the old destination block's fallthrough predecessor.
1616  // e.g.:
1617  //
1618  // s_cbranch_scc0 skip_long_branch:
1619  //
1620  // long_branch_bb:
1621  // spill s[8:9]
1622  // s_getpc_b64 s[8:9]
1623  // s_add_u32 s8, s8, restore_bb
1624  // s_addc_u32 s9, s9, 0
1625  // s_setpc_b64 s[8:9]
1626  //
1627  // skip_long_branch:
1628  // foo;
1629  //
1630  // .....
1631  //
1632  // dest_bb_fallthrough_predecessor:
1633  // bar;
1634  // s_branch dest_bb
1635  //
1636  // restore_bb:
1637  // restore s[8:9]
1638  // fallthrough dest_bb
1639  ///
1640  // dest_bb:
1641  // buzz;
1642 
1643  RS->enterBasicBlockEnd(MBB);
1644  unsigned Scav = RS->scavengeRegisterBackwards(
1645  AMDGPU::SReg_64RegClass,
1646  MachineBasicBlock::iterator(GetPC), false, 0);
1647  MRI.replaceRegWith(PCReg, Scav);
1648  MRI.clearVirtRegs();
1649  RS->setRegUsed(Scav);
1650 
1651  return 4 + 8 + 4 + 4;
1652 }
1653 
1654 unsigned SIInstrInfo::getBranchOpcode(SIInstrInfo::BranchPredicate Cond) {
1655  switch (Cond) {
1656  case SIInstrInfo::SCC_TRUE:
1657  return AMDGPU::S_CBRANCH_SCC1;
1658  case SIInstrInfo::SCC_FALSE:
1659  return AMDGPU::S_CBRANCH_SCC0;
1660  case SIInstrInfo::VCCNZ:
1661  return AMDGPU::S_CBRANCH_VCCNZ;
1662  case SIInstrInfo::VCCZ:
1663  return AMDGPU::S_CBRANCH_VCCZ;
1664  case SIInstrInfo::EXECNZ:
1665  return AMDGPU::S_CBRANCH_EXECNZ;
1666  case SIInstrInfo::EXECZ:
1667  return AMDGPU::S_CBRANCH_EXECZ;
1668  default:
1669  llvm_unreachable("invalid branch predicate");
1670  }
1671 }
1672 
1673 SIInstrInfo::BranchPredicate SIInstrInfo::getBranchPredicate(unsigned Opcode) {
1674  switch (Opcode) {
1675  case AMDGPU::S_CBRANCH_SCC0:
1676  return SCC_FALSE;
1677  case AMDGPU::S_CBRANCH_SCC1:
1678  return SCC_TRUE;
1679  case AMDGPU::S_CBRANCH_VCCNZ:
1680  return VCCNZ;
1681  case AMDGPU::S_CBRANCH_VCCZ:
1682  return VCCZ;
1683  case AMDGPU::S_CBRANCH_EXECNZ:
1684  return EXECNZ;
1685  case AMDGPU::S_CBRANCH_EXECZ:
1686  return EXECZ;
1687  default:
1688  return INVALID_BR;
1689  }
1690 }
1691 
1694  MachineBasicBlock *&TBB,
1695  MachineBasicBlock *&FBB,
1697  bool AllowModify) const {
1698  if (I->getOpcode() == AMDGPU::S_BRANCH) {
1699  // Unconditional Branch
1700  TBB = I->getOperand(0).getMBB();
1701  return false;
1702  }
1703 
1704  MachineBasicBlock *CondBB = nullptr;
1705 
1706  if (I->getOpcode() == AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO) {
1707  CondBB = I->getOperand(1).getMBB();
1708  Cond.push_back(I->getOperand(0));
1709  } else {
1710  BranchPredicate Pred = getBranchPredicate(I->getOpcode());
1711  if (Pred == INVALID_BR)
1712  return true;
1713 
1714  CondBB = I->getOperand(0).getMBB();
1716  Cond.push_back(I->getOperand(1)); // Save the branch register.
1717  }
1718  ++I;
1719 
1720  if (I == MBB.end()) {
1721  // Conditional branch followed by fall-through.
1722  TBB = CondBB;
1723  return false;
1724  }
1725 
1726  if (I->getOpcode() == AMDGPU::S_BRANCH) {
1727  TBB = CondBB;
1728  FBB = I->getOperand(0).getMBB();
1729  return false;
1730  }
1731 
1732  return true;
1733 }
1734 
1736  MachineBasicBlock *&FBB,
1738  bool AllowModify) const {
1740  auto E = MBB.end();
1741  if (I == E)
1742  return false;
1743 
1744  // Skip over the instructions that are artificially terminators for special
1745  // exec management.
1746  while (I != E && !I->isBranch() && !I->isReturn() &&
1747  I->getOpcode() != AMDGPU::SI_MASK_BRANCH) {
1748  switch (I->getOpcode()) {
1749  case AMDGPU::SI_MASK_BRANCH:
1750  case AMDGPU::S_MOV_B64_term:
1751  case AMDGPU::S_XOR_B64_term:
1752  case AMDGPU::S_ANDN2_B64_term:
1753  case AMDGPU::S_MOV_B32_term:
1754  case AMDGPU::S_XOR_B32_term:
1755  case AMDGPU::S_OR_B32_term:
1756  case AMDGPU::S_ANDN2_B32_term:
1757  break;
1758  case AMDGPU::SI_IF:
1759  case AMDGPU::SI_ELSE:
1760  case AMDGPU::SI_KILL_I1_TERMINATOR:
1761  case AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR:
1762  // FIXME: It's messy that these need to be considered here at all.
1763  return true;
1764  default:
1765  llvm_unreachable("unexpected non-branch terminator inst");
1766  }
1767 
1768  ++I;
1769  }
1770 
1771  if (I == E)
1772  return false;
1773 
1774  if (I->getOpcode() != AMDGPU::SI_MASK_BRANCH)
1775  return analyzeBranchImpl(MBB, I, TBB, FBB, Cond, AllowModify);
1776 
1777  ++I;
1778 
1779  // TODO: Should be able to treat as fallthrough?
1780  if (I == MBB.end())
1781  return true;
1782 
1783  if (analyzeBranchImpl(MBB, I, TBB, FBB, Cond, AllowModify))
1784  return true;
1785 
1786  MachineBasicBlock *MaskBrDest = I->getOperand(0).getMBB();
1787 
1788  // Specifically handle the case where the conditional branch is to the same
1789  // destination as the mask branch. e.g.
1790  //
1791  // si_mask_branch BB8
1792  // s_cbranch_execz BB8
1793  // s_cbranch BB9
1794  //
1795  // This is required to understand divergent loops which may need the branches
1796  // to be relaxed.
1797  if (TBB != MaskBrDest || Cond.empty())
1798  return true;
1799 
1800  auto Pred = Cond[0].getImm();
1801  return (Pred != EXECZ && Pred != EXECNZ);
1802 }
1803 
1805  int *BytesRemoved) const {
1807 
1808  unsigned Count = 0;
1809  unsigned RemovedSize = 0;
1810  while (I != MBB.end()) {
1811  MachineBasicBlock::iterator Next = std::next(I);
1812  if (I->getOpcode() == AMDGPU::SI_MASK_BRANCH) {
1813  I = Next;
1814  continue;
1815  }
1816 
1817  RemovedSize += getInstSizeInBytes(*I);
1818  I->eraseFromParent();
1819  ++Count;
1820  I = Next;
1821  }
1822 
1823  if (BytesRemoved)
1824  *BytesRemoved = RemovedSize;
1825 
1826  return Count;
1827 }
1828 
1829 // Copy the flags onto the implicit condition register operand.
1831  const MachineOperand &OrigCond) {
1832  CondReg.setIsUndef(OrigCond.isUndef());
1833  CondReg.setIsKill(OrigCond.isKill());
1834 }
1835 
1837  MachineBasicBlock *TBB,
1838  MachineBasicBlock *FBB,
1840  const DebugLoc &DL,
1841  int *BytesAdded) const {
1842  if (!FBB && Cond.empty()) {
1843  BuildMI(&MBB, DL, get(AMDGPU::S_BRANCH))
1844  .addMBB(TBB);
1845  if (BytesAdded)
1846  *BytesAdded = 4;
1847  return 1;
1848  }
1849 
1850  if(Cond.size() == 1 && Cond[0].isReg()) {
1851  BuildMI(&MBB, DL, get(AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO))
1852  .add(Cond[0])
1853  .addMBB(TBB);
1854  return 1;
1855  }
1856 
1857  assert(TBB && Cond[0].isImm());
1858 
1859  unsigned Opcode
1860  = getBranchOpcode(static_cast<BranchPredicate>(Cond[0].getImm()));
1861 
1862  if (!FBB) {
1863  Cond[1].isUndef();
1864  MachineInstr *CondBr =
1865  BuildMI(&MBB, DL, get(Opcode))
1866  .addMBB(TBB);
1867 
1868  // Copy the flags onto the implicit condition register operand.
1869  preserveCondRegFlags(CondBr->getOperand(1), Cond[1]);
1870 
1871  if (BytesAdded)
1872  *BytesAdded = 4;
1873  return 1;
1874  }
1875 
1876  assert(TBB && FBB);
1877 
1878  MachineInstr *CondBr =
1879  BuildMI(&MBB, DL, get(Opcode))
1880  .addMBB(TBB);
1881  BuildMI(&MBB, DL, get(AMDGPU::S_BRANCH))
1882  .addMBB(FBB);
1883 
1884  MachineOperand &CondReg = CondBr->getOperand(1);
1885  CondReg.setIsUndef(Cond[1].isUndef());
1886  CondReg.setIsKill(Cond[1].isKill());
1887 
1888  if (BytesAdded)
1889  *BytesAdded = 8;
1890 
1891  return 2;
1892 }
1893 
1895  SmallVectorImpl<MachineOperand> &Cond) const {
1896  if (Cond.size() != 2) {
1897  return true;
1898  }
1899 
1900  if (Cond[0].isImm()) {
1901  Cond[0].setImm(-Cond[0].getImm());
1902  return false;
1903  }
1904 
1905  return true;
1906 }
1907 
1910  unsigned TrueReg, unsigned FalseReg,
1911  int &CondCycles,
1912  int &TrueCycles, int &FalseCycles) const {
1913  switch (Cond[0].getImm()) {
1914  case VCCNZ:
1915  case VCCZ: {
1916  const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
1917  const TargetRegisterClass *RC = MRI.getRegClass(TrueReg);
1918  assert(MRI.getRegClass(FalseReg) == RC);
1919 
1920  int NumInsts = AMDGPU::getRegBitWidth(RC->getID()) / 32;
1921  CondCycles = TrueCycles = FalseCycles = NumInsts; // ???
1922 
1923  // Limit to equal cost for branch vs. N v_cndmask_b32s.
1924  return !RI.isSGPRClass(RC) && NumInsts <= 6;
1925  }
1926  case SCC_TRUE:
1927  case SCC_FALSE: {
1928  // FIXME: We could insert for VGPRs if we could replace the original compare
1929  // with a vector one.
1930  const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
1931  const TargetRegisterClass *RC = MRI.getRegClass(TrueReg);
1932  assert(MRI.getRegClass(FalseReg) == RC);
1933 
1934  int NumInsts = AMDGPU::getRegBitWidth(RC->getID()) / 32;
1935 
1936  // Multiples of 8 can do s_cselect_b64
1937  if (NumInsts % 2 == 0)
1938  NumInsts /= 2;
1939 
1940  CondCycles = TrueCycles = FalseCycles = NumInsts; // ???
1941  return RI.isSGPRClass(RC);
1942  }
1943  default:
1944  return false;
1945  }
1946 }
1947 
1950  unsigned DstReg, ArrayRef<MachineOperand> Cond,
1951  unsigned TrueReg, unsigned FalseReg) const {
1952  BranchPredicate Pred = static_cast<BranchPredicate>(Cond[0].getImm());
1953  if (Pred == VCCZ || Pred == SCC_FALSE) {
1954  Pred = static_cast<BranchPredicate>(-Pred);
1955  std::swap(TrueReg, FalseReg);
1956  }
1957 
1959  const TargetRegisterClass *DstRC = MRI.getRegClass(DstReg);
1960  unsigned DstSize = RI.getRegSizeInBits(*DstRC);
1961 
1962  if (DstSize == 32) {
1963  unsigned SelOp = Pred == SCC_TRUE ?
1964  AMDGPU::S_CSELECT_B32 : AMDGPU::V_CNDMASK_B32_e32;
1965 
1966  // Instruction's operands are backwards from what is expected.
1967  MachineInstr *Select =
1968  BuildMI(MBB, I, DL, get(SelOp), DstReg)
1969  .addReg(FalseReg)
1970  .addReg(TrueReg);
1971 
1972  preserveCondRegFlags(Select->getOperand(3), Cond[1]);
1973  return;
1974  }
1975 
1976  if (DstSize == 64 && Pred == SCC_TRUE) {
1977  MachineInstr *Select =
1978  BuildMI(MBB, I, DL, get(AMDGPU::S_CSELECT_B64), DstReg)
1979  .addReg(FalseReg)
1980  .addReg(TrueReg);
1981 
1982  preserveCondRegFlags(Select->getOperand(3), Cond[1]);
1983  return;
1984  }
1985 
1986  static const int16_t Sub0_15[] = {
1987  AMDGPU::sub0, AMDGPU::sub1, AMDGPU::sub2, AMDGPU::sub3,
1988  AMDGPU::sub4, AMDGPU::sub5, AMDGPU::sub6, AMDGPU::sub7,
1989  AMDGPU::sub8, AMDGPU::sub9, AMDGPU::sub10, AMDGPU::sub11,
1990  AMDGPU::sub12, AMDGPU::sub13, AMDGPU::sub14, AMDGPU::sub15,
1991  };
1992 
1993  static const int16_t Sub0_15_64[] = {
1994  AMDGPU::sub0_sub1, AMDGPU::sub2_sub3,
1995  AMDGPU::sub4_sub5, AMDGPU::sub6_sub7,
1996  AMDGPU::sub8_sub9, AMDGPU::sub10_sub11,
1997  AMDGPU::sub12_sub13, AMDGPU::sub14_sub15,
1998  };
1999 
2000  unsigned SelOp = AMDGPU::V_CNDMASK_B32_e32;
2001  const TargetRegisterClass *EltRC = &AMDGPU::VGPR_32RegClass;
2002  const int16_t *SubIndices = Sub0_15;
2003  int NElts = DstSize / 32;
2004 
2005  // 64-bit select is only available for SALU.
2006  // TODO: Split 96-bit into 64-bit and 32-bit, not 3x 32-bit.
2007  if (Pred == SCC_TRUE) {
2008  if (NElts % 2) {
2009  SelOp = AMDGPU::S_CSELECT_B32;
2010  EltRC = &AMDGPU::SGPR_32RegClass;
2011  } else {
2012  SelOp = AMDGPU::S_CSELECT_B64;
2013  EltRC = &AMDGPU::SGPR_64RegClass;
2014  SubIndices = Sub0_15_64;
2015  NElts /= 2;
2016  }
2017  }
2018 
2020  MBB, I, DL, get(AMDGPU::REG_SEQUENCE), DstReg);
2021 
2022  I = MIB->getIterator();
2023 
2025  for (int Idx = 0; Idx != NElts; ++Idx) {
2026  unsigned DstElt = MRI.createVirtualRegister(EltRC);
2027  Regs.push_back(DstElt);
2028 
2029  unsigned SubIdx = SubIndices[Idx];
2030 
2031  MachineInstr *Select =
2032  BuildMI(MBB, I, DL, get(SelOp), DstElt)
2033  .addReg(FalseReg, 0, SubIdx)
2034  .addReg(TrueReg, 0, SubIdx);
2035  preserveCondRegFlags(Select->getOperand(3), Cond[1]);
2036  fixImplicitOperands(*Select);
2037 
2038  MIB.addReg(DstElt)
2039  .addImm(SubIdx);
2040  }
2041 }
2042 
2044  switch (MI.getOpcode()) {
2045  case AMDGPU::V_MOV_B32_e32:
2046  case AMDGPU::V_MOV_B32_e64:
2047  case AMDGPU::V_MOV_B64_PSEUDO: {
2048  // If there are additional implicit register operands, this may be used for
2049  // register indexing so the source register operand isn't simply copied.
2050  unsigned NumOps = MI.getDesc().getNumOperands() +
2051  MI.getDesc().getNumImplicitUses();
2052 
2053  return MI.getNumOperands() == NumOps;
2054  }
2055  case AMDGPU::S_MOV_B32:
2056  case AMDGPU::S_MOV_B64:
2057  case AMDGPU::COPY:
2058  return true;
2059  default:
2060  return false;
2061  }
2062 }
2063 
2065  unsigned Kind) const {
2066  switch(Kind) {
2077  }
2078  return AMDGPUAS::FLAT_ADDRESS;
2079 }
2080 
2082  unsigned Opc = MI.getOpcode();
2083  int Src0ModIdx = AMDGPU::getNamedOperandIdx(Opc,
2084  AMDGPU::OpName::src0_modifiers);
2085  int Src1ModIdx = AMDGPU::getNamedOperandIdx(Opc,
2086  AMDGPU::OpName::src1_modifiers);
2087  int Src2ModIdx = AMDGPU::getNamedOperandIdx(Opc,
2088  AMDGPU::OpName::src2_modifiers);
2089 
2090  MI.RemoveOperand(Src2ModIdx);
2091  MI.RemoveOperand(Src1ModIdx);
2092  MI.RemoveOperand(Src0ModIdx);
2093 }
2094 
2096  unsigned Reg, MachineRegisterInfo *MRI) const {
2097  if (!MRI->hasOneNonDBGUse(Reg))
2098  return false;
2099 
2100  switch (DefMI.getOpcode()) {
2101  default:
2102  return false;
2103  case AMDGPU::S_MOV_B64:
2104  // TODO: We could fold 64-bit immediates, but this get compilicated
2105  // when there are sub-registers.
2106  return false;
2107 
2108  case AMDGPU::V_MOV_B32_e32:
2109  case AMDGPU::S_MOV_B32:
2110  break;
2111  }
2112 
2113  const MachineOperand *ImmOp = getNamedOperand(DefMI, AMDGPU::OpName::src0);
2114  assert(ImmOp);
2115  // FIXME: We could handle FrameIndex values here.
2116  if (!ImmOp->isImm())
2117  return false;
2118 
2119  unsigned Opc = UseMI.getOpcode();
2120  if (Opc == AMDGPU::COPY) {
2121  bool isVGPRCopy = RI.isVGPR(*MRI, UseMI.getOperand(0).getReg());
2122  unsigned NewOpc = isVGPRCopy ? AMDGPU::V_MOV_B32_e32 : AMDGPU::S_MOV_B32;
2123  UseMI.setDesc(get(NewOpc));
2124  UseMI.getOperand(1).ChangeToImmediate(ImmOp->getImm());
2125  UseMI.addImplicitDefUseOperands(*UseMI.getParent()->getParent());
2126  return true;
2127  }
2128 
2129  if (Opc == AMDGPU::V_MAD_F32 || Opc == AMDGPU::V_MAC_F32_e64 ||
2130  Opc == AMDGPU::V_MAD_F16 || Opc == AMDGPU::V_MAC_F16_e64 ||
2131  Opc == AMDGPU::V_FMA_F32 || Opc == AMDGPU::V_FMAC_F32_e64 ||
2132  Opc == AMDGPU::V_FMA_F16 || Opc == AMDGPU::V_FMAC_F16_e64) {
2133  // Don't fold if we are using source or output modifiers. The new VOP2
2134  // instructions don't have them.
2135  if (hasAnyModifiersSet(UseMI))
2136  return false;
2137 
2138  // If this is a free constant, there's no reason to do this.
2139  // TODO: We could fold this here instead of letting SIFoldOperands do it
2140  // later.
2141  MachineOperand *Src0 = getNamedOperand(UseMI, AMDGPU::OpName::src0);
2142 
2143  // Any src operand can be used for the legality check.
2144  if (isInlineConstant(UseMI, *Src0, *ImmOp))
2145  return false;
2146 
2147  bool IsF32 = Opc == AMDGPU::V_MAD_F32 || Opc == AMDGPU::V_MAC_F32_e64 ||
2148  Opc == AMDGPU::V_FMA_F32 || Opc == AMDGPU::V_FMAC_F32_e64;
2149  bool IsFMA = Opc == AMDGPU::V_FMA_F32 || Opc == AMDGPU::V_FMAC_F32_e64 ||
2150  Opc == AMDGPU::V_FMA_F16 || Opc == AMDGPU::V_FMAC_F16_e64;
2151  MachineOperand *Src1 = getNamedOperand(UseMI, AMDGPU::OpName::src1);
2152  MachineOperand *Src2 = getNamedOperand(UseMI, AMDGPU::OpName::src2);
2153 
2154  // Multiplied part is the constant: Use v_madmk_{f16, f32}.
2155  // We should only expect these to be on src0 due to canonicalizations.
2156  if (Src0->isReg() && Src0->getReg() == Reg) {
2157  if (!Src1->isReg() || RI.isSGPRClass(MRI->getRegClass(Src1->getReg())))
2158  return false;
2159 
2160  if (!Src2->isReg() || RI.isSGPRClass(MRI->getRegClass(Src2->getReg())))
2161  return false;
2162 
2163  unsigned NewOpc =
2164  IsFMA ? (IsF32 ? AMDGPU::V_FMAMK_F32 : AMDGPU::V_FMAMK_F16)
2165  : (IsF32 ? AMDGPU::V_MADMK_F32 : AMDGPU::V_MADMK_F16);
2166  if (pseudoToMCOpcode(NewOpc) == -1)
2167  return false;
2168 
2169  // We need to swap operands 0 and 1 since madmk constant is at operand 1.
2170 
2171  const int64_t Imm = ImmOp->getImm();
2172 
2173  // FIXME: This would be a lot easier if we could return a new instruction
2174  // instead of having to modify in place.
2175 
2176  // Remove these first since they are at the end.
2177  UseMI.RemoveOperand(
2178  AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::omod));
2179  UseMI.RemoveOperand(
2180  AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::clamp));
2181 
2182  unsigned Src1Reg = Src1->getReg();
2183  unsigned Src1SubReg = Src1->getSubReg();
2184  Src0->setReg(Src1Reg);
2185  Src0->setSubReg(Src1SubReg);
2186  Src0->setIsKill(Src1->isKill());
2187 
2188  if (Opc == AMDGPU::V_MAC_F32_e64 ||
2189  Opc == AMDGPU::V_MAC_F16_e64 ||
2190  Opc == AMDGPU::V_FMAC_F32_e64 ||
2191  Opc == AMDGPU::V_FMAC_F16_e64)
2192  UseMI.untieRegOperand(
2193  AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2));
2194 
2195  Src1->ChangeToImmediate(Imm);
2196 
2197  removeModOperands(UseMI);
2198  UseMI.setDesc(get(NewOpc));
2199 
2200  bool DeleteDef = MRI->hasOneNonDBGUse(Reg);
2201  if (DeleteDef)
2202  DefMI.eraseFromParent();
2203 
2204  return true;
2205  }
2206 
2207  // Added part is the constant: Use v_madak_{f16, f32}.
2208  if (Src2->isReg() && Src2->getReg() == Reg) {
2209  // Not allowed to use constant bus for another operand.
2210  // We can however allow an inline immediate as src0.
2211  bool Src0Inlined = false;
2212  if (Src0->isReg()) {
2213  // Try to inline constant if possible.
2214  // If the Def moves immediate and the use is single
2215  // We are saving VGPR here.
2216  MachineInstr *Def = MRI->getUniqueVRegDef(Src0->getReg());
2217  if (Def && Def->isMoveImmediate() &&
2218  isInlineConstant(Def->getOperand(1)) &&
2219  MRI->hasOneUse(Src0->getReg())) {
2220  Src0->ChangeToImmediate(Def->getOperand(1).getImm());
2221  Src0Inlined = true;
2222  } else if ((RI.isPhysicalRegister(Src0->getReg()) &&
2223  (ST.getConstantBusLimit(Opc) <= 1 &&
2224  RI.isSGPRClass(RI.getPhysRegClass(Src0->getReg())))) ||
2225  (RI.isVirtualRegister(Src0->getReg()) &&
2226  (ST.getConstantBusLimit(Opc) <= 1 &&
2227  RI.isSGPRClass(MRI->getRegClass(Src0->getReg())))))
2228  return false;
2229  // VGPR is okay as Src0 - fallthrough
2230  }
2231 
2232  if (Src1->isReg() && !Src0Inlined ) {
2233  // We have one slot for inlinable constant so far - try to fill it
2234  MachineInstr *Def = MRI->getUniqueVRegDef(Src1->getReg());
2235  if (Def && Def->isMoveImmediate() &&
2236  isInlineConstant(Def->getOperand(1)) &&
2237  MRI->hasOneUse(Src1->getReg()) &&
2238  commuteInstruction(UseMI)) {
2239  Src0->ChangeToImmediate(Def->getOperand(1).getImm());
2240  } else if ((RI.isPhysicalRegister(Src1->getReg()) &&
2241  RI.isSGPRClass(RI.getPhysRegClass(Src1->getReg()))) ||
2242  (RI.isVirtualRegister(Src1->getReg()) &&
2243  RI.isSGPRClass(MRI->getRegClass(Src1->getReg()))))
2244  return false;
2245  // VGPR is okay as Src1 - fallthrough
2246  }
2247 
2248  unsigned NewOpc =
2249  IsFMA ? (IsF32 ? AMDGPU::V_FMAAK_F32 : AMDGPU::V_FMAAK_F16)
2250  : (IsF32 ? AMDGPU::V_MADAK_F32 : AMDGPU::V_MADAK_F16);
2251  if (pseudoToMCOpcode(NewOpc) == -1)
2252  return false;
2253 
2254  const int64_t Imm = ImmOp->getImm();
2255 
2256  // FIXME: This would be a lot easier if we could return a new instruction
2257  // instead of having to modify in place.
2258 
2259  // Remove these first since they are at the end.
2260  UseMI.RemoveOperand(
2261  AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::omod));
2262  UseMI.RemoveOperand(
2263  AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::clamp));
2264 
2265  if (Opc == AMDGPU::V_MAC_F32_e64 ||
2266  Opc == AMDGPU::V_MAC_F16_e64 ||
2267  Opc == AMDGPU::V_FMAC_F32_e64 ||
2268  Opc == AMDGPU::V_FMAC_F16_e64)
2269  UseMI.untieRegOperand(
2270  AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2));
2271 
2272  // ChangingToImmediate adds Src2 back to the instruction.
2273  Src2->ChangeToImmediate(Imm);
2274 
2275  // These come before src2.
2276  removeModOperands(UseMI);
2277  UseMI.setDesc(get(NewOpc));
2278  // It might happen that UseMI was commuted
2279  // and we now have SGPR as SRC1. If so 2 inlined
2280  // constant and SGPR are illegal.
2281  legalizeOperands(UseMI);
2282 
2283  bool DeleteDef = MRI->hasOneNonDBGUse(Reg);
2284  if (DeleteDef)
2285  DefMI.eraseFromParent();
2286 
2287  return true;
2288  }
2289  }
2290 
2291  return false;
2292 }
2293 
2294 static bool offsetsDoNotOverlap(int WidthA, int OffsetA,
2295  int WidthB, int OffsetB) {
2296  int LowOffset = OffsetA < OffsetB ? OffsetA : OffsetB;
2297  int HighOffset = OffsetA < OffsetB ? OffsetB : OffsetA;
2298  int LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB;
2299  return LowOffset + LowWidth <= HighOffset;
2300 }
2301 
2302 bool SIInstrInfo::checkInstOffsetsDoNotOverlap(const MachineInstr &MIa,
2303  const MachineInstr &MIb) const {
2304  const MachineOperand *BaseOp0, *BaseOp1;
2305  int64_t Offset0, Offset1;
2306 
2307  if (getMemOperandWithOffset(MIa, BaseOp0, Offset0, &RI) &&
2308  getMemOperandWithOffset(MIb, BaseOp1, Offset1, &RI)) {
2309  if (!BaseOp0->isIdenticalTo(*BaseOp1))
2310  return false;
2311 
2312  if (!MIa.hasOneMemOperand() || !MIb.hasOneMemOperand()) {
2313  // FIXME: Handle ds_read2 / ds_write2.
2314  return false;
2315  }
2316  unsigned Width0 = (*MIa.memoperands_begin())->getSize();
2317  unsigned Width1 = (*MIb.memoperands_begin())->getSize();
2318  if (offsetsDoNotOverlap(Width0, Offset0, Width1, Offset1)) {
2319  return true;
2320  }
2321  }
2322 
2323  return false;
2324 }
2325 
2327  const MachineInstr &MIb,
2328  AliasAnalysis *AA) const {
2329  assert((MIa.mayLoad() || MIa.mayStore()) &&
2330  "MIa must load from or modify a memory location");
2331  assert((MIb.mayLoad() || MIb.mayStore()) &&
2332  "MIb must load from or modify a memory location");
2333 
2335  return false;
2336 
2337  // XXX - Can we relax this between address spaces?
2338  if (MIa.hasOrderedMemoryRef() || MIb.hasOrderedMemoryRef())
2339  return false;
2340 
2341  // TODO: Should we check the address space from the MachineMemOperand? That
2342  // would allow us to distinguish objects we know don't alias based on the
2343  // underlying address space, even if it was lowered to a different one,
2344  // e.g. private accesses lowered to use MUBUF instructions on a scratch
2345  // buffer.
2346  if (isDS(MIa)) {
2347  if (isDS(MIb))
2348  return checkInstOffsetsDoNotOverlap(MIa, MIb);
2349 
2350  return !isFLAT(MIb) || isSegmentSpecificFLAT(MIb);
2351  }
2352 
2353  if (isMUBUF(MIa) || isMTBUF(MIa)) {
2354  if (isMUBUF(MIb) || isMTBUF(MIb))
2355  return checkInstOffsetsDoNotOverlap(MIa, MIb);
2356 
2357  return !isFLAT(MIb) && !isSMRD(MIb);
2358  }
2359 
2360  if (isSMRD(MIa)) {
2361  if (isSMRD(MIb))
2362  return checkInstOffsetsDoNotOverlap(MIa, MIb);
2363 
2364  return !isFLAT(MIb) && !isMUBUF(MIa) && !isMTBUF(MIa);
2365  }
2366 
2367  if (isFLAT(MIa)) {
2368  if (isFLAT(MIb))
2369  return checkInstOffsetsDoNotOverlap(MIa, MIb);
2370 
2371  return false;
2372  }
2373 
2374  return false;
2375 }
2376 
2377 static int64_t getFoldableImm(const MachineOperand* MO) {
2378  if (!MO->isReg())
2379  return false;
2380  const MachineFunction *MF = MO->getParent()->getParent()->getParent();
2381  const MachineRegisterInfo &MRI = MF->getRegInfo();
2382  auto Def = MRI.getUniqueVRegDef(MO->getReg());
2383  if (Def && Def->getOpcode() == AMDGPU::V_MOV_B32_e32 &&
2384  Def->getOperand(1).isImm())
2385  return Def->getOperand(1).getImm();
2386  return AMDGPU::NoRegister;
2387 }
2388 
2390  MachineInstr &MI,
2391  LiveVariables *LV) const {
2392  unsigned Opc = MI.getOpcode();
2393  bool IsF16 = false;
2394  bool IsFMA = Opc == AMDGPU::V_FMAC_F32_e32 || Opc == AMDGPU::V_FMAC_F32_e64 ||
2395  Opc == AMDGPU::V_FMAC_F16_e32 || Opc == AMDGPU::V_FMAC_F16_e64;
2396 
2397  switch (Opc) {
2398  default:
2399  return nullptr;
2400  case AMDGPU::V_MAC_F16_e64:
2401  case AMDGPU::V_FMAC_F16_e64:
2402  IsF16 = true;
2404  case AMDGPU::V_MAC_F32_e64:
2405  case AMDGPU::V_FMAC_F32_e64:
2406  break;
2407  case AMDGPU::V_MAC_F16_e32:
2408  case AMDGPU::V_FMAC_F16_e32:
2409  IsF16 = true;
2411  case AMDGPU::V_MAC_F32_e32:
2412  case AMDGPU::V_FMAC_F32_e32: {
2413  int Src0Idx = AMDGPU::getNamedOperandIdx(MI.getOpcode(),
2414  AMDGPU::OpName::src0);
2415  const MachineOperand *Src0 = &MI.getOperand(Src0Idx);
2416  if (!Src0->isReg() && !Src0->isImm())
2417  return nullptr;
2418 
2419  if (Src0->isImm() && !isInlineConstant(MI, Src0Idx, *Src0))
2420  return nullptr;
2421 
2422  break;
2423  }
2424  }
2425 
2426  const MachineOperand *Dst = getNamedOperand(MI, AMDGPU::OpName::vdst);
2427  const MachineOperand *Src0 = getNamedOperand(MI, AMDGPU::OpName::src0);
2428  const MachineOperand *Src0Mods =
2429  getNamedOperand(MI, AMDGPU::OpName::src0_modifiers);
2430  const MachineOperand *Src1 = getNamedOperand(MI, AMDGPU::OpName::src1);
2431  const MachineOperand *Src1Mods =
2432  getNamedOperand(MI, AMDGPU::OpName::src1_modifiers);
2433  const MachineOperand *Src2 = getNamedOperand(MI, AMDGPU::OpName::src2);
2434  const MachineOperand *Clamp = getNamedOperand(MI, AMDGPU::OpName::clamp);
2435  const MachineOperand *Omod = getNamedOperand(MI, AMDGPU::OpName::omod);
2436 
2437  if (!Src0Mods && !Src1Mods && !Clamp && !Omod &&
2438  // If we have an SGPR input, we will violate the constant bus restriction.
2439  (ST.getConstantBusLimit(Opc) > 1 ||
2440  !Src0->isReg() ||
2441  !RI.isSGPRReg(MBB->getParent()->getRegInfo(), Src0->getReg()))) {
2442  if (auto Imm = getFoldableImm(Src2)) {
2443  unsigned NewOpc =
2444  IsFMA ? (IsF16 ? AMDGPU::V_FMAAK_F16 : AMDGPU::V_FMAAK_F32)
2445  : (IsF16 ? AMDGPU::V_MADAK_F16 : AMDGPU::V_MADAK_F32);
2446  if (pseudoToMCOpcode(NewOpc) != -1)
2447  return BuildMI(*MBB, MI, MI.getDebugLoc(), get(NewOpc))
2448  .add(*Dst)
2449  .add(*Src0)
2450  .add(*Src1)
2451  .addImm(Imm);
2452  }
2453  unsigned NewOpc =
2454  IsFMA ? (IsF16 ? AMDGPU::V_FMAMK_F16 : AMDGPU::V_FMAMK_F32)
2455  : (IsF16 ? AMDGPU::V_MADMK_F16 : AMDGPU::V_MADMK_F32);
2456  if (auto Imm = getFoldableImm(Src1)) {
2457  if (pseudoToMCOpcode(NewOpc) != -1)
2458  return BuildMI(*MBB, MI, MI.getDebugLoc(), get(NewOpc))
2459  .add(*Dst)
2460  .add(*Src0)
2461  .addImm(Imm)
2462  .add(*Src2);
2463  }
2464  if (auto Imm = getFoldableImm(Src0)) {
2465  if (pseudoToMCOpcode(NewOpc) != -1 &&
2467  AMDGPU::OpName::src0), Src1))
2468  return BuildMI(*MBB, MI, MI.getDebugLoc(), get(NewOpc))
2469  .add(*Dst)
2470  .add(*Src1)
2471  .addImm(Imm)
2472  .add(*Src2);
2473  }
2474  }
2475 
2476  unsigned NewOpc = IsFMA ? (IsF16 ? AMDGPU::V_FMA_F16 : AMDGPU::V_FMA_F32)
2477  : (IsF16 ? AMDGPU::V_MAD_F16 : AMDGPU::V_MAD_F32);
2478  if (pseudoToMCOpcode(NewOpc) == -1)
2479  return nullptr;
2480 
2481  return BuildMI(*MBB, MI, MI.getDebugLoc(), get(NewOpc))
2482  .add(*Dst)
2483  .addImm(Src0Mods ? Src0Mods->getImm() : 0)
2484  .add(*Src0)
2485  .addImm(Src1Mods ? Src1Mods->getImm() : 0)
2486  .add(*Src1)
2487  .addImm(0) // Src mods
2488  .add(*Src2)
2489  .addImm(Clamp ? Clamp->getImm() : 0)
2490  .addImm(Omod ? Omod->getImm() : 0);
2491 }
2492 
2493 // It's not generally safe to move VALU instructions across these since it will
2494 // start using the register as a base index rather than directly.
2495 // XXX - Why isn't hasSideEffects sufficient for these?
2497  switch (MI.getOpcode()) {
2498  case AMDGPU::S_SET_GPR_IDX_ON:
2499  case AMDGPU::S_SET_GPR_IDX_MODE:
2500  case AMDGPU::S_SET_GPR_IDX_OFF:
2501  return true;
2502  default:
2503  return false;
2504  }
2505 }
2506 
2508  const MachineBasicBlock *MBB,
2509  const MachineFunction &MF) const {
2510  // XXX - Do we want the SP check in the base implementation?
2511 
2512  // Target-independent instructions do not have an implicit-use of EXEC, even
2513  // when they operate on VGPRs. Treating EXEC modifications as scheduling
2514  // boundaries prevents incorrect movements of such instructions.
2515  return TargetInstrInfo::isSchedulingBoundary(MI, MBB, MF) ||
2516  MI.modifiesRegister(AMDGPU::EXEC, &RI) ||
2517  MI.getOpcode() == AMDGPU::S_SETREG_IMM32_B32 ||
2518  MI.getOpcode() == AMDGPU::S_SETREG_B32 ||
2520 }
2521 
2522 bool SIInstrInfo::isAlwaysGDS(uint16_t Opcode) const {
2523  return Opcode == AMDGPU::DS_ORDERED_COUNT ||
2524  Opcode == AMDGPU::DS_GWS_INIT ||
2525  Opcode == AMDGPU::DS_GWS_SEMA_V ||
2526  Opcode == AMDGPU::DS_GWS_SEMA_BR ||
2527  Opcode == AMDGPU::DS_GWS_SEMA_P ||
2528  Opcode == AMDGPU::DS_GWS_SEMA_RELEASE_ALL ||
2529  Opcode == AMDGPU::DS_GWS_BARRIER;
2530 }
2531 
2533  unsigned Opcode = MI.getOpcode();
2534 
2535  if (MI.mayStore() && isSMRD(MI))
2536  return true; // scalar store or atomic
2537 
2538  // This will terminate the function when other lanes may need to continue.
2539  if (MI.isReturn())
2540  return true;
2541 
2542  // These instructions cause shader I/O that may cause hardware lockups
2543  // when executed with an empty EXEC mask.
2544  //
2545  // Note: exp with VM = DONE = 0 is automatically skipped by hardware when
2546  // EXEC = 0, but checking for that case here seems not worth it
2547  // given the typical code patterns.
2548  if (Opcode == AMDGPU::S_SENDMSG || Opcode == AMDGPU::S_SENDMSGHALT ||
2549  Opcode == AMDGPU::EXP || Opcode == AMDGPU::EXP_DONE ||
2550  Opcode == AMDGPU::DS_ORDERED_COUNT || Opcode == AMDGPU::S_TRAP ||
2551  Opcode == AMDGPU::DS_GWS_INIT || Opcode == AMDGPU::DS_GWS_BARRIER)
2552  return true;
2553 
2554  if (MI.isCall() || MI.isInlineAsm())
2555  return true; // conservative assumption
2556 
2557  // These are like SALU instructions in terms of effects, so it's questionable
2558  // whether we should return true for those.
2559  //
2560  // However, executing them with EXEC = 0 causes them to operate on undefined
2561  // data, which we avoid by returning true here.
2562  if (Opcode == AMDGPU::V_READFIRSTLANE_B32 || Opcode == AMDGPU::V_READLANE_B32)
2563  return true;
2564 
2565  return false;
2566 }
2567 
2569  const MachineInstr &MI) const {
2570  if (MI.isMetaInstruction())
2571  return false;
2572 
2573  // This won't read exec if this is an SGPR->SGPR copy.
2574  if (MI.isCopyLike()) {
2575  if (!RI.isSGPRReg(MRI, MI.getOperand(0).getReg()))
2576  return true;
2577 
2578  // Make sure this isn't copying exec as a normal operand
2579  return MI.readsRegister(AMDGPU::EXEC, &RI);
2580  }
2581 
2582  // Make a conservative assumption about the callee.
2583  if (MI.isCall())
2584  return true;
2585 
2586  // Be conservative with any unhandled generic opcodes.
2587  if (!isTargetSpecificOpcode(MI.getOpcode()))
2588  return true;
2589 
2590  return !isSALU(MI) || MI.readsRegister(AMDGPU::EXEC, &RI);
2591 }
2592 
2593 bool SIInstrInfo::isInlineConstant(const APInt &Imm) const {
2594  switch (Imm.getBitWidth()) {
2595  case 1: // This likely will be a condition code mask.
2596  return true;
2597 
2598  case 32:
2600  ST.hasInv2PiInlineImm());
2601  case 64:
2603  ST.hasInv2PiInlineImm());
2604  case 16:
2605  return ST.has16BitInsts() &&
2607  ST.hasInv2PiInlineImm());
2608  default:
2609  llvm_unreachable("invalid bitwidth");
2610  }
2611 }
2612 
2614  uint8_t OperandType) const {
2615  if (!MO.isImm() ||
2616  OperandType < AMDGPU::OPERAND_SRC_FIRST ||
2617  OperandType > AMDGPU::OPERAND_SRC_LAST)
2618  return false;
2619 
2620  // MachineOperand provides no way to tell the true operand size, since it only
2621  // records a 64-bit value. We need to know the size to determine if a 32-bit
2622  // floating point immediate bit pattern is legal for an integer immediate. It
2623  // would be for any 32-bit integer operand, but would not be for a 64-bit one.
2624 
2625  int64_t Imm = MO.getImm();
2626  switch (OperandType) {
2631  int32_t Trunc = static_cast<int32_t>(Imm);
2633  }
2639  ST.hasInv2PiInlineImm());
2644  if (isInt<16>(Imm) || isUInt<16>(Imm)) {
2645  // A few special case instructions have 16-bit operands on subtargets
2646  // where 16-bit instructions are not legal.
2647  // TODO: Do the 32-bit immediates work? We shouldn't really need to handle
2648  // constants in these cases
2649  int16_t Trunc = static_cast<int16_t>(Imm);
2650  return ST.has16BitInsts() &&
2652  }
2653 
2654  return false;
2655  }
2660  uint32_t Trunc = static_cast<uint32_t>(Imm);
2662  }
2663  default:
2664  llvm_unreachable("invalid bitwidth");
2665  }
2666 }
2667 
2669  const MCOperandInfo &OpInfo) const {
2670  switch (MO.getType()) {
2672  return false;
2674  return !isInlineConstant(MO, OpInfo);
2680  return true;
2681  default:
2682  llvm_unreachable("unexpected operand type");
2683  }
2684 }
2685 
2686 static bool compareMachineOp(const MachineOperand &Op0,
2687  const MachineOperand &Op1) {
2688  if (Op0.getType() != Op1.getType())
2689  return false;
2690 
2691  switch (Op0.getType()) {
2693  return Op0.getReg() == Op1.getReg();
2695  return Op0.getImm() == Op1.getImm();
2696  default:
2697  llvm_unreachable("Didn't expect to be comparing these operand types");
2698  }
2699 }
2700 
2702  const MachineOperand &MO) const {
2703  const MCInstrDesc &InstDesc = MI.getDesc();
2704  const MCOperandInfo &OpInfo = InstDesc.OpInfo[OpNo];
2705 
2706  assert(MO.isImm() || MO.isTargetIndex() || MO.isFI());
2707 
2708  if (OpInfo.OperandType == MCOI::OPERAND_IMMEDIATE)
2709  return true;
2710 
2711  if (OpInfo.RegClass < 0)
2712  return false;
2713 
2714  if (MO.isImm() && isInlineConstant(MO, OpInfo))
2715  return RI.opCanUseInlineConstant(OpInfo.OperandType);
2716 
2717  if (!RI.opCanUseLiteralConstant(OpInfo.OperandType))
2718  return false;
2719 
2720  if (!isVOP3(MI) || !AMDGPU::isSISrcOperand(InstDesc, OpNo))
2721  return true;
2722 
2723  const MachineFunction *MF = MI.getParent()->getParent();
2724  const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
2725  return ST.hasVOP3Literal();
2726 }
2727 
2728 bool SIInstrInfo::hasVALU32BitEncoding(unsigned Opcode) const {
2729  int Op32 = AMDGPU::getVOPe32(Opcode);
2730  if (Op32 == -1)
2731  return false;
2732 
2733  return pseudoToMCOpcode(Op32) != -1;
2734 }
2735 
2736 bool SIInstrInfo::hasModifiers(unsigned Opcode) const {
2737  // The src0_modifier operand is present on all instructions
2738  // that have modifiers.
2739 
2740  return AMDGPU::getNamedOperandIdx(Opcode,
2741  AMDGPU::OpName::src0_modifiers) != -1;
2742 }
2743 
2745  unsigned OpName) const {
2746  const MachineOperand *Mods = getNamedOperand(MI, OpName);
2747  return Mods && Mods->getImm();
2748 }
2749 
2751  return hasModifiersSet(MI, AMDGPU::OpName::src0_modifiers) ||
2752  hasModifiersSet(MI, AMDGPU::OpName::src1_modifiers) ||
2753  hasModifiersSet(MI, AMDGPU::OpName::src2_modifiers) ||
2754  hasModifiersSet(MI, AMDGPU::OpName::clamp) ||
2755  hasModifiersSet(MI, AMDGPU::OpName::omod);
2756 }
2757 
2759  const MachineRegisterInfo &MRI) const {
2760  const MachineOperand *Src2 = getNamedOperand(MI, AMDGPU::OpName::src2);
2761  // Can't shrink instruction with three operands.
2762  // FIXME: v_cndmask_b32 has 3 operands and is shrinkable, but we need to add
2763  // a special case for it. It can only be shrunk if the third operand
2764  // is vcc, and src0_modifiers and src1_modifiers are not set.
2765  // We should handle this the same way we handle vopc, by addding
2766  // a register allocation hint pre-regalloc and then do the shrinking
2767  // post-regalloc.
2768  if (Src2) {
2769  switch (MI.getOpcode()) {
2770  default: return false;
2771 
2772  case AMDGPU::V_ADDC_U32_e64:
2773  case AMDGPU::V_SUBB_U32_e64:
2774  case AMDGPU::V_SUBBREV_U32_e64: {
2775  const MachineOperand *Src1
2776  = getNamedOperand(MI, AMDGPU::OpName::src1);
2777  if (!Src1->isReg() || !RI.isVGPR(MRI, Src1->getReg()))
2778  return false;
2779  // Additional verification is needed for sdst/src2.
2780  return true;
2781  }
2782  case AMDGPU::V_MAC_F32_e64:
2783  case AMDGPU::V_MAC_F16_e64:
2784  case AMDGPU::V_FMAC_F32_e64:
2785  case AMDGPU::V_FMAC_F16_e64:
2786  if (!Src2->isReg() || !RI.isVGPR(MRI, Src2->getReg()) ||
2787  hasModifiersSet(MI, AMDGPU::OpName::src2_modifiers))
2788  return false;
2789  break;
2790 
2791  case AMDGPU::V_CNDMASK_B32_e64:
2792  break;
2793  }
2794  }
2795 
2796  const MachineOperand *Src1 = getNamedOperand(MI, AMDGPU::OpName::src1);
2797  if (Src1 && (!Src1->isReg() || !RI.isVGPR(MRI, Src1->getReg()) ||
2798  hasModifiersSet(MI, AMDGPU::OpName::src1_modifiers)))
2799  return false;
2800 
2801  // We don't need to check src0, all input types are legal, so just make sure
2802  // src0 isn't using any modifiers.
2803  if (hasModifiersSet(MI, AMDGPU::OpName::src0_modifiers))
2804  return false;
2805 
2806  // Can it be shrunk to a valid 32 bit opcode?
2807  if (!hasVALU32BitEncoding(MI.getOpcode()))
2808  return false;
2809 
2810  // Check output modifiers
2811  return !hasModifiersSet(MI, AMDGPU::OpName::omod) &&
2812  !hasModifiersSet(MI, AMDGPU::OpName::clamp);
2813 }
2814 
2815 // Set VCC operand with all flags from \p Orig, except for setting it as
2816 // implicit.
2818  const MachineOperand &Orig) {
2819 
2820  for (MachineOperand &Use : MI.implicit_operands()) {
2821  if (Use.isUse() && Use.getReg() == AMDGPU::VCC) {
2822  Use.setIsUndef(Orig.isUndef());
2823  Use.setIsKill(Orig.isKill());
2824  return;
2825  }
2826  }
2827 }
2828 
2830  unsigned Op32) const {
2831  MachineBasicBlock *MBB = MI.getParent();;
2832  MachineInstrBuilder Inst32 =
2833  BuildMI(*MBB, MI, MI.getDebugLoc(), get(Op32));
2834 
2835  // Add the dst operand if the 32-bit encoding also has an explicit $vdst.
2836  // For VOPC instructions, this is replaced by an implicit def of vcc.
2837  int Op32DstIdx = AMDGPU::getNamedOperandIdx(Op32, AMDGPU::OpName::vdst);
2838  if (Op32DstIdx != -1) {
2839  // dst
2840  Inst32.add(MI.getOperand(0));
2841  } else {
2842  assert(((MI.getOperand(0).getReg() == AMDGPU::VCC) ||
2843  (MI.getOperand(0).getReg() == AMDGPU::VCC_LO)) &&
2844  "Unexpected case");
2845  }
2846 
2847  Inst32.add(*getNamedOperand(MI, AMDGPU::OpName::src0));
2848 
2849  const MachineOperand *Src1 = getNamedOperand(MI, AMDGPU::OpName::src1);
2850  if (Src1)
2851  Inst32.add(*Src1);
2852 
2853  const MachineOperand *Src2 = getNamedOperand(MI, AMDGPU::OpName::src2);
2854 
2855  if (Src2) {
2856  int Op32Src2Idx = AMDGPU::getNamedOperandIdx(Op32, AMDGPU::OpName::src2);
2857  if (Op32Src2Idx != -1) {
2858  Inst32.add(*Src2);
2859  } else {
2860  // In the case of V_CNDMASK_B32_e32, the explicit operand src2 is
2861  // replaced with an implicit read of vcc. This was already added
2862  // during the initial BuildMI, so find it to preserve the flags.
2863  copyFlagsToImplicitVCC(*Inst32, *Src2);
2864  }
2865  }
2866 
2867  return Inst32;
2868 }
2869 
2871  const MachineOperand &MO,
2872  const MCOperandInfo &OpInfo) const {
2873  // Literal constants use the constant bus.
2874  //if (isLiteralConstantLike(MO, OpInfo))
2875  // return true;
2876  if (MO.isImm())
2877  return !isInlineConstant(MO, OpInfo);
2878 
2879  if (!MO.isReg())
2880  return true; // Misc other operands like FrameIndex
2881 
2882  if (!MO.isUse())
2883  return false;
2884 
2886  return RI.isSGPRClass(MRI.getRegClass(MO.getReg()));
2887 
2888  // Null is free
2889  if (MO.getReg() == AMDGPU::SGPR_NULL)
2890  return false;
2891 
2892  // SGPRs use the constant bus
2893  if (MO.isImplicit()) {
2894  return MO.getReg() == AMDGPU::M0 ||
2895  MO.getReg() == AMDGPU::VCC ||
2896  MO.getReg() == AMDGPU::VCC_LO;
2897  } else {
2898  return AMDGPU::SReg_32RegClass.contains(MO.getReg()) ||
2899  AMDGPU::SReg_64RegClass.contains(MO.getReg());
2900  }
2901 }
2902 
2903 static unsigned findImplicitSGPRRead(const MachineInstr &MI) {
2904  for (const MachineOperand &MO : MI.implicit_operands()) {
2905  // We only care about reads.
2906  if (MO.isDef())
2907  continue;
2908 
2909  switch (MO.getReg()) {
2910  case AMDGPU::VCC:
2911  case AMDGPU::VCC_LO:
2912  case AMDGPU::VCC_HI:
2913  case AMDGPU::M0:
2914  case AMDGPU::FLAT_SCR:
2915  return MO.getReg();
2916 
2917  default:
2918  break;
2919  }
2920  }
2921 
2922  return AMDGPU::NoRegister;
2923 }
2924 
2925 static bool shouldReadExec(const MachineInstr &MI) {
2926  if (SIInstrInfo::isVALU(MI)) {
2927  switch (MI.getOpcode()) {
2928  case AMDGPU::V_READLANE_B32:
2929  case AMDGPU::V_READLANE_B32_gfx6_gfx7:
2930  case AMDGPU::V_READLANE_B32_gfx10:
2931  case AMDGPU::V_READLANE_B32_vi:
2932  case AMDGPU::V_WRITELANE_B32:
2933  case AMDGPU::V_WRITELANE_B32_gfx6_gfx7:
2934  case AMDGPU::V_WRITELANE_B32_gfx10:
2935  case AMDGPU::V_WRITELANE_B32_vi:
2936  return false;
2937  }
2938 
2939  return true;
2940  }
2941 
2942  if (SIInstrInfo::isGenericOpcode(MI.getOpcode()) ||
2943  SIInstrInfo::isSALU(MI) ||
2944  SIInstrInfo::isSMRD(MI))
2945  return false;
2946 
2947  return true;
2948 }
2949 
2950 static bool isSubRegOf(const SIRegisterInfo &TRI,
2951  const MachineOperand &SuperVec,
2952  const MachineOperand &SubReg) {
2954  return TRI.isSubRegister(SuperVec.getReg(), SubReg.getReg());
2955 
2956  return SubReg.getSubReg() != AMDGPU::NoSubRegister &&
2957  SubReg.getReg() == SuperVec.getReg();
2958 }
2959 
2961  StringRef &ErrInfo) const {
2962  uint16_t Opcode = MI.getOpcode();
2963  if (SIInstrInfo::isGenericOpcode(MI.getOpcode()))
2964  return true;
2965 
2966  const MachineFunction *MF = MI.getParent()->getParent();
2967  const MachineRegisterInfo &MRI = MF->getRegInfo();
2968 
2969  int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0);
2970  int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src1);
2971  int Src2Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src2);
2972 
2973  // Make sure the number of operands is correct.
2974  const MCInstrDesc &Desc = get(Opcode);
2975  if (!Desc.isVariadic() &&
2976  Desc.getNumOperands() != MI.getNumExplicitOperands()) {
2977  ErrInfo = "Instruction has wrong number of operands.";
2978  return false;
2979  }
2980 
2981  if (MI.isInlineAsm()) {
2982  // Verify register classes for inlineasm constraints.
2983  for (unsigned I = InlineAsm::MIOp_FirstOperand, E = MI.getNumOperands();
2984  I != E; ++I) {
2985  const TargetRegisterClass *RC = MI.getRegClassConstraint(I, this, &RI);
2986  if (!RC)
2987  continue;
2988 
2989  const MachineOperand &Op = MI.getOperand(I);
2990  if (!Op.isReg())
2991  continue;
2992 
2993  unsigned Reg = Op.getReg();
2994  if (!TargetRegisterInfo::isVirtualRegister(Reg) && !RC->contains(Reg)) {
2995  ErrInfo = "inlineasm operand has incorrect register class.";
2996  return false;
2997  }
2998  }
2999 
3000  return true;
3001  }
3002 
3003  // Make sure the register classes are correct.
3004  for (int i = 0, e = Desc.getNumOperands(); i != e; ++i) {
3005  if (MI.getOperand(i).isFPImm()) {
3006  ErrInfo = "FPImm Machine Operands are not supported. ISel should bitcast "
3007  "all fp values to integers.";
3008  return false;
3009  }
3010 
3011  int RegClass = Desc.OpInfo[i].RegClass;
3012 
3013  switch (Desc.OpInfo[i].OperandType) {
3015  if (MI.getOperand(i).isImm()) {
3016  ErrInfo = "Illegal immediate value for operand.";
3017  return false;
3018  }
3019  break;
3022  break;
3029  const MachineOperand &MO = MI.getOperand(i);
3030  if (!MO.isReg() && (!MO.isImm() || !isInlineConstant(MI, i))) {
3031  ErrInfo = "Illegal immediate value for operand.";
3032  return false;
3033  }
3034  break;
3035  }
3038  // Check if this operand is an immediate.
3039  // FrameIndex operands will be replaced by immediates, so they are
3040  // allowed.
3041  if (!MI.getOperand(i).isImm() && !MI.getOperand(i).isFI()) {
3042  ErrInfo = "Expected immediate, but got non-immediate";
3043  return false;
3044  }
3046  default:
3047  continue;
3048  }
3049 
3050  if (!MI.getOperand(i).isReg())
3051  continue;
3052 
3053  if (RegClass != -1) {
3054  unsigned Reg = MI.getOperand(i).getReg();
3055  if (Reg == AMDGPU::NoRegister ||
3057  continue;
3058 
3059  const TargetRegisterClass *RC = RI.getRegClass(RegClass);
3060  if (!RC->contains(Reg)) {
3061  ErrInfo = "Operand has incorrect register class.";
3062  return false;
3063  }
3064  }
3065  }
3066 
3067  // Verify SDWA
3068  if (isSDWA(MI)) {
3069  if (!ST.hasSDWA()) {
3070  ErrInfo = "SDWA is not supported on this target";
3071  return false;
3072  }
3073 
3074  int DstIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdst);
3075 
3076  const int OpIndicies[] = { DstIdx, Src0Idx, Src1Idx, Src2Idx };
3077 
3078  for (int OpIdx: OpIndicies) {
3079  if (OpIdx == -1)
3080  continue;
3081  const MachineOperand &MO = MI.getOperand(OpIdx);
3082 
3083  if (!ST.hasSDWAScalar()) {
3084  // Only VGPRS on VI
3085  if (!MO.isReg() || !RI.hasVGPRs(RI.getRegClassForReg(MRI, MO.getReg()))) {
3086  ErrInfo = "Only VGPRs allowed as operands in SDWA instructions on VI";
3087  return false;
3088  }
3089  } else {
3090  // No immediates on GFX9
3091  if (!MO.isReg()) {
3092  ErrInfo = "Only reg allowed as operands in SDWA instructions on GFX9";
3093  return false;
3094  }
3095  }
3096  }
3097 
3098  if (!ST.hasSDWAOmod()) {
3099  // No omod allowed on VI
3100  const MachineOperand *OMod = getNamedOperand(MI, AMDGPU::OpName::omod);
3101  if (OMod != nullptr &&
3102  (!OMod->isImm() || OMod->getImm() != 0)) {
3103  ErrInfo = "OMod not allowed in SDWA instructions on VI";
3104  return false;
3105  }
3106  }
3107 
3108  uint16_t BasicOpcode = AMDGPU::getBasicFromSDWAOp(Opcode);
3109  if (isVOPC(BasicOpcode)) {
3110  if (!ST.hasSDWASdst() && DstIdx != -1) {
3111  // Only vcc allowed as dst on VI for VOPC
3112  const MachineOperand &Dst = MI.getOperand(DstIdx);
3113  if (!Dst.isReg() || Dst.getReg() != AMDGPU::VCC) {
3114  ErrInfo = "Only VCC allowed as dst in SDWA instructions on VI";
3115  return false;
3116  }
3117  } else if (!ST.hasSDWAOutModsVOPC()) {
3118  // No clamp allowed on GFX9 for VOPC
3119  const MachineOperand *Clamp = getNamedOperand(MI, AMDGPU::OpName::clamp);
3120  if (Clamp && (!Clamp->isImm() || Clamp->getImm() != 0)) {
3121  ErrInfo = "Clamp not allowed in VOPC SDWA instructions on VI";
3122  return false;
3123  }
3124 
3125  // No omod allowed on GFX9 for VOPC
3126  const MachineOperand *OMod = getNamedOperand(MI, AMDGPU::OpName::omod);
3127  if (OMod && (!OMod->isImm() || OMod->getImm() != 0)) {
3128  ErrInfo = "OMod not allowed in VOPC SDWA instructions on VI";
3129  return false;
3130  }
3131  }
3132  }
3133 
3134  const MachineOperand *DstUnused = getNamedOperand(MI, AMDGPU::OpName::dst_unused);
3135  if (DstUnused && DstUnused->isImm() &&
3136  DstUnused->getImm() == AMDGPU::SDWA::UNUSED_PRESERVE) {
3137  const MachineOperand &Dst = MI.getOperand(DstIdx);
3138  if (!Dst.isReg() || !Dst.isTied()) {
3139  ErrInfo = "Dst register should have tied register";
3140  return false;
3141  }
3142 
3143  const MachineOperand &TiedMO =
3144  MI.getOperand(MI.findTiedOperandIdx(DstIdx));
3145  if (!TiedMO.isReg() || !TiedMO.isImplicit() || !TiedMO.isUse()) {
3146  ErrInfo =
3147  "Dst register should be tied to implicit use of preserved register";
3148  return false;
3149  } else if (TargetRegisterInfo::isPhysicalRegister(TiedMO.getReg()) &&
3150  Dst.getReg() != TiedMO.getReg()) {
3151  ErrInfo = "Dst register should use same physical register as preserved";
3152  return false;
3153  }
3154  }
3155  }
3156 
3157  // Verify MIMG
3158  if (isMIMG(MI.getOpcode()) && !MI.mayStore()) {
3159  // Ensure that the return type used is large enough for all the options
3160  // being used TFE/LWE require an extra result register.
3161  const MachineOperand *DMask = getNamedOperand(MI, AMDGPU::OpName::dmask);
3162  if (DMask) {
3163  uint64_t DMaskImm = DMask->getImm();
3164  uint32_t RegCount =
3165  isGather4(MI.getOpcode()) ? 4 : countPopulation(DMaskImm);
3166  const MachineOperand *TFE = getNamedOperand(MI, AMDGPU::OpName::tfe);
3167  const MachineOperand *LWE = getNamedOperand(MI, AMDGPU::OpName::lwe);
3168  const MachineOperand *D16 = getNamedOperand(MI, AMDGPU::OpName::d16);
3169 
3170  // Adjust for packed 16 bit values
3171  if (D16 && D16->getImm() && !ST.hasUnpackedD16VMem())
3172  RegCount >>= 1;
3173 
3174  // Adjust if using LWE or TFE
3175  if ((LWE && LWE->getImm()) || (TFE && TFE->getImm()))
3176  RegCount += 1;
3177 
3178  const uint32_t DstIdx =
3179  AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::vdata);
3180  const MachineOperand &Dst = MI.getOperand(DstIdx);
3181  if (Dst.isReg()) {
3182  const TargetRegisterClass *DstRC = getOpRegClass(MI, DstIdx);
3183  uint32_t DstSize = RI.getRegSizeInBits(*DstRC) / 32;
3184  if (RegCount > DstSize) {
3185  ErrInfo = "MIMG instruction returns too many registers for dst "
3186  "register class";
3187  return false;
3188  }
3189  }
3190  }
3191  }
3192 
3193  // Verify VOP*. Ignore multiple sgpr operands on writelane.
3194  if (Desc.getOpcode() != AMDGPU::V_WRITELANE_B32
3195  && (isVOP1(MI) || isVOP2(MI) || isVOP3(MI) || isVOPC(MI) || isSDWA(MI))) {
3196  // Only look at the true operands. Only a real operand can use the constant
3197  // bus, and we don't want to check pseudo-operands like the source modifier
3198  // flags.
3199  const int OpIndices[] = { Src0Idx, Src1Idx, Src2Idx };
3200 
3201  unsigned ConstantBusCount = 0;
3202  unsigned LiteralCount = 0;
3203 
3204  if (AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::imm) != -1)
3205  ++ConstantBusCount;
3206 
3207  SmallVector<unsigned, 2> SGPRsUsed;
3208  unsigned SGPRUsed = findImplicitSGPRRead(MI);
3209  if (SGPRUsed != AMDGPU::NoRegister) {
3210  ++ConstantBusCount;
3211  SGPRsUsed.push_back(SGPRUsed);
3212  }
3213 
3214  for (int OpIdx : OpIndices) {
3215  if (OpIdx == -1)
3216  break;
3217  const MachineOperand &MO = MI.getOperand(OpIdx);
3218  if (usesConstantBus(MRI, MO, MI.getDesc().OpInfo[OpIdx])) {
3219  if (MO.isReg()) {
3220  SGPRUsed = MO.getReg();
3221  if (llvm::all_of(SGPRsUsed, [this, SGPRUsed](unsigned SGPR) {
3222  return !RI.regsOverlap(SGPRUsed, SGPR);
3223  })) {
3224  ++ConstantBusCount;
3225  SGPRsUsed.push_back(SGPRUsed);
3226  }
3227  } else {
3228  ++ConstantBusCount;
3229  ++LiteralCount;
3230  }
3231  }
3232  }
3233  const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
3234  // v_writelane_b32 is an exception from constant bus restriction:
3235  // vsrc0 can be sgpr, const or m0 and lane select sgpr, m0 or inline-const
3236  if (ConstantBusCount > ST.getConstantBusLimit(Opcode) &&
3237  Opcode != AMDGPU::V_WRITELANE_B32) {
3238  ErrInfo = "VOP* instruction violates constant bus restriction";
3239  return false;
3240  }
3241 
3242  if (isVOP3(MI) && LiteralCount) {
3243  if (LiteralCount && !ST.hasVOP3Literal()) {
3244  ErrInfo = "VOP3 instruction uses literal";
3245  return false;
3246  }
3247  if (LiteralCount > 1) {
3248  ErrInfo = "VOP3 instruction uses more than one literal";
3249  return false;
3250  }
3251  }
3252  }
3253 
3254  // Verify misc. restrictions on specific instructions.
3255  if (Desc.getOpcode() == AMDGPU::V_DIV_SCALE_F32 ||
3256  Desc.getOpcode() == AMDGPU::V_DIV_SCALE_F64) {
3257  const MachineOperand &Src0 = MI.getOperand(Src0Idx);
3258  const MachineOperand &Src1 = MI.getOperand(Src1Idx);
3259  const MachineOperand &Src2 = MI.getOperand(Src2Idx);
3260  if (Src0.isReg() && Src1.isReg() && Src2.isReg()) {
3261  if (!compareMachineOp(Src0, Src1) &&
3262  !compareMachineOp(Src0, Src2)) {
3263  ErrInfo = "v_div_scale_{f32|f64} require src0 = src1 or src2";
3264  return false;
3265  }
3266  }
3267  }
3268 
3269  if (isSOP2(MI) || isSOPC(MI)) {
3270  const MachineOperand &Src0 = MI.getOperand(Src0Idx);
3271  const MachineOperand &Src1 = MI.getOperand(Src1Idx);
3272  unsigned Immediates = 0;
3273 
3274  if (!Src0.isReg() &&
3275  !isInlineConstant(Src0, Desc.OpInfo[Src0Idx].OperandType))
3276  Immediates++;
3277  if (!Src1.isReg() &&
3278  !isInlineConstant(Src1, Desc.OpInfo[Src1Idx].OperandType))
3279  Immediates++;
3280 
3281  if (Immediates > 1) {
3282  ErrInfo = "SOP2/SOPC instruction requires too many immediate constants";
3283  return false;
3284  }
3285  }
3286 
3287  if (isSOPK(MI)) {
3288  auto Op = getNamedOperand(MI, AMDGPU::OpName::simm16);
3289  if (Desc.isBranch()) {
3290  if (!Op->isMBB()) {
3291  ErrInfo = "invalid branch target for SOPK instruction";
3292  return false;
3293  }
3294  } else {
3295  uint64_t Imm = Op->getImm();
3296  if (sopkIsZext(MI)) {
3297  if (!isUInt<16>(Imm)) {
3298  ErrInfo = "invalid immediate for SOPK instruction";
3299  return false;
3300  }
3301  } else {
3302  if (!isInt<16>(Imm)) {
3303  ErrInfo = "invalid immediate for SOPK instruction";
3304  return false;
3305  }
3306  }
3307  }
3308  }
3309 
3310  if (Desc.getOpcode() == AMDGPU::V_MOVRELS_B32_e32 ||
3311  Desc.getOpcode() == AMDGPU::V_MOVRELS_B32_e64 ||
3312  Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e32 ||
3313  Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e64) {
3314  const bool IsDst = Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e32 ||
3315  Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e64;
3316 
3317  const unsigned StaticNumOps = Desc.getNumOperands() +
3318  Desc.getNumImplicitUses();
3319  const unsigned NumImplicitOps = IsDst ? 2 : 1;
3320 
3321  // Allow additional implicit operands. This allows a fixup done by the post
3322  // RA scheduler where the main implicit operand is killed and implicit-defs
3323  // are added for sub-registers that remain live after this instruction.
3324  if (MI.getNumOperands() < StaticNumOps + NumImplicitOps) {
3325  ErrInfo = "missing implicit register operands";
3326  return false;
3327  }
3328 
3329  const MachineOperand *Dst = getNamedOperand(MI, AMDGPU::OpName::vdst);
3330  if (IsDst) {
3331  if (!Dst->isUse()) {
3332  ErrInfo = "v_movreld_b32 vdst should be a use operand";
3333  return false;
3334  }
3335 
3336  unsigned UseOpIdx;
3337  if (!MI.isRegTiedToUseOperand(StaticNumOps, &UseOpIdx) ||
3338  UseOpIdx != StaticNumOps + 1) {
3339  ErrInfo = "movrel implicit operands should be tied";
3340  return false;
3341  }
3342  }
3343 
3344  const MachineOperand &Src0 = MI.getOperand(Src0Idx);
3345  const MachineOperand &ImpUse
3346  = MI.getOperand(StaticNumOps + NumImplicitOps - 1);
3347  if (!ImpUse.isReg() || !ImpUse.isUse() ||
3348  !isSubRegOf(RI, ImpUse, IsDst ? *Dst : Src0)) {
3349  ErrInfo = "src0 should be subreg of implicit vector use";
3350  return false;
3351  }
3352  }
3353 
3354  // Make sure we aren't losing exec uses in the td files. This mostly requires
3355  // being careful when using let Uses to try to add other use registers.
3356  if (shouldReadExec(MI)) {
3357  if (!MI.hasRegisterImplicitUseOperand(AMDGPU::EXEC)) {
3358  ErrInfo = "VALU instruction does not implicitly read exec mask";
3359  return false;
3360  }
3361  }
3362 
3363  if (isSMRD(MI)) {
3364  if (MI.mayStore()) {
3365  // The register offset form of scalar stores may only use m0 as the
3366  // soffset register.
3367  const MachineOperand *Soff = getNamedOperand(MI, AMDGPU::OpName::soff);
3368  if (Soff && Soff->getReg() != AMDGPU::M0) {
3369  ErrInfo = "scalar stores must use m0 as offset register";
3370  return false;
3371  }
3372  }
3373  }
3374 
3375  if (isFLAT(MI) && !MF->getSubtarget<GCNSubtarget>().hasFlatInstOffsets()) {
3376  const MachineOperand *Offset = getNamedOperand(MI, AMDGPU::OpName::offset);
3377  if (Offset->getImm() != 0) {
3378  ErrInfo = "subtarget does not support offsets in flat instructions";
3379  return false;
3380  }
3381  }
3382 
3383  if (isMIMG(MI)) {
3384  const MachineOperand *DimOp = getNamedOperand(MI, AMDGPU::OpName::dim);
3385  if (DimOp) {
3386  int VAddr0Idx = AMDGPU::getNamedOperandIdx(Opcode,
3387  AMDGPU::OpName::vaddr0);
3388  int SRsrcIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::srsrc);
3389  const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(Opcode);
3390  const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode =
3392  const AMDGPU::MIMGDimInfo *Dim =
3394 
3395  if (!Dim) {
3396  ErrInfo = "dim is out of range";
3397  return false;
3398  }
3399 
3400  bool IsNSA = SRsrcIdx - VAddr0Idx > 1;
3401  unsigned AddrWords = BaseOpcode->NumExtraArgs +
3402  (BaseOpcode->Gradients ? Dim->NumGradients : 0) +
3403  (BaseOpcode->Coordinates ? Dim->NumCoords : 0) +
3404  (BaseOpcode->LodOrClampOrMip ? 1 : 0);
3405 
3406  unsigned VAddrWords;
3407  if (IsNSA) {
3408  VAddrWords = SRsrcIdx - VAddr0Idx;
3409  } else {
3410  const TargetRegisterClass *RC = getOpRegClass(MI, VAddr0Idx);
3411  VAddrWords = MRI.getTargetRegisterInfo()->getRegSizeInBits(*RC) / 32;
3412  if (AddrWords > 8)
3413  AddrWords = 16;
3414  else if (AddrWords > 4)
3415  AddrWords = 8;
3416  else if (AddrWords == 3 && VAddrWords == 4) {
3417  // CodeGen uses the V4 variant of instructions for three addresses,
3418  // because the selection DAG does not support non-power-of-two types.
3419  AddrWords = 4;
3420  }
3421  }
3422 
3423  if (VAddrWords != AddrWords) {
3424  ErrInfo = "bad vaddr size";
3425  return false;
3426  }
3427  }
3428  }
3429 
3430  const MachineOperand *DppCt = getNamedOperand(MI, AMDGPU::OpName::dpp_ctrl);
3431  if (DppCt) {
3432  using namespace AMDGPU::DPP;
3433 
3434  unsigned DC = DppCt->getImm();
3435  if (DC == DppCtrl::DPP_UNUSED1 || DC == DppCtrl::DPP_UNUSED2 ||
3436  DC == DppCtrl::DPP_UNUSED3 || DC > DppCtrl::DPP_LAST ||
3442  ErrInfo = "Invalid dpp_ctrl value";
3443  return false;
3444  }
3445  if (DC >= DppCtrl::WAVE_SHL1 && DC <= DppCtrl::WAVE_ROR1 &&
3446  ST.getGeneration() >= AMDGPUSubtarget::GFX10) {
3447  ErrInfo = "Invalid dpp_ctrl value: "
3448  "wavefront shifts are not supported on GFX10+";
3449  return false;
3450  }
3451  if (DC >= DppCtrl::BCAST15 && DC <= DppCtrl::BCAST31 &&
3452  ST.getGeneration() >= AMDGPUSubtarget::GFX10) {
3453  ErrInfo = "Invalid dpp_ctrl value: "
3454  "broadcats are not supported on GFX10+";
3455  return false;
3456  }
3458  ST.getGeneration() < AMDGPUSubtarget::GFX10) {
3459  ErrInfo = "Invalid dpp_ctrl value: "
3460  "row_share and row_xmask are not supported before GFX10";
3461  return false;
3462  }
3463  }
3464 
3465  return true;
3466 }
3467 
3468 unsigned SIInstrInfo::getVALUOp(const MachineInstr &MI) const {
3469  switch (MI.getOpcode()) {
3470  default: return AMDGPU::INSTRUCTION_LIST_END;
3471  case AMDGPU::REG_SEQUENCE: return AMDGPU::REG_SEQUENCE;
3472  case AMDGPU::COPY: return AMDGPU::COPY;
3473  case AMDGPU::PHI: return AMDGPU::PHI;
3474  case AMDGPU::INSERT_SUBREG: return AMDGPU::INSERT_SUBREG;
3475  case AMDGPU::WQM: return AMDGPU::WQM;
3476  case AMDGPU::WWM: return AMDGPU::WWM;
3477  case AMDGPU::S_MOV_B32:
3478  return MI.getOperand(1).isReg() ?
3479  AMDGPU::COPY : AMDGPU::V_MOV_B32_e32;
3480  case AMDGPU::S_ADD_I32:
3481  return ST.hasAddNoCarry() ? AMDGPU::V_ADD_U32_e64 : AMDGPU::V_ADD_I32_e32;
3482  case AMDGPU::S_ADDC_U32:
3483  return AMDGPU::V_ADDC_U32_e32;
3484  case AMDGPU::S_SUB_I32:
3485  return ST.hasAddNoCarry() ? AMDGPU::V_SUB_U32_e64 : AMDGPU::V_SUB_I32_e32;
3486  // FIXME: These are not consistently handled, and selected when the carry is
3487  // used.
3488  case AMDGPU::S_ADD_U32:
3489  return AMDGPU::V_ADD_I32_e32;
3490  case AMDGPU::S_SUB_U32:
3491  return AMDGPU::V_SUB_I32_e32;
3492  case AMDGPU::S_SUBB_U32: return AMDGPU::V_SUBB_U32_e32;
3493  case AMDGPU::S_MUL_I32: return AMDGPU::V_MUL_LO_U32;
3494  case AMDGPU::S_MUL_HI_U32: return AMDGPU::V_MUL_HI_U32;
3495  case AMDGPU::S_MUL_HI_I32: return AMDGPU::V_MUL_HI_I32;
3496  case AMDGPU::S_AND_B32: return AMDGPU::V_AND_B32_e64;
3497  case AMDGPU::S_OR_B32: return AMDGPU::V_OR_B32_e64;
3498  case AMDGPU::S_XOR_B32: return AMDGPU::V_XOR_B32_e64;
3499  case AMDGPU::S_XNOR_B32:
3500  return ST.hasDLInsts() ? AMDGPU::V_XNOR_B32_e64 : AMDGPU::INSTRUCTION_LIST_END;
3501  case AMDGPU::S_MIN_I32: return AMDGPU::V_MIN_I32_e64;
3502  case AMDGPU::S_MIN_U32: return AMDGPU::V_MIN_U32_e64;
3503  case AMDGPU::S_MAX_I32: return AMDGPU::V_MAX_I32_e64;
3504  case AMDGPU::S_MAX_U32: return AMDGPU::V_MAX_U32_e64;
3505  case AMDGPU::S_ASHR_I32: return AMDGPU::V_ASHR_I32_e32;
3506  case AMDGPU::S_ASHR_I64: return AMDGPU::V_ASHR_I64;
3507  case AMDGPU::S_LSHL_B32: return AMDGPU::V_LSHL_B32_e32;
3508  case AMDGPU::S_LSHL_B64: return AMDGPU::V_LSHL_B64;
3509  case AMDGPU::S_LSHR_B32: return AMDGPU::V_LSHR_B32_e32;
3510  case AMDGPU::S_LSHR_B64: return AMDGPU::V_LSHR_B64;
3511  case AMDGPU::S_SEXT_I32_I8: return AMDGPU::V_BFE_I32;
3512  case AMDGPU::S_SEXT_I32_I16: return AMDGPU::V_BFE_I32;
3513  case AMDGPU::S_BFE_U32: return AMDGPU::V_BFE_U32;
3514  case AMDGPU::S_BFE_I32: return AMDGPU::V_BFE_I32;
3515  case AMDGPU::S_BFM_B32: return AMDGPU::V_BFM_B32_e64;
3516  case AMDGPU::S_BREV_B32: return AMDGPU::V_BFREV_B32_e32;
3517  case AMDGPU::S_NOT_B32: return AMDGPU::V_NOT_B32_e32;
3518  case AMDGPU::S_NOT_B64: return AMDGPU::V_NOT_B32_e32;
3519  case AMDGPU::S_CMP_EQ_I32: return AMDGPU::V_CMP_EQ_I32_e32;
3520  case AMDGPU::S_CMP_LG_I32: return AMDGPU::V_CMP_NE_I32_e32;
3521  case AMDGPU::S_CMP_GT_I32: return AMDGPU::V_CMP_GT_I32_e32;
3522  case AMDGPU::S_CMP_GE_I32: return AMDGPU::V_CMP_GE_I32_e32;
3523  case AMDGPU::S_CMP_LT_I32: return AMDGPU::V_CMP_LT_I32_e32;
3524  case AMDGPU::S_CMP_LE_I32: return AMDGPU::V_CMP_LE_I32_e32;
3525  case AMDGPU::S_CMP_EQ_U32: return AMDGPU::V_CMP_EQ_U32_e32;
3526  case AMDGPU::S_CMP_LG_U32: return AMDGPU::V_CMP_NE_U32_e32;
3527  case AMDGPU::S_CMP_GT_U32: return AMDGPU::V_CMP_GT_U32_e32;
3528  case AMDGPU::S_CMP_GE_U32: return AMDGPU::V_CMP_GE_U32_e32;
3529  case AMDGPU::S_CMP_LT_U32: return AMDGPU::V_CMP_LT_U32_e32;
3530  case AMDGPU::S_CMP_LE_U32: return AMDGPU::V_CMP_LE_U32_e32;
3531  case AMDGPU::S_CMP_EQ_U64: return AMDGPU::V_CMP_EQ_U64_e32;
3532  case AMDGPU::S_CMP_LG_U64: return AMDGPU::V_CMP_NE_U64_e32;
3533  case AMDGPU::S_BCNT1_I32_B32: return AMDGPU::V_BCNT_U32_B32_e64;
3534  case AMDGPU::S_FF1_I32_B32: return AMDGPU::V_FFBL_B32_e32;
3535  case AMDGPU::S_FLBIT_I32_B32: return AMDGPU::V_FFBH_U32_e32;
3536  case AMDGPU::S_FLBIT_I32: return AMDGPU::V_FFBH_I32_e64;
3537  case AMDGPU::S_CBRANCH_SCC0: return AMDGPU::S_CBRANCH_VCCZ;
3538  case AMDGPU::S_CBRANCH_SCC1: return AMDGPU::S_CBRANCH_VCCNZ;
3539  }
3541  "Unexpected scalar opcode without corresponding vector one!");
3542 }
3543 
3545  unsigned OpNo) const {
3547  const MCInstrDesc &Desc = get(MI.getOpcode());
3548  if (MI.isVariadic() || OpNo >= Desc.getNumOperands() ||
3549  Desc.OpInfo[OpNo].RegClass == -1) {
3550  unsigned Reg = MI.getOperand(OpNo).getReg();
3551 
3553  return MRI.getRegClass(Reg);
3554  return RI.getPhysRegClass(Reg);
3555  }
3556 
3557  unsigned RCID = Desc.OpInfo[OpNo].RegClass;
3558  return RI.getRegClass(RCID);
3559 }
3560 
3561 void SIInstrInfo::legalizeOpWithMove(MachineInstr &MI, unsigned OpIdx) const {
3563  MachineBasicBlock *MBB = MI.getParent();
3564  MachineOperand &MO = MI.getOperand(OpIdx);
3566  const SIRegisterInfo *TRI =
3567  static_cast<const SIRegisterInfo*>(MRI.getTargetRegisterInfo());
3568  unsigned RCID = get(MI.getOpcode()).OpInfo[OpIdx].RegClass;
3569  const TargetRegisterClass *RC = RI.getRegClass(RCID);
3570  unsigned Size = TRI->getRegSizeInBits(*RC);
3571  unsigned Opcode = (Size == 64) ? AMDGPU::V_MOV_B64_PSEUDO : AMDGPU::V_MOV_B32_e32;
3572  if (MO.isReg())
3573  Opcode = AMDGPU::COPY;
3574  else if (RI.isSGPRClass(RC))
3575  Opcode = (Size == 64) ? AMDGPU::S_MOV_B64 : AMDGPU::S_MOV_B32;
3576 
3577  const TargetRegisterClass *VRC = RI.getEquivalentVGPRClass(RC);
3578  if (RI.getCommonSubClass(&AMDGPU::VReg_64RegClass, VRC))
3579  VRC = &AMDGPU::VReg_64RegClass;
3580  else
3581  VRC = &AMDGPU::VGPR_32RegClass;
3582 
3583  unsigned Reg = MRI.createVirtualRegister(VRC);
3584  DebugLoc DL = MBB->findDebugLoc(I);
3585  BuildMI(*MI.getParent(), I, DL, get(Opcode), Reg).add(MO);
3586  MO.ChangeToRegister(Reg, false);
3587 }
3588 
3591  MachineOperand &SuperReg,
3592  const TargetRegisterClass *SuperRC,
3593  unsigned SubIdx,
3594  const TargetRegisterClass *SubRC)
3595  const {
3596  MachineBasicBlock *MBB = MI->getParent();
3597  DebugLoc DL = MI->getDebugLoc();
3598  unsigned SubReg = MRI.createVirtualRegister(SubRC);
3599 
3600  if (SuperReg.getSubReg() == AMDGPU::NoSubRegister) {
3601  BuildMI(*MBB, MI, DL, get(TargetOpcode::COPY), SubReg)
3602  .addReg(SuperReg.getReg(), 0, SubIdx);
3603  return SubReg;
3604  }
3605 
3606  // Just in case the super register is itself a sub-register, copy it to a new
3607  // value so we don't need to worry about merging its subreg index with the
3608  // SubIdx passed to this function. The register coalescer should be able to
3609  // eliminate this extra copy.
3610  unsigned NewSuperReg = MRI.createVirtualRegister(SuperRC);
3611 
3612  BuildMI(*MBB, MI, DL, get(TargetOpcode::COPY), NewSuperReg)
3613  .addReg(SuperReg.getReg(), 0, SuperReg.getSubReg());
3614 
3615  BuildMI(*MBB, MI, DL, get(TargetOpcode::COPY), SubReg)
3616  .addReg(NewSuperReg, 0, SubIdx);
3617 
3618  return SubReg;
3619 }
3620 
3624  MachineOperand &Op,
3625  const TargetRegisterClass *SuperRC,
3626  unsigned SubIdx,
3627  const TargetRegisterClass *SubRC) const {
3628  if (Op.isImm()) {
3629  if (SubIdx == AMDGPU::sub0)
3630  return MachineOperand::CreateImm(static_cast<int32_t>(Op.getImm()));
3631  if (SubIdx == AMDGPU::sub1)
3632  return MachineOperand::CreateImm(static_cast<int32_t>(Op.getImm() >> 32));
3633 
3634  llvm_unreachable("Unhandled register index for immediate");
3635  }
3636 
3637  unsigned SubReg = buildExtractSubReg(MII, MRI, Op, SuperRC,
3638  SubIdx, SubRC);
3639  return MachineOperand::CreateReg(SubReg, false);
3640 }
3641 
3642 // Change the order of operands from (0, 1, 2) to (0, 2, 1)
3643 void SIInstrInfo::swapOperands(MachineInstr &Inst) const {
3644  assert(Inst.getNumExplicitOperands() == 3);
3645  MachineOperand Op1 = Inst.getOperand(1);
3646  Inst.RemoveOperand(1);
3647  Inst.addOperand(Op1);
3648 }
3649 
3651  const MCOperandInfo &OpInfo,
3652  const MachineOperand &MO) const {
3653  if (!MO.isReg())
3654  return false;
3655 
3656  unsigned Reg = MO.getReg();
3657  const TargetRegisterClass *RC =
3659  MRI.getRegClass(Reg) :
3660  RI.getPhysRegClass(Reg);
3661 
3662  const SIRegisterInfo *TRI =
3663  static_cast<const SIRegisterInfo*>(MRI.getTargetRegisterInfo());
3664  RC = TRI->getSubRegClass(RC, MO.getSubReg());
3665 
3666  // In order to be legal, the common sub-class must be equal to the
3667  // class of the current operand. For example:
3668  //
3669  // v_mov_b32 s0 ; Operand defined as vsrc_b32
3670  // ; RI.getCommonSubClass(s0,vsrc_b32) = sgpr ; LEGAL
3671  //
3672  // s_sendmsg 0, s0 ; Operand defined as m0reg
3673  // ; RI.getCommonSubClass(s0,m0reg) = m0reg ; NOT LEGAL
3674 
3675  return RI.getCommonSubClass(RC, RI.getRegClass(OpInfo.RegClass)) == RC;
3676 }
3677 
3679  const MCOperandInfo &OpInfo,
3680  const MachineOperand &MO) const {
3681  if (MO.isReg())
3682  return isLegalRegOperand(MRI, OpInfo, MO);
3683 
3684  // Handle non-register types that are treated like immediates.
3685  assert(MO.isImm() || MO.isTargetIndex() || MO.isFI());
3686  return true;
3687 }
3688 
3689 bool SIInstrInfo::isOperandLegal(const MachineInstr &MI, unsigned OpIdx,
3690  const MachineOperand *MO) const {
3691  const MachineFunction &MF = *MI.getParent()->getParent();
3692  const MachineRegisterInfo &MRI = MF.getRegInfo();
3693  const MCInstrDesc &InstDesc = MI.getDesc();
3694  const MCOperandInfo &OpInfo = InstDesc.OpInfo[OpIdx];
3695  const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
3696  const TargetRegisterClass *DefinedRC =
3697  OpInfo.RegClass != -1 ? RI.getRegClass(OpInfo.RegClass) : nullptr;
3698  if (!MO)
3699  MO = &MI.getOperand(OpIdx);
3700 
3701  int ConstantBusLimit = ST.getConstantBusLimit(MI.getOpcode());
3702  int VOP3LiteralLimit = ST.hasVOP3Literal() ? 1 : 0;
3703  if (isVALU(MI) && usesConstantBus(MRI, *MO, OpInfo)) {
3704  if (isVOP3(MI) && isLiteralConstantLike(*MO, OpInfo) && !VOP3LiteralLimit--)
3705  return false;
3706 
3707  SmallDenseSet<RegSubRegPair> SGPRsUsed;
3708  if (MO->isReg())
3709  SGPRsUsed.insert(RegSubRegPair(MO->getReg(), MO->getSubReg()));
3710 
3711  for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
3712  if (i == OpIdx)
3713  continue;
3714  const MachineOperand &Op = MI.getOperand(i);
3715  if (Op.isReg()) {
3716  RegSubRegPair SGPR(Op.getReg(), Op.getSubReg());
3717  if (!SGPRsUsed.count(SGPR) &&
3718  usesConstantBus(MRI, Op, InstDesc.OpInfo[i])) {
3719  if (--ConstantBusLimit <= 0)
3720  return false;
3721  SGPRsUsed.insert(SGPR);
3722  }
3723  } else if (InstDesc.OpInfo[i].OperandType == AMDGPU::OPERAND_KIMM32) {
3724  if (--ConstantBusLimit <= 0)
3725  return false;
3726  } else if (isVOP3(MI) && AMDGPU::isSISrcOperand(InstDesc, i) &&
3727  isLiteralConstantLike(Op, InstDesc.OpInfo[i])) {
3728  if (!VOP3LiteralLimit--)
3729  return false;
3730  if (--ConstantBusLimit <= 0)
3731  return false;
3732  }
3733  }
3734  }
3735 
3736  if (MO->isReg()) {
3737  assert(DefinedRC);
3738  return isLegalRegOperand(MRI, OpInfo, *MO);
3739  }
3740 
3741  // Handle non-register types that are treated like immediates.
3742  assert(MO->isImm() || MO->isTargetIndex() || MO->isFI());
3743 
3744  if (!DefinedRC) {
3745  // This operand expects an immediate.
3746  return true;
3747  }
3748 
3749  return isImmOperandLegal(MI, OpIdx, *MO);
3750 }
3751 
3753  MachineInstr &MI) const {
3754  unsigned Opc = MI.getOpcode();
3755  const MCInstrDesc &InstrDesc = get(Opc);
3756 
3757  int Src1Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1);
3758  MachineOperand &Src1 = MI.getOperand(Src1Idx);
3759 
3760  // If there is an implicit SGPR use such as VCC use for v_addc_u32/v_subb_u32
3761  // we need to only have one constant bus use before GFX10.
3762  bool HasImplicitSGPR = findImplicitSGPRRead(MI) != AMDGPU::NoRegister;
3763  if (HasImplicitSGPR && ST.getConstantBusLimit(Opc) <= 1) {
3764  int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0);
3765  MachineOperand &Src0 = MI.getOperand(Src0Idx);
3766 
3767  if (Src0.isReg() && (RI.isSGPRReg(MRI, Src0.getReg()) ||
3768  isLiteralConstantLike(Src0, InstrDesc.OpInfo[Src0Idx])))
3769  legalizeOpWithMove(MI, Src0Idx);
3770  }
3771 
3772  // Special case: V_WRITELANE_B32 accepts only immediate or SGPR operands for
3773  // both the value to write (src0) and lane select (src1). Fix up non-SGPR
3774  // src0/src1 with V_READFIRSTLANE.
3775  if (Opc == AMDGPU::V_WRITELANE_B32) {
3776  int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0);
3777  MachineOperand &Src0 = MI.getOperand(Src0Idx);
3778  const DebugLoc &DL = MI.getDebugLoc();
3779  if (Src0.isReg() && RI.isVGPR(MRI, Src0.getReg())) {
3780  unsigned Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
3781  BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg)
3782  .add(Src0);
3783  Src0.ChangeToRegister(Reg, false);
3784  }
3785  if (Src1.isReg() && RI.isVGPR(MRI, Src1.getReg())) {
3786  unsigned Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
3787  const DebugLoc &DL = MI.getDebugLoc();
3788  BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg)
3789  .add(Src1);
3790  Src1.ChangeToRegister(Reg, false);
3791  }
3792  return;
3793  }
3794 
3795  // VOP2 src0 instructions support all operand types, so we don't need to check
3796  // their legality. If src1 is already legal, we don't need to do anything.
3797  if (isLegalRegOperand(MRI, InstrDesc.OpInfo[Src1Idx], Src1))
3798  return;
3799 
3800  // Special case: V_READLANE_B32 accepts only immediate or SGPR operands for
3801  // lane select. Fix up using V_READFIRSTLANE, since we assume that the lane
3802  // select is uniform.
3803  if (Opc == AMDGPU::V_READLANE_B32 && Src1.isReg() &&
3804  RI.isVGPR(MRI, Src1.getReg())) {
3805  unsigned Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
3806  const DebugLoc &DL = MI.getDebugLoc();
3807  BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg)
3808  .add(Src1);
3809  Src1.ChangeToRegister(Reg, false);
3810  return;
3811  }
3812 
3813  // We do not use commuteInstruction here because it is too aggressive and will
3814  // commute if it is possible. We only want to commute here if it improves
3815  // legality. This can be called a fairly large number of times so don't waste
3816  // compile time pointlessly swapping and checking legality again.
3817  if (HasImplicitSGPR || !MI.isCommutable()) {
3818  legalizeOpWithMove(MI, Src1Idx);
3819  return;
3820  }
3821 
3822  int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0);
3823  MachineOperand &Src0 = MI.getOperand(Src0Idx);
3824 
3825  // If src0 can be used as src1, commuting will make the operands legal.
3826  // Otherwise we have to give up and insert a move.
3827  //
3828  // TODO: Other immediate-like operand kinds could be commuted if there was a
3829  // MachineOperand::ChangeTo* for them.
3830  if ((!Src1.isImm() && !Src1.isReg()) ||
3831  !isLegalRegOperand(MRI, InstrDesc.OpInfo[Src1Idx], Src0)) {
3832  legalizeOpWithMove(MI, Src1Idx);
3833  return;
3834  }
3835 
3836  int CommutedOpc = commuteOpcode(MI);
3837  if (CommutedOpc == -1) {
3838  legalizeOpWithMove(MI, Src1Idx);
3839  return;
3840  }
3841 
3842  MI.setDesc(get(CommutedOpc));
3843 
3844  unsigned Src0Reg = Src0.getReg();
3845  unsigned Src0SubReg = Src0.getSubReg();
3846  bool Src0Kill = Src0.isKill();
3847 
3848  if (Src1.isImm())
3849  Src0.ChangeToImmediate(Src1.getImm());
3850  else if (Src1.isReg()) {
3851  Src0.ChangeToRegister(Src1.getReg(), false, false, Src1.isKill());
3852  Src0.setSubReg(Src1.getSubReg());
3853  } else
3854  llvm_unreachable("Should only have register or immediate operands");
3855 
3856  Src1.ChangeToRegister(Src0Reg, false, false, Src0Kill);
3857  Src1.setSubReg(Src0SubReg);
3858  fixImplicitOperands(MI);
3859 }
3860 
3861 // Legalize VOP3 operands. All operand types are supported for any operand
3862 // but only one literal constant and only starting from GFX10.
3864  MachineInstr &MI) const {
3865  unsigned Opc = MI.getOpcode();
3866 
3867  int VOP3Idx[3] = {
3868  AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0),
3869  AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1),
3870  AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2)
3871  };
3872 
3873  if (Opc == AMDGPU::V_PERMLANE16_B32 ||
3874  Opc == AMDGPU::V_PERMLANEX16_B32) {
3875  // src1 and src2 must be scalar
3876  MachineOperand &Src1 = MI.getOperand(VOP3Idx[1]);
3877  MachineOperand &Src2 = MI.getOperand(VOP3Idx[2]);
3878  const DebugLoc &DL = MI.getDebugLoc();
3879  if (Src1.isReg() && !RI.isSGPRClass(MRI.getRegClass(Src1.getReg()))) {
3880  unsigned Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
3881  BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg)
3882  .add(Src1);
3883  Src1.ChangeToRegister(Reg, false);
3884  }
3885  if (Src2.isReg() && !RI.isSGPRClass(MRI.getRegClass(Src2.getReg()))) {
3886  unsigned Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
3887  BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg)
3888  .add(Src2);
3889  Src2.ChangeToRegister(Reg, false);
3890  }
3891  }
3892 
3893  // Find the one SGPR operand we are allowed to use.
3894  int ConstantBusLimit = ST.getConstantBusLimit(Opc);
3895  int LiteralLimit = ST.hasVOP3Literal() ? 1 : 0;
3896  SmallDenseSet<unsigned> SGPRsUsed;
3897  unsigned SGPRReg = findUsedSGPR(MI, VOP3Idx);
3898  if (SGPRReg != AMDGPU::NoRegister) {
3899  SGPRsUsed.insert(SGPRReg);
3900  --ConstantBusLimit;
3901  }
3902 
3903  for (unsigned i = 0; i < 3; ++i) {
3904  int Idx = VOP3Idx[i];
3905  if (Idx == -1)
3906  break;
3907  MachineOperand &MO = MI.getOperand(Idx);
3908 
3909  if (!MO.isReg()) {
3910  if (!isLiteralConstantLike(MO, get(Opc).OpInfo[Idx]))
3911  continue;
3912 
3913  if (LiteralLimit > 0 && ConstantBusLimit > 0) {
3914  --LiteralLimit;
3915  --ConstantBusLimit;
3916  continue;
3917  }
3918 
3919  --LiteralLimit;
3920  --ConstantBusLimit;
3921  legalizeOpWithMove(MI, Idx);
3922  continue;
3923  }
3924 
3925  if (!RI.isSGPRClass(MRI.getRegClass(MO.getReg())))
3926  continue; // VGPRs are legal
3927 
3928  // We can use one SGPR in each VOP3 instruction prior to GFX10
3929  // and two starting from GFX10.
3930  if (SGPRsUsed.count(MO.getReg()))
3931  continue;
3932  if (ConstantBusLimit > 0) {
3933  SGPRsUsed.insert(MO.getReg());
3934  --ConstantBusLimit;
3935  continue;
3936  }
3937 
3938  // If we make it this far, then the operand is not legal and we must
3939  // legalize it.
3940  legalizeOpWithMove(MI, Idx);
3941  }
3942 }
3943 
3945  MachineRegisterInfo &MRI) const {
3946  const TargetRegisterClass *VRC = MRI.getRegClass(SrcReg);
3947  const TargetRegisterClass *SRC = RI.getEquivalentSGPRClass(VRC);
3948  unsigned DstReg = MRI.createVirtualRegister(SRC);
3949  unsigned SubRegs = RI.getRegSizeInBits(*VRC) / 32;
3950 
3951  if (SubRegs == 1) {
3952  BuildMI(*UseMI.getParent(), UseMI, UseMI.getDebugLoc(),
3953  get(AMDGPU::V_READFIRSTLANE_B32), DstReg)
3954  .addReg(SrcReg);
3955  return DstReg;
3956  }
3957 
3959  for (unsigned i = 0; i < SubRegs; ++i) {
3960  unsigned SGPR = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
3961  BuildMI(*UseMI.getParent(), UseMI, UseMI.getDebugLoc(),
3962  get(AMDGPU::V_READFIRSTLANE_B32), SGPR)
3963  .addReg(SrcReg, 0, RI.getSubRegFromChannel(i));
3964  SRegs.push_back(SGPR);
3965  }
3966 
3967  MachineInstrBuilder MIB =
3968  BuildMI(*UseMI.getParent(), UseMI, UseMI.getDebugLoc(),
3969  get(AMDGPU::REG_SEQUENCE), DstReg);
3970  for (unsigned i = 0; i < SubRegs; ++i) {
3971  MIB.addReg(SRegs[i]);
3972  MIB.addImm(RI.getSubRegFromChannel(i));
3973  }
3974  return DstReg;
3975 }
3976 
3978  MachineInstr &MI) const {
3979 
3980  // If the pointer is store in VGPRs, then we need to move them to
3981  // SGPRs using v_readfirstlane. This is safe because we only select
3982  // loads with uniform pointers to SMRD instruction so we know the
3983  // pointer value is uniform.
3984  MachineOperand *SBase = getNamedOperand(MI, AMDGPU::OpName::sbase);
3985  if (SBase && !RI.isSGPRClass(MRI.getRegClass(SBase->getReg()))) {
3986  unsigned SGPR = readlaneVGPRToSGPR(SBase->getReg(), MI, MRI);
3987  SBase->setReg(SGPR);
3988  }
3989  MachineOperand *SOff = getNamedOperand(MI, AMDGPU::OpName::soff);
3990  if (SOff && !RI.isSGPRClass(MRI.getRegClass(SOff->getReg()))) {
3991  unsigned SGPR = readlaneVGPRToSGPR(SOff->getReg(), MI, MRI);
3992  SOff->setReg(SGPR);
3993  }
3994 }
3995 
3998  const TargetRegisterClass *DstRC,
3999  MachineOperand &Op,
4001  const DebugLoc &DL) const {
4002  unsigned OpReg = Op.getReg();
4003  unsigned OpSubReg = Op.getSubReg();
4004 
4005  const TargetRegisterClass *OpRC = RI.getSubClassWithSubReg(
4006  RI.getRegClassForReg(MRI, OpReg), OpSubReg);
4007 
4008  // Check if operand is already the correct register class.
4009  if (DstRC == OpRC)
4010  return;
4011 
4012  unsigned DstReg = MRI.createVirtualRegister(DstRC);
4013  MachineInstr *Copy =
4014  BuildMI(InsertMBB, I, DL, get(AMDGPU::COPY), DstReg).add(Op);
4015 
4016  Op.setReg(DstReg);
4017  Op.setSubReg(0);
4018 
4019  MachineInstr *Def = MRI.getVRegDef(OpReg);
4020  if (!Def)
4021  return;
4022 
4023  // Try to eliminate the copy if it is copying an immediate value.
4024  if (Def->isMoveImmediate())
4025  FoldImmediate(*Copy, *Def, OpReg, &MRI);
4026 }
4027 
4028 // Emit the actual waterfall loop, executing the wrapped instruction for each
4029 // unique value of \p Rsrc across all lanes. In the best case we execute 1
4030 // iteration, in the worst case we execute 64 (once per lane).
4031 static void
4033  MachineBasicBlock &OrigBB, MachineBasicBlock &LoopBB,
4034  const DebugLoc &DL, MachineOperand &Rsrc) {
4035  MachineFunction &MF = *OrigBB.getParent();
4036  const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
4037  const SIRegisterInfo *TRI = ST.getRegisterInfo();
4038  unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
4039  unsigned SaveExecOpc =
4040  ST.isWave32() ? AMDGPU::S_AND_SAVEEXEC_B32 : AMDGPU::S_AND_SAVEEXEC_B64;
4041  unsigned XorTermOpc =
4042  ST.isWave32() ? AMDGPU::S_XOR_B32_term : AMDGPU::S_XOR_B64_term;
4043  unsigned AndOpc =
4044  ST.isWave32() ? AMDGPU::S_AND_B32 : AMDGPU::S_AND_B64;
4045  const auto *BoolXExecRC = TRI->getRegClass(AMDGPU::SReg_1_XEXECRegClassID);
4046 
4047  MachineBasicBlock::iterator I = LoopBB.begin();
4048 
4049  unsigned VRsrc = Rsrc.getReg();
4050  unsigned VRsrcUndef = getUndefRegState(Rsrc.isUndef());
4051 
4052  unsigned SaveExec = MRI.createVirtualRegister(BoolXExecRC);
4053  unsigned CondReg0 = MRI.createVirtualRegister(BoolXExecRC);
4054  unsigned CondReg1 = MRI.createVirtualRegister(BoolXExecRC);
4055  unsigned AndCond = MRI.createVirtualRegister(BoolXExecRC);
4056  unsigned SRsrcSub0 = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
4057  unsigned SRsrcSub1 = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
4058  unsigned SRsrcSub2 = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
4059  unsigned SRsrcSub3 = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
4060  unsigned SRsrc = MRI.createVirtualRegister(&AMDGPU::SReg_128RegClass);
4061 
4062  // Beginning of the loop, read the next Rsrc variant.
4063  BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), SRsrcSub0)
4064  .addReg(VRsrc, VRsrcUndef, AMDGPU::sub0);
4065  BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), SRsrcSub1)
4066  .addReg(VRsrc, VRsrcUndef, AMDGPU::sub1);
4067  BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), SRsrcSub2)
4068  .addReg(VRsrc, VRsrcUndef, AMDGPU::sub2);
4069  BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), SRsrcSub3)
4070  .addReg(VRsrc, VRsrcUndef, AMDGPU::sub3);
4071 
4072  BuildMI(LoopBB, I, DL, TII.get(AMDGPU::REG_SEQUENCE), SRsrc)
4073  .addReg(SRsrcSub0)
4074  .addImm(AMDGPU::sub0)
4075  .addReg(SRsrcSub1)
4076  .addImm(AMDGPU::sub1)
4077  .addReg(SRsrcSub2)
4078  .addImm(AMDGPU::sub2)
4079  .addReg(SRsrcSub3)
4080  .addImm(AMDGPU::sub3);
4081 
4082  // Update Rsrc operand to use the SGPR Rsrc.
4083  Rsrc.setReg(SRsrc);
4084  Rsrc.setIsKill(true);
4085 
4086  // Identify all lanes with identical Rsrc operands in their VGPRs.
4087  BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_CMP_EQ_U64_e64), CondReg0)
4088  .addReg(SRsrc, 0, AMDGPU::sub0_sub1)
4089  .addReg(VRsrc, 0, AMDGPU::sub0_sub1);
4090  BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_CMP_EQ_U64_e64), CondReg1)
4091  .addReg(SRsrc, 0, AMDGPU::sub2_sub3)
4092  .addReg(VRsrc, 0, AMDGPU::sub2_sub3);
4093  BuildMI(LoopBB, I, DL, TII.get(AndOpc), AndCond)
4094  .addReg(CondReg0)
4095  .addReg(CondReg1);
4096 
4097  MRI.setSimpleHint(SaveExec, AndCond);
4098 
4099  // Update EXEC to matching lanes, saving original to SaveExec.
4100  BuildMI(LoopBB, I, DL, TII.get(SaveExecOpc), SaveExec)
4101  .addReg(AndCond, RegState::Kill);
4102 
4103  // The original instruction is here; we insert the terminators after it.
4104  I = LoopBB.end();
4105 
4106  // Update EXEC, switch all done bits to 0 and all todo bits to 1.
4107  BuildMI(LoopBB, I, DL, TII.get(XorTermOpc), Exec)
4108  .addReg(Exec)
4109  .addReg(SaveExec);
4110  BuildMI(LoopBB, I, DL, TII.get(AMDGPU::S_CBRANCH_EXECNZ)).addMBB(&LoopBB);
4111 }
4112 
4113 // Build a waterfall loop around \p MI, replacing the VGPR \p Rsrc register
4114 // with SGPRs by iterating over all unique values across all lanes.
4116  MachineOperand &Rsrc, MachineDominatorTree *MDT) {
4117  MachineBasicBlock &MBB = *MI.getParent();
4118  MachineFunction &MF = *MBB.getParent();
4119  const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
4120  const SIRegisterInfo *TRI = ST.getRegisterInfo();
4123  const DebugLoc &DL = MI.getDebugLoc();
4124  unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
4125  unsigned MovExecOpc = ST.isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64;
4126  const auto *BoolXExecRC = TRI->getRegClass(AMDGPU::SReg_1_XEXECRegClassID);
4127 
4128  unsigned SaveExec = MRI.createVirtualRegister(BoolXExecRC);
4129 
4130  // Save the EXEC mask
4131  BuildMI(MBB, I, DL, TII.get(MovExecOpc), SaveExec).addReg(Exec);
4132 
4133  // Killed uses in the instruction we are waterfalling around will be
4134  // incorrect due to the added control-flow.
4135  for (auto &MO : MI.uses()) {
4136  if (MO.isReg() && MO.isUse()) {
4137  MRI.clearKillFlags(MO.getReg());
4138  }
4139  }
4140 
4141  // To insert the loop we need to split the block. Move everything after this
4142  // point to a new block, and insert a new empty block between the two.
4144  MachineBasicBlock *RemainderBB = MF.CreateMachineBasicBlock();
4145  MachineFunction::iterator MBBI(MBB);
4146  ++MBBI;
4147 
4148  MF.insert(MBBI, LoopBB);
4149  MF.insert(MBBI, RemainderBB);
4150 
4151  LoopBB->addSuccessor(LoopBB);
4152  LoopBB->addSuccessor(RemainderBB);
4153 
4154  // Move MI to the LoopBB, and the remainder of the block to RemainderBB.
4156  RemainderBB->transferSuccessorsAndUpdatePHIs(&MBB);
4157  RemainderBB->splice(RemainderBB->begin(), &MBB, I, MBB.end());
4158  LoopBB->splice(LoopBB->begin(), &MBB, J);
4159 
4160  MBB.addSuccessor(LoopBB);
4161 
4162  // Update dominators. We know that MBB immediately dominates LoopBB, that
4163  // LoopBB immediately dominates RemainderBB, and that RemainderBB immediately
4164  // dominates all of the successors transferred to it from MBB that MBB used
4165  // to dominate.
4166  if (MDT) {
4167  MDT->addNewBlock(LoopBB, &MBB);
4168  MDT->addNewBlock(RemainderBB, LoopBB);
4169  for (auto &Succ : RemainderBB->successors()) {
4170  if (MDT->dominates(&MBB, Succ)) {
4171  MDT->changeImmediateDominator(Succ, RemainderBB);
4172  }
4173  }
4174  }
4175 
4176  emitLoadSRsrcFromVGPRLoop(TII, MRI, MBB, *LoopBB, DL, Rsrc);
4177 
4178  // Restore the EXEC mask
4179  MachineBasicBlock::iterator First = RemainderBB->begin();
4180  BuildMI(*RemainderBB, First, DL, TII.get(MovExecOpc), Exec).addReg(SaveExec);
4181 }
4182 
4183 // Extract pointer from Rsrc and return a zero-value Rsrc replacement.
4184 static std::tuple<unsigned, unsigned>
4186  MachineBasicBlock &MBB = *MI.getParent();
4187  MachineFunction &MF = *MBB.getParent();
4189 
4190  // Extract the ptr from the resource descriptor.
4191  unsigned RsrcPtr =
4192  TII.buildExtractSubReg(MI, MRI, Rsrc, &AMDGPU::VReg_128RegClass,
4193  AMDGPU::sub0_sub1, &AMDGPU::VReg_64RegClass);
4194 
4195  // Create an empty resource descriptor
4196  unsigned Zero64 = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
4197  unsigned SRsrcFormatLo = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
4198  unsigned SRsrcFormatHi = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
4199  unsigned NewSRsrc = MRI.createVirtualRegister(&AMDGPU::SReg_128RegClass);
4200  uint64_t RsrcDataFormat = TII.getDefaultRsrcDataFormat();
4201 
4202  // Zero64 = 0
4203  BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::S_MOV_B64), Zero64)
4204  .addImm(0);
4205 
4206  // SRsrcFormatLo = RSRC_DATA_FORMAT{31-0}
4207  BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::S_MOV_B32), SRsrcFormatLo)
4208  .addImm(RsrcDataFormat & 0xFFFFFFFF);
4209 
4210  // SRsrcFormatHi = RSRC_DATA_FORMAT{63-32}
4211  BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::S_MOV_B32), SRsrcFormatHi)
4212  .addImm(RsrcDataFormat >> 32);
4213 
4214  // NewSRsrc = {Zero64, SRsrcFormat}
4215  BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::REG_SEQUENCE), NewSRsrc)
4216  .addReg(Zero64)
4217  .addImm(AMDGPU::sub0_sub1)
4218  .addReg(SRsrcFormatLo)
4219  .addImm(AMDGPU::sub2)
4220  .addReg(SRsrcFormatHi)
4221  .addImm(AMDGPU::sub3);
4222 
4223  return std::make_tuple(RsrcPtr, NewSRsrc);
4224 }
4225 
4227  MachineDominatorTree *MDT) const {
4228  MachineFunction &MF = *MI.getParent()->getParent();
4230 
4231  // Legalize VOP2
4232  if (isVOP2(MI) || isVOPC(MI)) {
4233  legalizeOperandsVOP2(MRI, MI);
4234  return;
4235  }
4236 
4237  // Legalize VOP3
4238  if (isVOP3(MI)) {
4239  legalizeOperandsVOP3(MRI, MI);
4240  return;
4241  }
4242 
4243  // Legalize SMRD
4244  if (isSMRD(MI)) {
4245  legalizeOperandsSMRD(MRI, MI);
4246  return;
4247  }
4248 
4249  // Legalize REG_SEQUENCE and PHI
4250  // The register class of the operands much be the same type as the register
4251  // class of the output.
4252  if (MI.getOpcode() == AMDGPU::PHI) {
4253  const TargetRegisterClass *RC = nullptr, *SRC = nullptr, *VRC = nullptr;
4254  for (unsigned i = 1, e = MI.getNumOperands(); i != e; i += 2) {
4255  if (!MI.getOperand(i).isReg() ||
4257  continue;
4258  const TargetRegisterClass *OpRC =
4259  MRI.getRegClass(MI.getOperand(i).getReg());
4260  if (RI.hasVGPRs(OpRC)) {
4261  VRC = OpRC;
4262  } else {
4263  SRC = OpRC;
4264  }
4265  }
4266 
4267  // If any of the operands are VGPR registers, then they all most be
4268  // otherwise we will create illegal VGPR->SGPR copies when legalizing
4269  // them.
4270  if (VRC || !RI.isSGPRClass(getOpRegClass(MI, 0))) {
4271  if (!VRC) {
4272  assert(SRC);
4273  VRC = RI.getEquivalentVGPRClass(SRC);
4274  }
4275  RC = VRC;
4276  } else {
4277  RC = SRC;
4278  }
4279 
4280  // Update all the operands so they have the same type.
4281  for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) {
4282  MachineOperand &Op = MI.getOperand(I);
4284  continue;
4285 
4286  // MI is a PHI instruction.
4287  MachineBasicBlock *InsertBB = MI.getOperand(I + 1).getMBB();
4288  MachineBasicBlock::iterator Insert = InsertBB->getFirstTerminator();
4289 
4290  // Avoid creating no-op copies with the same src and dst reg class. These
4291  // confuse some of the machine passes.
4292  legalizeGenericOperand(*InsertBB, Insert, RC, Op, MRI, MI.getDebugLoc());
4293  }
4294  }
4295 
4296  // REG_SEQUENCE doesn't really require operand legalization, but if one has a
4297  // VGPR dest type and SGPR sources, insert copies so all operands are
4298  // VGPRs. This seems to help operand folding / the register coalescer.
4299  if (MI.getOpcode() == AMDGPU::REG_SEQUENCE) {
4300  MachineBasicBlock *MBB = MI.getParent();
4301  const TargetRegisterClass *DstRC = getOpRegClass(MI, 0);
4302  if (RI.hasVGPRs(DstRC)) {
4303  // Update all the operands so they are VGPR register classes. These may
4304  // not be the same register class because REG_SEQUENCE supports mixing
4305  // subregister index types e.g. sub0_sub1 + sub2 + sub3
4306  for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) {
4307  MachineOperand &Op = MI.getOperand(I);
4309  continue;
4310 
4311  const TargetRegisterClass *OpRC = MRI.getRegClass(Op.getReg());
4312  const TargetRegisterClass *VRC = RI.getEquivalentVGPRClass(OpRC);
4313  if (VRC == OpRC)
4314  continue;
4315 
4316  legalizeGenericOperand(*MBB, MI, VRC, Op, MRI, MI.getDebugLoc());
4317  Op.setIsKill();
4318  }
4319  }
4320 
4321  return;
4322  }
4323 
4324  // Legalize INSERT_SUBREG
4325  // src0 must have the same register class as dst
4326  if (MI.getOpcode() == AMDGPU::INSERT_SUBREG) {
4327  unsigned Dst = MI.getOperand(0).getReg();
4328  unsigned Src0 = MI.getOperand(1).getReg();
4329  const TargetRegisterClass *DstRC = MRI.getRegClass(Dst);
4330  const TargetRegisterClass *Src0RC = MRI.getRegClass(Src0);
4331  if (DstRC != Src0RC) {
4332  MachineBasicBlock *MBB = MI.getParent();
4333  MachineOperand &Op = MI.getOperand(1);
4334  legalizeGenericOperand(*MBB, MI, DstRC, Op, MRI, MI.getDebugLoc());
4335  }
4336  return;
4337  }
4338 
4339  // Legalize SI_INIT_M0
4340  if (MI.getOpcode() == AMDGPU::SI_INIT_M0) {
4341  MachineOperand &Src = MI.getOperand(0);
4342  if (Src.isReg() && RI.hasVGPRs(MRI.getRegClass(Src.getReg())))
4343  Src.setReg(readlaneVGPRToSGPR(Src.getReg(), MI, MRI));
4344  return;
4345  }
4346 
4347  // Legalize MIMG and MUBUF/MTBUF for shaders.
4348  //
4349  // Shaders only generate MUBUF/MTBUF instructions via intrinsics or via
4350  // scratch memory access. In both cases, the legalization never involves
4351  // conversion to the addr64 form.
4352  if (isMIMG(MI) ||
4354  (isMUBUF(MI) || isMTBUF(MI)))) {
4355  MachineOperand *SRsrc = getNamedOperand(MI, AMDGPU::OpName::srsrc);
4356  if (SRsrc && !RI.isSGPRClass(MRI.getRegClass(SRsrc->getReg()))) {
4357  unsigned SGPR = readlaneVGPRToSGPR(SRsrc->getReg(), MI, MRI);
4358  SRsrc->setReg(SGPR);
4359  }
4360 
4361  MachineOperand *SSamp = getNamedOperand(MI, AMDGPU::OpName::ssamp);
4362  if (SSamp && !RI.isSGPRClass(MRI.getRegClass(SSamp->getReg()))) {
4363  unsigned SGPR = readlaneVGPRToSGPR(SSamp->getReg(), MI, MRI);
4364  SSamp->setReg(SGPR);
4365  }
4366  return;
4367  }
4368 
4369  // Legalize MUBUF* instructions.
4370  int RsrcIdx =
4371  AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::srsrc);
4372  if (RsrcIdx != -1) {
4373  // We have an MUBUF instruction
4374  MachineOperand *Rsrc = &MI.getOperand(RsrcIdx);
4375  unsigned RsrcRC = get(MI.getOpcode()).OpInfo[RsrcIdx].RegClass;
4376  if (RI.getCommonSubClass(MRI.getRegClass(Rsrc->getReg()),
4377  RI.getRegClass(RsrcRC))) {
4378  // The operands are legal.
4379  // FIXME: We may need to legalize operands besided srsrc.
4380  return;
4381  }
4382 
4383  // Legalize a VGPR Rsrc.
4384  //
4385  // If the instruction is _ADDR64, we can avoid a waterfall by extracting
4386  // the base pointer from the VGPR Rsrc, adding it to the VAddr, then using
4387  // a zero-value SRsrc.
4388  //
4389  // If the instruction is _OFFSET (both idxen and offen disabled), and we
4390  // support ADDR64 instructions, we can convert to ADDR64 and do the same as
4391  // above.
4392  //
4393  // Otherwise we are on non-ADDR64 hardware, and/or we have
4394  // idxen/offen/bothen and we fall back to a waterfall loop.
4395 
4396  MachineBasicBlock &MBB = *MI.getParent();
4397 
4398  MachineOperand *VAddr = getNamedOperand(MI, AMDGPU::OpName::vaddr);
4399  if (VAddr && AMDGPU::getIfAddr64Inst(MI.getOpcode()) != -1) {
4400  // This is already an ADDR64 instruction so we need to add the pointer
4401  // extracted from the resource descriptor to the current value of VAddr.
4402  unsigned NewVAddrLo = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4403  unsigned NewVAddrHi = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4404  unsigned NewVAddr = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass);
4405 
4406  unsigned RsrcPtr, NewSRsrc;
4407  std::tie(RsrcPtr, NewSRsrc) = extractRsrcPtr(*this, MI, *Rsrc);
4408 
4409  // NewVaddrLo = RsrcPtr:sub0 + VAddr:sub0
4410  DebugLoc DL = MI.getDebugLoc();
4412  BuildMI(MBB, MI, DL, get(AMDGPU::V_ADD_I32_e32), NewVAddrLo)
4413  .addReg(RsrcPtr, 0, AMDGPU::sub0)
4414  .addReg(VAddr->getReg(), 0, AMDGPU::sub0));
4415 
4416  // NewVaddrHi = RsrcPtr:sub1 + VAddr:sub1
4418  BuildMI(MBB, MI, DL, get(AMDGPU::V_ADDC_U32_e32), NewVAddrHi)
4419  .addReg(RsrcPtr, 0, AMDGPU::sub1)
4420  .addReg(VAddr->getReg(), 0, AMDGPU::sub1));
4421 
4422  // NewVaddr = {NewVaddrHi, NewVaddrLo}
4423  BuildMI(MBB, MI, MI.getDebugLoc(), get(AMDGPU::REG_SEQUENCE), NewVAddr)
4424  .addReg(NewVAddrLo)
4425  .addImm(AMDGPU::sub0)
4426  .addReg(NewVAddrHi)
4427  .addImm(AMDGPU::sub1);
4428 
4429  VAddr->setReg(NewVAddr);
4430  Rsrc->setReg(NewSRsrc);
4431  } else if (!VAddr && ST.hasAddr64()) {
4432  // This instructions is the _OFFSET variant, so we need to convert it to
4433  // ADDR64.
4434  assert(MBB.getParent()->getSubtarget<GCNSubtarget>().getGeneration()
4436  "FIXME: Need to emit flat atomics here");
4437 
4438  unsigned RsrcPtr, NewSRsrc;
4439  std::tie(RsrcPtr, NewSRsrc) = extractRsrcPtr(*this, MI, *Rsrc);
4440 
4441  unsigned NewVAddr = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass);
4442  MachineOperand *VData = getNamedOperand(MI, AMDGPU::OpName::vdata);
4443  MachineOperand *Offset = getNamedOperand(MI, AMDGPU::OpName::offset);
4444  MachineOperand *SOffset = getNamedOperand(MI, AMDGPU::OpName::soffset);
4445  unsigned Addr64Opcode = AMDGPU::getAddr64Inst(MI.getOpcode());
4446 
4447  // Atomics rith return have have an additional tied operand and are
4448  // missing some of the special bits.
4449  MachineOperand *VDataIn = getNamedOperand(MI, AMDGPU::OpName::vdata_in);
4450  MachineInstr *Addr64;
4451 
4452  if (!VDataIn) {
4453  // Regular buffer load / store.
4454  MachineInstrBuilder MIB =
4455  BuildMI(MBB, MI, MI.getDebugLoc(), get(Addr64Opcode))
4456  .add(*VData)
4457  .addReg(NewVAddr)
4458  .addReg(NewSRsrc)
4459  .add(*SOffset)
4460  .add(*Offset);
4461 
4462  // Atomics do not have this operand.
4463  if (const MachineOperand *GLC =
4464  getNamedOperand(MI, AMDGPU::OpName::glc)) {
4465  MIB.addImm(GLC->getImm());
4466  }
4467  if (const MachineOperand *DLC =
4468  getNamedOperand(MI, AMDGPU::OpName::dlc)) {
4469  MIB.addImm(DLC->getImm());
4470  }
4471 
4472  MIB.addImm(getNamedImmOperand(MI, AMDGPU::OpName::slc));
4473 
4474  if (const MachineOperand *TFE =
4475  getNamedOperand(MI, AMDGPU::OpName::tfe)) {
4476  MIB.addImm(TFE->getImm());
4477  }
4478 
4479  MIB.cloneMemRefs(MI);
4480  Addr64 = MIB;
4481  } else {
4482  // Atomics with return.
4483  Addr64 = BuildMI(MBB, MI, MI.getDebugLoc(), get(Addr64Opcode))
4484  .add(*VData)
4485  .add(*VDataIn)
4486  .addReg(NewVAddr)
4487  .addReg(NewSRsrc)
4488  .add(*SOffset)
4489  .add(*Offset)
4490  .addImm(getNamedImmOperand(MI, AMDGPU::OpName::slc))
4491  .cloneMemRefs(MI);
4492  }
4493 
4494  MI.removeFromParent();
4495 
4496  // NewVaddr = {NewVaddrHi, NewVaddrLo}
4497  BuildMI(MBB, Addr64, Addr64->getDebugLoc(), get(AMDGPU::REG_SEQUENCE),
4498  NewVAddr)
4499  .addReg(RsrcPtr, 0, AMDGPU::sub0)
4500  .addImm(AMDGPU::sub0)
4501  .addReg(RsrcPtr, 0, AMDGPU::sub1)
4502  .addImm(AMDGPU::sub1);
4503  } else {
4504  // This is another variant; legalize Rsrc with waterfall loop from VGPRs
4505  // to SGPRs.
4506  loadSRsrcFromVGPR(*this, MI, *Rsrc, MDT);
4507  }
4508  }
4509 }
4510 
4512  MachineDominatorTree *MDT) const {
4513  SetVectorType Worklist;
4514  Worklist.insert(&TopInst);
4515 
4516  while (!Worklist.empty()) {
4517  MachineInstr &Inst = *Worklist.pop_back_val();
4518  MachineBasicBlock *MBB = Inst.getParent();
4520 
4521  unsigned Opcode = Inst.getOpcode();
4522  unsigned NewOpcode = getVALUOp(Inst);
4523 
4524  // Handle some special cases
4525  switch (Opcode) {
4526  default:
4527  break;
4528  case AMDGPU::S_ADD_U64_PSEUDO:
4529  case AMDGPU::S_SUB_U64_PSEUDO:
4530  splitScalar64BitAddSub(Worklist, Inst, MDT);
4531  Inst.eraseFromParent();
4532  continue;
4533  case AMDGPU::S_ADD_I32:
4534  case AMDGPU::S_SUB_I32:
4535  // FIXME: The u32 versions currently selected use the carry.
4536  if (moveScalarAddSub(Worklist, Inst, MDT))
4537  continue;
4538 
4539  // Default handling
4540  break;
4541  case AMDGPU::S_AND_B64:
4542  splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_AND_B32, MDT);
4543  Inst.eraseFromParent();
4544  continue;
4545 
4546  case AMDGPU::S_OR_B64:
4547  splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_OR_B32, MDT);
4548  Inst.eraseFromParent();
4549  continue;
4550 
4551  case AMDGPU::S_XOR_B64:
4552  splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_XOR_B32, MDT);
4553  Inst.eraseFromParent();
4554  continue;
4555 
4556  case AMDGPU::S_NAND_B64:
4557  splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_NAND_B32, MDT);
4558  Inst.eraseFromParent();
4559  continue;
4560 
4561  case AMDGPU::S_NOR_B64:
4562  splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_NOR_B32, MDT);
4563  Inst.eraseFromParent();
4564  continue;
4565 
4566  case AMDGPU::S_XNOR_B64:
4567  if (ST.hasDLInsts())
4568  splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_XNOR_B32, MDT);
4569  else
4570  splitScalar64BitXnor(Worklist, Inst, MDT);
4571  Inst.eraseFromParent();
4572  continue;
4573 
4574  case AMDGPU::S_ANDN2_B64:
4575  splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_ANDN2_B32, MDT);
4576  Inst.eraseFromParent();
4577  continue;
4578 
4579  case AMDGPU::S_ORN2_B64:
4580  splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_ORN2_B32, MDT);
4581  Inst.eraseFromParent();
4582  continue;
4583 
4584  case AMDGPU::S_NOT_B64:
4585  splitScalar64BitUnaryOp(Worklist, Inst, AMDGPU::S_NOT_B32);
4586  Inst.eraseFromParent();
4587  continue;
4588 
4589  case AMDGPU::S_BCNT1_I32_B64:
4590  splitScalar64BitBCNT(Worklist, Inst);
4591  Inst.eraseFromParent();
4592  continue;
4593 
4594  case AMDGPU::S_BFE_I64:
4595  splitScalar64BitBFE(Worklist, Inst);
4596  Inst.eraseFromParent();
4597  continue;
4598 
4599  case AMDGPU::S_LSHL_B32:
4600  if (ST.getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS) {
4601  NewOpcode = AMDGPU::V_LSHLREV_B32_e64;
4602  swapOperands(Inst);
4603  }
4604  break;
4605  case AMDGPU::S_ASHR_I32:
4606  if (ST.getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS) {
4607  NewOpcode = AMDGPU::V_ASHRREV_I32_e64;
4608  swapOperands(Inst);
4609  }
4610  break;
4611  case AMDGPU::S_LSHR_B32:
4612  if (ST.getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS) {
4613  NewOpcode = AMDGPU::V_LSHRREV_B32_e64;
4614  swapOperands(Inst);
4615  }
4616  break;
4617  case AMDGPU::S_LSHL_B64:
4618  if (ST.getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS) {
4619  NewOpcode = AMDGPU::V_LSHLREV_B64;
4620  swapOperands(Inst);
4621  }
4622  break;
4623  case AMDGPU::S_ASHR_I64:
4624  if (ST.getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS) {
4625  NewOpcode = AMDGPU::V_ASHRREV_I64;
4626  swapOperands(Inst);
4627  }
4628  break;
4629  case AMDGPU::S_LSHR_B64:
4630  if (ST.getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS) {
4631  NewOpcode = AMDGPU::V_LSHRREV_B64;
4632  swapOperands(Inst);
4633  }
4634  break;
4635 
4636  case AMDGPU::S_ABS_I32:
4637  lowerScalarAbs(Worklist, Inst);
4638  Inst.eraseFromParent();
4639  continue;
4640 
4641  case AMDGPU::S_CBRANCH_SCC0:
4642  case AMDGPU::S_CBRANCH_SCC1:
4643  // Clear unused bits of vcc
4644  if (ST.isWave32())
4645  BuildMI(*MBB, Inst, Inst.getDebugLoc(), get(AMDGPU::S_AND_B32),
4646  AMDGPU::VCC_LO)
4647  .addReg(AMDGPU::EXEC_LO)
4648  .addReg(AMDGPU::VCC_LO);
4649  else
4650  BuildMI(*MBB, Inst, Inst.getDebugLoc(), get(AMDGPU::S_AND_B64),
4651  AMDGPU::VCC)
4652  .addReg(AMDGPU::EXEC)
4653  .addReg(AMDGPU::VCC);
4654  break;
4655 
4656  case AMDGPU::S_BFE_U64:
4657  case AMDGPU::S_BFM_B64:
4658  llvm_unreachable("Moving this op to VALU not implemented");
4659 
4660  case AMDGPU::S_PACK_LL_B32_B16:
4661  case AMDGPU::S_PACK_LH_B32_B16:
4662  case AMDGPU::S_PACK_HH_B32_B16:
4663  movePackToVALU(Worklist, MRI, Inst);
4664  Inst.eraseFromParent();
4665  continue;
4666 
4667  case AMDGPU::S_XNOR_B32:
4668  lowerScalarXnor(Worklist, Inst);
4669  Inst.eraseFromParent();
4670  continue;
4671 
4672  case AMDGPU::S_NAND_B32:
4673  splitScalarNotBinop(Worklist, Inst, AMDGPU::S_AND_B32);
4674  Inst.eraseFromParent();
4675  continue;
4676 
4677  case AMDGPU::S_NOR_B32:
4678  splitScalarNotBinop(Worklist, Inst, AMDGPU::S_OR_B32);
4679  Inst.eraseFromParent();
4680  continue;
4681 
4682  case AMDGPU::S_ANDN2_B32:
4683  splitScalarBinOpN2(Worklist, Inst, AMDGPU::S_AND_B32);
4684  Inst.eraseFromParent();
4685  continue;
4686 
4687  case AMDGPU::S_ORN2_B32:
4688  splitScalarBinOpN2(Worklist, Inst, AMDGPU::S_OR_B32);
4689  Inst.eraseFromParent();
4690  continue;
4691  }
4692 
4693  if (NewOpcode == AMDGPU::INSTRUCTION_LIST_END) {
4694  // We cannot move this instruction to the VALU, so we should try to
4695  // legalize its operands instead.
4696  legalizeOperands(Inst, MDT);
4697  continue;
4698  }
4699 
4700  // Use the new VALU Opcode.
4701  const MCInstrDesc &NewDesc = get(NewOpcode);
4702  Inst.setDesc(NewDesc);
4703 
4704  // Remove any references to SCC. Vector instructions can't read from it, and
4705  // We're just about to add the implicit use / defs of VCC, and we don't want
4706  // both.
4707  for (unsigned i = Inst.getNumOperands() - 1; i > 0; --i) {
4708  MachineOperand &Op = Inst.getOperand(i);
4709  if (Op.isReg() && Op.getReg() == AMDGPU::SCC) {
4710  // Only propagate through live-def of SCC.
4711  if (Op.isDef() && !Op.isDead())
4712  addSCCDefUsersToVALUWorklist(Op, Inst, Worklist);
4713  Inst.RemoveOperand(i);
4714  }
4715  }
4716 
4717  if (Opcode == AMDGPU::S_SEXT_I32_I8 || Opcode == AMDGPU::S_SEXT_I32_I16) {
4718  // We are converting these to a BFE, so we need to add the missing
4719  // operands for the size and offset.
4720  unsigned Size = (Opcode == AMDGPU::S_SEXT_I32_I8) ? 8 : 16;
4723 
4724  } else if (Opcode == AMDGPU::S_BCNT1_I32_B32) {
4725  // The VALU version adds the second operand to the result, so insert an
4726  // extra 0 operand.
4728  }
4729 
4731  fixImplicitOperands(Inst);
4732 
4733  if (Opcode == AMDGPU::S_BFE_I32 || Opcode == AMDGPU::S_BFE_U32) {
4734  const MachineOperand &OffsetWidthOp = Inst.getOperand(2);
4735  // If we need to move this to VGPRs, we need to unpack the second operand
4736  // back into the 2 separate ones for bit offset and width.
4737  assert(OffsetWidthOp.isImm() &&
4738  "Scalar BFE is only implemented for constant width and offset");
4739  uint32_t Imm = OffsetWidthOp.getImm();
4740 
4741  uint32_t Offset = Imm & 0x3f; // Extract bits [5:0].
4742  uint32_t BitWidth = (Imm & 0x7f0000) >> 16; // Extract bits [22:16].
4743  Inst.RemoveOperand(2); // Remove old immediate.
4744  Inst.addOperand(MachineOperand::CreateImm(Offset));
4745  Inst.addOperand(MachineOperand::CreateImm(BitWidth));
4746  }
4747 
4748  bool HasDst = Inst.getOperand(0).isReg() && Inst.getOperand(0).isDef();
4749  unsigned NewDstReg = AMDGPU::NoRegister;
4750  if (HasDst) {
4751  unsigned DstReg = Inst.getOperand(0).getReg();
4753  continue;
4754 
4755  // Update the destination register class.
4756  const TargetRegisterClass *NewDstRC = getDestEquivalentVGPRClass(Inst);
4757  if (!NewDstRC)
4758  continue;
4759 
4760  if (Inst.isCopy() &&
4762  NewDstRC == RI.getRegClassForReg(MRI, Inst.getOperand(1).getReg())) {
4763  // Instead of creating a copy where src and dst are the same register
4764  // class, we just replace all uses of dst with src. These kinds of
4765  // copies interfere with the heuristics MachineSink uses to decide
4766  // whether or not to split a critical edge. Since the pass assumes
4767  // that copies will end up as machine instructions and not be
4768  // eliminated.
4769  addUsersToMoveToVALUWorklist(DstReg, MRI, Worklist);
4770  MRI.replaceRegWith(DstReg, Inst.getOperand(1).getReg());
4771  MRI.clearKillFlags(Inst.getOperand(1).getReg());
4772  Inst.getOperand(0).setReg(DstReg);
4773 
4774  // Make sure we don't leave around a dead VGPR->SGPR copy. Normally
4775  // these are deleted later, but at -O0 it would leave a suspicious
4776  // looking illegal copy of an undef register.
4777  for (unsigned I = Inst.getNumOperands() - 1; I != 0; --I)
4778  Inst.RemoveOperand(I);
4779  Inst.setDesc(get(AMDGPU::IMPLICIT_DEF));
4780  continue;
4781  }
4782 
4783  NewDstReg = MRI.createVirtualRegister(NewDstRC);
4784  MRI.replaceRegWith(DstReg, NewDstReg);
4785  }
4786 
4787  // Legalize the operands
4788  legalizeOperands(Inst, MDT);
4789 
4790  if (HasDst)
4791  addUsersToMoveToVALUWorklist(NewDstReg, MRI, Worklist);
4792  }
4793 }
4794 
4795 // Add/sub require special handling to deal with carry outs.
4796 bool SIInstrInfo::moveScalarAddSub(SetVectorType &Worklist, MachineInstr &Inst,
4797  MachineDominatorTree *MDT) const {
4798  if (ST.hasAddNoCarry()) {
4799  // Assume there is no user of scc since we don't select this in that case.
4800  // Since scc isn't used, it doesn't really matter if the i32 or u32 variant
4801  // is used.
4802 
4803  MachineBasicBlock &MBB = *Inst.getParent();
4805 
4806  unsigned OldDstReg = Inst.getOperand(0).getReg();
4807  unsigned ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4808 
4809  unsigned Opc = Inst.getOpcode();
4810  assert(Opc == AMDGPU::S_ADD_I32 || Opc == AMDGPU::S_SUB_I32);
4811 
4812  unsigned NewOpc = Opc == AMDGPU::S_ADD_I32 ?
4813  AMDGPU::V_ADD_U32_e64 : AMDGPU::V_SUB_U32_e64;
4814 
4815  assert(Inst.getOperand(3).getReg() == AMDGPU::SCC);
4816  Inst.RemoveOperand(3);
4817 
4818  Inst.setDesc(get(NewOpc));
4819  Inst.addOperand(MachineOperand::CreateImm(0)); // clamp bit
4820  Inst.addImplicitDefUseOperands(*MBB.getParent());
4821  MRI.replaceRegWith(OldDstReg, ResultReg);
4822  legalizeOperands(Inst, MDT);
4823 
4824  addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist);
4825  return true;
4826  }
4827 
4828  return false;
4829 }
4830 
4831 void SIInstrInfo::lowerScalarAbs(SetVectorType &Worklist,
4832  MachineInstr &Inst) const {
4833  MachineBasicBlock &MBB = *Inst.getParent();
4835  MachineBasicBlock::iterator MII = Inst;
4836  DebugLoc DL = Inst.getDebugLoc();
4837 
4838  MachineOperand &Dest = Inst.getOperand(0);
4839  MachineOperand &Src = Inst.getOperand(1);
4840  unsigned TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4841  unsigned ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4842 
4843  unsigned SubOp = ST.hasAddNoCarry() ?
4844  AMDGPU::V_SUB_U32_e32 : AMDGPU::V_SUB_I32_e32;
4845 
4846  BuildMI(MBB, MII, DL, get(SubOp), TmpReg)
4847  .addImm(0)
4848  .addReg(Src.getReg());
4849 
4850  BuildMI(MBB, MII, DL, get(AMDGPU::V_MAX_I32_e64), ResultReg)
4851  .addReg(Src.getReg())
4852  .addReg(TmpReg);
4853 
4854  MRI.replaceRegWith(Dest.getReg(), ResultReg);
4855  addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist);
4856 }
4857 
4858 void SIInstrInfo::lowerScalarXnor(SetVectorType &Worklist,
4859  MachineInstr &Inst) const {
4860  MachineBasicBlock &MBB = *Inst.getParent();
4862  MachineBasicBlock::iterator MII = Inst;
4863  const DebugLoc &DL = Inst.getDebugLoc();
4864 
4865  MachineOperand &Dest = Inst.getOperand(0);
4866  MachineOperand &Src0 = Inst.getOperand(1);
4867  MachineOperand &Src1 = Inst.getOperand(2);
4868 
4869  if (ST.hasDLInsts()) {
4870  unsigned NewDest = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4871  legalizeGenericOperand(MBB, MII, &AMDGPU::VGPR_32RegClass, Src0, MRI, DL);
4872  legalizeGenericOperand(MBB, MII, &AMDGPU::VGPR_32RegClass, Src1, MRI, DL);
4873 
4874  BuildMI(MBB, MII, DL, get(AMDGPU::V_XNOR_B32_e64), NewDest)
4875  .add(Src0)
4876  .add(Src1);
4877 
4878  MRI.replaceRegWith(Dest.getReg(), NewDest);
4879  addUsersToMoveToVALUWorklist(NewDest, MRI, Worklist);
4880  } else {
4881  // Using the identity !(x ^ y) == (!x ^ y) == (x ^ !y), we can
4882  // invert either source and then perform the XOR. If either source is a
4883  // scalar register, then we can leave the inversion on the scalar unit to
4884  // acheive a better distrubution of scalar and vector instructions.
4885  bool Src0IsSGPR = Src0.isReg() &&
4886  RI.isSGPRClass(MRI.getRegClass(Src0.getReg()));
4887  bool Src1IsSGPR = Src1.isReg() &&
4888  RI.isSGPRClass(MRI.getRegClass(Src1.getReg()));
4889  MachineInstr *Not = nullptr;
4890  MachineInstr *Xor = nullptr;
4891  unsigned Temp = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
4892  unsigned NewDest = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
4893 
4894  // Build a pair of scalar instructions and add them to the work list.
4895  // The next iteration over the work list will lower these to the vector
4896  // unit as necessary.
4897  if (Src0IsSGPR) {
4898  Not = BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), Temp)
4899  .add(Src0);
4900  Xor = BuildMI(MBB, MII, DL, get(AMDGPU::S_XOR_B32), NewDest)
4901  .addReg(Temp)
4902  .add(Src1);
4903  } else if (Src1IsSGPR) {
4904  Not = BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), Temp)
4905  .add(Src1);
4906  Xor = BuildMI(MBB, MII, DL, get(AMDGPU::S_XOR_B32), NewDest)
4907  .add(Src0)
4908  .addReg(Temp);
4909  } else {
4910  Xor = BuildMI(MBB, MII, DL, get(AMDGPU::S_XOR_B32), Temp)
4911  .add(Src0)
4912  .add(Src1);
4913  Not = BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), NewDest)
4914  .addReg(Temp);
4915  Worklist.insert(Not);
4916  }
4917 
4918  MRI.replaceRegWith(Dest.getReg(), NewDest);
4919 
4920  Worklist.insert(Xor);
4921 
4922  addUsersToMoveToVALUWorklist(NewDest, MRI, Worklist);
4923  }
4924 }
4925 
4926 void SIInstrInfo::splitScalarNotBinop(SetVectorType &Worklist,
4927  MachineInstr &Inst,
4928  unsigned Opcode) const {
4929  MachineBasicBlock &MBB = *Inst.getParent();
4931  MachineBasicBlock::iterator MII = Inst;
4932  const DebugLoc &DL = Inst.getDebugLoc();
4933 
4934  MachineOperand &Dest = Inst.getOperand(0);
4935  MachineOperand &Src0 = Inst.getOperand(1);
4936  MachineOperand &Src1 = Inst.getOperand(2);
4937 
4938  unsigned NewDest = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
4939  unsigned Interm = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
4940 
4941  MachineInstr &Op = *BuildMI(MBB, MII, DL, get(Opcode), Interm)
4942  .add(Src0)
4943  .add(Src1);
4944 
4945  MachineInstr &Not = *BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), NewDest)
4946  .addReg(Interm);
4947 
4948  Worklist.insert(&Op);
4949  Worklist.insert(&Not);
4950 
4951  MRI.replaceRegWith(Dest.getReg(), NewDest);
4952  addUsersToMoveToVALUWorklist(NewDest, MRI, Worklist);
4953 }
4954 
4955 void SIInstrInfo::splitScalarBinOpN2(SetVectorType& Worklist,
4956  MachineInstr &Inst,
4957  unsigned Opcode) const {
4958  MachineBasicBlock &MBB = *Inst.getParent();
4960  MachineBasicBlock::iterator MII = Inst;
4961  const DebugLoc &DL = Inst.getDebugLoc();
4962 
4963  MachineOperand &Dest = Inst.getOperand(0);
4964  MachineOperand &Src0 = Inst.getOperand(1);
4965  MachineOperand &Src1 = Inst.getOperand(2);
4966 
4967  unsigned NewDest = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
4968  unsigned Interm = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
4969 
4970  MachineInstr &Not = *BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), Interm)
4971  .add(Src1);
4972 
4973  MachineInstr &Op = *BuildMI(MBB, MII, DL, get(Opcode), NewDest)
4974  .add(Src0)
4975  .addReg(Interm);
4976 
4977  Worklist.insert(&Not);
4978  Worklist.insert(&Op);
4979 
4980  MRI.replaceRegWith(Dest.getReg(), NewDest);
4981  addUsersToMoveToVALUWorklist(NewDest, MRI, Worklist);
4982 }
4983 
4984 void SIInstrInfo::splitScalar64BitUnaryOp(
4985  SetVectorType &Worklist, MachineInstr &Inst,
4986  unsigned Opcode) const {
4987  MachineBasicBlock &MBB = *Inst.getParent();
4989 
4990  MachineOperand &Dest = Inst.getOperand(0);
4991  MachineOperand &Src0 = Inst.getOperand(1);
4992  DebugLoc DL = Inst.getDebugLoc();
4993 
4994  MachineBasicBlock::iterator MII = Inst;
4995 
4996  const MCInstrDesc &InstDesc = get(Opcode);
4997  const TargetRegisterClass *Src0RC = Src0.isReg() ?
4998  MRI.getRegClass(Src0.getReg()) :
4999  &AMDGPU::SGPR_32RegClass;
5000 
5001  const TargetRegisterClass *Src0SubRC = RI.getSubRegClass(Src0RC, AMDGPU::sub0);
5002 
5003  MachineOperand SrcReg0Sub0 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC,
5004  AMDGPU::sub0, Src0SubRC);
5005 
5006  const TargetRegisterClass *DestRC = MRI.getRegClass(Dest.getReg());
5007  const TargetRegisterClass *NewDestRC = RI.getEquivalentVGPRClass(DestRC);
5008  const TargetRegisterClass *NewDestSubRC = RI.getSubRegClass(NewDestRC, AMDGPU::sub0);
5009 
5010  unsigned DestSub0 = MRI.createVirtualRegister(NewDestSubRC);
5011  MachineInstr &LoHalf = *BuildMI(MBB, MII, DL, InstDesc, DestSub0).add(SrcReg0Sub0);
5012 
5013  MachineOperand SrcReg0Sub1 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC,
5014  AMDGPU::sub1, Src0SubRC);
5015 
5016  unsigned DestSub1 = MRI.createVirtualRegister(NewDestSubRC);
5017  MachineInstr &HiHalf = *BuildMI(MBB, MII, DL, InstDesc, DestSub1).add(SrcReg0Sub1);
5018 
5019  unsigned FullDestReg = MRI.createVirtualRegister(NewDestRC);
5020  BuildMI(MBB, MII, DL, get(TargetOpcode::REG_SEQUENCE), FullDestReg)
5021  .addReg(DestSub0)
5022  .addImm(AMDGPU::sub0)
5023  .addReg(DestSub1)
5024  .addImm(AMDGPU::sub1);
5025 
5026  MRI.replaceRegWith(Dest.getReg(), FullDestReg);
5027 
5028  Worklist.insert(&LoHalf);
5029  Worklist.insert(&HiHalf);
5030 
5031  // We don't need to legalizeOperands here because for a single operand, src0
5032  // will support any kind of input.
5033 
5034  // Move all users of this moved value.
5035  addUsersToMoveToVALUWorklist(FullDestReg, MRI, Worklist);
5036 }
5037 
5038 void SIInstrInfo::splitScalar64BitAddSub(SetVectorType &Worklist,
5039  MachineInstr &Inst,
5040  MachineDominatorTree *MDT) const {
5041  bool IsAdd = (Inst.getOpcode() == AMDGPU::S_ADD_U64_PSEUDO);
5042 
5043  MachineBasicBlock &MBB = *Inst.getParent();
5045  const auto *CarryRC = RI.getRegClass(AMDGPU::SReg_1_XEXECRegClassID);
5046 
5047  unsigned FullDestReg = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass);
5048  unsigned DestSub0 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
5049  unsigned DestSub1 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
5050 
5051  unsigned CarryReg = MRI.createVirtualRegister(CarryRC);
5052  unsigned DeadCarryReg = MRI.createVirtualRegister(CarryRC);
5053 
5054  MachineOperand &Dest = Inst.getOperand(0);
5055  MachineOperand &Src0 = Inst.getOperand(1);
5056  MachineOperand &Src1 = Inst.getOperand(2);
5057  const DebugLoc &DL = Inst.getDebugLoc();
5058  MachineBasicBlock::iterator MII = Inst;
5059 
5060  const TargetRegisterClass *Src0RC = MRI.getRegClass(Src0.getReg());
5061  const TargetRegisterClass *Src1RC = MRI.getRegClass(Src1.getReg());
5062  const TargetRegisterClass *Src0SubRC = RI.getSubRegClass(Src0RC, AMDGPU::sub0);
5063  const TargetRegisterClass *Src1SubRC = RI.getSubRegClass(Src1RC, AMDGPU::sub0);
5064 
5065  MachineOperand SrcReg0Sub0 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC,
5066  AMDGPU::sub0, Src0SubRC);
5067  MachineOperand SrcReg1Sub0 = buildExtractSubRegOrImm(MII, MRI, Src1, Src1RC,
5068  AMDGPU::sub0, Src1SubRC);
5069 
5070 
5071  MachineOperand SrcReg0Sub1 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC,
5072  AMDGPU::sub1, Src0SubRC);
5073  MachineOperand SrcReg1Sub1 = buildExtractSubRegOrImm(MII, MRI, Src1, Src1RC,
5074  AMDGPU::sub1, Src1SubRC);
5075 
5076  unsigned LoOpc = IsAdd ? AMDGPU::V_ADD_I32_e64 : AMDGPU::V_SUB_I32_e64;
5077  MachineInstr *LoHalf =
5078  BuildMI(MBB, MII, DL, get(LoOpc), DestSub0)
5079  .addReg(CarryReg, RegState::Define)
5080  .add(SrcReg0Sub0)
5081  .add(SrcReg1Sub0)
5082  .addImm(0); // clamp bit
5083 
5084  unsigned HiOpc = IsAdd ? AMDGPU::V_ADDC_U32_e64 : AMDGPU::V_SUBB_U32_e64;
5085  MachineInstr *HiHalf =
5086  BuildMI(MBB, MII, DL, get(HiOpc), DestSub1)
5087  .addReg(DeadCarryReg, RegState::Define | RegState::Dead)
5088  .add(SrcReg0Sub1)
5089  .add(SrcReg1Sub1)
5090  .addReg(CarryReg, RegState::Kill)
5091  .addImm(0); // clamp bit
5092 
5093  BuildMI(MBB, MII, DL, get(TargetOpcode::REG_SEQUENCE), FullDestReg)
5094  .addReg(DestSub0)
5095  .addImm(AMDGPU::sub0)
5096  .addReg(DestSub1)
5097  .addImm(AMDGPU::sub1);
5098 
5099  MRI.replaceRegWith(Dest.getReg(), FullDestReg);
5100 
5101  // Try to legalize the operands in case we need to swap the order to keep it
5102  // valid.
5103  legalizeOperands(*LoHalf, MDT);
5104  legalizeOperands(*HiHalf, MDT);
5105 
5106  // Move all users of this moved vlaue.
5107  addUsersToMoveToVALUWorklist(FullDestReg, MRI, Worklist);
5108 }
5109 
5110 void SIInstrInfo::splitScalar64BitBinaryOp(SetVectorType &Worklist,
5111  MachineInstr &Inst, unsigned Opcode,
5112  MachineDominatorTree *MDT) const {
5113  MachineBasicBlock &MBB = *Inst.getParent();
5115 
5116  MachineOperand &Dest = Inst.getOperand(0);
5117  MachineOperand &Src0 = Inst.getOperand(1);
5118  MachineOperand &Src1 = Inst.getOperand(2);
5119  DebugLoc DL = Inst.getDebugLoc();
5120 
5121  MachineBasicBlock::iterator MII = Inst;
5122 
5123  const MCInstrDesc &InstDesc = get(Opcode);
5124  const TargetRegisterClass *Src0RC = Src0.isReg() ?
5125  MRI.getRegClass(Src0.getReg()) :
5126  &AMDGPU::SGPR_32RegClass;
5127 
5128  const TargetRegisterClass *Src0SubRC = RI.getSubRegClass(Src0RC, AMDGPU::sub0);
5129  const TargetRegisterClass *Src1RC = Src1.isReg() ?
5130  MRI.getRegClass(Src1.getReg()) :
5131  &AMDGPU::SGPR_32RegClass;
5132 
5133  const TargetRegisterClass *Src1SubRC = RI.getSubRegClass(Src1RC, AMDGPU::sub0);
5134 
5135  MachineOperand SrcReg0Sub0 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC,
5136  AMDGPU::sub0, Src0SubRC);
5137  MachineOperand SrcReg1Sub0 = buildExtractSubRegOrImm(MII, MRI, Src1, Src1RC,
5138  AMDGPU::sub0, Src1SubRC);
5139  MachineOperand SrcReg0Sub1 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC,
5140  AMDGPU::sub1, Src0SubRC);
5141  MachineOperand SrcReg1Sub1 = buildExtractSubRegOrImm(MII, MRI, Src1, Src1RC,
5142  AMDGPU::sub1, Src1SubRC);
5143 
5144  const TargetRegisterClass *DestRC = MRI.getRegClass(Dest.getReg());
5145  const TargetRegisterClass *NewDestRC = RI.getEquivalentVGPRClass(DestRC);
5146  const TargetRegisterClass *NewDestSubRC = RI.getSubRegClass(NewDestRC, AMDGPU::sub0);
5147 
5148  unsigned DestSub0 = MRI.createVirtualRegister(NewDestSubRC);
5149  MachineInstr &LoHalf = *BuildMI(MBB, MII, DL, InstDesc, DestSub0)
5150  .add(SrcReg0Sub0)
5151  .add(SrcReg1Sub0);
5152 
5153  unsigned DestSub1 = MRI.createVirtualRegister(NewDestSubRC);
5154  MachineInstr &HiHalf = *BuildMI(MBB, MII, DL, InstDesc, DestSub1)
5155  .add(SrcReg0Sub1)
5156  .add(SrcReg1Sub1);
5157 
5158  unsigned FullDestReg = MRI.createVirtualRegister(NewDestRC);
5159  BuildMI(MBB, MII, DL, get(TargetOpcode::REG_SEQUENCE), FullDestReg)
5160  .addReg(DestSub0)
5161  .addImm(AMDGPU::sub0)
5162  .addReg(DestSub1)
5163  .addImm(AMDGPU::sub1);
5164 
5165  MRI.replaceRegWith(Dest.getReg(), FullDestReg);
5166 
5167  Worklist.insert(&LoHalf);
5168  Worklist.insert(&HiHalf);
5169 
5170  // Move all users of this moved vlaue.
5171  addUsersToMoveToVALUWorklist(FullDestReg, MRI, Worklist);
5172 }
5173 
5174 void SIInstrInfo::splitScalar64BitXnor(SetVectorType &Worklist,
5175  MachineInstr &Inst,
5176  MachineDominatorTree *MDT) const {
5177  MachineBasicBlock &MBB = *Inst.getParent();
5179 
5180  MachineOperand &Dest = Inst.getOperand(0);
5181  MachineOperand &Src0 = Inst.getOperand(1);
5182  MachineOperand &Src1 = Inst.getOperand(2);
5183  const DebugLoc &DL = Inst.getDebugLoc();
5184 
5185  MachineBasicBlock::iterator MII = Inst;
5186 
5187  const TargetRegisterClass *DestRC = MRI.getRegClass(Dest.getReg());
5188 
5189  unsigned Interm = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
5190 
5191  MachineOperand* Op0;
5192  MachineOperand* Op1;
5193 
5194  if (Src0.isReg() && RI.isSGPRReg(MRI, Src0.getReg())) {
5195  Op0 = &Src0;
5196  Op1 = &Src1;
5197  } else {
5198  Op0 = &Src1;
5199  Op1 = &Src0;
5200  }
5201 
5202  BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B64), Interm)
5203  .add(*Op0);
5204 
5205  unsigned NewDest = MRI.createVirtualRegister(DestRC);
5206 
5207  MachineInstr &Xor = *BuildMI(MBB, MII, DL, get(AMDGPU::S_XOR_B64), NewDest)
5208  .addReg(Interm)
5209  .add(*Op1);
5210 
5211  MRI.replaceRegWith(Dest.getReg(), NewDest);
5212 
5213  Worklist.insert(&Xor);
5214 }
5215 
5216 void SIInstrInfo::splitScalar64BitBCNT(
5217  SetVectorType &Worklist, MachineInstr &Inst) const {
5218  MachineBasicBlock &MBB = *Inst.getParent();
5220 
5221  MachineBasicBlock::iterator MII = Inst;
5222  const DebugLoc &DL = Inst.getDebugLoc();
5223 
5224  MachineOperand &Dest = Inst.getOperand(0);
5225  MachineOperand &Src = Inst.getOperand(1);
5226 
5227  const MCInstrDesc &InstDesc = get(AMDGPU::V_BCNT_U32_B32_e64);
5228  const TargetRegisterClass *SrcRC = Src.isReg() ?
5229  MRI.getRegClass(Src.getReg()) :
5230  &AMDGPU::SGPR_32RegClass;
5231 
5232  unsigned MidReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
5233  unsigned ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
5234 
5235  const TargetRegisterClass *SrcSubRC = RI.getSubRegClass(SrcRC, AMDGPU::sub0);
5236 
5237  MachineOperand SrcRegSub0 = buildExtractSubRegOrImm(MII, MRI, Src, SrcRC,
5238  AMDGPU::sub0, SrcSubRC);
5239  MachineOperand SrcRegSub1 = buildExtractSubRegOrImm(MII, MRI, Src, SrcRC,
5240  AMDGPU::sub1, SrcSubRC);
5241 
5242  BuildMI(MBB, MII, DL, InstDesc, MidReg).add(SrcRegSub0).addImm(0);
5243 
5244  BuildMI(MBB, MII, DL, InstDesc, ResultReg).add(SrcRegSub1).addReg(MidReg);
5245 
5246  MRI.replaceRegWith(Dest.getReg(), ResultReg);
5247 
5248  // We don't need to legalize operands here. src0 for etiher instruction can be
5249  // an SGPR, and the second input is unused or determined here.
5250  addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist);
5251 }
5252 
5253 void SIInstrInfo::splitScalar64BitBFE(SetVectorType &Worklist,
5254  MachineInstr &Inst) const {
5255  MachineBasicBlock &MBB = *Inst.getParent();
5257  MachineBasicBlock::iterator MII = Inst;
5258  const DebugLoc &DL = Inst.getDebugLoc();
5259 
5260  MachineOperand &Dest = Inst.getOperand(0);
5261  uint32_t Imm = Inst.getOperand(2).getImm();
5262  uint32_t Offset = Imm & 0x3f; // Extract bits [5:0].
5263  uint32_t BitWidth = (Imm & 0x7f0000) >> 16; // Extract bits [22:16].
5264 
5265  (void) Offset;
5266 
5267  // Only sext_inreg cases handled.
5268  assert(Inst.getOpcode() == AMDGPU::S_BFE_I64 && BitWidth <= 32 &&
5269  Offset == 0 && "Not implemented");
5270 
5271  if (BitWidth < 32) {
5272  unsigned MidRegLo = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
5273  unsigned MidRegHi = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
5274  unsigned ResultReg = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass);
5275 
5276  BuildMI(MBB, MII, DL, get(AMDGPU::V_BFE_I32), MidRegLo)
5277  .addReg(Inst.getOperand(1).getReg(), 0, AMDGPU::sub0)
5278  .addImm(0)
5279  .addImm(BitWidth);
5280 
5281  BuildMI(MBB, MII, DL, get(AMDGPU::V_ASHRREV_I32_e32), MidRegHi)
5282  .addImm(31)
5283  .addReg(MidRegLo);
5284 
5285  BuildMI(MBB, MII, DL, get(TargetOpcode::REG_SEQUENCE), ResultReg)
5286  .addReg(MidRegLo)
5287  .addImm(AMDGPU::sub0)
5288  .