LLVM  6.0.0svn
SIPeepholeSDWA.cpp
Go to the documentation of this file.
1 //===- SIPeepholeSDWA.cpp - Peephole optimization for SDWA instructions ---===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 /// \file This pass tries to apply several peephole SDWA patterns.
11 ///
12 /// E.g. original:
13 /// V_LSHRREV_B32_e32 %vreg0, 16, %vreg1
14 /// V_ADD_I32_e32 %vreg2, %vreg0, %vreg3
15 /// V_LSHLREV_B32_e32 %vreg4, 16, %vreg2
16 ///
17 /// Replace:
18 /// V_ADD_I32_sdwa %vreg4, %vreg1, %vreg3
19 /// dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
20 ///
21 //===----------------------------------------------------------------------===//
22 
23 #include "AMDGPU.h"
24 #include "AMDGPUSubtarget.h"
25 #include "SIDefines.h"
26 #include "SIInstrInfo.h"
27 #include "SIRegisterInfo.h"
28 #include "Utils/AMDGPUBaseInfo.h"
29 #include "llvm/ADT/None.h"
30 #include "llvm/ADT/Optional.h"
31 #include "llvm/ADT/STLExtras.h"
32 #include "llvm/ADT/SmallVector.h"
33 #include "llvm/ADT/Statistic.h"
42 #include "llvm/MC/LaneBitmask.h"
43 #include "llvm/MC/MCInstrDesc.h"
44 #include "llvm/Pass.h"
45 #include "llvm/Support/Debug.h"
47 #include <algorithm>
48 #include <cassert>
49 #include <cstdint>
50 #include <memory>
51 #include <unordered_map>
52 
53 using namespace llvm;
54 
55 #define DEBUG_TYPE "si-peephole-sdwa"
56 
57 STATISTIC(NumSDWAPatternsFound, "Number of SDWA patterns found.");
58 STATISTIC(NumSDWAInstructionsPeepholed,
59  "Number of instruction converted to SDWA.");
60 
61 namespace {
62 
63 class SDWAOperand;
64 
65 class SIPeepholeSDWA : public MachineFunctionPass {
66 public:
67  using SDWAOperandsVector = SmallVector<SDWAOperand *, 4>;
68 
69 private:
71  const SIRegisterInfo *TRI;
72  const SIInstrInfo *TII;
73 
74  std::unordered_map<MachineInstr *, std::unique_ptr<SDWAOperand>> SDWAOperands;
75  std::unordered_map<MachineInstr *, SDWAOperandsVector> PotentialMatches;
76  SmallVector<MachineInstr *, 8> ConvertedInstructions;
77 
78  Optional<int64_t> foldToImm(const MachineOperand &Op) const;
79 
80 public:
81  static char ID;
82 
83  SIPeepholeSDWA() : MachineFunctionPass(ID) {
85  }
86 
87  bool runOnMachineFunction(MachineFunction &MF) override;
88  void matchSDWAOperands(MachineFunction &MF);
89  bool isConvertibleToSDWA(const MachineInstr &MI, const SISubtarget &ST) const;
90  bool convertToSDWA(MachineInstr &MI, const SDWAOperandsVector &SDWAOperands);
91  void legalizeScalarOperands(MachineInstr &MI, const SISubtarget &ST) const;
92 
93  StringRef getPassName() const override { return "SI Peephole SDWA"; }
94 
95  void getAnalysisUsage(AnalysisUsage &AU) const override {
96  AU.setPreservesCFG();
98  }
99 };
100 
101 class SDWAOperand {
102 private:
103  MachineOperand *Target; // Operand that would be used in converted instruction
104  MachineOperand *Replaced; // Operand that would be replace by Target
105 
106 public:
107  SDWAOperand(MachineOperand *TargetOp, MachineOperand *ReplacedOp)
108  : Target(TargetOp), Replaced(ReplacedOp) {
109  assert(Target->isReg());
110  assert(Replaced->isReg());
111  }
112 
113  virtual ~SDWAOperand() = default;
114 
115  virtual MachineInstr *potentialToConvert(const SIInstrInfo *TII) = 0;
116  virtual bool convertToSDWA(MachineInstr &MI, const SIInstrInfo *TII) = 0;
117 
118  MachineOperand *getTargetOperand() const { return Target; }
119  MachineOperand *getReplacedOperand() const { return Replaced; }
120  MachineInstr *getParentInst() const { return Target->getParent(); }
121 
122  MachineRegisterInfo *getMRI() const {
123  return &getParentInst()->getParent()->getParent()->getRegInfo();
124  }
125 };
126 
127 using namespace AMDGPU::SDWA;
128 
129 class SDWASrcOperand : public SDWAOperand {
130 private:
131  SdwaSel SrcSel;
132  bool Abs;
133  bool Neg;
134  bool Sext;
135 
136 public:
137  SDWASrcOperand(MachineOperand *TargetOp, MachineOperand *ReplacedOp,
138  SdwaSel SrcSel_ = DWORD, bool Abs_ = false, bool Neg_ = false,
139  bool Sext_ = false)
140  : SDWAOperand(TargetOp, ReplacedOp), SrcSel(SrcSel_), Abs(Abs_),
141  Neg(Neg_), Sext(Sext_) {}
142 
143  MachineInstr *potentialToConvert(const SIInstrInfo *TII) override;
144  bool convertToSDWA(MachineInstr &MI, const SIInstrInfo *TII) override;
145 
146  SdwaSel getSrcSel() const { return SrcSel; }
147  bool getAbs() const { return Abs; }
148  bool getNeg() const { return Neg; }
149  bool getSext() const { return Sext; }
150 
151  uint64_t getSrcMods(const SIInstrInfo *TII,
152  const MachineOperand *SrcOp) const;
153 };
154 
155 class SDWADstOperand : public SDWAOperand {
156 private:
157  SdwaSel DstSel;
158  DstUnused DstUn;
159 
160 public:
161  SDWADstOperand(MachineOperand *TargetOp, MachineOperand *ReplacedOp,
162  SdwaSel DstSel_ = DWORD, DstUnused DstUn_ = UNUSED_PAD)
163  : SDWAOperand(TargetOp, ReplacedOp), DstSel(DstSel_), DstUn(DstUn_) {}
164 
165  MachineInstr *potentialToConvert(const SIInstrInfo *TII) override;
166  bool convertToSDWA(MachineInstr &MI, const SIInstrInfo *TII) override;
167 
168  SdwaSel getDstSel() const { return DstSel; }
169  DstUnused getDstUnused() const { return DstUn; }
170 };
171 
172 } // end anonymous namespace
173 
174 INITIALIZE_PASS(SIPeepholeSDWA, DEBUG_TYPE, "SI Peephole SDWA", false, false)
175 
176 char SIPeepholeSDWA::ID = 0;
177 
178 char &llvm::SIPeepholeSDWAID = SIPeepholeSDWA::ID;
179 
181  return new SIPeepholeSDWA();
182 }
183 
184 #ifndef NDEBUG
185 static raw_ostream& operator<<(raw_ostream &OS, const SdwaSel &Sel) {
186  switch(Sel) {
187  case BYTE_0: OS << "BYTE_0"; break;
188  case BYTE_1: OS << "BYTE_1"; break;
189  case BYTE_2: OS << "BYTE_2"; break;
190  case BYTE_3: OS << "BYTE_3"; break;
191  case WORD_0: OS << "WORD_0"; break;
192  case WORD_1: OS << "WORD_1"; break;
193  case DWORD: OS << "DWORD"; break;
194  }
195  return OS;
196 }
197 
198 static raw_ostream& operator<<(raw_ostream &OS, const DstUnused &Un) {
199  switch(Un) {
200  case UNUSED_PAD: OS << "UNUSED_PAD"; break;
201  case UNUSED_SEXT: OS << "UNUSED_SEXT"; break;
202  case UNUSED_PRESERVE: OS << "UNUSED_PRESERVE"; break;
203  }
204  return OS;
205 }
206 
207 static raw_ostream& operator<<(raw_ostream &OS, const SDWASrcOperand &Src) {
208  OS << "SDWA src: " << *Src.getTargetOperand()
209  << " src_sel:" << Src.getSrcSel()
210  << " abs:" << Src.getAbs() << " neg:" << Src.getNeg()
211  << " sext:" << Src.getSext() << '\n';
212  return OS;
213 }
214 
215 static raw_ostream& operator<<(raw_ostream &OS, const SDWADstOperand &Dst) {
216  OS << "SDWA dst: " << *Dst.getTargetOperand()
217  << " dst_sel:" << Dst.getDstSel()
218  << " dst_unused:" << Dst.getDstUnused() << '\n';
219  return OS;
220 }
221 #endif
222 
223 static void copyRegOperand(MachineOperand &To, const MachineOperand &From) {
224  assert(To.isReg() && From.isReg());
225  To.setReg(From.getReg());
226  To.setSubReg(From.getSubReg());
227  To.setIsUndef(From.isUndef());
228  if (To.isUse()) {
229  To.setIsKill(From.isKill());
230  } else {
231  To.setIsDead(From.isDead());
232  }
233 }
234 
235 static bool isSameReg(const MachineOperand &LHS, const MachineOperand &RHS) {
236  return LHS.isReg() &&
237  RHS.isReg() &&
238  LHS.getReg() == RHS.getReg() &&
239  LHS.getSubReg() == RHS.getSubReg();
240 }
241 
242 static bool isSubregOf(const MachineOperand &SubReg,
243  const MachineOperand &SuperReg,
244  const TargetRegisterInfo *TRI) {
245 
246  if (!SuperReg.isReg() || !SubReg.isReg())
247  return false;
248 
249  if (isSameReg(SuperReg, SubReg))
250  return true;
251 
252  if (SuperReg.getReg() != SubReg.getReg())
253  return false;
254 
255  LaneBitmask SuperMask = TRI->getSubRegIndexLaneMask(SuperReg.getSubReg());
256  LaneBitmask SubMask = TRI->getSubRegIndexLaneMask(SubReg.getSubReg());
257  SuperMask |= ~SubMask;
258  return SuperMask.all();
259 }
260 
261 uint64_t SDWASrcOperand::getSrcMods(const SIInstrInfo *TII,
262  const MachineOperand *SrcOp) const {
263  uint64_t Mods = 0;
264  const auto *MI = SrcOp->getParent();
265  if (TII->getNamedOperand(*MI, AMDGPU::OpName::src0) == SrcOp) {
266  if (auto *Mod = TII->getNamedOperand(*MI, AMDGPU::OpName::src0_modifiers)) {
267  Mods = Mod->getImm();
268  }
269  } else if (TII->getNamedOperand(*MI, AMDGPU::OpName::src1) == SrcOp) {
270  if (auto *Mod = TII->getNamedOperand(*MI, AMDGPU::OpName::src1_modifiers)) {
271  Mods = Mod->getImm();
272  }
273  }
274  if (Abs || Neg) {
275  assert(!Sext &&
276  "Float and integer src modifiers can't be set simulteniously");
277  Mods |= Abs ? SISrcMods::ABS : 0;
278  Mods ^= Neg ? SISrcMods::NEG : 0;
279  } else if (Sext) {
280  Mods |= SISrcMods::SEXT;
281  }
282 
283  return Mods;
284 }
285 
286 MachineInstr *SDWASrcOperand::potentialToConvert(const SIInstrInfo *TII) {
287  // For SDWA src operand potential instruction is one that use register
288  // defined by parent instruction
289  MachineRegisterInfo *MRI = getMRI();
290  MachineOperand *Replaced = getReplacedOperand();
291  assert(Replaced->isReg());
292 
293  MachineInstr *PotentialMI = nullptr;
294  for (MachineOperand &PotentialMO : MRI->use_operands(Replaced->getReg())) {
295  // If this is use of another subreg of dst reg then do nothing
296  if (!isSubregOf(*Replaced, PotentialMO, MRI->getTargetRegisterInfo()))
297  continue;
298 
299  // If there exist use of superreg of dst then we should not combine this
300  // opernad
301  if (!isSameReg(PotentialMO, *Replaced))
302  return nullptr;
303 
304  // Check that PotentialMI is only instruction that uses dst reg
305  if (PotentialMI == nullptr) {
306  PotentialMI = PotentialMO.getParent();
307  } else if (PotentialMI != PotentialMO.getParent()) {
308  return nullptr;
309  }
310  }
311 
312  return PotentialMI;
313 }
314 
315 bool SDWASrcOperand::convertToSDWA(MachineInstr &MI, const SIInstrInfo *TII) {
316  // Find operand in instruction that matches source operand and replace it with
317  // target operand. Set corresponding src_sel
318 
319  MachineOperand *Src = TII->getNamedOperand(MI, AMDGPU::OpName::src0);
320  MachineOperand *SrcSel = TII->getNamedOperand(MI, AMDGPU::OpName::src0_sel);
321  MachineOperand *SrcMods =
322  TII->getNamedOperand(MI, AMDGPU::OpName::src0_modifiers);
323  assert(Src && (Src->isReg() || Src->isImm()));
324  if (!isSameReg(*Src, *getReplacedOperand())) {
325  // If this is not src0 then it should be src1
326  Src = TII->getNamedOperand(MI, AMDGPU::OpName::src1);
327  SrcSel = TII->getNamedOperand(MI, AMDGPU::OpName::src1_sel);
328  SrcMods = TII->getNamedOperand(MI, AMDGPU::OpName::src1_modifiers);
329 
330  assert(Src && Src->isReg());
331 
332  if ((MI.getOpcode() == AMDGPU::V_MAC_F16_sdwa ||
333  MI.getOpcode() == AMDGPU::V_MAC_F32_sdwa) &&
334  !isSameReg(*Src, *getReplacedOperand())) {
335  // In case of v_mac_f16/32_sdwa this pass can try to apply src operand to
336  // src2. This is not allowed.
337  return false;
338  }
339 
340  assert(isSameReg(*Src, *getReplacedOperand()) && SrcSel && SrcMods);
341  }
342  copyRegOperand(*Src, *getTargetOperand());
343  SrcSel->setImm(getSrcSel());
344  SrcMods->setImm(getSrcMods(TII, Src));
345  getTargetOperand()->setIsKill(false);
346  return true;
347 }
348 
349 MachineInstr *SDWADstOperand::potentialToConvert(const SIInstrInfo *TII) {
350  // For SDWA dst operand potential instruction is one that defines register
351  // that this operand uses
352  MachineRegisterInfo *MRI = getMRI();
353  MachineInstr *ParentMI = getParentInst();
354  MachineOperand *Replaced = getReplacedOperand();
355  assert(Replaced->isReg());
356 
357  for (MachineOperand &PotentialMO : MRI->def_operands(Replaced->getReg())) {
358  if (!isSubregOf(*Replaced, PotentialMO, MRI->getTargetRegisterInfo()))
359  continue;
360 
361  if (!isSameReg(*Replaced, PotentialMO))
362  return nullptr;
363 
364  // Check that ParentMI is the only instruction that uses replaced register
365  for (MachineOperand &UseMO : MRI->use_operands(PotentialMO.getReg())) {
366  if (isSubregOf(UseMO, PotentialMO, MRI->getTargetRegisterInfo()) &&
367  UseMO.getParent() != ParentMI) {
368  return nullptr;
369  }
370  }
371 
372  // Due to SSA this should be onle def of replaced register, so return it
373  return PotentialMO.getParent();
374  }
375 
376  return nullptr;
377 }
378 
379 bool SDWADstOperand::convertToSDWA(MachineInstr &MI, const SIInstrInfo *TII) {
380  // Replace vdst operand in MI with target operand. Set dst_sel and dst_unused
381 
382  if ((MI.getOpcode() == AMDGPU::V_MAC_F16_sdwa ||
383  MI.getOpcode() == AMDGPU::V_MAC_F32_sdwa) &&
384  getDstSel() != AMDGPU::SDWA::DWORD) {
385  // v_mac_f16/32_sdwa allow dst_sel to be equal only to DWORD
386  return false;
387  }
388 
389  MachineOperand *Operand = TII->getNamedOperand(MI, AMDGPU::OpName::vdst);
390  assert(Operand &&
391  Operand->isReg() &&
392  isSameReg(*Operand, *getReplacedOperand()));
393  copyRegOperand(*Operand, *getTargetOperand());
394  MachineOperand *DstSel= TII->getNamedOperand(MI, AMDGPU::OpName::dst_sel);
395  assert(DstSel);
396  DstSel->setImm(getDstSel());
397  MachineOperand *DstUnused= TII->getNamedOperand(MI, AMDGPU::OpName::dst_unused);
398  assert(DstUnused);
399  DstUnused->setImm(getDstUnused());
400 
401  // Remove original instruction because it would conflict with our new
402  // instruction by register definition
403  getParentInst()->eraseFromParent();
404  return true;
405 }
406 
407 Optional<int64_t> SIPeepholeSDWA::foldToImm(const MachineOperand &Op) const {
408  if (Op.isImm()) {
409  return Op.getImm();
410  }
411 
412  // If this is not immediate then it can be copy of immediate value, e.g.:
413  // %vreg1<def> = S_MOV_B32 255;
414  if (Op.isReg()) {
415  for (const MachineOperand &Def : MRI->def_operands(Op.getReg())) {
416  if (!isSameReg(Op, Def))
417  continue;
418 
419  const MachineInstr *DefInst = Def.getParent();
420  if (!TII->isFoldableCopy(*DefInst))
421  return None;
422 
423  const MachineOperand &Copied = DefInst->getOperand(1);
424  if (!Copied.isImm())
425  return None;
426 
427  return Copied.getImm();
428  }
429  }
430 
431  return None;
432 }
433 
434 void SIPeepholeSDWA::matchSDWAOperands(MachineFunction &MF) {
435  for (MachineBasicBlock &MBB : MF) {
436  for (MachineInstr &MI : MBB) {
437  unsigned Opcode = MI.getOpcode();
438  switch (Opcode) {
439  case AMDGPU::V_LSHRREV_B32_e32:
440  case AMDGPU::V_ASHRREV_I32_e32:
441  case AMDGPU::V_LSHLREV_B32_e32:
442  case AMDGPU::V_LSHRREV_B32_e64:
443  case AMDGPU::V_ASHRREV_I32_e64:
444  case AMDGPU::V_LSHLREV_B32_e64: {
445  // from: v_lshrrev_b32_e32 v1, 16/24, v0
446  // to SDWA src:v0 src_sel:WORD_1/BYTE_3
447 
448  // from: v_ashrrev_i32_e32 v1, 16/24, v0
449  // to SDWA src:v0 src_sel:WORD_1/BYTE_3 sext:1
450 
451  // from: v_lshlrev_b32_e32 v1, 16/24, v0
452  // to SDWA dst:v1 dst_sel:WORD_1/BYTE_3 dst_unused:UNUSED_PAD
453  MachineOperand *Src0 = TII->getNamedOperand(MI, AMDGPU::OpName::src0);
454  auto Imm = foldToImm(*Src0);
455  if (!Imm)
456  break;
457 
458  if (*Imm != 16 && *Imm != 24)
459  break;
460 
461  MachineOperand *Src1 = TII->getNamedOperand(MI, AMDGPU::OpName::src1);
462  MachineOperand *Dst = TII->getNamedOperand(MI, AMDGPU::OpName::vdst);
463  if (TRI->isPhysicalRegister(Src1->getReg()) ||
464  TRI->isPhysicalRegister(Dst->getReg()))
465  break;
466 
467  if (Opcode == AMDGPU::V_LSHLREV_B32_e32 ||
468  Opcode == AMDGPU::V_LSHLREV_B32_e64) {
469  auto SDWADst = make_unique<SDWADstOperand>(
470  Dst, Src1, *Imm == 16 ? WORD_1 : BYTE_3, UNUSED_PAD);
471  DEBUG(dbgs() << "Match: " << MI << "To: " << *SDWADst << '\n');
472  SDWAOperands[&MI] = std::move(SDWADst);
473  ++NumSDWAPatternsFound;
474  } else {
475  auto SDWASrc = make_unique<SDWASrcOperand>(
476  Src1, Dst, *Imm == 16 ? WORD_1 : BYTE_3, false, false,
477  Opcode != AMDGPU::V_LSHRREV_B32_e32 &&
478  Opcode != AMDGPU::V_LSHRREV_B32_e64);
479  DEBUG(dbgs() << "Match: " << MI << "To: " << *SDWASrc << '\n');
480  SDWAOperands[&MI] = std::move(SDWASrc);
481  ++NumSDWAPatternsFound;
482  }
483  break;
484  }
485 
486  case AMDGPU::V_LSHRREV_B16_e32:
487  case AMDGPU::V_ASHRREV_I16_e32:
488  case AMDGPU::V_LSHLREV_B16_e32:
489  case AMDGPU::V_LSHRREV_B16_e64:
490  case AMDGPU::V_ASHRREV_I16_e64:
491  case AMDGPU::V_LSHLREV_B16_e64: {
492  // from: v_lshrrev_b16_e32 v1, 8, v0
493  // to SDWA src:v0 src_sel:BYTE_1
494 
495  // from: v_ashrrev_i16_e32 v1, 8, v0
496  // to SDWA src:v0 src_sel:BYTE_1 sext:1
497 
498  // from: v_lshlrev_b16_e32 v1, 8, v0
499  // to SDWA dst:v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD
500  MachineOperand *Src0 = TII->getNamedOperand(MI, AMDGPU::OpName::src0);
501  auto Imm = foldToImm(*Src0);
502  if (!Imm || *Imm != 8)
503  break;
504 
505  MachineOperand *Src1 = TII->getNamedOperand(MI, AMDGPU::OpName::src1);
506  MachineOperand *Dst = TII->getNamedOperand(MI, AMDGPU::OpName::vdst);
507 
508  if (TRI->isPhysicalRegister(Src1->getReg()) ||
509  TRI->isPhysicalRegister(Dst->getReg()))
510  break;
511 
512  if (Opcode == AMDGPU::V_LSHLREV_B16_e32 ||
513  Opcode == AMDGPU::V_LSHLREV_B16_e64) {
514  auto SDWADst =
515  make_unique<SDWADstOperand>(Dst, Src1, BYTE_1, UNUSED_PAD);
516  DEBUG(dbgs() << "Match: " << MI << "To: " << *SDWADst << '\n');
517  SDWAOperands[&MI] = std::move(SDWADst);
518  ++NumSDWAPatternsFound;
519  } else {
520  auto SDWASrc = make_unique<SDWASrcOperand>(
521  Src1, Dst, BYTE_1, false, false,
522  Opcode != AMDGPU::V_LSHRREV_B16_e32 &&
523  Opcode != AMDGPU::V_LSHRREV_B16_e64);
524  DEBUG(dbgs() << "Match: " << MI << "To: " << *SDWASrc << '\n');
525  SDWAOperands[&MI] = std::move(SDWASrc);
526  ++NumSDWAPatternsFound;
527  }
528  break;
529  }
530 
531  case AMDGPU::V_BFE_I32:
532  case AMDGPU::V_BFE_U32: {
533  // e.g.:
534  // from: v_bfe_u32 v1, v0, 8, 8
535  // to SDWA src:v0 src_sel:BYTE_1
536 
537  // offset | width | src_sel
538  // ------------------------
539  // 0 | 8 | BYTE_0
540  // 0 | 16 | WORD_0
541  // 0 | 32 | DWORD ?
542  // 8 | 8 | BYTE_1
543  // 16 | 8 | BYTE_2
544  // 16 | 16 | WORD_1
545  // 24 | 8 | BYTE_3
546 
547  MachineOperand *Src1 = TII->getNamedOperand(MI, AMDGPU::OpName::src1);
548  auto Offset = foldToImm(*Src1);
549  if (!Offset)
550  break;
551 
552  MachineOperand *Src2 = TII->getNamedOperand(MI, AMDGPU::OpName::src2);
553  auto Width = foldToImm(*Src2);
554  if (!Width)
555  break;
556 
557  SdwaSel SrcSel = DWORD;
558 
559  if (*Offset == 0 && *Width == 8)
560  SrcSel = BYTE_0;
561  else if (*Offset == 0 && *Width == 16)
562  SrcSel = WORD_0;
563  else if (*Offset == 0 && *Width == 32)
564  SrcSel = DWORD;
565  else if (*Offset == 8 && *Width == 8)
566  SrcSel = BYTE_1;
567  else if (*Offset == 16 && *Width == 8)
568  SrcSel = BYTE_2;
569  else if (*Offset == 16 && *Width == 16)
570  SrcSel = WORD_1;
571  else if (*Offset == 24 && *Width == 8)
572  SrcSel = BYTE_3;
573  else
574  break;
575 
576  MachineOperand *Src0 = TII->getNamedOperand(MI, AMDGPU::OpName::src0);
577  MachineOperand *Dst = TII->getNamedOperand(MI, AMDGPU::OpName::vdst);
578 
579  if (TRI->isPhysicalRegister(Src0->getReg()) ||
580  TRI->isPhysicalRegister(Dst->getReg()))
581  break;
582 
583  auto SDWASrc = make_unique<SDWASrcOperand>(
584  Src0, Dst, SrcSel, false, false,
585  Opcode != AMDGPU::V_BFE_U32);
586  DEBUG(dbgs() << "Match: " << MI << "To: " << *SDWASrc << '\n');
587  SDWAOperands[&MI] = std::move(SDWASrc);
588  ++NumSDWAPatternsFound;
589  break;
590  }
591  case AMDGPU::V_AND_B32_e32:
592  case AMDGPU::V_AND_B32_e64: {
593  // e.g.:
594  // from: v_and_b32_e32 v1, 0x0000ffff/0x000000ff, v0
595  // to SDWA src:v0 src_sel:WORD_0/BYTE_0
596 
597  MachineOperand *Src0 = TII->getNamedOperand(MI, AMDGPU::OpName::src0);
598  MachineOperand *Src1 = TII->getNamedOperand(MI, AMDGPU::OpName::src1);
599  auto ValSrc = Src1;
600  auto Imm = foldToImm(*Src0);
601 
602  if (!Imm) {
603  Imm = foldToImm(*Src1);
604  ValSrc = Src0;
605  }
606 
607  if (!Imm || (*Imm != 0x0000ffff && *Imm != 0x000000ff))
608  break;
609 
610  MachineOperand *Dst = TII->getNamedOperand(MI, AMDGPU::OpName::vdst);
611 
612  if (TRI->isPhysicalRegister(Src1->getReg()) ||
613  TRI->isPhysicalRegister(Dst->getReg()))
614  break;
615 
616  auto SDWASrc = make_unique<SDWASrcOperand>(
617  ValSrc, Dst, *Imm == 0x0000ffff ? WORD_0 : BYTE_0);
618  DEBUG(dbgs() << "Match: " << MI << "To: " << *SDWASrc << '\n');
619  SDWAOperands[&MI] = std::move(SDWASrc);
620  ++NumSDWAPatternsFound;
621  break;
622  }
623  }
624  }
625  }
626 }
627 
628 bool SIPeepholeSDWA::isConvertibleToSDWA(const MachineInstr &MI,
629  const SISubtarget &ST) const {
630  // Check if this instruction has opcode that supports SDWA
631  int Opc = MI.getOpcode();
632  if (AMDGPU::getSDWAOp(Opc) == -1)
633  Opc = AMDGPU::getVOPe32(Opc);
634 
635  if (Opc == -1 || AMDGPU::getSDWAOp(Opc) == -1)
636  return false;
637 
638  if (!ST.hasSDWAOmod() && TII->hasModifiersSet(MI, AMDGPU::OpName::omod))
639  return false;
640 
641  if (TII->isVOPC(Opc)) {
642  if (!ST.hasSDWASdst()) {
643  const MachineOperand *SDst = TII->getNamedOperand(MI, AMDGPU::OpName::sdst);
644  if (SDst && SDst->getReg() != AMDGPU::VCC)
645  return false;
646  }
647 
648  if (!ST.hasSDWAOutModsVOPC() &&
649  (TII->hasModifiersSet(MI, AMDGPU::OpName::clamp) ||
650  TII->hasModifiersSet(MI, AMDGPU::OpName::omod)))
651  return false;
652 
653  } else if (TII->getNamedOperand(MI, AMDGPU::OpName::sdst) ||
654  !TII->getNamedOperand(MI, AMDGPU::OpName::vdst)) {
655  return false;
656  }
657 
658  if (!ST.hasSDWAMac() && (Opc == AMDGPU::V_MAC_F16_e32 ||
659  Opc == AMDGPU::V_MAC_F32_e32))
660  return false;
661 
662  return true;
663 }
664 
665 bool SIPeepholeSDWA::convertToSDWA(MachineInstr &MI,
666  const SDWAOperandsVector &SDWAOperands) {
667  // Convert to sdwa
668  int SDWAOpcode = AMDGPU::getSDWAOp(MI.getOpcode());
669  if (SDWAOpcode == -1)
670  SDWAOpcode = AMDGPU::getSDWAOp(AMDGPU::getVOPe32(MI.getOpcode()));
671  assert(SDWAOpcode != -1);
672 
673  const MCInstrDesc &SDWADesc = TII->get(SDWAOpcode);
674 
675  // Create SDWA version of instruction MI and initialize its operands
676  MachineInstrBuilder SDWAInst =
677  BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), SDWADesc);
678 
679  // Copy dst, if it is present in original then should also be present in SDWA
680  MachineOperand *Dst = TII->getNamedOperand(MI, AMDGPU::OpName::vdst);
681  if (Dst) {
682  assert(AMDGPU::getNamedOperandIdx(SDWAOpcode, AMDGPU::OpName::vdst) != -1);
683  SDWAInst.add(*Dst);
684  } else if ((Dst = TII->getNamedOperand(MI, AMDGPU::OpName::sdst))) {
685  assert(Dst &&
686  AMDGPU::getNamedOperandIdx(SDWAOpcode, AMDGPU::OpName::sdst) != -1);
687  SDWAInst.add(*Dst);
688  } else {
689  assert(AMDGPU::getNamedOperandIdx(SDWAOpcode, AMDGPU::OpName::sdst) != -1);
690  SDWAInst.addReg(AMDGPU::VCC, RegState::Define);
691  }
692 
693  // Copy src0, initialize src0_modifiers. All sdwa instructions has src0 and
694  // src0_modifiers (except for v_nop_sdwa, but it can't get here)
695  MachineOperand *Src0 = TII->getNamedOperand(MI, AMDGPU::OpName::src0);
696  assert(
697  Src0 &&
698  AMDGPU::getNamedOperandIdx(SDWAOpcode, AMDGPU::OpName::src0) != -1 &&
699  AMDGPU::getNamedOperandIdx(SDWAOpcode, AMDGPU::OpName::src0_modifiers) != -1);
700  if (auto *Mod = TII->getNamedOperand(MI, AMDGPU::OpName::src0_modifiers))
701  SDWAInst.addImm(Mod->getImm());
702  else
703  SDWAInst.addImm(0);
704  SDWAInst.add(*Src0);
705 
706  // Copy src1 if present, initialize src1_modifiers.
707  MachineOperand *Src1 = TII->getNamedOperand(MI, AMDGPU::OpName::src1);
708  if (Src1) {
709  assert(
710  AMDGPU::getNamedOperandIdx(SDWAOpcode, AMDGPU::OpName::src1) != -1 &&
711  AMDGPU::getNamedOperandIdx(SDWAOpcode, AMDGPU::OpName::src1_modifiers) != -1);
712  if (auto *Mod = TII->getNamedOperand(MI, AMDGPU::OpName::src1_modifiers))
713  SDWAInst.addImm(Mod->getImm());
714  else
715  SDWAInst.addImm(0);
716  SDWAInst.add(*Src1);
717  }
718 
719  if (SDWAOpcode == AMDGPU::V_MAC_F16_sdwa ||
720  SDWAOpcode == AMDGPU::V_MAC_F32_sdwa) {
721  // v_mac_f16/32 has additional src2 operand tied to vdst
722  MachineOperand *Src2 = TII->getNamedOperand(MI, AMDGPU::OpName::src2);
723  assert(Src2);
724  SDWAInst.add(*Src2);
725  }
726 
727  // Copy clamp if present, initialize otherwise
728  assert(AMDGPU::getNamedOperandIdx(SDWAOpcode, AMDGPU::OpName::clamp) != -1);
729  MachineOperand *Clamp = TII->getNamedOperand(MI, AMDGPU::OpName::clamp);
730  if (Clamp) {
731  SDWAInst.add(*Clamp);
732  } else {
733  SDWAInst.addImm(0);
734  }
735 
736  // Copy omod if present, initialize otherwise if needed
737  if (AMDGPU::getNamedOperandIdx(SDWAOpcode, AMDGPU::OpName::omod) != -1) {
738  MachineOperand *OMod = TII->getNamedOperand(MI, AMDGPU::OpName::omod);
739  if (OMod) {
740  SDWAInst.add(*OMod);
741  } else {
742  SDWAInst.addImm(0);
743  }
744  }
745 
746  // Initialize dst_sel if present
747  if (AMDGPU::getNamedOperandIdx(SDWAOpcode, AMDGPU::OpName::dst_sel) != -1) {
749  }
750 
751  // Initialize dst_unused if present
752  if (AMDGPU::getNamedOperandIdx(SDWAOpcode, AMDGPU::OpName::dst_unused) != -1) {
754  }
755 
756  // Initialize src0_sel
757  assert(AMDGPU::getNamedOperandIdx(SDWAOpcode, AMDGPU::OpName::src0_sel) != -1);
759 
760 
761  // Initialize src1_sel if present
762  if (Src1) {
763  assert(AMDGPU::getNamedOperandIdx(SDWAOpcode, AMDGPU::OpName::src1_sel) != -1);
765  }
766 
767  // Apply all sdwa operand pattenrs
768  bool Converted = false;
769  for (auto &Operand : SDWAOperands) {
770  // There should be no intesection between SDWA operands and potential MIs
771  // e.g.:
772  // v_and_b32 v0, 0xff, v1 -> src:v1 sel:BYTE_0
773  // v_and_b32 v2, 0xff, v0 -> src:v0 sel:BYTE_0
774  // v_add_u32 v3, v4, v2
775  //
776  // In that example it is possible that we would fold 2nd instruction into 3rd
777  // (v_add_u32_sdwa) and then try to fold 1st instruction into 2nd (that was
778  // already destroyed). So if SDWAOperand is also a potential MI then do not
779  // apply it.
780  if (PotentialMatches.count(Operand->getParentInst()) == 0)
781  Converted |= Operand->convertToSDWA(*SDWAInst, TII);
782  }
783  if (Converted) {
784  ConvertedInstructions.push_back(SDWAInst);
785  } else {
786  SDWAInst->eraseFromParent();
787  return false;
788  }
789 
790  DEBUG(dbgs() << "Convert instruction:" << MI
791  << "Into:" << *SDWAInst << '\n');
792  ++NumSDWAInstructionsPeepholed;
793 
794  MI.eraseFromParent();
795  return true;
796 }
797 
798 // If an instruction was converted to SDWA it should not have immediates or SGPR
799 // operands (allowed one SGPR on GFX9). Copy its scalar operands into VGPRs.
800 void SIPeepholeSDWA::legalizeScalarOperands(MachineInstr &MI, const SISubtarget &ST) const {
801  const MCInstrDesc &Desc = TII->get(MI.getOpcode());
802  unsigned ConstantBusCount = 0;
803  for (MachineOperand &Op: MI.explicit_uses()) {
804  if (!Op.isImm() && !(Op.isReg() && !TRI->isVGPR(*MRI, Op.getReg())))
805  continue;
806 
807  unsigned I = MI.getOperandNo(&Op);
808  if (Desc.OpInfo[I].RegClass == -1 ||
809  !TRI->hasVGPRs(TRI->getRegClass(Desc.OpInfo[I].RegClass)))
810  continue;
811 
812  if (ST.hasSDWAScalar() && ConstantBusCount == 0 && Op.isReg() &&
813  TRI->isSGPRReg(*MRI, Op.getReg())) {
814  ++ConstantBusCount;
815  continue;
816  }
817 
818  unsigned VGPR = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
819  auto Copy = BuildMI(*MI.getParent(), MI.getIterator(), MI.getDebugLoc(),
820  TII->get(AMDGPU::V_MOV_B32_e32), VGPR);
821  if (Op.isImm())
822  Copy.addImm(Op.getImm());
823  else if (Op.isReg())
824  Copy.addReg(Op.getReg(), Op.isKill() ? RegState::Kill : 0,
825  Op.getSubReg());
826  Op.ChangeToRegister(VGPR, false);
827  }
828 }
829 
830 bool SIPeepholeSDWA::runOnMachineFunction(MachineFunction &MF) {
831  const SISubtarget &ST = MF.getSubtarget<SISubtarget>();
832 
833  if (!ST.hasSDWA() || skipFunction(*MF.getFunction()))
834  return false;
835 
836  MRI = &MF.getRegInfo();
837  TRI = ST.getRegisterInfo();
838  TII = ST.getInstrInfo();
839 
840  // Find all SDWA operands in MF.
841  matchSDWAOperands(MF);
842 
843  for (const auto &OperandPair : SDWAOperands) {
844  const auto &Operand = OperandPair.second;
845  MachineInstr *PotentialMI = Operand->potentialToConvert(TII);
846  if (PotentialMI && isConvertibleToSDWA(*PotentialMI, ST)) {
847  PotentialMatches[PotentialMI].push_back(Operand.get());
848  }
849  }
850 
851  for (auto &PotentialPair : PotentialMatches) {
852  MachineInstr &PotentialMI = *PotentialPair.first;
853  convertToSDWA(PotentialMI, PotentialPair.second);
854  }
855 
856  PotentialMatches.clear();
857  SDWAOperands.clear();
858 
859  bool Ret = !ConvertedInstructions.empty();
860  while (!ConvertedInstructions.empty())
861  legalizeScalarOperands(*ConvertedInstructions.pop_back_val(), ST);
862 
863  return Ret;
864 }
const MachineInstrBuilder & add(const MachineOperand &MO) const
Interface definition for SIRegisterInfo.
A common definition of LaneBitmask for use in TableGen and CodeGen.
MachineInstr * getParent()
getParent - Return the instruction that this operand belongs to.
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
AMDGPU specific subclass of TargetSubtarget.
FunctionPass * createSIPeepholeSDWAPass()
Compute iterated dominance frontiers using a linear time algorithm.
Definition: AllocatorList.h:24
void ChangeToRegister(unsigned Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isDebug=false)
ChangeToRegister - Replace this operand with a new register operand of the specified value...
unsigned createVirtualRegister(const TargetRegisterClass *RegClass)
createVirtualRegister - Create and return a new virtual register in the function with the specified r...
char & SIPeepholeSDWAID
const DebugLoc & getDebugLoc() const
Returns the debug location id of this MachineInstr.
Definition: MachineInstr.h:268
bool hasModifiersSet(const MachineInstr &MI, unsigned OpName) const
Describe properties that are true of each instruction in the target description file.
Definition: MCInstrDesc.h:163
unsigned getReg() const
getReg - Returns the register number.
unsigned getOperandNo(const_mop_iterator I) const
Returns the number of the operand iterator I points to.
Definition: MachineInstr.h:384
void setIsUndef(bool Val=true)
const SIInstrInfo * getInstrInfo() const override
unsigned getSubReg() const
static bool isSameReg(const MachineOperand &LHS, const MachineOperand &RHS)
STATISTIC(NumFunctions, "Total number of functions")
void setIsDead(bool Val=true)
bool isImm() const
isImm - Tests if this is a MO_Immediate operand.
LLVM_READONLY int16_t getNamedOperandIdx(uint16_t Opcode, uint16_t NamedIdx)
#define DEBUG_TYPE
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
const HexagonInstrInfo * TII
bool isSGPRReg(const MachineRegisterInfo &MRI, unsigned Reg) const
void eraseFromParent()
Unlink &#39;this&#39; from the containing basic block and delete it.
unsigned SubReg
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
Definition: MachineInstr.h:290
bool hasSDWAOutModsVOPC() const
bool isVGPR(const MachineRegisterInfo &MRI, unsigned Reg) const
LLVM_READONLY MachineOperand * getNamedOperand(MachineInstr &MI, unsigned OperandName) const
Returns the operand named Op.
iterator_range< def_iterator > def_operands(unsigned Reg) const
LLVM_READONLY int getSDWAOp(uint16_t Opcode)
MachineInstrBuilder BuildMI(MachineFunction &MF, const DebugLoc &DL, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
const TargetRegisterInfo * getTargetRegisterInfo() const
unsigned const MachineRegisterInfo * MRI
bool isFoldableCopy(const MachineInstr &MI) const
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
static void copyRegOperand(MachineOperand &To, const MachineOperand &From)
bool hasVGPRs(const TargetRegisterClass *RC) const
void initializeSIPeepholeSDWAPass(PassRegistry &)
Represent the analysis usage information of a pass.
void setImm(int64_t immVal)
FunctionPass class - This class is used to implement most global optimizations.
Definition: Pass.h:285
self_iterator getIterator()
Definition: ilist_node.h:82
constexpr bool all() const
Definition: LaneBitmask.h:54
static bool isSubregOf(const MachineOperand &SubReg, const MachineOperand &SuperReg, const TargetRegisterInfo *TRI)
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
iterator_range< mop_iterator > explicit_uses()
Definition: MachineInstr.h:374
void setIsKill(bool Val=true)
#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:34
LLVM_READONLY int getVOPe32(uint16_t Opcode)
const SIRegisterInfo * getRegisterInfo() const override
iterator_range< use_iterator > use_operands(unsigned Reg) const
MachineOperand class - Representation of each machine instruction operand.
This is a &#39;vector&#39; (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:864
LLVM_NODISCARD T pop_back_val()
Definition: SmallVector.h:385
LaneBitmask getSubRegIndexLaneMask(unsigned SubIdx) const
Return a bitmask representing the parts of a register that are covered by SubIdx. ...
void setPreservesCFG()
This function should be called by the pass, iff they do not:
Definition: Pass.cpp:285
int64_t getImm() const
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:132
Target - Wrapper for Target specific information.
const MachineBasicBlock * getParent() const
Definition: MachineInstr.h:139
MachineRegisterInfo - Keep track of information for virtual and physical registers, including vreg register classes, use/def chains for registers, etc.
Representation of each machine instruction.
Definition: MachineInstr.h:59
Interface definition for SIInstrInfo.
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
LLVM_NODISCARD bool empty() const
Definition: SmallVector.h:61
static bool isVOPC(const MachineInstr &MI)
Definition: SIInstrInfo.h:387
int16_t RegClass
This specifies the register class enumeration of the operand if the operand is a register.
Definition: MCInstrDesc.h:76
void setReg(unsigned Reg)
Change the register this operand corresponds to.
#define I(x, y, z)
Definition: MD5.cpp:58
void setSubReg(unsigned subReg)
const MachineInstrBuilder & addReg(unsigned RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
raw_ostream & operator<<(raw_ostream &OS, const APInt &I)
Definition: APInt.h:2018
bool isReg() const
isReg - Tests if this is a MO_Register operand.
const Function * getFunction() const
getFunction - Return the LLVM function that this machine code represents
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
const MCOperandInfo * OpInfo
Definition: MCInstrDesc.h:174
This class implements an extremely fast bulk output stream that can only output to a stream...
Definition: raw_ostream.h:44
#define DEBUG(X)
Definition: Debug.h:118
IRTranslator LLVM IR MI
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:49
bool hasSDWAScalar() const
const MachineOperand & getOperand(unsigned i) const
Definition: MachineInstr.h:295