LLVM  10.0.0svn
SIFixSGPRCopies.cpp
Go to the documentation of this file.
1 //===- SIFixSGPRCopies.cpp - Remove potential VGPR => SGPR copies ---------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 /// \file
10 /// Copies from VGPR to SGPR registers are illegal and the register coalescer
11 /// will sometimes generate these illegal copies in situations like this:
12 ///
13 /// Register Class <vsrc> is the union of <vgpr> and <sgpr>
14 ///
15 /// BB0:
16 /// %0 <sgpr> = SCALAR_INST
17 /// %1 <vsrc> = COPY %0 <sgpr>
18 /// ...
19 /// BRANCH %cond BB1, BB2
20 /// BB1:
21 /// %2 <vgpr> = VECTOR_INST
22 /// %3 <vsrc> = COPY %2 <vgpr>
23 /// BB2:
24 /// %4 <vsrc> = PHI %1 <vsrc>, <%bb.0>, %3 <vrsc>, <%bb.1>
25 /// %5 <vgpr> = VECTOR_INST %4 <vsrc>
26 ///
27 ///
28 /// The coalescer will begin at BB0 and eliminate its copy, then the resulting
29 /// code will look like this:
30 ///
31 /// BB0:
32 /// %0 <sgpr> = SCALAR_INST
33 /// ...
34 /// BRANCH %cond BB1, BB2
35 /// BB1:
36 /// %2 <vgpr> = VECTOR_INST
37 /// %3 <vsrc> = COPY %2 <vgpr>
38 /// BB2:
39 /// %4 <sgpr> = PHI %0 <sgpr>, <%bb.0>, %3 <vsrc>, <%bb.1>
40 /// %5 <vgpr> = VECTOR_INST %4 <sgpr>
41 ///
42 /// Now that the result of the PHI instruction is an SGPR, the register
43 /// allocator is now forced to constrain the register class of %3 to
44 /// <sgpr> so we end up with final code like this:
45 ///
46 /// BB0:
47 /// %0 <sgpr> = SCALAR_INST
48 /// ...
49 /// BRANCH %cond BB1, BB2
50 /// BB1:
51 /// %2 <vgpr> = VECTOR_INST
52 /// %3 <sgpr> = COPY %2 <vgpr>
53 /// BB2:
54 /// %4 <sgpr> = PHI %0 <sgpr>, <%bb.0>, %3 <sgpr>, <%bb.1>
55 /// %5 <vgpr> = VECTOR_INST %4 <sgpr>
56 ///
57 /// Now this code contains an illegal copy from a VGPR to an SGPR.
58 ///
59 /// In order to avoid this problem, this pass searches for PHI instructions
60 /// which define a <vsrc> register and constrains its definition class to
61 /// <vgpr> if the user of the PHI's definition register is a vector instruction.
62 /// If the PHI's definition class is constrained to <vgpr> then the coalescer
63 /// will be unable to perform the COPY removal from the above example which
64 /// ultimately led to the creation of an illegal COPY.
65 //===----------------------------------------------------------------------===//
66 
67 #include "AMDGPU.h"
68 #include "AMDGPUSubtarget.h"
69 #include "SIInstrInfo.h"
70 #include "SIRegisterInfo.h"
72 #include "llvm/ADT/DenseSet.h"
73 #include "llvm/ADT/STLExtras.h"
74 #include "llvm/ADT/SmallSet.h"
75 #include "llvm/ADT/SmallVector.h"
85 #include "llvm/Pass.h"
86 #include "llvm/Support/CodeGen.h"
88 #include "llvm/Support/Debug.h"
91 #include <cassert>
92 #include <cstdint>
93 #include <iterator>
94 #include <list>
95 #include <map>
96 #include <tuple>
97 #include <utility>
98 
99 using namespace llvm;
100 
101 #define DEBUG_TYPE "si-fix-sgpr-copies"
102 
104  "amdgpu-enable-merge-m0",
105  cl::desc("Merge and hoist M0 initializations"),
106  cl::init(true));
107 
108 namespace {
109 
110 class SIFixSGPRCopies : public MachineFunctionPass {
112 
113 public:
114  static char ID;
115 
116  SIFixSGPRCopies() : MachineFunctionPass(ID) {}
117 
118  bool runOnMachineFunction(MachineFunction &MF) override;
119 
120  StringRef getPassName() const override { return "SI Fix SGPR copies"; }
121 
122  void getAnalysisUsage(AnalysisUsage &AU) const override {
125  AU.setPreservesCFG();
127  }
128 };
129 
130 } // end anonymous namespace
131 
132 INITIALIZE_PASS_BEGIN(SIFixSGPRCopies, DEBUG_TYPE,
133  "SI Fix SGPR copies", false, false)
136  "SI Fix SGPR copies", false, false)
137 
138 char SIFixSGPRCopies::ID = 0;
139 
140 char &llvm::SIFixSGPRCopiesID = SIFixSGPRCopies::ID;
141 
143  return new SIFixSGPRCopies();
144 }
145 
146 static bool hasVectorOperands(const MachineInstr &MI,
147  const SIRegisterInfo *TRI) {
149  for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
150  if (!MI.getOperand(i).isReg() ||
152  continue;
153 
154  if (TRI->hasVectorRegisters(MRI.getRegClass(MI.getOperand(i).getReg())))
155  return true;
156  }
157  return false;
158 }
159 
160 static std::pair<const TargetRegisterClass *, const TargetRegisterClass *>
162  const SIRegisterInfo &TRI,
163  const MachineRegisterInfo &MRI) {
164  Register DstReg = Copy.getOperand(0).getReg();
165  Register SrcReg = Copy.getOperand(1).getReg();
166 
167  const TargetRegisterClass *SrcRC = Register::isVirtualRegister(SrcReg)
168  ? MRI.getRegClass(SrcReg)
169  : TRI.getPhysRegClass(SrcReg);
170 
171  // We don't really care about the subregister here.
172  // SrcRC = TRI.getSubRegClass(SrcRC, Copy.getOperand(1).getSubReg());
173 
174  const TargetRegisterClass *DstRC = Register::isVirtualRegister(DstReg)
175  ? MRI.getRegClass(DstReg)
176  : TRI.getPhysRegClass(DstReg);
177 
178  return std::make_pair(SrcRC, DstRC);
179 }
180 
181 static bool isVGPRToSGPRCopy(const TargetRegisterClass *SrcRC,
182  const TargetRegisterClass *DstRC,
183  const SIRegisterInfo &TRI) {
184  return SrcRC != &AMDGPU::VReg_1RegClass && TRI.isSGPRClass(DstRC) &&
185  TRI.hasVectorRegisters(SrcRC);
186 }
187 
188 static bool isSGPRToVGPRCopy(const TargetRegisterClass *SrcRC,
189  const TargetRegisterClass *DstRC,
190  const SIRegisterInfo &TRI) {
191  return DstRC != &AMDGPU::VReg_1RegClass && TRI.isSGPRClass(SrcRC) &&
192  TRI.hasVectorRegisters(DstRC);
193 }
194 
196  const SIRegisterInfo *TRI,
197  const SIInstrInfo *TII) {
199  auto &Src = MI.getOperand(1);
200  Register DstReg = MI.getOperand(0).getReg();
201  Register SrcReg = Src.getReg();
202  if (!Register::isVirtualRegister(SrcReg) ||
204  return false;
205 
206  for (const auto &MO : MRI.reg_nodbg_operands(DstReg)) {
207  const auto *UseMI = MO.getParent();
208  if (UseMI == &MI)
209  continue;
210  if (MO.isDef() || UseMI->getParent() != MI.getParent() ||
211  UseMI->getOpcode() <= TargetOpcode::GENERIC_OP_END ||
212  !TII->isOperandLegal(*UseMI, UseMI->getOperandNo(&MO), &Src))
213  return false;
214  }
215  // Change VGPR to SGPR destination.
216  MRI.setRegClass(DstReg, TRI->getEquivalentSGPRClass(MRI.getRegClass(DstReg)));
217  return true;
218 }
219 
220 // Distribute an SGPR->VGPR copy of a REG_SEQUENCE into a VGPR REG_SEQUENCE.
221 //
222 // SGPRx = ...
223 // SGPRy = REG_SEQUENCE SGPRx, sub0 ...
224 // VGPRz = COPY SGPRy
225 //
226 // ==>
227 //
228 // VGPRx = COPY SGPRx
229 // VGPRz = REG_SEQUENCE VGPRx, sub0
230 //
231 // This exposes immediate folding opportunities when materializing 64-bit
232 // immediates.
234  const SIRegisterInfo *TRI,
235  const SIInstrInfo *TII,
237  assert(MI.isRegSequence());
238 
239  Register DstReg = MI.getOperand(0).getReg();
240  if (!TRI->isSGPRClass(MRI.getRegClass(DstReg)))
241  return false;
242 
243  if (!MRI.hasOneUse(DstReg))
244  return false;
245 
246  MachineInstr &CopyUse = *MRI.use_instr_begin(DstReg);
247  if (!CopyUse.isCopy())
248  return false;
249 
250  // It is illegal to have vreg inputs to a physreg defining reg_sequence.
252  return false;
253 
254  const TargetRegisterClass *SrcRC, *DstRC;
255  std::tie(SrcRC, DstRC) = getCopyRegClasses(CopyUse, *TRI, MRI);
256 
257  if (!isSGPRToVGPRCopy(SrcRC, DstRC, *TRI))
258  return false;
259 
260  if (tryChangeVGPRtoSGPRinCopy(CopyUse, TRI, TII))
261  return true;
262 
263  // TODO: Could have multiple extracts?
264  unsigned SubReg = CopyUse.getOperand(1).getSubReg();
265  if (SubReg != AMDGPU::NoSubRegister)
266  return false;
267 
268  MRI.setRegClass(DstReg, DstRC);
269 
270  // SGPRx = ...
271  // SGPRy = REG_SEQUENCE SGPRx, sub0 ...
272  // VGPRz = COPY SGPRy
273 
274  // =>
275  // VGPRx = COPY SGPRx
276  // VGPRz = REG_SEQUENCE VGPRx, sub0
277 
278  MI.getOperand(0).setReg(CopyUse.getOperand(0).getReg());
279  bool IsAGPR = TRI->hasAGPRs(DstRC);
280 
281  for (unsigned I = 1, N = MI.getNumOperands(); I != N; I += 2) {
282  Register SrcReg = MI.getOperand(I).getReg();
283  unsigned SrcSubReg = MI.getOperand(I).getSubReg();
284 
285  const TargetRegisterClass *SrcRC = MRI.getRegClass(SrcReg);
286  assert(TRI->isSGPRClass(SrcRC) &&
287  "Expected SGPR REG_SEQUENCE to only have SGPR inputs");
288 
289  SrcRC = TRI->getSubRegClass(SrcRC, SrcSubReg);
290  const TargetRegisterClass *NewSrcRC = TRI->getEquivalentVGPRClass(SrcRC);
291 
292  Register TmpReg = MRI.createVirtualRegister(NewSrcRC);
293 
294  BuildMI(*MI.getParent(), &MI, MI.getDebugLoc(), TII->get(AMDGPU::COPY),
295  TmpReg)
296  .add(MI.getOperand(I));
297 
298  if (IsAGPR) {
299  const TargetRegisterClass *NewSrcRC = TRI->getEquivalentAGPRClass(SrcRC);
300  Register TmpAReg = MRI.createVirtualRegister(NewSrcRC);
301  unsigned Opc = NewSrcRC == &AMDGPU::AGPR_32RegClass ?
302  AMDGPU::V_ACCVGPR_WRITE_B32 : AMDGPU::COPY;
303  BuildMI(*MI.getParent(), &MI, MI.getDebugLoc(), TII->get(Opc),
304  TmpAReg)
305  .addReg(TmpReg, RegState::Kill);
306  TmpReg = TmpAReg;
307  }
308 
309  MI.getOperand(I).setReg(TmpReg);
310  }
311 
312  CopyUse.eraseFromParent();
313  return true;
314 }
315 
316 static bool phiHasVGPROperands(const MachineInstr &PHI,
317  const MachineRegisterInfo &MRI,
318  const SIRegisterInfo *TRI,
319  const SIInstrInfo *TII) {
320  for (unsigned i = 1; i < PHI.getNumOperands(); i += 2) {
321  Register Reg = PHI.getOperand(i).getReg();
322  if (TRI->hasVGPRs(MRI.getRegClass(Reg)))
323  return true;
324  }
325  return false;
326 }
327 
328 static bool phiHasBreakDef(const MachineInstr &PHI,
329  const MachineRegisterInfo &MRI,
330  SmallSet<unsigned, 8> &Visited) {
331  for (unsigned i = 1; i < PHI.getNumOperands(); i += 2) {
332  Register Reg = PHI.getOperand(i).getReg();
333  if (Visited.count(Reg))
334  continue;
335 
336  Visited.insert(Reg);
337 
338  MachineInstr *DefInstr = MRI.getVRegDef(Reg);
339  switch (DefInstr->getOpcode()) {
340  default:
341  break;
342  case AMDGPU::SI_IF_BREAK:
343  return true;
344  case AMDGPU::PHI:
345  if (phiHasBreakDef(*DefInstr, MRI, Visited))
346  return true;
347  }
348  }
349  return false;
350 }
351 
353  const TargetRegisterInfo &TRI) {
355  E = MBB.end(); I != E; ++I) {
356  if (I->modifiesRegister(AMDGPU::EXEC, &TRI))
357  return true;
358  }
359  return false;
360 }
361 
362 static bool isSafeToFoldImmIntoCopy(const MachineInstr *Copy,
363  const MachineInstr *MoveImm,
364  const SIInstrInfo *TII,
365  unsigned &SMovOp,
366  int64_t &Imm) {
367  if (Copy->getOpcode() != AMDGPU::COPY)
368  return false;
369 
370  if (!MoveImm->isMoveImmediate())
371  return false;
372 
373  const MachineOperand *ImmOp =
374  TII->getNamedOperand(*MoveImm, AMDGPU::OpName::src0);
375  if (!ImmOp->isImm())
376  return false;
377 
378  // FIXME: Handle copies with sub-regs.
379  if (Copy->getOperand(0).getSubReg())
380  return false;
381 
382  switch (MoveImm->getOpcode()) {
383  default:
384  return false;
385  case AMDGPU::V_MOV_B32_e32:
386  SMovOp = AMDGPU::S_MOV_B32;
387  break;
388  case AMDGPU::V_MOV_B64_PSEUDO:
389  SMovOp = AMDGPU::S_MOV_B64;
390  break;
391  }
392  Imm = ImmOp->getImm();
393  return true;
394 }
395 
396 template <class UnaryPredicate>
398  const MachineBasicBlock *CutOff,
399  UnaryPredicate Predicate) {
400  if (MBB == CutOff)
401  return false;
402 
405  MBB->pred_end());
406 
407  while (!Worklist.empty()) {
408  MachineBasicBlock *MBB = Worklist.pop_back_val();
409 
410  if (!Visited.insert(MBB).second)
411  continue;
412  if (MBB == CutOff)
413  continue;
414  if (Predicate(MBB))
415  return true;
416 
417  Worklist.append(MBB->pred_begin(), MBB->pred_end());
418  }
419 
420  return false;
421 }
422 
424  const TargetRegisterInfo *TRI) {
425  return searchPredecessors(MBB, nullptr, [TRI](MachineBasicBlock *MBB) {
426  return hasTerminatorThatModifiesExec(*MBB, *TRI); });
427 }
428 
429 // Checks if there is potential path From instruction To instruction.
430 // If CutOff is specified and it sits in between of that path we ignore
431 // a higher portion of the path and report it is not reachable.
432 static bool isReachable(const MachineInstr *From,
433  const MachineInstr *To,
434  const MachineBasicBlock *CutOff,
435  MachineDominatorTree &MDT) {
436  // If either From block dominates To block or instructions are in the same
437  // block and From is higher.
438  if (MDT.dominates(From, To))
439  return true;
440 
441  const MachineBasicBlock *MBBFrom = From->getParent();
442  const MachineBasicBlock *MBBTo = To->getParent();
443  if (MBBFrom == MBBTo)
444  return false;
445 
446  // Instructions are in different blocks, do predecessor search.
447  // We should almost never get here since we do not usually produce M0 stores
448  // other than -1.
449  return searchPredecessors(MBBTo, CutOff, [MBBFrom]
450  (const MachineBasicBlock *MBB) { return MBB == MBBFrom; });
451 }
452 
453 // Return the first non-prologue instruction in the block.
457  while (I != MBB->end() && TII->isBasicBlockPrologue(*I))
458  ++I;
459 
460  return I;
461 }
462 
463 // Hoist and merge identical SGPR initializations into a common predecessor.
464 // This is intended to combine M0 initializations, but can work with any
465 // SGPR. A VGPR cannot be processed since we cannot guarantee vector
466 // executioon.
467 static bool hoistAndMergeSGPRInits(unsigned Reg,
468  const MachineRegisterInfo &MRI,
470  const TargetInstrInfo *TII) {
471  // List of inits by immediate value.
472  using InitListMap = std::map<unsigned, std::list<MachineInstr *>>;
473  InitListMap Inits;
474  // List of clobbering instructions.
476  // List of instructions marked for deletion.
477  SmallSet<MachineInstr*, 8> MergedInstrs;
478 
479  bool Changed = false;
480 
481  for (auto &MI : MRI.def_instructions(Reg)) {
482  MachineOperand *Imm = nullptr;
483  for (auto &MO: MI.operands()) {
484  if ((MO.isReg() && ((MO.isDef() && MO.getReg() != Reg) || !MO.isDef())) ||
485  (!MO.isImm() && !MO.isReg()) || (MO.isImm() && Imm)) {
486  Imm = nullptr;
487  break;
488  } else if (MO.isImm())
489  Imm = &MO;
490  }
491  if (Imm)
492  Inits[Imm->getImm()].push_front(&MI);
493  else
494  Clobbers.push_back(&MI);
495  }
496 
497  for (auto &Init : Inits) {
498  auto &Defs = Init.second;
499 
500  for (auto I1 = Defs.begin(), E = Defs.end(); I1 != E; ) {
501  MachineInstr *MI1 = *I1;
502 
503  for (auto I2 = std::next(I1); I2 != E; ) {
504  MachineInstr *MI2 = *I2;
505 
506  // Check any possible interference
507  auto interferes = [&](MachineBasicBlock::iterator From,
508  MachineBasicBlock::iterator To) -> bool {
509 
510  assert(MDT.dominates(&*To, &*From));
511 
512  auto interferes = [&MDT, From, To](MachineInstr* &Clobber) -> bool {
513  const MachineBasicBlock *MBBFrom = From->getParent();
514  const MachineBasicBlock *MBBTo = To->getParent();
515  bool MayClobberFrom = isReachable(Clobber, &*From, MBBTo, MDT);
516  bool MayClobberTo = isReachable(Clobber, &*To, MBBTo, MDT);
517  if (!MayClobberFrom && !MayClobberTo)
518  return false;
519  if ((MayClobberFrom && !MayClobberTo) ||
520  (!MayClobberFrom && MayClobberTo))
521  return true;
522  // Both can clobber, this is not an interference only if both are
523  // dominated by Clobber and belong to the same block or if Clobber
524  // properly dominates To, given that To >> From, so it dominates
525  // both and located in a common dominator.
526  return !((MBBFrom == MBBTo &&
527  MDT.dominates(Clobber, &*From) &&
528  MDT.dominates(Clobber, &*To)) ||
529  MDT.properlyDominates(Clobber->getParent(), MBBTo));
530  };
531 
532  return (llvm::any_of(Clobbers, interferes)) ||
533  (llvm::any_of(Inits, [&](InitListMap::value_type &C) {
534  return C.first != Init.first &&
535  llvm::any_of(C.second, interferes);
536  }));
537  };
538 
539  if (MDT.dominates(MI1, MI2)) {
540  if (!interferes(MI2, MI1)) {
541  LLVM_DEBUG(dbgs()
542  << "Erasing from "
543  << printMBBReference(*MI2->getParent()) << " " << *MI2);
544  MergedInstrs.insert(MI2);
545  Changed = true;
546  ++I2;
547  continue;
548  }
549  } else if (MDT.dominates(MI2, MI1)) {
550  if (!interferes(MI1, MI2)) {
551  LLVM_DEBUG(dbgs()
552  << "Erasing from "
553  << printMBBReference(*MI1->getParent()) << " " << *MI1);
554  MergedInstrs.insert(MI1);
555  Changed = true;
556  ++I1;
557  break;
558  }
559  } else {
560  auto *MBB = MDT.findNearestCommonDominator(MI1->getParent(),
561  MI2->getParent());
562  if (!MBB) {
563  ++I2;
564  continue;
565  }
566 
568  if (!interferes(MI1, I) && !interferes(MI2, I)) {
569  LLVM_DEBUG(dbgs()
570  << "Erasing from "
571  << printMBBReference(*MI1->getParent()) << " " << *MI1
572  << "and moving from "
573  << printMBBReference(*MI2->getParent()) << " to "
574  << printMBBReference(*I->getParent()) << " " << *MI2);
575  I->getParent()->splice(I, MI2->getParent(), MI2);
576  MergedInstrs.insert(MI1);
577  Changed = true;
578  ++I1;
579  break;
580  }
581  }
582  ++I2;
583  }
584  ++I1;
585  }
586  }
587 
588  for (auto MI : MergedInstrs)
589  MI->removeFromParent();
590 
591  if (Changed)
592  MRI.clearKillFlags(Reg);
593 
594  return Changed;
595 }
596 
597 bool SIFixSGPRCopies::runOnMachineFunction(MachineFunction &MF) {
598  const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
600  const SIRegisterInfo *TRI = ST.getRegisterInfo();
601  const SIInstrInfo *TII = ST.getInstrInfo();
602  MDT = &getAnalysis<MachineDominatorTree>();
603 
605 
606  for (MachineFunction::iterator BI = MF.begin(), BE = MF.end();
607  BI != BE; ++BI) {
608  MachineBasicBlock &MBB = *BI;
609  for (MachineBasicBlock::iterator I = MBB.begin(), E = MBB.end();
610  I != E; ++I) {
611  MachineInstr &MI = *I;
612 
613  switch (MI.getOpcode()) {
614  default:
615  continue;
616  case AMDGPU::COPY:
617  case AMDGPU::WQM:
618  case AMDGPU::SOFT_WQM:
619  case AMDGPU::WWM: {
620  Register DstReg = MI.getOperand(0).getReg();
621 
622  const TargetRegisterClass *SrcRC, *DstRC;
623  std::tie(SrcRC, DstRC) = getCopyRegClasses(MI, *TRI, MRI);
624 
625  if (!Register::isVirtualRegister(DstReg)) {
626  // If the destination register is a physical register there isn't
627  // really much we can do to fix this.
628  // Some special instructions use M0 as an input. Some even only use
629  // the first lane. Insert a readfirstlane and hope for the best.
630  if (DstReg == AMDGPU::M0 && TRI->hasVectorRegisters(SrcRC)) {
631  Register TmpReg
632  = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
633 
634  BuildMI(MBB, MI, MI.getDebugLoc(),
635  TII->get(AMDGPU::V_READFIRSTLANE_B32), TmpReg)
636  .add(MI.getOperand(1));
637  MI.getOperand(1).setReg(TmpReg);
638  }
639 
640  continue;
641  }
642 
643  if (isVGPRToSGPRCopy(SrcRC, DstRC, *TRI)) {
644  Register SrcReg = MI.getOperand(1).getReg();
645  if (!Register::isVirtualRegister(SrcReg)) {
646  TII->moveToVALU(MI, MDT);
647  break;
648  }
649 
650  MachineInstr *DefMI = MRI.getVRegDef(SrcReg);
651  unsigned SMovOp;
652  int64_t Imm;
653  // If we are just copying an immediate, we can replace the copy with
654  // s_mov_b32.
655  if (isSafeToFoldImmIntoCopy(&MI, DefMI, TII, SMovOp, Imm)) {
656  MI.getOperand(1).ChangeToImmediate(Imm);
658  MI.setDesc(TII->get(SMovOp));
659  break;
660  }
661  TII->moveToVALU(MI, MDT);
662  } else if (isSGPRToVGPRCopy(SrcRC, DstRC, *TRI)) {
663  tryChangeVGPRtoSGPRinCopy(MI, TRI, TII);
664  }
665 
666  break;
667  }
668  case AMDGPU::PHI: {
669  Register Reg = MI.getOperand(0).getReg();
670  if (!TRI->isSGPRClass(MRI.getRegClass(Reg)))
671  break;
672 
673  // We don't need to fix the PHI if the common dominator of the
674  // two incoming blocks terminates with a uniform branch.
675  bool HasVGPROperand = phiHasVGPROperands(MI, MRI, TRI, TII);
676  if (MI.getNumExplicitOperands() == 5 && !HasVGPROperand) {
677  MachineBasicBlock *MBB0 = MI.getOperand(2).getMBB();
678  MachineBasicBlock *MBB1 = MI.getOperand(4).getMBB();
679 
680  if (!predsHasDivergentTerminator(MBB0, TRI) &&
681  !predsHasDivergentTerminator(MBB1, TRI)) {
682  LLVM_DEBUG(dbgs()
683  << "Not fixing PHI for uniform branch: " << MI << '\n');
684  break;
685  }
686  }
687 
688  // If a PHI node defines an SGPR and any of its operands are VGPRs,
689  // then we need to move it to the VALU.
690  //
691  // Also, if a PHI node defines an SGPR and has all SGPR operands
692  // we must move it to the VALU, because the SGPR operands will
693  // all end up being assigned the same register, which means
694  // there is a potential for a conflict if different threads take
695  // different control flow paths.
696  //
697  // For Example:
698  //
699  // sgpr0 = def;
700  // ...
701  // sgpr1 = def;
702  // ...
703  // sgpr2 = PHI sgpr0, sgpr1
704  // use sgpr2;
705  //
706  // Will Become:
707  //
708  // sgpr2 = def;
709  // ...
710  // sgpr2 = def;
711  // ...
712  // use sgpr2
713  //
714  // The one exception to this rule is when one of the operands
715  // is defined by a SI_BREAK, SI_IF_BREAK, or SI_ELSE_BREAK
716  // instruction. In this case, there we know the program will
717  // never enter the second block (the loop) without entering
718  // the first block (where the condition is computed), so there
719  // is no chance for values to be over-written.
720 
721  SmallSet<unsigned, 8> Visited;
722  if (HasVGPROperand || !phiHasBreakDef(MI, MRI, Visited)) {
723  LLVM_DEBUG(dbgs() << "Fixing PHI: " << MI);
724  TII->moveToVALU(MI, MDT);
725  }
726 
727  break;
728  }
729  case AMDGPU::REG_SEQUENCE:
730  if (TRI->hasVectorRegisters(TII->getOpRegClass(MI, 0)) ||
731  !hasVectorOperands(MI, TRI)) {
732  foldVGPRCopyIntoRegSequence(MI, TRI, TII, MRI);
733  continue;
734  }
735 
736  LLVM_DEBUG(dbgs() << "Fixing REG_SEQUENCE: " << MI);
737 
738  TII->moveToVALU(MI, MDT);
739  break;
740  case AMDGPU::INSERT_SUBREG: {
741  const TargetRegisterClass *DstRC, *Src0RC, *Src1RC;
742  DstRC = MRI.getRegClass(MI.getOperand(0).getReg());
743  Src0RC = MRI.getRegClass(MI.getOperand(1).getReg());
744  Src1RC = MRI.getRegClass(MI.getOperand(2).getReg());
745  if (TRI->isSGPRClass(DstRC) &&
746  (TRI->hasVectorRegisters(Src0RC) ||
747  TRI->hasVectorRegisters(Src1RC))) {
748  LLVM_DEBUG(dbgs() << " Fixing INSERT_SUBREG: " << MI);
749  TII->moveToVALU(MI, MDT);
750  }
751  break;
752  }
753  }
754  }
755  }
756 
758  hoistAndMergeSGPRInits(AMDGPU::M0, MRI, *MDT, TII);
759 
760  return true;
761 }
uint64_t CallInst * C
Interface definition for SIRegisterInfo.
AnalysisUsage & addPreserved()
Add the specified Pass class to the set of analyses preserved by this pass.
AMDGPU specific subclass of TargetSubtarget.
MachineBasicBlock * getMBB() const
static bool isSafeToFoldImmIntoCopy(const MachineInstr *Copy, const MachineInstr *MoveImm, const SIInstrInfo *TII, unsigned &SMovOp, int64_t &Imm)
This class represents lattice values for constants.
Definition: AllocatorList.h:23
bool isOperandLegal(const MachineInstr &MI, unsigned OpIdx, const MachineOperand *MO=nullptr) const
Check if MO is a legal operand if it was the OpIdx Operand for MI.
Register createVirtualRegister(const TargetRegisterClass *RegClass, StringRef Name="")
createVirtualRegister - Create and return a new virtual register in the function with the specified r...
const TargetRegisterClass * getEquivalentVGPRClass(const TargetRegisterClass *SRC) const
Implements a dense probed hash-table based set.
Definition: DenseSet.h:249
const DebugLoc & getDebugLoc() const
Returns the debug location id of this MachineInstr.
Definition: MachineInstr.h:385
static bool isPhysicalRegister(unsigned Reg)
Return true if the specified register number is in the physical register namespace.
Definition: Register.h:63
const TargetRegisterClass * getOpRegClass(const MachineInstr &MI, unsigned OpNo) const
Return the correct register class for OpNo.
iterator getFirstTerminator()
Returns an iterator to the first terminator instruction of this basic block.
unsigned getOperandNo(const_mop_iterator I) const
Returns the number of the operand iterator I points to.
Definition: MachineInstr.h:511
iterator_range< reg_nodbg_iterator > reg_nodbg_operands(unsigned Reg) const
unsigned Reg
unsigned getSubReg() const
bool isRegSequence() const
const SIInstrInfo * getInstrInfo() const override
unsigned const TargetRegisterInfo * TRI
const TargetRegisterClass * getEquivalentSGPRClass(const TargetRegisterClass *VRC) const
bool isImm() const
isImm - Tests if this is a MO_Immediate operand.
bool isMoveImmediate(QueryType Type=IgnoreBundle) const
Return true if this instruction is a move immediate (including conditional moves) instruction...
Definition: MachineInstr.h:702
const TargetRegisterClass * getRegClass(Register Reg) const
Return the register class of the specified virtual register.
static bool foldVGPRCopyIntoRegSequence(MachineInstr &MI, const SIRegisterInfo *TRI, const SIInstrInfo *TII, MachineRegisterInfo &MRI)
static bool tryChangeVGPRtoSGPRinCopy(MachineInstr &MI, const SIRegisterInfo *TRI, const SIInstrInfo *TII)
AnalysisUsage & addRequired()
#define INITIALIZE_PASS_DEPENDENCY(depName)
Definition: PassSupport.h:50
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
const HexagonInstrInfo * TII
Printable printMBBReference(const MachineBasicBlock &MBB)
Prints a machine basic block reference.
unsigned getNumOperands() const
Retuns the total number of operands.
Definition: MachineInstr.h:414
void eraseFromParent()
Unlink &#39;this&#39; from the containing basic block and delete it.
unsigned SubReg
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
Definition: MachineInstr.h:411
virtual bool isBasicBlockPrologue(const MachineInstr &MI) const
True if the instruction is bound to the top of its basic block and no other instructions shall be ins...
bool isSGPRClass(const TargetRegisterClass *RC) const
MachineInstr * getVRegDef(unsigned Reg) const
getVRegDef - Return the machine instr that defines the specified virtual register or null if none is ...
SI Fix SGPR copies
static bool hoistAndMergeSGPRInits(unsigned Reg, const MachineRegisterInfo &MRI, MachineDominatorTree &MDT, const TargetInstrInfo *TII)
void clearKillFlags(unsigned Reg) const
clearKillFlags - Iterate over all the uses of the given register and clear the kill flag from the Mac...
INITIALIZE_PASS_BEGIN(SIFixSGPRCopies, DEBUG_TYPE, "SI Fix SGPR copies", false, false) INITIALIZE_PASS_END(SIFixSGPRCopies
LLVM_READONLY MachineOperand * getNamedOperand(MachineInstr &MI, unsigned OperandName) const
Returns the operand named Op.
const TargetRegisterClass * getEquivalentAGPRClass(const TargetRegisterClass *SRC) const
void setReg(Register Reg)
Change the register this operand corresponds to.
MachineBasicBlock * findNearestCommonDominator(MachineBasicBlock *A, MachineBasicBlock *B)
findNearestCommonDominator - Find nearest common dominator basic block for basic block A and B...
static bool isSGPRToVGPRCopy(const TargetRegisterClass *SrcRC, const TargetRegisterClass *DstRC, const SIRegisterInfo &TRI)
void ChangeToImmediate(int64_t ImmVal)
ChangeToImmediate - Replace this operand with a new immediate operand of the specified value...
TargetInstrInfo - Interface to description of machine instruction set.
MachineInstrBuilder BuildMI(MachineFunction &MF, const DebugLoc &DL, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:432
CodeGenOpt::Level getOptLevel() const
Returns the optimization level: None, Less, Default, or Aggressive.
unsigned const MachineRegisterInfo * MRI
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
MachineInstrBuilder & UseMI
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
bool hasVGPRs(const TargetRegisterClass *RC) const
Represent the analysis usage information of a pass.
use_instr_iterator use_instr_begin(unsigned RegNo) const
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly...
Definition: STLExtras.h:1184
FunctionPass class - This class is used to implement most global optimizations.
Definition: Pass.h:284
std::pair< NoneType, bool > insert(const T &V)
insert - Insert an element into the set if it isn&#39;t already there.
Definition: SmallSet.h:180
const TargetRegisterClass * getSubRegClass(const TargetRegisterClass *RC, unsigned SubIdx) const
bool isCopy() const
Falkor HW Prefetch Fix
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
INITIALIZE_PASS_END(RegBankSelect, DEBUG_TYPE, "Assign register bank of generic virtual registers", false, false) RegBankSelect
unsigned getNumExplicitOperands() const
Returns the number of non-implicit operands.
static bool phiHasBreakDef(const MachineInstr &PHI, const MachineRegisterInfo &MRI, SmallSet< unsigned, 8 > &Visited)
Iterator for intrusive lists based on ilist_node.
static std::pair< const TargetRegisterClass *, const TargetRegisterClass * > getCopyRegClasses(const MachineInstr &Copy, const SIRegisterInfo &TRI, const MachineRegisterInfo &MRI)
#define DEBUG_TYPE
void setDesc(const MCInstrDesc &tid)
Replace the instruction descriptor (thus opcode) of the current instruction with a new one...
BlockVerifier::State From
bool searchPredecessors(const MachineBasicBlock *MBB, const MachineBasicBlock *CutOff, UnaryPredicate Predicate)
bool hasAGPRs(const TargetRegisterClass *RC) const
static uint64_t add(uint64_t LeftOp, uint64_t RightOp)
Definition: FileCheck.cpp:243
MachineOperand class - Representation of each machine instruction operand.
This is a &#39;vector&#39; (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:837
bool dominates(const MachineDomTreeNode *A, const MachineDomTreeNode *B) const
MachineInstrBuilder MachineInstrBuilder & DefMI
Predicate
Predicate - These are "(BI << 5) | BO" for various predicates.
Definition: PPCPredicates.h:26
bool properlyDominates(const MachineDomTreeNode *A, const MachineDomTreeNode *B) const
void setPreservesCFG()
This function should be called by the pass, iff they do not:
Definition: Pass.cpp:301
static cl::opt< bool > EnableM0Merge("amdgpu-enable-merge-m0", cl::desc("Merge and hoist M0 initializations"), cl::init(true))
int64_t getImm() const
static bool hasTerminatorThatModifiesExec(const MachineBasicBlock &MBB, const TargetRegisterInfo &TRI)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:132
void addImplicitDefUseOperands(MachineFunction &MF)
Add all implicit def and use operands to this instruction.
FunctionPass * createSIFixSGPRCopiesPass()
static bool phiHasVGPROperands(const MachineInstr &PHI, const MachineRegisterInfo &MRI, const SIRegisterInfo *TRI, const SIInstrInfo *TII)
static bool isVGPRToSGPRCopy(const TargetRegisterClass *SrcRC, const TargetRegisterClass *DstRC, const SIRegisterInfo &TRI)
const MachineBasicBlock * getParent() const
Definition: MachineInstr.h:256
MachineRegisterInfo - Keep track of information for virtual and physical registers, including vreg register classes, use/def chains for registers, etc.
Provides AMDGPU specific target descriptions.
Representation of each machine instruction.
Definition: MachineInstr.h:64
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
Interface definition for SIInstrInfo.
void moveToVALU(MachineInstr &MI, MachineDominatorTree *MDT=nullptr) const
Replace this instruction&#39;s opcode with the equivalent VALU opcode.
bool hasOneUse(unsigned RegNo) const
hasOneUse - Return true if there is exactly one instruction using the specified register.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
char & SIFixSGPRCopiesID
#define I(x, y, z)
Definition: MD5.cpp:58
#define N
iterator getFirstNonPHI()
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
const LLVMTargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
static bool predsHasDivergentTerminator(MachineBasicBlock *MBB, const TargetRegisterInfo *TRI)
bool isReg() const
isReg - Tests if this is a MO_Register operand.
static MachineBasicBlock::iterator getFirstNonPrologue(MachineBasicBlock *MBB, const TargetInstrInfo *TII)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
bool hasVectorRegisters(const TargetRegisterClass *RC) const
const TargetRegisterClass * getPhysRegClass(unsigned Reg) const
Return the &#39;base&#39; register class for this register.
static bool isVirtualRegister(unsigned Reg)
Return true if the specified register number is in the virtual register namespace.
Definition: Register.h:69
iterator_range< def_instr_iterator > def_instructions(unsigned Reg) const
IRTranslator LLVM IR MI
void setRegClass(unsigned Reg, const TargetRegisterClass *RC)
setRegClass - Set the register class of the specified virtual register.
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:48
Register getReg() const
getReg - Returns the register number.
#define LLVM_DEBUG(X)
Definition: Debug.h:122
const MachineOperand & getOperand(unsigned i) const
Definition: MachineInstr.h:416
DominatorTree Class - Concrete subclass of DominatorTreeBase that is used to compute a normal dominat...
static bool isReachable(const MachineInstr *From, const MachineInstr *To, const MachineBasicBlock *CutOff, MachineDominatorTree &MDT)
static bool hasVectorOperands(const MachineInstr &MI, const SIRegisterInfo *TRI)
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
size_type count(const T &V) const
count - Return 1 if the element is in the set, 0 otherwise.
Definition: SmallSet.h:164
const SIRegisterInfo * getRegisterInfo() const override