LLVM  9.0.0svn
SIWholeQuadMode.cpp
Go to the documentation of this file.
1 //===-- SIWholeQuadMode.cpp - enter and suspend whole quad mode -----------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 /// \file
10 /// This pass adds instructions to enable whole quad mode for pixel
11 /// shaders, and whole wavefront mode for all programs.
12 ///
13 /// Whole quad mode is required for derivative computations, but it interferes
14 /// with shader side effects (stores and atomics). This pass is run on the
15 /// scheduled machine IR but before register coalescing, so that machine SSA is
16 /// available for analysis. It ensures that WQM is enabled when necessary, but
17 /// disabled around stores and atomics.
18 ///
19 /// When necessary, this pass creates a function prolog
20 ///
21 /// S_MOV_B64 LiveMask, EXEC
22 /// S_WQM_B64 EXEC, EXEC
23 ///
24 /// to enter WQM at the top of the function and surrounds blocks of Exact
25 /// instructions by
26 ///
27 /// S_AND_SAVEEXEC_B64 Tmp, LiveMask
28 /// ...
29 /// S_MOV_B64 EXEC, Tmp
30 ///
31 /// We also compute when a sequence of instructions requires Whole Wavefront
32 /// Mode (WWM) and insert instructions to save and restore it:
33 ///
34 /// S_OR_SAVEEXEC_B64 Tmp, -1
35 /// ...
36 /// S_MOV_B64 EXEC, Tmp
37 ///
38 /// In order to avoid excessive switching during sequences of Exact
39 /// instructions, the pass first analyzes which instructions must be run in WQM
40 /// (aka which instructions produce values that lead to derivative
41 /// computations).
42 ///
43 /// Basic blocks are always exited in WQM as long as some successor needs WQM.
44 ///
45 /// There is room for improvement given better control flow analysis:
46 ///
47 /// (1) at the top level (outside of control flow statements, and as long as
48 /// kill hasn't been used), one SGPR can be saved by recovering WQM from
49 /// the LiveMask (this is implemented for the entry block).
50 ///
51 /// (2) when entire regions (e.g. if-else blocks or entire loops) only
52 /// consist of exact and don't-care instructions, the switch only has to
53 /// be done at the entry and exit points rather than potentially in each
54 /// block of the region.
55 ///
56 //===----------------------------------------------------------------------===//
57 
58 #include "AMDGPU.h"
59 #include "AMDGPUSubtarget.h"
60 #include "SIInstrInfo.h"
61 #include "SIMachineFunctionInfo.h"
63 #include "llvm/ADT/DenseMap.h"
65 #include "llvm/ADT/SmallVector.h"
66 #include "llvm/ADT/StringRef.h"
78 #include "llvm/IR/CallingConv.h"
79 #include "llvm/IR/DebugLoc.h"
80 #include "llvm/MC/MCRegisterInfo.h"
81 #include "llvm/Pass.h"
82 #include "llvm/Support/Debug.h"
84 #include <cassert>
85 #include <vector>
86 
87 using namespace llvm;
88 
89 #define DEBUG_TYPE "si-wqm"
90 
91 namespace {
92 
93 enum {
94  StateWQM = 0x1,
95  StateWWM = 0x2,
96  StateExact = 0x4,
97 };
98 
99 struct PrintState {
100 public:
101  int State;
102 
103  explicit PrintState(int State) : State(State) {}
104 };
105 
106 #ifndef NDEBUG
107 static raw_ostream &operator<<(raw_ostream &OS, const PrintState &PS) {
108  if (PS.State & StateWQM)
109  OS << "WQM";
110  if (PS.State & StateWWM) {
111  if (PS.State & StateWQM)
112  OS << '|';
113  OS << "WWM";
114  }
115  if (PS.State & StateExact) {
116  if (PS.State & (StateWQM | StateWWM))
117  OS << '|';
118  OS << "Exact";
119  }
120 
121  return OS;
122 }
123 #endif
124 
125 struct InstrInfo {
126  char Needs = 0;
127  char Disabled = 0;
128  char OutNeeds = 0;
129 };
130 
131 struct BlockInfo {
132  char Needs = 0;
133  char InNeeds = 0;
134  char OutNeeds = 0;
135 };
136 
137 struct WorkItem {
138  MachineBasicBlock *MBB = nullptr;
139  MachineInstr *MI = nullptr;
140 
141  WorkItem() = default;
142  WorkItem(MachineBasicBlock *MBB) : MBB(MBB) {}
143  WorkItem(MachineInstr *MI) : MI(MI) {}
144 };
145 
146 class SIWholeQuadMode : public MachineFunctionPass {
147 private:
149  const SIInstrInfo *TII;
150  const SIRegisterInfo *TRI;
152  LiveIntervals *LIS;
153 
156  SmallVector<MachineInstr *, 1> LiveMaskQueries;
157  SmallVector<MachineInstr *, 4> LowerToCopyInstrs;
158 
159  void printInfo();
160 
161  void markInstruction(MachineInstr &MI, char Flag,
162  std::vector<WorkItem> &Worklist);
163  void markInstructionUses(const MachineInstr &MI, char Flag,
164  std::vector<WorkItem> &Worklist);
165  char scanInstructions(MachineFunction &MF, std::vector<WorkItem> &Worklist);
166  void propagateInstruction(MachineInstr &MI, std::vector<WorkItem> &Worklist);
167  void propagateBlock(MachineBasicBlock &MBB, std::vector<WorkItem> &Worklist);
168  char analyzeFunction(MachineFunction &MF);
169 
170  bool requiresCorrectState(const MachineInstr &MI) const;
171 
175  prepareInsertion(MachineBasicBlock &MBB, MachineBasicBlock::iterator First,
176  MachineBasicBlock::iterator Last, bool PreferLast,
177  bool SaveSCC);
178  void toExact(MachineBasicBlock &MBB, MachineBasicBlock::iterator Before,
179  unsigned SaveWQM, unsigned LiveMaskReg);
180  void toWQM(MachineBasicBlock &MBB, MachineBasicBlock::iterator Before,
181  unsigned SavedWQM);
182  void toWWM(MachineBasicBlock &MBB, MachineBasicBlock::iterator Before,
183  unsigned SaveOrig);
184  void fromWWM(MachineBasicBlock &MBB, MachineBasicBlock::iterator Before,
185  unsigned SavedOrig);
186  void processBlock(MachineBasicBlock &MBB, unsigned LiveMaskReg, bool isEntry);
187 
188  void lowerLiveMaskQueries(unsigned LiveMaskReg);
189  void lowerCopyInstrs();
190 
191 public:
192  static char ID;
193 
194  SIWholeQuadMode() :
195  MachineFunctionPass(ID) { }
196 
197  bool runOnMachineFunction(MachineFunction &MF) override;
198 
199  StringRef getPassName() const override { return "SI Whole Quad Mode"; }
200 
201  void getAnalysisUsage(AnalysisUsage &AU) const override {
203  AU.setPreservesCFG();
205  }
206 };
207 
208 } // end anonymous namespace
209 
210 char SIWholeQuadMode::ID = 0;
211 
212 INITIALIZE_PASS_BEGIN(SIWholeQuadMode, DEBUG_TYPE, "SI Whole Quad Mode", false,
213  false)
215 INITIALIZE_PASS_END(SIWholeQuadMode, DEBUG_TYPE, "SI Whole Quad Mode", false,
216  false)
217 
218 char &llvm::SIWholeQuadModeID = SIWholeQuadMode::ID;
219 
221  return new SIWholeQuadMode;
222 }
223 
224 #ifndef NDEBUG
225 LLVM_DUMP_METHOD void SIWholeQuadMode::printInfo() {
226  for (const auto &BII : Blocks) {
227  dbgs() << "\n"
228  << printMBBReference(*BII.first) << ":\n"
229  << " InNeeds = " << PrintState(BII.second.InNeeds)
230  << ", Needs = " << PrintState(BII.second.Needs)
231  << ", OutNeeds = " << PrintState(BII.second.OutNeeds) << "\n\n";
232 
233  for (const MachineInstr &MI : *BII.first) {
234  auto III = Instructions.find(&MI);
235  if (III == Instructions.end())
236  continue;
237 
238  dbgs() << " " << MI << " Needs = " << PrintState(III->second.Needs)
239  << ", OutNeeds = " << PrintState(III->second.OutNeeds) << '\n';
240  }
241  }
242 }
243 #endif
244 
245 void SIWholeQuadMode::markInstruction(MachineInstr &MI, char Flag,
246  std::vector<WorkItem> &Worklist) {
247  InstrInfo &II = Instructions[&MI];
248 
249  assert(!(Flag & StateExact) && Flag != 0);
250 
251  // Remove any disabled states from the flag. The user that required it gets
252  // an undefined value in the helper lanes. For example, this can happen if
253  // the result of an atomic is used by instruction that requires WQM, where
254  // ignoring the request for WQM is correct as per the relevant specs.
255  Flag &= ~II.Disabled;
256 
257  // Ignore if the flag is already encompassed by the existing needs, or we
258  // just disabled everything.
259  if ((II.Needs & Flag) == Flag)
260  return;
261 
262  II.Needs |= Flag;
263  Worklist.push_back(&MI);
264 }
265 
266 /// Mark all instructions defining the uses in \p MI with \p Flag.
267 void SIWholeQuadMode::markInstructionUses(const MachineInstr &MI, char Flag,
268  std::vector<WorkItem> &Worklist) {
269  for (const MachineOperand &Use : MI.uses()) {
270  if (!Use.isReg() || !Use.isUse())
271  continue;
272 
273  unsigned Reg = Use.getReg();
274 
275  // Handle physical registers that we need to track; this is mostly relevant
276  // for VCC, which can appear as the (implicit) input of a uniform branch,
277  // e.g. when a loop counter is stored in a VGPR.
279  if (Reg == AMDGPU::EXEC)
280  continue;
281 
282  for (MCRegUnitIterator RegUnit(Reg, TRI); RegUnit.isValid(); ++RegUnit) {
283  LiveRange &LR = LIS->getRegUnit(*RegUnit);
284  const VNInfo *Value = LR.Query(LIS->getInstructionIndex(MI)).valueIn();
285  if (!Value)
286  continue;
287 
288  // Since we're in machine SSA, we do not need to track physical
289  // registers across basic blocks.
290  if (Value->isPHIDef())
291  continue;
292 
293  markInstruction(*LIS->getInstructionFromIndex(Value->def), Flag,
294  Worklist);
295  }
296 
297  continue;
298  }
299 
300  for (MachineInstr &DefMI : MRI->def_instructions(Use.getReg()))
301  markInstruction(DefMI, Flag, Worklist);
302  }
303 }
304 
305 // Scan instructions to determine which ones require an Exact execmask and
306 // which ones seed WQM requirements.
307 char SIWholeQuadMode::scanInstructions(MachineFunction &MF,
308  std::vector<WorkItem> &Worklist) {
309  char GlobalFlags = 0;
310  bool WQMOutputs = MF.getFunction().hasFnAttribute("amdgpu-ps-wqm-outputs");
311  SmallVector<MachineInstr *, 4> SetInactiveInstrs;
312 
313  // We need to visit the basic blocks in reverse post-order so that we visit
314  // defs before uses, in particular so that we don't accidentally mark an
315  // instruction as needing e.g. WQM before visiting it and realizing it needs
316  // WQM disabled.
318  for (auto BI = RPOT.begin(), BE = RPOT.end(); BI != BE; ++BI) {
319  MachineBasicBlock &MBB = **BI;
320  BlockInfo &BBI = Blocks[&MBB];
321 
322  for (auto II = MBB.begin(), IE = MBB.end(); II != IE; ++II) {
323  MachineInstr &MI = *II;
324  InstrInfo &III = Instructions[&MI];
325  unsigned Opcode = MI.getOpcode();
326  char Flags = 0;
327 
328  if (TII->isWQM(Opcode)) {
329  // Sampling instructions don't need to produce results for all pixels
330  // in a quad, they just require all inputs of a quad to have been
331  // computed for derivatives.
332  markInstructionUses(MI, StateWQM, Worklist);
333  GlobalFlags |= StateWQM;
334  continue;
335  } else if (Opcode == AMDGPU::WQM) {
336  // The WQM intrinsic requires its output to have all the helper lanes
337  // correct, so we need it to be in WQM.
338  Flags = StateWQM;
339  LowerToCopyInstrs.push_back(&MI);
340  } else if (Opcode == AMDGPU::WWM) {
341  // The WWM intrinsic doesn't make the same guarantee, and plus it needs
342  // to be executed in WQM or Exact so that its copy doesn't clobber
343  // inactive lanes.
344  markInstructionUses(MI, StateWWM, Worklist);
345  GlobalFlags |= StateWWM;
346  LowerToCopyInstrs.push_back(&MI);
347  continue;
348  } else if (Opcode == AMDGPU::V_SET_INACTIVE_B32 ||
349  Opcode == AMDGPU::V_SET_INACTIVE_B64) {
350  III.Disabled = StateWWM;
351  MachineOperand &Inactive = MI.getOperand(2);
352  if (Inactive.isReg()) {
353  if (Inactive.isUndef()) {
354  LowerToCopyInstrs.push_back(&MI);
355  } else {
356  unsigned Reg = Inactive.getReg();
358  for (MachineInstr &DefMI : MRI->def_instructions(Reg))
359  markInstruction(DefMI, StateWWM, Worklist);
360  }
361  }
362  }
363  SetInactiveInstrs.push_back(&MI);
364  continue;
365  } else if (TII->isDisableWQM(MI)) {
366  BBI.Needs |= StateExact;
367  if (!(BBI.InNeeds & StateExact)) {
368  BBI.InNeeds |= StateExact;
369  Worklist.push_back(&MBB);
370  }
371  GlobalFlags |= StateExact;
372  III.Disabled = StateWQM | StateWWM;
373  continue;
374  } else {
375  if (Opcode == AMDGPU::SI_PS_LIVE) {
376  LiveMaskQueries.push_back(&MI);
377  } else if (WQMOutputs) {
378  // The function is in machine SSA form, which means that physical
379  // VGPRs correspond to shader inputs and outputs. Inputs are
380  // only used, outputs are only defined.
381  for (const MachineOperand &MO : MI.defs()) {
382  if (!MO.isReg())
383  continue;
384 
385  unsigned Reg = MO.getReg();
386 
387  if (!TRI->isVirtualRegister(Reg) &&
388  TRI->hasVGPRs(TRI->getPhysRegClass(Reg))) {
389  Flags = StateWQM;
390  break;
391  }
392  }
393  }
394 
395  if (!Flags)
396  continue;
397  }
398 
399  markInstruction(MI, Flags, Worklist);
400  GlobalFlags |= Flags;
401  }
402  }
403 
404  // Mark sure that any SET_INACTIVE instructions are computed in WQM if WQM is
405  // ever used anywhere in the function. This implements the corresponding
406  // semantics of @llvm.amdgcn.set.inactive.
407  if (GlobalFlags & StateWQM) {
408  for (MachineInstr *MI : SetInactiveInstrs)
409  markInstruction(*MI, StateWQM, Worklist);
410  }
411 
412  return GlobalFlags;
413 }
414 
415 void SIWholeQuadMode::propagateInstruction(MachineInstr &MI,
416  std::vector<WorkItem>& Worklist) {
417  MachineBasicBlock *MBB = MI.getParent();
418  InstrInfo II = Instructions[&MI]; // take a copy to prevent dangling references
419  BlockInfo &BI = Blocks[MBB];
420 
421  // Control flow-type instructions and stores to temporary memory that are
422  // followed by WQM computations must themselves be in WQM.
423  if ((II.OutNeeds & StateWQM) && !(II.Disabled & StateWQM) &&
424  (MI.isTerminator() || (TII->usesVM_CNT(MI) && MI.mayStore()))) {
425  Instructions[&MI].Needs = StateWQM;
426  II.Needs = StateWQM;
427  }
428 
429  // Propagate to block level
430  if (II.Needs & StateWQM) {
431  BI.Needs |= StateWQM;
432  if (!(BI.InNeeds & StateWQM)) {
433  BI.InNeeds |= StateWQM;
434  Worklist.push_back(MBB);
435  }
436  }
437 
438  // Propagate backwards within block
439  if (MachineInstr *PrevMI = MI.getPrevNode()) {
440  char InNeeds = (II.Needs & ~StateWWM) | II.OutNeeds;
441  if (!PrevMI->isPHI()) {
442  InstrInfo &PrevII = Instructions[PrevMI];
443  if ((PrevII.OutNeeds | InNeeds) != PrevII.OutNeeds) {
444  PrevII.OutNeeds |= InNeeds;
445  Worklist.push_back(PrevMI);
446  }
447  }
448  }
449 
450  // Propagate WQM flag to instruction inputs
451  assert(!(II.Needs & StateExact));
452 
453  if (II.Needs != 0)
454  markInstructionUses(MI, II.Needs, Worklist);
455 
456  // Ensure we process a block containing WWM, even if it does not require any
457  // WQM transitions.
458  if (II.Needs & StateWWM)
459  BI.Needs |= StateWWM;
460 }
461 
462 void SIWholeQuadMode::propagateBlock(MachineBasicBlock &MBB,
463  std::vector<WorkItem>& Worklist) {
464  BlockInfo BI = Blocks[&MBB]; // Make a copy to prevent dangling references.
465 
466  // Propagate through instructions
467  if (!MBB.empty()) {
468  MachineInstr *LastMI = &*MBB.rbegin();
469  InstrInfo &LastII = Instructions[LastMI];
470  if ((LastII.OutNeeds | BI.OutNeeds) != LastII.OutNeeds) {
471  LastII.OutNeeds |= BI.OutNeeds;
472  Worklist.push_back(LastMI);
473  }
474  }
475 
476  // Predecessor blocks must provide for our WQM/Exact needs.
477  for (MachineBasicBlock *Pred : MBB.predecessors()) {
478  BlockInfo &PredBI = Blocks[Pred];
479  if ((PredBI.OutNeeds | BI.InNeeds) == PredBI.OutNeeds)
480  continue;
481 
482  PredBI.OutNeeds |= BI.InNeeds;
483  PredBI.InNeeds |= BI.InNeeds;
484  Worklist.push_back(Pred);
485  }
486 
487  // All successors must be prepared to accept the same set of WQM/Exact data.
488  for (MachineBasicBlock *Succ : MBB.successors()) {
489  BlockInfo &SuccBI = Blocks[Succ];
490  if ((SuccBI.InNeeds | BI.OutNeeds) == SuccBI.InNeeds)
491  continue;
492 
493  SuccBI.InNeeds |= BI.OutNeeds;
494  Worklist.push_back(Succ);
495  }
496 }
497 
498 char SIWholeQuadMode::analyzeFunction(MachineFunction &MF) {
499  std::vector<WorkItem> Worklist;
500  char GlobalFlags = scanInstructions(MF, Worklist);
501 
502  while (!Worklist.empty()) {
503  WorkItem WI = Worklist.back();
504  Worklist.pop_back();
505 
506  if (WI.MI)
507  propagateInstruction(*WI.MI, Worklist);
508  else
509  propagateBlock(*WI.MBB, Worklist);
510  }
511 
512  return GlobalFlags;
513 }
514 
515 /// Whether \p MI really requires the exec state computed during analysis.
516 ///
517 /// Scalar instructions must occasionally be marked WQM for correct propagation
518 /// (e.g. thread masks leading up to branches), but when it comes to actual
519 /// execution, they don't care about EXEC.
520 bool SIWholeQuadMode::requiresCorrectState(const MachineInstr &MI) const {
521  if (MI.isTerminator())
522  return true;
523 
524  // Skip instructions that are not affected by EXEC
525  if (TII->isScalarUnit(MI))
526  return false;
527 
528  // Generic instructions such as COPY will either disappear by register
529  // coalescing or be lowered to SALU or VALU instructions.
530  if (MI.isTransient()) {
531  if (MI.getNumExplicitOperands() >= 1) {
532  const MachineOperand &Op = MI.getOperand(0);
533  if (Op.isReg()) {
534  if (TRI->isSGPRReg(*MRI, Op.getReg())) {
535  // SGPR instructions are not affected by EXEC
536  return false;
537  }
538  }
539  }
540  }
541 
542  return true;
543 }
544 
546 SIWholeQuadMode::saveSCC(MachineBasicBlock &MBB,
548  unsigned SaveReg = MRI->createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
549 
550  MachineInstr *Save =
551  BuildMI(MBB, Before, DebugLoc(), TII->get(AMDGPU::COPY), SaveReg)
552  .addReg(AMDGPU::SCC);
553  MachineInstr *Restore =
554  BuildMI(MBB, Before, DebugLoc(), TII->get(AMDGPU::COPY), AMDGPU::SCC)
555  .addReg(SaveReg);
556 
557  LIS->InsertMachineInstrInMaps(*Save);
558  LIS->InsertMachineInstrInMaps(*Restore);
559  LIS->createAndComputeVirtRegInterval(SaveReg);
560 
561  return Restore;
562 }
563 
564 // Return an iterator in the (inclusive) range [First, Last] at which
565 // instructions can be safely inserted, keeping in mind that some of the
566 // instructions we want to add necessarily clobber SCC.
567 MachineBasicBlock::iterator SIWholeQuadMode::prepareInsertion(
569  MachineBasicBlock::iterator Last, bool PreferLast, bool SaveSCC) {
570  if (!SaveSCC)
571  return PreferLast ? Last : First;
572 
573  LiveRange &LR = LIS->getRegUnit(*MCRegUnitIterator(AMDGPU::SCC, TRI));
574  auto MBBE = MBB.end();
575  SlotIndex FirstIdx = First != MBBE ? LIS->getInstructionIndex(*First)
576  : LIS->getMBBEndIdx(&MBB);
577  SlotIndex LastIdx =
578  Last != MBBE ? LIS->getInstructionIndex(*Last) : LIS->getMBBEndIdx(&MBB);
579  SlotIndex Idx = PreferLast ? LastIdx : FirstIdx;
580  const LiveRange::Segment *S;
581 
582  for (;;) {
583  S = LR.getSegmentContaining(Idx);
584  if (!S)
585  break;
586 
587  if (PreferLast) {
588  SlotIndex Next = S->start.getBaseIndex();
589  if (Next < FirstIdx)
590  break;
591  Idx = Next;
592  } else {
593  SlotIndex Next = S->end.getNextIndex().getBaseIndex();
594  if (Next > LastIdx)
595  break;
596  Idx = Next;
597  }
598  }
599 
601 
602  if (MachineInstr *MI = LIS->getInstructionFromIndex(Idx))
603  MBBI = MI;
604  else {
605  assert(Idx == LIS->getMBBEndIdx(&MBB));
606  MBBI = MBB.end();
607  }
608 
609  if (S)
610  MBBI = saveSCC(MBB, MBBI);
611 
612  return MBBI;
613 }
614 
615 void SIWholeQuadMode::toExact(MachineBasicBlock &MBB,
617  unsigned SaveWQM, unsigned LiveMaskReg) {
618  MachineInstr *MI;
619 
620  if (SaveWQM) {
621  MI = BuildMI(MBB, Before, DebugLoc(), TII->get(AMDGPU::S_AND_SAVEEXEC_B64),
622  SaveWQM)
623  .addReg(LiveMaskReg);
624  } else {
625  MI = BuildMI(MBB, Before, DebugLoc(), TII->get(AMDGPU::S_AND_B64),
626  AMDGPU::EXEC)
627  .addReg(AMDGPU::EXEC)
628  .addReg(LiveMaskReg);
629  }
630 
631  LIS->InsertMachineInstrInMaps(*MI);
632 }
633 
634 void SIWholeQuadMode::toWQM(MachineBasicBlock &MBB,
636  unsigned SavedWQM) {
637  MachineInstr *MI;
638 
639  if (SavedWQM) {
640  MI = BuildMI(MBB, Before, DebugLoc(), TII->get(AMDGPU::COPY), AMDGPU::EXEC)
641  .addReg(SavedWQM);
642  } else {
643  MI = BuildMI(MBB, Before, DebugLoc(), TII->get(AMDGPU::S_WQM_B64),
644  AMDGPU::EXEC)
645  .addReg(AMDGPU::EXEC);
646  }
647 
648  LIS->InsertMachineInstrInMaps(*MI);
649 }
650 
651 void SIWholeQuadMode::toWWM(MachineBasicBlock &MBB,
653  unsigned SaveOrig) {
654  MachineInstr *MI;
655 
656  assert(SaveOrig);
657  MI = BuildMI(MBB, Before, DebugLoc(), TII->get(AMDGPU::S_OR_SAVEEXEC_B64),
658  SaveOrig)
659  .addImm(-1);
660  LIS->InsertMachineInstrInMaps(*MI);
661 }
662 
663 void SIWholeQuadMode::fromWWM(MachineBasicBlock &MBB,
665  unsigned SavedOrig) {
666  MachineInstr *MI;
667 
668  assert(SavedOrig);
669  MI = BuildMI(MBB, Before, DebugLoc(), TII->get(AMDGPU::EXIT_WWM), AMDGPU::EXEC)
670  .addReg(SavedOrig);
671  LIS->InsertMachineInstrInMaps(*MI);
672 }
673 
674 void SIWholeQuadMode::processBlock(MachineBasicBlock &MBB, unsigned LiveMaskReg,
675  bool isEntry) {
676  auto BII = Blocks.find(&MBB);
677  if (BII == Blocks.end())
678  return;
679 
680  const BlockInfo &BI = BII->second;
681 
682  // This is a non-entry block that is WQM throughout, so no need to do
683  // anything.
684  if (!isEntry && BI.Needs == StateWQM && BI.OutNeeds != StateExact)
685  return;
686 
687  LLVM_DEBUG(dbgs() << "\nProcessing block " << printMBBReference(MBB)
688  << ":\n");
689 
690  unsigned SavedWQMReg = 0;
691  unsigned SavedNonWWMReg = 0;
692  bool WQMFromExec = isEntry;
693  char State = (isEntry || !(BI.InNeeds & StateWQM)) ? StateExact : StateWQM;
694  char NonWWMState = 0;
695 
696  auto II = MBB.getFirstNonPHI(), IE = MBB.end();
697  if (isEntry)
698  ++II; // Skip the instruction that saves LiveMask
699 
700  // This stores the first instruction where it's safe to switch from WQM to
701  // Exact or vice versa.
702  MachineBasicBlock::iterator FirstWQM = IE;
703 
704  // This stores the first instruction where it's safe to switch from WWM to
705  // Exact/WQM or to switch to WWM. It must always be the same as, or after,
706  // FirstWQM since if it's safe to switch to/from WWM, it must be safe to
707  // switch to/from WQM as well.
708  MachineBasicBlock::iterator FirstWWM = IE;
709  for (;;) {
710  MachineBasicBlock::iterator Next = II;
711  char Needs = StateExact | StateWQM; // WWM is disabled by default
712  char OutNeeds = 0;
713 
714  if (FirstWQM == IE)
715  FirstWQM = II;
716 
717  if (FirstWWM == IE)
718  FirstWWM = II;
719 
720  // First, figure out the allowed states (Needs) based on the propagated
721  // flags.
722  if (II != IE) {
723  MachineInstr &MI = *II;
724 
725  if (requiresCorrectState(MI)) {
726  auto III = Instructions.find(&MI);
727  if (III != Instructions.end()) {
728  if (III->second.Needs & StateWWM)
729  Needs = StateWWM;
730  else if (III->second.Needs & StateWQM)
731  Needs = StateWQM;
732  else
733  Needs &= ~III->second.Disabled;
734  OutNeeds = III->second.OutNeeds;
735  }
736  } else {
737  // If the instruction doesn't actually need a correct EXEC, then we can
738  // safely leave WWM enabled.
739  Needs = StateExact | StateWQM | StateWWM;
740  }
741 
742  if (MI.isTerminator() && OutNeeds == StateExact)
743  Needs = StateExact;
744 
745  if (MI.getOpcode() == AMDGPU::SI_ELSE && BI.OutNeeds == StateExact)
746  MI.getOperand(3).setImm(1);
747 
748  ++Next;
749  } else {
750  // End of basic block
751  if (BI.OutNeeds & StateWQM)
752  Needs = StateWQM;
753  else if (BI.OutNeeds == StateExact)
754  Needs = StateExact;
755  else
756  Needs = StateWQM | StateExact;
757  }
758 
759  // Now, transition if necessary.
760  if (!(Needs & State)) {
762  if (State == StateWWM || Needs == StateWWM) {
763  // We must switch to or from WWM
764  First = FirstWWM;
765  } else {
766  // We only need to switch to/from WQM, so we can use FirstWQM
767  First = FirstWQM;
768  }
769 
771  prepareInsertion(MBB, First, II, Needs == StateWQM,
772  Needs == StateExact || WQMFromExec);
773 
774  if (State == StateWWM) {
775  assert(SavedNonWWMReg);
776  fromWWM(MBB, Before, SavedNonWWMReg);
777  State = NonWWMState;
778  }
779 
780  if (Needs == StateWWM) {
781  NonWWMState = State;
782  SavedNonWWMReg = MRI->createVirtualRegister(&AMDGPU::SReg_64RegClass);
783  toWWM(MBB, Before, SavedNonWWMReg);
784  State = StateWWM;
785  } else {
786  if (State == StateWQM && (Needs & StateExact) && !(Needs & StateWQM)) {
787  if (!WQMFromExec && (OutNeeds & StateWQM))
788  SavedWQMReg = MRI->createVirtualRegister(&AMDGPU::SReg_64RegClass);
789 
790  toExact(MBB, Before, SavedWQMReg, LiveMaskReg);
791  State = StateExact;
792  } else if (State == StateExact && (Needs & StateWQM) &&
793  !(Needs & StateExact)) {
794  assert(WQMFromExec == (SavedWQMReg == 0));
795 
796  toWQM(MBB, Before, SavedWQMReg);
797 
798  if (SavedWQMReg) {
799  LIS->createAndComputeVirtRegInterval(SavedWQMReg);
800  SavedWQMReg = 0;
801  }
802  State = StateWQM;
803  } else {
804  // We can get here if we transitioned from WWM to a non-WWM state that
805  // already matches our needs, but we shouldn't need to do anything.
806  assert(Needs & State);
807  }
808  }
809  }
810 
811  if (Needs != (StateExact | StateWQM | StateWWM)) {
812  if (Needs != (StateExact | StateWQM))
813  FirstWQM = IE;
814  FirstWWM = IE;
815  }
816 
817  if (II == IE)
818  break;
819  II = Next;
820  }
821 }
822 
823 void SIWholeQuadMode::lowerLiveMaskQueries(unsigned LiveMaskReg) {
824  for (MachineInstr *MI : LiveMaskQueries) {
825  const DebugLoc &DL = MI->getDebugLoc();
826  unsigned Dest = MI->getOperand(0).getReg();
827  MachineInstr *Copy =
828  BuildMI(*MI->getParent(), MI, DL, TII->get(AMDGPU::COPY), Dest)
829  .addReg(LiveMaskReg);
830 
831  LIS->ReplaceMachineInstrInMaps(*MI, *Copy);
832  MI->eraseFromParent();
833  }
834 }
835 
836 void SIWholeQuadMode::lowerCopyInstrs() {
837  for (MachineInstr *MI : LowerToCopyInstrs) {
838  for (unsigned i = MI->getNumExplicitOperands() - 1; i > 1; i--)
839  MI->RemoveOperand(i);
840  MI->setDesc(TII->get(AMDGPU::COPY));
841  }
842 }
843 
844 bool SIWholeQuadMode::runOnMachineFunction(MachineFunction &MF) {
845  Instructions.clear();
846  Blocks.clear();
847  LiveMaskQueries.clear();
848  LowerToCopyInstrs.clear();
850 
851  const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
852 
853  TII = ST.getInstrInfo();
854  TRI = &TII->getRegisterInfo();
855  MRI = &MF.getRegInfo();
856  LIS = &getAnalysis<LiveIntervals>();
857 
858  char GlobalFlags = analyzeFunction(MF);
859  unsigned LiveMaskReg = 0;
860  if (!(GlobalFlags & StateWQM)) {
861  lowerLiveMaskQueries(AMDGPU::EXEC);
862  if (!(GlobalFlags & StateWWM))
863  return !LiveMaskQueries.empty();
864  } else {
865  // Store a copy of the original live mask when required
866  MachineBasicBlock &Entry = MF.front();
868 
869  if (GlobalFlags & StateExact || !LiveMaskQueries.empty()) {
870  LiveMaskReg = MRI->createVirtualRegister(&AMDGPU::SReg_64RegClass);
871  MachineInstr *MI = BuildMI(Entry, EntryMI, DebugLoc(),
872  TII->get(AMDGPU::COPY), LiveMaskReg)
873  .addReg(AMDGPU::EXEC);
874  LIS->InsertMachineInstrInMaps(*MI);
875  }
876 
877  lowerLiveMaskQueries(LiveMaskReg);
878 
879  if (GlobalFlags == StateWQM) {
880  // For a shader that needs only WQM, we can just set it once.
881  BuildMI(Entry, EntryMI, DebugLoc(), TII->get(AMDGPU::S_WQM_B64),
882  AMDGPU::EXEC)
883  .addReg(AMDGPU::EXEC);
884 
885  lowerCopyInstrs();
886  // EntryMI may become invalid here
887  return true;
888  }
889  }
890 
891  LLVM_DEBUG(printInfo());
892 
893  lowerCopyInstrs();
894 
895  // Handle the general case
896  for (auto BII : Blocks)
897  processBlock(*BII.first, LiveMaskReg, BII.first == &*MF.begin());
898 
899  // Physical registers like SCC aren't tracked by default anyway, so just
900  // removing the ranges we computed is the simplest option for maintaining
901  // the analysis results.
902  LIS->removeRegUnit(*MCRegUnitIterator(AMDGPU::SCC, TRI));
903 
904  return true;
905 }
bool isPHIDef() const
Returns true if this value is defined by a PHI instruction (or was, PHI instructions may have been el...
Definition: LiveInterval.h:77
char & SIWholeQuadModeID
AMDGPU specific subclass of TargetSubtarget.
SlotIndex getBaseIndex() const
Returns the base index for associated with this index.
Definition: SlotIndexes.h:241
SlotIndex def
The index of the defining instruction.
Definition: LiveInterval.h:60
SI Whole Quad Mode
This class represents lattice values for constants.
Definition: AllocatorList.h:23
#define LLVM_DUMP_METHOD
Mark debug helper function definitions like dump() that should not be stripped from debug builds...
Definition: Compiler.h:464
iterator_range< mop_iterator > uses()
Returns a range that includes all operands that are register uses.
Definition: MachineInstr.h:491
INITIALIZE_PASS_BEGIN(SIWholeQuadMode, DEBUG_TYPE, "SI Whole Quad Mode", false, false) INITIALIZE_PASS_END(SIWholeQuadMode
void push_back(const T &Elt)
Definition: SmallVector.h:211
const DebugLoc & getDebugLoc() const
Returns the debug location id of this MachineInstr.
Definition: MachineInstr.h:382
unsigned getReg() const
getReg - Returns the register number.
static bool isVirtualRegister(unsigned Reg)
Return true if the specified register number is in the virtual register namespace.
unsigned Reg
const SIInstrInfo * getInstrInfo() const override
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
Definition: Function.h:320
bool isTransient() const
Return true if this is a transient instruction that is either very likely to be eliminated during reg...
This represents a simple continuous liveness interval for a value.
Definition: LiveInterval.h:161
unsigned const TargetRegisterInfo * TRI
A debug info location.
Definition: DebugLoc.h:33
VNInfo - Value Number Information.
Definition: LiveInterval.h:52
iterator_range< succ_iterator > successors()
This class represents the liveness of a register, stack slot, etc.
Definition: LiveInterval.h:156
static const AMDGPUSubtarget & get(const MachineFunction &MF)
AnalysisUsage & addRequired()
#define INITIALIZE_PASS_DEPENDENCY(depName)
Definition: PassSupport.h:50
SlotIndex getNextIndex() const
Returns the next index.
Definition: SlotIndexes.h:279
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
const HexagonInstrInfo * TII
Printable printMBBReference(const MachineBasicBlock &MBB)
Prints a machine basic block reference.
A Use represents the edge between a Value definition and its users.
Definition: Use.h:55
void eraseFromParent()
Unlink &#39;this&#39; from the containing basic block and delete it.
bool isTerminator(QueryType Type=AnyInBundle) const
Returns true if this instruction part of the terminator for a basic block.
Definition: MachineInstr.h:648
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
Definition: MachineInstr.h:408
reverse_iterator rbegin()
LiveQueryResult Query(SlotIndex Idx) const
Query Liveness at Idx.
Definition: LiveInterval.h:528
Flag
These should be considered private to the implementation of the MCInstrDesc class.
Definition: MCInstrDesc.h:117
MachineInstrBuilder BuildMI(MachineFunction &MF, const DebugLoc &DL, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
bool mayStore(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly modify memory.
Definition: MachineInstr.h:819
* if(!EatIfPresent(lltok::kw_thread_local)) return false
ParseOptionalThreadLocal := /*empty.
unsigned const MachineRegisterInfo * MRI
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
FunctionPass * createSIWholeQuadModePass()
Represent the analysis usage information of a pass.
iterator_range< mop_iterator > defs()
Returns a range over all explicit operands that are register definitions.
Definition: MachineInstr.h:480
void setImm(int64_t immVal)
FunctionPass class - This class is used to implement most global optimizations.
Definition: Pass.h:284
iterator_range< pred_iterator > predecessors()
const MachineBasicBlock & front() const
INITIALIZE_PASS_END(RegBankSelect, DEBUG_TYPE, "Assign register bank of generic virtual registers", false, false) RegBankSelect
unsigned getNumExplicitOperands() const
Returns the number of non-implicit operands.
void setDesc(const MCInstrDesc &tid)
Replace the instruction descriptor (thus opcode) of the current instruction with a new one...
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
Definition: Function.h:212
MachineOperand class - Representation of each machine instruction operand.
This is a &#39;vector&#39; (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:839
MachineInstrBuilder MachineInstrBuilder & DefMI
void setPreservesCFG()
This function should be called by the pass, iff they do not:
Definition: Pass.cpp:285
const Function & getFunction() const
Return the LLVM function that this machine code represents.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:132
bool isValid() const
isValid - returns true if this iterator is not yet at the end.
#define DEBUG_TYPE
const MachineBasicBlock * getParent() const
Definition: MachineInstr.h:253
MachineRegisterInfo - Keep track of information for virtual and physical registers, including vreg register classes, use/def chains for registers, etc.
Provides AMDGPU specific target descriptions.
Representation of each machine instruction.
Definition: MachineInstr.h:63
Interface definition for SIInstrInfo.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
iterator getFirstNonPHI()
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
const MachineInstrBuilder & addReg(unsigned RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
raw_ostream & operator<<(raw_ostream &OS, const APInt &I)
Definition: APInt.h:2038
bool isReg() const
isReg - Tests if this is a MO_Register operand.
const Segment * getSegmentContaining(SlotIndex Idx) const
Return the segment that contains the specified index, or null if there is none.
Definition: LiveInterval.h:395
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
LLVM Value Representation.
Definition: Value.h:72
This class implements an extremely fast bulk output stream that can only output to a stream...
Definition: raw_ostream.h:45
IRTranslator LLVM IR MI
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:48
void RemoveOperand(unsigned OpNo)
Erase an operand from an instruction, leaving it with one fewer operand than it started with...
#define LLVM_DEBUG(X)
Definition: Debug.h:122
const MachineOperand & getOperand(unsigned i) const
Definition: MachineInstr.h:413
SlotIndex - An opaque wrapper around machine indexes.
Definition: SlotIndexes.h:83
const SIRegisterInfo * getRegisterInfo() const override