LCOV - code coverage report
Current view: top level - lib/Target/AMDGPU - SIWholeQuadMode.cpp (source / functions) Hit Total Coverage
Test: llvm-toolchain.info Lines: 330 347 95.1 %
Date: 2017-09-14 15:23:50 Functions: 24 25 96.0 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : //===-- SIWholeQuadMode.cpp - enter and suspend whole quad mode -----------===//
       2             : //
       3             : //                     The LLVM Compiler Infrastructure
       4             : //
       5             : // This file is distributed under the University of Illinois Open Source
       6             : // License. See LICENSE.TXT for details.
       7             : //
       8             : //===----------------------------------------------------------------------===//
       9             : //
      10             : /// \file
      11             : /// \brief This pass adds instructions to enable whole quad mode for pixel
      12             : /// shaders, and whole wavefront mode for all programs.
      13             : ///
      14             : /// Whole quad mode is required for derivative computations, but it interferes
      15             : /// with shader side effects (stores and atomics). This pass is run on the
      16             : /// scheduled machine IR but before register coalescing, so that machine SSA is
      17             : /// available for analysis. It ensures that WQM is enabled when necessary, but
      18             : /// disabled around stores and atomics.
      19             : ///
      20             : /// When necessary, this pass creates a function prolog
      21             : ///
      22             : ///   S_MOV_B64 LiveMask, EXEC
      23             : ///   S_WQM_B64 EXEC, EXEC
      24             : ///
      25             : /// to enter WQM at the top of the function and surrounds blocks of Exact
      26             : /// instructions by
      27             : ///
      28             : ///   S_AND_SAVEEXEC_B64 Tmp, LiveMask
      29             : ///   ...
      30             : ///   S_MOV_B64 EXEC, Tmp
      31             : ///
      32             : /// We also compute when a sequence of instructions requires Whole Wavefront
      33             : /// Mode (WWM) and insert instructions to save and restore it:
      34             : ///
      35             : /// S_OR_SAVEEXEC_B64 Tmp, -1
      36             : /// ...
      37             : /// S_MOV_B64 EXEC, Tmp
      38             : ///
      39             : /// In order to avoid excessive switching during sequences of Exact
      40             : /// instructions, the pass first analyzes which instructions must be run in WQM
      41             : /// (aka which instructions produce values that lead to derivative
      42             : /// computations).
      43             : ///
      44             : /// Basic blocks are always exited in WQM as long as some successor needs WQM.
      45             : ///
      46             : /// There is room for improvement given better control flow analysis:
      47             : ///
      48             : ///  (1) at the top level (outside of control flow statements, and as long as
      49             : ///      kill hasn't been used), one SGPR can be saved by recovering WQM from
      50             : ///      the LiveMask (this is implemented for the entry block).
      51             : ///
      52             : ///  (2) when entire regions (e.g. if-else blocks or entire loops) only
      53             : ///      consist of exact and don't-care instructions, the switch only has to
      54             : ///      be done at the entry and exit points rather than potentially in each
      55             : ///      block of the region.
      56             : ///
      57             : //===----------------------------------------------------------------------===//
      58             : 
      59             : #include "AMDGPU.h"
      60             : #include "AMDGPUSubtarget.h"
      61             : #include "SIInstrInfo.h"
      62             : #include "SIMachineFunctionInfo.h"
      63             : #include "llvm/ADT/DenseMap.h"
      64             : #include "llvm/ADT/PostOrderIterator.h"
      65             : #include "llvm/ADT/SmallVector.h"
      66             : #include "llvm/ADT/StringRef.h"
      67             : #include "llvm/CodeGen/LiveInterval.h"
      68             : #include "llvm/CodeGen/LiveIntervalAnalysis.h"
      69             : #include "llvm/CodeGen/MachineBasicBlock.h"
      70             : #include "llvm/CodeGen/MachineFunction.h"
      71             : #include "llvm/CodeGen/MachineFunctionPass.h"
      72             : #include "llvm/CodeGen/MachineInstr.h"
      73             : #include "llvm/CodeGen/MachineInstrBuilder.h"
      74             : #include "llvm/CodeGen/MachineOperand.h"
      75             : #include "llvm/CodeGen/MachineRegisterInfo.h"
      76             : #include "llvm/CodeGen/SlotIndexes.h"
      77             : #include "llvm/IR/CallingConv.h"
      78             : #include "llvm/IR/DebugLoc.h"
      79             : #include "llvm/MC/MCRegisterInfo.h"
      80             : #include "llvm/Pass.h"
      81             : #include "llvm/Support/Debug.h"
      82             : #include "llvm/Support/raw_ostream.h"
      83             : #include "llvm/Target/TargetRegisterInfo.h"
      84             : #include <cassert>
      85             : #include <vector>
      86             : 
      87             : using namespace llvm;
      88             : 
      89             : #define DEBUG_TYPE "si-wqm"
      90             : 
      91             : namespace {
      92             : 
      93             : enum {
      94             :   StateWQM = 0x1,
      95             :   StateWWM = 0x2,
      96             :   StateExact = 0x4,
      97             : };
      98             : 
      99             : struct PrintState {
     100             : public:
     101             :   int State;
     102             : 
     103             :   explicit PrintState(int State) : State(State) {}
     104             : };
     105             : 
     106             : static raw_ostream &operator<<(raw_ostream &OS, const PrintState &PS) {
     107             :   if (PS.State & StateWQM)
     108             :     OS << "WQM";
     109             :   if (PS.State & StateWWM) {
     110             :     if (PS.State & StateWQM)
     111             :       OS << '|';
     112             :     OS << "WWM";
     113             :   }
     114             :   if (PS.State & StateExact) {
     115             :     if (PS.State & (StateWQM | StateWWM))
     116             :       OS << '|';
     117             :     OS << "Exact";
     118             :   }
     119             : 
     120             :   return OS;
     121             : }
     122             : 
     123             : struct InstrInfo {
     124             :   char Needs = 0;
     125             :   char Disabled = 0;
     126             :   char OutNeeds = 0;
     127             : };
     128             : 
     129             : struct BlockInfo {
     130             :   char Needs = 0;
     131             :   char InNeeds = 0;
     132             :   char OutNeeds = 0;
     133             : };
     134             : 
     135             : struct WorkItem {
     136             :   MachineBasicBlock *MBB = nullptr;
     137             :   MachineInstr *MI = nullptr;
     138             : 
     139             :   WorkItem() = default;
     140        1625 :   WorkItem(MachineBasicBlock *MBB) : MBB(MBB) {}
     141       10635 :   WorkItem(MachineInstr *MI) : MI(MI) {}
     142             : };
     143             : 
     144        8766 : class SIWholeQuadMode : public MachineFunctionPass {
     145             : private:
     146             :   CallingConv::ID CallingConv;
     147             :   const SIInstrInfo *TII;
     148             :   const SIRegisterInfo *TRI;
     149             :   MachineRegisterInfo *MRI;
     150             :   LiveIntervals *LIS;
     151             : 
     152             :   DenseMap<const MachineInstr *, InstrInfo> Instructions;
     153             :   DenseMap<MachineBasicBlock *, BlockInfo> Blocks;
     154             :   SmallVector<MachineInstr *, 1> LiveMaskQueries;
     155             :   SmallVector<MachineInstr *, 4> LowerToCopyInstrs;
     156             : 
     157             :   void printInfo();
     158             : 
     159             :   void markInstruction(MachineInstr &MI, char Flag,
     160             :                        std::vector<WorkItem> &Worklist);
     161             :   void markInstructionUses(const MachineInstr &MI, char Flag,
     162             :                            std::vector<WorkItem> &Worklist);
     163             :   char scanInstructions(MachineFunction &MF, std::vector<WorkItem> &Worklist);
     164             :   void propagateInstruction(MachineInstr &MI, std::vector<WorkItem> &Worklist);
     165             :   void propagateBlock(MachineBasicBlock &MBB, std::vector<WorkItem> &Worklist);
     166             :   char analyzeFunction(MachineFunction &MF);
     167             : 
     168             :   bool requiresCorrectState(const MachineInstr &MI) const;
     169             : 
     170             :   MachineBasicBlock::iterator saveSCC(MachineBasicBlock &MBB,
     171             :                                       MachineBasicBlock::iterator Before);
     172             :   MachineBasicBlock::iterator
     173             :   prepareInsertion(MachineBasicBlock &MBB, MachineBasicBlock::iterator First,
     174             :                    MachineBasicBlock::iterator Last, bool PreferLast,
     175             :                    bool SaveSCC);
     176             :   void toExact(MachineBasicBlock &MBB, MachineBasicBlock::iterator Before,
     177             :                unsigned SaveWQM, unsigned LiveMaskReg);
     178             :   void toWQM(MachineBasicBlock &MBB, MachineBasicBlock::iterator Before,
     179             :              unsigned SavedWQM);
     180             :   void toWWM(MachineBasicBlock &MBB, MachineBasicBlock::iterator Before,
     181             :              unsigned SaveOrig);
     182             :   void fromWWM(MachineBasicBlock &MBB, MachineBasicBlock::iterator Before,
     183             :                unsigned SavedOrig);
     184             :   void processBlock(MachineBasicBlock &MBB, unsigned LiveMaskReg, bool isEntry);
     185             : 
     186             :   void lowerLiveMaskQueries(unsigned LiveMaskReg);
     187             :   void lowerCopyInstrs();
     188             : 
     189             : public:
     190             :   static char ID;
     191             : 
     192        1469 :   SIWholeQuadMode() :
     193        7345 :     MachineFunctionPass(ID) { }
     194             : 
     195             :   bool runOnMachineFunction(MachineFunction &MF) override;
     196             : 
     197        1463 :   StringRef getPassName() const override { return "SI Whole Quad Mode"; }
     198             : 
     199        1463 :   void getAnalysisUsage(AnalysisUsage &AU) const override {
     200        1463 :     AU.addRequired<LiveIntervals>();
     201        1463 :     AU.setPreservesCFG();
     202        1463 :     MachineFunctionPass::getAnalysisUsage(AU);
     203        1463 :   }
     204             : };
     205             : 
     206             : } // end anonymous namespace
     207             : 
     208             : char SIWholeQuadMode::ID = 0;
     209             : 
     210       53042 : INITIALIZE_PASS_BEGIN(SIWholeQuadMode, DEBUG_TYPE, "SI Whole Quad Mode", false,
     211             :                       false)
     212       53042 : INITIALIZE_PASS_DEPENDENCY(LiveIntervals)
     213      312538 : INITIALIZE_PASS_END(SIWholeQuadMode, DEBUG_TYPE, "SI Whole Quad Mode", false,
     214             :                     false)
     215             : 
     216             : char &llvm::SIWholeQuadModeID = SIWholeQuadMode::ID;
     217             : 
     218        1468 : FunctionPass *llvm::createSIWholeQuadModePass() {
     219        1468 :   return new SIWholeQuadMode;
     220             : }
     221             : 
     222             : void SIWholeQuadMode::printInfo() {
     223             :   for (const auto &BII : Blocks) {
     224             :     dbgs() << "\nBB#" << BII.first->getNumber() << ":\n"
     225             :            << "  InNeeds = " << PrintState(BII.second.InNeeds)
     226             :            << ", Needs = " << PrintState(BII.second.Needs)
     227             :            << ", OutNeeds = " << PrintState(BII.second.OutNeeds) << "\n\n";
     228             : 
     229             :     for (const MachineInstr &MI : *BII.first) {
     230             :       auto III = Instructions.find(&MI);
     231             :       if (III == Instructions.end())
     232             :         continue;
     233             : 
     234             :       dbgs() << "  " << MI << "    Needs = " << PrintState(III->second.Needs)
     235             :              << ", OutNeeds = " << PrintState(III->second.OutNeeds) << '\n';
     236             :     }
     237             :   }
     238             : }
     239             : 
     240        7470 : void SIWholeQuadMode::markInstruction(MachineInstr &MI, char Flag,
     241             :                                       std::vector<WorkItem> &Worklist) {
     242       14940 :   InstrInfo &II = Instructions[&MI];
     243             : 
     244             :   assert(!(Flag & StateExact) && Flag != 0);
     245             : 
     246             :   // Remove any disabled states from the flag. The user that required it gets
     247             :   // an undefined value in the helper lanes. For example, this can happen if
     248             :   // the result of an atomic is used by instruction that requires WQM, where
     249             :   // ignoring the request for WQM is correct as per the relevant specs.
     250        7470 :   Flag &= ~II.Disabled;
     251             : 
     252             :   // Ignore if the flag is already encompassed by the existing needs, or we
     253             :   // just disabled everything.
     254        7470 :   if ((II.Needs & Flag) == Flag)
     255             :     return;
     256             : 
     257        2545 :   II.Needs |= Flag;
     258        5090 :   Worklist.push_back(&MI);
     259             : }
     260             : 
     261             : /// Mark all instructions defining the uses in \p MI with \p Flag.
     262        4987 : void SIWholeQuadMode::markInstructionUses(const MachineInstr &MI, char Flag,
     263             :                                           std::vector<WorkItem> &Worklist) {
     264       24853 :   for (const MachineOperand &Use : MI.uses()) {
     265       30833 :     if (!Use.isReg() || !Use.isUse())
     266        9098 :       continue;
     267             : 
     268       10768 :     unsigned Reg = Use.getReg();
     269             : 
     270             :     // Handle physical registers that we need to track; this is mostly relevant
     271             :     // for VCC, which can appear as the (implicit) input of a uniform branch,
     272             :     // e.g. when a loop counter is stored in a VGPR.
     273       10768 :     if (!TargetRegisterInfo::isVirtualRegister(Reg)) {
     274        3388 :       if (Reg == AMDGPU::EXEC)
     275        1985 :         continue;
     276             : 
     277        4325 :       for (MCRegUnitIterator RegUnit(Reg, TRI); RegUnit.isValid(); ++RegUnit) {
     278        3038 :         LiveRange &LR = LIS->getRegUnit(*RegUnit);
     279        3038 :         const VNInfo *Value = LR.Query(LIS->getInstructionIndex(MI)).valueIn();
     280        1519 :         if (!Value)
     281         450 :           continue;
     282             : 
     283             :         // Since we're in machine SSA, we do not need to track physical
     284             :         // registers across basic blocks.
     285        1069 :         if (Value->isPHIDef())
     286        1035 :           continue;
     287             : 
     288          68 :         markInstruction(*LIS->getInstructionFromIndex(Value->def), Flag,
     289             :                         Worklist);
     290             :       }
     291             : 
     292        1403 :       continue;
     293             :     }
     294             : 
     295       36900 :     for (MachineInstr &DefMI : MRI->def_instructions(Use.getReg()))
     296        7380 :       markInstruction(DefMI, Flag, Worklist);
     297             :   }
     298        4987 : }
     299             : 
     300             : // Scan instructions to determine which ones require an Exact execmask and
     301             : // which ones seed WQM requirements.
     302       14834 : char SIWholeQuadMode::scanInstructions(MachineFunction &MF,
     303             :                                        std::vector<WorkItem> &Worklist) {
     304       14834 :   char GlobalFlags = 0;
     305       29668 :   bool WQMOutputs = MF.getFunction()->hasFnAttribute("amdgpu-ps-wqm-outputs");
     306       29668 :   SmallVector<MachineInstr *, 4> SetInactiveInstrs;
     307             : 
     308             :   // We need to visit the basic blocks in reverse post-order so that we visit
     309             :   // defs before uses, in particular so that we don't accidentally mark an
     310             :   // instruction as needing e.g. WQM before visiting it and realizing it needs
     311             :   // WQM disabled.
     312       29668 :   ReversePostOrderTraversal<MachineFunction *> RPOT(&MF);
     313       14834 :   for (auto BI = RPOT.begin(), BE = RPOT.end(); BI != BE; ++BI) {
     314       16894 :     MachineBasicBlock &MBB = **BI;
     315       33788 :     BlockInfo &BBI = Blocks[&MBB];
     316             : 
     317      392082 :     for (auto II = MBB.begin(), IE = MBB.end(); II != IE; ++II) {
     318      341400 :       MachineInstr &MI = *II;
     319      682800 :       InstrInfo &III = Instructions[&MI];
     320      341400 :       unsigned Opcode = MI.getOpcode();
     321      341400 :       char Flags = 0;
     322             : 
     323      682800 :       if (TII->isDS(Opcode) && CallingConv == CallingConv::AMDGPU_PS) {
     324             :         Flags = StateWQM;
     325      682921 :       } else if (TII->isWQM(Opcode)) {
     326             :         // Sampling instructions don't need to produce results for all pixels
     327             :         // in a quad, they just require all inputs of a quad to have been
     328             :         // computed for derivatives.
     329         205 :         markInstructionUses(MI, StateWQM, Worklist);
     330         205 :         GlobalFlags |= StateWQM;
     331         205 :         continue;
     332      341153 :       } else if (Opcode == AMDGPU::WQM) {
     333             :         // The WQM intrinsic requires its output to have all the helper lanes
     334             :         // correct, so we need it to be in WQM.
     335           8 :         Flags = StateWQM;
     336           8 :         LowerToCopyInstrs.push_back(&MI);
     337      341160 :       } else if (Opcode == AMDGPU::WWM) {
     338             :         // The WWM intrinsic doesn't make the same guarantee, and plus it needs
     339             :         // to be executed in WQM or Exact so that its copy doesn't clobber
     340             :         // inactive lanes.
     341          15 :         markInstructionUses(MI, StateWWM, Worklist);
     342          15 :         GlobalFlags |= StateWWM;
     343          15 :         LowerToCopyInstrs.push_back(&MI);
     344          15 :         continue;
     345      341130 :       } else if (Opcode == AMDGPU::V_SET_INACTIVE_B32 ||
     346             :                  Opcode == AMDGPU::V_SET_INACTIVE_B64) {
     347           8 :         III.Disabled = StateWWM;
     348           8 :         MachineOperand &Inactive = MI.getOperand(2);
     349           8 :         if (Inactive.isReg()) {
     350           2 :           if (Inactive.isUndef()) {
     351           0 :             LowerToCopyInstrs.push_back(&MI);
     352             :           } else {
     353           2 :             unsigned Reg = Inactive.getReg();
     354           2 :             if (TargetRegisterInfo::isVirtualRegister(Reg)) {
     355          10 :               for (MachineInstr &DefMI : MRI->def_instructions(Reg))
     356           2 :                 markInstruction(DefMI, StateWWM, Worklist);
     357             :             }
     358             :           }
     359             :         }
     360           8 :         SetInactiveInstrs.push_back(&MI);
     361           8 :         continue;
     362      342339 :       } else if (TII->isDisableWQM(MI)) {
     363        1217 :         BBI.Needs |= StateExact;
     364        1217 :         if (!(BBI.InNeeds & StateExact)) {
     365         909 :           BBI.InNeeds |= StateExact;
     366        1818 :           Worklist.push_back(&MBB);
     367             :         }
     368        1217 :         GlobalFlags |= StateExact;
     369        1217 :         III.Disabled = StateWQM | StateWWM;
     370        1217 :         continue;
     371             :       } else {
     372      339905 :         if (Opcode == AMDGPU::SI_PS_LIVE) {
     373           3 :           LiveMaskQueries.push_back(&MI);
     374      339902 :         } else if (WQMOutputs) {
     375             :           // The function is in machine SSA form, which means that physical
     376             :           // VGPRs correspond to shader inputs and outputs. Inputs are
     377             :           // only used, outputs are only defined.
     378          14 :           for (const MachineOperand &MO : MI.defs()) {
     379           8 :             if (!MO.isReg())
     380           0 :               continue;
     381             : 
     382           8 :             unsigned Reg = MO.getReg();
     383             : 
     384          10 :             if (!TRI->isVirtualRegister(Reg) &&
     385           2 :                 TRI->hasVGPRs(TRI->getPhysRegClass(Reg))) {
     386             :               Flags = StateWQM;
     387             :               break;
     388             :             }
     389             :           }
     390             :         }
     391             : 
     392           8 :         if (!Flags)
     393      339903 :           continue;
     394             :       }
     395             : 
     396          52 :       markInstruction(MI, Flags, Worklist);
     397          52 :       GlobalFlags |= Flags;
     398             :     }
     399             :   }
     400             : 
     401             :   // Mark sure that any SET_INACTIVE instructions are computed in WQM if WQM is
     402             :   // ever used anywhere in the function. This implements the corresponding
     403             :   // semantics of @llvm.amdgcn.set.inactive.
     404       14834 :   if (GlobalFlags & StateWQM) {
     405         578 :     for (MachineInstr *MI : SetInactiveInstrs)
     406           2 :       markInstruction(*MI, StateWQM, Worklist);
     407             :   }
     408             : 
     409       29668 :   return GlobalFlags;
     410             : }
     411             : 
     412       10635 : void SIWholeQuadMode::propagateInstruction(MachineInstr &MI,
     413             :                                            std::vector<WorkItem>& Worklist) {
     414       10635 :   MachineBasicBlock *MBB = MI.getParent();
     415       21270 :   InstrInfo II = Instructions[&MI]; // take a copy to prevent dangling references
     416       21270 :   BlockInfo &BI = Blocks[MBB];
     417             : 
     418             :   // Control flow-type instructions and stores to temporary memory that are
     419             :   // followed by WQM computations must themselves be in WQM.
     420       17300 :   if ((II.OutNeeds & StateWQM) && !(II.Disabled & StateWQM) &&
     421       13141 :       (MI.isTerminator() || (TII->usesVM_CNT(MI) && MI.mayStore()))) {
     422         382 :     Instructions[&MI].Needs = StateWQM;
     423         191 :     II.Needs = StateWQM;
     424             :   }
     425             : 
     426             :   // Propagate to block level
     427       10635 :   if (II.Needs & StateWQM) {
     428        4637 :     BI.Needs |= StateWQM;
     429        4637 :     if (!(BI.InNeeds & StateWQM)) {
     430         236 :       BI.InNeeds |= StateWQM;
     431         708 :       Worklist.push_back(MBB);
     432             :     }
     433             :   }
     434             : 
     435             :   // Propagate backwards within block
     436       20643 :   if (MachineInstr *PrevMI = MI.getPrevNode()) {
     437       10008 :     char InNeeds = (II.Needs & ~StateWWM) | II.OutNeeds;
     438        9841 :     if (!PrevMI->isPHI()) {
     439       19682 :       InstrInfo &PrevII = Instructions[PrevMI];
     440        9841 :       if ((PrevII.OutNeeds | InNeeds) != PrevII.OutNeeds) {
     441        7668 :         PrevII.OutNeeds |= InNeeds;
     442       15336 :         Worklist.push_back(PrevMI);
     443             :       }
     444             :     }
     445             :   }
     446             : 
     447             :   // Propagate WQM flag to instruction inputs
     448             :   assert(!(II.Needs & StateExact));
     449             : 
     450       10635 :   if (II.Needs != 0)
     451        4767 :     markInstructionUses(MI, II.Needs, Worklist);
     452       10635 : }
     453             : 
     454        1625 : void SIWholeQuadMode::propagateBlock(MachineBasicBlock &MBB,
     455             :                                      std::vector<WorkItem>& Worklist) {
     456        3250 :   BlockInfo BI = Blocks[&MBB]; // Make a copy to prevent dangling references.
     457             : 
     458             :   // Propagate through instructions
     459        1625 :   if (!MBB.empty()) {
     460        3242 :     MachineInstr *LastMI = &*MBB.rbegin();
     461        3242 :     InstrInfo &LastII = Instructions[LastMI];
     462        1621 :     if ((LastII.OutNeeds | BI.OutNeeds) != LastII.OutNeeds) {
     463         422 :       LastII.OutNeeds |= BI.OutNeeds;
     464         844 :       Worklist.push_back(LastMI);
     465             :     }
     466             :   }
     467             : 
     468             :   // Predecessor blocks must provide for our WQM/Exact needs.
     469        3996 :   for (MachineBasicBlock *Pred : MBB.predecessors()) {
     470        1492 :     BlockInfo &PredBI = Blocks[Pred];
     471         746 :     if ((PredBI.OutNeeds | BI.InNeeds) == PredBI.OutNeeds)
     472         313 :       continue;
     473             : 
     474         433 :     PredBI.OutNeeds |= BI.InNeeds;
     475         433 :     PredBI.InNeeds |= BI.InNeeds;
     476        1299 :     Worklist.push_back(Pred);
     477             :   }
     478             : 
     479             :   // All successors must be prepared to accept the same set of WQM/Exact data.
     480        4089 :   for (MachineBasicBlock *Succ : MBB.successors()) {
     481        1678 :     BlockInfo &SuccBI = Blocks[Succ];
     482         839 :     if ((SuccBI.InNeeds | BI.OutNeeds) == SuccBI.InNeeds)
     483         792 :       continue;
     484             : 
     485          47 :     SuccBI.InNeeds |= BI.OutNeeds;
     486         141 :     Worklist.push_back(Succ);
     487             :   }
     488        1625 : }
     489             : 
     490       14834 : char SIWholeQuadMode::analyzeFunction(MachineFunction &MF) {
     491       29668 :   std::vector<WorkItem> Worklist;
     492       14834 :   char GlobalFlags = scanInstructions(MF, Worklist);
     493             : 
     494       27094 :   while (!Worklist.empty()) {
     495       12260 :     WorkItem WI = Worklist.back();
     496       12260 :     Worklist.pop_back();
     497             : 
     498       12260 :     if (WI.MI)
     499       10635 :       propagateInstruction(*WI.MI, Worklist);
     500             :     else
     501        1625 :       propagateBlock(*WI.MBB, Worklist);
     502             :   }
     503             : 
     504       29668 :   return GlobalFlags;
     505             : }
     506             : 
     507             : /// Whether \p MI really requires the exec state computed during analysis.
     508             : ///
     509             : /// Scalar instructions must occasionally be marked WQM for correct propagation
     510             : /// (e.g. thread masks leading up to branches), but when it comes to actual
     511             : /// execution, they don't care about EXEC.
     512        4071 : bool SIWholeQuadMode::requiresCorrectState(const MachineInstr &MI) const {
     513        4071 :   if (MI.isTerminator())
     514             :     return true;
     515             : 
     516             :   // Skip instructions that are not affected by EXEC
     517        3821 :   if (TII->isScalarUnit(MI))
     518             :     return false;
     519             : 
     520             :   // Generic instructions such as COPY will either disappear by register
     521             :   // coalescing or be lowered to SALU or VALU instructions.
     522             :   if (MI.isTransient()) {
     523        1623 :     if (MI.getNumExplicitOperands() >= 1) {
     524        1623 :       const MachineOperand &Op = MI.getOperand(0);
     525        1623 :       if (Op.isReg()) {
     526        1623 :         if (TRI->isSGPRReg(*MRI, Op.getReg())) {
     527             :           // SGPR instructions are not affected by EXEC
     528             :           return false;
     529             :         }
     530             :       }
     531             :     }
     532             :   }
     533             : 
     534             :   return true;
     535             : }
     536             : 
     537             : MachineBasicBlock::iterator
     538           0 : SIWholeQuadMode::saveSCC(MachineBasicBlock &MBB,
     539             :                          MachineBasicBlock::iterator Before) {
     540           0 :   unsigned SaveReg = MRI->createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
     541             : 
     542             :   MachineInstr *Save =
     543           0 :       BuildMI(MBB, Before, DebugLoc(), TII->get(AMDGPU::COPY), SaveReg)
     544           0 :           .addReg(AMDGPU::SCC);
     545             :   MachineInstr *Restore =
     546           0 :       BuildMI(MBB, Before, DebugLoc(), TII->get(AMDGPU::COPY), AMDGPU::SCC)
     547           0 :           .addReg(SaveReg);
     548             : 
     549           0 :   LIS->InsertMachineInstrInMaps(*Save);
     550           0 :   LIS->InsertMachineInstrInMaps(*Restore);
     551           0 :   LIS->createAndComputeVirtRegInterval(SaveReg);
     552             : 
     553           0 :   return Restore;
     554             : }
     555             : 
     556             : // Return an iterator in the (inclusive) range [First, Last] at which
     557             : // instructions can be safely inserted, keeping in mind that some of the
     558             : // instructions we want to add necessarily clobber SCC.
     559         252 : MachineBasicBlock::iterator SIWholeQuadMode::prepareInsertion(
     560             :     MachineBasicBlock &MBB, MachineBasicBlock::iterator First,
     561             :     MachineBasicBlock::iterator Last, bool PreferLast, bool SaveSCC) {
     562         252 :   if (!SaveSCC)
     563          18 :     return PreferLast ? Last : First;
     564             : 
     565         702 :   LiveRange &LR = LIS->getRegUnit(*MCRegUnitIterator(AMDGPU::SCC, TRI));
     566         234 :   auto MBBE = MBB.end();
     567         702 :   SlotIndex FirstIdx = First != MBBE ? LIS->getInstructionIndex(*First)
     568         702 :                                      : LIS->getMBBEndIdx(&MBB);
     569             :   SlotIndex LastIdx =
     570         695 :       Last != MBBE ? LIS->getInstructionIndex(*Last) : LIS->getMBBEndIdx(&MBB);
     571         234 :   SlotIndex Idx = PreferLast ? LastIdx : FirstIdx;
     572             :   const LiveRange::Segment *S;
     573             : 
     574             :   for (;;) {
     575           2 :     S = LR.getSegmentContaining(Idx);
     576           2 :     if (!S)
     577             :       break;
     578             : 
     579           2 :     if (PreferLast) {
     580           4 :       SlotIndex Next = S->start.getBaseIndex();
     581           2 :       if (Next < FirstIdx)
     582             :         break;
     583             :       Idx = Next;
     584             :     } else {
     585           0 :       SlotIndex Next = S->end.getNextIndex().getBaseIndex();
     586           0 :       if (Next > LastIdx)
     587             :         break;
     588             :       Idx = Next;
     589             :     }
     590             :   }
     591             : 
     592         234 :   MachineBasicBlock::iterator MBBI;
     593             : 
     594         468 :   if (MachineInstr *MI = LIS->getInstructionFromIndex(Idx))
     595             :     MBBI = MI;
     596             :   else {
     597             :     assert(Idx == LIS->getMBBEndIdx(&MBB));
     598           0 :     MBBI = MBB.end();
     599             :   }
     600             : 
     601         234 :   if (S)
     602           0 :     MBBI = saveSCC(MBB, MBBI);
     603             : 
     604         234 :   return MBBI;
     605             : }
     606             : 
     607         115 : void SIWholeQuadMode::toExact(MachineBasicBlock &MBB,
     608             :                               MachineBasicBlock::iterator Before,
     609             :                               unsigned SaveWQM, unsigned LiveMaskReg) {
     610             :   MachineInstr *MI;
     611             : 
     612         115 :   if (SaveWQM) {
     613          30 :     MI = BuildMI(MBB, Before, DebugLoc(), TII->get(AMDGPU::S_AND_SAVEEXEC_B64),
     614          12 :                  SaveWQM)
     615           6 :              .addReg(LiveMaskReg);
     616             :   } else {
     617         545 :     MI = BuildMI(MBB, Before, DebugLoc(), TII->get(AMDGPU::S_AND_B64),
     618         218 :                  AMDGPU::EXEC)
     619         109 :              .addReg(AMDGPU::EXEC)
     620         109 :              .addReg(LiveMaskReg);
     621             :   }
     622             : 
     623         230 :   LIS->InsertMachineInstrInMaps(*MI);
     624         115 : }
     625             : 
     626         105 : void SIWholeQuadMode::toWQM(MachineBasicBlock &MBB,
     627             :                             MachineBasicBlock::iterator Before,
     628             :                             unsigned SavedWQM) {
     629             :   MachineInstr *MI;
     630             : 
     631         105 :   if (SavedWQM) {
     632          30 :     MI = BuildMI(MBB, Before, DebugLoc(), TII->get(AMDGPU::COPY), AMDGPU::EXEC)
     633           6 :              .addReg(SavedWQM);
     634             :   } else {
     635         495 :     MI = BuildMI(MBB, Before, DebugLoc(), TII->get(AMDGPU::S_WQM_B64),
     636         198 :                  AMDGPU::EXEC)
     637          99 :              .addReg(AMDGPU::EXEC);
     638             :   }
     639             : 
     640         210 :   LIS->InsertMachineInstrInMaps(*MI);
     641         105 : }
     642             : 
     643          17 : void SIWholeQuadMode::toWWM(MachineBasicBlock &MBB,
     644             :                             MachineBasicBlock::iterator Before,
     645             :                             unsigned SaveOrig) {
     646             :   MachineInstr *MI;
     647             : 
     648             :   assert(SaveOrig);
     649          85 :   MI = BuildMI(MBB, Before, DebugLoc(), TII->get(AMDGPU::S_OR_SAVEEXEC_B64),
     650          34 :                SaveOrig)
     651          17 :            .addImm(-1);
     652          34 :   LIS->InsertMachineInstrInMaps(*MI);
     653          17 : }
     654             : 
     655          17 : void SIWholeQuadMode::fromWWM(MachineBasicBlock &MBB,
     656             :                               MachineBasicBlock::iterator Before,
     657             :                               unsigned SavedOrig) {
     658             :   MachineInstr *MI;
     659             : 
     660             :   assert(SavedOrig);
     661          85 :   MI = BuildMI(MBB, Before, DebugLoc(), TII->get(AMDGPU::EXIT_WWM), AMDGPU::EXEC)
     662          17 :            .addReg(SavedOrig);
     663          34 :   LIS->InsertMachineInstrInMaps(*MI);
     664          17 : }
     665             : 
     666         315 : void SIWholeQuadMode::processBlock(MachineBasicBlock &MBB, unsigned LiveMaskReg,
     667             :                                    bool isEntry) {
     668         315 :   auto BII = Blocks.find(&MBB);
     669         945 :   if (BII == Blocks.end())
     670          84 :     return;
     671             : 
     672         315 :   const BlockInfo &BI = BII->second;
     673             : 
     674             :   // This is a non-entry block that is WQM throughout, so no need to do
     675             :   // anything.
     676         315 :   if (!isEntry && BI.Needs == StateWQM && BI.OutNeeds != StateExact)
     677             :     return;
     678             : 
     679             :   DEBUG(dbgs() << "\nProcessing block BB#" << MBB.getNumber() << ":\n");
     680             : 
     681         231 :   unsigned SavedWQMReg = 0;
     682         231 :   unsigned SavedNonWWMReg = 0;
     683         231 :   bool WQMFromExec = isEntry;
     684         231 :   char State = (isEntry || !(BI.InNeeds & StateWQM)) ? StateExact : StateWQM;
     685         231 :   char NonWWMState = 0;
     686             : 
     687         462 :   auto II = MBB.getFirstNonPHI(), IE = MBB.end();
     688         231 :   if (isEntry)
     689             :     ++II; // Skip the instruction that saves LiveMask
     690             : 
     691             :   // This stores the first instruction where it's safe to switch from WQM to
     692             :   // Exact or vice versa.
     693             :   MachineBasicBlock::iterator FirstWQM = IE;
     694             : 
     695             :   // This stores the first instruction where it's safe to switch from WWM to
     696             :   // Exact/WQM or to switch to WWM. It must always be the same as, or after,
     697             :   // FirstWQM since if it's safe to switch to/from WWM, it must be safe to
     698             :   // switch to/from WQM as well.
     699             :   MachineBasicBlock::iterator FirstWWM = IE;
     700             :   for (;;) {
     701        4302 :     MachineBasicBlock::iterator Next = II;
     702        4302 :     char Needs = StateExact | StateWQM; // WWM is disabled by default
     703        4302 :     char OutNeeds = 0;
     704             : 
     705        4302 :     if (FirstWQM == IE)
     706        1534 :       FirstWQM = II;
     707             : 
     708        4302 :     if (FirstWWM == IE)
     709        3020 :       FirstWWM = II;
     710             : 
     711             :     // First, figure out the allowed states (Needs) based on the propagated
     712             :     // flags.
     713        4302 :     if (II != IE) {
     714        4071 :       MachineInstr &MI = *II;
     715             : 
     716        4071 :       if (requiresCorrectState(MI)) {
     717        2789 :         auto III = Instructions.find(&MI);
     718        8367 :         if (III != Instructions.end()) {
     719        2789 :           if (III->second.Needs & StateWWM)
     720             :             Needs = StateWWM;
     721        2739 :           else if (III->second.Needs & StateWQM)
     722             :             Needs = StateWQM;
     723             :           else
     724        1704 :             Needs &= ~III->second.Disabled;
     725        2789 :           OutNeeds = III->second.OutNeeds;
     726             :         }
     727             :       } else {
     728             :         // If the instruction doesn't actually need a correct EXEC, then we can
     729             :         // safely leave WWM enabled.
     730             :         Needs = StateExact | StateWQM | StateWWM;
     731             :       }
     732             : 
     733        4071 :       if (MI.isTerminator() && OutNeeds == StateExact)
     734             :         Needs = StateExact;
     735             : 
     736        4071 :       if (MI.getOpcode() == AMDGPU::SI_ELSE && BI.OutNeeds == StateExact)
     737           6 :         MI.getOperand(3).setImm(1);
     738             : 
     739             :       ++Next;
     740             :     } else {
     741             :       // End of basic block
     742         231 :       if (BI.OutNeeds & StateWQM)
     743             :         Needs = StateWQM;
     744         181 :       else if (BI.OutNeeds == StateExact)
     745             :         Needs = StateExact;
     746             :       else
     747         103 :         Needs = StateWQM | StateExact;
     748             :     }
     749             : 
     750             :     // Now, transition if necessary.
     751        4302 :     if (!(Needs & State)) {
     752         252 :       MachineBasicBlock::iterator First;
     753         252 :       if (State == StateWWM || Needs == StateWWM) {
     754             :         // We must switch to or from WWM
     755          34 :         First = FirstWWM;
     756             :       } else {
     757             :         // We only need to switch to/from WQM, so we can use FirstWQM
     758         218 :         First = FirstWQM;
     759             :       }
     760             : 
     761             :       MachineBasicBlock::iterator Before =
     762             :           prepareInsertion(MBB, First, II, Needs == StateWQM,
     763         252 :                            Needs == StateExact || WQMFromExec);
     764             : 
     765         252 :       if (State == StateWWM) {
     766             :         assert(SavedNonWWMReg);
     767          17 :         fromWWM(MBB, Before, SavedNonWWMReg);
     768          17 :         State = NonWWMState;
     769             :       }
     770             : 
     771         252 :       if (Needs == StateWWM) {
     772          17 :         NonWWMState = State;
     773          17 :         SavedNonWWMReg = MRI->createVirtualRegister(&AMDGPU::SReg_64RegClass);
     774          17 :         toWWM(MBB, Before, SavedNonWWMReg);
     775          17 :         State = StateWWM;
     776             :       } else {
     777         235 :         if (State == StateWQM && (Needs & StateExact) && !(Needs & StateWQM)) {
     778         115 :           if (!WQMFromExec && (OutNeeds & StateWQM))
     779           6 :             SavedWQMReg = MRI->createVirtualRegister(&AMDGPU::SReg_64RegClass);
     780             : 
     781         115 :           toExact(MBB, Before, SavedWQMReg, LiveMaskReg);
     782         115 :           State = StateExact;
     783         120 :         } else if (State == StateExact && (Needs & StateWQM) &&
     784             :                    !(Needs & StateExact)) {
     785             :           assert(WQMFromExec == (SavedWQMReg == 0));
     786             : 
     787         105 :           toWQM(MBB, Before, SavedWQMReg);
     788             : 
     789         105 :           if (SavedWQMReg) {
     790          12 :             LIS->createAndComputeVirtRegInterval(SavedWQMReg);
     791           6 :             SavedWQMReg = 0;
     792             :           }
     793             :           State = StateWQM;
     794             :         } else {
     795             :           // We can get here if we transitioned from WWM to a non-WWM state that
     796             :           // already matches our needs, but we shouldn't need to do anything.
     797             :           assert(Needs & State);
     798             :         }
     799             :       }
     800             :     }
     801             : 
     802        4302 :     if (Needs != (StateExact | StateWQM | StateWWM)) {
     803        3020 :       if (Needs != (StateExact | StateWQM))
     804        1431 :         FirstWQM = IE;
     805             :       FirstWWM = IE;
     806             :     }
     807             : 
     808        4302 :     if (II == IE)
     809             :       break;
     810        4071 :     II = Next;
     811        4071 :   }
     812             : }
     813             : 
     814       14834 : void SIWholeQuadMode::lowerLiveMaskQueries(unsigned LiveMaskReg) {
     815       44505 :   for (MachineInstr *MI : LiveMaskQueries) {
     816           3 :     const DebugLoc &DL = MI->getDebugLoc();
     817           3 :     unsigned Dest = MI->getOperand(0).getReg();
     818             :     MachineInstr *Copy =
     819          12 :         BuildMI(*MI->getParent(), MI, DL, TII->get(AMDGPU::COPY), Dest)
     820           3 :             .addReg(LiveMaskReg);
     821             : 
     822           6 :     LIS->ReplaceMachineInstrInMaps(*MI, *Copy);
     823           3 :     MI->eraseFromParent();
     824             :   }
     825       14834 : }
     826             : 
     827         205 : void SIWholeQuadMode::lowerCopyInstrs() {
     828         638 :   for (MachineInstr *MI : LowerToCopyInstrs) {
     829          23 :     for (unsigned i = MI->getNumExplicitOperands() - 1; i > 1; i--)
     830           0 :       MI->RemoveOperand(i);
     831          69 :     MI->setDesc(TII->get(AMDGPU::COPY));
     832             :   }
     833         205 : }
     834             : 
     835       14834 : bool SIWholeQuadMode::runOnMachineFunction(MachineFunction &MF) {
     836       14834 :   Instructions.clear();
     837       14834 :   Blocks.clear();
     838       29668 :   LiveMaskQueries.clear();
     839       29668 :   LowerToCopyInstrs.clear();
     840       29668 :   CallingConv = MF.getFunction()->getCallingConv();
     841             : 
     842       14834 :   const SISubtarget &ST = MF.getSubtarget<SISubtarget>();
     843             : 
     844       14834 :   TII = ST.getInstrInfo();
     845       29668 :   TRI = &TII->getRegisterInfo();
     846       14834 :   MRI = &MF.getRegInfo();
     847       14834 :   LIS = &getAnalysis<LiveIntervals>();
     848             : 
     849       14834 :   char GlobalFlags = analyzeFunction(MF);
     850       14834 :   unsigned LiveMaskReg = 0;
     851       14834 :   if (!(GlobalFlags & StateWQM)) {
     852       14642 :     lowerLiveMaskQueries(AMDGPU::EXEC);
     853       14642 :     if (!(GlobalFlags & StateWWM))
     854       14629 :       return !LiveMaskQueries.empty();
     855             :   } else {
     856             :     // Store a copy of the original live mask when required
     857         192 :     MachineBasicBlock &Entry = MF.front();
     858         192 :     MachineBasicBlock::iterator EntryMI = Entry.getFirstNonPHI();
     859             : 
     860         192 :     if (GlobalFlags & StateExact || !LiveMaskQueries.empty()) {
     861          85 :       LiveMaskReg = MRI->createVirtualRegister(&AMDGPU::SReg_64RegClass);
     862         340 :       MachineInstr *MI = BuildMI(Entry, EntryMI, DebugLoc(),
     863         170 :                                  TII->get(AMDGPU::COPY), LiveMaskReg)
     864          85 :                              .addReg(AMDGPU::EXEC);
     865          85 :       LIS->InsertMachineInstrInMaps(*MI);
     866             :     }
     867             : 
     868         192 :     lowerLiveMaskQueries(LiveMaskReg);
     869             : 
     870         192 :     if (GlobalFlags == StateWQM) {
     871             :       // For a shader that needs only WQM, we can just set it once.
     872         535 :       BuildMI(Entry, EntryMI, DebugLoc(), TII->get(AMDGPU::S_WQM_B64),
     873         214 :               AMDGPU::EXEC)
     874         107 :           .addReg(AMDGPU::EXEC);
     875             : 
     876         107 :       lowerCopyInstrs();
     877             :       // EntryMI may become invalid here
     878         107 :       return true;
     879             :     }
     880             :   }
     881             : 
     882             :   DEBUG(printInfo());
     883             : 
     884          98 :   lowerCopyInstrs();
     885             : 
     886             :   // Handle the general case
     887         609 :   for (auto BII : Blocks)
     888         945 :     processBlock(*BII.first, LiveMaskReg, BII.first == &*MF.begin());
     889             : 
     890             :   // Physical registers like SCC aren't tracked by default anyway, so just
     891             :   // removing the ranges we computed is the simplest option for maintaining
     892             :   // the analysis results.
     893         294 :   LIS->removeRegUnit(*MCRegUnitIterator(AMDGPU::SCC, TRI));
     894             : 
     895          98 :   return true;
     896             : }

Generated by: LCOV version 1.13