LCOV - code coverage report
Current view: top level - lib/Target/AMDGPU - SIFormMemoryClauses.cpp (source / functions) Hit Total Coverage
Test: llvm-toolchain.info Lines: 151 159 95.0 %
Date: 2018-07-13 00:08:38 Functions: 19 22 86.4 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : //===-- SIFormMemoryClauses.cpp -------------------------------------------===//
       2             : //
       3             : //                     The LLVM Compiler Infrastructure
       4             : //
       5             : // This file is distributed under the University of Illinois Open Source
       6             : // License. See LICENSE.TXT for details.
       7             : //
       8             : //===----------------------------------------------------------------------===//
       9             : //
      10             : /// \file
      11             : /// This pass creates bundles of SMEM and VMEM instructions forming memory
      12             : /// clauses if XNACK is enabled. Def operands of clauses are marked as early
      13             : /// clobber to make sure we will not override any source within a clause.
      14             : ///
      15             : //===----------------------------------------------------------------------===//
      16             : 
      17             : #include "AMDGPU.h"
      18             : #include "AMDGPUSubtarget.h"
      19             : #include "GCNRegPressure.h"
      20             : #include "SIInstrInfo.h"
      21             : #include "SIMachineFunctionInfo.h"
      22             : #include "SIRegisterInfo.h"
      23             : #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
      24             : #include "llvm/ADT/DenseMap.h"
      25             : #include "llvm/CodeGen/LiveIntervals.h"
      26             : #include "llvm/CodeGen/MachineFunctionPass.h"
      27             : 
      28             : using namespace llvm;
      29             : 
      30             : #define DEBUG_TYPE "si-form-memory-clauses"
      31             : 
      32             : // Clauses longer then 15 instructions would overflow one of the counters
      33             : // and stall. They can stall even earlier if there are outstanding counters.
      34             : static cl::opt<unsigned>
      35      299229 : MaxClause("amdgpu-max-memory-clause", cl::Hidden, cl::init(15),
      36      199486 :           cl::desc("Maximum length of a memory clause, instructions"));
      37             : 
      38             : namespace {
      39             : 
      40        1736 : class SIFormMemoryClauses : public MachineFunctionPass {
      41             :   typedef DenseMap<unsigned, std::pair<unsigned, LaneBitmask>> RegUse;
      42             : 
      43             : public:
      44             :   static char ID;
      45             : 
      46             : public:
      47        1744 :   SIFormMemoryClauses() : MachineFunctionPass(ID) {
      48        1744 :     initializeSIFormMemoryClausesPass(*PassRegistry::getPassRegistry());
      49        1744 :   }
      50             : 
      51             :   bool runOnMachineFunction(MachineFunction &MF) override;
      52             : 
      53        1744 :   StringRef getPassName() const override {
      54        1744 :     return "SI Form memory clauses";
      55             :   }
      56             : 
      57        1744 :   void getAnalysisUsage(AnalysisUsage &AU) const override {
      58             :     AU.addRequired<LiveIntervals>();
      59             :     AU.setPreservesAll();
      60        1744 :     MachineFunctionPass::getAnalysisUsage(AU);
      61        1744 :   }
      62             : 
      63             : private:
      64             :   template <typename Callable>
      65             :   void forAllLanes(unsigned Reg, LaneBitmask LaneMask, Callable Func) const;
      66             : 
      67             :   bool canBundle(const MachineInstr &MI, RegUse &Defs, RegUse &Uses) const;
      68             :   bool checkPressure(const MachineInstr &MI, GCNDownwardRPTracker &RPT);
      69             :   void collectRegUses(const MachineInstr &MI, RegUse &Defs, RegUse &Uses) const;
      70             :   bool processRegUses(const MachineInstr &MI, RegUse &Defs, RegUse &Uses,
      71             :                       GCNDownwardRPTracker &RPT);
      72             : 
      73             :   const SISubtarget *ST;
      74             :   const SIRegisterInfo *TRI;
      75             :   const MachineRegisterInfo *MRI;
      76             :   SIMachineFunctionInfo *MFI;
      77             : 
      78             :   unsigned LastRecordedOccupancy;
      79             :   unsigned MaxVGPRs;
      80             :   unsigned MaxSGPRs;
      81             : };
      82             : 
      83             : } // End anonymous namespace.
      84             : 
      85       73254 : INITIALIZE_PASS_BEGIN(SIFormMemoryClauses, DEBUG_TYPE,
      86             :                       "SI Form memory clauses", false, false)
      87       73254 : INITIALIZE_PASS_DEPENDENCY(LiveIntervals)
      88      346058 : INITIALIZE_PASS_END(SIFormMemoryClauses, DEBUG_TYPE,
      89             :                     "SI Form memory clauses", false, false)
      90             : 
      91             : 
      92             : char SIFormMemoryClauses::ID = 0;
      93             : 
      94             : char &llvm::SIFormMemoryClausesID = SIFormMemoryClauses::ID;
      95             : 
      96           0 : FunctionPass *llvm::createSIFormMemoryClausesPass() {
      97           0 :   return new SIFormMemoryClauses();
      98             : }
      99             : 
     100             : static bool isVMEMClauseInst(const MachineInstr &MI) {
     101        1102 :   return SIInstrInfo::isFLAT(MI) || SIInstrInfo::isVMEM(MI);
     102             : }
     103             : 
     104             : static bool isSMEMClauseInst(const MachineInstr &MI) {
     105             :   return SIInstrInfo::isSMRD(MI);
     106             : }
     107             : 
     108             : // There no sense to create store clauses, they do not define anything,
     109             : // thus there is nothing to set early-clobber.
     110        1147 : static bool isValidClauseInst(const MachineInstr &MI, bool IsVMEMClause) {
     111        1147 :   if (MI.isDebugValue() || MI.isBundled())
     112             :     return false;
     113        1147 :   if (!MI.mayLoad() || MI.mayStore())
     114             :     return false;
     115         540 :   if (AMDGPU::getAtomicNoRetOp(MI.getOpcode()) != -1 ||
     116         180 :       AMDGPU::getAtomicRetOp(MI.getOpcode()) != -1)
     117             :     return false;
     118         180 :   if (IsVMEMClause && !isVMEMClauseInst(MI))
     119             :     return false;
     120         241 :   if (!IsVMEMClause && !isSMEMClauseInst(MI))
     121             :     return false;
     122             :   return true;
     123             : }
     124             : 
     125         518 : static unsigned getMopState(const MachineOperand &MO) {
     126             :   unsigned S = 0;
     127         518 :   if (MO.isImplicit())
     128             :     S |= RegState::Implicit;
     129         518 :   if (MO.isDead())
     130          32 :     S |= RegState::Dead;
     131         518 :   if (MO.isUndef())
     132          10 :     S |= RegState::Undef;
     133         518 :   if (MO.isKill())
     134           0 :     S |= RegState::Kill;
     135         518 :   if (MO.isEarlyClobber())
     136           0 :     S |= RegState::EarlyClobber;
     137        1036 :   if (TargetRegisterInfo::isPhysicalRegister(MO.getReg()) && MO.isRenamable())
     138           0 :     S |= RegState::Renamable;
     139         518 :   return S;
     140             : }
     141             : 
     142             : template <typename Callable>
     143         170 : void SIFormMemoryClauses::forAllLanes(unsigned Reg, LaneBitmask LaneMask,
     144             :                                       Callable Func) const {
     145         180 :   if (LaneMask.all() || TargetRegisterInfo::isPhysicalRegister(Reg) ||
     146           5 :       LaneMask == MRI->getMaxLaneMaskForVReg(Reg)) {
     147         111 :     Func(0);
     148         111 :     return;
     149             :   }
     150             : 
     151           1 :   const TargetRegisterClass *RC = MRI->getRegClass(Reg);
     152           1 :   unsigned E = TRI->getNumSubRegIndices();
     153             :   SmallVector<unsigned, AMDGPU::NUM_TARGET_SUBREGS> CoveringSubregs;
     154          68 :   for (unsigned Idx = 1; Idx < E; ++Idx) {
     155             :     // Is this index even compatible with the given class?
     156          67 :     if (TRI->getSubClassWithSubReg(RC, Idx) != RC)
     157          58 :       continue;
     158           9 :     LaneBitmask SubRegMask = TRI->getSubRegIndexLaneMask(Idx);
     159             :     // Early exit if we found a perfect match.
     160           9 :     if (SubRegMask == LaneMask) {
     161           0 :       Func(Idx);
     162             :       return;
     163             :     }
     164             : 
     165          18 :     if ((SubRegMask & ~LaneMask).any() || (SubRegMask & LaneMask).none())
     166           5 :       continue;
     167             : 
     168           4 :     CoveringSubregs.push_back(Idx);
     169             :   }
     170             : 
     171             :   llvm::sort(CoveringSubregs.begin(), CoveringSubregs.end(),
     172           3 :              [this](unsigned A, unsigned B) {
     173           3 :                LaneBitmask MaskA = TRI->getSubRegIndexLaneMask(A);
     174             :                LaneBitmask MaskB = TRI->getSubRegIndexLaneMask(B);
     175             :                unsigned NA = MaskA.getNumLanes();
     176             :                unsigned NB = MaskB.getNumLanes();
     177           3 :                if (NA != NB)
     178           1 :                  return NA > NB;
     179           2 :                return MaskA.getHighestLane() > MaskB.getHighestLane();
     180             :              });
     181             : 
     182           3 :   for (unsigned Idx : CoveringSubregs) {
     183           2 :     LaneBitmask SubRegMask = TRI->getSubRegIndexLaneMask(Idx);
     184           4 :     if ((SubRegMask & ~LaneMask).any() || (SubRegMask & LaneMask).none())
     185             :       continue;
     186             : 
     187           2 :     Func(Idx);
     188             :     LaneMask &= ~SubRegMask;
     189           2 :     if (LaneMask.none())
     190             :       return;
     191             :   }
     192             : 
     193           0 :   llvm_unreachable("Failed to find all subregs to cover lane mask");
     194             : }
     195             : 
     196             : // Returns false if there is a use of a def already in the map.
     197             : // In this case we must break the clause.
     198         180 : bool SIFormMemoryClauses::canBundle(const MachineInstr &MI,
     199             :                                     RegUse &Defs, RegUse &Uses) const {
     200             :   // Check interference with defs.
     201        2374 :   for (const MachineOperand &MO : MI.operands()) {
     202             :     // TODO: Prologue/Epilogue Insertion pass does not process bundled
     203             :     //       instructions.
     204        1106 :     if (MO.isFI())
     205           9 :       return false;
     206             : 
     207        1104 :     if (!MO.isReg())
     208        1097 :       continue;
     209             : 
     210         533 :     unsigned Reg = MO.getReg();
     211             : 
     212             :     // If it is tied we will need to write same register as we read.
     213         533 :     if (MO.isTied())
     214             :       return false;
     215             : 
     216         529 :     RegUse &Map = MO.isDef() ? Uses : Defs;
     217         529 :     auto Conflict = Map.find(Reg);
     218         529 :     if (Conflict == Map.end())
     219             :       continue;
     220             : 
     221           6 :     if (TargetRegisterInfo::isPhysicalRegister(Reg))
     222             :       return false;
     223             : 
     224           6 :     LaneBitmask Mask = TRI->getSubRegIndexLaneMask(MO.getSubReg());
     225           3 :     if ((Conflict->second.second & Mask).any())
     226             :       return false;
     227             :   }
     228             : 
     229             :   return true;
     230             : }
     231             : 
     232             : // Since all defs in the clause are early clobber we can run out of registers.
     233             : // Function returns false if pressure would hit the limit if instruction is
     234             : // bundled into a memory clause.
     235         171 : bool SIFormMemoryClauses::checkPressure(const MachineInstr &MI,
     236             :                                         GCNDownwardRPTracker &RPT) {
     237             :   // NB: skip advanceBeforeNext() call. Since all defs will be marked
     238             :   // early-clobber they will all stay alive at least to the end of the
     239             :   // clause. Therefor we should not decrease pressure even if load
     240             :   // pointer becomes dead and could otherwise be reused for destination.
     241         171 :   RPT.advanceToNext();
     242         171 :   GCNRegPressure MaxPressure = RPT.moveMaxPressure();
     243         171 :   unsigned Occupancy = MaxPressure.getOccupancy(*ST);
     244         341 :   if (Occupancy >= MFI->getMinAllowedOccupancy() &&
     245         341 :       MaxPressure.getVGPRNum() <= MaxVGPRs &&
     246         170 :       MaxPressure.getSGPRNum() <= MaxSGPRs) {
     247         170 :     LastRecordedOccupancy = Occupancy;
     248             :     return true;
     249             :   }
     250             :   return false;
     251             : }
     252             : 
     253             : // Collect register defs and uses along with their lane masks and states.
     254         170 : void SIFormMemoryClauses::collectRegUses(const MachineInstr &MI,
     255             :                                          RegUse &Defs, RegUse &Uses) const {
     256        2342 :   for (const MachineOperand &MO : MI.operands()) {
     257        1086 :     if (!MO.isReg())
     258         568 :       continue;
     259         518 :     unsigned Reg = MO.getReg();
     260         518 :     if (!Reg)
     261             :       continue;
     262             : 
     263             :     LaneBitmask Mask = TargetRegisterInfo::isVirtualRegister(Reg) ?
     264         362 :                          TRI->getSubRegIndexLaneMask(MO.getSubReg()) :
     265         880 :                          LaneBitmask::getAll();
     266         518 :     RegUse &Map = MO.isDef() ? Defs : Uses;
     267             : 
     268         518 :     auto Loc = Map.find(Reg);
     269         518 :     unsigned State = getMopState(MO);
     270         518 :     if (Loc == Map.end()) {
     271             :       Map[Reg] = std::make_pair(State, Mask);
     272             :     } else {
     273         177 :       Loc->second.first |= State;
     274             :       Loc->second.second |= Mask;
     275             :     }
     276             :   }
     277         170 : }
     278             : 
     279             : // Check register def/use conflicts, occupancy limits and collect def/use maps.
     280             : // Return true if instruction can be bundled with previous. It it cannot
     281             : // def/use maps are not updated.
     282         180 : bool SIFormMemoryClauses::processRegUses(const MachineInstr &MI,
     283             :                                          RegUse &Defs, RegUse &Uses,
     284             :                                          GCNDownwardRPTracker &RPT) {
     285         180 :   if (!canBundle(MI, Defs, Uses))
     286             :     return false;
     287             : 
     288         171 :   if (!checkPressure(MI, RPT))
     289             :     return false;
     290             : 
     291         170 :   collectRegUses(MI, Defs, Uses);
     292         170 :   return true;
     293             : }
     294             : 
     295       17709 : bool SIFormMemoryClauses::runOnMachineFunction(MachineFunction &MF) {
     296       17709 :   if (skipFunction(MF.getFunction()))
     297             :     return false;
     298             : 
     299       17707 :   ST = &MF.getSubtarget<SISubtarget>();
     300       17707 :   if (!ST->isXNACKEnabled())
     301             :     return false;
     302             : 
     303             :   const SIInstrInfo *TII = ST->getInstrInfo();
     304         126 :   TRI = ST->getRegisterInfo();
     305         126 :   MRI = &MF.getRegInfo();
     306         126 :   MFI = MF.getInfo<SIMachineFunctionInfo>();
     307         126 :   LiveIntervals *LIS = &getAnalysis<LiveIntervals>();
     308         126 :   SlotIndexes *Ind = LIS->getSlotIndexes();
     309             :   bool Changed = false;
     310             : 
     311         378 :   MaxVGPRs = TRI->getAllocatableSet(MF, &AMDGPU::VGPR_32RegClass).count();
     312         378 :   MaxSGPRs = TRI->getAllocatableSet(MF, &AMDGPU::SGPR_32RegClass).count();
     313             : 
     314         252 :   for (MachineBasicBlock &MBB : MF) {
     315             :     MachineBasicBlock::instr_iterator Next;
     316        1109 :     for (auto I = MBB.instr_begin(), E = MBB.instr_end(); I != E; I = Next) {
     317             :       MachineInstr &MI = *I;
     318             :       Next = std::next(I);
     319             : 
     320             :       bool IsVMEM = isVMEMClauseInst(MI);
     321             : 
     322         983 :       if (!isValidClauseInst(MI, IsVMEM))
     323        1838 :         continue;
     324             : 
     325             :       RegUse Defs, Uses;
     326             :       GCNDownwardRPTracker RPT(*LIS);
     327          93 :       RPT.reset(MI);
     328             : 
     329          93 :       if (!processRegUses(MI, Defs, Uses, RPT))
     330           6 :         continue;
     331             : 
     332             :       unsigned Length = 1;
     333         336 :       for ( ; Next != E && Length < MaxClause; ++Next) {
     334         164 :         if (!isValidClauseInst(*Next, IsVMEM))
     335             :           break;
     336             : 
     337             :         // A load from pointer which was loaded inside the same bundle is an
     338             :         // impossible clause because we will need to write and read the same
     339             :         // register inside. In this case processRegUses will return false.
     340          87 :         if (!processRegUses(*Next, Defs, Uses, RPT))
     341             :           break;
     342             : 
     343          83 :         ++Length;
     344             :       }
     345          87 :       if (Length < 2)
     346          52 :         continue;
     347             : 
     348             :       Changed = true;
     349          35 :       MFI->limitOccupancy(LastRecordedOccupancy);
     350             : 
     351         105 :       auto B = BuildMI(MBB, I, DebugLoc(), TII->get(TargetOpcode::BUNDLE));
     352          35 :       Ind->insertMachineInstrInMaps(*B);
     353             : 
     354         153 :       for (auto BI = I; BI != Next; ++BI) {
     355         118 :         BI->bundleWithPred();
     356         118 :         Ind->removeSingleMachineInstrFromMaps(*BI);
     357             : 
     358         354 :         for (MachineOperand &MO : BI->defs())
     359             :           if (MO.readsReg())
     360             :             MO.setIsInternalRead(true);
     361             :       }
     362             : 
     363         182 :       for (auto &&R : Defs) {
     364         338 :         forAllLanes(R.first, R.second.second, [&R, &B](unsigned SubReg) {
     365         113 :           unsigned S = R.second.first | RegState::EarlyClobber;
     366         113 :           if (!SubReg)
     367         111 :             S &= ~(RegState::Undef | RegState::Dead);
     368         113 :           B.addDef(R.first, S, SubReg);
     369         113 :         });
     370             :       }
     371             : 
     372         128 :       for (auto &&R : Uses) {
     373          58 :         forAllLanes(R.first, R.second.second, [&R, &B](unsigned SubReg) {
     374          58 :           B.addUse(R.first, R.second.first & ~RegState::Kill, SubReg);
     375             :         });
     376             :       }
     377             : 
     378         182 :       for (auto &&R : Defs) {
     379         112 :         unsigned Reg = R.first;
     380         112 :         Uses.erase(Reg);
     381         224 :         if (TargetRegisterInfo::isPhysicalRegister(Reg))
     382           0 :           continue;
     383         112 :         LIS->removeInterval(Reg);
     384         112 :         LIS->createAndComputeVirtRegInterval(Reg);
     385             :       }
     386             : 
     387         128 :       for (auto &&R : Uses) {
     388          58 :         unsigned Reg = R.first;
     389          58 :         if (TargetRegisterInfo::isPhysicalRegister(Reg))
     390          18 :           continue;
     391          40 :         LIS->removeInterval(Reg);
     392             :         LIS->createAndComputeVirtRegInterval(Reg);
     393             :       }
     394             :     }
     395             :   }
     396             : 
     397             :   return Changed;
     398      299229 : }

Generated by: LCOV version 1.13