LCOV - code coverage report
Current view: top level - lib/Target/AMDGPU - SIOptimizeExecMasking.cpp (source / functions) Hit Total Coverage
Test: llvm-toolchain.info Lines: 124 137 90.5 %
Date: 2017-09-14 15:23:50 Functions: 12 13 92.3 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : //===-- SIOptimizeExecMasking.cpp -----------------------------------------===//
       2             : //
       3             : //                     The LLVM Compiler Infrastructure
       4             : //
       5             : // This file is distributed under the University of Illinois Open Source
       6             : // License. See LICENSE.TXT for details.
       7             : //
       8             : //===----------------------------------------------------------------------===//
       9             : 
      10             : #include "AMDGPU.h"
      11             : #include "AMDGPUSubtarget.h"
      12             : #include "SIInstrInfo.h"
      13             : #include "llvm/CodeGen/LiveIntervalAnalysis.h"
      14             : #include "llvm/CodeGen/MachineFunctionPass.h"
      15             : #include "llvm/CodeGen/MachineInstrBuilder.h"
      16             : #include "llvm/CodeGen/MachineRegisterInfo.h"
      17             : #include "llvm/Support/Debug.h"
      18             : 
      19             : using namespace llvm;
      20             : 
      21             : #define DEBUG_TYPE "si-optimize-exec-masking"
      22             : 
      23             : namespace {
      24             : 
      25        1462 : class SIOptimizeExecMasking : public MachineFunctionPass {
      26             : public:
      27             :   static char ID;
      28             : 
      29             : public:
      30        1470 :   SIOptimizeExecMasking() : MachineFunctionPass(ID) {
      31        1470 :     initializeSIOptimizeExecMaskingPass(*PassRegistry::getPassRegistry());
      32        1470 :   }
      33             : 
      34             :   bool runOnMachineFunction(MachineFunction &MF) override;
      35             : 
      36        1465 :   StringRef getPassName() const override {
      37        1465 :     return "SI optimize exec mask operations";
      38             :   }
      39             : 
      40        1465 :   void getAnalysisUsage(AnalysisUsage &AU) const override {
      41        1465 :     AU.setPreservesCFG();
      42        1465 :     MachineFunctionPass::getAnalysisUsage(AU);
      43        1465 :   }
      44             : };
      45             : 
      46             : } // End anonymous namespace.
      47             : 
      48       53042 : INITIALIZE_PASS_BEGIN(SIOptimizeExecMasking, DEBUG_TYPE,
      49             :                       "SI optimize exec mask operations", false, false)
      50       53042 : INITIALIZE_PASS_DEPENDENCY(LiveIntervals)
      51      263906 : INITIALIZE_PASS_END(SIOptimizeExecMasking, DEBUG_TYPE,
      52             :                     "SI optimize exec mask operations", false, false)
      53             : 
      54             : char SIOptimizeExecMasking::ID = 0;
      55             : 
      56             : char &llvm::SIOptimizeExecMaskingID = SIOptimizeExecMasking::ID;
      57             : 
      58             : /// If \p MI is a copy from exec, return the register copied to.
      59             : static unsigned isCopyFromExec(const MachineInstr &MI) {
      60        1231 :   switch (MI.getOpcode()) {
      61         690 :   case AMDGPU::COPY:
      62             :   case AMDGPU::S_MOV_B64:
      63             :   case AMDGPU::S_MOV_B64_term: {
      64         690 :     const MachineOperand &Src = MI.getOperand(1);
      65         690 :     if (Src.isReg() && Src.getReg() == AMDGPU::EXEC)
      66         333 :       return MI.getOperand(0).getReg();
      67             :   }
      68             :   }
      69             : 
      70             :   return AMDGPU::NoRegister;
      71             : }
      72             : 
      73             : /// If \p MI is a copy to exec, return the register copied from.
      74             : static unsigned isCopyToExec(const MachineInstr &MI) {
      75       16140 :   switch (MI.getOpcode()) {
      76         647 :   case AMDGPU::COPY:
      77             :   case AMDGPU::S_MOV_B64: {
      78         647 :     const MachineOperand &Dst = MI.getOperand(0);
      79         647 :     if (Dst.isReg() && Dst.getReg() == AMDGPU::EXEC)
      80         351 :       return MI.getOperand(1).getReg();
      81             :     break;
      82             :   }
      83           0 :   case AMDGPU::S_MOV_B64_term:
      84           0 :     llvm_unreachable("should have been replaced");
      85             :   }
      86             : 
      87             :   return AMDGPU::NoRegister;
      88             : }
      89             : 
      90             : /// If \p MI is a logical operation on an exec value,
      91             : /// return the register copied to.
      92          15 : static unsigned isLogicalOpOnExec(const MachineInstr &MI) {
      93          30 :   switch (MI.getOpcode()) {
      94          11 :   case AMDGPU::S_AND_B64:
      95             :   case AMDGPU::S_OR_B64:
      96             :   case AMDGPU::S_XOR_B64:
      97             :   case AMDGPU::S_ANDN2_B64:
      98             :   case AMDGPU::S_ORN2_B64:
      99             :   case AMDGPU::S_NAND_B64:
     100             :   case AMDGPU::S_NOR_B64:
     101             :   case AMDGPU::S_XNOR_B64: {
     102          11 :     const MachineOperand &Src1 = MI.getOperand(1);
     103          11 :     if (Src1.isReg() && Src1.getReg() == AMDGPU::EXEC)
     104          10 :       return MI.getOperand(0).getReg();
     105           1 :     const MachineOperand &Src2 = MI.getOperand(2);
     106           1 :     if (Src2.isReg() && Src2.getReg() == AMDGPU::EXEC)
     107           1 :       return MI.getOperand(0).getReg();
     108             :   }
     109             :   }
     110             : 
     111             :   return AMDGPU::NoRegister;
     112             : }
     113             : 
     114         641 : static unsigned getSaveExecOp(unsigned Opc) {
     115         641 :   switch (Opc) {
     116             :   case AMDGPU::S_AND_B64:
     117             :     return AMDGPU::S_AND_SAVEEXEC_B64;
     118           2 :   case AMDGPU::S_OR_B64:
     119           2 :     return AMDGPU::S_OR_SAVEEXEC_B64;
     120           0 :   case AMDGPU::S_XOR_B64:
     121           0 :     return AMDGPU::S_XOR_SAVEEXEC_B64;
     122           3 :   case AMDGPU::S_ANDN2_B64:
     123           3 :     return AMDGPU::S_ANDN2_SAVEEXEC_B64;
     124           0 :   case AMDGPU::S_ORN2_B64:
     125           0 :     return AMDGPU::S_ORN2_SAVEEXEC_B64;
     126           0 :   case AMDGPU::S_NAND_B64:
     127           0 :     return AMDGPU::S_NAND_SAVEEXEC_B64;
     128           0 :   case AMDGPU::S_NOR_B64:
     129           0 :     return AMDGPU::S_NOR_SAVEEXEC_B64;
     130           0 :   case AMDGPU::S_XNOR_B64:
     131           0 :     return AMDGPU::S_XNOR_SAVEEXEC_B64;
     132           1 :   default:
     133           1 :     return AMDGPU::INSTRUCTION_LIST_END;
     134             :   }
     135             : }
     136             : 
     137             : // These are only terminators to get correct spill code placement during
     138             : // register allocation, so turn them back into normal instructions. Only one of
     139             : // these is expected per block.
     140             : static bool removeTerminatorBit(const SIInstrInfo &TII, MachineInstr &MI) {
     141       17546 :   switch (MI.getOpcode()) {
     142         335 :   case AMDGPU::S_MOV_B64_term: {
     143        1005 :     MI.setDesc(TII.get(AMDGPU::COPY));
     144             :     return true;
     145             :   }
     146          45 :   case AMDGPU::S_XOR_B64_term: {
     147             :     // This is only a terminator to get the correct spill code placement during
     148             :     // register allocation.
     149         135 :     MI.setDesc(TII.get(AMDGPU::S_XOR_B64));
     150             :     return true;
     151             :   }
     152          44 :   case AMDGPU::S_ANDN2_B64_term: {
     153             :     // This is only a terminator to get the correct spill code placement during
     154             :     // register allocation.
     155         132 :     MI.setDesc(TII.get(AMDGPU::S_ANDN2_B64));
     156             :     return true;
     157             :   }
     158             :   default:
     159             :     return false;
     160             :   }
     161             : }
     162             : 
     163       16965 : static MachineBasicBlock::reverse_iterator fixTerminators(
     164             :   const SIInstrInfo &TII,
     165             :   MachineBasicBlock &MBB) {
     166       33930 :   MachineBasicBlock::reverse_iterator I = MBB.rbegin(), E = MBB.rend();
     167       34087 :   for (; I != E; ++I) {
     168       66524 :     if (!I->isTerminator())
     169       15716 :       return I;
     170             : 
     171       35092 :     if (removeTerminatorBit(TII, *I))
     172         424 :       return I;
     173             :   }
     174             : 
     175         825 :   return E;
     176             : }
     177             : 
     178         351 : static MachineBasicBlock::reverse_iterator findExecCopy(
     179             :   const SIInstrInfo &TII,
     180             :   MachineBasicBlock &MBB,
     181             :   MachineBasicBlock::reverse_iterator I,
     182             :   unsigned CopyToExec) {
     183         351 :   const unsigned InstLimit = 25;
     184             : 
     185         351 :   auto E = MBB.rend();
     186        2498 :   for (unsigned N = 0; N <= InstLimit && I != E; ++I, ++N) {
     187        1564 :     unsigned CopyFromExec = isCopyFromExec(*I);
     188         333 :     if (CopyFromExec != AMDGPU::NoRegister)
     189         333 :       return I;
     190             :   }
     191             : 
     192          18 :   return E;
     193             : }
     194             : 
     195             : // XXX - Seems LivePhysRegs doesn't work correctly since it will incorrectly
     196             : // repor tthe register as unavailable because a super-register with a lane mask
     197             : // as unavailable.
     198         333 : static bool isLiveOut(const MachineBasicBlock &MBB, unsigned Reg) {
     199        1325 :   for (MachineBasicBlock *Succ : MBB.successors()) {
     200         661 :     if (Succ->isLiveIn(Reg))
     201             :       return true;
     202             :   }
     203             : 
     204             :   return false;
     205             : }
     206             : 
     207       14855 : bool SIOptimizeExecMasking::runOnMachineFunction(MachineFunction &MF) {
     208       14855 :   const SISubtarget &ST = MF.getSubtarget<SISubtarget>();
     209       14855 :   const SIRegisterInfo *TRI = ST.getRegisterInfo();
     210       14855 :   const SIInstrInfo *TII = ST.getInstrInfo();
     211             : 
     212             :   // Optimize sequences emitted for control flow lowering. They are originally
     213             :   // emitted as the separate operations because spill code may need to be
     214             :   // inserted for the saved copy of exec.
     215             :   //
     216             :   //     x = copy exec
     217             :   //     z = s_<op>_b64 x, y
     218             :   //     exec = copy z
     219             :   // =>
     220             :   //     x = s_<op>_saveexec_b64 y
     221             :   //
     222             : 
     223       61530 :   for (MachineBasicBlock &MBB : MF) {
     224       16965 :     MachineBasicBlock::reverse_iterator I = fixTerminators(*TII, MBB);
     225       16965 :     MachineBasicBlock::reverse_iterator E = MBB.rend();
     226       16965 :     if (I == E)
     227       17479 :       continue;
     228             : 
     229       16491 :     unsigned CopyToExec = isCopyToExec(*I);
     230       16140 :     if (CopyToExec == AMDGPU::NoRegister)
     231       15789 :       continue;
     232             : 
     233             :     // Scan backwards to find the def.
     234         351 :     auto CopyToExecInst = &*I;
     235         351 :     auto CopyFromExecInst = findExecCopy(*TII, MBB, I, CopyToExec);
     236         351 :     if (CopyFromExecInst == E) {
     237          18 :       auto PrepareExecInst = std::next(I);
     238          18 :       if (PrepareExecInst == E)
     239           2 :         continue;
     240             :       // Fold exec = COPY (S_AND_B64 reg, exec) -> exec = S_AND_B64 reg, exec
     241          47 :       if (CopyToExecInst->getOperand(1).isKill() &&
     242          15 :           isLogicalOpOnExec(*PrepareExecInst) == CopyToExec) {
     243             :         DEBUG(dbgs() << "Fold exec copy: " << *PrepareExecInst);
     244             : 
     245          11 :         PrepareExecInst->getOperand(0).setReg(AMDGPU::EXEC);
     246             : 
     247             :         DEBUG(dbgs() << "into: " << *PrepareExecInst << '\n');
     248             : 
     249          11 :         CopyToExecInst->eraseFromParent();
     250             :       }
     251             : 
     252          16 :       continue;
     253             :     }
     254             : 
     255         333 :     if (isLiveOut(MBB, CopyToExec)) {
     256             :       // The copied register is live out and has a second use in another block.
     257             :       DEBUG(dbgs() << "Exec copy source register is live out\n");
     258           2 :       continue;
     259             :     }
     260             : 
     261         331 :     unsigned CopyFromExec = CopyFromExecInst->getOperand(0).getReg();
     262         331 :     MachineInstr *SaveExecInst = nullptr;
     263         641 :     SmallVector<MachineInstr *, 4> OtherUseInsts;
     264             : 
     265             :     for (MachineBasicBlock::iterator J
     266        1986 :            = std::next(CopyFromExecInst->getIterator()), JE = I->getIterator();
     267         762 :          J != JE; ++J) {
     268         659 :       if (SaveExecInst && J->readsRegister(AMDGPU::EXEC, TRI)) {
     269             :         DEBUG(dbgs() << "exec read prevents saveexec: " << *J << '\n');
     270             :         // Make sure this is inserted after any VALU ops that may have been
     271             :         // scheduled in between.
     272             :         SaveExecInst = nullptr;
     273             :         break;
     274             :       }
     275             : 
     276         866 :       if (J->modifiesRegister(CopyToExec, TRI)) {
     277         332 :         if (SaveExecInst) {
     278             :           DEBUG(dbgs() << "Multiple instructions modify "
     279             :                 << PrintReg(CopyToExec, TRI) << '\n');
     280             :           SaveExecInst = nullptr;
     281             :           break;
     282             :         }
     283             : 
     284         662 :         unsigned SaveExecOp = getSaveExecOp(J->getOpcode());
     285         331 :         if (SaveExecOp == AMDGPU::INSTRUCTION_LIST_END)
     286             :           break;
     287             : 
     288         990 :         if (J->readsRegister(CopyFromExec, TRI)) {
     289         330 :           SaveExecInst = &*J;
     290             :           DEBUG(dbgs() << "Found save exec op: " << *SaveExecInst << '\n');
     291         330 :           continue;
     292             :         } else {
     293             :           DEBUG(dbgs() << "Instruction does not read exec copy: " << *J << '\n');
     294             :           break;
     295             :         }
     296             :       }
     297             : 
     298         271 :       if (SaveExecInst && J->readsRegister(CopyToExec, TRI)) {
     299             :         assert(SaveExecInst != &*J);
     300          85 :         OtherUseInsts.push_back(&*J);
     301             :       }
     302             :     }
     303             : 
     304         331 :     if (!SaveExecInst)
     305             :       continue;
     306             : 
     307             :     DEBUG(dbgs() << "Insert save exec op: " << *SaveExecInst << '\n');
     308             : 
     309         622 :     MachineOperand &Src0 = SaveExecInst->getOperand(1);
     310         622 :     MachineOperand &Src1 = SaveExecInst->getOperand(2);
     311             : 
     312         311 :     MachineOperand *OtherOp = nullptr;
     313             : 
     314         311 :     if (Src0.isReg() && Src0.getReg() == CopyFromExec) {
     315             :       OtherOp = &Src1;
     316           1 :     } else if (Src1.isReg() && Src1.getReg() == CopyFromExec) {
     317           1 :       if (!SaveExecInst->isCommutable())
     318             :         break;
     319             : 
     320             :       OtherOp = &Src0;
     321             :     } else
     322           0 :       llvm_unreachable("unexpected");
     323             : 
     324         310 :     CopyFromExecInst->eraseFromParent();
     325             : 
     326         620 :     auto InsPt = SaveExecInst->getIterator();
     327         310 :     const DebugLoc &DL = SaveExecInst->getDebugLoc();
     328             : 
     329         620 :     BuildMI(MBB, InsPt, DL, TII->get(getSaveExecOp(SaveExecInst->getOpcode())),
     330         930 :             CopyFromExec)
     331         310 :       .addReg(OtherOp->getReg());
     332         310 :     SaveExecInst->eraseFromParent();
     333             : 
     334         310 :     CopyToExecInst->eraseFromParent();
     335             : 
     336        1002 :     for (MachineInstr *OtherInst : OtherUseInsts) {
     337          72 :       OtherInst->substituteRegister(CopyToExec, AMDGPU::EXEC,
     338             :                                     AMDGPU::NoSubRegister, *TRI);
     339             :     }
     340             :   }
     341             : 
     342       14855 :   return true;
     343             : 
     344             : }

Generated by: LCOV version 1.13