LCOV - code coverage report
Current view: top level - lib/Target/AMDGPU - R600ControlFlowFinalizer.cpp (source / functions) Hit Total Coverage
Test: llvm-toolchain.info Lines: 277 303 91.4 %
Date: 2018-07-13 00:08:38 Functions: 19 20 95.0 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : //===- R600ControlFlowFinalizer.cpp - Finalize Control Flow Inst ----------===//
       2             : //
       3             : //                     The LLVM Compiler Infrastructure
       4             : //
       5             : // This file is distributed under the University of Illinois Open Source
       6             : // License. See LICENSE.TXT for details.
       7             : //
       8             : //===----------------------------------------------------------------------===//
       9             : //
      10             : /// \file
      11             : /// This pass compute turns all control flow pseudo instructions into native one
      12             : /// computing their address on the fly; it also sets STACK_SIZE info.
      13             : //
      14             : //===----------------------------------------------------------------------===//
      15             : 
      16             : #include "AMDGPU.h"
      17             : #include "AMDGPUSubtarget.h"
      18             : #include "R600Defines.h"
      19             : #include "R600InstrInfo.h"
      20             : #include "R600MachineFunctionInfo.h"
      21             : #include "R600RegisterInfo.h"
      22             : #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
      23             : #include "llvm/ADT/STLExtras.h"
      24             : #include "llvm/ADT/SmallVector.h"
      25             : #include "llvm/ADT/StringRef.h"
      26             : #include "llvm/CodeGen/MachineBasicBlock.h"
      27             : #include "llvm/CodeGen/MachineFunction.h"
      28             : #include "llvm/CodeGen/MachineFunctionPass.h"
      29             : #include "llvm/CodeGen/MachineInstr.h"
      30             : #include "llvm/CodeGen/MachineInstrBuilder.h"
      31             : #include "llvm/CodeGen/MachineOperand.h"
      32             : #include "llvm/IR/CallingConv.h"
      33             : #include "llvm/IR/DebugLoc.h"
      34             : #include "llvm/IR/Function.h"
      35             : #include "llvm/Pass.h"
      36             : #include "llvm/Support/Compiler.h"
      37             : #include "llvm/Support/Debug.h"
      38             : #include "llvm/Support/MathExtras.h"
      39             : #include "llvm/Support/raw_ostream.h"
      40             : #include <algorithm>
      41             : #include <cassert>
      42             : #include <cstdint>
      43             : #include <set>
      44             : #include <utility>
      45             : #include <vector>
      46             : 
      47             : using namespace llvm;
      48             : 
      49             : #define DEBUG_TYPE "r600cf"
      50             : 
      51             : namespace {
      52             : 
      53        4476 : struct CFStack {
      54             :   enum StackItem {
      55             :     ENTRY = 0,
      56             :     SUB_ENTRY = 1,
      57             :     FIRST_NON_WQM_PUSH = 2,
      58             :     FIRST_NON_WQM_PUSH_W_FULL_ENTRY = 3
      59             :   };
      60             : 
      61             :   const R600Subtarget *ST;
      62             :   std::vector<StackItem> BranchStack;
      63             :   std::vector<StackItem> LoopStack;
      64             :   unsigned MaxStackSize;
      65             :   unsigned CurrentEntries = 0;
      66             :   unsigned CurrentSubEntries = 0;
      67             : 
      68        2238 :   CFStack(const R600Subtarget *st, CallingConv::ID cc) : ST(st),
      69             :       // We need to reserve a stack entry for CALL_FS in vertex shaders.
      70        4476 :       MaxStackSize(cc == CallingConv::AMDGPU_VS ? 1 : 0) {}
      71             : 
      72             :   unsigned getLoopDepth();
      73             :   bool branchStackContains(CFStack::StackItem);
      74             :   bool requiresWorkAroundForInst(unsigned Opcode);
      75             :   unsigned getSubEntrySize(CFStack::StackItem Item);
      76             :   void updateMaxStackSize();
      77             :   void pushBranch(unsigned Opcode, bool isWQM = false);
      78             :   void pushLoop();
      79             :   void popBranch();
      80             :   void popLoop();
      81             : };
      82             : 
      83             : unsigned CFStack::getLoopDepth() {
      84          16 :   return LoopStack.size();
      85             : }
      86             : 
      87             : bool CFStack::branchStackContains(CFStack::StackItem Item) {
      88             :   for (std::vector<CFStack::StackItem>::const_iterator I = BranchStack.begin(),
      89          54 :        E = BranchStack.end(); I != E; ++I) {
      90          10 :     if (*I == Item)
      91             :       return true;
      92             :   }
      93             :   return false;
      94             : }
      95             : 
      96        9069 : bool CFStack::requiresWorkAroundForInst(unsigned Opcode) {
      97        9077 :   if (Opcode == R600::CF_ALU_PUSH_BEFORE && ST->hasCaymanISA() &&
      98             :       getLoopDepth() > 1)
      99             :     return true;
     100             : 
     101        9068 :   if (!ST->hasCFAluBug())
     102             :     return false;
     103             : 
     104             :   switch(Opcode) {
     105             :   default: return false;
     106          45 :   case R600::CF_ALU_PUSH_BEFORE:
     107             :   case R600::CF_ALU_ELSE_AFTER:
     108             :   case R600::CF_ALU_BREAK:
     109             :   case R600::CF_ALU_CONTINUE:
     110          45 :     if (CurrentSubEntries == 0)
     111             :       return false;
     112          10 :     if (ST->getWavefrontSize() == 64) {
     113             :       // We are being conservative here.  We only require this work-around if
     114             :       // CurrentSubEntries > 3 &&
     115             :       // (CurrentSubEntries % 4 == 3 || CurrentSubEntries % 4 == 0)
     116             :       //
     117             :       // We have to be conservative, because we don't know for certain that
     118             :       // our stack allocation algorithm for Evergreen/NI is correct.  Applying this
     119             :       // work-around when CurrentSubEntries > 3 allows us to over-allocate stack
     120             :       // resources without any problems.
     121           8 :       return CurrentSubEntries > 3;
     122             :     } else {
     123             :       assert(ST->getWavefrontSize() == 32);
     124             :       // We are being conservative here.  We only require the work-around if
     125             :       // CurrentSubEntries > 7 &&
     126             :       // (CurrentSubEntries % 8 == 7 || CurrentSubEntries % 8 == 0)
     127             :       // See the comment on the wavefront size == 64 case for why we are
     128             :       // being conservative.
     129           2 :       return CurrentSubEntries > 7;
     130             :     }
     131             :   }
     132             : }
     133             : 
     134             : unsigned CFStack::getSubEntrySize(CFStack::StackItem Item) {
     135         124 :   switch(Item) {
     136             :   default:
     137             :     return 0;
     138          88 :   case CFStack::FIRST_NON_WQM_PUSH:
     139             :   assert(!ST->hasCaymanISA());
     140          88 :   if (ST->getGeneration() <= R600Subtarget::R700) {
     141             :     // +1 For the push operation.
     142             :     // +2 Extra space required.
     143             :     return 3;
     144             :   } else {
     145             :     // Some documentation says that this is not necessary on Evergreen,
     146             :     // but experimentation has show that we need to allocate 1 extra
     147             :     // sub-entry for the first non-WQM push.
     148             :     // +1 For the push operation.
     149             :     // +1 Extra space required.
     150             :     return 2;
     151             :   }
     152           0 :   case CFStack::FIRST_NON_WQM_PUSH_W_FULL_ENTRY:
     153             :     assert(ST->getGeneration() >= R600Subtarget::EVERGREEN);
     154             :     // +1 For the push operation.
     155             :     // +1 Extra space required.
     156             :     return 2;
     157          36 :   case CFStack::SUB_ENTRY:
     158             :     return 1;
     159             :   }
     160             : }
     161             : 
     162             : void CFStack::updateMaxStackSize() {
     163             :   unsigned CurrentStackSize =
     164         160 :       CurrentEntries + (alignTo(CurrentSubEntries, 4) / 4);
     165         160 :   MaxStackSize = std::max(CurrentStackSize, MaxStackSize);
     166             : }
     167             : 
     168          62 : void CFStack::pushBranch(unsigned Opcode, bool isWQM) {
     169          62 :   CFStack::StackItem Item = CFStack::ENTRY;
     170          62 :   switch(Opcode) {
     171          62 :   case R600::CF_PUSH_EG:
     172             :   case R600::CF_ALU_PUSH_BEFORE:
     173          62 :     if (!isWQM) {
     174         116 :       if (!ST->hasCaymanISA() &&
     175             :           !branchStackContains(CFStack::FIRST_NON_WQM_PUSH))
     176          44 :         Item = CFStack::FIRST_NON_WQM_PUSH;  // May not be required on Evergreen/NI
     177             :                                              // See comment in
     178             :                                              // CFStack::getSubEntrySize()
     179          31 :       else if (CurrentEntries > 0 &&
     180          18 :                ST->getGeneration() > R600Subtarget::EVERGREEN &&
     181          18 :                !ST->hasCaymanISA() &&
     182             :                !branchStackContains(CFStack::FIRST_NON_WQM_PUSH_W_FULL_ENTRY))
     183           0 :         Item = CFStack::FIRST_NON_WQM_PUSH_W_FULL_ENTRY;
     184             :       else
     185          18 :         Item = CFStack::SUB_ENTRY;
     186             :     } else
     187             :       Item = CFStack::ENTRY;
     188             :     break;
     189             :   }
     190          62 :   BranchStack.push_back(Item);
     191          62 :   if (Item == CFStack::ENTRY)
     192           0 :     CurrentEntries++;
     193             :   else
     194          62 :     CurrentSubEntries += getSubEntrySize(Item);
     195             :   updateMaxStackSize();
     196          62 : }
     197             : 
     198          18 : void CFStack::pushLoop() {
     199          36 :   LoopStack.push_back(CFStack::ENTRY);
     200          18 :   CurrentEntries++;
     201             :   updateMaxStackSize();
     202          18 : }
     203             : 
     204          62 : void CFStack::popBranch() {
     205          62 :   CFStack::StackItem Top = BranchStack.back();
     206          62 :   if (Top == CFStack::ENTRY)
     207           0 :     CurrentEntries--;
     208             :   else
     209          62 :     CurrentSubEntries-= getSubEntrySize(Top);
     210             :   BranchStack.pop_back();
     211          62 : }
     212             : 
     213             : void CFStack::popLoop() {
     214          18 :   CurrentEntries--;
     215             :   LoopStack.pop_back();
     216             : }
     217             : 
     218         277 : class R600ControlFlowFinalizer : public MachineFunctionPass {
     219             : private:
     220             :   using ClauseFile = std::pair<MachineInstr *, std::vector<MachineInstr *>>;
     221             : 
     222             :   enum ControlFlowInstruction {
     223             :     CF_TC,
     224             :     CF_VC,
     225             :     CF_CALL_FS,
     226             :     CF_WHILE_LOOP,
     227             :     CF_END_LOOP,
     228             :     CF_LOOP_BREAK,
     229             :     CF_LOOP_CONTINUE,
     230             :     CF_JUMP,
     231             :     CF_ELSE,
     232             :     CF_POP,
     233             :     CF_END
     234             :   };
     235             : 
     236             :   const R600InstrInfo *TII = nullptr;
     237             :   const R600RegisterInfo *TRI = nullptr;
     238             :   unsigned MaxFetchInst;
     239             :   const R600Subtarget *ST = nullptr;
     240             : 
     241             :   bool IsTrivialInst(MachineInstr &MI) const {
     242       65368 :     switch (MI.getOpcode()) {
     243             :     case R600::KILL:
     244             :     case R600::RETURN:
     245             :       return true;
     246             :     default:
     247             :       return false;
     248             :     }
     249             :   }
     250             : 
     251        1208 :   const MCInstrDesc &getHWInstrDesc(ControlFlowInstruction CFI) const {
     252             :     unsigned Opcode = 0;
     253        3330 :     bool isEg = (ST->getGeneration() >= R600Subtarget::EVERGREEN);
     254        1208 :     switch (CFI) {
     255        1208 :     case CF_TC:
     256        1208 :       Opcode = isEg ? R600::CF_TC_EG : R600::CF_TC_R600;
     257             :       break;
     258           0 :     case CF_VC:
     259           0 :       Opcode = isEg ? R600::CF_VC_EG : R600::CF_VC_R600;
     260             :       break;
     261           0 :     case CF_CALL_FS:
     262          15 :       Opcode = isEg ? R600::CF_CALL_FS_EG : R600::CF_CALL_FS_R600;
     263             :       break;
     264           0 :     case CF_WHILE_LOOP:
     265          18 :       Opcode = isEg ? R600::WHILE_LOOP_EG : R600::WHILE_LOOP_R600;
     266             :       break;
     267           0 :     case CF_END_LOOP:
     268          18 :       Opcode = isEg ? R600::END_LOOP_EG : R600::END_LOOP_R600;
     269             :       break;
     270           0 :     case CF_LOOP_BREAK:
     271          18 :       Opcode = isEg ? R600::LOOP_BREAK_EG : R600::LOOP_BREAK_R600;
     272             :       break;
     273           0 :     case CF_LOOP_CONTINUE:
     274           0 :       Opcode = isEg ? R600::CF_CONTINUE_EG : R600::CF_CONTINUE_R600;
     275             :       break;
     276           0 :     case CF_JUMP:
     277          62 :       Opcode = isEg ? R600::CF_JUMP_EG : R600::CF_JUMP_R600;
     278             :       break;
     279           0 :     case CF_ELSE:
     280           3 :       Opcode = isEg ? R600::CF_ELSE_EG : R600::CF_ELSE_R600;
     281             :       break;
     282           0 :     case CF_POP:
     283          38 :       Opcode = isEg ? R600::POP_EG : R600::POP_R600;
     284             :       break;
     285           0 :     case CF_END:
     286        1950 :       if (ST->hasCaymanISA()) {
     287             :         Opcode = R600::CF_END_CM;
     288             :         break;
     289             :       }
     290        1664 :       Opcode = isEg ? R600::CF_END_EG : R600::CF_END_R600;
     291             :       break;
     292             :     }
     293             :     assert (Opcode && "No opcode selected");
     294        4538 :     return TII->get(Opcode);
     295             :   }
     296             : 
     297        1854 :   bool isCompatibleWithClause(const MachineInstr &MI,
     298             :                               std::set<unsigned> &DstRegs) const {
     299             :     unsigned DstMI, SrcMI;
     300       13650 :     for (MachineInstr::const_mop_iterator I = MI.operands_begin(),
     301        1854 :                                           E = MI.operands_end();
     302       13650 :          I != E; ++I) {
     303             :       const MachineOperand &MO = *I;
     304       11796 :       if (!MO.isReg())
     305             :         continue;
     306        3708 :       if (MO.isDef()) {
     307        1854 :         unsigned Reg = MO.getReg();
     308        3708 :         if (R600::R600_Reg128RegClass.contains(Reg))
     309         810 :           DstMI = Reg;
     310             :         else
     311        2088 :           DstMI = TRI->getMatchingSuperReg(Reg,
     312             :               AMDGPURegisterInfo::getSubRegFromChannel(TRI->getHWRegChan(Reg)),
     313             :               &R600::R600_Reg128RegClass);
     314             :       }
     315        3708 :       if (MO.isUse()) {
     316        1854 :         unsigned Reg = MO.getReg();
     317        3708 :         if (R600::R600_Reg128RegClass.contains(Reg))
     318             :           SrcMI = Reg;
     319             :         else
     320        1562 :           SrcMI = TRI->getMatchingSuperReg(Reg,
     321             :               AMDGPURegisterInfo::getSubRegFromChannel(TRI->getHWRegChan(Reg)),
     322             :               &R600::R600_Reg128RegClass);
     323             :       }
     324             :     }
     325        1854 :     if ((DstRegs.find(SrcMI) == DstRegs.end())) {
     326             :       DstRegs.insert(DstMI);
     327             :       return true;
     328             :     } else
     329             :       return false;
     330             :   }
     331             : 
     332             :   ClauseFile
     333        1208 :   MakeFetchClause(MachineBasicBlock &MBB, MachineBasicBlock::iterator &I)
     334             :       const {
     335        1208 :     MachineBasicBlock::iterator ClauseHead = I;
     336             :     std::vector<MachineInstr *> ClauseContent;
     337             :     unsigned AluInstCount = 0;
     338        1208 :     bool IsTex = TII->usesTextureCache(*ClauseHead);
     339             :     std::set<unsigned> DstRegs;
     340        3045 :     for (MachineBasicBlock::iterator E = MBB.end(); I != E; ++I) {
     341           0 :       if (IsTrivialInst(*I))
     342           0 :         continue;
     343        3045 :       if (AluInstCount >= MaxFetchInst)
     344             :         break;
     345        3045 :       if ((IsTex && !TII->usesTextureCache(*I)) ||
     346           0 :           (!IsTex && !TII->usesVertexCache(*I)))
     347             :         break;
     348        1854 :       if (!isCompatibleWithClause(*I, DstRegs))
     349             :         break;
     350        1837 :       AluInstCount ++;
     351        3674 :       ClauseContent.push_back(&*I);
     352             :     }
     353        2416 :     MachineInstr *MIb = BuildMI(MBB, ClauseHead, MBB.findDebugLoc(ClauseHead),
     354        1208 :         getHWInstrDesc(IsTex?CF_TC:CF_VC))
     355             :         .addImm(0) // ADDR
     356        1208 :         .addImm(AluInstCount - 1); // COUNT
     357        1208 :     return ClauseFile(MIb, std::move(ClauseContent));
     358             :   }
     359             : 
     360       49559 :   void getLiteral(MachineInstr &MI, std::vector<MachineOperand *> &Lits) const {
     361             :     static const unsigned LiteralRegs[] = {
     362             :       R600::ALU_LITERAL_X,
     363             :       R600::ALU_LITERAL_Y,
     364             :       R600::ALU_LITERAL_Z,
     365             :       R600::ALU_LITERAL_W
     366             :     };
     367             :     const SmallVector<std::pair<MachineOperand *, int64_t>, 3> Srcs =
     368       49559 :         TII->getSrcs(MI);
     369      241533 :     for (const auto &Src:Srcs) {
     370       95987 :       if (Src.first->getReg() != R600::ALU_LITERAL_X)
     371       75180 :         continue;
     372       20807 :       int64_t Imm = Src.second;
     373             :       std::vector<MachineOperand *>::iterator It =
     374             :           llvm::find_if(Lits, [&](MachineOperand *val) {
     375       10390 :             return val->isImm() && (val->getImm() == Imm);
     376             :           });
     377             : 
     378             :       // Get corresponding Operand
     379             :       MachineOperand &Operand = MI.getOperand(
     380       41614 :           TII->getOperandIdx(MI.getOpcode(), R600::OpName::literal));
     381             : 
     382       20807 :       if (It != Lits.end()) {
     383             :         // Reuse existing literal reg
     384        1860 :         unsigned Index = It - Lits.begin();
     385        1860 :         Src.first->setReg(LiteralRegs[Index]);
     386             :       } else {
     387             :         // Allocate new literal reg
     388             :         assert(Lits.size() < 4 && "Too many literals in Instruction Group");
     389       37894 :         Src.first->setReg(LiteralRegs[Lits.size()]);
     390       37894 :         Lits.push_back(&Operand);
     391             :       }
     392             :     }
     393       49559 :   }
     394             : 
     395             :   MachineBasicBlock::iterator insertLiterals(
     396             :       MachineBasicBlock::iterator InsertPos,
     397             :       const std::vector<unsigned> &Literals) const {
     398             :     MachineBasicBlock *MBB = InsertPos->getParent();
     399             :     for (unsigned i = 0, e = Literals.size(); i < e; i+=2) {
     400             :       unsigned LiteralPair0 = Literals[i];
     401             :       unsigned LiteralPair1 = (i + 1 < e)?Literals[i + 1]:0;
     402             :       InsertPos = BuildMI(MBB, InsertPos->getDebugLoc(),
     403             :           TII->get(R600::LITERALS))
     404             :           .addImm(LiteralPair0)
     405             :           .addImm(LiteralPair1);
     406             :     }
     407             :     return InsertPos;
     408             :   }
     409             : 
     410             :   ClauseFile
     411        3681 :   MakeALUClause(MachineBasicBlock &MBB, MachineBasicBlock::iterator &I)
     412             :       const {
     413             :     MachineInstr &ClauseHead = *I;
     414             :     std::vector<MachineInstr *> ClauseContent;
     415             :     I++;
     416       29925 :     for (MachineBasicBlock::instr_iterator E = MBB.instr_end(); I != E;) {
     417         286 :       if (IsTrivialInst(*I)) {
     418             :         ++I;
     419         286 :         continue;
     420             :       }
     421       29353 :       if (!I->isBundle() && !TII->isALUInstr(I->getOpcode()))
     422             :         break;
     423             :       std::vector<MachineOperand *>Literals;
     424       25958 :       if (I->isBundle()) {
     425             :         MachineInstr &DeleteMI = *I;
     426             :         MachineBasicBlock::instr_iterator BI = I.getInstrIterator();
     427      138893 :         while (++BI != E && BI->isBundledWithPred()) {
     428       37219 :           BI->unbundleFromPred();
     429     1484573 :           for (MachineOperand &MO : BI->operands()) {
     430      873056 :             if (MO.isReg() && MO.isInternalRead())
     431             :               MO.setIsInternalRead(false);
     432             :           }
     433       37219 :           getLiteral(*BI, Literals);
     434       74438 :           ClauseContent.push_back(&*BI);
     435             :         }
     436       13618 :         I = BI;
     437       13618 :         DeleteMI.eraseFromParent();
     438             :       } else {
     439       12340 :         getLiteral(*I, Literals);
     440       24680 :         ClauseContent.push_back(&*I);
     441             :         I++;
     442             :       }
     443       65486 :       for (unsigned i = 0, e = Literals.size(); i < e; i += 2) {
     444             :         MachineInstrBuilder MILit = BuildMI(MBB, I, I->getDebugLoc(),
     445       27140 :             TII->get(R600::LITERALS));
     446       40710 :         if (Literals[i]->isImm()) {
     447       13555 :             MILit.addImm(Literals[i]->getImm());
     448             :         } else {
     449             :             MILit.addGlobalAddress(Literals[i]->getGlobal(),
     450          30 :                                    Literals[i]->getOffset());
     451             :         }
     452       13570 :         if (i + 1 < e) {
     453       16131 :           if (Literals[i + 1]->isImm()) {
     454        5377 :             MILit.addImm(Literals[i + 1]->getImm());
     455             :           } else {
     456             :             MILit.addGlobalAddress(Literals[i + 1]->getGlobal(),
     457           0 :                                    Literals[i + 1]->getOffset());
     458             :           }
     459             :         } else
     460             :           MILit.addImm(0);
     461       27140 :         ClauseContent.push_back(MILit);
     462             :       }
     463             :     }
     464             :     assert(ClauseContent.size() < 128 && "ALU clause is too big");
     465        7362 :     ClauseHead.getOperand(7).setImm(ClauseContent.size() - 1);
     466        3681 :     return ClauseFile(&ClauseHead, std::move(ClauseContent));
     467             :   }
     468             : 
     469        1195 :   void EmitFetchClause(MachineBasicBlock::iterator InsertPos,
     470             :                        const DebugLoc &DL, ClauseFile &Clause,
     471             :                        unsigned &CfCount) {
     472        1195 :     CounterPropagateAddr(*Clause.first, CfCount);
     473        1195 :     MachineBasicBlock *BB = Clause.first->getParent();
     474        2390 :     BuildMI(BB, DL, TII->get(R600::FETCH_CLAUSE)).addImm(CfCount);
     475        4194 :     for (unsigned i = 0, e = Clause.second.size(); i < e; ++i) {
     476        5412 :       BB->splice(InsertPos, BB, Clause.second[i]);
     477             :     }
     478        2390 :     CfCount += 2 * Clause.second.size();
     479        1195 :   }
     480             : 
     481        3304 :   void EmitALUClause(MachineBasicBlock::iterator InsertPos, const DebugLoc &DL,
     482             :                      ClauseFile &Clause, unsigned &CfCount) {
     483        3304 :     Clause.first->getOperand(0).setImm(0);
     484        3304 :     CounterPropagateAddr(*Clause.first, CfCount);
     485        3304 :     MachineBasicBlock *BB = Clause.first->getParent();
     486        6608 :     BuildMI(BB, DL, TII->get(R600::ALU_CLAUSE)).addImm(CfCount);
     487       54258 :     for (unsigned i = 0, e = Clause.second.size(); i < e; ++i) {
     488      142950 :       BB->splice(InsertPos, BB, Clause.second[i]);
     489             :     }
     490        6608 :     CfCount += Clause.second.size();
     491        3304 :   }
     492             : 
     493             :   void CounterPropagateAddr(MachineInstr &MI, unsigned Addr) const {
     494        4600 :     MI.getOperand(0).setImm(Addr + MI.getOperand(0).getImm());
     495             :   }
     496          18 :   void CounterPropagateAddr(const std::set<MachineInstr *> &MIs,
     497             :                             unsigned Addr) const {
     498          54 :     for (MachineInstr *MI : MIs) {
     499             :       CounterPropagateAddr(*MI, Addr);
     500             :     }
     501          18 :   }
     502             : 
     503             : public:
     504             :   static char ID;
     505             : 
     506         278 :   R600ControlFlowFinalizer() : MachineFunctionPass(ID) {}
     507             : 
     508        2238 :   bool runOnMachineFunction(MachineFunction &MF) override {
     509        2238 :     ST = &MF.getSubtarget<R600Subtarget>();
     510        2238 :     MaxFetchInst = ST->getTexVTXClauseSize();
     511        2238 :     TII = ST->getInstrInfo();
     512        2238 :     TRI = ST->getRegisterInfo();
     513             : 
     514        2238 :     R600MachineFunctionInfo *MFI = MF.getInfo<R600MachineFunctionInfo>();
     515             : 
     516        6714 :     CFStack CFStack(ST, MF.getFunction().getCallingConv());
     517        4476 :     for (MachineFunction::iterator MB = MF.begin(), ME = MF.end(); MB != ME;
     518             :         ++MB) {
     519             :       MachineBasicBlock &MBB = *MB;
     520        2238 :       unsigned CfCount = 0;
     521        2238 :       std::vector<std::pair<unsigned, std::set<MachineInstr *>>> LoopStack;
     522             :       std::vector<MachineInstr * > IfThenElseStack;
     523        4476 :       if (MF.getFunction().getCallingConv() == CallingConv::AMDGPU_VS) {
     524          30 :         BuildMI(MBB, MBB.begin(), MBB.findDebugLoc(MBB.begin()),
     525          15 :             getHWInstrDesc(CF_CALL_FS));
     526          15 :         CfCount++;
     527             :       }
     528        2238 :       std::vector<ClauseFile> FetchClauses, AluClauses;
     529        2238 :       std::vector<MachineInstr *> LastAlu(1);
     530             :       std::vector<MachineInstr *> ToPopAfter;
     531             : 
     532        2238 :       for (MachineBasicBlock::iterator I = MBB.begin(), E = MBB.end();
     533       12515 :           I != E;) {
     534       20554 :         if (TII->usesTextureCache(*I) || TII->usesVertexCache(*I)) {
     535             :           LLVM_DEBUG(dbgs() << CfCount << ":"; I->dump(););
     536        2416 :           FetchClauses.push_back(MakeFetchClause(MBB, I));
     537        1208 :           CfCount++;
     538        1208 :           LastAlu.back() = nullptr;
     539             :           continue;
     540             :         }
     541             : 
     542        9069 :         MachineBasicBlock::iterator MI = I;
     543       18138 :         if (MI->getOpcode() != R600::ENDIF)
     544        9007 :           LastAlu.back() = nullptr;
     545        9069 :         if (MI->getOpcode() == R600::CF_ALU)
     546        3619 :           LastAlu.back() = &*MI;
     547             :         I++;
     548             :         bool RequiresWorkAround =
     549       18138 :             CFStack.requiresWorkAroundForInst(MI->getOpcode());
     550        9069 :         switch (MI->getOpcode()) {
     551          62 :         case R600::CF_ALU_PUSH_BEFORE:
     552          62 :           if (RequiresWorkAround) {
     553             :             LLVM_DEBUG(dbgs()
     554             :                        << "Applying bug work-around for ALU_PUSH_BEFORE\n");
     555           3 :             BuildMI(MBB, MI, MBB.findDebugLoc(MI), TII->get(R600::CF_PUSH_EG))
     556           1 :                 .addImm(CfCount + 1)
     557             :                 .addImm(1);
     558           1 :             MI->setDesc(TII->get(R600::CF_ALU));
     559           1 :             CfCount++;
     560           1 :             CFStack.pushBranch(R600::CF_PUSH_EG);
     561             :           } else
     562          61 :             CFStack.pushBranch(R600::CF_ALU_PUSH_BEFORE);
     563             :           LLVM_FALLTHROUGH;
     564             :         case R600::CF_ALU:
     565        3681 :           I = MI;
     566        7362 :           AluClauses.push_back(MakeALUClause(MBB, I));
     567             :           LLVM_DEBUG(dbgs() << CfCount << ":"; MI->dump(););
     568        3681 :           CfCount++;
     569        3681 :           break;
     570          18 :         case R600::WHILELOOP: {
     571          18 :           CFStack.pushLoop();
     572          54 :           MachineInstr *MIb = BuildMI(MBB, MI, MBB.findDebugLoc(MI),
     573          18 :               getHWInstrDesc(CF_WHILE_LOOP))
     574          18 :               .addImm(1);
     575             :           std::pair<unsigned, std::set<MachineInstr *>> Pair(CfCount,
     576          18 :               std::set<MachineInstr *>());
     577             :           Pair.second.insert(MIb);
     578             :           LoopStack.push_back(std::move(Pair));
     579          18 :           MI->eraseFromParent();
     580          18 :           CfCount++;
     581             :           break;
     582             :         }
     583             :         case R600::ENDLOOP: {
     584             :           CFStack.popLoop();
     585             :           std::pair<unsigned, std::set<MachineInstr *>> Pair =
     586             :               std::move(LoopStack.back());
     587             :           LoopStack.pop_back();
     588          18 :           CounterPropagateAddr(Pair.second, CfCount);
     589          72 :           BuildMI(MBB, MI, MBB.findDebugLoc(MI), getHWInstrDesc(CF_END_LOOP))
     590          18 :               .addImm(Pair.first + 1);
     591          18 :           MI->eraseFromParent();
     592          18 :           CfCount++;
     593             :           break;
     594             :         }
     595          62 :         case R600::IF_PREDICATE_SET: {
     596         124 :           LastAlu.push_back(nullptr);
     597         124 :           MachineInstr *MIb = BuildMI(MBB, MI, MBB.findDebugLoc(MI),
     598          62 :               getHWInstrDesc(CF_JUMP))
     599             :               .addImm(0)
     600          62 :               .addImm(0);
     601          62 :           IfThenElseStack.push_back(MIb);
     602             :           LLVM_DEBUG(dbgs() << CfCount << ":"; MIb->dump(););
     603          62 :           MI->eraseFromParent();
     604          62 :           CfCount++;
     605             :           break;
     606             :         }
     607             :         case R600::ELSE: {
     608           3 :           MachineInstr * JumpInst = IfThenElseStack.back();
     609             :           IfThenElseStack.pop_back();
     610           3 :           CounterPropagateAddr(*JumpInst, CfCount);
     611           6 :           MachineInstr *MIb = BuildMI(MBB, MI, MBB.findDebugLoc(MI),
     612           3 :               getHWInstrDesc(CF_ELSE))
     613             :               .addImm(0)
     614           3 :               .addImm(0);
     615             :           LLVM_DEBUG(dbgs() << CfCount << ":"; MIb->dump(););
     616           3 :           IfThenElseStack.push_back(MIb);
     617           3 :           MI->eraseFromParent();
     618           3 :           CfCount++;
     619             :           break;
     620             :         }
     621          62 :         case R600::ENDIF: {
     622          62 :           CFStack.popBranch();
     623          62 :           if (LastAlu.back()) {
     624          24 :             ToPopAfter.push_back(LastAlu.back());
     625             :           } else {
     626          76 :             MachineInstr *MIb = BuildMI(MBB, MI, MBB.findDebugLoc(MI),
     627          38 :                 getHWInstrDesc(CF_POP))
     628          38 :                 .addImm(CfCount + 1)
     629             :                 .addImm(1);
     630             :             (void)MIb;
     631             :             LLVM_DEBUG(dbgs() << CfCount << ":"; MIb->dump(););
     632          38 :             CfCount++;
     633             :           }
     634             : 
     635          62 :           MachineInstr *IfOrElseInst = IfThenElseStack.back();
     636             :           IfThenElseStack.pop_back();
     637          62 :           CounterPropagateAddr(*IfOrElseInst, CfCount);
     638             :           IfOrElseInst->getOperand(1).setImm(1);
     639             :           LastAlu.pop_back();
     640          62 :           MI->eraseFromParent();
     641          62 :           break;
     642             :         }
     643          18 :         case R600::BREAK: {
     644          18 :           CfCount ++;
     645          54 :           MachineInstr *MIb = BuildMI(MBB, MI, MBB.findDebugLoc(MI),
     646          18 :               getHWInstrDesc(CF_LOOP_BREAK))
     647          18 :               .addImm(0);
     648             :           LoopStack.back().second.insert(MIb);
     649          18 :           MI->eraseFromParent();
     650             :           break;
     651             :         }
     652           0 :         case R600::CONTINUE: {
     653           0 :           MachineInstr *MIb = BuildMI(MBB, MI, MBB.findDebugLoc(MI),
     654           0 :               getHWInstrDesc(CF_LOOP_CONTINUE))
     655           0 :               .addImm(0);
     656             :           LoopStack.back().second.insert(MIb);
     657           0 :           MI->eraseFromParent();
     658           0 :           CfCount++;
     659             :           break;
     660             :         }
     661        1950 :         case R600::RETURN: {
     662             :           DebugLoc DL = MBB.findDebugLoc(MI);
     663        3900 :           BuildMI(MBB, MI, DL, getHWInstrDesc(CF_END));
     664        1950 :           CfCount++;
     665        1950 :           if (CfCount % 2) {
     666        3216 :             BuildMI(MBB, I, DL, TII->get(R600::PAD));
     667        1608 :             CfCount++;
     668             :           }
     669        1950 :           MI->eraseFromParent();
     670        5095 :           for (unsigned i = 0, e = FetchClauses.size(); i < e; i++)
     671        2390 :             EmitFetchClause(I, DL, FetchClauses[i], CfCount);
     672        7204 :           for (unsigned i = 0, e = AluClauses.size(); i < e; i++)
     673        6608 :             EmitALUClause(I, DL, AluClauses[i], CfCount);
     674             :           break;
     675             :         }
     676        3257 :         default:
     677        3257 :           if (TII->isExport(MI->getOpcode())) {
     678             :             LLVM_DEBUG(dbgs() << CfCount << ":"; MI->dump(););
     679        3240 :             CfCount++;
     680             :           }
     681             :           break;
     682             :         }
     683             :       }
     684        4500 :       for (unsigned i = 0, e = ToPopAfter.size(); i < e; ++i) {
     685          48 :         MachineInstr *Alu = ToPopAfter[i];
     686          24 :         BuildMI(MBB, Alu, MBB.findDebugLoc((MachineBasicBlock::iterator)Alu),
     687          24 :             TII->get(R600::CF_ALU_POP_AFTER))
     688          24 :             .addImm(Alu->getOperand(0).getImm())
     689          24 :             .addImm(Alu->getOperand(1).getImm())
     690          24 :             .addImm(Alu->getOperand(2).getImm())
     691          24 :             .addImm(Alu->getOperand(3).getImm())
     692          24 :             .addImm(Alu->getOperand(4).getImm())
     693          24 :             .addImm(Alu->getOperand(5).getImm())
     694          24 :             .addImm(Alu->getOperand(6).getImm())
     695          24 :             .addImm(Alu->getOperand(7).getImm())
     696          24 :             .addImm(Alu->getOperand(8).getImm());
     697          24 :         Alu->eraseFromParent();
     698             :       }
     699        2238 :       MFI->CFStackSize = CFStack.MaxStackSize;
     700             :     }
     701             : 
     702        2238 :     return false;
     703             :   }
     704             : 
     705         278 :   StringRef getPassName() const override {
     706         278 :     return "R600 Control Flow Finalizer Pass";
     707             :   }
     708             : };
     709             : 
     710             : } // end anonymous namespace
     711             : 
     712       73254 : INITIALIZE_PASS_BEGIN(R600ControlFlowFinalizer, DEBUG_TYPE,
     713             :                      "R600 Control Flow Finalizer", false, false)
     714      342570 : INITIALIZE_PASS_END(R600ControlFlowFinalizer, DEBUG_TYPE,
     715             :                     "R600 Control Flow Finalizer", false, false)
     716             : 
     717             : char R600ControlFlowFinalizer::ID = 0;
     718             : 
     719             : char &llvm::R600ControlFlowFinalizerID = R600ControlFlowFinalizer::ID;
     720             : 
     721         278 : FunctionPass *llvm::createR600ControlFlowFinalizer() {
     722         556 :   return new R600ControlFlowFinalizer();
     723             : }

Generated by: LCOV version 1.13