LCOV - code coverage report
Current view: top level - lib/Target/AMDGPU - R600Packetizer.cpp (source / functions) Hit Total Coverage
Test: llvm-toolchain.info Lines: 155 157 98.7 %
Date: 2018-07-13 00:08:38 Functions: 16 19 84.2 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : //===----- R600Packetizer.cpp - VLIW packetizer ---------------------------===//
       2             : //
       3             : //                     The LLVM Compiler Infrastructure
       4             : //
       5             : // This file is distributed under the University of Illinois Open Source
       6             : // License. See LICENSE.TXT for details.
       7             : //
       8             : //===----------------------------------------------------------------------===//
       9             : //
      10             : /// \file
      11             : /// This pass implements instructions packetization for R600. It unsets isLast
      12             : /// bit of instructions inside a bundle and substitutes src register with
      13             : /// PreviousVector when applicable.
      14             : //
      15             : //===----------------------------------------------------------------------===//
      16             : 
      17             : #include "AMDGPU.h"
      18             : #include "AMDGPUSubtarget.h"
      19             : #include "R600InstrInfo.h"
      20             : #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
      21             : #include "llvm/CodeGen/DFAPacketizer.h"
      22             : #include "llvm/CodeGen/MachineDominators.h"
      23             : #include "llvm/CodeGen/MachineFunctionPass.h"
      24             : #include "llvm/CodeGen/MachineLoopInfo.h"
      25             : #include "llvm/CodeGen/Passes.h"
      26             : #include "llvm/CodeGen/ScheduleDAG.h"
      27             : #include "llvm/Support/Debug.h"
      28             : #include "llvm/Support/raw_ostream.h"
      29             : 
      30             : using namespace llvm;
      31             : 
      32             : #define DEBUG_TYPE "packets"
      33             : 
      34             : namespace {
      35             : 
      36         277 : class R600Packetizer : public MachineFunctionPass {
      37             : 
      38             : public:
      39             :   static char ID;
      40         278 :   R600Packetizer() : MachineFunctionPass(ID) {}
      41             : 
      42         278 :   void getAnalysisUsage(AnalysisUsage &AU) const override {
      43         278 :     AU.setPreservesCFG();
      44             :     AU.addRequired<MachineDominatorTree>();
      45             :     AU.addPreserved<MachineDominatorTree>();
      46             :     AU.addRequired<MachineLoopInfo>();
      47             :     AU.addPreserved<MachineLoopInfo>();
      48         278 :     MachineFunctionPass::getAnalysisUsage(AU);
      49         278 :   }
      50             : 
      51         278 :   StringRef getPassName() const override { return "R600 Packetizer"; }
      52             : 
      53             :   bool runOnMachineFunction(MachineFunction &Fn) override;
      54             : };
      55             : 
      56        2238 : class R600PacketizerList : public VLIWPacketizerList {
      57             : private:
      58             :   const R600InstrInfo *TII;
      59             :   const R600RegisterInfo &TRI;
      60             :   bool VLIW5;
      61             :   bool ConsideredInstUsesAlreadyWrittenVectorElement;
      62             : 
      63             :   unsigned getSlot(const MachineInstr &MI) const {
      64      258137 :     return TRI.getHWRegChan(MI.getOperand(0).getReg());
      65             :   }
      66             : 
      67             :   /// \returns register to PV chan mapping for bundle/single instructions that
      68             :   /// immediately precedes I.
      69       45816 :   DenseMap<unsigned, unsigned> getPreviousVector(MachineBasicBlock::iterator I)
      70             :       const {
      71             :     DenseMap<unsigned, unsigned> Result;
      72             :     I--;
      73      168326 :     if (!TII->isALUInstr(I->getOpcode()) && !I->isBundle())
      74             :       return Result;
      75             :     MachineBasicBlock::instr_iterator BI = I.getInstrIterator();
      76       40220 :     if (I->isBundle())
      77             :       BI++;
      78             :     int LastDstChan = -1;
      79             :     do {
      80             :       bool isTrans = false;
      81      197698 :       int BISlot = getSlot(*BI);
      82       98849 :       if (LastDstChan >= BISlot)
      83             :         isTrans = true;
      84             :       LastDstChan = BISlot;
      85       98849 :       if (TII->isPredicated(*BI))
      86       25756 :         continue;
      87      196814 :       int OperandIdx = TII->getOperandIdx(BI->getOpcode(), R600::OpName::write);
      88      172672 :       if (OperandIdx > -1 && BI->getOperand(OperandIdx).getImm() == 0)
      89         588 :         continue;
      90      195638 :       int DstIdx = TII->getOperandIdx(BI->getOpcode(), R600::OpName::dst);
      91      101970 :       if (DstIdx == -1) {
      92        4151 :         continue;
      93             :       }
      94      187336 :       unsigned Dst = BI->getOperand(DstIdx).getReg();
      95      112180 :       if (isTrans || TII->isTransOnly(*BI)) {
      96       18512 :         Result[Dst] = R600::PS;
      97       18512 :         continue;
      98             :       }
      99      225484 :       if (BI->getOpcode() == R600::DOT4_r600 ||
     100             :           BI->getOpcode() == R600::DOT4_eg) {
     101          24 :         Result[Dst] = R600::PV_X;
     102          24 :         continue;
     103             :       }
     104       76729 :       if (Dst == R600::OQAP) {
     105        1597 :         continue;
     106             :       }
     107             :       unsigned PVReg = 0;
     108       73535 :       switch (TRI.getHWRegChan(Dst)) {
     109             :       case 0:
     110             :         PVReg = R600::PV_X;
     111             :         break;
     112       14292 :       case 1:
     113             :         PVReg = R600::PV_Y;
     114       14292 :         break;
     115       15499 :       case 2:
     116             :         PVReg = R600::PV_Z;
     117       15499 :         break;
     118       28704 :       case 3:
     119             :         PVReg = R600::PV_W;
     120       28704 :         break;
     121           0 :       default:
     122           0 :         llvm_unreachable("Invalid Chan");
     123             :       }
     124       73535 :       Result[Dst] = PVReg;
     125       98849 :     } while ((++BI)->isBundledWithPred());
     126             :     return Result;
     127             :   }
     128             : 
     129       42196 :   void substitutePV(MachineInstr &MI, const DenseMap<unsigned, unsigned> &PVs)
     130             :       const {
     131       42196 :     unsigned Ops[] = {
     132             :       R600::OpName::src0,
     133             :       R600::OpName::src1,
     134             :       R600::OpName::src2
     135             :     };
     136      295372 :     for (unsigned i = 0; i < 3; i++) {
     137      253176 :       int OperandIdx = TII->getOperandIdx(MI.getOpcode(), Ops[i]);
     138      126588 :       if (OperandIdx < 0)
     139       43477 :         continue;
     140      166222 :       unsigned Src = MI.getOperand(OperandIdx).getReg();
     141       83111 :       const DenseMap<unsigned, unsigned>::const_iterator It = PVs.find(Src);
     142       83111 :       if (It != PVs.end())
     143       45728 :         MI.getOperand(OperandIdx).setReg(It->second);
     144             :     }
     145       42196 :   }
     146             : public:
     147             :   // Ctor.
     148             :   R600PacketizerList(MachineFunction &MF, const R600Subtarget &ST,
     149             :                      MachineLoopInfo &MLI)
     150        2238 :       : VLIWPacketizerList(MF, MLI, nullptr),
     151             :         TII(ST.getInstrInfo()),
     152        6714 :         TRI(TII->getRegisterInfo()) {
     153        2238 :     VLIW5 = !ST.hasCaymanISA();
     154             :   }
     155             : 
     156             :   // initPacketizerState - initialize some internal flags.
     157       60396 :   void initPacketizerState() override {
     158       60396 :     ConsideredInstUsesAlreadyWrittenVectorElement = false;
     159       60396 :   }
     160             : 
     161             :   // ignorePseudoInstruction - Ignore bundling of pseudo instructions.
     162       45816 :   bool ignorePseudoInstruction(const MachineInstr &MI,
     163             :                                const MachineBasicBlock *MBB) override {
     164       45816 :     return false;
     165             :   }
     166             : 
     167             :   // isSoloInstruction - return true if instruction MI can not be packetized
     168             :   // with any other instruction, which means that MI itself is a packet.
     169       60396 :   bool isSoloInstruction(const MachineInstr &MI) override {
     170       60396 :     if (TII->isVector(MI))
     171             :       return true;
     172      120792 :     if (!TII->isALUInstr(MI.getOpcode()))
     173             :       return true;
     174       98458 :     if (MI.getOpcode() == R600::GROUP_BARRIER)
     175             :       return true;
     176             :     // XXX: This can be removed once the packetizer properly handles all the
     177             :     // LDS instruction group restrictions.
     178       49225 :     return TII->isLDSInstr(MI.getOpcode());
     179             :   }
     180             : 
     181             :   // isLegalToPacketizeTogether - Is it legal to packetize SUI and SUJ
     182             :   // together.
     183       53128 :   bool isLegalToPacketizeTogether(SUnit *SUI, SUnit *SUJ) override {
     184       53128 :     MachineInstr *MII = SUI->getInstr(), *MIJ = SUJ->getInstr();
     185      159384 :     if (getSlot(*MII) == getSlot(*MIJ))
     186       11893 :       ConsideredInstUsesAlreadyWrittenVectorElement = true;
     187             :     // Does MII and MIJ share the same pred_sel ?
     188      106256 :     int OpI = TII->getOperandIdx(MII->getOpcode(), R600::OpName::pred_sel),
     189      106256 :         OpJ = TII->getOperandIdx(MIJ->getOpcode(), R600::OpName::pred_sel);
     190      106256 :     unsigned PredI = (OpI > -1)?MII->getOperand(OpI).getReg():0,
     191      106256 :         PredJ = (OpJ > -1)?MIJ->getOperand(OpJ).getReg():0;
     192       53128 :     if (PredI != PredJ)
     193             :       return false;
     194       53092 :     if (SUJ->isSucc(SUI)) {
     195       37686 :       for (unsigned i = 0, e = SUJ->Succs.size(); i < e; ++i) {
     196       31920 :         const SDep &Dep = SUJ->Succs[i];
     197       31920 :         if (Dep.getSUnit() != SUI)
     198       20827 :           continue;
     199       11093 :         if (Dep.getKind() == SDep::Anti)
     200        5764 :           continue;
     201        5329 :         if (Dep.getKind() == SDep::Output)
     202         855 :           if (MII->getOperand(0).getReg() != MIJ->getOperand(0).getReg())
     203           8 :             continue;
     204             :         return false;
     205             :       }
     206             :     }
     207             : 
     208             :     bool ARDef =
     209       47771 :         TII->definesAddressRegister(*MII) || TII->definesAddressRegister(*MIJ);
     210             :     bool ARUse =
     211       47771 :         TII->usesAddressRegister(*MII) || TII->usesAddressRegister(*MIJ);
     212             : 
     213       47771 :     return !ARDef || !ARUse;
     214             :   }
     215             : 
     216             :   // isLegalToPruneDependencies - Is it legal to prune dependece between SUI
     217             :   // and SUJ.
     218        5494 :   bool isLegalToPruneDependencies(SUnit *SUI, SUnit *SUJ) override {
     219        5494 :     return false;
     220             :   }
     221             : 
     222             :   void setIsLastBit(MachineInstr *MI, unsigned Bit) const {
     223       46710 :     unsigned LastOp = TII->getOperandIdx(MI->getOpcode(), R600::OpName::last);
     224       23355 :     MI->getOperand(LastOp).setImm(Bit);
     225             :   }
     226             : 
     227       45816 :   bool isBundlableWithCurrentPMI(MachineInstr &MI,
     228             :                                  const DenseMap<unsigned, unsigned> &PV,
     229             :                                  std::vector<R600InstrInfo::BankSwizzle> &BS,
     230             :                                  bool &isTransSlot) {
     231       45816 :     isTransSlot = TII->isTransOnly(MI);
     232             :     assert (!isTransSlot || VLIW5);
     233             : 
     234             :     // Is the dst reg sequence legal ?
     235       90673 :     if (!isTransSlot && !CurrentPacketMIs.empty()) {
     236       79548 :       if (getSlot(MI) <= getSlot(*CurrentPacketMIs.back())) {
     237       20669 :         if (ConsideredInstUsesAlreadyWrittenVectorElement &&
     238       20377 :             !TII->isVectorOnly(MI) && VLIW5) {
     239        8607 :           isTransSlot = true;
     240             :           LLVM_DEBUG({
     241             :             dbgs() << "Considering as Trans Inst :";
     242             :             MI.dump();
     243             :           });
     244             :         }
     245             :         else
     246             :           return false;
     247             :       }
     248             :     }
     249             : 
     250             :     // Are the Constants limitations met ?
     251       86116 :     CurrentPacketMIs.push_back(&MI);
     252       43058 :     if (!TII->fitsConstReadLimitations(CurrentPacketMIs)) {
     253             :       LLVM_DEBUG({
     254             :         dbgs() << "Couldn't pack :\n";
     255             :         MI.dump();
     256             :         dbgs() << "with the following packets :\n";
     257             :         for (unsigned i = 0, e = CurrentPacketMIs.size() - 1; i < e; i++) {
     258             :           CurrentPacketMIs[i]->dump();
     259             :           dbgs() << "\n";
     260             :         }
     261             :         dbgs() << "because of Consts read limitations\n";
     262             :       });
     263             :       CurrentPacketMIs.pop_back();
     264         341 :       return false;
     265             :     }
     266             : 
     267             :     // Is there a BankSwizzle set that meet Read Port limitations ?
     268       85434 :     if (!TII->fitsReadPortLimitations(CurrentPacketMIs,
     269       42717 :             PV, BS, isTransSlot)) {
     270             :       LLVM_DEBUG({
     271             :         dbgs() << "Couldn't pack :\n";
     272             :         MI.dump();
     273             :         dbgs() << "with the following packets :\n";
     274             :         for (unsigned i = 0, e = CurrentPacketMIs.size() - 1; i < e; i++) {
     275             :           CurrentPacketMIs[i]->dump();
     276             :           dbgs() << "\n";
     277             :         }
     278             :         dbgs() << "because of Read port limitations\n";
     279             :       });
     280             :       CurrentPacketMIs.pop_back();
     281         521 :       return false;
     282             :     }
     283             : 
     284             :     // We cannot read LDS source registers from the Trans slot.
     285       42196 :     if (isTransSlot && TII->readsLDSSrcReg(MI))
     286             :       return false;
     287             : 
     288             :     CurrentPacketMIs.pop_back();
     289       42196 :     return true;
     290             :   }
     291             : 
     292       45816 :   MachineBasicBlock::iterator addToPacket(MachineInstr &MI) override {
     293             :     MachineBasicBlock::iterator FirstInBundle =
     294       45816 :         CurrentPacketMIs.empty() ? &MI : CurrentPacketMIs.front();
     295             :     const DenseMap<unsigned, unsigned> &PV =
     296       45816 :         getPreviousVector(FirstInBundle);
     297             :     std::vector<R600InstrInfo::BankSwizzle> BS;
     298             :     bool isTransSlot;
     299             : 
     300       45816 :     if (isBundlableWithCurrentPMI(MI, PV, BS, isTransSlot)) {
     301      124131 :       for (unsigned i = 0, e = CurrentPacketMIs.size(); i < e; i++) {
     302       79478 :         MachineInstr *MI = CurrentPacketMIs[i];
     303       79478 :         unsigned Op = TII->getOperandIdx(MI->getOpcode(),
     304       39739 :             R600::OpName::bank_swizzle);
     305      119217 :         MI->getOperand(Op).setImm(BS[i]);
     306             :       }
     307             :       unsigned Op =
     308       84392 :           TII->getOperandIdx(MI.getOpcode(), R600::OpName::bank_swizzle);
     309       84392 :       MI.getOperand(Op).setImm(BS.back());
     310       42196 :       if (!CurrentPacketMIs.empty())
     311       23355 :         setIsLastBit(CurrentPacketMIs.back(), 0);
     312       42196 :       substitutePV(MI, PV);
     313       42196 :       MachineBasicBlock::iterator It = VLIWPacketizerList::addToPacket(MI);
     314       42196 :       if (isTransSlot) {
     315        9191 :         endPacket(std::next(It)->getParent(), std::next(It));
     316             :       }
     317       42196 :       return It;
     318             :     }
     319        7240 :     endPacket(MI.getParent(), MI);
     320        3620 :     if (TII->isTransOnly(MI))
     321          11 :       return MI;
     322        3609 :     return VLIWPacketizerList::addToPacket(MI);
     323             :   }
     324             : };
     325             : 
     326        2238 : bool R600Packetizer::runOnMachineFunction(MachineFunction &Fn) {
     327        2238 :   const R600Subtarget &ST = Fn.getSubtarget<R600Subtarget>();
     328             :   const R600InstrInfo *TII = ST.getInstrInfo();
     329             : 
     330        2238 :   MachineLoopInfo &MLI = getAnalysis<MachineLoopInfo>();
     331             : 
     332             :   // Instantiate the packetizer.
     333             :   R600PacketizerList Packetizer(Fn, ST, MLI);
     334             : 
     335             :   // DFA state table should not be empty.
     336             :   assert(Packetizer.getResourceTracker() && "Empty DFA table!");
     337             :   assert(Packetizer.getResourceTracker()->getInstrItins());
     338             : 
     339        2238 :   if (Packetizer.getResourceTracker()->getInstrItins()->isEmpty())
     340             :     return false;
     341             : 
     342             :   //
     343             :   // Loop over all basic blocks and remove KILL pseudo-instructions
     344             :   // These instructions confuse the dependence analysis. Consider:
     345             :   // D0 = ...   (Insn 0)
     346             :   // R0 = KILL R0, D0 (Insn 1)
     347             :   // R0 = ... (Insn 2)
     348             :   // Here, Insn 1 will result in the dependence graph not emitting an output
     349             :   // dependence between Insn 0 and Insn 2. This can lead to incorrect
     350             :   // packetization
     351             :   //
     352             :   for (MachineFunction::iterator MBB = Fn.begin(), MBBe = Fn.end();
     353        4474 :        MBB != MBBe; ++MBB) {
     354             :     MachineBasicBlock::iterator End = MBB->end();
     355        2237 :     MachineBasicBlock::iterator MI = MBB->begin();
     356       62740 :     while (MI != End) {
     357      121006 :       if (MI->isKill() || MI->getOpcode() == R600::IMPLICIT_DEF ||
     358        3618 :           (MI->getOpcode() == R600::CF_ALU && !MI->getOperand(8).getImm())) {
     359          17 :         MachineBasicBlock::iterator DeleteMI = MI;
     360             :         ++MI;
     361          17 :         MBB->erase(DeleteMI);
     362             :         End = MBB->end();
     363          17 :         continue;
     364             :       }
     365             :       ++MI;
     366             :     }
     367             :   }
     368             : 
     369             :   // Loop over all of the basic blocks.
     370             :   for (MachineFunction::iterator MBB = Fn.begin(), MBBe = Fn.end();
     371        4474 :        MBB != MBBe; ++MBB) {
     372             :     // Find scheduling regions and schedule / packetize each region.
     373             :     unsigned RemainingCount = MBB->size();
     374             :     for(MachineBasicBlock::iterator RegionEnd = MBB->end();
     375        4472 :         RegionEnd != MBB->begin();) {
     376             :       // The next region starts above the previous region. Look backward in the
     377             :       // instruction stream until we find the nearest boundary.
     378        2235 :       MachineBasicBlock::iterator I = RegionEnd;
     379        2235 :       for(;I != MBB->begin(); --I, --RemainingCount) {
     380        4470 :         if (TII->isSchedulingBoundary(*std::prev(I), &*MBB, Fn))
     381             :           break;
     382             :       }
     383        2235 :       I = MBB->begin();
     384             : 
     385             :       // Skip empty scheduling regions.
     386        2235 :       if (I == RegionEnd) {
     387             :         RegionEnd = std::prev(RegionEnd);
     388             :         --RemainingCount;
     389          90 :         continue;
     390             :       }
     391             :       // Skip regions with one instruction.
     392        2325 :       if (I == std::prev(RegionEnd)) {
     393             :         RegionEnd = std::prev(RegionEnd);
     394          90 :         continue;
     395             :       }
     396             : 
     397        2145 :       Packetizer.PacketizeMIs(&*MBB, &*I, RegionEnd);
     398        2145 :       RegionEnd = I;
     399             :     }
     400             :   }
     401             : 
     402             :   return true;
     403             : 
     404             : }
     405             : 
     406             : } // end anonymous namespace
     407             : 
     408       73254 : INITIALIZE_PASS_BEGIN(R600Packetizer, DEBUG_TYPE,
     409             :                      "R600 Packetizer", false, false)
     410      342570 : INITIALIZE_PASS_END(R600Packetizer, DEBUG_TYPE,
     411             :                     "R600 Packetizer", false, false)
     412             : 
     413             : char R600Packetizer::ID = 0;
     414             : 
     415             : char &llvm::R600PacketizerID = R600Packetizer::ID;
     416             : 
     417         278 : llvm::FunctionPass *llvm::createR600Packetizer() {
     418         556 :   return new R600Packetizer();
     419             : }

Generated by: LCOV version 1.13