LCOV - code coverage report
Current view: top level - lib/Target/AMDGPU - R600InstrInfo.cpp (source / functions) Hit Total Coverage
Test: llvm-toolchain.info Lines: 585 640 91.4 %
Date: 2018-10-14 09:39:32 Functions: 75 76 98.7 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : //===-- R600InstrInfo.cpp - R600 Instruction Information ------------------===//
       2             : //
       3             : //                     The LLVM Compiler Infrastructure
       4             : //
       5             : // This file is distributed under the University of Illinois Open Source
       6             : // License. See LICENSE.TXT for details.
       7             : //
       8             : //===----------------------------------------------------------------------===//
       9             : //
      10             : /// \file
      11             : /// R600 Implementation of TargetInstrInfo.
      12             : //
      13             : //===----------------------------------------------------------------------===//
      14             : 
      15             : #include "R600InstrInfo.h"
      16             : #include "AMDGPU.h"
      17             : #include "AMDGPUInstrInfo.h"
      18             : #include "AMDGPUSubtarget.h"
      19             : #include "R600Defines.h"
      20             : #include "R600FrameLowering.h"
      21             : #include "R600RegisterInfo.h"
      22             : #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
      23             : #include "Utils/AMDGPUBaseInfo.h"
      24             : #include "llvm/ADT/BitVector.h"
      25             : #include "llvm/ADT/SmallSet.h"
      26             : #include "llvm/ADT/SmallVector.h"
      27             : #include "llvm/CodeGen/MachineBasicBlock.h"
      28             : #include "llvm/CodeGen/MachineFrameInfo.h"
      29             : #include "llvm/CodeGen/MachineFunction.h"
      30             : #include "llvm/CodeGen/MachineInstr.h"
      31             : #include "llvm/CodeGen/MachineInstrBuilder.h"
      32             : #include "llvm/CodeGen/MachineOperand.h"
      33             : #include "llvm/CodeGen/MachineRegisterInfo.h"
      34             : #include "llvm/CodeGen/TargetRegisterInfo.h"
      35             : #include "llvm/CodeGen/TargetSubtargetInfo.h"
      36             : #include "llvm/Support/ErrorHandling.h"
      37             : #include <algorithm>
      38             : #include <cassert>
      39             : #include <cstdint>
      40             : #include <cstring>
      41             : #include <iterator>
      42             : #include <utility>
      43             : #include <vector>
      44             : 
      45             : using namespace llvm;
      46             : 
      47             : #define GET_INSTRINFO_CTOR_DTOR
      48             : #include "R600GenDFAPacketizer.inc"
      49             : 
      50             : #define GET_INSTRINFO_CTOR_DTOR
      51             : #define GET_INSTRMAP_INFO
      52             : #define GET_INSTRINFO_NAMED_OPS
      53             : #include "R600GenInstrInfo.inc"
      54             : 
      55         291 : R600InstrInfo::R600InstrInfo(const R600Subtarget &ST)
      56         291 :   : R600GenInstrInfo(-1, -1), RI(), ST(ST) {}
      57             : 
      58      284358 : bool R600InstrInfo::isVector(const MachineInstr &MI) const {
      59      853074 :   return get(MI.getOpcode()).TSFlags & R600_InstFlag::VECTOR;
      60             : }
      61             : 
      62        2123 : void R600InstrInfo::copyPhysReg(MachineBasicBlock &MBB,
      63             :                                 MachineBasicBlock::iterator MI,
      64             :                                 const DebugLoc &DL, unsigned DestReg,
      65             :                                 unsigned SrcReg, bool KillSrc) const {
      66             :   unsigned VectorComponents = 0;
      67        4246 :   if ((R600::R600_Reg128RegClass.contains(DestReg) ||
      68        2123 :       R600::R600_Reg128VerticalRegClass.contains(DestReg)) &&
      69           0 :       (R600::R600_Reg128RegClass.contains(SrcReg) ||
      70           0 :        R600::R600_Reg128VerticalRegClass.contains(SrcReg))) {
      71             :     VectorComponents = 4;
      72        4246 :   } else if((R600::R600_Reg64RegClass.contains(DestReg) ||
      73        2131 :             R600::R600_Reg64VerticalRegClass.contains(DestReg)) &&
      74             :             (R600::R600_Reg64RegClass.contains(SrcReg) ||
      75           0 :              R600::R600_Reg64VerticalRegClass.contains(SrcReg))) {
      76             :     VectorComponents = 2;
      77             :   }
      78             : 
      79             :   if (VectorComponents > 0) {
      80          24 :     for (unsigned I = 0; I < VectorComponents; I++) {
      81          16 :       unsigned SubRegIndex = AMDGPURegisterInfo::getSubRegFromChannel(I);
      82          16 :       buildDefaultInstruction(MBB, MI, R600::MOV,
      83             :                               RI.getSubReg(DestReg, SubRegIndex),
      84          16 :                               RI.getSubReg(SrcReg, SubRegIndex))
      85             :                               .addReg(DestReg,
      86          16 :                                       RegState::Define | RegState::Implicit);
      87             :     }
      88             :   } else {
      89        2115 :     MachineInstr *NewMI = buildDefaultInstruction(MBB, MI, R600::MOV,
      90        2115 :                                                   DestReg, SrcReg);
      91        2115 :     NewMI->getOperand(getOperandIdx(*NewMI, R600::OpName::src0))
      92             :                                     .setIsKill(KillSrc);
      93             :   }
      94        2123 : }
      95             : 
      96             : /// \returns true if \p MBBI can be moved into a new basic.
      97           0 : bool R600InstrInfo::isLegalToSplitMBBAt(MachineBasicBlock &MBB,
      98             :                                        MachineBasicBlock::iterator MBBI) const {
      99           0 :   for (MachineInstr::const_mop_iterator I = MBBI->operands_begin(),
     100           0 :                                         E = MBBI->operands_end(); I != E; ++I) {
     101           0 :     if (I->isReg() && !TargetRegisterInfo::isVirtualRegister(I->getReg()) &&
     102           0 :         I->isUse() && RI.isPhysRegLiveAcrossClauses(I->getReg()))
     103             :       return false;
     104             :   }
     105             :   return true;
     106             : }
     107             : 
     108        2437 : bool R600InstrInfo::isMov(unsigned Opcode) const {
     109        2437 :   switch(Opcode) {
     110             :   default:
     111             :     return false;
     112          62 :   case R600::MOV:
     113             :   case R600::MOV_IMM_F32:
     114             :   case R600::MOV_IMM_I32:
     115          62 :     return true;
     116             :   }
     117             : }
     118             : 
     119      198810 : bool R600InstrInfo::isReductionOp(unsigned Opcode) const {
     120      198810 :   return false;
     121             : }
     122             : 
     123      221649 : bool R600InstrInfo::isCubeOp(unsigned Opcode) const {
     124      221649 :   switch(Opcode) {
     125             :     default: return false;
     126           9 :     case R600::CUBE_r600_pseudo:
     127             :     case R600::CUBE_r600_real:
     128             :     case R600::CUBE_eg_pseudo:
     129             :     case R600::CUBE_eg_real:
     130           9 :       return true;
     131             :   }
     132             : }
     133             : 
     134      541049 : bool R600InstrInfo::isALUInstr(unsigned Opcode) const {
     135     1082098 :   unsigned TargetFlags = get(Opcode).TSFlags;
     136             : 
     137      541049 :   return (TargetFlags & R600_InstFlag::ALU_INST);
     138             : }
     139             : 
     140      158406 : bool R600InstrInfo::hasInstrModifiers(unsigned Opcode) const {
     141      316812 :   unsigned TargetFlags = get(Opcode).TSFlags;
     142             : 
     143             :   return ((TargetFlags & R600_InstFlag::OP1) |
     144      158406 :           (TargetFlags & R600_InstFlag::OP2) |
     145      158406 :           (TargetFlags & R600_InstFlag::OP3));
     146             : }
     147             : 
     148      250633 : bool R600InstrInfo::isLDSInstr(unsigned Opcode) const {
     149      501266 :   unsigned TargetFlags = get(Opcode).TSFlags;
     150             : 
     151             :   return ((TargetFlags & R600_InstFlag::LDS_1A) |
     152      250633 :           (TargetFlags & R600_InstFlag::LDS_1A1D) |
     153      250633 :           (TargetFlags & R600_InstFlag::LDS_1A2D));
     154             : }
     155             : 
     156      111061 : bool R600InstrInfo::isLDSRetInstr(unsigned Opcode) const {
     157      111061 :   return isLDSInstr(Opcode) && getOperandIdx(Opcode, R600::OpName::dst) != -1;
     158             : }
     159             : 
     160       60602 : bool R600InstrInfo::canBeConsideredALU(const MachineInstr &MI) const {
     161      121204 :   if (isALUInstr(MI.getOpcode()))
     162             :     return true;
     163       11775 :   if (isVector(MI) || isCubeOp(MI.getOpcode()))
     164           2 :     return true;
     165       23546 :   switch (MI.getOpcode()) {
     166             :   case R600::PRED_X:
     167             :   case R600::INTERP_PAIR_XY:
     168             :   case R600::INTERP_PAIR_ZW:
     169             :   case R600::INTERP_VEC_LOAD:
     170             :   case R600::COPY:
     171             :   case R600::DOT_4:
     172             :     return true;
     173       11655 :   default:
     174       11655 :     return false;
     175             :   }
     176             : }
     177             : 
     178      218415 : bool R600InstrInfo::isTransOnly(unsigned Opcode) const {
     179      218415 :   if (ST.hasCaymanISA())
     180             :     return false;
     181      391320 :   return (get(Opcode).getSchedClass() == R600::Sched::TransALU);
     182             : }
     183             : 
     184      218415 : bool R600InstrInfo::isTransOnly(const MachineInstr &MI) const {
     185      436830 :   return isTransOnly(MI.getOpcode());
     186             : }
     187             : 
     188       22744 : bool R600InstrInfo::isVectorOnly(unsigned Opcode) const {
     189       45488 :   return (get(Opcode).getSchedClass() == R600::Sched::VecALU);
     190             : }
     191             : 
     192       22744 : bool R600InstrInfo::isVectorOnly(const MachineInstr &MI) const {
     193       45488 :   return isVectorOnly(MI.getOpcode());
     194             : }
     195             : 
     196        3360 : bool R600InstrInfo::isExport(unsigned Opcode) const {
     197        6720 :   return (get(Opcode).TSFlags & R600_InstFlag::IS_EXPORT);
     198             : }
     199             : 
     200       66864 : bool R600InstrInfo::usesVertexCache(unsigned Opcode) const {
     201       66864 :   return ST.hasVertexCache() && IS_VTX(get(Opcode));
     202             : }
     203             : 
     204        9393 : bool R600InstrInfo::usesVertexCache(const MachineInstr &MI) const {
     205        9393 :   const MachineFunction *MF = MI.getParent()->getParent();
     206       19061 :   return !AMDGPU::isCompute(MF->getFunction().getCallingConv()) &&
     207         550 :          usesVertexCache(MI.getOpcode());
     208             : }
     209             : 
     210       64454 : bool R600InstrInfo::usesTextureCache(unsigned Opcode) const {
     211       64454 :   return (!ST.hasVertexCache() && IS_VTX(get(Opcode))) || IS_TEX(get(Opcode));
     212             : }
     213             : 
     214       15327 : bool R600InstrInfo::usesTextureCache(const MachineInstr &MI) const {
     215       15327 :   const MachineFunction *MF = MI.getParent()->getParent();
     216       29983 :   return (AMDGPU::isCompute(MF->getFunction().getCallingConv()) &&
     217       27254 :           usesVertexCache(MI.getOpcode())) ||
     218       23854 :           usesTextureCache(MI.getOpcode());
     219             : }
     220             : 
     221      101744 : bool R600InstrInfo::mustBeLastInClause(unsigned Opcode) const {
     222      101744 :   switch (Opcode) {
     223             :   case R600::KILLGT:
     224             :   case R600::GROUP_BARRIER:
     225             :     return true;
     226      101736 :   default:
     227      101736 :     return false;
     228             :   }
     229             : }
     230             : 
     231       96220 : bool R600InstrInfo::usesAddressRegister(MachineInstr &MI) const {
     232       96220 :   return MI.findRegisterUseOperandIdx(R600::AR_X) != -1;
     233             : }
     234             : 
     235       96005 : bool R600InstrInfo::definesAddressRegister(MachineInstr &MI) const {
     236       96005 :   return MI.findRegisterDefOperandIdx(R600::AR_X) != -1;
     237             : }
     238             : 
     239       37627 : bool R600InstrInfo::readsLDSSrcReg(const MachineInstr &MI) const {
     240       75254 :   if (!isALUInstr(MI.getOpcode())) {
     241             :     return false;
     242             :   }
     243      753854 :   for (MachineInstr::const_mop_iterator I = MI.operands_begin(),
     244       37294 :                                         E = MI.operands_end();
     245      791148 :        I != E; ++I) {
     246      753854 :     if (!I->isReg() || !I->isUse() ||
     247      114735 :         TargetRegisterInfo::isVirtualRegister(I->getReg()))
     248             :       continue;
     249             : 
     250       79820 :     if (R600::R600_LDS_SRC_REGRegClass.contains(I->getReg()))
     251             :       return true;
     252             :   }
     253             :   return false;
     254             : }
     255             : 
     256      326551 : int R600InstrInfo::getSelIdx(unsigned Opcode, unsigned SrcIdx) const {
     257             :   static const unsigned SrcSelTable[][2] = {
     258             :     {R600::OpName::src0, R600::OpName::src0_sel},
     259             :     {R600::OpName::src1, R600::OpName::src1_sel},
     260             :     {R600::OpName::src2, R600::OpName::src2_sel},
     261             :     {R600::OpName::src0_X, R600::OpName::src0_sel_X},
     262             :     {R600::OpName::src0_Y, R600::OpName::src0_sel_Y},
     263             :     {R600::OpName::src0_Z, R600::OpName::src0_sel_Z},
     264             :     {R600::OpName::src0_W, R600::OpName::src0_sel_W},
     265             :     {R600::OpName::src1_X, R600::OpName::src1_sel_X},
     266             :     {R600::OpName::src1_Y, R600::OpName::src1_sel_Y},
     267             :     {R600::OpName::src1_Z, R600::OpName::src1_sel_Z},
     268             :     {R600::OpName::src1_W, R600::OpName::src1_sel_W}
     269             :   };
     270             : 
     271      645710 :   for (const auto &Row : SrcSelTable) {
     272      645710 :     if (getOperandIdx(Opcode, Row[0]) == (int)SrcIdx) {
     273      326551 :       return getOperandIdx(Opcode, Row[1]);
     274             :     }
     275             :   }
     276             :   return -1;
     277             : }
     278             : 
     279             : SmallVector<std::pair<MachineOperand *, int64_t>, 3>
     280      337373 : R600InstrInfo::getSrcs(MachineInstr &MI) const {
     281             :   SmallVector<std::pair<MachineOperand *, int64_t>, 3> Result;
     282             : 
     283      674746 :   if (MI.getOpcode() == R600::DOT_4) {
     284             :     static const unsigned OpTable[8][2] = {
     285             :       {R600::OpName::src0_X, R600::OpName::src0_sel_X},
     286             :       {R600::OpName::src0_Y, R600::OpName::src0_sel_Y},
     287             :       {R600::OpName::src0_Z, R600::OpName::src0_sel_Z},
     288             :       {R600::OpName::src0_W, R600::OpName::src0_sel_W},
     289             :       {R600::OpName::src1_X, R600::OpName::src1_sel_X},
     290             :       {R600::OpName::src1_Y, R600::OpName::src1_sel_Y},
     291             :       {R600::OpName::src1_Z, R600::OpName::src1_sel_Z},
     292             :       {R600::OpName::src1_W, R600::OpName::src1_sel_W},
     293             :     };
     294             : 
     295         288 :     for (unsigned j = 0; j < 8; j++) {
     296             :       MachineOperand &MO =
     297         512 :           MI.getOperand(getOperandIdx(MI.getOpcode(), OpTable[j][0]));
     298         256 :       unsigned Reg = MO.getReg();
     299         256 :       if (Reg == R600::ALU_CONST) {
     300             :         MachineOperand &Sel =
     301          42 :             MI.getOperand(getOperandIdx(MI.getOpcode(), OpTable[j][1]));
     302          21 :         Result.push_back(std::make_pair(&MO, Sel.getImm()));
     303          21 :         continue;
     304             :       }
     305             : 
     306             :     }
     307             :     return Result;
     308             :   }
     309             : 
     310             :   static const unsigned OpTable[3][2] = {
     311             :     {R600::OpName::src0, R600::OpName::src0_sel},
     312             :     {R600::OpName::src1, R600::OpName::src1_sel},
     313             :     {R600::OpName::src2, R600::OpName::src2_sel},
     314             :   };
     315             : 
     316     1017615 :   for (unsigned j = 0; j < 3; j++) {
     317     1914116 :     int SrcIdx = getOperandIdx(MI.getOpcode(), OpTable[j][0]);
     318      957058 :     if (SrcIdx < 0)
     319             :       break;
     320      680274 :     MachineOperand &MO = MI.getOperand(SrcIdx);
     321      680274 :     unsigned Reg = MO.getReg();
     322      680274 :     if (Reg == R600::ALU_CONST) {
     323             :       MachineOperand &Sel =
     324       56838 :           MI.getOperand(getOperandIdx(MI.getOpcode(), OpTable[j][1]));
     325       28419 :       Result.push_back(std::make_pair(&MO, Sel.getImm()));
     326       28419 :       continue;
     327             :     }
     328      651855 :     if (Reg == R600::ALU_LITERAL_X) {
     329             :       MachineOperand &Operand =
     330      290220 :           MI.getOperand(getOperandIdx(MI.getOpcode(), R600::OpName::literal));
     331      145110 :       if (Operand.isImm()) {
     332      145035 :         Result.push_back(std::make_pair(&MO, Operand.getImm()));
     333      145035 :         continue;
     334             :       }
     335             :       assert(Operand.isGlobal());
     336             :     }
     337     1013640 :     Result.push_back(std::make_pair(&MO, 0));
     338             :   }
     339             :   return Result;
     340             : }
     341             : 
     342             : std::vector<std::pair<int, unsigned>>
     343       84288 : R600InstrInfo::ExtractSrcs(MachineInstr &MI,
     344             :                            const DenseMap<unsigned, unsigned> &PV,
     345             :                            unsigned &ConstCount) const {
     346       84288 :   ConstCount = 0;
     347       84288 :   const std::pair<int, unsigned> DummyPair(-1, 0);
     348             :   std::vector<std::pair<int, unsigned>> Result;
     349             :   unsigned i = 0;
     350      340043 :   for (const auto &Src : getSrcs(MI)) {
     351      171467 :     ++i;
     352      171467 :     unsigned Reg = Src.first->getReg();
     353      171467 :     int Index = RI.getEncodingValue(Reg) & 0xff;
     354      171467 :     if (Reg == R600::OQAP) {
     355        1654 :       Result.push_back(std::make_pair(Index, 0U));
     356             :     }
     357      171467 :     if (PV.find(Reg) != PV.end()) {
     358             :       // 255 is used to tells its a PS/PV reg
     359       23093 :       Result.push_back(std::make_pair(255, 0U));
     360      118866 :       continue;
     361             :     }
     362      148374 :     if (Index > 127) {
     363       95773 :       ConstCount++;
     364       95773 :       Result.push_back(DummyPair);
     365       95773 :       continue;
     366             :     }
     367       52601 :     unsigned Chan = RI.getHWRegChan(Reg);
     368       52601 :     Result.push_back(std::make_pair(Index, Chan));
     369             :   }
     370      165685 :   for (; i < 3; ++i)
     371       81397 :     Result.push_back(DummyPair);
     372       84288 :   return Result;
     373             : }
     374             : 
     375             : static std::vector<std::pair<int, unsigned>>
     376     1931903 : Swizzle(std::vector<std::pair<int, unsigned>> Src,
     377             :         R600InstrInfo::BankSwizzle Swz) {
     378     1931903 :   if (Src[0] == Src[1])
     379      320503 :     Src[1].first = -1;
     380     1931903 :   switch (Swz) {
     381             :   case R600InstrInfo::ALU_VEC_012_SCL_210:
     382             :     break;
     383      311296 :   case R600InstrInfo::ALU_VEC_021_SCL_122:
     384             :     std::swap(Src[1], Src[2]);
     385             :     break;
     386      305695 :   case R600InstrInfo::ALU_VEC_102_SCL_221:
     387             :     std::swap(Src[0], Src[1]);
     388             :     break;
     389      309431 :   case R600InstrInfo::ALU_VEC_120_SCL_212:
     390             :     std::swap(Src[0], Src[1]);
     391             :     std::swap(Src[0], Src[2]);
     392             :     break;
     393      305325 :   case R600InstrInfo::ALU_VEC_201:
     394             :     std::swap(Src[0], Src[2]);
     395             :     std::swap(Src[0], Src[1]);
     396             :     break;
     397      301275 :   case R600InstrInfo::ALU_VEC_210:
     398             :     std::swap(Src[0], Src[2]);
     399             :     break;
     400             :   }
     401     1931903 :   return Src;
     402             : }
     403             : 
     404      266736 : static unsigned getTransSwizzle(R600InstrInfo::BankSwizzle Swz, unsigned Op) {
     405      266736 :   switch (Swz) {
     406      127642 :   case R600InstrInfo::ALU_VEC_012_SCL_210: {
     407      127642 :     unsigned Cycles[3] = { 2, 1, 0};
     408      127642 :     return Cycles[Op];
     409             :   }
     410       46154 :   case R600InstrInfo::ALU_VEC_021_SCL_122: {
     411       46154 :     unsigned Cycles[3] = { 1, 2, 2};
     412       46154 :     return Cycles[Op];
     413             :   }
     414       46314 :   case R600InstrInfo::ALU_VEC_120_SCL_212: {
     415       46314 :     unsigned Cycles[3] = { 2, 1, 2};
     416       46314 :     return Cycles[Op];
     417             :   }
     418       46626 :   case R600InstrInfo::ALU_VEC_102_SCL_221: {
     419       46626 :     unsigned Cycles[3] = { 2, 2, 1};
     420       46626 :     return Cycles[Op];
     421             :   }
     422           0 :   default:
     423           0 :     llvm_unreachable("Wrong Swizzle for Trans Slot");
     424             :   }
     425             : }
     426             : 
     427             : /// returns how many MIs (whose inputs are represented by IGSrcs) can be packed
     428             : /// in the same Instruction Group while meeting read port limitations given a
     429             : /// Swz swizzle sequence.
     430      530380 : unsigned  R600InstrInfo::isLegalUpTo(
     431             :     const std::vector<std::vector<std::pair<int, unsigned>>> &IGSrcs,
     432             :     const std::vector<R600InstrInfo::BankSwizzle> &Swz,
     433             :     const std::vector<std::pair<int, unsigned>> &TransSrcs,
     434             :     R600InstrInfo::BankSwizzle TransSwz) const {
     435             :   int Vector[4][3];
     436      530380 :   memset(Vector, -1, sizeof(Vector));
     437     2644785 :   for (unsigned i = 0, e = IGSrcs.size(); i < e; i++) {
     438             :     const std::vector<std::pair<int, unsigned>> &Srcs =
     439     7727612 :         Swizzle(IGSrcs[i], Swz[i]);
     440     6982024 :     for (unsigned j = 0; j < 3; j++) {
     441     5397999 :       const std::pair<int, unsigned> &Src = Srcs[j];
     442     5397999 :       if (Src.first < 0 || Src.first == 255)
     443             :         continue;
     444     5057860 :       if (Src.first == GET_REG_INDEX(RI.getEncodingValue(R600::OQAP))) {
     445        3324 :         if (Swz[i] != R600InstrInfo::ALU_VEC_012_SCL_210 &&
     446             :             Swz[i] != R600InstrInfo::ALU_VEC_021_SCL_122) {
     447             :             // The value from output queue A (denoted by register OQAP) can
     448             :             // only be fetched during the first cycle.
     449             :             return false;
     450             :         }
     451             :         // OQAP does not count towards the normal read port restrictions
     452             :         continue;
     453             :       }
     454     2527268 :       if (Vector[Src.second][j] < 0)
     455     2114203 :         Vector[Src.second][j] = Src.first;
     456     2527268 :       if (Vector[Src.second][j] != Src.first)
     457      347878 :         return i;
     458             :     }
     459             :   }
     460             :   // Now check Trans Alu
     461      461408 :   for (unsigned i = 0, e = TransSrcs.size(); i < e; ++i) {
     462      236234 :     const std::pair<int, unsigned> &Src = TransSrcs[i];
     463      236234 :     unsigned Cycle = getTransSwizzle(TransSwz, i);
     464      236234 :     if (Src.first < 0)
     465             :       continue;
     466      217856 :     if (Src.first == 255)
     467             :       continue;
     468      191588 :     if (Vector[Src.second][Cycle] < 0)
     469       48578 :       Vector[Src.second][Cycle] = Src.first;
     470      191588 :     if (Vector[Src.second][Cycle] != Src.first)
     471      279660 :       return IGSrcs.size() - 1;
     472             :   }
     473       85344 :   return IGSrcs.size();
     474             : }
     475             : 
     476             : /// Given a swizzle sequence SwzCandidate and an index Idx, returns the next
     477             : /// (in lexicographic term) swizzle sequence assuming that all swizzles after
     478             : /// Idx can be skipped
     479             : static bool
     480      487708 : NextPossibleSolution(
     481             :     std::vector<R600InstrInfo::BankSwizzle> &SwzCandidate,
     482             :     unsigned Idx) {
     483             :   assert(Idx < SwzCandidate.size());
     484      487708 :   int ResetIdx = Idx;
     485      584411 :   while (ResetIdx > -1 && SwzCandidate[ResetIdx] == R600InstrInfo::ALU_VEC_210)
     486       96703 :     ResetIdx --;
     487     1118957 :   for (unsigned i = ResetIdx + 1, e = SwzCandidate.size(); i < e; i++) {
     488      287082 :     SwzCandidate[i] = R600InstrInfo::ALU_VEC_012_SCL_210;
     489             :   }
     490      487708 :   if (ResetIdx == -1)
     491             :     return false;
     492      486508 :   int NextSwizzle = SwzCandidate[ResetIdx] + 1;
     493      486508 :   SwzCandidate[ResetIdx] = (R600InstrInfo::BankSwizzle)NextSwizzle;
     494      486508 :   return true;
     495             : }
     496             : 
     497             : /// Enumerate all possible Swizzle sequence to find one that can meet all
     498             : /// read port requirements.
     499       43872 : bool R600InstrInfo::FindSwizzleForVectorSlot(
     500             :     const std::vector<std::vector<std::pair<int, unsigned>>> &IGSrcs,
     501             :     std::vector<R600InstrInfo::BankSwizzle> &SwzCandidate,
     502             :     const std::vector<std::pair<int, unsigned>> &TransSrcs,
     503             :     R600InstrInfo::BankSwizzle TransSwz) const {
     504             :   unsigned ValidUpTo = 0;
     505             :   do {
     506      530380 :     ValidUpTo = isLegalUpTo(IGSrcs, SwzCandidate, TransSrcs, TransSwz);
     507     1060760 :     if (ValidUpTo == IGSrcs.size())
     508             :       return true;
     509      487708 :   } while (NextPossibleSolution(SwzCandidate, ValidUpTo));
     510             :   return false;
     511             : }
     512             : 
     513             : /// Instructions in Trans slot can't read gpr at cycle 0 if they also read
     514             : /// a const, and can't read a gpr at cycle 1 if they read 2 const.
     515             : static bool
     516       10184 : isConstCompatible(R600InstrInfo::BankSwizzle TransSwz,
     517             :                   const std::vector<std::pair<int, unsigned>> &TransOps,
     518             :                   unsigned ConstCount) {
     519             :   // TransALU can't read 3 constants
     520       10184 :   if (ConstCount > 2)
     521             :     return false;
     522       50816 :   for (unsigned i = 0, e = TransOps.size(); i < e; ++i) {
     523       30502 :     const std::pair<int, unsigned> &Src = TransOps[i];
     524       30502 :     unsigned Cycle = getTransSwizzle(TransSwz, i);
     525       30502 :     if (Src.first < 0)
     526             :       continue;
     527       11457 :     if (ConstCount > 0 && Cycle == 0)
     528             :       return false;
     529       11436 :     if (ConstCount > 1 && Cycle == 1)
     530             :       return false;
     531             :   }
     532             :   return true;
     533             : }
     534             : 
     535             : bool
     536       43193 : R600InstrInfo::fitsReadPortLimitations(const std::vector<MachineInstr *> &IG,
     537             :                                        const DenseMap<unsigned, unsigned> &PV,
     538             :                                        std::vector<BankSwizzle> &ValidSwizzle,
     539             :                                        bool isLastAluTrans)
     540             :     const {
     541             :   //Todo : support shared src0 - src1 operand
     542             : 
     543       43193 :   std::vector<std::vector<std::pair<int, unsigned>>> IGSrcs;
     544             :   ValidSwizzle.clear();
     545             :   unsigned ConstCount;
     546       43193 :   BankSwizzle TransBS = ALU_VEC_012_SCL_210;
     547      170674 :   for (unsigned i = 0, e = IG.size(); i < e; ++i) {
     548      168576 :     IGSrcs.push_back(ExtractSrcs(*IG[i], PV, ConstCount));
     549      252864 :     unsigned Op = getOperandIdx(IG[i]->getOpcode(),
     550       84288 :         R600::OpName::bank_swizzle);
     551       84288 :     ValidSwizzle.push_back( (R600InstrInfo::BankSwizzle)
     552      252864 :         IG[i]->getOperand(Op).getImm());
     553             :   }
     554             :   std::vector<std::pair<int, unsigned>> TransOps;
     555       43193 :   if (!isLastAluTrans)
     556       33726 :     return FindSwizzleForVectorSlot(IGSrcs, ValidSwizzle, TransOps, TransBS);
     557             : 
     558             :   TransOps = std::move(IGSrcs.back());
     559             :   IGSrcs.pop_back();
     560             :   ValidSwizzle.pop_back();
     561             : 
     562             :   static const R600InstrInfo::BankSwizzle TransSwz[] = {
     563             :     ALU_VEC_012_SCL_210,
     564             :     ALU_VEC_021_SCL_122,
     565             :     ALU_VEC_120_SCL_212,
     566             :     ALU_VEC_102_SCL_221
     567             :   };
     568       10416 :   for (unsigned i = 0; i < 4; i++) {
     569       10184 :     TransBS = TransSwz[i];
     570       10184 :     if (!isConstCompatible(TransBS, TransOps, ConstCount))
     571             :       continue;
     572       10146 :     bool Result = FindSwizzleForVectorSlot(IGSrcs, ValidSwizzle, TransOps,
     573             :         TransBS);
     574       10146 :     if (Result) {
     575        9235 :       ValidSwizzle.push_back(TransBS);
     576        9235 :       return true;
     577             :     }
     578             :   }
     579             : 
     580             :   return false;
     581             : }
     582             : 
     583             : bool
     584       99528 : R600InstrInfo::fitsConstReadLimitations(const std::vector<unsigned> &Consts)
     585             :     const {
     586             :   assert (Consts.size() <= 12 && "Too many operands in instructions group");
     587             :   unsigned Pair1 = 0, Pair2 = 0;
     588      239465 :   for (unsigned i = 0, n = Consts.size(); i < n; ++i) {
     589       41305 :     unsigned ReadConstHalf = Consts[i] & 2;
     590             :     unsigned ReadConstIndex = Consts[i] & (~3);
     591       41305 :     unsigned ReadHalfConst = ReadConstIndex | ReadConstHalf;
     592       41305 :     if (!Pair1) {
     593             :       Pair1 = ReadHalfConst;
     594             :       continue;
     595             :     }
     596        7087 :     if (Pair1 == ReadHalfConst)
     597             :       continue;
     598        5966 :     if (!Pair2) {
     599             :       Pair2 = ReadHalfConst;
     600             :       continue;
     601             :     }
     602        1703 :     if (Pair2 != ReadHalfConst)
     603             :       return false;
     604             :   }
     605             :   return true;
     606             : }
     607             : 
     608             : bool
     609       90098 : R600InstrInfo::fitsConstReadLimitations(const std::vector<MachineInstr *> &MIs)
     610             :     const {
     611             :   std::vector<unsigned> Consts;
     612       90098 :   SmallSet<int64_t, 4> Literals;
     613      334844 :   for (unsigned i = 0, n = MIs.size(); i < n; i++) {
     614      154674 :     MachineInstr &MI = *MIs[i];
     615      309348 :     if (!isALUInstr(MI.getOpcode()))
     616             :       continue;
     617             : 
     618      622662 :     for (const auto &Src : getSrcs(MI)) {
     619      315802 :       if (Src.first->getReg() == R600::ALU_LITERAL_X)
     620       67248 :         Literals.insert(Src.second);
     621      315802 :       if (Literals.size() > 4)
     622             :         return false;
     623      315776 :       if (Src.first->getReg() == R600::ALU_CONST)
     624       16250 :         Consts.push_back(Src.second);
     625      533328 :       if (R600::R600_KC0RegClass.contains(Src.first->getReg()) ||
     626      300692 :           R600::R600_KC1RegClass.contains(Src.first->getReg())) {
     627       15084 :         unsigned Index = RI.getEncodingValue(Src.first->getReg()) & 0xff;
     628       15084 :         unsigned Chan = RI.getHWRegChan(Src.first->getReg());
     629       15084 :         Consts.push_back((Index << 2) | Chan);
     630             :       }
     631             :     }
     632             :   }
     633       90072 :   return fitsConstReadLimitations(Consts);
     634             : }
     635             : 
     636             : DFAPacketizer *
     637        2297 : R600InstrInfo::CreateTargetScheduleState(const TargetSubtargetInfo &STI) const {
     638        2297 :   const InstrItineraryData *II = STI.getInstrItineraryData();
     639        2297 :   return static_cast<const R600Subtarget &>(STI).createDFAPacketizer(II);
     640             : }
     641             : 
     642             : static bool
     643             : isPredicateSetter(unsigned Opcode) {
     644        6418 :   switch (Opcode) {
     645             :   case R600::PRED_X:
     646             :     return true;
     647        3400 :   default:
     648             :     return false;
     649             :   }
     650             : }
     651             : 
     652             : static MachineInstr *
     653             : findFirstPredicateSetterFrom(MachineBasicBlock &MBB,
     654             :                              MachineBasicBlock::iterator I) {
     655         489 :   while (I != MBB.begin()) {
     656             :     --I;
     657             :     MachineInstr &MI = *I;
     658         489 :     if (isPredicateSetter(MI.getOpcode()))
     659             :       return &MI;
     660             :   }
     661             : 
     662             :   return nullptr;
     663             : }
     664             : 
     665             : static
     666             : bool isJump(unsigned Opcode) {
     667       42863 :   return Opcode == R600::JUMP || Opcode == R600::JUMP_COND;
     668             : }
     669             : 
     670             : static bool isBranch(unsigned Opcode) {
     671       40222 :   return Opcode == R600::BRANCH || Opcode == R600::BRANCH_COND_i32 ||
     672             :       Opcode == R600::BRANCH_COND_f32;
     673             : }
     674             : 
     675       40304 : bool R600InstrInfo::analyzeBranch(MachineBasicBlock &MBB,
     676             :                                   MachineBasicBlock *&TBB,
     677             :                                   MachineBasicBlock *&FBB,
     678             :                                   SmallVectorImpl<MachineOperand> &Cond,
     679             :                                   bool AllowModify) const {
     680             :   // Most of the following comes from the ARM implementation of AnalyzeBranch
     681             : 
     682             :   // If the block has no terminators, it just falls into the block after it.
     683       40304 :   MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr();
     684       40304 :   if (I == MBB.end())
     685             :     return false;
     686             : 
     687             :   // R600::BRANCH* instructions are only available after isel and are not
     688             :   // handled
     689       40222 :   if (isBranch(I->getOpcode()))
     690             :     return true;
     691       40211 :   if (!isJump(I->getOpcode())) {
     692             :     return false;
     693             :   }
     694             : 
     695             :   // Remove successive JUMP
     696        7962 :   while (I != MBB.begin() && std::prev(I)->getOpcode() == R600::JUMP) {
     697           0 :       MachineBasicBlock::iterator PriorI = std::prev(I);
     698           0 :       if (AllowModify)
     699           0 :         I->removeFromParent();
     700           0 :       I = PriorI;
     701             :   }
     702             :   MachineInstr &LastInst = *I;
     703             : 
     704             :   // If there is only one terminator instruction, process it.
     705        2658 :   unsigned LastOpc = LastInst.getOpcode();
     706        7962 :   if (I == MBB.begin() || !isJump((--I)->getOpcode())) {
     707        2215 :     if (LastOpc == R600::JUMP) {
     708         574 :       TBB = LastInst.getOperand(0).getMBB();
     709         574 :       return false;
     710        1641 :     } else if (LastOpc == R600::JUMP_COND) {
     711        1641 :       auto predSet = I;
     712        2027 :       while (!isPredicateSetter(predSet->getOpcode())) {
     713         386 :         predSet = --I;
     714             :       }
     715        1641 :       TBB = LastInst.getOperand(0).getMBB();
     716        3282 :       Cond.push_back(predSet->getOperand(1));
     717        3282 :       Cond.push_back(predSet->getOperand(2));
     718        1641 :       Cond.push_back(MachineOperand::CreateReg(R600::PRED_SEL_ONE, false));
     719             :       return false;
     720             :     }
     721             :     return true;  // Can't handle indirect branch.
     722             :   }
     723             : 
     724             :   // Get the instruction before it if it is a terminator.
     725             :   MachineInstr &SecondLastInst = *I;
     726             :   unsigned SecondLastOpc = SecondLastInst.getOpcode();
     727             : 
     728             :   // If the block ends with a B and a Bcc, handle it.
     729         443 :   if (SecondLastOpc == R600::JUMP_COND && LastOpc == R600::JUMP) {
     730         443 :     auto predSet = --I;
     731         502 :     while (!isPredicateSetter(predSet->getOpcode())) {
     732          59 :       predSet = --I;
     733             :     }
     734         443 :     TBB = SecondLastInst.getOperand(0).getMBB();
     735         443 :     FBB = LastInst.getOperand(0).getMBB();
     736         886 :     Cond.push_back(predSet->getOperand(1));
     737         886 :     Cond.push_back(predSet->getOperand(2));
     738         443 :     Cond.push_back(MachineOperand::CreateReg(R600::PRED_SEL_ONE, false));
     739             :     return false;
     740             :   }
     741             : 
     742             :   // Otherwise, can't handle this.
     743             :   return true;
     744             : }
     745             : 
     746             : static
     747         428 : MachineBasicBlock::iterator FindLastAluClause(MachineBasicBlock &MBB) {
     748             :   for (MachineBasicBlock::reverse_iterator It = MBB.rbegin(), E = MBB.rend();
     749        2157 :       It != E; ++It) {
     750        3970 :     if (It->getOpcode() == R600::CF_ALU ||
     751             :         It->getOpcode() == R600::CF_ALU_PUSH_BEFORE)
     752             :       return It.getReverse();
     753             :   }
     754             :   return MBB.end();
     755             : }
     756             : 
     757         228 : unsigned R600InstrInfo::insertBranch(MachineBasicBlock &MBB,
     758             :                                      MachineBasicBlock *TBB,
     759             :                                      MachineBasicBlock *FBB,
     760             :                                      ArrayRef<MachineOperand> Cond,
     761             :                                      const DebugLoc &DL,
     762             :                                      int *BytesAdded) const {
     763             :   assert(TBB && "insertBranch must not be told to insert a fallthrough");
     764             :   assert(!BytesAdded && "code size not handled");
     765             : 
     766         228 :   if (!FBB) {
     767         228 :     if (Cond.empty()) {
     768          26 :       BuildMI(&MBB, DL, get(R600::JUMP)).addMBB(TBB);
     769          26 :       return 1;
     770             :     } else {
     771         202 :       MachineInstr *PredSet = findFirstPredicateSetterFrom(MBB, MBB.end());
     772             :       assert(PredSet && "No previous predicate !");
     773         202 :       addFlag(*PredSet, 0, MO_FLAG_PUSH);
     774         202 :       PredSet->getOperand(2).setImm(Cond[1].getImm());
     775             : 
     776         202 :       BuildMI(&MBB, DL, get(R600::JUMP_COND))
     777             :              .addMBB(TBB)
     778         202 :              .addReg(R600::PREDICATE_BIT, RegState::Kill);
     779         202 :       MachineBasicBlock::iterator CfAlu = FindLastAluClause(MBB);
     780         202 :       if (CfAlu == MBB.end())
     781             :         return 1;
     782             :       assert (CfAlu->getOpcode() == R600::CF_ALU);
     783         116 :       CfAlu->setDesc(get(R600::CF_ALU_PUSH_BEFORE));
     784         116 :       return 1;
     785             :     }
     786             :   } else {
     787           0 :     MachineInstr *PredSet = findFirstPredicateSetterFrom(MBB, MBB.end());
     788             :     assert(PredSet && "No previous predicate !");
     789           0 :     addFlag(*PredSet, 0, MO_FLAG_PUSH);
     790           0 :     PredSet->getOperand(2).setImm(Cond[1].getImm());
     791           0 :     BuildMI(&MBB, DL, get(R600::JUMP_COND))
     792             :             .addMBB(TBB)
     793           0 :             .addReg(R600::PREDICATE_BIT, RegState::Kill);
     794           0 :     BuildMI(&MBB, DL, get(R600::JUMP)).addMBB(FBB);
     795           0 :     MachineBasicBlock::iterator CfAlu = FindLastAluClause(MBB);
     796           0 :     if (CfAlu == MBB.end())
     797             :       return 2;
     798             :     assert (CfAlu->getOpcode() == R600::CF_ALU);
     799           0 :     CfAlu->setDesc(get(R600::CF_ALU_PUSH_BEFORE));
     800           0 :     return 2;
     801             :   }
     802             : }
     803             : 
     804         316 : unsigned R600InstrInfo::removeBranch(MachineBasicBlock &MBB,
     805             :                                      int *BytesRemoved) const {
     806             :   assert(!BytesRemoved && "code size not handled");
     807             : 
     808             :   // Note : we leave PRED* instructions there.
     809             :   // They may be needed when predicating instructions.
     810             : 
     811         316 :   MachineBasicBlock::iterator I = MBB.end();
     812             : 
     813         316 :   if (I == MBB.begin()) {
     814             :     return 0;
     815             :   }
     816             :   --I;
     817         632 :   switch (I->getOpcode()) {
     818             :   default:
     819             :     return 0;
     820         139 :   case R600::JUMP_COND: {
     821         139 :     MachineInstr *predSet = findFirstPredicateSetterFrom(MBB, I);
     822         139 :     clearFlag(*predSet, 0, MO_FLAG_PUSH);
     823         139 :     I->eraseFromParent();
     824         139 :     MachineBasicBlock::iterator CfAlu = FindLastAluClause(MBB);
     825         139 :     if (CfAlu == MBB.end())
     826             :       break;
     827             :     assert (CfAlu->getOpcode() == R600::CF_ALU_PUSH_BEFORE);
     828         139 :     CfAlu->setDesc(get(R600::CF_ALU));
     829             :     break;
     830             :   }
     831             :   case R600::JUMP:
     832         158 :     I->eraseFromParent();
     833         158 :     break;
     834             :   }
     835         297 :   I = MBB.end();
     836             : 
     837         297 :   if (I == MBB.begin()) {
     838             :     return 1;
     839             :   }
     840             :   --I;
     841         588 :   switch (I->getOpcode()) {
     842             :     // FIXME: only one case??
     843             :   default:
     844             :     return 1;
     845          87 :   case R600::JUMP_COND: {
     846          87 :     MachineInstr *predSet = findFirstPredicateSetterFrom(MBB, I);
     847          87 :     clearFlag(*predSet, 0, MO_FLAG_PUSH);
     848          87 :     I->eraseFromParent();
     849          87 :     MachineBasicBlock::iterator CfAlu = FindLastAluClause(MBB);
     850          87 :     if (CfAlu == MBB.end())
     851             :       break;
     852             :     assert (CfAlu->getOpcode() == R600::CF_ALU_PUSH_BEFORE);
     853           1 :     CfAlu->setDesc(get(R600::CF_ALU));
     854             :     break;
     855             :   }
     856             :   case R600::JUMP:
     857           0 :     I->eraseFromParent();
     858           0 :     break;
     859             :   }
     860             :   return 2;
     861             : }
     862             : 
     863      130027 : bool R600InstrInfo::isPredicated(const MachineInstr &MI) const {
     864      130027 :   int idx = MI.findFirstPredOperandIdx();
     865      130027 :   if (idx < 0)
     866             :     return false;
     867             : 
     868      200574 :   unsigned Reg = MI.getOperand(idx).getReg();
     869             :   switch (Reg) {
     870             :   default: return false;
     871             :   case R600::PRED_SEL_ONE:
     872             :   case R600::PRED_SEL_ZERO:
     873             :   case R600::PREDICATE_BIT:
     874             :     return true;
     875             :   }
     876             : }
     877             : 
     878        3400 : bool R600InstrInfo::isPredicable(const MachineInstr &MI) const {
     879             :   // XXX: KILL* instructions can be predicated, but they must be the last
     880             :   // instruction in a clause, so this means any instructions after them cannot
     881             :   // be predicated.  Until we have proper support for instruction clauses in the
     882             :   // backend, we will mark KILL* instructions as unpredicable.
     883             : 
     884        6800 :   if (MI.getOpcode() == R600::KILLGT) {
     885             :     return false;
     886        3400 :   } else if (MI.getOpcode() == R600::CF_ALU) {
     887             :     // If the clause start in the middle of MBB then the MBB has more
     888             :     // than a single clause, unable to predicate several clauses.
     889        4690 :     if (MI.getParent()->begin() != MachineBasicBlock::const_iterator(MI))
     890             :       return false;
     891             :     // TODO: We don't support KC merging atm
     892        2345 :     return MI.getOperand(3).getImm() == 0 && MI.getOperand(4).getImm() == 0;
     893        1055 :   } else if (isVector(MI)) {
     894             :     return false;
     895             :   } else {
     896        1054 :     return TargetInstrInfo::isPredicable(MI);
     897             :   }
     898             : }
     899             : 
     900             : bool
     901         100 : R600InstrInfo::isProfitableToIfCvt(MachineBasicBlock &MBB,
     902             :                                    unsigned NumCycles,
     903             :                                    unsigned ExtraPredCycles,
     904             :                                    BranchProbability Probability) const{
     905         100 :   return true;
     906             : }
     907             : 
     908             : bool
     909           2 : R600InstrInfo::isProfitableToIfCvt(MachineBasicBlock &TMBB,
     910             :                                    unsigned NumTCycles,
     911             :                                    unsigned ExtraTCycles,
     912             :                                    MachineBasicBlock &FMBB,
     913             :                                    unsigned NumFCycles,
     914             :                                    unsigned ExtraFCycles,
     915             :                                    BranchProbability Probability) const {
     916           2 :   return true;
     917             : }
     918             : 
     919             : bool
     920         138 : R600InstrInfo::isProfitableToDupForIfCvt(MachineBasicBlock &MBB,
     921             :                                          unsigned NumCycles,
     922             :                                          BranchProbability Probability)
     923             :                                          const {
     924         138 :   return true;
     925             : }
     926             : 
     927             : bool
     928           2 : R600InstrInfo::isProfitableToUnpredicate(MachineBasicBlock &TMBB,
     929             :                                          MachineBasicBlock &FMBB) const {
     930           2 :   return false;
     931             : }
     932             : 
     933             : bool
     934         318 : R600InstrInfo::reverseBranchCondition(SmallVectorImpl<MachineOperand> &Cond) const {
     935             :   MachineOperand &MO = Cond[1];
     936         318 :   switch (MO.getImm()) {
     937          62 :   case R600::PRED_SETE_INT:
     938             :     MO.setImm(R600::PRED_SETNE_INT);
     939             :     break;
     940         256 :   case R600::PRED_SETNE_INT:
     941             :     MO.setImm(R600::PRED_SETE_INT);
     942             :     break;
     943           0 :   case R600::PRED_SETE:
     944             :     MO.setImm(R600::PRED_SETNE);
     945             :     break;
     946           0 :   case R600::PRED_SETNE:
     947             :     MO.setImm(R600::PRED_SETE);
     948             :     break;
     949             :   default:
     950             :     return true;
     951             :   }
     952             : 
     953             :   MachineOperand &MO2 = Cond[2];
     954         318 :   switch (MO2.getReg()) {
     955           0 :   case R600::PRED_SEL_ZERO:
     956           0 :     MO2.setReg(R600::PRED_SEL_ONE);
     957           0 :     break;
     958         318 :   case R600::PRED_SEL_ONE:
     959         318 :     MO2.setReg(R600::PRED_SEL_ZERO);
     960         318 :     break;
     961             :   default:
     962             :     return true;
     963             :   }
     964             :   return false;
     965             : }
     966             : 
     967        3400 : bool R600InstrInfo::DefinesPredicate(MachineInstr &MI,
     968             :                                      std::vector<MachineOperand> &Pred) const {
     969        3400 :   return isPredicateSetter(MI.getOpcode());
     970             : }
     971             : 
     972         235 : bool R600InstrInfo::PredicateInstruction(MachineInstr &MI,
     973             :                                          ArrayRef<MachineOperand> Pred) const {
     974         235 :   int PIdx = MI.findFirstPredOperandIdx();
     975             : 
     976         470 :   if (MI.getOpcode() == R600::CF_ALU) {
     977          26 :     MI.getOperand(8).setImm(0);
     978          26 :     return true;
     979             :   }
     980             : 
     981         209 :   if (MI.getOpcode() == R600::DOT_4) {
     982          15 :     MI.getOperand(getOperandIdx(MI, R600::OpName::pred_sel_X))
     983          15 :         .setReg(Pred[2].getReg());
     984          15 :     MI.getOperand(getOperandIdx(MI, R600::OpName::pred_sel_Y))
     985          15 :         .setReg(Pred[2].getReg());
     986          15 :     MI.getOperand(getOperandIdx(MI, R600::OpName::pred_sel_Z))
     987          15 :         .setReg(Pred[2].getReg());
     988          15 :     MI.getOperand(getOperandIdx(MI, R600::OpName::pred_sel_W))
     989          15 :         .setReg(Pred[2].getReg());
     990          15 :     MachineInstrBuilder MIB(*MI.getParent()->getParent(), MI);
     991          15 :     MIB.addReg(R600::PREDICATE_BIT, RegState::Implicit);
     992             :     return true;
     993             :   }
     994             : 
     995         194 :   if (PIdx != -1) {
     996         194 :     MachineOperand &PMO = MI.getOperand(PIdx);
     997         194 :     PMO.setReg(Pred[2].getReg());
     998         194 :     MachineInstrBuilder MIB(*MI.getParent()->getParent(), MI);
     999         194 :     MIB.addReg(R600::PREDICATE_BIT, RegState::Implicit);
    1000             :     return true;
    1001             :   }
    1002             : 
    1003             :   return false;
    1004             : }
    1005             : 
    1006        3400 : unsigned int R600InstrInfo::getPredicationCost(const MachineInstr &) const {
    1007        3400 :   return 2;
    1008             : }
    1009             : 
    1010      257013 : unsigned int R600InstrInfo::getInstrLatency(const InstrItineraryData *ItinData,
    1011             :                                             const MachineInstr &,
    1012             :                                             unsigned *PredCost) const {
    1013      257013 :   if (PredCost)
    1014           0 :     *PredCost = 2;
    1015      257013 :   return 2;
    1016             : }
    1017             : 
    1018        2225 : unsigned R600InstrInfo::calculateIndirectAddress(unsigned RegIndex,
    1019             :                                                    unsigned Channel) const {
    1020             :   assert(Channel == 0);
    1021        2225 :   return RegIndex;
    1022             : }
    1023             : 
    1024        6781 : bool R600InstrInfo::expandPostRAPseudo(MachineInstr &MI) const {
    1025       13562 :   switch (MI.getOpcode()) {
    1026        6766 :   default: {
    1027        6766 :     MachineBasicBlock *MBB = MI.getParent();
    1028             :     int OffsetOpIdx =
    1029        6766 :         R600::getNamedOperandIdx(MI.getOpcode(), R600::OpName::addr);
    1030             :     // addr is a custom operand with multiple MI operands, and only the
    1031             :     // first MI operand is given a name.
    1032        6766 :     int RegOpIdx = OffsetOpIdx + 1;
    1033             :     int ChanOpIdx =
    1034        6766 :         R600::getNamedOperandIdx(MI.getOpcode(), R600::OpName::chan);
    1035        6766 :     if (isRegisterLoad(MI)) {
    1036             :       int DstOpIdx =
    1037        1148 :           R600::getNamedOperandIdx(MI.getOpcode(), R600::OpName::dst);
    1038        1148 :       unsigned RegIndex = MI.getOperand(RegOpIdx).getImm();
    1039        1148 :       unsigned Channel = MI.getOperand(ChanOpIdx).getImm();
    1040        1148 :       unsigned Address = calculateIndirectAddress(RegIndex, Channel);
    1041        1148 :       unsigned OffsetReg = MI.getOperand(OffsetOpIdx).getReg();
    1042        1148 :       if (OffsetReg == R600::INDIRECT_BASE_ADDR) {
    1043         995 :         buildMovInstr(MBB, MI, MI.getOperand(DstOpIdx).getReg(),
    1044             :                       getIndirectAddrRegClass()->getRegister(Address));
    1045             :       } else {
    1046         153 :         buildIndirectRead(MBB, MI, MI.getOperand(DstOpIdx).getReg(), Address,
    1047         306 :                           OffsetReg);
    1048             :       }
    1049        5618 :     } else if (isRegisterStore(MI)) {
    1050             :       int ValOpIdx =
    1051         953 :           R600::getNamedOperandIdx(MI.getOpcode(), R600::OpName::val);
    1052         953 :       unsigned RegIndex = MI.getOperand(RegOpIdx).getImm();
    1053         953 :       unsigned Channel = MI.getOperand(ChanOpIdx).getImm();
    1054         953 :       unsigned Address = calculateIndirectAddress(RegIndex, Channel);
    1055         953 :       unsigned OffsetReg = MI.getOperand(OffsetOpIdx).getReg();
    1056         953 :       if (OffsetReg == R600::INDIRECT_BASE_ADDR) {
    1057         829 :         buildMovInstr(MBB, MI, getIndirectAddrRegClass()->getRegister(Address),
    1058         829 :                       MI.getOperand(ValOpIdx).getReg());
    1059             :       } else {
    1060         124 :         buildIndirectWrite(MBB, MI, MI.getOperand(ValOpIdx).getReg(),
    1061             :                            calculateIndirectAddress(RegIndex, Channel),
    1062         124 :                            OffsetReg);
    1063             :       }
    1064             :     } else {
    1065             :       return false;
    1066             :     }
    1067             : 
    1068        2101 :     MBB->erase(MI);
    1069        2101 :     return true;
    1070             :   }
    1071          14 :   case R600::R600_EXTRACT_ELT_V2:
    1072             :   case R600::R600_EXTRACT_ELT_V4:
    1073          14 :     buildIndirectRead(MI.getParent(), MI, MI.getOperand(0).getReg(),
    1074             :                       RI.getHWRegIndex(MI.getOperand(1).getReg()), //  Address
    1075          14 :                       MI.getOperand(2).getReg(),
    1076          14 :                       RI.getHWRegChan(MI.getOperand(1).getReg()));
    1077          14 :     break;
    1078           1 :   case R600::R600_INSERT_ELT_V2:
    1079             :   case R600::R600_INSERT_ELT_V4:
    1080           1 :     buildIndirectWrite(MI.getParent(), MI, MI.getOperand(2).getReg(), // Value
    1081             :                        RI.getHWRegIndex(MI.getOperand(1).getReg()),   // Address
    1082           1 :                        MI.getOperand(3).getReg(),                     // Offset
    1083           1 :                        RI.getHWRegChan(MI.getOperand(1).getReg()));   // Channel
    1084           1 :     break;
    1085             :   }
    1086          15 :   MI.eraseFromParent();
    1087          15 :   return true;
    1088             : }
    1089             : 
    1090        4594 : void R600InstrInfo::reserveIndirectRegisters(BitVector &Reserved,
    1091             :                                              const MachineFunction &MF,
    1092             :                                              const R600RegisterInfo &TRI) const {
    1093        4594 :   const R600Subtarget &ST = MF.getSubtarget<R600Subtarget>();
    1094             :   const R600FrameLowering *TFL = ST.getFrameLowering();
    1095             : 
    1096        4594 :   unsigned StackWidth = TFL->getStackWidth(MF);
    1097        4594 :   int End = getIndirectIndexEnd(MF);
    1098             : 
    1099        4594 :   if (End == -1)
    1100             :     return;
    1101             : 
    1102        8662 :   for (int Index = getIndirectIndexBegin(MF); Index <= End; ++Index) {
    1103       15704 :     for (unsigned Chan = 0; Chan < StackWidth; ++Chan) {
    1104        7852 :       unsigned Reg = R600::R600_TReg32RegClass.getRegister((4 * Index) + Chan);
    1105        7852 :       TRI.reserveRegisterTuples(Reserved, Reg);
    1106             :     }
    1107             :   }
    1108             : }
    1109             : 
    1110        1872 : const TargetRegisterClass *R600InstrInfo::getIndirectAddrRegClass() const {
    1111        1872 :   return &R600::R600_TReg32_XRegClass;
    1112             : }
    1113             : 
    1114         124 : MachineInstrBuilder R600InstrInfo::buildIndirectWrite(MachineBasicBlock *MBB,
    1115             :                                        MachineBasicBlock::iterator I,
    1116             :                                        unsigned ValueReg, unsigned Address,
    1117             :                                        unsigned OffsetReg) const {
    1118         124 :   return buildIndirectWrite(MBB, I, ValueReg, Address, OffsetReg, 0);
    1119             : }
    1120             : 
    1121         125 : MachineInstrBuilder R600InstrInfo::buildIndirectWrite(MachineBasicBlock *MBB,
    1122             :                                        MachineBasicBlock::iterator I,
    1123             :                                        unsigned ValueReg, unsigned Address,
    1124             :                                        unsigned OffsetReg,
    1125             :                                        unsigned AddrChan) const {
    1126             :   unsigned AddrReg;
    1127         125 :   switch (AddrChan) {
    1128           0 :     default: llvm_unreachable("Invalid Channel");
    1129         248 :     case 0: AddrReg = R600::R600_AddrRegClass.getRegister(Address); break;
    1130           0 :     case 1: AddrReg = R600::R600_Addr_YRegClass.getRegister(Address); break;
    1131           0 :     case 2: AddrReg = R600::R600_Addr_ZRegClass.getRegister(Address); break;
    1132           2 :     case 3: AddrReg = R600::R600_Addr_WRegClass.getRegister(Address); break;
    1133             :   }
    1134         125 :   MachineInstr *MOVA = buildDefaultInstruction(*MBB, I, R600::MOVA_INT_eg,
    1135         125 :                                                R600::AR_X, OffsetReg);
    1136         125 :   setImmOperand(*MOVA, R600::OpName::write, 0);
    1137             : 
    1138         125 :   MachineInstrBuilder Mov = buildDefaultInstruction(*MBB, I, R600::MOV,
    1139         125 :                                       AddrReg, ValueReg)
    1140             :                                       .addReg(R600::AR_X,
    1141         125 :                                            RegState::Implicit | RegState::Kill);
    1142         125 :   setImmOperand(*Mov, R600::OpName::dst_rel, 1);
    1143         125 :   return Mov;
    1144             : }
    1145             : 
    1146         153 : MachineInstrBuilder R600InstrInfo::buildIndirectRead(MachineBasicBlock *MBB,
    1147             :                                        MachineBasicBlock::iterator I,
    1148             :                                        unsigned ValueReg, unsigned Address,
    1149             :                                        unsigned OffsetReg) const {
    1150         153 :   return buildIndirectRead(MBB, I, ValueReg, Address, OffsetReg, 0);
    1151             : }
    1152             : 
    1153         167 : MachineInstrBuilder R600InstrInfo::buildIndirectRead(MachineBasicBlock *MBB,
    1154             :                                        MachineBasicBlock::iterator I,
    1155             :                                        unsigned ValueReg, unsigned Address,
    1156             :                                        unsigned OffsetReg,
    1157             :                                        unsigned AddrChan) const {
    1158             :   unsigned AddrReg;
    1159         167 :   switch (AddrChan) {
    1160           0 :     default: llvm_unreachable("Invalid Channel");
    1161         310 :     case 0: AddrReg = R600::R600_AddrRegClass.getRegister(Address); break;
    1162          12 :     case 1: AddrReg = R600::R600_Addr_YRegClass.getRegister(Address); break;
    1163           0 :     case 2: AddrReg = R600::R600_Addr_ZRegClass.getRegister(Address); break;
    1164          12 :     case 3: AddrReg = R600::R600_Addr_WRegClass.getRegister(Address); break;
    1165             :   }
    1166         167 :   MachineInstr *MOVA = buildDefaultInstruction(*MBB, I, R600::MOVA_INT_eg,
    1167             :                                                        R600::AR_X,
    1168         167 :                                                        OffsetReg);
    1169         167 :   setImmOperand(*MOVA, R600::OpName::write, 0);
    1170         167 :   MachineInstrBuilder Mov = buildDefaultInstruction(*MBB, I, R600::MOV,
    1171             :                                       ValueReg,
    1172         167 :                                       AddrReg)
    1173             :                                       .addReg(R600::AR_X,
    1174         167 :                                            RegState::Implicit | RegState::Kill);
    1175         167 :   setImmOperand(*Mov, R600::OpName::src0_rel, 1);
    1176             : 
    1177         167 :   return Mov;
    1178             : }
    1179             : 
    1180        1620 : int R600InstrInfo::getIndirectIndexBegin(const MachineFunction &MF) const {
    1181        1620 :   const MachineRegisterInfo &MRI = MF.getRegInfo();
    1182        1620 :   const MachineFrameInfo &MFI = MF.getFrameInfo();
    1183        1620 :   int Offset = -1;
    1184             : 
    1185        1620 :   if (MFI.getNumObjects() == 0) {
    1186             :     return -1;
    1187             :   }
    1188             : 
    1189        1620 :   if (MRI.livein_empty()) {
    1190             :     return 0;
    1191             :   }
    1192             : 
    1193          48 :   const TargetRegisterClass *IndirectRC = getIndirectAddrRegClass();
    1194          96 :   for (std::pair<unsigned, unsigned> LI : MRI.liveins()) {
    1195             :     unsigned Reg = LI.first;
    1196          96 :     if (TargetRegisterInfo::isVirtualRegister(Reg) ||
    1197          48 :         !IndirectRC->contains(Reg))
    1198             :       continue;
    1199             : 
    1200             :     unsigned RegIndex;
    1201             :     unsigned RegEnd;
    1202          48 :     for (RegIndex = 0, RegEnd = IndirectRC->getNumRegs(); RegIndex != RegEnd;
    1203             :                                                           ++RegIndex) {
    1204          48 :       if (IndirectRC->getRegister(RegIndex) == Reg)
    1205             :         break;
    1206             :     }
    1207          96 :     Offset = std::max(Offset, (int)RegIndex);
    1208             :   }
    1209             : 
    1210          48 :   return Offset + 1;
    1211             : }
    1212             : 
    1213        4594 : int R600InstrInfo::getIndirectIndexEnd(const MachineFunction &MF) const {
    1214             :   int Offset = 0;
    1215        4594 :   const MachineFrameInfo &MFI = MF.getFrameInfo();
    1216             : 
    1217             :   // Variable sized objects are not supported
    1218        4594 :   if (MFI.hasVarSizedObjects()) {
    1219             :     return -1;
    1220             :   }
    1221             : 
    1222        4592 :   if (MFI.getNumObjects() == 0) {
    1223             :     return -1;
    1224             :   }
    1225             : 
    1226         810 :   const R600Subtarget &ST = MF.getSubtarget<R600Subtarget>();
    1227             :   const R600FrameLowering *TFL = ST.getFrameLowering();
    1228             : 
    1229             :   unsigned IgnoredFrameReg;
    1230         810 :   Offset = TFL->getFrameIndexReference(MF, -1, IgnoredFrameReg);
    1231             : 
    1232         810 :   return getIndirectIndexBegin(MF) + Offset;
    1233             : }
    1234             : 
    1235       52794 : unsigned R600InstrInfo::getMaxAlusPerClause() const {
    1236       52794 :   return 115;
    1237             : }
    1238             : 
    1239        9192 : MachineInstrBuilder R600InstrInfo::buildDefaultInstruction(MachineBasicBlock &MBB,
    1240             :                                                   MachineBasicBlock::iterator I,
    1241             :                                                   unsigned Opcode,
    1242             :                                                   unsigned DstReg,
    1243             :                                                   unsigned Src0Reg,
    1244             :                                                   unsigned Src1Reg) const {
    1245        9192 :   MachineInstrBuilder MIB = BuildMI(MBB, I, MBB.findDebugLoc(I), get(Opcode),
    1246        9192 :     DstReg);           // $dst
    1247             : 
    1248        9192 :   if (Src1Reg) {
    1249             :     MIB.addImm(0)     // $update_exec_mask
    1250             :        .addImm(0);    // $update_predicate
    1251             :   }
    1252             :   MIB.addImm(1)        // $write
    1253             :      .addImm(0)        // $omod
    1254             :      .addImm(0)        // $dst_rel
    1255             :      .addImm(0)        // $dst_clamp
    1256        9192 :      .addReg(Src0Reg)  // $src0
    1257             :      .addImm(0)        // $src0_neg
    1258             :      .addImm(0)        // $src0_rel
    1259             :      .addImm(0)        // $src0_abs
    1260             :      .addImm(-1);       // $src0_sel
    1261             : 
    1262        9192 :   if (Src1Reg) {
    1263         258 :     MIB.addReg(Src1Reg) // $src1
    1264             :        .addImm(0)       // $src1_neg
    1265             :        .addImm(0)       // $src1_rel
    1266             :        .addImm(0)       // $src1_abs
    1267             :        .addImm(-1);      // $src1_sel
    1268             :   }
    1269             : 
    1270             :   //XXX: The r600g finalizer expects this to be 1, once we've moved the
    1271             :   //scheduling to the backend, we can change the default to 0.
    1272             :   MIB.addImm(1)        // $last
    1273        9192 :       .addReg(R600::PRED_SEL_OFF) // $pred_sel
    1274             :       .addImm(0)         // $literal
    1275             :       .addImm(0);        // $bank_swizzle
    1276             : 
    1277        9192 :   return MIB;
    1278             : }
    1279             : 
    1280             : #define OPERAND_CASE(Label) \
    1281             :   case Label: { \
    1282             :     static const unsigned Ops[] = \
    1283             :     { \
    1284             :       Label##_X, \
    1285             :       Label##_Y, \
    1286             :       Label##_Z, \
    1287             :       Label##_W \
    1288             :     }; \
    1289             :     return Ops[Slot]; \
    1290             :   }
    1291             : 
    1292        1792 : static unsigned getSlotedOps(unsigned  Op, unsigned Slot) {
    1293        1792 :   switch (Op) {
    1294         128 :   OPERAND_CASE(R600::OpName::update_exec_mask)
    1295         128 :   OPERAND_CASE(R600::OpName::update_pred)
    1296         128 :   OPERAND_CASE(R600::OpName::write)
    1297         128 :   OPERAND_CASE(R600::OpName::omod)
    1298         128 :   OPERAND_CASE(R600::OpName::dst_rel)
    1299         128 :   OPERAND_CASE(R600::OpName::clamp)
    1300           0 :   OPERAND_CASE(R600::OpName::src0)
    1301         128 :   OPERAND_CASE(R600::OpName::src0_neg)
    1302         128 :   OPERAND_CASE(R600::OpName::src0_rel)
    1303         128 :   OPERAND_CASE(R600::OpName::src0_abs)
    1304         128 :   OPERAND_CASE(R600::OpName::src0_sel)
    1305           0 :   OPERAND_CASE(R600::OpName::src1)
    1306         128 :   OPERAND_CASE(R600::OpName::src1_neg)
    1307         128 :   OPERAND_CASE(R600::OpName::src1_rel)
    1308         128 :   OPERAND_CASE(R600::OpName::src1_abs)
    1309         128 :   OPERAND_CASE(R600::OpName::src1_sel)
    1310           0 :   OPERAND_CASE(R600::OpName::pred_sel)
    1311           0 :   default:
    1312           0 :     llvm_unreachable("Wrong Operand");
    1313             :   }
    1314             : }
    1315             : 
    1316             : #undef OPERAND_CASE
    1317             : 
    1318         128 : MachineInstr *R600InstrInfo::buildSlotOfVectorInstruction(
    1319             :     MachineBasicBlock &MBB, MachineInstr *MI, unsigned Slot, unsigned DstReg)
    1320             :     const {
    1321             :   assert (MI->getOpcode() == R600::DOT_4 && "Not Implemented");
    1322             :   unsigned Opcode;
    1323         128 :   if (ST.getGeneration() <= AMDGPUSubtarget::R700)
    1324             :     Opcode = R600::DOT4_r600;
    1325             :   else
    1326             :     Opcode = R600::DOT4_eg;
    1327             :   MachineBasicBlock::iterator I = MI;
    1328             :   MachineOperand &Src0 = MI->getOperand(
    1329         256 :       getOperandIdx(MI->getOpcode(), getSlotedOps(R600::OpName::src0, Slot)));
    1330             :   MachineOperand &Src1 = MI->getOperand(
    1331         256 :       getOperandIdx(MI->getOpcode(), getSlotedOps(R600::OpName::src1, Slot)));
    1332         128 :   MachineInstr *MIB = buildDefaultInstruction(
    1333         128 :       MBB, I, Opcode, DstReg, Src0.getReg(), Src1.getReg());
    1334             :   static const unsigned  Operands[14] = {
    1335             :     R600::OpName::update_exec_mask,
    1336             :     R600::OpName::update_pred,
    1337             :     R600::OpName::write,
    1338             :     R600::OpName::omod,
    1339             :     R600::OpName::dst_rel,
    1340             :     R600::OpName::clamp,
    1341             :     R600::OpName::src0_neg,
    1342             :     R600::OpName::src0_rel,
    1343             :     R600::OpName::src0_abs,
    1344             :     R600::OpName::src0_sel,
    1345             :     R600::OpName::src1_neg,
    1346             :     R600::OpName::src1_rel,
    1347             :     R600::OpName::src1_abs,
    1348             :     R600::OpName::src1_sel,
    1349             :   };
    1350             : 
    1351         256 :   MachineOperand &MO = MI->getOperand(getOperandIdx(MI->getOpcode(),
    1352         128 :       getSlotedOps(R600::OpName::pred_sel, Slot)));
    1353         128 :   MIB->getOperand(getOperandIdx(Opcode, R600::OpName::pred_sel))
    1354         128 :       .setReg(MO.getReg());
    1355             : 
    1356        1920 :   for (unsigned i = 0; i < 14; i++) {
    1357             :     MachineOperand &MO = MI->getOperand(
    1358        3584 :         getOperandIdx(MI->getOpcode(), getSlotedOps(Operands[i], Slot)));
    1359             :     assert (MO.isImm());
    1360        1792 :     setImmOperand(*MIB, Operands[i], MO.getImm());
    1361             :   }
    1362         128 :   MIB->getOperand(20).setImm(0);
    1363         128 :   return MIB;
    1364             : }
    1365             : 
    1366         562 : MachineInstr *R600InstrInfo::buildMovImm(MachineBasicBlock &BB,
    1367             :                                          MachineBasicBlock::iterator I,
    1368             :                                          unsigned DstReg,
    1369             :                                          uint64_t Imm) const {
    1370         562 :   MachineInstr *MovImm = buildDefaultInstruction(BB, I, R600::MOV, DstReg,
    1371         562 :                                                   R600::ALU_LITERAL_X);
    1372         562 :   setImmOperand(*MovImm, R600::OpName::literal, Imm);
    1373         562 :   return MovImm;
    1374             : }
    1375             : 
    1376        2706 : MachineInstr *R600InstrInfo::buildMovInstr(MachineBasicBlock *MBB,
    1377             :                                        MachineBasicBlock::iterator I,
    1378             :                                        unsigned DstReg, unsigned SrcReg) const {
    1379        2706 :   return buildDefaultInstruction(*MBB, I, R600::MOV, DstReg, SrcReg);
    1380             : }
    1381             : 
    1382       11770 : int R600InstrInfo::getOperandIdx(const MachineInstr &MI, unsigned Op) const {
    1383       23540 :   return getOperandIdx(MI.getOpcode(), Op);
    1384             : }
    1385             : 
    1386     4228030 : int R600InstrInfo::getOperandIdx(unsigned Opcode, unsigned Op) const {
    1387     4228030 :   return R600::getNamedOperandIdx(Opcode, Op);
    1388             : }
    1389             : 
    1390        6618 : void R600InstrInfo::setImmOperand(MachineInstr &MI, unsigned Op,
    1391             :                                   int64_t Imm) const {
    1392        6618 :   int Idx = getOperandIdx(MI, Op);
    1393             :   assert(Idx != -1 && "Operand not supported for this instruction.");
    1394             :   assert(MI.getOperand(Idx).isImm());
    1395        6618 :   MI.getOperand(Idx).setImm(Imm);
    1396        6618 : }
    1397             : 
    1398             : //===----------------------------------------------------------------------===//
    1399             : // Instruction flag getters/setters
    1400             : //===----------------------------------------------------------------------===//
    1401             : 
    1402        1784 : MachineOperand &R600InstrInfo::getFlagOp(MachineInstr &MI, unsigned SrcIdx,
    1403             :                                          unsigned Flag) const {
    1404        3568 :   unsigned TargetFlags = get(MI.getOpcode()).TSFlags;
    1405             :   int FlagIndex = 0;
    1406        1784 :   if (Flag != 0) {
    1407             :     // If we pass something other than the default value of Flag to this
    1408             :     // function, it means we are want to set a flag on an instruction
    1409             :     // that uses native encoding.
    1410             :     assert(HAS_NATIVE_OPERANDS(TargetFlags));
    1411             :     bool IsOP3 = (TargetFlags & R600_InstFlag::OP3) == R600_InstFlag::OP3;
    1412        1184 :     switch (Flag) {
    1413           0 :     case MO_FLAG_CLAMP:
    1414           0 :       FlagIndex = getOperandIdx(MI, R600::OpName::clamp);
    1415           0 :       break;
    1416         652 :     case MO_FLAG_MASK:
    1417         652 :       FlagIndex = getOperandIdx(MI, R600::OpName::write);
    1418         652 :       break;
    1419         492 :     case MO_FLAG_NOT_LAST:
    1420             :     case MO_FLAG_LAST:
    1421         492 :       FlagIndex = getOperandIdx(MI, R600::OpName::last);
    1422         492 :       break;
    1423          20 :     case MO_FLAG_NEG:
    1424             :       switch (SrcIdx) {
    1425          20 :       case 0:
    1426          20 :         FlagIndex = getOperandIdx(MI, R600::OpName::src0_neg);
    1427          20 :         break;
    1428           0 :       case 1:
    1429           0 :         FlagIndex = getOperandIdx(MI, R600::OpName::src1_neg);
    1430           0 :         break;
    1431           0 :       case 2:
    1432           0 :         FlagIndex = getOperandIdx(MI, R600::OpName::src2_neg);
    1433           0 :         break;
    1434             :       }
    1435             :       break;
    1436             : 
    1437          20 :     case MO_FLAG_ABS:
    1438             :       assert(!IsOP3 && "Cannot set absolute value modifier for OP3 "
    1439             :                        "instructions.");
    1440             :       (void)IsOP3;
    1441             :       switch (SrcIdx) {
    1442          20 :       case 0:
    1443          20 :         FlagIndex = getOperandIdx(MI, R600::OpName::src0_abs);
    1444          20 :         break;
    1445           0 :       case 1:
    1446           0 :         FlagIndex = getOperandIdx(MI, R600::OpName::src1_abs);
    1447           0 :         break;
    1448             :       }
    1449             :       break;
    1450             : 
    1451             :     default:
    1452             :       FlagIndex = -1;
    1453             :       break;
    1454             :     }
    1455             :     assert(FlagIndex != -1 && "Flag not supported for this instruction");
    1456             :   } else {
    1457         600 :       FlagIndex = GET_FLAG_OPERAND_IDX(TargetFlags);
    1458             :       assert(FlagIndex != 0 &&
    1459             :          "Instruction flags not supported for this instruction");
    1460             :   }
    1461             : 
    1462        1784 :   MachineOperand &FlagOp = MI.getOperand(FlagIndex);
    1463             :   assert(FlagOp.isImm());
    1464        1784 :   return FlagOp;
    1465             : }
    1466             : 
    1467         900 : void R600InstrInfo::addFlag(MachineInstr &MI, unsigned Operand,
    1468             :                             unsigned Flag) const {
    1469        1800 :   unsigned TargetFlags = get(MI.getOpcode()).TSFlags;
    1470         900 :   if (Flag == 0) {
    1471             :     return;
    1472             :   }
    1473         900 :   if (HAS_NATIVE_OPERANDS(TargetFlags)) {
    1474         612 :     MachineOperand &FlagOp = getFlagOp(MI, Operand, Flag);
    1475         612 :     if (Flag == MO_FLAG_NOT_LAST) {
    1476         246 :       clearFlag(MI, Operand, MO_FLAG_LAST);
    1477         366 :     } else if (Flag == MO_FLAG_MASK) {
    1478         326 :       clearFlag(MI, Operand, Flag);
    1479             :     } else {
    1480             :       FlagOp.setImm(1);
    1481             :     }
    1482             :   } else {
    1483         288 :       MachineOperand &FlagOp = getFlagOp(MI, Operand);
    1484         288 :       FlagOp.setImm(FlagOp.getImm() | (Flag << (NUM_MO_FLAGS * Operand)));
    1485             :   }
    1486             : }
    1487             : 
    1488         798 : void R600InstrInfo::clearFlag(MachineInstr &MI, unsigned Operand,
    1489             :                               unsigned Flag) const {
    1490        1596 :   unsigned TargetFlags = get(MI.getOpcode()).TSFlags;
    1491         798 :   if (HAS_NATIVE_OPERANDS(TargetFlags)) {
    1492         572 :     MachineOperand &FlagOp = getFlagOp(MI, Operand, Flag);
    1493             :     FlagOp.setImm(0);
    1494             :   } else {
    1495         226 :     MachineOperand &FlagOp = getFlagOp(MI);
    1496         226 :     unsigned InstFlags = FlagOp.getImm();
    1497         226 :     InstFlags &= ~(Flag << (NUM_MO_FLAGS * Operand));
    1498         226 :     FlagOp.setImm(InstFlags);
    1499             :   }
    1500         798 : }
    1501             : 
    1502       10765 : unsigned R600InstrInfo::getAddressSpaceForPseudoSourceKind(
    1503             :     unsigned Kind) const {
    1504             :   switch (Kind) {
    1505             :   case PseudoSourceValue::Stack:
    1506             :   case PseudoSourceValue::FixedStack:
    1507             :     return AMDGPUAS::PRIVATE_ADDRESS;
    1508             :   case PseudoSourceValue::ConstantPool:
    1509             :   case PseudoSourceValue::GOT:
    1510             :   case PseudoSourceValue::JumpTable:
    1511             :   case PseudoSourceValue::GlobalValueCallEntry:
    1512             :   case PseudoSourceValue::ExternalSymbolCallEntry:
    1513             :   case PseudoSourceValue::TargetCustom:
    1514             :     return AMDGPUAS::CONSTANT_ADDRESS;
    1515             :   }
    1516             : 
    1517           0 :   llvm_unreachable("Invalid pseudo source kind");
    1518             : }

Generated by: LCOV version 1.13