LCOV - code coverage report
Current view: top level - lib/CodeGen/GlobalISel - LegalizerHelper.cpp (source / functions) Hit Total Coverage
Test: llvm-toolchain.info Lines: 451 498 90.6 %
Date: 2017-09-14 15:23:50 Functions: 12 12 100.0 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : //===-- llvm/CodeGen/GlobalISel/LegalizerHelper.cpp -----------------------===//
       2             : //
       3             : //                     The LLVM Compiler Infrastructure
       4             : //
       5             : // This file is distributed under the University of Illinois Open Source
       6             : // License. See LICENSE.TXT for details.
       7             : //
       8             : //===----------------------------------------------------------------------===//
       9             : //
      10             : /// \file This file implements the LegalizerHelper class to legalize
      11             : /// individual instructions and the LegalizeMachineIR wrapper pass for the
      12             : /// primary legalization.
      13             : //
      14             : //===----------------------------------------------------------------------===//
      15             : 
      16             : #include "llvm/CodeGen/GlobalISel/LegalizerHelper.h"
      17             : #include "llvm/CodeGen/GlobalISel/CallLowering.h"
      18             : #include "llvm/CodeGen/GlobalISel/LegalizerInfo.h"
      19             : #include "llvm/CodeGen/MachineRegisterInfo.h"
      20             : #include "llvm/Support/Debug.h"
      21             : #include "llvm/Support/raw_ostream.h"
      22             : #include "llvm/Target/TargetLowering.h"
      23             : #include "llvm/Target/TargetSubtargetInfo.h"
      24             : 
      25             : #include <sstream>
      26             : 
      27             : #define DEBUG_TYPE "legalizer"
      28             : 
      29             : using namespace llvm;
      30             : 
      31         986 : LegalizerHelper::LegalizerHelper(MachineFunction &MF)
      32        1972 :     : MRI(MF.getRegInfo()), LI(*MF.getSubtarget().getLegalizerInfo()) {
      33         986 :   MIRBuilder.setMF(MF);
      34         986 : }
      35             : 
      36             : LegalizerHelper::LegalizeResult
      37        3647 : LegalizerHelper::legalizeInstrStep(MachineInstr &MI) {
      38             :   DEBUG(dbgs() << "Legalizing: "; MI.print(dbgs()));
      39             : 
      40        3647 :   auto Action = LI.getAction(MI, MRI);
      41        3647 :   switch (std::get<0>(Action)) {
      42             :   case LegalizerInfo::Legal:
      43             :     DEBUG(dbgs() << ".. Already legal\n");
      44             :     return AlreadyLegal;
      45          72 :   case LegalizerInfo::Libcall:
      46             :     DEBUG(dbgs() << ".. Convert to libcall\n");
      47          72 :     return libcall(MI);
      48          38 :   case LegalizerInfo::NarrowScalar:
      49             :     DEBUG(dbgs() << ".. Narrow scalar\n");
      50          76 :     return narrowScalar(MI, std::get<1>(Action), std::get<2>(Action));
      51         204 :   case LegalizerInfo::WidenScalar:
      52             :     DEBUG(dbgs() << ".. Widen scalar\n");
      53         408 :     return widenScalar(MI, std::get<1>(Action), std::get<2>(Action));
      54          30 :   case LegalizerInfo::Lower:
      55             :     DEBUG(dbgs() << ".. Lower\n");
      56          60 :     return lower(MI, std::get<1>(Action), std::get<2>(Action));
      57          33 :   case LegalizerInfo::FewerElements:
      58             :     DEBUG(dbgs() << ".. Reduce number of elements\n");
      59          66 :     return fewerElementsVector(MI, std::get<1>(Action), std::get<2>(Action));
      60          83 :   case LegalizerInfo::Custom:
      61             :     DEBUG(dbgs() << ".. Custom legalization\n");
      62          83 :     return LI.legalizeCustom(MI, MRI, MIRBuilder) ? Legalized
      63             :                                                   : UnableToLegalize;
      64          10 :   default:
      65             :     DEBUG(dbgs() << ".. Unable to legalize\n");
      66          10 :     return UnableToLegalize;
      67             :   }
      68             : }
      69             : 
      70          90 : void LegalizerHelper::extractParts(unsigned Reg, LLT Ty, int NumParts,
      71             :                                    SmallVectorImpl<unsigned> &VRegs) {
      72         306 :   for (int i = 0; i < NumParts; ++i)
      73         216 :     VRegs.push_back(MRI.createGenericVirtualRegister(Ty));
      74         180 :   MIRBuilder.buildUnmerge(VRegs, Reg);
      75          90 : }
      76             : 
      77          72 : static RTLIB::Libcall getRTLibDesc(unsigned Opcode, unsigned Size) {
      78          72 :   switch (Opcode) {
      79             :   case TargetOpcode::G_SDIV:
      80             :     assert(Size == 32 && "Unsupported size");
      81             :     return RTLIB::SDIV_I32;
      82          12 :   case TargetOpcode::G_UDIV:
      83             :     assert(Size == 32 && "Unsupported size");
      84          12 :     return RTLIB::UDIV_I32;
      85           6 :   case TargetOpcode::G_SREM:
      86             :     assert(Size == 32 && "Unsupported size");
      87           6 :     return RTLIB::SREM_I32;
      88           6 :   case TargetOpcode::G_UREM:
      89             :     assert(Size == 32 && "Unsupported size");
      90           6 :     return RTLIB::UREM_I32;
      91           8 :   case TargetOpcode::G_FADD:
      92             :     assert((Size == 32 || Size == 64) && "Unsupported size");
      93           8 :     return Size == 64 ? RTLIB::ADD_F64 : RTLIB::ADD_F32;
      94          14 :   case TargetOpcode::G_FREM:
      95          14 :     return Size == 64 ? RTLIB::REM_F64 : RTLIB::REM_F32;
      96          14 :   case TargetOpcode::G_FPOW:
      97          14 :     return Size == 64 ? RTLIB::POW_F64 : RTLIB::POW_F32;
      98             :   }
      99           0 :   llvm_unreachable("Unknown libcall function");
     100             : }
     101             : 
     102             : LegalizerHelper::LegalizeResult
     103         154 : llvm::createLibcall(MachineIRBuilder &MIRBuilder, RTLIB::Libcall Libcall,
     104             :                     const CallLowering::ArgInfo &Result,
     105             :                     ArrayRef<CallLowering::ArgInfo> Args) {
     106         154 :   auto &CLI = *MIRBuilder.getMF().getSubtarget().getCallLowering();
     107         154 :   auto &TLI = *MIRBuilder.getMF().getSubtarget().getTargetLowering();
     108         308 :   const char *Name = TLI.getLibcallName(Libcall);
     109             : 
     110         308 :   MIRBuilder.getMF().getFrameInfo().setHasCalls(true);
     111         462 :   if (!CLI.lowerCall(MIRBuilder, TLI.getLibcallCallingConv(Libcall),
     112         462 :                      MachineOperand::CreateES(Name), Result, Args))
     113             :     return LegalizerHelper::UnableToLegalize;
     114             : 
     115         154 :   return LegalizerHelper::Legalized;
     116             : }
     117             : 
     118             : static LegalizerHelper::LegalizeResult
     119          72 : simpleLibcall(MachineInstr &MI, MachineIRBuilder &MIRBuilder, unsigned Size,
     120             :               Type *OpType) {
     121         144 :   auto Libcall = getRTLibDesc(MI.getOpcode(), Size);
     122         504 :   return createLibcall(MIRBuilder, Libcall, {MI.getOperand(0).getReg(), OpType},
     123          72 :                        {{MI.getOperand(1).getReg(), OpType},
     124         216 :                         {MI.getOperand(2).getReg(), OpType}});
     125             : }
     126             : 
     127             : LegalizerHelper::LegalizeResult
     128          72 : LegalizerHelper::libcall(MachineInstr &MI) {
     129          72 :   LLT LLTy = MRI.getType(MI.getOperand(0).getReg());
     130          72 :   unsigned Size = LLTy.getSizeInBits();
     131          72 :   auto &Ctx = MIRBuilder.getMF().getFunction()->getContext();
     132             : 
     133          72 :   MIRBuilder.setInstr(MI);
     134             : 
     135          72 :   switch (MI.getOpcode()) {
     136             :   default:
     137             :     return UnableToLegalize;
     138          36 :   case TargetOpcode::G_SDIV:
     139             :   case TargetOpcode::G_UDIV:
     140             :   case TargetOpcode::G_SREM:
     141             :   case TargetOpcode::G_UREM: {
     142          36 :     Type *HLTy = Type::getInt32Ty(Ctx);
     143          36 :     auto Status = simpleLibcall(MI, MIRBuilder, Size, HLTy);
     144          36 :     if (Status != Legalized)
     145             :       return Status;
     146             :     break;
     147             :   }
     148          36 :   case TargetOpcode::G_FADD:
     149             :   case TargetOpcode::G_FPOW:
     150             :   case TargetOpcode::G_FREM: {
     151          36 :     Type *HLTy = Size == 64 ? Type::getDoubleTy(Ctx) : Type::getFloatTy(Ctx);
     152          36 :     auto Status = simpleLibcall(MI, MIRBuilder, Size, HLTy);
     153          36 :     if (Status != Legalized)
     154             :       return Status;
     155             :     break;
     156             :   }
     157             :   }
     158             : 
     159          72 :   MI.eraseFromParent();
     160          72 :   return Legalized;
     161             : }
     162             : 
     163          38 : LegalizerHelper::LegalizeResult LegalizerHelper::narrowScalar(MachineInstr &MI,
     164             :                                                               unsigned TypeIdx,
     165             :                                                               LLT NarrowTy) {
     166             :   // FIXME: Don't know how to handle secondary types yet.
     167          46 :   if (TypeIdx != 0 && MI.getOpcode() != TargetOpcode::G_EXTRACT)
     168             :     return UnableToLegalize;
     169             : 
     170          38 :   MIRBuilder.setInstr(MI);
     171             : 
     172          76 :   switch (MI.getOpcode()) {
     173             :   default:
     174             :     return UnableToLegalize;
     175           3 :   case TargetOpcode::G_IMPLICIT_DEF: {
     176           9 :     int NumParts = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits() /
     177           6 :                    NarrowTy.getSizeInBits();
     178             : 
     179           6 :     SmallVector<unsigned, 2> DstRegs;
     180           9 :     for (int i = 0; i < NumParts; ++i) {
     181           6 :       unsigned Dst = MRI.createGenericVirtualRegister(NarrowTy);
     182           6 :       MIRBuilder.buildUndef(Dst);
     183           6 :       DstRegs.push_back(Dst);
     184             :     }
     185           6 :     MIRBuilder.buildMerge(MI.getOperand(0).getReg(), DstRegs);
     186           3 :     MI.eraseFromParent();
     187           3 :     return Legalized;
     188             :   }
     189           3 :   case TargetOpcode::G_ADD: {
     190             :     // Expand in terms of carry-setting/consuming G_ADDE instructions.
     191           9 :     int NumParts = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits() /
     192           6 :                    NarrowTy.getSizeInBits();
     193             : 
     194          18 :     SmallVector<unsigned, 2> Src1Regs, Src2Regs, DstRegs;
     195           3 :     extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, Src1Regs);
     196           3 :     extractParts(MI.getOperand(2).getReg(), NarrowTy, NumParts, Src2Regs);
     197             : 
     198           6 :     unsigned CarryIn = MRI.createGenericVirtualRegister(LLT::scalar(1));
     199           3 :     MIRBuilder.buildConstant(CarryIn, 0);
     200             : 
     201           9 :     for (int i = 0; i < NumParts; ++i) {
     202           6 :       unsigned DstReg = MRI.createGenericVirtualRegister(NarrowTy);
     203          12 :       unsigned CarryOut = MRI.createGenericVirtualRegister(LLT::scalar(1));
     204             : 
     205          18 :       MIRBuilder.buildUAdde(DstReg, CarryOut, Src1Regs[i],
     206          18 :                             Src2Regs[i], CarryIn);
     207             : 
     208           6 :       DstRegs.push_back(DstReg);
     209           6 :       CarryIn = CarryOut;
     210             :     }
     211           3 :     unsigned DstReg = MI.getOperand(0).getReg();
     212           6 :     MIRBuilder.buildMerge(DstReg, DstRegs);
     213           3 :     MI.eraseFromParent();
     214           3 :     return Legalized;
     215             :   }
     216           8 :   case TargetOpcode::G_EXTRACT: {
     217           8 :     if (TypeIdx != 1)
     218             :       return UnableToLegalize;
     219             : 
     220           8 :     int64_t NarrowSize = NarrowTy.getSizeInBits();
     221             :     int NumParts =
     222           8 :         MRI.getType(MI.getOperand(1).getReg()).getSizeInBits() / NarrowSize;
     223             : 
     224          24 :     SmallVector<unsigned, 2> SrcRegs, DstRegs;
     225          16 :     SmallVector<uint64_t, 2> Indexes;
     226           8 :     extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, SrcRegs);
     227             : 
     228           8 :     unsigned OpReg = MI.getOperand(0).getReg();
     229           8 :     int64_t OpStart = MI.getOperand(2).getImm();
     230           8 :     int64_t OpSize = MRI.getType(OpReg).getSizeInBits();
     231          24 :     for (int i = 0; i < NumParts; ++i) {
     232          16 :       unsigned SrcStart = i * NarrowSize;
     233             : 
     234          16 :       if (SrcStart + NarrowSize <= OpStart || SrcStart >= OpStart + OpSize) {
     235             :         // No part of the extract uses this subregister, ignore it.
     236          17 :         continue;
     237          21 :       } else if (SrcStart == OpStart && NarrowTy == MRI.getType(OpReg)) {
     238             :         // The entire subregister is extracted, forward the value.
     239           6 :         DstRegs.push_back(SrcRegs[i]);
     240           3 :         continue;
     241             :       }
     242             : 
     243             :       // OpSegStart is where this destination segment would start in OpReg if it
     244             :       // extended infinitely in both directions.
     245             :       int64_t ExtractOffset, SegSize;
     246           6 :       if (OpStart < SrcStart) {
     247           1 :         ExtractOffset = 0;
     248           2 :         SegSize = std::min(NarrowSize, OpStart + OpSize - SrcStart);
     249             :       } else {
     250           5 :         ExtractOffset = OpStart - SrcStart;
     251          10 :         SegSize = std::min(SrcStart + NarrowSize - OpStart, OpSize);
     252             :       }
     253             : 
     254          12 :       unsigned SegReg = SrcRegs[i];
     255           6 :       if (ExtractOffset != 0 || SegSize != NarrowSize) {
     256             :         // A genuine extract is needed.
     257          10 :         SegReg = MRI.createGenericVirtualRegister(LLT::scalar(SegSize));
     258          10 :         MIRBuilder.buildExtract(SegReg, SrcRegs[i], ExtractOffset);
     259             :       }
     260             : 
     261           6 :       DstRegs.push_back(SegReg);
     262             :     }
     263             : 
     264          16 :     MIRBuilder.buildMerge(MI.getOperand(0).getReg(), DstRegs);
     265           8 :     MI.eraseFromParent();
     266           8 :     return Legalized;
     267             :   }
     268           8 :   case TargetOpcode::G_INSERT: {
     269           8 :     if (TypeIdx != 0)
     270             :       return UnableToLegalize;
     271             : 
     272           8 :     int64_t NarrowSize = NarrowTy.getSizeInBits();
     273             :     int NumParts =
     274           8 :         MRI.getType(MI.getOperand(0).getReg()).getSizeInBits() / NarrowSize;
     275             : 
     276          24 :     SmallVector<unsigned, 2> SrcRegs, DstRegs;
     277          16 :     SmallVector<uint64_t, 2> Indexes;
     278           8 :     extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, SrcRegs);
     279             : 
     280           8 :     unsigned OpReg = MI.getOperand(2).getReg();
     281           8 :     int64_t OpStart = MI.getOperand(3).getImm();
     282           8 :     int64_t OpSize = MRI.getType(OpReg).getSizeInBits();
     283          24 :     for (int i = 0; i < NumParts; ++i) {
     284          16 :       unsigned DstStart = i * NarrowSize;
     285             : 
     286          23 :       if (DstStart + NarrowSize <= OpStart || DstStart >= OpStart + OpSize) {
     287             :         // No part of the insert affects this subregister, forward the original.
     288          14 :         DstRegs.push_back(SrcRegs[i]);
     289          16 :         continue;
     290          20 :       } else if (DstStart == OpStart && NarrowTy == MRI.getType(OpReg)) {
     291             :         // The entire subregister is defined by this insert, forward the new
     292             :         // value.
     293           2 :         DstRegs.push_back(OpReg);
     294           2 :         continue;
     295             :       }
     296             : 
     297             :       // OpSegStart is where this destination segment would start in OpReg if it
     298             :       // extended infinitely in both directions.
     299             :       int64_t ExtractOffset, InsertOffset, SegSize;
     300           7 :       if (OpStart < DstStart) {
     301           1 :         InsertOffset = 0;
     302           1 :         ExtractOffset = DstStart - OpStart;
     303           2 :         SegSize = std::min(NarrowSize, OpStart + OpSize - DstStart);
     304             :       } else {
     305           6 :         InsertOffset = OpStart - DstStart;
     306           6 :         ExtractOffset = 0;
     307           6 :         SegSize =
     308          18 :             std::min(NarrowSize - InsertOffset, OpStart + OpSize - DstStart);
     309             :       }
     310             : 
     311           7 :       unsigned SegReg = OpReg;
     312           7 :       if (ExtractOffset != 0 || SegSize != OpSize) {
     313             :         // A genuine extract is needed.
     314           4 :         SegReg = MRI.createGenericVirtualRegister(LLT::scalar(SegSize));
     315           2 :         MIRBuilder.buildExtract(SegReg, OpReg, ExtractOffset);
     316             :       }
     317             : 
     318           7 :       unsigned DstReg = MRI.createGenericVirtualRegister(NarrowTy);
     319          14 :       MIRBuilder.buildInsert(DstReg, SrcRegs[i], SegReg, InsertOffset);
     320           7 :       DstRegs.push_back(DstReg);
     321             :     }
     322             : 
     323             :     assert(DstRegs.size() == (unsigned)NumParts && "not all parts covered");
     324          16 :     MIRBuilder.buildMerge(MI.getOperand(0).getReg(), DstRegs);
     325           8 :     MI.eraseFromParent();
     326           8 :     return Legalized;
     327             :   }
     328           8 :   case TargetOpcode::G_LOAD: {
     329           8 :     unsigned NarrowSize = NarrowTy.getSizeInBits();
     330             :     int NumParts =
     331           8 :         MRI.getType(MI.getOperand(0).getReg()).getSizeInBits() / NarrowSize;
     332             :     LLT OffsetTy = LLT::scalar(
     333          24 :         MRI.getType(MI.getOperand(1).getReg()).getScalarSizeInBits());
     334             : 
     335          16 :     SmallVector<unsigned, 2> DstRegs;
     336          24 :     for (int i = 0; i < NumParts; ++i) {
     337          16 :       unsigned DstReg = MRI.createGenericVirtualRegister(NarrowTy);
     338          16 :       unsigned SrcReg = 0;
     339          16 :       unsigned Adjustment = i * NarrowSize / 8;
     340             : 
     341          32 :       MIRBuilder.materializeGEP(SrcReg, MI.getOperand(1).getReg(), OffsetTy,
     342             :                                 Adjustment);
     343             : 
     344             :       // TODO: This is conservatively correct, but we probably want to split the
     345             :       // memory operands in the future.
     346          16 :       MIRBuilder.buildLoad(DstReg, SrcReg, **MI.memoperands_begin());
     347             : 
     348          16 :       DstRegs.push_back(DstReg);
     349             :     }
     350           8 :     unsigned DstReg = MI.getOperand(0).getReg();
     351          16 :     MIRBuilder.buildMerge(DstReg, DstRegs);
     352           8 :     MI.eraseFromParent();
     353           8 :     return Legalized;
     354             :   }
     355           6 :   case TargetOpcode::G_STORE: {
     356           6 :     unsigned NarrowSize = NarrowTy.getSizeInBits();
     357             :     int NumParts =
     358           6 :         MRI.getType(MI.getOperand(0).getReg()).getSizeInBits() / NarrowSize;
     359             :     LLT OffsetTy = LLT::scalar(
     360          18 :         MRI.getType(MI.getOperand(1).getReg()).getScalarSizeInBits());
     361             : 
     362          12 :     SmallVector<unsigned, 2> SrcRegs;
     363           6 :     extractParts(MI.getOperand(0).getReg(), NarrowTy, NumParts, SrcRegs);
     364             : 
     365          18 :     for (int i = 0; i < NumParts; ++i) {
     366          12 :       unsigned DstReg = 0;
     367          12 :       unsigned Adjustment = i * NarrowSize / 8;
     368             : 
     369          24 :       MIRBuilder.materializeGEP(DstReg, MI.getOperand(1).getReg(), OffsetTy,
     370             :                                 Adjustment);
     371             : 
     372             :       // TODO: This is conservatively correct, but we probably want to split the
     373             :       // memory operands in the future.
     374          24 :       MIRBuilder.buildStore(SrcRegs[i], DstReg, **MI.memoperands_begin());
     375             :     }
     376           6 :     MI.eraseFromParent();
     377           6 :     return Legalized;
     378             :   }
     379           2 :   case TargetOpcode::G_CONSTANT: {
     380           2 :     unsigned NarrowSize = NarrowTy.getSizeInBits();
     381             :     int NumParts =
     382           2 :         MRI.getType(MI.getOperand(0).getReg()).getSizeInBits() / NarrowSize;
     383           4 :     const APInt &Cst = MI.getOperand(1).getCImm()->getValue();
     384           2 :     LLVMContext &Ctx = MIRBuilder.getMF().getFunction()->getContext();
     385             : 
     386           4 :     SmallVector<unsigned, 2> DstRegs;
     387           6 :     for (int i = 0; i < NumParts; ++i) {
     388           4 :       unsigned DstReg = MRI.createGenericVirtualRegister(NarrowTy);
     389             :       ConstantInt *CI =
     390          12 :           ConstantInt::get(Ctx, Cst.lshr(NarrowSize * i).trunc(NarrowSize));
     391           4 :       MIRBuilder.buildConstant(DstReg, *CI);
     392           4 :       DstRegs.push_back(DstReg);
     393             :     }
     394           2 :     unsigned DstReg = MI.getOperand(0).getReg();
     395           4 :     MIRBuilder.buildMerge(DstReg, DstRegs);
     396           2 :     MI.eraseFromParent();
     397           2 :     return Legalized;
     398             :   }
     399             :   }
     400             : }
     401             : 
     402             : LegalizerHelper::LegalizeResult
     403         204 : LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
     404         204 :   MIRBuilder.setInstr(MI);
     405             : 
     406         408 :   switch (MI.getOpcode()) {
     407             :   default:
     408             :     return UnableToLegalize;
     409          49 :   case TargetOpcode::G_ADD:
     410             :   case TargetOpcode::G_AND:
     411             :   case TargetOpcode::G_MUL:
     412             :   case TargetOpcode::G_OR:
     413             :   case TargetOpcode::G_XOR:
     414             :   case TargetOpcode::G_SUB:
     415             :   case TargetOpcode::G_SHL: {
     416             :     // Perform operation at larger width (any extension is fine here, high bits
     417             :     // don't affect the result) and then truncate the result back to the
     418             :     // original type.
     419          49 :     unsigned Src1Ext = MRI.createGenericVirtualRegister(WideTy);
     420          49 :     unsigned Src2Ext = MRI.createGenericVirtualRegister(WideTy);
     421          49 :     MIRBuilder.buildAnyExt(Src1Ext, MI.getOperand(1).getReg());
     422          49 :     MIRBuilder.buildAnyExt(Src2Ext, MI.getOperand(2).getReg());
     423             : 
     424          49 :     unsigned DstExt = MRI.createGenericVirtualRegister(WideTy);
     425         147 :     MIRBuilder.buildInstr(MI.getOpcode())
     426          49 :         .addDef(DstExt)
     427          49 :         .addUse(Src1Ext)
     428          49 :         .addUse(Src2Ext);
     429             : 
     430          49 :     MIRBuilder.buildTrunc(MI.getOperand(0).getReg(), DstExt);
     431          49 :     MI.eraseFromParent();
     432          49 :     return Legalized;
     433             :   }
     434          69 :   case TargetOpcode::G_SDIV:
     435             :   case TargetOpcode::G_UDIV:
     436             :   case TargetOpcode::G_SREM:
     437             :   case TargetOpcode::G_UREM:
     438             :   case TargetOpcode::G_ASHR:
     439             :   case TargetOpcode::G_LSHR: {
     440         120 :     unsigned ExtOp = MI.getOpcode() == TargetOpcode::G_SDIV ||
     441          86 :                              MI.getOpcode() == TargetOpcode::G_SREM ||
     442          35 :                              MI.getOpcode() == TargetOpcode::G_ASHR
     443          69 :                          ? TargetOpcode::G_SEXT
     444          69 :                          : TargetOpcode::G_ZEXT;
     445             : 
     446          69 :     unsigned LHSExt = MRI.createGenericVirtualRegister(WideTy);
     447         207 :     MIRBuilder.buildInstr(ExtOp).addDef(LHSExt).addUse(
     448          69 :         MI.getOperand(1).getReg());
     449             : 
     450          69 :     unsigned RHSExt = MRI.createGenericVirtualRegister(WideTy);
     451         207 :     MIRBuilder.buildInstr(ExtOp).addDef(RHSExt).addUse(
     452          69 :         MI.getOperand(2).getReg());
     453             : 
     454          69 :     unsigned ResExt = MRI.createGenericVirtualRegister(WideTy);
     455         207 :     MIRBuilder.buildInstr(MI.getOpcode())
     456          69 :         .addDef(ResExt)
     457          69 :         .addUse(LHSExt)
     458          69 :         .addUse(RHSExt);
     459             : 
     460          69 :     MIRBuilder.buildTrunc(MI.getOperand(0).getReg(), ResExt);
     461          69 :     MI.eraseFromParent();
     462          69 :     return Legalized;
     463             :   }
     464           3 :   case TargetOpcode::G_SELECT: {
     465           3 :     if (TypeIdx != 0)
     466             :       return UnableToLegalize;
     467             : 
     468             :     // Perform operation at larger width (any extension is fine here, high bits
     469             :     // don't affect the result) and then truncate the result back to the
     470             :     // original type.
     471           3 :     unsigned Src1Ext = MRI.createGenericVirtualRegister(WideTy);
     472           3 :     unsigned Src2Ext = MRI.createGenericVirtualRegister(WideTy);
     473           3 :     MIRBuilder.buildAnyExt(Src1Ext, MI.getOperand(2).getReg());
     474           3 :     MIRBuilder.buildAnyExt(Src2Ext, MI.getOperand(3).getReg());
     475             : 
     476           3 :     unsigned DstExt = MRI.createGenericVirtualRegister(WideTy);
     477           6 :     MIRBuilder.buildInstr(TargetOpcode::G_SELECT)
     478           3 :         .addDef(DstExt)
     479           3 :         .addReg(MI.getOperand(1).getReg())
     480           3 :         .addUse(Src1Ext)
     481           3 :         .addUse(Src2Ext);
     482             : 
     483           3 :     MIRBuilder.buildTrunc(MI.getOperand(0).getReg(), DstExt);
     484           3 :     MI.eraseFromParent();
     485           3 :     return Legalized;
     486             :   }
     487           7 :   case TargetOpcode::G_FPTOSI:
     488             :   case TargetOpcode::G_FPTOUI: {
     489           7 :     if (TypeIdx != 0)
     490             :       return UnableToLegalize;
     491             : 
     492           7 :     unsigned DstExt = MRI.createGenericVirtualRegister(WideTy);
     493          21 :     MIRBuilder.buildInstr(MI.getOpcode())
     494           7 :         .addDef(DstExt)
     495          14 :         .addUse(MI.getOperand(1).getReg());
     496             : 
     497           7 :     MIRBuilder.buildTrunc(MI.getOperand(0).getReg(), DstExt);
     498           7 :     MI.eraseFromParent();
     499           7 :     return Legalized;
     500             :   }
     501           6 :   case TargetOpcode::G_SITOFP:
     502             :   case TargetOpcode::G_UITOFP: {
     503           6 :     if (TypeIdx != 1)
     504             :       return UnableToLegalize;
     505             : 
     506           6 :     unsigned Src = MI.getOperand(1).getReg();
     507           6 :     unsigned SrcExt = MRI.createGenericVirtualRegister(WideTy);
     508             : 
     509          12 :     if (MI.getOpcode() == TargetOpcode::G_SITOFP) {
     510           3 :       MIRBuilder.buildSExt(SrcExt, Src);
     511             :     } else {
     512             :       assert(MI.getOpcode() == TargetOpcode::G_UITOFP && "Unexpected conv op");
     513           3 :       MIRBuilder.buildZExt(SrcExt, Src);
     514             :     }
     515             : 
     516          18 :     MIRBuilder.buildInstr(MI.getOpcode())
     517          12 :         .addDef(MI.getOperand(0).getReg())
     518           6 :         .addUse(SrcExt);
     519             : 
     520           6 :     MI.eraseFromParent();
     521           6 :     return Legalized;
     522             :   }
     523           1 :   case TargetOpcode::G_INSERT: {
     524           1 :     if (TypeIdx != 0)
     525             :       return UnableToLegalize;
     526             : 
     527           1 :     unsigned Src = MI.getOperand(1).getReg();
     528           1 :     unsigned SrcExt = MRI.createGenericVirtualRegister(WideTy);
     529           1 :     MIRBuilder.buildAnyExt(SrcExt, Src);
     530             : 
     531           1 :     unsigned DstExt = MRI.createGenericVirtualRegister(WideTy);
     532           1 :     auto MIB = MIRBuilder.buildInsert(DstExt, SrcExt, MI.getOperand(2).getReg(),
     533           2 :                                       MI.getOperand(3).getImm());
     534           1 :     for (unsigned OpNum = 4; OpNum < MI.getNumOperands(); OpNum += 2) {
     535           0 :       MIB.addReg(MI.getOperand(OpNum).getReg());
     536           0 :       MIB.addImm(MI.getOperand(OpNum + 1).getImm());
     537             :     }
     538             : 
     539           1 :     MIRBuilder.buildTrunc(MI.getOperand(0).getReg(), DstExt);
     540           1 :     MI.eraseFromParent();
     541           1 :     return Legalized;
     542             :   }
     543           9 :   case TargetOpcode::G_LOAD: {
     544             :     assert(alignTo(MRI.getType(MI.getOperand(0).getReg()).getSizeInBits(), 8) ==
     545             :                WideTy.getSizeInBits() &&
     546             :            "illegal to increase number of bytes loaded");
     547             : 
     548           9 :     unsigned DstExt = MRI.createGenericVirtualRegister(WideTy);
     549           9 :     MIRBuilder.buildLoad(DstExt, MI.getOperand(1).getReg(),
     550           9 :                          **MI.memoperands_begin());
     551           9 :     MIRBuilder.buildTrunc(MI.getOperand(0).getReg(), DstExt);
     552           9 :     MI.eraseFromParent();
     553           9 :     return Legalized;
     554             :   }
     555           7 :   case TargetOpcode::G_STORE: {
     556          21 :     if (MRI.getType(MI.getOperand(0).getReg()) != LLT::scalar(1) ||
     557          14 :         WideTy != LLT::scalar(8))
     558             :       return UnableToLegalize;
     559             : 
     560           7 :     auto &TLI = *MIRBuilder.getMF().getSubtarget().getTargetLowering();
     561          14 :     auto Content = TLI.getBooleanContents(false, false);
     562             : 
     563           7 :     unsigned ExtOp = TargetOpcode::G_ANYEXT;
     564           7 :     if (Content == TargetLoweringBase::ZeroOrOneBooleanContent)
     565             :       ExtOp = TargetOpcode::G_ZEXT;
     566           0 :     else if (Content == TargetLoweringBase::ZeroOrNegativeOneBooleanContent)
     567             :       ExtOp = TargetOpcode::G_SEXT;
     568             :     else
     569           0 :       ExtOp = TargetOpcode::G_ANYEXT;
     570             : 
     571           7 :     unsigned SrcExt = MRI.createGenericVirtualRegister(WideTy);
     572          21 :     MIRBuilder.buildInstr(ExtOp).addDef(SrcExt).addUse(
     573           7 :         MI.getOperand(0).getReg());
     574           7 :     MIRBuilder.buildStore(SrcExt, MI.getOperand(1).getReg(),
     575           7 :                           **MI.memoperands_begin());
     576           7 :     MI.eraseFromParent();
     577           7 :     return Legalized;
     578             :   }
     579          25 :   case TargetOpcode::G_CONSTANT: {
     580          25 :     unsigned DstExt = MRI.createGenericVirtualRegister(WideTy);
     581          25 :     MIRBuilder.buildConstant(DstExt, *MI.getOperand(1).getCImm());
     582          25 :     MIRBuilder.buildTrunc(MI.getOperand(0).getReg(), DstExt);
     583          25 :     MI.eraseFromParent();
     584          25 :     return Legalized;
     585             :   }
     586           1 :   case TargetOpcode::G_FCONSTANT: {
     587           1 :     unsigned DstExt = MRI.createGenericVirtualRegister(WideTy);
     588           1 :     MIRBuilder.buildFConstant(DstExt, *MI.getOperand(1).getFPImm());
     589           1 :     MIRBuilder.buildFPTrunc(MI.getOperand(0).getReg(), DstExt);
     590           1 :     MI.eraseFromParent();
     591           1 :     return Legalized;
     592             :   }
     593           0 :   case TargetOpcode::G_BRCOND: {
     594           0 :     unsigned TstExt = MRI.createGenericVirtualRegister(WideTy);
     595           0 :     MIRBuilder.buildAnyExt(TstExt, MI.getOperand(0).getReg());
     596           0 :     MIRBuilder.buildBrCond(TstExt, *MI.getOperand(1).getMBB());
     597           0 :     MI.eraseFromParent();
     598           0 :     return Legalized;
     599             :   }
     600           0 :   case TargetOpcode::G_FCMP: {
     601             :     unsigned Op0Ext, Op1Ext, DstReg;
     602           0 :     unsigned Cmp1 = MI.getOperand(2).getReg();
     603           0 :     unsigned Cmp2 = MI.getOperand(3).getReg();
     604           0 :     if (TypeIdx == 0) {
     605           0 :       Op0Ext = Cmp1;
     606           0 :       Op1Ext = Cmp2;
     607           0 :       DstReg = MRI.createGenericVirtualRegister(WideTy);
     608             :     } else {
     609           0 :       Op0Ext = MRI.createGenericVirtualRegister(WideTy);
     610           0 :       Op1Ext = MRI.createGenericVirtualRegister(WideTy);
     611           0 :       DstReg = MI.getOperand(0).getReg();
     612           0 :       MIRBuilder.buildInstr(TargetOpcode::G_FPEXT, Op0Ext, Cmp1);
     613           0 :       MIRBuilder.buildInstr(TargetOpcode::G_FPEXT, Op1Ext, Cmp2);
     614             :     }
     615           0 :     MIRBuilder.buildFCmp(
     616           0 :         static_cast<CmpInst::Predicate>(MI.getOperand(1).getPredicate()),
     617           0 :         DstReg, Op0Ext, Op1Ext);
     618           0 :     if (TypeIdx == 0)
     619           0 :       MIRBuilder.buildInstr(TargetOpcode::G_TRUNC, MI.getOperand(0).getReg(),
     620             :                             DstReg);
     621           0 :     MI.eraseFromParent();
     622             :     return Legalized;
     623             :   }
     624          15 :   case TargetOpcode::G_ICMP: {
     625             :     bool IsSigned = CmpInst::isSigned(
     626          15 :         static_cast<CmpInst::Predicate>(MI.getOperand(1).getPredicate()));
     627          15 :     unsigned Cmp1 = MI.getOperand(2).getReg();
     628          15 :     unsigned Cmp2 = MI.getOperand(3).getReg();
     629             :     unsigned Op0Ext, Op1Ext, DstReg;
     630          15 :     if (TypeIdx == 0) {
     631          11 :       Op0Ext = Cmp1;
     632          11 :       Op1Ext = Cmp2;
     633          11 :       DstReg = MRI.createGenericVirtualRegister(WideTy);
     634             :     } else {
     635           4 :       Op0Ext = MRI.createGenericVirtualRegister(WideTy);
     636           4 :       Op1Ext = MRI.createGenericVirtualRegister(WideTy);
     637           4 :       DstReg = MI.getOperand(0).getReg();
     638           4 :       if (IsSigned) {
     639           2 :         MIRBuilder.buildSExt(Op0Ext, Cmp1);
     640           2 :         MIRBuilder.buildSExt(Op1Ext, Cmp2);
     641             :       } else {
     642           2 :         MIRBuilder.buildZExt(Op0Ext, Cmp1);
     643           2 :         MIRBuilder.buildZExt(Op1Ext, Cmp2);
     644             :       }
     645             :     }
     646          15 :     MIRBuilder.buildICmp(
     647          15 :         static_cast<CmpInst::Predicate>(MI.getOperand(1).getPredicate()),
     648          15 :         DstReg, Op0Ext, Op1Ext);
     649          15 :     if (TypeIdx == 0)
     650          11 :       MIRBuilder.buildInstr(TargetOpcode::G_TRUNC, MI.getOperand(0).getReg(),
     651             :                             DstReg);
     652          15 :     MI.eraseFromParent();
     653             :     return Legalized;
     654             :   }
     655           3 :   case TargetOpcode::G_GEP: {
     656             :     assert(TypeIdx == 1 && "unable to legalize pointer of GEP");
     657           3 :     unsigned OffsetExt = MRI.createGenericVirtualRegister(WideTy);
     658           3 :     MIRBuilder.buildSExt(OffsetExt, MI.getOperand(2).getReg());
     659           6 :     MI.getOperand(2).setReg(OffsetExt);
     660           3 :     return Legalized;
     661             :   }
     662           9 :   case TargetOpcode::G_PHI: {
     663             :     assert(TypeIdx == 0 && "Expecting only Idx 0");
     664          18 :     auto getExtendedReg = [&](unsigned Reg, MachineBasicBlock &MBB) {
     665          18 :       auto FirstTermIt = MBB.getFirstTerminator();
     666          36 :       MIRBuilder.setInsertPt(MBB, FirstTermIt);
     667          18 :       MachineInstr *DefMI = MRI.getVRegDef(Reg);
     668          18 :       MachineInstrBuilder MIB;
     669          36 :       if (DefMI->getOpcode() == TargetOpcode::G_TRUNC)
     670          18 :         MIB = MIRBuilder.buildAnyExtOrTrunc(WideTy,
     671          14 :                                             DefMI->getOperand(1).getReg());
     672             :       else
     673           8 :         MIB = MIRBuilder.buildAnyExt(WideTy, Reg);
     674          18 :       return MIB->getOperand(0).getReg();
     675           9 :     };
     676           9 :     auto MIB = MIRBuilder.buildInstr(TargetOpcode::G_PHI, WideTy);
     677          18 :     for (auto OpIt = MI.operands_begin() + 1, OpE = MI.operands_end();
     678          27 :          OpIt != OpE;) {
     679          18 :       unsigned Reg = OpIt++->getReg();
     680          18 :       MachineBasicBlock *OpMBB = OpIt++->getMBB();
     681          18 :       MIB.addReg(getExtendedReg(Reg, *OpMBB));
     682             :       MIB.addMBB(OpMBB);
     683             :     }
     684           9 :     auto *MBB = MI.getParent();
     685           9 :     MIRBuilder.setInsertPt(*MBB, MBB->getFirstNonPHI());
     686           9 :     MIRBuilder.buildTrunc(MI.getOperand(0).getReg(),
     687           9 :                           MIB->getOperand(0).getReg());
     688           9 :     MI.eraseFromParent();
     689             :     return Legalized;
     690             :   }
     691             :   }
     692             : }
     693             : 
     694             : LegalizerHelper::LegalizeResult
     695          30 : LegalizerHelper::lower(MachineInstr &MI, unsigned TypeIdx, LLT Ty) {
     696             :   using namespace TargetOpcode;
     697          30 :   MIRBuilder.setInstr(MI);
     698             : 
     699          60 :   switch(MI.getOpcode()) {
     700             :   default:
     701             :     return UnableToLegalize;
     702          27 :   case TargetOpcode::G_SREM:
     703             :   case TargetOpcode::G_UREM: {
     704          27 :     unsigned QuotReg = MRI.createGenericVirtualRegister(Ty);
     705          81 :     MIRBuilder.buildInstr(MI.getOpcode() == G_SREM ? G_SDIV : G_UDIV)
     706          27 :         .addDef(QuotReg)
     707          54 :         .addUse(MI.getOperand(1).getReg())
     708          54 :         .addUse(MI.getOperand(2).getReg());
     709             : 
     710          27 :     unsigned ProdReg = MRI.createGenericVirtualRegister(Ty);
     711          27 :     MIRBuilder.buildMul(ProdReg, QuotReg, MI.getOperand(2).getReg());
     712          27 :     MIRBuilder.buildSub(MI.getOperand(0).getReg(), MI.getOperand(1).getReg(),
     713             :                         ProdReg);
     714          27 :     MI.eraseFromParent();
     715          27 :     return Legalized;
     716             :   }
     717           1 :   case TargetOpcode::G_SMULO:
     718             :   case TargetOpcode::G_UMULO: {
     719             :     // Generate G_UMULH/G_SMULH to check for overflow and a normal G_MUL for the
     720             :     // result.
     721           1 :     unsigned Res = MI.getOperand(0).getReg();
     722           1 :     unsigned Overflow = MI.getOperand(1).getReg();
     723           1 :     unsigned LHS = MI.getOperand(2).getReg();
     724           1 :     unsigned RHS = MI.getOperand(3).getReg();
     725             : 
     726           1 :     MIRBuilder.buildMul(Res, LHS, RHS);
     727             : 
     728           1 :     unsigned Opcode = MI.getOpcode() == TargetOpcode::G_SMULO
     729           1 :                           ? TargetOpcode::G_SMULH
     730           1 :                           : TargetOpcode::G_UMULH;
     731             : 
     732           1 :     unsigned HiPart = MRI.createGenericVirtualRegister(Ty);
     733           2 :     MIRBuilder.buildInstr(Opcode)
     734           1 :       .addDef(HiPart)
     735           1 :       .addUse(LHS)
     736           1 :       .addUse(RHS);
     737             : 
     738           1 :     unsigned Zero = MRI.createGenericVirtualRegister(Ty);
     739           1 :     MIRBuilder.buildConstant(Zero, 0);
     740           1 :     MIRBuilder.buildICmp(CmpInst::ICMP_NE, Overflow, HiPart, Zero);
     741           1 :     MI.eraseFromParent();
     742           1 :     return Legalized;
     743             :   }
     744           2 :   case TargetOpcode::G_FNEG: {
     745             :     // TODO: Handle vector types once we are able to
     746             :     // represent them.
     747           2 :     if (Ty.isVector())
     748             :       return UnableToLegalize;
     749           2 :     unsigned Res = MI.getOperand(0).getReg();
     750             :     Type *ZeroTy;
     751           2 :     LLVMContext &Ctx = MIRBuilder.getMF().getFunction()->getContext();
     752           2 :     switch (Ty.getSizeInBits()) {
     753           0 :     case 16:
     754           0 :       ZeroTy = Type::getHalfTy(Ctx);
     755           0 :       break;
     756           1 :     case 32:
     757           1 :       ZeroTy = Type::getFloatTy(Ctx);
     758           1 :       break;
     759           1 :     case 64:
     760           1 :       ZeroTy = Type::getDoubleTy(Ctx);
     761           1 :       break;
     762           0 :     default:
     763           0 :       llvm_unreachable("unexpected floating-point type");
     764             :     }
     765             :     ConstantFP &ZeroForNegation =
     766           4 :         *cast<ConstantFP>(ConstantFP::getZeroValueForNegation(ZeroTy));
     767           2 :     unsigned Zero = MRI.createGenericVirtualRegister(Ty);
     768           2 :     MIRBuilder.buildFConstant(Zero, ZeroForNegation);
     769           4 :     MIRBuilder.buildInstr(TargetOpcode::G_FSUB)
     770           2 :         .addDef(Res)
     771           2 :         .addUse(Zero)
     772           4 :         .addUse(MI.getOperand(1).getReg());
     773           2 :     MI.eraseFromParent();
     774           2 :     return Legalized;
     775             :   }
     776           0 :   case TargetOpcode::G_FSUB: {
     777             :     // Lower (G_FSUB LHS, RHS) to (G_FADD LHS, (G_FNEG RHS)).
     778             :     // First, check if G_FNEG is marked as Lower. If so, we may
     779             :     // end up with an infinite loop as G_FSUB is used to legalize G_FNEG.
     780           0 :     if (LI.getAction({G_FNEG, Ty}).first == LegalizerInfo::Lower)
     781             :       return UnableToLegalize;
     782           0 :     unsigned Res = MI.getOperand(0).getReg();
     783           0 :     unsigned LHS = MI.getOperand(1).getReg();
     784           0 :     unsigned RHS = MI.getOperand(2).getReg();
     785           0 :     unsigned Neg = MRI.createGenericVirtualRegister(Ty);
     786           0 :     MIRBuilder.buildInstr(TargetOpcode::G_FNEG).addDef(Neg).addUse(RHS);
     787           0 :     MIRBuilder.buildInstr(TargetOpcode::G_FADD)
     788           0 :         .addDef(Res)
     789           0 :         .addUse(LHS)
     790           0 :         .addUse(Neg);
     791           0 :     MI.eraseFromParent();
     792           0 :     return Legalized;
     793             :   }
     794             :   }
     795             : }
     796             : 
     797             : LegalizerHelper::LegalizeResult
     798          33 : LegalizerHelper::fewerElementsVector(MachineInstr &MI, unsigned TypeIdx,
     799             :                                      LLT NarrowTy) {
     800             :   // FIXME: Don't know how to handle secondary types yet.
     801          33 :   if (TypeIdx != 0)
     802             :     return UnableToLegalize;
     803          66 :   switch (MI.getOpcode()) {
     804             :   default:
     805             :     return UnableToLegalize;
     806          31 :   case TargetOpcode::G_ADD: {
     807          31 :     unsigned NarrowSize = NarrowTy.getSizeInBits();
     808          31 :     unsigned DstReg = MI.getOperand(0).getReg();
     809          31 :     int NumParts = MRI.getType(DstReg).getSizeInBits() / NarrowSize;
     810             : 
     811          31 :     MIRBuilder.setInstr(MI);
     812             : 
     813         186 :     SmallVector<unsigned, 2> Src1Regs, Src2Regs, DstRegs;
     814          31 :     extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, Src1Regs);
     815          31 :     extractParts(MI.getOperand(2).getReg(), NarrowTy, NumParts, Src2Regs);
     816             : 
     817         111 :     for (int i = 0; i < NumParts; ++i) {
     818          80 :       unsigned DstReg = MRI.createGenericVirtualRegister(NarrowTy);
     819         240 :       MIRBuilder.buildAdd(DstReg, Src1Regs[i], Src2Regs[i]);
     820          80 :       DstRegs.push_back(DstReg);
     821             :     }
     822             : 
     823          62 :     MIRBuilder.buildMerge(DstReg, DstRegs);
     824          31 :     MI.eraseFromParent();
     825          31 :     return Legalized;
     826             :   }
     827             :   }
     828             : }

Generated by: LCOV version 1.13