LCOV - code coverage report
Current view: top level - lib/CodeGen/GlobalISel - LegalizerHelper.cpp (source / functions) Hit Total Coverage
Test: llvm-toolchain.info Lines: 496 549 90.3 %
Date: 2018-06-17 00:07:59 Functions: 15 15 100.0 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : //===-- llvm/CodeGen/GlobalISel/LegalizerHelper.cpp -----------------------===//
       2             : //
       3             : //                     The LLVM Compiler Infrastructure
       4             : //
       5             : // This file is distributed under the University of Illinois Open Source
       6             : // License. See LICENSE.TXT for details.
       7             : //
       8             : //===----------------------------------------------------------------------===//
       9             : //
      10             : /// \file This file implements the LegalizerHelper class to legalize
      11             : /// individual instructions and the LegalizeMachineIR wrapper pass for the
      12             : /// primary legalization.
      13             : //
      14             : //===----------------------------------------------------------------------===//
      15             : 
      16             : #include "llvm/CodeGen/GlobalISel/LegalizerHelper.h"
      17             : #include "llvm/CodeGen/GlobalISel/CallLowering.h"
      18             : #include "llvm/CodeGen/GlobalISel/LegalizerInfo.h"
      19             : #include "llvm/CodeGen/MachineRegisterInfo.h"
      20             : #include "llvm/CodeGen/TargetLowering.h"
      21             : #include "llvm/CodeGen/TargetSubtargetInfo.h"
      22             : #include "llvm/Support/Debug.h"
      23             : #include "llvm/Support/raw_ostream.h"
      24             : 
      25             : 
      26             : #define DEBUG_TYPE "legalizer"
      27             : 
      28             : using namespace llvm;
      29             : using namespace LegalizeActions;
      30             : 
      31        1312 : LegalizerHelper::LegalizerHelper(MachineFunction &MF)
      32        1312 :     : MRI(MF.getRegInfo()), LI(*MF.getSubtarget().getLegalizerInfo()) {
      33        1312 :   MIRBuilder.setMF(MF);
      34        1312 : }
      35             : 
      36             : LegalizerHelper::LegalizeResult
      37        4682 : LegalizerHelper::legalizeInstrStep(MachineInstr &MI) {
      38             :   LLVM_DEBUG(dbgs() << "Legalizing: "; MI.print(dbgs()));
      39             : 
      40        4682 :   auto Step = LI.getAction(MI, MRI);
      41        4682 :   switch (Step.Action) {
      42             :   case Legal:
      43             :     LLVM_DEBUG(dbgs() << ".. Already legal\n");
      44             :     return AlreadyLegal;
      45         118 :   case Libcall:
      46             :     LLVM_DEBUG(dbgs() << ".. Convert to libcall\n");
      47         118 :     return libcall(MI);
      48          57 :   case NarrowScalar:
      49             :     LLVM_DEBUG(dbgs() << ".. Narrow scalar\n");
      50          57 :     return narrowScalar(MI, Step.TypeIdx, Step.NewType);
      51         269 :   case WidenScalar:
      52             :     LLVM_DEBUG(dbgs() << ".. Widen scalar\n");
      53         269 :     return widenScalar(MI, Step.TypeIdx, Step.NewType);
      54          39 :   case Lower:
      55             :     LLVM_DEBUG(dbgs() << ".. Lower\n");
      56          39 :     return lower(MI, Step.TypeIdx, Step.NewType);
      57          32 :   case FewerElements:
      58             :     LLVM_DEBUG(dbgs() << ".. Reduce number of elements\n");
      59          32 :     return fewerElementsVector(MI, Step.TypeIdx, Step.NewType);
      60          91 :   case Custom:
      61             :     LLVM_DEBUG(dbgs() << ".. Custom legalization\n");
      62          91 :     return LI.legalizeCustom(MI, MRI, MIRBuilder) ? Legalized
      63             :                                                   : UnableToLegalize;
      64           9 :   default:
      65             :     LLVM_DEBUG(dbgs() << ".. Unable to legalize\n");
      66           9 :     return UnableToLegalize;
      67             :   }
      68             : }
      69             : 
      70          94 : void LegalizerHelper::extractParts(unsigned Reg, LLT Ty, int NumParts,
      71             :                                    SmallVectorImpl<unsigned> &VRegs) {
      72         552 :   for (int i = 0; i < NumParts; ++i)
      73         458 :     VRegs.push_back(MRI.createGenericVirtualRegister(Ty));
      74         188 :   MIRBuilder.buildUnmerge(VRegs, Reg);
      75          94 : }
      76             : 
      77          98 : static RTLIB::Libcall getRTLibDesc(unsigned Opcode, unsigned Size) {
      78          98 :   switch (Opcode) {
      79             :   case TargetOpcode::G_SDIV:
      80             :     assert(Size == 32 && "Unsupported size");
      81             :     return RTLIB::SDIV_I32;
      82          12 :   case TargetOpcode::G_UDIV:
      83             :     assert(Size == 32 && "Unsupported size");
      84          12 :     return RTLIB::UDIV_I32;
      85           6 :   case TargetOpcode::G_SREM:
      86             :     assert(Size == 32 && "Unsupported size");
      87           6 :     return RTLIB::SREM_I32;
      88           6 :   case TargetOpcode::G_UREM:
      89             :     assert(Size == 32 && "Unsupported size");
      90           6 :     return RTLIB::UREM_I32;
      91           8 :   case TargetOpcode::G_FADD:
      92             :     assert((Size == 32 || Size == 64) && "Unsupported size");
      93           8 :     return Size == 64 ? RTLIB::ADD_F64 : RTLIB::ADD_F32;
      94          12 :   case TargetOpcode::G_FSUB:
      95             :     assert((Size == 32 || Size == 64) && "Unsupported size");
      96          12 :     return Size == 64 ? RTLIB::SUB_F64 : RTLIB::SUB_F32;
      97           4 :   case TargetOpcode::G_FMUL:
      98             :     assert((Size == 32 || Size == 64) && "Unsupported size");
      99           4 :     return Size == 64 ? RTLIB::MUL_F64 : RTLIB::MUL_F32;
     100           4 :   case TargetOpcode::G_FDIV:
     101             :     assert((Size == 32 || Size == 64) && "Unsupported size");
     102           4 :     return Size == 64 ? RTLIB::DIV_F64 : RTLIB::DIV_F32;
     103          14 :   case TargetOpcode::G_FREM:
     104          14 :     return Size == 64 ? RTLIB::REM_F64 : RTLIB::REM_F32;
     105          14 :   case TargetOpcode::G_FPOW:
     106          14 :     return Size == 64 ? RTLIB::POW_F64 : RTLIB::POW_F32;
     107           6 :   case TargetOpcode::G_FMA:
     108             :     assert((Size == 32 || Size == 64) && "Unsupported size");
     109           6 :     return Size == 64 ? RTLIB::FMA_F64 : RTLIB::FMA_F32;
     110             :   }
     111           0 :   llvm_unreachable("Unknown libcall function");
     112             : }
     113             : 
     114             : LegalizerHelper::LegalizeResult
     115         200 : llvm::createLibcall(MachineIRBuilder &MIRBuilder, RTLIB::Libcall Libcall,
     116             :                     const CallLowering::ArgInfo &Result,
     117             :                     ArrayRef<CallLowering::ArgInfo> Args) {
     118         200 :   auto &CLI = *MIRBuilder.getMF().getSubtarget().getCallLowering();
     119         200 :   auto &TLI = *MIRBuilder.getMF().getSubtarget().getTargetLowering();
     120             :   const char *Name = TLI.getLibcallName(Libcall);
     121             : 
     122         200 :   MIRBuilder.getMF().getFrameInfo().setHasCalls(true);
     123         400 :   if (!CLI.lowerCall(MIRBuilder, TLI.getLibcallCallingConv(Libcall),
     124         400 :                      MachineOperand::CreateES(Name), Result, Args))
     125             :     return LegalizerHelper::UnableToLegalize;
     126             : 
     127         200 :   return LegalizerHelper::Legalized;
     128             : }
     129             : 
     130             : // Useful for libcalls where all operands have the same type.
     131             : static LegalizerHelper::LegalizeResult
     132          98 : simpleLibcall(MachineInstr &MI, MachineIRBuilder &MIRBuilder, unsigned Size,
     133             :               Type *OpType) {
     134         196 :   auto Libcall = getRTLibDesc(MI.getOpcode(), Size);
     135             : 
     136             :   SmallVector<CallLowering::ArgInfo, 3> Args;
     137         502 :   for (unsigned i = 1; i < MI.getNumOperands(); i++)
     138         606 :     Args.push_back({MI.getOperand(i).getReg(), OpType});
     139         196 :   return createLibcall(MIRBuilder, Libcall, {MI.getOperand(0).getReg(), OpType},
     140         196 :                        Args);
     141             : }
     142             : 
     143          20 : static RTLIB::Libcall getConvRTLibDesc(unsigned Opcode, Type *ToType,
     144             :                                        Type *FromType) {
     145          20 :   auto ToMVT = MVT::getVT(ToType);
     146          20 :   auto FromMVT = MVT::getVT(FromType);
     147             : 
     148          20 :   switch (Opcode) {
     149           2 :   case TargetOpcode::G_FPEXT:
     150           2 :     return RTLIB::getFPEXT(FromMVT, ToMVT);
     151           2 :   case TargetOpcode::G_FPTRUNC:
     152           2 :     return RTLIB::getFPROUND(FromMVT, ToMVT);
     153           4 :   case TargetOpcode::G_FPTOSI:
     154           4 :     return RTLIB::getFPTOSINT(FromMVT, ToMVT);
     155           4 :   case TargetOpcode::G_FPTOUI:
     156           4 :     return RTLIB::getFPTOUINT(FromMVT, ToMVT);
     157           4 :   case TargetOpcode::G_SITOFP:
     158           4 :     return RTLIB::getSINTTOFP(FromMVT, ToMVT);
     159           4 :   case TargetOpcode::G_UITOFP:
     160           4 :     return RTLIB::getUINTTOFP(FromMVT, ToMVT);
     161             :   }
     162           0 :   llvm_unreachable("Unsupported libcall function");
     163             : }
     164             : 
     165             : static LegalizerHelper::LegalizeResult
     166          20 : conversionLibcall(MachineInstr &MI, MachineIRBuilder &MIRBuilder, Type *ToType,
     167             :                   Type *FromType) {
     168          40 :   RTLIB::Libcall Libcall = getConvRTLibDesc(MI.getOpcode(), ToType, FromType);
     169          60 :   return createLibcall(MIRBuilder, Libcall, {MI.getOperand(0).getReg(), ToType},
     170          60 :                        {{MI.getOperand(1).getReg(), FromType}});
     171             : }
     172             : 
     173             : LegalizerHelper::LegalizeResult
     174         118 : LegalizerHelper::libcall(MachineInstr &MI) {
     175         236 :   LLT LLTy = MRI.getType(MI.getOperand(0).getReg());
     176         118 :   unsigned Size = LLTy.getSizeInBits();
     177         118 :   auto &Ctx = MIRBuilder.getMF().getFunction().getContext();
     178             : 
     179         118 :   MIRBuilder.setInstr(MI);
     180             : 
     181         236 :   switch (MI.getOpcode()) {
     182             :   default:
     183             :     return UnableToLegalize;
     184          36 :   case TargetOpcode::G_SDIV:
     185             :   case TargetOpcode::G_UDIV:
     186             :   case TargetOpcode::G_SREM:
     187             :   case TargetOpcode::G_UREM: {
     188          36 :     Type *HLTy = Type::getInt32Ty(Ctx);
     189          36 :     auto Status = simpleLibcall(MI, MIRBuilder, Size, HLTy);
     190          36 :     if (Status != Legalized)
     191             :       return Status;
     192             :     break;
     193             :   }
     194          62 :   case TargetOpcode::G_FADD:
     195             :   case TargetOpcode::G_FSUB:
     196             :   case TargetOpcode::G_FMUL:
     197             :   case TargetOpcode::G_FDIV:
     198             :   case TargetOpcode::G_FMA:
     199             :   case TargetOpcode::G_FPOW:
     200             :   case TargetOpcode::G_FREM: {
     201          62 :     Type *HLTy = Size == 64 ? Type::getDoubleTy(Ctx) : Type::getFloatTy(Ctx);
     202          62 :     auto Status = simpleLibcall(MI, MIRBuilder, Size, HLTy);
     203          62 :     if (Status != Legalized)
     204             :       return Status;
     205             :     break;
     206             :   }
     207           2 :   case TargetOpcode::G_FPEXT: {
     208             :     // FIXME: Support other floating point types (half, fp128 etc)
     209           4 :     unsigned FromSize = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
     210           4 :     unsigned ToSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
     211           2 :     if (ToSize != 64 || FromSize != 32)
     212             :       return UnableToLegalize;
     213           2 :     LegalizeResult Status = conversionLibcall(
     214           2 :         MI, MIRBuilder, Type::getDoubleTy(Ctx), Type::getFloatTy(Ctx));
     215           2 :     if (Status != Legalized)
     216             :       return Status;
     217             :     break;
     218             :   }
     219           2 :   case TargetOpcode::G_FPTRUNC: {
     220             :     // FIXME: Support other floating point types (half, fp128 etc)
     221           4 :     unsigned FromSize = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
     222           4 :     unsigned ToSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
     223           2 :     if (ToSize != 32 || FromSize != 64)
     224             :       return UnableToLegalize;
     225           2 :     LegalizeResult Status = conversionLibcall(
     226           2 :         MI, MIRBuilder, Type::getFloatTy(Ctx), Type::getDoubleTy(Ctx));
     227           2 :     if (Status != Legalized)
     228             :       return Status;
     229             :     break;
     230             :   }
     231           8 :   case TargetOpcode::G_FPTOSI:
     232             :   case TargetOpcode::G_FPTOUI: {
     233             :     // FIXME: Support other types
     234          16 :     unsigned FromSize = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
     235          16 :     unsigned ToSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
     236           8 :     if (ToSize != 32 || (FromSize != 32 && FromSize != 64))
     237             :       return UnableToLegalize;
     238          16 :     LegalizeResult Status = conversionLibcall(
     239           8 :         MI, MIRBuilder, Type::getInt32Ty(Ctx),
     240           8 :         FromSize == 64 ? Type::getDoubleTy(Ctx) : Type::getFloatTy(Ctx));
     241           8 :     if (Status != Legalized)
     242             :       return Status;
     243             :     break;
     244             :   }
     245           8 :   case TargetOpcode::G_SITOFP:
     246             :   case TargetOpcode::G_UITOFP: {
     247             :     // FIXME: Support other types
     248          16 :     unsigned FromSize = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
     249          16 :     unsigned ToSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
     250           8 :     if (FromSize != 32 || (ToSize != 32 && ToSize != 64))
     251             :       return UnableToLegalize;
     252           8 :     LegalizeResult Status = conversionLibcall(
     253             :         MI, MIRBuilder,
     254             :         ToSize == 64 ? Type::getDoubleTy(Ctx) : Type::getFloatTy(Ctx),
     255          16 :         Type::getInt32Ty(Ctx));
     256           8 :     if (Status != Legalized)
     257             :       return Status;
     258             :     break;
     259             :   }
     260             :   }
     261             : 
     262         118 :   MI.eraseFromParent();
     263         118 :   return Legalized;
     264             : }
     265             : 
     266          57 : LegalizerHelper::LegalizeResult LegalizerHelper::narrowScalar(MachineInstr &MI,
     267             :                                                               unsigned TypeIdx,
     268             :                                                               LLT NarrowTy) {
     269             :   // FIXME: Don't know how to handle secondary types yet.
     270          63 :   if (TypeIdx != 0 && MI.getOpcode() != TargetOpcode::G_EXTRACT)
     271             :     return UnableToLegalize;
     272             : 
     273          57 :   MIRBuilder.setInstr(MI);
     274             : 
     275         114 :   uint64_t SizeOp0 = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
     276          57 :   uint64_t NarrowSize = NarrowTy.getSizeInBits();
     277             : 
     278         114 :   switch (MI.getOpcode()) {
     279             :   default:
     280             :     return UnableToLegalize;
     281           2 :   case TargetOpcode::G_IMPLICIT_DEF: {
     282             :     // FIXME: add support for when SizeOp0 isn't an exact multiple of
     283             :     // NarrowSize.
     284           2 :     if (SizeOp0 % NarrowSize != 0)
     285             :       return UnableToLegalize;
     286           2 :     int NumParts = SizeOp0 / NarrowSize;
     287             : 
     288             :     SmallVector<unsigned, 2> DstRegs;
     289          10 :     for (int i = 0; i < NumParts; ++i)
     290           4 :       DstRegs.push_back(
     291           8 :           MIRBuilder.buildUndef(NarrowTy)->getOperand(0).getReg());
     292           2 :     MIRBuilder.buildMerge(MI.getOperand(0).getReg(), DstRegs);
     293           2 :     MI.eraseFromParent();
     294             :     return Legalized;
     295             :   }
     296           4 :   case TargetOpcode::G_ADD: {
     297             :     // FIXME: add support for when SizeOp0 isn't an exact multiple of
     298             :     // NarrowSize.
     299           4 :     if (SizeOp0 % NarrowSize != 0)
     300             :       return UnableToLegalize;
     301             :     // Expand in terms of carry-setting/consuming G_ADDE instructions.
     302           4 :     int NumParts = SizeOp0 / NarrowTy.getSizeInBits();
     303             : 
     304             :     SmallVector<unsigned, 2> Src1Regs, Src2Regs, DstRegs;
     305           4 :     extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, Src1Regs);
     306           4 :     extractParts(MI.getOperand(2).getReg(), NarrowTy, NumParts, Src2Regs);
     307             : 
     308           8 :     unsigned CarryIn = MRI.createGenericVirtualRegister(LLT::scalar(1));
     309           4 :     MIRBuilder.buildConstant(CarryIn, 0);
     310             : 
     311          22 :     for (int i = 0; i < NumParts; ++i) {
     312          18 :       unsigned DstReg = MRI.createGenericVirtualRegister(NarrowTy);
     313          18 :       unsigned CarryOut = MRI.createGenericVirtualRegister(LLT::scalar(1));
     314             : 
     315           9 :       MIRBuilder.buildUAdde(DstReg, CarryOut, Src1Regs[i],
     316          27 :                             Src2Regs[i], CarryIn);
     317             : 
     318           9 :       DstRegs.push_back(DstReg);
     319             :       CarryIn = CarryOut;
     320             :     }
     321           4 :     unsigned DstReg = MI.getOperand(0).getReg();
     322           4 :     MIRBuilder.buildMerge(DstReg, DstRegs);
     323           4 :     MI.eraseFromParent();
     324             :     return Legalized;
     325             :   }
     326           6 :   case TargetOpcode::G_EXTRACT: {
     327           6 :     if (TypeIdx != 1)
     328             :       return UnableToLegalize;
     329             : 
     330          12 :     int64_t SizeOp1 = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
     331             :     // FIXME: add support for when SizeOp1 isn't an exact multiple of
     332             :     // NarrowSize.
     333           6 :     if (SizeOp1 % NarrowSize != 0)
     334             :       return UnableToLegalize;
     335           6 :     int NumParts = SizeOp1 / NarrowSize;
     336             : 
     337             :     SmallVector<unsigned, 2> SrcRegs, DstRegs;
     338             :     SmallVector<uint64_t, 2> Indexes;
     339           6 :     extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, SrcRegs);
     340             : 
     341           6 :     unsigned OpReg = MI.getOperand(0).getReg();
     342           6 :     uint64_t OpStart = MI.getOperand(2).getImm();
     343          12 :     uint64_t OpSize = MRI.getType(OpReg).getSizeInBits();
     344          30 :     for (int i = 0; i < NumParts; ++i) {
     345          12 :       unsigned SrcStart = i * NarrowSize;
     346             : 
     347          12 :       if (SrcStart + NarrowSize <= OpStart || SrcStart >= OpStart + OpSize) {
     348             :         // No part of the extract uses this subregister, ignore it.
     349          13 :         continue;
     350          10 :       } else if (SrcStart == OpStart && NarrowTy == MRI.getType(OpReg)) {
     351             :         // The entire subregister is extracted, forward the value.
     352           3 :         DstRegs.push_back(SrcRegs[i]);
     353           3 :         continue;
     354             :       }
     355             : 
     356             :       // OpSegStart is where this destination segment would start in OpReg if it
     357             :       // extended infinitely in both directions.
     358             :       int64_t ExtractOffset;
     359             :       uint64_t SegSize;
     360           4 :       if (OpStart < SrcStart) {
     361             :         ExtractOffset = 0;
     362           2 :         SegSize = std::min(NarrowSize, OpStart + OpSize - SrcStart);
     363             :       } else {
     364           3 :         ExtractOffset = OpStart - SrcStart;
     365           6 :         SegSize = std::min(SrcStart + NarrowSize - OpStart, OpSize);
     366             :       }
     367             : 
     368           4 :       unsigned SegReg = SrcRegs[i];
     369           4 :       if (ExtractOffset != 0 || SegSize != NarrowSize) {
     370             :         // A genuine extract is needed.
     371          12 :         SegReg = MRI.createGenericVirtualRegister(LLT::scalar(SegSize));
     372           8 :         MIRBuilder.buildExtract(SegReg, SrcRegs[i], ExtractOffset);
     373             :       }
     374             : 
     375           4 :       DstRegs.push_back(SegReg);
     376             :     }
     377             : 
     378           6 :     MIRBuilder.buildMerge(MI.getOperand(0).getReg(), DstRegs);
     379           6 :     MI.eraseFromParent();
     380             :     return Legalized;
     381             :   }
     382           7 :   case TargetOpcode::G_INSERT: {
     383             :     // FIXME: add support for when SizeOp0 isn't an exact multiple of
     384             :     // NarrowSize.
     385           7 :     if (SizeOp0 % NarrowSize != 0)
     386             :       return UnableToLegalize;
     387             : 
     388           7 :     int NumParts = SizeOp0 / NarrowSize;
     389             : 
     390             :     SmallVector<unsigned, 2> SrcRegs, DstRegs;
     391             :     SmallVector<uint64_t, 2> Indexes;
     392           7 :     extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, SrcRegs);
     393             : 
     394           7 :     unsigned OpReg = MI.getOperand(2).getReg();
     395           7 :     uint64_t OpStart = MI.getOperand(3).getImm();
     396          14 :     uint64_t OpSize = MRI.getType(OpReg).getSizeInBits();
     397          37 :     for (int i = 0; i < NumParts; ++i) {
     398          15 :       unsigned DstStart = i * NarrowSize;
     399             : 
     400          22 :       if (DstStart + NarrowSize <= OpStart || DstStart >= OpStart + OpSize) {
     401             :         // No part of the insert affects this subregister, forward the original.
     402           7 :         DstRegs.push_back(SrcRegs[i]);
     403          17 :         continue;
     404          11 :       } else if (DstStart == OpStart && NarrowTy == MRI.getType(OpReg)) {
     405             :         // The entire subregister is defined by this insert, forward the new
     406             :         // value.
     407           3 :         DstRegs.push_back(OpReg);
     408           3 :         continue;
     409             :       }
     410             : 
     411             :       // OpSegStart is where this destination segment would start in OpReg if it
     412             :       // extended infinitely in both directions.
     413             :       int64_t ExtractOffset, InsertOffset;
     414             :       uint64_t SegSize;
     415           5 :       if (OpStart < DstStart) {
     416             :         InsertOffset = 0;
     417           1 :         ExtractOffset = DstStart - OpStart;
     418           2 :         SegSize = std::min(NarrowSize, OpStart + OpSize - DstStart);
     419             :       } else {
     420           4 :         InsertOffset = OpStart - DstStart;
     421             :         ExtractOffset = 0;
     422           4 :         SegSize =
     423           8 :             std::min(NarrowSize - InsertOffset, OpStart + OpSize - DstStart);
     424             :       }
     425             : 
     426           5 :       unsigned SegReg = OpReg;
     427           5 :       if (ExtractOffset != 0 || SegSize != OpSize) {
     428             :         // A genuine extract is needed.
     429           6 :         SegReg = MRI.createGenericVirtualRegister(LLT::scalar(SegSize));
     430           2 :         MIRBuilder.buildExtract(SegReg, OpReg, ExtractOffset);
     431             :       }
     432             : 
     433          10 :       unsigned DstReg = MRI.createGenericVirtualRegister(NarrowTy);
     434          10 :       MIRBuilder.buildInsert(DstReg, SrcRegs[i], SegReg, InsertOffset);
     435           5 :       DstRegs.push_back(DstReg);
     436             :     }
     437             : 
     438             :     assert(DstRegs.size() == (unsigned)NumParts && "not all parts covered");
     439           7 :     MIRBuilder.buildMerge(MI.getOperand(0).getReg(), DstRegs);
     440           7 :     MI.eraseFromParent();
     441             :     return Legalized;
     442             :   }
     443           8 :   case TargetOpcode::G_LOAD: {
     444             :     // FIXME: add support for when SizeOp0 isn't an exact multiple of
     445             :     // NarrowSize.
     446           8 :     if (SizeOp0 % NarrowSize != 0)
     447             :       return UnableToLegalize;
     448             : 
     449           8 :     const auto &MMO = **MI.memoperands_begin();
     450             :     // This implementation doesn't work for atomics. Give up instead of doing
     451             :     // something invalid.
     452          16 :     if (MMO.getOrdering() != AtomicOrdering::NotAtomic ||
     453             :         MMO.getFailureOrdering() != AtomicOrdering::NotAtomic)
     454             :       return UnableToLegalize;
     455             : 
     456           8 :     int NumParts = SizeOp0 / NarrowSize;
     457             :     LLT OffsetTy = LLT::scalar(
     458          16 :         MRI.getType(MI.getOperand(1).getReg()).getScalarSizeInBits());
     459             : 
     460             :     SmallVector<unsigned, 2> DstRegs;
     461          40 :     for (int i = 0; i < NumParts; ++i) {
     462          32 :       unsigned DstReg = MRI.createGenericVirtualRegister(NarrowTy);
     463          16 :       unsigned SrcReg = 0;
     464          16 :       unsigned Adjustment = i * NarrowSize / 8;
     465             : 
     466          96 :       MachineMemOperand *SplitMMO = MIRBuilder.getMF().getMachineMemOperand(
     467          16 :           MMO.getPointerInfo().getWithOffset(Adjustment), MMO.getFlags(),
     468          16 :           NarrowSize / 8, i == 0 ? MMO.getAlignment() : NarrowSize / 8,
     469          16 :           MMO.getAAInfo(), MMO.getRanges(), MMO.getSyncScopeID(),
     470          16 :           MMO.getOrdering(), MMO.getFailureOrdering());
     471             : 
     472          16 :       MIRBuilder.materializeGEP(SrcReg, MI.getOperand(1).getReg(), OffsetTy,
     473             :                                 Adjustment);
     474             : 
     475          16 :       MIRBuilder.buildLoad(DstReg, SrcReg, *SplitMMO);
     476             : 
     477          16 :       DstRegs.push_back(DstReg);
     478             :     }
     479           8 :     unsigned DstReg = MI.getOperand(0).getReg();
     480           8 :     MIRBuilder.buildMerge(DstReg, DstRegs);
     481           8 :     MI.eraseFromParent();
     482             :     return Legalized;
     483             :   }
     484          20 :   case TargetOpcode::G_STORE: {
     485             :     // FIXME: add support for when SizeOp0 isn't an exact multiple of
     486             :     // NarrowSize.
     487          20 :     if (SizeOp0 % NarrowSize != 0)
     488             :       return UnableToLegalize;
     489             : 
     490           9 :     const auto &MMO = **MI.memoperands_begin();
     491             :     // This implementation doesn't work for atomics. Give up instead of doing
     492             :     // something invalid.
     493          18 :     if (MMO.getOrdering() != AtomicOrdering::NotAtomic ||
     494             :         MMO.getFailureOrdering() != AtomicOrdering::NotAtomic)
     495             :       return UnableToLegalize;
     496             : 
     497           9 :     int NumParts = SizeOp0 / NarrowSize;
     498             :     LLT OffsetTy = LLT::scalar(
     499          18 :         MRI.getType(MI.getOperand(1).getReg()).getScalarSizeInBits());
     500             : 
     501             :     SmallVector<unsigned, 2> SrcRegs;
     502           9 :     extractParts(MI.getOperand(0).getReg(), NarrowTy, NumParts, SrcRegs);
     503             : 
     504          45 :     for (int i = 0; i < NumParts; ++i) {
     505          18 :       unsigned DstReg = 0;
     506          18 :       unsigned Adjustment = i * NarrowSize / 8;
     507             : 
     508         108 :       MachineMemOperand *SplitMMO = MIRBuilder.getMF().getMachineMemOperand(
     509          18 :           MMO.getPointerInfo().getWithOffset(Adjustment), MMO.getFlags(),
     510          18 :           NarrowSize / 8, i == 0 ? MMO.getAlignment() : NarrowSize / 8,
     511          18 :           MMO.getAAInfo(), MMO.getRanges(), MMO.getSyncScopeID(),
     512          18 :           MMO.getOrdering(), MMO.getFailureOrdering());
     513             : 
     514          18 :       MIRBuilder.materializeGEP(DstReg, MI.getOperand(1).getReg(), OffsetTy,
     515             :                                 Adjustment);
     516             : 
     517          36 :       MIRBuilder.buildStore(SrcRegs[i], DstReg, *SplitMMO);
     518             :     }
     519           9 :     MI.eraseFromParent();
     520             :     return Legalized;
     521             :   }
     522           7 :   case TargetOpcode::G_CONSTANT: {
     523             :     // FIXME: add support for when SizeOp0 isn't an exact multiple of
     524             :     // NarrowSize.
     525           7 :     if (SizeOp0 % NarrowSize != 0)
     526             :       return UnableToLegalize;
     527           7 :     int NumParts = SizeOp0 / NarrowSize;
     528           7 :     const APInt &Cst = MI.getOperand(1).getCImm()->getValue();
     529           7 :     LLVMContext &Ctx = MIRBuilder.getMF().getFunction().getContext();
     530             : 
     531             :     SmallVector<unsigned, 2> DstRegs;
     532          35 :     for (int i = 0; i < NumParts; ++i) {
     533          28 :       unsigned DstReg = MRI.createGenericVirtualRegister(NarrowTy);
     534             :       ConstantInt *CI =
     535          42 :           ConstantInt::get(Ctx, Cst.lshr(NarrowSize * i).trunc(NarrowSize));
     536          14 :       MIRBuilder.buildConstant(DstReg, *CI);
     537          14 :       DstRegs.push_back(DstReg);
     538             :     }
     539           7 :     unsigned DstReg = MI.getOperand(0).getReg();
     540           7 :     MIRBuilder.buildMerge(DstReg, DstRegs);
     541           7 :     MI.eraseFromParent();
     542             :     return Legalized;
     543             :   }
     544           1 :   case TargetOpcode::G_OR: {
     545             :     // Legalize bitwise operation:
     546             :     // A = BinOp<Ty> B, C
     547             :     // into:
     548             :     // B1, ..., BN = G_UNMERGE_VALUES B
     549             :     // C1, ..., CN = G_UNMERGE_VALUES C
     550             :     // A1 = BinOp<Ty/N> B1, C2
     551             :     // ...
     552             :     // AN = BinOp<Ty/N> BN, CN
     553             :     // A = G_MERGE_VALUES A1, ..., AN
     554             : 
     555             :     // FIXME: add support for when SizeOp0 isn't an exact multiple of
     556             :     // NarrowSize.
     557           1 :     if (SizeOp0 % NarrowSize != 0)
     558             :       return UnableToLegalize;
     559           1 :     int NumParts = SizeOp0 / NarrowSize;
     560             : 
     561             :     // List the registers where the destination will be scattered.
     562             :     SmallVector<unsigned, 2> DstRegs;
     563             :     // List the registers where the first argument will be split.
     564             :     SmallVector<unsigned, 2> SrcsReg1;
     565             :     // List the registers where the second argument will be split.
     566             :     SmallVector<unsigned, 2> SrcsReg2;
     567             :     // Create all the temporary registers.
     568           5 :     for (int i = 0; i < NumParts; ++i) {
     569           4 :       unsigned DstReg = MRI.createGenericVirtualRegister(NarrowTy);
     570           4 :       unsigned SrcReg1 = MRI.createGenericVirtualRegister(NarrowTy);
     571           4 :       unsigned SrcReg2 = MRI.createGenericVirtualRegister(NarrowTy);
     572             : 
     573           2 :       DstRegs.push_back(DstReg);
     574           2 :       SrcsReg1.push_back(SrcReg1);
     575           2 :       SrcsReg2.push_back(SrcReg2);
     576             :     }
     577             :     // Explode the big arguments into smaller chunks.
     578           2 :     MIRBuilder.buildUnmerge(SrcsReg1, MI.getOperand(1).getReg());
     579           2 :     MIRBuilder.buildUnmerge(SrcsReg2, MI.getOperand(2).getReg());
     580             : 
     581             :     // Do the operation on each small part.
     582           5 :     for (int i = 0; i < NumParts; ++i)
     583           8 :       MIRBuilder.buildOr(DstRegs[i], SrcsReg1[i], SrcsReg2[i]);
     584             : 
     585             :     // Gather the destination registers into the final destination.
     586           1 :     unsigned DstReg = MI.getOperand(0).getReg();
     587           1 :     MIRBuilder.buildMerge(DstReg, DstRegs);
     588           1 :     MI.eraseFromParent();
     589             :     return Legalized;
     590             :   }
     591             :   }
     592             : }
     593             : 
     594         341 : void LegalizerHelper::widenScalarSrc(MachineInstr &MI, LLT WideTy,
     595             :                                      unsigned OpIdx, unsigned ExtOpcode) {
     596         341 :   MachineOperand &MO = MI.getOperand(OpIdx);
     597         341 :   auto ExtB = MIRBuilder.buildInstr(ExtOpcode, WideTy, MO.getReg());
     598         341 :   MO.setReg(ExtB->getOperand(0).getReg());
     599         341 : }
     600             : 
     601         234 : void LegalizerHelper::widenScalarDst(MachineInstr &MI, LLT WideTy,
     602             :                                      unsigned OpIdx, unsigned TruncOpcode) {
     603         234 :   MachineOperand &MO = MI.getOperand(OpIdx);
     604         468 :   unsigned DstExt = MRI.createGenericVirtualRegister(WideTy);
     605         468 :   MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt());
     606         234 :   MIRBuilder.buildInstr(TruncOpcode, MO.getReg(), DstExt);
     607         234 :   MO.setReg(DstExt);
     608         234 : }
     609             : 
     610             : LegalizerHelper::LegalizeResult
     611         269 : LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
     612         269 :   MIRBuilder.setInstr(MI);
     613             : 
     614         538 :   switch (MI.getOpcode()) {
     615             :   default:
     616             :     return UnableToLegalize;
     617             : 
     618          64 :   case TargetOpcode::G_ADD:
     619             :   case TargetOpcode::G_AND:
     620             :   case TargetOpcode::G_MUL:
     621             :   case TargetOpcode::G_OR:
     622             :   case TargetOpcode::G_XOR:
     623             :   case TargetOpcode::G_SUB:
     624             :     // Perform operation at larger width (any extension is fine here, high bits
     625             :     // don't affect the result) and then truncate the result back to the
     626             :     // original type.
     627          64 :     widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
     628          64 :     widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ANYEXT);
     629          64 :     widenScalarDst(MI, WideTy);
     630          64 :     MIRBuilder.recordInsertion(&MI);
     631          64 :     return Legalized;
     632             : 
     633           4 :   case TargetOpcode::G_SHL:
     634           4 :     widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
     635             :     // The "number of bits to shift" operand must preserve its value as an
     636             :     // unsigned integer:
     637           4 :     widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ZEXT);
     638           4 :     widenScalarDst(MI, WideTy);
     639           4 :     MIRBuilder.recordInsertion(&MI);
     640           4 :     return Legalized;
     641             : 
     642          34 :   case TargetOpcode::G_SDIV:
     643             :   case TargetOpcode::G_SREM:
     644          34 :     widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_SEXT);
     645          34 :     widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_SEXT);
     646          34 :     widenScalarDst(MI, WideTy);
     647          34 :     MIRBuilder.recordInsertion(&MI);
     648          34 :     return Legalized;
     649             : 
     650           3 :   case TargetOpcode::G_ASHR:
     651           3 :     widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_SEXT);
     652             :     // The "number of bits to shift" operand must preserve its value as an
     653             :     // unsigned integer:
     654           3 :     widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ZEXT);
     655           3 :     widenScalarDst(MI, WideTy);
     656           3 :     MIRBuilder.recordInsertion(&MI);
     657           3 :     return Legalized;
     658             : 
     659          36 :   case TargetOpcode::G_UDIV:
     660             :   case TargetOpcode::G_UREM:
     661             :   case TargetOpcode::G_LSHR:
     662          36 :     widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ZEXT);
     663          36 :     widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ZEXT);
     664          36 :     widenScalarDst(MI, WideTy);
     665          36 :     MIRBuilder.recordInsertion(&MI);
     666          36 :     return Legalized;
     667             : 
     668           3 :   case TargetOpcode::G_SELECT:
     669           3 :     if (TypeIdx != 0)
     670             :       return UnableToLegalize;
     671             :     // Perform operation at larger width (any extension is fine here, high bits
     672             :     // don't affect the result) and then truncate the result back to the
     673             :     // original type.
     674           3 :     widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ANYEXT);
     675           3 :     widenScalarSrc(MI, WideTy, 3, TargetOpcode::G_ANYEXT);
     676           3 :     widenScalarDst(MI, WideTy);
     677           3 :     MIRBuilder.recordInsertion(&MI);
     678           3 :     return Legalized;
     679             : 
     680           8 :   case TargetOpcode::G_FPTOSI:
     681             :   case TargetOpcode::G_FPTOUI:
     682           8 :     if (TypeIdx != 0)
     683             :       return UnableToLegalize;
     684           7 :     widenScalarDst(MI, WideTy);
     685           7 :     MIRBuilder.recordInsertion(&MI);
     686           7 :     return Legalized;
     687             : 
     688           3 :   case TargetOpcode::G_SITOFP:
     689           3 :     if (TypeIdx != 1)
     690             :       return UnableToLegalize;
     691           3 :     widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_SEXT);
     692           3 :     MIRBuilder.recordInsertion(&MI);
     693           3 :     return Legalized;
     694             : 
     695           3 :   case TargetOpcode::G_UITOFP:
     696           3 :     if (TypeIdx != 1)
     697             :       return UnableToLegalize;
     698           3 :     widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ZEXT);
     699           3 :     MIRBuilder.recordInsertion(&MI);
     700           3 :     return Legalized;
     701             : 
     702           1 :   case TargetOpcode::G_INSERT:
     703           1 :     if (TypeIdx != 0)
     704             :       return UnableToLegalize;
     705           1 :     widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
     706           1 :     widenScalarDst(MI, WideTy);
     707           1 :     MIRBuilder.recordInsertion(&MI);
     708           1 :     return Legalized;
     709             : 
     710          11 :   case TargetOpcode::G_LOAD:
     711             :     // For some types like i24, we might try to widen to i32. To properly handle
     712             :     // this we should be using a dedicated extending load, until then avoid
     713             :     // trying to legalize.
     714          33 :     if (alignTo(MRI.getType(MI.getOperand(0).getReg()).getSizeInBits(), 8) !=
     715          11 :         WideTy.getSizeInBits())
     716             :       return UnableToLegalize;
     717             :     LLVM_FALLTHROUGH;
     718             :   case TargetOpcode::G_SEXTLOAD:
     719             :   case TargetOpcode::G_ZEXTLOAD:
     720           9 :     widenScalarDst(MI, WideTy);
     721           9 :     MIRBuilder.recordInsertion(&MI);
     722           9 :     return Legalized;
     723             : 
     724             :   case TargetOpcode::G_STORE: {
     725          17 :     if (MRI.getType(MI.getOperand(0).getReg()) != LLT::scalar(1) ||
     726             :         WideTy != LLT::scalar(8))
     727             :       return UnableToLegalize;
     728             : 
     729          15 :     widenScalarSrc(MI, WideTy, 0, TargetOpcode::G_ZEXT);
     730          15 :     MIRBuilder.recordInsertion(&MI);
     731          15 :     return Legalized;
     732             :   }
     733          44 :   case TargetOpcode::G_CONSTANT: {
     734          44 :     MachineOperand &SrcMO = MI.getOperand(1);
     735          44 :     LLVMContext &Ctx = MIRBuilder.getMF().getFunction().getContext();
     736          88 :     const APInt &Val = SrcMO.getCImm()->getValue().sext(WideTy.getSizeInBits());
     737          44 :     SrcMO.setCImm(ConstantInt::get(Ctx, Val));
     738             : 
     739          44 :     widenScalarDst(MI, WideTy);
     740          44 :     MIRBuilder.recordInsertion(&MI);
     741             :     return Legalized;
     742             :   }
     743           1 :   case TargetOpcode::G_FCONSTANT: {
     744           1 :     MachineOperand &SrcMO = MI.getOperand(1);
     745           1 :     LLVMContext &Ctx = MIRBuilder.getMF().getFunction().getContext();
     746           1 :     APFloat Val = SrcMO.getFPImm()->getValueAPF();
     747             :     bool LosesInfo;
     748           1 :     switch (WideTy.getSizeInBits()) {
     749           1 :     case 32:
     750           1 :       Val.convert(APFloat::IEEEsingle(), APFloat::rmTowardZero, &LosesInfo);
     751           1 :       break;
     752           0 :     case 64:
     753           0 :       Val.convert(APFloat::IEEEdouble(), APFloat::rmTowardZero, &LosesInfo);
     754           0 :       break;
     755           0 :     default:
     756           0 :       llvm_unreachable("Unhandled fp widen type");
     757             :     }
     758           1 :     SrcMO.setFPImm(ConstantFP::get(Ctx, Val));
     759             : 
     760           1 :     widenScalarDst(MI, WideTy, 0, TargetOpcode::G_FPTRUNC);
     761           1 :     MIRBuilder.recordInsertion(&MI);
     762             :     return Legalized;
     763             :   }
     764           0 :   case TargetOpcode::G_BRCOND:
     765           0 :     widenScalarSrc(MI, WideTy, 0, TargetOpcode::G_ANYEXT);
     766           0 :     MIRBuilder.recordInsertion(&MI);
     767           0 :     return Legalized;
     768             : 
     769           0 :   case TargetOpcode::G_FCMP:
     770           0 :     if (TypeIdx == 0)
     771           0 :       widenScalarDst(MI, WideTy);
     772             :     else {
     773           0 :       widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_FPEXT);
     774           0 :       widenScalarSrc(MI, WideTy, 3, TargetOpcode::G_FPEXT);
     775             :     }
     776           0 :     MIRBuilder.recordInsertion(&MI);
     777           0 :     return Legalized;
     778             : 
     779          22 :   case TargetOpcode::G_ICMP:
     780          22 :     if (TypeIdx == 0)
     781          18 :       widenScalarDst(MI, WideTy);
     782             :     else {
     783           4 :       unsigned ExtOpcode = CmpInst::isSigned(static_cast<CmpInst::Predicate>(
     784           4 :                                MI.getOperand(1).getPredicate()))
     785           4 :                                ? TargetOpcode::G_SEXT
     786             :                                : TargetOpcode::G_ZEXT;
     787           4 :       widenScalarSrc(MI, WideTy, 2, ExtOpcode);
     788           4 :       widenScalarSrc(MI, WideTy, 3, ExtOpcode);
     789             :     }
     790          22 :     MIRBuilder.recordInsertion(&MI);
     791          22 :     return Legalized;
     792             : 
     793           3 :   case TargetOpcode::G_GEP:
     794             :     assert(TypeIdx == 1 && "unable to legalize pointer of GEP");
     795           3 :     widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_SEXT);
     796           3 :     MIRBuilder.recordInsertion(&MI);
     797           3 :     return Legalized;
     798             : 
     799             :   case TargetOpcode::G_PHI: {
     800             :     assert(TypeIdx == 0 && "Expecting only Idx 0");
     801             : 
     802          50 :     for (unsigned I = 1; I < MI.getNumOperands(); I += 2) {
     803          40 :       MachineBasicBlock &OpMBB = *MI.getOperand(I + 1).getMBB();
     804          20 :       MIRBuilder.setInsertPt(OpMBB, OpMBB.getFirstTerminator());
     805          20 :       widenScalarSrc(MI, WideTy, I, TargetOpcode::G_ANYEXT);
     806             :     }
     807             : 
     808          10 :     MachineBasicBlock &MBB = *MI.getParent();
     809          20 :     MIRBuilder.setInsertPt(MBB, --MBB.getFirstNonPHI());
     810          10 :     widenScalarDst(MI, WideTy);
     811          10 :     MIRBuilder.recordInsertion(&MI);
     812          10 :     return Legalized;
     813             :   }
     814             :   }
     815             : }
     816             : 
     817             : LegalizerHelper::LegalizeResult
     818          39 : LegalizerHelper::lower(MachineInstr &MI, unsigned TypeIdx, LLT Ty) {
     819             :   using namespace TargetOpcode;
     820          39 :   MIRBuilder.setInstr(MI);
     821             : 
     822          78 :   switch(MI.getOpcode()) {
     823             :   default:
     824             :     return UnableToLegalize;
     825          27 :   case TargetOpcode::G_SREM:
     826             :   case TargetOpcode::G_UREM: {
     827          54 :     unsigned QuotReg = MRI.createGenericVirtualRegister(Ty);
     828          81 :     MIRBuilder.buildInstr(MI.getOpcode() == G_SREM ? G_SDIV : G_UDIV)
     829             :         .addDef(QuotReg)
     830          27 :         .addUse(MI.getOperand(1).getReg())
     831          27 :         .addUse(MI.getOperand(2).getReg());
     832             : 
     833          54 :     unsigned ProdReg = MRI.createGenericVirtualRegister(Ty);
     834          27 :     MIRBuilder.buildMul(ProdReg, QuotReg, MI.getOperand(2).getReg());
     835          27 :     MIRBuilder.buildSub(MI.getOperand(0).getReg(), MI.getOperand(1).getReg(),
     836          27 :                         ProdReg);
     837          27 :     MI.eraseFromParent();
     838          27 :     return Legalized;
     839             :   }
     840           2 :   case TargetOpcode::G_SMULO:
     841             :   case TargetOpcode::G_UMULO: {
     842             :     // Generate G_UMULH/G_SMULH to check for overflow and a normal G_MUL for the
     843             :     // result.
     844           2 :     unsigned Res = MI.getOperand(0).getReg();
     845           2 :     unsigned Overflow = MI.getOperand(1).getReg();
     846           2 :     unsigned LHS = MI.getOperand(2).getReg();
     847           2 :     unsigned RHS = MI.getOperand(3).getReg();
     848             : 
     849           2 :     MIRBuilder.buildMul(Res, LHS, RHS);
     850             : 
     851           2 :     unsigned Opcode = MI.getOpcode() == TargetOpcode::G_SMULO
     852           2 :                           ? TargetOpcode::G_SMULH
     853             :                           : TargetOpcode::G_UMULH;
     854             : 
     855           4 :     unsigned HiPart = MRI.createGenericVirtualRegister(Ty);
     856           4 :     MIRBuilder.buildInstr(Opcode)
     857             :       .addDef(HiPart)
     858             :       .addUse(LHS)
     859             :       .addUse(RHS);
     860             : 
     861           4 :     unsigned Zero = MRI.createGenericVirtualRegister(Ty);
     862           2 :     MIRBuilder.buildConstant(Zero, 0);
     863             : 
     864             :     // For *signed* multiply, overflow is detected by checking:
     865             :     // (hi != (lo >> bitwidth-1))
     866           2 :     if (Opcode == TargetOpcode::G_SMULH) {
     867           2 :       unsigned Shifted = MRI.createGenericVirtualRegister(Ty);
     868           2 :       unsigned ShiftAmt = MRI.createGenericVirtualRegister(Ty);
     869           1 :       MIRBuilder.buildConstant(ShiftAmt, Ty.getSizeInBits() - 1);
     870           2 :       MIRBuilder.buildInstr(TargetOpcode::G_ASHR)
     871             :         .addDef(Shifted)
     872             :         .addUse(Res)
     873             :         .addUse(ShiftAmt);
     874           1 :       MIRBuilder.buildICmp(CmpInst::ICMP_NE, Overflow, HiPart, Shifted);
     875             :     } else {
     876           1 :       MIRBuilder.buildICmp(CmpInst::ICMP_NE, Overflow, HiPart, Zero);
     877             :     }
     878           2 :     MI.eraseFromParent();
     879           2 :     return Legalized;
     880             :   }
     881             :   case TargetOpcode::G_FNEG: {
     882             :     // TODO: Handle vector types once we are able to
     883             :     // represent them.
     884             :     if (Ty.isVector())
     885             :       return UnableToLegalize;
     886           7 :     unsigned Res = MI.getOperand(0).getReg();
     887             :     Type *ZeroTy;
     888           7 :     LLVMContext &Ctx = MIRBuilder.getMF().getFunction().getContext();
     889           7 :     switch (Ty.getSizeInBits()) {
     890           0 :     case 16:
     891           0 :       ZeroTy = Type::getHalfTy(Ctx);
     892           0 :       break;
     893           3 :     case 32:
     894           3 :       ZeroTy = Type::getFloatTy(Ctx);
     895           3 :       break;
     896           3 :     case 64:
     897           3 :       ZeroTy = Type::getDoubleTy(Ctx);
     898           3 :       break;
     899           1 :     case 128:
     900           1 :       ZeroTy = Type::getFP128Ty(Ctx);
     901           1 :       break;
     902           0 :     default:
     903           0 :       llvm_unreachable("unexpected floating-point type");
     904             :     }
     905             :     ConstantFP &ZeroForNegation =
     906           7 :         *cast<ConstantFP>(ConstantFP::getZeroValueForNegation(ZeroTy));
     907           7 :     auto Zero = MIRBuilder.buildFConstant(Ty, ZeroForNegation);
     908          14 :     MIRBuilder.buildInstr(TargetOpcode::G_FSUB)
     909             :         .addDef(Res)
     910           7 :         .addUse(Zero->getOperand(0).getReg())
     911           7 :         .addUse(MI.getOperand(1).getReg());
     912           7 :     MI.eraseFromParent();
     913           7 :     return Legalized;
     914             :   }
     915           0 :   case TargetOpcode::G_FSUB: {
     916             :     // Lower (G_FSUB LHS, RHS) to (G_FADD LHS, (G_FNEG RHS)).
     917             :     // First, check if G_FNEG is marked as Lower. If so, we may
     918             :     // end up with an infinite loop as G_FSUB is used to legalize G_FNEG.
     919           0 :     if (LI.getAction({G_FNEG, {Ty}}).Action == Lower)
     920             :       return UnableToLegalize;
     921           0 :     unsigned Res = MI.getOperand(0).getReg();
     922           0 :     unsigned LHS = MI.getOperand(1).getReg();
     923           0 :     unsigned RHS = MI.getOperand(2).getReg();
     924           0 :     unsigned Neg = MRI.createGenericVirtualRegister(Ty);
     925           0 :     MIRBuilder.buildInstr(TargetOpcode::G_FNEG).addDef(Neg).addUse(RHS);
     926           0 :     MIRBuilder.buildInstr(TargetOpcode::G_FADD)
     927             :         .addDef(Res)
     928             :         .addUse(LHS)
     929             :         .addUse(Neg);
     930           0 :     MI.eraseFromParent();
     931           0 :     return Legalized;
     932             :   }
     933           2 :   case TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS: {
     934           2 :     unsigned OldValRes = MI.getOperand(0).getReg();
     935           2 :     unsigned SuccessRes = MI.getOperand(1).getReg();
     936           2 :     unsigned Addr = MI.getOperand(2).getReg();
     937           2 :     unsigned CmpVal = MI.getOperand(3).getReg();
     938           2 :     unsigned NewVal = MI.getOperand(4).getReg();
     939           2 :     MIRBuilder.buildAtomicCmpXchg(OldValRes, Addr, CmpVal, NewVal,
     940           2 :                                   **MI.memoperands_begin());
     941           2 :     MIRBuilder.buildICmp(CmpInst::ICMP_EQ, SuccessRes, OldValRes, CmpVal);
     942           2 :     MI.eraseFromParent();
     943           2 :     return Legalized;
     944             :   }
     945           1 :   case TargetOpcode::G_LOAD:
     946             :   case TargetOpcode::G_SEXTLOAD:
     947             :   case TargetOpcode::G_ZEXTLOAD: {
     948             :     // Lower to a memory-width G_LOAD and a G_SEXT/G_ZEXT/G_ANYEXT
     949           1 :     unsigned DstReg = MI.getOperand(0).getReg();
     950           1 :     unsigned PtrReg = MI.getOperand(1).getReg();
     951           2 :     LLT DstTy = MRI.getType(DstReg);
     952           1 :     auto &MMO = **MI.memoperands_begin();
     953             : 
     954           1 :     if (DstTy.getSizeInBits() == MMO.getSize() /* in bytes */ * 8) {
     955             :       // In the case of G_LOAD, this was a non-extending load already and we're
     956             :       // about to lower to the same instruction.
     957           1 :       if (MI.getOpcode() == TargetOpcode::G_LOAD)
     958             :           return UnableToLegalize;
     959           0 :       MIRBuilder.buildLoad(DstReg, PtrReg, MMO);
     960           0 :       MI.eraseFromParent();
     961           0 :       return Legalized;
     962             :     }
     963             : 
     964             :     if (DstTy.isScalar()) {
     965           0 :       unsigned TmpReg = MRI.createGenericVirtualRegister(
     966           0 :           LLT::scalar(MMO.getSize() /* in bytes */ * 8));
     967           0 :       MIRBuilder.buildLoad(TmpReg, PtrReg, MMO);
     968           0 :       switch (MI.getOpcode()) {
     969           0 :       default:
     970           0 :         llvm_unreachable("Unexpected opcode");
     971           0 :       case TargetOpcode::G_LOAD:
     972           0 :         MIRBuilder.buildAnyExt(DstReg, TmpReg);
     973           0 :         break;
     974           0 :       case TargetOpcode::G_SEXTLOAD:
     975           0 :         MIRBuilder.buildSExt(DstReg, TmpReg);
     976           0 :         break;
     977           0 :       case TargetOpcode::G_ZEXTLOAD:
     978           0 :         MIRBuilder.buildZExt(DstReg, TmpReg);
     979           0 :         break;
     980             :       }
     981           0 :       MI.eraseFromParent();
     982           0 :       return Legalized;
     983             :     }
     984             : 
     985             :     return UnableToLegalize;
     986             :   }
     987             :   }
     988             : }
     989             : 
     990             : LegalizerHelper::LegalizeResult
     991          32 : LegalizerHelper::fewerElementsVector(MachineInstr &MI, unsigned TypeIdx,
     992             :                                      LLT NarrowTy) {
     993             :   // FIXME: Don't know how to handle secondary types yet.
     994          32 :   if (TypeIdx != 0)
     995             :     return UnableToLegalize;
     996          64 :   switch (MI.getOpcode()) {
     997             :   default:
     998             :     return UnableToLegalize;
     999          32 :   case TargetOpcode::G_ADD: {
    1000          32 :     unsigned NarrowSize = NarrowTy.getSizeInBits();
    1001          32 :     unsigned DstReg = MI.getOperand(0).getReg();
    1002          64 :     unsigned Size = MRI.getType(DstReg).getSizeInBits();
    1003          32 :     int NumParts = Size / NarrowSize;
    1004             :     // FIXME: Don't know how to handle the situation where the small vectors
    1005             :     // aren't all the same size yet.
    1006          32 :     if (Size % NarrowSize != 0)
    1007             :       return UnableToLegalize;
    1008             : 
    1009          32 :     MIRBuilder.setInstr(MI);
    1010             : 
    1011             :     SmallVector<unsigned, 2> Src1Regs, Src2Regs, DstRegs;
    1012          32 :     extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, Src1Regs);
    1013          32 :     extractParts(MI.getOperand(2).getReg(), NarrowTy, NumParts, Src2Regs);
    1014             : 
    1015         198 :     for (int i = 0; i < NumParts; ++i) {
    1016         166 :       unsigned DstReg = MRI.createGenericVirtualRegister(NarrowTy);
    1017         249 :       MIRBuilder.buildAdd(DstReg, Src1Regs[i], Src2Regs[i]);
    1018          83 :       DstRegs.push_back(DstReg);
    1019             :     }
    1020             : 
    1021          32 :     MIRBuilder.buildMerge(DstReg, DstRegs);
    1022          32 :     MI.eraseFromParent();
    1023             :     return Legalized;
    1024             :   }
    1025             :   }
    1026             : }

Generated by: LCOV version 1.13