LCOV - code coverage report
Current view: top level - lib/CodeGen/GlobalISel - LegalizerHelper.cpp (source / functions) Hit Total Coverage
Test: llvm-toolchain.info Lines: 599 646 92.7 %
Date: 2018-10-20 13:21:21 Functions: 17 17 100.0 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : //===-- llvm/CodeGen/GlobalISel/LegalizerHelper.cpp -----------------------===//
       2             : //
       3             : //                     The LLVM Compiler Infrastructure
       4             : //
       5             : // This file is distributed under the University of Illinois Open Source
       6             : // License. See LICENSE.TXT for details.
       7             : //
       8             : //===----------------------------------------------------------------------===//
       9             : //
      10             : /// \file This file implements the LegalizerHelper class to legalize
      11             : /// individual instructions and the LegalizeMachineIR wrapper pass for the
      12             : /// primary legalization.
      13             : //
      14             : //===----------------------------------------------------------------------===//
      15             : 
      16             : #include "llvm/CodeGen/GlobalISel/LegalizerHelper.h"
      17             : #include "llvm/CodeGen/GlobalISel/CallLowering.h"
      18             : #include "llvm/CodeGen/GlobalISel/LegalizerInfo.h"
      19             : #include "llvm/CodeGen/MachineRegisterInfo.h"
      20             : #include "llvm/CodeGen/TargetInstrInfo.h"
      21             : #include "llvm/CodeGen/TargetLowering.h"
      22             : #include "llvm/CodeGen/TargetSubtargetInfo.h"
      23             : #include "llvm/Support/Debug.h"
      24             : #include "llvm/Support/MathExtras.h"
      25             : #include "llvm/Support/raw_ostream.h"
      26             : 
      27             : #define DEBUG_TYPE "legalizer"
      28             : 
      29             : using namespace llvm;
      30             : using namespace LegalizeActions;
      31             : 
      32        1512 : LegalizerHelper::LegalizerHelper(MachineFunction &MF)
      33        1512 :     : MRI(MF.getRegInfo()), LI(*MF.getSubtarget().getLegalizerInfo()) {
      34        1512 :   MIRBuilder.setMF(MF);
      35        1512 : }
      36             : 
      37          13 : LegalizerHelper::LegalizerHelper(MachineFunction &MF, const LegalizerInfo &LI)
      38          13 :     : MRI(MF.getRegInfo()), LI(LI) {
      39          13 :   MIRBuilder.setMF(MF);
      40          13 : }
      41             : LegalizerHelper::LegalizeResult
      42        5877 : LegalizerHelper::legalizeInstrStep(MachineInstr &MI) {
      43             :   LLVM_DEBUG(dbgs() << "Legalizing: "; MI.print(dbgs()));
      44             : 
      45        5877 :   auto Step = LI.getAction(MI, MRI);
      46        5877 :   switch (Step.Action) {
      47             :   case Legal:
      48             :     LLVM_DEBUG(dbgs() << ".. Already legal\n");
      49             :     return AlreadyLegal;
      50         118 :   case Libcall:
      51             :     LLVM_DEBUG(dbgs() << ".. Convert to libcall\n");
      52         118 :     return libcall(MI);
      53          65 :   case NarrowScalar:
      54             :     LLVM_DEBUG(dbgs() << ".. Narrow scalar\n");
      55          65 :     return narrowScalar(MI, Step.TypeIdx, Step.NewType);
      56         347 :   case WidenScalar:
      57             :     LLVM_DEBUG(dbgs() << ".. Widen scalar\n");
      58         347 :     return widenScalar(MI, Step.TypeIdx, Step.NewType);
      59          40 :   case Lower:
      60             :     LLVM_DEBUG(dbgs() << ".. Lower\n");
      61          40 :     return lower(MI, Step.TypeIdx, Step.NewType);
      62          32 :   case FewerElements:
      63             :     LLVM_DEBUG(dbgs() << ".. Reduce number of elements\n");
      64          32 :     return fewerElementsVector(MI, Step.TypeIdx, Step.NewType);
      65          95 :   case Custom:
      66             :     LLVM_DEBUG(dbgs() << ".. Custom legalization\n");
      67          95 :     return LI.legalizeCustom(MI, MRI, MIRBuilder) ? Legalized
      68             :                                                   : UnableToLegalize;
      69          11 :   default:
      70             :     LLVM_DEBUG(dbgs() << ".. Unable to legalize\n");
      71          11 :     return UnableToLegalize;
      72             :   }
      73             : }
      74             : 
      75         100 : void LegalizerHelper::extractParts(unsigned Reg, LLT Ty, int NumParts,
      76             :                                    SmallVectorImpl<unsigned> &VRegs) {
      77         341 :   for (int i = 0; i < NumParts; ++i)
      78         482 :     VRegs.push_back(MRI.createGenericVirtualRegister(Ty));
      79         200 :   MIRBuilder.buildUnmerge(VRegs, Reg);
      80         100 : }
      81             : 
      82          98 : static RTLIB::Libcall getRTLibDesc(unsigned Opcode, unsigned Size) {
      83          98 :   switch (Opcode) {
      84             :   case TargetOpcode::G_SDIV:
      85             :     assert(Size == 32 && "Unsupported size");
      86             :     return RTLIB::SDIV_I32;
      87          12 :   case TargetOpcode::G_UDIV:
      88             :     assert(Size == 32 && "Unsupported size");
      89          12 :     return RTLIB::UDIV_I32;
      90           6 :   case TargetOpcode::G_SREM:
      91             :     assert(Size == 32 && "Unsupported size");
      92           6 :     return RTLIB::SREM_I32;
      93           6 :   case TargetOpcode::G_UREM:
      94             :     assert(Size == 32 && "Unsupported size");
      95           6 :     return RTLIB::UREM_I32;
      96           8 :   case TargetOpcode::G_FADD:
      97             :     assert((Size == 32 || Size == 64) && "Unsupported size");
      98           8 :     return Size == 64 ? RTLIB::ADD_F64 : RTLIB::ADD_F32;
      99          12 :   case TargetOpcode::G_FSUB:
     100             :     assert((Size == 32 || Size == 64) && "Unsupported size");
     101          12 :     return Size == 64 ? RTLIB::SUB_F64 : RTLIB::SUB_F32;
     102           4 :   case TargetOpcode::G_FMUL:
     103             :     assert((Size == 32 || Size == 64) && "Unsupported size");
     104           4 :     return Size == 64 ? RTLIB::MUL_F64 : RTLIB::MUL_F32;
     105           4 :   case TargetOpcode::G_FDIV:
     106             :     assert((Size == 32 || Size == 64) && "Unsupported size");
     107           4 :     return Size == 64 ? RTLIB::DIV_F64 : RTLIB::DIV_F32;
     108          14 :   case TargetOpcode::G_FREM:
     109          14 :     return Size == 64 ? RTLIB::REM_F64 : RTLIB::REM_F32;
     110          14 :   case TargetOpcode::G_FPOW:
     111          14 :     return Size == 64 ? RTLIB::POW_F64 : RTLIB::POW_F32;
     112           6 :   case TargetOpcode::G_FMA:
     113             :     assert((Size == 32 || Size == 64) && "Unsupported size");
     114           6 :     return Size == 64 ? RTLIB::FMA_F64 : RTLIB::FMA_F32;
     115             :   }
     116           0 :   llvm_unreachable("Unknown libcall function");
     117             : }
     118             : 
     119             : LegalizerHelper::LegalizeResult
     120         200 : llvm::createLibcall(MachineIRBuilder &MIRBuilder, RTLIB::Libcall Libcall,
     121             :                     const CallLowering::ArgInfo &Result,
     122             :                     ArrayRef<CallLowering::ArgInfo> Args) {
     123         200 :   auto &CLI = *MIRBuilder.getMF().getSubtarget().getCallLowering();
     124         200 :   auto &TLI = *MIRBuilder.getMF().getSubtarget().getTargetLowering();
     125             :   const char *Name = TLI.getLibcallName(Libcall);
     126             : 
     127         200 :   MIRBuilder.getMF().getFrameInfo().setHasCalls(true);
     128         200 :   if (!CLI.lowerCall(MIRBuilder, TLI.getLibcallCallingConv(Libcall),
     129         200 :                      MachineOperand::CreateES(Name), Result, Args))
     130           0 :     return LegalizerHelper::UnableToLegalize;
     131             : 
     132             :   return LegalizerHelper::Legalized;
     133             : }
     134             : 
     135             : // Useful for libcalls where all operands have the same type.
     136             : static LegalizerHelper::LegalizeResult
     137          98 : simpleLibcall(MachineInstr &MI, MachineIRBuilder &MIRBuilder, unsigned Size,
     138             :               Type *OpType) {
     139         196 :   auto Libcall = getRTLibDesc(MI.getOpcode(), Size);
     140             : 
     141             :   SmallVector<CallLowering::ArgInfo, 3> Args;
     142         300 :   for (unsigned i = 1; i < MI.getNumOperands(); i++)
     143         606 :     Args.push_back({MI.getOperand(i).getReg(), OpType});
     144         196 :   return createLibcall(MIRBuilder, Libcall, {MI.getOperand(0).getReg(), OpType},
     145          98 :                        Args);
     146             : }
     147             : 
     148          20 : static RTLIB::Libcall getConvRTLibDesc(unsigned Opcode, Type *ToType,
     149             :                                        Type *FromType) {
     150          20 :   auto ToMVT = MVT::getVT(ToType);
     151          20 :   auto FromMVT = MVT::getVT(FromType);
     152             : 
     153          20 :   switch (Opcode) {
     154           2 :   case TargetOpcode::G_FPEXT:
     155           2 :     return RTLIB::getFPEXT(FromMVT, ToMVT);
     156           2 :   case TargetOpcode::G_FPTRUNC:
     157           2 :     return RTLIB::getFPROUND(FromMVT, ToMVT);
     158           4 :   case TargetOpcode::G_FPTOSI:
     159           4 :     return RTLIB::getFPTOSINT(FromMVT, ToMVT);
     160           4 :   case TargetOpcode::G_FPTOUI:
     161           4 :     return RTLIB::getFPTOUINT(FromMVT, ToMVT);
     162           4 :   case TargetOpcode::G_SITOFP:
     163           4 :     return RTLIB::getSINTTOFP(FromMVT, ToMVT);
     164           4 :   case TargetOpcode::G_UITOFP:
     165           4 :     return RTLIB::getUINTTOFP(FromMVT, ToMVT);
     166             :   }
     167           0 :   llvm_unreachable("Unsupported libcall function");
     168             : }
     169             : 
     170             : static LegalizerHelper::LegalizeResult
     171          20 : conversionLibcall(MachineInstr &MI, MachineIRBuilder &MIRBuilder, Type *ToType,
     172             :                   Type *FromType) {
     173          40 :   RTLIB::Libcall Libcall = getConvRTLibDesc(MI.getOpcode(), ToType, FromType);
     174          60 :   return createLibcall(MIRBuilder, Libcall, {MI.getOperand(0).getReg(), ToType},
     175          20 :                        {{MI.getOperand(1).getReg(), FromType}});
     176             : }
     177             : 
     178             : LegalizerHelper::LegalizeResult
     179         118 : LegalizerHelper::libcall(MachineInstr &MI) {
     180         118 :   LLT LLTy = MRI.getType(MI.getOperand(0).getReg());
     181         118 :   unsigned Size = LLTy.getSizeInBits();
     182         118 :   auto &Ctx = MIRBuilder.getMF().getFunction().getContext();
     183             : 
     184         118 :   MIRBuilder.setInstr(MI);
     185             : 
     186         236 :   switch (MI.getOpcode()) {
     187             :   default:
     188             :     return UnableToLegalize;
     189          36 :   case TargetOpcode::G_SDIV:
     190             :   case TargetOpcode::G_UDIV:
     191             :   case TargetOpcode::G_SREM:
     192             :   case TargetOpcode::G_UREM: {
     193          36 :     Type *HLTy = Type::getInt32Ty(Ctx);
     194          36 :     auto Status = simpleLibcall(MI, MIRBuilder, Size, HLTy);
     195          36 :     if (Status != Legalized)
     196             :       return Status;
     197             :     break;
     198             :   }
     199          62 :   case TargetOpcode::G_FADD:
     200             :   case TargetOpcode::G_FSUB:
     201             :   case TargetOpcode::G_FMUL:
     202             :   case TargetOpcode::G_FDIV:
     203             :   case TargetOpcode::G_FMA:
     204             :   case TargetOpcode::G_FPOW:
     205             :   case TargetOpcode::G_FREM: {
     206          62 :     Type *HLTy = Size == 64 ? Type::getDoubleTy(Ctx) : Type::getFloatTy(Ctx);
     207          62 :     auto Status = simpleLibcall(MI, MIRBuilder, Size, HLTy);
     208          62 :     if (Status != Legalized)
     209             :       return Status;
     210             :     break;
     211             :   }
     212           2 :   case TargetOpcode::G_FPEXT: {
     213             :     // FIXME: Support other floating point types (half, fp128 etc)
     214           2 :     unsigned FromSize = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
     215           2 :     unsigned ToSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
     216           2 :     if (ToSize != 64 || FromSize != 32)
     217             :       return UnableToLegalize;
     218           2 :     LegalizeResult Status = conversionLibcall(
     219           2 :         MI, MIRBuilder, Type::getDoubleTy(Ctx), Type::getFloatTy(Ctx));
     220           2 :     if (Status != Legalized)
     221             :       return Status;
     222             :     break;
     223             :   }
     224           2 :   case TargetOpcode::G_FPTRUNC: {
     225             :     // FIXME: Support other floating point types (half, fp128 etc)
     226           2 :     unsigned FromSize = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
     227           2 :     unsigned ToSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
     228           2 :     if (ToSize != 32 || FromSize != 64)
     229             :       return UnableToLegalize;
     230           2 :     LegalizeResult Status = conversionLibcall(
     231           2 :         MI, MIRBuilder, Type::getFloatTy(Ctx), Type::getDoubleTy(Ctx));
     232           2 :     if (Status != Legalized)
     233             :       return Status;
     234             :     break;
     235             :   }
     236           8 :   case TargetOpcode::G_FPTOSI:
     237             :   case TargetOpcode::G_FPTOUI: {
     238             :     // FIXME: Support other types
     239           8 :     unsigned FromSize = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
     240           8 :     unsigned ToSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
     241           8 :     if (ToSize != 32 || (FromSize != 32 && FromSize != 64))
     242             :       return UnableToLegalize;
     243           8 :     LegalizeResult Status = conversionLibcall(
     244           8 :         MI, MIRBuilder, Type::getInt32Ty(Ctx),
     245           8 :         FromSize == 64 ? Type::getDoubleTy(Ctx) : Type::getFloatTy(Ctx));
     246           8 :     if (Status != Legalized)
     247             :       return Status;
     248             :     break;
     249             :   }
     250           8 :   case TargetOpcode::G_SITOFP:
     251             :   case TargetOpcode::G_UITOFP: {
     252             :     // FIXME: Support other types
     253           8 :     unsigned FromSize = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
     254           8 :     unsigned ToSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
     255           8 :     if (FromSize != 32 || (ToSize != 32 && ToSize != 64))
     256             :       return UnableToLegalize;
     257           8 :     LegalizeResult Status = conversionLibcall(
     258           8 :         MI, MIRBuilder,
     259           8 :         ToSize == 64 ? Type::getDoubleTy(Ctx) : Type::getFloatTy(Ctx),
     260           8 :         Type::getInt32Ty(Ctx));
     261           8 :     if (Status != Legalized)
     262             :       return Status;
     263             :     break;
     264             :   }
     265             :   }
     266             : 
     267         118 :   MI.eraseFromParent();
     268         118 :   return Legalized;
     269             : }
     270             : 
     271          65 : LegalizerHelper::LegalizeResult LegalizerHelper::narrowScalar(MachineInstr &MI,
     272             :                                                               unsigned TypeIdx,
     273             :                                                               LLT NarrowTy) {
     274             :   // FIXME: Don't know how to handle secondary types yet.
     275          65 :   if (TypeIdx != 0 && MI.getOpcode() != TargetOpcode::G_EXTRACT)
     276             :     return UnableToLegalize;
     277             : 
     278          65 :   MIRBuilder.setInstr(MI);
     279             : 
     280          65 :   uint64_t SizeOp0 = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
     281          65 :   uint64_t NarrowSize = NarrowTy.getSizeInBits();
     282             : 
     283         130 :   switch (MI.getOpcode()) {
     284             :   default:
     285             :     return UnableToLegalize;
     286           4 :   case TargetOpcode::G_IMPLICIT_DEF: {
     287             :     // FIXME: add support for when SizeOp0 isn't an exact multiple of
     288             :     // NarrowSize.
     289           4 :     if (SizeOp0 % NarrowSize != 0)
     290             :       return UnableToLegalize;
     291           4 :     int NumParts = SizeOp0 / NarrowSize;
     292             : 
     293             :     SmallVector<unsigned, 2> DstRegs;
     294          12 :     for (int i = 0; i < NumParts; ++i)
     295           8 :       DstRegs.push_back(
     296           8 :           MIRBuilder.buildUndef(NarrowTy)->getOperand(0).getReg());
     297           4 :     MIRBuilder.buildMerge(MI.getOperand(0).getReg(), DstRegs);
     298           4 :     MI.eraseFromParent();
     299             :     return Legalized;
     300             :   }
     301           4 :   case TargetOpcode::G_ADD: {
     302             :     // FIXME: add support for when SizeOp0 isn't an exact multiple of
     303             :     // NarrowSize.
     304           4 :     if (SizeOp0 % NarrowSize != 0)
     305             :       return UnableToLegalize;
     306             :     // Expand in terms of carry-setting/consuming G_ADDE instructions.
     307           4 :     int NumParts = SizeOp0 / NarrowTy.getSizeInBits();
     308             : 
     309             :     SmallVector<unsigned, 2> Src1Regs, Src2Regs, DstRegs;
     310           4 :     extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, Src1Regs);
     311           4 :     extractParts(MI.getOperand(2).getReg(), NarrowTy, NumParts, Src2Regs);
     312             : 
     313           8 :     unsigned CarryIn = MRI.createGenericVirtualRegister(LLT::scalar(1));
     314           4 :     MIRBuilder.buildConstant(CarryIn, 0);
     315             : 
     316          13 :     for (int i = 0; i < NumParts; ++i) {
     317          18 :       unsigned DstReg = MRI.createGenericVirtualRegister(NarrowTy);
     318          18 :       unsigned CarryOut = MRI.createGenericVirtualRegister(LLT::scalar(1));
     319             : 
     320             :       MIRBuilder.buildUAdde(DstReg, CarryOut, Src1Regs[i],
     321          27 :                             Src2Regs[i], CarryIn);
     322             : 
     323           9 :       DstRegs.push_back(DstReg);
     324             :       CarryIn = CarryOut;
     325             :     }
     326           4 :     unsigned DstReg = MI.getOperand(0).getReg();
     327           4 :     MIRBuilder.buildMerge(DstReg, DstRegs);
     328           4 :     MI.eraseFromParent();
     329             :     return Legalized;
     330             :   }
     331           8 :   case TargetOpcode::G_EXTRACT: {
     332           8 :     if (TypeIdx != 1)
     333             :       return UnableToLegalize;
     334             : 
     335           8 :     int64_t SizeOp1 = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
     336             :     // FIXME: add support for when SizeOp1 isn't an exact multiple of
     337             :     // NarrowSize.
     338           8 :     if (SizeOp1 % NarrowSize != 0)
     339             :       return UnableToLegalize;
     340           8 :     int NumParts = SizeOp1 / NarrowSize;
     341             : 
     342             :     SmallVector<unsigned, 2> SrcRegs, DstRegs;
     343             :     SmallVector<uint64_t, 2> Indexes;
     344           8 :     extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, SrcRegs);
     345             : 
     346           8 :     unsigned OpReg = MI.getOperand(0).getReg();
     347           8 :     uint64_t OpStart = MI.getOperand(2).getImm();
     348          16 :     uint64_t OpSize = MRI.getType(OpReg).getSizeInBits();
     349          24 :     for (int i = 0; i < NumParts; ++i) {
     350          16 :       unsigned SrcStart = i * NarrowSize;
     351             : 
     352          16 :       if (SrcStart + NarrowSize <= OpStart || SrcStart >= OpStart + OpSize) {
     353             :         // No part of the extract uses this subregister, ignore it.
     354          12 :         continue;
     355          15 :       } else if (SrcStart == OpStart && NarrowTy == MRI.getType(OpReg)) {
     356             :         // The entire subregister is extracted, forward the value.
     357           5 :         DstRegs.push_back(SrcRegs[i]);
     358           5 :         continue;
     359             :       }
     360             : 
     361             :       // OpSegStart is where this destination segment would start in OpReg if it
     362             :       // extended infinitely in both directions.
     363             :       int64_t ExtractOffset;
     364             :       uint64_t SegSize;
     365           4 :       if (OpStart < SrcStart) {
     366             :         ExtractOffset = 0;
     367           2 :         SegSize = std::min(NarrowSize, OpStart + OpSize - SrcStart);
     368             :       } else {
     369           3 :         ExtractOffset = OpStart - SrcStart;
     370           4 :         SegSize = std::min(SrcStart + NarrowSize - OpStart, OpSize);
     371             :       }
     372             : 
     373           4 :       unsigned SegReg = SrcRegs[i];
     374           4 :       if (ExtractOffset != 0 || SegSize != NarrowSize) {
     375             :         // A genuine extract is needed.
     376          12 :         SegReg = MRI.createGenericVirtualRegister(LLT::scalar(SegSize));
     377           8 :         MIRBuilder.buildExtract(SegReg, SrcRegs[i], ExtractOffset);
     378             :       }
     379             : 
     380           4 :       DstRegs.push_back(SegReg);
     381             :     }
     382             : 
     383           8 :     MIRBuilder.buildMerge(MI.getOperand(0).getReg(), DstRegs);
     384           8 :     MI.eraseFromParent();
     385             :     return Legalized;
     386             :   }
     387          11 :   case TargetOpcode::G_INSERT: {
     388             :     // FIXME: add support for when SizeOp0 isn't an exact multiple of
     389             :     // NarrowSize.
     390          11 :     if (SizeOp0 % NarrowSize != 0)
     391             :       return UnableToLegalize;
     392             : 
     393          11 :     int NumParts = SizeOp0 / NarrowSize;
     394             : 
     395             :     SmallVector<unsigned, 2> SrcRegs, DstRegs;
     396             :     SmallVector<uint64_t, 2> Indexes;
     397          11 :     extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, SrcRegs);
     398             : 
     399          11 :     unsigned OpReg = MI.getOperand(2).getReg();
     400          11 :     uint64_t OpStart = MI.getOperand(3).getImm();
     401          22 :     uint64_t OpSize = MRI.getType(OpReg).getSizeInBits();
     402          34 :     for (int i = 0; i < NumParts; ++i) {
     403          23 :       unsigned DstStart = i * NarrowSize;
     404             : 
     405          23 :       if (DstStart + NarrowSize <= OpStart || DstStart >= OpStart + OpSize) {
     406             :         // No part of the insert affects this subregister, forward the original.
     407          11 :         DstRegs.push_back(SrcRegs[i]);
     408          18 :         continue;
     409          14 :       } else if (DstStart == OpStart && NarrowTy == MRI.getType(OpReg)) {
     410             :         // The entire subregister is defined by this insert, forward the new
     411             :         // value.
     412           7 :         DstRegs.push_back(OpReg);
     413           7 :         continue;
     414             :       }
     415             : 
     416             :       // OpSegStart is where this destination segment would start in OpReg if it
     417             :       // extended infinitely in both directions.
     418             :       int64_t ExtractOffset, InsertOffset;
     419             :       uint64_t SegSize;
     420           5 :       if (OpStart < DstStart) {
     421             :         InsertOffset = 0;
     422           1 :         ExtractOffset = DstStart - OpStart;
     423           2 :         SegSize = std::min(NarrowSize, OpStart + OpSize - DstStart);
     424             :       } else {
     425           4 :         InsertOffset = OpStart - DstStart;
     426             :         ExtractOffset = 0;
     427           4 :         SegSize =
     428           5 :             std::min(NarrowSize - InsertOffset, OpStart + OpSize - DstStart);
     429             :       }
     430             : 
     431           5 :       unsigned SegReg = OpReg;
     432           5 :       if (ExtractOffset != 0 || SegSize != OpSize) {
     433             :         // A genuine extract is needed.
     434           6 :         SegReg = MRI.createGenericVirtualRegister(LLT::scalar(SegSize));
     435           2 :         MIRBuilder.buildExtract(SegReg, OpReg, ExtractOffset);
     436             :       }
     437             : 
     438          10 :       unsigned DstReg = MRI.createGenericVirtualRegister(NarrowTy);
     439          10 :       MIRBuilder.buildInsert(DstReg, SrcRegs[i], SegReg, InsertOffset);
     440           5 :       DstRegs.push_back(DstReg);
     441             :     }
     442             : 
     443             :     assert(DstRegs.size() == (unsigned)NumParts && "not all parts covered");
     444          11 :     MIRBuilder.buildMerge(MI.getOperand(0).getReg(), DstRegs);
     445          11 :     MI.eraseFromParent();
     446             :     return Legalized;
     447             :   }
     448           8 :   case TargetOpcode::G_LOAD: {
     449             :     // FIXME: add support for when SizeOp0 isn't an exact multiple of
     450             :     // NarrowSize.
     451           8 :     if (SizeOp0 % NarrowSize != 0)
     452             :       return UnableToLegalize;
     453             : 
     454           8 :     const auto &MMO = **MI.memoperands_begin();
     455             :     // This implementation doesn't work for atomics. Give up instead of doing
     456             :     // something invalid.
     457           8 :     if (MMO.getOrdering() != AtomicOrdering::NotAtomic ||
     458             :         MMO.getFailureOrdering() != AtomicOrdering::NotAtomic)
     459             :       return UnableToLegalize;
     460             : 
     461           8 :     int NumParts = SizeOp0 / NarrowSize;
     462             :     LLT OffsetTy = LLT::scalar(
     463          16 :         MRI.getType(MI.getOperand(1).getReg()).getScalarSizeInBits());
     464             : 
     465             :     SmallVector<unsigned, 2> DstRegs;
     466          24 :     for (int i = 0; i < NumParts; ++i) {
     467          32 :       unsigned DstReg = MRI.createGenericVirtualRegister(NarrowTy);
     468          16 :       unsigned SrcReg = 0;
     469          16 :       unsigned Adjustment = i * NarrowSize / 8;
     470             : 
     471          64 :       MachineMemOperand *SplitMMO = MIRBuilder.getMF().getMachineMemOperand(
     472          16 :           MMO.getPointerInfo().getWithOffset(Adjustment), MMO.getFlags(),
     473          16 :           NarrowSize / 8, i == 0 ? MMO.getAlignment() : NarrowSize / 8,
     474          16 :           MMO.getAAInfo(), MMO.getRanges(), MMO.getSyncScopeID(),
     475             :           MMO.getOrdering(), MMO.getFailureOrdering());
     476             : 
     477          16 :       MIRBuilder.materializeGEP(SrcReg, MI.getOperand(1).getReg(), OffsetTy,
     478             :                                 Adjustment);
     479             : 
     480          16 :       MIRBuilder.buildLoad(DstReg, SrcReg, *SplitMMO);
     481             : 
     482          16 :       DstRegs.push_back(DstReg);
     483             :     }
     484           8 :     unsigned DstReg = MI.getOperand(0).getReg();
     485           8 :     MIRBuilder.buildMerge(DstReg, DstRegs);
     486           8 :     MI.eraseFromParent();
     487             :     return Legalized;
     488             :   }
     489          20 :   case TargetOpcode::G_STORE: {
     490             :     // FIXME: add support for when SizeOp0 isn't an exact multiple of
     491             :     // NarrowSize.
     492          20 :     if (SizeOp0 % NarrowSize != 0)
     493             :       return UnableToLegalize;
     494             : 
     495           9 :     const auto &MMO = **MI.memoperands_begin();
     496             :     // This implementation doesn't work for atomics. Give up instead of doing
     497             :     // something invalid.
     498           9 :     if (MMO.getOrdering() != AtomicOrdering::NotAtomic ||
     499             :         MMO.getFailureOrdering() != AtomicOrdering::NotAtomic)
     500             :       return UnableToLegalize;
     501             : 
     502           9 :     int NumParts = SizeOp0 / NarrowSize;
     503             :     LLT OffsetTy = LLT::scalar(
     504          18 :         MRI.getType(MI.getOperand(1).getReg()).getScalarSizeInBits());
     505             : 
     506             :     SmallVector<unsigned, 2> SrcRegs;
     507           9 :     extractParts(MI.getOperand(0).getReg(), NarrowTy, NumParts, SrcRegs);
     508             : 
     509          27 :     for (int i = 0; i < NumParts; ++i) {
     510          18 :       unsigned DstReg = 0;
     511          18 :       unsigned Adjustment = i * NarrowSize / 8;
     512             : 
     513          72 :       MachineMemOperand *SplitMMO = MIRBuilder.getMF().getMachineMemOperand(
     514          18 :           MMO.getPointerInfo().getWithOffset(Adjustment), MMO.getFlags(),
     515          18 :           NarrowSize / 8, i == 0 ? MMO.getAlignment() : NarrowSize / 8,
     516          18 :           MMO.getAAInfo(), MMO.getRanges(), MMO.getSyncScopeID(),
     517             :           MMO.getOrdering(), MMO.getFailureOrdering());
     518             : 
     519          18 :       MIRBuilder.materializeGEP(DstReg, MI.getOperand(1).getReg(), OffsetTy,
     520             :                                 Adjustment);
     521             : 
     522          36 :       MIRBuilder.buildStore(SrcRegs[i], DstReg, *SplitMMO);
     523             :     }
     524           9 :     MI.eraseFromParent();
     525             :     return Legalized;
     526             :   }
     527           7 :   case TargetOpcode::G_CONSTANT: {
     528             :     // FIXME: add support for when SizeOp0 isn't an exact multiple of
     529             :     // NarrowSize.
     530           7 :     if (SizeOp0 % NarrowSize != 0)
     531             :       return UnableToLegalize;
     532           7 :     int NumParts = SizeOp0 / NarrowSize;
     533           7 :     const APInt &Cst = MI.getOperand(1).getCImm()->getValue();
     534           7 :     LLVMContext &Ctx = MIRBuilder.getMF().getFunction().getContext();
     535             : 
     536             :     SmallVector<unsigned, 2> DstRegs;
     537          21 :     for (int i = 0; i < NumParts; ++i) {
     538          28 :       unsigned DstReg = MRI.createGenericVirtualRegister(NarrowTy);
     539             :       ConstantInt *CI =
     540          28 :           ConstantInt::get(Ctx, Cst.lshr(NarrowSize * i).trunc(NarrowSize));
     541          14 :       MIRBuilder.buildConstant(DstReg, *CI);
     542          14 :       DstRegs.push_back(DstReg);
     543             :     }
     544           7 :     unsigned DstReg = MI.getOperand(0).getReg();
     545           7 :     MIRBuilder.buildMerge(DstReg, DstRegs);
     546           7 :     MI.eraseFromParent();
     547             :     return Legalized;
     548             :   }
     549           1 :   case TargetOpcode::G_OR: {
     550             :     // Legalize bitwise operation:
     551             :     // A = BinOp<Ty> B, C
     552             :     // into:
     553             :     // B1, ..., BN = G_UNMERGE_VALUES B
     554             :     // C1, ..., CN = G_UNMERGE_VALUES C
     555             :     // A1 = BinOp<Ty/N> B1, C2
     556             :     // ...
     557             :     // AN = BinOp<Ty/N> BN, CN
     558             :     // A = G_MERGE_VALUES A1, ..., AN
     559             : 
     560             :     // FIXME: add support for when SizeOp0 isn't an exact multiple of
     561             :     // NarrowSize.
     562           1 :     if (SizeOp0 % NarrowSize != 0)
     563             :       return UnableToLegalize;
     564           1 :     int NumParts = SizeOp0 / NarrowSize;
     565             : 
     566             :     // List the registers where the destination will be scattered.
     567             :     SmallVector<unsigned, 2> DstRegs;
     568             :     // List the registers where the first argument will be split.
     569             :     SmallVector<unsigned, 2> SrcsReg1;
     570             :     // List the registers where the second argument will be split.
     571             :     SmallVector<unsigned, 2> SrcsReg2;
     572             :     // Create all the temporary registers.
     573           3 :     for (int i = 0; i < NumParts; ++i) {
     574           4 :       unsigned DstReg = MRI.createGenericVirtualRegister(NarrowTy);
     575           4 :       unsigned SrcReg1 = MRI.createGenericVirtualRegister(NarrowTy);
     576           4 :       unsigned SrcReg2 = MRI.createGenericVirtualRegister(NarrowTy);
     577             : 
     578           2 :       DstRegs.push_back(DstReg);
     579           2 :       SrcsReg1.push_back(SrcReg1);
     580           2 :       SrcsReg2.push_back(SrcReg2);
     581             :     }
     582             :     // Explode the big arguments into smaller chunks.
     583           2 :     MIRBuilder.buildUnmerge(SrcsReg1, MI.getOperand(1).getReg());
     584           2 :     MIRBuilder.buildUnmerge(SrcsReg2, MI.getOperand(2).getReg());
     585             : 
     586             :     // Do the operation on each small part.
     587           3 :     for (int i = 0; i < NumParts; ++i)
     588           8 :       MIRBuilder.buildOr(DstRegs[i], SrcsReg1[i], SrcsReg2[i]);
     589             : 
     590             :     // Gather the destination registers into the final destination.
     591           1 :     unsigned DstReg = MI.getOperand(0).getReg();
     592           1 :     MIRBuilder.buildMerge(DstReg, DstRegs);
     593           1 :     MI.eraseFromParent();
     594             :     return Legalized;
     595             :   }
     596             :   }
     597             : }
     598             : 
     599         369 : void LegalizerHelper::widenScalarSrc(MachineInstr &MI, LLT WideTy,
     600             :                                      unsigned OpIdx, unsigned ExtOpcode) {
     601         369 :   MachineOperand &MO = MI.getOperand(OpIdx);
     602         369 :   auto ExtB = MIRBuilder.buildInstr(ExtOpcode, WideTy, MO.getReg());
     603         369 :   MO.setReg(ExtB->getOperand(0).getReg());
     604         369 : }
     605             : 
     606         310 : void LegalizerHelper::widenScalarDst(MachineInstr &MI, LLT WideTy,
     607             :                                      unsigned OpIdx, unsigned TruncOpcode) {
     608         310 :   MachineOperand &MO = MI.getOperand(OpIdx);
     609         620 :   unsigned DstExt = MRI.createGenericVirtualRegister(WideTy);
     610         620 :   MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt());
     611         310 :   MIRBuilder.buildInstr(TruncOpcode, MO.getReg(), DstExt);
     612         310 :   MO.setReg(DstExt);
     613         310 : }
     614             : 
     615             : LegalizerHelper::LegalizeResult
     616         354 : LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
     617         354 :   MIRBuilder.setInstr(MI);
     618             : 
     619         708 :   switch (MI.getOpcode()) {
     620             :   default:
     621             :     return UnableToLegalize;
     622           2 :   case TargetOpcode::G_UADDO:
     623             :   case TargetOpcode::G_USUBO: {
     624           2 :     if (TypeIdx == 1)
     625             :       return UnableToLegalize; // TODO
     626             :     auto LHSZext = MIRBuilder.buildInstr(TargetOpcode::G_ZEXT, WideTy,
     627           2 :                                          MI.getOperand(2).getReg());
     628             :     auto RHSZext = MIRBuilder.buildInstr(TargetOpcode::G_ZEXT, WideTy,
     629           2 :                                          MI.getOperand(3).getReg());
     630           2 :     unsigned Opcode = MI.getOpcode() == TargetOpcode::G_UADDO
     631           2 :                           ? TargetOpcode::G_ADD
     632             :                           : TargetOpcode::G_SUB;
     633             :     // Do the arithmetic in the larger type.
     634           2 :     auto NewOp = MIRBuilder.buildInstr(Opcode, WideTy, LHSZext, RHSZext);
     635           2 :     LLT OrigTy = MRI.getType(MI.getOperand(0).getReg());
     636           2 :     APInt Mask = APInt::getAllOnesValue(OrigTy.getSizeInBits());
     637             :     auto AndOp = MIRBuilder.buildInstr(
     638             :         TargetOpcode::G_AND, WideTy, NewOp,
     639           2 :         MIRBuilder.buildConstant(WideTy, Mask.getZExtValue()));
     640             :     // There is no overflow if the AndOp is the same as NewOp.
     641           2 :     MIRBuilder.buildICmp(CmpInst::ICMP_NE, MI.getOperand(1).getReg(), NewOp,
     642             :                          AndOp);
     643             :     // Now trunc the NewOp to the original result.
     644           2 :     MIRBuilder.buildTrunc(MI.getOperand(0).getReg(), NewOp);
     645           2 :     MI.eraseFromParent();
     646             :     return Legalized;
     647             :   }
     648           5 :   case TargetOpcode::G_CTTZ:
     649             :   case TargetOpcode::G_CTTZ_ZERO_UNDEF:
     650             :   case TargetOpcode::G_CTLZ:
     651             :   case TargetOpcode::G_CTLZ_ZERO_UNDEF:
     652             :   case TargetOpcode::G_CTPOP: {
     653             :     // First ZEXT the input.
     654           5 :     auto MIBSrc = MIRBuilder.buildZExt(WideTy, MI.getOperand(1).getReg());
     655           5 :     LLT CurTy = MRI.getType(MI.getOperand(0).getReg());
     656          10 :     if (MI.getOpcode() == TargetOpcode::G_CTTZ) {
     657             :       // The count is the same in the larger type except if the original
     658             :       // value was zero.  This can be handled by setting the bit just off
     659             :       // the top of the original type.
     660             :       auto TopBit =
     661           1 :           APInt::getOneBitSet(WideTy.getSizeInBits(), CurTy.getSizeInBits());
     662             :       MIBSrc = MIRBuilder.buildInstr(
     663             :           TargetOpcode::G_OR, WideTy, MIBSrc,
     664           2 :           MIRBuilder.buildConstant(WideTy, TopBit.getSExtValue()));
     665             :     }
     666             :     // Perform the operation at the larger size.
     667          10 :     auto MIBNewOp = MIRBuilder.buildInstr(MI.getOpcode(), WideTy, MIBSrc);
     668             :     // This is already the correct result for CTPOP and CTTZs
     669          10 :     if (MI.getOpcode() == TargetOpcode::G_CTLZ ||
     670             :         MI.getOpcode() == TargetOpcode::G_CTLZ_ZERO_UNDEF) {
     671             :       // The correct result is NewOp - (Difference in widety and current ty).
     672           2 :       unsigned SizeDiff = WideTy.getSizeInBits() - CurTy.getSizeInBits();
     673             :       MIBNewOp =
     674             :           MIRBuilder.buildInstr(TargetOpcode::G_SUB, WideTy, MIBNewOp,
     675           2 :                                 MIRBuilder.buildConstant(WideTy, SizeDiff));
     676             :     }
     677           5 :     auto &TII = *MI.getMF()->getSubtarget().getInstrInfo();
     678             :     // Make the original instruction a trunc now, and update it's source.
     679           5 :     MI.setDesc(TII.get(TargetOpcode::G_TRUNC));
     680          10 :     MI.getOperand(1).setReg(MIBNewOp->getOperand(0).getReg());
     681           5 :     MIRBuilder.recordInsertion(&MI);
     682             :     return Legalized;
     683             :   }
     684             : 
     685          76 :   case TargetOpcode::G_ADD:
     686             :   case TargetOpcode::G_AND:
     687             :   case TargetOpcode::G_MUL:
     688             :   case TargetOpcode::G_OR:
     689             :   case TargetOpcode::G_XOR:
     690             :   case TargetOpcode::G_SUB:
     691             :     // Perform operation at larger width (any extension is fine here, high bits
     692             :     // don't affect the result) and then truncate the result back to the
     693             :     // original type.
     694          76 :     widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
     695          76 :     widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ANYEXT);
     696          76 :     widenScalarDst(MI, WideTy);
     697          76 :     MIRBuilder.recordInsertion(&MI);
     698          76 :     return Legalized;
     699             : 
     700           4 :   case TargetOpcode::G_SHL:
     701           4 :     widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
     702             :     // The "number of bits to shift" operand must preserve its value as an
     703             :     // unsigned integer:
     704           4 :     widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ZEXT);
     705           4 :     widenScalarDst(MI, WideTy);
     706           4 :     MIRBuilder.recordInsertion(&MI);
     707           4 :     return Legalized;
     708             : 
     709          34 :   case TargetOpcode::G_SDIV:
     710             :   case TargetOpcode::G_SREM:
     711          34 :     widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_SEXT);
     712          34 :     widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_SEXT);
     713          34 :     widenScalarDst(MI, WideTy);
     714          34 :     MIRBuilder.recordInsertion(&MI);
     715          34 :     return Legalized;
     716             : 
     717           3 :   case TargetOpcode::G_ASHR:
     718           3 :     widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_SEXT);
     719             :     // The "number of bits to shift" operand must preserve its value as an
     720             :     // unsigned integer:
     721           3 :     widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ZEXT);
     722           3 :     widenScalarDst(MI, WideTy);
     723           3 :     MIRBuilder.recordInsertion(&MI);
     724           3 :     return Legalized;
     725             : 
     726          36 :   case TargetOpcode::G_UDIV:
     727             :   case TargetOpcode::G_UREM:
     728             :   case TargetOpcode::G_LSHR:
     729          36 :     widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ZEXT);
     730          36 :     widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ZEXT);
     731          36 :     widenScalarDst(MI, WideTy);
     732          36 :     MIRBuilder.recordInsertion(&MI);
     733          36 :     return Legalized;
     734             : 
     735           3 :   case TargetOpcode::G_SELECT:
     736           3 :     if (TypeIdx != 0)
     737             :       return UnableToLegalize;
     738             :     // Perform operation at larger width (any extension is fine here, high bits
     739             :     // don't affect the result) and then truncate the result back to the
     740             :     // original type.
     741           3 :     widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ANYEXT);
     742           3 :     widenScalarSrc(MI, WideTy, 3, TargetOpcode::G_ANYEXT);
     743           3 :     widenScalarDst(MI, WideTy);
     744           3 :     MIRBuilder.recordInsertion(&MI);
     745           3 :     return Legalized;
     746             : 
     747          12 :   case TargetOpcode::G_FPTOSI:
     748             :   case TargetOpcode::G_FPTOUI:
     749          12 :     if (TypeIdx != 0)
     750             :       return UnableToLegalize;
     751          11 :     widenScalarDst(MI, WideTy);
     752          11 :     MIRBuilder.recordInsertion(&MI);
     753          11 :     return Legalized;
     754             : 
     755           7 :   case TargetOpcode::G_SITOFP:
     756           7 :     if (TypeIdx != 1)
     757             :       return UnableToLegalize;
     758           7 :     widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_SEXT);
     759           7 :     MIRBuilder.recordInsertion(&MI);
     760           7 :     return Legalized;
     761             : 
     762           3 :   case TargetOpcode::G_UITOFP:
     763           3 :     if (TypeIdx != 1)
     764             :       return UnableToLegalize;
     765           3 :     widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ZEXT);
     766           3 :     MIRBuilder.recordInsertion(&MI);
     767           3 :     return Legalized;
     768             : 
     769           1 :   case TargetOpcode::G_INSERT:
     770           1 :     if (TypeIdx != 0)
     771             :       return UnableToLegalize;
     772           1 :     widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
     773           1 :     widenScalarDst(MI, WideTy);
     774           1 :     MIRBuilder.recordInsertion(&MI);
     775           1 :     return Legalized;
     776             : 
     777           9 :   case TargetOpcode::G_LOAD:
     778             :     // For some types like i24, we might try to widen to i32. To properly handle
     779             :     // this we should be using a dedicated extending load, until then avoid
     780             :     // trying to legalize.
     781          18 :     if (alignTo(MRI.getType(MI.getOperand(0).getReg()).getSizeInBits(), 8) !=
     782           9 :         WideTy.getSizeInBits())
     783             :       return UnableToLegalize;
     784             :     LLVM_FALLTHROUGH;
     785             :   case TargetOpcode::G_SEXTLOAD:
     786             :   case TargetOpcode::G_ZEXTLOAD:
     787           9 :     widenScalarDst(MI, WideTy);
     788           9 :     MIRBuilder.recordInsertion(&MI);
     789           9 :     return Legalized;
     790             : 
     791             :   case TargetOpcode::G_STORE: {
     792          17 :     if (MRI.getType(MI.getOperand(0).getReg()) != LLT::scalar(1) ||
     793           0 :         WideTy != LLT::scalar(8))
     794           2 :       return UnableToLegalize;
     795             : 
     796          15 :     widenScalarSrc(MI, WideTy, 0, TargetOpcode::G_ZEXT);
     797          15 :     MIRBuilder.recordInsertion(&MI);
     798          15 :     return Legalized;
     799             :   }
     800          56 :   case TargetOpcode::G_CONSTANT: {
     801          56 :     MachineOperand &SrcMO = MI.getOperand(1);
     802          56 :     LLVMContext &Ctx = MIRBuilder.getMF().getFunction().getContext();
     803         112 :     const APInt &Val = SrcMO.getCImm()->getValue().sext(WideTy.getSizeInBits());
     804          56 :     SrcMO.setCImm(ConstantInt::get(Ctx, Val));
     805             : 
     806          56 :     widenScalarDst(MI, WideTy);
     807          56 :     MIRBuilder.recordInsertion(&MI);
     808             :     return Legalized;
     809             :   }
     810           1 :   case TargetOpcode::G_FCONSTANT: {
     811           1 :     MachineOperand &SrcMO = MI.getOperand(1);
     812           1 :     LLVMContext &Ctx = MIRBuilder.getMF().getFunction().getContext();
     813           1 :     APFloat Val = SrcMO.getFPImm()->getValueAPF();
     814             :     bool LosesInfo;
     815           1 :     switch (WideTy.getSizeInBits()) {
     816           1 :     case 32:
     817           1 :       Val.convert(APFloat::IEEEsingle(), APFloat::rmTowardZero, &LosesInfo);
     818           1 :       break;
     819           0 :     case 64:
     820           0 :       Val.convert(APFloat::IEEEdouble(), APFloat::rmTowardZero, &LosesInfo);
     821           0 :       break;
     822           0 :     default:
     823           0 :       llvm_unreachable("Unhandled fp widen type");
     824             :     }
     825           1 :     SrcMO.setFPImm(ConstantFP::get(Ctx, Val));
     826             : 
     827           1 :     widenScalarDst(MI, WideTy, 0, TargetOpcode::G_FPTRUNC);
     828           1 :     MIRBuilder.recordInsertion(&MI);
     829             :     return Legalized;
     830             :   }
     831           0 :   case TargetOpcode::G_BRCOND:
     832           0 :     widenScalarSrc(MI, WideTy, 0, TargetOpcode::G_ANYEXT);
     833           0 :     MIRBuilder.recordInsertion(&MI);
     834           0 :     return Legalized;
     835             : 
     836          28 :   case TargetOpcode::G_FCMP:
     837          28 :     if (TypeIdx == 0)
     838          28 :       widenScalarDst(MI, WideTy);
     839             :     else {
     840           0 :       widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_FPEXT);
     841           0 :       widenScalarSrc(MI, WideTy, 3, TargetOpcode::G_FPEXT);
     842             :     }
     843          28 :     MIRBuilder.recordInsertion(&MI);
     844          28 :     return Legalized;
     845             : 
     846          42 :   case TargetOpcode::G_ICMP:
     847          42 :     if (TypeIdx == 0)
     848          38 :       widenScalarDst(MI, WideTy);
     849             :     else {
     850           4 :       unsigned ExtOpcode = CmpInst::isSigned(static_cast<CmpInst::Predicate>(
     851           4 :                                MI.getOperand(1).getPredicate()))
     852           4 :                                ? TargetOpcode::G_SEXT
     853             :                                : TargetOpcode::G_ZEXT;
     854           4 :       widenScalarSrc(MI, WideTy, 2, ExtOpcode);
     855           4 :       widenScalarSrc(MI, WideTy, 3, ExtOpcode);
     856             :     }
     857          42 :     MIRBuilder.recordInsertion(&MI);
     858          42 :     return Legalized;
     859             : 
     860           3 :   case TargetOpcode::G_GEP:
     861             :     assert(TypeIdx == 1 && "unable to legalize pointer of GEP");
     862           3 :     widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_SEXT);
     863           3 :     MIRBuilder.recordInsertion(&MI);
     864           3 :     return Legalized;
     865             : 
     866             :   case TargetOpcode::G_PHI: {
     867             :     assert(TypeIdx == 0 && "Expecting only Idx 0");
     868             : 
     869          30 :     for (unsigned I = 1; I < MI.getNumOperands(); I += 2) {
     870          20 :       MachineBasicBlock &OpMBB = *MI.getOperand(I + 1).getMBB();
     871          20 :       MIRBuilder.setInsertPt(OpMBB, OpMBB.getFirstTerminator());
     872          20 :       widenScalarSrc(MI, WideTy, I, TargetOpcode::G_ANYEXT);
     873             :     }
     874             : 
     875          10 :     MachineBasicBlock &MBB = *MI.getParent();
     876          20 :     MIRBuilder.setInsertPt(MBB, --MBB.getFirstNonPHI());
     877          10 :     widenScalarDst(MI, WideTy);
     878          10 :     MIRBuilder.recordInsertion(&MI);
     879          10 :     return Legalized;
     880             :   }
     881             :   }
     882             : }
     883             : 
     884             : LegalizerHelper::LegalizeResult
     885          46 : LegalizerHelper::lower(MachineInstr &MI, unsigned TypeIdx, LLT Ty) {
     886             :   using namespace TargetOpcode;
     887          46 :   MIRBuilder.setInstr(MI);
     888             : 
     889          92 :   switch(MI.getOpcode()) {
     890             :   default:
     891             :     return UnableToLegalize;
     892          27 :   case TargetOpcode::G_SREM:
     893             :   case TargetOpcode::G_UREM: {
     894          54 :     unsigned QuotReg = MRI.createGenericVirtualRegister(Ty);
     895          67 :     MIRBuilder.buildInstr(MI.getOpcode() == G_SREM ? G_SDIV : G_UDIV)
     896             :         .addDef(QuotReg)
     897          27 :         .addUse(MI.getOperand(1).getReg())
     898          27 :         .addUse(MI.getOperand(2).getReg());
     899             : 
     900          54 :     unsigned ProdReg = MRI.createGenericVirtualRegister(Ty);
     901          27 :     MIRBuilder.buildMul(ProdReg, QuotReg, MI.getOperand(2).getReg());
     902          27 :     MIRBuilder.buildSub(MI.getOperand(0).getReg(), MI.getOperand(1).getReg(),
     903          27 :                         ProdReg);
     904          27 :     MI.eraseFromParent();
     905          27 :     return Legalized;
     906             :   }
     907           2 :   case TargetOpcode::G_SMULO:
     908             :   case TargetOpcode::G_UMULO: {
     909             :     // Generate G_UMULH/G_SMULH to check for overflow and a normal G_MUL for the
     910             :     // result.
     911           2 :     unsigned Res = MI.getOperand(0).getReg();
     912           2 :     unsigned Overflow = MI.getOperand(1).getReg();
     913           2 :     unsigned LHS = MI.getOperand(2).getReg();
     914           2 :     unsigned RHS = MI.getOperand(3).getReg();
     915             : 
     916           2 :     MIRBuilder.buildMul(Res, LHS, RHS);
     917             : 
     918           2 :     unsigned Opcode = MI.getOpcode() == TargetOpcode::G_SMULO
     919           2 :                           ? TargetOpcode::G_SMULH
     920             :                           : TargetOpcode::G_UMULH;
     921             : 
     922           4 :     unsigned HiPart = MRI.createGenericVirtualRegister(Ty);
     923           2 :     MIRBuilder.buildInstr(Opcode)
     924             :       .addDef(HiPart)
     925             :       .addUse(LHS)
     926             :       .addUse(RHS);
     927             : 
     928           4 :     unsigned Zero = MRI.createGenericVirtualRegister(Ty);
     929           2 :     MIRBuilder.buildConstant(Zero, 0);
     930             : 
     931             :     // For *signed* multiply, overflow is detected by checking:
     932             :     // (hi != (lo >> bitwidth-1))
     933           2 :     if (Opcode == TargetOpcode::G_SMULH) {
     934           2 :       unsigned Shifted = MRI.createGenericVirtualRegister(Ty);
     935           2 :       unsigned ShiftAmt = MRI.createGenericVirtualRegister(Ty);
     936           1 :       MIRBuilder.buildConstant(ShiftAmt, Ty.getSizeInBits() - 1);
     937           1 :       MIRBuilder.buildInstr(TargetOpcode::G_ASHR)
     938             :         .addDef(Shifted)
     939             :         .addUse(Res)
     940             :         .addUse(ShiftAmt);
     941           1 :       MIRBuilder.buildICmp(CmpInst::ICMP_NE, Overflow, HiPart, Shifted);
     942             :     } else {
     943           1 :       MIRBuilder.buildICmp(CmpInst::ICMP_NE, Overflow, HiPart, Zero);
     944             :     }
     945           2 :     MI.eraseFromParent();
     946           2 :     return Legalized;
     947             :   }
     948           7 :   case TargetOpcode::G_FNEG: {
     949             :     // TODO: Handle vector types once we are able to
     950             :     // represent them.
     951           7 :     if (Ty.isVector())
     952           0 :       return UnableToLegalize;
     953           7 :     unsigned Res = MI.getOperand(0).getReg();
     954             :     Type *ZeroTy;
     955           7 :     LLVMContext &Ctx = MIRBuilder.getMF().getFunction().getContext();
     956           7 :     switch (Ty.getSizeInBits()) {
     957           0 :     case 16:
     958           0 :       ZeroTy = Type::getHalfTy(Ctx);
     959           0 :       break;
     960           3 :     case 32:
     961           3 :       ZeroTy = Type::getFloatTy(Ctx);
     962           3 :       break;
     963           3 :     case 64:
     964           3 :       ZeroTy = Type::getDoubleTy(Ctx);
     965           3 :       break;
     966           1 :     case 128:
     967           1 :       ZeroTy = Type::getFP128Ty(Ctx);
     968           1 :       break;
     969           0 :     default:
     970           0 :       llvm_unreachable("unexpected floating-point type");
     971             :     }
     972             :     ConstantFP &ZeroForNegation =
     973           7 :         *cast<ConstantFP>(ConstantFP::getZeroValueForNegation(ZeroTy));
     974           7 :     auto Zero = MIRBuilder.buildFConstant(Ty, ZeroForNegation);
     975           7 :     MIRBuilder.buildInstr(TargetOpcode::G_FSUB)
     976             :         .addDef(Res)
     977           7 :         .addUse(Zero->getOperand(0).getReg())
     978           7 :         .addUse(MI.getOperand(1).getReg());
     979           7 :     MI.eraseFromParent();
     980           7 :     return Legalized;
     981             :   }
     982           0 :   case TargetOpcode::G_FSUB: {
     983             :     // Lower (G_FSUB LHS, RHS) to (G_FADD LHS, (G_FNEG RHS)).
     984             :     // First, check if G_FNEG is marked as Lower. If so, we may
     985             :     // end up with an infinite loop as G_FSUB is used to legalize G_FNEG.
     986           0 :     if (LI.getAction({G_FNEG, {Ty}}).Action == Lower)
     987             :       return UnableToLegalize;
     988           0 :     unsigned Res = MI.getOperand(0).getReg();
     989           0 :     unsigned LHS = MI.getOperand(1).getReg();
     990           0 :     unsigned RHS = MI.getOperand(2).getReg();
     991           0 :     unsigned Neg = MRI.createGenericVirtualRegister(Ty);
     992           0 :     MIRBuilder.buildInstr(TargetOpcode::G_FNEG).addDef(Neg).addUse(RHS);
     993           0 :     MIRBuilder.buildInstr(TargetOpcode::G_FADD)
     994             :         .addDef(Res)
     995             :         .addUse(LHS)
     996             :         .addUse(Neg);
     997           0 :     MI.eraseFromParent();
     998           0 :     return Legalized;
     999             :   }
    1000           2 :   case TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS: {
    1001           2 :     unsigned OldValRes = MI.getOperand(0).getReg();
    1002           2 :     unsigned SuccessRes = MI.getOperand(1).getReg();
    1003           2 :     unsigned Addr = MI.getOperand(2).getReg();
    1004           2 :     unsigned CmpVal = MI.getOperand(3).getReg();
    1005           2 :     unsigned NewVal = MI.getOperand(4).getReg();
    1006             :     MIRBuilder.buildAtomicCmpXchg(OldValRes, Addr, CmpVal, NewVal,
    1007           2 :                                   **MI.memoperands_begin());
    1008           2 :     MIRBuilder.buildICmp(CmpInst::ICMP_EQ, SuccessRes, OldValRes, CmpVal);
    1009           2 :     MI.eraseFromParent();
    1010           2 :     return Legalized;
    1011             :   }
    1012           2 :   case TargetOpcode::G_LOAD:
    1013             :   case TargetOpcode::G_SEXTLOAD:
    1014             :   case TargetOpcode::G_ZEXTLOAD: {
    1015             :     // Lower to a memory-width G_LOAD and a G_SEXT/G_ZEXT/G_ANYEXT
    1016           2 :     unsigned DstReg = MI.getOperand(0).getReg();
    1017           2 :     unsigned PtrReg = MI.getOperand(1).getReg();
    1018           4 :     LLT DstTy = MRI.getType(DstReg);
    1019           2 :     auto &MMO = **MI.memoperands_begin();
    1020             : 
    1021           2 :     if (DstTy.getSizeInBits() == MMO.getSize() /* in bytes */ * 8) {
    1022             :       // In the case of G_LOAD, this was a non-extending load already and we're
    1023             :       // about to lower to the same instruction.
    1024           2 :       if (MI.getOpcode() == TargetOpcode::G_LOAD)
    1025             :           return UnableToLegalize;
    1026           0 :       MIRBuilder.buildLoad(DstReg, PtrReg, MMO);
    1027           0 :       MI.eraseFromParent();
    1028           0 :       return Legalized;
    1029             :     }
    1030             : 
    1031           0 :     if (DstTy.isScalar()) {
    1032           3 :       unsigned TmpReg = MRI.createGenericVirtualRegister(
    1033             :           LLT::scalar(MMO.getSize() /* in bytes */ * 8));
    1034           1 :       MIRBuilder.buildLoad(TmpReg, PtrReg, MMO);
    1035           1 :       switch (MI.getOpcode()) {
    1036           0 :       default:
    1037           0 :         llvm_unreachable("Unexpected opcode");
    1038           0 :       case TargetOpcode::G_LOAD:
    1039           0 :         MIRBuilder.buildAnyExt(DstReg, TmpReg);
    1040           0 :         break;
    1041           0 :       case TargetOpcode::G_SEXTLOAD:
    1042           0 :         MIRBuilder.buildSExt(DstReg, TmpReg);
    1043           0 :         break;
    1044           1 :       case TargetOpcode::G_ZEXTLOAD:
    1045           1 :         MIRBuilder.buildZExt(DstReg, TmpReg);
    1046           1 :         break;
    1047             :       }
    1048           1 :       MI.eraseFromParent();
    1049           1 :       return Legalized;
    1050             :     }
    1051             : 
    1052             :     return UnableToLegalize;
    1053             :   }
    1054           6 :   case TargetOpcode::G_CTLZ_ZERO_UNDEF:
    1055             :   case TargetOpcode::G_CTTZ_ZERO_UNDEF:
    1056             :   case TargetOpcode::G_CTLZ:
    1057             :   case TargetOpcode::G_CTTZ:
    1058             :   case TargetOpcode::G_CTPOP:
    1059           6 :     return lowerBitCount(MI, TypeIdx, Ty);
    1060             :   }
    1061             : }
    1062             : 
    1063             : LegalizerHelper::LegalizeResult
    1064          32 : LegalizerHelper::fewerElementsVector(MachineInstr &MI, unsigned TypeIdx,
    1065             :                                      LLT NarrowTy) {
    1066             :   // FIXME: Don't know how to handle secondary types yet.
    1067          32 :   if (TypeIdx != 0)
    1068             :     return UnableToLegalize;
    1069          64 :   switch (MI.getOpcode()) {
    1070             :   default:
    1071             :     return UnableToLegalize;
    1072          32 :   case TargetOpcode::G_ADD: {
    1073          32 :     unsigned NarrowSize = NarrowTy.getSizeInBits();
    1074          32 :     unsigned DstReg = MI.getOperand(0).getReg();
    1075          32 :     unsigned Size = MRI.getType(DstReg).getSizeInBits();
    1076          32 :     int NumParts = Size / NarrowSize;
    1077             :     // FIXME: Don't know how to handle the situation where the small vectors
    1078             :     // aren't all the same size yet.
    1079          32 :     if (Size % NarrowSize != 0)
    1080             :       return UnableToLegalize;
    1081             : 
    1082          32 :     MIRBuilder.setInstr(MI);
    1083             : 
    1084             :     SmallVector<unsigned, 2> Src1Regs, Src2Regs, DstRegs;
    1085          32 :     extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, Src1Regs);
    1086          32 :     extractParts(MI.getOperand(2).getReg(), NarrowTy, NumParts, Src2Regs);
    1087             : 
    1088         115 :     for (int i = 0; i < NumParts; ++i) {
    1089         166 :       unsigned DstReg = MRI.createGenericVirtualRegister(NarrowTy);
    1090         249 :       MIRBuilder.buildAdd(DstReg, Src1Regs[i], Src2Regs[i]);
    1091          83 :       DstRegs.push_back(DstReg);
    1092             :     }
    1093             : 
    1094          32 :     MIRBuilder.buildMerge(DstReg, DstRegs);
    1095          32 :     MI.eraseFromParent();
    1096             :     return Legalized;
    1097             :   }
    1098             :   }
    1099             : }
    1100             : 
    1101             : LegalizerHelper::LegalizeResult
    1102           6 : LegalizerHelper::lowerBitCount(MachineInstr &MI, unsigned TypeIdx, LLT Ty) {
    1103           6 :   unsigned Opc = MI.getOpcode();
    1104           6 :   auto &TII = *MI.getMF()->getSubtarget().getInstrInfo();
    1105             :   auto isLegalOrCustom = [this](const LegalityQuery &Q) {
    1106           8 :     auto QAction = LI.getAction(Q).Action;
    1107           8 :     return QAction == Legal || QAction == Custom;
    1108             :   };
    1109           6 :   switch (Opc) {
    1110             :   default:
    1111             :     return UnableToLegalize;
    1112           0 :   case TargetOpcode::G_CTLZ_ZERO_UNDEF: {
    1113             :     // This trivially expands to CTLZ.
    1114           0 :     MI.setDesc(TII.get(TargetOpcode::G_CTLZ));
    1115           0 :     MIRBuilder.recordInsertion(&MI);
    1116           0 :     return Legalized;
    1117             :   }
    1118           2 :   case TargetOpcode::G_CTLZ: {
    1119           2 :     unsigned SrcReg = MI.getOperand(1).getReg();
    1120           2 :     unsigned Len = Ty.getSizeInBits();
    1121           2 :     if (isLegalOrCustom({TargetOpcode::G_CTLZ_ZERO_UNDEF, {Ty}})) {
    1122             :       // If CTLZ_ZERO_UNDEF is legal or custom, emit that and a select with
    1123             :       // zero.
    1124             :       auto MIBCtlzZU =
    1125           1 :           MIRBuilder.buildInstr(TargetOpcode::G_CTLZ_ZERO_UNDEF, Ty, SrcReg);
    1126           1 :       auto MIBZero = MIRBuilder.buildConstant(Ty, 0);
    1127           1 :       auto MIBLen = MIRBuilder.buildConstant(Ty, Len);
    1128           2 :       auto MIBICmp = MIRBuilder.buildICmp(CmpInst::ICMP_EQ, LLT::scalar(1),
    1129           1 :                                           SrcReg, MIBZero);
    1130           1 :       MIRBuilder.buildSelect(MI.getOperand(0).getReg(), MIBICmp, MIBLen,
    1131           1 :                              MIBCtlzZU);
    1132           1 :       MI.eraseFromParent();
    1133             :       return Legalized;
    1134             :     }
    1135             :     // for now, we do this:
    1136             :     // NewLen = NextPowerOf2(Len);
    1137             :     // x = x | (x >> 1);
    1138             :     // x = x | (x >> 2);
    1139             :     // ...
    1140             :     // x = x | (x >>16);
    1141             :     // x = x | (x >>32); // for 64-bit input
    1142             :     // Upto NewLen/2
    1143             :     // return Len - popcount(x);
    1144             :     //
    1145             :     // Ref: "Hacker's Delight" by Henry Warren
    1146           1 :     unsigned Op = SrcReg;
    1147           2 :     unsigned NewLen = PowerOf2Ceil(Len);
    1148           4 :     for (unsigned i = 0; (1U << i) <= (NewLen / 2); ++i) {
    1149           3 :       auto MIBShiftAmt = MIRBuilder.buildConstant(Ty, 1ULL << i);
    1150             :       auto MIBOp = MIRBuilder.buildInstr(
    1151             :           TargetOpcode::G_OR, Ty, Op,
    1152           3 :           MIRBuilder.buildInstr(TargetOpcode::G_LSHR, Ty, Op, MIBShiftAmt));
    1153           3 :       Op = MIBOp->getOperand(0).getReg();
    1154             :     }
    1155           1 :     auto MIBPop = MIRBuilder.buildInstr(TargetOpcode::G_CTPOP, Ty, Op);
    1156           2 :     MIRBuilder.buildInstr(TargetOpcode::G_SUB, MI.getOperand(0).getReg(),
    1157           1 :                           MIRBuilder.buildConstant(Ty, Len), MIBPop);
    1158           1 :     MI.eraseFromParent();
    1159           1 :     return Legalized;
    1160             :   }
    1161           1 :   case TargetOpcode::G_CTTZ_ZERO_UNDEF: {
    1162             :     // This trivially expands to CTTZ.
    1163           1 :     MI.setDesc(TII.get(TargetOpcode::G_CTTZ));
    1164           1 :     MIRBuilder.recordInsertion(&MI);
    1165           1 :     return Legalized;
    1166             :   }
    1167           3 :   case TargetOpcode::G_CTTZ: {
    1168           3 :     unsigned SrcReg = MI.getOperand(1).getReg();
    1169           3 :     unsigned Len = Ty.getSizeInBits();
    1170           3 :     if (isLegalOrCustom({TargetOpcode::G_CTTZ_ZERO_UNDEF, {Ty}})) {
    1171             :       // If CTTZ_ZERO_UNDEF is legal or custom, emit that and a select with
    1172             :       // zero.
    1173             :       auto MIBCttzZU =
    1174           1 :           MIRBuilder.buildInstr(TargetOpcode::G_CTTZ_ZERO_UNDEF, Ty, SrcReg);
    1175           1 :       auto MIBZero = MIRBuilder.buildConstant(Ty, 0);
    1176           1 :       auto MIBLen = MIRBuilder.buildConstant(Ty, Len);
    1177           2 :       auto MIBICmp = MIRBuilder.buildICmp(CmpInst::ICMP_EQ, LLT::scalar(1),
    1178           1 :                                           SrcReg, MIBZero);
    1179           1 :       MIRBuilder.buildSelect(MI.getOperand(0).getReg(), MIBICmp, MIBLen,
    1180           1 :                              MIBCttzZU);
    1181           1 :       MI.eraseFromParent();
    1182             :       return Legalized;
    1183             :     }
    1184             :     // for now, we use: { return popcount(~x & (x - 1)); }
    1185             :     // unless the target has ctlz but not ctpop, in which case we use:
    1186             :     // { return 32 - nlz(~x & (x-1)); }
    1187             :     // Ref: "Hacker's Delight" by Henry Warren
    1188           2 :     auto MIBCstNeg1 = MIRBuilder.buildConstant(Ty, -1);
    1189             :     auto MIBNot =
    1190           2 :         MIRBuilder.buildInstr(TargetOpcode::G_XOR, Ty, SrcReg, MIBCstNeg1);
    1191             :     auto MIBTmp = MIRBuilder.buildInstr(
    1192             :         TargetOpcode::G_AND, Ty, MIBNot,
    1193           2 :         MIRBuilder.buildInstr(TargetOpcode::G_ADD, Ty, SrcReg, MIBCstNeg1));
    1194           3 :     if (!isLegalOrCustom({TargetOpcode::G_CTPOP, {Ty}}) &&
    1195           1 :         isLegalOrCustom({TargetOpcode::G_CTLZ, {Ty}})) {
    1196           1 :       auto MIBCstLen = MIRBuilder.buildConstant(Ty, Len);
    1197             :       MIRBuilder.buildInstr(
    1198           2 :           TargetOpcode::G_SUB, MI.getOperand(0).getReg(),
    1199             :           MIBCstLen,
    1200           1 :           MIRBuilder.buildInstr(TargetOpcode::G_CTLZ, Ty, MIBTmp));
    1201           1 :       MI.eraseFromParent();
    1202             :       return Legalized;
    1203             :     }
    1204           1 :     MI.setDesc(TII.get(TargetOpcode::G_CTPOP));
    1205           2 :     MI.getOperand(1).setReg(MIBTmp->getOperand(0).getReg());
    1206           1 :     return Legalized;
    1207             :   }
    1208             :   }
    1209             : }

Generated by: LCOV version 1.13