LCOV - code coverage report
Current view: top level - lib/Target/AArch64 - AArch64InstructionSelector.cpp (source / functions) Hit Total Coverage
Test: llvm-toolchain.info Lines: 503 689 73.0 %
Date: 2018-10-20 13:21:21 Functions: 21 29 72.4 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : //===- AArch64InstructionSelector.cpp ----------------------------*- C++ -*-==//
       2             : //
       3             : //                     The LLVM Compiler Infrastructure
       4             : //
       5             : // This file is distributed under the University of Illinois Open Source
       6             : // License. See LICENSE.TXT for details.
       7             : //
       8             : //===----------------------------------------------------------------------===//
       9             : /// \file
      10             : /// This file implements the targeting of the InstructionSelector class for
      11             : /// AArch64.
      12             : /// \todo This should be generated by TableGen.
      13             : //===----------------------------------------------------------------------===//
      14             : 
      15             : #include "AArch64InstrInfo.h"
      16             : #include "AArch64MachineFunctionInfo.h"
      17             : #include "AArch64RegisterBankInfo.h"
      18             : #include "AArch64RegisterInfo.h"
      19             : #include "AArch64Subtarget.h"
      20             : #include "AArch64TargetMachine.h"
      21             : #include "MCTargetDesc/AArch64AddressingModes.h"
      22             : #include "llvm/CodeGen/GlobalISel/InstructionSelector.h"
      23             : #include "llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h"
      24             : #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
      25             : #include "llvm/CodeGen/GlobalISel/Utils.h"
      26             : #include "llvm/CodeGen/MachineBasicBlock.h"
      27             : #include "llvm/CodeGen/MachineFunction.h"
      28             : #include "llvm/CodeGen/MachineInstr.h"
      29             : #include "llvm/CodeGen/MachineInstrBuilder.h"
      30             : #include "llvm/CodeGen/MachineOperand.h"
      31             : #include "llvm/CodeGen/MachineRegisterInfo.h"
      32             : #include "llvm/IR/Type.h"
      33             : #include "llvm/Support/Debug.h"
      34             : #include "llvm/Support/raw_ostream.h"
      35             : 
      36             : #define DEBUG_TYPE "aarch64-isel"
      37             : 
      38             : using namespace llvm;
      39             : 
      40             : namespace {
      41             : 
      42             : #define GET_GLOBALISEL_PREDICATE_BITSET
      43             : #include "AArch64GenGlobalISel.inc"
      44             : #undef GET_GLOBALISEL_PREDICATE_BITSET
      45             : 
      46             : class AArch64InstructionSelector : public InstructionSelector {
      47             : public:
      48             :   AArch64InstructionSelector(const AArch64TargetMachine &TM,
      49             :                              const AArch64Subtarget &STI,
      50             :                              const AArch64RegisterBankInfo &RBI);
      51             : 
      52             :   bool select(MachineInstr &I, CodeGenCoverage &CoverageInfo) const override;
      53             :   static const char *getName() { return DEBUG_TYPE; }
      54             : 
      55             : private:
      56             :   /// tblgen-erated 'select' implementation, used as the initial selector for
      57             :   /// the patterns that don't require complex C++.
      58             :   bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const;
      59             : 
      60             :   bool selectVaStartAAPCS(MachineInstr &I, MachineFunction &MF,
      61             :                           MachineRegisterInfo &MRI) const;
      62             :   bool selectVaStartDarwin(MachineInstr &I, MachineFunction &MF,
      63             :                            MachineRegisterInfo &MRI) const;
      64             : 
      65             :   bool selectCompareBranch(MachineInstr &I, MachineFunction &MF,
      66             :                            MachineRegisterInfo &MRI) const;
      67             : 
      68             :   ComplexRendererFns selectArithImmed(MachineOperand &Root) const;
      69             : 
      70             :   ComplexRendererFns selectAddrModeUnscaled(MachineOperand &Root,
      71             :                                             unsigned Size) const;
      72             : 
      73           0 :   ComplexRendererFns selectAddrModeUnscaled8(MachineOperand &Root) const {
      74           0 :     return selectAddrModeUnscaled(Root, 1);
      75             :   }
      76           0 :   ComplexRendererFns selectAddrModeUnscaled16(MachineOperand &Root) const {
      77           0 :     return selectAddrModeUnscaled(Root, 2);
      78             :   }
      79           0 :   ComplexRendererFns selectAddrModeUnscaled32(MachineOperand &Root) const {
      80           0 :     return selectAddrModeUnscaled(Root, 4);
      81             :   }
      82           0 :   ComplexRendererFns selectAddrModeUnscaled64(MachineOperand &Root) const {
      83           0 :     return selectAddrModeUnscaled(Root, 8);
      84             :   }
      85           0 :   ComplexRendererFns selectAddrModeUnscaled128(MachineOperand &Root) const {
      86           0 :     return selectAddrModeUnscaled(Root, 16);
      87             :   }
      88             : 
      89             :   ComplexRendererFns selectAddrModeIndexed(MachineOperand &Root,
      90             :                                            unsigned Size) const;
      91             :   template <int Width>
      92         297 :   ComplexRendererFns selectAddrModeIndexed(MachineOperand &Root) const {
      93         297 :     return selectAddrModeIndexed(Root, Width / 8);
      94             :   }
      95           4 : 
      96           4 :   void renderTruncImm(MachineInstrBuilder &MIB, const MachineInstr &MI) const;
      97             : 
      98         218 :   // Materialize a GlobalValue or BlockAddress using a movz+movk sequence.
      99         218 :   void materializeLargeCMVal(MachineInstr &I, const Value *V,
     100             :                              unsigned char OpFlags) const;
     101          58 : 
     102          58 :   const AArch64TargetMachine &TM;
     103             :   const AArch64Subtarget &STI;
     104           7 :   const AArch64InstrInfo &TII;
     105           7 :   const AArch64RegisterInfo &TRI;
     106             :   const AArch64RegisterBankInfo &RBI;
     107          10 : 
     108          10 : #define GET_GLOBALISEL_PREDICATES_DECL
     109             : #include "AArch64GenGlobalISel.inc"
     110             : #undef GET_GLOBALISEL_PREDICATES_DECL
     111             : 
     112             : // We declare the temporaries used by selectImpl() in the class to minimize the
     113             : // cost of constructing placeholder values.
     114             : #define GET_GLOBALISEL_TEMPORARIES_DECL
     115             : #include "AArch64GenGlobalISel.inc"
     116             : #undef GET_GLOBALISEL_TEMPORARIES_DECL
     117             : };
     118             : 
     119             : } // end anonymous namespace
     120             : 
     121             : #define GET_GLOBALISEL_IMPL
     122             : #include "AArch64GenGlobalISel.inc"
     123             : #undef GET_GLOBALISEL_IMPL
     124             : 
     125             : AArch64InstructionSelector::AArch64InstructionSelector(
     126             :     const AArch64TargetMachine &TM, const AArch64Subtarget &STI,
     127             :     const AArch64RegisterBankInfo &RBI)
     128             :     : InstructionSelector(), TM(TM), STI(STI), TII(*STI.getInstrInfo()),
     129             :       TRI(*STI.getRegisterInfo()), RBI(RBI),
     130             : #define GET_GLOBALISEL_PREDICATES_INIT
     131             : #include "AArch64GenGlobalISel.inc"
     132             : #undef GET_GLOBALISEL_PREDICATES_INIT
     133             : #define GET_GLOBALISEL_TEMPORARIES_INIT
     134             : #include "AArch64GenGlobalISel.inc"
     135             : #undef GET_GLOBALISEL_TEMPORARIES_INIT
     136             : {
     137             : }
     138             : 
     139             : // FIXME: This should be target-independent, inferred from the types declared
     140        1570 : // for each class in the bank.
     141             : static const TargetRegisterClass *
     142        1570 : getRegClassForTypeOnBank(LLT Ty, const RegisterBank &RB,
     143             :                          const RegisterBankInfo &RBI,
     144             :                          bool GetAllRegSet = false) {
     145             :   if (RB.getID() == AArch64::GPRRegBankID) {
     146             :     if (Ty.getSizeInBits() <= 32)
     147             :       return GetAllRegSet ? &AArch64::GPR32allRegClass
     148             :                           : &AArch64::GPR32RegClass;
     149             :     if (Ty.getSizeInBits() == 64)
     150             :       return GetAllRegSet ? &AArch64::GPR64allRegClass
     151             :                           : &AArch64::GPR64RegClass;
     152        1570 :     return nullptr;
     153             :   }
     154             : 
     155             :   if (RB.getID() == AArch64::FPRRegBankID) {
     156             :     if (Ty.getSizeInBits() <= 16)
     157         909 :       return &AArch64::FPR16RegClass;
     158             :     if (Ty.getSizeInBits() == 32)
     159             :       return &AArch64::FPR32RegClass;
     160         909 :     if (Ty.getSizeInBits() == 64)
     161         534 :       return &AArch64::FPR64RegClass;
     162         293 :     if (Ty.getSizeInBits() == 128)
     163         293 :       return &AArch64::FPR128RegClass;
     164         241 :     return nullptr;
     165         241 :   }
     166         241 : 
     167             :   return nullptr;
     168             : }
     169             : 
     170         375 : /// Check whether \p I is a currently unsupported binary operation:
     171         375 : /// - it has an unsized type
     172             : /// - an operand is not a vreg
     173         363 : /// - all operands are not in the same bank
     174             : /// These are checks that should someday live in the verifier, but right now,
     175         303 : /// these are mostly limitations of the aarch64 selector.
     176             : static bool unsupportedBinOp(const MachineInstr &I,
     177         116 :                              const AArch64RegisterBankInfo &RBI,
     178             :                              const MachineRegisterInfo &MRI,
     179           0 :                              const AArch64RegisterInfo &TRI) {
     180             :   LLT Ty = MRI.getType(I.getOperand(0).getReg());
     181             :   if (!Ty.isValid()) {
     182             :     LLVM_DEBUG(dbgs() << "Generic binop register should be typed\n");
     183             :     return true;
     184             :   }
     185             : 
     186             :   const RegisterBank *PrevOpBank = nullptr;
     187             :   for (auto &MO : I.operands()) {
     188             :     // FIXME: Support non-register operands.
     189             :     if (!MO.isReg()) {
     190             :       LLVM_DEBUG(dbgs() << "Generic inst non-reg operands are unsupported\n");
     191          16 :       return true;
     192             :     }
     193             : 
     194             :     // FIXME: Can generic operations have physical registers operands? If
     195          16 :     // so, this will need to be taught about that, and we'll need to get the
     196          16 :     // bank out of the minimal class for the register.
     197             :     // Either way, this needs to be documented (and possibly verified).
     198             :     if (!TargetRegisterInfo::isVirtualRegister(MO.getReg())) {
     199             :       LLVM_DEBUG(dbgs() << "Generic inst has physical register operand\n");
     200             :       return true;
     201             :     }
     202          64 : 
     203             :     const RegisterBank *OpBank = RBI.getRegBank(MO.getReg(), MRI, TRI);
     204          48 :     if (!OpBank) {
     205             :       LLVM_DEBUG(dbgs() << "Generic register has no bank or class\n");
     206             :       return true;
     207             :     }
     208             : 
     209             :     if (PrevOpBank && OpBank != PrevOpBank) {
     210             :       LLVM_DEBUG(dbgs() << "Generic inst operands have different banks\n");
     211             :       return true;
     212             :     }
     213          96 :     PrevOpBank = OpBank;
     214             :   }
     215             :   return false;
     216             : }
     217             : 
     218          48 : /// Select the AArch64 opcode for the basic binary operation \p GenericOpc
     219          48 : /// (such as G_OR or G_SDIV), appropriate for the register bank \p RegBankID
     220             : /// and of size \p OpSize.
     221             : /// \returns \p GenericOpc if the combination is unsupported.
     222             : static unsigned selectBinaryOp(unsigned GenericOpc, unsigned RegBankID,
     223             :                                unsigned OpSize) {
     224          48 :   switch (RegBankID) {
     225             :   case AArch64::GPRRegBankID:
     226             :     if (OpSize == 32) {
     227             :       switch (GenericOpc) {
     228             :       case TargetOpcode::G_SHL:
     229             :         return AArch64::LSLVWr;
     230             :       case TargetOpcode::G_LSHR:
     231             :         return AArch64::LSRVWr;
     232             :       case TargetOpcode::G_ASHR:
     233             :         return AArch64::ASRVWr;
     234             :       default:
     235             :         return GenericOpc;
     236             :       }
     237          14 :     } else if (OpSize == 64) {
     238             :       switch (GenericOpc) {
     239          14 :       case TargetOpcode::G_GEP:
     240          14 :         return AArch64::ADDXrr;
     241          14 :       case TargetOpcode::G_SHL:
     242           4 :         return AArch64::LSLVXr;
     243             :       case TargetOpcode::G_LSHR:
     244             :         return AArch64::LSRVXr;
     245           1 :       case TargetOpcode::G_ASHR:
     246           1 :         return AArch64::ASRVXr;
     247           1 :       default:
     248           1 :         return GenericOpc;
     249           0 :       }
     250           0 :     }
     251             :     break;
     252          10 :   case AArch64::FPRRegBankID:
     253          10 :     switch (OpSize) {
     254             :     case 32:
     255             :       switch (GenericOpc) {
     256           0 :       case TargetOpcode::G_FADD:
     257           0 :         return AArch64::FADDSrr;
     258           0 :       case TargetOpcode::G_FSUB:
     259           0 :         return AArch64::FSUBSrr;
     260           0 :       case TargetOpcode::G_FMUL:
     261           0 :         return AArch64::FMULSrr;
     262           0 :       case TargetOpcode::G_FDIV:
     263           0 :         return AArch64::FDIVSrr;
     264             :       default:
     265             :         return GenericOpc;
     266             :       }
     267           0 :     case 64:
     268             :       switch (GenericOpc) {
     269           0 :       case TargetOpcode::G_FADD:
     270           0 :         return AArch64::FADDDrr;
     271             :       case TargetOpcode::G_FSUB:
     272             :         return AArch64::FSUBDrr;
     273           0 :       case TargetOpcode::G_FMUL:
     274           0 :         return AArch64::FMULDrr;
     275           0 :       case TargetOpcode::G_FDIV:
     276           0 :         return AArch64::FDIVDrr;
     277           0 :       case TargetOpcode::G_OR:
     278           0 :         return AArch64::ORRv8i8;
     279           0 :       default:
     280           0 :         return GenericOpc;
     281             :       }
     282           0 :     }
     283           0 :     break;
     284             :   }
     285             :   return GenericOpc;
     286           0 : }
     287           0 : 
     288           0 : /// Select the AArch64 opcode for the G_LOAD or G_STORE operation \p GenericOpc,
     289           0 : /// appropriate for the (value) register bank \p RegBankID and of memory access
     290           0 : /// size \p OpSize.  This returns the variant with the base+unsigned-immediate
     291           0 : /// addressing mode (e.g., LDRXui).
     292           0 : /// \returns \p GenericOpc if the combination is unsupported.
     293           0 : static unsigned selectLoadStoreUIOp(unsigned GenericOpc, unsigned RegBankID,
     294           0 :                                     unsigned OpSize) {
     295           0 :   const bool isStore = GenericOpc == TargetOpcode::G_STORE;
     296             :   switch (RegBankID) {
     297             :   case AArch64::GPRRegBankID:
     298             :     switch (OpSize) {
     299             :     case 8:
     300             :       return isStore ? AArch64::STRBBui : AArch64::LDRBBui;
     301             :     case 16:
     302             :       return isStore ? AArch64::STRHHui : AArch64::LDRHHui;
     303             :     case 32:
     304             :       return isStore ? AArch64::STRWui : AArch64::LDRWui;
     305             :     case 64:
     306             :       return isStore ? AArch64::STRXui : AArch64::LDRXui;
     307             :     }
     308          76 :     break;
     309             :   case AArch64::FPRRegBankID:
     310             :     switch (OpSize) {
     311          76 :     case 8:
     312          73 :       return isStore ? AArch64::STRBui : AArch64::LDRBui;
     313             :     case 16:
     314          45 :       return isStore ? AArch64::STRHui : AArch64::LDRHui;
     315          45 :     case 32:
     316           7 :       return isStore ? AArch64::STRSui : AArch64::LDRSui;
     317           7 :     case 64:
     318           0 :       return isStore ? AArch64::STRDui : AArch64::LDRDui;
     319           0 :     }
     320          21 :     break;
     321          21 :   }
     322             :   return GenericOpc;
     323             : }
     324           3 : 
     325             : static bool selectFP16CopyFromGPR32(MachineInstr &I, const TargetInstrInfo &TII,
     326           3 :                                     MachineRegisterInfo &MRI, unsigned SrcReg) {
     327           3 :   // Copies from gpr32 to fpr16 need to use a sub-register copy.
     328           0 :   unsigned CopyReg = MRI.createVirtualRegister(&AArch64::FPR32RegClass);
     329           0 :   BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(AArch64::COPY))
     330           0 :       .addDef(CopyReg)
     331           0 :       .addUse(SrcReg);
     332           0 :   unsigned SubRegCopy = MRI.createVirtualRegister(&AArch64::FPR16RegClass);
     333           0 :   BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(TargetOpcode::COPY))
     334             :       .addDef(SubRegCopy)
     335             :       .addUse(CopyReg, 0, AArch64::hsub);
     336             : 
     337             :   MachineOperand &RegOp = I.getOperand(1);
     338             :   RegOp.setReg(SubRegCopy);
     339             :   return true;
     340           4 : }
     341             : 
     342             : static bool selectCopy(MachineInstr &I, const TargetInstrInfo &TII,
     343           4 :                        MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI,
     344          12 :                        const RegisterBankInfo &RBI) {
     345             : 
     346             :   unsigned DstReg = I.getOperand(0).getReg();
     347           4 :   unsigned SrcReg = I.getOperand(1).getReg();
     348           8 : 
     349             :   if (TargetRegisterInfo::isPhysicalRegister(DstReg)) {
     350             :     if (TRI.getRegClass(AArch64::FPR16RegClassID)->contains(DstReg) &&
     351             :         !TargetRegisterInfo::isPhysicalRegister(SrcReg)) {
     352           4 :       const RegisterBank &RegBank = *RBI.getRegBank(SrcReg, MRI, TRI);
     353           4 :       const TargetRegisterClass *SrcRC = getRegClassForTypeOnBank(
     354           4 :           MRI.getType(SrcReg), RegBank, RBI, /* GetAllRegSet */ true);
     355             :       if (SrcRC == &AArch64::GPR32allRegClass)
     356             :         return selectFP16CopyFromGPR32(I, TII, MRI, SrcReg);
     357         955 :     }
     358             :     assert(I.isCopy() && "Generic operators do not allow physical registers");
     359             :     return true;
     360             :   }
     361         955 : 
     362         955 :   const RegisterBank &RegBank = *RBI.getRegBank(DstReg, MRI, TRI);
     363             :   const unsigned DstSize = MRI.getType(DstReg).getSizeInBits();
     364         955 :   (void)DstSize;
     365         524 :   const unsigned SrcSize = RBI.getSizeInBits(SrcReg, MRI, TRI);
     366             :   (void)SrcSize;
     367           8 :   assert((!TargetRegisterInfo::isPhysicalRegister(SrcReg) || I.isCopy()) &&
     368           8 :          "No phys reg on generic operators");
     369             :   assert(
     370           8 :       (DstSize == SrcSize ||
     371           3 :        // Copies are a mean to setup initial types, the number of
     372             :        // bits may not exactly match.
     373             :        (TargetRegisterInfo::isPhysicalRegister(SrcReg) &&
     374         259 :         DstSize <= RBI.getSizeInBits(SrcReg, MRI, TRI)) ||
     375             :        // Copies are a mean to copy bits around, as long as we are
     376             :        // on the same register class, that's fine. Otherwise, that
     377         693 :        // means we need some SUBREG_TO_REG or AND & co.
     378         693 :        (((DstSize + 31) / 32 == (SrcSize + 31) / 32) && DstSize > SrcSize)) &&
     379             :       "Copy with different width?!");
     380         693 :   assert((DstSize <= 64 || RegBank.getID() == AArch64::FPRRegBankID) &&
     381             :          "GPRs cannot get more than 64-bit width values");
     382             : 
     383             :   const TargetRegisterClass *RC = getRegClassForTypeOnBank(
     384             :       MRI.getType(DstReg), RegBank, RBI, /* GetAllRegSet */ true);
     385             :   if (!RC) {
     386             :     LLVM_DEBUG(dbgs() << "Unexpected bitcast size " << DstSize << '\n');
     387             :     return false;
     388             :   }
     389             : 
     390             :   if (!TargetRegisterInfo::isPhysicalRegister(SrcReg)) {
     391             :     const RegClassOrRegBank &RegClassOrBank = MRI.getRegClassOrRegBank(SrcReg);
     392             :     const TargetRegisterClass *SrcRC =
     393             :         RegClassOrBank.dyn_cast<const TargetRegisterClass *>();
     394             :     const RegisterBank *RB = nullptr;
     395             :     if (!SrcRC) {
     396             :       RB = RegClassOrBank.get<const RegisterBank *>();
     397             :       SrcRC = getRegClassForTypeOnBank(MRI.getType(SrcReg), *RB, RBI, true);
     398         693 :     }
     399             :     // Copies from fpr16 to gpr32 need to use SUBREG_TO_REG.
     400         693 :     if (RC == &AArch64::GPR32allRegClass && SrcRC == &AArch64::FPR16RegClass) {
     401             :       unsigned PromoteReg = MRI.createVirtualRegister(&AArch64::FPR32RegClass);
     402             :       BuildMI(*I.getParent(), I, I.getDebugLoc(),
     403             :               TII.get(AArch64::SUBREG_TO_REG))
     404             :           .addDef(PromoteReg)
     405         693 :           .addImm(0)
     406             :           .addUse(SrcReg)
     407             :           .addImm(AArch64::hsub);
     408             :       MachineOperand &RegOp = I.getOperand(1);
     409             :       RegOp.setReg(PromoteReg);
     410           2 :     } else if (RC == &AArch64::FPR16RegClass &&
     411             :                SrcRC == &AArch64::GPR32allRegClass) {
     412          82 :       selectFP16CopyFromGPR32(I, TII, MRI, SrcReg);
     413             :     }
     414             :   }
     415          84 : 
     416           2 :   // No need to constrain SrcReg. It will get constrained when
     417           4 :   // we hit another of its use or its defs.
     418           4 :   // Copies do not have constraints.
     419             :   if (!RBI.constrainGenericRegister(DstReg, *RC, MRI)) {
     420             :     LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())
     421             :                       << " operand\n");
     422             :     return false;
     423           2 :   }
     424           2 :   I.setDesc(TII.get(AArch64::COPY));
     425          82 :   return true;
     426             : }
     427           1 : 
     428             : static unsigned selectFPConvOpc(unsigned GenericOpc, LLT DstTy, LLT SrcTy) {
     429             :   if (!DstTy.isScalar() || !SrcTy.isScalar())
     430             :     return GenericOpc;
     431             : 
     432             :   const unsigned DstSize = DstTy.getSizeInBits();
     433             :   const unsigned SrcSize = SrcTy.getSizeInBits();
     434         693 : 
     435             :   switch (DstSize) {
     436             :   case 32:
     437             :     switch (SrcSize) {
     438             :     case 32:
     439         693 :       switch (GenericOpc) {
     440         693 :       case TargetOpcode::G_SITOFP:
     441             :         return AArch64::SCVTFUWSri;
     442             :       case TargetOpcode::G_UITOFP:
     443           0 :         return AArch64::UCVTFUWSri;
     444           0 :       case TargetOpcode::G_FPTOSI:
     445           0 :         return AArch64::FCVTZSUWSr;
     446             :       case TargetOpcode::G_FPTOUI:
     447           0 :         return AArch64::FCVTZUUWSr;
     448           0 :       default:
     449             :         return GenericOpc;
     450           0 :       }
     451           0 :     case 64:
     452           0 :       switch (GenericOpc) {
     453           0 :       case TargetOpcode::G_SITOFP:
     454             :         return AArch64::SCVTFUXSri;
     455             :       case TargetOpcode::G_UITOFP:
     456             :         return AArch64::UCVTFUXSri;
     457             :       case TargetOpcode::G_FPTOSI:
     458             :         return AArch64::FCVTZSUWDr;
     459             :       case TargetOpcode::G_FPTOUI:
     460             :         return AArch64::FCVTZUUWDr;
     461             :       default:
     462             :         return GenericOpc;
     463             :       }
     464             :     default:
     465             :       return GenericOpc;
     466           0 :     }
     467             :   case 64:
     468             :     switch (SrcSize) {
     469             :     case 32:
     470             :       switch (GenericOpc) {
     471             :       case TargetOpcode::G_SITOFP:
     472             :         return AArch64::SCVTFUWDri;
     473             :       case TargetOpcode::G_UITOFP:
     474             :         return AArch64::UCVTFUWDri;
     475             :       case TargetOpcode::G_FPTOSI:
     476             :         return AArch64::FCVTZSUXSr;
     477             :       case TargetOpcode::G_FPTOUI:
     478             :         return AArch64::FCVTZUUXSr;
     479             :       default:
     480             :         return GenericOpc;
     481             :       }
     482           0 :     case 64:
     483           0 :       switch (GenericOpc) {
     484           0 :       case TargetOpcode::G_SITOFP:
     485             :         return AArch64::SCVTFUXDri;
     486             :       case TargetOpcode::G_UITOFP:
     487             :         return AArch64::UCVTFUXDri;
     488             :       case TargetOpcode::G_FPTOSI:
     489             :         return AArch64::FCVTZSUXDr;
     490             :       case TargetOpcode::G_FPTOUI:
     491             :         return AArch64::FCVTZUUXDr;
     492             :       default:
     493             :         return GenericOpc;
     494             :       }
     495             :     default:
     496             :       return GenericOpc;
     497           0 :     }
     498             :   default:
     499             :     return GenericOpc;
     500             :   };
     501             :   return GenericOpc;
     502             : }
     503             : 
     504             : static AArch64CC::CondCode changeICMPPredToAArch64CC(CmpInst::Predicate P) {
     505             :   switch (P) {
     506             :   default:
     507             :     llvm_unreachable("Unknown condition code!");
     508             :   case CmpInst::ICMP_NE:
     509             :     return AArch64CC::NE;
     510             :   case CmpInst::ICMP_EQ:
     511             :     return AArch64CC::EQ;
     512             :   case CmpInst::ICMP_SGT:
     513             :     return AArch64CC::GT;
     514             :   case CmpInst::ICMP_SGE:
     515             :     return AArch64CC::GE;
     516             :   case CmpInst::ICMP_SLT:
     517             :     return AArch64CC::LT;
     518             :   case CmpInst::ICMP_SLE:
     519             :     return AArch64CC::LE;
     520             :   case CmpInst::ICMP_UGT:
     521           0 :     return AArch64CC::HI;
     522           0 :   case CmpInst::ICMP_UGE:
     523             :     return AArch64CC::HS;
     524             :   case CmpInst::ICMP_ULT:
     525             :     return AArch64CC::LO;
     526             :   case CmpInst::ICMP_ULE:
     527             :     return AArch64CC::LS;
     528             :   }
     529             : }
     530             : 
     531             : static void changeFCMPPredToAArch64CC(CmpInst::Predicate P,
     532             :                                       AArch64CC::CondCode &CondCode,
     533             :                                       AArch64CC::CondCode &CondCode2) {
     534             :   CondCode2 = AArch64CC::AL;
     535             :   switch (P) {
     536             :   default:
     537             :     llvm_unreachable("Unknown FP condition!");
     538             :   case CmpInst::FCMP_OEQ:
     539             :     CondCode = AArch64CC::EQ;
     540             :     break;
     541             :   case CmpInst::FCMP_OGT:
     542             :     CondCode = AArch64CC::GT;
     543             :     break;
     544             :   case CmpInst::FCMP_OGE:
     545             :     CondCode = AArch64CC::GE;
     546           6 :     break;
     547             :   case CmpInst::FCMP_OLT:
     548             :     CondCode = AArch64CC::MI;
     549           6 :     break;
     550           6 :   case CmpInst::FCMP_OLE:
     551           0 :     CondCode = AArch64CC::LS;
     552           0 :     break;
     553           0 :   case CmpInst::FCMP_ONE:
     554           0 :     CondCode = AArch64CC::MI;
     555           0 :     CondCode2 = AArch64CC::GT;
     556           0 :     break;
     557           0 :   case CmpInst::FCMP_ORD:
     558           0 :     CondCode = AArch64CC::VC;
     559           0 :     break;
     560           0 :   case CmpInst::FCMP_UNO:
     561           0 :     CondCode = AArch64CC::VS;
     562           0 :     break;
     563           0 :   case CmpInst::FCMP_UEQ:
     564           0 :     CondCode = AArch64CC::EQ;
     565           0 :     CondCode2 = AArch64CC::VS;
     566           0 :     break;
     567           0 :   case CmpInst::FCMP_UGT:
     568           3 :     CondCode = AArch64CC::HI;
     569           3 :     break;
     570           3 :   case CmpInst::FCMP_UGE:
     571           3 :     CondCode = AArch64CC::PL;
     572           0 :     break;
     573           0 :   case CmpInst::FCMP_ULT:
     574           0 :     CondCode = AArch64CC::LT;
     575           0 :     break;
     576           0 :   case CmpInst::FCMP_ULE:
     577           0 :     CondCode = AArch64CC::LE;
     578           0 :     break;
     579           0 :   case CmpInst::FCMP_UNE:
     580           0 :     CondCode = AArch64CC::NE;
     581           0 :     break;
     582           0 :   }
     583           0 : }
     584           0 : 
     585           3 : bool AArch64InstructionSelector::selectCompareBranch(
     586           3 :     MachineInstr &I, MachineFunction &MF, MachineRegisterInfo &MRI) const {
     587           3 : 
     588           0 :   const unsigned CondReg = I.getOperand(0).getReg();
     589           0 :   MachineBasicBlock *DestMBB = I.getOperand(1).getMBB();
     590           0 :   MachineInstr *CCMI = MRI.getVRegDef(CondReg);
     591           0 :   if (CCMI->getOpcode() == TargetOpcode::G_TRUNC)
     592           0 :     CCMI = MRI.getVRegDef(CCMI->getOperand(1).getReg());
     593           0 :   if (CCMI->getOpcode() != TargetOpcode::G_ICMP)
     594           0 :     return false;
     595           0 : 
     596           0 :   unsigned LHS = CCMI->getOperand(2).getReg();
     597             :   unsigned RHS = CCMI->getOperand(3).getReg();
     598           6 :   if (!getConstantVRegVal(RHS, MRI))
     599             :     std::swap(RHS, LHS);
     600           0 : 
     601             :   const auto RHSImm = getConstantVRegVal(RHS, MRI);
     602             :   if (!RHSImm || *RHSImm != 0)
     603           0 :     return false;
     604           0 : 
     605           0 :   const RegisterBank &RB = *RBI.getRegBank(LHS, MRI, TRI);
     606           0 :   if (RB.getID() != AArch64::GPRRegBankID)
     607           0 :     return false;
     608           0 : 
     609           0 :   const auto Pred = (CmpInst::Predicate)CCMI->getOperand(1).getPredicate();
     610             :   if (Pred != CmpInst::ICMP_NE && Pred != CmpInst::ICMP_EQ)
     611           0 :     return false;
     612           0 : 
     613           0 :   const unsigned CmpWidth = MRI.getType(LHS).getSizeInBits();
     614             :   unsigned CBOpc = 0;
     615             :   if (CmpWidth <= 32)
     616           0 :     CBOpc = (Pred == CmpInst::ICMP_EQ ? AArch64::CBZW : AArch64::CBNZW);
     617           0 :   else if (CmpWidth == 64)
     618           0 :     CBOpc = (Pred == CmpInst::ICMP_EQ ? AArch64::CBZX : AArch64::CBNZX);
     619             :   else
     620           0 :     return false;
     621           0 : 
     622           0 :   BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(CBOpc))
     623             :       .addUse(LHS)
     624           0 :       .addMBB(DestMBB)
     625           0 :       .constrainAllUses(TII, TRI, RBI);
     626           0 : 
     627             :   I.eraseFromParent();
     628           0 :   return true;
     629             : }
     630           0 : 
     631           0 : bool AArch64InstructionSelector::selectVaStartAAPCS(
     632           0 :     MachineInstr &I, MachineFunction &MF, MachineRegisterInfo &MRI) const {
     633           0 :   return false;
     634             : }
     635           0 : 
     636             : bool AArch64InstructionSelector::selectVaStartDarwin(
     637           0 :     MachineInstr &I, MachineFunction &MF, MachineRegisterInfo &MRI) const {
     638             :   AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>();
     639             :   unsigned ListReg = I.getOperand(0).getReg();
     640           0 : 
     641             :   unsigned ArgsAddrReg = MRI.createVirtualRegister(&AArch64::GPR64RegClass);
     642           0 : 
     643           0 :   auto MIB =
     644             :       BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(AArch64::ADDXri))
     645             :           .addDef(ArgsAddrReg)
     646           0 :           .addFrameIndex(FuncInfo->getVarArgsStackIndex())
     647             :           .addImm(0)
     648           0 :           .addImm(0);
     649             : 
     650             :   constrainSelectedInstRegOperands(*MIB, TII, TRI, RBI);
     651           1 : 
     652             :   MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(AArch64::STRXui))
     653           1 :             .addUse(ArgsAddrReg)
     654           1 :             .addUse(ListReg)
     655             :             .addImm(0)
     656           1 :             .addMemOperand(*I.memoperands_begin());
     657             : 
     658             :   constrainSelectedInstRegOperands(*MIB, TII, TRI, RBI);
     659           2 :   I.eraseFromParent();
     660             :   return true;
     661           1 : }
     662             : 
     663           1 : void AArch64InstructionSelector::materializeLargeCMVal(
     664             :     MachineInstr &I, const Value *V, unsigned char OpFlags) const {
     665           1 :   MachineBasicBlock &MBB = *I.getParent();
     666             :   MachineFunction &MF = *MBB.getParent();
     667           2 :   MachineRegisterInfo &MRI = MF.getRegInfo();
     668             :   MachineIRBuilder MIB(I);
     669             : 
     670             :   auto MovZ = MIB.buildInstr(AArch64::MOVZXi, &AArch64::GPR64RegClass);
     671           1 :   MovZ->addOperand(MF, I.getOperand(1));
     672             :   MovZ->getOperand(1).setTargetFlags(OpFlags | AArch64II::MO_G0 |
     673           1 :                                      AArch64II::MO_NC);
     674           1 :   MovZ->addOperand(MF, MachineOperand::CreateImm(0));
     675           1 :   constrainSelectedInstRegOperands(*MovZ, TII, TRI, RBI);
     676             : 
     677             :   auto BuildMovK = [&](unsigned SrcReg, unsigned char Flags, unsigned Offset,
     678           4 :                        unsigned ForceDstReg) {
     679             :     unsigned DstReg = ForceDstReg
     680           4 :                           ? ForceDstReg
     681           4 :                           : MRI.createVirtualRegister(&AArch64::GPR64RegClass);
     682           4 :     auto MovI = MIB.buildInstr(AArch64::MOVKXi).addDef(DstReg).addUse(SrcReg);
     683             :     if (auto *GV = dyn_cast<GlobalValue>(V)) {
     684             :       MovI->addOperand(MF, MachineOperand::CreateGA(
     685           4 :                                GV, MovZ->getOperand(1).getOffset(), Flags));
     686           8 :     } else {
     687           4 :       MovI->addOperand(
     688             :           MF, MachineOperand::CreateBA(cast<BlockAddress>(V),
     689           4 :                                        MovZ->getOperand(1).getOffset(), Flags));
     690           4 :     }
     691             :     MovI->addOperand(MF, MachineOperand::CreateImm(Offset));
     692             :     constrainSelectedInstRegOperands(*MovI, TII, TRI, RBI);
     693             :     return DstReg;
     694             :   };
     695             :   unsigned DstReg = BuildMovK(MovZ->getOperand(0).getReg(),
     696             :                               AArch64II::MO_G1 | AArch64II::MO_NC, 16, 0);
     697             :   DstReg = BuildMovK(DstReg, AArch64II::MO_G2 | AArch64II::MO_NC, 32, 0);
     698             :   BuildMovK(DstReg, AArch64II::MO_G3, 48, I.getOperand(0).getReg());
     699             :   return;
     700             : }
     701             : 
     702             : bool AArch64InstructionSelector::select(MachineInstr &I,
     703             :                                         CodeGenCoverage &CoverageInfo) const {
     704             :   assert(I.getParent() && "Instruction should be in a basic block!");
     705             :   assert(I.getParent()->getParent() && "Instruction should be in a function!");
     706             : 
     707             :   MachineBasicBlock &MBB = *I.getParent();
     708             :   MachineFunction &MF = *MBB.getParent();
     709           4 :   MachineRegisterInfo &MRI = MF.getRegInfo();
     710           4 : 
     711             :   unsigned Opcode = I.getOpcode();
     712           4 :   // G_PHI requires same handling as PHI
     713           4 :   if (!isPreISelGenericOpcode(Opcode) || Opcode == TargetOpcode::G_PHI) {
     714           4 :     // Certain non-generic instructions also need some special handling.
     715             : 
     716             :     if (Opcode ==  TargetOpcode::LOAD_STACK_GUARD)
     717        2311 :       return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
     718             : 
     719             :     if (Opcode == TargetOpcode::PHI || Opcode == TargetOpcode::G_PHI) {
     720             :       const unsigned DefReg = I.getOperand(0).getReg();
     721             :       const LLT DefTy = MRI.getType(DefReg);
     722        2311 : 
     723        2311 :       const TargetRegisterClass *DefRC = nullptr;
     724        2311 :       if (TargetRegisterInfo::isPhysicalRegister(DefReg)) {
     725             :         DefRC = TRI.getRegClass(DefReg);
     726        2311 :       } else {
     727             :         const RegClassOrRegBank &RegClassOrBank =
     728        2311 :             MRI.getRegClassOrRegBank(DefReg);
     729             : 
     730             :         DefRC = RegClassOrBank.dyn_cast<const TargetRegisterClass *>();
     731        1229 :         if (!DefRC) {
     732           0 :           if (!DefTy.isValid()) {
     733             :             LLVM_DEBUG(dbgs() << "PHI operand has no type, not a gvreg?\n");
     734        1229 :             return false;
     735           9 :           }
     736             :           const RegisterBank &RB = *RegClassOrBank.get<const RegisterBank *>();
     737             :           DefRC = getRegClassForTypeOnBank(DefTy, RB, RBI);
     738             :           if (!DefRC) {
     739           9 :             LLVM_DEBUG(dbgs() << "PHI operand has unexpected size/bank\n");
     740           0 :             return false;
     741             :           }
     742             :         }
     743             :       }
     744             :       I.setDesc(TII.get(TargetOpcode::PHI));
     745             : 
     746           2 :       return RBI.constrainGenericRegister(DefReg, *DefRC, MRI);
     747           7 :     }
     748             : 
     749             :     if (I.isCopy())
     750             :       return selectCopy(I, TII, MRI, TRI, RBI);
     751             : 
     752           7 :     return true;
     753           7 :   }
     754             : 
     755             : 
     756             :   if (I.getNumOperands() != I.getNumExplicitOperands()) {
     757             :     LLVM_DEBUG(
     758             :         dbgs() << "Generic instruction has unexpected implicit operands\n");
     759           9 :     return false;
     760             :   }
     761           9 : 
     762             :   if (selectImpl(I, CoverageInfo))
     763             :     return true;
     764        1220 : 
     765         921 :   LLT Ty =
     766             :       I.getOperand(0).isReg() ? MRI.getType(I.getOperand(0).getReg()) : LLT{};
     767             : 
     768             :   switch (Opcode) {
     769             :   case TargetOpcode::G_BRCOND: {
     770             :     if (Ty.getSizeInBits() > 32) {
     771        1082 :       // We shouldn't need this on AArch64, but it would be implemented as an
     772             :       // EXTRACT_SUBREG followed by a TBNZW because TBNZX has no encoding if the
     773             :       // bit being tested is < 32.
     774             :       LLVM_DEBUG(dbgs() << "G_BRCOND has type: " << Ty
     775             :                         << ", expected at most 32-bits");
     776             :       return false;
     777        1082 :     }
     778             : 
     779             :     const unsigned CondReg = I.getOperand(0).getReg();
     780             :     MachineBasicBlock *DestMBB = I.getOperand(1).getMBB();
     781        1172 : 
     782             :     if (selectCompareBranch(I, MF, MRI))
     783         391 :       return true;
     784          17 : 
     785          17 :     auto MIB = BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::TBNZW))
     786             :                    .addUse(CondReg)
     787             :                    .addImm(/*bit offset=*/0)
     788             :                    .addMBB(DestMBB);
     789             : 
     790             :     I.eraseFromParent();
     791             :     return constrainSelectedInstRegOperands(*MIB.getInstr(), TII, TRI, RBI);
     792             :   }
     793             : 
     794          17 :   case TargetOpcode::G_BRINDIRECT: {
     795          17 :     I.setDesc(TII.get(AArch64::BR));
     796             :     return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
     797          17 :   }
     798             : 
     799             :   case TargetOpcode::G_FCONSTANT:
     800          26 :   case TargetOpcode::G_CONSTANT: {
     801             :     const bool isFP = Opcode == TargetOpcode::G_FCONSTANT;
     802             : 
     803          13 :     const LLT s32 = LLT::scalar(32);
     804             :     const LLT s64 = LLT::scalar(64);
     805          13 :     const LLT p0 = LLT::pointer(0, 64);
     806          13 : 
     807             :     const unsigned DefReg = I.getOperand(0).getReg();
     808             :     const LLT DefTy = MRI.getType(DefReg);
     809           4 :     const unsigned DefSize = DefTy.getSizeInBits();
     810           4 :     const RegisterBank &RB = *RBI.getRegBank(DefReg, MRI, TRI);
     811           4 : 
     812             :     // FIXME: Redundant check, but even less readable when factored out.
     813             :     if (isFP) {
     814          28 :       if (Ty != s32 && Ty != s64) {
     815             :         LLVM_DEBUG(dbgs() << "Unable to materialize FP " << Ty
     816             :                           << " constant, expected: " << s32 << " or " << s64
     817             :                           << '\n');
     818             :         return false;
     819             :       }
     820             : 
     821             :       if (RB.getID() != AArch64::FPRRegBankID) {
     822          28 :         LLVM_DEBUG(dbgs() << "Unable to materialize FP " << Ty
     823          28 :                           << " constant on bank: " << RB
     824          28 :                           << ", expected: FPR\n");
     825          28 :         return false;
     826             :       }
     827             : 
     828          28 :       // The case when we have 0.0 is covered by tablegen. Reject it here so we
     829          42 :       // can be sure tablegen works correctly and isn't rescued by this code.
     830             :       if (I.getOperand(1).getFPImm()->getValueAPF().isExactlyValue(0.0))
     831             :         return false;
     832             :     } else {
     833           0 :       // s32 and s64 are covered by tablegen.
     834             :       if (Ty != p0) {
     835             :         LLVM_DEBUG(dbgs() << "Unable to materialize integer " << Ty
     836          28 :                           << " constant, expected: " << s32 << ", " << s64
     837             :                           << ", or " << p0 << '\n');
     838             :         return false;
     839             :       }
     840             : 
     841             :       if (RB.getID() != AArch64::GPRRegBankID) {
     842             :         LLVM_DEBUG(dbgs() << "Unable to materialize integer " << Ty
     843             :                           << " constant on bank: " << RB
     844             :                           << ", expected: GPR\n");
     845          56 :         return false;
     846             :       }
     847             :     }
     848             : 
     849           0 :     const unsigned MovOpc =
     850             :         DefSize == 32 ? AArch64::MOVi32imm : AArch64::MOVi64imm;
     851             : 
     852             :     I.setDesc(TII.get(MovOpc));
     853           0 : 
     854             :     if (isFP) {
     855             :       const TargetRegisterClass &GPRRC =
     856           0 :           DefSize == 32 ? AArch64::GPR32RegClass : AArch64::GPR64RegClass;
     857             :       const TargetRegisterClass &FPRRC =
     858             :           DefSize == 32 ? AArch64::FPR32RegClass : AArch64::FPR64RegClass;
     859             : 
     860             :       const unsigned DefGPRReg = MRI.createVirtualRegister(&GPRRC);
     861             :       MachineOperand &RegOp = I.getOperand(0);
     862             :       RegOp.setReg(DefGPRReg);
     863             : 
     864             :       BuildMI(MBB, std::next(I.getIterator()), I.getDebugLoc(),
     865          28 :               TII.get(AArch64::COPY))
     866             :           .addDef(DefReg)
     867          28 :           .addUse(DefGPRReg);
     868             : 
     869          28 :       if (!RBI.constrainGenericRegister(DefReg, FPRRC, MRI)) {
     870          28 :         LLVM_DEBUG(dbgs() << "Failed to constrain G_FCONSTANT def operand\n");
     871             :         return false;
     872          28 :       }
     873             : 
     874             :       MachineOperand &ImmOp = I.getOperand(1);
     875          28 :       // FIXME: Is going through int64_t always correct?
     876          28 :       ImmOp.ChangeToImmediate(
     877          28 :           ImmOp.getFPImm()->getValueAPF().bitcastToAPInt().getZExtValue());
     878             :     } else if (I.getOperand(1).isCImm()) {
     879          28 :       uint64_t Val = I.getOperand(1).getCImm()->getZExtValue();
     880          84 :       I.getOperand(1).ChangeToImmediate(Val);
     881             :     } else if (I.getOperand(1).isImm()) {
     882             :       uint64_t Val = I.getOperand(1).getImm();
     883             :       I.getOperand(1).ChangeToImmediate(Val);
     884          28 :     }
     885             : 
     886             :     constrainSelectedInstRegOperands(I, TII, TRI, RBI);
     887             :     return true;
     888             :   }
     889          28 :   case TargetOpcode::G_EXTRACT: {
     890             :     LLT SrcTy = MRI.getType(I.getOperand(1).getReg());
     891          28 :     LLT DstTy = MRI.getType(I.getOperand(0).getReg());
     892          84 :     (void)DstTy;
     893           0 :     unsigned SrcSize = SrcTy.getSizeInBits();
     894           0 :     // Larger extracts are vectors, same-size extracts should be something else
     895           0 :     // by now (either split up or simplified to a COPY).
     896           0 :     if (SrcTy.getSizeInBits() > 64 || Ty.getSizeInBits() > 32)
     897           0 :       return false;
     898           0 : 
     899             :     I.setDesc(TII.get(SrcSize == 64 ? AArch64::UBFMXri : AArch64::UBFMWri));
     900             :     MachineInstrBuilder(MF, I).addImm(I.getOperand(2).getImm() +
     901          28 :                                       Ty.getSizeInBits() - 1);
     902          28 : 
     903             :     if (SrcSize < 64) {
     904           4 :       assert(SrcSize == 32 && DstTy.getSizeInBits() == 16 &&
     905           4 :              "unexpected G_EXTRACT types");
     906             :       return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
     907             :     }
     908           4 : 
     909             :     unsigned DstReg = MRI.createGenericVirtualRegister(LLT::scalar(64));
     910             :     BuildMI(MBB, std::next(I.getIterator()), I.getDebugLoc(),
     911           4 :             TII.get(AArch64::COPY))
     912             :         .addDef(I.getOperand(0).getReg())
     913             :         .addUse(DstReg, 0, AArch64::sub_32);
     914           6 :     RBI.constrainGenericRegister(I.getOperand(0).getReg(),
     915           8 :                                  AArch64::GPR32RegClass, MRI);
     916           4 :     I.getOperand(0).setReg(DstReg);
     917             : 
     918           4 :     return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
     919             :   }
     920             : 
     921           2 :   case TargetOpcode::G_INSERT: {
     922             :     LLT SrcTy = MRI.getType(I.getOperand(2).getReg());
     923             :     LLT DstTy = MRI.getType(I.getOperand(0).getReg());
     924           2 :     unsigned DstSize = DstTy.getSizeInBits();
     925           2 :     // Larger inserts are vectors, same-size ones should be something else by
     926           6 :     // now (split up or turned into COPYs).
     927           2 :     if (Ty.getSizeInBits() > 64 || SrcTy.getSizeInBits() > 32)
     928             :       return false;
     929           2 : 
     930             :     I.setDesc(TII.get(DstSize == 64 ? AArch64::BFMXri : AArch64::BFMWri));
     931           2 :     unsigned LSB = I.getOperand(3).getImm();
     932             :     unsigned Width = MRI.getType(I.getOperand(2).getReg()).getSizeInBits();
     933           2 :     I.getOperand(3).setImm((DstSize - LSB) % DstSize);
     934             :     MachineInstrBuilder(MF, I).addImm(Width - 1);
     935             : 
     936           6 :     if (DstSize < 64) {
     937           6 :       assert(DstSize == 32 && SrcTy.getSizeInBits() == 16 &&
     938           6 :              "unexpected G_INSERT types");
     939           6 :       return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
     940             :     }
     941             : 
     942           6 :     unsigned SrcReg = MRI.createGenericVirtualRegister(LLT::scalar(64));
     943             :     BuildMI(MBB, I.getIterator(), I.getDebugLoc(),
     944             :             TII.get(AArch64::SUBREG_TO_REG))
     945          10 :         .addDef(SrcReg)
     946           6 :         .addImm(0)
     947           6 :         .addUse(I.getOperand(2).getReg())
     948           6 :         .addImm(AArch64::sub_32);
     949           6 :     RBI.constrainGenericRegister(I.getOperand(2).getReg(),
     950             :                                  AArch64::GPR32RegClass, MRI);
     951           6 :     I.getOperand(2).setReg(SrcReg);
     952             : 
     953             :     return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
     954           4 :   }
     955             :   case TargetOpcode::G_FRAME_INDEX: {
     956             :     // allocas and G_FRAME_INDEX are only supported in addrspace(0).
     957           2 :     if (Ty != LLT::pointer(0, 64)) {
     958           2 :       LLVM_DEBUG(dbgs() << "G_FRAME_INDEX pointer has type: " << Ty
     959           4 :                         << ", expected: " << LLT::pointer(0, 64) << '\n');
     960             :       return false;
     961             :     }
     962           2 :     I.setDesc(TII.get(AArch64::ADDXri));
     963             : 
     964           2 :     // MOs for a #0 shifted immediate.
     965             :     I.addOperand(MachineOperand::CreateImm(0));
     966           4 :     I.addOperand(MachineOperand::CreateImm(0));
     967             : 
     968           2 :     return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
     969             :   }
     970             : 
     971             :   case TargetOpcode::G_GLOBAL_VALUE: {
     972           3 :     auto GV = I.getOperand(1).getGlobal();
     973             :     if (GV->isThreadLocal()) {
     974             :       // FIXME: we don't support TLS yet.
     975           0 :       return false;
     976             :     }
     977           3 :     unsigned char OpFlags = STI.ClassifyGlobalReference(GV, TM);
     978             :     if (OpFlags & AArch64II::MO_GOT) {
     979             :       I.setDesc(TII.get(AArch64::LOADgot));
     980           3 :       I.getOperand(1).setTargetFlags(OpFlags);
     981           3 :     } else if (TM.getCodeModel() == CodeModel::Large) {
     982             :       // Materialize the global using movz/movk instructions.
     983           3 :       materializeLargeCMVal(I, GV, OpFlags);
     984             :       I.eraseFromParent();
     985             :       return true;
     986          85 :     } else if (TM.getCodeModel() == CodeModel::Tiny) {
     987          85 :       I.setDesc(TII.get(AArch64::ADR));
     988          85 :       I.getOperand(1).setTargetFlags(OpFlags);
     989             :     } else {
     990             :       I.setDesc(TII.get(AArch64::MOVaddr));
     991             :       I.getOperand(1).setTargetFlags(OpFlags | AArch64II::MO_PAGE);
     992          85 :       MachineInstrBuilder MIB(MF, I);
     993          85 :       MIB.addGlobalAddress(GV, I.getOperand(1).getOffset(),
     994          18 :                            OpFlags | AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
     995          18 :     }
     996          67 :     return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
     997             :   }
     998           3 : 
     999           3 :   case TargetOpcode::G_LOAD:
    1000           3 :   case TargetOpcode::G_STORE: {
    1001          64 :     LLT PtrTy = MRI.getType(I.getOperand(1).getReg());
    1002          34 : 
    1003          34 :     if (PtrTy != LLT::pointer(0, 64)) {
    1004             :       LLVM_DEBUG(dbgs() << "Load/Store pointer has type: " << PtrTy
    1005          30 :                         << ", expected: " << LLT::pointer(0, 64) << '\n');
    1006          30 :       return false;
    1007             :     }
    1008          30 : 
    1009          30 :     auto &MemOp = **I.memoperands_begin();
    1010             :     if (MemOp.getOrdering() != AtomicOrdering::NotAtomic) {
    1011          82 :       LLVM_DEBUG(dbgs() << "Atomic load/store not supported yet\n");
    1012             :       return false;
    1013             :     }
    1014          80 :     unsigned MemSizeInBits = MemOp.getSize() * 8;
    1015             : 
    1016          80 :     // FIXME: PR36018: Volatile loads in some cases are incorrectly selected by
    1017             :     // folding with an extend. Until we have a G_SEXTLOAD solution bail out if
    1018             :     // we hit one.
    1019             :     if (Opcode == TargetOpcode::G_LOAD && MemOp.isVolatile())
    1020             :       return false;
    1021             : 
    1022             :     const unsigned PtrReg = I.getOperand(1).getReg();
    1023             : #ifndef NDEBUG
    1024          80 :     const RegisterBank &PtrRB = *RBI.getRegBank(PtrReg, MRI, TRI);
    1025          80 :     // Sanity-check the pointer register.
    1026             :     assert(PtrRB.getID() == AArch64::GPRRegBankID &&
    1027             :            "Load/Store pointer operand isn't a GPR");
    1028             :     assert(MRI.getType(PtrReg).isPointer() &&
    1029          76 :            "Load/Store pointer operand isn't a pointer");
    1030             : #endif
    1031             : 
    1032             :     const unsigned ValReg = I.getOperand(0).getReg();
    1033             :     const RegisterBank &RB = *RBI.getRegBank(ValReg, MRI, TRI);
    1034          76 : 
    1035             :     const unsigned NewOpc =
    1036             :         selectLoadStoreUIOp(I.getOpcode(), RB.getID(), MemSizeInBits);
    1037          76 :     if (NewOpc == I.getOpcode())
    1038             :       return false;
    1039             : 
    1040             :     I.setDesc(TII.get(NewOpc));
    1041             : 
    1042             :     uint64_t Offset = 0;
    1043             :     auto *PtrMI = MRI.getVRegDef(PtrReg);
    1044             : 
    1045             :     // Try to fold a GEP into our unsigned immediate addressing mode.
    1046             :     if (PtrMI->getOpcode() == TargetOpcode::G_GEP) {
    1047          76 :       if (auto COff = getConstantVRegVal(PtrMI->getOperand(2).getReg(), MRI)) {
    1048          76 :         int64_t Imm = *COff;
    1049             :         const unsigned Size = MemSizeInBits / 8;
    1050             :         const unsigned Scale = Log2_32(Size);
    1051          76 :         if ((Imm & (Size - 1)) == 0 && Imm >= 0 && Imm < (0x1000 << Scale)) {
    1052          76 :           unsigned Ptr2Reg = PtrMI->getOperand(1).getReg();
    1053             :           I.getOperand(1).setReg(Ptr2Reg);
    1054             :           PtrMI = MRI.getVRegDef(Ptr2Reg);
    1055          76 :           Offset = Imm / Size;
    1056             :         }
    1057             :       }
    1058          76 :     }
    1059             : 
    1060             :     // If we haven't folded anything into our addressing mode yet, try to fold
    1061         152 :     // a frame index into the base+offset.
    1062          19 :     if (!Offset && PtrMI->getOpcode() == TargetOpcode::G_FRAME_INDEX)
    1063          19 :       I.getOperand(1).ChangeToFrameIndex(PtrMI->getOperand(1).getIndex());
    1064          19 : 
    1065             :     I.addOperand(MachineOperand::CreateImm(Offset));
    1066          19 : 
    1067          19 :     // If we're storing a 0, use WZR/XZR.
    1068          38 :     if (auto CVal = getConstantVRegVal(ValReg, MRI)) {
    1069          19 :       if (*CVal == 0 && Opcode == TargetOpcode::G_STORE) {
    1070          19 :         if (I.getOpcode() == AArch64::STRWui)
    1071             :           I.getOperand(0).setReg(AArch64::WZR);
    1072             :         else if (I.getOpcode() == AArch64::STRXui)
    1073             :           I.getOperand(0).setReg(AArch64::XZR);
    1074             :       }
    1075             :     }
    1076             : 
    1077         133 :     return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
    1078           6 :   }
    1079             : 
    1080         152 :   case TargetOpcode::G_SMULH:
    1081             :   case TargetOpcode::G_UMULH: {
    1082             :     // Reject the various things we don't support yet.
    1083          76 :     if (unsupportedBinOp(I, RBI, MRI, TRI))
    1084           0 :       return false;
    1085           0 : 
    1086           0 :     const unsigned DefReg = I.getOperand(0).getReg();
    1087           0 :     const RegisterBank &RB = *RBI.getRegBank(DefReg, MRI, TRI);
    1088           0 : 
    1089             :     if (RB.getID() != AArch64::GPRRegBankID) {
    1090             :       LLVM_DEBUG(dbgs() << "G_[SU]MULH on bank: " << RB << ", expected: GPR\n");
    1091             :       return false;
    1092          76 :     }
    1093             : 
    1094             :     if (Ty != LLT::scalar(64)) {
    1095           2 :       LLVM_DEBUG(dbgs() << "G_[SU]MULH has type: " << Ty
    1096             :                         << ", expected: " << LLT::scalar(64) << '\n');
    1097             :       return false;
    1098           2 :     }
    1099             : 
    1100             :     unsigned NewOpc = I.getOpcode() == TargetOpcode::G_SMULH ? AArch64::SMULHrr
    1101           2 :                                                              : AArch64::UMULHrr;
    1102           2 :     I.setDesc(TII.get(NewOpc));
    1103             : 
    1104           2 :     // Now that we selected an opcode, we need to constrain the register
    1105             :     // operands to use appropriate classes.
    1106             :     return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
    1107             :   }
    1108             :   case TargetOpcode::G_FADD:
    1109           2 :   case TargetOpcode::G_FSUB:
    1110             :   case TargetOpcode::G_FMUL:
    1111             :   case TargetOpcode::G_FDIV:
    1112           0 : 
    1113             :   case TargetOpcode::G_OR:
    1114             :   case TargetOpcode::G_SHL:
    1115           4 :   case TargetOpcode::G_LSHR:
    1116             :   case TargetOpcode::G_ASHR:
    1117           2 :   case TargetOpcode::G_GEP: {
    1118             :     // Reject the various things we don't support yet.
    1119             :     if (unsupportedBinOp(I, RBI, MRI, TRI))
    1120             :       return false;
    1121           2 : 
    1122             :     const unsigned OpSize = Ty.getSizeInBits();
    1123          14 : 
    1124             :     const unsigned DefReg = I.getOperand(0).getReg();
    1125             :     const RegisterBank &RB = *RBI.getRegBank(DefReg, MRI, TRI);
    1126             : 
    1127             :     const unsigned NewOpc = selectBinaryOp(I.getOpcode(), RB.getID(), OpSize);
    1128             :     if (NewOpc == I.getOpcode())
    1129             :       return false;
    1130             : 
    1131             :     I.setDesc(TII.get(NewOpc));
    1132             :     // FIXME: Should the type be always reset in setDesc?
    1133             : 
    1134          14 :     // Now that we selected an opcode, we need to constrain the register
    1135             :     // operands to use appropriate classes.
    1136             :     return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
    1137          14 :   }
    1138             : 
    1139          14 :   case TargetOpcode::G_PTR_MASK: {
    1140          14 :     uint64_t Align = I.getOperand(2).getImm();
    1141             :     if (Align >= 64 || Align == 0)
    1142          14 :       return false;
    1143          14 : 
    1144             :     uint64_t Mask = ~((1ULL << Align) - 1);
    1145             :     I.setDesc(TII.get(AArch64::ANDXri));
    1146          14 :     I.getOperand(2).setImm(AArch64_AM::encodeLogicalImmediate(Mask, 64));
    1147             : 
    1148             :     return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
    1149             :   }
    1150             :   case TargetOpcode::G_PTRTOINT:
    1151          14 :   case TargetOpcode::G_TRUNC: {
    1152             :     const LLT DstTy = MRI.getType(I.getOperand(0).getReg());
    1153             :     const LLT SrcTy = MRI.getType(I.getOperand(1).getReg());
    1154           3 : 
    1155           3 :     const unsigned DstReg = I.getOperand(0).getReg();
    1156           3 :     const unsigned SrcReg = I.getOperand(1).getReg();
    1157             : 
    1158             :     const RegisterBank &DstRB = *RBI.getRegBank(DstReg, MRI, TRI);
    1159           3 :     const RegisterBank &SrcRB = *RBI.getRegBank(SrcReg, MRI, TRI);
    1160           3 : 
    1161           3 :     if (DstRB.getID() != SrcRB.getID()) {
    1162             :       LLVM_DEBUG(
    1163           3 :           dbgs() << "G_TRUNC/G_PTRTOINT input/output on different banks\n");
    1164             :       return false;
    1165          56 :     }
    1166             : 
    1167          56 :     if (DstRB.getID() == AArch64::GPRRegBankID) {
    1168          56 :       const TargetRegisterClass *DstRC =
    1169             :           getRegClassForTypeOnBank(DstTy, DstRB, RBI);
    1170             :       if (!DstRC)
    1171             :         return false;
    1172             : 
    1173          56 :       const TargetRegisterClass *SrcRC =
    1174          56 :           getRegClassForTypeOnBank(SrcTy, SrcRB, RBI);
    1175             :       if (!SrcRC)
    1176          56 :         return false;
    1177             : 
    1178             :       if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, MRI) ||
    1179             :           !RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) {
    1180             :         LLVM_DEBUG(dbgs() << "Failed to constrain G_TRUNC/G_PTRTOINT\n");
    1181             :         return false;
    1182          56 :       }
    1183             : 
    1184          56 :       if (DstRC == SrcRC) {
    1185          56 :         // Nothing to be done
    1186             :       } else if (Opcode == TargetOpcode::G_TRUNC && DstTy == LLT::scalar(32) &&
    1187             :                  SrcTy == LLT::scalar(64)) {
    1188             :         llvm_unreachable("TableGen can import this case");
    1189          56 :         return false;
    1190          56 :       } else if (DstRC == &AArch64::GPR32RegClass &&
    1191             :                  SrcRC == &AArch64::GPR64RegClass) {
    1192             :         I.getOperand(1).setSubReg(AArch64::sub_32);
    1193         112 :       } else {
    1194          56 :         LLVM_DEBUG(
    1195             :             dbgs() << "Unhandled mismatched classes in G_TRUNC/G_PTRTOINT\n");
    1196           0 :         return false;
    1197             :       }
    1198             : 
    1199          56 :       I.setDesc(TII.get(TargetOpcode::COPY));
    1200             :       return true;
    1201           5 :     } else if (DstRB.getID() == AArch64::FPRRegBankID) {
    1202             :       if (DstTy == LLT::vector(4, 16) && SrcTy == LLT::vector(4, 32)) {
    1203           0 :         I.setDesc(TII.get(AArch64::XTNv4i16));
    1204             :         constrainSelectedInstRegOperands(I, TII, TRI, RBI);
    1205           5 :         return true;
    1206             :       }
    1207           5 :     }
    1208             : 
    1209             :     return false;
    1210             :   }
    1211             : 
    1212             :   case TargetOpcode::G_ANYEXT: {
    1213             :     const unsigned DstReg = I.getOperand(0).getReg();
    1214          56 :     const unsigned SrcReg = I.getOperand(1).getReg();
    1215          56 : 
    1216           0 :     const RegisterBank &RBDst = *RBI.getRegBank(DstReg, MRI, TRI);
    1217             :     if (RBDst.getID() != AArch64::GPRRegBankID) {
    1218           0 :       LLVM_DEBUG(dbgs() << "G_ANYEXT on bank: " << RBDst
    1219           0 :                         << ", expected: GPR\n");
    1220           0 :       return false;
    1221             :     }
    1222             : 
    1223             :     const RegisterBank &RBSrc = *RBI.getRegBank(SrcReg, MRI, TRI);
    1224             :     if (RBSrc.getID() != AArch64::GPRRegBankID) {
    1225             :       LLVM_DEBUG(dbgs() << "G_ANYEXT on bank: " << RBSrc
    1226             :                         << ", expected: GPR\n");
    1227          30 :       return false;
    1228          30 :     }
    1229          30 : 
    1230             :     const unsigned DstSize = MRI.getType(DstReg).getSizeInBits();
    1231          30 : 
    1232          30 :     if (DstSize == 0) {
    1233             :       LLVM_DEBUG(dbgs() << "G_ANYEXT operand has no size, not a gvreg?\n");
    1234             :       return false;
    1235             :     }
    1236             : 
    1237             :     if (DstSize != 64 && DstSize > 32) {
    1238          30 :       LLVM_DEBUG(dbgs() << "G_ANYEXT to size: " << DstSize
    1239          30 :                         << ", expected: 32 or 64\n");
    1240             :       return false;
    1241             :     }
    1242             :     // At this point G_ANYEXT is just like a plain COPY, but we need
    1243             :     // to explicitly form the 64-bit value if any.
    1244             :     if (DstSize > 32) {
    1245          30 :       unsigned ExtSrc = MRI.createVirtualRegister(&AArch64::GPR64allRegClass);
    1246             :       BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::SUBREG_TO_REG))
    1247          30 :           .addDef(ExtSrc)
    1248             :           .addImm(0)
    1249             :           .addUse(SrcReg)
    1250             :           .addImm(AArch64::sub_32);
    1251             :       I.getOperand(1).setReg(ExtSrc);
    1252          30 :     }
    1253             :     return selectCopy(I, TII, MRI, TRI, RBI);
    1254             :   }
    1255             : 
    1256             :   case TargetOpcode::G_ZEXT:
    1257             :   case TargetOpcode::G_SEXT: {
    1258             :     unsigned Opcode = I.getOpcode();
    1259          30 :     const LLT DstTy = MRI.getType(I.getOperand(0).getReg()),
    1260           2 :               SrcTy = MRI.getType(I.getOperand(1).getReg());
    1261           4 :     const bool isSigned = Opcode == TargetOpcode::G_SEXT;
    1262             :     const unsigned DefReg = I.getOperand(0).getReg();
    1263             :     const unsigned SrcReg = I.getOperand(1).getReg();
    1264             :     const RegisterBank &RB = *RBI.getRegBank(DefReg, MRI, TRI);
    1265             : 
    1266           4 :     if (RB.getID() != AArch64::GPRRegBankID) {
    1267             :       LLVM_DEBUG(dbgs() << TII.getName(I.getOpcode()) << " on bank: " << RB
    1268          30 :                         << ", expected: GPR\n");
    1269             :       return false;
    1270             :     }
    1271          14 : 
    1272             :     MachineInstr *ExtI;
    1273          14 :     if (DstTy == LLT::scalar(64)) {
    1274          14 :       // FIXME: Can we avoid manually doing this?
    1275          14 :       if (!RBI.constrainGenericRegister(SrcReg, AArch64::GPR32RegClass, MRI)) {
    1276             :         LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(Opcode)
    1277             :                           << " operand\n");
    1278             :         return false;
    1279          14 :       }
    1280             : 
    1281          14 :       const unsigned SrcXReg =
    1282             :           MRI.createVirtualRegister(&AArch64::GPR64RegClass);
    1283             :       BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::SUBREG_TO_REG))
    1284             :           .addDef(SrcXReg)
    1285             :           .addImm(0)
    1286             :           .addUse(SrcReg)
    1287             :           .addImm(AArch64::sub_32);
    1288           9 : 
    1289             :       const unsigned NewOpc = isSigned ? AArch64::SBFMXri : AArch64::UBFMXri;
    1290           5 :       ExtI = BuildMI(MBB, I, I.getDebugLoc(), TII.get(NewOpc))
    1291             :                  .addDef(DefReg)
    1292             :                  .addUse(SrcXReg)
    1293             :                  .addImm(0)
    1294             :                  .addImm(SrcTy.getSizeInBits() - 1);
    1295             :     } else if (DstTy.isScalar() && DstTy.getSizeInBits() <= 32) {
    1296             :       const unsigned NewOpc = isSigned ? AArch64::SBFMWri : AArch64::UBFMWri;
    1297           5 :       ExtI = BuildMI(MBB, I, I.getDebugLoc(), TII.get(NewOpc))
    1298          10 :                  .addDef(DefReg)
    1299             :                  .addUse(SrcReg)
    1300             :                  .addImm(0)
    1301             :                  .addImm(SrcTy.getSizeInBits() - 1);
    1302             :     } else {
    1303             :       return false;
    1304           5 :     }
    1305          10 : 
    1306             :     constrainSelectedInstRegOperands(*ExtI, TII, TRI, RBI);
    1307             : 
    1308             :     I.eraseFromParent();
    1309           5 :     return true;
    1310           9 :   }
    1311           9 : 
    1312          18 :   case TargetOpcode::G_SITOFP:
    1313             :   case TargetOpcode::G_UITOFP:
    1314             :   case TargetOpcode::G_FPTOSI:
    1315             :   case TargetOpcode::G_FPTOUI: {
    1316           9 :     const LLT DstTy = MRI.getType(I.getOperand(0).getReg()),
    1317             :               SrcTy = MRI.getType(I.getOperand(1).getReg());
    1318           0 :     const unsigned NewOpc = selectFPConvOpc(Opcode, DstTy, SrcTy);
    1319             :     if (NewOpc == Opcode)
    1320             :       return false;
    1321          14 : 
    1322             :     I.setDesc(TII.get(NewOpc));
    1323          14 :     constrainSelectedInstRegOperands(I, TII, TRI, RBI);
    1324          14 : 
    1325             :     return true;
    1326             :   }
    1327           0 : 
    1328             : 
    1329             :   case TargetOpcode::G_INTTOPTR:
    1330             :     // The importer is currently unable to import pointer types since they
    1331           0 :     // didn't exist in SelectionDAG.
    1332           0 :     return selectCopy(I, TII, MRI, TRI, RBI);
    1333           0 : 
    1334           0 :   case TargetOpcode::G_BITCAST:
    1335             :     // Imported SelectionDAG rules can handle every bitcast except those that
    1336             :     // bitcast from a type to the same type. Ideally, these shouldn't occur
    1337           0 :     // but we might not run an optimizer that deletes them.
    1338           0 :     if (MRI.getType(I.getOperand(0).getReg()) ==
    1339             :         MRI.getType(I.getOperand(1).getReg()))
    1340           0 :       return selectCopy(I, TII, MRI, TRI, RBI);
    1341             :     return false;
    1342             : 
    1343             :   case TargetOpcode::G_SELECT: {
    1344           1 :     if (MRI.getType(I.getOperand(1).getReg()) != LLT::scalar(1)) {
    1345             :       LLVM_DEBUG(dbgs() << "G_SELECT cond has type: " << Ty
    1346             :                         << ", expected: " << LLT::scalar(1) << '\n');
    1347           1 :       return false;
    1348             :     }
    1349           3 : 
    1350             :     const unsigned CondReg = I.getOperand(1).getReg();
    1351             :     const unsigned TReg = I.getOperand(2).getReg();
    1352             :     const unsigned FReg = I.getOperand(3).getReg();
    1353           3 : 
    1354           3 :     unsigned CSelOpc = 0;
    1355           3 : 
    1356             :     if (Ty == LLT::scalar(32)) {
    1357             :       CSelOpc = AArch64::CSELWr;
    1358             :     } else if (Ty == LLT::scalar(64) || Ty == LLT::pointer(0, 64)) {
    1359           9 :       CSelOpc = AArch64::CSELXr;
    1360             :     } else {
    1361             :       return false;
    1362             :     }
    1363             : 
    1364             :     MachineInstr &TstMI =
    1365             :         *BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::ANDSWri))
    1366           9 :              .addDef(AArch64::WZR)
    1367           9 :              .addUse(CondReg)
    1368             :              .addImm(AArch64_AM::encodeLogicalImmediate(1, 32));
    1369             : 
    1370             :     MachineInstr &CSelMI = *BuildMI(MBB, I, I.getDebugLoc(), TII.get(CSelOpc))
    1371           9 :                                 .addDef(I.getOperand(0).getReg())
    1372             :                                 .addUse(TReg)
    1373           9 :                                 .addUse(FReg)
    1374             :                                 .addImm(AArch64CC::NE);
    1375             : 
    1376           0 :     constrainSelectedInstRegOperands(TstMI, TII, TRI, RBI);
    1377             :     constrainSelectedInstRegOperands(CSelMI, TII, TRI, RBI);
    1378             : 
    1379             :     I.eraseFromParent();
    1380          18 :     return true;
    1381             :   }
    1382             :   case TargetOpcode::G_ICMP: {
    1383           9 :     if (Ty != LLT::scalar(32)) {
    1384             :       LLVM_DEBUG(dbgs() << "G_ICMP result has type: " << Ty
    1385          18 :                         << ", expected: " << LLT::scalar(32) << '\n');
    1386           9 :       return false;
    1387             :     }
    1388             : 
    1389           9 :     unsigned CmpOpc = 0;
    1390             :     unsigned ZReg = 0;
    1391           9 : 
    1392           9 :     LLT CmpTy = MRI.getType(I.getOperand(2).getReg());
    1393             :     if (CmpTy == LLT::scalar(32)) {
    1394           9 :       CmpOpc = AArch64::SUBSWrr;
    1395           9 :       ZReg = AArch64::WZR;
    1396             :     } else if (CmpTy == LLT::scalar(64) || CmpTy.isPointer()) {
    1397             :       CmpOpc = AArch64::SUBSXrr;
    1398          14 :       ZReg = AArch64::XZR;
    1399             :     } else {
    1400             :       return false;
    1401           0 :     }
    1402             : 
    1403             :     // CSINC increments the result by one when the condition code is false.
    1404             :     // Therefore, we have to invert the predicate to get an increment by 1 when
    1405             :     // the predicate is true.
    1406             :     const AArch64CC::CondCode invCC =
    1407          14 :         changeICMPPredToAArch64CC(CmpInst::getInversePredicate(
    1408             :             (CmpInst::Predicate)I.getOperand(1).getPredicate()));
    1409             : 
    1410             :     MachineInstr &CmpMI = *BuildMI(MBB, I, I.getDebugLoc(), TII.get(CmpOpc))
    1411             :                                .addDef(ZReg)
    1412             :                                .addUse(I.getOperand(2).getReg())
    1413             :                                .addUse(I.getOperand(3).getReg());
    1414             : 
    1415           0 :     MachineInstr &CSetMI =
    1416             :         *BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::CSINCWr))
    1417             :              .addDef(I.getOperand(0).getReg())
    1418             :              .addUse(AArch64::WZR)
    1419             :              .addUse(AArch64::WZR)
    1420             :              .addImm(invCC);
    1421             : 
    1422          14 :     constrainSelectedInstRegOperands(CmpMI, TII, TRI, RBI);
    1423          14 :     constrainSelectedInstRegOperands(CSetMI, TII, TRI, RBI);
    1424             : 
    1425          28 :     I.eraseFromParent();
    1426             :     return true;
    1427          14 :   }
    1428          14 : 
    1429             :   case TargetOpcode::G_FCMP: {
    1430             :     if (Ty != LLT::scalar(32)) {
    1431          28 :       LLVM_DEBUG(dbgs() << "G_FCMP result has type: " << Ty
    1432          14 :                         << ", expected: " << LLT::scalar(32) << '\n');
    1433             :       return false;
    1434             :     }
    1435          14 : 
    1436             :     unsigned CmpOpc = 0;
    1437          14 :     LLT CmpTy = MRI.getType(I.getOperand(2).getReg());
    1438          14 :     if (CmpTy == LLT::scalar(32)) {
    1439             :       CmpOpc = AArch64::FCMPSrr;
    1440          14 :     } else if (CmpTy == LLT::scalar(64)) {
    1441          14 :       CmpOpc = AArch64::FCMPDrr;
    1442             :     } else {
    1443             :       return false;
    1444             :     }
    1445           6 : 
    1446             :     // FIXME: regbank
    1447             : 
    1448           0 :     AArch64CC::CondCode CC1, CC2;
    1449             :     changeFCMPPredToAArch64CC(
    1450             :         (CmpInst::Predicate)I.getOperand(1).getPredicate(), CC1, CC2);
    1451             : 
    1452           6 :     MachineInstr &CmpMI = *BuildMI(MBB, I, I.getDebugLoc(), TII.get(CmpOpc))
    1453             :                                .addUse(I.getOperand(2).getReg())
    1454             :                                .addUse(I.getOperand(3).getReg());
    1455             : 
    1456             :     const unsigned DefReg = I.getOperand(0).getReg();
    1457             :     unsigned Def1Reg = DefReg;
    1458             :     if (CC2 != AArch64CC::AL)
    1459             :       Def1Reg = MRI.createVirtualRegister(&AArch64::GPR32RegClass);
    1460             : 
    1461             :     MachineInstr &CSetMI =
    1462             :         *BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::CSINCWr))
    1463             :              .addDef(Def1Reg)
    1464           6 :              .addUse(AArch64::WZR)
    1465           6 :              .addUse(AArch64::WZR)
    1466             :              .addImm(getInvertedCondCode(CC1));
    1467          12 : 
    1468           6 :     if (CC2 != AArch64CC::AL) {
    1469           6 :       unsigned Def2Reg = MRI.createVirtualRegister(&AArch64::GPR32RegClass);
    1470             :       MachineInstr &CSet2MI =
    1471           6 :           *BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::CSINCWr))
    1472             :                .addDef(Def2Reg)
    1473           6 :                .addUse(AArch64::WZR)
    1474           3 :                .addUse(AArch64::WZR)
    1475             :                .addImm(getInvertedCondCode(CC2));
    1476             :       MachineInstr &OrMI =
    1477          12 :           *BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::ORRWrr))
    1478             :                .addDef(DefReg)
    1479             :                .addUse(Def1Reg)
    1480             :                .addUse(Def2Reg);
    1481          12 :       constrainSelectedInstRegOperands(OrMI, TII, TRI, RBI);
    1482             :       constrainSelectedInstRegOperands(CSet2MI, TII, TRI, RBI);
    1483           6 :     }
    1484           3 : 
    1485             :     constrainSelectedInstRegOperands(CmpMI, TII, TRI, RBI);
    1486           6 :     constrainSelectedInstRegOperands(CSetMI, TII, TRI, RBI);
    1487             : 
    1488             :     I.eraseFromParent();
    1489             :     return true;
    1490           6 :   }
    1491             :   case TargetOpcode::G_VASTART:
    1492           6 :     return STI.isTargetDarwin() ? selectVaStartDarwin(I, MF, MRI)
    1493             :                                 : selectVaStartAAPCS(I, MF, MRI);
    1494             :   case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS:
    1495           3 :     if (!I.getOperand(0).isIntrinsicID())
    1496           3 :       return false;
    1497           3 :     if (I.getOperand(0).getIntrinsicID() != Intrinsic::trap)
    1498             :       return false;
    1499             :     BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::BRK))
    1500           6 :       .addImm(1);
    1501           6 :     I.eraseFromParent();
    1502             :     return true;
    1503           6 :   case TargetOpcode::G_IMPLICIT_DEF: {
    1504           6 :     I.setDesc(TII.get(TargetOpcode::IMPLICIT_DEF));
    1505             :     const LLT DstTy = MRI.getType(I.getOperand(0).getReg());
    1506           1 :     const unsigned DstReg = I.getOperand(0).getReg();
    1507           2 :     const RegisterBank &DstRB = *RBI.getRegBank(DstReg, MRI, TRI);
    1508             :     const TargetRegisterClass *DstRC =
    1509           2 :         getRegClassForTypeOnBank(DstTy, DstRB, RBI);
    1510           2 :     RBI.constrainGenericRegister(DstReg, *DstRC, MRI);
    1511             :     return true;
    1512           1 :   }
    1513             :   case TargetOpcode::G_BLOCK_ADDR: {
    1514           2 :     if (TM.getCodeModel() == CodeModel::Large) {
    1515             :       materializeLargeCMVal(I, I.getOperand(1).getBlockAddress(), 0);
    1516           1 :       I.eraseFromParent();
    1517           1 :       return true;
    1518           7 :     } else {
    1519           7 :       I.setDesc(TII.get(AArch64::MOVaddrBA));
    1520           7 :       auto MovMI = BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::MOVaddrBA),
    1521             :                            I.getOperand(0).getReg())
    1522           7 :                        .addBlockAddress(I.getOperand(1).getBlockAddress(),
    1523             :                                         /* Offset */ 0, AArch64II::MO_PAGE)
    1524           7 :                        .addBlockAddress(
    1525           7 :                            I.getOperand(1).getBlockAddress(), /* Offset */ 0,
    1526             :                            AArch64II::MO_NC | AArch64II::MO_PAGEOFF);
    1527             :       I.eraseFromParent();
    1528           2 :       return constrainSelectedInstRegOperands(*MovMI, TII, TRI, RBI);
    1529           2 :     }
    1530           1 :   }
    1531           1 :   }
    1532           1 : 
    1533             :   return false;
    1534           1 : }
    1535             : 
    1536           2 : /// SelectArithImmed - Select an immediate value that can be represented as
    1537           1 : /// a 12-bit value shifted left by either 0 or 12.  If so, return true with
    1538           1 : /// Val set to the 12-bit value and Shift set to the shifter operand.
    1539             : InstructionSelector::ComplexRendererFns
    1540           1 : AArch64InstructionSelector::selectArithImmed(MachineOperand &Root) const {
    1541           1 :   MachineInstr &MI = *Root.getParent();
    1542           1 :   MachineBasicBlock &MBB = *MI.getParent();
    1543           1 :   MachineFunction &MF = *MBB.getParent();
    1544             :   MachineRegisterInfo &MRI = MF.getRegInfo();
    1545             : 
    1546             :   // This function is called from the addsub_shifted_imm ComplexPattern,
    1547             :   // which lists [imm] as the list of opcode it's interested in, however
    1548             :   // we still need to check whether the operand is actually an immediate
    1549             :   // here because the ComplexPattern opcode list is only used in
    1550             :   // root-level opcode matching.
    1551             :   uint64_t Immed;
    1552             :   if (Root.isImm())
    1553             :     Immed = Root.getImm();
    1554             :   else if (Root.isCImm())
    1555          40 :     Immed = Root.getCImm()->getZExtValue();
    1556          40 :   else if (Root.isReg()) {
    1557          40 :     MachineInstr *Def = MRI.getVRegDef(Root.getReg());
    1558          40 :     if (Def->getOpcode() != TargetOpcode::G_CONSTANT)
    1559          40 :       return None;
    1560             :     MachineOperand &Op1 = Def->getOperand(1);
    1561             :     if (!Op1.isCImm() || Op1.getCImm()->getBitWidth() > 64)
    1562             :       return None;
    1563             :     Immed = Op1.getCImm()->getZExtValue();
    1564             :   } else
    1565             :     return None;
    1566             : 
    1567          40 :   unsigned ShiftAmt;
    1568           0 : 
    1569          40 :   if (Immed >> 12 == 0) {
    1570           0 :     ShiftAmt = 0;
    1571          40 :   } else if ((Immed & 0xfff) == 0 && Immed >> 24 == 0) {
    1572          40 :     ShiftAmt = 12;
    1573          80 :     Immed = Immed >> 12;
    1574             :   } else
    1575          11 :     return None;
    1576          11 : 
    1577             :   unsigned ShVal = AArch64_AM::getShifterImm(AArch64_AM::LSL, ShiftAmt);
    1578             :   return {{
    1579             :       [=](MachineInstrBuilder &MIB) { MIB.addImm(Immed); },
    1580             :       [=](MachineInstrBuilder &MIB) { MIB.addImm(ShVal); },
    1581             :   }};
    1582             : }
    1583             : 
    1584          11 : /// Select a "register plus unscaled signed 9-bit immediate" address.  This
    1585             : /// should only match when there is an offset that is not valid for a scaled
    1586           0 : /// immediate addressing mode.  The "Size" argument is the size in bytes of the
    1587             : /// memory reference, which is needed here to know what is valid for a scaled
    1588             : /// immediate.
    1589             : InstructionSelector::ComplexRendererFns
    1590             : AArch64InstructionSelector::selectAddrModeUnscaled(MachineOperand &Root,
    1591             :                                                    unsigned Size) const {
    1592             :   MachineRegisterInfo &MRI =
    1593             :       Root.getParent()->getParent()->getParent()->getRegInfo();
    1594          11 : 
    1595          11 :   if (!Root.isReg())
    1596          44 :     return None;
    1597             : 
    1598             :   if (!isBaseWithConstantOffset(Root, MRI))
    1599             :     return None;
    1600             : 
    1601             :   MachineInstr *RootDef = MRI.getVRegDef(Root.getReg());
    1602             :   if (!RootDef)
    1603             :     return None;
    1604             : 
    1605          91 :   MachineOperand &OffImm = RootDef->getOperand(2);
    1606             :   if (!OffImm.isReg())
    1607             :     return None;
    1608          91 :   MachineInstr *RHS = MRI.getVRegDef(OffImm.getReg());
    1609             :   if (!RHS || RHS->getOpcode() != TargetOpcode::G_CONSTANT)
    1610          91 :     return None;
    1611             :   int64_t RHSC;
    1612             :   MachineOperand &RHSOp1 = RHS->getOperand(1);
    1613          91 :   if (!RHSOp1.isCImm() || RHSOp1.getCImm()->getBitWidth() > 64)
    1614             :     return None;
    1615             :   RHSC = RHSOp1.getCImm()->getSExtValue();
    1616           3 : 
    1617           3 :   // If the offset is valid as a scaled immediate, don't match here.
    1618             :   if ((RHSC & (Size - 1)) == 0 && RHSC >= 0 && RHSC < (0x1000 << Log2_32(Size)))
    1619             :     return None;
    1620           3 :   if (RHSC >= -256 && RHSC < 256) {
    1621           3 :     MachineOperand &Base = RootDef->getOperand(1);
    1622             :     return {{
    1623           3 :         [=](MachineInstrBuilder &MIB) { MIB.add(Base); },
    1624           3 :         [=](MachineInstrBuilder &MIB) { MIB.addImm(RHSC); },
    1625             :     }};
    1626             :   }
    1627           3 :   return None;
    1628           3 : }
    1629             : 
    1630             : /// Select a "register plus scaled unsigned 12-bit immediate" address.  The
    1631             : /// "Size" argument is the size in bytes of the memory reference, which
    1632             : /// determines the scale.
    1633           6 : InstructionSelector::ComplexRendererFns
    1634             : AArch64InstructionSelector::selectAddrModeIndexed(MachineOperand &Root,
    1635           3 :                                                   unsigned Size) const {
    1636           0 :   MachineRegisterInfo &MRI =
    1637             :       Root.getParent()->getParent()->getParent()->getRegInfo();
    1638           0 : 
    1639             :   if (!Root.isReg())
    1640           0 :     return None;
    1641             : 
    1642             :   MachineInstr *RootDef = MRI.getVRegDef(Root.getReg());
    1643             :   if (!RootDef)
    1644             :     return None;
    1645             : 
    1646             :   if (RootDef->getOpcode() == TargetOpcode::G_FRAME_INDEX) {
    1647             :     return {{
    1648             :         [=](MachineInstrBuilder &MIB) { MIB.add(RootDef->getOperand(1)); },
    1649         297 :         [=](MachineInstrBuilder &MIB) { MIB.addImm(0); },
    1650             :     }};
    1651             :   }
    1652         297 : 
    1653             :   if (isBaseWithConstantOffset(Root, MRI)) {
    1654         297 :     MachineOperand &LHS = RootDef->getOperand(1);
    1655             :     MachineOperand &RHS = RootDef->getOperand(2);
    1656             :     MachineInstr *LHSDef = MRI.getVRegDef(LHS.getReg());
    1657         297 :     MachineInstr *RHSDef = MRI.getVRegDef(RHS.getReg());
    1658         297 :     if (LHSDef && RHSDef) {
    1659             :       int64_t RHSC = (int64_t)RHSDef->getOperand(1).getCImm()->getZExtValue();
    1660             :       unsigned Scale = Log2_32(Size);
    1661         594 :       if ((RHSC & (Size - 1)) == 0 && RHSC >= 0 && RHSC < (0x1000 << Scale)) {
    1662             :         if (LHSDef->getOpcode() == TargetOpcode::G_FRAME_INDEX)
    1663          23 :           return {{
    1664             :               [=](MachineInstrBuilder &MIB) { MIB.add(LHSDef->getOperand(1)); },
    1665          92 :               [=](MachineInstrBuilder &MIB) { MIB.addImm(RHSC >> Scale); },
    1666             :           }};
    1667             : 
    1668         274 :         return {{
    1669         186 :             [=](MachineInstrBuilder &MIB) { MIB.add(LHS); },
    1670             :             [=](MachineInstrBuilder &MIB) { MIB.addImm(RHSC >> Scale); },
    1671         186 :         }};
    1672         186 :       }
    1673         186 :     }
    1674         372 :   }
    1675             : 
    1676         186 :   // Before falling back to our general case, check if the unscaled
    1677         366 :   // instructions can handle this. If so, that's preferable.
    1678             :   if (selectAddrModeUnscaled(Root, Size).hasValue())
    1679           0 :     return None;
    1680           0 : 
    1681           0 :   return {{
    1682             :       [=](MachineInstrBuilder &MIB) { MIB.add(Root); },
    1683             :       [=](MachineInstrBuilder &MIB) { MIB.addImm(0); },
    1684         183 :   }};
    1685         183 : }
    1686         732 : 
    1687             : void AArch64InstructionSelector::renderTruncImm(MachineInstrBuilder &MIB,
    1688             :                                                 const MachineInstr &MI) const {
    1689             :   const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
    1690             :   assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && "Expected G_CONSTANT");
    1691             :   Optional<int64_t> CstVal = getConstantVRegVal(MI.getOperand(0).getReg(), MRI);
    1692             :   assert(CstVal && "Expected constant value");
    1693          91 :   MIB.addImm(CstVal.getValue());
    1694             : }
    1695             : 
    1696             : namespace llvm {
    1697          91 : InstructionSelector *
    1698             : createAArch64InstructionSelector(const AArch64TargetMachine &TM,
    1699         364 :                                  AArch64Subtarget &Subtarget,
    1700             :                                  AArch64RegisterBankInfo &RBI) {
    1701             :   return new AArch64InstructionSelector(TM, Subtarget, RBI);
    1702           1 : }
    1703             : }

Generated by: LCOV version 1.13