LCOV - code coverage report
Current view: top level - lib/Target/AArch64 - AArch64InstructionSelector.cpp (source / functions) Hit Total Coverage
Test: llvm-toolchain.info Lines: 514 701 73.3 %
Date: 2018-07-13 00:08:38 Functions: 27 33 81.8 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : //===- AArch64InstructionSelector.cpp ----------------------------*- C++ -*-==//
       2             : //
       3             : //                     The LLVM Compiler Infrastructure
       4             : //
       5             : // This file is distributed under the University of Illinois Open Source
       6             : // License. See LICENSE.TXT for details.
       7             : //
       8             : //===----------------------------------------------------------------------===//
       9             : /// \file
      10             : /// This file implements the targeting of the InstructionSelector class for
      11             : /// AArch64.
      12             : /// \todo This should be generated by TableGen.
      13             : //===----------------------------------------------------------------------===//
      14             : 
      15             : #include "AArch64InstrInfo.h"
      16             : #include "AArch64MachineFunctionInfo.h"
      17             : #include "AArch64RegisterBankInfo.h"
      18             : #include "AArch64RegisterInfo.h"
      19             : #include "AArch64Subtarget.h"
      20             : #include "AArch64TargetMachine.h"
      21             : #include "MCTargetDesc/AArch64AddressingModes.h"
      22             : #include "llvm/CodeGen/GlobalISel/InstructionSelector.h"
      23             : #include "llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h"
      24             : #include "llvm/CodeGen/GlobalISel/Utils.h"
      25             : #include "llvm/CodeGen/MachineBasicBlock.h"
      26             : #include "llvm/CodeGen/MachineFunction.h"
      27             : #include "llvm/CodeGen/MachineInstr.h"
      28             : #include "llvm/CodeGen/MachineInstrBuilder.h"
      29             : #include "llvm/CodeGen/MachineOperand.h"
      30             : #include "llvm/CodeGen/MachineRegisterInfo.h"
      31             : #include "llvm/IR/Type.h"
      32             : #include "llvm/Support/Debug.h"
      33             : #include "llvm/Support/raw_ostream.h"
      34             : 
      35             : #define DEBUG_TYPE "aarch64-isel"
      36             : 
      37             : using namespace llvm;
      38             : 
      39             : namespace {
      40             : 
      41             : #define GET_GLOBALISEL_PREDICATE_BITSET
      42             : #include "AArch64GenGlobalISel.inc"
      43             : #undef GET_GLOBALISEL_PREDICATE_BITSET
      44             : 
      45        4224 : class AArch64InstructionSelector : public InstructionSelector {
      46             : public:
      47             :   AArch64InstructionSelector(const AArch64TargetMachine &TM,
      48             :                              const AArch64Subtarget &STI,
      49             :                              const AArch64RegisterBankInfo &RBI);
      50             : 
      51             :   bool select(MachineInstr &I, CodeGenCoverage &CoverageInfo) const override;
      52             :   static const char *getName() { return DEBUG_TYPE; }
      53             : 
      54             : private:
      55             :   /// tblgen-erated 'select' implementation, used as the initial selector for
      56             :   /// the patterns that don't require complex C++.
      57             :   bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const;
      58             : 
      59             :   bool selectVaStartAAPCS(MachineInstr &I, MachineFunction &MF,
      60             :                           MachineRegisterInfo &MRI) const;
      61             :   bool selectVaStartDarwin(MachineInstr &I, MachineFunction &MF,
      62             :                            MachineRegisterInfo &MRI) const;
      63             : 
      64             :   bool selectCompareBranch(MachineInstr &I, MachineFunction &MF,
      65             :                            MachineRegisterInfo &MRI) const;
      66             : 
      67             :   ComplexRendererFns selectArithImmed(MachineOperand &Root) const;
      68             : 
      69             :   ComplexRendererFns selectAddrModeUnscaled(MachineOperand &Root,
      70             :                                             unsigned Size) const;
      71             : 
      72           0 :   ComplexRendererFns selectAddrModeUnscaled8(MachineOperand &Root) const {
      73           0 :     return selectAddrModeUnscaled(Root, 1);
      74             :   }
      75           0 :   ComplexRendererFns selectAddrModeUnscaled16(MachineOperand &Root) const {
      76           0 :     return selectAddrModeUnscaled(Root, 2);
      77             :   }
      78           0 :   ComplexRendererFns selectAddrModeUnscaled32(MachineOperand &Root) const {
      79           0 :     return selectAddrModeUnscaled(Root, 4);
      80             :   }
      81           0 :   ComplexRendererFns selectAddrModeUnscaled64(MachineOperand &Root) const {
      82           0 :     return selectAddrModeUnscaled(Root, 8);
      83             :   }
      84           0 :   ComplexRendererFns selectAddrModeUnscaled128(MachineOperand &Root) const {
      85           0 :     return selectAddrModeUnscaled(Root, 16);
      86             :   }
      87             : 
      88             :   ComplexRendererFns selectAddrModeIndexed(MachineOperand &Root,
      89             :                                            unsigned Size) const;
      90             :   template <int Width>
      91         111 :   ComplexRendererFns selectAddrModeIndexed(MachineOperand &Root) const {
      92         111 :     return selectAddrModeIndexed(Root, Width / 8);
      93             :   }
      94             : 
      95             :   void renderTruncImm(MachineInstrBuilder &MIB, const MachineInstr &MI) const;
      96             : 
      97             :   const AArch64TargetMachine &TM;
      98             :   const AArch64Subtarget &STI;
      99             :   const AArch64InstrInfo &TII;
     100             :   const AArch64RegisterInfo &TRI;
     101             :   const AArch64RegisterBankInfo &RBI;
     102             : 
     103             : #define GET_GLOBALISEL_PREDICATES_DECL
     104             : #include "AArch64GenGlobalISel.inc"
     105             : #undef GET_GLOBALISEL_PREDICATES_DECL
     106             : 
     107             : // We declare the temporaries used by selectImpl() in the class to minimize the
     108             : // cost of constructing placeholder values.
     109             : #define GET_GLOBALISEL_TEMPORARIES_DECL
     110             : #include "AArch64GenGlobalISel.inc"
     111             : #undef GET_GLOBALISEL_TEMPORARIES_DECL
     112             : };
     113             : 
     114             : } // end anonymous namespace
     115             : 
     116             : #define GET_GLOBALISEL_IMPL
     117             : #include "AArch64GenGlobalISel.inc"
     118             : #undef GET_GLOBALISEL_IMPL
     119             : 
     120        1438 : AArch64InstructionSelector::AArch64InstructionSelector(
     121             :     const AArch64TargetMachine &TM, const AArch64Subtarget &STI,
     122        1438 :     const AArch64RegisterBankInfo &RBI)
     123             :     : InstructionSelector(), TM(TM), STI(STI), TII(*STI.getInstrInfo()),
     124             :       TRI(*STI.getRegisterInfo()), RBI(RBI),
     125             : #define GET_GLOBALISEL_PREDICATES_INIT
     126             : #include "AArch64GenGlobalISel.inc"
     127             : #undef GET_GLOBALISEL_PREDICATES_INIT
     128             : #define GET_GLOBALISEL_TEMPORARIES_INIT
     129             : #include "AArch64GenGlobalISel.inc"
     130             : #undef GET_GLOBALISEL_TEMPORARIES_INIT
     131             : {
     132        1438 : }
     133             : 
     134             : // FIXME: This should be target-independent, inferred from the types declared
     135             : // for each class in the bank.
     136             : static const TargetRegisterClass *
     137         854 : getRegClassForTypeOnBank(LLT Ty, const RegisterBank &RB,
     138             :                          const RegisterBankInfo &RBI,
     139             :                          bool GetAllRegSet = false) {
     140         854 :   if (RB.getID() == AArch64::GPRRegBankID) {
     141         479 :     if (Ty.getSizeInBits() <= 32)
     142         291 :       return GetAllRegSet ? &AArch64::GPR32allRegClass
     143             :                           : &AArch64::GPR32RegClass;
     144         188 :     if (Ty.getSizeInBits() == 64)
     145         188 :       return GetAllRegSet ? &AArch64::GPR64allRegClass
     146             :                           : &AArch64::GPR64RegClass;
     147             :     return nullptr;
     148             :   }
     149             : 
     150         375 :   if (RB.getID() == AArch64::FPRRegBankID) {
     151         375 :     if (Ty.getSizeInBits() <= 16)
     152             :       return &AArch64::FPR16RegClass;
     153         363 :     if (Ty.getSizeInBits() == 32)
     154             :       return &AArch64::FPR32RegClass;
     155         303 :     if (Ty.getSizeInBits() == 64)
     156             :       return &AArch64::FPR64RegClass;
     157         116 :     if (Ty.getSizeInBits() == 128)
     158             :       return &AArch64::FPR128RegClass;
     159           0 :     return nullptr;
     160             :   }
     161             : 
     162             :   return nullptr;
     163             : }
     164             : 
     165             : /// Check whether \p I is a currently unsupported binary operation:
     166             : /// - it has an unsized type
     167             : /// - an operand is not a vreg
     168             : /// - all operands are not in the same bank
     169             : /// These are checks that should someday live in the verifier, but right now,
     170             : /// these are mostly limitations of the aarch64 selector.
     171          16 : static bool unsupportedBinOp(const MachineInstr &I,
     172             :                              const AArch64RegisterBankInfo &RBI,
     173             :                              const MachineRegisterInfo &MRI,
     174             :                              const AArch64RegisterInfo &TRI) {
     175          16 :   LLT Ty = MRI.getType(I.getOperand(0).getReg());
     176          16 :   if (!Ty.isValid()) {
     177             :     LLVM_DEBUG(dbgs() << "Generic binop register should be typed\n");
     178             :     return true;
     179             :   }
     180             : 
     181             :   const RegisterBank *PrevOpBank = nullptr;
     182         112 :   for (auto &MO : I.operands()) {
     183             :     // FIXME: Support non-register operands.
     184          48 :     if (!MO.isReg()) {
     185             :       LLVM_DEBUG(dbgs() << "Generic inst non-reg operands are unsupported\n");
     186             :       return true;
     187             :     }
     188             : 
     189             :     // FIXME: Can generic operations have physical registers operands? If
     190             :     // so, this will need to be taught about that, and we'll need to get the
     191             :     // bank out of the minimal class for the register.
     192             :     // Either way, this needs to be documented (and possibly verified).
     193          96 :     if (!TargetRegisterInfo::isVirtualRegister(MO.getReg())) {
     194             :       LLVM_DEBUG(dbgs() << "Generic inst has physical register operand\n");
     195             :       return true;
     196             :     }
     197             : 
     198          48 :     const RegisterBank *OpBank = RBI.getRegBank(MO.getReg(), MRI, TRI);
     199          48 :     if (!OpBank) {
     200             :       LLVM_DEBUG(dbgs() << "Generic register has no bank or class\n");
     201             :       return true;
     202             :     }
     203             : 
     204          48 :     if (PrevOpBank && OpBank != PrevOpBank) {
     205             :       LLVM_DEBUG(dbgs() << "Generic inst operands have different banks\n");
     206             :       return true;
     207             :     }
     208             :     PrevOpBank = OpBank;
     209             :   }
     210             :   return false;
     211             : }
     212             : 
     213             : /// Select the AArch64 opcode for the basic binary operation \p GenericOpc
     214             : /// (such as G_OR or G_SDIV), appropriate for the register bank \p RegBankID
     215             : /// and of size \p OpSize.
     216             : /// \returns \p GenericOpc if the combination is unsupported.
     217          14 : static unsigned selectBinaryOp(unsigned GenericOpc, unsigned RegBankID,
     218             :                                unsigned OpSize) {
     219          14 :   switch (RegBankID) {
     220          14 :   case AArch64::GPRRegBankID:
     221          14 :     if (OpSize == 32) {
     222           4 :       switch (GenericOpc) {
     223             :       case TargetOpcode::G_SHL:
     224             :         return AArch64::LSLVWr;
     225           1 :       case TargetOpcode::G_LSHR:
     226           1 :         return AArch64::LSRVWr;
     227           1 :       case TargetOpcode::G_ASHR:
     228           1 :         return AArch64::ASRVWr;
     229           0 :       default:
     230           0 :         return GenericOpc;
     231             :       }
     232          10 :     } else if (OpSize == 64) {
     233          10 :       switch (GenericOpc) {
     234             :       case TargetOpcode::G_GEP:
     235             :         return AArch64::ADDXrr;
     236           0 :       case TargetOpcode::G_SHL:
     237           0 :         return AArch64::LSLVXr;
     238           0 :       case TargetOpcode::G_LSHR:
     239           0 :         return AArch64::LSRVXr;
     240           0 :       case TargetOpcode::G_ASHR:
     241           0 :         return AArch64::ASRVXr;
     242           0 :       default:
     243           0 :         return GenericOpc;
     244             :       }
     245             :     }
     246             :     break;
     247           0 :   case AArch64::FPRRegBankID:
     248           0 :     switch (OpSize) {
     249           0 :     case 32:
     250           0 :       switch (GenericOpc) {
     251             :       case TargetOpcode::G_FADD:
     252             :         return AArch64::FADDSrr;
     253           0 :       case TargetOpcode::G_FSUB:
     254           0 :         return AArch64::FSUBSrr;
     255           0 :       case TargetOpcode::G_FMUL:
     256           0 :         return AArch64::FMULSrr;
     257           0 :       case TargetOpcode::G_FDIV:
     258           0 :         return AArch64::FDIVSrr;
     259           0 :       default:
     260           0 :         return GenericOpc;
     261             :       }
     262           0 :     case 64:
     263           0 :       switch (GenericOpc) {
     264             :       case TargetOpcode::G_FADD:
     265             :         return AArch64::FADDDrr;
     266           0 :       case TargetOpcode::G_FSUB:
     267           0 :         return AArch64::FSUBDrr;
     268           0 :       case TargetOpcode::G_FMUL:
     269           0 :         return AArch64::FMULDrr;
     270           0 :       case TargetOpcode::G_FDIV:
     271           0 :         return AArch64::FDIVDrr;
     272           0 :       case TargetOpcode::G_OR:
     273           0 :         return AArch64::ORRv8i8;
     274           0 :       default:
     275           0 :         return GenericOpc;
     276             :       }
     277             :     }
     278             :     break;
     279             :   }
     280             :   return GenericOpc;
     281             : }
     282             : 
     283             : /// Select the AArch64 opcode for the G_LOAD or G_STORE operation \p GenericOpc,
     284             : /// appropriate for the (value) register bank \p RegBankID and of memory access
     285             : /// size \p OpSize.  This returns the variant with the base+unsigned-immediate
     286             : /// addressing mode (e.g., LDRXui).
     287             : /// \returns \p GenericOpc if the combination is unsupported.
     288          39 : static unsigned selectLoadStoreUIOp(unsigned GenericOpc, unsigned RegBankID,
     289             :                                     unsigned OpSize) {
     290             :   const bool isStore = GenericOpc == TargetOpcode::G_STORE;
     291          39 :   switch (RegBankID) {
     292          36 :   case AArch64::GPRRegBankID:
     293          36 :     switch (OpSize) {
     294          22 :     case 8:
     295          22 :       return isStore ? AArch64::STRBBui : AArch64::LDRBBui;
     296           7 :     case 16:
     297           7 :       return isStore ? AArch64::STRHHui : AArch64::LDRHHui;
     298           0 :     case 32:
     299           0 :       return isStore ? AArch64::STRWui : AArch64::LDRWui;
     300           7 :     case 64:
     301           7 :       return isStore ? AArch64::STRXui : AArch64::LDRXui;
     302             :     }
     303             :     break;
     304           3 :   case AArch64::FPRRegBankID:
     305           3 :     switch (OpSize) {
     306           3 :     case 8:
     307           3 :       return isStore ? AArch64::STRBui : AArch64::LDRBui;
     308           0 :     case 16:
     309           0 :       return isStore ? AArch64::STRHui : AArch64::LDRHui;
     310           0 :     case 32:
     311           0 :       return isStore ? AArch64::STRSui : AArch64::LDRSui;
     312           0 :     case 64:
     313           0 :       return isStore ? AArch64::STRDui : AArch64::LDRDui;
     314             :     }
     315             :     break;
     316             :   }
     317             :   return GenericOpc;
     318             : }
     319             : 
     320           4 : static bool selectFP16CopyFromGPR32(MachineInstr &I, const TargetInstrInfo &TII,
     321             :                                     MachineRegisterInfo &MRI, unsigned SrcReg) {
     322             :   // Copies from gpr32 to fpr16 need to use a sub-register copy.
     323           4 :   unsigned CopyReg = MRI.createVirtualRegister(&AArch64::FPR32RegClass);
     324          12 :   BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(AArch64::COPY))
     325             :       .addDef(CopyReg)
     326             :       .addUse(SrcReg);
     327           4 :   unsigned SubRegCopy = MRI.createVirtualRegister(&AArch64::FPR16RegClass);
     328          12 :   BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(TargetOpcode::COPY))
     329             :       .addDef(SubRegCopy)
     330             :       .addUse(CopyReg, 0, AArch64::hsub);
     331             : 
     332           4 :   MachineOperand &RegOp = I.getOperand(1);
     333           4 :   RegOp.setReg(SubRegCopy);
     334           4 :   return true;
     335             : }
     336             : 
     337         916 : static bool selectCopy(MachineInstr &I, const TargetInstrInfo &TII,
     338             :                        MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI,
     339             :                        const RegisterBankInfo &RBI) {
     340             : 
     341         916 :   unsigned DstReg = I.getOperand(0).getReg();
     342         916 :   unsigned SrcReg = I.getOperand(1).getReg();
     343             : 
     344         916 :   if (TargetRegisterInfo::isPhysicalRegister(DstReg)) {
     345         545 :     if (TRI.getRegClass(AArch64::FPR16RegClassID)->contains(DstReg) &&
     346             :         !TargetRegisterInfo::isPhysicalRegister(SrcReg)) {
     347           8 :       const RegisterBank &RegBank = *RBI.getRegBank(SrcReg, MRI, TRI);
     348             :       const TargetRegisterClass *SrcRC = getRegClassForTypeOnBank(
     349           8 :           MRI.getType(SrcReg), RegBank, RBI, /* GetAllRegSet */ true);
     350           8 :       if (SrcRC == &AArch64::GPR32allRegClass)
     351           3 :         return selectFP16CopyFromGPR32(I, TII, MRI, SrcReg);
     352             :     }
     353             :     assert(I.isCopy() && "Generic operators do not allow physical registers");
     354             :     return true;
     355             :   }
     356             : 
     357         664 :   const RegisterBank &RegBank = *RBI.getRegBank(DstReg, MRI, TRI);
     358         664 :   const unsigned DstSize = MRI.getType(DstReg).getSizeInBits();
     359             :   (void)DstSize;
     360         664 :   const unsigned SrcSize = RBI.getSizeInBits(SrcReg, MRI, TRI);
     361             :   (void)SrcSize;
     362             :   assert((!TargetRegisterInfo::isPhysicalRegister(SrcReg) || I.isCopy()) &&
     363             :          "No phys reg on generic operators");
     364             :   assert(
     365             :       (DstSize == SrcSize ||
     366             :        // Copies are a mean to setup initial types, the number of
     367             :        // bits may not exactly match.
     368             :        (TargetRegisterInfo::isPhysicalRegister(SrcReg) &&
     369             :         DstSize <= RBI.getSizeInBits(SrcReg, MRI, TRI)) ||
     370             :        // Copies are a mean to copy bits around, as long as we are
     371             :        // on the same register class, that's fine. Otherwise, that
     372             :        // means we need some SUBREG_TO_REG or AND & co.
     373             :        (((DstSize + 31) / 32 == (SrcSize + 31) / 32) && DstSize > SrcSize)) &&
     374             :       "Copy with different width?!");
     375             :   assert((DstSize <= 64 || RegBank.getID() == AArch64::FPRRegBankID) &&
     376             :          "GPRs cannot get more than 64-bit width values");
     377             : 
     378             :   const TargetRegisterClass *RC = getRegClassForTypeOnBank(
     379         664 :       MRI.getType(DstReg), RegBank, RBI, /* GetAllRegSet */ true);
     380         664 :   if (!RC) {
     381             :     LLVM_DEBUG(dbgs() << "Unexpected bitcast size " << DstSize << '\n');
     382             :     return false;
     383             :   }
     384             : 
     385         664 :   if (!TargetRegisterInfo::isPhysicalRegister(SrcReg)) {
     386             :     const RegClassOrRegBank &RegClassOrBank = MRI.getRegClassOrRegBank(SrcReg);
     387             :     const TargetRegisterClass *SrcRC =
     388             :         RegClassOrBank.dyn_cast<const TargetRegisterClass *>();
     389             :     const RegisterBank *RB = nullptr;
     390           2 :     if (!SrcRC) {
     391             :       RB = RegClassOrBank.get<const RegisterBank *>();
     392          58 :       SrcRC = getRegClassForTypeOnBank(MRI.getType(SrcReg), *RB, RBI, true);
     393             :     }
     394             :     // Copies from fpr16 to gpr32 need to use SUBREG_TO_REG.
     395          60 :     if (RC == &AArch64::GPR32allRegClass && SrcRC == &AArch64::FPR16RegClass) {
     396           2 :       unsigned PromoteReg = MRI.createVirtualRegister(&AArch64::FPR32RegClass);
     397           4 :       BuildMI(*I.getParent(), I, I.getDebugLoc(),
     398           2 :               TII.get(AArch64::SUBREG_TO_REG))
     399             :           .addDef(PromoteReg)
     400             :           .addImm(0)
     401             :           .addUse(SrcReg)
     402             :           .addImm(AArch64::hsub);
     403           2 :       MachineOperand &RegOp = I.getOperand(1);
     404           2 :       RegOp.setReg(PromoteReg);
     405          58 :     } else if (RC == &AArch64::FPR16RegClass &&
     406             :                SrcRC == &AArch64::GPR32allRegClass) {
     407           1 :       selectFP16CopyFromGPR32(I, TII, MRI, SrcReg);
     408             :     }
     409             :   }
     410             : 
     411             :   // No need to constrain SrcReg. It will get constrained when
     412             :   // we hit another of its use or its defs.
     413             :   // Copies do not have constraints.
     414         664 :   if (!RBI.constrainGenericRegister(DstReg, *RC, MRI)) {
     415             :     LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())
     416             :                       << " operand\n");
     417             :     return false;
     418             :   }
     419         664 :   I.setDesc(TII.get(AArch64::COPY));
     420         664 :   return true;
     421             : }
     422             : 
     423           0 : static unsigned selectFPConvOpc(unsigned GenericOpc, LLT DstTy, LLT SrcTy) {
     424             :   if (!DstTy.isScalar() || !SrcTy.isScalar())
     425             :     return GenericOpc;
     426             : 
     427           0 :   const unsigned DstSize = DstTy.getSizeInBits();
     428           0 :   const unsigned SrcSize = SrcTy.getSizeInBits();
     429             : 
     430           0 :   switch (DstSize) {
     431           0 :   case 32:
     432           0 :     switch (SrcSize) {
     433           0 :     case 32:
     434           0 :       switch (GenericOpc) {
     435             :       case TargetOpcode::G_SITOFP:
     436             :         return AArch64::SCVTFUWSri;
     437           0 :       case TargetOpcode::G_UITOFP:
     438           0 :         return AArch64::UCVTFUWSri;
     439           0 :       case TargetOpcode::G_FPTOSI:
     440           0 :         return AArch64::FCVTZSUWSr;
     441           0 :       case TargetOpcode::G_FPTOUI:
     442           0 :         return AArch64::FCVTZUUWSr;
     443           0 :       default:
     444           0 :         return GenericOpc;
     445             :       }
     446           0 :     case 64:
     447           0 :       switch (GenericOpc) {
     448             :       case TargetOpcode::G_SITOFP:
     449             :         return AArch64::SCVTFUXSri;
     450           0 :       case TargetOpcode::G_UITOFP:
     451           0 :         return AArch64::UCVTFUXSri;
     452           0 :       case TargetOpcode::G_FPTOSI:
     453           0 :         return AArch64::FCVTZSUWDr;
     454           0 :       case TargetOpcode::G_FPTOUI:
     455           0 :         return AArch64::FCVTZUUWDr;
     456           0 :       default:
     457           0 :         return GenericOpc;
     458             :       }
     459             :     default:
     460             :       return GenericOpc;
     461             :     }
     462           0 :   case 64:
     463           0 :     switch (SrcSize) {
     464           0 :     case 32:
     465           0 :       switch (GenericOpc) {
     466             :       case TargetOpcode::G_SITOFP:
     467             :         return AArch64::SCVTFUWDri;
     468           0 :       case TargetOpcode::G_UITOFP:
     469           0 :         return AArch64::UCVTFUWDri;
     470           0 :       case TargetOpcode::G_FPTOSI:
     471           0 :         return AArch64::FCVTZSUXSr;
     472           0 :       case TargetOpcode::G_FPTOUI:
     473           0 :         return AArch64::FCVTZUUXSr;
     474           0 :       default:
     475           0 :         return GenericOpc;
     476             :       }
     477           0 :     case 64:
     478           0 :       switch (GenericOpc) {
     479             :       case TargetOpcode::G_SITOFP:
     480             :         return AArch64::SCVTFUXDri;
     481           0 :       case TargetOpcode::G_UITOFP:
     482           0 :         return AArch64::UCVTFUXDri;
     483           0 :       case TargetOpcode::G_FPTOSI:
     484           0 :         return AArch64::FCVTZSUXDr;
     485           0 :       case TargetOpcode::G_FPTOUI:
     486           0 :         return AArch64::FCVTZUUXDr;
     487           0 :       default:
     488           0 :         return GenericOpc;
     489             :       }
     490             :     default:
     491             :       return GenericOpc;
     492             :     }
     493             :   default:
     494             :     return GenericOpc;
     495             :   };
     496             :   return GenericOpc;
     497             : }
     498             : 
     499          14 : static AArch64CC::CondCode changeICMPPredToAArch64CC(CmpInst::Predicate P) {
     500          14 :   switch (P) {
     501           0 :   default:
     502           0 :     llvm_unreachable("Unknown condition code!");
     503             :   case CmpInst::ICMP_NE:
     504             :     return AArch64CC::NE;
     505           3 :   case CmpInst::ICMP_EQ:
     506           3 :     return AArch64CC::EQ;
     507           0 :   case CmpInst::ICMP_SGT:
     508           0 :     return AArch64CC::GT;
     509           1 :   case CmpInst::ICMP_SGE:
     510           1 :     return AArch64CC::GE;
     511           0 :   case CmpInst::ICMP_SLT:
     512           0 :     return AArch64CC::LT;
     513           0 :   case CmpInst::ICMP_SLE:
     514           0 :     return AArch64CC::LE;
     515           0 :   case CmpInst::ICMP_UGT:
     516           0 :     return AArch64CC::HI;
     517           3 :   case CmpInst::ICMP_UGE:
     518           3 :     return AArch64CC::HS;
     519           3 :   case CmpInst::ICMP_ULT:
     520           3 :     return AArch64CC::LO;
     521           1 :   case CmpInst::ICMP_ULE:
     522           1 :     return AArch64CC::LS;
     523             :   }
     524             : }
     525             : 
     526           6 : static void changeFCMPPredToAArch64CC(CmpInst::Predicate P,
     527             :                                       AArch64CC::CondCode &CondCode,
     528             :                                       AArch64CC::CondCode &CondCode2) {
     529           6 :   CondCode2 = AArch64CC::AL;
     530           6 :   switch (P) {
     531           0 :   default:
     532           0 :     llvm_unreachable("Unknown FP condition!");
     533           0 :   case CmpInst::FCMP_OEQ:
     534           0 :     CondCode = AArch64CC::EQ;
     535           0 :     break;
     536           0 :   case CmpInst::FCMP_OGT:
     537           0 :     CondCode = AArch64CC::GT;
     538           0 :     break;
     539           0 :   case CmpInst::FCMP_OGE:
     540           0 :     CondCode = AArch64CC::GE;
     541           0 :     break;
     542           0 :   case CmpInst::FCMP_OLT:
     543           0 :     CondCode = AArch64CC::MI;
     544           0 :     break;
     545           0 :   case CmpInst::FCMP_OLE:
     546           0 :     CondCode = AArch64CC::LS;
     547           0 :     break;
     548           3 :   case CmpInst::FCMP_ONE:
     549           3 :     CondCode = AArch64CC::MI;
     550           3 :     CondCode2 = AArch64CC::GT;
     551           3 :     break;
     552           0 :   case CmpInst::FCMP_ORD:
     553           0 :     CondCode = AArch64CC::VC;
     554           0 :     break;
     555           0 :   case CmpInst::FCMP_UNO:
     556           0 :     CondCode = AArch64CC::VS;
     557           0 :     break;
     558           0 :   case CmpInst::FCMP_UEQ:
     559           0 :     CondCode = AArch64CC::EQ;
     560           0 :     CondCode2 = AArch64CC::VS;
     561           0 :     break;
     562           0 :   case CmpInst::FCMP_UGT:
     563           0 :     CondCode = AArch64CC::HI;
     564           0 :     break;
     565           3 :   case CmpInst::FCMP_UGE:
     566           3 :     CondCode = AArch64CC::PL;
     567           3 :     break;
     568           0 :   case CmpInst::FCMP_ULT:
     569           0 :     CondCode = AArch64CC::LT;
     570           0 :     break;
     571           0 :   case CmpInst::FCMP_ULE:
     572           0 :     CondCode = AArch64CC::LE;
     573           0 :     break;
     574           0 :   case CmpInst::FCMP_UNE:
     575           0 :     CondCode = AArch64CC::NE;
     576           0 :     break;
     577             :   }
     578           6 : }
     579             : 
     580          17 : bool AArch64InstructionSelector::selectCompareBranch(
     581             :     MachineInstr &I, MachineFunction &MF, MachineRegisterInfo &MRI) const {
     582             : 
     583          17 :   const unsigned CondReg = I.getOperand(0).getReg();
     584          17 :   MachineBasicBlock *DestMBB = I.getOperand(1).getMBB();
     585          17 :   MachineInstr *CCMI = MRI.getVRegDef(CondReg);
     586          34 :   if (CCMI->getOpcode() == TargetOpcode::G_TRUNC)
     587          17 :     CCMI = MRI.getVRegDef(CCMI->getOperand(1).getReg());
     588          34 :   if (CCMI->getOpcode() != TargetOpcode::G_ICMP)
     589             :     return false;
     590             : 
     591           9 :   unsigned LHS = CCMI->getOperand(2).getReg();
     592           9 :   unsigned RHS = CCMI->getOperand(3).getReg();
     593           9 :   if (!getConstantVRegVal(RHS, MRI))
     594             :     std::swap(RHS, LHS);
     595             : 
     596           9 :   const auto RHSImm = getConstantVRegVal(RHS, MRI);
     597           9 :   if (!RHSImm || *RHSImm != 0)
     598             :     return false;
     599             : 
     600           9 :   const RegisterBank &RB = *RBI.getRegBank(LHS, MRI, TRI);
     601           9 :   if (RB.getID() != AArch64::GPRRegBankID)
     602             :     return false;
     603             : 
     604           9 :   const auto Pred = (CmpInst::Predicate)CCMI->getOperand(1).getPredicate();
     605           9 :   if (Pred != CmpInst::ICMP_NE && Pred != CmpInst::ICMP_EQ)
     606             :     return false;
     607             : 
     608           4 :   const unsigned CmpWidth = MRI.getType(LHS).getSizeInBits();
     609             :   unsigned CBOpc = 0;
     610           4 :   if (CmpWidth <= 32)
     611           2 :     CBOpc = (Pred == CmpInst::ICMP_EQ ? AArch64::CBZW : AArch64::CBNZW);
     612           2 :   else if (CmpWidth == 64)
     613           2 :     CBOpc = (Pred == CmpInst::ICMP_EQ ? AArch64::CBZX : AArch64::CBNZX);
     614             :   else
     615             :     return false;
     616             : 
     617          12 :   BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(CBOpc))
     618             :       .addUse(LHS)
     619             :       .addMBB(DestMBB)
     620           4 :       .constrainAllUses(TII, TRI, RBI);
     621             : 
     622           4 :   I.eraseFromParent();
     623             :   return true;
     624             : }
     625             : 
     626             : bool AArch64InstructionSelector::selectVaStartAAPCS(
     627             :     MachineInstr &I, MachineFunction &MF, MachineRegisterInfo &MRI) const {
     628             :   return false;
     629             : }
     630             : 
     631           1 : bool AArch64InstructionSelector::selectVaStartDarwin(
     632             :     MachineInstr &I, MachineFunction &MF, MachineRegisterInfo &MRI) const {
     633           1 :   AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>();
     634           1 :   unsigned ListReg = I.getOperand(0).getReg();
     635             : 
     636           1 :   unsigned ArgsAddrReg = MRI.createVirtualRegister(&AArch64::GPR64RegClass);
     637             : 
     638             :   auto MIB =
     639           3 :       BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(AArch64::ADDXri))
     640             :           .addDef(ArgsAddrReg)
     641           1 :           .addFrameIndex(FuncInfo->getVarArgsStackIndex())
     642             :           .addImm(0)
     643           1 :           .addImm(0);
     644             : 
     645           1 :   constrainSelectedInstRegOperands(*MIB, TII, TRI, RBI);
     646             : 
     647           3 :   MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(AArch64::STRXui))
     648             :             .addUse(ArgsAddrReg)
     649             :             .addUse(ListReg)
     650             :             .addImm(0)
     651           1 :             .addMemOperand(*I.memoperands_begin());
     652             : 
     653           1 :   constrainSelectedInstRegOperands(*MIB, TII, TRI, RBI);
     654           1 :   I.eraseFromParent();
     655           1 :   return true;
     656             : }
     657             : 
     658        1946 : bool AArch64InstructionSelector::select(MachineInstr &I,
     659             :                                         CodeGenCoverage &CoverageInfo) const {
     660             :   assert(I.getParent() && "Instruction should be in a basic block!");
     661             :   assert(I.getParent()->getParent() && "Instruction should be in a function!");
     662             : 
     663        1946 :   MachineBasicBlock &MBB = *I.getParent();
     664        1946 :   MachineFunction &MF = *MBB.getParent();
     665        1946 :   MachineRegisterInfo &MRI = MF.getRegInfo();
     666             : 
     667        1946 :   unsigned Opcode = I.getOpcode();
     668             :   // G_PHI requires same handling as PHI
     669        1946 :   if (!isPreISelGenericOpcode(Opcode) || Opcode == TargetOpcode::G_PHI) {
     670             :     // Certain non-generic instructions also need some special handling.
     671             : 
     672        1143 :     if (Opcode ==  TargetOpcode::LOAD_STACK_GUARD)
     673           0 :       return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
     674             : 
     675        1143 :     if (Opcode == TargetOpcode::PHI || Opcode == TargetOpcode::G_PHI) {
     676           9 :       const unsigned DefReg = I.getOperand(0).getReg();
     677             :       const LLT DefTy = MRI.getType(DefReg);
     678             : 
     679             :       const TargetRegisterClass *DefRC = nullptr;
     680           9 :       if (TargetRegisterInfo::isPhysicalRegister(DefReg)) {
     681           0 :         DefRC = TRI.getRegClass(DefReg);
     682             :       } else {
     683             :         const RegClassOrRegBank &RegClassOrBank =
     684             :             MRI.getRegClassOrRegBank(DefReg);
     685             : 
     686             :         DefRC = RegClassOrBank.dyn_cast<const TargetRegisterClass *>();
     687           2 :         if (!DefRC) {
     688           7 :           if (!DefTy.isValid()) {
     689             :             LLVM_DEBUG(dbgs() << "PHI operand has no type, not a gvreg?\n");
     690             :             return false;
     691             :           }
     692             :           const RegisterBank &RB = *RegClassOrBank.get<const RegisterBank *>();
     693           7 :           DefRC = getRegClassForTypeOnBank(DefTy, RB, RBI);
     694           7 :           if (!DefRC) {
     695             :             LLVM_DEBUG(dbgs() << "PHI operand has unexpected size/bank\n");
     696             :             return false;
     697             :           }
     698             :         }
     699             :       }
     700           9 :       I.setDesc(TII.get(TargetOpcode::PHI));
     701             : 
     702           9 :       return RBI.constrainGenericRegister(DefReg, *DefRC, MRI);
     703             :     }
     704             : 
     705        1134 :     if (I.isCopy())
     706         882 :       return selectCopy(I, TII, MRI, TRI, RBI);
     707             : 
     708             :     return true;
     709             :   }
     710             : 
     711             : 
     712         803 :   if (I.getNumOperands() != I.getNumExplicitOperands()) {
     713             :     LLVM_DEBUG(
     714             :         dbgs() << "Generic instruction has unexpected implicit operands\n");
     715             :     return false;
     716             :   }
     717             : 
     718         803 :   if (selectImpl(I, CoverageInfo))
     719             :     return true;
     720             : 
     721             :   LLT Ty =
     722         911 :       I.getOperand(0).isReg() ? MRI.getType(I.getOperand(0).getReg()) : LLT{};
     723             : 
     724         304 :   switch (Opcode) {
     725          17 :   case TargetOpcode::G_BRCOND: {
     726          17 :     if (Ty.getSizeInBits() > 32) {
     727             :       // We shouldn't need this on AArch64, but it would be implemented as an
     728             :       // EXTRACT_SUBREG followed by a TBNZW because TBNZX has no encoding if the
     729             :       // bit being tested is < 32.
     730             :       LLVM_DEBUG(dbgs() << "G_BRCOND has type: " << Ty
     731             :                         << ", expected at most 32-bits");
     732             :       return false;
     733             :     }
     734             : 
     735          17 :     const unsigned CondReg = I.getOperand(0).getReg();
     736          17 :     MachineBasicBlock *DestMBB = I.getOperand(1).getMBB();
     737             : 
     738          17 :     if (selectCompareBranch(I, MF, MRI))
     739             :       return true;
     740             : 
     741          39 :     auto MIB = BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::TBNZW))
     742             :                    .addUse(CondReg)
     743             :                    .addImm(/*bit offset=*/0)
     744          13 :                    .addMBB(DestMBB);
     745             : 
     746          13 :     I.eraseFromParent();
     747          13 :     return constrainSelectedInstRegOperands(*MIB.getInstr(), TII, TRI, RBI);
     748             :   }
     749             : 
     750           2 :   case TargetOpcode::G_BRINDIRECT: {
     751           2 :     I.setDesc(TII.get(AArch64::BR));
     752           2 :     return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
     753             :   }
     754             : 
     755          28 :   case TargetOpcode::G_FCONSTANT:
     756             :   case TargetOpcode::G_CONSTANT: {
     757             :     const bool isFP = Opcode == TargetOpcode::G_FCONSTANT;
     758             : 
     759             :     const LLT s32 = LLT::scalar(32);
     760             :     const LLT s64 = LLT::scalar(64);
     761             :     const LLT p0 = LLT::pointer(0, 64);
     762             : 
     763          28 :     const unsigned DefReg = I.getOperand(0).getReg();
     764          28 :     const LLT DefTy = MRI.getType(DefReg);
     765          28 :     const unsigned DefSize = DefTy.getSizeInBits();
     766          28 :     const RegisterBank &RB = *RBI.getRegBank(DefReg, MRI, TRI);
     767             : 
     768             :     // FIXME: Redundant check, but even less readable when factored out.
     769          28 :     if (isFP) {
     770             :       if (Ty != s32 && Ty != s64) {
     771             :         LLVM_DEBUG(dbgs() << "Unable to materialize FP " << Ty
     772             :                           << " constant, expected: " << s32 << " or " << s64
     773             :                           << '\n');
     774             :         return false;
     775             :       }
     776             : 
     777          28 :       if (RB.getID() != AArch64::FPRRegBankID) {
     778             :         LLVM_DEBUG(dbgs() << "Unable to materialize FP " << Ty
     779             :                           << " constant on bank: " << RB
     780             :                           << ", expected: FPR\n");
     781             :         return false;
     782             :       }
     783             : 
     784             :       // The case when we have 0.0 is covered by tablegen. Reject it here so we
     785             :       // can be sure tablegen works correctly and isn't rescued by this code.
     786          56 :       if (I.getOperand(1).getFPImm()->getValueAPF().isExactlyValue(0.0))
     787             :         return false;
     788             :     } else {
     789             :       // s32 and s64 are covered by tablegen.
     790             :       if (Ty != p0) {
     791             :         LLVM_DEBUG(dbgs() << "Unable to materialize integer " << Ty
     792             :                           << " constant, expected: " << s32 << ", " << s64
     793             :                           << ", or " << p0 << '\n');
     794             :         return false;
     795             :       }
     796             : 
     797           0 :       if (RB.getID() != AArch64::GPRRegBankID) {
     798             :         LLVM_DEBUG(dbgs() << "Unable to materialize integer " << Ty
     799             :                           << " constant on bank: " << RB
     800             :                           << ", expected: GPR\n");
     801             :         return false;
     802             :       }
     803             :     }
     804             : 
     805             :     const unsigned MovOpc =
     806          28 :         DefSize == 32 ? AArch64::MOVi32imm : AArch64::MOVi64imm;
     807             : 
     808          28 :     I.setDesc(TII.get(MovOpc));
     809             : 
     810          28 :     if (isFP) {
     811          28 :       const TargetRegisterClass &GPRRC =
     812             :           DefSize == 32 ? AArch64::GPR32RegClass : AArch64::GPR64RegClass;
     813          28 :       const TargetRegisterClass &FPRRC =
     814             :           DefSize == 32 ? AArch64::FPR32RegClass : AArch64::FPR64RegClass;
     815             : 
     816          28 :       const unsigned DefGPRReg = MRI.createVirtualRegister(&GPRRC);
     817          28 :       MachineOperand &RegOp = I.getOperand(0);
     818          28 :       RegOp.setReg(DefGPRReg);
     819             : 
     820          56 :       BuildMI(MBB, std::next(I.getIterator()), I.getDebugLoc(),
     821          56 :               TII.get(AArch64::COPY))
     822             :           .addDef(DefReg)
     823             :           .addUse(DefGPRReg);
     824             : 
     825          28 :       if (!RBI.constrainGenericRegister(DefReg, FPRRC, MRI)) {
     826             :         LLVM_DEBUG(dbgs() << "Failed to constrain G_FCONSTANT def operand\n");
     827             :         return false;
     828             :       }
     829             : 
     830          28 :       MachineOperand &ImmOp = I.getOperand(1);
     831             :       // FIXME: Is going through int64_t always correct?
     832          28 :       ImmOp.ChangeToImmediate(
     833          84 :           ImmOp.getFPImm()->getValueAPF().bitcastToAPInt().getZExtValue());
     834           0 :     } else if (I.getOperand(1).isCImm()) {
     835           0 :       uint64_t Val = I.getOperand(1).getCImm()->getZExtValue();
     836           0 :       I.getOperand(1).ChangeToImmediate(Val);
     837           0 :     } else if (I.getOperand(1).isImm()) {
     838           0 :       uint64_t Val = I.getOperand(1).getImm();
     839           0 :       I.getOperand(1).ChangeToImmediate(Val);
     840             :     }
     841             : 
     842          28 :     constrainSelectedInstRegOperands(I, TII, TRI, RBI);
     843          28 :     return true;
     844             :   }
     845           4 :   case TargetOpcode::G_EXTRACT: {
     846           8 :     LLT SrcTy = MRI.getType(I.getOperand(1).getReg());
     847             :     LLT DstTy = MRI.getType(I.getOperand(0).getReg());
     848             :     (void)DstTy;
     849           4 :     unsigned SrcSize = SrcTy.getSizeInBits();
     850             :     // Larger extracts are vectors, same-size extracts should be something else
     851             :     // by now (either split up or simplified to a COPY).
     852           4 :     if (SrcTy.getSizeInBits() > 64 || Ty.getSizeInBits() > 32)
     853             :       return false;
     854             : 
     855           4 :     I.setDesc(TII.get(SrcSize == 64 ? AArch64::UBFMXri : AArch64::UBFMWri));
     856           8 :     MachineInstrBuilder(MF, I).addImm(I.getOperand(2).getImm() +
     857           8 :                                       Ty.getSizeInBits() - 1);
     858             : 
     859           4 :     if (SrcSize < 64) {
     860             :       assert(SrcSize == 32 && DstTy.getSizeInBits() == 16 &&
     861             :              "unexpected G_EXTRACT types");
     862           2 :       return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
     863             :     }
     864             : 
     865           2 :     unsigned DstReg = MRI.createGenericVirtualRegister(LLT::scalar(64));
     866           4 :     BuildMI(MBB, std::next(I.getIterator()), I.getDebugLoc(),
     867           4 :             TII.get(AArch64::COPY))
     868           2 :         .addDef(I.getOperand(0).getReg())
     869             :         .addUse(DstReg, 0, AArch64::sub_32);
     870           2 :     RBI.constrainGenericRegister(I.getOperand(0).getReg(),
     871             :                                  AArch64::GPR32RegClass, MRI);
     872           2 :     I.getOperand(0).setReg(DstReg);
     873             : 
     874           2 :     return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
     875             :   }
     876             : 
     877           6 :   case TargetOpcode::G_INSERT: {
     878          12 :     LLT SrcTy = MRI.getType(I.getOperand(2).getReg());
     879          12 :     LLT DstTy = MRI.getType(I.getOperand(0).getReg());
     880           6 :     unsigned DstSize = DstTy.getSizeInBits();
     881             :     // Larger inserts are vectors, same-size ones should be something else by
     882             :     // now (split up or turned into COPYs).
     883           6 :     if (Ty.getSizeInBits() > 64 || SrcTy.getSizeInBits() > 32)
     884             :       return false;
     885             : 
     886           6 :     I.setDesc(TII.get(DstSize == 64 ? AArch64::BFMXri : AArch64::BFMWri));
     887           6 :     unsigned LSB = I.getOperand(3).getImm();
     888           6 :     unsigned Width = MRI.getType(I.getOperand(2).getReg()).getSizeInBits();
     889           6 :     I.getOperand(3).setImm((DstSize - LSB) % DstSize);
     890           6 :     MachineInstrBuilder(MF, I).addImm(Width - 1);
     891             : 
     892           6 :     if (DstSize < 64) {
     893             :       assert(DstSize == 32 && SrcTy.getSizeInBits() == 16 &&
     894             :              "unexpected G_INSERT types");
     895           4 :       return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
     896             :     }
     897             : 
     898           2 :     unsigned SrcReg = MRI.createGenericVirtualRegister(LLT::scalar(64));
     899           4 :     BuildMI(MBB, I.getIterator(), I.getDebugLoc(),
     900           4 :             TII.get(AArch64::SUBREG_TO_REG))
     901             :         .addDef(SrcReg)
     902             :         .addImm(0)
     903           2 :         .addUse(I.getOperand(2).getReg())
     904             :         .addImm(AArch64::sub_32);
     905           2 :     RBI.constrainGenericRegister(I.getOperand(2).getReg(),
     906             :                                  AArch64::GPR32RegClass, MRI);
     907           4 :     I.getOperand(2).setReg(SrcReg);
     908             : 
     909           2 :     return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
     910             :   }
     911             :   case TargetOpcode::G_FRAME_INDEX: {
     912             :     // allocas and G_FRAME_INDEX are only supported in addrspace(0).
     913             :     if (Ty != LLT::pointer(0, 64)) {
     914             :       LLVM_DEBUG(dbgs() << "G_FRAME_INDEX pointer has type: " << Ty
     915             :                         << ", expected: " << LLT::pointer(0, 64) << '\n');
     916             :       return false;
     917             :     }
     918           3 :     I.setDesc(TII.get(AArch64::ADDXri));
     919             : 
     920             :     // MOs for a #0 shifted immediate.
     921           3 :     I.addOperand(MachineOperand::CreateImm(0));
     922           3 :     I.addOperand(MachineOperand::CreateImm(0));
     923             : 
     924           3 :     return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
     925             :   }
     926             : 
     927          37 :   case TargetOpcode::G_GLOBAL_VALUE: {
     928          37 :     auto GV = I.getOperand(1).getGlobal();
     929          37 :     if (GV->isThreadLocal()) {
     930             :       // FIXME: we don't support TLS yet.
     931             :       return false;
     932             :     }
     933          37 :     unsigned char OpFlags = STI.ClassifyGlobalReference(GV, TM);
     934          37 :     if (OpFlags & AArch64II::MO_GOT) {
     935          10 :       I.setDesc(TII.get(AArch64::LOADgot));
     936          10 :       I.getOperand(1).setTargetFlags(OpFlags);
     937          27 :     } else if (TM.getCodeModel() == CodeModel::Large) {
     938             :       // Materialize the global using movz/movk instructions.
     939           2 :       unsigned MovZDstReg = MRI.createVirtualRegister(&AArch64::GPR64RegClass);
     940           4 :       auto InsertPt = std::next(I.getIterator());
     941             :       auto MovZ =
     942           6 :           BuildMI(MBB, InsertPt, I.getDebugLoc(), TII.get(AArch64::MOVZXi))
     943           2 :               .addDef(MovZDstReg);
     944           4 :       MovZ->addOperand(MF, I.getOperand(1));
     945           2 :       MovZ->getOperand(1).setTargetFlags(OpFlags | AArch64II::MO_G0 |
     946             :                                          AArch64II::MO_NC);
     947           2 :       MovZ->addOperand(MF, MachineOperand::CreateImm(0));
     948           2 :       constrainSelectedInstRegOperands(*MovZ, TII, TRI, RBI);
     949             : 
     950             :       auto BuildMovK = [&](unsigned SrcReg, unsigned char Flags,
     951           6 :                            unsigned Offset, unsigned ForceDstReg) {
     952             :         unsigned DstReg =
     953          10 :             ForceDstReg ? ForceDstReg
     954          10 :                         : MRI.createVirtualRegister(&AArch64::GPR64RegClass);
     955          18 :         auto MovI = BuildMI(MBB, InsertPt, MovZ->getDebugLoc(),
     956          18 :                             TII.get(AArch64::MOVKXi))
     957             :                         .addDef(DstReg)
     958           6 :                         .addReg(SrcReg);
     959          12 :         MovI->addOperand(MF, MachineOperand::CreateGA(
     960          12 :                                  GV, MovZ->getOperand(1).getOffset(), Flags));
     961          12 :         MovI->addOperand(MF, MachineOperand::CreateImm(Offset));
     962           6 :         constrainSelectedInstRegOperands(*MovI, TII, TRI, RBI);
     963           6 :         return DstReg;
     964           2 :       };
     965           2 :       unsigned DstReg = BuildMovK(MovZ->getOperand(0).getReg(),
     966           2 :                                   AArch64II::MO_G1 | AArch64II::MO_NC, 16, 0);
     967           2 :       DstReg = BuildMovK(DstReg, AArch64II::MO_G2 | AArch64II::MO_NC, 32, 0);
     968           2 :       BuildMovK(DstReg, AArch64II::MO_G3, 48, I.getOperand(0).getReg());
     969           2 :       I.eraseFromParent();
     970             :       return true;
     971             :     } else {
     972          25 :       I.setDesc(TII.get(AArch64::MOVaddr));
     973          25 :       I.getOperand(1).setTargetFlags(OpFlags | AArch64II::MO_PAGE);
     974             :       MachineInstrBuilder MIB(MF, I);
     975          50 :       MIB.addGlobalAddress(GV, I.getOperand(1).getOffset(),
     976          50 :                            OpFlags | AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
     977             :     }
     978          35 :     return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
     979             :   }
     980             : 
     981          44 :   case TargetOpcode::G_LOAD:
     982             :   case TargetOpcode::G_STORE: {
     983          44 :     LLT PtrTy = MRI.getType(I.getOperand(1).getReg());
     984             : 
     985             :     if (PtrTy != LLT::pointer(0, 64)) {
     986             :       LLVM_DEBUG(dbgs() << "Load/Store pointer has type: " << PtrTy
     987             :                         << ", expected: " << LLT::pointer(0, 64) << '\n');
     988             :       return false;
     989             :     }
     990             : 
     991          44 :     auto &MemOp = **I.memoperands_begin();
     992          44 :     if (MemOp.getOrdering() != AtomicOrdering::NotAtomic) {
     993             :       LLVM_DEBUG(dbgs() << "Atomic load/store not supported yet\n");
     994             :       return false;
     995             :     }
     996          40 :     unsigned MemSizeInBits = MemOp.getSize() * 8;
     997             : 
     998             :     // FIXME: PR36018: Volatile loads in some cases are incorrectly selected by
     999             :     // folding with an extend. Until we have a G_SEXTLOAD solution bail out if
    1000             :     // we hit one.
    1001          56 :     if (Opcode == TargetOpcode::G_LOAD && MemOp.isVolatile())
    1002             :       return false;
    1003             : 
    1004             :     const unsigned PtrReg = I.getOperand(1).getReg();
    1005             : #ifndef NDEBUG
    1006             :     const RegisterBank &PtrRB = *RBI.getRegBank(PtrReg, MRI, TRI);
    1007             :     // Sanity-check the pointer register.
    1008             :     assert(PtrRB.getID() == AArch64::GPRRegBankID &&
    1009             :            "Load/Store pointer operand isn't a GPR");
    1010             :     assert(MRI.getType(PtrReg).isPointer() &&
    1011             :            "Load/Store pointer operand isn't a pointer");
    1012             : #endif
    1013             : 
    1014          39 :     const unsigned ValReg = I.getOperand(0).getReg();
    1015          39 :     const RegisterBank &RB = *RBI.getRegBank(ValReg, MRI, TRI);
    1016             : 
    1017             :     const unsigned NewOpc =
    1018          78 :         selectLoadStoreUIOp(I.getOpcode(), RB.getID(), MemSizeInBits);
    1019          39 :     if (NewOpc == I.getOpcode())
    1020             :       return false;
    1021             : 
    1022          39 :     I.setDesc(TII.get(NewOpc));
    1023             : 
    1024             :     uint64_t Offset = 0;
    1025          39 :     auto *PtrMI = MRI.getVRegDef(PtrReg);
    1026             : 
    1027             :     // Try to fold a GEP into our unsigned immediate addressing mode.
    1028          78 :     if (PtrMI->getOpcode() == TargetOpcode::G_GEP) {
    1029          20 :       if (auto COff = getConstantVRegVal(PtrMI->getOperand(2).getReg(), MRI)) {
    1030          20 :         int64_t Imm = *COff;
    1031          20 :         const unsigned Size = MemSizeInBits / 8;
    1032             :         const unsigned Scale = Log2_32(Size);
    1033          20 :         if ((Imm & (Size - 1)) == 0 && Imm >= 0 && Imm < (0x1000 << Scale)) {
    1034          19 :           unsigned Ptr2Reg = PtrMI->getOperand(1).getReg();
    1035          38 :           I.getOperand(1).setReg(Ptr2Reg);
    1036          19 :           PtrMI = MRI.getVRegDef(Ptr2Reg);
    1037          19 :           Offset = Imm / Size;
    1038             :         }
    1039             :       }
    1040             :     }
    1041             : 
    1042             :     // If we haven't folded anything into our addressing mode yet, try to fold
    1043             :     // a frame index into the base+offset.
    1044          39 :     if (!Offset && PtrMI->getOpcode() == TargetOpcode::G_FRAME_INDEX)
    1045           6 :       I.getOperand(1).ChangeToFrameIndex(PtrMI->getOperand(1).getIndex());
    1046             : 
    1047          78 :     I.addOperand(MachineOperand::CreateImm(Offset));
    1048             : 
    1049             :     // If we're storing a 0, use WZR/XZR.
    1050          39 :     if (auto CVal = getConstantVRegVal(ValReg, MRI)) {
    1051           0 :       if (*CVal == 0 && Opcode == TargetOpcode::G_STORE) {
    1052           0 :         if (I.getOpcode() == AArch64::STRWui)
    1053           0 :           I.getOperand(0).setReg(AArch64::WZR);
    1054           0 :         else if (I.getOpcode() == AArch64::STRXui)
    1055           0 :           I.getOperand(0).setReg(AArch64::XZR);
    1056             :       }
    1057             :     }
    1058             : 
    1059          39 :     return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
    1060             :   }
    1061             : 
    1062           2 :   case TargetOpcode::G_SMULH:
    1063             :   case TargetOpcode::G_UMULH: {
    1064             :     // Reject the various things we don't support yet.
    1065           2 :     if (unsupportedBinOp(I, RBI, MRI, TRI))
    1066             :       return false;
    1067             : 
    1068           2 :     const unsigned DefReg = I.getOperand(0).getReg();
    1069           2 :     const RegisterBank &RB = *RBI.getRegBank(DefReg, MRI, TRI);
    1070             : 
    1071           2 :     if (RB.getID() != AArch64::GPRRegBankID) {
    1072             :       LLVM_DEBUG(dbgs() << "G_[SU]MULH on bank: " << RB << ", expected: GPR\n");
    1073             :       return false;
    1074             :     }
    1075             : 
    1076             :     if (Ty != LLT::scalar(64)) {
    1077             :       LLVM_DEBUG(dbgs() << "G_[SU]MULH has type: " << Ty
    1078             :                         << ", expected: " << LLT::scalar(64) << '\n');
    1079             :       return false;
    1080             :     }
    1081             : 
    1082           4 :     unsigned NewOpc = I.getOpcode() == TargetOpcode::G_SMULH ? AArch64::SMULHrr
    1083             :                                                              : AArch64::UMULHrr;
    1084           2 :     I.setDesc(TII.get(NewOpc));
    1085             : 
    1086             :     // Now that we selected an opcode, we need to constrain the register
    1087             :     // operands to use appropriate classes.
    1088           2 :     return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
    1089             :   }
    1090          14 :   case TargetOpcode::G_FADD:
    1091             :   case TargetOpcode::G_FSUB:
    1092             :   case TargetOpcode::G_FMUL:
    1093             :   case TargetOpcode::G_FDIV:
    1094             : 
    1095             :   case TargetOpcode::G_OR:
    1096             :   case TargetOpcode::G_SHL:
    1097             :   case TargetOpcode::G_LSHR:
    1098             :   case TargetOpcode::G_ASHR:
    1099             :   case TargetOpcode::G_GEP: {
    1100             :     // Reject the various things we don't support yet.
    1101          14 :     if (unsupportedBinOp(I, RBI, MRI, TRI))
    1102             :       return false;
    1103             : 
    1104          14 :     const unsigned OpSize = Ty.getSizeInBits();
    1105             : 
    1106          14 :     const unsigned DefReg = I.getOperand(0).getReg();
    1107          14 :     const RegisterBank &RB = *RBI.getRegBank(DefReg, MRI, TRI);
    1108             : 
    1109          28 :     const unsigned NewOpc = selectBinaryOp(I.getOpcode(), RB.getID(), OpSize);
    1110          14 :     if (NewOpc == I.getOpcode())
    1111             :       return false;
    1112             : 
    1113          14 :     I.setDesc(TII.get(NewOpc));
    1114             :     // FIXME: Should the type be always reset in setDesc?
    1115             : 
    1116             :     // Now that we selected an opcode, we need to constrain the register
    1117             :     // operands to use appropriate classes.
    1118          14 :     return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
    1119             :   }
    1120             : 
    1121           3 :   case TargetOpcode::G_PTR_MASK: {
    1122           3 :     uint64_t Align = I.getOperand(2).getImm();
    1123           3 :     if (Align >= 64 || Align == 0)
    1124             :       return false;
    1125             : 
    1126           3 :     uint64_t Mask = ~((1ULL << Align) - 1);
    1127           3 :     I.setDesc(TII.get(AArch64::ANDXri));
    1128           3 :     I.getOperand(2).setImm(AArch64_AM::encodeLogicalImmediate(Mask, 64));
    1129             : 
    1130           3 :     return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
    1131             :   }
    1132          56 :   case TargetOpcode::G_PTRTOINT:
    1133             :   case TargetOpcode::G_TRUNC: {
    1134          56 :     const LLT DstTy = MRI.getType(I.getOperand(0).getReg());
    1135          56 :     const LLT SrcTy = MRI.getType(I.getOperand(1).getReg());
    1136             : 
    1137             :     const unsigned DstReg = I.getOperand(0).getReg();
    1138             :     const unsigned SrcReg = I.getOperand(1).getReg();
    1139             : 
    1140          56 :     const RegisterBank &DstRB = *RBI.getRegBank(DstReg, MRI, TRI);
    1141          56 :     const RegisterBank &SrcRB = *RBI.getRegBank(SrcReg, MRI, TRI);
    1142             : 
    1143          56 :     if (DstRB.getID() != SrcRB.getID()) {
    1144             :       LLVM_DEBUG(
    1145             :           dbgs() << "G_TRUNC/G_PTRTOINT input/output on different banks\n");
    1146             :       return false;
    1147             :     }
    1148             : 
    1149          56 :     if (DstRB.getID() == AArch64::GPRRegBankID) {
    1150             :       const TargetRegisterClass *DstRC =
    1151          56 :           getRegClassForTypeOnBank(DstTy, DstRB, RBI);
    1152          56 :       if (!DstRC)
    1153             :         return false;
    1154             : 
    1155             :       const TargetRegisterClass *SrcRC =
    1156          56 :           getRegClassForTypeOnBank(SrcTy, SrcRB, RBI);
    1157          56 :       if (!SrcRC)
    1158             :         return false;
    1159             : 
    1160         112 :       if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, MRI) ||
    1161          56 :           !RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) {
    1162             :         LLVM_DEBUG(dbgs() << "Failed to constrain G_TRUNC/G_PTRTOINT\n");
    1163             :         return false;
    1164             :       }
    1165             : 
    1166          56 :       if (DstRC == SrcRC) {
    1167             :         // Nothing to be done
    1168           5 :       } else if (Opcode == TargetOpcode::G_TRUNC && DstTy == LLT::scalar(32) &&
    1169             :                  SrcTy == LLT::scalar(64)) {
    1170           0 :         llvm_unreachable("TableGen can import this case");
    1171             :         return false;
    1172          10 :       } else if (DstRC == &AArch64::GPR32RegClass &&
    1173             :                  SrcRC == &AArch64::GPR64RegClass) {
    1174           5 :         I.getOperand(1).setSubReg(AArch64::sub_32);
    1175             :       } else {
    1176             :         LLVM_DEBUG(
    1177             :             dbgs() << "Unhandled mismatched classes in G_TRUNC/G_PTRTOINT\n");
    1178             :         return false;
    1179             :       }
    1180             : 
    1181          56 :       I.setDesc(TII.get(TargetOpcode::COPY));
    1182          56 :       return true;
    1183           0 :     } else if (DstRB.getID() == AArch64::FPRRegBankID) {
    1184             :       if (DstTy == LLT::vector(4, 16) && SrcTy == LLT::vector(4, 32)) {
    1185           0 :         I.setDesc(TII.get(AArch64::XTNv4i16));
    1186           0 :         constrainSelectedInstRegOperands(I, TII, TRI, RBI);
    1187           0 :         return true;
    1188             :       }
    1189             :     }
    1190             : 
    1191             :     return false;
    1192             :   }
    1193             : 
    1194          30 :   case TargetOpcode::G_ANYEXT: {
    1195          30 :     const unsigned DstReg = I.getOperand(0).getReg();
    1196          30 :     const unsigned SrcReg = I.getOperand(1).getReg();
    1197             : 
    1198          30 :     const RegisterBank &RBDst = *RBI.getRegBank(DstReg, MRI, TRI);
    1199          30 :     if (RBDst.getID() != AArch64::GPRRegBankID) {
    1200             :       LLVM_DEBUG(dbgs() << "G_ANYEXT on bank: " << RBDst
    1201             :                         << ", expected: GPR\n");
    1202             :       return false;
    1203             :     }
    1204             : 
    1205          30 :     const RegisterBank &RBSrc = *RBI.getRegBank(SrcReg, MRI, TRI);
    1206          30 :     if (RBSrc.getID() != AArch64::GPRRegBankID) {
    1207             :       LLVM_DEBUG(dbgs() << "G_ANYEXT on bank: " << RBSrc
    1208             :                         << ", expected: GPR\n");
    1209             :       return false;
    1210             :     }
    1211             : 
    1212          30 :     const unsigned DstSize = MRI.getType(DstReg).getSizeInBits();
    1213             : 
    1214          30 :     if (DstSize == 0) {
    1215             :       LLVM_DEBUG(dbgs() << "G_ANYEXT operand has no size, not a gvreg?\n");
    1216             :       return false;
    1217             :     }
    1218             : 
    1219          30 :     if (DstSize != 64 && DstSize > 32) {
    1220             :       LLVM_DEBUG(dbgs() << "G_ANYEXT to size: " << DstSize
    1221             :                         << ", expected: 32 or 64\n");
    1222             :       return false;
    1223             :     }
    1224             :     // At this point G_ANYEXT is just like a plain COPY, but we need
    1225             :     // to explicitly form the 64-bit value if any.
    1226          30 :     if (DstSize > 32) {
    1227           2 :       unsigned ExtSrc = MRI.createVirtualRegister(&AArch64::GPR64allRegClass);
    1228           6 :       BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::SUBREG_TO_REG))
    1229             :           .addDef(ExtSrc)
    1230             :           .addImm(0)
    1231             :           .addUse(SrcReg)
    1232             :           .addImm(AArch64::sub_32);
    1233           4 :       I.getOperand(1).setReg(ExtSrc);
    1234             :     }
    1235          30 :     return selectCopy(I, TII, MRI, TRI, RBI);
    1236             :   }
    1237             : 
    1238          17 :   case TargetOpcode::G_ZEXT:
    1239             :   case TargetOpcode::G_SEXT: {
    1240          17 :     unsigned Opcode = I.getOpcode();
    1241          34 :     const LLT DstTy = MRI.getType(I.getOperand(0).getReg()),
    1242          34 :               SrcTy = MRI.getType(I.getOperand(1).getReg());
    1243             :     const bool isSigned = Opcode == TargetOpcode::G_SEXT;
    1244             :     const unsigned DefReg = I.getOperand(0).getReg();
    1245             :     const unsigned SrcReg = I.getOperand(1).getReg();
    1246          17 :     const RegisterBank &RB = *RBI.getRegBank(DefReg, MRI, TRI);
    1247             : 
    1248          17 :     if (RB.getID() != AArch64::GPRRegBankID) {
    1249             :       LLVM_DEBUG(dbgs() << TII.getName(I.getOpcode()) << " on bank: " << RB
    1250             :                         << ", expected: GPR\n");
    1251             :       return false;
    1252             :     }
    1253             : 
    1254             :     MachineInstr *ExtI;
    1255             :     if (DstTy == LLT::scalar(64)) {
    1256             :       // FIXME: Can we avoid manually doing this?
    1257           5 :       if (!RBI.constrainGenericRegister(SrcReg, AArch64::GPR32RegClass, MRI)) {
    1258             :         LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(Opcode)
    1259             :                           << " operand\n");
    1260             :         return false;
    1261             :       }
    1262             : 
    1263             :       const unsigned SrcXReg =
    1264           5 :           MRI.createVirtualRegister(&AArch64::GPR64RegClass);
    1265          15 :       BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::SUBREG_TO_REG))
    1266             :           .addDef(SrcXReg)
    1267             :           .addImm(0)
    1268             :           .addUse(SrcReg)
    1269             :           .addImm(AArch64::sub_32);
    1270             : 
    1271           5 :       const unsigned NewOpc = isSigned ? AArch64::SBFMXri : AArch64::UBFMXri;
    1272          15 :       ExtI = BuildMI(MBB, I, I.getDebugLoc(), TII.get(NewOpc))
    1273             :                  .addDef(DefReg)
    1274             :                  .addUse(SrcXReg)
    1275             :                  .addImm(0)
    1276           5 :                  .addImm(SrcTy.getSizeInBits() - 1);
    1277          12 :     } else if (DstTy.isScalar() && DstTy.getSizeInBits() <= 32) {
    1278          12 :       const unsigned NewOpc = isSigned ? AArch64::SBFMWri : AArch64::UBFMWri;
    1279          36 :       ExtI = BuildMI(MBB, I, I.getDebugLoc(), TII.get(NewOpc))
    1280             :                  .addDef(DefReg)
    1281             :                  .addUse(SrcReg)
    1282             :                  .addImm(0)
    1283          12 :                  .addImm(SrcTy.getSizeInBits() - 1);
    1284             :     } else {
    1285             :       return false;
    1286             :     }
    1287             : 
    1288          17 :     constrainSelectedInstRegOperands(*ExtI, TII, TRI, RBI);
    1289             : 
    1290          17 :     I.eraseFromParent();
    1291          17 :     return true;
    1292             :   }
    1293             : 
    1294           0 :   case TargetOpcode::G_SITOFP:
    1295             :   case TargetOpcode::G_UITOFP:
    1296             :   case TargetOpcode::G_FPTOSI:
    1297             :   case TargetOpcode::G_FPTOUI: {
    1298           0 :     const LLT DstTy = MRI.getType(I.getOperand(0).getReg()),
    1299           0 :               SrcTy = MRI.getType(I.getOperand(1).getReg());
    1300           0 :     const unsigned NewOpc = selectFPConvOpc(Opcode, DstTy, SrcTy);
    1301           0 :     if (NewOpc == Opcode)
    1302             :       return false;
    1303             : 
    1304           0 :     I.setDesc(TII.get(NewOpc));
    1305           0 :     constrainSelectedInstRegOperands(I, TII, TRI, RBI);
    1306             : 
    1307           0 :     return true;
    1308             :   }
    1309             : 
    1310             : 
    1311           1 :   case TargetOpcode::G_INTTOPTR:
    1312             :     // The importer is currently unable to import pointer types since they
    1313             :     // didn't exist in SelectionDAG.
    1314           1 :     return selectCopy(I, TII, MRI, TRI, RBI);
    1315             : 
    1316           3 :   case TargetOpcode::G_BITCAST:
    1317             :     // Imported SelectionDAG rules can handle every bitcast except those that
    1318             :     // bitcast from a type to the same type. Ideally, these shouldn't occur
    1319             :     // but we might not run an optimizer that deletes them.
    1320           3 :     if (MRI.getType(I.getOperand(0).getReg()) ==
    1321           3 :         MRI.getType(I.getOperand(1).getReg()))
    1322           3 :       return selectCopy(I, TII, MRI, TRI, RBI);
    1323             :     return false;
    1324             : 
    1325             :   case TargetOpcode::G_SELECT: {
    1326           9 :     if (MRI.getType(I.getOperand(1).getReg()) != LLT::scalar(1)) {
    1327             :       LLVM_DEBUG(dbgs() << "G_SELECT cond has type: " << Ty
    1328             :                         << ", expected: " << LLT::scalar(1) << '\n');
    1329             :       return false;
    1330             :     }
    1331             : 
    1332             :     const unsigned CondReg = I.getOperand(1).getReg();
    1333           9 :     const unsigned TReg = I.getOperand(2).getReg();
    1334           9 :     const unsigned FReg = I.getOperand(3).getReg();
    1335             : 
    1336             :     unsigned CSelOpc = 0;
    1337             : 
    1338             :     if (Ty == LLT::scalar(32)) {
    1339             :       CSelOpc = AArch64::CSELWr;
    1340             :     } else if (Ty == LLT::scalar(64) || Ty == LLT::pointer(0, 64)) {
    1341             :       CSelOpc = AArch64::CSELXr;
    1342             :     } else {
    1343             :       return false;
    1344             :     }
    1345             : 
    1346             :     MachineInstr &TstMI =
    1347          27 :         *BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::ANDSWri))
    1348             :              .addDef(AArch64::WZR)
    1349             :              .addUse(CondReg)
    1350          18 :              .addImm(AArch64_AM::encodeLogicalImmediate(1, 32));
    1351             : 
    1352          27 :     MachineInstr &CSelMI = *BuildMI(MBB, I, I.getDebugLoc(), TII.get(CSelOpc))
    1353           9 :                                 .addDef(I.getOperand(0).getReg())
    1354             :                                 .addUse(TReg)
    1355             :                                 .addUse(FReg)
    1356           9 :                                 .addImm(AArch64CC::NE);
    1357             : 
    1358           9 :     constrainSelectedInstRegOperands(TstMI, TII, TRI, RBI);
    1359           9 :     constrainSelectedInstRegOperands(CSelMI, TII, TRI, RBI);
    1360             : 
    1361           9 :     I.eraseFromParent();
    1362           9 :     return true;
    1363             :   }
    1364             :   case TargetOpcode::G_ICMP: {
    1365             :     if (Ty != LLT::scalar(32)) {
    1366             :       LLVM_DEBUG(dbgs() << "G_ICMP result has type: " << Ty
    1367             :                         << ", expected: " << LLT::scalar(32) << '\n');
    1368             :       return false;
    1369             :     }
    1370             : 
    1371             :     unsigned CmpOpc = 0;
    1372             :     unsigned ZReg = 0;
    1373             : 
    1374          14 :     LLT CmpTy = MRI.getType(I.getOperand(2).getReg());
    1375             :     if (CmpTy == LLT::scalar(32)) {
    1376             :       CmpOpc = AArch64::SUBSWrr;
    1377             :       ZReg = AArch64::WZR;
    1378             :     } else if (CmpTy == LLT::scalar(64) || CmpTy.isPointer()) {
    1379             :       CmpOpc = AArch64::SUBSXrr;
    1380             :       ZReg = AArch64::XZR;
    1381             :     } else {
    1382             :       return false;
    1383             :     }
    1384             : 
    1385             :     // CSINC increments the result by one when the condition code is false.
    1386             :     // Therefore, we have to invert the predicate to get an increment by 1 when
    1387             :     // the predicate is true.
    1388             :     const AArch64CC::CondCode invCC =
    1389          14 :         changeICMPPredToAArch64CC(CmpInst::getInversePredicate(
    1390          28 :             (CmpInst::Predicate)I.getOperand(1).getPredicate()));
    1391             : 
    1392          42 :     MachineInstr &CmpMI = *BuildMI(MBB, I, I.getDebugLoc(), TII.get(CmpOpc))
    1393             :                                .addDef(ZReg)
    1394          14 :                                .addUse(I.getOperand(2).getReg())
    1395          28 :                                .addUse(I.getOperand(3).getReg());
    1396             : 
    1397             :     MachineInstr &CSetMI =
    1398          42 :         *BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::CSINCWr))
    1399          14 :              .addDef(I.getOperand(0).getReg())
    1400             :              .addUse(AArch64::WZR)
    1401             :              .addUse(AArch64::WZR)
    1402          28 :              .addImm(invCC);
    1403             : 
    1404          14 :     constrainSelectedInstRegOperands(CmpMI, TII, TRI, RBI);
    1405          14 :     constrainSelectedInstRegOperands(CSetMI, TII, TRI, RBI);
    1406             : 
    1407          14 :     I.eraseFromParent();
    1408          14 :     return true;
    1409             :   }
    1410             : 
    1411             :   case TargetOpcode::G_FCMP: {
    1412             :     if (Ty != LLT::scalar(32)) {
    1413             :       LLVM_DEBUG(dbgs() << "G_FCMP result has type: " << Ty
    1414             :                         << ", expected: " << LLT::scalar(32) << '\n');
    1415             :       return false;
    1416             :     }
    1417             : 
    1418             :     unsigned CmpOpc = 0;
    1419           6 :     LLT CmpTy = MRI.getType(I.getOperand(2).getReg());
    1420             :     if (CmpTy == LLT::scalar(32)) {
    1421             :       CmpOpc = AArch64::FCMPSrr;
    1422             :     } else if (CmpTy == LLT::scalar(64)) {
    1423             :       CmpOpc = AArch64::FCMPDrr;
    1424             :     } else {
    1425             :       return false;
    1426             :     }
    1427             : 
    1428             :     // FIXME: regbank
    1429             : 
    1430             :     AArch64CC::CondCode CC1, CC2;
    1431           6 :     changeFCMPPredToAArch64CC(
    1432           6 :         (CmpInst::Predicate)I.getOperand(1).getPredicate(), CC1, CC2);
    1433             : 
    1434          18 :     MachineInstr &CmpMI = *BuildMI(MBB, I, I.getDebugLoc(), TII.get(CmpOpc))
    1435           6 :                                .addUse(I.getOperand(2).getReg())
    1436          12 :                                .addUse(I.getOperand(3).getReg());
    1437             : 
    1438           6 :     const unsigned DefReg = I.getOperand(0).getReg();
    1439             :     unsigned Def1Reg = DefReg;
    1440           6 :     if (CC2 != AArch64CC::AL)
    1441           3 :       Def1Reg = MRI.createVirtualRegister(&AArch64::GPR32RegClass);
    1442             : 
    1443             :     MachineInstr &CSetMI =
    1444          18 :         *BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::CSINCWr))
    1445             :              .addDef(Def1Reg)
    1446             :              .addUse(AArch64::WZR)
    1447             :              .addUse(AArch64::WZR)
    1448          18 :              .addImm(getInvertedCondCode(CC1));
    1449             : 
    1450           6 :     if (CC2 != AArch64CC::AL) {
    1451           3 :       unsigned Def2Reg = MRI.createVirtualRegister(&AArch64::GPR32RegClass);
    1452             :       MachineInstr &CSet2MI =
    1453           9 :           *BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::CSINCWr))
    1454             :                .addDef(Def2Reg)
    1455             :                .addUse(AArch64::WZR)
    1456             :                .addUse(AArch64::WZR)
    1457           9 :                .addImm(getInvertedCondCode(CC2));
    1458             :       MachineInstr &OrMI =
    1459           9 :           *BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::ORRWrr))
    1460             :                .addDef(DefReg)
    1461             :                .addUse(Def1Reg)
    1462           3 :                .addUse(Def2Reg);
    1463           3 :       constrainSelectedInstRegOperands(OrMI, TII, TRI, RBI);
    1464           3 :       constrainSelectedInstRegOperands(CSet2MI, TII, TRI, RBI);
    1465             :     }
    1466             : 
    1467           6 :     constrainSelectedInstRegOperands(CmpMI, TII, TRI, RBI);
    1468           6 :     constrainSelectedInstRegOperands(CSetMI, TII, TRI, RBI);
    1469             : 
    1470           6 :     I.eraseFromParent();
    1471           6 :     return true;
    1472             :   }
    1473           1 :   case TargetOpcode::G_VASTART:
    1474           2 :     return STI.isTargetDarwin() ? selectVaStartDarwin(I, MF, MRI)
    1475             :                                 : selectVaStartAAPCS(I, MF, MRI);
    1476           2 :   case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS:
    1477           4 :     if (!I.getOperand(0).isIntrinsicID())
    1478             :       return false;
    1479           1 :     if (I.getOperand(0).getIntrinsicID() != Intrinsic::trap)
    1480             :       return false;
    1481           3 :     BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::BRK))
    1482             :       .addImm(1);
    1483           1 :     I.eraseFromParent();
    1484           1 :     return true;
    1485           5 :   case TargetOpcode::G_IMPLICIT_DEF:
    1486           5 :     I.setDesc(TII.get(TargetOpcode::IMPLICIT_DEF));
    1487          10 :     const LLT DstTy = MRI.getType(I.getOperand(0).getReg());
    1488             :     const unsigned DstReg = I.getOperand(0).getReg();
    1489           5 :     const RegisterBank &DstRB = *RBI.getRegBank(DstReg, MRI, TRI);
    1490             :     const TargetRegisterClass *DstRC =
    1491           5 :         getRegClassForTypeOnBank(DstTy, DstRB, RBI);
    1492           5 :     RBI.constrainGenericRegister(DstReg, *DstRC, MRI);
    1493           5 :     return true;
    1494             :   }
    1495             : 
    1496             :   return false;
    1497             : }
    1498             : 
    1499             : /// SelectArithImmed - Select an immediate value that can be represented as
    1500             : /// a 12-bit value shifted left by either 0 or 12.  If so, return true with
    1501             : /// Val set to the 12-bit value and Shift set to the shifter operand.
    1502             : InstructionSelector::ComplexRendererFns
    1503          34 : AArch64InstructionSelector::selectArithImmed(MachineOperand &Root) const {
    1504          34 :   MachineInstr &MI = *Root.getParent();
    1505          34 :   MachineBasicBlock &MBB = *MI.getParent();
    1506          34 :   MachineFunction &MF = *MBB.getParent();
    1507          34 :   MachineRegisterInfo &MRI = MF.getRegInfo();
    1508             : 
    1509             :   // This function is called from the addsub_shifted_imm ComplexPattern,
    1510             :   // which lists [imm] as the list of opcode it's interested in, however
    1511             :   // we still need to check whether the operand is actually an immediate
    1512             :   // here because the ComplexPattern opcode list is only used in
    1513             :   // root-level opcode matching.
    1514             :   uint64_t Immed;
    1515          34 :   if (Root.isImm())
    1516           0 :     Immed = Root.getImm();
    1517          34 :   else if (Root.isCImm())
    1518           0 :     Immed = Root.getCImm()->getZExtValue();
    1519          34 :   else if (Root.isReg()) {
    1520          34 :     MachineInstr *Def = MRI.getVRegDef(Root.getReg());
    1521          68 :     if (Def->getOpcode() != TargetOpcode::G_CONSTANT)
    1522             :       return None;
    1523          10 :     MachineOperand &Op1 = Def->getOperand(1);
    1524          20 :     if (!Op1.isCImm() || Op1.getCImm()->getBitWidth() > 64)
    1525             :       return None;
    1526             :     Immed = Op1.getCImm()->getZExtValue();
    1527             :   } else
    1528             :     return None;
    1529             : 
    1530             :   unsigned ShiftAmt;
    1531             : 
    1532          10 :   if (Immed >> 12 == 0) {
    1533             :     ShiftAmt = 0;
    1534           0 :   } else if ((Immed & 0xfff) == 0 && Immed >> 24 == 0) {
    1535             :     ShiftAmt = 12;
    1536             :     Immed = Immed >> 12;
    1537             :   } else
    1538             :     return None;
    1539             : 
    1540             :   unsigned ShVal = AArch64_AM::getShifterImm(AArch64_AM::LSL, ShiftAmt);
    1541             :   return {{
    1542          10 :       [=](MachineInstrBuilder &MIB) { MIB.addImm(Immed); },
    1543          10 :       [=](MachineInstrBuilder &MIB) { MIB.addImm(ShVal); },
    1544          20 :   }};
    1545             : }
    1546             : 
    1547             : /// Select a "register plus unscaled signed 9-bit immediate" address.  This
    1548             : /// should only match when there is an offset that is not valid for a scaled
    1549             : /// immediate addressing mode.  The "Size" argument is the size in bytes of the
    1550             : /// memory reference, which is needed here to know what is valid for a scaled
    1551             : /// immediate.
    1552             : InstructionSelector::ComplexRendererFns
    1553          80 : AArch64InstructionSelector::selectAddrModeUnscaled(MachineOperand &Root,
    1554             :                                                    unsigned Size) const {
    1555             :   MachineRegisterInfo &MRI =
    1556          80 :       Root.getParent()->getParent()->getParent()->getRegInfo();
    1557             : 
    1558          80 :   if (!Root.isReg())
    1559             :     return None;
    1560             : 
    1561          80 :   if (!isBaseWithConstantOffset(Root, MRI))
    1562             :     return None;
    1563             : 
    1564           2 :   MachineInstr *RootDef = MRI.getVRegDef(Root.getReg());
    1565           2 :   if (!RootDef)
    1566             :     return None;
    1567             : 
    1568           2 :   MachineOperand &OffImm = RootDef->getOperand(2);
    1569           2 :   if (!OffImm.isReg())
    1570             :     return None;
    1571           2 :   MachineInstr *RHS = MRI.getVRegDef(OffImm.getReg());
    1572           4 :   if (!RHS || RHS->getOpcode() != TargetOpcode::G_CONSTANT)
    1573             :     return None;
    1574             :   int64_t RHSC;
    1575           2 :   MachineOperand &RHSOp1 = RHS->getOperand(1);
    1576           4 :   if (!RHSOp1.isCImm() || RHSOp1.getCImm()->getBitWidth() > 64)
    1577             :     return None;
    1578             :   RHSC = RHSOp1.getCImm()->getSExtValue();
    1579             : 
    1580             :   // If the offset is valid as a scaled immediate, don't match here.
    1581           6 :   if ((RHSC & (Size - 1)) == 0 && RHSC >= 0 && RHSC < (0x1000 << Log2_32(Size)))
    1582             :     return None;
    1583           2 :   if (RHSC >= -256 && RHSC < 256) {
    1584           0 :     MachineOperand &Base = RootDef->getOperand(1);
    1585             :     return {{
    1586           0 :         [=](MachineInstrBuilder &MIB) { MIB.add(Base); },
    1587           0 :         [=](MachineInstrBuilder &MIB) { MIB.addImm(RHSC); },
    1588           0 :     }};
    1589             :   }
    1590             :   return None;
    1591             : }
    1592             : 
    1593             : /// Select a "register plus scaled unsigned 12-bit immediate" address.  The
    1594             : /// "Size" argument is the size in bytes of the memory reference, which
    1595             : /// determines the scale.
    1596             : InstructionSelector::ComplexRendererFns
    1597         111 : AArch64InstructionSelector::selectAddrModeIndexed(MachineOperand &Root,
    1598             :                                                   unsigned Size) const {
    1599             :   MachineRegisterInfo &MRI =
    1600         111 :       Root.getParent()->getParent()->getParent()->getRegInfo();
    1601             : 
    1602         111 :   if (!Root.isReg())
    1603             :     return None;
    1604             : 
    1605         111 :   MachineInstr *RootDef = MRI.getVRegDef(Root.getReg());
    1606         111 :   if (!RootDef)
    1607             :     return None;
    1608             : 
    1609         222 :   if (RootDef->getOpcode() == TargetOpcode::G_FRAME_INDEX) {
    1610             :     return {{
    1611          22 :         [=](MachineInstrBuilder &MIB) { MIB.add(RootDef->getOperand(1)); },
    1612             :         [=](MachineInstrBuilder &MIB) { MIB.addImm(0); },
    1613          44 :     }};
    1614             :   }
    1615             : 
    1616          89 :   if (isBaseWithConstantOffset(Root, MRI)) {
    1617          11 :     MachineOperand &LHS = RootDef->getOperand(1);
    1618             :     MachineOperand &RHS = RootDef->getOperand(2);
    1619          11 :     MachineInstr *LHSDef = MRI.getVRegDef(LHS.getReg());
    1620          11 :     MachineInstr *RHSDef = MRI.getVRegDef(RHS.getReg());
    1621          11 :     if (LHSDef && RHSDef) {
    1622          22 :       int64_t RHSC = (int64_t)RHSDef->getOperand(1).getCImm()->getZExtValue();
    1623             :       unsigned Scale = Log2_32(Size);
    1624          11 :       if ((RHSC & (Size - 1)) == 0 && RHSC >= 0 && RHSC < (0x1000 << Scale)) {
    1625          18 :         if (LHSDef->getOpcode() == TargetOpcode::G_FRAME_INDEX)
    1626             :           return {{
    1627           0 :               [=](MachineInstrBuilder &MIB) { MIB.add(LHSDef->getOperand(1)); },
    1628           0 :               [=](MachineInstrBuilder &MIB) { MIB.addImm(RHSC >> Scale); },
    1629           0 :           }};
    1630             : 
    1631             :         return {{
    1632           9 :             [=](MachineInstrBuilder &MIB) { MIB.add(LHS); },
    1633           9 :             [=](MachineInstrBuilder &MIB) { MIB.addImm(RHSC >> Scale); },
    1634          27 :         }};
    1635             :       }
    1636             :     }
    1637             :   }
    1638             : 
    1639             :   // Before falling back to our general case, check if the unscaled
    1640             :   // instructions can handle this. If so, that's preferable.
    1641         160 :   if (selectAddrModeUnscaled(Root, Size).hasValue())
    1642             :     return None;
    1643             : 
    1644             :   return {{
    1645          80 :       [=](MachineInstrBuilder &MIB) { MIB.add(Root); },
    1646             :       [=](MachineInstrBuilder &MIB) { MIB.addImm(0); },
    1647         240 :   }};
    1648             : }
    1649             : 
    1650           1 : void AArch64InstructionSelector::renderTruncImm(MachineInstrBuilder &MIB,
    1651             :                                                 const MachineInstr &MI) const {
    1652           1 :   const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
    1653             :   assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && "Expected G_CONSTANT");
    1654           1 :   Optional<int64_t> CstVal = getConstantVRegVal(MI.getOperand(0).getReg(), MRI);
    1655             :   assert(CstVal && "Expected constant value");
    1656           1 :   MIB.addImm(CstVal.getValue());
    1657           1 : }
    1658             : 
    1659             : namespace llvm {
    1660             : InstructionSelector *
    1661        1438 : createAArch64InstructionSelector(const AArch64TargetMachine &TM,
    1662             :                                  AArch64Subtarget &Subtarget,
    1663             :                                  AArch64RegisterBankInfo &RBI) {
    1664        1438 :   return new AArch64InstructionSelector(TM, Subtarget, RBI);
    1665             : }
    1666      299229 : }

Generated by: LCOV version 1.13