LCOV - code coverage report
Current view: top level - lib/Target/AArch64 - AArch64InstructionSelector.cpp (source / functions) Hit Total Coverage
Test: llvm-toolchain.info Lines: 518 705 73.5 %
Date: 2018-02-21 17:27:13 Functions: 25 33 75.8 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : //===- AArch64InstructionSelector.cpp ----------------------------*- C++ -*-==//
       2             : //
       3             : //                     The LLVM Compiler Infrastructure
       4             : //
       5             : // This file is distributed under the University of Illinois Open Source
       6             : // License. See LICENSE.TXT for details.
       7             : //
       8             : //===----------------------------------------------------------------------===//
       9             : /// \file
      10             : /// This file implements the targeting of the InstructionSelector class for
      11             : /// AArch64.
      12             : /// \todo This should be generated by TableGen.
      13             : //===----------------------------------------------------------------------===//
      14             : 
      15             : #include "AArch64InstrInfo.h"
      16             : #include "AArch64MachineFunctionInfo.h"
      17             : #include "AArch64RegisterBankInfo.h"
      18             : #include "AArch64RegisterInfo.h"
      19             : #include "AArch64Subtarget.h"
      20             : #include "AArch64TargetMachine.h"
      21             : #include "MCTargetDesc/AArch64AddressingModes.h"
      22             : #include "llvm/CodeGen/GlobalISel/InstructionSelector.h"
      23             : #include "llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h"
      24             : #include "llvm/CodeGen/GlobalISel/Utils.h"
      25             : #include "llvm/CodeGen/MachineBasicBlock.h"
      26             : #include "llvm/CodeGen/MachineFunction.h"
      27             : #include "llvm/CodeGen/MachineInstr.h"
      28             : #include "llvm/CodeGen/MachineInstrBuilder.h"
      29             : #include "llvm/CodeGen/MachineOperand.h"
      30             : #include "llvm/CodeGen/MachineRegisterInfo.h"
      31             : #include "llvm/IR/Type.h"
      32             : #include "llvm/Support/Debug.h"
      33             : #include "llvm/Support/raw_ostream.h"
      34             : 
      35             : #define DEBUG_TYPE "aarch64-isel"
      36             : 
      37             : using namespace llvm;
      38             : 
      39             : namespace {
      40             : 
      41             : #define GET_GLOBALISEL_PREDICATE_BITSET
      42             : #include "AArch64GenGlobalISel.inc"
      43             : #undef GET_GLOBALISEL_PREDICATE_BITSET
      44             : 
      45        1294 : class AArch64InstructionSelector : public InstructionSelector {
      46             : public:
      47             :   AArch64InstructionSelector(const AArch64TargetMachine &TM,
      48             :                              const AArch64Subtarget &STI,
      49             :                              const AArch64RegisterBankInfo &RBI);
      50             : 
      51             :   bool select(MachineInstr &I, CodeGenCoverage &CoverageInfo) const override;
      52             :   static const char *getName() { return DEBUG_TYPE; }
      53             : 
      54             : private:
      55             :   /// tblgen-erated 'select' implementation, used as the initial selector for
      56             :   /// the patterns that don't require complex C++.
      57             :   bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const;
      58             : 
      59             :   bool selectVaStartAAPCS(MachineInstr &I, MachineFunction &MF,
      60             :                           MachineRegisterInfo &MRI) const;
      61             :   bool selectVaStartDarwin(MachineInstr &I, MachineFunction &MF,
      62             :                            MachineRegisterInfo &MRI) const;
      63             : 
      64             :   bool selectCompareBranch(MachineInstr &I, MachineFunction &MF,
      65             :                            MachineRegisterInfo &MRI) const;
      66             : 
      67             :   ComplexRendererFns selectArithImmed(MachineOperand &Root) const;
      68             : 
      69             :   ComplexRendererFns selectAddrModeUnscaled(MachineOperand &Root,
      70             :                                             unsigned Size) const;
      71             : 
      72           0 :   ComplexRendererFns selectAddrModeUnscaled8(MachineOperand &Root) const {
      73           0 :     return selectAddrModeUnscaled(Root, 1);
      74             :   }
      75           0 :   ComplexRendererFns selectAddrModeUnscaled16(MachineOperand &Root) const {
      76           0 :     return selectAddrModeUnscaled(Root, 2);
      77             :   }
      78           0 :   ComplexRendererFns selectAddrModeUnscaled32(MachineOperand &Root) const {
      79           0 :     return selectAddrModeUnscaled(Root, 4);
      80             :   }
      81           0 :   ComplexRendererFns selectAddrModeUnscaled64(MachineOperand &Root) const {
      82           0 :     return selectAddrModeUnscaled(Root, 8);
      83             :   }
      84           0 :   ComplexRendererFns selectAddrModeUnscaled128(MachineOperand &Root) const {
      85           0 :     return selectAddrModeUnscaled(Root, 16);
      86             :   }
      87             : 
      88             :   ComplexRendererFns selectAddrModeIndexed(MachineOperand &Root,
      89             :                                            unsigned Size) const;
      90             :   template <int Width>
      91         101 :   ComplexRendererFns selectAddrModeIndexed(MachineOperand &Root) const {
      92         101 :     return selectAddrModeIndexed(Root, Width / 8);
      93             :   }
      94             : 
      95             :   void renderTruncImm(MachineInstrBuilder &MIB, const MachineInstr &MI) const;
      96             : 
      97             :   const AArch64TargetMachine &TM;
      98             :   const AArch64Subtarget &STI;
      99             :   const AArch64InstrInfo &TII;
     100             :   const AArch64RegisterInfo &TRI;
     101             :   const AArch64RegisterBankInfo &RBI;
     102             : 
     103             : #define GET_GLOBALISEL_PREDICATES_DECL
     104             : #include "AArch64GenGlobalISel.inc"
     105             : #undef GET_GLOBALISEL_PREDICATES_DECL
     106             : 
     107             : // We declare the temporaries used by selectImpl() in the class to minimize the
     108             : // cost of constructing placeholder values.
     109             : #define GET_GLOBALISEL_TEMPORARIES_DECL
     110             : #include "AArch64GenGlobalISel.inc"
     111             : #undef GET_GLOBALISEL_TEMPORARIES_DECL
     112             : };
     113             : 
     114             : } // end anonymous namespace
     115             : 
     116             : #define GET_GLOBALISEL_IMPL
     117             : #include "AArch64GenGlobalISel.inc"
     118             : #undef GET_GLOBALISEL_IMPL
     119             : 
     120        1322 : AArch64InstructionSelector::AArch64InstructionSelector(
     121             :     const AArch64TargetMachine &TM, const AArch64Subtarget &STI,
     122        1322 :     const AArch64RegisterBankInfo &RBI)
     123             :     : InstructionSelector(), TM(TM), STI(STI), TII(*STI.getInstrInfo()),
     124             :       TRI(*STI.getRegisterInfo()), RBI(RBI),
     125             : #define GET_GLOBALISEL_PREDICATES_INIT
     126             : #include "AArch64GenGlobalISel.inc"
     127             : #undef GET_GLOBALISEL_PREDICATES_INIT
     128             : #define GET_GLOBALISEL_TEMPORARIES_INIT
     129             : #include "AArch64GenGlobalISel.inc"
     130             : #undef GET_GLOBALISEL_TEMPORARIES_INIT
     131             : {
     132        1322 : }
     133             : 
     134             : // FIXME: This should be target-independent, inferred from the types declared
     135             : // for each class in the bank.
     136             : static const TargetRegisterClass *
     137         539 : getRegClassForTypeOnBank(LLT Ty, const RegisterBank &RB,
     138             :                          const RegisterBankInfo &RBI,
     139             :                          bool GetAllRegSet = false) {
     140         539 :   if (RB.getID() == AArch64::GPRRegBankID) {
     141         453 :     if (Ty.getSizeInBits() <= 32)
     142         275 :       return GetAllRegSet ? &AArch64::GPR32allRegClass
     143             :                           : &AArch64::GPR32RegClass;
     144         178 :     if (Ty.getSizeInBits() == 64)
     145         178 :       return GetAllRegSet ? &AArch64::GPR64allRegClass
     146             :                           : &AArch64::GPR64RegClass;
     147             :     return nullptr;
     148             :   }
     149             : 
     150          86 :   if (RB.getID() == AArch64::FPRRegBankID) {
     151          86 :     if (Ty.getSizeInBits() <= 16)
     152             :       return &AArch64::FPR16RegClass;
     153          74 :     if (Ty.getSizeInBits() == 32)
     154             :       return &AArch64::FPR32RegClass;
     155          35 :     if (Ty.getSizeInBits() == 64)
     156             :       return &AArch64::FPR64RegClass;
     157           2 :     if (Ty.getSizeInBits() == 128)
     158             :       return &AArch64::FPR128RegClass;
     159           0 :     return nullptr;
     160             :   }
     161             : 
     162             :   return nullptr;
     163             : }
     164             : 
     165             : /// Check whether \p I is a currently unsupported binary operation:
     166             : /// - it has an unsized type
     167             : /// - an operand is not a vreg
     168             : /// - all operands are not in the same bank
     169             : /// These are checks that should someday live in the verifier, but right now,
     170             : /// these are mostly limitations of the aarch64 selector.
     171          16 : static bool unsupportedBinOp(const MachineInstr &I,
     172             :                              const AArch64RegisterBankInfo &RBI,
     173             :                              const MachineRegisterInfo &MRI,
     174             :                              const AArch64RegisterInfo &TRI) {
     175          16 :   LLT Ty = MRI.getType(I.getOperand(0).getReg());
     176          16 :   if (!Ty.isValid()) {
     177             :     DEBUG(dbgs() << "Generic binop register should be typed\n");
     178             :     return true;
     179             :   }
     180             : 
     181             :   const RegisterBank *PrevOpBank = nullptr;
     182         112 :   for (auto &MO : I.operands()) {
     183             :     // FIXME: Support non-register operands.
     184          48 :     if (!MO.isReg()) {
     185             :       DEBUG(dbgs() << "Generic inst non-reg operands are unsupported\n");
     186             :       return true;
     187             :     }
     188             : 
     189             :     // FIXME: Can generic operations have physical registers operands? If
     190             :     // so, this will need to be taught about that, and we'll need to get the
     191             :     // bank out of the minimal class for the register.
     192             :     // Either way, this needs to be documented (and possibly verified).
     193          96 :     if (!TargetRegisterInfo::isVirtualRegister(MO.getReg())) {
     194             :       DEBUG(dbgs() << "Generic inst has physical register operand\n");
     195             :       return true;
     196             :     }
     197             : 
     198          48 :     const RegisterBank *OpBank = RBI.getRegBank(MO.getReg(), MRI, TRI);
     199          48 :     if (!OpBank) {
     200             :       DEBUG(dbgs() << "Generic register has no bank or class\n");
     201             :       return true;
     202             :     }
     203             : 
     204          48 :     if (PrevOpBank && OpBank != PrevOpBank) {
     205             :       DEBUG(dbgs() << "Generic inst operands have different banks\n");
     206             :       return true;
     207             :     }
     208             :     PrevOpBank = OpBank;
     209             :   }
     210             :   return false;
     211             : }
     212             : 
     213             : /// Select the AArch64 opcode for the basic binary operation \p GenericOpc
     214             : /// (such as G_OR or G_SDIV), appropriate for the register bank \p RegBankID
     215             : /// and of size \p OpSize.
     216             : /// \returns \p GenericOpc if the combination is unsupported.
     217          14 : static unsigned selectBinaryOp(unsigned GenericOpc, unsigned RegBankID,
     218             :                                unsigned OpSize) {
     219          14 :   switch (RegBankID) {
     220          14 :   case AArch64::GPRRegBankID:
     221          14 :     if (OpSize == 32) {
     222           4 :       switch (GenericOpc) {
     223             :       case TargetOpcode::G_SHL:
     224             :         return AArch64::LSLVWr;
     225           1 :       case TargetOpcode::G_LSHR:
     226           1 :         return AArch64::LSRVWr;
     227           1 :       case TargetOpcode::G_ASHR:
     228           1 :         return AArch64::ASRVWr;
     229           0 :       default:
     230           0 :         return GenericOpc;
     231             :       }
     232          10 :     } else if (OpSize == 64) {
     233          10 :       switch (GenericOpc) {
     234             :       case TargetOpcode::G_GEP:
     235             :         return AArch64::ADDXrr;
     236           0 :       case TargetOpcode::G_SHL:
     237           0 :         return AArch64::LSLVXr;
     238           0 :       case TargetOpcode::G_LSHR:
     239           0 :         return AArch64::LSRVXr;
     240           0 :       case TargetOpcode::G_ASHR:
     241           0 :         return AArch64::ASRVXr;
     242           0 :       default:
     243           0 :         return GenericOpc;
     244             :       }
     245             :     }
     246             :     break;
     247           0 :   case AArch64::FPRRegBankID:
     248           0 :     switch (OpSize) {
     249           0 :     case 32:
     250           0 :       switch (GenericOpc) {
     251             :       case TargetOpcode::G_FADD:
     252             :         return AArch64::FADDSrr;
     253           0 :       case TargetOpcode::G_FSUB:
     254           0 :         return AArch64::FSUBSrr;
     255           0 :       case TargetOpcode::G_FMUL:
     256           0 :         return AArch64::FMULSrr;
     257           0 :       case TargetOpcode::G_FDIV:
     258           0 :         return AArch64::FDIVSrr;
     259           0 :       default:
     260           0 :         return GenericOpc;
     261             :       }
     262           0 :     case 64:
     263           0 :       switch (GenericOpc) {
     264             :       case TargetOpcode::G_FADD:
     265             :         return AArch64::FADDDrr;
     266           0 :       case TargetOpcode::G_FSUB:
     267           0 :         return AArch64::FSUBDrr;
     268           0 :       case TargetOpcode::G_FMUL:
     269           0 :         return AArch64::FMULDrr;
     270           0 :       case TargetOpcode::G_FDIV:
     271           0 :         return AArch64::FDIVDrr;
     272           0 :       case TargetOpcode::G_OR:
     273           0 :         return AArch64::ORRv8i8;
     274           0 :       default:
     275           0 :         return GenericOpc;
     276             :       }
     277             :     }
     278             :     break;
     279             :   }
     280             :   return GenericOpc;
     281             : }
     282             : 
     283             : /// Select the AArch64 opcode for the G_LOAD or G_STORE operation \p GenericOpc,
     284             : /// appropriate for the (value) register bank \p RegBankID and of memory access
     285             : /// size \p OpSize.  This returns the variant with the base+unsigned-immediate
     286             : /// addressing mode (e.g., LDRXui).
     287             : /// \returns \p GenericOpc if the combination is unsupported.
     288          13 : static unsigned selectLoadStoreUIOp(unsigned GenericOpc, unsigned RegBankID,
     289             :                                     unsigned OpSize) {
     290             :   const bool isStore = GenericOpc == TargetOpcode::G_STORE;
     291          13 :   switch (RegBankID) {
     292          11 :   case AArch64::GPRRegBankID:
     293          11 :     switch (OpSize) {
     294           2 :     case 8:
     295           2 :       return isStore ? AArch64::STRBBui : AArch64::LDRBBui;
     296           2 :     case 16:
     297           2 :       return isStore ? AArch64::STRHHui : AArch64::LDRHHui;
     298           0 :     case 32:
     299           0 :       return isStore ? AArch64::STRWui : AArch64::LDRWui;
     300           7 :     case 64:
     301           7 :       return isStore ? AArch64::STRXui : AArch64::LDRXui;
     302             :     }
     303             :     break;
     304           2 :   case AArch64::FPRRegBankID:
     305           2 :     switch (OpSize) {
     306           2 :     case 8:
     307           2 :       return isStore ? AArch64::STRBui : AArch64::LDRBui;
     308           0 :     case 16:
     309           0 :       return isStore ? AArch64::STRHui : AArch64::LDRHui;
     310           0 :     case 32:
     311           0 :       return isStore ? AArch64::STRSui : AArch64::LDRSui;
     312           0 :     case 64:
     313           0 :       return isStore ? AArch64::STRDui : AArch64::LDRDui;
     314             :     }
     315             :     break;
     316             :   }
     317             :   return GenericOpc;
     318             : }
     319             : 
     320           4 : static bool selectFP16CopyFromGPR32(MachineInstr &I, const TargetInstrInfo &TII,
     321             :                                     MachineRegisterInfo &MRI, unsigned SrcReg) {
     322             :   // Copies from gpr32 to fpr16 need to use a sub-register copy.
     323           4 :   unsigned CopyReg = MRI.createVirtualRegister(&AArch64::FPR32RegClass);
     324          12 :   BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(AArch64::COPY))
     325             :       .addDef(CopyReg)
     326             :       .addUse(SrcReg);
     327           4 :   unsigned SubRegCopy = MRI.createVirtualRegister(&AArch64::FPR16RegClass);
     328          12 :   BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(TargetOpcode::COPY))
     329             :       .addDef(SubRegCopy)
     330             :       .addUse(CopyReg, 0, AArch64::hsub);
     331             : 
     332           4 :   MachineOperand &RegOp = I.getOperand(1);
     333           4 :   RegOp.setReg(SubRegCopy);
     334           4 :   return true;
     335             : }
     336             : 
     337         608 : static bool selectCopy(MachineInstr &I, const TargetInstrInfo &TII,
     338             :                        MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI,
     339             :                        const RegisterBankInfo &RBI) {
     340             : 
     341         608 :   unsigned DstReg = I.getOperand(0).getReg();
     342         608 :   unsigned SrcReg = I.getOperand(1).getReg();
     343             : 
     344         608 :   if (TargetRegisterInfo::isPhysicalRegister(DstReg)) {
     345         537 :     if (TRI.getRegClass(AArch64::FPR16RegClassID)->contains(DstReg) &&
     346             :         !TargetRegisterInfo::isPhysicalRegister(SrcReg)) {
     347           8 :       const RegisterBank &RegBank = *RBI.getRegBank(SrcReg, MRI, TRI);
     348           8 :       const TargetRegisterClass *SrcRC = getRegClassForTypeOnBank(
     349           8 :           MRI.getType(SrcReg), RegBank, RBI, /* GetAllRegSet */ true);
     350           8 :       if (SrcRC == &AArch64::GPR32allRegClass)
     351           3 :         return selectFP16CopyFromGPR32(I, TII, MRI, SrcReg);
     352             :     }
     353             :     assert(I.isCopy() && "Generic operators do not allow physical registers");
     354             :     return true;
     355             :   }
     356             : 
     357         360 :   const RegisterBank &RegBank = *RBI.getRegBank(DstReg, MRI, TRI);
     358         360 :   const unsigned DstSize = MRI.getType(DstReg).getSizeInBits();
     359             :   (void)DstSize;
     360         360 :   const unsigned SrcSize = RBI.getSizeInBits(SrcReg, MRI, TRI);
     361             :   (void)SrcSize;
     362             :   assert((!TargetRegisterInfo::isPhysicalRegister(SrcReg) || I.isCopy()) &&
     363             :          "No phys reg on generic operators");
     364             :   assert(
     365             :       (DstSize == SrcSize ||
     366             :        // Copies are a mean to setup initial types, the number of
     367             :        // bits may not exactly match.
     368             :        (TargetRegisterInfo::isPhysicalRegister(SrcReg) &&
     369             :         DstSize <= RBI.getSizeInBits(SrcReg, MRI, TRI)) ||
     370             :        // Copies are a mean to copy bits around, as long as we are
     371             :        // on the same register class, that's fine. Otherwise, that
     372             :        // means we need some SUBREG_TO_REG or AND & co.
     373             :        (((DstSize + 31) / 32 == (SrcSize + 31) / 32) && DstSize > SrcSize)) &&
     374             :       "Copy with different width?!");
     375             :   assert((DstSize <= 64 || RegBank.getID() == AArch64::FPRRegBankID) &&
     376             :          "GPRs cannot get more than 64-bit width values");
     377             : 
     378         360 :   const TargetRegisterClass *RC = getRegClassForTypeOnBank(
     379         360 :       MRI.getType(DstReg), RegBank, RBI, /* GetAllRegSet */ true);
     380         360 :   if (!RC) {
     381             :     DEBUG(dbgs() << "Unexpected bitcast size " << DstSize << '\n');
     382             :     return false;
     383             :   }
     384             : 
     385         360 :   if (!TargetRegisterInfo::isPhysicalRegister(SrcReg)) {
     386             :     const RegClassOrRegBank &RegClassOrBank = MRI.getRegClassOrRegBank(SrcReg);
     387             :     const TargetRegisterClass *SrcRC =
     388             :         RegClassOrBank.dyn_cast<const TargetRegisterClass *>();
     389             :     const RegisterBank *RB = nullptr;
     390           2 :     if (!SrcRC) {
     391             :       RB = RegClassOrBank.get<const RegisterBank *>();
     392          53 :       SrcRC = getRegClassForTypeOnBank(MRI.getType(SrcReg), *RB, RBI, true);
     393             :     }
     394             :     // Copies from fpr16 to gpr32 need to use SUBREG_TO_REG.
     395          55 :     if (RC == &AArch64::GPR32allRegClass && SrcRC == &AArch64::FPR16RegClass) {
     396           2 :       unsigned PromoteReg = MRI.createVirtualRegister(&AArch64::FPR32RegClass);
     397           4 :       BuildMI(*I.getParent(), I, I.getDebugLoc(),
     398           2 :               TII.get(AArch64::SUBREG_TO_REG))
     399             :           .addDef(PromoteReg)
     400             :           .addImm(0)
     401             :           .addUse(SrcReg)
     402             :           .addImm(AArch64::hsub);
     403           2 :       MachineOperand &RegOp = I.getOperand(1);
     404           2 :       RegOp.setReg(PromoteReg);
     405          53 :     } else if (RC == &AArch64::FPR16RegClass &&
     406             :                SrcRC == &AArch64::GPR32allRegClass) {
     407           1 :       selectFP16CopyFromGPR32(I, TII, MRI, SrcReg);
     408             :     }
     409             :   }
     410             : 
     411             :   // No need to constrain SrcReg. It will get constrained when
     412             :   // we hit another of its use or its defs.
     413             :   // Copies do not have constraints.
     414         360 :   if (!RBI.constrainGenericRegister(DstReg, *RC, MRI)) {
     415             :     DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())
     416             :                  << " operand\n");
     417             :     return false;
     418             :   }
     419         360 :   I.setDesc(TII.get(AArch64::COPY));
     420         360 :   return true;
     421             : }
     422             : 
     423           0 : static unsigned selectFPConvOpc(unsigned GenericOpc, LLT DstTy, LLT SrcTy) {
     424             :   if (!DstTy.isScalar() || !SrcTy.isScalar())
     425             :     return GenericOpc;
     426             : 
     427           0 :   const unsigned DstSize = DstTy.getSizeInBits();
     428           0 :   const unsigned SrcSize = SrcTy.getSizeInBits();
     429             : 
     430           0 :   switch (DstSize) {
     431           0 :   case 32:
     432           0 :     switch (SrcSize) {
     433           0 :     case 32:
     434           0 :       switch (GenericOpc) {
     435             :       case TargetOpcode::G_SITOFP:
     436             :         return AArch64::SCVTFUWSri;
     437           0 :       case TargetOpcode::G_UITOFP:
     438           0 :         return AArch64::UCVTFUWSri;
     439           0 :       case TargetOpcode::G_FPTOSI:
     440           0 :         return AArch64::FCVTZSUWSr;
     441           0 :       case TargetOpcode::G_FPTOUI:
     442           0 :         return AArch64::FCVTZUUWSr;
     443           0 :       default:
     444           0 :         return GenericOpc;
     445             :       }
     446           0 :     case 64:
     447           0 :       switch (GenericOpc) {
     448             :       case TargetOpcode::G_SITOFP:
     449             :         return AArch64::SCVTFUXSri;
     450           0 :       case TargetOpcode::G_UITOFP:
     451           0 :         return AArch64::UCVTFUXSri;
     452           0 :       case TargetOpcode::G_FPTOSI:
     453           0 :         return AArch64::FCVTZSUWDr;
     454           0 :       case TargetOpcode::G_FPTOUI:
     455           0 :         return AArch64::FCVTZUUWDr;
     456           0 :       default:
     457           0 :         return GenericOpc;
     458             :       }
     459             :     default:
     460             :       return GenericOpc;
     461             :     }
     462           0 :   case 64:
     463           0 :     switch (SrcSize) {
     464           0 :     case 32:
     465           0 :       switch (GenericOpc) {
     466             :       case TargetOpcode::G_SITOFP:
     467             :         return AArch64::SCVTFUWDri;
     468           0 :       case TargetOpcode::G_UITOFP:
     469           0 :         return AArch64::UCVTFUWDri;
     470           0 :       case TargetOpcode::G_FPTOSI:
     471           0 :         return AArch64::FCVTZSUXSr;
     472           0 :       case TargetOpcode::G_FPTOUI:
     473           0 :         return AArch64::FCVTZUUXSr;
     474           0 :       default:
     475           0 :         return GenericOpc;
     476             :       }
     477           0 :     case 64:
     478           0 :       switch (GenericOpc) {
     479             :       case TargetOpcode::G_SITOFP:
     480             :         return AArch64::SCVTFUXDri;
     481           0 :       case TargetOpcode::G_UITOFP:
     482           0 :         return AArch64::UCVTFUXDri;
     483           0 :       case TargetOpcode::G_FPTOSI:
     484           0 :         return AArch64::FCVTZSUXDr;
     485           0 :       case TargetOpcode::G_FPTOUI:
     486           0 :         return AArch64::FCVTZUUXDr;
     487           0 :       default:
     488           0 :         return GenericOpc;
     489             :       }
     490             :     default:
     491             :       return GenericOpc;
     492             :     }
     493             :   default:
     494             :     return GenericOpc;
     495             :   };
     496             :   return GenericOpc;
     497             : }
     498             : 
     499          14 : static AArch64CC::CondCode changeICMPPredToAArch64CC(CmpInst::Predicate P) {
     500          14 :   switch (P) {
     501           0 :   default:
     502           0 :     llvm_unreachable("Unknown condition code!");
     503             :   case CmpInst::ICMP_NE:
     504             :     return AArch64CC::NE;
     505           3 :   case CmpInst::ICMP_EQ:
     506           3 :     return AArch64CC::EQ;
     507           0 :   case CmpInst::ICMP_SGT:
     508           0 :     return AArch64CC::GT;
     509           1 :   case CmpInst::ICMP_SGE:
     510           1 :     return AArch64CC::GE;
     511           0 :   case CmpInst::ICMP_SLT:
     512           0 :     return AArch64CC::LT;
     513           0 :   case CmpInst::ICMP_SLE:
     514           0 :     return AArch64CC::LE;
     515           0 :   case CmpInst::ICMP_UGT:
     516           0 :     return AArch64CC::HI;
     517           3 :   case CmpInst::ICMP_UGE:
     518           3 :     return AArch64CC::HS;
     519           3 :   case CmpInst::ICMP_ULT:
     520           3 :     return AArch64CC::LO;
     521           1 :   case CmpInst::ICMP_ULE:
     522           1 :     return AArch64CC::LS;
     523             :   }
     524             : }
     525             : 
     526           6 : static void changeFCMPPredToAArch64CC(CmpInst::Predicate P,
     527             :                                       AArch64CC::CondCode &CondCode,
     528             :                                       AArch64CC::CondCode &CondCode2) {
     529           6 :   CondCode2 = AArch64CC::AL;
     530           6 :   switch (P) {
     531           0 :   default:
     532           0 :     llvm_unreachable("Unknown FP condition!");
     533           0 :   case CmpInst::FCMP_OEQ:
     534           0 :     CondCode = AArch64CC::EQ;
     535           0 :     break;
     536           0 :   case CmpInst::FCMP_OGT:
     537           0 :     CondCode = AArch64CC::GT;
     538           0 :     break;
     539           0 :   case CmpInst::FCMP_OGE:
     540           0 :     CondCode = AArch64CC::GE;
     541           0 :     break;
     542           0 :   case CmpInst::FCMP_OLT:
     543           0 :     CondCode = AArch64CC::MI;
     544           0 :     break;
     545           0 :   case CmpInst::FCMP_OLE:
     546           0 :     CondCode = AArch64CC::LS;
     547           0 :     break;
     548           3 :   case CmpInst::FCMP_ONE:
     549           3 :     CondCode = AArch64CC::MI;
     550           3 :     CondCode2 = AArch64CC::GT;
     551           3 :     break;
     552           0 :   case CmpInst::FCMP_ORD:
     553           0 :     CondCode = AArch64CC::VC;
     554           0 :     break;
     555           0 :   case CmpInst::FCMP_UNO:
     556           0 :     CondCode = AArch64CC::VS;
     557           0 :     break;
     558           0 :   case CmpInst::FCMP_UEQ:
     559           0 :     CondCode = AArch64CC::EQ;
     560           0 :     CondCode2 = AArch64CC::VS;
     561           0 :     break;
     562           0 :   case CmpInst::FCMP_UGT:
     563           0 :     CondCode = AArch64CC::HI;
     564           0 :     break;
     565           3 :   case CmpInst::FCMP_UGE:
     566           3 :     CondCode = AArch64CC::PL;
     567           3 :     break;
     568           0 :   case CmpInst::FCMP_ULT:
     569           0 :     CondCode = AArch64CC::LT;
     570           0 :     break;
     571           0 :   case CmpInst::FCMP_ULE:
     572           0 :     CondCode = AArch64CC::LE;
     573           0 :     break;
     574           0 :   case CmpInst::FCMP_UNE:
     575           0 :     CondCode = AArch64CC::NE;
     576           0 :     break;
     577             :   }
     578           6 : }
     579             : 
     580          17 : bool AArch64InstructionSelector::selectCompareBranch(
     581             :     MachineInstr &I, MachineFunction &MF, MachineRegisterInfo &MRI) const {
     582             : 
     583          17 :   const unsigned CondReg = I.getOperand(0).getReg();
     584          17 :   MachineBasicBlock *DestMBB = I.getOperand(1).getMBB();
     585          17 :   MachineInstr *CCMI = MRI.getVRegDef(CondReg);
     586          34 :   if (CCMI->getOpcode() == TargetOpcode::G_TRUNC)
     587          17 :     CCMI = MRI.getVRegDef(CCMI->getOperand(1).getReg());
     588          34 :   if (CCMI->getOpcode() != TargetOpcode::G_ICMP)
     589             :     return false;
     590             : 
     591           9 :   unsigned LHS = CCMI->getOperand(2).getReg();
     592           9 :   unsigned RHS = CCMI->getOperand(3).getReg();
     593           9 :   if (!getConstantVRegVal(RHS, MRI))
     594             :     std::swap(RHS, LHS);
     595             : 
     596           9 :   const auto RHSImm = getConstantVRegVal(RHS, MRI);
     597           9 :   if (!RHSImm || *RHSImm != 0)
     598             :     return false;
     599             : 
     600           9 :   const RegisterBank &RB = *RBI.getRegBank(LHS, MRI, TRI);
     601           9 :   if (RB.getID() != AArch64::GPRRegBankID)
     602             :     return false;
     603             : 
     604           9 :   const auto Pred = (CmpInst::Predicate)CCMI->getOperand(1).getPredicate();
     605           9 :   if (Pred != CmpInst::ICMP_NE && Pred != CmpInst::ICMP_EQ)
     606             :     return false;
     607             : 
     608           4 :   const unsigned CmpWidth = MRI.getType(LHS).getSizeInBits();
     609             :   unsigned CBOpc = 0;
     610           4 :   if (CmpWidth <= 32)
     611           2 :     CBOpc = (Pred == CmpInst::ICMP_EQ ? AArch64::CBZW : AArch64::CBNZW);
     612           2 :   else if (CmpWidth == 64)
     613           2 :     CBOpc = (Pred == CmpInst::ICMP_EQ ? AArch64::CBZX : AArch64::CBNZX);
     614             :   else
     615             :     return false;
     616             : 
     617          12 :   BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(CBOpc))
     618             :       .addUse(LHS)
     619             :       .addMBB(DestMBB)
     620           4 :       .constrainAllUses(TII, TRI, RBI);
     621             : 
     622           4 :   I.eraseFromParent();
     623             :   return true;
     624             : }
     625             : 
     626             : bool AArch64InstructionSelector::selectVaStartAAPCS(
     627             :     MachineInstr &I, MachineFunction &MF, MachineRegisterInfo &MRI) const {
     628             :   return false;
     629             : }
     630             : 
     631           1 : bool AArch64InstructionSelector::selectVaStartDarwin(
     632             :     MachineInstr &I, MachineFunction &MF, MachineRegisterInfo &MRI) const {
     633             :   AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>();
     634           1 :   unsigned ListReg = I.getOperand(0).getReg();
     635             : 
     636           1 :   unsigned ArgsAddrReg = MRI.createVirtualRegister(&AArch64::GPR64RegClass);
     637             : 
     638             :   auto MIB =
     639           3 :       BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(AArch64::ADDXri))
     640             :           .addDef(ArgsAddrReg)
     641           1 :           .addFrameIndex(FuncInfo->getVarArgsStackIndex())
     642             :           .addImm(0)
     643           1 :           .addImm(0);
     644             : 
     645           1 :   constrainSelectedInstRegOperands(*MIB, TII, TRI, RBI);
     646             : 
     647           3 :   MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(AArch64::STRXui))
     648             :             .addUse(ArgsAddrReg)
     649             :             .addUse(ListReg)
     650             :             .addImm(0)
     651           1 :             .addMemOperand(*I.memoperands_begin());
     652             : 
     653           1 :   constrainSelectedInstRegOperands(*MIB, TII, TRI, RBI);
     654           1 :   I.eraseFromParent();
     655           1 :   return true;
     656             : }
     657             : 
     658        1301 : bool AArch64InstructionSelector::select(MachineInstr &I,
     659             :                                         CodeGenCoverage &CoverageInfo) const {
     660             :   assert(I.getParent() && "Instruction should be in a basic block!");
     661             :   assert(I.getParent()->getParent() && "Instruction should be in a function!");
     662             : 
     663        1301 :   MachineBasicBlock &MBB = *I.getParent();
     664        1301 :   MachineFunction &MF = *MBB.getParent();
     665        1301 :   MachineRegisterInfo &MRI = MF.getRegInfo();
     666             : 
     667        1301 :   unsigned Opcode = I.getOpcode();
     668             :   // G_PHI requires same handling as PHI
     669        1301 :   if (!isPreISelGenericOpcode(Opcode) || Opcode == TargetOpcode::G_PHI) {
     670             :     // Certain non-generic instructions also need some special handling.
     671             : 
     672         689 :     if (Opcode ==  TargetOpcode::LOAD_STACK_GUARD)
     673           0 :       return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
     674             : 
     675         689 :     if (Opcode == TargetOpcode::PHI || Opcode == TargetOpcode::G_PHI) {
     676           9 :       const unsigned DefReg = I.getOperand(0).getReg();
     677           9 :       const LLT DefTy = MRI.getType(DefReg);
     678             : 
     679             :       const TargetRegisterClass *DefRC = nullptr;
     680           9 :       if (TargetRegisterInfo::isPhysicalRegister(DefReg)) {
     681           0 :         DefRC = TRI.getRegClass(DefReg);
     682             :       } else {
     683             :         const RegClassOrRegBank &RegClassOrBank =
     684             :             MRI.getRegClassOrRegBank(DefReg);
     685             : 
     686             :         DefRC = RegClassOrBank.dyn_cast<const TargetRegisterClass *>();
     687           2 :         if (!DefRC) {
     688           7 :           if (!DefTy.isValid()) {
     689             :             DEBUG(dbgs() << "PHI operand has no type, not a gvreg?\n");
     690             :             return false;
     691             :           }
     692             :           const RegisterBank &RB = *RegClassOrBank.get<const RegisterBank *>();
     693           7 :           DefRC = getRegClassForTypeOnBank(DefTy, RB, RBI);
     694           7 :           if (!DefRC) {
     695             :             DEBUG(dbgs() << "PHI operand has unexpected size/bank\n");
     696             :             return false;
     697             :           }
     698             :         }
     699             :       }
     700           9 :       I.setDesc(TII.get(TargetOpcode::PHI));
     701             : 
     702           9 :       return RBI.constrainGenericRegister(DefReg, *DefRC, MRI);
     703             :     }
     704             : 
     705         680 :     if (I.isCopy())
     706         579 :       return selectCopy(I, TII, MRI, TRI, RBI);
     707             : 
     708             :     return true;
     709             :   }
     710             : 
     711             : 
     712         612 :   if (I.getNumOperands() != I.getNumExplicitOperands()) {
     713             :     DEBUG(dbgs() << "Generic instruction has unexpected implicit operands\n");
     714             :     return false;
     715             :   }
     716             : 
     717         612 :   if (selectImpl(I, CoverageInfo))
     718             :     return true;
     719             : 
     720             :   LLT Ty =
     721         524 :       I.getOperand(0).isReg() ? MRI.getType(I.getOperand(0).getReg()) : LLT{};
     722             : 
     723         262 :   switch (Opcode) {
     724          17 :   case TargetOpcode::G_BRCOND: {
     725          17 :     if (Ty.getSizeInBits() > 32) {
     726             :       // We shouldn't need this on AArch64, but it would be implemented as an
     727             :       // EXTRACT_SUBREG followed by a TBNZW because TBNZX has no encoding if the
     728             :       // bit being tested is < 32.
     729             :       DEBUG(dbgs() << "G_BRCOND has type: " << Ty
     730             :                    << ", expected at most 32-bits");
     731             :       return false;
     732             :     }
     733             : 
     734          17 :     const unsigned CondReg = I.getOperand(0).getReg();
     735          17 :     MachineBasicBlock *DestMBB = I.getOperand(1).getMBB();
     736             : 
     737          17 :     if (selectCompareBranch(I, MF, MRI))
     738             :       return true;
     739             : 
     740          39 :     auto MIB = BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::TBNZW))
     741             :                    .addUse(CondReg)
     742             :                    .addImm(/*bit offset=*/0)
     743          13 :                    .addMBB(DestMBB);
     744             : 
     745          13 :     I.eraseFromParent();
     746          13 :     return constrainSelectedInstRegOperands(*MIB.getInstr(), TII, TRI, RBI);
     747             :   }
     748             : 
     749           2 :   case TargetOpcode::G_BRINDIRECT: {
     750           2 :     I.setDesc(TII.get(AArch64::BR));
     751           2 :     return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
     752             :   }
     753             : 
     754          28 :   case TargetOpcode::G_FCONSTANT:
     755             :   case TargetOpcode::G_CONSTANT: {
     756             :     const bool isFP = Opcode == TargetOpcode::G_FCONSTANT;
     757             : 
     758             :     const LLT s32 = LLT::scalar(32);
     759             :     const LLT s64 = LLT::scalar(64);
     760             :     const LLT p0 = LLT::pointer(0, 64);
     761             : 
     762          28 :     const unsigned DefReg = I.getOperand(0).getReg();
     763          28 :     const LLT DefTy = MRI.getType(DefReg);
     764          28 :     const unsigned DefSize = DefTy.getSizeInBits();
     765          28 :     const RegisterBank &RB = *RBI.getRegBank(DefReg, MRI, TRI);
     766             : 
     767             :     // FIXME: Redundant check, but even less readable when factored out.
     768          28 :     if (isFP) {
     769             :       if (Ty != s32 && Ty != s64) {
     770             :         DEBUG(dbgs() << "Unable to materialize FP " << Ty
     771             :                      << " constant, expected: " << s32 << " or " << s64
     772             :                      << '\n');
     773             :         return false;
     774             :       }
     775             : 
     776          28 :       if (RB.getID() != AArch64::FPRRegBankID) {
     777             :         DEBUG(dbgs() << "Unable to materialize FP " << Ty
     778             :                      << " constant on bank: " << RB << ", expected: FPR\n");
     779             :         return false;
     780             :       }
     781             : 
     782             :       // The case when we have 0.0 is covered by tablegen. Reject it here so we
     783             :       // can be sure tablegen works correctly and isn't rescued by this code.
     784          56 :       if (I.getOperand(1).getFPImm()->getValueAPF().isExactlyValue(0.0))
     785             :         return false;
     786             :     } else {
     787             :       // s32 and s64 are covered by tablegen.
     788             :       if (Ty != p0) {
     789             :         DEBUG(dbgs() << "Unable to materialize integer " << Ty
     790             :                      << " constant, expected: " << s32 << ", " << s64 << ", or "
     791             :                      << p0 << '\n');
     792             :         return false;
     793             :       }
     794             : 
     795           0 :       if (RB.getID() != AArch64::GPRRegBankID) {
     796             :         DEBUG(dbgs() << "Unable to materialize integer " << Ty
     797             :                      << " constant on bank: " << RB << ", expected: GPR\n");
     798             :         return false;
     799             :       }
     800             :     }
     801             : 
     802             :     const unsigned MovOpc =
     803          28 :         DefSize == 32 ? AArch64::MOVi32imm : AArch64::MOVi64imm;
     804             : 
     805          28 :     I.setDesc(TII.get(MovOpc));
     806             : 
     807          28 :     if (isFP) {
     808          28 :       const TargetRegisterClass &GPRRC =
     809             :           DefSize == 32 ? AArch64::GPR32RegClass : AArch64::GPR64RegClass;
     810          28 :       const TargetRegisterClass &FPRRC =
     811             :           DefSize == 32 ? AArch64::FPR32RegClass : AArch64::FPR64RegClass;
     812             : 
     813          28 :       const unsigned DefGPRReg = MRI.createVirtualRegister(&GPRRC);
     814          28 :       MachineOperand &RegOp = I.getOperand(0);
     815          28 :       RegOp.setReg(DefGPRReg);
     816             : 
     817          56 :       BuildMI(MBB, std::next(I.getIterator()), I.getDebugLoc(),
     818          56 :               TII.get(AArch64::COPY))
     819             :           .addDef(DefReg)
     820             :           .addUse(DefGPRReg);
     821             : 
     822          28 :       if (!RBI.constrainGenericRegister(DefReg, FPRRC, MRI)) {
     823             :         DEBUG(dbgs() << "Failed to constrain G_FCONSTANT def operand\n");
     824             :         return false;
     825             :       }
     826             : 
     827          28 :       MachineOperand &ImmOp = I.getOperand(1);
     828             :       // FIXME: Is going through int64_t always correct?
     829          28 :       ImmOp.ChangeToImmediate(
     830          84 :           ImmOp.getFPImm()->getValueAPF().bitcastToAPInt().getZExtValue());
     831           0 :     } else if (I.getOperand(1).isCImm()) {
     832           0 :       uint64_t Val = I.getOperand(1).getCImm()->getZExtValue();
     833           0 :       I.getOperand(1).ChangeToImmediate(Val);
     834           0 :     } else if (I.getOperand(1).isImm()) {
     835           0 :       uint64_t Val = I.getOperand(1).getImm();
     836           0 :       I.getOperand(1).ChangeToImmediate(Val);
     837             :     }
     838             : 
     839          28 :     constrainSelectedInstRegOperands(I, TII, TRI, RBI);
     840          28 :     return true;
     841             :   }
     842           4 :   case TargetOpcode::G_EXTRACT: {
     843           4 :     LLT SrcTy = MRI.getType(I.getOperand(1).getReg());
     844           4 :     LLT DstTy = MRI.getType(I.getOperand(0).getReg());
     845             :     (void)DstTy;
     846           4 :     unsigned SrcSize = SrcTy.getSizeInBits();
     847             :     // Larger extracts are vectors, same-size extracts should be something else
     848             :     // by now (either split up or simplified to a COPY).
     849           4 :     if (SrcTy.getSizeInBits() > 64 || Ty.getSizeInBits() > 32)
     850             :       return false;
     851             : 
     852           4 :     I.setDesc(TII.get(SrcSize == 64 ? AArch64::UBFMXri : AArch64::UBFMWri));
     853           8 :     MachineInstrBuilder(MF, I).addImm(I.getOperand(2).getImm() +
     854           8 :                                       Ty.getSizeInBits() - 1);
     855             : 
     856           4 :     if (SrcSize < 64) {
     857             :       assert(SrcSize == 32 && DstTy.getSizeInBits() == 16 &&
     858             :              "unexpected G_EXTRACT types");
     859           2 :       return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
     860             :     }
     861             : 
     862           2 :     unsigned DstReg = MRI.createGenericVirtualRegister(LLT::scalar(64));
     863           4 :     BuildMI(MBB, std::next(I.getIterator()), I.getDebugLoc(),
     864           4 :             TII.get(AArch64::COPY))
     865           2 :         .addDef(I.getOperand(0).getReg())
     866             :         .addUse(DstReg, 0, AArch64::sub_32);
     867           2 :     RBI.constrainGenericRegister(I.getOperand(0).getReg(),
     868             :                                  AArch64::GPR32RegClass, MRI);
     869           2 :     I.getOperand(0).setReg(DstReg);
     870             : 
     871           2 :     return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
     872             :   }
     873             : 
     874           6 :   case TargetOpcode::G_INSERT: {
     875           6 :     LLT SrcTy = MRI.getType(I.getOperand(2).getReg());
     876           6 :     LLT DstTy = MRI.getType(I.getOperand(0).getReg());
     877           6 :     unsigned DstSize = DstTy.getSizeInBits();
     878             :     // Larger inserts are vectors, same-size ones should be something else by
     879             :     // now (split up or turned into COPYs).
     880           6 :     if (Ty.getSizeInBits() > 64 || SrcTy.getSizeInBits() > 32)
     881             :       return false;
     882             : 
     883           6 :     I.setDesc(TII.get(DstSize == 64 ? AArch64::BFMXri : AArch64::BFMWri));
     884           6 :     unsigned LSB = I.getOperand(3).getImm();
     885           6 :     unsigned Width = MRI.getType(I.getOperand(2).getReg()).getSizeInBits();
     886           6 :     I.getOperand(3).setImm((DstSize - LSB) % DstSize);
     887           6 :     MachineInstrBuilder(MF, I).addImm(Width - 1);
     888             : 
     889           6 :     if (DstSize < 64) {
     890             :       assert(DstSize == 32 && SrcTy.getSizeInBits() == 16 &&
     891             :              "unexpected G_INSERT types");
     892           4 :       return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
     893             :     }
     894             : 
     895           2 :     unsigned SrcReg = MRI.createGenericVirtualRegister(LLT::scalar(64));
     896           4 :     BuildMI(MBB, I.getIterator(), I.getDebugLoc(),
     897           4 :             TII.get(AArch64::SUBREG_TO_REG))
     898             :         .addDef(SrcReg)
     899             :         .addImm(0)
     900           2 :         .addUse(I.getOperand(2).getReg())
     901             :         .addImm(AArch64::sub_32);
     902           2 :     RBI.constrainGenericRegister(I.getOperand(2).getReg(),
     903             :                                  AArch64::GPR32RegClass, MRI);
     904           4 :     I.getOperand(2).setReg(SrcReg);
     905             : 
     906           2 :     return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
     907             :   }
     908             :   case TargetOpcode::G_FRAME_INDEX: {
     909             :     // allocas and G_FRAME_INDEX are only supported in addrspace(0).
     910             :     if (Ty != LLT::pointer(0, 64)) {
     911             :       DEBUG(dbgs() << "G_FRAME_INDEX pointer has type: " << Ty
     912             :             << ", expected: " << LLT::pointer(0, 64) << '\n');
     913             :       return false;
     914             :     }
     915           3 :     I.setDesc(TII.get(AArch64::ADDXri));
     916             : 
     917             :     // MOs for a #0 shifted immediate.
     918           3 :     I.addOperand(MachineOperand::CreateImm(0));
     919           3 :     I.addOperand(MachineOperand::CreateImm(0));
     920             : 
     921           3 :     return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
     922             :   }
     923             : 
     924          37 :   case TargetOpcode::G_GLOBAL_VALUE: {
     925          37 :     auto GV = I.getOperand(1).getGlobal();
     926          37 :     if (GV->isThreadLocal()) {
     927             :       // FIXME: we don't support TLS yet.
     928             :       return false;
     929             :     }
     930          37 :     unsigned char OpFlags = STI.ClassifyGlobalReference(GV, TM);
     931          37 :     if (OpFlags & AArch64II::MO_GOT) {
     932          10 :       I.setDesc(TII.get(AArch64::LOADgot));
     933          10 :       I.getOperand(1).setTargetFlags(OpFlags);
     934          27 :     } else if (TM.getCodeModel() == CodeModel::Large) {
     935             :       // Materialize the global using movz/movk instructions.
     936           2 :       unsigned MovZDstReg = MRI.createVirtualRegister(&AArch64::GPR64RegClass);
     937           4 :       auto InsertPt = std::next(I.getIterator());
     938             :       auto MovZ =
     939           6 :           BuildMI(MBB, InsertPt, I.getDebugLoc(), TII.get(AArch64::MOVZXi))
     940           2 :               .addDef(MovZDstReg);
     941           4 :       MovZ->addOperand(MF, I.getOperand(1));
     942           2 :       MovZ->getOperand(1).setTargetFlags(OpFlags | AArch64II::MO_G0 |
     943             :                                          AArch64II::MO_NC);
     944           2 :       MovZ->addOperand(MF, MachineOperand::CreateImm(0));
     945           2 :       constrainSelectedInstRegOperands(*MovZ, TII, TRI, RBI);
     946             : 
     947             :       auto BuildMovK = [&](unsigned SrcReg, unsigned char Flags,
     948           6 :                            unsigned Offset, unsigned ForceDstReg) {
     949             :         unsigned DstReg =
     950          10 :             ForceDstReg ? ForceDstReg
     951           4 :                         : MRI.createVirtualRegister(&AArch64::GPR64RegClass);
     952          18 :         auto MovI = BuildMI(MBB, InsertPt, MovZ->getDebugLoc(),
     953          18 :                             TII.get(AArch64::MOVKXi))
     954             :                         .addDef(DstReg)
     955           6 :                         .addReg(SrcReg);
     956          12 :         MovI->addOperand(MF, MachineOperand::CreateGA(
     957          12 :                                  GV, MovZ->getOperand(1).getOffset(), Flags));
     958          12 :         MovI->addOperand(MF, MachineOperand::CreateImm(Offset));
     959           6 :         constrainSelectedInstRegOperands(*MovI, TII, TRI, RBI);
     960           6 :         return DstReg;
     961           2 :       };
     962           2 :       unsigned DstReg = BuildMovK(MovZ->getOperand(0).getReg(),
     963           2 :                                   AArch64II::MO_G1 | AArch64II::MO_NC, 16, 0);
     964           2 :       DstReg = BuildMovK(DstReg, AArch64II::MO_G2 | AArch64II::MO_NC, 32, 0);
     965           2 :       BuildMovK(DstReg, AArch64II::MO_G3, 48, I.getOperand(0).getReg());
     966           2 :       I.eraseFromParent();
     967             :       return true;
     968             :     } else {
     969          25 :       I.setDesc(TII.get(AArch64::MOVaddr));
     970          25 :       I.getOperand(1).setTargetFlags(OpFlags | AArch64II::MO_PAGE);
     971             :       MachineInstrBuilder MIB(MF, I);
     972          50 :       MIB.addGlobalAddress(GV, I.getOperand(1).getOffset(),
     973          50 :                            OpFlags | AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
     974             :     }
     975          35 :     return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
     976             :   }
     977             : 
     978          18 :   case TargetOpcode::G_LOAD:
     979             :   case TargetOpcode::G_STORE: {
     980          18 :     LLT MemTy = Ty;
     981          18 :     LLT PtrTy = MRI.getType(I.getOperand(1).getReg());
     982             : 
     983             :     if (PtrTy != LLT::pointer(0, 64)) {
     984             :       DEBUG(dbgs() << "Load/Store pointer has type: " << PtrTy
     985             :                    << ", expected: " << LLT::pointer(0, 64) << '\n');
     986             :       return false;
     987             :     }
     988             : 
     989          18 :     auto &MemOp = **I.memoperands_begin();
     990          18 :     if (MemOp.getOrdering() != AtomicOrdering::NotAtomic) {
     991             :       DEBUG(dbgs() << "Atomic load/store not supported yet\n");
     992             :       return false;
     993             :     }
     994             : 
     995             :     // FIXME: PR36018: Volatile loads in some cases are incorrectly selected by
     996             :     // folding with an extend. Until we have a G_SEXTLOAD solution bail out if
     997             :     // we hit one.
     998          20 :     if (Opcode == TargetOpcode::G_LOAD && MemOp.isVolatile())
     999             :       return false;
    1000             : 
    1001          13 :     const unsigned PtrReg = I.getOperand(1).getReg();
    1002             : #ifndef NDEBUG
    1003             :     const RegisterBank &PtrRB = *RBI.getRegBank(PtrReg, MRI, TRI);
    1004             :     // Sanity-check the pointer register.
    1005             :     assert(PtrRB.getID() == AArch64::GPRRegBankID &&
    1006             :            "Load/Store pointer operand isn't a GPR");
    1007             :     assert(MRI.getType(PtrReg).isPointer() &&
    1008             :            "Load/Store pointer operand isn't a pointer");
    1009             : #endif
    1010             : 
    1011          13 :     const unsigned ValReg = I.getOperand(0).getReg();
    1012          13 :     const RegisterBank &RB = *RBI.getRegBank(ValReg, MRI, TRI);
    1013             : 
    1014             :     const unsigned NewOpc =
    1015          26 :         selectLoadStoreUIOp(I.getOpcode(), RB.getID(), MemTy.getSizeInBits());
    1016          13 :     if (NewOpc == I.getOpcode())
    1017             :       return false;
    1018             : 
    1019          13 :     I.setDesc(TII.get(NewOpc));
    1020             : 
    1021             :     uint64_t Offset = 0;
    1022          13 :     auto *PtrMI = MRI.getVRegDef(PtrReg);
    1023             : 
    1024             :     // Try to fold a GEP into our unsigned immediate addressing mode.
    1025          26 :     if (PtrMI->getOpcode() == TargetOpcode::G_GEP) {
    1026           3 :       if (auto COff = getConstantVRegVal(PtrMI->getOperand(2).getReg(), MRI)) {
    1027           3 :         int64_t Imm = *COff;
    1028           3 :         const unsigned Size = MemTy.getSizeInBits() / 8;
    1029             :         const unsigned Scale = Log2_32(Size);
    1030           3 :         if ((Imm & (Size - 1)) == 0 && Imm >= 0 && Imm < (0x1000 << Scale)) {
    1031           3 :           unsigned Ptr2Reg = PtrMI->getOperand(1).getReg();
    1032           6 :           I.getOperand(1).setReg(Ptr2Reg);
    1033           3 :           PtrMI = MRI.getVRegDef(Ptr2Reg);
    1034           3 :           Offset = Imm / Size;
    1035             :         }
    1036             :       }
    1037             :     }
    1038             : 
    1039             :     // If we haven't folded anything into our addressing mode yet, try to fold
    1040             :     // a frame index into the base+offset.
    1041          13 :     if (!Offset && PtrMI->getOpcode() == TargetOpcode::G_FRAME_INDEX)
    1042           6 :       I.getOperand(1).ChangeToFrameIndex(PtrMI->getOperand(1).getIndex());
    1043             : 
    1044          26 :     I.addOperand(MachineOperand::CreateImm(Offset));
    1045             : 
    1046             :     // If we're storing a 0, use WZR/XZR.
    1047          13 :     if (auto CVal = getConstantVRegVal(ValReg, MRI)) {
    1048           0 :       if (*CVal == 0 && Opcode == TargetOpcode::G_STORE) {
    1049           0 :         if (I.getOpcode() == AArch64::STRWui)
    1050           0 :           I.getOperand(0).setReg(AArch64::WZR);
    1051           0 :         else if (I.getOpcode() == AArch64::STRXui)
    1052           0 :           I.getOperand(0).setReg(AArch64::XZR);
    1053             :       }
    1054             :     }
    1055             : 
    1056          13 :     return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
    1057             :   }
    1058             : 
    1059           2 :   case TargetOpcode::G_SMULH:
    1060             :   case TargetOpcode::G_UMULH: {
    1061             :     // Reject the various things we don't support yet.
    1062           2 :     if (unsupportedBinOp(I, RBI, MRI, TRI))
    1063             :       return false;
    1064             : 
    1065           2 :     const unsigned DefReg = I.getOperand(0).getReg();
    1066           2 :     const RegisterBank &RB = *RBI.getRegBank(DefReg, MRI, TRI);
    1067             : 
    1068           2 :     if (RB.getID() != AArch64::GPRRegBankID) {
    1069             :       DEBUG(dbgs() << "G_[SU]MULH on bank: " << RB << ", expected: GPR\n");
    1070             :       return false;
    1071             :     }
    1072             : 
    1073             :     if (Ty != LLT::scalar(64)) {
    1074             :       DEBUG(dbgs() << "G_[SU]MULH has type: " << Ty
    1075             :                    << ", expected: " << LLT::scalar(64) << '\n');
    1076             :       return false;
    1077             :     }
    1078             : 
    1079           4 :     unsigned NewOpc = I.getOpcode() == TargetOpcode::G_SMULH ? AArch64::SMULHrr
    1080             :                                                              : AArch64::UMULHrr;
    1081           2 :     I.setDesc(TII.get(NewOpc));
    1082             : 
    1083             :     // Now that we selected an opcode, we need to constrain the register
    1084             :     // operands to use appropriate classes.
    1085           2 :     return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
    1086             :   }
    1087          14 :   case TargetOpcode::G_FADD:
    1088             :   case TargetOpcode::G_FSUB:
    1089             :   case TargetOpcode::G_FMUL:
    1090             :   case TargetOpcode::G_FDIV:
    1091             : 
    1092             :   case TargetOpcode::G_OR:
    1093             :   case TargetOpcode::G_SHL:
    1094             :   case TargetOpcode::G_LSHR:
    1095             :   case TargetOpcode::G_ASHR:
    1096             :   case TargetOpcode::G_GEP: {
    1097             :     // Reject the various things we don't support yet.
    1098          14 :     if (unsupportedBinOp(I, RBI, MRI, TRI))
    1099             :       return false;
    1100             : 
    1101          14 :     const unsigned OpSize = Ty.getSizeInBits();
    1102             : 
    1103          14 :     const unsigned DefReg = I.getOperand(0).getReg();
    1104          14 :     const RegisterBank &RB = *RBI.getRegBank(DefReg, MRI, TRI);
    1105             : 
    1106          28 :     const unsigned NewOpc = selectBinaryOp(I.getOpcode(), RB.getID(), OpSize);
    1107          14 :     if (NewOpc == I.getOpcode())
    1108             :       return false;
    1109             : 
    1110          14 :     I.setDesc(TII.get(NewOpc));
    1111             :     // FIXME: Should the type be always reset in setDesc?
    1112             : 
    1113             :     // Now that we selected an opcode, we need to constrain the register
    1114             :     // operands to use appropriate classes.
    1115          14 :     return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
    1116             :   }
    1117             : 
    1118           3 :   case TargetOpcode::G_PTR_MASK: {
    1119           3 :     uint64_t Align = I.getOperand(2).getImm();
    1120           3 :     if (Align >= 64 || Align == 0)
    1121             :       return false;
    1122             : 
    1123           3 :     uint64_t Mask = ~((1ULL << Align) - 1);
    1124           3 :     I.setDesc(TII.get(AArch64::ANDXri));
    1125           3 :     I.getOperand(2).setImm(AArch64_AM::encodeLogicalImmediate(Mask, 64));
    1126             : 
    1127           3 :     return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
    1128             :   }
    1129          53 :   case TargetOpcode::G_PTRTOINT:
    1130             :   case TargetOpcode::G_TRUNC: {
    1131          53 :     const LLT DstTy = MRI.getType(I.getOperand(0).getReg());
    1132          53 :     const LLT SrcTy = MRI.getType(I.getOperand(1).getReg());
    1133             : 
    1134          53 :     const unsigned DstReg = I.getOperand(0).getReg();
    1135          53 :     const unsigned SrcReg = I.getOperand(1).getReg();
    1136             : 
    1137          53 :     const RegisterBank &DstRB = *RBI.getRegBank(DstReg, MRI, TRI);
    1138          53 :     const RegisterBank &SrcRB = *RBI.getRegBank(SrcReg, MRI, TRI);
    1139             : 
    1140          53 :     if (DstRB.getID() != SrcRB.getID()) {
    1141             :       DEBUG(dbgs() << "G_TRUNC/G_PTRTOINT input/output on different banks\n");
    1142             :       return false;
    1143             :     }
    1144             : 
    1145          53 :     if (DstRB.getID() == AArch64::GPRRegBankID) {
    1146             :       const TargetRegisterClass *DstRC =
    1147          53 :           getRegClassForTypeOnBank(DstTy, DstRB, RBI);
    1148          53 :       if (!DstRC)
    1149             :         return false;
    1150             : 
    1151             :       const TargetRegisterClass *SrcRC =
    1152          53 :           getRegClassForTypeOnBank(SrcTy, SrcRB, RBI);
    1153          53 :       if (!SrcRC)
    1154             :         return false;
    1155             : 
    1156         106 :       if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, MRI) ||
    1157          53 :           !RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) {
    1158             :         DEBUG(dbgs() << "Failed to constrain G_TRUNC/G_PTRTOINT\n");
    1159             :         return false;
    1160             :       }
    1161             : 
    1162          53 :       if (DstRC == SrcRC) {
    1163             :         // Nothing to be done
    1164           5 :       } else if (Opcode == TargetOpcode::G_TRUNC && DstTy == LLT::scalar(32) &&
    1165             :                  SrcTy == LLT::scalar(64)) {
    1166           0 :         llvm_unreachable("TableGen can import this case");
    1167             :         return false;
    1168          10 :       } else if (DstRC == &AArch64::GPR32RegClass &&
    1169             :                  SrcRC == &AArch64::GPR64RegClass) {
    1170           5 :         I.getOperand(1).setSubReg(AArch64::sub_32);
    1171             :       } else {
    1172             :         DEBUG(dbgs() << "Unhandled mismatched classes in G_TRUNC/G_PTRTOINT\n");
    1173             :         return false;
    1174             :       }
    1175             : 
    1176          53 :       I.setDesc(TII.get(TargetOpcode::COPY));
    1177          53 :       return true;
    1178           0 :     } else if (DstRB.getID() == AArch64::FPRRegBankID) {
    1179             :       if (DstTy == LLT::vector(4, 16) && SrcTy == LLT::vector(4, 32)) {
    1180           0 :         I.setDesc(TII.get(AArch64::XTNv4i16));
    1181           0 :         constrainSelectedInstRegOperands(I, TII, TRI, RBI);
    1182           0 :         return true;
    1183             :       }
    1184             :     }
    1185             : 
    1186             :     return false;
    1187             :   }
    1188             : 
    1189          25 :   case TargetOpcode::G_ANYEXT: {
    1190          25 :     const unsigned DstReg = I.getOperand(0).getReg();
    1191          25 :     const unsigned SrcReg = I.getOperand(1).getReg();
    1192             : 
    1193          25 :     const RegisterBank &RBDst = *RBI.getRegBank(DstReg, MRI, TRI);
    1194          25 :     if (RBDst.getID() != AArch64::GPRRegBankID) {
    1195             :       DEBUG(dbgs() << "G_ANYEXT on bank: " << RBDst << ", expected: GPR\n");
    1196             :       return false;
    1197             :     }
    1198             : 
    1199          25 :     const RegisterBank &RBSrc = *RBI.getRegBank(SrcReg, MRI, TRI);
    1200          25 :     if (RBSrc.getID() != AArch64::GPRRegBankID) {
    1201             :       DEBUG(dbgs() << "G_ANYEXT on bank: " << RBSrc << ", expected: GPR\n");
    1202             :       return false;
    1203             :     }
    1204             : 
    1205          25 :     const unsigned DstSize = MRI.getType(DstReg).getSizeInBits();
    1206             : 
    1207          25 :     if (DstSize == 0) {
    1208             :       DEBUG(dbgs() << "G_ANYEXT operand has no size, not a gvreg?\n");
    1209             :       return false;
    1210             :     }
    1211             : 
    1212          25 :     if (DstSize != 64 && DstSize > 32) {
    1213             :       DEBUG(dbgs() << "G_ANYEXT to size: " << DstSize
    1214             :                    << ", expected: 32 or 64\n");
    1215             :       return false;
    1216             :     }
    1217             :     // At this point G_ANYEXT is just like a plain COPY, but we need
    1218             :     // to explicitly form the 64-bit value if any.
    1219          25 :     if (DstSize > 32) {
    1220           2 :       unsigned ExtSrc = MRI.createVirtualRegister(&AArch64::GPR64allRegClass);
    1221           6 :       BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::SUBREG_TO_REG))
    1222             :           .addDef(ExtSrc)
    1223             :           .addImm(0)
    1224             :           .addUse(SrcReg)
    1225             :           .addImm(AArch64::sub_32);
    1226           4 :       I.getOperand(1).setReg(ExtSrc);
    1227             :     }
    1228          25 :     return selectCopy(I, TII, MRI, TRI, RBI);
    1229             :   }
    1230             : 
    1231          11 :   case TargetOpcode::G_ZEXT:
    1232             :   case TargetOpcode::G_SEXT: {
    1233          11 :     unsigned Opcode = I.getOpcode();
    1234          11 :     const LLT DstTy = MRI.getType(I.getOperand(0).getReg()),
    1235          11 :               SrcTy = MRI.getType(I.getOperand(1).getReg());
    1236             :     const bool isSigned = Opcode == TargetOpcode::G_SEXT;
    1237          11 :     const unsigned DefReg = I.getOperand(0).getReg();
    1238          11 :     const unsigned SrcReg = I.getOperand(1).getReg();
    1239          11 :     const RegisterBank &RB = *RBI.getRegBank(DefReg, MRI, TRI);
    1240             : 
    1241          11 :     if (RB.getID() != AArch64::GPRRegBankID) {
    1242             :       DEBUG(dbgs() << TII.getName(I.getOpcode()) << " on bank: " << RB
    1243             :                    << ", expected: GPR\n");
    1244             :       return false;
    1245             :     }
    1246             : 
    1247             :     MachineInstr *ExtI;
    1248             :     if (DstTy == LLT::scalar(64)) {
    1249             :       // FIXME: Can we avoid manually doing this?
    1250           5 :       if (!RBI.constrainGenericRegister(SrcReg, AArch64::GPR32RegClass, MRI)) {
    1251             :         DEBUG(dbgs() << "Failed to constrain " << TII.getName(Opcode)
    1252             :                      << " operand\n");
    1253             :         return false;
    1254             :       }
    1255             : 
    1256             :       const unsigned SrcXReg =
    1257           5 :           MRI.createVirtualRegister(&AArch64::GPR64RegClass);
    1258          15 :       BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::SUBREG_TO_REG))
    1259             :           .addDef(SrcXReg)
    1260             :           .addImm(0)
    1261             :           .addUse(SrcReg)
    1262             :           .addImm(AArch64::sub_32);
    1263             : 
    1264           5 :       const unsigned NewOpc = isSigned ? AArch64::SBFMXri : AArch64::UBFMXri;
    1265          15 :       ExtI = BuildMI(MBB, I, I.getDebugLoc(), TII.get(NewOpc))
    1266             :                  .addDef(DefReg)
    1267             :                  .addUse(SrcXReg)
    1268             :                  .addImm(0)
    1269           5 :                  .addImm(SrcTy.getSizeInBits() - 1);
    1270           6 :     } else if (DstTy.isScalar() && DstTy.getSizeInBits() <= 32) {
    1271           6 :       const unsigned NewOpc = isSigned ? AArch64::SBFMWri : AArch64::UBFMWri;
    1272          18 :       ExtI = BuildMI(MBB, I, I.getDebugLoc(), TII.get(NewOpc))
    1273             :                  .addDef(DefReg)
    1274             :                  .addUse(SrcReg)
    1275             :                  .addImm(0)
    1276           6 :                  .addImm(SrcTy.getSizeInBits() - 1);
    1277             :     } else {
    1278             :       return false;
    1279             :     }
    1280             : 
    1281          11 :     constrainSelectedInstRegOperands(*ExtI, TII, TRI, RBI);
    1282             : 
    1283          11 :     I.eraseFromParent();
    1284          11 :     return true;
    1285             :   }
    1286             : 
    1287           0 :   case TargetOpcode::G_SITOFP:
    1288             :   case TargetOpcode::G_UITOFP:
    1289             :   case TargetOpcode::G_FPTOSI:
    1290             :   case TargetOpcode::G_FPTOUI: {
    1291           0 :     const LLT DstTy = MRI.getType(I.getOperand(0).getReg()),
    1292           0 :               SrcTy = MRI.getType(I.getOperand(1).getReg());
    1293           0 :     const unsigned NewOpc = selectFPConvOpc(Opcode, DstTy, SrcTy);
    1294           0 :     if (NewOpc == Opcode)
    1295             :       return false;
    1296             : 
    1297           0 :     I.setDesc(TII.get(NewOpc));
    1298           0 :     constrainSelectedInstRegOperands(I, TII, TRI, RBI);
    1299             : 
    1300           0 :     return true;
    1301             :   }
    1302             : 
    1303             : 
    1304           1 :   case TargetOpcode::G_INTTOPTR:
    1305             :     // The importer is currently unable to import pointer types since they
    1306             :     // didn't exist in SelectionDAG.
    1307           1 :     return selectCopy(I, TII, MRI, TRI, RBI);
    1308             : 
    1309           3 :   case TargetOpcode::G_BITCAST:
    1310             :     // Imported SelectionDAG rules can handle every bitcast except those that
    1311             :     // bitcast from a type to the same type. Ideally, these shouldn't occur
    1312             :     // but we might not run an optimizer that deletes them.
    1313           6 :     if (MRI.getType(I.getOperand(0).getReg()) ==
    1314           6 :         MRI.getType(I.getOperand(1).getReg()))
    1315           3 :       return selectCopy(I, TII, MRI, TRI, RBI);
    1316             :     return false;
    1317             : 
    1318             :   case TargetOpcode::G_SELECT: {
    1319           9 :     if (MRI.getType(I.getOperand(1).getReg()) != LLT::scalar(1)) {
    1320             :       DEBUG(dbgs() << "G_SELECT cond has type: " << Ty
    1321             :                    << ", expected: " << LLT::scalar(1) << '\n');
    1322             :       return false;
    1323             :     }
    1324             : 
    1325           9 :     const unsigned CondReg = I.getOperand(1).getReg();
    1326           9 :     const unsigned TReg = I.getOperand(2).getReg();
    1327           9 :     const unsigned FReg = I.getOperand(3).getReg();
    1328             : 
    1329             :     unsigned CSelOpc = 0;
    1330             : 
    1331             :     if (Ty == LLT::scalar(32)) {
    1332             :       CSelOpc = AArch64::CSELWr;
    1333             :     } else if (Ty == LLT::scalar(64) || Ty == LLT::pointer(0, 64)) {
    1334             :       CSelOpc = AArch64::CSELXr;
    1335             :     } else {
    1336             :       return false;
    1337             :     }
    1338             : 
    1339             :     MachineInstr &TstMI =
    1340          27 :         *BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::ANDSWri))
    1341             :              .addDef(AArch64::WZR)
    1342             :              .addUse(CondReg)
    1343          18 :              .addImm(AArch64_AM::encodeLogicalImmediate(1, 32));
    1344             : 
    1345          27 :     MachineInstr &CSelMI = *BuildMI(MBB, I, I.getDebugLoc(), TII.get(CSelOpc))
    1346           9 :                                 .addDef(I.getOperand(0).getReg())
    1347             :                                 .addUse(TReg)
    1348             :                                 .addUse(FReg)
    1349           9 :                                 .addImm(AArch64CC::NE);
    1350             : 
    1351           9 :     constrainSelectedInstRegOperands(TstMI, TII, TRI, RBI);
    1352           9 :     constrainSelectedInstRegOperands(CSelMI, TII, TRI, RBI);
    1353             : 
    1354           9 :     I.eraseFromParent();
    1355           9 :     return true;
    1356             :   }
    1357             :   case TargetOpcode::G_ICMP: {
    1358             :     if (Ty != LLT::scalar(32)) {
    1359             :       DEBUG(dbgs() << "G_ICMP result has type: " << Ty
    1360             :                    << ", expected: " << LLT::scalar(32) << '\n');
    1361             :       return false;
    1362             :     }
    1363             : 
    1364             :     unsigned CmpOpc = 0;
    1365             :     unsigned ZReg = 0;
    1366             : 
    1367          14 :     LLT CmpTy = MRI.getType(I.getOperand(2).getReg());
    1368             :     if (CmpTy == LLT::scalar(32)) {
    1369             :       CmpOpc = AArch64::SUBSWrr;
    1370             :       ZReg = AArch64::WZR;
    1371             :     } else if (CmpTy == LLT::scalar(64) || CmpTy.isPointer()) {
    1372             :       CmpOpc = AArch64::SUBSXrr;
    1373             :       ZReg = AArch64::XZR;
    1374             :     } else {
    1375             :       return false;
    1376             :     }
    1377             : 
    1378             :     // CSINC increments the result by one when the condition code is false.
    1379             :     // Therefore, we have to invert the predicate to get an increment by 1 when
    1380             :     // the predicate is true.
    1381             :     const AArch64CC::CondCode invCC =
    1382          14 :         changeICMPPredToAArch64CC(CmpInst::getInversePredicate(
    1383          28 :             (CmpInst::Predicate)I.getOperand(1).getPredicate()));
    1384             : 
    1385          42 :     MachineInstr &CmpMI = *BuildMI(MBB, I, I.getDebugLoc(), TII.get(CmpOpc))
    1386             :                                .addDef(ZReg)
    1387          14 :                                .addUse(I.getOperand(2).getReg())
    1388          28 :                                .addUse(I.getOperand(3).getReg());
    1389             : 
    1390             :     MachineInstr &CSetMI =
    1391          42 :         *BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::CSINCWr))
    1392          14 :              .addDef(I.getOperand(0).getReg())
    1393             :              .addUse(AArch64::WZR)
    1394             :              .addUse(AArch64::WZR)
    1395          28 :              .addImm(invCC);
    1396             : 
    1397          14 :     constrainSelectedInstRegOperands(CmpMI, TII, TRI, RBI);
    1398          14 :     constrainSelectedInstRegOperands(CSetMI, TII, TRI, RBI);
    1399             : 
    1400          14 :     I.eraseFromParent();
    1401          14 :     return true;
    1402             :   }
    1403             : 
    1404             :   case TargetOpcode::G_FCMP: {
    1405             :     if (Ty != LLT::scalar(32)) {
    1406             :       DEBUG(dbgs() << "G_FCMP result has type: " << Ty
    1407             :                    << ", expected: " << LLT::scalar(32) << '\n');
    1408             :       return false;
    1409             :     }
    1410             : 
    1411             :     unsigned CmpOpc = 0;
    1412           6 :     LLT CmpTy = MRI.getType(I.getOperand(2).getReg());
    1413             :     if (CmpTy == LLT::scalar(32)) {
    1414             :       CmpOpc = AArch64::FCMPSrr;
    1415             :     } else if (CmpTy == LLT::scalar(64)) {
    1416             :       CmpOpc = AArch64::FCMPDrr;
    1417             :     } else {
    1418             :       return false;
    1419             :     }
    1420             : 
    1421             :     // FIXME: regbank
    1422             : 
    1423             :     AArch64CC::CondCode CC1, CC2;
    1424           6 :     changeFCMPPredToAArch64CC(
    1425           6 :         (CmpInst::Predicate)I.getOperand(1).getPredicate(), CC1, CC2);
    1426             : 
    1427          18 :     MachineInstr &CmpMI = *BuildMI(MBB, I, I.getDebugLoc(), TII.get(CmpOpc))
    1428           6 :                                .addUse(I.getOperand(2).getReg())
    1429          12 :                                .addUse(I.getOperand(3).getReg());
    1430             : 
    1431           6 :     const unsigned DefReg = I.getOperand(0).getReg();
    1432             :     unsigned Def1Reg = DefReg;
    1433           6 :     if (CC2 != AArch64CC::AL)
    1434           3 :       Def1Reg = MRI.createVirtualRegister(&AArch64::GPR32RegClass);
    1435             : 
    1436             :     MachineInstr &CSetMI =
    1437          18 :         *BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::CSINCWr))
    1438             :              .addDef(Def1Reg)
    1439             :              .addUse(AArch64::WZR)
    1440             :              .addUse(AArch64::WZR)
    1441          18 :              .addImm(getInvertedCondCode(CC1));
    1442             : 
    1443           6 :     if (CC2 != AArch64CC::AL) {
    1444           3 :       unsigned Def2Reg = MRI.createVirtualRegister(&AArch64::GPR32RegClass);
    1445             :       MachineInstr &CSet2MI =
    1446           9 :           *BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::CSINCWr))
    1447             :                .addDef(Def2Reg)
    1448             :                .addUse(AArch64::WZR)
    1449             :                .addUse(AArch64::WZR)
    1450           9 :                .addImm(getInvertedCondCode(CC2));
    1451             :       MachineInstr &OrMI =
    1452           9 :           *BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::ORRWrr))
    1453             :                .addDef(DefReg)
    1454             :                .addUse(Def1Reg)
    1455           3 :                .addUse(Def2Reg);
    1456           3 :       constrainSelectedInstRegOperands(OrMI, TII, TRI, RBI);
    1457           3 :       constrainSelectedInstRegOperands(CSet2MI, TII, TRI, RBI);
    1458             :     }
    1459             : 
    1460           6 :     constrainSelectedInstRegOperands(CmpMI, TII, TRI, RBI);
    1461           6 :     constrainSelectedInstRegOperands(CSetMI, TII, TRI, RBI);
    1462             : 
    1463           6 :     I.eraseFromParent();
    1464           6 :     return true;
    1465             :   }
    1466           1 :   case TargetOpcode::G_VASTART:
    1467           2 :     return STI.isTargetDarwin() ? selectVaStartDarwin(I, MF, MRI)
    1468             :                                 : selectVaStartAAPCS(I, MF, MRI);
    1469           5 :   case TargetOpcode::G_IMPLICIT_DEF:
    1470           5 :     I.setDesc(TII.get(TargetOpcode::IMPLICIT_DEF));
    1471           5 :     const LLT DstTy = MRI.getType(I.getOperand(0).getReg());
    1472           5 :     const unsigned DstReg = I.getOperand(0).getReg();
    1473           5 :     const RegisterBank &DstRB = *RBI.getRegBank(DstReg, MRI, TRI);
    1474             :     const TargetRegisterClass *DstRC =
    1475           5 :         getRegClassForTypeOnBank(DstTy, DstRB, RBI);
    1476           5 :     RBI.constrainGenericRegister(DstReg, *DstRC, MRI);
    1477           5 :     return true;
    1478             :   }
    1479             : 
    1480             :   return false;
    1481             : }
    1482             : 
    1483             : /// SelectArithImmed - Select an immediate value that can be represented as
    1484             : /// a 12-bit value shifted left by either 0 or 12.  If so, return true with
    1485             : /// Val set to the 12-bit value and Shift set to the shifter operand.
    1486             : InstructionSelector::ComplexRendererFns
    1487          34 : AArch64InstructionSelector::selectArithImmed(MachineOperand &Root) const {
    1488          34 :   MachineInstr &MI = *Root.getParent();
    1489          34 :   MachineBasicBlock &MBB = *MI.getParent();
    1490          34 :   MachineFunction &MF = *MBB.getParent();
    1491          34 :   MachineRegisterInfo &MRI = MF.getRegInfo();
    1492             : 
    1493             :   // This function is called from the addsub_shifted_imm ComplexPattern,
    1494             :   // which lists [imm] as the list of opcode it's interested in, however
    1495             :   // we still need to check whether the operand is actually an immediate
    1496             :   // here because the ComplexPattern opcode list is only used in
    1497             :   // root-level opcode matching.
    1498             :   uint64_t Immed;
    1499          34 :   if (Root.isImm())
    1500           0 :     Immed = Root.getImm();
    1501          34 :   else if (Root.isCImm())
    1502           0 :     Immed = Root.getCImm()->getZExtValue();
    1503          34 :   else if (Root.isReg()) {
    1504          34 :     MachineInstr *Def = MRI.getVRegDef(Root.getReg());
    1505          68 :     if (Def->getOpcode() != TargetOpcode::G_CONSTANT)
    1506             :       return None;
    1507          10 :     MachineOperand &Op1 = Def->getOperand(1);
    1508          20 :     if (!Op1.isCImm() || Op1.getCImm()->getBitWidth() > 64)
    1509             :       return None;
    1510             :     Immed = Op1.getCImm()->getZExtValue();
    1511             :   } else
    1512             :     return None;
    1513             : 
    1514             :   unsigned ShiftAmt;
    1515             : 
    1516          10 :   if (Immed >> 12 == 0) {
    1517             :     ShiftAmt = 0;
    1518           0 :   } else if ((Immed & 0xfff) == 0 && Immed >> 24 == 0) {
    1519             :     ShiftAmt = 12;
    1520             :     Immed = Immed >> 12;
    1521             :   } else
    1522             :     return None;
    1523             : 
    1524             :   unsigned ShVal = AArch64_AM::getShifterImm(AArch64_AM::LSL, ShiftAmt);
    1525             :   return {{
    1526          10 :       [=](MachineInstrBuilder &MIB) { MIB.addImm(Immed); },
    1527          10 :       [=](MachineInstrBuilder &MIB) { MIB.addImm(ShVal); },
    1528          20 :   }};
    1529             : }
    1530             : 
    1531             : /// Select a "register plus unscaled signed 9-bit immediate" address.  This
    1532             : /// should only match when there is an offset that is not valid for a scaled
    1533             : /// immediate addressing mode.  The "Size" argument is the size in bytes of the
    1534             : /// memory reference, which is needed here to know what is valid for a scaled
    1535             : /// immediate.
    1536             : InstructionSelector::ComplexRendererFns
    1537          68 : AArch64InstructionSelector::selectAddrModeUnscaled(MachineOperand &Root,
    1538             :                                                    unsigned Size) const {
    1539             :   MachineRegisterInfo &MRI =
    1540          68 :       Root.getParent()->getParent()->getParent()->getRegInfo();
    1541             : 
    1542          68 :   if (!Root.isReg())
    1543             :     return None;
    1544             : 
    1545          68 :   if (!isBaseWithConstantOffset(Root, MRI))
    1546             :     return None;
    1547             : 
    1548           3 :   MachineInstr *RootDef = MRI.getVRegDef(Root.getReg());
    1549           3 :   if (!RootDef)
    1550             :     return None;
    1551             : 
    1552           3 :   MachineOperand &OffImm = RootDef->getOperand(2);
    1553           3 :   if (!OffImm.isReg())
    1554             :     return None;
    1555           3 :   MachineInstr *RHS = MRI.getVRegDef(OffImm.getReg());
    1556           6 :   if (!RHS || RHS->getOpcode() != TargetOpcode::G_CONSTANT)
    1557             :     return None;
    1558             :   int64_t RHSC;
    1559           3 :   MachineOperand &RHSOp1 = RHS->getOperand(1);
    1560           6 :   if (!RHSOp1.isCImm() || RHSOp1.getCImm()->getBitWidth() > 64)
    1561             :     return None;
    1562             :   RHSC = RHSOp1.getCImm()->getSExtValue();
    1563             : 
    1564             :   // If the offset is valid as a scaled immediate, don't match here.
    1565           9 :   if ((RHSC & (Size - 1)) == 0 && RHSC >= 0 && RHSC < (0x1000 << Log2_32(Size)))
    1566             :     return None;
    1567           3 :   if (RHSC >= -256 && RHSC < 256) {
    1568           0 :     MachineOperand &Base = RootDef->getOperand(1);
    1569             :     return {{
    1570           0 :         [=](MachineInstrBuilder &MIB) { MIB.add(Base); },
    1571           0 :         [=](MachineInstrBuilder &MIB) { MIB.addImm(RHSC); },
    1572           0 :     }};
    1573             :   }
    1574             :   return None;
    1575             : }
    1576             : 
    1577             : /// Select a "register plus scaled unsigned 12-bit immediate" address.  The
    1578             : /// "Size" argument is the size in bytes of the memory reference, which
    1579             : /// determines the scale.
    1580             : InstructionSelector::ComplexRendererFns
    1581         101 : AArch64InstructionSelector::selectAddrModeIndexed(MachineOperand &Root,
    1582             :                                                   unsigned Size) const {
    1583             :   MachineRegisterInfo &MRI =
    1584         101 :       Root.getParent()->getParent()->getParent()->getRegInfo();
    1585             : 
    1586         101 :   if (!Root.isReg())
    1587             :     return None;
    1588             : 
    1589         101 :   MachineInstr *RootDef = MRI.getVRegDef(Root.getReg());
    1590         101 :   if (!RootDef)
    1591             :     return None;
    1592             : 
    1593         202 :   if (RootDef->getOpcode() == TargetOpcode::G_FRAME_INDEX) {
    1594             :     return {{
    1595          22 :         [=](MachineInstrBuilder &MIB) { MIB.add(RootDef->getOperand(1)); },
    1596             :         [=](MachineInstrBuilder &MIB) { MIB.addImm(0); },
    1597          44 :     }};
    1598             :   }
    1599             : 
    1600          79 :   if (isBaseWithConstantOffset(Root, MRI)) {
    1601          14 :     MachineOperand &LHS = RootDef->getOperand(1);
    1602             :     MachineOperand &RHS = RootDef->getOperand(2);
    1603          14 :     MachineInstr *LHSDef = MRI.getVRegDef(LHS.getReg());
    1604          14 :     MachineInstr *RHSDef = MRI.getVRegDef(RHS.getReg());
    1605          14 :     if (LHSDef && RHSDef) {
    1606          28 :       int64_t RHSC = (int64_t)RHSDef->getOperand(1).getCImm()->getZExtValue();
    1607             :       unsigned Scale = Log2_32(Size);
    1608          14 :       if ((RHSC & (Size - 1)) == 0 && RHSC >= 0 && RHSC < (0x1000 << Scale)) {
    1609          22 :         if (LHSDef->getOpcode() == TargetOpcode::G_FRAME_INDEX)
    1610             :           return {{
    1611           0 :               [=](MachineInstrBuilder &MIB) { MIB.add(LHSDef->getOperand(1)); },
    1612           0 :               [=](MachineInstrBuilder &MIB) { MIB.addImm(RHSC >> Scale); },
    1613           0 :           }};
    1614             : 
    1615             :         return {{
    1616          11 :             [=](MachineInstrBuilder &MIB) { MIB.add(LHS); },
    1617          11 :             [=](MachineInstrBuilder &MIB) { MIB.addImm(RHSC >> Scale); },
    1618          33 :         }};
    1619             :       }
    1620             :     }
    1621             :   }
    1622             : 
    1623             :   // Before falling back to our general case, check if the unscaled
    1624             :   // instructions can handle this. If so, that's preferable.
    1625         136 :   if (selectAddrModeUnscaled(Root, Size).hasValue())
    1626             :     return None;
    1627             : 
    1628             :   return {{
    1629          68 :       [=](MachineInstrBuilder &MIB) { MIB.add(Root); },
    1630             :       [=](MachineInstrBuilder &MIB) { MIB.addImm(0); },
    1631         204 :   }};
    1632             : }
    1633             : 
    1634           1 : void AArch64InstructionSelector::renderTruncImm(MachineInstrBuilder &MIB,
    1635             :                                                 const MachineInstr &MI) const {
    1636           1 :   const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
    1637             :   assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && "Expected G_CONSTANT");
    1638           1 :   Optional<int64_t> CstVal = getConstantVRegVal(MI.getOperand(0).getReg(), MRI);
    1639             :   assert(CstVal && "Expected constant value");
    1640           1 :   MIB.addImm(CstVal.getValue());
    1641           1 : }
    1642             : 
    1643             : namespace llvm {
    1644             : InstructionSelector *
    1645        1322 : createAArch64InstructionSelector(const AArch64TargetMachine &TM,
    1646             :                                  AArch64Subtarget &Subtarget,
    1647             :                                  AArch64RegisterBankInfo &RBI) {
    1648        1322 :   return new AArch64InstructionSelector(TM, Subtarget, RBI);
    1649             : }
    1650      291582 : }

Generated by: LCOV version 1.13