LCOV - code coverage report
Current view: top level - lib/Target/ARM - ARMISelDAGToDAG.cpp (source / functions) Hit Total Coverage
Test: llvm-toolchain.info Lines: 1875 2086 89.9 %
Date: 2017-09-14 15:23:50 Functions: 68 70 97.1 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : //===-- ARMISelDAGToDAG.cpp - A dag to dag inst selector for ARM ----------===//
       2             : //
       3             : //                     The LLVM Compiler Infrastructure
       4             : //
       5             : // This file is distributed under the University of Illinois Open Source
       6             : // License. See LICENSE.TXT for details.
       7             : //
       8             : //===----------------------------------------------------------------------===//
       9             : //
      10             : // This file defines an instruction selector for the ARM target.
      11             : //
      12             : //===----------------------------------------------------------------------===//
      13             : 
      14             : #include "ARM.h"
      15             : #include "Utils/ARMBaseInfo.h"
      16             : #include "ARMBaseInstrInfo.h"
      17             : #include "ARMTargetMachine.h"
      18             : #include "MCTargetDesc/ARMAddressingModes.h"
      19             : #include "llvm/ADT/StringSwitch.h"
      20             : #include "llvm/CodeGen/MachineFrameInfo.h"
      21             : #include "llvm/CodeGen/MachineFunction.h"
      22             : #include "llvm/CodeGen/MachineInstrBuilder.h"
      23             : #include "llvm/CodeGen/MachineRegisterInfo.h"
      24             : #include "llvm/CodeGen/SelectionDAG.h"
      25             : #include "llvm/CodeGen/SelectionDAGISel.h"
      26             : #include "llvm/IR/CallingConv.h"
      27             : #include "llvm/IR/Constants.h"
      28             : #include "llvm/IR/DerivedTypes.h"
      29             : #include "llvm/IR/Function.h"
      30             : #include "llvm/IR/Intrinsics.h"
      31             : #include "llvm/IR/LLVMContext.h"
      32             : #include "llvm/Support/CommandLine.h"
      33             : #include "llvm/Support/Debug.h"
      34             : #include "llvm/Support/ErrorHandling.h"
      35             : #include "llvm/Target/TargetLowering.h"
      36             : #include "llvm/Target/TargetOptions.h"
      37             : 
      38             : using namespace llvm;
      39             : 
      40             : #define DEBUG_TYPE "arm-isel"
      41             : 
      42             : static cl::opt<bool>
      43       72306 : DisableShifterOp("disable-shifter-op", cl::Hidden,
      44      216918 :   cl::desc("Disable isel of shifter-op"),
      45      289224 :   cl::init(false));
      46             : 
      47             : //===--------------------------------------------------------------------===//
      48             : /// ARMDAGToDAGISel - ARM specific code to select ARM machine
      49             : /// instructions for SelectionDAG operations.
      50             : ///
      51             : namespace {
      52             : 
      53             : enum AddrMode2Type {
      54             :   AM2_BASE, // Simple AM2 (+-imm12)
      55             :   AM2_SHOP  // Shifter-op AM2
      56             : };
      57             : 
      58        2524 : class ARMDAGToDAGISel : public SelectionDAGISel {
      59             :   /// Subtarget - Keep a pointer to the ARMSubtarget around so that we can
      60             :   /// make the right decision when generating code for different targets.
      61             :   const ARMSubtarget *Subtarget;
      62             : 
      63             : public:
      64             :   explicit ARMDAGToDAGISel(ARMBaseTargetMachine &tm, CodeGenOpt::Level OptLevel)
      65        2551 :       : SelectionDAGISel(tm, OptLevel) {}
      66             : 
      67       11835 :   bool runOnMachineFunction(MachineFunction &MF) override {
      68             :     // Reset the subtarget each time through.
      69       11835 :     Subtarget = &MF.getSubtarget<ARMSubtarget>();
      70       11835 :     SelectionDAGISel::runOnMachineFunction(MF);
      71       11808 :     return true;
      72             :   }
      73             : 
      74           0 :   StringRef getPassName() const override { return "ARM Instruction Selection"; }
      75             : 
      76             :   void PreprocessISelDAG() override;
      77             : 
      78             :   /// getI32Imm - Return a target constant of type i32 with the specified
      79             :   /// value.
      80             :   inline SDValue getI32Imm(unsigned Imm, const SDLoc &dl) {
      81         297 :     return CurDAG->getTargetConstant(Imm, dl, MVT::i32);
      82             :   }
      83             : 
      84             :   void Select(SDNode *N) override;
      85             : 
      86             :   bool hasNoVMLxHazardUse(SDNode *N) const;
      87             :   bool isShifterOpProfitable(const SDValue &Shift,
      88             :                              ARM_AM::ShiftOpc ShOpcVal, unsigned ShAmt);
      89             :   bool SelectRegShifterOperand(SDValue N, SDValue &A,
      90             :                                SDValue &B, SDValue &C,
      91             :                                bool CheckProfitability = true);
      92             :   bool SelectImmShifterOperand(SDValue N, SDValue &A,
      93             :                                SDValue &B, bool CheckProfitability = true);
      94             :   bool SelectShiftRegShifterOperand(SDValue N, SDValue &A,
      95             :                                     SDValue &B, SDValue &C) {
      96             :     // Don't apply the profitability check
      97         330 :     return SelectRegShifterOperand(N, A, B, C, false);
      98             :   }
      99             :   bool SelectShiftImmShifterOperand(SDValue N, SDValue &A,
     100             :                                     SDValue &B) {
     101             :     // Don't apply the profitability check
     102        5324 :     return SelectImmShifterOperand(N, A, B, false);
     103             :   }
     104             : 
     105             :   bool SelectAddrModeImm12(SDValue N, SDValue &Base, SDValue &OffImm);
     106             :   bool SelectLdStSOReg(SDValue N, SDValue &Base, SDValue &Offset, SDValue &Opc);
     107             : 
     108             :   AddrMode2Type SelectAddrMode2Worker(SDValue N, SDValue &Base,
     109             :                                       SDValue &Offset, SDValue &Opc);
     110             :   bool SelectAddrMode2Base(SDValue N, SDValue &Base, SDValue &Offset,
     111             :                            SDValue &Opc) {
     112             :     return SelectAddrMode2Worker(N, Base, Offset, Opc) == AM2_BASE;
     113             :   }
     114             : 
     115             :   bool SelectAddrMode2ShOp(SDValue N, SDValue &Base, SDValue &Offset,
     116             :                            SDValue &Opc) {
     117             :     return SelectAddrMode2Worker(N, Base, Offset, Opc) == AM2_SHOP;
     118             :   }
     119             : 
     120             :   bool SelectAddrMode2(SDValue N, SDValue &Base, SDValue &Offset,
     121             :                        SDValue &Opc) {
     122           2 :     SelectAddrMode2Worker(N, Base, Offset, Opc);
     123             : //    return SelectAddrMode2ShOp(N, Base, Offset, Opc);
     124             :     // This always matches one way or another.
     125             :     return true;
     126             :   }
     127             : 
     128        1111 :   bool SelectCMOVPred(SDValue N, SDValue &Pred, SDValue &Reg) {
     129        1111 :     const ConstantSDNode *CN = cast<ConstantSDNode>(N);
     130        6666 :     Pred = CurDAG->getTargetConstant(CN->getZExtValue(), SDLoc(N), MVT::i32);
     131        2222 :     Reg = CurDAG->getRegister(ARM::CPSR, MVT::i32);
     132        1111 :     return true;
     133             :   }
     134             : 
     135             :   bool SelectAddrMode2OffsetReg(SDNode *Op, SDValue N,
     136             :                              SDValue &Offset, SDValue &Opc);
     137             :   bool SelectAddrMode2OffsetImm(SDNode *Op, SDValue N,
     138             :                              SDValue &Offset, SDValue &Opc);
     139             :   bool SelectAddrMode2OffsetImmPre(SDNode *Op, SDValue N,
     140             :                              SDValue &Offset, SDValue &Opc);
     141             :   bool SelectAddrOffsetNone(SDValue N, SDValue &Base);
     142             :   bool SelectAddrMode3(SDValue N, SDValue &Base,
     143             :                        SDValue &Offset, SDValue &Opc);
     144             :   bool SelectAddrMode3Offset(SDNode *Op, SDValue N,
     145             :                              SDValue &Offset, SDValue &Opc);
     146             :   bool SelectAddrMode5(SDValue N, SDValue &Base,
     147             :                        SDValue &Offset);
     148             :   bool SelectAddrMode6(SDNode *Parent, SDValue N, SDValue &Addr,SDValue &Align);
     149             :   bool SelectAddrMode6Offset(SDNode *Op, SDValue N, SDValue &Offset);
     150             : 
     151             :   bool SelectAddrModePC(SDValue N, SDValue &Offset, SDValue &Label);
     152             : 
     153             :   // Thumb Addressing Modes:
     154             :   bool SelectThumbAddrModeRR(SDValue N, SDValue &Base, SDValue &Offset);
     155             :   bool SelectThumbAddrModeImm5S(SDValue N, unsigned Scale, SDValue &Base,
     156             :                                 SDValue &OffImm);
     157             :   bool SelectThumbAddrModeImm5S1(SDValue N, SDValue &Base,
     158             :                                  SDValue &OffImm);
     159             :   bool SelectThumbAddrModeImm5S2(SDValue N, SDValue &Base,
     160             :                                  SDValue &OffImm);
     161             :   bool SelectThumbAddrModeImm5S4(SDValue N, SDValue &Base,
     162             :                                  SDValue &OffImm);
     163             :   bool SelectThumbAddrModeSP(SDValue N, SDValue &Base, SDValue &OffImm);
     164             : 
     165             :   // Thumb 2 Addressing Modes:
     166             :   bool SelectT2AddrModeImm12(SDValue N, SDValue &Base, SDValue &OffImm);
     167             :   bool SelectT2AddrModeImm8(SDValue N, SDValue &Base,
     168             :                             SDValue &OffImm);
     169             :   bool SelectT2AddrModeImm8Offset(SDNode *Op, SDValue N,
     170             :                                  SDValue &OffImm);
     171             :   bool SelectT2AddrModeSoReg(SDValue N, SDValue &Base,
     172             :                              SDValue &OffReg, SDValue &ShImm);
     173             :   bool SelectT2AddrModeExclusive(SDValue N, SDValue &Base, SDValue &OffImm);
     174             : 
     175             :   inline bool is_so_imm(unsigned Imm) const {
     176             :     return ARM_AM::getSOImmVal(Imm) != -1;
     177             :   }
     178             : 
     179             :   inline bool is_so_imm_not(unsigned Imm) const {
     180             :     return ARM_AM::getSOImmVal(~Imm) != -1;
     181             :   }
     182             : 
     183             :   inline bool is_t2_so_imm(unsigned Imm) const {
     184         210 :     return ARM_AM::getT2SOImmVal(Imm) != -1;
     185             :   }
     186             : 
     187             :   inline bool is_t2_so_imm_not(unsigned Imm) const {
     188          78 :     return ARM_AM::getT2SOImmVal(~Imm) != -1;
     189             :   }
     190             : 
     191             :   // Include the pieces autogenerated from the target description.
     192             : #include "ARMGenDAGISel.inc"
     193             : 
     194             : private:
     195             :   void transferMemOperands(SDNode *Src, SDNode *Dst);
     196             : 
     197             :   /// Indexed (pre/post inc/dec) load matching code for ARM.
     198             :   bool tryARMIndexedLoad(SDNode *N);
     199             :   bool tryT1IndexedLoad(SDNode *N);
     200             :   bool tryT2IndexedLoad(SDNode *N);
     201             : 
     202             :   /// SelectVLD - Select NEON load intrinsics.  NumVecs should be
     203             :   /// 1, 2, 3 or 4.  The opcode arrays specify the instructions used for
     204             :   /// loads of D registers and even subregs and odd subregs of Q registers.
     205             :   /// For NumVecs <= 2, QOpcodes1 is not used.
     206             :   void SelectVLD(SDNode *N, bool isUpdating, unsigned NumVecs,
     207             :                  const uint16_t *DOpcodes, const uint16_t *QOpcodes0,
     208             :                  const uint16_t *QOpcodes1);
     209             : 
     210             :   /// SelectVST - Select NEON store intrinsics.  NumVecs should
     211             :   /// be 1, 2, 3 or 4.  The opcode arrays specify the instructions used for
     212             :   /// stores of D registers and even subregs and odd subregs of Q registers.
     213             :   /// For NumVecs <= 2, QOpcodes1 is not used.
     214             :   void SelectVST(SDNode *N, bool isUpdating, unsigned NumVecs,
     215             :                  const uint16_t *DOpcodes, const uint16_t *QOpcodes0,
     216             :                  const uint16_t *QOpcodes1);
     217             : 
     218             :   /// SelectVLDSTLane - Select NEON load/store lane intrinsics.  NumVecs should
     219             :   /// be 2, 3 or 4.  The opcode arrays specify the instructions used for
     220             :   /// load/store of D registers and Q registers.
     221             :   void SelectVLDSTLane(SDNode *N, bool IsLoad, bool isUpdating,
     222             :                        unsigned NumVecs, const uint16_t *DOpcodes,
     223             :                        const uint16_t *QOpcodes);
     224             : 
     225             :   /// SelectVLDDup - Select NEON load-duplicate intrinsics.  NumVecs
     226             :   /// should be 1, 2, 3 or 4.  The opcode array specifies the instructions used
     227             :   /// for loading D registers.  (Q registers are not supported.)
     228             :   void SelectVLDDup(SDNode *N, bool isUpdating, unsigned NumVecs,
     229             :                     const uint16_t *DOpcodes,
     230             :                     const uint16_t *QOpcodes = nullptr);
     231             : 
     232             :   /// Try to select SBFX/UBFX instructions for ARM.
     233             :   bool tryV6T2BitfieldExtractOp(SDNode *N, bool isSigned);
     234             : 
     235             :   // Select special operations if node forms integer ABS pattern
     236             :   bool tryABSOp(SDNode *N);
     237             : 
     238             :   bool tryReadRegister(SDNode *N);
     239             :   bool tryWriteRegister(SDNode *N);
     240             : 
     241             :   bool tryInlineAsm(SDNode *N);
     242             : 
     243             :   void SelectCMPZ(SDNode *N, bool &SwitchEQNEToPLMI);
     244             : 
     245             :   void SelectCMP_SWAP(SDNode *N);
     246             : 
     247             :   /// SelectInlineAsmMemoryOperand - Implement addressing mode selection for
     248             :   /// inline asm expressions.
     249             :   bool SelectInlineAsmMemoryOperand(const SDValue &Op, unsigned ConstraintID,
     250             :                                     std::vector<SDValue> &OutOps) override;
     251             : 
     252             :   // Form pairs of consecutive R, S, D, or Q registers.
     253             :   SDNode *createGPRPairNode(EVT VT, SDValue V0, SDValue V1);
     254             :   SDNode *createSRegPairNode(EVT VT, SDValue V0, SDValue V1);
     255             :   SDNode *createDRegPairNode(EVT VT, SDValue V0, SDValue V1);
     256             :   SDNode *createQRegPairNode(EVT VT, SDValue V0, SDValue V1);
     257             : 
     258             :   // Form sequences of 4 consecutive S, D, or Q registers.
     259             :   SDNode *createQuadSRegsNode(EVT VT, SDValue V0, SDValue V1, SDValue V2, SDValue V3);
     260             :   SDNode *createQuadDRegsNode(EVT VT, SDValue V0, SDValue V1, SDValue V2, SDValue V3);
     261             :   SDNode *createQuadQRegsNode(EVT VT, SDValue V0, SDValue V1, SDValue V2, SDValue V3);
     262             : 
     263             :   // Get the alignment operand for a NEON VLD or VST instruction.
     264             :   SDValue GetVLDSTAlign(SDValue Align, const SDLoc &dl, unsigned NumVecs,
     265             :                         bool is64BitVector);
     266             : 
     267             :   /// Returns the number of instructions required to materialize the given
     268             :   /// constant in a register, or 3 if a literal pool load is needed.
     269             :   unsigned ConstantMaterializationCost(unsigned Val) const;
     270             : 
     271             :   /// Checks if N is a multiplication by a constant where we can extract out a
     272             :   /// power of two from the constant so that it can be used in a shift, but only
     273             :   /// if it simplifies the materialization of the constant. Returns true if it
     274             :   /// is, and assigns to PowerOfTwo the power of two that should be extracted
     275             :   /// out and to NewMulConst the new constant to be multiplied by.
     276             :   bool canExtractShiftFromMul(const SDValue &N, unsigned MaxShift,
     277             :                               unsigned &PowerOfTwo, SDValue &NewMulConst) const;
     278             : 
     279             :   /// Replace N with M in CurDAG, in a way that also ensures that M gets
     280             :   /// selected when N would have been selected.
     281             :   void replaceDAGValue(const SDValue &N, SDValue M);
     282             : };
     283             : }
     284             : 
     285             : /// isInt32Immediate - This method tests to see if the node is a 32-bit constant
     286             : /// operand. If so Imm will receive the 32-bit value.
     287             : static bool isInt32Immediate(SDNode *N, unsigned &Imm) {
     288        1574 :   if (N->getOpcode() == ISD::Constant && N->getValueType(0) == MVT::i32) {
     289         970 :     Imm = cast<ConstantSDNode>(N)->getZExtValue();
     290             :     return true;
     291             :   }
     292             :   return false;
     293             : }
     294             : 
     295             : // isInt32Immediate - This method tests to see if a constant operand.
     296             : // If so Imm will receive the 32 bit value.
     297             : static bool isInt32Immediate(SDValue N, unsigned &Imm) {
     298          42 :   return isInt32Immediate(N.getNode(), Imm);
     299             : }
     300             : 
     301             : // isOpcWithIntImmediate - This method tests to see if the node is a specific
     302             : // opcode and that it has a immediate integer right operand.
     303             : // If so Imm will receive the 32 bit value.
     304             : static bool isOpcWithIntImmediate(SDNode *N, unsigned Opc, unsigned& Imm) {
     305       12246 :   return N->getOpcode() == Opc &&
     306        1630 :          isInt32Immediate(N->getOperand(1).getNode(), Imm);
     307             : }
     308             : 
     309             : /// \brief Check whether a particular node is a constant value representable as
     310             : /// (N * Scale) where (N in [\p RangeMin, \p RangeMax).
     311             : ///
     312             : /// \param ScaledConstant [out] - On success, the pre-scaled constant value.
     313             : static bool isScaledConstantInRange(SDValue Node, int Scale,
     314             :                                     int RangeMin, int RangeMax,
     315             :                                     int &ScaledConstant) {
     316             :   assert(Scale > 0 && "Invalid scale!");
     317             : 
     318             :   // Check that this is a constant.
     319        4155 :   const ConstantSDNode *C = dyn_cast<ConstantSDNode>(Node);
     320             :   if (!C)
     321             :     return false;
     322             : 
     323        4155 :   ScaledConstant = (int) C->getZExtValue();
     324        1009 :   if ((ScaledConstant % Scale) != 0)
     325             :     return false;
     326             : 
     327        4151 :   ScaledConstant /= Scale;
     328        4151 :   return ScaledConstant >= RangeMin && ScaledConstant < RangeMax;
     329             : }
     330             : 
     331       15703 : void ARMDAGToDAGISel::PreprocessISelDAG() {
     332       15703 :   if (!Subtarget->hasV6T2Ops())
     333             :     return;
     334             : 
     335       10211 :   bool isThumb2 = Subtarget->isThumb();
     336       20422 :   for (SelectionDAG::allnodes_iterator I = CurDAG->allnodes_begin(),
     337       20422 :        E = CurDAG->allnodes_end(); I != E; ) {
     338      567108 :     SDNode *N = &*I++; // Preincrement iterator to avoid invalidation issues.
     339             : 
     340      189036 :     if (N->getOpcode() != ISD::ADD)
     341      372644 :       continue;
     342             : 
     343             :     // Look for (add X1, (and (srl X2, c1), c2)) where c2 is constant with
     344             :     // leading zeros, followed by consecutive set bits, followed by 1 or 2
     345             :     // trailing zeros, e.g. 1020.
     346             :     // Transform the expression to
     347             :     // (add X1, (shl (and (srl X2, c1), (c2>>tz)), tz)) where tz is the number
     348             :     // of trailing zeros of c2. The left shift would be folded as an shifter
     349             :     // operand of 'add' and the 'and' and 'srl' would become a bits extraction
     350             :     // node (UBFX).
     351             : 
     352       10850 :     SDValue N0 = N->getOperand(0);
     353       10850 :     SDValue N1 = N->getOperand(1);
     354        5425 :     unsigned And_imm = 0;
     355       10850 :     if (!isOpcWithIntImmediate(N1.getNode(), ISD::AND, And_imm)) {
     356        5376 :       if (isOpcWithIntImmediate(N0.getNode(), ISD::AND, And_imm))
     357             :         std::swap(N0, N1);
     358             :     }
     359        5425 :     if (!And_imm)
     360        5350 :       continue;
     361             : 
     362             :     // Check if the AND mask is an immediate of the form: 000.....1111111100
     363         150 :     unsigned TZ = countTrailingZeros(And_imm);
     364          75 :     if (TZ != 1 && TZ != 2)
     365             :       // Be conservative here. Shifter operands aren't always free. e.g. On
     366             :       // Swift, left shifter operand of 1 / 2 for free but others are not.
     367             :       // e.g.
     368             :       //  ubfx   r3, r1, #16, #8
     369             :       //  ldr.w  r3, [r0, r3, lsl #2]
     370             :       // vs.
     371             :       //  mov.w  r9, #1020
     372             :       //  and.w  r2, r9, r1, lsr #14
     373             :       //  ldr    r2, [r0, r2]
     374          69 :       continue;
     375           6 :     And_imm >>= TZ;
     376           6 :     if (And_imm & (And_imm + 1))
     377           0 :       continue;
     378             : 
     379             :     // Look for (and (srl X, c1), c2).
     380          12 :     SDValue Srl = N1.getOperand(0);
     381           6 :     unsigned Srl_imm = 0;
     382          15 :     if (!isOpcWithIntImmediate(Srl.getNode(), ISD::SRL, Srl_imm) ||
     383             :         (Srl_imm <= 2))
     384           3 :       continue;
     385             : 
     386             :     // Make sure first operand is not a shifter operand which would prevent
     387             :     // folding of the left shift.
     388           3 :     SDValue CPTmp0;
     389           3 :     SDValue CPTmp1;
     390           3 :     SDValue CPTmp2;
     391           3 :     if (isThumb2) {
     392           0 :       if (SelectImmShifterOperand(N0, CPTmp0, CPTmp1))
     393           0 :         continue;
     394             :     } else {
     395           6 :       if (SelectImmShifterOperand(N0, CPTmp0, CPTmp1) ||
     396           3 :           SelectRegShifterOperand(N0, CPTmp0, CPTmp1, CPTmp2))
     397           0 :         continue;
     398             :     }
     399             : 
     400             :     // Now make the transformation.
     401           9 :     Srl = CurDAG->getNode(ISD::SRL, SDLoc(Srl), MVT::i32,
     402           6 :                           Srl.getOperand(0),
     403           9 :                           CurDAG->getConstant(Srl_imm + TZ, SDLoc(Srl),
     404          18 :                                               MVT::i32));
     405           9 :     N1 = CurDAG->getNode(ISD::AND, SDLoc(N1), MVT::i32,
     406             :                          Srl,
     407          18 :                          CurDAG->getConstant(And_imm, SDLoc(Srl), MVT::i32));
     408           9 :     N1 = CurDAG->getNode(ISD::SHL, SDLoc(N1), MVT::i32,
     409          18 :                          N1, CurDAG->getConstant(TZ, SDLoc(Srl), MVT::i32));
     410           3 :     CurDAG->UpdateNodeOperands(N, N0, N1);
     411             :   }
     412             : }
     413             : 
     414             : /// hasNoVMLxHazardUse - Return true if it's desirable to select a FP MLA / MLS
     415             : /// node. VFP / NEON fp VMLA / VMLS instructions have special RAW hazards (at
     416             : /// least on current ARM implementations) which should be avoidded.
     417         348 : bool ARMDAGToDAGISel::hasNoVMLxHazardUse(SDNode *N) const {
     418         348 :   if (OptLevel == CodeGenOpt::None)
     419             :     return true;
     420             : 
     421         348 :   if (!Subtarget->hasVMLxHazards())
     422             :     return true;
     423             : 
     424         166 :   if (!N->hasOneUse())
     425             :     return false;
     426             : 
     427         498 :   SDNode *Use = *N->use_begin();
     428         166 :   if (Use->getOpcode() == ISD::CopyToReg)
     429             :     return true;
     430         158 :   if (Use->isMachineOpcode()) {
     431             :     const ARMBaseInstrInfo *TII = static_cast<const ARMBaseInstrInfo *>(
     432         316 :         CurDAG->getSubtarget().getInstrInfo());
     433             : 
     434         474 :     const MCInstrDesc &MCID = TII->get(Use->getMachineOpcode());
     435         158 :     if (MCID.mayStore())
     436             :       return true;
     437         139 :     unsigned Opcode = MCID.getOpcode();
     438         139 :     if (Opcode == ARM::VMOVRS || Opcode == ARM::VMOVRRD)
     439             :       return true;
     440             :     // vmlx feeding into another vmlx. We actually want to unfold
     441             :     // the use later in the MLxExpansion pass. e.g.
     442             :     // vmla
     443             :     // vmla (stall 8 cycles)
     444             :     //
     445             :     // vmul (5 cycles)
     446             :     // vadd (5 cycles)
     447             :     // vmla
     448             :     // This adds up to about 18 - 19 cycles.
     449             :     //
     450             :     // vmla
     451             :     // vmul (stall 4 cycles)
     452             :     // vadd adds up to about 14 cycles.
     453         103 :     return TII->isFpMLxInstruction(Opcode);
     454             :   }
     455             : 
     456             :   return false;
     457             : }
     458             : 
     459         131 : bool ARMDAGToDAGISel::isShifterOpProfitable(const SDValue &Shift,
     460             :                                             ARM_AM::ShiftOpc ShOpcVal,
     461             :                                             unsigned ShAmt) {
     462         247 :   if (!Subtarget->isLikeA9() && !Subtarget->isSwift())
     463             :     return true;
     464          32 :   if (Shift.hasOneUse())
     465             :     return true;
     466             :   // R << 2 is free.
     467           3 :   return ShOpcVal == ARM_AM::lsl &&
     468           2 :          (ShAmt == 2 || (Subtarget->isSwift() && ShAmt == 1));
     469             : }
     470             : 
     471        5160 : unsigned ARMDAGToDAGISel::ConstantMaterializationCost(unsigned Val) const {
     472        5160 :   if (Subtarget->isThumb()) {
     473        2777 :     if (Val <= 255) return 1;                               // MOV
     474         983 :     if (Subtarget->hasV6T2Ops() &&
     475         513 :         (Val <= 0xffff || ARM_AM::getT2SOImmValSplatVal(Val) != -1))
     476             :       return 1; // MOVW
     477         697 :     if (Val <= 510) return 2;                               // MOV + ADDi8
     478         657 :     if (~Val <= 255) return 2;                              // MOV + MVN
     479         609 :     if (ARM_AM::isThumbImmShiftedVal(Val)) return 2;        // MOV + LSL
     480             :   } else {
     481        2383 :     if (ARM_AM::getSOImmVal(Val) != -1) return 1;           // MOV
     482         528 :     if (ARM_AM::getSOImmVal(~Val) != -1) return 1;          // MVN
     483         432 :     if (Subtarget->hasV6T2Ops() && Val <= 0xffff) return 1; // MOVW
     484         368 :     if (ARM_AM::isSOImmTwoPartVal(Val)) return 2;           // two instrs
     485             :   }
     486         802 :   if (Subtarget->useMovt(*MF)) return 2; // MOVW + MOVT
     487             :   return 3; // Literal pool load
     488             : }
     489             : 
     490         182 : bool ARMDAGToDAGISel::canExtractShiftFromMul(const SDValue &N,
     491             :                                              unsigned MaxShift,
     492             :                                              unsigned &PowerOfTwo,
     493             :                                              SDValue &NewMulConst) const {
     494             :   assert(N.getOpcode() == ISD::MUL);
     495             :   assert(MaxShift > 0);
     496             : 
     497             :   // If the multiply is used in more than one place then changing the constant
     498             :   // will make other uses incorrect, so don't.
     499         364 :   if (!N.hasOneUse()) return false;
     500             :   // Check if the multiply is by a constant
     501         362 :   ConstantSDNode *MulConst = dyn_cast<ConstantSDNode>(N.getOperand(1));
     502             :   if (!MulConst) return false;
     503             :   // If the constant is used in more than one place then modifying it will mean
     504             :   // we need to materialize two constants instead of one, which is a bad idea.
     505         128 :   if (!MulConst->hasOneUse()) return false;
     506          60 :   unsigned MulConstVal = MulConst->getZExtValue();
     507          60 :   if (MulConstVal == 0) return false;
     508             : 
     509             :   // Find the largest power of 2 that MulConstVal is a multiple of
     510          60 :   PowerOfTwo = MaxShift;
     511        1427 :   while ((MulConstVal % (1 << PowerOfTwo)) != 0) {
     512        1377 :     --PowerOfTwo;
     513        1377 :     if (PowerOfTwo == 0) return false;
     514             :   }
     515             : 
     516             :   // Only optimise if the new cost is better
     517          50 :   unsigned NewMulConstVal = MulConstVal / (1 << PowerOfTwo);
     518         200 :   NewMulConst = CurDAG->getConstant(NewMulConstVal, SDLoc(N), MVT::i32);
     519          50 :   unsigned OldCost = ConstantMaterializationCost(MulConstVal);
     520          50 :   unsigned NewCost = ConstantMaterializationCost(NewMulConstVal);
     521          50 :   return NewCost < OldCost;
     522             : }
     523             : 
     524          33 : void ARMDAGToDAGISel::replaceDAGValue(const SDValue &N, SDValue M) {
     525          99 :   CurDAG->RepositionNode(N.getNode()->getIterator(), M.getNode());
     526          33 :   CurDAG->ReplaceAllUsesWith(N, M);
     527          33 : }
     528             : 
     529       12808 : bool ARMDAGToDAGISel::SelectImmShifterOperand(SDValue N,
     530             :                                               SDValue &BaseReg,
     531             :                                               SDValue &Opc,
     532             :                                               bool CheckProfitability) {
     533       12808 :   if (DisableShifterOp)
     534             :     return false;
     535             : 
     536             :   // If N is a multiply-by-constant and it's profitable to extract a shift and
     537             :   // use it in a shifted operand do so.
     538       25616 :   if (N.getOpcode() == ISD::MUL) {
     539         154 :     unsigned PowerOfTwo = 0;
     540         154 :     SDValue NewMulConst;
     541         154 :     if (canExtractShiftFromMul(N, 31, PowerOfTwo, NewMulConst)) {
     542          36 :       HandleSDNode Handle(N);
     543          36 :       SDLoc Loc(N);
     544          36 :       replaceDAGValue(N.getOperand(1), NewMulConst);
     545          18 :       BaseReg = Handle.getValue();
     546          36 :       Opc = CurDAG->getTargetConstant(
     547          36 :           ARM_AM::getSORegOpc(ARM_AM::lsl, PowerOfTwo), Loc, MVT::i32);
     548          18 :       return true;
     549             :     }
     550             :   }
     551             : 
     552       25580 :   ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOpcode());
     553             : 
     554             :   // Don't match base register only case. That is matched to a separate
     555             :   // lower complexity pattern with explicit register operand.
     556         942 :   if (ShOpcVal == ARM_AM::no_shift) return false;
     557             : 
     558        1884 :   BaseReg = N.getOperand(0);
     559         942 :   unsigned ShImmVal = 0;
     560        2810 :   ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1));
     561             :   if (!RHS) return false;
     562         926 :   ShImmVal = RHS->getZExtValue() & 31;
     563        2778 :   Opc = CurDAG->getTargetConstant(ARM_AM::getSORegOpc(ShOpcVal, ShImmVal),
     564        3704 :                                   SDLoc(N), MVT::i32);
     565         926 :   return true;
     566             : }
     567             : 
     568        8099 : bool ARMDAGToDAGISel::SelectRegShifterOperand(SDValue N,
     569             :                                               SDValue &BaseReg,
     570             :                                               SDValue &ShReg,
     571             :                                               SDValue &Opc,
     572             :                                               bool CheckProfitability) {
     573        8099 :   if (DisableShifterOp)
     574             :     return false;
     575             : 
     576       16198 :   ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOpcode());
     577             : 
     578             :   // Don't match base register only case. That is matched to a separate
     579             :   // lower complexity pattern with explicit register operand.
     580         837 :   if (ShOpcVal == ARM_AM::no_shift) return false;
     581             : 
     582        1674 :   BaseReg = N.getOperand(0);
     583         837 :   unsigned ShImmVal = 0;
     584        1674 :   ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1));
     585             :   if (RHS) return false;
     586             : 
     587          72 :   ShReg = N.getOperand(1);
     588          36 :   if (CheckProfitability && !isShifterOpProfitable(N, ShOpcVal, ShImmVal))
     589             :     return false;
     590         108 :   Opc = CurDAG->getTargetConstant(ARM_AM::getSORegOpc(ShOpcVal, ShImmVal),
     591         144 :                                   SDLoc(N), MVT::i32);
     592          36 :   return true;
     593             : }
     594             : 
     595             : 
     596        6003 : bool ARMDAGToDAGISel::SelectAddrModeImm12(SDValue N,
     597             :                                           SDValue &Base,
     598             :                                           SDValue &OffImm) {
     599             :   // Match simple R + imm12 operands.
     600             : 
     601             :   // Base only.
     602       18414 :   if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB &&
     603        3204 :       !CurDAG->isBaseWithConstantOffset(N)) {
     604        6088 :     if (N.getOpcode() == ISD::FrameIndex) {
     605             :       // Match frame index.
     606        1594 :       int FI = cast<FrameIndexSDNode>(N)->getIndex();
     607        3188 :       Base = CurDAG->getTargetFrameIndex(
     608        4782 :           FI, TLI->getPointerTy(CurDAG->getDataLayout()));
     609        7970 :       OffImm  = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
     610             :       return true;
     611             :     }
     612             : 
     613        1881 :     if (N.getOpcode() == ARMISD::Wrapper &&
     614        1165 :         N.getOperand(0).getOpcode() != ISD::TargetGlobalAddress &&
     615        2359 :         N.getOperand(0).getOpcode() != ISD::TargetExternalSymbol &&
     616         606 :         N.getOperand(0).getOpcode() != ISD::TargetGlobalTLSAddress) {
     617         594 :       Base = N.getOperand(0);
     618             :     } else
     619        1153 :       Base = N;
     620        7250 :     OffImm  = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
     621             :     return true;
     622             :   }
     623             : 
     624        8877 :   if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
     625        2959 :     int RHSC = (int)RHS->getSExtValue();
     626        5918 :     if (N.getOpcode() == ISD::SUB)
     627           0 :       RHSC = -RHSC;
     628             : 
     629        2959 :     if (RHSC > -0x1000 && RHSC < 0x1000) { // 12 bits
     630        5918 :       Base   = N.getOperand(0);
     631        5918 :       if (Base.getOpcode() == ISD::FrameIndex) {
     632        1325 :         int FI = cast<FrameIndexSDNode>(Base)->getIndex();
     633        2650 :         Base = CurDAG->getTargetFrameIndex(
     634        3975 :             FI, TLI->getPointerTy(CurDAG->getDataLayout()));
     635             :       }
     636       14795 :       OffImm = CurDAG->getTargetConstant(RHSC, SDLoc(N), MVT::i32);
     637             :       return true;
     638             :     }
     639             :   }
     640             : 
     641             :   // Base only.
     642           0 :   Base = N;
     643           0 :   OffImm  = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
     644             :   return true;
     645             : }
     646             : 
     647             : 
     648             : 
     649        6137 : bool ARMDAGToDAGISel::SelectLdStSOReg(SDValue N, SDValue &Base, SDValue &Offset,
     650             :                                       SDValue &Opc) {
     651       12274 :   if (N.getOpcode() == ISD::MUL &&
     652           0 :       ((!Subtarget->isLikeA9() && !Subtarget->isSwift()) || N.hasOneUse())) {
     653           0 :     if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
     654             :       // X * [3,5,9] -> X + X * [2,4,8] etc.
     655           0 :       int RHSC = (int)RHS->getZExtValue();
     656           0 :       if (RHSC & 1) {
     657           0 :         RHSC = RHSC & ~1;
     658           0 :         ARM_AM::AddrOpc AddSub = ARM_AM::add;
     659           0 :         if (RHSC < 0) {
     660           0 :           AddSub = ARM_AM::sub;
     661           0 :           RHSC = - RHSC;
     662             :         }
     663           0 :         if (isPowerOf2_32(RHSC)) {
     664           0 :           unsigned ShAmt = Log2_32(RHSC);
     665           0 :           Base = Offset = N.getOperand(0);
     666           0 :           Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt,
     667             :                                                             ARM_AM::lsl),
     668           0 :                                           SDLoc(N), MVT::i32);
     669           0 :           return true;
     670             :         }
     671             :       }
     672             :     }
     673             :   }
     674             : 
     675       18684 :   if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB &&
     676             :       // ISD::OR that is equivalent to an ISD::ADD.
     677        3204 :       !CurDAG->isBaseWithConstantOffset(N))
     678             :     return false;
     679             : 
     680             :   // Leave simple R +/- imm12 operands for LDRi12
     681        6348 :   if (N.getOpcode() == ISD::ADD || N.getOpcode() == ISD::OR) {
     682             :     int RHSC;
     683        6182 :     if (isScaledConstantInRange(N.getOperand(1), /*Scale=*/1,
     684             :                                 -0x1000+1, 0x1000, RHSC)) // 12 bits.
     685             :       return false;
     686             :   }
     687             : 
     688             :   // Otherwise this is R +/- [possibly shifted] R.
     689         268 :   ARM_AM::AddrOpc AddSub = N.getOpcode() == ISD::SUB ? ARM_AM::sub:ARM_AM::add;
     690             :   ARM_AM::ShiftOpc ShOpcVal =
     691         402 :     ARM_AM::getShiftOpcForNode(N.getOperand(1).getOpcode());
     692         134 :   unsigned ShAmt = 0;
     693             : 
     694         268 :   Base   = N.getOperand(0);
     695         268 :   Offset = N.getOperand(1);
     696             : 
     697         134 :   if (ShOpcVal != ARM_AM::no_shift) {
     698             :     // Check to see if the RHS of the shift is a constant, if not, we can't fold
     699             :     // it.
     700             :     if (ConstantSDNode *Sh =
     701         224 :            dyn_cast<ConstantSDNode>(N.getOperand(1).getOperand(1))) {
     702          56 :       ShAmt = Sh->getZExtValue();
     703          56 :       if (isShifterOpProfitable(Offset, ShOpcVal, ShAmt))
     704         168 :         Offset = N.getOperand(1).getOperand(0);
     705             :       else {
     706             :         ShAmt = 0;
     707             :         ShOpcVal = ARM_AM::no_shift;
     708             :       }
     709             :     } else {
     710             :       ShOpcVal = ARM_AM::no_shift;
     711             :     }
     712             :   }
     713             : 
     714             :   // Try matching (R shl C) + (R).
     715         268 :   if (N.getOpcode() != ISD::SUB && ShOpcVal == ARM_AM::no_shift &&
     716         208 :       !(Subtarget->isLikeA9() || Subtarget->isSwift() ||
     717         130 :         N.getOperand(0).hasOneUse())) {
     718          15 :     ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOperand(0).getOpcode());
     719           0 :     if (ShOpcVal != ARM_AM::no_shift) {
     720             :       // Check to see if the RHS of the shift is a constant, if not, we can't
     721             :       // fold it.
     722             :       if (ConstantSDNode *Sh =
     723           0 :           dyn_cast<ConstantSDNode>(N.getOperand(0).getOperand(1))) {
     724           0 :         ShAmt = Sh->getZExtValue();
     725           0 :         if (isShifterOpProfitable(N.getOperand(0), ShOpcVal, ShAmt)) {
     726           0 :           Offset = N.getOperand(0).getOperand(0);
     727           0 :           Base = N.getOperand(1);
     728             :         } else {
     729             :           ShAmt = 0;
     730             :           ShOpcVal = ARM_AM::no_shift;
     731             :         }
     732             :       } else {
     733             :         ShOpcVal = ARM_AM::no_shift;
     734             :       }
     735             :     }
     736             :   }
     737             : 
     738             :   // If Offset is a multiply-by-constant and it's profitable to extract a shift
     739             :   // and use it in a shifted operand do so.
     740         285 :   if (Offset.getOpcode() == ISD::MUL && N.hasOneUse()) {
     741          17 :     unsigned PowerOfTwo = 0;
     742          17 :     SDValue NewMulConst;
     743          17 :     if (canExtractShiftFromMul(Offset, 31, PowerOfTwo, NewMulConst)) {
     744          22 :       HandleSDNode Handle(Offset);
     745          22 :       replaceDAGValue(Offset.getOperand(1), NewMulConst);
     746          11 :       Offset = Handle.getValue();
     747          11 :       ShAmt = PowerOfTwo;
     748          11 :       ShOpcVal = ARM_AM::lsl;
     749             :     }
     750             :   }
     751             : 
     752         402 :   Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, ShOpcVal),
     753         536 :                                   SDLoc(N), MVT::i32);
     754         134 :   return true;
     755             : }
     756             : 
     757             : 
     758             : //-----
     759             : 
     760           2 : AddrMode2Type ARMDAGToDAGISel::SelectAddrMode2Worker(SDValue N,
     761             :                                                      SDValue &Base,
     762             :                                                      SDValue &Offset,
     763             :                                                      SDValue &Opc) {
     764           4 :   if (N.getOpcode() == ISD::MUL &&
     765           0 :       (!(Subtarget->isLikeA9() || Subtarget->isSwift()) || N.hasOneUse())) {
     766           0 :     if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
     767             :       // X * [3,5,9] -> X + X * [2,4,8] etc.
     768           0 :       int RHSC = (int)RHS->getZExtValue();
     769           0 :       if (RHSC & 1) {
     770           0 :         RHSC = RHSC & ~1;
     771           0 :         ARM_AM::AddrOpc AddSub = ARM_AM::add;
     772           0 :         if (RHSC < 0) {
     773           0 :           AddSub = ARM_AM::sub;
     774           0 :           RHSC = - RHSC;
     775             :         }
     776           0 :         if (isPowerOf2_32(RHSC)) {
     777           0 :           unsigned ShAmt = Log2_32(RHSC);
     778           0 :           Base = Offset = N.getOperand(0);
     779           0 :           Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt,
     780             :                                                             ARM_AM::lsl),
     781           0 :                                           SDLoc(N), MVT::i32);
     782           0 :           return AM2_SHOP;
     783             :         }
     784             :       }
     785             :     }
     786             :   }
     787             : 
     788           4 :   if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB &&
     789             :       // ISD::OR that is equivalent to an ADD.
     790           0 :       !CurDAG->isBaseWithConstantOffset(N)) {
     791           0 :     Base = N;
     792           0 :     if (N.getOpcode() == ISD::FrameIndex) {
     793           0 :       int FI = cast<FrameIndexSDNode>(N)->getIndex();
     794           0 :       Base = CurDAG->getTargetFrameIndex(
     795           0 :           FI, TLI->getPointerTy(CurDAG->getDataLayout()));
     796           0 :     } else if (N.getOpcode() == ARMISD::Wrapper &&
     797           0 :                N.getOperand(0).getOpcode() != ISD::TargetGlobalAddress &&
     798           0 :                N.getOperand(0).getOpcode() != ISD::TargetExternalSymbol &&
     799           0 :                N.getOperand(0).getOpcode() != ISD::TargetGlobalTLSAddress) {
     800           0 :       Base = N.getOperand(0);
     801             :     }
     802           0 :     Offset = CurDAG->getRegister(0, MVT::i32);
     803           0 :     Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(ARM_AM::add, 0,
     804             :                                                       ARM_AM::no_shift),
     805           0 :                                     SDLoc(N), MVT::i32);
     806           0 :     return AM2_BASE;
     807             :   }
     808             : 
     809             :   // Match simple R +/- imm12 operands.
     810           4 :   if (N.getOpcode() != ISD::SUB) {
     811             :     int RHSC;
     812           4 :     if (isScaledConstantInRange(N.getOperand(1), /*Scale=*/1,
     813             :                                 -0x1000+1, 0x1000, RHSC)) { // 12 bits.
     814           0 :       Base = N.getOperand(0);
     815           0 :       if (Base.getOpcode() == ISD::FrameIndex) {
     816           0 :         int FI = cast<FrameIndexSDNode>(Base)->getIndex();
     817           0 :         Base = CurDAG->getTargetFrameIndex(
     818           0 :             FI, TLI->getPointerTy(CurDAG->getDataLayout()));
     819             :       }
     820           0 :       Offset = CurDAG->getRegister(0, MVT::i32);
     821             : 
     822           0 :       ARM_AM::AddrOpc AddSub = ARM_AM::add;
     823           0 :       if (RHSC < 0) {
     824           0 :         AddSub = ARM_AM::sub;
     825           0 :         RHSC = - RHSC;
     826             :       }
     827           0 :       Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, RHSC,
     828             :                                                         ARM_AM::no_shift),
     829           0 :                                       SDLoc(N), MVT::i32);
     830           0 :       return AM2_BASE;
     831             :     }
     832             :   }
     833             : 
     834           4 :   if ((Subtarget->isLikeA9() || Subtarget->isSwift()) && !N.hasOneUse()) {
     835             :     // Compute R +/- (R << N) and reuse it.
     836           0 :     Base = N;
     837           0 :     Offset = CurDAG->getRegister(0, MVT::i32);
     838           0 :     Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(ARM_AM::add, 0,
     839             :                                                       ARM_AM::no_shift),
     840           0 :                                     SDLoc(N), MVT::i32);
     841           0 :     return AM2_BASE;
     842             :   }
     843             : 
     844             :   // Otherwise this is R +/- [possibly shifted] R.
     845           6 :   ARM_AM::AddrOpc AddSub = N.getOpcode() != ISD::SUB ? ARM_AM::add:ARM_AM::sub;
     846             :   ARM_AM::ShiftOpc ShOpcVal =
     847           6 :     ARM_AM::getShiftOpcForNode(N.getOperand(1).getOpcode());
     848           2 :   unsigned ShAmt = 0;
     849             : 
     850           4 :   Base   = N.getOperand(0);
     851           4 :   Offset = N.getOperand(1);
     852             : 
     853           2 :   if (ShOpcVal != ARM_AM::no_shift) {
     854             :     // Check to see if the RHS of the shift is a constant, if not, we can't fold
     855             :     // it.
     856             :     if (ConstantSDNode *Sh =
     857           0 :            dyn_cast<ConstantSDNode>(N.getOperand(1).getOperand(1))) {
     858           0 :       ShAmt = Sh->getZExtValue();
     859           0 :       if (isShifterOpProfitable(Offset, ShOpcVal, ShAmt))
     860           0 :         Offset = N.getOperand(1).getOperand(0);
     861             :       else {
     862             :         ShAmt = 0;
     863             :         ShOpcVal = ARM_AM::no_shift;
     864             :       }
     865             :     } else {
     866             :       ShOpcVal = ARM_AM::no_shift;
     867             :     }
     868             :   }
     869             : 
     870             :   // Try matching (R shl C) + (R).
     871           4 :   if (N.getOpcode() != ISD::SUB && ShOpcVal == ARM_AM::no_shift &&
     872           6 :       !(Subtarget->isLikeA9() || Subtarget->isSwift() ||
     873           4 :         N.getOperand(0).hasOneUse())) {
     874           0 :     ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOperand(0).getOpcode());
     875           0 :     if (ShOpcVal != ARM_AM::no_shift) {
     876             :       // Check to see if the RHS of the shift is a constant, if not, we can't
     877             :       // fold it.
     878             :       if (ConstantSDNode *Sh =
     879           0 :           dyn_cast<ConstantSDNode>(N.getOperand(0).getOperand(1))) {
     880           0 :         ShAmt = Sh->getZExtValue();
     881           0 :         if (isShifterOpProfitable(N.getOperand(0), ShOpcVal, ShAmt)) {
     882           0 :           Offset = N.getOperand(0).getOperand(0);
     883           0 :           Base = N.getOperand(1);
     884             :         } else {
     885             :           ShAmt = 0;
     886             :           ShOpcVal = ARM_AM::no_shift;
     887             :         }
     888             :       } else {
     889             :         ShOpcVal = ARM_AM::no_shift;
     890             :       }
     891             :     }
     892             :   }
     893             : 
     894           6 :   Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, ShOpcVal),
     895           8 :                                   SDLoc(N), MVT::i32);
     896           2 :   return AM2_SHOP;
     897             : }
     898             : 
     899          23 : bool ARMDAGToDAGISel::SelectAddrMode2OffsetReg(SDNode *Op, SDValue N,
     900             :                                             SDValue &Offset, SDValue &Opc) {
     901          23 :   unsigned Opcode = Op->getOpcode();
     902             :   ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
     903          60 :     ? cast<LoadSDNode>(Op)->getAddressingMode()
     904          32 :     : cast<StoreSDNode>(Op)->getAddressingMode();
     905          46 :   ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC)
     906          23 :     ? ARM_AM::add : ARM_AM::sub;
     907             :   int Val;
     908          23 :   if (isScaledConstantInRange(N, /*Scale=*/1, 0, 0x1000, Val))
     909             :     return false;
     910             : 
     911          17 :   Offset = N;
     912          34 :   ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOpcode());
     913           5 :   unsigned ShAmt = 0;
     914           5 :   if (ShOpcVal != ARM_AM::no_shift) {
     915             :     // Check to see if the RHS of the shift is a constant, if not, we can't fold
     916             :     // it.
     917          15 :     if (ConstantSDNode *Sh = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
     918           5 :       ShAmt = Sh->getZExtValue();
     919           5 :       if (isShifterOpProfitable(N, ShOpcVal, ShAmt))
     920           6 :         Offset = N.getOperand(0);
     921             :       else {
     922             :         ShAmt = 0;
     923             :         ShOpcVal = ARM_AM::no_shift;
     924             :       }
     925             :     } else {
     926             :       ShOpcVal = ARM_AM::no_shift;
     927             :     }
     928             :   }
     929             : 
     930          51 :   Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, ShOpcVal),
     931          68 :                                   SDLoc(N), MVT::i32);
     932          17 :   return true;
     933             : }
     934             : 
     935          18 : bool ARMDAGToDAGISel::SelectAddrMode2OffsetImmPre(SDNode *Op, SDValue N,
     936             :                                             SDValue &Offset, SDValue &Opc) {
     937          18 :   unsigned Opcode = Op->getOpcode();
     938             :   ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
     939          54 :     ? cast<LoadSDNode>(Op)->getAddressingMode()
     940          18 :     : cast<StoreSDNode>(Op)->getAddressingMode();
     941          36 :   ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC)
     942          18 :     ? ARM_AM::add : ARM_AM::sub;
     943             :   int Val;
     944          27 :   if (isScaledConstantInRange(N, /*Scale=*/1, 0, 0x1000, Val)) { // 12 bits.
     945           9 :     if (AddSub == ARM_AM::sub) Val *= -1;
     946          18 :     Offset = CurDAG->getRegister(0, MVT::i32);
     947          45 :     Opc = CurDAG->getTargetConstant(Val, SDLoc(Op), MVT::i32);
     948             :     return true;
     949             :   }
     950             : 
     951             :   return false;
     952             : }
     953             : 
     954             : 
     955          32 : bool ARMDAGToDAGISel::SelectAddrMode2OffsetImm(SDNode *Op, SDValue N,
     956             :                                             SDValue &Offset, SDValue &Opc) {
     957          32 :   unsigned Opcode = Op->getOpcode();
     958             :   ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
     959          78 :     ? cast<LoadSDNode>(Op)->getAddressingMode()
     960          50 :     : cast<StoreSDNode>(Op)->getAddressingMode();
     961          64 :   ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC)
     962          32 :     ? ARM_AM::add : ARM_AM::sub;
     963             :   int Val;
     964          57 :   if (isScaledConstantInRange(N, /*Scale=*/1, 0, 0x1000, Val)) { // 12 bits.
     965          50 :     Offset = CurDAG->getRegister(0, MVT::i32);
     966         100 :     Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, Val,
     967             :                                                       ARM_AM::no_shift),
     968         100 :                                     SDLoc(Op), MVT::i32);
     969             :     return true;
     970             :   }
     971             : 
     972             :   return false;
     973             : }
     974             : 
     975             : bool ARMDAGToDAGISel::SelectAddrOffsetNone(SDValue N, SDValue &Base) {
     976         421 :   Base = N;
     977             :   return true;
     978             : }
     979             : 
     980         531 : bool ARMDAGToDAGISel::SelectAddrMode3(SDValue N,
     981             :                                       SDValue &Base, SDValue &Offset,
     982             :                                       SDValue &Opc) {
     983        1062 :   if (N.getOpcode() == ISD::SUB) {
     984             :     // X - C  is canonicalize to X + -C, no need to handle it here.
     985           0 :     Base = N.getOperand(0);
     986           0 :     Offset = N.getOperand(1);
     987           0 :     Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::sub, 0), SDLoc(N),
     988           0 :                                     MVT::i32);
     989             :     return true;
     990             :   }
     991             : 
     992         531 :   if (!CurDAG->isBaseWithConstantOffset(N)) {
     993         453 :     Base = N;
     994         906 :     if (N.getOpcode() == ISD::FrameIndex) {
     995          27 :       int FI = cast<FrameIndexSDNode>(N)->getIndex();
     996          54 :       Base = CurDAG->getTargetFrameIndex(
     997          81 :           FI, TLI->getPointerTy(CurDAG->getDataLayout()));
     998             :     }
     999         906 :     Offset = CurDAG->getRegister(0, MVT::i32);
    1000        2265 :     Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::add, 0), SDLoc(N),
    1001         906 :                                     MVT::i32);
    1002             :     return true;
    1003             :   }
    1004             : 
    1005             :   // If the RHS is +/- imm8, fold into addr mode.
    1006             :   int RHSC;
    1007         234 :   if (isScaledConstantInRange(N.getOperand(1), /*Scale=*/1,
    1008             :                               -256 + 1, 256, RHSC)) { // 8 bits.
    1009         156 :     Base = N.getOperand(0);
    1010         156 :     if (Base.getOpcode() == ISD::FrameIndex) {
    1011          21 :       int FI = cast<FrameIndexSDNode>(Base)->getIndex();
    1012          42 :       Base = CurDAG->getTargetFrameIndex(
    1013          63 :           FI, TLI->getPointerTy(CurDAG->getDataLayout()));
    1014             :     }
    1015         156 :     Offset = CurDAG->getRegister(0, MVT::i32);
    1016             : 
    1017          78 :     ARM_AM::AddrOpc AddSub = ARM_AM::add;
    1018          78 :     if (RHSC < 0) {
    1019           0 :       AddSub = ARM_AM::sub;
    1020           0 :       RHSC = -RHSC;
    1021             :     }
    1022         390 :     Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, RHSC), SDLoc(N),
    1023         156 :                                     MVT::i32);
    1024             :     return true;
    1025             :   }
    1026             : 
    1027           0 :   Base = N.getOperand(0);
    1028           0 :   Offset = N.getOperand(1);
    1029           0 :   Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::add, 0), SDLoc(N),
    1030           0 :                                   MVT::i32);
    1031             :   return true;
    1032             : }
    1033             : 
    1034          14 : bool ARMDAGToDAGISel::SelectAddrMode3Offset(SDNode *Op, SDValue N,
    1035             :                                             SDValue &Offset, SDValue &Opc) {
    1036          14 :   unsigned Opcode = Op->getOpcode();
    1037             :   ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
    1038          37 :     ? cast<LoadSDNode>(Op)->getAddressingMode()
    1039          19 :     : cast<StoreSDNode>(Op)->getAddressingMode();
    1040          28 :   ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC)
    1041          14 :     ? ARM_AM::add : ARM_AM::sub;
    1042             :   int Val;
    1043          28 :   if (isScaledConstantInRange(N, /*Scale=*/1, 0, 256, Val)) { // 12 bits.
    1044          28 :     Offset = CurDAG->getRegister(0, MVT::i32);
    1045          70 :     Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, Val), SDLoc(Op),
    1046          28 :                                     MVT::i32);
    1047             :     return true;
    1048             :   }
    1049             : 
    1050           0 :   Offset = N;
    1051           0 :   Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, 0), SDLoc(Op),
    1052           0 :                                   MVT::i32);
    1053             :   return true;
    1054             : }
    1055             : 
    1056        3013 : bool ARMDAGToDAGISel::SelectAddrMode5(SDValue N,
    1057             :                                       SDValue &Base, SDValue &Offset) {
    1058        3013 :   if (!CurDAG->isBaseWithConstantOffset(N)) {
    1059        2785 :     Base = N;
    1060        5570 :     if (N.getOpcode() == ISD::FrameIndex) {
    1061         281 :       int FI = cast<FrameIndexSDNode>(N)->getIndex();
    1062         562 :       Base = CurDAG->getTargetFrameIndex(
    1063         843 :           FI, TLI->getPointerTy(CurDAG->getDataLayout()));
    1064        3247 :     } else if (N.getOpcode() == ARMISD::Wrapper &&
    1065        2140 :                N.getOperand(0).getOpcode() != ISD::TargetGlobalAddress &&
    1066        4466 :                N.getOperand(0).getOpcode() != ISD::TargetExternalSymbol &&
    1067        1308 :                N.getOperand(0).getOpcode() != ISD::TargetGlobalTLSAddress) {
    1068        1308 :       Base = N.getOperand(0);
    1069             :     }
    1070        8355 :     Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(ARM_AM::add, 0),
    1071       11140 :                                        SDLoc(N), MVT::i32);
    1072             :     return true;
    1073             :   }
    1074             : 
    1075             :   // If the RHS is +/- imm8, fold into addr mode.
    1076             :   int RHSC;
    1077         684 :   if (isScaledConstantInRange(N.getOperand(1), /*Scale=*/4,
    1078             :                               -256 + 1, 256, RHSC)) {
    1079         456 :     Base = N.getOperand(0);
    1080         456 :     if (Base.getOpcode() == ISD::FrameIndex) {
    1081           3 :       int FI = cast<FrameIndexSDNode>(Base)->getIndex();
    1082           6 :       Base = CurDAG->getTargetFrameIndex(
    1083           9 :           FI, TLI->getPointerTy(CurDAG->getDataLayout()));
    1084             :     }
    1085             : 
    1086         228 :     ARM_AM::AddrOpc AddSub = ARM_AM::add;
    1087         228 :     if (RHSC < 0) {
    1088           2 :       AddSub = ARM_AM::sub;
    1089           2 :       RHSC = -RHSC;
    1090             :     }
    1091         912 :     Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(AddSub, RHSC),
    1092         912 :                                        SDLoc(N), MVT::i32);
    1093             :     return true;
    1094             :   }
    1095             : 
    1096           0 :   Base = N;
    1097           0 :   Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(ARM_AM::add, 0),
    1098           0 :                                      SDLoc(N), MVT::i32);
    1099             :   return true;
    1100             : }
    1101             : 
    1102        3727 : bool ARMDAGToDAGISel::SelectAddrMode6(SDNode *Parent, SDValue N, SDValue &Addr,
    1103             :                                       SDValue &Align) {
    1104        3727 :   Addr = N;
    1105             : 
    1106        3727 :   unsigned Alignment = 0;
    1107             : 
    1108        3727 :   MemSDNode *MemN = cast<MemSDNode>(Parent);
    1109             : 
    1110        4305 :   if (isa<LSBaseSDNode>(MemN) ||
    1111        1066 :       ((MemN->getOpcode() == ARMISD::VST1_UPD ||
    1112         678 :         MemN->getOpcode() == ARMISD::VLD1_UPD) &&
    1113         570 :        MemN->getConstantOperandVal(MemN->getNumOperands() - 1) == 1)) {
    1114             :     // This case occurs only for VLD1-lane/dup and VST1-lane instructions.
    1115             :     // The maximum alignment is equal to the memory size being referenced.
    1116        6672 :     unsigned MMOAlign = MemN->getAlignment();
    1117        3336 :     unsigned MemSize = MemN->getMemoryVT().getSizeInBits() / 8;
    1118        3336 :     if (MMOAlign >= MemSize && MemSize > 1)
    1119        1605 :       Alignment = MemSize;
    1120             :   } else {
    1121             :     // All other uses of addrmode6 are for intrinsics.  For now just record
    1122             :     // the raw alignment value; it will be refined later based on the legal
    1123             :     // alignment operands for the intrinsic.
    1124         782 :     Alignment = MemN->getAlignment();
    1125             :   }
    1126             : 
    1127       18635 :   Align = CurDAG->getTargetConstant(Alignment, SDLoc(N), MVT::i32);
    1128        3727 :   return true;
    1129             : }
    1130             : 
    1131           4 : bool ARMDAGToDAGISel::SelectAddrMode6Offset(SDNode *Op, SDValue N,
    1132             :                                             SDValue &Offset) {
    1133           4 :   LSBaseSDNode *LdSt = cast<LSBaseSDNode>(Op);
    1134           4 :   ISD::MemIndexedMode AM = LdSt->getAddressingMode();
    1135           4 :   if (AM != ISD::POST_INC)
    1136             :     return false;
    1137           4 :   Offset = N;
    1138           4 :   if (ConstantSDNode *NC = dyn_cast<ConstantSDNode>(N)) {
    1139           4 :     if (NC->getZExtValue() * 8 == LdSt->getMemoryVT().getSizeInBits())
    1140           6 :       Offset = CurDAG->getRegister(0, MVT::i32);
    1141             :   }
    1142             :   return true;
    1143             : }
    1144             : 
    1145        6785 : bool ARMDAGToDAGISel::SelectAddrModePC(SDValue N,
    1146             :                                        SDValue &Offset, SDValue &Label) {
    1147       13613 :   if (N.getOpcode() == ARMISD::PIC_ADD && N.hasOneUse()) {
    1148          86 :     Offset = N.getOperand(0);
    1149          86 :     SDValue N1 = N.getOperand(1);
    1150          86 :     Label = CurDAG->getTargetConstant(cast<ConstantSDNode>(N1)->getZExtValue(),
    1151         215 :                                       SDLoc(N), MVT::i32);
    1152             :     return true;
    1153             :   }
    1154             : 
    1155             :   return false;
    1156             : }
    1157             : 
    1158             : 
    1159             : //===----------------------------------------------------------------------===//
    1160             : //                         Thumb Addressing Modes
    1161             : //===----------------------------------------------------------------------===//
    1162             : 
    1163         259 : bool ARMDAGToDAGISel::SelectThumbAddrModeRR(SDValue N,
    1164             :                                             SDValue &Base, SDValue &Offset){
    1165         518 :   if (N.getOpcode() != ISD::ADD && !CurDAG->isBaseWithConstantOffset(N)) {
    1166           2 :     ConstantSDNode *NC = dyn_cast<ConstantSDNode>(N);
    1167           2 :     if (!NC || !NC->isNullValue())
    1168             :       return false;
    1169             : 
    1170           2 :     Base = Offset = N;
    1171             :     return true;
    1172             :   }
    1173             : 
    1174         480 :   Base = N.getOperand(0);
    1175         480 :   Offset = N.getOperand(1);
    1176             :   return true;
    1177             : }
    1178             : 
    1179             : bool
    1180         960 : ARMDAGToDAGISel::SelectThumbAddrModeImm5S(SDValue N, unsigned Scale,
    1181             :                                           SDValue &Base, SDValue &OffImm) {
    1182         960 :   if (!CurDAG->isBaseWithConstantOffset(N)) {
    1183        1360 :     if (N.getOpcode() == ISD::ADD) {
    1184             :       return false; // We want to select register offset instead
    1185         546 :     } else if (N.getOpcode() == ARMISD::Wrapper &&
    1186          46 :         N.getOperand(0).getOpcode() != ISD::TargetGlobalAddress &&
    1187          30 :         N.getOperand(0).getOpcode() != ISD::TargetExternalSymbol &&
    1188         548 :         N.getOperand(0).getOpcode() != ISD::TargetConstantPool &&
    1189           0 :         N.getOperand(0).getOpcode() != ISD::TargetGlobalTLSAddress) {
    1190           0 :       Base = N.getOperand(0);
    1191             :     } else {
    1192         528 :       Base = N;
    1193             :     }
    1194             : 
    1195        2640 :     OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
    1196             :     return true;
    1197             :   }
    1198             : 
    1199             :   // If the RHS is + imm5 * scale, fold into addr mode.
    1200             :   int RHSC;
    1201         765 :   if (isScaledConstantInRange(N.getOperand(1), Scale, 0, 32, RHSC)) {
    1202         410 :     Base = N.getOperand(0);
    1203        1025 :     OffImm = CurDAG->getTargetConstant(RHSC, SDLoc(N), MVT::i32);
    1204             :     return true;
    1205             :   }
    1206             : 
    1207             :   // Offset is too large, so use register offset instead.
    1208             :   return false;
    1209             : }
    1210             : 
    1211             : bool
    1212             : ARMDAGToDAGISel::SelectThumbAddrModeImm5S4(SDValue N, SDValue &Base,
    1213             :                                            SDValue &OffImm) {
    1214         811 :   return SelectThumbAddrModeImm5S(N, 4, Base, OffImm);
    1215             : }
    1216             : 
    1217             : bool
    1218             : ARMDAGToDAGISel::SelectThumbAddrModeImm5S2(SDValue N, SDValue &Base,
    1219             :                                            SDValue &OffImm) {
    1220          85 :   return SelectThumbAddrModeImm5S(N, 2, Base, OffImm);
    1221             : }
    1222             : 
    1223             : bool
    1224             : ARMDAGToDAGISel::SelectThumbAddrModeImm5S1(SDValue N, SDValue &Base,
    1225             :                                            SDValue &OffImm) {
    1226          64 :   return SelectThumbAddrModeImm5S(N, 1, Base, OffImm);
    1227             : }
    1228             : 
    1229        1665 : bool ARMDAGToDAGISel::SelectThumbAddrModeSP(SDValue N,
    1230             :                                             SDValue &Base, SDValue &OffImm) {
    1231        3330 :   if (N.getOpcode() == ISD::FrameIndex) {
    1232         335 :     int FI = cast<FrameIndexSDNode>(N)->getIndex();
    1233             :     // Only multiples of 4 are allowed for the offset, so the frame object
    1234             :     // alignment must be at least 4.
    1235         335 :     MachineFrameInfo &MFI = MF->getFrameInfo();
    1236         335 :     if (MFI.getObjectAlignment(FI) < 4)
    1237             :       MFI.setObjectAlignment(FI, 4);
    1238         670 :     Base = CurDAG->getTargetFrameIndex(
    1239        1005 :         FI, TLI->getPointerTy(CurDAG->getDataLayout()));
    1240        1675 :     OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
    1241             :     return true;
    1242             :   }
    1243             : 
    1244        1330 :   if (!CurDAG->isBaseWithConstantOffset(N))
    1245             :     return false;
    1246             : 
    1247        2247 :   RegisterSDNode *LHSR = dyn_cast<RegisterSDNode>(N.getOperand(0));
    1248        2247 :   if (N.getOperand(0).getOpcode() == ISD::FrameIndex ||
    1249           0 :       (LHSR && LHSR->getReg() == ARM::SP)) {
    1250             :     // If the RHS is + imm8 * scale, fold into addr mode.
    1251             :     int RHSC;
    1252        1503 :     if (isScaledConstantInRange(N.getOperand(1), /*Scale=*/4, 0, 256, RHSC)) {
    1253        1002 :       Base = N.getOperand(0);
    1254        1002 :       if (Base.getOpcode() == ISD::FrameIndex) {
    1255         501 :         int FI = cast<FrameIndexSDNode>(Base)->getIndex();
    1256             :         // For LHS+RHS to result in an offset that's a multiple of 4 the object
    1257             :         // indexed by the LHS must be 4-byte aligned.
    1258         501 :         MachineFrameInfo &MFI = MF->getFrameInfo();
    1259         501 :         if (MFI.getObjectAlignment(FI) < 4)
    1260             :           MFI.setObjectAlignment(FI, 4);
    1261        1002 :         Base = CurDAG->getTargetFrameIndex(
    1262        1503 :             FI, TLI->getPointerTy(CurDAG->getDataLayout()));
    1263             :       }
    1264        2505 :       OffImm = CurDAG->getTargetConstant(RHSC, SDLoc(N), MVT::i32);
    1265         501 :       return true;
    1266             :     }
    1267             :   }
    1268             : 
    1269             :   return false;
    1270             : }
    1271             : 
    1272             : 
    1273             : //===----------------------------------------------------------------------===//
    1274             : //                        Thumb 2 Addressing Modes
    1275             : //===----------------------------------------------------------------------===//
    1276             : 
    1277             : 
    1278        3830 : bool ARMDAGToDAGISel::SelectT2AddrModeImm12(SDValue N,
    1279             :                                             SDValue &Base, SDValue &OffImm) {
    1280             :   // Match simple R + imm12 operands.
    1281             : 
    1282             :   // Base only.
    1283       12078 :   if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB &&
    1284        2209 :       !CurDAG->isBaseWithConstantOffset(N)) {
    1285        4304 :     if (N.getOpcode() == ISD::FrameIndex) {
    1286             :       // Match frame index.
    1287         926 :       int FI = cast<FrameIndexSDNode>(N)->getIndex();
    1288        1852 :       Base = CurDAG->getTargetFrameIndex(
    1289        2778 :           FI, TLI->getPointerTy(CurDAG->getDataLayout()));
    1290        4630 :       OffImm  = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
    1291         926 :       return true;
    1292             :     }
    1293             : 
    1294        1437 :     if (N.getOpcode() == ARMISD::Wrapper &&
    1295         499 :         N.getOperand(0).getOpcode() != ISD::TargetGlobalAddress &&
    1296        1450 :         N.getOperand(0).getOpcode() != ISD::TargetExternalSymbol &&
    1297         140 :         N.getOperand(0).getOpcode() != ISD::TargetGlobalTLSAddress) {
    1298         128 :       Base = N.getOperand(0);
    1299         128 :       if (Base.getOpcode() == ISD::TargetConstantPool)
    1300             :         return false;  // We want to select t2LDRpci instead.
    1301             :     } else
    1302        1162 :       Base = N;
    1303        5810 :     OffImm  = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
    1304        1162 :     return true;
    1305             :   }
    1306             : 
    1307        5034 :   if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
    1308        1678 :     if (SelectT2AddrModeImm8(N, Base, OffImm))
    1309             :       // Let t2LDRi8 handle (R - imm8).
    1310             :       return false;
    1311             : 
    1312        1653 :     int RHSC = (int)RHS->getZExtValue();
    1313        3306 :     if (N.getOpcode() == ISD::SUB)
    1314           0 :       RHSC = -RHSC;
    1315             : 
    1316        1653 :     if (RHSC >= 0 && RHSC < 0x1000) { // 12 bits (unsigned)
    1317        3306 :       Base   = N.getOperand(0);
    1318        3306 :       if (Base.getOpcode() == ISD::FrameIndex) {
    1319         764 :         int FI = cast<FrameIndexSDNode>(Base)->getIndex();
    1320        1528 :         Base = CurDAG->getTargetFrameIndex(
    1321        2292 :             FI, TLI->getPointerTy(CurDAG->getDataLayout()));
    1322             :       }
    1323        8265 :       OffImm = CurDAG->getTargetConstant(RHSC, SDLoc(N), MVT::i32);
    1324        1653 :       return true;
    1325             :     }
    1326             :   }
    1327             : 
    1328             :   // Base only.
    1329           0 :   Base = N;
    1330           0 :   OffImm  = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
    1331           0 :   return true;
    1332             : }
    1333             : 
    1334        1767 : bool ARMDAGToDAGISel::SelectT2AddrModeImm8(SDValue N,
    1335             :                                            SDValue &Base, SDValue &OffImm) {
    1336             :   // Match simple R - imm8 operands.
    1337        3776 :   if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB &&
    1338         121 :       !CurDAG->isBaseWithConstantOffset(N))
    1339             :     return false;
    1340             : 
    1341        5109 :   if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
    1342        1703 :     int RHSC = (int)RHS->getSExtValue();
    1343        3406 :     if (N.getOpcode() == ISD::SUB)
    1344           0 :       RHSC = -RHSC;
    1345             : 
    1346        1703 :     if ((RHSC >= -255) && (RHSC < 0)) { // 8 bits (always negative)
    1347         100 :       Base = N.getOperand(0);
    1348         100 :       if (Base.getOpcode() == ISD::FrameIndex) {
    1349           0 :         int FI = cast<FrameIndexSDNode>(Base)->getIndex();
    1350           0 :         Base = CurDAG->getTargetFrameIndex(
    1351           0 :             FI, TLI->getPointerTy(CurDAG->getDataLayout()));
    1352             :       }
    1353         250 :       OffImm = CurDAG->getTargetConstant(RHSC, SDLoc(N), MVT::i32);
    1354             :       return true;
    1355             :     }
    1356             :   }
    1357             : 
    1358             :   return false;
    1359             : }
    1360             : 
    1361          48 : bool ARMDAGToDAGISel::SelectT2AddrModeImm8Offset(SDNode *Op, SDValue N,
    1362             :                                                  SDValue &OffImm){
    1363          48 :   unsigned Opcode = Op->getOpcode();
    1364             :   ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
    1365         119 :     ? cast<LoadSDNode>(Op)->getAddressingMode()
    1366          73 :     : cast<StoreSDNode>(Op)->getAddressingMode();
    1367             :   int RHSC;
    1368          96 :   if (isScaledConstantInRange(N, /*Scale=*/1, 0, 0x100, RHSC)) { // 8 bits.
    1369          96 :     OffImm = ((AM == ISD::PRE_INC) || (AM == ISD::POST_INC))
    1370         272 :       ? CurDAG->getTargetConstant(RHSC, SDLoc(N), MVT::i32)
    1371         120 :       : CurDAG->getTargetConstant(-RHSC, SDLoc(N), MVT::i32);
    1372             :     return true;
    1373             :   }
    1374             : 
    1375             :   return false;
    1376             : }
    1377             : 
    1378        3982 : bool ARMDAGToDAGISel::SelectT2AddrModeSoReg(SDValue N,
    1379             :                                             SDValue &Base,
    1380             :                                             SDValue &OffReg, SDValue &ShImm) {
    1381             :   // (R - imm8) should be handled by t2LDRi8. The rest are handled by t2LDRi12.
    1382        7964 :   if (N.getOpcode() != ISD::ADD && !CurDAG->isBaseWithConstantOffset(N))
    1383             :     return false;
    1384             : 
    1385             :   // Leave (R + imm12) for t2LDRi12, (R - imm8) for t2LDRi8.
    1386        5353 :   if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
    1387        1693 :     int RHSC = (int)RHS->getZExtValue();
    1388        1693 :     if (RHSC >= 0 && RHSC < 0x1000) // 12 bits (unsigned)
    1389             :       return false;
    1390          40 :     else if (RHSC < 0 && RHSC >= -255) // 8 bits
    1391             :       return false;
    1392             :   }
    1393             : 
    1394             :   // Look for (R + R) or (R + (R << [1,2,3])).
    1395         152 :   unsigned ShAmt = 0;
    1396         304 :   Base   = N.getOperand(0);
    1397         304 :   OffReg = N.getOperand(1);
    1398             : 
    1399             :   // Swap if it is ((R << c) + R).
    1400         304 :   ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(OffReg.getOpcode());
    1401          55 :   if (ShOpcVal != ARM_AM::lsl) {
    1402         208 :     ShOpcVal = ARM_AM::getShiftOpcForNode(Base.getOpcode());
    1403           0 :     if (ShOpcVal == ARM_AM::lsl)
    1404             :       std::swap(Base, OffReg);
    1405             :   }
    1406             : 
    1407         152 :   if (ShOpcVal == ARM_AM::lsl) {
    1408             :     // Check to see if the RHS of the shift is a constant, if not, we can't fold
    1409             :     // it.
    1410         144 :     if (ConstantSDNode *Sh = dyn_cast<ConstantSDNode>(OffReg.getOperand(1))) {
    1411          48 :       ShAmt = Sh->getZExtValue();
    1412          48 :       if (ShAmt < 4 && isShifterOpProfitable(OffReg, ShOpcVal, ShAmt))
    1413          96 :         OffReg = OffReg.getOperand(0);
    1414             :       else {
    1415             :         ShAmt = 0;
    1416             :       }
    1417             :     }
    1418             :   }
    1419             : 
    1420             :   // If OffReg is a multiply-by-constant and it's profitable to extract a shift
    1421             :   // and use it in a shifted operand do so.
    1422         316 :   if (OffReg.getOpcode() == ISD::MUL && N.hasOneUse()) {
    1423          11 :     unsigned PowerOfTwo = 0;
    1424          11 :     SDValue NewMulConst;
    1425          11 :     if (canExtractShiftFromMul(OffReg, 3, PowerOfTwo, NewMulConst)) {
    1426           8 :       HandleSDNode Handle(OffReg);
    1427           8 :       replaceDAGValue(OffReg.getOperand(1), NewMulConst);
    1428           4 :       OffReg = Handle.getValue();
    1429           4 :       ShAmt = PowerOfTwo;
    1430             :     }
    1431             :   }
    1432             : 
    1433         760 :   ShImm = CurDAG->getTargetConstant(ShAmt, SDLoc(N), MVT::i32);
    1434             : 
    1435         152 :   return true;
    1436             : }
    1437             : 
    1438          79 : bool ARMDAGToDAGISel::SelectT2AddrModeExclusive(SDValue N, SDValue &Base,
    1439             :                                                 SDValue &OffImm) {
    1440             :   // This *must* succeed since it's used for the irreplaceable ldrex and strex
    1441             :   // instructions.
    1442          79 :   Base = N;
    1443         395 :   OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
    1444             : 
    1445         158 :   if (N.getOpcode() != ISD::ADD || !CurDAG->isBaseWithConstantOffset(N))
    1446             :     return true;
    1447             : 
    1448          18 :   ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1));
    1449             :   if (!RHS)
    1450             :     return true;
    1451             : 
    1452           6 :   uint32_t RHSC = (int)RHS->getZExtValue();
    1453           6 :   if (RHSC > 1020 || RHSC % 4 != 0)
    1454             :     return true;
    1455             : 
    1456           4 :   Base = N.getOperand(0);
    1457           4 :   if (Base.getOpcode() == ISD::FrameIndex) {
    1458           0 :     int FI = cast<FrameIndexSDNode>(Base)->getIndex();
    1459           0 :     Base = CurDAG->getTargetFrameIndex(
    1460           0 :         FI, TLI->getPointerTy(CurDAG->getDataLayout()));
    1461             :   }
    1462             : 
    1463          10 :   OffImm = CurDAG->getTargetConstant(RHSC/4, SDLoc(N), MVT::i32);
    1464             :   return true;
    1465             : }
    1466             : 
    1467             : //===--------------------------------------------------------------------===//
    1468             : 
    1469             : /// getAL - Returns a ARMCC::AL immediate node.
    1470        3457 : static inline SDValue getAL(SelectionDAG *CurDAG, const SDLoc &dl) {
    1471        6914 :   return CurDAG->getTargetConstant((uint64_t)ARMCC::AL, dl, MVT::i32);
    1472             : }
    1473             : 
    1474             : void ARMDAGToDAGISel::transferMemOperands(SDNode *N, SDNode *Result) {
    1475          66 :   MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1);
    1476          66 :   MemOp[0] = cast<MemSDNode>(N)->getMemOperand();
    1477         132 :   cast<MachineSDNode>(Result)->setMemRefs(MemOp, MemOp + 1);
    1478             : }
    1479             : 
    1480        6899 : bool ARMDAGToDAGISel::tryARMIndexedLoad(SDNode *N) {
    1481        6899 :   LoadSDNode *LD = cast<LoadSDNode>(N);
    1482       13798 :   ISD::MemIndexedMode AM = LD->getAddressingMode();
    1483        6899 :   if (AM == ISD::UNINDEXED)
    1484             :     return false;
    1485             : 
    1486          41 :   EVT LoadedVT = LD->getMemoryVT();
    1487          41 :   SDValue Offset, AMOpc;
    1488          41 :   bool isPre = (AM == ISD::PRE_INC) || (AM == ISD::PRE_DEC);
    1489          41 :   unsigned Opcode = 0;
    1490          41 :   bool Match = false;
    1491          77 :   if (LoadedVT == MVT::i32 && isPre &&
    1492          11 :       SelectAddrMode2OffsetImmPre(N, LD->getOffset(), Offset, AMOpc)) {
    1493             :     Opcode = ARM::LDR_PRE_IMM;
    1494             :     Match = true;
    1495          76 :   } else if (LoadedVT == MVT::i32 && !isPre &&
    1496          14 :       SelectAddrMode2OffsetImm(N, LD->getOffset(), Offset, AMOpc)) {
    1497             :     Opcode = ARM::LDR_POST_IMM;
    1498             :     Match = true;
    1499          44 :   } else if (LoadedVT == MVT::i32 &&
    1500          14 :       SelectAddrMode2OffsetReg(N, LD->getOffset(), Offset, AMOpc)) {
    1501          14 :     Opcode = isPre ? ARM::LDR_PRE_REG : ARM::LDR_POST_REG;
    1502             :     Match = true;
    1503             : 
    1504          20 :   } else if (LoadedVT == MVT::i16 &&
    1505           4 :              SelectAddrMode3Offset(N, LD->getOffset(), Offset, AMOpc)) {
    1506           4 :     Match = true;
    1507           4 :     Opcode = (LD->getExtensionType() == ISD::SEXTLOAD)
    1508           4 :       ? (isPre ? ARM::LDRSH_PRE : ARM::LDRSH_POST)
    1509             :       : (isPre ? ARM::LDRH_PRE : ARM::LDRH_POST);
    1510          12 :   } else if (LoadedVT == MVT::i8 || LoadedVT == MVT::i1) {
    1511          12 :     if (LD->getExtensionType() == ISD::SEXTLOAD) {
    1512           5 :       if (SelectAddrMode3Offset(N, LD->getOffset(), Offset, AMOpc)) {
    1513           5 :         Match = true;
    1514           5 :         Opcode = isPre ? ARM::LDRSB_PRE : ARM::LDRSB_POST;
    1515             :       }
    1516             :     } else {
    1517          14 :       if (isPre &&
    1518           7 :           SelectAddrMode2OffsetImmPre(N, LD->getOffset(), Offset, AMOpc)) {
    1519             :         Match = true;
    1520             :         Opcode = ARM::LDRB_PRE_IMM;
    1521           0 :       } else if (!isPre &&
    1522           0 :                   SelectAddrMode2OffsetImm(N, LD->getOffset(), Offset, AMOpc)) {
    1523             :         Match = true;
    1524             :         Opcode = ARM::LDRB_POST_IMM;
    1525           0 :       } else if (SelectAddrMode2OffsetReg(N, LD->getOffset(), Offset, AMOpc)) {
    1526           0 :         Match = true;
    1527           0 :         Opcode = isPre ? ARM::LDRB_PRE_REG : ARM::LDRB_POST_REG;
    1528             :       }
    1529             :     }
    1530             :   }
    1531             : 
    1532             :   if (Match) {
    1533          41 :     if (Opcode == ARM::LDR_PRE_IMM || Opcode == ARM::LDRB_PRE_IMM) {
    1534          18 :       SDValue Chain = LD->getChain();
    1535           9 :       SDValue Base = LD->getBasePtr();
    1536          18 :       SDValue Ops[]= { Base, AMOpc, getAL(CurDAG, SDLoc(N)),
    1537          36 :                        CurDAG->getRegister(0, MVT::i32), Chain };
    1538          27 :       SDNode *New = CurDAG->getMachineNode(Opcode, SDLoc(N), MVT::i32, MVT::i32,
    1539          45 :                                            MVT::Other, Ops);
    1540          18 :       transferMemOperands(N, New);
    1541          18 :       ReplaceNode(N, New);
    1542             :       return true;
    1543             :     } else {
    1544          64 :       SDValue Chain = LD->getChain();
    1545          32 :       SDValue Base = LD->getBasePtr();
    1546          64 :       SDValue Ops[]= { Base, Offset, AMOpc, getAL(CurDAG, SDLoc(N)),
    1547         128 :                        CurDAG->getRegister(0, MVT::i32), Chain };
    1548          96 :       SDNode *New = CurDAG->getMachineNode(Opcode, SDLoc(N), MVT::i32, MVT::i32,
    1549         160 :                                            MVT::Other, Ops);
    1550          64 :       transferMemOperands(N, New);
    1551          64 :       ReplaceNode(N, New);
    1552             :       return true;
    1553             :     }
    1554             :   }
    1555             : 
    1556             :   return false;
    1557             : }
    1558             : 
    1559        1518 : bool ARMDAGToDAGISel::tryT1IndexedLoad(SDNode *N) {
    1560        1518 :   LoadSDNode *LD = cast<LoadSDNode>(N);
    1561        1518 :   EVT LoadedVT = LD->getMemoryVT();
    1562        3036 :   ISD::MemIndexedMode AM = LD->getAddressingMode();
    1563        1522 :   if (AM != ISD::POST_INC || LD->getExtensionType() != ISD::NON_EXTLOAD ||
    1564           2 :       LoadedVT.getSimpleVT().SimpleTy != MVT::i32)
    1565             :     return false;
    1566             : 
    1567           4 :   auto *COffs = dyn_cast<ConstantSDNode>(LD->getOffset());
    1568           2 :   if (!COffs || COffs->getZExtValue() != 4)
    1569             :     return false;
    1570             : 
    1571             :   // A T1 post-indexed load is just a single register LDM: LDM r0!, {r1}.
    1572             :   // The encoding of LDM is not how the rest of ISel expects a post-inc load to
    1573             :   // look however, so we use a pseudo here and switch it for a tLDMIA_UPD after
    1574             :   // ISel.
    1575           4 :   SDValue Chain = LD->getChain();
    1576           2 :   SDValue Base = LD->getBasePtr();
    1577           4 :   SDValue Ops[]= { Base, getAL(CurDAG, SDLoc(N)),
    1578           8 :                    CurDAG->getRegister(0, MVT::i32), Chain };
    1579           6 :   SDNode *New = CurDAG->getMachineNode(ARM::tLDR_postidx, SDLoc(N), MVT::i32,
    1580          10 :                                        MVT::i32, MVT::Other, Ops);
    1581           4 :   transferMemOperands(N, New);
    1582           4 :   ReplaceNode(N, New);
    1583           2 :   return true;
    1584             : }
    1585             : 
    1586        3527 : bool ARMDAGToDAGISel::tryT2IndexedLoad(SDNode *N) {
    1587        3527 :   LoadSDNode *LD = cast<LoadSDNode>(N);
    1588        7054 :   ISD::MemIndexedMode AM = LD->getAddressingMode();
    1589        3527 :   if (AM == ISD::UNINDEXED)
    1590             :     return false;
    1591             : 
    1592          23 :   EVT LoadedVT = LD->getMemoryVT();
    1593          23 :   bool isSExtLd = LD->getExtensionType() == ISD::SEXTLOAD;
    1594          23 :   SDValue Offset;
    1595          23 :   bool isPre = (AM == ISD::PRE_INC) || (AM == ISD::PRE_DEC);
    1596          23 :   unsigned Opcode = 0;
    1597          23 :   bool Match = false;
    1598          23 :   if (SelectT2AddrModeImm8Offset(N, LD->getOffset(), Offset)) {
    1599          23 :     switch (LoadedVT.getSimpleVT().SimpleTy) {
    1600          16 :     case MVT::i32:
    1601          16 :       Opcode = isPre ? ARM::t2LDR_PRE : ARM::t2LDR_POST;
    1602             :       break;
    1603           0 :     case MVT::i16:
    1604           0 :       if (isSExtLd)
    1605           0 :         Opcode = isPre ? ARM::t2LDRSH_PRE : ARM::t2LDRSH_POST;
    1606             :       else
    1607           0 :         Opcode = isPre ? ARM::t2LDRH_PRE : ARM::t2LDRH_POST;
    1608             :       break;
    1609           7 :     case MVT::i8:
    1610             :     case MVT::i1:
    1611           7 :       if (isSExtLd)
    1612           2 :         Opcode = isPre ? ARM::t2LDRSB_PRE : ARM::t2LDRSB_POST;
    1613             :       else
    1614           5 :         Opcode = isPre ? ARM::t2LDRB_PRE : ARM::t2LDRB_POST;
    1615             :       break;
    1616             :     default:
    1617             :       return false;
    1618             :     }
    1619          23 :     Match = true;
    1620             :   }
    1621             : 
    1622             :   if (Match) {
    1623          46 :     SDValue Chain = LD->getChain();
    1624          23 :     SDValue Base = LD->getBasePtr();
    1625          46 :     SDValue Ops[]= { Base, Offset, getAL(CurDAG, SDLoc(N)),
    1626          92 :                      CurDAG->getRegister(0, MVT::i32), Chain };
    1627          69 :     SDNode *New = CurDAG->getMachineNode(Opcode, SDLoc(N), MVT::i32, MVT::i32,
    1628         115 :                                          MVT::Other, Ops);
    1629          46 :     transferMemOperands(N, New);
    1630          46 :     ReplaceNode(N, New);
    1631             :     return true;
    1632             :   }
    1633             : 
    1634             :   return false;
    1635             : }
    1636             : 
    1637             : /// \brief Form a GPRPair pseudo register from a pair of GPR regs.
    1638         109 : SDNode *ARMDAGToDAGISel::createGPRPairNode(EVT VT, SDValue V0, SDValue V1) {
    1639         327 :   SDLoc dl(V0.getNode());
    1640             :   SDValue RegClass =
    1641         327 :     CurDAG->getTargetConstant(ARM::GPRPairRegClassID, dl, MVT::i32);
    1642         327 :   SDValue SubReg0 = CurDAG->getTargetConstant(ARM::gsub_0, dl, MVT::i32);
    1643         327 :   SDValue SubReg1 = CurDAG->getTargetConstant(ARM::gsub_1, dl, MVT::i32);
    1644         109 :   const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 };
    1645         327 :   return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
    1646             : }
    1647             : 
    1648             : /// \brief Form a D register from a pair of S registers.
    1649          38 : SDNode *ARMDAGToDAGISel::createSRegPairNode(EVT VT, SDValue V0, SDValue V1) {
    1650         114 :   SDLoc dl(V0.getNode());
    1651             :   SDValue RegClass =
    1652         114 :     CurDAG->getTargetConstant(ARM::DPR_VFP2RegClassID, dl, MVT::i32);
    1653         114 :   SDValue SubReg0 = CurDAG->getTargetConstant(ARM::ssub_0, dl, MVT::i32);
    1654         114 :   SDValue SubReg1 = CurDAG->getTargetConstant(ARM::ssub_1, dl, MVT::i32);
    1655          38 :   const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 };
    1656         114 :   return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
    1657             : }
    1658             : 
    1659             : /// \brief Form a quad register from a pair of D registers.
    1660         332 : SDNode *ARMDAGToDAGISel::createDRegPairNode(EVT VT, SDValue V0, SDValue V1) {
    1661         996 :   SDLoc dl(V0.getNode());
    1662         332 :   SDValue RegClass = CurDAG->getTargetConstant(ARM::QPRRegClassID, dl,
    1663         664 :                                                MVT::i32);
    1664         996 :   SDValue SubReg0 = CurDAG->getTargetConstant(ARM::dsub_0, dl, MVT::i32);
    1665         996 :   SDValue SubReg1 = CurDAG->getTargetConstant(ARM::dsub_1, dl, MVT::i32);
    1666         332 :   const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 };
    1667         996 :   return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
    1668             : }
    1669             : 
    1670             : /// \brief Form 4 consecutive D registers from a pair of Q registers.
    1671          26 : SDNode *ARMDAGToDAGISel::createQRegPairNode(EVT VT, SDValue V0, SDValue V1) {
    1672          78 :   SDLoc dl(V0.getNode());
    1673          26 :   SDValue RegClass = CurDAG->getTargetConstant(ARM::QQPRRegClassID, dl,
    1674          52 :                                                MVT::i32);
    1675          78 :   SDValue SubReg0 = CurDAG->getTargetConstant(ARM::qsub_0, dl, MVT::i32);
    1676          78 :   SDValue SubReg1 = CurDAG->getTargetConstant(ARM::qsub_1, dl, MVT::i32);
    1677          26 :   const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 };
    1678          78 :   return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
    1679             : }
    1680             : 
    1681             : /// \brief Form 4 consecutive S registers.
    1682          68 : SDNode *ARMDAGToDAGISel::createQuadSRegsNode(EVT VT, SDValue V0, SDValue V1,
    1683             :                                    SDValue V2, SDValue V3) {
    1684         204 :   SDLoc dl(V0.getNode());
    1685             :   SDValue RegClass =
    1686         204 :     CurDAG->getTargetConstant(ARM::QPR_VFP2RegClassID, dl, MVT::i32);
    1687         204 :   SDValue SubReg0 = CurDAG->getTargetConstant(ARM::ssub_0, dl, MVT::i32);
    1688         204 :   SDValue SubReg1 = CurDAG->getTargetConstant(ARM::ssub_1, dl, MVT::i32);
    1689         204 :   SDValue SubReg2 = CurDAG->getTargetConstant(ARM::ssub_2, dl, MVT::i32);
    1690         204 :   SDValue SubReg3 = CurDAG->getTargetConstant(ARM::ssub_3, dl, MVT::i32);
    1691             :   const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1,
    1692          68 :                                     V2, SubReg2, V3, SubReg3 };
    1693         204 :   return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
    1694             : }
    1695             : 
    1696             : /// \brief Form 4 consecutive D registers.
    1697          57 : SDNode *ARMDAGToDAGISel::createQuadDRegsNode(EVT VT, SDValue V0, SDValue V1,
    1698             :                                    SDValue V2, SDValue V3) {
    1699         171 :   SDLoc dl(V0.getNode());
    1700          57 :   SDValue RegClass = CurDAG->getTargetConstant(ARM::QQPRRegClassID, dl,
    1701         114 :                                                MVT::i32);
    1702         171 :   SDValue SubReg0 = CurDAG->getTargetConstant(ARM::dsub_0, dl, MVT::i32);
    1703         171 :   SDValue SubReg1 = CurDAG->getTargetConstant(ARM::dsub_1, dl, MVT::i32);
    1704         171 :   SDValue SubReg2 = CurDAG->getTargetConstant(ARM::dsub_2, dl, MVT::i32);
    1705         171 :   SDValue SubReg3 = CurDAG->getTargetConstant(ARM::dsub_3, dl, MVT::i32);
    1706             :   const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1,
    1707          57 :                                     V2, SubReg2, V3, SubReg3 };
    1708         171 :   return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
    1709             : }
    1710             : 
    1711             : /// \brief Form 4 consecutive Q registers.
    1712          42 : SDNode *ARMDAGToDAGISel::createQuadQRegsNode(EVT VT, SDValue V0, SDValue V1,
    1713             :                                    SDValue V2, SDValue V3) {
    1714         126 :   SDLoc dl(V0.getNode());
    1715          42 :   SDValue RegClass = CurDAG->getTargetConstant(ARM::QQQQPRRegClassID, dl,
    1716          84 :                                                MVT::i32);
    1717         126 :   SDValue SubReg0 = CurDAG->getTargetConstant(ARM::qsub_0, dl, MVT::i32);
    1718         126 :   SDValue SubReg1 = CurDAG->getTargetConstant(ARM::qsub_1, dl, MVT::i32);
    1719         126 :   SDValue SubReg2 = CurDAG->getTargetConstant(ARM::qsub_2, dl, MVT::i32);
    1720         126 :   SDValue SubReg3 = CurDAG->getTargetConstant(ARM::qsub_3, dl, MVT::i32);
    1721             :   const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1,
    1722          42 :                                     V2, SubReg2, V3, SubReg3 };
    1723         126 :   return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
    1724             : }
    1725             : 
    1726             : /// GetVLDSTAlign - Get the alignment (in bytes) for the alignment operand
    1727             : /// of a NEON VLD or VST instruction.  The supported values depend on the
    1728             : /// number of registers being loaded.
    1729         445 : SDValue ARMDAGToDAGISel::GetVLDSTAlign(SDValue Align, const SDLoc &dl,
    1730             :                                        unsigned NumVecs, bool is64BitVector) {
    1731         445 :   unsigned NumRegs = NumVecs;
    1732         445 :   if (!is64BitVector && NumVecs < 3)
    1733         250 :     NumRegs *= 2;
    1734             : 
    1735         890 :   unsigned Alignment = cast<ConstantSDNode>(Align)->getZExtValue();
    1736         445 :   if (Alignment >= 32 && NumRegs == 4)
    1737             :     Alignment = 32;
    1738         436 :   else if (Alignment >= 16 && (NumRegs == 2 || NumRegs == 4))
    1739             :     Alignment = 16;
    1740         344 :   else if (Alignment >= 8)
    1741             :     Alignment = 8;
    1742             :   else
    1743         300 :     Alignment = 0;
    1744             : 
    1745        1335 :   return CurDAG->getTargetConstant(Alignment, dl, MVT::i32);
    1746             : }
    1747             : 
    1748             : static bool isVLDfixed(unsigned Opc)
    1749             : {
    1750             :   switch (Opc) {
    1751             :   default: return false;
    1752             :   case ARM::VLD1d8wb_fixed : return true;
    1753             :   case ARM::VLD1d16wb_fixed : return true;
    1754             :   case ARM::VLD1d64Qwb_fixed : return true;
    1755             :   case ARM::VLD1d32wb_fixed : return true;
    1756             :   case ARM::VLD1d64wb_fixed : return true;
    1757             :   case ARM::VLD1d64TPseudoWB_fixed : return true;
    1758             :   case ARM::VLD1d64QPseudoWB_fixed : return true;
    1759             :   case ARM::VLD1q8wb_fixed : return true;
    1760             :   case ARM::VLD1q16wb_fixed : return true;
    1761             :   case ARM::VLD1q32wb_fixed : return true;
    1762             :   case ARM::VLD1q64wb_fixed : return true;
    1763             :   case ARM::VLD1DUPd8wb_fixed : return true;
    1764             :   case ARM::VLD1DUPd16wb_fixed : return true;
    1765             :   case ARM::VLD1DUPd32wb_fixed : return true;
    1766             :   case ARM::VLD1DUPq8wb_fixed : return true;
    1767             :   case ARM::VLD1DUPq16wb_fixed : return true;
    1768             :   case ARM::VLD1DUPq32wb_fixed : return true;
    1769             :   case ARM::VLD2d8wb_fixed : return true;
    1770             :   case ARM::VLD2d16wb_fixed : return true;
    1771             :   case ARM::VLD2d32wb_fixed : return true;
    1772             :   case ARM::VLD2q8PseudoWB_fixed : return true;
    1773             :   case ARM::VLD2q16PseudoWB_fixed : return true;
    1774             :   case ARM::VLD2q32PseudoWB_fixed : return true;
    1775             :   case ARM::VLD2DUPd8wb_fixed : return true;
    1776             :   case ARM::VLD2DUPd16wb_fixed : return true;
    1777             :   case ARM::VLD2DUPd32wb_fixed : return true;
    1778             :   }
    1779             : }
    1780             : 
    1781             : static bool isVSTfixed(unsigned Opc)
    1782             : {
    1783             :   switch (Opc) {
    1784             :   default: return false;
    1785             :   case ARM::VST1d8wb_fixed : return true;
    1786             :   case ARM::VST1d16wb_fixed : return true;
    1787             :   case ARM::VST1d32wb_fixed : return true;
    1788             :   case ARM::VST1d64wb_fixed : return true;
    1789             :   case ARM::VST1q8wb_fixed : return true;
    1790             :   case ARM::VST1q16wb_fixed : return true;
    1791             :   case ARM::VST1q32wb_fixed : return true;
    1792             :   case ARM::VST1q64wb_fixed : return true;
    1793             :   case ARM::VST1d64TPseudoWB_fixed : return true;
    1794             :   case ARM::VST1d64QPseudoWB_fixed : return true;
    1795             :   case ARM::VST2d8wb_fixed : return true;
    1796             :   case ARM::VST2d16wb_fixed : return true;
    1797             :   case ARM::VST2d32wb_fixed : return true;
    1798             :   case ARM::VST2q8PseudoWB_fixed : return true;
    1799             :   case ARM::VST2q16PseudoWB_fixed : return true;
    1800             :   case ARM::VST2q32PseudoWB_fixed : return true;
    1801             :   }
    1802             : }
    1803             : 
    1804             : // Get the register stride update opcode of a VLD/VST instruction that
    1805             : // is otherwise equivalent to the given fixed stride updating instruction.
    1806          81 : static unsigned getVLDSTRegisterUpdateOpcode(unsigned Opc) {
    1807             :   assert((isVLDfixed(Opc) || isVSTfixed(Opc))
    1808             :     && "Incorrect fixed stride updating instruction.");
    1809          81 :   switch (Opc) {
    1810             :   default: break;
    1811             :   case ARM::VLD1d8wb_fixed: return ARM::VLD1d8wb_register;
    1812           0 :   case ARM::VLD1d16wb_fixed: return ARM::VLD1d16wb_register;
    1813           2 :   case ARM::VLD1d32wb_fixed: return ARM::VLD1d32wb_register;
    1814           8 :   case ARM::VLD1d64wb_fixed: return ARM::VLD1d64wb_register;
    1815           6 :   case ARM::VLD1q8wb_fixed: return ARM::VLD1q8wb_register;
    1816           0 :   case ARM::VLD1q16wb_fixed: return ARM::VLD1q16wb_register;
    1817           7 :   case ARM::VLD1q32wb_fixed: return ARM::VLD1q32wb_register;
    1818           7 :   case ARM::VLD1q64wb_fixed: return ARM::VLD1q64wb_register;
    1819           0 :   case ARM::VLD1d64Twb_fixed: return ARM::VLD1d64Twb_register;
    1820           0 :   case ARM::VLD1d64Qwb_fixed: return ARM::VLD1d64Qwb_register;
    1821           0 :   case ARM::VLD1d64TPseudoWB_fixed: return ARM::VLD1d64TPseudoWB_register;
    1822           0 :   case ARM::VLD1d64QPseudoWB_fixed: return ARM::VLD1d64QPseudoWB_register;
    1823           1 :   case ARM::VLD1DUPd8wb_fixed : return ARM::VLD1DUPd8wb_register;
    1824           0 :   case ARM::VLD1DUPd16wb_fixed : return ARM::VLD1DUPd16wb_register;
    1825           0 :   case ARM::VLD1DUPd32wb_fixed : return ARM::VLD1DUPd32wb_register;
    1826           1 :   case ARM::VLD1DUPq8wb_fixed : return ARM::VLD1DUPq8wb_register;
    1827           0 :   case ARM::VLD1DUPq16wb_fixed : return ARM::VLD1DUPq16wb_register;
    1828           0 :   case ARM::VLD1DUPq32wb_fixed : return ARM::VLD1DUPq32wb_register;
    1829             : 
    1830           6 :   case ARM::VST1d8wb_fixed: return ARM::VST1d8wb_register;
    1831           0 :   case ARM::VST1d16wb_fixed: return ARM::VST1d16wb_register;
    1832           4 :   case ARM::VST1d32wb_fixed: return ARM::VST1d32wb_register;
    1833           0 :   case ARM::VST1d64wb_fixed: return ARM::VST1d64wb_register;
    1834           5 :   case ARM::VST1q8wb_fixed: return ARM::VST1q8wb_register;
    1835           1 :   case ARM::VST1q16wb_fixed: return ARM::VST1q16wb_register;
    1836           5 :   case ARM::VST1q32wb_fixed: return ARM::VST1q32wb_register;
    1837          11 :   case ARM::VST1q64wb_fixed: return ARM::VST1q64wb_register;
    1838           0 :   case ARM::VST1d64TPseudoWB_fixed: return ARM::VST1d64TPseudoWB_register;
    1839           0 :   case ARM::VST1d64QPseudoWB_fixed: return ARM::VST1d64QPseudoWB_register;
    1840             : 
    1841           0 :   case ARM::VLD2d8wb_fixed: return ARM::VLD2d8wb_register;
    1842           0 :   case ARM::VLD2d16wb_fixed: return ARM::VLD2d16wb_register;
    1843           0 :   case ARM::VLD2d32wb_fixed: return ARM::VLD2d32wb_register;
    1844           1 :   case ARM::VLD2q8PseudoWB_fixed: return ARM::VLD2q8PseudoWB_register;
    1845           0 :   case ARM::VLD2q16PseudoWB_fixed: return ARM::VLD2q16PseudoWB_register;
    1846           0 :   case ARM::VLD2q32PseudoWB_fixed: return ARM::VLD2q32PseudoWB_register;
    1847             : 
    1848           1 :   case ARM::VST2d8wb_fixed: return ARM::VST2d8wb_register;
    1849           0 :   case ARM::VST2d16wb_fixed: return ARM::VST2d16wb_register;
    1850           0 :   case ARM::VST2d32wb_fixed: return ARM::VST2d32wb_register;
    1851           0 :   case ARM::VST2q8PseudoWB_fixed: return ARM::VST2q8PseudoWB_register;
    1852           0 :   case ARM::VST2q16PseudoWB_fixed: return ARM::VST2q16PseudoWB_register;
    1853           0 :   case ARM::VST2q32PseudoWB_fixed: return ARM::VST2q32PseudoWB_register;
    1854             : 
    1855           1 :   case ARM::VLD2DUPd8wb_fixed: return ARM::VLD2DUPd8wb_register;
    1856           1 :   case ARM::VLD2DUPd16wb_fixed: return ARM::VLD2DUPd16wb_register;
    1857           0 :   case ARM::VLD2DUPd32wb_fixed: return ARM::VLD2DUPd32wb_register;
    1858             :   }
    1859           0 :   return Opc; // If not one we handle, return it unchanged.
    1860             : }
    1861             : 
    1862             : /// Returns true if the given increment is a Constant known to be equal to the
    1863             : /// access size performed by a NEON load/store. This means the "[rN]!" form can
    1864             : /// be used.
    1865         235 : static bool isPerfectIncrement(SDValue Inc, EVT VecTy, unsigned NumVecs) {
    1866         193 :   auto C = dyn_cast<ConstantSDNode>(Inc);
    1867         193 :   return C && C->getZExtValue() == VecTy.getSizeInBits() / 8 * NumVecs;
    1868             : }
    1869             : 
    1870         249 : void ARMDAGToDAGISel::SelectVLD(SDNode *N, bool isUpdating, unsigned NumVecs,
    1871             :                                 const uint16_t *DOpcodes,
    1872             :                                 const uint16_t *QOpcodes0,
    1873             :                                 const uint16_t *QOpcodes1) {
    1874             :   assert(NumVecs >= 1 && NumVecs <= 4 && "VLD NumVecs out-of-range");
    1875         330 :   SDLoc dl(N);
    1876             : 
    1877         249 :   SDValue MemAddr, Align;
    1878         249 :   unsigned AddrOpIdx = isUpdating ? 1 : 2;
    1879         498 :   if (!SelectAddrMode6(N, N->getOperand(AddrOpIdx), MemAddr, Align))
    1880         168 :     return;
    1881             : 
    1882         498 :   SDValue Chain = N->getOperand(0);
    1883         498 :   EVT VT = N->getValueType(0);
    1884         249 :   bool is64BitVector = VT.is64BitVector();
    1885         249 :   Align = GetVLDSTAlign(Align, dl, NumVecs, is64BitVector);
    1886             : 
    1887             :   unsigned OpcodeIndex;
    1888         249 :   switch (VT.getSimpleVT().SimpleTy) {
    1889           0 :   default: llvm_unreachable("unhandled vld type");
    1890             :     // Double-register operations:
    1891             :   case MVT::v8i8:  OpcodeIndex = 0; break;
    1892          10 :   case MVT::v4i16: OpcodeIndex = 1; break;
    1893          22 :   case MVT::v2f32:
    1894          22 :   case MVT::v2i32: OpcodeIndex = 2; break;
    1895          17 :   case MVT::v1i64: OpcodeIndex = 3; break;
    1896             :     // Quad-register operations:
    1897             :   case MVT::v16i8: OpcodeIndex = 0; break;
    1898          17 :   case MVT::v8i16: OpcodeIndex = 1; break;
    1899          84 :   case MVT::v4f32:
    1900          84 :   case MVT::v4i32: OpcodeIndex = 2; break;
    1901          26 :   case MVT::v2f64:
    1902          26 :   case MVT::v2i64: OpcodeIndex = 3;
    1903             :     assert(NumVecs == 1 && "v2i64 type only supported for VLD1");
    1904          26 :     break;
    1905             :   }
    1906             : 
    1907         249 :   EVT ResTy;
    1908         249 :   if (NumVecs == 1)
    1909         168 :     ResTy = VT;
    1910             :   else {
    1911          81 :     unsigned ResTyElts = (NumVecs == 3) ? 4 : NumVecs;
    1912          81 :     if (!is64BitVector)
    1913          36 :       ResTyElts *= 2;
    1914          81 :     ResTy = EVT::getVectorVT(*CurDAG->getContext(), MVT::i64, ResTyElts);
    1915             :   }
    1916         330 :   std::vector<EVT> ResTys;
    1917         249 :   ResTys.push_back(ResTy);
    1918         249 :   if (isUpdating)
    1919         234 :     ResTys.push_back(MVT::i32);
    1920         498 :   ResTys.push_back(MVT::Other);
    1921             : 
    1922         249 :   SDValue Pred = getAL(CurDAG, dl);
    1923         498 :   SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
    1924             :   SDNode *VLd;
    1925         330 :   SmallVector<SDValue, 7> Ops;
    1926             : 
    1927             :   // Double registers and VLD1/VLD2 quad registers are directly supported.
    1928         249 :   if (is64BitVector || NumVecs <= 2) {
    1929         369 :     unsigned Opc = (is64BitVector ? DOpcodes[OpcodeIndex] :
    1930         140 :                     QOpcodes0[OpcodeIndex]);
    1931         229 :     Ops.push_back(MemAddr);
    1932         229 :     Ops.push_back(Align);
    1933         229 :     if (isUpdating) {
    1934         226 :       SDValue Inc = N->getOperand(AddrOpIdx + 1);
    1935             :       // FIXME: VLD1/VLD2 fixed increment doesn't need Reg0. Remove the reg0
    1936             :       // case entirely when the rest are updated to that form, too.
    1937         113 :       bool IsImmUpdate = isPerfectIncrement(Inc, VT, NumVecs);
    1938         113 :       if ((NumVecs <= 2) && !IsImmUpdate)
    1939          44 :         Opc = getVLDSTRegisterUpdateOpcode(Opc);
    1940             :       // FIXME: We use a VLD1 for v1i64 even if the pseudo says vld2/3/4, so
    1941             :       // check for that explicitly too. Horribly hacky, but temporary.
    1942         113 :       if ((NumVecs > 2 && !isVLDfixed(Opc)) || !IsImmUpdate)
    1943          47 :         Ops.push_back(IsImmUpdate ? Reg0 : Inc);
    1944             :     }
    1945         229 :     Ops.push_back(Pred);
    1946         229 :     Ops.push_back(Reg0);
    1947         229 :     Ops.push_back(Chain);
    1948         687 :     VLd = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
    1949             : 
    1950             :   } else {
    1951             :     // Otherwise, quad registers are loaded with two separate instructions,
    1952             :     // where one loads the even registers and the other loads the odd registers.
    1953          40 :     EVT AddrTy = MemAddr.getValueType();
    1954             : 
    1955             :     // Load the even subregs.  This is always an updating load, so that it
    1956             :     // provides the address to the second load for the odd subregs.
    1957             :     SDValue ImplDef =
    1958          40 :       SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, ResTy), 0);
    1959          20 :     const SDValue OpsA[] = { MemAddr, Align, Reg0, ImplDef, Pred, Reg0, Chain };
    1960          60 :     SDNode *VLdA = CurDAG->getMachineNode(QOpcodes0[OpcodeIndex], dl,
    1961          60 :                                           ResTy, AddrTy, MVT::Other, OpsA);
    1962          20 :     Chain = SDValue(VLdA, 2);
    1963             : 
    1964             :     // Load the odd subregs.
    1965          20 :     Ops.push_back(SDValue(VLdA, 1));
    1966          20 :     Ops.push_back(Align);
    1967          20 :     if (isUpdating) {
    1968           8 :       SDValue Inc = N->getOperand(AddrOpIdx + 1);
    1969             :       assert(isa<ConstantSDNode>(Inc.getNode()) &&
    1970             :              "only constant post-increment update allowed for VLD3/4");
    1971             :       (void)Inc;
    1972           4 :       Ops.push_back(Reg0);
    1973             :     }
    1974          20 :     Ops.push_back(SDValue(VLdA, 0));
    1975          20 :     Ops.push_back(Pred);
    1976          20 :     Ops.push_back(Reg0);
    1977          20 :     Ops.push_back(Chain);
    1978          60 :     VLd = CurDAG->getMachineNode(QOpcodes1[OpcodeIndex], dl, ResTys, Ops);
    1979             :   }
    1980             : 
    1981             :   // Transfer memoperands.
    1982         249 :   MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1);
    1983         249 :   MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand();
    1984         498 :   cast<MachineSDNode>(VLd)->setMemRefs(MemOp, MemOp + 1);
    1985             : 
    1986         249 :   if (NumVecs == 1) {
    1987         336 :     ReplaceNode(N, VLd);
    1988         168 :     return;
    1989             :   }
    1990             : 
    1991             :   // Extract out the subregisters.
    1992          81 :   SDValue SuperReg = SDValue(VLd, 0);
    1993             :   static_assert(ARM::dsub_7 == ARM::dsub_0 + 7 &&
    1994             :                     ARM::qsub_3 == ARM::qsub_0 + 3,
    1995             :                 "Unexpected subreg numbering");
    1996          81 :   unsigned Sub0 = (is64BitVector ? ARM::dsub_0 : ARM::qsub_0);
    1997         315 :   for (unsigned Vec = 0; Vec < NumVecs; ++Vec)
    1998         468 :     ReplaceUses(SDValue(N, Vec),
    1999         234 :                 CurDAG->getTargetExtractSubreg(Sub0 + Vec, dl, VT, SuperReg));
    2000          81 :   ReplaceUses(SDValue(N, NumVecs), SDValue(VLd, 1));
    2001          81 :   if (isUpdating)
    2002          17 :     ReplaceUses(SDValue(N, NumVecs + 1), SDValue(VLd, 2));
    2003          81 :   CurDAG->RemoveDeadNode(N);
    2004             : }
    2005             : 
    2006         196 : void ARMDAGToDAGISel::SelectVST(SDNode *N, bool isUpdating, unsigned NumVecs,
    2007             :                                 const uint16_t *DOpcodes,
    2008             :                                 const uint16_t *QOpcodes0,
    2009             :                                 const uint16_t *QOpcodes1) {
    2010             :   assert(NumVecs >= 1 && NumVecs <= 4 && "VST NumVecs out-of-range");
    2011         214 :   SDLoc dl(N);
    2012             : 
    2013         196 :   SDValue MemAddr, Align;
    2014         196 :   unsigned AddrOpIdx = isUpdating ? 1 : 2;
    2015         196 :   unsigned Vec0Idx = 3; // AddrOpIdx + (isUpdating ? 2 : 1)
    2016         392 :   if (!SelectAddrMode6(N, N->getOperand(AddrOpIdx), MemAddr, Align))
    2017         178 :     return;
    2018             : 
    2019         196 :   MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1);
    2020         196 :   MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand();
    2021             : 
    2022         392 :   SDValue Chain = N->getOperand(0);
    2023         588 :   EVT VT = N->getOperand(Vec0Idx).getValueType();
    2024         196 :   bool is64BitVector = VT.is64BitVector();
    2025         196 :   Align = GetVLDSTAlign(Align, dl, NumVecs, is64BitVector);
    2026             : 
    2027             :   unsigned OpcodeIndex;
    2028         196 :   switch (VT.getSimpleVT().SimpleTy) {
    2029           0 :   default: llvm_unreachable("unhandled vst type");
    2030             :     // Double-register operations:
    2031             :   case MVT::v8i8:  OpcodeIndex = 0; break;
    2032           6 :   case MVT::v4i16: OpcodeIndex = 1; break;
    2033          21 :   case MVT::v2f32:
    2034          21 :   case MVT::v2i32: OpcodeIndex = 2; break;
    2035           8 :   case MVT::v1i64: OpcodeIndex = 3; break;
    2036             :     // Quad-register operations:
    2037             :   case MVT::v16i8: OpcodeIndex = 0; break;
    2038          22 :   case MVT::v8i16: OpcodeIndex = 1; break;
    2039          62 :   case MVT::v4f32:
    2040          62 :   case MVT::v4i32: OpcodeIndex = 2; break;
    2041          21 :   case MVT::v2f64:
    2042          21 :   case MVT::v2i64: OpcodeIndex = 3;
    2043             :     assert(NumVecs == 1 && "v2i64 type only supported for VST1");
    2044          21 :     break;
    2045             :   }
    2046             : 
    2047         214 :   std::vector<EVT> ResTys;
    2048         196 :   if (isUpdating)
    2049         202 :     ResTys.push_back(MVT::i32);
    2050         392 :   ResTys.push_back(MVT::Other);
    2051             : 
    2052         196 :   SDValue Pred = getAL(CurDAG, dl);
    2053         392 :   SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
    2054         214 :   SmallVector<SDValue, 7> Ops;
    2055             : 
    2056             :   // Double registers and VST1/VST2 quad registers are directly supported.
    2057         196 :   if (is64BitVector || NumVecs <= 2) {
    2058         178 :     SDValue SrcReg;
    2059         178 :     if (NumVecs == 1) {
    2060         254 :       SrcReg = N->getOperand(Vec0Idx);
    2061          51 :     } else if (is64BitVector) {
    2062             :       // Form a REG_SEQUENCE to force register allocation.
    2063          72 :       SDValue V0 = N->getOperand(Vec0Idx + 0);
    2064          72 :       SDValue V1 = N->getOperand(Vec0Idx + 1);
    2065          36 :       if (NumVecs == 2)
    2066          10 :         SrcReg = SDValue(createDRegPairNode(MVT::v2i64, V0, V1), 0);
    2067             :       else {
    2068          52 :         SDValue V2 = N->getOperand(Vec0Idx + 2);
    2069             :         // If it's a vst3, form a quad D-register and leave the last part as
    2070             :         // an undef.
    2071             :         SDValue V3 = (NumVecs == 3)
    2072          17 :           ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF,dl,VT), 0)
    2073          44 :           : N->getOperand(Vec0Idx + 3);
    2074          26 :         SrcReg = SDValue(createQuadDRegsNode(MVT::v4i64, V0, V1, V2, V3), 0);
    2075             :       }
    2076             :     } else {
    2077             :       // Form a QQ register.
    2078          30 :       SDValue Q0 = N->getOperand(Vec0Idx);
    2079          30 :       SDValue Q1 = N->getOperand(Vec0Idx + 1);
    2080          15 :       SrcReg = SDValue(createQRegPairNode(MVT::v4i64, Q0, Q1), 0);
    2081             :     }
    2082             : 
    2083         288 :     unsigned Opc = (is64BitVector ? DOpcodes[OpcodeIndex] :
    2084         110 :                     QOpcodes0[OpcodeIndex]);
    2085         178 :     Ops.push_back(MemAddr);
    2086         178 :     Ops.push_back(Align);
    2087         178 :     if (isUpdating) {
    2088         198 :       SDValue Inc = N->getOperand(AddrOpIdx + 1);
    2089             :       // FIXME: VST1/VST2 fixed increment doesn't need Reg0. Remove the reg0
    2090             :       // case entirely when the rest are updated to that form, too.
    2091          99 :       bool IsImmUpdate = isPerfectIncrement(Inc, VT, NumVecs);
    2092          99 :       if (NumVecs <= 2 && !IsImmUpdate)
    2093          33 :         Opc = getVLDSTRegisterUpdateOpcode(Opc);
    2094             :       // FIXME: We use a VST1 for v1i64 even if the pseudo says vld2/3/4, so
    2095             :       // check for that explicitly too. Horribly hacky, but temporary.
    2096          99 :       if  (!IsImmUpdate)
    2097          34 :         Ops.push_back(Inc);
    2098          65 :       else if (NumVecs > 2 && !isVSTfixed(Opc))
    2099           1 :         Ops.push_back(Reg0);
    2100             :     }
    2101         178 :     Ops.push_back(SrcReg);
    2102         178 :     Ops.push_back(Pred);
    2103         178 :     Ops.push_back(Reg0);
    2104         178 :     Ops.push_back(Chain);
    2105         534 :     SDNode *VSt = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
    2106             : 
    2107             :     // Transfer memoperands.
    2108         356 :     cast<MachineSDNode>(VSt)->setMemRefs(MemOp, MemOp + 1);
    2109             : 
    2110         356 :     ReplaceNode(N, VSt);
    2111             :     return;
    2112             :   }
    2113             : 
    2114             :   // Otherwise, quad registers are stored with two separate instructions,
    2115             :   // where one stores the even registers and the other stores the odd registers.
    2116             : 
    2117             :   // Form the QQQQ REG_SEQUENCE.
    2118          36 :   SDValue V0 = N->getOperand(Vec0Idx + 0);
    2119          36 :   SDValue V1 = N->getOperand(Vec0Idx + 1);
    2120          36 :   SDValue V2 = N->getOperand(Vec0Idx + 2);
    2121             :   SDValue V3 = (NumVecs == 3)
    2122           5 :     ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, VT), 0)
    2123          44 :     : N->getOperand(Vec0Idx + 3);
    2124          36 :   SDValue RegSeq = SDValue(createQuadQRegsNode(MVT::v8i64, V0, V1, V2, V3), 0);
    2125             : 
    2126             :   // Store the even D registers.  This is always an updating store, so that it
    2127             :   // provides the address to the second store for the odd subregs.
    2128          18 :   const SDValue OpsA[] = { MemAddr, Align, Reg0, RegSeq, Pred, Reg0, Chain };
    2129          72 :   SDNode *VStA = CurDAG->getMachineNode(QOpcodes0[OpcodeIndex], dl,
    2130             :                                         MemAddr.getValueType(),
    2131          54 :                                         MVT::Other, OpsA);
    2132          36 :   cast<MachineSDNode>(VStA)->setMemRefs(MemOp, MemOp + 1);
    2133          18 :   Chain = SDValue(VStA, 1);
    2134             : 
    2135             :   // Store the odd D registers.
    2136          18 :   Ops.push_back(SDValue(VStA, 0));
    2137          18 :   Ops.push_back(Align);
    2138          18 :   if (isUpdating) {
    2139           4 :     SDValue Inc = N->getOperand(AddrOpIdx + 1);
    2140             :     assert(isa<ConstantSDNode>(Inc.getNode()) &&
    2141             :            "only constant post-increment update allowed for VST3/4");
    2142             :     (void)Inc;
    2143           2 :     Ops.push_back(Reg0);
    2144             :   }
    2145          18 :   Ops.push_back(RegSeq);
    2146          18 :   Ops.push_back(Pred);
    2147          18 :   Ops.push_back(Reg0);
    2148          18 :   Ops.push_back(Chain);
    2149          72 :   SDNode *VStB = CurDAG->getMachineNode(QOpcodes1[OpcodeIndex], dl, ResTys,
    2150          18 :                                         Ops);
    2151          36 :   cast<MachineSDNode>(VStB)->setMemRefs(MemOp, MemOp + 1);
    2152          36 :   ReplaceNode(N, VStB);
    2153             : }
    2154             : 
    2155          87 : void ARMDAGToDAGISel::SelectVLDSTLane(SDNode *N, bool IsLoad, bool isUpdating,
    2156             :                                       unsigned NumVecs,
    2157             :                                       const uint16_t *DOpcodes,
    2158             :                                       const uint16_t *QOpcodes) {
    2159             :   assert(NumVecs >=2 && NumVecs <= 4 && "VLDSTLane NumVecs out-of-range");
    2160         147 :   SDLoc dl(N);
    2161             : 
    2162          87 :   SDValue MemAddr, Align;
    2163          87 :   unsigned AddrOpIdx = isUpdating ? 1 : 2;
    2164          87 :   unsigned Vec0Idx = 3; // AddrOpIdx + (isUpdating ? 2 : 1)
    2165         174 :   if (!SelectAddrMode6(N, N->getOperand(AddrOpIdx), MemAddr, Align))
    2166          27 :     return;
    2167             : 
    2168          87 :   MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1);
    2169          87 :   MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand();
    2170             : 
    2171         174 :   SDValue Chain = N->getOperand(0);
    2172             :   unsigned Lane =
    2173         348 :     cast<ConstantSDNode>(N->getOperand(Vec0Idx + NumVecs))->getZExtValue();
    2174         261 :   EVT VT = N->getOperand(Vec0Idx).getValueType();
    2175          87 :   bool is64BitVector = VT.is64BitVector();
    2176             : 
    2177          87 :   unsigned Alignment = 0;
    2178          87 :   if (NumVecs != 3) {
    2179         116 :     Alignment = cast<ConstantSDNode>(Align)->getZExtValue();
    2180          58 :     unsigned NumBytes = NumVecs * VT.getScalarSizeInBits() / 8;
    2181          58 :     if (Alignment > NumBytes)
    2182          18 :       Alignment = NumBytes;
    2183          58 :     if (Alignment < 8 && Alignment < NumBytes)
    2184          37 :       Alignment = 0;
    2185             :     // Alignment must be a power of two; make sure of that.
    2186          58 :     Alignment = (Alignment & -Alignment);
    2187          58 :     if (Alignment == 1)
    2188           0 :       Alignment = 0;
    2189             :   }
    2190         261 :   Align = CurDAG->getTargetConstant(Alignment, dl, MVT::i32);
    2191             : 
    2192             :   unsigned OpcodeIndex;
    2193          87 :   switch (VT.getSimpleVT().SimpleTy) {
    2194           0 :   default: llvm_unreachable("unhandled vld/vst lane type");
    2195             :     // Double-register operations:
    2196             :   case MVT::v8i8:  OpcodeIndex = 0; break;
    2197          10 :   case MVT::v4i16: OpcodeIndex = 1; break;
    2198          28 :   case MVT::v2f32:
    2199          28 :   case MVT::v2i32: OpcodeIndex = 2; break;
    2200             :     // Quad-register operations:
    2201             :   case MVT::v8i16: OpcodeIndex = 0; break;
    2202          19 :   case MVT::v4f32:
    2203          19 :   case MVT::v4i32: OpcodeIndex = 1; break;
    2204             :   }
    2205             : 
    2206         147 :   std::vector<EVT> ResTys;
    2207          87 :   if (IsLoad) {
    2208          60 :     unsigned ResTyElts = (NumVecs == 3) ? 4 : NumVecs;
    2209          60 :     if (!is64BitVector)
    2210          25 :       ResTyElts *= 2;
    2211         120 :     ResTys.push_back(EVT::getVectorVT(*CurDAG->getContext(),
    2212         120 :                                       MVT::i64, ResTyElts));
    2213             :   }
    2214          87 :   if (isUpdating)
    2215          22 :     ResTys.push_back(MVT::i32);
    2216         174 :   ResTys.push_back(MVT::Other);
    2217             : 
    2218          87 :   SDValue Pred = getAL(CurDAG, dl);
    2219         174 :   SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
    2220             : 
    2221         147 :   SmallVector<SDValue, 8> Ops;
    2222          87 :   Ops.push_back(MemAddr);
    2223          87 :   Ops.push_back(Align);
    2224          87 :   if (isUpdating) {
    2225          22 :     SDValue Inc = N->getOperand(AddrOpIdx + 1);
    2226             :     bool IsImmUpdate =
    2227          11 :         isPerfectIncrement(Inc, VT.getVectorElementType(), NumVecs);
    2228          11 :     Ops.push_back(IsImmUpdate ? Reg0 : Inc);
    2229             :   }
    2230             : 
    2231          87 :   SDValue SuperReg;
    2232         174 :   SDValue V0 = N->getOperand(Vec0Idx + 0);
    2233         174 :   SDValue V1 = N->getOperand(Vec0Idx + 1);
    2234          87 :   if (NumVecs == 2) {
    2235          32 :     if (is64BitVector)
    2236          21 :       SuperReg = SDValue(createDRegPairNode(MVT::v2i64, V0, V1), 0);
    2237             :     else
    2238          11 :       SuperReg = SDValue(createQRegPairNode(MVT::v4i64, V0, V1), 0);
    2239             :   } else {
    2240         110 :     SDValue V2 = N->getOperand(Vec0Idx + 2);
    2241             :     SDValue V3 = (NumVecs == 3)
    2242          29 :       ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, VT), 0)
    2243         107 :       : N->getOperand(Vec0Idx + 3);
    2244          55 :     if (is64BitVector)
    2245          31 :       SuperReg = SDValue(createQuadDRegsNode(MVT::v4i64, V0, V1, V2, V3), 0);
    2246             :     else
    2247          24 :       SuperReg = SDValue(createQuadQRegsNode(MVT::v8i64, V0, V1, V2, V3), 0);
    2248             :   }
    2249          87 :   Ops.push_back(SuperReg);
    2250         174 :   Ops.push_back(getI32Imm(Lane, dl));
    2251          87 :   Ops.push_back(Pred);
    2252          87 :   Ops.push_back(Reg0);
    2253          87 :   Ops.push_back(Chain);
    2254             : 
    2255         122 :   unsigned Opc = (is64BitVector ? DOpcodes[OpcodeIndex] :
    2256          35 :                                   QOpcodes[OpcodeIndex]);
    2257         261 :   SDNode *VLdLn = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
    2258         174 :   cast<MachineSDNode>(VLdLn)->setMemRefs(MemOp, MemOp + 1);
    2259          87 :   if (!IsLoad) {
    2260          54 :     ReplaceNode(N, VLdLn);
    2261          27 :     return;
    2262             :   }
    2263             : 
    2264             :   // Extract the subregisters.
    2265          60 :   SuperReg = SDValue(VLdLn, 0);
    2266             :   static_assert(ARM::dsub_7 == ARM::dsub_0 + 7 &&
    2267             :                     ARM::qsub_3 == ARM::qsub_0 + 3,
    2268             :                 "Unexpected subreg numbering");
    2269          60 :   unsigned Sub0 = is64BitVector ? ARM::dsub_0 : ARM::qsub_0;
    2270         234 :   for (unsigned Vec = 0; Vec < NumVecs; ++Vec)
    2271         348 :     ReplaceUses(SDValue(N, Vec),
    2272         174 :                 CurDAG->getTargetExtractSubreg(Sub0 + Vec, dl, VT, SuperReg));
    2273          60 :   ReplaceUses(SDValue(N, NumVecs), SDValue(VLdLn, 1));
    2274          60 :   if (isUpdating)
    2275           8 :     ReplaceUses(SDValue(N, NumVecs + 1), SDValue(VLdLn, 2));
    2276          60 :   CurDAG->RemoveDeadNode(N);
    2277             : }
    2278             : 
    2279          46 : void ARMDAGToDAGISel::SelectVLDDup(SDNode *N, bool isUpdating, unsigned NumVecs,
    2280             :                                    const uint16_t *DOpcodes,
    2281             :                                    const uint16_t *QOpcodes) {
    2282             :   assert(NumVecs >= 1 && NumVecs <= 4 && "VLDDup NumVecs out-of-range");
    2283          92 :   SDLoc dl(N);
    2284             : 
    2285          46 :   SDValue MemAddr, Align;
    2286          92 :   if (!SelectAddrMode6(N, N->getOperand(1), MemAddr, Align))
    2287           0 :     return;
    2288             : 
    2289          46 :   MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1);
    2290          46 :   MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand();
    2291             : 
    2292          92 :   SDValue Chain = N->getOperand(0);
    2293          92 :   EVT VT = N->getValueType(0);
    2294             : 
    2295          46 :   unsigned Alignment = 0;
    2296          46 :   if (NumVecs != 3) {
    2297          88 :     Alignment = cast<ConstantSDNode>(Align)->getZExtValue();
    2298          44 :     unsigned NumBytes = NumVecs * VT.getScalarSizeInBits() / 8;
    2299          44 :     if (Alignment > NumBytes)
    2300           7 :       Alignment = NumBytes;
    2301          44 :     if (Alignment < 8 && Alignment < NumBytes)
    2302           9 :       Alignment = 0;
    2303             :     // Alignment must be a power of two; make sure of that.
    2304          44 :     Alignment = (Alignment & -Alignment);
    2305          44 :     if (Alignment == 1)
    2306          11 :       Alignment = 0;
    2307             :   }
    2308         138 :   Align = CurDAG->getTargetConstant(Alignment, dl, MVT::i32);
    2309             : 
    2310             :   unsigned Opc;
    2311          46 :   switch (VT.getSimpleVT().SimpleTy) {
    2312           0 :   default: llvm_unreachable("unhandled vld-dup type");
    2313          12 :   case MVT::v8i8:  Opc = DOpcodes[0]; break;
    2314           4 :   case MVT::v16i8: Opc = QOpcodes[0]; break;
    2315          13 :   case MVT::v4i16: Opc = DOpcodes[1]; break;
    2316           1 :   case MVT::v8i16: Opc = QOpcodes[1]; break;
    2317           9 :   case MVT::v2f32:
    2318           9 :   case MVT::v2i32: Opc = DOpcodes[2]; break;
    2319           7 :   case MVT::v4f32:
    2320           7 :   case MVT::v4i32: Opc = QOpcodes[2]; break;
    2321             :   }
    2322             : 
    2323          46 :   SDValue Pred = getAL(CurDAG, dl);
    2324          92 :   SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
    2325          92 :   SmallVector<SDValue, 6> Ops;
    2326          46 :   Ops.push_back(MemAddr);
    2327          46 :   Ops.push_back(Align);
    2328          46 :   if (isUpdating) {
    2329             :     // fixed-stride update instructions don't have an explicit writeback
    2330             :     // operand. It's implicit in the opcode itself.
    2331          24 :     SDValue Inc = N->getOperand(2);
    2332             :     bool IsImmUpdate =
    2333          12 :         isPerfectIncrement(Inc, VT.getVectorElementType(), NumVecs);
    2334          12 :     if (NumVecs <= 2 && !IsImmUpdate)
    2335           4 :       Opc = getVLDSTRegisterUpdateOpcode(Opc);
    2336          12 :     if (!IsImmUpdate)
    2337           5 :       Ops.push_back(Inc);
    2338             :     // FIXME: VLD3 and VLD4 haven't been updated to that form yet.
    2339           7 :     else if (NumVecs > 2)
    2340           1 :       Ops.push_back(Reg0);
    2341             :   }
    2342          46 :   Ops.push_back(Pred);
    2343          46 :   Ops.push_back(Reg0);
    2344          46 :   Ops.push_back(Chain);
    2345             : 
    2346          46 :   unsigned ResTyElts = (NumVecs == 3) ? 4 : NumVecs;
    2347          92 :   std::vector<EVT> ResTys;
    2348          92 :   ResTys.push_back(EVT::getVectorVT(*CurDAG->getContext(), MVT::i64,ResTyElts));
    2349          46 :   if (isUpdating)
    2350          24 :     ResTys.push_back(MVT::i32);
    2351          92 :   ResTys.push_back(MVT::Other);
    2352         138 :   SDNode *VLdDup = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
    2353          92 :   cast<MachineSDNode>(VLdDup)->setMemRefs(MemOp, MemOp + 1);
    2354             : 
    2355             :   // Extract the subregisters.
    2356          46 :   if (NumVecs == 1) {
    2357          34 :     ReplaceUses(SDValue(N, 0), SDValue(VLdDup, 0));
    2358             :   } else {
    2359          12 :     SDValue SuperReg = SDValue(VLdDup, 0);
    2360             :     static_assert(ARM::dsub_7 == ARM::dsub_0 + 7, "Unexpected subreg numbering");
    2361          12 :     unsigned SubIdx = ARM::dsub_0;
    2362          42 :     for (unsigned Vec = 0; Vec < NumVecs; ++Vec)
    2363          60 :       ReplaceUses(SDValue(N, Vec),
    2364          30 :                   CurDAG->getTargetExtractSubreg(SubIdx+Vec, dl, VT, SuperReg));
    2365             :   }
    2366          46 :   ReplaceUses(SDValue(N, NumVecs), SDValue(VLdDup, 1));
    2367          46 :   if (isUpdating)
    2368          12 :     ReplaceUses(SDValue(N, NumVecs + 1), SDValue(VLdDup, 2));
    2369          46 :   CurDAG->RemoveDeadNode(N);
    2370             : }
    2371             : 
    2372        1640 : bool ARMDAGToDAGISel::tryV6T2BitfieldExtractOp(SDNode *N, bool isSigned) {
    2373        1640 :   if (!Subtarget->hasV6T2Ops())
    2374             :     return false;
    2375             : 
    2376             :   unsigned Opc = isSigned
    2377        1308 :     ? (Subtarget->isThumb() ? ARM::t2SBFX : ARM::SBFX)
    2378        1308 :     : (Subtarget->isThumb() ? ARM::t2UBFX : ARM::UBFX);
    2379         740 :   SDLoc dl(N);
    2380             : 
    2381             :   // For unsigned extracts, check for a shift right and mask
    2382         740 :   unsigned And_imm = 0;
    2383        1480 :   if (N->getOpcode() == ISD::AND) {
    2384         324 :     if (isOpcWithIntImmediate(N, ISD::AND, And_imm)) {
    2385             : 
    2386             :       // The immediate is a mask of the low bits iff imm & (imm+1) == 0
    2387         324 :       if (And_imm & (And_imm + 1))
    2388             :         return false;
    2389             : 
    2390         204 :       unsigned Srl_imm = 0;
    2391         433 :       if (isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::SRL,
    2392             :                                 Srl_imm)) {
    2393             :         assert(Srl_imm > 0 && Srl_imm < 32 && "bad amount in shift node!");
    2394             : 
    2395             :         // Note: The width operand is encoded as width-1.
    2396          50 :         unsigned Width = countTrailingOnes(And_imm) - 1;
    2397          25 :         unsigned LSB = Srl_imm;
    2398             : 
    2399          50 :         SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
    2400             : 
    2401          50 :         if ((LSB + Width + 1) == N->getValueType(0).getSizeInBits()) {
    2402             :           // It's cheaper to use a right shift to extract the top bits.
    2403           1 :           if (Subtarget->isThumb()) {
    2404           0 :             Opc = isSigned ? ARM::t2ASRri : ARM::t2LSRri;
    2405           0 :             SDValue Ops[] = { N->getOperand(0).getOperand(0),
    2406           0 :                               CurDAG->getTargetConstant(LSB, dl, MVT::i32),
    2407           0 :                               getAL(CurDAG, dl), Reg0, Reg0 };
    2408           0 :             CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops);
    2409             :             return true;
    2410             :           }
    2411             : 
    2412             :           // ARM models shift instructions as MOVsi with shifter operand.
    2413           1 :           ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(ISD::SRL);
    2414             :           SDValue ShOpc =
    2415           2 :             CurDAG->getTargetConstant(ARM_AM::getSORegOpc(ShOpcVal, LSB), dl,
    2416           3 :                                       MVT::i32);
    2417           3 :           SDValue Ops[] = { N->getOperand(0).getOperand(0), ShOpc,
    2418           1 :                             getAL(CurDAG, dl), Reg0, Reg0 };
    2419           3 :           CurDAG->SelectNodeTo(N, ARM::MOVsi, MVT::i32, Ops);
    2420           1 :           return true;
    2421             :         }
    2422             : 
    2423          72 :         SDValue Ops[] = { N->getOperand(0).getOperand(0),
    2424          48 :                           CurDAG->getTargetConstant(LSB, dl, MVT::i32),
    2425          48 :                           CurDAG->getTargetConstant(Width, dl, MVT::i32),
    2426          72 :                           getAL(CurDAG, dl), Reg0 };
    2427          72 :         CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops);
    2428          24 :         return true;
    2429             :       }
    2430             :     }
    2431             :     return false;
    2432             :   }
    2433             : 
    2434             :   // Otherwise, we're looking for a shift of a shift
    2435         297 :   unsigned Shl_imm = 0;
    2436         606 :   if (isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::SHL, Shl_imm)) {
    2437             :     assert(Shl_imm > 0 && Shl_imm < 32 && "bad amount in shift node!");
    2438          12 :     unsigned Srl_imm = 0;
    2439          36 :     if (isInt32Immediate(N->getOperand(1), Srl_imm)) {
    2440             :       assert(Srl_imm > 0 && Srl_imm < 32 && "bad amount in shift node!");
    2441             :       // Note: The width operand is encoded as width-1.
    2442          12 :       unsigned Width = 32 - Srl_imm - 1;
    2443          12 :       int LSB = Srl_imm - Shl_imm;
    2444          12 :       if (LSB < 0)
    2445             :         return false;
    2446          22 :       SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
    2447          33 :       SDValue Ops[] = { N->getOperand(0).getOperand(0),
    2448          22 :                         CurDAG->getTargetConstant(LSB, dl, MVT::i32),
    2449          22 :                         CurDAG->getTargetConstant(Width, dl, MVT::i32),
    2450          33 :                         getAL(CurDAG, dl), Reg0 };
    2451          33 :       CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops);
    2452          11 :       return true;
    2453             :     }
    2454             :   }
    2455             : 
    2456             :   // Or we are looking for a shift of an and, with a mask operand
    2457         582 :   if (isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::AND, And_imm) &&
    2458          21 :       isShiftedMask_32(And_imm)) {
    2459           9 :     unsigned Srl_imm = 0;
    2460          18 :     unsigned LSB = countTrailingZeros(And_imm);
    2461             :     // Shift must be the same as the ands lsb
    2462          27 :     if (isInt32Immediate(N->getOperand(1), Srl_imm) && Srl_imm == LSB) {
    2463             :       assert(Srl_imm > 0 && Srl_imm < 32 && "bad amount in shift node!");
    2464          10 :       unsigned MSB = 31 - countLeadingZeros(And_imm);
    2465             :       // Note: The width operand is encoded as width-1.
    2466           5 :       unsigned Width = MSB - LSB;
    2467          10 :       SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
    2468          15 :       SDValue Ops[] = { N->getOperand(0).getOperand(0),
    2469          10 :                         CurDAG->getTargetConstant(Srl_imm, dl, MVT::i32),
    2470          10 :                         CurDAG->getTargetConstant(Width, dl, MVT::i32),
    2471          15 :                         getAL(CurDAG, dl), Reg0 };
    2472          15 :       CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops);
    2473             :       return true;
    2474             :     }
    2475             :   }
    2476             : 
    2477         280 :   if (N->getOpcode() == ISD::SIGN_EXTEND_INREG) {
    2478         354 :     unsigned Width = cast<VTSDNode>(N->getOperand(1))->getVT().getSizeInBits();
    2479         118 :     unsigned LSB = 0;
    2480         459 :     if (!isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::SRL, LSB) &&
    2481         315 :         !isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::SRA, LSB))
    2482             :       return false;
    2483             : 
    2484          13 :     if (LSB + Width > 32)
    2485             :       return false;
    2486             : 
    2487          26 :     SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
    2488          39 :     SDValue Ops[] = { N->getOperand(0).getOperand(0),
    2489          26 :                       CurDAG->getTargetConstant(LSB, dl, MVT::i32),
    2490          26 :                       CurDAG->getTargetConstant(Width - 1, dl, MVT::i32),
    2491          39 :                       getAL(CurDAG, dl), Reg0 };
    2492          39 :     CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops);
    2493          13 :     return true;
    2494             :   }
    2495             : 
    2496             :   return false;
    2497             : }
    2498             : 
    2499             : /// Target-specific DAG combining for ISD::XOR.
    2500             : /// Target-independent combining lowers SELECT_CC nodes of the form
    2501             : /// select_cc setg[ge] X,  0,  X, -X
    2502             : /// select_cc setgt    X, -1,  X, -X
    2503             : /// select_cc setl[te] X,  0, -X,  X
    2504             : /// select_cc setlt    X,  1, -X,  X
    2505             : /// which represent Integer ABS into:
    2506             : /// Y = sra (X, size(X)-1); xor (add (X, Y), Y)
    2507             : /// ARM instruction selection detects the latter and matches it to
    2508             : /// ARM::ABS or ARM::t2ABS machine node.
    2509         344 : bool ARMDAGToDAGISel::tryABSOp(SDNode *N){
    2510         688 :   SDValue XORSrc0 = N->getOperand(0);
    2511         688 :   SDValue XORSrc1 = N->getOperand(1);
    2512         688 :   EVT VT = N->getValueType(0);
    2513             : 
    2514         344 :   if (Subtarget->isThumb1Only())
    2515             :     return false;
    2516             : 
    2517         475 :   if (XORSrc0.getOpcode() != ISD::ADD || XORSrc1.getOpcode() != ISD::SRA)
    2518             :     return false;
    2519             : 
    2520          16 :   SDValue ADDSrc0 = XORSrc0.getOperand(0);
    2521          16 :   SDValue ADDSrc1 = XORSrc0.getOperand(1);
    2522          16 :   SDValue SRASrc0 = XORSrc1.getOperand(0);
    2523          16 :   SDValue SRASrc1 = XORSrc1.getOperand(1);
    2524           8 :   ConstantSDNode *SRAConstant =  dyn_cast<ConstantSDNode>(SRASrc1);
    2525          16 :   EVT XType = SRASrc0.getValueType();
    2526           8 :   unsigned Size = XType.getSizeInBits() - 1;
    2527             : 
    2528          24 :   if (ADDSrc1 == XORSrc1 && ADDSrc0 == SRASrc0 &&
    2529          24 :       XType.isInteger() && SRAConstant != nullptr &&
    2530           8 :       Size == SRAConstant->getZExtValue()) {
    2531          16 :     unsigned Opcode = Subtarget->isThumb2() ? ARM::t2ABS : ARM::ABS;
    2532           8 :     CurDAG->SelectNodeTo(N, Opcode, VT, ADDSrc0);
    2533             :     return true;
    2534             :   }
    2535             : 
    2536             :   return false;
    2537             : }
    2538             : 
    2539             : /// We've got special pseudo-instructions for these
    2540           6 : void ARMDAGToDAGISel::SelectCMP_SWAP(SDNode *N) {
    2541             :   unsigned Opcode;
    2542           6 :   EVT MemTy = cast<MemSDNode>(N)->getMemoryVT();
    2543          10 :   if (MemTy == MVT::i8)
    2544             :     Opcode = ARM::CMP_SWAP_8;
    2545           6 :   else if (MemTy == MVT::i16)
    2546             :     Opcode = ARM::CMP_SWAP_16;
    2547           2 :   else if (MemTy == MVT::i32)
    2548             :     Opcode = ARM::CMP_SWAP_32;
    2549             :   else
    2550           0 :     llvm_unreachable("Unknown AtomicCmpSwap type");
    2551             : 
    2552          36 :   SDValue Ops[] = {N->getOperand(1), N->getOperand(2), N->getOperand(3),
    2553          24 :                    N->getOperand(0)};
    2554          18 :   SDNode *CmpSwap = CurDAG->getMachineNode(
    2555          12 :       Opcode, SDLoc(N),
    2556          30 :       CurDAG->getVTList(MVT::i32, MVT::i32, MVT::Other), Ops);
    2557             : 
    2558           6 :   MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1);
    2559           6 :   MemOp[0] = cast<MemSDNode>(N)->getMemOperand();
    2560          12 :   cast<MachineSDNode>(CmpSwap)->setMemRefs(MemOp, MemOp + 1);
    2561             : 
    2562           6 :   ReplaceUses(SDValue(N, 0), SDValue(CmpSwap, 0));
    2563           6 :   ReplaceUses(SDValue(N, 1), SDValue(CmpSwap, 2));
    2564           6 :   CurDAG->RemoveDeadNode(N);
    2565           6 : }
    2566             : 
    2567             : static Optional<std::pair<unsigned, unsigned>>
    2568          68 : getContiguousRangeOfSetBits(const APInt &A) {
    2569          68 :   unsigned FirstOne = A.getBitWidth() - A.countLeadingZeros() - 1;
    2570          68 :   unsigned LastOne = A.countTrailingZeros();
    2571          68 :   if (A.countPopulation() != (FirstOne - LastOne + 1))
    2572             :     return Optional<std::pair<unsigned,unsigned>>();
    2573         180 :   return std::make_pair(FirstOne, LastOne);
    2574             : }
    2575             : 
    2576        1950 : void ARMDAGToDAGISel::SelectCMPZ(SDNode *N, bool &SwitchEQNEToPLMI) {
    2577             :   assert(N->getOpcode() == ARMISD::CMPZ);
    2578        1950 :   SwitchEQNEToPLMI = false;
    2579             : 
    2580        1950 :   if (!Subtarget->isThumb())
    2581             :     // FIXME: Work out whether it is profitable to do this in A32 mode - LSL and
    2582             :     // LSR don't exist as standalone instructions - they need the barrel shifter.
    2583        1890 :     return;
    2584             : 
    2585             :   // select (cmpz (and X, C), #0) -> (LSLS X) or (LSRS X) or (LSRS (LSLS X))
    2586        2302 :   SDValue And = N->getOperand(0);
    2587        1750 :   if (!And->hasOneUse())
    2588             :     return;
    2589             : 
    2590        1198 :   SDValue Zero = N->getOperand(1);
    2591        1518 :   if (!isa<ConstantSDNode>(Zero) || !cast<ConstantSDNode>(Zero)->isNullValue() ||
    2592         428 :       And->getOpcode() != ISD::AND)
    2593             :     return;
    2594         174 :   SDValue X = And.getOperand(0);
    2595         254 :   auto C = dyn_cast<ConstantSDNode>(And.getOperand(1));
    2596             : 
    2597         148 :   if (!C || !X->hasOneUse())
    2598             :     return;
    2599         128 :   auto Range = getContiguousRangeOfSetBits(C->getAPIntValue());
    2600          68 :   if (!Range)
    2601             :     return;
    2602             : 
    2603             :   // There are several ways to lower this:
    2604             :   SDNode *NewN;
    2605         120 :   SDLoc dl(N);
    2606             : 
    2607          60 :   auto EmitShift = [&](unsigned Opc, SDValue Src, unsigned Imm) -> SDNode* {
    2608         300 :     if (Subtarget->isThumb2()) {
    2609          96 :       Opc = (Opc == ARM::tLSLri) ? ARM::t2LSLri : ARM::t2LSRri;
    2610         228 :       SDValue Ops[] = { Src, CurDAG->getTargetConstant(Imm, dl, MVT::i32),
    2611         192 :                         getAL(CurDAG, dl), CurDAG->getRegister(0, MVT::i32),
    2612         288 :                         CurDAG->getRegister(0, MVT::i32) };
    2613         192 :       return CurDAG->getMachineNode(Opc, dl, MVT::i32, Ops);
    2614             :     } else {
    2615          36 :       SDValue Ops[] = {CurDAG->getRegister(ARM::CPSR, MVT::i32), Src,
    2616          36 :                        CurDAG->getTargetConstant(Imm, dl, MVT::i32),
    2617          72 :                        getAL(CurDAG, dl), CurDAG->getRegister(0, MVT::i32)};
    2618          48 :       return CurDAG->getMachineNode(Opc, dl, MVT::i32, Ops);
    2619             :     }
    2620          60 :   };
    2621             :   
    2622          60 :   if (Range->second == 0) {
    2623             :     //  1. Mask includes the LSB -> Simply shift the top N bits off
    2624          49 :     NewN = EmitShift(ARM::tLSLri, X, 31 - Range->first);
    2625          49 :     ReplaceNode(And.getNode(), NewN);
    2626          11 :   } else if (Range->first == 31) {
    2627             :     //  2. Mask includes the MSB -> Simply shift the bottom N bits off
    2628           2 :     NewN = EmitShift(ARM::tLSRri, X, Range->second);
    2629           2 :     ReplaceNode(And.getNode(), NewN);
    2630          18 :   } else if (Range->first == Range->second) {
    2631             :     //  3. Only one bit is set. We can shift this into the sign bit and use a
    2632             :     //     PL/MI comparison.
    2633           7 :     NewN = EmitShift(ARM::tLSLri, X, 31 - Range->first);
    2634          14 :     ReplaceNode(And.getNode(), NewN);
    2635             : 
    2636           7 :     SwitchEQNEToPLMI = true;
    2637           2 :   } else if (!Subtarget->hasV6T2Ops()) {
    2638             :     //  4. Do a double shift to clear bottom and top bits, but only in
    2639             :     //     thumb-1 mode as in thumb-2 we can use UBFX.
    2640           1 :     NewN = EmitShift(ARM::tLSLri, X, 31 - Range->first);
    2641           2 :     NewN = EmitShift(ARM::tLSRri, SDValue(NewN, 0),
    2642           2 :                      Range->second + (31 - Range->first));
    2643           1 :     ReplaceNode(And.getNode(), NewN);
    2644             :   }
    2645             : 
    2646             : }
    2647             : 
    2648      257802 : void ARMDAGToDAGISel::Select(SDNode *N) {
    2649      509652 :   SDLoc dl(N);
    2650             : 
    2651      257802 :   if (N->isMachineOpcode()) {
    2652          87 :     N->setNodeId(-1);
    2653        5925 :     return;   // Already selected.
    2654             :   }
    2655             : 
    2656      515430 :   switch (N->getOpcode()) {
    2657             :   default: break;
    2658         135 :   case ISD::WRITE_REGISTER:
    2659         135 :     if (tryWriteRegister(N))
    2660             :       return;
    2661             :     break;
    2662          95 :   case ISD::READ_REGISTER:
    2663          95 :     if (tryReadRegister(N))
    2664             :       return;
    2665             :     break;
    2666        3477 :   case ISD::INLINEASM:
    2667        3477 :     if (tryInlineAsm(N))
    2668             :       return;
    2669             :     break;
    2670         344 :   case ISD::XOR:
    2671             :     // Select special operations if XOR node forms integer ABS pattern
    2672         344 :     if (tryABSOp(N))
    2673             :       return;
    2674             :     // Other cases are autogenerated.
    2675             :     break;
    2676        4878 :   case ISD::Constant: {
    2677        9756 :     unsigned Val = cast<ConstantSDNode>(N)->getZExtValue();
    2678             :     // If we can't materialize the constant we need to use a literal pool
    2679        4878 :     if (ConstantMaterializationCost(Val) > 2) {
    2680         281 :       SDValue CPIdx = CurDAG->getTargetConstantPool(
    2681         281 :           ConstantInt::get(Type::getInt32Ty(*CurDAG->getContext()), Val),
    2682        1124 :           TLI->getPointerTy(CurDAG->getDataLayout()));
    2683             : 
    2684             :       SDNode *ResNode;
    2685         281 :       if (Subtarget->isThumb()) {
    2686             :         SDValue Ops[] = {
    2687             :           CPIdx,
    2688         166 :           getAL(CurDAG, dl),
    2689         498 :           CurDAG->getRegister(0, MVT::i32),
    2690         166 :           CurDAG->getEntryNode()
    2691         664 :         };
    2692         332 :         ResNode = CurDAG->getMachineNode(ARM::tLDRpci, dl, MVT::i32, MVT::Other,
    2693         498 :                                          Ops);
    2694             :       } else {
    2695             :         SDValue Ops[] = {
    2696             :           CPIdx,
    2697         230 :           CurDAG->getTargetConstant(0, dl, MVT::i32),
    2698         115 :           getAL(CurDAG, dl),
    2699         345 :           CurDAG->getRegister(0, MVT::i32),
    2700         115 :           CurDAG->getEntryNode()
    2701         575 :         };
    2702         230 :         ResNode = CurDAG->getMachineNode(ARM::LDRcp, dl, MVT::i32, MVT::Other,
    2703         345 :                                          Ops);
    2704             :       }
    2705             :       // Annotate the Node with memory operand information so that MachineInstr
    2706             :       // queries work properly. This e.g. gives the register allocation the
    2707             :       // required information for rematerialization.
    2708         281 :       MachineFunction& MF = CurDAG->getMachineFunction();
    2709         281 :       MachineSDNode::mmo_iterator MemOp = MF.allocateMemRefsArray(1);
    2710         281 :       MemOp[0] = MF.getMachineMemOperand(
    2711             :           MachinePointerInfo::getConstantPool(MF),
    2712             :           MachineMemOperand::MOLoad, 4, 4);
    2713             : 
    2714         562 :       cast<MachineSDNode>(ResNode)->setMemRefs(MemOp, MemOp+1);
    2715             :         
    2716         562 :       ReplaceNode(N, ResNode);
    2717             :       return;
    2718             :     }
    2719             : 
    2720             :     // Other cases are autogenerated.
    2721             :     break;
    2722             :   }
    2723        1989 :   case ISD::FrameIndex: {
    2724             :     // Selects to ADDri FI, 0 which in turn will become ADDri SP, imm.
    2725        1989 :     int FI = cast<FrameIndexSDNode>(N)->getIndex();
    2726        1989 :     SDValue TFI = CurDAG->getTargetFrameIndex(
    2727        7956 :         FI, TLI->getPointerTy(CurDAG->getDataLayout()));
    2728        1989 :     if (Subtarget->isThumb1Only()) {
    2729             :       // Set the alignment of the frame object to 4, to avoid having to generate
    2730             :       // more than one ADD
    2731         254 :       MachineFrameInfo &MFI = MF->getFrameInfo();
    2732         254 :       if (MFI.getObjectAlignment(FI) < 4)
    2733             :         MFI.setObjectAlignment(FI, 4);
    2734         762 :       CurDAG->SelectNodeTo(N, ARM::tADDframe, MVT::i32, TFI,
    2735         762 :                            CurDAG->getTargetConstant(0, dl, MVT::i32));
    2736         254 :       return;
    2737             :     } else {
    2738        2391 :       unsigned Opc = ((Subtarget->isThumb() && Subtarget->hasThumb2()) ?
    2739        1735 :                       ARM::t2ADDri : ARM::ADDri);
    2740        3470 :       SDValue Ops[] = { TFI, CurDAG->getTargetConstant(0, dl, MVT::i32),
    2741        6940 :                         getAL(CurDAG, dl), CurDAG->getRegister(0, MVT::i32),
    2742       10410 :                         CurDAG->getRegister(0, MVT::i32) };
    2743        5205 :       CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops);
    2744             :       return;
    2745             :     }
    2746             :   }
    2747         296 :   case ISD::SRL:
    2748         296 :     if (tryV6T2BitfieldExtractOp(N, false))
    2749             :       return;
    2750             :     break;
    2751         418 :   case ISD::SIGN_EXTEND_INREG:
    2752             :   case ISD::SRA:
    2753         418 :     if (tryV6T2BitfieldExtractOp(N, true))
    2754             :       return;
    2755             :     break;
    2756         286 :   case ISD::MUL:
    2757         286 :     if (Subtarget->isThumb1Only())
    2758             :       break;
    2759         563 :     if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(1))) {
    2760          73 :       unsigned RHSV = C->getZExtValue();
    2761          73 :       if (!RHSV) break;
    2762         146 :       if (isPowerOf2_32(RHSV-1)) {  // 2^n+1?
    2763           0 :         unsigned ShImm = Log2_32(RHSV-1);
    2764             :         if (ShImm >= 32)
    2765             :           break;
    2766           0 :         SDValue V = N->getOperand(0);
    2767           0 :         ShImm = ARM_AM::getSORegOpc(ARM_AM::lsl, ShImm);
    2768           0 :         SDValue ShImmOp = CurDAG->getTargetConstant(ShImm, dl, MVT::i32);
    2769           0 :         SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
    2770           0 :         if (Subtarget->isThumb()) {
    2771           0 :           SDValue Ops[] = { V, V, ShImmOp, getAL(CurDAG, dl), Reg0, Reg0 };
    2772           0 :           CurDAG->SelectNodeTo(N, ARM::t2ADDrs, MVT::i32, Ops);
    2773             :           return;
    2774             :         } else {
    2775           0 :           SDValue Ops[] = { V, V, Reg0, ShImmOp, getAL(CurDAG, dl), Reg0,
    2776           0 :                             Reg0 };
    2777           0 :           CurDAG->SelectNodeTo(N, ARM::ADDrsi, MVT::i32, Ops);
    2778             :           return;
    2779             :         }
    2780             :       }
    2781          73 :       if (isPowerOf2_32(RHSV+1)) {  // 2^n-1?
    2782           0 :         unsigned ShImm = Log2_32(RHSV+1);
    2783             :         if (ShImm >= 32)
    2784             :           break;
    2785           0 :         SDValue V = N->getOperand(0);
    2786           0 :         ShImm = ARM_AM::getSORegOpc(ARM_AM::lsl, ShImm);
    2787           0 :         SDValue ShImmOp = CurDAG->getTargetConstant(ShImm, dl, MVT::i32);
    2788           0 :         SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
    2789           0 :         if (Subtarget->isThumb()) {
    2790           0 :           SDValue Ops[] = { V, V, ShImmOp, getAL(CurDAG, dl), Reg0, Reg0 };
    2791           0 :           CurDAG->SelectNodeTo(N, ARM::t2RSBrs, MVT::i32, Ops);
    2792             :           return;
    2793             :         } else {
    2794           0 :           SDValue Ops[] = { V, V, Reg0, ShImmOp, getAL(CurDAG, dl), Reg0,
    2795           0 :                             Reg0 };
    2796           0 :           CurDAG->SelectNodeTo(N, ARM::RSBrsi, MVT::i32, Ops);
    2797             :           return;
    2798             :         }
    2799             :       }
    2800             :     }
    2801             :     break;
    2802         926 :   case ISD::AND: {
    2803             :     // Check for unsigned bitfield extract
    2804         926 :     if (tryV6T2BitfieldExtractOp(N, false))
    2805          41 :       return;
    2806             : 
    2807             :     // If an immediate is used in an AND node, it is possible that the immediate
    2808             :     // can be more optimally materialized when negated. If this is the case we
    2809             :     // can negate the immediate and use a BIC instead.
    2810        2485 :     auto *N1C = dyn_cast<ConstantSDNode>(N->getOperand(1));
    2811        1127 :     if (N1C && N1C->hasOneUse() && Subtarget->isThumb()) {
    2812         244 :       uint32_t Imm = (uint32_t) N1C->getZExtValue();
    2813             : 
    2814             :       // In Thumb2 mode, an AND can take a 12-bit immediate. If this
    2815             :       // immediate can be negated and fit in the immediate operand of
    2816             :       // a t2BIC, don't do any manual transform here as this can be
    2817             :       // handled by the generic ISel machinery.
    2818             :       bool PreferImmediateEncoding =
    2819         532 :         Subtarget->hasThumb2() && (is_t2_so_imm(Imm) || is_t2_so_imm_not(Imm));
    2820          91 :       if (!PreferImmediateEncoding &&
    2821          91 :           ConstantMaterializationCost(Imm) >
    2822          91 :               ConstantMaterializationCost(~Imm)) {
    2823             :         // The current immediate costs more to materialize than a negated
    2824             :         // immediate, so negate the immediate and use a BIC.
    2825             :         SDValue NewImm =
    2826          42 :           CurDAG->getConstant(~N1C->getZExtValue(), dl, MVT::i32);
    2827             :         // If the new constant didn't exist before, reposition it in the topological
    2828             :         // ordering so it is just before N. Otherwise, don't touch its location.
    2829          14 :         if (NewImm->getNodeId() == -1)
    2830          14 :           CurDAG->RepositionNode(N->getIterator(), NewImm.getNode());
    2831             : 
    2832          14 :         if (!Subtarget->hasThumb2()) {
    2833          30 :           SDValue Ops[] = {CurDAG->getRegister(ARM::CPSR, MVT::i32),
    2834          30 :                            N->getOperand(0), NewImm, getAL(CurDAG, dl),
    2835          40 :                            CurDAG->getRegister(0, MVT::i32)};
    2836          40 :           ReplaceNode(N, CurDAG->getMachineNode(ARM::tBIC, dl, MVT::i32, Ops));
    2837             :           return;
    2838             :         } else {
    2839          12 :           SDValue Ops[] = {N->getOperand(0), NewImm, getAL(CurDAG, dl),
    2840          12 :                            CurDAG->getRegister(0, MVT::i32),
    2841          20 :                            CurDAG->getRegister(0, MVT::i32)};
    2842          12 :           ReplaceNode(N,
    2843          16 :                       CurDAG->getMachineNode(ARM::t2BICrr, dl, MVT::i32, Ops));
    2844             :           return;
    2845             :         }
    2846             :       }
    2847             :     }
    2848             : 
    2849             :     // (and (or x, c2), c1) and top 16-bits of c1 and c2 match, lower 16-bits
    2850             :     // of c1 are 0xffff, and lower 16-bit of c2 are 0. That is, the top 16-bits
    2851             :     // are entirely contributed by c2 and lower 16-bits are entirely contributed
    2852             :     // by x. That's equal to (or (and x, 0xffff), (and c1, 0xffff0000)).
    2853             :     // Select it to: "movt x, ((c1 & 0xffff) >> 16)
    2854        1774 :     EVT VT = N->getValueType(0);
    2855         887 :     if (VT != MVT::i32)
    2856             :       break;
    2857        1242 :     unsigned Opc = (Subtarget->isThumb() && Subtarget->hasThumb2())
    2858        1398 :       ? ARM::t2MOVTi16
    2859         565 :       : (Subtarget->hasV6T2Ops() ? ARM::MOVTi16 : 0);
    2860             :     if (!Opc)
    2861             :       break;
    2862        1104 :     SDValue N0 = N->getOperand(0), N1 = N->getOperand(1);
    2863         295 :     N1C = dyn_cast<ConstantSDNode>(N1);
    2864             :     if (!N1C)
    2865             :       break;
    2866         592 :     if (N0.getOpcode() == ISD::OR && N0.getNode()->hasOneUse()) {
    2867           4 :       SDValue N2 = N0.getOperand(1);
    2868           2 :       ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2);
    2869             :       if (!N2C)
    2870             :         break;
    2871           2 :       unsigned N1CVal = N1C->getZExtValue();
    2872           2 :       unsigned N2CVal = N2C->getZExtValue();
    2873           4 :       if ((N1CVal & 0xffff0000U) == (N2CVal & 0xffff0000U) &&
    2874           4 :           (N1CVal & 0xffffU) == 0xffffU &&
    2875           2 :           (N2CVal & 0xffffU) == 0x0U) {
    2876           4 :         SDValue Imm16 = CurDAG->getTargetConstant((N2CVal & 0xFFFF0000U) >> 16,
    2877           6 :                                                   dl, MVT::i32);
    2878           4 :         SDValue Ops[] = { N0.getOperand(0), Imm16,
    2879           4 :                           getAL(CurDAG, dl), CurDAG->getRegister(0, MVT::i32) };
    2880           6 :         ReplaceNode(N, CurDAG->getMachineNode(Opc, dl, VT, Ops));
    2881             :         return;
    2882             :       }
    2883             :     }
    2884             : 
    2885             :     break;
    2886             :   }
    2887          12 :   case ARMISD::UMAAL: {
    2888          12 :     unsigned Opc = Subtarget->isThumb() ? ARM::t2UMAAL : ARM::UMAAL;
    2889          48 :     SDValue Ops[] = { N->getOperand(0), N->getOperand(1),
    2890          48 :                       N->getOperand(2), N->getOperand(3),
    2891          12 :                       getAL(CurDAG, dl),
    2892          72 :                       CurDAG->getRegister(0, MVT::i32) };
    2893          60 :     ReplaceNode(N, CurDAG->getMachineNode(Opc, dl, MVT::i32, MVT::i32, Ops));
    2894             :     return;
    2895             :   }
    2896          30 :   case ARMISD::UMLAL:{
    2897          30 :     if (Subtarget->isThumb()) {
    2898          84 :       SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2),
    2899          42 :                         N->getOperand(3), getAL(CurDAG, dl),
    2900          84 :                         CurDAG->getRegister(0, MVT::i32)};
    2901          42 :       ReplaceNode(
    2902          70 :           N, CurDAG->getMachineNode(ARM::t2UMLAL, dl, MVT::i32, MVT::i32, Ops));
    2903             :       return;
    2904             :     }else{
    2905          96 :       SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2),
    2906          48 :                         N->getOperand(3), getAL(CurDAG, dl),
    2907          48 :                         CurDAG->getRegister(0, MVT::i32),
    2908         128 :                         CurDAG->getRegister(0, MVT::i32) };
    2909          48 :       ReplaceNode(N, CurDAG->getMachineNode(
    2910          16 :                          Subtarget->hasV6Ops() ? ARM::UMLAL : ARM::UMLALv5, dl,
    2911          48 :                          MVT::i32, MVT::i32, Ops));
    2912             :       return;
    2913             :     }
    2914             :   }
    2915          32 :   case ARMISD::SMLAL:{
    2916          32 :     if (Subtarget->isThumb()) {
    2917         102 :       SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2),
    2918          51 :                         N->getOperand(3), getAL(CurDAG, dl),
    2919         102 :                         CurDAG->getRegister(0, MVT::i32)};
    2920          51 :       ReplaceNode(
    2921          85 :           N, CurDAG->getMachineNode(ARM::t2SMLAL, dl, MVT::i32, MVT::i32, Ops));
    2922             :       return;
    2923             :     }else{
    2924          90 :       SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2),
    2925          45 :                         N->getOperand(3), getAL(CurDAG, dl),
    2926          45 :                         CurDAG->getRegister(0, MVT::i32),
    2927         120 :                         CurDAG->getRegister(0, MVT::i32) };
    2928          45 :       ReplaceNode(N, CurDAG->getMachineNode(
    2929          15 :                          Subtarget->hasV6Ops() ? ARM::SMLAL : ARM::SMLALv5, dl,
    2930          45 :                          MVT::i32, MVT::i32, Ops));
    2931             :       return;
    2932             :     }
    2933             :   }
    2934         127 :   case ARMISD::SUBE: {
    2935         127 :     if (!Subtarget->hasV6Ops())
    2936             :       break;
    2937             :     // Look for a pattern to match SMMLS
    2938             :     // (sube a, (smul_loHi a, b), (subc 0, (smul_LOhi(a, b))))
    2939         156 :     if (N->getOperand(1).getOpcode() != ISD::SMUL_LOHI ||
    2940          90 :         N->getOperand(2).getOpcode() != ARMISD::SUBC ||
    2941           8 :         !SDValue(N, 1).use_empty())
    2942             :       break;
    2943             : 
    2944           4 :     if (Subtarget->isThumb())
    2945             :       assert(Subtarget->hasThumb2() &&
    2946             :              "This pattern should not be generated for Thumb");
    2947             : 
    2948           8 :     SDValue SmulLoHi = N->getOperand(1);
    2949           8 :     SDValue Subc = N->getOperand(2);
    2950          12 :     auto *Zero = dyn_cast<ConstantSDNode>(Subc.getOperand(0));
    2951             : 
    2952           4 :     if (!Zero || Zero->getZExtValue() != 0 ||
    2953          16 :         Subc.getOperand(1) != SmulLoHi.getValue(0) ||
    2954          16 :         N->getOperand(1) != SmulLoHi.getValue(1) ||
    2955          16 :         N->getOperand(2) != Subc.getValue(1))
    2956             :       break;
    2957             : 
    2958           8 :     unsigned Opc = Subtarget->isThumb2() ? ARM::t2SMMLS : ARM::SMMLS;
    2959          16 :     SDValue Ops[] = { SmulLoHi.getOperand(0), SmulLoHi.getOperand(1),
    2960          12 :                       N->getOperand(0), getAL(CurDAG, dl),
    2961          20 :                       CurDAG->getRegister(0, MVT::i32) };
    2962          16 :     ReplaceNode(N, CurDAG->getMachineNode(Opc, dl, MVT::i32, Ops));
    2963           4 :     return;
    2964             :   }
    2965       11944 :   case ISD::LOAD: {
    2966       11944 :     if (Subtarget->isThumb() && Subtarget->hasThumb2()) {
    2967        3527 :       if (tryT2IndexedLoad(N))
    2968             :         return;
    2969        8417 :     } else if (Subtarget->isThumb()) {
    2970        1518 :       if (tryT1IndexedLoad(N))
    2971             :         return;
    2972        6899 :     } else if (tryARMIndexedLoad(N))
    2973             :       return;
    2974             :     // Other cases are autogenerated.
    2975             :     break;
    2976             :   }
    2977        1738 :   case ARMISD::BRCOND: {
    2978             :     // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc)
    2979             :     // Emits: (Bcc:void (bb:Other):$dst, (imm:i32):$cc)
    2980             :     // Pattern complexity = 6  cost = 1  size = 0
    2981             : 
    2982             :     // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc)
    2983             :     // Emits: (tBcc:void (bb:Other):$dst, (imm:i32):$cc)
    2984             :     // Pattern complexity = 6  cost = 1  size = 0
    2985             : 
    2986             :     // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc)
    2987             :     // Emits: (t2Bcc:void (bb:Other):$dst, (imm:i32):$cc)
    2988             :     // Pattern complexity = 6  cost = 1  size = 0
    2989             : 
    2990        2794 :     unsigned Opc = Subtarget->isThumb() ?
    2991        2794 :       ((Subtarget->hasThumb2()) ? ARM::t2Bcc : ARM::tBcc) : ARM::Bcc;
    2992        3476 :     SDValue Chain = N->getOperand(0);
    2993        3476 :     SDValue N1 = N->getOperand(1);
    2994        3476 :     SDValue N2 = N->getOperand(2);
    2995        3476 :     SDValue N3 = N->getOperand(3);
    2996        3476 :     SDValue InFlag = N->getOperand(4);
    2997             :     assert(N1.getOpcode() == ISD::BasicBlock);
    2998             :     assert(N2.getOpcode() == ISD::Constant);
    2999             :     assert(N3.getOpcode() == ISD::Register);
    3000             : 
    3001        3476 :     unsigned CC = (unsigned) cast<ConstantSDNode>(N2)->getZExtValue();
    3002             :     
    3003        3476 :     if (InFlag.getOpcode() == ARMISD::CMPZ) {
    3004             :       bool SwitchEQNEToPLMI;
    3005        1467 :       SelectCMPZ(InFlag.getNode(), SwitchEQNEToPLMI);
    3006        2934 :       InFlag = N->getOperand(4);
    3007             : 
    3008        1467 :       if (SwitchEQNEToPLMI) {
    3009           5 :         switch ((ARMCC::CondCodes)CC) {
    3010           0 :         default: llvm_unreachable("CMPZ must be either NE or EQ!");
    3011             :         case ARMCC::NE:
    3012             :           CC = (unsigned)ARMCC::MI;
    3013             :           break;
    3014           4 :         case ARMCC::EQ:
    3015           4 :           CC = (unsigned)ARMCC::PL;
    3016           4 :           break;
    3017             :         }
    3018             :       }
    3019             :     }
    3020             : 
    3021        5214 :     SDValue Tmp2 = CurDAG->getTargetConstant(CC, dl, MVT::i32);
    3022        1738 :     SDValue Ops[] = { N1, Tmp2, N3, Chain, InFlag };
    3023        3476 :     SDNode *ResNode = CurDAG->getMachineNode(Opc, dl, MVT::Other,
    3024        5214 :                                              MVT::Glue, Ops);
    3025        1738 :     Chain = SDValue(ResNode, 0);
    3026        1738 :     if (N->getNumValues() == 2) {
    3027          45 :       InFlag = SDValue(ResNode, 1);
    3028          45 :       ReplaceUses(SDValue(N, 1), InFlag);
    3029             :     }
    3030        1738 :     ReplaceUses(SDValue(N, 0),
    3031             :                 SDValue(Chain.getNode(), Chain.getResNo()));
    3032        1738 :     CurDAG->RemoveDeadNode(N);
    3033             :     return;
    3034             :   }
    3035             : 
    3036        1950 :   case ARMISD::CMPZ: {
    3037             :     // select (CMPZ X, #-C) -> (CMPZ (ADDS X, #C), #0)
    3038             :     //   This allows us to avoid materializing the expensive negative constant.
    3039             :     //   The CMPZ #0 is useless and will be peepholed away but we need to keep it
    3040             :     //   for its glue output.
    3041        3900 :     SDValue X = N->getOperand(0);
    3042        5698 :     auto *C = dyn_cast<ConstantSDNode>(N->getOperand(1).getNode());
    3043        1798 :     if (C && C->getSExtValue() < 0 && Subtarget->isThumb()) {
    3044          26 :       int64_t Addend = -C->getSExtValue();
    3045             : 
    3046          26 :       SDNode *Add = nullptr;
    3047             :       // ADDS can be better than CMN if the immediate fits in a
    3048             :       // 16-bit ADDS, which means either [0,256) for tADDi8 or [0,8) for tADDi3.
    3049             :       // Outside that range we can just use a CMN which is 32-bit but has a
    3050             :       // 12-bit immediate range.
    3051          26 :       if (Addend < 1<<8) {
    3052          22 :         if (Subtarget->isThumb2()) {
    3053          40 :           SDValue Ops[] = { X, CurDAG->getTargetConstant(Addend, dl, MVT::i32),
    3054          80 :                             getAL(CurDAG, dl), CurDAG->getRegister(0, MVT::i32),
    3055         120 :                             CurDAG->getRegister(0, MVT::i32) };
    3056          60 :           Add = CurDAG->getMachineNode(ARM::t2ADDri, dl, MVT::i32, Ops);
    3057             :         } else {
    3058           4 :           unsigned Opc = (Addend < 1<<3) ? ARM::tADDi3 : ARM::tADDi8;
    3059           6 :           SDValue Ops[] = {CurDAG->getRegister(ARM::CPSR, MVT::i32), X,
    3060           4 :                            CurDAG->getTargetConstant(Addend, dl, MVT::i32),
    3061           8 :                            getAL(CurDAG, dl), CurDAG->getRegister(0, MVT::i32)};
    3062           6 :           Add = CurDAG->getMachineNode(Opc, dl, MVT::i32, Ops);
    3063             :         }
    3064             :       }
    3065          22 :       if (Add) {
    3066          66 :         SDValue Ops2[] = {SDValue(Add, 0), CurDAG->getConstant(0, dl, MVT::i32)};
    3067          66 :         CurDAG->MorphNodeTo(N, ARMISD::CMPZ, CurDAG->getVTList(MVT::Glue), Ops2);
    3068             :       }
    3069             :     }
    3070             :     // Other cases are autogenerated.
    3071             :     break;
    3072             :   }
    3073             : 
    3074        1171 :   case ARMISD::CMOV: {
    3075        2342 :     SDValue InFlag = N->getOperand(4);
    3076             : 
    3077        2342 :     if (InFlag.getOpcode() == ARMISD::CMPZ) {
    3078             :       bool SwitchEQNEToPLMI;
    3079         483 :       SelectCMPZ(InFlag.getNode(), SwitchEQNEToPLMI);
    3080             : 
    3081         483 :       if (SwitchEQNEToPLMI) {
    3082           4 :         SDValue ARMcc = N->getOperand(2);
    3083             :         ARMCC::CondCodes CC =
    3084           4 :           (ARMCC::CondCodes)cast<ConstantSDNode>(ARMcc)->getZExtValue();
    3085             : 
    3086           2 :         switch (CC) {
    3087           0 :         default: llvm_unreachable("CMPZ must be either NE or EQ!");
    3088             :         case ARMCC::NE:
    3089             :           CC = ARMCC::MI;
    3090             :           break;
    3091           2 :         case ARMCC::EQ:
    3092           2 :           CC = ARMCC::PL;
    3093           2 :           break;
    3094             :         }
    3095           4 :         SDValue NewARMcc = CurDAG->getConstant((unsigned)CC, dl, MVT::i32);
    3096           8 :         SDValue Ops[] = {N->getOperand(0), N->getOperand(1), NewARMcc,
    3097           8 :                          N->getOperand(3), N->getOperand(4)};
    3098           6 :         CurDAG->MorphNodeTo(N, ARMISD::CMOV, N->getVTList(), Ops);
    3099             :       }
    3100             : 
    3101             :     }
    3102             :     // Other cases are autogenerated.
    3103             :     break;
    3104             :   }
    3105             :     
    3106          27 :   case ARMISD::VZIP: {
    3107          27 :     unsigned Opc = 0;
    3108          54 :     EVT VT = N->getValueType(0);
    3109          27 :     switch (VT.getSimpleVT().SimpleTy) {
    3110             :     default: return;
    3111             :     case MVT::v8i8:  Opc = ARM::VZIPd8; break;
    3112           7 :     case MVT::v4i16: Opc = ARM::VZIPd16; break;
    3113           0 :     case MVT::v2f32:
    3114             :     // vzip.32 Dd, Dm is a pseudo-instruction expanded to vtrn.32 Dd, Dm.
    3115           0 :     case MVT::v2i32: Opc = ARM::VTRNd32; break;
    3116           4 :     case MVT::v16i8: Opc = ARM::VZIPq8; break;
    3117           2 :     case MVT::v8i16: Opc = ARM::VZIPq16; break;
    3118           9 :     case MVT::v4f32:
    3119           9 :     case MVT::v4i32: Opc = ARM::VZIPq32; break;
    3120             :     }
    3121          27 :     SDValue Pred = getAL(CurDAG, dl);
    3122          54 :     SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
    3123          81 :     SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg };
    3124          81 :     ReplaceNode(N, CurDAG->getMachineNode(Opc, dl, VT, VT, Ops));
    3125          27 :     return;
    3126             :   }
    3127          55 :   case ARMISD::VUZP: {
    3128          55 :     unsigned Opc = 0;
    3129         110 :     EVT VT = N->getValueType(0);
    3130          55 :     switch (VT.getSimpleVT().SimpleTy) {
    3131             :     default: return;
    3132             :     case MVT::v8i8:  Opc = ARM::VUZPd8; break;
    3133          15 :     case MVT::v4i16: Opc = ARM::VUZPd16; break;
    3134           0 :     case MVT::v2f32:
    3135             :     // vuzp.32 Dd, Dm is a pseudo-instruction expanded to vtrn.32 Dd, Dm.
    3136           0 :     case MVT::v2i32: Opc = ARM::VTRNd32; break;
    3137           4 :     case MVT::v16i8: Opc = ARM::VUZPq8; break;
    3138           7 :     case MVT::v8i16: Opc = ARM::VUZPq16; break;
    3139           8 :     case MVT::v4f32:
    3140           8 :     case MVT::v4i32: Opc = ARM::VUZPq32; break;
    3141             :     }
    3142          55 :     SDValue Pred = getAL(CurDAG, dl);
    3143         110 :     SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
    3144         165 :     SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg };
    3145         165 :     ReplaceNode(N, CurDAG->getMachineNode(Opc, dl, VT, VT, Ops));
    3146          55 :     return;
    3147             :   }
    3148          29 :   case ARMISD::VTRN: {
    3149          29 :     unsigned Opc = 0;
    3150          58 :     EVT VT = N->getValueType(0);
    3151          29 :     switch (VT.getSimpleVT().SimpleTy) {
    3152             :     default: return;
    3153             :     case MVT::v8i8:  Opc = ARM::VTRNd8; break;
    3154           7 :     case MVT::v4i16: Opc = ARM::VTRNd16; break;
    3155           4 :     case MVT::v2f32:
    3156           4 :     case MVT::v2i32: Opc = ARM::VTRNd32; break;
    3157           2 :     case MVT::v16i8: Opc = ARM::VTRNq8; break;
    3158           6 :     case MVT::v8i16: Opc = ARM::VTRNq16; break;
    3159           4 :     case MVT::v4f32:
    3160           4 :     case MVT::v4i32: Opc = ARM::VTRNq32; break;
    3161             :     }
    3162          29 :     SDValue Pred = getAL(CurDAG, dl);
    3163          58 :     SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
    3164          87 :     SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg };
    3165          87 :     ReplaceNode(N, CurDAG->getMachineNode(Opc, dl, VT, VT, Ops));
    3166          29 :     return;
    3167             :   }
    3168         407 :   case ARMISD::BUILD_VECTOR: {
    3169         814 :     EVT VecVT = N->getValueType(0);
    3170         407 :     EVT EltVT = VecVT.getVectorElementType();
    3171         407 :     unsigned NumElts = VecVT.getVectorNumElements();
    3172         814 :     if (EltVT == MVT::f64) {
    3173             :       assert(NumElts == 2 && "unexpected type for BUILD_VECTOR");
    3174         602 :       ReplaceNode(
    3175         903 :           N, createDRegPairNode(VecVT, N->getOperand(0), N->getOperand(1)));
    3176             :       return;
    3177             :     }
    3178             :     assert(EltVT == MVT::f32 && "unexpected type for BUILD_VECTOR");
    3179         106 :     if (NumElts == 2) {
    3180          76 :       ReplaceNode(
    3181         114 :           N, createSRegPairNode(VecVT, N->getOperand(0), N->getOperand(1)));
    3182             :       return;
    3183             :     }
    3184             :     assert(NumElts == 4 && "unexpected type for BUILD_VECTOR");
    3185         136 :     ReplaceNode(N,
    3186         204 :                 createQuadSRegsNode(VecVT, N->getOperand(0), N->getOperand(1),
    3187         204 :                                     N->getOperand(2), N->getOperand(3)));
    3188             :     return;
    3189             :   }
    3190             : 
    3191          28 :   case ARMISD::VLD1DUP: {
    3192             :     static const uint16_t DOpcodes[] = { ARM::VLD1DUPd8, ARM::VLD1DUPd16,
    3193             :                                          ARM::VLD1DUPd32 };
    3194             :     static const uint16_t QOpcodes[] = { ARM::VLD1DUPq8, ARM::VLD1DUPq16,
    3195             :                                          ARM::VLD1DUPq32 };
    3196          28 :     SelectVLDDup(N, false, 1, DOpcodes, QOpcodes);
    3197          28 :     return;
    3198             :   }
    3199             : 
    3200           4 :   case ARMISD::VLD2DUP: {
    3201             :     static const uint16_t Opcodes[] = { ARM::VLD2DUPd8, ARM::VLD2DUPd16,
    3202             :                                         ARM::VLD2DUPd32 };
    3203           4 :     SelectVLDDup(N, false, 2, Opcodes);
    3204           4 :     return;
    3205             :   }
    3206             : 
    3207           1 :   case ARMISD::VLD3DUP: {
    3208             :     static const uint16_t Opcodes[] = { ARM::VLD3DUPd8Pseudo,
    3209             :                                         ARM::VLD3DUPd16Pseudo,
    3210             :                                         ARM::VLD3DUPd32Pseudo };
    3211           1 :     SelectVLDDup(N, false, 3, Opcodes);
    3212           1 :     return;
    3213             :   }
    3214             : 
    3215           1 :   case ARMISD::VLD4DUP: {
    3216             :     static const uint16_t Opcodes[] = { ARM::VLD4DUPd8Pseudo,
    3217             :                                         ARM::VLD4DUPd16Pseudo,
    3218             :                                         ARM::VLD4DUPd32Pseudo };
    3219           1 :     SelectVLDDup(N, false, 4, Opcodes);
    3220           1 :     return;
    3221             :   }
    3222             : 
    3223           6 :   case ARMISD::VLD1DUP_UPD: {
    3224             :     static const uint16_t DOpcodes[] = { ARM::VLD1DUPd8wb_fixed,
    3225             :                                          ARM::VLD1DUPd16wb_fixed,
    3226             :                                          ARM::VLD1DUPd32wb_fixed };
    3227             :     static const uint16_t QOpcodes[] = { ARM::VLD1DUPq8wb_fixed,
    3228             :                                          ARM::VLD1DUPq16wb_fixed,
    3229             :                                          ARM::VLD1DUPq32wb_fixed };
    3230           6 :     SelectVLDDup(N, true, 1, DOpcodes, QOpcodes);
    3231           6 :     return;
    3232             :   }
    3233             : 
    3234           4 :   case ARMISD::VLD2DUP_UPD: {
    3235             :     static const uint16_t Opcodes[] = { ARM::VLD2DUPd8wb_fixed,
    3236             :                                         ARM::VLD2DUPd16wb_fixed,
    3237             :                                         ARM::VLD2DUPd32wb_fixed };
    3238           4 :     SelectVLDDup(N, true, 2, Opcodes);
    3239           4 :     return;
    3240             :   }
    3241             : 
    3242           1 :   case ARMISD::VLD3DUP_UPD: {
    3243             :     static const uint16_t Opcodes[] = { ARM::VLD3DUPd8Pseudo_UPD,
    3244             :                                         ARM::VLD3DUPd16Pseudo_UPD,
    3245             :                                         ARM::VLD3DUPd32Pseudo_UPD };
    3246           1 :     SelectVLDDup(N, true, 3, Opcodes);
    3247           1 :     return;
    3248             :   }
    3249             : 
    3250           1 :   case ARMISD::VLD4DUP_UPD: {
    3251             :     static const uint16_t Opcodes[] = { ARM::VLD4DUPd8Pseudo_UPD,
    3252             :                                         ARM::VLD4DUPd16Pseudo_UPD,
    3253             :                                         ARM::VLD4DUPd32Pseudo_UPD };
    3254           1 :     SelectVLDDup(N, true, 4, Opcodes);
    3255           1 :     return;
    3256             :   }
    3257             : 
    3258         100 :   case ARMISD::VLD1_UPD: {
    3259             :     static const uint16_t DOpcodes[] = { ARM::VLD1d8wb_fixed,
    3260             :                                          ARM::VLD1d16wb_fixed,
    3261             :                                          ARM::VLD1d32wb_fixed,
    3262             :                                          ARM::VLD1d64wb_fixed };
    3263             :     static const uint16_t QOpcodes[] = { ARM::VLD1q8wb_fixed,
    3264             :                                          ARM::VLD1q16wb_fixed,
    3265             :                                          ARM::VLD1q32wb_fixed,
    3266             :                                          ARM::VLD1q64wb_fixed };
    3267         100 :     SelectVLD(N, true, 1, DOpcodes, QOpcodes, nullptr);
    3268         100 :     return;
    3269             :   }
    3270             : 
    3271           7 :   case ARMISD::VLD2_UPD: {
    3272             :     static const uint16_t DOpcodes[] = { ARM::VLD2d8wb_fixed,
    3273             :                                          ARM::VLD2d16wb_fixed,
    3274             :                                          ARM::VLD2d32wb_fixed,
    3275             :                                          ARM::VLD1q64wb_fixed};
    3276             :     static const uint16_t QOpcodes[] = { ARM::VLD2q8PseudoWB_fixed,
    3277             :                                          ARM::VLD2q16PseudoWB_fixed,
    3278             :                                          ARM::VLD2q32PseudoWB_fixed };
    3279           7 :     SelectVLD(N, true, 2, DOpcodes, QOpcodes, nullptr);
    3280           7 :     return;
    3281             :   }
    3282             : 
    3283           7 :   case ARMISD::VLD3_UPD: {
    3284             :     static const uint16_t DOpcodes[] = { ARM::VLD3d8Pseudo_UPD,
    3285             :                                          ARM::VLD3d16Pseudo_UPD,
    3286             :                                          ARM::VLD3d32Pseudo_UPD,
    3287             :                                          ARM::VLD1d64TPseudoWB_fixed};
    3288             :     static const uint16_t QOpcodes0[] = { ARM::VLD3q8Pseudo_UPD,
    3289             :                                           ARM::VLD3q16Pseudo_UPD,
    3290             :                                           ARM::VLD3q32Pseudo_UPD };
    3291             :     static const uint16_t QOpcodes1[] = { ARM::VLD3q8oddPseudo_UPD,
    3292             :                                           ARM::VLD3q16oddPseudo_UPD,
    3293             :                                           ARM::VLD3q32oddPseudo_UPD };
    3294           7 :     SelectVLD(N, true, 3, DOpcodes, QOpcodes0, QOpcodes1);
    3295           7 :     return;
    3296             :   }
    3297             : 
    3298           3 :   case ARMISD::VLD4_UPD: {
    3299             :     static const uint16_t DOpcodes[] = { ARM::VLD4d8Pseudo_UPD,
    3300             :                                          ARM::VLD4d16Pseudo_UPD,
    3301             :                                          ARM::VLD4d32Pseudo_UPD,
    3302             :                                          ARM::VLD1d64QPseudoWB_fixed};
    3303             :     static const uint16_t QOpcodes0[] = { ARM::VLD4q8Pseudo_UPD,
    3304             :                                           ARM::VLD4q16Pseudo_UPD,
    3305             :                                           ARM::VLD4q32Pseudo_UPD };
    3306             :     static const uint16_t QOpcodes1[] = { ARM::VLD4q8oddPseudo_UPD,
    3307             :                                           ARM::VLD4q16oddPseudo_UPD,
    3308             :                                           ARM::VLD4q32oddPseudo_UPD };
    3309           3 :     SelectVLD(N, true, 4, DOpcodes, QOpcodes0, QOpcodes1);
    3310           3 :     return;
    3311             :   }
    3312             : 
    3313           4 :   case ARMISD::VLD2LN_UPD: {
    3314             :     static const uint16_t DOpcodes[] = { ARM::VLD2LNd8Pseudo_UPD,
    3315             :                                          ARM::VLD2LNd16Pseudo_UPD,
    3316             :                                          ARM::VLD2LNd32Pseudo_UPD };
    3317             :     static const uint16_t QOpcodes[] = { ARM::VLD2LNq16Pseudo_UPD,
    3318             :                                          ARM::VLD2LNq32Pseudo_UPD };
    3319           4 :     SelectVLDSTLane(N, true, true, 2, DOpcodes, QOpcodes);
    3320           4 :     return;
    3321             :   }
    3322             : 
    3323           2 :   case ARMISD::VLD3LN_UPD: {
    3324             :     static const uint16_t DOpcodes[] = { ARM::VLD3LNd8Pseudo_UPD,
    3325             :                                          ARM::VLD3LNd16Pseudo_UPD,
    3326             :                                          ARM::VLD3LNd32Pseudo_UPD };
    3327             :     static const uint16_t QOpcodes[] = { ARM::VLD3LNq16Pseudo_UPD,
    3328             :                                          ARM::VLD3LNq32Pseudo_UPD };
    3329           2 :     SelectVLDSTLane(N, true, true, 3, DOpcodes, QOpcodes);
    3330           2 :     return;
    3331             :   }
    3332             : 
    3333           2 :   case ARMISD::VLD4LN_UPD: {
    3334             :     static const uint16_t DOpcodes[] = { ARM::VLD4LNd8Pseudo_UPD,
    3335             :                                          ARM::VLD4LNd16Pseudo_UPD,
    3336             :                                          ARM::VLD4LNd32Pseudo_UPD };
    3337             :     static const uint16_t QOpcodes[] = { ARM::VLD4LNq16Pseudo_UPD,
    3338             :                                          ARM::VLD4LNq32Pseudo_UPD };
    3339           2 :     SelectVLDSTLane(N, true, true, 4, DOpcodes, QOpcodes);
    3340           2 :     return;
    3341             :   }
    3342             : 
    3343          90 :   case ARMISD::VST1_UPD: {
    3344             :     static const uint16_t DOpcodes[] = { ARM::VST1d8wb_fixed,
    3345             :                                          ARM::VST1d16wb_fixed,
    3346             :                                          ARM::VST1d32wb_fixed,
    3347             :                                          ARM::VST1d64wb_fixed };
    3348             :     static const uint16_t QOpcodes[] = { ARM::VST1q8wb_fixed,
    3349             :                                          ARM::VST1q16wb_fixed,
    3350             :                                          ARM::VST1q32wb_fixed,
    3351             :                                          ARM::VST1q64wb_fixed };
    3352          90 :     SelectVST(N, true, 1, DOpcodes, QOpcodes, nullptr);
    3353          90 :     return;
    3354             :   }
    3355             : 
    3356           5 :   case ARMISD::VST2_UPD: {
    3357             :     static const uint16_t DOpcodes[] = { ARM::VST2d8wb_fixed,
    3358             :                                          ARM::VST2d16wb_fixed,
    3359             :                                          ARM::VST2d32wb_fixed,
    3360             :                                          ARM::VST1q64wb_fixed};
    3361             :     static const uint16_t QOpcodes[] = { ARM::VST2q8PseudoWB_fixed,
    3362             :                                          ARM::VST2q16PseudoWB_fixed,
    3363             :                                          ARM::VST2q32PseudoWB_fixed };
    3364           5 :     SelectVST(N, true, 2, DOpcodes, QOpcodes, nullptr);
    3365           5 :     return;
    3366             :   }
    3367             : 
    3368           3 :   case ARMISD::VST3_UPD: {
    3369             :     static const uint16_t DOpcodes[] = { ARM::VST3d8Pseudo_UPD,
    3370             :                                          ARM::VST3d16Pseudo_UPD,
    3371             :                                          ARM::VST3d32Pseudo_UPD,
    3372             :                                          ARM::VST1d64TPseudoWB_fixed};
    3373             :     static const uint16_t QOpcodes0[] = { ARM::VST3q8Pseudo_UPD,
    3374             :                                           ARM::VST3q16Pseudo_UPD,
    3375             :                                           ARM::VST3q32Pseudo_UPD };
    3376             :     static const uint16_t QOpcodes1[] = { ARM::VST3q8oddPseudo_UPD,
    3377             :                                           ARM::VST3q16oddPseudo_UPD,
    3378             :                                           ARM::VST3q32oddPseudo_UPD };
    3379           3 :     SelectVST(N, true, 3, DOpcodes, QOpcodes0, QOpcodes1);
    3380           3 :     return;
    3381             :   }
    3382             : 
    3383           3 :   case ARMISD::VST4_UPD: {
    3384             :     static const uint16_t DOpcodes[] = { ARM::VST4d8Pseudo_UPD,
    3385             :                                          ARM::VST4d16Pseudo_UPD,
    3386             :                                          ARM::VST4d32Pseudo_UPD,
    3387             :                                          ARM::VST1d64QPseudoWB_fixed};
    3388             :     static const uint16_t QOpcodes0[] = { ARM::VST4q8Pseudo_UPD,
    3389             :                                           ARM::VST4q16Pseudo_UPD,
    3390             :                                           ARM::VST4q32Pseudo_UPD };
    3391             :     static const uint16_t QOpcodes1[] = { ARM::VST4q8oddPseudo_UPD,
    3392             :                                           ARM::VST4q16oddPseudo_UPD,
    3393             :                                           ARM::VST4q32oddPseudo_UPD };
    3394           3 :     SelectVST(N, true, 4, DOpcodes, QOpcodes0, QOpcodes1);
    3395           3 :     return;
    3396             :   }
    3397             : 
    3398           1 :   case ARMISD::VST2LN_UPD: {
    3399             :     static const uint16_t DOpcodes[] = { ARM::VST2LNd8Pseudo_UPD,
    3400             :                                          ARM::VST2LNd16Pseudo_UPD,
    3401             :                                          ARM::VST2LNd32Pseudo_UPD };
    3402             :     static const uint16_t QOpcodes[] = { ARM::VST2LNq16Pseudo_UPD,
    3403             :                                          ARM::VST2LNq32Pseudo_UPD };
    3404           1 :     SelectVLDSTLane(N, false, true, 2, DOpcodes, QOpcodes);
    3405           1 :     return;
    3406             :   }
    3407             : 
    3408           1 :   case ARMISD::VST3LN_UPD: {
    3409             :     static const uint16_t DOpcodes[] = { ARM::VST3LNd8Pseudo_UPD,
    3410             :                                          ARM::VST3LNd16Pseudo_UPD,
    3411             :                                          ARM::VST3LNd32Pseudo_UPD };
    3412             :     static const uint16_t QOpcodes[] = { ARM::VST3LNq16Pseudo_UPD,
    3413             :                                          ARM::VST3LNq32Pseudo_UPD };
    3414           1 :     SelectVLDSTLane(N, false, true, 3, DOpcodes, QOpcodes);
    3415           1 :     return;
    3416             :   }
    3417             : 
    3418           1 :   case ARMISD::VST4LN_UPD: {
    3419             :     static const uint16_t DOpcodes[] = { ARM::VST4LNd8Pseudo_UPD,
    3420             :                                          ARM::VST4LNd16Pseudo_UPD,
    3421             :                                          ARM::VST4LNd32Pseudo_UPD };
    3422             :     static const uint16_t QOpcodes[] = { ARM::VST4LNq16Pseudo_UPD,
    3423             :                                          ARM::VST4LNq32Pseudo_UPD };
    3424           1 :     SelectVLDSTLane(N, false, true, 4, DOpcodes, QOpcodes);
    3425           1 :     return;
    3426             :   }
    3427             : 
    3428        1536 :   case ISD::INTRINSIC_VOID:
    3429             :   case ISD::INTRINSIC_W_CHAIN: {
    3430        6144 :     unsigned IntNo = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue();
    3431        1536 :     switch (IntNo) {
    3432             :     default:
    3433             :       break;
    3434             : 
    3435           4 :     case Intrinsic::arm_mrrc:
    3436             :     case Intrinsic::arm_mrrc2: {
    3437           8 :       SDLoc dl(N);
    3438           8 :       SDValue Chain = N->getOperand(0);
    3439             :       unsigned Opc;
    3440             : 
    3441           4 :       if (Subtarget->isThumb())
    3442           2 :         Opc = (IntNo == Intrinsic::arm_mrrc ? ARM::t2MRRC : ARM::t2MRRC2);
    3443             :       else
    3444           2 :         Opc = (IntNo == Intrinsic::arm_mrrc ? ARM::MRRC : ARM::MRRC2);
    3445             : 
    3446           8 :       SmallVector<SDValue, 5> Ops;
    3447          20 :       Ops.push_back(getI32Imm(cast<ConstantSDNode>(N->getOperand(2))->getZExtValue(), dl)); /* coproc */
    3448          20 :       Ops.push_back(getI32Imm(cast<ConstantSDNode>(N->getOperand(3))->getZExtValue(), dl)); /* opc */
    3449          20 :       Ops.push_back(getI32Imm(cast<ConstantSDNode>(N->getOperand(4))->getZExtValue(), dl)); /* CRm */
    3450             : 
    3451             :       // The mrrc2 instruction in ARM doesn't allow predicates, the top 4 bits of the encoded
    3452             :       // instruction will always be '1111' but it is possible in assembly language to specify
    3453             :       // AL as a predicate to mrrc2 but it doesn't make any difference to the encoded instruction.
    3454           4 :       if (Opc != ARM::MRRC2) {
    3455           3 :         Ops.push_back(getAL(CurDAG, dl));
    3456           6 :         Ops.push_back(CurDAG->getRegister(0, MVT::i32));
    3457             :       }
    3458             : 
    3459           4 :       Ops.push_back(Chain);
    3460             : 
    3461             :       // Writes to two registers.
    3462           4 :       const EVT RetType[] = {MVT::i32, MVT::i32, MVT::Other};
    3463             : 
    3464          16 :       ReplaceNode(N, CurDAG->getMachineNode(Opc, dl, RetType, Ops));
    3465             :       return;
    3466             :     }
    3467         127 :     case Intrinsic::arm_ldaexd:
    3468             :     case Intrinsic::arm_ldrexd: {
    3469         254 :       SDLoc dl(N);
    3470         254 :       SDValue Chain = N->getOperand(0);
    3471         254 :       SDValue MemAddr = N->getOperand(2);
    3472         185 :       bool isThumb = Subtarget->isThumb() && Subtarget->hasV8MBaselineOps();
    3473             : 
    3474         127 :       bool IsAcquire = IntNo == Intrinsic::arm_ldaexd;
    3475         127 :       unsigned NewOpc = isThumb ? (IsAcquire ? ARM::t2LDAEXD : ARM::t2LDREXD)
    3476             :                                 : (IsAcquire ? ARM::LDAEXD : ARM::LDREXD);
    3477             : 
    3478             :       // arm_ldrexd returns a i64 value in {i32, i32}
    3479         254 :       std::vector<EVT> ResTys;
    3480         127 :       if (isThumb) {
    3481         116 :         ResTys.push_back(MVT::i32);
    3482         116 :         ResTys.push_back(MVT::i32);
    3483             :       } else
    3484         138 :         ResTys.push_back(MVT::Untyped);
    3485         254 :       ResTys.push_back(MVT::Other);
    3486             : 
    3487             :       // Place arguments in the right order.
    3488         127 :       SDValue Ops[] = {MemAddr, getAL(CurDAG, dl),
    3489         381 :                        CurDAG->getRegister(0, MVT::i32), Chain};
    3490         381 :       SDNode *Ld = CurDAG->getMachineNode(NewOpc, dl, ResTys, Ops);
    3491             :       // Transfer memoperands.
    3492         127 :       MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1);
    3493         127 :       MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand();
    3494         254 :       cast<MachineSDNode>(Ld)->setMemRefs(MemOp, MemOp + 1);
    3495             : 
    3496             :       // Remap uses.
    3497         127 :       SDValue OutChain = isThumb ? SDValue(Ld, 2) : SDValue(Ld, 1);
    3498         254 :       if (!SDValue(N, 0).use_empty()) {
    3499         109 :         SDValue Result;
    3500         109 :         if (isThumb)
    3501             :           Result = SDValue(Ld, 0);
    3502             :         else {
    3503             :           SDValue SubRegIdx =
    3504         168 :             CurDAG->getTargetConstant(ARM::gsub_0, dl, MVT::i32);
    3505         168 :           SDNode *ResNode = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
    3506          56 :               dl, MVT::i32, SDValue(Ld, 0), SubRegIdx);
    3507          56 :           Result = SDValue(ResNode,0);
    3508             :         }
    3509         109 :         ReplaceUses(SDValue(N, 0), Result);
    3510             :       }
    3511         254 :       if (!SDValue(N, 1).use_empty()) {
    3512         108 :         SDValue Result;
    3513         108 :         if (isThumb)
    3514             :           Result = SDValue(Ld, 1);
    3515             :         else {
    3516             :           SDValue SubRegIdx =
    3517         165 :             CurDAG->getTargetConstant(ARM::gsub_1, dl, MVT::i32);
    3518         165 :           SDNode *ResNode = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
    3519          55 :               dl, MVT::i32, SDValue(Ld, 0), SubRegIdx);
    3520          55 :           Result = SDValue(ResNode,0);
    3521             :         }
    3522         108 :         ReplaceUses(SDValue(N, 1), Result);
    3523             :       }
    3524         127 :       ReplaceUses(SDValue(N, 2), OutChain);
    3525         127 :       CurDAG->RemoveDeadNode(N);
    3526             :       return;
    3527             :     }
    3528         109 :     case Intrinsic::arm_stlexd:
    3529             :     case Intrinsic::arm_strexd: {
    3530         218 :       SDLoc dl(N);
    3531         218 :       SDValue Chain = N->getOperand(0);
    3532         218 :       SDValue Val0 = N->getOperand(2);
    3533         218 :       SDValue Val1 = N->getOperand(3);
    3534         218 :       SDValue MemAddr = N->getOperand(4);
    3535             : 
    3536             :       // Store exclusive double return a i32 value which is the return status
    3537             :       // of the issued store.
    3538         109 :       const EVT ResTys[] = {MVT::i32, MVT::Other};
    3539             : 
    3540         109 :       bool isThumb = Subtarget->isThumb() && Subtarget->hasThumb2();
    3541             :       // Place arguments in the right order.
    3542         218 :       SmallVector<SDValue, 7> Ops;
    3543         109 :       if (isThumb) {
    3544          51 :         Ops.push_back(Val0);
    3545          51 :         Ops.push_back(Val1);
    3546             :       } else
    3547             :         // arm_strexd uses GPRPair.
    3548         116 :         Ops.push_back(SDValue(createGPRPairNode(MVT::Untyped, Val0, Val1), 0));
    3549         109 :       Ops.push_back(MemAddr);
    3550         109 :       Ops.push_back(getAL(CurDAG, dl));
    3551         218 :       Ops.push_back(CurDAG->getRegister(0, MVT::i32));
    3552         109 :       Ops.push_back(Chain);
    3553             : 
    3554         109 :       bool IsRelease = IntNo == Intrinsic::arm_stlexd;
    3555         109 :       unsigned NewOpc = isThumb ? (IsRelease ? ARM::t2STLEXD : ARM::t2STREXD)
    3556             :                                 : (IsRelease ? ARM::STLEXD : ARM::STREXD);
    3557             : 
    3558         327 :       SDNode *St = CurDAG->getMachineNode(NewOpc, dl, ResTys, Ops);
    3559             :       // Transfer memoperands.
    3560         109 :       MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1);
    3561         109 :       MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand();
    3562         218 :       cast<MachineSDNode>(St)->setMemRefs(MemOp, MemOp + 1);
    3563             : 
    3564         218 :       ReplaceNode(N, St);
    3565             :       return;
    3566             :     }
    3567             : 
    3568          68 :     case Intrinsic::arm_neon_vld1: {
    3569             :       static const uint16_t DOpcodes[] = { ARM::VLD1d8, ARM::VLD1d16,
    3570             :                                            ARM::VLD1d32, ARM::VLD1d64 };
    3571             :       static const uint16_t QOpcodes[] = { ARM::VLD1q8, ARM::VLD1q16,
    3572             :                                            ARM::VLD1q32, ARM::VLD1q64};
    3573          68 :       SelectVLD(N, false, 1, DOpcodes, QOpcodes, nullptr);
    3574          68 :       return;
    3575             :     }
    3576             : 
    3577          16 :     case Intrinsic::arm_neon_vld2: {
    3578             :       static const uint16_t DOpcodes[] = { ARM::VLD2d8, ARM::VLD2d16,
    3579             :                                            ARM::VLD2d32, ARM::VLD1q64 };
    3580             :       static const uint16_t QOpcodes[] = { ARM::VLD2q8Pseudo, ARM::VLD2q16Pseudo,
    3581             :                                            ARM::VLD2q32Pseudo };
    3582          16 :       SelectVLD(N, false, 2, DOpcodes, QOpcodes, nullptr);
    3583          16 :       return;
    3584             :     }
    3585             : 
    3586          37 :     case Intrinsic::arm_neon_vld3: {
    3587             :       static const uint16_t DOpcodes[] = { ARM::VLD3d8Pseudo,
    3588             :                                            ARM::VLD3d16Pseudo,
    3589             :                                            ARM::VLD3d32Pseudo,
    3590             :                                            ARM::VLD1d64TPseudo };
    3591             :       static const uint16_t QOpcodes0[] = { ARM::VLD3q8Pseudo_UPD,
    3592             :                                             ARM::VLD3q16Pseudo_UPD,
    3593             :                                             ARM::VLD3q32Pseudo_UPD };
    3594             :       static const uint16_t QOpcodes1[] = { ARM::VLD3q8oddPseudo,
    3595             :                                             ARM::VLD3q16oddPseudo,
    3596             :                                             ARM::VLD3q32oddPseudo };
    3597          37 :       SelectVLD(N, false, 3, DOpcodes, QOpcodes0, QOpcodes1);
    3598          37 :       return;
    3599             :     }
    3600             : 
    3601          11 :     case Intrinsic::arm_neon_vld4: {
    3602             :       static const uint16_t DOpcodes[] = { ARM::VLD4d8Pseudo,
    3603             :                                            ARM::VLD4d16Pseudo,
    3604             :                                            ARM::VLD4d32Pseudo,
    3605             :                                            ARM::VLD1d64QPseudo };
    3606             :       static const uint16_t QOpcodes0[] = { ARM::VLD4q8Pseudo_UPD,
    3607             :                                             ARM::VLD4q16Pseudo_UPD,
    3608             :                                             ARM::VLD4q32Pseudo_UPD };
    3609             :       static const uint16_t QOpcodes1[] = { ARM::VLD4q8oddPseudo,
    3610             :                                             ARM::VLD4q16oddPseudo,
    3611             :                                             ARM::VLD4q32oddPseudo };
    3612          11 :       SelectVLD(N, false, 4, DOpcodes, QOpcodes0, QOpcodes1);
    3613          11 :       return;
    3614             :     }
    3615             : 
    3616          19 :     case Intrinsic::arm_neon_vld2lane: {
    3617             :       static const uint16_t DOpcodes[] = { ARM::VLD2LNd8Pseudo,
    3618             :                                            ARM::VLD2LNd16Pseudo,
    3619             :                                            ARM::VLD2LNd32Pseudo };
    3620             :       static const uint16_t QOpcodes[] = { ARM::VLD2LNq16Pseudo,
    3621             :                                            ARM::VLD2LNq32Pseudo };
    3622          19 :       SelectVLDSTLane(N, true, false, 2, DOpcodes, QOpcodes);
    3623          19 :       return;
    3624             :     }
    3625             : 
    3626          18 :     case Intrinsic::arm_neon_vld3lane: {
    3627             :       static const uint16_t DOpcodes[] = { ARM::VLD3LNd8Pseudo,
    3628             :                                            ARM::VLD3LNd16Pseudo,
    3629             :                                            ARM::VLD3LNd32Pseudo };
    3630             :       static const uint16_t QOpcodes[] = { ARM::VLD3LNq16Pseudo,
    3631             :                                            ARM::VLD3LNq32Pseudo };
    3632          18 :       SelectVLDSTLane(N, true, false, 3, DOpcodes, QOpcodes);
    3633          18 :       return;
    3634             :     }
    3635             : 
    3636          15 :     case Intrinsic::arm_neon_vld4lane: {
    3637             :       static const uint16_t DOpcodes[] = { ARM::VLD4LNd8Pseudo,
    3638             :                                            ARM::VLD4LNd16Pseudo,
    3639             :                                            ARM::VLD4LNd32Pseudo };
    3640             :       static const uint16_t QOpcodes[] = { ARM::VLD4LNq16Pseudo,
    3641             :                                            ARM::VLD4LNq32Pseudo };
    3642          15 :       SelectVLDSTLane(N, true, false, 4, DOpcodes, QOpcodes);
    3643          15 :       return;
    3644             :     }
    3645             : 
    3646          37 :     case Intrinsic::arm_neon_vst1: {
    3647             :       static const uint16_t DOpcodes[] = { ARM::VST1d8, ARM::VST1d16,
    3648             :                                            ARM::VST1d32, ARM::VST1d64 };
    3649             :       static const uint16_t QOpcodes[] = { ARM::VST1q8, ARM::VST1q16,
    3650             :                                            ARM::VST1q32, ARM::VST1q64 };
    3651          37 :       SelectVST(N, false, 1, DOpcodes, QOpcodes, nullptr);
    3652          37 :       return;
    3653             :     }
    3654             : 
    3655          20 :     case Intrinsic::arm_neon_vst2: {
    3656             :       static const uint16_t DOpcodes[] = { ARM::VST2d8, ARM::VST2d16,
    3657             :                                            ARM::VST2d32, ARM::VST1q64 };
    3658             :       static const uint16_t QOpcodes[] = { ARM::VST2q8Pseudo, ARM::VST2q16Pseudo,
    3659             :                                            ARM::VST2q32Pseudo };
    3660          20 :       SelectVST(N, false, 2, DOpcodes, QOpcodes, nullptr);
    3661          20 :       return;
    3662             :     }
    3663             : 
    3664          19 :     case Intrinsic::arm_neon_vst3: {
    3665             :       static const uint16_t DOpcodes[] = { ARM::VST3d8Pseudo,
    3666             :                                            ARM::VST3d16Pseudo,
    3667             :                                            ARM::VST3d32Pseudo,
    3668             :                                            ARM::VST1d64TPseudo };
    3669             :       static const uint16_t QOpcodes0[] = { ARM::VST3q8Pseudo_UPD,
    3670             :                                             ARM::VST3q16Pseudo_UPD,
    3671             :                                             ARM::VST3q32Pseudo_UPD };
    3672             :       static const uint16_t QOpcodes1[] = { ARM::VST3q8oddPseudo,
    3673             :                                             ARM::VST3q16oddPseudo,
    3674             :                                             ARM::VST3q32oddPseudo };
    3675          19 :       SelectVST(N, false, 3, DOpcodes, QOpcodes0, QOpcodes1);
    3676          19 :       return;
    3677             :     }
    3678             : 
    3679          19 :     case Intrinsic::arm_neon_vst4: {
    3680             :       static const uint16_t DOpcodes[] = { ARM::VST4d8Pseudo,
    3681             :                                            ARM::VST4d16Pseudo,
    3682             :                                            ARM::VST4d32Pseudo,
    3683             :                                            ARM::VST1d64QPseudo };
    3684             :       static const uint16_t QOpcodes0[] = { ARM::VST4q8Pseudo_UPD,
    3685             :                                             ARM::VST4q16Pseudo_UPD,
    3686             :                                             ARM::VST4q32Pseudo_UPD };
    3687             :       static const uint16_t QOpcodes1[] = { ARM::VST4q8oddPseudo,
    3688             :                                             ARM::VST4q16oddPseudo,
    3689             :                                             ARM::VST4q32oddPseudo };
    3690          19 :       SelectVST(N, false, 4, DOpcodes, QOpcodes0, QOpcodes1);
    3691          19 :       return;
    3692             :     }
    3693             : 
    3694           8 :     case Intrinsic::arm_neon_vst2lane: {
    3695             :       static const uint16_t DOpcodes[] = { ARM::VST2LNd8Pseudo,
    3696             :                                            ARM::VST2LNd16Pseudo,
    3697             :                                            ARM::VST2LNd32Pseudo };
    3698             :       static const uint16_t QOpcodes[] = { ARM::VST2LNq16Pseudo,
    3699             :                                            ARM::VST2LNq32Pseudo };
    3700           8 :       SelectVLDSTLane(N, false, false, 2, DOpcodes, QOpcodes);
    3701           8 :       return;
    3702             :     }
    3703             : 
    3704           8 :     case Intrinsic::arm_neon_vst3lane: {
    3705             :       static const uint16_t DOpcodes[] = { ARM::VST3LNd8Pseudo,
    3706             :                                            ARM::VST3LNd16Pseudo,
    3707             :                                            ARM::VST3LNd32Pseudo };
    3708             :       static const uint16_t QOpcodes[] = { ARM::VST3LNq16Pseudo,
    3709             :                                            ARM::VST3LNq32Pseudo };
    3710           8 :       SelectVLDSTLane(N, false, false, 3, DOpcodes, QOpcodes);
    3711           8 :       return;
    3712             :     }
    3713             : 
    3714           8 :     case Intrinsic::arm_neon_vst4lane: {
    3715             :       static const uint16_t DOpcodes[] = { ARM::VST4LNd8Pseudo,
    3716             :                                            ARM::VST4LNd16Pseudo,
    3717             :                                            ARM::VST4LNd32Pseudo };
    3718             :       static const uint16_t QOpcodes[] = { ARM::VST4LNq16Pseudo,
    3719             :                                            ARM::VST4LNq32Pseudo };
    3720           8 :       SelectVLDSTLane(N, false, false, 4, DOpcodes, QOpcodes);
    3721           8 :       return;
    3722             :     }
    3723             :     }
    3724             :     break;
    3725             :   }
    3726             : 
    3727           6 :   case ISD::ATOMIC_CMP_SWAP:
    3728           6 :     SelectCMP_SWAP(N);
    3729           6 :     return;
    3730             :   }
    3731             : 
    3732      251850 :   SelectCode(N);
    3733             : }
    3734             : 
    3735             : // Inspect a register string of the form
    3736             : // cp<coprocessor>:<opc1>:c<CRn>:c<CRm>:<opc2> (32bit) or
    3737             : // cp<coprocessor>:<opc1>:c<CRm> (64bit) inspect the fields of the string
    3738             : // and obtain the integer operands from them, adding these operands to the
    3739             : // provided vector.
    3740         230 : static void getIntOperandsFromRegisterString(StringRef RegString,
    3741             :                                              SelectionDAG *CurDAG,
    3742             :                                              const SDLoc &DL,
    3743             :                                              std::vector<SDValue> &Ops) {
    3744         460 :   SmallVector<StringRef, 5> Fields;
    3745         230 :   RegString.split(Fields, ':');
    3746             : 
    3747         230 :   if (Fields.size() > 1) {
    3748             :     bool AllIntFields = true;
    3749             : 
    3750          72 :     for (StringRef Field : Fields) {
    3751             :       // Need to trim out leading 'cp' characters and get the integer field.
    3752             :       unsigned IntField;
    3753          96 :       AllIntFields &= !Field.trim("CPcp").getAsInteger(10, IntField);
    3754          96 :       Ops.push_back(CurDAG->getTargetConstant(IntField, DL, MVT::i32));
    3755             :     }
    3756             : 
    3757             :     assert(AllIntFields &&
    3758             :             "Unexpected non-integer value in special register string.");
    3759             :   }
    3760         230 : }
    3761             : 
    3762             : // Maps a Banked Register string to its mask value. The mask value returned is
    3763             : // for use in the MRSbanked / MSRbanked instruction nodes as the Banked Register
    3764             : // mask operand, which expresses which register is to be used, e.g. r8, and in
    3765             : // which mode it is to be used, e.g. usr. Returns -1 to signify that the string
    3766             : // was invalid.
    3767         222 : static inline int getBankedRegisterMask(StringRef RegString) {
    3768         666 :   auto TheReg = ARMBankedReg::lookupBankedRegByName(RegString.lower());
    3769         222 :   if (!TheReg)
    3770             :      return -1;
    3771           0 :   return TheReg->Encoding;
    3772             : }
    3773             : 
    3774             : // The flags here are common to those allowed for apsr in the A class cores and
    3775             : // those allowed for the special registers in the M class cores. Returns a
    3776             : // value representing which flags were present, -1 if invalid.
    3777          12 : static inline int getMClassFlagsMask(StringRef Flags) {
    3778          12 :   return StringSwitch<int>(Flags)
    3779          36 :           .Case("", 0x2) // no flags means nzcvq for psr registers, and 0x2 is
    3780             :                          // correct when flags are not permitted
    3781          36 :           .Case("g", 0x1)
    3782          36 :           .Case("nzcvq", 0x2)
    3783          36 :           .Case("nzcvqg", 0x3)
    3784          36 :           .Default(-1);
    3785             : }
    3786             : 
    3787             : // Maps MClass special registers string to its value for use in the
    3788             : // t2MRS_M/t2MSR_M instruction nodes as the SYSm value operand.
    3789             : // Returns -1 to signify that the string was invalid.
    3790         182 : static int getMClassRegisterMask(StringRef Reg, const ARMSubtarget *Subtarget) {
    3791         182 :   auto TheReg = ARMSysReg::lookupMClassSysRegByName(Reg);
    3792         182 :   const FeatureBitset &FeatureBits = Subtarget->getFeatureBits();
    3793         363 :   if (!TheReg || !TheReg->hasRequiredFeatures(FeatureBits))
    3794             :     return -1;
    3795         178 :   return (int)(TheReg->Encoding & 0xFFF); // SYSm value
    3796             : }
    3797             : 
    3798          24 : static int getARClassRegisterMask(StringRef Reg, StringRef Flags) {
    3799             :   // The mask operand contains the special register (R Bit) in bit 4, whether
    3800             :   // the register is spsr (R bit is 1) or one of cpsr/apsr (R bit is 0), and
    3801             :   // bits 3-0 contains the fields to be accessed in the special register, set by
    3802             :   // the flags provided with the register.
    3803          24 :   int Mask = 0;
    3804          36 :   if (Reg == "apsr") {
    3805             :     // The flags permitted for apsr are the same flags that are allowed in
    3806             :     // M class registers. We get the flag value and then shift the flags into
    3807             :     // the correct place to combine with the mask.
    3808          12 :     Mask = getMClassFlagsMask(Flags);
    3809          12 :     if (Mask == -1)
    3810             :       return -1;
    3811          12 :     return Mask << 2;
    3812             :   }
    3813             : 
    3814          19 :   if (Reg != "cpsr" && Reg != "spsr") {
    3815             :     return -1;
    3816             :   }
    3817             : 
    3818             :   // This is the same as if the flags were "fc"
    3819          10 :   if (Flags.empty() || Flags == "all")
    3820             :     return Mask | 0x9;
    3821             : 
    3822             :   // Inspect the supplied flags string and set the bits in the mask for
    3823             :   // the relevant and valid flags allowed for cpsr and spsr.
    3824          36 :   for (char Flag : Flags) {
    3825             :     int FlagVal;
    3826          16 :     switch (Flag) {
    3827             :       case 'c':
    3828             :         FlagVal = 0x1;
    3829             :         break;
    3830             :       case 'x':
    3831           4 :         FlagVal = 0x2;
    3832             :         break;
    3833             :       case 's':
    3834           4 :         FlagVal = 0x4;
    3835             :         break;
    3836             :       case 'f':
    3837             :         FlagVal = 0x8;
    3838             :         break;
    3839             :       default:
    3840             :         FlagVal = 0;
    3841             :     }
    3842             : 
    3843             :     // This avoids allowing strings where the same flag bit appears twice.
    3844          16 :     if (!FlagVal || (Mask & FlagVal))
    3845             :       return -1;
    3846          16 :     Mask |= FlagVal;
    3847             :   }
    3848             : 
    3849             :   // If the register is spsr then we need to set the R bit.
    3850          15 :   if (Reg == "spsr")
    3851           5 :     Mask |= 0x10;
    3852             : 
    3853             :   return Mask;
    3854             : }
    3855             : 
    3856             : // Lower the read_register intrinsic to ARM specific DAG nodes
    3857             : // using the supplied metadata string to select the instruction node to use
    3858             : // and the registers/masks to construct as operands for the node.
    3859          95 : bool ARMDAGToDAGISel::tryReadRegister(SDNode *N){
    3860         285 :   const MDNodeSDNode *MD = dyn_cast<MDNodeSDNode>(N->getOperand(1));
    3861         285 :   const MDString *RegString = dyn_cast<MDString>(MD->getMD()->getOperand(0));
    3862         190 :   bool IsThumb2 = Subtarget->isThumb2();
    3863         190 :   SDLoc DL(N);
    3864             : 
    3865         190 :   std::vector<SDValue> Ops;
    3866          95 :   getIntOperandsFromRegisterString(RegString->getString(), CurDAG, DL, Ops);
    3867             : 
    3868          95 :   if (!Ops.empty()) {
    3869             :     // If the special register string was constructed of fields (as defined
    3870             :     // in the ACLE) then need to lower to MRC node (32 bit) or
    3871             :     // MRRC node(64 bit), we can make the distinction based on the number of
    3872             :     // operands we have.
    3873             :     unsigned Opcode;
    3874           8 :     SmallVector<EVT, 3> ResTypes;
    3875           8 :     if (Ops.size() == 5){
    3876           2 :       Opcode = IsThumb2 ? ARM::t2MRC : ARM::MRC;
    3877           2 :       ResTypes.append({ MVT::i32, MVT::Other });
    3878             :     } else {
    3879             :       assert(Ops.size() == 3 &&
    3880             :               "Invalid number of fields in special register string.");
    3881           2 :       Opcode = IsThumb2 ? ARM::t2MRRC : ARM::MRRC;
    3882           2 :       ResTypes.append({ MVT::i32, MVT::i32, MVT::Other });
    3883             :     }
    3884             : 
    3885           8 :     Ops.push_back(getAL(CurDAG, DL));
    3886          12 :     Ops.push_back(CurDAG->getRegister(0, MVT::i32));
    3887           8 :     Ops.push_back(N->getOperand(0));
    3888          16 :     ReplaceNode(N, CurDAG->getMachineNode(Opcode, DL, ResTypes, Ops));
    3889           4 :     return true;
    3890             :   }
    3891             : 
    3892          91 :   std::string SpecialReg = RegString->getString().lower();
    3893             : 
    3894          91 :   int BankedReg = getBankedRegisterMask(SpecialReg);
    3895          91 :   if (BankedReg != -1) {
    3896           0 :     Ops = { CurDAG->getTargetConstant(BankedReg, DL, MVT::i32),
    3897           0 :             getAL(CurDAG, DL), CurDAG->getRegister(0, MVT::i32),
    3898           0 :             N->getOperand(0) };
    3899           0 :     ReplaceNode(
    3900           0 :         N, CurDAG->getMachineNode(IsThumb2 ? ARM::t2MRSbanked : ARM::MRSbanked,
    3901           0 :                                   DL, MVT::i32, MVT::Other, Ops));
    3902           0 :     return true;
    3903             :   }
    3904             : 
    3905             :   // The VFP registers are read by creating SelectionDAG nodes with opcodes
    3906             :   // corresponding to the register that is being read from. So we switch on the
    3907             :   // string to find which opcode we need to use.
    3908          91 :   unsigned Opcode = StringSwitch<unsigned>(SpecialReg)
    3909         273 :                     .Case("fpscr", ARM::VMRS)
    3910         273 :                     .Case("fpexc", ARM::VMRS_FPEXC)
    3911         273 :                     .Case("fpsid", ARM::VMRS_FPSID)
    3912         273 :                     .Case("mvfr0", ARM::VMRS_MVFR0)
    3913         273 :                     .Case("mvfr1", ARM::VMRS_MVFR1)
    3914         273 :                     .Case("mvfr2", ARM::VMRS_MVFR2)
    3915         273 :                     .Case("fpinst", ARM::VMRS_FPINST)
    3916         273 :                     .Case("fpinst2", ARM::VMRS_FPINST2)
    3917         182 :                     .Default(0);
    3918             : 
    3919             :   // If an opcode was found then we can lower the read to a VFP instruction.
    3920           2 :   if (Opcode) {
    3921           2 :     if (!Subtarget->hasVFP2())
    3922             :       return false;
    3923           2 :     if (Opcode == ARM::VMRS_MVFR2 && !Subtarget->hasFPARMv8())
    3924             :       return false;
    3925             : 
    3926           8 :     Ops = { getAL(CurDAG, DL), CurDAG->getRegister(0, MVT::i32),
    3927           4 :             N->getOperand(0) };
    3928           6 :     ReplaceNode(N,
    3929          10 :                 CurDAG->getMachineNode(Opcode, DL, MVT::i32, MVT::Other, Ops));
    3930           2 :     return true;
    3931             :   }
    3932             : 
    3933             :   // If the target is M Class then need to validate that the register string
    3934             :   // is an acceptable value, so check that a mask can be constructed from the
    3935             :   // string.
    3936          89 :   if (Subtarget->isMClass()) {
    3937         154 :     int SYSmValue = getMClassRegisterMask(SpecialReg, Subtarget);
    3938          77 :     if (SYSmValue == -1)
    3939             :       return false;
    3940             : 
    3941         148 :     SDValue Ops[] = { CurDAG->getTargetConstant(SYSmValue, DL, MVT::i32),
    3942         296 :                       getAL(CurDAG, DL), CurDAG->getRegister(0, MVT::i32),
    3943         370 :                       N->getOperand(0) };
    3944         222 :     ReplaceNode(
    3945         370 :         N, CurDAG->getMachineNode(ARM::t2MRS_M, DL, MVT::i32, MVT::Other, Ops));
    3946          74 :     return true;
    3947             :   }
    3948             : 
    3949             :   // Here we know the target is not M Class so we need to check if it is one
    3950             :   // of the remaining possible values which are apsr, cpsr or spsr.
    3951          22 :   if (SpecialReg == "apsr" || SpecialReg == "cpsr") {
    3952          16 :     Ops = { getAL(CurDAG, DL), CurDAG->getRegister(0, MVT::i32),
    3953           8 :             N->getOperand(0) };
    3954          16 :     ReplaceNode(N, CurDAG->getMachineNode(IsThumb2 ? ARM::t2MRS_AR : ARM::MRS,
    3955           8 :                                           DL, MVT::i32, MVT::Other, Ops));
    3956           4 :     return true;
    3957             :   }
    3958             : 
    3959           8 :   if (SpecialReg == "spsr") {
    3960           4 :     Ops = { getAL(CurDAG, DL), CurDAG->getRegister(0, MVT::i32),
    3961           2 :             N->getOperand(0) };
    3962           3 :     ReplaceNode(
    3963           4 :         N, CurDAG->getMachineNode(IsThumb2 ? ARM::t2MRSsys_AR : ARM::MRSsys, DL,
    3964           2 :                                   MVT::i32, MVT::Other, Ops));
    3965           1 :     return true;
    3966             :   }
    3967             : 
    3968             :   return false;
    3969             : }
    3970             : 
    3971             : // Lower the write_register intrinsic to ARM specific DAG nodes
    3972             : // using the supplied metadata string to select the instruction node to use
    3973             : // and the registers/masks to use in the nodes
    3974         135 : bool ARMDAGToDAGISel::tryWriteRegister(SDNode *N){
    3975         405 :   const MDNodeSDNode *MD = dyn_cast<MDNodeSDNode>(N->getOperand(1));
    3976         405 :   const MDString *RegString = dyn_cast<MDString>(MD->getMD()->getOperand(0));
    3977         270 :   bool IsThumb2 = Subtarget->isThumb2();
    3978         270 :   SDLoc DL(N);
    3979             : 
    3980         270 :   std::vector<SDValue> Ops;
    3981         135 :   getIntOperandsFromRegisterString(RegString->getString(), CurDAG, DL, Ops);
    3982             : 
    3983         135 :   if (!Ops.empty()) {
    3984             :     // If the special register string was constructed of fields (as defined
    3985             :     // in the ACLE) then need to lower to MCR node (32 bit) or
    3986             :     // MCRR node(64 bit), we can make the distinction based on the number of
    3987             :     // operands we have.
    3988             :     unsigned Opcode;
    3989           8 :     if (Ops.size() == 5) {
    3990           2 :       Opcode = IsThumb2 ? ARM::t2MCR : ARM::MCR;
    3991          10 :       Ops.insert(Ops.begin()+2, N->getOperand(2));
    3992             :     } else {
    3993             :       assert(Ops.size() == 3 &&
    3994             :               "Invalid number of fields in special register string.");
    3995           2 :       Opcode = IsThumb2 ? ARM::t2MCRR : ARM::MCRR;
    3996           6 :       SDValue WriteValue[] = { N->getOperand(2), N->getOperand(3) };
    3997          10 :       Ops.insert(Ops.begin()+2, WriteValue, WriteValue+2);
    3998             :     }
    3999             : 
    4000           8 :     Ops.push_back(getAL(CurDAG, DL));
    4001          12 :     Ops.push_back(CurDAG->getRegister(0, MVT::i32));
    4002           8 :     Ops.push_back(N->getOperand(0));
    4003             : 
    4004          16 :     ReplaceNode(N, CurDAG->getMachineNode(Opcode, DL, MVT::Other, Ops));
    4005           4 :     return true;
    4006             :   }
    4007             : 
    4008         131 :   std::string SpecialReg = RegString->getString().lower();
    4009         131 :   int BankedReg = getBankedRegisterMask(SpecialReg);
    4010         131 :   if (BankedReg != -1) {
    4011           0 :     Ops = { CurDAG->getTargetConstant(BankedReg, DL, MVT::i32), N->getOperand(2),
    4012           0 :             getAL(CurDAG, DL), CurDAG->getRegister(0, MVT::i32),
    4013           0 :             N->getOperand(0) };
    4014           0 :     ReplaceNode(
    4015           0 :         N, CurDAG->getMachineNode(IsThumb2 ? ARM::t2MSRbanked : ARM::MSRbanked,
    4016           0 :                                   DL, MVT::Other, Ops));
    4017           0 :     return true;
    4018             :   }
    4019             : 
    4020             :   // The VFP registers are written to by creating SelectionDAG nodes with
    4021             :   // opcodes corresponding to the register that is being written. So we switch
    4022             :   // on the string to find which opcode we need to use.
    4023         131 :   unsigned Opcode = StringSwitch<unsigned>(SpecialReg)
    4024         393 :                     .Case("fpscr", ARM::VMSR)
    4025         393 :                     .Case("fpexc", ARM::VMSR_FPEXC)
    4026         393 :                     .Case("fpsid", ARM::VMSR_FPSID)
    4027         393 :                     .Case("fpinst", ARM::VMSR_FPINST)
    4028         393 :                     .Case("fpinst2", ARM::VMSR_FPINST2)
    4029         262 :                     .Default(0);
    4030             : 
    4031           2 :   if (Opcode) {
    4032           2 :     if (!Subtarget->hasVFP2())
    4033             :       return false;
    4034          10 :     Ops = { N->getOperand(2), getAL(CurDAG, DL),
    4035          10 :             CurDAG->getRegister(0, MVT::i32), N->getOperand(0) };
    4036           8 :     ReplaceNode(N, CurDAG->getMachineNode(Opcode, DL, MVT::Other, Ops));
    4037           2 :     return true;
    4038             :   }
    4039             : 
    4040         129 :   std::pair<StringRef, StringRef> Fields;
    4041         258 :   Fields = StringRef(SpecialReg).rsplit('_');
    4042         129 :   std::string Reg = Fields.first.str();
    4043         129 :   StringRef Flags = Fields.second;
    4044             : 
    4045             :   // If the target was M Class then need to validate the special register value
    4046             :   // and retrieve the mask for use in the instruction node.
    4047         129 :   if (Subtarget->isMClass()) {
    4048         210 :     int SYSmValue = getMClassRegisterMask(SpecialReg, Subtarget);
    4049         105 :     if (SYSmValue == -1)
    4050             :       return false;
    4051             : 
    4052         208 :     SDValue Ops[] = { CurDAG->getTargetConstant(SYSmValue, DL, MVT::i32),
    4053         312 :                       N->getOperand(2), getAL(CurDAG, DL),
    4054         520 :                       CurDAG->getRegister(0, MVT::i32), N->getOperand(0) };
    4055         416 :     ReplaceNode(N, CurDAG->getMachineNode(ARM::t2MSR_M, DL, MVT::Other, Ops));
    4056         104 :     return true;
    4057             :   }
    4058             : 
    4059             :   // We then check to see if a valid mask can be constructed for one of the
    4060             :   // register string values permitted for the A and R class cores. These values
    4061             :   // are apsr, spsr and cpsr; these are also valid on older cores.
    4062          24 :   int Mask = getARClassRegisterMask(Reg, Flags);
    4063          24 :   if (Mask != -1) {
    4064         176 :     Ops = { CurDAG->getTargetConstant(Mask, DL, MVT::i32), N->getOperand(2),
    4065          88 :             getAL(CurDAG, DL), CurDAG->getRegister(0, MVT::i32),
    4066          44 :             N->getOperand(0) };
    4067          88 :     ReplaceNode(N, CurDAG->getMachineNode(IsThumb2 ? ARM::t2MSR_AR : ARM::MSR,
    4068          22 :                                           DL, MVT::Other, Ops));
    4069          22 :     return true;
    4070             :   }
    4071             : 
    4072             :   return false;
    4073             : }
    4074             : 
    4075        3477 : bool ARMDAGToDAGISel::tryInlineAsm(SDNode *N){
    4076        6954 :   std::vector<SDValue> AsmNodeOperands;
    4077             :   unsigned Flag, Kind;
    4078        3477 :   bool Changed = false;
    4079        6954 :   unsigned NumOps = N->getNumOperands();
    4080             : 
    4081             :   // Normally, i64 data is bounded to two arbitrary GRPs for "%r" constraint.
    4082             :   // However, some instrstions (e.g. ldrexd/strexd in ARM mode) require
    4083             :   // (even/even+1) GPRs and use %n and %Hn to refer to the individual regs
    4084             :   // respectively. Since there is no constraint to explicitly specify a
    4085             :   // reg pair, we use GPRPair reg class for "%r" for 64-bit data. For Thumb,
    4086             :   // the 64-bit data may be referred by H, Q, R modifiers, so we still pack
    4087             :   // them into a GPRPair.
    4088             : 
    4089        6954 :   SDLoc dl(N);
    4090         240 :   SDValue Glue = N->getGluedNode() ? N->getOperand(NumOps-1)
    4091         120 :                                    : SDValue(nullptr,0);
    4092             : 
    4093        6954 :   SmallVector<bool, 8> OpChanged;
    4094             :   // Glue node will be appended late.
    4095       25903 :   for(unsigned i = 0, e = N->getGluedNode() ? NumOps - 1 : NumOps; i < e; ++i) {
    4096       44612 :     SDValue op = N->getOperand(i);
    4097       22306 :     AsmNodeOperands.push_back(op);
    4098             : 
    4099       22306 :     if (i < InlineAsm::Op_FirstOperand)
    4100       36119 :       continue;
    4101             : 
    4102       25194 :     if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(i))) {
    4103        4279 :       Flag = C->getZExtValue();
    4104        4279 :       Kind = InlineAsm::getKind(Flag);
    4105             :     }
    4106             :     else
    4107        4119 :       continue;
    4108             : 
    4109             :     // Immediate operands to inline asm in the SelectionDAG are modeled with
    4110             :     // two operands. The first is a constant of value InlineAsm::Kind_Imm, and
    4111             :     // the second is a constant with the value of the immediate. If we get here
    4112             :     // and we have a Kind_Imm, skip the next operand, and continue.
    4113        4279 :     if (Kind == InlineAsm::Kind_Imm) {
    4114          80 :       SDValue op = N->getOperand(++i);
    4115          40 :       AsmNodeOperands.push_back(op);
    4116          40 :       continue;
    4117             :     }
    4118             : 
    4119        4239 :     unsigned NumRegs = InlineAsm::getNumOperandRegisters(Flag);
    4120        4239 :     if (NumRegs)
    4121        4239 :       OpChanged.push_back(false);
    4122             : 
    4123        4239 :     unsigned DefIdx = 0;
    4124        4239 :     bool IsTiedToChangedOp = false;
    4125             :     // If it's a use that is tied with a previous def, it has no
    4126             :     // reg class constraint.
    4127        4246 :     if (Changed && InlineAsm::isUseOperandTiedToDef(Flag, DefIdx))
    4128          14 :       IsTiedToChangedOp = OpChanged[DefIdx];
    4129             : 
    4130             :     // Memory operands to inline asm in the SelectionDAG are modeled with two
    4131             :     // operands: a constant of value InlineAsm::Kind_Mem followed by the input
    4132             :     // operand. If we get here and we have a Kind_Mem, skip the next operand (so
    4133             :     // it doesn't get misinterpreted), and continue. We do this here because
    4134             :     // it's important to update the OpChanged array correctly before moving on.
    4135        4239 :     if (Kind == InlineAsm::Kind_Mem) {
    4136          50 :       SDValue op = N->getOperand(++i);
    4137          25 :       AsmNodeOperands.push_back(op);
    4138          25 :       continue;
    4139             :     }
    4140             : 
    4141        4214 :     if (Kind != InlineAsm::Kind_RegUse && Kind != InlineAsm::Kind_RegDef
    4142        4214 :         && Kind != InlineAsm::Kind_RegDefEarlyClobber)
    4143        3829 :       continue;
    4144             : 
    4145             :     unsigned RC;
    4146         385 :     bool HasRC = InlineAsm::hasRegClassConstraint(Flag, RC);
    4147         675 :     if ((!IsTiedToChangedOp && (!HasRC || RC != ARM::GPRRegClassID))
    4148         286 :         || NumRegs != 2)
    4149         290 :       continue;
    4150             : 
    4151             :     assert((i+2 < NumOps) && "Invalid number of operands in inline asm");
    4152         190 :     SDValue V0 = N->getOperand(i+1);
    4153         190 :     SDValue V1 = N->getOperand(i+2);
    4154          95 :     unsigned Reg0 = cast<RegisterSDNode>(V0)->getReg();
    4155          95 :     unsigned Reg1 = cast<RegisterSDNode>(V1)->getReg();
    4156          95 :     SDValue PairedReg;
    4157          95 :     MachineRegisterInfo &MRI = MF->getRegInfo();
    4158             : 
    4159          95 :     if (Kind == InlineAsm::Kind_RegDef ||
    4160             :         Kind == InlineAsm::Kind_RegDefEarlyClobber) {
    4161             :       // Replace the two GPRs with 1 GPRPair and copy values from GPRPair to
    4162             :       // the original GPRs.
    4163             : 
    4164          44 :       unsigned GPVR = MRI.createVirtualRegister(&ARM::GPRPairRegClass);
    4165          88 :       PairedReg = CurDAG->getRegister(GPVR, MVT::Untyped);
    4166          44 :       SDValue Chain = SDValue(N,0);
    4167             : 
    4168          44 :       SDNode *GU = N->getGluedUser();
    4169          44 :       SDValue RegCopy = CurDAG->getCopyFromReg(Chain, dl, GPVR, MVT::Untyped,
    4170         132 :                                                Chain.getValue(1));
    4171             : 
    4172             :       // Extract values from a GPRPair reg and copy to the original GPR reg.
    4173          44 :       SDValue Sub0 = CurDAG->getTargetExtractSubreg(ARM::gsub_0, dl, MVT::i32,
    4174          44 :                                                     RegCopy);
    4175          44 :       SDValue Sub1 = CurDAG->getTargetExtractSubreg(ARM::gsub_1, dl, MVT::i32,
    4176          44 :                                                     RegCopy);
    4177          44 :       SDValue T0 = CurDAG->getCopyToReg(Sub0, dl, Reg0, Sub0,
    4178          88 :                                         RegCopy.getValue(1));
    4179          88 :       SDValue T1 = CurDAG->getCopyToReg(Sub1, dl, Reg1, Sub1, T0.getValue(1));
    4180             : 
    4181             :       // Update the original glue user.
    4182         176 :       std::vector<SDValue> Ops(GU->op_begin(), GU->op_end()-1);
    4183         132 :       Ops.push_back(T1.getValue(1));
    4184          88 :       CurDAG->UpdateNodeOperands(GU, Ops);
    4185             :     }
    4186             :     else {
    4187             :       // For Kind  == InlineAsm::Kind_RegUse, we first copy two GPRs into a
    4188             :       // GPRPair and then pass the GPRPair to the inline asm.
    4189          51 :       SDValue Chain = AsmNodeOperands[InlineAsm::Op_InputChain];
    4190             : 
    4191             :       // As REG_SEQ doesn't take RegisterSDNode, we copy them first.
    4192          51 :       SDValue T0 = CurDAG->getCopyFromReg(Chain, dl, Reg0, MVT::i32,
    4193         153 :                                           Chain.getValue(1));
    4194          51 :       SDValue T1 = CurDAG->getCopyFromReg(Chain, dl, Reg1, MVT::i32,
    4195         153 :                                           T0.getValue(1));
    4196         102 :       SDValue Pair = SDValue(createGPRPairNode(MVT::Untyped, T0, T1), 0);
    4197             : 
    4198             :       // Copy REG_SEQ into a GPRPair-typed VR and replace the original two
    4199             :       // i32 VRs of inline asm with it.
    4200          51 :       unsigned GPVR = MRI.createVirtualRegister(&ARM::GPRPairRegClass);
    4201         102 :       PairedReg = CurDAG->getRegister(GPVR, MVT::Untyped);
    4202         102 :       Chain = CurDAG->getCopyToReg(T1, dl, GPVR, Pair, T1.getValue(1));
    4203             : 
    4204          51 :       AsmNodeOperands[InlineAsm::Op_InputChain] = Chain;
    4205         102 :       Glue = Chain.getValue(1);
    4206             :     }
    4207             : 
    4208          95 :     Changed = true;
    4209             : 
    4210          95 :     if(PairedReg.getNode()) {
    4211         190 :       OpChanged[OpChanged.size() -1 ] = true;
    4212          95 :       Flag = InlineAsm::getFlagWord(Kind, 1 /* RegNum*/);
    4213          95 :       if (IsTiedToChangedOp)
    4214          14 :         Flag = InlineAsm::getFlagWordForMatchingOp(Flag, DefIdx);
    4215             :       else
    4216          88 :         Flag = InlineAsm::getFlagWordForRegClass(Flag, ARM::GPRPairRegClassID);
    4217             :       // Replace the current flag.
    4218         380 :       AsmNodeOperands[AsmNodeOperands.size() -1] = CurDAG->getTargetConstant(
    4219          95 :           Flag, dl, MVT::i32);
    4220             :       // Add the new register node and skip the original two GPRs.
    4221          95 :       AsmNodeOperands.push_back(PairedReg);
    4222             :       // Skip the next two GPRs.
    4223          95 :       i += 2;
    4224             :     }
    4225             :   }
    4226             : 
    4227        3477 :   if (Glue.getNode())
    4228         120 :     AsmNodeOperands.push_back(Glue);
    4229        3477 :   if (!Changed)
    4230             :     return false;
    4231             : 
    4232         147 :   SDValue New = CurDAG->getNode(ISD::INLINEASM, SDLoc(N),
    4233         245 :       CurDAG->getVTList(MVT::Other, MVT::Glue), AsmNodeOperands);
    4234          98 :   New->setNodeId(-1);
    4235          98 :   ReplaceNode(N, New.getNode());
    4236          49 :   return true;
    4237             : }
    4238             : 
    4239             : 
    4240          18 : bool ARMDAGToDAGISel::
    4241             : SelectInlineAsmMemoryOperand(const SDValue &Op, unsigned ConstraintID,
    4242             :                              std::vector<SDValue> &OutOps) {
    4243             :   switch(ConstraintID) {
    4244           0 :   default:
    4245           0 :     llvm_unreachable("Unexpected asm memory constraint");
    4246          18 :   case InlineAsm::Constraint_i:
    4247             :     // FIXME: It seems strange that 'i' is needed here since it's supposed to
    4248             :     //        be an immediate and not a memory constraint.
    4249             :     LLVM_FALLTHROUGH;
    4250             :   case InlineAsm::Constraint_m:
    4251             :   case InlineAsm::Constraint_o:
    4252             :   case InlineAsm::Constraint_Q:
    4253             :   case InlineAsm::Constraint_Um:
    4254             :   case InlineAsm::Constraint_Un:
    4255             :   case InlineAsm::Constraint_Uq:
    4256             :   case InlineAsm::Constraint_Us:
    4257             :   case InlineAsm::Constraint_Ut:
    4258             :   case InlineAsm::Constraint_Uv:
    4259             :   case InlineAsm::Constraint_Uy:
    4260             :     // Require the address to be in a register.  That is safe for all ARM
    4261             :     // variants and it is hard to do anything much smarter without knowing
    4262             :     // how the operand is used.
    4263          18 :     OutOps.push_back(Op);
    4264             :     return false;
    4265             :   }
    4266             :   return true;
    4267             : }
    4268             : 
    4269             : /// createARMISelDag - This pass converts a legalized DAG into a
    4270             : /// ARM-specific DAG, ready for instruction scheduling.
    4271             : ///
    4272        2551 : FunctionPass *llvm::createARMISelDag(ARMBaseTargetMachine &TM,
    4273             :                                      CodeGenOpt::Level OptLevel) {
    4274        5102 :   return new ARMDAGToDAGISel(TM, OptLevel);
    4275      216918 : }

Generated by: LCOV version 1.13