LCOV - code coverage report
Current view: top level - lib/Target/ARM - ARMExpandPseudoInsts.cpp (source / functions) Hit Total Coverage
Test: llvm-toolchain.info Lines: 440 718 61.3 %
Date: 2018-09-23 13:06:45 Functions: 13 20 65.0 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : //===-- ARMExpandPseudoInsts.cpp - Expand pseudo instructions -------------===//
       2             : //
       3             : //                     The LLVM Compiler Infrastructure
       4             : //
       5             : // This file is distributed under the University of Illinois Open Source
       6             : // License. See LICENSE.TXT for details.
       7             : //
       8             : //===----------------------------------------------------------------------===//
       9             : //
      10             : // This file contains a pass that expands pseudo instructions into target
      11             : // instructions to allow proper scheduling, if-conversion, and other late
      12             : // optimizations. This pass should be run after register allocation but before
      13             : // the post-regalloc scheduling pass.
      14             : //
      15             : //===----------------------------------------------------------------------===//
      16             : 
      17             : #include "ARM.h"
      18             : #include "ARMBaseInstrInfo.h"
      19             : #include "ARMBaseRegisterInfo.h"
      20             : #include "ARMConstantPoolValue.h"
      21             : #include "ARMMachineFunctionInfo.h"
      22             : #include "ARMSubtarget.h"
      23             : #include "MCTargetDesc/ARMAddressingModes.h"
      24             : #include "llvm/CodeGen/LivePhysRegs.h"
      25             : #include "llvm/CodeGen/MachineFrameInfo.h"
      26             : #include "llvm/CodeGen/MachineFunctionPass.h"
      27             : 
      28             : using namespace llvm;
      29             : 
      30             : #define DEBUG_TYPE "arm-pseudo"
      31             : 
      32             : static cl::opt<bool>
      33             : VerifyARMPseudo("verify-arm-pseudo-expand", cl::Hidden,
      34             :                 cl::desc("Verify machine code after expanding ARM pseudos"));
      35             : 
      36             : #define ARM_EXPAND_PSEUDO_NAME "ARM pseudo instruction expansion pass"
      37             : 
      38             : namespace {
      39             :   class ARMExpandPseudo : public MachineFunctionPass {
      40             :   public:
      41             :     static char ID;
      42        2817 :     ARMExpandPseudo() : MachineFunctionPass(ID) {}
      43             : 
      44             :     const ARMBaseInstrInfo *TII;
      45             :     const TargetRegisterInfo *TRI;
      46             :     const ARMSubtarget *STI;
      47             :     ARMFunctionInfo *AFI;
      48             : 
      49             :     bool runOnMachineFunction(MachineFunction &Fn) override;
      50             : 
      51        2804 :     MachineFunctionProperties getRequiredProperties() const override {
      52        2804 :       return MachineFunctionProperties().set(
      53        2804 :           MachineFunctionProperties::Property::NoVRegs);
      54             :     }
      55             : 
      56        2803 :     StringRef getPassName() const override {
      57        2803 :       return ARM_EXPAND_PSEUDO_NAME;
      58             :     }
      59             : 
      60             :   private:
      61             :     void TransferImpOps(MachineInstr &OldMI,
      62             :                         MachineInstrBuilder &UseMI, MachineInstrBuilder &DefMI);
      63             :     bool ExpandMI(MachineBasicBlock &MBB,
      64             :                   MachineBasicBlock::iterator MBBI,
      65             :                   MachineBasicBlock::iterator &NextMBBI);
      66             :     bool ExpandMBB(MachineBasicBlock &MBB);
      67             :     void ExpandVLD(MachineBasicBlock::iterator &MBBI);
      68             :     void ExpandVST(MachineBasicBlock::iterator &MBBI);
      69             :     void ExpandLaneOp(MachineBasicBlock::iterator &MBBI);
      70             :     void ExpandVTBL(MachineBasicBlock::iterator &MBBI,
      71             :                     unsigned Opc, bool IsExt);
      72             :     void ExpandMOV32BitImm(MachineBasicBlock &MBB,
      73             :                            MachineBasicBlock::iterator &MBBI);
      74             :     bool ExpandCMP_SWAP(MachineBasicBlock &MBB,
      75             :                         MachineBasicBlock::iterator MBBI, unsigned LdrexOp,
      76             :                         unsigned StrexOp, unsigned UxtOp,
      77             :                         MachineBasicBlock::iterator &NextMBBI);
      78             : 
      79             :     bool ExpandCMP_SWAP_64(MachineBasicBlock &MBB,
      80             :                            MachineBasicBlock::iterator MBBI,
      81             :                            MachineBasicBlock::iterator &NextMBBI);
      82             :   };
      83             :   char ARMExpandPseudo::ID = 0;
      84             : }
      85             : 
      86      217248 : INITIALIZE_PASS(ARMExpandPseudo, DEBUG_TYPE, ARM_EXPAND_PSEUDO_NAME, false,
      87             :                 false)
      88             : 
      89             : /// TransferImpOps - Transfer implicit operands on the pseudo instruction to
      90             : /// the instructions created from the expansion.
      91           0 : void ARMExpandPseudo::TransferImpOps(MachineInstr &OldMI,
      92             :                                      MachineInstrBuilder &UseMI,
      93             :                                      MachineInstrBuilder &DefMI) {
      94           0 :   const MCInstrDesc &Desc = OldMI.getDesc();
      95           0 :   for (unsigned i = Desc.getNumOperands(), e = OldMI.getNumOperands();
      96           0 :        i != e; ++i) {
      97           0 :     const MachineOperand &MO = OldMI.getOperand(i);
      98             :     assert(MO.isReg() && MO.getReg());
      99           0 :     if (MO.isUse())
     100             :       UseMI.add(MO);
     101             :     else
     102             :       DefMI.add(MO);
     103             :   }
     104           0 : }
     105             : 
     106             : namespace {
     107             :   // Constants for register spacing in NEON load/store instructions.
     108             :   // For quad-register load-lane and store-lane pseudo instructors, the
     109             :   // spacing is initially assumed to be EvenDblSpc, and that is changed to
     110             :   // OddDblSpc depending on the lane number operand.
     111             :   enum NEONRegSpacing {
     112             :     SingleSpc,
     113             :     SingleLowSpc ,  // Single spacing, low registers, three and four vectors.
     114             :     SingleHighQSpc, // Single spacing, high registers, four vectors.
     115             :     SingleHighTSpc, // Single spacing, high registers, three vectors.
     116             :     EvenDblSpc,
     117             :     OddDblSpc
     118             :   };
     119             : 
     120             :   // Entries for NEON load/store information table.  The table is sorted by
     121             :   // PseudoOpc for fast binary-search lookups.
     122             :   struct NEONLdStTableEntry {
     123             :     uint16_t PseudoOpc;
     124             :     uint16_t RealOpc;
     125             :     bool IsLoad;
     126             :     bool isUpdating;
     127             :     bool hasWritebackOperand;
     128             :     uint8_t RegSpacing; // One of type NEONRegSpacing
     129             :     uint8_t NumRegs; // D registers loaded or stored
     130             :     uint8_t RegElts; // elements per D register; used for lane ops
     131             :     // FIXME: Temporary flag to denote whether the real instruction takes
     132             :     // a single register (like the encoding) or all of the registers in
     133             :     // the list (like the asm syntax and the isel DAG). When all definitions
     134             :     // are converted to take only the single encoded register, this will
     135             :     // go away.
     136             :     bool copyAllListRegs;
     137             : 
     138             :     // Comparison methods for binary search of the table.
     139             :     bool operator<(const NEONLdStTableEntry &TE) const {
     140             :       return PseudoOpc < TE.PseudoOpc;
     141             :     }
     142           0 :     friend bool operator<(const NEONLdStTableEntry &TE, unsigned PseudoOpc) {
     143        3351 :       return TE.PseudoOpc < PseudoOpc;
     144             :     }
     145             :     friend bool LLVM_ATTRIBUTE_UNUSED operator<(unsigned PseudoOpc,
     146             :                                                 const NEONLdStTableEntry &TE) {
     147             :       return PseudoOpc < TE.PseudoOpc;
     148             :     }
     149             :   };
     150             : }
     151             : 
     152             : static const NEONLdStTableEntry NEONLdStTable[] = {
     153             : { ARM::VLD1LNq16Pseudo,     ARM::VLD1LNd16,     true, false, false, EvenDblSpc, 1, 4 ,true},
     154             : { ARM::VLD1LNq16Pseudo_UPD, ARM::VLD1LNd16_UPD, true, true, true,  EvenDblSpc, 1, 4 ,true},
     155             : { ARM::VLD1LNq32Pseudo,     ARM::VLD1LNd32,     true, false, false, EvenDblSpc, 1, 2 ,true},
     156             : { ARM::VLD1LNq32Pseudo_UPD, ARM::VLD1LNd32_UPD, true, true, true,  EvenDblSpc, 1, 2 ,true},
     157             : { ARM::VLD1LNq8Pseudo,      ARM::VLD1LNd8,      true, false, false, EvenDblSpc, 1, 8 ,true},
     158             : { ARM::VLD1LNq8Pseudo_UPD,  ARM::VLD1LNd8_UPD, true, true, true,  EvenDblSpc, 1, 8 ,true},
     159             : 
     160             : { ARM::VLD1d16QPseudo,      ARM::VLD1d16Q,     true,  false, false, SingleSpc,  4, 4 ,false},
     161             : { ARM::VLD1d16TPseudo,      ARM::VLD1d16T,     true,  false, false, SingleSpc,  3, 4 ,false},
     162             : { ARM::VLD1d32QPseudo,      ARM::VLD1d32Q,     true,  false, false, SingleSpc,  4, 2 ,false},
     163             : { ARM::VLD1d32TPseudo,      ARM::VLD1d32T,     true,  false, false, SingleSpc,  3, 2 ,false},
     164             : { ARM::VLD1d64QPseudo,      ARM::VLD1d64Q,     true,  false, false, SingleSpc,  4, 1 ,false},
     165             : { ARM::VLD1d64QPseudoWB_fixed,  ARM::VLD1d64Qwb_fixed,   true,  true, false, SingleSpc,  4, 1 ,false},
     166             : { ARM::VLD1d64QPseudoWB_register,  ARM::VLD1d64Qwb_register,   true,  true, true, SingleSpc,  4, 1 ,false},
     167             : { ARM::VLD1d64TPseudo,      ARM::VLD1d64T,     true,  false, false, SingleSpc,  3, 1 ,false},
     168             : { ARM::VLD1d64TPseudoWB_fixed,  ARM::VLD1d64Twb_fixed,   true,  true, false, SingleSpc,  3, 1 ,false},
     169             : { ARM::VLD1d64TPseudoWB_register,  ARM::VLD1d64Twb_register, true, true, true,  SingleSpc,  3, 1 ,false},
     170             : { ARM::VLD1d8QPseudo,       ARM::VLD1d8Q,      true,  false, false, SingleSpc,  4, 8 ,false},
     171             : { ARM::VLD1d8TPseudo,       ARM::VLD1d8T,      true,  false, false, SingleSpc,  3, 8 ,false},
     172             : { ARM::VLD1q16HighQPseudo,  ARM::VLD1d16Q,     true,  false, false, SingleHighQSpc,  4, 4 ,false},
     173             : { ARM::VLD1q16HighTPseudo,  ARM::VLD1d16T,     true,  false, false, SingleHighTSpc,  3, 4 ,false},
     174             : { ARM::VLD1q16LowQPseudo_UPD,  ARM::VLD1d16Qwb_fixed,   true,  true, true, SingleLowSpc,  4, 4 ,false},
     175             : { ARM::VLD1q16LowTPseudo_UPD,  ARM::VLD1d16Twb_fixed,   true,  true, true, SingleLowSpc,  3, 4 ,false},
     176             : { ARM::VLD1q32HighQPseudo,  ARM::VLD1d32Q,     true,  false, false, SingleHighQSpc,  4, 2 ,false},
     177             : { ARM::VLD1q32HighTPseudo,  ARM::VLD1d32T,     true,  false, false, SingleHighTSpc,  3, 2 ,false},
     178             : { ARM::VLD1q32LowQPseudo_UPD,  ARM::VLD1d32Qwb_fixed,   true,  true, true, SingleLowSpc,  4, 2 ,false},
     179             : { ARM::VLD1q32LowTPseudo_UPD,  ARM::VLD1d32Twb_fixed,   true,  true, true, SingleLowSpc,  3, 2 ,false},
     180             : { ARM::VLD1q64HighQPseudo,  ARM::VLD1d64Q,     true,  false, false, SingleHighQSpc,  4, 1 ,false},
     181             : { ARM::VLD1q64HighTPseudo,  ARM::VLD1d64T,     true,  false, false, SingleHighTSpc,  3, 1 ,false},
     182             : { ARM::VLD1q64LowQPseudo_UPD,  ARM::VLD1d64Qwb_fixed,   true,  true, true, SingleLowSpc,  4, 1 ,false},
     183             : { ARM::VLD1q64LowTPseudo_UPD,  ARM::VLD1d64Twb_fixed,   true,  true, true, SingleLowSpc,  3, 1 ,false},
     184             : { ARM::VLD1q8HighQPseudo,   ARM::VLD1d8Q,     true,  false, false, SingleHighQSpc,  4, 8 ,false},
     185             : { ARM::VLD1q8HighTPseudo,   ARM::VLD1d8T,     true,  false, false, SingleHighTSpc,  3, 8 ,false},
     186             : { ARM::VLD1q8LowQPseudo_UPD,  ARM::VLD1d8Qwb_fixed,   true,  true, true, SingleLowSpc,  4, 8 ,false},
     187             : { ARM::VLD1q8LowTPseudo_UPD,  ARM::VLD1d8Twb_fixed,   true,  true, true, SingleLowSpc,  3, 8 ,false},
     188             : 
     189             : { ARM::VLD2DUPq16EvenPseudo,  ARM::VLD2DUPd16x2,  true, false, false, EvenDblSpc, 2, 4 ,false},
     190             : { ARM::VLD2DUPq16OddPseudo,   ARM::VLD2DUPd16x2,  true, false, false, OddDblSpc,  2, 4 ,false},
     191             : { ARM::VLD2DUPq32EvenPseudo,  ARM::VLD2DUPd32x2,  true, false, false, EvenDblSpc, 2, 2 ,false},
     192             : { ARM::VLD2DUPq32OddPseudo,   ARM::VLD2DUPd32x2,  true, false, false, OddDblSpc,  2, 2 ,false},
     193             : { ARM::VLD2DUPq8EvenPseudo,   ARM::VLD2DUPd8x2,   true, false, false, EvenDblSpc, 2, 8 ,false},
     194             : { ARM::VLD2DUPq8OddPseudo,    ARM::VLD2DUPd8x2,   true, false, false, OddDblSpc,  2, 8 ,false},
     195             : 
     196             : { ARM::VLD2LNd16Pseudo,     ARM::VLD2LNd16,     true, false, false, SingleSpc,  2, 4 ,true},
     197             : { ARM::VLD2LNd16Pseudo_UPD, ARM::VLD2LNd16_UPD, true, true, true,  SingleSpc,  2, 4 ,true},
     198             : { ARM::VLD2LNd32Pseudo,     ARM::VLD2LNd32,     true, false, false, SingleSpc,  2, 2 ,true},
     199             : { ARM::VLD2LNd32Pseudo_UPD, ARM::VLD2LNd32_UPD, true, true, true,  SingleSpc,  2, 2 ,true},
     200             : { ARM::VLD2LNd8Pseudo,      ARM::VLD2LNd8,      true, false, false, SingleSpc,  2, 8 ,true},
     201             : { ARM::VLD2LNd8Pseudo_UPD,  ARM::VLD2LNd8_UPD, true, true, true,  SingleSpc,  2, 8 ,true},
     202             : { ARM::VLD2LNq16Pseudo,     ARM::VLD2LNq16,     true, false, false, EvenDblSpc, 2, 4 ,true},
     203             : { ARM::VLD2LNq16Pseudo_UPD, ARM::VLD2LNq16_UPD, true, true, true,  EvenDblSpc, 2, 4 ,true},
     204             : { ARM::VLD2LNq32Pseudo,     ARM::VLD2LNq32,     true, false, false, EvenDblSpc, 2, 2 ,true},
     205             : { ARM::VLD2LNq32Pseudo_UPD, ARM::VLD2LNq32_UPD, true, true, true,  EvenDblSpc, 2, 2 ,true},
     206             : 
     207             : { ARM::VLD2q16Pseudo,       ARM::VLD2q16,      true,  false, false, SingleSpc,  4, 4 ,false},
     208             : { ARM::VLD2q16PseudoWB_fixed,   ARM::VLD2q16wb_fixed, true, true, false,  SingleSpc,  4, 4 ,false},
     209             : { ARM::VLD2q16PseudoWB_register,   ARM::VLD2q16wb_register, true, true, true,  SingleSpc,  4, 4 ,false},
     210             : { ARM::VLD2q32Pseudo,       ARM::VLD2q32,      true,  false, false, SingleSpc,  4, 2 ,false},
     211             : { ARM::VLD2q32PseudoWB_fixed,   ARM::VLD2q32wb_fixed, true, true, false,  SingleSpc,  4, 2 ,false},
     212             : { ARM::VLD2q32PseudoWB_register,   ARM::VLD2q32wb_register, true, true, true,  SingleSpc,  4, 2 ,false},
     213             : { ARM::VLD2q8Pseudo,        ARM::VLD2q8,       true,  false, false, SingleSpc,  4, 8 ,false},
     214             : { ARM::VLD2q8PseudoWB_fixed,    ARM::VLD2q8wb_fixed, true, true, false,  SingleSpc,  4, 8 ,false},
     215             : { ARM::VLD2q8PseudoWB_register,    ARM::VLD2q8wb_register, true, true, true,  SingleSpc,  4, 8 ,false},
     216             : 
     217             : { ARM::VLD3DUPd16Pseudo,     ARM::VLD3DUPd16,     true, false, false, SingleSpc, 3, 4,true},
     218             : { ARM::VLD3DUPd16Pseudo_UPD, ARM::VLD3DUPd16_UPD, true, true, true,  SingleSpc, 3, 4,true},
     219             : { ARM::VLD3DUPd32Pseudo,     ARM::VLD3DUPd32,     true, false, false, SingleSpc, 3, 2,true},
     220             : { ARM::VLD3DUPd32Pseudo_UPD, ARM::VLD3DUPd32_UPD, true, true, true,  SingleSpc, 3, 2,true},
     221             : { ARM::VLD3DUPd8Pseudo,      ARM::VLD3DUPd8,      true, false, false, SingleSpc, 3, 8,true},
     222             : { ARM::VLD3DUPd8Pseudo_UPD,  ARM::VLD3DUPd8_UPD, true, true, true,  SingleSpc, 3, 8,true},
     223             : { ARM::VLD3DUPq16EvenPseudo, ARM::VLD3DUPq16,     true, false, false, EvenDblSpc, 3, 4 ,true},
     224             : { ARM::VLD3DUPq16OddPseudo,  ARM::VLD3DUPq16,     true, false, false, OddDblSpc,  3, 4 ,true},
     225             : { ARM::VLD3DUPq32EvenPseudo, ARM::VLD3DUPq32,     true, false, false, EvenDblSpc, 3, 2 ,true},
     226             : { ARM::VLD3DUPq32OddPseudo,  ARM::VLD3DUPq32,     true, false, false, OddDblSpc,  3, 2 ,true},
     227             : { ARM::VLD3DUPq8EvenPseudo,  ARM::VLD3DUPq8,      true, false, false, EvenDblSpc, 3, 8 ,true},
     228             : { ARM::VLD3DUPq8OddPseudo,   ARM::VLD3DUPq8,      true, false, false, OddDblSpc,  3, 8 ,true},
     229             : 
     230             : { ARM::VLD3LNd16Pseudo,     ARM::VLD3LNd16,     true, false, false, SingleSpc,  3, 4 ,true},
     231             : { ARM::VLD3LNd16Pseudo_UPD, ARM::VLD3LNd16_UPD, true, true, true,  SingleSpc,  3, 4 ,true},
     232             : { ARM::VLD3LNd32Pseudo,     ARM::VLD3LNd32,     true, false, false, SingleSpc,  3, 2 ,true},
     233             : { ARM::VLD3LNd32Pseudo_UPD, ARM::VLD3LNd32_UPD, true, true, true,  SingleSpc,  3, 2 ,true},
     234             : { ARM::VLD3LNd8Pseudo,      ARM::VLD3LNd8,      true, false, false, SingleSpc,  3, 8 ,true},
     235             : { ARM::VLD3LNd8Pseudo_UPD,  ARM::VLD3LNd8_UPD, true, true, true,  SingleSpc,  3, 8 ,true},
     236             : { ARM::VLD3LNq16Pseudo,     ARM::VLD3LNq16,     true, false, false, EvenDblSpc, 3, 4 ,true},
     237             : { ARM::VLD3LNq16Pseudo_UPD, ARM::VLD3LNq16_UPD, true, true, true,  EvenDblSpc, 3, 4 ,true},
     238             : { ARM::VLD3LNq32Pseudo,     ARM::VLD3LNq32,     true, false, false, EvenDblSpc, 3, 2 ,true},
     239             : { ARM::VLD3LNq32Pseudo_UPD, ARM::VLD3LNq32_UPD, true, true, true,  EvenDblSpc, 3, 2 ,true},
     240             : 
     241             : { ARM::VLD3d16Pseudo,       ARM::VLD3d16,      true,  false, false, SingleSpc,  3, 4 ,true},
     242             : { ARM::VLD3d16Pseudo_UPD,   ARM::VLD3d16_UPD, true, true, true,  SingleSpc,  3, 4 ,true},
     243             : { ARM::VLD3d32Pseudo,       ARM::VLD3d32,      true,  false, false, SingleSpc,  3, 2 ,true},
     244             : { ARM::VLD3d32Pseudo_UPD,   ARM::VLD3d32_UPD, true, true, true,  SingleSpc,  3, 2 ,true},
     245             : { ARM::VLD3d8Pseudo,        ARM::VLD3d8,       true,  false, false, SingleSpc,  3, 8 ,true},
     246             : { ARM::VLD3d8Pseudo_UPD,    ARM::VLD3d8_UPD, true, true, true,  SingleSpc,  3, 8 ,true},
     247             : 
     248             : { ARM::VLD3q16Pseudo_UPD,    ARM::VLD3q16_UPD, true, true, true,  EvenDblSpc, 3, 4 ,true},
     249             : { ARM::VLD3q16oddPseudo,     ARM::VLD3q16,     true,  false, false, OddDblSpc,  3, 4 ,true},
     250             : { ARM::VLD3q16oddPseudo_UPD, ARM::VLD3q16_UPD, true, true, true,  OddDblSpc,  3, 4 ,true},
     251             : { ARM::VLD3q32Pseudo_UPD,    ARM::VLD3q32_UPD, true, true, true,  EvenDblSpc, 3, 2 ,true},
     252             : { ARM::VLD3q32oddPseudo,     ARM::VLD3q32,     true,  false, false, OddDblSpc,  3, 2 ,true},
     253             : { ARM::VLD3q32oddPseudo_UPD, ARM::VLD3q32_UPD, true, true, true,  OddDblSpc,  3, 2 ,true},
     254             : { ARM::VLD3q8Pseudo_UPD,     ARM::VLD3q8_UPD, true, true, true,  EvenDblSpc, 3, 8 ,true},
     255             : { ARM::VLD3q8oddPseudo,      ARM::VLD3q8,      true,  false, false, OddDblSpc,  3, 8 ,true},
     256             : { ARM::VLD3q8oddPseudo_UPD,  ARM::VLD3q8_UPD, true, true, true,  OddDblSpc,  3, 8 ,true},
     257             : 
     258             : { ARM::VLD4DUPd16Pseudo,     ARM::VLD4DUPd16,     true, false, false, SingleSpc, 4, 4,true},
     259             : { ARM::VLD4DUPd16Pseudo_UPD, ARM::VLD4DUPd16_UPD, true, true, true,  SingleSpc, 4, 4,true},
     260             : { ARM::VLD4DUPd32Pseudo,     ARM::VLD4DUPd32,     true, false, false, SingleSpc, 4, 2,true},
     261             : { ARM::VLD4DUPd32Pseudo_UPD, ARM::VLD4DUPd32_UPD, true, true, true,  SingleSpc, 4, 2,true},
     262             : { ARM::VLD4DUPd8Pseudo,      ARM::VLD4DUPd8,      true, false, false, SingleSpc, 4, 8,true},
     263             : { ARM::VLD4DUPd8Pseudo_UPD,  ARM::VLD4DUPd8_UPD, true, true, true,  SingleSpc, 4, 8,true},
     264             : { ARM::VLD4DUPq16EvenPseudo, ARM::VLD4DUPq16,     true, false, false, EvenDblSpc, 4, 4 ,true},
     265             : { ARM::VLD4DUPq16OddPseudo,  ARM::VLD4DUPq16,     true, false, false, OddDblSpc,  4, 4 ,true},
     266             : { ARM::VLD4DUPq32EvenPseudo, ARM::VLD4DUPq32,     true, false, false, EvenDblSpc, 4, 2 ,true},
     267             : { ARM::VLD4DUPq32OddPseudo,  ARM::VLD4DUPq32,     true, false, false, OddDblSpc,  4, 2 ,true},
     268             : { ARM::VLD4DUPq8EvenPseudo,  ARM::VLD4DUPq8,      true, false, false, EvenDblSpc, 4, 8 ,true},
     269             : { ARM::VLD4DUPq8OddPseudo,   ARM::VLD4DUPq8,      true, false, false, OddDblSpc,  4, 8 ,true},
     270             : 
     271             : { ARM::VLD4LNd16Pseudo,     ARM::VLD4LNd16,     true, false, false, SingleSpc,  4, 4 ,true},
     272             : { ARM::VLD4LNd16Pseudo_UPD, ARM::VLD4LNd16_UPD, true, true, true,  SingleSpc,  4, 4 ,true},
     273             : { ARM::VLD4LNd32Pseudo,     ARM::VLD4LNd32,     true, false, false, SingleSpc,  4, 2 ,true},
     274             : { ARM::VLD4LNd32Pseudo_UPD, ARM::VLD4LNd32_UPD, true, true, true,  SingleSpc,  4, 2 ,true},
     275             : { ARM::VLD4LNd8Pseudo,      ARM::VLD4LNd8,      true, false, false, SingleSpc,  4, 8 ,true},
     276             : { ARM::VLD4LNd8Pseudo_UPD,  ARM::VLD4LNd8_UPD, true, true, true,  SingleSpc,  4, 8 ,true},
     277             : { ARM::VLD4LNq16Pseudo,     ARM::VLD4LNq16,     true, false, false, EvenDblSpc, 4, 4 ,true},
     278             : { ARM::VLD4LNq16Pseudo_UPD, ARM::VLD4LNq16_UPD, true, true, true,  EvenDblSpc, 4, 4 ,true},
     279             : { ARM::VLD4LNq32Pseudo,     ARM::VLD4LNq32,     true, false, false, EvenDblSpc, 4, 2 ,true},
     280             : { ARM::VLD4LNq32Pseudo_UPD, ARM::VLD4LNq32_UPD, true, true, true,  EvenDblSpc, 4, 2 ,true},
     281             : 
     282             : { ARM::VLD4d16Pseudo,       ARM::VLD4d16,      true,  false, false, SingleSpc,  4, 4 ,true},
     283             : { ARM::VLD4d16Pseudo_UPD,   ARM::VLD4d16_UPD, true, true, true,  SingleSpc,  4, 4 ,true},
     284             : { ARM::VLD4d32Pseudo,       ARM::VLD4d32,      true,  false, false, SingleSpc,  4, 2 ,true},
     285             : { ARM::VLD4d32Pseudo_UPD,   ARM::VLD4d32_UPD, true, true, true,  SingleSpc,  4, 2 ,true},
     286             : { ARM::VLD4d8Pseudo,        ARM::VLD4d8,       true,  false, false, SingleSpc,  4, 8 ,true},
     287             : { ARM::VLD4d8Pseudo_UPD,    ARM::VLD4d8_UPD, true, true, true,  SingleSpc,  4, 8 ,true},
     288             : 
     289             : { ARM::VLD4q16Pseudo_UPD,    ARM::VLD4q16_UPD, true, true, true,  EvenDblSpc, 4, 4 ,true},
     290             : { ARM::VLD4q16oddPseudo,     ARM::VLD4q16,     true,  false, false, OddDblSpc,  4, 4 ,true},
     291             : { ARM::VLD4q16oddPseudo_UPD, ARM::VLD4q16_UPD, true, true, true,  OddDblSpc,  4, 4 ,true},
     292             : { ARM::VLD4q32Pseudo_UPD,    ARM::VLD4q32_UPD, true, true, true,  EvenDblSpc, 4, 2 ,true},
     293             : { ARM::VLD4q32oddPseudo,     ARM::VLD4q32,     true,  false, false, OddDblSpc,  4, 2 ,true},
     294             : { ARM::VLD4q32oddPseudo_UPD, ARM::VLD4q32_UPD, true, true, true,  OddDblSpc,  4, 2 ,true},
     295             : { ARM::VLD4q8Pseudo_UPD,     ARM::VLD4q8_UPD, true, true, true,  EvenDblSpc, 4, 8 ,true},
     296             : { ARM::VLD4q8oddPseudo,      ARM::VLD4q8,      true,  false, false, OddDblSpc,  4, 8 ,true},
     297             : { ARM::VLD4q8oddPseudo_UPD,  ARM::VLD4q8_UPD, true, true, true,  OddDblSpc,  4, 8 ,true},
     298             : 
     299             : { ARM::VST1LNq16Pseudo,     ARM::VST1LNd16,    false, false, false, EvenDblSpc, 1, 4 ,true},
     300             : { ARM::VST1LNq16Pseudo_UPD, ARM::VST1LNd16_UPD, false, true, true,  EvenDblSpc, 1, 4 ,true},
     301             : { ARM::VST1LNq32Pseudo,     ARM::VST1LNd32,    false, false, false, EvenDblSpc, 1, 2 ,true},
     302             : { ARM::VST1LNq32Pseudo_UPD, ARM::VST1LNd32_UPD, false, true, true,  EvenDblSpc, 1, 2 ,true},
     303             : { ARM::VST1LNq8Pseudo,      ARM::VST1LNd8,     false, false, false, EvenDblSpc, 1, 8 ,true},
     304             : { ARM::VST1LNq8Pseudo_UPD,  ARM::VST1LNd8_UPD, false, true, true,  EvenDblSpc, 1, 8 ,true},
     305             : 
     306             : { ARM::VST1d16QPseudo,      ARM::VST1d16Q,     false, false, false, SingleSpc,  4, 4 ,false},
     307             : { ARM::VST1d16TPseudo,      ARM::VST1d16T,     false, false, false, SingleSpc,  3, 4 ,false},
     308             : { ARM::VST1d32QPseudo,      ARM::VST1d32Q,     false, false, false, SingleSpc,  4, 2 ,false},
     309             : { ARM::VST1d32TPseudo,      ARM::VST1d32T,     false, false, false, SingleSpc,  3, 2 ,false},
     310             : { ARM::VST1d64QPseudo,      ARM::VST1d64Q,     false, false, false, SingleSpc,  4, 1 ,false},
     311             : { ARM::VST1d64QPseudoWB_fixed,  ARM::VST1d64Qwb_fixed, false, true, false,  SingleSpc,  4, 1 ,false},
     312             : { ARM::VST1d64QPseudoWB_register, ARM::VST1d64Qwb_register, false, true, true,  SingleSpc,  4, 1 ,false},
     313             : { ARM::VST1d64TPseudo,      ARM::VST1d64T,     false, false, false, SingleSpc,  3, 1 ,false},
     314             : { ARM::VST1d64TPseudoWB_fixed,  ARM::VST1d64Twb_fixed, false, true, false,  SingleSpc,  3, 1 ,false},
     315             : { ARM::VST1d64TPseudoWB_register,  ARM::VST1d64Twb_register, false, true, true,  SingleSpc,  3, 1 ,false},
     316             : { ARM::VST1d8QPseudo,       ARM::VST1d8Q,      false, false, false, SingleSpc,  4, 8 ,false},
     317             : { ARM::VST1d8TPseudo,       ARM::VST1d8T,      false, false, false, SingleSpc,  3, 8 ,false},
     318             : { ARM::VST1q16HighQPseudo,  ARM::VST1d16Q,      false, false, false, SingleHighQSpc,   4, 4 ,false},
     319             : { ARM::VST1q16HighTPseudo,  ARM::VST1d16T,      false, false, false, SingleHighTSpc,   3, 4 ,false},
     320             : { ARM::VST1q16LowQPseudo_UPD,   ARM::VST1d16Qwb_fixed,  false, true, true, SingleLowSpc,   4, 4 ,false},
     321             : { ARM::VST1q16LowTPseudo_UPD,   ARM::VST1d16Twb_fixed,  false, true, true, SingleLowSpc,   3, 4 ,false},
     322             : { ARM::VST1q32HighQPseudo,  ARM::VST1d32Q,      false, false, false, SingleHighQSpc,   4, 2 ,false},
     323             : { ARM::VST1q32HighTPseudo,  ARM::VST1d32T,      false, false, false, SingleHighTSpc,   3, 2 ,false},
     324             : { ARM::VST1q32LowQPseudo_UPD,   ARM::VST1d32Qwb_fixed,  false, true, true, SingleLowSpc,   4, 2 ,false},
     325             : { ARM::VST1q32LowTPseudo_UPD,   ARM::VST1d32Twb_fixed,  false, true, true, SingleLowSpc,   3, 2 ,false},
     326             : { ARM::VST1q64HighQPseudo,  ARM::VST1d64Q,      false, false, false, SingleHighQSpc,   4, 1 ,false},
     327             : { ARM::VST1q64HighTPseudo,  ARM::VST1d64T,      false, false, false, SingleHighTSpc,   3, 1 ,false},
     328             : { ARM::VST1q64LowQPseudo_UPD,   ARM::VST1d64Qwb_fixed,  false, true, true, SingleLowSpc,   4, 1 ,false},
     329             : { ARM::VST1q64LowTPseudo_UPD,   ARM::VST1d64Twb_fixed,  false, true, true, SingleLowSpc,   3, 1 ,false},
     330             : { ARM::VST1q8HighQPseudo,   ARM::VST1d8Q,      false, false, false, SingleHighQSpc,   4, 8 ,false},
     331             : { ARM::VST1q8HighTPseudo,   ARM::VST1d8T,      false, false, false, SingleHighTSpc,   3, 8 ,false},
     332             : { ARM::VST1q8LowQPseudo_UPD,   ARM::VST1d8Qwb_fixed,  false, true, true, SingleLowSpc,   4, 8 ,false},
     333             : { ARM::VST1q8LowTPseudo_UPD,   ARM::VST1d8Twb_fixed,  false, true, true, SingleLowSpc,   3, 8 ,false},
     334             : 
     335             : { ARM::VST2LNd16Pseudo,     ARM::VST2LNd16,     false, false, false, SingleSpc, 2, 4 ,true},
     336             : { ARM::VST2LNd16Pseudo_UPD, ARM::VST2LNd16_UPD, false, true, true,  SingleSpc, 2, 4 ,true},
     337             : { ARM::VST2LNd32Pseudo,     ARM::VST2LNd32,     false, false, false, SingleSpc, 2, 2 ,true},
     338             : { ARM::VST2LNd32Pseudo_UPD, ARM::VST2LNd32_UPD, false, true, true,  SingleSpc, 2, 2 ,true},
     339             : { ARM::VST2LNd8Pseudo,      ARM::VST2LNd8,      false, false, false, SingleSpc, 2, 8 ,true},
     340             : { ARM::VST2LNd8Pseudo_UPD,  ARM::VST2LNd8_UPD, false, true, true,  SingleSpc, 2, 8 ,true},
     341             : { ARM::VST2LNq16Pseudo,     ARM::VST2LNq16,     false, false, false, EvenDblSpc, 2, 4,true},
     342             : { ARM::VST2LNq16Pseudo_UPD, ARM::VST2LNq16_UPD, false, true, true,  EvenDblSpc, 2, 4,true},
     343             : { ARM::VST2LNq32Pseudo,     ARM::VST2LNq32,     false, false, false, EvenDblSpc, 2, 2,true},
     344             : { ARM::VST2LNq32Pseudo_UPD, ARM::VST2LNq32_UPD, false, true, true,  EvenDblSpc, 2, 2,true},
     345             : 
     346             : { ARM::VST2q16Pseudo,       ARM::VST2q16,      false, false, false, SingleSpc,  4, 4 ,false},
     347             : { ARM::VST2q16PseudoWB_fixed,   ARM::VST2q16wb_fixed, false, true, false,  SingleSpc,  4, 4 ,false},
     348             : { ARM::VST2q16PseudoWB_register,   ARM::VST2q16wb_register, false, true, true,  SingleSpc,  4, 4 ,false},
     349             : { ARM::VST2q32Pseudo,       ARM::VST2q32,      false, false, false, SingleSpc,  4, 2 ,false},
     350             : { ARM::VST2q32PseudoWB_fixed,   ARM::VST2q32wb_fixed, false, true, false,  SingleSpc,  4, 2 ,false},
     351             : { ARM::VST2q32PseudoWB_register,   ARM::VST2q32wb_register, false, true, true,  SingleSpc,  4, 2 ,false},
     352             : { ARM::VST2q8Pseudo,        ARM::VST2q8,       false, false, false, SingleSpc,  4, 8 ,false},
     353             : { ARM::VST2q8PseudoWB_fixed,    ARM::VST2q8wb_fixed, false, true, false,  SingleSpc,  4, 8 ,false},
     354             : { ARM::VST2q8PseudoWB_register,    ARM::VST2q8wb_register, false, true, true,  SingleSpc,  4, 8 ,false},
     355             : 
     356             : { ARM::VST3LNd16Pseudo,     ARM::VST3LNd16,     false, false, false, SingleSpc, 3, 4 ,true},
     357             : { ARM::VST3LNd16Pseudo_UPD, ARM::VST3LNd16_UPD, false, true, true,  SingleSpc, 3, 4 ,true},
     358             : { ARM::VST3LNd32Pseudo,     ARM::VST3LNd32,     false, false, false, SingleSpc, 3, 2 ,true},
     359             : { ARM::VST3LNd32Pseudo_UPD, ARM::VST3LNd32_UPD, false, true, true,  SingleSpc, 3, 2 ,true},
     360             : { ARM::VST3LNd8Pseudo,      ARM::VST3LNd8,      false, false, false, SingleSpc, 3, 8 ,true},
     361             : { ARM::VST3LNd8Pseudo_UPD,  ARM::VST3LNd8_UPD, false, true, true,  SingleSpc, 3, 8 ,true},
     362             : { ARM::VST3LNq16Pseudo,     ARM::VST3LNq16,     false, false, false, EvenDblSpc, 3, 4,true},
     363             : { ARM::VST3LNq16Pseudo_UPD, ARM::VST3LNq16_UPD, false, true, true,  EvenDblSpc, 3, 4,true},
     364             : { ARM::VST3LNq32Pseudo,     ARM::VST3LNq32,     false, false, false, EvenDblSpc, 3, 2,true},
     365             : { ARM::VST3LNq32Pseudo_UPD, ARM::VST3LNq32_UPD, false, true, true,  EvenDblSpc, 3, 2,true},
     366             : 
     367             : { ARM::VST3d16Pseudo,       ARM::VST3d16,      false, false, false, SingleSpc,  3, 4 ,true},
     368             : { ARM::VST3d16Pseudo_UPD,   ARM::VST3d16_UPD, false, true, true,  SingleSpc,  3, 4 ,true},
     369             : { ARM::VST3d32Pseudo,       ARM::VST3d32,      false, false, false, SingleSpc,  3, 2 ,true},
     370             : { ARM::VST3d32Pseudo_UPD,   ARM::VST3d32_UPD, false, true, true,  SingleSpc,  3, 2 ,true},
     371             : { ARM::VST3d8Pseudo,        ARM::VST3d8,       false, false, false, SingleSpc,  3, 8 ,true},
     372             : { ARM::VST3d8Pseudo_UPD,    ARM::VST3d8_UPD, false, true, true,  SingleSpc,  3, 8 ,true},
     373             : 
     374             : { ARM::VST3q16Pseudo_UPD,    ARM::VST3q16_UPD, false, true, true,  EvenDblSpc, 3, 4 ,true},
     375             : { ARM::VST3q16oddPseudo,     ARM::VST3q16,     false, false, false, OddDblSpc,  3, 4 ,true},
     376             : { ARM::VST3q16oddPseudo_UPD, ARM::VST3q16_UPD, false, true, true,  OddDblSpc,  3, 4 ,true},
     377             : { ARM::VST3q32Pseudo_UPD,    ARM::VST3q32_UPD, false, true, true,  EvenDblSpc, 3, 2 ,true},
     378             : { ARM::VST3q32oddPseudo,     ARM::VST3q32,     false, false, false, OddDblSpc,  3, 2 ,true},
     379             : { ARM::VST3q32oddPseudo_UPD, ARM::VST3q32_UPD, false, true, true,  OddDblSpc,  3, 2 ,true},
     380             : { ARM::VST3q8Pseudo_UPD,     ARM::VST3q8_UPD, false, true, true,  EvenDblSpc, 3, 8 ,true},
     381             : { ARM::VST3q8oddPseudo,      ARM::VST3q8,      false, false, false, OddDblSpc,  3, 8 ,true},
     382             : { ARM::VST3q8oddPseudo_UPD,  ARM::VST3q8_UPD, false, true, true,  OddDblSpc,  3, 8 ,true},
     383             : 
     384             : { ARM::VST4LNd16Pseudo,     ARM::VST4LNd16,     false, false, false, SingleSpc, 4, 4 ,true},
     385             : { ARM::VST4LNd16Pseudo_UPD, ARM::VST4LNd16_UPD, false, true, true,  SingleSpc, 4, 4 ,true},
     386             : { ARM::VST4LNd32Pseudo,     ARM::VST4LNd32,     false, false, false, SingleSpc, 4, 2 ,true},
     387             : { ARM::VST4LNd32Pseudo_UPD, ARM::VST4LNd32_UPD, false, true, true,  SingleSpc, 4, 2 ,true},
     388             : { ARM::VST4LNd8Pseudo,      ARM::VST4LNd8,      false, false, false, SingleSpc, 4, 8 ,true},
     389             : { ARM::VST4LNd8Pseudo_UPD,  ARM::VST4LNd8_UPD, false, true, true,  SingleSpc, 4, 8 ,true},
     390             : { ARM::VST4LNq16Pseudo,     ARM::VST4LNq16,     false, false, false, EvenDblSpc, 4, 4,true},
     391             : { ARM::VST4LNq16Pseudo_UPD, ARM::VST4LNq16_UPD, false, true, true,  EvenDblSpc, 4, 4,true},
     392             : { ARM::VST4LNq32Pseudo,     ARM::VST4LNq32,     false, false, false, EvenDblSpc, 4, 2,true},
     393             : { ARM::VST4LNq32Pseudo_UPD, ARM::VST4LNq32_UPD, false, true, true,  EvenDblSpc, 4, 2,true},
     394             : 
     395             : { ARM::VST4d16Pseudo,       ARM::VST4d16,      false, false, false, SingleSpc,  4, 4 ,true},
     396             : { ARM::VST4d16Pseudo_UPD,   ARM::VST4d16_UPD, false, true, true,  SingleSpc,  4, 4 ,true},
     397             : { ARM::VST4d32Pseudo,       ARM::VST4d32,      false, false, false, SingleSpc,  4, 2 ,true},
     398             : { ARM::VST4d32Pseudo_UPD,   ARM::VST4d32_UPD, false, true, true,  SingleSpc,  4, 2 ,true},
     399             : { ARM::VST4d8Pseudo,        ARM::VST4d8,       false, false, false, SingleSpc,  4, 8 ,true},
     400             : { ARM::VST4d8Pseudo_UPD,    ARM::VST4d8_UPD, false, true, true,  SingleSpc,  4, 8 ,true},
     401             : 
     402             : { ARM::VST4q16Pseudo_UPD,    ARM::VST4q16_UPD, false, true, true,  EvenDblSpc, 4, 4 ,true},
     403             : { ARM::VST4q16oddPseudo,     ARM::VST4q16,     false, false, false, OddDblSpc,  4, 4 ,true},
     404             : { ARM::VST4q16oddPseudo_UPD, ARM::VST4q16_UPD, false, true, true,  OddDblSpc,  4, 4 ,true},
     405             : { ARM::VST4q32Pseudo_UPD,    ARM::VST4q32_UPD, false, true, true,  EvenDblSpc, 4, 2 ,true},
     406             : { ARM::VST4q32oddPseudo,     ARM::VST4q32,     false, false, false, OddDblSpc,  4, 2 ,true},
     407             : { ARM::VST4q32oddPseudo_UPD, ARM::VST4q32_UPD, false, true, true,  OddDblSpc,  4, 2 ,true},
     408             : { ARM::VST4q8Pseudo_UPD,     ARM::VST4q8_UPD, false, true, true,  EvenDblSpc, 4, 8 ,true},
     409             : { ARM::VST4q8oddPseudo,      ARM::VST4q8,      false, false, false, OddDblSpc,  4, 8 ,true},
     410             : { ARM::VST4q8oddPseudo_UPD,  ARM::VST4q8_UPD, false, true, true,  OddDblSpc,  4, 8 ,true}
     411             : };
     412             : 
     413             : /// LookupNEONLdSt - Search the NEONLdStTable for information about a NEON
     414             : /// load or store pseudo instruction.
     415         421 : static const NEONLdStTableEntry *LookupNEONLdSt(unsigned Opcode) {
     416             : #ifndef NDEBUG
     417             :   // Make sure the table is sorted.
     418             :   static std::atomic<bool> TableChecked(false);
     419             :   if (!TableChecked.load(std::memory_order_relaxed)) {
     420             :     assert(std::is_sorted(std::begin(NEONLdStTable), std::end(NEONLdStTable)) &&
     421             :            "NEONLdStTable is not sorted!");
     422             :     TableChecked.store(true, std::memory_order_relaxed);
     423             :   }
     424             : #endif
     425             : 
     426             :   auto I = std::lower_bound(std::begin(NEONLdStTable),
     427             :                             std::end(NEONLdStTable), Opcode);
     428         421 :   if (I != std::end(NEONLdStTable) && I->PseudoOpc == Opcode)
     429         421 :     return I;
     430             :   return nullptr;
     431             : }
     432             : 
     433             : /// GetDSubRegs - Get 4 D subregisters of a Q, QQ, or QQQQ register,
     434             : /// corresponding to the specified register spacing.  Not all of the results
     435             : /// are necessarily valid, e.g., a Q register only has 2 D subregisters.
     436         420 : static void GetDSubRegs(unsigned Reg, NEONRegSpacing RegSpc,
     437             :                         const TargetRegisterInfo *TRI, unsigned &D0,
     438             :                         unsigned &D1, unsigned &D2, unsigned &D3) {
     439         420 :   if (RegSpc == SingleSpc || RegSpc == SingleLowSpc) {
     440         207 :     D0 = TRI->getSubReg(Reg, ARM::dsub_0);
     441         207 :     D1 = TRI->getSubReg(Reg, ARM::dsub_1);
     442         207 :     D2 = TRI->getSubReg(Reg, ARM::dsub_2);
     443         207 :     D3 = TRI->getSubReg(Reg, ARM::dsub_3);
     444         213 :   } else if (RegSpc == SingleHighQSpc) {
     445           8 :     D0 = TRI->getSubReg(Reg, ARM::dsub_4);
     446           8 :     D1 = TRI->getSubReg(Reg, ARM::dsub_5);
     447           8 :     D2 = TRI->getSubReg(Reg, ARM::dsub_6);
     448           8 :     D3 = TRI->getSubReg(Reg, ARM::dsub_7);
     449         205 :   } else if (RegSpc == SingleHighTSpc) {
     450           8 :     D0 = TRI->getSubReg(Reg, ARM::dsub_3);
     451           8 :     D1 = TRI->getSubReg(Reg, ARM::dsub_4);
     452           8 :     D2 = TRI->getSubReg(Reg, ARM::dsub_5);
     453           8 :     D3 = TRI->getSubReg(Reg, ARM::dsub_6);
     454         197 :   } else if (RegSpc == EvenDblSpc) {
     455         102 :     D0 = TRI->getSubReg(Reg, ARM::dsub_0);
     456         102 :     D1 = TRI->getSubReg(Reg, ARM::dsub_2);
     457         102 :     D2 = TRI->getSubReg(Reg, ARM::dsub_4);
     458         102 :     D3 = TRI->getSubReg(Reg, ARM::dsub_6);
     459             :   } else {
     460             :     assert(RegSpc == OddDblSpc && "unknown register spacing");
     461          95 :     D0 = TRI->getSubReg(Reg, ARM::dsub_1);
     462          95 :     D1 = TRI->getSubReg(Reg, ARM::dsub_3);
     463          95 :     D2 = TRI->getSubReg(Reg, ARM::dsub_5);
     464          95 :     D3 = TRI->getSubReg(Reg, ARM::dsub_7);
     465             :   }
     466         420 : }
     467             : 
     468             : /// ExpandVLD - Translate VLD pseudo instructions with Q, QQ or QQQQ register
     469             : /// operands to real VLD instructions with D register operands.
     470           0 : void ARMExpandPseudo::ExpandVLD(MachineBasicBlock::iterator &MBBI) {
     471             :   MachineInstr &MI = *MBBI;
     472           0 :   MachineBasicBlock &MBB = *MI.getParent();
     473             : 
     474           0 :   const NEONLdStTableEntry *TableEntry = LookupNEONLdSt(MI.getOpcode());
     475             :   assert(TableEntry && TableEntry->IsLoad && "NEONLdStTable lookup failed");
     476           0 :   NEONRegSpacing RegSpc = (NEONRegSpacing)TableEntry->RegSpacing;
     477           0 :   unsigned NumRegs = TableEntry->NumRegs;
     478             : 
     479             :   MachineInstrBuilder MIB = BuildMI(MBB, MBBI, MI.getDebugLoc(),
     480           0 :                                     TII->get(TableEntry->RealOpc));
     481             :   unsigned OpIdx = 0;
     482             : 
     483           0 :   bool DstIsDead = MI.getOperand(OpIdx).isDead();
     484           0 :   unsigned DstReg = MI.getOperand(OpIdx++).getReg();
     485           0 :   if(TableEntry->RealOpc == ARM::VLD2DUPd8x2 ||
     486           0 :      TableEntry->RealOpc == ARM::VLD2DUPd16x2 ||
     487             :      TableEntry->RealOpc == ARM::VLD2DUPd32x2) {
     488             :     unsigned SubRegIndex;
     489           0 :     if (RegSpc == EvenDblSpc) {
     490             :       SubRegIndex = ARM::dsub_0;
     491             :     } else {
     492             :       assert(RegSpc == OddDblSpc && "Unexpected spacing!");
     493             :       SubRegIndex = ARM::dsub_1;
     494             :     }
     495           0 :     unsigned SubReg = TRI->getSubReg(DstReg, SubRegIndex);
     496           0 :     unsigned DstRegPair = TRI->getMatchingSuperReg(SubReg, ARM::dsub_0,
     497             :                                                    &ARM::DPairSpcRegClass);
     498           0 :     MIB.addReg(DstRegPair, RegState::Define | getDeadRegState(DstIsDead));
     499             :   } else {
     500             :     unsigned D0, D1, D2, D3;
     501           0 :     GetDSubRegs(DstReg, RegSpc, TRI, D0, D1, D2, D3);
     502           0 :     MIB.addReg(D0, RegState::Define | getDeadRegState(DstIsDead));
     503           0 :     if (NumRegs > 1 && TableEntry->copyAllListRegs)
     504           0 :       MIB.addReg(D1, RegState::Define | getDeadRegState(DstIsDead));
     505           0 :     if (NumRegs > 2 && TableEntry->copyAllListRegs)
     506           0 :       MIB.addReg(D2, RegState::Define | getDeadRegState(DstIsDead));
     507           0 :     if (NumRegs > 3 && TableEntry->copyAllListRegs)
     508           0 :       MIB.addReg(D3, RegState::Define | getDeadRegState(DstIsDead));
     509             :   }
     510             : 
     511           0 :   if (TableEntry->isUpdating)
     512           0 :     MIB.add(MI.getOperand(OpIdx++));
     513             : 
     514             :   // Copy the addrmode6 operands.
     515           0 :   MIB.add(MI.getOperand(OpIdx++));
     516           0 :   MIB.add(MI.getOperand(OpIdx++));
     517             : 
     518             :   // Copy the am6offset operand.
     519           0 :   if (TableEntry->hasWritebackOperand) {
     520             :     // TODO: The writing-back pseudo instructions we translate here are all
     521             :     // defined to take am6offset nodes that are capable to represent both fixed
     522             :     // and register forms. Some real instructions, however, do not rely on
     523             :     // am6offset and have separate definitions for such forms. When this is the
     524             :     // case, fixed forms do not take any offset nodes, so here we skip them for
     525             :     // such instructions. Once all real and pseudo writing-back instructions are
     526             :     // rewritten without use of am6offset nodes, this code will go away.
     527           0 :     const MachineOperand &AM6Offset = MI.getOperand(OpIdx++);
     528           0 :     if (TableEntry->RealOpc == ARM::VLD1d8Qwb_fixed ||
     529           0 :         TableEntry->RealOpc == ARM::VLD1d16Qwb_fixed ||
     530           0 :         TableEntry->RealOpc == ARM::VLD1d32Qwb_fixed ||
     531           0 :         TableEntry->RealOpc == ARM::VLD1d64Qwb_fixed ||
     532           0 :         TableEntry->RealOpc == ARM::VLD1d8Twb_fixed ||
     533           0 :         TableEntry->RealOpc == ARM::VLD1d16Twb_fixed ||
     534           0 :         TableEntry->RealOpc == ARM::VLD1d32Twb_fixed ||
     535             :         TableEntry->RealOpc == ARM::VLD1d64Twb_fixed) {
     536             :       assert(AM6Offset.getReg() == 0 &&
     537             :              "A fixed writing-back pseudo instruction provides an offset "
     538             :              "register!");
     539             :     } else {
     540             :       MIB.add(AM6Offset);
     541             :     }
     542             :   }
     543             : 
     544             :   // For an instruction writing double-spaced subregs, the pseudo instruction
     545             :   // has an extra operand that is a use of the super-register.  Record the
     546             :   // operand index and skip over it.
     547             :   unsigned SrcOpIdx = 0;
     548           0 :   if(TableEntry->RealOpc != ARM::VLD2DUPd8x2 &&
     549           0 :      TableEntry->RealOpc != ARM::VLD2DUPd16x2 &&
     550             :      TableEntry->RealOpc != ARM::VLD2DUPd32x2) {
     551           0 :     if (RegSpc == EvenDblSpc || RegSpc == OddDblSpc ||
     552           0 :         RegSpc == SingleLowSpc || RegSpc == SingleHighQSpc ||
     553             :         RegSpc == SingleHighTSpc)
     554           0 :       SrcOpIdx = OpIdx++;
     555             :   }
     556             : 
     557             :   // Copy the predicate operands.
     558           0 :   MIB.add(MI.getOperand(OpIdx++));
     559           0 :   MIB.add(MI.getOperand(OpIdx++));
     560             : 
     561             :   // Copy the super-register source operand used for double-spaced subregs over
     562             :   // to the new instruction as an implicit operand.
     563           0 :   if (SrcOpIdx != 0) {
     564           0 :     MachineOperand MO = MI.getOperand(SrcOpIdx);
     565             :     MO.setImplicit(true);
     566             :     MIB.add(MO);
     567             :   }
     568             :   // Add an implicit def for the super-register.
     569           0 :   MIB.addReg(DstReg, RegState::ImplicitDefine | getDeadRegState(DstIsDead));
     570           0 :   TransferImpOps(MI, MIB, MIB);
     571             : 
     572             :   // Transfer memoperands.
     573             :   MIB.cloneMemRefs(MI);
     574             : 
     575           0 :   MI.eraseFromParent();
     576           0 : }
     577             : 
     578             : /// ExpandVST - Translate VST pseudo instructions with Q, QQ or QQQQ register
     579             : /// operands to real VST instructions with D register operands.
     580           0 : void ARMExpandPseudo::ExpandVST(MachineBasicBlock::iterator &MBBI) {
     581             :   MachineInstr &MI = *MBBI;
     582           0 :   MachineBasicBlock &MBB = *MI.getParent();
     583             : 
     584           0 :   const NEONLdStTableEntry *TableEntry = LookupNEONLdSt(MI.getOpcode());
     585             :   assert(TableEntry && !TableEntry->IsLoad && "NEONLdStTable lookup failed");
     586           0 :   NEONRegSpacing RegSpc = (NEONRegSpacing)TableEntry->RegSpacing;
     587           0 :   unsigned NumRegs = TableEntry->NumRegs;
     588             : 
     589             :   MachineInstrBuilder MIB = BuildMI(MBB, MBBI, MI.getDebugLoc(),
     590           0 :                                     TII->get(TableEntry->RealOpc));
     591             :   unsigned OpIdx = 0;
     592           0 :   if (TableEntry->isUpdating)
     593           0 :     MIB.add(MI.getOperand(OpIdx++));
     594             : 
     595             :   // Copy the addrmode6 operands.
     596           0 :   MIB.add(MI.getOperand(OpIdx++));
     597           0 :   MIB.add(MI.getOperand(OpIdx++));
     598             : 
     599           0 :   if (TableEntry->hasWritebackOperand) {
     600             :     // TODO: The writing-back pseudo instructions we translate here are all
     601             :     // defined to take am6offset nodes that are capable to represent both fixed
     602             :     // and register forms. Some real instructions, however, do not rely on
     603             :     // am6offset and have separate definitions for such forms. When this is the
     604             :     // case, fixed forms do not take any offset nodes, so here we skip them for
     605             :     // such instructions. Once all real and pseudo writing-back instructions are
     606             :     // rewritten without use of am6offset nodes, this code will go away.
     607           0 :     const MachineOperand &AM6Offset = MI.getOperand(OpIdx++);
     608           0 :     if (TableEntry->RealOpc == ARM::VST1d8Qwb_fixed ||
     609           0 :         TableEntry->RealOpc == ARM::VST1d16Qwb_fixed ||
     610           0 :         TableEntry->RealOpc == ARM::VST1d32Qwb_fixed ||
     611           0 :         TableEntry->RealOpc == ARM::VST1d64Qwb_fixed ||
     612           0 :         TableEntry->RealOpc == ARM::VST1d8Twb_fixed ||
     613           0 :         TableEntry->RealOpc == ARM::VST1d16Twb_fixed ||
     614           0 :         TableEntry->RealOpc == ARM::VST1d32Twb_fixed ||
     615             :         TableEntry->RealOpc == ARM::VST1d64Twb_fixed) {
     616             :       assert(AM6Offset.getReg() == 0 &&
     617             :              "A fixed writing-back pseudo instruction provides an offset "
     618             :              "register!");
     619             :     } else {
     620             :       MIB.add(AM6Offset);
     621             :     }
     622             :   }
     623             : 
     624           0 :   bool SrcIsKill = MI.getOperand(OpIdx).isKill();
     625             :   bool SrcIsUndef = MI.getOperand(OpIdx).isUndef();
     626           0 :   unsigned SrcReg = MI.getOperand(OpIdx++).getReg();
     627             :   unsigned D0, D1, D2, D3;
     628           0 :   GetDSubRegs(SrcReg, RegSpc, TRI, D0, D1, D2, D3);
     629           0 :   MIB.addReg(D0, getUndefRegState(SrcIsUndef));
     630           0 :   if (NumRegs > 1 && TableEntry->copyAllListRegs)
     631           0 :     MIB.addReg(D1, getUndefRegState(SrcIsUndef));
     632           0 :   if (NumRegs > 2 && TableEntry->copyAllListRegs)
     633           0 :     MIB.addReg(D2, getUndefRegState(SrcIsUndef));
     634           0 :   if (NumRegs > 3 && TableEntry->copyAllListRegs)
     635           0 :     MIB.addReg(D3, getUndefRegState(SrcIsUndef));
     636             : 
     637             :   // Copy the predicate operands.
     638           0 :   MIB.add(MI.getOperand(OpIdx++));
     639           0 :   MIB.add(MI.getOperand(OpIdx++));
     640             : 
     641           0 :   if (SrcIsKill && !SrcIsUndef) // Add an implicit kill for the super-reg.
     642           0 :     MIB->addRegisterKilled(SrcReg, TRI, true);
     643           0 :   else if (!SrcIsUndef)
     644           0 :     MIB.addReg(SrcReg, RegState::Implicit); // Add implicit uses for src reg.
     645           0 :   TransferImpOps(MI, MIB, MIB);
     646             : 
     647             :   // Transfer memoperands.
     648             :   MIB.cloneMemRefs(MI);
     649             : 
     650           0 :   MI.eraseFromParent();
     651           0 : }
     652             : 
     653             : /// ExpandLaneOp - Translate VLD*LN and VST*LN instructions with Q, QQ or QQQQ
     654             : /// register operands to real instructions with D register operands.
     655           0 : void ARMExpandPseudo::ExpandLaneOp(MachineBasicBlock::iterator &MBBI) {
     656             :   MachineInstr &MI = *MBBI;
     657           0 :   MachineBasicBlock &MBB = *MI.getParent();
     658             : 
     659           0 :   const NEONLdStTableEntry *TableEntry = LookupNEONLdSt(MI.getOpcode());
     660             :   assert(TableEntry && "NEONLdStTable lookup failed");
     661           0 :   NEONRegSpacing RegSpc = (NEONRegSpacing)TableEntry->RegSpacing;
     662           0 :   unsigned NumRegs = TableEntry->NumRegs;
     663           0 :   unsigned RegElts = TableEntry->RegElts;
     664             : 
     665             :   MachineInstrBuilder MIB = BuildMI(MBB, MBBI, MI.getDebugLoc(),
     666           0 :                                     TII->get(TableEntry->RealOpc));
     667             :   unsigned OpIdx = 0;
     668             :   // The lane operand is always the 3rd from last operand, before the 2
     669             :   // predicate operands.
     670           0 :   unsigned Lane = MI.getOperand(MI.getDesc().getNumOperands() - 3).getImm();
     671             : 
     672             :   // Adjust the lane and spacing as needed for Q registers.
     673             :   assert(RegSpc != OddDblSpc && "unexpected register spacing for VLD/VST-lane");
     674           0 :   if (RegSpc == EvenDblSpc && Lane >= RegElts) {
     675             :     RegSpc = OddDblSpc;
     676           0 :     Lane -= RegElts;
     677             :   }
     678             :   assert(Lane < RegElts && "out of range lane for VLD/VST-lane");
     679             : 
     680           0 :   unsigned D0 = 0, D1 = 0, D2 = 0, D3 = 0;
     681             :   unsigned DstReg = 0;
     682             :   bool DstIsDead = false;
     683           0 :   if (TableEntry->IsLoad) {
     684             :     DstIsDead = MI.getOperand(OpIdx).isDead();
     685           0 :     DstReg = MI.getOperand(OpIdx++).getReg();
     686           0 :     GetDSubRegs(DstReg, RegSpc, TRI, D0, D1, D2, D3);
     687           0 :     MIB.addReg(D0, RegState::Define | getDeadRegState(DstIsDead));
     688           0 :     if (NumRegs > 1)
     689           0 :       MIB.addReg(D1, RegState::Define | getDeadRegState(DstIsDead));
     690           0 :     if (NumRegs > 2)
     691           0 :       MIB.addReg(D2, RegState::Define | getDeadRegState(DstIsDead));
     692           0 :     if (NumRegs > 3)
     693           0 :       MIB.addReg(D3, RegState::Define | getDeadRegState(DstIsDead));
     694             :   }
     695             : 
     696           0 :   if (TableEntry->isUpdating)
     697           0 :     MIB.add(MI.getOperand(OpIdx++));
     698             : 
     699             :   // Copy the addrmode6 operands.
     700           0 :   MIB.add(MI.getOperand(OpIdx++));
     701           0 :   MIB.add(MI.getOperand(OpIdx++));
     702             :   // Copy the am6offset operand.
     703           0 :   if (TableEntry->hasWritebackOperand)
     704           0 :     MIB.add(MI.getOperand(OpIdx++));
     705             : 
     706             :   // Grab the super-register source.
     707           0 :   MachineOperand MO = MI.getOperand(OpIdx++);
     708           0 :   if (!TableEntry->IsLoad)
     709           0 :     GetDSubRegs(MO.getReg(), RegSpc, TRI, D0, D1, D2, D3);
     710             : 
     711             :   // Add the subregs as sources of the new instruction.
     712             :   unsigned SrcFlags = (getUndefRegState(MO.isUndef()) |
     713           0 :                        getKillRegState(MO.isKill()));
     714           0 :   MIB.addReg(D0, SrcFlags);
     715           0 :   if (NumRegs > 1)
     716           0 :     MIB.addReg(D1, SrcFlags);
     717           0 :   if (NumRegs > 2)
     718           0 :     MIB.addReg(D2, SrcFlags);
     719           0 :   if (NumRegs > 3)
     720           0 :     MIB.addReg(D3, SrcFlags);
     721             : 
     722             :   // Add the lane number operand.
     723           0 :   MIB.addImm(Lane);
     724           0 :   OpIdx += 1;
     725             : 
     726             :   // Copy the predicate operands.
     727           0 :   MIB.add(MI.getOperand(OpIdx++));
     728           0 :   MIB.add(MI.getOperand(OpIdx++));
     729             : 
     730             :   // Copy the super-register source to be an implicit source.
     731             :   MO.setImplicit(true);
     732             :   MIB.add(MO);
     733           0 :   if (TableEntry->IsLoad)
     734             :     // Add an implicit def for the super-register.
     735           0 :     MIB.addReg(DstReg, RegState::ImplicitDefine | getDeadRegState(DstIsDead));
     736           0 :   TransferImpOps(MI, MIB, MIB);
     737             :   // Transfer memoperands.
     738             :   MIB.cloneMemRefs(MI);
     739           0 :   MI.eraseFromParent();
     740           0 : }
     741             : 
     742             : /// ExpandVTBL - Translate VTBL and VTBX pseudo instructions with Q or QQ
     743             : /// register operands to real instructions with D register operands.
     744           5 : void ARMExpandPseudo::ExpandVTBL(MachineBasicBlock::iterator &MBBI,
     745             :                                  unsigned Opc, bool IsExt) {
     746             :   MachineInstr &MI = *MBBI;
     747           5 :   MachineBasicBlock &MBB = *MI.getParent();
     748             : 
     749          10 :   MachineInstrBuilder MIB = BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(Opc));
     750             :   unsigned OpIdx = 0;
     751             : 
     752             :   // Transfer the destination register operand.
     753           5 :   MIB.add(MI.getOperand(OpIdx++));
     754           5 :   if (IsExt) {
     755           3 :     MachineOperand VdSrc(MI.getOperand(OpIdx++));
     756             :     MIB.add(VdSrc);
     757             :   }
     758             : 
     759           5 :   bool SrcIsKill = MI.getOperand(OpIdx).isKill();
     760           5 :   unsigned SrcReg = MI.getOperand(OpIdx++).getReg();
     761             :   unsigned D0, D1, D2, D3;
     762           5 :   GetDSubRegs(SrcReg, SingleSpc, TRI, D0, D1, D2, D3);
     763           5 :   MIB.addReg(D0);
     764             : 
     765             :   // Copy the other source register operand.
     766          10 :   MachineOperand VmSrc(MI.getOperand(OpIdx++));
     767             :   MIB.add(VmSrc);
     768             : 
     769             :   // Copy the predicate operands.
     770           5 :   MIB.add(MI.getOperand(OpIdx++));
     771           5 :   MIB.add(MI.getOperand(OpIdx++));
     772             : 
     773             :   // Add an implicit kill and use for the super-reg.
     774           5 :   MIB.addReg(SrcReg, RegState::Implicit | getKillRegState(SrcIsKill));
     775           5 :   TransferImpOps(MI, MIB, MIB);
     776           5 :   MI.eraseFromParent();
     777           5 : }
     778             : 
     779             : static bool IsAnAddressOperand(const MachineOperand &MO) {
     780             :   // This check is overly conservative.  Unless we are certain that the machine
     781             :   // operand is not a symbol reference, we return that it is a symbol reference.
     782             :   // This is important as the load pair may not be split up Windows.
     783             :   switch (MO.getType()) {
     784             :   case MachineOperand::MO_Register:
     785             :   case MachineOperand::MO_Immediate:
     786             :   case MachineOperand::MO_CImmediate:
     787             :   case MachineOperand::MO_FPImmediate:
     788             :     return false;
     789             :   case MachineOperand::MO_MachineBasicBlock:
     790             :     return true;
     791             :   case MachineOperand::MO_FrameIndex:
     792             :     return false;
     793             :   case MachineOperand::MO_ConstantPoolIndex:
     794             :   case MachineOperand::MO_TargetIndex:
     795             :   case MachineOperand::MO_JumpTableIndex:
     796             :   case MachineOperand::MO_ExternalSymbol:
     797             :   case MachineOperand::MO_GlobalAddress:
     798             :   case MachineOperand::MO_BlockAddress:
     799             :     return true;
     800             :   case MachineOperand::MO_RegisterMask:
     801             :   case MachineOperand::MO_RegisterLiveOut:
     802             :     return false;
     803             :   case MachineOperand::MO_Metadata:
     804             :   case MachineOperand::MO_MCSymbol:
     805             :     return true;
     806             :   case MachineOperand::MO_CFIIndex:
     807             :     return false;
     808             :   case MachineOperand::MO_IntrinsicID:
     809             :   case MachineOperand::MO_Predicate:
     810             :     llvm_unreachable("should not exist post-isel");
     811             :   }
     812           0 :   llvm_unreachable("unhandled machine operand type");
     813             : }
     814             : 
     815             : static MachineOperand makeImplicit(const MachineOperand &MO) {
     816        1341 :   MachineOperand NewMO = MO;
     817             :   NewMO.setImplicit();
     818             :   return NewMO;
     819             : }
     820             : 
     821           0 : void ARMExpandPseudo::ExpandMOV32BitImm(MachineBasicBlock &MBB,
     822             :                                         MachineBasicBlock::iterator &MBBI) {
     823             :   MachineInstr &MI = *MBBI;
     824           0 :   unsigned Opcode = MI.getOpcode();
     825           0 :   unsigned PredReg = 0;
     826           0 :   ARMCC::CondCodes Pred = getInstrPredicate(MI, PredReg);
     827           0 :   unsigned DstReg = MI.getOperand(0).getReg();
     828             :   bool DstIsDead = MI.getOperand(0).isDead();
     829           0 :   bool isCC = Opcode == ARM::MOVCCi32imm || Opcode == ARM::t2MOVCCi32imm;
     830           0 :   const MachineOperand &MO = MI.getOperand(isCC ? 2 : 1);
     831           0 :   bool RequiresBundling = STI->isTargetWindows() && IsAnAddressOperand(MO);
     832           0 :   MachineInstrBuilder LO16, HI16;
     833             : 
     834           0 :   if (!STI->hasV6T2Ops() &&
     835           0 :       (Opcode == ARM::MOVi32imm || Opcode == ARM::MOVCCi32imm)) {
     836             :     // FIXME Windows CE supports older ARM CPUs
     837             :     assert(!STI->isTargetWindows() && "Windows on ARM requires ARMv7+");
     838             : 
     839             :     // Expand into a movi + orr.
     840           0 :     LO16 = BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(ARM::MOVi), DstReg);
     841           0 :     HI16 = BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(ARM::ORRri))
     842           0 :       .addReg(DstReg, RegState::Define | getDeadRegState(DstIsDead))
     843           0 :       .addReg(DstReg);
     844             : 
     845             :     assert (MO.isImm() && "MOVi32imm w/ non-immediate source operand!");
     846           0 :     unsigned ImmVal = (unsigned)MO.getImm();
     847             :     unsigned SOImmValV1 = ARM_AM::getSOImmTwoPartFirst(ImmVal);
     848             :     unsigned SOImmValV2 = ARM_AM::getSOImmTwoPartSecond(ImmVal);
     849           0 :     LO16 = LO16.addImm(SOImmValV1);
     850           0 :     HI16 = HI16.addImm(SOImmValV2);
     851             :     LO16.cloneMemRefs(MI);
     852             :     HI16.cloneMemRefs(MI);
     853           0 :     LO16.addImm(Pred).addReg(PredReg).add(condCodeOp());
     854           0 :     HI16.addImm(Pred).addReg(PredReg).add(condCodeOp());
     855           0 :     if (isCC)
     856           0 :       LO16.add(makeImplicit(MI.getOperand(1)));
     857           0 :     TransferImpOps(MI, LO16, HI16);
     858           0 :     MI.eraseFromParent();
     859           0 :     return;
     860             :   }
     861             : 
     862             :   unsigned LO16Opc = 0;
     863             :   unsigned HI16Opc = 0;
     864           0 :   if (Opcode == ARM::t2MOVi32imm || Opcode == ARM::t2MOVCCi32imm) {
     865             :     LO16Opc = ARM::t2MOVi16;
     866             :     HI16Opc = ARM::t2MOVTi16;
     867             :   } else {
     868             :     LO16Opc = ARM::MOVi16;
     869             :     HI16Opc = ARM::MOVTi16;
     870             :   }
     871             : 
     872           0 :   LO16 = BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(LO16Opc), DstReg);
     873           0 :   HI16 = BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(HI16Opc))
     874           0 :     .addReg(DstReg, RegState::Define | getDeadRegState(DstIsDead))
     875           0 :     .addReg(DstReg);
     876             : 
     877           0 :   switch (MO.getType()) {
     878           0 :   case MachineOperand::MO_Immediate: {
     879           0 :     unsigned Imm = MO.getImm();
     880           0 :     unsigned Lo16 = Imm & 0xffff;
     881           0 :     unsigned Hi16 = (Imm >> 16) & 0xffff;
     882           0 :     LO16 = LO16.addImm(Lo16);
     883           0 :     HI16 = HI16.addImm(Hi16);
     884           0 :     break;
     885             :   }
     886           0 :   case MachineOperand::MO_ExternalSymbol: {
     887           0 :     const char *ES = MO.getSymbolName();
     888             :     unsigned TF = MO.getTargetFlags();
     889           0 :     LO16 = LO16.addExternalSymbol(ES, TF | ARMII::MO_LO16);
     890           0 :     HI16 = HI16.addExternalSymbol(ES, TF | ARMII::MO_HI16);
     891           0 :     break;
     892             :   }
     893           0 :   default: {
     894           0 :     const GlobalValue *GV = MO.getGlobal();
     895             :     unsigned TF = MO.getTargetFlags();
     896           0 :     LO16 = LO16.addGlobalAddress(GV, MO.getOffset(), TF | ARMII::MO_LO16);
     897           0 :     HI16 = HI16.addGlobalAddress(GV, MO.getOffset(), TF | ARMII::MO_HI16);
     898           0 :     break;
     899             :   }
     900             :   }
     901             : 
     902             :   LO16.cloneMemRefs(MI);
     903             :   HI16.cloneMemRefs(MI);
     904           0 :   LO16.addImm(Pred).addReg(PredReg);
     905           0 :   HI16.addImm(Pred).addReg(PredReg);
     906             : 
     907           0 :   if (RequiresBundling)
     908           0 :     finalizeBundle(MBB, LO16->getIterator(), MBBI->getIterator());
     909             : 
     910           0 :   if (isCC)
     911           0 :     LO16.add(makeImplicit(MI.getOperand(1)));
     912           0 :   TransferImpOps(MI, LO16, HI16);
     913           0 :   MI.eraseFromParent();
     914             : }
     915             : 
     916             : /// Expand a CMP_SWAP pseudo-inst to an ldrex/strex loop as simply as
     917             : /// possible. This only gets used at -O0 so we don't care about efficiency of
     918             : /// the generated code.
     919           0 : bool ARMExpandPseudo::ExpandCMP_SWAP(MachineBasicBlock &MBB,
     920             :                                      MachineBasicBlock::iterator MBBI,
     921             :                                      unsigned LdrexOp, unsigned StrexOp,
     922             :                                      unsigned UxtOp,
     923             :                                      MachineBasicBlock::iterator &NextMBBI) {
     924           0 :   bool IsThumb = STI->isThumb();
     925             :   MachineInstr &MI = *MBBI;
     926             :   DebugLoc DL = MI.getDebugLoc();
     927           0 :   const MachineOperand &Dest = MI.getOperand(0);
     928           0 :   unsigned TempReg = MI.getOperand(1).getReg();
     929             :   // Duplicating undef operands into 2 instructions does not guarantee the same
     930             :   // value on both; However undef should be replaced by xzr anyway.
     931             :   assert(!MI.getOperand(2).isUndef() && "cannot handle undef");
     932           0 :   unsigned AddrReg = MI.getOperand(2).getReg();
     933           0 :   unsigned DesiredReg = MI.getOperand(3).getReg();
     934           0 :   unsigned NewReg = MI.getOperand(4).getReg();
     935             : 
     936           0 :   MachineFunction *MF = MBB.getParent();
     937           0 :   auto LoadCmpBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
     938           0 :   auto StoreBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
     939           0 :   auto DoneBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
     940             : 
     941           0 :   MF->insert(++MBB.getIterator(), LoadCmpBB);
     942           0 :   MF->insert(++LoadCmpBB->getIterator(), StoreBB);
     943           0 :   MF->insert(++StoreBB->getIterator(), DoneBB);
     944             : 
     945           0 :   if (UxtOp) {
     946             :     MachineInstrBuilder MIB =
     947           0 :         BuildMI(MBB, MBBI, DL, TII->get(UxtOp), DesiredReg)
     948           0 :             .addReg(DesiredReg, RegState::Kill);
     949           0 :     if (!IsThumb)
     950             :       MIB.addImm(0);
     951           0 :     MIB.add(predOps(ARMCC::AL));
     952             :   }
     953             : 
     954             :   // .Lloadcmp:
     955             :   //     ldrex rDest, [rAddr]
     956             :   //     cmp rDest, rDesired
     957             :   //     bne .Ldone
     958             : 
     959           0 :   MachineInstrBuilder MIB;
     960           0 :   MIB = BuildMI(LoadCmpBB, DL, TII->get(LdrexOp), Dest.getReg());
     961           0 :   MIB.addReg(AddrReg);
     962           0 :   if (LdrexOp == ARM::t2LDREX)
     963             :     MIB.addImm(0); // a 32-bit Thumb ldrex (only) allows an offset.
     964           0 :   MIB.add(predOps(ARMCC::AL));
     965             : 
     966           0 :   unsigned CMPrr = IsThumb ? ARM::tCMPhir : ARM::CMPrr;
     967           0 :   BuildMI(LoadCmpBB, DL, TII->get(CMPrr))
     968           0 :       .addReg(Dest.getReg(), getKillRegState(Dest.isDead()))
     969           0 :       .addReg(DesiredReg)
     970           0 :       .add(predOps(ARMCC::AL));
     971           0 :   unsigned Bcc = IsThumb ? ARM::tBcc : ARM::Bcc;
     972           0 :   BuildMI(LoadCmpBB, DL, TII->get(Bcc))
     973             :       .addMBB(DoneBB)
     974             :       .addImm(ARMCC::NE)
     975           0 :       .addReg(ARM::CPSR, RegState::Kill);
     976           0 :   LoadCmpBB->addSuccessor(DoneBB);
     977           0 :   LoadCmpBB->addSuccessor(StoreBB);
     978             : 
     979             :   // .Lstore:
     980             :   //     strex rTempReg, rNew, [rAddr]
     981             :   //     cmp rTempReg, #0
     982             :   //     bne .Lloadcmp
     983           0 :   MIB = BuildMI(StoreBB, DL, TII->get(StrexOp), TempReg)
     984           0 :     .addReg(NewReg)
     985           0 :     .addReg(AddrReg);
     986           0 :   if (StrexOp == ARM::t2STREX)
     987             :     MIB.addImm(0); // a 32-bit Thumb strex (only) allows an offset.
     988           0 :   MIB.add(predOps(ARMCC::AL));
     989             : 
     990           0 :   unsigned CMPri = IsThumb ? ARM::t2CMPri : ARM::CMPri;
     991           0 :   BuildMI(StoreBB, DL, TII->get(CMPri))
     992           0 :       .addReg(TempReg, RegState::Kill)
     993             :       .addImm(0)
     994           0 :       .add(predOps(ARMCC::AL));
     995           0 :   BuildMI(StoreBB, DL, TII->get(Bcc))
     996             :       .addMBB(LoadCmpBB)
     997             :       .addImm(ARMCC::NE)
     998           0 :       .addReg(ARM::CPSR, RegState::Kill);
     999           0 :   StoreBB->addSuccessor(LoadCmpBB);
    1000           0 :   StoreBB->addSuccessor(DoneBB);
    1001             : 
    1002             :   DoneBB->splice(DoneBB->end(), &MBB, MI, MBB.end());
    1003           0 :   DoneBB->transferSuccessors(&MBB);
    1004             : 
    1005           0 :   MBB.addSuccessor(LoadCmpBB);
    1006             : 
    1007           0 :   NextMBBI = MBB.end();
    1008           0 :   MI.eraseFromParent();
    1009             : 
    1010             :   // Recompute livein lists.
    1011             :   LivePhysRegs LiveRegs;
    1012           0 :   computeAndAddLiveIns(LiveRegs, *DoneBB);
    1013           0 :   computeAndAddLiveIns(LiveRegs, *StoreBB);
    1014           0 :   computeAndAddLiveIns(LiveRegs, *LoadCmpBB);
    1015             :   // Do an extra pass around the loop to get loop carried registers right.
    1016           0 :   StoreBB->clearLiveIns();
    1017           0 :   computeAndAddLiveIns(LiveRegs, *StoreBB);
    1018           0 :   LoadCmpBB->clearLiveIns();
    1019           0 :   computeAndAddLiveIns(LiveRegs, *LoadCmpBB);
    1020             : 
    1021           0 :   return true;
    1022             : }
    1023             : 
    1024             : /// ARM's ldrexd/strexd take a consecutive register pair (represented as a
    1025             : /// single GPRPair register), Thumb's take two separate registers so we need to
    1026             : /// extract the subregs from the pair.
    1027          14 : static void addExclusiveRegPair(MachineInstrBuilder &MIB, MachineOperand &Reg,
    1028             :                                 unsigned Flags, bool IsThumb,
    1029             :                                 const TargetRegisterInfo *TRI) {
    1030          14 :   if (IsThumb) {
    1031           6 :     unsigned RegLo = TRI->getSubReg(Reg.getReg(), ARM::gsub_0);
    1032           6 :     unsigned RegHi = TRI->getSubReg(Reg.getReg(), ARM::gsub_1);
    1033           6 :     MIB.addReg(RegLo, Flags | getKillRegState(Reg.isDead()));
    1034           6 :     MIB.addReg(RegHi, Flags | getKillRegState(Reg.isDead()));
    1035             :   } else
    1036           8 :     MIB.addReg(Reg.getReg(), Flags | getKillRegState(Reg.isDead()));
    1037          14 : }
    1038             : 
    1039             : /// Expand a 64-bit CMP_SWAP to an ldrexd/strexd loop.
    1040           7 : bool ARMExpandPseudo::ExpandCMP_SWAP_64(MachineBasicBlock &MBB,
    1041             :                                         MachineBasicBlock::iterator MBBI,
    1042             :                                         MachineBasicBlock::iterator &NextMBBI) {
    1043           7 :   bool IsThumb = STI->isThumb();
    1044             :   MachineInstr &MI = *MBBI;
    1045             :   DebugLoc DL = MI.getDebugLoc();
    1046           7 :   MachineOperand &Dest = MI.getOperand(0);
    1047           7 :   unsigned TempReg = MI.getOperand(1).getReg();
    1048             :   // Duplicating undef operands into 2 instructions does not guarantee the same
    1049             :   // value on both; However undef should be replaced by xzr anyway.
    1050             :   assert(!MI.getOperand(2).isUndef() && "cannot handle undef");
    1051           7 :   unsigned AddrReg = MI.getOperand(2).getReg();
    1052           7 :   unsigned DesiredReg = MI.getOperand(3).getReg();
    1053           7 :   MachineOperand New = MI.getOperand(4);
    1054             :   New.setIsKill(false);
    1055             : 
    1056           7 :   unsigned DestLo = TRI->getSubReg(Dest.getReg(), ARM::gsub_0);
    1057           7 :   unsigned DestHi = TRI->getSubReg(Dest.getReg(), ARM::gsub_1);
    1058           7 :   unsigned DesiredLo = TRI->getSubReg(DesiredReg, ARM::gsub_0);
    1059           7 :   unsigned DesiredHi = TRI->getSubReg(DesiredReg, ARM::gsub_1);
    1060             : 
    1061           7 :   MachineFunction *MF = MBB.getParent();
    1062           7 :   auto LoadCmpBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
    1063           7 :   auto StoreBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
    1064           7 :   auto DoneBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
    1065             : 
    1066           7 :   MF->insert(++MBB.getIterator(), LoadCmpBB);
    1067           7 :   MF->insert(++LoadCmpBB->getIterator(), StoreBB);
    1068           7 :   MF->insert(++StoreBB->getIterator(), DoneBB);
    1069             : 
    1070             :   // .Lloadcmp:
    1071             :   //     ldrexd rDestLo, rDestHi, [rAddr]
    1072             :   //     cmp rDestLo, rDesiredLo
    1073             :   //     sbcs dead rTempReg, rDestHi, rDesiredHi
    1074             :   //     bne .Ldone
    1075           7 :   unsigned LDREXD = IsThumb ? ARM::t2LDREXD : ARM::LDREXD;
    1076           7 :   MachineInstrBuilder MIB;
    1077           7 :   MIB = BuildMI(LoadCmpBB, DL, TII->get(LDREXD));
    1078           7 :   addExclusiveRegPair(MIB, Dest, RegState::Define, IsThumb, TRI);
    1079           7 :   MIB.addReg(AddrReg).add(predOps(ARMCC::AL));
    1080             : 
    1081           7 :   unsigned CMPrr = IsThumb ? ARM::tCMPhir : ARM::CMPrr;
    1082           7 :   BuildMI(LoadCmpBB, DL, TII->get(CMPrr))
    1083           7 :       .addReg(DestLo, getKillRegState(Dest.isDead()))
    1084           7 :       .addReg(DesiredLo)
    1085           7 :       .add(predOps(ARMCC::AL));
    1086             : 
    1087           7 :   BuildMI(LoadCmpBB, DL, TII->get(CMPrr))
    1088           7 :       .addReg(DestHi, getKillRegState(Dest.isDead()))
    1089           7 :       .addReg(DesiredHi)
    1090           7 :       .addImm(ARMCC::EQ).addReg(ARM::CPSR, RegState::Kill);
    1091             : 
    1092           7 :   unsigned Bcc = IsThumb ? ARM::tBcc : ARM::Bcc;
    1093           7 :   BuildMI(LoadCmpBB, DL, TII->get(Bcc))
    1094             :       .addMBB(DoneBB)
    1095             :       .addImm(ARMCC::NE)
    1096           7 :       .addReg(ARM::CPSR, RegState::Kill);
    1097           7 :   LoadCmpBB->addSuccessor(DoneBB);
    1098           7 :   LoadCmpBB->addSuccessor(StoreBB);
    1099             : 
    1100             :   // .Lstore:
    1101             :   //     strexd rTempReg, rNewLo, rNewHi, [rAddr]
    1102             :   //     cmp rTempReg, #0
    1103             :   //     bne .Lloadcmp
    1104           7 :   unsigned STREXD = IsThumb ? ARM::t2STREXD : ARM::STREXD;
    1105           7 :   MIB = BuildMI(StoreBB, DL, TII->get(STREXD), TempReg);
    1106           7 :   addExclusiveRegPair(MIB, New, 0, IsThumb, TRI);
    1107           7 :   MIB.addReg(AddrReg).add(predOps(ARMCC::AL));
    1108             : 
    1109           7 :   unsigned CMPri = IsThumb ? ARM::t2CMPri : ARM::CMPri;
    1110           7 :   BuildMI(StoreBB, DL, TII->get(CMPri))
    1111           7 :       .addReg(TempReg, RegState::Kill)
    1112             :       .addImm(0)
    1113           7 :       .add(predOps(ARMCC::AL));
    1114           7 :   BuildMI(StoreBB, DL, TII->get(Bcc))
    1115             :       .addMBB(LoadCmpBB)
    1116             :       .addImm(ARMCC::NE)
    1117           7 :       .addReg(ARM::CPSR, RegState::Kill);
    1118           7 :   StoreBB->addSuccessor(LoadCmpBB);
    1119           7 :   StoreBB->addSuccessor(DoneBB);
    1120             : 
    1121             :   DoneBB->splice(DoneBB->end(), &MBB, MI, MBB.end());
    1122           7 :   DoneBB->transferSuccessors(&MBB);
    1123             : 
    1124           7 :   MBB.addSuccessor(LoadCmpBB);
    1125             : 
    1126           7 :   NextMBBI = MBB.end();
    1127           7 :   MI.eraseFromParent();
    1128             : 
    1129             :   // Recompute livein lists.
    1130             :   LivePhysRegs LiveRegs;
    1131           7 :   computeAndAddLiveIns(LiveRegs, *DoneBB);
    1132           7 :   computeAndAddLiveIns(LiveRegs, *StoreBB);
    1133           7 :   computeAndAddLiveIns(LiveRegs, *LoadCmpBB);
    1134             :   // Do an extra pass around the loop to get loop carried registers right.
    1135           7 :   StoreBB->clearLiveIns();
    1136           7 :   computeAndAddLiveIns(LiveRegs, *StoreBB);
    1137           7 :   LoadCmpBB->clearLiveIns();
    1138           7 :   computeAndAddLiveIns(LiveRegs, *LoadCmpBB);
    1139             : 
    1140           7 :   return true;
    1141             : }
    1142             : 
    1143             : 
    1144      161138 : bool ARMExpandPseudo::ExpandMI(MachineBasicBlock &MBB,
    1145             :                                MachineBasicBlock::iterator MBBI,
    1146             :                                MachineBasicBlock::iterator &NextMBBI) {
    1147             :   MachineInstr &MI = *MBBI;
    1148      161138 :   unsigned Opcode = MI.getOpcode();
    1149      161138 :   switch (Opcode) {
    1150             :     default:
    1151             :       return false;
    1152             : 
    1153         574 :     case ARM::TCRETURNdi:
    1154             :     case ARM::TCRETURNri: {
    1155         574 :       MachineBasicBlock::iterator MBBI = MBB.getLastNonDebugInstr();
    1156             :       assert(MBBI->isReturn() &&
    1157             :              "Can only insert epilog into returning blocks");
    1158         574 :       unsigned RetOpcode = MBBI->getOpcode();
    1159             :       DebugLoc dl = MBBI->getDebugLoc();
    1160             :       const ARMBaseInstrInfo &TII = *static_cast<const ARMBaseInstrInfo *>(
    1161         574 :           MBB.getParent()->getSubtarget().getInstrInfo());
    1162             : 
    1163             :       // Tail call return: adjust the stack pointer and jump to callee.
    1164         574 :       MBBI = MBB.getLastNonDebugInstr();
    1165         574 :       MachineOperand &JumpTarget = MBBI->getOperand(0);
    1166             : 
    1167             :       // Jump to label or value in register.
    1168         574 :       if (RetOpcode == ARM::TCRETURNdi) {
    1169             :         unsigned TCOpcode =
    1170         554 :             STI->isThumb()
    1171         554 :                 ? (STI->isTargetMachO() ? ARM::tTAILJMPd : ARM::tTAILJMPdND)
    1172             :                 : ARM::TAILJMPd;
    1173        1108 :         MachineInstrBuilder MIB = BuildMI(MBB, MBBI, dl, TII.get(TCOpcode));
    1174         554 :         if (JumpTarget.isGlobal())
    1175             :           MIB.addGlobalAddress(JumpTarget.getGlobal(), JumpTarget.getOffset(),
    1176         666 :                                JumpTarget.getTargetFlags());
    1177             :         else {
    1178             :           assert(JumpTarget.isSymbol());
    1179             :           MIB.addExternalSymbol(JumpTarget.getSymbolName(),
    1180         221 :                                 JumpTarget.getTargetFlags());
    1181             :         }
    1182             : 
    1183             :         // Add the default predicate in Thumb mode.
    1184         554 :         if (STI->isThumb())
    1185         363 :           MIB.add(predOps(ARMCC::AL));
    1186          20 :       } else if (RetOpcode == ARM::TCRETURNri) {
    1187             :         unsigned Opcode =
    1188          20 :           STI->isThumb() ? ARM::tTAILJMPr
    1189           9 :                          : (STI->hasV4TOps() ? ARM::TAILJMPr : ARM::TAILJMPr4);
    1190          20 :         BuildMI(MBB, MBBI, dl,
    1191          40 :                 TII.get(Opcode))
    1192          20 :             .addReg(JumpTarget.getReg(), RegState::Kill);
    1193             :       }
    1194             : 
    1195         574 :       auto NewMI = std::prev(MBBI);
    1196        1766 :       for (unsigned i = 1, e = MBBI->getNumOperands(); i != e; ++i)
    1197        2384 :         NewMI->addOperand(MBBI->getOperand(i));
    1198             : 
    1199             :       // Delete the pseudo instruction TCRETURN.
    1200         574 :       MBB.erase(MBBI);
    1201             :       MBBI = NewMI;
    1202             :       return true;
    1203             :     }
    1204         136 :     case ARM::VMOVScc:
    1205             :     case ARM::VMOVDcc: {
    1206         136 :       unsigned newOpc = Opcode == ARM::VMOVScc ? ARM::VMOVS : ARM::VMOVD;
    1207         136 :       BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(newOpc),
    1208         272 :               MI.getOperand(1).getReg())
    1209         136 :           .add(MI.getOperand(2))
    1210         136 :           .addImm(MI.getOperand(3).getImm()) // 'pred'
    1211         136 :           .add(MI.getOperand(4))
    1212         136 :           .add(makeImplicit(MI.getOperand(1)));
    1213             : 
    1214         136 :       MI.eraseFromParent();
    1215         136 :       return true;
    1216             :     }
    1217         304 :     case ARM::t2MOVCCr:
    1218             :     case ARM::MOVCCr: {
    1219         304 :       unsigned Opc = AFI->isThumbFunction() ? ARM::t2MOVr : ARM::MOVr;
    1220         304 :       BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(Opc),
    1221         608 :               MI.getOperand(1).getReg())
    1222         304 :           .add(MI.getOperand(2))
    1223         304 :           .addImm(MI.getOperand(3).getImm()) // 'pred'
    1224         304 :           .add(MI.getOperand(4))
    1225         304 :           .add(condCodeOp()) // 's' bit
    1226         304 :           .add(makeImplicit(MI.getOperand(1)));
    1227             : 
    1228         304 :       MI.eraseFromParent();
    1229         304 :       return true;
    1230             :     }
    1231           2 :     case ARM::MOVCCsi: {
    1232           2 :       BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(ARM::MOVsi),
    1233           4 :               (MI.getOperand(1).getReg()))
    1234           2 :           .add(MI.getOperand(2))
    1235           2 :           .addImm(MI.getOperand(3).getImm())
    1236           2 :           .addImm(MI.getOperand(4).getImm()) // 'pred'
    1237           2 :           .add(MI.getOperand(5))
    1238           2 :           .add(condCodeOp()) // 's' bit
    1239           2 :           .add(makeImplicit(MI.getOperand(1)));
    1240             : 
    1241           2 :       MI.eraseFromParent();
    1242           2 :       return true;
    1243             :     }
    1244           7 :     case ARM::MOVCCsr: {
    1245           7 :       BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(ARM::MOVsr),
    1246          14 :               (MI.getOperand(1).getReg()))
    1247           7 :           .add(MI.getOperand(2))
    1248           7 :           .add(MI.getOperand(3))
    1249           7 :           .addImm(MI.getOperand(4).getImm())
    1250           7 :           .addImm(MI.getOperand(5).getImm()) // 'pred'
    1251           7 :           .add(MI.getOperand(6))
    1252           7 :           .add(condCodeOp()) // 's' bit
    1253           7 :           .add(makeImplicit(MI.getOperand(1)));
    1254             : 
    1255           7 :       MI.eraseFromParent();
    1256           7 :       return true;
    1257             :     }
    1258          92 :     case ARM::t2MOVCCi16:
    1259             :     case ARM::MOVCCi16: {
    1260          92 :       unsigned NewOpc = AFI->isThumbFunction() ? ARM::t2MOVi16 : ARM::MOVi16;
    1261          92 :       BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(NewOpc),
    1262         184 :               MI.getOperand(1).getReg())
    1263          92 :           .addImm(MI.getOperand(2).getImm())
    1264          92 :           .addImm(MI.getOperand(3).getImm()) // 'pred'
    1265          92 :           .add(MI.getOperand(4))
    1266          92 :           .add(makeImplicit(MI.getOperand(1)));
    1267          92 :       MI.eraseFromParent();
    1268          92 :       return true;
    1269             :     }
    1270         736 :     case ARM::t2MOVCCi:
    1271             :     case ARM::MOVCCi: {
    1272         736 :       unsigned Opc = AFI->isThumbFunction() ? ARM::t2MOVi : ARM::MOVi;
    1273         736 :       BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(Opc),
    1274        1472 :               MI.getOperand(1).getReg())
    1275         736 :           .addImm(MI.getOperand(2).getImm())
    1276         736 :           .addImm(MI.getOperand(3).getImm()) // 'pred'
    1277         736 :           .add(MI.getOperand(4))
    1278         736 :           .add(condCodeOp()) // 's' bit
    1279         736 :           .add(makeImplicit(MI.getOperand(1)));
    1280             : 
    1281         736 :       MI.eraseFromParent();
    1282         736 :       return true;
    1283             :     }
    1284          60 :     case ARM::t2MVNCCi:
    1285             :     case ARM::MVNCCi: {
    1286          60 :       unsigned Opc = AFI->isThumbFunction() ? ARM::t2MVNi : ARM::MVNi;
    1287          60 :       BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(Opc),
    1288         120 :               MI.getOperand(1).getReg())
    1289          60 :           .addImm(MI.getOperand(2).getImm())
    1290          60 :           .addImm(MI.getOperand(3).getImm()) // 'pred'
    1291          60 :           .add(MI.getOperand(4))
    1292          60 :           .add(condCodeOp()) // 's' bit
    1293          60 :           .add(makeImplicit(MI.getOperand(1)));
    1294             : 
    1295          60 :       MI.eraseFromParent();
    1296          60 :       return true;
    1297             :     }
    1298           4 :     case ARM::t2MOVCClsl:
    1299             :     case ARM::t2MOVCClsr:
    1300             :     case ARM::t2MOVCCasr:
    1301             :     case ARM::t2MOVCCror: {
    1302             :       unsigned NewOpc;
    1303             :       switch (Opcode) {
    1304             :       case ARM::t2MOVCClsl: NewOpc = ARM::t2LSLri; break;
    1305           1 :       case ARM::t2MOVCClsr: NewOpc = ARM::t2LSRri; break;
    1306           0 :       case ARM::t2MOVCCasr: NewOpc = ARM::t2ASRri; break;
    1307           1 :       case ARM::t2MOVCCror: NewOpc = ARM::t2RORri; break;
    1308           0 :       default: llvm_unreachable("unexpeced conditional move");
    1309             :       }
    1310           4 :       BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(NewOpc),
    1311           8 :               MI.getOperand(1).getReg())
    1312           4 :           .add(MI.getOperand(2))
    1313           4 :           .addImm(MI.getOperand(3).getImm())
    1314           4 :           .addImm(MI.getOperand(4).getImm()) // 'pred'
    1315           4 :           .add(MI.getOperand(5))
    1316           4 :           .add(condCodeOp()) // 's' bit
    1317           4 :           .add(makeImplicit(MI.getOperand(1)));
    1318           4 :       MI.eraseFromParent();
    1319           4 :       return true;
    1320             :     }
    1321          32 :     case ARM::Int_eh_sjlj_dispatchsetup: {
    1322          32 :       MachineFunction &MF = *MI.getParent()->getParent();
    1323          32 :       const ARMBaseInstrInfo *AII =
    1324             :         static_cast<const ARMBaseInstrInfo*>(TII);
    1325          32 :       const ARMBaseRegisterInfo &RI = AII->getRegisterInfo();
    1326             :       // For functions using a base pointer, we rematerialize it (via the frame
    1327             :       // pointer) here since eh.sjlj.setjmp and eh.sjlj.longjmp don't do it
    1328             :       // for us. Otherwise, expand to nothing.
    1329          32 :       if (RI.hasBasePointer(MF)) {
    1330           0 :         int32_t NumBytes = AFI->getFramePtrSpillOffset();
    1331           0 :         unsigned FramePtr = RI.getFrameRegister(MF);
    1332             :         assert(MF.getSubtarget().getFrameLowering()->hasFP(MF) &&
    1333             :                "base pointer without frame pointer?");
    1334             : 
    1335           0 :         if (AFI->isThumb2Function()) {
    1336           0 :           emitT2RegPlusImmediate(MBB, MBBI, MI.getDebugLoc(), ARM::R6,
    1337           0 :                                  FramePtr, -NumBytes, ARMCC::AL, 0, *TII);
    1338           0 :         } else if (AFI->isThumbFunction()) {
    1339           0 :           emitThumbRegPlusImmediate(MBB, MBBI, MI.getDebugLoc(), ARM::R6,
    1340           0 :                                     FramePtr, -NumBytes, *TII, RI);
    1341             :         } else {
    1342           0 :           emitARMRegPlusImmediate(MBB, MBBI, MI.getDebugLoc(), ARM::R6,
    1343             :                                   FramePtr, -NumBytes, ARMCC::AL, 0,
    1344           0 :                                   *TII);
    1345             :         }
    1346             :         // If there's dynamic realignment, adjust for it.
    1347           0 :         if (RI.needsStackRealignment(MF)) {
    1348           0 :           MachineFrameInfo &MFI = MF.getFrameInfo();
    1349           0 :           unsigned MaxAlign = MFI.getMaxAlignment();
    1350             :           assert (!AFI->isThumb1OnlyFunction());
    1351             :           // Emit bic r6, r6, MaxAlign
    1352             :           assert(MaxAlign <= 256 && "The BIC instruction cannot encode "
    1353             :                                     "immediates larger than 256 with all lower "
    1354             :                                     "bits set.");
    1355           0 :           unsigned bicOpc = AFI->isThumbFunction() ?
    1356             :             ARM::t2BICri : ARM::BICri;
    1357           0 :           BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(bicOpc), ARM::R6)
    1358           0 :               .addReg(ARM::R6, RegState::Kill)
    1359           0 :               .addImm(MaxAlign - 1)
    1360           0 :               .add(predOps(ARMCC::AL))
    1361           0 :               .add(condCodeOp());
    1362             :         }
    1363             : 
    1364             :       }
    1365          32 :       MI.eraseFromParent();
    1366          32 :       return true;
    1367             :     }
    1368             : 
    1369           2 :     case ARM::MOVsrl_flag:
    1370             :     case ARM::MOVsra_flag: {
    1371             :       // These are just fancy MOVs instructions.
    1372           4 :       BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(ARM::MOVsi),
    1373           4 :               MI.getOperand(0).getReg())
    1374           2 :           .add(MI.getOperand(1))
    1375           2 :           .addImm(ARM_AM::getSORegOpc(
    1376           2 :               (Opcode == ARM::MOVsrl_flag ? ARM_AM::lsr : ARM_AM::asr), 1))
    1377           2 :           .add(predOps(ARMCC::AL))
    1378           2 :           .addReg(ARM::CPSR, RegState::Define);
    1379           2 :       MI.eraseFromParent();
    1380           2 :       return true;
    1381             :     }
    1382           2 :     case ARM::RRX: {
    1383             :       // This encodes as "MOVs Rd, Rm, rrx
    1384             :       MachineInstrBuilder MIB =
    1385           2 :           BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(ARM::MOVsi),
    1386           4 :                   MI.getOperand(0).getReg())
    1387           2 :               .add(MI.getOperand(1))
    1388             :               .addImm(ARM_AM::getSORegOpc(ARM_AM::rrx, 0))
    1389           2 :               .add(predOps(ARMCC::AL))
    1390           2 :               .add(condCodeOp());
    1391           2 :       TransferImpOps(MI, MIB, MIB);
    1392           2 :       MI.eraseFromParent();
    1393             :       return true;
    1394             :     }
    1395          39 :     case ARM::tTPsoft:
    1396             :     case ARM::TPsoft: {
    1397             :       const bool Thumb = Opcode == ARM::tTPsoft;
    1398             : 
    1399          39 :       MachineInstrBuilder MIB;
    1400          39 :       if (STI->genLongCalls()) {
    1401           2 :         MachineFunction *MF = MBB.getParent();
    1402           2 :         MachineConstantPool *MCP = MF->getConstantPool();
    1403           2 :         unsigned PCLabelID = AFI->createPICLabelUId();
    1404             :         MachineConstantPoolValue *CPV =
    1405           2 :             ARMConstantPoolSymbol::Create(MF->getFunction().getContext(),
    1406             :                                           "__aeabi_read_tp", PCLabelID, 0);
    1407           2 :         unsigned Reg = MI.getOperand(0).getReg();
    1408           2 :         MIB = BuildMI(MBB, MBBI, MI.getDebugLoc(),
    1409           4 :                       TII->get(Thumb ? ARM::tLDRpci : ARM::LDRi12), Reg)
    1410           2 :                   .addConstantPoolIndex(MCP->getConstantPoolIndex(CPV, 4));
    1411           2 :         if (!Thumb)
    1412             :           MIB.addImm(0);
    1413           2 :         MIB.add(predOps(ARMCC::AL));
    1414             : 
    1415             :         MIB = BuildMI(MBB, MBBI, MI.getDebugLoc(),
    1416           4 :                       TII->get(Thumb ? ARM::tBLXr : ARM::BLX));
    1417           2 :         if (Thumb)
    1418           1 :           MIB.add(predOps(ARMCC::AL));
    1419           2 :         MIB.addReg(Reg, RegState::Kill);
    1420             :       } else {
    1421             :         MIB = BuildMI(MBB, MBBI, MI.getDebugLoc(),
    1422          89 :                       TII->get(Thumb ? ARM::tBL : ARM::BL));
    1423          37 :         if (Thumb)
    1424          11 :           MIB.add(predOps(ARMCC::AL));
    1425             :         MIB.addExternalSymbol("__aeabi_read_tp", 0);
    1426             :       }
    1427             : 
    1428             :       MIB.cloneMemRefs(MI);
    1429          39 :       TransferImpOps(MI, MIB, MIB);
    1430          39 :       MI.eraseFromParent();
    1431             :       return true;
    1432             :     }
    1433          45 :     case ARM::tLDRpci_pic:
    1434             :     case ARM::t2LDRpci_pic: {
    1435             :       unsigned NewLdOpc = (Opcode == ARM::tLDRpci_pic)
    1436          45 :         ? ARM::tLDRpci : ARM::t2LDRpci;
    1437          45 :       unsigned DstReg = MI.getOperand(0).getReg();
    1438             :       bool DstIsDead = MI.getOperand(0).isDead();
    1439             :       MachineInstrBuilder MIB1 =
    1440          90 :           BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(NewLdOpc), DstReg)
    1441          45 :               .add(MI.getOperand(1))
    1442          45 :               .add(predOps(ARMCC::AL));
    1443             :       MIB1.cloneMemRefs(MI);
    1444             :       MachineInstrBuilder MIB2 =
    1445          90 :           BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(ARM::tPICADD))
    1446          45 :               .addReg(DstReg, RegState::Define | getDeadRegState(DstIsDead))
    1447          45 :               .addReg(DstReg)
    1448          45 :               .add(MI.getOperand(2));
    1449          45 :       TransferImpOps(MI, MIB1, MIB2);
    1450          45 :       MI.eraseFromParent();
    1451             :       return true;
    1452             :     }
    1453             : 
    1454         365 :     case ARM::LDRLIT_ga_abs:
    1455             :     case ARM::LDRLIT_ga_pcrel:
    1456             :     case ARM::LDRLIT_ga_pcrel_ldr:
    1457             :     case ARM::tLDRLIT_ga_abs:
    1458             :     case ARM::tLDRLIT_ga_pcrel: {
    1459         365 :       unsigned DstReg = MI.getOperand(0).getReg();
    1460             :       bool DstIsDead = MI.getOperand(0).isDead();
    1461             :       const MachineOperand &MO1 = MI.getOperand(1);
    1462             :       auto Flags = MO1.getTargetFlags();
    1463         365 :       const GlobalValue *GV = MO1.getGlobal();
    1464             :       bool IsARM =
    1465         365 :           Opcode != ARM::tLDRLIT_ga_pcrel && Opcode != ARM::tLDRLIT_ga_abs;
    1466         365 :       bool IsPIC =
    1467         365 :           Opcode != ARM::LDRLIT_ga_abs && Opcode != ARM::tLDRLIT_ga_abs;
    1468         365 :       unsigned LDRLITOpc = IsARM ? ARM::LDRi12 : ARM::tLDRpci;
    1469             :       unsigned PICAddOpc =
    1470             :           IsARM
    1471         365 :               ? (Opcode == ARM::LDRLIT_ga_pcrel_ldr ? ARM::PICLDR : ARM::PICADD)
    1472             :               : ARM::tPICADD;
    1473             : 
    1474             :       // We need a new const-pool entry to load from.
    1475         365 :       MachineConstantPool *MCP = MBB.getParent()->getConstantPool();
    1476             :       unsigned ARMPCLabelIndex = 0;
    1477             :       MachineConstantPoolValue *CPV;
    1478             : 
    1479         365 :       if (IsPIC) {
    1480         328 :         unsigned PCAdj = IsARM ? 8 : 4;
    1481         328 :         auto Modifier = (Flags & ARMII::MO_GOT)
    1482         328 :                             ? ARMCP::GOT_PREL
    1483             :                             : ARMCP::no_modifier;
    1484         328 :         ARMPCLabelIndex = AFI->createPICLabelUId();
    1485         328 :         CPV = ARMConstantPoolConstant::Create(
    1486             :             GV, ARMPCLabelIndex, ARMCP::CPValue, PCAdj, Modifier,
    1487             :             /*AddCurrentAddr*/ Modifier == ARMCP::GOT_PREL);
    1488             :       } else
    1489          37 :         CPV = ARMConstantPoolConstant::Create(GV, ARMCP::no_modifier);
    1490             : 
    1491             :       MachineInstrBuilder MIB =
    1492         730 :           BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(LDRLITOpc), DstReg)
    1493         365 :             .addConstantPoolIndex(MCP->getConstantPoolIndex(CPV, 4));
    1494         365 :       if (IsARM)
    1495             :         MIB.addImm(0);
    1496         365 :       MIB.add(predOps(ARMCC::AL));
    1497             : 
    1498         365 :       if (IsPIC) {
    1499             :         MachineInstrBuilder MIB =
    1500         656 :           BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(PICAddOpc))
    1501         328 :             .addReg(DstReg, RegState::Define | getDeadRegState(DstIsDead))
    1502         328 :             .addReg(DstReg)
    1503         328 :             .addImm(ARMPCLabelIndex);
    1504             : 
    1505         328 :         if (IsARM)
    1506         182 :           MIB.add(predOps(ARMCC::AL));
    1507             :       }
    1508             : 
    1509         365 :       MI.eraseFromParent();
    1510             :       return true;
    1511             :     }
    1512         624 :     case ARM::MOV_ga_pcrel:
    1513             :     case ARM::MOV_ga_pcrel_ldr:
    1514             :     case ARM::t2MOV_ga_pcrel: {
    1515             :       // Expand into movw + movw. Also "add pc" / ldr [pc] in PIC mode.
    1516         624 :       unsigned LabelId = AFI->createPICLabelUId();
    1517         624 :       unsigned DstReg = MI.getOperand(0).getReg();
    1518             :       bool DstIsDead = MI.getOperand(0).isDead();
    1519             :       const MachineOperand &MO1 = MI.getOperand(1);
    1520         624 :       const GlobalValue *GV = MO1.getGlobal();
    1521             :       unsigned TF = MO1.getTargetFlags();
    1522             :       bool isARM = Opcode != ARM::t2MOV_ga_pcrel;
    1523         624 :       unsigned LO16Opc = isARM ? ARM::MOVi16_ga_pcrel : ARM::t2MOVi16_ga_pcrel;
    1524         624 :       unsigned HI16Opc = isARM ? ARM::MOVTi16_ga_pcrel :ARM::t2MOVTi16_ga_pcrel;
    1525             :       unsigned LO16TF = TF | ARMII::MO_LO16;
    1526             :       unsigned HI16TF = TF | ARMII::MO_HI16;
    1527             :       unsigned PICAddOpc = isARM
    1528         624 :         ? (Opcode == ARM::MOV_ga_pcrel_ldr ? ARM::PICLDR : ARM::PICADD)
    1529             :         : ARM::tPICADD;
    1530             :       MachineInstrBuilder MIB1 = BuildMI(MBB, MBBI, MI.getDebugLoc(),
    1531        1248 :                                          TII->get(LO16Opc), DstReg)
    1532         624 :         .addGlobalAddress(GV, MO1.getOffset(), TF | LO16TF)
    1533         624 :         .addImm(LabelId);
    1534             : 
    1535        1248 :       BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(HI16Opc), DstReg)
    1536         624 :         .addReg(DstReg)
    1537         624 :         .addGlobalAddress(GV, MO1.getOffset(), TF | HI16TF)
    1538             :         .addImm(LabelId);
    1539             : 
    1540         624 :       MachineInstrBuilder MIB3 = BuildMI(MBB, MBBI, MI.getDebugLoc(),
    1541        1248 :                                          TII->get(PICAddOpc))
    1542         624 :         .addReg(DstReg, RegState::Define | getDeadRegState(DstIsDead))
    1543         624 :         .addReg(DstReg).addImm(LabelId);
    1544         624 :       if (isARM) {
    1545         213 :         MIB3.add(predOps(ARMCC::AL));
    1546         213 :         if (Opcode == ARM::MOV_ga_pcrel_ldr)
    1547             :           MIB3.cloneMemRefs(MI);
    1548             :       }
    1549         624 :       TransferImpOps(MI, MIB1, MIB3);
    1550         624 :       MI.eraseFromParent();
    1551             :       return true;
    1552             :     }
    1553             : 
    1554        1677 :     case ARM::MOVi32imm:
    1555             :     case ARM::MOVCCi32imm:
    1556             :     case ARM::t2MOVi32imm:
    1557             :     case ARM::t2MOVCCi32imm:
    1558        1677 :       ExpandMOV32BitImm(MBB, MBBI);
    1559        1677 :       return true;
    1560             : 
    1561           6 :     case ARM::SUBS_PC_LR: {
    1562             :       MachineInstrBuilder MIB =
    1563          12 :           BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(ARM::SUBri), ARM::PC)
    1564           6 :               .addReg(ARM::LR)
    1565           6 :               .add(MI.getOperand(0))
    1566           6 :               .add(MI.getOperand(1))
    1567           6 :               .add(MI.getOperand(2))
    1568           6 :               .addReg(ARM::CPSR, RegState::Undef);
    1569           6 :       TransferImpOps(MI, MIB, MIB);
    1570           6 :       MI.eraseFromParent();
    1571             :       return true;
    1572             :     }
    1573           3 :     case ARM::VLDMQIA: {
    1574             :       unsigned NewOpc = ARM::VLDMDIA;
    1575             :       MachineInstrBuilder MIB =
    1576           6 :         BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(NewOpc));
    1577             :       unsigned OpIdx = 0;
    1578             : 
    1579             :       // Grab the Q register destination.
    1580           3 :       bool DstIsDead = MI.getOperand(OpIdx).isDead();
    1581           3 :       unsigned DstReg = MI.getOperand(OpIdx++).getReg();
    1582             : 
    1583             :       // Copy the source register.
    1584             :       MIB.add(MI.getOperand(OpIdx++));
    1585             : 
    1586             :       // Copy the predicate operands.
    1587           3 :       MIB.add(MI.getOperand(OpIdx++));
    1588           3 :       MIB.add(MI.getOperand(OpIdx++));
    1589             : 
    1590             :       // Add the destination operands (D subregs).
    1591           3 :       unsigned D0 = TRI->getSubReg(DstReg, ARM::dsub_0);
    1592           3 :       unsigned D1 = TRI->getSubReg(DstReg, ARM::dsub_1);
    1593           3 :       MIB.addReg(D0, RegState::Define | getDeadRegState(DstIsDead))
    1594           3 :         .addReg(D1, RegState::Define | getDeadRegState(DstIsDead));
    1595             : 
    1596             :       // Add an implicit def for the super-register.
    1597           3 :       MIB.addReg(DstReg, RegState::ImplicitDefine | getDeadRegState(DstIsDead));
    1598           3 :       TransferImpOps(MI, MIB, MIB);
    1599             :       MIB.cloneMemRefs(MI);
    1600           3 :       MI.eraseFromParent();
    1601             :       return true;
    1602             :     }
    1603             : 
    1604           8 :     case ARM::VSTMQIA: {
    1605             :       unsigned NewOpc = ARM::VSTMDIA;
    1606             :       MachineInstrBuilder MIB =
    1607          16 :         BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(NewOpc));
    1608             :       unsigned OpIdx = 0;
    1609             : 
    1610             :       // Grab the Q register source.
    1611           8 :       bool SrcIsKill = MI.getOperand(OpIdx).isKill();
    1612           8 :       unsigned SrcReg = MI.getOperand(OpIdx++).getReg();
    1613             : 
    1614             :       // Copy the destination register.
    1615           8 :       MachineOperand Dst(MI.getOperand(OpIdx++));
    1616             :       MIB.add(Dst);
    1617             : 
    1618             :       // Copy the predicate operands.
    1619           8 :       MIB.add(MI.getOperand(OpIdx++));
    1620           8 :       MIB.add(MI.getOperand(OpIdx++));
    1621             : 
    1622             :       // Add the source operands (D subregs).
    1623           8 :       unsigned D0 = TRI->getSubReg(SrcReg, ARM::dsub_0);
    1624           8 :       unsigned D1 = TRI->getSubReg(SrcReg, ARM::dsub_1);
    1625           9 :       MIB.addReg(D0, SrcIsKill ? RegState::Kill : 0)
    1626           8 :          .addReg(D1, SrcIsKill ? RegState::Kill : 0);
    1627             : 
    1628           8 :       if (SrcIsKill)      // Add an implicit kill for the Q register.
    1629           7 :         MIB->addRegisterKilled(SrcReg, TRI, true);
    1630             : 
    1631           8 :       TransferImpOps(MI, MIB, MIB);
    1632             :       MIB.cloneMemRefs(MI);
    1633           8 :       MI.eraseFromParent();
    1634             :       return true;
    1635             :     }
    1636             : 
    1637         152 :     case ARM::VLD2q8Pseudo:
    1638             :     case ARM::VLD2q16Pseudo:
    1639             :     case ARM::VLD2q32Pseudo:
    1640             :     case ARM::VLD2q8PseudoWB_fixed:
    1641             :     case ARM::VLD2q16PseudoWB_fixed:
    1642             :     case ARM::VLD2q32PseudoWB_fixed:
    1643             :     case ARM::VLD2q8PseudoWB_register:
    1644             :     case ARM::VLD2q16PseudoWB_register:
    1645             :     case ARM::VLD2q32PseudoWB_register:
    1646             :     case ARM::VLD3d8Pseudo:
    1647             :     case ARM::VLD3d16Pseudo:
    1648             :     case ARM::VLD3d32Pseudo:
    1649             :     case ARM::VLD1d8TPseudo:
    1650             :     case ARM::VLD1d16TPseudo:
    1651             :     case ARM::VLD1d32TPseudo:
    1652             :     case ARM::VLD1d64TPseudo:
    1653             :     case ARM::VLD1d64TPseudoWB_fixed:
    1654             :     case ARM::VLD1d64TPseudoWB_register:
    1655             :     case ARM::VLD3d8Pseudo_UPD:
    1656             :     case ARM::VLD3d16Pseudo_UPD:
    1657             :     case ARM::VLD3d32Pseudo_UPD:
    1658             :     case ARM::VLD3q8Pseudo_UPD:
    1659             :     case ARM::VLD3q16Pseudo_UPD:
    1660             :     case ARM::VLD3q32Pseudo_UPD:
    1661             :     case ARM::VLD3q8oddPseudo:
    1662             :     case ARM::VLD3q16oddPseudo:
    1663             :     case ARM::VLD3q32oddPseudo:
    1664             :     case ARM::VLD3q8oddPseudo_UPD:
    1665             :     case ARM::VLD3q16oddPseudo_UPD:
    1666             :     case ARM::VLD3q32oddPseudo_UPD:
    1667             :     case ARM::VLD4d8Pseudo:
    1668             :     case ARM::VLD4d16Pseudo:
    1669             :     case ARM::VLD4d32Pseudo:
    1670             :     case ARM::VLD1d8QPseudo:
    1671             :     case ARM::VLD1d16QPseudo:
    1672             :     case ARM::VLD1d32QPseudo:
    1673             :     case ARM::VLD1d64QPseudo:
    1674             :     case ARM::VLD1d64QPseudoWB_fixed:
    1675             :     case ARM::VLD1d64QPseudoWB_register:
    1676             :     case ARM::VLD1q8HighQPseudo:
    1677             :     case ARM::VLD1q8LowQPseudo_UPD:
    1678             :     case ARM::VLD1q8HighTPseudo:
    1679             :     case ARM::VLD1q8LowTPseudo_UPD:
    1680             :     case ARM::VLD1q16HighQPseudo:
    1681             :     case ARM::VLD1q16LowQPseudo_UPD:
    1682             :     case ARM::VLD1q16HighTPseudo:
    1683             :     case ARM::VLD1q16LowTPseudo_UPD:
    1684             :     case ARM::VLD1q32HighQPseudo:
    1685             :     case ARM::VLD1q32LowQPseudo_UPD:
    1686             :     case ARM::VLD1q32HighTPseudo:
    1687             :     case ARM::VLD1q32LowTPseudo_UPD:
    1688             :     case ARM::VLD1q64HighQPseudo:
    1689             :     case ARM::VLD1q64LowQPseudo_UPD:
    1690             :     case ARM::VLD1q64HighTPseudo:
    1691             :     case ARM::VLD1q64LowTPseudo_UPD:
    1692             :     case ARM::VLD4d8Pseudo_UPD:
    1693             :     case ARM::VLD4d16Pseudo_UPD:
    1694             :     case ARM::VLD4d32Pseudo_UPD:
    1695             :     case ARM::VLD4q8Pseudo_UPD:
    1696             :     case ARM::VLD4q16Pseudo_UPD:
    1697             :     case ARM::VLD4q32Pseudo_UPD:
    1698             :     case ARM::VLD4q8oddPseudo:
    1699             :     case ARM::VLD4q16oddPseudo:
    1700             :     case ARM::VLD4q32oddPseudo:
    1701             :     case ARM::VLD4q8oddPseudo_UPD:
    1702             :     case ARM::VLD4q16oddPseudo_UPD:
    1703             :     case ARM::VLD4q32oddPseudo_UPD:
    1704             :     case ARM::VLD3DUPd8Pseudo:
    1705             :     case ARM::VLD3DUPd16Pseudo:
    1706             :     case ARM::VLD3DUPd32Pseudo:
    1707             :     case ARM::VLD3DUPd8Pseudo_UPD:
    1708             :     case ARM::VLD3DUPd16Pseudo_UPD:
    1709             :     case ARM::VLD3DUPd32Pseudo_UPD:
    1710             :     case ARM::VLD4DUPd8Pseudo:
    1711             :     case ARM::VLD4DUPd16Pseudo:
    1712             :     case ARM::VLD4DUPd32Pseudo:
    1713             :     case ARM::VLD4DUPd8Pseudo_UPD:
    1714             :     case ARM::VLD4DUPd16Pseudo_UPD:
    1715             :     case ARM::VLD4DUPd32Pseudo_UPD:
    1716             :     case ARM::VLD2DUPq8EvenPseudo:
    1717             :     case ARM::VLD2DUPq8OddPseudo:
    1718             :     case ARM::VLD2DUPq16EvenPseudo:
    1719             :     case ARM::VLD2DUPq16OddPseudo:
    1720             :     case ARM::VLD2DUPq32EvenPseudo:
    1721             :     case ARM::VLD2DUPq32OddPseudo:
    1722             :     case ARM::VLD3DUPq8EvenPseudo:
    1723             :     case ARM::VLD3DUPq8OddPseudo:
    1724             :     case ARM::VLD3DUPq16EvenPseudo:
    1725             :     case ARM::VLD3DUPq16OddPseudo:
    1726             :     case ARM::VLD3DUPq32EvenPseudo:
    1727             :     case ARM::VLD3DUPq32OddPseudo:
    1728             :     case ARM::VLD4DUPq8EvenPseudo:
    1729             :     case ARM::VLD4DUPq8OddPseudo:
    1730             :     case ARM::VLD4DUPq16EvenPseudo:
    1731             :     case ARM::VLD4DUPq16OddPseudo:
    1732             :     case ARM::VLD4DUPq32EvenPseudo:
    1733             :     case ARM::VLD4DUPq32OddPseudo:
    1734         152 :       ExpandVLD(MBBI);
    1735         152 :       return true;
    1736             : 
    1737         107 :     case ARM::VST2q8Pseudo:
    1738             :     case ARM::VST2q16Pseudo:
    1739             :     case ARM::VST2q32Pseudo:
    1740             :     case ARM::VST2q8PseudoWB_fixed:
    1741             :     case ARM::VST2q16PseudoWB_fixed:
    1742             :     case ARM::VST2q32PseudoWB_fixed:
    1743             :     case ARM::VST2q8PseudoWB_register:
    1744             :     case ARM::VST2q16PseudoWB_register:
    1745             :     case ARM::VST2q32PseudoWB_register:
    1746             :     case ARM::VST3d8Pseudo:
    1747             :     case ARM::VST3d16Pseudo:
    1748             :     case ARM::VST3d32Pseudo:
    1749             :     case ARM::VST1d8TPseudo:
    1750             :     case ARM::VST1d16TPseudo:
    1751             :     case ARM::VST1d32TPseudo:
    1752             :     case ARM::VST1d64TPseudo:
    1753             :     case ARM::VST3d8Pseudo_UPD:
    1754             :     case ARM::VST3d16Pseudo_UPD:
    1755             :     case ARM::VST3d32Pseudo_UPD:
    1756             :     case ARM::VST1d64TPseudoWB_fixed:
    1757             :     case ARM::VST1d64TPseudoWB_register:
    1758             :     case ARM::VST3q8Pseudo_UPD:
    1759             :     case ARM::VST3q16Pseudo_UPD:
    1760             :     case ARM::VST3q32Pseudo_UPD:
    1761             :     case ARM::VST3q8oddPseudo:
    1762             :     case ARM::VST3q16oddPseudo:
    1763             :     case ARM::VST3q32oddPseudo:
    1764             :     case ARM::VST3q8oddPseudo_UPD:
    1765             :     case ARM::VST3q16oddPseudo_UPD:
    1766             :     case ARM::VST3q32oddPseudo_UPD:
    1767             :     case ARM::VST4d8Pseudo:
    1768             :     case ARM::VST4d16Pseudo:
    1769             :     case ARM::VST4d32Pseudo:
    1770             :     case ARM::VST1d8QPseudo:
    1771             :     case ARM::VST1d16QPseudo:
    1772             :     case ARM::VST1d32QPseudo:
    1773             :     case ARM::VST1d64QPseudo:
    1774             :     case ARM::VST4d8Pseudo_UPD:
    1775             :     case ARM::VST4d16Pseudo_UPD:
    1776             :     case ARM::VST4d32Pseudo_UPD:
    1777             :     case ARM::VST1d64QPseudoWB_fixed:
    1778             :     case ARM::VST1d64QPseudoWB_register:
    1779             :     case ARM::VST1q8HighQPseudo:
    1780             :     case ARM::VST1q8LowQPseudo_UPD:
    1781             :     case ARM::VST1q8HighTPseudo:
    1782             :     case ARM::VST1q8LowTPseudo_UPD:
    1783             :     case ARM::VST1q16HighQPseudo:
    1784             :     case ARM::VST1q16LowQPseudo_UPD:
    1785             :     case ARM::VST1q16HighTPseudo:
    1786             :     case ARM::VST1q16LowTPseudo_UPD:
    1787             :     case ARM::VST1q32HighQPseudo:
    1788             :     case ARM::VST1q32LowQPseudo_UPD:
    1789             :     case ARM::VST1q32HighTPseudo:
    1790             :     case ARM::VST1q32LowTPseudo_UPD:
    1791             :     case ARM::VST1q64HighQPseudo:
    1792             :     case ARM::VST1q64LowQPseudo_UPD:
    1793             :     case ARM::VST1q64HighTPseudo:
    1794             :     case ARM::VST1q64LowTPseudo_UPD:
    1795             :     case ARM::VST4q8Pseudo_UPD:
    1796             :     case ARM::VST4q16Pseudo_UPD:
    1797             :     case ARM::VST4q32Pseudo_UPD:
    1798             :     case ARM::VST4q8oddPseudo:
    1799             :     case ARM::VST4q16oddPseudo:
    1800             :     case ARM::VST4q32oddPseudo:
    1801             :     case ARM::VST4q8oddPseudo_UPD:
    1802             :     case ARM::VST4q16oddPseudo_UPD:
    1803             :     case ARM::VST4q32oddPseudo_UPD:
    1804         107 :       ExpandVST(MBBI);
    1805         107 :       return true;
    1806             : 
    1807         162 :     case ARM::VLD1LNq8Pseudo:
    1808             :     case ARM::VLD1LNq16Pseudo:
    1809             :     case ARM::VLD1LNq32Pseudo:
    1810             :     case ARM::VLD1LNq8Pseudo_UPD:
    1811             :     case ARM::VLD1LNq16Pseudo_UPD:
    1812             :     case ARM::VLD1LNq32Pseudo_UPD:
    1813             :     case ARM::VLD2LNd8Pseudo:
    1814             :     case ARM::VLD2LNd16Pseudo:
    1815             :     case ARM::VLD2LNd32Pseudo:
    1816             :     case ARM::VLD2LNq16Pseudo:
    1817             :     case ARM::VLD2LNq32Pseudo:
    1818             :     case ARM::VLD2LNd8Pseudo_UPD:
    1819             :     case ARM::VLD2LNd16Pseudo_UPD:
    1820             :     case ARM::VLD2LNd32Pseudo_UPD:
    1821             :     case ARM::VLD2LNq16Pseudo_UPD:
    1822             :     case ARM::VLD2LNq32Pseudo_UPD:
    1823             :     case ARM::VLD3LNd8Pseudo:
    1824             :     case ARM::VLD3LNd16Pseudo:
    1825             :     case ARM::VLD3LNd32Pseudo:
    1826             :     case ARM::VLD3LNq16Pseudo:
    1827             :     case ARM::VLD3LNq32Pseudo:
    1828             :     case ARM::VLD3LNd8Pseudo_UPD:
    1829             :     case ARM::VLD3LNd16Pseudo_UPD:
    1830             :     case ARM::VLD3LNd32Pseudo_UPD:
    1831             :     case ARM::VLD3LNq16Pseudo_UPD:
    1832             :     case ARM::VLD3LNq32Pseudo_UPD:
    1833             :     case ARM::VLD4LNd8Pseudo:
    1834             :     case ARM::VLD4LNd16Pseudo:
    1835             :     case ARM::VLD4LNd32Pseudo:
    1836             :     case ARM::VLD4LNq16Pseudo:
    1837             :     case ARM::VLD4LNq32Pseudo:
    1838             :     case ARM::VLD4LNd8Pseudo_UPD:
    1839             :     case ARM::VLD4LNd16Pseudo_UPD:
    1840             :     case ARM::VLD4LNd32Pseudo_UPD:
    1841             :     case ARM::VLD4LNq16Pseudo_UPD:
    1842             :     case ARM::VLD4LNq32Pseudo_UPD:
    1843             :     case ARM::VST1LNq8Pseudo:
    1844             :     case ARM::VST1LNq16Pseudo:
    1845             :     case ARM::VST1LNq32Pseudo:
    1846             :     case ARM::VST1LNq8Pseudo_UPD:
    1847             :     case ARM::VST1LNq16Pseudo_UPD:
    1848             :     case ARM::VST1LNq32Pseudo_UPD:
    1849             :     case ARM::VST2LNd8Pseudo:
    1850             :     case ARM::VST2LNd16Pseudo:
    1851             :     case ARM::VST2LNd32Pseudo:
    1852             :     case ARM::VST2LNq16Pseudo:
    1853             :     case ARM::VST2LNq32Pseudo:
    1854             :     case ARM::VST2LNd8Pseudo_UPD:
    1855             :     case ARM::VST2LNd16Pseudo_UPD:
    1856             :     case ARM::VST2LNd32Pseudo_UPD:
    1857             :     case ARM::VST2LNq16Pseudo_UPD:
    1858             :     case ARM::VST2LNq32Pseudo_UPD:
    1859             :     case ARM::VST3LNd8Pseudo:
    1860             :     case ARM::VST3LNd16Pseudo:
    1861             :     case ARM::VST3LNd32Pseudo:
    1862             :     case ARM::VST3LNq16Pseudo:
    1863             :     case ARM::VST3LNq32Pseudo:
    1864             :     case ARM::VST3LNd8Pseudo_UPD:
    1865             :     case ARM::VST3LNd16Pseudo_UPD:
    1866             :     case ARM::VST3LNd32Pseudo_UPD:
    1867             :     case ARM::VST3LNq16Pseudo_UPD:
    1868             :     case ARM::VST3LNq32Pseudo_UPD:
    1869             :     case ARM::VST4LNd8Pseudo:
    1870             :     case ARM::VST4LNd16Pseudo:
    1871             :     case ARM::VST4LNd32Pseudo:
    1872             :     case ARM::VST4LNq16Pseudo:
    1873             :     case ARM::VST4LNq32Pseudo:
    1874             :     case ARM::VST4LNd8Pseudo_UPD:
    1875             :     case ARM::VST4LNd16Pseudo_UPD:
    1876             :     case ARM::VST4LNd32Pseudo_UPD:
    1877             :     case ARM::VST4LNq16Pseudo_UPD:
    1878             :     case ARM::VST4LNq32Pseudo_UPD:
    1879         162 :       ExpandLaneOp(MBBI);
    1880         162 :       return true;
    1881             : 
    1882           1 :     case ARM::VTBL3Pseudo: ExpandVTBL(MBBI, ARM::VTBL3, false); return true;
    1883           1 :     case ARM::VTBL4Pseudo: ExpandVTBL(MBBI, ARM::VTBL4, false); return true;
    1884           1 :     case ARM::VTBX3Pseudo: ExpandVTBL(MBBI, ARM::VTBX3, true); return true;
    1885           2 :     case ARM::VTBX4Pseudo: ExpandVTBL(MBBI, ARM::VTBX4, true); return true;
    1886             : 
    1887           2 :     case ARM::CMP_SWAP_8:
    1888           2 :       if (STI->isThumb())
    1889           1 :         return ExpandCMP_SWAP(MBB, MBBI, ARM::t2LDREXB, ARM::t2STREXB,
    1890           1 :                               ARM::tUXTB, NextMBBI);
    1891             :       else
    1892           1 :         return ExpandCMP_SWAP(MBB, MBBI, ARM::LDREXB, ARM::STREXB,
    1893           1 :                               ARM::UXTB, NextMBBI);
    1894           2 :     case ARM::CMP_SWAP_16:
    1895           2 :       if (STI->isThumb())
    1896           1 :         return ExpandCMP_SWAP(MBB, MBBI, ARM::t2LDREXH, ARM::t2STREXH,
    1897           1 :                               ARM::tUXTH, NextMBBI);
    1898             :       else
    1899           1 :         return ExpandCMP_SWAP(MBB, MBBI, ARM::LDREXH, ARM::STREXH,
    1900           1 :                               ARM::UXTH, NextMBBI);
    1901           2 :     case ARM::CMP_SWAP_32:
    1902           2 :       if (STI->isThumb())
    1903           1 :         return ExpandCMP_SWAP(MBB, MBBI, ARM::t2LDREX, ARM::t2STREX, 0,
    1904           1 :                               NextMBBI);
    1905             :       else
    1906           1 :         return ExpandCMP_SWAP(MBB, MBBI, ARM::LDREX, ARM::STREX, 0, NextMBBI);
    1907             : 
    1908           7 :     case ARM::CMP_SWAP_64:
    1909           7 :       return ExpandCMP_SWAP_64(MBB, MBBI, NextMBBI);
    1910             :   }
    1911             : }
    1912             : 
    1913       20028 : bool ARMExpandPseudo::ExpandMBB(MachineBasicBlock &MBB) {
    1914             :   bool Modified = false;
    1915             : 
    1916             :   MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();
    1917      181166 :   while (MBBI != E) {
    1918      161138 :     MachineBasicBlock::iterator NMBBI = std::next(MBBI);
    1919      161138 :     Modified |= ExpandMI(MBB, MBBI, NMBBI);
    1920      161138 :     MBBI = NMBBI;
    1921             :   }
    1922             : 
    1923       20028 :   return Modified;
    1924             : }
    1925             : 
    1926       14427 : bool ARMExpandPseudo::runOnMachineFunction(MachineFunction &MF) {
    1927       14427 :   STI = &static_cast<const ARMSubtarget &>(MF.getSubtarget());
    1928       14427 :   TII = STI->getInstrInfo();
    1929       14427 :   TRI = STI->getRegisterInfo();
    1930       14427 :   AFI = MF.getInfo<ARMFunctionInfo>();
    1931             : 
    1932             :   bool Modified = false;
    1933       34455 :   for (MachineBasicBlock &MBB : MF)
    1934       20028 :     Modified |= ExpandMBB(MBB);
    1935       14427 :   if (VerifyARMPseudo)
    1936           4 :     MF.verify(this, "After expanding ARM pseudo instructions.");
    1937       14427 :   return Modified;
    1938             : }
    1939             : 
    1940             : /// createARMExpandPseudoPass - returns an instance of the pseudo instruction
    1941             : /// expansion pass.
    1942        2816 : FunctionPass *llvm::createARMExpandPseudoPass() {
    1943        2816 :   return new ARMExpandPseudo();
    1944             : }

Generated by: LCOV version 1.13