LCOV - code coverage report
Current view: top level - lib/Target/ARM - ARMExpandPseudoInsts.cpp (source / functions) Hit Total Coverage
Test: llvm-toolchain.info Lines: 699 728 96.0 %
Date: 2018-06-17 00:07:59 Functions: 22 23 95.7 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : //===-- ARMExpandPseudoInsts.cpp - Expand pseudo instructions -------------===//
       2             : //
       3             : //                     The LLVM Compiler Infrastructure
       4             : //
       5             : // This file is distributed under the University of Illinois Open Source
       6             : // License. See LICENSE.TXT for details.
       7             : //
       8             : //===----------------------------------------------------------------------===//
       9             : //
      10             : // This file contains a pass that expands pseudo instructions into target
      11             : // instructions to allow proper scheduling, if-conversion, and other late
      12             : // optimizations. This pass should be run after register allocation but before
      13             : // the post-regalloc scheduling pass.
      14             : //
      15             : //===----------------------------------------------------------------------===//
      16             : 
      17             : #include "ARM.h"
      18             : #include "ARMBaseInstrInfo.h"
      19             : #include "ARMBaseRegisterInfo.h"
      20             : #include "ARMConstantPoolValue.h"
      21             : #include "ARMMachineFunctionInfo.h"
      22             : #include "ARMSubtarget.h"
      23             : #include "MCTargetDesc/ARMAddressingModes.h"
      24             : #include "llvm/CodeGen/LivePhysRegs.h"
      25             : #include "llvm/CodeGen/MachineFrameInfo.h"
      26             : #include "llvm/CodeGen/MachineFunctionPass.h"
      27             : 
      28             : using namespace llvm;
      29             : 
      30             : #define DEBUG_TYPE "arm-pseudo"
      31             : 
      32             : static cl::opt<bool>
      33      101169 : VerifyARMPseudo("verify-arm-pseudo-expand", cl::Hidden,
      34      101169 :                 cl::desc("Verify machine code after expanding ARM pseudos"));
      35             : 
      36             : #define ARM_EXPAND_PSEUDO_NAME "ARM pseudo instruction expansion pass"
      37             : 
      38             : namespace {
      39        2702 :   class ARMExpandPseudo : public MachineFunctionPass {
      40             :   public:
      41             :     static char ID;
      42        2729 :     ARMExpandPseudo() : MachineFunctionPass(ID) {}
      43             : 
      44             :     const ARMBaseInstrInfo *TII;
      45             :     const TargetRegisterInfo *TRI;
      46             :     const ARMSubtarget *STI;
      47             :     ARMFunctionInfo *AFI;
      48             : 
      49             :     bool runOnMachineFunction(MachineFunction &Fn) override;
      50             : 
      51        2716 :     MachineFunctionProperties getRequiredProperties() const override {
      52        5432 :       return MachineFunctionProperties().set(
      53        2716 :           MachineFunctionProperties::Property::NoVRegs);
      54             :     }
      55             : 
      56        2715 :     StringRef getPassName() const override {
      57        2715 :       return ARM_EXPAND_PSEUDO_NAME;
      58             :     }
      59             : 
      60             :   private:
      61             :     void TransferImpOps(MachineInstr &OldMI,
      62             :                         MachineInstrBuilder &UseMI, MachineInstrBuilder &DefMI);
      63             :     bool ExpandMI(MachineBasicBlock &MBB,
      64             :                   MachineBasicBlock::iterator MBBI,
      65             :                   MachineBasicBlock::iterator &NextMBBI);
      66             :     bool ExpandMBB(MachineBasicBlock &MBB);
      67             :     void ExpandVLD(MachineBasicBlock::iterator &MBBI);
      68             :     void ExpandVST(MachineBasicBlock::iterator &MBBI);
      69             :     void ExpandLaneOp(MachineBasicBlock::iterator &MBBI);
      70             :     void ExpandVTBL(MachineBasicBlock::iterator &MBBI,
      71             :                     unsigned Opc, bool IsExt);
      72             :     void ExpandMOV32BitImm(MachineBasicBlock &MBB,
      73             :                            MachineBasicBlock::iterator &MBBI);
      74             :     bool ExpandCMP_SWAP(MachineBasicBlock &MBB,
      75             :                         MachineBasicBlock::iterator MBBI, unsigned LdrexOp,
      76             :                         unsigned StrexOp, unsigned UxtOp,
      77             :                         MachineBasicBlock::iterator &NextMBBI);
      78             : 
      79             :     bool ExpandCMP_SWAP_64(MachineBasicBlock &MBB,
      80             :                            MachineBasicBlock::iterator MBBI,
      81             :                            MachineBasicBlock::iterator &NextMBBI);
      82             :   };
      83             :   char ARMExpandPseudo::ID = 0;
      84             : }
      85             : 
      86      357084 : INITIALIZE_PASS(ARMExpandPseudo, DEBUG_TYPE, ARM_EXPAND_PSEUDO_NAME, false,
      87             :                 false)
      88             : 
      89             : /// TransferImpOps - Transfer implicit operands on the pseudo instruction to
      90             : /// the instructions created from the expansion.
      91        2753 : void ARMExpandPseudo::TransferImpOps(MachineInstr &OldMI,
      92             :                                      MachineInstrBuilder &UseMI,
      93             :                                      MachineInstrBuilder &DefMI) {
      94        2753 :   const MCInstrDesc &Desc = OldMI.getDesc();
      95        5712 :   for (unsigned i = Desc.getNumOperands(), e = OldMI.getNumOperands();
      96        2959 :        i != e; ++i) {
      97         206 :     const MachineOperand &MO = OldMI.getOperand(i);
      98             :     assert(MO.isReg() && MO.getReg());
      99         206 :     if (MO.isUse())
     100             :       UseMI.add(MO);
     101             :     else
     102             :       DefMI.add(MO);
     103             :   }
     104        2753 : }
     105             : 
     106             : namespace {
     107             :   // Constants for register spacing in NEON load/store instructions.
     108             :   // For quad-register load-lane and store-lane pseudo instructors, the
     109             :   // spacing is initially assumed to be EvenDblSpc, and that is changed to
     110             :   // OddDblSpc depending on the lane number operand.
     111             :   enum NEONRegSpacing {
     112             :     SingleSpc,
     113             :     SingleLowSpc ,  // Single spacing, low registers, three and four vectors.
     114             :     SingleHighQSpc, // Single spacing, high registers, four vectors.
     115             :     SingleHighTSpc, // Single spacing, high registers, three vectors.
     116             :     EvenDblSpc,
     117             :     OddDblSpc
     118             :   };
     119             : 
     120             :   // Entries for NEON load/store information table.  The table is sorted by
     121             :   // PseudoOpc for fast binary-search lookups.
     122             :   struct NEONLdStTableEntry {
     123             :     uint16_t PseudoOpc;
     124             :     uint16_t RealOpc;
     125             :     bool IsLoad;
     126             :     bool isUpdating;
     127             :     bool hasWritebackOperand;
     128             :     uint8_t RegSpacing; // One of type NEONRegSpacing
     129             :     uint8_t NumRegs; // D registers loaded or stored
     130             :     uint8_t RegElts; // elements per D register; used for lane ops
     131             :     // FIXME: Temporary flag to denote whether the real instruction takes
     132             :     // a single register (like the encoding) or all of the registers in
     133             :     // the list (like the asm syntax and the isel DAG). When all definitions
     134             :     // are converted to take only the single encoded register, this will
     135             :     // go away.
     136             :     bool copyAllListRegs;
     137             : 
     138             :     // Comparison methods for binary search of the table.
     139             :     bool operator<(const NEONLdStTableEntry &TE) const {
     140             :       return PseudoOpc < TE.PseudoOpc;
     141             :     }
     142             :     friend bool operator<(const NEONLdStTableEntry &TE, unsigned PseudoOpc) {
     143             :       return TE.PseudoOpc < PseudoOpc;
     144             :     }
     145             :     friend bool LLVM_ATTRIBUTE_UNUSED operator<(unsigned PseudoOpc,
     146             :                                                 const NEONLdStTableEntry &TE) {
     147             :       return PseudoOpc < TE.PseudoOpc;
     148             :     }
     149             :   };
     150             : }
     151             : 
     152             : static const NEONLdStTableEntry NEONLdStTable[] = {
     153             : { ARM::VLD1LNq16Pseudo,     ARM::VLD1LNd16,     true, false, false, EvenDblSpc, 1, 4 ,true},
     154             : { ARM::VLD1LNq16Pseudo_UPD, ARM::VLD1LNd16_UPD, true, true, true,  EvenDblSpc, 1, 4 ,true},
     155             : { ARM::VLD1LNq32Pseudo,     ARM::VLD1LNd32,     true, false, false, EvenDblSpc, 1, 2 ,true},
     156             : { ARM::VLD1LNq32Pseudo_UPD, ARM::VLD1LNd32_UPD, true, true, true,  EvenDblSpc, 1, 2 ,true},
     157             : { ARM::VLD1LNq8Pseudo,      ARM::VLD1LNd8,      true, false, false, EvenDblSpc, 1, 8 ,true},
     158             : { ARM::VLD1LNq8Pseudo_UPD,  ARM::VLD1LNd8_UPD, true, true, true,  EvenDblSpc, 1, 8 ,true},
     159             : 
     160             : { ARM::VLD1d16QPseudo,      ARM::VLD1d16Q,     true,  false, false, SingleSpc,  4, 4 ,false},
     161             : { ARM::VLD1d16TPseudo,      ARM::VLD1d16T,     true,  false, false, SingleSpc,  3, 4 ,false},
     162             : { ARM::VLD1d32QPseudo,      ARM::VLD1d32Q,     true,  false, false, SingleSpc,  4, 2 ,false},
     163             : { ARM::VLD1d32TPseudo,      ARM::VLD1d32T,     true,  false, false, SingleSpc,  3, 2 ,false},
     164             : { ARM::VLD1d64QPseudo,      ARM::VLD1d64Q,     true,  false, false, SingleSpc,  4, 1 ,false},
     165             : { ARM::VLD1d64QPseudoWB_fixed,  ARM::VLD1d64Qwb_fixed,   true,  true, false, SingleSpc,  4, 1 ,false},
     166             : { ARM::VLD1d64QPseudoWB_register,  ARM::VLD1d64Qwb_register,   true,  true, true, SingleSpc,  4, 1 ,false},
     167             : { ARM::VLD1d64TPseudo,      ARM::VLD1d64T,     true,  false, false, SingleSpc,  3, 1 ,false},
     168             : { ARM::VLD1d64TPseudoWB_fixed,  ARM::VLD1d64Twb_fixed,   true,  true, false, SingleSpc,  3, 1 ,false},
     169             : { ARM::VLD1d64TPseudoWB_register,  ARM::VLD1d64Twb_register, true, true, true,  SingleSpc,  3, 1 ,false},
     170             : { ARM::VLD1d8QPseudo,       ARM::VLD1d8Q,      true,  false, false, SingleSpc,  4, 8 ,false},
     171             : { ARM::VLD1d8TPseudo,       ARM::VLD1d8T,      true,  false, false, SingleSpc,  3, 8 ,false},
     172             : { ARM::VLD1q16HighQPseudo,  ARM::VLD1d16Q,     true,  false, false, SingleHighQSpc,  4, 4 ,false},
     173             : { ARM::VLD1q16HighTPseudo,  ARM::VLD1d16T,     true,  false, false, SingleHighTSpc,  3, 4 ,false},
     174             : { ARM::VLD1q16LowQPseudo_UPD,  ARM::VLD1d16Qwb_fixed,   true,  true, true, SingleLowSpc,  4, 4 ,false},
     175             : { ARM::VLD1q16LowTPseudo_UPD,  ARM::VLD1d16Twb_fixed,   true,  true, true, SingleLowSpc,  3, 4 ,false},
     176             : { ARM::VLD1q32HighQPseudo,  ARM::VLD1d32Q,     true,  false, false, SingleHighQSpc,  4, 2 ,false},
     177             : { ARM::VLD1q32HighTPseudo,  ARM::VLD1d32T,     true,  false, false, SingleHighTSpc,  3, 2 ,false},
     178             : { ARM::VLD1q32LowQPseudo_UPD,  ARM::VLD1d32Qwb_fixed,   true,  true, true, SingleLowSpc,  4, 2 ,false},
     179             : { ARM::VLD1q32LowTPseudo_UPD,  ARM::VLD1d32Twb_fixed,   true,  true, true, SingleLowSpc,  3, 2 ,false},
     180             : { ARM::VLD1q64HighQPseudo,  ARM::VLD1d64Q,     true,  false, false, SingleHighQSpc,  4, 1 ,false},
     181             : { ARM::VLD1q64HighTPseudo,  ARM::VLD1d64T,     true,  false, false, SingleHighTSpc,  3, 1 ,false},
     182             : { ARM::VLD1q64LowQPseudo_UPD,  ARM::VLD1d64Qwb_fixed,   true,  true, true, SingleLowSpc,  4, 1 ,false},
     183             : { ARM::VLD1q64LowTPseudo_UPD,  ARM::VLD1d64Twb_fixed,   true,  true, true, SingleLowSpc,  3, 1 ,false},
     184             : { ARM::VLD1q8HighQPseudo,   ARM::VLD1d8Q,     true,  false, false, SingleHighQSpc,  4, 8 ,false},
     185             : { ARM::VLD1q8HighTPseudo,   ARM::VLD1d8T,     true,  false, false, SingleHighTSpc,  3, 8 ,false},
     186             : { ARM::VLD1q8LowQPseudo_UPD,  ARM::VLD1d8Qwb_fixed,   true,  true, true, SingleLowSpc,  4, 8 ,false},
     187             : { ARM::VLD1q8LowTPseudo_UPD,  ARM::VLD1d8Twb_fixed,   true,  true, true, SingleLowSpc,  3, 8 ,false},
     188             : 
     189             : { ARM::VLD2LNd16Pseudo,     ARM::VLD2LNd16,     true, false, false, SingleSpc,  2, 4 ,true},
     190             : { ARM::VLD2LNd16Pseudo_UPD, ARM::VLD2LNd16_UPD, true, true, true,  SingleSpc,  2, 4 ,true},
     191             : { ARM::VLD2LNd32Pseudo,     ARM::VLD2LNd32,     true, false, false, SingleSpc,  2, 2 ,true},
     192             : { ARM::VLD2LNd32Pseudo_UPD, ARM::VLD2LNd32_UPD, true, true, true,  SingleSpc,  2, 2 ,true},
     193             : { ARM::VLD2LNd8Pseudo,      ARM::VLD2LNd8,      true, false, false, SingleSpc,  2, 8 ,true},
     194             : { ARM::VLD2LNd8Pseudo_UPD,  ARM::VLD2LNd8_UPD, true, true, true,  SingleSpc,  2, 8 ,true},
     195             : { ARM::VLD2LNq16Pseudo,     ARM::VLD2LNq16,     true, false, false, EvenDblSpc, 2, 4 ,true},
     196             : { ARM::VLD2LNq16Pseudo_UPD, ARM::VLD2LNq16_UPD, true, true, true,  EvenDblSpc, 2, 4 ,true},
     197             : { ARM::VLD2LNq32Pseudo,     ARM::VLD2LNq32,     true, false, false, EvenDblSpc, 2, 2 ,true},
     198             : { ARM::VLD2LNq32Pseudo_UPD, ARM::VLD2LNq32_UPD, true, true, true,  EvenDblSpc, 2, 2 ,true},
     199             : 
     200             : { ARM::VLD2q16Pseudo,       ARM::VLD2q16,      true,  false, false, SingleSpc,  4, 4 ,false},
     201             : { ARM::VLD2q16PseudoWB_fixed,   ARM::VLD2q16wb_fixed, true, true, false,  SingleSpc,  4, 4 ,false},
     202             : { ARM::VLD2q16PseudoWB_register,   ARM::VLD2q16wb_register, true, true, true,  SingleSpc,  4, 4 ,false},
     203             : { ARM::VLD2q32Pseudo,       ARM::VLD2q32,      true,  false, false, SingleSpc,  4, 2 ,false},
     204             : { ARM::VLD2q32PseudoWB_fixed,   ARM::VLD2q32wb_fixed, true, true, false,  SingleSpc,  4, 2 ,false},
     205             : { ARM::VLD2q32PseudoWB_register,   ARM::VLD2q32wb_register, true, true, true,  SingleSpc,  4, 2 ,false},
     206             : { ARM::VLD2q8Pseudo,        ARM::VLD2q8,       true,  false, false, SingleSpc,  4, 8 ,false},
     207             : { ARM::VLD2q8PseudoWB_fixed,    ARM::VLD2q8wb_fixed, true, true, false,  SingleSpc,  4, 8 ,false},
     208             : { ARM::VLD2q8PseudoWB_register,    ARM::VLD2q8wb_register, true, true, true,  SingleSpc,  4, 8 ,false},
     209             : 
     210             : { ARM::VLD3DUPd16Pseudo,     ARM::VLD3DUPd16,     true, false, false, SingleSpc, 3, 4,true},
     211             : { ARM::VLD3DUPd16Pseudo_UPD, ARM::VLD3DUPd16_UPD, true, true, true,  SingleSpc, 3, 4,true},
     212             : { ARM::VLD3DUPd32Pseudo,     ARM::VLD3DUPd32,     true, false, false, SingleSpc, 3, 2,true},
     213             : { ARM::VLD3DUPd32Pseudo_UPD, ARM::VLD3DUPd32_UPD, true, true, true,  SingleSpc, 3, 2,true},
     214             : { ARM::VLD3DUPd8Pseudo,      ARM::VLD3DUPd8,      true, false, false, SingleSpc, 3, 8,true},
     215             : { ARM::VLD3DUPd8Pseudo_UPD,  ARM::VLD3DUPd8_UPD, true, true, true,  SingleSpc, 3, 8,true},
     216             : 
     217             : { ARM::VLD3LNd16Pseudo,     ARM::VLD3LNd16,     true, false, false, SingleSpc,  3, 4 ,true},
     218             : { ARM::VLD3LNd16Pseudo_UPD, ARM::VLD3LNd16_UPD, true, true, true,  SingleSpc,  3, 4 ,true},
     219             : { ARM::VLD3LNd32Pseudo,     ARM::VLD3LNd32,     true, false, false, SingleSpc,  3, 2 ,true},
     220             : { ARM::VLD3LNd32Pseudo_UPD, ARM::VLD3LNd32_UPD, true, true, true,  SingleSpc,  3, 2 ,true},
     221             : { ARM::VLD3LNd8Pseudo,      ARM::VLD3LNd8,      true, false, false, SingleSpc,  3, 8 ,true},
     222             : { ARM::VLD3LNd8Pseudo_UPD,  ARM::VLD3LNd8_UPD, true, true, true,  SingleSpc,  3, 8 ,true},
     223             : { ARM::VLD3LNq16Pseudo,     ARM::VLD3LNq16,     true, false, false, EvenDblSpc, 3, 4 ,true},
     224             : { ARM::VLD3LNq16Pseudo_UPD, ARM::VLD3LNq16_UPD, true, true, true,  EvenDblSpc, 3, 4 ,true},
     225             : { ARM::VLD3LNq32Pseudo,     ARM::VLD3LNq32,     true, false, false, EvenDblSpc, 3, 2 ,true},
     226             : { ARM::VLD3LNq32Pseudo_UPD, ARM::VLD3LNq32_UPD, true, true, true,  EvenDblSpc, 3, 2 ,true},
     227             : 
     228             : { ARM::VLD3d16Pseudo,       ARM::VLD3d16,      true,  false, false, SingleSpc,  3, 4 ,true},
     229             : { ARM::VLD3d16Pseudo_UPD,   ARM::VLD3d16_UPD, true, true, true,  SingleSpc,  3, 4 ,true},
     230             : { ARM::VLD3d32Pseudo,       ARM::VLD3d32,      true,  false, false, SingleSpc,  3, 2 ,true},
     231             : { ARM::VLD3d32Pseudo_UPD,   ARM::VLD3d32_UPD, true, true, true,  SingleSpc,  3, 2 ,true},
     232             : { ARM::VLD3d8Pseudo,        ARM::VLD3d8,       true,  false, false, SingleSpc,  3, 8 ,true},
     233             : { ARM::VLD3d8Pseudo_UPD,    ARM::VLD3d8_UPD, true, true, true,  SingleSpc,  3, 8 ,true},
     234             : 
     235             : { ARM::VLD3q16Pseudo_UPD,    ARM::VLD3q16_UPD, true, true, true,  EvenDblSpc, 3, 4 ,true},
     236             : { ARM::VLD3q16oddPseudo,     ARM::VLD3q16,     true,  false, false, OddDblSpc,  3, 4 ,true},
     237             : { ARM::VLD3q16oddPseudo_UPD, ARM::VLD3q16_UPD, true, true, true,  OddDblSpc,  3, 4 ,true},
     238             : { ARM::VLD3q32Pseudo_UPD,    ARM::VLD3q32_UPD, true, true, true,  EvenDblSpc, 3, 2 ,true},
     239             : { ARM::VLD3q32oddPseudo,     ARM::VLD3q32,     true,  false, false, OddDblSpc,  3, 2 ,true},
     240             : { ARM::VLD3q32oddPseudo_UPD, ARM::VLD3q32_UPD, true, true, true,  OddDblSpc,  3, 2 ,true},
     241             : { ARM::VLD3q8Pseudo_UPD,     ARM::VLD3q8_UPD, true, true, true,  EvenDblSpc, 3, 8 ,true},
     242             : { ARM::VLD3q8oddPseudo,      ARM::VLD3q8,      true,  false, false, OddDblSpc,  3, 8 ,true},
     243             : { ARM::VLD3q8oddPseudo_UPD,  ARM::VLD3q8_UPD, true, true, true,  OddDblSpc,  3, 8 ,true},
     244             : 
     245             : { ARM::VLD4DUPd16Pseudo,     ARM::VLD4DUPd16,     true, false, false, SingleSpc, 4, 4,true},
     246             : { ARM::VLD4DUPd16Pseudo_UPD, ARM::VLD4DUPd16_UPD, true, true, true,  SingleSpc, 4, 4,true},
     247             : { ARM::VLD4DUPd32Pseudo,     ARM::VLD4DUPd32,     true, false, false, SingleSpc, 4, 2,true},
     248             : { ARM::VLD4DUPd32Pseudo_UPD, ARM::VLD4DUPd32_UPD, true, true, true,  SingleSpc, 4, 2,true},
     249             : { ARM::VLD4DUPd8Pseudo,      ARM::VLD4DUPd8,      true, false, false, SingleSpc, 4, 8,true},
     250             : { ARM::VLD4DUPd8Pseudo_UPD,  ARM::VLD4DUPd8_UPD, true, true, true,  SingleSpc, 4, 8,true},
     251             : 
     252             : { ARM::VLD4LNd16Pseudo,     ARM::VLD4LNd16,     true, false, false, SingleSpc,  4, 4 ,true},
     253             : { ARM::VLD4LNd16Pseudo_UPD, ARM::VLD4LNd16_UPD, true, true, true,  SingleSpc,  4, 4 ,true},
     254             : { ARM::VLD4LNd32Pseudo,     ARM::VLD4LNd32,     true, false, false, SingleSpc,  4, 2 ,true},
     255             : { ARM::VLD4LNd32Pseudo_UPD, ARM::VLD4LNd32_UPD, true, true, true,  SingleSpc,  4, 2 ,true},
     256             : { ARM::VLD4LNd8Pseudo,      ARM::VLD4LNd8,      true, false, false, SingleSpc,  4, 8 ,true},
     257             : { ARM::VLD4LNd8Pseudo_UPD,  ARM::VLD4LNd8_UPD, true, true, true,  SingleSpc,  4, 8 ,true},
     258             : { ARM::VLD4LNq16Pseudo,     ARM::VLD4LNq16,     true, false, false, EvenDblSpc, 4, 4 ,true},
     259             : { ARM::VLD4LNq16Pseudo_UPD, ARM::VLD4LNq16_UPD, true, true, true,  EvenDblSpc, 4, 4 ,true},
     260             : { ARM::VLD4LNq32Pseudo,     ARM::VLD4LNq32,     true, false, false, EvenDblSpc, 4, 2 ,true},
     261             : { ARM::VLD4LNq32Pseudo_UPD, ARM::VLD4LNq32_UPD, true, true, true,  EvenDblSpc, 4, 2 ,true},
     262             : 
     263             : { ARM::VLD4d16Pseudo,       ARM::VLD4d16,      true,  false, false, SingleSpc,  4, 4 ,true},
     264             : { ARM::VLD4d16Pseudo_UPD,   ARM::VLD4d16_UPD, true, true, true,  SingleSpc,  4, 4 ,true},
     265             : { ARM::VLD4d32Pseudo,       ARM::VLD4d32,      true,  false, false, SingleSpc,  4, 2 ,true},
     266             : { ARM::VLD4d32Pseudo_UPD,   ARM::VLD4d32_UPD, true, true, true,  SingleSpc,  4, 2 ,true},
     267             : { ARM::VLD4d8Pseudo,        ARM::VLD4d8,       true,  false, false, SingleSpc,  4, 8 ,true},
     268             : { ARM::VLD4d8Pseudo_UPD,    ARM::VLD4d8_UPD, true, true, true,  SingleSpc,  4, 8 ,true},
     269             : 
     270             : { ARM::VLD4q16Pseudo_UPD,    ARM::VLD4q16_UPD, true, true, true,  EvenDblSpc, 4, 4 ,true},
     271             : { ARM::VLD4q16oddPseudo,     ARM::VLD4q16,     true,  false, false, OddDblSpc,  4, 4 ,true},
     272             : { ARM::VLD4q16oddPseudo_UPD, ARM::VLD4q16_UPD, true, true, true,  OddDblSpc,  4, 4 ,true},
     273             : { ARM::VLD4q32Pseudo_UPD,    ARM::VLD4q32_UPD, true, true, true,  EvenDblSpc, 4, 2 ,true},
     274             : { ARM::VLD4q32oddPseudo,     ARM::VLD4q32,     true,  false, false, OddDblSpc,  4, 2 ,true},
     275             : { ARM::VLD4q32oddPseudo_UPD, ARM::VLD4q32_UPD, true, true, true,  OddDblSpc,  4, 2 ,true},
     276             : { ARM::VLD4q8Pseudo_UPD,     ARM::VLD4q8_UPD, true, true, true,  EvenDblSpc, 4, 8 ,true},
     277             : { ARM::VLD4q8oddPseudo,      ARM::VLD4q8,      true,  false, false, OddDblSpc,  4, 8 ,true},
     278             : { ARM::VLD4q8oddPseudo_UPD,  ARM::VLD4q8_UPD, true, true, true,  OddDblSpc,  4, 8 ,true},
     279             : 
     280             : { ARM::VST1LNq16Pseudo,     ARM::VST1LNd16,    false, false, false, EvenDblSpc, 1, 4 ,true},
     281             : { ARM::VST1LNq16Pseudo_UPD, ARM::VST1LNd16_UPD, false, true, true,  EvenDblSpc, 1, 4 ,true},
     282             : { ARM::VST1LNq32Pseudo,     ARM::VST1LNd32,    false, false, false, EvenDblSpc, 1, 2 ,true},
     283             : { ARM::VST1LNq32Pseudo_UPD, ARM::VST1LNd32_UPD, false, true, true,  EvenDblSpc, 1, 2 ,true},
     284             : { ARM::VST1LNq8Pseudo,      ARM::VST1LNd8,     false, false, false, EvenDblSpc, 1, 8 ,true},
     285             : { ARM::VST1LNq8Pseudo_UPD,  ARM::VST1LNd8_UPD, false, true, true,  EvenDblSpc, 1, 8 ,true},
     286             : 
     287             : { ARM::VST1d16QPseudo,      ARM::VST1d16Q,     false, false, false, SingleSpc,  4, 4 ,false},
     288             : { ARM::VST1d16TPseudo,      ARM::VST1d16T,     false, false, false, SingleSpc,  3, 4 ,false},
     289             : { ARM::VST1d32QPseudo,      ARM::VST1d32Q,     false, false, false, SingleSpc,  4, 2 ,false},
     290             : { ARM::VST1d32TPseudo,      ARM::VST1d32T,     false, false, false, SingleSpc,  3, 2 ,false},
     291             : { ARM::VST1d64QPseudo,      ARM::VST1d64Q,     false, false, false, SingleSpc,  4, 1 ,false},
     292             : { ARM::VST1d64QPseudoWB_fixed,  ARM::VST1d64Qwb_fixed, false, true, false,  SingleSpc,  4, 1 ,false},
     293             : { ARM::VST1d64QPseudoWB_register, ARM::VST1d64Qwb_register, false, true, true,  SingleSpc,  4, 1 ,false},
     294             : { ARM::VST1d64TPseudo,      ARM::VST1d64T,     false, false, false, SingleSpc,  3, 1 ,false},
     295             : { ARM::VST1d64TPseudoWB_fixed,  ARM::VST1d64Twb_fixed, false, true, false,  SingleSpc,  3, 1 ,false},
     296             : { ARM::VST1d64TPseudoWB_register,  ARM::VST1d64Twb_register, false, true, true,  SingleSpc,  3, 1 ,false},
     297             : { ARM::VST1d8QPseudo,       ARM::VST1d8Q,      false, false, false, SingleSpc,  4, 8 ,false},
     298             : { ARM::VST1d8TPseudo,       ARM::VST1d8T,      false, false, false, SingleSpc,  3, 8 ,false},
     299             : { ARM::VST1q16HighQPseudo,  ARM::VST1d16Q,      false, false, false, SingleHighQSpc,   4, 4 ,false},
     300             : { ARM::VST1q16HighTPseudo,  ARM::VST1d16T,      false, false, false, SingleHighTSpc,   3, 4 ,false},
     301             : { ARM::VST1q16LowQPseudo_UPD,   ARM::VST1d16Qwb_fixed,  false, true, true, SingleLowSpc,   4, 4 ,false},
     302             : { ARM::VST1q16LowTPseudo_UPD,   ARM::VST1d16Twb_fixed,  false, true, true, SingleLowSpc,   3, 4 ,false},
     303             : { ARM::VST1q32HighQPseudo,  ARM::VST1d32Q,      false, false, false, SingleHighQSpc,   4, 2 ,false},
     304             : { ARM::VST1q32HighTPseudo,  ARM::VST1d32T,      false, false, false, SingleHighTSpc,   3, 2 ,false},
     305             : { ARM::VST1q32LowQPseudo_UPD,   ARM::VST1d32Qwb_fixed,  false, true, true, SingleLowSpc,   4, 2 ,false},
     306             : { ARM::VST1q32LowTPseudo_UPD,   ARM::VST1d32Twb_fixed,  false, true, true, SingleLowSpc,   3, 2 ,false},
     307             : { ARM::VST1q64HighQPseudo,  ARM::VST1d64Q,      false, false, false, SingleHighQSpc,   4, 1 ,false},
     308             : { ARM::VST1q64HighTPseudo,  ARM::VST1d64T,      false, false, false, SingleHighTSpc,   3, 1 ,false},
     309             : { ARM::VST1q64LowQPseudo_UPD,   ARM::VST1d64Qwb_fixed,  false, true, true, SingleLowSpc,   4, 1 ,false},
     310             : { ARM::VST1q64LowTPseudo_UPD,   ARM::VST1d64Twb_fixed,  false, true, true, SingleLowSpc,   3, 1 ,false},
     311             : { ARM::VST1q8HighQPseudo,   ARM::VST1d8Q,      false, false, false, SingleHighQSpc,   4, 8 ,false},
     312             : { ARM::VST1q8HighTPseudo,   ARM::VST1d8T,      false, false, false, SingleHighTSpc,   3, 8 ,false},
     313             : { ARM::VST1q8LowQPseudo_UPD,   ARM::VST1d8Qwb_fixed,  false, true, true, SingleLowSpc,   4, 8 ,false},
     314             : { ARM::VST1q8LowTPseudo_UPD,   ARM::VST1d8Twb_fixed,  false, true, true, SingleLowSpc,   3, 8 ,false},
     315             : 
     316             : { ARM::VST2LNd16Pseudo,     ARM::VST2LNd16,     false, false, false, SingleSpc, 2, 4 ,true},
     317             : { ARM::VST2LNd16Pseudo_UPD, ARM::VST2LNd16_UPD, false, true, true,  SingleSpc, 2, 4 ,true},
     318             : { ARM::VST2LNd32Pseudo,     ARM::VST2LNd32,     false, false, false, SingleSpc, 2, 2 ,true},
     319             : { ARM::VST2LNd32Pseudo_UPD, ARM::VST2LNd32_UPD, false, true, true,  SingleSpc, 2, 2 ,true},
     320             : { ARM::VST2LNd8Pseudo,      ARM::VST2LNd8,      false, false, false, SingleSpc, 2, 8 ,true},
     321             : { ARM::VST2LNd8Pseudo_UPD,  ARM::VST2LNd8_UPD, false, true, true,  SingleSpc, 2, 8 ,true},
     322             : { ARM::VST2LNq16Pseudo,     ARM::VST2LNq16,     false, false, false, EvenDblSpc, 2, 4,true},
     323             : { ARM::VST2LNq16Pseudo_UPD, ARM::VST2LNq16_UPD, false, true, true,  EvenDblSpc, 2, 4,true},
     324             : { ARM::VST2LNq32Pseudo,     ARM::VST2LNq32,     false, false, false, EvenDblSpc, 2, 2,true},
     325             : { ARM::VST2LNq32Pseudo_UPD, ARM::VST2LNq32_UPD, false, true, true,  EvenDblSpc, 2, 2,true},
     326             : 
     327             : { ARM::VST2q16Pseudo,       ARM::VST2q16,      false, false, false, SingleSpc,  4, 4 ,false},
     328             : { ARM::VST2q16PseudoWB_fixed,   ARM::VST2q16wb_fixed, false, true, false,  SingleSpc,  4, 4 ,false},
     329             : { ARM::VST2q16PseudoWB_register,   ARM::VST2q16wb_register, false, true, true,  SingleSpc,  4, 4 ,false},
     330             : { ARM::VST2q32Pseudo,       ARM::VST2q32,      false, false, false, SingleSpc,  4, 2 ,false},
     331             : { ARM::VST2q32PseudoWB_fixed,   ARM::VST2q32wb_fixed, false, true, false,  SingleSpc,  4, 2 ,false},
     332             : { ARM::VST2q32PseudoWB_register,   ARM::VST2q32wb_register, false, true, true,  SingleSpc,  4, 2 ,false},
     333             : { ARM::VST2q8Pseudo,        ARM::VST2q8,       false, false, false, SingleSpc,  4, 8 ,false},
     334             : { ARM::VST2q8PseudoWB_fixed,    ARM::VST2q8wb_fixed, false, true, false,  SingleSpc,  4, 8 ,false},
     335             : { ARM::VST2q8PseudoWB_register,    ARM::VST2q8wb_register, false, true, true,  SingleSpc,  4, 8 ,false},
     336             : 
     337             : { ARM::VST3LNd16Pseudo,     ARM::VST3LNd16,     false, false, false, SingleSpc, 3, 4 ,true},
     338             : { ARM::VST3LNd16Pseudo_UPD, ARM::VST3LNd16_UPD, false, true, true,  SingleSpc, 3, 4 ,true},
     339             : { ARM::VST3LNd32Pseudo,     ARM::VST3LNd32,     false, false, false, SingleSpc, 3, 2 ,true},
     340             : { ARM::VST3LNd32Pseudo_UPD, ARM::VST3LNd32_UPD, false, true, true,  SingleSpc, 3, 2 ,true},
     341             : { ARM::VST3LNd8Pseudo,      ARM::VST3LNd8,      false, false, false, SingleSpc, 3, 8 ,true},
     342             : { ARM::VST3LNd8Pseudo_UPD,  ARM::VST3LNd8_UPD, false, true, true,  SingleSpc, 3, 8 ,true},
     343             : { ARM::VST3LNq16Pseudo,     ARM::VST3LNq16,     false, false, false, EvenDblSpc, 3, 4,true},
     344             : { ARM::VST3LNq16Pseudo_UPD, ARM::VST3LNq16_UPD, false, true, true,  EvenDblSpc, 3, 4,true},
     345             : { ARM::VST3LNq32Pseudo,     ARM::VST3LNq32,     false, false, false, EvenDblSpc, 3, 2,true},
     346             : { ARM::VST3LNq32Pseudo_UPD, ARM::VST3LNq32_UPD, false, true, true,  EvenDblSpc, 3, 2,true},
     347             : 
     348             : { ARM::VST3d16Pseudo,       ARM::VST3d16,      false, false, false, SingleSpc,  3, 4 ,true},
     349             : { ARM::VST3d16Pseudo_UPD,   ARM::VST3d16_UPD, false, true, true,  SingleSpc,  3, 4 ,true},
     350             : { ARM::VST3d32Pseudo,       ARM::VST3d32,      false, false, false, SingleSpc,  3, 2 ,true},
     351             : { ARM::VST3d32Pseudo_UPD,   ARM::VST3d32_UPD, false, true, true,  SingleSpc,  3, 2 ,true},
     352             : { ARM::VST3d8Pseudo,        ARM::VST3d8,       false, false, false, SingleSpc,  3, 8 ,true},
     353             : { ARM::VST3d8Pseudo_UPD,    ARM::VST3d8_UPD, false, true, true,  SingleSpc,  3, 8 ,true},
     354             : 
     355             : { ARM::VST3q16Pseudo_UPD,    ARM::VST3q16_UPD, false, true, true,  EvenDblSpc, 3, 4 ,true},
     356             : { ARM::VST3q16oddPseudo,     ARM::VST3q16,     false, false, false, OddDblSpc,  3, 4 ,true},
     357             : { ARM::VST3q16oddPseudo_UPD, ARM::VST3q16_UPD, false, true, true,  OddDblSpc,  3, 4 ,true},
     358             : { ARM::VST3q32Pseudo_UPD,    ARM::VST3q32_UPD, false, true, true,  EvenDblSpc, 3, 2 ,true},
     359             : { ARM::VST3q32oddPseudo,     ARM::VST3q32,     false, false, false, OddDblSpc,  3, 2 ,true},
     360             : { ARM::VST3q32oddPseudo_UPD, ARM::VST3q32_UPD, false, true, true,  OddDblSpc,  3, 2 ,true},
     361             : { ARM::VST3q8Pseudo_UPD,     ARM::VST3q8_UPD, false, true, true,  EvenDblSpc, 3, 8 ,true},
     362             : { ARM::VST3q8oddPseudo,      ARM::VST3q8,      false, false, false, OddDblSpc,  3, 8 ,true},
     363             : { ARM::VST3q8oddPseudo_UPD,  ARM::VST3q8_UPD, false, true, true,  OddDblSpc,  3, 8 ,true},
     364             : 
     365             : { ARM::VST4LNd16Pseudo,     ARM::VST4LNd16,     false, false, false, SingleSpc, 4, 4 ,true},
     366             : { ARM::VST4LNd16Pseudo_UPD, ARM::VST4LNd16_UPD, false, true, true,  SingleSpc, 4, 4 ,true},
     367             : { ARM::VST4LNd32Pseudo,     ARM::VST4LNd32,     false, false, false, SingleSpc, 4, 2 ,true},
     368             : { ARM::VST4LNd32Pseudo_UPD, ARM::VST4LNd32_UPD, false, true, true,  SingleSpc, 4, 2 ,true},
     369             : { ARM::VST4LNd8Pseudo,      ARM::VST4LNd8,      false, false, false, SingleSpc, 4, 8 ,true},
     370             : { ARM::VST4LNd8Pseudo_UPD,  ARM::VST4LNd8_UPD, false, true, true,  SingleSpc, 4, 8 ,true},
     371             : { ARM::VST4LNq16Pseudo,     ARM::VST4LNq16,     false, false, false, EvenDblSpc, 4, 4,true},
     372             : { ARM::VST4LNq16Pseudo_UPD, ARM::VST4LNq16_UPD, false, true, true,  EvenDblSpc, 4, 4,true},
     373             : { ARM::VST4LNq32Pseudo,     ARM::VST4LNq32,     false, false, false, EvenDblSpc, 4, 2,true},
     374             : { ARM::VST4LNq32Pseudo_UPD, ARM::VST4LNq32_UPD, false, true, true,  EvenDblSpc, 4, 2,true},
     375             : 
     376             : { ARM::VST4d16Pseudo,       ARM::VST4d16,      false, false, false, SingleSpc,  4, 4 ,true},
     377             : { ARM::VST4d16Pseudo_UPD,   ARM::VST4d16_UPD, false, true, true,  SingleSpc,  4, 4 ,true},
     378             : { ARM::VST4d32Pseudo,       ARM::VST4d32,      false, false, false, SingleSpc,  4, 2 ,true},
     379             : { ARM::VST4d32Pseudo_UPD,   ARM::VST4d32_UPD, false, true, true,  SingleSpc,  4, 2 ,true},
     380             : { ARM::VST4d8Pseudo,        ARM::VST4d8,       false, false, false, SingleSpc,  4, 8 ,true},
     381             : { ARM::VST4d8Pseudo_UPD,    ARM::VST4d8_UPD, false, true, true,  SingleSpc,  4, 8 ,true},
     382             : 
     383             : { ARM::VST4q16Pseudo_UPD,    ARM::VST4q16_UPD, false, true, true,  EvenDblSpc, 4, 4 ,true},
     384             : { ARM::VST4q16oddPseudo,     ARM::VST4q16,     false, false, false, OddDblSpc,  4, 4 ,true},
     385             : { ARM::VST4q16oddPseudo_UPD, ARM::VST4q16_UPD, false, true, true,  OddDblSpc,  4, 4 ,true},
     386             : { ARM::VST4q32Pseudo_UPD,    ARM::VST4q32_UPD, false, true, true,  EvenDblSpc, 4, 2 ,true},
     387             : { ARM::VST4q32oddPseudo,     ARM::VST4q32,     false, false, false, OddDblSpc,  4, 2 ,true},
     388             : { ARM::VST4q32oddPseudo_UPD, ARM::VST4q32_UPD, false, true, true,  OddDblSpc,  4, 2 ,true},
     389             : { ARM::VST4q8Pseudo_UPD,     ARM::VST4q8_UPD, false, true, true,  EvenDblSpc, 4, 8 ,true},
     390             : { ARM::VST4q8oddPseudo,      ARM::VST4q8,      false, false, false, OddDblSpc,  4, 8 ,true},
     391             : { ARM::VST4q8oddPseudo_UPD,  ARM::VST4q8_UPD, false, true, true,  OddDblSpc,  4, 8 ,true}
     392             : };
     393             : 
     394             : /// LookupNEONLdSt - Search the NEONLdStTable for information about a NEON
     395             : /// load or store pseudo instruction.
     396             : static const NEONLdStTableEntry *LookupNEONLdSt(unsigned Opcode) {
     397             : #ifndef NDEBUG
     398             :   // Make sure the table is sorted.
     399             :   static bool TableChecked = false;
     400             :   if (!TableChecked) {
     401             :     assert(std::is_sorted(std::begin(NEONLdStTable), std::end(NEONLdStTable)) &&
     402             :            "NEONLdStTable is not sorted!");
     403             :     TableChecked = true;
     404             :   }
     405             : #endif
     406             : 
     407             :   auto I = std::lower_bound(std::begin(NEONLdStTable),
     408             :                             std::end(NEONLdStTable), Opcode);
     409         391 :   if (I != std::end(NEONLdStTable) && I->PseudoOpc == Opcode)
     410             :     return I;
     411             :   return nullptr;
     412             : }
     413             : 
     414             : /// GetDSubRegs - Get 4 D subregisters of a Q, QQ, or QQQQ register,
     415             : /// corresponding to the specified register spacing.  Not all of the results
     416             : /// are necessarily valid, e.g., a Q register only has 2 D subregisters.
     417         396 : static void GetDSubRegs(unsigned Reg, NEONRegSpacing RegSpc,
     418             :                         const TargetRegisterInfo *TRI, unsigned &D0,
     419             :                         unsigned &D1, unsigned &D2, unsigned &D3) {
     420         396 :   if (RegSpc == SingleSpc || RegSpc == SingleLowSpc) {
     421         197 :     D0 = TRI->getSubReg(Reg, ARM::dsub_0);
     422         197 :     D1 = TRI->getSubReg(Reg, ARM::dsub_1);
     423         197 :     D2 = TRI->getSubReg(Reg, ARM::dsub_2);
     424         197 :     D3 = TRI->getSubReg(Reg, ARM::dsub_3);
     425         199 :   } else if (RegSpc == SingleHighQSpc) {
     426           8 :     D0 = TRI->getSubReg(Reg, ARM::dsub_4);
     427           8 :     D1 = TRI->getSubReg(Reg, ARM::dsub_5);
     428           8 :     D2 = TRI->getSubReg(Reg, ARM::dsub_6);
     429           8 :     D3 = TRI->getSubReg(Reg, ARM::dsub_7);
     430         191 :   } else if (RegSpc == SingleHighTSpc) {
     431           8 :     D0 = TRI->getSubReg(Reg, ARM::dsub_3);
     432           8 :     D1 = TRI->getSubReg(Reg, ARM::dsub_4);
     433           8 :     D2 = TRI->getSubReg(Reg, ARM::dsub_5);
     434           8 :     D3 = TRI->getSubReg(Reg, ARM::dsub_6);
     435         183 :   } else if (RegSpc == EvenDblSpc) {
     436          94 :     D0 = TRI->getSubReg(Reg, ARM::dsub_0);
     437          94 :     D1 = TRI->getSubReg(Reg, ARM::dsub_2);
     438          94 :     D2 = TRI->getSubReg(Reg, ARM::dsub_4);
     439          94 :     D3 = TRI->getSubReg(Reg, ARM::dsub_6);
     440             :   } else {
     441             :     assert(RegSpc == OddDblSpc && "unknown register spacing");
     442          89 :     D0 = TRI->getSubReg(Reg, ARM::dsub_1);
     443          89 :     D1 = TRI->getSubReg(Reg, ARM::dsub_3);
     444          89 :     D2 = TRI->getSubReg(Reg, ARM::dsub_5);
     445          89 :     D3 = TRI->getSubReg(Reg, ARM::dsub_7);
     446             :   }
     447         396 : }
     448             : 
     449             : /// ExpandVLD - Translate VLD pseudo instructions with Q, QQ or QQQQ register
     450             : /// operands to real VLD instructions with D register operands.
     451         124 : void ARMExpandPseudo::ExpandVLD(MachineBasicBlock::iterator &MBBI) {
     452             :   MachineInstr &MI = *MBBI;
     453         124 :   MachineBasicBlock &MBB = *MI.getParent();
     454             : 
     455         124 :   const NEONLdStTableEntry *TableEntry = LookupNEONLdSt(MI.getOpcode());
     456             :   assert(TableEntry && TableEntry->IsLoad && "NEONLdStTable lookup failed");
     457         124 :   NEONRegSpacing RegSpc = (NEONRegSpacing)TableEntry->RegSpacing;
     458         124 :   unsigned NumRegs = TableEntry->NumRegs;
     459             : 
     460             :   MachineInstrBuilder MIB = BuildMI(MBB, MBBI, MI.getDebugLoc(),
     461         248 :                                     TII->get(TableEntry->RealOpc));
     462             :   unsigned OpIdx = 0;
     463             : 
     464         124 :   bool DstIsDead = MI.getOperand(OpIdx).isDead();
     465         124 :   unsigned DstReg = MI.getOperand(OpIdx++).getReg();
     466             :   unsigned D0, D1, D2, D3;
     467         124 :   GetDSubRegs(DstReg, RegSpc, TRI, D0, D1, D2, D3);
     468         124 :   MIB.addReg(D0, RegState::Define | getDeadRegState(DstIsDead));
     469         124 :   if (NumRegs > 1 && TableEntry->copyAllListRegs)
     470          71 :     MIB.addReg(D1, RegState::Define | getDeadRegState(DstIsDead));
     471         124 :   if (NumRegs > 2 && TableEntry->copyAllListRegs)
     472          71 :     MIB.addReg(D2, RegState::Define | getDeadRegState(DstIsDead));
     473         124 :   if (NumRegs > 3 && TableEntry->copyAllListRegs)
     474          20 :     MIB.addReg(D3, RegState::Define | getDeadRegState(DstIsDead));
     475             : 
     476         124 :   if (TableEntry->isUpdating)
     477          49 :     MIB.add(MI.getOperand(OpIdx++));
     478             : 
     479             :   // Copy the addrmode6 operands.
     480         124 :   MIB.add(MI.getOperand(OpIdx++));
     481         124 :   MIB.add(MI.getOperand(OpIdx++));
     482             : 
     483             :   // Copy the am6offset operand.
     484         124 :   if (TableEntry->hasWritebackOperand) {
     485             :     // TODO: The writing-back pseudo instructions we translate here are all
     486             :     // defined to take am6offset nodes that are capable to represent both fixed
     487             :     // and register forms. Some real instructions, however, do not rely on
     488             :     // am6offset and have separate definitions for such forms. When this is the
     489             :     // case, fixed forms do not take any offset nodes, so here we skip them for
     490             :     // such instructions. Once all real and pseudo writing-back instructions are
     491             :     // rewritten without use of am6offset nodes, this code will go away.
     492          41 :     const MachineOperand &AM6Offset = MI.getOperand(OpIdx++);
     493          41 :     if (TableEntry->RealOpc == ARM::VLD1d8Qwb_fixed ||
     494          39 :         TableEntry->RealOpc == ARM::VLD1d16Qwb_fixed ||
     495          38 :         TableEntry->RealOpc == ARM::VLD1d32Qwb_fixed ||
     496          37 :         TableEntry->RealOpc == ARM::VLD1d64Qwb_fixed ||
     497          36 :         TableEntry->RealOpc == ARM::VLD1d8Twb_fixed ||
     498          35 :         TableEntry->RealOpc == ARM::VLD1d16Twb_fixed ||
     499          34 :         TableEntry->RealOpc == ARM::VLD1d32Twb_fixed ||
     500             :         TableEntry->RealOpc == ARM::VLD1d64Twb_fixed) {
     501             :       assert(AM6Offset.getReg() == 0 &&
     502             :              "A fixed writing-back pseudo instruction provides an offset "
     503             :              "register!");
     504             :     } else {
     505             :       MIB.add(AM6Offset);
     506             :     }
     507             :   }
     508             : 
     509             :   // For an instruction writing double-spaced subregs, the pseudo instruction
     510             :   // has an extra operand that is a use of the super-register.  Record the
     511             :   // operand index and skip over it.
     512             :   unsigned SrcOpIdx = 0;
     513         248 :   if (RegSpc == EvenDblSpc || RegSpc == OddDblSpc ||
     514         201 :       RegSpc == SingleLowSpc || RegSpc == SingleHighQSpc ||
     515             :       RegSpc == SingleHighTSpc)
     516          55 :     SrcOpIdx = OpIdx++;
     517             : 
     518             :   // Copy the predicate operands.
     519         124 :   MIB.add(MI.getOperand(OpIdx++));
     520         124 :   MIB.add(MI.getOperand(OpIdx++));
     521             : 
     522             :   // Copy the super-register source operand used for double-spaced subregs over
     523             :   // to the new instruction as an implicit operand.
     524         124 :   if (SrcOpIdx != 0) {
     525         110 :     MachineOperand MO = MI.getOperand(SrcOpIdx);
     526             :     MO.setImplicit(true);
     527             :     MIB.add(MO);
     528             :   }
     529             :   // Add an implicit def for the super-register.
     530         124 :   MIB.addReg(DstReg, RegState::ImplicitDefine | getDeadRegState(DstIsDead));
     531         124 :   TransferImpOps(MI, MIB, MIB);
     532             : 
     533             :   // Transfer memoperands.
     534         124 :   MIB->setMemRefs(MI.memoperands_begin(), MI.memoperands_end());
     535             : 
     536         124 :   MI.eraseFromParent();
     537         124 : }
     538             : 
     539             : /// ExpandVST - Translate VST pseudo instructions with Q, QQ or QQQQ register
     540             : /// operands to real VST instructions with D register operands.
     541         107 : void ARMExpandPseudo::ExpandVST(MachineBasicBlock::iterator &MBBI) {
     542             :   MachineInstr &MI = *MBBI;
     543         107 :   MachineBasicBlock &MBB = *MI.getParent();
     544             : 
     545         107 :   const NEONLdStTableEntry *TableEntry = LookupNEONLdSt(MI.getOpcode());
     546             :   assert(TableEntry && !TableEntry->IsLoad && "NEONLdStTable lookup failed");
     547         107 :   NEONRegSpacing RegSpc = (NEONRegSpacing)TableEntry->RegSpacing;
     548         107 :   unsigned NumRegs = TableEntry->NumRegs;
     549             : 
     550             :   MachineInstrBuilder MIB = BuildMI(MBB, MBBI, MI.getDebugLoc(),
     551         214 :                                     TII->get(TableEntry->RealOpc));
     552             :   unsigned OpIdx = 0;
     553         107 :   if (TableEntry->isUpdating)
     554          36 :     MIB.add(MI.getOperand(OpIdx++));
     555             : 
     556             :   // Copy the addrmode6 operands.
     557         107 :   MIB.add(MI.getOperand(OpIdx++));
     558         107 :   MIB.add(MI.getOperand(OpIdx++));
     559             : 
     560         107 :   if (TableEntry->hasWritebackOperand) {
     561             :     // TODO: The writing-back pseudo instructions we translate here are all
     562             :     // defined to take am6offset nodes that are capable to represent both fixed
     563             :     // and register forms. Some real instructions, however, do not rely on
     564             :     // am6offset and have separate definitions for such forms. When this is the
     565             :     // case, fixed forms do not take any offset nodes, so here we skip them for
     566             :     // such instructions. Once all real and pseudo writing-back instructions are
     567             :     // rewritten without use of am6offset nodes, this code will go away.
     568          32 :     const MachineOperand &AM6Offset = MI.getOperand(OpIdx++);
     569          32 :     if (TableEntry->RealOpc == ARM::VST1d8Qwb_fixed ||
     570          30 :         TableEntry->RealOpc == ARM::VST1d16Qwb_fixed ||
     571          29 :         TableEntry->RealOpc == ARM::VST1d32Qwb_fixed ||
     572          28 :         TableEntry->RealOpc == ARM::VST1d64Qwb_fixed ||
     573          27 :         TableEntry->RealOpc == ARM::VST1d8Twb_fixed ||
     574          26 :         TableEntry->RealOpc == ARM::VST1d16Twb_fixed ||
     575          25 :         TableEntry->RealOpc == ARM::VST1d32Twb_fixed ||
     576             :         TableEntry->RealOpc == ARM::VST1d64Twb_fixed) {
     577             :       assert(AM6Offset.getReg() == 0 &&
     578             :              "A fixed writing-back pseudo instruction provides an offset "
     579             :              "register!");
     580             :     } else {
     581             :       MIB.add(AM6Offset);
     582             :     }
     583             :   }
     584             : 
     585         107 :   bool SrcIsKill = MI.getOperand(OpIdx).isKill();
     586             :   bool SrcIsUndef = MI.getOperand(OpIdx).isUndef();
     587         107 :   unsigned SrcReg = MI.getOperand(OpIdx++).getReg();
     588             :   unsigned D0, D1, D2, D3;
     589         107 :   GetDSubRegs(SrcReg, RegSpc, TRI, D0, D1, D2, D3);
     590         107 :   MIB.addReg(D0, getUndefRegState(SrcIsUndef));
     591         107 :   if (NumRegs > 1 && TableEntry->copyAllListRegs)
     592          57 :     MIB.addReg(D1, getUndefRegState(SrcIsUndef));
     593         107 :   if (NumRegs > 2 && TableEntry->copyAllListRegs)
     594          57 :     MIB.addReg(D2, getUndefRegState(SrcIsUndef));
     595         107 :   if (NumRegs > 3 && TableEntry->copyAllListRegs)
     596          32 :     MIB.addReg(D3, getUndefRegState(SrcIsUndef));
     597             : 
     598             :   // Copy the predicate operands.
     599         107 :   MIB.add(MI.getOperand(OpIdx++));
     600         107 :   MIB.add(MI.getOperand(OpIdx++));
     601             : 
     602         107 :   if (SrcIsKill && !SrcIsUndef) // Add an implicit kill for the super-reg.
     603          72 :     MIB->addRegisterKilled(SrcReg, TRI, true);
     604          35 :   else if (!SrcIsUndef)
     605          34 :     MIB.addReg(SrcReg, RegState::Implicit); // Add implicit uses for src reg.
     606         107 :   TransferImpOps(MI, MIB, MIB);
     607             : 
     608             :   // Transfer memoperands.
     609         107 :   MIB->setMemRefs(MI.memoperands_begin(), MI.memoperands_end());
     610             : 
     611         107 :   MI.eraseFromParent();
     612         107 : }
     613             : 
     614             : /// ExpandLaneOp - Translate VLD*LN and VST*LN instructions with Q, QQ or QQQQ
     615             : /// register operands to real instructions with D register operands.
     616         160 : void ARMExpandPseudo::ExpandLaneOp(MachineBasicBlock::iterator &MBBI) {
     617             :   MachineInstr &MI = *MBBI;
     618         160 :   MachineBasicBlock &MBB = *MI.getParent();
     619             : 
     620         160 :   const NEONLdStTableEntry *TableEntry = LookupNEONLdSt(MI.getOpcode());
     621             :   assert(TableEntry && "NEONLdStTable lookup failed");
     622         160 :   NEONRegSpacing RegSpc = (NEONRegSpacing)TableEntry->RegSpacing;
     623         160 :   unsigned NumRegs = TableEntry->NumRegs;
     624         160 :   unsigned RegElts = TableEntry->RegElts;
     625             : 
     626             :   MachineInstrBuilder MIB = BuildMI(MBB, MBBI, MI.getDebugLoc(),
     627         320 :                                     TII->get(TableEntry->RealOpc));
     628             :   unsigned OpIdx = 0;
     629             :   // The lane operand is always the 3rd from last operand, before the 2
     630             :   // predicate operands.
     631         480 :   unsigned Lane = MI.getOperand(MI.getDesc().getNumOperands() - 3).getImm();
     632             : 
     633             :   // Adjust the lane and spacing as needed for Q registers.
     634             :   assert(RegSpc != OddDblSpc && "unexpected register spacing for VLD/VST-lane");
     635         160 :   if (RegSpc == EvenDblSpc && Lane >= RegElts) {
     636             :     RegSpc = OddDblSpc;
     637          52 :     Lane -= RegElts;
     638             :   }
     639             :   assert(Lane < RegElts && "out of range lane for VLD/VST-lane");
     640             : 
     641         160 :   unsigned D0 = 0, D1 = 0, D2 = 0, D3 = 0;
     642             :   unsigned DstReg = 0;
     643             :   bool DstIsDead = false;
     644         160 :   if (TableEntry->IsLoad) {
     645             :     DstIsDead = MI.getOperand(OpIdx).isDead();
     646          75 :     DstReg = MI.getOperand(OpIdx++).getReg();
     647          75 :     GetDSubRegs(DstReg, RegSpc, TRI, D0, D1, D2, D3);
     648          75 :     MIB.addReg(D0, RegState::Define | getDeadRegState(DstIsDead));
     649          75 :     if (NumRegs > 1)
     650          60 :       MIB.addReg(D1, RegState::Define | getDeadRegState(DstIsDead));
     651          75 :     if (NumRegs > 2)
     652          37 :       MIB.addReg(D2, RegState::Define | getDeadRegState(DstIsDead));
     653          75 :     if (NumRegs > 3)
     654          17 :       MIB.addReg(D3, RegState::Define | getDeadRegState(DstIsDead));
     655             :   }
     656             : 
     657         160 :   if (TableEntry->isUpdating)
     658          11 :     MIB.add(MI.getOperand(OpIdx++));
     659             : 
     660             :   // Copy the addrmode6 operands.
     661         160 :   MIB.add(MI.getOperand(OpIdx++));
     662         160 :   MIB.add(MI.getOperand(OpIdx++));
     663             :   // Copy the am6offset operand.
     664         160 :   if (TableEntry->hasWritebackOperand)
     665          11 :     MIB.add(MI.getOperand(OpIdx++));
     666             : 
     667             :   // Grab the super-register source.
     668         320 :   MachineOperand MO = MI.getOperand(OpIdx++);
     669         160 :   if (!TableEntry->IsLoad)
     670          85 :     GetDSubRegs(MO.getReg(), RegSpc, TRI, D0, D1, D2, D3);
     671             : 
     672             :   // Add the subregs as sources of the new instruction.
     673             :   unsigned SrcFlags = (getUndefRegState(MO.isUndef()) |
     674         160 :                        getKillRegState(MO.isKill()));
     675         160 :   MIB.addReg(D0, SrcFlags);
     676         160 :   if (NumRegs > 1)
     677          87 :     MIB.addReg(D1, SrcFlags);
     678         160 :   if (NumRegs > 2)
     679          55 :     MIB.addReg(D2, SrcFlags);
     680         160 :   if (NumRegs > 3)
     681          26 :     MIB.addReg(D3, SrcFlags);
     682             : 
     683             :   // Add the lane number operand.
     684         160 :   MIB.addImm(Lane);
     685         160 :   OpIdx += 1;
     686             : 
     687             :   // Copy the predicate operands.
     688         160 :   MIB.add(MI.getOperand(OpIdx++));
     689         160 :   MIB.add(MI.getOperand(OpIdx++));
     690             : 
     691             :   // Copy the super-register source to be an implicit source.
     692             :   MO.setImplicit(true);
     693             :   MIB.add(MO);
     694         160 :   if (TableEntry->IsLoad)
     695             :     // Add an implicit def for the super-register.
     696          75 :     MIB.addReg(DstReg, RegState::ImplicitDefine | getDeadRegState(DstIsDead));
     697         160 :   TransferImpOps(MI, MIB, MIB);
     698             :   // Transfer memoperands.
     699         160 :   MIB->setMemRefs(MI.memoperands_begin(), MI.memoperands_end());
     700         160 :   MI.eraseFromParent();
     701         160 : }
     702             : 
     703             : /// ExpandVTBL - Translate VTBL and VTBX pseudo instructions with Q or QQ
     704             : /// register operands to real instructions with D register operands.
     705           5 : void ARMExpandPseudo::ExpandVTBL(MachineBasicBlock::iterator &MBBI,
     706             :                                  unsigned Opc, bool IsExt) {
     707             :   MachineInstr &MI = *MBBI;
     708           5 :   MachineBasicBlock &MBB = *MI.getParent();
     709             : 
     710          10 :   MachineInstrBuilder MIB = BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(Opc));
     711             :   unsigned OpIdx = 0;
     712             : 
     713             :   // Transfer the destination register operand.
     714           5 :   MIB.add(MI.getOperand(OpIdx++));
     715           5 :   if (IsExt) {
     716           3 :     MachineOperand VdSrc(MI.getOperand(OpIdx++));
     717             :     MIB.add(VdSrc);
     718             :   }
     719             : 
     720           5 :   bool SrcIsKill = MI.getOperand(OpIdx).isKill();
     721           5 :   unsigned SrcReg = MI.getOperand(OpIdx++).getReg();
     722             :   unsigned D0, D1, D2, D3;
     723           5 :   GetDSubRegs(SrcReg, SingleSpc, TRI, D0, D1, D2, D3);
     724           5 :   MIB.addReg(D0);
     725             : 
     726             :   // Copy the other source register operand.
     727          10 :   MachineOperand VmSrc(MI.getOperand(OpIdx++));
     728             :   MIB.add(VmSrc);
     729             : 
     730             :   // Copy the predicate operands.
     731           5 :   MIB.add(MI.getOperand(OpIdx++));
     732           5 :   MIB.add(MI.getOperand(OpIdx++));
     733             : 
     734             :   // Add an implicit kill and use for the super-reg.
     735           5 :   MIB.addReg(SrcReg, RegState::Implicit | getKillRegState(SrcIsKill));
     736           5 :   TransferImpOps(MI, MIB, MIB);
     737           5 :   MI.eraseFromParent();
     738           5 : }
     739             : 
     740          57 : static bool IsAnAddressOperand(const MachineOperand &MO) {
     741             :   // This check is overly conservative.  Unless we are certain that the machine
     742             :   // operand is not a symbol reference, we return that it is a symbol reference.
     743             :   // This is important as the load pair may not be split up Windows.
     744          57 :   switch (MO.getType()) {
     745             :   case MachineOperand::MO_Register:
     746             :   case MachineOperand::MO_Immediate:
     747             :   case MachineOperand::MO_CImmediate:
     748             :   case MachineOperand::MO_FPImmediate:
     749             :     return false;
     750           0 :   case MachineOperand::MO_MachineBasicBlock:
     751           0 :     return true;
     752             :   case MachineOperand::MO_FrameIndex:
     753             :     return false;
     754          57 :   case MachineOperand::MO_ConstantPoolIndex:
     755             :   case MachineOperand::MO_TargetIndex:
     756             :   case MachineOperand::MO_JumpTableIndex:
     757             :   case MachineOperand::MO_ExternalSymbol:
     758             :   case MachineOperand::MO_GlobalAddress:
     759             :   case MachineOperand::MO_BlockAddress:
     760          57 :     return true;
     761             :   case MachineOperand::MO_RegisterMask:
     762             :   case MachineOperand::MO_RegisterLiveOut:
     763             :     return false;
     764           0 :   case MachineOperand::MO_Metadata:
     765             :   case MachineOperand::MO_MCSymbol:
     766           0 :     return true;
     767             :   case MachineOperand::MO_CFIIndex:
     768             :     return false;
     769           0 :   case MachineOperand::MO_IntrinsicID:
     770             :   case MachineOperand::MO_Predicate:
     771           0 :     llvm_unreachable("should not exist post-isel");
     772             :   }
     773           0 :   llvm_unreachable("unhandled machine operand type");
     774             : }
     775             : 
     776             : static MachineOperand makeImplicit(const MachineOperand &MO) {
     777        1176 :   MachineOperand NewMO = MO;
     778             :   NewMO.setImplicit();
     779             :   return NewMO;
     780             : }
     781             : 
     782        1630 : void ARMExpandPseudo::ExpandMOV32BitImm(MachineBasicBlock &MBB,
     783             :                                         MachineBasicBlock::iterator &MBBI) {
     784             :   MachineInstr &MI = *MBBI;
     785        1630 :   unsigned Opcode = MI.getOpcode();
     786        1630 :   unsigned PredReg = 0;
     787        1630 :   ARMCC::CondCodes Pred = getInstrPredicate(MI, PredReg);
     788        1630 :   unsigned DstReg = MI.getOperand(0).getReg();
     789             :   bool DstIsDead = MI.getOperand(0).isDead();
     790        1630 :   bool isCC = Opcode == ARM::MOVCCi32imm || Opcode == ARM::t2MOVCCi32imm;
     791        1630 :   const MachineOperand &MO = MI.getOperand(isCC ? 2 : 1);
     792        3260 :   bool RequiresBundling = STI->isTargetWindows() && IsAnAddressOperand(MO);
     793        1630 :   MachineInstrBuilder LO16, HI16;
     794             : 
     795        1822 :   if (!STI->hasV6T2Ops() &&
     796         192 :       (Opcode == ARM::MOVi32imm || Opcode == ARM::MOVCCi32imm)) {
     797             :     // FIXME Windows CE supports older ARM CPUs
     798             :     assert(!STI->isTargetWindows() && "Windows on ARM requires ARMv7+");
     799             : 
     800             :     // Expand into a movi + orr.
     801         350 :     LO16 = BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(ARM::MOVi), DstReg);
     802         525 :     HI16 = BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(ARM::ORRri))
     803         175 :       .addReg(DstReg, RegState::Define | getDeadRegState(DstIsDead))
     804         175 :       .addReg(DstReg);
     805             : 
     806             :     assert (MO.isImm() && "MOVi32imm w/ non-immediate source operand!");
     807         175 :     unsigned ImmVal = (unsigned)MO.getImm();
     808             :     unsigned SOImmValV1 = ARM_AM::getSOImmTwoPartFirst(ImmVal);
     809             :     unsigned SOImmValV2 = ARM_AM::getSOImmTwoPartSecond(ImmVal);
     810         175 :     LO16 = LO16.addImm(SOImmValV1);
     811         175 :     HI16 = HI16.addImm(SOImmValV2);
     812         175 :     LO16->setMemRefs(MI.memoperands_begin(), MI.memoperands_end());
     813         175 :     HI16->setMemRefs(MI.memoperands_begin(), MI.memoperands_end());
     814         525 :     LO16.addImm(Pred).addReg(PredReg).add(condCodeOp());
     815         350 :     HI16.addImm(Pred).addReg(PredReg).add(condCodeOp());
     816         175 :     if (isCC)
     817           0 :       LO16.add(makeImplicit(MI.getOperand(1)));
     818         175 :     TransferImpOps(MI, LO16, HI16);
     819         175 :     MI.eraseFromParent();
     820         175 :     return;
     821             :   }
     822             : 
     823             :   unsigned LO16Opc = 0;
     824             :   unsigned HI16Opc = 0;
     825        1455 :   if (Opcode == ARM::t2MOVi32imm || Opcode == ARM::t2MOVCCi32imm) {
     826             :     LO16Opc = ARM::t2MOVi16;
     827             :     HI16Opc = ARM::t2MOVTi16;
     828             :   } else {
     829             :     LO16Opc = ARM::MOVi16;
     830             :     HI16Opc = ARM::MOVTi16;
     831             :   }
     832             : 
     833        2910 :   LO16 = BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(LO16Opc), DstReg);
     834        4365 :   HI16 = BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(HI16Opc))
     835        1455 :     .addReg(DstReg, RegState::Define | getDeadRegState(DstIsDead))
     836        1455 :     .addReg(DstReg);
     837             : 
     838        1455 :   switch (MO.getType()) {
     839         513 :   case MachineOperand::MO_Immediate: {
     840         513 :     unsigned Imm = MO.getImm();
     841         513 :     unsigned Lo16 = Imm & 0xffff;
     842         513 :     unsigned Hi16 = (Imm >> 16) & 0xffff;
     843         513 :     LO16 = LO16.addImm(Lo16);
     844         513 :     HI16 = HI16.addImm(Hi16);
     845             :     break;
     846             :   }
     847          10 :   case MachineOperand::MO_ExternalSymbol: {
     848          10 :     const char *ES = MO.getSymbolName();
     849             :     unsigned TF = MO.getTargetFlags();
     850          10 :     LO16 = LO16.addExternalSymbol(ES, TF | ARMII::MO_LO16);
     851          10 :     HI16 = HI16.addExternalSymbol(ES, TF | ARMII::MO_HI16);
     852             :     break;
     853             :   }
     854         932 :   default: {
     855         932 :     const GlobalValue *GV = MO.getGlobal();
     856             :     unsigned TF = MO.getTargetFlags();
     857         932 :     LO16 = LO16.addGlobalAddress(GV, MO.getOffset(), TF | ARMII::MO_LO16);
     858         932 :     HI16 = HI16.addGlobalAddress(GV, MO.getOffset(), TF | ARMII::MO_HI16);
     859             :     break;
     860             :   }
     861             :   }
     862             : 
     863        1455 :   LO16->setMemRefs(MI.memoperands_begin(), MI.memoperands_end());
     864        1455 :   HI16->setMemRefs(MI.memoperands_begin(), MI.memoperands_end());
     865        2910 :   LO16.addImm(Pred).addReg(PredReg);
     866        1455 :   HI16.addImm(Pred).addReg(PredReg);
     867             : 
     868        1455 :   if (RequiresBundling)
     869          57 :     finalizeBundle(MBB, LO16->getIterator(), MBBI->getIterator());
     870             : 
     871        1455 :   if (isCC)
     872          40 :     LO16.add(makeImplicit(MI.getOperand(1)));
     873        1455 :   TransferImpOps(MI, LO16, HI16);
     874        1455 :   MI.eraseFromParent();
     875             : }
     876             : 
     877             : /// Expand a CMP_SWAP pseudo-inst to an ldrex/strex loop as simply as
     878             : /// possible. This only gets used at -O0 so we don't care about efficiency of
     879             : /// the generated code.
     880           6 : bool ARMExpandPseudo::ExpandCMP_SWAP(MachineBasicBlock &MBB,
     881             :                                      MachineBasicBlock::iterator MBBI,
     882             :                                      unsigned LdrexOp, unsigned StrexOp,
     883             :                                      unsigned UxtOp,
     884             :                                      MachineBasicBlock::iterator &NextMBBI) {
     885           6 :   bool IsThumb = STI->isThumb();
     886             :   MachineInstr &MI = *MBBI;
     887             :   DebugLoc DL = MI.getDebugLoc();
     888           6 :   const MachineOperand &Dest = MI.getOperand(0);
     889           6 :   unsigned TempReg = MI.getOperand(1).getReg();
     890             :   // Duplicating undef operands into 2 instructions does not guarantee the same
     891             :   // value on both; However undef should be replaced by xzr anyway.
     892             :   assert(!MI.getOperand(2).isUndef() && "cannot handle undef");
     893           6 :   unsigned AddrReg = MI.getOperand(2).getReg();
     894           6 :   unsigned DesiredReg = MI.getOperand(3).getReg();
     895           6 :   unsigned NewReg = MI.getOperand(4).getReg();
     896             : 
     897           6 :   MachineFunction *MF = MBB.getParent();
     898           6 :   auto LoadCmpBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
     899           6 :   auto StoreBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
     900           6 :   auto DoneBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
     901             : 
     902           6 :   MF->insert(++MBB.getIterator(), LoadCmpBB);
     903           6 :   MF->insert(++LoadCmpBB->getIterator(), StoreBB);
     904           6 :   MF->insert(++StoreBB->getIterator(), DoneBB);
     905             : 
     906           6 :   if (UxtOp) {
     907             :     MachineInstrBuilder MIB =
     908          12 :         BuildMI(MBB, MBBI, DL, TII->get(UxtOp), DesiredReg)
     909           4 :             .addReg(DesiredReg, RegState::Kill);
     910           4 :     if (!IsThumb)
     911             :       MIB.addImm(0);
     912           4 :     MIB.add(predOps(ARMCC::AL));
     913             :   }
     914             : 
     915             :   // .Lloadcmp:
     916             :   //     ldrex rDest, [rAddr]
     917             :   //     cmp rDest, rDesired
     918             :   //     bne .Ldone
     919             : 
     920           6 :   MachineInstrBuilder MIB;
     921          12 :   MIB = BuildMI(LoadCmpBB, DL, TII->get(LdrexOp), Dest.getReg());
     922           6 :   MIB.addReg(AddrReg);
     923           6 :   if (LdrexOp == ARM::t2LDREX)
     924             :     MIB.addImm(0); // a 32-bit Thumb ldrex (only) allows an offset.
     925           6 :   MIB.add(predOps(ARMCC::AL));
     926             : 
     927           6 :   unsigned CMPrr = IsThumb ? ARM::tCMPhir : ARM::CMPrr;
     928          18 :   BuildMI(LoadCmpBB, DL, TII->get(CMPrr))
     929           6 :       .addReg(Dest.getReg(), getKillRegState(Dest.isDead()))
     930           6 :       .addReg(DesiredReg)
     931           6 :       .add(predOps(ARMCC::AL));
     932           6 :   unsigned Bcc = IsThumb ? ARM::tBcc : ARM::Bcc;
     933          18 :   BuildMI(LoadCmpBB, DL, TII->get(Bcc))
     934             :       .addMBB(DoneBB)
     935             :       .addImm(ARMCC::NE)
     936           6 :       .addReg(ARM::CPSR, RegState::Kill);
     937           6 :   LoadCmpBB->addSuccessor(DoneBB);
     938           6 :   LoadCmpBB->addSuccessor(StoreBB);
     939             : 
     940             :   // .Lstore:
     941             :   //     strex rTempReg, rNew, [rAddr]
     942             :   //     cmp rTempReg, #0
     943             :   //     bne .Lloadcmp
     944          18 :   MIB = BuildMI(StoreBB, DL, TII->get(StrexOp), TempReg)
     945           6 :     .addReg(NewReg)
     946           6 :     .addReg(AddrReg);
     947           6 :   if (StrexOp == ARM::t2STREX)
     948             :     MIB.addImm(0); // a 32-bit Thumb strex (only) allows an offset.
     949           6 :   MIB.add(predOps(ARMCC::AL));
     950             : 
     951           6 :   unsigned CMPri = IsThumb ? ARM::t2CMPri : ARM::CMPri;
     952          18 :   BuildMI(StoreBB, DL, TII->get(CMPri))
     953           6 :       .addReg(TempReg, RegState::Kill)
     954             :       .addImm(0)
     955           6 :       .add(predOps(ARMCC::AL));
     956          18 :   BuildMI(StoreBB, DL, TII->get(Bcc))
     957             :       .addMBB(LoadCmpBB)
     958             :       .addImm(ARMCC::NE)
     959           6 :       .addReg(ARM::CPSR, RegState::Kill);
     960           6 :   StoreBB->addSuccessor(LoadCmpBB);
     961           6 :   StoreBB->addSuccessor(DoneBB);
     962             : 
     963             :   DoneBB->splice(DoneBB->end(), &MBB, MI, MBB.end());
     964           6 :   DoneBB->transferSuccessors(&MBB);
     965             : 
     966           6 :   MBB.addSuccessor(LoadCmpBB);
     967             : 
     968           6 :   NextMBBI = MBB.end();
     969           6 :   MI.eraseFromParent();
     970             : 
     971             :   // Recompute livein lists.
     972             :   LivePhysRegs LiveRegs;
     973           6 :   computeAndAddLiveIns(LiveRegs, *DoneBB);
     974           6 :   computeAndAddLiveIns(LiveRegs, *StoreBB);
     975           6 :   computeAndAddLiveIns(LiveRegs, *LoadCmpBB);
     976             :   // Do an extra pass around the loop to get loop carried registers right.
     977           6 :   StoreBB->clearLiveIns();
     978           6 :   computeAndAddLiveIns(LiveRegs, *StoreBB);
     979           6 :   LoadCmpBB->clearLiveIns();
     980           6 :   computeAndAddLiveIns(LiveRegs, *LoadCmpBB);
     981             : 
     982           6 :   return true;
     983             : }
     984             : 
     985             : /// ARM's ldrexd/strexd take a consecutive register pair (represented as a
     986             : /// single GPRPair register), Thumb's take two separate registers so we need to
     987             : /// extract the subregs from the pair.
     988          14 : static void addExclusiveRegPair(MachineInstrBuilder &MIB, MachineOperand &Reg,
     989             :                                 unsigned Flags, bool IsThumb,
     990             :                                 const TargetRegisterInfo *TRI) {
     991          14 :   if (IsThumb) {
     992           6 :     unsigned RegLo = TRI->getSubReg(Reg.getReg(), ARM::gsub_0);
     993           6 :     unsigned RegHi = TRI->getSubReg(Reg.getReg(), ARM::gsub_1);
     994           6 :     MIB.addReg(RegLo, Flags | getKillRegState(Reg.isDead()));
     995           6 :     MIB.addReg(RegHi, Flags | getKillRegState(Reg.isDead()));
     996             :   } else
     997           8 :     MIB.addReg(Reg.getReg(), Flags | getKillRegState(Reg.isDead()));
     998          14 : }
     999             : 
    1000             : /// Expand a 64-bit CMP_SWAP to an ldrexd/strexd loop.
    1001           7 : bool ARMExpandPseudo::ExpandCMP_SWAP_64(MachineBasicBlock &MBB,
    1002             :                                         MachineBasicBlock::iterator MBBI,
    1003             :                                         MachineBasicBlock::iterator &NextMBBI) {
    1004           7 :   bool IsThumb = STI->isThumb();
    1005             :   MachineInstr &MI = *MBBI;
    1006             :   DebugLoc DL = MI.getDebugLoc();
    1007           7 :   MachineOperand &Dest = MI.getOperand(0);
    1008           7 :   unsigned TempReg = MI.getOperand(1).getReg();
    1009             :   // Duplicating undef operands into 2 instructions does not guarantee the same
    1010             :   // value on both; However undef should be replaced by xzr anyway.
    1011             :   assert(!MI.getOperand(2).isUndef() && "cannot handle undef");
    1012           7 :   unsigned AddrReg = MI.getOperand(2).getReg();
    1013           7 :   unsigned DesiredReg = MI.getOperand(3).getReg();
    1014           7 :   MachineOperand New = MI.getOperand(4);
    1015             :   New.setIsKill(false);
    1016             : 
    1017           7 :   unsigned DestLo = TRI->getSubReg(Dest.getReg(), ARM::gsub_0);
    1018           7 :   unsigned DestHi = TRI->getSubReg(Dest.getReg(), ARM::gsub_1);
    1019           7 :   unsigned DesiredLo = TRI->getSubReg(DesiredReg, ARM::gsub_0);
    1020           7 :   unsigned DesiredHi = TRI->getSubReg(DesiredReg, ARM::gsub_1);
    1021             : 
    1022           7 :   MachineFunction *MF = MBB.getParent();
    1023           7 :   auto LoadCmpBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
    1024           7 :   auto StoreBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
    1025           7 :   auto DoneBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
    1026             : 
    1027           7 :   MF->insert(++MBB.getIterator(), LoadCmpBB);
    1028           7 :   MF->insert(++LoadCmpBB->getIterator(), StoreBB);
    1029           7 :   MF->insert(++StoreBB->getIterator(), DoneBB);
    1030             : 
    1031             :   // .Lloadcmp:
    1032             :   //     ldrexd rDestLo, rDestHi, [rAddr]
    1033             :   //     cmp rDestLo, rDesiredLo
    1034             :   //     sbcs dead rTempReg, rDestHi, rDesiredHi
    1035             :   //     bne .Ldone
    1036           7 :   unsigned LDREXD = IsThumb ? ARM::t2LDREXD : ARM::LDREXD;
    1037           7 :   MachineInstrBuilder MIB;
    1038          14 :   MIB = BuildMI(LoadCmpBB, DL, TII->get(LDREXD));
    1039           7 :   addExclusiveRegPair(MIB, Dest, RegState::Define, IsThumb, TRI);
    1040          14 :   MIB.addReg(AddrReg).add(predOps(ARMCC::AL));
    1041             : 
    1042           7 :   unsigned CMPrr = IsThumb ? ARM::tCMPhir : ARM::CMPrr;
    1043          21 :   BuildMI(LoadCmpBB, DL, TII->get(CMPrr))
    1044           7 :       .addReg(DestLo, getKillRegState(Dest.isDead()))
    1045           7 :       .addReg(DesiredLo)
    1046           7 :       .add(predOps(ARMCC::AL));
    1047             : 
    1048          21 :   BuildMI(LoadCmpBB, DL, TII->get(CMPrr))
    1049           7 :       .addReg(DestHi, getKillRegState(Dest.isDead()))
    1050           7 :       .addReg(DesiredHi)
    1051           7 :       .addImm(ARMCC::EQ).addReg(ARM::CPSR, RegState::Kill);
    1052             : 
    1053           7 :   unsigned Bcc = IsThumb ? ARM::tBcc : ARM::Bcc;
    1054          21 :   BuildMI(LoadCmpBB, DL, TII->get(Bcc))
    1055             :       .addMBB(DoneBB)
    1056             :       .addImm(ARMCC::NE)
    1057           7 :       .addReg(ARM::CPSR, RegState::Kill);
    1058           7 :   LoadCmpBB->addSuccessor(DoneBB);
    1059           7 :   LoadCmpBB->addSuccessor(StoreBB);
    1060             : 
    1061             :   // .Lstore:
    1062             :   //     strexd rTempReg, rNewLo, rNewHi, [rAddr]
    1063             :   //     cmp rTempReg, #0
    1064             :   //     bne .Lloadcmp
    1065           7 :   unsigned STREXD = IsThumb ? ARM::t2STREXD : ARM::STREXD;
    1066          14 :   MIB = BuildMI(StoreBB, DL, TII->get(STREXD), TempReg);
    1067           7 :   addExclusiveRegPair(MIB, New, 0, IsThumb, TRI);
    1068          14 :   MIB.addReg(AddrReg).add(predOps(ARMCC::AL));
    1069             : 
    1070           7 :   unsigned CMPri = IsThumb ? ARM::t2CMPri : ARM::CMPri;
    1071          21 :   BuildMI(StoreBB, DL, TII->get(CMPri))
    1072           7 :       .addReg(TempReg, RegState::Kill)
    1073             :       .addImm(0)
    1074           7 :       .add(predOps(ARMCC::AL));
    1075          21 :   BuildMI(StoreBB, DL, TII->get(Bcc))
    1076             :       .addMBB(LoadCmpBB)
    1077             :       .addImm(ARMCC::NE)
    1078           7 :       .addReg(ARM::CPSR, RegState::Kill);
    1079           7 :   StoreBB->addSuccessor(LoadCmpBB);
    1080           7 :   StoreBB->addSuccessor(DoneBB);
    1081             : 
    1082             :   DoneBB->splice(DoneBB->end(), &MBB, MI, MBB.end());
    1083           7 :   DoneBB->transferSuccessors(&MBB);
    1084             : 
    1085           7 :   MBB.addSuccessor(LoadCmpBB);
    1086             : 
    1087           7 :   NextMBBI = MBB.end();
    1088           7 :   MI.eraseFromParent();
    1089             : 
    1090             :   // Recompute livein lists.
    1091             :   LivePhysRegs LiveRegs;
    1092           7 :   computeAndAddLiveIns(LiveRegs, *DoneBB);
    1093           7 :   computeAndAddLiveIns(LiveRegs, *StoreBB);
    1094           7 :   computeAndAddLiveIns(LiveRegs, *LoadCmpBB);
    1095             :   // Do an extra pass around the loop to get loop carried registers right.
    1096           7 :   StoreBB->clearLiveIns();
    1097           7 :   computeAndAddLiveIns(LiveRegs, *StoreBB);
    1098           7 :   LoadCmpBB->clearLiveIns();
    1099           7 :   computeAndAddLiveIns(LiveRegs, *LoadCmpBB);
    1100             : 
    1101           7 :   return true;
    1102             : }
    1103             : 
    1104             : 
    1105      156139 : bool ARMExpandPseudo::ExpandMI(MachineBasicBlock &MBB,
    1106             :                                MachineBasicBlock::iterator MBBI,
    1107             :                                MachineBasicBlock::iterator &NextMBBI) {
    1108             :   MachineInstr &MI = *MBBI;
    1109      156139 :   unsigned Opcode = MI.getOpcode();
    1110      156139 :   switch (Opcode) {
    1111             :     default:
    1112             :       return false;
    1113             : 
    1114         555 :     case ARM::TCRETURNdi:
    1115             :     case ARM::TCRETURNri: {
    1116         555 :       MachineBasicBlock::iterator MBBI = MBB.getLastNonDebugInstr();
    1117             :       assert(MBBI->isReturn() &&
    1118             :              "Can only insert epilog into returning blocks");
    1119         555 :       unsigned RetOpcode = MBBI->getOpcode();
    1120             :       DebugLoc dl = MBBI->getDebugLoc();
    1121             :       const ARMBaseInstrInfo &TII = *static_cast<const ARMBaseInstrInfo *>(
    1122         555 :           MBB.getParent()->getSubtarget().getInstrInfo());
    1123             : 
    1124             :       // Tail call return: adjust the stack pointer and jump to callee.
    1125         555 :       MBBI = MBB.getLastNonDebugInstr();
    1126         555 :       MachineOperand &JumpTarget = MBBI->getOperand(0);
    1127             : 
    1128             :       // Jump to label or value in register.
    1129         555 :       if (RetOpcode == ARM::TCRETURNdi) {
    1130             :         unsigned TCOpcode =
    1131         536 :             STI->isThumb()
    1132         888 :                 ? (STI->isTargetMachO() ? ARM::tTAILJMPd : ARM::tTAILJMPdND)
    1133             :                 : ARM::TAILJMPd;
    1134        1072 :         MachineInstrBuilder MIB = BuildMI(MBB, MBBI, dl, TII.get(TCOpcode));
    1135         536 :         if (JumpTarget.isGlobal())
    1136             :           MIB.addGlobalAddress(JumpTarget.getGlobal(), JumpTarget.getOffset(),
    1137         638 :                                JumpTarget.getTargetFlags());
    1138             :         else {
    1139             :           assert(JumpTarget.isSymbol());
    1140             :           MIB.addExternalSymbol(JumpTarget.getSymbolName(),
    1141         217 :                                 JumpTarget.getTargetFlags());
    1142             :         }
    1143             : 
    1144             :         // Add the default predicate in Thumb mode.
    1145         536 :         if (STI->isThumb())
    1146         352 :           MIB.add(predOps(ARMCC::AL));
    1147          19 :       } else if (RetOpcode == ARM::TCRETURNri) {
    1148             :         unsigned Opcode =
    1149          28 :           STI->isThumb() ? ARM::tTAILJMPr
    1150           9 :                          : (STI->hasV4TOps() ? ARM::TAILJMPr : ARM::TAILJMPr4);
    1151          38 :         BuildMI(MBB, MBBI, dl,
    1152          19 :                 TII.get(Opcode))
    1153          19 :             .addReg(JumpTarget.getReg(), RegState::Kill);
    1154             :       }
    1155             : 
    1156             :       auto NewMI = std::prev(MBBI);
    1157        1703 :       for (unsigned i = 1, e = MBBI->getNumOperands(); i != e; ++i)
    1158        2296 :         NewMI->addOperand(MBBI->getOperand(i));
    1159             : 
    1160             :       // Delete the pseudo instruction TCRETURN.
    1161         555 :       MBB.erase(MBBI);
    1162             :       MBBI = NewMI;
    1163             :       return true;
    1164             :     }
    1165         136 :     case ARM::VMOVScc:
    1166             :     case ARM::VMOVDcc: {
    1167         136 :       unsigned newOpc = Opcode == ARM::VMOVScc ? ARM::VMOVS : ARM::VMOVD;
    1168         272 :       BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(newOpc),
    1169         272 :               MI.getOperand(1).getReg())
    1170         136 :           .add(MI.getOperand(2))
    1171         136 :           .addImm(MI.getOperand(3).getImm()) // 'pred'
    1172         136 :           .add(MI.getOperand(4))
    1173         272 :           .add(makeImplicit(MI.getOperand(1)));
    1174             : 
    1175         136 :       MI.eraseFromParent();
    1176         136 :       return true;
    1177             :     }
    1178         267 :     case ARM::t2MOVCCr:
    1179             :     case ARM::MOVCCr: {
    1180         267 :       unsigned Opc = AFI->isThumbFunction() ? ARM::t2MOVr : ARM::MOVr;
    1181         534 :       BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(Opc),
    1182         534 :               MI.getOperand(1).getReg())
    1183         267 :           .add(MI.getOperand(2))
    1184         267 :           .addImm(MI.getOperand(3).getImm()) // 'pred'
    1185         267 :           .add(MI.getOperand(4))
    1186         267 :           .add(condCodeOp()) // 's' bit
    1187         534 :           .add(makeImplicit(MI.getOperand(1)));
    1188             : 
    1189         267 :       MI.eraseFromParent();
    1190         267 :       return true;
    1191             :     }
    1192           7 :     case ARM::MOVCCsi: {
    1193          14 :       BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(ARM::MOVsi),
    1194          14 :               (MI.getOperand(1).getReg()))
    1195           7 :           .add(MI.getOperand(2))
    1196           7 :           .addImm(MI.getOperand(3).getImm())
    1197           7 :           .addImm(MI.getOperand(4).getImm()) // 'pred'
    1198           7 :           .add(MI.getOperand(5))
    1199           7 :           .add(condCodeOp()) // 's' bit
    1200          14 :           .add(makeImplicit(MI.getOperand(1)));
    1201             : 
    1202           7 :       MI.eraseFromParent();
    1203           7 :       return true;
    1204             :     }
    1205           7 :     case ARM::MOVCCsr: {
    1206          14 :       BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(ARM::MOVsr),
    1207          14 :               (MI.getOperand(1).getReg()))
    1208           7 :           .add(MI.getOperand(2))
    1209           7 :           .add(MI.getOperand(3))
    1210           7 :           .addImm(MI.getOperand(4).getImm())
    1211           7 :           .addImm(MI.getOperand(5).getImm()) // 'pred'
    1212           7 :           .add(MI.getOperand(6))
    1213           7 :           .add(condCodeOp()) // 's' bit
    1214          14 :           .add(makeImplicit(MI.getOperand(1)));
    1215             : 
    1216           7 :       MI.eraseFromParent();
    1217           7 :       return true;
    1218             :     }
    1219          72 :     case ARM::t2MOVCCi16:
    1220             :     case ARM::MOVCCi16: {
    1221          72 :       unsigned NewOpc = AFI->isThumbFunction() ? ARM::t2MOVi16 : ARM::MOVi16;
    1222         144 :       BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(NewOpc),
    1223         144 :               MI.getOperand(1).getReg())
    1224          72 :           .addImm(MI.getOperand(2).getImm())
    1225          72 :           .addImm(MI.getOperand(3).getImm()) // 'pred'
    1226          72 :           .add(MI.getOperand(4))
    1227         144 :           .add(makeImplicit(MI.getOperand(1)));
    1228          72 :       MI.eraseFromParent();
    1229          72 :       return true;
    1230             :     }
    1231         608 :     case ARM::t2MOVCCi:
    1232             :     case ARM::MOVCCi: {
    1233         608 :       unsigned Opc = AFI->isThumbFunction() ? ARM::t2MOVi : ARM::MOVi;
    1234        1216 :       BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(Opc),
    1235        1216 :               MI.getOperand(1).getReg())
    1236         608 :           .addImm(MI.getOperand(2).getImm())
    1237         608 :           .addImm(MI.getOperand(3).getImm()) // 'pred'
    1238         608 :           .add(MI.getOperand(4))
    1239         608 :           .add(condCodeOp()) // 's' bit
    1240        1216 :           .add(makeImplicit(MI.getOperand(1)));
    1241             : 
    1242         608 :       MI.eraseFromParent();
    1243         608 :       return true;
    1244             :     }
    1245          56 :     case ARM::t2MVNCCi:
    1246             :     case ARM::MVNCCi: {
    1247          56 :       unsigned Opc = AFI->isThumbFunction() ? ARM::t2MVNi : ARM::MVNi;
    1248         112 :       BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(Opc),
    1249         112 :               MI.getOperand(1).getReg())
    1250          56 :           .addImm(MI.getOperand(2).getImm())
    1251          56 :           .addImm(MI.getOperand(3).getImm()) // 'pred'
    1252          56 :           .add(MI.getOperand(4))
    1253          56 :           .add(condCodeOp()) // 's' bit
    1254         112 :           .add(makeImplicit(MI.getOperand(1)));
    1255             : 
    1256          56 :       MI.eraseFromParent();
    1257          56 :       return true;
    1258             :     }
    1259           3 :     case ARM::t2MOVCClsl:
    1260             :     case ARM::t2MOVCClsr:
    1261             :     case ARM::t2MOVCCasr:
    1262             :     case ARM::t2MOVCCror: {
    1263             :       unsigned NewOpc;
    1264           3 :       switch (Opcode) {
    1265             :       case ARM::t2MOVCClsl: NewOpc = ARM::t2LSLri; break;
    1266           1 :       case ARM::t2MOVCClsr: NewOpc = ARM::t2LSRri; break;
    1267           0 :       case ARM::t2MOVCCasr: NewOpc = ARM::t2ASRri; break;
    1268           1 :       case ARM::t2MOVCCror: NewOpc = ARM::t2RORri; break;
    1269           0 :       default: llvm_unreachable("unexpeced conditional move");
    1270             :       }
    1271           6 :       BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(NewOpc),
    1272           6 :               MI.getOperand(1).getReg())
    1273           3 :           .add(MI.getOperand(2))
    1274           3 :           .addImm(MI.getOperand(3).getImm())
    1275           3 :           .addImm(MI.getOperand(4).getImm()) // 'pred'
    1276           3 :           .add(MI.getOperand(5))
    1277           3 :           .add(condCodeOp()) // 's' bit
    1278           6 :           .add(makeImplicit(MI.getOperand(1)));
    1279           3 :       MI.eraseFromParent();
    1280           3 :       return true;
    1281             :     }
    1282          32 :     case ARM::Int_eh_sjlj_dispatchsetup: {
    1283          32 :       MachineFunction &MF = *MI.getParent()->getParent();
    1284          32 :       const ARMBaseInstrInfo *AII =
    1285             :         static_cast<const ARMBaseInstrInfo*>(TII);
    1286          32 :       const ARMBaseRegisterInfo &RI = AII->getRegisterInfo();
    1287             :       // For functions using a base pointer, we rematerialize it (via the frame
    1288             :       // pointer) here since eh.sjlj.setjmp and eh.sjlj.longjmp don't do it
    1289             :       // for us. Otherwise, expand to nothing.
    1290          32 :       if (RI.hasBasePointer(MF)) {
    1291           0 :         int32_t NumBytes = AFI->getFramePtrSpillOffset();
    1292           0 :         unsigned FramePtr = RI.getFrameRegister(MF);
    1293             :         assert(MF.getSubtarget().getFrameLowering()->hasFP(MF) &&
    1294             :                "base pointer without frame pointer?");
    1295             : 
    1296           0 :         if (AFI->isThumb2Function()) {
    1297           0 :           emitT2RegPlusImmediate(MBB, MBBI, MI.getDebugLoc(), ARM::R6,
    1298           0 :                                  FramePtr, -NumBytes, ARMCC::AL, 0, *TII);
    1299           0 :         } else if (AFI->isThumbFunction()) {
    1300           0 :           emitThumbRegPlusImmediate(MBB, MBBI, MI.getDebugLoc(), ARM::R6,
    1301           0 :                                     FramePtr, -NumBytes, *TII, RI);
    1302             :         } else {
    1303           0 :           emitARMRegPlusImmediate(MBB, MBBI, MI.getDebugLoc(), ARM::R6,
    1304             :                                   FramePtr, -NumBytes, ARMCC::AL, 0,
    1305           0 :                                   *TII);
    1306             :         }
    1307             :         // If there's dynamic realignment, adjust for it.
    1308           0 :         if (RI.needsStackRealignment(MF)) {
    1309           0 :           MachineFrameInfo &MFI = MF.getFrameInfo();
    1310           0 :           unsigned MaxAlign = MFI.getMaxAlignment();
    1311             :           assert (!AFI->isThumb1OnlyFunction());
    1312             :           // Emit bic r6, r6, MaxAlign
    1313             :           assert(MaxAlign <= 256 && "The BIC instruction cannot encode "
    1314             :                                     "immediates larger than 256 with all lower "
    1315             :                                     "bits set.");
    1316           0 :           unsigned bicOpc = AFI->isThumbFunction() ?
    1317             :             ARM::t2BICri : ARM::BICri;
    1318           0 :           BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(bicOpc), ARM::R6)
    1319           0 :               .addReg(ARM::R6, RegState::Kill)
    1320           0 :               .addImm(MaxAlign - 1)
    1321           0 :               .add(predOps(ARMCC::AL))
    1322           0 :               .add(condCodeOp());
    1323             :         }
    1324             : 
    1325             :       }
    1326          32 :       MI.eraseFromParent();
    1327          32 :       return true;
    1328             :     }
    1329             : 
    1330           2 :     case ARM::MOVsrl_flag:
    1331             :     case ARM::MOVsra_flag: {
    1332             :       // These are just fancy MOVs instructions.
    1333           6 :       BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(ARM::MOVsi),
    1334           4 :               MI.getOperand(0).getReg())
    1335           2 :           .add(MI.getOperand(1))
    1336           2 :           .addImm(ARM_AM::getSORegOpc(
    1337           2 :               (Opcode == ARM::MOVsrl_flag ? ARM_AM::lsr : ARM_AM::asr), 1))
    1338           2 :           .add(predOps(ARMCC::AL))
    1339           2 :           .addReg(ARM::CPSR, RegState::Define);
    1340           2 :       MI.eraseFromParent();
    1341           2 :       return true;
    1342             :     }
    1343           2 :     case ARM::RRX: {
    1344             :       // This encodes as "MOVs Rd, Rm, rrx
    1345             :       MachineInstrBuilder MIB =
    1346           4 :           BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(ARM::MOVsi),
    1347           4 :                   MI.getOperand(0).getReg())
    1348           2 :               .add(MI.getOperand(1))
    1349             :               .addImm(ARM_AM::getSORegOpc(ARM_AM::rrx, 0))
    1350           2 :               .add(predOps(ARMCC::AL))
    1351           2 :               .add(condCodeOp());
    1352           2 :       TransferImpOps(MI, MIB, MIB);
    1353           2 :       MI.eraseFromParent();
    1354             :       return true;
    1355             :     }
    1356          39 :     case ARM::tTPsoft:
    1357             :     case ARM::TPsoft: {
    1358             :       const bool Thumb = Opcode == ARM::tTPsoft;
    1359             : 
    1360          39 :       MachineInstrBuilder MIB;
    1361          39 :       if (STI->genLongCalls()) {
    1362           2 :         MachineFunction *MF = MBB.getParent();
    1363           2 :         MachineConstantPool *MCP = MF->getConstantPool();
    1364           2 :         unsigned PCLabelID = AFI->createPICLabelUId();
    1365             :         MachineConstantPoolValue *CPV =
    1366           4 :             ARMConstantPoolSymbol::Create(MF->getFunction().getContext(),
    1367           2 :                                           "__aeabi_read_tp", PCLabelID, 0);
    1368           2 :         unsigned Reg = MI.getOperand(0).getReg();
    1369           4 :         MIB = BuildMI(MBB, MBBI, MI.getDebugLoc(),
    1370           2 :                       TII->get(Thumb ? ARM::tLDRpci : ARM::LDRi12), Reg)
    1371           2 :                   .addConstantPoolIndex(MCP->getConstantPoolIndex(CPV, 4));
    1372           2 :         if (!Thumb)
    1373             :           MIB.addImm(0);
    1374           2 :         MIB.add(predOps(ARMCC::AL));
    1375             : 
    1376           2 :         MIB = BuildMI(MBB, MBBI, MI.getDebugLoc(),
    1377           2 :                       TII->get(Thumb ? ARM::tBLXr : ARM::BLX));
    1378           2 :         if (Thumb)
    1379           1 :           MIB.add(predOps(ARMCC::AL));
    1380           2 :         MIB.addReg(Reg, RegState::Kill);
    1381             :       } else {
    1382          37 :         MIB = BuildMI(MBB, MBBI, MI.getDebugLoc(),
    1383          37 :                       TII->get(Thumb ? ARM::tBL : ARM::BL));
    1384          37 :         if (Thumb)
    1385          11 :           MIB.add(predOps(ARMCC::AL));
    1386             :         MIB.addExternalSymbol("__aeabi_read_tp", 0);
    1387             :       }
    1388             : 
    1389          39 :       MIB->setMemRefs(MI.memoperands_begin(), MI.memoperands_end());
    1390          39 :       TransferImpOps(MI, MIB, MIB);
    1391          39 :       MI.eraseFromParent();
    1392             :       return true;
    1393             :     }
    1394          45 :     case ARM::tLDRpci_pic:
    1395             :     case ARM::t2LDRpci_pic: {
    1396             :       unsigned NewLdOpc = (Opcode == ARM::tLDRpci_pic)
    1397          45 :         ? ARM::tLDRpci : ARM::t2LDRpci;
    1398          45 :       unsigned DstReg = MI.getOperand(0).getReg();
    1399             :       bool DstIsDead = MI.getOperand(0).isDead();
    1400             :       MachineInstrBuilder MIB1 =
    1401          90 :           BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(NewLdOpc), DstReg)
    1402          45 :               .add(MI.getOperand(1))
    1403          45 :               .add(predOps(ARMCC::AL));
    1404          45 :       MIB1->setMemRefs(MI.memoperands_begin(), MI.memoperands_end());
    1405             :       MachineInstrBuilder MIB2 =
    1406         135 :           BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(ARM::tPICADD))
    1407          45 :               .addReg(DstReg, RegState::Define | getDeadRegState(DstIsDead))
    1408          45 :               .addReg(DstReg)
    1409          90 :               .add(MI.getOperand(2));
    1410          45 :       TransferImpOps(MI, MIB1, MIB2);
    1411          45 :       MI.eraseFromParent();
    1412             :       return true;
    1413             :     }
    1414             : 
    1415         357 :     case ARM::LDRLIT_ga_abs:
    1416             :     case ARM::LDRLIT_ga_pcrel:
    1417             :     case ARM::LDRLIT_ga_pcrel_ldr:
    1418             :     case ARM::tLDRLIT_ga_abs:
    1419             :     case ARM::tLDRLIT_ga_pcrel: {
    1420         357 :       unsigned DstReg = MI.getOperand(0).getReg();
    1421             :       bool DstIsDead = MI.getOperand(0).isDead();
    1422             :       const MachineOperand &MO1 = MI.getOperand(1);
    1423             :       auto Flags = MO1.getTargetFlags();
    1424         357 :       const GlobalValue *GV = MO1.getGlobal();
    1425             :       bool IsARM =
    1426         357 :           Opcode != ARM::tLDRLIT_ga_pcrel && Opcode != ARM::tLDRLIT_ga_abs;
    1427         357 :       bool IsPIC =
    1428         357 :           Opcode != ARM::LDRLIT_ga_abs && Opcode != ARM::tLDRLIT_ga_abs;
    1429         357 :       unsigned LDRLITOpc = IsARM ? ARM::LDRi12 : ARM::tLDRpci;
    1430             :       unsigned PICAddOpc =
    1431             :           IsARM
    1432         357 :               ? (Opcode == ARM::LDRLIT_ga_pcrel_ldr ? ARM::PICLDR : ARM::PICADD)
    1433             :               : ARM::tPICADD;
    1434             : 
    1435             :       // We need a new const-pool entry to load from.
    1436         357 :       MachineConstantPool *MCP = MBB.getParent()->getConstantPool();
    1437             :       unsigned ARMPCLabelIndex = 0;
    1438             :       MachineConstantPoolValue *CPV;
    1439             : 
    1440         357 :       if (IsPIC) {
    1441         320 :         unsigned PCAdj = IsARM ? 8 : 4;
    1442         320 :         auto Modifier = (Flags & ARMII::MO_GOT)
    1443         320 :                             ? ARMCP::GOT_PREL
    1444             :                             : ARMCP::no_modifier;
    1445         320 :         ARMPCLabelIndex = AFI->createPICLabelUId();
    1446         320 :         CPV = ARMConstantPoolConstant::Create(
    1447             :             GV, ARMPCLabelIndex, ARMCP::CPValue, PCAdj, Modifier,
    1448             :             /*AddCurrentAddr*/ Modifier == ARMCP::GOT_PREL);
    1449             :       } else
    1450          37 :         CPV = ARMConstantPoolConstant::Create(GV, ARMCP::no_modifier);
    1451             : 
    1452             :       MachineInstrBuilder MIB =
    1453         714 :           BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(LDRLITOpc), DstReg)
    1454         357 :             .addConstantPoolIndex(MCP->getConstantPoolIndex(CPV, 4));
    1455         357 :       if (IsARM)
    1456             :         MIB.addImm(0);
    1457         357 :       MIB.add(predOps(ARMCC::AL));
    1458             : 
    1459         357 :       if (IsPIC) {
    1460             :         MachineInstrBuilder MIB =
    1461         960 :           BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(PICAddOpc))
    1462         320 :             .addReg(DstReg, RegState::Define | getDeadRegState(DstIsDead))
    1463         320 :             .addReg(DstReg)
    1464         640 :             .addImm(ARMPCLabelIndex);
    1465             : 
    1466         320 :         if (IsARM)
    1467         174 :           MIB.add(predOps(ARMCC::AL));
    1468             :       }
    1469             : 
    1470         357 :       MI.eraseFromParent();
    1471             :       return true;
    1472             :     }
    1473         624 :     case ARM::MOV_ga_pcrel:
    1474             :     case ARM::MOV_ga_pcrel_ldr:
    1475             :     case ARM::t2MOV_ga_pcrel: {
    1476             :       // Expand into movw + movw. Also "add pc" / ldr [pc] in PIC mode.
    1477         624 :       unsigned LabelId = AFI->createPICLabelUId();
    1478         624 :       unsigned DstReg = MI.getOperand(0).getReg();
    1479             :       bool DstIsDead = MI.getOperand(0).isDead();
    1480             :       const MachineOperand &MO1 = MI.getOperand(1);
    1481         624 :       const GlobalValue *GV = MO1.getGlobal();
    1482             :       unsigned TF = MO1.getTargetFlags();
    1483             :       bool isARM = Opcode != ARM::t2MOV_ga_pcrel;
    1484         624 :       unsigned LO16Opc = isARM ? ARM::MOVi16_ga_pcrel : ARM::t2MOVi16_ga_pcrel;
    1485         624 :       unsigned HI16Opc = isARM ? ARM::MOVTi16_ga_pcrel :ARM::t2MOVTi16_ga_pcrel;
    1486             :       unsigned LO16TF = TF | ARMII::MO_LO16;
    1487             :       unsigned HI16TF = TF | ARMII::MO_HI16;
    1488             :       unsigned PICAddOpc = isARM
    1489         624 :         ? (Opcode == ARM::MOV_ga_pcrel_ldr ? ARM::PICLDR : ARM::PICADD)
    1490             :         : ARM::tPICADD;
    1491         624 :       MachineInstrBuilder MIB1 = BuildMI(MBB, MBBI, MI.getDebugLoc(),
    1492         624 :                                          TII->get(LO16Opc), DstReg)
    1493         624 :         .addGlobalAddress(GV, MO1.getOffset(), TF | LO16TF)
    1494        1248 :         .addImm(LabelId);
    1495             : 
    1496        1872 :       BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(HI16Opc), DstReg)
    1497         624 :         .addReg(DstReg)
    1498         624 :         .addGlobalAddress(GV, MO1.getOffset(), TF | HI16TF)
    1499             :         .addImm(LabelId);
    1500             : 
    1501        1248 :       MachineInstrBuilder MIB3 = BuildMI(MBB, MBBI, MI.getDebugLoc(),
    1502         624 :                                          TII->get(PICAddOpc))
    1503         624 :         .addReg(DstReg, RegState::Define | getDeadRegState(DstIsDead))
    1504        1248 :         .addReg(DstReg).addImm(LabelId);
    1505         624 :       if (isARM) {
    1506         213 :         MIB3.add(predOps(ARMCC::AL));
    1507         213 :         if (Opcode == ARM::MOV_ga_pcrel_ldr)
    1508         128 :           MIB3->setMemRefs(MI.memoperands_begin(), MI.memoperands_end());
    1509             :       }
    1510         624 :       TransferImpOps(MI, MIB1, MIB3);
    1511         624 :       MI.eraseFromParent();
    1512             :       return true;
    1513             :     }
    1514             : 
    1515        1630 :     case ARM::MOVi32imm:
    1516             :     case ARM::MOVCCi32imm:
    1517             :     case ARM::t2MOVi32imm:
    1518             :     case ARM::t2MOVCCi32imm:
    1519        1630 :       ExpandMOV32BitImm(MBB, MBBI);
    1520        1630 :       return true;
    1521             : 
    1522           6 :     case ARM::SUBS_PC_LR: {
    1523             :       MachineInstrBuilder MIB =
    1524          18 :           BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(ARM::SUBri), ARM::PC)
    1525           6 :               .addReg(ARM::LR)
    1526           6 :               .add(MI.getOperand(0))
    1527           6 :               .add(MI.getOperand(1))
    1528           6 :               .add(MI.getOperand(2))
    1529           6 :               .addReg(ARM::CPSR, RegState::Undef);
    1530           6 :       TransferImpOps(MI, MIB, MIB);
    1531           6 :       MI.eraseFromParent();
    1532             :       return true;
    1533             :     }
    1534           3 :     case ARM::VLDMQIA: {
    1535             :       unsigned NewOpc = ARM::VLDMDIA;
    1536             :       MachineInstrBuilder MIB =
    1537           6 :         BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(NewOpc));
    1538             :       unsigned OpIdx = 0;
    1539             : 
    1540             :       // Grab the Q register destination.
    1541           3 :       bool DstIsDead = MI.getOperand(OpIdx).isDead();
    1542           3 :       unsigned DstReg = MI.getOperand(OpIdx++).getReg();
    1543             : 
    1544             :       // Copy the source register.
    1545             :       MIB.add(MI.getOperand(OpIdx++));
    1546             : 
    1547             :       // Copy the predicate operands.
    1548           3 :       MIB.add(MI.getOperand(OpIdx++));
    1549           3 :       MIB.add(MI.getOperand(OpIdx++));
    1550             : 
    1551             :       // Add the destination operands (D subregs).
    1552           3 :       unsigned D0 = TRI->getSubReg(DstReg, ARM::dsub_0);
    1553           3 :       unsigned D1 = TRI->getSubReg(DstReg, ARM::dsub_1);
    1554           3 :       MIB.addReg(D0, RegState::Define | getDeadRegState(DstIsDead))
    1555           3 :         .addReg(D1, RegState::Define | getDeadRegState(DstIsDead));
    1556             : 
    1557             :       // Add an implicit def for the super-register.
    1558           3 :       MIB.addReg(DstReg, RegState::ImplicitDefine | getDeadRegState(DstIsDead));
    1559           3 :       TransferImpOps(MI, MIB, MIB);
    1560           3 :       MIB.setMemRefs(MI.memoperands_begin(), MI.memoperands_end());
    1561           3 :       MI.eraseFromParent();
    1562             :       return true;
    1563             :     }
    1564             : 
    1565           8 :     case ARM::VSTMQIA: {
    1566             :       unsigned NewOpc = ARM::VSTMDIA;
    1567             :       MachineInstrBuilder MIB =
    1568          16 :         BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(NewOpc));
    1569             :       unsigned OpIdx = 0;
    1570             : 
    1571             :       // Grab the Q register source.
    1572           8 :       bool SrcIsKill = MI.getOperand(OpIdx).isKill();
    1573           8 :       unsigned SrcReg = MI.getOperand(OpIdx++).getReg();
    1574             : 
    1575             :       // Copy the destination register.
    1576           8 :       MachineOperand Dst(MI.getOperand(OpIdx++));
    1577             :       MIB.add(Dst);
    1578             : 
    1579             :       // Copy the predicate operands.
    1580           8 :       MIB.add(MI.getOperand(OpIdx++));
    1581           8 :       MIB.add(MI.getOperand(OpIdx++));
    1582             : 
    1583             :       // Add the source operands (D subregs).
    1584           8 :       unsigned D0 = TRI->getSubReg(SrcReg, ARM::dsub_0);
    1585           8 :       unsigned D1 = TRI->getSubReg(SrcReg, ARM::dsub_1);
    1586           8 :       MIB.addReg(D0, SrcIsKill ? RegState::Kill : 0)
    1587           8 :          .addReg(D1, SrcIsKill ? RegState::Kill : 0);
    1588             : 
    1589           8 :       if (SrcIsKill)      // Add an implicit kill for the Q register.
    1590           7 :         MIB->addRegisterKilled(SrcReg, TRI, true);
    1591             : 
    1592           8 :       TransferImpOps(MI, MIB, MIB);
    1593           8 :       MIB.setMemRefs(MI.memoperands_begin(), MI.memoperands_end());
    1594           8 :       MI.eraseFromParent();
    1595             :       return true;
    1596             :     }
    1597             : 
    1598         124 :     case ARM::VLD2q8Pseudo:
    1599             :     case ARM::VLD2q16Pseudo:
    1600             :     case ARM::VLD2q32Pseudo:
    1601             :     case ARM::VLD2q8PseudoWB_fixed:
    1602             :     case ARM::VLD2q16PseudoWB_fixed:
    1603             :     case ARM::VLD2q32PseudoWB_fixed:
    1604             :     case ARM::VLD2q8PseudoWB_register:
    1605             :     case ARM::VLD2q16PseudoWB_register:
    1606             :     case ARM::VLD2q32PseudoWB_register:
    1607             :     case ARM::VLD3d8Pseudo:
    1608             :     case ARM::VLD3d16Pseudo:
    1609             :     case ARM::VLD3d32Pseudo:
    1610             :     case ARM::VLD1d8TPseudo:
    1611             :     case ARM::VLD1d16TPseudo:
    1612             :     case ARM::VLD1d32TPseudo:
    1613             :     case ARM::VLD1d64TPseudo:
    1614             :     case ARM::VLD1d64TPseudoWB_fixed:
    1615             :     case ARM::VLD1d64TPseudoWB_register:
    1616             :     case ARM::VLD3d8Pseudo_UPD:
    1617             :     case ARM::VLD3d16Pseudo_UPD:
    1618             :     case ARM::VLD3d32Pseudo_UPD:
    1619             :     case ARM::VLD3q8Pseudo_UPD:
    1620             :     case ARM::VLD3q16Pseudo_UPD:
    1621             :     case ARM::VLD3q32Pseudo_UPD:
    1622             :     case ARM::VLD3q8oddPseudo:
    1623             :     case ARM::VLD3q16oddPseudo:
    1624             :     case ARM::VLD3q32oddPseudo:
    1625             :     case ARM::VLD3q8oddPseudo_UPD:
    1626             :     case ARM::VLD3q16oddPseudo_UPD:
    1627             :     case ARM::VLD3q32oddPseudo_UPD:
    1628             :     case ARM::VLD4d8Pseudo:
    1629             :     case ARM::VLD4d16Pseudo:
    1630             :     case ARM::VLD4d32Pseudo:
    1631             :     case ARM::VLD1d8QPseudo:
    1632             :     case ARM::VLD1d16QPseudo:
    1633             :     case ARM::VLD1d32QPseudo:
    1634             :     case ARM::VLD1d64QPseudo:
    1635             :     case ARM::VLD1d64QPseudoWB_fixed:
    1636             :     case ARM::VLD1d64QPseudoWB_register:
    1637             :     case ARM::VLD1q8HighQPseudo:
    1638             :     case ARM::VLD1q8LowQPseudo_UPD:
    1639             :     case ARM::VLD1q8HighTPseudo:
    1640             :     case ARM::VLD1q8LowTPseudo_UPD:
    1641             :     case ARM::VLD1q16HighQPseudo:
    1642             :     case ARM::VLD1q16LowQPseudo_UPD:
    1643             :     case ARM::VLD1q16HighTPseudo:
    1644             :     case ARM::VLD1q16LowTPseudo_UPD:
    1645             :     case ARM::VLD1q32HighQPseudo:
    1646             :     case ARM::VLD1q32LowQPseudo_UPD:
    1647             :     case ARM::VLD1q32HighTPseudo:
    1648             :     case ARM::VLD1q32LowTPseudo_UPD:
    1649             :     case ARM::VLD1q64HighQPseudo:
    1650             :     case ARM::VLD1q64LowQPseudo_UPD:
    1651             :     case ARM::VLD1q64HighTPseudo:
    1652             :     case ARM::VLD1q64LowTPseudo_UPD:
    1653             :     case ARM::VLD4d8Pseudo_UPD:
    1654             :     case ARM::VLD4d16Pseudo_UPD:
    1655             :     case ARM::VLD4d32Pseudo_UPD:
    1656             :     case ARM::VLD4q8Pseudo_UPD:
    1657             :     case ARM::VLD4q16Pseudo_UPD:
    1658             :     case ARM::VLD4q32Pseudo_UPD:
    1659             :     case ARM::VLD4q8oddPseudo:
    1660             :     case ARM::VLD4q16oddPseudo:
    1661             :     case ARM::VLD4q32oddPseudo:
    1662             :     case ARM::VLD4q8oddPseudo_UPD:
    1663             :     case ARM::VLD4q16oddPseudo_UPD:
    1664             :     case ARM::VLD4q32oddPseudo_UPD:
    1665             :     case ARM::VLD3DUPd8Pseudo:
    1666             :     case ARM::VLD3DUPd16Pseudo:
    1667             :     case ARM::VLD3DUPd32Pseudo:
    1668             :     case ARM::VLD3DUPd8Pseudo_UPD:
    1669             :     case ARM::VLD3DUPd16Pseudo_UPD:
    1670             :     case ARM::VLD3DUPd32Pseudo_UPD:
    1671             :     case ARM::VLD4DUPd8Pseudo:
    1672             :     case ARM::VLD4DUPd16Pseudo:
    1673             :     case ARM::VLD4DUPd32Pseudo:
    1674             :     case ARM::VLD4DUPd8Pseudo_UPD:
    1675             :     case ARM::VLD4DUPd16Pseudo_UPD:
    1676             :     case ARM::VLD4DUPd32Pseudo_UPD:
    1677         124 :       ExpandVLD(MBBI);
    1678         124 :       return true;
    1679             : 
    1680         107 :     case ARM::VST2q8Pseudo:
    1681             :     case ARM::VST2q16Pseudo:
    1682             :     case ARM::VST2q32Pseudo:
    1683             :     case ARM::VST2q8PseudoWB_fixed:
    1684             :     case ARM::VST2q16PseudoWB_fixed:
    1685             :     case ARM::VST2q32PseudoWB_fixed:
    1686             :     case ARM::VST2q8PseudoWB_register:
    1687             :     case ARM::VST2q16PseudoWB_register:
    1688             :     case ARM::VST2q32PseudoWB_register:
    1689             :     case ARM::VST3d8Pseudo:
    1690             :     case ARM::VST3d16Pseudo:
    1691             :     case ARM::VST3d32Pseudo:
    1692             :     case ARM::VST1d8TPseudo:
    1693             :     case ARM::VST1d16TPseudo:
    1694             :     case ARM::VST1d32TPseudo:
    1695             :     case ARM::VST1d64TPseudo:
    1696             :     case ARM::VST3d8Pseudo_UPD:
    1697             :     case ARM::VST3d16Pseudo_UPD:
    1698             :     case ARM::VST3d32Pseudo_UPD:
    1699             :     case ARM::VST1d64TPseudoWB_fixed:
    1700             :     case ARM::VST1d64TPseudoWB_register:
    1701             :     case ARM::VST3q8Pseudo_UPD:
    1702             :     case ARM::VST3q16Pseudo_UPD:
    1703             :     case ARM::VST3q32Pseudo_UPD:
    1704             :     case ARM::VST3q8oddPseudo:
    1705             :     case ARM::VST3q16oddPseudo:
    1706             :     case ARM::VST3q32oddPseudo:
    1707             :     case ARM::VST3q8oddPseudo_UPD:
    1708             :     case ARM::VST3q16oddPseudo_UPD:
    1709             :     case ARM::VST3q32oddPseudo_UPD:
    1710             :     case ARM::VST4d8Pseudo:
    1711             :     case ARM::VST4d16Pseudo:
    1712             :     case ARM::VST4d32Pseudo:
    1713             :     case ARM::VST1d8QPseudo:
    1714             :     case ARM::VST1d16QPseudo:
    1715             :     case ARM::VST1d32QPseudo:
    1716             :     case ARM::VST1d64QPseudo:
    1717             :     case ARM::VST4d8Pseudo_UPD:
    1718             :     case ARM::VST4d16Pseudo_UPD:
    1719             :     case ARM::VST4d32Pseudo_UPD:
    1720             :     case ARM::VST1d64QPseudoWB_fixed:
    1721             :     case ARM::VST1d64QPseudoWB_register:
    1722             :     case ARM::VST1q8HighQPseudo:
    1723             :     case ARM::VST1q8LowQPseudo_UPD:
    1724             :     case ARM::VST1q8HighTPseudo:
    1725             :     case ARM::VST1q8LowTPseudo_UPD:
    1726             :     case ARM::VST1q16HighQPseudo:
    1727             :     case ARM::VST1q16LowQPseudo_UPD:
    1728             :     case ARM::VST1q16HighTPseudo:
    1729             :     case ARM::VST1q16LowTPseudo_UPD:
    1730             :     case ARM::VST1q32HighQPseudo:
    1731             :     case ARM::VST1q32LowQPseudo_UPD:
    1732             :     case ARM::VST1q32HighTPseudo:
    1733             :     case ARM::VST1q32LowTPseudo_UPD:
    1734             :     case ARM::VST1q64HighQPseudo:
    1735             :     case ARM::VST1q64LowQPseudo_UPD:
    1736             :     case ARM::VST1q64HighTPseudo:
    1737             :     case ARM::VST1q64LowTPseudo_UPD:
    1738             :     case ARM::VST4q8Pseudo_UPD:
    1739             :     case ARM::VST4q16Pseudo_UPD:
    1740             :     case ARM::VST4q32Pseudo_UPD:
    1741             :     case ARM::VST4q8oddPseudo:
    1742             :     case ARM::VST4q16oddPseudo:
    1743             :     case ARM::VST4q32oddPseudo:
    1744             :     case ARM::VST4q8oddPseudo_UPD:
    1745             :     case ARM::VST4q16oddPseudo_UPD:
    1746             :     case ARM::VST4q32oddPseudo_UPD:
    1747         107 :       ExpandVST(MBBI);
    1748         107 :       return true;
    1749             : 
    1750         160 :     case ARM::VLD1LNq8Pseudo:
    1751             :     case ARM::VLD1LNq16Pseudo:
    1752             :     case ARM::VLD1LNq32Pseudo:
    1753             :     case ARM::VLD1LNq8Pseudo_UPD:
    1754             :     case ARM::VLD1LNq16Pseudo_UPD:
    1755             :     case ARM::VLD1LNq32Pseudo_UPD:
    1756             :     case ARM::VLD2LNd8Pseudo:
    1757             :     case ARM::VLD2LNd16Pseudo:
    1758             :     case ARM::VLD2LNd32Pseudo:
    1759             :     case ARM::VLD2LNq16Pseudo:
    1760             :     case ARM::VLD2LNq32Pseudo:
    1761             :     case ARM::VLD2LNd8Pseudo_UPD:
    1762             :     case ARM::VLD2LNd16Pseudo_UPD:
    1763             :     case ARM::VLD2LNd32Pseudo_UPD:
    1764             :     case ARM::VLD2LNq16Pseudo_UPD:
    1765             :     case ARM::VLD2LNq32Pseudo_UPD:
    1766             :     case ARM::VLD3LNd8Pseudo:
    1767             :     case ARM::VLD3LNd16Pseudo:
    1768             :     case ARM::VLD3LNd32Pseudo:
    1769             :     case ARM::VLD3LNq16Pseudo:
    1770             :     case ARM::VLD3LNq32Pseudo:
    1771             :     case ARM::VLD3LNd8Pseudo_UPD:
    1772             :     case ARM::VLD3LNd16Pseudo_UPD:
    1773             :     case ARM::VLD3LNd32Pseudo_UPD:
    1774             :     case ARM::VLD3LNq16Pseudo_UPD:
    1775             :     case ARM::VLD3LNq32Pseudo_UPD:
    1776             :     case ARM::VLD4LNd8Pseudo:
    1777             :     case ARM::VLD4LNd16Pseudo:
    1778             :     case ARM::VLD4LNd32Pseudo:
    1779             :     case ARM::VLD4LNq16Pseudo:
    1780             :     case ARM::VLD4LNq32Pseudo:
    1781             :     case ARM::VLD4LNd8Pseudo_UPD:
    1782             :     case ARM::VLD4LNd16Pseudo_UPD:
    1783             :     case ARM::VLD4LNd32Pseudo_UPD:
    1784             :     case ARM::VLD4LNq16Pseudo_UPD:
    1785             :     case ARM::VLD4LNq32Pseudo_UPD:
    1786             :     case ARM::VST1LNq8Pseudo:
    1787             :     case ARM::VST1LNq16Pseudo:
    1788             :     case ARM::VST1LNq32Pseudo:
    1789             :     case ARM::VST1LNq8Pseudo_UPD:
    1790             :     case ARM::VST1LNq16Pseudo_UPD:
    1791             :     case ARM::VST1LNq32Pseudo_UPD:
    1792             :     case ARM::VST2LNd8Pseudo:
    1793             :     case ARM::VST2LNd16Pseudo:
    1794             :     case ARM::VST2LNd32Pseudo:
    1795             :     case ARM::VST2LNq16Pseudo:
    1796             :     case ARM::VST2LNq32Pseudo:
    1797             :     case ARM::VST2LNd8Pseudo_UPD:
    1798             :     case ARM::VST2LNd16Pseudo_UPD:
    1799             :     case ARM::VST2LNd32Pseudo_UPD:
    1800             :     case ARM::VST2LNq16Pseudo_UPD:
    1801             :     case ARM::VST2LNq32Pseudo_UPD:
    1802             :     case ARM::VST3LNd8Pseudo:
    1803             :     case ARM::VST3LNd16Pseudo:
    1804             :     case ARM::VST3LNd32Pseudo:
    1805             :     case ARM::VST3LNq16Pseudo:
    1806             :     case ARM::VST3LNq32Pseudo:
    1807             :     case ARM::VST3LNd8Pseudo_UPD:
    1808             :     case ARM::VST3LNd16Pseudo_UPD:
    1809             :     case ARM::VST3LNd32Pseudo_UPD:
    1810             :     case ARM::VST3LNq16Pseudo_UPD:
    1811             :     case ARM::VST3LNq32Pseudo_UPD:
    1812             :     case ARM::VST4LNd8Pseudo:
    1813             :     case ARM::VST4LNd16Pseudo:
    1814             :     case ARM::VST4LNd32Pseudo:
    1815             :     case ARM::VST4LNq16Pseudo:
    1816             :     case ARM::VST4LNq32Pseudo:
    1817             :     case ARM::VST4LNd8Pseudo_UPD:
    1818             :     case ARM::VST4LNd16Pseudo_UPD:
    1819             :     case ARM::VST4LNd32Pseudo_UPD:
    1820             :     case ARM::VST4LNq16Pseudo_UPD:
    1821             :     case ARM::VST4LNq32Pseudo_UPD:
    1822         160 :       ExpandLaneOp(MBBI);
    1823         160 :       return true;
    1824             : 
    1825           1 :     case ARM::VTBL3Pseudo: ExpandVTBL(MBBI, ARM::VTBL3, false); return true;
    1826           1 :     case ARM::VTBL4Pseudo: ExpandVTBL(MBBI, ARM::VTBL4, false); return true;
    1827           1 :     case ARM::VTBX3Pseudo: ExpandVTBL(MBBI, ARM::VTBX3, true); return true;
    1828           2 :     case ARM::VTBX4Pseudo: ExpandVTBL(MBBI, ARM::VTBX4, true); return true;
    1829             : 
    1830           2 :     case ARM::CMP_SWAP_8:
    1831           2 :       if (STI->isThumb())
    1832             :         return ExpandCMP_SWAP(MBB, MBBI, ARM::t2LDREXB, ARM::t2STREXB,
    1833           1 :                               ARM::tUXTB, NextMBBI);
    1834             :       else
    1835             :         return ExpandCMP_SWAP(MBB, MBBI, ARM::LDREXB, ARM::STREXB,
    1836           1 :                               ARM::UXTB, NextMBBI);
    1837           2 :     case ARM::CMP_SWAP_16:
    1838           2 :       if (STI->isThumb())
    1839             :         return ExpandCMP_SWAP(MBB, MBBI, ARM::t2LDREXH, ARM::t2STREXH,
    1840           1 :                               ARM::tUXTH, NextMBBI);
    1841             :       else
    1842             :         return ExpandCMP_SWAP(MBB, MBBI, ARM::LDREXH, ARM::STREXH,
    1843           1 :                               ARM::UXTH, NextMBBI);
    1844           2 :     case ARM::CMP_SWAP_32:
    1845           2 :       if (STI->isThumb())
    1846             :         return ExpandCMP_SWAP(MBB, MBBI, ARM::t2LDREX, ARM::t2STREX, 0,
    1847           1 :                               NextMBBI);
    1848             :       else
    1849           1 :         return ExpandCMP_SWAP(MBB, MBBI, ARM::LDREX, ARM::STREX, 0, NextMBBI);
    1850             : 
    1851           7 :     case ARM::CMP_SWAP_64:
    1852           7 :       return ExpandCMP_SWAP_64(MBB, MBBI, NextMBBI);
    1853             :   }
    1854             : }
    1855             : 
    1856       18884 : bool ARMExpandPseudo::ExpandMBB(MachineBasicBlock &MBB) {
    1857             :   bool Modified = false;
    1858             : 
    1859             :   MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();
    1860      331162 :   while (MBBI != E) {
    1861      156139 :     MachineBasicBlock::iterator NMBBI = std::next(MBBI);
    1862      156139 :     Modified |= ExpandMI(MBB, MBBI, NMBBI);
    1863      156139 :     MBBI = NMBBI;
    1864             :   }
    1865             : 
    1866       18884 :   return Modified;
    1867             : }
    1868             : 
    1869       13636 : bool ARMExpandPseudo::runOnMachineFunction(MachineFunction &MF) {
    1870       13636 :   STI = &static_cast<const ARMSubtarget &>(MF.getSubtarget());
    1871       13636 :   TII = STI->getInstrInfo();
    1872       13636 :   TRI = STI->getRegisterInfo();
    1873       13636 :   AFI = MF.getInfo<ARMFunctionInfo>();
    1874             : 
    1875             :   bool Modified = false;
    1876       32520 :   for (MachineBasicBlock &MBB : MF)
    1877       18884 :     Modified |= ExpandMBB(MBB);
    1878       13636 :   if (VerifyARMPseudo)
    1879           4 :     MF.verify(this, "After expanding ARM pseudo instructions.");
    1880       13636 :   return Modified;
    1881             : }
    1882             : 
    1883             : /// createARMExpandPseudoPass - returns an instance of the pseudo instruction
    1884             : /// expansion pass.
    1885        2728 : FunctionPass *llvm::createARMExpandPseudoPass() {
    1886        5456 :   return new ARMExpandPseudo();
    1887      303507 : }

Generated by: LCOV version 1.13