LCOV - code coverage report
Current view: top level - lib/Target/ARM - ARMExpandPseudoInsts.cpp (source / functions) Hit Total Coverage
Test: llvm-toolchain.info Lines: 754 784 96.2 %
Date: 2017-09-14 15:23:50 Functions: 22 23 95.7 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : //===-- ARMExpandPseudoInsts.cpp - Expand pseudo instructions -------------===//
       2             : //
       3             : //                     The LLVM Compiler Infrastructure
       4             : //
       5             : // This file is distributed under the University of Illinois Open Source
       6             : // License. See LICENSE.TXT for details.
       7             : //
       8             : //===----------------------------------------------------------------------===//
       9             : //
      10             : // This file contains a pass that expands pseudo instructions into target
      11             : // instructions to allow proper scheduling, if-conversion, and other late
      12             : // optimizations. This pass should be run after register allocation but before
      13             : // the post-regalloc scheduling pass.
      14             : //
      15             : //===----------------------------------------------------------------------===//
      16             : 
      17             : #include "ARM.h"
      18             : #include "ARMBaseInstrInfo.h"
      19             : #include "ARMBaseRegisterInfo.h"
      20             : #include "ARMConstantPoolValue.h"
      21             : #include "ARMMachineFunctionInfo.h"
      22             : #include "ARMSubtarget.h"
      23             : #include "MCTargetDesc/ARMAddressingModes.h"
      24             : #include "llvm/CodeGen/LivePhysRegs.h"
      25             : #include "llvm/CodeGen/MachineFrameInfo.h"
      26             : #include "llvm/CodeGen/MachineFunctionPass.h"
      27             : 
      28             : using namespace llvm;
      29             : 
      30             : #define DEBUG_TYPE "arm-pseudo"
      31             : 
      32             : static cl::opt<bool>
      33       72306 : VerifyARMPseudo("verify-arm-pseudo-expand", cl::Hidden,
      34      144612 :                 cl::desc("Verify machine code after expanding ARM pseudos"));
      35             : 
      36             : #define ARM_EXPAND_PSEUDO_NAME "ARM pseudo instruction expansion pass"
      37             : 
      38             : namespace {
      39        2549 :   class ARMExpandPseudo : public MachineFunctionPass {
      40             :   public:
      41             :     static char ID;
      42        2576 :     ARMExpandPseudo() : MachineFunctionPass(ID) {}
      43             : 
      44             :     const ARMBaseInstrInfo *TII;
      45             :     const TargetRegisterInfo *TRI;
      46             :     const ARMSubtarget *STI;
      47             :     ARMFunctionInfo *AFI;
      48             : 
      49             :     bool runOnMachineFunction(MachineFunction &Fn) override;
      50             : 
      51        2570 :     MachineFunctionProperties getRequiredProperties() const override {
      52        7710 :       return MachineFunctionProperties().set(
      53        7710 :           MachineFunctionProperties::Property::NoVRegs);
      54             :     }
      55             : 
      56        2569 :     StringRef getPassName() const override {
      57        2569 :       return ARM_EXPAND_PSEUDO_NAME;
      58             :     }
      59             : 
      60             :   private:
      61             :     void TransferImpOps(MachineInstr &OldMI,
      62             :                         MachineInstrBuilder &UseMI, MachineInstrBuilder &DefMI);
      63             :     bool ExpandMI(MachineBasicBlock &MBB,
      64             :                   MachineBasicBlock::iterator MBBI,
      65             :                   MachineBasicBlock::iterator &NextMBBI);
      66             :     bool ExpandMBB(MachineBasicBlock &MBB);
      67             :     void ExpandVLD(MachineBasicBlock::iterator &MBBI);
      68             :     void ExpandVST(MachineBasicBlock::iterator &MBBI);
      69             :     void ExpandLaneOp(MachineBasicBlock::iterator &MBBI);
      70             :     void ExpandVTBL(MachineBasicBlock::iterator &MBBI,
      71             :                     unsigned Opc, bool IsExt);
      72             :     void ExpandMOV32BitImm(MachineBasicBlock &MBB,
      73             :                            MachineBasicBlock::iterator &MBBI);
      74             :     bool ExpandCMP_SWAP(MachineBasicBlock &MBB,
      75             :                         MachineBasicBlock::iterator MBBI, unsigned LdrexOp,
      76             :                         unsigned StrexOp, unsigned UxtOp,
      77             :                         MachineBasicBlock::iterator &NextMBBI);
      78             : 
      79             :     bool ExpandCMP_SWAP_64(MachineBasicBlock &MBB,
      80             :                            MachineBasicBlock::iterator MBBI,
      81             :                            MachineBasicBlock::iterator &NextMBBI);
      82             :   };
      83             :   char ARMExpandPseudo::ID = 0;
      84             : }
      85             : 
      86      312538 : INITIALIZE_PASS(ARMExpandPseudo, DEBUG_TYPE, ARM_EXPAND_PSEUDO_NAME, false,
      87             :                 false)
      88             : 
      89             : /// TransferImpOps - Transfer implicit operands on the pseudo instruction to
      90             : /// the instructions created from the expansion.
      91        2472 : void ARMExpandPseudo::TransferImpOps(MachineInstr &OldMI,
      92             :                                      MachineInstrBuilder &UseMI,
      93             :                                      MachineInstrBuilder &DefMI) {
      94        2472 :   const MCInstrDesc &Desc = OldMI.getDesc();
      95        2678 :   for (unsigned i = Desc.getNumOperands(), e = OldMI.getNumOperands();
      96        2678 :        i != e; ++i) {
      97         412 :     const MachineOperand &MO = OldMI.getOperand(i);
      98             :     assert(MO.isReg() && MO.getReg());
      99         206 :     if (MO.isUse())
     100             :       UseMI.add(MO);
     101             :     else
     102             :       DefMI.add(MO);
     103             :   }
     104        2472 : }
     105             : 
     106             : namespace {
     107             :   // Constants for register spacing in NEON load/store instructions.
     108             :   // For quad-register load-lane and store-lane pseudo instructors, the
     109             :   // spacing is initially assumed to be EvenDblSpc, and that is changed to
     110             :   // OddDblSpc depending on the lane number operand.
     111             :   enum NEONRegSpacing {
     112             :     SingleSpc,
     113             :     EvenDblSpc,
     114             :     OddDblSpc
     115             :   };
     116             : 
     117             :   // Entries for NEON load/store information table.  The table is sorted by
     118             :   // PseudoOpc for fast binary-search lookups.
     119             :   struct NEONLdStTableEntry {
     120             :     uint16_t PseudoOpc;
     121             :     uint16_t RealOpc;
     122             :     bool IsLoad;
     123             :     bool isUpdating;
     124             :     bool hasWritebackOperand;
     125             :     uint8_t RegSpacing; // One of type NEONRegSpacing
     126             :     uint8_t NumRegs; // D registers loaded or stored
     127             :     uint8_t RegElts; // elements per D register; used for lane ops
     128             :     // FIXME: Temporary flag to denote whether the real instruction takes
     129             :     // a single register (like the encoding) or all of the registers in
     130             :     // the list (like the asm syntax and the isel DAG). When all definitions
     131             :     // are converted to take only the single encoded register, this will
     132             :     // go away.
     133             :     bool copyAllListRegs;
     134             : 
     135             :     // Comparison methods for binary search of the table.
     136             :     bool operator<(const NEONLdStTableEntry &TE) const {
     137             :       return PseudoOpc < TE.PseudoOpc;
     138             :     }
     139             :     friend bool operator<(const NEONLdStTableEntry &TE, unsigned PseudoOpc) {
     140             :       return TE.PseudoOpc < PseudoOpc;
     141             :     }
     142             :     friend bool LLVM_ATTRIBUTE_UNUSED operator<(unsigned PseudoOpc,
     143             :                                                 const NEONLdStTableEntry &TE) {
     144             :       return PseudoOpc < TE.PseudoOpc;
     145             :     }
     146             :   };
     147             : }
     148             : 
     149             : static const NEONLdStTableEntry NEONLdStTable[] = {
     150             : { ARM::VLD1LNq16Pseudo,     ARM::VLD1LNd16,     true, false, false, EvenDblSpc, 1, 4 ,true},
     151             : { ARM::VLD1LNq16Pseudo_UPD, ARM::VLD1LNd16_UPD, true, true, true,  EvenDblSpc, 1, 4 ,true},
     152             : { ARM::VLD1LNq32Pseudo,     ARM::VLD1LNd32,     true, false, false, EvenDblSpc, 1, 2 ,true},
     153             : { ARM::VLD1LNq32Pseudo_UPD, ARM::VLD1LNd32_UPD, true, true, true,  EvenDblSpc, 1, 2 ,true},
     154             : { ARM::VLD1LNq8Pseudo,      ARM::VLD1LNd8,      true, false, false, EvenDblSpc, 1, 8 ,true},
     155             : { ARM::VLD1LNq8Pseudo_UPD,  ARM::VLD1LNd8_UPD, true, true, true,  EvenDblSpc, 1, 8 ,true},
     156             : 
     157             : { ARM::VLD1d64QPseudo,      ARM::VLD1d64Q,     true,  false, false, SingleSpc,  4, 1 ,false},
     158             : { ARM::VLD1d64QPseudoWB_fixed,  ARM::VLD1d64Qwb_fixed,   true,  true, false, SingleSpc,  4, 1 ,false},
     159             : { ARM::VLD1d64TPseudo,      ARM::VLD1d64T,     true,  false, false, SingleSpc,  3, 1 ,false},
     160             : { ARM::VLD1d64TPseudoWB_fixed,  ARM::VLD1d64Twb_fixed,   true,  true, false, SingleSpc,  3, 1 ,false},
     161             : 
     162             : { ARM::VLD2LNd16Pseudo,     ARM::VLD2LNd16,     true, false, false, SingleSpc,  2, 4 ,true},
     163             : { ARM::VLD2LNd16Pseudo_UPD, ARM::VLD2LNd16_UPD, true, true, true,  SingleSpc,  2, 4 ,true},
     164             : { ARM::VLD2LNd32Pseudo,     ARM::VLD2LNd32,     true, false, false, SingleSpc,  2, 2 ,true},
     165             : { ARM::VLD2LNd32Pseudo_UPD, ARM::VLD2LNd32_UPD, true, true, true,  SingleSpc,  2, 2 ,true},
     166             : { ARM::VLD2LNd8Pseudo,      ARM::VLD2LNd8,      true, false, false, SingleSpc,  2, 8 ,true},
     167             : { ARM::VLD2LNd8Pseudo_UPD,  ARM::VLD2LNd8_UPD, true, true, true,  SingleSpc,  2, 8 ,true},
     168             : { ARM::VLD2LNq16Pseudo,     ARM::VLD2LNq16,     true, false, false, EvenDblSpc, 2, 4 ,true},
     169             : { ARM::VLD2LNq16Pseudo_UPD, ARM::VLD2LNq16_UPD, true, true, true,  EvenDblSpc, 2, 4 ,true},
     170             : { ARM::VLD2LNq32Pseudo,     ARM::VLD2LNq32,     true, false, false, EvenDblSpc, 2, 2 ,true},
     171             : { ARM::VLD2LNq32Pseudo_UPD, ARM::VLD2LNq32_UPD, true, true, true,  EvenDblSpc, 2, 2 ,true},
     172             : 
     173             : { ARM::VLD2q16Pseudo,       ARM::VLD2q16,      true,  false, false, SingleSpc,  4, 4 ,false},
     174             : { ARM::VLD2q16PseudoWB_fixed,   ARM::VLD2q16wb_fixed, true, true, false,  SingleSpc,  4, 4 ,false},
     175             : { ARM::VLD2q16PseudoWB_register,   ARM::VLD2q16wb_register, true, true, true,  SingleSpc,  4, 4 ,false},
     176             : { ARM::VLD2q32Pseudo,       ARM::VLD2q32,      true,  false, false, SingleSpc,  4, 2 ,false},
     177             : { ARM::VLD2q32PseudoWB_fixed,   ARM::VLD2q32wb_fixed, true, true, false,  SingleSpc,  4, 2 ,false},
     178             : { ARM::VLD2q32PseudoWB_register,   ARM::VLD2q32wb_register, true, true, true,  SingleSpc,  4, 2 ,false},
     179             : { ARM::VLD2q8Pseudo,        ARM::VLD2q8,       true,  false, false, SingleSpc,  4, 8 ,false},
     180             : { ARM::VLD2q8PseudoWB_fixed,    ARM::VLD2q8wb_fixed, true, true, false,  SingleSpc,  4, 8 ,false},
     181             : { ARM::VLD2q8PseudoWB_register,    ARM::VLD2q8wb_register, true, true, true,  SingleSpc,  4, 8 ,false},
     182             : 
     183             : { ARM::VLD3DUPd16Pseudo,     ARM::VLD3DUPd16,     true, false, false, SingleSpc, 3, 4,true},
     184             : { ARM::VLD3DUPd16Pseudo_UPD, ARM::VLD3DUPd16_UPD, true, true, true,  SingleSpc, 3, 4,true},
     185             : { ARM::VLD3DUPd32Pseudo,     ARM::VLD3DUPd32,     true, false, false, SingleSpc, 3, 2,true},
     186             : { ARM::VLD3DUPd32Pseudo_UPD, ARM::VLD3DUPd32_UPD, true, true, true,  SingleSpc, 3, 2,true},
     187             : { ARM::VLD3DUPd8Pseudo,      ARM::VLD3DUPd8,      true, false, false, SingleSpc, 3, 8,true},
     188             : { ARM::VLD3DUPd8Pseudo_UPD,  ARM::VLD3DUPd8_UPD, true, true, true,  SingleSpc, 3, 8,true},
     189             : 
     190             : { ARM::VLD3LNd16Pseudo,     ARM::VLD3LNd16,     true, false, false, SingleSpc,  3, 4 ,true},
     191             : { ARM::VLD3LNd16Pseudo_UPD, ARM::VLD3LNd16_UPD, true, true, true,  SingleSpc,  3, 4 ,true},
     192             : { ARM::VLD3LNd32Pseudo,     ARM::VLD3LNd32,     true, false, false, SingleSpc,  3, 2 ,true},
     193             : { ARM::VLD3LNd32Pseudo_UPD, ARM::VLD3LNd32_UPD, true, true, true,  SingleSpc,  3, 2 ,true},
     194             : { ARM::VLD3LNd8Pseudo,      ARM::VLD3LNd8,      true, false, false, SingleSpc,  3, 8 ,true},
     195             : { ARM::VLD3LNd8Pseudo_UPD,  ARM::VLD3LNd8_UPD, true, true, true,  SingleSpc,  3, 8 ,true},
     196             : { ARM::VLD3LNq16Pseudo,     ARM::VLD3LNq16,     true, false, false, EvenDblSpc, 3, 4 ,true},
     197             : { ARM::VLD3LNq16Pseudo_UPD, ARM::VLD3LNq16_UPD, true, true, true,  EvenDblSpc, 3, 4 ,true},
     198             : { ARM::VLD3LNq32Pseudo,     ARM::VLD3LNq32,     true, false, false, EvenDblSpc, 3, 2 ,true},
     199             : { ARM::VLD3LNq32Pseudo_UPD, ARM::VLD3LNq32_UPD, true, true, true,  EvenDblSpc, 3, 2 ,true},
     200             : 
     201             : { ARM::VLD3d16Pseudo,       ARM::VLD3d16,      true,  false, false, SingleSpc,  3, 4 ,true},
     202             : { ARM::VLD3d16Pseudo_UPD,   ARM::VLD3d16_UPD, true, true, true,  SingleSpc,  3, 4 ,true},
     203             : { ARM::VLD3d32Pseudo,       ARM::VLD3d32,      true,  false, false, SingleSpc,  3, 2 ,true},
     204             : { ARM::VLD3d32Pseudo_UPD,   ARM::VLD3d32_UPD, true, true, true,  SingleSpc,  3, 2 ,true},
     205             : { ARM::VLD3d8Pseudo,        ARM::VLD3d8,       true,  false, false, SingleSpc,  3, 8 ,true},
     206             : { ARM::VLD3d8Pseudo_UPD,    ARM::VLD3d8_UPD, true, true, true,  SingleSpc,  3, 8 ,true},
     207             : 
     208             : { ARM::VLD3q16Pseudo_UPD,    ARM::VLD3q16_UPD, true, true, true,  EvenDblSpc, 3, 4 ,true},
     209             : { ARM::VLD3q16oddPseudo,     ARM::VLD3q16,     true,  false, false, OddDblSpc,  3, 4 ,true},
     210             : { ARM::VLD3q16oddPseudo_UPD, ARM::VLD3q16_UPD, true, true, true,  OddDblSpc,  3, 4 ,true},
     211             : { ARM::VLD3q32Pseudo_UPD,    ARM::VLD3q32_UPD, true, true, true,  EvenDblSpc, 3, 2 ,true},
     212             : { ARM::VLD3q32oddPseudo,     ARM::VLD3q32,     true,  false, false, OddDblSpc,  3, 2 ,true},
     213             : { ARM::VLD3q32oddPseudo_UPD, ARM::VLD3q32_UPD, true, true, true,  OddDblSpc,  3, 2 ,true},
     214             : { ARM::VLD3q8Pseudo_UPD,     ARM::VLD3q8_UPD, true, true, true,  EvenDblSpc, 3, 8 ,true},
     215             : { ARM::VLD3q8oddPseudo,      ARM::VLD3q8,      true,  false, false, OddDblSpc,  3, 8 ,true},
     216             : { ARM::VLD3q8oddPseudo_UPD,  ARM::VLD3q8_UPD, true, true, true,  OddDblSpc,  3, 8 ,true},
     217             : 
     218             : { ARM::VLD4DUPd16Pseudo,     ARM::VLD4DUPd16,     true, false, false, SingleSpc, 4, 4,true},
     219             : { ARM::VLD4DUPd16Pseudo_UPD, ARM::VLD4DUPd16_UPD, true, true, true,  SingleSpc, 4, 4,true},
     220             : { ARM::VLD4DUPd32Pseudo,     ARM::VLD4DUPd32,     true, false, false, SingleSpc, 4, 2,true},
     221             : { ARM::VLD4DUPd32Pseudo_UPD, ARM::VLD4DUPd32_UPD, true, true, true,  SingleSpc, 4, 2,true},
     222             : { ARM::VLD4DUPd8Pseudo,      ARM::VLD4DUPd8,      true, false, false, SingleSpc, 4, 8,true},
     223             : { ARM::VLD4DUPd8Pseudo_UPD,  ARM::VLD4DUPd8_UPD, true, true, true,  SingleSpc, 4, 8,true},
     224             : 
     225             : { ARM::VLD4LNd16Pseudo,     ARM::VLD4LNd16,     true, false, false, SingleSpc,  4, 4 ,true},
     226             : { ARM::VLD4LNd16Pseudo_UPD, ARM::VLD4LNd16_UPD, true, true, true,  SingleSpc,  4, 4 ,true},
     227             : { ARM::VLD4LNd32Pseudo,     ARM::VLD4LNd32,     true, false, false, SingleSpc,  4, 2 ,true},
     228             : { ARM::VLD4LNd32Pseudo_UPD, ARM::VLD4LNd32_UPD, true, true, true,  SingleSpc,  4, 2 ,true},
     229             : { ARM::VLD4LNd8Pseudo,      ARM::VLD4LNd8,      true, false, false, SingleSpc,  4, 8 ,true},
     230             : { ARM::VLD4LNd8Pseudo_UPD,  ARM::VLD4LNd8_UPD, true, true, true,  SingleSpc,  4, 8 ,true},
     231             : { ARM::VLD4LNq16Pseudo,     ARM::VLD4LNq16,     true, false, false, EvenDblSpc, 4, 4 ,true},
     232             : { ARM::VLD4LNq16Pseudo_UPD, ARM::VLD4LNq16_UPD, true, true, true,  EvenDblSpc, 4, 4 ,true},
     233             : { ARM::VLD4LNq32Pseudo,     ARM::VLD4LNq32,     true, false, false, EvenDblSpc, 4, 2 ,true},
     234             : { ARM::VLD4LNq32Pseudo_UPD, ARM::VLD4LNq32_UPD, true, true, true,  EvenDblSpc, 4, 2 ,true},
     235             : 
     236             : { ARM::VLD4d16Pseudo,       ARM::VLD4d16,      true,  false, false, SingleSpc,  4, 4 ,true},
     237             : { ARM::VLD4d16Pseudo_UPD,   ARM::VLD4d16_UPD, true, true, true,  SingleSpc,  4, 4 ,true},
     238             : { ARM::VLD4d32Pseudo,       ARM::VLD4d32,      true,  false, false, SingleSpc,  4, 2 ,true},
     239             : { ARM::VLD4d32Pseudo_UPD,   ARM::VLD4d32_UPD, true, true, true,  SingleSpc,  4, 2 ,true},
     240             : { ARM::VLD4d8Pseudo,        ARM::VLD4d8,       true,  false, false, SingleSpc,  4, 8 ,true},
     241             : { ARM::VLD4d8Pseudo_UPD,    ARM::VLD4d8_UPD, true, true, true,  SingleSpc,  4, 8 ,true},
     242             : 
     243             : { ARM::VLD4q16Pseudo_UPD,    ARM::VLD4q16_UPD, true, true, true,  EvenDblSpc, 4, 4 ,true},
     244             : { ARM::VLD4q16oddPseudo,     ARM::VLD4q16,     true,  false, false, OddDblSpc,  4, 4 ,true},
     245             : { ARM::VLD4q16oddPseudo_UPD, ARM::VLD4q16_UPD, true, true, true,  OddDblSpc,  4, 4 ,true},
     246             : { ARM::VLD4q32Pseudo_UPD,    ARM::VLD4q32_UPD, true, true, true,  EvenDblSpc, 4, 2 ,true},
     247             : { ARM::VLD4q32oddPseudo,     ARM::VLD4q32,     true,  false, false, OddDblSpc,  4, 2 ,true},
     248             : { ARM::VLD4q32oddPseudo_UPD, ARM::VLD4q32_UPD, true, true, true,  OddDblSpc,  4, 2 ,true},
     249             : { ARM::VLD4q8Pseudo_UPD,     ARM::VLD4q8_UPD, true, true, true,  EvenDblSpc, 4, 8 ,true},
     250             : { ARM::VLD4q8oddPseudo,      ARM::VLD4q8,      true,  false, false, OddDblSpc,  4, 8 ,true},
     251             : { ARM::VLD4q8oddPseudo_UPD,  ARM::VLD4q8_UPD, true, true, true,  OddDblSpc,  4, 8 ,true},
     252             : 
     253             : { ARM::VST1LNq16Pseudo,     ARM::VST1LNd16,    false, false, false, EvenDblSpc, 1, 4 ,true},
     254             : { ARM::VST1LNq16Pseudo_UPD, ARM::VST1LNd16_UPD, false, true, true,  EvenDblSpc, 1, 4 ,true},
     255             : { ARM::VST1LNq32Pseudo,     ARM::VST1LNd32,    false, false, false, EvenDblSpc, 1, 2 ,true},
     256             : { ARM::VST1LNq32Pseudo_UPD, ARM::VST1LNd32_UPD, false, true, true,  EvenDblSpc, 1, 2 ,true},
     257             : { ARM::VST1LNq8Pseudo,      ARM::VST1LNd8,     false, false, false, EvenDblSpc, 1, 8 ,true},
     258             : { ARM::VST1LNq8Pseudo_UPD,  ARM::VST1LNd8_UPD, false, true, true,  EvenDblSpc, 1, 8 ,true},
     259             : 
     260             : { ARM::VST1d64QPseudo,      ARM::VST1d64Q,     false, false, false, SingleSpc,  4, 1 ,false},
     261             : { ARM::VST1d64QPseudoWB_fixed,  ARM::VST1d64Qwb_fixed, false, true, false,  SingleSpc,  4, 1 ,false},
     262             : { ARM::VST1d64QPseudoWB_register, ARM::VST1d64Qwb_register, false, true, true,  SingleSpc,  4, 1 ,false},
     263             : { ARM::VST1d64TPseudo,      ARM::VST1d64T,     false, false, false, SingleSpc,  3, 1 ,false},
     264             : { ARM::VST1d64TPseudoWB_fixed,  ARM::VST1d64Twb_fixed, false, true, false,  SingleSpc,  3, 1 ,false},
     265             : { ARM::VST1d64TPseudoWB_register,  ARM::VST1d64Twb_register, false, true, true,  SingleSpc,  3, 1 ,false},
     266             : 
     267             : { ARM::VST2LNd16Pseudo,     ARM::VST2LNd16,     false, false, false, SingleSpc, 2, 4 ,true},
     268             : { ARM::VST2LNd16Pseudo_UPD, ARM::VST2LNd16_UPD, false, true, true,  SingleSpc, 2, 4 ,true},
     269             : { ARM::VST2LNd32Pseudo,     ARM::VST2LNd32,     false, false, false, SingleSpc, 2, 2 ,true},
     270             : { ARM::VST2LNd32Pseudo_UPD, ARM::VST2LNd32_UPD, false, true, true,  SingleSpc, 2, 2 ,true},
     271             : { ARM::VST2LNd8Pseudo,      ARM::VST2LNd8,      false, false, false, SingleSpc, 2, 8 ,true},
     272             : { ARM::VST2LNd8Pseudo_UPD,  ARM::VST2LNd8_UPD, false, true, true,  SingleSpc, 2, 8 ,true},
     273             : { ARM::VST2LNq16Pseudo,     ARM::VST2LNq16,     false, false, false, EvenDblSpc, 2, 4,true},
     274             : { ARM::VST2LNq16Pseudo_UPD, ARM::VST2LNq16_UPD, false, true, true,  EvenDblSpc, 2, 4,true},
     275             : { ARM::VST2LNq32Pseudo,     ARM::VST2LNq32,     false, false, false, EvenDblSpc, 2, 2,true},
     276             : { ARM::VST2LNq32Pseudo_UPD, ARM::VST2LNq32_UPD, false, true, true,  EvenDblSpc, 2, 2,true},
     277             : 
     278             : { ARM::VST2q16Pseudo,       ARM::VST2q16,      false, false, false, SingleSpc,  4, 4 ,false},
     279             : { ARM::VST2q16PseudoWB_fixed,   ARM::VST2q16wb_fixed, false, true, false,  SingleSpc,  4, 4 ,false},
     280             : { ARM::VST2q16PseudoWB_register,   ARM::VST2q16wb_register, false, true, true,  SingleSpc,  4, 4 ,false},
     281             : { ARM::VST2q32Pseudo,       ARM::VST2q32,      false, false, false, SingleSpc,  4, 2 ,false},
     282             : { ARM::VST2q32PseudoWB_fixed,   ARM::VST2q32wb_fixed, false, true, false,  SingleSpc,  4, 2 ,false},
     283             : { ARM::VST2q32PseudoWB_register,   ARM::VST2q32wb_register, false, true, true,  SingleSpc,  4, 2 ,false},
     284             : { ARM::VST2q8Pseudo,        ARM::VST2q8,       false, false, false, SingleSpc,  4, 8 ,false},
     285             : { ARM::VST2q8PseudoWB_fixed,    ARM::VST2q8wb_fixed, false, true, false,  SingleSpc,  4, 8 ,false},
     286             : { ARM::VST2q8PseudoWB_register,    ARM::VST2q8wb_register, false, true, true,  SingleSpc,  4, 8 ,false},
     287             : 
     288             : { ARM::VST3LNd16Pseudo,     ARM::VST3LNd16,     false, false, false, SingleSpc, 3, 4 ,true},
     289             : { ARM::VST3LNd16Pseudo_UPD, ARM::VST3LNd16_UPD, false, true, true,  SingleSpc, 3, 4 ,true},
     290             : { ARM::VST3LNd32Pseudo,     ARM::VST3LNd32,     false, false, false, SingleSpc, 3, 2 ,true},
     291             : { ARM::VST3LNd32Pseudo_UPD, ARM::VST3LNd32_UPD, false, true, true,  SingleSpc, 3, 2 ,true},
     292             : { ARM::VST3LNd8Pseudo,      ARM::VST3LNd8,      false, false, false, SingleSpc, 3, 8 ,true},
     293             : { ARM::VST3LNd8Pseudo_UPD,  ARM::VST3LNd8_UPD, false, true, true,  SingleSpc, 3, 8 ,true},
     294             : { ARM::VST3LNq16Pseudo,     ARM::VST3LNq16,     false, false, false, EvenDblSpc, 3, 4,true},
     295             : { ARM::VST3LNq16Pseudo_UPD, ARM::VST3LNq16_UPD, false, true, true,  EvenDblSpc, 3, 4,true},
     296             : { ARM::VST3LNq32Pseudo,     ARM::VST3LNq32,     false, false, false, EvenDblSpc, 3, 2,true},
     297             : { ARM::VST3LNq32Pseudo_UPD, ARM::VST3LNq32_UPD, false, true, true,  EvenDblSpc, 3, 2,true},
     298             : 
     299             : { ARM::VST3d16Pseudo,       ARM::VST3d16,      false, false, false, SingleSpc,  3, 4 ,true},
     300             : { ARM::VST3d16Pseudo_UPD,   ARM::VST3d16_UPD, false, true, true,  SingleSpc,  3, 4 ,true},
     301             : { ARM::VST3d32Pseudo,       ARM::VST3d32,      false, false, false, SingleSpc,  3, 2 ,true},
     302             : { ARM::VST3d32Pseudo_UPD,   ARM::VST3d32_UPD, false, true, true,  SingleSpc,  3, 2 ,true},
     303             : { ARM::VST3d8Pseudo,        ARM::VST3d8,       false, false, false, SingleSpc,  3, 8 ,true},
     304             : { ARM::VST3d8Pseudo_UPD,    ARM::VST3d8_UPD, false, true, true,  SingleSpc,  3, 8 ,true},
     305             : 
     306             : { ARM::VST3q16Pseudo_UPD,    ARM::VST3q16_UPD, false, true, true,  EvenDblSpc, 3, 4 ,true},
     307             : { ARM::VST3q16oddPseudo,     ARM::VST3q16,     false, false, false, OddDblSpc,  3, 4 ,true},
     308             : { ARM::VST3q16oddPseudo_UPD, ARM::VST3q16_UPD, false, true, true,  OddDblSpc,  3, 4 ,true},
     309             : { ARM::VST3q32Pseudo_UPD,    ARM::VST3q32_UPD, false, true, true,  EvenDblSpc, 3, 2 ,true},
     310             : { ARM::VST3q32oddPseudo,     ARM::VST3q32,     false, false, false, OddDblSpc,  3, 2 ,true},
     311             : { ARM::VST3q32oddPseudo_UPD, ARM::VST3q32_UPD, false, true, true,  OddDblSpc,  3, 2 ,true},
     312             : { ARM::VST3q8Pseudo_UPD,     ARM::VST3q8_UPD, false, true, true,  EvenDblSpc, 3, 8 ,true},
     313             : { ARM::VST3q8oddPseudo,      ARM::VST3q8,      false, false, false, OddDblSpc,  3, 8 ,true},
     314             : { ARM::VST3q8oddPseudo_UPD,  ARM::VST3q8_UPD, false, true, true,  OddDblSpc,  3, 8 ,true},
     315             : 
     316             : { ARM::VST4LNd16Pseudo,     ARM::VST4LNd16,     false, false, false, SingleSpc, 4, 4 ,true},
     317             : { ARM::VST4LNd16Pseudo_UPD, ARM::VST4LNd16_UPD, false, true, true,  SingleSpc, 4, 4 ,true},
     318             : { ARM::VST4LNd32Pseudo,     ARM::VST4LNd32,     false, false, false, SingleSpc, 4, 2 ,true},
     319             : { ARM::VST4LNd32Pseudo_UPD, ARM::VST4LNd32_UPD, false, true, true,  SingleSpc, 4, 2 ,true},
     320             : { ARM::VST4LNd8Pseudo,      ARM::VST4LNd8,      false, false, false, SingleSpc, 4, 8 ,true},
     321             : { ARM::VST4LNd8Pseudo_UPD,  ARM::VST4LNd8_UPD, false, true, true,  SingleSpc, 4, 8 ,true},
     322             : { ARM::VST4LNq16Pseudo,     ARM::VST4LNq16,     false, false, false, EvenDblSpc, 4, 4,true},
     323             : { ARM::VST4LNq16Pseudo_UPD, ARM::VST4LNq16_UPD, false, true, true,  EvenDblSpc, 4, 4,true},
     324             : { ARM::VST4LNq32Pseudo,     ARM::VST4LNq32,     false, false, false, EvenDblSpc, 4, 2,true},
     325             : { ARM::VST4LNq32Pseudo_UPD, ARM::VST4LNq32_UPD, false, true, true,  EvenDblSpc, 4, 2,true},
     326             : 
     327             : { ARM::VST4d16Pseudo,       ARM::VST4d16,      false, false, false, SingleSpc,  4, 4 ,true},
     328             : { ARM::VST4d16Pseudo_UPD,   ARM::VST4d16_UPD, false, true, true,  SingleSpc,  4, 4 ,true},
     329             : { ARM::VST4d32Pseudo,       ARM::VST4d32,      false, false, false, SingleSpc,  4, 2 ,true},
     330             : { ARM::VST4d32Pseudo_UPD,   ARM::VST4d32_UPD, false, true, true,  SingleSpc,  4, 2 ,true},
     331             : { ARM::VST4d8Pseudo,        ARM::VST4d8,       false, false, false, SingleSpc,  4, 8 ,true},
     332             : { ARM::VST4d8Pseudo_UPD,    ARM::VST4d8_UPD, false, true, true,  SingleSpc,  4, 8 ,true},
     333             : 
     334             : { ARM::VST4q16Pseudo_UPD,    ARM::VST4q16_UPD, false, true, true,  EvenDblSpc, 4, 4 ,true},
     335             : { ARM::VST4q16oddPseudo,     ARM::VST4q16,     false, false, false, OddDblSpc,  4, 4 ,true},
     336             : { ARM::VST4q16oddPseudo_UPD, ARM::VST4q16_UPD, false, true, true,  OddDblSpc,  4, 4 ,true},
     337             : { ARM::VST4q32Pseudo_UPD,    ARM::VST4q32_UPD, false, true, true,  EvenDblSpc, 4, 2 ,true},
     338             : { ARM::VST4q32oddPseudo,     ARM::VST4q32,     false, false, false, OddDblSpc,  4, 2 ,true},
     339             : { ARM::VST4q32oddPseudo_UPD, ARM::VST4q32_UPD, false, true, true,  OddDblSpc,  4, 2 ,true},
     340             : { ARM::VST4q8Pseudo_UPD,     ARM::VST4q8_UPD, false, true, true,  EvenDblSpc, 4, 8 ,true},
     341             : { ARM::VST4q8oddPseudo,      ARM::VST4q8,      false, false, false, OddDblSpc,  4, 8 ,true},
     342             : { ARM::VST4q8oddPseudo_UPD,  ARM::VST4q8_UPD, false, true, true,  OddDblSpc,  4, 8 ,true}
     343             : };
     344             : 
     345             : /// LookupNEONLdSt - Search the NEONLdStTable for information about a NEON
     346             : /// load or store pseudo instruction.
     347             : static const NEONLdStTableEntry *LookupNEONLdSt(unsigned Opcode) {
     348             : #ifndef NDEBUG
     349             :   // Make sure the table is sorted.
     350             :   static bool TableChecked = false;
     351             :   if (!TableChecked) {
     352             :     assert(std::is_sorted(std::begin(NEONLdStTable), std::end(NEONLdStTable)) &&
     353             :            "NEONLdStTable is not sorted!");
     354             :     TableChecked = true;
     355             :   }
     356             : #endif
     357             : 
     358             :   auto I = std::lower_bound(std::begin(NEONLdStTable),
     359         335 :                             std::end(NEONLdStTable), Opcode);
     360         335 :   if (I != std::end(NEONLdStTable) && I->PseudoOpc == Opcode)
     361             :     return I;
     362             :   return nullptr;
     363             : }
     364             : 
     365             : /// GetDSubRegs - Get 4 D subregisters of a Q, QQ, or QQQQ register,
     366             : /// corresponding to the specified register spacing.  Not all of the results
     367             : /// are necessarily valid, e.g., a Q register only has 2 D subregisters.
     368         340 : static void GetDSubRegs(unsigned Reg, NEONRegSpacing RegSpc,
     369             :                         const TargetRegisterInfo *TRI, unsigned &D0,
     370             :                         unsigned &D1, unsigned &D2, unsigned &D3) {
     371         340 :   if (RegSpc == SingleSpc) {
     372         154 :     D0 = TRI->getSubReg(Reg, ARM::dsub_0);
     373         154 :     D1 = TRI->getSubReg(Reg, ARM::dsub_1);
     374         154 :     D2 = TRI->getSubReg(Reg, ARM::dsub_2);
     375         154 :     D3 = TRI->getSubReg(Reg, ARM::dsub_3);
     376         186 :   } else if (RegSpc == EvenDblSpc) {
     377          94 :     D0 = TRI->getSubReg(Reg, ARM::dsub_0);
     378          94 :     D1 = TRI->getSubReg(Reg, ARM::dsub_2);
     379          94 :     D2 = TRI->getSubReg(Reg, ARM::dsub_4);
     380          94 :     D3 = TRI->getSubReg(Reg, ARM::dsub_6);
     381             :   } else {
     382             :     assert(RegSpc == OddDblSpc && "unknown register spacing");
     383          92 :     D0 = TRI->getSubReg(Reg, ARM::dsub_1);
     384          92 :     D1 = TRI->getSubReg(Reg, ARM::dsub_3);
     385          92 :     D2 = TRI->getSubReg(Reg, ARM::dsub_5);
     386          92 :     D3 = TRI->getSubReg(Reg, ARM::dsub_7);
     387             :   }
     388         340 : }
     389             : 
     390             : /// ExpandVLD - Translate VLD pseudo instructions with Q, QQ or QQQQ register
     391             : /// operands to real VLD instructions with D register operands.
     392          95 : void ARMExpandPseudo::ExpandVLD(MachineBasicBlock::iterator &MBBI) {
     393          95 :   MachineInstr &MI = *MBBI;
     394          95 :   MachineBasicBlock &MBB = *MI.getParent();
     395             : 
     396         190 :   const NEONLdStTableEntry *TableEntry = LookupNEONLdSt(MI.getOpcode());
     397             :   assert(TableEntry && TableEntry->IsLoad && "NEONLdStTable lookup failed");
     398          95 :   NEONRegSpacing RegSpc = (NEONRegSpacing)TableEntry->RegSpacing;
     399          95 :   unsigned NumRegs = TableEntry->NumRegs;
     400             : 
     401             :   MachineInstrBuilder MIB = BuildMI(MBB, MBBI, MI.getDebugLoc(),
     402         285 :                                     TII->get(TableEntry->RealOpc));
     403          95 :   unsigned OpIdx = 0;
     404             : 
     405         190 :   bool DstIsDead = MI.getOperand(OpIdx).isDead();
     406          95 :   unsigned DstReg = MI.getOperand(OpIdx++).getReg();
     407             :   unsigned D0, D1, D2, D3;
     408          95 :   GetDSubRegs(DstReg, RegSpc, TRI, D0, D1, D2, D3);
     409          95 :   MIB.addReg(D0, RegState::Define | getDeadRegState(DstIsDead));
     410          95 :   if (NumRegs > 1 && TableEntry->copyAllListRegs)
     411          73 :     MIB.addReg(D1, RegState::Define | getDeadRegState(DstIsDead));
     412          95 :   if (NumRegs > 2 && TableEntry->copyAllListRegs)
     413          73 :     MIB.addReg(D2, RegState::Define | getDeadRegState(DstIsDead));
     414          95 :   if (NumRegs > 3 && TableEntry->copyAllListRegs)
     415          20 :     MIB.addReg(D3, RegState::Define | getDeadRegState(DstIsDead));
     416             : 
     417          95 :   if (TableEntry->isUpdating)
     418          76 :     MIB.add(MI.getOperand(OpIdx++));
     419             : 
     420             :   // Copy the addrmode6 operands.
     421         285 :   MIB.add(MI.getOperand(OpIdx++));
     422         285 :   MIB.add(MI.getOperand(OpIdx++));
     423             :   // Copy the am6offset operand.
     424          95 :   if (TableEntry->hasWritebackOperand)
     425          60 :     MIB.add(MI.getOperand(OpIdx++));
     426             : 
     427             :   // For an instruction writing double-spaced subregs, the pseudo instruction
     428             :   // has an extra operand that is a use of the super-register.  Record the
     429             :   // operand index and skip over it.
     430          95 :   unsigned SrcOpIdx = 0;
     431          95 :   if (RegSpc == EvenDblSpc || RegSpc == OddDblSpc)
     432          39 :     SrcOpIdx = OpIdx++;
     433             : 
     434             :   // Copy the predicate operands.
     435         285 :   MIB.add(MI.getOperand(OpIdx++));
     436         285 :   MIB.add(MI.getOperand(OpIdx++));
     437             : 
     438             :   // Copy the super-register source operand used for double-spaced subregs over
     439             :   // to the new instruction as an implicit operand.
     440          95 :   if (SrcOpIdx != 0) {
     441          78 :     MachineOperand MO = MI.getOperand(SrcOpIdx);
     442          39 :     MO.setImplicit(true);
     443          39 :     MIB.add(MO);
     444             :   }
     445             :   // Add an implicit def for the super-register.
     446          95 :   MIB.addReg(DstReg, RegState::ImplicitDefine | getDeadRegState(DstIsDead));
     447          95 :   TransferImpOps(MI, MIB, MIB);
     448             : 
     449             :   // Transfer memoperands.
     450         285 :   MIB->setMemRefs(MI.memoperands_begin(), MI.memoperands_end());
     451             : 
     452          95 :   MI.eraseFromParent();
     453          95 : }
     454             : 
     455             : /// ExpandVST - Translate VST pseudo instructions with Q, QQ or QQQQ register
     456             : /// operands to real VST instructions with D register operands.
     457          77 : void ARMExpandPseudo::ExpandVST(MachineBasicBlock::iterator &MBBI) {
     458          77 :   MachineInstr &MI = *MBBI;
     459          77 :   MachineBasicBlock &MBB = *MI.getParent();
     460             : 
     461         154 :   const NEONLdStTableEntry *TableEntry = LookupNEONLdSt(MI.getOpcode());
     462             :   assert(TableEntry && !TableEntry->IsLoad && "NEONLdStTable lookup failed");
     463          77 :   NEONRegSpacing RegSpc = (NEONRegSpacing)TableEntry->RegSpacing;
     464          77 :   unsigned NumRegs = TableEntry->NumRegs;
     465             : 
     466             :   MachineInstrBuilder MIB = BuildMI(MBB, MBBI, MI.getDebugLoc(),
     467         231 :                                     TII->get(TableEntry->RealOpc));
     468          77 :   unsigned OpIdx = 0;
     469          77 :   if (TableEntry->isUpdating)
     470          26 :     MIB.add(MI.getOperand(OpIdx++));
     471             : 
     472             :   // Copy the addrmode6 operands.
     473         231 :   MIB.add(MI.getOperand(OpIdx++));
     474         231 :   MIB.add(MI.getOperand(OpIdx++));
     475             :   // Copy the am6offset operand.
     476          77 :   if (TableEntry->hasWritebackOperand)
     477          44 :     MIB.add(MI.getOperand(OpIdx++));
     478             : 
     479         231 :   bool SrcIsKill = MI.getOperand(OpIdx).isKill();
     480         154 :   bool SrcIsUndef = MI.getOperand(OpIdx).isUndef();
     481          77 :   unsigned SrcReg = MI.getOperand(OpIdx++).getReg();
     482             :   unsigned D0, D1, D2, D3;
     483          77 :   GetDSubRegs(SrcReg, RegSpc, TRI, D0, D1, D2, D3);
     484          77 :   MIB.addReg(D0, getUndefRegState(SrcIsUndef));
     485          77 :   if (NumRegs > 1 && TableEntry->copyAllListRegs)
     486          57 :     MIB.addReg(D1, getUndefRegState(SrcIsUndef));
     487          77 :   if (NumRegs > 2 && TableEntry->copyAllListRegs)
     488          57 :     MIB.addReg(D2, getUndefRegState(SrcIsUndef));
     489          77 :   if (NumRegs > 3 && TableEntry->copyAllListRegs)
     490          32 :     MIB.addReg(D3, getUndefRegState(SrcIsUndef));
     491             : 
     492             :   // Copy the predicate operands.
     493         231 :   MIB.add(MI.getOperand(OpIdx++));
     494         231 :   MIB.add(MI.getOperand(OpIdx++));
     495             : 
     496          77 :   if (SrcIsKill && !SrcIsUndef) // Add an implicit kill for the super-reg.
     497          51 :     MIB->addRegisterKilled(SrcReg, TRI, true);
     498          26 :   else if (!SrcIsUndef)
     499          25 :     MIB.addReg(SrcReg, RegState::Implicit); // Add implicit uses for src reg.
     500          77 :   TransferImpOps(MI, MIB, MIB);
     501             : 
     502             :   // Transfer memoperands.
     503         231 :   MIB->setMemRefs(MI.memoperands_begin(), MI.memoperands_end());
     504             : 
     505          77 :   MI.eraseFromParent();
     506          77 : }
     507             : 
     508             : /// ExpandLaneOp - Translate VLD*LN and VST*LN instructions with Q, QQ or QQQQ
     509             : /// register operands to real instructions with D register operands.
     510         163 : void ARMExpandPseudo::ExpandLaneOp(MachineBasicBlock::iterator &MBBI) {
     511         163 :   MachineInstr &MI = *MBBI;
     512         163 :   MachineBasicBlock &MBB = *MI.getParent();
     513             : 
     514         326 :   const NEONLdStTableEntry *TableEntry = LookupNEONLdSt(MI.getOpcode());
     515             :   assert(TableEntry && "NEONLdStTable lookup failed");
     516         163 :   NEONRegSpacing RegSpc = (NEONRegSpacing)TableEntry->RegSpacing;
     517         163 :   unsigned NumRegs = TableEntry->NumRegs;
     518         163 :   unsigned RegElts = TableEntry->RegElts;
     519             : 
     520             :   MachineInstrBuilder MIB = BuildMI(MBB, MBBI, MI.getDebugLoc(),
     521         489 :                                     TII->get(TableEntry->RealOpc));
     522         163 :   unsigned OpIdx = 0;
     523             :   // The lane operand is always the 3rd from last operand, before the 2
     524             :   // predicate operands.
     525         326 :   unsigned Lane = MI.getOperand(MI.getDesc().getNumOperands() - 3).getImm();
     526             : 
     527             :   // Adjust the lane and spacing as needed for Q registers.
     528             :   assert(RegSpc != OddDblSpc && "unexpected register spacing for VLD/VST-lane");
     529         163 :   if (RegSpc == EvenDblSpc && Lane >= RegElts) {
     530          55 :     RegSpc = OddDblSpc;
     531          55 :     Lane -= RegElts;
     532             :   }
     533             :   assert(Lane < RegElts && "out of range lane for VLD/VST-lane");
     534             : 
     535         163 :   unsigned D0 = 0, D1 = 0, D2 = 0, D3 = 0;
     536         163 :   unsigned DstReg = 0;
     537         163 :   bool DstIsDead = false;
     538         163 :   if (TableEntry->IsLoad) {
     539         148 :     DstIsDead = MI.getOperand(OpIdx).isDead();
     540          74 :     DstReg = MI.getOperand(OpIdx++).getReg();
     541          74 :     GetDSubRegs(DstReg, RegSpc, TRI, D0, D1, D2, D3);
     542          74 :     MIB.addReg(D0, RegState::Define | getDeadRegState(DstIsDead));
     543          74 :     if (NumRegs > 1)
     544          60 :       MIB.addReg(D1, RegState::Define | getDeadRegState(DstIsDead));
     545          74 :     if (NumRegs > 2)
     546          37 :       MIB.addReg(D2, RegState::Define | getDeadRegState(DstIsDead));
     547          74 :     if (NumRegs > 3)
     548          17 :       MIB.addReg(D3, RegState::Define | getDeadRegState(DstIsDead));
     549             :   }
     550             : 
     551         163 :   if (TableEntry->isUpdating)
     552          22 :     MIB.add(MI.getOperand(OpIdx++));
     553             : 
     554             :   // Copy the addrmode6 operands.
     555         489 :   MIB.add(MI.getOperand(OpIdx++));
     556         489 :   MIB.add(MI.getOperand(OpIdx++));
     557             :   // Copy the am6offset operand.
     558         163 :   if (TableEntry->hasWritebackOperand)
     559          22 :     MIB.add(MI.getOperand(OpIdx++));
     560             : 
     561             :   // Grab the super-register source.
     562         326 :   MachineOperand MO = MI.getOperand(OpIdx++);
     563         163 :   if (!TableEntry->IsLoad)
     564          89 :     GetDSubRegs(MO.getReg(), RegSpc, TRI, D0, D1, D2, D3);
     565             : 
     566             :   // Add the subregs as sources of the new instruction.
     567         326 :   unsigned SrcFlags = (getUndefRegState(MO.isUndef()) |
     568         326 :                        getKillRegState(MO.isKill()));
     569         163 :   MIB.addReg(D0, SrcFlags);
     570         163 :   if (NumRegs > 1)
     571          87 :     MIB.addReg(D1, SrcFlags);
     572         163 :   if (NumRegs > 2)
     573          55 :     MIB.addReg(D2, SrcFlags);
     574         163 :   if (NumRegs > 3)
     575          26 :     MIB.addReg(D3, SrcFlags);
     576             : 
     577             :   // Add the lane number operand.
     578         326 :   MIB.addImm(Lane);
     579         163 :   OpIdx += 1;
     580             : 
     581             :   // Copy the predicate operands.
     582         489 :   MIB.add(MI.getOperand(OpIdx++));
     583         489 :   MIB.add(MI.getOperand(OpIdx++));
     584             : 
     585             :   // Copy the super-register source to be an implicit source.
     586         163 :   MO.setImplicit(true);
     587         163 :   MIB.add(MO);
     588         163 :   if (TableEntry->IsLoad)
     589             :     // Add an implicit def for the super-register.
     590          74 :     MIB.addReg(DstReg, RegState::ImplicitDefine | getDeadRegState(DstIsDead));
     591         163 :   TransferImpOps(MI, MIB, MIB);
     592             :   // Transfer memoperands.
     593         489 :   MIB->setMemRefs(MI.memoperands_begin(), MI.memoperands_end());
     594         163 :   MI.eraseFromParent();
     595         163 : }
     596             : 
     597             : /// ExpandVTBL - Translate VTBL and VTBX pseudo instructions with Q or QQ
     598             : /// register operands to real instructions with D register operands.
     599           5 : void ARMExpandPseudo::ExpandVTBL(MachineBasicBlock::iterator &MBBI,
     600             :                                  unsigned Opc, bool IsExt) {
     601           5 :   MachineInstr &MI = *MBBI;
     602           5 :   MachineBasicBlock &MBB = *MI.getParent();
     603             : 
     604          15 :   MachineInstrBuilder MIB = BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(Opc));
     605           5 :   unsigned OpIdx = 0;
     606             : 
     607             :   // Transfer the destination register operand.
     608          10 :   MIB.add(MI.getOperand(OpIdx++));
     609           5 :   if (IsExt)
     610           6 :     MIB.add(MI.getOperand(OpIdx++));
     611             : 
     612          15 :   bool SrcIsKill = MI.getOperand(OpIdx).isKill();
     613           5 :   unsigned SrcReg = MI.getOperand(OpIdx++).getReg();
     614             :   unsigned D0, D1, D2, D3;
     615           5 :   GetDSubRegs(SrcReg, SingleSpc, TRI, D0, D1, D2, D3);
     616           5 :   MIB.addReg(D0);
     617             : 
     618             :   // Copy the other source register operand.
     619          15 :   MIB.add(MI.getOperand(OpIdx++));
     620             : 
     621             :   // Copy the predicate operands.
     622          15 :   MIB.add(MI.getOperand(OpIdx++));
     623          15 :   MIB.add(MI.getOperand(OpIdx++));
     624             : 
     625             :   // Add an implicit kill and use for the super-reg.
     626           5 :   MIB.addReg(SrcReg, RegState::Implicit | getKillRegState(SrcIsKill));
     627           5 :   TransferImpOps(MI, MIB, MIB);
     628           5 :   MI.eraseFromParent();
     629           5 : }
     630             : 
     631          45 : static bool IsAnAddressOperand(const MachineOperand &MO) {
     632             :   // This check is overly conservative.  Unless we are certain that the machine
     633             :   // operand is not a symbol reference, we return that it is a symbol reference.
     634             :   // This is important as the load pair may not be split up Windows.
     635          45 :   switch (MO.getType()) {
     636             :   case MachineOperand::MO_Register:
     637             :   case MachineOperand::MO_Immediate:
     638             :   case MachineOperand::MO_CImmediate:
     639             :   case MachineOperand::MO_FPImmediate:
     640             :     return false;
     641           0 :   case MachineOperand::MO_MachineBasicBlock:
     642           0 :     return true;
     643             :   case MachineOperand::MO_FrameIndex:
     644             :     return false;
     645          45 :   case MachineOperand::MO_ConstantPoolIndex:
     646             :   case MachineOperand::MO_TargetIndex:
     647             :   case MachineOperand::MO_JumpTableIndex:
     648             :   case MachineOperand::MO_ExternalSymbol:
     649             :   case MachineOperand::MO_GlobalAddress:
     650             :   case MachineOperand::MO_BlockAddress:
     651          45 :     return true;
     652             :   case MachineOperand::MO_RegisterMask:
     653             :   case MachineOperand::MO_RegisterLiveOut:
     654             :     return false;
     655           0 :   case MachineOperand::MO_Metadata:
     656             :   case MachineOperand::MO_MCSymbol:
     657           0 :     return true;
     658             :   case MachineOperand::MO_CFIIndex:
     659             :     return false;
     660           0 :   case MachineOperand::MO_IntrinsicID:
     661             :   case MachineOperand::MO_Predicate:
     662           0 :     llvm_unreachable("should not exist post-isel");
     663             :   }
     664           0 :   llvm_unreachable("unhandled machine operand type");
     665             : }
     666             : 
     667             : static MachineOperand makeImplicit(const MachineOperand &MO) {
     668         972 :   MachineOperand NewMO = MO;
     669         972 :   NewMO.setImplicit();
     670             :   return NewMO;
     671             : }
     672             : 
     673        1437 : void ARMExpandPseudo::ExpandMOV32BitImm(MachineBasicBlock &MBB,
     674             :                                         MachineBasicBlock::iterator &MBBI) {
     675        1437 :   MachineInstr &MI = *MBBI;
     676        1437 :   unsigned Opcode = MI.getOpcode();
     677        1437 :   unsigned PredReg = 0;
     678        1437 :   ARMCC::CondCodes Pred = getInstrPredicate(MI, PredReg);
     679        1437 :   unsigned DstReg = MI.getOperand(0).getReg();
     680        2874 :   bool DstIsDead = MI.getOperand(0).isDead();
     681        1437 :   bool isCC = Opcode == ARM::MOVCCi32imm || Opcode == ARM::t2MOVCCi32imm;
     682        2874 :   const MachineOperand &MO = MI.getOperand(isCC ? 2 : 1);
     683        2874 :   bool RequiresBundling = STI->isTargetWindows() && IsAnAddressOperand(MO);
     684        1437 :   MachineInstrBuilder LO16, HI16;
     685             : 
     686        1506 :   if (!STI->hasV6T2Ops() &&
     687          69 :       (Opcode == ARM::MOVi32imm || Opcode == ARM::MOVCCi32imm)) {
     688             :     // FIXME Windows CE supports older ARM CPUs
     689             :     assert(!STI->isTargetWindows() && "Windows on ARM requires ARMv7+");
     690             : 
     691             :     // Expand into a movi + orr.
     692         174 :     LO16 = BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(ARM::MOVi), DstReg);
     693         174 :     HI16 = BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(ARM::ORRri))
     694          58 :       .addReg(DstReg, RegState::Define | getDeadRegState(DstIsDead))
     695          58 :       .addReg(DstReg);
     696             : 
     697             :     assert (MO.isImm() && "MOVi32imm w/ non-immediate source operand!");
     698          58 :     unsigned ImmVal = (unsigned)MO.getImm();
     699          58 :     unsigned SOImmValV1 = ARM_AM::getSOImmTwoPartFirst(ImmVal);
     700          58 :     unsigned SOImmValV2 = ARM_AM::getSOImmTwoPartSecond(ImmVal);
     701         116 :     LO16 = LO16.addImm(SOImmValV1);
     702         116 :     HI16 = HI16.addImm(SOImmValV2);
     703         174 :     LO16->setMemRefs(MI.memoperands_begin(), MI.memoperands_end());
     704         174 :     HI16->setMemRefs(MI.memoperands_begin(), MI.memoperands_end());
     705         232 :     LO16.addImm(Pred).addReg(PredReg).add(condCodeOp());
     706         232 :     HI16.addImm(Pred).addReg(PredReg).add(condCodeOp());
     707          58 :     if (isCC)
     708           0 :       LO16.add(makeImplicit(MI.getOperand(1)));
     709          58 :     TransferImpOps(MI, LO16, HI16);
     710          58 :     MI.eraseFromParent();
     711          58 :     return;
     712             :   }
     713             : 
     714        1379 :   unsigned LO16Opc = 0;
     715        1379 :   unsigned HI16Opc = 0;
     716        1379 :   if (Opcode == ARM::t2MOVi32imm || Opcode == ARM::t2MOVCCi32imm) {
     717             :     LO16Opc = ARM::t2MOVi16;
     718             :     HI16Opc = ARM::t2MOVTi16;
     719             :   } else {
     720         755 :     LO16Opc = ARM::MOVi16;
     721         755 :     HI16Opc = ARM::MOVTi16;
     722             :   }
     723             : 
     724        4137 :   LO16 = BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(LO16Opc), DstReg);
     725        4137 :   HI16 = BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(HI16Opc))
     726        1379 :     .addReg(DstReg, RegState::Define | getDeadRegState(DstIsDead))
     727        1379 :     .addReg(DstReg);
     728             : 
     729        1379 :   switch (MO.getType()) {
     730         471 :   case MachineOperand::MO_Immediate: {
     731         471 :     unsigned Imm = MO.getImm();
     732         471 :     unsigned Lo16 = Imm & 0xffff;
     733         471 :     unsigned Hi16 = (Imm >> 16) & 0xffff;
     734         942 :     LO16 = LO16.addImm(Lo16);
     735         942 :     HI16 = HI16.addImm(Hi16);
     736             :     break;
     737             :   }
     738          10 :   case MachineOperand::MO_ExternalSymbol: {
     739          10 :     const char *ES = MO.getSymbolName();
     740          10 :     unsigned TF = MO.getTargetFlags();
     741          20 :     LO16 = LO16.addExternalSymbol(ES, TF | ARMII::MO_LO16);
     742          20 :     HI16 = HI16.addExternalSymbol(ES, TF | ARMII::MO_HI16);
     743             :     break;
     744             :   }
     745         898 :   default: {
     746         898 :     const GlobalValue *GV = MO.getGlobal();
     747         898 :     unsigned TF = MO.getTargetFlags();
     748        2694 :     LO16 = LO16.addGlobalAddress(GV, MO.getOffset(), TF | ARMII::MO_LO16);
     749        2694 :     HI16 = HI16.addGlobalAddress(GV, MO.getOffset(), TF | ARMII::MO_HI16);
     750             :     break;
     751             :   }
     752             :   }
     753             : 
     754        4137 :   LO16->setMemRefs(MI.memoperands_begin(), MI.memoperands_end());
     755        4137 :   HI16->setMemRefs(MI.memoperands_begin(), MI.memoperands_end());
     756        2758 :   LO16.addImm(Pred).addReg(PredReg);
     757        2758 :   HI16.addImm(Pred).addReg(PredReg);
     758             : 
     759        1379 :   if (RequiresBundling)
     760         135 :     finalizeBundle(MBB, LO16->getIterator(), MBBI->getIterator());
     761             : 
     762        1379 :   if (isCC)
     763          42 :     LO16.add(makeImplicit(MI.getOperand(1)));
     764        1379 :   TransferImpOps(MI, LO16, HI16);
     765        1379 :   MI.eraseFromParent();
     766             : }
     767             : 
     768             : /// Expand a CMP_SWAP pseudo-inst to an ldrex/strex loop as simply as
     769             : /// possible. This only gets used at -O0 so we don't care about efficiency of
     770             : /// the generated code.
     771           6 : bool ARMExpandPseudo::ExpandCMP_SWAP(MachineBasicBlock &MBB,
     772             :                                      MachineBasicBlock::iterator MBBI,
     773             :                                      unsigned LdrexOp, unsigned StrexOp,
     774             :                                      unsigned UxtOp,
     775             :                                      MachineBasicBlock::iterator &NextMBBI) {
     776           6 :   bool IsThumb = STI->isThumb();
     777           6 :   MachineInstr &MI = *MBBI;
     778          18 :   DebugLoc DL = MI.getDebugLoc();
     779           6 :   const MachineOperand &Dest = MI.getOperand(0);
     780           6 :   unsigned TempReg = MI.getOperand(1).getReg();
     781             :   // Duplicating undef operands into 2 instructions does not guarantee the same
     782             :   // value on both; However undef should be replaced by xzr anyway.
     783             :   assert(!MI.getOperand(2).isUndef() && "cannot handle undef");
     784           6 :   unsigned AddrReg = MI.getOperand(2).getReg();
     785           6 :   unsigned DesiredReg = MI.getOperand(3).getReg();
     786           6 :   unsigned NewReg = MI.getOperand(4).getReg();
     787             : 
     788           6 :   MachineFunction *MF = MBB.getParent();
     789           6 :   auto LoadCmpBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
     790           6 :   auto StoreBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
     791           6 :   auto DoneBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
     792             : 
     793          24 :   MF->insert(++MBB.getIterator(), LoadCmpBB);
     794          24 :   MF->insert(++LoadCmpBB->getIterator(), StoreBB);
     795          24 :   MF->insert(++StoreBB->getIterator(), DoneBB);
     796             : 
     797           6 :   if (UxtOp) {
     798             :     MachineInstrBuilder MIB =
     799          12 :         BuildMI(MBB, MBBI, DL, TII->get(UxtOp), DesiredReg)
     800           4 :             .addReg(DesiredReg, RegState::Kill);
     801           4 :     if (!IsThumb)
     802             :       MIB.addImm(0);
     803          12 :     MIB.add(predOps(ARMCC::AL));
     804             :   }
     805             : 
     806             :   // .Lloadcmp:
     807             :   //     ldrex rDest, [rAddr]
     808             :   //     cmp rDest, rDesired
     809             :   //     bne .Ldone
     810             : 
     811           6 :   MachineInstrBuilder MIB;
     812          18 :   MIB = BuildMI(LoadCmpBB, DL, TII->get(LdrexOp), Dest.getReg());
     813           6 :   MIB.addReg(AddrReg);
     814           6 :   if (LdrexOp == ARM::t2LDREX)
     815             :     MIB.addImm(0); // a 32-bit Thumb ldrex (only) allows an offset.
     816          18 :   MIB.add(predOps(ARMCC::AL));
     817             : 
     818           6 :   unsigned CMPrr = IsThumb ? ARM::tCMPhir : ARM::CMPrr;
     819          24 :   BuildMI(LoadCmpBB, DL, TII->get(CMPrr))
     820          12 :       .addReg(Dest.getReg(), getKillRegState(Dest.isDead()))
     821           6 :       .addReg(DesiredReg)
     822          18 :       .add(predOps(ARMCC::AL));
     823           6 :   unsigned Bcc = IsThumb ? ARM::tBcc : ARM::Bcc;
     824          24 :   BuildMI(LoadCmpBB, DL, TII->get(Bcc))
     825           6 :       .addMBB(DoneBB)
     826           6 :       .addImm(ARMCC::NE)
     827           6 :       .addReg(ARM::CPSR, RegState::Kill);
     828           6 :   LoadCmpBB->addSuccessor(DoneBB);
     829           6 :   LoadCmpBB->addSuccessor(StoreBB);
     830             : 
     831             :   // .Lstore:
     832             :   //     strex rTempReg, rNew, [rAddr]
     833             :   //     cmp rTempReg, #0
     834             :   //     bne .Lloadcmp
     835          24 :   MIB = BuildMI(StoreBB, DL, TII->get(StrexOp), TempReg)
     836           6 :     .addReg(NewReg)
     837           6 :     .addReg(AddrReg);
     838           6 :   if (StrexOp == ARM::t2STREX)
     839             :     MIB.addImm(0); // a 32-bit Thumb strex (only) allows an offset.
     840          18 :   MIB.add(predOps(ARMCC::AL));
     841             : 
     842           6 :   unsigned CMPri = IsThumb ? ARM::t2CMPri : ARM::CMPri;
     843          24 :   BuildMI(StoreBB, DL, TII->get(CMPri))
     844           6 :       .addReg(TempReg, RegState::Kill)
     845           6 :       .addImm(0)
     846          18 :       .add(predOps(ARMCC::AL));
     847          24 :   BuildMI(StoreBB, DL, TII->get(Bcc))
     848           6 :       .addMBB(LoadCmpBB)
     849           6 :       .addImm(ARMCC::NE)
     850           6 :       .addReg(ARM::CPSR, RegState::Kill);
     851           6 :   StoreBB->addSuccessor(LoadCmpBB);
     852           6 :   StoreBB->addSuccessor(DoneBB);
     853             : 
     854          24 :   DoneBB->splice(DoneBB->end(), &MBB, MI, MBB.end());
     855           6 :   DoneBB->transferSuccessors(&MBB);
     856             : 
     857           6 :   MBB.addSuccessor(LoadCmpBB);
     858             : 
     859           6 :   NextMBBI = MBB.end();
     860           6 :   MI.eraseFromParent();
     861             : 
     862             :   // Recompute livein lists.
     863          12 :   LivePhysRegs LiveRegs;
     864           6 :   computeAndAddLiveIns(LiveRegs, *DoneBB);
     865           6 :   computeAndAddLiveIns(LiveRegs, *StoreBB);
     866           6 :   computeAndAddLiveIns(LiveRegs, *LoadCmpBB);
     867             :   // Do an extra pass around the loop to get loop carried registers right.
     868           6 :   StoreBB->clearLiveIns();
     869           6 :   computeAndAddLiveIns(LiveRegs, *StoreBB);
     870           6 :   LoadCmpBB->clearLiveIns();
     871           6 :   computeAndAddLiveIns(LiveRegs, *LoadCmpBB);
     872             : 
     873          12 :   return true;
     874             : }
     875             : 
     876             : /// ARM's ldrexd/strexd take a consecutive register pair (represented as a
     877             : /// single GPRPair register), Thumb's take two separate registers so we need to
     878             : /// extract the subregs from the pair.
     879          14 : static void addExclusiveRegPair(MachineInstrBuilder &MIB, MachineOperand &Reg,
     880             :                                 unsigned Flags, bool IsThumb,
     881             :                                 const TargetRegisterInfo *TRI) {
     882          14 :   if (IsThumb) {
     883           6 :     unsigned RegLo = TRI->getSubReg(Reg.getReg(), ARM::gsub_0);
     884           6 :     unsigned RegHi = TRI->getSubReg(Reg.getReg(), ARM::gsub_1);
     885          12 :     MIB.addReg(RegLo, Flags | getKillRegState(Reg.isDead()));
     886          12 :     MIB.addReg(RegHi, Flags | getKillRegState(Reg.isDead()));
     887             :   } else
     888          16 :     MIB.addReg(Reg.getReg(), Flags | getKillRegState(Reg.isDead()));
     889          14 : }
     890             : 
     891             : /// Expand a 64-bit CMP_SWAP to an ldrexd/strexd loop.
     892           7 : bool ARMExpandPseudo::ExpandCMP_SWAP_64(MachineBasicBlock &MBB,
     893             :                                         MachineBasicBlock::iterator MBBI,
     894             :                                         MachineBasicBlock::iterator &NextMBBI) {
     895           7 :   bool IsThumb = STI->isThumb();
     896           7 :   MachineInstr &MI = *MBBI;
     897          21 :   DebugLoc DL = MI.getDebugLoc();
     898           7 :   MachineOperand &Dest = MI.getOperand(0);
     899           7 :   unsigned TempReg = MI.getOperand(1).getReg();
     900             :   // Duplicating undef operands into 2 instructions does not guarantee the same
     901             :   // value on both; However undef should be replaced by xzr anyway.
     902             :   assert(!MI.getOperand(2).isUndef() && "cannot handle undef");
     903           7 :   unsigned AddrReg = MI.getOperand(2).getReg();
     904           7 :   unsigned DesiredReg = MI.getOperand(3).getReg();
     905           7 :   MachineOperand New = MI.getOperand(4);
     906           7 :   New.setIsKill(false);
     907             : 
     908           7 :   unsigned DestLo = TRI->getSubReg(Dest.getReg(), ARM::gsub_0);
     909           7 :   unsigned DestHi = TRI->getSubReg(Dest.getReg(), ARM::gsub_1);
     910           7 :   unsigned DesiredLo = TRI->getSubReg(DesiredReg, ARM::gsub_0);
     911           7 :   unsigned DesiredHi = TRI->getSubReg(DesiredReg, ARM::gsub_1);
     912             : 
     913           7 :   MachineFunction *MF = MBB.getParent();
     914           7 :   auto LoadCmpBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
     915           7 :   auto StoreBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
     916           7 :   auto DoneBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
     917             : 
     918          28 :   MF->insert(++MBB.getIterator(), LoadCmpBB);
     919          28 :   MF->insert(++LoadCmpBB->getIterator(), StoreBB);
     920          28 :   MF->insert(++StoreBB->getIterator(), DoneBB);
     921             : 
     922             :   // .Lloadcmp:
     923             :   //     ldrexd rDestLo, rDestHi, [rAddr]
     924             :   //     cmp rDestLo, rDesiredLo
     925             :   //     sbcs rTempReg<dead>, rDestHi, rDesiredHi
     926             :   //     bne .Ldone
     927           7 :   unsigned LDREXD = IsThumb ? ARM::t2LDREXD : ARM::LDREXD;
     928           7 :   MachineInstrBuilder MIB;
     929          21 :   MIB = BuildMI(LoadCmpBB, DL, TII->get(LDREXD));
     930           7 :   addExclusiveRegPair(MIB, Dest, RegState::Define, IsThumb, TRI);
     931          28 :   MIB.addReg(AddrReg).add(predOps(ARMCC::AL));
     932             : 
     933           7 :   unsigned CMPrr = IsThumb ? ARM::tCMPhir : ARM::CMPrr;
     934          28 :   BuildMI(LoadCmpBB, DL, TII->get(CMPrr))
     935          14 :       .addReg(DestLo, getKillRegState(Dest.isDead()))
     936           7 :       .addReg(DesiredLo)
     937          21 :       .add(predOps(ARMCC::AL));
     938             : 
     939          28 :   BuildMI(LoadCmpBB, DL, TII->get(CMPrr))
     940          14 :       .addReg(DestHi, getKillRegState(Dest.isDead()))
     941           7 :       .addReg(DesiredHi)
     942           7 :       .addImm(ARMCC::EQ).addReg(ARM::CPSR, RegState::Kill);
     943             : 
     944           7 :   unsigned Bcc = IsThumb ? ARM::tBcc : ARM::Bcc;
     945          28 :   BuildMI(LoadCmpBB, DL, TII->get(Bcc))
     946           7 :       .addMBB(DoneBB)
     947           7 :       .addImm(ARMCC::NE)
     948           7 :       .addReg(ARM::CPSR, RegState::Kill);
     949           7 :   LoadCmpBB->addSuccessor(DoneBB);
     950           7 :   LoadCmpBB->addSuccessor(StoreBB);
     951             : 
     952             :   // .Lstore:
     953             :   //     strexd rTempReg, rNewLo, rNewHi, [rAddr]
     954             :   //     cmp rTempReg, #0
     955             :   //     bne .Lloadcmp
     956           7 :   unsigned STREXD = IsThumb ? ARM::t2STREXD : ARM::STREXD;
     957          21 :   MIB = BuildMI(StoreBB, DL, TII->get(STREXD), TempReg);
     958           7 :   addExclusiveRegPair(MIB, New, 0, IsThumb, TRI);
     959          28 :   MIB.addReg(AddrReg).add(predOps(ARMCC::AL));
     960             : 
     961           7 :   unsigned CMPri = IsThumb ? ARM::t2CMPri : ARM::CMPri;
     962          28 :   BuildMI(StoreBB, DL, TII->get(CMPri))
     963           7 :       .addReg(TempReg, RegState::Kill)
     964           7 :       .addImm(0)
     965          21 :       .add(predOps(ARMCC::AL));
     966          28 :   BuildMI(StoreBB, DL, TII->get(Bcc))
     967           7 :       .addMBB(LoadCmpBB)
     968           7 :       .addImm(ARMCC::NE)
     969           7 :       .addReg(ARM::CPSR, RegState::Kill);
     970           7 :   StoreBB->addSuccessor(LoadCmpBB);
     971           7 :   StoreBB->addSuccessor(DoneBB);
     972             : 
     973          28 :   DoneBB->splice(DoneBB->end(), &MBB, MI, MBB.end());
     974           7 :   DoneBB->transferSuccessors(&MBB);
     975             : 
     976           7 :   MBB.addSuccessor(LoadCmpBB);
     977             : 
     978           7 :   NextMBBI = MBB.end();
     979           7 :   MI.eraseFromParent();
     980             : 
     981             :   // Recompute livein lists.
     982          14 :   LivePhysRegs LiveRegs;
     983           7 :   computeAndAddLiveIns(LiveRegs, *DoneBB);
     984           7 :   computeAndAddLiveIns(LiveRegs, *StoreBB);
     985           7 :   computeAndAddLiveIns(LiveRegs, *LoadCmpBB);
     986             :   // Do an extra pass around the loop to get loop carried registers right.
     987           7 :   StoreBB->clearLiveIns();
     988           7 :   computeAndAddLiveIns(LiveRegs, *StoreBB);
     989           7 :   LoadCmpBB->clearLiveIns();
     990           7 :   computeAndAddLiveIns(LiveRegs, *LoadCmpBB);
     991             : 
     992          14 :   return true;
     993             : }
     994             : 
     995             : 
     996      142963 : bool ARMExpandPseudo::ExpandMI(MachineBasicBlock &MBB,
     997             :                                MachineBasicBlock::iterator MBBI,
     998             :                                MachineBasicBlock::iterator &NextMBBI) {
     999      142963 :   MachineInstr &MI = *MBBI;
    1000      142963 :   unsigned Opcode = MI.getOpcode();
    1001      142963 :   switch (Opcode) {
    1002             :     default:
    1003             :       return false;
    1004             : 
    1005         504 :     case ARM::TCRETURNdi:
    1006             :     case ARM::TCRETURNri: {
    1007         504 :       MachineBasicBlock::iterator MBBI = MBB.getLastNonDebugInstr();
    1008             :       assert(MBBI->isReturn() &&
    1009             :              "Can only insert epilog into returning blocks");
    1010        1008 :       unsigned RetOpcode = MBBI->getOpcode();
    1011        1512 :       DebugLoc dl = MBBI->getDebugLoc();
    1012             :       const ARMBaseInstrInfo &TII = *static_cast<const ARMBaseInstrInfo *>(
    1013         504 :           MBB.getParent()->getSubtarget().getInstrInfo());
    1014             : 
    1015             :       // Tail call return: adjust the stack pointer and jump to callee.
    1016         504 :       MBBI = MBB.getLastNonDebugInstr();
    1017         504 :       MachineOperand &JumpTarget = MBBI->getOperand(0);
    1018             : 
    1019             :       // Jump to label or value in register.
    1020         504 :       if (RetOpcode == ARM::TCRETURNdi) {
    1021             :         unsigned TCOpcode =
    1022         488 :             STI->isThumb()
    1023         795 :                 ? (STI->isTargetMachO() ? ARM::tTAILJMPd : ARM::tTAILJMPdND)
    1024         488 :                 : ARM::TAILJMPd;
    1025         976 :         MachineInstrBuilder MIB = BuildMI(MBB, MBBI, dl, TII.get(TCOpcode));
    1026         488 :         if (JumpTarget.isGlobal())
    1027             :           MIB.addGlobalAddress(JumpTarget.getGlobal(), JumpTarget.getOffset(),
    1028         618 :                                JumpTarget.getTargetFlags());
    1029             :         else {
    1030             :           assert(JumpTarget.isSymbol());
    1031             :           MIB.addExternalSymbol(JumpTarget.getSymbolName(),
    1032         179 :                                 JumpTarget.getTargetFlags());
    1033             :         }
    1034             : 
    1035             :         // Add the default predicate in Thumb mode.
    1036         488 :         if (STI->isThumb())
    1037         921 :           MIB.add(predOps(ARMCC::AL));
    1038          16 :       } else if (RetOpcode == ARM::TCRETURNri) {
    1039             :         unsigned Opcode =
    1040          25 :           STI->isThumb() ? ARM::tTAILJMPr
    1041          25 :                          : (STI->hasV4TOps() ? ARM::TAILJMPr : ARM::TAILJMPr4);
    1042          32 :         BuildMI(MBB, MBBI, dl,
    1043          32 :                 TII.get(Opcode))
    1044          16 :             .addReg(JumpTarget.getReg(), RegState::Kill);
    1045             :       }
    1046             : 
    1047         504 :       auto NewMI = std::prev(MBBI);
    1048        2013 :       for (unsigned i = 1, e = MBBI->getNumOperands(); i != e; ++i)
    1049        3015 :         NewMI->addOperand(MBBI->getOperand(i));
    1050             : 
    1051             :       // Delete the pseudo instruction TCRETURN.
    1052         504 :       MBB.erase(MBBI);
    1053         504 :       MBBI = NewMI;
    1054         504 :       return true;
    1055             :     }
    1056          47 :     case ARM::VMOVScc:
    1057             :     case ARM::VMOVDcc: {
    1058          47 :       unsigned newOpc = Opcode == ARM::VMOVScc ? ARM::VMOVS : ARM::VMOVD;
    1059          94 :       BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(newOpc),
    1060         188 :               MI.getOperand(1).getReg())
    1061         141 :           .add(MI.getOperand(2))
    1062          94 :           .addImm(MI.getOperand(3).getImm()) // 'pred'
    1063         141 :           .add(MI.getOperand(4))
    1064         141 :           .add(makeImplicit(MI.getOperand(1)));
    1065             : 
    1066          47 :       MI.eraseFromParent();
    1067          47 :       return true;
    1068             :     }
    1069         254 :     case ARM::t2MOVCCr:
    1070             :     case ARM::MOVCCr: {
    1071         254 :       unsigned Opc = AFI->isThumbFunction() ? ARM::t2MOVr : ARM::MOVr;
    1072         508 :       BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(Opc),
    1073        1016 :               MI.getOperand(1).getReg())
    1074         762 :           .add(MI.getOperand(2))
    1075         508 :           .addImm(MI.getOperand(3).getImm()) // 'pred'
    1076         762 :           .add(MI.getOperand(4))
    1077         762 :           .add(condCodeOp()) // 's' bit
    1078         762 :           .add(makeImplicit(MI.getOperand(1)));
    1079             : 
    1080         254 :       MI.eraseFromParent();
    1081         254 :       return true;
    1082             :     }
    1083           7 :     case ARM::MOVCCsi: {
    1084          14 :       BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(ARM::MOVsi),
    1085          28 :               (MI.getOperand(1).getReg()))
    1086          21 :           .add(MI.getOperand(2))
    1087          14 :           .addImm(MI.getOperand(3).getImm())
    1088          14 :           .addImm(MI.getOperand(4).getImm()) // 'pred'
    1089          21 :           .add(MI.getOperand(5))
    1090          21 :           .add(condCodeOp()) // 's' bit
    1091          21 :           .add(makeImplicit(MI.getOperand(1)));
    1092             : 
    1093           7 :       MI.eraseFromParent();
    1094           7 :       return true;
    1095             :     }
    1096           7 :     case ARM::MOVCCsr: {
    1097          14 :       BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(ARM::MOVsr),
    1098          28 :               (MI.getOperand(1).getReg()))
    1099          21 :           .add(MI.getOperand(2))
    1100          21 :           .add(MI.getOperand(3))
    1101          14 :           .addImm(MI.getOperand(4).getImm())
    1102          14 :           .addImm(MI.getOperand(5).getImm()) // 'pred'
    1103          21 :           .add(MI.getOperand(6))
    1104          21 :           .add(condCodeOp()) // 's' bit
    1105          21 :           .add(makeImplicit(MI.getOperand(1)));
    1106             : 
    1107           7 :       MI.eraseFromParent();
    1108           7 :       return true;
    1109             :     }
    1110          65 :     case ARM::t2MOVCCi16:
    1111             :     case ARM::MOVCCi16: {
    1112          65 :       unsigned NewOpc = AFI->isThumbFunction() ? ARM::t2MOVi16 : ARM::MOVi16;
    1113         130 :       BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(NewOpc),
    1114         260 :               MI.getOperand(1).getReg())
    1115         130 :           .addImm(MI.getOperand(2).getImm())
    1116         130 :           .addImm(MI.getOperand(3).getImm()) // 'pred'
    1117         195 :           .add(MI.getOperand(4))
    1118         195 :           .add(makeImplicit(MI.getOperand(1)));
    1119          65 :       MI.eraseFromParent();
    1120          65 :       return true;
    1121             :     }
    1122         528 :     case ARM::t2MOVCCi:
    1123             :     case ARM::MOVCCi: {
    1124         528 :       unsigned Opc = AFI->isThumbFunction() ? ARM::t2MOVi : ARM::MOVi;
    1125        1056 :       BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(Opc),
    1126        2112 :               MI.getOperand(1).getReg())
    1127        1056 :           .addImm(MI.getOperand(2).getImm())
    1128        1056 :           .addImm(MI.getOperand(3).getImm()) // 'pred'
    1129        1584 :           .add(MI.getOperand(4))
    1130        1584 :           .add(condCodeOp()) // 's' bit
    1131        1584 :           .add(makeImplicit(MI.getOperand(1)));
    1132             : 
    1133         528 :       MI.eraseFromParent();
    1134         528 :       return true;
    1135             :     }
    1136          47 :     case ARM::t2MVNCCi:
    1137             :     case ARM::MVNCCi: {
    1138          47 :       unsigned Opc = AFI->isThumbFunction() ? ARM::t2MVNi : ARM::MVNi;
    1139          94 :       BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(Opc),
    1140         188 :               MI.getOperand(1).getReg())
    1141          94 :           .addImm(MI.getOperand(2).getImm())
    1142          94 :           .addImm(MI.getOperand(3).getImm()) // 'pred'
    1143         141 :           .add(MI.getOperand(4))
    1144         141 :           .add(condCodeOp()) // 's' bit
    1145         141 :           .add(makeImplicit(MI.getOperand(1)));
    1146             : 
    1147          47 :       MI.eraseFromParent();
    1148          47 :       return true;
    1149             :     }
    1150           3 :     case ARM::t2MOVCClsl:
    1151             :     case ARM::t2MOVCClsr:
    1152             :     case ARM::t2MOVCCasr:
    1153             :     case ARM::t2MOVCCror: {
    1154             :       unsigned NewOpc;
    1155           3 :       switch (Opcode) {
    1156             :       case ARM::t2MOVCClsl: NewOpc = ARM::t2LSLri; break;
    1157           1 :       case ARM::t2MOVCClsr: NewOpc = ARM::t2LSRri; break;
    1158           0 :       case ARM::t2MOVCCasr: NewOpc = ARM::t2ASRri; break;
    1159           1 :       case ARM::t2MOVCCror: NewOpc = ARM::t2RORri; break;
    1160           0 :       default: llvm_unreachable("unexpeced conditional move");
    1161             :       }
    1162           6 :       BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(NewOpc),
    1163          12 :               MI.getOperand(1).getReg())
    1164           9 :           .add(MI.getOperand(2))
    1165           6 :           .addImm(MI.getOperand(3).getImm())
    1166           6 :           .addImm(MI.getOperand(4).getImm()) // 'pred'
    1167           9 :           .add(MI.getOperand(5))
    1168           9 :           .add(condCodeOp()) // 's' bit
    1169           9 :           .add(makeImplicit(MI.getOperand(1)));
    1170           3 :       MI.eraseFromParent();
    1171           3 :       return true;
    1172             :     }
    1173          28 :     case ARM::Int_eh_sjlj_dispatchsetup: {
    1174          28 :       MachineFunction &MF = *MI.getParent()->getParent();
    1175          28 :       const ARMBaseInstrInfo *AII =
    1176             :         static_cast<const ARMBaseInstrInfo*>(TII);
    1177          28 :       const ARMBaseRegisterInfo &RI = AII->getRegisterInfo();
    1178             :       // For functions using a base pointer, we rematerialize it (via the frame
    1179             :       // pointer) here since eh.sjlj.setjmp and eh.sjlj.longjmp don't do it
    1180             :       // for us. Otherwise, expand to nothing.
    1181          28 :       if (RI.hasBasePointer(MF)) {
    1182           0 :         int32_t NumBytes = AFI->getFramePtrSpillOffset();
    1183           0 :         unsigned FramePtr = RI.getFrameRegister(MF);
    1184             :         assert(MF.getSubtarget().getFrameLowering()->hasFP(MF) &&
    1185             :                "base pointer without frame pointer?");
    1186             : 
    1187           0 :         if (AFI->isThumb2Function()) {
    1188           0 :           emitT2RegPlusImmediate(MBB, MBBI, MI.getDebugLoc(), ARM::R6,
    1189           0 :                                  FramePtr, -NumBytes, ARMCC::AL, 0, *TII);
    1190           0 :         } else if (AFI->isThumbFunction()) {
    1191           0 :           emitThumbRegPlusImmediate(MBB, MBBI, MI.getDebugLoc(), ARM::R6,
    1192           0 :                                     FramePtr, -NumBytes, *TII, RI);
    1193             :         } else {
    1194           0 :           emitARMRegPlusImmediate(MBB, MBBI, MI.getDebugLoc(), ARM::R6,
    1195             :                                   FramePtr, -NumBytes, ARMCC::AL, 0,
    1196           0 :                                   *TII);
    1197             :         }
    1198             :         // If there's dynamic realignment, adjust for it.
    1199           0 :         if (RI.needsStackRealignment(MF)) {
    1200           0 :           MachineFrameInfo &MFI = MF.getFrameInfo();
    1201           0 :           unsigned MaxAlign = MFI.getMaxAlignment();
    1202             :           assert (!AFI->isThumb1OnlyFunction());
    1203             :           // Emit bic r6, r6, MaxAlign
    1204             :           assert(MaxAlign <= 256 && "The BIC instruction cannot encode "
    1205             :                                     "immediates larger than 256 with all lower "
    1206             :                                     "bits set.");
    1207           0 :           unsigned bicOpc = AFI->isThumbFunction() ?
    1208           0 :             ARM::t2BICri : ARM::BICri;
    1209           0 :           BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(bicOpc), ARM::R6)
    1210           0 :               .addReg(ARM::R6, RegState::Kill)
    1211           0 :               .addImm(MaxAlign - 1)
    1212           0 :               .add(predOps(ARMCC::AL))
    1213           0 :               .add(condCodeOp());
    1214             :         }
    1215             : 
    1216             :       }
    1217          28 :       MI.eraseFromParent();
    1218          28 :       return true;
    1219             :     }
    1220             : 
    1221           2 :     case ARM::MOVsrl_flag:
    1222             :     case ARM::MOVsra_flag: {
    1223             :       // These are just fancy MOVs instructions.
    1224           6 :       BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(ARM::MOVsi),
    1225           8 :               MI.getOperand(0).getReg())
    1226           6 :           .add(MI.getOperand(1))
    1227           4 :           .addImm(ARM_AM::getSORegOpc(
    1228           4 :               (Opcode == ARM::MOVsrl_flag ? ARM_AM::lsr : ARM_AM::asr), 1))
    1229           8 :           .add(predOps(ARMCC::AL))
    1230           2 :           .addReg(ARM::CPSR, RegState::Define);
    1231           2 :       MI.eraseFromParent();
    1232           2 :       return true;
    1233             :     }
    1234           2 :     case ARM::RRX: {
    1235             :       // This encodes as "MOVs Rd, Rm, rrx
    1236             :       MachineInstrBuilder MIB =
    1237           4 :           BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(ARM::MOVsi),
    1238           8 :                   MI.getOperand(0).getReg())
    1239           6 :               .add(MI.getOperand(1))
    1240           4 :               .addImm(ARM_AM::getSORegOpc(ARM_AM::rrx, 0))
    1241           8 :               .add(predOps(ARMCC::AL))
    1242           4 :               .add(condCodeOp());
    1243           2 :       TransferImpOps(MI, MIB, MIB);
    1244           2 :       MI.eraseFromParent();
    1245             :       return true;
    1246             :     }
    1247          39 :     case ARM::tTPsoft:
    1248             :     case ARM::TPsoft: {
    1249          39 :       const bool Thumb = Opcode == ARM::tTPsoft;
    1250             : 
    1251          39 :       MachineInstrBuilder MIB;
    1252          39 :       if (STI->genLongCalls()) {
    1253           2 :         MachineFunction *MF = MBB.getParent();
    1254           2 :         MachineConstantPool *MCP = MF->getConstantPool();
    1255           4 :         unsigned PCLabelID = AFI->createPICLabelUId();
    1256             :         MachineConstantPoolValue *CPV =
    1257           4 :             ARMConstantPoolSymbol::Create(MF->getFunction()->getContext(),
    1258           2 :                                           "__aeabi_read_tp", PCLabelID, 0);
    1259           2 :         unsigned Reg = MI.getOperand(0).getReg();
    1260           4 :         MIB = BuildMI(MBB, MBBI, MI.getDebugLoc(),
    1261           6 :                       TII->get(Thumb ? ARM::tLDRpci : ARM::LDRi12), Reg)
    1262           4 :                   .addConstantPoolIndex(MCP->getConstantPoolIndex(CPV, 4));
    1263           2 :         if (!Thumb)
    1264             :           MIB.addImm(0);
    1265           6 :         MIB.add(predOps(ARMCC::AL));
    1266             : 
    1267           2 :         MIB = BuildMI(MBB, MBBI, MI.getDebugLoc(),
    1268           4 :                       TII->get(Thumb ? ARM::tBLXr : ARM::BLX));
    1269           2 :         if (Thumb)
    1270           3 :           MIB.add(predOps(ARMCC::AL));
    1271           2 :         MIB.addReg(Reg, RegState::Kill);
    1272             :       } else {
    1273          37 :         MIB = BuildMI(MBB, MBBI, MI.getDebugLoc(),
    1274         111 :                       TII->get(Thumb ? ARM::tBL : ARM::BL));
    1275          37 :         if (Thumb)
    1276          33 :           MIB.add(predOps(ARMCC::AL));
    1277             :         MIB.addExternalSymbol("__aeabi_read_tp", 0);
    1278             :       }
    1279             : 
    1280         117 :       MIB->setMemRefs(MI.memoperands_begin(), MI.memoperands_end());
    1281          39 :       TransferImpOps(MI, MIB, MIB);
    1282          39 :       MI.eraseFromParent();
    1283             :       return true;
    1284             :     }
    1285          71 :     case ARM::tLDRpci_pic:
    1286             :     case ARM::t2LDRpci_pic: {
    1287          71 :       unsigned NewLdOpc = (Opcode == ARM::tLDRpci_pic)
    1288          71 :         ? ARM::tLDRpci : ARM::t2LDRpci;
    1289          71 :       unsigned DstReg = MI.getOperand(0).getReg();
    1290         142 :       bool DstIsDead = MI.getOperand(0).isDead();
    1291             :       MachineInstrBuilder MIB1 =
    1292         213 :           BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(NewLdOpc), DstReg)
    1293         213 :               .add(MI.getOperand(1))
    1294         213 :               .add(predOps(ARMCC::AL));
    1295         213 :       MIB1->setMemRefs(MI.memoperands_begin(), MI.memoperands_end());
    1296             :       MachineInstrBuilder MIB2 =
    1297         213 :           BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(ARM::tPICADD))
    1298          71 :               .addReg(DstReg, RegState::Define | getDeadRegState(DstIsDead))
    1299          71 :               .addReg(DstReg)
    1300         213 :               .add(MI.getOperand(2));
    1301          71 :       TransferImpOps(MI, MIB1, MIB2);
    1302          71 :       MI.eraseFromParent();
    1303             :       return true;
    1304             :     }
    1305             : 
    1306         244 :     case ARM::LDRLIT_ga_abs:
    1307             :     case ARM::LDRLIT_ga_pcrel:
    1308             :     case ARM::LDRLIT_ga_pcrel_ldr:
    1309             :     case ARM::tLDRLIT_ga_abs:
    1310             :     case ARM::tLDRLIT_ga_pcrel: {
    1311         244 :       unsigned DstReg = MI.getOperand(0).getReg();
    1312         488 :       bool DstIsDead = MI.getOperand(0).isDead();
    1313         244 :       const MachineOperand &MO1 = MI.getOperand(1);
    1314         244 :       const GlobalValue *GV = MO1.getGlobal();
    1315         244 :       bool IsARM =
    1316         244 :           Opcode != ARM::tLDRLIT_ga_pcrel && Opcode != ARM::tLDRLIT_ga_abs;
    1317         244 :       bool IsPIC =
    1318         244 :           Opcode != ARM::LDRLIT_ga_abs && Opcode != ARM::tLDRLIT_ga_abs;
    1319         244 :       unsigned LDRLITOpc = IsARM ? ARM::LDRi12 : ARM::tLDRpci;
    1320         244 :       unsigned PICAddOpc =
    1321             :           IsARM
    1322         244 :               ? (Opcode == ARM::LDRLIT_ga_pcrel_ldr ? ARM::PICLDR : ARM::PICADD)
    1323             :               : ARM::tPICADD;
    1324             : 
    1325             :       // We need a new const-pool entry to load from.
    1326         244 :       MachineConstantPool *MCP = MBB.getParent()->getConstantPool();
    1327         244 :       unsigned ARMPCLabelIndex = 0;
    1328             :       MachineConstantPoolValue *CPV;
    1329             : 
    1330         244 :       if (IsPIC) {
    1331         207 :         unsigned PCAdj = IsARM ? 8 : 4;
    1332         207 :         auto Modifier = STI->getCPModifier(GV);
    1333         414 :         ARMPCLabelIndex = AFI->createPICLabelUId();
    1334         207 :         CPV = ARMConstantPoolConstant::Create(
    1335             :             GV, ARMPCLabelIndex, ARMCP::CPValue, PCAdj, Modifier,
    1336             :             /*AddCurrentAddr*/ Modifier == ARMCP::GOT_PREL);
    1337             :       } else
    1338          37 :         CPV = ARMConstantPoolConstant::Create(GV, ARMCP::no_modifier);
    1339             : 
    1340             :       MachineInstrBuilder MIB =
    1341         732 :           BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(LDRLITOpc), DstReg)
    1342         488 :             .addConstantPoolIndex(MCP->getConstantPoolIndex(CPV, 4));
    1343         244 :       if (IsARM)
    1344             :         MIB.addImm(0);
    1345         732 :       MIB.add(predOps(ARMCC::AL));
    1346             : 
    1347         244 :       if (IsPIC) {
    1348             :         MachineInstrBuilder MIB =
    1349         621 :           BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(PICAddOpc))
    1350         207 :             .addReg(DstReg, RegState::Define | getDeadRegState(DstIsDead))
    1351         207 :             .addReg(DstReg)
    1352         414 :             .addImm(ARMPCLabelIndex);
    1353             : 
    1354         207 :         if (IsARM)
    1355         276 :           MIB.add(predOps(ARMCC::AL));
    1356             :       }
    1357             : 
    1358         244 :       MI.eraseFromParent();
    1359             :       return true;
    1360             :     }
    1361         566 :     case ARM::MOV_ga_pcrel:
    1362             :     case ARM::MOV_ga_pcrel_ldr:
    1363             :     case ARM::t2MOV_ga_pcrel: {
    1364             :       // Expand into movw + movw. Also "add pc" / ldr [pc] in PIC mode.
    1365        1132 :       unsigned LabelId = AFI->createPICLabelUId();
    1366         566 :       unsigned DstReg = MI.getOperand(0).getReg();
    1367        1132 :       bool DstIsDead = MI.getOperand(0).isDead();
    1368         566 :       const MachineOperand &MO1 = MI.getOperand(1);
    1369         566 :       const GlobalValue *GV = MO1.getGlobal();
    1370         566 :       unsigned TF = MO1.getTargetFlags();
    1371         566 :       bool isARM = Opcode != ARM::t2MOV_ga_pcrel;
    1372         566 :       unsigned LO16Opc = isARM ? ARM::MOVi16_ga_pcrel : ARM::t2MOVi16_ga_pcrel;
    1373         566 :       unsigned HI16Opc = isARM ? ARM::MOVTi16_ga_pcrel :ARM::t2MOVTi16_ga_pcrel;
    1374         566 :       unsigned LO16TF = TF | ARMII::MO_LO16;
    1375         566 :       unsigned HI16TF = TF | ARMII::MO_HI16;
    1376         566 :       unsigned PICAddOpc = isARM
    1377         566 :         ? (Opcode == ARM::MOV_ga_pcrel_ldr ? ARM::PICLDR : ARM::PICADD)
    1378             :         : ARM::tPICADD;
    1379         566 :       MachineInstrBuilder MIB1 = BuildMI(MBB, MBBI, MI.getDebugLoc(),
    1380        1698 :                                          TII->get(LO16Opc), DstReg)
    1381        1132 :         .addGlobalAddress(GV, MO1.getOffset(), TF | LO16TF)
    1382        1132 :         .addImm(LabelId);
    1383             : 
    1384        1698 :       BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(HI16Opc), DstReg)
    1385         566 :         .addReg(DstReg)
    1386        1132 :         .addGlobalAddress(GV, MO1.getOffset(), TF | HI16TF)
    1387        1132 :         .addImm(LabelId);
    1388             : 
    1389        1132 :       MachineInstrBuilder MIB3 = BuildMI(MBB, MBBI, MI.getDebugLoc(),
    1390        1132 :                                          TII->get(PICAddOpc))
    1391         566 :         .addReg(DstReg, RegState::Define | getDeadRegState(DstIsDead))
    1392        1132 :         .addReg(DstReg).addImm(LabelId);
    1393         566 :       if (isARM) {
    1394         639 :         MIB3.add(predOps(ARMCC::AL));
    1395         213 :         if (Opcode == ARM::MOV_ga_pcrel_ldr)
    1396         256 :           MIB3->setMemRefs(MI.memoperands_begin(), MI.memoperands_end());
    1397             :       }
    1398         566 :       TransferImpOps(MI, MIB1, MIB3);
    1399         566 :       MI.eraseFromParent();
    1400             :       return true;
    1401             :     }
    1402             : 
    1403        1437 :     case ARM::MOVi32imm:
    1404             :     case ARM::MOVCCi32imm:
    1405             :     case ARM::t2MOVi32imm:
    1406             :     case ARM::t2MOVCCi32imm:
    1407        1437 :       ExpandMOV32BitImm(MBB, MBBI);
    1408        1437 :       return true;
    1409             : 
    1410           6 :     case ARM::SUBS_PC_LR: {
    1411             :       MachineInstrBuilder MIB =
    1412          24 :           BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(ARM::SUBri), ARM::PC)
    1413           6 :               .addReg(ARM::LR)
    1414          12 :               .add(MI.getOperand(0))
    1415          18 :               .add(MI.getOperand(1))
    1416          18 :               .add(MI.getOperand(2))
    1417           6 :               .addReg(ARM::CPSR, RegState::Undef);
    1418           6 :       TransferImpOps(MI, MIB, MIB);
    1419           6 :       MI.eraseFromParent();
    1420             :       return true;
    1421             :     }
    1422           3 :     case ARM::VLDMQIA: {
    1423           3 :       unsigned NewOpc = ARM::VLDMDIA;
    1424             :       MachineInstrBuilder MIB =
    1425           9 :         BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(NewOpc));
    1426           3 :       unsigned OpIdx = 0;
    1427             : 
    1428             :       // Grab the Q register destination.
    1429           6 :       bool DstIsDead = MI.getOperand(OpIdx).isDead();
    1430           3 :       unsigned DstReg = MI.getOperand(OpIdx++).getReg();
    1431             : 
    1432             :       // Copy the source register.
    1433           9 :       MIB.add(MI.getOperand(OpIdx++));
    1434             : 
    1435             :       // Copy the predicate operands.
    1436           9 :       MIB.add(MI.getOperand(OpIdx++));
    1437           9 :       MIB.add(MI.getOperand(OpIdx++));
    1438             : 
    1439             :       // Add the destination operands (D subregs).
    1440           3 :       unsigned D0 = TRI->getSubReg(DstReg, ARM::dsub_0);
    1441           3 :       unsigned D1 = TRI->getSubReg(DstReg, ARM::dsub_1);
    1442           3 :       MIB.addReg(D0, RegState::Define | getDeadRegState(DstIsDead))
    1443           3 :         .addReg(D1, RegState::Define | getDeadRegState(DstIsDead));
    1444             : 
    1445             :       // Add an implicit def for the super-register.
    1446           3 :       MIB.addReg(DstReg, RegState::ImplicitDefine | getDeadRegState(DstIsDead));
    1447           3 :       TransferImpOps(MI, MIB, MIB);
    1448           6 :       MIB.setMemRefs(MI.memoperands_begin(), MI.memoperands_end());
    1449           3 :       MI.eraseFromParent();
    1450             :       return true;
    1451             :     }
    1452             : 
    1453           8 :     case ARM::VSTMQIA: {
    1454           8 :       unsigned NewOpc = ARM::VSTMDIA;
    1455             :       MachineInstrBuilder MIB =
    1456          24 :         BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(NewOpc));
    1457           8 :       unsigned OpIdx = 0;
    1458             : 
    1459             :       // Grab the Q register source.
    1460          16 :       bool SrcIsKill = MI.getOperand(OpIdx).isKill();
    1461           8 :       unsigned SrcReg = MI.getOperand(OpIdx++).getReg();
    1462             : 
    1463             :       // Copy the destination register.
    1464          24 :       MIB.add(MI.getOperand(OpIdx++));
    1465             : 
    1466             :       // Copy the predicate operands.
    1467          24 :       MIB.add(MI.getOperand(OpIdx++));
    1468          24 :       MIB.add(MI.getOperand(OpIdx++));
    1469             : 
    1470             :       // Add the source operands (D subregs).
    1471           8 :       unsigned D0 = TRI->getSubReg(SrcReg, ARM::dsub_0);
    1472           8 :       unsigned D1 = TRI->getSubReg(SrcReg, ARM::dsub_1);
    1473           8 :       MIB.addReg(D0, SrcIsKill ? RegState::Kill : 0)
    1474           8 :          .addReg(D1, SrcIsKill ? RegState::Kill : 0);
    1475             : 
    1476           8 :       if (SrcIsKill)      // Add an implicit kill for the Q register.
    1477           7 :         MIB->addRegisterKilled(SrcReg, TRI, true);
    1478             : 
    1479           8 :       TransferImpOps(MI, MIB, MIB);
    1480          16 :       MIB.setMemRefs(MI.memoperands_begin(), MI.memoperands_end());
    1481           8 :       MI.eraseFromParent();
    1482             :       return true;
    1483             :     }
    1484             : 
    1485          95 :     case ARM::VLD2q8Pseudo:
    1486             :     case ARM::VLD2q16Pseudo:
    1487             :     case ARM::VLD2q32Pseudo:
    1488             :     case ARM::VLD2q8PseudoWB_fixed:
    1489             :     case ARM::VLD2q16PseudoWB_fixed:
    1490             :     case ARM::VLD2q32PseudoWB_fixed:
    1491             :     case ARM::VLD2q8PseudoWB_register:
    1492             :     case ARM::VLD2q16PseudoWB_register:
    1493             :     case ARM::VLD2q32PseudoWB_register:
    1494             :     case ARM::VLD3d8Pseudo:
    1495             :     case ARM::VLD3d16Pseudo:
    1496             :     case ARM::VLD3d32Pseudo:
    1497             :     case ARM::VLD1d64TPseudo:
    1498             :     case ARM::VLD1d64TPseudoWB_fixed:
    1499             :     case ARM::VLD3d8Pseudo_UPD:
    1500             :     case ARM::VLD3d16Pseudo_UPD:
    1501             :     case ARM::VLD3d32Pseudo_UPD:
    1502             :     case ARM::VLD3q8Pseudo_UPD:
    1503             :     case ARM::VLD3q16Pseudo_UPD:
    1504             :     case ARM::VLD3q32Pseudo_UPD:
    1505             :     case ARM::VLD3q8oddPseudo:
    1506             :     case ARM::VLD3q16oddPseudo:
    1507             :     case ARM::VLD3q32oddPseudo:
    1508             :     case ARM::VLD3q8oddPseudo_UPD:
    1509             :     case ARM::VLD3q16oddPseudo_UPD:
    1510             :     case ARM::VLD3q32oddPseudo_UPD:
    1511             :     case ARM::VLD4d8Pseudo:
    1512             :     case ARM::VLD4d16Pseudo:
    1513             :     case ARM::VLD4d32Pseudo:
    1514             :     case ARM::VLD1d64QPseudo:
    1515             :     case ARM::VLD1d64QPseudoWB_fixed:
    1516             :     case ARM::VLD4d8Pseudo_UPD:
    1517             :     case ARM::VLD4d16Pseudo_UPD:
    1518             :     case ARM::VLD4d32Pseudo_UPD:
    1519             :     case ARM::VLD4q8Pseudo_UPD:
    1520             :     case ARM::VLD4q16Pseudo_UPD:
    1521             :     case ARM::VLD4q32Pseudo_UPD:
    1522             :     case ARM::VLD4q8oddPseudo:
    1523             :     case ARM::VLD4q16oddPseudo:
    1524             :     case ARM::VLD4q32oddPseudo:
    1525             :     case ARM::VLD4q8oddPseudo_UPD:
    1526             :     case ARM::VLD4q16oddPseudo_UPD:
    1527             :     case ARM::VLD4q32oddPseudo_UPD:
    1528             :     case ARM::VLD3DUPd8Pseudo:
    1529             :     case ARM::VLD3DUPd16Pseudo:
    1530             :     case ARM::VLD3DUPd32Pseudo:
    1531             :     case ARM::VLD3DUPd8Pseudo_UPD:
    1532             :     case ARM::VLD3DUPd16Pseudo_UPD:
    1533             :     case ARM::VLD3DUPd32Pseudo_UPD:
    1534             :     case ARM::VLD4DUPd8Pseudo:
    1535             :     case ARM::VLD4DUPd16Pseudo:
    1536             :     case ARM::VLD4DUPd32Pseudo:
    1537             :     case ARM::VLD4DUPd8Pseudo_UPD:
    1538             :     case ARM::VLD4DUPd16Pseudo_UPD:
    1539             :     case ARM::VLD4DUPd32Pseudo_UPD:
    1540          95 :       ExpandVLD(MBBI);
    1541          95 :       return true;
    1542             : 
    1543          77 :     case ARM::VST2q8Pseudo:
    1544             :     case ARM::VST2q16Pseudo:
    1545             :     case ARM::VST2q32Pseudo:
    1546             :     case ARM::VST2q8PseudoWB_fixed:
    1547             :     case ARM::VST2q16PseudoWB_fixed:
    1548             :     case ARM::VST2q32PseudoWB_fixed:
    1549             :     case ARM::VST2q8PseudoWB_register:
    1550             :     case ARM::VST2q16PseudoWB_register:
    1551             :     case ARM::VST2q32PseudoWB_register:
    1552             :     case ARM::VST3d8Pseudo:
    1553             :     case ARM::VST3d16Pseudo:
    1554             :     case ARM::VST3d32Pseudo:
    1555             :     case ARM::VST1d64TPseudo:
    1556             :     case ARM::VST3d8Pseudo_UPD:
    1557             :     case ARM::VST3d16Pseudo_UPD:
    1558             :     case ARM::VST3d32Pseudo_UPD:
    1559             :     case ARM::VST1d64TPseudoWB_fixed:
    1560             :     case ARM::VST1d64TPseudoWB_register:
    1561             :     case ARM::VST3q8Pseudo_UPD:
    1562             :     case ARM::VST3q16Pseudo_UPD:
    1563             :     case ARM::VST3q32Pseudo_UPD:
    1564             :     case ARM::VST3q8oddPseudo:
    1565             :     case ARM::VST3q16oddPseudo:
    1566             :     case ARM::VST3q32oddPseudo:
    1567             :     case ARM::VST3q8oddPseudo_UPD:
    1568             :     case ARM::VST3q16oddPseudo_UPD:
    1569             :     case ARM::VST3q32oddPseudo_UPD:
    1570             :     case ARM::VST4d8Pseudo:
    1571             :     case ARM::VST4d16Pseudo:
    1572             :     case ARM::VST4d32Pseudo:
    1573             :     case ARM::VST1d64QPseudo:
    1574             :     case ARM::VST4d8Pseudo_UPD:
    1575             :     case ARM::VST4d16Pseudo_UPD:
    1576             :     case ARM::VST4d32Pseudo_UPD:
    1577             :     case ARM::VST1d64QPseudoWB_fixed:
    1578             :     case ARM::VST1d64QPseudoWB_register:
    1579             :     case ARM::VST4q8Pseudo_UPD:
    1580             :     case ARM::VST4q16Pseudo_UPD:
    1581             :     case ARM::VST4q32Pseudo_UPD:
    1582             :     case ARM::VST4q8oddPseudo:
    1583             :     case ARM::VST4q16oddPseudo:
    1584             :     case ARM::VST4q32oddPseudo:
    1585             :     case ARM::VST4q8oddPseudo_UPD:
    1586             :     case ARM::VST4q16oddPseudo_UPD:
    1587             :     case ARM::VST4q32oddPseudo_UPD:
    1588          77 :       ExpandVST(MBBI);
    1589          77 :       return true;
    1590             : 
    1591         163 :     case ARM::VLD1LNq8Pseudo:
    1592             :     case ARM::VLD1LNq16Pseudo:
    1593             :     case ARM::VLD1LNq32Pseudo:
    1594             :     case ARM::VLD1LNq8Pseudo_UPD:
    1595             :     case ARM::VLD1LNq16Pseudo_UPD:
    1596             :     case ARM::VLD1LNq32Pseudo_UPD:
    1597             :     case ARM::VLD2LNd8Pseudo:
    1598             :     case ARM::VLD2LNd16Pseudo:
    1599             :     case ARM::VLD2LNd32Pseudo:
    1600             :     case ARM::VLD2LNq16Pseudo:
    1601             :     case ARM::VLD2LNq32Pseudo:
    1602             :     case ARM::VLD2LNd8Pseudo_UPD:
    1603             :     case ARM::VLD2LNd16Pseudo_UPD:
    1604             :     case ARM::VLD2LNd32Pseudo_UPD:
    1605             :     case ARM::VLD2LNq16Pseudo_UPD:
    1606             :     case ARM::VLD2LNq32Pseudo_UPD:
    1607             :     case ARM::VLD3LNd8Pseudo:
    1608             :     case ARM::VLD3LNd16Pseudo:
    1609             :     case ARM::VLD3LNd32Pseudo:
    1610             :     case ARM::VLD3LNq16Pseudo:
    1611             :     case ARM::VLD3LNq32Pseudo:
    1612             :     case ARM::VLD3LNd8Pseudo_UPD:
    1613             :     case ARM::VLD3LNd16Pseudo_UPD:
    1614             :     case ARM::VLD3LNd32Pseudo_UPD:
    1615             :     case ARM::VLD3LNq16Pseudo_UPD:
    1616             :     case ARM::VLD3LNq32Pseudo_UPD:
    1617             :     case ARM::VLD4LNd8Pseudo:
    1618             :     case ARM::VLD4LNd16Pseudo:
    1619             :     case ARM::VLD4LNd32Pseudo:
    1620             :     case ARM::VLD4LNq16Pseudo:
    1621             :     case ARM::VLD4LNq32Pseudo:
    1622             :     case ARM::VLD4LNd8Pseudo_UPD:
    1623             :     case ARM::VLD4LNd16Pseudo_UPD:
    1624             :     case ARM::VLD4LNd32Pseudo_UPD:
    1625             :     case ARM::VLD4LNq16Pseudo_UPD:
    1626             :     case ARM::VLD4LNq32Pseudo_UPD:
    1627             :     case ARM::VST1LNq8Pseudo:
    1628             :     case ARM::VST1LNq16Pseudo:
    1629             :     case ARM::VST1LNq32Pseudo:
    1630             :     case ARM::VST1LNq8Pseudo_UPD:
    1631             :     case ARM::VST1LNq16Pseudo_UPD:
    1632             :     case ARM::VST1LNq32Pseudo_UPD:
    1633             :     case ARM::VST2LNd8Pseudo:
    1634             :     case ARM::VST2LNd16Pseudo:
    1635             :     case ARM::VST2LNd32Pseudo:
    1636             :     case ARM::VST2LNq16Pseudo:
    1637             :     case ARM::VST2LNq32Pseudo:
    1638             :     case ARM::VST2LNd8Pseudo_UPD:
    1639             :     case ARM::VST2LNd16Pseudo_UPD:
    1640             :     case ARM::VST2LNd32Pseudo_UPD:
    1641             :     case ARM::VST2LNq16Pseudo_UPD:
    1642             :     case ARM::VST2LNq32Pseudo_UPD:
    1643             :     case ARM::VST3LNd8Pseudo:
    1644             :     case ARM::VST3LNd16Pseudo:
    1645             :     case ARM::VST3LNd32Pseudo:
    1646             :     case ARM::VST3LNq16Pseudo:
    1647             :     case ARM::VST3LNq32Pseudo:
    1648             :     case ARM::VST3LNd8Pseudo_UPD:
    1649             :     case ARM::VST3LNd16Pseudo_UPD:
    1650             :     case ARM::VST3LNd32Pseudo_UPD:
    1651             :     case ARM::VST3LNq16Pseudo_UPD:
    1652             :     case ARM::VST3LNq32Pseudo_UPD:
    1653             :     case ARM::VST4LNd8Pseudo:
    1654             :     case ARM::VST4LNd16Pseudo:
    1655             :     case ARM::VST4LNd32Pseudo:
    1656             :     case ARM::VST4LNq16Pseudo:
    1657             :     case ARM::VST4LNq32Pseudo:
    1658             :     case ARM::VST4LNd8Pseudo_UPD:
    1659             :     case ARM::VST4LNd16Pseudo_UPD:
    1660             :     case ARM::VST4LNd32Pseudo_UPD:
    1661             :     case ARM::VST4LNq16Pseudo_UPD:
    1662             :     case ARM::VST4LNq32Pseudo_UPD:
    1663         163 :       ExpandLaneOp(MBBI);
    1664         163 :       return true;
    1665             : 
    1666           1 :     case ARM::VTBL3Pseudo: ExpandVTBL(MBBI, ARM::VTBL3, false); return true;
    1667           1 :     case ARM::VTBL4Pseudo: ExpandVTBL(MBBI, ARM::VTBL4, false); return true;
    1668           1 :     case ARM::VTBX3Pseudo: ExpandVTBL(MBBI, ARM::VTBX3, true); return true;
    1669           2 :     case ARM::VTBX4Pseudo: ExpandVTBL(MBBI, ARM::VTBX4, true); return true;
    1670             : 
    1671           2 :     case ARM::CMP_SWAP_8:
    1672           2 :       if (STI->isThumb())
    1673             :         return ExpandCMP_SWAP(MBB, MBBI, ARM::t2LDREXB, ARM::t2STREXB,
    1674           1 :                               ARM::tUXTB, NextMBBI);
    1675             :       else
    1676             :         return ExpandCMP_SWAP(MBB, MBBI, ARM::LDREXB, ARM::STREXB,
    1677           1 :                               ARM::UXTB, NextMBBI);
    1678           2 :     case ARM::CMP_SWAP_16:
    1679           2 :       if (STI->isThumb())
    1680             :         return ExpandCMP_SWAP(MBB, MBBI, ARM::t2LDREXH, ARM::t2STREXH,
    1681           1 :                               ARM::tUXTH, NextMBBI);
    1682             :       else
    1683             :         return ExpandCMP_SWAP(MBB, MBBI, ARM::LDREXH, ARM::STREXH,
    1684           1 :                               ARM::UXTH, NextMBBI);
    1685           2 :     case ARM::CMP_SWAP_32:
    1686           2 :       if (STI->isThumb())
    1687             :         return ExpandCMP_SWAP(MBB, MBBI, ARM::t2LDREX, ARM::t2STREX, 0,
    1688           1 :                               NextMBBI);
    1689             :       else
    1690           1 :         return ExpandCMP_SWAP(MBB, MBBI, ARM::LDREX, ARM::STREX, 0, NextMBBI);
    1691             : 
    1692           7 :     case ARM::CMP_SWAP_64:
    1693           7 :       return ExpandCMP_SWAP_64(MBB, MBBI, NextMBBI);
    1694             :   }
    1695             : }
    1696             : 
    1697       17088 : bool ARMExpandPseudo::ExpandMBB(MachineBasicBlock &MBB) {
    1698       17088 :   bool Modified = false;
    1699             : 
    1700       34176 :   MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();
    1701      303014 :   while (MBBI != E) {
    1702      142963 :     MachineBasicBlock::iterator NMBBI = std::next(MBBI);
    1703      142963 :     Modified |= ExpandMI(MBB, MBBI, NMBBI);
    1704      142963 :     MBBI = NMBBI;
    1705             :   }
    1706             : 
    1707       17088 :   return Modified;
    1708             : }
    1709             : 
    1710       11976 : bool ARMExpandPseudo::runOnMachineFunction(MachineFunction &MF) {
    1711       11976 :   STI = &static_cast<const ARMSubtarget &>(MF.getSubtarget());
    1712       11976 :   TII = STI->getInstrInfo();
    1713       11976 :   TRI = STI->getRegisterInfo();
    1714       11976 :   AFI = MF.getInfo<ARMFunctionInfo>();
    1715             : 
    1716       11976 :   bool Modified = false;
    1717       53016 :   for (MachineBasicBlock &MBB : MF)
    1718       17088 :     Modified |= ExpandMBB(MBB);
    1719       11976 :   if (VerifyARMPseudo)
    1720           4 :     MF.verify(this, "After expanding ARM pseudo instructions.");
    1721       11976 :   return Modified;
    1722             : }
    1723             : 
    1724             : /// createARMExpandPseudoPass - returns an instance of the pseudo instruction
    1725             : /// expansion pass.
    1726        2575 : FunctionPass *llvm::createARMExpandPseudoPass() {
    1727        5150 :   return new ARMExpandPseudo();
    1728      216918 : }

Generated by: LCOV version 1.13