LCOV - code coverage report
Current view: top level - lib/Target/ARM - ARMExpandPseudoInsts.cpp (source / functions) Hit Total Coverage
Test: llvm-toolchain.info Lines: 677 706 95.9 %
Date: 2018-02-22 16:16:46 Functions: 22 23 95.7 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : //===-- ARMExpandPseudoInsts.cpp - Expand pseudo instructions -------------===//
       2             : //
       3             : //                     The LLVM Compiler Infrastructure
       4             : //
       5             : // This file is distributed under the University of Illinois Open Source
       6             : // License. See LICENSE.TXT for details.
       7             : //
       8             : //===----------------------------------------------------------------------===//
       9             : //
      10             : // This file contains a pass that expands pseudo instructions into target
      11             : // instructions to allow proper scheduling, if-conversion, and other late
      12             : // optimizations. This pass should be run after register allocation but before
      13             : // the post-regalloc scheduling pass.
      14             : //
      15             : //===----------------------------------------------------------------------===//
      16             : 
      17             : #include "ARM.h"
      18             : #include "ARMBaseInstrInfo.h"
      19             : #include "ARMBaseRegisterInfo.h"
      20             : #include "ARMConstantPoolValue.h"
      21             : #include "ARMMachineFunctionInfo.h"
      22             : #include "ARMSubtarget.h"
      23             : #include "MCTargetDesc/ARMAddressingModes.h"
      24             : #include "llvm/CodeGen/LivePhysRegs.h"
      25             : #include "llvm/CodeGen/MachineFrameInfo.h"
      26             : #include "llvm/CodeGen/MachineFunctionPass.h"
      27             : 
      28             : using namespace llvm;
      29             : 
      30             : #define DEBUG_TYPE "arm-pseudo"
      31             : 
      32             : static cl::opt<bool>
      33       81626 : VerifyARMPseudo("verify-arm-pseudo-expand", cl::Hidden,
      34       81626 :                 cl::desc("Verify machine code after expanding ARM pseudos"));
      35             : 
      36             : #define ARM_EXPAND_PSEUDO_NAME "ARM pseudo instruction expansion pass"
      37             : 
      38             : namespace {
      39        2649 :   class ARMExpandPseudo : public MachineFunctionPass {
      40             :   public:
      41             :     static char ID;
      42        2676 :     ARMExpandPseudo() : MachineFunctionPass(ID) {}
      43             : 
      44             :     const ARMBaseInstrInfo *TII;
      45             :     const TargetRegisterInfo *TRI;
      46             :     const ARMSubtarget *STI;
      47             :     ARMFunctionInfo *AFI;
      48             : 
      49             :     bool runOnMachineFunction(MachineFunction &Fn) override;
      50             : 
      51        2663 :     MachineFunctionProperties getRequiredProperties() const override {
      52        5326 :       return MachineFunctionProperties().set(
      53        2663 :           MachineFunctionProperties::Property::NoVRegs);
      54             :     }
      55             : 
      56        2662 :     StringRef getPassName() const override {
      57        2662 :       return ARM_EXPAND_PSEUDO_NAME;
      58             :     }
      59             : 
      60             :   private:
      61             :     void TransferImpOps(MachineInstr &OldMI,
      62             :                         MachineInstrBuilder &UseMI, MachineInstrBuilder &DefMI);
      63             :     bool ExpandMI(MachineBasicBlock &MBB,
      64             :                   MachineBasicBlock::iterator MBBI,
      65             :                   MachineBasicBlock::iterator &NextMBBI);
      66             :     bool ExpandMBB(MachineBasicBlock &MBB);
      67             :     void ExpandVLD(MachineBasicBlock::iterator &MBBI);
      68             :     void ExpandVST(MachineBasicBlock::iterator &MBBI);
      69             :     void ExpandLaneOp(MachineBasicBlock::iterator &MBBI);
      70             :     void ExpandVTBL(MachineBasicBlock::iterator &MBBI,
      71             :                     unsigned Opc, bool IsExt);
      72             :     void ExpandMOV32BitImm(MachineBasicBlock &MBB,
      73             :                            MachineBasicBlock::iterator &MBBI);
      74             :     bool ExpandCMP_SWAP(MachineBasicBlock &MBB,
      75             :                         MachineBasicBlock::iterator MBBI, unsigned LdrexOp,
      76             :                         unsigned StrexOp, unsigned UxtOp,
      77             :                         MachineBasicBlock::iterator &NextMBBI);
      78             : 
      79             :     bool ExpandCMP_SWAP_64(MachineBasicBlock &MBB,
      80             :                            MachineBasicBlock::iterator MBBI,
      81             :                            MachineBasicBlock::iterator &NextMBBI);
      82             :   };
      83             :   char ARMExpandPseudo::ID = 0;
      84             : }
      85             : 
      86      279510 : INITIALIZE_PASS(ARMExpandPseudo, DEBUG_TYPE, ARM_EXPAND_PSEUDO_NAME, false,
      87             :                 false)
      88             : 
      89             : /// TransferImpOps - Transfer implicit operands on the pseudo instruction to
      90             : /// the instructions created from the expansion.
      91        2675 : void ARMExpandPseudo::TransferImpOps(MachineInstr &OldMI,
      92             :                                      MachineInstrBuilder &UseMI,
      93             :                                      MachineInstrBuilder &DefMI) {
      94        2675 :   const MCInstrDesc &Desc = OldMI.getDesc();
      95        5556 :   for (unsigned i = Desc.getNumOperands(), e = OldMI.getNumOperands();
      96        2881 :        i != e; ++i) {
      97         206 :     const MachineOperand &MO = OldMI.getOperand(i);
      98             :     assert(MO.isReg() && MO.getReg());
      99         206 :     if (MO.isUse())
     100             :       UseMI.add(MO);
     101             :     else
     102             :       DefMI.add(MO);
     103             :   }
     104        2675 : }
     105             : 
     106             : namespace {
     107             :   // Constants for register spacing in NEON load/store instructions.
     108             :   // For quad-register load-lane and store-lane pseudo instructors, the
     109             :   // spacing is initially assumed to be EvenDblSpc, and that is changed to
     110             :   // OddDblSpc depending on the lane number operand.
     111             :   enum NEONRegSpacing {
     112             :     SingleSpc,
     113             :     EvenDblSpc,
     114             :     OddDblSpc
     115             :   };
     116             : 
     117             :   // Entries for NEON load/store information table.  The table is sorted by
     118             :   // PseudoOpc for fast binary-search lookups.
     119             :   struct NEONLdStTableEntry {
     120             :     uint16_t PseudoOpc;
     121             :     uint16_t RealOpc;
     122             :     bool IsLoad;
     123             :     bool isUpdating;
     124             :     bool hasWritebackOperand;
     125             :     uint8_t RegSpacing; // One of type NEONRegSpacing
     126             :     uint8_t NumRegs; // D registers loaded or stored
     127             :     uint8_t RegElts; // elements per D register; used for lane ops
     128             :     // FIXME: Temporary flag to denote whether the real instruction takes
     129             :     // a single register (like the encoding) or all of the registers in
     130             :     // the list (like the asm syntax and the isel DAG). When all definitions
     131             :     // are converted to take only the single encoded register, this will
     132             :     // go away.
     133             :     bool copyAllListRegs;
     134             : 
     135             :     // Comparison methods for binary search of the table.
     136             :     bool operator<(const NEONLdStTableEntry &TE) const {
     137             :       return PseudoOpc < TE.PseudoOpc;
     138             :     }
     139             :     friend bool operator<(const NEONLdStTableEntry &TE, unsigned PseudoOpc) {
     140             :       return TE.PseudoOpc < PseudoOpc;
     141             :     }
     142             :     friend bool LLVM_ATTRIBUTE_UNUSED operator<(unsigned PseudoOpc,
     143             :                                                 const NEONLdStTableEntry &TE) {
     144             :       return PseudoOpc < TE.PseudoOpc;
     145             :     }
     146             :   };
     147             : }
     148             : 
     149             : static const NEONLdStTableEntry NEONLdStTable[] = {
     150             : { ARM::VLD1LNq16Pseudo,     ARM::VLD1LNd16,     true, false, false, EvenDblSpc, 1, 4 ,true},
     151             : { ARM::VLD1LNq16Pseudo_UPD, ARM::VLD1LNd16_UPD, true, true, true,  EvenDblSpc, 1, 4 ,true},
     152             : { ARM::VLD1LNq32Pseudo,     ARM::VLD1LNd32,     true, false, false, EvenDblSpc, 1, 2 ,true},
     153             : { ARM::VLD1LNq32Pseudo_UPD, ARM::VLD1LNd32_UPD, true, true, true,  EvenDblSpc, 1, 2 ,true},
     154             : { ARM::VLD1LNq8Pseudo,      ARM::VLD1LNd8,      true, false, false, EvenDblSpc, 1, 8 ,true},
     155             : { ARM::VLD1LNq8Pseudo_UPD,  ARM::VLD1LNd8_UPD, true, true, true,  EvenDblSpc, 1, 8 ,true},
     156             : 
     157             : { ARM::VLD1d64QPseudo,      ARM::VLD1d64Q,     true,  false, false, SingleSpc,  4, 1 ,false},
     158             : { ARM::VLD1d64QPseudoWB_fixed,  ARM::VLD1d64Qwb_fixed,   true,  true, false, SingleSpc,  4, 1 ,false},
     159             : { ARM::VLD1d64TPseudo,      ARM::VLD1d64T,     true,  false, false, SingleSpc,  3, 1 ,false},
     160             : { ARM::VLD1d64TPseudoWB_fixed,  ARM::VLD1d64Twb_fixed,   true,  true, false, SingleSpc,  3, 1 ,false},
     161             : 
     162             : { ARM::VLD2LNd16Pseudo,     ARM::VLD2LNd16,     true, false, false, SingleSpc,  2, 4 ,true},
     163             : { ARM::VLD2LNd16Pseudo_UPD, ARM::VLD2LNd16_UPD, true, true, true,  SingleSpc,  2, 4 ,true},
     164             : { ARM::VLD2LNd32Pseudo,     ARM::VLD2LNd32,     true, false, false, SingleSpc,  2, 2 ,true},
     165             : { ARM::VLD2LNd32Pseudo_UPD, ARM::VLD2LNd32_UPD, true, true, true,  SingleSpc,  2, 2 ,true},
     166             : { ARM::VLD2LNd8Pseudo,      ARM::VLD2LNd8,      true, false, false, SingleSpc,  2, 8 ,true},
     167             : { ARM::VLD2LNd8Pseudo_UPD,  ARM::VLD2LNd8_UPD, true, true, true,  SingleSpc,  2, 8 ,true},
     168             : { ARM::VLD2LNq16Pseudo,     ARM::VLD2LNq16,     true, false, false, EvenDblSpc, 2, 4 ,true},
     169             : { ARM::VLD2LNq16Pseudo_UPD, ARM::VLD2LNq16_UPD, true, true, true,  EvenDblSpc, 2, 4 ,true},
     170             : { ARM::VLD2LNq32Pseudo,     ARM::VLD2LNq32,     true, false, false, EvenDblSpc, 2, 2 ,true},
     171             : { ARM::VLD2LNq32Pseudo_UPD, ARM::VLD2LNq32_UPD, true, true, true,  EvenDblSpc, 2, 2 ,true},
     172             : 
     173             : { ARM::VLD2q16Pseudo,       ARM::VLD2q16,      true,  false, false, SingleSpc,  4, 4 ,false},
     174             : { ARM::VLD2q16PseudoWB_fixed,   ARM::VLD2q16wb_fixed, true, true, false,  SingleSpc,  4, 4 ,false},
     175             : { ARM::VLD2q16PseudoWB_register,   ARM::VLD2q16wb_register, true, true, true,  SingleSpc,  4, 4 ,false},
     176             : { ARM::VLD2q32Pseudo,       ARM::VLD2q32,      true,  false, false, SingleSpc,  4, 2 ,false},
     177             : { ARM::VLD2q32PseudoWB_fixed,   ARM::VLD2q32wb_fixed, true, true, false,  SingleSpc,  4, 2 ,false},
     178             : { ARM::VLD2q32PseudoWB_register,   ARM::VLD2q32wb_register, true, true, true,  SingleSpc,  4, 2 ,false},
     179             : { ARM::VLD2q8Pseudo,        ARM::VLD2q8,       true,  false, false, SingleSpc,  4, 8 ,false},
     180             : { ARM::VLD2q8PseudoWB_fixed,    ARM::VLD2q8wb_fixed, true, true, false,  SingleSpc,  4, 8 ,false},
     181             : { ARM::VLD2q8PseudoWB_register,    ARM::VLD2q8wb_register, true, true, true,  SingleSpc,  4, 8 ,false},
     182             : 
     183             : { ARM::VLD3DUPd16Pseudo,     ARM::VLD3DUPd16,     true, false, false, SingleSpc, 3, 4,true},
     184             : { ARM::VLD3DUPd16Pseudo_UPD, ARM::VLD3DUPd16_UPD, true, true, true,  SingleSpc, 3, 4,true},
     185             : { ARM::VLD3DUPd32Pseudo,     ARM::VLD3DUPd32,     true, false, false, SingleSpc, 3, 2,true},
     186             : { ARM::VLD3DUPd32Pseudo_UPD, ARM::VLD3DUPd32_UPD, true, true, true,  SingleSpc, 3, 2,true},
     187             : { ARM::VLD3DUPd8Pseudo,      ARM::VLD3DUPd8,      true, false, false, SingleSpc, 3, 8,true},
     188             : { ARM::VLD3DUPd8Pseudo_UPD,  ARM::VLD3DUPd8_UPD, true, true, true,  SingleSpc, 3, 8,true},
     189             : 
     190             : { ARM::VLD3LNd16Pseudo,     ARM::VLD3LNd16,     true, false, false, SingleSpc,  3, 4 ,true},
     191             : { ARM::VLD3LNd16Pseudo_UPD, ARM::VLD3LNd16_UPD, true, true, true,  SingleSpc,  3, 4 ,true},
     192             : { ARM::VLD3LNd32Pseudo,     ARM::VLD3LNd32,     true, false, false, SingleSpc,  3, 2 ,true},
     193             : { ARM::VLD3LNd32Pseudo_UPD, ARM::VLD3LNd32_UPD, true, true, true,  SingleSpc,  3, 2 ,true},
     194             : { ARM::VLD3LNd8Pseudo,      ARM::VLD3LNd8,      true, false, false, SingleSpc,  3, 8 ,true},
     195             : { ARM::VLD3LNd8Pseudo_UPD,  ARM::VLD3LNd8_UPD, true, true, true,  SingleSpc,  3, 8 ,true},
     196             : { ARM::VLD3LNq16Pseudo,     ARM::VLD3LNq16,     true, false, false, EvenDblSpc, 3, 4 ,true},
     197             : { ARM::VLD3LNq16Pseudo_UPD, ARM::VLD3LNq16_UPD, true, true, true,  EvenDblSpc, 3, 4 ,true},
     198             : { ARM::VLD3LNq32Pseudo,     ARM::VLD3LNq32,     true, false, false, EvenDblSpc, 3, 2 ,true},
     199             : { ARM::VLD3LNq32Pseudo_UPD, ARM::VLD3LNq32_UPD, true, true, true,  EvenDblSpc, 3, 2 ,true},
     200             : 
     201             : { ARM::VLD3d16Pseudo,       ARM::VLD3d16,      true,  false, false, SingleSpc,  3, 4 ,true},
     202             : { ARM::VLD3d16Pseudo_UPD,   ARM::VLD3d16_UPD, true, true, true,  SingleSpc,  3, 4 ,true},
     203             : { ARM::VLD3d32Pseudo,       ARM::VLD3d32,      true,  false, false, SingleSpc,  3, 2 ,true},
     204             : { ARM::VLD3d32Pseudo_UPD,   ARM::VLD3d32_UPD, true, true, true,  SingleSpc,  3, 2 ,true},
     205             : { ARM::VLD3d8Pseudo,        ARM::VLD3d8,       true,  false, false, SingleSpc,  3, 8 ,true},
     206             : { ARM::VLD3d8Pseudo_UPD,    ARM::VLD3d8_UPD, true, true, true,  SingleSpc,  3, 8 ,true},
     207             : 
     208             : { ARM::VLD3q16Pseudo_UPD,    ARM::VLD3q16_UPD, true, true, true,  EvenDblSpc, 3, 4 ,true},
     209             : { ARM::VLD3q16oddPseudo,     ARM::VLD3q16,     true,  false, false, OddDblSpc,  3, 4 ,true},
     210             : { ARM::VLD3q16oddPseudo_UPD, ARM::VLD3q16_UPD, true, true, true,  OddDblSpc,  3, 4 ,true},
     211             : { ARM::VLD3q32Pseudo_UPD,    ARM::VLD3q32_UPD, true, true, true,  EvenDblSpc, 3, 2 ,true},
     212             : { ARM::VLD3q32oddPseudo,     ARM::VLD3q32,     true,  false, false, OddDblSpc,  3, 2 ,true},
     213             : { ARM::VLD3q32oddPseudo_UPD, ARM::VLD3q32_UPD, true, true, true,  OddDblSpc,  3, 2 ,true},
     214             : { ARM::VLD3q8Pseudo_UPD,     ARM::VLD3q8_UPD, true, true, true,  EvenDblSpc, 3, 8 ,true},
     215             : { ARM::VLD3q8oddPseudo,      ARM::VLD3q8,      true,  false, false, OddDblSpc,  3, 8 ,true},
     216             : { ARM::VLD3q8oddPseudo_UPD,  ARM::VLD3q8_UPD, true, true, true,  OddDblSpc,  3, 8 ,true},
     217             : 
     218             : { ARM::VLD4DUPd16Pseudo,     ARM::VLD4DUPd16,     true, false, false, SingleSpc, 4, 4,true},
     219             : { ARM::VLD4DUPd16Pseudo_UPD, ARM::VLD4DUPd16_UPD, true, true, true,  SingleSpc, 4, 4,true},
     220             : { ARM::VLD4DUPd32Pseudo,     ARM::VLD4DUPd32,     true, false, false, SingleSpc, 4, 2,true},
     221             : { ARM::VLD4DUPd32Pseudo_UPD, ARM::VLD4DUPd32_UPD, true, true, true,  SingleSpc, 4, 2,true},
     222             : { ARM::VLD4DUPd8Pseudo,      ARM::VLD4DUPd8,      true, false, false, SingleSpc, 4, 8,true},
     223             : { ARM::VLD4DUPd8Pseudo_UPD,  ARM::VLD4DUPd8_UPD, true, true, true,  SingleSpc, 4, 8,true},
     224             : 
     225             : { ARM::VLD4LNd16Pseudo,     ARM::VLD4LNd16,     true, false, false, SingleSpc,  4, 4 ,true},
     226             : { ARM::VLD4LNd16Pseudo_UPD, ARM::VLD4LNd16_UPD, true, true, true,  SingleSpc,  4, 4 ,true},
     227             : { ARM::VLD4LNd32Pseudo,     ARM::VLD4LNd32,     true, false, false, SingleSpc,  4, 2 ,true},
     228             : { ARM::VLD4LNd32Pseudo_UPD, ARM::VLD4LNd32_UPD, true, true, true,  SingleSpc,  4, 2 ,true},
     229             : { ARM::VLD4LNd8Pseudo,      ARM::VLD4LNd8,      true, false, false, SingleSpc,  4, 8 ,true},
     230             : { ARM::VLD4LNd8Pseudo_UPD,  ARM::VLD4LNd8_UPD, true, true, true,  SingleSpc,  4, 8 ,true},
     231             : { ARM::VLD4LNq16Pseudo,     ARM::VLD4LNq16,     true, false, false, EvenDblSpc, 4, 4 ,true},
     232             : { ARM::VLD4LNq16Pseudo_UPD, ARM::VLD4LNq16_UPD, true, true, true,  EvenDblSpc, 4, 4 ,true},
     233             : { ARM::VLD4LNq32Pseudo,     ARM::VLD4LNq32,     true, false, false, EvenDblSpc, 4, 2 ,true},
     234             : { ARM::VLD4LNq32Pseudo_UPD, ARM::VLD4LNq32_UPD, true, true, true,  EvenDblSpc, 4, 2 ,true},
     235             : 
     236             : { ARM::VLD4d16Pseudo,       ARM::VLD4d16,      true,  false, false, SingleSpc,  4, 4 ,true},
     237             : { ARM::VLD4d16Pseudo_UPD,   ARM::VLD4d16_UPD, true, true, true,  SingleSpc,  4, 4 ,true},
     238             : { ARM::VLD4d32Pseudo,       ARM::VLD4d32,      true,  false, false, SingleSpc,  4, 2 ,true},
     239             : { ARM::VLD4d32Pseudo_UPD,   ARM::VLD4d32_UPD, true, true, true,  SingleSpc,  4, 2 ,true},
     240             : { ARM::VLD4d8Pseudo,        ARM::VLD4d8,       true,  false, false, SingleSpc,  4, 8 ,true},
     241             : { ARM::VLD4d8Pseudo_UPD,    ARM::VLD4d8_UPD, true, true, true,  SingleSpc,  4, 8 ,true},
     242             : 
     243             : { ARM::VLD4q16Pseudo_UPD,    ARM::VLD4q16_UPD, true, true, true,  EvenDblSpc, 4, 4 ,true},
     244             : { ARM::VLD4q16oddPseudo,     ARM::VLD4q16,     true,  false, false, OddDblSpc,  4, 4 ,true},
     245             : { ARM::VLD4q16oddPseudo_UPD, ARM::VLD4q16_UPD, true, true, true,  OddDblSpc,  4, 4 ,true},
     246             : { ARM::VLD4q32Pseudo_UPD,    ARM::VLD4q32_UPD, true, true, true,  EvenDblSpc, 4, 2 ,true},
     247             : { ARM::VLD4q32oddPseudo,     ARM::VLD4q32,     true,  false, false, OddDblSpc,  4, 2 ,true},
     248             : { ARM::VLD4q32oddPseudo_UPD, ARM::VLD4q32_UPD, true, true, true,  OddDblSpc,  4, 2 ,true},
     249             : { ARM::VLD4q8Pseudo_UPD,     ARM::VLD4q8_UPD, true, true, true,  EvenDblSpc, 4, 8 ,true},
     250             : { ARM::VLD4q8oddPseudo,      ARM::VLD4q8,      true,  false, false, OddDblSpc,  4, 8 ,true},
     251             : { ARM::VLD4q8oddPseudo_UPD,  ARM::VLD4q8_UPD, true, true, true,  OddDblSpc,  4, 8 ,true},
     252             : 
     253             : { ARM::VST1LNq16Pseudo,     ARM::VST1LNd16,    false, false, false, EvenDblSpc, 1, 4 ,true},
     254             : { ARM::VST1LNq16Pseudo_UPD, ARM::VST1LNd16_UPD, false, true, true,  EvenDblSpc, 1, 4 ,true},
     255             : { ARM::VST1LNq32Pseudo,     ARM::VST1LNd32,    false, false, false, EvenDblSpc, 1, 2 ,true},
     256             : { ARM::VST1LNq32Pseudo_UPD, ARM::VST1LNd32_UPD, false, true, true,  EvenDblSpc, 1, 2 ,true},
     257             : { ARM::VST1LNq8Pseudo,      ARM::VST1LNd8,     false, false, false, EvenDblSpc, 1, 8 ,true},
     258             : { ARM::VST1LNq8Pseudo_UPD,  ARM::VST1LNd8_UPD, false, true, true,  EvenDblSpc, 1, 8 ,true},
     259             : 
     260             : { ARM::VST1d64QPseudo,      ARM::VST1d64Q,     false, false, false, SingleSpc,  4, 1 ,false},
     261             : { ARM::VST1d64QPseudoWB_fixed,  ARM::VST1d64Qwb_fixed, false, true, false,  SingleSpc,  4, 1 ,false},
     262             : { ARM::VST1d64QPseudoWB_register, ARM::VST1d64Qwb_register, false, true, true,  SingleSpc,  4, 1 ,false},
     263             : { ARM::VST1d64TPseudo,      ARM::VST1d64T,     false, false, false, SingleSpc,  3, 1 ,false},
     264             : { ARM::VST1d64TPseudoWB_fixed,  ARM::VST1d64Twb_fixed, false, true, false,  SingleSpc,  3, 1 ,false},
     265             : { ARM::VST1d64TPseudoWB_register,  ARM::VST1d64Twb_register, false, true, true,  SingleSpc,  3, 1 ,false},
     266             : 
     267             : { ARM::VST2LNd16Pseudo,     ARM::VST2LNd16,     false, false, false, SingleSpc, 2, 4 ,true},
     268             : { ARM::VST2LNd16Pseudo_UPD, ARM::VST2LNd16_UPD, false, true, true,  SingleSpc, 2, 4 ,true},
     269             : { ARM::VST2LNd32Pseudo,     ARM::VST2LNd32,     false, false, false, SingleSpc, 2, 2 ,true},
     270             : { ARM::VST2LNd32Pseudo_UPD, ARM::VST2LNd32_UPD, false, true, true,  SingleSpc, 2, 2 ,true},
     271             : { ARM::VST2LNd8Pseudo,      ARM::VST2LNd8,      false, false, false, SingleSpc, 2, 8 ,true},
     272             : { ARM::VST2LNd8Pseudo_UPD,  ARM::VST2LNd8_UPD, false, true, true,  SingleSpc, 2, 8 ,true},
     273             : { ARM::VST2LNq16Pseudo,     ARM::VST2LNq16,     false, false, false, EvenDblSpc, 2, 4,true},
     274             : { ARM::VST2LNq16Pseudo_UPD, ARM::VST2LNq16_UPD, false, true, true,  EvenDblSpc, 2, 4,true},
     275             : { ARM::VST2LNq32Pseudo,     ARM::VST2LNq32,     false, false, false, EvenDblSpc, 2, 2,true},
     276             : { ARM::VST2LNq32Pseudo_UPD, ARM::VST2LNq32_UPD, false, true, true,  EvenDblSpc, 2, 2,true},
     277             : 
     278             : { ARM::VST2q16Pseudo,       ARM::VST2q16,      false, false, false, SingleSpc,  4, 4 ,false},
     279             : { ARM::VST2q16PseudoWB_fixed,   ARM::VST2q16wb_fixed, false, true, false,  SingleSpc,  4, 4 ,false},
     280             : { ARM::VST2q16PseudoWB_register,   ARM::VST2q16wb_register, false, true, true,  SingleSpc,  4, 4 ,false},
     281             : { ARM::VST2q32Pseudo,       ARM::VST2q32,      false, false, false, SingleSpc,  4, 2 ,false},
     282             : { ARM::VST2q32PseudoWB_fixed,   ARM::VST2q32wb_fixed, false, true, false,  SingleSpc,  4, 2 ,false},
     283             : { ARM::VST2q32PseudoWB_register,   ARM::VST2q32wb_register, false, true, true,  SingleSpc,  4, 2 ,false},
     284             : { ARM::VST2q8Pseudo,        ARM::VST2q8,       false, false, false, SingleSpc,  4, 8 ,false},
     285             : { ARM::VST2q8PseudoWB_fixed,    ARM::VST2q8wb_fixed, false, true, false,  SingleSpc,  4, 8 ,false},
     286             : { ARM::VST2q8PseudoWB_register,    ARM::VST2q8wb_register, false, true, true,  SingleSpc,  4, 8 ,false},
     287             : 
     288             : { ARM::VST3LNd16Pseudo,     ARM::VST3LNd16,     false, false, false, SingleSpc, 3, 4 ,true},
     289             : { ARM::VST3LNd16Pseudo_UPD, ARM::VST3LNd16_UPD, false, true, true,  SingleSpc, 3, 4 ,true},
     290             : { ARM::VST3LNd32Pseudo,     ARM::VST3LNd32,     false, false, false, SingleSpc, 3, 2 ,true},
     291             : { ARM::VST3LNd32Pseudo_UPD, ARM::VST3LNd32_UPD, false, true, true,  SingleSpc, 3, 2 ,true},
     292             : { ARM::VST3LNd8Pseudo,      ARM::VST3LNd8,      false, false, false, SingleSpc, 3, 8 ,true},
     293             : { ARM::VST3LNd8Pseudo_UPD,  ARM::VST3LNd8_UPD, false, true, true,  SingleSpc, 3, 8 ,true},
     294             : { ARM::VST3LNq16Pseudo,     ARM::VST3LNq16,     false, false, false, EvenDblSpc, 3, 4,true},
     295             : { ARM::VST3LNq16Pseudo_UPD, ARM::VST3LNq16_UPD, false, true, true,  EvenDblSpc, 3, 4,true},
     296             : { ARM::VST3LNq32Pseudo,     ARM::VST3LNq32,     false, false, false, EvenDblSpc, 3, 2,true},
     297             : { ARM::VST3LNq32Pseudo_UPD, ARM::VST3LNq32_UPD, false, true, true,  EvenDblSpc, 3, 2,true},
     298             : 
     299             : { ARM::VST3d16Pseudo,       ARM::VST3d16,      false, false, false, SingleSpc,  3, 4 ,true},
     300             : { ARM::VST3d16Pseudo_UPD,   ARM::VST3d16_UPD, false, true, true,  SingleSpc,  3, 4 ,true},
     301             : { ARM::VST3d32Pseudo,       ARM::VST3d32,      false, false, false, SingleSpc,  3, 2 ,true},
     302             : { ARM::VST3d32Pseudo_UPD,   ARM::VST3d32_UPD, false, true, true,  SingleSpc,  3, 2 ,true},
     303             : { ARM::VST3d8Pseudo,        ARM::VST3d8,       false, false, false, SingleSpc,  3, 8 ,true},
     304             : { ARM::VST3d8Pseudo_UPD,    ARM::VST3d8_UPD, false, true, true,  SingleSpc,  3, 8 ,true},
     305             : 
     306             : { ARM::VST3q16Pseudo_UPD,    ARM::VST3q16_UPD, false, true, true,  EvenDblSpc, 3, 4 ,true},
     307             : { ARM::VST3q16oddPseudo,     ARM::VST3q16,     false, false, false, OddDblSpc,  3, 4 ,true},
     308             : { ARM::VST3q16oddPseudo_UPD, ARM::VST3q16_UPD, false, true, true,  OddDblSpc,  3, 4 ,true},
     309             : { ARM::VST3q32Pseudo_UPD,    ARM::VST3q32_UPD, false, true, true,  EvenDblSpc, 3, 2 ,true},
     310             : { ARM::VST3q32oddPseudo,     ARM::VST3q32,     false, false, false, OddDblSpc,  3, 2 ,true},
     311             : { ARM::VST3q32oddPseudo_UPD, ARM::VST3q32_UPD, false, true, true,  OddDblSpc,  3, 2 ,true},
     312             : { ARM::VST3q8Pseudo_UPD,     ARM::VST3q8_UPD, false, true, true,  EvenDblSpc, 3, 8 ,true},
     313             : { ARM::VST3q8oddPseudo,      ARM::VST3q8,      false, false, false, OddDblSpc,  3, 8 ,true},
     314             : { ARM::VST3q8oddPseudo_UPD,  ARM::VST3q8_UPD, false, true, true,  OddDblSpc,  3, 8 ,true},
     315             : 
     316             : { ARM::VST4LNd16Pseudo,     ARM::VST4LNd16,     false, false, false, SingleSpc, 4, 4 ,true},
     317             : { ARM::VST4LNd16Pseudo_UPD, ARM::VST4LNd16_UPD, false, true, true,  SingleSpc, 4, 4 ,true},
     318             : { ARM::VST4LNd32Pseudo,     ARM::VST4LNd32,     false, false, false, SingleSpc, 4, 2 ,true},
     319             : { ARM::VST4LNd32Pseudo_UPD, ARM::VST4LNd32_UPD, false, true, true,  SingleSpc, 4, 2 ,true},
     320             : { ARM::VST4LNd8Pseudo,      ARM::VST4LNd8,      false, false, false, SingleSpc, 4, 8 ,true},
     321             : { ARM::VST4LNd8Pseudo_UPD,  ARM::VST4LNd8_UPD, false, true, true,  SingleSpc, 4, 8 ,true},
     322             : { ARM::VST4LNq16Pseudo,     ARM::VST4LNq16,     false, false, false, EvenDblSpc, 4, 4,true},
     323             : { ARM::VST4LNq16Pseudo_UPD, ARM::VST4LNq16_UPD, false, true, true,  EvenDblSpc, 4, 4,true},
     324             : { ARM::VST4LNq32Pseudo,     ARM::VST4LNq32,     false, false, false, EvenDblSpc, 4, 2,true},
     325             : { ARM::VST4LNq32Pseudo_UPD, ARM::VST4LNq32_UPD, false, true, true,  EvenDblSpc, 4, 2,true},
     326             : 
     327             : { ARM::VST4d16Pseudo,       ARM::VST4d16,      false, false, false, SingleSpc,  4, 4 ,true},
     328             : { ARM::VST4d16Pseudo_UPD,   ARM::VST4d16_UPD, false, true, true,  SingleSpc,  4, 4 ,true},
     329             : { ARM::VST4d32Pseudo,       ARM::VST4d32,      false, false, false, SingleSpc,  4, 2 ,true},
     330             : { ARM::VST4d32Pseudo_UPD,   ARM::VST4d32_UPD, false, true, true,  SingleSpc,  4, 2 ,true},
     331             : { ARM::VST4d8Pseudo,        ARM::VST4d8,       false, false, false, SingleSpc,  4, 8 ,true},
     332             : { ARM::VST4d8Pseudo_UPD,    ARM::VST4d8_UPD, false, true, true,  SingleSpc,  4, 8 ,true},
     333             : 
     334             : { ARM::VST4q16Pseudo_UPD,    ARM::VST4q16_UPD, false, true, true,  EvenDblSpc, 4, 4 ,true},
     335             : { ARM::VST4q16oddPseudo,     ARM::VST4q16,     false, false, false, OddDblSpc,  4, 4 ,true},
     336             : { ARM::VST4q16oddPseudo_UPD, ARM::VST4q16_UPD, false, true, true,  OddDblSpc,  4, 4 ,true},
     337             : { ARM::VST4q32Pseudo_UPD,    ARM::VST4q32_UPD, false, true, true,  EvenDblSpc, 4, 2 ,true},
     338             : { ARM::VST4q32oddPseudo,     ARM::VST4q32,     false, false, false, OddDblSpc,  4, 2 ,true},
     339             : { ARM::VST4q32oddPseudo_UPD, ARM::VST4q32_UPD, false, true, true,  OddDblSpc,  4, 2 ,true},
     340             : { ARM::VST4q8Pseudo_UPD,     ARM::VST4q8_UPD, false, true, true,  EvenDblSpc, 4, 8 ,true},
     341             : { ARM::VST4q8oddPseudo,      ARM::VST4q8,      false, false, false, OddDblSpc,  4, 8 ,true},
     342             : { ARM::VST4q8oddPseudo_UPD,  ARM::VST4q8_UPD, false, true, true,  OddDblSpc,  4, 8 ,true}
     343             : };
     344             : 
     345             : /// LookupNEONLdSt - Search the NEONLdStTable for information about a NEON
     346             : /// load or store pseudo instruction.
     347             : static const NEONLdStTableEntry *LookupNEONLdSt(unsigned Opcode) {
     348             : #ifndef NDEBUG
     349             :   // Make sure the table is sorted.
     350             :   static bool TableChecked = false;
     351             :   if (!TableChecked) {
     352             :     assert(std::is_sorted(std::begin(NEONLdStTable), std::end(NEONLdStTable)) &&
     353             :            "NEONLdStTable is not sorted!");
     354             :     TableChecked = true;
     355             :   }
     356             : #endif
     357             : 
     358             :   auto I = std::lower_bound(std::begin(NEONLdStTable),
     359             :                             std::end(NEONLdStTable), Opcode);
     360         333 :   if (I != std::end(NEONLdStTable) && I->PseudoOpc == Opcode)
     361             :     return I;
     362             :   return nullptr;
     363             : }
     364             : 
     365             : /// GetDSubRegs - Get 4 D subregisters of a Q, QQ, or QQQQ register,
     366             : /// corresponding to the specified register spacing.  Not all of the results
     367             : /// are necessarily valid, e.g., a Q register only has 2 D subregisters.
     368         338 : static void GetDSubRegs(unsigned Reg, NEONRegSpacing RegSpc,
     369             :                         const TargetRegisterInfo *TRI, unsigned &D0,
     370             :                         unsigned &D1, unsigned &D2, unsigned &D3) {
     371         338 :   if (RegSpc == SingleSpc) {
     372         152 :     D0 = TRI->getSubReg(Reg, ARM::dsub_0);
     373         152 :     D1 = TRI->getSubReg(Reg, ARM::dsub_1);
     374         152 :     D2 = TRI->getSubReg(Reg, ARM::dsub_2);
     375         152 :     D3 = TRI->getSubReg(Reg, ARM::dsub_3);
     376         186 :   } else if (RegSpc == EvenDblSpc) {
     377          94 :     D0 = TRI->getSubReg(Reg, ARM::dsub_0);
     378          94 :     D1 = TRI->getSubReg(Reg, ARM::dsub_2);
     379          94 :     D2 = TRI->getSubReg(Reg, ARM::dsub_4);
     380          94 :     D3 = TRI->getSubReg(Reg, ARM::dsub_6);
     381             :   } else {
     382             :     assert(RegSpc == OddDblSpc && "unknown register spacing");
     383          92 :     D0 = TRI->getSubReg(Reg, ARM::dsub_1);
     384          92 :     D1 = TRI->getSubReg(Reg, ARM::dsub_3);
     385          92 :     D2 = TRI->getSubReg(Reg, ARM::dsub_5);
     386          92 :     D3 = TRI->getSubReg(Reg, ARM::dsub_7);
     387             :   }
     388         338 : }
     389             : 
     390             : /// ExpandVLD - Translate VLD pseudo instructions with Q, QQ or QQQQ register
     391             : /// operands to real VLD instructions with D register operands.
     392          93 : void ARMExpandPseudo::ExpandVLD(MachineBasicBlock::iterator &MBBI) {
     393             :   MachineInstr &MI = *MBBI;
     394          93 :   MachineBasicBlock &MBB = *MI.getParent();
     395             : 
     396          93 :   const NEONLdStTableEntry *TableEntry = LookupNEONLdSt(MI.getOpcode());
     397             :   assert(TableEntry && TableEntry->IsLoad && "NEONLdStTable lookup failed");
     398          93 :   NEONRegSpacing RegSpc = (NEONRegSpacing)TableEntry->RegSpacing;
     399          93 :   unsigned NumRegs = TableEntry->NumRegs;
     400             : 
     401             :   MachineInstrBuilder MIB = BuildMI(MBB, MBBI, MI.getDebugLoc(),
     402         186 :                                     TII->get(TableEntry->RealOpc));
     403             :   unsigned OpIdx = 0;
     404             : 
     405          93 :   bool DstIsDead = MI.getOperand(OpIdx).isDead();
     406          93 :   unsigned DstReg = MI.getOperand(OpIdx++).getReg();
     407             :   unsigned D0, D1, D2, D3;
     408          93 :   GetDSubRegs(DstReg, RegSpc, TRI, D0, D1, D2, D3);
     409          93 :   MIB.addReg(D0, RegState::Define | getDeadRegState(DstIsDead));
     410          93 :   if (NumRegs > 1 && TableEntry->copyAllListRegs)
     411          71 :     MIB.addReg(D1, RegState::Define | getDeadRegState(DstIsDead));
     412          93 :   if (NumRegs > 2 && TableEntry->copyAllListRegs)
     413          71 :     MIB.addReg(D2, RegState::Define | getDeadRegState(DstIsDead));
     414          93 :   if (NumRegs > 3 && TableEntry->copyAllListRegs)
     415          20 :     MIB.addReg(D3, RegState::Define | getDeadRegState(DstIsDead));
     416             : 
     417          93 :   if (TableEntry->isUpdating)
     418          38 :     MIB.add(MI.getOperand(OpIdx++));
     419             : 
     420             :   // Copy the addrmode6 operands.
     421          93 :   MIB.add(MI.getOperand(OpIdx++));
     422          93 :   MIB.add(MI.getOperand(OpIdx++));
     423             :   // Copy the am6offset operand.
     424          93 :   if (TableEntry->hasWritebackOperand)
     425          30 :     MIB.add(MI.getOperand(OpIdx++));
     426             : 
     427             :   // For an instruction writing double-spaced subregs, the pseudo instruction
     428             :   // has an extra operand that is a use of the super-register.  Record the
     429             :   // operand index and skip over it.
     430             :   unsigned SrcOpIdx = 0;
     431          93 :   if (RegSpc == EvenDblSpc || RegSpc == OddDblSpc)
     432          39 :     SrcOpIdx = OpIdx++;
     433             : 
     434             :   // Copy the predicate operands.
     435          93 :   MIB.add(MI.getOperand(OpIdx++));
     436          93 :   MIB.add(MI.getOperand(OpIdx++));
     437             : 
     438             :   // Copy the super-register source operand used for double-spaced subregs over
     439             :   // to the new instruction as an implicit operand.
     440          93 :   if (SrcOpIdx != 0) {
     441          78 :     MachineOperand MO = MI.getOperand(SrcOpIdx);
     442             :     MO.setImplicit(true);
     443             :     MIB.add(MO);
     444             :   }
     445             :   // Add an implicit def for the super-register.
     446          93 :   MIB.addReg(DstReg, RegState::ImplicitDefine | getDeadRegState(DstIsDead));
     447          93 :   TransferImpOps(MI, MIB, MIB);
     448             : 
     449             :   // Transfer memoperands.
     450          93 :   MIB->setMemRefs(MI.memoperands_begin(), MI.memoperands_end());
     451             : 
     452          93 :   MI.eraseFromParent();
     453          93 : }
     454             : 
     455             : /// ExpandVST - Translate VST pseudo instructions with Q, QQ or QQQQ register
     456             : /// operands to real VST instructions with D register operands.
     457          77 : void ARMExpandPseudo::ExpandVST(MachineBasicBlock::iterator &MBBI) {
     458             :   MachineInstr &MI = *MBBI;
     459          77 :   MachineBasicBlock &MBB = *MI.getParent();
     460             : 
     461          77 :   const NEONLdStTableEntry *TableEntry = LookupNEONLdSt(MI.getOpcode());
     462             :   assert(TableEntry && !TableEntry->IsLoad && "NEONLdStTable lookup failed");
     463          77 :   NEONRegSpacing RegSpc = (NEONRegSpacing)TableEntry->RegSpacing;
     464          77 :   unsigned NumRegs = TableEntry->NumRegs;
     465             : 
     466             :   MachineInstrBuilder MIB = BuildMI(MBB, MBBI, MI.getDebugLoc(),
     467         154 :                                     TII->get(TableEntry->RealOpc));
     468             :   unsigned OpIdx = 0;
     469          77 :   if (TableEntry->isUpdating)
     470          26 :     MIB.add(MI.getOperand(OpIdx++));
     471             : 
     472             :   // Copy the addrmode6 operands.
     473          77 :   MIB.add(MI.getOperand(OpIdx++));
     474          77 :   MIB.add(MI.getOperand(OpIdx++));
     475             :   // Copy the am6offset operand.
     476          77 :   if (TableEntry->hasWritebackOperand)
     477          22 :     MIB.add(MI.getOperand(OpIdx++));
     478             : 
     479          77 :   bool SrcIsKill = MI.getOperand(OpIdx).isKill();
     480             :   bool SrcIsUndef = MI.getOperand(OpIdx).isUndef();
     481          77 :   unsigned SrcReg = MI.getOperand(OpIdx++).getReg();
     482             :   unsigned D0, D1, D2, D3;
     483          77 :   GetDSubRegs(SrcReg, RegSpc, TRI, D0, D1, D2, D3);
     484          77 :   MIB.addReg(D0, getUndefRegState(SrcIsUndef));
     485          77 :   if (NumRegs > 1 && TableEntry->copyAllListRegs)
     486          57 :     MIB.addReg(D1, getUndefRegState(SrcIsUndef));
     487          77 :   if (NumRegs > 2 && TableEntry->copyAllListRegs)
     488          57 :     MIB.addReg(D2, getUndefRegState(SrcIsUndef));
     489          77 :   if (NumRegs > 3 && TableEntry->copyAllListRegs)
     490          32 :     MIB.addReg(D3, getUndefRegState(SrcIsUndef));
     491             : 
     492             :   // Copy the predicate operands.
     493          77 :   MIB.add(MI.getOperand(OpIdx++));
     494          77 :   MIB.add(MI.getOperand(OpIdx++));
     495             : 
     496          77 :   if (SrcIsKill && !SrcIsUndef) // Add an implicit kill for the super-reg.
     497          50 :     MIB->addRegisterKilled(SrcReg, TRI, true);
     498          27 :   else if (!SrcIsUndef)
     499          26 :     MIB.addReg(SrcReg, RegState::Implicit); // Add implicit uses for src reg.
     500          77 :   TransferImpOps(MI, MIB, MIB);
     501             : 
     502             :   // Transfer memoperands.
     503          77 :   MIB->setMemRefs(MI.memoperands_begin(), MI.memoperands_end());
     504             : 
     505          77 :   MI.eraseFromParent();
     506          77 : }
     507             : 
     508             : /// ExpandLaneOp - Translate VLD*LN and VST*LN instructions with Q, QQ or QQQQ
     509             : /// register operands to real instructions with D register operands.
     510         163 : void ARMExpandPseudo::ExpandLaneOp(MachineBasicBlock::iterator &MBBI) {
     511             :   MachineInstr &MI = *MBBI;
     512         163 :   MachineBasicBlock &MBB = *MI.getParent();
     513             : 
     514         163 :   const NEONLdStTableEntry *TableEntry = LookupNEONLdSt(MI.getOpcode());
     515             :   assert(TableEntry && "NEONLdStTable lookup failed");
     516         163 :   NEONRegSpacing RegSpc = (NEONRegSpacing)TableEntry->RegSpacing;
     517         163 :   unsigned NumRegs = TableEntry->NumRegs;
     518         163 :   unsigned RegElts = TableEntry->RegElts;
     519             : 
     520             :   MachineInstrBuilder MIB = BuildMI(MBB, MBBI, MI.getDebugLoc(),
     521         326 :                                     TII->get(TableEntry->RealOpc));
     522             :   unsigned OpIdx = 0;
     523             :   // The lane operand is always the 3rd from last operand, before the 2
     524             :   // predicate operands.
     525         489 :   unsigned Lane = MI.getOperand(MI.getDesc().getNumOperands() - 3).getImm();
     526             : 
     527             :   // Adjust the lane and spacing as needed for Q registers.
     528             :   assert(RegSpc != OddDblSpc && "unexpected register spacing for VLD/VST-lane");
     529         163 :   if (RegSpc == EvenDblSpc && Lane >= RegElts) {
     530             :     RegSpc = OddDblSpc;
     531          55 :     Lane -= RegElts;
     532             :   }
     533             :   assert(Lane < RegElts && "out of range lane for VLD/VST-lane");
     534             : 
     535         163 :   unsigned D0 = 0, D1 = 0, D2 = 0, D3 = 0;
     536             :   unsigned DstReg = 0;
     537             :   bool DstIsDead = false;
     538         163 :   if (TableEntry->IsLoad) {
     539             :     DstIsDead = MI.getOperand(OpIdx).isDead();
     540          74 :     DstReg = MI.getOperand(OpIdx++).getReg();
     541          74 :     GetDSubRegs(DstReg, RegSpc, TRI, D0, D1, D2, D3);
     542          74 :     MIB.addReg(D0, RegState::Define | getDeadRegState(DstIsDead));
     543          74 :     if (NumRegs > 1)
     544          60 :       MIB.addReg(D1, RegState::Define | getDeadRegState(DstIsDead));
     545          74 :     if (NumRegs > 2)
     546          37 :       MIB.addReg(D2, RegState::Define | getDeadRegState(DstIsDead));
     547          74 :     if (NumRegs > 3)
     548          17 :       MIB.addReg(D3, RegState::Define | getDeadRegState(DstIsDead));
     549             :   }
     550             : 
     551         163 :   if (TableEntry->isUpdating)
     552          11 :     MIB.add(MI.getOperand(OpIdx++));
     553             : 
     554             :   // Copy the addrmode6 operands.
     555         163 :   MIB.add(MI.getOperand(OpIdx++));
     556         163 :   MIB.add(MI.getOperand(OpIdx++));
     557             :   // Copy the am6offset operand.
     558         163 :   if (TableEntry->hasWritebackOperand)
     559          11 :     MIB.add(MI.getOperand(OpIdx++));
     560             : 
     561             :   // Grab the super-register source.
     562         326 :   MachineOperand MO = MI.getOperand(OpIdx++);
     563         163 :   if (!TableEntry->IsLoad)
     564          89 :     GetDSubRegs(MO.getReg(), RegSpc, TRI, D0, D1, D2, D3);
     565             : 
     566             :   // Add the subregs as sources of the new instruction.
     567             :   unsigned SrcFlags = (getUndefRegState(MO.isUndef()) |
     568         163 :                        getKillRegState(MO.isKill()));
     569         163 :   MIB.addReg(D0, SrcFlags);
     570         163 :   if (NumRegs > 1)
     571          87 :     MIB.addReg(D1, SrcFlags);
     572         163 :   if (NumRegs > 2)
     573          55 :     MIB.addReg(D2, SrcFlags);
     574         163 :   if (NumRegs > 3)
     575          26 :     MIB.addReg(D3, SrcFlags);
     576             : 
     577             :   // Add the lane number operand.
     578         163 :   MIB.addImm(Lane);
     579         163 :   OpIdx += 1;
     580             : 
     581             :   // Copy the predicate operands.
     582         163 :   MIB.add(MI.getOperand(OpIdx++));
     583         163 :   MIB.add(MI.getOperand(OpIdx++));
     584             : 
     585             :   // Copy the super-register source to be an implicit source.
     586             :   MO.setImplicit(true);
     587             :   MIB.add(MO);
     588         163 :   if (TableEntry->IsLoad)
     589             :     // Add an implicit def for the super-register.
     590          74 :     MIB.addReg(DstReg, RegState::ImplicitDefine | getDeadRegState(DstIsDead));
     591         163 :   TransferImpOps(MI, MIB, MIB);
     592             :   // Transfer memoperands.
     593         163 :   MIB->setMemRefs(MI.memoperands_begin(), MI.memoperands_end());
     594         163 :   MI.eraseFromParent();
     595         163 : }
     596             : 
     597             : /// ExpandVTBL - Translate VTBL and VTBX pseudo instructions with Q or QQ
     598             : /// register operands to real instructions with D register operands.
     599           5 : void ARMExpandPseudo::ExpandVTBL(MachineBasicBlock::iterator &MBBI,
     600             :                                  unsigned Opc, bool IsExt) {
     601             :   MachineInstr &MI = *MBBI;
     602           5 :   MachineBasicBlock &MBB = *MI.getParent();
     603             : 
     604          10 :   MachineInstrBuilder MIB = BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(Opc));
     605             :   unsigned OpIdx = 0;
     606             : 
     607             :   // Transfer the destination register operand.
     608           5 :   MIB.add(MI.getOperand(OpIdx++));
     609           5 :   if (IsExt) {
     610           3 :     MachineOperand VdSrc(MI.getOperand(OpIdx++));
     611           3 :     VdSrc.setIsRenamable(false);
     612             :     MIB.add(VdSrc);
     613             :   }
     614             : 
     615           5 :   bool SrcIsKill = MI.getOperand(OpIdx).isKill();
     616           5 :   unsigned SrcReg = MI.getOperand(OpIdx++).getReg();
     617             :   unsigned D0, D1, D2, D3;
     618           5 :   GetDSubRegs(SrcReg, SingleSpc, TRI, D0, D1, D2, D3);
     619           5 :   MIB.addReg(D0);
     620             : 
     621             :   // Copy the other source register operand.
     622          10 :   MachineOperand VmSrc(MI.getOperand(OpIdx++));
     623           5 :   VmSrc.setIsRenamable(false);
     624             :   MIB.add(VmSrc);
     625             : 
     626             :   // Copy the predicate operands.
     627           5 :   MIB.add(MI.getOperand(OpIdx++));
     628           5 :   MIB.add(MI.getOperand(OpIdx++));
     629             : 
     630             :   // Add an implicit kill and use for the super-reg.
     631           5 :   MIB.addReg(SrcReg, RegState::Implicit | getKillRegState(SrcIsKill));
     632           5 :   TransferImpOps(MI, MIB, MIB);
     633           5 :   MI.eraseFromParent();
     634           5 : }
     635             : 
     636          57 : static bool IsAnAddressOperand(const MachineOperand &MO) {
     637             :   // This check is overly conservative.  Unless we are certain that the machine
     638             :   // operand is not a symbol reference, we return that it is a symbol reference.
     639             :   // This is important as the load pair may not be split up Windows.
     640          57 :   switch (MO.getType()) {
     641             :   case MachineOperand::MO_Register:
     642             :   case MachineOperand::MO_Immediate:
     643             :   case MachineOperand::MO_CImmediate:
     644             :   case MachineOperand::MO_FPImmediate:
     645             :     return false;
     646           0 :   case MachineOperand::MO_MachineBasicBlock:
     647           0 :     return true;
     648             :   case MachineOperand::MO_FrameIndex:
     649             :     return false;
     650          57 :   case MachineOperand::MO_ConstantPoolIndex:
     651             :   case MachineOperand::MO_TargetIndex:
     652             :   case MachineOperand::MO_JumpTableIndex:
     653             :   case MachineOperand::MO_ExternalSymbol:
     654             :   case MachineOperand::MO_GlobalAddress:
     655             :   case MachineOperand::MO_BlockAddress:
     656          57 :     return true;
     657             :   case MachineOperand::MO_RegisterMask:
     658             :   case MachineOperand::MO_RegisterLiveOut:
     659             :     return false;
     660           0 :   case MachineOperand::MO_Metadata:
     661             :   case MachineOperand::MO_MCSymbol:
     662           0 :     return true;
     663             :   case MachineOperand::MO_CFIIndex:
     664             :     return false;
     665           0 :   case MachineOperand::MO_IntrinsicID:
     666             :   case MachineOperand::MO_Predicate:
     667           0 :     llvm_unreachable("should not exist post-isel");
     668             :   }
     669           0 :   llvm_unreachable("unhandled machine operand type");
     670             : }
     671             : 
     672             : static MachineOperand makeImplicit(const MachineOperand &MO) {
     673        1072 :   MachineOperand NewMO = MO;
     674             :   NewMO.setImplicit();
     675             :   return NewMO;
     676             : }
     677             : 
     678        1613 : void ARMExpandPseudo::ExpandMOV32BitImm(MachineBasicBlock &MBB,
     679             :                                         MachineBasicBlock::iterator &MBBI) {
     680             :   MachineInstr &MI = *MBBI;
     681        1613 :   unsigned Opcode = MI.getOpcode();
     682        1613 :   unsigned PredReg = 0;
     683        1613 :   ARMCC::CondCodes Pred = getInstrPredicate(MI, PredReg);
     684        1613 :   unsigned DstReg = MI.getOperand(0).getReg();
     685             :   bool DstIsDead = MI.getOperand(0).isDead();
     686        1613 :   bool isCC = Opcode == ARM::MOVCCi32imm || Opcode == ARM::t2MOVCCi32imm;
     687        1613 :   const MachineOperand &MO = MI.getOperand(isCC ? 2 : 1);
     688        3226 :   bool RequiresBundling = STI->isTargetWindows() && IsAnAddressOperand(MO);
     689        1613 :   MachineInstrBuilder LO16, HI16;
     690             : 
     691        1799 :   if (!STI->hasV6T2Ops() &&
     692         186 :       (Opcode == ARM::MOVi32imm || Opcode == ARM::MOVCCi32imm)) {
     693             :     // FIXME Windows CE supports older ARM CPUs
     694             :     assert(!STI->isTargetWindows() && "Windows on ARM requires ARMv7+");
     695             : 
     696             :     // Expand into a movi + orr.
     697         338 :     LO16 = BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(ARM::MOVi), DstReg);
     698         507 :     HI16 = BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(ARM::ORRri))
     699         169 :       .addReg(DstReg, RegState::Define | getDeadRegState(DstIsDead))
     700         169 :       .addReg(DstReg);
     701             : 
     702             :     assert (MO.isImm() && "MOVi32imm w/ non-immediate source operand!");
     703         169 :     unsigned ImmVal = (unsigned)MO.getImm();
     704             :     unsigned SOImmValV1 = ARM_AM::getSOImmTwoPartFirst(ImmVal);
     705             :     unsigned SOImmValV2 = ARM_AM::getSOImmTwoPartSecond(ImmVal);
     706         169 :     LO16 = LO16.addImm(SOImmValV1);
     707         169 :     HI16 = HI16.addImm(SOImmValV2);
     708         169 :     LO16->setMemRefs(MI.memoperands_begin(), MI.memoperands_end());
     709         169 :     HI16->setMemRefs(MI.memoperands_begin(), MI.memoperands_end());
     710         507 :     LO16.addImm(Pred).addReg(PredReg).add(condCodeOp());
     711         338 :     HI16.addImm(Pred).addReg(PredReg).add(condCodeOp());
     712         169 :     if (isCC)
     713           0 :       LO16.add(makeImplicit(MI.getOperand(1)));
     714         169 :     TransferImpOps(MI, LO16, HI16);
     715         169 :     MI.eraseFromParent();
     716         169 :     return;
     717             :   }
     718             : 
     719             :   unsigned LO16Opc = 0;
     720             :   unsigned HI16Opc = 0;
     721        1444 :   if (Opcode == ARM::t2MOVi32imm || Opcode == ARM::t2MOVCCi32imm) {
     722             :     LO16Opc = ARM::t2MOVi16;
     723             :     HI16Opc = ARM::t2MOVTi16;
     724             :   } else {
     725             :     LO16Opc = ARM::MOVi16;
     726             :     HI16Opc = ARM::MOVTi16;
     727             :   }
     728             : 
     729        2888 :   LO16 = BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(LO16Opc), DstReg);
     730        4332 :   HI16 = BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(HI16Opc))
     731        1444 :     .addReg(DstReg, RegState::Define | getDeadRegState(DstIsDead))
     732        1444 :     .addReg(DstReg);
     733             : 
     734        1444 :   switch (MO.getType()) {
     735         509 :   case MachineOperand::MO_Immediate: {
     736         509 :     unsigned Imm = MO.getImm();
     737         509 :     unsigned Lo16 = Imm & 0xffff;
     738         509 :     unsigned Hi16 = (Imm >> 16) & 0xffff;
     739         509 :     LO16 = LO16.addImm(Lo16);
     740         509 :     HI16 = HI16.addImm(Hi16);
     741             :     break;
     742             :   }
     743          10 :   case MachineOperand::MO_ExternalSymbol: {
     744          10 :     const char *ES = MO.getSymbolName();
     745             :     unsigned TF = MO.getTargetFlags();
     746          10 :     LO16 = LO16.addExternalSymbol(ES, TF | ARMII::MO_LO16);
     747          10 :     HI16 = HI16.addExternalSymbol(ES, TF | ARMII::MO_HI16);
     748             :     break;
     749             :   }
     750         925 :   default: {
     751         925 :     const GlobalValue *GV = MO.getGlobal();
     752             :     unsigned TF = MO.getTargetFlags();
     753         925 :     LO16 = LO16.addGlobalAddress(GV, MO.getOffset(), TF | ARMII::MO_LO16);
     754         925 :     HI16 = HI16.addGlobalAddress(GV, MO.getOffset(), TF | ARMII::MO_HI16);
     755             :     break;
     756             :   }
     757             :   }
     758             : 
     759        1444 :   LO16->setMemRefs(MI.memoperands_begin(), MI.memoperands_end());
     760        1444 :   HI16->setMemRefs(MI.memoperands_begin(), MI.memoperands_end());
     761        2888 :   LO16.addImm(Pred).addReg(PredReg);
     762        1444 :   HI16.addImm(Pred).addReg(PredReg);
     763             : 
     764        1444 :   if (RequiresBundling)
     765          57 :     finalizeBundle(MBB, LO16->getIterator(), MBBI->getIterator());
     766             : 
     767        1444 :   if (isCC)
     768          40 :     LO16.add(makeImplicit(MI.getOperand(1)));
     769        1444 :   TransferImpOps(MI, LO16, HI16);
     770        1444 :   MI.eraseFromParent();
     771             : }
     772             : 
     773             : /// Expand a CMP_SWAP pseudo-inst to an ldrex/strex loop as simply as
     774             : /// possible. This only gets used at -O0 so we don't care about efficiency of
     775             : /// the generated code.
     776           6 : bool ARMExpandPseudo::ExpandCMP_SWAP(MachineBasicBlock &MBB,
     777             :                                      MachineBasicBlock::iterator MBBI,
     778             :                                      unsigned LdrexOp, unsigned StrexOp,
     779             :                                      unsigned UxtOp,
     780             :                                      MachineBasicBlock::iterator &NextMBBI) {
     781           6 :   bool IsThumb = STI->isThumb();
     782             :   MachineInstr &MI = *MBBI;
     783             :   DebugLoc DL = MI.getDebugLoc();
     784           6 :   const MachineOperand &Dest = MI.getOperand(0);
     785           6 :   unsigned TempReg = MI.getOperand(1).getReg();
     786             :   // Duplicating undef operands into 2 instructions does not guarantee the same
     787             :   // value on both; However undef should be replaced by xzr anyway.
     788             :   assert(!MI.getOperand(2).isUndef() && "cannot handle undef");
     789           6 :   unsigned AddrReg = MI.getOperand(2).getReg();
     790           6 :   unsigned DesiredReg = MI.getOperand(3).getReg();
     791           6 :   unsigned NewReg = MI.getOperand(4).getReg();
     792             : 
     793           6 :   MachineFunction *MF = MBB.getParent();
     794           6 :   auto LoadCmpBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
     795           6 :   auto StoreBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
     796           6 :   auto DoneBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
     797             : 
     798           6 :   MF->insert(++MBB.getIterator(), LoadCmpBB);
     799           6 :   MF->insert(++LoadCmpBB->getIterator(), StoreBB);
     800           6 :   MF->insert(++StoreBB->getIterator(), DoneBB);
     801             : 
     802           6 :   if (UxtOp) {
     803             :     MachineInstrBuilder MIB =
     804          12 :         BuildMI(MBB, MBBI, DL, TII->get(UxtOp), DesiredReg)
     805           4 :             .addReg(DesiredReg, RegState::Kill);
     806           4 :     if (!IsThumb)
     807             :       MIB.addImm(0);
     808           4 :     MIB.add(predOps(ARMCC::AL));
     809             :   }
     810             : 
     811             :   // .Lloadcmp:
     812             :   //     ldrex rDest, [rAddr]
     813             :   //     cmp rDest, rDesired
     814             :   //     bne .Ldone
     815             : 
     816           6 :   MachineInstrBuilder MIB;
     817          12 :   MIB = BuildMI(LoadCmpBB, DL, TII->get(LdrexOp), Dest.getReg());
     818           6 :   MIB.addReg(AddrReg);
     819           6 :   if (LdrexOp == ARM::t2LDREX)
     820             :     MIB.addImm(0); // a 32-bit Thumb ldrex (only) allows an offset.
     821           6 :   MIB.add(predOps(ARMCC::AL));
     822             : 
     823           6 :   unsigned CMPrr = IsThumb ? ARM::tCMPhir : ARM::CMPrr;
     824          18 :   BuildMI(LoadCmpBB, DL, TII->get(CMPrr))
     825           6 :       .addReg(Dest.getReg(), getKillRegState(Dest.isDead()))
     826           6 :       .addReg(DesiredReg)
     827           6 :       .add(predOps(ARMCC::AL));
     828           6 :   unsigned Bcc = IsThumb ? ARM::tBcc : ARM::Bcc;
     829          18 :   BuildMI(LoadCmpBB, DL, TII->get(Bcc))
     830             :       .addMBB(DoneBB)
     831             :       .addImm(ARMCC::NE)
     832           6 :       .addReg(ARM::CPSR, RegState::Kill);
     833           6 :   LoadCmpBB->addSuccessor(DoneBB);
     834           6 :   LoadCmpBB->addSuccessor(StoreBB);
     835             : 
     836             :   // .Lstore:
     837             :   //     strex rTempReg, rNew, [rAddr]
     838             :   //     cmp rTempReg, #0
     839             :   //     bne .Lloadcmp
     840          18 :   MIB = BuildMI(StoreBB, DL, TII->get(StrexOp), TempReg)
     841           6 :     .addReg(NewReg)
     842           6 :     .addReg(AddrReg);
     843           6 :   if (StrexOp == ARM::t2STREX)
     844             :     MIB.addImm(0); // a 32-bit Thumb strex (only) allows an offset.
     845           6 :   MIB.add(predOps(ARMCC::AL));
     846             : 
     847           6 :   unsigned CMPri = IsThumb ? ARM::t2CMPri : ARM::CMPri;
     848          18 :   BuildMI(StoreBB, DL, TII->get(CMPri))
     849           6 :       .addReg(TempReg, RegState::Kill)
     850             :       .addImm(0)
     851           6 :       .add(predOps(ARMCC::AL));
     852          18 :   BuildMI(StoreBB, DL, TII->get(Bcc))
     853             :       .addMBB(LoadCmpBB)
     854             :       .addImm(ARMCC::NE)
     855           6 :       .addReg(ARM::CPSR, RegState::Kill);
     856           6 :   StoreBB->addSuccessor(LoadCmpBB);
     857           6 :   StoreBB->addSuccessor(DoneBB);
     858             : 
     859             :   DoneBB->splice(DoneBB->end(), &MBB, MI, MBB.end());
     860           6 :   DoneBB->transferSuccessors(&MBB);
     861             : 
     862           6 :   MBB.addSuccessor(LoadCmpBB);
     863             : 
     864           6 :   NextMBBI = MBB.end();
     865           6 :   MI.eraseFromParent();
     866             : 
     867             :   // Recompute livein lists.
     868             :   LivePhysRegs LiveRegs;
     869           6 :   computeAndAddLiveIns(LiveRegs, *DoneBB);
     870           6 :   computeAndAddLiveIns(LiveRegs, *StoreBB);
     871           6 :   computeAndAddLiveIns(LiveRegs, *LoadCmpBB);
     872             :   // Do an extra pass around the loop to get loop carried registers right.
     873           6 :   StoreBB->clearLiveIns();
     874           6 :   computeAndAddLiveIns(LiveRegs, *StoreBB);
     875           6 :   LoadCmpBB->clearLiveIns();
     876           6 :   computeAndAddLiveIns(LiveRegs, *LoadCmpBB);
     877             : 
     878           6 :   return true;
     879             : }
     880             : 
     881             : /// ARM's ldrexd/strexd take a consecutive register pair (represented as a
     882             : /// single GPRPair register), Thumb's take two separate registers so we need to
     883             : /// extract the subregs from the pair.
     884          14 : static void addExclusiveRegPair(MachineInstrBuilder &MIB, MachineOperand &Reg,
     885             :                                 unsigned Flags, bool IsThumb,
     886             :                                 const TargetRegisterInfo *TRI) {
     887          14 :   if (IsThumb) {
     888           6 :     unsigned RegLo = TRI->getSubReg(Reg.getReg(), ARM::gsub_0);
     889           6 :     unsigned RegHi = TRI->getSubReg(Reg.getReg(), ARM::gsub_1);
     890           6 :     MIB.addReg(RegLo, Flags | getKillRegState(Reg.isDead()));
     891           6 :     MIB.addReg(RegHi, Flags | getKillRegState(Reg.isDead()));
     892             :   } else
     893           8 :     MIB.addReg(Reg.getReg(), Flags | getKillRegState(Reg.isDead()));
     894          14 : }
     895             : 
     896             : /// Expand a 64-bit CMP_SWAP to an ldrexd/strexd loop.
     897           7 : bool ARMExpandPseudo::ExpandCMP_SWAP_64(MachineBasicBlock &MBB,
     898             :                                         MachineBasicBlock::iterator MBBI,
     899             :                                         MachineBasicBlock::iterator &NextMBBI) {
     900           7 :   bool IsThumb = STI->isThumb();
     901             :   MachineInstr &MI = *MBBI;
     902             :   DebugLoc DL = MI.getDebugLoc();
     903           7 :   MachineOperand &Dest = MI.getOperand(0);
     904           7 :   unsigned TempReg = MI.getOperand(1).getReg();
     905             :   // Duplicating undef operands into 2 instructions does not guarantee the same
     906             :   // value on both; However undef should be replaced by xzr anyway.
     907             :   assert(!MI.getOperand(2).isUndef() && "cannot handle undef");
     908           7 :   unsigned AddrReg = MI.getOperand(2).getReg();
     909           7 :   unsigned DesiredReg = MI.getOperand(3).getReg();
     910           7 :   MachineOperand New = MI.getOperand(4);
     911             :   New.setIsKill(false);
     912             : 
     913           7 :   unsigned DestLo = TRI->getSubReg(Dest.getReg(), ARM::gsub_0);
     914           7 :   unsigned DestHi = TRI->getSubReg(Dest.getReg(), ARM::gsub_1);
     915           7 :   unsigned DesiredLo = TRI->getSubReg(DesiredReg, ARM::gsub_0);
     916           7 :   unsigned DesiredHi = TRI->getSubReg(DesiredReg, ARM::gsub_1);
     917             : 
     918           7 :   MachineFunction *MF = MBB.getParent();
     919           7 :   auto LoadCmpBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
     920           7 :   auto StoreBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
     921           7 :   auto DoneBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
     922             : 
     923           7 :   MF->insert(++MBB.getIterator(), LoadCmpBB);
     924           7 :   MF->insert(++LoadCmpBB->getIterator(), StoreBB);
     925           7 :   MF->insert(++StoreBB->getIterator(), DoneBB);
     926             : 
     927             :   // .Lloadcmp:
     928             :   //     ldrexd rDestLo, rDestHi, [rAddr]
     929             :   //     cmp rDestLo, rDesiredLo
     930             :   //     sbcs dead rTempReg, rDestHi, rDesiredHi
     931             :   //     bne .Ldone
     932           7 :   unsigned LDREXD = IsThumb ? ARM::t2LDREXD : ARM::LDREXD;
     933           7 :   MachineInstrBuilder MIB;
     934          14 :   MIB = BuildMI(LoadCmpBB, DL, TII->get(LDREXD));
     935           7 :   addExclusiveRegPair(MIB, Dest, RegState::Define, IsThumb, TRI);
     936          14 :   MIB.addReg(AddrReg).add(predOps(ARMCC::AL));
     937             : 
     938           7 :   unsigned CMPrr = IsThumb ? ARM::tCMPhir : ARM::CMPrr;
     939          21 :   BuildMI(LoadCmpBB, DL, TII->get(CMPrr))
     940           7 :       .addReg(DestLo, getKillRegState(Dest.isDead()))
     941           7 :       .addReg(DesiredLo)
     942           7 :       .add(predOps(ARMCC::AL));
     943             : 
     944          21 :   BuildMI(LoadCmpBB, DL, TII->get(CMPrr))
     945           7 :       .addReg(DestHi, getKillRegState(Dest.isDead()))
     946           7 :       .addReg(DesiredHi)
     947           7 :       .addImm(ARMCC::EQ).addReg(ARM::CPSR, RegState::Kill);
     948             : 
     949           7 :   unsigned Bcc = IsThumb ? ARM::tBcc : ARM::Bcc;
     950          21 :   BuildMI(LoadCmpBB, DL, TII->get(Bcc))
     951             :       .addMBB(DoneBB)
     952             :       .addImm(ARMCC::NE)
     953           7 :       .addReg(ARM::CPSR, RegState::Kill);
     954           7 :   LoadCmpBB->addSuccessor(DoneBB);
     955           7 :   LoadCmpBB->addSuccessor(StoreBB);
     956             : 
     957             :   // .Lstore:
     958             :   //     strexd rTempReg, rNewLo, rNewHi, [rAddr]
     959             :   //     cmp rTempReg, #0
     960             :   //     bne .Lloadcmp
     961           7 :   unsigned STREXD = IsThumb ? ARM::t2STREXD : ARM::STREXD;
     962          14 :   MIB = BuildMI(StoreBB, DL, TII->get(STREXD), TempReg);
     963           7 :   addExclusiveRegPair(MIB, New, 0, IsThumb, TRI);
     964          14 :   MIB.addReg(AddrReg).add(predOps(ARMCC::AL));
     965             : 
     966           7 :   unsigned CMPri = IsThumb ? ARM::t2CMPri : ARM::CMPri;
     967          21 :   BuildMI(StoreBB, DL, TII->get(CMPri))
     968           7 :       .addReg(TempReg, RegState::Kill)
     969             :       .addImm(0)
     970           7 :       .add(predOps(ARMCC::AL));
     971          21 :   BuildMI(StoreBB, DL, TII->get(Bcc))
     972             :       .addMBB(LoadCmpBB)
     973             :       .addImm(ARMCC::NE)
     974           7 :       .addReg(ARM::CPSR, RegState::Kill);
     975           7 :   StoreBB->addSuccessor(LoadCmpBB);
     976           7 :   StoreBB->addSuccessor(DoneBB);
     977             : 
     978             :   DoneBB->splice(DoneBB->end(), &MBB, MI, MBB.end());
     979           7 :   DoneBB->transferSuccessors(&MBB);
     980             : 
     981           7 :   MBB.addSuccessor(LoadCmpBB);
     982             : 
     983           7 :   NextMBBI = MBB.end();
     984           7 :   MI.eraseFromParent();
     985             : 
     986             :   // Recompute livein lists.
     987             :   LivePhysRegs LiveRegs;
     988           7 :   computeAndAddLiveIns(LiveRegs, *DoneBB);
     989           7 :   computeAndAddLiveIns(LiveRegs, *StoreBB);
     990           7 :   computeAndAddLiveIns(LiveRegs, *LoadCmpBB);
     991             :   // Do an extra pass around the loop to get loop carried registers right.
     992           7 :   StoreBB->clearLiveIns();
     993           7 :   computeAndAddLiveIns(LiveRegs, *StoreBB);
     994           7 :   LoadCmpBB->clearLiveIns();
     995           7 :   computeAndAddLiveIns(LiveRegs, *LoadCmpBB);
     996             : 
     997           7 :   return true;
     998             : }
     999             : 
    1000             : 
    1001      153147 : bool ARMExpandPseudo::ExpandMI(MachineBasicBlock &MBB,
    1002             :                                MachineBasicBlock::iterator MBBI,
    1003             :                                MachineBasicBlock::iterator &NextMBBI) {
    1004             :   MachineInstr &MI = *MBBI;
    1005      153147 :   unsigned Opcode = MI.getOpcode();
    1006      153147 :   switch (Opcode) {
    1007             :     default:
    1008             :       return false;
    1009             : 
    1010         554 :     case ARM::TCRETURNdi:
    1011             :     case ARM::TCRETURNri: {
    1012         554 :       MachineBasicBlock::iterator MBBI = MBB.getLastNonDebugInstr();
    1013             :       assert(MBBI->isReturn() &&
    1014             :              "Can only insert epilog into returning blocks");
    1015         554 :       unsigned RetOpcode = MBBI->getOpcode();
    1016             :       DebugLoc dl = MBBI->getDebugLoc();
    1017             :       const ARMBaseInstrInfo &TII = *static_cast<const ARMBaseInstrInfo *>(
    1018         554 :           MBB.getParent()->getSubtarget().getInstrInfo());
    1019             : 
    1020             :       // Tail call return: adjust the stack pointer and jump to callee.
    1021         554 :       MBBI = MBB.getLastNonDebugInstr();
    1022         554 :       MachineOperand &JumpTarget = MBBI->getOperand(0);
    1023             : 
    1024             :       // Jump to label or value in register.
    1025         554 :       if (RetOpcode == ARM::TCRETURNdi) {
    1026             :         unsigned TCOpcode =
    1027         535 :             STI->isThumb()
    1028         886 :                 ? (STI->isTargetMachO() ? ARM::tTAILJMPd : ARM::tTAILJMPdND)
    1029             :                 : ARM::TAILJMPd;
    1030        1070 :         MachineInstrBuilder MIB = BuildMI(MBB, MBBI, dl, TII.get(TCOpcode));
    1031         535 :         if (JumpTarget.isGlobal())
    1032             :           MIB.addGlobalAddress(JumpTarget.getGlobal(), JumpTarget.getOffset(),
    1033         636 :                                JumpTarget.getTargetFlags());
    1034             :         else {
    1035             :           assert(JumpTarget.isSymbol());
    1036             :           MIB.addExternalSymbol(JumpTarget.getSymbolName(),
    1037         217 :                                 JumpTarget.getTargetFlags());
    1038             :         }
    1039             : 
    1040             :         // Add the default predicate in Thumb mode.
    1041         535 :         if (STI->isThumb())
    1042         351 :           MIB.add(predOps(ARMCC::AL));
    1043          19 :       } else if (RetOpcode == ARM::TCRETURNri) {
    1044             :         unsigned Opcode =
    1045          28 :           STI->isThumb() ? ARM::tTAILJMPr
    1046           9 :                          : (STI->hasV4TOps() ? ARM::TAILJMPr : ARM::TAILJMPr4);
    1047          38 :         BuildMI(MBB, MBBI, dl,
    1048          19 :                 TII.get(Opcode))
    1049          19 :             .addReg(JumpTarget.getReg(), RegState::Kill);
    1050             :       }
    1051             : 
    1052             :       auto NewMI = std::prev(MBBI);
    1053        1701 :       for (unsigned i = 1, e = MBBI->getNumOperands(); i != e; ++i)
    1054        2294 :         NewMI->addOperand(MBBI->getOperand(i));
    1055             : 
    1056             :       // Delete the pseudo instruction TCRETURN.
    1057         554 :       MBB.erase(MBBI);
    1058             :       MBBI = NewMI;
    1059             :       return true;
    1060             :     }
    1061          47 :     case ARM::VMOVScc:
    1062             :     case ARM::VMOVDcc: {
    1063          47 :       unsigned newOpc = Opcode == ARM::VMOVScc ? ARM::VMOVS : ARM::VMOVD;
    1064          94 :       BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(newOpc),
    1065          94 :               MI.getOperand(1).getReg())
    1066          47 :           .add(MI.getOperand(2))
    1067          47 :           .addImm(MI.getOperand(3).getImm()) // 'pred'
    1068          47 :           .add(MI.getOperand(4))
    1069          94 :           .add(makeImplicit(MI.getOperand(1)));
    1070             : 
    1071          47 :       MI.eraseFromParent();
    1072          47 :       return true;
    1073             :     }
    1074         273 :     case ARM::t2MOVCCr:
    1075             :     case ARM::MOVCCr: {
    1076         273 :       unsigned Opc = AFI->isThumbFunction() ? ARM::t2MOVr : ARM::MOVr;
    1077         546 :       BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(Opc),
    1078         546 :               MI.getOperand(1).getReg())
    1079         273 :           .add(MI.getOperand(2))
    1080         273 :           .addImm(MI.getOperand(3).getImm()) // 'pred'
    1081         273 :           .add(MI.getOperand(4))
    1082         273 :           .add(condCodeOp()) // 's' bit
    1083         546 :           .add(makeImplicit(MI.getOperand(1)));
    1084             : 
    1085         273 :       MI.eraseFromParent();
    1086         273 :       return true;
    1087             :     }
    1088           7 :     case ARM::MOVCCsi: {
    1089          14 :       BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(ARM::MOVsi),
    1090          14 :               (MI.getOperand(1).getReg()))
    1091           7 :           .add(MI.getOperand(2))
    1092           7 :           .addImm(MI.getOperand(3).getImm())
    1093           7 :           .addImm(MI.getOperand(4).getImm()) // 'pred'
    1094           7 :           .add(MI.getOperand(5))
    1095           7 :           .add(condCodeOp()) // 's' bit
    1096          14 :           .add(makeImplicit(MI.getOperand(1)));
    1097             : 
    1098           7 :       MI.eraseFromParent();
    1099           7 :       return true;
    1100             :     }
    1101           7 :     case ARM::MOVCCsr: {
    1102          14 :       BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(ARM::MOVsr),
    1103          14 :               (MI.getOperand(1).getReg()))
    1104           7 :           .add(MI.getOperand(2))
    1105           7 :           .add(MI.getOperand(3))
    1106           7 :           .addImm(MI.getOperand(4).getImm())
    1107           7 :           .addImm(MI.getOperand(5).getImm()) // 'pred'
    1108           7 :           .add(MI.getOperand(6))
    1109           7 :           .add(condCodeOp()) // 's' bit
    1110          14 :           .add(makeImplicit(MI.getOperand(1)));
    1111             : 
    1112           7 :       MI.eraseFromParent();
    1113           7 :       return true;
    1114             :     }
    1115          71 :     case ARM::t2MOVCCi16:
    1116             :     case ARM::MOVCCi16: {
    1117          71 :       unsigned NewOpc = AFI->isThumbFunction() ? ARM::t2MOVi16 : ARM::MOVi16;
    1118         142 :       BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(NewOpc),
    1119         142 :               MI.getOperand(1).getReg())
    1120          71 :           .addImm(MI.getOperand(2).getImm())
    1121          71 :           .addImm(MI.getOperand(3).getImm()) // 'pred'
    1122          71 :           .add(MI.getOperand(4))
    1123         142 :           .add(makeImplicit(MI.getOperand(1)));
    1124          71 :       MI.eraseFromParent();
    1125          71 :       return true;
    1126             :     }
    1127         591 :     case ARM::t2MOVCCi:
    1128             :     case ARM::MOVCCi: {
    1129         591 :       unsigned Opc = AFI->isThumbFunction() ? ARM::t2MOVi : ARM::MOVi;
    1130        1182 :       BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(Opc),
    1131        1182 :               MI.getOperand(1).getReg())
    1132         591 :           .addImm(MI.getOperand(2).getImm())
    1133         591 :           .addImm(MI.getOperand(3).getImm()) // 'pred'
    1134         591 :           .add(MI.getOperand(4))
    1135         591 :           .add(condCodeOp()) // 's' bit
    1136        1182 :           .add(makeImplicit(MI.getOperand(1)));
    1137             : 
    1138         591 :       MI.eraseFromParent();
    1139         591 :       return true;
    1140             :     }
    1141          53 :     case ARM::t2MVNCCi:
    1142             :     case ARM::MVNCCi: {
    1143          53 :       unsigned Opc = AFI->isThumbFunction() ? ARM::t2MVNi : ARM::MVNi;
    1144         106 :       BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(Opc),
    1145         106 :               MI.getOperand(1).getReg())
    1146          53 :           .addImm(MI.getOperand(2).getImm())
    1147          53 :           .addImm(MI.getOperand(3).getImm()) // 'pred'
    1148          53 :           .add(MI.getOperand(4))
    1149          53 :           .add(condCodeOp()) // 's' bit
    1150         106 :           .add(makeImplicit(MI.getOperand(1)));
    1151             : 
    1152          53 :       MI.eraseFromParent();
    1153          53 :       return true;
    1154             :     }
    1155           3 :     case ARM::t2MOVCClsl:
    1156             :     case ARM::t2MOVCClsr:
    1157             :     case ARM::t2MOVCCasr:
    1158             :     case ARM::t2MOVCCror: {
    1159             :       unsigned NewOpc;
    1160           3 :       switch (Opcode) {
    1161             :       case ARM::t2MOVCClsl: NewOpc = ARM::t2LSLri; break;
    1162           1 :       case ARM::t2MOVCClsr: NewOpc = ARM::t2LSRri; break;
    1163           0 :       case ARM::t2MOVCCasr: NewOpc = ARM::t2ASRri; break;
    1164           1 :       case ARM::t2MOVCCror: NewOpc = ARM::t2RORri; break;
    1165           0 :       default: llvm_unreachable("unexpeced conditional move");
    1166             :       }
    1167           6 :       BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(NewOpc),
    1168           6 :               MI.getOperand(1).getReg())
    1169           3 :           .add(MI.getOperand(2))
    1170           3 :           .addImm(MI.getOperand(3).getImm())
    1171           3 :           .addImm(MI.getOperand(4).getImm()) // 'pred'
    1172           3 :           .add(MI.getOperand(5))
    1173           3 :           .add(condCodeOp()) // 's' bit
    1174           6 :           .add(makeImplicit(MI.getOperand(1)));
    1175           3 :       MI.eraseFromParent();
    1176           3 :       return true;
    1177             :     }
    1178          32 :     case ARM::Int_eh_sjlj_dispatchsetup: {
    1179          32 :       MachineFunction &MF = *MI.getParent()->getParent();
    1180          32 :       const ARMBaseInstrInfo *AII =
    1181             :         static_cast<const ARMBaseInstrInfo*>(TII);
    1182          32 :       const ARMBaseRegisterInfo &RI = AII->getRegisterInfo();
    1183             :       // For functions using a base pointer, we rematerialize it (via the frame
    1184             :       // pointer) here since eh.sjlj.setjmp and eh.sjlj.longjmp don't do it
    1185             :       // for us. Otherwise, expand to nothing.
    1186          32 :       if (RI.hasBasePointer(MF)) {
    1187           0 :         int32_t NumBytes = AFI->getFramePtrSpillOffset();
    1188           0 :         unsigned FramePtr = RI.getFrameRegister(MF);
    1189             :         assert(MF.getSubtarget().getFrameLowering()->hasFP(MF) &&
    1190             :                "base pointer without frame pointer?");
    1191             : 
    1192           0 :         if (AFI->isThumb2Function()) {
    1193           0 :           emitT2RegPlusImmediate(MBB, MBBI, MI.getDebugLoc(), ARM::R6,
    1194           0 :                                  FramePtr, -NumBytes, ARMCC::AL, 0, *TII);
    1195           0 :         } else if (AFI->isThumbFunction()) {
    1196           0 :           emitThumbRegPlusImmediate(MBB, MBBI, MI.getDebugLoc(), ARM::R6,
    1197           0 :                                     FramePtr, -NumBytes, *TII, RI);
    1198             :         } else {
    1199           0 :           emitARMRegPlusImmediate(MBB, MBBI, MI.getDebugLoc(), ARM::R6,
    1200             :                                   FramePtr, -NumBytes, ARMCC::AL, 0,
    1201           0 :                                   *TII);
    1202             :         }
    1203             :         // If there's dynamic realignment, adjust for it.
    1204           0 :         if (RI.needsStackRealignment(MF)) {
    1205           0 :           MachineFrameInfo &MFI = MF.getFrameInfo();
    1206           0 :           unsigned MaxAlign = MFI.getMaxAlignment();
    1207             :           assert (!AFI->isThumb1OnlyFunction());
    1208             :           // Emit bic r6, r6, MaxAlign
    1209             :           assert(MaxAlign <= 256 && "The BIC instruction cannot encode "
    1210             :                                     "immediates larger than 256 with all lower "
    1211             :                                     "bits set.");
    1212           0 :           unsigned bicOpc = AFI->isThumbFunction() ?
    1213             :             ARM::t2BICri : ARM::BICri;
    1214           0 :           BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(bicOpc), ARM::R6)
    1215           0 :               .addReg(ARM::R6, RegState::Kill)
    1216           0 :               .addImm(MaxAlign - 1)
    1217           0 :               .add(predOps(ARMCC::AL))
    1218           0 :               .add(condCodeOp());
    1219             :         }
    1220             : 
    1221             :       }
    1222          32 :       MI.eraseFromParent();
    1223          32 :       return true;
    1224             :     }
    1225             : 
    1226           2 :     case ARM::MOVsrl_flag:
    1227             :     case ARM::MOVsra_flag: {
    1228             :       // These are just fancy MOVs instructions.
    1229           6 :       BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(ARM::MOVsi),
    1230           4 :               MI.getOperand(0).getReg())
    1231           2 :           .add(MI.getOperand(1))
    1232           2 :           .addImm(ARM_AM::getSORegOpc(
    1233           2 :               (Opcode == ARM::MOVsrl_flag ? ARM_AM::lsr : ARM_AM::asr), 1))
    1234           2 :           .add(predOps(ARMCC::AL))
    1235           2 :           .addReg(ARM::CPSR, RegState::Define);
    1236           2 :       MI.eraseFromParent();
    1237           2 :       return true;
    1238             :     }
    1239           2 :     case ARM::RRX: {
    1240             :       // This encodes as "MOVs Rd, Rm, rrx
    1241             :       MachineInstrBuilder MIB =
    1242           4 :           BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(ARM::MOVsi),
    1243           4 :                   MI.getOperand(0).getReg())
    1244           2 :               .add(MI.getOperand(1))
    1245             :               .addImm(ARM_AM::getSORegOpc(ARM_AM::rrx, 0))
    1246           2 :               .add(predOps(ARMCC::AL))
    1247           2 :               .add(condCodeOp());
    1248           2 :       TransferImpOps(MI, MIB, MIB);
    1249           2 :       MI.eraseFromParent();
    1250             :       return true;
    1251             :     }
    1252          39 :     case ARM::tTPsoft:
    1253             :     case ARM::TPsoft: {
    1254             :       const bool Thumb = Opcode == ARM::tTPsoft;
    1255             : 
    1256          39 :       MachineInstrBuilder MIB;
    1257          39 :       if (STI->genLongCalls()) {
    1258           2 :         MachineFunction *MF = MBB.getParent();
    1259           2 :         MachineConstantPool *MCP = MF->getConstantPool();
    1260           2 :         unsigned PCLabelID = AFI->createPICLabelUId();
    1261             :         MachineConstantPoolValue *CPV =
    1262           4 :             ARMConstantPoolSymbol::Create(MF->getFunction().getContext(),
    1263           2 :                                           "__aeabi_read_tp", PCLabelID, 0);
    1264           2 :         unsigned Reg = MI.getOperand(0).getReg();
    1265           4 :         MIB = BuildMI(MBB, MBBI, MI.getDebugLoc(),
    1266           2 :                       TII->get(Thumb ? ARM::tLDRpci : ARM::LDRi12), Reg)
    1267           2 :                   .addConstantPoolIndex(MCP->getConstantPoolIndex(CPV, 4));
    1268           2 :         if (!Thumb)
    1269             :           MIB.addImm(0);
    1270           2 :         MIB.add(predOps(ARMCC::AL));
    1271             : 
    1272           2 :         MIB = BuildMI(MBB, MBBI, MI.getDebugLoc(),
    1273           2 :                       TII->get(Thumb ? ARM::tBLXr : ARM::BLX));
    1274           2 :         if (Thumb)
    1275           1 :           MIB.add(predOps(ARMCC::AL));
    1276           2 :         MIB.addReg(Reg, RegState::Kill);
    1277             :       } else {
    1278          37 :         MIB = BuildMI(MBB, MBBI, MI.getDebugLoc(),
    1279          37 :                       TII->get(Thumb ? ARM::tBL : ARM::BL));
    1280          37 :         if (Thumb)
    1281          11 :           MIB.add(predOps(ARMCC::AL));
    1282             :         MIB.addExternalSymbol("__aeabi_read_tp", 0);
    1283             :       }
    1284             : 
    1285          39 :       MIB->setMemRefs(MI.memoperands_begin(), MI.memoperands_end());
    1286          39 :       TransferImpOps(MI, MIB, MIB);
    1287          39 :       MI.eraseFromParent();
    1288             :       return true;
    1289             :     }
    1290          42 :     case ARM::tLDRpci_pic:
    1291             :     case ARM::t2LDRpci_pic: {
    1292             :       unsigned NewLdOpc = (Opcode == ARM::tLDRpci_pic)
    1293          42 :         ? ARM::tLDRpci : ARM::t2LDRpci;
    1294          42 :       unsigned DstReg = MI.getOperand(0).getReg();
    1295             :       bool DstIsDead = MI.getOperand(0).isDead();
    1296             :       MachineInstrBuilder MIB1 =
    1297          84 :           BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(NewLdOpc), DstReg)
    1298          42 :               .add(MI.getOperand(1))
    1299          42 :               .add(predOps(ARMCC::AL));
    1300          42 :       MIB1->setMemRefs(MI.memoperands_begin(), MI.memoperands_end());
    1301             :       MachineInstrBuilder MIB2 =
    1302         126 :           BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(ARM::tPICADD))
    1303          42 :               .addReg(DstReg, RegState::Define | getDeadRegState(DstIsDead))
    1304          42 :               .addReg(DstReg)
    1305          84 :               .add(MI.getOperand(2));
    1306          42 :       TransferImpOps(MI, MIB1, MIB2);
    1307          42 :       MI.eraseFromParent();
    1308             :       return true;
    1309             :     }
    1310             : 
    1311         330 :     case ARM::LDRLIT_ga_abs:
    1312             :     case ARM::LDRLIT_ga_pcrel:
    1313             :     case ARM::LDRLIT_ga_pcrel_ldr:
    1314             :     case ARM::tLDRLIT_ga_abs:
    1315             :     case ARM::tLDRLIT_ga_pcrel: {
    1316         330 :       unsigned DstReg = MI.getOperand(0).getReg();
    1317             :       bool DstIsDead = MI.getOperand(0).isDead();
    1318             :       const MachineOperand &MO1 = MI.getOperand(1);
    1319             :       auto Flags = MO1.getTargetFlags();
    1320         330 :       const GlobalValue *GV = MO1.getGlobal();
    1321             :       bool IsARM =
    1322         330 :           Opcode != ARM::tLDRLIT_ga_pcrel && Opcode != ARM::tLDRLIT_ga_abs;
    1323         330 :       bool IsPIC =
    1324         330 :           Opcode != ARM::LDRLIT_ga_abs && Opcode != ARM::tLDRLIT_ga_abs;
    1325         330 :       unsigned LDRLITOpc = IsARM ? ARM::LDRi12 : ARM::tLDRpci;
    1326             :       unsigned PICAddOpc =
    1327             :           IsARM
    1328         330 :               ? (Opcode == ARM::LDRLIT_ga_pcrel_ldr ? ARM::PICLDR : ARM::PICADD)
    1329             :               : ARM::tPICADD;
    1330             : 
    1331             :       // We need a new const-pool entry to load from.
    1332         330 :       MachineConstantPool *MCP = MBB.getParent()->getConstantPool();
    1333             :       unsigned ARMPCLabelIndex = 0;
    1334             :       MachineConstantPoolValue *CPV;
    1335             : 
    1336         330 :       if (IsPIC) {
    1337         293 :         unsigned PCAdj = IsARM ? 8 : 4;
    1338         293 :         auto Modifier = (Flags & ARMII::MO_GOT)
    1339         293 :                             ? ARMCP::GOT_PREL
    1340             :                             : ARMCP::no_modifier;
    1341         293 :         ARMPCLabelIndex = AFI->createPICLabelUId();
    1342         293 :         CPV = ARMConstantPoolConstant::Create(
    1343             :             GV, ARMPCLabelIndex, ARMCP::CPValue, PCAdj, Modifier,
    1344             :             /*AddCurrentAddr*/ Modifier == ARMCP::GOT_PREL);
    1345             :       } else
    1346          37 :         CPV = ARMConstantPoolConstant::Create(GV, ARMCP::no_modifier);
    1347             : 
    1348             :       MachineInstrBuilder MIB =
    1349         660 :           BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(LDRLITOpc), DstReg)
    1350         330 :             .addConstantPoolIndex(MCP->getConstantPoolIndex(CPV, 4));
    1351         330 :       if (IsARM)
    1352             :         MIB.addImm(0);
    1353         330 :       MIB.add(predOps(ARMCC::AL));
    1354             : 
    1355         330 :       if (IsPIC) {
    1356             :         MachineInstrBuilder MIB =
    1357         879 :           BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(PICAddOpc))
    1358         293 :             .addReg(DstReg, RegState::Define | getDeadRegState(DstIsDead))
    1359         293 :             .addReg(DstReg)
    1360         586 :             .addImm(ARMPCLabelIndex);
    1361             : 
    1362         293 :         if (IsARM)
    1363         149 :           MIB.add(predOps(ARMCC::AL));
    1364             :       }
    1365             : 
    1366         330 :       MI.eraseFromParent();
    1367             :       return true;
    1368             :     }
    1369         624 :     case ARM::MOV_ga_pcrel:
    1370             :     case ARM::MOV_ga_pcrel_ldr:
    1371             :     case ARM::t2MOV_ga_pcrel: {
    1372             :       // Expand into movw + movw. Also "add pc" / ldr [pc] in PIC mode.
    1373         624 :       unsigned LabelId = AFI->createPICLabelUId();
    1374         624 :       unsigned DstReg = MI.getOperand(0).getReg();
    1375             :       bool DstIsDead = MI.getOperand(0).isDead();
    1376             :       const MachineOperand &MO1 = MI.getOperand(1);
    1377         624 :       const GlobalValue *GV = MO1.getGlobal();
    1378             :       unsigned TF = MO1.getTargetFlags();
    1379             :       bool isARM = Opcode != ARM::t2MOV_ga_pcrel;
    1380         624 :       unsigned LO16Opc = isARM ? ARM::MOVi16_ga_pcrel : ARM::t2MOVi16_ga_pcrel;
    1381         624 :       unsigned HI16Opc = isARM ? ARM::MOVTi16_ga_pcrel :ARM::t2MOVTi16_ga_pcrel;
    1382             :       unsigned LO16TF = TF | ARMII::MO_LO16;
    1383             :       unsigned HI16TF = TF | ARMII::MO_HI16;
    1384             :       unsigned PICAddOpc = isARM
    1385         624 :         ? (Opcode == ARM::MOV_ga_pcrel_ldr ? ARM::PICLDR : ARM::PICADD)
    1386             :         : ARM::tPICADD;
    1387         624 :       MachineInstrBuilder MIB1 = BuildMI(MBB, MBBI, MI.getDebugLoc(),
    1388         624 :                                          TII->get(LO16Opc), DstReg)
    1389         624 :         .addGlobalAddress(GV, MO1.getOffset(), TF | LO16TF)
    1390        1248 :         .addImm(LabelId);
    1391             : 
    1392        1872 :       BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(HI16Opc), DstReg)
    1393         624 :         .addReg(DstReg)
    1394         624 :         .addGlobalAddress(GV, MO1.getOffset(), TF | HI16TF)
    1395             :         .addImm(LabelId);
    1396             : 
    1397        1248 :       MachineInstrBuilder MIB3 = BuildMI(MBB, MBBI, MI.getDebugLoc(),
    1398         624 :                                          TII->get(PICAddOpc))
    1399         624 :         .addReg(DstReg, RegState::Define | getDeadRegState(DstIsDead))
    1400        1248 :         .addReg(DstReg).addImm(LabelId);
    1401         624 :       if (isARM) {
    1402         213 :         MIB3.add(predOps(ARMCC::AL));
    1403         213 :         if (Opcode == ARM::MOV_ga_pcrel_ldr)
    1404         128 :           MIB3->setMemRefs(MI.memoperands_begin(), MI.memoperands_end());
    1405             :       }
    1406         624 :       TransferImpOps(MI, MIB1, MIB3);
    1407         624 :       MI.eraseFromParent();
    1408             :       return true;
    1409             :     }
    1410             : 
    1411        1613 :     case ARM::MOVi32imm:
    1412             :     case ARM::MOVCCi32imm:
    1413             :     case ARM::t2MOVi32imm:
    1414             :     case ARM::t2MOVCCi32imm:
    1415        1613 :       ExpandMOV32BitImm(MBB, MBBI);
    1416        1613 :       return true;
    1417             : 
    1418           6 :     case ARM::SUBS_PC_LR: {
    1419             :       MachineInstrBuilder MIB =
    1420          18 :           BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(ARM::SUBri), ARM::PC)
    1421           6 :               .addReg(ARM::LR)
    1422           6 :               .add(MI.getOperand(0))
    1423           6 :               .add(MI.getOperand(1))
    1424           6 :               .add(MI.getOperand(2))
    1425           6 :               .addReg(ARM::CPSR, RegState::Undef);
    1426           6 :       TransferImpOps(MI, MIB, MIB);
    1427           6 :       MI.eraseFromParent();
    1428             :       return true;
    1429             :     }
    1430           3 :     case ARM::VLDMQIA: {
    1431             :       unsigned NewOpc = ARM::VLDMDIA;
    1432             :       MachineInstrBuilder MIB =
    1433           6 :         BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(NewOpc));
    1434             :       unsigned OpIdx = 0;
    1435             : 
    1436             :       // Grab the Q register destination.
    1437           3 :       bool DstIsDead = MI.getOperand(OpIdx).isDead();
    1438           3 :       unsigned DstReg = MI.getOperand(OpIdx++).getReg();
    1439             : 
    1440             :       // Copy the source register.
    1441             :       MIB.add(MI.getOperand(OpIdx++));
    1442             : 
    1443             :       // Copy the predicate operands.
    1444           3 :       MIB.add(MI.getOperand(OpIdx++));
    1445           3 :       MIB.add(MI.getOperand(OpIdx++));
    1446             : 
    1447             :       // Add the destination operands (D subregs).
    1448           3 :       unsigned D0 = TRI->getSubReg(DstReg, ARM::dsub_0);
    1449           3 :       unsigned D1 = TRI->getSubReg(DstReg, ARM::dsub_1);
    1450           3 :       MIB.addReg(D0, RegState::Define | getDeadRegState(DstIsDead))
    1451           3 :         .addReg(D1, RegState::Define | getDeadRegState(DstIsDead));
    1452             : 
    1453             :       // Add an implicit def for the super-register.
    1454           3 :       MIB.addReg(DstReg, RegState::ImplicitDefine | getDeadRegState(DstIsDead));
    1455           3 :       TransferImpOps(MI, MIB, MIB);
    1456           3 :       MIB.setMemRefs(MI.memoperands_begin(), MI.memoperands_end());
    1457           3 :       MI.eraseFromParent();
    1458             :       return true;
    1459             :     }
    1460             : 
    1461           8 :     case ARM::VSTMQIA: {
    1462             :       unsigned NewOpc = ARM::VSTMDIA;
    1463             :       MachineInstrBuilder MIB =
    1464          16 :         BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(NewOpc));
    1465             :       unsigned OpIdx = 0;
    1466             : 
    1467             :       // Grab the Q register source.
    1468           8 :       bool SrcIsKill = MI.getOperand(OpIdx).isKill();
    1469           8 :       unsigned SrcReg = MI.getOperand(OpIdx++).getReg();
    1470             : 
    1471             :       // Copy the destination register.
    1472           8 :       MachineOperand Dst(MI.getOperand(OpIdx++));
    1473           8 :       Dst.setIsRenamable(false);
    1474             :       MIB.add(Dst);
    1475             : 
    1476             :       // Copy the predicate operands.
    1477           8 :       MIB.add(MI.getOperand(OpIdx++));
    1478           8 :       MIB.add(MI.getOperand(OpIdx++));
    1479             : 
    1480             :       // Add the source operands (D subregs).
    1481           8 :       unsigned D0 = TRI->getSubReg(SrcReg, ARM::dsub_0);
    1482           8 :       unsigned D1 = TRI->getSubReg(SrcReg, ARM::dsub_1);
    1483           8 :       MIB.addReg(D0, SrcIsKill ? RegState::Kill : 0)
    1484           8 :          .addReg(D1, SrcIsKill ? RegState::Kill : 0);
    1485             : 
    1486           8 :       if (SrcIsKill)      // Add an implicit kill for the Q register.
    1487           7 :         MIB->addRegisterKilled(SrcReg, TRI, true);
    1488             : 
    1489           8 :       TransferImpOps(MI, MIB, MIB);
    1490           8 :       MIB.setMemRefs(MI.memoperands_begin(), MI.memoperands_end());
    1491           8 :       MI.eraseFromParent();
    1492             :       return true;
    1493             :     }
    1494             : 
    1495          93 :     case ARM::VLD2q8Pseudo:
    1496             :     case ARM::VLD2q16Pseudo:
    1497             :     case ARM::VLD2q32Pseudo:
    1498             :     case ARM::VLD2q8PseudoWB_fixed:
    1499             :     case ARM::VLD2q16PseudoWB_fixed:
    1500             :     case ARM::VLD2q32PseudoWB_fixed:
    1501             :     case ARM::VLD2q8PseudoWB_register:
    1502             :     case ARM::VLD2q16PseudoWB_register:
    1503             :     case ARM::VLD2q32PseudoWB_register:
    1504             :     case ARM::VLD3d8Pseudo:
    1505             :     case ARM::VLD3d16Pseudo:
    1506             :     case ARM::VLD3d32Pseudo:
    1507             :     case ARM::VLD1d64TPseudo:
    1508             :     case ARM::VLD1d64TPseudoWB_fixed:
    1509             :     case ARM::VLD3d8Pseudo_UPD:
    1510             :     case ARM::VLD3d16Pseudo_UPD:
    1511             :     case ARM::VLD3d32Pseudo_UPD:
    1512             :     case ARM::VLD3q8Pseudo_UPD:
    1513             :     case ARM::VLD3q16Pseudo_UPD:
    1514             :     case ARM::VLD3q32Pseudo_UPD:
    1515             :     case ARM::VLD3q8oddPseudo:
    1516             :     case ARM::VLD3q16oddPseudo:
    1517             :     case ARM::VLD3q32oddPseudo:
    1518             :     case ARM::VLD3q8oddPseudo_UPD:
    1519             :     case ARM::VLD3q16oddPseudo_UPD:
    1520             :     case ARM::VLD3q32oddPseudo_UPD:
    1521             :     case ARM::VLD4d8Pseudo:
    1522             :     case ARM::VLD4d16Pseudo:
    1523             :     case ARM::VLD4d32Pseudo:
    1524             :     case ARM::VLD1d64QPseudo:
    1525             :     case ARM::VLD1d64QPseudoWB_fixed:
    1526             :     case ARM::VLD4d8Pseudo_UPD:
    1527             :     case ARM::VLD4d16Pseudo_UPD:
    1528             :     case ARM::VLD4d32Pseudo_UPD:
    1529             :     case ARM::VLD4q8Pseudo_UPD:
    1530             :     case ARM::VLD4q16Pseudo_UPD:
    1531             :     case ARM::VLD4q32Pseudo_UPD:
    1532             :     case ARM::VLD4q8oddPseudo:
    1533             :     case ARM::VLD4q16oddPseudo:
    1534             :     case ARM::VLD4q32oddPseudo:
    1535             :     case ARM::VLD4q8oddPseudo_UPD:
    1536             :     case ARM::VLD4q16oddPseudo_UPD:
    1537             :     case ARM::VLD4q32oddPseudo_UPD:
    1538             :     case ARM::VLD3DUPd8Pseudo:
    1539             :     case ARM::VLD3DUPd16Pseudo:
    1540             :     case ARM::VLD3DUPd32Pseudo:
    1541             :     case ARM::VLD3DUPd8Pseudo_UPD:
    1542             :     case ARM::VLD3DUPd16Pseudo_UPD:
    1543             :     case ARM::VLD3DUPd32Pseudo_UPD:
    1544             :     case ARM::VLD4DUPd8Pseudo:
    1545             :     case ARM::VLD4DUPd16Pseudo:
    1546             :     case ARM::VLD4DUPd32Pseudo:
    1547             :     case ARM::VLD4DUPd8Pseudo_UPD:
    1548             :     case ARM::VLD4DUPd16Pseudo_UPD:
    1549             :     case ARM::VLD4DUPd32Pseudo_UPD:
    1550          93 :       ExpandVLD(MBBI);
    1551          93 :       return true;
    1552             : 
    1553          77 :     case ARM::VST2q8Pseudo:
    1554             :     case ARM::VST2q16Pseudo:
    1555             :     case ARM::VST2q32Pseudo:
    1556             :     case ARM::VST2q8PseudoWB_fixed:
    1557             :     case ARM::VST2q16PseudoWB_fixed:
    1558             :     case ARM::VST2q32PseudoWB_fixed:
    1559             :     case ARM::VST2q8PseudoWB_register:
    1560             :     case ARM::VST2q16PseudoWB_register:
    1561             :     case ARM::VST2q32PseudoWB_register:
    1562             :     case ARM::VST3d8Pseudo:
    1563             :     case ARM::VST3d16Pseudo:
    1564             :     case ARM::VST3d32Pseudo:
    1565             :     case ARM::VST1d64TPseudo:
    1566             :     case ARM::VST3d8Pseudo_UPD:
    1567             :     case ARM::VST3d16Pseudo_UPD:
    1568             :     case ARM::VST3d32Pseudo_UPD:
    1569             :     case ARM::VST1d64TPseudoWB_fixed:
    1570             :     case ARM::VST1d64TPseudoWB_register:
    1571             :     case ARM::VST3q8Pseudo_UPD:
    1572             :     case ARM::VST3q16Pseudo_UPD:
    1573             :     case ARM::VST3q32Pseudo_UPD:
    1574             :     case ARM::VST3q8oddPseudo:
    1575             :     case ARM::VST3q16oddPseudo:
    1576             :     case ARM::VST3q32oddPseudo:
    1577             :     case ARM::VST3q8oddPseudo_UPD:
    1578             :     case ARM::VST3q16oddPseudo_UPD:
    1579             :     case ARM::VST3q32oddPseudo_UPD:
    1580             :     case ARM::VST4d8Pseudo:
    1581             :     case ARM::VST4d16Pseudo:
    1582             :     case ARM::VST4d32Pseudo:
    1583             :     case ARM::VST1d64QPseudo:
    1584             :     case ARM::VST4d8Pseudo_UPD:
    1585             :     case ARM::VST4d16Pseudo_UPD:
    1586             :     case ARM::VST4d32Pseudo_UPD:
    1587             :     case ARM::VST1d64QPseudoWB_fixed:
    1588             :     case ARM::VST1d64QPseudoWB_register:
    1589             :     case ARM::VST4q8Pseudo_UPD:
    1590             :     case ARM::VST4q16Pseudo_UPD:
    1591             :     case ARM::VST4q32Pseudo_UPD:
    1592             :     case ARM::VST4q8oddPseudo:
    1593             :     case ARM::VST4q16oddPseudo:
    1594             :     case ARM::VST4q32oddPseudo:
    1595             :     case ARM::VST4q8oddPseudo_UPD:
    1596             :     case ARM::VST4q16oddPseudo_UPD:
    1597             :     case ARM::VST4q32oddPseudo_UPD:
    1598          77 :       ExpandVST(MBBI);
    1599          77 :       return true;
    1600             : 
    1601         163 :     case ARM::VLD1LNq8Pseudo:
    1602             :     case ARM::VLD1LNq16Pseudo:
    1603             :     case ARM::VLD1LNq32Pseudo:
    1604             :     case ARM::VLD1LNq8Pseudo_UPD:
    1605             :     case ARM::VLD1LNq16Pseudo_UPD:
    1606             :     case ARM::VLD1LNq32Pseudo_UPD:
    1607             :     case ARM::VLD2LNd8Pseudo:
    1608             :     case ARM::VLD2LNd16Pseudo:
    1609             :     case ARM::VLD2LNd32Pseudo:
    1610             :     case ARM::VLD2LNq16Pseudo:
    1611             :     case ARM::VLD2LNq32Pseudo:
    1612             :     case ARM::VLD2LNd8Pseudo_UPD:
    1613             :     case ARM::VLD2LNd16Pseudo_UPD:
    1614             :     case ARM::VLD2LNd32Pseudo_UPD:
    1615             :     case ARM::VLD2LNq16Pseudo_UPD:
    1616             :     case ARM::VLD2LNq32Pseudo_UPD:
    1617             :     case ARM::VLD3LNd8Pseudo:
    1618             :     case ARM::VLD3LNd16Pseudo:
    1619             :     case ARM::VLD3LNd32Pseudo:
    1620             :     case ARM::VLD3LNq16Pseudo:
    1621             :     case ARM::VLD3LNq32Pseudo:
    1622             :     case ARM::VLD3LNd8Pseudo_UPD:
    1623             :     case ARM::VLD3LNd16Pseudo_UPD:
    1624             :     case ARM::VLD3LNd32Pseudo_UPD:
    1625             :     case ARM::VLD3LNq16Pseudo_UPD:
    1626             :     case ARM::VLD3LNq32Pseudo_UPD:
    1627             :     case ARM::VLD4LNd8Pseudo:
    1628             :     case ARM::VLD4LNd16Pseudo:
    1629             :     case ARM::VLD4LNd32Pseudo:
    1630             :     case ARM::VLD4LNq16Pseudo:
    1631             :     case ARM::VLD4LNq32Pseudo:
    1632             :     case ARM::VLD4LNd8Pseudo_UPD:
    1633             :     case ARM::VLD4LNd16Pseudo_UPD:
    1634             :     case ARM::VLD4LNd32Pseudo_UPD:
    1635             :     case ARM::VLD4LNq16Pseudo_UPD:
    1636             :     case ARM::VLD4LNq32Pseudo_UPD:
    1637             :     case ARM::VST1LNq8Pseudo:
    1638             :     case ARM::VST1LNq16Pseudo:
    1639             :     case ARM::VST1LNq32Pseudo:
    1640             :     case ARM::VST1LNq8Pseudo_UPD:
    1641             :     case ARM::VST1LNq16Pseudo_UPD:
    1642             :     case ARM::VST1LNq32Pseudo_UPD:
    1643             :     case ARM::VST2LNd8Pseudo:
    1644             :     case ARM::VST2LNd16Pseudo:
    1645             :     case ARM::VST2LNd32Pseudo:
    1646             :     case ARM::VST2LNq16Pseudo:
    1647             :     case ARM::VST2LNq32Pseudo:
    1648             :     case ARM::VST2LNd8Pseudo_UPD:
    1649             :     case ARM::VST2LNd16Pseudo_UPD:
    1650             :     case ARM::VST2LNd32Pseudo_UPD:
    1651             :     case ARM::VST2LNq16Pseudo_UPD:
    1652             :     case ARM::VST2LNq32Pseudo_UPD:
    1653             :     case ARM::VST3LNd8Pseudo:
    1654             :     case ARM::VST3LNd16Pseudo:
    1655             :     case ARM::VST3LNd32Pseudo:
    1656             :     case ARM::VST3LNq16Pseudo:
    1657             :     case ARM::VST3LNq32Pseudo:
    1658             :     case ARM::VST3LNd8Pseudo_UPD:
    1659             :     case ARM::VST3LNd16Pseudo_UPD:
    1660             :     case ARM::VST3LNd32Pseudo_UPD:
    1661             :     case ARM::VST3LNq16Pseudo_UPD:
    1662             :     case ARM::VST3LNq32Pseudo_UPD:
    1663             :     case ARM::VST4LNd8Pseudo:
    1664             :     case ARM::VST4LNd16Pseudo:
    1665             :     case ARM::VST4LNd32Pseudo:
    1666             :     case ARM::VST4LNq16Pseudo:
    1667             :     case ARM::VST4LNq32Pseudo:
    1668             :     case ARM::VST4LNd8Pseudo_UPD:
    1669             :     case ARM::VST4LNd16Pseudo_UPD:
    1670             :     case ARM::VST4LNd32Pseudo_UPD:
    1671             :     case ARM::VST4LNq16Pseudo_UPD:
    1672             :     case ARM::VST4LNq32Pseudo_UPD:
    1673         163 :       ExpandLaneOp(MBBI);
    1674         163 :       return true;
    1675             : 
    1676           1 :     case ARM::VTBL3Pseudo: ExpandVTBL(MBBI, ARM::VTBL3, false); return true;
    1677           1 :     case ARM::VTBL4Pseudo: ExpandVTBL(MBBI, ARM::VTBL4, false); return true;
    1678           1 :     case ARM::VTBX3Pseudo: ExpandVTBL(MBBI, ARM::VTBX3, true); return true;
    1679           2 :     case ARM::VTBX4Pseudo: ExpandVTBL(MBBI, ARM::VTBX4, true); return true;
    1680             : 
    1681           2 :     case ARM::CMP_SWAP_8:
    1682           2 :       if (STI->isThumb())
    1683             :         return ExpandCMP_SWAP(MBB, MBBI, ARM::t2LDREXB, ARM::t2STREXB,
    1684           1 :                               ARM::tUXTB, NextMBBI);
    1685             :       else
    1686             :         return ExpandCMP_SWAP(MBB, MBBI, ARM::LDREXB, ARM::STREXB,
    1687           1 :                               ARM::UXTB, NextMBBI);
    1688           2 :     case ARM::CMP_SWAP_16:
    1689           2 :       if (STI->isThumb())
    1690             :         return ExpandCMP_SWAP(MBB, MBBI, ARM::t2LDREXH, ARM::t2STREXH,
    1691           1 :                               ARM::tUXTH, NextMBBI);
    1692             :       else
    1693             :         return ExpandCMP_SWAP(MBB, MBBI, ARM::LDREXH, ARM::STREXH,
    1694           1 :                               ARM::UXTH, NextMBBI);
    1695           2 :     case ARM::CMP_SWAP_32:
    1696           2 :       if (STI->isThumb())
    1697             :         return ExpandCMP_SWAP(MBB, MBBI, ARM::t2LDREX, ARM::t2STREX, 0,
    1698           1 :                               NextMBBI);
    1699             :       else
    1700           1 :         return ExpandCMP_SWAP(MBB, MBBI, ARM::LDREX, ARM::STREX, 0, NextMBBI);
    1701             : 
    1702           7 :     case ARM::CMP_SWAP_64:
    1703           7 :       return ExpandCMP_SWAP_64(MBB, MBBI, NextMBBI);
    1704             :   }
    1705             : }
    1706             : 
    1707       18372 : bool ARMExpandPseudo::ExpandMBB(MachineBasicBlock &MBB) {
    1708             :   bool Modified = false;
    1709             : 
    1710             :   MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();
    1711      324666 :   while (MBBI != E) {
    1712      153147 :     MachineBasicBlock::iterator NMBBI = std::next(MBBI);
    1713      153147 :     Modified |= ExpandMI(MBB, MBBI, NMBBI);
    1714      153147 :     MBBI = NMBBI;
    1715             :   }
    1716             : 
    1717       18372 :   return Modified;
    1718             : }
    1719             : 
    1720       13116 : bool ARMExpandPseudo::runOnMachineFunction(MachineFunction &MF) {
    1721       13116 :   STI = &static_cast<const ARMSubtarget &>(MF.getSubtarget());
    1722       13116 :   TII = STI->getInstrInfo();
    1723       13116 :   TRI = STI->getRegisterInfo();
    1724       13116 :   AFI = MF.getInfo<ARMFunctionInfo>();
    1725             : 
    1726             :   bool Modified = false;
    1727       31488 :   for (MachineBasicBlock &MBB : MF)
    1728       18372 :     Modified |= ExpandMBB(MBB);
    1729       13116 :   if (VerifyARMPseudo)
    1730           4 :     MF.verify(this, "After expanding ARM pseudo instructions.");
    1731       13116 :   return Modified;
    1732             : }
    1733             : 
    1734             : /// createARMExpandPseudoPass - returns an instance of the pseudo instruction
    1735             : /// expansion pass.
    1736        2675 : FunctionPass *llvm::createARMExpandPseudoPass() {
    1737        5350 :   return new ARMExpandPseudo();
    1738      244878 : }

Generated by: LCOV version 1.13