PPCInstrInfo.cpp

Go to the documentation of this file.

//===-- PPCInstrInfo.cpp - PowerPC Instruction Information ----------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This file contains the PowerPC implementation of the TargetInstrInfo class.
//
//===----------------------------------------------------------------------===//
 
#include "PPCInstrInfo.h"
#include "MCTargetDesc/PPCPredicates.h"
#include "PPC.h"
#include "PPCHazardRecognizers.h"
#include "PPCInstrBuilder.h"
#include "PPCMachineFunctionInfo.h"
#include "PPCTargetMachine.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/Analysis/AliasAnalysis.h"
#include "llvm/CodeGen/LiveIntervals.h"
#include "llvm/CodeGen/LivePhysRegs.h"
#include "llvm/CodeGen/MachineCombinerPattern.h"
#include "llvm/CodeGen/MachineConstantPool.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineFunctionPass.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
#include "llvm/CodeGen/MachineMemOperand.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/CodeGen/PseudoSourceValue.h"
#include "llvm/CodeGen/RegisterClassInfo.h"
#include "llvm/CodeGen/RegisterPressure.h"
#include "llvm/CodeGen/ScheduleDAG.h"
#include "llvm/CodeGen/SlotIndexes.h"
#include "llvm/CodeGen/StackMaps.h"
#include "llvm/MC/MCAsmInfo.h"
#include "llvm/MC/MCInst.h"
#include "llvm/MC/TargetRegistry.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/raw_ostream.h"
 
using namespace llvm;
 
#define DEBUG_TYPE "ppc-instr-info"
 
#define GET_INSTRMAP_INFO
#define GET_INSTRINFO_CTOR_DTOR
#include "PPCGenInstrInfo.inc"
 
STATISTIC(NumStoreSPILLVSRRCAsVec,
          "Number of spillvsrrc spilled to stack as vec");
STATISTIC(NumStoreSPILLVSRRCAsGpr,
          "Number of spillvsrrc spilled to stack as gpr");
STATISTIC(NumGPRtoVSRSpill, "Number of gpr spills to spillvsrrc");
STATISTIC(CmpIselsConverted,
          "Number of ISELs that depend on comparison of constants converted");
STATISTIC(MissedConvertibleImmediateInstrs,
          "Number of compare-immediate instructions fed by constants");
STATISTIC(NumRcRotatesConvertedToRcAnd,
          "Number of record-form rotates converted to record-form andi");
 
static cl::
opt<bool> DisableCTRLoopAnal("disable-ppc-ctrloop-analysis", cl::Hidden,
            cl::desc("Disable analysis for CTR loops"));
 
static cl::opt<bool> DisableCmpOpt("disable-ppc-cmp-opt",
cl::desc("Disable compare instruction optimization"), cl::Hidden);
 
static cl::opt<bool> VSXSelfCopyCrash("crash-on-ppc-vsx-self-copy",
cl::desc("Causes the backend to crash instead of generating a nop VSX copy"),
cl::Hidden);
 
static cl::opt<bool>
UseOldLatencyCalc("ppc-old-latency-calc", cl::Hidden,
  cl::desc("Use the old (incorrect) instruction latency calculation"));
 
static cl::opt<float>
    FMARPFactor("ppc-fma-rp-factor", cl::Hidden, cl::init(1.5),
                cl::desc("register pressure factor for the transformations."));
 
static cl::opt<bool> EnableFMARegPressureReduction(
    "ppc-fma-rp-reduction", cl::Hidden, cl::init(true),
    cl::desc("enable register pressure reduce in machine combiner pass."));
 
// Pin the vtable to this file.
void PPCInstrInfo::anchor() {}
 
PPCInstrInfo::PPCInstrInfo(PPCSubtarget &STI)
    : PPCGenInstrInfo(PPC::ADJCALLSTACKDOWN, PPC::ADJCALLSTACKUP,
                      /* CatchRetOpcode */ -1,
                      STI.isPPC64() ? PPC::BLR8 : PPC::BLR),
      Subtarget(STI), RI(STI.getTargetMachine()) {}
 
/// CreateTargetHazardRecognizer - Return the hazard recognizer to use for
/// this target when scheduling the DAG.
ScheduleHazardRecognizer *
PPCInstrInfo::CreateTargetHazardRecognizer(const TargetSubtargetInfo *STI,
                                           const ScheduleDAG *DAG) const {
  unsigned Directive =
      static_cast<const PPCSubtarget *>(STI)->getCPUDirective();
  if (Directive == PPC::DIR_440 || Directive == PPC::DIR_A2 ||
      Directive == PPC::DIR_E500mc || Directive == PPC::DIR_E5500) {
    const InstrItineraryData *II =
        static_cast<const PPCSubtarget *>(STI)->getInstrItineraryData();
    return new ScoreboardHazardRecognizer(II, DAG);
  }
 
  return TargetInstrInfo::CreateTargetHazardRecognizer(STI, DAG);
}
 
/// CreateTargetPostRAHazardRecognizer - Return the postRA hazard recognizer
/// to use for this target when scheduling the DAG.
ScheduleHazardRecognizer *
PPCInstrInfo::CreateTargetPostRAHazardRecognizer(const InstrItineraryData *II,
                                                 const ScheduleDAG *DAG) const {
  unsigned Directive =
      DAG->MF.getSubtarget<PPCSubtarget>().getCPUDirective();
 
  // FIXME: Leaving this as-is until we have POWER9 scheduling info
  if (Directive == PPC::DIR_PWR7 || Directive == PPC::DIR_PWR8)
    return new PPCDispatchGroupSBHazardRecognizer(II, DAG);
 
  // Most subtargets use a PPC970 recognizer.
  if (Directive != PPC::DIR_440 && Directive != PPC::DIR_A2 &&
      Directive != PPC::DIR_E500mc && Directive != PPC::DIR_E5500) {
    assert(DAG->TII && "No InstrInfo?");
 
    return new PPCHazardRecognizer970(*DAG);
  }
 
  return new ScoreboardHazardRecognizer(II, DAG);
}
 
unsigned PPCInstrInfo::getInstrLatency(const InstrItineraryData *ItinData,
                                       const MachineInstr &MI,
                                       unsigned *PredCost) const {
  if (!ItinData || UseOldLatencyCalc)
    return PPCGenInstrInfo::getInstrLatency(ItinData, MI, PredCost);
 
  // The default implementation of getInstrLatency calls getStageLatency, but
  // getStageLatency does not do the right thing for us. While we have
  // itinerary, most cores are fully pipelined, and so the itineraries only
  // express the first part of the pipeline, not every stage. Instead, we need
  // to use the listed output operand cycle number (using operand 0 here, which
  // is an output).
 
  unsigned Latency = 1;
  unsigned DefClass = MI.getDesc().getSchedClass();
  for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
    const MachineOperand &MO = MI.getOperand(i);
    if (!MO.isReg() || !MO.isDef() || MO.isImplicit())
      continue;
 
    std::optional<unsigned> Cycle = ItinData->getOperandCycle(DefClass, i);
    if (!Cycle)
      continue;
 
    Latency = std::max(Latency, *Cycle);
  }
 
  return Latency;
}
 
std::optional<unsigned> PPCInstrInfo::getOperandLatency(
    const InstrItineraryData *ItinData, const MachineInstr &DefMI,
    unsigned DefIdx, const MachineInstr &UseMI, unsigned UseIdx) const {
  std::optional<unsigned> Latency = PPCGenInstrInfo::getOperandLatency(
      ItinData, DefMI, DefIdx, UseMI, UseIdx);
 
  if (!DefMI.getParent())
    return Latency;
 
  const MachineOperand &DefMO = DefMI.getOperand(DefIdx);
  Register Reg = DefMO.getReg();
 
  bool IsRegCR;
  if (Reg.isVirtual()) {
    const MachineRegisterInfo *MRI =
        &DefMI.getParent()->getParent()->getRegInfo();
    IsRegCR = MRI->getRegClass(Reg)->hasSuperClassEq(&PPC::CRRCRegClass) ||
              MRI->getRegClass(Reg)->hasSuperClassEq(&PPC::CRBITRCRegClass);
  } else {
    IsRegCR = PPC::CRRCRegClass.contains(Reg) ||
              PPC::CRBITRCRegClass.contains(Reg);
  }
 
  if (UseMI.isBranch() && IsRegCR) {
    if (!Latency)
      Latency = getInstrLatency(ItinData, DefMI);
 
    // On some cores, there is an additional delay between writing to a condition
    // register, and using it from a branch.
    unsigned Directive = Subtarget.getCPUDirective();
    switch (Directive) {
    default: break;
    case PPC::DIR_7400:
    case PPC::DIR_750:
    case PPC::DIR_970:
    case PPC::DIR_E5500:
    case PPC::DIR_PWR4:
    case PPC::DIR_PWR5:
    case PPC::DIR_PWR5X:
    case PPC::DIR_PWR6:
    case PPC::DIR_PWR6X:
    case PPC::DIR_PWR7:
    case PPC::DIR_PWR8:
    // FIXME: Is this needed for POWER9?
    Latency = *Latency + 2;
    break;
    }
  }
 
  return Latency;
}
 
void PPCInstrInfo::setSpecialOperandAttr(MachineInstr &MI,
                                         uint32_t Flags) const {
  MI.setFlags(Flags);
  MI.clearFlag(MachineInstr::MIFlag::NoSWrap);
  MI.clearFlag(MachineInstr::MIFlag::NoUWrap);
  MI.clearFlag(MachineInstr::MIFlag::IsExact);
}
 
// This function does not list all associative and commutative operations, but
// only those worth feeding through the machine combiner in an attempt to
// reduce the critical path. Mostly, this means floating-point operations,
// because they have high latencies(>=5) (compared to other operations, such as
// and/or, which are also associative and commutative, but have low latencies).
bool PPCInstrInfo::isAssociativeAndCommutative(const MachineInstr &Inst,
                                               bool Invert) const {
  if (Invert)
    return false;
  switch (Inst.getOpcode()) {
  // Floating point:
  // FP Add:
  case PPC::FADD:
  case PPC::FADDS:
  // FP Multiply:
  case PPC::FMUL:
  case PPC::FMULS:
  // Altivec Add:
  case PPC::VADDFP:
  // VSX Add:
  case PPC::XSADDDP:
  case PPC::XVADDDP:
  case PPC::XVADDSP:
  case PPC::XSADDSP:
  // VSX Multiply:
  case PPC::XSMULDP:
  case PPC::XVMULDP:
  case PPC::XVMULSP:
  case PPC::XSMULSP:
    return Inst.getFlag(MachineInstr::MIFlag::FmReassoc) &&
           Inst.getFlag(MachineInstr::MIFlag::FmNsz);
  // Fixed point:
  // Multiply:
  case PPC::MULHD:
  case PPC::MULLD:
  case PPC::MULHW:
  case PPC::MULLW:
    return true;
  default:
    return false;
  }
}
 
#define InfoArrayIdxFMAInst 0
#define InfoArrayIdxFAddInst 1
#define InfoArrayIdxFMULInst 2
#define InfoArrayIdxAddOpIdx 3
#define InfoArrayIdxMULOpIdx 4
#define InfoArrayIdxFSubInst 5
// Array keeps info for FMA instructions:
// Index 0(InfoArrayIdxFMAInst): FMA instruction;
// Index 1(InfoArrayIdxFAddInst): ADD instruction associated with FMA;
// Index 2(InfoArrayIdxFMULInst): MUL instruction associated with FMA;
// Index 3(InfoArrayIdxAddOpIdx): ADD operand index in FMA operands;
// Index 4(InfoArrayIdxMULOpIdx): first MUL operand index in FMA operands;
//                                second MUL operand index is plus 1;
// Index 5(InfoArrayIdxFSubInst): SUB instruction associated with FMA.
static const uint16_t FMAOpIdxInfo[][6] = {
    // FIXME: Add more FMA instructions like XSNMADDADP and so on.
    {PPC::XSMADDADP, PPC::XSADDDP, PPC::XSMULDP, 1, 2, PPC::XSSUBDP},
    {PPC::XSMADDASP, PPC::XSADDSP, PPC::XSMULSP, 1, 2, PPC::XSSUBSP},
    {PPC::XVMADDADP, PPC::XVADDDP, PPC::XVMULDP, 1, 2, PPC::XVSUBDP},
    {PPC::XVMADDASP, PPC::XVADDSP, PPC::XVMULSP, 1, 2, PPC::XVSUBSP},
    {PPC::FMADD, PPC::FADD, PPC::FMUL, 3, 1, PPC::FSUB},
    {PPC::FMADDS, PPC::FADDS, PPC::FMULS, 3, 1, PPC::FSUBS}};
 
// Check if an opcode is a FMA instruction. If it is, return the index in array
// FMAOpIdxInfo. Otherwise, return -1.
int16_t PPCInstrInfo::getFMAOpIdxInfo(unsigned Opcode) const {
  for (unsigned I = 0; I < std::size(FMAOpIdxInfo); I++)
    if (FMAOpIdxInfo[I][InfoArrayIdxFMAInst] == Opcode)
      return I;
  return -1;
}
 
// On PowerPC target, we have two kinds of patterns related to FMA:
// 1: Improve ILP.
// Try to reassociate FMA chains like below:
//
// Pattern 1:
//   A =  FADD X,  Y          (Leaf)
//   B =  FMA  A,  M21,  M22  (Prev)
//   C =  FMA  B,  M31,  M32  (Root)
// -->
//   A =  FMA  X,  M21,  M22
//   B =  FMA  Y,  M31,  M32
//   C =  FADD A,  B
//
// Pattern 2:
//   A =  FMA  X,  M11,  M12  (Leaf)
//   B =  FMA  A,  M21,  M22  (Prev)
//   C =  FMA  B,  M31,  M32  (Root)
// -->
//   A =  FMUL M11,  M12
//   B =  FMA  X,  M21,  M22
//   D =  FMA  A,  M31,  M32
//   C =  FADD B,  D
//
// breaking the dependency between A and B, allowing FMA to be executed in
// parallel (or back-to-back in a pipeline) instead of depending on each other.
//
// 2: Reduce register pressure.
// Try to reassociate FMA with FSUB and a constant like below:
// C is a floating point const.
//
// Pattern 1:
//   A = FSUB  X,  Y      (Leaf)
//   D = FMA   B,  C,  A  (Root)
// -->
//   A = FMA   B,  Y,  -C
//   D = FMA   A,  X,  C
//
// Pattern 2:
//   A = FSUB  X,  Y      (Leaf)
//   D = FMA   B,  A,  C  (Root)
// -->
//   A = FMA   B,  Y,  -C
//   D = FMA   A,  X,  C
//
//  Before the transformation, A must be assigned with different hardware
//  register with D. After the transformation, A and D must be assigned with
//  same hardware register due to TIE attribute of FMA instructions.
//
bool PPCInstrInfo::getFMAPatterns(MachineInstr &Root,
                                  SmallVectorImpl<unsigned> &Patterns,
                                  bool DoRegPressureReduce) const {
  MachineBasicBlock *MBB = Root.getParent();
  const MachineRegisterInfo *MRI = &MBB->getParent()->getRegInfo();
  const TargetRegisterInfo *TRI = &getRegisterInfo();
 
  auto IsAllOpsVirtualReg = [](const MachineInstr &Instr) {
    for (const auto &MO : Instr.explicit_operands())
      if (!(MO.isReg() && MO.getReg().isVirtual()))
        return false;
    return true;
  };
 
  auto IsReassociableAddOrSub = [&](const MachineInstr &Instr,
                                    unsigned OpType) {
    if (Instr.getOpcode() !=
        FMAOpIdxInfo[getFMAOpIdxInfo(Root.getOpcode())][OpType])
      return false;
 
    // Instruction can be reassociated.
    // fast math flags may prohibit reassociation.
    if (!(Instr.getFlag(MachineInstr::MIFlag::FmReassoc) &&
          Instr.getFlag(MachineInstr::MIFlag::FmNsz)))
      return false;
 
    // Instruction operands are virtual registers for reassociation.
    if (!IsAllOpsVirtualReg(Instr))
      return false;
 
    // For register pressure reassociation, the FSub must have only one use as
    // we want to delete the sub to save its def.
    if (OpType == InfoArrayIdxFSubInst &&
        !MRI->hasOneNonDBGUse(Instr.getOperand(0).getReg()))
      return false;
 
    return true;
  };
 
  auto IsReassociableFMA = [&](const MachineInstr &Instr, int16_t &AddOpIdx,
                               int16_t &MulOpIdx, bool IsLeaf) {
    int16_t Idx = getFMAOpIdxInfo(Instr.getOpcode());
    if (Idx < 0)
      return false;
 
    // Instruction can be reassociated.
    // fast math flags may prohibit reassociation.
    if (!(Instr.getFlag(MachineInstr::MIFlag::FmReassoc) &&
          Instr.getFlag(MachineInstr::MIFlag::FmNsz)))
      return false;
 
    // Instruction operands are virtual registers for reassociation.
    if (!IsAllOpsVirtualReg(Instr))
      return false;
 
    MulOpIdx = FMAOpIdxInfo[Idx][InfoArrayIdxMULOpIdx];
    if (IsLeaf)
      return true;
 
    AddOpIdx = FMAOpIdxInfo[Idx][InfoArrayIdxAddOpIdx];
 
    const MachineOperand &OpAdd = Instr.getOperand(AddOpIdx);
    MachineInstr *MIAdd = MRI->getUniqueVRegDef(OpAdd.getReg());
    // If 'add' operand's def is not in current block, don't do ILP related opt.
    if (!MIAdd || MIAdd->getParent() != MBB)
      return false;
 
    // If this is not Leaf FMA Instr, its 'add' operand should only have one use
    // as this fma will be changed later.
    return IsLeaf ? true : MRI->hasOneNonDBGUse(OpAdd.getReg());
  };
 
  int16_t AddOpIdx = -1;
  int16_t MulOpIdx = -1;
 
  bool IsUsedOnceL = false;
  bool IsUsedOnceR = false;
  MachineInstr *MULInstrL = nullptr;
  MachineInstr *MULInstrR = nullptr;
 
  auto IsRPReductionCandidate = [&]() {
    // Currently, we only support float and double.
    // FIXME: add support for other types.
    unsigned Opcode = Root.getOpcode();
    if (Opcode != PPC::XSMADDASP && Opcode != PPC::XSMADDADP)
      return false;
 
    // Root must be a valid FMA like instruction.
    // Treat it as leaf as we don't care its add operand.
    if (IsReassociableFMA(Root, AddOpIdx, MulOpIdx, true)) {
      assert((MulOpIdx >= 0) && "mul operand index not right!");
      Register MULRegL = TRI->lookThruSingleUseCopyChain(
          Root.getOperand(MulOpIdx).getReg(), MRI);
      Register MULRegR = TRI->lookThruSingleUseCopyChain(
          Root.getOperand(MulOpIdx + 1).getReg(), MRI);
      if (!MULRegL && !MULRegR)
        return false;
 
      if (MULRegL && !MULRegR) {
        MULRegR =
            TRI->lookThruCopyLike(Root.getOperand(MulOpIdx + 1).getReg(), MRI);
        IsUsedOnceL = true;
      } else if (!MULRegL && MULRegR) {
        MULRegL =
            TRI->lookThruCopyLike(Root.getOperand(MulOpIdx).getReg(), MRI);
        IsUsedOnceR = true;
      } else {
        IsUsedOnceL = true;
        IsUsedOnceR = true;
      }
 
      if (!MULRegL.isVirtual() || !MULRegR.isVirtual())
        return false;
 
      MULInstrL = MRI->getVRegDef(MULRegL);
      MULInstrR = MRI->getVRegDef(MULRegR);
      return true;
    }
    return false;
  };
 
  // Register pressure fma reassociation patterns.
  if (DoRegPressureReduce && IsRPReductionCandidate()) {
    assert((MULInstrL && MULInstrR) && "wrong register preduction candidate!");
    // Register pressure pattern 1
    if (isLoadFromConstantPool(MULInstrL) && IsUsedOnceR &&
        IsReassociableAddOrSub(*MULInstrR, InfoArrayIdxFSubInst)) {
      LLVM_DEBUG(dbgs() << "add pattern REASSOC_XY_BCA\n");
      Patterns.push_back(PPCMachineCombinerPattern::REASSOC_XY_BCA);
      return true;
    }
 
    // Register pressure pattern 2
    if ((isLoadFromConstantPool(MULInstrR) && IsUsedOnceL &&
         IsReassociableAddOrSub(*MULInstrL, InfoArrayIdxFSubInst))) {
      LLVM_DEBUG(dbgs() << "add pattern REASSOC_XY_BAC\n");
      Patterns.push_back(PPCMachineCombinerPattern::REASSOC_XY_BAC);
      return true;
    }
  }
 
  // ILP fma reassociation patterns.
  // Root must be a valid FMA like instruction.
  AddOpIdx = -1;
  if (!IsReassociableFMA(Root, AddOpIdx, MulOpIdx, false))
    return false;
 
  assert((AddOpIdx >= 0) && "add operand index not right!");
 
  Register RegB = Root.getOperand(AddOpIdx).getReg();
  MachineInstr *Prev = MRI->getUniqueVRegDef(RegB);
 
  // Prev must be a valid FMA like instruction.
  AddOpIdx = -1;
  if (!IsReassociableFMA(*Prev, AddOpIdx, MulOpIdx, false))
    return false;
 
  assert((AddOpIdx >= 0) && "add operand index not right!");
 
  Register RegA = Prev->getOperand(AddOpIdx).getReg();
  MachineInstr *Leaf = MRI->getUniqueVRegDef(RegA);
  AddOpIdx = -1;
  if (IsReassociableFMA(*Leaf, AddOpIdx, MulOpIdx, true)) {
    Patterns.push_back(PPCMachineCombinerPattern::REASSOC_XMM_AMM_BMM);
    LLVM_DEBUG(dbgs() << "add pattern REASSOC_XMM_AMM_BMM\n");
    return true;
  }
  if (IsReassociableAddOrSub(*Leaf, InfoArrayIdxFAddInst)) {
    Patterns.push_back(PPCMachineCombinerPattern::REASSOC_XY_AMM_BMM);
    LLVM_DEBUG(dbgs() << "add pattern REASSOC_XY_AMM_BMM\n");
    return true;
  }
  return false;
}
 
void PPCInstrInfo::finalizeInsInstrs(
    MachineInstr &Root, unsigned &Pattern,
    SmallVectorImpl<MachineInstr *> &InsInstrs) const {
  assert(!InsInstrs.empty() && "Instructions set to be inserted is empty!");
 
  MachineFunction *MF = Root.getMF();
  MachineRegisterInfo *MRI = &MF->getRegInfo();
  const TargetRegisterInfo *TRI = &getRegisterInfo();
  MachineConstantPool *MCP = MF->getConstantPool();
 
  int16_t Idx = getFMAOpIdxInfo(Root.getOpcode());
  if (Idx < 0)
    return;
 
  uint16_t FirstMulOpIdx = FMAOpIdxInfo[Idx][InfoArrayIdxMULOpIdx];
 
  // For now we only need to fix up placeholder for register pressure reduce
  // patterns.
  Register ConstReg = 0;
  switch (Pattern) {
  case PPCMachineCombinerPattern::REASSOC_XY_BCA:
    ConstReg =
        TRI->lookThruCopyLike(Root.getOperand(FirstMulOpIdx).getReg(), MRI);
    break;
  case PPCMachineCombinerPattern::REASSOC_XY_BAC:
    ConstReg =
        TRI->lookThruCopyLike(Root.getOperand(FirstMulOpIdx + 1).getReg(), MRI);
    break;
  default:
    // Not register pressure reduce patterns.
    return;
  }
 
  MachineInstr *ConstDefInstr = MRI->getVRegDef(ConstReg);
  // Get const value from const pool.
  const Constant *C = getConstantFromConstantPool(ConstDefInstr);
  assert(isa<llvm::ConstantFP>(C) && "not a valid constant!");
 
  // Get negative fp const.
  APFloat F1((dyn_cast<ConstantFP>(C))->getValueAPF());
  F1.changeSign();
  Constant *NegC = ConstantFP::get(dyn_cast<ConstantFP>(C)->getContext(), F1);
  Align Alignment = MF->getDataLayout().getPrefTypeAlign(C->getType());
 
  // Put negative fp const into constant pool.
  unsigned ConstPoolIdx = MCP->getConstantPoolIndex(NegC, Alignment);
 
  MachineOperand *Placeholder = nullptr;
  // Record the placeholder PPC::ZERO8 we add in reassociateFMA.
  for (auto *Inst : InsInstrs) {
    for (MachineOperand &Operand : Inst->explicit_operands()) {
      assert(Operand.isReg() && "Invalid instruction in InsInstrs!");
      if (Operand.getReg() == PPC::ZERO8) {
        Placeholder = &Operand;
        break;
      }
    }
  }
 
  assert(Placeholder && "Placeholder does not exist!");
 
  // Generate instructions to load the const fp from constant pool.
  // We only support PPC64 and medium code model.
  Register LoadNewConst =
      generateLoadForNewConst(ConstPoolIdx, &Root, C->getType(), InsInstrs);
 
  // Fill the placeholder with the new load from constant pool.
  Placeholder->setReg(LoadNewConst);
}
 
bool PPCInstrInfo::shouldReduceRegisterPressure(
    const MachineBasicBlock *MBB, const RegisterClassInfo *RegClassInfo) const {
 
  if (!EnableFMARegPressureReduction)
    return false;
 
  // Currently, we only enable register pressure reducing in machine combiner
  // for: 1: PPC64; 2: Code Model is Medium; 3: Power9 which also has vector
  // support.
  //
  // So we need following instructions to access a TOC entry:
  //
  // %6:g8rc_and_g8rc_nox0 = ADDIStocHA8 $x2, %const.0
  // %7:vssrc = DFLOADf32 target-flags(ppc-toc-lo) %const.0,
  //   killed %6:g8rc_and_g8rc_nox0, implicit $x2 :: (load 4 from constant-pool)
  //
  // FIXME: add more supported targets, like Small and Large code model, PPC32,
  // AIX.
  if (!(Subtarget.isPPC64() && Subtarget.hasP9Vector() &&
        Subtarget.getTargetMachine().getCodeModel() == CodeModel::Medium))
    return false;
 
  const TargetRegisterInfo *TRI = &getRegisterInfo();
  const MachineFunction *MF = MBB->getParent();
  const MachineRegisterInfo *MRI = &MF->getRegInfo();
 
  auto GetMBBPressure =
      [&](const MachineBasicBlock *MBB) -> std::vector<unsigned> {
    RegionPressure Pressure;
    RegPressureTracker RPTracker(Pressure);
 
    // Initialize the register pressure tracker.
    RPTracker.init(MBB->getParent(), RegClassInfo, nullptr, MBB, MBB->end(),
                   /*TrackLaneMasks*/ false, /*TrackUntiedDefs=*/true);
 
    for (const auto &MI : reverse(*MBB)) {
      if (MI.isDebugValue() || MI.isDebugLabel())
        continue;
      RegisterOperands RegOpers;
      RegOpers.collect(MI, *TRI, *MRI, false, false);
      RPTracker.recedeSkipDebugValues();
      assert(&*RPTracker.getPos() == &MI && "RPTracker sync error!");
      RPTracker.recede(RegOpers);
    }
 
    // Close the RPTracker to finalize live ins.
    RPTracker.closeRegion();
 
    return RPTracker.getPressure().MaxSetPressure;
  };
 
  // For now we only care about float and double type fma.
  unsigned VSSRCLimit = TRI->getRegPressureSetLimit(
      *MBB->getParent(), PPC::RegisterPressureSets::VSSRC);
 
  // Only reduce register pressure when pressure is high.
  return GetMBBPressure(MBB)[PPC::RegisterPressureSets::VSSRC] >
         (float)VSSRCLimit * FMARPFactor;
}
 
bool PPCInstrInfo::isLoadFromConstantPool(MachineInstr *I) const {
  // I has only one memory operand which is load from constant pool.
  if (!I->hasOneMemOperand())
    return false;
 
  MachineMemOperand *Op = I->memoperands()[0];
  return Op->isLoad() && Op->getPseudoValue() &&
         Op->getPseudoValue()->kind() == PseudoSourceValue::ConstantPool;
}
 
Register PPCInstrInfo::generateLoadForNewConst(
    unsigned Idx, MachineInstr *MI, Type *Ty,
    SmallVectorImpl<MachineInstr *> &InsInstrs) const {
  // Now we only support PPC64, Medium code model and P9 with vector.
  // We have immutable pattern to access const pool. See function
  // shouldReduceRegisterPressure.
  assert((Subtarget.isPPC64() && Subtarget.hasP9Vector() &&
          Subtarget.getTargetMachine().getCodeModel() == CodeModel::Medium) &&
         "Target not supported!\n");
 
  MachineFunction *MF = MI->getMF();
  MachineRegisterInfo *MRI = &MF->getRegInfo();
 
  // Generate ADDIStocHA8
  Register VReg1 = MRI->createVirtualRegister(&PPC::G8RC_and_G8RC_NOX0RegClass);
  MachineInstrBuilder TOCOffset =
      BuildMI(*MF, MI->getDebugLoc(), get(PPC::ADDIStocHA8), VReg1)
          .addReg(PPC::X2)
          .addConstantPoolIndex(Idx);
 
  assert((Ty->isFloatTy() || Ty->isDoubleTy()) &&
         "Only float and double are supported!");
 
  unsigned LoadOpcode;
  // Should be float type or double type.
  if (Ty->isFloatTy())
    LoadOpcode = PPC::DFLOADf32;
  else
    LoadOpcode = PPC::DFLOADf64;
 
  const TargetRegisterClass *RC = MRI->getRegClass(MI->getOperand(0).getReg());
  Register VReg2 = MRI->createVirtualRegister(RC);
  MachineMemOperand *MMO = MF->getMachineMemOperand(
      MachinePointerInfo::getConstantPool(*MF), MachineMemOperand::MOLoad,
      Ty->getScalarSizeInBits() / 8, MF->getDataLayout().getPrefTypeAlign(Ty));
 
  // Generate Load from constant pool.
  MachineInstrBuilder Load =
      BuildMI(*MF, MI->getDebugLoc(), get(LoadOpcode), VReg2)
          .addConstantPoolIndex(Idx)
          .addReg(VReg1, getKillRegState(true))
          .addMemOperand(MMO);
 
  Load->getOperand(1).setTargetFlags(PPCII::MO_TOC_LO);
 
  // Insert the toc load instructions into InsInstrs.
  InsInstrs.insert(InsInstrs.begin(), Load);
  InsInstrs.insert(InsInstrs.begin(), TOCOffset);
  return VReg2;
}
 
// This function returns the const value in constant pool if the \p I is a load
// from constant pool.
const Constant *
PPCInstrInfo::getConstantFromConstantPool(MachineInstr *I) const {
  MachineFunction *MF = I->getMF();
  MachineRegisterInfo *MRI = &MF->getRegInfo();
  MachineConstantPool *MCP = MF->getConstantPool();
  assert(I->mayLoad() && "Should be a load instruction.\n");
  for (auto MO : I->uses()) {
    if (!MO.isReg())
      continue;
    Register Reg = MO.getReg();
    if (Reg == 0 || !Reg.isVirtual())
      continue;
    // Find the toc address.
    MachineInstr *DefMI = MRI->getVRegDef(Reg);
    for (auto MO2 : DefMI->uses())
      if (MO2.isCPI())
        return (MCP->getConstants())[MO2.getIndex()].Val.ConstVal;
  }
  return nullptr;
}
 
CombinerObjective PPCInstrInfo::getCombinerObjective(unsigned Pattern) const {
  switch (Pattern) {
  case PPCMachineCombinerPattern::REASSOC_XY_AMM_BMM:
  case PPCMachineCombinerPattern::REASSOC_XMM_AMM_BMM:
    return CombinerObjective::MustReduceDepth;
  case PPCMachineCombinerPattern::REASSOC_XY_BCA:
  case PPCMachineCombinerPattern::REASSOC_XY_BAC:
    return CombinerObjective::MustReduceRegisterPressure;
  default:
    return TargetInstrInfo::getCombinerObjective(Pattern);
  }
}
 
bool PPCInstrInfo::getMachineCombinerPatterns(
    MachineInstr &Root, SmallVectorImpl<unsigned> &Patterns,
    bool DoRegPressureReduce) const {
  // Using the machine combiner in this way is potentially expensive, so
  // restrict to when aggressive optimizations are desired.
  if (Subtarget.getTargetMachine().getOptLevel() != CodeGenOptLevel::Aggressive)
    return false;
 
  if (getFMAPatterns(Root, Patterns, DoRegPressureReduce))
    return true;
 
  return TargetInstrInfo::getMachineCombinerPatterns(Root, Patterns,
                                                     DoRegPressureReduce);
}
 
void PPCInstrInfo::genAlternativeCodeSequence(
    MachineInstr &Root, unsigned Pattern,
    SmallVectorImpl<MachineInstr *> &InsInstrs,
    SmallVectorImpl<MachineInstr *> &DelInstrs,
    DenseMap<unsigned, unsigned> &InstrIdxForVirtReg) const {
  switch (Pattern) {
  case PPCMachineCombinerPattern::REASSOC_XY_AMM_BMM:
  case PPCMachineCombinerPattern::REASSOC_XMM_AMM_BMM:
  case PPCMachineCombinerPattern::REASSOC_XY_BCA:
  case PPCMachineCombinerPattern::REASSOC_XY_BAC:
    reassociateFMA(Root, Pattern, InsInstrs, DelInstrs, InstrIdxForVirtReg);
    break;
  default:
    // Reassociate default patterns.
    TargetInstrInfo::genAlternativeCodeSequence(Root, Pattern, InsInstrs,
                                                DelInstrs, InstrIdxForVirtReg);
    break;
  }
}
 
void PPCInstrInfo::reassociateFMA(
    MachineInstr &Root, unsigned Pattern,
    SmallVectorImpl<MachineInstr *> &InsInstrs,
    SmallVectorImpl<MachineInstr *> &DelInstrs,
    DenseMap<unsigned, unsigned> &InstrIdxForVirtReg) const {
  MachineFunction *MF = Root.getMF();
  MachineRegisterInfo &MRI = MF->getRegInfo();
  const TargetRegisterInfo *TRI = &getRegisterInfo();
  MachineOperand &OpC = Root.getOperand(0);
  Register RegC = OpC.getReg();
  const TargetRegisterClass *RC = MRI.getRegClass(RegC);
  MRI.constrainRegClass(RegC, RC);
 
  unsigned FmaOp = Root.getOpcode();
  int16_t Idx = getFMAOpIdxInfo(FmaOp);
  assert(Idx >= 0 && "Root must be a FMA instruction");
 
  bool IsILPReassociate =
      (Pattern == PPCMachineCombinerPattern::REASSOC_XY_AMM_BMM) ||
      (Pattern == PPCMachineCombinerPattern::REASSOC_XMM_AMM_BMM);
 
  uint16_t AddOpIdx = FMAOpIdxInfo[Idx][InfoArrayIdxAddOpIdx];
  uint16_t FirstMulOpIdx = FMAOpIdxInfo[Idx][InfoArrayIdxMULOpIdx];
 
  MachineInstr *Prev = nullptr;
  MachineInstr *Leaf = nullptr;
  switch (Pattern) {
  default:
    llvm_unreachable("not recognized pattern!");
  case PPCMachineCombinerPattern::REASSOC_XY_AMM_BMM:
  case PPCMachineCombinerPattern::REASSOC_XMM_AMM_BMM:
    Prev = MRI.getUniqueVRegDef(Root.getOperand(AddOpIdx).getReg());
    Leaf = MRI.getUniqueVRegDef(Prev->getOperand(AddOpIdx).getReg());
    break;
  case PPCMachineCombinerPattern::REASSOC_XY_BAC: {
    Register MULReg =
        TRI->lookThruCopyLike(Root.getOperand(FirstMulOpIdx).getReg(), &MRI);
    Leaf = MRI.getVRegDef(MULReg);
    break;
  }
  case PPCMachineCombinerPattern::REASSOC_XY_BCA: {
    Register MULReg = TRI->lookThruCopyLike(
        Root.getOperand(FirstMulOpIdx + 1).getReg(), &MRI);
    Leaf = MRI.getVRegDef(MULReg);
    break;
  }
  }
 
  uint32_t IntersectedFlags = 0;
  if (IsILPReassociate)
    IntersectedFlags = Root.getFlags() & Prev->getFlags() & Leaf->getFlags();
  else
    IntersectedFlags = Root.getFlags() & Leaf->getFlags();
 
  auto GetOperandInfo = [&](const MachineOperand &Operand, Register &Reg,
                            bool &KillFlag) {
    Reg = Operand.getReg();
    MRI.constrainRegClass(Reg, RC);
    KillFlag = Operand.isKill();
  };
 
  auto GetFMAInstrInfo = [&](const MachineInstr &Instr, Register &MulOp1,
                             Register &MulOp2, Register &AddOp,
                             bool &MulOp1KillFlag, bool &MulOp2KillFlag,
                             bool &AddOpKillFlag) {
    GetOperandInfo(Instr.getOperand(FirstMulOpIdx), MulOp1, MulOp1KillFlag);
    GetOperandInfo(Instr.getOperand(FirstMulOpIdx + 1), MulOp2, MulOp2KillFlag);
    GetOperandInfo(Instr.getOperand(AddOpIdx), AddOp, AddOpKillFlag);
  };
 
  Register RegM11, RegM12, RegX, RegY, RegM21, RegM22, RegM31, RegM32, RegA11,
      RegA21, RegB;
  bool KillX = false, KillY = false, KillM11 = false, KillM12 = false,
       KillM21 = false, KillM22 = false, KillM31 = false, KillM32 = false,
       KillA11 = false, KillA21 = false, KillB = false;
 
  GetFMAInstrInfo(Root, RegM31, RegM32, RegB, KillM31, KillM32, KillB);
 
  if (IsILPReassociate)
    GetFMAInstrInfo(*Prev, RegM21, RegM22, RegA21, KillM21, KillM22, KillA21);
 
  if (Pattern == PPCMachineCombinerPattern::REASSOC_XMM_AMM_BMM) {
    GetFMAInstrInfo(*Leaf, RegM11, RegM12, RegA11, KillM11, KillM12, KillA11);
    GetOperandInfo(Leaf->getOperand(AddOpIdx), RegX, KillX);
  } else if (Pattern == PPCMachineCombinerPattern::REASSOC_XY_AMM_BMM) {
    GetOperandInfo(Leaf->getOperand(1), RegX, KillX);
    GetOperandInfo(Leaf->getOperand(2), RegY, KillY);
  } else {
    // Get FSUB instruction info.
    GetOperandInfo(Leaf->getOperand(1), RegX, KillX);
    GetOperandInfo(Leaf->getOperand(2), RegY, KillY);
  }
 
  // Create new virtual registers for the new results instead of
  // recycling legacy ones because the MachineCombiner's computation of the
  // critical path requires a new register definition rather than an existing
  // one.
  // For register pressure reassociation, we only need create one virtual
  // register for the new fma.
  Register NewVRA = MRI.createVirtualRegister(RC);
  InstrIdxForVirtReg.insert(std::make_pair(NewVRA, 0));
 
  Register NewVRB = 0;
  if (IsILPReassociate) {
    NewVRB = MRI.createVirtualRegister(RC);
    InstrIdxForVirtReg.insert(std::make_pair(NewVRB, 1));
  }
 
  Register NewVRD = 0;
  if (Pattern == PPCMachineCombinerPattern::REASSOC_XMM_AMM_BMM) {
    NewVRD = MRI.createVirtualRegister(RC);
    InstrIdxForVirtReg.insert(std::make_pair(NewVRD, 2));
  }
 
  auto AdjustOperandOrder = [&](MachineInstr *MI, Register RegAdd, bool KillAdd,
                                Register RegMul1, bool KillRegMul1,
                                Register RegMul2, bool KillRegMul2) {
    MI->getOperand(AddOpIdx).setReg(RegAdd);
    MI->getOperand(AddOpIdx).setIsKill(KillAdd);
    MI->getOperand(FirstMulOpIdx).setReg(RegMul1);
    MI->getOperand(FirstMulOpIdx).setIsKill(KillRegMul1);
    MI->getOperand(FirstMulOpIdx + 1).setReg(RegMul2);
    MI->getOperand(FirstMulOpIdx + 1).setIsKill(KillRegMul2);
  };
 
  MachineInstrBuilder NewARegPressure, NewCRegPressure;
  switch (Pattern) {
  default:
    llvm_unreachable("not recognized pattern!");
  case PPCMachineCombinerPattern::REASSOC_XY_AMM_BMM: {
    // Create new instructions for insertion.
    MachineInstrBuilder MINewB =
        BuildMI(*MF, Prev->getDebugLoc(), get(FmaOp), NewVRB)
            .addReg(RegX, getKillRegState(KillX))
            .addReg(RegM21, getKillRegState(KillM21))
            .addReg(RegM22, getKillRegState(KillM22));
    MachineInstrBuilder MINewA =
        BuildMI(*MF, Root.getDebugLoc(), get(FmaOp), NewVRA)
            .addReg(RegY, getKillRegState(KillY))
            .addReg(RegM31, getKillRegState(KillM31))
            .addReg(RegM32, getKillRegState(KillM32));
    // If AddOpIdx is not 1, adjust the order.
    if (AddOpIdx != 1) {
      AdjustOperandOrder(MINewB, RegX, KillX, RegM21, KillM21, RegM22, KillM22);
      AdjustOperandOrder(MINewA, RegY, KillY, RegM31, KillM31, RegM32, KillM32);
    }
 
    MachineInstrBuilder MINewC =
        BuildMI(*MF, Root.getDebugLoc(),
                get(FMAOpIdxInfo[Idx][InfoArrayIdxFAddInst]), RegC)
            .addReg(NewVRB, getKillRegState(true))
            .addReg(NewVRA, getKillRegState(true));
 
    // Update flags for newly created instructions.
    setSpecialOperandAttr(*MINewA, IntersectedFlags);
    setSpecialOperandAttr(*MINewB, IntersectedFlags);
    setSpecialOperandAttr(*MINewC, IntersectedFlags);
 
    // Record new instructions for insertion.
    InsInstrs.push_back(MINewA);
    InsInstrs.push_back(MINewB);
    InsInstrs.push_back(MINewC);
    break;
  }
  case PPCMachineCombinerPattern::REASSOC_XMM_AMM_BMM: {
    assert(NewVRD && "new FMA register not created!");
    // Create new instructions for insertion.
    MachineInstrBuilder MINewA =
        BuildMI(*MF, Leaf->getDebugLoc(),
                get(FMAOpIdxInfo[Idx][InfoArrayIdxFMULInst]), NewVRA)
            .addReg(RegM11, getKillRegState(KillM11))
            .addReg(RegM12, getKillRegState(KillM12));
    MachineInstrBuilder MINewB =
        BuildMI(*MF, Prev->getDebugLoc(), get(FmaOp), NewVRB)
            .addReg(RegX, getKillRegState(KillX))
            .addReg(RegM21, getKillRegState(KillM21))
            .addReg(RegM22, getKillRegState(KillM22));
    MachineInstrBuilder MINewD =
        BuildMI(*MF, Root.getDebugLoc(), get(FmaOp), NewVRD)
            .addReg(NewVRA, getKillRegState(true))
            .addReg(RegM31, getKillRegState(KillM31))
            .addReg(RegM32, getKillRegState(KillM32));
    // If AddOpIdx is not 1, adjust the order.
    if (AddOpIdx != 1) {
      AdjustOperandOrder(MINewB, RegX, KillX, RegM21, KillM21, RegM22, KillM22);
      AdjustOperandOrder(MINewD, NewVRA, true, RegM31, KillM31, RegM32,
                         KillM32);
    }
 
    MachineInstrBuilder MINewC =
        BuildMI(*MF, Root.getDebugLoc(),
                get(FMAOpIdxInfo[Idx][InfoArrayIdxFAddInst]), RegC)
            .addReg(NewVRB, getKillRegState(true))
            .addReg(NewVRD, getKillRegState(true));
 
    // Update flags for newly created instructions.
    setSpecialOperandAttr(*MINewA, IntersectedFlags);
    setSpecialOperandAttr(*MINewB, IntersectedFlags);
    setSpecialOperandAttr(*MINewD, IntersectedFlags);
    setSpecialOperandAttr(*MINewC, IntersectedFlags);
 
    // Record new instructions for insertion.
    InsInstrs.push_back(MINewA);
    InsInstrs.push_back(MINewB);
    InsInstrs.push_back(MINewD);
    InsInstrs.push_back(MINewC);
    break;
  }
  case PPCMachineCombinerPattern::REASSOC_XY_BAC:
  case PPCMachineCombinerPattern::REASSOC_XY_BCA: {
    Register VarReg;
    bool KillVarReg = false;
    if (Pattern == PPCMachineCombinerPattern::REASSOC_XY_BCA) {
      VarReg = RegM31;
      KillVarReg = KillM31;
    } else {
      VarReg = RegM32;
      KillVarReg = KillM32;
    }
    // We don't want to get negative const from memory pool too early, as the
    // created entry will not be deleted even if it has no users. Since all
    // operand of Leaf and Root are virtual register, we use zero register
    // here as a placeholder. When the InsInstrs is selected in
    // MachineCombiner, we call finalizeInsInstrs to replace the zero register
    // with a virtual register which is a load from constant pool.
    NewARegPressure = BuildMI(*MF, Root.getDebugLoc(), get(FmaOp), NewVRA)
                          .addReg(RegB, getKillRegState(RegB))
                          .addReg(RegY, getKillRegState(KillY))
                          .addReg(PPC::ZERO8);
    NewCRegPressure = BuildMI(*MF, Root.getDebugLoc(), get(FmaOp), RegC)
                          .addReg(NewVRA, getKillRegState(true))
                          .addReg(RegX, getKillRegState(KillX))
                          .addReg(VarReg, getKillRegState(KillVarReg));
    // For now, we only support xsmaddadp/xsmaddasp, their add operand are
    // both at index 1, no need to adjust.
    // FIXME: when add more fma instructions support, like fma/fmas, adjust
    // the operand index here.
    break;
  }
  }
 
  if (!IsILPReassociate) {
    setSpecialOperandAttr(*NewARegPressure, IntersectedFlags);
    setSpecialOperandAttr(*NewCRegPressure, IntersectedFlags);
 
    InsInstrs.push_back(NewARegPressure);
    InsInstrs.push_back(NewCRegPressure);
  }
 
  assert(!InsInstrs.empty() &&
         "Insertion instructions set should not be empty!");
 
  // Record old instructions for deletion.
  DelInstrs.push_back(Leaf);
  if (IsILPReassociate)
    DelInstrs.push_back(Prev);
  DelInstrs.push_back(&Root);
}
 
// Detect 32 -> 64-bit extensions where we may reuse the low sub-register.
bool PPCInstrInfo::isCoalescableExtInstr(const MachineInstr &MI,
                                         Register &SrcReg, Register &DstReg,
                                         unsigned &SubIdx) const {
  switch (MI.getOpcode()) {
  default: return false;
  case PPC::EXTSW:
  case PPC::EXTSW_32:
  case PPC::EXTSW_32_64:
    SrcReg = MI.getOperand(1).getReg();
    DstReg = MI.getOperand(0).getReg();
    SubIdx = PPC::sub_32;
    return true;
  }
}
 
Register PPCInstrInfo::isLoadFromStackSlot(const MachineInstr &MI,
                                           int &FrameIndex) const {
  if (llvm::is_contained(getLoadOpcodesForSpillArray(), MI.getOpcode())) {
    // Check for the operands added by addFrameReference (the immediate is the
    // offset which defaults to 0).
    if (MI.getOperand(1).isImm() && !MI.getOperand(1).getImm() &&
        MI.getOperand(2).isFI()) {
      FrameIndex = MI.getOperand(2).getIndex();
      return MI.getOperand(0).getReg();
    }
  }
  return 0;
}
 
// For opcodes with the ReMaterializable flag set, this function is called to
// verify the instruction is really rematable.
bool PPCInstrInfo::isReallyTriviallyReMaterializable(
    const MachineInstr &MI) const {
  switch (MI.getOpcode()) {
  default:
    // Let base implementaion decide.
    break;
  case PPC::LI:
  case PPC::LI8:
  case PPC::PLI:
  case PPC::PLI8:
  case PPC::LIS:
  case PPC::LIS8:
  case PPC::ADDIStocHA:
  case PPC::ADDIStocHA8:
  case PPC::ADDItocL:
  case PPC::ADDItocL8:
  case PPC::LOAD_STACK_GUARD:
  case PPC::PPCLdFixedAddr:
  case PPC::XXLXORz:
  case PPC::XXLXORspz:
  case PPC::XXLXORdpz:
  case PPC::XXLEQVOnes:
  case PPC::XXSPLTI32DX:
  case PPC::XXSPLTIW:
  case PPC::XXSPLTIDP:
  case PPC::V_SET0B:
  case PPC::V_SET0H:
  case PPC::V_SET0:
  case PPC::V_SETALLONESB:
  case PPC::V_SETALLONESH:
  case PPC::V_SETALLONES:
  case PPC::CRSET:
  case PPC::CRUNSET:
  case PPC::XXSETACCZ:
  case PPC::XXSETACCZW:
    return true;
  }
  return TargetInstrInfo::isReallyTriviallyReMaterializable(MI);
}
 
Register PPCInstrInfo::isStoreToStackSlot(const MachineInstr &MI,
                                          int &FrameIndex) const {
  if (llvm::is_contained(getStoreOpcodesForSpillArray(), MI.getOpcode())) {
    if (MI.getOperand(1).isImm() && !MI.getOperand(1).getImm() &&
        MI.getOperand(2).isFI()) {
      FrameIndex = MI.getOperand(2).getIndex();
      return MI.getOperand(0).getReg();
    }
  }
  return 0;
}
 
MachineInstr *PPCInstrInfo::commuteInstructionImpl(MachineInstr &MI, bool NewMI,
                                                   unsigned OpIdx1,
                                                   unsigned OpIdx2) const {
  MachineFunction &MF = *MI.getParent()->getParent();
 
  // Normal instructions can be commuted the obvious way.
  if (MI.getOpcode() != PPC::RLWIMI && MI.getOpcode() != PPC::RLWIMI_rec)
    return TargetInstrInfo::commuteInstructionImpl(MI, NewMI, OpIdx1, OpIdx2);
  // Note that RLWIMI can be commuted as a 32-bit instruction, but not as a
  // 64-bit instruction (so we don't handle PPC::RLWIMI8 here), because
  // changing the relative order of the mask operands might change what happens
  // to the high-bits of the mask (and, thus, the result).
 
  // Cannot commute if it has a non-zero rotate count.
  if (MI.getOperand(3).getImm() != 0)
    return nullptr;
 
  // If we have a zero rotate count, we have:
  //   M = mask(MB,ME)
  //   Op0 = (Op1 & ~M) | (Op2 & M)
  // Change this to:
  //   M = mask((ME+1)&31, (MB-1)&31)
  //   Op0 = (Op2 & ~M) | (Op1 & M)
 
  // Swap op1/op2
  assert(((OpIdx1 == 1 && OpIdx2 == 2) || (OpIdx1 == 2 && OpIdx2 == 1)) &&
         "Only the operands 1 and 2 can be swapped in RLSIMI/RLWIMI_rec.");
  Register Reg0 = MI.getOperand(0).getReg();
  Register Reg1 = MI.getOperand(1).getReg();
  Register Reg2 = MI.getOperand(2).getReg();
  unsigned SubReg1 = MI.getOperand(1).getSubReg();
  unsigned SubReg2 = MI.getOperand(2).getSubReg();
  bool Reg1IsKill = MI.getOperand(1).isKill();
  bool Reg2IsKill = MI.getOperand(2).isKill();
  bool ChangeReg0 = false;
  // If machine instrs are no longer in two-address forms, update
  // destination register as well.
  if (Reg0 == Reg1) {
    // Must be two address instruction (i.e. op1 is tied to op0).
    assert(MI.getDesc().getOperandConstraint(1, MCOI::TIED_TO) == 0 &&
           "Expecting a two-address instruction!");
    assert(MI.getOperand(0).getSubReg() == SubReg1 && "Tied subreg mismatch");
    Reg2IsKill = false;
    ChangeReg0 = true;
  }
 
  // Masks.
  unsigned MB = MI.getOperand(4).getImm();
  unsigned ME = MI.getOperand(5).getImm();
 
  // We can't commute a trivial mask (there is no way to represent an all-zero
  // mask).
  if (MB == 0 && ME == 31)
    return nullptr;
 
  if (NewMI) {
    // Create a new instruction.
    Register Reg0 = ChangeReg0 ? Reg2 : MI.getOperand(0).getReg();
    bool Reg0IsDead = MI.getOperand(0).isDead();
    return BuildMI(MF, MI.getDebugLoc(), MI.getDesc())
        .addReg(Reg0, RegState::Define | getDeadRegState(Reg0IsDead))
        .addReg(Reg2, getKillRegState(Reg2IsKill))
        .addReg(Reg1, getKillRegState(Reg1IsKill))
        .addImm((ME + 1) & 31)
        .addImm((MB - 1) & 31);
  }
 
  if (ChangeReg0) {
    MI.getOperand(0).setReg(Reg2);
    MI.getOperand(0).setSubReg(SubReg2);
  }
  MI.getOperand(2).setReg(Reg1);
  MI.getOperand(1).setReg(Reg2);
  MI.getOperand(2).setSubReg(SubReg1);
  MI.getOperand(1).setSubReg(SubReg2);
  MI.getOperand(2).setIsKill(Reg1IsKill);
  MI.getOperand(1).setIsKill(Reg2IsKill);
 
  // Swap the mask around.
  MI.getOperand(4).setImm((ME + 1) & 31);
  MI.getOperand(5).setImm((MB - 1) & 31);
  return &MI;
}
 
bool PPCInstrInfo::findCommutedOpIndices(const MachineInstr &MI,
                                         unsigned &SrcOpIdx1,
                                         unsigned &SrcOpIdx2) const {
  // For VSX A-Type FMA instructions, it is the first two operands that can be
  // commuted, however, because the non-encoded tied input operand is listed
  // first, the operands to swap are actually the second and third.
 
  int AltOpc = PPC::getAltVSXFMAOpcode(MI.getOpcode());
  if (AltOpc == -1)
    return TargetInstrInfo::findCommutedOpIndices(MI, SrcOpIdx1, SrcOpIdx2);
 
  // The commutable operand indices are 2 and 3. Return them in SrcOpIdx1
  // and SrcOpIdx2.
  return fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, 2, 3);
}
 
void PPCInstrInfo::insertNoop(MachineBasicBlock &MBB,
                              MachineBasicBlock::iterator MI) const {
  // This function is used for scheduling, and the nop wanted here is the type
  // that terminates dispatch groups on the POWER cores.
  unsigned Directive = Subtarget.getCPUDirective();
  unsigned Opcode;
  switch (Directive) {
  default:            Opcode = PPC::NOP; break;
  case PPC::DIR_PWR6: Opcode = PPC::NOP_GT_PWR6; break;
  case PPC::DIR_PWR7: Opcode = PPC::NOP_GT_PWR7; break;
  case PPC::DIR_PWR8: Opcode = PPC::NOP_GT_PWR7; break; /* FIXME: Update when P8 InstrScheduling model is ready */
  // FIXME: Update when POWER9 scheduling model is ready.
  case PPC::DIR_PWR9: Opcode = PPC::NOP_GT_PWR7; break;
  }
 
  DebugLoc DL;
  BuildMI(MBB, MI, DL, get(Opcode));
}
 
/// Return the noop instruction to use for a noop.
MCInst PPCInstrInfo::getNop() const {
  MCInst Nop;
  Nop.setOpcode(PPC::NOP);
  return Nop;
}
 
// Branch analysis.
// Note: If the condition register is set to CTR or CTR8 then this is a
// BDNZ (imm == 1) or BDZ (imm == 0) branch.
bool PPCInstrInfo::analyzeBranch(MachineBasicBlock &MBB,
                                 MachineBasicBlock *&TBB,
                                 MachineBasicBlock *&FBB,
                                 SmallVectorImpl<MachineOperand> &Cond,
                                 bool AllowModify) const {
  bool isPPC64 = Subtarget.isPPC64();
 
  // If the block has no terminators, it just falls into the block after it.
  MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr();
  if (I == MBB.end())
    return false;
 
  if (!isUnpredicatedTerminator(*I))
    return false;
 
  if (AllowModify) {
    // If the BB ends with an unconditional branch to the fallthrough BB,
    // we eliminate the branch instruction.
    if (I->getOpcode() == PPC::B &&
        MBB.isLayoutSuccessor(I->getOperand(0).getMBB())) {
      I->eraseFromParent();
 
      // We update iterator after deleting the last branch.
      I = MBB.getLastNonDebugInstr();
      if (I == MBB.end() || !isUnpredicatedTerminator(*I))
        return false;
    }
  }
 
  // Get the last instruction in the block.
  MachineInstr &LastInst = *I;
 
  // If there is only one terminator instruction, process it.
  if (I == MBB.begin() || !isUnpredicatedTerminator(*--I)) {
    if (LastInst.getOpcode() == PPC::B) {
      if (!LastInst.getOperand(0).isMBB())
        return true;
      TBB = LastInst.getOperand(0).getMBB();
      return false;
    } else if (LastInst.getOpcode() == PPC::BCC) {
      if (!LastInst.getOperand(2).isMBB())
        return true;
      // Block ends with fall-through condbranch.
      TBB = LastInst.getOperand(2).getMBB();
      Cond.push_back(LastInst.getOperand(0));
      Cond.push_back(LastInst.getOperand(1));
      return false;
    } else if (LastInst.getOpcode() == PPC::BC) {
      if (!LastInst.getOperand(1).isMBB())
        return true;
      // Block ends with fall-through condbranch.
      TBB = LastInst.getOperand(1).getMBB();
      Cond.push_back(MachineOperand::CreateImm(PPC::PRED_BIT_SET));
      Cond.push_back(LastInst.getOperand(0));
      return false;
    } else if (LastInst.getOpcode() == PPC::BCn) {
      if (!LastInst.getOperand(1).isMBB())
        return true;
      // Block ends with fall-through condbranch.
      TBB = LastInst.getOperand(1).getMBB();
      Cond.push_back(MachineOperand::CreateImm(PPC::PRED_BIT_UNSET));
      Cond.push_back(LastInst.getOperand(0));
      return false;
    } else if (LastInst.getOpcode() == PPC::BDNZ8 ||
               LastInst.getOpcode() == PPC::BDNZ) {
      if (!LastInst.getOperand(0).isMBB())
        return true;
      if (DisableCTRLoopAnal)
        return true;
      TBB = LastInst.getOperand(0).getMBB();
      Cond.push_back(MachineOperand::CreateImm(1));
      Cond.push_back(MachineOperand::CreateReg(isPPC64 ? PPC::CTR8 : PPC::CTR,
                                               true));
      return false;
    } else if (LastInst.getOpcode() == PPC::BDZ8 ||
               LastInst.getOpcode() == PPC::BDZ) {
      if (!LastInst.getOperand(0).isMBB())
        return true;
      if (DisableCTRLoopAnal)
        return true;
      TBB = LastInst.getOperand(0).getMBB();
      Cond.push_back(MachineOperand::CreateImm(0));
      Cond.push_back(MachineOperand::CreateReg(isPPC64 ? PPC::CTR8 : PPC::CTR,
                                               true));
      return false;
    }
 
    // Otherwise, don't know what this is.
    return true;
  }
 
  // Get the instruction before it if it's a terminator.
  MachineInstr &SecondLastInst = *I;
 
  // If there are three terminators, we don't know what sort of block this is.
  if (I != MBB.begin() && isUnpredicatedTerminator(*--I))
    return true;
 
  // If the block ends with PPC::B and PPC:BCC, handle it.
  if (SecondLastInst.getOpcode() == PPC::BCC &&
      LastInst.getOpcode() == PPC::B) {
    if (!SecondLastInst.getOperand(2).isMBB() ||
        !LastInst.getOperand(0).isMBB())
      return true;
    TBB = SecondLastInst.getOperand(2).getMBB();
    Cond.push_back(SecondLastInst.getOperand(0));
    Cond.push_back(SecondLastInst.getOperand(1));
    FBB = LastInst.getOperand(0).getMBB();
    return false;
  } else if (SecondLastInst.getOpcode() == PPC::BC &&
             LastInst.getOpcode() == PPC::B) {
    if (!SecondLastInst.getOperand(1).isMBB() ||
        !LastInst.getOperand(0).isMBB())
      return true;
    TBB = SecondLastInst.getOperand(1).getMBB();
    Cond.push_back(MachineOperand::CreateImm(PPC::PRED_BIT_SET));
    Cond.push_back(SecondLastInst.getOperand(0));
    FBB = LastInst.getOperand(0).getMBB();
    return false;
  } else if (SecondLastInst.getOpcode() == PPC::BCn &&
             LastInst.getOpcode() == PPC::B) {
    if (!SecondLastInst.getOperand(1).isMBB() ||
        !LastInst.getOperand(0).isMBB())
      return true;
    TBB = SecondLastInst.getOperand(1).getMBB();
    Cond.push_back(MachineOperand::CreateImm(PPC::PRED_BIT_UNSET));
    Cond.push_back(SecondLastInst.getOperand(0));
    FBB = LastInst.getOperand(0).getMBB();
    return false;
  } else if ((SecondLastInst.getOpcode() == PPC::BDNZ8 ||
              SecondLastInst.getOpcode() == PPC::BDNZ) &&
             LastInst.getOpcode() == PPC::B) {
    if (!SecondLastInst.getOperand(0).isMBB() ||
        !LastInst.getOperand(0).isMBB())
      return true;
    if (DisableCTRLoopAnal)
      return true;
    TBB = SecondLastInst.getOperand(0).getMBB();
    Cond.push_back(MachineOperand::CreateImm(1));
    Cond.push_back(MachineOperand::CreateReg(isPPC64 ? PPC::CTR8 : PPC::CTR,
                                             true));
    FBB = LastInst.getOperand(0).getMBB();
    return false;
  } else if ((SecondLastInst.getOpcode() == PPC::BDZ8 ||
              SecondLastInst.getOpcode() == PPC::BDZ) &&
             LastInst.getOpcode() == PPC::B) {
    if (!SecondLastInst.getOperand(0).isMBB() ||
        !LastInst.getOperand(0).isMBB())
      return true;
    if (DisableCTRLoopAnal)
      return true;
    TBB = SecondLastInst.getOperand(0).getMBB();
    Cond.push_back(MachineOperand::CreateImm(0));
    Cond.push_back(MachineOperand::CreateReg(isPPC64 ? PPC::CTR8 : PPC::CTR,
                                             true));
    FBB = LastInst.getOperand(0).getMBB();
    return false;
  }
 
  // If the block ends with two PPC:Bs, handle it.  The second one is not
  // executed, so remove it.
  if (SecondLastInst.getOpcode() == PPC::B && LastInst.getOpcode() == PPC::B) {
    if (!SecondLastInst.getOperand(0).isMBB())
      return true;
    TBB = SecondLastInst.getOperand(0).getMBB();
    I = LastInst;
    if (AllowModify)
      I->eraseFromParent();
    return false;
  }
 
  // Otherwise, can't handle this.
  return true;
}
 
unsigned PPCInstrInfo::removeBranch(MachineBasicBlock &MBB,
                                    int *BytesRemoved) const {
  assert(!BytesRemoved && "code size not handled");
 
  MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr();
  if (I == MBB.end())
    return 0;
 
  if (I->getOpcode() != PPC::B && I->getOpcode() != PPC::BCC &&
      I->getOpcode() != PPC::BC && I->getOpcode() != PPC::BCn &&
      I->getOpcode() != PPC::BDNZ8 && I->getOpcode() != PPC::BDNZ &&
      I->getOpcode() != PPC::BDZ8  && I->getOpcode() != PPC::BDZ)
    return 0;
 
  // Remove the branch.
  I->eraseFromParent();
 
  I = MBB.end();
 
  if (I == MBB.begin()) return 1;
  --I;
  if (I->getOpcode() != PPC::BCC &&
      I->getOpcode() != PPC::BC && I->getOpcode() != PPC::BCn &&
      I->getOpcode() != PPC::BDNZ8 && I->getOpcode() != PPC::BDNZ &&
      I->getOpcode() != PPC::BDZ8  && I->getOpcode() != PPC::BDZ)
    return 1;
 
  // Remove the branch.
  I->eraseFromParent();
  return 2;
}
 
unsigned PPCInstrInfo::insertBranch(MachineBasicBlock &MBB,
                                    MachineBasicBlock *TBB,
                                    MachineBasicBlock *FBB,
                                    ArrayRef<MachineOperand> Cond,
                                    const DebugLoc &DL,
                                    int *BytesAdded) const {
  // Shouldn't be a fall through.
  assert(TBB && "insertBranch must not be told to insert a fallthrough");
  assert((Cond.size() == 2 || Cond.size() == 0) &&
         "PPC branch conditions have two components!");
  assert(!BytesAdded && "code size not handled");
 
  bool isPPC64 = Subtarget.isPPC64();
 
  // One-way branch.
  if (!FBB) {
    if (Cond.empty())   // Unconditional branch
      BuildMI(&MBB, DL, get(PPC::B)).addMBB(TBB);
    else if (Cond[1].getReg() == PPC::CTR || Cond[1].getReg() == PPC::CTR8)
      BuildMI(&MBB, DL, get(Cond[0].getImm() ?
                              (isPPC64 ? PPC::BDNZ8 : PPC::BDNZ) :
                              (isPPC64 ? PPC::BDZ8  : PPC::BDZ))).addMBB(TBB);
    else if (Cond[0].getImm() == PPC::PRED_BIT_SET)
      BuildMI(&MBB, DL, get(PPC::BC)).add(Cond[1]).addMBB(TBB);
    else if (Cond[0].getImm() == PPC::PRED_BIT_UNSET)
      BuildMI(&MBB, DL, get(PPC::BCn)).add(Cond[1]).addMBB(TBB);
    else                // Conditional branch
      BuildMI(&MBB, DL, get(PPC::BCC))
          .addImm(Cond[0].getImm())
          .add(Cond[1])
          .addMBB(TBB);
    return 1;
  }
 
  // Two-way Conditional Branch.
  if (Cond[1].getReg() == PPC::CTR || Cond[1].getReg() == PPC::CTR8)
    BuildMI(&MBB, DL, get(Cond[0].getImm() ?
                            (isPPC64 ? PPC::BDNZ8 : PPC::BDNZ) :
                            (isPPC64 ? PPC::BDZ8  : PPC::BDZ))).addMBB(TBB);
  else if (Cond[0].getImm() == PPC::PRED_BIT_SET)
    BuildMI(&MBB, DL, get(PPC::BC)).add(Cond[1]).addMBB(TBB);
  else if (Cond[0].getImm() == PPC::PRED_BIT_UNSET)
    BuildMI(&MBB, DL, get(PPC::BCn)).add(Cond[1]).addMBB(TBB);
  else
    BuildMI(&MBB, DL, get(PPC::BCC))
        .addImm(Cond[0].getImm())
        .add(Cond[1])
        .addMBB(TBB);
  BuildMI(&MBB, DL, get(PPC::B)).addMBB(FBB);
  return 2;
}
 
// Select analysis.
bool PPCInstrInfo::canInsertSelect(const MachineBasicBlock &MBB,
                                   ArrayRef<MachineOperand> Cond,
                                   Register DstReg, Register TrueReg,
                                   Register FalseReg, int &CondCycles,
                                   int &TrueCycles, int &FalseCycles) const {
  if (!Subtarget.hasISEL())
    return false;
 
  if (Cond.size() != 2)
    return false;
 
  // If this is really a bdnz-like condition, then it cannot be turned into a
  // select.
  if (Cond[1].getReg() == PPC::CTR || Cond[1].getReg() == PPC::CTR8)
    return false;
 
  // If the conditional branch uses a physical register, then it cannot be
  // turned into a select.
  if (Cond[1].getReg().isPhysical())
    return false;
 
  // Check register classes.
  const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
  const TargetRegisterClass *RC =
    RI.getCommonSubClass(MRI.getRegClass(TrueReg), MRI.getRegClass(FalseReg));
  if (!RC)
    return false;
 
  // isel is for regular integer GPRs only.
  if (!PPC::GPRCRegClass.hasSubClassEq(RC) &&
      !PPC::GPRC_NOR0RegClass.hasSubClassEq(RC) &&
      !PPC::G8RCRegClass.hasSubClassEq(RC) &&
      !PPC::G8RC_NOX0RegClass.hasSubClassEq(RC))
    return false;
 
  // FIXME: These numbers are for the A2, how well they work for other cores is
  // an open question. On the A2, the isel instruction has a 2-cycle latency
  // but single-cycle throughput. These numbers are used in combination with
  // the MispredictPenalty setting from the active SchedMachineModel.
  CondCycles = 1;
  TrueCycles = 1;
  FalseCycles = 1;
 
  return true;
}
 
void PPCInstrInfo::insertSelect(MachineBasicBlock &MBB,
                                MachineBasicBlock::iterator MI,
                                const DebugLoc &dl, Register DestReg,
                                ArrayRef<MachineOperand> Cond, Register TrueReg,
                                Register FalseReg) const {
  assert(Cond.size() == 2 &&
         "PPC branch conditions have two components!");
 
  // Get the register classes.
  MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
  const TargetRegisterClass *RC =
    RI.getCommonSubClass(MRI.getRegClass(TrueReg), MRI.getRegClass(FalseReg));
  assert(RC && "TrueReg and FalseReg must have overlapping register classes");
 
  bool Is64Bit = PPC::G8RCRegClass.hasSubClassEq(RC) ||
                 PPC::G8RC_NOX0RegClass.hasSubClassEq(RC);
  assert((Is64Bit ||
          PPC::GPRCRegClass.hasSubClassEq(RC) ||
          PPC::GPRC_NOR0RegClass.hasSubClassEq(RC)) &&
         "isel is for regular integer GPRs only");
 
  unsigned OpCode = Is64Bit ? PPC::ISEL8 : PPC::ISEL;
  auto SelectPred = static_cast<PPC::Predicate>(Cond[0].getImm());
 
  unsigned SubIdx = 0;
  bool SwapOps = false;
  switch (SelectPred) {
  case PPC::PRED_EQ:
  case PPC::PRED_EQ_MINUS:
  case PPC::PRED_EQ_PLUS:
      SubIdx = PPC::sub_eq; SwapOps = false; break;
  case PPC::PRED_NE:
  case PPC::PRED_NE_MINUS:
  case PPC::PRED_NE_PLUS:
      SubIdx = PPC::sub_eq; SwapOps = true; break;
  case PPC::PRED_LT:
  case PPC::PRED_LT_MINUS:
  case PPC::PRED_LT_PLUS:
      SubIdx = PPC::sub_lt; SwapOps = false; break;
  case PPC::PRED_GE:
  case PPC::PRED_GE_MINUS:
  case PPC::PRED_GE_PLUS:
      SubIdx = PPC::sub_lt; SwapOps = true; break;
  case PPC::PRED_GT:
  case PPC::PRED_GT_MINUS:
  case PPC::PRED_GT_PLUS:
      SubIdx = PPC::sub_gt; SwapOps = false; break;
  case PPC::PRED_LE:
  case PPC::PRED_LE_MINUS:
  case PPC::PRED_LE_PLUS:
      SubIdx = PPC::sub_gt; SwapOps = true; break;
  case PPC::PRED_UN:
  case PPC::PRED_UN_MINUS:
  case PPC::PRED_UN_PLUS:
      SubIdx = PPC::sub_un; SwapOps = false; break;
  case PPC::PRED_NU:
  case PPC::PRED_NU_MINUS:
  case PPC::PRED_NU_PLUS:
      SubIdx = PPC::sub_un; SwapOps = true; break;
  case PPC::PRED_BIT_SET:   SubIdx = 0; SwapOps = false; break;
  case PPC::PRED_BIT_UNSET: SubIdx = 0; SwapOps = true; break;
  }
 
  Register FirstReg =  SwapOps ? FalseReg : TrueReg,
           SecondReg = SwapOps ? TrueReg  : FalseReg;
 
  // The first input register of isel cannot be r0. If it is a member
  // of a register class that can be r0, then copy it first (the
  // register allocator should eliminate the copy).
  if (MRI.getRegClass(FirstReg)->contains(PPC::R0) ||
      MRI.getRegClass(FirstReg)->contains(PPC::X0)) {
    const TargetRegisterClass *FirstRC =
      MRI.getRegClass(FirstReg)->contains(PPC::X0) ?
        &PPC::G8RC_NOX0RegClass : &PPC::GPRC_NOR0RegClass;
    Register OldFirstReg = FirstReg;
    FirstReg = MRI.createVirtualRegister(FirstRC);
    BuildMI(MBB, MI, dl, get(TargetOpcode::COPY), FirstReg)
      .addReg(OldFirstReg);
  }
 
  BuildMI(MBB, MI, dl, get(OpCode), DestReg)
    .addReg(FirstReg).addReg(SecondReg)
    .addReg(Cond[1].getReg(), 0, SubIdx);
}
 
static unsigned getCRBitValue(unsigned CRBit) {
  unsigned Ret = 4;
  if (CRBit == PPC::CR0LT || CRBit == PPC::CR1LT ||
      CRBit == PPC::CR2LT || CRBit == PPC::CR3LT ||
      CRBit == PPC::CR4LT || CRBit == PPC::CR5LT ||
      CRBit == PPC::CR6LT || CRBit == PPC::CR7LT)
    Ret = 3;
  if (CRBit == PPC::CR0GT || CRBit == PPC::CR1GT ||
      CRBit == PPC::CR2GT || CRBit == PPC::CR3GT ||
      CRBit == PPC::CR4GT || CRBit == PPC::CR5GT ||
      CRBit == PPC::CR6GT || CRBit == PPC::CR7GT)
    Ret = 2;
  if (CRBit == PPC::CR0EQ || CRBit == PPC::CR1EQ ||
      CRBit == PPC::CR2EQ || CRBit == PPC::CR3EQ ||
      CRBit == PPC::CR4EQ || CRBit == PPC::CR5EQ ||
      CRBit == PPC::CR6EQ || CRBit == PPC::CR7EQ)
    Ret = 1;
  if (CRBit == PPC::CR0UN || CRBit == PPC::CR1UN ||
      CRBit == PPC::CR2UN || CRBit == PPC::CR3UN ||
      CRBit == PPC::CR4UN || CRBit == PPC::CR5UN ||
      CRBit == PPC::CR6UN || CRBit == PPC::CR7UN)
    Ret = 0;
 
  assert(Ret != 4 && "Invalid CR bit register");
  return Ret;
}
 
void PPCInstrInfo::copyPhysReg(MachineBasicBlock &MBB,
                               MachineBasicBlock::iterator I,
                               const DebugLoc &DL, MCRegister DestReg,
                               MCRegister SrcReg, bool KillSrc) const {
  // We can end up with self copies and similar things as a result of VSX copy
  // legalization. Promote them here.
  const TargetRegisterInfo *TRI = &getRegisterInfo();
  if (PPC::F8RCRegClass.contains(DestReg) &&
      PPC::VSRCRegClass.contains(SrcReg)) {
    MCRegister SuperReg =
        TRI->getMatchingSuperReg(DestReg, PPC::sub_64, &PPC::VSRCRegClass);
 
    if (VSXSelfCopyCrash && SrcReg == SuperReg)
      llvm_unreachable("nop VSX copy");
 
    DestReg = SuperReg;
  } else if (PPC::F8RCRegClass.contains(SrcReg) &&
             PPC::VSRCRegClass.contains(DestReg)) {
    MCRegister SuperReg =
        TRI->getMatchingSuperReg(SrcReg, PPC::sub_64, &PPC::VSRCRegClass);
 
    if (VSXSelfCopyCrash && DestReg == SuperReg)
      llvm_unreachable("nop VSX copy");
 
    SrcReg = SuperReg;
  }
 
  // Different class register copy
  if (PPC::CRBITRCRegClass.contains(SrcReg) &&
      PPC::GPRCRegClass.contains(DestReg)) {
    MCRegister CRReg = getCRFromCRBit(SrcReg);
    BuildMI(MBB, I, DL, get(PPC::MFOCRF), DestReg).addReg(CRReg);
    getKillRegState(KillSrc);
    // Rotate the CR bit in the CR fields to be the least significant bit and
    // then mask with 0x1 (MB = ME = 31).
    BuildMI(MBB, I, DL, get(PPC::RLWINM), DestReg)
       .addReg(DestReg, RegState::Kill)
       .addImm(TRI->getEncodingValue(CRReg) * 4 + (4 - getCRBitValue(SrcReg)))
       .addImm(31)
       .addImm(31);
    return;
  } else if (PPC::CRRCRegClass.contains(SrcReg) &&
             (PPC::G8RCRegClass.contains(DestReg) ||
              PPC::GPRCRegClass.contains(DestReg))) {
    bool Is64Bit = PPC::G8RCRegClass.contains(DestReg);
    unsigned MvCode = Is64Bit ? PPC::MFOCRF8 : PPC::MFOCRF;
    unsigned ShCode = Is64Bit ? PPC::RLWINM8 : PPC::RLWINM;
    unsigned CRNum = TRI->getEncodingValue(SrcReg);
    BuildMI(MBB, I, DL, get(MvCode), DestReg).addReg(SrcReg);
    getKillRegState(KillSrc);
    if (CRNum == 7)
      return;
    // Shift the CR bits to make the CR field in the lowest 4 bits of GRC.
    BuildMI(MBB, I, DL, get(ShCode), DestReg)
        .addReg(DestReg, RegState::Kill)
        .addImm(CRNum * 4 + 4)
        .addImm(28)
        .addImm(31);
    return;
  } else if (PPC::G8RCRegClass.contains(SrcReg) &&
             PPC::VSFRCRegClass.contains(DestReg)) {
    assert(Subtarget.hasDirectMove() &&
           "Subtarget doesn't support directmove, don't know how to copy.");
    BuildMI(MBB, I, DL, get(PPC::MTVSRD), DestReg).addReg(SrcReg);
    NumGPRtoVSRSpill++;
    getKillRegState(KillSrc);
    return;
  } else if (PPC::VSFRCRegClass.contains(SrcReg) &&
             PPC::G8RCRegClass.contains(DestReg)) {
    assert(Subtarget.hasDirectMove() &&
           "Subtarget doesn't support directmove, don't know how to copy.");
    BuildMI(MBB, I, DL, get(PPC::MFVSRD), DestReg).addReg(SrcReg);
    getKillRegState(KillSrc);
    return;
  } else if (PPC::SPERCRegClass.contains(SrcReg) &&
             PPC::GPRCRegClass.contains(DestReg)) {
    BuildMI(MBB, I, DL, get(PPC::EFSCFD), DestReg).addReg(SrcReg);
    getKillRegState(KillSrc);
    return;
  } else if (PPC::GPRCRegClass.contains(SrcReg) &&
             PPC::SPERCRegClass.contains(DestReg)) {
    BuildMI(MBB, I, DL, get(PPC::EFDCFS), DestReg).addReg(SrcReg);
    getKillRegState(KillSrc);
    return;
  }
 
  unsigned Opc;
  if (PPC::GPRCRegClass.contains(DestReg, SrcReg))
    Opc = PPC::OR;
  else if (PPC::G8RCRegClass.contains(DestReg, SrcReg))
    Opc = PPC::OR8;
  else if (PPC::F4RCRegClass.contains(DestReg, SrcReg))
    Opc = PPC::FMR;
  else if (PPC::CRRCRegClass.contains(DestReg, SrcReg))
    Opc = PPC::MCRF;
  else if (PPC::VRRCRegClass.contains(DestReg, SrcReg))
    Opc = PPC::VOR;
  else if (PPC::VSRCRegClass.contains(DestReg, SrcReg))
    // There are two different ways this can be done:
    //   1. xxlor : This has lower latency (on the P7), 2 cycles, but can only
    //      issue in VSU pipeline 0.
    //   2. xmovdp/xmovsp: This has higher latency (on the P7), 6 cycles, but
    //      can go to either pipeline.
    // We'll always use xxlor here, because in practically all cases where
    // copies are generated, they are close enough to some use that the
    // lower-latency form is preferable.
    Opc = PPC::XXLOR;
  else if (PPC::VSFRCRegClass.contains(DestReg, SrcReg) ||
           PPC::VSSRCRegClass.contains(DestReg, SrcReg))
    Opc = (Subtarget.hasP9Vector()) ? PPC::XSCPSGNDP : PPC::XXLORf;
  else if (Subtarget.pairedVectorMemops() &&
           PPC::VSRpRCRegClass.contains(DestReg, SrcReg)) {
    if (SrcReg > PPC::VSRp15)
      SrcReg = PPC::V0 + (SrcReg - PPC::VSRp16) * 2;
    else
      SrcReg = PPC::VSL0 + (SrcReg - PPC::VSRp0) * 2;
    if (DestReg > PPC::VSRp15)
      DestReg = PPC::V0 + (DestReg - PPC::VSRp16) * 2;
    else
      DestReg = PPC::VSL0 + (DestReg - PPC::VSRp0) * 2;
    BuildMI(MBB, I, DL, get(PPC::XXLOR), DestReg).
      addReg(SrcReg).addReg(SrcReg, getKillRegState(KillSrc));
    BuildMI(MBB, I, DL, get(PPC::XXLOR), DestReg + 1).
      addReg(SrcReg + 1).addReg(SrcReg + 1, getKillRegState(KillSrc));
    return;
  }
  else if (PPC::CRBITRCRegClass.contains(DestReg, SrcReg))
    Opc = PPC::CROR;
  else if (PPC::SPERCRegClass.contains(DestReg, SrcReg))
    Opc = PPC::EVOR;
  else if ((PPC::ACCRCRegClass.contains(DestReg) ||
            PPC::UACCRCRegClass.contains(DestReg)) &&
           (PPC::ACCRCRegClass.contains(SrcReg) ||
            PPC::UACCRCRegClass.contains(SrcReg))) {
    // If primed, de-prime the source register, copy the individual registers
    // and prime the destination if needed. The vector subregisters are
    // vs[(u)acc * 4] - vs[(u)acc * 4 + 3]. If the copy is not a kill and the
    // source is primed, we need to re-prime it after the copy as well.
    PPCRegisterInfo::emitAccCopyInfo(MBB, DestReg, SrcReg);
    bool DestPrimed = PPC::ACCRCRegClass.contains(DestReg);
    bool SrcPrimed = PPC::ACCRCRegClass.contains(SrcReg);
    MCRegister VSLSrcReg =
        PPC::VSL0 + (SrcReg - (SrcPrimed ? PPC::ACC0 : PPC::UACC0)) * 4;
    MCRegister VSLDestReg =
        PPC::VSL0 + (DestReg - (DestPrimed ? PPC::ACC0 : PPC::UACC0)) * 4;
    if (SrcPrimed)
      BuildMI(MBB, I, DL, get(PPC::XXMFACC), SrcReg).addReg(SrcReg);
    for (unsigned Idx = 0; Idx < 4; Idx++)
      BuildMI(MBB, I, DL, get(PPC::XXLOR), VSLDestReg + Idx)
          .addReg(VSLSrcReg + Idx)
          .addReg(VSLSrcReg + Idx, getKillRegState(KillSrc));
    if (DestPrimed)
      BuildMI(MBB, I, DL, get(PPC::XXMTACC), DestReg).addReg(DestReg);
    if (SrcPrimed && !KillSrc)
      BuildMI(MBB, I, DL, get(PPC::XXMTACC), SrcReg).addReg(SrcReg);
    return;
  } else if (PPC::G8pRCRegClass.contains(DestReg) &&
             PPC::G8pRCRegClass.contains(SrcReg)) {
    // TODO: Handle G8RC to G8pRC (and vice versa) copy.
    unsigned DestRegIdx = DestReg - PPC::G8p0;
    MCRegister DestRegSub0 = PPC::X0 + 2 * DestRegIdx;
    MCRegister DestRegSub1 = PPC::X0 + 2 * DestRegIdx + 1;
    unsigned SrcRegIdx = SrcReg - PPC::G8p0;
    MCRegister SrcRegSub0 = PPC::X0 + 2 * SrcRegIdx;
    MCRegister SrcRegSub1 = PPC::X0 + 2 * SrcRegIdx + 1;
    BuildMI(MBB, I, DL, get(PPC::OR8), DestRegSub0)
        .addReg(SrcRegSub0)
        .addReg(SrcRegSub0, getKillRegState(KillSrc));
    BuildMI(MBB, I, DL, get(PPC::OR8), DestRegSub1)
        .addReg(SrcRegSub1)
        .addReg(SrcRegSub1, getKillRegState(KillSrc));
    return;
  } else
    llvm_unreachable("Impossible reg-to-reg copy");
 
  const MCInstrDesc &MCID = get(Opc);
  if (MCID.getNumOperands() == 3)
    BuildMI(MBB, I, DL, MCID, DestReg)
      .addReg(SrcReg).addReg(SrcReg, getKillRegState(KillSrc));
  else
    BuildMI(MBB, I, DL, MCID, DestReg).addReg(SrcReg, getKillRegState(KillSrc));
}
 
unsigned PPCInstrInfo::getSpillIndex(const TargetRegisterClass *RC) const {
  int OpcodeIndex = 0;
 
  if (PPC::GPRCRegClass.hasSubClassEq(RC) ||
      PPC::GPRC_NOR0RegClass.hasSubClassEq(RC)) {
    OpcodeIndex = SOK_Int4Spill;
  } else if (PPC::G8RCRegClass.hasSubClassEq(RC) ||
             PPC::G8RC_NOX0RegClass.hasSubClassEq(RC)) {
    OpcodeIndex = SOK_Int8Spill;
  } else if (PPC::F8RCRegClass.hasSubClassEq(RC)) {
    OpcodeIndex = SOK_Float8Spill;
  } else if (PPC::F4RCRegClass.hasSubClassEq(RC)) {
    OpcodeIndex = SOK_Float4Spill;
  } else if (PPC::SPERCRegClass.hasSubClassEq(RC)) {
    OpcodeIndex = SOK_SPESpill;
  } else if (PPC::CRRCRegClass.hasSubClassEq(RC)) {
    OpcodeIndex = SOK_CRSpill;
  } else if (PPC::CRBITRCRegClass.hasSubClassEq(RC)) {
    OpcodeIndex = SOK_CRBitSpill;
  } else if (PPC::VRRCRegClass.hasSubClassEq(RC)) {
    OpcodeIndex = SOK_VRVectorSpill;
  } else if (PPC::VSRCRegClass.hasSubClassEq(RC)) {
    OpcodeIndex = SOK_VSXVectorSpill;
  } else if (PPC::VSFRCRegClass.hasSubClassEq(RC)) {
    OpcodeIndex = SOK_VectorFloat8Spill;
  } else if (PPC::VSSRCRegClass.hasSubClassEq(RC)) {
    OpcodeIndex = SOK_VectorFloat4Spill;
  } else if (PPC::SPILLTOVSRRCRegClass.hasSubClassEq(RC)) {
    OpcodeIndex = SOK_SpillToVSR;
  } else if (PPC::ACCRCRegClass.hasSubClassEq(RC)) {
    assert(Subtarget.pairedVectorMemops() &&
           "Register unexpected when paired memops are disabled.");
    OpcodeIndex = SOK_AccumulatorSpill;
  } else if (PPC::UACCRCRegClass.hasSubClassEq(RC)) {
    assert(Subtarget.pairedVectorMemops() &&
           "Register unexpected when paired memops are disabled.");
    OpcodeIndex = SOK_UAccumulatorSpill;
  } else if (PPC::WACCRCRegClass.hasSubClassEq(RC)) {
    assert(Subtarget.pairedVectorMemops() &&
           "Register unexpected when paired memops are disabled.");
    OpcodeIndex = SOK_WAccumulatorSpill;
  } else if (PPC::VSRpRCRegClass.hasSubClassEq(RC)) {
    assert(Subtarget.pairedVectorMemops() &&
           "Register unexpected when paired memops are disabled.");
    OpcodeIndex = SOK_PairedVecSpill;
  } else if (PPC::G8pRCRegClass.hasSubClassEq(RC)) {
    OpcodeIndex = SOK_PairedG8Spill;
  } else {
    llvm_unreachable("Unknown regclass!");
  }
  return OpcodeIndex;
}
 
unsigned
PPCInstrInfo::getStoreOpcodeForSpill(const TargetRegisterClass *RC) const {
  ArrayRef<unsigned> OpcodesForSpill = getStoreOpcodesForSpillArray();
  return OpcodesForSpill[getSpillIndex(RC)];
}
 
unsigned
PPCInstrInfo::getLoadOpcodeForSpill(const TargetRegisterClass *RC) const {
  ArrayRef<unsigned> OpcodesForSpill = getLoadOpcodesForSpillArray();
  return OpcodesForSpill[getSpillIndex(RC)];
}
 
void PPCInstrInfo::StoreRegToStackSlot(
    MachineFunction &MF, unsigned SrcReg, bool isKill, int FrameIdx,
    const TargetRegisterClass *RC,
    SmallVectorImpl<MachineInstr *> &NewMIs) const {
  unsigned Opcode = getStoreOpcodeForSpill(RC);
  DebugLoc DL;
 
  PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>();
  FuncInfo->setHasSpills();
 
  NewMIs.push_back(addFrameReference(
      BuildMI(MF, DL, get(Opcode)).addReg(SrcReg, getKillRegState(isKill)),
      FrameIdx));
 
  if (PPC::CRRCRegClass.hasSubClassEq(RC) ||
      PPC::CRBITRCRegClass.hasSubClassEq(RC))
    FuncInfo->setSpillsCR();
 
  if (isXFormMemOp(Opcode))
    FuncInfo->setHasNonRISpills();
}
 
void PPCInstrInfo::storeRegToStackSlotNoUpd(
    MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, unsigned SrcReg,
    bool isKill, int FrameIdx, const TargetRegisterClass *RC,
    const TargetRegisterInfo *TRI) const {
  MachineFunction &MF = *MBB.getParent();
  SmallVector<MachineInstr *, 4> NewMIs;
 
  StoreRegToStackSlot(MF, SrcReg, isKill, FrameIdx, RC, NewMIs);
 
  for (MachineInstr *NewMI : NewMIs)
    MBB.insert(MI, NewMI);
 
  const MachineFrameInfo &MFI = MF.getFrameInfo();
  MachineMemOperand *MMO = MF.getMachineMemOperand(
      MachinePointerInfo::getFixedStack(MF, FrameIdx),
      MachineMemOperand::MOStore, MFI.getObjectSize(FrameIdx),
      MFI.getObjectAlign(FrameIdx));
  NewMIs.back()->addMemOperand(MF, MMO);
}
 
void PPCInstrInfo::storeRegToStackSlot(
    MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, Register SrcReg,
    bool isKill, int FrameIdx, const TargetRegisterClass *RC,
    const TargetRegisterInfo *TRI, Register VReg) const {
  // We need to avoid a situation in which the value from a VRRC register is
  // spilled using an Altivec instruction and reloaded into a VSRC register
  // using a VSX instruction. The issue with this is that the VSX
  // load/store instructions swap the doublewords in the vector and the Altivec
  // ones don't. The register classes on the spill/reload may be different if
  // the register is defined using an Altivec instruction and is then used by a
  // VSX instruction.
  RC = updatedRC(RC);
  storeRegToStackSlotNoUpd(MBB, MI, SrcReg, isKill, FrameIdx, RC, TRI);
}
 
void PPCInstrInfo::LoadRegFromStackSlot(MachineFunction &MF, const DebugLoc &DL,
                                        unsigned DestReg, int FrameIdx,
                                        const TargetRegisterClass *RC,
                                        SmallVectorImpl<MachineInstr *> &NewMIs)
                                        const {
  unsigned Opcode = getLoadOpcodeForSpill(RC);
  NewMIs.push_back(addFrameReference(BuildMI(MF, DL, get(Opcode), DestReg),
                                     FrameIdx));
}
 
void PPCInstrInfo::loadRegFromStackSlotNoUpd(
    MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, unsigned DestReg,
    int FrameIdx, const TargetRegisterClass *RC,
    const TargetRegisterInfo *TRI) const {
  MachineFunction &MF = *MBB.getParent();
  SmallVector<MachineInstr*, 4> NewMIs;
  DebugLoc DL;
  if (MI != MBB.end()) DL = MI->getDebugLoc();
 
  LoadRegFromStackSlot(MF, DL, DestReg, FrameIdx, RC, NewMIs);
 
  for (MachineInstr *NewMI : NewMIs)
    MBB.insert(MI, NewMI);
 
  const MachineFrameInfo &MFI = MF.getFrameInfo();
  MachineMemOperand *MMO = MF.getMachineMemOperand(
      MachinePointerInfo::getFixedStack(MF, FrameIdx),
      MachineMemOperand::MOLoad, MFI.getObjectSize(FrameIdx),
      MFI.getObjectAlign(FrameIdx));
  NewMIs.back()->addMemOperand(MF, MMO);
}
 
void PPCInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
                                        MachineBasicBlock::iterator MI,
                                        Register DestReg, int FrameIdx,
                                        const TargetRegisterClass *RC,
                                        const TargetRegisterInfo *TRI,
                                        Register VReg) const {
  // We need to avoid a situation in which the value from a VRRC register is
  // spilled using an Altivec instruction and reloaded into a VSRC register
  // using a VSX instruction. The issue with this is that the VSX
  // load/store instructions swap the doublewords in the vector and the Altivec
  // ones don't. The register classes on the spill/reload may be different if
  // the register is defined using an Altivec instruction and is then used by a
  // VSX instruction.
  RC = updatedRC(RC);
 
  loadRegFromStackSlotNoUpd(MBB, MI, DestReg, FrameIdx, RC, TRI);
}
 
bool PPCInstrInfo::
reverseBranchCondition(SmallVectorImpl<MachineOperand> &Cond) const {
  assert(Cond.size() == 2 && "Invalid PPC branch opcode!");
  if (Cond[1].getReg() == PPC::CTR8 || Cond[1].getReg() == PPC::CTR)
    Cond[0].setImm(Cond[0].getImm() == 0 ? 1 : 0);
  else
    // Leave the CR# the same, but invert the condition.
    Cond[0].setImm(PPC::InvertPredicate((PPC::Predicate)Cond[0].getImm()));
  return false;
}
 
// For some instructions, it is legal to fold ZERO into the RA register field.
// This function performs that fold by replacing the operand with PPC::ZERO,
// it does not consider whether the load immediate zero is no longer in use.
bool PPCInstrInfo::onlyFoldImmediate(MachineInstr &UseMI, MachineInstr &DefMI,
                                     Register Reg) const {
  // A zero immediate should always be loaded with a single li.
  unsigned DefOpc = DefMI.getOpcode();
  if (DefOpc != PPC::LI && DefOpc != PPC::LI8)
    return false;
  if (!DefMI.getOperand(1).isImm())
    return false;
  if (DefMI.getOperand(1).getImm() != 0)
    return false;
 
  // Note that we cannot here invert the arguments of an isel in order to fold
  // a ZERO into what is presented as the second argument. All we have here
  // is the condition bit, and that might come from a CR-logical bit operation.
 
  const MCInstrDesc &UseMCID = UseMI.getDesc();
 
  // Only fold into real machine instructions.
  if (UseMCID.isPseudo())
    return false;
 
  // We need to find which of the User's operands is to be folded, that will be
  // the operand that matches the given register ID.
  unsigned UseIdx;
  for (UseIdx = 0; UseIdx < UseMI.getNumOperands(); ++UseIdx)
    if (UseMI.getOperand(UseIdx).isReg() &&
        UseMI.getOperand(UseIdx).getReg() == Reg)
      break;
 
  assert(UseIdx < UseMI.getNumOperands() && "Cannot find Reg in UseMI");
  assert(UseIdx < UseMCID.getNumOperands() && "No operand description for Reg");
 
  const MCOperandInfo *UseInfo = &UseMCID.operands()[UseIdx];
 
  // We can fold the zero if this register requires a GPRC_NOR0/G8RC_NOX0
  // register (which might also be specified as a pointer class kind).
  if (UseInfo->isLookupPtrRegClass()) {
    if (UseInfo->RegClass /* Kind */ != 1)
      return false;
  } else {
    if (UseInfo->RegClass != PPC::GPRC_NOR0RegClassID &&
        UseInfo->RegClass != PPC::G8RC_NOX0RegClassID)
      return false;
  }
 
  // Make sure this is not tied to an output register (or otherwise
  // constrained). This is true for ST?UX registers, for example, which
  // are tied to their output registers.
  if (UseInfo->Constraints != 0)
    return false;
 
  MCRegister ZeroReg;
  if (UseInfo->isLookupPtrRegClass()) {
    bool isPPC64 = Subtarget.isPPC64();
    ZeroReg = isPPC64 ? PPC::ZERO8 : PPC::ZERO;
  } else {
    ZeroReg = UseInfo->RegClass == PPC::G8RC_NOX0RegClassID ?
              PPC::ZERO8 : PPC::ZERO;
  }
 
  LLVM_DEBUG(dbgs() << "Folded immediate zero for: ");
  LLVM_DEBUG(UseMI.dump());
  UseMI.getOperand(UseIdx).setReg(ZeroReg);
  LLVM_DEBUG(dbgs() << "Into: ");
  LLVM_DEBUG(UseMI.dump());
  return true;
}
 
// Folds zero into instructions which have a load immediate zero as an operand
// but also recognize zero as immediate zero. If the definition of the load
// has no more users it is deleted.
bool PPCInstrInfo::foldImmediate(MachineInstr &UseMI, MachineInstr &DefMI,
                                 Register Reg, MachineRegisterInfo *MRI) const {
  bool Changed = onlyFoldImmediate(UseMI, DefMI, Reg);
  if (MRI->use_nodbg_empty(Reg))
    DefMI.eraseFromParent();
  return Changed;
}
 
static bool MBBDefinesCTR(MachineBasicBlock &MBB) {
  for (MachineInstr &MI : MBB)
    if (MI.definesRegister(PPC::CTR, /*TRI=*/nullptr) ||
        MI.definesRegister(PPC::CTR8, /*TRI=*/nullptr))
      return true;
  return false;
}
 
// We should make sure that, if we're going to predicate both sides of a
// condition (a diamond), that both sides don't define the counter register. We
// can predicate counter-decrement-based branches, but while that predicates
// the branching, it does not predicate the counter decrement. If we tried to
// merge the triangle into one predicated block, we'd decrement the counter
// twice.
bool PPCInstrInfo::isProfitableToIfCvt(MachineBasicBlock &TMBB,
                     unsigned NumT, unsigned ExtraT,
                     MachineBasicBlock &FMBB,
                     unsigned NumF, unsigned ExtraF,
                     BranchProbability Probability) const {
  return !(MBBDefinesCTR(TMBB) && MBBDefinesCTR(FMBB));
}
 
 
bool PPCInstrInfo::isPredicated(const MachineInstr &MI) const {
  // The predicated branches are identified by their type, not really by the
  // explicit presence of a predicate. Furthermore, some of them can be
  // predicated more than once. Because if conversion won't try to predicate
  // any instruction which already claims to be predicated (by returning true
  // here), always return false. In doing so, we let isPredicable() be the
  // final word on whether not the instruction can be (further) predicated.
 
  return false;
}
 
bool PPCInstrInfo::isSchedulingBoundary(const MachineInstr &MI,
                                        const MachineBasicBlock *MBB,
                                        const MachineFunction &MF) const {
  switch (MI.getOpcode()) {
  default:
    break;
  // Set MFFS and MTFSF as scheduling boundary to avoid unexpected code motion
  // across them, since some FP operations may change content of FPSCR.
  // TODO: Model FPSCR in PPC instruction definitions and remove the workaround
  case PPC::MFFS:
  case PPC::MTFSF:
  case PPC::FENCE:
    return true;
  }
  return TargetInstrInfo::isSchedulingBoundary(MI, MBB, MF);
}
 
bool PPCInstrInfo::PredicateInstruction(MachineInstr &MI,
                                        ArrayRef<MachineOperand> Pred) const {
  unsigned OpC = MI.getOpcode();
  if (OpC == PPC::BLR || OpC == PPC::BLR8) {
    if (Pred[1].getReg() == PPC::CTR8 || Pred[1].getReg() == PPC::CTR) {
      bool isPPC64 = Subtarget.isPPC64();
      MI.setDesc(get(Pred[0].getImm() ? (isPPC64 ? PPC::BDNZLR8 : PPC::BDNZLR)
                                      : (isPPC64 ? PPC::BDZLR8 : PPC::BDZLR)));
      // Need add Def and Use for CTR implicit operand.
      MachineInstrBuilder(*MI.getParent()->getParent(), MI)
          .addReg(Pred[1].getReg(), RegState::Implicit)
          .addReg(Pred[1].getReg(), RegState::ImplicitDefine);
    } else if (Pred[0].getImm() == PPC::PRED_BIT_SET) {
      MI.setDesc(get(PPC::BCLR));
      MachineInstrBuilder(*MI.getParent()->getParent(), MI).add(Pred[1]);
    } else if (Pred[0].getImm() == PPC::PRED_BIT_UNSET) {
      MI.setDesc(get(PPC::BCLRn));
      MachineInstrBuilder(*MI.getParent()->getParent(), MI).add(Pred[1]);
    } else {
      MI.setDesc(get(PPC::BCCLR));
      MachineInstrBuilder(*MI.getParent()->getParent(), MI)
          .addImm(Pred[0].getImm())
          .add(Pred[1]);
    }
 
    return true;
  } else if (OpC == PPC::B) {
    if (Pred[1].getReg() == PPC::CTR8 || Pred[1].getReg() == PPC::CTR) {
      bool isPPC64 = Subtarget.isPPC64();
      MI.setDesc(get(Pred[0].getImm() ? (isPPC64 ? PPC::BDNZ8 : PPC::BDNZ)
                                      : (isPPC64 ? PPC::BDZ8 : PPC::BDZ)));
      // Need add Def and Use for CTR implicit operand.
      MachineInstrBuilder(*MI.getParent()->getParent(), MI)
          .addReg(Pred[1].getReg(), RegState::Implicit)
          .addReg(Pred[1].getReg(), RegState::ImplicitDefine);
    } else if (Pred[0].getImm() == PPC::PRED_BIT_SET) {
      MachineBasicBlock *MBB = MI.getOperand(0).getMBB();
      MI.removeOperand(0);
 
      MI.setDesc(get(PPC::BC));
      MachineInstrBuilder(*MI.getParent()->getParent(), MI)
          .add(Pred[1])
          .addMBB(MBB);
    } else if (Pred[0].getImm() == PPC::PRED_BIT_UNSET) {
      MachineBasicBlock *MBB = MI.getOperand(0).getMBB();
      MI.removeOperand(0);
 
      MI.setDesc(get(PPC::BCn));
      MachineInstrBuilder(*MI.getParent()->getParent(), MI)
          .add(Pred[1])
          .addMBB(MBB);
    } else {
      MachineBasicBlock *MBB = MI.getOperand(0).getMBB();
      MI.removeOperand(0);
 
      MI.setDesc(get(PPC::BCC));
      MachineInstrBuilder(*MI.getParent()->getParent(), MI)
          .addImm(Pred[0].getImm())
          .add(Pred[1])
          .addMBB(MBB);
    }
 
    return true;
  } else if (OpC == PPC::BCTR || OpC == PPC::BCTR8 || OpC == PPC::BCTRL ||
             OpC == PPC::BCTRL8 || OpC == PPC::BCTRL_RM ||
             OpC == PPC::BCTRL8_RM) {
    if (Pred[1].getReg() == PPC::CTR8 || Pred[1].getReg() == PPC::CTR)
      llvm_unreachable("Cannot predicate bctr[l] on the ctr register");
 
    bool setLR = OpC == PPC::BCTRL || OpC == PPC::BCTRL8 ||
                 OpC == PPC::BCTRL_RM || OpC == PPC::BCTRL8_RM;
    bool isPPC64 = Subtarget.isPPC64();
 
    if (Pred[0].getImm() == PPC::PRED_BIT_SET) {
      MI.setDesc(get(isPPC64 ? (setLR ? PPC::BCCTRL8 : PPC::BCCTR8)
                             : (setLR ? PPC::BCCTRL : PPC::BCCTR)));
      MachineInstrBuilder(*MI.getParent()->getParent(), MI).add(Pred[1]);
    } else if (Pred[0].getImm() == PPC::PRED_BIT_UNSET) {
      MI.setDesc(get(isPPC64 ? (setLR ? PPC::BCCTRL8n : PPC::BCCTR8n)
                             : (setLR ? PPC::BCCTRLn : PPC::BCCTRn)));
      MachineInstrBuilder(*MI.getParent()->getParent(), MI).add(Pred[1]);
    } else {
      MI.setDesc(get(isPPC64 ? (setLR ? PPC::BCCCTRL8 : PPC::BCCCTR8)
                             : (setLR ? PPC::BCCCTRL : PPC::BCCCTR)));
      MachineInstrBuilder(*MI.getParent()->getParent(), MI)
          .addImm(Pred[0].getImm())
          .add(Pred[1]);
    }
 
    // Need add Def and Use for LR implicit operand.
    if (setLR)
      MachineInstrBuilder(*MI.getParent()->getParent(), MI)
          .addReg(isPPC64 ? PPC::LR8 : PPC::LR, RegState::Implicit)
          .addReg(isPPC64 ? PPC::LR8 : PPC::LR, RegState::ImplicitDefine);
    if (OpC == PPC::BCTRL_RM || OpC == PPC::BCTRL8_RM)
      MachineInstrBuilder(*MI.getParent()->getParent(), MI)
          .addReg(PPC::RM, RegState::ImplicitDefine);
 
    return true;
  }
 
  return false;
}
 
bool PPCInstrInfo::SubsumesPredicate(ArrayRef<MachineOperand> Pred1,
                                     ArrayRef<MachineOperand> Pred2) const {
  assert(Pred1.size() == 2 && "Invalid PPC first predicate");
  assert(Pred2.size() == 2 && "Invalid PPC second predicate");
 
  if (Pred1[1].getReg() == PPC::CTR8 || Pred1[1].getReg() == PPC::CTR)
    return false;
  if (Pred2[1].getReg() == PPC::CTR8 || Pred2[1].getReg() == PPC::CTR)
    return false;
 
  // P1 can only subsume P2 if they test the same condition register.
  if (Pred1[1].getReg() != Pred2[1].getReg())
    return false;
 
  PPC::Predicate P1 = (PPC::Predicate) Pred1[0].getImm();
  PPC::Predicate P2 = (PPC::Predicate) Pred2[0].getImm();
 
  if (P1 == P2)
    return true;
 
  // Does P1 subsume P2, e.g. GE subsumes GT.
  if (P1 == PPC::PRED_LE &&
      (P2 == PPC::PRED_LT || P2 == PPC::PRED_EQ))
    return true;
  if (P1 == PPC::PRED_GE &&
      (P2 == PPC::PRED_GT || P2 == PPC::PRED_EQ))
    return true;
 
  return false;
}
 
bool PPCInstrInfo::ClobbersPredicate(MachineInstr &MI,
                                     std::vector<MachineOperand> &Pred,
                                     bool SkipDead) const {
  // Note: At the present time, the contents of Pred from this function is
  // unused by IfConversion. This implementation follows ARM by pushing the
  // CR-defining operand. Because the 'DZ' and 'DNZ' count as types of
  // predicate, instructions defining CTR or CTR8 are also included as
  // predicate-defining instructions.
 
  const TargetRegisterClass *RCs[] =
    { &PPC::CRRCRegClass, &PPC::CRBITRCRegClass,
      &PPC::CTRRCRegClass, &PPC::CTRRC8RegClass };
 
  bool Found = false;
  for (const MachineOperand &MO : MI.operands()) {
    for (unsigned c = 0; c < std::size(RCs) && !Found; ++c) {
      const TargetRegisterClass *RC = RCs[c];
      if (MO.isReg()) {
        if (MO.isDef() && RC->contains(MO.getReg())) {
          Pred.push_back(MO);
          Found = true;
        }
      } else if (MO.isRegMask()) {
        for (MCPhysReg R : *RC)
          if (MO.clobbersPhysReg(R)) {
            Pred.push_back(MO);
            Found = true;
          }
      }
    }
  }
 
  return Found;
}
 
bool PPCInstrInfo::analyzeCompare(const MachineInstr &MI, Register &SrcReg,
                                  Register &SrcReg2, int64_t &Mask,
                                  int64_t &Value) const {
  unsigned Opc = MI.getOpcode();
 
  switch (Opc) {
  default: return false;
  case PPC::CMPWI:
  case PPC::CMPLWI:
  case PPC::CMPDI:
  case PPC::CMPLDI:
    SrcReg = MI.getOperand(1).getReg();
    SrcReg2 = 0;
    Value = MI.getOperand(2).getImm();
    Mask = 0xFFFF;
    return true;
  case PPC::CMPW:
  case PPC::CMPLW:
  case PPC::CMPD:
  case PPC::CMPLD:
  case PPC::FCMPUS:
  case PPC::FCMPUD:
    SrcReg = MI.getOperand(1).getReg();
    SrcReg2 = MI.getOperand(2).getReg();
    Value = 0;
    Mask = 0;
    return true;
  }
}
 
bool PPCInstrInfo::optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg,
                                        Register SrcReg2, int64_t Mask,
                                        int64_t Value,
                                        const MachineRegisterInfo *MRI) const {
  if (DisableCmpOpt)
    return false;
 
  int OpC = CmpInstr.getOpcode();
  Register CRReg = CmpInstr.getOperand(0).getReg();
 
  // FP record forms set CR1 based on the exception status bits, not a
  // comparison with zero.
  if (OpC == PPC::FCMPUS || OpC == PPC::FCMPUD)
    return false;
 
  const TargetRegisterInfo *TRI = &getRegisterInfo();
  // The record forms set the condition register based on a signed comparison
  // with zero (so says the ISA manual). This is not as straightforward as it
  // seems, however, because this is always a 64-bit comparison on PPC64, even
  // for instructions that are 32-bit in nature (like slw for example).
  // So, on PPC32, for unsigned comparisons, we can use the record forms only
  // for equality checks (as those don't depend on the sign). On PPC64,
  // we are restricted to equality for unsigned 64-bit comparisons and for
  // signed 32-bit comparisons the applicability is more restricted.
  bool isPPC64 = Subtarget.isPPC64();
  bool is32BitSignedCompare   = OpC ==  PPC::CMPWI || OpC == PPC::CMPW;
  bool is32BitUnsignedCompare = OpC == PPC::CMPLWI || OpC == PPC::CMPLW;
  bool is64BitUnsignedCompare = OpC == PPC::CMPLDI || OpC == PPC::CMPLD;
 
  // Look through copies unless that gets us to a physical register.
  Register ActualSrc = TRI->lookThruCopyLike(SrcReg, MRI);
  if (ActualSrc.isVirtual())
    SrcReg = ActualSrc;
 
  // Get the unique definition of SrcReg.
  MachineInstr *MI = MRI->getUniqueVRegDef(SrcReg);
  if (!MI) return false;
 
  bool equalityOnly = false;
  bool noSub = false;
  if (isPPC64) {
    if (is32BitSignedCompare) {
      // We can perform this optimization only if SrcReg is sign-extending.
      if (isSignExtended(SrcReg, MRI))
        noSub = true;
      else
        return false;
    } else if (is32BitUnsignedCompare) {
      // We can perform this optimization, equality only, if SrcReg is
      // zero-extending.
      if (isZeroExtended(SrcReg, MRI)) {
        noSub = true;
        equalityOnly = true;
      } else
        return false;
    } else
      equalityOnly = is64BitUnsignedCompare;
  } else
    equalityOnly = is32BitUnsignedCompare;
 
  if (equalityOnly) {
    // We need to check the uses of the condition register in order to reject
    // non-equality comparisons.
    for (MachineRegisterInfo::use_instr_iterator
         I = MRI->use_instr_begin(CRReg), IE = MRI->use_instr_end();
         I != IE; ++I) {
      MachineInstr *UseMI = &*I;
      if (UseMI->getOpcode() == PPC::BCC) {
        PPC::Predicate Pred = (PPC::Predicate)UseMI->getOperand(0).getImm();
        unsigned PredCond = PPC::getPredicateCondition(Pred);
        // We ignore hint bits when checking for non-equality comparisons.
        if (PredCond != PPC::PRED_EQ && PredCond != PPC::PRED_NE)
          return false;
      } else if (UseMI->getOpcode() == PPC::ISEL ||
                 UseMI->getOpcode() == PPC::ISEL8) {
        unsigned SubIdx = UseMI->getOperand(3).getSubReg();
        if (SubIdx != PPC::sub_eq)
          return false;
      } else
        return false;
    }
  }
 
  MachineBasicBlock::iterator I = CmpInstr;
 
  // Scan forward to find the first use of the compare.
  for (MachineBasicBlock::iterator EL = CmpInstr.getParent()->end(); I != EL;
       ++I) {
    bool FoundUse = false;
    for (MachineRegisterInfo::use_instr_iterator
         J = MRI->use_instr_begin(CRReg), JE = MRI->use_instr_end();
         J != JE; ++J)
      if (&*J == &*I) {
        FoundUse = true;
        break;
      }
 
    if (FoundUse)
      break;
  }
 
  SmallVector<std::pair<MachineOperand*, PPC::Predicate>, 4> PredsToUpdate;
  SmallVector<std::pair<MachineOperand*, unsigned>, 4> SubRegsToUpdate;
 
  // There are two possible candidates which can be changed to set CR[01].
  // One is MI, the other is a SUB instruction.
  // For CMPrr(r1,r2), we are looking for SUB(r1,r2) or SUB(r2,r1).
  MachineInstr *Sub = nullptr;
  if (SrcReg2 != 0)
    // MI is not a candidate for CMPrr.
    MI = nullptr;
  // FIXME: Conservatively refuse to convert an instruction which isn't in the
  // same BB as the comparison. This is to allow the check below to avoid calls
  // (and other explicit clobbers); instead we should really check for these
  // more explicitly (in at least a few predecessors).
  else if (MI->getParent() != CmpInstr.getParent())
    return false;
  else if (Value != 0) {
    // The record-form instructions set CR bit based on signed comparison
    // against 0. We try to convert a compare against 1 or -1 into a compare
    // against 0 to exploit record-form instructions. For example, we change
    // the condition "greater than -1" into "greater than or equal to 0"
    // and "less than 1" into "less than or equal to 0".
 
    // Since we optimize comparison based on a specific branch condition,
    // we don't optimize if condition code is used by more than once.
    if (equalityOnly || !MRI->hasOneUse(CRReg))
      return false;
 
    MachineInstr *UseMI = &*MRI->use_instr_begin(CRReg);
    if (UseMI->getOpcode() != PPC::BCC)
      return false;
 
    PPC::Predicate Pred = (PPC::Predicate)UseMI->getOperand(0).getImm();
    unsigned PredCond = PPC::getPredicateCondition(Pred);
    unsigned PredHint = PPC::getPredicateHint(Pred);
    int16_t Immed = (int16_t)Value;
 
    // When modifying the condition in the predicate, we propagate hint bits
    // from the original predicate to the new one.
    if (Immed == -1 && PredCond == PPC::PRED_GT)
      // We convert "greater than -1" into "greater than or equal to 0",
      // since we are assuming signed comparison by !equalityOnly
      Pred = PPC::getPredicate(PPC::PRED_GE, PredHint);
    else if (Immed == -1 && PredCond == PPC::PRED_LE)
      // We convert "less than or equal to -1" into "less than 0".
      Pred = PPC::getPredicate(PPC::PRED_LT, PredHint);
    else if (Immed == 1 && PredCond == PPC::PRED_LT)
      // We convert "less than 1" into "less than or equal to 0".
      Pred = PPC::getPredicate(PPC::PRED_LE, PredHint);
    else if (Immed == 1 && PredCond == PPC::PRED_GE)
      // We convert "greater than or equal to 1" into "greater than 0".
      Pred = PPC::getPredicate(PPC::PRED_GT, PredHint);
    else
      return false;
 
    // Convert the comparison and its user to a compare against zero with the
    // appropriate predicate on the branch. Zero comparison might provide
    // optimization opportunities post-RA (see optimization in
    // PPCPreEmitPeephole.cpp).
    UseMI->getOperand(0).setImm(Pred);
    CmpInstr.getOperand(2).setImm(0);
  }
 
  // Search for Sub.
  --I;
 
  // Get ready to iterate backward from CmpInstr.
  MachineBasicBlock::iterator E = MI, B = CmpInstr.getParent()->begin();
 
  for (; I != E && !noSub; --I) {
    const MachineInstr &Instr = *I;
    unsigned IOpC = Instr.getOpcode();
 
    if (&*I != &CmpInstr && (Instr.modifiesRegister(PPC::CR0, TRI) ||
                             Instr.readsRegister(PPC::CR0, TRI)))
      // This instruction modifies or uses the record condition register after
      // the one we want to change. While we could do this transformation, it
      // would likely not be profitable. This transformation removes one
      // instruction, and so even forcing RA to generate one move probably
      // makes it unprofitable.
      return false;
 
    // Check whether CmpInstr can be made redundant by the current instruction.
    if ((OpC == PPC::CMPW || OpC == PPC::CMPLW ||
         OpC == PPC::CMPD || OpC == PPC::CMPLD) &&
        (IOpC == PPC::SUBF || IOpC == PPC::SUBF8) &&
        ((Instr.getOperand(1).getReg() == SrcReg &&
          Instr.getOperand(2).getReg() == SrcReg2) ||
        (Instr.getOperand(1).getReg() == SrcReg2 &&
         Instr.getOperand(2).getReg() == SrcReg))) {
      Sub = &*I;
      break;
    }
 
    if (I == B)
      // The 'and' is below the comparison instruction.
      return false;
  }
 
  // Return false if no candidates exist.
  if (!MI && !Sub)
    return false;
 
  // The single candidate is called MI.
  if (!MI) MI = Sub;
 
  int NewOpC = -1;
  int MIOpC = MI->getOpcode();
  if (MIOpC == PPC::ANDI_rec || MIOpC == PPC::ANDI8_rec ||
      MIOpC == PPC::ANDIS_rec || MIOpC == PPC::ANDIS8_rec)
    NewOpC = MIOpC;
  else {
    NewOpC = PPC::getRecordFormOpcode(MIOpC);
    if (NewOpC == -1 && PPC::getNonRecordFormOpcode(MIOpC) != -1)
      NewOpC = MIOpC;
  }
 
  // FIXME: On the non-embedded POWER architectures, only some of the record
  // forms are fast, and we should use only the fast ones.
 
  // The defining instruction has a record form (or is already a record
  // form). It is possible, however, that we'll need to reverse the condition
  // code of the users.
  if (NewOpC == -1)
    return false;
 
  // This transformation should not be performed if `nsw` is missing and is not
  // `equalityOnly` comparison. Since if there is overflow, sub_lt, sub_gt in
  // CRReg do not reflect correct order. If `equalityOnly` is true, sub_eq in
  // CRReg can reflect if compared values are equal, this optz is still valid.
  if (!equalityOnly && (NewOpC == PPC::SUBF_rec || NewOpC == PPC::SUBF8_rec) &&
      Sub && !Sub->getFlag(MachineInstr::NoSWrap))
    return false;
 
  // If we have SUB(r1, r2) and CMP(r2, r1), the condition code based on CMP
  // needs to be updated to be based on SUB.  Push the condition code
  // operands to OperandsToUpdate.  If it is safe to remove CmpInstr, the
  // condition code of these operands will be modified.
  // Here, Value == 0 means we haven't converted comparison against 1 or -1 to
  // comparison against 0, which may modify predicate.
  bool ShouldSwap = false;
  if (Sub && Value == 0) {
    ShouldSwap = SrcReg2 != 0 && Sub->getOperand(1).getReg() == SrcReg2 &&
      Sub->getOperand(2).getReg() == SrcReg;
 
    // The operands to subf are the opposite of sub, so only in the fixed-point
    // case, invert the order.
    ShouldSwap = !ShouldSwap;
  }
 
  if (ShouldSwap)
    for (MachineRegisterInfo::use_instr_iterator
         I = MRI->use_instr_begin(CRReg), IE = MRI->use_instr_end();
         I != IE; ++I) {
      MachineInstr *UseMI = &*I;
      if (UseMI->getOpcode() == PPC::BCC) {
        PPC::Predicate Pred = (PPC::Predicate) UseMI->getOperand(0).getImm();
        unsigned PredCond = PPC::getPredicateCondition(Pred);
        assert((!equalityOnly ||
                PredCond == PPC::PRED_EQ || PredCond == PPC::PRED_NE) &&
               "Invalid predicate for equality-only optimization");
        (void)PredCond; // To suppress warning in release build.
        PredsToUpdate.push_back(std::make_pair(&(UseMI->getOperand(0)),
                                PPC::getSwappedPredicate(Pred)));
      } else if (UseMI->getOpcode() == PPC::ISEL ||
                 UseMI->getOpcode() == PPC::ISEL8) {
        unsigned NewSubReg = UseMI->getOperand(3).getSubReg();
        assert((!equalityOnly || NewSubReg == PPC::sub_eq) &&
               "Invalid CR bit for equality-only optimization");
 
        if (NewSubReg == PPC::sub_lt)
          NewSubReg = PPC::sub_gt;
        else if (NewSubReg == PPC::sub_gt)
          NewSubReg = PPC::sub_lt;
 
        SubRegsToUpdate.push_back(std::make_pair(&(UseMI->getOperand(3)),
                                                 NewSubReg));
      } else // We need to abort on a user we don't understand.
        return false;
    }
  assert(!(Value != 0 && ShouldSwap) &&
         "Non-zero immediate support and ShouldSwap"
         "may conflict in updating predicate");
 
  // Create a new virtual register to hold the value of the CR set by the
  // record-form instruction. If the instruction was not previously in
  // record form, then set the kill flag on the CR.
  CmpInstr.eraseFromParent();
 
  MachineBasicBlock::iterator MII = MI;
  BuildMI(*MI->getParent(), std::next(MII), MI->getDebugLoc(),
          get(TargetOpcode::COPY), CRReg)
    .addReg(PPC::CR0, MIOpC != NewOpC ? RegState::Kill : 0);
 
  // Even if CR0 register were dead before, it is alive now since the
  // instruction we just built uses it.
  MI->clearRegisterDeads(PPC::CR0);
 
  if (MIOpC != NewOpC) {
    // We need to be careful here: we're replacing one instruction with
    // another, and we need to make sure that we get all of the right
    // implicit uses and defs. On the other hand, the caller may be holding
    // an iterator to this instruction, and so we can't delete it (this is
    // specifically the case if this is the instruction directly after the
    // compare).
 
    // Rotates are expensive instructions. If we're emitting a record-form
    // rotate that can just be an andi/andis, we should just emit that.
    if (MIOpC == PPC::RLWINM || MIOpC == PPC::RLWINM8) {
      Register GPRRes = MI->getOperand(0).getReg();
      int64_t SH = MI->getOperand(2).getImm();
      int64_t MB = MI->getOperand(3).getImm();
      int64_t ME = MI->getOperand(4).getImm();
      // We can only do this if both the start and end of the mask are in the
      // same halfword.
      bool MBInLoHWord = MB >= 16;
      bool MEInLoHWord = ME >= 16;
      uint64_t Mask = ~0LLU;
 
      if (MB <= ME && MBInLoHWord == MEInLoHWord && SH == 0) {
        Mask = ((1LLU << (32 - MB)) - 1) & ~((1LLU << (31 - ME)) - 1);
        // The mask value needs to shift right 16 if we're emitting andis.
        Mask >>= MBInLoHWord ? 0 : 16;
        NewOpC = MIOpC == PPC::RLWINM
                     ? (MBInLoHWord ? PPC::ANDI_rec : PPC::ANDIS_rec)
                     : (MBInLoHWord ? PPC::ANDI8_rec : PPC::ANDIS8_rec);
      } else if (MRI->use_empty(GPRRes) && (ME == 31) &&
                 (ME - MB + 1 == SH) && (MB >= 16)) {
        // If we are rotating by the exact number of bits as are in the mask
        // and the mask is in the least significant bits of the register,
        // that's just an andis. (as long as the GPR result has no uses).
        Mask = ((1LLU << 32) - 1) & ~((1LLU << (32 - SH)) - 1);
        Mask >>= 16;
        NewOpC = MIOpC == PPC::RLWINM ? PPC::ANDIS_rec : PPC::ANDIS8_rec;
      }
      // If we've set the mask, we can transform.
      if (Mask != ~0LLU) {
        MI->removeOperand(4);
        MI->removeOperand(3);
        MI->getOperand(2).setImm(Mask);
        NumRcRotatesConvertedToRcAnd++;
      }
    } else if (MIOpC == PPC::RLDICL && MI->getOperand(2).getImm() == 0) {
      int64_t MB = MI->getOperand(3).getImm();
      if (MB >= 48) {
        uint64_t Mask = (1LLU << (63 - MB + 1)) - 1;
        NewOpC = PPC::ANDI8_rec;
        MI->removeOperand(3);
        MI->getOperand(2).setImm(Mask);
        NumRcRotatesConvertedToRcAnd++;
      }
    }
 
    const MCInstrDesc &NewDesc = get(NewOpC);
    MI->setDesc(NewDesc);
 
    for (MCPhysReg ImpDef : NewDesc.implicit_defs()) {
      if (!MI->definesRegister(ImpDef, /*TRI=*/nullptr)) {
        MI->addOperand(*MI->getParent()->getParent(),
                       MachineOperand::CreateReg(ImpDef, true, true));
      }
    }
    for (MCPhysReg ImpUse : NewDesc.implicit_uses()) {
      if (!MI->readsRegister(ImpUse, /*TRI=*/nullptr)) {
        MI->addOperand(*MI->getParent()->getParent(),
                       MachineOperand::CreateReg(ImpUse, false, true));
      }
    }
  }
  assert(MI->definesRegister(PPC::CR0, /*TRI=*/nullptr) &&
         "Record-form instruction does not define cr0?");
 
  // Modify the condition code of operands in OperandsToUpdate.
  // Since we have SUB(r1, r2) and CMP(r2, r1), the condition code needs to
  // be changed from r2 > r1 to r1 < r2, from r2 < r1 to r1 > r2, etc.
  for (unsigned i = 0, e = PredsToUpdate.size(); i < e; i++)
    PredsToUpdate[i].first->setImm(PredsToUpdate[i].second);
 
  for (unsigned i = 0, e = SubRegsToUpdate.size(); i < e; i++)
    SubRegsToUpdate[i].first->setSubReg(SubRegsToUpdate[i].second);
 
  return true;
}
 
bool PPCInstrInfo::optimizeCmpPostRA(MachineInstr &CmpMI) const {
  MachineRegisterInfo *MRI = &CmpMI.getParent()->getParent()->getRegInfo();
  if (MRI->isSSA())
    return false;
 
  Register SrcReg, SrcReg2;
  int64_t CmpMask, CmpValue;
  if (!analyzeCompare(CmpMI, SrcReg, SrcReg2, CmpMask, CmpValue))
    return false;
 
  // Try to optimize the comparison against 0.
  if (CmpValue || !CmpMask || SrcReg2)
    return false;
 
  // The record forms set the condition register based on a signed comparison
  // with zero (see comments in optimizeCompareInstr). Since we can't do the
  // equality checks in post-RA, we are more restricted on a unsigned
  // comparison.
  unsigned Opc = CmpMI.getOpcode();
  if (Opc == PPC::CMPLWI || Opc == PPC::CMPLDI)
    return false;
 
  // The record forms are always based on a 64-bit comparison on PPC64
  // (similary, a 32-bit comparison on PPC32), while the CMPWI is a 32-bit
  // comparison. Since we can't do the equality checks in post-RA, we bail out
  // the case.
  if (Subtarget.isPPC64() && Opc == PPC::CMPWI)
    return false;
 
  // CmpMI can't be deleted if it has implicit def.
  if (CmpMI.hasImplicitDef())
    return false;
 
  bool SrcRegHasOtherUse = false;
  MachineInstr *SrcMI = getDefMIPostRA(SrcReg, CmpMI, SrcRegHasOtherUse);
  if (!SrcMI || !SrcMI->definesRegister(SrcReg, /*TRI=*/nullptr))
    return false;
 
  MachineOperand RegMO = CmpMI.getOperand(0);
  Register CRReg = RegMO.getReg();
  if (CRReg != PPC::CR0)
    return false;
 
  // Make sure there is no def/use of CRReg between SrcMI and CmpMI.
  bool SeenUseOfCRReg = false;
  bool IsCRRegKilled = false;
  if (!isRegElgibleForForwarding(RegMO, *SrcMI, CmpMI, false, IsCRRegKilled,
                                 SeenUseOfCRReg) ||
      SrcMI->definesRegister(CRReg, /*TRI=*/nullptr) || SeenUseOfCRReg)
    return false;
 
  int SrcMIOpc = SrcMI->getOpcode();
  int NewOpC = PPC::getRecordFormOpcode(SrcMIOpc);
  if (NewOpC == -1)
    return false;
 
  LLVM_DEBUG(dbgs() << "Replace Instr: ");
  LLVM_DEBUG(SrcMI->dump());
 
  const MCInstrDesc &NewDesc = get(NewOpC);
  SrcMI->setDesc(NewDesc);
  MachineInstrBuilder(*SrcMI->getParent()->getParent(), SrcMI)
      .addReg(CRReg, RegState::ImplicitDefine);
  SrcMI->clearRegisterDeads(CRReg);
 
  assert(SrcMI->definesRegister(PPC::CR0, /*TRI=*/nullptr) &&
         "Record-form instruction does not define cr0?");
 
  LLVM_DEBUG(dbgs() << "with: ");
  LLVM_DEBUG(SrcMI->dump());
  LLVM_DEBUG(dbgs() << "Delete dead instruction: ");
  LLVM_DEBUG(CmpMI.dump());
  return true;
}
 
bool PPCInstrInfo::getMemOperandsWithOffsetWidth(
    const MachineInstr &LdSt, SmallVectorImpl<const MachineOperand *> &BaseOps,
    int64_t &Offset, bool &OffsetIsScalable, LocationSize &Width,
    const TargetRegisterInfo *TRI) const {
  const MachineOperand *BaseOp;
  OffsetIsScalable = false;
  if (!getMemOperandWithOffsetWidth(LdSt, BaseOp, Offset, Width, TRI))
    return false;
  BaseOps.push_back(BaseOp);
  return true;
}
 
static bool isLdStSafeToCluster(const MachineInstr &LdSt,
                                const TargetRegisterInfo *TRI) {
  // If this is a volatile load/store, don't mess with it.
  if (LdSt.hasOrderedMemoryRef() || LdSt.getNumExplicitOperands() != 3)
    return false;
 
  if (LdSt.getOperand(2).isFI())
    return true;
 
  assert(LdSt.getOperand(2).isReg() && "Expected a reg operand.");
  // Can't cluster if the instruction modifies the base register
  // or it is update form. e.g. ld r2,3(r2)
  if (LdSt.modifiesRegister(LdSt.getOperand(2).getReg(), TRI))
    return false;
 
  return true;
}
 
// Only cluster instruction pair that have the same opcode, and they are
// clusterable according to PowerPC specification.
static bool isClusterableLdStOpcPair(unsigned FirstOpc, unsigned SecondOpc,
                                     const PPCSubtarget &Subtarget) {
  switch (FirstOpc) {
  default:
    return false;
  case PPC::STD:
  case PPC::STFD:
  case PPC::STXSD:
  case PPC::DFSTOREf64:
    return FirstOpc == SecondOpc;
  // PowerPC backend has opcode STW/STW8 for instruction "stw" to deal with
  // 32bit and 64bit instruction selection. They are clusterable pair though
  // they are different opcode.
  case PPC::STW:
  case PPC::STW8:
    return SecondOpc == PPC::STW || SecondOpc == PPC::STW8;
  }
}
 
bool PPCInstrInfo::shouldClusterMemOps(
    ArrayRef<const MachineOperand *> BaseOps1, int64_t OpOffset1,
    bool OffsetIsScalable1, ArrayRef<const MachineOperand *> BaseOps2,
    int64_t OpOffset2, bool OffsetIsScalable2, unsigned ClusterSize,
    unsigned NumBytes) const {
 
  assert(BaseOps1.size() == 1 && BaseOps2.size() == 1);
  const MachineOperand &BaseOp1 = *BaseOps1.front();
  const MachineOperand &BaseOp2 = *BaseOps2.front();
  assert((BaseOp1.isReg() || BaseOp1.isFI()) &&
         "Only base registers and frame indices are supported.");
 
  // ClusterSize means the number of memory operations that will have been
  // clustered if this hook returns true.
  // Don't cluster memory op if there are already two ops clustered at least.
  if (ClusterSize > 2)
    return false;
 
  // Cluster the load/store only when they have the same base
  // register or FI.
  if ((BaseOp1.isReg() != BaseOp2.isReg()) ||
      (BaseOp1.isReg() && BaseOp1.getReg() != BaseOp2.getReg()) ||
      (BaseOp1.isFI() && BaseOp1.getIndex() != BaseOp2.getIndex()))
    return false;
 
  // Check if the load/store are clusterable according to the PowerPC
  // specification.
  const MachineInstr &FirstLdSt = *BaseOp1.getParent();
  const MachineInstr &SecondLdSt = *BaseOp2.getParent();
  unsigned FirstOpc = FirstLdSt.getOpcode();
  unsigned SecondOpc = SecondLdSt.getOpcode();
  const TargetRegisterInfo *TRI = &getRegisterInfo();
  // Cluster the load/store only when they have the same opcode, and they are
  // clusterable opcode according to PowerPC specification.
  if (!isClusterableLdStOpcPair(FirstOpc, SecondOpc, Subtarget))
    return false;
 
  // Can't cluster load/store that have ordered or volatile memory reference.
  if (!isLdStSafeToCluster(FirstLdSt, TRI) ||
      !isLdStSafeToCluster(SecondLdSt, TRI))
    return false;
 
  int64_t Offset1 = 0, Offset2 = 0;
  LocationSize Width1 = 0, Width2 = 0;
  const MachineOperand *Base1 = nullptr, *Base2 = nullptr;
  if (!getMemOperandWithOffsetWidth(FirstLdSt, Base1, Offset1, Width1, TRI) ||
      !getMemOperandWithOffsetWidth(SecondLdSt, Base2, Offset2, Width2, TRI) ||
      Width1 != Width2)
    return false;
 
  assert(Base1 == &BaseOp1 && Base2 == &BaseOp2 &&
         "getMemOperandWithOffsetWidth return incorrect base op");
  // The caller should already have ordered FirstMemOp/SecondMemOp by offset.
  assert(Offset1 <= Offset2 && "Caller should have ordered offsets.");
  return Offset1 + (int64_t)Width1.getValue() == Offset2;
}
 
/// GetInstSize - Return the number of bytes of code the specified
/// instruction may be.  This returns the maximum number of bytes.
///
unsigned PPCInstrInfo::getInstSizeInBytes(const MachineInstr &MI) const {
  unsigned Opcode = MI.getOpcode();
 
  if (Opcode == PPC::INLINEASM || Opcode == PPC::INLINEASM_BR) {
    const MachineFunction *MF = MI.getParent()->getParent();
    const char *AsmStr = MI.getOperand(0).getSymbolName();
    return getInlineAsmLength(AsmStr, *MF->getTarget().getMCAsmInfo());
  } else if (Opcode == TargetOpcode::STACKMAP) {
    StackMapOpers Opers(&MI);
    return Opers.getNumPatchBytes();
  } else if (Opcode == TargetOpcode::PATCHPOINT) {
    PatchPointOpers Opers(&MI);
    return Opers.getNumPatchBytes();
  } else {
    return get(Opcode).getSize();
  }
}
 
std::pair<unsigned, unsigned>
PPCInstrInfo::decomposeMachineOperandsTargetFlags(unsigned TF) const {
  // PPC always uses a direct mask.
  return std::make_pair(TF, 0u);
}
 
ArrayRef<std::pair<unsigned, const char *>>
PPCInstrInfo::getSerializableDirectMachineOperandTargetFlags() const {
  using namespace PPCII;
  static const std::pair<unsigned, const char *> TargetFlags[] = {
      {MO_PLT, "ppc-plt"},
      {MO_PIC_FLAG, "ppc-pic"},
      {MO_PCREL_FLAG, "ppc-pcrel"},
      {MO_GOT_FLAG, "ppc-got"},
      {MO_PCREL_OPT_FLAG, "ppc-opt-pcrel"},
      {MO_TLSGD_FLAG, "ppc-tlsgd"},
      {MO_TPREL_FLAG, "ppc-tprel"},
      {MO_TLSLDM_FLAG, "ppc-tlsldm"},
      {MO_TLSLD_FLAG, "ppc-tlsld"},
      {MO_TLSGDM_FLAG, "ppc-tlsgdm"},
      {MO_GOT_TLSGD_PCREL_FLAG, "ppc-got-tlsgd-pcrel"},
      {MO_GOT_TLSLD_PCREL_FLAG, "ppc-got-tlsld-pcrel"},
      {MO_GOT_TPREL_PCREL_FLAG, "ppc-got-tprel-pcrel"},
      {MO_LO, "ppc-lo"},
      {MO_HA, "ppc-ha"},
      {MO_TPREL_LO, "ppc-tprel-lo"},
      {MO_TPREL_HA, "ppc-tprel-ha"},
      {MO_DTPREL_LO, "ppc-dtprel-lo"},
      {MO_TLSLD_LO, "ppc-tlsld-lo"},
      {MO_TOC_LO, "ppc-toc-lo"},
      {MO_TLS, "ppc-tls"},
      {MO_PIC_HA_FLAG, "ppc-ha-pic"},
      {MO_PIC_LO_FLAG, "ppc-lo-pic"},
      {MO_TPREL_PCREL_FLAG, "ppc-tprel-pcrel"},
      {MO_TLS_PCREL_FLAG, "ppc-tls-pcrel"},
      {MO_GOT_PCREL_FLAG, "ppc-got-pcrel"},
  };
  return ArrayRef(TargetFlags);
}
 
// Expand VSX Memory Pseudo instruction to either a VSX or a FP instruction.
// The VSX versions have the advantage of a full 64-register target whereas
// the FP ones have the advantage of lower latency and higher throughput. So
// what we are after is using the faster instructions in low register pressure
// situations and using the larger register file in high register pressure
// situations.
bool PPCInstrInfo::expandVSXMemPseudo(MachineInstr &MI) const {
    unsigned UpperOpcode, LowerOpcode;
    switch (MI.getOpcode()) {
    case PPC::DFLOADf32:
      UpperOpcode = PPC::LXSSP;
      LowerOpcode = PPC::LFS;
      break;
    case PPC::DFLOADf64:
      UpperOpcode = PPC::LXSD;
      LowerOpcode = PPC::LFD;
      break;
    case PPC::DFSTOREf32:
      UpperOpcode = PPC::STXSSP;
      LowerOpcode = PPC::STFS;
      break;
    case PPC::DFSTOREf64:
      UpperOpcode = PPC::STXSD;
      LowerOpcode = PPC::STFD;
      break;
    case PPC::XFLOADf32:
      UpperOpcode = PPC::LXSSPX;
      LowerOpcode = PPC::LFSX;
      break;
    case PPC::XFLOADf64:
      UpperOpcode = PPC::LXSDX;
      LowerOpcode = PPC::LFDX;
      break;
    case PPC::XFSTOREf32:
      UpperOpcode = PPC::STXSSPX;
      LowerOpcode = PPC::STFSX;
      break;
    case PPC::XFSTOREf64:
      UpperOpcode = PPC::STXSDX;
      LowerOpcode = PPC::STFDX;
      break;
    case PPC::LIWAX:
      UpperOpcode = PPC::LXSIWAX;
      LowerOpcode = PPC::LFIWAX;
      break;
    case PPC::LIWZX:
      UpperOpcode = PPC::LXSIWZX;
      LowerOpcode = PPC::LFIWZX;
      break;
    case PPC::STIWX:
      UpperOpcode = PPC::STXSIWX;
      LowerOpcode = PPC::STFIWX;
      break;
    default:
      llvm_unreachable("Unknown Operation!");
    }
 
    Register TargetReg = MI.getOperand(0).getReg();
    unsigned Opcode;
    if ((TargetReg >= PPC::F0 && TargetReg <= PPC::F31) ||
        (TargetReg >= PPC::VSL0 && TargetReg <= PPC::VSL31))
      Opcode = LowerOpcode;
    else
      Opcode = UpperOpcode;
    MI.setDesc(get(Opcode));
    return true;
}
 
static bool isAnImmediateOperand(const MachineOperand &MO) {
  return MO.isCPI() || MO.isGlobal() || MO.isImm();
}
 
bool PPCInstrInfo::expandPostRAPseudo(MachineInstr &MI) const {
  auto &MBB = *MI.getParent();
  auto DL = MI.getDebugLoc();
 
  switch (MI.getOpcode()) {
  case PPC::BUILD_UACC: {
    MCRegister ACC = MI.getOperand(0).getReg();
    MCRegister UACC = MI.getOperand(1).getReg();
    if (ACC - PPC::ACC0 != UACC - PPC::UACC0) {
      MCRegister SrcVSR = PPC::VSL0 + (UACC - PPC::UACC0) * 4;
      MCRegister DstVSR = PPC::VSL0 + (ACC - PPC::ACC0) * 4;
      // FIXME: This can easily be improved to look up to the top of the MBB
      // to see if the inputs are XXLOR's. If they are and SrcReg is killed,
      // we can just re-target any such XXLOR's to DstVSR + offset.
      for (int VecNo = 0; VecNo < 4; VecNo++)
        BuildMI(MBB, MI, DL, get(PPC::XXLOR), DstVSR + VecNo)
            .addReg(SrcVSR + VecNo)
            .addReg(SrcVSR + VecNo);
    }
    // BUILD_UACC is expanded to 4 copies of the underlying vsx registers.
    // So after building the 4 copies, we can replace the BUILD_UACC instruction
    // with a NOP.
    [[fallthrough]];
  }
  case PPC::KILL_PAIR: {
    MI.setDesc(get(PPC::UNENCODED_NOP));
    MI.removeOperand(1);
    MI.removeOperand(0);
    return true;
  }
  case TargetOpcode::LOAD_STACK_GUARD: {
    assert(Subtarget.isTargetLinux() &&
           "Only Linux target is expected to contain LOAD_STACK_GUARD");
    const int64_t Offset = Subtarget.isPPC64() ? -0x7010 : -0x7008;
    const unsigned Reg = Subtarget.isPPC64() ? PPC::X13 : PPC::R2;
    MI.setDesc(get(Subtarget.isPPC64() ? PPC::LD : PPC::LWZ));
    MachineInstrBuilder(*MI.getParent()->getParent(), MI)
        .addImm(Offset)
        .addReg(Reg);
    return true;
  }
  case PPC::PPCLdFixedAddr: {
    assert(Subtarget.getTargetTriple().isOSGlibc() &&
           "Only targets with Glibc expected to contain PPCLdFixedAddr");
    int64_t Offset = 0;
    const unsigned Reg = Subtarget.isPPC64() ? PPC::X13 : PPC::R2;
    MI.setDesc(get(PPC::LWZ));
    uint64_t FAType = MI.getOperand(1).getImm();
#undef PPC_LNX_FEATURE
#undef PPC_CPU
#define PPC_LNX_DEFINE_OFFSETS
#include "llvm/TargetParser/PPCTargetParser.def"
    bool IsLE = Subtarget.isLittleEndian();
    bool Is64 = Subtarget.isPPC64();
    if (FAType == PPC_FAWORD_HWCAP) {
      if (IsLE)
        Offset = Is64 ? PPC_HWCAP_OFFSET_LE64 : PPC_HWCAP_OFFSET_LE32;
      else
        Offset = Is64 ? PPC_HWCAP_OFFSET_BE64 : PPC_HWCAP_OFFSET_BE32;
    } else if (FAType == PPC_FAWORD_HWCAP2) {
      if (IsLE)
        Offset = Is64 ? PPC_HWCAP2_OFFSET_LE64 : PPC_HWCAP2_OFFSET_LE32;
      else
        Offset = Is64 ? PPC_HWCAP2_OFFSET_BE64 : PPC_HWCAP2_OFFSET_BE32;
    } else if (FAType == PPC_FAWORD_CPUID) {
      if (IsLE)
        Offset = Is64 ? PPC_CPUID_OFFSET_LE64 : PPC_CPUID_OFFSET_LE32;
      else
        Offset = Is64 ? PPC_CPUID_OFFSET_BE64 : PPC_CPUID_OFFSET_BE32;
    }
    assert(Offset && "Do not know the offset for this fixed addr load");
    MI.removeOperand(1);
    Subtarget.getTargetMachine().setGlibcHWCAPAccess();
    MachineInstrBuilder(*MI.getParent()->getParent(), MI)
        .addImm(Offset)
        .addReg(Reg);
    return true;
#define PPC_TGT_PARSER_UNDEF_MACROS
#include "llvm/TargetParser/PPCTargetParser.def"
#undef PPC_TGT_PARSER_UNDEF_MACROS
  }
  case PPC::DFLOADf32:
  case PPC::DFLOADf64:
  case PPC::DFSTOREf32:
  case PPC::DFSTOREf64: {
    assert(Subtarget.hasP9Vector() &&
           "Invalid D-Form Pseudo-ops on Pre-P9 target.");
    assert(MI.getOperand(2).isReg() &&
           isAnImmediateOperand(MI.getOperand(1)) &&
           "D-form op must have register and immediate operands");
    return expandVSXMemPseudo(MI);
  }
  case PPC::XFLOADf32:
  case PPC::XFSTOREf32:
  case PPC::LIWAX:
  case PPC::LIWZX:
  case PPC::STIWX: {
    assert(Subtarget.hasP8Vector() &&
           "Invalid X-Form Pseudo-ops on Pre-P8 target.");
    assert(MI.getOperand(2).isReg() && MI.getOperand(1).isReg() &&
           "X-form op must have register and register operands");
    return expandVSXMemPseudo(MI);
  }
  case PPC::XFLOADf64:
  case PPC::XFSTOREf64: {
    assert(Subtarget.hasVSX() &&
           "Invalid X-Form Pseudo-ops on target that has no VSX.");
    assert(MI.getOperand(2).isReg() && MI.getOperand(1).isReg() &&
           "X-form op must have register and register operands");
    return expandVSXMemPseudo(MI);
  }
  case PPC::SPILLTOVSR_LD: {
    Register TargetReg = MI.getOperand(0).getReg();
    if (PPC::VSFRCRegClass.contains(TargetReg)) {
      MI.setDesc(get(PPC::DFLOADf64));
      return expandPostRAPseudo(MI);
    }
    else
      MI.setDesc(get(PPC::LD));
    return true;
  }
  case PPC::SPILLTOVSR_ST: {
    Register SrcReg = MI.getOperand(0).getReg();
    if (PPC::VSFRCRegClass.contains(SrcReg)) {
      NumStoreSPILLVSRRCAsVec++;
      MI.setDesc(get(PPC::DFSTOREf64));
      return expandPostRAPseudo(MI);
    } else {
      NumStoreSPILLVSRRCAsGpr++;
      MI.setDesc(get(PPC::STD));
    }
    return true;
  }
  case PPC::SPILLTOVSR_LDX: {
    Register TargetReg = MI.getOperand(0).getReg();
    if (PPC::VSFRCRegClass.contains(TargetReg))
      MI.setDesc(get(PPC::LXSDX));
    else
      MI.setDesc(get(PPC::LDX));
    return true;
  }
  case PPC::SPILLTOVSR_STX: {
    Register SrcReg = MI.getOperand(0).getReg();
    if (PPC::VSFRCRegClass.contains(SrcReg)) {
      NumStoreSPILLVSRRCAsVec++;
      MI.setDesc(get(PPC::STXSDX));
    } else {
      NumStoreSPILLVSRRCAsGpr++;
      MI.setDesc(get(PPC::STDX));
    }
    return true;
  }
 
    // FIXME: Maybe we can expand it in 'PowerPC Expand Atomic' pass.
  case PPC::CFENCE:
  case PPC::CFENCE8: {
    auto Val = MI.getOperand(0).getReg();
    unsigned CmpOp = Subtarget.isPPC64() ? PPC::CMPD : PPC::CMPW;
    BuildMI(MBB, MI, DL, get(CmpOp), PPC::CR7).addReg(Val).addReg(Val);
    BuildMI(MBB, MI, DL, get(PPC::CTRL_DEP))
        .addImm(PPC::PRED_NE_MINUS)
        .addReg(PPC::CR7)
        .addImm(1);
    MI.setDesc(get(PPC::ISYNC));
    MI.removeOperand(0);
    return true;
  }
  }
  return false;
}
 
// Essentially a compile-time implementation of a compare->isel sequence.
// It takes two constants to compare, along with the true/false registers
// and the comparison type (as a subreg to a CR field) and returns one
// of the true/false registers, depending on the comparison results.
static unsigned selectReg(int64_t Imm1, int64_t Imm2, unsigned CompareOpc,
                          unsigned TrueReg, unsigned FalseReg,
                          unsigned CRSubReg) {
  // Signed comparisons. The immediates are assumed to be sign-extended.
  if (CompareOpc == PPC::CMPWI || CompareOpc == PPC::CMPDI) {
    switch (CRSubReg) {
    default: llvm_unreachable("Unknown integer comparison type.");
    case PPC::sub_lt:
      return Imm1 < Imm2 ? TrueReg : FalseReg;
    case PPC::sub_gt:
      return Imm1 > Imm2 ? TrueReg : FalseReg;
    case PPC::sub_eq:
      return Imm1 == Imm2 ? TrueReg : FalseReg;
    }
  }
  // Unsigned comparisons.
  else if (CompareOpc == PPC::CMPLWI || CompareOpc == PPC::CMPLDI) {
    switch (CRSubReg) {
    default: llvm_unreachable("Unknown integer comparison type.");
    case PPC::sub_lt:
      return (uint64_t)Imm1 < (uint64_t)Imm2 ? TrueReg : FalseReg;
    case PPC::sub_gt:
      return (uint64_t)Imm1 > (uint64_t)Imm2 ? TrueReg : FalseReg;
    case PPC::sub_eq:
      return Imm1 == Imm2 ? TrueReg : FalseReg;
    }
  }
  return PPC::NoRegister;
}
 
void PPCInstrInfo::replaceInstrOperandWithImm(MachineInstr &MI,
                                              unsigned OpNo,
                                              int64_t Imm) const {
  assert(MI.getOperand(OpNo).isReg() && "Operand must be a REG");
  // Replace the REG with the Immediate.
  Register InUseReg = MI.getOperand(OpNo).getReg();
  MI.getOperand(OpNo).ChangeToImmediate(Imm);
 
  // We need to make sure that the MI didn't have any implicit use
  // of this REG any more. We don't call MI.implicit_operands().empty() to
  // return early, since MI's MCID might be changed in calling context, as a
  // result its number of explicit operands may be changed, thus the begin of
  // implicit operand is changed.
  const TargetRegisterInfo *TRI = &getRegisterInfo();
  int UseOpIdx = MI.findRegisterUseOperandIdx(InUseReg, TRI, false);
  if (UseOpIdx >= 0) {
    MachineOperand &MO = MI.getOperand(UseOpIdx);
    if (MO.isImplicit())
      // The operands must always be in the following order:
      // - explicit reg defs,
      // - other explicit operands (reg uses, immediates, etc.),
      // - implicit reg defs
      // - implicit reg uses
      // Therefore, removing the implicit operand won't change the explicit
      // operands layout.
      MI.removeOperand(UseOpIdx);
  }
}
 
// Replace an instruction with one that materializes a constant (and sets
// CR0 if the original instruction was a record-form instruction).
void PPCInstrInfo::replaceInstrWithLI(MachineInstr &MI,
                                      const LoadImmediateInfo &LII) const {
  // Remove existing operands.
  int OperandToKeep = LII.SetCR ? 1 : 0;
  for (int i = MI.getNumOperands() - 1; i > OperandToKeep; i--)
    MI.removeOperand(i);
 
  // Replace the instruction.
  if (LII.SetCR) {
    MI.setDesc(get(LII.Is64Bit ? PPC::ANDI8_rec : PPC::ANDI_rec));
    // Set the immediate.
    MachineInstrBuilder(*MI.getParent()->getParent(), MI)
        .addImm(LII.Imm).addReg(PPC::CR0, RegState::ImplicitDefine);
    return;
  }
  else
    MI.setDesc(get(LII.Is64Bit ? PPC::LI8 : PPC::LI));
 
  // Set the immediate.
  MachineInstrBuilder(*MI.getParent()->getParent(), MI)
      .addImm(LII.Imm);
}
 
MachineInstr *PPCInstrInfo::getDefMIPostRA(unsigned Reg, MachineInstr &MI,
                                           bool &SeenIntermediateUse) const {
  assert(!MI.getParent()->getParent()->getRegInfo().isSSA() &&
         "Should be called after register allocation.");
  const TargetRegisterInfo *TRI = &getRegisterInfo();
  MachineBasicBlock::reverse_iterator E = MI.getParent()->rend(), It = MI;
  It++;
  SeenIntermediateUse = false;
  for (; It != E; ++It) {
    if (It->modifiesRegister(Reg, TRI))
      return &*It;
    if (It->readsRegister(Reg, TRI))
      SeenIntermediateUse = true;
  }
  return nullptr;
}
 
void PPCInstrInfo::materializeImmPostRA(MachineBasicBlock &MBB,
                                        MachineBasicBlock::iterator MBBI,
                                        const DebugLoc &DL, Register Reg,
                                        int64_t Imm) const {
  assert(!MBB.getParent()->getRegInfo().isSSA() &&
         "Register should be in non-SSA form after RA");
  bool isPPC64 = Subtarget.isPPC64();
  // FIXME: Materialization here is not optimal.
  // For some special bit patterns we can use less instructions.
  // See `selectI64ImmDirect` in PPCISelDAGToDAG.cpp.
  if (isInt<16>(Imm)) {
    BuildMI(MBB, MBBI, DL, get(isPPC64 ? PPC::LI8 : PPC::LI), Reg).addImm(Imm);
  } else if (isInt<32>(Imm)) {
    BuildMI(MBB, MBBI, DL, get(isPPC64 ? PPC::LIS8 : PPC::LIS), Reg)
        .addImm(Imm >> 16);
    if (Imm & 0xFFFF)
      BuildMI(MBB, MBBI, DL, get(isPPC64 ? PPC::ORI8 : PPC::ORI), Reg)
          .addReg(Reg, RegState::Kill)
          .addImm(Imm & 0xFFFF);
  } else {
    assert(isPPC64 && "Materializing 64-bit immediate to single register is "
                      "only supported in PPC64");
    BuildMI(MBB, MBBI, DL, get(PPC::LIS8), Reg).addImm(Imm >> 48);
    if ((Imm >> 32) & 0xFFFF)
      BuildMI(MBB, MBBI, DL, get(PPC::ORI8), Reg)
          .addReg(Reg, RegState::Kill)
          .addImm((Imm >> 32) & 0xFFFF);
    BuildMI(MBB, MBBI, DL, get(PPC::RLDICR), Reg)
        .addReg(Reg, RegState::Kill)
        .addImm(32)
        .addImm(31);
    BuildMI(MBB, MBBI, DL, get(PPC::ORIS8), Reg)
        .addReg(Reg, RegState::Kill)
        .addImm((Imm >> 16) & 0xFFFF);
    if (Imm & 0xFFFF)
      BuildMI(MBB, MBBI, DL, get(PPC::ORI8), Reg)
          .addReg(Reg, RegState::Kill)
          .addImm(Imm & 0xFFFF);
  }
}
 
MachineInstr *PPCInstrInfo::getForwardingDefMI(
  MachineInstr &MI,
  unsigned &OpNoForForwarding,
  bool &SeenIntermediateUse) const {
  OpNoForForwarding = ~0U;
  MachineInstr *DefMI = nullptr;
  MachineRegisterInfo *MRI = &MI.getParent()->getParent()->getRegInfo();
  const TargetRegisterInfo *TRI = &getRegisterInfo();
  // If we're in SSA, get the defs through the MRI. Otherwise, only look
  // within the basic block to see if the register is defined using an
  // LI/LI8/ADDI/ADDI8.
  if (MRI->isSSA()) {
    for (int i = 1, e = MI.getNumOperands(); i < e; i++) {
      if (!MI.getOperand(i).isReg())
        continue;
      Register Reg = MI.getOperand(i).getReg();
      if (!Reg.isVirtual())
        continue;
      Register TrueReg = TRI->lookThruCopyLike(Reg, MRI);
      if (TrueReg.isVirtual()) {
        MachineInstr *DefMIForTrueReg = MRI->getVRegDef(TrueReg);
        if (DefMIForTrueReg->getOpcode() == PPC::LI ||
            DefMIForTrueReg->getOpcode() == PPC::LI8 ||
            DefMIForTrueReg->getOpcode() == PPC::ADDI ||
            DefMIForTrueReg->getOpcode() == PPC::ADDI8) {
          OpNoForForwarding = i;
          DefMI = DefMIForTrueReg;
          // The ADDI and LI operand maybe exist in one instruction at same
          // time. we prefer to fold LI operand as LI only has one Imm operand
          // and is more possible to be converted. So if current DefMI is
          // ADDI/ADDI8, we continue to find possible LI/LI8.
          if (DefMI->getOpcode() == PPC::LI || DefMI->getOpcode() == PPC::LI8)
            break;
        }
      }
    }
  } else {
    // Looking back through the definition for each operand could be expensive,
    // so exit early if this isn't an instruction that either has an immediate
    // form or is already an immediate form that we can handle.
    ImmInstrInfo III;
    unsigned Opc = MI.getOpcode();
    bool ConvertibleImmForm =
        Opc == PPC::CMPWI || Opc == PPC::CMPLWI || Opc == PPC::CMPDI ||
        Opc == PPC::CMPLDI || Opc == PPC::ADDI || Opc == PPC::ADDI8 ||
        Opc == PPC::ORI || Opc == PPC::ORI8 || Opc == PPC::XORI ||
        Opc == PPC::XORI8 || Opc == PPC::RLDICL || Opc == PPC::RLDICL_rec ||
        Opc == PPC::RLDICL_32 || Opc == PPC::RLDICL_32_64 ||
        Opc == PPC::RLWINM || Opc == PPC::RLWINM_rec || Opc == PPC::RLWINM8 ||
        Opc == PPC::RLWINM8_rec;
    bool IsVFReg = (MI.getNumOperands() && MI.getOperand(0).isReg())
                       ? PPC::isVFRegister(MI.getOperand(0).getReg())
                       : false;
    if (!ConvertibleImmForm && !instrHasImmForm(Opc, IsVFReg, III, true))
      return nullptr;
 
    // Don't convert or %X, %Y, %Y since that's just a register move.
    if ((Opc == PPC::OR || Opc == PPC::OR8) &&
        MI.getOperand(1).getReg() == MI.getOperand(2).getReg())
      return nullptr;
    for (int i = 1, e = MI.getNumOperands(); i < e; i++) {
      MachineOperand &MO = MI.getOperand(i);
      SeenIntermediateUse = false;
      if (MO.isReg() && MO.isUse() && !MO.isImplicit()) {
        Register Reg = MI.getOperand(i).getReg();
        // If we see another use of this reg between the def and the MI,
        // we want to flag it so the def isn't deleted.
        MachineInstr *DefMI = getDefMIPostRA(Reg, MI, SeenIntermediateUse);
        if (DefMI) {
          // Is this register defined by some form of add-immediate (including
          // load-immediate) within this basic block?
          switch (DefMI->getOpcode()) {
          default:
            break;
          case PPC::LI:
          case PPC::LI8:
          case PPC::ADDItocL8:
          case PPC::ADDI:
          case PPC::ADDI8:
            OpNoForForwarding = i;
            return DefMI;
          }
        }
      }
    }
  }
  return OpNoForForwarding == ~0U ? nullptr : DefMI;
}
 
unsigned PPCInstrInfo::getSpillTarget() const {
  // With P10, we may need to spill paired vector registers or accumulator
  // registers. MMA implies paired vectors, so we can just check that.
  bool IsP10Variant = Subtarget.isISA3_1() || Subtarget.pairedVectorMemops();
  // P11 uses the P10 target.
  return Subtarget.isISAFuture() ? 3 : IsP10Variant ?
                                   2 : Subtarget.hasP9Vector() ?
                                   1 : 0;
}
 
ArrayRef<unsigned> PPCInstrInfo::getStoreOpcodesForSpillArray() const {
  return {StoreSpillOpcodesArray[getSpillTarget()], SOK_LastOpcodeSpill};
}
 
ArrayRef<unsigned> PPCInstrInfo::getLoadOpcodesForSpillArray() const {
  return {LoadSpillOpcodesArray[getSpillTarget()], SOK_LastOpcodeSpill};
}
 
// This opt tries to convert the following imm form to an index form to save an
// add for stack variables.
// Return false if no such pattern found.
//
// ADDI instr: ToBeChangedReg = ADDI FrameBaseReg, OffsetAddi
// ADD instr:  ToBeDeletedReg = ADD ToBeChangedReg(killed), ScaleReg
// Imm instr:  Reg            = op OffsetImm, ToBeDeletedReg(killed)
//
// can be converted to:
//
// new ADDI instr: ToBeChangedReg = ADDI FrameBaseReg, (OffsetAddi + OffsetImm)
// Index instr:    Reg            = opx ScaleReg, ToBeChangedReg(killed)
//
// In order to eliminate ADD instr, make sure that:
// 1: (OffsetAddi + OffsetImm) must be int16 since this offset will be used in
//    new ADDI instr and ADDI can only take int16 Imm.
// 2: ToBeChangedReg must be killed in ADD instr and there is no other use
//    between ADDI and ADD instr since its original def in ADDI will be changed
//    in new ADDI instr. And also there should be no new def for it between
//    ADD and Imm instr as ToBeChangedReg will be used in Index instr.
// 3: ToBeDeletedReg must be killed in Imm instr and there is no other use
//    between ADD and Imm instr since ADD instr will be eliminated.
// 4: ScaleReg must not be redefined between ADD and Imm instr since it will be
//    moved to Index instr.
bool PPCInstrInfo::foldFrameOffset(MachineInstr &MI) const {
  MachineFunction *MF = MI.getParent()->getParent();
  MachineRegisterInfo *MRI = &MF->getRegInfo();
  bool PostRA = !MRI->isSSA();
  // Do this opt after PEI which is after RA. The reason is stack slot expansion
  // in PEI may expose such opportunities since in PEI, stack slot offsets to
  // frame base(OffsetAddi) are determined.
  if (!PostRA)
    return false;
  unsigned ToBeDeletedReg = 0;
  int64_t OffsetImm = 0;
  unsigned XFormOpcode = 0;
  ImmInstrInfo III;
 
  // Check if Imm instr meets requirement.
  if (!isImmInstrEligibleForFolding(MI, ToBeDeletedReg, XFormOpcode, OffsetImm,
                                    III))
    return false;
 
  bool OtherIntermediateUse = false;
  MachineInstr *ADDMI = getDefMIPostRA(ToBeDeletedReg, MI, OtherIntermediateUse);
 
  // Exit if there is other use between ADD and Imm instr or no def found.
  if (OtherIntermediateUse || !ADDMI)
    return false;
 
  // Check if ADD instr meets requirement.
  if (!isADDInstrEligibleForFolding(*ADDMI))
    return false;
 
  unsigned ScaleRegIdx = 0;
  int64_t OffsetAddi = 0;
  MachineInstr *ADDIMI = nullptr;
 
  // Check if there is a valid ToBeChangedReg in ADDMI.
  // 1: It must be killed.
  // 2: Its definition must be a valid ADDIMI.
  // 3: It must satify int16 offset requirement.
  if (isValidToBeChangedReg(ADDMI, 1, ADDIMI, OffsetAddi, OffsetImm))
    ScaleRegIdx = 2;
  else if (isValidToBeChangedReg(ADDMI, 2, ADDIMI, OffsetAddi, OffsetImm))
    ScaleRegIdx = 1;
  else
    return false;
 
  assert(ADDIMI && "There should be ADDIMI for valid ToBeChangedReg.");
  Register ToBeChangedReg = ADDIMI->getOperand(0).getReg();
  Register ScaleReg = ADDMI->getOperand(ScaleRegIdx).getReg();
  auto NewDefFor = [&](unsigned Reg, MachineBasicBlock::iterator Start,
                       MachineBasicBlock::iterator End) {
    for (auto It = ++Start; It != End; It++)
      if (It->modifiesRegister(Reg, &getRegisterInfo()))
        return true;
    return false;
  };
 
  // We are trying to replace the ImmOpNo with ScaleReg. Give up if it is
  // treated as special zero when ScaleReg is R0/X0 register.
  if (III.ZeroIsSpecialOrig == III.ImmOpNo &&
      (ScaleReg == PPC::R0 || ScaleReg == PPC::X0))
    return false;
 
  // Make sure no other def for ToBeChangedReg and ScaleReg between ADD Instr
  // and Imm Instr.
  if (NewDefFor(ToBeChangedReg, *ADDMI, MI) || NewDefFor(ScaleReg, *ADDMI, MI))
    return false;
 
  // Now start to do the transformation.
  LLVM_DEBUG(dbgs() << "Replace instruction: "
                    << "\n");
  LLVM_DEBUG(ADDIMI->dump());
  LLVM_DEBUG(ADDMI->dump());
  LLVM_DEBUG(MI.dump());
  LLVM_DEBUG(dbgs() << "with: "
                    << "\n");
 
  // Update ADDI instr.
  ADDIMI->getOperand(2).setImm(OffsetAddi + OffsetImm);
 
  // Update Imm instr.
  MI.setDesc(get(XFormOpcode));
  MI.getOperand(III.ImmOpNo)
      .ChangeToRegister(ScaleReg, false, false,
                        ADDMI->getOperand(ScaleRegIdx).isKill());
 
  MI.getOperand(III.OpNoForForwarding)
      .ChangeToRegister(ToBeChangedReg, false, false, true);
 
  // Eliminate ADD instr.
  ADDMI->eraseFromParent();
 
  LLVM_DEBUG(ADDIMI->dump());
  LLVM_DEBUG(MI.dump());
 
  return true;
}
 
bool PPCInstrInfo::isADDIInstrEligibleForFolding(MachineInstr &ADDIMI,
                                                 int64_t &Imm) const {
  unsigned Opc = ADDIMI.getOpcode();
 
  // Exit if the instruction is not ADDI.
  if (Opc != PPC::ADDI && Opc != PPC::ADDI8)
    return false;
 
  // The operand may not necessarily be an immediate - it could be a relocation.
  if (!ADDIMI.getOperand(2).isImm())
    return false;
 
  Imm = ADDIMI.getOperand(2).getImm();
 
  return true;
}
 
bool PPCInstrInfo::isADDInstrEligibleForFolding(MachineInstr &ADDMI) const {
  unsigned Opc = ADDMI.getOpcode();
 
  // Exit if the instruction is not ADD.
  return Opc == PPC::ADD4 || Opc == PPC::ADD8;
}
 
bool PPCInstrInfo::isImmInstrEligibleForFolding(MachineInstr &MI,
                                                unsigned &ToBeDeletedReg,
                                                unsigned &XFormOpcode,
                                                int64_t &OffsetImm,
                                                ImmInstrInfo &III) const {
  // Only handle load/store.
  if (!MI.mayLoadOrStore())
    return false;
 
  unsigned Opc = MI.getOpcode();
 
  XFormOpcode = RI.getMappedIdxOpcForImmOpc(Opc);
 
  // Exit if instruction has no index form.
  if (XFormOpcode == PPC::INSTRUCTION_LIST_END)
    return false;
 
  // TODO: sync the logic between instrHasImmForm() and ImmToIdxMap.
  if (!instrHasImmForm(XFormOpcode,
                       PPC::isVFRegister(MI.getOperand(0).getReg()), III, true))
    return false;
 
  if (!III.IsSummingOperands)
    return false;
 
  MachineOperand ImmOperand = MI.getOperand(III.ImmOpNo);
  MachineOperand RegOperand = MI.getOperand(III.OpNoForForwarding);
  // Only support imm operands, not relocation slots or others.
  if (!ImmOperand.isImm())
    return false;
 
  assert(RegOperand.isReg() && "Instruction format is not right");
 
  // There are other use for ToBeDeletedReg after Imm instr, can not delete it.
  if (!RegOperand.isKill())
    return false;
 
  ToBeDeletedReg = RegOperand.getReg();
  OffsetImm = ImmOperand.getImm();
 
  return true;
}
 
bool PPCInstrInfo::isValidToBeChangedReg(MachineInstr *ADDMI, unsigned Index,
                                         MachineInstr *&ADDIMI,
                                         int64_t &OffsetAddi,
                                         int64_t OffsetImm) const {
  assert((Index == 1 || Index == 2) && "Invalid operand index for add.");
  MachineOperand &MO = ADDMI->getOperand(Index);
 
  if (!MO.isKill())
    return false;
 
  bool OtherIntermediateUse = false;
 
  ADDIMI = getDefMIPostRA(MO.getReg(), *ADDMI, OtherIntermediateUse);
  // Currently handle only one "add + Imminstr" pair case, exit if other
  // intermediate use for ToBeChangedReg found.
  // TODO: handle the cases where there are other "add + Imminstr" pairs
  // with same offset in Imminstr which is like:
  //
  // ADDI instr: ToBeChangedReg  = ADDI FrameBaseReg, OffsetAddi
  // ADD instr1: ToBeDeletedReg1 = ADD ToBeChangedReg, ScaleReg1
  // Imm instr1: Reg1            = op1 OffsetImm, ToBeDeletedReg1(killed)
  // ADD instr2: ToBeDeletedReg2 = ADD ToBeChangedReg(killed), ScaleReg2
  // Imm instr2: Reg2            = op2 OffsetImm, ToBeDeletedReg2(killed)
  //
  // can be converted to:
  //
  // new ADDI instr: ToBeChangedReg = ADDI FrameBaseReg,
  //                                       (OffsetAddi + OffsetImm)
  // Index instr1:   Reg1           = opx1 ScaleReg1, ToBeChangedReg
  // Index instr2:   Reg2           = opx2 ScaleReg2, ToBeChangedReg(killed)
 
  if (OtherIntermediateUse || !ADDIMI)
    return false;
  // Check if ADDI instr meets requirement.
  if (!isADDIInstrEligibleForFolding(*ADDIMI, OffsetAddi))
    return false;
 
  if (isInt<16>(OffsetAddi + OffsetImm))
    return true;
  return false;
}
 
// If this instruction has an immediate form and one of its operands is a
// result of a load-immediate or an add-immediate, convert it to
// the immediate form if the constant is in range.
bool PPCInstrInfo::convertToImmediateForm(MachineInstr &MI,
                                          SmallSet<Register, 4> &RegsToUpdate,
                                          MachineInstr **KilledDef) const {
  MachineFunction *MF = MI.getParent()->getParent();
  MachineRegisterInfo *MRI = &MF->getRegInfo();
  bool PostRA = !MRI->isSSA();
  bool SeenIntermediateUse = true;
  unsigned ForwardingOperand = ~0U;
  MachineInstr *DefMI = getForwardingDefMI(MI, ForwardingOperand,
                                           SeenIntermediateUse);
  if (!DefMI)
    return false;
  assert(ForwardingOperand < MI.getNumOperands() &&
         "The forwarding operand needs to be valid at this point");
  bool IsForwardingOperandKilled = MI.getOperand(ForwardingOperand).isKill();
  bool KillFwdDefMI = !SeenIntermediateUse && IsForwardingOperandKilled;
  if (KilledDef && KillFwdDefMI)
    *KilledDef = DefMI;
 
  // Conservatively add defs from DefMI and defs/uses from MI to the set of
  // registers that need their kill flags updated.
  for (const MachineOperand &MO : DefMI->operands())
    if (MO.isReg() && MO.isDef())
      RegsToUpdate.insert(MO.getReg());
  for (const MachineOperand &MO : MI.operands())
    if (MO.isReg())
      RegsToUpdate.insert(MO.getReg());
 
  // If this is a imm instruction and its register operands is produced by ADDI,
  // put the imm into imm inst directly.
  if (RI.getMappedIdxOpcForImmOpc(MI.getOpcode()) !=
          PPC::INSTRUCTION_LIST_END &&
      transformToNewImmFormFedByAdd(MI, *DefMI, ForwardingOperand))
    return true;
 
  ImmInstrInfo III;
  bool IsVFReg = MI.getOperand(0).isReg()
                     ? PPC::isVFRegister(MI.getOperand(0).getReg())
                     : false;
  bool HasImmForm = instrHasImmForm(MI.getOpcode(), IsVFReg, III, PostRA);
  // If this is a reg+reg instruction that has a reg+imm form,
  // and one of the operands is produced by an add-immediate,
  // try to convert it.
  if (HasImmForm &&
      transformToImmFormFedByAdd(MI, III, ForwardingOperand, *DefMI,
                                 KillFwdDefMI))
    return true;
 
  // If this is a reg+reg instruction that has a reg+imm form,
  // and one of the operands is produced by LI, convert it now.
  if (HasImmForm &&
      transformToImmFormFedByLI(MI, III, ForwardingOperand, *DefMI))
    return true;
 
  // If this is not a reg+reg, but the DefMI is LI/LI8, check if its user MI
  // can be simpified to LI.
  if (!HasImmForm && simplifyToLI(MI, *DefMI, ForwardingOperand, KilledDef))
    return true;
 
  return false;
}
 
bool PPCInstrInfo::combineRLWINM(MachineInstr &MI,
                                 MachineInstr **ToErase) const {
  MachineRegisterInfo *MRI = &MI.getParent()->getParent()->getRegInfo();
  Register FoldingReg = MI.getOperand(1).getReg();
  if (!FoldingReg.isVirtual())
    return false;
  MachineInstr *SrcMI = MRI->getVRegDef(FoldingReg);
  if (SrcMI->getOpcode() != PPC::RLWINM &&
      SrcMI->getOpcode() != PPC::RLWINM_rec &&
      SrcMI->getOpcode() != PPC::RLWINM8 &&
      SrcMI->getOpcode() != PPC::RLWINM8_rec)
    return false;
  assert((MI.getOperand(2).isImm() && MI.getOperand(3).isImm() &&
          MI.getOperand(4).isImm() && SrcMI->getOperand(2).isImm() &&
          SrcMI->getOperand(3).isImm() && SrcMI->getOperand(4).isImm()) &&
         "Invalid PPC::RLWINM Instruction!");
  uint64_t SHSrc = SrcMI->getOperand(2).getImm();
  uint64_t SHMI = MI.getOperand(2).getImm();
  uint64_t MBSrc = SrcMI->getOperand(3).getImm();
  uint64_t MBMI = MI.getOperand(3).getImm();
  uint64_t MESrc = SrcMI->getOperand(4).getImm();
  uint64_t MEMI = MI.getOperand(4).getImm();
 
  assert((MEMI < 32 && MESrc < 32 && MBMI < 32 && MBSrc < 32) &&
         "Invalid PPC::RLWINM Instruction!");
  // If MBMI is bigger than MEMI, we always can not get run of ones.
  // RotatedSrcMask non-wrap:
  //                 0........31|32........63
  // RotatedSrcMask:   B---E        B---E
  // MaskMI:         -----------|--E  B------
  // Result:           -----          ---      (Bad candidate)
  //
  // RotatedSrcMask wrap:
  //                 0........31|32........63
  // RotatedSrcMask: --E   B----|--E    B----
  // MaskMI:         -----------|--E  B------
  // Result:         ---   -----|---    -----  (Bad candidate)
  //
  // One special case is RotatedSrcMask is a full set mask.
  // RotatedSrcMask full:
  //                 0........31|32........63
  // RotatedSrcMask: ------EB---|-------EB---
  // MaskMI:         -----------|--E  B------
  // Result:         -----------|---  -------  (Good candidate)
 
  // Mark special case.
  bool SrcMaskFull = (MBSrc - MESrc == 1) || (MBSrc == 0 && MESrc == 31);
 
  // For other MBMI > MEMI cases, just return.
  if ((MBMI > MEMI) && !SrcMaskFull)
    return false;
 
  // Handle MBMI <= MEMI cases.
  APInt MaskMI = APInt::getBitsSetWithWrap(32, 32 - MEMI - 1, 32 - MBMI);
  // In MI, we only need low 32 bits of SrcMI, just consider about low 32
  // bit of SrcMI mask. Note that in APInt, lowerest bit is at index 0,
  // while in PowerPC ISA, lowerest bit is at index 63.
  APInt MaskSrc = APInt::getBitsSetWithWrap(32, 32 - MESrc - 1, 32 - MBSrc);
 
  APInt RotatedSrcMask = MaskSrc.rotl(SHMI);
  APInt FinalMask = RotatedSrcMask & MaskMI;
  uint32_t NewMB, NewME;
  bool Simplified = false;
 
  // If final mask is 0, MI result should be 0 too.
  if (FinalMask.isZero()) {
    bool Is64Bit =
        (MI.getOpcode() == PPC::RLWINM8 || MI.getOpcode() == PPC::RLWINM8_rec);
    Simplified = true;
    LLVM_DEBUG(dbgs() << "Replace Instr: ");
    LLVM_DEBUG(MI.dump());
 
    if (MI.getOpcode() == PPC::RLWINM || MI.getOpcode() == PPC::RLWINM8) {
      // Replace MI with "LI 0"
      MI.removeOperand(4);
      MI.removeOperand(3);
      MI.removeOperand(2);
      MI.getOperand(1).ChangeToImmediate(0);
      MI.setDesc(get(Is64Bit ? PPC::LI8 : PPC::LI));
    } else {
      // Replace MI with "ANDI_rec reg, 0"
      MI.removeOperand(4);
      MI.removeOperand(3);
      MI.getOperand(2).setImm(0);
      MI.setDesc(get(Is64Bit ? PPC::ANDI8_rec : PPC::ANDI_rec));
      MI.getOperand(1).setReg(SrcMI->getOperand(1).getReg());
      if (SrcMI->getOperand(1).isKill()) {
        MI.getOperand(1).setIsKill(true);
        SrcMI->getOperand(1).setIsKill(false);
      } else
        // About to replace MI.getOperand(1), clear its kill flag.
        MI.getOperand(1).setIsKill(false);
    }
 
    LLVM_DEBUG(dbgs() << "With: ");
    LLVM_DEBUG(MI.dump());
 
  } else if ((isRunOfOnes((unsigned)(FinalMask.getZExtValue()), NewMB, NewME) &&
              NewMB <= NewME) ||
             SrcMaskFull) {
    // Here we only handle MBMI <= MEMI case, so NewMB must be no bigger
    // than NewME. Otherwise we get a 64 bit value after folding, but MI
    // return a 32 bit value.
    Simplified = true;
    LLVM_DEBUG(dbgs() << "Converting Instr: ");
    LLVM_DEBUG(MI.dump());
 
    uint16_t NewSH = (SHSrc + SHMI) % 32;
    MI.getOperand(2).setImm(NewSH);
    // If SrcMI mask is full, no need to update MBMI and MEMI.
    if (!SrcMaskFull) {
      MI.getOperand(3).setImm(NewMB);
      MI.getOperand(4).setImm(NewME);
    }
    MI.getOperand(1).setReg(SrcMI->getOperand(1).getReg());
    if (SrcMI->getOperand(1).isKill()) {
      MI.getOperand(1).setIsKill(true);
      SrcMI->getOperand(1).setIsKill(false);
    } else
      // About to replace MI.getOperand(1), clear its kill flag.
      MI.getOperand(1).setIsKill(false);
 
    LLVM_DEBUG(dbgs() << "To: ");
    LLVM_DEBUG(MI.dump());
  }
  if (Simplified & MRI->use_nodbg_empty(FoldingReg) &&
      !SrcMI->hasImplicitDef()) {
    // If FoldingReg has no non-debug use and it has no implicit def (it
    // is not RLWINMO or RLWINM8o), it's safe to delete its def SrcMI.
    // Otherwise keep it.
    *ToErase = SrcMI;
    LLVM_DEBUG(dbgs() << "Delete dead instruction: ");
    LLVM_DEBUG(SrcMI->dump());
  }
  return Simplified;
}
 
bool PPCInstrInfo::instrHasImmForm(unsigned Opc, bool IsVFReg,
                                   ImmInstrInfo &III, bool PostRA) const {
  // The vast majority of the instructions would need their operand 2 replaced
  // with an immediate when switching to the reg+imm form. A marked exception
  // are the update form loads/stores for which a constant operand 2 would need
  // to turn into a displacement and move operand 1 to the operand 2 position.
  III.ImmOpNo = 2;
  III.OpNoForForwarding = 2;
  III.ImmWidth = 16;
  III.ImmMustBeMultipleOf = 1;
  III.TruncateImmTo = 0;
  III.IsSummingOperands = false;
  switch (Opc) {
  default: return false;
  case PPC::ADD4:
  case PPC::ADD8:
    III.SignedImm = true;
    III.ZeroIsSpecialOrig = 0;
    III.ZeroIsSpecialNew = 1;
    III.IsCommutative = true;
    III.IsSummingOperands = true;
    III.ImmOpcode = Opc == PPC::ADD4 ? PPC::ADDI : PPC::ADDI8;
    break;
  case PPC::ADDC:
  case PPC::ADDC8:
    III.SignedImm = true;
    III.ZeroIsSpecialOrig = 0;
    III.ZeroIsSpecialNew = 0;
    III.IsCommutative = true;
    III.IsSummingOperands = true;
    III.ImmOpcode = Opc == PPC::ADDC ? PPC::ADDIC : PPC::ADDIC8;
    break;
  case PPC::ADDC_rec:
    III.SignedImm = true;
    III.ZeroIsSpecialOrig = 0;
    III.ZeroIsSpecialNew = 0;
    III.IsCommutative = true;
    III.IsSummingOperands = true;
    III.ImmOpcode = PPC::ADDIC_rec;
    break;
  case PPC::SUBFC:
  case PPC::SUBFC8:
    III.SignedImm = true;
    III.ZeroIsSpecialOrig = 0;
    III.ZeroIsSpecialNew = 0;
    III.IsCommutative = false;
    III.ImmOpcode = Opc == PPC::SUBFC ? PPC::SUBFIC : PPC::SUBFIC8;
    break;
  case PPC::CMPW:
  case PPC::CMPD:
    III.SignedImm = true;
    III.ZeroIsSpecialOrig = 0;
    III.ZeroIsSpecialNew = 0;
    III.IsCommutative = false;
    III.ImmOpcode = Opc == PPC::CMPW ? PPC::CMPWI : PPC::CMPDI;
    break;
  case PPC::CMPLW:
  case PPC::CMPLD:
    III.SignedImm = false;
    III.ZeroIsSpecialOrig = 0;
    III.ZeroIsSpecialNew = 0;
    III.IsCommutative = false;
    III.ImmOpcode = Opc == PPC::CMPLW ? PPC::CMPLWI : PPC::CMPLDI;
    break;
  case PPC::AND_rec:
  case PPC::AND8_rec:
  case PPC::OR:
  case PPC::OR8:
  case PPC::XOR:
  case PPC::XOR8:
    III.SignedImm = false;
    III.ZeroIsSpecialOrig = 0;
    III.ZeroIsSpecialNew = 0;
    III.IsCommutative = true;
    switch(Opc) {
    default: llvm_unreachable("Unknown opcode");
    case PPC::AND_rec:
      III.ImmOpcode = PPC::ANDI_rec;
      break;
    case PPC::AND8_rec:
      III.ImmOpcode = PPC::ANDI8_rec;
      break;
    case PPC::OR: III.ImmOpcode = PPC::ORI; break;
    case PPC::OR8: III.ImmOpcode = PPC::ORI8; break;
    case PPC::XOR: III.ImmOpcode = PPC::XORI; break;
    case PPC::XOR8: III.ImmOpcode = PPC::XORI8; break;
    }
    break;
  case PPC::RLWNM:
  case PPC::RLWNM8:
  case PPC::RLWNM_rec:
  case PPC::RLWNM8_rec:
  case PPC::SLW:
  case PPC::SLW8:
  case PPC::SLW_rec:
  case PPC::SLW8_rec:
  case PPC::SRW:
  case PPC::SRW8:
  case PPC::SRW_rec:
  case PPC::SRW8_rec:
  case PPC::SRAW:
  case PPC::SRAW_rec:
    III.SignedImm = false;
    III.ZeroIsSpecialOrig = 0;
    III.ZeroIsSpecialNew = 0;
    III.IsCommutative = false;
    // This isn't actually true, but the instructions ignore any of the
    // upper bits, so any immediate loaded with an LI is acceptable.
    // This does not apply to shift right algebraic because a value
    // out of range will produce a -1/0.
    III.ImmWidth = 16;
    if (Opc == PPC::RLWNM || Opc == PPC::RLWNM8 || Opc == PPC::RLWNM_rec ||
        Opc == PPC::RLWNM8_rec)
      III.TruncateImmTo = 5;
    else
      III.TruncateImmTo = 6;
    switch(Opc) {
    default: llvm_unreachable("Unknown opcode");
    case PPC::RLWNM: III.ImmOpcode = PPC::RLWINM; break;
    case PPC::RLWNM8: III.ImmOpcode = PPC::RLWINM8; break;
    case PPC::RLWNM_rec:
      III.ImmOpcode = PPC::RLWINM_rec;
      break;
    case PPC::RLWNM8_rec:
      III.ImmOpcode = PPC::RLWINM8_rec;
      break;
    case PPC::SLW: III.ImmOpcode = PPC::RLWINM; break;
    case PPC::SLW8: III.ImmOpcode = PPC::RLWINM8; break;
    case PPC::SLW_rec:
      III.ImmOpcode = PPC::RLWINM_rec;
      break;
    case PPC::SLW8_rec:
      III.ImmOpcode = PPC::RLWINM8_rec;
      break;
    case PPC::SRW: III.ImmOpcode = PPC::RLWINM; break;
    case PPC::SRW8: III.ImmOpcode = PPC::RLWINM8; break;
    case PPC::SRW_rec:
      III.ImmOpcode = PPC::RLWINM_rec;
      break;
    case PPC::SRW8_rec:
      III.ImmOpcode = PPC::RLWINM8_rec;
      break;
    case PPC::SRAW:
      III.ImmWidth = 5;
      III.TruncateImmTo = 0;
      III.ImmOpcode = PPC::SRAWI;
      break;
    case PPC::SRAW_rec:
      III.ImmWidth = 5;
      III.TruncateImmTo = 0;
      III.ImmOpcode = PPC::SRAWI_rec;
      break;
    }
    break;
  case PPC::RLDCL:
  case PPC::RLDCL_rec:
  case PPC::RLDCR:
  case PPC::RLDCR_rec:
  case PPC::SLD:
  case PPC::SLD_rec:
  case PPC::SRD:
  case PPC::SRD_rec:
  case PPC::SRAD:
  case PPC::SRAD_rec:
    III.SignedImm = false;
    III.ZeroIsSpecialOrig = 0;
    III.ZeroIsSpecialNew = 0;
    III.IsCommutative = false;
    // This isn't actually true, but the instructions ignore any of the
    // upper bits, so any immediate loaded with an LI is acceptable.
    // This does not apply to shift right algebraic because a value
    // out of range will produce a -1/0.
    III.ImmWidth = 16;
    if (Opc == PPC::RLDCL || Opc == PPC::RLDCL_rec || Opc == PPC::RLDCR ||
        Opc == PPC::RLDCR_rec)
      III.TruncateImmTo = 6;
    else
      III.TruncateImmTo = 7;
    switch(Opc) {
    default: llvm_unreachable("Unknown opcode");
    case PPC::RLDCL: III.ImmOpcode = PPC::RLDICL; break;
    case PPC::RLDCL_rec:
      III.ImmOpcode = PPC::RLDICL_rec;
      break;
    case PPC::RLDCR: III.ImmOpcode = PPC::RLDICR; break;
    case PPC::RLDCR_rec:
      III.ImmOpcode = PPC::RLDICR_rec;
      break;
    case PPC::SLD: III.ImmOpcode = PPC::RLDICR; break;
    case PPC::SLD_rec:
      III.ImmOpcode = PPC::RLDICR_rec;
      break;
    case PPC::SRD: III.ImmOpcode = PPC::RLDICL; break;
    case PPC::SRD_rec:
      III.ImmOpcode = PPC::RLDICL_rec;
      break;
    case PPC::SRAD:
      III.ImmWidth = 6;
      III.TruncateImmTo = 0;
      III.ImmOpcode = PPC::SRADI;
       break;
    case PPC::SRAD_rec:
      III.ImmWidth = 6;
      III.TruncateImmTo = 0;
      III.ImmOpcode = PPC::SRADI_rec;
      break;
    }
    break;
  // Loads and stores:
  case PPC::LBZX:
  case PPC::LBZX8:
  case PPC::LHZX:
  case PPC::LHZX8:
  case PPC::LHAX:
  case PPC::LHAX8:
  case PPC::LWZX:
  case PPC::LWZX8:
  case PPC::LWAX:
  case PPC::LDX:
  case PPC::LFSX:
  case PPC::LFDX:
  case PPC::STBX:
  case PPC::STBX8:
  case PPC::STHX:
  case PPC::STHX8:
  case PPC::STWX:
  case PPC::STWX8:
  case PPC::STDX:
  case PPC::STFSX:
  case PPC::STFDX:
    III.SignedImm = true;
    III.ZeroIsSpecialOrig = 1;
    III.ZeroIsSpecialNew = 2;
    III.IsCommutative = true;
    III.IsSummingOperands = true;
    III.ImmOpNo = 1;
    III.OpNoForForwarding = 2;
    switch(Opc) {
    default: llvm_unreachable("Unknown opcode");
    case PPC::LBZX: III.ImmOpcode = PPC::LBZ; break;
    case PPC::LBZX8: III.ImmOpcode = PPC::LBZ8; break;
    case PPC::LHZX: III.ImmOpcode = PPC::LHZ; break;
    case PPC::LHZX8: III.ImmOpcode = PPC::LHZ8; break;
    case PPC::LHAX: III.ImmOpcode = PPC::LHA; break;
    case PPC::LHAX8: III.ImmOpcode = PPC::LHA8; break;
    case PPC::LWZX: III.ImmOpcode = PPC::LWZ; break;
    case PPC::LWZX8: III.ImmOpcode = PPC::LWZ8; break;
    case PPC::LWAX:
      III.ImmOpcode = PPC::LWA;
      III.ImmMustBeMultipleOf = 4;
      break;
    case PPC::LDX: III.ImmOpcode = PPC::LD; III.ImmMustBeMultipleOf = 4; break;
    case PPC::LFSX: III.ImmOpcode = PPC::LFS; break;
    case PPC::LFDX: III.ImmOpcode = PPC::LFD; break;
    case PPC::STBX: III.ImmOpcode = PPC::STB; break;
    case PPC::STBX8: III.ImmOpcode = PPC::STB8; break;
    case PPC::STHX: III.ImmOpcode = PPC::STH; break;
    case PPC::STHX8: III.ImmOpcode = PPC::STH8; break;
    case PPC::STWX: III.ImmOpcode = PPC::STW; break;
    case PPC::STWX8: III.ImmOpcode = PPC::STW8; break;
    case PPC::STDX:
      III.ImmOpcode = PPC::STD;
      III.ImmMustBeMultipleOf = 4;
      break;
    case PPC::STFSX: III.ImmOpcode = PPC::STFS; break;
    case PPC::STFDX: III.ImmOpcode = PPC::STFD; break;
    }
    break;
  case PPC::LBZUX:
  case PPC::LBZUX8:
  case PPC::LHZUX:
  case PPC::LHZUX8:
  case PPC::LHAUX:
  case PPC::LHAUX8:
  case PPC::LWZUX:
  case PPC::LWZUX8:
  case PPC::LDUX:
  case PPC::LFSUX:
  case PPC::LFDUX:
  case PPC::STBUX:
  case PPC::STBUX8:
  case PPC::STHUX:
  case PPC::STHUX8:
  case PPC::STWUX:
  case PPC::STWUX8:
  case PPC::STDUX:
  case PPC::STFSUX:
  case PPC::STFDUX:
    III.SignedImm = true;
    III.ZeroIsSpecialOrig = 2;
    III.ZeroIsSpecialNew = 3;
    III.IsCommutative = false;
    III.IsSummingOperands = true;
    III.ImmOpNo = 2;
    III.OpNoForForwarding = 3;
    switch(Opc) {
    default: llvm_unreachable("Unknown opcode");
    case PPC::LBZUX: III.ImmOpcode = PPC::LBZU; break;
    case PPC::LBZUX8: III.ImmOpcode = PPC::LBZU8; break;
    case PPC::LHZUX: III.ImmOpcode = PPC::LHZU; break;
    case PPC::LHZUX8: III.ImmOpcode = PPC::LHZU8; break;
    case PPC::LHAUX: III.ImmOpcode = PPC::LHAU; break;
    case PPC::LHAUX8: III.ImmOpcode = PPC::LHAU8; break;
    case PPC::LWZUX: III.ImmOpcode = PPC::LWZU; break;
    case PPC::LWZUX8: III.ImmOpcode = PPC::LWZU8; break;
    case PPC::LDUX:
      III.ImmOpcode = PPC::LDU;
      III.ImmMustBeMultipleOf = 4;
      break;
    case PPC::LFSUX: III.ImmOpcode = PPC::LFSU; break;
    case PPC::LFDUX: III.ImmOpcode = PPC::LFDU; break;
    case PPC::STBUX: III.ImmOpcode = PPC::STBU; break;
    case PPC::STBUX8: III.ImmOpcode = PPC::STBU8; break;
    case PPC::STHUX: III.ImmOpcode = PPC::STHU; break;
    case PPC::STHUX8: III.ImmOpcode = PPC::STHU8; break;
    case PPC::STWUX: III.ImmOpcode = PPC::STWU; break;
    case PPC::STWUX8: III.ImmOpcode = PPC::STWU8; break;
    case PPC::STDUX:
      III.ImmOpcode = PPC::STDU;
      III.ImmMustBeMultipleOf = 4;
      break;
    case PPC::STFSUX: III.ImmOpcode = PPC::STFSU; break;
    case PPC::STFDUX: III.ImmOpcode = PPC::STFDU; break;
    }
    break;
  // Power9 and up only. For some of these, the X-Form version has access to all
  // 64 VSR's whereas the D-Form only has access to the VR's. We replace those
  // with pseudo-ops pre-ra and for post-ra, we check that the register loaded
  // into or stored from is one of the VR registers.
  case PPC::LXVX:
  case PPC::LXSSPX:
  case PPC::LXSDX:
  case PPC::STXVX:
  case PPC::STXSSPX:
  case PPC::STXSDX:
  case PPC::XFLOADf32:
  case PPC::XFLOADf64:
  case PPC::XFSTOREf32:
  case PPC::XFSTOREf64:
    if (!Subtarget.hasP9Vector())
      return false;
    III.SignedImm = true;
    III.ZeroIsSpecialOrig = 1;
    III.ZeroIsSpecialNew = 2;
    III.IsCommutative = true;
    III.IsSummingOperands = true;
    III.ImmOpNo = 1;
    III.OpNoForForwarding = 2;
    III.ImmMustBeMultipleOf = 4;
    switch(Opc) {
    default: llvm_unreachable("Unknown opcode");
    case PPC::LXVX:
      III.ImmOpcode = PPC::LXV;
      III.ImmMustBeMultipleOf = 16;
      break;
    case PPC::LXSSPX:
      if (PostRA) {
        if (IsVFReg)
          III.ImmOpcode = PPC::LXSSP;
        else {
          III.ImmOpcode = PPC::LFS;
          III.ImmMustBeMultipleOf = 1;
        }
        break;
      }
      [[fallthrough]];
    case PPC::XFLOADf32:
      III.ImmOpcode = PPC::DFLOADf32;
      break;
    case PPC::LXSDX:
      if (PostRA) {
        if (IsVFReg)
          III.ImmOpcode = PPC::LXSD;
        else {
          III.ImmOpcode = PPC::LFD;
          III.ImmMustBeMultipleOf = 1;
        }
        break;
      }
      [[fallthrough]];
    case PPC::XFLOADf64:
      III.ImmOpcode = PPC::DFLOADf64;
      break;
    case PPC::STXVX:
      III.ImmOpcode = PPC::STXV;
      III.ImmMustBeMultipleOf = 16;
      break;
    case PPC::STXSSPX:
      if (PostRA) {
        if (IsVFReg)
          III.ImmOpcode = PPC::STXSSP;
        else {
          III.ImmOpcode = PPC::STFS;
          III.ImmMustBeMultipleOf = 1;
        }
        break;
      }
      [[fallthrough]];
    case PPC::XFSTOREf32:
      III.ImmOpcode = PPC::DFSTOREf32;
      break;
    case PPC::STXSDX:
      if (PostRA) {
        if (IsVFReg)
          III.ImmOpcode = PPC::STXSD;
        else {
          III.ImmOpcode = PPC::STFD;
          III.ImmMustBeMultipleOf = 1;
        }
        break;
      }
      [[fallthrough]];
    case PPC::XFSTOREf64:
      III.ImmOpcode = PPC::DFSTOREf64;
      break;
    }
    break;
  }
  return true;
}
 
// Utility function for swaping two arbitrary operands of an instruction.
static void swapMIOperands(MachineInstr &MI, unsigned Op1, unsigned Op2) {
  assert(Op1 != Op2 && "Cannot swap operand with itself.");
 
  unsigned MaxOp = std::max(Op1, Op2);
  unsigned MinOp = std::min(Op1, Op2);
  MachineOperand MOp1 = MI.getOperand(MinOp);
  MachineOperand MOp2 = MI.getOperand(MaxOp);
  MI.removeOperand(std::max(Op1, Op2));
  MI.removeOperand(std::min(Op1, Op2));
 
  // If the operands we are swapping are the two at the end (the common case)
  // we can just remove both and add them in the opposite order.
  if (MaxOp - MinOp == 1 && MI.getNumOperands() == MinOp) {
    MI.addOperand(MOp2);
    MI.addOperand(MOp1);
  } else {
    // Store all operands in a temporary vector, remove them and re-add in the
    // right order.
    SmallVector<MachineOperand, 2> MOps;
    unsigned TotalOps = MI.getNumOperands() + 2; // We've already removed 2 ops.
    for (unsigned i = MI.getNumOperands() - 1; i >= MinOp; i--) {
      MOps.push_back(MI.getOperand(i));
      MI.removeOperand(i);
    }
    // MOp2 needs to be added next.
    MI.addOperand(MOp2);
    // Now add the rest.
    for (unsigned i = MI.getNumOperands(); i < TotalOps; i++) {
      if (i == MaxOp)
        MI.addOperand(MOp1);
      else {
        MI.addOperand(MOps.back());
        MOps.pop_back();
      }
    }
  }
}
 
// Check if the 'MI' that has the index OpNoForForwarding
// meets the requirement described in the ImmInstrInfo.
bool PPCInstrInfo::isUseMIElgibleForForwarding(MachineInstr &MI,
                                               const ImmInstrInfo &III,
                                               unsigned OpNoForForwarding
                                               ) const {
  // As the algorithm of checking for PPC::ZERO/PPC::ZERO8
  // would not work pre-RA, we can only do the check post RA.
  MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
  if (MRI.isSSA())
    return false;
 
  // Cannot do the transform if MI isn't summing the operands.
  if (!III.IsSummingOperands)
    return false;
 
  // The instruction we are trying to replace must have the ZeroIsSpecialOrig set.
  if (!III.ZeroIsSpecialOrig)
    return false;
 
  // We cannot do the transform if the operand we are trying to replace
  // isn't the same as the operand the instruction allows.
  if (OpNoForForwarding != III.OpNoForForwarding)
    return false;
 
  // Check if the instruction we are trying to transform really has
  // the special zero register as its operand.
  if (MI.getOperand(III.ZeroIsSpecialOrig).getReg() != PPC::ZERO &&
      MI.getOperand(III.ZeroIsSpecialOrig).getReg() != PPC::ZERO8)
    return false;
 
  // This machine instruction is convertible if it is,
  // 1. summing the operands.
  // 2. one of the operands is special zero register.
  // 3. the operand we are trying to replace is allowed by the MI.
  return true;
}
 
// Check if the DefMI is the add inst and set the ImmMO and RegMO
// accordingly.
bool PPCInstrInfo::isDefMIElgibleForForwarding(MachineInstr &DefMI,
                                               const ImmInstrInfo &III,
                                               MachineOperand *&ImmMO,
                                               MachineOperand *&RegMO) const {
  unsigned Opc = DefMI.getOpcode();
  if (Opc != PPC::ADDItocL8 && Opc != PPC::ADDI && Opc != PPC::ADDI8)
    return false;
 
  // Skip the optimization of transformTo[NewImm|Imm]FormFedByAdd for ADDItocL8
  // on AIX which is used for toc-data access. TODO: Follow up to see if it can
  // apply for AIX toc-data as well.
  if (Opc == PPC::ADDItocL8 && Subtarget.isAIX())
    return false;
 
  assert(DefMI.getNumOperands() >= 3 &&
         "Add inst must have at least three operands");
  RegMO = &DefMI.getOperand(1);
  ImmMO = &DefMI.getOperand(2);
 
  // Before RA, ADDI first operand could be a frame index.
  if (!RegMO->isReg())
    return false;
 
  // This DefMI is elgible for forwarding if it is:
  // 1. add inst
  // 2. one of the operands is Imm/CPI/Global.
  return isAnImmediateOperand(*ImmMO);
}
 
bool PPCInstrInfo::isRegElgibleForForwarding(
    const MachineOperand &RegMO, const MachineInstr &DefMI,
    const MachineInstr &MI, bool KillDefMI,
    bool &IsFwdFeederRegKilled, bool &SeenIntermediateUse) const {
  // x = addi y, imm
  // ...
  // z = lfdx 0, x   -> z = lfd imm(y)
  // The Reg "y" can be forwarded to the MI(z) only when there is no DEF
  // of "y" between the DEF of "x" and "z".
  // The query is only valid post RA.
  const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
  if (MRI.isSSA())
    return false;
 
  Register Reg = RegMO.getReg();
 
  // Walking the inst in reverse(MI-->DefMI) to get the last DEF of the Reg.
  MachineBasicBlock::const_reverse_iterator It = MI;
  MachineBasicBlock::const_reverse_iterator E = MI.getParent()->rend();
  It++;
  for (; It != E; ++It) {
    if (It->modifiesRegister(Reg, &getRegisterInfo()) && (&*It) != &DefMI)
      return false;
    else if (It->killsRegister(Reg, &getRegisterInfo()) && (&*It) != &DefMI)
      IsFwdFeederRegKilled = true;
    if (It->readsRegister(Reg, &getRegisterInfo()) && (&*It) != &DefMI)
      SeenIntermediateUse = true;
    // Made it to DefMI without encountering a clobber.
    if ((&*It) == &DefMI)
      break;
  }
  assert((&*It) == &DefMI && "DefMI is missing");
 
  // If DefMI also defines the register to be forwarded, we can only forward it
  // if DefMI is being erased.
  if (DefMI.modifiesRegister(Reg, &getRegisterInfo()))
    return KillDefMI;
 
  return true;
}
 
bool PPCInstrInfo::isImmElgibleForForwarding(const MachineOperand &ImmMO,
                                             const MachineInstr &DefMI,
                                             const ImmInstrInfo &III,
                                             int64_t &Imm,
                                             int64_t BaseImm) const {
  assert(isAnImmediateOperand(ImmMO) && "ImmMO is NOT an immediate");
  if (DefMI.getOpcode() == PPC::ADDItocL8) {
    // The operand for ADDItocL8 is CPI, which isn't imm at compiling time,
    // However, we know that, it is 16-bit width, and has the alignment of 4.
    // Check if the instruction met the requirement.
    if (III.ImmMustBeMultipleOf > 4 ||
       III.TruncateImmTo || III.ImmWidth != 16)
      return false;
 
    // Going from XForm to DForm loads means that the displacement needs to be
    // not just an immediate but also a multiple of 4, or 16 depending on the
    // load. A DForm load cannot be represented if it is a multiple of say 2.
    // XForm loads do not have this restriction.
    if (ImmMO.isGlobal()) {
      const DataLayout &DL = ImmMO.getGlobal()->getDataLayout();
      if (ImmMO.getGlobal()->getPointerAlignment(DL) < III.ImmMustBeMultipleOf)
        return false;
    }
 
    return true;
  }
 
  if (ImmMO.isImm()) {
    // It is Imm, we need to check if the Imm fit the range.
    // Sign-extend to 64-bits.
    // DefMI may be folded with another imm form instruction, the result Imm is
    // the sum of Imm of DefMI and BaseImm which is from imm form instruction.
    APInt ActualValue(64, ImmMO.getImm() + BaseImm, true);
    if (III.SignedImm && !ActualValue.isSignedIntN(III.ImmWidth))
      return false;
    if (!III.SignedImm && !ActualValue.isIntN(III.ImmWidth))
      return false;
    Imm = SignExtend64<16>(ImmMO.getImm() + BaseImm);
 
    if (Imm % III.ImmMustBeMultipleOf)
      return false;
    if (III.TruncateImmTo)
      Imm &= ((1 << III.TruncateImmTo) - 1);
  }
  else
    return false;
 
  // This ImmMO is forwarded if it meets the requriement describle
  // in ImmInstrInfo
  return true;
}
 
bool PPCInstrInfo::simplifyToLI(MachineInstr &MI, MachineInstr &DefMI,
                                unsigned OpNoForForwarding,
                                MachineInstr **KilledDef) const {
  if ((DefMI.getOpcode() != PPC::LI && DefMI.getOpcode() != PPC::LI8) ||
      !DefMI.getOperand(1).isImm())
    return false;
 
  MachineFunction *MF = MI.getParent()->getParent();
  MachineRegisterInfo *MRI = &MF->getRegInfo();
  bool PostRA = !MRI->isSSA();
 
  int64_t Immediate = DefMI.getOperand(1).getImm();
  // Sign-extend to 64-bits.
  int64_t SExtImm = SignExtend64<16>(Immediate);
 
  bool ReplaceWithLI = false;
  bool Is64BitLI = false;
  int64_t NewImm = 0;
  bool SetCR = false;
  unsigned Opc = MI.getOpcode();
  switch (Opc) {
  default:
    return false;
 
  // FIXME: Any branches conditional on such a comparison can be made
  // unconditional. At this time, this happens too infrequently to be worth
  // the implementation effort, but if that ever changes, we could convert
  // such a pattern here.
  case PPC::CMPWI:
  case PPC::CMPLWI:
  case PPC::CMPDI:
  case PPC::CMPLDI: {
    // Doing this post-RA would require dataflow analysis to reliably find uses
    // of the CR register set by the compare.
    // No need to fixup killed/dead flag since this transformation is only valid
    // before RA.
    if (PostRA)
      return false;
    // If a compare-immediate is fed by an immediate and is itself an input of
    // an ISEL (the most common case) into a COPY of the correct register.
    bool Changed = false;
    Register DefReg = MI.getOperand(0).getReg();
    int64_t Comparand = MI.getOperand(2).getImm();
    int64_t SExtComparand = ((uint64_t)Comparand & ~0x7FFFuLL) != 0
                                ? (Comparand | 0xFFFFFFFFFFFF0000)
                                : Comparand;
 
    for (auto &CompareUseMI : MRI->use_instructions(DefReg)) {
      unsigned UseOpc = CompareUseMI.getOpcode();
      if (UseOpc != PPC::ISEL && UseOpc != PPC::ISEL8)
        continue;
      unsigned CRSubReg = CompareUseMI.getOperand(3).getSubReg();
      Register TrueReg = CompareUseMI.getOperand(1).getReg();
      Register FalseReg = CompareUseMI.getOperand(2).getReg();
      unsigned RegToCopy =
          selectReg(SExtImm, SExtComparand, Opc, TrueReg, FalseReg, CRSubReg);
      if (RegToCopy == PPC::NoRegister)
        continue;
      // Can't use PPC::COPY to copy PPC::ZERO[8]. Convert it to LI[8] 0.
      if (RegToCopy == PPC::ZERO || RegToCopy == PPC::ZERO8) {
        CompareUseMI.setDesc(get(UseOpc == PPC::ISEL8 ? PPC::LI8 : PPC::LI));
        replaceInstrOperandWithImm(CompareUseMI, 1, 0);
        CompareUseMI.removeOperand(3);
        CompareUseMI.removeOperand(2);
        continue;
      }
      LLVM_DEBUG(
          dbgs() << "Found LI -> CMPI -> ISEL, replacing with a copy.\n");
      LLVM_DEBUG(DefMI.dump(); MI.dump(); CompareUseMI.dump());
      LLVM_DEBUG(dbgs() << "Is converted to:\n");
      // Convert to copy and remove unneeded operands.
      CompareUseMI.setDesc(get(PPC::COPY));
      CompareUseMI.removeOperand(3);
      CompareUseMI.removeOperand(RegToCopy == TrueReg ? 2 : 1);
      CmpIselsConverted++;
      Changed = true;
      LLVM_DEBUG(CompareUseMI.dump());
    }
    if (Changed)
      return true;
    // This may end up incremented multiple times since this function is called
    // during a fixed-point transformation, but it is only meant to indicate the
    // presence of this opportunity.
    MissedConvertibleImmediateInstrs++;
    return false;
  }
 
  // Immediate forms - may simply be convertable to an LI.
  case PPC::ADDI:
  case PPC::ADDI8: {
    // Does the sum fit in a 16-bit signed field?
    int64_t Addend = MI.getOperand(2).getImm();
    if (isInt<16>(Addend + SExtImm)) {
      ReplaceWithLI = true;
      Is64BitLI = Opc == PPC::ADDI8;
      NewImm = Addend + SExtImm;
      break;
    }
    return false;
  }
  case PPC::SUBFIC:
  case PPC::SUBFIC8: {
    // Only transform this if the CARRY implicit operand is dead.
    if (MI.getNumOperands() > 3 && !MI.getOperand(3).isDead())
      return false;
    int64_t Minuend = MI.getOperand(2).getImm();
    if (isInt<16>(Minuend - SExtImm)) {
      ReplaceWithLI = true;
      Is64BitLI = Opc == PPC::SUBFIC8;
      NewImm = Minuend - SExtImm;
      break;
    }
    return false;
  }
  case PPC::RLDICL:
  case PPC::RLDICL_rec:
  case PPC::RLDICL_32:
  case PPC::RLDICL_32_64: {
    // Use APInt's rotate function.
    int64_t SH = MI.getOperand(2).getImm();
    int64_t MB = MI.getOperand(3).getImm();
    APInt InVal((Opc == PPC::RLDICL || Opc == PPC::RLDICL_rec) ? 64 : 32,
                SExtImm, true);
    InVal = InVal.rotl(SH);
    uint64_t Mask = MB == 0 ? -1LLU : (1LLU << (63 - MB + 1)) - 1;
    InVal &= Mask;
    // Can't replace negative values with an LI as that will sign-extend
    // and not clear the left bits. If we're setting the CR bit, we will use
    // ANDI_rec which won't sign extend, so that's safe.
    if (isUInt<15>(InVal.getSExtValue()) ||
        (Opc == PPC::RLDICL_rec && isUInt<16>(InVal.getSExtValue()))) {
      ReplaceWithLI = true;
      Is64BitLI = Opc != PPC::RLDICL_32;
      NewImm = InVal.getSExtValue();
      SetCR = Opc == PPC::RLDICL_rec;
      break;
    }
    return false;
  }
  case PPC::RLWINM:
  case PPC::RLWINM8:
  case PPC::RLWINM_rec:
  case PPC::RLWINM8_rec: {
    int64_t SH = MI.getOperand(2).getImm();
    int64_t MB = MI.getOperand(3).getImm();
    int64_t ME = MI.getOperand(4).getImm();
    APInt InVal(32, SExtImm, true);
    InVal = InVal.rotl(SH);
    APInt Mask = APInt::getBitsSetWithWrap(32, 32 - ME - 1, 32 - MB);
    InVal &= Mask;
    // Can't replace negative values with an LI as that will sign-extend
    // and not clear the left bits. If we're setting the CR bit, we will use
    // ANDI_rec which won't sign extend, so that's safe.
    bool ValueFits = isUInt<15>(InVal.getSExtValue());
    ValueFits |= ((Opc == PPC::RLWINM_rec || Opc == PPC::RLWINM8_rec) &&
                  isUInt<16>(InVal.getSExtValue()));
    if (ValueFits) {
      ReplaceWithLI = true;
      Is64BitLI = Opc == PPC::RLWINM8 || Opc == PPC::RLWINM8_rec;
      NewImm = InVal.getSExtValue();
      SetCR = Opc == PPC::RLWINM_rec || Opc == PPC::RLWINM8_rec;
      break;
    }
    return false;
  }
  case PPC::ORI: