/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp

Bug Summary

File:	llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
Warning:	line 900, column 7 6th function call argument is an uninitialized value

Annotated Source Code

Press '?' to see keyboard shortcuts

Show analyzer invocation

clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name AArch64InstructionSelector.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -fhalf-no-semantic-interposition -mframe-pointer=none -fmath-errno -fno-rounding-math -mconstructor-aliases -munwind-tables -target-cpu x86-64 -tune-cpu generic -fno-split-dwarf-inlining -debugger-tuning=gdb -ffunction-sections -fdata-sections -resource-dir /usr/lib/llvm-12/lib/clang/12.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/build-llvm/lib/Target/AArch64 -I /build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64 -I /build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/build-llvm/include -I /build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/include -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0/backward -internal-isystem /usr/local/include -internal-isystem /usr/lib/llvm-12/lib/clang/12.0.0/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-comment -std=c++14 -fdeprecated-macro -fdebug-compilation-dir /build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/build-llvm/lib/Target/AArch64 -fdebug-prefix-map=/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf=. -ferror-limit 19 -fvisibility hidden -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -o /tmp/scan-build-2021-01-20-132452-38772-1 -x c++ /build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp

/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp

→

1//===- AArch64InstructionSelector.cpp ----------------------------*- C++ -*-==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file implements the targeting of the InstructionSelector class for
10/// AArch64.
11/// \todo This should be generated by TableGen.
12//===----------------------------------------------------------------------===//

14#include "AArch64InstrInfo.h"
15#include "AArch64MachineFunctionInfo.h"
16#include "AArch64RegisterBankInfo.h"
17#include "AArch64RegisterInfo.h"
18#include "AArch64Subtarget.h"
19#include "AArch64TargetMachine.h"
20#include "MCTargetDesc/AArch64AddressingModes.h"
21#include "MCTargetDesc/AArch64MCTargetDesc.h"
22#include "llvm/ADT/Optional.h"
23#include "llvm/CodeGen/GlobalISel/InstructionSelector.h"
24#include "llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h"
25#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
26#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
27#include "llvm/CodeGen/GlobalISel/Utils.h"
28#include "llvm/CodeGen/MachineBasicBlock.h"
29#include "llvm/CodeGen/MachineConstantPool.h"
30#include "llvm/CodeGen/MachineFunction.h"
31#include "llvm/CodeGen/MachineInstr.h"
32#include "llvm/CodeGen/MachineInstrBuilder.h"
33#include "llvm/CodeGen/MachineOperand.h"
34#include "llvm/CodeGen/MachineRegisterInfo.h"
35#include "llvm/CodeGen/TargetOpcodes.h"
36#include "llvm/IR/Constants.h"
37#include "llvm/IR/Instructions.h"
38#include "llvm/IR/PatternMatch.h"
39#include "llvm/IR/Type.h"
40#include "llvm/IR/IntrinsicsAArch64.h"
41#include "llvm/Pass.h"
42#include "llvm/Support/Debug.h"
43#include "llvm/Support/raw_ostream.h"

45#define DEBUG_TYPE"aarch64-isel" "aarch64-isel"

47using namespace llvm;
48using namespace MIPatternMatch;

50namespace {

52#define GET_GLOBALISEL_PREDICATE_BITSET
53#include "AArch64GenGlobalISel.inc"
54#undef GET_GLOBALISEL_PREDICATE_BITSET

56class AArch64InstructionSelector : public InstructionSelector {
57public:
AArch64InstructionSelector(const AArch64TargetMachine &TM,
                           const AArch64Subtarget &STI,
                           const AArch64RegisterBankInfo &RBI);

bool select(MachineInstr &I) override;
static const char *getName() { return DEBUG_TYPE"aarch64-isel"; }

void setupMF(MachineFunction &MF, GISelKnownBits &KB,
             CodeGenCoverage &CoverageInfo) override {
  InstructionSelector::setupMF(MF, KB, CoverageInfo);

  // hasFnAttribute() is expensive to call on every BRCOND selection, so
  // cache it here for each run of the selector.
  ProduceNonFlagSettingCondBr =
      !MF.getFunction().hasFnAttribute(Attribute::SpeculativeLoadHardening);
  MFReturnAddr = Register();

  processPHIs(MF);
}

78private:
/// tblgen-erated 'select' implementation, used as the initial selector for
/// the patterns that don't require complex C++.
bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const;

// A lowering phase that runs before any selection attempts.
// Returns true if the instruction was modified.
bool preISelLower(MachineInstr &I);

// An early selection function that runs before the selectImpl() call.
bool earlySelect(MachineInstr &I) const;

// Do some preprocessing of G_PHIs before we begin selection.
void processPHIs(MachineFunction &MF);

bool earlySelectSHL(MachineInstr &I, MachineRegisterInfo &MRI) const;

/// Eliminate same-sized cross-bank copies into stores before selectImpl().
bool contractCrossBankCopyIntoStore(MachineInstr &I,
                                    MachineRegisterInfo &MRI);

bool convertPtrAddToAdd(MachineInstr &I, MachineRegisterInfo &MRI);

bool selectVaStartAAPCS(MachineInstr &I, MachineFunction &MF,
                        MachineRegisterInfo &MRI) const;
bool selectVaStartDarwin(MachineInstr &I, MachineFunction &MF,
                         MachineRegisterInfo &MRI) const;

///@{
/// Helper functions for selectCompareBranch.
bool selectCompareBranchFedByFCmp(MachineInstr &I, MachineInstr &FCmp,
                                  MachineIRBuilder &MIB) const;
bool selectCompareBranchFedByICmp(MachineInstr &I, MachineInstr &ICmp,
                                  MachineIRBuilder &MIB) const;
bool tryOptCompareBranchFedByICmp(MachineInstr &I, MachineInstr &ICmp,
                                  MachineIRBuilder &MIB) const;
bool tryOptAndIntoCompareBranch(MachineInstr &AndInst, bool Invert,
                                MachineBasicBlock *DstMBB,
                                MachineIRBuilder &MIB) const;
///@}

bool selectCompareBranch(MachineInstr &I, MachineFunction &MF,
                         MachineRegisterInfo &MRI) const;

bool selectVectorAshrLshr(MachineInstr &I, MachineRegisterInfo &MRI) const;
bool selectVectorSHL(MachineInstr &I, MachineRegisterInfo &MRI) const;

// Helper to generate an equivalent of scalar_to_vector into a new register,
// returned via 'Dst'.
MachineInstr *emitScalarToVector(unsigned EltSize,
                                 const TargetRegisterClass *DstRC,
                                 Register Scalar,
                                 MachineIRBuilder &MIRBuilder) const;

/// Emit a lane insert into \p DstReg, or a new vector register if None is
/// provided.
///
/// The lane inserted into is defined by \p LaneIdx. The vector source
/// register is given by \p SrcReg. The register containing the element is
/// given by \p EltReg.
MachineInstr *emitLaneInsert(Optional<Register> DstReg, Register SrcReg,
                             Register EltReg, unsigned LaneIdx,
                             const RegisterBank &RB,
                             MachineIRBuilder &MIRBuilder) const;
bool selectInsertElt(MachineInstr &I, MachineRegisterInfo &MRI) const;
bool tryOptConstantBuildVec(MachineInstr &MI, LLT DstTy,
                            MachineRegisterInfo &MRI) const;
bool selectBuildVector(MachineInstr &I, MachineRegisterInfo &MRI) const;
bool selectMergeValues(MachineInstr &I, MachineRegisterInfo &MRI) const;
bool selectUnmergeValues(MachineInstr &I, MachineRegisterInfo &MRI) const;

bool selectShuffleVector(MachineInstr &I, MachineRegisterInfo &MRI) const;
bool selectExtractElt(MachineInstr &I, MachineRegisterInfo &MRI) const;
bool selectConcatVectors(MachineInstr &I, MachineRegisterInfo &MRI) const;
bool selectSplitVectorUnmerge(MachineInstr &I,
                              MachineRegisterInfo &MRI) const;
bool selectIntrinsicWithSideEffects(MachineInstr &I,
                                    MachineRegisterInfo &MRI) const;
bool selectIntrinsic(MachineInstr &I, MachineRegisterInfo &MRI);
bool selectVectorICmp(MachineInstr &I, MachineRegisterInfo &MRI) const;
bool selectIntrinsicTrunc(MachineInstr &I, MachineRegisterInfo &MRI) const;
bool selectIntrinsicRound(MachineInstr &I, MachineRegisterInfo &MRI) const;
bool selectJumpTable(MachineInstr &I, MachineRegisterInfo &MRI) const;
bool selectBrJT(MachineInstr &I, MachineRegisterInfo &MRI) const;
bool selectTLSGlobalValue(MachineInstr &I, MachineRegisterInfo &MRI) const;
bool selectReduction(MachineInstr &I, MachineRegisterInfo &MRI) const;

unsigned emitConstantPoolEntry(const Constant *CPVal,
                               MachineFunction &MF) const;
MachineInstr *emitLoadFromConstantPool(const Constant *CPVal,
                                       MachineIRBuilder &MIRBuilder) const;

// Emit a vector concat operation.
MachineInstr *emitVectorConcat(Optional<Register> Dst, Register Op1,
                               Register Op2,
                               MachineIRBuilder &MIRBuilder) const;

// Emit an integer compare between LHS and RHS, which checks for Predicate.
MachineInstr *emitIntegerCompare(MachineOperand &LHS, MachineOperand &RHS,
                                 MachineOperand &Predicate,
                                 MachineIRBuilder &MIRBuilder) const;

/// Emit a floating point comparison between \p LHS and \p RHS.
/// \p Pred if given is the intended predicate to use.
MachineInstr *emitFPCompare(Register LHS, Register RHS,
                            MachineIRBuilder &MIRBuilder,
                            Optional<CmpInst::Predicate> = None) const;

MachineInstr *emitInstr(unsigned Opcode,
                        std::initializer_list<llvm::DstOp> DstOps,
                        std::initializer_list<llvm::SrcOp> SrcOps,
                        MachineIRBuilder &MIRBuilder,
                        const ComplexRendererFns &RenderFns = None) const;
/// Helper function to emit an add or sub instruction.
///
/// \p AddrModeAndSizeToOpcode must contain each of the opcode variants above
/// in a specific order.
///
/// Below is an example of the expected input to \p AddrModeAndSizeToOpcode.
///
/// \code
///   const std::array<std::array<unsigned, 2>, 4> Table {
///    {{AArch64::ADDXri, AArch64::ADDWri},
///     {AArch64::ADDXrs, AArch64::ADDWrs},
///     {AArch64::ADDXrr, AArch64::ADDWrr},
///     {AArch64::SUBXri, AArch64::SUBWri},
///     {AArch64::ADDXrx, AArch64::ADDWrx}}};
/// \endcode
///
/// Each row in the table corresponds to a different addressing mode. Each
/// column corresponds to a different register size.
///
/// \attention Rows must be structured as follows:
///   - Row 0: The ri opcode variants
///   - Row 1: The rs opcode variants
///   - Row 2: The rr opcode variants
///   - Row 3: The ri opcode variants for negative immediates
///   - Row 4: The rx opcode variants
///
/// \attention Columns must be structured as follows:
///   - Column 0: The 64-bit opcode variants
///   - Column 1: The 32-bit opcode variants
///
/// \p Dst is the destination register of the binop to emit.
/// \p LHS is the left-hand operand of the binop to emit.
/// \p RHS is the right-hand operand of the binop to emit.
MachineInstr *emitAddSub(
    const std::array<std::array<unsigned, 2>, 5> &AddrModeAndSizeToOpcode,
    Register Dst, MachineOperand &LHS, MachineOperand &RHS,
    MachineIRBuilder &MIRBuilder) const;
MachineInstr *emitADD(Register DefReg, MachineOperand &LHS,
                      MachineOperand &RHS,
                      MachineIRBuilder &MIRBuilder) const;
MachineInstr *emitADDS(Register Dst, MachineOperand &LHS, MachineOperand &RHS,
                       MachineIRBuilder &MIRBuilder) const;
MachineInstr *emitSUBS(Register Dst, MachineOperand &LHS, MachineOperand &RHS,
                       MachineIRBuilder &MIRBuilder) const;
MachineInstr *emitCMN(MachineOperand &LHS, MachineOperand &RHS,
                      MachineIRBuilder &MIRBuilder) const;
MachineInstr *emitTST(MachineOperand &LHS, MachineOperand &RHS,
                      MachineIRBuilder &MIRBuilder) const;
MachineInstr *emitSelect(Register Dst, Register LHS, Register RHS,
                         AArch64CC::CondCode CC,
                         MachineIRBuilder &MIRBuilder) const;
MachineInstr *emitExtractVectorElt(Optional<Register> DstReg,
                                   const RegisterBank &DstRB, LLT ScalarTy,
                                   Register VecReg, unsigned LaneIdx,
                                   MachineIRBuilder &MIRBuilder) const;

/// Helper function for selecting G_FCONSTANT. If the G_FCONSTANT can be
/// materialized using a FMOV instruction, then update MI and return it.
/// Otherwise, do nothing and return a nullptr.
MachineInstr *emitFMovForFConstant(MachineInstr &MI,
                                   MachineRegisterInfo &MRI) const;

/// Emit a CSet for an integer compare.
///
/// \p DefReg is expected to be a 32-bit scalar register.
MachineInstr *emitCSetForICMP(Register DefReg, unsigned Pred,
                              MachineIRBuilder &MIRBuilder) const;
/// Emit a CSet for a FP compare.
///
/// \p Dst is expected to be a 32-bit scalar register.
MachineInstr *emitCSetForFCmp(Register Dst, CmpInst::Predicate Pred,
                              MachineIRBuilder &MIRBuilder) const;

/// Emit the overflow op for \p Opcode.
///
/// \p Opcode is expected to be an overflow op's opcode, e.g. G_UADDO,
/// G_USUBO, etc.
std::pair<MachineInstr *, AArch64CC::CondCode>
emitOverflowOp(unsigned Opcode, Register Dst, MachineOperand &LHS,
               MachineOperand &RHS, MachineIRBuilder &MIRBuilder) const;

/// Emit a TB(N)Z instruction which tests \p Bit in \p TestReg.
/// \p IsNegative is true if the test should be "not zero".
/// This will also optimize the test bit instruction when possible.
MachineInstr *emitTestBit(Register TestReg, uint64_t Bit, bool IsNegative,
                          MachineBasicBlock *DstMBB,
                          MachineIRBuilder &MIB) const;

/// Emit a CB(N)Z instruction which branches to \p DestMBB.
MachineInstr *emitCBZ(Register CompareReg, bool IsNegative,
                      MachineBasicBlock *DestMBB,
                      MachineIRBuilder &MIB) const;

// Equivalent to the i32shift_a and friends from AArch64InstrInfo.td.
// We use these manually instead of using the importer since it doesn't
// support SDNodeXForm.
ComplexRendererFns selectShiftA_32(const MachineOperand &Root) const;
ComplexRendererFns selectShiftB_32(const MachineOperand &Root) const;
ComplexRendererFns selectShiftA_64(const MachineOperand &Root) const;
ComplexRendererFns selectShiftB_64(const MachineOperand &Root) const;

ComplexRendererFns select12BitValueWithLeftShift(uint64_t Immed) const;
ComplexRendererFns selectArithImmed(MachineOperand &Root) const;
ComplexRendererFns selectNegArithImmed(MachineOperand &Root) const;

ComplexRendererFns selectAddrModeUnscaled(MachineOperand &Root,
                                          unsigned Size) const;

ComplexRendererFns selectAddrModeUnscaled8(MachineOperand &Root) const {
  return selectAddrModeUnscaled(Root, 1);
}
ComplexRendererFns selectAddrModeUnscaled16(MachineOperand &Root) const {
  return selectAddrModeUnscaled(Root, 2);
}
ComplexRendererFns selectAddrModeUnscaled32(MachineOperand &Root) const {
  return selectAddrModeUnscaled(Root, 4);
}
ComplexRendererFns selectAddrModeUnscaled64(MachineOperand &Root) const {
  return selectAddrModeUnscaled(Root, 8);
}
ComplexRendererFns selectAddrModeUnscaled128(MachineOperand &Root) const {
  return selectAddrModeUnscaled(Root, 16);
}

/// Helper to try to fold in a GISEL_ADD_LOW into an immediate, to be used
/// from complex pattern matchers like selectAddrModeIndexed().
ComplexRendererFns tryFoldAddLowIntoImm(MachineInstr &RootDef, unsigned Size,
                                        MachineRegisterInfo &MRI) const;

ComplexRendererFns selectAddrModeIndexed(MachineOperand &Root,
                                         unsigned Size) const;
template <int Width>
ComplexRendererFns selectAddrModeIndexed(MachineOperand &Root) const {
  return selectAddrModeIndexed(Root, Width / 8);
}

bool isWorthFoldingIntoExtendedReg(MachineInstr &MI,
                                   const MachineRegisterInfo &MRI) const;
ComplexRendererFns
selectAddrModeShiftedExtendXReg(MachineOperand &Root,
                                unsigned SizeInBytes) const;

/// Returns a \p ComplexRendererFns which contains a base, offset, and whether
/// or not a shift + extend should be folded into an addressing mode. Returns
/// None when this is not profitable or possible.
ComplexRendererFns
selectExtendedSHL(MachineOperand &Root, MachineOperand &Base,
                  MachineOperand &Offset, unsigned SizeInBytes,
                  bool WantsExt) const;
ComplexRendererFns selectAddrModeRegisterOffset(MachineOperand &Root) const;
ComplexRendererFns selectAddrModeXRO(MachineOperand &Root,
                                     unsigned SizeInBytes) const;
template <int Width>
ComplexRendererFns selectAddrModeXRO(MachineOperand &Root) const {
  return selectAddrModeXRO(Root, Width / 8);
}

ComplexRendererFns selectAddrModeWRO(MachineOperand &Root,
                                     unsigned SizeInBytes) const;
template <int Width>
ComplexRendererFns selectAddrModeWRO(MachineOperand &Root) const {
  return selectAddrModeWRO(Root, Width / 8);
}

ComplexRendererFns selectShiftedRegister(MachineOperand &Root) const;

ComplexRendererFns selectArithShiftedRegister(MachineOperand &Root) const {
  return selectShiftedRegister(Root);
}

ComplexRendererFns selectLogicalShiftedRegister(MachineOperand &Root) const {
  // TODO: selectShiftedRegister should allow for rotates on logical shifts.
  // For now, make them the same. The only difference between the two is that
  // logical shifts are allowed to fold in rotates. Otherwise, these are
  // functionally the same.
  return selectShiftedRegister(Root);
}

/// Given an extend instruction, determine the correct shift-extend type for
/// that instruction.
///
/// If the instruction is going to be used in a load or store, pass
/// \p IsLoadStore = true.
AArch64_AM::ShiftExtendType
getExtendTypeForInst(MachineInstr &MI, MachineRegisterInfo &MRI,
                     bool IsLoadStore = false) const;

/// Move \p Reg to \p RC if \p Reg is not already on \p RC.
///
/// \returns Either \p Reg if no change was necessary, or the new register
/// created by moving \p Reg.
///
/// Note: This uses emitCopy right now.
Register moveScalarRegClass(Register Reg, const TargetRegisterClass &RC,
                            MachineIRBuilder &MIB) const;

ComplexRendererFns selectArithExtendedRegister(MachineOperand &Root) const;

void renderTruncImm(MachineInstrBuilder &MIB, const MachineInstr &MI,
                    int OpIdx = -1) const;
void renderLogicalImm32(MachineInstrBuilder &MIB, const MachineInstr &I,
                        int OpIdx = -1) const;
void renderLogicalImm64(MachineInstrBuilder &MIB, const MachineInstr &I,
                        int OpIdx = -1) const;

// Materialize a GlobalValue or BlockAddress using a movz+movk sequence.
void materializeLargeCMVal(MachineInstr &I, const Value *V,
                           unsigned OpFlags) const;

// Optimization methods.
bool tryOptSelect(MachineInstr &MI) const;
MachineInstr *tryFoldIntegerCompare(MachineOperand &LHS, MachineOperand &RHS,
                                    MachineOperand &Predicate,
                                    MachineIRBuilder &MIRBuilder) const;

/// Return true if \p MI is a load or store of \p NumBytes bytes.
bool isLoadStoreOfNumBytes(const MachineInstr &MI, unsigned NumBytes) const;

/// Returns true if \p MI is guaranteed to have the high-half of a 64-bit
/// register zeroed out. In other words, the result of MI has been explicitly
/// zero extended.
bool isDef32(const MachineInstr &MI) const;

const AArch64TargetMachine &TM;
const AArch64Subtarget &STI;
const AArch64InstrInfo &TII;
const AArch64RegisterInfo &TRI;
const AArch64RegisterBankInfo &RBI;

bool ProduceNonFlagSettingCondBr = false;

// Some cached values used during selection.
// We use LR as a live-in register, and we keep track of it here as it can be
// clobbered by calls.
Register MFReturnAddr;

427#define GET_GLOBALISEL_PREDICATES_DECL
428#include "AArch64GenGlobalISel.inc"
429#undef GET_GLOBALISEL_PREDICATES_DECL

431// We declare the temporaries used by selectImpl() in the class to minimize the
432// cost of constructing placeholder values.
433#define GET_GLOBALISEL_TEMPORARIES_DECL
434#include "AArch64GenGlobalISel.inc"
435#undef GET_GLOBALISEL_TEMPORARIES_DECL
436};

438} // end anonymous namespace

440#define GET_GLOBALISEL_IMPL
441#include "AArch64GenGlobalISel.inc"
442#undef GET_GLOBALISEL_IMPL

444AArch64InstructionSelector::AArch64InstructionSelector(
  const AArch64TargetMachine &TM, const AArch64Subtarget &STI,
  const AArch64RegisterBankInfo &RBI)
  : InstructionSelector(), TM(TM), STI(STI), TII(*STI.getInstrInfo()),
    TRI(*STI.getRegisterInfo()), RBI(RBI),
449#define GET_GLOBALISEL_PREDICATES_INIT
450#include "AArch64GenGlobalISel.inc"
451#undef GET_GLOBALISEL_PREDICATES_INIT
452#define GET_GLOBALISEL_TEMPORARIES_INIT
453#include "AArch64GenGlobalISel.inc"
454#undef GET_GLOBALISEL_TEMPORARIES_INIT
455{
456}

458// FIXME: This should be target-independent, inferred from the types declared
459// for each class in the bank.
460static const TargetRegisterClass *
461getRegClassForTypeOnBank(LLT Ty, const RegisterBank &RB,
                       const RegisterBankInfo &RBI,
                       bool GetAllRegSet = false) {
if (RB.getID() == AArch64::GPRRegBankID) {
  if (Ty.getSizeInBits() <= 32)
    return GetAllRegSet ? &AArch64::GPR32allRegClass
                        : &AArch64::GPR32RegClass;
  if (Ty.getSizeInBits() == 64)
    return GetAllRegSet ? &AArch64::GPR64allRegClass
                        : &AArch64::GPR64RegClass;
  return nullptr;
}

if (RB.getID() == AArch64::FPRRegBankID) {
  if (Ty.getSizeInBits() <= 16)
    return &AArch64::FPR16RegClass;
  if (Ty.getSizeInBits() == 32)
    return &AArch64::FPR32RegClass;
  if (Ty.getSizeInBits() == 64)
    return &AArch64::FPR64RegClass;
  if (Ty.getSizeInBits() == 128)
    return &AArch64::FPR128RegClass;
  return nullptr;
}

return nullptr;
487}

489/// Given a register bank, and size in bits, return the smallest register class
490/// that can represent that combination.
491static const TargetRegisterClass *
492getMinClassForRegBank(const RegisterBank &RB, unsigned SizeInBits,
                    bool GetAllRegSet = false) {
unsigned RegBankID = RB.getID();

if (RegBankID == AArch64::GPRRegBankID) {
  if (SizeInBits <= 32)
    return GetAllRegSet ? &AArch64::GPR32allRegClass
                        : &AArch64::GPR32RegClass;
  if (SizeInBits == 64)
    return GetAllRegSet ? &AArch64::GPR64allRegClass
                        : &AArch64::GPR64RegClass;
}

if (RegBankID == AArch64::FPRRegBankID) {
  switch (SizeInBits) {
  default:
    return nullptr;
  case 8:
    return &AArch64::FPR8RegClass;
  case 16:
    return &AArch64::FPR16RegClass;
  case 32:
    return &AArch64::FPR32RegClass;
  case 64:
    return &AArch64::FPR64RegClass;
  case 128:
    return &AArch64::FPR128RegClass;
  }
}

return nullptr;
523}

525/// Returns the correct subregister to use for a given register class.
526static bool getSubRegForClass(const TargetRegisterClass *RC,
                            const TargetRegisterInfo &TRI, unsigned &SubReg) {
switch (TRI.getRegSizeInBits(*RC)) {
55
←
Control jumps to the 'default' case at line 544→
case 8:
  SubReg = AArch64::bsub;
  break;
case 16:
  SubReg = AArch64::hsub;
  break;
case 32:
  if (RC != &AArch64::FPR32RegClass)
    SubReg = AArch64::sub_32;
  else
    SubReg = AArch64::ssub;
  break;
case 64:
  SubReg = AArch64::dsub;
  break;
default:
  LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Couldn't find appropriate subregister for register class."
; } } while (false)
56
←
Assuming 'DebugFlag' is false→
57
←
Loop condition is false.  Exiting loop→
      dbgs() << "Couldn't find appropriate subregister for register class.")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Couldn't find appropriate subregister for register class."
; } } while (false);
  return false;
58
←
Returning without writing to 'SubReg'→
}

return true;
551}

553/// Returns the minimum size the given register bank can hold.
554static unsigned getMinSizeForRegBank(const RegisterBank &RB) {
switch (RB.getID()) {
case AArch64::GPRRegBankID:
  return 32;
case AArch64::FPRRegBankID:
  return 8;
default:
  llvm_unreachable("Tried to get minimum size for unknown register bank.")::llvm::llvm_unreachable_internal("Tried to get minimum size for unknown register bank."
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 561);
}
563}

565static Optional<uint64_t> getImmedFromMO(const MachineOperand &Root) {
auto &MI = *Root.getParent();
auto &MBB = *MI.getParent();
auto &MF = *MBB.getParent();
auto &MRI = MF.getRegInfo();
uint64_t Immed;
if (Root.isImm())
  Immed = Root.getImm();
else if (Root.isCImm())
  Immed = Root.getCImm()->getZExtValue();
else if (Root.isReg()) {
  auto ValAndVReg =
      getConstantVRegValWithLookThrough(Root.getReg(), MRI, true);
  if (!ValAndVReg)
    return None;
  Immed = ValAndVReg->Value.getSExtValue();
} else
  return None;
return Immed;
584}

586/// Check whether \p I is a currently unsupported binary operation:
587/// - it has an unsized type
588/// - an operand is not a vreg
589/// - all operands are not in the same bank
590/// These are checks that should someday live in the verifier, but right now,
591/// these are mostly limitations of the aarch64 selector.
592static bool unsupportedBinOp(const MachineInstr &I,
                           const AArch64RegisterBankInfo &RBI,
                           const MachineRegisterInfo &MRI,
                           const AArch64RegisterInfo &TRI) {
LLT Ty = MRI.getType(I.getOperand(0).getReg());
if (!Ty.isValid()) {
  LLVM_DEBUG(dbgs() << "Generic binop register should be typed\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Generic binop register should be typed\n"
; } } while (false);
  return true;
}

const RegisterBank *PrevOpBank = nullptr;
for (auto &MO : I.operands()) {
  // FIXME: Support non-register operands.
  if (!MO.isReg()) {
    LLVM_DEBUG(dbgs() << "Generic inst non-reg operands are unsupported\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Generic inst non-reg operands are unsupported\n"
; } } while (false);
    return true;
  }

  // FIXME: Can generic operations have physical registers operands? If
  // so, this will need to be taught about that, and we'll need to get the
  // bank out of the minimal class for the register.
  // Either way, this needs to be documented (and possibly verified).
  if (!Register::isVirtualRegister(MO.getReg())) {
    LLVM_DEBUG(dbgs() << "Generic inst has physical register operand\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Generic inst has physical register operand\n"
; } } while (false);
    return true;
  }

  const RegisterBank *OpBank = RBI.getRegBank(MO.getReg(), MRI, TRI);
  if (!OpBank) {
    LLVM_DEBUG(dbgs() << "Generic register has no bank or class\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Generic register has no bank or class\n"
; } } while (false);
    return true;
  }

  if (PrevOpBank && OpBank != PrevOpBank) {
    LLVM_DEBUG(dbgs() << "Generic inst operands have different banks\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Generic inst operands have different banks\n"
; } } while (false);
    return true;
  }
  PrevOpBank = OpBank;
}
return false;
632}

634/// Select the AArch64 opcode for the basic binary operation \p GenericOpc
635/// (such as G_OR or G_SDIV), appropriate for the register bank \p RegBankID
636/// and of size \p OpSize.
637/// \returns \p GenericOpc if the combination is unsupported.
638static unsigned selectBinaryOp(unsigned GenericOpc, unsigned RegBankID,
                             unsigned OpSize) {
switch (RegBankID) {
case AArch64::GPRRegBankID:
  if (OpSize == 32) {
    switch (GenericOpc) {
    case TargetOpcode::G_SHL:
      return AArch64::LSLVWr;
    case TargetOpcode::G_LSHR:
      return AArch64::LSRVWr;
    case TargetOpcode::G_ASHR:
      return AArch64::ASRVWr;
    default:
      return GenericOpc;
    }
  } else if (OpSize == 64) {
    switch (GenericOpc) {
    case TargetOpcode::G_PTR_ADD:
      return AArch64::ADDXrr;
    case TargetOpcode::G_SHL:
      return AArch64::LSLVXr;
    case TargetOpcode::G_LSHR:
      return AArch64::LSRVXr;
    case TargetOpcode::G_ASHR:
      return AArch64::ASRVXr;
    default:
      return GenericOpc;
    }
  }
  break;
case AArch64::FPRRegBankID:
  switch (OpSize) {
  case 32:
    switch (GenericOpc) {
    case TargetOpcode::G_FADD:
      return AArch64::FADDSrr;
    case TargetOpcode::G_FSUB:
      return AArch64::FSUBSrr;
    case TargetOpcode::G_FMUL:
      return AArch64::FMULSrr;
    case TargetOpcode::G_FDIV:
      return AArch64::FDIVSrr;
    default:
      return GenericOpc;
    }
  case 64:
    switch (GenericOpc) {
    case TargetOpcode::G_FADD:
      return AArch64::FADDDrr;
    case TargetOpcode::G_FSUB:
      return AArch64::FSUBDrr;
    case TargetOpcode::G_FMUL:
      return AArch64::FMULDrr;
    case TargetOpcode::G_FDIV:
      return AArch64::FDIVDrr;
    case TargetOpcode::G_OR:
      return AArch64::ORRv8i8;
    default:
      return GenericOpc;
    }
  }
  break;
}
return GenericOpc;
702}

704/// Select the AArch64 opcode for the G_LOAD or G_STORE operation \p GenericOpc,
705/// appropriate for the (value) register bank \p RegBankID and of memory access
706/// size \p OpSize.  This returns the variant with the base+unsigned-immediate
707/// addressing mode (e.g., LDRXui).
708/// \returns \p GenericOpc if the combination is unsupported.
709static unsigned selectLoadStoreUIOp(unsigned GenericOpc, unsigned RegBankID,
                                  unsigned OpSize) {
const bool isStore = GenericOpc == TargetOpcode::G_STORE;
switch (RegBankID) {
case AArch64::GPRRegBankID:
  switch (OpSize) {
  case 8:
    return isStore ? AArch64::STRBBui : AArch64::LDRBBui;
  case 16:
    return isStore ? AArch64::STRHHui : AArch64::LDRHHui;
  case 32:
    return isStore ? AArch64::STRWui : AArch64::LDRWui;
  case 64:
    return isStore ? AArch64::STRXui : AArch64::LDRXui;
  }
  break;
case AArch64::FPRRegBankID:
  switch (OpSize) {
  case 8:
    return isStore ? AArch64::STRBui : AArch64::LDRBui;
  case 16:
    return isStore ? AArch64::STRHui : AArch64::LDRHui;
  case 32:
    return isStore ? AArch64::STRSui : AArch64::LDRSui;
  case 64:
    return isStore ? AArch64::STRDui : AArch64::LDRDui;
  }
  break;
}
return GenericOpc;
739}

741#ifndef NDEBUG
742/// Helper function that verifies that we have a valid copy at the end of
743/// selectCopy. Verifies that the source and dest have the expected sizes and
744/// then returns true.
745static bool isValidCopy(const MachineInstr &I, const RegisterBank &DstBank,
                      const MachineRegisterInfo &MRI,
                      const TargetRegisterInfo &TRI,
                      const RegisterBankInfo &RBI) {
const Register DstReg = I.getOperand(0).getReg();
const Register SrcReg = I.getOperand(1).getReg();
const unsigned DstSize = RBI.getSizeInBits(DstReg, MRI, TRI);
const unsigned SrcSize = RBI.getSizeInBits(SrcReg, MRI, TRI);

// Make sure the size of the source and dest line up.
assert((((DstSize == SrcSize || (Register::isPhysicalRegister(SrcReg
) && DstSize <= SrcSize) || (((DstSize + 31) / 32 ==
 (SrcSize + 31) / 32) && DstSize > SrcSize)) &&
 "Copy with different width?!") ? static_cast<void> (0)
 : __assert_fail ("(DstSize == SrcSize || (Register::isPhysicalRegister(SrcReg) && DstSize <= SrcSize) || (((DstSize + 31) / 32 == (SrcSize + 31) / 32) && DstSize > SrcSize)) && \"Copy with different width?!\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 764, __PRETTY_FUNCTION__))
    (DstSize == SrcSize ||(((DstSize == SrcSize || (Register::isPhysicalRegister(SrcReg
) && DstSize <= SrcSize) || (((DstSize + 31) / 32 ==
 (SrcSize + 31) / 32) && DstSize > SrcSize)) &&
 "Copy with different width?!") ? static_cast<void> (0)
 : __assert_fail ("(DstSize == SrcSize || (Register::isPhysicalRegister(SrcReg) && DstSize <= SrcSize) || (((DstSize + 31) / 32 == (SrcSize + 31) / 32) && DstSize > SrcSize)) && \"Copy with different width?!\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 764, __PRETTY_FUNCTION__))
     // Copies are a mean to setup initial types, the number of(((DstSize == SrcSize || (Register::isPhysicalRegister(SrcReg
) && DstSize <= SrcSize) || (((DstSize + 31) / 32 ==
 (SrcSize + 31) / 32) && DstSize > SrcSize)) &&
 "Copy with different width?!") ? static_cast<void> (0)
 : __assert_fail ("(DstSize == SrcSize || (Register::isPhysicalRegister(SrcReg) && DstSize <= SrcSize) || (((DstSize + 31) / 32 == (SrcSize + 31) / 32) && DstSize > SrcSize)) && \"Copy with different width?!\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 764, __PRETTY_FUNCTION__))
     // bits may not exactly match.(((DstSize == SrcSize || (Register::isPhysicalRegister(SrcReg
) && DstSize <= SrcSize) || (((DstSize + 31) / 32 ==
 (SrcSize + 31) / 32) && DstSize > SrcSize)) &&
 "Copy with different width?!") ? static_cast<void> (0)
 : __assert_fail ("(DstSize == SrcSize || (Register::isPhysicalRegister(SrcReg) && DstSize <= SrcSize) || (((DstSize + 31) / 32 == (SrcSize + 31) / 32) && DstSize > SrcSize)) && \"Copy with different width?!\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 764, __PRETTY_FUNCTION__))
     (Register::isPhysicalRegister(SrcReg) && DstSize <= SrcSize) ||(((DstSize == SrcSize || (Register::isPhysicalRegister(SrcReg
) && DstSize <= SrcSize) || (((DstSize + 31) / 32 ==
 (SrcSize + 31) / 32) && DstSize > SrcSize)) &&
 "Copy with different width?!") ? static_cast<void> (0)
 : __assert_fail ("(DstSize == SrcSize || (Register::isPhysicalRegister(SrcReg) && DstSize <= SrcSize) || (((DstSize + 31) / 32 == (SrcSize + 31) / 32) && DstSize > SrcSize)) && \"Copy with different width?!\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 764, __PRETTY_FUNCTION__))
     // Copies are a mean to copy bits around, as long as we are(((DstSize == SrcSize || (Register::isPhysicalRegister(SrcReg
) && DstSize <= SrcSize) || (((DstSize + 31) / 32 ==
 (SrcSize + 31) / 32) && DstSize > SrcSize)) &&
 "Copy with different width?!") ? static_cast<void> (0)
 : __assert_fail ("(DstSize == SrcSize || (Register::isPhysicalRegister(SrcReg) && DstSize <= SrcSize) || (((DstSize + 31) / 32 == (SrcSize + 31) / 32) && DstSize > SrcSize)) && \"Copy with different width?!\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 764, __PRETTY_FUNCTION__))
     // on the same register class, that's fine. Otherwise, that(((DstSize == SrcSize || (Register::isPhysicalRegister(SrcReg
) && DstSize <= SrcSize) || (((DstSize + 31) / 32 ==
 (SrcSize + 31) / 32) && DstSize > SrcSize)) &&
 "Copy with different width?!") ? static_cast<void> (0)
 : __assert_fail ("(DstSize == SrcSize || (Register::isPhysicalRegister(SrcReg) && DstSize <= SrcSize) || (((DstSize + 31) / 32 == (SrcSize + 31) / 32) && DstSize > SrcSize)) && \"Copy with different width?!\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 764, __PRETTY_FUNCTION__))
     // means we need some SUBREG_TO_REG or AND & co.(((DstSize == SrcSize || (Register::isPhysicalRegister(SrcReg
) && DstSize <= SrcSize) || (((DstSize + 31) / 32 ==
 (SrcSize + 31) / 32) && DstSize > SrcSize)) &&
 "Copy with different width?!") ? static_cast<void> (0)
 : __assert_fail ("(DstSize == SrcSize || (Register::isPhysicalRegister(SrcReg) && DstSize <= SrcSize) || (((DstSize + 31) / 32 == (SrcSize + 31) / 32) && DstSize > SrcSize)) && \"Copy with different width?!\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 764, __PRETTY_FUNCTION__))
     (((DstSize + 31) / 32 == (SrcSize + 31) / 32) && DstSize > SrcSize)) &&(((DstSize == SrcSize || (Register::isPhysicalRegister(SrcReg
) && DstSize <= SrcSize) || (((DstSize + 31) / 32 ==
 (SrcSize + 31) / 32) && DstSize > SrcSize)) &&
 "Copy with different width?!") ? static_cast<void> (0)
 : __assert_fail ("(DstSize == SrcSize || (Register::isPhysicalRegister(SrcReg) && DstSize <= SrcSize) || (((DstSize + 31) / 32 == (SrcSize + 31) / 32) && DstSize > SrcSize)) && \"Copy with different width?!\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 764, __PRETTY_FUNCTION__))
    "Copy with different width?!")(((DstSize == SrcSize || (Register::isPhysicalRegister(SrcReg
) && DstSize <= SrcSize) || (((DstSize + 31) / 32 ==
 (SrcSize + 31) / 32) && DstSize > SrcSize)) &&
 "Copy with different width?!") ? static_cast<void> (0)
 : __assert_fail ("(DstSize == SrcSize || (Register::isPhysicalRegister(SrcReg) && DstSize <= SrcSize) || (((DstSize + 31) / 32 == (SrcSize + 31) / 32) && DstSize > SrcSize)) && \"Copy with different width?!\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 764, __PRETTY_FUNCTION__));

// Check the size of the destination.
assert((DstSize <= 64 || DstBank.getID() == AArch64::FPRRegBankID) &&(((DstSize <= 64 || DstBank.getID() == AArch64::FPRRegBankID
) && "GPRs cannot get more than 64-bit width values")
 ? static_cast<void> (0) : __assert_fail ("(DstSize <= 64 || DstBank.getID() == AArch64::FPRRegBankID) && \"GPRs cannot get more than 64-bit width values\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 768, __PRETTY_FUNCTION__))
       "GPRs cannot get more than 64-bit width values")(((DstSize <= 64 || DstBank.getID() == AArch64::FPRRegBankID
) && "GPRs cannot get more than 64-bit width values")
 ? static_cast<void> (0) : __assert_fail ("(DstSize <= 64 || DstBank.getID() == AArch64::FPRRegBankID) && \"GPRs cannot get more than 64-bit width values\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 768, __PRETTY_FUNCTION__));

return true;
771}
772#endif

774/// Helper function for selectCopy. Inserts a subregister copy from \p SrcReg
775/// to \p *To.
776///
777/// E.g "To = COPY SrcReg:SubReg"
778static bool copySubReg(MachineInstr &I, MachineRegisterInfo &MRI,
                     const RegisterBankInfo &RBI, Register SrcReg,
                     const TargetRegisterClass *To, unsigned SubReg) {
assert(SrcReg.isValid() && "Expected a valid source register?")((SrcReg.isValid() && "Expected a valid source register?"
) ? static_cast<void> (0) : __assert_fail ("SrcReg.isValid() && \"Expected a valid source register?\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 781, __PRETTY_FUNCTION__));
assert(To && "Destination register class cannot be null")((To && "Destination register class cannot be null") ?
 static_cast<void> (0) : __assert_fail ("To && \"Destination register class cannot be null\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 782, __PRETTY_FUNCTION__));
assert(SubReg && "Expected a valid subregister")((SubReg && "Expected a valid subregister") ? static_cast
<void> (0) : __assert_fail ("SubReg && \"Expected a valid subregister\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 783, __PRETTY_FUNCTION__));

MachineIRBuilder MIB(I);
auto SubRegCopy =
    MIB.buildInstr(TargetOpcode::COPY, {To}, {}).addReg(SrcReg, 0, SubReg);
MachineOperand &RegOp = I.getOperand(1);
RegOp.setReg(SubRegCopy.getReg(0));

// It's possible that the destination register won't be constrained. Make
// sure that happens.
if (!Register::isPhysicalRegister(I.getOperand(0).getReg()))
  RBI.constrainGenericRegister(I.getOperand(0).getReg(), *To, MRI);

return true;
797}

799/// Helper function to get the source and destination register classes for a
800/// copy. Returns a std::pair containing the source register class for the
801/// copy, and the destination register class for the copy. If a register class
802/// cannot be determined, then it will be nullptr.
803static std::pair<const TargetRegisterClass *, const TargetRegisterClass *>
804getRegClassesForCopy(MachineInstr &I, const TargetInstrInfo &TII,
                   MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI,
                   const RegisterBankInfo &RBI) {
Register DstReg = I.getOperand(0).getReg();
Register SrcReg = I.getOperand(1).getReg();
const RegisterBank &DstRegBank = *RBI.getRegBank(DstReg, MRI, TRI);
const RegisterBank &SrcRegBank = *RBI.getRegBank(SrcReg, MRI, TRI);
unsigned DstSize = RBI.getSizeInBits(DstReg, MRI, TRI);
unsigned SrcSize = RBI.getSizeInBits(SrcReg, MRI, TRI);

// Special casing for cross-bank copies of s1s. We can technically represent
// a 1-bit value with any size of register. The minimum size for a GPR is 32
// bits. So, we need to put the FPR on 32 bits as well.
//
// FIXME: I'm not sure if this case holds true outside of copies. If it does,
// then we can pull it into the helpers that get the appropriate class for a
// register bank. Or make a new helper that carries along some constraint
// information.
if (SrcRegBank != DstRegBank && (DstSize == 1 && SrcSize == 1))
  SrcSize = DstSize = 32;

return {getMinClassForRegBank(SrcRegBank, SrcSize, true),
        getMinClassForRegBank(DstRegBank, DstSize, true)};
827}

829static bool selectCopy(MachineInstr &I, const TargetInstrInfo &TII,
                     MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI,
                     const RegisterBankInfo &RBI) {
Register DstReg = I.getOperand(0).getReg();
Register SrcReg = I.getOperand(1).getReg();
const RegisterBank &DstRegBank = *RBI.getRegBank(DstReg, MRI, TRI);
const RegisterBank &SrcRegBank = *RBI.getRegBank(SrcReg, MRI, TRI);

// Find the correct register classes for the source and destination registers.
const TargetRegisterClass *SrcRC;
const TargetRegisterClass *DstRC;
std::tie(SrcRC, DstRC) = getRegClassesForCopy(I, TII, MRI, TRI, RBI);
24
←
Calling 'tie<const llvm::TargetRegisterClass *, const llvm::TargetRegisterClass *>'→
35
←
Returning from 'tie<const llvm::TargetRegisterClass *, const llvm::TargetRegisterClass *>'→
36
←
Calling 'tuple::operator='→
39
←
Returning from 'tuple::operator='→

if (!DstRC) {
40
←
Assuming 'DstRC' is non-null→
41
←
Taking false branch→
  LLVM_DEBUG(dbgs() << "Unexpected dest size "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unexpected dest size " <<
 RBI.getSizeInBits(DstReg, MRI, TRI) << '\n'; } } while
 (false)
                    << RBI.getSizeInBits(DstReg, MRI, TRI) << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unexpected dest size " <<
 RBI.getSizeInBits(DstReg, MRI, TRI) << '\n'; } } while
 (false);
  return false;
}

// A couple helpers below, for making sure that the copy we produce is valid.

// Set to true if we insert a SUBREG_TO_REG. If we do this, then we don't want
// to verify that the src and dst are the same size, since that's handled by
// the SUBREG_TO_REG.
bool KnownValid = false;

// Returns true, or asserts if something we don't expect happens. Instead of
// returning true, we return isValidCopy() to ensure that we verify the
// result.
auto CheckCopy = [&]() {
  // If we have a bitcast or something, we can't have physical registers.
  assert((I.isCopy() ||(((I.isCopy() || (!Register::isPhysicalRegister(I.getOperand(
0).getReg()) && !Register::isPhysicalRegister(I.getOperand
(1).getReg()))) && "No phys reg on generic operator!"
) ? static_cast<void> (0) : __assert_fail ("(I.isCopy() || (!Register::isPhysicalRegister(I.getOperand(0).getReg()) && !Register::isPhysicalRegister(I.getOperand(1).getReg()))) && \"No phys reg on generic operator!\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 863, __PRETTY_FUNCTION__))
          (!Register::isPhysicalRegister(I.getOperand(0).getReg()) &&(((I.isCopy() || (!Register::isPhysicalRegister(I.getOperand(
0).getReg()) && !Register::isPhysicalRegister(I.getOperand
(1).getReg()))) && "No phys reg on generic operator!"
) ? static_cast<void> (0) : __assert_fail ("(I.isCopy() || (!Register::isPhysicalRegister(I.getOperand(0).getReg()) && !Register::isPhysicalRegister(I.getOperand(1).getReg()))) && \"No phys reg on generic operator!\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 863, __PRETTY_FUNCTION__))
           !Register::isPhysicalRegister(I.getOperand(1).getReg()))) &&(((I.isCopy() || (!Register::isPhysicalRegister(I.getOperand(
0).getReg()) && !Register::isPhysicalRegister(I.getOperand
(1).getReg()))) && "No phys reg on generic operator!"
) ? static_cast<void> (0) : __assert_fail ("(I.isCopy() || (!Register::isPhysicalRegister(I.getOperand(0).getReg()) && !Register::isPhysicalRegister(I.getOperand(1).getReg()))) && \"No phys reg on generic operator!\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 863, __PRETTY_FUNCTION__))
         "No phys reg on generic operator!")(((I.isCopy() || (!Register::isPhysicalRegister(I.getOperand(
0).getReg()) && !Register::isPhysicalRegister(I.getOperand
(1).getReg()))) && "No phys reg on generic operator!"
) ? static_cast<void> (0) : __assert_fail ("(I.isCopy() || (!Register::isPhysicalRegister(I.getOperand(0).getReg()) && !Register::isPhysicalRegister(I.getOperand(1).getReg()))) && \"No phys reg on generic operator!\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 863, __PRETTY_FUNCTION__));
  bool ValidCopy = true;
865#ifndef NDEBUG
  ValidCopy = KnownValid || isValidCopy(I, DstRegBank, MRI, TRI, RBI);
  assert(ValidCopy && "Invalid copy.")((ValidCopy && "Invalid copy.") ? static_cast<void
> (0) : __assert_fail ("ValidCopy && \"Invalid copy.\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 867, __PRETTY_FUNCTION__));
868#endif
  return ValidCopy;
};

// Is this a copy? If so, then we may need to insert a subregister copy.
if (I.isCopy()) {
42
←
Calling 'MachineInstr::isCopy'→
45
←
Returning from 'MachineInstr::isCopy'→
46
←
Taking true branch→
  // Yes. Check if there's anything to fix up.
  if (!SrcRC) {
47
←
Assuming 'SrcRC' is non-null→
48
←
Taking false branch→
    LLVM_DEBUG(dbgs() << "Couldn't determine source register class\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Couldn't determine source register class\n"
; } } while (false);
    return false;
  }

  unsigned SrcSize = TRI.getRegSizeInBits(*SrcRC);
  unsigned DstSize = TRI.getRegSizeInBits(*DstRC);
  unsigned SubReg;
49
←
'SubReg' declared without an initial value→

  // If the source bank doesn't support a subregister copy small enough,
  // then we first need to copy to the destination bank.
  if (getMinSizeForRegBank(SrcRegBank) > DstSize) {
50
←
Assuming the condition is false→
51
←
Taking false branch→
    const TargetRegisterClass *DstTempRC =
        getMinClassForRegBank(DstRegBank, SrcSize, /* GetAllRegSet */ true);
    getSubRegForClass(DstRC, TRI, SubReg);

    MachineIRBuilder MIB(I);
    auto Copy = MIB.buildCopy({DstTempRC}, {SrcReg});
    copySubReg(I, MRI, RBI, Copy.getReg(0), DstRC, SubReg);
  } else if (SrcSize > DstSize) {
52
←
Assuming 'SrcSize' is > 'DstSize'→
53
←
Taking true branch→
    // If the source register is bigger than the destination we need to
    // perform a subregister copy.
    const TargetRegisterClass *SubRegRC =
        getMinClassForRegBank(SrcRegBank, DstSize, /* GetAllRegSet */ true);
    getSubRegForClass(SubRegRC, TRI, SubReg);
54
←
Calling 'getSubRegForClass'→
59
←
Returning from 'getSubRegForClass'→
    copySubReg(I, MRI, RBI, SrcReg, DstRC, SubReg);
60
←
6th function call argument is an uninitialized value
  } else if (DstSize > SrcSize) {
    // If the destination register is bigger than the source we need to do
    // a promotion using SUBREG_TO_REG.
    const TargetRegisterClass *PromotionRC =
        getMinClassForRegBank(SrcRegBank, DstSize, /* GetAllRegSet */ true);
    getSubRegForClass(SrcRC, TRI, SubReg);

    Register PromoteReg = MRI.createVirtualRegister(PromotionRC);
    BuildMI(*I.getParent(), I, I.getDebugLoc(),
            TII.get(AArch64::SUBREG_TO_REG), PromoteReg)
        .addImm(0)
        .addUse(SrcReg)
        .addImm(SubReg);
    MachineOperand &RegOp = I.getOperand(1);
    RegOp.setReg(PromoteReg);

    // Promise that the copy is implicitly validated by the SUBREG_TO_REG.
    KnownValid = true;
  }

  // If the destination is a physical register, then there's nothing to
  // change, so we're done.
  if (Register::isPhysicalRegister(DstReg))
    return CheckCopy();
}

// No need to constrain SrcReg. It will get constrained when we hit another
// of its use or its defs. Copies do not have constraints.
if (!RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) {
  LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Failed to constrain " <<
 TII.getName(I.getOpcode()) << " operand\n"; } } while (
false)
                    << " operand\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Failed to constrain " <<
 TII.getName(I.getOpcode()) << " operand\n"; } } while (
false);
  return false;
}
I.setDesc(TII.get(AArch64::COPY));
return CheckCopy();
936}

938static unsigned selectFPConvOpc(unsigned GenericOpc, LLT DstTy, LLT SrcTy) {
if (!DstTy.isScalar() || !SrcTy.isScalar())
  return GenericOpc;

const unsigned DstSize = DstTy.getSizeInBits();
const unsigned SrcSize = SrcTy.getSizeInBits();

switch (DstSize) {
case 32:
  switch (SrcSize) {
  case 32:
    switch (GenericOpc) {
    case TargetOpcode::G_SITOFP:
      return AArch64::SCVTFUWSri;
    case TargetOpcode::G_UITOFP:
      return AArch64::UCVTFUWSri;
    case TargetOpcode::G_FPTOSI:
      return AArch64::FCVTZSUWSr;
    case TargetOpcode::G_FPTOUI:
      return AArch64::FCVTZUUWSr;
    default:
      return GenericOpc;
    }
  case 64:
    switch (GenericOpc) {
    case TargetOpcode::G_SITOFP:
      return AArch64::SCVTFUXSri;
    case TargetOpcode::G_UITOFP:
      return AArch64::UCVTFUXSri;
    case TargetOpcode::G_FPTOSI:
      return AArch64::FCVTZSUWDr;
    case TargetOpcode::G_FPTOUI:
      return AArch64::FCVTZUUWDr;
    default:
      return GenericOpc;
    }
  default:
    return GenericOpc;
  }
case 64:
  switch (SrcSize) {
  case 32:
    switch (GenericOpc) {
    case TargetOpcode::G_SITOFP:
      return AArch64::SCVTFUWDri;
    case TargetOpcode::G_UITOFP:
      return AArch64::UCVTFUWDri;
    case TargetOpcode::G_FPTOSI:
      return AArch64::FCVTZSUXSr;
    case TargetOpcode::G_FPTOUI:
      return AArch64::FCVTZUUXSr;
    default:
      return GenericOpc;
    }
  case 64:
    switch (GenericOpc) {
    case TargetOpcode::G_SITOFP:
      return AArch64::SCVTFUXDri;
    case TargetOpcode::G_UITOFP:
      return AArch64::UCVTFUXDri;
    case TargetOpcode::G_FPTOSI:
      return AArch64::FCVTZSUXDr;
    case TargetOpcode::G_FPTOUI:
      return AArch64::FCVTZUUXDr;
    default:
      return GenericOpc;
    }
  default:
    return GenericOpc;
  }
default:
  return GenericOpc;
};
return GenericOpc;
1012}

1014MachineInstr *
1015AArch64InstructionSelector::emitSelect(Register Dst, Register True,
                                     Register False, AArch64CC::CondCode CC,
                                     MachineIRBuilder &MIB) const {
MachineRegisterInfo &MRI = *MIB.getMRI();
assert(RBI.getRegBank(False, MRI, TRI)->getID() ==((RBI.getRegBank(False, MRI, TRI)->getID() == RBI.getRegBank
(True, MRI, TRI)->getID() && "Expected both select operands to have the same regbank?"
) ? static_cast<void> (0) : __assert_fail ("RBI.getRegBank(False, MRI, TRI)->getID() == RBI.getRegBank(True, MRI, TRI)->getID() && \"Expected both select operands to have the same regbank?\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1021, __PRETTY_FUNCTION__))
           RBI.getRegBank(True, MRI, TRI)->getID() &&((RBI.getRegBank(False, MRI, TRI)->getID() == RBI.getRegBank
(True, MRI, TRI)->getID() && "Expected both select operands to have the same regbank?"
) ? static_cast<void> (0) : __assert_fail ("RBI.getRegBank(False, MRI, TRI)->getID() == RBI.getRegBank(True, MRI, TRI)->getID() && \"Expected both select operands to have the same regbank?\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1021, __PRETTY_FUNCTION__))
       "Expected both select operands to have the same regbank?")((RBI.getRegBank(False, MRI, TRI)->getID() == RBI.getRegBank
(True, MRI, TRI)->getID() && "Expected both select operands to have the same regbank?"
) ? static_cast<void> (0) : __assert_fail ("RBI.getRegBank(False, MRI, TRI)->getID() == RBI.getRegBank(True, MRI, TRI)->getID() && \"Expected both select operands to have the same regbank?\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1021, __PRETTY_FUNCTION__));
LLT Ty = MRI.getType(True);
if (Ty.isVector())
  return nullptr;
const unsigned Size = Ty.getSizeInBits();
assert((Size == 32 || Size == 64) &&(((Size == 32 || Size == 64) && "Expected 32 bit or 64 bit select only?"
) ? static_cast<void> (0) : __assert_fail ("(Size == 32 || Size == 64) && \"Expected 32 bit or 64 bit select only?\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1027, __PRETTY_FUNCTION__))
       "Expected 32 bit or 64 bit select only?")(((Size == 32 || Size == 64) && "Expected 32 bit or 64 bit select only?"
) ? static_cast<void> (0) : __assert_fail ("(Size == 32 || Size == 64) && \"Expected 32 bit or 64 bit select only?\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1027, __PRETTY_FUNCTION__));
const bool Is32Bit = Size == 32;
if (RBI.getRegBank(True, MRI, TRI)->getID() != AArch64::GPRRegBankID) {
  unsigned Opc = Is32Bit ? AArch64::FCSELSrrr : AArch64::FCSELDrrr;
  auto FCSel = MIB.buildInstr(Opc, {Dst}, {True, False}).addImm(CC);
  constrainSelectedInstRegOperands(*FCSel, TII, TRI, RBI);
  return &*FCSel;
}

// By default, we'll try and emit a CSEL.
unsigned Opc = Is32Bit ? AArch64::CSELWr : AArch64::CSELXr;
bool Optimized = false;
auto TryFoldBinOpIntoSelect = [&Opc, Is32Bit, &CC, &MRI,
                               &Optimized](Register &Reg, Register &OtherReg,
                                           bool Invert) {
  if (Optimized)
    return false;

  // Attempt to fold:
  //
  // %sub = G_SUB 0, %x
  // %select = G_SELECT cc, %reg, %sub
  //
  // Into:
  // %select = CSNEG %reg, %x, cc
  Register MatchReg;
  if (mi_match(Reg, MRI, m_Neg(m_Reg(MatchReg)))) {
    Opc = Is32Bit ? AArch64::CSNEGWr : AArch64::CSNEGXr;
    Reg = MatchReg;
    if (Invert) {
      CC = AArch64CC::getInvertedCondCode(CC);
      std::swap(Reg, OtherReg);
    }
    return true;
  }

  // Attempt to fold:
  //
  // %xor = G_XOR %x, -1
  // %select = G_SELECT cc, %reg, %xor
  //
  // Into:
  // %select = CSINV %reg, %x, cc
  if (mi_match(Reg, MRI, m_Not(m_Reg(MatchReg)))) {
    Opc = Is32Bit ? AArch64::CSINVWr : AArch64::CSINVXr;
    Reg = MatchReg;
    if (Invert) {
      CC = AArch64CC::getInvertedCondCode(CC);
      std::swap(Reg, OtherReg);
    }
    return true;
  }

  // Attempt to fold:
  //
  // %add = G_ADD %x, 1
  // %select = G_SELECT cc, %reg, %add
  //
  // Into:
  // %select = CSINC %reg, %x, cc
  if (mi_match(Reg, MRI, m_GAdd(m_Reg(MatchReg), m_SpecificICst(1)))) {
    Opc = Is32Bit ? AArch64::CSINCWr : AArch64::CSINCXr;
    Reg = MatchReg;
    if (Invert) {
      CC = AArch64CC::getInvertedCondCode(CC);
      std::swap(Reg, OtherReg);
    }
    return true;
  }

  return false;
};

// Helper lambda which tries to use CSINC/CSINV for the instruction when its
// true/false values are constants.
// FIXME: All of these patterns already exist in tablegen. We should be
// able to import these.
auto TryOptSelectCst = [&Opc, &True, &False, &CC, Is32Bit, &MRI,
                        &Optimized]() {
  if (Optimized)
    return false;
  auto TrueCst = getConstantVRegValWithLookThrough(True, MRI);
  auto FalseCst = getConstantVRegValWithLookThrough(False, MRI);
  if (!TrueCst && !FalseCst)
    return false;

  Register ZReg = Is32Bit ? AArch64::WZR : AArch64::XZR;
  if (TrueCst && FalseCst) {
    int64_t T = TrueCst->Value.getSExtValue();
    int64_t F = FalseCst->Value.getSExtValue();

    if (T == 0 && F == 1) {
      // G_SELECT cc, 0, 1 -> CSINC zreg, zreg, cc
      Opc = Is32Bit ? AArch64::CSINCWr : AArch64::CSINCXr;
      True = ZReg;
      False = ZReg;
      return true;
    }

    if (T == 0 && F == -1) {
      // G_SELECT cc 0, -1 -> CSINV zreg, zreg cc
      Opc = Is32Bit ? AArch64::CSINVWr : AArch64::CSINVXr;
      True = ZReg;
      False = ZReg;
      return true;
    }
  }

  if (TrueCst) {
    int64_t T = TrueCst->Value.getSExtValue();
    if (T == 1) {
      // G_SELECT cc, 1, f -> CSINC f, zreg, inv_cc
      Opc = Is32Bit ? AArch64::CSINCWr : AArch64::CSINCXr;
      True = False;
      False = ZReg;
      CC = AArch64CC::getInvertedCondCode(CC);
      return true;
    }

    if (T == -1) {
      // G_SELECT cc, -1, f -> CSINV f, zreg, inv_cc
      Opc = Is32Bit ? AArch64::CSINVWr : AArch64::CSINVXr;
      True = False;
      False = ZReg;
      CC = AArch64CC::getInvertedCondCode(CC);
      return true;
    }
  }

  if (FalseCst) {
    int64_t F = FalseCst->Value.getSExtValue();
    if (F == 1) {
      // G_SELECT cc, t, 1 -> CSINC t, zreg, cc
      Opc = Is32Bit ? AArch64::CSINCWr : AArch64::CSINCXr;
      False = ZReg;
      return true;
    }

    if (F == -1) {
      // G_SELECT cc, t, -1 -> CSINC t, zreg, cc
      Opc = Is32Bit ? AArch64::CSINVWr : AArch64::CSINVXr;
      False = ZReg;
      return true;
    }
  }
  return false;
};

Optimized |= TryFoldBinOpIntoSelect(False, True, /*Invert = */ false);
Optimized |= TryFoldBinOpIntoSelect(True, False, /*Invert = */ true);
Optimized |= TryOptSelectCst();
auto SelectInst = MIB.buildInstr(Opc, {Dst}, {True, False}).addImm(CC);
constrainSelectedInstRegOperands(*SelectInst, TII, TRI, RBI);
return &*SelectInst;
1181}

1183static AArch64CC::CondCode changeICMPPredToAArch64CC(CmpInst::Predicate P) {
switch (P) {
default:
  llvm_unreachable("Unknown condition code!")::llvm::llvm_unreachable_internal("Unknown condition code!", "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1186);
case CmpInst::ICMP_NE:
  return AArch64CC::NE;
case CmpInst::ICMP_EQ:
  return AArch64CC::EQ;
case CmpInst::ICMP_SGT:
  return AArch64CC::GT;
case CmpInst::ICMP_SGE:
  return AArch64CC::GE;
case CmpInst::ICMP_SLT:
  return AArch64CC::LT;
case CmpInst::ICMP_SLE:
  return AArch64CC::LE;
case CmpInst::ICMP_UGT:
  return AArch64CC::HI;
case CmpInst::ICMP_UGE:
  return AArch64CC::HS;
case CmpInst::ICMP_ULT:
  return AArch64CC::LO;
case CmpInst::ICMP_ULE:
  return AArch64CC::LS;
}
1208}

1210static void changeFCMPPredToAArch64CC(CmpInst::Predicate P,
                                    AArch64CC::CondCode &CondCode,
                                    AArch64CC::CondCode &CondCode2) {
CondCode2 = AArch64CC::AL;
switch (P) {
default:
  llvm_unreachable("Unknown FP condition!")::llvm::llvm_unreachable_internal("Unknown FP condition!", "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1216);
case CmpInst::FCMP_OEQ:
  CondCode = AArch64CC::EQ;
  break;
case CmpInst::FCMP_OGT:
  CondCode = AArch64CC::GT;
  break;
case CmpInst::FCMP_OGE:
  CondCode = AArch64CC::GE;
  break;
case CmpInst::FCMP_OLT:
  CondCode = AArch64CC::MI;
  break;
case CmpInst::FCMP_OLE:
  CondCode = AArch64CC::LS;
  break;
case CmpInst::FCMP_ONE:
  CondCode = AArch64CC::MI;
  CondCode2 = AArch64CC::GT;
  break;
case CmpInst::FCMP_ORD:
  CondCode = AArch64CC::VC;
  break;
case CmpInst::FCMP_UNO:
  CondCode = AArch64CC::VS;
  break;
case CmpInst::FCMP_UEQ:
  CondCode = AArch64CC::EQ;
  CondCode2 = AArch64CC::VS;
  break;
case CmpInst::FCMP_UGT:
  CondCode = AArch64CC::HI;
  break;
case CmpInst::FCMP_UGE:
  CondCode = AArch64CC::PL;
  break;
case CmpInst::FCMP_ULT:
  CondCode = AArch64CC::LT;
  break;
case CmpInst::FCMP_ULE:
  CondCode = AArch64CC::LE;
  break;
case CmpInst::FCMP_UNE:
  CondCode = AArch64CC::NE;
  break;
}
1262}

1264/// Return a register which can be used as a bit to test in a TB(N)Z.
1265static Register getTestBitReg(Register Reg, uint64_t &Bit, bool &Invert,
                            MachineRegisterInfo &MRI) {
assert(Reg.isValid() && "Expected valid register!")((Reg.isValid() && "Expected valid register!") ? static_cast
<void> (0) : __assert_fail ("Reg.isValid() && \"Expected valid register!\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1267, __PRETTY_FUNCTION__));
while (MachineInstr *MI = getDefIgnoringCopies(Reg, MRI)) {
  unsigned Opc = MI->getOpcode();

  if (!MI->getOperand(0).isReg() ||
      !MRI.hasOneNonDBGUse(MI->getOperand(0).getReg()))
    break;

  // (tbz (any_ext x), b) -> (tbz x, b) if we don't use the extended bits.
  //
  // (tbz (trunc x), b) -> (tbz x, b) is always safe, because the bit number
  // on the truncated x is the same as the bit number on x.
  if (Opc == TargetOpcode::G_ANYEXT || Opc == TargetOpcode::G_ZEXT ||
      Opc == TargetOpcode::G_TRUNC) {
    Register NextReg = MI->getOperand(1).getReg();
    // Did we find something worth folding?
    if (!NextReg.isValid() || !MRI.hasOneNonDBGUse(NextReg))
      break;

    // NextReg is worth folding. Keep looking.
    Reg = NextReg;
    continue;
  }

  // Attempt to find a suitable operation with a constant on one side.
  Optional<uint64_t> C;
  Register TestReg;
  switch (Opc) {
  default:
    break;
  case TargetOpcode::G_AND:
  case TargetOpcode::G_XOR: {
    TestReg = MI->getOperand(1).getReg();
    Register ConstantReg = MI->getOperand(2).getReg();
    auto VRegAndVal = getConstantVRegValWithLookThrough(ConstantReg, MRI);
    if (!VRegAndVal) {
      // AND commutes, check the other side for a constant.
      // FIXME: Can we canonicalize the constant so that it's always on the
      // same side at some point earlier?
      std::swap(ConstantReg, TestReg);
      VRegAndVal = getConstantVRegValWithLookThrough(ConstantReg, MRI);
    }
    if (VRegAndVal)
      C = VRegAndVal->Value.getSExtValue();
    break;
  }
  case TargetOpcode::G_ASHR:
  case TargetOpcode::G_LSHR:
  case TargetOpcode::G_SHL: {
    TestReg = MI->getOperand(1).getReg();
    auto VRegAndVal =
        getConstantVRegValWithLookThrough(MI->getOperand(2).getReg(), MRI);
    if (VRegAndVal)
      C = VRegAndVal->Value.getSExtValue();
    break;
  }
  }

  // Didn't find a constant or viable register. Bail out of the loop.
  if (!C || !TestReg.isValid())
    break;

  // We found a suitable instruction with a constant. Check to see if we can
  // walk through the instruction.
  Register NextReg;
  unsigned TestRegSize = MRI.getType(TestReg).getSizeInBits();
  switch (Opc) {
  default:
    break;
  case TargetOpcode::G_AND:
    // (tbz (and x, m), b) -> (tbz x, b) when the b-th bit of m is set.
    if ((*C >> Bit) & 1)
      NextReg = TestReg;
    break;
  case TargetOpcode::G_SHL:
    // (tbz (shl x, c), b) -> (tbz x, b-c) when b-c is positive and fits in
    // the type of the register.
    if (*C <= Bit && (Bit - *C) < TestRegSize) {
      NextReg = TestReg;
      Bit = Bit - *C;
    }
    break;
  case TargetOpcode::G_ASHR:
    // (tbz (ashr x, c), b) -> (tbz x, b+c) or (tbz x, msb) if b+c is > # bits
    // in x
    NextReg = TestReg;
    Bit = Bit + *C;
    if (Bit >= TestRegSize)
      Bit = TestRegSize - 1;
    break;
  case TargetOpcode::G_LSHR:
    // (tbz (lshr x, c), b) -> (tbz x, b+c) when b + c is < # bits in x
    if ((Bit + *C) < TestRegSize) {
      NextReg = TestReg;
      Bit = Bit + *C;
    }
    break;
  case TargetOpcode::G_XOR:
    // We can walk through a G_XOR by inverting whether we use tbz/tbnz when
    // appropriate.
    //
    // e.g. If x' = xor x, c, and the b-th bit is set in c then
    //
    // tbz x', b -> tbnz x, b
    //
    // Because x' only has the b-th bit set if x does not.
    if ((*C >> Bit) & 1)
      Invert = !Invert;
    NextReg = TestReg;
    break;
  }

  // Check if we found anything worth folding.
  if (!NextReg.isValid())
    return Reg;
  Reg = NextReg;
}

return Reg;
1386}

1388MachineInstr *AArch64InstructionSelector::emitTestBit(
  Register TestReg, uint64_t Bit, bool IsNegative, MachineBasicBlock *DstMBB,
  MachineIRBuilder &MIB) const {
assert(TestReg.isValid())((TestReg.isValid()) ? static_cast<void> (0) : __assert_fail
 ("TestReg.isValid()", "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1391, __PRETTY_FUNCTION__));
assert(ProduceNonFlagSettingCondBr &&((ProduceNonFlagSettingCondBr && "Cannot emit TB(N)Z with speculation tracking!"
) ? static_cast<void> (0) : __assert_fail ("ProduceNonFlagSettingCondBr && \"Cannot emit TB(N)Z with speculation tracking!\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1393, __PRETTY_FUNCTION__))
       "Cannot emit TB(N)Z with speculation tracking!")((ProduceNonFlagSettingCondBr && "Cannot emit TB(N)Z with speculation tracking!"
) ? static_cast<void> (0) : __assert_fail ("ProduceNonFlagSettingCondBr && \"Cannot emit TB(N)Z with speculation tracking!\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1393, __PRETTY_FUNCTION__));
MachineRegisterInfo &MRI = *MIB.getMRI();

// Attempt to optimize the test bit by walking over instructions.
TestReg = getTestBitReg(TestReg, Bit, IsNegative, MRI);
LLT Ty = MRI.getType(TestReg);
unsigned Size = Ty.getSizeInBits();
assert(!Ty.isVector() && "Expected a scalar!")((!Ty.isVector() && "Expected a scalar!") ? static_cast
<void> (0) : __assert_fail ("!Ty.isVector() && \"Expected a scalar!\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1400, __PRETTY_FUNCTION__));
assert(Bit < 64 && "Bit is too large!")((Bit < 64 && "Bit is too large!") ? static_cast<
void> (0) : __assert_fail ("Bit < 64 && \"Bit is too large!\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1401, __PRETTY_FUNCTION__));

// When the test register is a 64-bit register, we have to narrow to make
// TBNZW work.
bool UseWReg = Bit < 32;
unsigned NecessarySize = UseWReg ? 32 : 64;
if (Size != NecessarySize)
  TestReg = moveScalarRegClass(
      TestReg, UseWReg ? AArch64::GPR32RegClass : AArch64::GPR64RegClass,
      MIB);

static const unsigned OpcTable[2][2] = {{AArch64::TBZX, AArch64::TBNZX},
                                        {AArch64::TBZW, AArch64::TBNZW}};
unsigned Opc = OpcTable[UseWReg][IsNegative];
auto TestBitMI =
    MIB.buildInstr(Opc).addReg(TestReg).addImm(Bit).addMBB(DstMBB);
constrainSelectedInstRegOperands(*TestBitMI, TII, TRI, RBI);
return &*TestBitMI;
1419}

1421bool AArch64InstructionSelector::tryOptAndIntoCompareBranch(
  MachineInstr &AndInst, bool Invert, MachineBasicBlock *DstMBB,
  MachineIRBuilder &MIB) const {
assert(AndInst.getOpcode() == TargetOpcode::G_AND && "Expected G_AND only?")((AndInst.getOpcode() == TargetOpcode::G_AND && "Expected G_AND only?"
) ? static_cast<void> (0) : __assert_fail ("AndInst.getOpcode() == TargetOpcode::G_AND && \"Expected G_AND only?\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1424, __PRETTY_FUNCTION__));
// Given something like this:
//
//  %x = ...Something...
//  %one = G_CONSTANT i64 1
//  %zero = G_CONSTANT i64 0
//  %and = G_AND %x, %one
//  %cmp = G_ICMP intpred(ne), %and, %zero
//  %cmp_trunc = G_TRUNC %cmp
//  G_BRCOND %cmp_trunc, %bb.3
//
// We want to try and fold the AND into the G_BRCOND and produce either a
// TBNZ (when we have intpred(ne)) or a TBZ (when we have intpred(eq)).
//
// In this case, we'd get
//
// TBNZ %x %bb.3
//

// Check if the AND has a constant on its RHS which we can use as a mask.
// If it's a power of 2, then it's the same as checking a specific bit.
// (e.g, ANDing with 8 == ANDing with 000...100 == testing if bit 3 is set)
auto MaybeBit = getConstantVRegValWithLookThrough(
    AndInst.getOperand(2).getReg(), *MIB.getMRI());
if (!MaybeBit)
  return false;

int32_t Bit = MaybeBit->Value.exactLogBase2();
if (Bit < 0)
  return false;

Register TestReg = AndInst.getOperand(1).getReg();

// Emit a TB(N)Z.
emitTestBit(TestReg, Bit, Invert, DstMBB, MIB);
return true;
1460}

1462MachineInstr *AArch64InstructionSelector::emitCBZ(Register CompareReg,
                                                bool IsNegative,
                                                MachineBasicBlock *DestMBB,
                                                MachineIRBuilder &MIB) const {
assert(ProduceNonFlagSettingCondBr && "CBZ does not set flags!")((ProduceNonFlagSettingCondBr && "CBZ does not set flags!"
) ? static_cast<void> (0) : __assert_fail ("ProduceNonFlagSettingCondBr && \"CBZ does not set flags!\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1466, __PRETTY_FUNCTION__));
MachineRegisterInfo &MRI = *MIB.getMRI();
assert(RBI.getRegBank(CompareReg, MRI, TRI)->getID() ==((RBI.getRegBank(CompareReg, MRI, TRI)->getID() == AArch64
::GPRRegBankID && "Expected GPRs only?") ? static_cast
<void> (0) : __assert_fail ("RBI.getRegBank(CompareReg, MRI, TRI)->getID() == AArch64::GPRRegBankID && \"Expected GPRs only?\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1470, __PRETTY_FUNCTION__))
           AArch64::GPRRegBankID &&((RBI.getRegBank(CompareReg, MRI, TRI)->getID() == AArch64
::GPRRegBankID && "Expected GPRs only?") ? static_cast
<void> (0) : __assert_fail ("RBI.getRegBank(CompareReg, MRI, TRI)->getID() == AArch64::GPRRegBankID && \"Expected GPRs only?\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1470, __PRETTY_FUNCTION__))
       "Expected GPRs only?")((RBI.getRegBank(CompareReg, MRI, TRI)->getID() == AArch64
::GPRRegBankID && "Expected GPRs only?") ? static_cast
<void> (0) : __assert_fail ("RBI.getRegBank(CompareReg, MRI, TRI)->getID() == AArch64::GPRRegBankID && \"Expected GPRs only?\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1470, __PRETTY_FUNCTION__));
auto Ty = MRI.getType(CompareReg);
unsigned Width = Ty.getSizeInBits();
assert(!Ty.isVector() && "Expected scalar only?")((!Ty.isVector() && "Expected scalar only?") ? static_cast
<void> (0) : __assert_fail ("!Ty.isVector() && \"Expected scalar only?\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1473, __PRETTY_FUNCTION__));
assert(Width <= 64 && "Expected width to be at most 64?")((Width <= 64 && "Expected width to be at most 64?"
) ? static_cast<void> (0) : __assert_fail ("Width <= 64 && \"Expected width to be at most 64?\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1474, __PRETTY_FUNCTION__));
static const unsigned OpcTable[2][2] = {{AArch64::CBZW, AArch64::CBZX},
                                        {AArch64::CBNZW, AArch64::CBNZX}};
unsigned Opc = OpcTable[IsNegative][Width == 64];
auto BranchMI = MIB.buildInstr(Opc, {}, {CompareReg}).addMBB(DestMBB);
constrainSelectedInstRegOperands(*BranchMI, TII, TRI, RBI);
return &*BranchMI;
1481}

1483bool AArch64InstructionSelector::selectCompareBranchFedByFCmp(
  MachineInstr &I, MachineInstr &FCmp, MachineIRBuilder &MIB) const {
assert(FCmp.getOpcode() == TargetOpcode::G_FCMP)((FCmp.getOpcode() == TargetOpcode::G_FCMP) ? static_cast<
void> (0) : __assert_fail ("FCmp.getOpcode() == TargetOpcode::G_FCMP"
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1485, __PRETTY_FUNCTION__));
assert(I.getOpcode() == TargetOpcode::G_BRCOND)((I.getOpcode() == TargetOpcode::G_BRCOND) ? static_cast<void
> (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_BRCOND"
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1486, __PRETTY_FUNCTION__));
// Unfortunately, the mapping of LLVM FP CC's onto AArch64 CC's isn't
// totally clean.  Some of them require two branches to implement.
auto Pred = (CmpInst::Predicate)FCmp.getOperand(1).getPredicate();
emitFPCompare(FCmp.getOperand(2).getReg(), FCmp.getOperand(3).getReg(), MIB,
              Pred);
AArch64CC::CondCode CC1, CC2;
changeFCMPPredToAArch64CC(static_cast<CmpInst::Predicate>(Pred), CC1, CC2);
MachineBasicBlock *DestMBB = I.getOperand(1).getMBB();
MIB.buildInstr(AArch64::Bcc, {}, {}).addImm(CC1).addMBB(DestMBB);
if (CC2 != AArch64CC::AL)
  MIB.buildInstr(AArch64::Bcc, {}, {}).addImm(CC2).addMBB(DestMBB);
I.eraseFromParent();
return true;
1500}

1502bool AArch64InstructionSelector::tryOptCompareBranchFedByICmp(
  MachineInstr &I, MachineInstr &ICmp, MachineIRBuilder &MIB) const {
assert(ICmp.getOpcode() == TargetOpcode::G_ICMP)((ICmp.getOpcode() == TargetOpcode::G_ICMP) ? static_cast<
void> (0) : __assert_fail ("ICmp.getOpcode() == TargetOpcode::G_ICMP"
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1504, __PRETTY_FUNCTION__));
assert(I.getOpcode() == TargetOpcode::G_BRCOND)((I.getOpcode() == TargetOpcode::G_BRCOND) ? static_cast<void
> (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_BRCOND"
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1505, __PRETTY_FUNCTION__));
// Attempt to optimize the G_BRCOND + G_ICMP into a TB(N)Z/CB(N)Z.
//
// Speculation tracking/SLH assumes that optimized TB(N)Z/CB(N)Z
// instructions will not be produced, as they are conditional branch
// instructions that do not set flags.
if (!ProduceNonFlagSettingCondBr)
  return false;

MachineRegisterInfo &MRI = *MIB.getMRI();
MachineBasicBlock *DestMBB = I.getOperand(1).getMBB();
auto Pred =
    static_cast<CmpInst::Predicate>(ICmp.getOperand(1).getPredicate());
Register LHS = ICmp.getOperand(2).getReg();
Register RHS = ICmp.getOperand(3).getReg();

// We're allowed to emit a TB(N)Z/CB(N)Z. Try to do that.
auto VRegAndVal = getConstantVRegValWithLookThrough(RHS, MRI);
MachineInstr *AndInst = getOpcodeDef(TargetOpcode::G_AND, LHS, MRI);

// When we can emit a TB(N)Z, prefer that.
//
// Handle non-commutative condition codes first.
// Note that we don't want to do this when we have a G_AND because it can
// become a tst. The tst will make the test bit in the TB(N)Z redundant.
if (VRegAndVal && !AndInst) {
  int64_t C = VRegAndVal->Value.getSExtValue();

  // When we have a greater-than comparison, we can just test if the msb is
  // zero.
  if (C == -1 && Pred == CmpInst::ICMP_SGT) {
    uint64_t Bit = MRI.getType(LHS).getSizeInBits() - 1;
    emitTestBit(LHS, Bit, /*IsNegative = */ false, DestMBB, MIB);
    I.eraseFromParent();
    return true;
  }

  // When we have a less than comparison, we can just test if the msb is not
  // zero.
  if (C == 0 && Pred == CmpInst::ICMP_SLT) {
    uint64_t Bit = MRI.getType(LHS).getSizeInBits() - 1;
    emitTestBit(LHS, Bit, /*IsNegative = */ true, DestMBB, MIB);
    I.eraseFromParent();
    return true;
  }
}

// Attempt to handle commutative condition codes. Right now, that's only
// eq/ne.
if (ICmpInst::isEquality(Pred)) {
  if (!VRegAndVal) {
    std::swap(RHS, LHS);
    VRegAndVal = getConstantVRegValWithLookThrough(RHS, MRI);
    AndInst = getOpcodeDef(TargetOpcode::G_AND, LHS, MRI);
  }

  if (VRegAndVal && VRegAndVal->Value == 0) {
    // If there's a G_AND feeding into this branch, try to fold it away by
    // emitting a TB(N)Z instead.
    //
    // Note: If we have LT, then it *is* possible to fold, but it wouldn't be
    // beneficial. When we have an AND and LT, we need a TST/ANDS, so folding
    // would be redundant.
    if (AndInst &&
        tryOptAndIntoCompareBranch(
            *AndInst, /*Invert = */ Pred == CmpInst::ICMP_NE, DestMBB, MIB)) {
      I.eraseFromParent();
      return true;
    }

    // Otherwise, try to emit a CB(N)Z instead.
    auto LHSTy = MRI.getType(LHS);
    if (!LHSTy.isVector() && LHSTy.getSizeInBits() <= 64) {
      emitCBZ(LHS, /*IsNegative = */ Pred == CmpInst::ICMP_NE, DestMBB, MIB);
      I.eraseFromParent();
      return true;
    }
  }
}

return false;
1586}

1588bool AArch64InstructionSelector::selectCompareBranchFedByICmp(
  MachineInstr &I, MachineInstr &ICmp, MachineIRBuilder &MIB) const {
assert(ICmp.getOpcode() == TargetOpcode::G_ICMP)((ICmp.getOpcode() == TargetOpcode::G_ICMP) ? static_cast<
void> (0) : __assert_fail ("ICmp.getOpcode() == TargetOpcode::G_ICMP"
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1590, __PRETTY_FUNCTION__));
assert(I.getOpcode() == TargetOpcode::G_BRCOND)((I.getOpcode() == TargetOpcode::G_BRCOND) ? static_cast<void
> (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_BRCOND"
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1591, __PRETTY_FUNCTION__));
if (tryOptCompareBranchFedByICmp(I, ICmp, MIB))
  return true;

// Couldn't optimize. Emit a compare + a Bcc.
MachineBasicBlock *DestMBB = I.getOperand(1).getMBB();
auto PredOp = ICmp.getOperand(1);
emitIntegerCompare(ICmp.getOperand(2), ICmp.getOperand(3), PredOp, MIB);
const AArch64CC::CondCode CC = changeICMPPredToAArch64CC(
    static_cast<CmpInst::Predicate>(PredOp.getPredicate()));
MIB.buildInstr(AArch64::Bcc, {}, {}).addImm(CC).addMBB(DestMBB);
I.eraseFromParent();
return true;
1604}

1606bool AArch64InstructionSelector::selectCompareBranch(
  MachineInstr &I, MachineFunction &MF, MachineRegisterInfo &MRI) const {
Register CondReg = I.getOperand(0).getReg();
MachineInstr *CCMI = MRI.getVRegDef(CondReg);
if (CCMI->getOpcode() == TargetOpcode::G_TRUNC) {
  CondReg = CCMI->getOperand(1).getReg();
  CCMI = MRI.getVRegDef(CondReg);
}

// Try to select the G_BRCOND using whatever is feeding the condition if
// possible.
MachineIRBuilder MIB(I);
unsigned CCMIOpc = CCMI->getOpcode();
if (CCMIOpc == TargetOpcode::G_FCMP)
  return selectCompareBranchFedByFCmp(I, *CCMI, MIB);
if (CCMIOpc == TargetOpcode::G_ICMP)
  return selectCompareBranchFedByICmp(I, *CCMI, MIB);

// Speculation tracking/SLH assumes that optimized TB(N)Z/CB(N)Z
// instructions will not be produced, as they are conditional branch
// instructions that do not set flags.
if (ProduceNonFlagSettingCondBr) {
  emitTestBit(CondReg, /*Bit = */ 0, /*IsNegative = */ true,
              I.getOperand(1).getMBB(), MIB);
  I.eraseFromParent();
  return true;
}

// Can't emit TB(N)Z/CB(N)Z. Emit a tst + bcc instead.
auto TstMI =
    MIB.buildInstr(AArch64::ANDSWri, {LLT::scalar(32)}, {CondReg}).addImm(1);
constrainSelectedInstRegOperands(*TstMI, TII, TRI, RBI);
auto Bcc = MIB.buildInstr(AArch64::Bcc)
               .addImm(AArch64CC::EQ)
               .addMBB(I.getOperand(1).getMBB());
I.eraseFromParent();
return constrainSelectedInstRegOperands(*Bcc, TII, TRI, RBI);
1643}

1645/// Returns the element immediate value of a vector shift operand if found.
1646/// This needs to detect a splat-like operation, e.g. a G_BUILD_VECTOR.
1647static Optional<int64_t> getVectorShiftImm(Register Reg,
                                         MachineRegisterInfo &MRI) {
assert(MRI.getType(Reg).isVector() && "Expected a *vector* shift operand")((MRI.getType(Reg).isVector() && "Expected a *vector* shift operand"
) ? static_cast<void> (0) : __assert_fail ("MRI.getType(Reg).isVector() && \"Expected a *vector* shift operand\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1649, __PRETTY_FUNCTION__));
MachineInstr *OpMI = MRI.getVRegDef(Reg);
assert(OpMI && "Expected to find a vreg def for vector shift operand")((OpMI && "Expected to find a vreg def for vector shift operand"
) ? static_cast<void> (0) : __assert_fail ("OpMI && \"Expected to find a vreg def for vector shift operand\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1651, __PRETTY_FUNCTION__));
if (OpMI->getOpcode() != TargetOpcode::G_BUILD_VECTOR)
  return None;

// Check all operands are identical immediates.
int64_t ImmVal = 0;
for (unsigned Idx = 1; Idx < OpMI->getNumOperands(); ++Idx) {
  auto VRegAndVal = getConstantVRegValWithLookThrough(OpMI->getOperand(Idx).getReg(), MRI);
  if (!VRegAndVal)
    return None;

  if (Idx == 1)
    ImmVal = VRegAndVal->Value.getSExtValue();
  if (ImmVal != VRegAndVal->Value.getSExtValue())
    return None;
}

return ImmVal;
1669}

1671/// Matches and returns the shift immediate value for a SHL instruction given
1672/// a shift operand.
1673static Optional<int64_t> getVectorSHLImm(LLT SrcTy, Register Reg, MachineRegisterInfo &MRI) {
Optional<int64_t> ShiftImm = getVectorShiftImm(Reg, MRI);
if (!ShiftImm)
  return None;
// Check the immediate is in range for a SHL.
int64_t Imm = *ShiftImm;
if (Imm < 0)
  return None;
switch (SrcTy.getElementType().getSizeInBits()) {
default:
  LLVM_DEBUG(dbgs() << "Unhandled element type for vector shift")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unhandled element type for vector shift"
; } } while (false);
  return None;
case 8:
  if (Imm > 7)
    return None;
  break;
case 16:
  if (Imm > 15)
    return None;
  break;
case 32:
  if (Imm > 31)
    return None;
  break;
case 64:
  if (Imm > 63)
    return None;
  break;
}
return Imm;
1703}

1705bool AArch64InstructionSelector::selectVectorSHL(
  MachineInstr &I, MachineRegisterInfo &MRI) const {
assert(I.getOpcode() == TargetOpcode::G_SHL)((I.getOpcode() == TargetOpcode::G_SHL) ? static_cast<void
> (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_SHL"
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1707, __PRETTY_FUNCTION__));
Register DstReg = I.getOperand(0).getReg();
const LLT Ty = MRI.getType(DstReg);
Register Src1Reg = I.getOperand(1).getReg();
Register Src2Reg = I.getOperand(2).getReg();

if (!Ty.isVector())
  return false;

// Check if we have a vector of constants on RHS that we can select as the
// immediate form.
Optional<int64_t> ImmVal = getVectorSHLImm(Ty, Src2Reg, MRI);

unsigned Opc = 0;
if (Ty == LLT::vector(2, 64)) {
  Opc = ImmVal ? AArch64::SHLv2i64_shift : AArch64::USHLv2i64;
} else if (Ty == LLT::vector(4, 32)) {
  Opc = ImmVal ? AArch64::SHLv4i32_shift : AArch64::USHLv4i32;
} else if (Ty == LLT::vector(2, 32)) {
  Opc = ImmVal ? AArch64::SHLv2i32_shift : AArch64::USHLv2i32;
} else if (Ty == LLT::vector(4, 16)) {
  Opc = ImmVal ? AArch64::SHLv4i16_shift : AArch64::USHLv4i16;
} else if (Ty == LLT::vector(8, 16)) {
  Opc = ImmVal ? AArch64::SHLv8i16_shift : AArch64::USHLv8i16;
} else if (Ty == LLT::vector(16, 8)) {
  Opc = ImmVal ? AArch64::SHLv16i8_shift : AArch64::USHLv16i8;
} else if (Ty == LLT::vector(8, 8)) {
  Opc = ImmVal ? AArch64::SHLv8i8_shift : AArch64::USHLv8i8;
} else {
  LLVM_DEBUG(dbgs() << "Unhandled G_SHL type")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unhandled G_SHL type"; }
 } while (false);
  return false;
}

MachineIRBuilder MIB(I);
auto Shl = MIB.buildInstr(Opc, {DstReg}, {Src1Reg});
if (ImmVal)
  Shl.addImm(*ImmVal);
else
  Shl.addUse(Src2Reg);
constrainSelectedInstRegOperands(*Shl, TII, TRI, RBI);
I.eraseFromParent();
return true;
1749}

1751bool AArch64InstructionSelector::selectVectorAshrLshr(
  MachineInstr &I, MachineRegisterInfo &MRI) const {
assert(I.getOpcode() == TargetOpcode::G_ASHR ||((I.getOpcode() == TargetOpcode::G_ASHR || I.getOpcode() == TargetOpcode
::G_LSHR) ? static_cast<void> (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_ASHR || I.getOpcode() == TargetOpcode::G_LSHR"
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1754, __PRETTY_FUNCTION__))
       I.getOpcode() == TargetOpcode::G_LSHR)((I.getOpcode() == TargetOpcode::G_ASHR || I.getOpcode() == TargetOpcode
::G_LSHR) ? static_cast<void> (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_ASHR || I.getOpcode() == TargetOpcode::G_LSHR"
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1754, __PRETTY_FUNCTION__));
Register DstReg = I.getOperand(0).getReg();
const LLT Ty = MRI.getType(DstReg);
Register Src1Reg = I.getOperand(1).getReg();
Register Src2Reg = I.getOperand(2).getReg();

if (!Ty.isVector())
  return false;

bool IsASHR = I.getOpcode() == TargetOpcode::G_ASHR;

// We expect the immediate case to be lowered in the PostLegalCombiner to
// AArch64ISD::VASHR or AArch64ISD::VLSHR equivalents.

// There is not a shift right register instruction, but the shift left
// register instruction takes a signed value, where negative numbers specify a
// right shift.

unsigned Opc = 0;
unsigned NegOpc = 0;
const TargetRegisterClass *RC =
    getRegClassForTypeOnBank(Ty, RBI.getRegBank(AArch64::FPRRegBankID), RBI);
if (Ty == LLT::vector(2, 64)) {
  Opc = IsASHR ? AArch64::SSHLv2i64 : AArch64::USHLv2i64;
  NegOpc = AArch64::NEGv2i64;
} else if (Ty == LLT::vector(4, 32)) {
  Opc = IsASHR ? AArch64::SSHLv4i32 : AArch64::USHLv4i32;
  NegOpc = AArch64::NEGv4i32;
} else if (Ty == LLT::vector(2, 32)) {
  Opc = IsASHR ? AArch64::SSHLv2i32 : AArch64::USHLv2i32;
  NegOpc = AArch64::NEGv2i32;
} else if (Ty == LLT::vector(4, 16)) {
  Opc = IsASHR ? AArch64::SSHLv4i16 : AArch64::USHLv4i16;
  NegOpc = AArch64::NEGv4i16;
} else if (Ty == LLT::vector(8, 16)) {
  Opc = IsASHR ? AArch64::SSHLv8i16 : AArch64::USHLv8i16;
  NegOpc = AArch64::NEGv8i16;
} else if (Ty == LLT::vector(16, 8)) {
  Opc = IsASHR ? AArch64::SSHLv16i8 : AArch64::USHLv16i8;
  NegOpc = AArch64::NEGv8i16;
} else if (Ty == LLT::vector(8, 8)) {
  Opc = IsASHR ? AArch64::SSHLv8i8 : AArch64::USHLv8i8;
  NegOpc = AArch64::NEGv8i8;
} else {
  LLVM_DEBUG(dbgs() << "Unhandled G_ASHR type")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unhandled G_ASHR type"; }
 } while (false);
  return false;
}

MachineIRBuilder MIB(I);
auto Neg = MIB.buildInstr(NegOpc, {RC}, {Src2Reg});
constrainSelectedInstRegOperands(*Neg, TII, TRI, RBI);
auto SShl = MIB.buildInstr(Opc, {DstReg}, {Src1Reg, Neg});
constrainSelectedInstRegOperands(*SShl, TII, TRI, RBI);
I.eraseFromParent();
return true;
1809}

1811bool AArch64InstructionSelector::selectVaStartAAPCS(
  MachineInstr &I, MachineFunction &MF, MachineRegisterInfo &MRI) const {
return false;
1814}

1816bool AArch64InstructionSelector::selectVaStartDarwin(
  MachineInstr &I, MachineFunction &MF, MachineRegisterInfo &MRI) const {
AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>();
Register ListReg = I.getOperand(0).getReg();

Register ArgsAddrReg = MRI.createVirtualRegister(&AArch64::GPR64RegClass);

auto MIB =
    BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(AArch64::ADDXri))
        .addDef(ArgsAddrReg)
        .addFrameIndex(FuncInfo->getVarArgsStackIndex())
        .addImm(0)
        .addImm(0);

constrainSelectedInstRegOperands(*MIB, TII, TRI, RBI);

MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(AArch64::STRXui))
          .addUse(ArgsAddrReg)
          .addUse(ListReg)
          .addImm(0)
          .addMemOperand(*I.memoperands_begin());

constrainSelectedInstRegOperands(*MIB, TII, TRI, RBI);
I.eraseFromParent();
return true;
1841}

1843void AArch64InstructionSelector::materializeLargeCMVal(
  MachineInstr &I, const Value *V, unsigned OpFlags) const {
MachineBasicBlock &MBB = *I.getParent();
MachineFunction &MF = *MBB.getParent();
MachineRegisterInfo &MRI = MF.getRegInfo();
MachineIRBuilder MIB(I);

auto MovZ = MIB.buildInstr(AArch64::MOVZXi, {&AArch64::GPR64RegClass}, {});
MovZ->addOperand(MF, I.getOperand(1));
MovZ->getOperand(1).setTargetFlags(OpFlags | AArch64II::MO_G0 |
                                   AArch64II::MO_NC);
MovZ->addOperand(MF, MachineOperand::CreateImm(0));
constrainSelectedInstRegOperands(*MovZ, TII, TRI, RBI);

auto BuildMovK = [&](Register SrcReg, unsigned char Flags, unsigned Offset,
                     Register ForceDstReg) {
  Register DstReg = ForceDstReg
                        ? ForceDstReg
                        : MRI.createVirtualRegister(&AArch64::GPR64RegClass);
  auto MovI = MIB.buildInstr(AArch64::MOVKXi).addDef(DstReg).addUse(SrcReg);
  if (auto *GV = dyn_cast<GlobalValue>(V)) {
    MovI->addOperand(MF, MachineOperand::CreateGA(
                             GV, MovZ->getOperand(1).getOffset(), Flags));
  } else {
    MovI->addOperand(
        MF, MachineOperand::CreateBA(cast<BlockAddress>(V),
                                     MovZ->getOperand(1).getOffset(), Flags));
  }
  MovI->addOperand(MF, MachineOperand::CreateImm(Offset));
  constrainSelectedInstRegOperands(*MovI, TII, TRI, RBI);
  return DstReg;
};
Register DstReg = BuildMovK(MovZ.getReg(0),
                            AArch64II::MO_G1 | AArch64II::MO_NC, 16, 0);
DstReg = BuildMovK(DstReg, AArch64II::MO_G2 | AArch64II::MO_NC, 32, 0);
BuildMovK(DstReg, AArch64II::MO_G3, 48, I.getOperand(0).getReg());
1879}

1881bool AArch64InstructionSelector::preISelLower(MachineInstr &I) {
MachineBasicBlock &MBB = *I.getParent();
MachineFunction &MF = *MBB.getParent();
MachineRegisterInfo &MRI = MF.getRegInfo();

switch (I.getOpcode()) {
case TargetOpcode::G_SHL:
case TargetOpcode::G_ASHR:
case TargetOpcode::G_LSHR: {
  // These shifts are legalized to have 64 bit shift amounts because we want
  // to take advantage of the existing imported selection patterns that assume
  // the immediates are s64s. However, if the shifted type is 32 bits and for
  // some reason we receive input GMIR that has an s64 shift amount that's not
  // a G_CONSTANT, insert a truncate so that we can still select the s32
  // register-register variant.
  Register SrcReg = I.getOperand(1).getReg();
  Register ShiftReg = I.getOperand(2).getReg();
  const LLT ShiftTy = MRI.getType(ShiftReg);
  const LLT SrcTy = MRI.getType(SrcReg);
  if (SrcTy.isVector())
    return false;
  assert(!ShiftTy.isVector() && "unexpected vector shift ty")((!ShiftTy.isVector() && "unexpected vector shift ty"
) ? static_cast<void> (0) : __assert_fail ("!ShiftTy.isVector() && \"unexpected vector shift ty\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1902, __PRETTY_FUNCTION__));
  if (SrcTy.getSizeInBits() != 32 || ShiftTy.getSizeInBits() != 64)
    return false;
  auto *AmtMI = MRI.getVRegDef(ShiftReg);
  assert(AmtMI && "could not find a vreg definition for shift amount")((AmtMI && "could not find a vreg definition for shift amount"
) ? static_cast<void> (0) : __assert_fail ("AmtMI && \"could not find a vreg definition for shift amount\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1906, __PRETTY_FUNCTION__));
  if (AmtMI->getOpcode() != TargetOpcode::G_CONSTANT) {
    // Insert a subregister copy to implement a 64->32 trunc
    MachineIRBuilder MIB(I);
    auto Trunc = MIB.buildInstr(TargetOpcode::COPY, {SrcTy}, {})
                     .addReg(ShiftReg, 0, AArch64::sub_32);
    MRI.setRegBank(Trunc.getReg(0), RBI.getRegBank(AArch64::GPRRegBankID));
    I.getOperand(2).setReg(Trunc.getReg(0));
  }
  return true;
}
case TargetOpcode::G_STORE:
  return contractCrossBankCopyIntoStore(I, MRI);
case TargetOpcode::G_PTR_ADD:
  return convertPtrAddToAdd(I, MRI);
case TargetOpcode::G_LOAD: {
  // For scalar loads of pointers, we try to convert the dest type from p0
  // to s64 so that our imported patterns can match. Like with the G_PTR_ADD
  // conversion, this should be ok because all users should have been
  // selected already, so the type doesn't matter for them.
  Register DstReg = I.getOperand(0).getReg();
  const LLT DstTy = MRI.getType(DstReg);
  if (!DstTy.isPointer())
    return false;
  MRI.setType(DstReg, LLT::scalar(64));
  return true;
}
case AArch64::G_DUP: {
  // Convert the type from p0 to s64 to help selection.
  LLT DstTy = MRI.getType(I.getOperand(0).getReg());
  if (!DstTy.getElementType().isPointer())
    return false;
  MachineIRBuilder MIB(I);
  auto NewSrc = MIB.buildCopy(LLT::scalar(64), I.getOperand(1).getReg());
  MRI.setType(I.getOperand(0).getReg(),
              DstTy.changeElementType(LLT::scalar(64)));
  MRI.setRegBank(NewSrc.getReg(0), RBI.getRegBank(AArch64::GPRRegBankID));
  I.getOperand(1).setReg(NewSrc.getReg(0));
  return true;
}
case TargetOpcode::G_UITOFP:
case TargetOpcode::G_SITOFP: {
  // If both source and destination regbanks are FPR, then convert the opcode
  // to G_SITOF so that the importer can select it to an fpr variant.
  // Otherwise, it ends up matching an fpr/gpr variant and adding a cross-bank
  // copy.
  Register SrcReg = I.getOperand(1).getReg();
  LLT SrcTy = MRI.getType(SrcReg);
  LLT DstTy = MRI.getType(I.getOperand(0).getReg());
  if (SrcTy.isVector() || SrcTy.getSizeInBits() != DstTy.getSizeInBits())
    return false;

  if (RBI.getRegBank(SrcReg, MRI, TRI)->getID() == AArch64::FPRRegBankID) {
    if (I.getOpcode() == TargetOpcode::G_SITOFP)
      I.setDesc(TII.get(AArch64::G_SITOF));
    else
      I.setDesc(TII.get(AArch64::G_UITOF));
    return true;
  }
  return false;
}
default:
  return false;
}
1970}

1972/// This lowering tries to look for G_PTR_ADD instructions and then converts
1973/// them to a standard G_ADD with a COPY on the source.
1974///
1975/// The motivation behind this is to expose the add semantics to the imported
1976/// tablegen patterns. We shouldn't need to check for uses being loads/stores,
1977/// because the selector works bottom up, uses before defs. By the time we
1978/// end up trying to select a G_PTR_ADD, we should have already attempted to
1979/// fold this into addressing modes and were therefore unsuccessful.
1980bool AArch64InstructionSelector::convertPtrAddToAdd(
  MachineInstr &I, MachineRegisterInfo &MRI) {
assert(I.getOpcode() == TargetOpcode::G_PTR_ADD && "Expected G_PTR_ADD")((I.getOpcode() == TargetOpcode::G_PTR_ADD && "Expected G_PTR_ADD"
) ? static_cast<void> (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_PTR_ADD && \"Expected G_PTR_ADD\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1982, __PRETTY_FUNCTION__));
Register DstReg = I.getOperand(0).getReg();
Register AddOp1Reg = I.getOperand(1).getReg();
const LLT PtrTy = MRI.getType(DstReg);
if (PtrTy.getAddressSpace() != 0)
  return false;

MachineIRBuilder MIB(I);
const LLT CastPtrTy = PtrTy.isVector() ? LLT::vector(2, 64) : LLT::scalar(64);
auto PtrToInt = MIB.buildPtrToInt(CastPtrTy, AddOp1Reg);
// Set regbanks on the registers.
if (PtrTy.isVector())
  MRI.setRegBank(PtrToInt.getReg(0), RBI.getRegBank(AArch64::FPRRegBankID));
else
  MRI.setRegBank(PtrToInt.getReg(0), RBI.getRegBank(AArch64::GPRRegBankID));

// Now turn the %dst(p0) = G_PTR_ADD %base, off into:
// %dst(intty) = G_ADD %intbase, off
I.setDesc(TII.get(TargetOpcode::G_ADD));
MRI.setType(DstReg, CastPtrTy);
I.getOperand(1).setReg(PtrToInt.getReg(0));
if (!select(*PtrToInt)) {
  LLVM_DEBUG(dbgs() << "Failed to select G_PTRTOINT in convertPtrAddToAdd")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Failed to select G_PTRTOINT in convertPtrAddToAdd"
; } } while (false);
  return false;
}

// Also take the opportunity here to try to do some optimization.
// Try to convert this into a G_SUB if the offset is a 0-x negate idiom.
Register NegatedReg;
if (!mi_match(I.getOperand(2).getReg(), MRI, m_Neg(m_Reg(NegatedReg))))
  return true;
I.getOperand(2).setReg(NegatedReg);
I.setDesc(TII.get(TargetOpcode::G_SUB));
return true;
2016}

2018bool AArch64InstructionSelector::earlySelectSHL(
  MachineInstr &I, MachineRegisterInfo &MRI) const {
// We try to match the immediate variant of LSL, which is actually an alias
// for a special case of UBFM. Otherwise, we fall back to the imported
// selector which will match the register variant.
assert(I.getOpcode() == TargetOpcode::G_SHL && "unexpected op")((I.getOpcode() == TargetOpcode::G_SHL && "unexpected op"
) ? static_cast<void> (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_SHL && \"unexpected op\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2023, __PRETTY_FUNCTION__));
const auto &MO = I.getOperand(2);
auto VRegAndVal = getConstantVRegVal(MO.getReg(), MRI);
if (!VRegAndVal)
  return false;

const LLT DstTy = MRI.getType(I.getOperand(0).getReg());
if (DstTy.isVector())
  return false;
bool Is64Bit = DstTy.getSizeInBits() == 64;
auto Imm1Fn = Is64Bit ? selectShiftA_64(MO) : selectShiftA_32(MO);
auto Imm2Fn = Is64Bit ? selectShiftB_64(MO) : selectShiftB_32(MO);
MachineIRBuilder MIB(I);

if (!Imm1Fn || !Imm2Fn)
  return false;

auto NewI =
    MIB.buildInstr(Is64Bit ? AArch64::UBFMXri : AArch64::UBFMWri,
                   {I.getOperand(0).getReg()}, {I.getOperand(1).getReg()});

for (auto &RenderFn : *Imm1Fn)
  RenderFn(NewI);
for (auto &RenderFn : *Imm2Fn)
  RenderFn(NewI);

I.eraseFromParent();
return constrainSelectedInstRegOperands(*NewI, TII, TRI, RBI);
2051}

2053bool AArch64InstructionSelector::contractCrossBankCopyIntoStore(
  MachineInstr &I, MachineRegisterInfo &MRI) {
assert(I.getOpcode() == TargetOpcode::G_STORE && "Expected G_STORE")((I.getOpcode() == TargetOpcode::G_STORE && "Expected G_STORE"
) ? static_cast<void> (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_STORE && \"Expected G_STORE\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2055, __PRETTY_FUNCTION__));
// If we're storing a scalar, it doesn't matter what register bank that
// scalar is on. All that matters is the size.
//
// So, if we see something like this (with a 32-bit scalar as an example):
//
// %x:gpr(s32) = ... something ...
// %y:fpr(s32) = COPY %x:gpr(s32)
// G_STORE %y:fpr(s32)
//
// We can fix this up into something like this:
//
// G_STORE %x:gpr(s32)
//
// And then continue the selection process normally.
Register DefDstReg = getSrcRegIgnoringCopies(I.getOperand(0).getReg(), MRI);
if (!DefDstReg.isValid())
  return false;
LLT DefDstTy = MRI.getType(DefDstReg);
Register StoreSrcReg = I.getOperand(0).getReg();
LLT StoreSrcTy = MRI.getType(StoreSrcReg);

// If we get something strange like a physical register, then we shouldn't
// go any further.
if (!DefDstTy.isValid())
  return false;

// Are the source and dst types the same size?
if (DefDstTy.getSizeInBits() != StoreSrcTy.getSizeInBits())
  return false;

if (RBI.getRegBank(StoreSrcReg, MRI, TRI) ==
    RBI.getRegBank(DefDstReg, MRI, TRI))
  return false;

// We have a cross-bank copy, which is entering a store. Let's fold it.
I.getOperand(0).setReg(DefDstReg);
return true;
2093}

2095bool AArch64InstructionSelector::earlySelect(MachineInstr &I) const {
assert(I.getParent() && "Instruction should be in a basic block!")((I.getParent() && "Instruction should be in a basic block!"
) ? static_cast<void> (0) : __assert_fail ("I.getParent() && \"Instruction should be in a basic block!\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2096, __PRETTY_FUNCTION__));
assert(I.getParent()->getParent() && "Instruction should be in a function!")((I.getParent()->getParent() && "Instruction should be in a function!"
) ? static_cast<void> (0) : __assert_fail ("I.getParent()->getParent() && \"Instruction should be in a function!\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2097, __PRETTY_FUNCTION__));

MachineBasicBlock &MBB = *I.getParent();
MachineFunction &MF = *MBB.getParent();
MachineRegisterInfo &MRI = MF.getRegInfo();

switch (I.getOpcode()) {
case TargetOpcode::G_BR: {
  // If the branch jumps to the fallthrough block, don't bother emitting it.
  // Only do this for -O0 for a good code size improvement, because when
  // optimizations are enabled we want to leave this choice to
  // MachineBlockPlacement.
  bool EnableOpt = MF.getTarget().getOptLevel() != CodeGenOpt::None;
  if (EnableOpt || !MBB.isLayoutSuccessor(I.getOperand(0).getMBB()))
    return false;
  I.eraseFromParent();
  return true;
}
case TargetOpcode::G_SHL:
  return earlySelectSHL(I, MRI);
case TargetOpcode::G_CONSTANT: {
  bool IsZero = false;
  if (I.getOperand(1).isCImm())
    IsZero = I.getOperand(1).getCImm()->getZExtValue() == 0;
  else if (I.getOperand(1).isImm())
    IsZero = I.getOperand(1).getImm() == 0;

  if (!IsZero)
    return false;

  Register DefReg = I.getOperand(0).getReg();
  LLT Ty = MRI.getType(DefReg);
  if (Ty.getSizeInBits() == 64) {
    I.getOperand(1).ChangeToRegister(AArch64::XZR, false);
    RBI.constrainGenericRegister(DefReg, AArch64::GPR64RegClass, MRI);
  } else if (Ty.getSizeInBits() == 32) {
    I.getOperand(1).ChangeToRegister(AArch64::WZR, false);
    RBI.constrainGenericRegister(DefReg, AArch64::GPR32RegClass, MRI);
  } else
    return false;

  I.setDesc(TII.get(TargetOpcode::COPY));
  return true;
}
default:
  return false;
}
2144}

2146bool AArch64InstructionSelector::select(MachineInstr &I) {
assert(I.getParent() && "Instruction should be in a basic block!")((I.getParent() && "Instruction should be in a basic block!"
) ? static_cast<void> (0) : __assert_fail ("I.getParent() && \"Instruction should be in a basic block!\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2147, __PRETTY_FUNCTION__));
assert(I.getParent()->getParent() && "Instruction should be in a function!")((I.getParent()->getParent() && "Instruction should be in a function!"
) ? static_cast<void> (0) : __assert_fail ("I.getParent()->getParent() && \"Instruction should be in a function!\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2148, __PRETTY_FUNCTION__));

MachineBasicBlock &MBB = *I.getParent();
MachineFunction &MF = *MBB.getParent();
MachineRegisterInfo &MRI = MF.getRegInfo();

const AArch64Subtarget *Subtarget =
    &static_cast<const AArch64Subtarget &>(MF.getSubtarget());
if (Subtarget->requiresStrictAlign()) {
  // We don't support this feature yet.
  LLVM_DEBUG(dbgs() << "AArch64 GISel does not support strict-align yet\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "AArch64 GISel does not support strict-align yet\n"
; } } while (false);
  return false;
}

unsigned Opcode = I.getOpcode();
// G_PHI requires same handling as PHI
if (!I.isPreISelOpcode() || Opcode == TargetOpcode::G_PHI) {
  // Certain non-generic instructions also need some special handling.

  if (Opcode ==  TargetOpcode::LOAD_STACK_GUARD)
    return constrainSelectedInstRegOperands(I, TII, TRI, RBI);

  if (Opcode == TargetOpcode::PHI || Opcode == TargetOpcode::G_PHI) {
    const Register DefReg = I.getOperand(0).getReg();
    const LLT DefTy = MRI.getType(DefReg);

    const RegClassOrRegBank &RegClassOrBank =
      MRI.getRegClassOrRegBank(DefReg);

    const TargetRegisterClass *DefRC
      = RegClassOrBank.dyn_cast<const TargetRegisterClass *>();
    if (!DefRC) {
      if (!DefTy.isValid()) {
        LLVM_DEBUG(dbgs() << "PHI operand has no type, not a gvreg?\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "PHI operand has no type, not a gvreg?\n"
; } } while (false);
        return false;
      }
      const RegisterBank &RB = *RegClassOrBank.get<const RegisterBank *>();
      DefRC = getRegClassForTypeOnBank(DefTy, RB, RBI);
      if (!DefRC) {
        LLVM_DEBUG(dbgs() << "PHI operand has unexpected size/bank\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "PHI operand has unexpected size/bank\n"
; } } while (false);
        return false;
      }
    }

    I.setDesc(TII.get(TargetOpcode::PHI));

    return RBI.constrainGenericRegister(DefReg, *DefRC, MRI);
  }

  if (I.isCopy())
    return selectCopy(I, TII, MRI, TRI, RBI);

  return true;
}


if (I.getNumOperands() != I.getNumExplicitOperands()) {
  LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Generic instruction has unexpected implicit operands\n"
; } } while (false)
      dbgs() << "Generic instruction has unexpected implicit operands\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Generic instruction has unexpected implicit operands\n"
; } } while (false);
  return false;
}

// Try to do some lowering before we start instruction selecting. These
// lowerings are purely transformations on the input G_MIR and so selection
// must continue after any modification of the instruction.
if (preISelLower(I)) {
  Opcode = I.getOpcode(); // The opcode may have been modified, refresh it.
}

// There may be patterns where the importer can't deal with them optimally,
// but does select it to a suboptimal sequence so our custom C++ selection
// code later never has a chance to work on it. Therefore, we have an early
// selection attempt here to give priority to certain selection routines
// over the imported ones.
if (earlySelect(I))
  return true;

if (selectImpl(I, *CoverageInfo))
  return true;

LLT Ty =
    I.getOperand(0).isReg() ? MRI.getType(I.getOperand(0).getReg()) : LLT{};

MachineIRBuilder MIB(I);

switch (Opcode) {
case TargetOpcode::G_BRCOND:
  return selectCompareBranch(I, MF, MRI);

case TargetOpcode::G_BRINDIRECT: {
  I.setDesc(TII.get(AArch64::BR));
  return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
}

case TargetOpcode::G_BRJT:
  return selectBrJT(I, MRI);

case AArch64::G_ADD_LOW: {
  // This op may have been separated from it's ADRP companion by the localizer
  // or some other code motion pass. Given that many CPUs will try to
  // macro fuse these operations anyway, select this into a MOVaddr pseudo
  // which will later be expanded into an ADRP+ADD pair after scheduling.
  MachineInstr *BaseMI = MRI.getVRegDef(I.getOperand(1).getReg());
  if (BaseMI->getOpcode() != AArch64::ADRP) {
    I.setDesc(TII.get(AArch64::ADDXri));
    I.addOperand(MachineOperand::CreateImm(0));
    return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
  }
  assert(TM.getCodeModel() == CodeModel::Small &&((TM.getCodeModel() == CodeModel::Small && "Expected small code model"
) ? static_cast<void> (0) : __assert_fail ("TM.getCodeModel() == CodeModel::Small && \"Expected small code model\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2257, __PRETTY_FUNCTION__))
         "Expected small code model")((TM.getCodeModel() == CodeModel::Small && "Expected small code model"
) ? static_cast<void> (0) : __assert_fail ("TM.getCodeModel() == CodeModel::Small && \"Expected small code model\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2257, __PRETTY_FUNCTION__));
  MachineIRBuilder MIB(I);
  auto Op1 = BaseMI->getOperand(1);
  auto Op2 = I.getOperand(2);
  auto MovAddr = MIB.buildInstr(AArch64::MOVaddr, {I.getOperand(0)}, {})
                     .addGlobalAddress(Op1.getGlobal(), Op1.getOffset(),
                                       Op1.getTargetFlags())
                     .addGlobalAddress(Op2.getGlobal(), Op2.getOffset(),
                                       Op2.getTargetFlags());
  I.eraseFromParent();
  return constrainSelectedInstRegOperands(*MovAddr, TII, TRI, RBI);
}

case TargetOpcode::G_BSWAP: {
  // Handle vector types for G_BSWAP directly.
  Register DstReg = I.getOperand(0).getReg();
  LLT DstTy = MRI.getType(DstReg);

  // We should only get vector types here; everything else is handled by the
  // importer right now.
  if (!DstTy.isVector() || DstTy.getSizeInBits() > 128) {
    LLVM_DEBUG(dbgs() << "Dst type for G_BSWAP currently unsupported.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Dst type for G_BSWAP currently unsupported.\n"
; } } while (false);
    return false;
  }

  // Only handle 4 and 2 element vectors for now.
  // TODO: 16-bit elements.
  unsigned NumElts = DstTy.getNumElements();
  if (NumElts != 4 && NumElts != 2) {
    LLVM_DEBUG(dbgs() << "Unsupported number of elements for G_BSWAP.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unsupported number of elements for G_BSWAP.\n"
; } } while (false);
    return false;
  }

  // Choose the correct opcode for the supported types. Right now, that's
  // v2s32, v4s32, and v2s64.
  unsigned Opc = 0;
  unsigned EltSize = DstTy.getElementType().getSizeInBits();
  if (EltSize == 32)
    Opc = (DstTy.getNumElements() == 2) ? AArch64::REV32v8i8
                                        : AArch64::REV32v16i8;
  else if (EltSize == 64)
    Opc = AArch64::REV64v16i8;

  // We should always get something by the time we get here...
  assert(Opc != 0 && "Didn't get an opcode for G_BSWAP?")((Opc != 0 && "Didn't get an opcode for G_BSWAP?") ? static_cast
<void> (0) : __assert_fail ("Opc != 0 && \"Didn't get an opcode for G_BSWAP?\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2301, __PRETTY_FUNCTION__));

  I.setDesc(TII.get(Opc));
  return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
}

case TargetOpcode::G_FCONSTANT:
case TargetOpcode::G_CONSTANT: {
  const bool isFP = Opcode == TargetOpcode::G_FCONSTANT;

  const LLT s8 = LLT::scalar(8);
  const LLT s16 = LLT::scalar(16);
  const LLT s32 = LLT::scalar(32);
  const LLT s64 = LLT::scalar(64);
  const LLT s128 = LLT::scalar(128);
  const LLT p0 = LLT::pointer(0, 64);

  const Register DefReg = I.getOperand(0).getReg();
  const LLT DefTy = MRI.getType(DefReg);
  const unsigned DefSize = DefTy.getSizeInBits();
  const RegisterBank &RB = *RBI.getRegBank(DefReg, MRI, TRI);

  // FIXME: Redundant check, but even less readable when factored out.
  if (isFP) {
    if (Ty != s32 && Ty != s64 && Ty != s128) {
      LLVM_DEBUG(dbgs() << "Unable to materialize FP " << Tydo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unable to materialize FP "
 << Ty << " constant, expected: " << s32 <<
 " or " << s64 << " or " << s128 << '\n'
; } } while (false)
                        << " constant, expected: " << s32 << " or " << s64do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unable to materialize FP "
 << Ty << " constant, expected: " << s32 <<
 " or " << s64 << " or " << s128 << '\n'
; } } while (false)
                        << " or " << s128 << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unable to materialize FP "
 << Ty << " constant, expected: " << s32 <<
 " or " << s64 << " or " << s128 << '\n'
; } } while (false);
      return false;
    }

    if (RB.getID() != AArch64::FPRRegBankID) {
      LLVM_DEBUG(dbgs() << "Unable to materialize FP " << Tydo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unable to materialize FP "
 << Ty << " constant on bank: " << RB <<
 ", expected: FPR\n"; } } while (false)
                        << " constant on bank: " << RBdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unable to materialize FP "
 << Ty << " constant on bank: " << RB <<
 ", expected: FPR\n"; } } while (false)
                        << ", expected: FPR\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unable to materialize FP "
 << Ty << " constant on bank: " << RB <<
 ", expected: FPR\n"; } } while (false);
      return false;
    }

    // The case when we have 0.0 is covered by tablegen. Reject it here so we
    // can be sure tablegen works correctly and isn't rescued by this code.
    // 0.0 is not covered by tablegen for FP128. So we will handle this 
    // scenario in the code here.
    if (DefSize != 128 && I.getOperand(1).getFPImm()->isExactlyValue(0.0))
      return false;
  } else {
    // s32 and s64 are covered by tablegen.
    if (Ty != p0 && Ty != s8 && Ty != s16) {
      LLVM_DEBUG(dbgs() << "Unable to materialize integer " << Tydo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unable to materialize integer "
 << Ty << " constant, expected: " << s32 <<
 ", " << s64 << ", or " << p0 << '\n'
; } } while (false)
                        << " constant, expected: " << s32 << ", " << s64do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unable to materialize integer "
 << Ty << " constant, expected: " << s32 <<
 ", " << s64 << ", or " << p0 << '\n'
; } } while (false)
                        << ", or " << p0 << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unable to materialize integer "
 << Ty << " constant, expected: " << s32 <<
 ", " << s64 << ", or " << p0 << '\n'
; } } while (false);
      return false;
    }

    if (RB.getID() != AArch64::GPRRegBankID) {
      LLVM_DEBUG(dbgs() << "Unable to materialize integer " << Tydo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unable to materialize integer "
 << Ty << " constant on bank: " << RB <<
 ", expected: GPR\n"; } } while (false)
                        << " constant on bank: " << RBdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unable to materialize integer "
 << Ty << " constant on bank: " << RB <<
 ", expected: GPR\n"; } } while (false)
                        << ", expected: GPR\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unable to materialize integer "
 << Ty << " constant on bank: " << RB <<
 ", expected: GPR\n"; } } while (false);
      return false;
    }
  }

  // We allow G_CONSTANT of types < 32b.
  const unsigned MovOpc =
      DefSize == 64 ? AArch64::MOVi64imm : AArch64::MOVi32imm;

  if (isFP) {
    // Either emit a FMOV, or emit a copy to emit a normal mov.
    const TargetRegisterClass &GPRRC =
        DefSize == 32 ? AArch64::GPR32RegClass : AArch64::GPR64RegClass;
    const TargetRegisterClass &FPRRC = 
        DefSize == 32 ? AArch64::FPR32RegClass 
                      : (DefSize == 64 ? AArch64::FPR64RegClass 
                                       : AArch64::FPR128RegClass);

    // Can we use a FMOV instruction to represent the immediate?
    if (emitFMovForFConstant(I, MRI))
      return true;

    // For 64b values, emit a constant pool load instead.
    if (DefSize == 64 || DefSize == 128) {
      auto *FPImm = I.getOperand(1).getFPImm();
      MachineIRBuilder MIB(I);
      auto *LoadMI = emitLoadFromConstantPool(FPImm, MIB);
      if (!LoadMI) {
        LLVM_DEBUG(dbgs() << "Failed to load double constant pool entry\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Failed to load double constant pool entry\n"
; } } while (false);
        return false;
      }
      MIB.buildCopy({DefReg}, {LoadMI->getOperand(0).getReg()});
      I.eraseFromParent();
      return RBI.constrainGenericRegister(DefReg, FPRRC, MRI);
    }

    // Nope. Emit a copy and use a normal mov instead.
    const Register DefGPRReg = MRI.createVirtualRegister(&GPRRC);
    MachineOperand &RegOp = I.getOperand(0);
    RegOp.setReg(DefGPRReg);
    MIB.setInsertPt(MIB.getMBB(), std::next(I.getIterator()));
    MIB.buildCopy({DefReg}, {DefGPRReg});

    if (!RBI.constrainGenericRegister(DefReg, FPRRC, MRI)) {
      LLVM_DEBUG(dbgs() << "Failed to constrain G_FCONSTANT def operand\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Failed to constrain G_FCONSTANT def operand\n"
; } } while (false);
      return false;
    }

    MachineOperand &ImmOp = I.getOperand(1);
    // FIXME: Is going through int64_t always correct?
    ImmOp.ChangeToImmediate(
        ImmOp.getFPImm()->getValueAPF().bitcastToAPInt().getZExtValue());
  } else if (I.getOperand(1).isCImm()) {
    uint64_t Val = I.getOperand(1).getCImm()->getZExtValue();
    I.getOperand(1).ChangeToImmediate(Val);
  } else if (I.getOperand(1).isImm()) {
    uint64_t Val = I.getOperand(1).getImm();
    I.getOperand(1).ChangeToImmediate(Val);
  }

  I.setDesc(TII.get(MovOpc));
  constrainSelectedInstRegOperands(I, TII, TRI, RBI);
  return true;
}
case TargetOpcode::G_EXTRACT: {
  Register DstReg = I.getOperand(0).getReg();
  Register SrcReg = I.getOperand(1).getReg();
  LLT SrcTy = MRI.getType(SrcReg);
  LLT DstTy = MRI.getType(DstReg);
  (void)DstTy;
  unsigned SrcSize = SrcTy.getSizeInBits();

  if (SrcTy.getSizeInBits() > 64) {
    // This should be an extract of an s128, which is like a vector extract.
    if (SrcTy.getSizeInBits() != 128)
      return false;
    // Only support extracting 64 bits from an s128 at the moment.
    if (DstTy.getSizeInBits() != 64)
      return false;

    const RegisterBank &SrcRB = *RBI.getRegBank(SrcReg, MRI, TRI);
    const RegisterBank &DstRB = *RBI.getRegBank(DstReg, MRI, TRI);
    // Check we have the right regbank always.
    assert(SrcRB.getID() == AArch64::FPRRegBankID &&((SrcRB.getID() == AArch64::FPRRegBankID && DstRB.getID
() == AArch64::FPRRegBankID && "Wrong extract regbank!"
) ? static_cast<void> (0) : __assert_fail ("SrcRB.getID() == AArch64::FPRRegBankID && DstRB.getID() == AArch64::FPRRegBankID && \"Wrong extract regbank!\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2442, __PRETTY_FUNCTION__))
           DstRB.getID() == AArch64::FPRRegBankID &&((SrcRB.getID() == AArch64::FPRRegBankID && DstRB.getID
() == AArch64::FPRRegBankID && "Wrong extract regbank!"
) ? static_cast<void> (0) : __assert_fail ("SrcRB.getID() == AArch64::FPRRegBankID && DstRB.getID() == AArch64::FPRRegBankID && \"Wrong extract regbank!\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2442, __PRETTY_FUNCTION__))
           "Wrong extract regbank!")((SrcRB.getID() == AArch64::FPRRegBankID && DstRB.getID
() == AArch64::FPRRegBankID && "Wrong extract regbank!"
) ? static_cast<void> (0) : __assert_fail ("SrcRB.getID() == AArch64::FPRRegBankID && DstRB.getID() == AArch64::FPRRegBankID && \"Wrong extract regbank!\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2442, __PRETTY_FUNCTION__));
    (void)SrcRB;

    // Emit the same code as a vector extract.
    // Offset must be a multiple of 64.
    unsigned Offset = I.getOperand(2).getImm();
    if (Offset % 64 != 0)
      return false;
    unsigned LaneIdx = Offset / 64;
    MachineIRBuilder MIB(I);
    MachineInstr *Extract = emitExtractVectorElt(
        DstReg, DstRB, LLT::scalar(64), SrcReg, LaneIdx, MIB);
    if (!Extract)
      return false;
    I.eraseFromParent();
    return true;
  }

  I.setDesc(TII.get(SrcSize == 64 ? AArch64::UBFMXri : AArch64::UBFMWri));
  MachineInstrBuilder(MF, I).addImm(I.getOperand(2).getImm() +
                                    Ty.getSizeInBits() - 1);

  if (SrcSize < 64) {
    assert(SrcSize == 32 && DstTy.getSizeInBits() == 16 &&((SrcSize == 32 && DstTy.getSizeInBits() == 16 &&
 "unexpected G_EXTRACT types") ? static_cast<void> (0) :
 __assert_fail ("SrcSize == 32 && DstTy.getSizeInBits() == 16 && \"unexpected G_EXTRACT types\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2466, __PRETTY_FUNCTION__))
           "unexpected G_EXTRACT types")((SrcSize == 32 && DstTy.getSizeInBits() == 16 &&
 "unexpected G_EXTRACT types") ? static_cast<void> (0) :
 __assert_fail ("SrcSize == 32 && DstTy.getSizeInBits() == 16 && \"unexpected G_EXTRACT types\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2466, __PRETTY_FUNCTION__));
    return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
  }

  DstReg = MRI.createGenericVirtualRegister(LLT::scalar(64));
  MIB.setInsertPt(MIB.getMBB(), std::next(I.getIterator()));
  MIB.buildInstr(TargetOpcode::COPY, {I.getOperand(0).getReg()}, {})
      .addReg(DstReg, 0, AArch64::sub_32);
  RBI.constrainGenericRegister(I.getOperand(0).getReg(),
                               AArch64::GPR32RegClass, MRI);
  I.getOperand(0).setReg(DstReg);

  return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
}

case TargetOpcode::G_INSERT: {
  LLT SrcTy = MRI.getType(I.getOperand(2).getReg());
  LLT DstTy = MRI.getType(I.getOperand(0).getReg());
  unsigned DstSize = DstTy.getSizeInBits();
  // Larger inserts are vectors, same-size ones should be something else by
  // now (split up or turned into COPYs).
  if (Ty.getSizeInBits() > 64 || SrcTy.getSizeInBits() > 32)
    return false;

  I.setDesc(TII.get(DstSize == 64 ? AArch64::BFMXri : AArch64::BFMWri));
  unsigned LSB = I.getOperand(3).getImm();
  unsigned Width = MRI.getType(I.getOperand(2).getReg()).getSizeInBits();
  I.getOperand(3).setImm((DstSize - LSB) % DstSize);
  MachineInstrBuilder(MF, I).addImm(Width - 1);

  if (DstSize < 64) {
    assert(DstSize == 32 && SrcTy.getSizeInBits() == 16 &&((DstSize == 32 && SrcTy.getSizeInBits() == 16 &&
 "unexpected G_INSERT types") ? static_cast<void> (0) :
 __assert_fail ("DstSize == 32 && SrcTy.getSizeInBits() == 16 && \"unexpected G_INSERT types\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2498, __PRETTY_FUNCTION__))
           "unexpected G_INSERT types")((DstSize == 32 && SrcTy.getSizeInBits() == 16 &&
 "unexpected G_INSERT types") ? static_cast<void> (0) :
 __assert_fail ("DstSize == 32 && SrcTy.getSizeInBits() == 16 && \"unexpected G_INSERT types\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2498, __PRETTY_FUNCTION__));
    return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
  }

  Register SrcReg = MRI.createGenericVirtualRegister(LLT::scalar(64));
  BuildMI(MBB, I.getIterator(), I.getDebugLoc(),
          TII.get(AArch64::SUBREG_TO_REG))
      .addDef(SrcReg)
      .addImm(0)
      .addUse(I.getOperand(2).getReg())
      .addImm(AArch64::sub_32);
  RBI.constrainGenericRegister(I.getOperand(2).getReg(),
                               AArch64::GPR32RegClass, MRI);
  I.getOperand(2).setReg(SrcReg);

  return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
}
case TargetOpcode::G_FRAME_INDEX: {
  // allocas and G_FRAME_INDEX are only supported in addrspace(0).
  if (Ty != LLT::pointer(0, 64)) {
    LLVM_DEBUG(dbgs() << "G_FRAME_INDEX pointer has type: " << Tydo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "G_FRAME_INDEX pointer has type: "
 << Ty << ", expected: " << LLT::pointer(0,
 64) << '\n'; } } while (false)
                      << ", expected: " << LLT::pointer(0, 64) << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "G_FRAME_INDEX pointer has type: "
 << Ty << ", expected: " << LLT::pointer(0,
 64) << '\n'; } } while (false);
    return false;
  }
  I.setDesc(TII.get(AArch64::ADDXri));

  // MOs for a #0 shifted immediate.
  I.addOperand(MachineOperand::CreateImm(0));
  I.addOperand(MachineOperand::CreateImm(0));

  return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
}

case TargetOpcode::G_GLOBAL_VALUE: {
  auto GV = I.getOperand(1).getGlobal();
  if (GV->isThreadLocal())
    return selectTLSGlobalValue(I, MRI);

  unsigned OpFlags = STI.ClassifyGlobalReference(GV, TM);
  if (OpFlags & AArch64II::MO_GOT) {
    I.setDesc(TII.get(AArch64::LOADgot));
    I.getOperand(1).setTargetFlags(OpFlags);
  } else if (TM.getCodeModel() == CodeModel::Large) {
    // Materialize the global using movz/movk instructions.
    materializeLargeCMVal(I, GV, OpFlags);
    I.eraseFromParent();
    return true;
  } else if (TM.getCodeModel() == CodeModel::Tiny) {
    I.setDesc(TII.get(AArch64::ADR));
    I.getOperand(1).setTargetFlags(OpFlags);
  } else {
    I.setDesc(TII.get(AArch64::MOVaddr));
    I.getOperand(1).setTargetFlags(OpFlags | AArch64II::MO_PAGE);
    MachineInstrBuilder MIB(MF, I);
    MIB.addGlobalAddress(GV, I.getOperand(1).getOffset(),
                         OpFlags | AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
  }
  return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
}

case TargetOpcode::G_ZEXTLOAD:
case TargetOpcode::G_LOAD:
case TargetOpcode::G_STORE: {
  bool IsZExtLoad = I.getOpcode() == TargetOpcode::G_ZEXTLOAD;
  MachineIRBuilder MIB(I);

  LLT PtrTy = MRI.getType(I.getOperand(1).getReg());

  if (PtrTy != LLT::pointer(0, 64)) {
    LLVM_DEBUG(dbgs() << "Load/Store pointer has type: " << PtrTydo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Load/Store pointer has type: "
 << PtrTy << ", expected: " << LLT::pointer
(0, 64) << '\n'; } } while (false)
                      << ", expected: " << LLT::pointer(0, 64) << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Load/Store pointer has type: "
 << PtrTy << ", expected: " << LLT::pointer
(0, 64) << '\n'; } } while (false);
    return false;
  }

  auto &MemOp = **I.memoperands_begin();
  uint64_t MemSizeInBytes = MemOp.getSize();
  if (MemOp.isAtomic()) {
    // For now we just support s8 acquire loads to be able to compile stack
    // protector code.
    if (MemOp.getOrdering() == AtomicOrdering::Acquire &&
        MemSizeInBytes == 1) {
      I.setDesc(TII.get(AArch64::LDARB));
      return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
    }
    LLVM_DEBUG(dbgs() << "Atomic load/store not fully supported yet\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Atomic load/store not fully supported yet\n"
; } } while (false);
    return false;
  }
  unsigned MemSizeInBits = MemSizeInBytes * 8;

2587#ifndef NDEBUG
  const Register PtrReg = I.getOperand(1).getReg();
  const RegisterBank &PtrRB = *RBI.getRegBank(PtrReg, MRI, TRI);
  // Sanity-check the pointer register.
  assert(PtrRB.getID() == AArch64::GPRRegBankID &&((PtrRB.getID() == AArch64::GPRRegBankID && "Load/Store pointer operand isn't a GPR"
) ? static_cast<void> (0) : __assert_fail ("PtrRB.getID() == AArch64::GPRRegBankID && \"Load/Store pointer operand isn't a GPR\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2592, __PRETTY_FUNCTION__))
         "Load/Store pointer operand isn't a GPR")((PtrRB.getID() == AArch64::GPRRegBankID && "Load/Store pointer operand isn't a GPR"
) ? static_cast<void> (0) : __assert_fail ("PtrRB.getID() == AArch64::GPRRegBankID && \"Load/Store pointer operand isn't a GPR\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2592, __PRETTY_FUNCTION__));
  assert(MRI.getType(PtrReg).isPointer() &&((MRI.getType(PtrReg).isPointer() && "Load/Store pointer operand isn't a pointer"
) ? static_cast<void> (0) : __assert_fail ("MRI.getType(PtrReg).isPointer() && \"Load/Store pointer operand isn't a pointer\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2594, __PRETTY_FUNCTION__))
         "Load/Store pointer operand isn't a pointer")((MRI.getType(PtrReg).isPointer() && "Load/Store pointer operand isn't a pointer"
) ? static_cast<void> (0) : __assert_fail ("MRI.getType(PtrReg).isPointer() && \"Load/Store pointer operand isn't a pointer\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2594, __PRETTY_FUNCTION__));
2595#endif

  const Register ValReg = I.getOperand(0).getReg();
  const RegisterBank &RB = *RBI.getRegBank(ValReg, MRI, TRI);

  // Helper lambda for partially selecting I. Either returns the original
  // instruction with an updated opcode, or a new instruction.
  auto SelectLoadStoreAddressingMode = [&]() -> MachineInstr * {
    bool IsStore = I.getOpcode() == TargetOpcode::G_STORE;
    const unsigned NewOpc =
        selectLoadStoreUIOp(I.getOpcode(), RB.getID(), MemSizeInBits);
    if (NewOpc == I.getOpcode())
      return nullptr;
    // Check if we can fold anything into the addressing mode.
    auto AddrModeFns =
        selectAddrModeIndexed(I.getOperand(1), MemSizeInBytes);
    if (!AddrModeFns) {
      // Can't fold anything. Use the original instruction.
      I.setDesc(TII.get(NewOpc));
      I.addOperand(MachineOperand::CreateImm(0));
      return &I;
    }

    // Folded something. Create a new instruction and return it.
    auto NewInst = MIB.buildInstr(NewOpc, {}, {}, I.getFlags());
    IsStore ? NewInst.addUse(ValReg) : NewInst.addDef(ValReg);
    NewInst.cloneMemRefs(I);
    for (auto &Fn : *AddrModeFns)
      Fn(NewInst);
    I.eraseFromParent();
    return &*NewInst;
  };

  MachineInstr *LoadStore = SelectLoadStoreAddressingMode();
  if (!LoadStore)
    return false;

  // If we're storing a 0, use WZR/XZR.
  if (Opcode == TargetOpcode::G_STORE) {
    auto CVal = getConstantVRegValWithLookThrough(
        LoadStore->getOperand(0).getReg(), MRI, /*LookThroughInstrs = */ true,
        /*HandleFConstants = */ false);
    if (CVal && CVal->Value == 0) {
      switch (LoadStore->getOpcode()) {
      case AArch64::STRWui:
      case AArch64::STRHHui:
      case AArch64::STRBBui:
        LoadStore->getOperand(0).setReg(AArch64::WZR);
        break;
      case AArch64::STRXui:
        LoadStore->getOperand(0).setReg(AArch64::XZR);
        break;
      }
    }
  }

  if (IsZExtLoad) {
    // The zextload from a smaller type to i32 should be handled by the
    // importer.
    if (MRI.getType(LoadStore->getOperand(0).getReg()).getSizeInBits() != 64)
      return false;
    // If we have a ZEXTLOAD then change the load's type to be a narrower reg
    // and zero_extend with SUBREG_TO_REG.
    Register LdReg = MRI.createVirtualRegister(&AArch64::GPR32RegClass);
    Register DstReg = LoadStore->getOperand(0).getReg();
    LoadStore->getOperand(0).setReg(LdReg);

    MIB.setInsertPt(MIB.getMBB(), std::next(LoadStore->getIterator()));
    MIB.buildInstr(AArch64::SUBREG_TO_REG, {DstReg}, {})
        .addImm(0)
        .addUse(LdReg)
        .addImm(AArch64::sub_32);
    constrainSelectedInstRegOperands(*LoadStore, TII, TRI, RBI);
    return RBI.constrainGenericRegister(DstReg, AArch64::GPR64allRegClass,
                                        MRI);
  }
  return constrainSelectedInstRegOperands(*LoadStore, TII, TRI, RBI);
}

case TargetOpcode::G_SMULH:
case TargetOpcode::G_UMULH: {
  // Reject the various things we don't support yet.
  if (unsupportedBinOp(I, RBI, MRI, TRI))
    return false;

  const Register DefReg = I.getOperand(0).getReg();
  const RegisterBank &RB = *RBI.getRegBank(DefReg, MRI, TRI);

  if (RB.getID() != AArch64::GPRRegBankID) {
    LLVM_DEBUG(dbgs() << "G_[SU]MULH on bank: " << RB << ", expected: GPR\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "G_[SU]MULH on bank: " <<
 RB << ", expected: GPR\n"; } } while (false);
    return false;
  }

  if (Ty != LLT::scalar(64)) {
    LLVM_DEBUG(dbgs() << "G_[SU]MULH has type: " << Tydo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "G_[SU]MULH has type: " <<
 Ty << ", expected: " << LLT::scalar(64) <<
 '\n'; } } while (false)
                      << ", expected: " << LLT::scalar(64) << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "G_[SU]MULH has type: " <<
 Ty << ", expected: " << LLT::scalar(64) <<
 '\n'; } } while (false);
    return false;
  }

  unsigned NewOpc = I.getOpcode() == TargetOpcode::G_SMULH ? AArch64::SMULHrr
                                                           : AArch64::UMULHrr;
  I.setDesc(TII.get(NewOpc));

  // Now that we selected an opcode, we need to constrain the register
  // operands to use appropriate classes.
  return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
}
case TargetOpcode::G_LSHR:
case TargetOpcode::G_ASHR:
  if (MRI.getType(I.getOperand(0).getReg()).isVector())
    return selectVectorAshrLshr(I, MRI);
  LLVM_FALLTHROUGH[[gnu::fallthrough]];
case TargetOpcode::G_SHL:
  if (Opcode == TargetOpcode::G_SHL &&
      MRI.getType(I.getOperand(0).getReg()).isVector())
    return selectVectorSHL(I, MRI);
  LLVM_FALLTHROUGH[[gnu::fallthrough]];
case TargetOpcode::G_FADD:
case TargetOpcode::G_FSUB:
case TargetOpcode::G_FMUL:
case TargetOpcode::G_FDIV:
case TargetOpcode::G_OR: {
  // Reject the various things we don't support yet.
  if (unsupportedBinOp(I, RBI, MRI, TRI))
    return false;

  const unsigned OpSize = Ty.getSizeInBits();

  const Register DefReg = I.getOperand(0).getReg();
  const RegisterBank &RB = *RBI.getRegBank(DefReg, MRI, TRI);

  const unsigned NewOpc = selectBinaryOp(I.getOpcode(), RB.getID(), OpSize);
  if (NewOpc == I.getOpcode())
    return false;

  I.setDesc(TII.get(NewOpc));
  // FIXME: Should the type be always reset in setDesc?

  // Now that we selected an opcode, we need to constrain the register
  // operands to use appropriate classes.
  return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
}

case TargetOpcode::G_PTR_ADD: {
  MachineIRBuilder MIRBuilder(I);
  emitADD(I.getOperand(0).getReg(), I.getOperand(1), I.getOperand(2),
          MIRBuilder);
  I.eraseFromParent();
  return true;
}
case TargetOpcode::G_SADDO:
case TargetOpcode::G_UADDO:
case TargetOpcode::G_SSUBO: {
  // Emit the operation and get the correct condition code.
  MachineIRBuilder MIRBuilder(I);
  auto OpAndCC = emitOverflowOp(Opcode, I.getOperand(0).getReg(),
                                I.getOperand(2), I.getOperand(3), MIRBuilder);

  // Now, put the overflow result in the register given by the first operand
  // to the overflow op. CSINC increments the result when the predicate is
  // false, so to get the increment when it's true, we need to use the
  // inverse. In this case, we want to increment when carry is set.
  Register ZReg = AArch64::WZR;
  auto CsetMI = MIRBuilder
                    .buildInstr(AArch64::CSINCWr, {I.getOperand(1).getReg()},
                                {ZReg, ZReg})
                    .addImm(getInvertedCondCode(OpAndCC.second));
  constrainSelectedInstRegOperands(*CsetMI, TII, TRI, RBI);
  I.eraseFromParent();
  return true;
}

case TargetOpcode::G_PTRMASK: {
  Register MaskReg = I.getOperand(2).getReg();
  Optional<int64_t> MaskVal = getConstantVRegSExtVal(MaskReg, MRI);
  // TODO: Implement arbitrary cases
  if (!MaskVal || !isShiftedMask_64(*MaskVal))
    return false;

  uint64_t Mask = *MaskVal;
  I.setDesc(TII.get(AArch64::ANDXri));
  I.getOperand(2).ChangeToImmediate(
      AArch64_AM::encodeLogicalImmediate(Mask, 64));

  return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
}
case TargetOpcode::G_PTRTOINT:
case TargetOpcode::G_TRUNC: {
  const LLT DstTy = MRI.getType(I.getOperand(0).getReg());
  const LLT SrcTy = MRI.getType(I.getOperand(1).getReg());

  const Register DstReg = I.getOperand(0).getReg();
  const Register SrcReg = I.getOperand(1).getReg();

  const RegisterBank &DstRB = *RBI.getRegBank(DstReg, MRI, TRI);
  const RegisterBank &SrcRB = *RBI.getRegBank(SrcReg, MRI, TRI);

  if (DstRB.getID() != SrcRB.getID()) {
    LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "G_TRUNC/G_PTRTOINT input/output on different banks\n"
; } } while (false)
        dbgs() << "G_TRUNC/G_PTRTOINT input/output on different banks\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "G_TRUNC/G_PTRTOINT input/output on different banks\n"
; } } while (false);
    return false;
  }

  if (DstRB.getID() == AArch64::GPRRegBankID) {
    const TargetRegisterClass *DstRC =
        getRegClassForTypeOnBank(DstTy, DstRB, RBI);
    if (!DstRC)
      return false;

    const TargetRegisterClass *SrcRC =
        getRegClassForTypeOnBank(SrcTy, SrcRB, RBI);
    if (!SrcRC)
      return false;

    if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, MRI) ||
        !RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) {
      LLVM_DEBUG(dbgs() << "Failed to constrain G_TRUNC/G_PTRTOINT\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Failed to constrain G_TRUNC/G_PTRTOINT\n"
; } } while (false);
      return false;
    }

    if (DstRC == SrcRC) {
      // Nothing to be done
    } else if (Opcode == TargetOpcode::G_TRUNC && DstTy == LLT::scalar(32) &&
               SrcTy == LLT::scalar(64)) {
      llvm_unreachable("TableGen can import this case")::llvm::llvm_unreachable_internal("TableGen can import this case"
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2819);
      return false;
    } else if (DstRC == &AArch64::GPR32RegClass &&
               SrcRC == &AArch64::GPR64RegClass) {
      I.getOperand(1).setSubReg(AArch64::sub_32);
    } else {
      LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unhandled mismatched classes in G_TRUNC/G_PTRTOINT\n"
; } } while (false)
          dbgs() << "Unhandled mismatched classes in G_TRUNC/G_PTRTOINT\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unhandled mismatched classes in G_TRUNC/G_PTRTOINT\n"
; } } while (false);
      return false;
    }

    I.setDesc(TII.get(TargetOpcode::COPY));
    return true;
  } else if (DstRB.getID() == AArch64::FPRRegBankID) {
    if (DstTy == LLT::vector(4, 16) && SrcTy == LLT::vector(4, 32)) {
      I.setDesc(TII.get(AArch64::XTNv4i16));
      constrainSelectedInstRegOperands(I, TII, TRI, RBI);
      return true;
    }

    if (!SrcTy.isVector() && SrcTy.getSizeInBits() == 128) {
      MachineIRBuilder MIB(I);
      MachineInstr *Extract = emitExtractVectorElt(
          DstReg, DstRB, LLT::scalar(DstTy.getSizeInBits()), SrcReg, 0, MIB);
      if (!Extract)
        return false;
      I.eraseFromParent();
      return true;
    }

    // We might have a vector G_PTRTOINT, in which case just emit a COPY.
    if (Opcode == TargetOpcode::G_PTRTOINT) {
      assert(DstTy.isVector() && "Expected an FPR ptrtoint to be a vector")((DstTy.isVector() && "Expected an FPR ptrtoint to be a vector"
) ? static_cast<void> (0) : __assert_fail ("DstTy.isVector() && \"Expected an FPR ptrtoint to be a vector\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2851, __PRETTY_FUNCTION__));
      I.setDesc(TII.get(TargetOpcode::COPY));
      return true;
    }
  }

  return false;
}

case TargetOpcode::G_ANYEXT: {
  const Register DstReg = I.getOperand(0).getReg();
  const Register SrcReg = I.getOperand(1).getReg();

  const RegisterBank &RBDst = *RBI.getRegBank(DstReg, MRI, TRI);
  if (RBDst.getID() != AArch64::GPRRegBankID) {
    LLVM_DEBUG(dbgs() << "G_ANYEXT on bank: " << RBDstdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "G_ANYEXT on bank: " <<
 RBDst << ", expected: GPR\n"; } } while (false)
                      << ", expected: GPR\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "G_ANYEXT on bank: " <<
 RBDst << ", expected: GPR\n"; } } while (false);
    return false;
  }

  const RegisterBank &RBSrc = *RBI.getRegBank(SrcReg, MRI, TRI);
  if (RBSrc.getID() != AArch64::GPRRegBankID) {
    LLVM_DEBUG(dbgs() << "G_ANYEXT on bank: " << RBSrcdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "G_ANYEXT on bank: " <<
 RBSrc << ", expected: GPR\n"; } } while (false)
                      << ", expected: GPR\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "G_ANYEXT on bank: " <<
 RBSrc << ", expected: GPR\n"; } } while (false);
    return false;
  }

  const unsigned DstSize = MRI.getType(DstReg).getSizeInBits();

  if (DstSize == 0) {
    LLVM_DEBUG(dbgs() << "G_ANYEXT operand has no size, not a gvreg?\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "G_ANYEXT operand has no size, not a gvreg?\n"
; } } while (false);
    return false;
  }

  if (DstSize != 64 && DstSize > 32) {
    LLVM_DEBUG(dbgs() << "G_ANYEXT to size: " << DstSizedo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "G_ANYEXT to size: " <<
 DstSize << ", expected: 32 or 64\n"; } } while (false)
                      << ", expected: 32 or 64\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "G_ANYEXT to size: " <<
 DstSize << ", expected: 32 or 64\n"; } } while (false);
    return false;
  }
  // At this point G_ANYEXT is just like a plain COPY, but we need
  // to explicitly form the 64-bit value if any.
  if (DstSize > 32) {
    Register ExtSrc = MRI.createVirtualRegister(&AArch64::GPR64allRegClass);
    BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::SUBREG_TO_REG))
        .addDef(ExtSrc)
        .addImm(0)
        .addUse(SrcReg)
        .addImm(AArch64::sub_32);
    I.getOperand(1).setReg(ExtSrc);
  }
  return selectCopy(I, TII, MRI, TRI, RBI);
}

case TargetOpcode::G_ZEXT:
case TargetOpcode::G_SEXT_INREG:
case TargetOpcode::G_SEXT: {
  unsigned Opcode = I.getOpcode();
  const bool IsSigned = Opcode != TargetOpcode::G_ZEXT;
  const Register DefReg = I.getOperand(0).getReg();
  Register SrcReg = I.getOperand(1).getReg();
  const LLT DstTy = MRI.getType(DefReg);
  const LLT SrcTy = MRI.getType(SrcReg);
  unsigned DstSize = DstTy.getSizeInBits();
  unsigned SrcSize = SrcTy.getSizeInBits();

  // SEXT_INREG has the same src reg size as dst, the size of the value to be
  // extended is encoded in the imm.
  if (Opcode == TargetOpcode::G_SEXT_INREG)
    SrcSize = I.getOperand(2).getImm();

  if (DstTy.isVector())
    return false; // Should be handled by imported patterns.

  assert((*RBI.getRegBank(DefReg, MRI, TRI)).getID() ==(((*RBI.getRegBank(DefReg, MRI, TRI)).getID() == AArch64::GPRRegBankID
 && "Unexpected ext regbank") ? static_cast<void>
 (0) : __assert_fail ("(*RBI.getRegBank(DefReg, MRI, TRI)).getID() == AArch64::GPRRegBankID && \"Unexpected ext regbank\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2926, __PRETTY_FUNCTION__))
             AArch64::GPRRegBankID &&(((*RBI.getRegBank(DefReg, MRI, TRI)).getID() == AArch64::GPRRegBankID
 && "Unexpected ext regbank") ? static_cast<void>
 (0) : __assert_fail ("(*RBI.getRegBank(DefReg, MRI, TRI)).getID() == AArch64::GPRRegBankID && \"Unexpected ext regbank\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2926, __PRETTY_FUNCTION__))
         "Unexpected ext regbank")(((*RBI.getRegBank(DefReg, MRI, TRI)).getID() == AArch64::GPRRegBankID
 && "Unexpected ext regbank") ? static_cast<void>
 (0) : __assert_fail ("(*RBI.getRegBank(DefReg, MRI, TRI)).getID() == AArch64::GPRRegBankID && \"Unexpected ext regbank\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2926, __PRETTY_FUNCTION__));

  MachineIRBuilder MIB(I);
  MachineInstr *ExtI;

  // First check if we're extending the result of a load which has a dest type
  // smaller than 32 bits, then this zext is redundant. GPR32 is the smallest
  // GPR register on AArch64 and all loads which are smaller automatically
  // zero-extend the upper bits. E.g.
  // %v(s8) = G_LOAD %p, :: (load 1)
  // %v2(s32) = G_ZEXT %v(s8)
  if (!IsSigned) {
    auto *LoadMI = getOpcodeDef(TargetOpcode::G_LOAD, SrcReg, MRI);
    bool IsGPR =
        RBI.getRegBank(SrcReg, MRI, TRI)->getID() == AArch64::GPRRegBankID;
    if (LoadMI && IsGPR) {
      const MachineMemOperand *MemOp = *LoadMI->memoperands_begin();
      unsigned BytesLoaded = MemOp->getSize();
      if (BytesLoaded < 4 && SrcTy.getSizeInBytes() == BytesLoaded)
        return selectCopy(I, TII, MRI, TRI, RBI);
    }

    // If we are zero extending from 32 bits to 64 bits, it's possible that
    // the instruction implicitly does the zero extend for us. In that case,
    // we can just emit a SUBREG_TO_REG.
    if (IsGPR && SrcSize == 32 && DstSize == 64) {
      // Unlike with the G_LOAD case, we don't want to look through copies
      // here.
      MachineInstr *Def = MRI.getVRegDef(SrcReg);
      if (Def && isDef32(*Def)) {
        MIB.buildInstr(AArch64::SUBREG_TO_REG, {DefReg}, {})
            .addImm(0)
            .addUse(SrcReg)
            .addImm(AArch64::sub_32);

        if (!RBI.constrainGenericRegister(DefReg, AArch64::GPR64RegClass,
                                          MRI)) {
          LLVM_DEBUG(dbgs() << "Failed to constrain G_ZEXT destination\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Failed to constrain G_ZEXT destination\n"
; } } while (false);
          return false;
        }

        if (!RBI.constrainGenericRegister(SrcReg, AArch64::GPR32RegClass,
                                          MRI)) {
          LLVM_DEBUG(dbgs() << "Failed to constrain G_ZEXT source\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Failed to constrain G_ZEXT source\n"
; } } while (false);
          return false;
        }

        I.eraseFromParent();
        return true;
      }
    }
  }

  if (DstSize == 64) {
    if (Opcode != TargetOpcode::G_SEXT_INREG) {
      // FIXME: Can we avoid manually doing this?
      if (!RBI.constrainGenericRegister(SrcReg, AArch64::GPR32RegClass,
                                        MRI)) {
        LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(Opcode)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Failed to constrain " <<
 TII.getName(Opcode) << " operand\n"; } } while (false)
                          << " operand\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Failed to constrain " <<
 TII.getName(Opcode) << " operand\n"; } } while (false);
        return false;
      }
      SrcReg = MIB.buildInstr(AArch64::SUBREG_TO_REG,
                              {&AArch64::GPR64RegClass}, {})
                   .addImm(0)
                   .addUse(SrcReg)
                   .addImm(AArch64::sub_32)
                   .getReg(0);
    }

    ExtI = MIB.buildInstr(IsSigned ? AArch64::SBFMXri : AArch64::UBFMXri,
                           {DefReg}, {SrcReg})
                .addImm(0)
                .addImm(SrcSize - 1);
  } else if (DstSize <= 32) {
    ExtI = MIB.buildInstr(IsSigned ? AArch64::SBFMWri : AArch64::UBFMWri,
                           {DefReg}, {SrcReg})
                .addImm(0)
                .addImm(SrcSize - 1);
  } else {
    return false;
  }

  constrainSelectedInstRegOperands(*ExtI, TII, TRI, RBI);
  I.eraseFromParent();
  return true;
}

case TargetOpcode::G_SITOFP:
case TargetOpcode::G_UITOFP:
case TargetOpcode::G_FPTOSI:
case TargetOpcode::G_FPTOUI: {
  const LLT DstTy = MRI.getType(I.getOperand(0).getReg()),
            SrcTy = MRI.getType(I.getOperand(1).getReg());
  const unsigned NewOpc = selectFPConvOpc(Opcode, DstTy, SrcTy);
  if (NewOpc == Opcode)
    return false;

  I.setDesc(TII.get(NewOpc));
  constrainSelectedInstRegOperands(I, TII, TRI, RBI);

  return true;
}

case TargetOpcode::G_FREEZE:
  return selectCopy(I, TII, MRI, TRI, RBI);

case TargetOpcode::G_INTTOPTR:
  // The importer is currently unable to import pointer types since they
  // didn't exist in SelectionDAG.
  return selectCopy(I, TII, MRI, TRI, RBI);

case TargetOpcode::G_BITCAST:
  // Imported SelectionDAG rules can handle every bitcast except those that
  // bitcast from a type to the same type. Ideally, these shouldn't occur
  // but we might not run an optimizer that deletes them. The other exception
  // is bitcasts involving pointer types, as SelectionDAG has no knowledge
  // of them.
  return selectCopy(I, TII, MRI, TRI, RBI);

case TargetOpcode::G_SELECT: {
  if (MRI.getType(I.getOperand(1).getReg()) != LLT::scalar(1)) {
    LLVM_DEBUG(dbgs() << "G_SELECT cond has type: " << Tydo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "G_SELECT cond has type: "
 << Ty << ", expected: " << LLT::scalar(1) <<
 '\n'; } } while (false)
                      << ", expected: " << LLT::scalar(1) << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "G_SELECT cond has type: "
 << Ty << ", expected: " << LLT::scalar(1) <<
 '\n'; } } while (false);
    return false;
  }

  const Register CondReg = I.getOperand(1).getReg();
  const Register TReg = I.getOperand(2).getReg();
  const Register FReg = I.getOperand(3).getReg();

  if (tryOptSelect(I))
    return true;

  // Make sure to use an unused vreg instead of wzr, so that the peephole
  // optimizations will be able to optimize these.
  MachineIRBuilder MIB(I);
  Register DeadVReg = MRI.createVirtualRegister(&AArch64::GPR32RegClass);
  auto TstMI = MIB.buildInstr(AArch64::ANDSWri, {DeadVReg}, {CondReg})
                   .addImm(AArch64_AM::encodeLogicalImmediate(1, 32));
  constrainSelectedInstRegOperands(*TstMI, TII, TRI, RBI);
  if (!emitSelect(I.getOperand(0).getReg(), TReg, FReg, AArch64CC::NE, MIB))
    return false;
  I.eraseFromParent();
  return true;
}
case TargetOpcode::G_ICMP: {
  if (Ty.isVector())
    return selectVectorICmp(I, MRI);

  if (Ty != LLT::scalar(32)) {
    LLVM_DEBUG(dbgs() << "G_ICMP result has type: " << Tydo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "G_ICMP result has type: "
 << Ty << ", expected: " << LLT::scalar(32)
 << '\n'; } } while (false)
                      << ", expected: " << LLT::scalar(32) << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "G_ICMP result has type: "
 << Ty << ", expected: " << LLT::scalar(32)
 << '\n'; } } while (false);
    return false;
  }

  MachineIRBuilder MIRBuilder(I);
  auto Pred = static_cast<CmpInst::Predicate>(I.getOperand(1).getPredicate());
  emitIntegerCompare(I.getOperand(2), I.getOperand(3), I.getOperand(1),
                     MIRBuilder);
  emitCSetForICMP(I.getOperand(0).getReg(), Pred, MIRBuilder);
  I.eraseFromParent();
  return true;
}

case TargetOpcode::G_FCMP: {
  MachineIRBuilder MIRBuilder(I);
  CmpInst::Predicate Pred =
      static_cast<CmpInst::Predicate>(I.getOperand(1).getPredicate());
  if (!emitFPCompare(I.getOperand(2).getReg(), I.getOperand(3).getReg(),
                     MIRBuilder, Pred) ||
      !emitCSetForFCmp(I.getOperand(0).getReg(), Pred, MIRBuilder))
    return false;
  I.eraseFromParent();
  return true;
}
case TargetOpcode::G_VASTART:
  return STI.isTargetDarwin() ? selectVaStartDarwin(I, MF, MRI)
                              : selectVaStartAAPCS(I, MF, MRI);
case TargetOpcode::G_INTRINSIC:
  return selectIntrinsic(I, MRI);
case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS:
  return selectIntrinsicWithSideEffects(I, MRI);
case TargetOpcode::G_IMPLICIT_DEF: {
  I.setDesc(TII.get(TargetOpcode::IMPLICIT_DEF));
  const LLT DstTy = MRI.getType(I.getOperand(0).getReg());
  const Register DstReg = I.getOperand(0).getReg();
  const RegisterBank &DstRB = *RBI.getRegBank(DstReg, MRI, TRI);
  const TargetRegisterClass *DstRC =
      getRegClassForTypeOnBank(DstTy, DstRB, RBI);
  RBI.constrainGenericRegister(DstReg, *DstRC, MRI);
  return true;
}
case TargetOpcode::G_BLOCK_ADDR: {
  if (TM.getCodeModel() == CodeModel::Large) {
    materializeLargeCMVal(I, I.getOperand(1).getBlockAddress(), 0);
    I.eraseFromParent();
    return true;
  } else {
    I.setDesc(TII.get(AArch64::MOVaddrBA));
    auto MovMI = BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::MOVaddrBA),
                         I.getOperand(0).getReg())
                     .addBlockAddress(I.getOperand(1).getBlockAddress(),
                                      /* Offset */ 0, AArch64II::MO_PAGE)
                     .addBlockAddress(
                         I.getOperand(1).getBlockAddress(), /* Offset */ 0,
                         AArch64II::MO_NC | AArch64II::MO_PAGEOFF);
    I.eraseFromParent();
    return constrainSelectedInstRegOperands(*MovMI, TII, TRI, RBI);
  }
}
case AArch64::G_DUP: {
  // When the scalar of G_DUP is an s8/s16 gpr, they can't be selected by
  // imported patterns. Do it manually here. Avoiding generating s16 gpr is
  // difficult because at RBS we may end up pessimizing the fpr case if we
  // decided to add an anyextend to fix this. Manual selection is the most
  // robust solution for now.
  Register SrcReg = I.getOperand(1).getReg();
  if (RBI.getRegBank(SrcReg, MRI, TRI)->getID() != AArch64::GPRRegBankID)
    return false; // We expect the fpr regbank case to be imported.
  LLT SrcTy = MRI.getType(SrcReg);
  if (SrcTy.getSizeInBits() == 16)
    I.setDesc(TII.get(AArch64::DUPv8i16gpr));
  else if (SrcTy.getSizeInBits() == 8)
    I.setDesc(TII.get(AArch64::DUPv16i8gpr));
  else
    return false;
  return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
}
case TargetOpcode::G_INTRINSIC_TRUNC:
  return selectIntrinsicTrunc(I, MRI);
case TargetOpcode::G_INTRINSIC_ROUND:
  return selectIntrinsicRound(I, MRI);
case TargetOpcode::G_BUILD_VECTOR:
  return selectBuildVector(I, MRI);
case TargetOpcode::G_MERGE_VALUES:
  return selectMergeValues(I, MRI);
case TargetOpcode::G_UNMERGE_VALUES:
  return selectUnmergeValues(I, MRI);
case TargetOpcode::G_SHUFFLE_VECTOR:
  return selectShuffleVector(I, MRI);
case TargetOpcode::G_EXTRACT_VECTOR_ELT:
  return selectExtractElt(I, MRI);
case TargetOpcode::G_INSERT_VECTOR_ELT:
  return selectInsertElt(I, MRI);
case TargetOpcode::G_CONCAT_VECTORS:
  return selectConcatVectors(I, MRI);
case TargetOpcode::G_JUMP_TABLE:
  return selectJumpTable(I, MRI);
case TargetOpcode::G_VECREDUCE_FADD:
case TargetOpcode::G_VECREDUCE_ADD:
  return selectReduction(I, MRI);
}

return false;
3181}

3183bool AArch64InstructionSelector::selectReduction(
  MachineInstr &I, MachineRegisterInfo &MRI) const {
Register VecReg = I.getOperand(1).getReg();
LLT VecTy = MRI.getType(VecReg);
if (I.getOpcode() == TargetOpcode::G_VECREDUCE_ADD) {
  unsigned Opc = 0;
  if (VecTy == LLT::vector(16, 8))
    Opc = AArch64::ADDVv16i8v;
  else if (VecTy == LLT::vector(8, 16))
    Opc = AArch64::ADDVv8i16v;
  else if (VecTy == LLT::vector(4, 32))
    Opc = AArch64::ADDVv4i32v;
  else if (VecTy == LLT::vector(2, 64))
    Opc = AArch64::ADDPv2i64p;
  else {
    LLVM_DEBUG(dbgs() << "Unhandled type for add reduction")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unhandled type for add reduction"
; } } while (false);
    return false;
  }
  I.setDesc(TII.get(Opc));
  return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
}

if (I.getOpcode() == TargetOpcode::G_VECREDUCE_FADD) {
  unsigned Opc = 0;
  if (VecTy == LLT::vector(2, 32))
    Opc = AArch64::FADDPv2i32p;
  else if (VecTy == LLT::vector(2, 64))
    Opc = AArch64::FADDPv2i64p;
  else {
    LLVM_DEBUG(dbgs() << "Unhandled type for fadd reduction")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unhandled type for fadd reduction"
; } } while (false);
    return false;
  }
  I.setDesc(TII.get(Opc));
  return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
}
return false;
3219}

3221bool AArch64InstructionSelector::selectBrJT(MachineInstr &I,
                                          MachineRegisterInfo &MRI) const {
assert(I.getOpcode() == TargetOpcode::G_BRJT && "Expected G_BRJT")((I.getOpcode() == TargetOpcode::G_BRJT && "Expected G_BRJT"
) ? static_cast<void> (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_BRJT && \"Expected G_BRJT\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3223, __PRETTY_FUNCTION__));
Register JTAddr = I.getOperand(0).getReg();
unsigned JTI = I.getOperand(1).getIndex();
Register Index = I.getOperand(2).getReg();
MachineIRBuilder MIB(I);

Register TargetReg = MRI.createVirtualRegister(&AArch64::GPR64RegClass);
Register ScratchReg = MRI.createVirtualRegister(&AArch64::GPR64spRegClass);

MF->getInfo<AArch64FunctionInfo>()->setJumpTableEntryInfo(JTI, 4, nullptr);
auto JumpTableInst = MIB.buildInstr(AArch64::JumpTableDest32,
                                    {TargetReg, ScratchReg}, {JTAddr, Index})
                         .addJumpTableIndex(JTI);
// Build the indirect branch.
MIB.buildInstr(AArch64::BR, {}, {TargetReg});
I.eraseFromParent();
return constrainSelectedInstRegOperands(*JumpTableInst, TII, TRI, RBI);
3240}

3242bool AArch64InstructionSelector::selectJumpTable(
  MachineInstr &I, MachineRegisterInfo &MRI) const {
assert(I.getOpcode() == TargetOpcode::G_JUMP_TABLE && "Expected jump table")((I.getOpcode() == TargetOpcode::G_JUMP_TABLE && "Expected jump table"
) ? static_cast<void> (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_JUMP_TABLE && \"Expected jump table\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3244, __PRETTY_FUNCTION__));
assert(I.getOperand(1).isJTI() && "Jump table op should have a JTI!")((I.getOperand(1).isJTI() && "Jump table op should have a JTI!"
) ? static_cast<void> (0) : __assert_fail ("I.getOperand(1).isJTI() && \"Jump table op should have a JTI!\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3245, __PRETTY_FUNCTION__));

Register DstReg = I.getOperand(0).getReg();
unsigned JTI = I.getOperand(1).getIndex();
// We generate a MOVaddrJT which will get expanded to an ADRP + ADD later.
MachineIRBuilder MIB(I);
auto MovMI =
  MIB.buildInstr(AArch64::MOVaddrJT, {DstReg}, {})
        .addJumpTableIndex(JTI, AArch64II::MO_PAGE)
        .addJumpTableIndex(JTI, AArch64II::MO_NC | AArch64II::MO_PAGEOFF);
I.eraseFromParent();
return constrainSelectedInstRegOperands(*MovMI, TII, TRI, RBI);
3257}

3259bool AArch64InstructionSelector::selectTLSGlobalValue(
  MachineInstr &I, MachineRegisterInfo &MRI) const {
if (!STI.isTargetMachO())
  return false;
MachineFunction &MF = *I.getParent()->getParent();
MF.getFrameInfo().setAdjustsStack(true);

const GlobalValue &GV = *I.getOperand(1).getGlobal();
MachineIRBuilder MIB(I);

auto LoadGOT =
    MIB.buildInstr(AArch64::LOADgot, {&AArch64::GPR64commonRegClass}, {})
        .addGlobalAddress(&GV, 0, AArch64II::MO_TLS);

auto Load = MIB.buildInstr(AArch64::LDRXui, {&AArch64::GPR64commonRegClass},
                           {LoadGOT.getReg(0)})
                .addImm(0);

MIB.buildCopy(Register(AArch64::X0), LoadGOT.getReg(0));
// TLS calls preserve all registers except those that absolutely must be
// trashed: X0 (it takes an argument), LR (it's a call) and NZCV (let's not be
// silly).
MIB.buildInstr(getBLRCallOpcode(MF), {}, {Load})
    .addUse(AArch64::X0, RegState::Implicit)
    .addDef(AArch64::X0, RegState::Implicit)
    .addRegMask(TRI.getTLSCallPreservedMask());

MIB.buildCopy(I.getOperand(0).getReg(), Register(AArch64::X0));
RBI.constrainGenericRegister(I.getOperand(0).getReg(), AArch64::GPR64RegClass,
                             MRI);
I.eraseFromParent();
return true;
3291}

3293bool AArch64InstructionSelector::selectIntrinsicTrunc(
  MachineInstr &I, MachineRegisterInfo &MRI) const {
const LLT SrcTy = MRI.getType(I.getOperand(0).getReg());

// Select the correct opcode.
unsigned Opc = 0;
if (!SrcTy.isVector()) {
  switch (SrcTy.getSizeInBits()) {
  default:
  case 16:
    Opc = AArch64::FRINTZHr;
    break;
  case 32:
    Opc = AArch64::FRINTZSr;
    break;
  case 64:
    Opc = AArch64::FRINTZDr;
    break;
  }
} else {
  unsigned NumElts = SrcTy.getNumElements();
  switch (SrcTy.getElementType().getSizeInBits()) {
  default:
    break;
  case 16:
    if (NumElts == 4)
      Opc = AArch64::FRINTZv4f16;
    else if (NumElts == 8)
      Opc = AArch64::FRINTZv8f16;
    break;
  case 32:
    if (NumElts == 2)
      Opc = AArch64::FRINTZv2f32;
    else if (NumElts == 4)
      Opc = AArch64::FRINTZv4f32;
    break;
  case 64:
    if (NumElts == 2)
      Opc = AArch64::FRINTZv2f64;
    break;
  }
}

if (!Opc) {
  // Didn't get an opcode above, bail.
  LLVM_DEBUG(dbgs() << "Unsupported type for G_INTRINSIC_TRUNC!\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unsupported type for G_INTRINSIC_TRUNC!\n"
; } } while (false);
  return false;
}

// Legalization would have set us up perfectly for this; we just need to
// set the opcode and move on.
I.setDesc(TII.get(Opc));
return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
3346}

3348bool AArch64InstructionSelector::selectIntrinsicRound(
  MachineInstr &I, MachineRegisterInfo &MRI) const {
const LLT SrcTy = MRI.getType(I.getOperand(0).getReg());

// Select the correct opcode.
unsigned Opc = 0;
if (!SrcTy.isVector()) {
  switch (SrcTy.getSizeInBits()) {
  default:
  case 16:
    Opc = AArch64::FRINTAHr;
    break;
  case 32:
    Opc = AArch64::FRINTASr;
    break;
  case 64:
    Opc = AArch64::FRINTADr;
    break;
  }
} else {
  unsigned NumElts = SrcTy.getNumElements();
  switch (SrcTy.getElementType().getSizeInBits()) {
  default:
    break;
  case 16:
    if (NumElts == 4)
      Opc = AArch64::FRINTAv4f16;
    else if (NumElts == 8)
      Opc = AArch64::FRINTAv8f16;
    break;
  case 32:
    if (NumElts == 2)
      Opc = AArch64::FRINTAv2f32;
    else if (NumElts == 4)
      Opc = AArch64::FRINTAv4f32;
    break;
  case 64:
    if (NumElts == 2)
      Opc = AArch64::FRINTAv2f64;
    break;
  }
}

if (!Opc) {
  // Didn't get an opcode above, bail.
  LLVM_DEBUG(dbgs() << "Unsupported type for G_INTRINSIC_ROUND!\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unsupported type for G_INTRINSIC_ROUND!\n"
; } } while (false);
  return false;
}

// Legalization would have set us up perfectly for this; we just need to
// set the opcode and move on.
I.setDesc(TII.get(Opc));
return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
3401}

3403bool AArch64InstructionSelector::selectVectorICmp(
  MachineInstr &I, MachineRegisterInfo &MRI) const {
Register DstReg = I.getOperand(0).getReg();
LLT DstTy = MRI.getType(DstReg);
Register SrcReg = I.getOperand(2).getReg();
Register Src2Reg = I.getOperand(3).getReg();
LLT SrcTy = MRI.getType(SrcReg);

unsigned SrcEltSize = SrcTy.getElementType().getSizeInBits();
unsigned NumElts = DstTy.getNumElements();

// First index is element size, 0 == 8b, 1 == 16b, 2 == 32b, 3 == 64b
// Second index is num elts, 0 == v2, 1 == v4, 2 == v8, 3 == v16
// Third index is cc opcode:
// 0 == eq
// 1 == ugt
// 2 == uge
// 3 == ult
// 4 == ule
// 5 == sgt
// 6 == sge
// 7 == slt
// 8 == sle
// ne is done by negating 'eq' result.

// This table below assumes that for some comparisons the operands will be
// commuted.
// ult op == commute + ugt op
// ule op == commute + uge op
// slt op == commute + sgt op
// sle op == commute + sge op
unsigned PredIdx = 0;
bool SwapOperands = false;
CmpInst::Predicate Pred = (CmpInst::Predicate)I.getOperand(1).getPredicate();
switch (Pred) {
case CmpInst::ICMP_NE:
case CmpInst::ICMP_EQ:
  PredIdx = 0;
  break;
case CmpInst::ICMP_UGT:
  PredIdx = 1;
  break;
case CmpInst::ICMP_UGE:
  PredIdx = 2;
  break;
case CmpInst::ICMP_ULT:
  PredIdx = 3;
  SwapOperands = true;
  break;
case CmpInst::ICMP_ULE:
  PredIdx = 4;
  SwapOperands = true;
  break;
case CmpInst::ICMP_SGT:
  PredIdx = 5;
  break;
case CmpInst::ICMP_SGE:
  PredIdx = 6;
  break;
case CmpInst::ICMP_SLT:
  PredIdx = 7;
  SwapOperands = true;
  break;
case CmpInst::ICMP_SLE:
  PredIdx = 8;
  SwapOperands = true;
  break;
default:
  llvm_unreachable("Unhandled icmp predicate")::llvm::llvm_unreachable_internal("Unhandled icmp predicate",
 "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3471);
  return false;
}

// This table obviously should be tablegen'd when we have our GISel native
// tablegen selector.

static const unsigned OpcTable[4][4][9] = {
    {
        {0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
         0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
         0 /* invalid */},
        {0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
         0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
         0 /* invalid */},
        {AArch64::CMEQv8i8, AArch64::CMHIv8i8, AArch64::CMHSv8i8,
         AArch64::CMHIv8i8, AArch64::CMHSv8i8, AArch64::CMGTv8i8,
         AArch64::CMGEv8i8, AArch64::CMGTv8i8, AArch64::CMGEv8i8},
        {AArch64::CMEQv16i8, AArch64::CMHIv16i8, AArch64::CMHSv16i8,
         AArch64::CMHIv16i8, AArch64::CMHSv16i8, AArch64::CMGTv16i8,
         AArch64::CMGEv16i8, AArch64::CMGTv16i8, AArch64::CMGEv16i8}
    },
    {
        {0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
         0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
         0 /* invalid */},
        {AArch64::CMEQv4i16, AArch64::CMHIv4i16, AArch64::CMHSv4i16,
         AArch64::CMHIv4i16, AArch64::CMHSv4i16, AArch64::CMGTv4i16,
         AArch64::CMGEv4i16, AArch64::CMGTv4i16, AArch64::CMGEv4i16},
        {AArch64::CMEQv8i16, AArch64::CMHIv8i16, AArch64::CMHSv8i16,
         AArch64::CMHIv8i16, AArch64::CMHSv8i16, AArch64::CMGTv8i16,
         AArch64::CMGEv8i16, AArch64::CMGTv8i16, AArch64::CMGEv8i16},
        {0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
         0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
         0 /* invalid */}
    },
    {
        {AArch64::CMEQv2i32, AArch64::CMHIv2i32, AArch64::CMHSv2i32,
         AArch64::CMHIv2i32, AArch64::CMHSv2i32, AArch64::CMGTv2i32,
         AArch64::CMGEv2i32, AArch64::CMGTv2i32, AArch64::CMGEv2i32},
        {AArch64::CMEQv4i32, AArch64::CMHIv4i32, AArch64::CMHSv4i32,
         AArch64::CMHIv4i32, AArch64::CMHSv4i32, AArch64::CMGTv4i32,
         AArch64::CMGEv4i32, AArch64::CMGTv4i32, AArch64::CMGEv4i32},
        {0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
         0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
         0 /* invalid */},
        {0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
         0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
         0 /* invalid */}
    },
    {
        {AArch64::CMEQv2i64, AArch64::CMHIv2i64, AArch64::CMHSv2i64,
         AArch64::CMHIv2i64, AArch64::CMHSv2i64, AArch64::CMGTv2i64,
         AArch64::CMGEv2i64, AArch64::CMGTv2i64, AArch64::CMGEv2i64},
        {0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
         0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
         0 /* invalid */},
        {0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
         0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
         0 /* invalid */},
        {0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
         0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
         0 /* invalid */}
    },
};
unsigned EltIdx = Log2_32(SrcEltSize / 8);
unsigned NumEltsIdx = Log2_32(NumElts / 2);
unsigned Opc = OpcTable[EltIdx][NumEltsIdx][PredIdx];
if (!Opc) {
  LLVM_DEBUG(dbgs() << "Could not map G_ICMP to cmp opcode")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Could not map G_ICMP to cmp opcode"
; } } while (false);
  return false;
}

const RegisterBank &VecRB = *RBI.getRegBank(SrcReg, MRI, TRI);
const TargetRegisterClass *SrcRC =
    getRegClassForTypeOnBank(SrcTy, VecRB, RBI, true);
if (!SrcRC) {
  LLVM_DEBUG(dbgs() << "Could not determine source register class.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Could not determine source register class.\n"
; } } while (false);
  return false;
}

unsigned NotOpc = Pred == ICmpInst::ICMP_NE ? AArch64::NOTv8i8 : 0;
if (SrcTy.getSizeInBits() == 128)
  NotOpc = NotOpc ? AArch64::NOTv16i8 : 0;

if (SwapOperands)
  std::swap(SrcReg, Src2Reg);

MachineIRBuilder MIB(I);
auto Cmp = MIB.buildInstr(Opc, {SrcRC}, {SrcReg, Src2Reg});
constrainSelectedInstRegOperands(*Cmp, TII, TRI, RBI);

// Invert if we had a 'ne' cc.
if (NotOpc) {
  Cmp = MIB.buildInstr(NotOpc, {DstReg}, {Cmp});
  constrainSelectedInstRegOperands(*Cmp, TII, TRI, RBI);
} else {
  MIB.buildCopy(DstReg, Cmp.getReg(0));
}
RBI.constrainGenericRegister(DstReg, *SrcRC, MRI);
I.eraseFromParent();
return true;
3573}

3575MachineInstr *AArch64InstructionSelector::emitScalarToVector(
  unsigned EltSize, const TargetRegisterClass *DstRC, Register Scalar,
  MachineIRBuilder &MIRBuilder) const {
auto Undef = MIRBuilder.buildInstr(TargetOpcode::IMPLICIT_DEF, {DstRC}, {});

auto BuildFn = [&](unsigned SubregIndex) {
  auto Ins =
      MIRBuilder
          .buildInstr(TargetOpcode::INSERT_SUBREG, {DstRC}, {Undef, Scalar})
          .addImm(SubregIndex);
  constrainSelectedInstRegOperands(*Undef, TII, TRI, RBI);
  constrainSelectedInstRegOperands(*Ins, TII, TRI, RBI);
  return &*Ins;
};

switch (EltSize) {
case 16:
  return BuildFn(AArch64::hsub);
case 32:
  return BuildFn(AArch64::ssub);
case 64:
  return BuildFn(AArch64::dsub);
default:
  return nullptr;
}
3600}

3602bool AArch64InstructionSelector::selectMergeValues(
  MachineInstr &I, MachineRegisterInfo &MRI) const {
assert(I.getOpcode() == TargetOpcode::G_MERGE_VALUES && "unexpected opcode")((I.getOpcode() == TargetOpcode::G_MERGE_VALUES && "unexpected opcode"
) ? static_cast<void> (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_MERGE_VALUES && \"unexpected opcode\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3604, __PRETTY_FUNCTION__));
const LLT DstTy = MRI.getType(I.getOperand(0).getReg());
const LLT SrcTy = MRI.getType(I.getOperand(1).getReg());
assert(!DstTy.isVector() && !SrcTy.isVector() && "invalid merge operation")((!DstTy.isVector() && !SrcTy.isVector() && "invalid merge operation"
) ? static_cast<void> (0) : __assert_fail ("!DstTy.isVector() && !SrcTy.isVector() && \"invalid merge operation\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3607, __PRETTY_FUNCTION__));
const RegisterBank &RB = *RBI.getRegBank(I.getOperand(1).getReg(), MRI, TRI);

if (I.getNumOperands() != 3)
  return false;

// Merging 2 s64s into an s128.
if (DstTy == LLT::scalar(128)) {
  if (SrcTy.getSizeInBits() != 64)
    return false;
  MachineIRBuilder MIB(I);
  Register DstReg = I.getOperand(0).getReg();
  Register Src1Reg = I.getOperand(1).getReg();
  Register Src2Reg = I.getOperand(2).getReg();
  auto Tmp = MIB.buildInstr(TargetOpcode::IMPLICIT_DEF, {DstTy}, {});
  MachineInstr *InsMI =
      emitLaneInsert(None, Tmp.getReg(0), Src1Reg, /* LaneIdx */ 0, RB, MIB);
  if (!InsMI)
    return false;
  MachineInstr *Ins2MI = emitLaneInsert(DstReg, InsMI->getOperand(0).getReg(),
                                        Src2Reg, /* LaneIdx */ 1, RB, MIB);
  if (!Ins2MI)
    return false;
  constrainSelectedInstRegOperands(*InsMI, TII, TRI, RBI);
  constrainSelectedInstRegOperands(*Ins2MI, TII, TRI, RBI);
  I.eraseFromParent();
  return true;
}

if (RB.getID() != AArch64::GPRRegBankID)
  return false;

if (DstTy.getSizeInBits() != 64 || SrcTy.getSizeInBits() != 32)
  return false;

auto *DstRC = &AArch64::GPR64RegClass;
Register SubToRegDef = MRI.createVirtualRegister(DstRC);
MachineInstr &SubRegMI = *BuildMI(*I.getParent(), I, I.getDebugLoc(),
                                  TII.get(TargetOpcode::SUBREG_TO_REG))
                              .addDef(SubToRegDef)
                              .addImm(0)
                              .addUse(I.getOperand(1).getReg())
                              .addImm(AArch64::sub_32);
Register SubToRegDef2 = MRI.createVirtualRegister(DstRC);
// Need to anyext the second scalar before we can use bfm
MachineInstr &SubRegMI2 = *BuildMI(*I.getParent(), I, I.getDebugLoc(),
                                  TII.get(TargetOpcode::SUBREG_TO_REG))
                              .addDef(SubToRegDef2)
                              .addImm(0)
                              .addUse(I.getOperand(2).getReg())
                              .addImm(AArch64::sub_32);
MachineInstr &BFM =
    *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(AArch64::BFMXri))
         .addDef(I.getOperand(0).getReg())
         .addUse(SubToRegDef)
         .addUse(SubToRegDef2)
         .addImm(32)
         .addImm(31);
constrainSelectedInstRegOperands(SubRegMI, TII, TRI, RBI);
constrainSelectedInstRegOperands(SubRegMI2, TII, TRI, RBI);
constrainSelectedInstRegOperands(BFM, TII, TRI, RBI);
I.eraseFromParent();
return true;
3670}

3672static bool getLaneCopyOpcode(unsigned &CopyOpc, unsigned &ExtractSubReg,
                            const unsigned EltSize) {
// Choose a lane copy opcode and subregister based off of the size of the
// vector's elements.
switch (EltSize) {
case 16:
  CopyOpc = AArch64::CPYi16;
  ExtractSubReg = AArch64::hsub;
  break;
case 32:
  CopyOpc = AArch64::CPYi32;
  ExtractSubReg = AArch64::ssub;
  break;
case 64:
  CopyOpc = AArch64::CPYi64;
  ExtractSubReg = AArch64::dsub;
  break;
default:
  // Unknown size, bail out.
  LLVM_DEBUG(dbgs() << "Elt size '" << EltSize << "' unsupported.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Elt size '" << EltSize
 << "' unsupported.\n"; } } while (false);
  return false;
}
return true;
3695}

3697MachineInstr *AArch64InstructionSelector::emitExtractVectorElt(
  Optional<Register> DstReg, const RegisterBank &DstRB, LLT ScalarTy,
  Register VecReg, unsigned LaneIdx, MachineIRBuilder &MIRBuilder) const {
MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
unsigned CopyOpc = 0;
unsigned ExtractSubReg = 0;
if (!getLaneCopyOpcode(CopyOpc, ExtractSubReg, ScalarTy.getSizeInBits())) {
  LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Couldn't determine lane copy opcode for instruction.\n"
; } } while (false)
      dbgs() << "Couldn't determine lane copy opcode for instruction.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Couldn't determine lane copy opcode for instruction.\n"
; } } while (false);
  return nullptr;
}

const TargetRegisterClass *DstRC =
    getRegClassForTypeOnBank(ScalarTy, DstRB, RBI, true);
if (!DstRC) {
  LLVM_DEBUG(dbgs() << "Could not determine destination register class.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Could not determine destination register class.\n"
; } } while (false);
  return nullptr;
}

const RegisterBank &VecRB = *RBI.getRegBank(VecReg, MRI, TRI);
const LLT &VecTy = MRI.getType(VecReg);
const TargetRegisterClass *VecRC =
    getRegClassForTypeOnBank(VecTy, VecRB, RBI, true);
if (!VecRC) {
  LLVM_DEBUG(dbgs() << "Could not determine source register class.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Could not determine source register class.\n"
; } } while (false);
  return nullptr;
}

// The register that we're going to copy into.
Register InsertReg = VecReg;
if (!DstReg)
  DstReg = MRI.createVirtualRegister(DstRC);
// If the lane index is 0, we just use a subregister COPY.
if (LaneIdx == 0) {
  auto Copy = MIRBuilder.buildInstr(TargetOpcode::COPY, {*DstReg}, {})
                  .addReg(VecReg, 0, ExtractSubReg);
  RBI.constrainGenericRegister(*DstReg, *DstRC, MRI);
  return &*Copy;
}

// Lane copies require 128-bit wide registers. If we're dealing with an
// unpacked vector, then we need to move up to that width. Insert an implicit
// def and a subregister insert to get us there.
if (VecTy.getSizeInBits() != 128) {
  MachineInstr *ScalarToVector = emitScalarToVector(
      VecTy.getSizeInBits(), &AArch64::FPR128RegClass, VecReg, MIRBuilder);
  if (!ScalarToVector)
    return nullptr;
  InsertReg = ScalarToVector->getOperand(0).getReg();
}

MachineInstr *LaneCopyMI =
    MIRBuilder.buildInstr(CopyOpc, {*DstReg}, {InsertReg}).addImm(LaneIdx);
constrainSelectedInstRegOperands(*LaneCopyMI, TII, TRI, RBI);

// Make sure that we actually constrain the initial copy.
RBI.constrainGenericRegister(*DstReg, *DstRC, MRI);
return LaneCopyMI;
3755}

3757bool AArch64InstructionSelector::selectExtractElt(
  MachineInstr &I, MachineRegisterInfo &MRI) const {
assert(I.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT &&((I.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT &&
 "unexpected opcode!") ? static_cast<void> (0) : __assert_fail
 ("I.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT && \"unexpected opcode!\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3760, __PRETTY_FUNCTION__))
       "unexpected opcode!")((I.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT &&
 "unexpected opcode!") ? static_cast<void> (0) : __assert_fail
 ("I.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT && \"unexpected opcode!\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3760, __PRETTY_FUNCTION__));
Register DstReg = I.getOperand(0).getReg();
const LLT NarrowTy = MRI.getType(DstReg);
const Register SrcReg = I.getOperand(1).getReg();
const LLT WideTy = MRI.getType(SrcReg);
(void)WideTy;
assert(WideTy.getSizeInBits() >= NarrowTy.getSizeInBits() &&((WideTy.getSizeInBits() >= NarrowTy.getSizeInBits() &&
 "source register size too small!") ? static_cast<void>
 (0) : __assert_fail ("WideTy.getSizeInBits() >= NarrowTy.getSizeInBits() && \"source register size too small!\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3767, __PRETTY_FUNCTION__))
       "source register size too small!")((WideTy.getSizeInBits() >= NarrowTy.getSizeInBits() &&
 "source register size too small!") ? static_cast<void>
 (0) : __assert_fail ("WideTy.getSizeInBits() >= NarrowTy.getSizeInBits() && \"source register size too small!\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3767, __PRETTY_FUNCTION__));
assert(!NarrowTy.isVector() && "cannot extract vector into vector!")((!NarrowTy.isVector() && "cannot extract vector into vector!"
) ? static_cast<void> (0) : __assert_fail ("!NarrowTy.isVector() && \"cannot extract vector into vector!\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3768, __PRETTY_FUNCTION__));

// Need the lane index to determine the correct copy opcode.
MachineOperand &LaneIdxOp = I.getOperand(2);
assert(LaneIdxOp.isReg() && "Lane index operand was not a register?")((LaneIdxOp.isReg() && "Lane index operand was not a register?"
) ? static_cast<void> (0) : __assert_fail ("LaneIdxOp.isReg() && \"Lane index operand was not a register?\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3772, __PRETTY_FUNCTION__));

if (RBI.getRegBank(DstReg, MRI, TRI)->getID() != AArch64::FPRRegBankID) {
  LLVM_DEBUG(dbgs() << "Cannot extract into GPR.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Cannot extract into GPR.\n"
; } } while (false);
  return false;
}

// Find the index to extract from.
auto VRegAndVal = getConstantVRegValWithLookThrough(LaneIdxOp.getReg(), MRI);
if (!VRegAndVal)
  return false;
unsigned LaneIdx = VRegAndVal->Value.getSExtValue();

MachineIRBuilder MIRBuilder(I);

const RegisterBank &DstRB = *RBI.getRegBank(DstReg, MRI, TRI);
MachineInstr *Extract = emitExtractVectorElt(DstReg, DstRB, NarrowTy, SrcReg,
                                             LaneIdx, MIRBuilder);
if (!Extract)
  return false;

I.eraseFromParent();
return true;
3795}

3797bool AArch64InstructionSelector::selectSplitVectorUnmerge(
  MachineInstr &I, MachineRegisterInfo &MRI) const {
unsigned NumElts = I.getNumOperands() - 1;
Register SrcReg = I.getOperand(NumElts).getReg();
const LLT NarrowTy = MRI.getType(I.getOperand(0).getReg());
const LLT SrcTy = MRI.getType(SrcReg);

assert(NarrowTy.isVector() && "Expected an unmerge into vectors")((NarrowTy.isVector() && "Expected an unmerge into vectors"
) ? static_cast<void> (0) : __assert_fail ("NarrowTy.isVector() && \"Expected an unmerge into vectors\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3804, __PRETTY_FUNCTION__));
if (SrcTy.getSizeInBits() > 128) {
  LLVM_DEBUG(dbgs() << "Unexpected vector type for vec split unmerge")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unexpected vector type for vec split unmerge"
; } } while (false);
  return false;
}

MachineIRBuilder MIB(I);

// We implement a split vector operation by treating the sub-vectors as
// scalars and extracting them.
const RegisterBank &DstRB =
    *RBI.getRegBank(I.getOperand(0).getReg(), MRI, TRI);
for (unsigned OpIdx = 0; OpIdx < NumElts; ++OpIdx) {
  Register Dst = I.getOperand(OpIdx).getReg();
  MachineInstr *Extract =
      emitExtractVectorElt(Dst, DstRB, NarrowTy, SrcReg, OpIdx, MIB);
  if (!Extract)
    return false;
}
I.eraseFromParent();
return true;
3825}

3827bool AArch64InstructionSelector::selectUnmergeValues(
  MachineInstr &I, MachineRegisterInfo &MRI) const {
assert(I.getOpcode() == TargetOpcode::G_UNMERGE_VALUES &&((I.getOpcode() == TargetOpcode::G_UNMERGE_VALUES && "unexpected opcode"
) ? static_cast<void> (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_UNMERGE_VALUES && \"unexpected opcode\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3830, __PRETTY_FUNCTION__))
       "unexpected opcode")((I.getOpcode() == TargetOpcode::G_UNMERGE_VALUES && "unexpected opcode"
) ? static_cast<void> (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_UNMERGE_VALUES && \"unexpected opcode\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3830, __PRETTY_FUNCTION__));

// TODO: Handle unmerging into GPRs and from scalars to scalars.
if (RBI.getRegBank(I.getOperand(0).getReg(), MRI, TRI)->getID() !=
        AArch64::FPRRegBankID ||
    RBI.getRegBank(I.getOperand(1).getReg(), MRI, TRI)->getID() !=
        AArch64::FPRRegBankID) {
  LLVM_DEBUG(dbgs() << "Unmerging vector-to-gpr and scalar-to-scalar "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unmerging vector-to-gpr and scalar-to-scalar "
 "currently unsupported.\n"; } } while (false)
                       "currently unsupported.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unmerging vector-to-gpr and scalar-to-scalar "
 "currently unsupported.\n"; } } while (false);
  return false;
}

// The last operand is the vector source register, and every other operand is
// a register to unpack into.
unsigned NumElts = I.getNumOperands() - 1;
Register SrcReg = I.getOperand(NumElts).getReg();
const LLT NarrowTy = MRI.getType(I.getOperand(0).getReg());
const LLT WideTy = MRI.getType(SrcReg);
(void)WideTy;
assert((WideTy.isVector() || WideTy.getSizeInBits() == 128) &&(((WideTy.isVector() || WideTy.getSizeInBits() == 128) &&
 "can only unmerge from vector or s128 types!") ? static_cast
<void> (0) : __assert_fail ("(WideTy.isVector() || WideTy.getSizeInBits() == 128) && \"can only unmerge from vector or s128 types!\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3850, __PRETTY_FUNCTION__))
       "can only unmerge from vector or s128 types!")(((WideTy.isVector() || WideTy.getSizeInBits() == 128) &&
 "can only unmerge from vector or s128 types!") ? static_cast
<void> (0) : __assert_fail ("(WideTy.isVector() || WideTy.getSizeInBits() == 128) && \"can only unmerge from vector or s128 types!\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3850, __PRETTY_FUNCTION__));
assert(WideTy.getSizeInBits() > NarrowTy.getSizeInBits() &&((WideTy.getSizeInBits() > NarrowTy.getSizeInBits() &&
 "source register size too small!") ? static_cast<void>
 (0) : __assert_fail ("WideTy.getSizeInBits() > NarrowTy.getSizeInBits() && \"source register size too small!\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3852, __PRETTY_FUNCTION__))
       "source register size too small!")((WideTy.getSizeInBits() > NarrowTy.getSizeInBits() &&
 "source register size too small!") ? static_cast<void>
 (0) : __assert_fail ("WideTy.getSizeInBits() > NarrowTy.getSizeInBits() && \"source register size too small!\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3852, __PRETTY_FUNCTION__));

if (!NarrowTy.isScalar())
  return selectSplitVectorUnmerge(I, MRI);

MachineIRBuilder MIB(I);

// Choose a lane copy opcode and subregister based off of the size of the
// vector's elements.
unsigned CopyOpc = 0;
unsigned ExtractSubReg = 0;
if (!getLaneCopyOpcode(CopyOpc, ExtractSubReg, NarrowTy.getSizeInBits()))
  return false;

// Set up for the lane copies.
MachineBasicBlock &MBB = *I.getParent();

// Stores the registers we'll be copying from.
SmallVector<Register, 4> InsertRegs;

// We'll use the first register twice, so we only need NumElts-1 registers.
unsigned NumInsertRegs = NumElts - 1;

// If our elements fit into exactly 128 bits, then we can copy from the source
// directly. Otherwise, we need to do a bit of setup with some subregister
// inserts.
if (NarrowTy.getSizeInBits() * NumElts == 128) {
  InsertRegs = SmallVector<Register, 4>(NumInsertRegs, SrcReg);
} else {
  // No. We have to perform subregister inserts. For each insert, create an
  // implicit def and a subregister insert, and save the register we create.
  for (unsigned Idx = 0; Idx < NumInsertRegs; ++Idx) {
    Register ImpDefReg = MRI.createVirtualRegister(&AArch64::FPR128RegClass);
    MachineInstr &ImpDefMI =
        *BuildMI(MBB, I, I.getDebugLoc(), TII.get(TargetOpcode::IMPLICIT_DEF),
                 ImpDefReg);

    // Now, create the subregister insert from SrcReg.
    Register InsertReg = MRI.createVirtualRegister(&AArch64::FPR128RegClass);
    MachineInstr &InsMI =
        *BuildMI(MBB, I, I.getDebugLoc(),
                 TII.get(TargetOpcode::INSERT_SUBREG), InsertReg)
             .addUse(ImpDefReg)
             .addUse(SrcReg)
             .addImm(AArch64::dsub);

    constrainSelectedInstRegOperands(ImpDefMI, TII, TRI, RBI);
    constrainSelectedInstRegOperands(InsMI, TII, TRI, RBI);

    // Save the register so that we can copy from it after.
    InsertRegs.push_back(InsertReg);
  }
}

// Now that we've created any necessary subregister inserts, we can
// create the copies.
//
// Perform the first copy separately as a subregister copy.
Register CopyTo = I.getOperand(0).getReg();
auto FirstCopy = MIB.buildInstr(TargetOpcode::COPY, {CopyTo}, {})
                     .addReg(InsertRegs[0], 0, ExtractSubReg);
constrainSelectedInstRegOperands(*FirstCopy, TII, TRI, RBI);

// Now, perform the remaining copies as vector lane copies.
unsigned LaneIdx = 1;
for (Register InsReg : InsertRegs) {
  Register CopyTo = I.getOperand(LaneIdx).getReg();
  MachineInstr &CopyInst =
      *BuildMI(MBB, I, I.getDebugLoc(), TII.get(CopyOpc), CopyTo)
           .addUse(InsReg)
           .addImm(LaneIdx);
  constrainSelectedInstRegOperands(CopyInst, TII, TRI, RBI);
  ++LaneIdx;
}

// Separately constrain the first copy's destination. Because of the
// limitation in constrainOperandRegClass, we can't guarantee that this will
// actually be constrained. So, do it ourselves using the second operand.
const TargetRegisterClass *RC =
    MRI.getRegClassOrNull(I.getOperand(1).getReg());
if (!RC) {
  LLVM_DEBUG(dbgs() << "Couldn't constrain copy destination.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Couldn't constrain copy destination.\n"
; } } while (false);
  return false;
}

RBI.constrainGenericRegister(CopyTo, *RC, MRI);
I.eraseFromParent();
return true;
3940}

3942bool AArch64InstructionSelector::selectConcatVectors(
  MachineInstr &I, MachineRegisterInfo &MRI) const {
assert(I.getOpcode() == TargetOpcode::G_CONCAT_VECTORS &&((I.getOpcode() == TargetOpcode::G_CONCAT_VECTORS && "Unexpected opcode"
) ? static_cast<void> (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_CONCAT_VECTORS && \"Unexpected opcode\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3945, __PRETTY_FUNCTION__))
       "Unexpected opcode")((I.getOpcode() == TargetOpcode::G_CONCAT_VECTORS && "Unexpected opcode"
) ? static_cast<void> (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_CONCAT_VECTORS && \"Unexpected opcode\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3945, __PRETTY_FUNCTION__));
Register Dst = I.getOperand(0).getReg();
Register Op1 = I.getOperand(1).getReg();
Register Op2 = I.getOperand(2).getReg();
MachineIRBuilder MIRBuilder(I);
MachineInstr *ConcatMI = emitVectorConcat(Dst, Op1, Op2, MIRBuilder);
if (!ConcatMI)
  return false;
I.eraseFromParent();
return true;
3955}

3957unsigned
3958AArch64InstructionSelector::emitConstantPoolEntry(const Constant *CPVal,
                                                MachineFunction &MF) const {
Type *CPTy = CPVal->getType();
Align Alignment = MF.getDataLayout().getPrefTypeAlign(CPTy);

MachineConstantPool *MCP = MF.getConstantPool();
return MCP->getConstantPoolIndex(CPVal, Alignment);
3965}

3967MachineInstr *AArch64InstructionSelector::emitLoadFromConstantPool(
  const Constant *CPVal, MachineIRBuilder &MIRBuilder) const {
unsigned CPIdx = emitConstantPoolEntry(CPVal, MIRBuilder.getMF());

auto Adrp =
    MIRBuilder.buildInstr(AArch64::ADRP, {&AArch64::GPR64RegClass}, {})
        .addConstantPoolIndex(CPIdx, 0, AArch64II::MO_PAGE);

MachineInstr *LoadMI = nullptr;
switch (MIRBuilder.getDataLayout().getTypeStoreSize(CPVal->getType())) {
case 16:
  LoadMI =
      &*MIRBuilder
            .buildInstr(AArch64::LDRQui, {&AArch64::FPR128RegClass}, {Adrp})
            .addConstantPoolIndex(CPIdx, 0,
                                  AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
  break;
case 8:
  LoadMI = &*MIRBuilder
               .buildInstr(AArch64::LDRDui, {&AArch64::FPR64RegClass}, {Adrp})
               .addConstantPoolIndex(
                   CPIdx, 0, AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
  break;
default:
  LLVM_DEBUG(dbgs() << "Could not load from constant pool of type "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Could not load from constant pool of type "
 << *CPVal->getType(); } } while (false)
                    << *CPVal->getType())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Could not load from constant pool of type "
 << *CPVal->getType(); } } while (false);
  return nullptr;
}
constrainSelectedInstRegOperands(*Adrp, TII, TRI, RBI);
constrainSelectedInstRegOperands(*LoadMI, TII, TRI, RBI);
return LoadMI;
3998}

4000/// Return an <Opcode, SubregIndex> pair to do an vector elt insert of a given
4001/// size and RB.
4002static std::pair<unsigned, unsigned>
4003getInsertVecEltOpInfo(const RegisterBank &RB, unsigned EltSize) {
unsigned Opc, SubregIdx;
if (RB.getID() == AArch64::GPRRegBankID) {
  if (EltSize == 16) {
    Opc = AArch64::INSvi16gpr;
    SubregIdx = AArch64::ssub;
  } else if (EltSize == 32) {
    Opc = AArch64::INSvi32gpr;
    SubregIdx = AArch64::ssub;
  } else if (EltSize == 64) {
    Opc = AArch64::INSvi64gpr;
    SubregIdx = AArch64::dsub;
  } else {
    llvm_unreachable("invalid elt size!")::llvm::llvm_unreachable_internal("invalid elt size!", "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4016);
  }
} else {
  if (EltSize == 8) {
    Opc = AArch64::INSvi8lane;
    SubregIdx = AArch64::bsub;
  } else if (EltSize == 16) {
    Opc = AArch64::INSvi16lane;
    SubregIdx = AArch64::hsub;
  } else if (EltSize == 32) {
    Opc = AArch64::INSvi32lane;
    SubregIdx = AArch64::ssub;
  } else if (EltSize == 64) {
    Opc = AArch64::INSvi64lane;
    SubregIdx = AArch64::dsub;
  } else {
    llvm_unreachable("invalid elt size!")::llvm::llvm_unreachable_internal("invalid elt size!", "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4032);
  }
}
return std::make_pair(Opc, SubregIdx);
4036}

4038MachineInstr *AArch64InstructionSelector::emitInstr(
  unsigned Opcode, std::initializer_list<llvm::DstOp> DstOps,
  std::initializer_list<llvm::SrcOp> SrcOps, MachineIRBuilder &MIRBuilder,
  const ComplexRendererFns &RenderFns) const {
assert(Opcode && "Expected an opcode?")((Opcode && "Expected an opcode?") ? static_cast<void
> (0) : __assert_fail ("Opcode && \"Expected an opcode?\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4042, __PRETTY_FUNCTION__));
assert(!isPreISelGenericOpcode(Opcode) &&((!isPreISelGenericOpcode(Opcode) && "Function should only be used to produce selected instructions!"
) ? static_cast<void> (0) : __assert_fail ("!isPreISelGenericOpcode(Opcode) && \"Function should only be used to produce selected instructions!\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4044, __PRETTY_FUNCTION__))
       "Function should only be used to produce selected instructions!")((!isPreISelGenericOpcode(Opcode) && "Function should only be used to produce selected instructions!"
) ? static_cast<void> (0) : __assert_fail ("!isPreISelGenericOpcode(Opcode) && \"Function should only be used to produce selected instructions!\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4044, __PRETTY_FUNCTION__));
auto MI = MIRBuilder.buildInstr(Opcode, DstOps, SrcOps);
if (RenderFns)
  for (auto &Fn : *RenderFns)
    Fn(MI);
constrainSelectedInstRegOperands(*MI, TII, TRI, RBI);
return &*MI;
4051}

4053MachineInstr *AArch64InstructionSelector::emitAddSub(
  const std::array<std::array<unsigned, 2>, 5> &AddrModeAndSizeToOpcode,
  Register Dst, MachineOperand &LHS, MachineOperand &RHS,
  MachineIRBuilder &MIRBuilder) const {
MachineRegisterInfo &MRI = MIRBuilder.getMF().getRegInfo();
assert(LHS.isReg() && RHS.isReg() && "Expected register operands?")((LHS.isReg() && RHS.isReg() && "Expected register operands?"
) ? static_cast<void> (0) : __assert_fail ("LHS.isReg() && RHS.isReg() && \"Expected register operands?\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4058, __PRETTY_FUNCTION__));
2
←
'?' condition is true→
auto Ty = MRI.getType(LHS.getReg());
assert(!Ty.isVector() && "Expected a scalar or pointer?")((!Ty.isVector() && "Expected a scalar or pointer?") ?
 static_cast<void> (0) : __assert_fail ("!Ty.isVector() && \"Expected a scalar or pointer?\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4060, __PRETTY_FUNCTION__));
3
←
'?' condition is true→
unsigned Size = Ty.getSizeInBits();
assert((Size == 32 || Size == 64) && "Expected a 32-bit or 64-bit type only")(((Size == 32 || Size == 64) && "Expected a 32-bit or 64-bit type only"
) ? static_cast<void> (0) : __assert_fail ("(Size == 32 || Size == 64) && \"Expected a 32-bit or 64-bit type only\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4062, __PRETTY_FUNCTION__));
4
←
Assuming 'Size' is not equal to 32→
5
←
Assuming 'Size' is equal to 64→
6
←
'?' condition is true→
bool Is32Bit = Size == 32;

// INSTRri form with positive arithmetic immediate.
if (auto Fns = selectArithImmed(RHS))
7
←
Taking false branch→
  return emitInstr(AddrModeAndSizeToOpcode[0][Is32Bit], {Dst}, {LHS},
                   MIRBuilder, Fns);

// INSTRri form with negative arithmetic immediate.
if (auto Fns = selectNegArithImmed(RHS))
8
←
Taking false branch→
  return emitInstr(AddrModeAndSizeToOpcode[3][Is32Bit], {Dst}, {LHS},
                   MIRBuilder, Fns);

// INSTRrx form.
if (auto Fns = selectArithExtendedRegister(RHS))
9
←
Calling 'AArch64InstructionSelector::selectArithExtendedRegister'→
  return emitInstr(AddrModeAndSizeToOpcode[4][Is32Bit], {Dst}, {LHS},
                   MIRBuilder, Fns);

// INSTRrs form.
if (auto Fns = selectShiftedRegister(RHS))
  return emitInstr(AddrModeAndSizeToOpcode[1][Is32Bit], {Dst}, {LHS},
                   MIRBuilder, Fns);
return emitInstr(AddrModeAndSizeToOpcode[2][Is32Bit], {Dst}, {LHS, RHS},
                 MIRBuilder);
4086}

4088MachineInstr *
4089AArch64InstructionSelector::emitADD(Register DefReg, MachineOperand &LHS,
                                  MachineOperand &RHS,
                                  MachineIRBuilder &MIRBuilder) const {
const std::array<std::array<unsigned, 2>, 5> OpcTable{
    {{AArch64::ADDXri, AArch64::ADDWri},
     {AArch64::ADDXrs, AArch64::ADDWrs},
     {AArch64::ADDXrr, AArch64::ADDWrr},
     {AArch64::SUBXri, AArch64::SUBWri},
     {AArch64::ADDXrx, AArch64::ADDWrx}}};
return emitAddSub(OpcTable, DefReg, LHS, RHS, MIRBuilder);
1
Calling 'AArch64InstructionSelector::emitAddSub'→
4099}

4101MachineInstr *
4102AArch64InstructionSelector::emitADDS(Register Dst, MachineOperand &LHS,
                                   MachineOperand &RHS,
                                   MachineIRBuilder &MIRBuilder) const {
const std::array<std::array<unsigned, 2>, 5> OpcTable{
    {{AArch64::ADDSXri, AArch64::ADDSWri},
     {AArch64::ADDSXrs, AArch64::ADDSWrs},
     {AArch64::ADDSXrr, AArch64::ADDSWrr},
     {AArch64::SUBSXri, AArch64::SUBSWri},
     {AArch64::ADDSXrx, AArch64::ADDSWrx}}};
return emitAddSub(OpcTable, Dst, LHS, RHS, MIRBuilder);
4112}

4114MachineInstr *
4115AArch64InstructionSelector::emitSUBS(Register Dst, MachineOperand &LHS,
                                   MachineOperand &RHS,
                                   MachineIRBuilder &MIRBuilder) const {
const std::array<std::array<unsigned, 2>, 5> OpcTable{
    {{AArch64::SUBSXri, AArch64::SUBSWri},
     {AArch64::SUBSXrs, AArch64::SUBSWrs},
     {AArch64::SUBSXrr, AArch64::SUBSWrr},
     {AArch64::ADDSXri, AArch64::ADDSWri},
     {AArch64::SUBSXrx, AArch64::SUBSWrx}}};
return emitAddSub(OpcTable, Dst, LHS, RHS, MIRBuilder);
4125}

4127MachineInstr *
4128AArch64InstructionSelector::emitCMN(MachineOperand &LHS, MachineOperand &RHS,
                                  MachineIRBuilder &MIRBuilder) const {
MachineRegisterInfo &MRI = MIRBuilder.getMF().getRegInfo();
bool Is32Bit = (MRI.getType(LHS.getReg()).getSizeInBits() == 32);
auto RC = Is32Bit ? &AArch64::GPR32RegClass : &AArch64::GPR64RegClass;
return emitADDS(MRI.createVirtualRegister(RC), LHS, RHS, MIRBuilder);
4134}

4136MachineInstr *
4137AArch64InstructionSelector::emitTST(MachineOperand &LHS, MachineOperand &RHS,
                                  MachineIRBuilder &MIRBuilder) const {
assert(LHS.isReg() && RHS.isReg() && "Expected register operands?")((LHS.isReg() && RHS.isReg() && "Expected register operands?"
) ? static_cast<void> (0) : __assert_fail ("LHS.isReg() && RHS.isReg() && \"Expected register operands?\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4139, __PRETTY_FUNCTION__));
MachineRegisterInfo &MRI = MIRBuilder.getMF().getRegInfo();
LLT Ty = MRI.getType(LHS.getReg());
unsigned RegSize = Ty.getSizeInBits();
bool Is32Bit = (RegSize == 32);
const unsigned OpcTable[3][2] = {{AArch64::ANDSXri, AArch64::ANDSWri},
                                 {AArch64::ANDSXrs, AArch64::ANDSWrs},
                                 {AArch64::ANDSXrr, AArch64::ANDSWrr}};
// ANDS needs a logical immediate for its immediate form. Check if we can
// fold one in.
if (auto ValAndVReg = getConstantVRegValWithLookThrough(RHS.getReg(), MRI)) {
  int64_t Imm = ValAndVReg->Value.getSExtValue();

  if (AArch64_AM::isLogicalImmediate(Imm, RegSize)) {
    auto TstMI = MIRBuilder.buildInstr(OpcTable[0][Is32Bit], {Ty}, {LHS});
    TstMI.addImm(AArch64_AM::encodeLogicalImmediate(Imm, RegSize));
    constrainSelectedInstRegOperands(*TstMI, TII, TRI, RBI);
    return &*TstMI;
  }
}

if (auto Fns = selectLogicalShiftedRegister(RHS))
  return emitInstr(OpcTable[1][Is32Bit], {Ty}, {LHS}, MIRBuilder, Fns);
return emitInstr(OpcTable[2][Is32Bit], {Ty}, {LHS, RHS}, MIRBuilder);
4163}

4165MachineInstr *AArch64InstructionSelector::emitIntegerCompare(
  MachineOperand &LHS, MachineOperand &RHS, MachineOperand &Predicate,
  MachineIRBuilder &MIRBuilder) const {
assert(LHS.isReg() && RHS.isReg() && "Expected LHS and RHS to be registers!")((LHS.isReg() && RHS.isReg() && "Expected LHS and RHS to be registers!"
) ? static_cast<void> (0) : __assert_fail ("LHS.isReg() && RHS.isReg() && \"Expected LHS and RHS to be registers!\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4168, __PRETTY_FUNCTION__));
assert(Predicate.isPredicate() && "Expected predicate?")((Predicate.isPredicate() && "Expected predicate?") ?
 static_cast<void> (0) : __assert_fail ("Predicate.isPredicate() && \"Expected predicate?\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4169, __PRETTY_FUNCTION__));
MachineRegisterInfo &MRI = MIRBuilder.getMF().getRegInfo();
LLT CmpTy = MRI.getType(LHS.getReg());
assert(!CmpTy.isVector() && "Expected scalar or pointer")((!CmpTy.isVector() && "Expected scalar or pointer") ?
 static_cast<void> (0) : __assert_fail ("!CmpTy.isVector() && \"Expected scalar or pointer\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4172, __PRETTY_FUNCTION__));
unsigned Size = CmpTy.getSizeInBits();
(void)Size;
assert((Size == 32 || Size == 64) && "Expected a 32-bit or 64-bit LHS/RHS?")(((Size == 32 || Size == 64) && "Expected a 32-bit or 64-bit LHS/RHS?"
) ? static_cast<void> (0) : __assert_fail ("(Size == 32 || Size == 64) && \"Expected a 32-bit or 64-bit LHS/RHS?\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4175, __PRETTY_FUNCTION__));
// Fold the compare into a cmn or tst if possible.
if (auto FoldCmp = tryFoldIntegerCompare(LHS, RHS, Predicate, MIRBuilder))
  return FoldCmp;
auto Dst = MRI.cloneVirtualRegister(LHS.getReg());
return emitSUBS(Dst, LHS, RHS, MIRBuilder);
4181}

4183MachineInstr *AArch64InstructionSelector::emitCSetForFCmp(
  Register Dst, CmpInst::Predicate Pred, MachineIRBuilder &MIRBuilder) const {
MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
4186#ifndef NDEBUG
LLT Ty = MRI.getType(Dst);
assert(!Ty.isVector() && Ty.getSizeInBits() == 32 &&((!Ty.isVector() && Ty.getSizeInBits() == 32 &&
 "Expected a 32-bit scalar register?") ? static_cast<void>
 (0) : __assert_fail ("!Ty.isVector() && Ty.getSizeInBits() == 32 && \"Expected a 32-bit scalar register?\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4189, __PRETTY_FUNCTION__))
       "Expected a 32-bit scalar register?")((!Ty.isVector() && Ty.getSizeInBits() == 32 &&
 "Expected a 32-bit scalar register?") ? static_cast<void>
 (0) : __assert_fail ("!Ty.isVector() && Ty.getSizeInBits() == 32 && \"Expected a 32-bit scalar register?\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4189, __PRETTY_FUNCTION__));
4190#endif
const Register ZeroReg = AArch64::WZR;
auto EmitCSet = [&](Register CsetDst, AArch64CC::CondCode CC) {
  auto CSet =
      MIRBuilder.buildInstr(AArch64::CSINCWr, {CsetDst}, {ZeroReg, ZeroReg})
          .addImm(getInvertedCondCode(CC));
  constrainSelectedInstRegOperands(*CSet, TII, TRI, RBI);
  return &*CSet;
};

AArch64CC::CondCode CC1, CC2;
changeFCMPPredToAArch64CC(Pred, CC1, CC2);
if (CC2 == AArch64CC::AL)
  return EmitCSet(Dst, CC1);

const TargetRegisterClass *RC = &AArch64::GPR32RegClass;
Register Def1Reg = MRI.createVirtualRegister(RC);
Register Def2Reg = MRI.createVirtualRegister(RC);
EmitCSet(Def1Reg, CC1);
EmitCSet(Def2Reg, CC2);
auto OrMI = MIRBuilder.buildInstr(AArch64::ORRWrr, {Dst}, {Def1Reg, Def2Reg});
constrainSelectedInstRegOperands(*OrMI, TII, TRI, RBI);
return &*OrMI;
4213}

4215MachineInstr *
4216AArch64InstructionSelector::emitFPCompare(Register LHS, Register RHS,
                                        MachineIRBuilder &MIRBuilder,
                                        Optional<CmpInst::Predicate> Pred) const {
MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
LLT Ty = MRI.getType(LHS);
if (Ty.isVector())
  return nullptr;
unsigned OpSize = Ty.getSizeInBits();
if (OpSize != 32 && OpSize != 64)
  return nullptr;

// If this is a compare against +0.0, then we don't have
// to explicitly materialize a constant.
const ConstantFP *FPImm = getConstantFPVRegVal(RHS, MRI);
bool ShouldUseImm = FPImm && (FPImm->isZero() && !FPImm->isNegative());

auto IsEqualityPred = [](CmpInst::Predicate P) {
  return P == CmpInst::FCMP_OEQ || P == CmpInst::FCMP_ONE ||
         P == CmpInst::FCMP_UEQ || P == CmpInst::FCMP_UNE;
};
if (!ShouldUseImm && Pred && IsEqualityPred(*Pred)) {
  // Try commutating the operands.
  const ConstantFP *LHSImm = getConstantFPVRegVal(LHS, MRI);
  if (LHSImm && (LHSImm->isZero() && !LHSImm->isNegative())) {
    ShouldUseImm = true;
    std::swap(LHS, RHS);
  }
}
unsigned CmpOpcTbl[2][2] = {{AArch64::FCMPSrr, AArch64::FCMPDrr},
                            {AArch64::FCMPSri, AArch64::FCMPDri}};
unsigned CmpOpc = CmpOpcTbl[ShouldUseImm][OpSize == 64];

// Partially build the compare. Decide if we need to add a use for the
// third operand based off whether or not we're comparing against 0.0.
auto CmpMI = MIRBuilder.buildInstr(CmpOpc).addUse(LHS);
if (!ShouldUseImm)
  CmpMI.addUse(RHS);
constrainSelectedInstRegOperands(*CmpMI, TII, TRI, RBI);
return &*CmpMI;
4255}

4257MachineInstr *AArch64InstructionSelector::emitVectorConcat(
  Optional<Register> Dst, Register Op1, Register Op2,
  MachineIRBuilder &MIRBuilder) const {
// We implement a vector concat by:
// 1. Use scalar_to_vector to insert the lower vector into the larger dest
// 2. Insert the upper vector into the destination's upper element
// TODO: some of this code is common with G_BUILD_VECTOR handling.
MachineRegisterInfo &MRI = MIRBuilder.getMF().getRegInfo();

const LLT Op1Ty = MRI.getType(Op1);
const LLT Op2Ty = MRI.getType(Op2);

if (Op1Ty != Op2Ty) {
  LLVM_DEBUG(dbgs() << "Could not do vector concat of differing vector tys")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Could not do vector concat of differing vector tys"
; } } while (false);
  return nullptr;
}
assert(Op1Ty.isVector() && "Expected a vector for vector concat")((Op1Ty.isVector() && "Expected a vector for vector concat"
) ? static_cast<void> (0) : __assert_fail ("Op1Ty.isVector() && \"Expected a vector for vector concat\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4273, __PRETTY_FUNCTION__));

if (Op1Ty.getSizeInBits() >= 128) {
  LLVM_DEBUG(dbgs() << "Vector concat not supported for full size vectors")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Vector concat not supported for full size vectors"
; } } while (false);
  return nullptr;
}

// At the moment we just support 64 bit vector concats.
if (Op1Ty.getSizeInBits() != 64) {
  LLVM_DEBUG(dbgs() << "Vector concat supported for 64b vectors")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Vector concat supported for 64b vectors"
; } } while (false);
  return nullptr;
}

const LLT ScalarTy = LLT::scalar(Op1Ty.getSizeInBits());
const RegisterBank &FPRBank = *RBI.getRegBank(Op1, MRI, TRI);
const TargetRegisterClass *DstRC =
    getMinClassForRegBank(FPRBank, Op1Ty.getSizeInBits() * 2);

MachineInstr *WidenedOp1 =
    emitScalarToVector(ScalarTy.getSizeInBits(), DstRC, Op1, MIRBuilder);
MachineInstr *WidenedOp2 =
    emitScalarToVector(ScalarTy.getSizeInBits(), DstRC, Op2, MIRBuilder);
if (!WidenedOp1 || !WidenedOp2) {
  LLVM_DEBUG(dbgs() << "Could not emit a vector from scalar value")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Could not emit a vector from scalar value"
; } } while (false);
  return nullptr;
}

// Now do the insert of the upper element.
unsigned InsertOpc, InsSubRegIdx;
std::tie(InsertOpc, InsSubRegIdx) =
    getInsertVecEltOpInfo(FPRBank, ScalarTy.getSizeInBits());

if (!Dst)
  Dst = MRI.createVirtualRegister(DstRC);
auto InsElt =
    MIRBuilder
        .buildInstr(InsertOpc, {*Dst}, {WidenedOp1->getOperand(0).getReg()})
        .addImm(1) /* Lane index */
        .addUse(WidenedOp2->getOperand(0).getReg())
        .addImm(0);
constrainSelectedInstRegOperands(*InsElt, TII, TRI, RBI);
return &*InsElt;
4315}

4317MachineInstr *AArch64InstructionSelector::emitFMovForFConstant(
  MachineInstr &I, MachineRegisterInfo &MRI) const {
assert(I.getOpcode() == TargetOpcode::G_FCONSTANT &&((I.getOpcode() == TargetOpcode::G_FCONSTANT && "Expected a G_FCONSTANT!"
) ? static_cast<void> (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_FCONSTANT && \"Expected a G_FCONSTANT!\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4320, __PRETTY_FUNCTION__))
       "Expected a G_FCONSTANT!")((I.getOpcode() == TargetOpcode::G_FCONSTANT && "Expected a G_FCONSTANT!"
) ? static_cast<void> (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_FCONSTANT && \"Expected a G_FCONSTANT!\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4320, __PRETTY_FUNCTION__));
MachineOperand &ImmOp = I.getOperand(1);
unsigned DefSize = MRI.getType(I.getOperand(0).getReg()).getSizeInBits();

// Only handle 32 and 64 bit defs for now.
if (DefSize != 32 && DefSize != 64)
  return nullptr;

// Don't handle null values using FMOV.
if (ImmOp.getFPImm()->isNullValue())
  return nullptr;

// Get the immediate representation for the FMOV.
const APFloat &ImmValAPF = ImmOp.getFPImm()->getValueAPF();
int Imm = DefSize == 32 ? AArch64_AM::getFP32Imm(ImmValAPF)
                        : AArch64_AM::getFP64Imm(ImmValAPF);

// If this is -1, it means the immediate can't be represented as the requested
// floating point value. Bail.
if (Imm == -1)
  return nullptr;

// Update MI to represent the new FMOV instruction, constrain it, and return.
ImmOp.ChangeToImmediate(Imm);
unsigned MovOpc = DefSize == 32 ? AArch64::FMOVSi : AArch64::FMOVDi;
I.setDesc(TII.get(MovOpc));
constrainSelectedInstRegOperands(I, TII, TRI, RBI);
return &I;
4348}

4350MachineInstr *
4351AArch64InstructionSelector::emitCSetForICMP(Register DefReg, unsigned Pred,
                                   MachineIRBuilder &MIRBuilder) const {
// CSINC increments the result when the predicate is false. Invert it.
const AArch64CC::CondCode InvCC = changeICMPPredToAArch64CC(
    CmpInst::getInversePredicate((CmpInst::Predicate)Pred));
auto I =
    MIRBuilder
  .buildInstr(AArch64::CSINCWr, {DefReg}, {Register(AArch64::WZR), Register(AArch64::WZR)})
        .addImm(InvCC);
constrainSelectedInstRegOperands(*I, TII, TRI, RBI);
return &*I;
4362}

4364std::pair<MachineInstr *, AArch64CC::CondCode>
4365AArch64InstructionSelector::emitOverflowOp(unsigned Opcode, Register Dst,
                                         MachineOperand &LHS,
                                         MachineOperand &RHS,
                                         MachineIRBuilder &MIRBuilder) const {
switch (Opcode) {
default:
  llvm_unreachable("Unexpected opcode!")::llvm::llvm_unreachable_internal("Unexpected opcode!", "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4371);
case TargetOpcode::G_SADDO:
  return std::make_pair(emitADDS(Dst, LHS, RHS, MIRBuilder), AArch64CC::VS);
case TargetOpcode::G_UADDO:
  return std::make_pair(emitADDS(Dst, LHS, RHS, MIRBuilder), AArch64CC::HS);
case TargetOpcode::G_SSUBO:
  return std::make_pair(emitSUBS(Dst, LHS, RHS, MIRBuilder), AArch64CC::VS);
}
4379}

4381bool AArch64InstructionSelector::tryOptSelect(MachineInstr &I) const {
MachineIRBuilder MIB(I);
MachineRegisterInfo &MRI = *MIB.getMRI();
// We want to recognize this pattern:
//
// $z = G_FCMP pred, $x, $y
// ...
// $w = G_SELECT $z, $a, $b
//
// Where the value of $z is *only* ever used by the G_SELECT (possibly with
// some copies/truncs in between.)
//
// If we see this, then we can emit something like this:
//
// fcmp $x, $y
// fcsel $w, $a, $b, pred
//
// Rather than emitting both of the rather long sequences in the standard
// G_FCMP/G_SELECT select methods.

// First, check if the condition is defined by a compare.
MachineInstr *CondDef = MRI.getVRegDef(I.getOperand(1).getReg());
while (CondDef) {
  // We can only fold if all of the defs have one use.
  Register CondDefReg = CondDef->getOperand(0).getReg();
  if (!MRI.hasOneNonDBGUse(CondDefReg)) {
    // Unless it's another select.
    for (const MachineInstr &UI : MRI.use_nodbg_instructions(CondDefReg)) {
      if (CondDef == &UI)
        continue;
      if (UI.getOpcode() != TargetOpcode::G_SELECT)
        return false;
    }
  }

  // We can skip over G_TRUNC since the condition is 1-bit.
  // Truncating/extending can have no impact on the value.
  unsigned Opc = CondDef->getOpcode();
  if (Opc != TargetOpcode::COPY && Opc != TargetOpcode::G_TRUNC)
    break;

  // Can't see past copies from physregs.
  if (Opc == TargetOpcode::COPY &&
      Register::isPhysicalRegister(CondDef->getOperand(1).getReg()))
    return false;

  CondDef = MRI.getVRegDef(CondDef->getOperand(1).getReg());
}

// Is the condition defined by a compare?
if (!CondDef)
  return false;

unsigned CondOpc = CondDef->getOpcode();
if (CondOpc != TargetOpcode::G_ICMP && CondOpc != TargetOpcode::G_FCMP)
  return false;

AArch64CC::CondCode CondCode;
if (CondOpc == TargetOpcode::G_ICMP) {
  auto Pred =
      static_cast<CmpInst::Predicate>(CondDef->getOperand(1).getPredicate());
  CondCode = changeICMPPredToAArch64CC(Pred);
  emitIntegerCompare(CondDef->getOperand(2), CondDef->getOperand(3),
                     CondDef->getOperand(1), MIB);
} else {
  // Get the condition code for the select.
  auto Pred =
      static_cast<CmpInst::Predicate>(CondDef->getOperand(1).getPredicate());
  AArch64CC::CondCode CondCode2;
  changeFCMPPredToAArch64CC(Pred, CondCode, CondCode2);

  // changeFCMPPredToAArch64CC sets CondCode2 to AL when we require two
  // instructions to emit the comparison.
  // TODO: Handle FCMP_UEQ and FCMP_ONE. After that, this check will be
  // unnecessary.
  if (CondCode2 != AArch64CC::AL)
    return false;

  if (!emitFPCompare(CondDef->getOperand(2).getReg(),
                     CondDef->getOperand(3).getReg(), MIB)) {
    LLVM_DEBUG(dbgs() << "Couldn't emit compare for select!\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Couldn't emit compare for select!\n"
; } } while (false);
    return false;
  }
}

// Emit the select.
emitSelect(I.getOperand(0).getReg(), I.getOperand(2).getReg(),
           I.getOperand(3).getReg(), CondCode, MIB);
I.eraseFromParent();
return true;
4471}

4473MachineInstr *AArch64InstructionSelector::tryFoldIntegerCompare(
  MachineOperand &LHS, MachineOperand &RHS, MachineOperand &Predicate,
  MachineIRBuilder &MIRBuilder) const {
assert(LHS.isReg() && RHS.isReg() && Predicate.isPredicate() &&((LHS.isReg() && RHS.isReg() && Predicate.isPredicate
() && "Unexpected MachineOperand") ? static_cast<void
> (0) : __assert_fail ("LHS.isReg() && RHS.isReg() && Predicate.isPredicate() && \"Unexpected MachineOperand\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4477, __PRETTY_FUNCTION__))
       "Unexpected MachineOperand")((LHS.isReg() && RHS.isReg() && Predicate.isPredicate
() && "Unexpected MachineOperand") ? static_cast<void
> (0) : __assert_fail ("LHS.isReg() && RHS.isReg() && Predicate.isPredicate() && \"Unexpected MachineOperand\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4477, __PRETTY_FUNCTION__));
MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
// We want to find this sort of thing:
// x = G_SUB 0, y
// G_ICMP z, x
//
// In this case, we can fold the G_SUB into the G_ICMP using a CMN instead.
// e.g:
//
// cmn z, y

// Helper lambda to detect the subtract followed by the compare.
// Takes in the def of the LHS or RHS, and checks if it's a subtract from 0.
auto IsCMN = [&](MachineInstr *DefMI, const AArch64CC::CondCode &CC) {
  if (!DefMI || DefMI->getOpcode() != TargetOpcode::G_SUB)
    return false;

  // Need to make sure NZCV is the same at the end of the transformation.
  if (CC != AArch64CC::EQ && CC != AArch64CC::NE)
    return false;

  // We want to match against SUBs.
  if (DefMI->getOpcode() != TargetOpcode::G_SUB)
    return false;

  // Make sure that we're getting
  // x = G_SUB 0, y
  auto ValAndVReg =
      getConstantVRegValWithLookThrough(DefMI->getOperand(1).getReg(), MRI);
  if (!ValAndVReg || ValAndVReg->Value != 0)
    return false;

  // This can safely be represented as a CMN.
  return true;
};

// Check if the RHS or LHS of the G_ICMP is defined by a SUB
MachineInstr *LHSDef = getDefIgnoringCopies(LHS.getReg(), MRI);
MachineInstr *RHSDef = getDefIgnoringCopies(RHS.getReg(), MRI);
CmpInst::Predicate P = (CmpInst::Predicate)Predicate.getPredicate();
const AArch64CC::CondCode CC = changeICMPPredToAArch64CC(P);

// Given this:
//
// x = G_SUB 0, y
// G_ICMP x, z
//
// Produce this:
//
// cmn y, z
if (IsCMN(LHSDef, CC))
  return emitCMN(LHSDef->getOperand(2), RHS, MIRBuilder);

// Same idea here, but with the RHS of the compare instead:
//
// Given this:
//
// x = G_SUB 0, y
// G_ICMP z, x
//
// Produce this:
//
// cmn z, y
if (IsCMN(RHSDef, CC))
  return emitCMN(LHS, RHSDef->getOperand(2), MIRBuilder);

// Given this:
//
// z = G_AND x, y
// G_ICMP z, 0
//
// Produce this if the compare is signed:
//
// tst x, y
if (!CmpInst::isUnsigned(P) && LHSDef &&
    LHSDef->getOpcode() == TargetOpcode::G_AND) {
  // Make sure that the RHS is 0.
  auto ValAndVReg = getConstantVRegValWithLookThrough(RHS.getReg(), MRI);
  if (!ValAndVReg || ValAndVReg->Value != 0)
    return nullptr;

  return emitTST(LHSDef->getOperand(1),
                 LHSDef->getOperand(2), MIRBuilder);
}

return nullptr;
4563}

4565bool AArch64InstructionSelector::selectShuffleVector(
  MachineInstr &I, MachineRegisterInfo &MRI) const {
const LLT DstTy = MRI.getType(I.getOperand(0).getReg());
Register Src1Reg = I.getOperand(1).getReg();
const LLT Src1Ty = MRI.getType(Src1Reg);
Register Src2Reg = I.getOperand(2).getReg();
const LLT Src2Ty = MRI.getType(Src2Reg);
ArrayRef<int> Mask = I.getOperand(3).getShuffleMask();

MachineBasicBlock &MBB = *I.getParent();
MachineFunction &MF = *MBB.getParent();
LLVMContext &Ctx = MF.getFunction().getContext();

// G_SHUFFLE_VECTOR is weird in that the source operands can be scalars, if
// it's originated from a <1 x T> type. Those should have been lowered into
// G_BUILD_VECTOR earlier.
if (!Src1Ty.isVector() || !Src2Ty.isVector()) {
  LLVM_DEBUG(dbgs() << "Could not select a \"scalar\" G_SHUFFLE_VECTOR\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Could not select a \"scalar\" G_SHUFFLE_VECTOR\n"
; } } while (false);
  return false;
}

unsigned BytesPerElt = DstTy.getElementType().getSizeInBits() / 8;

SmallVector<Constant *, 64> CstIdxs;
for (int Val : Mask) {
  // For now, any undef indexes we'll just assume to be 0. This should be
  // optimized in future, e.g. to select DUP etc.
  Val = Val < 0 ? 0 : Val;
  for (unsigned Byte = 0; Byte < BytesPerElt; ++Byte) {
    unsigned Offset = Byte + Val * BytesPerElt;
    CstIdxs.emplace_back(ConstantInt::get(Type::getInt8Ty(Ctx), Offset));
  }
}

MachineIRBuilder MIRBuilder(I);

// Use a constant pool to load the index vector for TBL.
Constant *CPVal = ConstantVector::get(CstIdxs);
MachineInstr *IndexLoad = emitLoadFromConstantPool(CPVal, MIRBuilder);
if (!IndexLoad) {
  LLVM_DEBUG(dbgs() << "Could not load from a constant pool")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Could not load from a constant pool"
; } } while (false);
  return false;
}

if (DstTy.getSizeInBits() != 128) {
  assert(DstTy.getSizeInBits() == 64 && "Unexpected shuffle result ty")((DstTy.getSizeInBits() == 64 && "Unexpected shuffle result ty"
) ? static_cast<void> (0) : __assert_fail ("DstTy.getSizeInBits() == 64 && \"Unexpected shuffle result ty\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4610, __PRETTY_FUNCTION__));
  // This case can be done with TBL1.
  MachineInstr *Concat = emitVectorConcat(None, Src1Reg, Src2Reg, MIRBuilder);
  if (!Concat) {
    LLVM_DEBUG(dbgs() << "Could not do vector concat for tbl1")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Could not do vector concat for tbl1"
; } } while (false);
    return false;
  }

  // The constant pool load will be 64 bits, so need to convert to FPR128 reg.
  IndexLoad =
      emitScalarToVector(64, &AArch64::FPR128RegClass,
                         IndexLoad->getOperand(0).getReg(), MIRBuilder);

  auto TBL1 = MIRBuilder.buildInstr(
      AArch64::TBLv16i8One, {&AArch64::FPR128RegClass},
      {Concat->getOperand(0).getReg(), IndexLoad->getOperand(0).getReg()});
  constrainSelectedInstRegOperands(*TBL1, TII, TRI, RBI);

  auto Copy =
      MIRBuilder
          .buildInstr(TargetOpcode::COPY, {I.getOperand(0).getReg()}, {})
          .addReg(TBL1.getReg(0), 0, AArch64::dsub);
  RBI.constrainGenericRegister(Copy.getReg(0), AArch64::FPR64RegClass, MRI);
  I.eraseFromParent();
  return true;
}

// For TBL2 we need to emit a REG_SEQUENCE to tie together two consecutive
// Q registers for regalloc.
auto RegSeq = MIRBuilder
                  .buildInstr(TargetOpcode::REG_SEQUENCE,
                              {&AArch64::QQRegClass}, {Src1Reg})
                  .addImm(AArch64::qsub0)
                  .addUse(Src2Reg)
                  .addImm(AArch64::qsub1);

auto TBL2 = MIRBuilder.buildInstr(AArch64::TBLv16i8Two, {I.getOperand(0)},
                                  {RegSeq, IndexLoad->getOperand(0)});
constrainSelectedInstRegOperands(*RegSeq, TII, TRI, RBI);
constrainSelectedInstRegOperands(*TBL2, TII, TRI, RBI);
I.eraseFromParent();
return true;
4652}

4654MachineInstr *AArch64InstructionSelector::emitLaneInsert(
  Optional<Register> DstReg, Register SrcReg, Register EltReg,
  unsigned LaneIdx, const RegisterBank &RB,
  MachineIRBuilder &MIRBuilder) const {
MachineInstr *InsElt = nullptr;
const TargetRegisterClass *DstRC = &AArch64::FPR128RegClass;
MachineRegisterInfo &MRI = *MIRBuilder.getMRI();

// Create a register to define with the insert if one wasn't passed in.
if (!DstReg)
  DstReg = MRI.createVirtualRegister(DstRC);

unsigned EltSize = MRI.getType(EltReg).getSizeInBits();
unsigned Opc = getInsertVecEltOpInfo(RB, EltSize).first;

if (RB.getID() == AArch64::FPRRegBankID) {
  auto InsSub = emitScalarToVector(EltSize, DstRC, EltReg, MIRBuilder);
  InsElt = MIRBuilder.buildInstr(Opc, {*DstReg}, {SrcReg})
               .addImm(LaneIdx)
               .addUse(InsSub->getOperand(0).getReg())
               .addImm(0);
} else {
  InsElt = MIRBuilder.buildInstr(Opc, {*DstReg}, {SrcReg})
               .addImm(LaneIdx)
               .addUse(EltReg);
}

constrainSelectedInstRegOperands(*InsElt, TII, TRI, RBI);
return InsElt;
4683}

4685bool AArch64InstructionSelector::selectInsertElt(
  MachineInstr &I, MachineRegisterInfo &MRI) const {
assert(I.getOpcode() == TargetOpcode::G_INSERT_VECTOR_ELT)((I.getOpcode() == TargetOpcode::G_INSERT_VECTOR_ELT) ? static_cast
<void> (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_INSERT_VECTOR_ELT"
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4687, __PRETTY_FUNCTION__));

// Get information on the destination.
Register DstReg = I.getOperand(0).getReg();
const LLT DstTy = MRI.getType(DstReg);
unsigned VecSize = DstTy.getSizeInBits();

// Get information on the element we want to insert into the destination.
Register EltReg = I.getOperand(2).getReg();
const LLT EltTy = MRI.getType(EltReg);
unsigned EltSize = EltTy.getSizeInBits();
if (EltSize < 16 || EltSize > 64)
  return false; // Don't support all element types yet.

// Find the definition of the index. Bail out if it's not defined by a
// G_CONSTANT.
Register IdxReg = I.getOperand(3).getReg();
auto VRegAndVal = getConstantVRegValWithLookThrough(IdxReg, MRI);
if (!VRegAndVal)
  return false;
unsigned LaneIdx = VRegAndVal->Value.getSExtValue();

// Perform the lane insert.
Register SrcReg = I.getOperand(1).getReg();
const RegisterBank &EltRB = *RBI.getRegBank(EltReg, MRI, TRI);
MachineIRBuilder MIRBuilder(I);

if (VecSize < 128) {
  // If the vector we're inserting into is smaller than 128 bits, widen it
  // to 128 to do the insert.
  MachineInstr *ScalarToVec = emitScalarToVector(
      VecSize, &AArch64::FPR128RegClass, SrcReg, MIRBuilder);
  if (!ScalarToVec)
    return false;
  SrcReg = ScalarToVec->getOperand(0).getReg();
}

// Create an insert into a new FPR128 register.
// Note that if our vector is already 128 bits, we end up emitting an extra
// register.
MachineInstr *InsMI =
    emitLaneInsert(None, SrcReg, EltReg, LaneIdx, EltRB, MIRBuilder);

if (VecSize < 128) {
  // If we had to widen to perform the insert, then we have to demote back to
  // the original size to get the result we want.
  Register DemoteVec = InsMI->getOperand(0).getReg();
  const TargetRegisterClass *RC =
      getMinClassForRegBank(*RBI.getRegBank(DemoteVec, MRI, TRI), VecSize);
  if (RC != &AArch64::FPR32RegClass && RC != &AArch64::FPR64RegClass) {
    LLVM_DEBUG(dbgs() << "Unsupported register class!\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unsupported register class!\n"
; } } while (false);
    return false;
  }
  unsigned SubReg = 0;
  if (!getSubRegForClass(RC, TRI, SubReg))
    return false;
  if (SubReg != AArch64::ssub && SubReg != AArch64::dsub) {
    LLVM_DEBUG(dbgs() << "Unsupported destination size! (" << VecSizedo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unsupported destination size! ("
 << VecSize << "\n"; } } while (false)
                      << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unsupported destination size! ("
 << VecSize << "\n"; } } while (false);
    return false;
  }
  MIRBuilder.buildInstr(TargetOpcode::COPY, {DstReg}, {})
      .addReg(DemoteVec, 0, SubReg);
  RBI.constrainGenericRegister(DstReg, *RC, MRI);
} else {
  // No widening needed.
  InsMI->getOperand(0).setReg(DstReg);
  constrainSelectedInstRegOperands(*InsMI, TII, TRI, RBI);
}

I.eraseFromParent();
return true;
4759}

4761bool AArch64InstructionSelector::tryOptConstantBuildVec(
  MachineInstr &I, LLT DstTy, MachineRegisterInfo &MRI) const {
assert(I.getOpcode() == TargetOpcode::G_BUILD_VECTOR)((I.getOpcode() == TargetOpcode::G_BUILD_VECTOR) ? static_cast
<void> (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_BUILD_VECTOR"
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4763, __PRETTY_FUNCTION__));
unsigned DstSize = DstTy.getSizeInBits();
assert(DstSize <= 128 && "Unexpected build_vec type!")((DstSize <= 128 && "Unexpected build_vec type!") ?
 static_cast<void> (0) : __assert_fail ("DstSize <= 128 && \"Unexpected build_vec type!\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4765, __PRETTY_FUNCTION__));
if (DstSize < 32)
  return false;
// Check if we're building a constant vector, in which case we want to
// generate a constant pool load instead of a vector insert sequence.
SmallVector<Constant *, 16> Csts;
for (unsigned Idx = 1; Idx < I.getNumOperands(); ++Idx) {
  // Try to find G_CONSTANT or G_FCONSTANT
  auto *OpMI =
      getOpcodeDef(TargetOpcode::G_CONSTANT, I.getOperand(Idx).getReg(), MRI);
  if (OpMI)
    Csts.emplace_back(
        const_cast<ConstantInt *>(OpMI->getOperand(1).getCImm()));
  else if ((OpMI = getOpcodeDef(TargetOpcode::G_FCONSTANT,
                                I.getOperand(Idx).getReg(), MRI)))
    Csts.emplace_back(
        const_cast<ConstantFP *>(OpMI->getOperand(1).getFPImm()));
  else
    return false;
}
Constant *CV = ConstantVector::get(Csts);
MachineIRBuilder MIB(I);
if (CV->isNullValue()) {
  // Until the importer can support immAllZerosV in pattern leaf nodes,
  // select a zero move manually here.
  Register DstReg = I.getOperand(0).getReg();
  if (DstSize == 128) {
    auto Mov = MIB.buildInstr(AArch64::MOVIv2d_ns, {DstReg}, {}).addImm(0);
    I.eraseFromParent();
    return constrainSelectedInstRegOperands(*Mov, TII, TRI, RBI);
  } else if (DstSize == 64) {
    auto Mov =
        MIB.buildInstr(AArch64::MOVIv2d_ns, {&AArch64::FPR128RegClass}, {})
            .addImm(0);
    MIB.buildInstr(TargetOpcode::COPY, {DstReg}, {})
        .addReg(Mov.getReg(0), 0, AArch64::dsub);
    I.eraseFromParent();
    return RBI.constrainGenericRegister(DstReg, AArch64::FPR64RegClass, MRI);
  }
}
auto *CPLoad = emitLoadFromConstantPool(CV, MIB);
if (!CPLoad) {
  LLVM_DEBUG(dbgs() << "Could not generate cp load for build_vector")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Could not generate cp load for build_vector"
; } } while (false);
  return false;
}
MIB.buildCopy(I.getOperand(0), CPLoad->getOperand(0));
RBI.constrainGenericRegister(I.getOperand(0).getReg(),
                             *MRI.getRegClass(CPLoad->getOperand(0).getReg()),
                             MRI);
I.eraseFromParent();
return true;
4816}

4818bool AArch64InstructionSelector::selectBuildVector(
  MachineInstr &I, MachineRegisterInfo &MRI) const {
assert(I.getOpcode() == TargetOpcode::G_BUILD_VECTOR)((I.getOpcode() == TargetOpcode::G_BUILD_VECTOR) ? static_cast
<void> (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_BUILD_VECTOR"
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4820, __PRETTY_FUNCTION__));
// Until we port more of the optimized selections, for now just use a vector
// insert sequence.
const LLT DstTy = MRI.getType(I.getOperand(0).getReg());
const LLT EltTy = MRI.getType(I.getOperand(1).getReg());
unsigned EltSize = EltTy.getSizeInBits();

if (tryOptConstantBuildVec(I, DstTy, MRI))
  return true;
if (EltSize < 16 || EltSize > 64)
  return false; // Don't support all element types yet.
const RegisterBank &RB = *RBI.getRegBank(I.getOperand(1).getReg(), MRI, TRI);
MachineIRBuilder MIRBuilder(I);

const TargetRegisterClass *DstRC = &AArch64::FPR128RegClass;
MachineInstr *ScalarToVec =
    emitScalarToVector(DstTy.getElementType().getSizeInBits(), DstRC,
                       I.getOperand(1).getReg(), MIRBuilder);
if (!ScalarToVec)
  return false;

Register DstVec = ScalarToVec->getOperand(0).getReg();
unsigned DstSize = DstTy.getSizeInBits();

// Keep track of the last MI we inserted. Later on, we might be able to save
// a copy using it.
MachineInstr *PrevMI = nullptr;
for (unsigned i = 2, e = DstSize / EltSize + 1; i < e; ++i) {
  // Note that if we don't do a subregister copy, we can end up making an
  // extra register.
  PrevMI = &*emitLaneInsert(None, DstVec, I.getOperand(i).getReg(), i - 1, RB,
                            MIRBuilder);
  DstVec = PrevMI->getOperand(0).getReg();
}

// If DstTy's size in bits is less than 128, then emit a subregister copy
// from DstVec to the last register we've defined.
if (DstSize < 128) {
  // Force this to be FPR using the destination vector.
  const TargetRegisterClass *RC =
      getMinClassForRegBank(*RBI.getRegBank(DstVec, MRI, TRI), DstSize);
  if (!RC)
    return false;
  if (RC != &AArch64::FPR32RegClass && RC != &AArch64::FPR64RegClass) {
    LLVM_DEBUG(dbgs() << "Unsupported register class!\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unsupported register class!\n"
; } } while (false);
    return false;
  }

  unsigned SubReg = 0;
  if (!getSubRegForClass(RC, TRI, SubReg))
    return false;
  if (SubReg != AArch64::ssub && SubReg != AArch64::dsub) {
    LLVM_DEBUG(dbgs() << "Unsupported destination size! (" << DstSizedo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unsupported destination size! ("
 << DstSize << "\n"; } } while (false)
                      << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unsupported destination size! ("
 << DstSize << "\n"; } } while (false);
    return false;
  }

  Register Reg = MRI.createVirtualRegister(RC);
  Register DstReg = I.getOperand(0).getReg();

  MIRBuilder.buildInstr(TargetOpcode::COPY, {DstReg}, {})
      .addReg(DstVec, 0, SubReg);
  MachineOperand &RegOp = I.getOperand(1);
  RegOp.setReg(Reg);
  RBI.constrainGenericRegister(DstReg, *RC, MRI);
} else {
  // We don't need a subregister copy. Save a copy by re-using the
  // destination register on the final insert.
  assert(PrevMI && "PrevMI was null?")((PrevMI && "PrevMI was null?") ? static_cast<void
> (0) : __assert_fail ("PrevMI && \"PrevMI was null?\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4888, __PRETTY_FUNCTION__));
  PrevMI->getOperand(0).setReg(I.getOperand(0).getReg());
  constrainSelectedInstRegOperands(*PrevMI, TII, TRI, RBI);
}

I.eraseFromParent();
return true;
4895}

4897/// Helper function to find an intrinsic ID on an a MachineInstr. Returns the
4898/// ID if it exists, and 0 otherwise.
4899static unsigned findIntrinsicID(MachineInstr &I) {
auto IntrinOp = find_if(I.operands(), [&](const MachineOperand &Op) {
  return Op.isIntrinsicID();
});
if (IntrinOp == I.operands_end())
  return 0;
return IntrinOp->getIntrinsicID();
4906}

4908bool AArch64InstructionSelector::selectIntrinsicWithSideEffects(
  MachineInstr &I, MachineRegisterInfo &MRI) const {
// Find the intrinsic ID.
unsigned IntrinID = findIntrinsicID(I);
if (!IntrinID)
  return false;
MachineIRBuilder MIRBuilder(I);

// Select the instruction.
switch (IntrinID) {
default:
  return false;
case Intrinsic::trap:
  MIRBuilder.buildInstr(AArch64::BRK, {}, {}).addImm(1);
  break;
case Intrinsic::debugtrap:
  MIRBuilder.buildInstr(AArch64::BRK, {}, {}).addImm(0xF000);
  break;
case Intrinsic::ubsantrap:
  MIRBuilder.buildInstr(AArch64::BRK, {}, {})
      .addImm(I.getOperand(1).getImm() | ('U' << 8));
  break;
}

I.eraseFromParent();
return true;
4934}

4936bool AArch64InstructionSelector::selectIntrinsic(MachineInstr &I,
                                               MachineRegisterInfo &MRI) {
unsigned IntrinID = findIntrinsicID(I);
if (!IntrinID)
  return false;
MachineIRBuilder MIRBuilder(I);

switch (IntrinID) {
default:
  break;
case Intrinsic::aarch64_crypto_sha1h: {
  Register DstReg = I.getOperand(0).getReg();
  Register SrcReg = I.getOperand(2).getReg();

  // FIXME: Should this be an assert?
  if (MRI.getType(DstReg).getSizeInBits() != 32 ||
      MRI.getType(SrcReg).getSizeInBits() != 32)
    return false;

  // The operation has to happen on FPRs. Set up some new FPR registers for
  // the source and destination if they are on GPRs.
  if (RBI.getRegBank(SrcReg, MRI, TRI)->getID() != AArch64::FPRRegBankID) {
    SrcReg = MRI.createVirtualRegister(&AArch64::FPR32RegClass);
    MIRBuilder.buildCopy({SrcReg}, {I.getOperand(2)});

    // Make sure the copy ends up getting constrained properly.
    RBI.constrainGenericRegister(I.getOperand(2).getReg(),
                                 AArch64::GPR32RegClass, MRI);
  }

  if (RBI.getRegBank(DstReg, MRI, TRI)->getID() != AArch64::FPRRegBankID)
    DstReg = MRI.createVirtualRegister(&AArch64::FPR32RegClass);

  // Actually insert the instruction.
  auto SHA1Inst = MIRBuilder.buildInstr(AArch64::SHA1Hrr, {DstReg}, {SrcReg});
  constrainSelectedInstRegOperands(*SHA1Inst, TII, TRI, RBI);

  // Did we create a new register for the destination?
  if (DstReg != I.getOperand(0).getReg()) {
    // Yep. Copy the result of the instruction back into the original
    // destination.
    MIRBuilder.buildCopy({I.getOperand(0)}, {DstReg});
    RBI.constrainGenericRegister(I.getOperand(0).getReg(),
                                 AArch64::GPR32RegClass, MRI);
  }

  I.eraseFromParent();
  return true;
}
case Intrinsic::frameaddress:
case Intrinsic::returnaddress: {
  MachineFunction &MF = *I.getParent()->getParent();
  MachineFrameInfo &MFI = MF.getFrameInfo();

  unsigned Depth = I.getOperand(2).getImm();
  Register DstReg = I.getOperand(0).getReg();
  RBI.constrainGenericRegister(DstReg, AArch64::GPR64RegClass, MRI);

  if (Depth == 0 && IntrinID == Intrinsic::returnaddress) {
    if (!MFReturnAddr) {
      // Insert the copy from LR/X30 into the entry block, before it can be
      // clobbered by anything.
      MFI.setReturnAddressIsTaken(true);
      MFReturnAddr = getFunctionLiveInPhysReg(MF, TII, AArch64::LR,
                                              AArch64::GPR64RegClass);
    }

    if (STI.hasPAuth()) {
      MIRBuilder.buildInstr(AArch64::XPACI, {DstReg}, {MFReturnAddr});
    } else {
      MIRBuilder.buildCopy({Register(AArch64::LR)}, {MFReturnAddr});
      MIRBuilder.buildInstr(AArch64::XPACLRI);
      MIRBuilder.buildCopy({DstReg}, {Register(AArch64::LR)});
    }

    I.eraseFromParent();
    return true;
  }

  MFI.setFrameAddressIsTaken(true);
  Register FrameAddr(AArch64::FP);
  while (Depth--) {
    Register NextFrame = MRI.createVirtualRegister(&AArch64::GPR64spRegClass);
    auto Ldr =
        MIRBuilder.buildInstr(AArch64::LDRXui, {NextFrame}, {FrameAddr})
            .addImm(0);
    constrainSelectedInstRegOperands(*Ldr, TII, TRI, RBI);
    FrameAddr = NextFrame;
  }

  if (IntrinID == Intrinsic::frameaddress)
    MIRBuilder.buildCopy({DstReg}, {FrameAddr});
  else {
    MFI.setReturnAddressIsTaken(true);

    if (STI.hasPAuth()) {
      Register TmpReg = MRI.createVirtualRegister(&AArch64::GPR64RegClass);
      MIRBuilder.buildInstr(AArch64::LDRXui, {TmpReg}, {FrameAddr}).addImm(1);
      MIRBuilder.buildInstr(AArch64::XPACI, {DstReg}, {TmpReg});
    } else {
      MIRBuilder.buildInstr(AArch64::LDRXui, {Register(AArch64::LR)}, {FrameAddr}).addImm(1);
      MIRBuilder.buildInstr(AArch64::XPACLRI);
      MIRBuilder.buildCopy({DstReg}, {Register(AArch64::LR)});
    }
  }

  I.eraseFromParent();
  return true;
}
}
return false;
5047}

5049InstructionSelector::ComplexRendererFns
5050AArch64InstructionSelector::selectShiftA_32(const MachineOperand &Root) const {
auto MaybeImmed = getImmedFromMO(Root);
if (MaybeImmed == None || *MaybeImmed > 31)
  return None;
uint64_t Enc = (32 - *MaybeImmed) & 0x1f;
return {{[=](MachineInstrBuilder &MIB) { MIB.addImm(Enc); }}};
5056}

5058InstructionSelector::ComplexRendererFns
5059AArch64InstructionSelector::selectShiftB_32(const MachineOperand &Root) const {
auto MaybeImmed = getImmedFromMO(Root);
if (MaybeImmed == None || *MaybeImmed > 31)
  return None;
uint64_t Enc = 31 - *MaybeImmed;
return {{[=](MachineInstrBuilder &MIB) { MIB.addImm(Enc); }}};
5065}

5067InstructionSelector::ComplexRendererFns
5068AArch64InstructionSelector::selectShiftA_64(const MachineOperand &Root) const {
auto MaybeImmed = getImmedFromMO(Root);
if (MaybeImmed == None || *MaybeImmed > 63)
  return None;
uint64_t Enc = (64 - *MaybeImmed) & 0x3f;
return {{[=](MachineInstrBuilder &MIB) { MIB.addImm(Enc); }}};
5074}

5076InstructionSelector::ComplexRendererFns
5077AArch64InstructionSelector::selectShiftB_64(const MachineOperand &Root) const {
auto MaybeImmed = getImmedFromMO(Root);
if (MaybeImmed == None || *MaybeImmed > 63)
  return None;
uint64_t Enc = 63 - *MaybeImmed;
return {{[=](MachineInstrBuilder &MIB) { MIB.addImm(Enc); }}};
5083}

5085/// Helper to select an immediate value that can be represented as a 12-bit
5086/// value shifted left by either 0 or 12. If it is possible to do so, return
5087/// the immediate and shift value. If not, return None.
5088///
5089/// Used by selectArithImmed and selectNegArithImmed.
5090InstructionSelector::ComplexRendererFns
5091AArch64InstructionSelector::select12BitValueWithLeftShift(
  uint64_t Immed) const {
unsigned ShiftAmt;
if (Immed >> 12 == 0) {
  ShiftAmt = 0;
} else if ((Immed & 0xfff) == 0 && Immed >> 24 == 0) {
  ShiftAmt = 12;
  Immed = Immed >> 12;
} else
  return None;

unsigned ShVal = AArch64_AM::getShifterImm(AArch64_AM::LSL, ShiftAmt);
return {{
    [=](MachineInstrBuilder &MIB) { MIB.addImm(Immed); },
    [=](MachineInstrBuilder &MIB) { MIB.addImm(ShVal); },
}};
5107}

5109/// SelectArithImmed - Select an immediate value that can be represented as
5110/// a 12-bit value shifted left by either 0 or 12.  If so, return true with
5111/// Val set to the 12-bit value and Shift set to the shifter operand.
5112InstructionSelector::ComplexRendererFns
5113AArch64InstructionSelector::selectArithImmed(MachineOperand &Root) const {
// This function is called from the addsub_shifted_imm ComplexPattern,
// which lists [imm] as the list of opcode it's interested in, however
// we still need to check whether the operand is actually an immediate
// here because the ComplexPattern opcode list is only used in
// root-level opcode matching.
auto MaybeImmed = getImmedFromMO(Root);
if (MaybeImmed == None)
  return None;
return select12BitValueWithLeftShift(*MaybeImmed);
5123}

5125/// SelectNegArithImmed - As above, but negates the value before trying to
5126/// select it.
5127InstructionSelector::ComplexRendererFns
5128AArch64InstructionSelector::selectNegArithImmed(MachineOperand &Root) const {
// We need a register here, because we need to know if we have a 64 or 32
// bit immediate.
if (!Root.isReg())
  return None;
auto MaybeImmed = getImmedFromMO(Root);
if (MaybeImmed == None)
  return None;
uint64_t Immed = *MaybeImmed;

// This negation is almost always valid, but "cmp wN, #0" and "cmn wN, #0"
// have the opposite effect on the C flag, so this pattern mustn't match under
// those circumstances.
if (Immed == 0)
  return None;

// Check if we're dealing with a 32-bit type on the root or a 64-bit type on
// the root.
MachineRegisterInfo &MRI = Root.getParent()->getMF()->getRegInfo();
if (MRI.getType(Root.getReg()).getSizeInBits() == 32)
  Immed = ~((uint32_t)Immed) + 1;
else
  Immed = ~Immed + 1ULL;

if (Immed & 0xFFFFFFFFFF000000ULL)
  return None;

Immed &= 0xFFFFFFULL;
return select12BitValueWithLeftShift(Immed);
5157}

5159/// Return true if it is worth folding MI into an extended register. That is,
5160/// if it's safe to pull it into the addressing mode of a load or store as a
5161/// shift.
5162bool AArch64InstructionSelector::isWorthFoldingIntoExtendedReg(
  MachineInstr &MI, const MachineRegisterInfo &MRI) const {
// Always fold if there is one use, or if we're optimizing for size.
Register DefReg = MI.getOperand(0).getReg();
if (MRI.hasOneNonDBGUse(DefReg) ||
    MI.getParent()->getParent()->getFunction().hasMinSize())
  return true;

// It's better to avoid folding and recomputing shifts when we don't have a
// fastpath.
if (!STI.hasLSLFast())
  return false;

// We have a fastpath, so folding a shift in and potentially computing it
// many times may be beneficial. Check if this is only used in memory ops.
// If it is, then we should fold.
return all_of(MRI.use_nodbg_instructions(DefReg),
              [](MachineInstr &Use) { return Use.mayLoadOrStore(); });
5180}

5182static bool isSignExtendShiftType(AArch64_AM::ShiftExtendType Type) {
switch (Type) {
case AArch64_AM::SXTB:
case AArch64_AM::SXTH:
case AArch64_AM::SXTW:
  return true;
default:
  return false;
}
5191}

5193InstructionSelector::ComplexRendererFns
5194AArch64InstructionSelector::selectExtendedSHL(
  MachineOperand &Root, MachineOperand &Base, MachineOperand &Offset,
  unsigned SizeInBytes, bool WantsExt) const {
assert(Base.isReg() && "Expected base to be a register operand")((Base.isReg() && "Expected base to be a register operand"
) ? static_cast<void> (0) : __assert_fail ("Base.isReg() && \"Expected base to be a register operand\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 5197, __PRETTY_FUNCTION__));
assert(Offset.isReg() && "Expected offset to be a register operand")((Offset.isReg() && "Expected offset to be a register operand"
) ? static_cast<void> (0) : __assert_fail ("Offset.isReg() && \"Expected offset to be a register operand\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 5198, __PRETTY_FUNCTION__));

MachineRegisterInfo &MRI = Root.getParent()->getMF()->getRegInfo();
MachineInstr *OffsetInst = MRI.getVRegDef(Offset.getReg());
if (!OffsetInst)
  return None;

unsigned OffsetOpc = OffsetInst->getOpcode();
bool LookedThroughZExt = false;
if (OffsetOpc != TargetOpcode::G_SHL && OffsetOpc != TargetOpcode::G_MUL) {
  // Try to look through a ZEXT.
  if (OffsetOpc != TargetOpcode::G_ZEXT || !WantsExt)
    return None;

  OffsetInst = MRI.getVRegDef(OffsetInst->getOperand(1).getReg());
  OffsetOpc = OffsetInst->getOpcode();
  LookedThroughZExt = true;

  if (OffsetOpc != TargetOpcode::G_SHL && OffsetOpc != TargetOpcode::G_MUL)
    return None;
}
// Make sure that the memory op is a valid size.
int64_t LegalShiftVal = Log2_32(SizeInBytes);
if (LegalShiftVal == 0)
  return None;
if (!isWorthFoldingIntoExtendedReg(*OffsetInst, MRI))
  return None;

// Now, try to find the specific G_CONSTANT. Start by assuming that the
// register we will offset is the LHS, and the register containing the
// constant is the RHS.
Register OffsetReg = OffsetInst->getOperand(1).getReg();
Register ConstantReg = OffsetInst->getOperand(2).getReg();
auto ValAndVReg = getConstantVRegValWithLookThrough(ConstantReg, MRI);
if (!ValAndVReg) {
  // We didn't get a constant on the RHS. If the opcode is a shift, then
  // we're done.
  if (OffsetOpc == TargetOpcode::G_SHL)
    return None;

  // If we have a G_MUL, we can use either register. Try looking at the RHS.
  std::swap(OffsetReg, ConstantReg);
  ValAndVReg = getConstantVRegValWithLookThrough(ConstantReg, MRI);
  if (!ValAndVReg)
    return None;
}

// The value must fit into 3 bits, and must be positive. Make sure that is
// true.
int64_t ImmVal = ValAndVReg->Value.getSExtValue();

// Since we're going to pull this into a shift, the constant value must be
// a power of 2. If we got a multiply, then we need to check this.
if (OffsetOpc == TargetOpcode::G_MUL) {
  if (!isPowerOf2_32(ImmVal))
    return None;

  // Got a power of 2. So, the amount we'll shift is the log base-2 of that.
  ImmVal = Log2_32(ImmVal);
}

if ((ImmVal & 0x7) != ImmVal)
  return None;

// We are only allowed to shift by LegalShiftVal. This shift value is built
// into the instruction, so we can't just use whatever we want.
if (ImmVal != LegalShiftVal)
  return None;

unsigned SignExtend = 0;
if (WantsExt) {
  // Check if the offset is defined by an extend, unless we looked through a
  // G_ZEXT earlier.
  if (!LookedThroughZExt) {
    MachineInstr *ExtInst = getDefIgnoringCopies(OffsetReg, MRI);
    auto Ext = getExtendTypeForInst(*ExtInst, MRI, true);
    if (Ext == AArch64_AM::InvalidShiftExtend)
      return None;

    SignExtend = isSignExtendShiftType(Ext) ? 1 : 0;
    // We only support SXTW for signed extension here.
    if (SignExtend && Ext != AArch64_AM::SXTW)
      return None;
    OffsetReg = ExtInst->getOperand(1).getReg();
  }

  // Need a 32-bit wide register here.
  MachineIRBuilder MIB(*MRI.getVRegDef(Root.getReg()));
  OffsetReg = moveScalarRegClass(OffsetReg, AArch64::GPR32RegClass, MIB);
}

// We can use the LHS of the GEP as the base, and the LHS of the shift as an
// offset. Signify that we are shifting by setting the shift flag to 1.
return {{[=](MachineInstrBuilder &MIB) { MIB.addUse(Base.getReg()); },
         [=](MachineInstrBuilder &MIB) { MIB.addUse(OffsetReg); },
         [=](MachineInstrBuilder &MIB) {
           // Need to add both immediates here to make sure that they are both
           // added to the instruction.
           MIB.addImm(SignExtend);
           MIB.addImm(1);
         }}};
5299}

5301/// This is used for computing addresses like this:
5302///
5303/// ldr x1, [x2, x3, lsl #3]
5304///
5305/// Where x2 is the base register, and x3 is an offset register. The shift-left
5306/// is a constant value specific to this load instruction. That is, we'll never
5307/// see anything other than a 3 here (which corresponds to the size of the
5308/// element being loaded.)
5309InstructionSelector::ComplexRendererFns
5310AArch64InstructionSelector::selectAddrModeShiftedExtendXReg(
  MachineOperand &Root, unsigned SizeInBytes) const {
if (!Root.isReg())
  return None;
MachineRegisterInfo &MRI = Root.getParent()->getMF()->getRegInfo();

// We want to find something like this:
//
// val = G_CONSTANT LegalShiftVal
// shift = G_SHL off_reg val
// ptr = G_PTR_ADD base_reg shift
// x = G_LOAD ptr
//
// And fold it into this addressing mode:
//
// ldr x, [base_reg, off_reg, lsl #LegalShiftVal]

// Check if we can find the G_PTR_ADD.
MachineInstr *PtrAdd =
    getOpcodeDef(TargetOpcode::G_PTR_ADD, Root.getReg(), MRI);
if (!PtrAdd || !isWorthFoldingIntoExtendedReg(*PtrAdd, MRI))
  return None;

// Now, try to match an opcode which will match our specific offset.
// We want a G_SHL or a G_MUL.
MachineInstr *OffsetInst =
    getDefIgnoringCopies(PtrAdd->getOperand(2).getReg(), MRI);
return selectExtendedSHL(Root, PtrAdd->getOperand(1),
                         OffsetInst->getOperand(0), SizeInBytes,
                         /*WantsExt=*/false);
5340}

5342/// This is used for computing addresses like this:
5343///
5344/// ldr x1, [x2, x3]
5345///
5346/// Where x2 is the base register, and x3 is an offset register.
5347///
5348/// When possible (or profitable) to fold a G_PTR_ADD into the address calculation,
5349/// this will do so. Otherwise, it will return None.
5350InstructionSelector::ComplexRendererFns
5351AArch64InstructionSelector::selectAddrModeRegisterOffset(
  MachineOperand &Root) const {
MachineRegisterInfo &MRI = Root.getParent()->getMF()->getRegInfo();

// We need a GEP.
MachineInstr *Gep = MRI.getVRegDef(Root.getReg());
if (!Gep || Gep->getOpcode() != TargetOpcode::G_PTR_ADD)
  return None;

// If this is used more than once, let's not bother folding.
// TODO: Check if they are memory ops. If they are, then we can still fold
// without having to recompute anything.
if (!MRI.hasOneNonDBGUse(Gep->getOperand(0).getReg()))
  return None;

// Base is the GEP's LHS, offset is its RHS.
return {{[=](MachineInstrBuilder &MIB) {
           MIB.addUse(Gep->getOperand(1).getReg());
         },
         [=](MachineInstrBuilder &MIB) {
           MIB.addUse(Gep->getOperand(2).getReg());
         },
         [=](MachineInstrBuilder &MIB) {
           // Need to add both immediates here to make sure that they are both
           // added to the instruction.
           MIB.addImm(0);
           MIB.addImm(0);
         }}};
5379}

5381/// This is intended to be equivalent to selectAddrModeXRO in
5382/// AArch64ISelDAGtoDAG. It's used for selecting X register offset loads.
5383InstructionSelector::ComplexRendererFns
5384AArch64InstructionSelector::selectAddrModeXRO(MachineOperand &Root,
                                            unsigned SizeInBytes) const {
MachineRegisterInfo &MRI = Root.getParent()->getMF()->getRegInfo();
if (!Root.isReg())
  return None;
MachineInstr *PtrAdd =
    getOpcodeDef(TargetOpcode::G_PTR_ADD, Root.getReg(), MRI);
if (!PtrAdd)
  return None;

// Check for an immediates which cannot be encoded in the [base + imm]
// addressing mode, and can't be encoded in an add/sub. If this happens, we'll
// end up with code like:
//
// mov x0, wide
// add x1 base, x0
// ldr x2, [x1, x0]
//
// In this situation, we can use the [base, xreg] addressing mode to save an
// add/sub:
//
// mov x0, wide
// ldr x2, [base, x0]
auto ValAndVReg =
    getConstantVRegValWithLookThrough(PtrAdd->getOperand(2).getReg(), MRI);
if (ValAndVReg) {
  unsigned Scale = Log2_32(SizeInBytes);
  int64_t ImmOff = ValAndVReg->Value.getSExtValue();

  // Skip immediates that can be selected in the load/store addresing
  // mode.
  if (ImmOff % SizeInBytes == 0 && ImmOff >= 0 &&
      ImmOff < (0x1000 << Scale))
    return None;

  // Helper lambda to decide whether or not it is preferable to emit an add.
  auto isPreferredADD = [](int64_t ImmOff) {
    // Constants in [0x0, 0xfff] can be encoded in an add.
    if ((ImmOff & 0xfffffffffffff000LL) == 0x0LL)
      return true;

    // Can it be encoded in an add lsl #12?
    if ((ImmOff & 0xffffffffff000fffLL) != 0x0LL)
      return false;

    // It can be encoded in an add lsl #12, but we may not want to. If it is
    // possible to select this as a single movz, then prefer that. A single
    // movz is faster than an add with a shift.
    return (ImmOff & 0xffffffffff00ffffLL) != 0x0LL &&
           (ImmOff & 0xffffffffffff0fffLL) != 0x0LL;
  };

  // If the immediate can be encoded in a single add/sub, then bail out.
  if (isPreferredADD(ImmOff) || isPreferredADD(-ImmOff))
    return None;
}

// Try to fold shifts into the addressing mode.
auto AddrModeFns = selectAddrModeShiftedExtendXReg(Root, SizeInBytes);
if (AddrModeFns)
  return AddrModeFns;

// If that doesn't work, see if it's possible to fold in registers from
// a GEP.
return selectAddrModeRegisterOffset(Root);
5449}

5451/// This is used for computing addresses like this:
5452///
5453/// ldr x0, [xBase, wOffset, sxtw #LegalShiftVal]
5454///
5455/// Where we have a 64-bit base register, a 32-bit offset register, and an
5456/// extend (which may or may not be signed).
5457InstructionSelector::ComplexRendererFns
5458AArch64InstructionSelector::selectAddrModeWRO(MachineOperand &Root,
                                            unsigned SizeInBytes) const {
MachineRegisterInfo &MRI = Root.getParent()->getMF()->getRegInfo();

MachineInstr *PtrAdd =
    getOpcodeDef(TargetOpcode::G_PTR_ADD, Root.getReg(), MRI);
if (!PtrAdd || !isWorthFoldingIntoExtendedReg(*PtrAdd, MRI))
  return None;

MachineOperand &LHS = PtrAdd->getOperand(1);
MachineOperand &RHS = PtrAdd->getOperand(2);
MachineInstr *OffsetInst = getDefIgnoringCopies(RHS.getReg(), MRI);

// The first case is the same as selectAddrModeXRO, except we need an extend.
// In this case, we try to find a shift and extend, and fold them into the
// addressing mode.
//
// E.g.
//
// off_reg = G_Z/S/ANYEXT ext_reg
// val = G_CONSTANT LegalShiftVal
// shift = G_SHL off_reg val
// ptr = G_PTR_ADD base_reg shift
// x = G_LOAD ptr
//
// In this case we can get a load like this:
//
// ldr x0, [base_reg, ext_reg, sxtw #LegalShiftVal]
auto ExtendedShl = selectExtendedSHL(Root, LHS, OffsetInst->getOperand(0),
                                     SizeInBytes, /*WantsExt=*/true);
if (ExtendedShl)
  return ExtendedShl;

// There was no shift. We can try and fold a G_Z/S/ANYEXT in alone though.
//
// e.g.
// ldr something, [base_reg, ext_reg, sxtw]
if (!isWorthFoldingIntoExtendedReg(*OffsetInst, MRI))
  return None;

// Check if this is an extend. We'll get an extend type if it is.
AArch64_AM::ShiftExtendType Ext =
    getExtendTypeForInst(*OffsetInst, MRI, /*IsLoadStore=*/true);
if (Ext == AArch64_AM::InvalidShiftExtend)
  return None;

// Need a 32-bit wide register.
MachineIRBuilder MIB(*PtrAdd);
Register ExtReg = moveScalarRegClass(OffsetInst->getOperand(1).getReg(),
                                     AArch64::GPR32RegClass, MIB);
unsigned SignExtend = Ext == AArch64_AM::SXTW;

// Base is LHS, offset is ExtReg.
return {{[=](MachineInstrBuilder &MIB) { MIB.addUse(LHS.getReg()); },
         [=](MachineInstrBuilder &MIB) { MIB.addUse(ExtReg); },
         [=](MachineInstrBuilder &MIB) {
           MIB.addImm(SignExtend);
           MIB.addImm(0);
         }}};
5517}

5519/// Select a "register plus unscaled signed 9-bit immediate" address.  This
5520/// should only match when there is an offset that is not valid for a scaled
5521/// immediate addressing mode.  The "Size" argument is the size in bytes of the
5522/// memory reference, which is needed here to know what is valid for a scaled
5523/// immediate.
5524InstructionSelector::ComplexRendererFns
5525AArch64InstructionSelector::selectAddrModeUnscaled(MachineOperand &Root,
                                                 unsigned Size) const {
MachineRegisterInfo &MRI =
    Root.getParent()->getParent()->getParent()->getRegInfo();

if (!Root.isReg())
  return None;

if (!isBaseWithConstantOffset(Root, MRI))
  return None;

MachineInstr *RootDef = MRI.getVRegDef(Root.getReg());
if (!RootDef)
  return None;

MachineOperand &OffImm = RootDef->getOperand(2);
if (!OffImm.isReg())
  return None;
MachineInstr *RHS = MRI.getVRegDef(OffImm.getReg());
if (!RHS || RHS->getOpcode() != TargetOpcode::G_CONSTANT)
  return None;
int64_t RHSC;
MachineOperand &RHSOp1 = RHS->getOperand(1);
if (!RHSOp1.isCImm() || RHSOp1.getCImm()->getBitWidth() > 64)
  return None;
RHSC = RHSOp1.getCImm()->getSExtValue();

// If the offset is valid as a scaled immediate, don't match here.
if ((RHSC & (Size - 1)) == 0 && RHSC >= 0 && RHSC < (0x1000 << Log2_32(Size)))
  return None;
if (RHSC >= -256 && RHSC < 256) {
  MachineOperand &Base = RootDef->getOperand(1);
  return {{
      [=](MachineInstrBuilder &MIB) { MIB.add(Base); },
      [=](MachineInstrBuilder &MIB) { MIB.addImm(RHSC); },
  }};
}
return None;
5563}

5565InstructionSelector::ComplexRendererFns
5566AArch64InstructionSelector::tryFoldAddLowIntoImm(MachineInstr &RootDef,
                                               unsigned Size,
                                               MachineRegisterInfo &MRI) const {
if (RootDef.getOpcode() != AArch64::G_ADD_LOW)
  return None;
MachineInstr &Adrp = *MRI.getVRegDef(RootDef.getOperand(1).getReg());
if (Adrp.getOpcode() != AArch64::ADRP)
  return None;

// TODO: add heuristics like isWorthFoldingADDlow() from SelectionDAG.
// TODO: Need to check GV's offset % size if doing offset folding into globals.
assert(Adrp.getOperand(1).getOffset() == 0 && "Unexpected offset in global")((Adrp.getOperand(1).getOffset() == 0 && "Unexpected offset in global"
) ? static_cast<void> (0) : __assert_fail ("Adrp.getOperand(1).getOffset() == 0 && \"Unexpected offset in global\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 5577, __PRETTY_FUNCTION__));
auto GV = Adrp.getOperand(1).getGlobal();
if (GV->isThreadLocal())
  return None;

auto &MF = *RootDef.getParent()->getParent();
if (GV->getPointerAlignment(MF.getDataLayout()) < Size)
  return None;

unsigned OpFlags = STI.ClassifyGlobalReference(GV, MF.getTarget());
MachineIRBuilder MIRBuilder(RootDef);
Register AdrpReg = Adrp.getOperand(0).getReg();
return {{[=](MachineInstrBuilder &MIB) { MIB.addUse(AdrpReg); },
         [=](MachineInstrBuilder &MIB) {
           MIB.addGlobalAddress(GV, /* Offset */ 0,
                                OpFlags | AArch64II::MO_PAGEOFF |
                                    AArch64II::MO_NC);
         }}};
5595}

5597/// Select a "register plus scaled unsigned 12-bit immediate" address.  The
5598/// "Size" argument is the size in bytes of the memory reference, which
5599/// determines the scale.
5600InstructionSelector::ComplexRendererFns
5601AArch64InstructionSelector::selectAddrModeIndexed(MachineOperand &Root,
                                                unsigned Size) const {
MachineFunction &MF = *Root.getParent()->getParent()->getParent();
MachineRegisterInfo &MRI = MF.getRegInfo();

if (!Root.isReg())
  return None;

MachineInstr *RootDef = MRI.getVRegDef(Root.getReg());
if (!RootDef)
  return None;

if (RootDef->getOpcode() == TargetOpcode::G_FRAME_INDEX) {
  return {{
      [=](MachineInstrBuilder &MIB) { MIB.add(RootDef->getOperand(1)); },
      [=](MachineInstrBuilder &MIB) { MIB.addImm(0); },
  }};
}

CodeModel::Model CM = MF.getTarget().getCodeModel();
// Check if we can fold in the ADD of small code model ADRP + ADD address.
if (CM == CodeModel::Small) {
  auto OpFns = tryFoldAddLowIntoImm(*RootDef, Size, MRI);
  if (OpFns)
    return OpFns;
}

if (isBaseWithConstantOffset(Root, MRI)) {
  MachineOperand &LHS = RootDef->getOperand(1);
  MachineOperand &RHS = RootDef->getOperand(2);
  MachineInstr *LHSDef = MRI.getVRegDef(LHS.getReg());
  MachineInstr *RHSDef = MRI.getVRegDef(RHS.getReg());
  if (LHSDef && RHSDef) {
    int64_t RHSC = (int64_t)RHSDef->getOperand(1).getCImm()->getZExtValue();
    unsigned Scale = Log2_32(Size);
    if ((RHSC & (Size - 1)) == 0 && RHSC >= 0 && RHSC < (0x1000 << Scale)) {
      if (LHSDef->getOpcode() == TargetOpcode::G_FRAME_INDEX)
        return {{
            [=](MachineInstrBuilder &MIB) { MIB.add(LHSDef->getOperand(1)); },
            [=](MachineInstrBuilder &MIB) { MIB.addImm(RHSC >> Scale); },
        }};

      return {{
          [=](MachineInstrBuilder &MIB) { MIB.add(LHS); },
          [=](MachineInstrBuilder &MIB) { MIB.addImm(RHSC >> Scale); },
      }};
    }
  }
}

// Before falling back to our general case, check if the unscaled
// instructions can handle this. If so, that's preferable.
if (selectAddrModeUnscaled(Root, Size).hasValue())
  return None;

return {{
    [=](MachineInstrBuilder &MIB) { MIB.add(Root); },
    [=](MachineInstrBuilder &MIB) { MIB.addImm(0); },
}};
5660}

5662/// Given a shift instruction, return the correct shift type for that
5663/// instruction.
5664static AArch64_AM::ShiftExtendType getShiftTypeForInst(MachineInstr &MI) {
// TODO: Handle AArch64_AM::ROR
switch (MI.getOpcode()) {
default:
  return AArch64_AM::InvalidShiftExtend;
case TargetOpcode::G_SHL:
  return AArch64_AM::LSL;
case TargetOpcode::G_LSHR:
  return AArch64_AM::LSR;
case TargetOpcode::G_ASHR:
  return AArch64_AM::ASR;
}
5676}

5678/// Select a "shifted register" operand. If the value is not shifted, set the
5679/// shift operand to a default value of "lsl 0".
5680///
5681/// TODO: Allow shifted register to be rotated in logical instructions.
5682InstructionSelector::ComplexRendererFns
5683AArch64InstructionSelector::selectShiftedRegister(MachineOperand &Root) const {
if (!Root.isReg())
  return None;
MachineRegisterInfo &MRI =
    Root.getParent()->getParent()->getParent()->getRegInfo();

// Check if the operand is defined by an instruction which corresponds to
// a ShiftExtendType. E.g. a G_SHL, G_LSHR, etc.
//
// TODO: Handle AArch64_AM::ROR for logical instructions.
MachineInstr *ShiftInst = MRI.getVRegDef(Root.getReg());
if (!ShiftInst)
  return None;
AArch64_AM::ShiftExtendType ShType = getShiftTypeForInst(*ShiftInst);
if (ShType == AArch64_AM::InvalidShiftExtend)
  return None;
if (!isWorthFoldingIntoExtendedReg(*ShiftInst, MRI))
  return None;

// Need an immediate on the RHS.
MachineOperand &ShiftRHS = ShiftInst->getOperand(2);
auto Immed = getImmedFromMO(ShiftRHS);
if (!Immed)
  return None;

// We have something that we can fold. Fold in the shift's LHS and RHS into
// the instruction.
MachineOperand &ShiftLHS = ShiftInst->getOperand(1);
Register ShiftReg = ShiftLHS.getReg();

unsigned NumBits = MRI.getType(ShiftReg).getSizeInBits();
unsigned Val = *Immed & (NumBits - 1);
unsigned ShiftVal = AArch64_AM::getShifterImm(ShType, Val);

return {{[=](MachineInstrBuilder &MIB) { MIB.addUse(ShiftReg); },
         [=](MachineInstrBuilder &MIB) { MIB.addImm(ShiftVal); }}};
5719}

5721AArch64_AM::ShiftExtendType AArch64InstructionSelector::getExtendTypeForInst(
  MachineInstr &MI, MachineRegisterInfo &MRI, bool IsLoadStore) const {
unsigned Opc = MI.getOpcode();

// Handle explicit extend instructions first.
if (Opc == TargetOpcode::G_SEXT || Opc == TargetOpcode::G_SEXT_INREG) {
  unsigned Size;
  if (Opc == TargetOpcode::G_SEXT)
    Size = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
  else
    Size = MI.getOperand(2).getImm();
  assert(Size != 64 && "Extend from 64 bits?")((Size != 64 && "Extend from 64 bits?") ? static_cast
<void> (0) : __assert_fail ("Size != 64 && \"Extend from 64 bits?\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 5732, __PRETTY_FUNCTION__));
  switch (Size) {
  case 8:
    return AArch64_AM::SXTB;
  case 16:
    return AArch64_AM::SXTH;
  case 32:
    return AArch64_AM::SXTW;
  default:
    return AArch64_AM::InvalidShiftExtend;
  }
}

if (Opc == TargetOpcode::G_ZEXT || Opc == TargetOpcode::G_ANYEXT) {
  unsigned Size = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
  assert(Size != 64 && "Extend from 64 bits?")((Size != 64 && "Extend from 64 bits?") ? static_cast
<void> (0) : __assert_fail ("Size != 64 && \"Extend from 64 bits?\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 5747, __PRETTY_FUNCTION__));
  switch (Size) {
  case 8:
    return AArch64_AM::UXTB;
  case 16:
    return AArch64_AM::UXTH;
  case 32:
    return AArch64_AM::UXTW;
  default:
    return AArch64_AM::InvalidShiftExtend;
  }
}

// Don't have an explicit extend. Try to handle a G_AND with a constant mask
// on the RHS.
if (Opc != TargetOpcode::G_AND)
  return AArch64_AM::InvalidShiftExtend;

Optional<uint64_t> MaybeAndMask = getImmedFromMO(MI.getOperand(2));
if (!MaybeAndMask)
  return AArch64_AM::InvalidShiftExtend;
uint64_t AndMask = *MaybeAndMask;
switch (AndMask) {
default:
  return AArch64_AM::InvalidShiftExtend;
case 0xFF:
  return !IsLoadStore ? AArch64_AM::UXTB : AArch64_AM::InvalidShiftExtend;
case 0xFFFF:
  return !IsLoadStore ? AArch64_AM::UXTH : AArch64_AM::InvalidShiftExtend;
case 0xFFFFFFFF:
  return AArch64_AM::UXTW;
}
5779}

5781Register AArch64InstructionSelector::moveScalarRegClass(
  Register Reg, const TargetRegisterClass &RC, MachineIRBuilder &MIB) const {
MachineRegisterInfo &MRI = *MIB.getMRI();
auto Ty = MRI.getType(Reg);
assert(!Ty.isVector() && "Expected scalars only!")((!Ty.isVector() && "Expected scalars only!") ? static_cast
<void> (0) : __assert_fail ("!Ty.isVector() && \"Expected scalars only!\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 5785, __PRETTY_FUNCTION__));
20
←
'?' condition is true→
if (Ty.getSizeInBits() == TRI.getRegSizeInBits(RC))
21
←
Assuming the condition is false→
22
←
Taking false branch→
  return Reg;

// Create a copy and immediately select it.
// FIXME: We should have an emitCopy function?
auto Copy = MIB.buildCopy({&RC}, {Reg});
selectCopy(*Copy, TII, MRI, TRI, RBI);
23
←
Calling 'selectCopy'→
return Copy.getReg(0);
5794}

5796/// Select an "extended register" operand. This operand folds in an extend
5797/// followed by an optional left shift.
5798InstructionSelector::ComplexRendererFns
5799AArch64InstructionSelector::selectArithExtendedRegister(
  MachineOperand &Root) const {
if (!Root.isReg())
10
←
Taking false branch→
  return None;
MachineRegisterInfo &MRI =
    Root.getParent()->getParent()->getParent()->getRegInfo();

uint64_t ShiftVal = 0;
Register ExtReg;
AArch64_AM::ShiftExtendType Ext;
MachineInstr *RootDef = getDefIgnoringCopies(Root.getReg(), MRI);
if (!RootDef)
11
←
Assuming 'RootDef' is non-null→
12
←
Taking false branch→
  return None;

if (!isWorthFoldingIntoExtendedReg(*RootDef, MRI))
13
←
Assuming the condition is false→
14
←
Taking false branch→
  return None;

// Check if we can fold a shift and an extend.
if (RootDef->getOpcode() == TargetOpcode::G_SHL) {
15
←
Assuming the condition is false→
16
←
Taking false branch→
  // Look for a constant on the RHS of the shift.
  MachineOperand &RHS = RootDef->getOperand(2);
  Optional<uint64_t> MaybeShiftVal = getImmedFromMO(RHS);
  if (!MaybeShiftVal)
    return None;
  ShiftVal = *MaybeShiftVal;
  if (ShiftVal > 4)
    return None;
  // Look for a valid extend instruction on the LHS of the shift.
  MachineOperand &LHS = RootDef->getOperand(1);
  MachineInstr *ExtDef = getDefIgnoringCopies(LHS.getReg(), MRI);
  if (!ExtDef)
    return None;
  Ext = getExtendTypeForInst(*ExtDef, MRI);
  if (Ext == AArch64_AM::InvalidShiftExtend)
    return None;
  ExtReg = ExtDef->getOperand(1).getReg();
} else {
  // Didn't get a shift. Try just folding an extend.
  Ext = getExtendTypeForInst(*RootDef, MRI);
  if (Ext16.1
'Ext' is not equal to InvalidShiftExtend
1
'Ext' is not equal to InvalidShiftExtend
1
'Ext' is not equal to InvalidShiftExtend
 == AArch64_AM::InvalidShiftExtend)
17
←
Taking false branch→
    return None;
  ExtReg = RootDef->getOperand(1).getReg();

  // If we have a 32 bit instruction which zeroes out the high half of a
  // register, we get an implicit zero extend for free. Check if we have one.
  // FIXME: We actually emit the extend right now even though we don't have
  // to.
  if (Ext17.1
'Ext' is not equal to UXTW
1
'Ext' is not equal to UXTW
1
'Ext' is not equal to UXTW
 == AArch64_AM::UXTW && MRI.getType(ExtReg).getSizeInBits() == 32) {
18
←
Taking false branch→
    MachineInstr *ExtInst = MRI.getVRegDef(ExtReg);
    if (ExtInst && isDef32(*ExtInst))
      return None;
  }
}

// We require a GPR32 here. Narrow the ExtReg if needed using a subregister
// copy.
MachineIRBuilder MIB(*RootDef);
ExtReg = moveScalarRegClass(ExtReg, AArch64::GPR32RegClass, MIB);
19
←
Calling 'AArch64InstructionSelector::moveScalarRegClass'→

return {{[=](MachineInstrBuilder &MIB) { MIB.addUse(ExtReg); },
         [=](MachineInstrBuilder &MIB) {
           MIB.addImm(getArithExtendImm(Ext, ShiftVal));
         }}};
5862}

5864void AArch64InstructionSelector::renderTruncImm(MachineInstrBuilder &MIB,
                                              const MachineInstr &MI,
                                              int OpIdx) const {
const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&((MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx
 == -1 && "Expected G_CONSTANT") ? static_cast<void
> (0) : __assert_fail ("MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 && \"Expected G_CONSTANT\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 5869, __PRETTY_FUNCTION__))
       "Expected G_CONSTANT")((MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx
 == -1 && "Expected G_CONSTANT") ? static_cast<void
> (0) : __assert_fail ("MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 && \"Expected G_CONSTANT\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 5869, __PRETTY_FUNCTION__));
Optional<int64_t> CstVal =
    getConstantVRegSExtVal(MI.getOperand(0).getReg(), MRI);
assert(CstVal && "Expected constant value")((CstVal && "Expected constant value") ? static_cast<
void> (0) : __assert_fail ("CstVal && \"Expected constant value\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 5872, __PRETTY_FUNCTION__));
MIB.addImm(CstVal.getValue());
5874}

5876void AArch64InstructionSelector::renderLogicalImm32(
MachineInstrBuilder &MIB, const MachineInstr &I, int OpIdx) const {
assert(I.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&((I.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx ==
 -1 && "Expected G_CONSTANT") ? static_cast<void>
 (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 && \"Expected G_CONSTANT\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 5879, __PRETTY_FUNCTION__))
       "Expected G_CONSTANT")((I.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx ==
 -1 && "Expected G_CONSTANT") ? static_cast<void>
 (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 && \"Expected G_CONSTANT\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 5879, __PRETTY_FUNCTION__));
uint64_t CstVal = I.getOperand(1).getCImm()->getZExtValue();
uint64_t Enc = AArch64_AM::encodeLogicalImmediate(CstVal, 32);
MIB.addImm(Enc);
5883}

5885void AArch64InstructionSelector::renderLogicalImm64(
MachineInstrBuilder &MIB, const MachineInstr &I, int OpIdx) const {
assert(I.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&((I.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx ==
 -1 && "Expected G_CONSTANT") ? static_cast<void>
 (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 && \"Expected G_CONSTANT\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 5888, __PRETTY_FUNCTION__))
       "Expected G_CONSTANT")((I.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx ==
 -1 && "Expected G_CONSTANT") ? static_cast<void>
 (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 && \"Expected G_CONSTANT\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 5888, __PRETTY_FUNCTION__));
uint64_t CstVal = I.getOperand(1).getCImm()->getZExtValue();
uint64_t Enc = AArch64_AM::encodeLogicalImmediate(CstVal, 64);
MIB.addImm(Enc);
5892}

5894bool AArch64InstructionSelector::isLoadStoreOfNumBytes(
  const MachineInstr &MI, unsigned NumBytes) const {
if (!MI.mayLoadOrStore())
  return false;
assert(MI.hasOneMemOperand() &&((MI.hasOneMemOperand() && "Expected load/store to have only one mem op!"
) ? static_cast<void> (0) : __assert_fail ("MI.hasOneMemOperand() && \"Expected load/store to have only one mem op!\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 5899, __PRETTY_FUNCTION__))
       "Expected load/store to have only one mem op!")((MI.hasOneMemOperand() && "Expected load/store to have only one mem op!"
) ? static_cast<void> (0) : __assert_fail ("MI.hasOneMemOperand() && \"Expected load/store to have only one mem op!\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 5899, __PRETTY_FUNCTION__));
return (*MI.memoperands_begin())->getSize() == NumBytes;
5901}

5903bool AArch64InstructionSelector::isDef32(const MachineInstr &MI) const {
const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
if (MRI.getType(MI.getOperand(0).getReg()).getSizeInBits() != 32)
  return false;

// Only return true if we know the operation will zero-out the high half of
// the 64-bit register. Truncates can be subregister copies, which don't
// zero out the high bits. Copies and other copy-like instructions can be
// fed by truncates, or could be lowered as subregister copies.
switch (MI.getOpcode()) {
default:
  return true;
case TargetOpcode::COPY:
case TargetOpcode::G_BITCAST:
case TargetOpcode::G_TRUNC:
case TargetOpcode::G_PHI:
  return false;
}
5921}


5924// Perform fixups on the given PHI instruction's operands to force them all
5925// to be the same as the destination regbank.
5926static void fixupPHIOpBanks(MachineInstr &MI, MachineRegisterInfo &MRI,
                          const AArch64RegisterBankInfo &RBI) {
assert(MI.getOpcode() == TargetOpcode::G_PHI && "Expected a G_PHI")((MI.getOpcode() == TargetOpcode::G_PHI && "Expected a G_PHI"
) ? static_cast<void> (0) : __assert_fail ("MI.getOpcode() == TargetOpcode::G_PHI && \"Expected a G_PHI\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 5928, __PRETTY_FUNCTION__));
Register DstReg = MI.getOperand(0).getReg();
const RegisterBank *DstRB = MRI.getRegBankOrNull(DstReg);
assert(DstRB && "Expected PHI dst to have regbank assigned")((DstRB && "Expected PHI dst to have regbank assigned"
) ? static_cast<void> (0) : __assert_fail ("DstRB && \"Expected PHI dst to have regbank assigned\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 5931, __PRETTY_FUNCTION__));
MachineIRBuilder MIB(MI);

// Go through each operand and ensure it has the same regbank.
for (unsigned OpIdx = 1; OpIdx < MI.getNumOperands(); ++OpIdx) {
  MachineOperand &MO = MI.getOperand(OpIdx);
  if (!MO.isReg())
    continue;
  Register OpReg = MO.getReg();
  const RegisterBank *RB = MRI.getRegBankOrNull(OpReg);
  if (RB != DstRB) {
    // Insert a cross-bank copy.
    auto *OpDef = MRI.getVRegDef(OpReg);
    const LLT &Ty = MRI.getType(OpReg);
    MIB.setInsertPt(*OpDef->getParent(), std::next(OpDef->getIterator()));
    auto Copy = MIB.buildCopy(Ty, OpReg);
    MRI.setRegBank(Copy.getReg(0), *DstRB);
    MO.setReg(Copy.getReg(0));
  }
}
5951}

5953void AArch64InstructionSelector::processPHIs(MachineFunction &MF) {
// We're looking for PHIs, build a list so we don't invalidate iterators.
MachineRegisterInfo &MRI = MF.getRegInfo();
SmallVector<MachineInstr *, 32> Phis;
for (auto &BB : MF) {
  for (auto &MI : BB) {
    if (MI.getOpcode() == TargetOpcode::G_PHI)
      Phis.emplace_back(&MI);
  }
}

for (auto *MI : Phis) {
  // We need to do some work here if the operand types are < 16 bit and they
  // are split across fpr/gpr banks. Since all types <32b on gpr
  // end up being assigned gpr32 regclasses, we can end up with PHIs here
  // which try to select between a gpr32 and an fpr16. Ideally RBS shouldn't
  // be selecting heterogenous regbanks for operands if possible, but we
  // still need to be able to deal with it here.
  //
  // To fix this, if we have a gpr-bank operand < 32b in size and at least
  // one other operand is on the fpr bank, then we add cross-bank copies
  // to homogenize the operand banks. For simplicity the bank that we choose
  // to settle on is whatever bank the def operand has. For example:
  //
  // %endbb:
  //   %dst:gpr(s16) = G_PHI %in1:gpr(s16), %bb1, %in2:fpr(s16), %bb2
  //  =>
  // %bb2:
  //   ...
  //   %in2_copy:gpr(s16) = COPY %in2:fpr(s16)
  //   ...
  // %endbb:
  //   %dst:gpr(s16) = G_PHI %in1:gpr(s16), %bb1, %in2_copy:gpr(s16), %bb2
  bool HasGPROp = false, HasFPROp = false;
  for (unsigned OpIdx = 1; OpIdx < MI->getNumOperands(); ++OpIdx) {
    const auto &MO = MI->getOperand(OpIdx);
    if (!MO.isReg())
      continue;
    const LLT &Ty = MRI.getType(MO.getReg());
    if (!Ty.isValid() || !Ty.isScalar())
      break;
    if (Ty.getSizeInBits() >= 32)
      break;
    const RegisterBank *RB = MRI.getRegBankOrNull(MO.getReg());
    // If for some reason we don't have a regbank yet. Don't try anything.
    if (!RB)
      break;

    if (RB->getID() == AArch64::GPRRegBankID)
      HasGPROp = true;
    else
      HasFPROp = true;
  }
  // We have heterogenous regbanks, need to fixup.
  if (HasGPROp && HasFPROp)
    fixupPHIOpBanks(*MI, MRI, RBI);
}
6010}

6012namespace llvm {
6013InstructionSelector *
6014createAArch64InstructionSelector(const AArch64TargetMachine &TM,
                               AArch64Subtarget &Subtarget,
                               AArch64RegisterBankInfo &RBI) {
return new AArch64InstructionSelector(TM, Subtarget, RBI);
6018}
6019}

←

/usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0/tuple

→

1// <tuple> -*- C++ -*-

3// Copyright (C) 2007-2016 Free Software Foundation, Inc.
4//
5// This file is part of the GNU ISO C++ Library.  This library is free
6// software; you can redistribute it and/or modify it under the
7// terms of the GNU General Public License as published by the
8// Free Software Foundation; either version 3, or (at your option)
9// any later version.

11// This library is distributed in the hope that it will be useful,
12// but WITHOUT ANY WARRANTY; without even the implied warranty of
13// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
14// GNU General Public License for more details.

16// Under Section 7 of GPL version 3, you are granted additional
17// permissions described in the GCC Runtime Library Exception, version
18// 3.1, as published by the Free Software Foundation.

20// You should have received a copy of the GNU General Public License and
21// a copy of the GCC Runtime Library Exception along with this program;
22// see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
23// <http://www.gnu.org/licenses/>.

25/** @file include/tuple
*  This is a Standard C++ Library header.
*/

29#ifndef _GLIBCXX_TUPLE1
30#define _GLIBCXX_TUPLE1 1

32#pragma GCC system_header

34#if __cplusplus201402L < 201103L
35# include <bits/c++0x_warning.h>
36#else

38#include <utility>
39#include <array>
40#include <bits/uses_allocator.h>

42namespace std _GLIBCXX_VISIBILITY(default)__attribute__ ((__visibility__ ("default")))
43{
44_GLIBCXX_BEGIN_NAMESPACE_VERSION

/**
 *  @addtogroup utilities
 *  @{
 */

template<std::size_t _Idx, typename _Head, bool _IsEmptyNotFinal>
  struct _Head_base;

template<std::size_t _Idx, typename _Head>
  struct _Head_base<_Idx, _Head, true>
  : public _Head
  {
    constexpr _Head_base()
    : _Head() { }

    constexpr _Head_base(const _Head& __h)
    : _Head(__h) { }

    constexpr _Head_base(const _Head_base&) = default;
    constexpr _Head_base(_Head_base&&) = default;

    template<typename _UHead>
      constexpr _Head_base(_UHead&& __h)
: _Head(std::forward<_UHead>(__h)) { }

    _Head_base(allocator_arg_t, __uses_alloc0)
    : _Head() { }

    template<typename _Alloc>
_Head_base(allocator_arg_t, __uses_alloc1<_Alloc> __a)
: _Head(allocator_arg, *__a._M_a) { }

    template<typename _Alloc>
_Head_base(allocator_arg_t, __uses_alloc2<_Alloc> __a)
: _Head(*__a._M_a) { }

    template<typename _UHead>
_Head_base(__uses_alloc0, _UHead&& __uhead)
: _Head(std::forward<_UHead>(__uhead)) { }

    template<typename _Alloc, typename _UHead>
_Head_base(__uses_alloc1<_Alloc> __a, _UHead&& __uhead)
: _Head(allocator_arg, *__a._M_a, std::forward<_UHead>(__uhead)) { }

    template<typename _Alloc, typename _UHead>
_Head_base(__uses_alloc2<_Alloc> __a, _UHead&& __uhead)
: _Head(std::forward<_UHead>(__uhead), *__a._M_a) { }

    static constexpr _Head&
    _M_head(_Head_base& __b) noexcept { return __b; }

    static constexpr const _Head&
    _M_head(const _Head_base& __b) noexcept { return __b; }
  };

template<std::size_t _Idx, typename _Head>
  struct _Head_base<_Idx, _Head, false>
  {
    constexpr _Head_base()
    : _M_head_impl() { }

    constexpr _Head_base(const _Head& __h)
    : _M_head_impl(__h) { }

    constexpr _Head_base(const _Head_base&) = default;
    constexpr _Head_base(_Head_base&&) = default;

    template<typename _UHead>
      constexpr _Head_base(_UHead&& __h)
: _M_head_impl(std::forward<_UHead>(__h)) { }

    _Head_base(allocator_arg_t, __uses_alloc0)
    : _M_head_impl() { }

    template<typename _Alloc>
_Head_base(allocator_arg_t, __uses_alloc1<_Alloc> __a)
: _M_head_impl(allocator_arg, *__a._M_a) { }

    template<typename _Alloc>
_Head_base(allocator_arg_t, __uses_alloc2<_Alloc> __a)
: _M_head_impl(*__a._M_a) { }

    template<typename _UHead>
_Head_base(__uses_alloc0, _UHead&& __uhead)
: _M_head_impl(std::forward<_UHead>(__uhead)) { }

    template<typename _Alloc, typename _UHead>
_Head_base(__uses_alloc1<_Alloc> __a, _UHead&& __uhead)
: _M_head_impl(allocator_arg, *__a._M_a, std::forward<_UHead>(__uhead))
{ }

    template<typename _Alloc, typename _UHead>
_Head_base(__uses_alloc2<_Alloc> __a, _UHead&& __uhead)
: _M_head_impl(std::forward<_UHead>(__uhead), *__a._M_a) { }

    static constexpr _Head&
    _M_head(_Head_base& __b) noexcept { return __b._M_head_impl; }

    static constexpr const _Head&
    _M_head(const _Head_base& __b) noexcept { return __b._M_head_impl; }

    _Head _M_head_impl;
  };

/**
 * Contains the actual implementation of the @c tuple template, stored
 * as a recursive inheritance hierarchy from the first element (most
 * derived class) to the last (least derived class). The @c Idx
 * parameter gives the 0-based index of the element stored at this
 * point in the hierarchy; we use it to implement a constant-time
 * get() operation.
 */
template<std::size_t _Idx, typename... _Elements>
  struct _Tuple_impl; 

template<typename _Tp>
  struct __is_empty_non_tuple : is_empty<_Tp> { };

// Using EBO for elements that are tuples causes ambiguous base errors.
template<typename _El0, typename... _El>
  struct __is_empty_non_tuple<tuple<_El0, _El...>> : false_type { };

// Use the Empty Base-class Optimization for empty, non-final types.
template<typename _Tp>
  using __empty_not_final
  = typename conditional<__is_final(_Tp), false_type,
	   __is_empty_non_tuple<_Tp>>::type;

/**
 * Recursive tuple implementation. Here we store the @c Head element
 * and derive from a @c Tuple_impl containing the remaining elements
 * (which contains the @c Tail).
 */
template<std::size_t _Idx, typename _Head, typename... _Tail>
  struct _Tuple_impl<_Idx, _Head, _Tail...>
  : public _Tuple_impl<_Idx + 1, _Tail...>,
    private _Head_base<_Idx, _Head, __empty_not_final<_Head>::value>
  {
    template<std::size_t, typename...> friend class _Tuple_impl;

    typedef _Tuple_impl<_Idx + 1, _Tail...> _Inherited;
    typedef _Head_base<_Idx, _Head, __empty_not_final<_Head>::value> _Base;

    static constexpr _Head&  
    _M_head(_Tuple_impl& __t) noexcept { return _Base::_M_head(__t); }

    static constexpr const _Head&
    _M_head(const _Tuple_impl& __t) noexcept { return _Base::_M_head(__t); }

    static constexpr _Inherited&
    _M_tail(_Tuple_impl& __t) noexcept { return __t; }

    static constexpr const _Inherited&
    _M_tail(const _Tuple_impl& __t) noexcept { return __t; }

    constexpr _Tuple_impl()
    : _Inherited(), _Base() { }

    explicit 
    constexpr _Tuple_impl(const _Head& __head, const _Tail&... __tail)
    : _Inherited(__tail...), _Base(__head) { }
27
←
Calling constructor for '_Tuple_impl<1, const llvm::TargetRegisterClass *&>'→
30
←
Returning from constructor for '_Tuple_impl<1, const llvm::TargetRegisterClass *&>'→
31
←
Calling constructor for '_Head_base<0, const llvm::TargetRegisterClass *&, false>'→
32
←
Returning from constructor for '_Head_base<0, const llvm::TargetRegisterClass *&, false>'→

    template<typename _UHead, typename... _UTail, typename = typename
             enable_if<sizeof...(_Tail) == sizeof...(_UTail)>::type> 
      explicit
      constexpr _Tuple_impl(_UHead&& __head, _UTail&&... __tail)
: _Inherited(std::forward<_UTail>(__tail)...),
 _Base(std::forward<_UHead>(__head)) { }

    constexpr _Tuple_impl(const _Tuple_impl&) = default;

    constexpr
    _Tuple_impl(_Tuple_impl&& __in)
    noexcept(__and_<is_nothrow_move_constructible<_Head>,
             is_nothrow_move_constructible<_Inherited>>::value)
    : _Inherited(std::move(_M_tail(__in))), 
_Base(std::forward<_Head>(_M_head(__in))) { }

    template<typename... _UElements>
      constexpr _Tuple_impl(const _Tuple_impl<_Idx, _UElements...>& __in)
: _Inherited(_Tuple_impl<_Idx, _UElements...>::_M_tail(__in)),
 _Base(_Tuple_impl<_Idx, _UElements...>::_M_head(__in)) { }

    template<typename _UHead, typename... _UTails>
      constexpr _Tuple_impl(_Tuple_impl<_Idx, _UHead, _UTails...>&& __in)
: _Inherited(std::move
     (_Tuple_impl<_Idx, _UHead, _UTails...>::_M_tail(__in))),
 _Base(std::forward<_UHead>
(_Tuple_impl<_Idx, _UHead, _UTails...>::_M_head(__in))) { }

    template<typename _Alloc>
_Tuple_impl(allocator_arg_t __tag, const _Alloc& __a)
: _Inherited(__tag, __a),
        _Base(__tag, __use_alloc<_Head>(__a)) { }

    template<typename _Alloc>
_Tuple_impl(allocator_arg_t __tag, const _Alloc& __a,
    const _Head& __head, const _Tail&... __tail)
: _Inherited(__tag, __a, __tail...),
        _Base(__use_alloc<_Head, _Alloc, _Head>(__a), __head) { }

    template<typename _Alloc, typename _UHead, typename... _UTail,
             typename = typename enable_if<sizeof...(_Tail)
			     == sizeof...(_UTail)>::type>
_Tuple_impl(allocator_arg_t __tag, const _Alloc& __a,
           _UHead&& __head, _UTail&&... __tail)
: _Inherited(__tag, __a, std::forward<_UTail>(__tail)...),
        _Base(__use_alloc<_Head, _Alloc, _UHead>(__a),
       std::forward<_UHead>(__head)) { }

    template<typename _Alloc>
      _Tuple_impl(allocator_arg_t __tag, const _Alloc& __a,
           const _Tuple_impl& __in)
: _Inherited(__tag, __a, _M_tail(__in)), 
        _Base(__use_alloc<_Head, _Alloc, _Head>(__a), _M_head(__in)) { }

    template<typename _Alloc>
_Tuple_impl(allocator_arg_t __tag, const _Alloc& __a,
           _Tuple_impl&& __in)
: _Inherited(__tag, __a, std::move(_M_tail(__in))), 
 _Base(__use_alloc<_Head, _Alloc, _Head>(__a),
       std::forward<_Head>(_M_head(__in))) { }

    template<typename _Alloc, typename... _UElements>
_Tuple_impl(allocator_arg_t __tag, const _Alloc& __a,
           const _Tuple_impl<_Idx, _UElements...>& __in)
: _Inherited(__tag, __a,
     _Tuple_impl<_Idx, _UElements...>::_M_tail(__in)),
 _Base(__use_alloc<_Head, _Alloc, _Head>(__a),
_Tuple_impl<_Idx, _UElements...>::_M_head(__in)) { }

    template<typename _Alloc, typename _UHead, typename... _UTails>
_Tuple_impl(allocator_arg_t __tag, const _Alloc& __a,
           _Tuple_impl<_Idx, _UHead, _UTails...>&& __in)
: _Inherited(__tag, __a, std::move
     (_Tuple_impl<_Idx, _UHead, _UTails...>::_M_tail(__in))),
 _Base(__use_alloc<_Head, _Alloc, _UHead>(__a),
              std::forward<_UHead>
(_Tuple_impl<_Idx, _UHead, _UTails...>::_M_head(__in))) { }

    _Tuple_impl&
    operator=(const _Tuple_impl& __in)
    {
_M_head(*this) = _M_head(__in);
_M_tail(*this) = _M_tail(__in);
return *this;
    }

    _Tuple_impl&
    operator=(_Tuple_impl&& __in)
    noexcept(__and_<is_nothrow_move_assignable<_Head>,
             is_nothrow_move_assignable<_Inherited>>::value)
    {
_M_head(*this) = std::forward<_Head>(_M_head(__in));
_M_tail(*this) = std::move(_M_tail(__in));
return *this;
    }

    template<typename... _UElements>
      _Tuple_impl&
      operator=(const _Tuple_impl<_Idx, _UElements...>& __in)
      {
 _M_head(*this) = _Tuple_impl<_Idx, _UElements...>::_M_head(__in);
 _M_tail(*this) = _Tuple_impl<_Idx, _UElements...>::_M_tail(__in);
 return *this;
}

    template<typename _UHead, typename... _UTails>
      _Tuple_impl&
      operator=(_Tuple_impl<_Idx, _UHead, _UTails...>&& __in)
      {
 _M_head(*this) = std::forward<_UHead>
   (_Tuple_impl<_Idx, _UHead, _UTails...>::_M_head(__in));
 _M_tail(*this) = std::move
   (_Tuple_impl<_Idx, _UHead, _UTails...>::_M_tail(__in));
 return *this;
}

  protected:
    void
    _M_swap(_Tuple_impl& __in)
    noexcept(__is_nothrow_swappable<_Head>::value
             && noexcept(_M_tail(__in)._M_swap(_M_tail(__in))))
    {
using std::swap;
swap(_M_head(*this), _M_head(__in));
_Inherited::_M_swap(_M_tail(__in));
    }
  };

// Basis case of inheritance recursion.
template<std::size_t _Idx, typename _Head>
  struct _Tuple_impl<_Idx, _Head>
  : private _Head_base<_Idx, _Head, __empty_not_final<_Head>::value>
  {
    template<std::size_t, typename...> friend class _Tuple_impl;

    typedef _Head_base<_Idx, _Head, __empty_not_final<_Head>::value> _Base;

    static constexpr _Head&
    _M_head(_Tuple_impl& __t) noexcept { return _Base::_M_head(__t); }

    static constexpr const _Head&
    _M_head(const _Tuple_impl& __t) noexcept { return _Base::_M_head(__t); }

    constexpr _Tuple_impl()
    : _Base() { }

    explicit
    constexpr _Tuple_impl(const _Head& __head)
    : _Base(__head) { }
28
←
Calling constructor for '_Head_base<1, const llvm::TargetRegisterClass *&, false>'→
29
←
Returning from constructor for '_Head_base<1, const llvm::TargetRegisterClass *&, false>'→

    template<typename _UHead>
      explicit
      constexpr _Tuple_impl(_UHead&& __head)
: _Base(std::forward<_UHead>(__head)) { }

    constexpr _Tuple_impl(const _Tuple_impl&) = default;

    constexpr
    _Tuple_impl(_Tuple_impl&& __in)
    noexcept(is_nothrow_move_constructible<_Head>::value)
    : _Base(std::forward<_Head>(_M_head(__in))) { }

    template<typename _UHead>
      constexpr _Tuple_impl(const _Tuple_impl<_Idx, _UHead>& __in)
: _Base(_Tuple_impl<_Idx, _UHead>::_M_head(__in)) { }

    template<typename _UHead>
      constexpr _Tuple_impl(_Tuple_impl<_Idx, _UHead>&& __in)
: _Base(std::forward<_UHead>(_Tuple_impl<_Idx, _UHead>::_M_head(__in)))
{ }

    template<typename _Alloc>
_Tuple_impl(allocator_arg_t __tag, const _Alloc& __a)
: _Base(__tag, __use_alloc<_Head>(__a)) { }

    template<typename _Alloc>
_Tuple_impl(allocator_arg_t __tag, const _Alloc& __a,
    const _Head& __head)
: _Base(__use_alloc<_Head, _Alloc, _Head>(__a), __head) { }

    template<typename _Alloc, typename _UHead>
_Tuple_impl(allocator_arg_t __tag, const _Alloc& __a,
           _UHead&& __head)
: _Base(__use_alloc<_Head, _Alloc, _UHead>(__a),
       std::forward<_UHead>(__head)) { }

    template<typename _Alloc>
      _Tuple_impl(allocator_arg_t __tag, const _Alloc& __a,
           const _Tuple_impl& __in)
: _Base(__use_alloc<_Head, _Alloc, _Head>(__a), _M_head(__in)) { }

    template<typename _Alloc>
_Tuple_impl(allocator_arg_t __tag, const _Alloc& __a,
           _Tuple_impl&& __in)
: _Base(__use_alloc<_Head, _Alloc, _Head>(__a),
       std::forward<_Head>(_M_head(__in))) { }

    template<typename _Alloc, typename _UHead>
_Tuple_impl(allocator_arg_t __tag, const _Alloc& __a,
           const _Tuple_impl<_Idx, _UHead>& __in)
: _Base(__use_alloc<_Head, _Alloc, _Head>(__a),
_Tuple_impl<_Idx, _UHead>::_M_head(__in)) { }

    template<typename _Alloc, typename _UHead>
_Tuple_impl(allocator_arg_t __tag, const _Alloc& __a,
           _Tuple_impl<_Idx, _UHead>&& __in)
: _Base(__use_alloc<_Head, _Alloc, _UHead>(__a),
              std::forward<_UHead>(_Tuple_impl<_Idx, _UHead>::_M_head(__in)))
{ }

    _Tuple_impl&
    operator=(const _Tuple_impl& __in)
    {
_M_head(*this) = _M_head(__in);
return *this;
    }

    _Tuple_impl&
    operator=(_Tuple_impl&& __in)
    noexcept(is_nothrow_move_assignable<_Head>::value)
    {
_M_head(*this) = std::forward<_Head>(_M_head(__in));
return *this;
    }

    template<typename _UHead>
      _Tuple_impl&
      operator=(const _Tuple_impl<_Idx, _UHead>& __in)
      {
 _M_head(*this) = _Tuple_impl<_Idx, _UHead>::_M_head(__in);
 return *this;
}

    template<typename _UHead>
      _Tuple_impl&
      operator=(_Tuple_impl<_Idx, _UHead>&& __in)
      {
 _M_head(*this)
   = std::forward<_UHead>(_Tuple_impl<_Idx, _UHead>::_M_head(__in));
 return *this;
}

  protected:
    void
    _M_swap(_Tuple_impl& __in)
    noexcept(__is_nothrow_swappable<_Head>::value)
    {
using std::swap;
swap(_M_head(*this), _M_head(__in));
    }
  };

template<typename... _Elements>
  class tuple;

// Concept utility functions, reused in conditionally-explicit
// constructors.
template<bool, typename... _Elements>
struct _TC
{
  template<typename... _UElements>
  static constexpr bool _ConstructibleTuple()
  {
    return __and_<is_constructible<_Elements, const _UElements&>...>::value;
  }

  template<typename... _UElements>
  static constexpr bool _ImplicitlyConvertibleTuple()
  {
    return __and_<is_convertible<const _UElements&, _Elements>...>::value;
  }

  template<typename... _UElements>
  static constexpr bool _MoveConstructibleTuple()
  {
    return __and_<is_constructible<_Elements, _UElements&&>...>::value;
  }

  template<typename... _UElements>
  static constexpr bool _ImplicitlyMoveConvertibleTuple()
  {
    return __and_<is_convertible<_UElements&&, _Elements>...>::value;
  }

  template<typename _SrcTuple>
  static constexpr bool _NonNestedTuple()
  {
    return  __and_<__not_<is_same<tuple<_Elements...>,
                                 typename remove_cv<
                                   typename remove_reference<_SrcTuple>::type
                                 >::type>>,
                   __not_<is_convertible<_SrcTuple, _Elements...>>,
                   __not_<is_constructible<_Elements..., _SrcTuple>>
            >::value;
  }
  template<typename... _UElements>
  static constexpr bool _NotSameTuple()
  {
    return  __not_<is_same<tuple<_Elements...>,
	     typename remove_const<
	       typename remove_reference<_UElements...>::type
	       >::type>>::value;
  }
};

template<typename... _Elements>
struct _TC<false, _Elements...>
{
  template<typename... _UElements>
  static constexpr bool _ConstructibleTuple()
  {
    return false;
  }

  template<typename... _UElements>
  static constexpr bool _ImplicitlyConvertibleTuple()
  {
    return false;
  }

  template<typename... _UElements>
  static constexpr bool _MoveConstructibleTuple()
  {
    return false;
  }

  template<typename... _UElements>
  static constexpr bool _ImplicitlyMoveConvertibleTuple()
  {
    return false;
  }

  template<typename... _UElements>
  static constexpr bool _NonNestedTuple()
  {
    return true;
  }
  template<typename... _UElements>
  static constexpr bool _NotSameTuple()
  {
    return  true;
  }
};

/// Primary class template, tuple
template<typename... _Elements> 
  class tuple : public _Tuple_impl<0, _Elements...>
  {
    typedef _Tuple_impl<0, _Elements...> _Inherited;

    // Used for constraining the default constructor so
    // that it becomes dependent on the constraints.
    template<typename _Dummy>
    struct _TC2
    {
      static constexpr bool _DefaultConstructibleTuple()
      {
        return __and_<is_default_constructible<_Elements>...>::value;
      }
      static constexpr bool _ImplicitlyDefaultConstructibleTuple()
      {
        return __and_<__is_implicitly_default_constructible<_Elements>...>
          ::value;
      }
    };

  public:
    template<typename _Dummy = void,
             typename enable_if<_TC2<_Dummy>::
                                  _ImplicitlyDefaultConstructibleTuple(),
                                bool>::type = true>
    constexpr tuple()
    : _Inherited() { }

    template<typename _Dummy = void,
             typename enable_if<_TC2<_Dummy>::
                                  _DefaultConstructibleTuple()
                                &&
                                !_TC2<_Dummy>::
                                  _ImplicitlyDefaultConstructibleTuple(),
                                bool>::type = false>
    explicit constexpr tuple()
    : _Inherited() { }

    // Shortcut for the cases where constructors taking _Elements...
    // need to be constrained.
    template<typename _Dummy> using _TCC =
      _TC<is_same<_Dummy, void>::value,
          _Elements...>;

    template<typename _Dummy = void,
             typename enable_if<
               _TCC<_Dummy>::template
                 _ConstructibleTuple<_Elements...>()
               && _TCC<_Dummy>::template
                 _ImplicitlyConvertibleTuple<_Elements...>()
               && (sizeof...(_Elements) >= 1),
             bool>::type=true>
      constexpr tuple(const _Elements&... __elements)
    : _Inherited(__elements...) { }

    template<typename _Dummy = void,
             typename enable_if<
               _TCC<_Dummy>::template
                 _ConstructibleTuple<_Elements...>()
               && !_TCC<_Dummy>::template
                 _ImplicitlyConvertibleTuple<_Elements...>()
               && (sizeof...(_Elements) >= 1),
             bool>::type=false>
    explicit constexpr tuple(const _Elements&... __elements)
    : _Inherited(__elements...) { }

    // Shortcut for the cases where constructors taking _UElements...
    // need to be constrained.
    template<typename... _UElements> using _TMC =
                _TC<(sizeof...(_Elements) == sizeof...(_UElements)),
                    _Elements...>;

    template<typename... _UElements, typename
      enable_if<
  _TC<sizeof...(_UElements) == 1, _Elements...>::template
    _NotSameTuple<_UElements...>()
  && _TMC<_UElements...>::template
                  _MoveConstructibleTuple<_UElements...>()
                && _TMC<_UElements...>::template
                  _ImplicitlyMoveConvertibleTuple<_UElements...>()
                && (sizeof...(_Elements) >= 1),
      bool>::type=true>
      constexpr tuple(_UElements&&... __elements)
      : _Inherited(std::forward<_UElements>(__elements)...) { }

    template<typename... _UElements, typename
      enable_if<
  _TC<sizeof...(_UElements) == 1, _Elements...>::template
    _NotSameTuple<_UElements...>()
  && _TMC<_UElements...>::template
                  _MoveConstructibleTuple<_UElements...>()
                && !_TMC<_UElements...>::template
                  _ImplicitlyMoveConvertibleTuple<_UElements...>()
                && (sizeof...(_Elements) >= 1),
      bool>::type=false>
      explicit constexpr tuple(_UElements&&... __elements)
: _Inherited(std::forward<_UElements>(__elements)...) {	}

    constexpr tuple(const tuple&) = default;

    constexpr tuple(tuple&&) = default; 

    // Shortcut for the cases where constructors taking tuples
    // must avoid creating temporaries.
    template<typename _Dummy> using _TNTC =
      _TC<is_same<_Dummy, void>::value && sizeof...(_Elements) == 1,
          _Elements...>;

    template<typename... _UElements, typename _Dummy = void, typename
      enable_if<_TMC<_UElements...>::template
                  _ConstructibleTuple<_UElements...>()
                && _TMC<_UElements...>::template
                  _ImplicitlyConvertibleTuple<_UElements...>()
                && _TNTC<_Dummy>::template
                  _NonNestedTuple<const tuple<_UElements...>&>(),
      bool>::type=true>
      constexpr tuple(const tuple<_UElements...>& __in)
      : _Inherited(static_cast<const _Tuple_impl<0, _UElements...>&>(__in))
      { }

    template<typename... _UElements, typename _Dummy = void, typename
      enable_if<_TMC<_UElements...>::template
                  _ConstructibleTuple<_UElements...>()
                && !_TMC<_UElements...>::template
                  _ImplicitlyConvertibleTuple<_UElements...>()
                && _TNTC<_Dummy>::template
                  _NonNestedTuple<const tuple<_UElements...>&>(),
      bool>::type=false>
      explicit constexpr tuple(const tuple<_UElements...>& __in)
      : _Inherited(static_cast<const _Tuple_impl<0, _UElements...>&>(__in))
      { }

    template<typename... _UElements, typename _Dummy = void, typename
      enable_if<_TMC<_UElements...>::template
                  _MoveConstructibleTuple<_UElements...>()
                && _TMC<_UElements...>::template
                  _ImplicitlyMoveConvertibleTuple<_UElements...>()
                && _TNTC<_Dummy>::template
                  _NonNestedTuple<tuple<_UElements...>&&>(),
      bool>::type=true>
      constexpr tuple(tuple<_UElements...>&& __in)
      : _Inherited(static_cast<_Tuple_impl<0, _UElements...>&&>(__in)) { }

    template<typename... _UElements, typename _Dummy = void, typename
      enable_if<_TMC<_UElements...>::template
                  _MoveConstructibleTuple<_UElements...>()
                && !_TMC<_UElements...>::template
                  _ImplicitlyMoveConvertibleTuple<_UElements...>()
                && _TNTC<_Dummy>::template
                  _NonNestedTuple<tuple<_UElements...>&&>(),
      bool>::type=false>
      explicit constexpr tuple(tuple<_UElements...>&& __in)
      : _Inherited(static_cast<_Tuple_impl<0, _UElements...>&&>(__in)) { }

    // Allocator-extended constructors.

    template<typename _Alloc>
tuple(allocator_arg_t __tag, const _Alloc& __a)
: _Inherited(__tag, __a) { }

    template<typename _Alloc, typename _Dummy = void,
             typename enable_if<
               _TCC<_Dummy>::template
                 _ConstructibleTuple<_Elements...>()
               && _TCC<_Dummy>::template
                 _ImplicitlyConvertibleTuple<_Elements...>(),
             bool>::type=true>
tuple(allocator_arg_t __tag, const _Alloc& __a,
     const _Elements&... __elements)
: _Inherited(__tag, __a, __elements...) { }

    template<typename _Alloc, typename _Dummy = void,
             typename enable_if<
               _TCC<_Dummy>::template
                 _ConstructibleTuple<_Elements...>()
               && !_TCC<_Dummy>::template
                 _ImplicitlyConvertibleTuple<_Elements...>(),
             bool>::type=false>
explicit tuple(allocator_arg_t __tag, const _Alloc& __a,
                     const _Elements&... __elements)
: _Inherited(__tag, __a, __elements...) { }

    template<typename _Alloc, typename... _UElements, typename
      enable_if<_TMC<_UElements...>::template
                  _MoveConstructibleTuple<_UElements...>()
                && _TMC<_UElements...>::template
                  _ImplicitlyMoveConvertibleTuple<_UElements...>(),
      bool>::type=true>
tuple(allocator_arg_t __tag, const _Alloc& __a,
     _UElements&&... __elements)
: _Inherited(__tag, __a, std::forward<_UElements>(__elements)...)
     	{ }

    template<typename _Alloc, typename... _UElements, typename
      enable_if<_TMC<_UElements...>::template
                  _MoveConstructibleTuple<_UElements...>()
                && !_TMC<_UElements...>::template
                  _ImplicitlyMoveConvertibleTuple<_UElements...>(),
      bool>::type=false>
explicit tuple(allocator_arg_t __tag, const _Alloc& __a,
     _UElements&&... __elements)
: _Inherited(__tag, __a, std::forward<_UElements>(__elements)...)
      { }

    template<typename _Alloc>
tuple(allocator_arg_t __tag, const _Alloc& __a, const tuple& __in)
: _Inherited(__tag, __a, static_cast<const _Inherited&>(__in)) { }

    template<typename _Alloc>
tuple(allocator_arg_t __tag, const _Alloc& __a, tuple&& __in)
: _Inherited(__tag, __a, static_cast<_Inherited&&>(__in)) { }

    template<typename _Alloc, typename... _UElements, typename
      enable_if<_TMC<_UElements...>::template
                  _ConstructibleTuple<_UElements...>()
                && _TMC<_UElements...>::template
                  _ImplicitlyConvertibleTuple<_UElements...>(),
      bool>::type=true>
tuple(allocator_arg_t __tag, const _Alloc& __a,
     const tuple<_UElements...>& __in)
: _Inherited(__tag, __a,
            static_cast<const _Tuple_impl<0, _UElements...>&>(__in))
{ }

    template<typename _Alloc, typename... _UElements, typename
      enable_if<_TMC<_UElements...>::template
                  _ConstructibleTuple<_UElements...>()
                && !_TMC<_UElements...>::template
                  _ImplicitlyConvertibleTuple<_UElements...>(),
      bool>::type=false>
explicit tuple(allocator_arg_t __tag, const _Alloc& __a,
     const tuple<_UElements...>& __in)
: _Inherited(__tag, __a,
            static_cast<const _Tuple_impl<0, _UElements...>&>(__in))
{ }

    template<typename _Alloc, typename... _UElements, typename
      enable_if<_TMC<_UElements...>::template
                  _MoveConstructibleTuple<_UElements...>()
                && _TMC<_UElements...>::template
                  _ImplicitlyMoveConvertibleTuple<_UElements...>(),
      bool>::type=true>
tuple(allocator_arg_t __tag, const _Alloc& __a,
     tuple<_UElements...>&& __in)
: _Inherited(__tag, __a,
            static_cast<_Tuple_impl<0, _UElements...>&&>(__in))
{ }

    template<typename _Alloc, typename... _UElements, typename
      enable_if<_TMC<_UElements...>::template
                  _MoveConstructibleTuple<_UElements...>()
                && !_TMC<_UElements...>::template
                  _ImplicitlyMoveConvertibleTuple<_UElements...>(),
      bool>::type=false>
explicit tuple(allocator_arg_t __tag, const _Alloc& __a,
     tuple<_UElements...>&& __in)
: _Inherited(__tag, __a,
            static_cast<_Tuple_impl<0, _UElements...>&&>(__in))
{ }

    tuple&
    operator=(const tuple& __in)
    {
static_cast<_Inherited&>(*this) = __in;
return *this;
    }

    tuple&
    operator=(tuple&& __in)
    noexcept(is_nothrow_move_assignable<_Inherited>::value)
    {
static_cast<_Inherited&>(*this) = std::move(__in);
return *this;
    }

    template<typename... _UElements, typename = typename
      enable_if<sizeof...(_UElements)
	 == sizeof...(_Elements)>::type>
      tuple&
      operator=(const tuple<_UElements...>& __in)
      {
 static_cast<_Inherited&>(*this) = __in;
 return *this;
}

    template<typename... _UElements, typename = typename
      enable_if<sizeof...(_UElements)
	 == sizeof...(_Elements)>::type>
      tuple&
      operator=(tuple<_UElements...>&& __in)
      {
 static_cast<_Inherited&>(*this) = std::move(__in);
 return *this;
}

    void
    swap(tuple& __in)
    noexcept(noexcept(__in._M_swap(__in)))
    { _Inherited::_M_swap(__in); }
  };

// Explicit specialization, zero-element tuple.
template<>
  class tuple<>
  {
  public:
    void swap(tuple&) noexcept { /* no-op */ }
  };

/// Partial specialization, 2-element tuple.
/// Includes construction and assignment from a pair.
template<typename _T1, typename _T2>
  class tuple<_T1, _T2> : public _Tuple_impl<0, _T1, _T2>
  {
    typedef _Tuple_impl<0, _T1, _T2> _Inherited;

  public:
    template <typename _U1 = _T1,
              typename _U2 = _T2,
              typename enable_if<__and_<
                                   __is_implicitly_default_constructible<_U1>,
                                   __is_implicitly_default_constructible<_U2>>
                                 ::value, bool>::type = true>

    constexpr tuple()
    : _Inherited() { }

    template <typename _U1 = _T1,
              typename _U2 = _T2,
              typename enable_if<
                __and_<
                  is_default_constructible<_U1>,
                  is_default_constructible<_U2>,
                  __not_<
                    __and_<__is_implicitly_default_constructible<_U1>,
                           __is_implicitly_default_constructible<_U2>>>>
                ::value, bool>::type = false>

    explicit constexpr tuple()
    : _Inherited() { }

    // Shortcut for the cases where constructors taking _T1, _T2
    // need to be constrained.
    template<typename _Dummy> using _TCC =
      _TC<is_same<_Dummy, void>::value, _T1, _T2>;

    template<typename _Dummy = void, typename
             enable_if<_TCC<_Dummy>::template
                         _ConstructibleTuple<_T1, _T2>()
                       && _TCC<_Dummy>::template
                         _ImplicitlyConvertibleTuple<_T1, _T2>(),
bool>::type = true>
      constexpr tuple(const _T1& __a1, const _T2& __a2)
      : _Inherited(__a1, __a2) { }
26
←
Calling constructor for '_Tuple_impl<0, const llvm::TargetRegisterClass *&, const llvm::TargetRegisterClass *&>'→
33
←
Returning from constructor for '_Tuple_impl<0, const llvm::TargetRegisterClass *&, const llvm::TargetRegisterClass *&>'→

    template<typename _Dummy = void, typename
             enable_if<_TCC<_Dummy>::template
                         _ConstructibleTuple<_T1, _T2>()
                       && !_TCC<_Dummy>::template
                         _ImplicitlyConvertibleTuple<_T1, _T2>(),
bool>::type = false>
      explicit constexpr tuple(const _T1& __a1, const _T2& __a2)
      : _Inherited(__a1, __a2) { }

    // Shortcut for the cases where constructors taking _U1, _U2
    // need to be constrained.
    using _TMC = _TC<true, _T1, _T2>;

    template<typename _U1, typename _U2, typename
      enable_if<_TMC::template
                  _MoveConstructibleTuple<_U1, _U2>()
                && _TMC::template
                  _ImplicitlyMoveConvertibleTuple<_U1, _U2>()
         && !is_same<typename decay<_U1>::type,
	      allocator_arg_t>::value,
bool>::type = true>
      constexpr tuple(_U1&& __a1, _U2&& __a2)
: _Inherited(std::forward<_U1>(__a1), std::forward<_U2>(__a2)) { }

    template<typename _U1, typename _U2, typename
      enable_if<_TMC::template
                  _MoveConstructibleTuple<_U1, _U2>()
                && !_TMC::template
                  _ImplicitlyMoveConvertibleTuple<_U1, _U2>()
         && !is_same<typename decay<_U1>::type,
	      allocator_arg_t>::value,
bool>::type = false>
      explicit constexpr tuple(_U1&& __a1, _U2&& __a2)
: _Inherited(std::forward<_U1>(__a1), std::forward<_U2>(__a2)) { }

    constexpr tuple(const tuple&) = default;

    constexpr tuple(tuple&&) = default;

    template<typename _U1, typename _U2, typename
      enable_if<_TMC::template
                  _ConstructibleTuple<_U1, _U2>()
                && _TMC::template
                  _ImplicitlyConvertibleTuple<_U1, _U2>(),
bool>::type = true>
      constexpr tuple(const tuple<_U1, _U2>& __in)
: _Inherited(static_cast<const _Tuple_impl<0, _U1, _U2>&>(__in)) { }

    template<typename _U1, typename _U2, typename
      enable_if<_TMC::template
                  _ConstructibleTuple<_U1, _U2>()
                && !_TMC::template
                  _ImplicitlyConvertibleTuple<_U1, _U2>(),
bool>::type = false>
      explicit constexpr tuple(const tuple<_U1, _U2>& __in)
: _Inherited(static_cast<const _Tuple_impl<0, _U1, _U2>&>(__in)) { }

    template<typename _U1, typename _U2, typename
      enable_if<_TMC::template
                  _MoveConstructibleTuple<_U1, _U2>()
                && _TMC::template
                  _ImplicitlyMoveConvertibleTuple<_U1, _U2>(),
bool>::type = true>
      constexpr tuple(tuple<_U1, _U2>&& __in)
: _Inherited(static_cast<_Tuple_impl<0, _U1, _U2>&&>(__in)) { }

    template<typename _U1, typename _U2, typename
      enable_if<_TMC::template
                  _MoveConstructibleTuple<_U1, _U2>()
                && !_TMC::template
                  _ImplicitlyMoveConvertibleTuple<_U1, _U2>(),
bool>::type = false>
      explicit constexpr tuple(tuple<_U1, _U2>&& __in)
: _Inherited(static_cast<_Tuple_impl<0, _U1, _U2>&&>(__in)) { }

    template<typename _U1, typename _U2, typename
      enable_if<_TMC::template
                  _ConstructibleTuple<_U1, _U2>()
                && _TMC::template
                  _ImplicitlyConvertibleTuple<_U1, _U2>(),
bool>::type = true>
      constexpr tuple(const pair<_U1, _U2>& __in)
: _Inherited(__in.first, __in.second) { }

    template<typename _U1, typename _U2, typename
      enable_if<_TMC::template
                  _ConstructibleTuple<_U1, _U2>()
                && !_TMC::template
                  _ImplicitlyConvertibleTuple<_U1, _U2>(),
bool>::type = false>
      explicit constexpr tuple(const pair<_U1, _U2>& __in)
: _Inherited(__in.first, __in.second) { }

    template<typename _U1, typename _U2, typename
      enable_if<_TMC::template
                  _MoveConstructibleTuple<_U1, _U2>()
                && _TMC::template
                  _ImplicitlyMoveConvertibleTuple<_U1, _U2>(),
bool>::type = true>
      constexpr tuple(pair<_U1, _U2>&& __in)
: _Inherited(std::forward<_U1>(__in.first),
     std::forward<_U2>(__in.second)) { }

    template<typename _U1, typename _U2, typename
      enable_if<_TMC::template
                  _MoveConstructibleTuple<_U1, _U2>()
                && !_TMC::template
                  _ImplicitlyMoveConvertibleTuple<_U1, _U2>(),
bool>::type = false>
      explicit constexpr tuple(pair<_U1, _U2>&& __in)
: _Inherited(std::forward<_U1>(__in.first),
     std::forward<_U2>(__in.second)) { }

    // Allocator-extended constructors.

    template<typename _Alloc>
tuple(allocator_arg_t __tag, const _Alloc& __a)
: _Inherited(__tag, __a) { }

    template<typename _Alloc, typename _Dummy = void,
             typename enable_if<
               _TCC<_Dummy>::template
                 _ConstructibleTuple<_T1, _T2>()
               && _TCC<_Dummy>::template
                 _ImplicitlyConvertibleTuple<_T1, _T2>(),
             bool>::type=true>

tuple(allocator_arg_t __tag, const _Alloc& __a,
     const _T1& __a1, const _T2& __a2)
: _Inherited(__tag, __a, __a1, __a2) { }

    template<typename _Alloc, typename _Dummy = void,
             typename enable_if<
               _TCC<_Dummy>::template
                 _ConstructibleTuple<_T1, _T2>()
               && !_TCC<_Dummy>::template
                 _ImplicitlyConvertibleTuple<_T1, _T2>(),
             bool>::type=false>

explicit tuple(allocator_arg_t __tag, const _Alloc& __a,
     const _T1& __a1, const _T2& __a2)
: _Inherited(__tag, __a, __a1, __a2) { }

    template<typename _Alloc, typename _U1, typename _U2, typename
      enable_if<_TMC::template
                  _MoveConstructibleTuple<_U1, _U2>()
                && _TMC::template
                  _ImplicitlyMoveConvertibleTuple<_U1, _U2>(),
bool>::type = true>
tuple(allocator_arg_t __tag, const _Alloc& __a, _U1&& __a1, _U2&& __a2)
: _Inherited(__tag, __a, std::forward<_U1>(__a1),
            std::forward<_U2>(__a2)) { }

    template<typename _Alloc, typename _U1, typename _U2, typename
      enable_if<_TMC::template
                  _MoveConstructibleTuple<_U1, _U2>()
                && !_TMC::template
                  _ImplicitlyMoveConvertibleTuple<_U1, _U2>(),
bool>::type = false>
explicit tuple(allocator_arg_t __tag, const _Alloc& __a,
                     _U1&& __a1, _U2&& __a2)
: _Inherited(__tag, __a, std::forward<_U1>(__a1),
            std::forward<_U2>(__a2)) { }

    template<typename _Alloc>
tuple(allocator_arg_t __tag, const _Alloc& __a, const tuple& __in)
: _Inherited(__tag, __a, static_cast<const _Inherited&>(__in)) { }

    template<typename _Alloc>
tuple(allocator_arg_t __tag, const _Alloc& __a, tuple&& __in)
: _Inherited(__tag, __a, static_cast<_Inherited&&>(__in)) { }

    template<typename _Alloc, typename _U1, typename _U2, typename
      enable_if<_TMC::template
                  _ConstructibleTuple<_U1, _U2>()
                && _TMC::template
                  _ImplicitlyConvertibleTuple<_U1, _U2>(),
bool>::type = true>
tuple(allocator_arg_t __tag, const _Alloc& __a,
     const tuple<_U1, _U2>& __in)
: _Inherited(__tag, __a,
            static_cast<const _Tuple_impl<0, _U1, _U2>&>(__in))
{ }

    template<typename _Alloc, typename _U1, typename _U2, typename
      enable_if<_TMC::template
                  _ConstructibleTuple<_U1, _U2>()
                && !_TMC::template
                  _ImplicitlyConvertibleTuple<_U1, _U2>(),
bool>::type = false>
explicit tuple(allocator_arg_t __tag, const _Alloc& __a,
     const tuple<_U1, _U2>& __in)
: _Inherited(__tag, __a,
            static_cast<const _Tuple_impl<0, _U1, _U2>&>(__in))
{ }

    template<typename _Alloc, typename _U1, typename _U2, typename
      enable_if<_TMC::template
                  _MoveConstructibleTuple<_U1, _U2>()
                && _TMC::template
                  _ImplicitlyMoveConvertibleTuple<_U1, _U2>(),
bool>::type = true>
tuple(allocator_arg_t __tag, const _Alloc& __a, tuple<_U1, _U2>&& __in)
: _Inherited(__tag, __a, static_cast<_Tuple_impl<0, _U1, _U2>&&>(__in))
{ }

    template<typename _Alloc, typename _U1, typename _U2, typename
      enable_if<_TMC::template
                  _MoveConstructibleTuple<_U1, _U2>()
                && !_TMC::template
                  _ImplicitlyMoveConvertibleTuple<_U1, _U2>(),
bool>::type = false>
explicit tuple(allocator_arg_t __tag, const _Alloc& __a,
                     tuple<_U1, _U2>&& __in)
: _Inherited(__tag, __a, static_cast<_Tuple_impl<0, _U1, _U2>&&>(__in))
{ }

    template<typename _Alloc, typename _U1, typename _U2, typename
      enable_if<_TMC::template
                  _ConstructibleTuple<_U1, _U2>()
                && _TMC::template
                  _ImplicitlyConvertibleTuple<_U1, _U2>(),
bool>::type = true>
      tuple(allocator_arg_t __tag, const _Alloc& __a,
     const pair<_U1, _U2>& __in)
: _Inherited(__tag, __a, __in.first, __in.second) { }

    template<typename _Alloc, typename _U1, typename _U2, typename
      enable_if<_TMC::template
                  _ConstructibleTuple<_U1, _U2>()
                && !_TMC::template
                  _ImplicitlyConvertibleTuple<_U1, _U2>(),
bool>::type = false>
      explicit tuple(allocator_arg_t __tag, const _Alloc& __a,
     const pair<_U1, _U2>& __in)
: _Inherited(__tag, __a, __in.first, __in.second) { }

    template<typename _Alloc, typename _U1, typename _U2, typename
      enable_if<_TMC::template
                  _MoveConstructibleTuple<_U1, _U2>()
                && _TMC::template
                  _ImplicitlyMoveConvertibleTuple<_U1, _U2>(),
bool>::type = true>
      tuple(allocator_arg_t __tag, const _Alloc& __a, pair<_U1, _U2>&& __in)
: _Inherited(__tag, __a, std::forward<_U1>(__in.first),
     std::forward<_U2>(__in.second)) { }

    template<typename _Alloc, typename _U1, typename _U2, typename
      enable_if<_TMC::template
                  _MoveConstructibleTuple<_U1, _U2>()
                && !_TMC::template
                  _ImplicitlyMoveConvertibleTuple<_U1, _U2>(),
bool>::type = false>
      explicit tuple(allocator_arg_t __tag, const _Alloc& __a,
                     pair<_U1, _U2>&& __in)
: _Inherited(__tag, __a, std::forward<_U1>(__in.first),
     std::forward<_U2>(__in.second)) { }

    tuple&
    operator=(const tuple& __in)
    {
static_cast<_Inherited&>(*this) = __in;
return *this;
    }

    tuple&
    operator=(tuple&& __in)
    noexcept(is_nothrow_move_assignable<_Inherited>::value)
    {
static_cast<_Inherited&>(*this) = std::move(__in);
return *this;
    }

    template<typename _U1, typename _U2>
      tuple&
      operator=(const tuple<_U1, _U2>& __in)
      {
 static_cast<_Inherited&>(*this) = __in;
 return *this;
}

    template<typename _U1, typename _U2>
      tuple&
      operator=(tuple<_U1, _U2>&& __in)
      {
 static_cast<_Inherited&>(*this) = std::move(__in);
 return *this;
}

    template<typename _U1, typename _U2>
      tuple&
      operator=(const pair<_U1, _U2>& __in)
      {
 this->_M_head(*this) = __in.first;
 this->_M_tail(*this)._M_head(*this) = __in.second;
 return *this;
}

    template<typename _U1, typename _U2>
      tuple&
      operator=(pair<_U1, _U2>&& __in)
      {
 this->_M_head(*this) = std::forward<_U1>(__in.first);
37
←
Value assigned to 'SrcRC', which participates in a condition later→
 this->_M_tail(*this)._M_head(*this) = std::forward<_U2>(__in.second);
38
←
Value assigned to 'DstRC', which participates in a condition later→
 return *this;
}

    void
    swap(tuple& __in)
    noexcept(noexcept(__in._M_swap(__in)))
    { _Inherited::_M_swap(__in); }
  };


/**
 * Recursive case for tuple_element: strip off the first element in
 * the tuple and retrieve the (i-1)th element of the remaining tuple.
 */
template<std::size_t __i, typename _Head, typename... _Tail>
  struct tuple_element<__i, tuple<_Head, _Tail...> >
  : tuple_element<__i - 1, tuple<_Tail...> > { };

/**
 * Basis case for tuple_element: The first element is the one we're seeking.
 */
template<typename _Head, typename... _Tail>
  struct tuple_element<0, tuple<_Head, _Tail...> >
  {
    typedef _Head type;
  };

/// class tuple_size
template<typename... _Elements>
  struct tuple_size<tuple<_Elements...>>
  : public integral_constant<std::size_t, sizeof...(_Elements)> { };

template<std::size_t __i, typename _Head, typename... _Tail>
  constexpr _Head&
  __get_helper(_Tuple_impl<__i, _Head, _Tail...>& __t) noexcept
  { return _Tuple_impl<__i, _Head, _Tail...>::_M_head(__t); }

template<std::size_t __i, typename _Head, typename... _Tail>
  constexpr const _Head&
  __get_helper(const _Tuple_impl<__i, _Head, _Tail...>& __t) noexcept
  { return _Tuple_impl<__i, _Head, _Tail...>::_M_head(__t); }

/// Return a reference to the ith element of a tuple.
template<std::size_t __i, typename... _Elements>
  constexpr __tuple_element_t<__i, tuple<_Elements...>>&
  get(tuple<_Elements...>& __t) noexcept
  { return std::__get_helper<__i>(__t); }

/// Return a const reference to the ith element of a const tuple.
template<std::size_t __i, typename... _Elements>
  constexpr const __tuple_element_t<__i, tuple<_Elements...>>&
  get(const tuple<_Elements...>& __t) noexcept
  { return std::__get_helper<__i>(__t); }

/// Return an rvalue reference to the ith element of a tuple rvalue.
template<std::size_t __i, typename... _Elements>
  constexpr __tuple_element_t<__i, tuple<_Elements...>>&&
  get(tuple<_Elements...>&& __t) noexcept
  {
    typedef __tuple_element_t<__i, tuple<_Elements...>> __element_type;
    return std::forward<__element_type&&>(std::get<__i>(__t));
  }

1276#if __cplusplus201402L > 201103L

1278#define __cpp_lib_tuples_by_type201304 201304

template<typename _Head, size_t __i, typename... _Tail>
  constexpr _Head&
  __get_helper2(_Tuple_impl<__i, _Head, _Tail...>& __t) noexcept
  { return _Tuple_impl<__i, _Head, _Tail...>::_M_head(__t); }

template<typename _Head, size_t __i, typename... _Tail>
  constexpr const _Head&
  __get_helper2(const _Tuple_impl<__i, _Head, _Tail...>& __t) noexcept
  { return _Tuple_impl<__i, _Head, _Tail...>::_M_head(__t); }

/// Return a reference to the unique element of type _Tp of a tuple.
template <typename _Tp, typename... _Types>
  constexpr _Tp&
  get(tuple<_Types...>& __t) noexcept
  { return std::__get_helper2<_Tp>(__t); }

/// Return a reference to the unique element of type _Tp of a tuple rvalue.
template <typename _Tp, typename... _Types>
  constexpr _Tp&&
  get(tuple<_Types...>&& __t) noexcept
  { return std::forward<_Tp&&>(std::__get_helper2<_Tp>(__t)); }

/// Return a const reference to the unique element of type _Tp of a tuple.
template <typename _Tp, typename... _Types>
  constexpr const _Tp&
  get(const tuple<_Types...>& __t) noexcept
  { return std::__get_helper2<_Tp>(__t); }
1307#endif

// This class performs the comparison operations on tuples
template<typename _Tp, typename _Up, size_t __i, size_t __size>
  struct __tuple_compare
  {
    static constexpr bool
    __eq(const _Tp& __t, const _Up& __u)
    {
return bool(std::get<__i>(__t) == std::get<__i>(__u))
 && __tuple_compare<_Tp, _Up, __i + 1, __size>::__eq(__t, __u);
    }
 
    static constexpr bool
    __less(const _Tp& __t, const _Up& __u)
    {
return bool(std::get<__i>(__t) < std::get<__i>(__u))
 || (!bool(std::get<__i>(__u) < std::get<__i>(__t))
     && __tuple_compare<_Tp, _Up, __i + 1, __size>::__less(__t, __u));
    }
  };

template<typename _Tp, typename _Up, size_t __size>
  struct __tuple_compare<_Tp, _Up, __size, __size>
  {
    static constexpr bool
    __eq(const _Tp&, const _Up&) { return true; }
 
    static constexpr bool
    __less(const _Tp&, const _Up&) { return false; }
  };

template<typename... _TElements, typename... _UElements>
  constexpr bool
  operator==(const tuple<_TElements...>& __t,
      const tuple<_UElements...>& __u)
  {
    static_assert(sizeof...(_TElements) == sizeof...(_UElements),
 "tuple objects can only be compared if they have equal sizes.");
    using __compare = __tuple_compare<tuple<_TElements...>,
			tuple<_UElements...>,
			0, sizeof...(_TElements)>;
    return __compare::__eq(__t, __u);
  }

template<typename... _TElements, typename... _UElements>
  constexpr bool
  operator<(const tuple<_TElements...>& __t,
     const tuple<_UElements...>& __u)
  {
    static_assert(sizeof...(_TElements) == sizeof...(_UElements),
 "tuple objects can only be compared if they have equal sizes.");
    using __compare = __tuple_compare<tuple<_TElements...>,
			tuple<_UElements...>,
			0, sizeof...(_TElements)>;
    return __compare::__less(__t, __u);
  }

template<typename... _TElements, typename... _UElements>
  constexpr bool
  operator!=(const tuple<_TElements...>& __t,
      const tuple<_UElements...>& __u)
  { return !(__t == __u); }

template<typename... _TElements, typename... _UElements>
  constexpr bool
  operator>(const tuple<_TElements...>& __t,
     const tuple<_UElements...>& __u)
  { return __u < __t; }

template<typename... _TElements, typename... _UElements>
  constexpr bool
  operator<=(const tuple<_TElements...>& __t,
      const tuple<_UElements...>& __u)
  { return !(__u < __t); }

template<typename... _TElements, typename... _UElements>
  constexpr bool
  operator>=(const tuple<_TElements...>& __t,
      const tuple<_UElements...>& __u)
  { return !(__t < __u); }

// NB: DR 705.
template<typename... _Elements>
  constexpr tuple<typename __decay_and_strip<_Elements>::__type...>
  make_tuple(_Elements&&... __args)
  {
    typedef tuple<typename __decay_and_strip<_Elements>::__type...>
__result_type;
    return __result_type(std::forward<_Elements>(__args)...);
  }

// _GLIBCXX_RESOLVE_LIB_DEFECTS
// 2275. Why is forward_as_tuple not constexpr?
template<typename... _Elements>
  constexpr tuple<_Elements&&...>
  forward_as_tuple(_Elements&&... __args) noexcept
  { return tuple<_Elements&&...>(std::forward<_Elements>(__args)...); }

template<typename... _Tps>
  struct __is_tuple_like_impl<tuple<_Tps...>> : true_type
  { };

// Internal type trait that allows us to sfinae-protect tuple_cat.
template<typename _Tp>
  struct __is_tuple_like
  : public __is_tuple_like_impl<typename std::remove_cv
          <typename std::remove_reference<_Tp>::type>::type>::type
  { };

template<size_t, typename, typename, size_t>
  struct __make_tuple_impl;

template<size_t _Idx, typename _Tuple, typename... _Tp, size_t _Nm>
  struct __make_tuple_impl<_Idx, tuple<_Tp...>, _Tuple, _Nm>
  : __make_tuple_impl<_Idx + 1,
	tuple<_Tp..., __tuple_element_t<_Idx, _Tuple>>,
	_Tuple, _Nm>
  { };

template<std::size_t _Nm, typename _Tuple, typename... _Tp>
  struct __make_tuple_impl<_Nm, tuple<_Tp...>, _Tuple, _Nm>
  {
    typedef tuple<_Tp...> __type;
  };

template<typename _Tuple>
  struct __do_make_tuple
  : __make_tuple_impl<0, tuple<>, _Tuple, std::tuple_size<_Tuple>::value>
  { };

// Returns the std::tuple equivalent of a tuple-like type.
template<typename _Tuple>
  struct __make_tuple
  : public __do_make_tuple<typename std::remove_cv
          <typename std::remove_reference<_Tuple>::type>::type>
  { };

// Combines several std::tuple's into a single one.
template<typename...>
  struct __combine_tuples;

template<>
  struct __combine_tuples<>
  {
    typedef tuple<> __type;
  };

template<typename... _Ts>
  struct __combine_tuples<tuple<_Ts...>>
  {
    typedef tuple<_Ts...> __type;
  };

template<typename... _T1s, typename... _T2s, typename... _Rem>
  struct __combine_tuples<tuple<_T1s...>, tuple<_T2s...>, _Rem...>
  {
    typedef typename __combine_tuples<tuple<_T1s..., _T2s...>,
			_Rem...>::__type __type;
  };

// Computes the result type of tuple_cat given a set of tuple-like types.
template<typename... _Tpls>
  struct __tuple_cat_result
  {
    typedef typename __combine_tuples
      <typename __make_tuple<_Tpls>::__type...>::__type __type;
  };

// Helper to determine the index set for the first tuple-like
// type of a given set.
template<typename...>
  struct __make_1st_indices;

template<>
  struct __make_1st_indices<>
  {
    typedef std::_Index_tuple<> __type;
  };

template<typename _Tp, typename... _Tpls>
  struct __make_1st_indices<_Tp, _Tpls...>
  {
    typedef typename std::_Build_index_tuple<std::tuple_size<
typename std::remove_reference<_Tp>::type>::value>::__type __type;
  };

// Performs the actual concatenation by step-wise expanding tuple-like
// objects into the elements,  which are finally forwarded into the
// result tuple.
template<typename _Ret, typename _Indices, typename... _Tpls>
  struct __tuple_concater;

template<typename _Ret, std::size_t... _Is, typename _Tp, typename... _Tpls>
  struct __tuple_concater<_Ret, std::_Index_tuple<_Is...>, _Tp, _Tpls...>
  {
    template<typename... _Us>
      static constexpr _Ret
      _S_do(_Tp&& __tp, _Tpls&&... __tps, _Us&&... __us)
      {
 typedef typename __make_1st_indices<_Tpls...>::__type __idx;
 typedef __tuple_concater<_Ret, __idx, _Tpls...>      __next;
 return __next::_S_do(std::forward<_Tpls>(__tps)...,
	       std::forward<_Us>(__us)...,
	       std::get<_Is>(std::forward<_Tp>(__tp))...);
}
  };

template<typename _Ret>
  struct __tuple_concater<_Ret, std::_Index_tuple<>>
  {
    template<typename... _Us>
static constexpr _Ret
_S_do(_Us&&... __us)
      {
 return _Ret(std::forward<_Us>(__us)...);
}
  };

/// tuple_cat
template<typename... _Tpls, typename = typename
         enable_if<__and_<__is_tuple_like<_Tpls>...>::value>::type>
  constexpr auto
  tuple_cat(_Tpls&&... __tpls)
  -> typename __tuple_cat_result<_Tpls...>::__type
  {
    typedef typename __tuple_cat_result<_Tpls...>::__type __ret;
    typedef typename __make_1st_indices<_Tpls...>::__type __idx;
    typedef __tuple_concater<__ret, __idx, _Tpls...> __concater;
    return __concater::_S_do(std::forward<_Tpls>(__tpls)...);
  }

// _GLIBCXX_RESOLVE_LIB_DEFECTS
// 2301. Why is tie not constexpr?
/// tie
template<typename... _Elements>
  constexpr tuple<_Elements&...>
  tie(_Elements&... __args) noexcept
  { return tuple<_Elements&...>(__args...); }
25
←
Calling constructor for 'tuple<const llvm::TargetRegisterClass *&, const llvm::TargetRegisterClass *&>'→
34
←
Returning from constructor for 'tuple<const llvm::TargetRegisterClass *&, const llvm::TargetRegisterClass *&>'→

/// swap
template<typename... _Elements>
  inline void 
  swap(tuple<_Elements...>& __x, tuple<_Elements...>& __y)
  noexcept(noexcept(__x.swap(__y)))
  { __x.swap(__y); }

// A class (and instance) which can be used in 'tie' when an element
// of a tuple is not required
struct _Swallow_assign
{
  template<class _Tp>
    const _Swallow_assign&
    operator=(const _Tp&) const
    { return *this; }
};

const _Swallow_assign ignore{};

/// Partial specialization for tuples
template<typename... _Types, typename _Alloc>
  struct uses_allocator<tuple<_Types...>, _Alloc> : true_type { };

// See stl_pair.h...
template<class _T1, class _T2>
  template<typename... _Args1, typename... _Args2>
    inline
    pair<_T1, _T2>::
    pair(piecewise_construct_t,
  tuple<_Args1...> __first, tuple<_Args2...> __second)
    : pair(__first, __second,
    typename _Build_index_tuple<sizeof...(_Args1)>::__type(),
    typename _Build_index_tuple<sizeof...(_Args2)>::__type())
    { }

template<class _T1, class _T2>
  template<typename... _Args1, std::size_t... _Indexes1,
           typename... _Args2, std::size_t... _Indexes2>
    inline
    pair<_T1, _T2>::
    pair(tuple<_Args1...>& __tuple1, tuple<_Args2...>& __tuple2,
  _Index_tuple<_Indexes1...>, _Index_tuple<_Indexes2...>)
    : first(std::forward<_Args1>(std::get<_Indexes1>(__tuple1))...),
      second(std::forward<_Args2>(std::get<_Indexes2>(__tuple2))...)
    { }

/// @}

1595_GLIBCXX_END_NAMESPACE_VERSION
1596} // namespace std

1598#endif // C++11

1600#endif // _GLIBCXX_TUPLE

←

/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/include/llvm/CodeGen/MachineInstr.h

1//===- llvm/CodeGen/MachineInstr.h - MachineInstr class ---------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains the declaration of the MachineInstr class, which is the
10// basic representation for all target dependent machine instructions used by
11// the back end.
12//
13//===----------------------------------------------------------------------===//

15#ifndef LLVM_CODEGEN_MACHINEINSTR_H
16#define LLVM_CODEGEN_MACHINEINSTR_H

18#include "llvm/ADT/DenseMapInfo.h"
19#include "llvm/ADT/PointerSumType.h"
20#include "llvm/ADT/ilist.h"
21#include "llvm/ADT/ilist_node.h"
22#include "llvm/ADT/iterator_range.h"
23#include "llvm/CodeGen/MachineMemOperand.h"
24#include "llvm/CodeGen/MachineOperand.h"
25#include "llvm/CodeGen/TargetOpcodes.h"
26#include "llvm/IR/DebugLoc.h"
27#include "llvm/IR/InlineAsm.h"
28#include "llvm/MC/MCInstrDesc.h"
29#include "llvm/MC/MCSymbol.h"
30#include "llvm/Support/ArrayRecycler.h"
31#include "llvm/Support/TrailingObjects.h"
32#include <algorithm>
33#include <cassert>
34#include <cstdint>
35#include <utility>

37namespace llvm {

39class AAResults;
40template <typename T> class ArrayRef;
41class DIExpression;
42class DILocalVariable;
43class MachineBasicBlock;
44class MachineFunction;
45class MachineRegisterInfo;
46class ModuleSlotTracker;
47class raw_ostream;
48template <typename T> class SmallVectorImpl;
49class SmallBitVector;
50class StringRef;
51class TargetInstrInfo;
52class TargetRegisterClass;
53class TargetRegisterInfo;

55//===----------------------------------------------------------------------===//
56/// Representation of each machine instruction.
57///
58/// This class isn't a POD type, but it must have a trivial destructor. When a
59/// MachineFunction is deleted, all the contained MachineInstrs are deallocated
60/// without having their destructor called.
61///
62class MachineInstr
  : public ilist_node_with_parent<MachineInstr, MachineBasicBlock,
                                  ilist_sentinel_tracking<true>> {
65public:
using mmo_iterator = ArrayRef<MachineMemOperand *>::iterator;

/// Flags to specify different kinds of comments to output in
/// assembly code.  These flags carry semantic information not
/// otherwise easily derivable from the IR text.
///
enum CommentFlag {
  ReloadReuse = 0x1,    // higher bits are reserved for target dep comments.
  NoSchedComment = 0x2,
  TAsmComments = 0x4    // Target Asm comments should start from this value.
};

enum MIFlag {
  NoFlags      = 0,
  FrameSetup   = 1 << 0,              // Instruction is used as a part of
                                      // function frame setup code.
  FrameDestroy = 1 << 1,              // Instruction is used as a part of
                                      // function frame destruction code.
  BundledPred  = 1 << 2,              // Instruction has bundled predecessors.
  BundledSucc  = 1 << 3,              // Instruction has bundled successors.
  FmNoNans     = 1 << 4,              // Instruction does not support Fast
                                      // math nan values.
  FmNoInfs     = 1 << 5,              // Instruction does not support Fast
                                      // math infinity values.
  FmNsz        = 1 << 6,              // Instruction is not required to retain
                                      // signed zero values.
  FmArcp       = 1 << 7,              // Instruction supports Fast math
                                      // reciprocal approximations.
  FmContract   = 1 << 8,              // Instruction supports Fast math
                                      // contraction operations like fma.
  FmAfn        = 1 << 9,              // Instruction may map to Fast math
                                      // instrinsic approximation.
  FmReassoc    = 1 << 10,             // Instruction supports Fast math
                                      // reassociation of operand order.
  NoUWrap      = 1 << 11,             // Instruction supports binary operator
                                      // no unsigned wrap.
  NoSWrap      = 1 << 12,             // Instruction supports binary operator
                                      // no signed wrap.
  IsExact      = 1 << 13,             // Instruction supports division is
                                      // known to be exact.
  NoFPExcept   = 1 << 14,             // Instruction does not raise
                                      // floatint-point exceptions.
  NoMerge      = 1 << 15,             // Passes that drop source location info
                                      // (e.g. branch folding) should skip
                                      // this instruction.
};

113private:
const MCInstrDesc *MCID;              // Instruction descriptor.
MachineBasicBlock *Parent = nullptr;  // Pointer to the owning basic block.

// Operands are allocated by an ArrayRecycler.
MachineOperand *Operands = nullptr;   // Pointer to the first operand.
unsigned NumOperands = 0;             // Number of operands on instruction.

uint16_t Flags = 0;                   // Various bits of additional
                                      // information about machine
                                      // instruction.

uint8_t AsmPrinterFlags = 0;          // Various bits of information used by
                                      // the AsmPrinter to emit helpful
                                      // comments.  This is *not* semantic
                                      // information.  Do not use this for
                                      // anything other than to convey comment
                                      // information to AsmPrinter.

// OperandCapacity has uint8_t size, so it should be next to AsmPrinterFlags
// to properly pack.
using OperandCapacity = ArrayRecycler<MachineOperand>::Capacity;
OperandCapacity CapOperands;          // Capacity of the Operands array.

/// Internal implementation detail class that provides out-of-line storage for
/// extra info used by the machine instruction when this info cannot be stored
/// in-line within the instruction itself.
///
/// This has to be defined eagerly due to the implementation constraints of
/// `PointerSumType` where it is used.
class ExtraInfo final
    : TrailingObjects<ExtraInfo, MachineMemOperand *, MCSymbol *, MDNode *> {
public:
  static ExtraInfo *create(BumpPtrAllocator &Allocator,
                           ArrayRef<MachineMemOperand *> MMOs,
                           MCSymbol *PreInstrSymbol = nullptr,
                           MCSymbol *PostInstrSymbol = nullptr,
                           MDNode *HeapAllocMarker = nullptr) {
    bool HasPreInstrSymbol = PreInstrSymbol != nullptr;
    bool HasPostInstrSymbol = PostInstrSymbol != nullptr;
    bool HasHeapAllocMarker = HeapAllocMarker != nullptr;
    auto *Result = new (Allocator.Allocate(
        totalSizeToAlloc<MachineMemOperand *, MCSymbol *, MDNode *>(
            MMOs.size(), HasPreInstrSymbol + HasPostInstrSymbol,
            HasHeapAllocMarker),
        alignof(ExtraInfo)))
        ExtraInfo(MMOs.size(), HasPreInstrSymbol, HasPostInstrSymbol,
                  HasHeapAllocMarker);

    // Copy the actual data into the trailing objects.
    std::copy(MMOs.begin(), MMOs.end(),
              Result->getTrailingObjects<MachineMemOperand *>());

    if (HasPreInstrSymbol)
      Result->getTrailingObjects<MCSymbol *>()[0] = PreInstrSymbol;
    if (HasPostInstrSymbol)
      Result->getTrailingObjects<MCSymbol *>()[HasPreInstrSymbol] =
          PostInstrSymbol;
    if (HasHeapAllocMarker)
      Result->getTrailingObjects<MDNode *>()[0] = HeapAllocMarker;

    return Result;
  }

  ArrayRef<MachineMemOperand *> getMMOs() const {
    return makeArrayRef(getTrailingObjects<MachineMemOperand *>(), NumMMOs);
  }

  MCSymbol *getPreInstrSymbol() const {
    return HasPreInstrSymbol ? getTrailingObjects<MCSymbol *>()[0] : nullptr;
  }

  MCSymbol *getPostInstrSymbol() const {
    return HasPostInstrSymbol
               ? getTrailingObjects<MCSymbol *>()[HasPreInstrSymbol]
               : nullptr;
  }

  MDNode *getHeapAllocMarker() const {
    return HasHeapAllocMarker ? getTrailingObjects<MDNode *>()[0] : nullptr;
  }

private:
  friend TrailingObjects;

  // Description of the extra info, used to interpret the actual optional
  // data appended.
  //
  // Note that this is not terribly space optimized. This leaves a great deal
  // of flexibility to fit more in here later.
  const int NumMMOs;
  const bool HasPreInstrSymbol;
  const bool HasPostInstrSymbol;
  const bool HasHeapAllocMarker;

  // Implement the `TrailingObjects` internal API.
  size_t numTrailingObjects(OverloadToken<MachineMemOperand *>) const {
    return NumMMOs;
  }
  size_t numTrailingObjects(OverloadToken<MCSymbol *>) const {
    return HasPreInstrSymbol + HasPostInstrSymbol;
  }
  size_t numTrailingObjects(OverloadToken<MDNode *>) const {
    return HasHeapAllocMarker;
  }

  // Just a boring constructor to allow us to initialize the sizes. Always use
  // the `create` routine above.
  ExtraInfo(int NumMMOs, bool HasPreInstrSymbol, bool HasPostInstrSymbol,
            bool HasHeapAllocMarker)
      : NumMMOs(NumMMOs), HasPreInstrSymbol(HasPreInstrSymbol),
        HasPostInstrSymbol(HasPostInstrSymbol),
        HasHeapAllocMarker(HasHeapAllocMarker) {}
};

/// Enumeration of the kinds of inline extra info available. It is important
/// that the `MachineMemOperand` inline kind has a tag value of zero to make
/// it accessible as an `ArrayRef`.
enum ExtraInfoInlineKinds {
  EIIK_MMO = 0,
  EIIK_PreInstrSymbol,
  EIIK_PostInstrSymbol,
  EIIK_OutOfLine
};

// We store extra information about the instruction here. The common case is
// expected to be nothing or a single pointer (typically a MMO or a symbol).
// We work to optimize this common case by storing it inline here rather than
// requiring a separate allocation, but we fall back to an allocation when
// multiple pointers are needed.
PointerSumType<ExtraInfoInlineKinds,
               PointerSumTypeMember<EIIK_MMO, MachineMemOperand *>,
               PointerSumTypeMember<EIIK_PreInstrSymbol, MCSymbol *>,
               PointerSumTypeMember<EIIK_PostInstrSymbol, MCSymbol *>,
               PointerSumTypeMember<EIIK_OutOfLine, ExtraInfo *>>
    Info;

DebugLoc debugLoc;                    // Source line information.

/// Unique instruction number. Used by DBG_INSTR_REFs to refer to the values
/// defined by this instruction.
unsigned DebugInstrNum;

// Intrusive list support
friend struct ilist_traits<MachineInstr>;
friend struct ilist_callback_traits<MachineBasicBlock>;
void setParent(MachineBasicBlock *P) { Parent = P; }

/// This constructor creates a copy of the given
/// MachineInstr in the given MachineFunction.
MachineInstr(MachineFunction &, const MachineInstr &);

/// This constructor create a MachineInstr and add the implicit operands.
/// It reserves space for number of operands specified by
/// MCInstrDesc.  An explicit DebugLoc is supplied.
MachineInstr(MachineFunction &, const MCInstrDesc &tid, DebugLoc dl,
             bool NoImp = false);

// MachineInstrs are pool-allocated and owned by MachineFunction.
friend class MachineFunction;

void
dumprImpl(const MachineRegisterInfo &MRI, unsigned Depth, unsigned MaxDepth,
          SmallPtrSetImpl<const MachineInstr *> &AlreadySeenInstrs) const;

278public:
MachineInstr(const MachineInstr &) = delete;
MachineInstr &operator=(const MachineInstr &) = delete;
// Use MachineFunction::DeleteMachineInstr() instead.
~MachineInstr() = delete;

const MachineBasicBlock* getParent() const { return Parent; }
MachineBasicBlock* getParent() { return Parent; }

/// Move the instruction before \p MovePos.
void moveBefore(MachineInstr *MovePos);

/// Return the function that contains the basic block that this instruction
/// belongs to.
///
/// Note: this is undefined behaviour if the instruction does not have a
/// parent.
const MachineFunction *getMF() const;
MachineFunction *getMF() {
  return const_cast<MachineFunction *>(
      static_cast<const MachineInstr *>(this)->getMF());
}

/// Return the asm printer flags bitvector.
uint8_t getAsmPrinterFlags() const { return AsmPrinterFlags; }

/// Clear the AsmPrinter bitvector.
void clearAsmPrinterFlags() { AsmPrinterFlags = 0; }

/// Return whether an AsmPrinter flag is set.
bool getAsmPrinterFlag(CommentFlag Flag) const {
  return AsmPrinterFlags & Flag;
}

/// Set a flag for the AsmPrinter.
void setAsmPrinterFlag(uint8_t Flag) {
  AsmPrinterFlags |= Flag;
}

/// Clear specific AsmPrinter flags.
void clearAsmPrinterFlag(CommentFlag Flag) {
  AsmPrinterFlags &= ~Flag;
}

/// Return the MI flags bitvector.
uint16_t getFlags() const {
  return Flags;
}

/// Return whether an MI flag is set.
bool getFlag(MIFlag Flag) const {
  return Flags & Flag;
}

/// Set a MI flag.
void setFlag(MIFlag Flag) {
  Flags |= (uint16_t)Flag;
}

void setFlags(unsigned flags) {
  // Filter out the automatically maintained flags.
  unsigned Mask = BundledPred | BundledSucc;
  Flags = (Flags & Mask) | (flags & ~Mask);
}

/// clearFlag - Clear a MI flag.
void clearFlag(MIFlag Flag) {
  Flags &= ~((uint16_t)Flag);
}

/// Return true if MI is in a bundle (but not the first MI in a bundle).
///
/// A bundle looks like this before it's finalized:
///   ----------------
///   |      MI      |
///   ----------------
///          |
///   ----------------
///   |      MI    * |
///   ----------------
///          |
///   ----------------
///   |      MI    * |
///   ----------------
/// In this case, the first MI starts a bundle but is not inside a bundle, the
/// next 2 MIs are considered "inside" the bundle.
///
/// After a bundle is finalized, it looks like this:
///   ----------------
///   |    Bundle    |
///   ----------------
///          |
///   ----------------
///   |      MI    * |
///   ----------------
///          |
///   ----------------
///   |      MI    * |
///   ----------------
///          |
///   ----------------
///   |      MI    * |
///   ----------------
/// The first instruction has the special opcode "BUNDLE". It's not "inside"
/// a bundle, but the next three MIs are.
bool isInsideBundle() const {
  return getFlag(BundledPred);
}

/// Return true if this instruction part of a bundle. This is true
/// if either itself or its following instruction is marked "InsideBundle".
bool isBundled() const {
  return isBundledWithPred() || isBundledWithSucc();
}

/// Return true if this instruction is part of a bundle, and it is not the
/// first instruction in the bundle.
bool isBundledWithPred() const { return getFlag(BundledPred); }

/// Return true if this instruction is part of a bundle, and it is not the
/// last instruction in the bundle.
bool isBundledWithSucc() const { return getFlag(BundledSucc); }

/// Bundle this instruction with its predecessor. This can be an unbundled
/// instruction, or it can be the first instruction in a bundle.
void bundleWithPred();

/// Bundle this instruction with its successor. This can be an unbundled
/// instruction, or it can be the last instruction in a bundle.
void bundleWithSucc();

/// Break bundle above this instruction.
void unbundleFromPred();

/// Break bundle below this instruction.
void unbundleFromSucc();

/// Returns the debug location id of this MachineInstr.
const DebugLoc &getDebugLoc() const { return debugLoc; }

/// Return the operand containing the offset to be used if this DBG_VALUE
/// instruction is indirect; will be an invalid register if this value is
/// not indirect, and an immediate with value 0 otherwise.
const MachineOperand &getDebugOffset() const {
  assert(isDebugValue() && "not a DBG_VALUE")((isDebugValue() && "not a DBG_VALUE") ? static_cast<
void> (0) : __assert_fail ("isDebugValue() && \"not a DBG_VALUE\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/include/llvm/CodeGen/MachineInstr.h"
, 422, __PRETTY_FUNCTION__));
  return getOperand(1);
}
MachineOperand &getDebugOffset() {
  assert(isDebugValue() && "not a DBG_VALUE")((isDebugValue() && "not a DBG_VALUE") ? static_cast<
void> (0) : __assert_fail ("isDebugValue() && \"not a DBG_VALUE\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/include/llvm/CodeGen/MachineInstr.h"
, 426, __PRETTY_FUNCTION__));
  return getOperand(1);
}

/// Return the operand for the debug variable referenced by
/// this DBG_VALUE instruction.
const MachineOperand &getDebugVariableOp() const;
MachineOperand &getDebugVariableOp();

/// Return the debug variable referenced by
/// this DBG_VALUE instruction.
const DILocalVariable *getDebugVariable() const;

/// Return the operand for the complex address expression referenced by
/// this DBG_VALUE instruction.
MachineOperand &getDebugExpressionOp();

/// Return the complex address expression referenced by
/// this DBG_VALUE instruction.
const DIExpression *getDebugExpression() const;

/// Return the debug label referenced by
/// this DBG_LABEL instruction.
const DILabel *getDebugLabel() const;

/// Fetch the instruction number of this MachineInstr. If it does not have
/// one already, a new and unique number will be assigned.
unsigned getDebugInstrNum();

/// Examine the instruction number of this MachineInstr. May be zero if
/// it hasn't been assigned a number yet.
unsigned peekDebugInstrNum() const { return DebugInstrNum; }

/// Set instruction number of this MachineInstr. Avoid using unless you're
/// deserializing this information.
void setDebugInstrNum(unsigned Num) { DebugInstrNum = Num; }

/// Emit an error referring to the source location of this instruction.
/// This should only be used for inline assembly that is somehow
/// impossible to compile. Other errors should have been handled much
/// earlier.
///
/// If this method returns, the caller should try to recover from the error.
void emitError(StringRef Msg) const;

/// Returns the target instruction descriptor of this MachineInstr.
const MCInstrDesc &getDesc() const { return *MCID; }

/// Returns the opcode of this MachineInstr.
unsigned getOpcode() const { return MCID->Opcode; }

/// Retuns the total number of operands.
unsigned getNumOperands() const { return NumOperands; }

/// Returns the total number of operands which are debug locations.
unsigned getNumDebugOperands() const {
  return std::distance(debug_operands().begin(), debug_operands().end());
}

const MachineOperand& getOperand(unsigned i) const {
  assert(i < getNumOperands() && "getOperand() out of range!")((i < getNumOperands() && "getOperand() out of range!"
) ? static_cast<void> (0) : __assert_fail ("i < getNumOperands() && \"getOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/include/llvm/CodeGen/MachineInstr.h"
, 486, __PRETTY_FUNCTION__));
  return Operands[i];
}
MachineOperand& getOperand(unsigned i) {
  assert(i < getNumOperands() && "getOperand() out of range!")((i < getNumOperands() && "getOperand() out of range!"
) ? static_cast<void> (0) : __assert_fail ("i < getNumOperands() && \"getOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/include/llvm/CodeGen/MachineInstr.h"
, 490, __PRETTY_FUNCTION__));
  return Operands[i];
}

MachineOperand &getDebugOperand(unsigned Index) {
  assert(Index < getNumDebugOperands() && "getDebugOperand() out of range!")((Index < getNumDebugOperands() && "getDebugOperand() out of range!"
) ? static_cast<void> (0) : __assert_fail ("Index < getNumDebugOperands() && \"getDebugOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/include/llvm/CodeGen/MachineInstr.h"
, 495, __PRETTY_FUNCTION__));
  return *(debug_operands().begin() + Index);
}
const MachineOperand &getDebugOperand(unsigned Index) const {
  assert(Index < getNumDebugOperands() && "getDebugOperand() out of range!")((Index < getNumDebugOperands() && "getDebugOperand() out of range!"
) ? static_cast<void> (0) : __assert_fail ("Index < getNumDebugOperands() && \"getDebugOperand() out of range!\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/include/llvm/CodeGen/MachineInstr.h"
, 499, __PRETTY_FUNCTION__));
  return *(debug_operands().begin() + Index);
}

/// Returns a pointer to the operand corresponding to a debug use of Reg, or
/// nullptr if Reg is not used in any debug operand.
const MachineOperand *getDebugOperandForReg(Register Reg) const {
  const MachineOperand *RegOp =
      find_if(debug_operands(), [Reg](const MachineOperand &Op) {
        return Op.isReg() && Op.getReg() == Reg;
      });
  return RegOp == adl_end(debug_operands()) ? nullptr : RegOp;
}
MachineOperand *getDebugOperandForReg(Register Reg) {
  MachineOperand *RegOp =
      find_if(debug_operands(), [Reg](const MachineOperand &Op) {
        return Op.isReg() && Op.getReg() == Reg;
      });
  return RegOp == adl_end(debug_operands()) ? nullptr : RegOp;
}

unsigned getDebugOperandIndex(const MachineOperand *Op) const {
  assert(Op >= adl_begin(debug_operands()) &&((Op >= adl_begin(debug_operands()) && Op <= adl_end
(debug_operands()) && "Expected a debug operand.") ? static_cast
<void> (0) : __assert_fail ("Op >= adl_begin(debug_operands()) && Op <= adl_end(debug_operands()) && \"Expected a debug operand.\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/include/llvm/CodeGen/MachineInstr.h"
, 522, __PRETTY_FUNCTION__))
         Op <= adl_end(debug_operands()) && "Expected a debug operand.")((Op >= adl_begin(debug_operands()) && Op <= adl_end
(debug_operands()) && "Expected a debug operand.") ? static_cast
<void> (0) : __assert_fail ("Op >= adl_begin(debug_operands()) && Op <= adl_end(debug_operands()) && \"Expected a debug operand.\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/include/llvm/CodeGen/MachineInstr.h"
, 522, __PRETTY_FUNCTION__));
  return std::distance(adl_begin(debug_operands()), Op);
}

/// Returns the total number of definitions.
unsigned getNumDefs() const {
  return getNumExplicitDefs() + MCID->getNumImplicitDefs();
}

/// Returns true if the instruction has implicit definition.
bool hasImplicitDef() const {
  for (unsigned I = getNumExplicitOperands(), E = getNumOperands();
    I != E; ++I) {
    const MachineOperand &MO = getOperand(I);
    if (MO.isDef() && MO.isImplicit())
      return true;
  }
  return false;
}

/// Returns the implicit operands number.
unsigned getNumImplicitOperands() const {
  return getNumOperands() - getNumExplicitOperands();
}

/// Return true if operand \p OpIdx is a subregister index.
bool isOperandSubregIdx(unsigned OpIdx) const {
  assert(getOperand(OpIdx).getType() == MachineOperand::MO_Immediate &&((getOperand(OpIdx).getType() == MachineOperand::MO_Immediate
 && "Expected MO_Immediate operand type.") ? static_cast
<void> (0) : __assert_fail ("getOperand(OpIdx).getType() == MachineOperand::MO_Immediate && \"Expected MO_Immediate operand type.\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/include/llvm/CodeGen/MachineInstr.h"
, 550, __PRETTY_FUNCTION__))
         "Expected MO_Immediate operand type.")((getOperand(OpIdx).getType() == MachineOperand::MO_Immediate
 && "Expected MO_Immediate operand type.") ? static_cast
<void> (0) : __assert_fail ("getOperand(OpIdx).getType() == MachineOperand::MO_Immediate && \"Expected MO_Immediate operand type.\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/include/llvm/CodeGen/MachineInstr.h"
, 550, __PRETTY_FUNCTION__));
  if (isExtractSubreg() && OpIdx == 2)
    return true;
  if (isInsertSubreg() && OpIdx == 3)
    return true;
  if (isRegSequence() && OpIdx > 1 && (OpIdx % 2) == 0)
    return true;
  if (isSubregToReg() && OpIdx == 3)
    return true;
  return false;
}

/// Returns the number of non-implicit operands.
unsigned getNumExplicitOperands() const;

/// Returns the number of non-implicit definitions.
unsigned getNumExplicitDefs() const;

/// iterator/begin/end - Iterate over all operands of a machine instruction.
using mop_iterator = MachineOperand *;
using const_mop_iterator = const MachineOperand *;

mop_iterator operands_begin() { return Operands; }
mop_iterator operands_end() { return Operands + NumOperands; }

const_mop_iterator operands_begin() const { return Operands; }
const_mop_iterator operands_end() const { return Operands + NumOperands; }

iterator_range<mop_iterator> operands() {
  return make_range(operands_begin(), operands_end());
}
iterator_range<const_mop_iterator> operands() const {
  return make_range(operands_begin(), operands_end());
}
iterator_range<mop_iterator> explicit_operands() {
  return make_range(operands_begin(),
                    operands_begin() + getNumExplicitOperands());
}
iterator_range<const_mop_iterator> explicit_operands() const {
  return make_range(operands_begin(),
                    operands_begin() + getNumExplicitOperands());
}
iterator_range<mop_iterator> implicit_operands() {
  return make_range(explicit_operands().end(), operands_end());
}
iterator_range<const_mop_iterator> implicit_operands() const {
  return make_range(explicit_operands().end(), operands_end());
}
/// Returns a range over all operands that are used to determine the variable
/// location for this DBG_VALUE instruction.
iterator_range<mop_iterator> debug_operands() {
  assert(isDebugValue() && "Must be a debug value instruction.")((isDebugValue() && "Must be a debug value instruction."
) ? static_cast<void> (0) : __assert_fail ("isDebugValue() && \"Must be a debug value instruction.\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/include/llvm/CodeGen/MachineInstr.h"
, 601, __PRETTY_FUNCTION__));
  return make_range(operands_begin(), operands_begin() + 1);
}
/// \copydoc debug_operands()
iterator_range<const_mop_iterator> debug_operands() const {
  assert(isDebugValue() && "Must be a debug value instruction.")((isDebugValue() && "Must be a debug value instruction."
) ? static_cast<void> (0) : __assert_fail ("isDebugValue() && \"Must be a debug value instruction.\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/include/llvm/CodeGen/MachineInstr.h"
, 606, __PRETTY_FUNCTION__));
  return make_range(operands_begin(), operands_begin() + 1);
}
/// Returns a range over all explicit operands that are register definitions.
/// Implicit definition are not included!
iterator_range<mop_iterator> defs() {
  return make_range(operands_begin(),
                    operands_begin() + getNumExplicitDefs());
}
/// \copydoc defs()
iterator_range<const_mop_iterator> defs() const {
  return make_range(operands_begin(),
                    operands_begin() + getNumExplicitDefs());
}
/// Returns a range that includes all operands that are register uses.
/// This may include unrelated operands which are not register uses.
iterator_range<mop_iterator> uses() {
  return make_range(operands_begin() + getNumExplicitDefs(), operands_end());
}
/// \copydoc uses()
iterator_range<const_mop_iterator> uses() const {
  return make_range(operands_begin() + getNumExplicitDefs(), operands_end());
}
iterator_range<mop_iterator> explicit_uses() {
  return make_range(operands_begin() + getNumExplicitDefs(),
                    operands_begin() + getNumExplicitOperands());
}
iterator_range<const_mop_iterator> explicit_uses() const {
  return make_range(operands_begin() + getNumExplicitDefs(),
                    operands_begin() + getNumExplicitOperands());
}

/// Returns the number of the operand iterator \p I points to.
unsigned getOperandNo(const_mop_iterator I) const {
  return I - operands_begin();
}

/// Access to memory operands of the instruction. If there are none, that does
/// not imply anything about whether the function accesses memory. Instead,
/// the caller must behave conservatively.
ArrayRef<MachineMemOperand *> memoperands() const {
  if (!Info)
    return {};

  if (Info.is<EIIK_MMO>())
    return makeArrayRef(Info.getAddrOfZeroTagPointer(), 1);

  if (ExtraInfo *EI = Info.get<EIIK_OutOfLine>())
    return EI->getMMOs();

  return {};
}

/// Access to memory operands of the instruction.
///
/// If `memoperands_begin() == memoperands_end()`, that does not imply
/// anything about whether the function accesses memory. Instead, the caller
/// must behave conservatively.
mmo_iterator memoperands_begin() const { return memoperands().begin(); }

/// Access to memory operands of the instruction.
///
/// If `memoperands_begin() == memoperands_end()`, that does not imply
/// anything about whether the function accesses memory. Instead, the caller
/// must behave conservatively.
mmo_iterator memoperands_end() const { return memoperands().end(); }

/// Return true if we don't have any memory operands which described the
/// memory access done by this instruction.  If this is true, calling code
/// must be conservative.
bool memoperands_empty() const { return memoperands().empty(); }

/// Return true if this instruction has exactly one MachineMemOperand.
bool hasOneMemOperand() const { return memoperands().size() == 1; }

/// Return the number of memory operands.
unsigned getNumMemOperands() const { return memoperands().size(); }

/// Helper to extract a pre-instruction symbol if one has been added.
MCSymbol *getPreInstrSymbol() const {
  if (!Info)
    return nullptr;
  if (MCSymbol *S = Info.get<EIIK_PreInstrSymbol>())
    return S;
  if (ExtraInfo *EI = Info.get<EIIK_OutOfLine>())
    return EI->getPreInstrSymbol();

  return nullptr;
}

/// Helper to extract a post-instruction symbol if one has been added.
MCSymbol *getPostInstrSymbol() const {
  if (!Info)
    return nullptr;
  if (MCSymbol *S = Info.get<EIIK_PostInstrSymbol>())
    return S;
  if (ExtraInfo *EI = Info.get<EIIK_OutOfLine>())
    return EI->getPostInstrSymbol();

  return nullptr;
}

/// Helper to extract a heap alloc marker if one has been added.
MDNode *getHeapAllocMarker() const {
  if (!Info)
    return nullptr;
  if (ExtraInfo *EI = Info.get<EIIK_OutOfLine>())
    return EI->getHeapAllocMarker();

  return nullptr;
}

/// API for querying MachineInstr properties. They are the same as MCInstrDesc
/// queries but they are bundle aware.

enum QueryType {
  IgnoreBundle,    // Ignore bundles
  AnyInBundle,     // Return true if any instruction in bundle has property
  AllInBundle      // Return true if all instructions in bundle have property
};

/// Return true if the instruction (or in the case of a bundle,
/// the instructions inside the bundle) has the specified property.
/// The first argument is the property being queried.
/// The second argument indicates whether the query should look inside
/// instruction bundles.
bool hasProperty(unsigned MCFlag, QueryType Type = AnyInBundle) const {
  assert(MCFlag < 64 &&((MCFlag < 64 && "MCFlag out of range for bit mask in getFlags/hasPropertyInBundle."
) ? static_cast<void> (0) : __assert_fail ("MCFlag < 64 && \"MCFlag out of range for bit mask in getFlags/hasPropertyInBundle.\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/include/llvm/CodeGen/MachineInstr.h"
, 734, __PRETTY_FUNCTION__))
         "MCFlag out of range for bit mask in getFlags/hasPropertyInBundle.")((MCFlag < 64 && "MCFlag out of range for bit mask in getFlags/hasPropertyInBundle."
) ? static_cast<void> (0) : __assert_fail ("MCFlag < 64 && \"MCFlag out of range for bit mask in getFlags/hasPropertyInBundle.\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/include/llvm/CodeGen/MachineInstr.h"
, 734, __PRETTY_FUNCTION__));
  // Inline the fast path for unbundled or bundle-internal instructions.
  if (Type == IgnoreBundle || !isBundled() || isBundledWithPred())
    return getDesc().getFlags() & (1ULL << MCFlag);

  // If this is the first instruction in a bundle, take the slow path.
  return hasPropertyInBundle(1ULL << MCFlag, Type);
}

/// Return true if this is an instruction that should go through the usual
/// legalization steps.
bool isPreISelOpcode(QueryType Type = IgnoreBundle) const {
  return hasProperty(MCID::PreISelOpcode, Type);
}

/// Return true if this instruction can have a variable number of operands.
/// In this case, the variable operands will be after the normal
/// operands but before the implicit definitions and uses (if any are
/// present).
bool isVariadic(QueryType Type = IgnoreBundle) const {
  return hasProperty(MCID::Variadic, Type);
}

/// Set if this instruction has an optional definition, e.g.
/// ARM instructions which can set condition code if 's' bit is set.
bool hasOptionalDef(QueryType Type = IgnoreBundle) const {
  return hasProperty(MCID::HasOptionalDef, Type);
}

/// Return true if this is a pseudo instruction that doesn't
/// correspond to a real machine instruction.
bool isPseudo(QueryType Type = IgnoreBundle) const {
  return hasProperty(MCID::Pseudo, Type);
}

bool isReturn(QueryType Type = AnyInBundle) const {
  return hasProperty(MCID::Return, Type);
}

/// Return true if this is an instruction that marks the end of an EH scope,
/// i.e., a catchpad or a cleanuppad instruction.
bool isEHScopeReturn(QueryType Type = AnyInBundle) const {
  return hasProperty(MCID::EHScopeReturn, Type);
}

bool isCall(QueryType Type = AnyInBundle) const {
  return hasProperty(MCID::Call, Type);
}

/// Return true if this is a call instruction that may have an associated
/// call site entry in the debug info.
bool isCandidateForCallSiteEntry(QueryType Type = IgnoreBundle) const;
/// Return true if copying, moving, or erasing this instruction requires
/// updating Call Site Info (see \ref copyCallSiteInfo, \ref moveCallSiteInfo,
/// \ref eraseCallSiteInfo).
bool shouldUpdateCallSiteInfo() const;

/// Returns true if the specified instruction stops control flow
/// from executing the instruction immediately following it.  Examples include
/// unconditional branches and return instructions.
bool isBarrier(QueryType Type = AnyInBundle) const {
  return hasProperty(MCID::Barrier, Type);
}

/// Returns true if this instruction part of the terminator for a basic block.
/// Typically this is things like return and branch instructions.
///
/// Various passes use this to insert code into the bottom of a basic block,
/// but before control flow occurs.
bool isTerminator(QueryType Type = AnyInBundle) const {
  return hasProperty(MCID::Terminator, Type);
}

/// Returns true if this is a conditional, unconditional, or indirect branch.
/// Predicates below can be used to discriminate between
/// these cases, and the TargetInstrInfo::analyzeBranch method can be used to
/// get more information.
bool isBranch(QueryType Type = AnyInBundle) const {
  return hasProperty(MCID::Branch, Type);
}

/// Return true if this is an indirect branch, such as a
/// branch through a register.
bool isIndirectBranch(QueryType Type = AnyInBundle) const {
  return hasProperty(MCID::IndirectBranch, Type);
}

/// Return true if this is a branch which may fall
/// through to the next instruction or may transfer control flow to some other
/// block.  The TargetInstrInfo::analyzeBranch method can be used to get more
/// information about this branch.
bool isConditionalBranch(QueryType Type = AnyInBundle) const {
  return isBranch(Type) && !isBarrier(Type) && !isIndirectBranch(Type);
}

/// Return true if this is a branch which always
/// transfers control flow to some other block.  The
/// TargetInstrInfo::analyzeBranch method can be used to get more information
/// about this branch.
bool isUnconditionalBranch(QueryType Type = AnyInBundle) const {
  return isBranch(Type) && isBarrier(Type) && !isIndirectBranch(Type);
}

/// Return true if this instruction has a predicate operand that
/// controls execution.  It may be set to 'always', or may be set to other
/// values.   There are various methods in TargetInstrInfo that can be used to
/// control and modify the predicate in this instruction.
bool isPredicable(QueryType Type = AllInBundle) const {
  // If it's a bundle than all bundled instructions must be predicable for this
  // to return true.
  return hasProperty(MCID::Predicable, Type);
}

/// Return true if this instruction is a comparison.
bool isCompare(QueryType Type = IgnoreBundle) const {
  return hasProperty(MCID::Compare, Type);
}

/// Return true if this instruction is a move immediate
/// (including conditional moves) instruction.
bool isMoveImmediate(QueryType Type = IgnoreBundle) const {
  return hasProperty(MCID::MoveImm, Type);
}

/// Return true if this instruction is a register move.
/// (including moving values from subreg to reg)
bool isMoveReg(QueryType Type = IgnoreBundle) const {
  return hasProperty(MCID::MoveReg, Type);
}

/// Return true if this instruction is a bitcast instruction.
bool isBitcast(QueryType Type = IgnoreBundle) const {
  return hasProperty(MCID::Bitcast, Type);
}

/// Return true if this instruction is a select instruction.
bool isSelect(QueryType Type = IgnoreBundle) const {
  return hasProperty(MCID::Select, Type);
}

/// Return true if this instruction cannot be safely duplicated.
/// For example, if the instruction has a unique labels attached
/// to it, duplicating it would cause multiple definition errors.
bool isNotDuplicable(QueryType Type = AnyInBundle) const {
  return hasProperty(MCID::NotDuplicable, Type);
}

/// Return true if this instruction is convergent.
/// Convergent instructions can not be made control-dependent on any
/// additional values.
bool isConvergent(QueryType Type = AnyInBundle) const {
  if (isInlineAsm()) {
    unsigned ExtraInfo = getOperand(InlineAsm::MIOp_ExtraInfo).getImm();
    if (ExtraInfo & InlineAsm::Extra_IsConvergent)
      return true;
  }
  return hasProperty(MCID::Convergent, Type);
}

/// Returns true if the specified instruction has a delay slot
/// which must be filled by the code generator.
bool hasDelaySlot(QueryType Type = AnyInBundle) const {
  return hasProperty(MCID::DelaySlot, Type);
}

/// Return true for instructions that can be folded as
/// memory operands in other instructions. The most common use for this
/// is instructions that are simple loads from memory that don't modify
/// the loaded value in any way, but it can also be used for instructions
/// that can be expressed as constant-pool loads, such as V_SETALLONES
/// on x86, to allow them to be folded when it is beneficial.
/// This should only be set on instructions that return a value in their
/// only virtual register definition.
bool canFoldAsLoad(QueryType Type = IgnoreBundle) const {
  return hasProperty(MCID::FoldableAsLoad, Type);
}

/// Return true if this instruction behaves
/// the same way as the generic REG_SEQUENCE instructions.
/// E.g., on ARM,
/// dX VMOVDRR rY, rZ
/// is equivalent to
/// dX = REG_SEQUENCE rY, ssub_0, rZ, ssub_1.
///
/// Note that for the optimizers to be able to take advantage of
/// this property, TargetInstrInfo::getRegSequenceLikeInputs has to be
/// override accordingly.
bool isRegSequenceLike(QueryType Type = IgnoreBundle) const {
  return hasProperty(MCID::RegSequence, Type);
}

/// Return true if this instruction behaves
/// the same way as the generic EXTRACT_SUBREG instructions.
/// E.g., on ARM,
/// rX, rY VMOVRRD dZ
/// is equivalent to two EXTRACT_SUBREG:
/// rX = EXTRACT_SUBREG dZ, ssub_0
/// rY = EXTRACT_SUBREG dZ, ssub_1
///
/// Note that for the optimizers to be able to take advantage of
/// this property, TargetInstrInfo::getExtractSubregLikeInputs has to be
/// override accordingly.
bool isExtractSubregLike(QueryType Type = IgnoreBundle) const {
  return hasProperty(MCID::ExtractSubreg, Type);
}

/// Return true if this instruction behaves
/// the same way as the generic INSERT_SUBREG instructions.
/// E.g., on ARM,
/// dX = VSETLNi32 dY, rZ, Imm
/// is equivalent to a INSERT_SUBREG:
/// dX = INSERT_SUBREG dY, rZ, translateImmToSubIdx(Imm)
///
/// Note that for the optimizers to be able to take advantage of
/// this property, TargetInstrInfo::getInsertSubregLikeInputs has to be
/// override accordingly.
bool isInsertSubregLike(QueryType Type = IgnoreBundle) const {
  return hasProperty(MCID::InsertSubreg, Type);
}

//===--------------------------------------------------------------------===//
// Side Effect Analysis
//===--------------------------------------------------------------------===//

/// Return true if this instruction could possibly read memory.
/// Instructions with this flag set are not necessarily simple load
/// instructions, they may load a value and modify it, for example.
bool mayLoad(QueryType Type = AnyInBundle) const {
  if (isInlineAsm()) {
    unsigned ExtraInfo = getOperand(InlineAsm::MIOp_ExtraInfo).getImm();
    if (ExtraInfo & InlineAsm::Extra_MayLoad)
      return true;
  }
  return hasProperty(MCID::MayLoad, Type);
}

/// Return true if this instruction could possibly modify memory.
/// Instructions with this flag set are not necessarily simple store
/// instructions, they may store a modified value based on their operands, or
/// may not actually modify anything, for example.
bool mayStore(QueryType Type = AnyInBundle) const {
  if (isInlineAsm()) {
    unsigned ExtraInfo = getOperand(InlineAsm::MIOp_ExtraInfo).getImm();
    if (ExtraInfo & InlineAsm::Extra_MayStore)
      return true;
  }
  return hasProperty(MCID::MayStore, Type);
}

/// Return true if this instruction could possibly read or modify memory.
bool mayLoadOrStore(QueryType Type = AnyInBundle) const {
  return mayLoad(Type) || mayStore(Type);
}

/// Return true if this instruction could possibly raise a floating-point
/// exception.  This is the case if the instruction is a floating-point
/// instruction that can in principle raise an exception, as indicated
/// by the MCID::MayRaiseFPException property, *and* at the same time,
/// the instruction is used in a context where we expect floating-point
/// exceptions are not disabled, as indicated by the NoFPExcept MI flag.
bool mayRaiseFPException() const {
  return hasProperty(MCID::MayRaiseFPException) &&
         !getFlag(MachineInstr::MIFlag::NoFPExcept);
}

//===--------------------------------------------------------------------===//
// Flags that indicate whether an instruction can be modified by a method.
//===--------------------------------------------------------------------===//

/// Return true if this may be a 2- or 3-address
/// instruction (of the form "X = op Y, Z, ..."), which produces the same
/// result if Y and Z are exchanged.  If this flag is set, then the
/// TargetInstrInfo::commuteInstruction method may be used to hack on the
/// instruction.
///
/// Note that this flag may be set on instructions that are only commutable
/// sometimes.  In these cases, the call to commuteInstruction will fail.
/// Also note that some instructions require non-trivial modification to
/// commute them.
bool isCommutable(QueryType Type = IgnoreBundle) const {
  return hasProperty(MCID::Commutable, Type);
}

/// Return true if this is a 2-address instruction
/// which can be changed into a 3-address instruction if needed.  Doing this
/// transformation can be profitable in the register allocator, because it
/// means that the instruction can use a 2-address form if possible, but
/// degrade into a less efficient form if the source and dest register cannot
/// be assigned to the same register.  For example, this allows the x86
/// backend to turn a "shl reg, 3" instruction into an LEA instruction, which
/// is the same speed as the shift but has bigger code size.
///
/// If this returns true, then the target must implement the
/// TargetInstrInfo::convertToThreeAddress method for this instruction, which
/// is allowed to fail if the transformation isn't valid for this specific
/// instruction (e.g. shl reg, 4 on x86).
///
bool isConvertibleTo3Addr(QueryType Type = IgnoreBundle) const {
  return hasProperty(MCID::ConvertibleTo3Addr, Type);
}

/// Return true if this instruction requires
/// custom insertion support when the DAG scheduler is inserting it into a
/// machine basic block.  If this is true for the instruction, it basically
/// means that it is a pseudo instruction used at SelectionDAG time that is
/// expanded out into magic code by the target when MachineInstrs are formed.
///
/// If this is true, the TargetLoweringInfo::InsertAtEndOfBasicBlock method
/// is used to insert this into the MachineBasicBlock.
bool usesCustomInsertionHook(QueryType Type = IgnoreBundle) const {
  return hasProperty(MCID::UsesCustomInserter, Type);
}

/// Return true if this instruction requires *adjustment*
/// after instruction selection by calling a target hook. For example, this
/// can be used to fill in ARM 's' optional operand depending on whether
/// the conditional flag register is used.
bool hasPostISelHook(QueryType Type = IgnoreBundle) const {
  return hasProperty(MCID::HasPostISelHook, Type);
}

/// Returns true if this instruction is a candidate for remat.
/// This flag is deprecated, please don't use it anymore.  If this
/// flag is set, the isReallyTriviallyReMaterializable() method is called to
/// verify the instruction is really rematable.
bool isRematerializable(QueryType Type = AllInBundle) const {
  // It's only possible to re-mat a bundle if all bundled instructions are
  // re-materializable.
  return hasProperty(MCID::Rematerializable, Type);
}

/// Returns true if this instruction has the same cost (or less) than a move
/// instruction. This is useful during certain types of optimizations
/// (e.g., remat during two-address conversion or machine licm)
/// where we would like to remat or hoist the instruction, but not if it costs
/// more than moving the instruction into the appropriate register. Note, we
/// are not marking copies from and to the same register class with this flag.
bool isAsCheapAsAMove(QueryType Type = AllInBundle) const {
  // Only returns true for a bundle if all bundled instructions are cheap.
  return hasProperty(MCID::CheapAsAMove, Type);
}

/// Returns true if this instruction source operands
/// have special register allocation requirements that are not captured by the
/// operand register classes. e.g. ARM::STRD's two source registers must be an
/// even / odd pair, ARM::STM registers have to be in ascending order.
/// Post-register allocation passes should not attempt to change allocations
/// for sources of instructions with this flag.
bool hasExtraSrcRegAllocReq(QueryType Type = AnyInBundle) const {
  return hasProperty(MCID::ExtraSrcRegAllocReq, Type);
}

/// Returns true if this instruction def operands
/// have special register allocation requirements that are not captured by the
/// operand register classes. e.g. ARM::LDRD's two def registers must be an
/// even / odd pair, ARM::LDM registers have to be in ascending order.
/// Post-register allocation passes should not attempt to change allocations
/// for definitions of instructions with this flag.
bool hasExtraDefRegAllocReq(QueryType Type = AnyInBundle) const {
  return hasProperty(MCID::ExtraDefRegAllocReq, Type);
}

enum MICheckType {
  CheckDefs,      // Check all operands for equality
  CheckKillDead,  // Check all operands including kill / dead markers
  IgnoreDefs,     // Ignore all definitions
  IgnoreVRegDefs  // Ignore virtual register definitions
};

/// Return true if this instruction is identical to \p Other.
/// Two instructions are identical if they have the same opcode and all their
/// operands are identical (with respect to MachineOperand::isIdenticalTo()).
/// Note that this means liveness related flags (dead, undef, kill) do not
/// affect the notion of identical.
bool isIdenticalTo(const MachineInstr &Other,
                   MICheckType Check = CheckDefs) const;

/// Unlink 'this' from the containing basic block, and return it without
/// deleting it.
///
/// This function can not be used on bundled instructions, use
/// removeFromBundle() to remove individual instructions from a bundle.
MachineInstr *removeFromParent();

/// Unlink this instruction from its basic block and return it without
/// deleting it.
///
/// If the instruction is part of a bundle, the other instructions in the
/// bundle remain bundled.
MachineInstr *removeFromBundle();

/// Unlink 'this' from the containing basic block and delete it.
///
/// If this instruction is the header of a bundle, the whole bundle is erased.
/// This function can not be used for instructions inside a bundle, use
/// eraseFromBundle() to erase individual bundled instructions.
void eraseFromParent();

/// Unlink 'this' from the containing basic block and delete it.
///
/// For all definitions mark their uses in DBG_VALUE nodes
/// as undefined. Otherwise like eraseFromParent().
void eraseFromParentAndMarkDBGValuesForRemoval();

/// Unlink 'this' form its basic block and delete it.
///
/// If the instruction is part of a bundle, the other instructions in the
/// bundle remain bundled.
void eraseFromBundle();

bool isEHLabel() const { return getOpcode() == TargetOpcode::EH_LABEL; }
bool isGCLabel() const { return getOpcode() == TargetOpcode::GC_LABEL; }
bool isAnnotationLabel() const {
  return getOpcode() == TargetOpcode::ANNOTATION_LABEL;
}

/// Returns true if the MachineInstr represents a label.
bool isLabel() const {
  return isEHLabel() || isGCLabel() || isAnnotationLabel();
}

bool isCFIInstruction() const {
  return getOpcode() == TargetOpcode::CFI_INSTRUCTION;
}

// True if the instruction represents a position in the function.
bool isPosition() const { return isLabel() || isCFIInstruction(); }

bool isDebugValue() const { return getOpcode() == TargetOpcode::DBG_VALUE; }
bool isDebugLabel() const { return getOpcode() == TargetOpcode::DBG_LABEL; }
bool isDebugRef() const { return getOpcode() == TargetOpcode::DBG_INSTR_REF; }
bool isDebugInstr() const {
  return isDebugValue() || isDebugLabel() || isDebugRef();
}

bool isDebugOffsetImm() const { return getDebugOffset().isImm(); }

/// A DBG_VALUE is indirect iff the location operand is a register and
/// the offset operand is an immediate.
bool isIndirectDebugValue() const {
  return isDebugValue() && getDebugOperand(0).isReg() && isDebugOffsetImm();
}

/// A DBG_VALUE is an entry value iff its debug expression contains the
/// DW_OP_LLVM_entry_value operation.
bool isDebugEntryValue() const;

/// Return true if the instruction is a debug value which describes a part of
/// a variable as unavailable.
bool isUndefDebugValue() const {
  return isDebugValue() && getDebugOperand(0).isReg() &&
         !getDebugOperand(0).getReg().isValid();
}

bool isPHI() const {
  return getOpcode() == TargetOpcode::PHI ||
         getOpcode() == TargetOpcode::G_PHI;
}
bool isKill() const { return getOpcode() == TargetOpcode::KILL; }
bool isImplicitDef() const { return getOpcode()==TargetOpcode::IMPLICIT_DEF; }
bool isInlineAsm() const {
  return getOpcode() == TargetOpcode::INLINEASM ||
         getOpcode() == TargetOpcode::INLINEASM_BR;
}

/// FIXME: Seems like a layering violation that the AsmDialect, which is X86
/// specific, be attached to a generic MachineInstr.
bool isMSInlineAsm() const {
  return isInlineAsm() && getInlineAsmDialect() == InlineAsm::AD_Intel;
}

bool isStackAligningInlineAsm() const;
InlineAsm::AsmDialect getInlineAsmDialect() const;

bool isInsertSubreg() const {
  return getOpcode() == TargetOpcode::INSERT_SUBREG;
}

bool isSubregToReg() const {
  return getOpcode() == TargetOpcode::SUBREG_TO_REG;
}

bool isRegSequence() const {
  return getOpcode() == TargetOpcode::REG_SEQUENCE;
}

bool isBundle() const {
  return getOpcode() == TargetOpcode::BUNDLE;
}

bool isCopy() const {
  return getOpcode() == TargetOpcode::COPY;
43
←
Assuming the condition is true→
44
←
Returning the value 1, which participates in a condition later→
}

bool isFullCopy() const {
  return isCopy() && !getOperand(0).getSubReg() && !getOperand(1).getSubReg();
}

bool isExtractSubreg() const {
  return getOpcode() == TargetOpcode::EXTRACT_SUBREG;
}

/// Return true if the instruction behaves like a copy.
/// This does not include native copy instructions.
bool isCopyLike() const {
  return isCopy() || isSubregToReg();
}

/// Return true is the instruction is an identity copy.
bool isIdentityCopy() const {
  return isCopy() && getOperand(0).getReg() == getOperand(1).getReg() &&
    getOperand(0).getSubReg() == getOperand(1).getSubReg();
}

/// Return true if this instruction doesn't produce any output in the form of
/// executable instructions.
bool isMetaInstruction() const {
  switch (getOpcode()) {
  default:
    return false;
  case TargetOpcode::IMPLICIT_DEF:
  case TargetOpcode::KILL:
  case TargetOpcode::CFI_INSTRUCTION:
  case TargetOpcode::EH_LABEL:
  case TargetOpcode::GC_LABEL:
  case TargetOpcode::DBG_VALUE:
  case TargetOpcode::DBG_INSTR_REF:
  case TargetOpcode::DBG_LABEL:
  case TargetOpcode::LIFETIME_START:
  case TargetOpcode::LIFETIME_END:
  case TargetOpcode::PSEUDO_PROBE:
    return true;
  }
}

/// Return true if this is a transient instruction that is either very likely
/// to be eliminated during register allocation (such as copy-like
/// instructions), or if this instruction doesn't have an execution-time cost.
bool isTransient() const {
  switch (getOpcode()) {
  default:
    return isMetaInstruction();
  // Copy-like instructions are usually eliminated during register allocation.
  case TargetOpcode::PHI:
  case TargetOpcode::G_PHI:
  case TargetOpcode::COPY:
  case TargetOpcode::INSERT_SUBREG:
  case TargetOpcode::SUBREG_TO_REG:
  case TargetOpcode::REG_SEQUENCE:
    return true;
  }
}

/// Return the number of instructions inside the MI bundle, excluding the
/// bundle header.
///
/// This is the number of instructions that MachineBasicBlock::iterator
/// skips, 0 for unbundled instructions.
unsigned getBundleSize() const;

/// Return true if the MachineInstr reads the specified register.
/// If TargetRegisterInfo is passed, then it also checks if there
/// is a read of a super-register.
/// This does not count partial redefines of virtual registers as reads:
///   %reg1024:6 = OP.
bool readsRegister(Register Reg,
                   const TargetRegisterInfo *TRI = nullptr) const {
  return findRegisterUseOperandIdx(Reg, false, TRI) != -1;
}

/// Return true if the MachineInstr reads the specified virtual register.
/// Take into account that a partial define is a
/// read-modify-write operation.
bool readsVirtualRegister(Register Reg) const {
  return readsWritesVirtualRegister(Reg).first;
}

/// Return a pair of bools (reads, writes) indicating if this instruction
/// reads or writes Reg. This also considers partial defines.
/// If Ops is not null, all operand indices for Reg are added.
std::pair<bool,bool> readsWritesVirtualRegister(Register Reg,
                              SmallVectorImpl<unsigned> *Ops = nullptr) const;

/// Return true if the MachineInstr kills the specified register.
/// If TargetRegisterInfo is passed, then it also checks if there is
/// a kill of a super-register.
bool killsRegister(Register Reg,
                   const TargetRegisterInfo *TRI = nullptr) const {
  return findRegisterUseOperandIdx(Reg, true, TRI) != -1;
}

/// Return true if the MachineInstr fully defines the specified register.
/// If TargetRegisterInfo is passed, then it also checks
/// if there is a def of a super-register.
/// NOTE: It's ignoring subreg indices on virtual registers.
bool definesRegister(Register Reg,
                     const TargetRegisterInfo *TRI = nullptr) const {
  return findRegisterDefOperandIdx(Reg, false, false, TRI) != -1;
}

/// Return true if the MachineInstr modifies (fully define or partially
/// define) the specified register.
/// NOTE: It's ignoring subreg indices on virtual registers.
bool modifiesRegister(Register Reg,
                      const TargetRegisterInfo *TRI = nullptr) const {
  return findRegisterDefOperandIdx(Reg, false, true, TRI) != -1;
}

/// Returns true if the register is dead in this machine instruction.
/// If TargetRegisterInfo is passed, then it also checks
/// if there is a dead def of a super-register.
bool registerDefIsDead(Register Reg,
                       const TargetRegisterInfo *TRI = nullptr) const {
  return findRegisterDefOperandIdx(Reg, true, false, TRI) != -1;
}

/// Returns true if the MachineInstr has an implicit-use operand of exactly
/// the given register (not considering sub/super-registers).
bool hasRegisterImplicitUseOperand(Register Reg) const;

/// Returns the operand index that is a use of the specific register or -1
/// if it is not found. It further tightens the search criteria to a use
/// that kills the register if isKill is true.
int findRegisterUseOperandIdx(Register Reg, bool isKill = false,
                              const TargetRegisterInfo *TRI = nullptr) const;

/// Wrapper for findRegisterUseOperandIdx, it returns
/// a pointer to the MachineOperand rather than an index.
MachineOperand *findRegisterUseOperand(Register Reg, bool isKill = false,
                                    const TargetRegisterInfo *TRI = nullptr) {
  int Idx = findRegisterUseOperandIdx(Reg, isKill, TRI);
  return (Idx == -1) ? nullptr : &getOperand(Idx);
}

const MachineOperand *findRegisterUseOperand(
  Register Reg, bool isKill = false,
  const TargetRegisterInfo *TRI = nullptr) const {
  return const_cast<MachineInstr *>(this)->
    findRegisterUseOperand(Reg, isKill, TRI);
}

/// Returns the operand index that is a def of the specified register or
/// -1 if it is not found. If isDead is true, defs that are not dead are
/// skipped. If Overlap is true, then it also looks for defs that merely
/// overlap the specified register. If TargetRegisterInfo is non-null,
/// then it also checks if there is a def of a super-register.
/// This may also return a register mask operand when Overlap is true.
int findRegisterDefOperandIdx(Register Reg,
                              bool isDead = false, bool Overlap = false,
                              const TargetRegisterInfo *TRI = nullptr) const;

/// Wrapper for findRegisterDefOperandIdx, it returns
/// a pointer to the MachineOperand rather than an index.
MachineOperand *
findRegisterDefOperand(Register Reg, bool isDead = false,
                       bool Overlap = false,
                       const TargetRegisterInfo *TRI = nullptr) {
  int Idx = findRegisterDefOperandIdx(Reg, isDead, Overlap, TRI);
  return (Idx == -1) ? nullptr : &getOperand(Idx);
}

const MachineOperand *
findRegisterDefOperand(Register Reg, bool isDead = false,
                       bool Overlap = false,
                       const TargetRegisterInfo *TRI = nullptr) const {
  return const_cast<MachineInstr *>(this)->findRegisterDefOperand(
      Reg, isDead, Overlap, TRI);
}

/// Find the index of the first operand in the
/// operand list that is used to represent the predicate. It returns -1 if
/// none is found.
int findFirstPredOperandIdx() const;

/// Find the index of the flag word operand that
/// corresponds to operand OpIdx on an inline asm instruction.  Returns -1 if
/// getOperand(OpIdx) does not belong to an inline asm operand group.
///
/// If GroupNo is not NULL, it will receive the number of the operand group
/// containing OpIdx.
///
/// The flag operand is an immediate that can be decoded with methods like
/// InlineAsm::hasRegClassConstraint().
int findInlineAsmFlagIdx(unsigned OpIdx, unsigned *GroupNo = nullptr) const;

/// Compute the static register class constraint for operand OpIdx.
/// For normal instructions, this is derived from the MCInstrDesc.
/// For inline assembly it is derived from the flag words.
///
/// Returns NULL if the static register class constraint cannot be
/// determined.
const TargetRegisterClass*
getRegClassConstraint(unsigned OpIdx,
                      const TargetInstrInfo *TII,
                      const TargetRegisterInfo *TRI) const;

/// Applies the constraints (def/use) implied by this MI on \p Reg to
/// the given \p CurRC.
/// If \p ExploreBundle is set and MI is part of a bundle, all the
/// instructions inside the bundle will be taken into account. In other words,
/// this method accumulates all the constraints of the operand of this MI and
/// the related bundle if MI is a bundle or inside a bundle.
///
/// Returns the register class that satisfies both \p CurRC and the
/// constraints set by MI. Returns NULL if such a register class does not
/// exist.
///
/// \pre CurRC must not be NULL.
const TargetRegisterClass *getRegClassConstraintEffectForVReg(
    Register Reg, const TargetRegisterClass *CurRC,
    const TargetInstrInfo *TII, const TargetRegisterInfo *TRI,
    bool ExploreBundle = false) const;

/// Applies the constraints (def/use) implied by the \p OpIdx operand
/// to the given \p CurRC.
///
/// Returns the register class that satisfies both \p CurRC and the
/// constraints set by \p OpIdx MI. Returns NULL if such a register class
/// does not exist.
///
/// \pre CurRC must not be NULL.
/// \pre The operand at \p OpIdx must be a register.
const TargetRegisterClass *
getRegClassConstraintEffect(unsigned OpIdx, const TargetRegisterClass *CurRC,
                            const TargetInstrInfo *TII,
                            const TargetRegisterInfo *TRI) const;

/// Add a tie between the register operands at DefIdx and UseIdx.
/// The tie will cause the register allocator to ensure that the two
/// operands are assigned the same physical register.
///
/// Tied operands are managed automatically for explicit operands in the
/// MCInstrDesc. This method is for exceptional cases like inline asm.
void tieOperands(unsigned DefIdx, unsigned UseIdx);

/// Given the index of a tied register operand, find the
/// operand it is tied to. Defs are tied to uses and vice versa. Returns the
/// index of the tied operand which must exist.
unsigned findTiedOperandIdx(unsigned OpIdx) const;

/// Given the index of a register def operand,
/// check if the register def is tied to a source operand, due to either
/// two-address elimination or inline assembly constraints. Returns the
/// first tied use operand index by reference if UseOpIdx is not null.
bool isRegTiedToUseOperand(unsigned DefOpIdx,
                           unsigned *UseOpIdx = nullptr) const {
  const MachineOperand &MO = getOperand(DefOpIdx);
  if (!MO.isReg() || !MO.isDef() || !MO.isTied())
    return false;
  if (UseOpIdx)
    *UseOpIdx = findTiedOperandIdx(DefOpIdx);
  return true;
}

/// Return true if the use operand of the specified index is tied to a def
/// operand. It also returns the def operand index by reference if DefOpIdx
/// is not null.
bool isRegTiedToDefOperand(unsigned UseOpIdx,
                           unsigned *DefOpIdx = nullptr) const {
  const MachineOperand &MO = getOperand(UseOpIdx);
  if (!MO.isReg() || !MO.isUse() || !MO.isTied())
    return false;
  if (DefOpIdx)
    *DefOpIdx = findTiedOperandIdx(UseOpIdx);
  return true;
}

/// Clears kill flags on all operands.
void clearKillInfo();

/// Replace all occurrences of FromReg with ToReg:SubIdx,
/// properly composing subreg indices where necessary.
void substituteRegister(Register FromReg, Register ToReg, unsigned SubIdx,
                        const TargetRegisterInfo &RegInfo);

/// We have determined MI kills a register. Look for the
/// operand that uses it and mark it as IsKill. If AddIfNotFound is true,
/// add a implicit operand if it's not found. Returns true if the operand
/// exists / is added.
bool addRegisterKilled(Register IncomingReg,
                       const TargetRegisterInfo *RegInfo,
                       bool AddIfNotFound = false);

/// Clear all kill flags affecting Reg.  If RegInfo is provided, this includes
/// all aliasing registers.
void clearRegisterKills(Register Reg, const TargetRegisterInfo *RegInfo);

/// We have determined MI defined a register without a use.
/// Look for the operand that defines it and mark it as IsDead. If
/// AddIfNotFound is true, add a implicit operand if it's not found. Returns
/// true if the operand exists / is added.
bool addRegisterDead(Register Reg, const TargetRegisterInfo *RegInfo,
                     bool AddIfNotFound = false);

/// Clear all dead flags on operands defining register @p Reg.
void clearRegisterDeads(Register Reg);

/// Mark all subregister defs of register @p Reg with the undef flag.
/// This function is used when we determined to have a subregister def in an
/// otherwise undefined super register.
void setRegisterDefReadUndef(Register Reg, bool IsUndef = true);

/// We have determined MI defines a register. Make sure there is an operand
/// defining Reg.
void addRegisterDefined(Register Reg,
                        const TargetRegisterInfo *RegInfo = nullptr);

/// Mark every physreg used by this instruction as
/// dead except those in the UsedRegs list.
///
/// On instructions with register mask operands, also add implicit-def
/// operands for all registers in UsedRegs.
void setPhysRegsDeadExcept(ArrayRef<Register> UsedRegs,
                           const TargetRegisterInfo &TRI);

/// Return true if it is safe to move this instruction. If
/// SawStore is set to true, it means that there is a store (or call) between
/// the instruction's location and its intended destination.
bool isSafeToMove(AAResults *AA, bool &SawStore) const;

/// Returns true if this instruction's memory access aliases the memory
/// access of Other.
//
/// Assumes any physical registers used to compute addresses
/// have the same value for both instructions.  Returns false if neither
/// instruction writes to memory.
///
/// @param AA Optional alias analysis, used to compare memory operands.
/// @param Other MachineInstr to check aliasing against.
/// @param UseTBAA Whether to pass TBAA information to alias analysis.
bool mayAlias(AAResults *AA, const MachineInstr &Other, bool UseTBAA) const;

/// Return true if this instruction may have an ordered
/// or volatile memory reference, or if the information describing the memory
/// reference is not available. Return false if it is known to have no
/// ordered or volatile memory references.
bool hasOrderedMemoryRef() const;

/// Return true if this load instruction never traps and points to a memory
/// location whose value doesn't change during the execution of this function.
///
/// Examples include loading a value from the constant pool or from the
/// argument area of a function (if it does not change).  If the instruction
/// does multiple loads, this returns true only if all of the loads are
/// dereferenceable and invariant.
bool isDereferenceableInvariantLoad(AAResults *AA) const;

/// If the specified instruction is a PHI that always merges together the
/// same virtual register, return the register, otherwise return 0.
unsigned isConstantValuePHI() const;

/// Return true if this instruction has side effects that are not modeled
/// by mayLoad / mayStore, etc.
/// For all instructions, the property is encoded in MCInstrDesc::Flags
/// (see MCInstrDesc::hasUnmodeledSideEffects(). The only exception is
/// INLINEASM instruction, in which case the side effect property is encoded
/// in one of its operands (see InlineAsm::Extra_HasSideEffect).
///
bool hasUnmodeledSideEffects() const;

/// Returns true if it is illegal to fold a load across this instruction.
bool isLoadFoldBarrier() const;

/// Return true if all the defs of this instruction are dead.
bool allDefsAreDead() const;

/// Return a valid size if the instruction is a spill instruction.
Optional<unsigned> getSpillSize(const TargetInstrInfo *TII) const;

/// Return a valid size if the instruction is a folded spill instruction.
Optional<unsigned> getFoldedSpillSize(const TargetInstrInfo *TII) const;

/// Return a valid size if the instruction is a restore instruction.
Optional<unsigned> getRestoreSize(const TargetInstrInfo *TII) const;

/// Return a valid size if the instruction is a folded restore instruction.
Optional<unsigned>
getFoldedRestoreSize(const TargetInstrInfo *TII) const;

/// Copy implicit register operands from specified
/// instruction to this instruction.
void copyImplicitOps(MachineFunction &MF, const MachineInstr &MI);

/// Debugging support
/// @{
/// Determine the generic type to be printed (if needed) on uses and defs.
LLT getTypeToPrint(unsigned OpIdx, SmallBitVector &PrintedTypes,
                   const MachineRegisterInfo &MRI) const;

/// Return true when an instruction has tied register that can't be determined
/// by the instruction's descriptor. This is useful for MIR printing, to
/// determine whether we need to print the ties or not.
bool hasComplexRegisterTies() const;

/// Print this MI to \p OS.
/// Don't print information that can be inferred from other instructions if
/// \p IsStandalone is false. It is usually true when only a fragment of the
/// function is printed.
/// Only print the defs and the opcode if \p SkipOpers is true.
/// Otherwise, also print operands if \p SkipDebugLoc is true.
/// Otherwise, also print the debug loc, with a terminating newline.
/// \p TII is used to print the opcode name.  If it's not present, but the
/// MI is in a function, the opcode will be printed using the function's TII.
void print(raw_ostream &OS, bool IsStandalone = true, bool SkipOpers = false,
           bool SkipDebugLoc = false, bool AddNewLine = true,
           const TargetInstrInfo *TII = nullptr) const;
void print(raw_ostream &OS, ModuleSlotTracker &MST, bool IsStandalone = true,
           bool SkipOpers = false, bool SkipDebugLoc = false,
           bool AddNewLine = true,
           const TargetInstrInfo *TII = nullptr) const;
void dump() const;
/// Print on dbgs() the current instruction and the instructions defining its
/// operands and so on until we reach \p MaxDepth.
void dumpr(const MachineRegisterInfo &MRI,
           unsigned MaxDepth = UINT_MAX(2147483647 *2U +1U)) const;
/// @}

//===--------------------------------------------------------------------===//
// Accessors used to build up machine instructions.

/// Add the specified operand to the instruction.  If it is an implicit
/// operand, it is added to the end of the operand list.  If it is an
/// explicit operand it is added at the end of the explicit operand list
/// (before the first implicit operand).
///
/// MF must be the machine function that was used to allocate this
/// instruction.
///
/// MachineInstrBuilder provides a more convenient interface for creating
/// instructions and adding operands.
void addOperand(MachineFunction &MF, const MachineOperand &Op);

/// Add an operand without providing an MF reference. This only works for
/// instructions that are inserted in a basic block.
///
/// MachineInstrBuilder and the two-argument addOperand(MF, MO) should be
/// preferred.
void addOperand(const MachineOperand &Op);

/// Replace the instruction descriptor (thus opcode) of
/// the current instruction with a new one.
void setDesc(const MCInstrDesc &tid) { MCID = &tid; }

/// Replace current source information with new such.
/// Avoid using this, the constructor argument is preferable.
void setDebugLoc(DebugLoc dl) {
  debugLoc = std::move(dl);
  assert(debugLoc.hasTrivialDestructor() && "Expected trivial destructor")((debugLoc.hasTrivialDestructor() && "Expected trivial destructor"
) ? static_cast<void> (0) : __assert_fail ("debugLoc.hasTrivialDestructor() && \"Expected trivial destructor\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/include/llvm/CodeGen/MachineInstr.h"
, 1681, __PRETTY_FUNCTION__));
}

/// Erase an operand from an instruction, leaving it with one
/// fewer operand than it started with.
void RemoveOperand(unsigned OpNo);

/// Clear this MachineInstr's memory reference descriptor list.  This resets
/// the memrefs to their most conservative state.  This should be used only
/// as a last resort since it greatly pessimizes our knowledge of the memory
/// access performed by the instruction.
void dropMemRefs(MachineFunction &MF);

/// Assign this MachineInstr's memory reference descriptor list.
///
/// Unlike other methods, this *will* allocate them into a new array
/// associated with the provided `MachineFunction`.
void setMemRefs(MachineFunction &MF, ArrayRef<MachineMemOperand *> MemRefs);

/// Add a MachineMemOperand to the machine instruction.
/// This function should be used only occasionally. The setMemRefs function
/// is the primary method for setting up a MachineInstr's MemRefs list.
void addMemOperand(MachineFunction &MF, MachineMemOperand *MO);

/// Clone another MachineInstr's memory reference descriptor list and replace
/// ours with it.
///
/// Note that `*this` may be the incoming MI!
///
/// Prefer this API whenever possible as it can avoid allocations in common
/// cases.
void cloneMemRefs(MachineFunction &MF, const MachineInstr &MI);

/// Clone the merge of multiple MachineInstrs' memory reference descriptors
/// list and replace ours with it.
///
/// Note that `*this` may be one of the incoming MIs!
///
/// Prefer this API whenever possible as it can avoid allocations in common
/// cases.
void cloneMergedMemRefs(MachineFunction &MF,
                        ArrayRef<const MachineInstr *> MIs);

/// Set a symbol that will be emitted just prior to the instruction itself.
///
/// Setting this to a null pointer will remove any such symbol.
///
/// FIXME: This is not fully implemented yet.
void setPreInstrSymbol(MachineFunction &MF, MCSymbol *Symbol);

/// Set a symbol that will be emitted just after the instruction itself.
///
/// Setting this to a null pointer will remove any such symbol.
///
/// FIXME: This is not fully implemented yet.
void setPostInstrSymbol(MachineFunction &MF, MCSymbol *Symbol);

/// Clone another MachineInstr's pre- and post- instruction symbols and
/// replace ours with it.
void cloneInstrSymbols(MachineFunction &MF, const MachineInstr &MI);

/// Set a marker on instructions that denotes where we should create and emit
/// heap alloc site labels. This waits until after instruction selection and
/// optimizations to create the label, so it should still work if the
/// instruction is removed or duplicated.
void setHeapAllocMarker(MachineFunction &MF, MDNode *MD);

/// Return the MIFlags which represent both MachineInstrs. This
/// should be used when merging two MachineInstrs into one. This routine does
/// not modify the MIFlags of this MachineInstr.
uint16_t mergeFlagsWith(const MachineInstr& Other) const;

static uint16_t copyFlagsFromInstruction(const Instruction &I);

/// Copy all flags to MachineInst MIFlags
void copyIRFlags(const Instruction &I);

/// Break any tie involving OpIdx.
void untieRegOperand(unsigned OpIdx) {
  MachineOperand &MO = getOperand(OpIdx);
  if (MO.isReg() && MO.isTied()) {
    getOperand(findTiedOperandIdx(OpIdx)).TiedTo = 0;
    MO.TiedTo = 0;
  }
}

/// Add all implicit def and use operands to this instruction.
void addImplicitDefUseOperands(MachineFunction &MF);

/// Scan instructions immediately following MI and collect any matching
/// DBG_VALUEs.
void collectDebugValues(SmallVectorImpl<MachineInstr *> &DbgValues);

/// Find all DBG_VALUEs that point to the register def in this instruction
/// and point them to \p Reg instead.
void changeDebugValuesDefReg(Register Reg);

/// Returns the Intrinsic::ID for this instruction.
/// \pre Must have an intrinsic ID operand.
unsigned getIntrinsicID() const {
  return getOperand(getNumExplicitDefs()).getIntrinsicID();
}

/// Sets all register debug operands in this debug value instruction to be
/// undef.
void setDebugValueUndef() {
  assert(isDebugValue() && "Must be a debug value instruction.")((isDebugValue() && "Must be a debug value instruction."
) ? static_cast<void> (0) : __assert_fail ("isDebugValue() && \"Must be a debug value instruction.\""
, "/build/llvm-toolchain-snapshot-12~++20210120111114+fc6677f0bbaf/llvm/include/llvm/CodeGen/MachineInstr.h"
, 1787, __PRETTY_FUNCTION__));
  for (MachineOperand &MO : debug_operands()) {
    if (MO.isReg()) {
      MO.setReg(0);
      MO.setSubReg(0);
    }
  }
}

1796private:
/// If this instruction is embedded into a MachineFunction, return the
/// MachineRegisterInfo object for the current function, otherwise
/// return null.
MachineRegisterInfo *getRegInfo();

/// Unlink all of the register operands in this instruction from their
/// respective use lists.  This requires that the operands already be on their
/// use lists.
void RemoveRegOperandsFromUseLists(MachineRegisterInfo&);

/// Add all of the register operands in this instruction from their
/// respective use lists.  This requires that the operands not be on their
/// use lists yet.
void AddRegOperandsToUseLists(MachineRegisterInfo&);

/// Slow path for hasProperty when we're dealing with a bundle.
bool hasPropertyInBundle(uint64_t Mask, QueryType Type) const;

/// Implements the logic of getRegClassConstraintEffectForVReg for the
/// this MI and the given operand index \p OpIdx.
/// If the related operand does not constrained Reg, this returns CurRC.
const TargetRegisterClass *getRegClassConstraintEffectForVRegImpl(
    unsigned OpIdx, Register Reg, const TargetRegisterClass *CurRC,
    const TargetInstrInfo *TII, const TargetRegisterInfo *TRI) const;

/// Stores extra instruction information inline or allocates as ExtraInfo
/// based on the number of pointers.
void setExtraInfo(MachineFunction &MF, ArrayRef<MachineMemOperand *> MMOs,
                  MCSymbol *PreInstrSymbol, MCSymbol *PostInstrSymbol,
                  MDNode *HeapAllocMarker);
1827};

1829/// Special DenseMapInfo traits to compare MachineInstr* by *value* of the
1830/// instruction rather than by pointer value.
1831/// The hashing and equality testing functions ignore definitions so this is
1832/// useful for CSE, etc.
1833struct MachineInstrExpressionTrait : DenseMapInfo<MachineInstr*> {
static inline MachineInstr *getEmptyKey() {
  return nullptr;
}

static inline MachineInstr *getTombstoneKey() {
  return reinterpret_cast<MachineInstr*>(-1);
}

static unsigned getHashValue(const MachineInstr* const &MI);

static bool isEqual(const MachineInstr* const &LHS,
                    const MachineInstr* const &RHS) {
  if (RHS == getEmptyKey() || RHS == getTombstoneKey() ||
      LHS == getEmptyKey() || LHS == getTombstoneKey())
    return LHS == RHS;
  return LHS->isIdenticalTo(*RHS, MachineInstr::IgnoreVRegDefs);
}
1851};

1853//===----------------------------------------------------------------------===//
1854// Debugging Support

1856inline raw_ostream& operator<<(raw_ostream &OS, const MachineInstr &MI) {
MI.print(OS);
return OS;
1859}

1861} // end namespace llvm

1863#endif // LLVM_CODEGEN_MACHINEINSTR_H