/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp

Bug Summary

File:	llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
Warning:	line 5879, column 63 The result of the left shift is undefined due to shifting by '4294967295', which is greater or equal to the width of type 'int'

Annotated Source Code

Press '?' to see keyboard shortcuts

Show analyzer invocation

clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name AArch64InstructionSelector.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mframe-pointer=none -fmath-errno -fno-rounding-math -mconstructor-aliases -munwind-tables -target-cpu x86-64 -tune-cpu generic -debugger-tuning=gdb -ffunction-sections -fdata-sections -fcoverage-compilation-dir=/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/build-llvm/lib/Target/AArch64 -resource-dir /usr/lib/llvm-14/lib/clang/14.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/build-llvm/lib/Target/AArch64 -I /build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64 -I /build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/build-llvm/include -I /build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include -D NDEBUG -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/x86_64-linux-gnu/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10/backward -internal-isystem /usr/lib/llvm-14/lib/clang/14.0.0/include -internal-isystem /usr/local/include -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../x86_64-linux-gnu/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-class-memaccess -Wno-redundant-move -Wno-pessimizing-move -Wno-noexcept-type -Wno-comment -std=c++14 -fdeprecated-macro -fdebug-compilation-dir=/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/build-llvm/lib/Target/AArch64 -fdebug-prefix-map=/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0=. -ferror-limit 19 -fvisibility hidden -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -D__GCC_HAVE_DWARF2_CFI_ASM=1 -o /tmp/scan-build-2021-08-28-193554-24367-1 -x c++ /build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp

/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp

→

1//===- AArch64InstructionSelector.cpp ----------------------------*- C++ -*-==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file implements the targeting of the InstructionSelector class for
10/// AArch64.
11/// \todo This should be generated by TableGen.
12//===----------------------------------------------------------------------===//

14#include "AArch64GlobalISelUtils.h"
15#include "AArch64InstrInfo.h"
16#include "AArch64MachineFunctionInfo.h"
17#include "AArch64RegisterBankInfo.h"
18#include "AArch64RegisterInfo.h"
19#include "AArch64Subtarget.h"
20#include "AArch64TargetMachine.h"
21#include "AArch64GlobalISelUtils.h"
22#include "MCTargetDesc/AArch64AddressingModes.h"
23#include "MCTargetDesc/AArch64MCTargetDesc.h"
24#include "llvm/ADT/Optional.h"
25#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
26#include "llvm/CodeGen/GlobalISel/InstructionSelector.h"
27#include "llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h"
28#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
29#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
30#include "llvm/CodeGen/MachineBasicBlock.h"
31#include "llvm/CodeGen/MachineConstantPool.h"
32#include "llvm/CodeGen/MachineFunction.h"
33#include "llvm/CodeGen/MachineInstr.h"
34#include "llvm/CodeGen/MachineInstrBuilder.h"
35#include "llvm/CodeGen/MachineMemOperand.h"
36#include "llvm/CodeGen/MachineOperand.h"
37#include "llvm/CodeGen/MachineRegisterInfo.h"
38#include "llvm/CodeGen/TargetOpcodes.h"
39#include "llvm/IR/Constants.h"
40#include "llvm/IR/DerivedTypes.h"
41#include "llvm/IR/Instructions.h"
42#include "llvm/IR/PatternMatch.h"
43#include "llvm/IR/Type.h"
44#include "llvm/IR/IntrinsicsAArch64.h"
45#include "llvm/Pass.h"
46#include "llvm/Support/Debug.h"
47#include "llvm/Support/raw_ostream.h"

49#define DEBUG_TYPE"aarch64-isel" "aarch64-isel"

51using namespace llvm;
52using namespace MIPatternMatch;
53using namespace AArch64GISelUtils;

55namespace llvm {
56class BlockFrequencyInfo;
57class ProfileSummaryInfo;
58}

60namespace {

62#define GET_GLOBALISEL_PREDICATE_BITSET
63#include "AArch64GenGlobalISel.inc"
64#undef GET_GLOBALISEL_PREDICATE_BITSET

66class AArch64InstructionSelector : public InstructionSelector {
67public:
AArch64InstructionSelector(const AArch64TargetMachine &TM,
                           const AArch64Subtarget &STI,
                           const AArch64RegisterBankInfo &RBI);

bool select(MachineInstr &I) override;
static const char *getName() { return DEBUG_TYPE"aarch64-isel"; }

void setupMF(MachineFunction &MF, GISelKnownBits *KB,
             CodeGenCoverage &CoverageInfo, ProfileSummaryInfo *PSI,
             BlockFrequencyInfo *BFI) override {
  InstructionSelector::setupMF(MF, KB, CoverageInfo, PSI, BFI);
  MIB.setMF(MF);

  // hasFnAttribute() is expensive to call on every BRCOND selection, so
  // cache it here for each run of the selector.
  ProduceNonFlagSettingCondBr =
      !MF.getFunction().hasFnAttribute(Attribute::SpeculativeLoadHardening);
  MFReturnAddr = Register();

  processPHIs(MF);
}

90private:
/// tblgen-erated 'select' implementation, used as the initial selector for
/// the patterns that don't require complex C++.
bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const;

// A lowering phase that runs before any selection attempts.
// Returns true if the instruction was modified.
bool preISelLower(MachineInstr &I);

// An early selection function that runs before the selectImpl() call.
bool earlySelect(MachineInstr &I);

// Do some preprocessing of G_PHIs before we begin selection.
void processPHIs(MachineFunction &MF);

bool earlySelectSHL(MachineInstr &I, MachineRegisterInfo &MRI);

/// Eliminate same-sized cross-bank copies into stores before selectImpl().
bool contractCrossBankCopyIntoStore(MachineInstr &I,
                                    MachineRegisterInfo &MRI);

bool convertPtrAddToAdd(MachineInstr &I, MachineRegisterInfo &MRI);

bool selectVaStartAAPCS(MachineInstr &I, MachineFunction &MF,
                        MachineRegisterInfo &MRI) const;
bool selectVaStartDarwin(MachineInstr &I, MachineFunction &MF,
                         MachineRegisterInfo &MRI) const;

///@{
/// Helper functions for selectCompareBranch.
bool selectCompareBranchFedByFCmp(MachineInstr &I, MachineInstr &FCmp,
                                  MachineIRBuilder &MIB) const;
bool selectCompareBranchFedByICmp(MachineInstr &I, MachineInstr &ICmp,
                                  MachineIRBuilder &MIB) const;
bool tryOptCompareBranchFedByICmp(MachineInstr &I, MachineInstr &ICmp,
                                  MachineIRBuilder &MIB) const;
bool tryOptAndIntoCompareBranch(MachineInstr &AndInst, bool Invert,
                                MachineBasicBlock *DstMBB,
                                MachineIRBuilder &MIB) const;
///@}

bool selectCompareBranch(MachineInstr &I, MachineFunction &MF,
                         MachineRegisterInfo &MRI);

bool selectVectorAshrLshr(MachineInstr &I, MachineRegisterInfo &MRI);
bool selectVectorSHL(MachineInstr &I, MachineRegisterInfo &MRI);

// Helper to generate an equivalent of scalar_to_vector into a new register,
// returned via 'Dst'.
MachineInstr *emitScalarToVector(unsigned EltSize,
                                 const TargetRegisterClass *DstRC,
                                 Register Scalar,
                                 MachineIRBuilder &MIRBuilder) const;

/// Emit a lane insert into \p DstReg, or a new vector register if None is
/// provided.
///
/// The lane inserted into is defined by \p LaneIdx. The vector source
/// register is given by \p SrcReg. The register containing the element is
/// given by \p EltReg.
MachineInstr *emitLaneInsert(Optional<Register> DstReg, Register SrcReg,
                             Register EltReg, unsigned LaneIdx,
                             const RegisterBank &RB,
                             MachineIRBuilder &MIRBuilder) const;

/// Emit a sequence of instructions representing a constant \p CV for a
/// vector register \p Dst. (E.g. a MOV, or a load from a constant pool.)
///
/// \returns the last instruction in the sequence on success, and nullptr
/// otherwise.
MachineInstr *emitConstantVector(Register Dst, Constant *CV,
                                 MachineIRBuilder &MIRBuilder,
                                 MachineRegisterInfo &MRI);

bool selectInsertElt(MachineInstr &I, MachineRegisterInfo &MRI);
bool tryOptConstantBuildVec(MachineInstr &MI, LLT DstTy,
                            MachineRegisterInfo &MRI);
/// \returns true if a G_BUILD_VECTOR instruction \p MI can be selected as a
/// SUBREG_TO_REG.
bool tryOptBuildVecToSubregToReg(MachineInstr &MI, MachineRegisterInfo &MRI);
bool selectBuildVector(MachineInstr &I, MachineRegisterInfo &MRI);
bool selectMergeValues(MachineInstr &I, MachineRegisterInfo &MRI);
bool selectUnmergeValues(MachineInstr &I, MachineRegisterInfo &MRI);

bool selectShuffleVector(MachineInstr &I, MachineRegisterInfo &MRI);
bool selectExtractElt(MachineInstr &I, MachineRegisterInfo &MRI);
bool selectConcatVectors(MachineInstr &I, MachineRegisterInfo &MRI);
bool selectSplitVectorUnmerge(MachineInstr &I, MachineRegisterInfo &MRI);

/// Helper function to select vector load intrinsics like
/// @llvm.aarch64.neon.ld2.*, @llvm.aarch64.neon.ld4.*, etc.
/// \p Opc is the opcode that the selected instruction should use.
/// \p NumVecs is the number of vector destinations for the instruction.
/// \p I is the original G_INTRINSIC_W_SIDE_EFFECTS instruction.
bool selectVectorLoadIntrinsic(unsigned Opc, unsigned NumVecs,
                               MachineInstr &I);
bool selectIntrinsicWithSideEffects(MachineInstr &I,
                                    MachineRegisterInfo &MRI);
bool selectIntrinsic(MachineInstr &I, MachineRegisterInfo &MRI);
bool selectVectorICmp(MachineInstr &I, MachineRegisterInfo &MRI);
bool selectIntrinsicTrunc(MachineInstr &I, MachineRegisterInfo &MRI) const;
bool selectIntrinsicRound(MachineInstr &I, MachineRegisterInfo &MRI) const;
bool selectJumpTable(MachineInstr &I, MachineRegisterInfo &MRI);
bool selectBrJT(MachineInstr &I, MachineRegisterInfo &MRI);
bool selectTLSGlobalValue(MachineInstr &I, MachineRegisterInfo &MRI);
bool selectReduction(MachineInstr &I, MachineRegisterInfo &MRI);

unsigned emitConstantPoolEntry(const Constant *CPVal,
                               MachineFunction &MF) const;
MachineInstr *emitLoadFromConstantPool(const Constant *CPVal,
                                       MachineIRBuilder &MIRBuilder) const;

// Emit a vector concat operation.
MachineInstr *emitVectorConcat(Optional<Register> Dst, Register Op1,
                               Register Op2,
                               MachineIRBuilder &MIRBuilder) const;

// Emit an integer compare between LHS and RHS, which checks for Predicate.
MachineInstr *emitIntegerCompare(MachineOperand &LHS, MachineOperand &RHS,
                                 MachineOperand &Predicate,
                                 MachineIRBuilder &MIRBuilder) const;

/// Emit a floating point comparison between \p LHS and \p RHS.
/// \p Pred if given is the intended predicate to use.
MachineInstr *emitFPCompare(Register LHS, Register RHS,
                            MachineIRBuilder &MIRBuilder,
                            Optional<CmpInst::Predicate> = None) const;

MachineInstr *emitInstr(unsigned Opcode,
                        std::initializer_list<llvm::DstOp> DstOps,
                        std::initializer_list<llvm::SrcOp> SrcOps,
                        MachineIRBuilder &MIRBuilder,
                        const ComplexRendererFns &RenderFns = None) const;
/// Helper function to emit an add or sub instruction.
///
/// \p AddrModeAndSizeToOpcode must contain each of the opcode variants above
/// in a specific order.
///
/// Below is an example of the expected input to \p AddrModeAndSizeToOpcode.
///
/// \code
///   const std::array<std::array<unsigned, 2>, 4> Table {
///    {{AArch64::ADDXri, AArch64::ADDWri},
///     {AArch64::ADDXrs, AArch64::ADDWrs},
///     {AArch64::ADDXrr, AArch64::ADDWrr},
///     {AArch64::SUBXri, AArch64::SUBWri},
///     {AArch64::ADDXrx, AArch64::ADDWrx}}};
/// \endcode
///
/// Each row in the table corresponds to a different addressing mode. Each
/// column corresponds to a different register size.
///
/// \attention Rows must be structured as follows:
///   - Row 0: The ri opcode variants
///   - Row 1: The rs opcode variants
///   - Row 2: The rr opcode variants
///   - Row 3: The ri opcode variants for negative immediates
///   - Row 4: The rx opcode variants
///
/// \attention Columns must be structured as follows:
///   - Column 0: The 64-bit opcode variants
///   - Column 1: The 32-bit opcode variants
///
/// \p Dst is the destination register of the binop to emit.
/// \p LHS is the left-hand operand of the binop to emit.
/// \p RHS is the right-hand operand of the binop to emit.
MachineInstr *emitAddSub(
    const std::array<std::array<unsigned, 2>, 5> &AddrModeAndSizeToOpcode,
    Register Dst, MachineOperand &LHS, MachineOperand &RHS,
    MachineIRBuilder &MIRBuilder) const;
MachineInstr *emitADD(Register DefReg, MachineOperand &LHS,
                      MachineOperand &RHS,
                      MachineIRBuilder &MIRBuilder) const;
MachineInstr *emitADDS(Register Dst, MachineOperand &LHS, MachineOperand &RHS,
                       MachineIRBuilder &MIRBuilder) const;
MachineInstr *emitSUBS(Register Dst, MachineOperand &LHS, MachineOperand &RHS,
                       MachineIRBuilder &MIRBuilder) const;
MachineInstr *emitCMN(MachineOperand &LHS, MachineOperand &RHS,
                      MachineIRBuilder &MIRBuilder) const;
MachineInstr *emitTST(MachineOperand &LHS, MachineOperand &RHS,
                      MachineIRBuilder &MIRBuilder) const;
MachineInstr *emitSelect(Register Dst, Register LHS, Register RHS,
                         AArch64CC::CondCode CC,
                         MachineIRBuilder &MIRBuilder) const;
MachineInstr *emitExtractVectorElt(Optional<Register> DstReg,
                                   const RegisterBank &DstRB, LLT ScalarTy,
                                   Register VecReg, unsigned LaneIdx,
                                   MachineIRBuilder &MIRBuilder) const;

/// Emit a CSet for an integer compare.
///
/// \p DefReg and \p SrcReg are expected to be 32-bit scalar registers.
MachineInstr *emitCSetForICMP(Register DefReg, unsigned Pred,
                              MachineIRBuilder &MIRBuilder,
                              Register SrcReg = AArch64::WZR) const;
/// Emit a CSet for a FP compare.
///
/// \p Dst is expected to be a 32-bit scalar register.
MachineInstr *emitCSetForFCmp(Register Dst, CmpInst::Predicate Pred,
                              MachineIRBuilder &MIRBuilder) const;

/// Emit the overflow op for \p Opcode.
///
/// \p Opcode is expected to be an overflow op's opcode, e.g. G_UADDO,
/// G_USUBO, etc.
std::pair<MachineInstr *, AArch64CC::CondCode>
emitOverflowOp(unsigned Opcode, Register Dst, MachineOperand &LHS,
               MachineOperand &RHS, MachineIRBuilder &MIRBuilder) const;

/// Emit a TB(N)Z instruction which tests \p Bit in \p TestReg.
/// \p IsNegative is true if the test should be "not zero".
/// This will also optimize the test bit instruction when possible.
MachineInstr *emitTestBit(Register TestReg, uint64_t Bit, bool IsNegative,
                          MachineBasicBlock *DstMBB,
                          MachineIRBuilder &MIB) const;

/// Emit a CB(N)Z instruction which branches to \p DestMBB.
MachineInstr *emitCBZ(Register CompareReg, bool IsNegative,
                      MachineBasicBlock *DestMBB,
                      MachineIRBuilder &MIB) const;

// Equivalent to the i32shift_a and friends from AArch64InstrInfo.td.
// We use these manually instead of using the importer since it doesn't
// support SDNodeXForm.
ComplexRendererFns selectShiftA_32(const MachineOperand &Root) const;
ComplexRendererFns selectShiftB_32(const MachineOperand &Root) const;
ComplexRendererFns selectShiftA_64(const MachineOperand &Root) const;
ComplexRendererFns selectShiftB_64(const MachineOperand &Root) const;

ComplexRendererFns select12BitValueWithLeftShift(uint64_t Immed) const;
ComplexRendererFns selectArithImmed(MachineOperand &Root) const;
ComplexRendererFns selectNegArithImmed(MachineOperand &Root) const;

ComplexRendererFns selectAddrModeUnscaled(MachineOperand &Root,
                                          unsigned Size) const;

ComplexRendererFns selectAddrModeUnscaled8(MachineOperand &Root) const {
  return selectAddrModeUnscaled(Root, 1);
}
ComplexRendererFns selectAddrModeUnscaled16(MachineOperand &Root) const {
  return selectAddrModeUnscaled(Root, 2);
}
ComplexRendererFns selectAddrModeUnscaled32(MachineOperand &Root) const {
  return selectAddrModeUnscaled(Root, 4);
}
ComplexRendererFns selectAddrModeUnscaled64(MachineOperand &Root) const {
  return selectAddrModeUnscaled(Root, 8);
}
ComplexRendererFns selectAddrModeUnscaled128(MachineOperand &Root) const {
  return selectAddrModeUnscaled(Root, 16);
}

/// Helper to try to fold in a GISEL_ADD_LOW into an immediate, to be used
/// from complex pattern matchers like selectAddrModeIndexed().
ComplexRendererFns tryFoldAddLowIntoImm(MachineInstr &RootDef, unsigned Size,
                                        MachineRegisterInfo &MRI) const;

ComplexRendererFns selectAddrModeIndexed(MachineOperand &Root,
                                         unsigned Size) const;
template <int Width>
ComplexRendererFns selectAddrModeIndexed(MachineOperand &Root) const {
  return selectAddrModeIndexed(Root, Width / 8);
}

bool isWorthFoldingIntoExtendedReg(MachineInstr &MI,
                                   const MachineRegisterInfo &MRI) const;
ComplexRendererFns
selectAddrModeShiftedExtendXReg(MachineOperand &Root,
                                unsigned SizeInBytes) const;

/// Returns a \p ComplexRendererFns which contains a base, offset, and whether
/// or not a shift + extend should be folded into an addressing mode. Returns
/// None when this is not profitable or possible.
ComplexRendererFns
selectExtendedSHL(MachineOperand &Root, MachineOperand &Base,
                  MachineOperand &Offset, unsigned SizeInBytes,
                  bool WantsExt) const;
ComplexRendererFns selectAddrModeRegisterOffset(MachineOperand &Root) const;
ComplexRendererFns selectAddrModeXRO(MachineOperand &Root,
                                     unsigned SizeInBytes) const;
template <int Width>
ComplexRendererFns selectAddrModeXRO(MachineOperand &Root) const {
  return selectAddrModeXRO(Root, Width / 8);
}

ComplexRendererFns selectAddrModeWRO(MachineOperand &Root,
                                     unsigned SizeInBytes) const;
template <int Width>
ComplexRendererFns selectAddrModeWRO(MachineOperand &Root) const {
  return selectAddrModeWRO(Root, Width / 8);
}

ComplexRendererFns selectShiftedRegister(MachineOperand &Root) const;

ComplexRendererFns selectArithShiftedRegister(MachineOperand &Root) const {
  return selectShiftedRegister(Root);
}

ComplexRendererFns selectLogicalShiftedRegister(MachineOperand &Root) const {
  // TODO: selectShiftedRegister should allow for rotates on logical shifts.
  // For now, make them the same. The only difference between the two is that
  // logical shifts are allowed to fold in rotates. Otherwise, these are
  // functionally the same.
  return selectShiftedRegister(Root);
}

/// Given an extend instruction, determine the correct shift-extend type for
/// that instruction.
///
/// If the instruction is going to be used in a load or store, pass
/// \p IsLoadStore = true.
AArch64_AM::ShiftExtendType
getExtendTypeForInst(MachineInstr &MI, MachineRegisterInfo &MRI,
                     bool IsLoadStore = false) const;

/// Move \p Reg to \p RC if \p Reg is not already on \p RC.
///
/// \returns Either \p Reg if no change was necessary, or the new register
/// created by moving \p Reg.
///
/// Note: This uses emitCopy right now.
Register moveScalarRegClass(Register Reg, const TargetRegisterClass &RC,
                            MachineIRBuilder &MIB) const;

ComplexRendererFns selectArithExtendedRegister(MachineOperand &Root) const;

void renderTruncImm(MachineInstrBuilder &MIB, const MachineInstr &MI,
                    int OpIdx = -1) const;
void renderLogicalImm32(MachineInstrBuilder &MIB, const MachineInstr &I,
                        int OpIdx = -1) const;
void renderLogicalImm64(MachineInstrBuilder &MIB, const MachineInstr &I,
                        int OpIdx = -1) const;
void renderFPImm16(MachineInstrBuilder &MIB, const MachineInstr &MI,
                   int OpIdx = -1) const;
void renderFPImm32(MachineInstrBuilder &MIB, const MachineInstr &MI,
                   int OpIdx = -1) const;
void renderFPImm64(MachineInstrBuilder &MIB, const MachineInstr &MI,
                   int OpIdx = -1) const;

// Materialize a GlobalValue or BlockAddress using a movz+movk sequence.
void materializeLargeCMVal(MachineInstr &I, const Value *V, unsigned OpFlags);

// Optimization methods.
bool tryOptSelect(MachineInstr &MI);
MachineInstr *tryFoldIntegerCompare(MachineOperand &LHS, MachineOperand &RHS,
                                    MachineOperand &Predicate,
                                    MachineIRBuilder &MIRBuilder) const;

/// Return true if \p MI is a load or store of \p NumBytes bytes.
bool isLoadStoreOfNumBytes(const MachineInstr &MI, unsigned NumBytes) const;

/// Returns true if \p MI is guaranteed to have the high-half of a 64-bit
/// register zeroed out. In other words, the result of MI has been explicitly
/// zero extended.
bool isDef32(const MachineInstr &MI) const;

const AArch64TargetMachine &TM;
const AArch64Subtarget &STI;
const AArch64InstrInfo &TII;
const AArch64RegisterInfo &TRI;
const AArch64RegisterBankInfo &RBI;

bool ProduceNonFlagSettingCondBr = false;

// Some cached values used during selection.
// We use LR as a live-in register, and we keep track of it here as it can be
// clobbered by calls.
Register MFReturnAddr;

MachineIRBuilder MIB;

461#define GET_GLOBALISEL_PREDICATES_DECL
462#include "AArch64GenGlobalISel.inc"
463#undef GET_GLOBALISEL_PREDICATES_DECL

465// We declare the temporaries used by selectImpl() in the class to minimize the
466// cost of constructing placeholder values.
467#define GET_GLOBALISEL_TEMPORARIES_DECL
468#include "AArch64GenGlobalISel.inc"
469#undef GET_GLOBALISEL_TEMPORARIES_DECL
470};

472} // end anonymous namespace

474#define GET_GLOBALISEL_IMPL
475#include "AArch64GenGlobalISel.inc"
476#undef GET_GLOBALISEL_IMPL

478AArch64InstructionSelector::AArch64InstructionSelector(
  const AArch64TargetMachine &TM, const AArch64Subtarget &STI,
  const AArch64RegisterBankInfo &RBI)
  : InstructionSelector(), TM(TM), STI(STI), TII(*STI.getInstrInfo()),
    TRI(*STI.getRegisterInfo()), RBI(RBI),
483#define GET_GLOBALISEL_PREDICATES_INIT
484#include "AArch64GenGlobalISel.inc"
485#undef GET_GLOBALISEL_PREDICATES_INIT
486#define GET_GLOBALISEL_TEMPORARIES_INIT
487#include "AArch64GenGlobalISel.inc"
488#undef GET_GLOBALISEL_TEMPORARIES_INIT
489{
490}

492// FIXME: This should be target-independent, inferred from the types declared
493// for each class in the bank.
494static const TargetRegisterClass *
495getRegClassForTypeOnBank(LLT Ty, const RegisterBank &RB,
                       const RegisterBankInfo &RBI,
                       bool GetAllRegSet = false) {
if (RB.getID() == AArch64::GPRRegBankID) {
  if (Ty.getSizeInBits() <= 32)
    return GetAllRegSet ? &AArch64::GPR32allRegClass
                        : &AArch64::GPR32RegClass;
  if (Ty.getSizeInBits() == 64)
    return GetAllRegSet ? &AArch64::GPR64allRegClass
                        : &AArch64::GPR64RegClass;
  if (Ty.getSizeInBits() == 128)
    return &AArch64::XSeqPairsClassRegClass;
  return nullptr;
}

if (RB.getID() == AArch64::FPRRegBankID) {
  switch (Ty.getSizeInBits()) {
  case 8:
    return &AArch64::FPR8RegClass;
  case 16:
    return &AArch64::FPR16RegClass;
  case 32:
    return &AArch64::FPR32RegClass;
  case 64:
    return &AArch64::FPR64RegClass;
  case 128:
    return &AArch64::FPR128RegClass;
  }
  return nullptr;
}

return nullptr;
527}

529/// Given a register bank, and size in bits, return the smallest register class
530/// that can represent that combination.
531static const TargetRegisterClass *
532getMinClassForRegBank(const RegisterBank &RB, unsigned SizeInBits,
                    bool GetAllRegSet = false) {
unsigned RegBankID = RB.getID();

if (RegBankID == AArch64::GPRRegBankID) {
  if (SizeInBits <= 32)
    return GetAllRegSet ? &AArch64::GPR32allRegClass
                        : &AArch64::GPR32RegClass;
  if (SizeInBits == 64)
    return GetAllRegSet ? &AArch64::GPR64allRegClass
                        : &AArch64::GPR64RegClass;
  if (SizeInBits == 128)
    return &AArch64::XSeqPairsClassRegClass;
}

if (RegBankID == AArch64::FPRRegBankID) {
  switch (SizeInBits) {
  default:
    return nullptr;
  case 8:
    return &AArch64::FPR8RegClass;
  case 16:
    return &AArch64::FPR16RegClass;
  case 32:
    return &AArch64::FPR32RegClass;
  case 64:
    return &AArch64::FPR64RegClass;
  case 128:
    return &AArch64::FPR128RegClass;
  }
}

return nullptr;
565}

567/// Returns the correct subregister to use for a given register class.
568static bool getSubRegForClass(const TargetRegisterClass *RC,
                            const TargetRegisterInfo &TRI, unsigned &SubReg) {
switch (TRI.getRegSizeInBits(*RC)) {
case 8:
  SubReg = AArch64::bsub;
  break;
case 16:
  SubReg = AArch64::hsub;
  break;
case 32:
  if (RC != &AArch64::FPR32RegClass)
    SubReg = AArch64::sub_32;
  else
    SubReg = AArch64::ssub;
  break;
case 64:
  SubReg = AArch64::dsub;
  break;
default:
  LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Couldn't find appropriate subregister for register class."
; } } while (false)
      dbgs() << "Couldn't find appropriate subregister for register class.")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Couldn't find appropriate subregister for register class."
; } } while (false);
  return false;
}

return true;
593}

595/// Returns the minimum size the given register bank can hold.
596static unsigned getMinSizeForRegBank(const RegisterBank &RB) {
switch (RB.getID()) {
case AArch64::GPRRegBankID:
  return 32;
case AArch64::FPRRegBankID:
  return 8;
default:
  llvm_unreachable("Tried to get minimum size for unknown register bank.")::llvm::llvm_unreachable_internal("Tried to get minimum size for unknown register bank."
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 603);
}
605}

607/// Create a REG_SEQUENCE instruction using the registers in \p Regs.
608/// Helper function for functions like createDTuple and createQTuple.
609///
610/// \p RegClassIDs - The list of register class IDs available for some tuple of
611/// a scalar class. E.g. QQRegClassID, QQQRegClassID, QQQQRegClassID. This is
612/// expected to contain between 2 and 4 tuple classes.
613///
614/// \p SubRegs - The list of subregister classes associated with each register
615/// class ID in \p RegClassIDs. E.g., QQRegClassID should use the qsub0
616/// subregister class. The index of each subregister class is expected to
617/// correspond with the index of each register class.
618///
619/// \returns Either the destination register of REG_SEQUENCE instruction that
620/// was created, or the 0th element of \p Regs if \p Regs contains a single
621/// element.
622static Register createTuple(ArrayRef<Register> Regs,
                          const unsigned RegClassIDs[],
                          const unsigned SubRegs[], MachineIRBuilder &MIB) {
unsigned NumRegs = Regs.size();
if (NumRegs == 1)
  return Regs[0];
assert(NumRegs >= 2 && NumRegs <= 4 &&(static_cast <bool> (NumRegs >= 2 && NumRegs
 <= 4 && "Only support between two and 4 registers in a tuple!"
) ? void (0) : __assert_fail ("NumRegs >= 2 && NumRegs <= 4 && \"Only support between two and 4 registers in a tuple!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 629, __extension__ __PRETTY_FUNCTION__))
       "Only support between two and 4 registers in a tuple!")(static_cast <bool> (NumRegs >= 2 && NumRegs
 <= 4 && "Only support between two and 4 registers in a tuple!"
) ? void (0) : __assert_fail ("NumRegs >= 2 && NumRegs <= 4 && \"Only support between two and 4 registers in a tuple!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 629, __extension__ __PRETTY_FUNCTION__));
const TargetRegisterInfo *TRI = MIB.getMF().getSubtarget().getRegisterInfo();
auto *DesiredClass = TRI->getRegClass(RegClassIDs[NumRegs - 2]);
auto RegSequence =
    MIB.buildInstr(TargetOpcode::REG_SEQUENCE, {DesiredClass}, {});
for (unsigned I = 0, E = Regs.size(); I < E; ++I) {
  RegSequence.addUse(Regs[I]);
  RegSequence.addImm(SubRegs[I]);
}
return RegSequence.getReg(0);
639}

641/// Create a tuple of D-registers using the registers in \p Regs.
642static Register createDTuple(ArrayRef<Register> Regs, MachineIRBuilder &MIB) {
static const unsigned RegClassIDs[] = {
    AArch64::DDRegClassID, AArch64::DDDRegClassID, AArch64::DDDDRegClassID};
static const unsigned SubRegs[] = {AArch64::dsub0, AArch64::dsub1,
                                   AArch64::dsub2, AArch64::dsub3};
return createTuple(Regs, RegClassIDs, SubRegs, MIB);
648}

650/// Create a tuple of Q-registers using the registers in \p Regs.
651static Register createQTuple(ArrayRef<Register> Regs, MachineIRBuilder &MIB) {
static const unsigned RegClassIDs[] = {
    AArch64::QQRegClassID, AArch64::QQQRegClassID, AArch64::QQQQRegClassID};
static const unsigned SubRegs[] = {AArch64::qsub0, AArch64::qsub1,
                                   AArch64::qsub2, AArch64::qsub3};
return createTuple(Regs, RegClassIDs, SubRegs, MIB);
657}

659static Optional<uint64_t> getImmedFromMO(const MachineOperand &Root) {
auto &MI = *Root.getParent();
auto &MBB = *MI.getParent();
auto &MF = *MBB.getParent();
auto &MRI = MF.getRegInfo();
uint64_t Immed;
if (Root.isImm())
  Immed = Root.getImm();
else if (Root.isCImm())
  Immed = Root.getCImm()->getZExtValue();
else if (Root.isReg()) {
  auto ValAndVReg =
      getConstantVRegValWithLookThrough(Root.getReg(), MRI, true);
  if (!ValAndVReg)
    return None;
  Immed = ValAndVReg->Value.getSExtValue();
} else
  return None;
return Immed;
678}

680/// Check whether \p I is a currently unsupported binary operation:
681/// - it has an unsized type
682/// - an operand is not a vreg
683/// - all operands are not in the same bank
684/// These are checks that should someday live in the verifier, but right now,
685/// these are mostly limitations of the aarch64 selector.
686static bool unsupportedBinOp(const MachineInstr &I,
                           const AArch64RegisterBankInfo &RBI,
                           const MachineRegisterInfo &MRI,
                           const AArch64RegisterInfo &TRI) {
LLT Ty = MRI.getType(I.getOperand(0).getReg());
if (!Ty.isValid()) {
  LLVM_DEBUG(dbgs() << "Generic binop register should be typed\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Generic binop register should be typed\n"
; } } while (false);
  return true;
}

const RegisterBank *PrevOpBank = nullptr;
for (auto &MO : I.operands()) {
  // FIXME: Support non-register operands.
  if (!MO.isReg()) {
    LLVM_DEBUG(dbgs() << "Generic inst non-reg operands are unsupported\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Generic inst non-reg operands are unsupported\n"
; } } while (false);
    return true;
  }

  // FIXME: Can generic operations have physical registers operands? If
  // so, this will need to be taught about that, and we'll need to get the
  // bank out of the minimal class for the register.
  // Either way, this needs to be documented (and possibly verified).
  if (!Register::isVirtualRegister(MO.getReg())) {
    LLVM_DEBUG(dbgs() << "Generic inst has physical register operand\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Generic inst has physical register operand\n"
; } } while (false);
    return true;
  }

  const RegisterBank *OpBank = RBI.getRegBank(MO.getReg(), MRI, TRI);
  if (!OpBank) {
    LLVM_DEBUG(dbgs() << "Generic register has no bank or class\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Generic register has no bank or class\n"
; } } while (false);
    return true;
  }

  if (PrevOpBank && OpBank != PrevOpBank) {
    LLVM_DEBUG(dbgs() << "Generic inst operands have different banks\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Generic inst operands have different banks\n"
; } } while (false);
    return true;
  }
  PrevOpBank = OpBank;
}
return false;
726}

728/// Select the AArch64 opcode for the basic binary operation \p GenericOpc
729/// (such as G_OR or G_SDIV), appropriate for the register bank \p RegBankID
730/// and of size \p OpSize.
731/// \returns \p GenericOpc if the combination is unsupported.
732static unsigned selectBinaryOp(unsigned GenericOpc, unsigned RegBankID,
                             unsigned OpSize) {
switch (RegBankID) {
case AArch64::GPRRegBankID:
  if (OpSize == 32) {
    switch (GenericOpc) {
    case TargetOpcode::G_SHL:
      return AArch64::LSLVWr;
    case TargetOpcode::G_LSHR:
      return AArch64::LSRVWr;
    case TargetOpcode::G_ASHR:
      return AArch64::ASRVWr;
    default:
      return GenericOpc;
    }
  } else if (OpSize == 64) {
    switch (GenericOpc) {
    case TargetOpcode::G_PTR_ADD:
      return AArch64::ADDXrr;
    case TargetOpcode::G_SHL:
      return AArch64::LSLVXr;
    case TargetOpcode::G_LSHR:
      return AArch64::LSRVXr;
    case TargetOpcode::G_ASHR:
      return AArch64::ASRVXr;
    default:
      return GenericOpc;
    }
  }
  break;
case AArch64::FPRRegBankID:
  switch (OpSize) {
  case 32:
    switch (GenericOpc) {
    case TargetOpcode::G_FADD:
      return AArch64::FADDSrr;
    case TargetOpcode::G_FSUB:
      return AArch64::FSUBSrr;
    case TargetOpcode::G_FMUL:
      return AArch64::FMULSrr;
    case TargetOpcode::G_FDIV:
      return AArch64::FDIVSrr;
    default:
      return GenericOpc;
    }
  case 64:
    switch (GenericOpc) {
    case TargetOpcode::G_FADD:
      return AArch64::FADDDrr;
    case TargetOpcode::G_FSUB:
      return AArch64::FSUBDrr;
    case TargetOpcode::G_FMUL:
      return AArch64::FMULDrr;
    case TargetOpcode::G_FDIV:
      return AArch64::FDIVDrr;
    case TargetOpcode::G_OR:
      return AArch64::ORRv8i8;
    default:
      return GenericOpc;
    }
  }
  break;
}
return GenericOpc;
796}

798/// Select the AArch64 opcode for the G_LOAD or G_STORE operation \p GenericOpc,
799/// appropriate for the (value) register bank \p RegBankID and of memory access
800/// size \p OpSize.  This returns the variant with the base+unsigned-immediate
801/// addressing mode (e.g., LDRXui).
802/// \returns \p GenericOpc if the combination is unsupported.
803static unsigned selectLoadStoreUIOp(unsigned GenericOpc, unsigned RegBankID,
                                  unsigned OpSize) {
const bool isStore = GenericOpc == TargetOpcode::G_STORE;
switch (RegBankID) {
case AArch64::GPRRegBankID:
  switch (OpSize) {
  case 8:
    return isStore ? AArch64::STRBBui : AArch64::LDRBBui;
  case 16:
    return isStore ? AArch64::STRHHui : AArch64::LDRHHui;
  case 32:
    return isStore ? AArch64::STRWui : AArch64::LDRWui;
  case 64:
    return isStore ? AArch64::STRXui : AArch64::LDRXui;
  }
  break;
case AArch64::FPRRegBankID:
  switch (OpSize) {
  case 8:
    return isStore ? AArch64::STRBui : AArch64::LDRBui;
  case 16:
    return isStore ? AArch64::STRHui : AArch64::LDRHui;
  case 32:
    return isStore ? AArch64::STRSui : AArch64::LDRSui;
  case 64:
    return isStore ? AArch64::STRDui : AArch64::LDRDui;
  }
  break;
}
return GenericOpc;
833}

835#ifndef NDEBUG
836/// Helper function that verifies that we have a valid copy at the end of
837/// selectCopy. Verifies that the source and dest have the expected sizes and
838/// then returns true.
839static bool isValidCopy(const MachineInstr &I, const RegisterBank &DstBank,
                      const MachineRegisterInfo &MRI,
                      const TargetRegisterInfo &TRI,
                      const RegisterBankInfo &RBI) {
const Register DstReg = I.getOperand(0).getReg();
const Register SrcReg = I.getOperand(1).getReg();
const unsigned DstSize = RBI.getSizeInBits(DstReg, MRI, TRI);
const unsigned SrcSize = RBI.getSizeInBits(SrcReg, MRI, TRI);

// Make sure the size of the source and dest line up.
assert((static_cast <bool> ((DstSize == SrcSize || (Register::
isPhysicalRegister(SrcReg) && DstSize <= SrcSize) ||
 (((DstSize + 31) / 32 == (SrcSize + 31) / 32) && DstSize
 > SrcSize)) && "Copy with different width?!") ? void
 (0) : __assert_fail ("(DstSize == SrcSize || (Register::isPhysicalRegister(SrcReg) && DstSize <= SrcSize) || (((DstSize + 31) / 32 == (SrcSize + 31) / 32) && DstSize > SrcSize)) && \"Copy with different width?!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 858, __extension__ __PRETTY_FUNCTION__))
    (DstSize == SrcSize ||(static_cast <bool> ((DstSize == SrcSize || (Register::
isPhysicalRegister(SrcReg) && DstSize <= SrcSize) ||
 (((DstSize + 31) / 32 == (SrcSize + 31) / 32) && DstSize
 > SrcSize)) && "Copy with different width?!") ? void
 (0) : __assert_fail ("(DstSize == SrcSize || (Register::isPhysicalRegister(SrcReg) && DstSize <= SrcSize) || (((DstSize + 31) / 32 == (SrcSize + 31) / 32) && DstSize > SrcSize)) && \"Copy with different width?!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 858, __extension__ __PRETTY_FUNCTION__))
     // Copies are a mean to setup initial types, the number of(static_cast <bool> ((DstSize == SrcSize || (Register::
isPhysicalRegister(SrcReg) && DstSize <= SrcSize) ||
 (((DstSize + 31) / 32 == (SrcSize + 31) / 32) && DstSize
 > SrcSize)) && "Copy with different width?!") ? void
 (0) : __assert_fail ("(DstSize == SrcSize || (Register::isPhysicalRegister(SrcReg) && DstSize <= SrcSize) || (((DstSize + 31) / 32 == (SrcSize + 31) / 32) && DstSize > SrcSize)) && \"Copy with different width?!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 858, __extension__ __PRETTY_FUNCTION__))
     // bits may not exactly match.(static_cast <bool> ((DstSize == SrcSize || (Register::
isPhysicalRegister(SrcReg) && DstSize <= SrcSize) ||
 (((DstSize + 31) / 32 == (SrcSize + 31) / 32) && DstSize
 > SrcSize)) && "Copy with different width?!") ? void
 (0) : __assert_fail ("(DstSize == SrcSize || (Register::isPhysicalRegister(SrcReg) && DstSize <= SrcSize) || (((DstSize + 31) / 32 == (SrcSize + 31) / 32) && DstSize > SrcSize)) && \"Copy with different width?!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 858, __extension__ __PRETTY_FUNCTION__))
     (Register::isPhysicalRegister(SrcReg) && DstSize <= SrcSize) ||(static_cast <bool> ((DstSize == SrcSize || (Register::
isPhysicalRegister(SrcReg) && DstSize <= SrcSize) ||
 (((DstSize + 31) / 32 == (SrcSize + 31) / 32) && DstSize
 > SrcSize)) && "Copy with different width?!") ? void
 (0) : __assert_fail ("(DstSize == SrcSize || (Register::isPhysicalRegister(SrcReg) && DstSize <= SrcSize) || (((DstSize + 31) / 32 == (SrcSize + 31) / 32) && DstSize > SrcSize)) && \"Copy with different width?!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 858, __extension__ __PRETTY_FUNCTION__))
     // Copies are a mean to copy bits around, as long as we are(static_cast <bool> ((DstSize == SrcSize || (Register::
isPhysicalRegister(SrcReg) && DstSize <= SrcSize) ||
 (((DstSize + 31) / 32 == (SrcSize + 31) / 32) && DstSize
 > SrcSize)) && "Copy with different width?!") ? void
 (0) : __assert_fail ("(DstSize == SrcSize || (Register::isPhysicalRegister(SrcReg) && DstSize <= SrcSize) || (((DstSize + 31) / 32 == (SrcSize + 31) / 32) && DstSize > SrcSize)) && \"Copy with different width?!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 858, __extension__ __PRETTY_FUNCTION__))
     // on the same register class, that's fine. Otherwise, that(static_cast <bool> ((DstSize == SrcSize || (Register::
isPhysicalRegister(SrcReg) && DstSize <= SrcSize) ||
 (((DstSize + 31) / 32 == (SrcSize + 31) / 32) && DstSize
 > SrcSize)) && "Copy with different width?!") ? void
 (0) : __assert_fail ("(DstSize == SrcSize || (Register::isPhysicalRegister(SrcReg) && DstSize <= SrcSize) || (((DstSize + 31) / 32 == (SrcSize + 31) / 32) && DstSize > SrcSize)) && \"Copy with different width?!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 858, __extension__ __PRETTY_FUNCTION__))
     // means we need some SUBREG_TO_REG or AND & co.(static_cast <bool> ((DstSize == SrcSize || (Register::
isPhysicalRegister(SrcReg) && DstSize <= SrcSize) ||
 (((DstSize + 31) / 32 == (SrcSize + 31) / 32) && DstSize
 > SrcSize)) && "Copy with different width?!") ? void
 (0) : __assert_fail ("(DstSize == SrcSize || (Register::isPhysicalRegister(SrcReg) && DstSize <= SrcSize) || (((DstSize + 31) / 32 == (SrcSize + 31) / 32) && DstSize > SrcSize)) && \"Copy with different width?!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 858, __extension__ __PRETTY_FUNCTION__))
     (((DstSize + 31) / 32 == (SrcSize + 31) / 32) && DstSize > SrcSize)) &&(static_cast <bool> ((DstSize == SrcSize || (Register::
isPhysicalRegister(SrcReg) && DstSize <= SrcSize) ||
 (((DstSize + 31) / 32 == (SrcSize + 31) / 32) && DstSize
 > SrcSize)) && "Copy with different width?!") ? void
 (0) : __assert_fail ("(DstSize == SrcSize || (Register::isPhysicalRegister(SrcReg) && DstSize <= SrcSize) || (((DstSize + 31) / 32 == (SrcSize + 31) / 32) && DstSize > SrcSize)) && \"Copy with different width?!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 858, __extension__ __PRETTY_FUNCTION__))
    "Copy with different width?!")(static_cast <bool> ((DstSize == SrcSize || (Register::
isPhysicalRegister(SrcReg) && DstSize <= SrcSize) ||
 (((DstSize + 31) / 32 == (SrcSize + 31) / 32) && DstSize
 > SrcSize)) && "Copy with different width?!") ? void
 (0) : __assert_fail ("(DstSize == SrcSize || (Register::isPhysicalRegister(SrcReg) && DstSize <= SrcSize) || (((DstSize + 31) / 32 == (SrcSize + 31) / 32) && DstSize > SrcSize)) && \"Copy with different width?!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 858, __extension__ __PRETTY_FUNCTION__));

// Check the size of the destination.
assert((DstSize <= 64 || DstBank.getID() == AArch64::FPRRegBankID) &&(static_cast <bool> ((DstSize <= 64 || DstBank.getID
() == AArch64::FPRRegBankID) && "GPRs cannot get more than 64-bit width values"
) ? void (0) : __assert_fail ("(DstSize <= 64 || DstBank.getID() == AArch64::FPRRegBankID) && \"GPRs cannot get more than 64-bit width values\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 862, __extension__ __PRETTY_FUNCTION__))
       "GPRs cannot get more than 64-bit width values")(static_cast <bool> ((DstSize <= 64 || DstBank.getID
() == AArch64::FPRRegBankID) && "GPRs cannot get more than 64-bit width values"
) ? void (0) : __assert_fail ("(DstSize <= 64 || DstBank.getID() == AArch64::FPRRegBankID) && \"GPRs cannot get more than 64-bit width values\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 862, __extension__ __PRETTY_FUNCTION__));

return true;
865}
866#endif

868/// Helper function for selectCopy. Inserts a subregister copy from \p SrcReg
869/// to \p *To.
870///
871/// E.g "To = COPY SrcReg:SubReg"
872static bool copySubReg(MachineInstr &I, MachineRegisterInfo &MRI,
                     const RegisterBankInfo &RBI, Register SrcReg,
                     const TargetRegisterClass *To, unsigned SubReg) {
assert(SrcReg.isValid() && "Expected a valid source register?")(static_cast <bool> (SrcReg.isValid() && "Expected a valid source register?"
) ? void (0) : __assert_fail ("SrcReg.isValid() && \"Expected a valid source register?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 875, __extension__ __PRETTY_FUNCTION__));
assert(To && "Destination register class cannot be null")(static_cast <bool> (To && "Destination register class cannot be null"
) ? void (0) : __assert_fail ("To && \"Destination register class cannot be null\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 876, __extension__ __PRETTY_FUNCTION__));
assert(SubReg && "Expected a valid subregister")(static_cast <bool> (SubReg && "Expected a valid subregister"
) ? void (0) : __assert_fail ("SubReg && \"Expected a valid subregister\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 877, __extension__ __PRETTY_FUNCTION__));

MachineIRBuilder MIB(I);
auto SubRegCopy =
    MIB.buildInstr(TargetOpcode::COPY, {To}, {}).addReg(SrcReg, 0, SubReg);
MachineOperand &RegOp = I.getOperand(1);
RegOp.setReg(SubRegCopy.getReg(0));

// It's possible that the destination register won't be constrained. Make
// sure that happens.
if (!Register::isPhysicalRegister(I.getOperand(0).getReg()))
  RBI.constrainGenericRegister(I.getOperand(0).getReg(), *To, MRI);

return true;
891}

893/// Helper function to get the source and destination register classes for a
894/// copy. Returns a std::pair containing the source register class for the
895/// copy, and the destination register class for the copy. If a register class
896/// cannot be determined, then it will be nullptr.
897static std::pair<const TargetRegisterClass *, const TargetRegisterClass *>
898getRegClassesForCopy(MachineInstr &I, const TargetInstrInfo &TII,
                   MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI,
                   const RegisterBankInfo &RBI) {
Register DstReg = I.getOperand(0).getReg();
Register SrcReg = I.getOperand(1).getReg();
const RegisterBank &DstRegBank = *RBI.getRegBank(DstReg, MRI, TRI);
const RegisterBank &SrcRegBank = *RBI.getRegBank(SrcReg, MRI, TRI);
unsigned DstSize = RBI.getSizeInBits(DstReg, MRI, TRI);
unsigned SrcSize = RBI.getSizeInBits(SrcReg, MRI, TRI);

// Special casing for cross-bank copies of s1s. We can technically represent
// a 1-bit value with any size of register. The minimum size for a GPR is 32
// bits. So, we need to put the FPR on 32 bits as well.
//
// FIXME: I'm not sure if this case holds true outside of copies. If it does,
// then we can pull it into the helpers that get the appropriate class for a
// register bank. Or make a new helper that carries along some constraint
// information.
if (SrcRegBank != DstRegBank && (DstSize == 1 && SrcSize == 1))
  SrcSize = DstSize = 32;

return {getMinClassForRegBank(SrcRegBank, SrcSize, true),
        getMinClassForRegBank(DstRegBank, DstSize, true)};
921}

923static bool selectCopy(MachineInstr &I, const TargetInstrInfo &TII,
                     MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI,
                     const RegisterBankInfo &RBI) {
Register DstReg = I.getOperand(0).getReg();
Register SrcReg = I.getOperand(1).getReg();
const RegisterBank &DstRegBank = *RBI.getRegBank(DstReg, MRI, TRI);
const RegisterBank &SrcRegBank = *RBI.getRegBank(SrcReg, MRI, TRI);

// Find the correct register classes for the source and destination registers.
const TargetRegisterClass *SrcRC;
const TargetRegisterClass *DstRC;
std::tie(SrcRC, DstRC) = getRegClassesForCopy(I, TII, MRI, TRI, RBI);

if (!DstRC) {
  LLVM_DEBUG(dbgs() << "Unexpected dest size "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unexpected dest size " <<
 RBI.getSizeInBits(DstReg, MRI, TRI) << '\n'; } } while
 (false)
                    << RBI.getSizeInBits(DstReg, MRI, TRI) << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unexpected dest size " <<
 RBI.getSizeInBits(DstReg, MRI, TRI) << '\n'; } } while
 (false);
  return false;
}

// A couple helpers below, for making sure that the copy we produce is valid.

// Set to true if we insert a SUBREG_TO_REG. If we do this, then we don't want
// to verify that the src and dst are the same size, since that's handled by
// the SUBREG_TO_REG.
bool KnownValid = false;

// Returns true, or asserts if something we don't expect happens. Instead of
// returning true, we return isValidCopy() to ensure that we verify the
// result.
auto CheckCopy = [&]() {
  // If we have a bitcast or something, we can't have physical registers.
  assert((I.isCopy() ||(static_cast <bool> ((I.isCopy() || (!Register::isPhysicalRegister
(I.getOperand(0).getReg()) && !Register::isPhysicalRegister
(I.getOperand(1).getReg()))) && "No phys reg on generic operator!"
) ? void (0) : __assert_fail ("(I.isCopy() || (!Register::isPhysicalRegister(I.getOperand(0).getReg()) && !Register::isPhysicalRegister(I.getOperand(1).getReg()))) && \"No phys reg on generic operator!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 957, __extension__ __PRETTY_FUNCTION__))
          (!Register::isPhysicalRegister(I.getOperand(0).getReg()) &&(static_cast <bool> ((I.isCopy() || (!Register::isPhysicalRegister
(I.getOperand(0).getReg()) && !Register::isPhysicalRegister
(I.getOperand(1).getReg()))) && "No phys reg on generic operator!"
) ? void (0) : __assert_fail ("(I.isCopy() || (!Register::isPhysicalRegister(I.getOperand(0).getReg()) && !Register::isPhysicalRegister(I.getOperand(1).getReg()))) && \"No phys reg on generic operator!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 957, __extension__ __PRETTY_FUNCTION__))
           !Register::isPhysicalRegister(I.getOperand(1).getReg()))) &&(static_cast <bool> ((I.isCopy() || (!Register::isPhysicalRegister
(I.getOperand(0).getReg()) && !Register::isPhysicalRegister
(I.getOperand(1).getReg()))) && "No phys reg on generic operator!"
) ? void (0) : __assert_fail ("(I.isCopy() || (!Register::isPhysicalRegister(I.getOperand(0).getReg()) && !Register::isPhysicalRegister(I.getOperand(1).getReg()))) && \"No phys reg on generic operator!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 957, __extension__ __PRETTY_FUNCTION__))
         "No phys reg on generic operator!")(static_cast <bool> ((I.isCopy() || (!Register::isPhysicalRegister
(I.getOperand(0).getReg()) && !Register::isPhysicalRegister
(I.getOperand(1).getReg()))) && "No phys reg on generic operator!"
) ? void (0) : __assert_fail ("(I.isCopy() || (!Register::isPhysicalRegister(I.getOperand(0).getReg()) && !Register::isPhysicalRegister(I.getOperand(1).getReg()))) && \"No phys reg on generic operator!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 957, __extension__ __PRETTY_FUNCTION__));
  bool ValidCopy = true;
959#ifndef NDEBUG
  ValidCopy = KnownValid || isValidCopy(I, DstRegBank, MRI, TRI, RBI);
  assert(ValidCopy && "Invalid copy.")(static_cast <bool> (ValidCopy && "Invalid copy."
) ? void (0) : __assert_fail ("ValidCopy && \"Invalid copy.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 961, __extension__ __PRETTY_FUNCTION__));
962#endif
  (void)KnownValid;
  return ValidCopy;
};

// Is this a copy? If so, then we may need to insert a subregister copy.
if (I.isCopy()) {
  // Yes. Check if there's anything to fix up.
  if (!SrcRC) {
    LLVM_DEBUG(dbgs() << "Couldn't determine source register class\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Couldn't determine source register class\n"
; } } while (false);
    return false;
  }

  unsigned SrcSize = TRI.getRegSizeInBits(*SrcRC);
  unsigned DstSize = TRI.getRegSizeInBits(*DstRC);
  unsigned SubReg;

  // If the source bank doesn't support a subregister copy small enough,
  // then we first need to copy to the destination bank.
  if (getMinSizeForRegBank(SrcRegBank) > DstSize) {
    const TargetRegisterClass *DstTempRC =
        getMinClassForRegBank(DstRegBank, SrcSize, /* GetAllRegSet */ true);
    getSubRegForClass(DstRC, TRI, SubReg);

    MachineIRBuilder MIB(I);
    auto Copy = MIB.buildCopy({DstTempRC}, {SrcReg});
    copySubReg(I, MRI, RBI, Copy.getReg(0), DstRC, SubReg);
  } else if (SrcSize > DstSize) {
    // If the source register is bigger than the destination we need to
    // perform a subregister copy.
    const TargetRegisterClass *SubRegRC =
        getMinClassForRegBank(SrcRegBank, DstSize, /* GetAllRegSet */ true);
    getSubRegForClass(SubRegRC, TRI, SubReg);
    copySubReg(I, MRI, RBI, SrcReg, DstRC, SubReg);
  } else if (DstSize > SrcSize) {
    // If the destination register is bigger than the source we need to do
    // a promotion using SUBREG_TO_REG.
    const TargetRegisterClass *PromotionRC =
        getMinClassForRegBank(SrcRegBank, DstSize, /* GetAllRegSet */ true);
    getSubRegForClass(SrcRC, TRI, SubReg);

    Register PromoteReg = MRI.createVirtualRegister(PromotionRC);
    BuildMI(*I.getParent(), I, I.getDebugLoc(),
            TII.get(AArch64::SUBREG_TO_REG), PromoteReg)
        .addImm(0)
        .addUse(SrcReg)
        .addImm(SubReg);
    MachineOperand &RegOp = I.getOperand(1);
    RegOp.setReg(PromoteReg);

    // Promise that the copy is implicitly validated by the SUBREG_TO_REG.
    KnownValid = true;
  }

  // If the destination is a physical register, then there's nothing to
  // change, so we're done.
  if (Register::isPhysicalRegister(DstReg))
    return CheckCopy();
}

// No need to constrain SrcReg. It will get constrained when we hit another
// of its use or its defs. Copies do not have constraints.
if (!RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) {
  LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Failed to constrain " <<
 TII.getName(I.getOpcode()) << " operand\n"; } } while (
false)
                    << " operand\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Failed to constrain " <<
 TII.getName(I.getOpcode()) << " operand\n"; } } while (
false);
  return false;
}

// If this a GPR ZEXT that we want to just reduce down into a copy.
// The sizes will be mismatched with the source < 32b but that's ok.
if (I.getOpcode() == TargetOpcode::G_ZEXT) {
  I.setDesc(TII.get(AArch64::COPY));
  assert(SrcRegBank.getID() == AArch64::GPRRegBankID)(static_cast <bool> (SrcRegBank.getID() == AArch64::GPRRegBankID
) ? void (0) : __assert_fail ("SrcRegBank.getID() == AArch64::GPRRegBankID"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1034, __extension__ __PRETTY_FUNCTION__));
  return selectCopy(I, TII, MRI, TRI, RBI);
}

I.setDesc(TII.get(AArch64::COPY));
return CheckCopy();
1040}

1042static unsigned selectFPConvOpc(unsigned GenericOpc, LLT DstTy, LLT SrcTy) {
if (!DstTy.isScalar() || !SrcTy.isScalar())
  return GenericOpc;

const unsigned DstSize = DstTy.getSizeInBits();
const unsigned SrcSize = SrcTy.getSizeInBits();

switch (DstSize) {
case 32:
  switch (SrcSize) {
  case 32:
    switch (GenericOpc) {
    case TargetOpcode::G_SITOFP:
      return AArch64::SCVTFUWSri;
    case TargetOpcode::G_UITOFP:
      return AArch64::UCVTFUWSri;
    case TargetOpcode::G_FPTOSI:
      return AArch64::FCVTZSUWSr;
    case TargetOpcode::G_FPTOUI:
      return AArch64::FCVTZUUWSr;
    default:
      return GenericOpc;
    }
  case 64:
    switch (GenericOpc) {
    case TargetOpcode::G_SITOFP:
      return AArch64::SCVTFUXSri;
    case TargetOpcode::G_UITOFP:
      return AArch64::UCVTFUXSri;
    case TargetOpcode::G_FPTOSI:
      return AArch64::FCVTZSUWDr;
    case TargetOpcode::G_FPTOUI:
      return AArch64::FCVTZUUWDr;
    default:
      return GenericOpc;
    }
  default:
    return GenericOpc;
  }
case 64:
  switch (SrcSize) {
  case 32:
    switch (GenericOpc) {
    case TargetOpcode::G_SITOFP:
      return AArch64::SCVTFUWDri;
    case TargetOpcode::G_UITOFP:
      return AArch64::UCVTFUWDri;
    case TargetOpcode::G_FPTOSI:
      return AArch64::FCVTZSUXSr;
    case TargetOpcode::G_FPTOUI:
      return AArch64::FCVTZUUXSr;
    default:
      return GenericOpc;
    }
  case 64:
    switch (GenericOpc) {
    case TargetOpcode::G_SITOFP:
      return AArch64::SCVTFUXDri;
    case TargetOpcode::G_UITOFP:
      return AArch64::UCVTFUXDri;
    case TargetOpcode::G_FPTOSI:
      return AArch64::FCVTZSUXDr;
    case TargetOpcode::G_FPTOUI:
      return AArch64::FCVTZUUXDr;
    default:
      return GenericOpc;
    }
  default:
    return GenericOpc;
  }
default:
  return GenericOpc;
};
return GenericOpc;
1116}

1118MachineInstr *
1119AArch64InstructionSelector::emitSelect(Register Dst, Register True,
                                     Register False, AArch64CC::CondCode CC,
                                     MachineIRBuilder &MIB) const {
MachineRegisterInfo &MRI = *MIB.getMRI();
assert(RBI.getRegBank(False, MRI, TRI)->getID() ==(static_cast <bool> (RBI.getRegBank(False, MRI, TRI)->
getID() == RBI.getRegBank(True, MRI, TRI)->getID() &&
 "Expected both select operands to have the same regbank?") ?
 void (0) : __assert_fail ("RBI.getRegBank(False, MRI, TRI)->getID() == RBI.getRegBank(True, MRI, TRI)->getID() && \"Expected both select operands to have the same regbank?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1125, __extension__ __PRETTY_FUNCTION__))
           RBI.getRegBank(True, MRI, TRI)->getID() &&(static_cast <bool> (RBI.getRegBank(False, MRI, TRI)->
getID() == RBI.getRegBank(True, MRI, TRI)->getID() &&
 "Expected both select operands to have the same regbank?") ?
 void (0) : __assert_fail ("RBI.getRegBank(False, MRI, TRI)->getID() == RBI.getRegBank(True, MRI, TRI)->getID() && \"Expected both select operands to have the same regbank?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1125, __extension__ __PRETTY_FUNCTION__))
       "Expected both select operands to have the same regbank?")(static_cast <bool> (RBI.getRegBank(False, MRI, TRI)->
getID() == RBI.getRegBank(True, MRI, TRI)->getID() &&
 "Expected both select operands to have the same regbank?") ?
 void (0) : __assert_fail ("RBI.getRegBank(False, MRI, TRI)->getID() == RBI.getRegBank(True, MRI, TRI)->getID() && \"Expected both select operands to have the same regbank?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1125, __extension__ __PRETTY_FUNCTION__));
LLT Ty = MRI.getType(True);
if (Ty.isVector())
  return nullptr;
const unsigned Size = Ty.getSizeInBits();
assert((Size == 32 || Size == 64) &&(static_cast <bool> ((Size == 32 || Size == 64) &&
 "Expected 32 bit or 64 bit select only?") ? void (0) : __assert_fail
 ("(Size == 32 || Size == 64) && \"Expected 32 bit or 64 bit select only?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1131, __extension__ __PRETTY_FUNCTION__))
       "Expected 32 bit or 64 bit select only?")(static_cast <bool> ((Size == 32 || Size == 64) &&
 "Expected 32 bit or 64 bit select only?") ? void (0) : __assert_fail
 ("(Size == 32 || Size == 64) && \"Expected 32 bit or 64 bit select only?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1131, __extension__ __PRETTY_FUNCTION__));
const bool Is32Bit = Size == 32;
if (RBI.getRegBank(True, MRI, TRI)->getID() != AArch64::GPRRegBankID) {
  unsigned Opc = Is32Bit ? AArch64::FCSELSrrr : AArch64::FCSELDrrr;
  auto FCSel = MIB.buildInstr(Opc, {Dst}, {True, False}).addImm(CC);
  constrainSelectedInstRegOperands(*FCSel, TII, TRI, RBI);
  return &*FCSel;
}

// By default, we'll try and emit a CSEL.
unsigned Opc = Is32Bit ? AArch64::CSELWr : AArch64::CSELXr;
bool Optimized = false;
auto TryFoldBinOpIntoSelect = [&Opc, Is32Bit, &CC, &MRI,
                               &Optimized](Register &Reg, Register &OtherReg,
                                           bool Invert) {
  if (Optimized)
    return false;

  // Attempt to fold:
  //
  // %sub = G_SUB 0, %x
  // %select = G_SELECT cc, %reg, %sub
  //
  // Into:
  // %select = CSNEG %reg, %x, cc
  Register MatchReg;
  if (mi_match(Reg, MRI, m_Neg(m_Reg(MatchReg)))) {
    Opc = Is32Bit ? AArch64::CSNEGWr : AArch64::CSNEGXr;
    Reg = MatchReg;
    if (Invert) {
      CC = AArch64CC::getInvertedCondCode(CC);
      std::swap(Reg, OtherReg);
    }
    return true;
  }

  // Attempt to fold:
  //
  // %xor = G_XOR %x, -1
  // %select = G_SELECT cc, %reg, %xor
  //
  // Into:
  // %select = CSINV %reg, %x, cc
  if (mi_match(Reg, MRI, m_Not(m_Reg(MatchReg)))) {
    Opc = Is32Bit ? AArch64::CSINVWr : AArch64::CSINVXr;
    Reg = MatchReg;
    if (Invert) {
      CC = AArch64CC::getInvertedCondCode(CC);
      std::swap(Reg, OtherReg);
    }
    return true;
  }

  // Attempt to fold:
  //
  // %add = G_ADD %x, 1
  // %select = G_SELECT cc, %reg, %add
  //
  // Into:
  // %select = CSINC %reg, %x, cc
  if (mi_match(Reg, MRI,
               m_any_of(m_GAdd(m_Reg(MatchReg), m_SpecificICst(1)),
                        m_GPtrAdd(m_Reg(MatchReg), m_SpecificICst(1))))) {
    Opc = Is32Bit ? AArch64::CSINCWr : AArch64::CSINCXr;
    Reg = MatchReg;
    if (Invert) {
      CC = AArch64CC::getInvertedCondCode(CC);
      std::swap(Reg, OtherReg);
    }
    return true;
  }

  return false;
};

// Helper lambda which tries to use CSINC/CSINV for the instruction when its
// true/false values are constants.
// FIXME: All of these patterns already exist in tablegen. We should be
// able to import these.
auto TryOptSelectCst = [&Opc, &True, &False, &CC, Is32Bit, &MRI,
                        &Optimized]() {
  if (Optimized)
    return false;
  auto TrueCst = getConstantVRegValWithLookThrough(True, MRI);
  auto FalseCst = getConstantVRegValWithLookThrough(False, MRI);
  if (!TrueCst && !FalseCst)
    return false;

  Register ZReg = Is32Bit ? AArch64::WZR : AArch64::XZR;
  if (TrueCst && FalseCst) {
    int64_t T = TrueCst->Value.getSExtValue();
    int64_t F = FalseCst->Value.getSExtValue();

    if (T == 0 && F == 1) {
      // G_SELECT cc, 0, 1 -> CSINC zreg, zreg, cc
      Opc = Is32Bit ? AArch64::CSINCWr : AArch64::CSINCXr;
      True = ZReg;
      False = ZReg;
      return true;
    }

    if (T == 0 && F == -1) {
      // G_SELECT cc 0, -1 -> CSINV zreg, zreg cc
      Opc = Is32Bit ? AArch64::CSINVWr : AArch64::CSINVXr;
      True = ZReg;
      False = ZReg;
      return true;
    }
  }

  if (TrueCst) {
    int64_t T = TrueCst->Value.getSExtValue();
    if (T == 1) {
      // G_SELECT cc, 1, f -> CSINC f, zreg, inv_cc
      Opc = Is32Bit ? AArch64::CSINCWr : AArch64::CSINCXr;
      True = False;
      False = ZReg;
      CC = AArch64CC::getInvertedCondCode(CC);
      return true;
    }

    if (T == -1) {
      // G_SELECT cc, -1, f -> CSINV f, zreg, inv_cc
      Opc = Is32Bit ? AArch64::CSINVWr : AArch64::CSINVXr;
      True = False;
      False = ZReg;
      CC = AArch64CC::getInvertedCondCode(CC);
      return true;
    }
  }

  if (FalseCst) {
    int64_t F = FalseCst->Value.getSExtValue();
    if (F == 1) {
      // G_SELECT cc, t, 1 -> CSINC t, zreg, cc
      Opc = Is32Bit ? AArch64::CSINCWr : AArch64::CSINCXr;
      False = ZReg;
      return true;
    }

    if (F == -1) {
      // G_SELECT cc, t, -1 -> CSINC t, zreg, cc
      Opc = Is32Bit ? AArch64::CSINVWr : AArch64::CSINVXr;
      False = ZReg;
      return true;
    }
  }
  return false;
};

Optimized |= TryFoldBinOpIntoSelect(False, True, /*Invert = */ false);
Optimized |= TryFoldBinOpIntoSelect(True, False, /*Invert = */ true);
Optimized |= TryOptSelectCst();
auto SelectInst = MIB.buildInstr(Opc, {Dst}, {True, False}).addImm(CC);
constrainSelectedInstRegOperands(*SelectInst, TII, TRI, RBI);
return &*SelectInst;
1287}

1289static AArch64CC::CondCode changeICMPPredToAArch64CC(CmpInst::Predicate P) {
switch (P) {
default:
  llvm_unreachable("Unknown condition code!")::llvm::llvm_unreachable_internal("Unknown condition code!", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1292);
case CmpInst::ICMP_NE:
  return AArch64CC::NE;
case CmpInst::ICMP_EQ:
  return AArch64CC::EQ;
case CmpInst::ICMP_SGT:
  return AArch64CC::GT;
case CmpInst::ICMP_SGE:
  return AArch64CC::GE;
case CmpInst::ICMP_SLT:
  return AArch64CC::LT;
case CmpInst::ICMP_SLE:
  return AArch64CC::LE;
case CmpInst::ICMP_UGT:
  return AArch64CC::HI;
case CmpInst::ICMP_UGE:
  return AArch64CC::HS;
case CmpInst::ICMP_ULT:
  return AArch64CC::LO;
case CmpInst::ICMP_ULE:
  return AArch64CC::LS;
}
1314}

1316/// Return a register which can be used as a bit to test in a TB(N)Z.
1317static Register getTestBitReg(Register Reg, uint64_t &Bit, bool &Invert,
                            MachineRegisterInfo &MRI) {
assert(Reg.isValid() && "Expected valid register!")(static_cast <bool> (Reg.isValid() && "Expected valid register!"
) ? void (0) : __assert_fail ("Reg.isValid() && \"Expected valid register!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1319, __extension__ __PRETTY_FUNCTION__));
while (MachineInstr *MI = getDefIgnoringCopies(Reg, MRI)) {
  unsigned Opc = MI->getOpcode();

  if (!MI->getOperand(0).isReg() ||
      !MRI.hasOneNonDBGUse(MI->getOperand(0).getReg()))
    break;

  // (tbz (any_ext x), b) -> (tbz x, b) if we don't use the extended bits.
  //
  // (tbz (trunc x), b) -> (tbz x, b) is always safe, because the bit number
  // on the truncated x is the same as the bit number on x.
  if (Opc == TargetOpcode::G_ANYEXT || Opc == TargetOpcode::G_ZEXT ||
      Opc == TargetOpcode::G_TRUNC) {
    Register NextReg = MI->getOperand(1).getReg();
    // Did we find something worth folding?
    if (!NextReg.isValid() || !MRI.hasOneNonDBGUse(NextReg))
      break;

    // NextReg is worth folding. Keep looking.
    Reg = NextReg;
    continue;
  }

  // Attempt to find a suitable operation with a constant on one side.
  Optional<uint64_t> C;
  Register TestReg;
  switch (Opc) {
  default:
    break;
  case TargetOpcode::G_AND:
  case TargetOpcode::G_XOR: {
    TestReg = MI->getOperand(1).getReg();
    Register ConstantReg = MI->getOperand(2).getReg();
    auto VRegAndVal = getConstantVRegValWithLookThrough(ConstantReg, MRI);
    if (!VRegAndVal) {
      // AND commutes, check the other side for a constant.
      // FIXME: Can we canonicalize the constant so that it's always on the
      // same side at some point earlier?
      std::swap(ConstantReg, TestReg);
      VRegAndVal = getConstantVRegValWithLookThrough(ConstantReg, MRI);
    }
    if (VRegAndVal)
      C = VRegAndVal->Value.getSExtValue();
    break;
  }
  case TargetOpcode::G_ASHR:
  case TargetOpcode::G_LSHR:
  case TargetOpcode::G_SHL: {
    TestReg = MI->getOperand(1).getReg();
    auto VRegAndVal =
        getConstantVRegValWithLookThrough(MI->getOperand(2).getReg(), MRI);
    if (VRegAndVal)
      C = VRegAndVal->Value.getSExtValue();
    break;
  }
  }

  // Didn't find a constant or viable register. Bail out of the loop.
  if (!C || !TestReg.isValid())
    break;

  // We found a suitable instruction with a constant. Check to see if we can
  // walk through the instruction.
  Register NextReg;
  unsigned TestRegSize = MRI.getType(TestReg).getSizeInBits();
  switch (Opc) {
  default:
    break;
  case TargetOpcode::G_AND:
    // (tbz (and x, m), b) -> (tbz x, b) when the b-th bit of m is set.
    if ((*C >> Bit) & 1)
      NextReg = TestReg;
    break;
  case TargetOpcode::G_SHL:
    // (tbz (shl x, c), b) -> (tbz x, b-c) when b-c is positive and fits in
    // the type of the register.
    if (*C <= Bit && (Bit - *C) < TestRegSize) {
      NextReg = TestReg;
      Bit = Bit - *C;
    }
    break;
  case TargetOpcode::G_ASHR:
    // (tbz (ashr x, c), b) -> (tbz x, b+c) or (tbz x, msb) if b+c is > # bits
    // in x
    NextReg = TestReg;
    Bit = Bit + *C;
    if (Bit >= TestRegSize)
      Bit = TestRegSize - 1;
    break;
  case TargetOpcode::G_LSHR:
    // (tbz (lshr x, c), b) -> (tbz x, b+c) when b + c is < # bits in x
    if ((Bit + *C) < TestRegSize) {
      NextReg = TestReg;
      Bit = Bit + *C;
    }
    break;
  case TargetOpcode::G_XOR:
    // We can walk through a G_XOR by inverting whether we use tbz/tbnz when
    // appropriate.
    //
    // e.g. If x' = xor x, c, and the b-th bit is set in c then
    //
    // tbz x', b -> tbnz x, b
    //
    // Because x' only has the b-th bit set if x does not.
    if ((*C >> Bit) & 1)
      Invert = !Invert;
    NextReg = TestReg;
    break;
  }

  // Check if we found anything worth folding.
  if (!NextReg.isValid())
    return Reg;
  Reg = NextReg;
}

return Reg;
1438}

1440MachineInstr *AArch64InstructionSelector::emitTestBit(
  Register TestReg, uint64_t Bit, bool IsNegative, MachineBasicBlock *DstMBB,
  MachineIRBuilder &MIB) const {
assert(TestReg.isValid())(static_cast <bool> (TestReg.isValid()) ? void (0) : __assert_fail
 ("TestReg.isValid()", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1443, __extension__ __PRETTY_FUNCTION__));
assert(ProduceNonFlagSettingCondBr &&(static_cast <bool> (ProduceNonFlagSettingCondBr &&
 "Cannot emit TB(N)Z with speculation tracking!") ? void (0) :
 __assert_fail ("ProduceNonFlagSettingCondBr && \"Cannot emit TB(N)Z with speculation tracking!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1445, __extension__ __PRETTY_FUNCTION__))
       "Cannot emit TB(N)Z with speculation tracking!")(static_cast <bool> (ProduceNonFlagSettingCondBr &&
 "Cannot emit TB(N)Z with speculation tracking!") ? void (0) :
 __assert_fail ("ProduceNonFlagSettingCondBr && \"Cannot emit TB(N)Z with speculation tracking!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1445, __extension__ __PRETTY_FUNCTION__));
MachineRegisterInfo &MRI = *MIB.getMRI();

// Attempt to optimize the test bit by walking over instructions.
TestReg = getTestBitReg(TestReg, Bit, IsNegative, MRI);
LLT Ty = MRI.getType(TestReg);
unsigned Size = Ty.getSizeInBits();
assert(!Ty.isVector() && "Expected a scalar!")(static_cast <bool> (!Ty.isVector() && "Expected a scalar!"
) ? void (0) : __assert_fail ("!Ty.isVector() && \"Expected a scalar!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1452, __extension__ __PRETTY_FUNCTION__));
assert(Bit < 64 && "Bit is too large!")(static_cast <bool> (Bit < 64 && "Bit is too large!"
) ? void (0) : __assert_fail ("Bit < 64 && \"Bit is too large!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1453, __extension__ __PRETTY_FUNCTION__));

// When the test register is a 64-bit register, we have to narrow to make
// TBNZW work.
bool UseWReg = Bit < 32;
unsigned NecessarySize = UseWReg ? 32 : 64;
if (Size != NecessarySize)
  TestReg = moveScalarRegClass(
      TestReg, UseWReg ? AArch64::GPR32RegClass : AArch64::GPR64RegClass,
      MIB);

static const unsigned OpcTable[2][2] = {{AArch64::TBZX, AArch64::TBNZX},
                                        {AArch64::TBZW, AArch64::TBNZW}};
unsigned Opc = OpcTable[UseWReg][IsNegative];
auto TestBitMI =
    MIB.buildInstr(Opc).addReg(TestReg).addImm(Bit).addMBB(DstMBB);
constrainSelectedInstRegOperands(*TestBitMI, TII, TRI, RBI);
return &*TestBitMI;
1471}

1473bool AArch64InstructionSelector::tryOptAndIntoCompareBranch(
  MachineInstr &AndInst, bool Invert, MachineBasicBlock *DstMBB,
  MachineIRBuilder &MIB) const {
assert(AndInst.getOpcode() == TargetOpcode::G_AND && "Expected G_AND only?")(static_cast <bool> (AndInst.getOpcode() == TargetOpcode
::G_AND && "Expected G_AND only?") ? void (0) : __assert_fail
 ("AndInst.getOpcode() == TargetOpcode::G_AND && \"Expected G_AND only?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1476, __extension__ __PRETTY_FUNCTION__));
// Given something like this:
//
//  %x = ...Something...
//  %one = G_CONSTANT i64 1
//  %zero = G_CONSTANT i64 0
//  %and = G_AND %x, %one
//  %cmp = G_ICMP intpred(ne), %and, %zero
//  %cmp_trunc = G_TRUNC %cmp
//  G_BRCOND %cmp_trunc, %bb.3
//
// We want to try and fold the AND into the G_BRCOND and produce either a
// TBNZ (when we have intpred(ne)) or a TBZ (when we have intpred(eq)).
//
// In this case, we'd get
//
// TBNZ %x %bb.3
//

// Check if the AND has a constant on its RHS which we can use as a mask.
// If it's a power of 2, then it's the same as checking a specific bit.
// (e.g, ANDing with 8 == ANDing with 000...100 == testing if bit 3 is set)
auto MaybeBit = getConstantVRegValWithLookThrough(
    AndInst.getOperand(2).getReg(), *MIB.getMRI());
if (!MaybeBit)
  return false;

int32_t Bit = MaybeBit->Value.exactLogBase2();
if (Bit < 0)
  return false;

Register TestReg = AndInst.getOperand(1).getReg();

// Emit a TB(N)Z.
emitTestBit(TestReg, Bit, Invert, DstMBB, MIB);
return true;
1512}

1514MachineInstr *AArch64InstructionSelector::emitCBZ(Register CompareReg,
                                                bool IsNegative,
                                                MachineBasicBlock *DestMBB,
                                                MachineIRBuilder &MIB) const {
assert(ProduceNonFlagSettingCondBr && "CBZ does not set flags!")(static_cast <bool> (ProduceNonFlagSettingCondBr &&
 "CBZ does not set flags!") ? void (0) : __assert_fail ("ProduceNonFlagSettingCondBr && \"CBZ does not set flags!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1518, __extension__ __PRETTY_FUNCTION__));
MachineRegisterInfo &MRI = *MIB.getMRI();
assert(RBI.getRegBank(CompareReg, MRI, TRI)->getID() ==(static_cast <bool> (RBI.getRegBank(CompareReg, MRI, TRI
)->getID() == AArch64::GPRRegBankID && "Expected GPRs only?"
) ? void (0) : __assert_fail ("RBI.getRegBank(CompareReg, MRI, TRI)->getID() == AArch64::GPRRegBankID && \"Expected GPRs only?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1522, __extension__ __PRETTY_FUNCTION__))
           AArch64::GPRRegBankID &&(static_cast <bool> (RBI.getRegBank(CompareReg, MRI, TRI
)->getID() == AArch64::GPRRegBankID && "Expected GPRs only?"
) ? void (0) : __assert_fail ("RBI.getRegBank(CompareReg, MRI, TRI)->getID() == AArch64::GPRRegBankID && \"Expected GPRs only?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1522, __extension__ __PRETTY_FUNCTION__))
       "Expected GPRs only?")(static_cast <bool> (RBI.getRegBank(CompareReg, MRI, TRI
)->getID() == AArch64::GPRRegBankID && "Expected GPRs only?"
) ? void (0) : __assert_fail ("RBI.getRegBank(CompareReg, MRI, TRI)->getID() == AArch64::GPRRegBankID && \"Expected GPRs only?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1522, __extension__ __PRETTY_FUNCTION__));
auto Ty = MRI.getType(CompareReg);
unsigned Width = Ty.getSizeInBits();
assert(!Ty.isVector() && "Expected scalar only?")(static_cast <bool> (!Ty.isVector() && "Expected scalar only?"
) ? void (0) : __assert_fail ("!Ty.isVector() && \"Expected scalar only?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1525, __extension__ __PRETTY_FUNCTION__));
assert(Width <= 64 && "Expected width to be at most 64?")(static_cast <bool> (Width <= 64 && "Expected width to be at most 64?"
) ? void (0) : __assert_fail ("Width <= 64 && \"Expected width to be at most 64?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1526, __extension__ __PRETTY_FUNCTION__));
static const unsigned OpcTable[2][2] = {{AArch64::CBZW, AArch64::CBZX},
                                        {AArch64::CBNZW, AArch64::CBNZX}};
unsigned Opc = OpcTable[IsNegative][Width == 64];
auto BranchMI = MIB.buildInstr(Opc, {}, {CompareReg}).addMBB(DestMBB);
constrainSelectedInstRegOperands(*BranchMI, TII, TRI, RBI);
return &*BranchMI;
1533}

1535bool AArch64InstructionSelector::selectCompareBranchFedByFCmp(
  MachineInstr &I, MachineInstr &FCmp, MachineIRBuilder &MIB) const {
assert(FCmp.getOpcode() == TargetOpcode::G_FCMP)(static_cast <bool> (FCmp.getOpcode() == TargetOpcode::
G_FCMP) ? void (0) : __assert_fail ("FCmp.getOpcode() == TargetOpcode::G_FCMP"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1537, __extension__ __PRETTY_FUNCTION__));
assert(I.getOpcode() == TargetOpcode::G_BRCOND)(static_cast <bool> (I.getOpcode() == TargetOpcode::G_BRCOND
) ? void (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_BRCOND"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1538, __extension__ __PRETTY_FUNCTION__));
// Unfortunately, the mapping of LLVM FP CC's onto AArch64 CC's isn't
// totally clean.  Some of them require two branches to implement.
auto Pred = (CmpInst::Predicate)FCmp.getOperand(1).getPredicate();
emitFPCompare(FCmp.getOperand(2).getReg(), FCmp.getOperand(3).getReg(), MIB,
              Pred);
AArch64CC::CondCode CC1, CC2;
changeFCMPPredToAArch64CC(static_cast<CmpInst::Predicate>(Pred), CC1, CC2);
MachineBasicBlock *DestMBB = I.getOperand(1).getMBB();
MIB.buildInstr(AArch64::Bcc, {}, {}).addImm(CC1).addMBB(DestMBB);
if (CC2 != AArch64CC::AL)
  MIB.buildInstr(AArch64::Bcc, {}, {}).addImm(CC2).addMBB(DestMBB);
I.eraseFromParent();
return true;
1552}

1554bool AArch64InstructionSelector::tryOptCompareBranchFedByICmp(
  MachineInstr &I, MachineInstr &ICmp, MachineIRBuilder &MIB) const {
assert(ICmp.getOpcode() == TargetOpcode::G_ICMP)(static_cast <bool> (ICmp.getOpcode() == TargetOpcode::
G_ICMP) ? void (0) : __assert_fail ("ICmp.getOpcode() == TargetOpcode::G_ICMP"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1556, __extension__ __PRETTY_FUNCTION__));
assert(I.getOpcode() == TargetOpcode::G_BRCOND)(static_cast <bool> (I.getOpcode() == TargetOpcode::G_BRCOND
) ? void (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_BRCOND"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1557, __extension__ __PRETTY_FUNCTION__));
// Attempt to optimize the G_BRCOND + G_ICMP into a TB(N)Z/CB(N)Z.
//
// Speculation tracking/SLH assumes that optimized TB(N)Z/CB(N)Z
// instructions will not be produced, as they are conditional branch
// instructions that do not set flags.
if (!ProduceNonFlagSettingCondBr)
  return false;

MachineRegisterInfo &MRI = *MIB.getMRI();
MachineBasicBlock *DestMBB = I.getOperand(1).getMBB();
auto Pred =
    static_cast<CmpInst::Predicate>(ICmp.getOperand(1).getPredicate());
Register LHS = ICmp.getOperand(2).getReg();
Register RHS = ICmp.getOperand(3).getReg();

// We're allowed to emit a TB(N)Z/CB(N)Z. Try to do that.
auto VRegAndVal = getConstantVRegValWithLookThrough(RHS, MRI);
MachineInstr *AndInst = getOpcodeDef(TargetOpcode::G_AND, LHS, MRI);

// When we can emit a TB(N)Z, prefer that.
//
// Handle non-commutative condition codes first.
// Note that we don't want to do this when we have a G_AND because it can
// become a tst. The tst will make the test bit in the TB(N)Z redundant.
if (VRegAndVal && !AndInst) {
  int64_t C = VRegAndVal->Value.getSExtValue();

  // When we have a greater-than comparison, we can just test if the msb is
  // zero.
  if (C == -1 && Pred == CmpInst::ICMP_SGT) {
    uint64_t Bit = MRI.getType(LHS).getSizeInBits() - 1;
    emitTestBit(LHS, Bit, /*IsNegative = */ false, DestMBB, MIB);
    I.eraseFromParent();
    return true;
  }

  // When we have a less than comparison, we can just test if the msb is not
  // zero.
  if (C == 0 && Pred == CmpInst::ICMP_SLT) {
    uint64_t Bit = MRI.getType(LHS).getSizeInBits() - 1;
    emitTestBit(LHS, Bit, /*IsNegative = */ true, DestMBB, MIB);
    I.eraseFromParent();
    return true;
  }
}

// Attempt to handle commutative condition codes. Right now, that's only
// eq/ne.
if (ICmpInst::isEquality(Pred)) {
  if (!VRegAndVal) {
    std::swap(RHS, LHS);
    VRegAndVal = getConstantVRegValWithLookThrough(RHS, MRI);
    AndInst = getOpcodeDef(TargetOpcode::G_AND, LHS, MRI);
  }

  if (VRegAndVal && VRegAndVal->Value == 0) {
    // If there's a G_AND feeding into this branch, try to fold it away by
    // emitting a TB(N)Z instead.
    //
    // Note: If we have LT, then it *is* possible to fold, but it wouldn't be
    // beneficial. When we have an AND and LT, we need a TST/ANDS, so folding
    // would be redundant.
    if (AndInst &&
        tryOptAndIntoCompareBranch(
            *AndInst, /*Invert = */ Pred == CmpInst::ICMP_NE, DestMBB, MIB)) {
      I.eraseFromParent();
      return true;
    }

    // Otherwise, try to emit a CB(N)Z instead.
    auto LHSTy = MRI.getType(LHS);
    if (!LHSTy.isVector() && LHSTy.getSizeInBits() <= 64) {
      emitCBZ(LHS, /*IsNegative = */ Pred == CmpInst::ICMP_NE, DestMBB, MIB);
      I.eraseFromParent();
      return true;
    }
  }
}

return false;
1638}

1640bool AArch64InstructionSelector::selectCompareBranchFedByICmp(
  MachineInstr &I, MachineInstr &ICmp, MachineIRBuilder &MIB) const {
assert(ICmp.getOpcode() == TargetOpcode::G_ICMP)(static_cast <bool> (ICmp.getOpcode() == TargetOpcode::
G_ICMP) ? void (0) : __assert_fail ("ICmp.getOpcode() == TargetOpcode::G_ICMP"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1642, __extension__ __PRETTY_FUNCTION__));
assert(I.getOpcode() == TargetOpcode::G_BRCOND)(static_cast <bool> (I.getOpcode() == TargetOpcode::G_BRCOND
) ? void (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_BRCOND"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1643, __extension__ __PRETTY_FUNCTION__));
if (tryOptCompareBranchFedByICmp(I, ICmp, MIB))
  return true;

// Couldn't optimize. Emit a compare + a Bcc.
MachineBasicBlock *DestMBB = I.getOperand(1).getMBB();
auto PredOp = ICmp.getOperand(1);
emitIntegerCompare(ICmp.getOperand(2), ICmp.getOperand(3), PredOp, MIB);
const AArch64CC::CondCode CC = changeICMPPredToAArch64CC(
    static_cast<CmpInst::Predicate>(PredOp.getPredicate()));
MIB.buildInstr(AArch64::Bcc, {}, {}).addImm(CC).addMBB(DestMBB);
I.eraseFromParent();
return true;
1656}

1658bool AArch64InstructionSelector::selectCompareBranch(
  MachineInstr &I, MachineFunction &MF, MachineRegisterInfo &MRI) {
Register CondReg = I.getOperand(0).getReg();
MachineInstr *CCMI = MRI.getVRegDef(CondReg);
if (CCMI->getOpcode() == TargetOpcode::G_TRUNC) {
  CondReg = CCMI->getOperand(1).getReg();
  CCMI = MRI.getVRegDef(CondReg);
}

// Try to select the G_BRCOND using whatever is feeding the condition if
// possible.
unsigned CCMIOpc = CCMI->getOpcode();
if (CCMIOpc == TargetOpcode::G_FCMP)
  return selectCompareBranchFedByFCmp(I, *CCMI, MIB);
if (CCMIOpc == TargetOpcode::G_ICMP)
  return selectCompareBranchFedByICmp(I, *CCMI, MIB);

// Speculation tracking/SLH assumes that optimized TB(N)Z/CB(N)Z
// instructions will not be produced, as they are conditional branch
// instructions that do not set flags.
if (ProduceNonFlagSettingCondBr) {
  emitTestBit(CondReg, /*Bit = */ 0, /*IsNegative = */ true,
              I.getOperand(1).getMBB(), MIB);
  I.eraseFromParent();
  return true;
}

// Can't emit TB(N)Z/CB(N)Z. Emit a tst + bcc instead.
auto TstMI =
    MIB.buildInstr(AArch64::ANDSWri, {LLT::scalar(32)}, {CondReg}).addImm(1);
constrainSelectedInstRegOperands(*TstMI, TII, TRI, RBI);
auto Bcc = MIB.buildInstr(AArch64::Bcc)
               .addImm(AArch64CC::EQ)
               .addMBB(I.getOperand(1).getMBB());
I.eraseFromParent();
return constrainSelectedInstRegOperands(*Bcc, TII, TRI, RBI);
1694}

1696/// Returns the element immediate value of a vector shift operand if found.
1697/// This needs to detect a splat-like operation, e.g. a G_BUILD_VECTOR.
1698static Optional<int64_t> getVectorShiftImm(Register Reg,
                                         MachineRegisterInfo &MRI) {
assert(MRI.getType(Reg).isVector() && "Expected a *vector* shift operand")(static_cast <bool> (MRI.getType(Reg).isVector() &&
 "Expected a *vector* shift operand") ? void (0) : __assert_fail
 ("MRI.getType(Reg).isVector() && \"Expected a *vector* shift operand\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1700, __extension__ __PRETTY_FUNCTION__));
MachineInstr *OpMI = MRI.getVRegDef(Reg);
assert(OpMI && "Expected to find a vreg def for vector shift operand")(static_cast <bool> (OpMI && "Expected to find a vreg def for vector shift operand"
) ? void (0) : __assert_fail ("OpMI && \"Expected to find a vreg def for vector shift operand\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1702, __extension__ __PRETTY_FUNCTION__));
return getAArch64VectorSplatScalar(*OpMI, MRI);
1704}

1706/// Matches and returns the shift immediate value for a SHL instruction given
1707/// a shift operand.
1708static Optional<int64_t> getVectorSHLImm(LLT SrcTy, Register Reg, MachineRegisterInfo &MRI) {
Optional<int64_t> ShiftImm = getVectorShiftImm(Reg, MRI);
if (!ShiftImm)
  return None;
// Check the immediate is in range for a SHL.
int64_t Imm = *ShiftImm;
if (Imm < 0)
  return None;
switch (SrcTy.getElementType().getSizeInBits()) {
default:
  LLVM_DEBUG(dbgs() << "Unhandled element type for vector shift")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unhandled element type for vector shift"
; } } while (false);
  return None;
case 8:
  if (Imm > 7)
    return None;
  break;
case 16:
  if (Imm > 15)
    return None;
  break;
case 32:
  if (Imm > 31)
    return None;
  break;
case 64:
  if (Imm > 63)
    return None;
  break;
}
return Imm;
1738}

1740bool AArch64InstructionSelector::selectVectorSHL(MachineInstr &I,
                                               MachineRegisterInfo &MRI) {
assert(I.getOpcode() == TargetOpcode::G_SHL)(static_cast <bool> (I.getOpcode() == TargetOpcode::G_SHL
) ? void (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_SHL"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1742, __extension__ __PRETTY_FUNCTION__));
Register DstReg = I.getOperand(0).getReg();
const LLT Ty = MRI.getType(DstReg);
Register Src1Reg = I.getOperand(1).getReg();
Register Src2Reg = I.getOperand(2).getReg();

if (!Ty.isVector())
  return false;

// Check if we have a vector of constants on RHS that we can select as the
// immediate form.
Optional<int64_t> ImmVal = getVectorSHLImm(Ty, Src2Reg, MRI);

unsigned Opc = 0;
if (Ty == LLT::fixed_vector(2, 64)) {
  Opc = ImmVal ? AArch64::SHLv2i64_shift : AArch64::USHLv2i64;
} else if (Ty == LLT::fixed_vector(4, 32)) {
  Opc = ImmVal ? AArch64::SHLv4i32_shift : AArch64::USHLv4i32;
} else if (Ty == LLT::fixed_vector(2, 32)) {
  Opc = ImmVal ? AArch64::SHLv2i32_shift : AArch64::USHLv2i32;
} else if (Ty == LLT::fixed_vector(4, 16)) {
  Opc = ImmVal ? AArch64::SHLv4i16_shift : AArch64::USHLv4i16;
} else if (Ty == LLT::fixed_vector(8, 16)) {
  Opc = ImmVal ? AArch64::SHLv8i16_shift : AArch64::USHLv8i16;
} else if (Ty == LLT::fixed_vector(16, 8)) {
  Opc = ImmVal ? AArch64::SHLv16i8_shift : AArch64::USHLv16i8;
} else if (Ty == LLT::fixed_vector(8, 8)) {
  Opc = ImmVal ? AArch64::SHLv8i8_shift : AArch64::USHLv8i8;
} else {
  LLVM_DEBUG(dbgs() << "Unhandled G_SHL type")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unhandled G_SHL type"; }
 } while (false);
  return false;
}

auto Shl = MIB.buildInstr(Opc, {DstReg}, {Src1Reg});
if (ImmVal)
  Shl.addImm(*ImmVal);
else
  Shl.addUse(Src2Reg);
constrainSelectedInstRegOperands(*Shl, TII, TRI, RBI);
I.eraseFromParent();
return true;
1783}

1785bool AArch64InstructionSelector::selectVectorAshrLshr(
  MachineInstr &I, MachineRegisterInfo &MRI) {
assert(I.getOpcode() == TargetOpcode::G_ASHR ||(static_cast <bool> (I.getOpcode() == TargetOpcode::G_ASHR
 || I.getOpcode() == TargetOpcode::G_LSHR) ? void (0) : __assert_fail
 ("I.getOpcode() == TargetOpcode::G_ASHR || I.getOpcode() == TargetOpcode::G_LSHR"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1788, __extension__ __PRETTY_FUNCTION__))
       I.getOpcode() == TargetOpcode::G_LSHR)(static_cast <bool> (I.getOpcode() == TargetOpcode::G_ASHR
 || I.getOpcode() == TargetOpcode::G_LSHR) ? void (0) : __assert_fail
 ("I.getOpcode() == TargetOpcode::G_ASHR || I.getOpcode() == TargetOpcode::G_LSHR"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1788, __extension__ __PRETTY_FUNCTION__));
Register DstReg = I.getOperand(0).getReg();
const LLT Ty = MRI.getType(DstReg);
Register Src1Reg = I.getOperand(1).getReg();
Register Src2Reg = I.getOperand(2).getReg();

if (!Ty.isVector())
  return false;

bool IsASHR = I.getOpcode() == TargetOpcode::G_ASHR;

// We expect the immediate case to be lowered in the PostLegalCombiner to
// AArch64ISD::VASHR or AArch64ISD::VLSHR equivalents.

// There is not a shift right register instruction, but the shift left
// register instruction takes a signed value, where negative numbers specify a
// right shift.

unsigned Opc = 0;
unsigned NegOpc = 0;
const TargetRegisterClass *RC =
    getRegClassForTypeOnBank(Ty, RBI.getRegBank(AArch64::FPRRegBankID), RBI);
if (Ty == LLT::fixed_vector(2, 64)) {
  Opc = IsASHR ? AArch64::SSHLv2i64 : AArch64::USHLv2i64;
  NegOpc = AArch64::NEGv2i64;
} else if (Ty == LLT::fixed_vector(4, 32)) {
  Opc = IsASHR ? AArch64::SSHLv4i32 : AArch64::USHLv4i32;
  NegOpc = AArch64::NEGv4i32;
} else if (Ty == LLT::fixed_vector(2, 32)) {
  Opc = IsASHR ? AArch64::SSHLv2i32 : AArch64::USHLv2i32;
  NegOpc = AArch64::NEGv2i32;
} else if (Ty == LLT::fixed_vector(4, 16)) {
  Opc = IsASHR ? AArch64::SSHLv4i16 : AArch64::USHLv4i16;
  NegOpc = AArch64::NEGv4i16;
} else if (Ty == LLT::fixed_vector(8, 16)) {
  Opc = IsASHR ? AArch64::SSHLv8i16 : AArch64::USHLv8i16;
  NegOpc = AArch64::NEGv8i16;
} else if (Ty == LLT::fixed_vector(16, 8)) {
  Opc = IsASHR ? AArch64::SSHLv16i8 : AArch64::USHLv16i8;
  NegOpc = AArch64::NEGv16i8;
} else if (Ty == LLT::fixed_vector(8, 8)) {
  Opc = IsASHR ? AArch64::SSHLv8i8 : AArch64::USHLv8i8;
  NegOpc = AArch64::NEGv8i8;
} else {
  LLVM_DEBUG(dbgs() << "Unhandled G_ASHR type")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unhandled G_ASHR type"; }
 } while (false);
  return false;
}

auto Neg = MIB.buildInstr(NegOpc, {RC}, {Src2Reg});
constrainSelectedInstRegOperands(*Neg, TII, TRI, RBI);
auto SShl = MIB.buildInstr(Opc, {DstReg}, {Src1Reg, Neg});
constrainSelectedInstRegOperands(*SShl, TII, TRI, RBI);
I.eraseFromParent();
return true;
1842}

1844bool AArch64InstructionSelector::selectVaStartAAPCS(
  MachineInstr &I, MachineFunction &MF, MachineRegisterInfo &MRI) const {
return false;
1847}

1849bool AArch64InstructionSelector::selectVaStartDarwin(
  MachineInstr &I, MachineFunction &MF, MachineRegisterInfo &MRI) const {
AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>();
Register ListReg = I.getOperand(0).getReg();

Register ArgsAddrReg = MRI.createVirtualRegister(&AArch64::GPR64RegClass);

auto MIB =
    BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(AArch64::ADDXri))
        .addDef(ArgsAddrReg)
        .addFrameIndex(FuncInfo->getVarArgsStackIndex())
        .addImm(0)
        .addImm(0);

constrainSelectedInstRegOperands(*MIB, TII, TRI, RBI);

MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(AArch64::STRXui))
          .addUse(ArgsAddrReg)
          .addUse(ListReg)
          .addImm(0)
          .addMemOperand(*I.memoperands_begin());

constrainSelectedInstRegOperands(*MIB, TII, TRI, RBI);
I.eraseFromParent();
return true;
1874}

1876void AArch64InstructionSelector::materializeLargeCMVal(
  MachineInstr &I, const Value *V, unsigned OpFlags) {
MachineBasicBlock &MBB = *I.getParent();
MachineFunction &MF = *MBB.getParent();
MachineRegisterInfo &MRI = MF.getRegInfo();

auto MovZ = MIB.buildInstr(AArch64::MOVZXi, {&AArch64::GPR64RegClass}, {});
MovZ->addOperand(MF, I.getOperand(1));
MovZ->getOperand(1).setTargetFlags(OpFlags | AArch64II::MO_G0 |
                                   AArch64II::MO_NC);
MovZ->addOperand(MF, MachineOperand::CreateImm(0));
constrainSelectedInstRegOperands(*MovZ, TII, TRI, RBI);

auto BuildMovK = [&](Register SrcReg, unsigned char Flags, unsigned Offset,
                     Register ForceDstReg) {
  Register DstReg = ForceDstReg
                        ? ForceDstReg
                        : MRI.createVirtualRegister(&AArch64::GPR64RegClass);
  auto MovI = MIB.buildInstr(AArch64::MOVKXi).addDef(DstReg).addUse(SrcReg);
  if (auto *GV = dyn_cast<GlobalValue>(V)) {
    MovI->addOperand(MF, MachineOperand::CreateGA(
                             GV, MovZ->getOperand(1).getOffset(), Flags));
  } else {
    MovI->addOperand(
        MF, MachineOperand::CreateBA(cast<BlockAddress>(V),
                                     MovZ->getOperand(1).getOffset(), Flags));
  }
  MovI->addOperand(MF, MachineOperand::CreateImm(Offset));
  constrainSelectedInstRegOperands(*MovI, TII, TRI, RBI);
  return DstReg;
};
Register DstReg = BuildMovK(MovZ.getReg(0),
                            AArch64II::MO_G1 | AArch64II::MO_NC, 16, 0);
DstReg = BuildMovK(DstReg, AArch64II::MO_G2 | AArch64II::MO_NC, 32, 0);
BuildMovK(DstReg, AArch64II::MO_G3, 48, I.getOperand(0).getReg());
1911}

1913bool AArch64InstructionSelector::preISelLower(MachineInstr &I) {
MachineBasicBlock &MBB = *I.getParent();
MachineFunction &MF = *MBB.getParent();
MachineRegisterInfo &MRI = MF.getRegInfo();

switch (I.getOpcode()) {
case TargetOpcode::G_SHL:
case TargetOpcode::G_ASHR:
case TargetOpcode::G_LSHR: {
  // These shifts are legalized to have 64 bit shift amounts because we want
  // to take advantage of the existing imported selection patterns that assume
  // the immediates are s64s. However, if the shifted type is 32 bits and for
  // some reason we receive input GMIR that has an s64 shift amount that's not
  // a G_CONSTANT, insert a truncate so that we can still select the s32
  // register-register variant.
  Register SrcReg = I.getOperand(1).getReg();
  Register ShiftReg = I.getOperand(2).getReg();
  const LLT ShiftTy = MRI.getType(ShiftReg);
  const LLT SrcTy = MRI.getType(SrcReg);
  if (SrcTy.isVector())
    return false;
  assert(!ShiftTy.isVector() && "unexpected vector shift ty")(static_cast <bool> (!ShiftTy.isVector() && "unexpected vector shift ty"
) ? void (0) : __assert_fail ("!ShiftTy.isVector() && \"unexpected vector shift ty\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1934, __extension__ __PRETTY_FUNCTION__));
  if (SrcTy.getSizeInBits() != 32 || ShiftTy.getSizeInBits() != 64)
    return false;
  auto *AmtMI = MRI.getVRegDef(ShiftReg);
  assert(AmtMI && "could not find a vreg definition for shift amount")(static_cast <bool> (AmtMI && "could not find a vreg definition for shift amount"
) ? void (0) : __assert_fail ("AmtMI && \"could not find a vreg definition for shift amount\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1938, __extension__ __PRETTY_FUNCTION__));
  if (AmtMI->getOpcode() != TargetOpcode::G_CONSTANT) {
    // Insert a subregister copy to implement a 64->32 trunc
    auto Trunc = MIB.buildInstr(TargetOpcode::COPY, {SrcTy}, {})
                     .addReg(ShiftReg, 0, AArch64::sub_32);
    MRI.setRegBank(Trunc.getReg(0), RBI.getRegBank(AArch64::GPRRegBankID));
    I.getOperand(2).setReg(Trunc.getReg(0));
  }
  return true;
}
case TargetOpcode::G_STORE: {
  bool Changed = contractCrossBankCopyIntoStore(I, MRI);
  MachineOperand &SrcOp = I.getOperand(0);
  if (MRI.getType(SrcOp.getReg()).isPointer()) {
    // Allow matching with imported patterns for stores of pointers. Unlike
    // G_LOAD/G_PTR_ADD, we may not have selected all users. So, emit a copy
    // and constrain.
    auto Copy = MIB.buildCopy(LLT::scalar(64), SrcOp);
    Register NewSrc = Copy.getReg(0);
    SrcOp.setReg(NewSrc);
    RBI.constrainGenericRegister(NewSrc, AArch64::GPR64RegClass, MRI);
    Changed = true;
  }
  return Changed;
}
case TargetOpcode::G_PTR_ADD:
  return convertPtrAddToAdd(I, MRI);
case TargetOpcode::G_LOAD: {
  // For scalar loads of pointers, we try to convert the dest type from p0
  // to s64 so that our imported patterns can match. Like with the G_PTR_ADD
  // conversion, this should be ok because all users should have been
  // selected already, so the type doesn't matter for them.
  Register DstReg = I.getOperand(0).getReg();
  const LLT DstTy = MRI.getType(DstReg);
  if (!DstTy.isPointer())
    return false;
  MRI.setType(DstReg, LLT::scalar(64));
  return true;
}
case AArch64::G_DUP: {
  // Convert the type from p0 to s64 to help selection.
  LLT DstTy = MRI.getType(I.getOperand(0).getReg());
  if (!DstTy.getElementType().isPointer())
    return false;
  auto NewSrc = MIB.buildCopy(LLT::scalar(64), I.getOperand(1).getReg());
  MRI.setType(I.getOperand(0).getReg(),
              DstTy.changeElementType(LLT::scalar(64)));
  MRI.setRegClass(NewSrc.getReg(0), &AArch64::GPR64RegClass);
  I.getOperand(1).setReg(NewSrc.getReg(0));
  return true;
}
case TargetOpcode::G_UITOFP:
case TargetOpcode::G_SITOFP: {
  // If both source and destination regbanks are FPR, then convert the opcode
  // to G_SITOF so that the importer can select it to an fpr variant.
  // Otherwise, it ends up matching an fpr/gpr variant and adding a cross-bank
  // copy.
  Register SrcReg = I.getOperand(1).getReg();
  LLT SrcTy = MRI.getType(SrcReg);
  LLT DstTy = MRI.getType(I.getOperand(0).getReg());
  if (SrcTy.isVector() || SrcTy.getSizeInBits() != DstTy.getSizeInBits())
    return false;

  if (RBI.getRegBank(SrcReg, MRI, TRI)->getID() == AArch64::FPRRegBankID) {
    if (I.getOpcode() == TargetOpcode::G_SITOFP)
      I.setDesc(TII.get(AArch64::G_SITOF));
    else
      I.setDesc(TII.get(AArch64::G_UITOF));
    return true;
  }
  return false;
}
default:
  return false;
}
2013}

2015/// This lowering tries to look for G_PTR_ADD instructions and then converts
2016/// them to a standard G_ADD with a COPY on the source.
2017///
2018/// The motivation behind this is to expose the add semantics to the imported
2019/// tablegen patterns. We shouldn't need to check for uses being loads/stores,
2020/// because the selector works bottom up, uses before defs. By the time we
2021/// end up trying to select a G_PTR_ADD, we should have already attempted to
2022/// fold this into addressing modes and were therefore unsuccessful.
2023bool AArch64InstructionSelector::convertPtrAddToAdd(
  MachineInstr &I, MachineRegisterInfo &MRI) {
assert(I.getOpcode() == TargetOpcode::G_PTR_ADD && "Expected G_PTR_ADD")(static_cast <bool> (I.getOpcode() == TargetOpcode::G_PTR_ADD
 && "Expected G_PTR_ADD") ? void (0) : __assert_fail (
"I.getOpcode() == TargetOpcode::G_PTR_ADD && \"Expected G_PTR_ADD\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2025, __extension__ __PRETTY_FUNCTION__));
Register DstReg = I.getOperand(0).getReg();
Register AddOp1Reg = I.getOperand(1).getReg();
const LLT PtrTy = MRI.getType(DstReg);
if (PtrTy.getAddressSpace() != 0)
  return false;

const LLT CastPtrTy =
    PtrTy.isVector() ? LLT::fixed_vector(2, 64) : LLT::scalar(64);
auto PtrToInt = MIB.buildPtrToInt(CastPtrTy, AddOp1Reg);
// Set regbanks on the registers.
if (PtrTy.isVector())
  MRI.setRegBank(PtrToInt.getReg(0), RBI.getRegBank(AArch64::FPRRegBankID));
else
  MRI.setRegBank(PtrToInt.getReg(0), RBI.getRegBank(AArch64::GPRRegBankID));

// Now turn the %dst(p0) = G_PTR_ADD %base, off into:
// %dst(intty) = G_ADD %intbase, off
I.setDesc(TII.get(TargetOpcode::G_ADD));
MRI.setType(DstReg, CastPtrTy);
I.getOperand(1).setReg(PtrToInt.getReg(0));
if (!select(*PtrToInt)) {
  LLVM_DEBUG(dbgs() << "Failed to select G_PTRTOINT in convertPtrAddToAdd")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Failed to select G_PTRTOINT in convertPtrAddToAdd"
; } } while (false);
  return false;
}

// Also take the opportunity here to try to do some optimization.
// Try to convert this into a G_SUB if the offset is a 0-x negate idiom.
Register NegatedReg;
if (!mi_match(I.getOperand(2).getReg(), MRI, m_Neg(m_Reg(NegatedReg))))
  return true;
I.getOperand(2).setReg(NegatedReg);
I.setDesc(TII.get(TargetOpcode::G_SUB));
return true;
2059}

2061bool AArch64InstructionSelector::earlySelectSHL(MachineInstr &I,
                                              MachineRegisterInfo &MRI) {
// We try to match the immediate variant of LSL, which is actually an alias
// for a special case of UBFM. Otherwise, we fall back to the imported
// selector which will match the register variant.
assert(I.getOpcode() == TargetOpcode::G_SHL && "unexpected op")(static_cast <bool> (I.getOpcode() == TargetOpcode::G_SHL
 && "unexpected op") ? void (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_SHL && \"unexpected op\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2066, __extension__ __PRETTY_FUNCTION__));
const auto &MO = I.getOperand(2);
auto VRegAndVal = getConstantVRegVal(MO.getReg(), MRI);
if (!VRegAndVal)
  return false;

const LLT DstTy = MRI.getType(I.getOperand(0).getReg());
if (DstTy.isVector())
  return false;
bool Is64Bit = DstTy.getSizeInBits() == 64;
auto Imm1Fn = Is64Bit ? selectShiftA_64(MO) : selectShiftA_32(MO);
auto Imm2Fn = Is64Bit ? selectShiftB_64(MO) : selectShiftB_32(MO);

if (!Imm1Fn || !Imm2Fn)
  return false;

auto NewI =
    MIB.buildInstr(Is64Bit ? AArch64::UBFMXri : AArch64::UBFMWri,
                   {I.getOperand(0).getReg()}, {I.getOperand(1).getReg()});

for (auto &RenderFn : *Imm1Fn)
  RenderFn(NewI);
for (auto &RenderFn : *Imm2Fn)
  RenderFn(NewI);

I.eraseFromParent();
return constrainSelectedInstRegOperands(*NewI, TII, TRI, RBI);
2093}

2095bool AArch64InstructionSelector::contractCrossBankCopyIntoStore(
  MachineInstr &I, MachineRegisterInfo &MRI) {
assert(I.getOpcode() == TargetOpcode::G_STORE && "Expected G_STORE")(static_cast <bool> (I.getOpcode() == TargetOpcode::G_STORE
 && "Expected G_STORE") ? void (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_STORE && \"Expected G_STORE\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2097, __extension__ __PRETTY_FUNCTION__));
// If we're storing a scalar, it doesn't matter what register bank that
// scalar is on. All that matters is the size.
//
// So, if we see something like this (with a 32-bit scalar as an example):
//
// %x:gpr(s32) = ... something ...
// %y:fpr(s32) = COPY %x:gpr(s32)
// G_STORE %y:fpr(s32)
//
// We can fix this up into something like this:
//
// G_STORE %x:gpr(s32)
//
// And then continue the selection process normally.
Register DefDstReg = getSrcRegIgnoringCopies(I.getOperand(0).getReg(), MRI);
if (!DefDstReg.isValid())
  return false;
LLT DefDstTy = MRI.getType(DefDstReg);
Register StoreSrcReg = I.getOperand(0).getReg();
LLT StoreSrcTy = MRI.getType(StoreSrcReg);

// If we get something strange like a physical register, then we shouldn't
// go any further.
if (!DefDstTy.isValid())
  return false;

// Are the source and dst types the same size?
if (DefDstTy.getSizeInBits() != StoreSrcTy.getSizeInBits())
  return false;

if (RBI.getRegBank(StoreSrcReg, MRI, TRI) ==
    RBI.getRegBank(DefDstReg, MRI, TRI))
  return false;

// We have a cross-bank copy, which is entering a store. Let's fold it.
I.getOperand(0).setReg(DefDstReg);
return true;
2135}

2137bool AArch64InstructionSelector::earlySelect(MachineInstr &I) {
assert(I.getParent() && "Instruction should be in a basic block!")(static_cast <bool> (I.getParent() && "Instruction should be in a basic block!"
) ? void (0) : __assert_fail ("I.getParent() && \"Instruction should be in a basic block!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2138, __extension__ __PRETTY_FUNCTION__));
assert(I.getParent()->getParent() && "Instruction should be in a function!")(static_cast <bool> (I.getParent()->getParent() &&
 "Instruction should be in a function!") ? void (0) : __assert_fail
 ("I.getParent()->getParent() && \"Instruction should be in a function!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2139, __extension__ __PRETTY_FUNCTION__));

MachineBasicBlock &MBB = *I.getParent();
MachineFunction &MF = *MBB.getParent();
MachineRegisterInfo &MRI = MF.getRegInfo();

switch (I.getOpcode()) {
case AArch64::G_DUP: {
  // Before selecting a DUP instruction, check if it is better selected as a
  // MOV or load from a constant pool.
  Register Src = I.getOperand(1).getReg();
  auto ValAndVReg = getConstantVRegValWithLookThrough(Src, MRI);
  if (!ValAndVReg)
    return false;
  LLVMContext &Ctx = MF.getFunction().getContext();
  Register Dst = I.getOperand(0).getReg();
  auto *CV = ConstantDataVector::getSplat(
      MRI.getType(Dst).getNumElements(),
      ConstantInt::get(Type::getIntNTy(Ctx, MRI.getType(Src).getSizeInBits()),
                       ValAndVReg->Value));
  if (!emitConstantVector(Dst, CV, MIB, MRI))
    return false;
  I.eraseFromParent();
  return true;
}
case TargetOpcode::G_BR:
  return false;
case TargetOpcode::G_SHL:
  return earlySelectSHL(I, MRI);
case TargetOpcode::G_CONSTANT: {
  bool IsZero = false;
  if (I.getOperand(1).isCImm())
    IsZero = I.getOperand(1).getCImm()->getZExtValue() == 0;
  else if (I.getOperand(1).isImm())
    IsZero = I.getOperand(1).getImm() == 0;

  if (!IsZero)
    return false;

  Register DefReg = I.getOperand(0).getReg();
  LLT Ty = MRI.getType(DefReg);
  if (Ty.getSizeInBits() == 64) {
    I.getOperand(1).ChangeToRegister(AArch64::XZR, false);
    RBI.constrainGenericRegister(DefReg, AArch64::GPR64RegClass, MRI);
  } else if (Ty.getSizeInBits() == 32) {
    I.getOperand(1).ChangeToRegister(AArch64::WZR, false);
    RBI.constrainGenericRegister(DefReg, AArch64::GPR32RegClass, MRI);
  } else
    return false;

  I.setDesc(TII.get(TargetOpcode::COPY));
  return true;
}

case TargetOpcode::G_ADD: {
  // Check if this is being fed by a G_ICMP on either side.
  //
  // (cmp pred, x, y) + z
  //
  // In the above case, when the cmp is true, we increment z by 1. So, we can
  // fold the add into the cset for the cmp by using cinc.
  //
  // FIXME: This would probably be a lot nicer in PostLegalizerLowering.
  Register X = I.getOperand(1).getReg();

  // Only handle scalars. Scalar G_ICMP is only legal for s32, so bail out
  // early if we see it.
  LLT Ty = MRI.getType(X);
  if (Ty.isVector() || Ty.getSizeInBits() != 32)
    return false;

  Register CmpReg = I.getOperand(2).getReg();
  MachineInstr *Cmp = getOpcodeDef(TargetOpcode::G_ICMP, CmpReg, MRI);
  if (!Cmp) {
    std::swap(X, CmpReg);
    Cmp = getOpcodeDef(TargetOpcode::G_ICMP, CmpReg, MRI);
    if (!Cmp)
      return false;
  }
  auto Pred =
      static_cast<CmpInst::Predicate>(Cmp->getOperand(1).getPredicate());
  emitIntegerCompare(Cmp->getOperand(2), Cmp->getOperand(3),
                     Cmp->getOperand(1), MIB);
  emitCSetForICMP(I.getOperand(0).getReg(), Pred, MIB, X);
  I.eraseFromParent();
  return true;
}
case TargetOpcode::G_OR: {
  // Look for operations that take the lower `Width=Size-ShiftImm` bits of
  // `ShiftSrc` and insert them into the upper `Width` bits of `MaskSrc` via
  // shifting and masking that we can replace with a BFI (encoded as a BFM).
  Register Dst = I.getOperand(0).getReg();
  LLT Ty = MRI.getType(Dst);

  if (!Ty.isScalar())
    return false;

  unsigned Size = Ty.getSizeInBits();
  if (Size != 32 && Size != 64)
    return false;

  Register ShiftSrc;
  int64_t ShiftImm;
  Register MaskSrc;
  int64_t MaskImm;
  if (!mi_match(
          Dst, MRI,
          m_GOr(m_OneNonDBGUse(m_GShl(m_Reg(ShiftSrc), m_ICst(ShiftImm))),
                m_OneNonDBGUse(m_GAnd(m_Reg(MaskSrc), m_ICst(MaskImm))))))
    return false;

  if (ShiftImm > Size || ((1ULL << ShiftImm) - 1ULL) != uint64_t(MaskImm))
    return false;

  int64_t Immr = Size - ShiftImm;
  int64_t Imms = Size - ShiftImm - 1;
  unsigned Opc = Size == 32 ? AArch64::BFMWri : AArch64::BFMXri;
  emitInstr(Opc, {Dst}, {MaskSrc, ShiftSrc, Immr, Imms}, MIB);
  I.eraseFromParent();
  return true;
}
default:
  return false;
}
2263}

2265bool AArch64InstructionSelector::select(MachineInstr &I) {
assert(I.getParent() && "Instruction should be in a basic block!")(static_cast <bool> (I.getParent() && "Instruction should be in a basic block!"
) ? void (0) : __assert_fail ("I.getParent() && \"Instruction should be in a basic block!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2266, __extension__ __PRETTY_FUNCTION__));
assert(I.getParent()->getParent() && "Instruction should be in a function!")(static_cast <bool> (I.getParent()->getParent() &&
 "Instruction should be in a function!") ? void (0) : __assert_fail
 ("I.getParent()->getParent() && \"Instruction should be in a function!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2267, __extension__ __PRETTY_FUNCTION__));

MachineBasicBlock &MBB = *I.getParent();
MachineFunction &MF = *MBB.getParent();
MachineRegisterInfo &MRI = MF.getRegInfo();

const AArch64Subtarget *Subtarget =
    &static_cast<const AArch64Subtarget &>(MF.getSubtarget());
if (Subtarget->requiresStrictAlign()) {
  // We don't support this feature yet.
  LLVM_DEBUG(dbgs() << "AArch64 GISel does not support strict-align yet\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "AArch64 GISel does not support strict-align yet\n"
; } } while (false);
  return false;
}

MIB.setInstrAndDebugLoc(I);

unsigned Opcode = I.getOpcode();
// G_PHI requires same handling as PHI
if (!I.isPreISelOpcode() || Opcode == TargetOpcode::G_PHI) {
  // Certain non-generic instructions also need some special handling.

  if (Opcode ==  TargetOpcode::LOAD_STACK_GUARD)
    return constrainSelectedInstRegOperands(I, TII, TRI, RBI);

  if (Opcode == TargetOpcode::PHI || Opcode == TargetOpcode::G_PHI) {
    const Register DefReg = I.getOperand(0).getReg();
    const LLT DefTy = MRI.getType(DefReg);

    const RegClassOrRegBank &RegClassOrBank =
      MRI.getRegClassOrRegBank(DefReg);

    const TargetRegisterClass *DefRC
      = RegClassOrBank.dyn_cast<const TargetRegisterClass *>();
    if (!DefRC) {
      if (!DefTy.isValid()) {
        LLVM_DEBUG(dbgs() << "PHI operand has no type, not a gvreg?\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "PHI operand has no type, not a gvreg?\n"
; } } while (false);
        return false;
      }
      const RegisterBank &RB = *RegClassOrBank.get<const RegisterBank *>();
      DefRC = getRegClassForTypeOnBank(DefTy, RB, RBI);
      if (!DefRC) {
        LLVM_DEBUG(dbgs() << "PHI operand has unexpected size/bank\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "PHI operand has unexpected size/bank\n"
; } } while (false);
        return false;
      }
    }

    I.setDesc(TII.get(TargetOpcode::PHI));

    return RBI.constrainGenericRegister(DefReg, *DefRC, MRI);
  }

  if (I.isCopy())
    return selectCopy(I, TII, MRI, TRI, RBI);

  return true;
}


if (I.getNumOperands() != I.getNumExplicitOperands()) {
  LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Generic instruction has unexpected implicit operands\n"
; } } while (false)
      dbgs() << "Generic instruction has unexpected implicit operands\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Generic instruction has unexpected implicit operands\n"
; } } while (false);
  return false;
}

// Try to do some lowering before we start instruction selecting. These
// lowerings are purely transformations on the input G_MIR and so selection
// must continue after any modification of the instruction.
if (preISelLower(I)) {
  Opcode = I.getOpcode(); // The opcode may have been modified, refresh it.
}

// There may be patterns where the importer can't deal with them optimally,
// but does select it to a suboptimal sequence so our custom C++ selection
// code later never has a chance to work on it. Therefore, we have an early
// selection attempt here to give priority to certain selection routines
// over the imported ones.
if (earlySelect(I))
  return true;

if (selectImpl(I, *CoverageInfo))
  return true;

LLT Ty =
    I.getOperand(0).isReg() ? MRI.getType(I.getOperand(0).getReg()) : LLT{};

switch (Opcode) {
case TargetOpcode::G_SBFX:
case TargetOpcode::G_UBFX: {
  static const unsigned OpcTable[2][2] = {
      {AArch64::UBFMWri, AArch64::UBFMXri},
      {AArch64::SBFMWri, AArch64::SBFMXri}};
  bool IsSigned = Opcode == TargetOpcode::G_SBFX;
  unsigned Size = Ty.getSizeInBits();
  unsigned Opc = OpcTable[IsSigned][Size == 64];
  auto Cst1 =
      getConstantVRegValWithLookThrough(I.getOperand(2).getReg(), MRI);
  assert(Cst1 && "Should have gotten a constant for src 1?")(static_cast <bool> (Cst1 && "Should have gotten a constant for src 1?"
) ? void (0) : __assert_fail ("Cst1 && \"Should have gotten a constant for src 1?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2363, __extension__ __PRETTY_FUNCTION__));
  auto Cst2 =
      getConstantVRegValWithLookThrough(I.getOperand(3).getReg(), MRI);
  assert(Cst2 && "Should have gotten a constant for src 2?")(static_cast <bool> (Cst2 && "Should have gotten a constant for src 2?"
) ? void (0) : __assert_fail ("Cst2 && \"Should have gotten a constant for src 2?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2366, __extension__ __PRETTY_FUNCTION__));
  auto LSB = Cst1->Value.getZExtValue();
  auto Width = Cst2->Value.getZExtValue();
  auto BitfieldInst =
      MIB.buildInstr(Opc, {I.getOperand(0)}, {I.getOperand(1)})
          .addImm(LSB)
          .addImm(LSB + Width - 1);
  I.eraseFromParent();
  return constrainSelectedInstRegOperands(*BitfieldInst, TII, TRI, RBI);
}
case TargetOpcode::G_BRCOND:
  return selectCompareBranch(I, MF, MRI);

case TargetOpcode::G_BRINDIRECT: {
  I.setDesc(TII.get(AArch64::BR));
  return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
}

case TargetOpcode::G_BRJT:
  return selectBrJT(I, MRI);

case AArch64::G_ADD_LOW: {
  // This op may have been separated from it's ADRP companion by the localizer
  // or some other code motion pass. Given that many CPUs will try to
  // macro fuse these operations anyway, select this into a MOVaddr pseudo
  // which will later be expanded into an ADRP+ADD pair after scheduling.
  MachineInstr *BaseMI = MRI.getVRegDef(I.getOperand(1).getReg());
  if (BaseMI->getOpcode() != AArch64::ADRP) {
    I.setDesc(TII.get(AArch64::ADDXri));
    I.addOperand(MachineOperand::CreateImm(0));
    return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
  }
  assert(TM.getCodeModel() == CodeModel::Small &&(static_cast <bool> (TM.getCodeModel() == CodeModel::Small
 && "Expected small code model") ? void (0) : __assert_fail
 ("TM.getCodeModel() == CodeModel::Small && \"Expected small code model\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2399, __extension__ __PRETTY_FUNCTION__))
         "Expected small code model")(static_cast <bool> (TM.getCodeModel() == CodeModel::Small
 && "Expected small code model") ? void (0) : __assert_fail
 ("TM.getCodeModel() == CodeModel::Small && \"Expected small code model\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2399, __extension__ __PRETTY_FUNCTION__));
  auto Op1 = BaseMI->getOperand(1);
  auto Op2 = I.getOperand(2);
  auto MovAddr = MIB.buildInstr(AArch64::MOVaddr, {I.getOperand(0)}, {})
                     .addGlobalAddress(Op1.getGlobal(), Op1.getOffset(),
                                       Op1.getTargetFlags())
                     .addGlobalAddress(Op2.getGlobal(), Op2.getOffset(),
                                       Op2.getTargetFlags());
  I.eraseFromParent();
  return constrainSelectedInstRegOperands(*MovAddr, TII, TRI, RBI);
}

case TargetOpcode::G_BSWAP: {
  // Handle vector types for G_BSWAP directly.
  Register DstReg = I.getOperand(0).getReg();
  LLT DstTy = MRI.getType(DstReg);

  // We should only get vector types here; everything else is handled by the
  // importer right now.
  if (!DstTy.isVector() || DstTy.getSizeInBits() > 128) {
    LLVM_DEBUG(dbgs() << "Dst type for G_BSWAP currently unsupported.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Dst type for G_BSWAP currently unsupported.\n"
; } } while (false);
    return false;
  }

  // Only handle 4 and 2 element vectors for now.
  // TODO: 16-bit elements.
  unsigned NumElts = DstTy.getNumElements();
  if (NumElts != 4 && NumElts != 2) {
    LLVM_DEBUG(dbgs() << "Unsupported number of elements for G_BSWAP.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unsupported number of elements for G_BSWAP.\n"
; } } while (false);
    return false;
  }

  // Choose the correct opcode for the supported types. Right now, that's
  // v2s32, v4s32, and v2s64.
  unsigned Opc = 0;
  unsigned EltSize = DstTy.getElementType().getSizeInBits();
  if (EltSize == 32)
    Opc = (DstTy.getNumElements() == 2) ? AArch64::REV32v8i8
                                        : AArch64::REV32v16i8;
  else if (EltSize == 64)
    Opc = AArch64::REV64v16i8;

  // We should always get something by the time we get here...
  assert(Opc != 0 && "Didn't get an opcode for G_BSWAP?")(static_cast <bool> (Opc != 0 && "Didn't get an opcode for G_BSWAP?"
) ? void (0) : __assert_fail ("Opc != 0 && \"Didn't get an opcode for G_BSWAP?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2442, __extension__ __PRETTY_FUNCTION__));

  I.setDesc(TII.get(Opc));
  return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
}

case TargetOpcode::G_FCONSTANT:
case TargetOpcode::G_CONSTANT: {
  const bool isFP = Opcode == TargetOpcode::G_FCONSTANT;

  const LLT s8 = LLT::scalar(8);
  const LLT s16 = LLT::scalar(16);
  const LLT s32 = LLT::scalar(32);
  const LLT s64 = LLT::scalar(64);
  const LLT s128 = LLT::scalar(128);
  const LLT p0 = LLT::pointer(0, 64);

  const Register DefReg = I.getOperand(0).getReg();
  const LLT DefTy = MRI.getType(DefReg);
  const unsigned DefSize = DefTy.getSizeInBits();
  const RegisterBank &RB = *RBI.getRegBank(DefReg, MRI, TRI);

  // FIXME: Redundant check, but even less readable when factored out.
  if (isFP) {
    if (Ty != s32 && Ty != s64 && Ty != s128) {
      LLVM_DEBUG(dbgs() << "Unable to materialize FP " << Tydo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unable to materialize FP "
 << Ty << " constant, expected: " << s32 <<
 " or " << s64 << " or " << s128 << '\n'
; } } while (false)
                        << " constant, expected: " << s32 << " or " << s64do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unable to materialize FP "
 << Ty << " constant, expected: " << s32 <<
 " or " << s64 << " or " << s128 << '\n'
; } } while (false)
                        << " or " << s128 << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unable to materialize FP "
 << Ty << " constant, expected: " << s32 <<
 " or " << s64 << " or " << s128 << '\n'
; } } while (false);
      return false;
    }

    if (RB.getID() != AArch64::FPRRegBankID) {
      LLVM_DEBUG(dbgs() << "Unable to materialize FP " << Tydo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unable to materialize FP "
 << Ty << " constant on bank: " << RB <<
 ", expected: FPR\n"; } } while (false)
                        << " constant on bank: " << RBdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unable to materialize FP "
 << Ty << " constant on bank: " << RB <<
 ", expected: FPR\n"; } } while (false)
                        << ", expected: FPR\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unable to materialize FP "
 << Ty << " constant on bank: " << RB <<
 ", expected: FPR\n"; } } while (false);
      return false;
    }

    // The case when we have 0.0 is covered by tablegen. Reject it here so we
    // can be sure tablegen works correctly and isn't rescued by this code.
    // 0.0 is not covered by tablegen for FP128. So we will handle this 
    // scenario in the code here.
    if (DefSize != 128 && I.getOperand(1).getFPImm()->isExactlyValue(0.0))
      return false;
  } else {
    // s32 and s64 are covered by tablegen.
    if (Ty != p0 && Ty != s8 && Ty != s16) {
      LLVM_DEBUG(dbgs() << "Unable to materialize integer " << Tydo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unable to materialize integer "
 << Ty << " constant, expected: " << s32 <<
 ", " << s64 << ", or " << p0 << '\n'
; } } while (false)
                        << " constant, expected: " << s32 << ", " << s64do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unable to materialize integer "
 << Ty << " constant, expected: " << s32 <<
 ", " << s64 << ", or " << p0 << '\n'
; } } while (false)
                        << ", or " << p0 << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unable to materialize integer "
 << Ty << " constant, expected: " << s32 <<
 ", " << s64 << ", or " << p0 << '\n'
; } } while (false);
      return false;
    }

    if (RB.getID() != AArch64::GPRRegBankID) {
      LLVM_DEBUG(dbgs() << "Unable to materialize integer " << Tydo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unable to materialize integer "
 << Ty << " constant on bank: " << RB <<
 ", expected: GPR\n"; } } while (false)
                        << " constant on bank: " << RBdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unable to materialize integer "
 << Ty << " constant on bank: " << RB <<
 ", expected: GPR\n"; } } while (false)
                        << ", expected: GPR\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unable to materialize integer "
 << Ty << " constant on bank: " << RB <<
 ", expected: GPR\n"; } } while (false);
      return false;
    }
  }

  // We allow G_CONSTANT of types < 32b.
  const unsigned MovOpc =
      DefSize == 64 ? AArch64::MOVi64imm : AArch64::MOVi32imm;

  if (isFP) {
    // Either emit a FMOV, or emit a copy to emit a normal mov.
    const TargetRegisterClass &GPRRC =
        DefSize == 32 ? AArch64::GPR32RegClass : AArch64::GPR64RegClass;
    const TargetRegisterClass &FPRRC = 
        DefSize == 32 ? AArch64::FPR32RegClass 
                      : (DefSize == 64 ? AArch64::FPR64RegClass 
                                       : AArch64::FPR128RegClass);

    // For 64b values, emit a constant pool load instead.
    // For s32, use a cp load if we have optsize/minsize.
    if (DefSize == 64 || DefSize == 128 ||
        (DefSize == 32 && shouldOptForSize(&MF))) {
      auto *FPImm = I.getOperand(1).getFPImm();
      auto *LoadMI = emitLoadFromConstantPool(FPImm, MIB);
      if (!LoadMI) {
        LLVM_DEBUG(dbgs() << "Failed to load double constant pool entry\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Failed to load double constant pool entry\n"
; } } while (false);
        return false;
      }
      MIB.buildCopy({DefReg}, {LoadMI->getOperand(0).getReg()});
      I.eraseFromParent();
      return RBI.constrainGenericRegister(DefReg, FPRRC, MRI);
    }

    // Nope. Emit a copy and use a normal mov instead.
    const Register DefGPRReg = MRI.createVirtualRegister(&GPRRC);
    MachineOperand &RegOp = I.getOperand(0);
    RegOp.setReg(DefGPRReg);
    MIB.setInsertPt(MIB.getMBB(), std::next(I.getIterator()));
    MIB.buildCopy({DefReg}, {DefGPRReg});

    if (!RBI.constrainGenericRegister(DefReg, FPRRC, MRI)) {
      LLVM_DEBUG(dbgs() << "Failed to constrain G_FCONSTANT def operand\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Failed to constrain G_FCONSTANT def operand\n"
; } } while (false);
      return false;
    }

    MachineOperand &ImmOp = I.getOperand(1);
    // FIXME: Is going through int64_t always correct?
    ImmOp.ChangeToImmediate(
        ImmOp.getFPImm()->getValueAPF().bitcastToAPInt().getZExtValue());
  } else if (I.getOperand(1).isCImm()) {
    uint64_t Val = I.getOperand(1).getCImm()->getZExtValue();
    I.getOperand(1).ChangeToImmediate(Val);
  } else if (I.getOperand(1).isImm()) {
    uint64_t Val = I.getOperand(1).getImm();
    I.getOperand(1).ChangeToImmediate(Val);
  }

  I.setDesc(TII.get(MovOpc));
  constrainSelectedInstRegOperands(I, TII, TRI, RBI);
  return true;
}
case TargetOpcode::G_EXTRACT: {
  Register DstReg = I.getOperand(0).getReg();
  Register SrcReg = I.getOperand(1).getReg();
  LLT SrcTy = MRI.getType(SrcReg);
  LLT DstTy = MRI.getType(DstReg);
  (void)DstTy;
  unsigned SrcSize = SrcTy.getSizeInBits();

  if (SrcTy.getSizeInBits() > 64) {
    // This should be an extract of an s128, which is like a vector extract.
    if (SrcTy.getSizeInBits() != 128)
      return false;
    // Only support extracting 64 bits from an s128 at the moment.
    if (DstTy.getSizeInBits() != 64)
      return false;

    unsigned Offset = I.getOperand(2).getImm();
    if (Offset % 64 != 0)
      return false;

    // Check we have the right regbank always.
    const RegisterBank &SrcRB = *RBI.getRegBank(SrcReg, MRI, TRI);
    const RegisterBank &DstRB = *RBI.getRegBank(DstReg, MRI, TRI);
    assert(SrcRB.getID() == DstRB.getID() && "Wrong extract regbank!")(static_cast <bool> (SrcRB.getID() == DstRB.getID() &&
 "Wrong extract regbank!") ? void (0) : __assert_fail ("SrcRB.getID() == DstRB.getID() && \"Wrong extract regbank!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2582, __extension__ __PRETTY_FUNCTION__));

    if (SrcRB.getID() == AArch64::GPRRegBankID) {
      MIB.buildInstr(TargetOpcode::COPY, {DstReg}, {})
          .addUse(SrcReg, 0, Offset == 0 ? AArch64::sube64 : AArch64::subo64);
      I.eraseFromParent();
      return true;
    }

    // Emit the same code as a vector extract.
    // Offset must be a multiple of 64.
    unsigned LaneIdx = Offset / 64;
    MachineInstr *Extract = emitExtractVectorElt(
        DstReg, DstRB, LLT::scalar(64), SrcReg, LaneIdx, MIB);
    if (!Extract)
      return false;
    I.eraseFromParent();
    return true;
  }

  I.setDesc(TII.get(SrcSize == 64 ? AArch64::UBFMXri : AArch64::UBFMWri));
  MachineInstrBuilder(MF, I).addImm(I.getOperand(2).getImm() +
                                    Ty.getSizeInBits() - 1);

  if (SrcSize < 64) {
    assert(SrcSize == 32 && DstTy.getSizeInBits() == 16 &&(static_cast <bool> (SrcSize == 32 && DstTy.getSizeInBits
() == 16 && "unexpected G_EXTRACT types") ? void (0) :
 __assert_fail ("SrcSize == 32 && DstTy.getSizeInBits() == 16 && \"unexpected G_EXTRACT types\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2608, __extension__ __PRETTY_FUNCTION__))
           "unexpected G_EXTRACT types")(static_cast <bool> (SrcSize == 32 && DstTy.getSizeInBits
() == 16 && "unexpected G_EXTRACT types") ? void (0) :
 __assert_fail ("SrcSize == 32 && DstTy.getSizeInBits() == 16 && \"unexpected G_EXTRACT types\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2608, __extension__ __PRETTY_FUNCTION__));
    return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
  }

  DstReg = MRI.createGenericVirtualRegister(LLT::scalar(64));
  MIB.setInsertPt(MIB.getMBB(), std::next(I.getIterator()));
  MIB.buildInstr(TargetOpcode::COPY, {I.getOperand(0).getReg()}, {})
      .addReg(DstReg, 0, AArch64::sub_32);
  RBI.constrainGenericRegister(I.getOperand(0).getReg(),
                               AArch64::GPR32RegClass, MRI);
  I.getOperand(0).setReg(DstReg);

  return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
}

case TargetOpcode::G_INSERT: {
  LLT SrcTy = MRI.getType(I.getOperand(2).getReg());
  LLT DstTy = MRI.getType(I.getOperand(0).getReg());
  unsigned DstSize = DstTy.getSizeInBits();
  // Larger inserts are vectors, same-size ones should be something else by
  // now (split up or turned into COPYs).
  if (Ty.getSizeInBits() > 64 || SrcTy.getSizeInBits() > 32)
    return false;

  I.setDesc(TII.get(DstSize == 64 ? AArch64::BFMXri : AArch64::BFMWri));
  unsigned LSB = I.getOperand(3).getImm();
  unsigned Width = MRI.getType(I.getOperand(2).getReg()).getSizeInBits();
  I.getOperand(3).setImm((DstSize - LSB) % DstSize);
  MachineInstrBuilder(MF, I).addImm(Width - 1);

  if (DstSize < 64) {
    assert(DstSize == 32 && SrcTy.getSizeInBits() == 16 &&(static_cast <bool> (DstSize == 32 && SrcTy.getSizeInBits
() == 16 && "unexpected G_INSERT types") ? void (0) :
 __assert_fail ("DstSize == 32 && SrcTy.getSizeInBits() == 16 && \"unexpected G_INSERT types\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2640, __extension__ __PRETTY_FUNCTION__))
           "unexpected G_INSERT types")(static_cast <bool> (DstSize == 32 && SrcTy.getSizeInBits
() == 16 && "unexpected G_INSERT types") ? void (0) :
 __assert_fail ("DstSize == 32 && SrcTy.getSizeInBits() == 16 && \"unexpected G_INSERT types\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2640, __extension__ __PRETTY_FUNCTION__));
    return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
  }

  Register SrcReg = MRI.createGenericVirtualRegister(LLT::scalar(64));
  BuildMI(MBB, I.getIterator(), I.getDebugLoc(),
          TII.get(AArch64::SUBREG_TO_REG))
      .addDef(SrcReg)
      .addImm(0)
      .addUse(I.getOperand(2).getReg())
      .addImm(AArch64::sub_32);
  RBI.constrainGenericRegister(I.getOperand(2).getReg(),
                               AArch64::GPR32RegClass, MRI);
  I.getOperand(2).setReg(SrcReg);

  return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
}
case TargetOpcode::G_FRAME_INDEX: {
  // allocas and G_FRAME_INDEX are only supported in addrspace(0).
  if (Ty != LLT::pointer(0, 64)) {
    LLVM_DEBUG(dbgs() << "G_FRAME_INDEX pointer has type: " << Tydo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "G_FRAME_INDEX pointer has type: "
 << Ty << ", expected: " << LLT::pointer(0,
 64) << '\n'; } } while (false)
                      << ", expected: " << LLT::pointer(0, 64) << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "G_FRAME_INDEX pointer has type: "
 << Ty << ", expected: " << LLT::pointer(0,
 64) << '\n'; } } while (false);
    return false;
  }
  I.setDesc(TII.get(AArch64::ADDXri));

  // MOs for a #0 shifted immediate.
  I.addOperand(MachineOperand::CreateImm(0));
  I.addOperand(MachineOperand::CreateImm(0));

  return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
}

case TargetOpcode::G_GLOBAL_VALUE: {
  auto GV = I.getOperand(1).getGlobal();
  if (GV->isThreadLocal())
    return selectTLSGlobalValue(I, MRI);

  unsigned OpFlags = STI.ClassifyGlobalReference(GV, TM);
  if (OpFlags & AArch64II::MO_GOT) {
    I.setDesc(TII.get(AArch64::LOADgot));
    I.getOperand(1).setTargetFlags(OpFlags);
  } else if (TM.getCodeModel() == CodeModel::Large) {
    // Materialize the global using movz/movk instructions.
    materializeLargeCMVal(I, GV, OpFlags);
    I.eraseFromParent();
    return true;
  } else if (TM.getCodeModel() == CodeModel::Tiny) {
    I.setDesc(TII.get(AArch64::ADR));
    I.getOperand(1).setTargetFlags(OpFlags);
  } else {
    I.setDesc(TII.get(AArch64::MOVaddr));
    I.getOperand(1).setTargetFlags(OpFlags | AArch64II::MO_PAGE);
    MachineInstrBuilder MIB(MF, I);
    MIB.addGlobalAddress(GV, I.getOperand(1).getOffset(),
                         OpFlags | AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
  }
  return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
}

case TargetOpcode::G_ZEXTLOAD:
case TargetOpcode::G_LOAD:
case TargetOpcode::G_STORE: {
  GLoadStore &LdSt = cast<GLoadStore>(I);
  bool IsZExtLoad = I.getOpcode() == TargetOpcode::G_ZEXTLOAD;
  LLT PtrTy = MRI.getType(LdSt.getPointerReg());

  if (PtrTy != LLT::pointer(0, 64)) {
    LLVM_DEBUG(dbgs() << "Load/Store pointer has type: " << PtrTydo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Load/Store pointer has type: "
 << PtrTy << ", expected: " << LLT::pointer
(0, 64) << '\n'; } } while (false)
                      << ", expected: " << LLT::pointer(0, 64) << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Load/Store pointer has type: "
 << PtrTy << ", expected: " << LLT::pointer
(0, 64) << '\n'; } } while (false);
    return false;
  }

  uint64_t MemSizeInBytes = LdSt.getMemSize();
  unsigned MemSizeInBits = LdSt.getMemSizeInBits();
  AtomicOrdering Order = LdSt.getMMO().getSuccessOrdering();

  // Need special instructions for atomics that affect ordering.
  if (Order != AtomicOrdering::NotAtomic &&
      Order != AtomicOrdering::Unordered &&
      Order != AtomicOrdering::Monotonic) {
    assert(!isa<GZExtLoad>(LdSt))(static_cast <bool> (!isa<GZExtLoad>(LdSt)) ? void
 (0) : __assert_fail ("!isa<GZExtLoad>(LdSt)", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2721, __extension__ __PRETTY_FUNCTION__));
    if (MemSizeInBytes > 64)
      return false;

    if (isa<GLoad>(LdSt)) {
      static unsigned Opcodes[] = {AArch64::LDARB, AArch64::LDARH,
                                   AArch64::LDARW, AArch64::LDARX};
      I.setDesc(TII.get(Opcodes[Log2_32(MemSizeInBytes)]));
    } else {
      static unsigned Opcodes[] = {AArch64::STLRB, AArch64::STLRH,
                                   AArch64::STLRW, AArch64::STLRX};
      I.setDesc(TII.get(Opcodes[Log2_32(MemSizeInBytes)]));
    }
    constrainSelectedInstRegOperands(I, TII, TRI, RBI);
    return true;
  }

2738#ifndef NDEBUG
  const Register PtrReg = LdSt.getPointerReg();
  const RegisterBank &PtrRB = *RBI.getRegBank(PtrReg, MRI, TRI);
  // Sanity-check the pointer register.
  assert(PtrRB.getID() == AArch64::GPRRegBankID &&(static_cast <bool> (PtrRB.getID() == AArch64::GPRRegBankID
 && "Load/Store pointer operand isn't a GPR") ? void (
0) : __assert_fail ("PtrRB.getID() == AArch64::GPRRegBankID && \"Load/Store pointer operand isn't a GPR\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2743, __extension__ __PRETTY_FUNCTION__))
         "Load/Store pointer operand isn't a GPR")(static_cast <bool> (PtrRB.getID() == AArch64::GPRRegBankID
 && "Load/Store pointer operand isn't a GPR") ? void (
0) : __assert_fail ("PtrRB.getID() == AArch64::GPRRegBankID && \"Load/Store pointer operand isn't a GPR\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2743, __extension__ __PRETTY_FUNCTION__));
  assert(MRI.getType(PtrReg).isPointer() &&(static_cast <bool> (MRI.getType(PtrReg).isPointer() &&
 "Load/Store pointer operand isn't a pointer") ? void (0) : __assert_fail
 ("MRI.getType(PtrReg).isPointer() && \"Load/Store pointer operand isn't a pointer\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2745, __extension__ __PRETTY_FUNCTION__))
         "Load/Store pointer operand isn't a pointer")(static_cast <bool> (MRI.getType(PtrReg).isPointer() &&
 "Load/Store pointer operand isn't a pointer") ? void (0) : __assert_fail
 ("MRI.getType(PtrReg).isPointer() && \"Load/Store pointer operand isn't a pointer\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2745, __extension__ __PRETTY_FUNCTION__));
2746#endif

  const Register ValReg = LdSt.getReg(0);
  const LLT ValTy = MRI.getType(ValReg);
  const RegisterBank &RB = *RBI.getRegBank(ValReg, MRI, TRI);

  // The code below doesn't support truncating stores, so we need to split it
  // again.
  if (isa<GStore>(LdSt) && ValTy.getSizeInBits() > MemSizeInBits) {
    unsigned SubReg;
    LLT MemTy = LdSt.getMMO().getMemoryType();
    auto *RC = getRegClassForTypeOnBank(MemTy, RB, RBI);
    if (!getSubRegForClass(RC, TRI, SubReg))
      return false;

    // Generate a subreg copy.
    auto Copy = MIB.buildInstr(TargetOpcode::COPY, {MemTy}, {})
                    .addReg(ValReg, 0, SubReg)
                    .getReg(0);
    RBI.constrainGenericRegister(Copy, *RC, MRI);
    LdSt.getOperand(0).setReg(Copy);
  }

  // Helper lambda for partially selecting I. Either returns the original
  // instruction with an updated opcode, or a new instruction.
  auto SelectLoadStoreAddressingMode = [&]() -> MachineInstr * {
    bool IsStore = isa<GStore>(I);
1
Assuming 'I' is not a 'GStore'→
    const unsigned NewOpc =
        selectLoadStoreUIOp(I.getOpcode(), RB.getID(), MemSizeInBits);
    if (NewOpc == I.getOpcode())
2
←
Taking false branch→
      return nullptr;
    // Check if we can fold anything into the addressing mode.
    auto AddrModeFns =
        selectAddrModeIndexed(I.getOperand(1), MemSizeInBytes);
3
←
Calling 'AArch64InstructionSelector::selectAddrModeIndexed'→
    if (!AddrModeFns) {
      // Can't fold anything. Use the original instruction.
      I.setDesc(TII.get(NewOpc));
      I.addOperand(MachineOperand::CreateImm(0));
      return &I;
    }

    // Folded something. Create a new instruction and return it.
    auto NewInst = MIB.buildInstr(NewOpc, {}, {}, I.getFlags());
    Register CurValReg = I.getOperand(0).getReg();
    IsStore ? NewInst.addUse(CurValReg) : NewInst.addDef(CurValReg);
    NewInst.cloneMemRefs(I);
    for (auto &Fn : *AddrModeFns)
      Fn(NewInst);
    I.eraseFromParent();
    return &*NewInst;
  };

  MachineInstr *LoadStore = SelectLoadStoreAddressingMode();
  if (!LoadStore)
    return false;

  // If we're storing a 0, use WZR/XZR.
  if (Opcode == TargetOpcode::G_STORE) {
    auto CVal = getConstantVRegValWithLookThrough(
        LoadStore->getOperand(0).getReg(), MRI, /*LookThroughInstrs = */ true,
        /*HandleFConstants = */ false);
    if (CVal && CVal->Value == 0) {
      switch (LoadStore->getOpcode()) {
      case AArch64::STRWui:
      case AArch64::STRHHui:
      case AArch64::STRBBui:
        LoadStore->getOperand(0).setReg(AArch64::WZR);
        break;
      case AArch64::STRXui:
        LoadStore->getOperand(0).setReg(AArch64::XZR);
        break;
      }
    }
  }

  if (IsZExtLoad) {
    // The zextload from a smaller type to i32 should be handled by the
    // importer.
    if (MRI.getType(LoadStore->getOperand(0).getReg()).getSizeInBits() != 64)
      return false;
    // If we have a ZEXTLOAD then change the load's type to be a narrower reg
    // and zero_extend with SUBREG_TO_REG.
    Register LdReg = MRI.createVirtualRegister(&AArch64::GPR32RegClass);
    Register DstReg = LoadStore->getOperand(0).getReg();
    LoadStore->getOperand(0).setReg(LdReg);

    MIB.setInsertPt(MIB.getMBB(), std::next(LoadStore->getIterator()));
    MIB.buildInstr(AArch64::SUBREG_TO_REG, {DstReg}, {})
        .addImm(0)
        .addUse(LdReg)
        .addImm(AArch64::sub_32);
    constrainSelectedInstRegOperands(*LoadStore, TII, TRI, RBI);
    return RBI.constrainGenericRegister(DstReg, AArch64::GPR64allRegClass,
                                        MRI);
  }
  return constrainSelectedInstRegOperands(*LoadStore, TII, TRI, RBI);
}

case TargetOpcode::G_SMULH:
case TargetOpcode::G_UMULH: {
  // Reject the various things we don't support yet.
  if (unsupportedBinOp(I, RBI, MRI, TRI))
    return false;

  const Register DefReg = I.getOperand(0).getReg();
  const RegisterBank &RB = *RBI.getRegBank(DefReg, MRI, TRI);

  if (RB.getID() != AArch64::GPRRegBankID) {
    LLVM_DEBUG(dbgs() << "G_[SU]MULH on bank: " << RB << ", expected: GPR\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "G_[SU]MULH on bank: " <<
 RB << ", expected: GPR\n"; } } while (false);
    return false;
  }

  if (Ty != LLT::scalar(64)) {
    LLVM_DEBUG(dbgs() << "G_[SU]MULH has type: " << Tydo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "G_[SU]MULH has type: " <<
 Ty << ", expected: " << LLT::scalar(64) <<
 '\n'; } } while (false)
                      << ", expected: " << LLT::scalar(64) << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "G_[SU]MULH has type: " <<
 Ty << ", expected: " << LLT::scalar(64) <<
 '\n'; } } while (false);
    return false;
  }

  unsigned NewOpc = I.getOpcode() == TargetOpcode::G_SMULH ? AArch64::SMULHrr
                                                           : AArch64::UMULHrr;
  I.setDesc(TII.get(NewOpc));

  // Now that we selected an opcode, we need to constrain the register
  // operands to use appropriate classes.
  return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
}
case TargetOpcode::G_LSHR:
case TargetOpcode::G_ASHR:
  if (MRI.getType(I.getOperand(0).getReg()).isVector())
    return selectVectorAshrLshr(I, MRI);
  LLVM_FALLTHROUGH[[gnu::fallthrough]];
case TargetOpcode::G_SHL:
  if (Opcode == TargetOpcode::G_SHL &&
      MRI.getType(I.getOperand(0).getReg()).isVector())
    return selectVectorSHL(I, MRI);
  LLVM_FALLTHROUGH[[gnu::fallthrough]];
case TargetOpcode::G_FADD:
case TargetOpcode::G_FSUB:
case TargetOpcode::G_FMUL:
case TargetOpcode::G_FDIV:
case TargetOpcode::G_OR: {
  // Reject the various things we don't support yet.
  if (unsupportedBinOp(I, RBI, MRI, TRI))
    return false;

  const unsigned OpSize = Ty.getSizeInBits();

  const Register DefReg = I.getOperand(0).getReg();
  const RegisterBank &RB = *RBI.getRegBank(DefReg, MRI, TRI);

  const unsigned NewOpc = selectBinaryOp(I.getOpcode(), RB.getID(), OpSize);
  if (NewOpc == I.getOpcode())
    return false;

  I.setDesc(TII.get(NewOpc));
  // FIXME: Should the type be always reset in setDesc?

  // Now that we selected an opcode, we need to constrain the register
  // operands to use appropriate classes.
  return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
}

case TargetOpcode::G_PTR_ADD: {
  emitADD(I.getOperand(0).getReg(), I.getOperand(1), I.getOperand(2), MIB);
  I.eraseFromParent();
  return true;
}
case TargetOpcode::G_SADDO:
case TargetOpcode::G_UADDO:
case TargetOpcode::G_SSUBO:
case TargetOpcode::G_USUBO: {
  // Emit the operation and get the correct condition code.
  auto OpAndCC = emitOverflowOp(Opcode, I.getOperand(0).getReg(),
                                I.getOperand(2), I.getOperand(3), MIB);

  // Now, put the overflow result in the register given by the first operand
  // to the overflow op. CSINC increments the result when the predicate is
  // false, so to get the increment when it's true, we need to use the
  // inverse. In this case, we want to increment when carry is set.
  Register ZReg = AArch64::WZR;
  auto CsetMI = MIB.buildInstr(AArch64::CSINCWr, {I.getOperand(1).getReg()},
                               {ZReg, ZReg})
                    .addImm(getInvertedCondCode(OpAndCC.second));
  constrainSelectedInstRegOperands(*CsetMI, TII, TRI, RBI);
  I.eraseFromParent();
  return true;
}

case TargetOpcode::G_PTRMASK: {
  Register MaskReg = I.getOperand(2).getReg();
  Optional<int64_t> MaskVal = getConstantVRegSExtVal(MaskReg, MRI);
  // TODO: Implement arbitrary cases
  if (!MaskVal || !isShiftedMask_64(*MaskVal))
    return false;

  uint64_t Mask = *MaskVal;
  I.setDesc(TII.get(AArch64::ANDXri));
  I.getOperand(2).ChangeToImmediate(
      AArch64_AM::encodeLogicalImmediate(Mask, 64));

  return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
}
case TargetOpcode::G_PTRTOINT:
case TargetOpcode::G_TRUNC: {
  const LLT DstTy = MRI.getType(I.getOperand(0).getReg());
  const LLT SrcTy = MRI.getType(I.getOperand(1).getReg());

  const Register DstReg = I.getOperand(0).getReg();
  const Register SrcReg = I.getOperand(1).getReg();

  const RegisterBank &DstRB = *RBI.getRegBank(DstReg, MRI, TRI);
  const RegisterBank &SrcRB = *RBI.getRegBank(SrcReg, MRI, TRI);

  if (DstRB.getID() != SrcRB.getID()) {
    LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "G_TRUNC/G_PTRTOINT input/output on different banks\n"
; } } while (false)
        dbgs() << "G_TRUNC/G_PTRTOINT input/output on different banks\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "G_TRUNC/G_PTRTOINT input/output on different banks\n"
; } } while (false);
    return false;
  }

  if (DstRB.getID() == AArch64::GPRRegBankID) {
    const TargetRegisterClass *DstRC =
        getRegClassForTypeOnBank(DstTy, DstRB, RBI);
    if (!DstRC)
      return false;

    const TargetRegisterClass *SrcRC =
        getRegClassForTypeOnBank(SrcTy, SrcRB, RBI);
    if (!SrcRC)
      return false;

    if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, MRI) ||
        !RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) {
      LLVM_DEBUG(dbgs() << "Failed to constrain G_TRUNC/G_PTRTOINT\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Failed to constrain G_TRUNC/G_PTRTOINT\n"
; } } while (false);
      return false;
    }

    if (DstRC == SrcRC) {
      // Nothing to be done
    } else if (Opcode == TargetOpcode::G_TRUNC && DstTy == LLT::scalar(32) &&
               SrcTy == LLT::scalar(64)) {
      llvm_unreachable("TableGen can import this case")::llvm::llvm_unreachable_internal("TableGen can import this case"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2986);
      return false;
    } else if (DstRC == &AArch64::GPR32RegClass &&
               SrcRC == &AArch64::GPR64RegClass) {
      I.getOperand(1).setSubReg(AArch64::sub_32);
    } else {
      LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unhandled mismatched classes in G_TRUNC/G_PTRTOINT\n"
; } } while (false)
          dbgs() << "Unhandled mismatched classes in G_TRUNC/G_PTRTOINT\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unhandled mismatched classes in G_TRUNC/G_PTRTOINT\n"
; } } while (false);
      return false;
    }

    I.setDesc(TII.get(TargetOpcode::COPY));
    return true;
  } else if (DstRB.getID() == AArch64::FPRRegBankID) {
    if (DstTy == LLT::fixed_vector(4, 16) &&
        SrcTy == LLT::fixed_vector(4, 32)) {
      I.setDesc(TII.get(AArch64::XTNv4i16));
      constrainSelectedInstRegOperands(I, TII, TRI, RBI);
      return true;
    }

    if (!SrcTy.isVector() && SrcTy.getSizeInBits() == 128) {
      MachineInstr *Extract = emitExtractVectorElt(
          DstReg, DstRB, LLT::scalar(DstTy.getSizeInBits()), SrcReg, 0, MIB);
      if (!Extract)
        return false;
      I.eraseFromParent();
      return true;
    }

    // We might have a vector G_PTRTOINT, in which case just emit a COPY.
    if (Opcode == TargetOpcode::G_PTRTOINT) {
      assert(DstTy.isVector() && "Expected an FPR ptrtoint to be a vector")(static_cast <bool> (DstTy.isVector() && "Expected an FPR ptrtoint to be a vector"
) ? void (0) : __assert_fail ("DstTy.isVector() && \"Expected an FPR ptrtoint to be a vector\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3018, __extension__ __PRETTY_FUNCTION__));
      I.setDesc(TII.get(TargetOpcode::COPY));
      return selectCopy(I, TII, MRI, TRI, RBI);
    }
  }

  return false;
}

case TargetOpcode::G_ANYEXT: {
  const Register DstReg = I.getOperand(0).getReg();
  const Register SrcReg = I.getOperand(1).getReg();

  const RegisterBank &RBDst = *RBI.getRegBank(DstReg, MRI, TRI);
  if (RBDst.getID() != AArch64::GPRRegBankID) {
    LLVM_DEBUG(dbgs() << "G_ANYEXT on bank: " << RBDstdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "G_ANYEXT on bank: " <<
 RBDst << ", expected: GPR\n"; } } while (false)
                      << ", expected: GPR\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "G_ANYEXT on bank: " <<
 RBDst << ", expected: GPR\n"; } } while (false);
    return false;
  }

  const RegisterBank &RBSrc = *RBI.getRegBank(SrcReg, MRI, TRI);
  if (RBSrc.getID() != AArch64::GPRRegBankID) {
    LLVM_DEBUG(dbgs() << "G_ANYEXT on bank: " << RBSrcdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "G_ANYEXT on bank: " <<
 RBSrc << ", expected: GPR\n"; } } while (false)
                      << ", expected: GPR\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "G_ANYEXT on bank: " <<
 RBSrc << ", expected: GPR\n"; } } while (false);
    return false;
  }

  const unsigned DstSize = MRI.getType(DstReg).getSizeInBits();

  if (DstSize == 0) {
    LLVM_DEBUG(dbgs() << "G_ANYEXT operand has no size, not a gvreg?\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "G_ANYEXT operand has no size, not a gvreg?\n"
; } } while (false);
    return false;
  }

  if (DstSize != 64 && DstSize > 32) {
    LLVM_DEBUG(dbgs() << "G_ANYEXT to size: " << DstSizedo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "G_ANYEXT to size: " <<
 DstSize << ", expected: 32 or 64\n"; } } while (false)
                      << ", expected: 32 or 64\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "G_ANYEXT to size: " <<
 DstSize << ", expected: 32 or 64\n"; } } while (false);
    return false;
  }
  // At this point G_ANYEXT is just like a plain COPY, but we need
  // to explicitly form the 64-bit value if any.
  if (DstSize > 32) {
    Register ExtSrc = MRI.createVirtualRegister(&AArch64::GPR64allRegClass);
    BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::SUBREG_TO_REG))
        .addDef(ExtSrc)
        .addImm(0)
        .addUse(SrcReg)
        .addImm(AArch64::sub_32);
    I.getOperand(1).setReg(ExtSrc);
  }
  return selectCopy(I, TII, MRI, TRI, RBI);
}

case TargetOpcode::G_ZEXT:
case TargetOpcode::G_SEXT_INREG:
case TargetOpcode::G_SEXT: {
  unsigned Opcode = I.getOpcode();
  const bool IsSigned = Opcode != TargetOpcode::G_ZEXT;
  const Register DefReg = I.getOperand(0).getReg();
  Register SrcReg = I.getOperand(1).getReg();
  const LLT DstTy = MRI.getType(DefReg);
  const LLT SrcTy = MRI.getType(SrcReg);
  unsigned DstSize = DstTy.getSizeInBits();
  unsigned SrcSize = SrcTy.getSizeInBits();

  // SEXT_INREG has the same src reg size as dst, the size of the value to be
  // extended is encoded in the imm.
  if (Opcode == TargetOpcode::G_SEXT_INREG)
    SrcSize = I.getOperand(2).getImm();

  if (DstTy.isVector())
    return false; // Should be handled by imported patterns.

  assert((*RBI.getRegBank(DefReg, MRI, TRI)).getID() ==(static_cast <bool> ((*RBI.getRegBank(DefReg, MRI, TRI)
).getID() == AArch64::GPRRegBankID && "Unexpected ext regbank"
) ? void (0) : __assert_fail ("(*RBI.getRegBank(DefReg, MRI, TRI)).getID() == AArch64::GPRRegBankID && \"Unexpected ext regbank\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3093, __extension__ __PRETTY_FUNCTION__))
             AArch64::GPRRegBankID &&(static_cast <bool> ((*RBI.getRegBank(DefReg, MRI, TRI)
).getID() == AArch64::GPRRegBankID && "Unexpected ext regbank"
) ? void (0) : __assert_fail ("(*RBI.getRegBank(DefReg, MRI, TRI)).getID() == AArch64::GPRRegBankID && \"Unexpected ext regbank\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3093, __extension__ __PRETTY_FUNCTION__))
         "Unexpected ext regbank")(static_cast <bool> ((*RBI.getRegBank(DefReg, MRI, TRI)
).getID() == AArch64::GPRRegBankID && "Unexpected ext regbank"
) ? void (0) : __assert_fail ("(*RBI.getRegBank(DefReg, MRI, TRI)).getID() == AArch64::GPRRegBankID && \"Unexpected ext regbank\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3093, __extension__ __PRETTY_FUNCTION__));

  MachineInstr *ExtI;

  // First check if we're extending the result of a load which has a dest type
  // smaller than 32 bits, then this zext is redundant. GPR32 is the smallest
  // GPR register on AArch64 and all loads which are smaller automatically
  // zero-extend the upper bits. E.g.
  // %v(s8) = G_LOAD %p, :: (load 1)
  // %v2(s32) = G_ZEXT %v(s8)
  if (!IsSigned) {
    auto *LoadMI = getOpcodeDef(TargetOpcode::G_LOAD, SrcReg, MRI);
    bool IsGPR =
        RBI.getRegBank(SrcReg, MRI, TRI)->getID() == AArch64::GPRRegBankID;
    if (LoadMI && IsGPR) {
      const MachineMemOperand *MemOp = *LoadMI->memoperands_begin();
      unsigned BytesLoaded = MemOp->getSize();
      if (BytesLoaded < 4 && SrcTy.getSizeInBytes() == BytesLoaded)
        return selectCopy(I, TII, MRI, TRI, RBI);
    }

    // For the 32-bit -> 64-bit case, we can emit a mov (ORRWrs)
    // + SUBREG_TO_REG.
    //
    // If we are zero extending from 32 bits to 64 bits, it's possible that
    // the instruction implicitly does the zero extend for us. In that case,
    // we only need the SUBREG_TO_REG.
    if (IsGPR && SrcSize == 32 && DstSize == 64) {
      // Unlike with the G_LOAD case, we don't want to look through copies
      // here. (See isDef32.)
      MachineInstr *Def = MRI.getVRegDef(SrcReg);
      Register SubregToRegSrc = SrcReg;

      // Does the instruction implicitly zero extend?
      if (!Def || !isDef32(*Def)) {
        // No. Zero out using an OR.
        Register OrDst = MRI.createVirtualRegister(&AArch64::GPR32RegClass);
        const Register ZReg = AArch64::WZR;
        MIB.buildInstr(AArch64::ORRWrs, {OrDst}, {ZReg, SrcReg}).addImm(0);
        SubregToRegSrc = OrDst;
      }

      MIB.buildInstr(AArch64::SUBREG_TO_REG, {DefReg}, {})
          .addImm(0)
          .addUse(SubregToRegSrc)
          .addImm(AArch64::sub_32);

      if (!RBI.constrainGenericRegister(DefReg, AArch64::GPR64RegClass,
                                        MRI)) {
        LLVM_DEBUG(dbgs() << "Failed to constrain G_ZEXT destination\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Failed to constrain G_ZEXT destination\n"
; } } while (false);
        return false;
      }

      if (!RBI.constrainGenericRegister(SrcReg, AArch64::GPR32RegClass,
                                        MRI)) {
        LLVM_DEBUG(dbgs() << "Failed to constrain G_ZEXT source\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Failed to constrain G_ZEXT source\n"
; } } while (false);
        return false;
      }

      I.eraseFromParent();
      return true;
    }
  }

  if (DstSize == 64) {
    if (Opcode != TargetOpcode::G_SEXT_INREG) {
      // FIXME: Can we avoid manually doing this?
      if (!RBI.constrainGenericRegister(SrcReg, AArch64::GPR32RegClass,
                                        MRI)) {
        LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(Opcode)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Failed to constrain " <<
 TII.getName(Opcode) << " operand\n"; } } while (false)
                          << " operand\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Failed to constrain " <<
 TII.getName(Opcode) << " operand\n"; } } while (false);
        return false;
      }
      SrcReg = MIB.buildInstr(AArch64::SUBREG_TO_REG,
                              {&AArch64::GPR64RegClass}, {})
                   .addImm(0)
                   .addUse(SrcReg)
                   .addImm(AArch64::sub_32)
                   .getReg(0);
    }

    ExtI = MIB.buildInstr(IsSigned ? AArch64::SBFMXri : AArch64::UBFMXri,
                           {DefReg}, {SrcReg})
                .addImm(0)
                .addImm(SrcSize - 1);
  } else if (DstSize <= 32) {
    ExtI = MIB.buildInstr(IsSigned ? AArch64::SBFMWri : AArch64::UBFMWri,
                           {DefReg}, {SrcReg})
                .addImm(0)
                .addImm(SrcSize - 1);
  } else {
    return false;
  }

  constrainSelectedInstRegOperands(*ExtI, TII, TRI, RBI);
  I.eraseFromParent();
  return true;
}

case TargetOpcode::G_SITOFP:
case TargetOpcode::G_UITOFP:
case TargetOpcode::G_FPTOSI:
case TargetOpcode::G_FPTOUI: {
  const LLT DstTy = MRI.getType(I.getOperand(0).getReg()),
            SrcTy = MRI.getType(I.getOperand(1).getReg());
  const unsigned NewOpc = selectFPConvOpc(Opcode, DstTy, SrcTy);
  if (NewOpc == Opcode)
    return false;

  I.setDesc(TII.get(NewOpc));
  constrainSelectedInstRegOperands(I, TII, TRI, RBI);

  return true;
}

case TargetOpcode::G_FREEZE:
  return selectCopy(I, TII, MRI, TRI, RBI);

case TargetOpcode::G_INTTOPTR:
  // The importer is currently unable to import pointer types since they
  // didn't exist in SelectionDAG.
  return selectCopy(I, TII, MRI, TRI, RBI);

case TargetOpcode::G_BITCAST:
  // Imported SelectionDAG rules can handle every bitcast except those that
  // bitcast from a type to the same type. Ideally, these shouldn't occur
  // but we might not run an optimizer that deletes them. The other exception
  // is bitcasts involving pointer types, as SelectionDAG has no knowledge
  // of them.
  return selectCopy(I, TII, MRI, TRI, RBI);

case TargetOpcode::G_SELECT: {
  if (MRI.getType(I.getOperand(1).getReg()) != LLT::scalar(1)) {
    LLVM_DEBUG(dbgs() << "G_SELECT cond has type: " << Tydo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "G_SELECT cond has type: "
 << Ty << ", expected: " << LLT::scalar(1) <<
 '\n'; } } while (false)
                      << ", expected: " << LLT::scalar(1) << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "G_SELECT cond has type: "
 << Ty << ", expected: " << LLT::scalar(1) <<
 '\n'; } } while (false);
    return false;
  }

  const Register CondReg = I.getOperand(1).getReg();
  const Register TReg = I.getOperand(2).getReg();
  const Register FReg = I.getOperand(3).getReg();

  if (tryOptSelect(I))
    return true;

  // Make sure to use an unused vreg instead of wzr, so that the peephole
  // optimizations will be able to optimize these.
  Register DeadVReg = MRI.createVirtualRegister(&AArch64::GPR32RegClass);
  auto TstMI = MIB.buildInstr(AArch64::ANDSWri, {DeadVReg}, {CondReg})
                   .addImm(AArch64_AM::encodeLogicalImmediate(1, 32));
  constrainSelectedInstRegOperands(*TstMI, TII, TRI, RBI);
  if (!emitSelect(I.getOperand(0).getReg(), TReg, FReg, AArch64CC::NE, MIB))
    return false;
  I.eraseFromParent();
  return true;
}
case TargetOpcode::G_ICMP: {
  if (Ty.isVector())
    return selectVectorICmp(I, MRI);

  if (Ty != LLT::scalar(32)) {
    LLVM_DEBUG(dbgs() << "G_ICMP result has type: " << Tydo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "G_ICMP result has type: "
 << Ty << ", expected: " << LLT::scalar(32)
 << '\n'; } } while (false)
                      << ", expected: " << LLT::scalar(32) << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "G_ICMP result has type: "
 << Ty << ", expected: " << LLT::scalar(32)
 << '\n'; } } while (false);
    return false;
  }

  auto Pred = static_cast<CmpInst::Predicate>(I.getOperand(1).getPredicate());
  emitIntegerCompare(I.getOperand(2), I.getOperand(3), I.getOperand(1),
                     MIB);
  emitCSetForICMP(I.getOperand(0).getReg(), Pred, MIB);
  I.eraseFromParent();
  return true;
}

case TargetOpcode::G_FCMP: {
  CmpInst::Predicate Pred =
      static_cast<CmpInst::Predicate>(I.getOperand(1).getPredicate());
  if (!emitFPCompare(I.getOperand(2).getReg(), I.getOperand(3).getReg(), MIB,
                     Pred) ||
      !emitCSetForFCmp(I.getOperand(0).getReg(), Pred, MIB))
    return false;
  I.eraseFromParent();
  return true;
}
case TargetOpcode::G_VASTART:
  return STI.isTargetDarwin() ? selectVaStartDarwin(I, MF, MRI)
                              : selectVaStartAAPCS(I, MF, MRI);
case TargetOpcode::G_INTRINSIC:
  return selectIntrinsic(I, MRI);
case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS:
  return selectIntrinsicWithSideEffects(I, MRI);
case TargetOpcode::G_IMPLICIT_DEF: {
  I.setDesc(TII.get(TargetOpcode::IMPLICIT_DEF));
  const LLT DstTy = MRI.getType(I.getOperand(0).getReg());
  const Register DstReg = I.getOperand(0).getReg();
  const RegisterBank &DstRB = *RBI.getRegBank(DstReg, MRI, TRI);
  const TargetRegisterClass *DstRC =
      getRegClassForTypeOnBank(DstTy, DstRB, RBI);
  RBI.constrainGenericRegister(DstReg, *DstRC, MRI);
  return true;
}
case TargetOpcode::G_BLOCK_ADDR: {
  if (TM.getCodeModel() == CodeModel::Large) {
    materializeLargeCMVal(I, I.getOperand(1).getBlockAddress(), 0);
    I.eraseFromParent();
    return true;
  } else {
    I.setDesc(TII.get(AArch64::MOVaddrBA));
    auto MovMI = BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::MOVaddrBA),
                         I.getOperand(0).getReg())
                     .addBlockAddress(I.getOperand(1).getBlockAddress(),
                                      /* Offset */ 0, AArch64II::MO_PAGE)
                     .addBlockAddress(
                         I.getOperand(1).getBlockAddress(), /* Offset */ 0,
                         AArch64II::MO_NC | AArch64II::MO_PAGEOFF);
    I.eraseFromParent();
    return constrainSelectedInstRegOperands(*MovMI, TII, TRI, RBI);
  }
}
case AArch64::G_DUP: {
  // When the scalar of G_DUP is an s8/s16 gpr, they can't be selected by
  // imported patterns. Do it manually here. Avoiding generating s16 gpr is
  // difficult because at RBS we may end up pessimizing the fpr case if we
  // decided to add an anyextend to fix this. Manual selection is the most
  // robust solution for now.
  if (RBI.getRegBank(I.getOperand(1).getReg(), MRI, TRI)->getID() !=
      AArch64::GPRRegBankID)
    return false; // We expect the fpr regbank case to be imported.
  LLT VecTy = MRI.getType(I.getOperand(0).getReg());
  if (VecTy == LLT::fixed_vector(8, 8))
    I.setDesc(TII.get(AArch64::DUPv8i8gpr));
  else if (VecTy == LLT::fixed_vector(16, 8))
    I.setDesc(TII.get(AArch64::DUPv16i8gpr));
  else if (VecTy == LLT::fixed_vector(4, 16))
    I.setDesc(TII.get(AArch64::DUPv4i16gpr));
  else if (VecTy == LLT::fixed_vector(8, 16))
    I.setDesc(TII.get(AArch64::DUPv8i16gpr));
  else
    return false;
  return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
}
case TargetOpcode::G_INTRINSIC_TRUNC:
  return selectIntrinsicTrunc(I, MRI);
case TargetOpcode::G_INTRINSIC_ROUND:
  return selectIntrinsicRound(I, MRI);
case TargetOpcode::G_BUILD_VECTOR:
  return selectBuildVector(I, MRI);
case TargetOpcode::G_MERGE_VALUES:
  return selectMergeValues(I, MRI);
case TargetOpcode::G_UNMERGE_VALUES:
  return selectUnmergeValues(I, MRI);
case TargetOpcode::G_SHUFFLE_VECTOR:
  return selectShuffleVector(I, MRI);
case TargetOpcode::G_EXTRACT_VECTOR_ELT:
  return selectExtractElt(I, MRI);
case TargetOpcode::G_INSERT_VECTOR_ELT:
  return selectInsertElt(I, MRI);
case TargetOpcode::G_CONCAT_VECTORS:
  return selectConcatVectors(I, MRI);
case TargetOpcode::G_JUMP_TABLE:
  return selectJumpTable(I, MRI);
case TargetOpcode::G_VECREDUCE_FADD:
case TargetOpcode::G_VECREDUCE_ADD:
  return selectReduction(I, MRI);
}

return false;
3360}

3362bool AArch64InstructionSelector::selectReduction(MachineInstr &I,
                                               MachineRegisterInfo &MRI) {
Register VecReg = I.getOperand(1).getReg();
LLT VecTy = MRI.getType(VecReg);
if (I.getOpcode() == TargetOpcode::G_VECREDUCE_ADD) {
  // For <2 x i32> ADDPv2i32 generates an FPR64 value, so we need to emit
  // a subregister copy afterwards.
  if (VecTy == LLT::fixed_vector(2, 32)) {
    Register DstReg = I.getOperand(0).getReg();
    auto AddP = MIB.buildInstr(AArch64::ADDPv2i32, {&AArch64::FPR64RegClass},
                               {VecReg, VecReg});
    auto Copy = MIB.buildInstr(TargetOpcode::COPY, {DstReg}, {})
                    .addReg(AddP.getReg(0), 0, AArch64::ssub)
                    .getReg(0);
    RBI.constrainGenericRegister(Copy, AArch64::FPR32RegClass, MRI);
    I.eraseFromParent();
    return constrainSelectedInstRegOperands(*AddP, TII, TRI, RBI);
  }

  unsigned Opc = 0;
  if (VecTy == LLT::fixed_vector(16, 8))
    Opc = AArch64::ADDVv16i8v;
  else if (VecTy == LLT::fixed_vector(8, 16))
    Opc = AArch64::ADDVv8i16v;
  else if (VecTy == LLT::fixed_vector(4, 32))
    Opc = AArch64::ADDVv4i32v;
  else if (VecTy == LLT::fixed_vector(2, 64))
    Opc = AArch64::ADDPv2i64p;
  else {
    LLVM_DEBUG(dbgs() << "Unhandled type for add reduction")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unhandled type for add reduction"
; } } while (false);
    return false;
  }
  I.setDesc(TII.get(Opc));
  return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
}

if (I.getOpcode() == TargetOpcode::G_VECREDUCE_FADD) {
  unsigned Opc = 0;
  if (VecTy == LLT::fixed_vector(2, 32))
    Opc = AArch64::FADDPv2i32p;
  else if (VecTy == LLT::fixed_vector(2, 64))
    Opc = AArch64::FADDPv2i64p;
  else {
    LLVM_DEBUG(dbgs() << "Unhandled type for fadd reduction")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unhandled type for fadd reduction"
; } } while (false);
    return false;
  }
  I.setDesc(TII.get(Opc));
  return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
}
return false;
3412}

3414bool AArch64InstructionSelector::selectBrJT(MachineInstr &I,
                                          MachineRegisterInfo &MRI) {
assert(I.getOpcode() == TargetOpcode::G_BRJT && "Expected G_BRJT")(static_cast <bool> (I.getOpcode() == TargetOpcode::G_BRJT
 && "Expected G_BRJT") ? void (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_BRJT && \"Expected G_BRJT\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3416, __extension__ __PRETTY_FUNCTION__));
Register JTAddr = I.getOperand(0).getReg();
unsigned JTI = I.getOperand(1).getIndex();
Register Index = I.getOperand(2).getReg();

Register TargetReg = MRI.createVirtualRegister(&AArch64::GPR64RegClass);
Register ScratchReg = MRI.createVirtualRegister(&AArch64::GPR64spRegClass);

MF->getInfo<AArch64FunctionInfo>()->setJumpTableEntryInfo(JTI, 4, nullptr);
auto JumpTableInst = MIB.buildInstr(AArch64::JumpTableDest32,
                                    {TargetReg, ScratchReg}, {JTAddr, Index})
                         .addJumpTableIndex(JTI);
// Build the indirect branch.
MIB.buildInstr(AArch64::BR, {}, {TargetReg});
I.eraseFromParent();
return constrainSelectedInstRegOperands(*JumpTableInst, TII, TRI, RBI);
3432}

3434bool AArch64InstructionSelector::selectJumpTable(MachineInstr &I,
                                               MachineRegisterInfo &MRI) {
assert(I.getOpcode() == TargetOpcode::G_JUMP_TABLE && "Expected jump table")(static_cast <bool> (I.getOpcode() == TargetOpcode::G_JUMP_TABLE
 && "Expected jump table") ? void (0) : __assert_fail
 ("I.getOpcode() == TargetOpcode::G_JUMP_TABLE && \"Expected jump table\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3436, __extension__ __PRETTY_FUNCTION__));
assert(I.getOperand(1).isJTI() && "Jump table op should have a JTI!")(static_cast <bool> (I.getOperand(1).isJTI() &&
 "Jump table op should have a JTI!") ? void (0) : __assert_fail
 ("I.getOperand(1).isJTI() && \"Jump table op should have a JTI!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3437, __extension__ __PRETTY_FUNCTION__));

Register DstReg = I.getOperand(0).getReg();
unsigned JTI = I.getOperand(1).getIndex();
// We generate a MOVaddrJT which will get expanded to an ADRP + ADD later.
auto MovMI =
  MIB.buildInstr(AArch64::MOVaddrJT, {DstReg}, {})
        .addJumpTableIndex(JTI, AArch64II::MO_PAGE)
        .addJumpTableIndex(JTI, AArch64II::MO_NC | AArch64II::MO_PAGEOFF);
I.eraseFromParent();
return constrainSelectedInstRegOperands(*MovMI, TII, TRI, RBI);
3448}

3450bool AArch64InstructionSelector::selectTLSGlobalValue(
  MachineInstr &I, MachineRegisterInfo &MRI) {
if (!STI.isTargetMachO())
  return false;
MachineFunction &MF = *I.getParent()->getParent();
MF.getFrameInfo().setAdjustsStack(true);

const auto &GlobalOp = I.getOperand(1);
assert(GlobalOp.getOffset() == 0 &&(static_cast <bool> (GlobalOp.getOffset() == 0 &&
 "Shouldn't have an offset on TLS globals!") ? void (0) : __assert_fail
 ("GlobalOp.getOffset() == 0 && \"Shouldn't have an offset on TLS globals!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3459, __extension__ __PRETTY_FUNCTION__))
       "Shouldn't have an offset on TLS globals!")(static_cast <bool> (GlobalOp.getOffset() == 0 &&
 "Shouldn't have an offset on TLS globals!") ? void (0) : __assert_fail
 ("GlobalOp.getOffset() == 0 && \"Shouldn't have an offset on TLS globals!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3459, __extension__ __PRETTY_FUNCTION__));
const GlobalValue &GV = *GlobalOp.getGlobal();

auto LoadGOT =
    MIB.buildInstr(AArch64::LOADgot, {&AArch64::GPR64commonRegClass}, {})
        .addGlobalAddress(&GV, 0, AArch64II::MO_TLS);

auto Load = MIB.buildInstr(AArch64::LDRXui, {&AArch64::GPR64commonRegClass},
                           {LoadGOT.getReg(0)})
                .addImm(0);

MIB.buildCopy(Register(AArch64::X0), LoadGOT.getReg(0));
// TLS calls preserve all registers except those that absolutely must be
// trashed: X0 (it takes an argument), LR (it's a call) and NZCV (let's not be
// silly).
MIB.buildInstr(getBLRCallOpcode(MF), {}, {Load})
    .addUse(AArch64::X0, RegState::Implicit)
    .addDef(AArch64::X0, RegState::Implicit)
    .addRegMask(TRI.getTLSCallPreservedMask());

MIB.buildCopy(I.getOperand(0).getReg(), Register(AArch64::X0));
RBI.constrainGenericRegister(I.getOperand(0).getReg(), AArch64::GPR64RegClass,
                             MRI);
I.eraseFromParent();
return true;
3484}

3486bool AArch64InstructionSelector::selectIntrinsicTrunc(
  MachineInstr &I, MachineRegisterInfo &MRI) const {
const LLT SrcTy = MRI.getType(I.getOperand(0).getReg());

// Select the correct opcode.
unsigned Opc = 0;
if (!SrcTy.isVector()) {
  switch (SrcTy.getSizeInBits()) {
  default:
  case 16:
    Opc = AArch64::FRINTZHr;
    break;
  case 32:
    Opc = AArch64::FRINTZSr;
    break;
  case 64:
    Opc = AArch64::FRINTZDr;
    break;
  }
} else {
  unsigned NumElts = SrcTy.getNumElements();
  switch (SrcTy.getElementType().getSizeInBits()) {
  default:
    break;
  case 16:
    if (NumElts == 4)
      Opc = AArch64::FRINTZv4f16;
    else if (NumElts == 8)
      Opc = AArch64::FRINTZv8f16;
    break;
  case 32:
    if (NumElts == 2)
      Opc = AArch64::FRINTZv2f32;
    else if (NumElts == 4)
      Opc = AArch64::FRINTZv4f32;
    break;
  case 64:
    if (NumElts == 2)
      Opc = AArch64::FRINTZv2f64;
    break;
  }
}

if (!Opc) {
  // Didn't get an opcode above, bail.
  LLVM_DEBUG(dbgs() << "Unsupported type for G_INTRINSIC_TRUNC!\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unsupported type for G_INTRINSIC_TRUNC!\n"
; } } while (false);
  return false;
}

// Legalization would have set us up perfectly for this; we just need to
// set the opcode and move on.
I.setDesc(TII.get(Opc));
return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
3539}

3541bool AArch64InstructionSelector::selectIntrinsicRound(
  MachineInstr &I, MachineRegisterInfo &MRI) const {
const LLT SrcTy = MRI.getType(I.getOperand(0).getReg());

// Select the correct opcode.
unsigned Opc = 0;
if (!SrcTy.isVector()) {
  switch (SrcTy.getSizeInBits()) {
  default:
  case 16:
    Opc = AArch64::FRINTAHr;
    break;
  case 32:
    Opc = AArch64::FRINTASr;
    break;
  case 64:
    Opc = AArch64::FRINTADr;
    break;
  }
} else {
  unsigned NumElts = SrcTy.getNumElements();
  switch (SrcTy.getElementType().getSizeInBits()) {
  default:
    break;
  case 16:
    if (NumElts == 4)
      Opc = AArch64::FRINTAv4f16;
    else if (NumElts == 8)
      Opc = AArch64::FRINTAv8f16;
    break;
  case 32:
    if (NumElts == 2)
      Opc = AArch64::FRINTAv2f32;
    else if (NumElts == 4)
      Opc = AArch64::FRINTAv4f32;
    break;
  case 64:
    if (NumElts == 2)
      Opc = AArch64::FRINTAv2f64;
    break;
  }
}

if (!Opc) {
  // Didn't get an opcode above, bail.
  LLVM_DEBUG(dbgs() << "Unsupported type for G_INTRINSIC_ROUND!\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unsupported type for G_INTRINSIC_ROUND!\n"
; } } while (false);
  return false;
}

// Legalization would have set us up perfectly for this; we just need to
// set the opcode and move on.
I.setDesc(TII.get(Opc));
return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
3594}

3596bool AArch64InstructionSelector::selectVectorICmp(
  MachineInstr &I, MachineRegisterInfo &MRI) {
Register DstReg = I.getOperand(0).getReg();
LLT DstTy = MRI.getType(DstReg);
Register SrcReg = I.getOperand(2).getReg();
Register Src2Reg = I.getOperand(3).getReg();
LLT SrcTy = MRI.getType(SrcReg);

unsigned SrcEltSize = SrcTy.getElementType().getSizeInBits();
unsigned NumElts = DstTy.getNumElements();

// First index is element size, 0 == 8b, 1 == 16b, 2 == 32b, 3 == 64b
// Second index is num elts, 0 == v2, 1 == v4, 2 == v8, 3 == v16
// Third index is cc opcode:
// 0 == eq
// 1 == ugt
// 2 == uge
// 3 == ult
// 4 == ule
// 5 == sgt
// 6 == sge
// 7 == slt
// 8 == sle
// ne is done by negating 'eq' result.

// This table below assumes that for some comparisons the operands will be
// commuted.
// ult op == commute + ugt op
// ule op == commute + uge op
// slt op == commute + sgt op
// sle op == commute + sge op
unsigned PredIdx = 0;
bool SwapOperands = false;
CmpInst::Predicate Pred = (CmpInst::Predicate)I.getOperand(1).getPredicate();
switch (Pred) {
case CmpInst::ICMP_NE:
case CmpInst::ICMP_EQ:
  PredIdx = 0;
  break;
case CmpInst::ICMP_UGT:
  PredIdx = 1;
  break;
case CmpInst::ICMP_UGE:
  PredIdx = 2;
  break;
case CmpInst::ICMP_ULT:
  PredIdx = 3;
  SwapOperands = true;
  break;
case CmpInst::ICMP_ULE:
  PredIdx = 4;
  SwapOperands = true;
  break;
case CmpInst::ICMP_SGT:
  PredIdx = 5;
  break;
case CmpInst::ICMP_SGE:
  PredIdx = 6;
  break;
case CmpInst::ICMP_SLT:
  PredIdx = 7;
  SwapOperands = true;
  break;
case CmpInst::ICMP_SLE:
  PredIdx = 8;
  SwapOperands = true;
  break;
default:
  llvm_unreachable("Unhandled icmp predicate")::llvm::llvm_unreachable_internal("Unhandled icmp predicate",
 "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3664);
  return false;
}

// This table obviously should be tablegen'd when we have our GISel native
// tablegen selector.

static const unsigned OpcTable[4][4][9] = {
    {
        {0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
         0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
         0 /* invalid */},
        {0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
         0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
         0 /* invalid */},
        {AArch64::CMEQv8i8, AArch64::CMHIv8i8, AArch64::CMHSv8i8,
         AArch64::CMHIv8i8, AArch64::CMHSv8i8, AArch64::CMGTv8i8,
         AArch64::CMGEv8i8, AArch64::CMGTv8i8, AArch64::CMGEv8i8},
        {AArch64::CMEQv16i8, AArch64::CMHIv16i8, AArch64::CMHSv16i8,
         AArch64::CMHIv16i8, AArch64::CMHSv16i8, AArch64::CMGTv16i8,
         AArch64::CMGEv16i8, AArch64::CMGTv16i8, AArch64::CMGEv16i8}
    },
    {
        {0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
         0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
         0 /* invalid */},
        {AArch64::CMEQv4i16, AArch64::CMHIv4i16, AArch64::CMHSv4i16,
         AArch64::CMHIv4i16, AArch64::CMHSv4i16, AArch64::CMGTv4i16,
         AArch64::CMGEv4i16, AArch64::CMGTv4i16, AArch64::CMGEv4i16},
        {AArch64::CMEQv8i16, AArch64::CMHIv8i16, AArch64::CMHSv8i16,
         AArch64::CMHIv8i16, AArch64::CMHSv8i16, AArch64::CMGTv8i16,
         AArch64::CMGEv8i16, AArch64::CMGTv8i16, AArch64::CMGEv8i16},
        {0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
         0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
         0 /* invalid */}
    },
    {
        {AArch64::CMEQv2i32, AArch64::CMHIv2i32, AArch64::CMHSv2i32,
         AArch64::CMHIv2i32, AArch64::CMHSv2i32, AArch64::CMGTv2i32,
         AArch64::CMGEv2i32, AArch64::CMGTv2i32, AArch64::CMGEv2i32},
        {AArch64::CMEQv4i32, AArch64::CMHIv4i32, AArch64::CMHSv4i32,
         AArch64::CMHIv4i32, AArch64::CMHSv4i32, AArch64::CMGTv4i32,
         AArch64::CMGEv4i32, AArch64::CMGTv4i32, AArch64::CMGEv4i32},
        {0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
         0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
         0 /* invalid */},
        {0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
         0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
         0 /* invalid */}
    },
    {
        {AArch64::CMEQv2i64, AArch64::CMHIv2i64, AArch64::CMHSv2i64,
         AArch64::CMHIv2i64, AArch64::CMHSv2i64, AArch64::CMGTv2i64,
         AArch64::CMGEv2i64, AArch64::CMGTv2i64, AArch64::CMGEv2i64},
        {0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
         0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
         0 /* invalid */},
        {0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
         0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
         0 /* invalid */},
        {0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
         0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
         0 /* invalid */}
    },
};
unsigned EltIdx = Log2_32(SrcEltSize / 8);
unsigned NumEltsIdx = Log2_32(NumElts / 2);
unsigned Opc = OpcTable[EltIdx][NumEltsIdx][PredIdx];
if (!Opc) {
  LLVM_DEBUG(dbgs() << "Could not map G_ICMP to cmp opcode")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Could not map G_ICMP to cmp opcode"
; } } while (false);
  return false;
}

const RegisterBank &VecRB = *RBI.getRegBank(SrcReg, MRI, TRI);
const TargetRegisterClass *SrcRC =
    getRegClassForTypeOnBank(SrcTy, VecRB, RBI, true);
if (!SrcRC) {
  LLVM_DEBUG(dbgs() << "Could not determine source register class.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Could not determine source register class.\n"
; } } while (false);
  return false;
}

unsigned NotOpc = Pred == ICmpInst::ICMP_NE ? AArch64::NOTv8i8 : 0;
if (SrcTy.getSizeInBits() == 128)
  NotOpc = NotOpc ? AArch64::NOTv16i8 : 0;

if (SwapOperands)
  std::swap(SrcReg, Src2Reg);

auto Cmp = MIB.buildInstr(Opc, {SrcRC}, {SrcReg, Src2Reg});
constrainSelectedInstRegOperands(*Cmp, TII, TRI, RBI);

// Invert if we had a 'ne' cc.
if (NotOpc) {
  Cmp = MIB.buildInstr(NotOpc, {DstReg}, {Cmp});
  constrainSelectedInstRegOperands(*Cmp, TII, TRI, RBI);
} else {
  MIB.buildCopy(DstReg, Cmp.getReg(0));
}
RBI.constrainGenericRegister(DstReg, *SrcRC, MRI);
I.eraseFromParent();
return true;
3765}

3767MachineInstr *AArch64InstructionSelector::emitScalarToVector(
  unsigned EltSize, const TargetRegisterClass *DstRC, Register Scalar,
  MachineIRBuilder &MIRBuilder) const {
auto Undef = MIRBuilder.buildInstr(TargetOpcode::IMPLICIT_DEF, {DstRC}, {});

auto BuildFn = [&](unsigned SubregIndex) {
  auto Ins =
      MIRBuilder
          .buildInstr(TargetOpcode::INSERT_SUBREG, {DstRC}, {Undef, Scalar})
          .addImm(SubregIndex);
  constrainSelectedInstRegOperands(*Undef, TII, TRI, RBI);
  constrainSelectedInstRegOperands(*Ins, TII, TRI, RBI);
  return &*Ins;
};

switch (EltSize) {
case 16:
  return BuildFn(AArch64::hsub);
case 32:
  return BuildFn(AArch64::ssub);
case 64:
  return BuildFn(AArch64::dsub);
default:
  return nullptr;
}
3792}

3794bool AArch64InstructionSelector::selectMergeValues(
  MachineInstr &I, MachineRegisterInfo &MRI) {
assert(I.getOpcode() == TargetOpcode::G_MERGE_VALUES && "unexpected opcode")(static_cast <bool> (I.getOpcode() == TargetOpcode::G_MERGE_VALUES
 && "unexpected opcode") ? void (0) : __assert_fail (
"I.getOpcode() == TargetOpcode::G_MERGE_VALUES && \"unexpected opcode\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3796, __extension__ __PRETTY_FUNCTION__));
const LLT DstTy = MRI.getType(I.getOperand(0).getReg());
const LLT SrcTy = MRI.getType(I.getOperand(1).getReg());
assert(!DstTy.isVector() && !SrcTy.isVector() && "invalid merge operation")(static_cast <bool> (!DstTy.isVector() && !SrcTy
.isVector() && "invalid merge operation") ? void (0) :
 __assert_fail ("!DstTy.isVector() && !SrcTy.isVector() && \"invalid merge operation\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3799, __extension__ __PRETTY_FUNCTION__));
const RegisterBank &RB = *RBI.getRegBank(I.getOperand(1).getReg(), MRI, TRI);

if (I.getNumOperands() != 3)
  return false;

// Merging 2 s64s into an s128.
if (DstTy == LLT::scalar(128)) {
  if (SrcTy.getSizeInBits() != 64)
    return false;
  Register DstReg = I.getOperand(0).getReg();
  Register Src1Reg = I.getOperand(1).getReg();
  Register Src2Reg = I.getOperand(2).getReg();
  auto Tmp = MIB.buildInstr(TargetOpcode::IMPLICIT_DEF, {DstTy}, {});
  MachineInstr *InsMI =
      emitLaneInsert(None, Tmp.getReg(0), Src1Reg, /* LaneIdx */ 0, RB, MIB);
  if (!InsMI)
    return false;
  MachineInstr *Ins2MI = emitLaneInsert(DstReg, InsMI->getOperand(0).getReg(),
                                        Src2Reg, /* LaneIdx */ 1, RB, MIB);
  if (!Ins2MI)
    return false;
  constrainSelectedInstRegOperands(*InsMI, TII, TRI, RBI);
  constrainSelectedInstRegOperands(*Ins2MI, TII, TRI, RBI);
  I.eraseFromParent();
  return true;
}

if (RB.getID() != AArch64::GPRRegBankID)
  return false;

if (DstTy.getSizeInBits() != 64 || SrcTy.getSizeInBits() != 32)
  return false;

auto *DstRC = &AArch64::GPR64RegClass;
Register SubToRegDef = MRI.createVirtualRegister(DstRC);
MachineInstr &SubRegMI = *BuildMI(*I.getParent(), I, I.getDebugLoc(),
                                  TII.get(TargetOpcode::SUBREG_TO_REG))
                              .addDef(SubToRegDef)
                              .addImm(0)
                              .addUse(I.getOperand(1).getReg())
                              .addImm(AArch64::sub_32);
Register SubToRegDef2 = MRI.createVirtualRegister(DstRC);
// Need to anyext the second scalar before we can use bfm
MachineInstr &SubRegMI2 = *BuildMI(*I.getParent(), I, I.getDebugLoc(),
                                  TII.get(TargetOpcode::SUBREG_TO_REG))
                              .addDef(SubToRegDef2)
                              .addImm(0)
                              .addUse(I.getOperand(2).getReg())
                              .addImm(AArch64::sub_32);
MachineInstr &BFM =
    *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(AArch64::BFMXri))
         .addDef(I.getOperand(0).getReg())
         .addUse(SubToRegDef)
         .addUse(SubToRegDef2)
         .addImm(32)
         .addImm(31);
constrainSelectedInstRegOperands(SubRegMI, TII, TRI, RBI);
constrainSelectedInstRegOperands(SubRegMI2, TII, TRI, RBI);
constrainSelectedInstRegOperands(BFM, TII, TRI, RBI);
I.eraseFromParent();
return true;
3861}

3863static bool getLaneCopyOpcode(unsigned &CopyOpc, unsigned &ExtractSubReg,
                            const unsigned EltSize) {
// Choose a lane copy opcode and subregister based off of the size of the
// vector's elements.
switch (EltSize) {
case 8:
  CopyOpc = AArch64::CPYi8;
  ExtractSubReg = AArch64::bsub;
  break;
case 16:
  CopyOpc = AArch64::CPYi16;
  ExtractSubReg = AArch64::hsub;
  break;
case 32:
  CopyOpc = AArch64::CPYi32;
  ExtractSubReg = AArch64::ssub;
  break;
case 64:
  CopyOpc = AArch64::CPYi64;
  ExtractSubReg = AArch64::dsub;
  break;
default:
  // Unknown size, bail out.
  LLVM_DEBUG(dbgs() << "Elt size '" << EltSize << "' unsupported.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Elt size '" << EltSize
 << "' unsupported.\n"; } } while (false);
  return false;
}
return true;
3890}

3892MachineInstr *AArch64InstructionSelector::emitExtractVectorElt(
  Optional<Register> DstReg, const RegisterBank &DstRB, LLT ScalarTy,
  Register VecReg, unsigned LaneIdx, MachineIRBuilder &MIRBuilder) const {
MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
unsigned CopyOpc = 0;
unsigned ExtractSubReg = 0;
if (!getLaneCopyOpcode(CopyOpc, ExtractSubReg, ScalarTy.getSizeInBits())) {
  LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Couldn't determine lane copy opcode for instruction.\n"
; } } while (false)
      dbgs() << "Couldn't determine lane copy opcode for instruction.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Couldn't determine lane copy opcode for instruction.\n"
; } } while (false);
  return nullptr;
}

const TargetRegisterClass *DstRC =
    getRegClassForTypeOnBank(ScalarTy, DstRB, RBI, true);
if (!DstRC) {
  LLVM_DEBUG(dbgs() << "Could not determine destination register class.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Could not determine destination register class.\n"
; } } while (false);
  return nullptr;
}

const RegisterBank &VecRB = *RBI.getRegBank(VecReg, MRI, TRI);
const LLT &VecTy = MRI.getType(VecReg);
const TargetRegisterClass *VecRC =
    getRegClassForTypeOnBank(VecTy, VecRB, RBI, true);
if (!VecRC) {
  LLVM_DEBUG(dbgs() << "Could not determine source register class.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Could not determine source register class.\n"
; } } while (false);
  return nullptr;
}

// The register that we're going to copy into.
Register InsertReg = VecReg;
if (!DstReg)
  DstReg = MRI.createVirtualRegister(DstRC);
// If the lane index is 0, we just use a subregister COPY.
if (LaneIdx == 0) {
  auto Copy = MIRBuilder.buildInstr(TargetOpcode::COPY, {*DstReg}, {})
                  .addReg(VecReg, 0, ExtractSubReg);
  RBI.constrainGenericRegister(*DstReg, *DstRC, MRI);
  return &*Copy;
}

// Lane copies require 128-bit wide registers. If we're dealing with an
// unpacked vector, then we need to move up to that width. Insert an implicit
// def and a subregister insert to get us there.
if (VecTy.getSizeInBits() != 128) {
  MachineInstr *ScalarToVector = emitScalarToVector(
      VecTy.getSizeInBits(), &AArch64::FPR128RegClass, VecReg, MIRBuilder);
  if (!ScalarToVector)
    return nullptr;
  InsertReg = ScalarToVector->getOperand(0).getReg();
}

MachineInstr *LaneCopyMI =
    MIRBuilder.buildInstr(CopyOpc, {*DstReg}, {InsertReg}).addImm(LaneIdx);
constrainSelectedInstRegOperands(*LaneCopyMI, TII, TRI, RBI);

// Make sure that we actually constrain the initial copy.
RBI.constrainGenericRegister(*DstReg, *DstRC, MRI);
return LaneCopyMI;
3950}

3952bool AArch64InstructionSelector::selectExtractElt(
  MachineInstr &I, MachineRegisterInfo &MRI) {
assert(I.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT &&(static_cast <bool> (I.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT
 && "unexpected opcode!") ? void (0) : __assert_fail (
"I.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT && \"unexpected opcode!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3955, __extension__ __PRETTY_FUNCTION__))
       "unexpected opcode!")(static_cast <bool> (I.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT
 && "unexpected opcode!") ? void (0) : __assert_fail (
"I.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT && \"unexpected opcode!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3955, __extension__ __PRETTY_FUNCTION__));
Register DstReg = I.getOperand(0).getReg();
const LLT NarrowTy = MRI.getType(DstReg);
const Register SrcReg = I.getOperand(1).getReg();
const LLT WideTy = MRI.getType(SrcReg);
(void)WideTy;
assert(WideTy.getSizeInBits() >= NarrowTy.getSizeInBits() &&(static_cast <bool> (WideTy.getSizeInBits() >= NarrowTy
.getSizeInBits() && "source register size too small!"
) ? void (0) : __assert_fail ("WideTy.getSizeInBits() >= NarrowTy.getSizeInBits() && \"source register size too small!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3962, __extension__ __PRETTY_FUNCTION__))
       "source register size too small!")(static_cast <bool> (WideTy.getSizeInBits() >= NarrowTy
.getSizeInBits() && "source register size too small!"
) ? void (0) : __assert_fail ("WideTy.getSizeInBits() >= NarrowTy.getSizeInBits() && \"source register size too small!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3962, __extension__ __PRETTY_FUNCTION__));
assert(!NarrowTy.isVector() && "cannot extract vector into vector!")(static_cast <bool> (!NarrowTy.isVector() && "cannot extract vector into vector!"
) ? void (0) : __assert_fail ("!NarrowTy.isVector() && \"cannot extract vector into vector!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3963, __extension__ __PRETTY_FUNCTION__));

// Need the lane index to determine the correct copy opcode.
MachineOperand &LaneIdxOp = I.getOperand(2);
assert(LaneIdxOp.isReg() && "Lane index operand was not a register?")(static_cast <bool> (LaneIdxOp.isReg() && "Lane index operand was not a register?"
) ? void (0) : __assert_fail ("LaneIdxOp.isReg() && \"Lane index operand was not a register?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3967, __extension__ __PRETTY_FUNCTION__));

if (RBI.getRegBank(DstReg, MRI, TRI)->getID() != AArch64::FPRRegBankID) {
  LLVM_DEBUG(dbgs() << "Cannot extract into GPR.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Cannot extract into GPR.\n"
; } } while (false);
  return false;
}

// Find the index to extract from.
auto VRegAndVal = getConstantVRegValWithLookThrough(LaneIdxOp.getReg(), MRI);
if (!VRegAndVal)
  return false;
unsigned LaneIdx = VRegAndVal->Value.getSExtValue();


const RegisterBank &DstRB = *RBI.getRegBank(DstReg, MRI, TRI);
MachineInstr *Extract = emitExtractVectorElt(DstReg, DstRB, NarrowTy, SrcReg,
                                             LaneIdx, MIB);
if (!Extract)
  return false;

I.eraseFromParent();
return true;
3989}

3991bool AArch64InstructionSelector::selectSplitVectorUnmerge(
  MachineInstr &I, MachineRegisterInfo &MRI) {
unsigned NumElts = I.getNumOperands() - 1;
Register SrcReg = I.getOperand(NumElts).getReg();
const LLT NarrowTy = MRI.getType(I.getOperand(0).getReg());
const LLT SrcTy = MRI.getType(SrcReg);

assert(NarrowTy.isVector() && "Expected an unmerge into vectors")(static_cast <bool> (NarrowTy.isVector() && "Expected an unmerge into vectors"
) ? void (0) : __assert_fail ("NarrowTy.isVector() && \"Expected an unmerge into vectors\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3998, __extension__ __PRETTY_FUNCTION__));
if (SrcTy.getSizeInBits() > 128) {
  LLVM_DEBUG(dbgs() << "Unexpected vector type for vec split unmerge")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unexpected vector type for vec split unmerge"
; } } while (false);
  return false;
}

// We implement a split vector operation by treating the sub-vectors as
// scalars and extracting them.
const RegisterBank &DstRB =
    *RBI.getRegBank(I.getOperand(0).getReg(), MRI, TRI);
for (unsigned OpIdx = 0; OpIdx < NumElts; ++OpIdx) {
  Register Dst = I.getOperand(OpIdx).getReg();
  MachineInstr *Extract =
      emitExtractVectorElt(Dst, DstRB, NarrowTy, SrcReg, OpIdx, MIB);
  if (!Extract)
    return false;
}
I.eraseFromParent();
return true;
4017}

4019bool AArch64InstructionSelector::selectUnmergeValues(MachineInstr &I,
                                                   MachineRegisterInfo &MRI) {
assert(I.getOpcode() == TargetOpcode::G_UNMERGE_VALUES &&(static_cast <bool> (I.getOpcode() == TargetOpcode::G_UNMERGE_VALUES
 && "unexpected opcode") ? void (0) : __assert_fail (
"I.getOpcode() == TargetOpcode::G_UNMERGE_VALUES && \"unexpected opcode\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4022, __extension__ __PRETTY_FUNCTION__))
       "unexpected opcode")(static_cast <bool> (I.getOpcode() == TargetOpcode::G_UNMERGE_VALUES
 && "unexpected opcode") ? void (0) : __assert_fail (
"I.getOpcode() == TargetOpcode::G_UNMERGE_VALUES && \"unexpected opcode\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4022, __extension__ __PRETTY_FUNCTION__));

// TODO: Handle unmerging into GPRs and from scalars to scalars.
if (RBI.getRegBank(I.getOperand(0).getReg(), MRI, TRI)->getID() !=
        AArch64::FPRRegBankID ||
    RBI.getRegBank(I.getOperand(1).getReg(), MRI, TRI)->getID() !=
        AArch64::FPRRegBankID) {
  LLVM_DEBUG(dbgs() << "Unmerging vector-to-gpr and scalar-to-scalar "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unmerging vector-to-gpr and scalar-to-scalar "
 "currently unsupported.\n"; } } while (false)
                       "currently unsupported.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unmerging vector-to-gpr and scalar-to-scalar "
 "currently unsupported.\n"; } } while (false);
  return false;
}

// The last operand is the vector source register, and every other operand is
// a register to unpack into.
unsigned NumElts = I.getNumOperands() - 1;
Register SrcReg = I.getOperand(NumElts).getReg();
const LLT NarrowTy = MRI.getType(I.getOperand(0).getReg());
const LLT WideTy = MRI.getType(SrcReg);
(void)WideTy;
assert((WideTy.isVector() || WideTy.getSizeInBits() == 128) &&(static_cast <bool> ((WideTy.isVector() || WideTy.getSizeInBits
() == 128) && "can only unmerge from vector or s128 types!"
) ? void (0) : __assert_fail ("(WideTy.isVector() || WideTy.getSizeInBits() == 128) && \"can only unmerge from vector or s128 types!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4042, __extension__ __PRETTY_FUNCTION__))
       "can only unmerge from vector or s128 types!")(static_cast <bool> ((WideTy.isVector() || WideTy.getSizeInBits
() == 128) && "can only unmerge from vector or s128 types!"
) ? void (0) : __assert_fail ("(WideTy.isVector() || WideTy.getSizeInBits() == 128) && \"can only unmerge from vector or s128 types!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4042, __extension__ __PRETTY_FUNCTION__));
assert(WideTy.getSizeInBits() > NarrowTy.getSizeInBits() &&(static_cast <bool> (WideTy.getSizeInBits() > NarrowTy
.getSizeInBits() && "source register size too small!"
) ? void (0) : __assert_fail ("WideTy.getSizeInBits() > NarrowTy.getSizeInBits() && \"source register size too small!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4044, __extension__ __PRETTY_FUNCTION__))
       "source register size too small!")(static_cast <bool> (WideTy.getSizeInBits() > NarrowTy
.getSizeInBits() && "source register size too small!"
) ? void (0) : __assert_fail ("WideTy.getSizeInBits() > NarrowTy.getSizeInBits() && \"source register size too small!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4044, __extension__ __PRETTY_FUNCTION__));

if (!NarrowTy.isScalar())
  return selectSplitVectorUnmerge(I, MRI);

// Choose a lane copy opcode and subregister based off of the size of the
// vector's elements.
unsigned CopyOpc = 0;
unsigned ExtractSubReg = 0;
if (!getLaneCopyOpcode(CopyOpc, ExtractSubReg, NarrowTy.getSizeInBits()))
  return false;

// Set up for the lane copies.
MachineBasicBlock &MBB = *I.getParent();

// Stores the registers we'll be copying from.
SmallVector<Register, 4> InsertRegs;

// We'll use the first register twice, so we only need NumElts-1 registers.
unsigned NumInsertRegs = NumElts - 1;

// If our elements fit into exactly 128 bits, then we can copy from the source
// directly. Otherwise, we need to do a bit of setup with some subregister
// inserts.
if (NarrowTy.getSizeInBits() * NumElts == 128) {
  InsertRegs = SmallVector<Register, 4>(NumInsertRegs, SrcReg);
} else {
  // No. We have to perform subregister inserts. For each insert, create an
  // implicit def and a subregister insert, and save the register we create.
  const TargetRegisterClass *RC =
      getMinClassForRegBank(*RBI.getRegBank(SrcReg, MRI, TRI),
                            WideTy.getScalarSizeInBits() * NumElts);
  unsigned SubReg = 0;
  bool Found = getSubRegForClass(RC, TRI, SubReg);
  (void)Found;
  assert(Found && "expected to find last operand's subeg idx")(static_cast <bool> (Found && "expected to find last operand's subeg idx"
) ? void (0) : __assert_fail ("Found && \"expected to find last operand's subeg idx\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4079, __extension__ __PRETTY_FUNCTION__));
  for (unsigned Idx = 0; Idx < NumInsertRegs; ++Idx) {
    Register ImpDefReg = MRI.createVirtualRegister(&AArch64::FPR128RegClass);
    MachineInstr &ImpDefMI =
        *BuildMI(MBB, I, I.getDebugLoc(), TII.get(TargetOpcode::IMPLICIT_DEF),
                 ImpDefReg);

    // Now, create the subregister insert from SrcReg.
    Register InsertReg = MRI.createVirtualRegister(&AArch64::FPR128RegClass);
    MachineInstr &InsMI =
        *BuildMI(MBB, I, I.getDebugLoc(),
                 TII.get(TargetOpcode::INSERT_SUBREG), InsertReg)
             .addUse(ImpDefReg)
             .addUse(SrcReg)
             .addImm(SubReg);

    constrainSelectedInstRegOperands(ImpDefMI, TII, TRI, RBI);
    constrainSelectedInstRegOperands(InsMI, TII, TRI, RBI);

    // Save the register so that we can copy from it after.
    InsertRegs.push_back(InsertReg);
  }
}

// Now that we've created any necessary subregister inserts, we can
// create the copies.
//
// Perform the first copy separately as a subregister copy.
Register CopyTo = I.getOperand(0).getReg();
auto FirstCopy = MIB.buildInstr(TargetOpcode::COPY, {CopyTo}, {})
                     .addReg(InsertRegs[0], 0, ExtractSubReg);
constrainSelectedInstRegOperands(*FirstCopy, TII, TRI, RBI);

// Now, perform the remaining copies as vector lane copies.
unsigned LaneIdx = 1;
for (Register InsReg : InsertRegs) {
  Register CopyTo = I.getOperand(LaneIdx).getReg();
  MachineInstr &CopyInst =
      *BuildMI(MBB, I, I.getDebugLoc(), TII.get(CopyOpc), CopyTo)
           .addUse(InsReg)
           .addImm(LaneIdx);
  constrainSelectedInstRegOperands(CopyInst, TII, TRI, RBI);
  ++LaneIdx;
}

// Separately constrain the first copy's destination. Because of the
// limitation in constrainOperandRegClass, we can't guarantee that this will
// actually be constrained. So, do it ourselves using the second operand.
const TargetRegisterClass *RC =
    MRI.getRegClassOrNull(I.getOperand(1).getReg());
if (!RC) {
  LLVM_DEBUG(dbgs() << "Couldn't constrain copy destination.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Couldn't constrain copy destination.\n"
; } } while (false);
  return false;
}

RBI.constrainGenericRegister(CopyTo, *RC, MRI);
I.eraseFromParent();
return true;
4137}

4139bool AArch64InstructionSelector::selectConcatVectors(
  MachineInstr &I, MachineRegisterInfo &MRI)  {
assert(I.getOpcode() == TargetOpcode::G_CONCAT_VECTORS &&(static_cast <bool> (I.getOpcode() == TargetOpcode::G_CONCAT_VECTORS
 && "Unexpected opcode") ? void (0) : __assert_fail (
"I.getOpcode() == TargetOpcode::G_CONCAT_VECTORS && \"Unexpected opcode\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4142, __extension__ __PRETTY_FUNCTION__))
       "Unexpected opcode")(static_cast <bool> (I.getOpcode() == TargetOpcode::G_CONCAT_VECTORS
 && "Unexpected opcode") ? void (0) : __assert_fail (
"I.getOpcode() == TargetOpcode::G_CONCAT_VECTORS && \"Unexpected opcode\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4142, __extension__ __PRETTY_FUNCTION__));
Register Dst = I.getOperand(0).getReg();
Register Op1 = I.getOperand(1).getReg();
Register Op2 = I.getOperand(2).getReg();
MachineInstr *ConcatMI = emitVectorConcat(Dst, Op1, Op2, MIB);
if (!ConcatMI)
  return false;
I.eraseFromParent();
return true;
4151}

4153unsigned
4154AArch64InstructionSelector::emitConstantPoolEntry(const Constant *CPVal,
                                                MachineFunction &MF) const {
Type *CPTy = CPVal->getType();
Align Alignment = MF.getDataLayout().getPrefTypeAlign(CPTy);

MachineConstantPool *MCP = MF.getConstantPool();
return MCP->getConstantPoolIndex(CPVal, Alignment);
4161}

4163MachineInstr *AArch64InstructionSelector::emitLoadFromConstantPool(
  const Constant *CPVal, MachineIRBuilder &MIRBuilder) const {
auto &MF = MIRBuilder.getMF();
unsigned CPIdx = emitConstantPoolEntry(CPVal, MF);

auto Adrp =
    MIRBuilder.buildInstr(AArch64::ADRP, {&AArch64::GPR64RegClass}, {})
        .addConstantPoolIndex(CPIdx, 0, AArch64II::MO_PAGE);

MachineInstr *LoadMI = nullptr;
MachinePointerInfo PtrInfo = MachinePointerInfo::getConstantPool(MF);
unsigned Size = MIRBuilder.getDataLayout().getTypeStoreSize(CPVal->getType());
switch (Size) {
case 16:
  LoadMI =
      &*MIRBuilder
            .buildInstr(AArch64::LDRQui, {&AArch64::FPR128RegClass}, {Adrp})
            .addConstantPoolIndex(CPIdx, 0,
                                  AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
  break;
case 8:
  LoadMI =
      &*MIRBuilder
            .buildInstr(AArch64::LDRDui, {&AArch64::FPR64RegClass}, {Adrp})
            .addConstantPoolIndex(CPIdx, 0,
                                  AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
  break;
case 4:
  LoadMI =
      &*MIRBuilder
            .buildInstr(AArch64::LDRSui, {&AArch64::FPR32RegClass}, {Adrp})
            .addConstantPoolIndex(CPIdx, 0,
                                  AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
  break;
default:
  LLVM_DEBUG(dbgs() << "Could not load from constant pool of type "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Could not load from constant pool of type "
 << *CPVal->getType(); } } while (false)
                    << *CPVal->getType())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Could not load from constant pool of type "
 << *CPVal->getType(); } } while (false);
  return nullptr;
}
LoadMI->addMemOperand(MF, MF.getMachineMemOperand(PtrInfo,
                                                  MachineMemOperand::MOLoad,
                                                  Size, Align(Size)));
constrainSelectedInstRegOperands(*Adrp, TII, TRI, RBI);
constrainSelectedInstRegOperands(*LoadMI, TII, TRI, RBI);
return LoadMI;
4208}

4210/// Return an <Opcode, SubregIndex> pair to do an vector elt insert of a given
4211/// size and RB.
4212static std::pair<unsigned, unsigned>
4213getInsertVecEltOpInfo(const RegisterBank &RB, unsigned EltSize) {
unsigned Opc, SubregIdx;
if (RB.getID() == AArch64::GPRRegBankID) {
  if (EltSize == 16) {
    Opc = AArch64::INSvi16gpr;
    SubregIdx = AArch64::ssub;
  } else if (EltSize == 32) {
    Opc = AArch64::INSvi32gpr;
    SubregIdx = AArch64::ssub;
  } else if (EltSize == 64) {
    Opc = AArch64::INSvi64gpr;
    SubregIdx = AArch64::dsub;
  } else {
    llvm_unreachable("invalid elt size!")::llvm::llvm_unreachable_internal("invalid elt size!", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4226);
  }
} else {
  if (EltSize == 8) {
    Opc = AArch64::INSvi8lane;
    SubregIdx = AArch64::bsub;
  } else if (EltSize == 16) {
    Opc = AArch64::INSvi16lane;
    SubregIdx = AArch64::hsub;
  } else if (EltSize == 32) {
    Opc = AArch64::INSvi32lane;
    SubregIdx = AArch64::ssub;
  } else if (EltSize == 64) {
    Opc = AArch64::INSvi64lane;
    SubregIdx = AArch64::dsub;
  } else {
    llvm_unreachable("invalid elt size!")::llvm::llvm_unreachable_internal("invalid elt size!", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4242);
  }
}
return std::make_pair(Opc, SubregIdx);
4246}

4248MachineInstr *AArch64InstructionSelector::emitInstr(
  unsigned Opcode, std::initializer_list<llvm::DstOp> DstOps,
  std::initializer_list<llvm::SrcOp> SrcOps, MachineIRBuilder &MIRBuilder,
  const ComplexRendererFns &RenderFns) const {
assert(Opcode && "Expected an opcode?")(static_cast <bool> (Opcode && "Expected an opcode?"
) ? void (0) : __assert_fail ("Opcode && \"Expected an opcode?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4252, __extension__ __PRETTY_FUNCTION__));
assert(!isPreISelGenericOpcode(Opcode) &&(static_cast <bool> (!isPreISelGenericOpcode(Opcode) &&
 "Function should only be used to produce selected instructions!"
) ? void (0) : __assert_fail ("!isPreISelGenericOpcode(Opcode) && \"Function should only be used to produce selected instructions!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4254, __extension__ __PRETTY_FUNCTION__))
       "Function should only be used to produce selected instructions!")(static_cast <bool> (!isPreISelGenericOpcode(Opcode) &&
 "Function should only be used to produce selected instructions!"
) ? void (0) : __assert_fail ("!isPreISelGenericOpcode(Opcode) && \"Function should only be used to produce selected instructions!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4254, __extension__ __PRETTY_FUNCTION__));
auto MI = MIRBuilder.buildInstr(Opcode, DstOps, SrcOps);
if (RenderFns)
  for (auto &Fn : *RenderFns)
    Fn(MI);
constrainSelectedInstRegOperands(*MI, TII, TRI, RBI);
return &*MI;
4261}

4263MachineInstr *AArch64InstructionSelector::emitAddSub(
  const std::array<std::array<unsigned, 2>, 5> &AddrModeAndSizeToOpcode,
  Register Dst, MachineOperand &LHS, MachineOperand &RHS,
  MachineIRBuilder &MIRBuilder) const {
MachineRegisterInfo &MRI = MIRBuilder.getMF().getRegInfo();
assert(LHS.isReg() && RHS.isReg() && "Expected register operands?")(static_cast <bool> (LHS.isReg() && RHS.isReg()
 && "Expected register operands?") ? void (0) : __assert_fail
 ("LHS.isReg() && RHS.isReg() && \"Expected register operands?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4268, __extension__ __PRETTY_FUNCTION__));
auto Ty = MRI.getType(LHS.getReg());
assert(!Ty.isVector() && "Expected a scalar or pointer?")(static_cast <bool> (!Ty.isVector() && "Expected a scalar or pointer?"
) ? void (0) : __assert_fail ("!Ty.isVector() && \"Expected a scalar or pointer?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4270, __extension__ __PRETTY_FUNCTION__));
unsigned Size = Ty.getSizeInBits();
assert((Size == 32 || Size == 64) && "Expected a 32-bit or 64-bit type only")(static_cast <bool> ((Size == 32 || Size == 64) &&
 "Expected a 32-bit or 64-bit type only") ? void (0) : __assert_fail
 ("(Size == 32 || Size == 64) && \"Expected a 32-bit or 64-bit type only\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4272, __extension__ __PRETTY_FUNCTION__));
bool Is32Bit = Size == 32;

// INSTRri form with positive arithmetic immediate.
if (auto Fns = selectArithImmed(RHS))
  return emitInstr(AddrModeAndSizeToOpcode[0][Is32Bit], {Dst}, {LHS},
                   MIRBuilder, Fns);

// INSTRri form with negative arithmetic immediate.
if (auto Fns = selectNegArithImmed(RHS))
  return emitInstr(AddrModeAndSizeToOpcode[3][Is32Bit], {Dst}, {LHS},
                   MIRBuilder, Fns);

// INSTRrx form.
if (auto Fns = selectArithExtendedRegister(RHS))
  return emitInstr(AddrModeAndSizeToOpcode[4][Is32Bit], {Dst}, {LHS},
                   MIRBuilder, Fns);

// INSTRrs form.
if (auto Fns = selectShiftedRegister(RHS))
  return emitInstr(AddrModeAndSizeToOpcode[1][Is32Bit], {Dst}, {LHS},
                   MIRBuilder, Fns);
return emitInstr(AddrModeAndSizeToOpcode[2][Is32Bit], {Dst}, {LHS, RHS},
                 MIRBuilder);
4296}

4298MachineInstr *
4299AArch64InstructionSelector::emitADD(Register DefReg, MachineOperand &LHS,
                                  MachineOperand &RHS,
                                  MachineIRBuilder &MIRBuilder) const {
const std::array<std::array<unsigned, 2>, 5> OpcTable{
    {{AArch64::ADDXri, AArch64::ADDWri},
     {AArch64::ADDXrs, AArch64::ADDWrs},
     {AArch64::ADDXrr, AArch64::ADDWrr},
     {AArch64::SUBXri, AArch64::SUBWri},
     {AArch64::ADDXrx, AArch64::ADDWrx}}};
return emitAddSub(OpcTable, DefReg, LHS, RHS, MIRBuilder);
4309}

4311MachineInstr *
4312AArch64InstructionSelector::emitADDS(Register Dst, MachineOperand &LHS,
                                   MachineOperand &RHS,
                                   MachineIRBuilder &MIRBuilder) const {
const std::array<std::array<unsigned, 2>, 5> OpcTable{
    {{AArch64::ADDSXri, AArch64::ADDSWri},
     {AArch64::ADDSXrs, AArch64::ADDSWrs},
     {AArch64::ADDSXrr, AArch64::ADDSWrr},
     {AArch64::SUBSXri, AArch64::SUBSWri},
     {AArch64::ADDSXrx, AArch64::ADDSWrx}}};
return emitAddSub(OpcTable, Dst, LHS, RHS, MIRBuilder);
4322}

4324MachineInstr *
4325AArch64InstructionSelector::emitSUBS(Register Dst, MachineOperand &LHS,
                                   MachineOperand &RHS,
                                   MachineIRBuilder &MIRBuilder) const {
const std::array<std::array<unsigned, 2>, 5> OpcTable{
    {{AArch64::SUBSXri, AArch64::SUBSWri},
     {AArch64::SUBSXrs, AArch64::SUBSWrs},
     {AArch64::SUBSXrr, AArch64::SUBSWrr},
     {AArch64::ADDSXri, AArch64::ADDSWri},
     {AArch64::SUBSXrx, AArch64::SUBSWrx}}};
return emitAddSub(OpcTable, Dst, LHS, RHS, MIRBuilder);
4335}

4337MachineInstr *
4338AArch64InstructionSelector::emitCMN(MachineOperand &LHS, MachineOperand &RHS,
                                  MachineIRBuilder &MIRBuilder) const {
MachineRegisterInfo &MRI = MIRBuilder.getMF().getRegInfo();
bool Is32Bit = (MRI.getType(LHS.getReg()).getSizeInBits() == 32);
auto RC = Is32Bit ? &AArch64::GPR32RegClass : &AArch64::GPR64RegClass;
return emitADDS(MRI.createVirtualRegister(RC), LHS, RHS, MIRBuilder);
4344}

4346MachineInstr *
4347AArch64InstructionSelector::emitTST(MachineOperand &LHS, MachineOperand &RHS,
                                  MachineIRBuilder &MIRBuilder) const {
assert(LHS.isReg() && RHS.isReg() && "Expected register operands?")(static_cast <bool> (LHS.isReg() && RHS.isReg()
 && "Expected register operands?") ? void (0) : __assert_fail
 ("LHS.isReg() && RHS.isReg() && \"Expected register operands?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4349, __extension__ __PRETTY_FUNCTION__));
MachineRegisterInfo &MRI = MIRBuilder.getMF().getRegInfo();
LLT Ty = MRI.getType(LHS.getReg());
unsigned RegSize = Ty.getSizeInBits();
bool Is32Bit = (RegSize == 32);
const unsigned OpcTable[3][2] = {{AArch64::ANDSXri, AArch64::ANDSWri},
                                 {AArch64::ANDSXrs, AArch64::ANDSWrs},
                                 {AArch64::ANDSXrr, AArch64::ANDSWrr}};
// ANDS needs a logical immediate for its immediate form. Check if we can
// fold one in.
if (auto ValAndVReg = getConstantVRegValWithLookThrough(RHS.getReg(), MRI)) {
  int64_t Imm = ValAndVReg->Value.getSExtValue();

  if (AArch64_AM::isLogicalImmediate(Imm, RegSize)) {
    auto TstMI = MIRBuilder.buildInstr(OpcTable[0][Is32Bit], {Ty}, {LHS});
    TstMI.addImm(AArch64_AM::encodeLogicalImmediate(Imm, RegSize));
    constrainSelectedInstRegOperands(*TstMI, TII, TRI, RBI);
    return &*TstMI;
  }
}

if (auto Fns = selectLogicalShiftedRegister(RHS))
  return emitInstr(OpcTable[1][Is32Bit], {Ty}, {LHS}, MIRBuilder, Fns);
return emitInstr(OpcTable[2][Is32Bit], {Ty}, {LHS, RHS}, MIRBuilder);
4373}

4375MachineInstr *AArch64InstructionSelector::emitIntegerCompare(
  MachineOperand &LHS, MachineOperand &RHS, MachineOperand &Predicate,
  MachineIRBuilder &MIRBuilder) const {
assert(LHS.isReg() && RHS.isReg() && "Expected LHS and RHS to be registers!")(static_cast <bool> (LHS.isReg() && RHS.isReg()
 && "Expected LHS and RHS to be registers!") ? void (
0) : __assert_fail ("LHS.isReg() && RHS.isReg() && \"Expected LHS and RHS to be registers!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4378, __extension__ __PRETTY_FUNCTION__));
assert(Predicate.isPredicate() && "Expected predicate?")(static_cast <bool> (Predicate.isPredicate() &&
 "Expected predicate?") ? void (0) : __assert_fail ("Predicate.isPredicate() && \"Expected predicate?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4379, __extension__ __PRETTY_FUNCTION__));
MachineRegisterInfo &MRI = MIRBuilder.getMF().getRegInfo();
LLT CmpTy = MRI.getType(LHS.getReg());
assert(!CmpTy.isVector() && "Expected scalar or pointer")(static_cast <bool> (!CmpTy.isVector() && "Expected scalar or pointer"
) ? void (0) : __assert_fail ("!CmpTy.isVector() && \"Expected scalar or pointer\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4382, __extension__ __PRETTY_FUNCTION__));
unsigned Size = CmpTy.getSizeInBits();
(void)Size;
assert((Size == 32 || Size == 64) && "Expected a 32-bit or 64-bit LHS/RHS?")(static_cast <bool> ((Size == 32 || Size == 64) &&
 "Expected a 32-bit or 64-bit LHS/RHS?") ? void (0) : __assert_fail
 ("(Size == 32 || Size == 64) && \"Expected a 32-bit or 64-bit LHS/RHS?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4385, __extension__ __PRETTY_FUNCTION__));
// Fold the compare into a cmn or tst if possible.
if (auto FoldCmp = tryFoldIntegerCompare(LHS, RHS, Predicate, MIRBuilder))
  return FoldCmp;
auto Dst = MRI.cloneVirtualRegister(LHS.getReg());
return emitSUBS(Dst, LHS, RHS, MIRBuilder);
4391}

4393MachineInstr *AArch64InstructionSelector::emitCSetForFCmp(
  Register Dst, CmpInst::Predicate Pred, MachineIRBuilder &MIRBuilder) const {
MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
4396#ifndef NDEBUG
LLT Ty = MRI.getType(Dst);
assert(!Ty.isVector() && Ty.getSizeInBits() == 32 &&(static_cast <bool> (!Ty.isVector() && Ty.getSizeInBits
() == 32 && "Expected a 32-bit scalar register?") ? void
 (0) : __assert_fail ("!Ty.isVector() && Ty.getSizeInBits() == 32 && \"Expected a 32-bit scalar register?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4399, __extension__ __PRETTY_FUNCTION__))
       "Expected a 32-bit scalar register?")(static_cast <bool> (!Ty.isVector() && Ty.getSizeInBits
() == 32 && "Expected a 32-bit scalar register?") ? void
 (0) : __assert_fail ("!Ty.isVector() && Ty.getSizeInBits() == 32 && \"Expected a 32-bit scalar register?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4399, __extension__ __PRETTY_FUNCTION__));
4400#endif
const Register ZeroReg = AArch64::WZR;
auto EmitCSet = [&](Register CsetDst, AArch64CC::CondCode CC) {
  auto CSet =
      MIRBuilder.buildInstr(AArch64::CSINCWr, {CsetDst}, {ZeroReg, ZeroReg})
          .addImm(getInvertedCondCode(CC));
  constrainSelectedInstRegOperands(*CSet, TII, TRI, RBI);
  return &*CSet;
};

AArch64CC::CondCode CC1, CC2;
changeFCMPPredToAArch64CC(Pred, CC1, CC2);
if (CC2 == AArch64CC::AL)
  return EmitCSet(Dst, CC1);

const TargetRegisterClass *RC = &AArch64::GPR32RegClass;
Register Def1Reg = MRI.createVirtualRegister(RC);
Register Def2Reg = MRI.createVirtualRegister(RC);
EmitCSet(Def1Reg, CC1);
EmitCSet(Def2Reg, CC2);
auto OrMI = MIRBuilder.buildInstr(AArch64::ORRWrr, {Dst}, {Def1Reg, Def2Reg});
constrainSelectedInstRegOperands(*OrMI, TII, TRI, RBI);
return &*OrMI;
4423}

4425MachineInstr *
4426AArch64InstructionSelector::emitFPCompare(Register LHS, Register RHS,
                                        MachineIRBuilder &MIRBuilder,
                                        Optional<CmpInst::Predicate> Pred) const {
MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
LLT Ty = MRI.getType(LHS);
if (Ty.isVector())
  return nullptr;
unsigned OpSize = Ty.getSizeInBits();
if (OpSize != 32 && OpSize != 64)
  return nullptr;

// If this is a compare against +0.0, then we don't have
// to explicitly materialize a constant.
const ConstantFP *FPImm = getConstantFPVRegVal(RHS, MRI);
bool ShouldUseImm = FPImm && (FPImm->isZero() && !FPImm->isNegative());

auto IsEqualityPred = [](CmpInst::Predicate P) {
  return P == CmpInst::FCMP_OEQ || P == CmpInst::FCMP_ONE ||
         P == CmpInst::FCMP_UEQ || P == CmpInst::FCMP_UNE;
};
if (!ShouldUseImm && Pred && IsEqualityPred(*Pred)) {
  // Try commutating the operands.
  const ConstantFP *LHSImm = getConstantFPVRegVal(LHS, MRI);
  if (LHSImm && (LHSImm->isZero() && !LHSImm->isNegative())) {
    ShouldUseImm = true;
    std::swap(LHS, RHS);
  }
}
unsigned CmpOpcTbl[2][2] = {{AArch64::FCMPSrr, AArch64::FCMPDrr},
                            {AArch64::FCMPSri, AArch64::FCMPDri}};
unsigned CmpOpc = CmpOpcTbl[ShouldUseImm][OpSize == 64];

// Partially build the compare. Decide if we need to add a use for the
// third operand based off whether or not we're comparing against 0.0.
auto CmpMI = MIRBuilder.buildInstr(CmpOpc).addUse(LHS);
if (!ShouldUseImm)
  CmpMI.addUse(RHS);
constrainSelectedInstRegOperands(*CmpMI, TII, TRI, RBI);
return &*CmpMI;
4465}

4467MachineInstr *AArch64InstructionSelector::emitVectorConcat(
  Optional<Register> Dst, Register Op1, Register Op2,
  MachineIRBuilder &MIRBuilder) const {
// We implement a vector concat by:
// 1. Use scalar_to_vector to insert the lower vector into the larger dest
// 2. Insert the upper vector into the destination's upper element
// TODO: some of this code is common with G_BUILD_VECTOR handling.
MachineRegisterInfo &MRI = MIRBuilder.getMF().getRegInfo();

const LLT Op1Ty = MRI.getType(Op1);
const LLT Op2Ty = MRI.getType(Op2);

if (Op1Ty != Op2Ty) {
  LLVM_DEBUG(dbgs() << "Could not do vector concat of differing vector tys")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Could not do vector concat of differing vector tys"
; } } while (false);
  return nullptr;
}
assert(Op1Ty.isVector() && "Expected a vector for vector concat")(static_cast <bool> (Op1Ty.isVector() && "Expected a vector for vector concat"
) ? void (0) : __assert_fail ("Op1Ty.isVector() && \"Expected a vector for vector concat\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4483, __extension__ __PRETTY_FUNCTION__));

if (Op1Ty.getSizeInBits() >= 128) {
  LLVM_DEBUG(dbgs() << "Vector concat not supported for full size vectors")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Vector concat not supported for full size vectors"
; } } while (false);
  return nullptr;
}

// At the moment we just support 64 bit vector concats.
if (Op1Ty.getSizeInBits() != 64) {
  LLVM_DEBUG(dbgs() << "Vector concat supported for 64b vectors")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Vector concat supported for 64b vectors"
; } } while (false);
  return nullptr;
}

const LLT ScalarTy = LLT::scalar(Op1Ty.getSizeInBits());
const RegisterBank &FPRBank = *RBI.getRegBank(Op1, MRI, TRI);
const TargetRegisterClass *DstRC =
    getMinClassForRegBank(FPRBank, Op1Ty.getSizeInBits() * 2);

MachineInstr *WidenedOp1 =
    emitScalarToVector(ScalarTy.getSizeInBits(), DstRC, Op1, MIRBuilder);
MachineInstr *WidenedOp2 =
    emitScalarToVector(ScalarTy.getSizeInBits(), DstRC, Op2, MIRBuilder);
if (!WidenedOp1 || !WidenedOp2) {
  LLVM_DEBUG(dbgs() << "Could not emit a vector from scalar value")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Could not emit a vector from scalar value"
; } } while (false);
  return nullptr;
}

// Now do the insert of the upper element.
unsigned InsertOpc, InsSubRegIdx;
std::tie(InsertOpc, InsSubRegIdx) =
    getInsertVecEltOpInfo(FPRBank, ScalarTy.getSizeInBits());

if (!Dst)
  Dst = MRI.createVirtualRegister(DstRC);
auto InsElt =
    MIRBuilder
        .buildInstr(InsertOpc, {*Dst}, {WidenedOp1->getOperand(0).getReg()})
        .addImm(1) /* Lane index */
        .addUse(WidenedOp2->getOperand(0).getReg())
        .addImm(0);
constrainSelectedInstRegOperands(*InsElt, TII, TRI, RBI);
return &*InsElt;
4525}

4527MachineInstr *
4528AArch64InstructionSelector::emitCSetForICMP(Register DefReg, unsigned Pred,
                                          MachineIRBuilder &MIRBuilder,
                                          Register SrcReg) const {
// CSINC increments the result when the predicate is false. Invert it.
const AArch64CC::CondCode InvCC = changeICMPPredToAArch64CC(
    CmpInst::getInversePredicate((CmpInst::Predicate)Pred));
auto I = MIRBuilder.buildInstr(AArch64::CSINCWr, {DefReg}, {SrcReg, SrcReg})
             .addImm(InvCC);
constrainSelectedInstRegOperands(*I, TII, TRI, RBI);
return &*I;
4538}

4540std::pair<MachineInstr *, AArch64CC::CondCode>
4541AArch64InstructionSelector::emitOverflowOp(unsigned Opcode, Register Dst,
                                         MachineOperand &LHS,
                                         MachineOperand &RHS,
                                         MachineIRBuilder &MIRBuilder) const {
switch (Opcode) {
default:
  llvm_unreachable("Unexpected opcode!")::llvm::llvm_unreachable_internal("Unexpected opcode!", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4547);
case TargetOpcode::G_SADDO:
  return std::make_pair(emitADDS(Dst, LHS, RHS, MIRBuilder), AArch64CC::VS);
case TargetOpcode::G_UADDO:
  return std::make_pair(emitADDS(Dst, LHS, RHS, MIRBuilder), AArch64CC::HS);
case TargetOpcode::G_SSUBO:
  return std::make_pair(emitSUBS(Dst, LHS, RHS, MIRBuilder), AArch64CC::VS);
case TargetOpcode::G_USUBO:
  return std::make_pair(emitSUBS(Dst, LHS, RHS, MIRBuilder), AArch64CC::LO);
}
4557}

4559bool AArch64InstructionSelector::tryOptSelect(MachineInstr &I) {
MachineRegisterInfo &MRI = *MIB.getMRI();
// We want to recognize this pattern:
//
// $z = G_FCMP pred, $x, $y
// ...
// $w = G_SELECT $z, $a, $b
//
// Where the value of $z is *only* ever used by the G_SELECT (possibly with
// some copies/truncs in between.)
//
// If we see this, then we can emit something like this:
//
// fcmp $x, $y
// fcsel $w, $a, $b, pred
//
// Rather than emitting both of the rather long sequences in the standard
// G_FCMP/G_SELECT select methods.

// First, check if the condition is defined by a compare.
MachineInstr *CondDef = MRI.getVRegDef(I.getOperand(1).getReg());
while (CondDef) {
  // We can only fold if all of the defs have one use.
  Register CondDefReg = CondDef->getOperand(0).getReg();
  if (!MRI.hasOneNonDBGUse(CondDefReg)) {
    // Unless it's another select.
    for (const MachineInstr &UI : MRI.use_nodbg_instructions(CondDefReg)) {
      if (CondDef == &UI)
        continue;
      if (UI.getOpcode() != TargetOpcode::G_SELECT)
        return false;
    }
  }

  // We can skip over G_TRUNC since the condition is 1-bit.
  // Truncating/extending can have no impact on the value.
  unsigned Opc = CondDef->getOpcode();
  if (Opc != TargetOpcode::COPY && Opc != TargetOpcode::G_TRUNC)
    break;

  // Can't see past copies from physregs.
  if (Opc == TargetOpcode::COPY &&
      Register::isPhysicalRegister(CondDef->getOperand(1).getReg()))
    return false;

  CondDef = MRI.getVRegDef(CondDef->getOperand(1).getReg());
}

// Is the condition defined by a compare?
if (!CondDef)
  return false;

unsigned CondOpc = CondDef->getOpcode();
if (CondOpc != TargetOpcode::G_ICMP && CondOpc != TargetOpcode::G_FCMP)
  return false;

AArch64CC::CondCode CondCode;
if (CondOpc == TargetOpcode::G_ICMP) {
  auto Pred =
      static_cast<CmpInst::Predicate>(CondDef->getOperand(1).getPredicate());
  CondCode = changeICMPPredToAArch64CC(Pred);
  emitIntegerCompare(CondDef->getOperand(2), CondDef->getOperand(3),
                     CondDef->getOperand(1), MIB);
} else {
  // Get the condition code for the select.
  auto Pred =
      static_cast<CmpInst::Predicate>(CondDef->getOperand(1).getPredicate());
  AArch64CC::CondCode CondCode2;
  changeFCMPPredToAArch64CC(Pred, CondCode, CondCode2);

  // changeFCMPPredToAArch64CC sets CondCode2 to AL when we require two
  // instructions to emit the comparison.
  // TODO: Handle FCMP_UEQ and FCMP_ONE. After that, this check will be
  // unnecessary.
  if (CondCode2 != AArch64CC::AL)
    return false;

  if (!emitFPCompare(CondDef->getOperand(2).getReg(),
                     CondDef->getOperand(3).getReg(), MIB)) {
    LLVM_DEBUG(dbgs() << "Couldn't emit compare for select!\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Couldn't emit compare for select!\n"
; } } while (false);
    return false;
  }
}

// Emit the select.
emitSelect(I.getOperand(0).getReg(), I.getOperand(2).getReg(),
           I.getOperand(3).getReg(), CondCode, MIB);
I.eraseFromParent();
return true;
4648}

4650MachineInstr *AArch64InstructionSelector::tryFoldIntegerCompare(
  MachineOperand &LHS, MachineOperand &RHS, MachineOperand &Predicate,
  MachineIRBuilder &MIRBuilder) const {
assert(LHS.isReg() && RHS.isReg() && Predicate.isPredicate() &&(static_cast <bool> (LHS.isReg() && RHS.isReg()
 && Predicate.isPredicate() && "Unexpected MachineOperand"
) ? void (0) : __assert_fail ("LHS.isReg() && RHS.isReg() && Predicate.isPredicate() && \"Unexpected MachineOperand\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4654, __extension__ __PRETTY_FUNCTION__))
       "Unexpected MachineOperand")(static_cast <bool> (LHS.isReg() && RHS.isReg()
 && Predicate.isPredicate() && "Unexpected MachineOperand"
) ? void (0) : __assert_fail ("LHS.isReg() && RHS.isReg() && Predicate.isPredicate() && \"Unexpected MachineOperand\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4654, __extension__ __PRETTY_FUNCTION__));
MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
// We want to find this sort of thing:
// x = G_SUB 0, y
// G_ICMP z, x
//
// In this case, we can fold the G_SUB into the G_ICMP using a CMN instead.
// e.g:
//
// cmn z, y

// Check if the RHS or LHS of the G_ICMP is defined by a SUB
MachineInstr *LHSDef = getDefIgnoringCopies(LHS.getReg(), MRI);
MachineInstr *RHSDef = getDefIgnoringCopies(RHS.getReg(), MRI);
auto P = static_cast<CmpInst::Predicate>(Predicate.getPredicate());
// Given this:
//
// x = G_SUB 0, y
// G_ICMP x, z
//
// Produce this:
//
// cmn y, z
if (isCMN(LHSDef, P, MRI))
  return emitCMN(LHSDef->getOperand(2), RHS, MIRBuilder);

// Same idea here, but with the RHS of the compare instead:
//
// Given this:
//
// x = G_SUB 0, y
// G_ICMP z, x
//
// Produce this:
//
// cmn z, y
if (isCMN(RHSDef, P, MRI))
  return emitCMN(LHS, RHSDef->getOperand(2), MIRBuilder);

// Given this:
//
// z = G_AND x, y
// G_ICMP z, 0
//
// Produce this if the compare is signed:
//
// tst x, y
if (!CmpInst::isUnsigned(P) && LHSDef &&
    LHSDef->getOpcode() == TargetOpcode::G_AND) {
  // Make sure that the RHS is 0.
  auto ValAndVReg = getConstantVRegValWithLookThrough(RHS.getReg(), MRI);
  if (!ValAndVReg || ValAndVReg->Value != 0)
    return nullptr;

  return emitTST(LHSDef->getOperand(1),
                 LHSDef->getOperand(2), MIRBuilder);
}

return nullptr;
4713}

4715bool AArch64InstructionSelector::selectShuffleVector(
  MachineInstr &I, MachineRegisterInfo &MRI) {
const LLT DstTy = MRI.getType(I.getOperand(0).getReg());
Register Src1Reg = I.getOperand(1).getReg();
const LLT Src1Ty = MRI.getType(Src1Reg);
Register Src2Reg = I.getOperand(2).getReg();
const LLT Src2Ty = MRI.getType(Src2Reg);
ArrayRef<int> Mask = I.getOperand(3).getShuffleMask();

MachineBasicBlock &MBB = *I.getParent();
MachineFunction &MF = *MBB.getParent();
LLVMContext &Ctx = MF.getFunction().getContext();

// G_SHUFFLE_VECTOR is weird in that the source operands can be scalars, if
// it's originated from a <1 x T> type. Those should have been lowered into
// G_BUILD_VECTOR earlier.
if (!Src1Ty.isVector() || !Src2Ty.isVector()) {
  LLVM_DEBUG(dbgs() << "Could not select a \"scalar\" G_SHUFFLE_VECTOR\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Could not select a \"scalar\" G_SHUFFLE_VECTOR\n"
; } } while (false);
  return false;
}

unsigned BytesPerElt = DstTy.getElementType().getSizeInBits() / 8;

SmallVector<Constant *, 64> CstIdxs;
for (int Val : Mask) {
  // For now, any undef indexes we'll just assume to be 0. This should be
  // optimized in future, e.g. to select DUP etc.
  Val = Val < 0 ? 0 : Val;
  for (unsigned Byte = 0; Byte < BytesPerElt; ++Byte) {
    unsigned Offset = Byte + Val * BytesPerElt;
    CstIdxs.emplace_back(ConstantInt::get(Type::getInt8Ty(Ctx), Offset));
  }
}

// Use a constant pool to load the index vector for TBL.
Constant *CPVal = ConstantVector::get(CstIdxs);
MachineInstr *IndexLoad = emitLoadFromConstantPool(CPVal, MIB);
if (!IndexLoad) {
  LLVM_DEBUG(dbgs() << "Could not load from a constant pool")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Could not load from a constant pool"
; } } while (false);
  return false;
}

if (DstTy.getSizeInBits() != 128) {
  assert(DstTy.getSizeInBits() == 64 && "Unexpected shuffle result ty")(static_cast <bool> (DstTy.getSizeInBits() == 64 &&
 "Unexpected shuffle result ty") ? void (0) : __assert_fail (
"DstTy.getSizeInBits() == 64 && \"Unexpected shuffle result ty\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4758, __extension__ __PRETTY_FUNCTION__));
  // This case can be done with TBL1.
  MachineInstr *Concat = emitVectorConcat(None, Src1Reg, Src2Reg, MIB);
  if (!Concat) {
    LLVM_DEBUG(dbgs() << "Could not do vector concat for tbl1")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Could not do vector concat for tbl1"
; } } while (false);
    return false;
  }

  // The constant pool load will be 64 bits, so need to convert to FPR128 reg.
  IndexLoad = emitScalarToVector(64, &AArch64::FPR128RegClass,
                                 IndexLoad->getOperand(0).getReg(), MIB);

  auto TBL1 = MIB.buildInstr(
      AArch64::TBLv16i8One, {&AArch64::FPR128RegClass},
      {Concat->getOperand(0).getReg(), IndexLoad->getOperand(0).getReg()});
  constrainSelectedInstRegOperands(*TBL1, TII, TRI, RBI);

  auto Copy =
      MIB.buildInstr(TargetOpcode::COPY, {I.getOperand(0).getReg()}, {})
          .addReg(TBL1.getReg(0), 0, AArch64::dsub);
  RBI.constrainGenericRegister(Copy.getReg(0), AArch64::FPR64RegClass, MRI);
  I.eraseFromParent();
  return true;
}

// For TBL2 we need to emit a REG_SEQUENCE to tie together two consecutive
// Q registers for regalloc.
SmallVector<Register, 2> Regs = {Src1Reg, Src2Reg};
auto RegSeq = createQTuple(Regs, MIB);
auto TBL2 = MIB.buildInstr(AArch64::TBLv16i8Two, {I.getOperand(0)},
                           {RegSeq, IndexLoad->getOperand(0)});
constrainSelectedInstRegOperands(*TBL2, TII, TRI, RBI);
I.eraseFromParent();
return true;
4792}

4794MachineInstr *AArch64InstructionSelector::emitLaneInsert(
  Optional<Register> DstReg, Register SrcReg, Register EltReg,
  unsigned LaneIdx, const RegisterBank &RB,
  MachineIRBuilder &MIRBuilder) const {
MachineInstr *InsElt = nullptr;
const TargetRegisterClass *DstRC = &AArch64::FPR128RegClass;
MachineRegisterInfo &MRI = *MIRBuilder.getMRI();

// Create a register to define with the insert if one wasn't passed in.
if (!DstReg)
  DstReg = MRI.createVirtualRegister(DstRC);

unsigned EltSize = MRI.getType(EltReg).getSizeInBits();
unsigned Opc = getInsertVecEltOpInfo(RB, EltSize).first;

if (RB.getID() == AArch64::FPRRegBankID) {
  auto InsSub = emitScalarToVector(EltSize, DstRC, EltReg, MIRBuilder);
  InsElt = MIRBuilder.buildInstr(Opc, {*DstReg}, {SrcReg})
               .addImm(LaneIdx)
               .addUse(InsSub->getOperand(0).getReg())
               .addImm(0);
} else {
  InsElt = MIRBuilder.buildInstr(Opc, {*DstReg}, {SrcReg})
               .addImm(LaneIdx)
               .addUse(EltReg);
}

constrainSelectedInstRegOperands(*InsElt, TII, TRI, RBI);
return InsElt;
4823}

4825bool AArch64InstructionSelector::selectInsertElt(MachineInstr &I,
                                               MachineRegisterInfo &MRI) {
assert(I.getOpcode() == TargetOpcode::G_INSERT_VECTOR_ELT)(static_cast <bool> (I.getOpcode() == TargetOpcode::G_INSERT_VECTOR_ELT
) ? void (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_INSERT_VECTOR_ELT"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4827, __extension__ __PRETTY_FUNCTION__));

// Get information on the destination.
Register DstReg = I.getOperand(0).getReg();
const LLT DstTy = MRI.getType(DstReg);
unsigned VecSize = DstTy.getSizeInBits();

// Get information on the element we want to insert into the destination.
Register EltReg = I.getOperand(2).getReg();
const LLT EltTy = MRI.getType(EltReg);
unsigned EltSize = EltTy.getSizeInBits();
if (EltSize < 16 || EltSize > 64)
  return false; // Don't support all element types yet.

// Find the definition of the index. Bail out if it's not defined by a
// G_CONSTANT.
Register IdxReg = I.getOperand(3).getReg();
auto VRegAndVal = getConstantVRegValWithLookThrough(IdxReg, MRI);
if (!VRegAndVal)
  return false;
unsigned LaneIdx = VRegAndVal->Value.getSExtValue();

// Perform the lane insert.
Register SrcReg = I.getOperand(1).getReg();
const RegisterBank &EltRB = *RBI.getRegBank(EltReg, MRI, TRI);

if (VecSize < 128) {
  // If the vector we're inserting into is smaller than 128 bits, widen it
  // to 128 to do the insert.
  MachineInstr *ScalarToVec =
      emitScalarToVector(VecSize, &AArch64::FPR128RegClass, SrcReg, MIB);
  if (!ScalarToVec)
    return false;
  SrcReg = ScalarToVec->getOperand(0).getReg();
}

// Create an insert into a new FPR128 register.
// Note that if our vector is already 128 bits, we end up emitting an extra
// register.
MachineInstr *InsMI =
    emitLaneInsert(None, SrcReg, EltReg, LaneIdx, EltRB, MIB);

if (VecSize < 128) {
  // If we had to widen to perform the insert, then we have to demote back to
  // the original size to get the result we want.
  Register DemoteVec = InsMI->getOperand(0).getReg();
  const TargetRegisterClass *RC =
      getMinClassForRegBank(*RBI.getRegBank(DemoteVec, MRI, TRI), VecSize);
  if (RC != &AArch64::FPR32RegClass && RC != &AArch64::FPR64RegClass) {
    LLVM_DEBUG(dbgs() << "Unsupported register class!\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unsupported register class!\n"
; } } while (false);
    return false;
  }
  unsigned SubReg = 0;
  if (!getSubRegForClass(RC, TRI, SubReg))
    return false;
  if (SubReg != AArch64::ssub && SubReg != AArch64::dsub) {
    LLVM_DEBUG(dbgs() << "Unsupported destination size! (" << VecSizedo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unsupported destination size! ("
 << VecSize << "\n"; } } while (false)
                      << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unsupported destination size! ("
 << VecSize << "\n"; } } while (false);
    return false;
  }
  MIB.buildInstr(TargetOpcode::COPY, {DstReg}, {})
      .addReg(DemoteVec, 0, SubReg);
  RBI.constrainGenericRegister(DstReg, *RC, MRI);
} else {
  // No widening needed.
  InsMI->getOperand(0).setReg(DstReg);
  constrainSelectedInstRegOperands(*InsMI, TII, TRI, RBI);
}

I.eraseFromParent();
return true;
4898}

4900MachineInstr *
4901AArch64InstructionSelector::emitConstantVector(Register Dst, Constant *CV,
                                             MachineIRBuilder &MIRBuilder,
                                             MachineRegisterInfo &MRI) {
LLT DstTy = MRI.getType(Dst);
unsigned DstSize = DstTy.getSizeInBits();
if (CV->isNullValue()) {
  if (DstSize == 128) {
    auto Mov =
        MIRBuilder.buildInstr(AArch64::MOVIv2d_ns, {Dst}, {}).addImm(0);
    constrainSelectedInstRegOperands(*Mov, TII, TRI, RBI);
    return &*Mov;
  }

  if (DstSize == 64) {
    auto Mov =
        MIRBuilder
            .buildInstr(AArch64::MOVIv2d_ns, {&AArch64::FPR128RegClass}, {})
            .addImm(0);
    auto Copy = MIRBuilder.buildInstr(TargetOpcode::COPY, {Dst}, {})
                    .addReg(Mov.getReg(0), 0, AArch64::dsub);
    RBI.constrainGenericRegister(Dst, AArch64::FPR64RegClass, MRI);
    return &*Copy;
  }
}

auto *CPLoad = emitLoadFromConstantPool(CV, MIRBuilder);
if (!CPLoad) {
  LLVM_DEBUG(dbgs() << "Could not generate cp load for constant vector!")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Could not generate cp load for constant vector!"
; } } while (false);
  return nullptr;
}

auto Copy = MIRBuilder.buildCopy(Dst, CPLoad->getOperand(0));
RBI.constrainGenericRegister(
    Dst, *MRI.getRegClass(CPLoad->getOperand(0).getReg()), MRI);
return &*Copy;
4936}

4938bool AArch64InstructionSelector::tryOptConstantBuildVec(
  MachineInstr &I, LLT DstTy, MachineRegisterInfo &MRI) {
assert(I.getOpcode() == TargetOpcode::G_BUILD_VECTOR)(static_cast <bool> (I.getOpcode() == TargetOpcode::G_BUILD_VECTOR
) ? void (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_BUILD_VECTOR"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4940, __extension__ __PRETTY_FUNCTION__));
unsigned DstSize = DstTy.getSizeInBits();
assert(DstSize <= 128 && "Unexpected build_vec type!")(static_cast <bool> (DstSize <= 128 && "Unexpected build_vec type!"
) ? void (0) : __assert_fail ("DstSize <= 128 && \"Unexpected build_vec type!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4942, __extension__ __PRETTY_FUNCTION__));
if (DstSize < 32)
  return false;
// Check if we're building a constant vector, in which case we want to
// generate a constant pool load instead of a vector insert sequence.
SmallVector<Constant *, 16> Csts;
for (unsigned Idx = 1; Idx < I.getNumOperands(); ++Idx) {
  // Try to find G_CONSTANT or G_FCONSTANT
  auto *OpMI =
      getOpcodeDef(TargetOpcode::G_CONSTANT, I.getOperand(Idx).getReg(), MRI);
  if (OpMI)
    Csts.emplace_back(
        const_cast<ConstantInt *>(OpMI->getOperand(1).getCImm()));
  else if ((OpMI = getOpcodeDef(TargetOpcode::G_FCONSTANT,
                                I.getOperand(Idx).getReg(), MRI)))
    Csts.emplace_back(
        const_cast<ConstantFP *>(OpMI->getOperand(1).getFPImm()));
  else
    return false;
}
Constant *CV = ConstantVector::get(Csts);
if (!emitConstantVector(I.getOperand(0).getReg(), CV, MIB, MRI))
  return false;
I.eraseFromParent();
return true;
4967}

4969bool AArch64InstructionSelector::tryOptBuildVecToSubregToReg(
  MachineInstr &I, MachineRegisterInfo &MRI) {
// Given:
//  %vec = G_BUILD_VECTOR %elt, %undef, %undef, ... %undef
//
// Select the G_BUILD_VECTOR as a SUBREG_TO_REG from %elt.
Register Dst = I.getOperand(0).getReg();
Register EltReg = I.getOperand(1).getReg();
LLT EltTy = MRI.getType(EltReg);
// If the index isn't on the same bank as its elements, then this can't be a
// SUBREG_TO_REG.
const RegisterBank &EltRB = *RBI.getRegBank(EltReg, MRI, TRI);
const RegisterBank &DstRB = *RBI.getRegBank(Dst, MRI, TRI);
if (EltRB != DstRB)
  return false;
if (any_of(make_range(I.operands_begin() + 2, I.operands_end()),
           [&MRI](const MachineOperand &Op) {
             return !getOpcodeDef(TargetOpcode::G_IMPLICIT_DEF, Op.getReg(),
                                  MRI);
           }))
  return false;
unsigned SubReg;
const TargetRegisterClass *EltRC =
    getMinClassForRegBank(EltRB, EltTy.getSizeInBits());
if (!EltRC)
  return false;
const TargetRegisterClass *DstRC =
    getMinClassForRegBank(DstRB, MRI.getType(Dst).getSizeInBits());
if (!DstRC)
  return false;
if (!getSubRegForClass(EltRC, TRI, SubReg))
  return false;
auto SubregToReg = MIB.buildInstr(AArch64::SUBREG_TO_REG, {Dst}, {})
                       .addImm(0)
                       .addUse(EltReg)
                       .addImm(SubReg);
I.eraseFromParent();
constrainSelectedInstRegOperands(*SubregToReg, TII, TRI, RBI);
return RBI.constrainGenericRegister(Dst, *DstRC, MRI);
5008}

5010bool AArch64InstructionSelector::selectBuildVector(MachineInstr &I,
                                                 MachineRegisterInfo &MRI) {
assert(I.getOpcode() == TargetOpcode::G_BUILD_VECTOR)(static_cast <bool> (I.getOpcode() == TargetOpcode::G_BUILD_VECTOR
) ? void (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_BUILD_VECTOR"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 5012, __extension__ __PRETTY_FUNCTION__));
// Until we port more of the optimized selections, for now just use a vector
// insert sequence.
const LLT DstTy = MRI.getType(I.getOperand(0).getReg());
const LLT EltTy = MRI.getType(I.getOperand(1).getReg());
unsigned EltSize = EltTy.getSizeInBits();

if (tryOptConstantBuildVec(I, DstTy, MRI))
  return true;
if (tryOptBuildVecToSubregToReg(I, MRI))
  return true;

if (EltSize < 16 || EltSize > 64)
  return false; // Don't support all element types yet.
const RegisterBank &RB = *RBI.getRegBank(I.getOperand(1).getReg(), MRI, TRI);

const TargetRegisterClass *DstRC = &AArch64::FPR128RegClass;
MachineInstr *ScalarToVec =
    emitScalarToVector(DstTy.getElementType().getSizeInBits(), DstRC,
                       I.getOperand(1).getReg(), MIB);
if (!ScalarToVec)
  return false;

Register DstVec = ScalarToVec->getOperand(0).getReg();
unsigned DstSize = DstTy.getSizeInBits();

// Keep track of the last MI we inserted. Later on, we might be able to save
// a copy using it.
MachineInstr *PrevMI = nullptr;
for (unsigned i = 2, e = DstSize / EltSize + 1; i < e; ++i) {
  // Note that if we don't do a subregister copy, we can end up making an
  // extra register.
  PrevMI = &*emitLaneInsert(None, DstVec, I.getOperand(i).getReg(), i - 1, RB,
                            MIB);
  DstVec = PrevMI->getOperand(0).getReg();
}

// If DstTy's size in bits is less than 128, then emit a subregister copy
// from DstVec to the last register we've defined.
if (DstSize < 128) {
  // Force this to be FPR using the destination vector.
  const TargetRegisterClass *RC =
      getMinClassForRegBank(*RBI.getRegBank(DstVec, MRI, TRI), DstSize);
  if (!RC)
    return false;
  if (RC != &AArch64::FPR32RegClass && RC != &AArch64::FPR64RegClass) {
    LLVM_DEBUG(dbgs() << "Unsupported register class!\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unsupported register class!\n"
; } } while (false);
    return false;
  }

  unsigned SubReg = 0;
  if (!getSubRegForClass(RC, TRI, SubReg))
    return false;
  if (SubReg != AArch64::ssub && SubReg != AArch64::dsub) {
    LLVM_DEBUG(dbgs() << "Unsupported destination size! (" << DstSizedo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unsupported destination size! ("
 << DstSize << "\n"; } } while (false)
                      << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unsupported destination size! ("
 << DstSize << "\n"; } } while (false);
    return false;
  }

  Register Reg = MRI.createVirtualRegister(RC);
  Register DstReg = I.getOperand(0).getReg();

  MIB.buildInstr(TargetOpcode::COPY, {DstReg}, {}).addReg(DstVec, 0, SubReg);
  MachineOperand &RegOp = I.getOperand(1);
  RegOp.setReg(Reg);
  RBI.constrainGenericRegister(DstReg, *RC, MRI);
} else {
  // We don't need a subregister copy. Save a copy by re-using the
  // destination register on the final insert.
  assert(PrevMI && "PrevMI was null?")(static_cast <bool> (PrevMI && "PrevMI was null?"
) ? void (0) : __assert_fail ("PrevMI && \"PrevMI was null?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 5081, __extension__ __PRETTY_FUNCTION__));
  PrevMI->getOperand(0).setReg(I.getOperand(0).getReg());
  constrainSelectedInstRegOperands(*PrevMI, TII, TRI, RBI);
}

I.eraseFromParent();
return true;
5088}

5090/// Helper function to find an intrinsic ID on an a MachineInstr. Returns the
5091/// ID if it exists, and 0 otherwise.
5092static unsigned findIntrinsicID(MachineInstr &I) {
auto IntrinOp = find_if(I.operands(), [&](const MachineOperand &Op) {
  return Op.isIntrinsicID();
});
if (IntrinOp == I.operands_end())
  return 0;
return IntrinOp->getIntrinsicID();
5099}

5101bool AArch64InstructionSelector::selectVectorLoadIntrinsic(unsigned Opc,
                                                         unsigned NumVecs,
                                                         MachineInstr &I) {
assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS)(static_cast <bool> (I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS
) ? void (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS"
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 5104, __extension__ __PRETTY_FUNCTION__));
assert(Opc && "Expected an opcode?")(static_cast <bool> (Opc && "Expected an opcode?"
) ? void (0) : __assert_fail ("Opc && \"Expected an opcode?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 5105, __extension__ __PRETTY_FUNCTION__));
assert(NumVecs > 1 && NumVecs < 5 && "Only support 2, 3, or 4 vectors")(static_cast <bool> (NumVecs > 1 && NumVecs <
&& "Only support 2, 3, or 4 vectors") ? void (0) :
 __assert_fail ("NumVecs > 1 && NumVecs < 5 && \"Only support 2, 3, or 4 vectors\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 5106, __extension__ __PRETTY_FUNCTION__));
auto &MRI = *MIB.getMRI();
LLT Ty = MRI.getType(I.getOperand(0).getReg());
unsigned Size = Ty.getSizeInBits();
assert((Size == 64 || Size == 128) &&(static_cast <bool> ((Size == 64 || Size == 128) &&
 "Destination must be 64 bits or 128 bits?") ? void (0) : __assert_fail
 ("(Size == 64 || Size == 128) && \"Destination must be 64 bits or 128 bits?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 5111, __extension__ __PRETTY_FUNCTION__))
       "Destination must be 64 bits or 128 bits?")(static_cast <bool> ((Size == 64 || Size == 128) &&
 "Destination must be 64 bits or 128 bits?") ? void (0) : __assert_fail
 ("(Size == 64 || Size == 128) && \"Destination must be 64 bits or 128 bits?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 5111, __extension__ __PRETTY_FUNCTION__));
unsigned SubReg = Size == 64 ? AArch64::dsub0 : AArch64::qsub0;
auto Ptr = I.getOperand(I.getNumOperands() - 1).getReg();
assert(MRI.getType(Ptr).isPointer() && "Expected a pointer type?")(static_cast <bool> (MRI.getType(Ptr).isPointer() &&
 "Expected a pointer type?") ? void (0) : __assert_fail ("MRI.getType(Ptr).isPointer() && \"Expected a pointer type?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 5114, __extension__ __PRETTY_FUNCTION__));
auto Load = MIB.buildInstr(Opc, {Ty}, {Ptr});
Load.cloneMemRefs(I);
constrainSelectedInstRegOperands(*Load, TII, TRI, RBI);
Register SelectedLoadDst = Load->getOperand(0).getReg();
for (unsigned Idx = 0; Idx < NumVecs; ++Idx) {
  auto Vec = MIB.buildInstr(TargetOpcode::COPY, {I.getOperand(Idx)}, {})
                 .addReg(SelectedLoadDst, 0, SubReg + Idx);
  // Emit the subreg copies and immediately select them.
  // FIXME: We should refactor our copy code into an emitCopy helper and
  // clean up uses of this pattern elsewhere in the selector.
  selectCopy(*Vec, TII, MRI, TRI, RBI);
}
return true;
5128}

5130bool AArch64InstructionSelector::selectIntrinsicWithSideEffects(
  MachineInstr &I, MachineRegisterInfo &MRI) {
// Find the intrinsic ID.
unsigned IntrinID = findIntrinsicID(I);
if (!IntrinID)
  return false;

const LLT S8 = LLT::scalar(8);
const LLT S16 = LLT::scalar(16);
const LLT S32 = LLT::scalar(32);
const LLT S64 = LLT::scalar(64);
const LLT P0 = LLT::pointer(0, 64);
// Select the instruction.
switch (IntrinID) {
default:
  return false;
case Intrinsic::aarch64_ldxp:
case Intrinsic::aarch64_ldaxp: {
  auto NewI = MIB.buildInstr(
      IntrinID == Intrinsic::aarch64_ldxp ? AArch64::LDXPX : AArch64::LDAXPX,
      {I.getOperand(0).getReg(), I.getOperand(1).getReg()},
      {I.getOperand(3)});
  NewI.cloneMemRefs(I);
  constrainSelectedInstRegOperands(*NewI, TII, TRI, RBI);
  break;
}
case Intrinsic::trap:
  MIB.buildInstr(AArch64::BRK, {}, {}).addImm(1);
  break;
case Intrinsic::debugtrap:
  MIB.buildInstr(AArch64::BRK, {}, {}).addImm(0xF000);
  break;
case Intrinsic::ubsantrap:
  MIB.buildInstr(AArch64::BRK, {}, {})
      .addImm(I.getOperand(1).getImm() | ('U' << 8));
  break;
case Intrinsic::aarch64_neon_ld2: {
  LLT Ty = MRI.getType(I.getOperand(0).getReg());
  unsigned Opc = 0;
  if (Ty == LLT::fixed_vector(8, S8))
    Opc = AArch64::LD2Twov8b;
  else if (Ty == LLT::fixed_vector(16, S8))
    Opc = AArch64::LD2Twov16b;
  else if (Ty == LLT::fixed_vector(4, S16))
    Opc = AArch64::LD2Twov4h;
  else if (Ty == LLT::fixed_vector(8, S16))
    Opc = AArch64::LD2Twov8h;
  else if (Ty == LLT::fixed_vector(2, S32))
    Opc = AArch64::LD2Twov2s;
  else if (Ty == LLT::fixed_vector(4, S32))
    Opc = AArch64::LD2Twov4s;
  else if (Ty == LLT::fixed_vector(2, S64) || Ty == LLT::fixed_vector(2, P0))
    Opc = AArch64::LD2Twov2d;
  else if (Ty == S64 || Ty == P0)
    Opc = AArch64::LD1Twov1d;
  else
    llvm_unreachable("Unexpected type for ld2!")::llvm::llvm_unreachable_internal("Unexpected type for ld2!",
 "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 5186);
  selectVectorLoadIntrinsic(Opc, 2, I);
  break;
}
case Intrinsic::aarch64_neon_ld4: {
  LLT Ty = MRI.getType(I.getOperand(0).getReg());
  unsigned Opc = 0;
  if (Ty == LLT::fixed_vector(8, S8))
    Opc = AArch64::LD4Fourv8b;
  else if (Ty == LLT::fixed_vector(16, S8))
    Opc = AArch64::LD4Fourv16b;
  else if (Ty == LLT::fixed_vector(4, S16))
    Opc = AArch64::LD4Fourv4h;
  else if (Ty == LLT::fixed_vector(8, S16))
    Opc = AArch64::LD4Fourv8h;
  else if (Ty == LLT::fixed_vector(2, S32))
    Opc = AArch64::LD4Fourv2s;
  else if (Ty == LLT::fixed_vector(4, S32))
    Opc = AArch64::LD4Fourv4s;
  else if (Ty == LLT::fixed_vector(2, S64) || Ty == LLT::fixed_vector(2, P0))
    Opc = AArch64::LD4Fourv2d;
  else if (Ty == S64 || Ty == P0)
    Opc = AArch64::LD1Fourv1d;
  else
    llvm_unreachable("Unexpected type for ld4!")::llvm::llvm_unreachable_internal("Unexpected type for ld4!",
 "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 5210);
  selectVectorLoadIntrinsic(Opc, 4, I);
  break;
}
case Intrinsic::aarch64_neon_st2: {
  Register Src1 = I.getOperand(1).getReg();
  Register Src2 = I.getOperand(2).getReg();
  Register Ptr = I.getOperand(3).getReg();
  LLT Ty = MRI.getType(Src1);
  unsigned Opc;
  if (Ty == LLT::fixed_vector(8, S8))
    Opc = AArch64::ST2Twov8b;
  else if (Ty == LLT::fixed_vector(16, S8))
    Opc = AArch64::ST2Twov16b;
  else if (Ty == LLT::fixed_vector(4, S16))
    Opc = AArch64::ST2Twov4h;
  else if (Ty == LLT::fixed_vector(8, S16))
    Opc = AArch64::ST2Twov8h;
  else if (Ty == LLT::fixed_vector(2, S32))
    Opc = AArch64::ST2Twov2s;
  else if (Ty == LLT::fixed_vector(4, S32))
    Opc = AArch64::ST2Twov4s;
  else if (Ty == LLT::fixed_vector(2, S64) || Ty == LLT::fixed_vector(2, P0))
    Opc = AArch64::ST2Twov2d;
  else if (Ty == S64 || Ty == P0)
    Opc = AArch64::ST1Twov1d;
  else
    llvm_unreachable("Unexpected type for st2!")::llvm::llvm_unreachable_internal("Unexpected type for st2!",
 "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 5237);
  SmallVector<Register, 2> Regs = {Src1, Src2};
  Register Tuple = Ty.getSizeInBits() == 128 ? createQTuple(Regs, MIB)
                                             : createDTuple(Regs, MIB);
  auto Store = MIB.buildInstr(Opc, {}, {Tuple, Ptr});
  Store.cloneMemRefs(I);
  constrainSelectedInstRegOperands(*Store, TII, TRI, RBI);
  break;
}
}

I.eraseFromParent();
return true;
5250}

5252bool AArch64InstructionSelector::selectIntrinsic(MachineInstr &I,
                                               MachineRegisterInfo &MRI) {
unsigned IntrinID = findIntrinsicID(I);
if (!IntrinID)
  return false;

switch (IntrinID) {
default:
  break;
case Intrinsic::aarch64_crypto_sha1h: {
  Register DstReg = I.getOperand(0).getReg();
  Register SrcReg = I.getOperand(2).getReg();

  // FIXME: Should this be an assert?
  if (MRI.getType(DstReg).getSizeInBits() != 32 ||
      MRI.getType(SrcReg).getSizeInBits() != 32)
    return false;

  // The operation has to happen on FPRs. Set up some new FPR registers for
  // the source and destination if they are on GPRs.
  if (RBI.getRegBank(SrcReg, MRI, TRI)->getID() != AArch64::FPRRegBankID) {
    SrcReg = MRI.createVirtualRegister(&AArch64::FPR32RegClass);
    MIB.buildCopy({SrcReg}, {I.getOperand(2)});

    // Make sure the copy ends up getting constrained properly.
    RBI.constrainGenericRegister(I.getOperand(2).getReg(),
                                 AArch64::GPR32RegClass, MRI);
  }

  if (RBI.getRegBank(DstReg, MRI, TRI)->getID() != AArch64::FPRRegBankID)
    DstReg = MRI.createVirtualRegister(&AArch64::FPR32RegClass);

  // Actually insert the instruction.
  auto SHA1Inst = MIB.buildInstr(AArch64::SHA1Hrr, {DstReg}, {SrcReg});
  constrainSelectedInstRegOperands(*SHA1Inst, TII, TRI, RBI);

  // Did we create a new register for the destination?
  if (DstReg != I.getOperand(0).getReg()) {
    // Yep. Copy the result of the instruction back into the original
    // destination.
    MIB.buildCopy({I.getOperand(0)}, {DstReg});
    RBI.constrainGenericRegister(I.getOperand(0).getReg(),
                                 AArch64::GPR32RegClass, MRI);
  }

  I.eraseFromParent();
  return true;
}
case Intrinsic::frameaddress:
case Intrinsic::returnaddress: {
  MachineFunction &MF = *I.getParent()->getParent();
  MachineFrameInfo &MFI = MF.getFrameInfo();

  unsigned Depth = I.getOperand(2).getImm();
  Register DstReg = I.getOperand(0).getReg();
  RBI.constrainGenericRegister(DstReg, AArch64::GPR64RegClass, MRI);

  if (Depth == 0 && IntrinID == Intrinsic::returnaddress) {
    if (!MFReturnAddr) {
      // Insert the copy from LR/X30 into the entry block, before it can be
      // clobbered by anything.
      MFI.setReturnAddressIsTaken(true);
      MFReturnAddr = getFunctionLiveInPhysReg(MF, TII, AArch64::LR,
                                              AArch64::GPR64RegClass);
    }

    if (STI.hasPAuth()) {
      MIB.buildInstr(AArch64::XPACI, {DstReg}, {MFReturnAddr});
    } else {
      MIB.buildCopy({Register(AArch64::LR)}, {MFReturnAddr});
      MIB.buildInstr(AArch64::XPACLRI);
      MIB.buildCopy({DstReg}, {Register(AArch64::LR)});
    }

    I.eraseFromParent();
    return true;
  }

  MFI.setFrameAddressIsTaken(true);
  Register FrameAddr(AArch64::FP);
  while (Depth--) {
    Register NextFrame = MRI.createVirtualRegister(&AArch64::GPR64spRegClass);
    auto Ldr =
        MIB.buildInstr(AArch64::LDRXui, {NextFrame}, {FrameAddr}).addImm(0);
    constrainSelectedInstRegOperands(*Ldr, TII, TRI, RBI);
    FrameAddr = NextFrame;
  }

  if (IntrinID == Intrinsic::frameaddress)
    MIB.buildCopy({DstReg}, {FrameAddr});
  else {
    MFI.setReturnAddressIsTaken(true);

    if (STI.hasPAuth()) {
      Register TmpReg = MRI.createVirtualRegister(&AArch64::GPR64RegClass);
      MIB.buildInstr(AArch64::LDRXui, {TmpReg}, {FrameAddr}).addImm(1);
      MIB.buildInstr(AArch64::XPACI, {DstReg}, {TmpReg});
    } else {
      MIB.buildInstr(AArch64::LDRXui, {Register(AArch64::LR)}, {FrameAddr})
          .addImm(1);
      MIB.buildInstr(AArch64::XPACLRI);
      MIB.buildCopy({DstReg}, {Register(AArch64::LR)});
    }
  }

  I.eraseFromParent();
  return true;
}
case Intrinsic::swift_async_context_addr:
  auto Sub = MIB.buildInstr(AArch64::SUBXri, {I.getOperand(0).getReg()},
                            {Register(AArch64::FP)})
                 .addImm(8)
                 .addImm(0);
  constrainSelectedInstRegOperands(*Sub, TII, TRI, RBI);

  MF->getFrameInfo().setFrameAddressIsTaken(true);
  MF->getInfo<AArch64FunctionInfo>()->setHasSwiftAsyncContext(true);
  I.eraseFromParent();
  return true;
}
return false;
5373}

5375InstructionSelector::ComplexRendererFns
5376AArch64InstructionSelector::selectShiftA_32(const MachineOperand &Root) const {
auto MaybeImmed = getImmedFromMO(Root);
if (MaybeImmed == None || *MaybeImmed > 31)
  return None;
uint64_t Enc = (32 - *MaybeImmed) & 0x1f;
return {{[=](MachineInstrBuilder &MIB) { MIB.addImm(Enc); }}};
5382}

5384InstructionSelector::ComplexRendererFns
5385AArch64InstructionSelector::selectShiftB_32(const MachineOperand &Root) const {
auto MaybeImmed = getImmedFromMO(Root);
if (MaybeImmed == None || *MaybeImmed > 31)
  return None;
uint64_t Enc = 31 - *MaybeImmed;
return {{[=](MachineInstrBuilder &MIB) { MIB.addImm(Enc); }}};
5391}

5393InstructionSelector::ComplexRendererFns
5394AArch64InstructionSelector::selectShiftA_64(const MachineOperand &Root) const {
auto MaybeImmed = getImmedFromMO(Root);
if (MaybeImmed == None || *MaybeImmed > 63)
  return None;
uint64_t Enc = (64 - *MaybeImmed) & 0x3f;
return {{[=](MachineInstrBuilder &MIB) { MIB.addImm(Enc); }}};
5400}

5402InstructionSelector::ComplexRendererFns
5403AArch64InstructionSelector::selectShiftB_64(const MachineOperand &Root) const {
auto MaybeImmed = getImmedFromMO(Root);
if (MaybeImmed == None || *MaybeImmed > 63)
  return None;
uint64_t Enc = 63 - *MaybeImmed;
return {{[=](MachineInstrBuilder &MIB) { MIB.addImm(Enc); }}};
5409}

5411/// Helper to select an immediate value that can be represented as a 12-bit
5412/// value shifted left by either 0 or 12. If it is possible to do so, return
5413/// the immediate and shift value. If not, return None.
5414///
5415/// Used by selectArithImmed and selectNegArithImmed.
5416InstructionSelector::ComplexRendererFns
5417AArch64InstructionSelector::select12BitValueWithLeftShift(
  uint64_t Immed) const {
unsigned ShiftAmt;
if (Immed >> 12 == 0) {
  ShiftAmt = 0;
} else if ((Immed & 0xfff) == 0 && Immed >> 24 == 0) {
  ShiftAmt = 12;
  Immed = Immed >> 12;
} else
  return None;

unsigned ShVal = AArch64_AM::getShifterImm(AArch64_AM::LSL, ShiftAmt);
return {{
    [=](MachineInstrBuilder &MIB) { MIB.addImm(Immed); },
    [=](MachineInstrBuilder &MIB) { MIB.addImm(ShVal); },
}};
5433}

5435/// SelectArithImmed - Select an immediate value that can be represented as
5436/// a 12-bit value shifted left by either 0 or 12.  If so, return true with
5437/// Val set to the 12-bit value and Shift set to the shifter operand.
5438InstructionSelector::ComplexRendererFns
5439AArch64InstructionSelector::selectArithImmed(MachineOperand &Root) const {
// This function is called from the addsub_shifted_imm ComplexPattern,
// which lists [imm] as the list of opcode it's interested in, however
// we still need to check whether the operand is actually an immediate
// here because the ComplexPattern opcode list is only used in
// root-level opcode matching.
auto MaybeImmed = getImmedFromMO(Root);
if (MaybeImmed == None)
  return None;
return select12BitValueWithLeftShift(*MaybeImmed);
5449}

5451/// SelectNegArithImmed - As above, but negates the value before trying to
5452/// select it.
5453InstructionSelector::ComplexRendererFns
5454AArch64InstructionSelector::selectNegArithImmed(MachineOperand &Root) const {
// We need a register here, because we need to know if we have a 64 or 32
// bit immediate.
if (!Root.isReg())
  return None;
auto MaybeImmed = getImmedFromMO(Root);
if (MaybeImmed == None)
  return None;
uint64_t Immed = *MaybeImmed;

// This negation is almost always valid, but "cmp wN, #0" and "cmn wN, #0"
// have the opposite effect on the C flag, so this pattern mustn't match under
// those circumstances.
if (Immed == 0)
  return None;

// Check if we're dealing with a 32-bit type on the root or a 64-bit type on
// the root.
MachineRegisterInfo &MRI = Root.getParent()->getMF()->getRegInfo();
if (MRI.getType(Root.getReg()).getSizeInBits() == 32)
  Immed = ~((uint32_t)Immed) + 1;
else
  Immed = ~Immed + 1ULL;

if (Immed & 0xFFFFFFFFFF000000ULL)
  return None;

Immed &= 0xFFFFFFULL;
return select12BitValueWithLeftShift(Immed);
5483}

5485/// Return true if it is worth folding MI into an extended register. That is,
5486/// if it's safe to pull it into the addressing mode of a load or store as a
5487/// shift.
5488bool AArch64InstructionSelector::isWorthFoldingIntoExtendedReg(
  MachineInstr &MI, const MachineRegisterInfo &MRI) const {
// Always fold if there is one use, or if we're optimizing for size.
Register DefReg = MI.getOperand(0).getReg();
if (MRI.hasOneNonDBGUse(DefReg) ||
    MI.getParent()->getParent()->getFunction().hasOptSize())
  return true;

// It's better to avoid folding and recomputing shifts when we don't have a
// fastpath.
if (!STI.hasLSLFast())
  return false;

// We have a fastpath, so folding a shift in and potentially computing it
// many times may be beneficial. Check if this is only used in memory ops.
// If it is, then we should fold.
return all_of(MRI.use_nodbg_instructions(DefReg),
              [](MachineInstr &Use) { return Use.mayLoadOrStore(); });
5506}

5508static bool isSignExtendShiftType(AArch64_AM::ShiftExtendType Type) {
switch (Type) {
case AArch64_AM::SXTB:
case AArch64_AM::SXTH:
case AArch64_AM::SXTW:
  return true;
default:
  return false;
}
5517}

5519InstructionSelector::ComplexRendererFns
5520AArch64InstructionSelector::selectExtendedSHL(
  MachineOperand &Root, MachineOperand &Base, MachineOperand &Offset,
  unsigned SizeInBytes, bool WantsExt) const {
assert(Base.isReg() && "Expected base to be a register operand")(static_cast <bool> (Base.isReg() && "Expected base to be a register operand"
) ? void (0) : __assert_fail ("Base.isReg() && \"Expected base to be a register operand\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 5523, __extension__ __PRETTY_FUNCTION__));
assert(Offset.isReg() && "Expected offset to be a register operand")(static_cast <bool> (Offset.isReg() && "Expected offset to be a register operand"
) ? void (0) : __assert_fail ("Offset.isReg() && \"Expected offset to be a register operand\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 5524, __extension__ __PRETTY_FUNCTION__));

MachineRegisterInfo &MRI = Root.getParent()->getMF()->getRegInfo();
MachineInstr *OffsetInst = MRI.getVRegDef(Offset.getReg());
if (!OffsetInst)
  return None;

unsigned OffsetOpc = OffsetInst->getOpcode();
bool LookedThroughZExt = false;
if (OffsetOpc != TargetOpcode::G_SHL && OffsetOpc != TargetOpcode::G_MUL) {
  // Try to look through a ZEXT.
  if (OffsetOpc != TargetOpcode::G_ZEXT || !WantsExt)
    return None;

  OffsetInst = MRI.getVRegDef(OffsetInst->getOperand(1).getReg());
  OffsetOpc = OffsetInst->getOpcode();
  LookedThroughZExt = true;

  if (OffsetOpc != TargetOpcode::G_SHL && OffsetOpc != TargetOpcode::G_MUL)
    return None;
}
// Make sure that the memory op is a valid size.
int64_t LegalShiftVal = Log2_32(SizeInBytes);
if (LegalShiftVal == 0)
  return None;
if (!isWorthFoldingIntoExtendedReg(*OffsetInst, MRI))
  return None;

// Now, try to find the specific G_CONSTANT. Start by assuming that the
// register we will offset is the LHS, and the register containing the
// constant is the RHS.
Register OffsetReg = OffsetInst->getOperand(1).getReg();
Register ConstantReg = OffsetInst->getOperand(2).getReg();
auto ValAndVReg = getConstantVRegValWithLookThrough(ConstantReg, MRI);
if (!ValAndVReg) {
  // We didn't get a constant on the RHS. If the opcode is a shift, then
  // we're done.
  if (OffsetOpc == TargetOpcode::G_SHL)
    return None;

  // If we have a G_MUL, we can use either register. Try looking at the RHS.
  std::swap(OffsetReg, ConstantReg);
  ValAndVReg = getConstantVRegValWithLookThrough(ConstantReg, MRI);
  if (!ValAndVReg)
    return None;
}

// The value must fit into 3 bits, and must be positive. Make sure that is
// true.
int64_t ImmVal = ValAndVReg->Value.getSExtValue();

// Since we're going to pull this into a shift, the constant value must be
// a power of 2. If we got a multiply, then we need to check this.
if (OffsetOpc == TargetOpcode::G_MUL) {
  if (!isPowerOf2_32(ImmVal))
    return None;

  // Got a power of 2. So, the amount we'll shift is the log base-2 of that.
  ImmVal = Log2_32(ImmVal);
}

if ((ImmVal & 0x7) != ImmVal)
  return None;

// We are only allowed to shift by LegalShiftVal. This shift value is built
// into the instruction, so we can't just use whatever we want.
if (ImmVal != LegalShiftVal)
  return None;

unsigned SignExtend = 0;
if (WantsExt) {
  // Check if the offset is defined by an extend, unless we looked through a
  // G_ZEXT earlier.
  if (!LookedThroughZExt) {
    MachineInstr *ExtInst = getDefIgnoringCopies(OffsetReg, MRI);
    auto Ext = getExtendTypeForInst(*ExtInst, MRI, true);
    if (Ext == AArch64_AM::InvalidShiftExtend)
      return None;

    SignExtend = isSignExtendShiftType(Ext) ? 1 : 0;
    // We only support SXTW for signed extension here.
    if (SignExtend && Ext != AArch64_AM::SXTW)
      return None;
    OffsetReg = ExtInst->getOperand(1).getReg();
  }

  // Need a 32-bit wide register here.
  MachineIRBuilder MIB(*MRI.getVRegDef(Root.getReg()));
  OffsetReg = moveScalarRegClass(OffsetReg, AArch64::GPR32RegClass, MIB);
}

// We can use the LHS of the GEP as the base, and the LHS of the shift as an
// offset. Signify that we are shifting by setting the shift flag to 1.
return {{[=](MachineInstrBuilder &MIB) { MIB.addUse(Base.getReg()); },
         [=](MachineInstrBuilder &MIB) { MIB.addUse(OffsetReg); },
         [=](MachineInstrBuilder &MIB) {
           // Need to add both immediates here to make sure that they are both
           // added to the instruction.
           MIB.addImm(SignExtend);
           MIB.addImm(1);
         }}};
5625}

5627/// This is used for computing addresses like this:
5628///
5629/// ldr x1, [x2, x3, lsl #3]
5630///
5631/// Where x2 is the base register, and x3 is an offset register. The shift-left
5632/// is a constant value specific to this load instruction. That is, we'll never
5633/// see anything other than a 3 here (which corresponds to the size of the
5634/// element being loaded.)
5635InstructionSelector::ComplexRendererFns
5636AArch64InstructionSelector::selectAddrModeShiftedExtendXReg(
  MachineOperand &Root, unsigned SizeInBytes) const {
if (!Root.isReg())
  return None;
MachineRegisterInfo &MRI = Root.getParent()->getMF()->getRegInfo();

// We want to find something like this:
//
// val = G_CONSTANT LegalShiftVal
// shift = G_SHL off_reg val
// ptr = G_PTR_ADD base_reg shift
// x = G_LOAD ptr
//
// And fold it into this addressing mode:
//
// ldr x, [base_reg, off_reg, lsl #LegalShiftVal]

// Check if we can find the G_PTR_ADD.
MachineInstr *PtrAdd =
    getOpcodeDef(TargetOpcode::G_PTR_ADD, Root.getReg(), MRI);
if (!PtrAdd || !isWorthFoldingIntoExtendedReg(*PtrAdd, MRI))
  return None;

// Now, try to match an opcode which will match our specific offset.
// We want a G_SHL or a G_MUL.
MachineInstr *OffsetInst =
    getDefIgnoringCopies(PtrAdd->getOperand(2).getReg(), MRI);
return selectExtendedSHL(Root, PtrAdd->getOperand(1),
                         OffsetInst->getOperand(0), SizeInBytes,
                         /*WantsExt=*/false);
5666}

5668/// This is used for computing addresses like this:
5669///
5670/// ldr x1, [x2, x3]
5671///
5672/// Where x2 is the base register, and x3 is an offset register.
5673///
5674/// When possible (or profitable) to fold a G_PTR_ADD into the address calculation,
5675/// this will do so. Otherwise, it will return None.
5676InstructionSelector::ComplexRendererFns
5677AArch64InstructionSelector::selectAddrModeRegisterOffset(
  MachineOperand &Root) const {
MachineRegisterInfo &MRI = Root.getParent()->getMF()->getRegInfo();

// We need a GEP.
MachineInstr *Gep = MRI.getVRegDef(Root.getReg());
if (!Gep || Gep->getOpcode() != TargetOpcode::G_PTR_ADD)
  return None;

// If this is used more than once, let's not bother folding.
// TODO: Check if they are memory ops. If they are, then we can still fold
// without having to recompute anything.
if (!MRI.hasOneNonDBGUse(Gep->getOperand(0).getReg()))
  return None;

// Base is the GEP's LHS, offset is its RHS.
return {{[=](MachineInstrBuilder &MIB) {
           MIB.addUse(Gep->getOperand(1).getReg());
         },
         [=](MachineInstrBuilder &MIB) {
           MIB.addUse(Gep->getOperand(2).getReg());
         },
         [=](MachineInstrBuilder &MIB) {
           // Need to add both immediates here to make sure that they are both
           // added to the instruction.
           MIB.addImm(0);
           MIB.addImm(0);
         }}};
5705}

5707/// This is intended to be equivalent to selectAddrModeXRO in
5708/// AArch64ISelDAGtoDAG. It's used for selecting X register offset loads.
5709InstructionSelector::ComplexRendererFns
5710AArch64InstructionSelector::selectAddrModeXRO(MachineOperand &Root,
                                            unsigned SizeInBytes) const {
MachineRegisterInfo &MRI = Root.getParent()->getMF()->getRegInfo();
if (!Root.isReg())
  return None;
MachineInstr *PtrAdd =
    getOpcodeDef(TargetOpcode::G_PTR_ADD, Root.getReg(), MRI);
if (!PtrAdd)
  return None;

// Check for an immediates which cannot be encoded in the [base + imm]
// addressing mode, and can't be encoded in an add/sub. If this happens, we'll
// end up with code like:
//
// mov x0, wide
// add x1 base, x0
// ldr x2, [x1, x0]
//
// In this situation, we can use the [base, xreg] addressing mode to save an
// add/sub:
//
// mov x0, wide
// ldr x2, [base, x0]
auto ValAndVReg =
    getConstantVRegValWithLookThrough(PtrAdd->getOperand(2).getReg(), MRI);
if (ValAndVReg) {
  unsigned Scale = Log2_32(SizeInBytes);
  int64_t ImmOff = ValAndVReg->Value.getSExtValue();

  // Skip immediates that can be selected in the load/store addresing
  // mode.
  if (ImmOff % SizeInBytes == 0 && ImmOff >= 0 &&
      ImmOff < (0x1000 << Scale))
    return None;

  // Helper lambda to decide whether or not it is preferable to emit an add.
  auto isPreferredADD = [](int64_t ImmOff) {
    // Constants in [0x0, 0xfff] can be encoded in an add.
    if ((ImmOff & 0xfffffffffffff000LL) == 0x0LL)
      return true;

    // Can it be encoded in an add lsl #12?
    if ((ImmOff & 0xffffffffff000fffLL) != 0x0LL)
      return false;

    // It can be encoded in an add lsl #12, but we may not want to. If it is
    // possible to select this as a single movz, then prefer that. A single
    // movz is faster than an add with a shift.
    return (ImmOff & 0xffffffffff00ffffLL) != 0x0LL &&
           (ImmOff & 0xffffffffffff0fffLL) != 0x0LL;
  };

  // If the immediate can be encoded in a single add/sub, then bail out.
  if (isPreferredADD(ImmOff) || isPreferredADD(-ImmOff))
    return None;
}

// Try to fold shifts into the addressing mode.
auto AddrModeFns = selectAddrModeShiftedExtendXReg(Root, SizeInBytes);
if (AddrModeFns)
  return AddrModeFns;

// If that doesn't work, see if it's possible to fold in registers from
// a GEP.
return selectAddrModeRegisterOffset(Root);
5775}

5777/// This is used for computing addresses like this:
5778///
5779/// ldr x0, [xBase, wOffset, sxtw #LegalShiftVal]
5780///
5781/// Where we have a 64-bit base register, a 32-bit offset register, and an
5782/// extend (which may or may not be signed).
5783InstructionSelector::ComplexRendererFns
5784AArch64InstructionSelector::selectAddrModeWRO(MachineOperand &Root,
                                            unsigned SizeInBytes) const {
MachineRegisterInfo &MRI = Root.getParent()->getMF()->getRegInfo();

MachineInstr *PtrAdd =
    getOpcodeDef(TargetOpcode::G_PTR_ADD, Root.getReg(), MRI);
if (!PtrAdd || !isWorthFoldingIntoExtendedReg(*PtrAdd, MRI))
  return None;

MachineOperand &LHS = PtrAdd->getOperand(1);
MachineOperand &RHS = PtrAdd->getOperand(2);
MachineInstr *OffsetInst = getDefIgnoringCopies(RHS.getReg(), MRI);

// The first case is the same as selectAddrModeXRO, except we need an extend.
// In this case, we try to find a shift and extend, and fold them into the
// addressing mode.
//
// E.g.
//
// off_reg = G_Z/S/ANYEXT ext_reg
// val = G_CONSTANT LegalShiftVal
// shift = G_SHL off_reg val
// ptr = G_PTR_ADD base_reg shift
// x = G_LOAD ptr
//
// In this case we can get a load like this:
//
// ldr x0, [base_reg, ext_reg, sxtw #LegalShiftVal]
auto ExtendedShl = selectExtendedSHL(Root, LHS, OffsetInst->getOperand(0),
                                     SizeInBytes, /*WantsExt=*/true);
if (ExtendedShl)
  return ExtendedShl;

// There was no shift. We can try and fold a G_Z/S/ANYEXT in alone though.
//
// e.g.
// ldr something, [base_reg, ext_reg, sxtw]
if (!isWorthFoldingIntoExtendedReg(*OffsetInst, MRI))
  return None;

// Check if this is an extend. We'll get an extend type if it is.
AArch64_AM::ShiftExtendType Ext =
    getExtendTypeForInst(*OffsetInst, MRI, /*IsLoadStore=*/true);
if (Ext == AArch64_AM::InvalidShiftExtend)
  return None;

// Need a 32-bit wide register.
MachineIRBuilder MIB(*PtrAdd);
Register ExtReg = moveScalarRegClass(OffsetInst->getOperand(1).getReg(),
                                     AArch64::GPR32RegClass, MIB);
unsigned SignExtend = Ext == AArch64_AM::SXTW;

// Base is LHS, offset is ExtReg.
return {{[=](MachineInstrBuilder &MIB) { MIB.addUse(LHS.getReg()); },
         [=](MachineInstrBuilder &MIB) { MIB.addUse(ExtReg); },
         [=](MachineInstrBuilder &MIB) {
           MIB.addImm(SignExtend);
           MIB.addImm(0);
         }}};
5843}

5845/// Select a "register plus unscaled signed 9-bit immediate" address.  This
5846/// should only match when there is an offset that is not valid for a scaled
5847/// immediate addressing mode.  The "Size" argument is the size in bytes of the
5848/// memory reference, which is needed here to know what is valid for a scaled
5849/// immediate.
5850InstructionSelector::ComplexRendererFns
5851AArch64InstructionSelector::selectAddrModeUnscaled(MachineOperand &Root,
                                                 unsigned Size) const {
MachineRegisterInfo &MRI =
    Root.getParent()->getParent()->getParent()->getRegInfo();

if (!Root.isReg())
14
←
Calling 'MachineOperand::isReg'→
16
←
Returning from 'MachineOperand::isReg'→
17
←
Taking false branch→
  return None;

if (!isBaseWithConstantOffset(Root, MRI))
18
←
Assuming the condition is false→
19
←
Taking false branch→
  return None;

MachineInstr *RootDef = MRI.getVRegDef(Root.getReg());
if (!RootDef)
20
←
Assuming 'RootDef' is non-null→
21
←
Taking false branch→
  return None;

MachineOperand &OffImm = RootDef->getOperand(2);
if (!OffImm.isReg())
22
←
Calling 'MachineOperand::isReg'→
25
←
Returning from 'MachineOperand::isReg'→
26
←
Taking false branch→
  return None;
MachineInstr *RHS = MRI.getVRegDef(OffImm.getReg());
if (!RHS || RHS->getOpcode() != TargetOpcode::G_CONSTANT)
27
←
Assuming 'RHS' is non-null→
28
←
Assuming the condition is false→
29
←
Taking false branch→
  return None;
int64_t RHSC;
MachineOperand &RHSOp1 = RHS->getOperand(1);
if (!RHSOp1.isCImm() || RHSOp1.getCImm()->getBitWidth() > 64)
30
←
Calling 'MachineOperand::isCImm'→
33
←
Returning from 'MachineOperand::isCImm'→
34
←
Assuming the condition is false→
35
←
Taking false branch→
  return None;
RHSC = RHSOp1.getCImm()->getSExtValue();

// If the offset is valid as a scaled immediate, don't match here.
if ((RHSC & (Size - 1)) == 0 && RHSC >= 0 && RHSC < (0x1000 << Log2_32(Size)))
36
←
Assuming the condition is true→
37
←
Assuming 'RHSC' is >= 0→
38
←
Calling 'Log2_32'→
40
←
Returning from 'Log2_32'→
41
←
The result of the left shift is undefined due to shifting by '4294967295', which is greater or equal to the width of type 'int'
  return None;
if (RHSC >= -256 && RHSC < 256) {
  MachineOperand &Base = RootDef->getOperand(1);
  return {{
      [=](MachineInstrBuilder &MIB) { MIB.add(Base); },
      [=](MachineInstrBuilder &MIB) { MIB.addImm(RHSC); },
  }};
}
return None;
5889}

5891InstructionSelector::ComplexRendererFns
5892AArch64InstructionSelector::tryFoldAddLowIntoImm(MachineInstr &RootDef,
                                               unsigned Size,
                                               MachineRegisterInfo &MRI) const {
if (RootDef.getOpcode() != AArch64::G_ADD_LOW)
  return None;
MachineInstr &Adrp = *MRI.getVRegDef(RootDef.getOperand(1).getReg());
if (Adrp.getOpcode() != AArch64::ADRP)
  return None;

// TODO: add heuristics like isWorthFoldingADDlow() from SelectionDAG.
auto Offset = Adrp.getOperand(1).getOffset();
if (Offset % Size != 0)
  return None;

auto GV = Adrp.getOperand(1).getGlobal();
if (GV->isThreadLocal())
  return None;

auto &MF = *RootDef.getParent()->getParent();
if (GV->getPointerAlignment(MF.getDataLayout()) < Size)
  return None;

unsigned OpFlags = STI.ClassifyGlobalReference(GV, MF.getTarget());
MachineIRBuilder MIRBuilder(RootDef);
Register AdrpReg = Adrp.getOperand(0).getReg();
return {{[=](MachineInstrBuilder &MIB) { MIB.addUse(AdrpReg); },
         [=](MachineInstrBuilder &MIB) {
           MIB.addGlobalAddress(GV, Offset,
                                OpFlags | AArch64II::MO_PAGEOFF |
                                    AArch64II::MO_NC);
         }}};
5923}

5925/// Select a "register plus scaled unsigned 12-bit immediate" address.  The
5926/// "Size" argument is the size in bytes of the memory reference, which
5927/// determines the scale.
5928InstructionSelector::ComplexRendererFns
5929AArch64InstructionSelector::selectAddrModeIndexed(MachineOperand &Root,
                                                unsigned Size) const {
MachineFunction &MF = *Root.getParent()->getParent()->getParent();
MachineRegisterInfo &MRI = MF.getRegInfo();

if (!Root.isReg())
4
←
Taking false branch→
  return None;

MachineInstr *RootDef = MRI.getVRegDef(Root.getReg());
if (!RootDef)
5
←
Assuming 'RootDef' is non-null→
6
←
Taking false branch→
  return None;

if (RootDef->getOpcode() == TargetOpcode::G_FRAME_INDEX) {
7
←
Assuming the condition is false→
8
←
Taking false branch→
  return {{
      [=](MachineInstrBuilder &MIB) { MIB.add(RootDef->getOperand(1)); },
      [=](MachineInstrBuilder &MIB) { MIB.addImm(0); },
  }};
}

CodeModel::Model CM = MF.getTarget().getCodeModel();
// Check if we can fold in the ADD of small code model ADRP + ADD address.
if (CM == CodeModel::Small) {
9
←
Assuming 'CM' is not equal to Small→
10
←
Taking false branch→
  auto OpFns = tryFoldAddLowIntoImm(*RootDef, Size, MRI);
  if (OpFns)
    return OpFns;
}

if (isBaseWithConstantOffset(Root, MRI)) {
11
←
Assuming the condition is false→
12
←
Taking false branch→
  MachineOperand &LHS = RootDef->getOperand(1);
  MachineOperand &RHS = RootDef->getOperand(2);
  MachineInstr *LHSDef = MRI.getVRegDef(LHS.getReg());
  MachineInstr *RHSDef = MRI.getVRegDef(RHS.getReg());
  if (LHSDef && RHSDef) {
    int64_t RHSC = (int64_t)RHSDef->getOperand(1).getCImm()->getZExtValue();
    unsigned Scale = Log2_32(Size);
    if ((RHSC & (Size - 1)) == 0 && RHSC >= 0 && RHSC < (0x1000 << Scale)) {
      if (LHSDef->getOpcode() == TargetOpcode::G_FRAME_INDEX)
        return {{
            [=](MachineInstrBuilder &MIB) { MIB.add(LHSDef->getOperand(1)); },
            [=](MachineInstrBuilder &MIB) { MIB.addImm(RHSC >> Scale); },
        }};

      return {{
          [=](MachineInstrBuilder &MIB) { MIB.add(LHS); },
          [=](MachineInstrBuilder &MIB) { MIB.addImm(RHSC >> Scale); },
      }};
    }
  }
}

// Before falling back to our general case, check if the unscaled
// instructions can handle this. If so, that's preferable.
if (selectAddrModeUnscaled(Root, Size).hasValue())
13
←
Calling 'AArch64InstructionSelector::selectAddrModeUnscaled'→
  return None;

return {{
    [=](MachineInstrBuilder &MIB) { MIB.add(Root); },
    [=](MachineInstrBuilder &MIB) { MIB.addImm(0); },
}};
5988}

5990/// Given a shift instruction, return the correct shift type for that
5991/// instruction.
5992static AArch64_AM::ShiftExtendType getShiftTypeForInst(MachineInstr &MI) {
// TODO: Handle AArch64_AM::ROR
switch (MI.getOpcode()) {
default:
  return AArch64_AM::InvalidShiftExtend;
case TargetOpcode::G_SHL:
  return AArch64_AM::LSL;
case TargetOpcode::G_LSHR:
  return AArch64_AM::LSR;
case TargetOpcode::G_ASHR:
  return AArch64_AM::ASR;
}
6004}

6006/// Select a "shifted register" operand. If the value is not shifted, set the
6007/// shift operand to a default value of "lsl 0".
6008///
6009/// TODO: Allow shifted register to be rotated in logical instructions.
6010InstructionSelector::ComplexRendererFns
6011AArch64InstructionSelector::selectShiftedRegister(MachineOperand &Root) const {
if (!Root.isReg())
  return None;
MachineRegisterInfo &MRI =
    Root.getParent()->getParent()->getParent()->getRegInfo();

// Check if the operand is defined by an instruction which corresponds to
// a ShiftExtendType. E.g. a G_SHL, G_LSHR, etc.
//
// TODO: Handle AArch64_AM::ROR for logical instructions.
MachineInstr *ShiftInst = MRI.getVRegDef(Root.getReg());
if (!ShiftInst)
  return None;
AArch64_AM::ShiftExtendType ShType = getShiftTypeForInst(*ShiftInst);
if (ShType == AArch64_AM::InvalidShiftExtend)
  return None;
if (!isWorthFoldingIntoExtendedReg(*ShiftInst, MRI))
  return None;

// Need an immediate on the RHS.
MachineOperand &ShiftRHS = ShiftInst->getOperand(2);
auto Immed = getImmedFromMO(ShiftRHS);
if (!Immed)
  return None;

// We have something that we can fold. Fold in the shift's LHS and RHS into
// the instruction.
MachineOperand &ShiftLHS = ShiftInst->getOperand(1);
Register ShiftReg = ShiftLHS.getReg();

unsigned NumBits = MRI.getType(ShiftReg).getSizeInBits();
unsigned Val = *Immed & (NumBits - 1);
unsigned ShiftVal = AArch64_AM::getShifterImm(ShType, Val);

return {{[=](MachineInstrBuilder &MIB) { MIB.addUse(ShiftReg); },
         [=](MachineInstrBuilder &MIB) { MIB.addImm(ShiftVal); }}};
6047}

6049AArch64_AM::ShiftExtendType AArch64InstructionSelector::getExtendTypeForInst(
  MachineInstr &MI, MachineRegisterInfo &MRI, bool IsLoadStore) const {
unsigned Opc = MI.getOpcode();

// Handle explicit extend instructions first.
if (Opc == TargetOpcode::G_SEXT || Opc == TargetOpcode::G_SEXT_INREG) {
  unsigned Size;
  if (Opc == TargetOpcode::G_SEXT)
    Size = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
  else
    Size = MI.getOperand(2).getImm();
  assert(Size != 64 && "Extend from 64 bits?")(static_cast <bool> (Size != 64 && "Extend from 64 bits?"
) ? void (0) : __assert_fail ("Size != 64 && \"Extend from 64 bits?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 6060, __extension__ __PRETTY_FUNCTION__));
  switch (Size) {
  case 8:
    return IsLoadStore ? AArch64_AM::InvalidShiftExtend : AArch64_AM::SXTB;
  case 16:
    return IsLoadStore ? AArch64_AM::InvalidShiftExtend : AArch64_AM::SXTH;
  case 32:
    return AArch64_AM::SXTW;
  default:
    return AArch64_AM::InvalidShiftExtend;
  }
}

if (Opc == TargetOpcode::G_ZEXT || Opc == TargetOpcode::G_ANYEXT) {
  unsigned Size = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
  assert(Size != 64 && "Extend from 64 bits?")(static_cast <bool> (Size != 64 && "Extend from 64 bits?"
) ? void (0) : __assert_fail ("Size != 64 && \"Extend from 64 bits?\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 6075, __extension__ __PRETTY_FUNCTION__));
  switch (Size) {
  case 8:
    return IsLoadStore ? AArch64_AM::InvalidShiftExtend : AArch64_AM::UXTB;
  case 16:
    return IsLoadStore ? AArch64_AM::InvalidShiftExtend : AArch64_AM::UXTH;
  case 32:
    return AArch64_AM::UXTW;
  default:
    return AArch64_AM::InvalidShiftExtend;
  }
}

// Don't have an explicit extend. Try to handle a G_AND with a constant mask
// on the RHS.
if (Opc != TargetOpcode::G_AND)
  return AArch64_AM::InvalidShiftExtend;

Optional<uint64_t> MaybeAndMask = getImmedFromMO(MI.getOperand(2));
if (!MaybeAndMask)
  return AArch64_AM::InvalidShiftExtend;
uint64_t AndMask = *MaybeAndMask;
switch (AndMask) {
default:
  return AArch64_AM::InvalidShiftExtend;
case 0xFF:
  return !IsLoadStore ? AArch64_AM::UXTB : AArch64_AM::InvalidShiftExtend;
case 0xFFFF:
  return !IsLoadStore ? AArch64_AM::UXTH : AArch64_AM::InvalidShiftExtend;
case 0xFFFFFFFF:
  return AArch64_AM::UXTW;
}
6107}

6109Register AArch64InstructionSelector::moveScalarRegClass(
  Register Reg, const TargetRegisterClass &RC, MachineIRBuilder &MIB) const {
MachineRegisterInfo &MRI = *MIB.getMRI();
auto Ty = MRI.getType(Reg);
assert(!Ty.isVector() && "Expected scalars only!")(static_cast <bool> (!Ty.isVector() && "Expected scalars only!"
) ? void (0) : __assert_fail ("!Ty.isVector() && \"Expected scalars only!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 6113, __extension__ __PRETTY_FUNCTION__));
if (Ty.getSizeInBits() == TRI.getRegSizeInBits(RC))
  return Reg;

// Create a copy and immediately select it.
// FIXME: We should have an emitCopy function?
auto Copy = MIB.buildCopy({&RC}, {Reg});
selectCopy(*Copy, TII, MRI, TRI, RBI);
return Copy.getReg(0);
6122}

6124/// Select an "extended register" operand. This operand folds in an extend
6125/// followed by an optional left shift.
6126InstructionSelector::ComplexRendererFns
6127AArch64InstructionSelector::selectArithExtendedRegister(
  MachineOperand &Root) const {
if (!Root.isReg())
  return None;
MachineRegisterInfo &MRI =
    Root.getParent()->getParent()->getParent()->getRegInfo();

uint64_t ShiftVal = 0;
Register ExtReg;
AArch64_AM::ShiftExtendType Ext;
MachineInstr *RootDef = getDefIgnoringCopies(Root.getReg(), MRI);
if (!RootDef)
  return None;

if (!isWorthFoldingIntoExtendedReg(*RootDef, MRI))
  return None;

// Check if we can fold a shift and an extend.
if (RootDef->getOpcode() == TargetOpcode::G_SHL) {
  // Look for a constant on the RHS of the shift.
  MachineOperand &RHS = RootDef->getOperand(2);
  Optional<uint64_t> MaybeShiftVal = getImmedFromMO(RHS);
  if (!MaybeShiftVal)
    return None;
  ShiftVal = *MaybeShiftVal;
  if (ShiftVal > 4)
    return None;
  // Look for a valid extend instruction on the LHS of the shift.
  MachineOperand &LHS = RootDef->getOperand(1);
  MachineInstr *ExtDef = getDefIgnoringCopies(LHS.getReg(), MRI);
  if (!ExtDef)
    return None;
  Ext = getExtendTypeForInst(*ExtDef, MRI);
  if (Ext == AArch64_AM::InvalidShiftExtend)
    return None;
  ExtReg = ExtDef->getOperand(1).getReg();
} else {
  // Didn't get a shift. Try just folding an extend.
  Ext = getExtendTypeForInst(*RootDef, MRI);
  if (Ext == AArch64_AM::InvalidShiftExtend)
    return None;
  ExtReg = RootDef->getOperand(1).getReg();

  // If we have a 32 bit instruction which zeroes out the high half of a
  // register, we get an implicit zero extend for free. Check if we have one.
  // FIXME: We actually emit the extend right now even though we don't have
  // to.
  if (Ext == AArch64_AM::UXTW && MRI.getType(ExtReg).getSizeInBits() == 32) {
    MachineInstr *ExtInst = MRI.getVRegDef(ExtReg);
    if (ExtInst && isDef32(*ExtInst))
      return None;
  }
}

// We require a GPR32 here. Narrow the ExtReg if needed using a subregister
// copy.
MachineIRBuilder MIB(*RootDef);
ExtReg = moveScalarRegClass(ExtReg, AArch64::GPR32RegClass, MIB);

return {{[=](MachineInstrBuilder &MIB) { MIB.addUse(ExtReg); },
         [=](MachineInstrBuilder &MIB) {
           MIB.addImm(getArithExtendImm(Ext, ShiftVal));
         }}};
6190}

6192void AArch64InstructionSelector::renderTruncImm(MachineInstrBuilder &MIB,
                                              const MachineInstr &MI,
                                              int OpIdx) const {
const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&(static_cast <bool> (MI.getOpcode() == TargetOpcode::G_CONSTANT
 && OpIdx == -1 && "Expected G_CONSTANT") ? void
 (0) : __assert_fail ("MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 && \"Expected G_CONSTANT\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 6197, __extension__ __PRETTY_FUNCTION__))
       "Expected G_CONSTANT")(static_cast <bool> (MI.getOpcode() == TargetOpcode::G_CONSTANT
 && OpIdx == -1 && "Expected G_CONSTANT") ? void
 (0) : __assert_fail ("MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 && \"Expected G_CONSTANT\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 6197, __extension__ __PRETTY_FUNCTION__));
Optional<int64_t> CstVal =
    getConstantVRegSExtVal(MI.getOperand(0).getReg(), MRI);
assert(CstVal && "Expected constant value")(static_cast <bool> (CstVal && "Expected constant value"
) ? void (0) : __assert_fail ("CstVal && \"Expected constant value\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 6200, __extension__ __PRETTY_FUNCTION__));
MIB.addImm(CstVal.getValue());
6202}

6204void AArch64InstructionSelector::renderLogicalImm32(
MachineInstrBuilder &MIB, const MachineInstr &I, int OpIdx) const {
assert(I.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&(static_cast <bool> (I.getOpcode() == TargetOpcode::G_CONSTANT
 && OpIdx == -1 && "Expected G_CONSTANT") ? void
 (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 && \"Expected G_CONSTANT\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 6207, __extension__ __PRETTY_FUNCTION__))
       "Expected G_CONSTANT")(static_cast <bool> (I.getOpcode() == TargetOpcode::G_CONSTANT
 && OpIdx == -1 && "Expected G_CONSTANT") ? void
 (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 && \"Expected G_CONSTANT\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 6207, __extension__ __PRETTY_FUNCTION__));
uint64_t CstVal = I.getOperand(1).getCImm()->getZExtValue();
uint64_t Enc = AArch64_AM::encodeLogicalImmediate(CstVal, 32);
MIB.addImm(Enc);
6211}

6213void AArch64InstructionSelector::renderLogicalImm64(
MachineInstrBuilder &MIB, const MachineInstr &I, int OpIdx) const {
assert(I.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&(static_cast <bool> (I.getOpcode() == TargetOpcode::G_CONSTANT
 && OpIdx == -1 && "Expected G_CONSTANT") ? void
 (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 && \"Expected G_CONSTANT\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 6216, __extension__ __PRETTY_FUNCTION__))
       "Expected G_CONSTANT")(static_cast <bool> (I.getOpcode() == TargetOpcode::G_CONSTANT
 && OpIdx == -1 && "Expected G_CONSTANT") ? void
 (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 && \"Expected G_CONSTANT\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 6216, __extension__ __PRETTY_FUNCTION__));
uint64_t CstVal = I.getOperand(1).getCImm()->getZExtValue();
uint64_t Enc = AArch64_AM::encodeLogicalImmediate(CstVal, 64);
MIB.addImm(Enc);
6220}

6222void AArch64InstructionSelector::renderFPImm16(MachineInstrBuilder &MIB,
                                             const MachineInstr &MI,
                                             int OpIdx) const {
assert(MI.getOpcode() == TargetOpcode::G_FCONSTANT && OpIdx == -1 &&(static_cast <bool> (MI.getOpcode() == TargetOpcode::G_FCONSTANT
 && OpIdx == -1 && "Expected G_FCONSTANT") ? void
 (0) : __assert_fail ("MI.getOpcode() == TargetOpcode::G_FCONSTANT && OpIdx == -1 && \"Expected G_FCONSTANT\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 6226, __extension__ __PRETTY_FUNCTION__))
       "Expected G_FCONSTANT")(static_cast <bool> (MI.getOpcode() == TargetOpcode::G_FCONSTANT
 && OpIdx == -1 && "Expected G_FCONSTANT") ? void
 (0) : __assert_fail ("MI.getOpcode() == TargetOpcode::G_FCONSTANT && OpIdx == -1 && \"Expected G_FCONSTANT\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 6226, __extension__ __PRETTY_FUNCTION__));
MIB.addImm(
    AArch64_AM::getFP16Imm(MI.getOperand(1).getFPImm()->getValueAPF()));
6229}

6231void AArch64InstructionSelector::renderFPImm32(MachineInstrBuilder &MIB,
                                             const MachineInstr &MI,
                                             int OpIdx) const {
assert(MI.getOpcode() == TargetOpcode::G_FCONSTANT && OpIdx == -1 &&(static_cast <bool> (MI.getOpcode() == TargetOpcode::G_FCONSTANT
 && OpIdx == -1 && "Expected G_FCONSTANT") ? void
 (0) : __assert_fail ("MI.getOpcode() == TargetOpcode::G_FCONSTANT && OpIdx == -1 && \"Expected G_FCONSTANT\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 6235, __extension__ __PRETTY_FUNCTION__))
       "Expected G_FCONSTANT")(static_cast <bool> (MI.getOpcode() == TargetOpcode::G_FCONSTANT
 && OpIdx == -1 && "Expected G_FCONSTANT") ? void
 (0) : __assert_fail ("MI.getOpcode() == TargetOpcode::G_FCONSTANT && OpIdx == -1 && \"Expected G_FCONSTANT\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 6235, __extension__ __PRETTY_FUNCTION__));
MIB.addImm(
    AArch64_AM::getFP32Imm(MI.getOperand(1).getFPImm()->getValueAPF()));
6238}

6240void AArch64InstructionSelector::renderFPImm64(MachineInstrBuilder &MIB,
                                             const MachineInstr &MI,
                                             int OpIdx) const {
assert(MI.getOpcode() == TargetOpcode::G_FCONSTANT && OpIdx == -1 &&(static_cast <bool> (MI.getOpcode() == TargetOpcode::G_FCONSTANT
 && OpIdx == -1 && "Expected G_FCONSTANT") ? void
 (0) : __assert_fail ("MI.getOpcode() == TargetOpcode::G_FCONSTANT && OpIdx == -1 && \"Expected G_FCONSTANT\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 6244, __extension__ __PRETTY_FUNCTION__))
       "Expected G_FCONSTANT")(static_cast <bool> (MI.getOpcode() == TargetOpcode::G_FCONSTANT
 && OpIdx == -1 && "Expected G_FCONSTANT") ? void
 (0) : __assert_fail ("MI.getOpcode() == TargetOpcode::G_FCONSTANT && OpIdx == -1 && \"Expected G_FCONSTANT\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 6244, __extension__ __PRETTY_FUNCTION__));
MIB.addImm(
    AArch64_AM::getFP64Imm(MI.getOperand(1).getFPImm()->getValueAPF()));
6247}

6249bool AArch64InstructionSelector::isLoadStoreOfNumBytes(
  const MachineInstr &MI, unsigned NumBytes) const {
if (!MI.mayLoadOrStore())
  return false;
assert(MI.hasOneMemOperand() &&(static_cast <bool> (MI.hasOneMemOperand() && "Expected load/store to have only one mem op!"
) ? void (0) : __assert_fail ("MI.hasOneMemOperand() && \"Expected load/store to have only one mem op!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 6254, __extension__ __PRETTY_FUNCTION__))
       "Expected load/store to have only one mem op!")(static_cast <bool> (MI.hasOneMemOperand() && "Expected load/store to have only one mem op!"
) ? void (0) : __assert_fail ("MI.hasOneMemOperand() && \"Expected load/store to have only one mem op!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 6254, __extension__ __PRETTY_FUNCTION__));
return (*MI.memoperands_begin())->getSize() == NumBytes;
6256}

6258bool AArch64InstructionSelector::isDef32(const MachineInstr &MI) const {
const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
if (MRI.getType(MI.getOperand(0).getReg()).getSizeInBits() != 32)
  return false;

// Only return true if we know the operation will zero-out the high half of
// the 64-bit register. Truncates can be subregister copies, which don't
// zero out the high bits. Copies and other copy-like instructions can be
// fed by truncates, or could be lowered as subregister copies.
switch (MI.getOpcode()) {
default:
  return true;
case TargetOpcode::COPY:
case TargetOpcode::G_BITCAST:
case TargetOpcode::G_TRUNC:
case TargetOpcode::G_PHI:
  return false;
}
6276}


6279// Perform fixups on the given PHI instruction's operands to force them all
6280// to be the same as the destination regbank.
6281static void fixupPHIOpBanks(MachineInstr &MI, MachineRegisterInfo &MRI,
                          const AArch64RegisterBankInfo &RBI) {
assert(MI.getOpcode() == TargetOpcode::G_PHI && "Expected a G_PHI")(static_cast <bool> (MI.getOpcode() == TargetOpcode::G_PHI
 && "Expected a G_PHI") ? void (0) : __assert_fail ("MI.getOpcode() == TargetOpcode::G_PHI && \"Expected a G_PHI\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 6283, __extension__ __PRETTY_FUNCTION__));
Register DstReg = MI.getOperand(0).getReg();
const RegisterBank *DstRB = MRI.getRegBankOrNull(DstReg);
assert(DstRB && "Expected PHI dst to have regbank assigned")(static_cast <bool> (DstRB && "Expected PHI dst to have regbank assigned"
) ? void (0) : __assert_fail ("DstRB && \"Expected PHI dst to have regbank assigned\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 6286, __extension__ __PRETTY_FUNCTION__));
MachineIRBuilder MIB(MI);

// Go through each operand and ensure it has the same regbank.
for (unsigned OpIdx = 1; OpIdx < MI.getNumOperands(); ++OpIdx) {
  MachineOperand &MO = MI.getOperand(OpIdx);
  if (!MO.isReg())
    continue;
  Register OpReg = MO.getReg();
  const RegisterBank *RB = MRI.getRegBankOrNull(OpReg);
  if (RB != DstRB) {
    // Insert a cross-bank copy.
    auto *OpDef = MRI.getVRegDef(OpReg);
    const LLT &Ty = MRI.getType(OpReg);
    MachineBasicBlock &OpDefBB = *OpDef->getParent();

    // Any instruction we insert must appear after all PHIs in the block
    // for the block to be valid MIR.
    MachineBasicBlock::iterator InsertPt = std::next(OpDef->getIterator());
    if (InsertPt != OpDefBB.end() && InsertPt->isPHI())
      InsertPt = OpDefBB.getFirstNonPHI();
    MIB.setInsertPt(*OpDef->getParent(), InsertPt);
    auto Copy = MIB.buildCopy(Ty, OpReg);
    MRI.setRegBank(Copy.getReg(0), *DstRB);
    MO.setReg(Copy.getReg(0));
  }
}
6313}

6315void AArch64InstructionSelector::processPHIs(MachineFunction &MF) {
// We're looking for PHIs, build a list so we don't invalidate iterators.
MachineRegisterInfo &MRI = MF.getRegInfo();
SmallVector<MachineInstr *, 32> Phis;
for (auto &BB : MF) {
  for (auto &MI : BB) {
    if (MI.getOpcode() == TargetOpcode::G_PHI)
      Phis.emplace_back(&MI);
  }
}

for (auto *MI : Phis) {
  // We need to do some work here if the operand types are < 16 bit and they
  // are split across fpr/gpr banks. Since all types <32b on gpr
  // end up being assigned gpr32 regclasses, we can end up with PHIs here
  // which try to select between a gpr32 and an fpr16. Ideally RBS shouldn't
  // be selecting heterogenous regbanks for operands if possible, but we
  // still need to be able to deal with it here.
  //
  // To fix this, if we have a gpr-bank operand < 32b in size and at least
  // one other operand is on the fpr bank, then we add cross-bank copies
  // to homogenize the operand banks. For simplicity the bank that we choose
  // to settle on is whatever bank the def operand has. For example:
  //
  // %endbb:
  //   %dst:gpr(s16) = G_PHI %in1:gpr(s16), %bb1, %in2:fpr(s16), %bb2
  //  =>
  // %bb2:
  //   ...
  //   %in2_copy:gpr(s16) = COPY %in2:fpr(s16)
  //   ...
  // %endbb:
  //   %dst:gpr(s16) = G_PHI %in1:gpr(s16), %bb1, %in2_copy:gpr(s16), %bb2
  bool HasGPROp = false, HasFPROp = false;
  for (unsigned OpIdx = 1; OpIdx < MI->getNumOperands(); ++OpIdx) {
    const auto &MO = MI->getOperand(OpIdx);
    if (!MO.isReg())
      continue;
    const LLT &Ty = MRI.getType(MO.getReg());
    if (!Ty.isValid() || !Ty.isScalar())
      break;
    if (Ty.getSizeInBits() >= 32)
      break;
    const RegisterBank *RB = MRI.getRegBankOrNull(MO.getReg());
    // If for some reason we don't have a regbank yet. Don't try anything.
    if (!RB)
      break;

    if (RB->getID() == AArch64::GPRRegBankID)
      HasGPROp = true;
    else
      HasFPROp = true;
  }
  // We have heterogenous regbanks, need to fixup.
  if (HasGPROp && HasFPROp)
    fixupPHIOpBanks(*MI, MRI, RBI);
}
6372}

6374namespace llvm {
6375InstructionSelector *
6376createAArch64InstructionSelector(const AArch64TargetMachine &TM,
                               AArch64Subtarget &Subtarget,
                               AArch64RegisterBankInfo &RBI) {
return new AArch64InstructionSelector(TM, Subtarget, RBI);
6380}
6381}

←

/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/MachineOperand.h

→

1//===-- llvm/CodeGen/MachineOperand.h - MachineOperand class ----*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains the declaration of the MachineOperand class.
10//
11//===----------------------------------------------------------------------===//

13#ifndef LLVM_CODEGEN_MACHINEOPERAND_H
14#define LLVM_CODEGEN_MACHINEOPERAND_H

16#include "llvm/ADT/DenseMap.h"
17#include "llvm/CodeGen/Register.h"
18#include "llvm/IR/Intrinsics.h"
19#include "llvm/Support/DataTypes.h"
20#include "llvm/Support/LowLevelTypeImpl.h"
21#include <cassert>

23namespace llvm {

25class BlockAddress;
26class Constant;
27class ConstantFP;
28class ConstantInt;
29class GlobalValue;
30class MachineBasicBlock;
31class MachineInstr;
32class MachineRegisterInfo;
33class MCCFIInstruction;
34class MDNode;
35class ModuleSlotTracker;
36class TargetIntrinsicInfo;
37class TargetRegisterInfo;
38class hash_code;
39class raw_ostream;
40class MCSymbol;

42/// MachineOperand class - Representation of each machine instruction operand.
43///
44/// This class isn't a POD type because it has a private constructor, but its
45/// destructor must be trivial. Functions like MachineInstr::addOperand(),
46/// MachineRegisterInfo::moveOperands(), and MF::DeleteMachineInstr() depend on
47/// not having to call the MachineOperand destructor.
48///
49class MachineOperand {
50public:
enum MachineOperandType : unsigned char {
  MO_Register,          ///< Register operand.
  MO_Immediate,         ///< Immediate operand
  MO_CImmediate,        ///< Immediate >64bit operand
  MO_FPImmediate,       ///< Floating-point immediate operand
  MO_MachineBasicBlock, ///< MachineBasicBlock reference
  MO_FrameIndex,        ///< Abstract Stack Frame Index
  MO_ConstantPoolIndex, ///< Address of indexed Constant in Constant Pool
  MO_TargetIndex,       ///< Target-dependent index+offset operand.
  MO_JumpTableIndex,    ///< Address of indexed Jump Table for switch
  MO_ExternalSymbol,    ///< Name of external global symbol
  MO_GlobalAddress,     ///< Address of a global value
  MO_BlockAddress,      ///< Address of a basic block
  MO_RegisterMask,      ///< Mask of preserved registers.
  MO_RegisterLiveOut,   ///< Mask of live-out registers.
  MO_Metadata,          ///< Metadata reference (for debug info)
  MO_MCSymbol,          ///< MCSymbol reference (for debug/eh info)
  MO_CFIIndex,          ///< MCCFIInstruction index.
  MO_IntrinsicID,       ///< Intrinsic ID for ISel
  MO_Predicate,         ///< Generic predicate for ISel
  MO_ShuffleMask,       ///< Other IR Constant for ISel (shuffle masks)
  MO_Last = MO_ShuffleMask
};

75private:
/// OpKind - Specify what kind of operand this is.  This discriminates the
/// union.
unsigned OpKind : 8;

/// Subregister number for MO_Register.  A value of 0 indicates the
/// MO_Register has no subReg.
///
/// For all other kinds of operands, this field holds target-specific flags.
unsigned SubReg_TargetFlags : 12;

/// TiedTo - Non-zero when this register operand is tied to another register
/// operand. The encoding of this field is described in the block comment
/// before MachineInstr::tieOperands().
unsigned TiedTo : 4;

/// IsDef - True if this is a def, false if this is a use of the register.
/// This is only valid on register operands.
///
unsigned IsDef : 1;

/// IsImp - True if this is an implicit def or use, false if it is explicit.
/// This is only valid on register opderands.
///
unsigned IsImp : 1;

/// IsDeadOrKill
/// For uses: IsKill - Conservatively indicates the last use of a register
/// on this path through the function. A register operand with true value of
/// this flag must be the last use of the register, a register operand with
/// false value may or may not be the last use of the register. After regalloc
/// we can use recomputeLivenessFlags to get precise kill flags.
/// For defs: IsDead - True if this register is never used by a subsequent
/// instruction.
/// This is only valid on register operands.
unsigned IsDeadOrKill : 1;

/// See isRenamable().
unsigned IsRenamable : 1;

/// IsUndef - True if this register operand reads an "undef" value, i.e. the
/// read value doesn't matter.  This flag can be set on both use and def
/// operands.  On a sub-register def operand, it refers to the part of the
/// register that isn't written.  On a full-register def operand, it is a
/// noop.  See readsReg().
///
/// This is only valid on registers.
///
/// Note that an instruction may have multiple <undef> operands referring to
/// the same register.  In that case, the instruction may depend on those
/// operands reading the same dont-care value.  For example:
///
///   %1 = XOR undef %2, undef %2
///
/// Any register can be used for %2, and its value doesn't matter, but
/// the two operands must be the same register.
///
unsigned IsUndef : 1;

/// IsInternalRead - True if this operand reads a value that was defined
/// inside the same instruction or bundle.  This flag can be set on both use
/// and def operands.  On a sub-register def operand, it refers to the part
/// of the register that isn't written.  On a full-register def operand, it
/// is a noop.
///
/// When this flag is set, the instruction bundle must contain at least one
/// other def of the register.  If multiple instructions in the bundle define
/// the register, the meaning is target-defined.
unsigned IsInternalRead : 1;

/// IsEarlyClobber - True if this MO_Register 'def' operand is written to
/// by the MachineInstr before all input registers are read.  This is used to
/// model the GCC inline asm '&' constraint modifier.
unsigned IsEarlyClobber : 1;

/// IsDebug - True if this MO_Register 'use' operand is in a debug pseudo,
/// not a real instruction.  Such uses should be ignored during codegen.
unsigned IsDebug : 1;

/// SmallContents - This really should be part of the Contents union, but
/// lives out here so we can get a better packed struct.
/// MO_Register: Register number.
/// OffsetedInfo: Low bits of offset.
union {
  unsigned RegNo;           // For MO_Register.
  unsigned OffsetLo;        // Matches Contents.OffsetedInfo.OffsetHi.
} SmallContents;

/// ParentMI - This is the instruction that this operand is embedded into.
/// This is valid for all operand types, when the operand is in an instr.
MachineInstr *ParentMI;

/// Contents union - This contains the payload for the various operand types.
union ContentsUnion {
  ContentsUnion() {}
  MachineBasicBlock *MBB;  // For MO_MachineBasicBlock.
  const ConstantFP *CFP;   // For MO_FPImmediate.
  const ConstantInt *CI;   // For MO_CImmediate. Integers > 64bit.
  int64_t ImmVal;          // For MO_Immediate.
  const uint32_t *RegMask; // For MO_RegisterMask and MO_RegisterLiveOut.
  const MDNode *MD;        // For MO_Metadata.
  MCSymbol *Sym;           // For MO_MCSymbol.
  unsigned CFIIndex;       // For MO_CFI.
  Intrinsic::ID IntrinsicID; // For MO_IntrinsicID.
  unsigned Pred;           // For MO_Predicate
  ArrayRef<int> ShuffleMask; // For MO_ShuffleMask

  struct {                  // For MO_Register.
    // Register number is in SmallContents.RegNo.
    MachineOperand *Prev;   // Access list for register. See MRI.
    MachineOperand *Next;
  } Reg;

  /// OffsetedInfo - This struct contains the offset and an object identifier.
  /// this represent the object as with an optional offset from it.
  struct {
    union {
      int Index;                // For MO_*Index - The index itself.
      const char *SymbolName;   // For MO_ExternalSymbol.
      const GlobalValue *GV;    // For MO_GlobalAddress.
      const BlockAddress *BA;   // For MO_BlockAddress.
    } Val;
    // Low bits of offset are in SmallContents.OffsetLo.
    int OffsetHi;               // An offset from the object, high 32 bits.
  } OffsetedInfo;
} Contents;

explicit MachineOperand(MachineOperandType K)
  : OpKind(K), SubReg_TargetFlags(0), ParentMI(nullptr) {
  // Assert that the layout is what we expect. It's easy to grow this object.
  static_assert(alignof(MachineOperand) <= alignof(int64_t),
                "MachineOperand shouldn't be more than 8 byte aligned");
  static_assert(sizeof(Contents) <= 2 * sizeof(void *),
                "Contents should be at most two pointers");
  static_assert(sizeof(MachineOperand) <=
                    alignTo<alignof(int64_t)>(2 * sizeof(unsigned) +
                                              3 * sizeof(void *)),
                "MachineOperand too big. Should be Kind, SmallContents, "
                "ParentMI, and Contents");
}

216public:
/// getType - Returns the MachineOperandType for this operand.
///
MachineOperandType getType() const { return (MachineOperandType)OpKind; }

unsigned getTargetFlags() const {
  return isReg() ? 0 : SubReg_TargetFlags;
}
void setTargetFlags(unsigned F) {
  assert(!isReg() && "Register operands can't have target flags")(static_cast <bool> (!isReg() && "Register operands can't have target flags"
) ? void (0) : __assert_fail ("!isReg() && \"Register operands can't have target flags\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/MachineOperand.h"
, 225, __extension__ __PRETTY_FUNCTION__));
  SubReg_TargetFlags = F;
  assert(SubReg_TargetFlags == F && "Target flags out of range")(static_cast <bool> (SubReg_TargetFlags == F &&
 "Target flags out of range") ? void (0) : __assert_fail ("SubReg_TargetFlags == F && \"Target flags out of range\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/MachineOperand.h"
, 227, __extension__ __PRETTY_FUNCTION__));
}
void addTargetFlag(unsigned F) {
  assert(!isReg() && "Register operands can't have target flags")(static_cast <bool> (!isReg() && "Register operands can't have target flags"
) ? void (0) : __assert_fail ("!isReg() && \"Register operands can't have target flags\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/MachineOperand.h"
, 230, __extension__ __PRETTY_FUNCTION__));
  SubReg_TargetFlags |= F;
  assert((SubReg_TargetFlags & F) && "Target flags out of range")(static_cast <bool> ((SubReg_TargetFlags & F) &&
 "Target flags out of range") ? void (0) : __assert_fail ("(SubReg_TargetFlags & F) && \"Target flags out of range\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/MachineOperand.h"
, 232, __extension__ __PRETTY_FUNCTION__));
}


/// getParent - Return the instruction that this operand belongs to.
///
MachineInstr *getParent() { return ParentMI; }
const MachineInstr *getParent() const { return ParentMI; }

/// clearParent - Reset the parent pointer.
///
/// The MachineOperand copy constructor also copies ParentMI, expecting the
/// original to be deleted. If a MachineOperand is ever stored outside a
/// MachineInstr, the parent pointer must be cleared.
///
/// Never call clearParent() on an operand in a MachineInstr.
///
void clearParent() { ParentMI = nullptr; }

/// Print a subreg index operand.
/// MO_Immediate operands can also be subreg idices. If it's the case, the
/// subreg index name will be printed. MachineInstr::isOperandSubregIdx can be
/// called to check this.
static void printSubRegIdx(raw_ostream &OS, uint64_t Index,
                           const TargetRegisterInfo *TRI);

/// Print operand target flags.
static void printTargetFlags(raw_ostream& OS, const MachineOperand &Op);

/// Print a MCSymbol as an operand.
static void printSymbol(raw_ostream &OS, MCSymbol &Sym);

/// Print a stack object reference.
static void printStackObjectReference(raw_ostream &OS, unsigned FrameIndex,
                                      bool IsFixed, StringRef Name);

/// Print the offset with explicit +/- signs.
static void printOperandOffset(raw_ostream &OS, int64_t Offset);

/// Print an IRSlotNumber.
static void printIRSlotNumber(raw_ostream &OS, int Slot);

/// Print the MachineOperand to \p os.
/// Providing a valid \p TRI and \p IntrinsicInfo results in a more
/// target-specific printing. If \p TRI and \p IntrinsicInfo are null, the
/// function will try to pick it up from the parent.
void print(raw_ostream &os, const TargetRegisterInfo *TRI = nullptr,
           const TargetIntrinsicInfo *IntrinsicInfo = nullptr) const;

/// More complex way of printing a MachineOperand.
/// \param TypeToPrint specifies the generic type to be printed on uses and
/// defs. It can be determined using MachineInstr::getTypeToPrint.
/// \param OpIdx - specifies the index of the operand in machine instruction.
/// This will be used by target dependent MIR formatter. Could be None if the
/// index is unknown, e.g. called by dump().
/// \param PrintDef - whether we want to print `def` on an operand which
/// isDef. Sometimes, if the operand is printed before '=', we don't print
/// `def`.
/// \param IsStandalone - whether we want a verbose output of the MO. This
/// prints extra information that can be easily inferred when printing the
/// whole function, but not when printing only a fragment of it.
/// \param ShouldPrintRegisterTies - whether we want to print register ties.
/// Sometimes they are easily determined by the instruction's descriptor
/// (MachineInstr::hasComplexRegiterTies can determine if it's needed).
/// \param TiedOperandIdx - if we need to print register ties this needs to
/// provide the index of the tied register. If not, it will be ignored.
/// \param TRI - provide more target-specific information to the printer.
/// Unlike the previous function, this one will not try and get the
/// information from it's parent.
/// \param IntrinsicInfo - same as \p TRI.
void print(raw_ostream &os, ModuleSlotTracker &MST, LLT TypeToPrint,
           Optional<unsigned> OpIdx, bool PrintDef, bool IsStandalone,
           bool ShouldPrintRegisterTies, unsigned TiedOperandIdx,
           const TargetRegisterInfo *TRI,
           const TargetIntrinsicInfo *IntrinsicInfo) const;

/// Same as print(os, TRI, IntrinsicInfo), but allows to specify the low-level
/// type to be printed the same way the full version of print(...) does it.
void print(raw_ostream &os, LLT TypeToPrint,
           const TargetRegisterInfo *TRI = nullptr,
           const TargetIntrinsicInfo *IntrinsicInfo = nullptr) const;

void dump() const;

//===--------------------------------------------------------------------===//
// Accessors that tell you what kind of MachineOperand you're looking at.
//===--------------------------------------------------------------------===//

/// isReg - Tests if this is a MO_Register operand.
bool isReg() const { return OpKind == MO_Register; }
15
←
Returning the value 1, which participates in a condition later→
23
←
Assuming field 'OpKind' is equal to MO_Register→
24
←
Returning the value 1, which participates in a condition later→
/// isImm - Tests if this is a MO_Immediate operand.
bool isImm() const { return OpKind == MO_Immediate; }
/// isCImm - Test if this is a MO_CImmediate operand.
bool isCImm() const { return OpKind == MO_CImmediate; }
31
←
Assuming field 'OpKind' is equal to MO_CImmediate→
32
←
Returning the value 1, which participates in a condition later→
/// isFPImm - Tests if this is a MO_FPImmediate operand.
bool isFPImm() const { return OpKind == MO_FPImmediate; }
/// isMBB - Tests if this is a MO_MachineBasicBlock operand.
bool isMBB() const { return OpKind == MO_MachineBasicBlock; }
/// isFI - Tests if this is a MO_FrameIndex operand.
bool isFI() const { return OpKind == MO_FrameIndex; }
/// isCPI - Tests if this is a MO_ConstantPoolIndex operand.
bool isCPI() const { return OpKind == MO_ConstantPoolIndex; }
/// isTargetIndex - Tests if this is a MO_TargetIndex operand.
bool isTargetIndex() const { return OpKind == MO_TargetIndex; }
/// isJTI - Tests if this is a MO_JumpTableIndex operand.
bool isJTI() const { return OpKind == MO_JumpTableIndex; }
/// isGlobal - Tests if this is a MO_GlobalAddress operand.
bool isGlobal() const { return OpKind == MO_GlobalAddress; }
/// isSymbol - Tests if this is a MO_ExternalSymbol operand.
bool isSymbol() const { return OpKind == MO_ExternalSymbol; }
/// isBlockAddress - Tests if this is a MO_BlockAddress operand.
bool isBlockAddress() const { return OpKind == MO_BlockAddress; }
/// isRegMask - Tests if this is a MO_RegisterMask operand.
bool isRegMask() const { return OpKind == MO_RegisterMask; }
/// isRegLiveOut - Tests if this is a MO_RegisterLiveOut operand.
bool isRegLiveOut() const { return OpKind == MO_RegisterLiveOut; }
/// isMetadata - Tests if this is a MO_Metadata operand.
bool isMetadata() const { return OpKind == MO_Metadata; }
bool isMCSymbol() const { return OpKind == MO_MCSymbol; }
bool isCFIIndex() const { return OpKind == MO_CFIIndex; }
bool isIntrinsicID() const { return OpKind == MO_IntrinsicID; }
bool isPredicate() const { return OpKind == MO_Predicate; }
bool isShuffleMask() const { return OpKind == MO_ShuffleMask; }
//===--------------------------------------------------------------------===//
// Accessors for Register Operands
//===--------------------------------------------------------------------===//

/// getReg - Returns the register number.
Register getReg() const {
  assert(isReg() && "This is not a register operand!")(static_cast <bool> (isReg() && "This is not a register operand!"
) ? void (0) : __assert_fail ("isReg() && \"This is not a register operand!\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/MachineOperand.h"
, 361, __extension__ __PRETTY_FUNCTION__));
  return Register(SmallContents.RegNo);
}

unsigned getSubReg() const {
  assert(isReg() && "Wrong MachineOperand accessor")(static_cast <bool> (isReg() && "Wrong MachineOperand accessor"
) ? void (0) : __assert_fail ("isReg() && \"Wrong MachineOperand accessor\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/MachineOperand.h"
, 366, __extension__ __PRETTY_FUNCTION__));
  return SubReg_TargetFlags;
}

bool isUse() const {
  assert(isReg() && "Wrong MachineOperand accessor")(static_cast <bool> (isReg() && "Wrong MachineOperand accessor"
) ? void (0) : __assert_fail ("isReg() && \"Wrong MachineOperand accessor\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/MachineOperand.h"
, 371, __extension__ __PRETTY_FUNCTION__));
  return !IsDef;
}

bool isDef() const {
  assert(isReg() && "Wrong MachineOperand accessor")(static_cast <bool> (isReg() && "Wrong MachineOperand accessor"
) ? void (0) : __assert_fail ("isReg() && \"Wrong MachineOperand accessor\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/MachineOperand.h"
, 376, __extension__ __PRETTY_FUNCTION__));
  return IsDef;
}

bool isImplicit() const {
  assert(isReg() && "Wrong MachineOperand accessor")(static_cast <bool> (isReg() && "Wrong MachineOperand accessor"
) ? void (0) : __assert_fail ("isReg() && \"Wrong MachineOperand accessor\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/MachineOperand.h"
, 381, __extension__ __PRETTY_FUNCTION__));
  return IsImp;
}

bool isDead() const {
  assert(isReg() && "Wrong MachineOperand accessor")(static_cast <bool> (isReg() && "Wrong MachineOperand accessor"
) ? void (0) : __assert_fail ("isReg() && \"Wrong MachineOperand accessor\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/MachineOperand.h"
, 386, __extension__ __PRETTY_FUNCTION__));
  return IsDeadOrKill & IsDef;
}

bool isKill() const {
  assert(isReg() && "Wrong MachineOperand accessor")(static_cast <bool> (isReg() && "Wrong MachineOperand accessor"
) ? void (0) : __assert_fail ("isReg() && \"Wrong MachineOperand accessor\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/MachineOperand.h"
, 391, __extension__ __PRETTY_FUNCTION__));
  return IsDeadOrKill & !IsDef;
}

bool isUndef() const {
  assert(isReg() && "Wrong MachineOperand accessor")(static_cast <bool> (isReg() && "Wrong MachineOperand accessor"
) ? void (0) : __assert_fail ("isReg() && \"Wrong MachineOperand accessor\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/MachineOperand.h"
, 396, __extension__ __PRETTY_FUNCTION__));
  return IsUndef;
}

/// isRenamable - Returns true if this register may be renamed, i.e. it does
/// not generate a value that is somehow read in a way that is not represented
/// by the Machine IR (e.g. to meet an ABI or ISA requirement).  This is only
/// valid on physical register operands.  Virtual registers are assumed to
/// always be renamable regardless of the value of this field.
///
/// Operands that are renamable can freely be changed to any other register
/// that is a member of the register class returned by
/// MI->getRegClassConstraint().
///
/// isRenamable can return false for several different reasons:
///
/// - ABI constraints (since liveness is not always precisely modeled).  We
///   conservatively handle these cases by setting all physical register
///   operands that didn’t start out as virtual regs to not be renamable.
///   Also any physical register operands created after register allocation or
///   whose register is changed after register allocation will not be
///   renamable.  This state is tracked in the MachineOperand::IsRenamable
///   bit.
///
/// - Opcode/target constraints: for opcodes that have complex register class
///   requirements (e.g. that depend on other operands/instructions), we set
///   hasExtraSrcRegAllocReq/hasExtraDstRegAllocReq in the machine opcode
///   description.  Operands belonging to instructions with opcodes that are
///   marked hasExtraSrcRegAllocReq/hasExtraDstRegAllocReq return false from
///   isRenamable().  Additionally, the AllowRegisterRenaming target property
///   prevents any operands from being marked renamable for targets that don't
///   have detailed opcode hasExtraSrcRegAllocReq/hasExtraDstRegAllocReq
///   values.
bool isRenamable() const;

bool isInternalRead() const {
  assert(isReg() && "Wrong MachineOperand accessor")(static_cast <bool> (isReg() && "Wrong MachineOperand accessor"
) ? void (0) : __assert_fail ("isReg() && \"Wrong MachineOperand accessor\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/MachineOperand.h"
, 432, __extension__ __PRETTY_FUNCTION__));
  return IsInternalRead;
}

bool isEarlyClobber() const {
  assert(isReg() && "Wrong MachineOperand accessor")(static_cast <bool> (isReg() && "Wrong MachineOperand accessor"
) ? void (0) : __assert_fail ("isReg() && \"Wrong MachineOperand accessor\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/MachineOperand.h"
, 437, __extension__ __PRETTY_FUNCTION__));
  return IsEarlyClobber;
}

bool isTied() const {
  assert(isReg() && "Wrong MachineOperand accessor")(static_cast <bool> (isReg() && "Wrong MachineOperand accessor"
) ? void (0) : __assert_fail ("isReg() && \"Wrong MachineOperand accessor\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/MachineOperand.h"
, 442, __extension__ __PRETTY_FUNCTION__));
  return TiedTo;
}

bool isDebug() const {
  assert(isReg() && "Wrong MachineOperand accessor")(static_cast <bool> (isReg() && "Wrong MachineOperand accessor"
) ? void (0) : __assert_fail ("isReg() && \"Wrong MachineOperand accessor\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/MachineOperand.h"
, 447, __extension__ __PRETTY_FUNCTION__));
  return IsDebug;
}

/// readsReg - Returns true if this operand reads the previous value of its
/// register.  A use operand with the <undef> flag set doesn't read its
/// register.  A sub-register def implicitly reads the other parts of the
/// register being redefined unless the <undef> flag is set.
///
/// This refers to reading the register value from before the current
/// instruction or bundle. Internal bundle reads are not included.
bool readsReg() const {
  assert(isReg() && "Wrong MachineOperand accessor")(static_cast <bool> (isReg() && "Wrong MachineOperand accessor"
) ? void (0) : __assert_fail ("isReg() && \"Wrong MachineOperand accessor\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/MachineOperand.h"
, 459, __extension__ __PRETTY_FUNCTION__));
  return !isUndef() && !isInternalRead() && (isUse() || getSubReg());
}

//===--------------------------------------------------------------------===//
// Mutators for Register Operands
//===--------------------------------------------------------------------===//

/// Change the register this operand corresponds to.
///
void setReg(Register Reg);

void setSubReg(unsigned subReg) {
  assert(isReg() && "Wrong MachineOperand mutator")(static_cast <bool> (isReg() && "Wrong MachineOperand mutator"
) ? void (0) : __assert_fail ("isReg() && \"Wrong MachineOperand mutator\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/MachineOperand.h"
, 472, __extension__ __PRETTY_FUNCTION__));
  SubReg_TargetFlags = subReg;
  assert(SubReg_TargetFlags == subReg && "SubReg out of range")(static_cast <bool> (SubReg_TargetFlags == subReg &&
 "SubReg out of range") ? void (0) : __assert_fail ("SubReg_TargetFlags == subReg && \"SubReg out of range\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/MachineOperand.h"
, 474, __extension__ __PRETTY_FUNCTION__));
}

/// substVirtReg - Substitute the current register with the virtual
/// subregister Reg:SubReg. Take any existing SubReg index into account,
/// using TargetRegisterInfo to compose the subreg indices if necessary.
/// Reg must be a virtual register, SubIdx can be 0.
///
void substVirtReg(Register Reg, unsigned SubIdx, const TargetRegisterInfo&);

/// substPhysReg - Substitute the current register with the physical register
/// Reg, taking any existing SubReg into account. For instance,
/// substPhysReg(%eax) will change %reg1024:sub_8bit to %al.
///
void substPhysReg(MCRegister Reg, const TargetRegisterInfo&);

void setIsUse(bool Val = true) { setIsDef(!Val); }

/// Change a def to a use, or a use to a def.
void setIsDef(bool Val = true);

void setImplicit(bool Val = true) {
  assert(isReg() && "Wrong MachineOperand mutator")(static_cast <bool> (isReg() && "Wrong MachineOperand mutator"
) ? void (0) : __assert_fail ("isReg() && \"Wrong MachineOperand mutator\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/MachineOperand.h"
, 496, __extension__ __PRETTY_FUNCTION__));
  IsImp = Val;
}

void setIsKill(bool Val = true) {
  assert(isReg() && !IsDef && "Wrong MachineOperand mutator")(static_cast <bool> (isReg() && !IsDef &&
 "Wrong MachineOperand mutator") ? void (0) : __assert_fail (
"isReg() && !IsDef && \"Wrong MachineOperand mutator\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/MachineOperand.h"
, 501, __extension__ __PRETTY_FUNCTION__));
  assert((!Val || !isDebug()) && "Marking a debug operation as kill")(static_cast <bool> ((!Val || !isDebug()) && "Marking a debug operation as kill"
) ? void (0) : __assert_fail ("(!Val || !isDebug()) && \"Marking a debug operation as kill\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/MachineOperand.h"
, 502, __extension__ __PRETTY_FUNCTION__));
  IsDeadOrKill = Val;
}

void setIsDead(bool Val = true) {
  assert(isReg() && IsDef && "Wrong MachineOperand mutator")(static_cast <bool> (isReg() && IsDef &&
 "Wrong MachineOperand mutator") ? void (0) : __assert_fail (
"isReg() && IsDef && \"Wrong MachineOperand mutator\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/MachineOperand.h"
, 507, __extension__ __PRETTY_FUNCTION__));
  IsDeadOrKill = Val;
}

void setIsUndef(bool Val = true) {
  assert(isReg() && "Wrong MachineOperand mutator")(static_cast <bool> (isReg() && "Wrong MachineOperand mutator"
) ? void (0) : __assert_fail ("isReg() && \"Wrong MachineOperand mutator\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/MachineOperand.h"
, 512, __extension__ __PRETTY_FUNCTION__));
  IsUndef = Val;
}

void setIsRenamable(bool Val = true);

void setIsInternalRead(bool Val = true) {
  assert(isReg() && "Wrong MachineOperand mutator")(static_cast <bool> (isReg() && "Wrong MachineOperand mutator"
) ? void (0) : __assert_fail ("isReg() && \"Wrong MachineOperand mutator\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/MachineOperand.h"
, 519, __extension__ __PRETTY_FUNCTION__));
  IsInternalRead = Val;
}

void setIsEarlyClobber(bool Val = true) {
  assert(isReg() && IsDef && "Wrong MachineOperand mutator")(static_cast <bool> (isReg() && IsDef &&
 "Wrong MachineOperand mutator") ? void (0) : __assert_fail (
"isReg() && IsDef && \"Wrong MachineOperand mutator\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/MachineOperand.h"
, 524, __extension__ __PRETTY_FUNCTION__));
  IsEarlyClobber = Val;
}

void setIsDebug(bool Val = true) {
  assert(isReg() && !IsDef && "Wrong MachineOperand mutator")(static_cast <bool> (isReg() && !IsDef &&
 "Wrong MachineOperand mutator") ? void (0) : __assert_fail (
"isReg() && !IsDef && \"Wrong MachineOperand mutator\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/MachineOperand.h"
, 529, __extension__ __PRETTY_FUNCTION__));
  IsDebug = Val;
}

//===--------------------------------------------------------------------===//
// Accessors for various operand types.
//===--------------------------------------------------------------------===//

int64_t getImm() const {
  assert(isImm() && "Wrong MachineOperand accessor")(static_cast <bool> (isImm() && "Wrong MachineOperand accessor"
) ? void (0) : __assert_fail ("isImm() && \"Wrong MachineOperand accessor\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/MachineOperand.h"
, 538, __extension__ __PRETTY_FUNCTION__));
  return Contents.ImmVal;
}

const ConstantInt *getCImm() const {
  assert(isCImm() && "Wrong MachineOperand accessor")(static_cast <bool> (isCImm() && "Wrong MachineOperand accessor"
) ? void (0) : __assert_fail ("isCImm() && \"Wrong MachineOperand accessor\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/MachineOperand.h"
, 543, __extension__ __PRETTY_FUNCTION__));
  return Contents.CI;
}

const ConstantFP *getFPImm() const {
  assert(isFPImm() && "Wrong MachineOperand accessor")(static_cast <bool> (isFPImm() && "Wrong MachineOperand accessor"
) ? void (0) : __assert_fail ("isFPImm() && \"Wrong MachineOperand accessor\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/MachineOperand.h"
, 548, __extension__ __PRETTY_FUNCTION__));
  return Contents.CFP;
}

MachineBasicBlock *getMBB() const {
  assert(isMBB() && "Wrong MachineOperand accessor")(static_cast <bool> (isMBB() && "Wrong MachineOperand accessor"
) ? void (0) : __assert_fail ("isMBB() && \"Wrong MachineOperand accessor\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/MachineOperand.h"
, 553, __extension__ __PRETTY_FUNCTION__));
  return Contents.MBB;
}

int getIndex() const {
  assert((isFI() || isCPI() || isTargetIndex() || isJTI()) &&(static_cast <bool> ((isFI() || isCPI() || isTargetIndex
() || isJTI()) && "Wrong MachineOperand accessor") ? void
 (0) : __assert_fail ("(isFI() || isCPI() || isTargetIndex() || isJTI()) && \"Wrong MachineOperand accessor\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/MachineOperand.h"
, 559, __extension__ __PRETTY_FUNCTION__))
         "Wrong MachineOperand accessor")(static_cast <bool> ((isFI() || isCPI() || isTargetIndex
() || isJTI()) && "Wrong MachineOperand accessor") ? void
 (0) : __assert_fail ("(isFI() || isCPI() || isTargetIndex() || isJTI()) && \"Wrong MachineOperand accessor\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/MachineOperand.h"
, 559, __extension__ __PRETTY_FUNCTION__));
  return Contents.OffsetedInfo.Val.Index;
}

const GlobalValue *getGlobal() const {
  assert(isGlobal() && "Wrong MachineOperand accessor")(static_cast <bool> (isGlobal() && "Wrong MachineOperand accessor"
) ? void (0) : __assert_fail ("isGlobal() && \"Wrong MachineOperand accessor\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/MachineOperand.h"
, 564, __extension__ __PRETTY_FUNCTION__));
  return Contents.OffsetedInfo.Val.GV;
}

const BlockAddress *getBlockAddress() const {
  assert(isBlockAddress() && "Wrong MachineOperand accessor")(static_cast <bool> (isBlockAddress() && "Wrong MachineOperand accessor"
) ? void (0) : __assert_fail ("isBlockAddress() && \"Wrong MachineOperand accessor\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/MachineOperand.h"
, 569, __extension__ __PRETTY_FUNCTION__));
  return Contents.OffsetedInfo.Val.BA;
}

MCSymbol *getMCSymbol() const {
  assert(isMCSymbol() && "Wrong MachineOperand accessor")(static_cast <bool> (isMCSymbol() && "Wrong MachineOperand accessor"
) ? void (0) : __assert_fail ("isMCSymbol() && \"Wrong MachineOperand accessor\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/MachineOperand.h"
, 574, __extension__ __PRETTY_FUNCTION__));
  return Contents.Sym;
}

unsigned getCFIIndex() const {
  assert(isCFIIndex() && "Wrong MachineOperand accessor")(static_cast <bool> (isCFIIndex() && "Wrong MachineOperand accessor"
) ? void (0) : __assert_fail ("isCFIIndex() && \"Wrong MachineOperand accessor\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/MachineOperand.h"
, 579, __extension__ __PRETTY_FUNCTION__));
  return Contents.CFIIndex;
}

Intrinsic::ID getIntrinsicID() const {
  assert(isIntrinsicID() && "Wrong MachineOperand accessor")(static_cast <bool> (isIntrinsicID() && "Wrong MachineOperand accessor"
) ? void (0) : __assert_fail ("isIntrinsicID() && \"Wrong MachineOperand accessor\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/MachineOperand.h"
, 584, __extension__ __PRETTY_FUNCTION__));
  return Contents.IntrinsicID;
}

unsigned getPredicate() const {
  assert(isPredicate() && "Wrong MachineOperand accessor")(static_cast <bool> (isPredicate() && "Wrong MachineOperand accessor"
) ? void (0) : __assert_fail ("isPredicate() && \"Wrong MachineOperand accessor\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/MachineOperand.h"
, 589, __extension__ __PRETTY_FUNCTION__));
  return Contents.Pred;
}

ArrayRef<int> getShuffleMask() const {
  assert(isShuffleMask() && "Wrong MachineOperand accessor")(static_cast <bool> (isShuffleMask() && "Wrong MachineOperand accessor"
) ? void (0) : __assert_fail ("isShuffleMask() && \"Wrong MachineOperand accessor\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/MachineOperand.h"
, 594, __extension__ __PRETTY_FUNCTION__));
  return Contents.ShuffleMask;
}

/// Return the offset from the symbol in this operand. This always returns 0
/// for ExternalSymbol operands.
int64_t getOffset() const {
  assert((isGlobal() || isSymbol() || isMCSymbol() || isCPI() ||(static_cast <bool> ((isGlobal() || isSymbol() || isMCSymbol
() || isCPI() || isTargetIndex() || isBlockAddress()) &&
 "Wrong MachineOperand accessor") ? void (0) : __assert_fail (
"(isGlobal() || isSymbol() || isMCSymbol() || isCPI() || isTargetIndex() || isBlockAddress()) && \"Wrong MachineOperand accessor\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/MachineOperand.h"
, 603, __extension__ __PRETTY_FUNCTION__))
          isTargetIndex() || isBlockAddress()) &&(static_cast <bool> ((isGlobal() || isSymbol() || isMCSymbol
() || isCPI() || isTargetIndex() || isBlockAddress()) &&
 "Wrong MachineOperand accessor") ? void (0) : __assert_fail (
"(isGlobal() || isSymbol() || isMCSymbol() || isCPI() || isTargetIndex() || isBlockAddress()) && \"Wrong MachineOperand accessor\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/MachineOperand.h"
, 603, __extension__ __PRETTY_FUNCTION__))
         "Wrong MachineOperand accessor")(static_cast <bool> ((isGlobal() || isSymbol() || isMCSymbol
() || isCPI() || isTargetIndex() || isBlockAddress()) &&
 "Wrong MachineOperand accessor") ? void (0) : __assert_fail (
"(isGlobal() || isSymbol() || isMCSymbol() || isCPI() || isTargetIndex() || isBlockAddress()) && \"Wrong MachineOperand accessor\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/MachineOperand.h"
, 603, __extension__ __PRETTY_FUNCTION__));
  return int64_t(uint64_t(Contents.OffsetedInfo.OffsetHi) << 32) |
         SmallContents.OffsetLo;
}

const char *getSymbolName() const {
  assert(isSymbol() && "Wrong MachineOperand accessor")(static_cast <bool> (isSymbol() && "Wrong MachineOperand accessor"
) ? void (0) : __assert_fail ("isSymbol() && \"Wrong MachineOperand accessor\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/MachineOperand.h"
, 609, __extension__ __PRETTY_FUNCTION__));
  return Contents.OffsetedInfo.Val.SymbolName;
}

/// clobbersPhysReg - Returns true if this RegMask clobbers PhysReg.
/// It is sometimes necessary to detach the register mask pointer from its
/// machine operand. This static method can be used for such detached bit
/// mask pointers.
static bool clobbersPhysReg(const uint32_t *RegMask, MCRegister PhysReg) {
  // See TargetRegisterInfo.h.
  assert(PhysReg < (1u << 30) && "Not a physical register")(static_cast <bool> (PhysReg < (1u << 30) &&
 "Not a physical register") ? void (0) : __assert_fail ("PhysReg < (1u << 30) && \"Not a physical register\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/MachineOperand.h"
, 619, __extension__ __PRETTY_FUNCTION__));
  return !(RegMask[PhysReg / 32] & (1u << PhysReg % 32));
}

/// clobbersPhysReg - Returns true if this RegMask operand clobbers PhysReg.
bool clobbersPhysReg(MCRegister PhysReg) const {
   return clobbersPhysReg(getRegMask(), PhysReg);
}

/// getRegMask - Returns a bit mask of registers preserved by this RegMask
/// operand.
const uint32_t *getRegMask() const {
  assert(isRegMask() && "Wrong MachineOperand accessor")(static_cast <bool> (isRegMask() && "Wrong MachineOperand accessor"
) ? void (0) : __assert_fail ("isRegMask() && \"Wrong MachineOperand accessor\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/MachineOperand.h"
, 631, __extension__ __PRETTY_FUNCTION__));
  return Contents.RegMask;
}

/// Returns number of elements needed for a regmask array.
static unsigned getRegMaskSize(unsigned NumRegs) {
  return (NumRegs + 31) / 32;
}

/// getRegLiveOut - Returns a bit mask of live-out registers.
const uint32_t *getRegLiveOut() const {
  assert(isRegLiveOut() && "Wrong MachineOperand accessor")(static_cast <bool> (isRegLiveOut() && "Wrong MachineOperand accessor"
) ? void (0) : __assert_fail ("isRegLiveOut() && \"Wrong MachineOperand accessor\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/MachineOperand.h"
, 642, __extension__ __PRETTY_FUNCTION__));
  return Contents.RegMask;
}

const MDNode *getMetadata() const {
  assert(isMetadata() && "Wrong MachineOperand accessor")(static_cast <bool> (isMetadata() && "Wrong MachineOperand accessor"
) ? void (0) : __assert_fail ("isMetadata() && \"Wrong MachineOperand accessor\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/MachineOperand.h"
, 647, __extension__ __PRETTY_FUNCTION__));
  return Contents.MD;
}

//===--------------------------------------------------------------------===//
// Mutators for various operand types.
//===--------------------------------------------------------------------===//

void setImm(int64_t immVal) {
  assert(isImm() && "Wrong MachineOperand mutator")(static_cast <bool> (isImm() && "Wrong MachineOperand mutator"
) ? void (0) : __assert_fail ("isImm() && \"Wrong MachineOperand mutator\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/MachineOperand.h"
, 656, __extension__ __PRETTY_FUNCTION__));
  Contents.ImmVal = immVal;
}

void setCImm(const ConstantInt *CI) {
  assert(isCImm() && "Wrong MachineOperand mutator")(static_cast <bool> (isCImm() && "Wrong MachineOperand mutator"
) ? void (0) : __assert_fail ("isCImm() && \"Wrong MachineOperand mutator\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/MachineOperand.h"
, 661, __extension__ __PRETTY_FUNCTION__));
  Contents.CI = CI;
}

void setFPImm(const ConstantFP *CFP) {
  assert(isFPImm() && "Wrong MachineOperand mutator")(static_cast <bool> (isFPImm() && "Wrong MachineOperand mutator"
) ? void (0) : __assert_fail ("isFPImm() && \"Wrong MachineOperand mutator\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/MachineOperand.h"
, 666, __extension__ __PRETTY_FUNCTION__));
  Contents.CFP = CFP;
}

void setOffset(int64_t Offset) {
  assert((isGlobal() || isSymbol() || isMCSymbol() || isCPI() ||(static_cast <bool> ((isGlobal() || isSymbol() || isMCSymbol
() || isCPI() || isTargetIndex() || isBlockAddress()) &&
 "Wrong MachineOperand mutator") ? void (0) : __assert_fail (
"(isGlobal() || isSymbol() || isMCSymbol() || isCPI() || isTargetIndex() || isBlockAddress()) && \"Wrong MachineOperand mutator\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/MachineOperand.h"
, 673, __extension__ __PRETTY_FUNCTION__))
          isTargetIndex() || isBlockAddress()) &&(static_cast <bool> ((isGlobal() || isSymbol() || isMCSymbol
() || isCPI() || isTargetIndex() || isBlockAddress()) &&
 "Wrong MachineOperand mutator") ? void (0) : __assert_fail (
"(isGlobal() || isSymbol() || isMCSymbol() || isCPI() || isTargetIndex() || isBlockAddress()) && \"Wrong MachineOperand mutator\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/MachineOperand.h"
, 673, __extension__ __PRETTY_FUNCTION__))
         "Wrong MachineOperand mutator")(static_cast <bool> ((isGlobal() || isSymbol() || isMCSymbol
() || isCPI() || isTargetIndex() || isBlockAddress()) &&
 "Wrong MachineOperand mutator") ? void (0) : __assert_fail (
"(isGlobal() || isSymbol() || isMCSymbol() || isCPI() || isTargetIndex() || isBlockAddress()) && \"Wrong MachineOperand mutator\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/MachineOperand.h"
, 673, __extension__ __PRETTY_FUNCTION__));
  SmallContents.OffsetLo = unsigned(Offset);
  Contents.OffsetedInfo.OffsetHi = int(Offset >> 32);
}

void setIndex(int Idx) {
  assert((isFI() || isCPI() || isTargetIndex() || isJTI()) &&(static_cast <bool> ((isFI() || isCPI() || isTargetIndex
() || isJTI()) && "Wrong MachineOperand mutator") ? void
 (0) : __assert_fail ("(isFI() || isCPI() || isTargetIndex() || isJTI()) && \"Wrong MachineOperand mutator\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/MachineOperand.h"
, 680, __extension__ __PRETTY_FUNCTION__))
         "Wrong MachineOperand mutator")(static_cast <bool> ((isFI() || isCPI() || isTargetIndex
() || isJTI()) && "Wrong MachineOperand mutator") ? void
 (0) : __assert_fail ("(isFI() || isCPI() || isTargetIndex() || isJTI()) && \"Wrong MachineOperand mutator\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/MachineOperand.h"
, 680, __extension__ __PRETTY_FUNCTION__));
  Contents.OffsetedInfo.Val.Index = Idx;
}

void setMetadata(const MDNode *MD) {
  assert(isMetadata() && "Wrong MachineOperand mutator")(static_cast <bool> (isMetadata() && "Wrong MachineOperand mutator"
) ? void (0) : __assert_fail ("isMetadata() && \"Wrong MachineOperand mutator\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/MachineOperand.h"
, 685, __extension__ __PRETTY_FUNCTION__));
  Contents.MD = MD;
}

void setMBB(MachineBasicBlock *MBB) {
  assert(isMBB() && "Wrong MachineOperand mutator")(static_cast <bool> (isMBB() && "Wrong MachineOperand mutator"
) ? void (0) : __assert_fail ("isMBB() && \"Wrong MachineOperand mutator\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/MachineOperand.h"
, 690, __extension__ __PRETTY_FUNCTION__));
  Contents.MBB = MBB;
}

/// Sets value of register mask operand referencing Mask.  The
/// operand does not take ownership of the memory referenced by Mask, it must
/// remain valid for the lifetime of the operand. See CreateRegMask().
/// Any physreg with a 0 bit in the mask is clobbered by the instruction.
void setRegMask(const uint32_t *RegMaskPtr) {
  assert(isRegMask() && "Wrong MachineOperand mutator")(static_cast <bool> (isRegMask() && "Wrong MachineOperand mutator"
) ? void (0) : __assert_fail ("isRegMask() && \"Wrong MachineOperand mutator\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/MachineOperand.h"
, 699, __extension__ __PRETTY_FUNCTION__));
  Contents.RegMask = RegMaskPtr;
}

void setIntrinsicID(Intrinsic::ID IID) {
  assert(isIntrinsicID() && "Wrong MachineOperand mutator")(static_cast <bool> (isIntrinsicID() && "Wrong MachineOperand mutator"
) ? void (0) : __assert_fail ("isIntrinsicID() && \"Wrong MachineOperand mutator\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/MachineOperand.h"
, 704, __extension__ __PRETTY_FUNCTION__));
  Contents.IntrinsicID = IID;
}

void setPredicate(unsigned Predicate) {
  assert(isPredicate() && "Wrong MachineOperand mutator")(static_cast <bool> (isPredicate() && "Wrong MachineOperand mutator"
) ? void (0) : __assert_fail ("isPredicate() && \"Wrong MachineOperand mutator\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/MachineOperand.h"
, 709, __extension__ __PRETTY_FUNCTION__));
  Contents.Pred = Predicate;
}

//===--------------------------------------------------------------------===//
// Other methods.
//===--------------------------------------------------------------------===//

/// Returns true if this operand is identical to the specified operand except
/// for liveness related flags (isKill, isUndef and isDead). Note that this
/// should stay in sync with the hash_value overload below.
bool isIdenticalTo(const MachineOperand &Other) const;

/// MachineOperand hash_value overload.
///
/// Note that this includes the same information in the hash that
/// isIdenticalTo uses for comparison. It is thus suited for use in hash
/// tables which use that function for equality comparisons only. This must
/// stay exactly in sync with isIdenticalTo above.
friend hash_code hash_value(const MachineOperand &MO);

/// ChangeToImmediate - Replace this operand with a new immediate operand of
/// the specified value.  If an operand is known to be an immediate already,
/// the setImm method should be used.
void ChangeToImmediate(int64_t ImmVal, unsigned TargetFlags = 0);

/// ChangeToFPImmediate - Replace this operand with a new FP immediate operand
/// of the specified value.  If an operand is known to be an FP immediate
/// already, the setFPImm method should be used.
void ChangeToFPImmediate(const ConstantFP *FPImm, unsigned TargetFlags = 0);

/// ChangeToES - Replace this operand with a new external symbol operand.
void ChangeToES(const char *SymName, unsigned TargetFlags = 0);

/// ChangeToGA - Replace this operand with a new global address operand.
void ChangeToGA(const GlobalValue *GV, int64_t Offset,
                unsigned TargetFlags = 0);

/// ChangeToMCSymbol - Replace this operand with a new MC symbol operand.
void ChangeToMCSymbol(MCSymbol *Sym, unsigned TargetFlags = 0);

/// Replace this operand with a frame index.
void ChangeToFrameIndex(int Idx, unsigned TargetFlags = 0);

/// Replace this operand with a target index.
void ChangeToTargetIndex(unsigned Idx, int64_t Offset,
                         unsigned TargetFlags = 0);

/// ChangeToRegister - Replace this operand with a new register operand of
/// the specified value.  If an operand is known to be an register already,
/// the setReg method should be used.
void ChangeToRegister(Register Reg, bool isDef, bool isImp = false,
                      bool isKill = false, bool isDead = false,
                      bool isUndef = false, bool isDebug = false);

/// getTargetIndexName - If this MachineOperand is a TargetIndex that has a
/// name, attempt to get the name. Returns nullptr if the TargetIndex does not
/// have a name. Asserts if MO is not a TargetIndex.
const char *getTargetIndexName() const;

//===--------------------------------------------------------------------===//
// Construction methods.
//===--------------------------------------------------------------------===//

static MachineOperand CreateImm(int64_t Val) {
  MachineOperand Op(MachineOperand::MO_Immediate);
  Op.setImm(Val);
  return Op;
}

static MachineOperand CreateCImm(const ConstantInt *CI) {
  MachineOperand Op(MachineOperand::MO_CImmediate);
  Op.Contents.CI = CI;
  return Op;
}

static MachineOperand CreateFPImm(const ConstantFP *CFP) {
  MachineOperand Op(MachineOperand::MO_FPImmediate);
  Op.Contents.CFP = CFP;
  return Op;
}

static MachineOperand CreateReg(Register Reg, bool isDef, bool isImp = false,
                                bool isKill = false, bool isDead = false,
                                bool isUndef = false,
                                bool isEarlyClobber = false,
                                unsigned SubReg = 0, bool isDebug = false,
                                bool isInternalRead = false,
                                bool isRenamable = false) {
  assert(!(isDead && !isDef) && "Dead flag on non-def")(static_cast <bool> (!(isDead && !isDef) &&
 "Dead flag on non-def") ? void (0) : __assert_fail ("!(isDead && !isDef) && \"Dead flag on non-def\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/MachineOperand.h"
, 798, __extension__ __PRETTY_FUNCTION__));
  assert(!(isKill && isDef) && "Kill flag on def")(static_cast <bool> (!(isKill && isDef) &&
 "Kill flag on def") ? void (0) : __assert_fail ("!(isKill && isDef) && \"Kill flag on def\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/MachineOperand.h"
, 799, __extension__ __PRETTY_FUNCTION__));
  MachineOperand Op(MachineOperand::MO_Register);
  Op.IsDef = isDef;
  Op.IsImp = isImp;
  Op.IsDeadOrKill = isKill | isDead;
  Op.IsRenamable = isRenamable;
  Op.IsUndef = isUndef;
  Op.IsInternalRead = isInternalRead;
  Op.IsEarlyClobber = isEarlyClobber;
  Op.TiedTo = 0;
  Op.IsDebug = isDebug;
  Op.SmallContents.RegNo = Reg;
  Op.Contents.Reg.Prev = nullptr;
  Op.Contents.Reg.Next = nullptr;
  Op.setSubReg(SubReg);
  return Op;
}
static MachineOperand CreateMBB(MachineBasicBlock *MBB,
                                unsigned TargetFlags = 0) {
  MachineOperand Op(MachineOperand::MO_MachineBasicBlock);
  Op.setMBB(MBB);
  Op.setTargetFlags(TargetFlags);
  return Op;
}
static MachineOperand CreateFI(int Idx) {
  MachineOperand Op(MachineOperand::MO_FrameIndex);
  Op.setIndex(Idx);
  return Op;
}
static MachineOperand CreateCPI(unsigned Idx, int Offset,
                                unsigned TargetFlags = 0) {
  MachineOperand Op(MachineOperand::MO_ConstantPoolIndex);
  Op.setIndex(Idx);
  Op.setOffset(Offset);
  Op.setTargetFlags(TargetFlags);
  return Op;
}
static MachineOperand CreateTargetIndex(unsigned Idx, int64_t Offset,
                                        unsigned TargetFlags = 0) {
  MachineOperand Op(MachineOperand::MO_TargetIndex);
  Op.setIndex(Idx);
  Op.setOffset(Offset);
  Op.setTargetFlags(TargetFlags);
  return Op;
}
static MachineOperand CreateJTI(unsigned Idx, unsigned TargetFlags = 0) {
  MachineOperand Op(MachineOperand::MO_JumpTableIndex);
  Op.setIndex(Idx);
  Op.setTargetFlags(TargetFlags);
  return Op;
}
static MachineOperand CreateGA(const GlobalValue *GV, int64_t Offset,
                               unsigned TargetFlags = 0) {
  MachineOperand Op(MachineOperand::MO_GlobalAddress);
  Op.Contents.OffsetedInfo.Val.GV = GV;
  Op.setOffset(Offset);
  Op.setTargetFlags(TargetFlags);
  return Op;
}
static MachineOperand CreateES(const char *SymName,
                               unsigned TargetFlags = 0) {
  MachineOperand Op(MachineOperand::MO_ExternalSymbol);
  Op.Contents.OffsetedInfo.Val.SymbolName = SymName;
  Op.setOffset(0); // Offset is always 0.
  Op.setTargetFlags(TargetFlags);
  return Op;
}
static MachineOperand CreateBA(const BlockAddress *BA, int64_t Offset,
                               unsigned TargetFlags = 0) {
  MachineOperand Op(MachineOperand::MO_BlockAddress);
  Op.Contents.OffsetedInfo.Val.BA = BA;
  Op.setOffset(Offset);
  Op.setTargetFlags(TargetFlags);
  return Op;
}
/// CreateRegMask - Creates a register mask operand referencing Mask.  The
/// operand does not take ownership of the memory referenced by Mask, it
/// must remain valid for the lifetime of the operand.
///
/// A RegMask operand represents a set of non-clobbered physical registers
/// on an instruction that clobbers many registers, typically a call.  The
/// bit mask has a bit set for each physreg that is preserved by this
/// instruction, as described in the documentation for
/// TargetRegisterInfo::getCallPreservedMask().
///
/// Any physreg with a 0 bit in the mask is clobbered by the instruction.
///
static MachineOperand CreateRegMask(const uint32_t *Mask) {
  assert(Mask && "Missing register mask")(static_cast <bool> (Mask && "Missing register mask"
) ? void (0) : __assert_fail ("Mask && \"Missing register mask\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/MachineOperand.h"
, 887, __extension__ __PRETTY_FUNCTION__));
  MachineOperand Op(MachineOperand::MO_RegisterMask);
  Op.Contents.RegMask = Mask;
  return Op;
}
static MachineOperand CreateRegLiveOut(const uint32_t *Mask) {
  assert(Mask && "Missing live-out register mask")(static_cast <bool> (Mask && "Missing live-out register mask"
) ? void (0) : __assert_fail ("Mask && \"Missing live-out register mask\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/MachineOperand.h"
, 893, __extension__ __PRETTY_FUNCTION__));
  MachineOperand Op(MachineOperand::MO_RegisterLiveOut);
  Op.Contents.RegMask = Mask;
  return Op;
}
static MachineOperand CreateMetadata(const MDNode *Meta) {
  MachineOperand Op(MachineOperand::MO_Metadata);
  Op.Contents.MD = Meta;
  return Op;
}

static MachineOperand CreateMCSymbol(MCSymbol *Sym,
                                     unsigned TargetFlags = 0) {
  MachineOperand Op(MachineOperand::MO_MCSymbol);
  Op.Contents.Sym = Sym;
  Op.setOffset(0);
  Op.setTargetFlags(TargetFlags);
  return Op;
}

static MachineOperand CreateCFIIndex(unsigned CFIIndex) {
  MachineOperand Op(MachineOperand::MO_CFIIndex);
  Op.Contents.CFIIndex = CFIIndex;
  return Op;
}

static MachineOperand CreateIntrinsicID(Intrinsic::ID ID) {
  MachineOperand Op(MachineOperand::MO_IntrinsicID);
  Op.Contents.IntrinsicID = ID;
  return Op;
}

static MachineOperand CreatePredicate(unsigned Pred) {
  MachineOperand Op(MachineOperand::MO_Predicate);
  Op.Contents.Pred = Pred;
  return Op;
}

static MachineOperand CreateShuffleMask(ArrayRef<int> Mask) {
  MachineOperand Op(MachineOperand::MO_ShuffleMask);
  Op.Contents.ShuffleMask = Mask;
  return Op;
}

friend class MachineInstr;
friend class MachineRegisterInfo;

940private:
// If this operand is currently a register operand, and if this is in a
// function, deregister the operand from the register's use/def list.
void removeRegFromUses();

/// Artificial kinds for DenseMap usage.
enum : unsigned char {
  MO_Empty = MO_Last + 1,
  MO_Tombstone,
};

friend struct DenseMapInfo<MachineOperand>;

//===--------------------------------------------------------------------===//
// Methods for handling register use/def lists.
//===--------------------------------------------------------------------===//

/// isOnRegUseList - Return true if this operand is on a register use/def
/// list or false if not.  This can only be called for register operands
/// that are part of a machine instruction.
bool isOnRegUseList() const {
  assert(isReg() && "Can only add reg operand to use lists")(static_cast <bool> (isReg() && "Can only add reg operand to use lists"
) ? void (0) : __assert_fail ("isReg() && \"Can only add reg operand to use lists\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/MachineOperand.h"
, 961, __extension__ __PRETTY_FUNCTION__));
  return Contents.Reg.Prev != nullptr;
}
964};

966template <> struct DenseMapInfo<MachineOperand> {
static MachineOperand getEmptyKey() {
  return MachineOperand(static_cast<MachineOperand::MachineOperandType>(
      MachineOperand::MO_Empty));
}
static MachineOperand getTombstoneKey() {
  return MachineOperand(static_cast<MachineOperand::MachineOperandType>(
      MachineOperand::MO_Tombstone));
}
static unsigned getHashValue(const MachineOperand &MO) {
  return hash_value(MO);
}
static bool isEqual(const MachineOperand &LHS, const MachineOperand &RHS) {
  if (LHS.getType() == static_cast<MachineOperand::MachineOperandType>(
                           MachineOperand::MO_Empty) ||
      LHS.getType() == static_cast<MachineOperand::MachineOperandType>(
                           MachineOperand::MO_Tombstone))
    return LHS.getType() == RHS.getType();
  return LHS.isIdenticalTo(RHS);
}
986};

988inline raw_ostream &operator<<(raw_ostream &OS, const MachineOperand &MO) {
MO.print(OS);
return OS;
991}

993// See friend declaration above. This additional declaration is required in
994// order to compile LLVM with IBM xlC compiler.
995hash_code hash_value(const MachineOperand &MO);
996} // namespace llvm

998#endif

←

/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/Support/MathExtras.h

1//===-- llvm/Support/MathExtras.h - Useful math functions -------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains some functions that are useful for math stuff.
10//
11//===----------------------------------------------------------------------===//
12 
13#ifndef LLVM_SUPPORT_MATHEXTRAS_H
14#define LLVM_SUPPORT_MATHEXTRAS_H
15 
16#include "llvm/Support/Compiler.h"
17#include <cassert>
18#include <climits>
19#include <cmath>
20#include <cstdint>
21#include <cstring>
22#include <limits>
23#include <type_traits>
24 
25#ifdef __ANDROID_NDK__
26#include <android/api-level.h>
27#endif
28 
29#ifdef _MSC_VER
30// Declare these intrinsics manually rather including intrin.h. It's very
31// expensive, and MathExtras.h is popular.
32// #include <intrin.h>
33extern "C" {
34unsigned char _BitScanForward(unsigned long *_Index, unsigned long _Mask);
35unsigned char _BitScanForward64(unsigned long *_Index, unsigned __int64 _Mask);
36unsigned char _BitScanReverse(unsigned long *_Index, unsigned long _Mask);
37unsigned char _BitScanReverse64(unsigned long *_Index, unsigned __int64 _Mask);
38}
39#endif
40 
41namespace llvm {
42 
43/// The behavior an operation has on an input of 0.
44enum ZeroBehavior {
45  /// The returned value is undefined.
46  ZB_Undefined,
47  /// The returned value is numeric_limits<T>::max()
48  ZB_Max,
49  /// The returned value is numeric_limits<T>::digits
50  ZB_Width
51};
52 
53/// Mathematical constants.
54namespace numbers {
55// TODO: Track C++20 std::numbers.
56// TODO: Favor using the hexadecimal FP constants (requires C++17).
57constexpr double e          = 2.7182818284590452354, // (0x1.5bf0a8b145749P+1) https://oeis.org/A001113
58                 egamma     = .57721566490153286061, // (0x1.2788cfc6fb619P-1) https://oeis.org/A001620
59                 ln2        = .69314718055994530942, // (0x1.62e42fefa39efP-1) https://oeis.org/A002162
60                 ln10       = 2.3025850929940456840, // (0x1.24bb1bbb55516P+1) https://oeis.org/A002392
61                 log2e      = 1.4426950408889634074, // (0x1.71547652b82feP+0)
62                 log10e     = .43429448190325182765, // (0x1.bcb7b1526e50eP-2)
63                 pi         = 3.1415926535897932385, // (0x1.921fb54442d18P+1) https://oeis.org/A000796
64                 inv_pi     = .31830988618379067154, // (0x1.45f306bc9c883P-2) https://oeis.org/A049541
65                 sqrtpi     = 1.7724538509055160273, // (0x1.c5bf891b4ef6bP+0) https://oeis.org/A002161
66                 inv_sqrtpi = .56418958354775628695, // (0x1.20dd750429b6dP-1) https://oeis.org/A087197
67                 sqrt2      = 1.4142135623730950488, // (0x1.6a09e667f3bcdP+0) https://oeis.org/A00219
68                 inv_sqrt2  = .70710678118654752440, // (0x1.6a09e667f3bcdP-1)
69                 sqrt3      = 1.7320508075688772935, // (0x1.bb67ae8584caaP+0) https://oeis.org/A002194
70                 inv_sqrt3  = .57735026918962576451, // (0x1.279a74590331cP-1)
71                 phi        = 1.6180339887498948482; // (0x1.9e3779b97f4a8P+0) https://oeis.org/A001622
72constexpr float ef          = 2.71828183F, // (0x1.5bf0a8P+1) https://oeis.org/A001113
73                egammaf     = .577215665F, // (0x1.2788d0P-1) https://oeis.org/A001620
74                ln2f        = .693147181F, // (0x1.62e430P-1) https://oeis.org/A002162
75                ln10f       = 2.30258509F, // (0x1.26bb1cP+1) https://oeis.org/A002392
76                log2ef      = 1.44269504F, // (0x1.715476P+0)
77                log10ef     = .434294482F, // (0x1.bcb7b2P-2)
78                pif         = 3.14159265F, // (0x1.921fb6P+1) https://oeis.org/A000796
79                inv_pif     = .318309886F, // (0x1.45f306P-2) https://oeis.org/A049541
80                sqrtpif     = 1.77245385F, // (0x1.c5bf8aP+0) https://oeis.org/A002161
81                inv_sqrtpif = .564189584F, // (0x1.20dd76P-1) https://oeis.org/A087197
82                sqrt2f      = 1.41421356F, // (0x1.6a09e6P+0) https://oeis.org/A002193
83                inv_sqrt2f  = .707106781F, // (0x1.6a09e6P-1)
84                sqrt3f      = 1.73205081F, // (0x1.bb67aeP+0) https://oeis.org/A002194
85                inv_sqrt3f  = .577350269F, // (0x1.279a74P-1)
86                phif        = 1.61803399F; // (0x1.9e377aP+0) https://oeis.org/A001622
87} // namespace numbers
88 
89namespace detail {
90template <typename T, std::size_t SizeOfT> struct TrailingZerosCounter {
91  static unsigned count(T Val, ZeroBehavior) {
92    if (!Val)
93      return std::numeric_limits<T>::digits;
94    if (Val & 0x1)
95      return 0;
96 
97    // Bisection method.
98    unsigned ZeroBits = 0;
99    T Shift = std::numeric_limits<T>::digits >> 1;
100    T Mask = std::numeric_limits<T>::max() >> Shift;
101    while (Shift) {
102      if ((Val & Mask) == 0) {
103        Val >>= Shift;
104        ZeroBits |= Shift;
105      }
106      Shift >>= 1;
107      Mask >>= Shift;
108    }
109    return ZeroBits;
110  }
111};
112 
113#if defined(__GNUC__4) || defined(_MSC_VER)
114template <typename T> struct TrailingZerosCounter<T, 4> {
115  static unsigned count(T Val, ZeroBehavior ZB) {
116    if (ZB != ZB_Undefined && Val == 0)
117      return 32;
118 
119#if __has_builtin(__builtin_ctz)1 || defined(__GNUC__4)
120    return __builtin_ctz(Val);
121#elif defined(_MSC_VER)
122    unsigned long Index;
123    _BitScanForward(&Index, Val);
124    return Index;
125#endif
126  }
127};
128 
129#if !defined(_MSC_VER) || defined(_M_X64)
130template <typename T> struct TrailingZerosCounter<T, 8> {
131  static unsigned count(T Val, ZeroBehavior ZB) {
132    if (ZB != ZB_Undefined && Val == 0)
133      return 64;
134 
135#if __has_builtin(__builtin_ctzll)1 || defined(__GNUC__4)
136    return __builtin_ctzll(Val);
137#elif defined(_MSC_VER)
138    unsigned long Index;
139    _BitScanForward64(&Index, Val);
140    return Index;
141#endif
142  }
143};
144#endif
145#endif
146} // namespace detail
147 
148/// Count number of 0's from the least significant bit to the most
149///   stopping at the first 1.
150///
151/// Only unsigned integral types are allowed.
152///
153/// \param ZB the behavior on an input of 0. Only ZB_Width and ZB_Undefined are
154///   valid arguments.
155template <typename T>
156unsigned countTrailingZeros(T Val, ZeroBehavior ZB = ZB_Width) {
157  static_assert(std::numeric_limits<T>::is_integer &&
158                    !std::numeric_limits<T>::is_signed,
159                "Only unsigned integral types are allowed.");
160  return llvm::detail::TrailingZerosCounter<T, sizeof(T)>::count(Val, ZB);
161}
162 
163namespace detail {
164template <typename T, std::size_t SizeOfT> struct LeadingZerosCounter {
165  static unsigned count(T Val, ZeroBehavior) {
166    if (!Val)
167      return std::numeric_limits<T>::digits;
168 
169    // Bisection method.
170    unsigned ZeroBits = 0;
171    for (T Shift = std::numeric_limits<T>::digits >> 1; Shift; Shift >>= 1) {
172      T Tmp = Val >> Shift;
173      if (Tmp)
174        Val = Tmp;
175      else
176        ZeroBits |= Shift;
177    }
178    return ZeroBits;
179  }
180};
181 
182#if defined(__GNUC__4) || defined(_MSC_VER)
183template <typename T> struct LeadingZerosCounter<T, 4> {
184  static unsigned count(T Val, ZeroBehavior ZB) {
185    if (ZB != ZB_Undefined && Val == 0)
186      return 32;
187 
188#if __has_builtin(__builtin_clz)1 || defined(__GNUC__4)
189    return __builtin_clz(Val);
190#elif defined(_MSC_VER)
191    unsigned long Index;
192    _BitScanReverse(&Index, Val);
193    return Index ^ 31;
194#endif
195  }
196};
197 
198#if !defined(_MSC_VER) || defined(_M_X64)
199template <typename T> struct LeadingZerosCounter<T, 8> {
200  static unsigned count(T Val, ZeroBehavior ZB) {
201    if (ZB != ZB_Undefined && Val == 0)
202      return 64;
203 
204#if __has_builtin(__builtin_clzll)1 || defined(__GNUC__4)
205    return __builtin_clzll(Val);
206#elif defined(_MSC_VER)
207    unsigned long Index;
208    _BitScanReverse64(&Index, Val);
209    return Index ^ 63;
210#endif
211  }
212};
213#endif
214#endif
215} // namespace detail
216 
217/// Count number of 0's from the most significant bit to the least
218///   stopping at the first 1.
219///
220/// Only unsigned integral types are allowed.
221///
222/// \param ZB the behavior on an input of 0. Only ZB_Width and ZB_Undefined are
223///   valid arguments.
224template <typename T>
225unsigned countLeadingZeros(T Val, ZeroBehavior ZB = ZB_Width) {
226  static_assert(std::numeric_limits<T>::is_integer &&
227                    !std::numeric_limits<T>::is_signed,
228                "Only unsigned integral types are allowed.");
229  return llvm::detail::LeadingZerosCounter<T, sizeof(T)>::count(Val, ZB);
230}
231 
232/// Get the index of the first set bit starting from the least
233///   significant bit.
234///
235/// Only unsigned integral types are allowed.
236///
237/// \param ZB the behavior on an input of 0. Only ZB_Max and ZB_Undefined are
238///   valid arguments.
239template <typename T> T findFirstSet(T Val, ZeroBehavior ZB = ZB_Max) {
240  if (ZB == ZB_Max && Val == 0)
241    return std::numeric_limits<T>::max();
242 
243  return countTrailingZeros(Val, ZB_Undefined);
244}
245 
246/// Create a bitmask with the N right-most bits set to 1, and all other
247/// bits set to 0.  Only unsigned types are allowed.
248template <typename T> T maskTrailingOnes(unsigned N) {
249  static_assert(std::is_unsigned<T>::value, "Invalid type!");
250  const unsigned Bits = CHAR_BIT8 * sizeof(T);
251  assert(N <= Bits && "Invalid bit index")(static_cast <bool> (N <= Bits && "Invalid bit index"
) ? void (0) : __assert_fail ("N <= Bits && \"Invalid bit index\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/Support/MathExtras.h"
, 251, __extension__ __PRETTY_FUNCTION__));
252  return N == 0 ? 0 : (T(-1) >> (Bits - N));
253}
254 
255/// Create a bitmask with the N left-most bits set to 1, and all other
256/// bits set to 0.  Only unsigned types are allowed.
257template <typename T> T maskLeadingOnes(unsigned N) {
258  return ~maskTrailingOnes<T>(CHAR_BIT8 * sizeof(T) - N);
259}
260 
261/// Create a bitmask with the N right-most bits set to 0, and all other
262/// bits set to 1.  Only unsigned types are allowed.
263template <typename T> T maskTrailingZeros(unsigned N) {
264  return maskLeadingOnes<T>(CHAR_BIT8 * sizeof(T) - N);
265}
266 
267/// Create a bitmask with the N left-most bits set to 0, and all other
268/// bits set to 1.  Only unsigned types are allowed.
269template <typename T> T maskLeadingZeros(unsigned N) {
270  return maskTrailingOnes<T>(CHAR_BIT8 * sizeof(T) - N);
271}
272 
273/// Get the index of the last set bit starting from the least
274///   significant bit.
275///
276/// Only unsigned integral types are allowed.
277///
278/// \param ZB the behavior on an input of 0. Only ZB_Max and ZB_Undefined are
279///   valid arguments.
280template <typename T> T findLastSet(T Val, ZeroBehavior ZB = ZB_Max) {
281  if (ZB == ZB_Max && Val == 0)
282    return std::numeric_limits<T>::max();
283 
284  // Use ^ instead of - because both gcc and llvm can remove the associated ^
285  // in the __builtin_clz intrinsic on x86.
286  return countLeadingZeros(Val, ZB_Undefined) ^
287         (std::numeric_limits<T>::digits - 1);
288}
289 
290/// Macro compressed bit reversal table for 256 bits.
291///
292/// http://graphics.stanford.edu/~seander/bithacks.html#BitReverseTable
293static const unsigned char BitReverseTable256[256] = {
294#define R2(n) n, n + 2 * 64, n + 1 * 64, n + 3 * 64
295#define R4(n) R2(n), R2(n + 2 * 16), R2(n + 1 * 16), R2(n + 3 * 16)
296#define R6(n) R4(n), R4(n + 2 * 4), R4(n + 1 * 4), R4(n + 3 * 4)
297  R6(0), R6(2), R6(1), R6(3)
298#undef R2
299#undef R4
300#undef R6
301};
302 
303/// Reverse the bits in \p Val.
304template <typename T>
305T reverseBits(T Val) {
306  unsigned char in[sizeof(Val)];
307  unsigned char out[sizeof(Val)];
308  std::memcpy(in, &Val, sizeof(Val));
309  for (unsigned i = 0; i < sizeof(Val); ++i)
310    out[(sizeof(Val) - i) - 1] = BitReverseTable256[in[i]];
311  std::memcpy(&Val, out, sizeof(Val));
312  return Val;
313}
314 
315#if __has_builtin(__builtin_bitreverse8)1
316template<>
317inline uint8_t reverseBits<uint8_t>(uint8_t Val) {
318  return __builtin_bitreverse8(Val);
319}
320#endif
321 
322#if __has_builtin(__builtin_bitreverse16)1
323template<>
324inline uint16_t reverseBits<uint16_t>(uint16_t Val) {
325  return __builtin_bitreverse16(Val);
326}
327#endif
328 
329#if __has_builtin(__builtin_bitreverse32)1
330template<>
331inline uint32_t reverseBits<uint32_t>(uint32_t Val) {
332  return __builtin_bitreverse32(Val);
333}
334#endif
335 
336#if __has_builtin(__builtin_bitreverse64)1
337template<>
338inline uint64_t reverseBits<uint64_t>(uint64_t Val) {
339  return __builtin_bitreverse64(Val);
340}
341#endif
342 
343// NOTE: The following support functions use the _32/_64 extensions instead of
344// type overloading so that signed and unsigned integers can be used without
345// ambiguity.
346 
347/// Return the high 32 bits of a 64 bit value.
348constexpr inline uint32_t Hi_32(uint64_t Value) {
349  return static_cast<uint32_t>(Value >> 32);
350}
351 
352/// Return the low 32 bits of a 64 bit value.
353constexpr inline uint32_t Lo_32(uint64_t Value) {
354  return static_cast<uint32_t>(Value);
355}
356 
357/// Make a 64-bit integer from a high / low pair of 32-bit integers.
358constexpr inline uint64_t Make_64(uint32_t High, uint32_t Low) {
359  return ((uint64_t)High << 32) | (uint64_t)Low;
360}
361 
362/// Checks if an integer fits into the given bit width.
363template <unsigned N> constexpr inline bool isInt(int64_t x) {
364  return N >= 64 || (-(INT64_C(1)1L<<(N-1)) <= x && x < (INT64_C(1)1L<<(N-1)));
365}
366// Template specializations to get better code for common cases.
367template <> constexpr inline bool isInt<8>(int64_t x) {
368  return static_cast<int8_t>(x) == x;
369}
370template <> constexpr inline bool isInt<16>(int64_t x) {
371  return static_cast<int16_t>(x) == x;
372}
373template <> constexpr inline bool isInt<32>(int64_t x) {
374  return static_cast<int32_t>(x) == x;
375}
376 
377/// Checks if a signed integer is an N bit number shifted left by S.
378template <unsigned N, unsigned S>
379constexpr inline bool isShiftedInt(int64_t x) {
380  static_assert(
381      N > 0, "isShiftedInt<0> doesn't make sense (refers to a 0-bit number.");
382  static_assert(N + S <= 64, "isShiftedInt<N, S> with N + S > 64 is too wide.");
383  return isInt<N + S>(x) && (x % (UINT64_C(1)1UL << S) == 0);
384}
385 
386/// Checks if an unsigned integer fits into the given bit width.
387///
388/// This is written as two functions rather than as simply
389///
390///   return N >= 64 || X < (UINT64_C(1) << N);
391///
392/// to keep MSVC from (incorrectly) warning on isUInt<64> that we're shifting
393/// left too many places.
394template <unsigned N>
395constexpr inline std::enable_if_t<(N < 64), bool> isUInt(uint64_t X) {
396  static_assert(N > 0, "isUInt<0> doesn't make sense");
397  return X < (UINT64_C(1)1UL << (N));
398}
399template <unsigned N>
400constexpr inline std::enable_if_t<N >= 64, bool> isUInt(uint64_t) {
401  return true;
402}
403 
404// Template specializations to get better code for common cases.
405template <> constexpr inline bool isUInt<8>(uint64_t x) {
406  return static_cast<uint8_t>(x) == x;
407}
408template <> constexpr inline bool isUInt<16>(uint64_t x) {
409  return static_cast<uint16_t>(x) == x;
410}
411template <> constexpr inline bool isUInt<32>(uint64_t x) {
412  return static_cast<uint32_t>(x) == x;
413}
414 
415/// Checks if a unsigned integer is an N bit number shifted left by S.
416template <unsigned N, unsigned S>
417constexpr inline bool isShiftedUInt(uint64_t x) {
418  static_assert(
419      N > 0, "isShiftedUInt<0> doesn't make sense (refers to a 0-bit number)");
420  static_assert(N + S <= 64,
421                "isShiftedUInt<N, S> with N + S > 64 is too wide.");
422  // Per the two static_asserts above, S must be strictly less than 64.  So
423  // 1 << S is not undefined behavior.
424  return isUInt<N + S>(x) && (x % (UINT64_C(1)1UL << S) == 0);
425}
426 
427/// Gets the maximum value for a N-bit unsigned integer.
428inline uint64_t maxUIntN(uint64_t N) {
429  assert(N > 0 && N <= 64 && "integer width out of range")(static_cast <bool> (N > 0 && N <= 64 &&
 "integer width out of range") ? void (0) : __assert_fail ("N > 0 && N <= 64 && \"integer width out of range\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/Support/MathExtras.h"
, 429, __extension__ __PRETTY_FUNCTION__));
430 
431  // uint64_t(1) << 64 is undefined behavior, so we can't do
432  //   (uint64_t(1) << N) - 1
433  // without checking first that N != 64.  But this works and doesn't have a
434  // branch.
435  return UINT64_MAX(18446744073709551615UL) >> (64 - N);
436}
437 
438/// Gets the minimum value for a N-bit signed integer.
439inline int64_t minIntN(int64_t N) {
440  assert(N > 0 && N <= 64 && "integer width out of range")(static_cast <bool> (N > 0 && N <= 64 &&
 "integer width out of range") ? void (0) : __assert_fail ("N > 0 && N <= 64 && \"integer width out of range\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/Support/MathExtras.h"
, 440, __extension__ __PRETTY_FUNCTION__));
441 
442  return UINT64_C(1)1UL + ~(UINT64_C(1)1UL << (N - 1));
443}
444 
445/// Gets the maximum value for a N-bit signed integer.
446inline int64_t maxIntN(int64_t N) {
447  assert(N > 0 && N <= 64 && "integer width out of range")(static_cast <bool> (N > 0 && N <= 64 &&
 "integer width out of range") ? void (0) : __assert_fail ("N > 0 && N <= 64 && \"integer width out of range\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/Support/MathExtras.h"
, 447, __extension__ __PRETTY_FUNCTION__));
448 
449  // This relies on two's complement wraparound when N == 64, so we convert to
450  // int64_t only at the very end to avoid UB.
451  return (UINT64_C(1)1UL << (N - 1)) - 1;
452}
453 
454/// Checks if an unsigned integer fits into the given (dynamic) bit width.
455inline bool isUIntN(unsigned N, uint64_t x) {
456  return N >= 64 || x <= maxUIntN(N);
457}
458 
459/// Checks if an signed integer fits into the given (dynamic) bit width.
460inline bool isIntN(unsigned N, int64_t x) {
461  return N >= 64 || (minIntN(N) <= x && x <= maxIntN(N));
462}
463 
464/// Return true if the argument is a non-empty sequence of ones starting at the
465/// least significant bit with the remainder zero (32 bit version).
466/// Ex. isMask_32(0x0000FFFFU) == true.
467constexpr inline bool isMask_32(uint32_t Value) {
468  return Value && ((Value + 1) & Value) == 0;
469}
470 
471/// Return true if the argument is a non-empty sequence of ones starting at the
472/// least significant bit with the remainder zero (64 bit version).
473constexpr inline bool isMask_64(uint64_t Value) {
474  return Value && ((Value + 1) & Value) == 0;
475}
476 
477/// Return true if the argument contains a non-empty sequence of ones with the
478/// remainder zero (32 bit version.) Ex. isShiftedMask_32(0x0000FF00U) == true.
479constexpr inline bool isShiftedMask_32(uint32_t Value) {
480  return Value && isMask_32((Value - 1) | Value);
481}
482 
483/// Return true if the argument contains a non-empty sequence of ones with the
484/// remainder zero (64 bit version.)
485constexpr inline bool isShiftedMask_64(uint64_t Value) {
486  return Value && isMask_64((Value - 1) | Value);
487}
488 
489/// Return true if the argument is a power of two > 0.
490/// Ex. isPowerOf2_32(0x00100000U) == true (32 bit edition.)
491constexpr inline bool isPowerOf2_32(uint32_t Value) {
492  return Value && !(Value & (Value - 1));
493}
494 
495/// Return true if the argument is a power of two > 0 (64 bit edition.)
496constexpr inline bool isPowerOf2_64(uint64_t Value) {
497  return Value && !(Value & (Value - 1));
498}
499 
500/// Count the number of ones from the most significant bit to the first
501/// zero bit.
502///
503/// Ex. countLeadingOnes(0xFF0FFF00) == 8.
504/// Only unsigned integral types are allowed.
505///
506/// \param ZB the behavior on an input of all ones. Only ZB_Width and
507/// ZB_Undefined are valid arguments.
508template <typename T>
509unsigned countLeadingOnes(T Value, ZeroBehavior ZB = ZB_Width) {
510  static_assert(std::numeric_limits<T>::is_integer &&
511                    !std::numeric_limits<T>::is_signed,
512                "Only unsigned integral types are allowed.");
513  return countLeadingZeros<T>(~Value, ZB);
514}
515 
516/// Count the number of ones from the least significant bit to the first
517/// zero bit.
518///
519/// Ex. countTrailingOnes(0x00FF00FF) == 8.
520/// Only unsigned integral types are allowed.
521///
522/// \param ZB the behavior on an input of all ones. Only ZB_Width and
523/// ZB_Undefined are valid arguments.
524template <typename T>
525unsigned countTrailingOnes(T Value, ZeroBehavior ZB = ZB_Width) {
526  static_assert(std::numeric_limits<T>::is_integer &&
527                    !std::numeric_limits<T>::is_signed,
528                "Only unsigned integral types are allowed.");
529  return countTrailingZeros<T>(~Value, ZB);
530}
531 
532namespace detail {
533template <typename T, std::size_t SizeOfT> struct PopulationCounter {
534  static unsigned count(T Value) {
535    // Generic version, forward to 32 bits.
536    static_assert(SizeOfT <= 4, "Not implemented!");
537#if defined(__GNUC__4)
538    return __builtin_popcount(Value);
539#else
540    uint32_t v = Value;
541    v = v - ((v >> 1) & 0x55555555);
542    v = (v & 0x33333333) + ((v >> 2) & 0x33333333);
543    return ((v + (v >> 4) & 0xF0F0F0F) * 0x1010101) >> 24;
544#endif
545  }
546};
547 
548template <typename T> struct PopulationCounter<T, 8> {
549  static unsigned count(T Value) {
550#if defined(__GNUC__4)
551    return __builtin_popcountll(Value);
552#else
553    uint64_t v = Value;
554    v = v - ((v >> 1) & 0x5555555555555555ULL);
555    v = (v & 0x3333333333333333ULL) + ((v >> 2) & 0x3333333333333333ULL);
556    v = (v + (v >> 4)) & 0x0F0F0F0F0F0F0F0FULL;
557    return unsigned((uint64_t)(v * 0x0101010101010101ULL) >> 56);
558#endif
559  }
560};
561} // namespace detail
562 
563/// Count the number of set bits in a value.
564/// Ex. countPopulation(0xF000F000) = 8
565/// Returns 0 if the word is zero.
566template <typename T>
567inline unsigned countPopulation(T Value) {
568  static_assert(std::numeric_limits<T>::is_integer &&
569                    !std::numeric_limits<T>::is_signed,
570                "Only unsigned integral types are allowed.");
571  return detail::PopulationCounter<T, sizeof(T)>::count(Value);
572}
573 
574/// Compile time Log2.
575/// Valid only for positive powers of two.
576template <size_t kValue> constexpr inline size_t CTLog2() {
577  static_assert(kValue > 0 && llvm::isPowerOf2_64(kValue),
578                "Value is not a valid power of 2");
579  return 1 + CTLog2<kValue / 2>();
580}
581 
582template <> constexpr inline size_t CTLog2<1>() { return 0; }
583 
584/// Return the log base 2 of the specified value.
585inline double Log2(double Value) {
586#if defined(__ANDROID_API__) && __ANDROID_API__ < 18
587  return __builtin_log(Value) / __builtin_log(2.0);
588#else
589  return log2(Value);
590#endif
591}
592 
593/// Return the floor log base 2 of the specified value, -1 if the value is zero.
594/// (32 bit edition.)
595/// Ex. Log2_32(32) == 5, Log2_32(1) == 0, Log2_32(0) == -1, Log2_32(6) == 2
596inline unsigned Log2_32(uint32_t Value) {
597  return 31 - countLeadingZeros(Value);
39
←
Returning the value 4294967295→
598}
599 
600/// Return the floor log base 2 of the specified value, -1 if the value is zero.
601/// (64 bit edition.)
602inline unsigned Log2_64(uint64_t Value) {
603  return 63 - countLeadingZeros(Value);
604}
605 
606/// Return the ceil log base 2 of the specified value, 32 if the value is zero.
607/// (32 bit edition).
608/// Ex. Log2_32_Ceil(32) == 5, Log2_32_Ceil(1) == 0, Log2_32_Ceil(6) == 3
609inline unsigned Log2_32_Ceil(uint32_t Value) {
610  return 32 - countLeadingZeros(Value - 1);
611}
612 
613/// Return the ceil log base 2 of the specified value, 64 if the value is zero.
614/// (64 bit edition.)
615inline unsigned Log2_64_Ceil(uint64_t Value) {
616  return 64 - countLeadingZeros(Value - 1);
617}
618 
619/// Return the greatest common divisor of the values using Euclid's algorithm.
620template <typename T>
621inline T greatestCommonDivisor(T A, T B) {
622  while (B) {
623    T Tmp = B;
624    B = A % B;
625    A = Tmp;
626  }
627  return A;
628}
629 
630inline uint64_t GreatestCommonDivisor64(uint64_t A, uint64_t B) {
631  return greatestCommonDivisor<uint64_t>(A, B);
632}
633 
634/// This function takes a 64-bit integer and returns the bit equivalent double.
635inline double BitsToDouble(uint64_t Bits) {
636  double D;
637  static_assert(sizeof(uint64_t) == sizeof(double), "Unexpected type sizes");
638  memcpy(&D, &Bits, sizeof(Bits));
639  return D;
640}
641 
642/// This function takes a 32-bit integer and returns the bit equivalent float.
643inline float BitsToFloat(uint32_t Bits) {
644  float F;
645  static_assert(sizeof(uint32_t) == sizeof(float), "Unexpected type sizes");
646  memcpy(&F, &Bits, sizeof(Bits));
647  return F;
648}
649 
650/// This function takes a double and returns the bit equivalent 64-bit integer.
651/// Note that copying doubles around changes the bits of NaNs on some hosts,
652/// notably x86, so this routine cannot be used if these bits are needed.
653inline uint64_t DoubleToBits(double Double) {
654  uint64_t Bits;
655  static_assert(sizeof(uint64_t) == sizeof(double), "Unexpected type sizes");
656  memcpy(&Bits, &Double, sizeof(Double));
657  return Bits;
658}
659 
660/// This function takes a float and returns the bit equivalent 32-bit integer.
661/// Note that copying floats around changes the bits of NaNs on some hosts,
662/// notably x86, so this routine cannot be used if these bits are needed.
663inline uint32_t FloatToBits(float Float) {
664  uint32_t Bits;
665  static_assert(sizeof(uint32_t) == sizeof(float), "Unexpected type sizes");
666  memcpy(&Bits, &Float, sizeof(Float));
667  return Bits;
668}
669 
670/// A and B are either alignments or offsets. Return the minimum alignment that
671/// may be assumed after adding the two together.
672constexpr inline uint64_t MinAlign(uint64_t A, uint64_t B) {
673  // The largest power of 2 that divides both A and B.
674  //
675  // Replace "-Value" by "1+~Value" in the following commented code to avoid
676  // MSVC warning C4146
677  //    return (A | B) & -(A | B);
678  return (A | B) & (1 + ~(A | B));
679}
680 
681/// Returns the next power of two (in 64-bits) that is strictly greater than A.
682/// Returns zero on overflow.
683inline uint64_t NextPowerOf2(uint64_t A) {
684  A |= (A >> 1);
685  A |= (A >> 2);
686  A |= (A >> 4);
687  A |= (A >> 8);
688  A |= (A >> 16);
689  A |= (A >> 32);
690  return A + 1;
691}
692 
693/// Returns the power of two which is less than or equal to the given value.
694/// Essentially, it is a floor operation across the domain of powers of two.
695inline uint64_t PowerOf2Floor(uint64_t A) {
696  if (!A) return 0;
697  return 1ull << (63 - countLeadingZeros(A, ZB_Undefined));
698}
699 
700/// Returns the power of two which is greater than or equal to the given value.
701/// Essentially, it is a ceil operation across the domain of powers of two.
702inline uint64_t PowerOf2Ceil(uint64_t A) {
703  if (!A)
704    return 0;
705  return NextPowerOf2(A - 1);
706}
707 
708/// Returns the next integer (mod 2**64) that is greater than or equal to
709/// \p Value and is a multiple of \p Align. \p Align must be non-zero.
710///
711/// If non-zero \p Skew is specified, the return value will be a minimal
712/// integer that is greater than or equal to \p Value and equal to
713/// \p Align * N + \p Skew for some integer N. If \p Skew is larger than
714/// \p Align, its value is adjusted to '\p Skew mod \p Align'.
715///
716/// Examples:
717/// \code
718///   alignTo(5, 8) = 8
719///   alignTo(17, 8) = 24
720///   alignTo(~0LL, 8) = 0
721///   alignTo(321, 255) = 510
722///
723///   alignTo(5, 8, 7) = 7
724///   alignTo(17, 8, 1) = 17
725///   alignTo(~0LL, 8, 3) = 3
726///   alignTo(321, 255, 42) = 552
727/// \endcode
728inline uint64_t alignTo(uint64_t Value, uint64_t Align, uint64_t Skew = 0) {
729  assert(Align != 0u && "Align can't be 0.")(static_cast <bool> (Align != 0u && "Align can't be 0."
) ? void (0) : __assert_fail ("Align != 0u && \"Align can't be 0.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/Support/MathExtras.h"
, 729, __extension__ __PRETTY_FUNCTION__));
730  Skew %= Align;
731  return (Value + Align - 1 - Skew) / Align * Align + Skew;
732}
733 
734/// Returns the next integer (mod 2**64) that is greater than or equal to
735/// \p Value and is a multiple of \c Align. \c Align must be non-zero.
736template <uint64_t Align> constexpr inline uint64_t alignTo(uint64_t Value) {
737  static_assert(Align != 0u, "Align must be non-zero");
738  return (Value + Align - 1) / Align * Align;
739}
740 
741/// Returns the integer ceil(Numerator / Denominator).
742inline uint64_t divideCeil(uint64_t Numerator, uint64_t Denominator) {
743  return alignTo(Numerator, Denominator) / Denominator;
744}
745 
746/// Returns the integer nearest(Numerator / Denominator).
747inline uint64_t divideNearest(uint64_t Numerator, uint64_t Denominator) {
748  return (Numerator + (Denominator / 2)) / Denominator;
749}
750 
751/// Returns the largest uint64_t less than or equal to \p Value and is
752/// \p Skew mod \p Align. \p Align must be non-zero
753inline uint64_t alignDown(uint64_t Value, uint64_t Align, uint64_t Skew = 0) {
754  assert(Align != 0u && "Align can't be 0.")(static_cast <bool> (Align != 0u && "Align can't be 0."
) ? void (0) : __assert_fail ("Align != 0u && \"Align can't be 0.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/Support/MathExtras.h"
, 754, __extension__ __PRETTY_FUNCTION__));
755  Skew %= Align;
756  return (Value - Skew) / Align * Align + Skew;
757}
758 
759/// Sign-extend the number in the bottom B bits of X to a 32-bit integer.
760/// Requires 0 < B <= 32.
761template <unsigned B> constexpr inline int32_t SignExtend32(uint32_t X) {
762  static_assert(B > 0, "Bit width can't be 0.");
763  static_assert(B <= 32, "Bit width out of range.");
764  return int32_t(X << (32 - B)) >> (32 - B);
765}
766 
767/// Sign-extend the number in the bottom B bits of X to a 32-bit integer.
768/// Requires 0 < B <= 32.
769inline int32_t SignExtend32(uint32_t X, unsigned B) {
770  assert(B > 0 && "Bit width can't be 0.")(static_cast <bool> (B > 0 && "Bit width can't be 0."
) ? void (0) : __assert_fail ("B > 0 && \"Bit width can't be 0.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/Support/MathExtras.h"
, 770, __extension__ __PRETTY_FUNCTION__));
771  assert(B <= 32 && "Bit width out of range.")(static_cast <bool> (B <= 32 && "Bit width out of range."
) ? void (0) : __assert_fail ("B <= 32 && \"Bit width out of range.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/Support/MathExtras.h"
, 771, __extension__ __PRETTY_FUNCTION__));
772  return int32_t(X << (32 - B)) >> (32 - B);
773}
774 
775/// Sign-extend the number in the bottom B bits of X to a 64-bit integer.
776/// Requires 0 < B <= 64.
777template <unsigned B> constexpr inline int64_t SignExtend64(uint64_t x) {
778  static_assert(B > 0, "Bit width can't be 0.");
779  static_assert(B <= 64, "Bit width out of range.");
780  return int64_t(x << (64 - B)) >> (64 - B);
781}
782 
783/// Sign-extend the number in the bottom B bits of X to a 64-bit integer.
784/// Requires 0 < B <= 64.
785inline int64_t SignExtend64(uint64_t X, unsigned B) {
786  assert(B > 0 && "Bit width can't be 0.")(static_cast <bool> (B > 0 && "Bit width can't be 0."
) ? void (0) : __assert_fail ("B > 0 && \"Bit width can't be 0.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/Support/MathExtras.h"
, 786, __extension__ __PRETTY_FUNCTION__));
787  assert(B <= 64 && "Bit width out of range.")(static_cast <bool> (B <= 64 && "Bit width out of range."
) ? void (0) : __assert_fail ("B <= 64 && \"Bit width out of range.\""
, "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/Support/MathExtras.h"
, 787, __extension__ __PRETTY_FUNCTION__));
788  return int64_t(X << (64 - B)) >> (64 - B);
789}
790 
791/// Subtract two unsigned integers, X and Y, of type T and return the absolute
792/// value of the result.
793template <typename T>
794std::enable_if_t<std::is_unsigned<T>::value, T> AbsoluteDifference(T X, T Y) {
795  return X > Y ? (X - Y) : (Y - X);
796}
797 
798/// Add two unsigned integers, X and Y, of type T.  Clamp the result to the
799/// maximum representable value of T on overflow.  ResultOverflowed indicates if
800/// the result is larger than the maximum representable value of type T.
801template <typename T>
802std::enable_if_t<std::is_unsigned<T>::value, T>
803SaturatingAdd(T X, T Y, bool *ResultOverflowed = nullptr) {
804  bool Dummy;
805  bool &Overflowed = ResultOverflowed ? *ResultOverflowed : Dummy;
806  // Hacker's Delight, p. 29
807  T Z = X + Y;
808  Overflowed = (Z < X || Z < Y);
809  if (Overflowed)
810    return std::numeric_limits<T>::max();
811  else
812    return Z;
813}
814 
815/// Multiply two unsigned integers, X and Y, of type T.  Clamp the result to the
816/// maximum representable value of T on overflow.  ResultOverflowed indicates if
817/// the result is larger than the maximum representable value of type T.
818template <typename T>
819std::enable_if_t<std::is_unsigned<T>::value, T>
820SaturatingMultiply(T X, T Y, bool *ResultOverflowed = nullptr) {
821  bool Dummy;
822  bool &Overflowed = ResultOverflowed ? *ResultOverflowed : Dummy;
823 
824  // Hacker's Delight, p. 30 has a different algorithm, but we don't use that
825  // because it fails for uint16_t (where multiplication can have undefined
826  // behavior due to promotion to int), and requires a division in addition
827  // to the multiplication.
828 
829  Overflowed = false;
830 
831  // Log2(Z) would be either Log2Z or Log2Z + 1.
832  // Special case: if X or Y is 0, Log2_64 gives -1, and Log2Z
833  // will necessarily be less than Log2Max as desired.
834  int Log2Z = Log2_64(X) + Log2_64(Y);
835  const T Max = std::numeric_limits<T>::max();
836  int Log2Max = Log2_64(Max);
837  if (Log2Z < Log2Max) {
838    return X * Y;
839  }
840  if (Log2Z > Log2Max) {
841    Overflowed = true;
842    return Max;
843  }
844 
845  // We're going to use the top bit, and maybe overflow one
846  // bit past it. Multiply all but the bottom bit then add
847  // that on at the end.
848  T Z = (X >> 1) * Y;
849  if (Z & ~(Max >> 1)) {
850    Overflowed = true;
851    return Max;
852  }
853  Z <<= 1;
854  if (X & 1)
855    return SaturatingAdd(Z, Y, ResultOverflowed);
856 
857  return Z;
858}
859 
860/// Multiply two unsigned integers, X and Y, and add the unsigned integer, A to
861/// the product. Clamp the result to the maximum representable value of T on
862/// overflow. ResultOverflowed indicates if the result is larger than the
863/// maximum representable value of type T.
864template <typename T>
865std::enable_if_t<std::is_unsigned<T>::value, T>
866SaturatingMultiplyAdd(T X, T Y, T A, bool *ResultOverflowed = nullptr) {
867  bool Dummy;
868  bool &Overflowed = ResultOverflowed ? *ResultOverflowed : Dummy;
869 
870  T Product = SaturatingMultiply(X, Y, &Overflowed);
871  if (Overflowed)
872    return Product;
873 
874  return SaturatingAdd(A, Product, &Overflowed);
875}
876 
877/// Use this rather than HUGE_VALF; the latter causes warnings on MSVC.
878extern const float huge_valf;
879 
880 
881/// Add two signed integers, computing the two's complement truncated result,
882/// returning true if overflow occured.
883template <typename T>
884std::enable_if_t<std::is_signed<T>::value, T> AddOverflow(T X, T Y, T &Result) {
885#if __has_builtin(__builtin_add_overflow)1
886  return __builtin_add_overflow(X, Y, &Result);
887#else
888  // Perform the unsigned addition.
889  using U = std::make_unsigned_t<T>;
890  const U UX = static_cast<U>(X);
891  const U UY = static_cast<U>(Y);
892  const U UResult = UX + UY;
893 
894  // Convert to signed.
895  Result = static_cast<T>(UResult);
896 
897  // Adding two positive numbers should result in a positive number.
898  if (X > 0 && Y > 0)
899    return Result <= 0;
900  // Adding two negatives should result in a negative number.
901  if (X < 0 && Y < 0)
902    return Result >= 0;
903  return false;
904#endif
905}
906 
907/// Subtract two signed integers, computing the two's complement truncated
908/// result, returning true if an overflow ocurred.
909template <typename T>
910std::enable_if_t<std::is_signed<T>::value, T> SubOverflow(T X, T Y, T &Result) {
911#if __has_builtin(__builtin_sub_overflow)1
912  return __builtin_sub_overflow(X, Y, &Result);
913#else
914  // Perform the unsigned addition.
915  using U = std::make_unsigned_t<T>;
916  const U UX = static_cast<U>(X);
917  const U UY = static_cast<U>(Y);
918  const U UResult = UX - UY;
919 
920  // Convert to signed.
921  Result = static_cast<T>(UResult);
922 
923  // Subtracting a positive number from a negative results in a negative number.
924  if (X <= 0 && Y > 0)
925    return Result >= 0;
926  // Subtracting a negative number from a positive results in a positive number.
927  if (X >= 0 && Y < 0)
928    return Result <= 0;
929  return false;
930#endif
931}
932 
933/// Multiply two signed integers, computing the two's complement truncated
934/// result, returning true if an overflow ocurred.
935template <typename T>
936std::enable_if_t<std::is_signed<T>::value, T> MulOverflow(T X, T Y, T &Result) {
937  // Perform the unsigned multiplication on absolute values.
938  using U = std::make_unsigned_t<T>;
939  const U UX = X < 0 ? (0 - static_cast<U>(X)) : static_cast<U>(X);
940  const U UY = Y < 0 ? (0 - static_cast<U>(Y)) : static_cast<U>(Y);
941  const U UResult = UX * UY;
942 
943  // Convert to signed.
944  const bool IsNegative = (X < 0) ^ (Y < 0);
945  Result = IsNegative ? (0 - UResult) : UResult;
946 
947  // If any of the args was 0, result is 0 and no overflow occurs.
948  if (UX == 0 || UY == 0)
949    return false;
950 
951  // UX and UY are in [1, 2^n], where n is the number of digits.
952  // Check how the max allowed absolute value (2^n for negative, 2^(n-1) for
953  // positive) divided by an argument compares to the other.
954  if (IsNegative)
955    return UX > (static_cast<U>(std::numeric_limits<T>::max()) + U(1)) / UY;
956  else
957    return UX > (static_cast<U>(std::numeric_limits<T>::max())) / UY;
958}
959 
960} // End llvm namespace
961 
962#endif