/build/source/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp

Bug Summary

File:	build/source/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
Warning:	line 6466, column 63 The result of the left shift is undefined due to shifting by '4294967295', which is greater or equal to the width of type 'int'

Annotated Source Code

Press '?' to see keyboard shortcuts

Show analyzer invocation

clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -clear-ast-before-backend -disable-llvm-verifier -discard-value-names -main-file-name AArch64InstructionSelector.cpp -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mframe-pointer=none -fmath-errno -ffp-contract=on -fno-rounding-math -mconstructor-aliases -funwind-tables=2 -target-cpu x86-64 -tune-cpu generic -debugger-tuning=gdb -ffunction-sections -fdata-sections -fcoverage-compilation-dir=/build/source/build-llvm/tools/clang/stage2-bins -resource-dir /usr/lib/llvm-17/lib/clang/17 -D _DEBUG -D _GLIBCXX_ASSERTIONS -D _GNU_SOURCE -D _LIBCPP_ENABLE_ASSERTIONS -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I lib/Target/AArch64 -I /build/source/llvm/lib/Target/AArch64 -I include -I /build/source/llvm/include -D _FORTIFY_SOURCE=2 -D NDEBUG -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/x86_64-linux-gnu/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10/backward -internal-isystem /usr/lib/llvm-17/lib/clang/17/include -internal-isystem /usr/local/include -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../x86_64-linux-gnu/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -fmacro-prefix-map=/build/source/build-llvm/tools/clang/stage2-bins=build-llvm/tools/clang/stage2-bins -fmacro-prefix-map=/build/source/= -fcoverage-prefix-map=/build/source/build-llvm/tools/clang/stage2-bins=build-llvm/tools/clang/stage2-bins -fcoverage-prefix-map=/build/source/= -source-date-epoch 1683717183 -O2 -Wno-unused-command-line-argument -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-class-memaccess -Wno-redundant-move -Wno-pessimizing-move -Wno-noexcept-type -Wno-comment -Wno-misleading-indentation -std=c++17 -fdeprecated-macro -fdebug-compilation-dir=/build/source/build-llvm/tools/clang/stage2-bins -fdebug-prefix-map=/build/source/build-llvm/tools/clang/stage2-bins=build-llvm/tools/clang/stage2-bins -fdebug-prefix-map=/build/source/= -ferror-limit 19 -fvisibility=hidden -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -fcolor-diagnostics -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -D__GCC_HAVE_DWARF2_CFI_ASM=1 -o /tmp/scan-build-2023-05-10-133810-16478-1 -x c++ /build/source/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp

/build/source/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp

→

1//===- AArch64InstructionSelector.cpp ----------------------------*- C++ -*-==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file implements the targeting of the InstructionSelector class for
10/// AArch64.
11/// \todo This should be generated by TableGen.
12//===----------------------------------------------------------------------===//

14#include "AArch64GlobalISelUtils.h"
15#include "AArch64InstrInfo.h"
16#include "AArch64MachineFunctionInfo.h"
17#include "AArch64RegisterBankInfo.h"
18#include "AArch64RegisterInfo.h"
19#include "AArch64Subtarget.h"
20#include "AArch64TargetMachine.h"
21#include "MCTargetDesc/AArch64AddressingModes.h"
22#include "MCTargetDesc/AArch64MCTargetDesc.h"
23#include "llvm/BinaryFormat/Dwarf.h"
24#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
25#include "llvm/CodeGen/GlobalISel/InstructionSelector.h"
26#include "llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h"
27#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
28#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
29#include "llvm/CodeGen/GlobalISel/Utils.h"
30#include "llvm/CodeGen/MachineBasicBlock.h"
31#include "llvm/CodeGen/MachineConstantPool.h"
32#include "llvm/CodeGen/MachineFrameInfo.h"
33#include "llvm/CodeGen/MachineFunction.h"
34#include "llvm/CodeGen/MachineInstr.h"
35#include "llvm/CodeGen/MachineInstrBuilder.h"
36#include "llvm/CodeGen/MachineMemOperand.h"
37#include "llvm/CodeGen/MachineOperand.h"
38#include "llvm/CodeGen/MachineRegisterInfo.h"
39#include "llvm/CodeGen/TargetOpcodes.h"
40#include "llvm/IR/Constants.h"
41#include "llvm/IR/DerivedTypes.h"
42#include "llvm/IR/Instructions.h"
43#include "llvm/IR/IntrinsicsAArch64.h"
44#include "llvm/IR/PatternMatch.h"
45#include "llvm/IR/Type.h"
46#include "llvm/Pass.h"
47#include "llvm/Support/Debug.h"
48#include "llvm/Support/raw_ostream.h"
49#include <optional>

51#define DEBUG_TYPE"aarch64-isel" "aarch64-isel"

53using namespace llvm;
54using namespace MIPatternMatch;
55using namespace AArch64GISelUtils;

57namespace llvm {
58class BlockFrequencyInfo;
59class ProfileSummaryInfo;
60}

62namespace {

64#define GET_GLOBALISEL_PREDICATE_BITSET
65#include "AArch64GenGlobalISel.inc"
66#undef GET_GLOBALISEL_PREDICATE_BITSET


69class AArch64InstructionSelector : public InstructionSelector {
70public:
AArch64InstructionSelector(const AArch64TargetMachine &TM,
                           const AArch64Subtarget &STI,
                           const AArch64RegisterBankInfo &RBI);

bool select(MachineInstr &I) override;
static const char *getName() { return DEBUG_TYPE"aarch64-isel"; }

void setupMF(MachineFunction &MF, GISelKnownBits *KB,
             CodeGenCoverage &CoverageInfo, ProfileSummaryInfo *PSI,
             BlockFrequencyInfo *BFI) override {
  InstructionSelector::setupMF(MF, KB, CoverageInfo, PSI, BFI);
  MIB.setMF(MF);

  // hasFnAttribute() is expensive to call on every BRCOND selection, so
  // cache it here for each run of the selector.
  ProduceNonFlagSettingCondBr =
      !MF.getFunction().hasFnAttribute(Attribute::SpeculativeLoadHardening);
  MFReturnAddr = Register();

  processPHIs(MF);
}

93private:
/// tblgen-erated 'select' implementation, used as the initial selector for
/// the patterns that don't require complex C++.
bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const;

// A lowering phase that runs before any selection attempts.
// Returns true if the instruction was modified.
bool preISelLower(MachineInstr &I);

// An early selection function that runs before the selectImpl() call.
bool earlySelect(MachineInstr &I);

// Do some preprocessing of G_PHIs before we begin selection.
void processPHIs(MachineFunction &MF);

bool earlySelectSHL(MachineInstr &I, MachineRegisterInfo &MRI);

/// Eliminate same-sized cross-bank copies into stores before selectImpl().
bool contractCrossBankCopyIntoStore(MachineInstr &I,
                                    MachineRegisterInfo &MRI);

bool convertPtrAddToAdd(MachineInstr &I, MachineRegisterInfo &MRI);

bool selectVaStartAAPCS(MachineInstr &I, MachineFunction &MF,
                        MachineRegisterInfo &MRI) const;
bool selectVaStartDarwin(MachineInstr &I, MachineFunction &MF,
                         MachineRegisterInfo &MRI) const;

///@{
/// Helper functions for selectCompareBranch.
bool selectCompareBranchFedByFCmp(MachineInstr &I, MachineInstr &FCmp,
                                  MachineIRBuilder &MIB) const;
bool selectCompareBranchFedByICmp(MachineInstr &I, MachineInstr &ICmp,
                                  MachineIRBuilder &MIB) const;
bool tryOptCompareBranchFedByICmp(MachineInstr &I, MachineInstr &ICmp,
                                  MachineIRBuilder &MIB) const;
bool tryOptAndIntoCompareBranch(MachineInstr &AndInst, bool Invert,
                                MachineBasicBlock *DstMBB,
                                MachineIRBuilder &MIB) const;
///@}

bool selectCompareBranch(MachineInstr &I, MachineFunction &MF,
                         MachineRegisterInfo &MRI);

bool selectVectorAshrLshr(MachineInstr &I, MachineRegisterInfo &MRI);
bool selectVectorSHL(MachineInstr &I, MachineRegisterInfo &MRI);

// Helper to generate an equivalent of scalar_to_vector into a new register,
// returned via 'Dst'.
MachineInstr *emitScalarToVector(unsigned EltSize,
                                 const TargetRegisterClass *DstRC,
                                 Register Scalar,
                                 MachineIRBuilder &MIRBuilder) const;

/// Emit a lane insert into \p DstReg, or a new vector register if
/// std::nullopt is provided.
///
/// The lane inserted into is defined by \p LaneIdx. The vector source
/// register is given by \p SrcReg. The register containing the element is
/// given by \p EltReg.
MachineInstr *emitLaneInsert(std::optional<Register> DstReg, Register SrcReg,
                             Register EltReg, unsigned LaneIdx,
                             const RegisterBank &RB,
                             MachineIRBuilder &MIRBuilder) const;

/// Emit a sequence of instructions representing a constant \p CV for a
/// vector register \p Dst. (E.g. a MOV, or a load from a constant pool.)
///
/// \returns the last instruction in the sequence on success, and nullptr
/// otherwise.
MachineInstr *emitConstantVector(Register Dst, Constant *CV,
                                 MachineIRBuilder &MIRBuilder,
                                 MachineRegisterInfo &MRI);

bool selectInsertElt(MachineInstr &I, MachineRegisterInfo &MRI);
bool tryOptConstantBuildVec(MachineInstr &MI, LLT DstTy,
                            MachineRegisterInfo &MRI);
/// \returns true if a G_BUILD_VECTOR instruction \p MI can be selected as a
/// SUBREG_TO_REG.
bool tryOptBuildVecToSubregToReg(MachineInstr &MI, MachineRegisterInfo &MRI);
bool selectBuildVector(MachineInstr &I, MachineRegisterInfo &MRI);
bool selectMergeValues(MachineInstr &I, MachineRegisterInfo &MRI);
bool selectUnmergeValues(MachineInstr &I, MachineRegisterInfo &MRI);

bool selectShuffleVector(MachineInstr &I, MachineRegisterInfo &MRI);
bool selectExtractElt(MachineInstr &I, MachineRegisterInfo &MRI);
bool selectConcatVectors(MachineInstr &I, MachineRegisterInfo &MRI);
bool selectSplitVectorUnmerge(MachineInstr &I, MachineRegisterInfo &MRI);

/// Helper function to select vector load intrinsics like
/// @llvm.aarch64.neon.ld2.*, @llvm.aarch64.neon.ld4.*, etc.
/// \p Opc is the opcode that the selected instruction should use.
/// \p NumVecs is the number of vector destinations for the instruction.
/// \p I is the original G_INTRINSIC_W_SIDE_EFFECTS instruction.
bool selectVectorLoadIntrinsic(unsigned Opc, unsigned NumVecs,
                               MachineInstr &I);
bool selectIntrinsicWithSideEffects(MachineInstr &I,
                                    MachineRegisterInfo &MRI);
bool selectIntrinsic(MachineInstr &I, MachineRegisterInfo &MRI);
bool selectVectorICmp(MachineInstr &I, MachineRegisterInfo &MRI);
bool selectIntrinsicTrunc(MachineInstr &I, MachineRegisterInfo &MRI) const;
bool selectIntrinsicRound(MachineInstr &I, MachineRegisterInfo &MRI) const;
bool selectJumpTable(MachineInstr &I, MachineRegisterInfo &MRI);
bool selectBrJT(MachineInstr &I, MachineRegisterInfo &MRI);
bool selectTLSGlobalValue(MachineInstr &I, MachineRegisterInfo &MRI);
bool selectReduction(MachineInstr &I, MachineRegisterInfo &MRI);
bool selectMOPS(MachineInstr &I, MachineRegisterInfo &MRI);
bool selectUSMovFromExtend(MachineInstr &I, MachineRegisterInfo &MRI);

unsigned emitConstantPoolEntry(const Constant *CPVal,
                               MachineFunction &MF) const;
MachineInstr *emitLoadFromConstantPool(const Constant *CPVal,
                                       MachineIRBuilder &MIRBuilder) const;

// Emit a vector concat operation.
MachineInstr *emitVectorConcat(std::optional<Register> Dst, Register Op1,
                               Register Op2,
                               MachineIRBuilder &MIRBuilder) const;

// Emit an integer compare between LHS and RHS, which checks for Predicate.
MachineInstr *emitIntegerCompare(MachineOperand &LHS, MachineOperand &RHS,
                                 MachineOperand &Predicate,
                                 MachineIRBuilder &MIRBuilder) const;

/// Emit a floating point comparison between \p LHS and \p RHS.
/// \p Pred if given is the intended predicate to use.
MachineInstr *
emitFPCompare(Register LHS, Register RHS, MachineIRBuilder &MIRBuilder,
              std::optional<CmpInst::Predicate> = std::nullopt) const;

MachineInstr *
emitInstr(unsigned Opcode, std::initializer_list<llvm::DstOp> DstOps,
          std::initializer_list<llvm::SrcOp> SrcOps,
          MachineIRBuilder &MIRBuilder,
          const ComplexRendererFns &RenderFns = std::nullopt) const;
/// Helper function to emit an add or sub instruction.
///
/// \p AddrModeAndSizeToOpcode must contain each of the opcode variants above
/// in a specific order.
///
/// Below is an example of the expected input to \p AddrModeAndSizeToOpcode.
///
/// \code
///   const std::array<std::array<unsigned, 2>, 4> Table {
///    {{AArch64::ADDXri, AArch64::ADDWri},
///     {AArch64::ADDXrs, AArch64::ADDWrs},
///     {AArch64::ADDXrr, AArch64::ADDWrr},
///     {AArch64::SUBXri, AArch64::SUBWri},
///     {AArch64::ADDXrx, AArch64::ADDWrx}}};
/// \endcode
///
/// Each row in the table corresponds to a different addressing mode. Each
/// column corresponds to a different register size.
///
/// \attention Rows must be structured as follows:
///   - Row 0: The ri opcode variants
///   - Row 1: The rs opcode variants
///   - Row 2: The rr opcode variants
///   - Row 3: The ri opcode variants for negative immediates
///   - Row 4: The rx opcode variants
///
/// \attention Columns must be structured as follows:
///   - Column 0: The 64-bit opcode variants
///   - Column 1: The 32-bit opcode variants
///
/// \p Dst is the destination register of the binop to emit.
/// \p LHS is the left-hand operand of the binop to emit.
/// \p RHS is the right-hand operand of the binop to emit.
MachineInstr *emitAddSub(
    const std::array<std::array<unsigned, 2>, 5> &AddrModeAndSizeToOpcode,
    Register Dst, MachineOperand &LHS, MachineOperand &RHS,
    MachineIRBuilder &MIRBuilder) const;
MachineInstr *emitADD(Register DefReg, MachineOperand &LHS,
                      MachineOperand &RHS,
                      MachineIRBuilder &MIRBuilder) const;
MachineInstr *emitADDS(Register Dst, MachineOperand &LHS, MachineOperand &RHS,
                       MachineIRBuilder &MIRBuilder) const;
MachineInstr *emitSUBS(Register Dst, MachineOperand &LHS, MachineOperand &RHS,
                       MachineIRBuilder &MIRBuilder) const;
MachineInstr *emitCMN(MachineOperand &LHS, MachineOperand &RHS,
                      MachineIRBuilder &MIRBuilder) const;
MachineInstr *emitTST(MachineOperand &LHS, MachineOperand &RHS,
                      MachineIRBuilder &MIRBuilder) const;
MachineInstr *emitSelect(Register Dst, Register LHS, Register RHS,
                         AArch64CC::CondCode CC,
                         MachineIRBuilder &MIRBuilder) const;
MachineInstr *emitExtractVectorElt(std::optional<Register> DstReg,
                                   const RegisterBank &DstRB, LLT ScalarTy,
                                   Register VecReg, unsigned LaneIdx,
                                   MachineIRBuilder &MIRBuilder) const;
MachineInstr *emitCSINC(Register Dst, Register Src1, Register Src2,
                        AArch64CC::CondCode Pred,
                        MachineIRBuilder &MIRBuilder) const;
/// Emit a CSet for a FP compare.
///
/// \p Dst is expected to be a 32-bit scalar register.
MachineInstr *emitCSetForFCmp(Register Dst, CmpInst::Predicate Pred,
                              MachineIRBuilder &MIRBuilder) const;

/// Emit the overflow op for \p Opcode.
///
/// \p Opcode is expected to be an overflow op's opcode, e.g. G_UADDO,
/// G_USUBO, etc.
std::pair<MachineInstr *, AArch64CC::CondCode>
emitOverflowOp(unsigned Opcode, Register Dst, MachineOperand &LHS,
               MachineOperand &RHS, MachineIRBuilder &MIRBuilder) const;

/// Emit expression as a conjunction (a series of CCMP/CFCMP ops).
/// In some cases this is even possible with OR operations in the expression.
MachineInstr *emitConjunction(Register Val, AArch64CC::CondCode &OutCC,
                              MachineIRBuilder &MIB) const;
MachineInstr *emitConditionalComparison(Register LHS, Register RHS,
                                        CmpInst::Predicate CC,
                                        AArch64CC::CondCode Predicate,
                                        AArch64CC::CondCode OutCC,
                                        MachineIRBuilder &MIB) const;
MachineInstr *emitConjunctionRec(Register Val, AArch64CC::CondCode &OutCC,
                                 bool Negate, Register CCOp,
                                 AArch64CC::CondCode Predicate,
                                 MachineIRBuilder &MIB) const;

/// Emit a TB(N)Z instruction which tests \p Bit in \p TestReg.
/// \p IsNegative is true if the test should be "not zero".
/// This will also optimize the test bit instruction when possible.
MachineInstr *emitTestBit(Register TestReg, uint64_t Bit, bool IsNegative,
                          MachineBasicBlock *DstMBB,
                          MachineIRBuilder &MIB) const;

/// Emit a CB(N)Z instruction which branches to \p DestMBB.
MachineInstr *emitCBZ(Register CompareReg, bool IsNegative,
                      MachineBasicBlock *DestMBB,
                      MachineIRBuilder &MIB) const;

// Equivalent to the i32shift_a and friends from AArch64InstrInfo.td.
// We use these manually instead of using the importer since it doesn't
// support SDNodeXForm.
ComplexRendererFns selectShiftA_32(const MachineOperand &Root) const;
ComplexRendererFns selectShiftB_32(const MachineOperand &Root) const;
ComplexRendererFns selectShiftA_64(const MachineOperand &Root) const;
ComplexRendererFns selectShiftB_64(const MachineOperand &Root) const;

ComplexRendererFns select12BitValueWithLeftShift(uint64_t Immed) const;
ComplexRendererFns selectArithImmed(MachineOperand &Root) const;
ComplexRendererFns selectNegArithImmed(MachineOperand &Root) const;

ComplexRendererFns selectAddrModeUnscaled(MachineOperand &Root,
                                          unsigned Size) const;

ComplexRendererFns selectAddrModeUnscaled8(MachineOperand &Root) const {
  return selectAddrModeUnscaled(Root, 1);
}
ComplexRendererFns selectAddrModeUnscaled16(MachineOperand &Root) const {
  return selectAddrModeUnscaled(Root, 2);
}
ComplexRendererFns selectAddrModeUnscaled32(MachineOperand &Root) const {
  return selectAddrModeUnscaled(Root, 4);
}
ComplexRendererFns selectAddrModeUnscaled64(MachineOperand &Root) const {
  return selectAddrModeUnscaled(Root, 8);
}
ComplexRendererFns selectAddrModeUnscaled128(MachineOperand &Root) const {
  return selectAddrModeUnscaled(Root, 16);
}

/// Helper to try to fold in a GISEL_ADD_LOW into an immediate, to be used
/// from complex pattern matchers like selectAddrModeIndexed().
ComplexRendererFns tryFoldAddLowIntoImm(MachineInstr &RootDef, unsigned Size,
                                        MachineRegisterInfo &MRI) const;

ComplexRendererFns selectAddrModeIndexed(MachineOperand &Root,
                                         unsigned Size) const;
template <int Width>
ComplexRendererFns selectAddrModeIndexed(MachineOperand &Root) const {
  return selectAddrModeIndexed(Root, Width / 8);
}

bool isWorthFoldingIntoExtendedReg(MachineInstr &MI,
                                   const MachineRegisterInfo &MRI) const;
ComplexRendererFns
selectAddrModeShiftedExtendXReg(MachineOperand &Root,
                                unsigned SizeInBytes) const;

/// Returns a \p ComplexRendererFns which contains a base, offset, and whether
/// or not a shift + extend should be folded into an addressing mode. Returns
/// None when this is not profitable or possible.
ComplexRendererFns
selectExtendedSHL(MachineOperand &Root, MachineOperand &Base,
                  MachineOperand &Offset, unsigned SizeInBytes,
                  bool WantsExt) const;
ComplexRendererFns selectAddrModeRegisterOffset(MachineOperand &Root) const;
ComplexRendererFns selectAddrModeXRO(MachineOperand &Root,
                                     unsigned SizeInBytes) const;
template <int Width>
ComplexRendererFns selectAddrModeXRO(MachineOperand &Root) const {
  return selectAddrModeXRO(Root, Width / 8);
}

ComplexRendererFns selectAddrModeWRO(MachineOperand &Root,
                                     unsigned SizeInBytes) const;
template <int Width>
ComplexRendererFns selectAddrModeWRO(MachineOperand &Root) const {
  return selectAddrModeWRO(Root, Width / 8);
}

ComplexRendererFns selectShiftedRegister(MachineOperand &Root,
                                         bool AllowROR = false) const;

ComplexRendererFns selectArithShiftedRegister(MachineOperand &Root) const {
  return selectShiftedRegister(Root);
}

ComplexRendererFns selectLogicalShiftedRegister(MachineOperand &Root) const {
  return selectShiftedRegister(Root, true);
}

/// Given an extend instruction, determine the correct shift-extend type for
/// that instruction.
///
/// If the instruction is going to be used in a load or store, pass
/// \p IsLoadStore = true.
AArch64_AM::ShiftExtendType
getExtendTypeForInst(MachineInstr &MI, MachineRegisterInfo &MRI,
                     bool IsLoadStore = false) const;

/// Move \p Reg to \p RC if \p Reg is not already on \p RC.
///
/// \returns Either \p Reg if no change was necessary, or the new register
/// created by moving \p Reg.
///
/// Note: This uses emitCopy right now.
Register moveScalarRegClass(Register Reg, const TargetRegisterClass &RC,
                            MachineIRBuilder &MIB) const;

ComplexRendererFns selectArithExtendedRegister(MachineOperand &Root) const;

void renderTruncImm(MachineInstrBuilder &MIB, const MachineInstr &MI,
                    int OpIdx = -1) const;
void renderLogicalImm32(MachineInstrBuilder &MIB, const MachineInstr &I,
                        int OpIdx = -1) const;
void renderLogicalImm64(MachineInstrBuilder &MIB, const MachineInstr &I,
                        int OpIdx = -1) const;
void renderFPImm16(MachineInstrBuilder &MIB, const MachineInstr &MI,
                   int OpIdx = -1) const;
void renderFPImm32(MachineInstrBuilder &MIB, const MachineInstr &MI,
                   int OpIdx = -1) const;
void renderFPImm64(MachineInstrBuilder &MIB, const MachineInstr &MI,
                   int OpIdx = -1) const;
void renderFPImm32SIMDModImmType4(MachineInstrBuilder &MIB,
                                  const MachineInstr &MI,
                                  int OpIdx = -1) const;

// Materialize a GlobalValue or BlockAddress using a movz+movk sequence.
void materializeLargeCMVal(MachineInstr &I, const Value *V, unsigned OpFlags);

// Optimization methods.
bool tryOptSelect(GSelect &Sel);
bool tryOptSelectConjunction(GSelect &Sel, MachineInstr &CondMI);
MachineInstr *tryFoldIntegerCompare(MachineOperand &LHS, MachineOperand &RHS,
                                    MachineOperand &Predicate,
                                    MachineIRBuilder &MIRBuilder) const;

/// Return true if \p MI is a load or store of \p NumBytes bytes.
bool isLoadStoreOfNumBytes(const MachineInstr &MI, unsigned NumBytes) const;

/// Returns true if \p MI is guaranteed to have the high-half of a 64-bit
/// register zeroed out. In other words, the result of MI has been explicitly
/// zero extended.
bool isDef32(const MachineInstr &MI) const;

const AArch64TargetMachine &TM;
const AArch64Subtarget &STI;
const AArch64InstrInfo &TII;
const AArch64RegisterInfo &TRI;
const AArch64RegisterBankInfo &RBI;

bool ProduceNonFlagSettingCondBr = false;

// Some cached values used during selection.
// We use LR as a live-in register, and we keep track of it here as it can be
// clobbered by calls.
Register MFReturnAddr;

MachineIRBuilder MIB;

477#define GET_GLOBALISEL_PREDICATES_DECL
478#include "AArch64GenGlobalISel.inc"
479#undef GET_GLOBALISEL_PREDICATES_DECL

481// We declare the temporaries used by selectImpl() in the class to minimize the
482// cost of constructing placeholder values.
483#define GET_GLOBALISEL_TEMPORARIES_DECL
484#include "AArch64GenGlobalISel.inc"
485#undef GET_GLOBALISEL_TEMPORARIES_DECL
486};

488} // end anonymous namespace

490#define GET_GLOBALISEL_IMPL
491#include "AArch64GenGlobalISel.inc"
492#undef GET_GLOBALISEL_IMPL

494AArch64InstructionSelector::AArch64InstructionSelector(
  const AArch64TargetMachine &TM, const AArch64Subtarget &STI,
  const AArch64RegisterBankInfo &RBI)
  : TM(TM), STI(STI), TII(*STI.getInstrInfo()), TRI(*STI.getRegisterInfo()),
    RBI(RBI),
499#define GET_GLOBALISEL_PREDICATES_INIT
500#include "AArch64GenGlobalISel.inc"
501#undef GET_GLOBALISEL_PREDICATES_INIT
502#define GET_GLOBALISEL_TEMPORARIES_INIT
503#include "AArch64GenGlobalISel.inc"
504#undef GET_GLOBALISEL_TEMPORARIES_INIT
505{
506}

508// FIXME: This should be target-independent, inferred from the types declared
509// for each class in the bank.
510//
511/// Given a register bank, and a type, return the smallest register class that
512/// can represent that combination.
513static const TargetRegisterClass *
514getRegClassForTypeOnBank(LLT Ty, const RegisterBank &RB,
                       bool GetAllRegSet = false) {
if (RB.getID() == AArch64::GPRRegBankID) {
  if (Ty.getSizeInBits() <= 32)
    return GetAllRegSet ? &AArch64::GPR32allRegClass
                        : &AArch64::GPR32RegClass;
  if (Ty.getSizeInBits() == 64)
    return GetAllRegSet ? &AArch64::GPR64allRegClass
                        : &AArch64::GPR64RegClass;
  if (Ty.getSizeInBits() == 128)
    return &AArch64::XSeqPairsClassRegClass;
  return nullptr;
}

if (RB.getID() == AArch64::FPRRegBankID) {
  switch (Ty.getSizeInBits()) {
  case 8:
    return &AArch64::FPR8RegClass;
  case 16:
    return &AArch64::FPR16RegClass;
  case 32:
    return &AArch64::FPR32RegClass;
  case 64:
    return &AArch64::FPR64RegClass;
  case 128:
    return &AArch64::FPR128RegClass;
  }
  return nullptr;
}

return nullptr;
545}

547/// Given a register bank, and size in bits, return the smallest register class
548/// that can represent that combination.
549static const TargetRegisterClass *
550getMinClassForRegBank(const RegisterBank &RB, unsigned SizeInBits,
                    bool GetAllRegSet = false) {
unsigned RegBankID = RB.getID();

if (RegBankID == AArch64::GPRRegBankID) {
  if (SizeInBits <= 32)
    return GetAllRegSet ? &AArch64::GPR32allRegClass
                        : &AArch64::GPR32RegClass;
  if (SizeInBits == 64)
    return GetAllRegSet ? &AArch64::GPR64allRegClass
                        : &AArch64::GPR64RegClass;
  if (SizeInBits == 128)
    return &AArch64::XSeqPairsClassRegClass;
}

if (RegBankID == AArch64::FPRRegBankID) {
  switch (SizeInBits) {
  default:
    return nullptr;
  case 8:
    return &AArch64::FPR8RegClass;
  case 16:
    return &AArch64::FPR16RegClass;
  case 32:
    return &AArch64::FPR32RegClass;
  case 64:
    return &AArch64::FPR64RegClass;
  case 128:
    return &AArch64::FPR128RegClass;
  }
}

return nullptr;
583}

585/// Returns the correct subregister to use for a given register class.
586static bool getSubRegForClass(const TargetRegisterClass *RC,
                            const TargetRegisterInfo &TRI, unsigned &SubReg) {
switch (TRI.getRegSizeInBits(*RC)) {
case 8:
  SubReg = AArch64::bsub;
  break;
case 16:
  SubReg = AArch64::hsub;
  break;
case 32:
  if (RC != &AArch64::FPR32RegClass)
    SubReg = AArch64::sub_32;
  else
    SubReg = AArch64::ssub;
  break;
case 64:
  SubReg = AArch64::dsub;
  break;
default:
  LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Couldn't find appropriate subregister for register class."
; } } while (false)
      dbgs() << "Couldn't find appropriate subregister for register class.")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Couldn't find appropriate subregister for register class."
; } } while (false);
  return false;
}

return true;
611}

613/// Returns the minimum size the given register bank can hold.
614static unsigned getMinSizeForRegBank(const RegisterBank &RB) {
switch (RB.getID()) {
case AArch64::GPRRegBankID:
  return 32;
case AArch64::FPRRegBankID:
  return 8;
default:
  llvm_unreachable("Tried to get minimum size for unknown register bank.")::llvm::llvm_unreachable_internal("Tried to get minimum size for unknown register bank."
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 621);
}
623}

625/// Create a REG_SEQUENCE instruction using the registers in \p Regs.
626/// Helper function for functions like createDTuple and createQTuple.
627///
628/// \p RegClassIDs - The list of register class IDs available for some tuple of
629/// a scalar class. E.g. QQRegClassID, QQQRegClassID, QQQQRegClassID. This is
630/// expected to contain between 2 and 4 tuple classes.
631///
632/// \p SubRegs - The list of subregister classes associated with each register
633/// class ID in \p RegClassIDs. E.g., QQRegClassID should use the qsub0
634/// subregister class. The index of each subregister class is expected to
635/// correspond with the index of each register class.
636///
637/// \returns Either the destination register of REG_SEQUENCE instruction that
638/// was created, or the 0th element of \p Regs if \p Regs contains a single
639/// element.
640static Register createTuple(ArrayRef<Register> Regs,
                          const unsigned RegClassIDs[],
                          const unsigned SubRegs[], MachineIRBuilder &MIB) {
unsigned NumRegs = Regs.size();
if (NumRegs == 1)
  return Regs[0];
assert(NumRegs >= 2 && NumRegs <= 4 &&(static_cast <bool> (NumRegs >= 2 && NumRegs
 <= 4 && "Only support between two and 4 registers in a tuple!"
) ? void (0) : __assert_fail ("NumRegs >= 2 && NumRegs <= 4 && \"Only support between two and 4 registers in a tuple!\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 647, __extension__ __PRETTY_FUNCTION__))
       "Only support between two and 4 registers in a tuple!")(static_cast <bool> (NumRegs >= 2 && NumRegs
 <= 4 && "Only support between two and 4 registers in a tuple!"
) ? void (0) : __assert_fail ("NumRegs >= 2 && NumRegs <= 4 && \"Only support between two and 4 registers in a tuple!\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 647, __extension__ __PRETTY_FUNCTION__));
const TargetRegisterInfo *TRI = MIB.getMF().getSubtarget().getRegisterInfo();
auto *DesiredClass = TRI->getRegClass(RegClassIDs[NumRegs - 2]);
auto RegSequence =
    MIB.buildInstr(TargetOpcode::REG_SEQUENCE, {DesiredClass}, {});
for (unsigned I = 0, E = Regs.size(); I < E; ++I) {
  RegSequence.addUse(Regs[I]);
  RegSequence.addImm(SubRegs[I]);
}
return RegSequence.getReg(0);
657}

659/// Create a tuple of D-registers using the registers in \p Regs.
660static Register createDTuple(ArrayRef<Register> Regs, MachineIRBuilder &MIB) {
static const unsigned RegClassIDs[] = {
    AArch64::DDRegClassID, AArch64::DDDRegClassID, AArch64::DDDDRegClassID};
static const unsigned SubRegs[] = {AArch64::dsub0, AArch64::dsub1,
                                   AArch64::dsub2, AArch64::dsub3};
return createTuple(Regs, RegClassIDs, SubRegs, MIB);
666}

668/// Create a tuple of Q-registers using the registers in \p Regs.
669static Register createQTuple(ArrayRef<Register> Regs, MachineIRBuilder &MIB) {
static const unsigned RegClassIDs[] = {
    AArch64::QQRegClassID, AArch64::QQQRegClassID, AArch64::QQQQRegClassID};
static const unsigned SubRegs[] = {AArch64::qsub0, AArch64::qsub1,
                                   AArch64::qsub2, AArch64::qsub3};
return createTuple(Regs, RegClassIDs, SubRegs, MIB);
675}

677static std::optional<uint64_t> getImmedFromMO(const MachineOperand &Root) {
auto &MI = *Root.getParent();
auto &MBB = *MI.getParent();
auto &MF = *MBB.getParent();
auto &MRI = MF.getRegInfo();
uint64_t Immed;
if (Root.isImm())
  Immed = Root.getImm();
else if (Root.isCImm())
  Immed = Root.getCImm()->getZExtValue();
else if (Root.isReg()) {
  auto ValAndVReg =
      getIConstantVRegValWithLookThrough(Root.getReg(), MRI, true);
  if (!ValAndVReg)
    return std::nullopt;
  Immed = ValAndVReg->Value.getSExtValue();
} else
  return std::nullopt;
return Immed;
696}

698/// Check whether \p I is a currently unsupported binary operation:
699/// - it has an unsized type
700/// - an operand is not a vreg
701/// - all operands are not in the same bank
702/// These are checks that should someday live in the verifier, but right now,
703/// these are mostly limitations of the aarch64 selector.
704static bool unsupportedBinOp(const MachineInstr &I,
                           const AArch64RegisterBankInfo &RBI,
                           const MachineRegisterInfo &MRI,
                           const AArch64RegisterInfo &TRI) {
LLT Ty = MRI.getType(I.getOperand(0).getReg());
if (!Ty.isValid()) {
  LLVM_DEBUG(dbgs() << "Generic binop register should be typed\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Generic binop register should be typed\n"
; } } while (false);
  return true;
}

const RegisterBank *PrevOpBank = nullptr;
for (auto &MO : I.operands()) {
  // FIXME: Support non-register operands.
  if (!MO.isReg()) {
    LLVM_DEBUG(dbgs() << "Generic inst non-reg operands are unsupported\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Generic inst non-reg operands are unsupported\n"
; } } while (false);
    return true;
  }

  // FIXME: Can generic operations have physical registers operands? If
  // so, this will need to be taught about that, and we'll need to get the
  // bank out of the minimal class for the register.
  // Either way, this needs to be documented (and possibly verified).
  if (!MO.getReg().isVirtual()) {
    LLVM_DEBUG(dbgs() << "Generic inst has physical register operand\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Generic inst has physical register operand\n"
; } } while (false);
    return true;
  }

  const RegisterBank *OpBank = RBI.getRegBank(MO.getReg(), MRI, TRI);
  if (!OpBank) {
    LLVM_DEBUG(dbgs() << "Generic register has no bank or class\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Generic register has no bank or class\n"
; } } while (false);
    return true;
  }

  if (PrevOpBank && OpBank != PrevOpBank) {
    LLVM_DEBUG(dbgs() << "Generic inst operands have different banks\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Generic inst operands have different banks\n"
; } } while (false);
    return true;
  }
  PrevOpBank = OpBank;
}
return false;
744}

746/// Select the AArch64 opcode for the basic binary operation \p GenericOpc
747/// (such as G_OR or G_SDIV), appropriate for the register bank \p RegBankID
748/// and of size \p OpSize.
749/// \returns \p GenericOpc if the combination is unsupported.
750static unsigned selectBinaryOp(unsigned GenericOpc, unsigned RegBankID,
                             unsigned OpSize) {
switch (RegBankID) {
case AArch64::GPRRegBankID:
  if (OpSize == 32) {
    switch (GenericOpc) {
    case TargetOpcode::G_SHL:
      return AArch64::LSLVWr;
    case TargetOpcode::G_LSHR:
      return AArch64::LSRVWr;
    case TargetOpcode::G_ASHR:
      return AArch64::ASRVWr;
    default:
      return GenericOpc;
    }
  } else if (OpSize == 64) {
    switch (GenericOpc) {
    case TargetOpcode::G_PTR_ADD:
      return AArch64::ADDXrr;
    case TargetOpcode::G_SHL:
      return AArch64::LSLVXr;
    case TargetOpcode::G_LSHR:
      return AArch64::LSRVXr;
    case TargetOpcode::G_ASHR:
      return AArch64::ASRVXr;
    default:
      return GenericOpc;
    }
  }
  break;
case AArch64::FPRRegBankID:
  switch (OpSize) {
  case 32:
    switch (GenericOpc) {
    case TargetOpcode::G_FADD:
      return AArch64::FADDSrr;
    case TargetOpcode::G_FSUB:
      return AArch64::FSUBSrr;
    case TargetOpcode::G_FMUL:
      return AArch64::FMULSrr;
    case TargetOpcode::G_FDIV:
      return AArch64::FDIVSrr;
    default:
      return GenericOpc;
    }
  case 64:
    switch (GenericOpc) {
    case TargetOpcode::G_FADD:
      return AArch64::FADDDrr;
    case TargetOpcode::G_FSUB:
      return AArch64::FSUBDrr;
    case TargetOpcode::G_FMUL:
      return AArch64::FMULDrr;
    case TargetOpcode::G_FDIV:
      return AArch64::FDIVDrr;
    case TargetOpcode::G_OR:
      return AArch64::ORRv8i8;
    default:
      return GenericOpc;
    }
  }
  break;
}
return GenericOpc;
814}

816/// Select the AArch64 opcode for the G_LOAD or G_STORE operation \p GenericOpc,
817/// appropriate for the (value) register bank \p RegBankID and of memory access
818/// size \p OpSize.  This returns the variant with the base+unsigned-immediate
819/// addressing mode (e.g., LDRXui).
820/// \returns \p GenericOpc if the combination is unsupported.
821static unsigned selectLoadStoreUIOp(unsigned GenericOpc, unsigned RegBankID,
                                  unsigned OpSize) {
const bool isStore = GenericOpc == TargetOpcode::G_STORE;
switch (RegBankID) {
case AArch64::GPRRegBankID:
  switch (OpSize) {
  case 8:
    return isStore ? AArch64::STRBBui : AArch64::LDRBBui;
  case 16:
    return isStore ? AArch64::STRHHui : AArch64::LDRHHui;
  case 32:
    return isStore ? AArch64::STRWui : AArch64::LDRWui;
  case 64:
    return isStore ? AArch64::STRXui : AArch64::LDRXui;
  }
  break;
case AArch64::FPRRegBankID:
  switch (OpSize) {
  case 8:
    return isStore ? AArch64::STRBui : AArch64::LDRBui;
  case 16:
    return isStore ? AArch64::STRHui : AArch64::LDRHui;
  case 32:
    return isStore ? AArch64::STRSui : AArch64::LDRSui;
  case 64:
    return isStore ? AArch64::STRDui : AArch64::LDRDui;
  case 128:
    return isStore ? AArch64::STRQui : AArch64::LDRQui;
  }
  break;
}
return GenericOpc;
853}

855/// Helper function for selectCopy. Inserts a subregister copy from \p SrcReg
856/// to \p *To.
857///
858/// E.g "To = COPY SrcReg:SubReg"
859static bool copySubReg(MachineInstr &I, MachineRegisterInfo &MRI,
                     const RegisterBankInfo &RBI, Register SrcReg,
                     const TargetRegisterClass *To, unsigned SubReg) {
assert(SrcReg.isValid() && "Expected a valid source register?")(static_cast <bool> (SrcReg.isValid() && "Expected a valid source register?"
) ? void (0) : __assert_fail ("SrcReg.isValid() && \"Expected a valid source register?\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 862, __extension__ __PRETTY_FUNCTION__));
assert(To && "Destination register class cannot be null")(static_cast <bool> (To && "Destination register class cannot be null"
) ? void (0) : __assert_fail ("To && \"Destination register class cannot be null\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 863, __extension__ __PRETTY_FUNCTION__));
assert(SubReg && "Expected a valid subregister")(static_cast <bool> (SubReg && "Expected a valid subregister"
) ? void (0) : __assert_fail ("SubReg && \"Expected a valid subregister\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 864, __extension__ __PRETTY_FUNCTION__));

MachineIRBuilder MIB(I);
auto SubRegCopy =
    MIB.buildInstr(TargetOpcode::COPY, {To}, {}).addReg(SrcReg, 0, SubReg);
MachineOperand &RegOp = I.getOperand(1);
RegOp.setReg(SubRegCopy.getReg(0));

// It's possible that the destination register won't be constrained. Make
// sure that happens.
if (!I.getOperand(0).getReg().isPhysical())
  RBI.constrainGenericRegister(I.getOperand(0).getReg(), *To, MRI);

return true;
878}

880/// Helper function to get the source and destination register classes for a
881/// copy. Returns a std::pair containing the source register class for the
882/// copy, and the destination register class for the copy. If a register class
883/// cannot be determined, then it will be nullptr.
884static std::pair<const TargetRegisterClass *, const TargetRegisterClass *>
885getRegClassesForCopy(MachineInstr &I, const TargetInstrInfo &TII,
                   MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI,
                   const RegisterBankInfo &RBI) {
Register DstReg = I.getOperand(0).getReg();
Register SrcReg = I.getOperand(1).getReg();
const RegisterBank &DstRegBank = *RBI.getRegBank(DstReg, MRI, TRI);
const RegisterBank &SrcRegBank = *RBI.getRegBank(SrcReg, MRI, TRI);
unsigned DstSize = RBI.getSizeInBits(DstReg, MRI, TRI);
unsigned SrcSize = RBI.getSizeInBits(SrcReg, MRI, TRI);

// Special casing for cross-bank copies of s1s. We can technically represent
// a 1-bit value with any size of register. The minimum size for a GPR is 32
// bits. So, we need to put the FPR on 32 bits as well.
//
// FIXME: I'm not sure if this case holds true outside of copies. If it does,
// then we can pull it into the helpers that get the appropriate class for a
// register bank. Or make a new helper that carries along some constraint
// information.
if (SrcRegBank != DstRegBank && (DstSize == 1 && SrcSize == 1))
  SrcSize = DstSize = 32;

return {getMinClassForRegBank(SrcRegBank, SrcSize, true),
        getMinClassForRegBank(DstRegBank, DstSize, true)};
908}

910// FIXME: We need some sort of API in RBI/TRI to allow generic code to
911// constrain operands of simple instructions given a TargetRegisterClass
912// and LLT
913static bool selectDebugInstr(MachineInstr &I, MachineRegisterInfo &MRI,
                           const RegisterBankInfo &RBI) {
for (MachineOperand &MO : I.operands()) {
  if (!MO.isReg())
    continue;
  Register Reg = MO.getReg();
  if (!Reg)
    continue;
  if (Reg.isPhysical())
    continue;
  LLT Ty = MRI.getType(Reg);
  const RegClassOrRegBank &RegClassOrBank = MRI.getRegClassOrRegBank(Reg);
  const TargetRegisterClass *RC =
      RegClassOrBank.dyn_cast<const TargetRegisterClass *>();
  if (!RC) {
    const RegisterBank &RB = *RegClassOrBank.get<const RegisterBank *>();
    RC = getRegClassForTypeOnBank(Ty, RB);
    if (!RC) {
      LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Warning: DBG_VALUE operand has unexpected size/bank\n"
; } } while (false)
          dbgs() << "Warning: DBG_VALUE operand has unexpected size/bank\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Warning: DBG_VALUE operand has unexpected size/bank\n"
; } } while (false);
      break;
    }
  }
  RBI.constrainGenericRegister(Reg, *RC, MRI);
}

return true;
940}

942static bool selectCopy(MachineInstr &I, const TargetInstrInfo &TII,
                     MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI,
                     const RegisterBankInfo &RBI) {
Register DstReg = I.getOperand(0).getReg();
Register SrcReg = I.getOperand(1).getReg();
const RegisterBank &DstRegBank = *RBI.getRegBank(DstReg, MRI, TRI);
const RegisterBank &SrcRegBank = *RBI.getRegBank(SrcReg, MRI, TRI);

// Find the correct register classes for the source and destination registers.
const TargetRegisterClass *SrcRC;
const TargetRegisterClass *DstRC;
std::tie(SrcRC, DstRC) = getRegClassesForCopy(I, TII, MRI, TRI, RBI);

if (!DstRC) {
  LLVM_DEBUG(dbgs() << "Unexpected dest size "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unexpected dest size " <<
 RBI.getSizeInBits(DstReg, MRI, TRI) << '\n'; } } while
 (false)
                    << RBI.getSizeInBits(DstReg, MRI, TRI) << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unexpected dest size " <<
 RBI.getSizeInBits(DstReg, MRI, TRI) << '\n'; } } while
 (false);
  return false;
}

// Is this a copy? If so, then we may need to insert a subregister copy.
if (I.isCopy()) {
  // Yes. Check if there's anything to fix up.
  if (!SrcRC) {
    LLVM_DEBUG(dbgs() << "Couldn't determine source register class\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Couldn't determine source register class\n"
; } } while (false);
    return false;
  }

  unsigned SrcSize = TRI.getRegSizeInBits(*SrcRC);
  unsigned DstSize = TRI.getRegSizeInBits(*DstRC);
  unsigned SubReg;

  // If the source bank doesn't support a subregister copy small enough,
  // then we first need to copy to the destination bank.
  if (getMinSizeForRegBank(SrcRegBank) > DstSize) {
    const TargetRegisterClass *DstTempRC =
        getMinClassForRegBank(DstRegBank, SrcSize, /* GetAllRegSet */ true);
    getSubRegForClass(DstRC, TRI, SubReg);

    MachineIRBuilder MIB(I);
    auto Copy = MIB.buildCopy({DstTempRC}, {SrcReg});
    copySubReg(I, MRI, RBI, Copy.getReg(0), DstRC, SubReg);
  } else if (SrcSize > DstSize) {
    // If the source register is bigger than the destination we need to
    // perform a subregister copy.
    const TargetRegisterClass *SubRegRC =
        getMinClassForRegBank(SrcRegBank, DstSize, /* GetAllRegSet */ true);
    getSubRegForClass(SubRegRC, TRI, SubReg);
    copySubReg(I, MRI, RBI, SrcReg, DstRC, SubReg);
  } else if (DstSize > SrcSize) {
    // If the destination register is bigger than the source we need to do
    // a promotion using SUBREG_TO_REG.
    const TargetRegisterClass *PromotionRC =
        getMinClassForRegBank(SrcRegBank, DstSize, /* GetAllRegSet */ true);
    getSubRegForClass(SrcRC, TRI, SubReg);

    Register PromoteReg = MRI.createVirtualRegister(PromotionRC);
    BuildMI(*I.getParent(), I, I.getDebugLoc(),
            TII.get(AArch64::SUBREG_TO_REG), PromoteReg)
        .addImm(0)
        .addUse(SrcReg)
        .addImm(SubReg);
    MachineOperand &RegOp = I.getOperand(1);
    RegOp.setReg(PromoteReg);
  }

  // If the destination is a physical register, then there's nothing to
  // change, so we're done.
  if (DstReg.isPhysical())
    return true;
}

// No need to constrain SrcReg. It will get constrained when we hit another
// of its use or its defs. Copies do not have constraints.
if (!RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) {
  LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Failed to constrain " <<
 TII.getName(I.getOpcode()) << " operand\n"; } } while (
false)
                    << " operand\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Failed to constrain " <<
 TII.getName(I.getOpcode()) << " operand\n"; } } while (
false);
  return false;
}

// If this a GPR ZEXT that we want to just reduce down into a copy.
// The sizes will be mismatched with the source < 32b but that's ok.
if (I.getOpcode() == TargetOpcode::G_ZEXT) {
  I.setDesc(TII.get(AArch64::COPY));
  assert(SrcRegBank.getID() == AArch64::GPRRegBankID)(static_cast <bool> (SrcRegBank.getID() == AArch64::GPRRegBankID
) ? void (0) : __assert_fail ("SrcRegBank.getID() == AArch64::GPRRegBankID"
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1025, __extension__ __PRETTY_FUNCTION__));
  return selectCopy(I, TII, MRI, TRI, RBI);
}

I.setDesc(TII.get(AArch64::COPY));
return true;
1031}

1033static unsigned selectFPConvOpc(unsigned GenericOpc, LLT DstTy, LLT SrcTy) {
if (!DstTy.isScalar() || !SrcTy.isScalar())
  return GenericOpc;

const unsigned DstSize = DstTy.getSizeInBits();
const unsigned SrcSize = SrcTy.getSizeInBits();

switch (DstSize) {
case 32:
  switch (SrcSize) {
  case 32:
    switch (GenericOpc) {
    case TargetOpcode::G_SITOFP:
      return AArch64::SCVTFUWSri;
    case TargetOpcode::G_UITOFP:
      return AArch64::UCVTFUWSri;
    case TargetOpcode::G_FPTOSI:
      return AArch64::FCVTZSUWSr;
    case TargetOpcode::G_FPTOUI:
      return AArch64::FCVTZUUWSr;
    default:
      return GenericOpc;
    }
  case 64:
    switch (GenericOpc) {
    case TargetOpcode::G_SITOFP:
      return AArch64::SCVTFUXSri;
    case TargetOpcode::G_UITOFP:
      return AArch64::UCVTFUXSri;
    case TargetOpcode::G_FPTOSI:
      return AArch64::FCVTZSUWDr;
    case TargetOpcode::G_FPTOUI:
      return AArch64::FCVTZUUWDr;
    default:
      return GenericOpc;
    }
  default:
    return GenericOpc;
  }
case 64:
  switch (SrcSize) {
  case 32:
    switch (GenericOpc) {
    case TargetOpcode::G_SITOFP:
      return AArch64::SCVTFUWDri;
    case TargetOpcode::G_UITOFP:
      return AArch64::UCVTFUWDri;
    case TargetOpcode::G_FPTOSI:
      return AArch64::FCVTZSUXSr;
    case TargetOpcode::G_FPTOUI:
      return AArch64::FCVTZUUXSr;
    default:
      return GenericOpc;
    }
  case 64:
    switch (GenericOpc) {
    case TargetOpcode::G_SITOFP:
      return AArch64::SCVTFUXDri;
    case TargetOpcode::G_UITOFP:
      return AArch64::UCVTFUXDri;
    case TargetOpcode::G_FPTOSI:
      return AArch64::FCVTZSUXDr;
    case TargetOpcode::G_FPTOUI:
      return AArch64::FCVTZUUXDr;
    default:
      return GenericOpc;
    }
  default:
    return GenericOpc;
  }
default:
  return GenericOpc;
};
return GenericOpc;
1107}

1109MachineInstr *
1110AArch64InstructionSelector::emitSelect(Register Dst, Register True,
                                     Register False, AArch64CC::CondCode CC,
                                     MachineIRBuilder &MIB) const {
MachineRegisterInfo &MRI = *MIB.getMRI();
assert(RBI.getRegBank(False, MRI, TRI)->getID() ==(static_cast <bool> (RBI.getRegBank(False, MRI, TRI)->
getID() == RBI.getRegBank(True, MRI, TRI)->getID() &&
 "Expected both select operands to have the same regbank?") ?
 void (0) : __assert_fail ("RBI.getRegBank(False, MRI, TRI)->getID() == RBI.getRegBank(True, MRI, TRI)->getID() && \"Expected both select operands to have the same regbank?\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1116, __extension__ __PRETTY_FUNCTION__))
           RBI.getRegBank(True, MRI, TRI)->getID() &&(static_cast <bool> (RBI.getRegBank(False, MRI, TRI)->
getID() == RBI.getRegBank(True, MRI, TRI)->getID() &&
 "Expected both select operands to have the same regbank?") ?
 void (0) : __assert_fail ("RBI.getRegBank(False, MRI, TRI)->getID() == RBI.getRegBank(True, MRI, TRI)->getID() && \"Expected both select operands to have the same regbank?\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1116, __extension__ __PRETTY_FUNCTION__))
       "Expected both select operands to have the same regbank?")(static_cast <bool> (RBI.getRegBank(False, MRI, TRI)->
getID() == RBI.getRegBank(True, MRI, TRI)->getID() &&
 "Expected both select operands to have the same regbank?") ?
 void (0) : __assert_fail ("RBI.getRegBank(False, MRI, TRI)->getID() == RBI.getRegBank(True, MRI, TRI)->getID() && \"Expected both select operands to have the same regbank?\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1116, __extension__ __PRETTY_FUNCTION__));
LLT Ty = MRI.getType(True);
if (Ty.isVector())
  return nullptr;
const unsigned Size = Ty.getSizeInBits();
assert((Size == 32 || Size == 64) &&(static_cast <bool> ((Size == 32 || Size == 64) &&
 "Expected 32 bit or 64 bit select only?") ? void (0) : __assert_fail
 ("(Size == 32 || Size == 64) && \"Expected 32 bit or 64 bit select only?\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1122, __extension__ __PRETTY_FUNCTION__))
       "Expected 32 bit or 64 bit select only?")(static_cast <bool> ((Size == 32 || Size == 64) &&
 "Expected 32 bit or 64 bit select only?") ? void (0) : __assert_fail
 ("(Size == 32 || Size == 64) && \"Expected 32 bit or 64 bit select only?\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1122, __extension__ __PRETTY_FUNCTION__));
const bool Is32Bit = Size == 32;
if (RBI.getRegBank(True, MRI, TRI)->getID() != AArch64::GPRRegBankID) {
  unsigned Opc = Is32Bit ? AArch64::FCSELSrrr : AArch64::FCSELDrrr;
  auto FCSel = MIB.buildInstr(Opc, {Dst}, {True, False}).addImm(CC);
  constrainSelectedInstRegOperands(*FCSel, TII, TRI, RBI);
  return &*FCSel;
}

// By default, we'll try and emit a CSEL.
unsigned Opc = Is32Bit ? AArch64::CSELWr : AArch64::CSELXr;
bool Optimized = false;
auto TryFoldBinOpIntoSelect = [&Opc, Is32Bit, &CC, &MRI,
                               &Optimized](Register &Reg, Register &OtherReg,
                                           bool Invert) {
  if (Optimized)
    return false;

  // Attempt to fold:
  //
  // %sub = G_SUB 0, %x
  // %select = G_SELECT cc, %reg, %sub
  //
  // Into:
  // %select = CSNEG %reg, %x, cc
  Register MatchReg;
  if (mi_match(Reg, MRI, m_Neg(m_Reg(MatchReg)))) {
    Opc = Is32Bit ? AArch64::CSNEGWr : AArch64::CSNEGXr;
    Reg = MatchReg;
    if (Invert) {
      CC = AArch64CC::getInvertedCondCode(CC);
      std::swap(Reg, OtherReg);
    }
    return true;
  }

  // Attempt to fold:
  //
  // %xor = G_XOR %x, -1
  // %select = G_SELECT cc, %reg, %xor
  //
  // Into:
  // %select = CSINV %reg, %x, cc
  if (mi_match(Reg, MRI, m_Not(m_Reg(MatchReg)))) {
    Opc = Is32Bit ? AArch64::CSINVWr : AArch64::CSINVXr;
    Reg = MatchReg;
    if (Invert) {
      CC = AArch64CC::getInvertedCondCode(CC);
      std::swap(Reg, OtherReg);
    }
    return true;
  }

  // Attempt to fold:
  //
  // %add = G_ADD %x, 1
  // %select = G_SELECT cc, %reg, %add
  //
  // Into:
  // %select = CSINC %reg, %x, cc
  if (mi_match(Reg, MRI,
               m_any_of(m_GAdd(m_Reg(MatchReg), m_SpecificICst(1)),
                        m_GPtrAdd(m_Reg(MatchReg), m_SpecificICst(1))))) {
    Opc = Is32Bit ? AArch64::CSINCWr : AArch64::CSINCXr;
    Reg = MatchReg;
    if (Invert) {
      CC = AArch64CC::getInvertedCondCode(CC);
      std::swap(Reg, OtherReg);
    }
    return true;
  }

  return false;
};

// Helper lambda which tries to use CSINC/CSINV for the instruction when its
// true/false values are constants.
// FIXME: All of these patterns already exist in tablegen. We should be
// able to import these.
auto TryOptSelectCst = [&Opc, &True, &False, &CC, Is32Bit, &MRI,
                        &Optimized]() {
  if (Optimized)
    return false;
  auto TrueCst = getIConstantVRegValWithLookThrough(True, MRI);
  auto FalseCst = getIConstantVRegValWithLookThrough(False, MRI);
  if (!TrueCst && !FalseCst)
    return false;

  Register ZReg = Is32Bit ? AArch64::WZR : AArch64::XZR;
  if (TrueCst && FalseCst) {
    int64_t T = TrueCst->Value.getSExtValue();
    int64_t F = FalseCst->Value.getSExtValue();

    if (T == 0 && F == 1) {
      // G_SELECT cc, 0, 1 -> CSINC zreg, zreg, cc
      Opc = Is32Bit ? AArch64::CSINCWr : AArch64::CSINCXr;
      True = ZReg;
      False = ZReg;
      return true;
    }

    if (T == 0 && F == -1) {
      // G_SELECT cc 0, -1 -> CSINV zreg, zreg cc
      Opc = Is32Bit ? AArch64::CSINVWr : AArch64::CSINVXr;
      True = ZReg;
      False = ZReg;
      return true;
    }
  }

  if (TrueCst) {
    int64_t T = TrueCst->Value.getSExtValue();
    if (T == 1) {
      // G_SELECT cc, 1, f -> CSINC f, zreg, inv_cc
      Opc = Is32Bit ? AArch64::CSINCWr : AArch64::CSINCXr;
      True = False;
      False = ZReg;
      CC = AArch64CC::getInvertedCondCode(CC);
      return true;
    }

    if (T == -1) {
      // G_SELECT cc, -1, f -> CSINV f, zreg, inv_cc
      Opc = Is32Bit ? AArch64::CSINVWr : AArch64::CSINVXr;
      True = False;
      False = ZReg;
      CC = AArch64CC::getInvertedCondCode(CC);
      return true;
    }
  }

  if (FalseCst) {
    int64_t F = FalseCst->Value.getSExtValue();
    if (F == 1) {
      // G_SELECT cc, t, 1 -> CSINC t, zreg, cc
      Opc = Is32Bit ? AArch64::CSINCWr : AArch64::CSINCXr;
      False = ZReg;
      return true;
    }

    if (F == -1) {
      // G_SELECT cc, t, -1 -> CSINC t, zreg, cc
      Opc = Is32Bit ? AArch64::CSINVWr : AArch64::CSINVXr;
      False = ZReg;
      return true;
    }
  }
  return false;
};

Optimized |= TryFoldBinOpIntoSelect(False, True, /*Invert = */ false);
Optimized |= TryFoldBinOpIntoSelect(True, False, /*Invert = */ true);
Optimized |= TryOptSelectCst();
auto SelectInst = MIB.buildInstr(Opc, {Dst}, {True, False}).addImm(CC);
constrainSelectedInstRegOperands(*SelectInst, TII, TRI, RBI);
return &*SelectInst;
1278}

1280static AArch64CC::CondCode changeICMPPredToAArch64CC(CmpInst::Predicate P) {
switch (P) {
default:
  llvm_unreachable("Unknown condition code!")::llvm::llvm_unreachable_internal("Unknown condition code!", "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1283);
case CmpInst::ICMP_NE:
  return AArch64CC::NE;
case CmpInst::ICMP_EQ:
  return AArch64CC::EQ;
case CmpInst::ICMP_SGT:
  return AArch64CC::GT;
case CmpInst::ICMP_SGE:
  return AArch64CC::GE;
case CmpInst::ICMP_SLT:
  return AArch64CC::LT;
case CmpInst::ICMP_SLE:
  return AArch64CC::LE;
case CmpInst::ICMP_UGT:
  return AArch64CC::HI;
case CmpInst::ICMP_UGE:
  return AArch64CC::HS;
case CmpInst::ICMP_ULT:
  return AArch64CC::LO;
case CmpInst::ICMP_ULE:
  return AArch64CC::LS;
}
1305}

1307/// changeFPCCToORAArch64CC - Convert an IR fp condition code to an AArch64 CC.
1308static void changeFPCCToORAArch64CC(CmpInst::Predicate CC,
                                  AArch64CC::CondCode &CondCode,
                                  AArch64CC::CondCode &CondCode2) {
CondCode2 = AArch64CC::AL;
switch (CC) {
default:
  llvm_unreachable("Unknown FP condition!")::llvm::llvm_unreachable_internal("Unknown FP condition!", "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1314);
case CmpInst::FCMP_OEQ:
  CondCode = AArch64CC::EQ;
  break;
case CmpInst::FCMP_OGT:
  CondCode = AArch64CC::GT;
  break;
case CmpInst::FCMP_OGE:
  CondCode = AArch64CC::GE;
  break;
case CmpInst::FCMP_OLT:
  CondCode = AArch64CC::MI;
  break;
case CmpInst::FCMP_OLE:
  CondCode = AArch64CC::LS;
  break;
case CmpInst::FCMP_ONE:
  CondCode = AArch64CC::MI;
  CondCode2 = AArch64CC::GT;
  break;
case CmpInst::FCMP_ORD:
  CondCode = AArch64CC::VC;
  break;
case CmpInst::FCMP_UNO:
  CondCode = AArch64CC::VS;
  break;
case CmpInst::FCMP_UEQ:
  CondCode = AArch64CC::EQ;
  CondCode2 = AArch64CC::VS;
  break;
case CmpInst::FCMP_UGT:
  CondCode = AArch64CC::HI;
  break;
case CmpInst::FCMP_UGE:
  CondCode = AArch64CC::PL;
  break;
case CmpInst::FCMP_ULT:
  CondCode = AArch64CC::LT;
  break;
case CmpInst::FCMP_ULE:
  CondCode = AArch64CC::LE;
  break;
case CmpInst::FCMP_UNE:
  CondCode = AArch64CC::NE;
  break;
}
1360}

1362/// Convert an IR fp condition code to an AArch64 CC.
1363/// This differs from changeFPCCToAArch64CC in that it returns cond codes that
1364/// should be AND'ed instead of OR'ed.
1365static void changeFPCCToANDAArch64CC(CmpInst::Predicate CC,
                                   AArch64CC::CondCode &CondCode,
                                   AArch64CC::CondCode &CondCode2) {
CondCode2 = AArch64CC::AL;
switch (CC) {
default:
  changeFPCCToORAArch64CC(CC, CondCode, CondCode2);
  assert(CondCode2 == AArch64CC::AL)(static_cast <bool> (CondCode2 == AArch64CC::AL) ? void
 (0) : __assert_fail ("CondCode2 == AArch64CC::AL", "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1372, __extension__ __PRETTY_FUNCTION__));
  break;
case CmpInst::FCMP_ONE:
  // (a one b)
  // == ((a olt b) || (a ogt b))
  // == ((a ord b) && (a une b))
  CondCode = AArch64CC::VC;
  CondCode2 = AArch64CC::NE;
  break;
case CmpInst::FCMP_UEQ:
  // (a ueq b)
  // == ((a uno b) || (a oeq b))
  // == ((a ule b) && (a uge b))
  CondCode = AArch64CC::PL;
  CondCode2 = AArch64CC::LE;
  break;
}
1389}

1391/// Return a register which can be used as a bit to test in a TB(N)Z.
1392static Register getTestBitReg(Register Reg, uint64_t &Bit, bool &Invert,
                            MachineRegisterInfo &MRI) {
assert(Reg.isValid() && "Expected valid register!")(static_cast <bool> (Reg.isValid() && "Expected valid register!"
) ? void (0) : __assert_fail ("Reg.isValid() && \"Expected valid register!\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1394, __extension__ __PRETTY_FUNCTION__));
bool HasZext = false;
while (MachineInstr *MI = getDefIgnoringCopies(Reg, MRI)) {
  unsigned Opc = MI->getOpcode();

  if (!MI->getOperand(0).isReg() ||
      !MRI.hasOneNonDBGUse(MI->getOperand(0).getReg()))
    break;

  // (tbz (any_ext x), b) -> (tbz x, b) if we don't use the extended bits.
  //
  // (tbz (trunc x), b) -> (tbz x, b) is always safe, because the bit number
  // on the truncated x is the same as the bit number on x.
  if (Opc == TargetOpcode::G_ANYEXT || Opc == TargetOpcode::G_ZEXT ||
      Opc == TargetOpcode::G_TRUNC) {
    if (Opc == TargetOpcode::G_ZEXT)
      HasZext = true;

    Register NextReg = MI->getOperand(1).getReg();
    // Did we find something worth folding?
    if (!NextReg.isValid() || !MRI.hasOneNonDBGUse(NextReg))
      break;

    // NextReg is worth folding. Keep looking.
    Reg = NextReg;
    continue;
  }

  // Attempt to find a suitable operation with a constant on one side.
  std::optional<uint64_t> C;
  Register TestReg;
  switch (Opc) {
  default:
    break;
  case TargetOpcode::G_AND:
  case TargetOpcode::G_XOR: {
    TestReg = MI->getOperand(1).getReg();
    Register ConstantReg = MI->getOperand(2).getReg();
    auto VRegAndVal = getIConstantVRegValWithLookThrough(ConstantReg, MRI);
    if (!VRegAndVal) {
      // AND commutes, check the other side for a constant.
      // FIXME: Can we canonicalize the constant so that it's always on the
      // same side at some point earlier?
      std::swap(ConstantReg, TestReg);
      VRegAndVal = getIConstantVRegValWithLookThrough(ConstantReg, MRI);
    }
    if (VRegAndVal) {
      if (HasZext)
        C = VRegAndVal->Value.getZExtValue();
      else
        C = VRegAndVal->Value.getSExtValue();
    }
    break;
  }
  case TargetOpcode::G_ASHR:
  case TargetOpcode::G_LSHR:
  case TargetOpcode::G_SHL: {
    TestReg = MI->getOperand(1).getReg();
    auto VRegAndVal =
        getIConstantVRegValWithLookThrough(MI->getOperand(2).getReg(), MRI);
    if (VRegAndVal)
      C = VRegAndVal->Value.getSExtValue();
    break;
  }
  }

  // Didn't find a constant or viable register. Bail out of the loop.
  if (!C || !TestReg.isValid())
    break;

  // We found a suitable instruction with a constant. Check to see if we can
  // walk through the instruction.
  Register NextReg;
  unsigned TestRegSize = MRI.getType(TestReg).getSizeInBits();
  switch (Opc) {
  default:
    break;
  case TargetOpcode::G_AND:
    // (tbz (and x, m), b) -> (tbz x, b) when the b-th bit of m is set.
    if ((*C >> Bit) & 1)
      NextReg = TestReg;
    break;
  case TargetOpcode::G_SHL:
    // (tbz (shl x, c), b) -> (tbz x, b-c) when b-c is positive and fits in
    // the type of the register.
    if (*C <= Bit && (Bit - *C) < TestRegSize) {
      NextReg = TestReg;
      Bit = Bit - *C;
    }
    break;
  case TargetOpcode::G_ASHR:
    // (tbz (ashr x, c), b) -> (tbz x, b+c) or (tbz x, msb) if b+c is > # bits
    // in x
    NextReg = TestReg;
    Bit = Bit + *C;
    if (Bit >= TestRegSize)
      Bit = TestRegSize - 1;
    break;
  case TargetOpcode::G_LSHR:
    // (tbz (lshr x, c), b) -> (tbz x, b+c) when b + c is < # bits in x
    if ((Bit + *C) < TestRegSize) {
      NextReg = TestReg;
      Bit = Bit + *C;
    }
    break;
  case TargetOpcode::G_XOR:
    // We can walk through a G_XOR by inverting whether we use tbz/tbnz when
    // appropriate.
    //
    // e.g. If x' = xor x, c, and the b-th bit is set in c then
    //
    // tbz x', b -> tbnz x, b
    //
    // Because x' only has the b-th bit set if x does not.
    if ((*C >> Bit) & 1)
      Invert = !Invert;
    NextReg = TestReg;
    break;
  }

  // Check if we found anything worth folding.
  if (!NextReg.isValid())
    return Reg;
  Reg = NextReg;
}

return Reg;
1521}

1523MachineInstr *AArch64InstructionSelector::emitTestBit(
  Register TestReg, uint64_t Bit, bool IsNegative, MachineBasicBlock *DstMBB,
  MachineIRBuilder &MIB) const {
assert(TestReg.isValid())(static_cast <bool> (TestReg.isValid()) ? void (0) : __assert_fail
 ("TestReg.isValid()", "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1526, __extension__ __PRETTY_FUNCTION__));
assert(ProduceNonFlagSettingCondBr &&(static_cast <bool> (ProduceNonFlagSettingCondBr &&
 "Cannot emit TB(N)Z with speculation tracking!") ? void (0) :
 __assert_fail ("ProduceNonFlagSettingCondBr && \"Cannot emit TB(N)Z with speculation tracking!\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1528, __extension__ __PRETTY_FUNCTION__))
       "Cannot emit TB(N)Z with speculation tracking!")(static_cast <bool> (ProduceNonFlagSettingCondBr &&
 "Cannot emit TB(N)Z with speculation tracking!") ? void (0) :
 __assert_fail ("ProduceNonFlagSettingCondBr && \"Cannot emit TB(N)Z with speculation tracking!\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1528, __extension__ __PRETTY_FUNCTION__));
MachineRegisterInfo &MRI = *MIB.getMRI();

// Attempt to optimize the test bit by walking over instructions.
TestReg = getTestBitReg(TestReg, Bit, IsNegative, MRI);
LLT Ty = MRI.getType(TestReg);
unsigned Size = Ty.getSizeInBits();
assert(!Ty.isVector() && "Expected a scalar!")(static_cast <bool> (!Ty.isVector() && "Expected a scalar!"
) ? void (0) : __assert_fail ("!Ty.isVector() && \"Expected a scalar!\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1535, __extension__ __PRETTY_FUNCTION__));
assert(Bit < 64 && "Bit is too large!")(static_cast <bool> (Bit < 64 && "Bit is too large!"
) ? void (0) : __assert_fail ("Bit < 64 && \"Bit is too large!\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1536, __extension__ __PRETTY_FUNCTION__));

// When the test register is a 64-bit register, we have to narrow to make
// TBNZW work.
bool UseWReg = Bit < 32;
unsigned NecessarySize = UseWReg ? 32 : 64;
if (Size != NecessarySize)
  TestReg = moveScalarRegClass(
      TestReg, UseWReg ? AArch64::GPR32RegClass : AArch64::GPR64RegClass,
      MIB);

static const unsigned OpcTable[2][2] = {{AArch64::TBZX, AArch64::TBNZX},
                                        {AArch64::TBZW, AArch64::TBNZW}};
unsigned Opc = OpcTable[UseWReg][IsNegative];
auto TestBitMI =
    MIB.buildInstr(Opc).addReg(TestReg).addImm(Bit).addMBB(DstMBB);
constrainSelectedInstRegOperands(*TestBitMI, TII, TRI, RBI);
return &*TestBitMI;
1554}

1556bool AArch64InstructionSelector::tryOptAndIntoCompareBranch(
  MachineInstr &AndInst, bool Invert, MachineBasicBlock *DstMBB,
  MachineIRBuilder &MIB) const {
assert(AndInst.getOpcode() == TargetOpcode::G_AND && "Expected G_AND only?")(static_cast <bool> (AndInst.getOpcode() == TargetOpcode
::G_AND && "Expected G_AND only?") ? void (0) : __assert_fail
 ("AndInst.getOpcode() == TargetOpcode::G_AND && \"Expected G_AND only?\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1559, __extension__ __PRETTY_FUNCTION__));
// Given something like this:
//
//  %x = ...Something...
//  %one = G_CONSTANT i64 1
//  %zero = G_CONSTANT i64 0
//  %and = G_AND %x, %one
//  %cmp = G_ICMP intpred(ne), %and, %zero
//  %cmp_trunc = G_TRUNC %cmp
//  G_BRCOND %cmp_trunc, %bb.3
//
// We want to try and fold the AND into the G_BRCOND and produce either a
// TBNZ (when we have intpred(ne)) or a TBZ (when we have intpred(eq)).
//
// In this case, we'd get
//
// TBNZ %x %bb.3
//

// Check if the AND has a constant on its RHS which we can use as a mask.
// If it's a power of 2, then it's the same as checking a specific bit.
// (e.g, ANDing with 8 == ANDing with 000...100 == testing if bit 3 is set)
auto MaybeBit = getIConstantVRegValWithLookThrough(
    AndInst.getOperand(2).getReg(), *MIB.getMRI());
if (!MaybeBit)
  return false;

int32_t Bit = MaybeBit->Value.exactLogBase2();
if (Bit < 0)
  return false;

Register TestReg = AndInst.getOperand(1).getReg();

// Emit a TB(N)Z.
emitTestBit(TestReg, Bit, Invert, DstMBB, MIB);
return true;
1595}

1597MachineInstr *AArch64InstructionSelector::emitCBZ(Register CompareReg,
                                                bool IsNegative,
                                                MachineBasicBlock *DestMBB,
                                                MachineIRBuilder &MIB) const {
assert(ProduceNonFlagSettingCondBr && "CBZ does not set flags!")(static_cast <bool> (ProduceNonFlagSettingCondBr &&
 "CBZ does not set flags!") ? void (0) : __assert_fail ("ProduceNonFlagSettingCondBr && \"CBZ does not set flags!\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1601, __extension__ __PRETTY_FUNCTION__));
MachineRegisterInfo &MRI = *MIB.getMRI();
assert(RBI.getRegBank(CompareReg, MRI, TRI)->getID() ==(static_cast <bool> (RBI.getRegBank(CompareReg, MRI, TRI
)->getID() == AArch64::GPRRegBankID && "Expected GPRs only?"
) ? void (0) : __assert_fail ("RBI.getRegBank(CompareReg, MRI, TRI)->getID() == AArch64::GPRRegBankID && \"Expected GPRs only?\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1605, __extension__ __PRETTY_FUNCTION__))
           AArch64::GPRRegBankID &&(static_cast <bool> (RBI.getRegBank(CompareReg, MRI, TRI
)->getID() == AArch64::GPRRegBankID && "Expected GPRs only?"
) ? void (0) : __assert_fail ("RBI.getRegBank(CompareReg, MRI, TRI)->getID() == AArch64::GPRRegBankID && \"Expected GPRs only?\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1605, __extension__ __PRETTY_FUNCTION__))
       "Expected GPRs only?")(static_cast <bool> (RBI.getRegBank(CompareReg, MRI, TRI
)->getID() == AArch64::GPRRegBankID && "Expected GPRs only?"
) ? void (0) : __assert_fail ("RBI.getRegBank(CompareReg, MRI, TRI)->getID() == AArch64::GPRRegBankID && \"Expected GPRs only?\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1605, __extension__ __PRETTY_FUNCTION__));
auto Ty = MRI.getType(CompareReg);
unsigned Width = Ty.getSizeInBits();
assert(!Ty.isVector() && "Expected scalar only?")(static_cast <bool> (!Ty.isVector() && "Expected scalar only?"
) ? void (0) : __assert_fail ("!Ty.isVector() && \"Expected scalar only?\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1608, __extension__ __PRETTY_FUNCTION__));
assert(Width <= 64 && "Expected width to be at most 64?")(static_cast <bool> (Width <= 64 && "Expected width to be at most 64?"
) ? void (0) : __assert_fail ("Width <= 64 && \"Expected width to be at most 64?\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1609, __extension__ __PRETTY_FUNCTION__));
static const unsigned OpcTable[2][2] = {{AArch64::CBZW, AArch64::CBZX},
                                        {AArch64::CBNZW, AArch64::CBNZX}};
unsigned Opc = OpcTable[IsNegative][Width == 64];
auto BranchMI = MIB.buildInstr(Opc, {}, {CompareReg}).addMBB(DestMBB);
constrainSelectedInstRegOperands(*BranchMI, TII, TRI, RBI);
return &*BranchMI;
1616}

1618bool AArch64InstructionSelector::selectCompareBranchFedByFCmp(
  MachineInstr &I, MachineInstr &FCmp, MachineIRBuilder &MIB) const {
assert(FCmp.getOpcode() == TargetOpcode::G_FCMP)(static_cast <bool> (FCmp.getOpcode() == TargetOpcode::
G_FCMP) ? void (0) : __assert_fail ("FCmp.getOpcode() == TargetOpcode::G_FCMP"
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1620, __extension__ __PRETTY_FUNCTION__));
assert(I.getOpcode() == TargetOpcode::G_BRCOND)(static_cast <bool> (I.getOpcode() == TargetOpcode::G_BRCOND
) ? void (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_BRCOND"
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1621, __extension__ __PRETTY_FUNCTION__));
// Unfortunately, the mapping of LLVM FP CC's onto AArch64 CC's isn't
// totally clean.  Some of them require two branches to implement.
auto Pred = (CmpInst::Predicate)FCmp.getOperand(1).getPredicate();
emitFPCompare(FCmp.getOperand(2).getReg(), FCmp.getOperand(3).getReg(), MIB,
              Pred);
AArch64CC::CondCode CC1, CC2;
changeFCMPPredToAArch64CC(static_cast<CmpInst::Predicate>(Pred), CC1, CC2);
MachineBasicBlock *DestMBB = I.getOperand(1).getMBB();
MIB.buildInstr(AArch64::Bcc, {}, {}).addImm(CC1).addMBB(DestMBB);
if (CC2 != AArch64CC::AL)
  MIB.buildInstr(AArch64::Bcc, {}, {}).addImm(CC2).addMBB(DestMBB);
I.eraseFromParent();
return true;
1635}

1637bool AArch64InstructionSelector::tryOptCompareBranchFedByICmp(
  MachineInstr &I, MachineInstr &ICmp, MachineIRBuilder &MIB) const {
assert(ICmp.getOpcode() == TargetOpcode::G_ICMP)(static_cast <bool> (ICmp.getOpcode() == TargetOpcode::
G_ICMP) ? void (0) : __assert_fail ("ICmp.getOpcode() == TargetOpcode::G_ICMP"
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1639, __extension__ __PRETTY_FUNCTION__));
assert(I.getOpcode() == TargetOpcode::G_BRCOND)(static_cast <bool> (I.getOpcode() == TargetOpcode::G_BRCOND
) ? void (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_BRCOND"
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1640, __extension__ __PRETTY_FUNCTION__));
// Attempt to optimize the G_BRCOND + G_ICMP into a TB(N)Z/CB(N)Z.
//
// Speculation tracking/SLH assumes that optimized TB(N)Z/CB(N)Z
// instructions will not be produced, as they are conditional branch
// instructions that do not set flags.
if (!ProduceNonFlagSettingCondBr)
  return false;

MachineRegisterInfo &MRI = *MIB.getMRI();
MachineBasicBlock *DestMBB = I.getOperand(1).getMBB();
auto Pred =
    static_cast<CmpInst::Predicate>(ICmp.getOperand(1).getPredicate());
Register LHS = ICmp.getOperand(2).getReg();
Register RHS = ICmp.getOperand(3).getReg();

// We're allowed to emit a TB(N)Z/CB(N)Z. Try to do that.
auto VRegAndVal = getIConstantVRegValWithLookThrough(RHS, MRI);
MachineInstr *AndInst = getOpcodeDef(TargetOpcode::G_AND, LHS, MRI);

// When we can emit a TB(N)Z, prefer that.
//
// Handle non-commutative condition codes first.
// Note that we don't want to do this when we have a G_AND because it can
// become a tst. The tst will make the test bit in the TB(N)Z redundant.
if (VRegAndVal && !AndInst) {
  int64_t C = VRegAndVal->Value.getSExtValue();

  // When we have a greater-than comparison, we can just test if the msb is
  // zero.
  if (C == -1 && Pred == CmpInst::ICMP_SGT) {
    uint64_t Bit = MRI.getType(LHS).getSizeInBits() - 1;
    emitTestBit(LHS, Bit, /*IsNegative = */ false, DestMBB, MIB);
    I.eraseFromParent();
    return true;
  }

  // When we have a less than comparison, we can just test if the msb is not
  // zero.
  if (C == 0 && Pred == CmpInst::ICMP_SLT) {
    uint64_t Bit = MRI.getType(LHS).getSizeInBits() - 1;
    emitTestBit(LHS, Bit, /*IsNegative = */ true, DestMBB, MIB);
    I.eraseFromParent();
    return true;
  }

  // Inversely, if we have a signed greater-than-or-equal comparison to zero,
  // we can test if the msb is zero.
  if (C == 0 && Pred == CmpInst::ICMP_SGE) {
    uint64_t Bit = MRI.getType(LHS).getSizeInBits() - 1;
    emitTestBit(LHS, Bit, /*IsNegative = */ false, DestMBB, MIB);
    I.eraseFromParent();
    return true;
  }
}

// Attempt to handle commutative condition codes. Right now, that's only
// eq/ne.
if (ICmpInst::isEquality(Pred)) {
  if (!VRegAndVal) {
    std::swap(RHS, LHS);
    VRegAndVal = getIConstantVRegValWithLookThrough(RHS, MRI);
    AndInst = getOpcodeDef(TargetOpcode::G_AND, LHS, MRI);
  }

  if (VRegAndVal && VRegAndVal->Value == 0) {
    // If there's a G_AND feeding into this branch, try to fold it away by
    // emitting a TB(N)Z instead.
    //
    // Note: If we have LT, then it *is* possible to fold, but it wouldn't be
    // beneficial. When we have an AND and LT, we need a TST/ANDS, so folding
    // would be redundant.
    if (AndInst &&
        tryOptAndIntoCompareBranch(
            *AndInst, /*Invert = */ Pred == CmpInst::ICMP_NE, DestMBB, MIB)) {
      I.eraseFromParent();
      return true;
    }

    // Otherwise, try to emit a CB(N)Z instead.
    auto LHSTy = MRI.getType(LHS);
    if (!LHSTy.isVector() && LHSTy.getSizeInBits() <= 64) {
      emitCBZ(LHS, /*IsNegative = */ Pred == CmpInst::ICMP_NE, DestMBB, MIB);
      I.eraseFromParent();
      return true;
    }
  }
}

return false;
1730}

1732bool AArch64InstructionSelector::selectCompareBranchFedByICmp(
  MachineInstr &I, MachineInstr &ICmp, MachineIRBuilder &MIB) const {
assert(ICmp.getOpcode() == TargetOpcode::G_ICMP)(static_cast <bool> (ICmp.getOpcode() == TargetOpcode::
G_ICMP) ? void (0) : __assert_fail ("ICmp.getOpcode() == TargetOpcode::G_ICMP"
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1734, __extension__ __PRETTY_FUNCTION__));
assert(I.getOpcode() == TargetOpcode::G_BRCOND)(static_cast <bool> (I.getOpcode() == TargetOpcode::G_BRCOND
) ? void (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_BRCOND"
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1735, __extension__ __PRETTY_FUNCTION__));
if (tryOptCompareBranchFedByICmp(I, ICmp, MIB))
  return true;

// Couldn't optimize. Emit a compare + a Bcc.
MachineBasicBlock *DestMBB = I.getOperand(1).getMBB();
auto PredOp = ICmp.getOperand(1);
emitIntegerCompare(ICmp.getOperand(2), ICmp.getOperand(3), PredOp, MIB);
const AArch64CC::CondCode CC = changeICMPPredToAArch64CC(
    static_cast<CmpInst::Predicate>(PredOp.getPredicate()));
MIB.buildInstr(AArch64::Bcc, {}, {}).addImm(CC).addMBB(DestMBB);
I.eraseFromParent();
return true;
1748}

1750bool AArch64InstructionSelector::selectCompareBranch(
  MachineInstr &I, MachineFunction &MF, MachineRegisterInfo &MRI) {
Register CondReg = I.getOperand(0).getReg();
MachineInstr *CCMI = MRI.getVRegDef(CondReg);
// Try to select the G_BRCOND using whatever is feeding the condition if
// possible.
unsigned CCMIOpc = CCMI->getOpcode();
if (CCMIOpc == TargetOpcode::G_FCMP)
  return selectCompareBranchFedByFCmp(I, *CCMI, MIB);
if (CCMIOpc == TargetOpcode::G_ICMP)
  return selectCompareBranchFedByICmp(I, *CCMI, MIB);

// Speculation tracking/SLH assumes that optimized TB(N)Z/CB(N)Z
// instructions will not be produced, as they are conditional branch
// instructions that do not set flags.
if (ProduceNonFlagSettingCondBr) {
  emitTestBit(CondReg, /*Bit = */ 0, /*IsNegative = */ true,
              I.getOperand(1).getMBB(), MIB);
  I.eraseFromParent();
  return true;
}

// Can't emit TB(N)Z/CB(N)Z. Emit a tst + bcc instead.
auto TstMI =
    MIB.buildInstr(AArch64::ANDSWri, {LLT::scalar(32)}, {CondReg}).addImm(1);
constrainSelectedInstRegOperands(*TstMI, TII, TRI, RBI);
auto Bcc = MIB.buildInstr(AArch64::Bcc)
               .addImm(AArch64CC::EQ)
               .addMBB(I.getOperand(1).getMBB());
I.eraseFromParent();
return constrainSelectedInstRegOperands(*Bcc, TII, TRI, RBI);
1781}

1783/// Returns the element immediate value of a vector shift operand if found.
1784/// This needs to detect a splat-like operation, e.g. a G_BUILD_VECTOR.
1785static std::optional<int64_t> getVectorShiftImm(Register Reg,
                                              MachineRegisterInfo &MRI) {
assert(MRI.getType(Reg).isVector() && "Expected a *vector* shift operand")(static_cast <bool> (MRI.getType(Reg).isVector() &&
 "Expected a *vector* shift operand") ? void (0) : __assert_fail
 ("MRI.getType(Reg).isVector() && \"Expected a *vector* shift operand\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1787, __extension__ __PRETTY_FUNCTION__));
MachineInstr *OpMI = MRI.getVRegDef(Reg);
return getAArch64VectorSplatScalar(*OpMI, MRI);
1790}

1792/// Matches and returns the shift immediate value for a SHL instruction given
1793/// a shift operand.
1794static std::optional<int64_t> getVectorSHLImm(LLT SrcTy, Register Reg,
                                            MachineRegisterInfo &MRI) {
std::optional<int64_t> ShiftImm = getVectorShiftImm(Reg, MRI);
if (!ShiftImm)
  return std::nullopt;
// Check the immediate is in range for a SHL.
int64_t Imm = *ShiftImm;
if (Imm < 0)
  return std::nullopt;
switch (SrcTy.getElementType().getSizeInBits()) {
default:
  LLVM_DEBUG(dbgs() << "Unhandled element type for vector shift")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unhandled element type for vector shift"
; } } while (false);
  return std::nullopt;
case 8:
  if (Imm > 7)
    return std::nullopt;
  break;
case 16:
  if (Imm > 15)
    return std::nullopt;
  break;
case 32:
  if (Imm > 31)
    return std::nullopt;
  break;
case 64:
  if (Imm > 63)
    return std::nullopt;
  break;
}
return Imm;
1825}

1827bool AArch64InstructionSelector::selectVectorSHL(MachineInstr &I,
                                               MachineRegisterInfo &MRI) {
assert(I.getOpcode() == TargetOpcode::G_SHL)(static_cast <bool> (I.getOpcode() == TargetOpcode::G_SHL
) ? void (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_SHL"
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1829, __extension__ __PRETTY_FUNCTION__));
Register DstReg = I.getOperand(0).getReg();
const LLT Ty = MRI.getType(DstReg);
Register Src1Reg = I.getOperand(1).getReg();
Register Src2Reg = I.getOperand(2).getReg();

if (!Ty.isVector())
  return false;

// Check if we have a vector of constants on RHS that we can select as the
// immediate form.
std::optional<int64_t> ImmVal = getVectorSHLImm(Ty, Src2Reg, MRI);

unsigned Opc = 0;
if (Ty == LLT::fixed_vector(2, 64)) {
  Opc = ImmVal ? AArch64::SHLv2i64_shift : AArch64::USHLv2i64;
} else if (Ty == LLT::fixed_vector(4, 32)) {
  Opc = ImmVal ? AArch64::SHLv4i32_shift : AArch64::USHLv4i32;
} else if (Ty == LLT::fixed_vector(2, 32)) {
  Opc = ImmVal ? AArch64::SHLv2i32_shift : AArch64::USHLv2i32;
} else if (Ty == LLT::fixed_vector(4, 16)) {
  Opc = ImmVal ? AArch64::SHLv4i16_shift : AArch64::USHLv4i16;
} else if (Ty == LLT::fixed_vector(8, 16)) {
  Opc = ImmVal ? AArch64::SHLv8i16_shift : AArch64::USHLv8i16;
} else if (Ty == LLT::fixed_vector(16, 8)) {
  Opc = ImmVal ? AArch64::SHLv16i8_shift : AArch64::USHLv16i8;
} else if (Ty == LLT::fixed_vector(8, 8)) {
  Opc = ImmVal ? AArch64::SHLv8i8_shift : AArch64::USHLv8i8;
} else {
  LLVM_DEBUG(dbgs() << "Unhandled G_SHL type")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unhandled G_SHL type"; }
 } while (false);
  return false;
}

auto Shl = MIB.buildInstr(Opc, {DstReg}, {Src1Reg});
if (ImmVal)
  Shl.addImm(*ImmVal);
else
  Shl.addUse(Src2Reg);
constrainSelectedInstRegOperands(*Shl, TII, TRI, RBI);
I.eraseFromParent();
return true;
1870}

1872bool AArch64InstructionSelector::selectVectorAshrLshr(
  MachineInstr &I, MachineRegisterInfo &MRI) {
assert(I.getOpcode() == TargetOpcode::G_ASHR ||(static_cast <bool> (I.getOpcode() == TargetOpcode::G_ASHR
 || I.getOpcode() == TargetOpcode::G_LSHR) ? void (0) : __assert_fail
 ("I.getOpcode() == TargetOpcode::G_ASHR || I.getOpcode() == TargetOpcode::G_LSHR"
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1875, __extension__ __PRETTY_FUNCTION__))
       I.getOpcode() == TargetOpcode::G_LSHR)(static_cast <bool> (I.getOpcode() == TargetOpcode::G_ASHR
 || I.getOpcode() == TargetOpcode::G_LSHR) ? void (0) : __assert_fail
 ("I.getOpcode() == TargetOpcode::G_ASHR || I.getOpcode() == TargetOpcode::G_LSHR"
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 1875, __extension__ __PRETTY_FUNCTION__));
Register DstReg = I.getOperand(0).getReg();
const LLT Ty = MRI.getType(DstReg);
Register Src1Reg = I.getOperand(1).getReg();
Register Src2Reg = I.getOperand(2).getReg();

if (!Ty.isVector())
  return false;

bool IsASHR = I.getOpcode() == TargetOpcode::G_ASHR;

// We expect the immediate case to be lowered in the PostLegalCombiner to
// AArch64ISD::VASHR or AArch64ISD::VLSHR equivalents.

// There is not a shift right register instruction, but the shift left
// register instruction takes a signed value, where negative numbers specify a
// right shift.

unsigned Opc = 0;
unsigned NegOpc = 0;
const TargetRegisterClass *RC =
    getRegClassForTypeOnBank(Ty, RBI.getRegBank(AArch64::FPRRegBankID));
if (Ty == LLT::fixed_vector(2, 64)) {
  Opc = IsASHR ? AArch64::SSHLv2i64 : AArch64::USHLv2i64;
  NegOpc = AArch64::NEGv2i64;
} else if (Ty == LLT::fixed_vector(4, 32)) {
  Opc = IsASHR ? AArch64::SSHLv4i32 : AArch64::USHLv4i32;
  NegOpc = AArch64::NEGv4i32;
} else if (Ty == LLT::fixed_vector(2, 32)) {
  Opc = IsASHR ? AArch64::SSHLv2i32 : AArch64::USHLv2i32;
  NegOpc = AArch64::NEGv2i32;
} else if (Ty == LLT::fixed_vector(4, 16)) {
  Opc = IsASHR ? AArch64::SSHLv4i16 : AArch64::USHLv4i16;
  NegOpc = AArch64::NEGv4i16;
} else if (Ty == LLT::fixed_vector(8, 16)) {
  Opc = IsASHR ? AArch64::SSHLv8i16 : AArch64::USHLv8i16;
  NegOpc = AArch64::NEGv8i16;
} else if (Ty == LLT::fixed_vector(16, 8)) {
  Opc = IsASHR ? AArch64::SSHLv16i8 : AArch64::USHLv16i8;
  NegOpc = AArch64::NEGv16i8;
} else if (Ty == LLT::fixed_vector(8, 8)) {
  Opc = IsASHR ? AArch64::SSHLv8i8 : AArch64::USHLv8i8;
  NegOpc = AArch64::NEGv8i8;
} else {
  LLVM_DEBUG(dbgs() << "Unhandled G_ASHR type")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unhandled G_ASHR type"; }
 } while (false);
  return false;
}

auto Neg = MIB.buildInstr(NegOpc, {RC}, {Src2Reg});
constrainSelectedInstRegOperands(*Neg, TII, TRI, RBI);
auto SShl = MIB.buildInstr(Opc, {DstReg}, {Src1Reg, Neg});
constrainSelectedInstRegOperands(*SShl, TII, TRI, RBI);
I.eraseFromParent();
return true;
1929}

1931bool AArch64InstructionSelector::selectVaStartAAPCS(
  MachineInstr &I, MachineFunction &MF, MachineRegisterInfo &MRI) const {
return false;
1934}

1936bool AArch64InstructionSelector::selectVaStartDarwin(
  MachineInstr &I, MachineFunction &MF, MachineRegisterInfo &MRI) const {
AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>();
Register ListReg = I.getOperand(0).getReg();

Register ArgsAddrReg = MRI.createVirtualRegister(&AArch64::GPR64RegClass);

int FrameIdx = FuncInfo->getVarArgsStackIndex();
if (MF.getSubtarget<AArch64Subtarget>().isCallingConvWin64(
        MF.getFunction().getCallingConv())) {
  FrameIdx = FuncInfo->getVarArgsGPRSize() > 0
                 ? FuncInfo->getVarArgsGPRIndex()
                 : FuncInfo->getVarArgsStackIndex();
}

auto MIB =
    BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(AArch64::ADDXri))
        .addDef(ArgsAddrReg)
        .addFrameIndex(FrameIdx)
        .addImm(0)
        .addImm(0);

constrainSelectedInstRegOperands(*MIB, TII, TRI, RBI);

MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(AArch64::STRXui))
          .addUse(ArgsAddrReg)
          .addUse(ListReg)
          .addImm(0)
          .addMemOperand(*I.memoperands_begin());

constrainSelectedInstRegOperands(*MIB, TII, TRI, RBI);
I.eraseFromParent();
return true;
1969}

1971void AArch64InstructionSelector::materializeLargeCMVal(
  MachineInstr &I, const Value *V, unsigned OpFlags) {
MachineBasicBlock &MBB = *I.getParent();
MachineFunction &MF = *MBB.getParent();
MachineRegisterInfo &MRI = MF.getRegInfo();

auto MovZ = MIB.buildInstr(AArch64::MOVZXi, {&AArch64::GPR64RegClass}, {});
MovZ->addOperand(MF, I.getOperand(1));
MovZ->getOperand(1).setTargetFlags(OpFlags | AArch64II::MO_G0 |
                                   AArch64II::MO_NC);
MovZ->addOperand(MF, MachineOperand::CreateImm(0));
constrainSelectedInstRegOperands(*MovZ, TII, TRI, RBI);

auto BuildMovK = [&](Register SrcReg, unsigned char Flags, unsigned Offset,
                     Register ForceDstReg) {
  Register DstReg = ForceDstReg
                        ? ForceDstReg
                        : MRI.createVirtualRegister(&AArch64::GPR64RegClass);
  auto MovI = MIB.buildInstr(AArch64::MOVKXi).addDef(DstReg).addUse(SrcReg);
  if (auto *GV = dyn_cast<GlobalValue>(V)) {
    MovI->addOperand(MF, MachineOperand::CreateGA(
                             GV, MovZ->getOperand(1).getOffset(), Flags));
  } else {
    MovI->addOperand(
        MF, MachineOperand::CreateBA(cast<BlockAddress>(V),
                                     MovZ->getOperand(1).getOffset(), Flags));
  }
  MovI->addOperand(MF, MachineOperand::CreateImm(Offset));
  constrainSelectedInstRegOperands(*MovI, TII, TRI, RBI);
  return DstReg;
};
Register DstReg = BuildMovK(MovZ.getReg(0),
                            AArch64II::MO_G1 | AArch64II::MO_NC, 16, 0);
DstReg = BuildMovK(DstReg, AArch64II::MO_G2 | AArch64II::MO_NC, 32, 0);
BuildMovK(DstReg, AArch64II::MO_G3, 48, I.getOperand(0).getReg());
2006}

2008bool AArch64InstructionSelector::preISelLower(MachineInstr &I) {
MachineBasicBlock &MBB = *I.getParent();
MachineFunction &MF = *MBB.getParent();
MachineRegisterInfo &MRI = MF.getRegInfo();

switch (I.getOpcode()) {
case TargetOpcode::G_STORE: {
  bool Changed = contractCrossBankCopyIntoStore(I, MRI);
  MachineOperand &SrcOp = I.getOperand(0);
  if (MRI.getType(SrcOp.getReg()).isPointer()) {
    // Allow matching with imported patterns for stores of pointers. Unlike
    // G_LOAD/G_PTR_ADD, we may not have selected all users. So, emit a copy
    // and constrain.
    auto Copy = MIB.buildCopy(LLT::scalar(64), SrcOp);
    Register NewSrc = Copy.getReg(0);
    SrcOp.setReg(NewSrc);
    RBI.constrainGenericRegister(NewSrc, AArch64::GPR64RegClass, MRI);
    Changed = true;
  }
  return Changed;
}
case TargetOpcode::G_PTR_ADD:
  return convertPtrAddToAdd(I, MRI);
case TargetOpcode::G_LOAD: {
  // For scalar loads of pointers, we try to convert the dest type from p0
  // to s64 so that our imported patterns can match. Like with the G_PTR_ADD
  // conversion, this should be ok because all users should have been
  // selected already, so the type doesn't matter for them.
  Register DstReg = I.getOperand(0).getReg();
  const LLT DstTy = MRI.getType(DstReg);
  if (!DstTy.isPointer())
    return false;
  MRI.setType(DstReg, LLT::scalar(64));
  return true;
}
case AArch64::G_DUP: {
  // Convert the type from p0 to s64 to help selection.
  LLT DstTy = MRI.getType(I.getOperand(0).getReg());
  if (!DstTy.getElementType().isPointer())
    return false;
  auto NewSrc = MIB.buildCopy(LLT::scalar(64), I.getOperand(1).getReg());
  MRI.setType(I.getOperand(0).getReg(),
              DstTy.changeElementType(LLT::scalar(64)));
  MRI.setRegClass(NewSrc.getReg(0), &AArch64::GPR64RegClass);
  I.getOperand(1).setReg(NewSrc.getReg(0));
  return true;
}
case TargetOpcode::G_UITOFP:
case TargetOpcode::G_SITOFP: {
  // If both source and destination regbanks are FPR, then convert the opcode
  // to G_SITOF so that the importer can select it to an fpr variant.
  // Otherwise, it ends up matching an fpr/gpr variant and adding a cross-bank
  // copy.
  Register SrcReg = I.getOperand(1).getReg();
  LLT SrcTy = MRI.getType(SrcReg);
  LLT DstTy = MRI.getType(I.getOperand(0).getReg());
  if (SrcTy.isVector() || SrcTy.getSizeInBits() != DstTy.getSizeInBits())
    return false;

  if (RBI.getRegBank(SrcReg, MRI, TRI)->getID() == AArch64::FPRRegBankID) {
    if (I.getOpcode() == TargetOpcode::G_SITOFP)
      I.setDesc(TII.get(AArch64::G_SITOF));
    else
      I.setDesc(TII.get(AArch64::G_UITOF));
    return true;
  }
  return false;
}
default:
  return false;
}
2079}

2081/// This lowering tries to look for G_PTR_ADD instructions and then converts
2082/// them to a standard G_ADD with a COPY on the source.
2083///
2084/// The motivation behind this is to expose the add semantics to the imported
2085/// tablegen patterns. We shouldn't need to check for uses being loads/stores,
2086/// because the selector works bottom up, uses before defs. By the time we
2087/// end up trying to select a G_PTR_ADD, we should have already attempted to
2088/// fold this into addressing modes and were therefore unsuccessful.
2089bool AArch64InstructionSelector::convertPtrAddToAdd(
  MachineInstr &I, MachineRegisterInfo &MRI) {
assert(I.getOpcode() == TargetOpcode::G_PTR_ADD && "Expected G_PTR_ADD")(static_cast <bool> (I.getOpcode() == TargetOpcode::G_PTR_ADD
 && "Expected G_PTR_ADD") ? void (0) : __assert_fail (
"I.getOpcode() == TargetOpcode::G_PTR_ADD && \"Expected G_PTR_ADD\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2091, __extension__ __PRETTY_FUNCTION__));
Register DstReg = I.getOperand(0).getReg();
Register AddOp1Reg = I.getOperand(1).getReg();
const LLT PtrTy = MRI.getType(DstReg);
if (PtrTy.getAddressSpace() != 0)
  return false;

const LLT CastPtrTy =
    PtrTy.isVector() ? LLT::fixed_vector(2, 64) : LLT::scalar(64);
auto PtrToInt = MIB.buildPtrToInt(CastPtrTy, AddOp1Reg);
// Set regbanks on the registers.
if (PtrTy.isVector())
  MRI.setRegBank(PtrToInt.getReg(0), RBI.getRegBank(AArch64::FPRRegBankID));
else
  MRI.setRegBank(PtrToInt.getReg(0), RBI.getRegBank(AArch64::GPRRegBankID));

// Now turn the %dst(p0) = G_PTR_ADD %base, off into:
// %dst(intty) = G_ADD %intbase, off
I.setDesc(TII.get(TargetOpcode::G_ADD));
MRI.setType(DstReg, CastPtrTy);
I.getOperand(1).setReg(PtrToInt.getReg(0));
if (!select(*PtrToInt)) {
  LLVM_DEBUG(dbgs() << "Failed to select G_PTRTOINT in convertPtrAddToAdd")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Failed to select G_PTRTOINT in convertPtrAddToAdd"
; } } while (false);
  return false;
}

// Also take the opportunity here to try to do some optimization.
// Try to convert this into a G_SUB if the offset is a 0-x negate idiom.
Register NegatedReg;
if (!mi_match(I.getOperand(2).getReg(), MRI, m_Neg(m_Reg(NegatedReg))))
  return true;
I.getOperand(2).setReg(NegatedReg);
I.setDesc(TII.get(TargetOpcode::G_SUB));
return true;
2125}

2127bool AArch64InstructionSelector::earlySelectSHL(MachineInstr &I,
                                              MachineRegisterInfo &MRI) {
// We try to match the immediate variant of LSL, which is actually an alias
// for a special case of UBFM. Otherwise, we fall back to the imported
// selector which will match the register variant.
assert(I.getOpcode() == TargetOpcode::G_SHL && "unexpected op")(static_cast <bool> (I.getOpcode() == TargetOpcode::G_SHL
 && "unexpected op") ? void (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_SHL && \"unexpected op\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2132, __extension__ __PRETTY_FUNCTION__));
const auto &MO = I.getOperand(2);
auto VRegAndVal = getIConstantVRegVal(MO.getReg(), MRI);
if (!VRegAndVal)
  return false;

const LLT DstTy = MRI.getType(I.getOperand(0).getReg());
if (DstTy.isVector())
  return false;
bool Is64Bit = DstTy.getSizeInBits() == 64;
auto Imm1Fn = Is64Bit ? selectShiftA_64(MO) : selectShiftA_32(MO);
auto Imm2Fn = Is64Bit ? selectShiftB_64(MO) : selectShiftB_32(MO);

if (!Imm1Fn || !Imm2Fn)
  return false;

auto NewI =
    MIB.buildInstr(Is64Bit ? AArch64::UBFMXri : AArch64::UBFMWri,
                   {I.getOperand(0).getReg()}, {I.getOperand(1).getReg()});

for (auto &RenderFn : *Imm1Fn)
  RenderFn(NewI);
for (auto &RenderFn : *Imm2Fn)
  RenderFn(NewI);

I.eraseFromParent();
return constrainSelectedInstRegOperands(*NewI, TII, TRI, RBI);
2159}

2161bool AArch64InstructionSelector::contractCrossBankCopyIntoStore(
  MachineInstr &I, MachineRegisterInfo &MRI) {
assert(I.getOpcode() == TargetOpcode::G_STORE && "Expected G_STORE")(static_cast <bool> (I.getOpcode() == TargetOpcode::G_STORE
 && "Expected G_STORE") ? void (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_STORE && \"Expected G_STORE\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2163, __extension__ __PRETTY_FUNCTION__));
// If we're storing a scalar, it doesn't matter what register bank that
// scalar is on. All that matters is the size.
//
// So, if we see something like this (with a 32-bit scalar as an example):
//
// %x:gpr(s32) = ... something ...
// %y:fpr(s32) = COPY %x:gpr(s32)
// G_STORE %y:fpr(s32)
//
// We can fix this up into something like this:
//
// G_STORE %x:gpr(s32)
//
// And then continue the selection process normally.
Register DefDstReg = getSrcRegIgnoringCopies(I.getOperand(0).getReg(), MRI);
if (!DefDstReg.isValid())
  return false;
LLT DefDstTy = MRI.getType(DefDstReg);
Register StoreSrcReg = I.getOperand(0).getReg();
LLT StoreSrcTy = MRI.getType(StoreSrcReg);

// If we get something strange like a physical register, then we shouldn't
// go any further.
if (!DefDstTy.isValid())
  return false;

// Are the source and dst types the same size?
if (DefDstTy.getSizeInBits() != StoreSrcTy.getSizeInBits())
  return false;

if (RBI.getRegBank(StoreSrcReg, MRI, TRI) ==
    RBI.getRegBank(DefDstReg, MRI, TRI))
  return false;

// We have a cross-bank copy, which is entering a store. Let's fold it.
I.getOperand(0).setReg(DefDstReg);
return true;
2201}

2203bool AArch64InstructionSelector::earlySelect(MachineInstr &I) {
assert(I.getParent() && "Instruction should be in a basic block!")(static_cast <bool> (I.getParent() && "Instruction should be in a basic block!"
) ? void (0) : __assert_fail ("I.getParent() && \"Instruction should be in a basic block!\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2204, __extension__ __PRETTY_FUNCTION__));
assert(I.getParent()->getParent() && "Instruction should be in a function!")(static_cast <bool> (I.getParent()->getParent() &&
 "Instruction should be in a function!") ? void (0) : __assert_fail
 ("I.getParent()->getParent() && \"Instruction should be in a function!\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2205, __extension__ __PRETTY_FUNCTION__));

MachineBasicBlock &MBB = *I.getParent();
MachineFunction &MF = *MBB.getParent();
MachineRegisterInfo &MRI = MF.getRegInfo();

switch (I.getOpcode()) {
case AArch64::G_DUP: {
  // Before selecting a DUP instruction, check if it is better selected as a
  // MOV or load from a constant pool.
  Register Src = I.getOperand(1).getReg();
  auto ValAndVReg = getIConstantVRegValWithLookThrough(Src, MRI);
  if (!ValAndVReg)
    return false;
  LLVMContext &Ctx = MF.getFunction().getContext();
  Register Dst = I.getOperand(0).getReg();
  auto *CV = ConstantDataVector::getSplat(
      MRI.getType(Dst).getNumElements(),
      ConstantInt::get(Type::getIntNTy(Ctx, MRI.getType(Src).getSizeInBits()),
                       ValAndVReg->Value));
  if (!emitConstantVector(Dst, CV, MIB, MRI))
    return false;
  I.eraseFromParent();
  return true;
}
case TargetOpcode::G_SEXT:
  // Check for i64 sext(i32 vector_extract) prior to tablegen to select SMOV
  // over a normal extend.
  if (selectUSMovFromExtend(I, MRI))
    return true;
  return false;
case TargetOpcode::G_BR:
  return false;
case TargetOpcode::G_SHL:
  return earlySelectSHL(I, MRI);
case TargetOpcode::G_CONSTANT: {
  bool IsZero = false;
  if (I.getOperand(1).isCImm())
    IsZero = I.getOperand(1).getCImm()->isZero();
  else if (I.getOperand(1).isImm())
    IsZero = I.getOperand(1).getImm() == 0;

  if (!IsZero)
    return false;

  Register DefReg = I.getOperand(0).getReg();
  LLT Ty = MRI.getType(DefReg);
  if (Ty.getSizeInBits() == 64) {
    I.getOperand(1).ChangeToRegister(AArch64::XZR, false);
    RBI.constrainGenericRegister(DefReg, AArch64::GPR64RegClass, MRI);
  } else if (Ty.getSizeInBits() == 32) {
    I.getOperand(1).ChangeToRegister(AArch64::WZR, false);
    RBI.constrainGenericRegister(DefReg, AArch64::GPR32RegClass, MRI);
  } else
    return false;

  I.setDesc(TII.get(TargetOpcode::COPY));
  return true;
}

case TargetOpcode::G_ADD: {
  // Check if this is being fed by a G_ICMP on either side.
  //
  // (cmp pred, x, y) + z
  //
  // In the above case, when the cmp is true, we increment z by 1. So, we can
  // fold the add into the cset for the cmp by using cinc.
  //
  // FIXME: This would probably be a lot nicer in PostLegalizerLowering.
  Register AddDst = I.getOperand(0).getReg();
  Register AddLHS = I.getOperand(1).getReg();
  Register AddRHS = I.getOperand(2).getReg();
  // Only handle scalars.
  LLT Ty = MRI.getType(AddLHS);
  if (Ty.isVector())
    return false;
  // Since G_ICMP is modeled as ADDS/SUBS/ANDS, we can handle 32 bits or 64
  // bits.
  unsigned Size = Ty.getSizeInBits();
  if (Size != 32 && Size != 64)
    return false;
  auto MatchCmp = [&](Register Reg) -> MachineInstr * {
    if (!MRI.hasOneNonDBGUse(Reg))
      return nullptr;
    // If the LHS of the add is 32 bits, then we want to fold a 32-bit
    // compare.
    if (Size == 32)
      return getOpcodeDef(TargetOpcode::G_ICMP, Reg, MRI);
    // We model scalar compares using 32-bit destinations right now.
    // If it's a 64-bit compare, it'll have 64-bit sources.
    Register ZExt;
    if (!mi_match(Reg, MRI,
                  m_OneNonDBGUse(m_GZExt(m_OneNonDBGUse(m_Reg(ZExt))))))
      return nullptr;
    auto *Cmp = getOpcodeDef(TargetOpcode::G_ICMP, ZExt, MRI);
    if (!Cmp ||
        MRI.getType(Cmp->getOperand(2).getReg()).getSizeInBits() != 64)
      return nullptr;
    return Cmp;
  };
  // Try to match
  // z + (cmp pred, x, y)
  MachineInstr *Cmp = MatchCmp(AddRHS);
  if (!Cmp) {
    // (cmp pred, x, y) + z
    std::swap(AddLHS, AddRHS);
    Cmp = MatchCmp(AddRHS);
    if (!Cmp)
      return false;
  }
  auto &PredOp = Cmp->getOperand(1);
  auto Pred = static_cast<CmpInst::Predicate>(PredOp.getPredicate());
  const AArch64CC::CondCode InvCC =
      changeICMPPredToAArch64CC(CmpInst::getInversePredicate(Pred));
  MIB.setInstrAndDebugLoc(I);
  emitIntegerCompare(/*LHS=*/Cmp->getOperand(2),
                     /*RHS=*/Cmp->getOperand(3), PredOp, MIB);
  emitCSINC(/*Dst=*/AddDst, /*Src =*/AddLHS, /*Src2=*/AddLHS, InvCC, MIB);
  I.eraseFromParent();
  return true;
}
case TargetOpcode::G_OR: {
  // Look for operations that take the lower `Width=Size-ShiftImm` bits of
  // `ShiftSrc` and insert them into the upper `Width` bits of `MaskSrc` via
  // shifting and masking that we can replace with a BFI (encoded as a BFM).
  Register Dst = I.getOperand(0).getReg();
  LLT Ty = MRI.getType(Dst);

  if (!Ty.isScalar())
    return false;

  unsigned Size = Ty.getSizeInBits();
  if (Size != 32 && Size != 64)
    return false;

  Register ShiftSrc;
  int64_t ShiftImm;
  Register MaskSrc;
  int64_t MaskImm;
  if (!mi_match(
          Dst, MRI,
          m_GOr(m_OneNonDBGUse(m_GShl(m_Reg(ShiftSrc), m_ICst(ShiftImm))),
                m_OneNonDBGUse(m_GAnd(m_Reg(MaskSrc), m_ICst(MaskImm))))))
    return false;

  if (ShiftImm > Size || ((1ULL << ShiftImm) - 1ULL) != uint64_t(MaskImm))
    return false;

  int64_t Immr = Size - ShiftImm;
  int64_t Imms = Size - ShiftImm - 1;
  unsigned Opc = Size == 32 ? AArch64::BFMWri : AArch64::BFMXri;
  emitInstr(Opc, {Dst}, {MaskSrc, ShiftSrc, Immr, Imms}, MIB);
  I.eraseFromParent();
  return true;
}
case TargetOpcode::G_FENCE: {
  if (I.getOperand(1).getImm() == 0)
    BuildMI(MBB, I, MIMetadata(I), TII.get(TargetOpcode::MEMBARRIER));
  else
    BuildMI(MBB, I, MIMetadata(I), TII.get(AArch64::DMB))
        .addImm(I.getOperand(0).getImm() == 4 ? 0x9 : 0xb);
  I.eraseFromParent();
  return true;
}
default:
  return false;
}
2372}

2374bool AArch64InstructionSelector::select(MachineInstr &I) {
assert(I.getParent() && "Instruction should be in a basic block!")(static_cast <bool> (I.getParent() && "Instruction should be in a basic block!"
) ? void (0) : __assert_fail ("I.getParent() && \"Instruction should be in a basic block!\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2375, __extension__ __PRETTY_FUNCTION__));
assert(I.getParent()->getParent() && "Instruction should be in a function!")(static_cast <bool> (I.getParent()->getParent() &&
 "Instruction should be in a function!") ? void (0) : __assert_fail
 ("I.getParent()->getParent() && \"Instruction should be in a function!\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2376, __extension__ __PRETTY_FUNCTION__));

MachineBasicBlock &MBB = *I.getParent();
MachineFunction &MF = *MBB.getParent();
MachineRegisterInfo &MRI = MF.getRegInfo();

const AArch64Subtarget *Subtarget = &MF.getSubtarget<AArch64Subtarget>();
if (Subtarget->requiresStrictAlign()) {
  // We don't support this feature yet.
  LLVM_DEBUG(dbgs() << "AArch64 GISel does not support strict-align yet\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "AArch64 GISel does not support strict-align yet\n"
; } } while (false);
  return false;
}

MIB.setInstrAndDebugLoc(I);

unsigned Opcode = I.getOpcode();
// G_PHI requires same handling as PHI
if (!I.isPreISelOpcode() || Opcode == TargetOpcode::G_PHI) {
  // Certain non-generic instructions also need some special handling.

  if (Opcode ==  TargetOpcode::LOAD_STACK_GUARD)
    return constrainSelectedInstRegOperands(I, TII, TRI, RBI);

  if (Opcode == TargetOpcode::PHI || Opcode == TargetOpcode::G_PHI) {
    const Register DefReg = I.getOperand(0).getReg();
    const LLT DefTy = MRI.getType(DefReg);

    const RegClassOrRegBank &RegClassOrBank =
      MRI.getRegClassOrRegBank(DefReg);

    const TargetRegisterClass *DefRC
      = RegClassOrBank.dyn_cast<const TargetRegisterClass *>();
    if (!DefRC) {
      if (!DefTy.isValid()) {
        LLVM_DEBUG(dbgs() << "PHI operand has no type, not a gvreg?\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "PHI operand has no type, not a gvreg?\n"
; } } while (false);
        return false;
      }
      const RegisterBank &RB = *RegClassOrBank.get<const RegisterBank *>();
      DefRC = getRegClassForTypeOnBank(DefTy, RB);
      if (!DefRC) {
        LLVM_DEBUG(dbgs() << "PHI operand has unexpected size/bank\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "PHI operand has unexpected size/bank\n"
; } } while (false);
        return false;
      }
    }

    I.setDesc(TII.get(TargetOpcode::PHI));

    return RBI.constrainGenericRegister(DefReg, *DefRC, MRI);
  }

  if (I.isCopy())
    return selectCopy(I, TII, MRI, TRI, RBI);

  if (I.isDebugInstr())
    return selectDebugInstr(I, MRI, RBI);

  return true;
}


if (I.getNumOperands() != I.getNumExplicitOperands()) {
  LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Generic instruction has unexpected implicit operands\n"
; } } while (false)
      dbgs() << "Generic instruction has unexpected implicit operands\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Generic instruction has unexpected implicit operands\n"
; } } while (false);
  return false;
}

// Try to do some lowering before we start instruction selecting. These
// lowerings are purely transformations on the input G_MIR and so selection
// must continue after any modification of the instruction.
if (preISelLower(I)) {
  Opcode = I.getOpcode(); // The opcode may have been modified, refresh it.
}

// There may be patterns where the importer can't deal with them optimally,
// but does select it to a suboptimal sequence so our custom C++ selection
// code later never has a chance to work on it. Therefore, we have an early
// selection attempt here to give priority to certain selection routines
// over the imported ones.
if (earlySelect(I))
  return true;

if (selectImpl(I, *CoverageInfo))
  return true;

LLT Ty =
    I.getOperand(0).isReg() ? MRI.getType(I.getOperand(0).getReg()) : LLT{};

switch (Opcode) {
case TargetOpcode::G_SBFX:
case TargetOpcode::G_UBFX: {
  static const unsigned OpcTable[2][2] = {
      {AArch64::UBFMWri, AArch64::UBFMXri},
      {AArch64::SBFMWri, AArch64::SBFMXri}};
  bool IsSigned = Opcode == TargetOpcode::G_SBFX;
  unsigned Size = Ty.getSizeInBits();
  unsigned Opc = OpcTable[IsSigned][Size == 64];
  auto Cst1 =
      getIConstantVRegValWithLookThrough(I.getOperand(2).getReg(), MRI);
  assert(Cst1 && "Should have gotten a constant for src 1?")(static_cast <bool> (Cst1 && "Should have gotten a constant for src 1?"
) ? void (0) : __assert_fail ("Cst1 && \"Should have gotten a constant for src 1?\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2474, __extension__ __PRETTY_FUNCTION__));
  auto Cst2 =
      getIConstantVRegValWithLookThrough(I.getOperand(3).getReg(), MRI);
  assert(Cst2 && "Should have gotten a constant for src 2?")(static_cast <bool> (Cst2 && "Should have gotten a constant for src 2?"
) ? void (0) : __assert_fail ("Cst2 && \"Should have gotten a constant for src 2?\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2477, __extension__ __PRETTY_FUNCTION__));
  auto LSB = Cst1->Value.getZExtValue();
  auto Width = Cst2->Value.getZExtValue();
  auto BitfieldInst =
      MIB.buildInstr(Opc, {I.getOperand(0)}, {I.getOperand(1)})
          .addImm(LSB)
          .addImm(LSB + Width - 1);
  I.eraseFromParent();
  return constrainSelectedInstRegOperands(*BitfieldInst, TII, TRI, RBI);
}
case TargetOpcode::G_BRCOND:
  return selectCompareBranch(I, MF, MRI);

case TargetOpcode::G_BRINDIRECT: {
  I.setDesc(TII.get(AArch64::BR));
  return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
}

case TargetOpcode::G_BRJT:
  return selectBrJT(I, MRI);

case AArch64::G_ADD_LOW: {
  // This op may have been separated from it's ADRP companion by the localizer
  // or some other code motion pass. Given that many CPUs will try to
  // macro fuse these operations anyway, select this into a MOVaddr pseudo
  // which will later be expanded into an ADRP+ADD pair after scheduling.
  MachineInstr *BaseMI = MRI.getVRegDef(I.getOperand(1).getReg());
  if (BaseMI->getOpcode() != AArch64::ADRP) {
    I.setDesc(TII.get(AArch64::ADDXri));
    I.addOperand(MachineOperand::CreateImm(0));
    return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
  }
  assert(TM.getCodeModel() == CodeModel::Small &&(static_cast <bool> (TM.getCodeModel() == CodeModel::Small
 && "Expected small code model") ? void (0) : __assert_fail
 ("TM.getCodeModel() == CodeModel::Small && \"Expected small code model\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2510, __extension__ __PRETTY_FUNCTION__))
         "Expected small code model")(static_cast <bool> (TM.getCodeModel() == CodeModel::Small
 && "Expected small code model") ? void (0) : __assert_fail
 ("TM.getCodeModel() == CodeModel::Small && \"Expected small code model\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2510, __extension__ __PRETTY_FUNCTION__));
  auto Op1 = BaseMI->getOperand(1);
  auto Op2 = I.getOperand(2);
  auto MovAddr = MIB.buildInstr(AArch64::MOVaddr, {I.getOperand(0)}, {})
                     .addGlobalAddress(Op1.getGlobal(), Op1.getOffset(),
                                       Op1.getTargetFlags())
                     .addGlobalAddress(Op2.getGlobal(), Op2.getOffset(),
                                       Op2.getTargetFlags());
  I.eraseFromParent();
  return constrainSelectedInstRegOperands(*MovAddr, TII, TRI, RBI);
}

case TargetOpcode::G_BSWAP: {
  // Handle vector types for G_BSWAP directly.
  Register DstReg = I.getOperand(0).getReg();
  LLT DstTy = MRI.getType(DstReg);

  // We should only get vector types here; everything else is handled by the
  // importer right now.
  if (!DstTy.isVector() || DstTy.getSizeInBits() > 128) {
    LLVM_DEBUG(dbgs() << "Dst type for G_BSWAP currently unsupported.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Dst type for G_BSWAP currently unsupported.\n"
; } } while (false);
    return false;
  }

  // Only handle 4 and 2 element vectors for now.
  // TODO: 16-bit elements.
  unsigned NumElts = DstTy.getNumElements();
  if (NumElts != 4 && NumElts != 2) {
    LLVM_DEBUG(dbgs() << "Unsupported number of elements for G_BSWAP.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unsupported number of elements for G_BSWAP.\n"
; } } while (false);
    return false;
  }

  // Choose the correct opcode for the supported types. Right now, that's
  // v2s32, v4s32, and v2s64.
  unsigned Opc = 0;
  unsigned EltSize = DstTy.getElementType().getSizeInBits();
  if (EltSize == 32)
    Opc = (DstTy.getNumElements() == 2) ? AArch64::REV32v8i8
                                        : AArch64::REV32v16i8;
  else if (EltSize == 64)
    Opc = AArch64::REV64v16i8;

  // We should always get something by the time we get here...
  assert(Opc != 0 && "Didn't get an opcode for G_BSWAP?")(static_cast <bool> (Opc != 0 && "Didn't get an opcode for G_BSWAP?"
) ? void (0) : __assert_fail ("Opc != 0 && \"Didn't get an opcode for G_BSWAP?\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2553, __extension__ __PRETTY_FUNCTION__));

  I.setDesc(TII.get(Opc));
  return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
}

case TargetOpcode::G_FCONSTANT:
case TargetOpcode::G_CONSTANT: {
  const bool isFP = Opcode == TargetOpcode::G_FCONSTANT;

  const LLT s8 = LLT::scalar(8);
  const LLT s16 = LLT::scalar(16);
  const LLT s32 = LLT::scalar(32);
  const LLT s64 = LLT::scalar(64);
  const LLT s128 = LLT::scalar(128);
  const LLT p0 = LLT::pointer(0, 64);

  const Register DefReg = I.getOperand(0).getReg();
  const LLT DefTy = MRI.getType(DefReg);
  const unsigned DefSize = DefTy.getSizeInBits();
  const RegisterBank &RB = *RBI.getRegBank(DefReg, MRI, TRI);

  // FIXME: Redundant check, but even less readable when factored out.
  if (isFP) {
    if (Ty != s16 && Ty != s32 && Ty != s64 && Ty != s128) {
      LLVM_DEBUG(dbgs() << "Unable to materialize FP " << Tydo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unable to materialize FP "
 << Ty << " constant, expected: " << s16 <<
 " or " << s32 << " or " << s64 << " or "
 << s128 << '\n'; } } while (false)
                        << " constant, expected: " << s16 << " or " << s32do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unable to materialize FP "
 << Ty << " constant, expected: " << s16 <<
 " or " << s32 << " or " << s64 << " or "
 << s128 << '\n'; } } while (false)
                        << " or " << s64 << " or " << s128 << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unable to materialize FP "
 << Ty << " constant, expected: " << s16 <<
 " or " << s32 << " or " << s64 << " or "
 << s128 << '\n'; } } while (false);
      return false;
    }

    if (RB.getID() != AArch64::FPRRegBankID) {
      LLVM_DEBUG(dbgs() << "Unable to materialize FP " << Tydo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unable to materialize FP "
 << Ty << " constant on bank: " << RB <<
 ", expected: FPR\n"; } } while (false)
                        << " constant on bank: " << RBdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unable to materialize FP "
 << Ty << " constant on bank: " << RB <<
 ", expected: FPR\n"; } } while (false)
                        << ", expected: FPR\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unable to materialize FP "
 << Ty << " constant on bank: " << RB <<
 ", expected: FPR\n"; } } while (false);
      return false;
    }

    // The case when we have 0.0 is covered by tablegen. Reject it here so we
    // can be sure tablegen works correctly and isn't rescued by this code.
    // 0.0 is not covered by tablegen for FP128. So we will handle this
    // scenario in the code here.
    if (DefSize != 128 && I.getOperand(1).getFPImm()->isExactlyValue(0.0))
      return false;
  } else {
    // s32 and s64 are covered by tablegen.
    if (Ty != p0 && Ty != s8 && Ty != s16) {
      LLVM_DEBUG(dbgs() << "Unable to materialize integer " << Tydo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unable to materialize integer "
 << Ty << " constant, expected: " << s32 <<
 ", " << s64 << ", or " << p0 << '\n'
; } } while (false)
                        << " constant, expected: " << s32 << ", " << s64do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unable to materialize integer "
 << Ty << " constant, expected: " << s32 <<
 ", " << s64 << ", or " << p0 << '\n'
; } } while (false)
                        << ", or " << p0 << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unable to materialize integer "
 << Ty << " constant, expected: " << s32 <<
 ", " << s64 << ", or " << p0 << '\n'
; } } while (false);
      return false;
    }

    if (RB.getID() != AArch64::GPRRegBankID) {
      LLVM_DEBUG(dbgs() << "Unable to materialize integer " << Tydo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unable to materialize integer "
 << Ty << " constant on bank: " << RB <<
 ", expected: GPR\n"; } } while (false)
                        << " constant on bank: " << RBdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unable to materialize integer "
 << Ty << " constant on bank: " << RB <<
 ", expected: GPR\n"; } } while (false)
                        << ", expected: GPR\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unable to materialize integer "
 << Ty << " constant on bank: " << RB <<
 ", expected: GPR\n"; } } while (false);
      return false;
    }
  }

  if (isFP) {
    const TargetRegisterClass &FPRRC = *getRegClassForTypeOnBank(DefTy, RB);
    // For 16, 64, and 128b values, emit a constant pool load.
    switch (DefSize) {
    default:
      llvm_unreachable("Unexpected destination size for G_FCONSTANT?")::llvm::llvm_unreachable_internal("Unexpected destination size for G_FCONSTANT?"
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2619);
    case 32:
      // For s32, use a cp load if we have optsize/minsize.
      if (!shouldOptForSize(&MF))
        break;
      [[fallthrough]];
    case 16:
    case 64:
    case 128: {
      auto *FPImm = I.getOperand(1).getFPImm();
      auto *LoadMI = emitLoadFromConstantPool(FPImm, MIB);
      if (!LoadMI) {
        LLVM_DEBUG(dbgs() << "Failed to load double constant pool entry\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Failed to load double constant pool entry\n"
; } } while (false);
        return false;
      }
      MIB.buildCopy({DefReg}, {LoadMI->getOperand(0).getReg()});
      I.eraseFromParent();
      return RBI.constrainGenericRegister(DefReg, FPRRC, MRI);
    }
    }

    // Either emit a FMOV, or emit a copy to emit a normal mov.
    assert(DefSize == 32 &&(static_cast <bool> (DefSize == 32 && "Expected constant pool loads for all sizes other than 32!"
) ? void (0) : __assert_fail ("DefSize == 32 && \"Expected constant pool loads for all sizes other than 32!\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2642, __extension__ __PRETTY_FUNCTION__))
           "Expected constant pool loads for all sizes other than 32!")(static_cast <bool> (DefSize == 32 && "Expected constant pool loads for all sizes other than 32!"
) ? void (0) : __assert_fail ("DefSize == 32 && \"Expected constant pool loads for all sizes other than 32!\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2642, __extension__ __PRETTY_FUNCTION__));
    const Register DefGPRReg =
        MRI.createVirtualRegister(&AArch64::GPR32RegClass);
    MachineOperand &RegOp = I.getOperand(0);
    RegOp.setReg(DefGPRReg);
    MIB.setInsertPt(MIB.getMBB(), std::next(I.getIterator()));
    MIB.buildCopy({DefReg}, {DefGPRReg});

    if (!RBI.constrainGenericRegister(DefReg, FPRRC, MRI)) {
      LLVM_DEBUG(dbgs() << "Failed to constrain G_FCONSTANT def operand\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Failed to constrain G_FCONSTANT def operand\n"
; } } while (false);
      return false;
    }

    MachineOperand &ImmOp = I.getOperand(1);
    // FIXME: Is going through int64_t always correct?
    ImmOp.ChangeToImmediate(
        ImmOp.getFPImm()->getValueAPF().bitcastToAPInt().getZExtValue());
  } else if (I.getOperand(1).isCImm()) {
    uint64_t Val = I.getOperand(1).getCImm()->getZExtValue();
    I.getOperand(1).ChangeToImmediate(Val);
  } else if (I.getOperand(1).isImm()) {
    uint64_t Val = I.getOperand(1).getImm();
    I.getOperand(1).ChangeToImmediate(Val);
  }

  const unsigned MovOpc =
      DefSize == 64 ? AArch64::MOVi64imm : AArch64::MOVi32imm;
  I.setDesc(TII.get(MovOpc));
  constrainSelectedInstRegOperands(I, TII, TRI, RBI);
  return true;
}
case TargetOpcode::G_EXTRACT: {
  Register DstReg = I.getOperand(0).getReg();
  Register SrcReg = I.getOperand(1).getReg();
  LLT SrcTy = MRI.getType(SrcReg);
  LLT DstTy = MRI.getType(DstReg);
  (void)DstTy;
  unsigned SrcSize = SrcTy.getSizeInBits();

  if (SrcTy.getSizeInBits() > 64) {
    // This should be an extract of an s128, which is like a vector extract.
    if (SrcTy.getSizeInBits() != 128)
      return false;
    // Only support extracting 64 bits from an s128 at the moment.
    if (DstTy.getSizeInBits() != 64)
      return false;

    unsigned Offset = I.getOperand(2).getImm();
    if (Offset % 64 != 0)
      return false;

    // Check we have the right regbank always.
    const RegisterBank &SrcRB = *RBI.getRegBank(SrcReg, MRI, TRI);
    const RegisterBank &DstRB = *RBI.getRegBank(DstReg, MRI, TRI);
    assert(SrcRB.getID() == DstRB.getID() && "Wrong extract regbank!")(static_cast <bool> (SrcRB.getID() == DstRB.getID() &&
 "Wrong extract regbank!") ? void (0) : __assert_fail ("SrcRB.getID() == DstRB.getID() && \"Wrong extract regbank!\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2696, __extension__ __PRETTY_FUNCTION__));

    if (SrcRB.getID() == AArch64::GPRRegBankID) {
      auto NewI =
          MIB.buildInstr(TargetOpcode::COPY, {DstReg}, {})
              .addUse(SrcReg, 0,
                      Offset == 0 ? AArch64::sube64 : AArch64::subo64);
      constrainOperandRegClass(MF, TRI, MRI, TII, RBI, *NewI,
                               AArch64::GPR64RegClass, NewI->getOperand(0));
      I.eraseFromParent();
      return true;
    }

    // Emit the same code as a vector extract.
    // Offset must be a multiple of 64.
    unsigned LaneIdx = Offset / 64;
    MachineInstr *Extract = emitExtractVectorElt(
        DstReg, DstRB, LLT::scalar(64), SrcReg, LaneIdx, MIB);
    if (!Extract)
      return false;
    I.eraseFromParent();
    return true;
  }

  I.setDesc(TII.get(SrcSize == 64 ? AArch64::UBFMXri : AArch64::UBFMWri));
  MachineInstrBuilder(MF, I).addImm(I.getOperand(2).getImm() +
                                    Ty.getSizeInBits() - 1);

  if (SrcSize < 64) {
    assert(SrcSize == 32 && DstTy.getSizeInBits() == 16 &&(static_cast <bool> (SrcSize == 32 && DstTy.getSizeInBits
() == 16 && "unexpected G_EXTRACT types") ? void (0) :
 __assert_fail ("SrcSize == 32 && DstTy.getSizeInBits() == 16 && \"unexpected G_EXTRACT types\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2726, __extension__ __PRETTY_FUNCTION__))
           "unexpected G_EXTRACT types")(static_cast <bool> (SrcSize == 32 && DstTy.getSizeInBits
() == 16 && "unexpected G_EXTRACT types") ? void (0) :
 __assert_fail ("SrcSize == 32 && DstTy.getSizeInBits() == 16 && \"unexpected G_EXTRACT types\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2726, __extension__ __PRETTY_FUNCTION__));
    return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
  }

  DstReg = MRI.createGenericVirtualRegister(LLT::scalar(64));
  MIB.setInsertPt(MIB.getMBB(), std::next(I.getIterator()));
  MIB.buildInstr(TargetOpcode::COPY, {I.getOperand(0).getReg()}, {})
      .addReg(DstReg, 0, AArch64::sub_32);
  RBI.constrainGenericRegister(I.getOperand(0).getReg(),
                               AArch64::GPR32RegClass, MRI);
  I.getOperand(0).setReg(DstReg);

  return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
}

case TargetOpcode::G_INSERT: {
  LLT SrcTy = MRI.getType(I.getOperand(2).getReg());
  LLT DstTy = MRI.getType(I.getOperand(0).getReg());
  unsigned DstSize = DstTy.getSizeInBits();
  // Larger inserts are vectors, same-size ones should be something else by
  // now (split up or turned into COPYs).
  if (Ty.getSizeInBits() > 64 || SrcTy.getSizeInBits() > 32)
    return false;

  I.setDesc(TII.get(DstSize == 64 ? AArch64::BFMXri : AArch64::BFMWri));
  unsigned LSB = I.getOperand(3).getImm();
  unsigned Width = MRI.getType(I.getOperand(2).getReg()).getSizeInBits();
  I.getOperand(3).setImm((DstSize - LSB) % DstSize);
  MachineInstrBuilder(MF, I).addImm(Width - 1);

  if (DstSize < 64) {
    assert(DstSize == 32 && SrcTy.getSizeInBits() == 16 &&(static_cast <bool> (DstSize == 32 && SrcTy.getSizeInBits
() == 16 && "unexpected G_INSERT types") ? void (0) :
 __assert_fail ("DstSize == 32 && SrcTy.getSizeInBits() == 16 && \"unexpected G_INSERT types\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2758, __extension__ __PRETTY_FUNCTION__))
           "unexpected G_INSERT types")(static_cast <bool> (DstSize == 32 && SrcTy.getSizeInBits
() == 16 && "unexpected G_INSERT types") ? void (0) :
 __assert_fail ("DstSize == 32 && SrcTy.getSizeInBits() == 16 && \"unexpected G_INSERT types\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2758, __extension__ __PRETTY_FUNCTION__));
    return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
  }

  Register SrcReg = MRI.createGenericVirtualRegister(LLT::scalar(64));
  BuildMI(MBB, I.getIterator(), I.getDebugLoc(),
          TII.get(AArch64::SUBREG_TO_REG))
      .addDef(SrcReg)
      .addImm(0)
      .addUse(I.getOperand(2).getReg())
      .addImm(AArch64::sub_32);
  RBI.constrainGenericRegister(I.getOperand(2).getReg(),
                               AArch64::GPR32RegClass, MRI);
  I.getOperand(2).setReg(SrcReg);

  return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
}
case TargetOpcode::G_FRAME_INDEX: {
  // allocas and G_FRAME_INDEX are only supported in addrspace(0).
  if (Ty != LLT::pointer(0, 64)) {
    LLVM_DEBUG(dbgs() << "G_FRAME_INDEX pointer has type: " << Tydo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "G_FRAME_INDEX pointer has type: "
 << Ty << ", expected: " << LLT::pointer(0,
 64) << '\n'; } } while (false)
                      << ", expected: " << LLT::pointer(0, 64) << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "G_FRAME_INDEX pointer has type: "
 << Ty << ", expected: " << LLT::pointer(0,
 64) << '\n'; } } while (false);
    return false;
  }
  I.setDesc(TII.get(AArch64::ADDXri));

  // MOs for a #0 shifted immediate.
  I.addOperand(MachineOperand::CreateImm(0));
  I.addOperand(MachineOperand::CreateImm(0));

  return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
}

case TargetOpcode::G_GLOBAL_VALUE: {
  auto GV = I.getOperand(1).getGlobal();
  if (GV->isThreadLocal())
    return selectTLSGlobalValue(I, MRI);

  unsigned OpFlags = STI.ClassifyGlobalReference(GV, TM);
  if (OpFlags & AArch64II::MO_GOT) {
    I.setDesc(TII.get(AArch64::LOADgot));
    I.getOperand(1).setTargetFlags(OpFlags);
  } else if (TM.getCodeModel() == CodeModel::Large) {
    // Materialize the global using movz/movk instructions.
    materializeLargeCMVal(I, GV, OpFlags);
    I.eraseFromParent();
    return true;
  } else if (TM.getCodeModel() == CodeModel::Tiny) {
    I.setDesc(TII.get(AArch64::ADR));
    I.getOperand(1).setTargetFlags(OpFlags);
  } else {
    I.setDesc(TII.get(AArch64::MOVaddr));
    I.getOperand(1).setTargetFlags(OpFlags | AArch64II::MO_PAGE);
    MachineInstrBuilder MIB(MF, I);
    MIB.addGlobalAddress(GV, I.getOperand(1).getOffset(),
                         OpFlags | AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
  }
  return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
}

case TargetOpcode::G_ZEXTLOAD:
case TargetOpcode::G_LOAD:
case TargetOpcode::G_STORE: {
  GLoadStore &LdSt = cast<GLoadStore>(I);
  bool IsZExtLoad = I.getOpcode() == TargetOpcode::G_ZEXTLOAD;
  LLT PtrTy = MRI.getType(LdSt.getPointerReg());

  if (PtrTy != LLT::pointer(0, 64)) {
    LLVM_DEBUG(dbgs() << "Load/Store pointer has type: " << PtrTydo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Load/Store pointer has type: "
 << PtrTy << ", expected: " << LLT::pointer
(0, 64) << '\n'; } } while (false)
                      << ", expected: " << LLT::pointer(0, 64) << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Load/Store pointer has type: "
 << PtrTy << ", expected: " << LLT::pointer
(0, 64) << '\n'; } } while (false);
    return false;
  }

  uint64_t MemSizeInBytes = LdSt.getMemSize();
  unsigned MemSizeInBits = LdSt.getMemSizeInBits();
  AtomicOrdering Order = LdSt.getMMO().getSuccessOrdering();

  // Need special instructions for atomics that affect ordering.
  if (Order != AtomicOrdering::NotAtomic &&
      Order != AtomicOrdering::Unordered &&
      Order != AtomicOrdering::Monotonic) {
    assert(!isa<GZExtLoad>(LdSt))(static_cast <bool> (!isa<GZExtLoad>(LdSt)) ? void
 (0) : __assert_fail ("!isa<GZExtLoad>(LdSt)", "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2839, __extension__ __PRETTY_FUNCTION__));
    if (MemSizeInBytes > 64)
      return false;

    if (isa<GLoad>(LdSt)) {
      static constexpr unsigned LDAPROpcodes[] = {
          AArch64::LDAPRB, AArch64::LDAPRH, AArch64::LDAPRW, AArch64::LDAPRX};
      static constexpr unsigned LDAROpcodes[] = {
          AArch64::LDARB, AArch64::LDARH, AArch64::LDARW, AArch64::LDARX};
      ArrayRef<unsigned> Opcodes =
          STI.hasRCPC() && Order != AtomicOrdering::SequentiallyConsistent
              ? LDAPROpcodes
              : LDAROpcodes;
      I.setDesc(TII.get(Opcodes[Log2_32(MemSizeInBytes)]));
    } else {
      static constexpr unsigned Opcodes[] = {AArch64::STLRB, AArch64::STLRH,
                                             AArch64::STLRW, AArch64::STLRX};
      Register ValReg = LdSt.getReg(0);
      if (MRI.getType(ValReg).getSizeInBits() == 64 && MemSizeInBits != 64) {
        // Emit a subreg copy of 32 bits.
        Register NewVal = MRI.createVirtualRegister(&AArch64::GPR32RegClass);
        MIB.buildInstr(TargetOpcode::COPY, {NewVal}, {})
            .addReg(I.getOperand(0).getReg(), 0, AArch64::sub_32);
        I.getOperand(0).setReg(NewVal);
      }
      I.setDesc(TII.get(Opcodes[Log2_32(MemSizeInBytes)]));
    }
    constrainSelectedInstRegOperands(I, TII, TRI, RBI);
    return true;
  }

2870#ifndef NDEBUG
  const Register PtrReg = LdSt.getPointerReg();
  const RegisterBank &PtrRB = *RBI.getRegBank(PtrReg, MRI, TRI);
  // Check that the pointer register is valid.
  assert(PtrRB.getID() == AArch64::GPRRegBankID &&(static_cast <bool> (PtrRB.getID() == AArch64::GPRRegBankID
 && "Load/Store pointer operand isn't a GPR") ? void (
0) : __assert_fail ("PtrRB.getID() == AArch64::GPRRegBankID && \"Load/Store pointer operand isn't a GPR\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2875, __extension__ __PRETTY_FUNCTION__))
         "Load/Store pointer operand isn't a GPR")(static_cast <bool> (PtrRB.getID() == AArch64::GPRRegBankID
 && "Load/Store pointer operand isn't a GPR") ? void (
0) : __assert_fail ("PtrRB.getID() == AArch64::GPRRegBankID && \"Load/Store pointer operand isn't a GPR\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2875, __extension__ __PRETTY_FUNCTION__));
  assert(MRI.getType(PtrReg).isPointer() &&(static_cast <bool> (MRI.getType(PtrReg).isPointer() &&
 "Load/Store pointer operand isn't a pointer") ? void (0) : __assert_fail
 ("MRI.getType(PtrReg).isPointer() && \"Load/Store pointer operand isn't a pointer\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2877, __extension__ __PRETTY_FUNCTION__))
         "Load/Store pointer operand isn't a pointer")(static_cast <bool> (MRI.getType(PtrReg).isPointer() &&
 "Load/Store pointer operand isn't a pointer") ? void (0) : __assert_fail
 ("MRI.getType(PtrReg).isPointer() && \"Load/Store pointer operand isn't a pointer\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 2877, __extension__ __PRETTY_FUNCTION__));
2878#endif

  const Register ValReg = LdSt.getReg(0);
  const LLT ValTy = MRI.getType(ValReg);
  const RegisterBank &RB = *RBI.getRegBank(ValReg, MRI, TRI);

  // The code below doesn't support truncating stores, so we need to split it
  // again.
  if (isa<GStore>(LdSt) && ValTy.getSizeInBits() > MemSizeInBits) {
    unsigned SubReg;
    LLT MemTy = LdSt.getMMO().getMemoryType();
    auto *RC = getRegClassForTypeOnBank(MemTy, RB);
    if (!getSubRegForClass(RC, TRI, SubReg))
      return false;

    // Generate a subreg copy.
    auto Copy = MIB.buildInstr(TargetOpcode::COPY, {MemTy}, {})
                    .addReg(ValReg, 0, SubReg)
                    .getReg(0);
    RBI.constrainGenericRegister(Copy, *RC, MRI);
    LdSt.getOperand(0).setReg(Copy);
  } else if (isa<GLoad>(LdSt) && ValTy.getSizeInBits() > MemSizeInBits) {
    // If this is an any-extending load from the FPR bank, split it into a regular
    // load + extend.
    if (RB.getID() == AArch64::FPRRegBankID) {
      unsigned SubReg;
      LLT MemTy = LdSt.getMMO().getMemoryType();
      auto *RC = getRegClassForTypeOnBank(MemTy, RB);
      if (!getSubRegForClass(RC, TRI, SubReg))
        return false;
      Register OldDst = LdSt.getReg(0);
      Register NewDst =
          MRI.createGenericVirtualRegister(LdSt.getMMO().getMemoryType());
      LdSt.getOperand(0).setReg(NewDst);
      MRI.setRegBank(NewDst, RB);
      // Generate a SUBREG_TO_REG to extend it.
      MIB.setInsertPt(MIB.getMBB(), std::next(LdSt.getIterator()));
      MIB.buildInstr(AArch64::SUBREG_TO_REG, {OldDst}, {})
          .addImm(0)
          .addUse(NewDst)
          .addImm(SubReg);
      auto SubRegRC = getRegClassForTypeOnBank(MRI.getType(OldDst), RB);
      RBI.constrainGenericRegister(OldDst, *SubRegRC, MRI);
      MIB.setInstr(LdSt);
    }
  }

  // Helper lambda for partially selecting I. Either returns the original
  // instruction with an updated opcode, or a new instruction.
  auto SelectLoadStoreAddressingMode = [&]() -> MachineInstr * {
    bool IsStore = isa<GStore>(I);
1
Assuming 'I' is not a 'GStore'→
    const unsigned NewOpc =
        selectLoadStoreUIOp(I.getOpcode(), RB.getID(), MemSizeInBits);
    if (NewOpc == I.getOpcode())
2
←
Taking false branch→
      return nullptr;
    // Check if we can fold anything into the addressing mode.
    auto AddrModeFns =
        selectAddrModeIndexed(I.getOperand(1), MemSizeInBytes);
3
←
Calling 'AArch64InstructionSelector::selectAddrModeIndexed'→
    if (!AddrModeFns) {
      // Can't fold anything. Use the original instruction.
      I.setDesc(TII.get(NewOpc));
      I.addOperand(MachineOperand::CreateImm(0));
      return &I;
    }

    // Folded something. Create a new instruction and return it.
    auto NewInst = MIB.buildInstr(NewOpc, {}, {}, I.getFlags());
    Register CurValReg = I.getOperand(0).getReg();
    IsStore ? NewInst.addUse(CurValReg) : NewInst.addDef(CurValReg);
    NewInst.cloneMemRefs(I);
    for (auto &Fn : *AddrModeFns)
      Fn(NewInst);
    I.eraseFromParent();
    return &*NewInst;
  };

  MachineInstr *LoadStore = SelectLoadStoreAddressingMode();
  if (!LoadStore)
    return false;

  // If we're storing a 0, use WZR/XZR.
  if (Opcode == TargetOpcode::G_STORE) {
    auto CVal = getIConstantVRegValWithLookThrough(
        LoadStore->getOperand(0).getReg(), MRI);
    if (CVal && CVal->Value == 0) {
      switch (LoadStore->getOpcode()) {
      case AArch64::STRWui:
      case AArch64::STRHHui:
      case AArch64::STRBBui:
        LoadStore->getOperand(0).setReg(AArch64::WZR);
        break;
      case AArch64::STRXui:
        LoadStore->getOperand(0).setReg(AArch64::XZR);
        break;
      }
    }
  }

  if (IsZExtLoad) {
    // The zextload from a smaller type to i32 should be handled by the
    // importer.
    if (MRI.getType(LoadStore->getOperand(0).getReg()).getSizeInBits() != 64)
      return false;
    // If we have a ZEXTLOAD then change the load's type to be a narrower reg
    // and zero_extend with SUBREG_TO_REG.
    Register LdReg = MRI.createVirtualRegister(&AArch64::GPR32RegClass);
    Register DstReg = LoadStore->getOperand(0).getReg();
    LoadStore->getOperand(0).setReg(LdReg);

    MIB.setInsertPt(MIB.getMBB(), std::next(LoadStore->getIterator()));
    MIB.buildInstr(AArch64::SUBREG_TO_REG, {DstReg}, {})
        .addImm(0)
        .addUse(LdReg)
        .addImm(AArch64::sub_32);
    constrainSelectedInstRegOperands(*LoadStore, TII, TRI, RBI);
    return RBI.constrainGenericRegister(DstReg, AArch64::GPR64allRegClass,
                                        MRI);
  }
  return constrainSelectedInstRegOperands(*LoadStore, TII, TRI, RBI);
}

case TargetOpcode::G_SMULH:
case TargetOpcode::G_UMULH: {
  // Reject the various things we don't support yet.
  if (unsupportedBinOp(I, RBI, MRI, TRI))
    return false;

  const Register DefReg = I.getOperand(0).getReg();
  const RegisterBank &RB = *RBI.getRegBank(DefReg, MRI, TRI);

  if (RB.getID() != AArch64::GPRRegBankID) {
    LLVM_DEBUG(dbgs() << "G_[SU]MULH on bank: " << RB << ", expected: GPR\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "G_[SU]MULH on bank: " <<
 RB << ", expected: GPR\n"; } } while (false);
    return false;
  }

  if (Ty != LLT::scalar(64)) {
    LLVM_DEBUG(dbgs() << "G_[SU]MULH has type: " << Tydo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "G_[SU]MULH has type: " <<
 Ty << ", expected: " << LLT::scalar(64) <<
 '\n'; } } while (false)
                      << ", expected: " << LLT::scalar(64) << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "G_[SU]MULH has type: " <<
 Ty << ", expected: " << LLT::scalar(64) <<
 '\n'; } } while (false);
    return false;
  }

  unsigned NewOpc = I.getOpcode() == TargetOpcode::G_SMULH ? AArch64::SMULHrr
                                                           : AArch64::UMULHrr;
  I.setDesc(TII.get(NewOpc));

  // Now that we selected an opcode, we need to constrain the register
  // operands to use appropriate classes.
  return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
}
case TargetOpcode::G_LSHR:
case TargetOpcode::G_ASHR:
  if (MRI.getType(I.getOperand(0).getReg()).isVector())
    return selectVectorAshrLshr(I, MRI);
  [[fallthrough]];
case TargetOpcode::G_SHL:
  if (Opcode == TargetOpcode::G_SHL &&
      MRI.getType(I.getOperand(0).getReg()).isVector())
    return selectVectorSHL(I, MRI);

  // These shifts were legalized to have 64 bit shift amounts because we
  // want to take advantage of the selection patterns that assume the
  // immediates are s64s, however, selectBinaryOp will assume both operands
  // will have the same bit size.
  {
    Register SrcReg = I.getOperand(1).getReg();
    Register ShiftReg = I.getOperand(2).getReg();
    const LLT ShiftTy = MRI.getType(ShiftReg);
    const LLT SrcTy = MRI.getType(SrcReg);
    if (!SrcTy.isVector() && SrcTy.getSizeInBits() == 32 &&
        ShiftTy.getSizeInBits() == 64) {
      assert(!ShiftTy.isVector() && "unexpected vector shift ty")(static_cast <bool> (!ShiftTy.isVector() && "unexpected vector shift ty"
) ? void (0) : __assert_fail ("!ShiftTy.isVector() && \"unexpected vector shift ty\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3048, __extension__ __PRETTY_FUNCTION__));
      // Insert a subregister copy to implement a 64->32 trunc
      auto Trunc = MIB.buildInstr(TargetOpcode::COPY, {SrcTy}, {})
                       .addReg(ShiftReg, 0, AArch64::sub_32);
      MRI.setRegBank(Trunc.getReg(0), RBI.getRegBank(AArch64::GPRRegBankID));
      I.getOperand(2).setReg(Trunc.getReg(0));
    }
  }
  [[fallthrough]];
case TargetOpcode::G_OR: {
  // Reject the various things we don't support yet.
  if (unsupportedBinOp(I, RBI, MRI, TRI))
    return false;

  const unsigned OpSize = Ty.getSizeInBits();

  const Register DefReg = I.getOperand(0).getReg();
  const RegisterBank &RB = *RBI.getRegBank(DefReg, MRI, TRI);

  const unsigned NewOpc = selectBinaryOp(I.getOpcode(), RB.getID(), OpSize);
  if (NewOpc == I.getOpcode())
    return false;

  I.setDesc(TII.get(NewOpc));
  // FIXME: Should the type be always reset in setDesc?

  // Now that we selected an opcode, we need to constrain the register
  // operands to use appropriate classes.
  return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
}

case TargetOpcode::G_PTR_ADD: {
  emitADD(I.getOperand(0).getReg(), I.getOperand(1), I.getOperand(2), MIB);
  I.eraseFromParent();
  return true;
}
case TargetOpcode::G_SADDO:
case TargetOpcode::G_UADDO:
case TargetOpcode::G_SSUBO:
case TargetOpcode::G_USUBO: {
  // Emit the operation and get the correct condition code.
  auto OpAndCC = emitOverflowOp(Opcode, I.getOperand(0).getReg(),
                                I.getOperand(2), I.getOperand(3), MIB);

  // Now, put the overflow result in the register given by the first operand
  // to the overflow op. CSINC increments the result when the predicate is
  // false, so to get the increment when it's true, we need to use the
  // inverse. In this case, we want to increment when carry is set.
  Register ZReg = AArch64::WZR;
  emitCSINC(/*Dst=*/I.getOperand(1).getReg(), /*Src1=*/ZReg, /*Src2=*/ZReg,
            getInvertedCondCode(OpAndCC.second), MIB);
  I.eraseFromParent();
  return true;
}

case TargetOpcode::G_PTRMASK: {
  Register MaskReg = I.getOperand(2).getReg();
  std::optional<int64_t> MaskVal = getIConstantVRegSExtVal(MaskReg, MRI);
  // TODO: Implement arbitrary cases
  if (!MaskVal || !isShiftedMask_64(*MaskVal))
    return false;

  uint64_t Mask = *MaskVal;
  I.setDesc(TII.get(AArch64::ANDXri));
  I.getOperand(2).ChangeToImmediate(
      AArch64_AM::encodeLogicalImmediate(Mask, 64));

  return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
}
case TargetOpcode::G_PTRTOINT:
case TargetOpcode::G_TRUNC: {
  const LLT DstTy = MRI.getType(I.getOperand(0).getReg());
  const LLT SrcTy = MRI.getType(I.getOperand(1).getReg());

  const Register DstReg = I.getOperand(0).getReg();
  const Register SrcReg = I.getOperand(1).getReg();

  const RegisterBank &DstRB = *RBI.getRegBank(DstReg, MRI, TRI);
  const RegisterBank &SrcRB = *RBI.getRegBank(SrcReg, MRI, TRI);

  if (DstRB.getID() != SrcRB.getID()) {
    LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "G_TRUNC/G_PTRTOINT input/output on different banks\n"
; } } while (false)
        dbgs() << "G_TRUNC/G_PTRTOINT input/output on different banks\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "G_TRUNC/G_PTRTOINT input/output on different banks\n"
; } } while (false);
    return false;
  }

  if (DstRB.getID() == AArch64::GPRRegBankID) {
    const TargetRegisterClass *DstRC = getRegClassForTypeOnBank(DstTy, DstRB);
    if (!DstRC)
      return false;

    const TargetRegisterClass *SrcRC = getRegClassForTypeOnBank(SrcTy, SrcRB);
    if (!SrcRC)
      return false;

    if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, MRI) ||
        !RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) {
      LLVM_DEBUG(dbgs() << "Failed to constrain G_TRUNC/G_PTRTOINT\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Failed to constrain G_TRUNC/G_PTRTOINT\n"
; } } while (false);
      return false;
    }

    if (DstRC == SrcRC) {
      // Nothing to be done
    } else if (Opcode == TargetOpcode::G_TRUNC && DstTy == LLT::scalar(32) &&
               SrcTy == LLT::scalar(64)) {
      llvm_unreachable("TableGen can import this case")::llvm::llvm_unreachable_internal("TableGen can import this case"
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3153);
      return false;
    } else if (DstRC == &AArch64::GPR32RegClass &&
               SrcRC == &AArch64::GPR64RegClass) {
      I.getOperand(1).setSubReg(AArch64::sub_32);
    } else {
      LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unhandled mismatched classes in G_TRUNC/G_PTRTOINT\n"
; } } while (false)
          dbgs() << "Unhandled mismatched classes in G_TRUNC/G_PTRTOINT\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unhandled mismatched classes in G_TRUNC/G_PTRTOINT\n"
; } } while (false);
      return false;
    }

    I.setDesc(TII.get(TargetOpcode::COPY));
    return true;
  } else if (DstRB.getID() == AArch64::FPRRegBankID) {
    if (DstTy == LLT::fixed_vector(4, 16) &&
        SrcTy == LLT::fixed_vector(4, 32)) {
      I.setDesc(TII.get(AArch64::XTNv4i16));
      constrainSelectedInstRegOperands(I, TII, TRI, RBI);
      return true;
    }

    if (!SrcTy.isVector() && SrcTy.getSizeInBits() == 128) {
      MachineInstr *Extract = emitExtractVectorElt(
          DstReg, DstRB, LLT::scalar(DstTy.getSizeInBits()), SrcReg, 0, MIB);
      if (!Extract)
        return false;
      I.eraseFromParent();
      return true;
    }

    // We might have a vector G_PTRTOINT, in which case just emit a COPY.
    if (Opcode == TargetOpcode::G_PTRTOINT) {
      assert(DstTy.isVector() && "Expected an FPR ptrtoint to be a vector")(static_cast <bool> (DstTy.isVector() && "Expected an FPR ptrtoint to be a vector"
) ? void (0) : __assert_fail ("DstTy.isVector() && \"Expected an FPR ptrtoint to be a vector\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3185, __extension__ __PRETTY_FUNCTION__));
      I.setDesc(TII.get(TargetOpcode::COPY));
      return selectCopy(I, TII, MRI, TRI, RBI);
    }
  }

  return false;
}

case TargetOpcode::G_ANYEXT: {
  if (selectUSMovFromExtend(I, MRI))
    return true;

  const Register DstReg = I.getOperand(0).getReg();
  const Register SrcReg = I.getOperand(1).getReg();

  const RegisterBank &RBDst = *RBI.getRegBank(DstReg, MRI, TRI);
  if (RBDst.getID() != AArch64::GPRRegBankID) {
    LLVM_DEBUG(dbgs() << "G_ANYEXT on bank: " << RBDstdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "G_ANYEXT on bank: " <<
 RBDst << ", expected: GPR\n"; } } while (false)
                      << ", expected: GPR\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "G_ANYEXT on bank: " <<
 RBDst << ", expected: GPR\n"; } } while (false);
    return false;
  }

  const RegisterBank &RBSrc = *RBI.getRegBank(SrcReg, MRI, TRI);
  if (RBSrc.getID() != AArch64::GPRRegBankID) {
    LLVM_DEBUG(dbgs() << "G_ANYEXT on bank: " << RBSrcdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "G_ANYEXT on bank: " <<
 RBSrc << ", expected: GPR\n"; } } while (false)
                      << ", expected: GPR\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "G_ANYEXT on bank: " <<
 RBSrc << ", expected: GPR\n"; } } while (false);
    return false;
  }

  const unsigned DstSize = MRI.getType(DstReg).getSizeInBits();

  if (DstSize == 0) {
    LLVM_DEBUG(dbgs() << "G_ANYEXT operand has no size, not a gvreg?\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "G_ANYEXT operand has no size, not a gvreg?\n"
; } } while (false);
    return false;
  }

  if (DstSize != 64 && DstSize > 32) {
    LLVM_DEBUG(dbgs() << "G_ANYEXT to size: " << DstSizedo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "G_ANYEXT to size: " <<
 DstSize << ", expected: 32 or 64\n"; } } while (false)
                      << ", expected: 32 or 64\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "G_ANYEXT to size: " <<
 DstSize << ", expected: 32 or 64\n"; } } while (false);
    return false;
  }
  // At this point G_ANYEXT is just like a plain COPY, but we need
  // to explicitly form the 64-bit value if any.
  if (DstSize > 32) {
    Register ExtSrc = MRI.createVirtualRegister(&AArch64::GPR64allRegClass);
    BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::SUBREG_TO_REG))
        .addDef(ExtSrc)
        .addImm(0)
        .addUse(SrcReg)
        .addImm(AArch64::sub_32);
    I.getOperand(1).setReg(ExtSrc);
  }
  return selectCopy(I, TII, MRI, TRI, RBI);
}

case TargetOpcode::G_ZEXT:
case TargetOpcode::G_SEXT_INREG:
case TargetOpcode::G_SEXT: {
  if (selectUSMovFromExtend(I, MRI))
    return true;

  unsigned Opcode = I.getOpcode();
  const bool IsSigned = Opcode != TargetOpcode::G_ZEXT;
  const Register DefReg = I.getOperand(0).getReg();
  Register SrcReg = I.getOperand(1).getReg();
  const LLT DstTy = MRI.getType(DefReg);
  const LLT SrcTy = MRI.getType(SrcReg);
  unsigned DstSize = DstTy.getSizeInBits();
  unsigned SrcSize = SrcTy.getSizeInBits();

  // SEXT_INREG has the same src reg size as dst, the size of the value to be
  // extended is encoded in the imm.
  if (Opcode == TargetOpcode::G_SEXT_INREG)
    SrcSize = I.getOperand(2).getImm();

  if (DstTy.isVector())
    return false; // Should be handled by imported patterns.

  assert((*RBI.getRegBank(DefReg, MRI, TRI)).getID() ==(static_cast <bool> ((*RBI.getRegBank(DefReg, MRI, TRI)
).getID() == AArch64::GPRRegBankID && "Unexpected ext regbank"
) ? void (0) : __assert_fail ("(*RBI.getRegBank(DefReg, MRI, TRI)).getID() == AArch64::GPRRegBankID && \"Unexpected ext regbank\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3266, __extension__ __PRETTY_FUNCTION__))
             AArch64::GPRRegBankID &&(static_cast <bool> ((*RBI.getRegBank(DefReg, MRI, TRI)
).getID() == AArch64::GPRRegBankID && "Unexpected ext regbank"
) ? void (0) : __assert_fail ("(*RBI.getRegBank(DefReg, MRI, TRI)).getID() == AArch64::GPRRegBankID && \"Unexpected ext regbank\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3266, __extension__ __PRETTY_FUNCTION__))
         "Unexpected ext regbank")(static_cast <bool> ((*RBI.getRegBank(DefReg, MRI, TRI)
).getID() == AArch64::GPRRegBankID && "Unexpected ext regbank"
) ? void (0) : __assert_fail ("(*RBI.getRegBank(DefReg, MRI, TRI)).getID() == AArch64::GPRRegBankID && \"Unexpected ext regbank\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3266, __extension__ __PRETTY_FUNCTION__));

  MachineInstr *ExtI;

  // First check if we're extending the result of a load which has a dest type
  // smaller than 32 bits, then this zext is redundant. GPR32 is the smallest
  // GPR register on AArch64 and all loads which are smaller automatically
  // zero-extend the upper bits. E.g.
  // %v(s8) = G_LOAD %p, :: (load 1)
  // %v2(s32) = G_ZEXT %v(s8)
  if (!IsSigned) {
    auto *LoadMI = getOpcodeDef(TargetOpcode::G_LOAD, SrcReg, MRI);
    bool IsGPR =
        RBI.getRegBank(SrcReg, MRI, TRI)->getID() == AArch64::GPRRegBankID;
    if (LoadMI && IsGPR) {
      const MachineMemOperand *MemOp = *LoadMI->memoperands_begin();
      unsigned BytesLoaded = MemOp->getSize();
      if (BytesLoaded < 4 && SrcTy.getSizeInBytes() == BytesLoaded)
        return selectCopy(I, TII, MRI, TRI, RBI);
    }

    // For the 32-bit -> 64-bit case, we can emit a mov (ORRWrs)
    // + SUBREG_TO_REG.
    if (IsGPR && SrcSize == 32 && DstSize == 64) {
      Register SubregToRegSrc =
          MRI.createVirtualRegister(&AArch64::GPR32RegClass);
      const Register ZReg = AArch64::WZR;
      MIB.buildInstr(AArch64::ORRWrs, {SubregToRegSrc}, {ZReg, SrcReg})
          .addImm(0);

      MIB.buildInstr(AArch64::SUBREG_TO_REG, {DefReg}, {})
          .addImm(0)
          .addUse(SubregToRegSrc)
          .addImm(AArch64::sub_32);

      if (!RBI.constrainGenericRegister(DefReg, AArch64::GPR64RegClass,
                                        MRI)) {
        LLVM_DEBUG(dbgs() << "Failed to constrain G_ZEXT destination\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Failed to constrain G_ZEXT destination\n"
; } } while (false);
        return false;
      }

      if (!RBI.constrainGenericRegister(SrcReg, AArch64::GPR32RegClass,
                                        MRI)) {
        LLVM_DEBUG(dbgs() << "Failed to constrain G_ZEXT source\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Failed to constrain G_ZEXT source\n"
; } } while (false);
        return false;
      }

      I.eraseFromParent();
      return true;
    }
  }

  if (DstSize == 64) {
    if (Opcode != TargetOpcode::G_SEXT_INREG) {
      // FIXME: Can we avoid manually doing this?
      if (!RBI.constrainGenericRegister(SrcReg, AArch64::GPR32RegClass,
                                        MRI)) {
        LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(Opcode)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Failed to constrain " <<
 TII.getName(Opcode) << " operand\n"; } } while (false)
                          << " operand\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Failed to constrain " <<
 TII.getName(Opcode) << " operand\n"; } } while (false);
        return false;
      }
      SrcReg = MIB.buildInstr(AArch64::SUBREG_TO_REG,
                              {&AArch64::GPR64RegClass}, {})
                   .addImm(0)
                   .addUse(SrcReg)
                   .addImm(AArch64::sub_32)
                   .getReg(0);
    }

    ExtI = MIB.buildInstr(IsSigned ? AArch64::SBFMXri : AArch64::UBFMXri,
                           {DefReg}, {SrcReg})
                .addImm(0)
                .addImm(SrcSize - 1);
  } else if (DstSize <= 32) {
    ExtI = MIB.buildInstr(IsSigned ? AArch64::SBFMWri : AArch64::UBFMWri,
                           {DefReg}, {SrcReg})
                .addImm(0)
                .addImm(SrcSize - 1);
  } else {
    return false;
  }

  constrainSelectedInstRegOperands(*ExtI, TII, TRI, RBI);
  I.eraseFromParent();
  return true;
}

case TargetOpcode::G_SITOFP:
case TargetOpcode::G_UITOFP:
case TargetOpcode::G_FPTOSI:
case TargetOpcode::G_FPTOUI: {
  const LLT DstTy = MRI.getType(I.getOperand(0).getReg()),
            SrcTy = MRI.getType(I.getOperand(1).getReg());
  const unsigned NewOpc = selectFPConvOpc(Opcode, DstTy, SrcTy);
  if (NewOpc == Opcode)
    return false;

  I.setDesc(TII.get(NewOpc));
  constrainSelectedInstRegOperands(I, TII, TRI, RBI);
  I.setFlags(MachineInstr::NoFPExcept);

  return true;
}

case TargetOpcode::G_FREEZE:
  return selectCopy(I, TII, MRI, TRI, RBI);

case TargetOpcode::G_INTTOPTR:
  // The importer is currently unable to import pointer types since they
  // didn't exist in SelectionDAG.
  return selectCopy(I, TII, MRI, TRI, RBI);

case TargetOpcode::G_BITCAST:
  // Imported SelectionDAG rules can handle every bitcast except those that
  // bitcast from a type to the same type. Ideally, these shouldn't occur
  // but we might not run an optimizer that deletes them. The other exception
  // is bitcasts involving pointer types, as SelectionDAG has no knowledge
  // of them.
  return selectCopy(I, TII, MRI, TRI, RBI);

case TargetOpcode::G_SELECT: {
  auto &Sel = cast<GSelect>(I);
  const Register CondReg = Sel.getCondReg();
  const Register TReg = Sel.getTrueReg();
  const Register FReg = Sel.getFalseReg();

  if (tryOptSelect(Sel))
    return true;

  // Make sure to use an unused vreg instead of wzr, so that the peephole
  // optimizations will be able to optimize these.
  Register DeadVReg = MRI.createVirtualRegister(&AArch64::GPR32RegClass);
  auto TstMI = MIB.buildInstr(AArch64::ANDSWri, {DeadVReg}, {CondReg})
                   .addImm(AArch64_AM::encodeLogicalImmediate(1, 32));
  constrainSelectedInstRegOperands(*TstMI, TII, TRI, RBI);
  if (!emitSelect(Sel.getReg(0), TReg, FReg, AArch64CC::NE, MIB))
    return false;
  Sel.eraseFromParent();
  return true;
}
case TargetOpcode::G_ICMP: {
  if (Ty.isVector())
    return selectVectorICmp(I, MRI);

  if (Ty != LLT::scalar(32)) {
    LLVM_DEBUG(dbgs() << "G_ICMP result has type: " << Tydo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "G_ICMP result has type: "
 << Ty << ", expected: " << LLT::scalar(32)
 << '\n'; } } while (false)
                      << ", expected: " << LLT::scalar(32) << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "G_ICMP result has type: "
 << Ty << ", expected: " << LLT::scalar(32)
 << '\n'; } } while (false);
    return false;
  }

  auto Pred = static_cast<CmpInst::Predicate>(I.getOperand(1).getPredicate());
  const AArch64CC::CondCode InvCC =
      changeICMPPredToAArch64CC(CmpInst::getInversePredicate(Pred));
  emitIntegerCompare(I.getOperand(2), I.getOperand(3), I.getOperand(1), MIB);
  emitCSINC(/*Dst=*/I.getOperand(0).getReg(), /*Src1=*/AArch64::WZR,
            /*Src2=*/AArch64::WZR, InvCC, MIB);
  I.eraseFromParent();
  return true;
}

case TargetOpcode::G_FCMP: {
  CmpInst::Predicate Pred =
      static_cast<CmpInst::Predicate>(I.getOperand(1).getPredicate());
  if (!emitFPCompare(I.getOperand(2).getReg(), I.getOperand(3).getReg(), MIB,
                     Pred) ||
      !emitCSetForFCmp(I.getOperand(0).getReg(), Pred, MIB))
    return false;
  I.eraseFromParent();
  return true;
}
case TargetOpcode::G_VASTART:
  return STI.isTargetDarwin() ? selectVaStartDarwin(I, MF, MRI)
                              : selectVaStartAAPCS(I, MF, MRI);
case TargetOpcode::G_INTRINSIC:
  return selectIntrinsic(I, MRI);
case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS:
  return selectIntrinsicWithSideEffects(I, MRI);
case TargetOpcode::G_IMPLICIT_DEF: {
  I.setDesc(TII.get(TargetOpcode::IMPLICIT_DEF));
  const LLT DstTy = MRI.getType(I.getOperand(0).getReg());
  const Register DstReg = I.getOperand(0).getReg();
  const RegisterBank &DstRB = *RBI.getRegBank(DstReg, MRI, TRI);
  const TargetRegisterClass *DstRC = getRegClassForTypeOnBank(DstTy, DstRB);
  RBI.constrainGenericRegister(DstReg, *DstRC, MRI);
  return true;
}
case TargetOpcode::G_BLOCK_ADDR: {
  if (TM.getCodeModel() == CodeModel::Large) {
    materializeLargeCMVal(I, I.getOperand(1).getBlockAddress(), 0);
    I.eraseFromParent();
    return true;
  } else {
    I.setDesc(TII.get(AArch64::MOVaddrBA));
    auto MovMI = BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::MOVaddrBA),
                         I.getOperand(0).getReg())
                     .addBlockAddress(I.getOperand(1).getBlockAddress(),
                                      /* Offset */ 0, AArch64II::MO_PAGE)
                     .addBlockAddress(
                         I.getOperand(1).getBlockAddress(), /* Offset */ 0,
                         AArch64II::MO_NC | AArch64II::MO_PAGEOFF);
    I.eraseFromParent();
    return constrainSelectedInstRegOperands(*MovMI, TII, TRI, RBI);
  }
}
case AArch64::G_DUP: {
  // When the scalar of G_DUP is an s8/s16 gpr, they can't be selected by
  // imported patterns. Do it manually here. Avoiding generating s16 gpr is
  // difficult because at RBS we may end up pessimizing the fpr case if we
  // decided to add an anyextend to fix this. Manual selection is the most
  // robust solution for now.
  if (RBI.getRegBank(I.getOperand(1).getReg(), MRI, TRI)->getID() !=
      AArch64::GPRRegBankID)
    return false; // We expect the fpr regbank case to be imported.
  LLT VecTy = MRI.getType(I.getOperand(0).getReg());
  if (VecTy == LLT::fixed_vector(8, 8))
    I.setDesc(TII.get(AArch64::DUPv8i8gpr));
  else if (VecTy == LLT::fixed_vector(16, 8))
    I.setDesc(TII.get(AArch64::DUPv16i8gpr));
  else if (VecTy == LLT::fixed_vector(4, 16))
    I.setDesc(TII.get(AArch64::DUPv4i16gpr));
  else if (VecTy == LLT::fixed_vector(8, 16))
    I.setDesc(TII.get(AArch64::DUPv8i16gpr));
  else
    return false;
  return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
}
case TargetOpcode::G_INTRINSIC_TRUNC:
  return selectIntrinsicTrunc(I, MRI);
case TargetOpcode::G_INTRINSIC_ROUND:
  return selectIntrinsicRound(I, MRI);
case TargetOpcode::G_BUILD_VECTOR:
  return selectBuildVector(I, MRI);
case TargetOpcode::G_MERGE_VALUES:
  return selectMergeValues(I, MRI);
case TargetOpcode::G_UNMERGE_VALUES:
  return selectUnmergeValues(I, MRI);
case TargetOpcode::G_SHUFFLE_VECTOR:
  return selectShuffleVector(I, MRI);
case TargetOpcode::G_EXTRACT_VECTOR_ELT:
  return selectExtractElt(I, MRI);
case TargetOpcode::G_INSERT_VECTOR_ELT:
  return selectInsertElt(I, MRI);
case TargetOpcode::G_CONCAT_VECTORS:
  return selectConcatVectors(I, MRI);
case TargetOpcode::G_JUMP_TABLE:
  return selectJumpTable(I, MRI);
case TargetOpcode::G_VECREDUCE_FADD:
case TargetOpcode::G_VECREDUCE_ADD:
  return selectReduction(I, MRI);
case TargetOpcode::G_MEMCPY:
case TargetOpcode::G_MEMCPY_INLINE:
case TargetOpcode::G_MEMMOVE:
case TargetOpcode::G_MEMSET:
  assert(STI.hasMOPS() && "Shouldn't get here without +mops feature")(static_cast <bool> (STI.hasMOPS() && "Shouldn't get here without +mops feature"
) ? void (0) : __assert_fail ("STI.hasMOPS() && \"Shouldn't get here without +mops feature\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3519, __extension__ __PRETTY_FUNCTION__));
  return selectMOPS(I, MRI);
}

return false;
3524}

3526bool AArch64InstructionSelector::selectReduction(MachineInstr &I,
                                               MachineRegisterInfo &MRI) {
Register VecReg = I.getOperand(1).getReg();
LLT VecTy = MRI.getType(VecReg);
if (I.getOpcode() == TargetOpcode::G_VECREDUCE_ADD) {
  // For <2 x i32> ADDPv2i32 generates an FPR64 value, so we need to emit
  // a subregister copy afterwards.
  if (VecTy == LLT::fixed_vector(2, 32)) {
    Register DstReg = I.getOperand(0).getReg();
    auto AddP = MIB.buildInstr(AArch64::ADDPv2i32, {&AArch64::FPR64RegClass},
                               {VecReg, VecReg});
    auto Copy = MIB.buildInstr(TargetOpcode::COPY, {DstReg}, {})
                    .addReg(AddP.getReg(0), 0, AArch64::ssub)
                    .getReg(0);
    RBI.constrainGenericRegister(Copy, AArch64::FPR32RegClass, MRI);
    I.eraseFromParent();
    return constrainSelectedInstRegOperands(*AddP, TII, TRI, RBI);
  }

  unsigned Opc = 0;
  if (VecTy == LLT::fixed_vector(16, 8))
    Opc = AArch64::ADDVv16i8v;
  else if (VecTy == LLT::fixed_vector(8, 16))
    Opc = AArch64::ADDVv8i16v;
  else if (VecTy == LLT::fixed_vector(4, 32))
    Opc = AArch64::ADDVv4i32v;
  else if (VecTy == LLT::fixed_vector(2, 64))
    Opc = AArch64::ADDPv2i64p;
  else {
    LLVM_DEBUG(dbgs() << "Unhandled type for add reduction")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unhandled type for add reduction"
; } } while (false);
    return false;
  }
  I.setDesc(TII.get(Opc));
  return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
}

if (I.getOpcode() == TargetOpcode::G_VECREDUCE_FADD) {
  unsigned Opc = 0;
  if (VecTy == LLT::fixed_vector(2, 32))
    Opc = AArch64::FADDPv2i32p;
  else if (VecTy == LLT::fixed_vector(2, 64))
    Opc = AArch64::FADDPv2i64p;
  else {
    LLVM_DEBUG(dbgs() << "Unhandled type for fadd reduction")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unhandled type for fadd reduction"
; } } while (false);
    return false;
  }
  I.setDesc(TII.get(Opc));
  return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
}
return false;
3576}

3578bool AArch64InstructionSelector::selectMOPS(MachineInstr &GI,
                                          MachineRegisterInfo &MRI) {
unsigned Mopcode;
switch (GI.getOpcode()) {
case TargetOpcode::G_MEMCPY:
case TargetOpcode::G_MEMCPY_INLINE:
  Mopcode = AArch64::MOPSMemoryCopyPseudo;
  break;
case TargetOpcode::G_MEMMOVE:
  Mopcode = AArch64::MOPSMemoryMovePseudo;
  break;
case TargetOpcode::G_MEMSET:
  // For tagged memset see llvm.aarch64.mops.memset.tag
  Mopcode = AArch64::MOPSMemorySetPseudo;
  break;
}

auto &DstPtr = GI.getOperand(0);
auto &SrcOrVal = GI.getOperand(1);
auto &Size = GI.getOperand(2);

// Create copies of the registers that can be clobbered.
const Register DstPtrCopy = MRI.cloneVirtualRegister(DstPtr.getReg());
const Register SrcValCopy = MRI.cloneVirtualRegister(SrcOrVal.getReg());
const Register SizeCopy = MRI.cloneVirtualRegister(Size.getReg());

const bool IsSet = Mopcode == AArch64::MOPSMemorySetPseudo;
const auto &SrcValRegClass =
    IsSet ? AArch64::GPR64RegClass : AArch64::GPR64commonRegClass;

// Constrain to specific registers
RBI.constrainGenericRegister(DstPtrCopy, AArch64::GPR64commonRegClass, MRI);
RBI.constrainGenericRegister(SrcValCopy, SrcValRegClass, MRI);
RBI.constrainGenericRegister(SizeCopy, AArch64::GPR64RegClass, MRI);

MIB.buildCopy(DstPtrCopy, DstPtr);
MIB.buildCopy(SrcValCopy, SrcOrVal);
MIB.buildCopy(SizeCopy, Size);

// New instruction uses the copied registers because it must update them.
// The defs are not used since they don't exist in G_MEM*. They are still
// tied.
// Note: order of operands is different from G_MEMSET, G_MEMCPY, G_MEMMOVE
Register DefDstPtr = MRI.createVirtualRegister(&AArch64::GPR64commonRegClass);
Register DefSize = MRI.createVirtualRegister(&AArch64::GPR64RegClass);
if (IsSet) {
  MIB.buildInstr(Mopcode, {DefDstPtr, DefSize},
                 {DstPtrCopy, SizeCopy, SrcValCopy});
} else {
  Register DefSrcPtr = MRI.createVirtualRegister(&SrcValRegClass);
  MIB.buildInstr(Mopcode, {DefDstPtr, DefSrcPtr, DefSize},
                 {DstPtrCopy, SrcValCopy, SizeCopy});
}

GI.eraseFromParent();
return true;
3634}

3636bool AArch64InstructionSelector::selectBrJT(MachineInstr &I,
                                          MachineRegisterInfo &MRI) {
assert(I.getOpcode() == TargetOpcode::G_BRJT && "Expected G_BRJT")(static_cast <bool> (I.getOpcode() == TargetOpcode::G_BRJT
 && "Expected G_BRJT") ? void (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_BRJT && \"Expected G_BRJT\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3638, __extension__ __PRETTY_FUNCTION__));
Register JTAddr = I.getOperand(0).getReg();
unsigned JTI = I.getOperand(1).getIndex();
Register Index = I.getOperand(2).getReg();

Register TargetReg = MRI.createVirtualRegister(&AArch64::GPR64RegClass);
Register ScratchReg = MRI.createVirtualRegister(&AArch64::GPR64spRegClass);

MF->getInfo<AArch64FunctionInfo>()->setJumpTableEntryInfo(JTI, 4, nullptr);
auto JumpTableInst = MIB.buildInstr(AArch64::JumpTableDest32,
                                    {TargetReg, ScratchReg}, {JTAddr, Index})
                         .addJumpTableIndex(JTI);
// Build the indirect branch.
MIB.buildInstr(AArch64::BR, {}, {TargetReg});
I.eraseFromParent();
return constrainSelectedInstRegOperands(*JumpTableInst, TII, TRI, RBI);
3654}

3656bool AArch64InstructionSelector::selectJumpTable(MachineInstr &I,
                                               MachineRegisterInfo &MRI) {
assert(I.getOpcode() == TargetOpcode::G_JUMP_TABLE && "Expected jump table")(static_cast <bool> (I.getOpcode() == TargetOpcode::G_JUMP_TABLE
 && "Expected jump table") ? void (0) : __assert_fail
 ("I.getOpcode() == TargetOpcode::G_JUMP_TABLE && \"Expected jump table\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3658, __extension__ __PRETTY_FUNCTION__));
assert(I.getOperand(1).isJTI() && "Jump table op should have a JTI!")(static_cast <bool> (I.getOperand(1).isJTI() &&
 "Jump table op should have a JTI!") ? void (0) : __assert_fail
 ("I.getOperand(1).isJTI() && \"Jump table op should have a JTI!\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3659, __extension__ __PRETTY_FUNCTION__));

Register DstReg = I.getOperand(0).getReg();
unsigned JTI = I.getOperand(1).getIndex();
// We generate a MOVaddrJT which will get expanded to an ADRP + ADD later.
auto MovMI =
  MIB.buildInstr(AArch64::MOVaddrJT, {DstReg}, {})
        .addJumpTableIndex(JTI, AArch64II::MO_PAGE)
        .addJumpTableIndex(JTI, AArch64II::MO_NC | AArch64II::MO_PAGEOFF);
I.eraseFromParent();
return constrainSelectedInstRegOperands(*MovMI, TII, TRI, RBI);
3670}

3672bool AArch64InstructionSelector::selectTLSGlobalValue(
  MachineInstr &I, MachineRegisterInfo &MRI) {
if (!STI.isTargetMachO())
  return false;
MachineFunction &MF = *I.getParent()->getParent();
MF.getFrameInfo().setAdjustsStack(true);

const auto &GlobalOp = I.getOperand(1);
assert(GlobalOp.getOffset() == 0 &&(static_cast <bool> (GlobalOp.getOffset() == 0 &&
 "Shouldn't have an offset on TLS globals!") ? void (0) : __assert_fail
 ("GlobalOp.getOffset() == 0 && \"Shouldn't have an offset on TLS globals!\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3681, __extension__ __PRETTY_FUNCTION__))
       "Shouldn't have an offset on TLS globals!")(static_cast <bool> (GlobalOp.getOffset() == 0 &&
 "Shouldn't have an offset on TLS globals!") ? void (0) : __assert_fail
 ("GlobalOp.getOffset() == 0 && \"Shouldn't have an offset on TLS globals!\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3681, __extension__ __PRETTY_FUNCTION__));
const GlobalValue &GV = *GlobalOp.getGlobal();

auto LoadGOT =
    MIB.buildInstr(AArch64::LOADgot, {&AArch64::GPR64commonRegClass}, {})
        .addGlobalAddress(&GV, 0, AArch64II::MO_TLS);

auto Load = MIB.buildInstr(AArch64::LDRXui, {&AArch64::GPR64commonRegClass},
                           {LoadGOT.getReg(0)})
                .addImm(0);

MIB.buildCopy(Register(AArch64::X0), LoadGOT.getReg(0));
// TLS calls preserve all registers except those that absolutely must be
// trashed: X0 (it takes an argument), LR (it's a call) and NZCV (let's not be
// silly).
MIB.buildInstr(getBLRCallOpcode(MF), {}, {Load})
    .addUse(AArch64::X0, RegState::Implicit)
    .addDef(AArch64::X0, RegState::Implicit)
    .addRegMask(TRI.getTLSCallPreservedMask());

MIB.buildCopy(I.getOperand(0).getReg(), Register(AArch64::X0));
RBI.constrainGenericRegister(I.getOperand(0).getReg(), AArch64::GPR64RegClass,
                             MRI);
I.eraseFromParent();
return true;
3706}

3708bool AArch64InstructionSelector::selectIntrinsicTrunc(
  MachineInstr &I, MachineRegisterInfo &MRI) const {
const LLT SrcTy = MRI.getType(I.getOperand(0).getReg());

// Select the correct opcode.
unsigned Opc = 0;
if (!SrcTy.isVector()) {
  switch (SrcTy.getSizeInBits()) {
  default:
  case 16:
    Opc = AArch64::FRINTZHr;
    break;
  case 32:
    Opc = AArch64::FRINTZSr;
    break;
  case 64:
    Opc = AArch64::FRINTZDr;
    break;
  }
} else {
  unsigned NumElts = SrcTy.getNumElements();
  switch (SrcTy.getElementType().getSizeInBits()) {
  default:
    break;
  case 16:
    if (NumElts == 4)
      Opc = AArch64::FRINTZv4f16;
    else if (NumElts == 8)
      Opc = AArch64::FRINTZv8f16;
    break;
  case 32:
    if (NumElts == 2)
      Opc = AArch64::FRINTZv2f32;
    else if (NumElts == 4)
      Opc = AArch64::FRINTZv4f32;
    break;
  case 64:
    if (NumElts == 2)
      Opc = AArch64::FRINTZv2f64;
    break;
  }
}

if (!Opc) {
  // Didn't get an opcode above, bail.
  LLVM_DEBUG(dbgs() << "Unsupported type for G_INTRINSIC_TRUNC!\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unsupported type for G_INTRINSIC_TRUNC!\n"
; } } while (false);
  return false;
}

// Legalization would have set us up perfectly for this; we just need to
// set the opcode and move on.
I.setDesc(TII.get(Opc));
return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
3761}

3763bool AArch64InstructionSelector::selectIntrinsicRound(
  MachineInstr &I, MachineRegisterInfo &MRI) const {
const LLT SrcTy = MRI.getType(I.getOperand(0).getReg());

// Select the correct opcode.
unsigned Opc = 0;
if (!SrcTy.isVector()) {
  switch (SrcTy.getSizeInBits()) {
  default:
  case 16:
    Opc = AArch64::FRINTAHr;
    break;
  case 32:
    Opc = AArch64::FRINTASr;
    break;
  case 64:
    Opc = AArch64::FRINTADr;
    break;
  }
} else {
  unsigned NumElts = SrcTy.getNumElements();
  switch (SrcTy.getElementType().getSizeInBits()) {
  default:
    break;
  case 16:
    if (NumElts == 4)
      Opc = AArch64::FRINTAv4f16;
    else if (NumElts == 8)
      Opc = AArch64::FRINTAv8f16;
    break;
  case 32:
    if (NumElts == 2)
      Opc = AArch64::FRINTAv2f32;
    else if (NumElts == 4)
      Opc = AArch64::FRINTAv4f32;
    break;
  case 64:
    if (NumElts == 2)
      Opc = AArch64::FRINTAv2f64;
    break;
  }
}

if (!Opc) {
  // Didn't get an opcode above, bail.
  LLVM_DEBUG(dbgs() << "Unsupported type for G_INTRINSIC_ROUND!\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unsupported type for G_INTRINSIC_ROUND!\n"
; } } while (false);
  return false;
}

// Legalization would have set us up perfectly for this; we just need to
// set the opcode and move on.
I.setDesc(TII.get(Opc));
return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
3816}

3818bool AArch64InstructionSelector::selectVectorICmp(
  MachineInstr &I, MachineRegisterInfo &MRI) {
Register DstReg = I.getOperand(0).getReg();
LLT DstTy = MRI.getType(DstReg);
Register SrcReg = I.getOperand(2).getReg();
Register Src2Reg = I.getOperand(3).getReg();
LLT SrcTy = MRI.getType(SrcReg);

unsigned SrcEltSize = SrcTy.getElementType().getSizeInBits();
unsigned NumElts = DstTy.getNumElements();

// First index is element size, 0 == 8b, 1 == 16b, 2 == 32b, 3 == 64b
// Second index is num elts, 0 == v2, 1 == v4, 2 == v8, 3 == v16
// Third index is cc opcode:
// 0 == eq
// 1 == ugt
// 2 == uge
// 3 == ult
// 4 == ule
// 5 == sgt
// 6 == sge
// 7 == slt
// 8 == sle
// ne is done by negating 'eq' result.

// This table below assumes that for some comparisons the operands will be
// commuted.
// ult op == commute + ugt op
// ule op == commute + uge op
// slt op == commute + sgt op
// sle op == commute + sge op
unsigned PredIdx = 0;
bool SwapOperands = false;
CmpInst::Predicate Pred = (CmpInst::Predicate)I.getOperand(1).getPredicate();
switch (Pred) {
case CmpInst::ICMP_NE:
case CmpInst::ICMP_EQ:
  PredIdx = 0;
  break;
case CmpInst::ICMP_UGT:
  PredIdx = 1;
  break;
case CmpInst::ICMP_UGE:
  PredIdx = 2;
  break;
case CmpInst::ICMP_ULT:
  PredIdx = 3;
  SwapOperands = true;
  break;
case CmpInst::ICMP_ULE:
  PredIdx = 4;
  SwapOperands = true;
  break;
case CmpInst::ICMP_SGT:
  PredIdx = 5;
  break;
case CmpInst::ICMP_SGE:
  PredIdx = 6;
  break;
case CmpInst::ICMP_SLT:
  PredIdx = 7;
  SwapOperands = true;
  break;
case CmpInst::ICMP_SLE:
  PredIdx = 8;
  SwapOperands = true;
  break;
default:
  llvm_unreachable("Unhandled icmp predicate")::llvm::llvm_unreachable_internal("Unhandled icmp predicate",
 "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 3886);
  return false;
}

// This table obviously should be tablegen'd when we have our GISel native
// tablegen selector.

static const unsigned OpcTable[4][4][9] = {
    {
        {0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
         0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
         0 /* invalid */},
        {0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
         0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
         0 /* invalid */},
        {AArch64::CMEQv8i8, AArch64::CMHIv8i8, AArch64::CMHSv8i8,
         AArch64::CMHIv8i8, AArch64::CMHSv8i8, AArch64::CMGTv8i8,
         AArch64::CMGEv8i8, AArch64::CMGTv8i8, AArch64::CMGEv8i8},
        {AArch64::CMEQv16i8, AArch64::CMHIv16i8, AArch64::CMHSv16i8,
         AArch64::CMHIv16i8, AArch64::CMHSv16i8, AArch64::CMGTv16i8,
         AArch64::CMGEv16i8, AArch64::CMGTv16i8, AArch64::CMGEv16i8}
    },
    {
        {0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
         0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
         0 /* invalid */},
        {AArch64::CMEQv4i16, AArch64::CMHIv4i16, AArch64::CMHSv4i16,
         AArch64::CMHIv4i16, AArch64::CMHSv4i16, AArch64::CMGTv4i16,
         AArch64::CMGEv4i16, AArch64::CMGTv4i16, AArch64::CMGEv4i16},
        {AArch64::CMEQv8i16, AArch64::CMHIv8i16, AArch64::CMHSv8i16,
         AArch64::CMHIv8i16, AArch64::CMHSv8i16, AArch64::CMGTv8i16,
         AArch64::CMGEv8i16, AArch64::CMGTv8i16, AArch64::CMGEv8i16},
        {0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
         0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
         0 /* invalid */}
    },
    {
        {AArch64::CMEQv2i32, AArch64::CMHIv2i32, AArch64::CMHSv2i32,
         AArch64::CMHIv2i32, AArch64::CMHSv2i32, AArch64::CMGTv2i32,
         AArch64::CMGEv2i32, AArch64::CMGTv2i32, AArch64::CMGEv2i32},
        {AArch64::CMEQv4i32, AArch64::CMHIv4i32, AArch64::CMHSv4i32,
         AArch64::CMHIv4i32, AArch64::CMHSv4i32, AArch64::CMGTv4i32,
         AArch64::CMGEv4i32, AArch64::CMGTv4i32, AArch64::CMGEv4i32},
        {0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
         0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
         0 /* invalid */},
        {0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
         0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
         0 /* invalid */}
    },
    {
        {AArch64::CMEQv2i64, AArch64::CMHIv2i64, AArch64::CMHSv2i64,
         AArch64::CMHIv2i64, AArch64::CMHSv2i64, AArch64::CMGTv2i64,
         AArch64::CMGEv2i64, AArch64::CMGTv2i64, AArch64::CMGEv2i64},
        {0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
         0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
         0 /* invalid */},
        {0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
         0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
         0 /* invalid */},
        {0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
         0 /* invalid */, 0 /* invalid */, 0 /* invalid */, 0 /* invalid */,
         0 /* invalid */}
    },
};
unsigned EltIdx = Log2_32(SrcEltSize / 8);
unsigned NumEltsIdx = Log2_32(NumElts / 2);
unsigned Opc = OpcTable[EltIdx][NumEltsIdx][PredIdx];
if (!Opc) {
  LLVM_DEBUG(dbgs() << "Could not map G_ICMP to cmp opcode")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Could not map G_ICMP to cmp opcode"
; } } while (false);
  return false;
}

const RegisterBank &VecRB = *RBI.getRegBank(SrcReg, MRI, TRI);
const TargetRegisterClass *SrcRC =
    getRegClassForTypeOnBank(SrcTy, VecRB, true);
if (!SrcRC) {
  LLVM_DEBUG(dbgs() << "Could not determine source register class.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Could not determine source register class.\n"
; } } while (false);
  return false;
}

unsigned NotOpc = Pred == ICmpInst::ICMP_NE ? AArch64::NOTv8i8 : 0;
if (SrcTy.getSizeInBits() == 128)
  NotOpc = NotOpc ? AArch64::NOTv16i8 : 0;

if (SwapOperands)
  std::swap(SrcReg, Src2Reg);

auto Cmp = MIB.buildInstr(Opc, {SrcRC}, {SrcReg, Src2Reg});
constrainSelectedInstRegOperands(*Cmp, TII, TRI, RBI);

// Invert if we had a 'ne' cc.
if (NotOpc) {
  Cmp = MIB.buildInstr(NotOpc, {DstReg}, {Cmp});
  constrainSelectedInstRegOperands(*Cmp, TII, TRI, RBI);
} else {
  MIB.buildCopy(DstReg, Cmp.getReg(0));
}
RBI.constrainGenericRegister(DstReg, *SrcRC, MRI);
I.eraseFromParent();
return true;
3987}

3989MachineInstr *AArch64InstructionSelector::emitScalarToVector(
  unsigned EltSize, const TargetRegisterClass *DstRC, Register Scalar,
  MachineIRBuilder &MIRBuilder) const {
auto Undef = MIRBuilder.buildInstr(TargetOpcode::IMPLICIT_DEF, {DstRC}, {});

auto BuildFn = [&](unsigned SubregIndex) {
  auto Ins =
      MIRBuilder
          .buildInstr(TargetOpcode::INSERT_SUBREG, {DstRC}, {Undef, Scalar})
          .addImm(SubregIndex);
  constrainSelectedInstRegOperands(*Undef, TII, TRI, RBI);
  constrainSelectedInstRegOperands(*Ins, TII, TRI, RBI);
  return &*Ins;
};

switch (EltSize) {
case 8:
  return BuildFn(AArch64::bsub);
case 16:
  return BuildFn(AArch64::hsub);
case 32:
  return BuildFn(AArch64::ssub);
case 64:
  return BuildFn(AArch64::dsub);
default:
  return nullptr;
}
4016}

4018bool AArch64InstructionSelector::selectMergeValues(
  MachineInstr &I, MachineRegisterInfo &MRI) {
assert(I.getOpcode() == TargetOpcode::G_MERGE_VALUES && "unexpected opcode")(static_cast <bool> (I.getOpcode() == TargetOpcode::G_MERGE_VALUES
 && "unexpected opcode") ? void (0) : __assert_fail (
"I.getOpcode() == TargetOpcode::G_MERGE_VALUES && \"unexpected opcode\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4020, __extension__ __PRETTY_FUNCTION__));
const LLT DstTy = MRI.getType(I.getOperand(0).getReg());
const LLT SrcTy = MRI.getType(I.getOperand(1).getReg());
assert(!DstTy.isVector() && !SrcTy.isVector() && "invalid merge operation")(static_cast <bool> (!DstTy.isVector() && !SrcTy
.isVector() && "invalid merge operation") ? void (0) :
 __assert_fail ("!DstTy.isVector() && !SrcTy.isVector() && \"invalid merge operation\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4023, __extension__ __PRETTY_FUNCTION__));
const RegisterBank &RB = *RBI.getRegBank(I.getOperand(1).getReg(), MRI, TRI);

if (I.getNumOperands() != 3)
  return false;

// Merging 2 s64s into an s128.
if (DstTy == LLT::scalar(128)) {
  if (SrcTy.getSizeInBits() != 64)
    return false;
  Register DstReg = I.getOperand(0).getReg();
  Register Src1Reg = I.getOperand(1).getReg();
  Register Src2Reg = I.getOperand(2).getReg();
  auto Tmp = MIB.buildInstr(TargetOpcode::IMPLICIT_DEF, {DstTy}, {});
  MachineInstr *InsMI = emitLaneInsert(std::nullopt, Tmp.getReg(0), Src1Reg,
                                       /* LaneIdx */ 0, RB, MIB);
  if (!InsMI)
    return false;
  MachineInstr *Ins2MI = emitLaneInsert(DstReg, InsMI->getOperand(0).getReg(),
                                        Src2Reg, /* LaneIdx */ 1, RB, MIB);
  if (!Ins2MI)
    return false;
  constrainSelectedInstRegOperands(*InsMI, TII, TRI, RBI);
  constrainSelectedInstRegOperands(*Ins2MI, TII, TRI, RBI);
  I.eraseFromParent();
  return true;
}

if (RB.getID() != AArch64::GPRRegBankID)
  return false;

if (DstTy.getSizeInBits() != 64 || SrcTy.getSizeInBits() != 32)
  return false;

auto *DstRC = &AArch64::GPR64RegClass;
Register SubToRegDef = MRI.createVirtualRegister(DstRC);
MachineInstr &SubRegMI = *BuildMI(*I.getParent(), I, I.getDebugLoc(),
                                  TII.get(TargetOpcode::SUBREG_TO_REG))
                              .addDef(SubToRegDef)
                              .addImm(0)
                              .addUse(I.getOperand(1).getReg())
                              .addImm(AArch64::sub_32);
Register SubToRegDef2 = MRI.createVirtualRegister(DstRC);
// Need to anyext the second scalar before we can use bfm
MachineInstr &SubRegMI2 = *BuildMI(*I.getParent(), I, I.getDebugLoc(),
                                  TII.get(TargetOpcode::SUBREG_TO_REG))
                              .addDef(SubToRegDef2)
                              .addImm(0)
                              .addUse(I.getOperand(2).getReg())
                              .addImm(AArch64::sub_32);
MachineInstr &BFM =
    *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(AArch64::BFMXri))
         .addDef(I.getOperand(0).getReg())
         .addUse(SubToRegDef)
         .addUse(SubToRegDef2)
         .addImm(32)
         .addImm(31);
constrainSelectedInstRegOperands(SubRegMI, TII, TRI, RBI);
constrainSelectedInstRegOperands(SubRegMI2, TII, TRI, RBI);
constrainSelectedInstRegOperands(BFM, TII, TRI, RBI);
I.eraseFromParent();
return true;
4085}

4087static bool getLaneCopyOpcode(unsigned &CopyOpc, unsigned &ExtractSubReg,
                            const unsigned EltSize) {
// Choose a lane copy opcode and subregister based off of the size of the
// vector's elements.
switch (EltSize) {
case 8:
  CopyOpc = AArch64::DUPi8;
  ExtractSubReg = AArch64::bsub;
  break;
case 16:
  CopyOpc = AArch64::DUPi16;
  ExtractSubReg = AArch64::hsub;
  break;
case 32:
  CopyOpc = AArch64::DUPi32;
  ExtractSubReg = AArch64::ssub;
  break;
case 64:
  CopyOpc = AArch64::DUPi64;
  ExtractSubReg = AArch64::dsub;
  break;
default:
  // Unknown size, bail out.
  LLVM_DEBUG(dbgs() << "Elt size '" << EltSize << "' unsupported.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Elt size '" << EltSize
 << "' unsupported.\n"; } } while (false);
  return false;
}
return true;
4114}

4116MachineInstr *AArch64InstructionSelector::emitExtractVectorElt(
  std::optional<Register> DstReg, const RegisterBank &DstRB, LLT ScalarTy,
  Register VecReg, unsigned LaneIdx, MachineIRBuilder &MIRBuilder) const {
MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
unsigned CopyOpc = 0;
unsigned ExtractSubReg = 0;
if (!getLaneCopyOpcode(CopyOpc, ExtractSubReg, ScalarTy.getSizeInBits())) {
  LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Couldn't determine lane copy opcode for instruction.\n"
; } } while (false)
      dbgs() << "Couldn't determine lane copy opcode for instruction.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Couldn't determine lane copy opcode for instruction.\n"
; } } while (false);
  return nullptr;
}

const TargetRegisterClass *DstRC =
    getRegClassForTypeOnBank(ScalarTy, DstRB, true);
if (!DstRC) {
  LLVM_DEBUG(dbgs() << "Could not determine destination register class.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Could not determine destination register class.\n"
; } } while (false);
  return nullptr;
}

const RegisterBank &VecRB = *RBI.getRegBank(VecReg, MRI, TRI);
const LLT &VecTy = MRI.getType(VecReg);
const TargetRegisterClass *VecRC =
    getRegClassForTypeOnBank(VecTy, VecRB, true);
if (!VecRC) {
  LLVM_DEBUG(dbgs() << "Could not determine source register class.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Could not determine source register class.\n"
; } } while (false);
  return nullptr;
}

// The register that we're going to copy into.
Register InsertReg = VecReg;
if (!DstReg)
  DstReg = MRI.createVirtualRegister(DstRC);
// If the lane index is 0, we just use a subregister COPY.
if (LaneIdx == 0) {
  auto Copy = MIRBuilder.buildInstr(TargetOpcode::COPY, {*DstReg}, {})
                  .addReg(VecReg, 0, ExtractSubReg);
  RBI.constrainGenericRegister(*DstReg, *DstRC, MRI);
  return &*Copy;
}

// Lane copies require 128-bit wide registers. If we're dealing with an
// unpacked vector, then we need to move up to that width. Insert an implicit
// def and a subregister insert to get us there.
if (VecTy.getSizeInBits() != 128) {
  MachineInstr *ScalarToVector = emitScalarToVector(
      VecTy.getSizeInBits(), &AArch64::FPR128RegClass, VecReg, MIRBuilder);
  if (!ScalarToVector)
    return nullptr;
  InsertReg = ScalarToVector->getOperand(0).getReg();
}

MachineInstr *LaneCopyMI =
    MIRBuilder.buildInstr(CopyOpc, {*DstReg}, {InsertReg}).addImm(LaneIdx);
constrainSelectedInstRegOperands(*LaneCopyMI, TII, TRI, RBI);

// Make sure that we actually constrain the initial copy.
RBI.constrainGenericRegister(*DstReg, *DstRC, MRI);
return LaneCopyMI;
4174}

4176bool AArch64InstructionSelector::selectExtractElt(
  MachineInstr &I, MachineRegisterInfo &MRI) {
assert(I.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT &&(static_cast <bool> (I.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT
 && "unexpected opcode!") ? void (0) : __assert_fail (
"I.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT && \"unexpected opcode!\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4179, __extension__ __PRETTY_FUNCTION__))
       "unexpected opcode!")(static_cast <bool> (I.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT
 && "unexpected opcode!") ? void (0) : __assert_fail (
"I.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT && \"unexpected opcode!\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4179, __extension__ __PRETTY_FUNCTION__));
Register DstReg = I.getOperand(0).getReg();
const LLT NarrowTy = MRI.getType(DstReg);
const Register SrcReg = I.getOperand(1).getReg();
const LLT WideTy = MRI.getType(SrcReg);
(void)WideTy;
assert(WideTy.getSizeInBits() >= NarrowTy.getSizeInBits() &&(static_cast <bool> (WideTy.getSizeInBits() >= NarrowTy
.getSizeInBits() && "source register size too small!"
) ? void (0) : __assert_fail ("WideTy.getSizeInBits() >= NarrowTy.getSizeInBits() && \"source register size too small!\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4186, __extension__ __PRETTY_FUNCTION__))
       "source register size too small!")(static_cast <bool> (WideTy.getSizeInBits() >= NarrowTy
.getSizeInBits() && "source register size too small!"
) ? void (0) : __assert_fail ("WideTy.getSizeInBits() >= NarrowTy.getSizeInBits() && \"source register size too small!\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4186, __extension__ __PRETTY_FUNCTION__));
assert(!NarrowTy.isVector() && "cannot extract vector into vector!")(static_cast <bool> (!NarrowTy.isVector() && "cannot extract vector into vector!"
) ? void (0) : __assert_fail ("!NarrowTy.isVector() && \"cannot extract vector into vector!\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4187, __extension__ __PRETTY_FUNCTION__));

// Need the lane index to determine the correct copy opcode.
MachineOperand &LaneIdxOp = I.getOperand(2);
assert(LaneIdxOp.isReg() && "Lane index operand was not a register?")(static_cast <bool> (LaneIdxOp.isReg() && "Lane index operand was not a register?"
) ? void (0) : __assert_fail ("LaneIdxOp.isReg() && \"Lane index operand was not a register?\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4191, __extension__ __PRETTY_FUNCTION__));

if (RBI.getRegBank(DstReg, MRI, TRI)->getID() != AArch64::FPRRegBankID) {
  LLVM_DEBUG(dbgs() << "Cannot extract into GPR.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Cannot extract into GPR.\n"
; } } while (false);
  return false;
}

// Find the index to extract from.
auto VRegAndVal = getIConstantVRegValWithLookThrough(LaneIdxOp.getReg(), MRI);
if (!VRegAndVal)
  return false;
unsigned LaneIdx = VRegAndVal->Value.getSExtValue();


const RegisterBank &DstRB = *RBI.getRegBank(DstReg, MRI, TRI);
MachineInstr *Extract = emitExtractVectorElt(DstReg, DstRB, NarrowTy, SrcReg,
                                             LaneIdx, MIB);
if (!Extract)
  return false;

I.eraseFromParent();
return true;
4213}

4215bool AArch64InstructionSelector::selectSplitVectorUnmerge(
  MachineInstr &I, MachineRegisterInfo &MRI) {
unsigned NumElts = I.getNumOperands() - 1;
Register SrcReg = I.getOperand(NumElts).getReg();
const LLT NarrowTy = MRI.getType(I.getOperand(0).getReg());
const LLT SrcTy = MRI.getType(SrcReg);

assert(NarrowTy.isVector() && "Expected an unmerge into vectors")(static_cast <bool> (NarrowTy.isVector() && "Expected an unmerge into vectors"
) ? void (0) : __assert_fail ("NarrowTy.isVector() && \"Expected an unmerge into vectors\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4222, __extension__ __PRETTY_FUNCTION__));
if (SrcTy.getSizeInBits() > 128) {
  LLVM_DEBUG(dbgs() << "Unexpected vector type for vec split unmerge")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unexpected vector type for vec split unmerge"
; } } while (false);
  return false;
}

// We implement a split vector operation by treating the sub-vectors as
// scalars and extracting them.
const RegisterBank &DstRB =
    *RBI.getRegBank(I.getOperand(0).getReg(), MRI, TRI);
for (unsigned OpIdx = 0; OpIdx < NumElts; ++OpIdx) {
  Register Dst = I.getOperand(OpIdx).getReg();
  MachineInstr *Extract =
      emitExtractVectorElt(Dst, DstRB, NarrowTy, SrcReg, OpIdx, MIB);
  if (!Extract)
    return false;
}
I.eraseFromParent();
return true;
4241}

4243bool AArch64InstructionSelector::selectUnmergeValues(MachineInstr &I,
                                                   MachineRegisterInfo &MRI) {
assert(I.getOpcode() == TargetOpcode::G_UNMERGE_VALUES &&(static_cast <bool> (I.getOpcode() == TargetOpcode::G_UNMERGE_VALUES
 && "unexpected opcode") ? void (0) : __assert_fail (
"I.getOpcode() == TargetOpcode::G_UNMERGE_VALUES && \"unexpected opcode\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4246, __extension__ __PRETTY_FUNCTION__))
       "unexpected opcode")(static_cast <bool> (I.getOpcode() == TargetOpcode::G_UNMERGE_VALUES
 && "unexpected opcode") ? void (0) : __assert_fail (
"I.getOpcode() == TargetOpcode::G_UNMERGE_VALUES && \"unexpected opcode\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4246, __extension__ __PRETTY_FUNCTION__));

// TODO: Handle unmerging into GPRs and from scalars to scalars.
if (RBI.getRegBank(I.getOperand(0).getReg(), MRI, TRI)->getID() !=
        AArch64::FPRRegBankID ||
    RBI.getRegBank(I.getOperand(1).getReg(), MRI, TRI)->getID() !=
        AArch64::FPRRegBankID) {
  LLVM_DEBUG(dbgs() << "Unmerging vector-to-gpr and scalar-to-scalar "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unmerging vector-to-gpr and scalar-to-scalar "
 "currently unsupported.\n"; } } while (false)
                       "currently unsupported.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unmerging vector-to-gpr and scalar-to-scalar "
 "currently unsupported.\n"; } } while (false);
  return false;
}

// The last operand is the vector source register, and every other operand is
// a register to unpack into.
unsigned NumElts = I.getNumOperands() - 1;
Register SrcReg = I.getOperand(NumElts).getReg();
const LLT NarrowTy = MRI.getType(I.getOperand(0).getReg());
const LLT WideTy = MRI.getType(SrcReg);
(void)WideTy;
assert((WideTy.isVector() || WideTy.getSizeInBits() == 128) &&(static_cast <bool> ((WideTy.isVector() || WideTy.getSizeInBits
() == 128) && "can only unmerge from vector or s128 types!"
) ? void (0) : __assert_fail ("(WideTy.isVector() || WideTy.getSizeInBits() == 128) && \"can only unmerge from vector or s128 types!\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4266, __extension__ __PRETTY_FUNCTION__))
       "can only unmerge from vector or s128 types!")(static_cast <bool> ((WideTy.isVector() || WideTy.getSizeInBits
() == 128) && "can only unmerge from vector or s128 types!"
) ? void (0) : __assert_fail ("(WideTy.isVector() || WideTy.getSizeInBits() == 128) && \"can only unmerge from vector or s128 types!\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4266, __extension__ __PRETTY_FUNCTION__));
assert(WideTy.getSizeInBits() > NarrowTy.getSizeInBits() &&(static_cast <bool> (WideTy.getSizeInBits() > NarrowTy
.getSizeInBits() && "source register size too small!"
) ? void (0) : __assert_fail ("WideTy.getSizeInBits() > NarrowTy.getSizeInBits() && \"source register size too small!\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4268, __extension__ __PRETTY_FUNCTION__))
       "source register size too small!")(static_cast <bool> (WideTy.getSizeInBits() > NarrowTy
.getSizeInBits() && "source register size too small!"
) ? void (0) : __assert_fail ("WideTy.getSizeInBits() > NarrowTy.getSizeInBits() && \"source register size too small!\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4268, __extension__ __PRETTY_FUNCTION__));

if (!NarrowTy.isScalar())
  return selectSplitVectorUnmerge(I, MRI);

// Choose a lane copy opcode and subregister based off of the size of the
// vector's elements.
unsigned CopyOpc = 0;
unsigned ExtractSubReg = 0;
if (!getLaneCopyOpcode(CopyOpc, ExtractSubReg, NarrowTy.getSizeInBits()))
  return false;

// Set up for the lane copies.
MachineBasicBlock &MBB = *I.getParent();

// Stores the registers we'll be copying from.
SmallVector<Register, 4> InsertRegs;

// We'll use the first register twice, so we only need NumElts-1 registers.
unsigned NumInsertRegs = NumElts - 1;

// If our elements fit into exactly 128 bits, then we can copy from the source
// directly. Otherwise, we need to do a bit of setup with some subregister
// inserts.
if (NarrowTy.getSizeInBits() * NumElts == 128) {
  InsertRegs = SmallVector<Register, 4>(NumInsertRegs, SrcReg);
} else {
  // No. We have to perform subregister inserts. For each insert, create an
  // implicit def and a subregister insert, and save the register we create.
  const TargetRegisterClass *RC = getRegClassForTypeOnBank(
      LLT::fixed_vector(NumElts, WideTy.getScalarSizeInBits()),
      *RBI.getRegBank(SrcReg, MRI, TRI));
  unsigned SubReg = 0;
  bool Found = getSubRegForClass(RC, TRI, SubReg);
  (void)Found;
  assert(Found && "expected to find last operand's subeg idx")(static_cast <bool> (Found && "expected to find last operand's subeg idx"
) ? void (0) : __assert_fail ("Found && \"expected to find last operand's subeg idx\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4303, __extension__ __PRETTY_FUNCTION__));
  for (unsigned Idx = 0; Idx < NumInsertRegs; ++Idx) {
    Register ImpDefReg = MRI.createVirtualRegister(&AArch64::FPR128RegClass);
    MachineInstr &ImpDefMI =
        *BuildMI(MBB, I, I.getDebugLoc(), TII.get(TargetOpcode::IMPLICIT_DEF),
                 ImpDefReg);

    // Now, create the subregister insert from SrcReg.
    Register InsertReg = MRI.createVirtualRegister(&AArch64::FPR128RegClass);
    MachineInstr &InsMI =
        *BuildMI(MBB, I, I.getDebugLoc(),
                 TII.get(TargetOpcode::INSERT_SUBREG), InsertReg)
             .addUse(ImpDefReg)
             .addUse(SrcReg)
             .addImm(SubReg);

    constrainSelectedInstRegOperands(ImpDefMI, TII, TRI, RBI);
    constrainSelectedInstRegOperands(InsMI, TII, TRI, RBI);

    // Save the register so that we can copy from it after.
    InsertRegs.push_back(InsertReg);
  }
}

// Now that we've created any necessary subregister inserts, we can
// create the copies.
//
// Perform the first copy separately as a subregister copy.
Register CopyTo = I.getOperand(0).getReg();
auto FirstCopy = MIB.buildInstr(TargetOpcode::COPY, {CopyTo}, {})
                     .addReg(InsertRegs[0], 0, ExtractSubReg);
constrainSelectedInstRegOperands(*FirstCopy, TII, TRI, RBI);

// Now, perform the remaining copies as vector lane copies.
unsigned LaneIdx = 1;
for (Register InsReg : InsertRegs) {
  Register CopyTo = I.getOperand(LaneIdx).getReg();
  MachineInstr &CopyInst =
      *BuildMI(MBB, I, I.getDebugLoc(), TII.get(CopyOpc), CopyTo)
           .addUse(InsReg)
           .addImm(LaneIdx);
  constrainSelectedInstRegOperands(CopyInst, TII, TRI, RBI);
  ++LaneIdx;
}

// Separately constrain the first copy's destination. Because of the
// limitation in constrainOperandRegClass, we can't guarantee that this will
// actually be constrained. So, do it ourselves using the second operand.
const TargetRegisterClass *RC =
    MRI.getRegClassOrNull(I.getOperand(1).getReg());
if (!RC) {
  LLVM_DEBUG(dbgs() << "Couldn't constrain copy destination.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Couldn't constrain copy destination.\n"
; } } while (false);
  return false;
}

RBI.constrainGenericRegister(CopyTo, *RC, MRI);
I.eraseFromParent();
return true;
4361}

4363bool AArch64InstructionSelector::selectConcatVectors(
  MachineInstr &I, MachineRegisterInfo &MRI)  {
assert(I.getOpcode() == TargetOpcode::G_CONCAT_VECTORS &&(static_cast <bool> (I.getOpcode() == TargetOpcode::G_CONCAT_VECTORS
 && "Unexpected opcode") ? void (0) : __assert_fail (
"I.getOpcode() == TargetOpcode::G_CONCAT_VECTORS && \"Unexpected opcode\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4366, __extension__ __PRETTY_FUNCTION__))
       "Unexpected opcode")(static_cast <bool> (I.getOpcode() == TargetOpcode::G_CONCAT_VECTORS
 && "Unexpected opcode") ? void (0) : __assert_fail (
"I.getOpcode() == TargetOpcode::G_CONCAT_VECTORS && \"Unexpected opcode\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4366, __extension__ __PRETTY_FUNCTION__));
Register Dst = I.getOperand(0).getReg();
Register Op1 = I.getOperand(1).getReg();
Register Op2 = I.getOperand(2).getReg();
MachineInstr *ConcatMI = emitVectorConcat(Dst, Op1, Op2, MIB);
if (!ConcatMI)
  return false;
I.eraseFromParent();
return true;
4375}

4377unsigned
4378AArch64InstructionSelector::emitConstantPoolEntry(const Constant *CPVal,
                                                MachineFunction &MF) const {
Type *CPTy = CPVal->getType();
Align Alignment = MF.getDataLayout().getPrefTypeAlign(CPTy);

MachineConstantPool *MCP = MF.getConstantPool();
return MCP->getConstantPoolIndex(CPVal, Alignment);
4385}

4387MachineInstr *AArch64InstructionSelector::emitLoadFromConstantPool(
  const Constant *CPVal, MachineIRBuilder &MIRBuilder) const {
const TargetRegisterClass *RC;
unsigned Opc;
bool IsTiny = TM.getCodeModel() == CodeModel::Tiny;
unsigned Size = MIRBuilder.getDataLayout().getTypeStoreSize(CPVal->getType());
switch (Size) {
case 16:
  RC = &AArch64::FPR128RegClass;
  Opc = IsTiny ? AArch64::LDRQl : AArch64::LDRQui;
  break;
case 8:
  RC = &AArch64::FPR64RegClass;
  Opc = IsTiny ? AArch64::LDRDl : AArch64::LDRDui;
  break;
case 4:
  RC = &AArch64::FPR32RegClass;
  Opc = IsTiny ? AArch64::LDRSl : AArch64::LDRSui;
  break;
case 2:
  RC = &AArch64::FPR16RegClass;
  Opc = AArch64::LDRHui;
  break;
default:
  LLVM_DEBUG(dbgs() << "Could not load from constant pool of type "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Could not load from constant pool of type "
 << *CPVal->getType(); } } while (false)
                    << *CPVal->getType())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Could not load from constant pool of type "
 << *CPVal->getType(); } } while (false);
  return nullptr;
}

MachineInstr *LoadMI = nullptr;
auto &MF = MIRBuilder.getMF();
unsigned CPIdx = emitConstantPoolEntry(CPVal, MF);
if (IsTiny && (Size == 16 || Size == 8 || Size == 4)) {
  // Use load(literal) for tiny code model.
  LoadMI = &*MIRBuilder.buildInstr(Opc, {RC}, {}).addConstantPoolIndex(CPIdx);
} else {
  auto Adrp =
      MIRBuilder.buildInstr(AArch64::ADRP, {&AArch64::GPR64RegClass}, {})
          .addConstantPoolIndex(CPIdx, 0, AArch64II::MO_PAGE);

  LoadMI = &*MIRBuilder.buildInstr(Opc, {RC}, {Adrp})
                 .addConstantPoolIndex(
                     CPIdx, 0, AArch64II::MO_PAGEOFF | AArch64II::MO_NC);

  constrainSelectedInstRegOperands(*Adrp, TII, TRI, RBI);
}

MachinePointerInfo PtrInfo = MachinePointerInfo::getConstantPool(MF);
LoadMI->addMemOperand(MF, MF.getMachineMemOperand(PtrInfo,
                                                  MachineMemOperand::MOLoad,
                                                  Size, Align(Size)));
constrainSelectedInstRegOperands(*LoadMI, TII, TRI, RBI);
return LoadMI;
4440}

4442/// Return an <Opcode, SubregIndex> pair to do an vector elt insert of a given
4443/// size and RB.
4444static std::pair<unsigned, unsigned>
4445getInsertVecEltOpInfo(const RegisterBank &RB, unsigned EltSize) {
unsigned Opc, SubregIdx;
if (RB.getID() == AArch64::GPRRegBankID) {
  if (EltSize == 16) {
    Opc = AArch64::INSvi16gpr;
    SubregIdx = AArch64::ssub;
  } else if (EltSize == 32) {
    Opc = AArch64::INSvi32gpr;
    SubregIdx = AArch64::ssub;
  } else if (EltSize == 64) {
    Opc = AArch64::INSvi64gpr;
    SubregIdx = AArch64::dsub;
  } else {
    llvm_unreachable("invalid elt size!")::llvm::llvm_unreachable_internal("invalid elt size!", "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4458);
  }
} else {
  if (EltSize == 8) {
    Opc = AArch64::INSvi8lane;
    SubregIdx = AArch64::bsub;
  } else if (EltSize == 16) {
    Opc = AArch64::INSvi16lane;
    SubregIdx = AArch64::hsub;
  } else if (EltSize == 32) {
    Opc = AArch64::INSvi32lane;
    SubregIdx = AArch64::ssub;
  } else if (EltSize == 64) {
    Opc = AArch64::INSvi64lane;
    SubregIdx = AArch64::dsub;
  } else {
    llvm_unreachable("invalid elt size!")::llvm::llvm_unreachable_internal("invalid elt size!", "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4474);
  }
}
return std::make_pair(Opc, SubregIdx);
4478}

4480MachineInstr *AArch64InstructionSelector::emitInstr(
  unsigned Opcode, std::initializer_list<llvm::DstOp> DstOps,
  std::initializer_list<llvm::SrcOp> SrcOps, MachineIRBuilder &MIRBuilder,
  const ComplexRendererFns &RenderFns) const {
assert(Opcode && "Expected an opcode?")(static_cast <bool> (Opcode && "Expected an opcode?"
) ? void (0) : __assert_fail ("Opcode && \"Expected an opcode?\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4484, __extension__ __PRETTY_FUNCTION__));
assert(!isPreISelGenericOpcode(Opcode) &&(static_cast <bool> (!isPreISelGenericOpcode(Opcode) &&
 "Function should only be used to produce selected instructions!"
) ? void (0) : __assert_fail ("!isPreISelGenericOpcode(Opcode) && \"Function should only be used to produce selected instructions!\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4486, __extension__ __PRETTY_FUNCTION__))
       "Function should only be used to produce selected instructions!")(static_cast <bool> (!isPreISelGenericOpcode(Opcode) &&
 "Function should only be used to produce selected instructions!"
) ? void (0) : __assert_fail ("!isPreISelGenericOpcode(Opcode) && \"Function should only be used to produce selected instructions!\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4486, __extension__ __PRETTY_FUNCTION__));
auto MI = MIRBuilder.buildInstr(Opcode, DstOps, SrcOps);
if (RenderFns)
  for (auto &Fn : *RenderFns)
    Fn(MI);
constrainSelectedInstRegOperands(*MI, TII, TRI, RBI);
return &*MI;
4493}

4495MachineInstr *AArch64InstructionSelector::emitAddSub(
  const std::array<std::array<unsigned, 2>, 5> &AddrModeAndSizeToOpcode,
  Register Dst, MachineOperand &LHS, MachineOperand &RHS,
  MachineIRBuilder &MIRBuilder) const {
MachineRegisterInfo &MRI = MIRBuilder.getMF().getRegInfo();
assert(LHS.isReg() && RHS.isReg() && "Expected register operands?")(static_cast <bool> (LHS.isReg() && RHS.isReg()
 && "Expected register operands?") ? void (0) : __assert_fail
 ("LHS.isReg() && RHS.isReg() && \"Expected register operands?\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4500, __extension__ __PRETTY_FUNCTION__));
auto Ty = MRI.getType(LHS.getReg());
assert(!Ty.isVector() && "Expected a scalar or pointer?")(static_cast <bool> (!Ty.isVector() && "Expected a scalar or pointer?"
) ? void (0) : __assert_fail ("!Ty.isVector() && \"Expected a scalar or pointer?\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4502, __extension__ __PRETTY_FUNCTION__));
unsigned Size = Ty.getSizeInBits();
assert((Size == 32 || Size == 64) && "Expected a 32-bit or 64-bit type only")(static_cast <bool> ((Size == 32 || Size == 64) &&
 "Expected a 32-bit or 64-bit type only") ? void (0) : __assert_fail
 ("(Size == 32 || Size == 64) && \"Expected a 32-bit or 64-bit type only\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4504, __extension__ __PRETTY_FUNCTION__));
bool Is32Bit = Size == 32;

// INSTRri form with positive arithmetic immediate.
if (auto Fns = selectArithImmed(RHS))
  return emitInstr(AddrModeAndSizeToOpcode[0][Is32Bit], {Dst}, {LHS},
                   MIRBuilder, Fns);

// INSTRri form with negative arithmetic immediate.
if (auto Fns = selectNegArithImmed(RHS))
  return emitInstr(AddrModeAndSizeToOpcode[3][Is32Bit], {Dst}, {LHS},
                   MIRBuilder, Fns);

// INSTRrx form.
if (auto Fns = selectArithExtendedRegister(RHS))
  return emitInstr(AddrModeAndSizeToOpcode[4][Is32Bit], {Dst}, {LHS},
                   MIRBuilder, Fns);

// INSTRrs form.
if (auto Fns = selectShiftedRegister(RHS))
  return emitInstr(AddrModeAndSizeToOpcode[1][Is32Bit], {Dst}, {LHS},
                   MIRBuilder, Fns);
return emitInstr(AddrModeAndSizeToOpcode[2][Is32Bit], {Dst}, {LHS, RHS},
                 MIRBuilder);
4528}

4530MachineInstr *
4531AArch64InstructionSelector::emitADD(Register DefReg, MachineOperand &LHS,
                                  MachineOperand &RHS,
                                  MachineIRBuilder &MIRBuilder) const {
const std::array<std::array<unsigned, 2>, 5> OpcTable{
    {{AArch64::ADDXri, AArch64::ADDWri},
     {AArch64::ADDXrs, AArch64::ADDWrs},
     {AArch64::ADDXrr, AArch64::ADDWrr},
     {AArch64::SUBXri, AArch64::SUBWri},
     {AArch64::ADDXrx, AArch64::ADDWrx}}};
return emitAddSub(OpcTable, DefReg, LHS, RHS, MIRBuilder);
4541}

4543MachineInstr *
4544AArch64InstructionSelector::emitADDS(Register Dst, MachineOperand &LHS,
                                   MachineOperand &RHS,
                                   MachineIRBuilder &MIRBuilder) const {
const std::array<std::array<unsigned, 2>, 5> OpcTable{
    {{AArch64::ADDSXri, AArch64::ADDSWri},
     {AArch64::ADDSXrs, AArch64::ADDSWrs},
     {AArch64::ADDSXrr, AArch64::ADDSWrr},
     {AArch64::SUBSXri, AArch64::SUBSWri},
     {AArch64::ADDSXrx, AArch64::ADDSWrx}}};
return emitAddSub(OpcTable, Dst, LHS, RHS, MIRBuilder);
4554}

4556MachineInstr *
4557AArch64InstructionSelector::emitSUBS(Register Dst, MachineOperand &LHS,
                                   MachineOperand &RHS,
                                   MachineIRBuilder &MIRBuilder) const {
const std::array<std::array<unsigned, 2>, 5> OpcTable{
    {{AArch64::SUBSXri, AArch64::SUBSWri},
     {AArch64::SUBSXrs, AArch64::SUBSWrs},
     {AArch64::SUBSXrr, AArch64::SUBSWrr},
     {AArch64::ADDSXri, AArch64::ADDSWri},
     {AArch64::SUBSXrx, AArch64::SUBSWrx}}};
return emitAddSub(OpcTable, Dst, LHS, RHS, MIRBuilder);
4567}

4569MachineInstr *
4570AArch64InstructionSelector::emitCMN(MachineOperand &LHS, MachineOperand &RHS,
                                  MachineIRBuilder &MIRBuilder) const {
MachineRegisterInfo &MRI = MIRBuilder.getMF().getRegInfo();
bool Is32Bit = (MRI.getType(LHS.getReg()).getSizeInBits() == 32);
auto RC = Is32Bit ? &AArch64::GPR32RegClass : &AArch64::GPR64RegClass;
return emitADDS(MRI.createVirtualRegister(RC), LHS, RHS, MIRBuilder);
4576}

4578MachineInstr *
4579AArch64InstructionSelector::emitTST(MachineOperand &LHS, MachineOperand &RHS,
                                  MachineIRBuilder &MIRBuilder) const {
assert(LHS.isReg() && RHS.isReg() && "Expected register operands?")(static_cast <bool> (LHS.isReg() && RHS.isReg()
 && "Expected register operands?") ? void (0) : __assert_fail
 ("LHS.isReg() && RHS.isReg() && \"Expected register operands?\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4581, __extension__ __PRETTY_FUNCTION__));
MachineRegisterInfo &MRI = MIRBuilder.getMF().getRegInfo();
LLT Ty = MRI.getType(LHS.getReg());
unsigned RegSize = Ty.getSizeInBits();
bool Is32Bit = (RegSize == 32);
const unsigned OpcTable[3][2] = {{AArch64::ANDSXri, AArch64::ANDSWri},
                                 {AArch64::ANDSXrs, AArch64::ANDSWrs},
                                 {AArch64::ANDSXrr, AArch64::ANDSWrr}};
// ANDS needs a logical immediate for its immediate form. Check if we can
// fold one in.
if (auto ValAndVReg = getIConstantVRegValWithLookThrough(RHS.getReg(), MRI)) {
  int64_t Imm = ValAndVReg->Value.getSExtValue();

  if (AArch64_AM::isLogicalImmediate(Imm, RegSize)) {
    auto TstMI = MIRBuilder.buildInstr(OpcTable[0][Is32Bit], {Ty}, {LHS});
    TstMI.addImm(AArch64_AM::encodeLogicalImmediate(Imm, RegSize));
    constrainSelectedInstRegOperands(*TstMI, TII, TRI, RBI);
    return &*TstMI;
  }
}

if (auto Fns = selectLogicalShiftedRegister(RHS))
  return emitInstr(OpcTable[1][Is32Bit], {Ty}, {LHS}, MIRBuilder, Fns);
return emitInstr(OpcTable[2][Is32Bit], {Ty}, {LHS, RHS}, MIRBuilder);
4605}

4607MachineInstr *AArch64InstructionSelector::emitIntegerCompare(
  MachineOperand &LHS, MachineOperand &RHS, MachineOperand &Predicate,
  MachineIRBuilder &MIRBuilder) const {
assert(LHS.isReg() && RHS.isReg() && "Expected LHS and RHS to be registers!")(static_cast <bool> (LHS.isReg() && RHS.isReg()
 && "Expected LHS and RHS to be registers!") ? void (
0) : __assert_fail ("LHS.isReg() && RHS.isReg() && \"Expected LHS and RHS to be registers!\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4610, __extension__ __PRETTY_FUNCTION__));
assert(Predicate.isPredicate() && "Expected predicate?")(static_cast <bool> (Predicate.isPredicate() &&
 "Expected predicate?") ? void (0) : __assert_fail ("Predicate.isPredicate() && \"Expected predicate?\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4611, __extension__ __PRETTY_FUNCTION__));
MachineRegisterInfo &MRI = MIRBuilder.getMF().getRegInfo();
LLT CmpTy = MRI.getType(LHS.getReg());
assert(!CmpTy.isVector() && "Expected scalar or pointer")(static_cast <bool> (!CmpTy.isVector() && "Expected scalar or pointer"
) ? void (0) : __assert_fail ("!CmpTy.isVector() && \"Expected scalar or pointer\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4614, __extension__ __PRETTY_FUNCTION__));
unsigned Size = CmpTy.getSizeInBits();
(void)Size;
assert((Size == 32 || Size == 64) && "Expected a 32-bit or 64-bit LHS/RHS?")(static_cast <bool> ((Size == 32 || Size == 64) &&
 "Expected a 32-bit or 64-bit LHS/RHS?") ? void (0) : __assert_fail
 ("(Size == 32 || Size == 64) && \"Expected a 32-bit or 64-bit LHS/RHS?\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4617, __extension__ __PRETTY_FUNCTION__));
// Fold the compare into a cmn or tst if possible.
if (auto FoldCmp = tryFoldIntegerCompare(LHS, RHS, Predicate, MIRBuilder))
  return FoldCmp;
auto Dst = MRI.cloneVirtualRegister(LHS.getReg());
return emitSUBS(Dst, LHS, RHS, MIRBuilder);
4623}

4625MachineInstr *AArch64InstructionSelector::emitCSetForFCmp(
  Register Dst, CmpInst::Predicate Pred, MachineIRBuilder &MIRBuilder) const {
MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
4628#ifndef NDEBUG
LLT Ty = MRI.getType(Dst);
assert(!Ty.isVector() && Ty.getSizeInBits() == 32 &&(static_cast <bool> (!Ty.isVector() && Ty.getSizeInBits
() == 32 && "Expected a 32-bit scalar register?") ? void
 (0) : __assert_fail ("!Ty.isVector() && Ty.getSizeInBits() == 32 && \"Expected a 32-bit scalar register?\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4631, __extension__ __PRETTY_FUNCTION__))
       "Expected a 32-bit scalar register?")(static_cast <bool> (!Ty.isVector() && Ty.getSizeInBits
() == 32 && "Expected a 32-bit scalar register?") ? void
 (0) : __assert_fail ("!Ty.isVector() && Ty.getSizeInBits() == 32 && \"Expected a 32-bit scalar register?\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4631, __extension__ __PRETTY_FUNCTION__));
4632#endif
const Register ZReg = AArch64::WZR;
AArch64CC::CondCode CC1, CC2;
changeFCMPPredToAArch64CC(Pred, CC1, CC2);
auto InvCC1 = AArch64CC::getInvertedCondCode(CC1);
if (CC2 == AArch64CC::AL)
  return emitCSINC(/*Dst=*/Dst, /*Src1=*/ZReg, /*Src2=*/ZReg, InvCC1,
                   MIRBuilder);
const TargetRegisterClass *RC = &AArch64::GPR32RegClass;
Register Def1Reg = MRI.createVirtualRegister(RC);
Register Def2Reg = MRI.createVirtualRegister(RC);
auto InvCC2 = AArch64CC::getInvertedCondCode(CC2);
emitCSINC(/*Dst=*/Def1Reg, /*Src1=*/ZReg, /*Src2=*/ZReg, InvCC1, MIRBuilder);
emitCSINC(/*Dst=*/Def2Reg, /*Src1=*/ZReg, /*Src2=*/ZReg, InvCC2, MIRBuilder);
auto OrMI = MIRBuilder.buildInstr(AArch64::ORRWrr, {Dst}, {Def1Reg, Def2Reg});
constrainSelectedInstRegOperands(*OrMI, TII, TRI, RBI);
return &*OrMI;
4649}

4651MachineInstr *AArch64InstructionSelector::emitFPCompare(
  Register LHS, Register RHS, MachineIRBuilder &MIRBuilder,
  std::optional<CmpInst::Predicate> Pred) const {
MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
LLT Ty = MRI.getType(LHS);
if (Ty.isVector())
  return nullptr;
unsigned OpSize = Ty.getSizeInBits();
if (OpSize != 32 && OpSize != 64)
  return nullptr;

// If this is a compare against +0.0, then we don't have
// to explicitly materialize a constant.
const ConstantFP *FPImm = getConstantFPVRegVal(RHS, MRI);
bool ShouldUseImm = FPImm && (FPImm->isZero() && !FPImm->isNegative());

auto IsEqualityPred = [](CmpInst::Predicate P) {
  return P == CmpInst::FCMP_OEQ || P == CmpInst::FCMP_ONE ||
         P == CmpInst::FCMP_UEQ || P == CmpInst::FCMP_UNE;
};
if (!ShouldUseImm && Pred && IsEqualityPred(*Pred)) {
  // Try commutating the operands.
  const ConstantFP *LHSImm = getConstantFPVRegVal(LHS, MRI);
  if (LHSImm && (LHSImm->isZero() && !LHSImm->isNegative())) {
    ShouldUseImm = true;
    std::swap(LHS, RHS);
  }
}
unsigned CmpOpcTbl[2][2] = {{AArch64::FCMPSrr, AArch64::FCMPDrr},
                            {AArch64::FCMPSri, AArch64::FCMPDri}};
unsigned CmpOpc = CmpOpcTbl[ShouldUseImm][OpSize == 64];

// Partially build the compare. Decide if we need to add a use for the
// third operand based off whether or not we're comparing against 0.0.
auto CmpMI = MIRBuilder.buildInstr(CmpOpc).addUse(LHS);
CmpMI.setMIFlags(MachineInstr::NoFPExcept);
if (!ShouldUseImm)
  CmpMI.addUse(RHS);
constrainSelectedInstRegOperands(*CmpMI, TII, TRI, RBI);
return &*CmpMI;
4691}

4693MachineInstr *AArch64InstructionSelector::emitVectorConcat(
  std::optional<Register> Dst, Register Op1, Register Op2,
  MachineIRBuilder &MIRBuilder) const {
// We implement a vector concat by:
// 1. Use scalar_to_vector to insert the lower vector into the larger dest
// 2. Insert the upper vector into the destination's upper element
// TODO: some of this code is common with G_BUILD_VECTOR handling.
MachineRegisterInfo &MRI = MIRBuilder.getMF().getRegInfo();

const LLT Op1Ty = MRI.getType(Op1);
const LLT Op2Ty = MRI.getType(Op2);

if (Op1Ty != Op2Ty) {
  LLVM_DEBUG(dbgs() << "Could not do vector concat of differing vector tys")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Could not do vector concat of differing vector tys"
; } } while (false);
  return nullptr;
}
assert(Op1Ty.isVector() && "Expected a vector for vector concat")(static_cast <bool> (Op1Ty.isVector() && "Expected a vector for vector concat"
) ? void (0) : __assert_fail ("Op1Ty.isVector() && \"Expected a vector for vector concat\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4709, __extension__ __PRETTY_FUNCTION__));

if (Op1Ty.getSizeInBits() >= 128) {
  LLVM_DEBUG(dbgs() << "Vector concat not supported for full size vectors")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Vector concat not supported for full size vectors"
; } } while (false);
  return nullptr;
}

// At the moment we just support 64 bit vector concats.
if (Op1Ty.getSizeInBits() != 64) {
  LLVM_DEBUG(dbgs() << "Vector concat supported for 64b vectors")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Vector concat supported for 64b vectors"
; } } while (false);
  return nullptr;
}

const LLT ScalarTy = LLT::scalar(Op1Ty.getSizeInBits());
const RegisterBank &FPRBank = *RBI.getRegBank(Op1, MRI, TRI);
const TargetRegisterClass *DstRC =
    getRegClassForTypeOnBank(Op1Ty.multiplyElements(2), FPRBank);

MachineInstr *WidenedOp1 =
    emitScalarToVector(ScalarTy.getSizeInBits(), DstRC, Op1, MIRBuilder);
MachineInstr *WidenedOp2 =
    emitScalarToVector(ScalarTy.getSizeInBits(), DstRC, Op2, MIRBuilder);
if (!WidenedOp1 || !WidenedOp2) {
  LLVM_DEBUG(dbgs() << "Could not emit a vector from scalar value")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Could not emit a vector from scalar value"
; } } while (false);
  return nullptr;
}

// Now do the insert of the upper element.
unsigned InsertOpc, InsSubRegIdx;
std::tie(InsertOpc, InsSubRegIdx) =
    getInsertVecEltOpInfo(FPRBank, ScalarTy.getSizeInBits());

if (!Dst)
  Dst = MRI.createVirtualRegister(DstRC);
auto InsElt =
    MIRBuilder
        .buildInstr(InsertOpc, {*Dst}, {WidenedOp1->getOperand(0).getReg()})
        .addImm(1) /* Lane index */
        .addUse(WidenedOp2->getOperand(0).getReg())
        .addImm(0);
constrainSelectedInstRegOperands(*InsElt, TII, TRI, RBI);
return &*InsElt;
4751}

4753MachineInstr *
4754AArch64InstructionSelector::emitCSINC(Register Dst, Register Src1,
                                    Register Src2, AArch64CC::CondCode Pred,
                                    MachineIRBuilder &MIRBuilder) const {
auto &MRI = *MIRBuilder.getMRI();
const RegClassOrRegBank &RegClassOrBank = MRI.getRegClassOrRegBank(Dst);
// If we used a register class, then this won't necessarily have an LLT.
// Compute the size based off whether or not we have a class or bank.
unsigned Size;
if (const auto *RC = RegClassOrBank.dyn_cast<const TargetRegisterClass *>())
  Size = TRI.getRegSizeInBits(*RC);
else
  Size = MRI.getType(Dst).getSizeInBits();
// Some opcodes use s1.
assert(Size <= 64 && "Expected 64 bits or less only!")(static_cast <bool> (Size <= 64 && "Expected 64 bits or less only!"
) ? void (0) : __assert_fail ("Size <= 64 && \"Expected 64 bits or less only!\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4767, __extension__ __PRETTY_FUNCTION__));
static const unsigned OpcTable[2] = {AArch64::CSINCWr, AArch64::CSINCXr};
unsigned Opc = OpcTable[Size == 64];
auto CSINC = MIRBuilder.buildInstr(Opc, {Dst}, {Src1, Src2}).addImm(Pred);
constrainSelectedInstRegOperands(*CSINC, TII, TRI, RBI);
return &*CSINC;
4773}

4775std::pair<MachineInstr *, AArch64CC::CondCode>
4776AArch64InstructionSelector::emitOverflowOp(unsigned Opcode, Register Dst,
                                         MachineOperand &LHS,
                                         MachineOperand &RHS,
                                         MachineIRBuilder &MIRBuilder) const {
switch (Opcode) {
default:
  llvm_unreachable("Unexpected opcode!")::llvm::llvm_unreachable_internal("Unexpected opcode!", "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4782);
case TargetOpcode::G_SADDO:
  return std::make_pair(emitADDS(Dst, LHS, RHS, MIRBuilder), AArch64CC::VS);
case TargetOpcode::G_UADDO:
  return std::make_pair(emitADDS(Dst, LHS, RHS, MIRBuilder), AArch64CC::HS);
case TargetOpcode::G_SSUBO:
  return std::make_pair(emitSUBS(Dst, LHS, RHS, MIRBuilder), AArch64CC::VS);
case TargetOpcode::G_USUBO:
  return std::make_pair(emitSUBS(Dst, LHS, RHS, MIRBuilder), AArch64CC::LO);
}
4792}

4794/// Returns true if @p Val is a tree of AND/OR/CMP operations that can be
4795/// expressed as a conjunction.
4796/// \param CanNegate    Set to true if we can negate the whole sub-tree just by
4797///                     changing the conditions on the CMP tests.
4798///                     (this means we can call emitConjunctionRec() with
4799///                      Negate==true on this sub-tree)
4800/// \param MustBeFirst  Set to true if this subtree needs to be negated and we
4801///                     cannot do the negation naturally. We are required to
4802///                     emit the subtree first in this case.
4803/// \param WillNegate   Is true if are called when the result of this
4804///                     subexpression must be negated. This happens when the
4805///                     outer expression is an OR. We can use this fact to know
4806///                     that we have a double negation (or (or ...) ...) that
4807///                     can be implemented for free.
4808static bool canEmitConjunction(Register Val, bool &CanNegate, bool &MustBeFirst,
                             bool WillNegate, MachineRegisterInfo &MRI,
                             unsigned Depth = 0) {
if (!MRI.hasOneNonDBGUse(Val))
  return false;
MachineInstr *ValDef = MRI.getVRegDef(Val);
unsigned Opcode = ValDef->getOpcode();
if (isa<GAnyCmp>(ValDef)) {
  CanNegate = true;
  MustBeFirst = false;
  return true;
}
// Protect against exponential runtime and stack overflow.
if (Depth > 6)
  return false;
if (Opcode == TargetOpcode::G_AND || Opcode == TargetOpcode::G_OR) {
  bool IsOR = Opcode == TargetOpcode::G_OR;
  Register O0 = ValDef->getOperand(1).getReg();
  Register O1 = ValDef->getOperand(2).getReg();
  bool CanNegateL;
  bool MustBeFirstL;
  if (!canEmitConjunction(O0, CanNegateL, MustBeFirstL, IsOR, MRI, Depth + 1))
    return false;
  bool CanNegateR;
  bool MustBeFirstR;
  if (!canEmitConjunction(O1, CanNegateR, MustBeFirstR, IsOR, MRI, Depth + 1))
    return false;

  if (MustBeFirstL && MustBeFirstR)
    return false;

  if (IsOR) {
    // For an OR expression we need to be able to naturally negate at least
    // one side or we cannot do the transformation at all.
    if (!CanNegateL && !CanNegateR)
      return false;
    // If we the result of the OR will be negated and we can naturally negate
    // the leaves, then this sub-tree as a whole negates naturally.
    CanNegate = WillNegate && CanNegateL && CanNegateR;
    // If we cannot naturally negate the whole sub-tree, then this must be
    // emitted first.
    MustBeFirst = !CanNegate;
  } else {
    assert(Opcode == TargetOpcode::G_AND && "Must be G_AND")(static_cast <bool> (Opcode == TargetOpcode::G_AND &&
 "Must be G_AND") ? void (0) : __assert_fail ("Opcode == TargetOpcode::G_AND && \"Must be G_AND\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4851, __extension__ __PRETTY_FUNCTION__));
    // We cannot naturally negate an AND operation.
    CanNegate = false;
    MustBeFirst = MustBeFirstL || MustBeFirstR;
  }
  return true;
}
return false;
4859}

4861MachineInstr *AArch64InstructionSelector::emitConditionalComparison(
  Register LHS, Register RHS, CmpInst::Predicate CC,
  AArch64CC::CondCode Predicate, AArch64CC::CondCode OutCC,
  MachineIRBuilder &MIB) const {
// TODO: emit CMN as an optimization.
auto &MRI = *MIB.getMRI();
LLT OpTy = MRI.getType(LHS);
assert(OpTy.getSizeInBits() == 32 || OpTy.getSizeInBits() == 64)(static_cast <bool> (OpTy.getSizeInBits() == 32 || OpTy
.getSizeInBits() == 64) ? void (0) : __assert_fail ("OpTy.getSizeInBits() == 32 || OpTy.getSizeInBits() == 64"
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4868, __extension__ __PRETTY_FUNCTION__));
unsigned CCmpOpc;
std::optional<ValueAndVReg> C;
if (CmpInst::isIntPredicate(CC)) {
  C = getIConstantVRegValWithLookThrough(RHS, MRI);
  if (C && C->Value.ult(32))
    CCmpOpc = OpTy.getSizeInBits() == 32 ? AArch64::CCMPWi : AArch64::CCMPXi;
  else
    CCmpOpc = OpTy.getSizeInBits() == 32 ? AArch64::CCMPWr : AArch64::CCMPXr;
} else {
  switch (OpTy.getSizeInBits()) {
  case 16:
    CCmpOpc = AArch64::FCCMPHrr;
    break;
  case 32:
    CCmpOpc = AArch64::FCCMPSrr;
    break;
  case 64:
    CCmpOpc = AArch64::FCCMPDrr;
    break;
  default:
    return nullptr;
  }
}
AArch64CC::CondCode InvOutCC = AArch64CC::getInvertedCondCode(OutCC);
unsigned NZCV = AArch64CC::getNZCVToSatisfyCondCode(InvOutCC);
auto CCmp =
    MIB.buildInstr(CCmpOpc, {}, {LHS});
if (CCmpOpc == AArch64::CCMPWi || CCmpOpc == AArch64::CCMPXi)
  CCmp.addImm(C->Value.getZExtValue());
else
  CCmp.addReg(RHS);
CCmp.addImm(NZCV).addImm(Predicate);
constrainSelectedInstRegOperands(*CCmp, TII, TRI, RBI);
return &*CCmp;
4903}

4905MachineInstr *AArch64InstructionSelector::emitConjunctionRec(
  Register Val, AArch64CC::CondCode &OutCC, bool Negate, Register CCOp,
  AArch64CC::CondCode Predicate, MachineIRBuilder &MIB) const {
// We're at a tree leaf, produce a conditional comparison operation.
auto &MRI = *MIB.getMRI();
MachineInstr *ValDef = MRI.getVRegDef(Val);
unsigned Opcode = ValDef->getOpcode();
if (auto *Cmp = dyn_cast<GAnyCmp>(ValDef)) {
  Register LHS = Cmp->getLHSReg();
  Register RHS = Cmp->getRHSReg();
  CmpInst::Predicate CC = Cmp->getCond();
  if (Negate)
    CC = CmpInst::getInversePredicate(CC);
  if (isa<GICmp>(Cmp)) {
    OutCC = changeICMPPredToAArch64CC(CC);
  } else {
    // Handle special FP cases.
    AArch64CC::CondCode ExtraCC;
    changeFPCCToANDAArch64CC(CC, OutCC, ExtraCC);
    // Some floating point conditions can't be tested with a single condition
    // code. Construct an additional comparison in this case.
    if (ExtraCC != AArch64CC::AL) {
      MachineInstr *ExtraCmp;
      if (!CCOp)
        ExtraCmp = emitFPCompare(LHS, RHS, MIB, CC);
      else
        ExtraCmp =
            emitConditionalComparison(LHS, RHS, CC, Predicate, ExtraCC, MIB);
      CCOp = ExtraCmp->getOperand(0).getReg();
      Predicate = ExtraCC;
    }
  }

  // Produce a normal comparison if we are first in the chain
  if (!CCOp) {
    auto Dst = MRI.cloneVirtualRegister(LHS);
    if (isa<GICmp>(Cmp))
      return emitSUBS(Dst, Cmp->getOperand(2), Cmp->getOperand(3), MIB);
    return emitFPCompare(Cmp->getOperand(2).getReg(),
                         Cmp->getOperand(3).getReg(), MIB);
  }
  // Otherwise produce a ccmp.
  return emitConditionalComparison(LHS, RHS, CC, Predicate, OutCC, MIB);
}
assert(MRI.hasOneNonDBGUse(Val) && "Valid conjunction/disjunction tree")(static_cast <bool> (MRI.hasOneNonDBGUse(Val) &&
 "Valid conjunction/disjunction tree") ? void (0) : __assert_fail
 ("MRI.hasOneNonDBGUse(Val) && \"Valid conjunction/disjunction tree\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4949, __extension__ __PRETTY_FUNCTION__));

bool IsOR = Opcode == TargetOpcode::G_OR;

Register LHS = ValDef->getOperand(1).getReg();
bool CanNegateL;
bool MustBeFirstL;
bool ValidL = canEmitConjunction(LHS, CanNegateL, MustBeFirstL, IsOR, MRI);
assert(ValidL && "Valid conjunction/disjunction tree")(static_cast <bool> (ValidL && "Valid conjunction/disjunction tree"
) ? void (0) : __assert_fail ("ValidL && \"Valid conjunction/disjunction tree\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4957, __extension__ __PRETTY_FUNCTION__));
(void)ValidL;

Register RHS = ValDef->getOperand(2).getReg();
bool CanNegateR;
bool MustBeFirstR;
bool ValidR = canEmitConjunction(RHS, CanNegateR, MustBeFirstR, IsOR, MRI);
assert(ValidR && "Valid conjunction/disjunction tree")(static_cast <bool> (ValidR && "Valid conjunction/disjunction tree"
) ? void (0) : __assert_fail ("ValidR && \"Valid conjunction/disjunction tree\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4964, __extension__ __PRETTY_FUNCTION__));
(void)ValidR;

// Swap sub-tree that must come first to the right side.
if (MustBeFirstL) {
  assert(!MustBeFirstR && "Valid conjunction/disjunction tree")(static_cast <bool> (!MustBeFirstR && "Valid conjunction/disjunction tree"
) ? void (0) : __assert_fail ("!MustBeFirstR && \"Valid conjunction/disjunction tree\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4969, __extension__ __PRETTY_FUNCTION__));
  std::swap(LHS, RHS);
  std::swap(CanNegateL, CanNegateR);
  std::swap(MustBeFirstL, MustBeFirstR);
}

bool NegateR;
bool NegateAfterR;
bool NegateL;
bool NegateAfterAll;
if (Opcode == TargetOpcode::G_OR) {
  // Swap the sub-tree that we can negate naturally to the left.
  if (!CanNegateL) {
    assert(CanNegateR && "at least one side must be negatable")(static_cast <bool> (CanNegateR && "at least one side must be negatable"
) ? void (0) : __assert_fail ("CanNegateR && \"at least one side must be negatable\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4982, __extension__ __PRETTY_FUNCTION__));
    assert(!MustBeFirstR && "invalid conjunction/disjunction tree")(static_cast <bool> (!MustBeFirstR && "invalid conjunction/disjunction tree"
) ? void (0) : __assert_fail ("!MustBeFirstR && \"invalid conjunction/disjunction tree\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4983, __extension__ __PRETTY_FUNCTION__));
    assert(!Negate)(static_cast <bool> (!Negate) ? void (0) : __assert_fail
 ("!Negate", "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4984, __extension__ __PRETTY_FUNCTION__));
    std::swap(LHS, RHS);
    NegateR = false;
    NegateAfterR = true;
  } else {
    // Negate the left sub-tree if possible, otherwise negate the result.
    NegateR = CanNegateR;
    NegateAfterR = !CanNegateR;
  }
  NegateL = true;
  NegateAfterAll = !Negate;
} else {
  assert(Opcode == TargetOpcode::G_AND &&(static_cast <bool> (Opcode == TargetOpcode::G_AND &&
 "Valid conjunction/disjunction tree") ? void (0) : __assert_fail
 ("Opcode == TargetOpcode::G_AND && \"Valid conjunction/disjunction tree\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4997, __extension__ __PRETTY_FUNCTION__))
         "Valid conjunction/disjunction tree")(static_cast <bool> (Opcode == TargetOpcode::G_AND &&
 "Valid conjunction/disjunction tree") ? void (0) : __assert_fail
 ("Opcode == TargetOpcode::G_AND && \"Valid conjunction/disjunction tree\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4997, __extension__ __PRETTY_FUNCTION__));
  assert(!Negate && "Valid conjunction/disjunction tree")(static_cast <bool> (!Negate && "Valid conjunction/disjunction tree"
) ? void (0) : __assert_fail ("!Negate && \"Valid conjunction/disjunction tree\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 4998, __extension__ __PRETTY_FUNCTION__));

  NegateL = false;
  NegateR = false;
  NegateAfterR = false;
  NegateAfterAll = false;
}

// Emit sub-trees.
AArch64CC::CondCode RHSCC;
MachineInstr *CmpR =
    emitConjunctionRec(RHS, RHSCC, NegateR, CCOp, Predicate, MIB);
if (NegateAfterR)
  RHSCC = AArch64CC::getInvertedCondCode(RHSCC);
MachineInstr *CmpL = emitConjunctionRec(
    LHS, OutCC, NegateL, CmpR->getOperand(0).getReg(), RHSCC, MIB);
if (NegateAfterAll)
  OutCC = AArch64CC::getInvertedCondCode(OutCC);
return CmpL;
5017}

5019MachineInstr *AArch64InstructionSelector::emitConjunction(
  Register Val, AArch64CC::CondCode &OutCC, MachineIRBuilder &MIB) const {
bool DummyCanNegate;
bool DummyMustBeFirst;
if (!canEmitConjunction(Val, DummyCanNegate, DummyMustBeFirst, false,
                        *MIB.getMRI()))
  return nullptr;
return emitConjunctionRec(Val, OutCC, false, Register(), AArch64CC::AL, MIB);
5027}

5029bool AArch64InstructionSelector::tryOptSelectConjunction(GSelect &SelI,
                                                       MachineInstr &CondMI) {
AArch64CC::CondCode AArch64CC;
MachineInstr *ConjMI = emitConjunction(SelI.getCondReg(), AArch64CC, MIB);
if (!ConjMI)
  return false;

emitSelect(SelI.getReg(0), SelI.getTrueReg(), SelI.getFalseReg(), AArch64CC, MIB);
SelI.eraseFromParent();
return true;
5039}

5041bool AArch64InstructionSelector::tryOptSelect(GSelect &I) {
MachineRegisterInfo &MRI = *MIB.getMRI();
// We want to recognize this pattern:
//
// $z = G_FCMP pred, $x, $y
// ...
// $w = G_SELECT $z, $a, $b
//
// Where the value of $z is *only* ever used by the G_SELECT (possibly with
// some copies/truncs in between.)
//
// If we see this, then we can emit something like this:
//
// fcmp $x, $y
// fcsel $w, $a, $b, pred
//
// Rather than emitting both of the rather long sequences in the standard
// G_FCMP/G_SELECT select methods.

// First, check if the condition is defined by a compare.
MachineInstr *CondDef = MRI.getVRegDef(I.getOperand(1).getReg());

// We can only fold if all of the defs have one use.
Register CondDefReg = CondDef->getOperand(0).getReg();
if (!MRI.hasOneNonDBGUse(CondDefReg)) {
  // Unless it's another select.
  for (const MachineInstr &UI : MRI.use_nodbg_instructions(CondDefReg)) {
    if (CondDef == &UI)
      continue;
    if (UI.getOpcode() != TargetOpcode::G_SELECT)
      return false;
  }
}

// Is the condition defined by a compare?
unsigned CondOpc = CondDef->getOpcode();
if (CondOpc != TargetOpcode::G_ICMP && CondOpc != TargetOpcode::G_FCMP) {
  if (tryOptSelectConjunction(I, *CondDef))
    return true;
  return false;
}

AArch64CC::CondCode CondCode;
if (CondOpc == TargetOpcode::G_ICMP) {
  auto Pred =
      static_cast<CmpInst::Predicate>(CondDef->getOperand(1).getPredicate());
  CondCode = changeICMPPredToAArch64CC(Pred);
  emitIntegerCompare(CondDef->getOperand(2), CondDef->getOperand(3),
                     CondDef->getOperand(1), MIB);
} else {
  // Get the condition code for the select.
  auto Pred =
      static_cast<CmpInst::Predicate>(CondDef->getOperand(1).getPredicate());
  AArch64CC::CondCode CondCode2;
  changeFCMPPredToAArch64CC(Pred, CondCode, CondCode2);

  // changeFCMPPredToAArch64CC sets CondCode2 to AL when we require two
  // instructions to emit the comparison.
  // TODO: Handle FCMP_UEQ and FCMP_ONE. After that, this check will be
  // unnecessary.
  if (CondCode2 != AArch64CC::AL)
    return false;

  if (!emitFPCompare(CondDef->getOperand(2).getReg(),
                     CondDef->getOperand(3).getReg(), MIB)) {
    LLVM_DEBUG(dbgs() << "Couldn't emit compare for select!\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Couldn't emit compare for select!\n"
; } } while (false);
    return false;
  }
}

// Emit the select.
emitSelect(I.getOperand(0).getReg(), I.getOperand(2).getReg(),
           I.getOperand(3).getReg(), CondCode, MIB);
I.eraseFromParent();
return true;
5116}

5118MachineInstr *AArch64InstructionSelector::tryFoldIntegerCompare(
  MachineOperand &LHS, MachineOperand &RHS, MachineOperand &Predicate,
  MachineIRBuilder &MIRBuilder) const {
assert(LHS.isReg() && RHS.isReg() && Predicate.isPredicate() &&(static_cast <bool> (LHS.isReg() && RHS.isReg()
 && Predicate.isPredicate() && "Unexpected MachineOperand"
) ? void (0) : __assert_fail ("LHS.isReg() && RHS.isReg() && Predicate.isPredicate() && \"Unexpected MachineOperand\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 5122, __extension__ __PRETTY_FUNCTION__))
       "Unexpected MachineOperand")(static_cast <bool> (LHS.isReg() && RHS.isReg()
 && Predicate.isPredicate() && "Unexpected MachineOperand"
) ? void (0) : __assert_fail ("LHS.isReg() && RHS.isReg() && Predicate.isPredicate() && \"Unexpected MachineOperand\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 5122, __extension__ __PRETTY_FUNCTION__));
MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
// We want to find this sort of thing:
// x = G_SUB 0, y
// G_ICMP z, x
//
// In this case, we can fold the G_SUB into the G_ICMP using a CMN instead.
// e.g:
//
// cmn z, y

// Check if the RHS or LHS of the G_ICMP is defined by a SUB
MachineInstr *LHSDef = getDefIgnoringCopies(LHS.getReg(), MRI);
MachineInstr *RHSDef = getDefIgnoringCopies(RHS.getReg(), MRI);
auto P = static_cast<CmpInst::Predicate>(Predicate.getPredicate());
// Given this:
//
// x = G_SUB 0, y
// G_ICMP x, z
//
// Produce this:
//
// cmn y, z
if (isCMN(LHSDef, P, MRI))
  return emitCMN(LHSDef->getOperand(2), RHS, MIRBuilder);

// Same idea here, but with the RHS of the compare instead:
//
// Given this:
//
// x = G_SUB 0, y
// G_ICMP z, x
//
// Produce this:
//
// cmn z, y
if (isCMN(RHSDef, P, MRI))
  return emitCMN(LHS, RHSDef->getOperand(2), MIRBuilder);

// Given this:
//
// z = G_AND x, y
// G_ICMP z, 0
//
// Produce this if the compare is signed:
//
// tst x, y
if (!CmpInst::isUnsigned(P) && LHSDef &&
    LHSDef->getOpcode() == TargetOpcode::G_AND) {
  // Make sure that the RHS is 0.
  auto ValAndVReg = getIConstantVRegValWithLookThrough(RHS.getReg(), MRI);
  if (!ValAndVReg || ValAndVReg->Value != 0)
    return nullptr;

  return emitTST(LHSDef->getOperand(1),
                 LHSDef->getOperand(2), MIRBuilder);
}

return nullptr;
5181}

5183bool AArch64InstructionSelector::selectShuffleVector(
  MachineInstr &I, MachineRegisterInfo &MRI) {
const LLT DstTy = MRI.getType(I.getOperand(0).getReg());
Register Src1Reg = I.getOperand(1).getReg();
const LLT Src1Ty = MRI.getType(Src1Reg);
Register Src2Reg = I.getOperand(2).getReg();
const LLT Src2Ty = MRI.getType(Src2Reg);
ArrayRef<int> Mask = I.getOperand(3).getShuffleMask();

MachineBasicBlock &MBB = *I.getParent();
MachineFunction &MF = *MBB.getParent();
LLVMContext &Ctx = MF.getFunction().getContext();

// G_SHUFFLE_VECTOR is weird in that the source operands can be scalars, if
// it's originated from a <1 x T> type. Those should have been lowered into
// G_BUILD_VECTOR earlier.
if (!Src1Ty.isVector() || !Src2Ty.isVector()) {
  LLVM_DEBUG(dbgs() << "Could not select a \"scalar\" G_SHUFFLE_VECTOR\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Could not select a \"scalar\" G_SHUFFLE_VECTOR\n"
; } } while (false);
  return false;
}

unsigned BytesPerElt = DstTy.getElementType().getSizeInBits() / 8;

SmallVector<Constant *, 64> CstIdxs;
for (int Val : Mask) {
  // For now, any undef indexes we'll just assume to be 0. This should be
  // optimized in future, e.g. to select DUP etc.
  Val = Val < 0 ? 0 : Val;
  for (unsigned Byte = 0; Byte < BytesPerElt; ++Byte) {
    unsigned Offset = Byte + Val * BytesPerElt;
    CstIdxs.emplace_back(ConstantInt::get(Type::getInt8Ty(Ctx), Offset));
  }
}

// Use a constant pool to load the index vector for TBL.
Constant *CPVal = ConstantVector::get(CstIdxs);
MachineInstr *IndexLoad = emitLoadFromConstantPool(CPVal, MIB);
if (!IndexLoad) {
  LLVM_DEBUG(dbgs() << "Could not load from a constant pool")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Could not load from a constant pool"
; } } while (false);
  return false;
}

if (DstTy.getSizeInBits() != 128) {
  assert(DstTy.getSizeInBits() == 64 && "Unexpected shuffle result ty")(static_cast <bool> (DstTy.getSizeInBits() == 64 &&
 "Unexpected shuffle result ty") ? void (0) : __assert_fail (
"DstTy.getSizeInBits() == 64 && \"Unexpected shuffle result ty\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 5226, __extension__ __PRETTY_FUNCTION__));
  // This case can be done with TBL1.
  MachineInstr *Concat =
      emitVectorConcat(std::nullopt, Src1Reg, Src2Reg, MIB);
  if (!Concat) {
    LLVM_DEBUG(dbgs() << "Could not do vector concat for tbl1")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Could not do vector concat for tbl1"
; } } while (false);
    return false;
  }

  // The constant pool load will be 64 bits, so need to convert to FPR128 reg.
  IndexLoad = emitScalarToVector(64, &AArch64::FPR128RegClass,
                                 IndexLoad->getOperand(0).getReg(), MIB);

  auto TBL1 = MIB.buildInstr(
      AArch64::TBLv16i8One, {&AArch64::FPR128RegClass},
      {Concat->getOperand(0).getReg(), IndexLoad->getOperand(0).getReg()});
  constrainSelectedInstRegOperands(*TBL1, TII, TRI, RBI);

  auto Copy =
      MIB.buildInstr(TargetOpcode::COPY, {I.getOperand(0).getReg()}, {})
          .addReg(TBL1.getReg(0), 0, AArch64::dsub);
  RBI.constrainGenericRegister(Copy.getReg(0), AArch64::FPR64RegClass, MRI);
  I.eraseFromParent();
  return true;
}

// For TBL2 we need to emit a REG_SEQUENCE to tie together two consecutive
// Q registers for regalloc.
SmallVector<Register, 2> Regs = {Src1Reg, Src2Reg};
auto RegSeq = createQTuple(Regs, MIB);
auto TBL2 = MIB.buildInstr(AArch64::TBLv16i8Two, {I.getOperand(0)},
                           {RegSeq, IndexLoad->getOperand(0)});
constrainSelectedInstRegOperands(*TBL2, TII, TRI, RBI);
I.eraseFromParent();
return true;
5261}

5263MachineInstr *AArch64InstructionSelector::emitLaneInsert(
  std::optional<Register> DstReg, Register SrcReg, Register EltReg,
  unsigned LaneIdx, const RegisterBank &RB,
  MachineIRBuilder &MIRBuilder) const {
MachineInstr *InsElt = nullptr;
const TargetRegisterClass *DstRC = &AArch64::FPR128RegClass;
MachineRegisterInfo &MRI = *MIRBuilder.getMRI();

// Create a register to define with the insert if one wasn't passed in.
if (!DstReg)
  DstReg = MRI.createVirtualRegister(DstRC);

unsigned EltSize = MRI.getType(EltReg).getSizeInBits();
unsigned Opc = getInsertVecEltOpInfo(RB, EltSize).first;

if (RB.getID() == AArch64::FPRRegBankID) {
  auto InsSub = emitScalarToVector(EltSize, DstRC, EltReg, MIRBuilder);
  InsElt = MIRBuilder.buildInstr(Opc, {*DstReg}, {SrcReg})
               .addImm(LaneIdx)
               .addUse(InsSub->getOperand(0).getReg())
               .addImm(0);
} else {
  InsElt = MIRBuilder.buildInstr(Opc, {*DstReg}, {SrcReg})
               .addImm(LaneIdx)
               .addUse(EltReg);
}

constrainSelectedInstRegOperands(*InsElt, TII, TRI, RBI);
return InsElt;
5292}

5294bool AArch64InstructionSelector::selectUSMovFromExtend(
  MachineInstr &MI, MachineRegisterInfo &MRI) {
if (MI.getOpcode() != TargetOpcode::G_SEXT &&
    MI.getOpcode() != TargetOpcode::G_ZEXT &&
    MI.getOpcode() != TargetOpcode::G_ANYEXT)
  return false;
bool IsSigned = MI.getOpcode() == TargetOpcode::G_SEXT;
const Register DefReg = MI.getOperand(0).getReg();
const LLT DstTy = MRI.getType(DefReg);
unsigned DstSize = DstTy.getSizeInBits();

if (DstSize != 32 && DstSize != 64)
  return false;

MachineInstr *Extract = getOpcodeDef(TargetOpcode::G_EXTRACT_VECTOR_ELT,
                                     MI.getOperand(1).getReg(), MRI);
int64_t Lane;
if (!Extract || !mi_match(Extract->getOperand(2).getReg(), MRI, m_ICst(Lane)))
  return false;
Register Src0 = Extract->getOperand(1).getReg();

const LLT &VecTy = MRI.getType(Src0);

if (VecTy.getSizeInBits() != 128) {
  const MachineInstr *ScalarToVector = emitScalarToVector(
      VecTy.getSizeInBits(), &AArch64::FPR128RegClass, Src0, MIB);
  assert(ScalarToVector && "Didn't expect emitScalarToVector to fail!")(static_cast <bool> (ScalarToVector && "Didn't expect emitScalarToVector to fail!"
) ? void (0) : __assert_fail ("ScalarToVector && \"Didn't expect emitScalarToVector to fail!\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 5320, __extension__ __PRETTY_FUNCTION__));
  Src0 = ScalarToVector->getOperand(0).getReg();
}

unsigned Opcode;
if (DstSize == 64 && VecTy.getScalarSizeInBits() == 32)
  Opcode = IsSigned ? AArch64::SMOVvi32to64 : AArch64::UMOVvi32;
else if (DstSize == 64 && VecTy.getScalarSizeInBits() == 16)
  Opcode = IsSigned ? AArch64::SMOVvi16to64 : AArch64::UMOVvi16;
else if (DstSize == 64 && VecTy.getScalarSizeInBits() == 8)
  Opcode = IsSigned ? AArch64::SMOVvi8to64 : AArch64::UMOVvi8;
else if (DstSize == 32 && VecTy.getScalarSizeInBits() == 16)
  Opcode = IsSigned ? AArch64::SMOVvi16to32 : AArch64::UMOVvi16;
else if (DstSize == 32 && VecTy.getScalarSizeInBits() == 8)
  Opcode = IsSigned ? AArch64::SMOVvi8to32 : AArch64::UMOVvi8;
else
  llvm_unreachable("Unexpected type combo for S/UMov!")::llvm::llvm_unreachable_internal("Unexpected type combo for S/UMov!"
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 5336);

// We may need to generate one of these, depending on the type and sign of the
// input:
//  DstReg = SMOV Src0, Lane;
//  NewReg = UMOV Src0, Lane; DstReg = SUBREG_TO_REG NewReg, sub_32;
MachineInstr *ExtI = nullptr;
if (DstSize == 64 && !IsSigned) {
  Register NewReg = MRI.createVirtualRegister(&AArch64::GPR32RegClass);
  MIB.buildInstr(Opcode, {NewReg}, {Src0}).addImm(Lane);
  ExtI = MIB.buildInstr(AArch64::SUBREG_TO_REG, {DefReg}, {})
             .addImm(0)
             .addUse(NewReg)
             .addImm(AArch64::sub_32);
  RBI.constrainGenericRegister(DefReg, AArch64::GPR64RegClass, MRI);
} else
  ExtI = MIB.buildInstr(Opcode, {DefReg}, {Src0}).addImm(Lane);

constrainSelectedInstRegOperands(*ExtI, TII, TRI, RBI);
MI.eraseFromParent();
return true;
5357}

5359bool AArch64InstructionSelector::selectInsertElt(MachineInstr &I,
                                               MachineRegisterInfo &MRI) {
assert(I.getOpcode() == TargetOpcode::G_INSERT_VECTOR_ELT)(static_cast <bool> (I.getOpcode() == TargetOpcode::G_INSERT_VECTOR_ELT
) ? void (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_INSERT_VECTOR_ELT"
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 5361, __extension__ __PRETTY_FUNCTION__));

// Get information on the destination.
Register DstReg = I.getOperand(0).getReg();
const LLT DstTy = MRI.getType(DstReg);
unsigned VecSize = DstTy.getSizeInBits();

// Get information on the element we want to insert into the destination.
Register EltReg = I.getOperand(2).getReg();
const LLT EltTy = MRI.getType(EltReg);
unsigned EltSize = EltTy.getSizeInBits();
if (EltSize < 16 || EltSize > 64)
  return false; // Don't support all element types yet.

// Find the definition of the index. Bail out if it's not defined by a
// G_CONSTANT.
Register IdxReg = I.getOperand(3).getReg();
auto VRegAndVal = getIConstantVRegValWithLookThrough(IdxReg, MRI);
if (!VRegAndVal)
  return false;
unsigned LaneIdx = VRegAndVal->Value.getSExtValue();

// Perform the lane insert.
Register SrcReg = I.getOperand(1).getReg();
const RegisterBank &EltRB = *RBI.getRegBank(EltReg, MRI, TRI);

if (VecSize < 128) {
  // If the vector we're inserting into is smaller than 128 bits, widen it
  // to 128 to do the insert.
  MachineInstr *ScalarToVec =
      emitScalarToVector(VecSize, &AArch64::FPR128RegClass, SrcReg, MIB);
  if (!ScalarToVec)
    return false;
  SrcReg = ScalarToVec->getOperand(0).getReg();
}

// Create an insert into a new FPR128 register.
// Note that if our vector is already 128 bits, we end up emitting an extra
// register.
MachineInstr *InsMI =
    emitLaneInsert(std::nullopt, SrcReg, EltReg, LaneIdx, EltRB, MIB);

if (VecSize < 128) {
  // If we had to widen to perform the insert, then we have to demote back to
  // the original size to get the result we want.
  Register DemoteVec = InsMI->getOperand(0).getReg();
  const TargetRegisterClass *RC =
      getRegClassForTypeOnBank(DstTy, *RBI.getRegBank(DemoteVec, MRI, TRI));
  if (RC != &AArch64::FPR32RegClass && RC != &AArch64::FPR64RegClass) {
    LLVM_DEBUG(dbgs() << "Unsupported register class!\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unsupported register class!\n"
; } } while (false);
    return false;
  }
  unsigned SubReg = 0;
  if (!getSubRegForClass(RC, TRI, SubReg))
    return false;
  if (SubReg != AArch64::ssub && SubReg != AArch64::dsub) {
    LLVM_DEBUG(dbgs() << "Unsupported destination size! (" << VecSizedo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unsupported destination size! ("
 << VecSize << "\n"; } } while (false)
                      << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unsupported destination size! ("
 << VecSize << "\n"; } } while (false);
    return false;
  }
  MIB.buildInstr(TargetOpcode::COPY, {DstReg}, {})
      .addReg(DemoteVec, 0, SubReg);
  RBI.constrainGenericRegister(DstReg, *RC, MRI);
} else {
  // No widening needed.
  InsMI->getOperand(0).setReg(DstReg);
  constrainSelectedInstRegOperands(*InsMI, TII, TRI, RBI);
}

I.eraseFromParent();
return true;
5432}

5434MachineInstr *
5435AArch64InstructionSelector::emitConstantVector(Register Dst, Constant *CV,
                                             MachineIRBuilder &MIRBuilder,
                                             MachineRegisterInfo &MRI) {
LLT DstTy = MRI.getType(Dst);
unsigned DstSize = DstTy.getSizeInBits();
if (CV->isNullValue()) {
  if (DstSize == 128) {
    auto Mov =
        MIRBuilder.buildInstr(AArch64::MOVIv2d_ns, {Dst}, {}).addImm(0);
    constrainSelectedInstRegOperands(*Mov, TII, TRI, RBI);
    return &*Mov;
  }

  if (DstSize == 64) {
    auto Mov =
        MIRBuilder
            .buildInstr(AArch64::MOVIv2d_ns, {&AArch64::FPR128RegClass}, {})
            .addImm(0);
    auto Copy = MIRBuilder.buildInstr(TargetOpcode::COPY, {Dst}, {})
                    .addReg(Mov.getReg(0), 0, AArch64::dsub);
    RBI.constrainGenericRegister(Dst, AArch64::FPR64RegClass, MRI);
    return &*Copy;
  }
}

auto *CPLoad = emitLoadFromConstantPool(CV, MIRBuilder);
if (!CPLoad) {
  LLVM_DEBUG(dbgs() << "Could not generate cp load for constant vector!")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Could not generate cp load for constant vector!"
; } } while (false);
  return nullptr;
}

auto Copy = MIRBuilder.buildCopy(Dst, CPLoad->getOperand(0));
RBI.constrainGenericRegister(
    Dst, *MRI.getRegClass(CPLoad->getOperand(0).getReg()), MRI);
return &*Copy;
5470}

5472bool AArch64InstructionSelector::tryOptConstantBuildVec(
  MachineInstr &I, LLT DstTy, MachineRegisterInfo &MRI) {
assert(I.getOpcode() == TargetOpcode::G_BUILD_VECTOR)(static_cast <bool> (I.getOpcode() == TargetOpcode::G_BUILD_VECTOR
) ? void (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_BUILD_VECTOR"
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 5474, __extension__ __PRETTY_FUNCTION__));
unsigned DstSize = DstTy.getSizeInBits();
assert(DstSize <= 128 && "Unexpected build_vec type!")(static_cast <bool> (DstSize <= 128 && "Unexpected build_vec type!"
) ? void (0) : __assert_fail ("DstSize <= 128 && \"Unexpected build_vec type!\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 5476, __extension__ __PRETTY_FUNCTION__));
if (DstSize < 32)
  return false;
// Check if we're building a constant vector, in which case we want to
// generate a constant pool load instead of a vector insert sequence.
SmallVector<Constant *, 16> Csts;
for (unsigned Idx = 1; Idx < I.getNumOperands(); ++Idx) {
  // Try to find G_CONSTANT or G_FCONSTANT
  auto *OpMI =
      getOpcodeDef(TargetOpcode::G_CONSTANT, I.getOperand(Idx).getReg(), MRI);
  if (OpMI)
    Csts.emplace_back(
        const_cast<ConstantInt *>(OpMI->getOperand(1).getCImm()));
  else if ((OpMI = getOpcodeDef(TargetOpcode::G_FCONSTANT,
                                I.getOperand(Idx).getReg(), MRI)))
    Csts.emplace_back(
        const_cast<ConstantFP *>(OpMI->getOperand(1).getFPImm()));
  else
    return false;
}
Constant *CV = ConstantVector::get(Csts);
if (!emitConstantVector(I.getOperand(0).getReg(), CV, MIB, MRI))
  return false;
I.eraseFromParent();
return true;
5501}

5503bool AArch64InstructionSelector::tryOptBuildVecToSubregToReg(
  MachineInstr &I, MachineRegisterInfo &MRI) {
// Given:
//  %vec = G_BUILD_VECTOR %elt, %undef, %undef, ... %undef
//
// Select the G_BUILD_VECTOR as a SUBREG_TO_REG from %elt.
Register Dst = I.getOperand(0).getReg();
Register EltReg = I.getOperand(1).getReg();
LLT EltTy = MRI.getType(EltReg);
// If the index isn't on the same bank as its elements, then this can't be a
// SUBREG_TO_REG.
const RegisterBank &EltRB = *RBI.getRegBank(EltReg, MRI, TRI);
const RegisterBank &DstRB = *RBI.getRegBank(Dst, MRI, TRI);
if (EltRB != DstRB)
  return false;
if (any_of(make_range(I.operands_begin() + 2, I.operands_end()),
           [&MRI](const MachineOperand &Op) {
             return !getOpcodeDef(TargetOpcode::G_IMPLICIT_DEF, Op.getReg(),
                                  MRI);
           }))
  return false;
unsigned SubReg;
const TargetRegisterClass *EltRC = getRegClassForTypeOnBank(EltTy, EltRB);
if (!EltRC)
  return false;
const TargetRegisterClass *DstRC =
    getRegClassForTypeOnBank(MRI.getType(Dst), DstRB);
if (!DstRC)
  return false;
if (!getSubRegForClass(EltRC, TRI, SubReg))
  return false;
auto SubregToReg = MIB.buildInstr(AArch64::SUBREG_TO_REG, {Dst}, {})
                       .addImm(0)
                       .addUse(EltReg)
                       .addImm(SubReg);
I.eraseFromParent();
constrainSelectedInstRegOperands(*SubregToReg, TII, TRI, RBI);
return RBI.constrainGenericRegister(Dst, *DstRC, MRI);
5541}

5543bool AArch64InstructionSelector::selectBuildVector(MachineInstr &I,
                                                 MachineRegisterInfo &MRI) {
assert(I.getOpcode() == TargetOpcode::G_BUILD_VECTOR)(static_cast <bool> (I.getOpcode() == TargetOpcode::G_BUILD_VECTOR
) ? void (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_BUILD_VECTOR"
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 5545, __extension__ __PRETTY_FUNCTION__));
// Until we port more of the optimized selections, for now just use a vector
// insert sequence.
const LLT DstTy = MRI.getType(I.getOperand(0).getReg());
const LLT EltTy = MRI.getType(I.getOperand(1).getReg());
unsigned EltSize = EltTy.getSizeInBits();

if (tryOptConstantBuildVec(I, DstTy, MRI))
  return true;
if (tryOptBuildVecToSubregToReg(I, MRI))
  return true;

if (EltSize != 8 && EltSize != 16 && EltSize != 32 && EltSize != 64)
  return false; // Don't support all element types yet.
const RegisterBank &RB = *RBI.getRegBank(I.getOperand(1).getReg(), MRI, TRI);

const TargetRegisterClass *DstRC = &AArch64::FPR128RegClass;
MachineInstr *ScalarToVec =
    emitScalarToVector(DstTy.getElementType().getSizeInBits(), DstRC,
                       I.getOperand(1).getReg(), MIB);
if (!ScalarToVec)
  return false;

Register DstVec = ScalarToVec->getOperand(0).getReg();
unsigned DstSize = DstTy.getSizeInBits();

// Keep track of the last MI we inserted. Later on, we might be able to save
// a copy using it.
MachineInstr *PrevMI = nullptr;
for (unsigned i = 2, e = DstSize / EltSize + 1; i < e; ++i) {
  // Note that if we don't do a subregister copy, we can end up making an
  // extra register.
  PrevMI = &*emitLaneInsert(std::nullopt, DstVec, I.getOperand(i).getReg(),
                            i - 1, RB, MIB);
  DstVec = PrevMI->getOperand(0).getReg();
}

// If DstTy's size in bits is less than 128, then emit a subregister copy
// from DstVec to the last register we've defined.
if (DstSize < 128) {
  // Force this to be FPR using the destination vector.
  const TargetRegisterClass *RC =
      getRegClassForTypeOnBank(DstTy, *RBI.getRegBank(DstVec, MRI, TRI));
  if (!RC)
    return false;
  if (RC != &AArch64::FPR32RegClass && RC != &AArch64::FPR64RegClass) {
    LLVM_DEBUG(dbgs() << "Unsupported register class!\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unsupported register class!\n"
; } } while (false);
    return false;
  }

  unsigned SubReg = 0;
  if (!getSubRegForClass(RC, TRI, SubReg))
    return false;
  if (SubReg != AArch64::ssub && SubReg != AArch64::dsub) {
    LLVM_DEBUG(dbgs() << "Unsupported destination size! (" << DstSizedo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unsupported destination size! ("
 << DstSize << "\n"; } } while (false)
                      << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "Unsupported destination size! ("
 << DstSize << "\n"; } } while (false);
    return false;
  }

  Register Reg = MRI.createVirtualRegister(RC);
  Register DstReg = I.getOperand(0).getReg();

  MIB.buildInstr(TargetOpcode::COPY, {DstReg}, {}).addReg(DstVec, 0, SubReg);
  MachineOperand &RegOp = I.getOperand(1);
  RegOp.setReg(Reg);
  RBI.constrainGenericRegister(DstReg, *RC, MRI);
} else {
  // We don't need a subregister copy. Save a copy by re-using the
  // destination register on the final insert.
  assert(PrevMI && "PrevMI was null?")(static_cast <bool> (PrevMI && "PrevMI was null?"
) ? void (0) : __assert_fail ("PrevMI && \"PrevMI was null?\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 5614, __extension__ __PRETTY_FUNCTION__));
  PrevMI->getOperand(0).setReg(I.getOperand(0).getReg());
  constrainSelectedInstRegOperands(*PrevMI, TII, TRI, RBI);
}

I.eraseFromParent();
return true;
5621}

5623bool AArch64InstructionSelector::selectVectorLoadIntrinsic(unsigned Opc,
                                                         unsigned NumVecs,
                                                         MachineInstr &I) {
assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS)(static_cast <bool> (I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS
) ? void (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS"
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 5626, __extension__ __PRETTY_FUNCTION__));
assert(Opc && "Expected an opcode?")(static_cast <bool> (Opc && "Expected an opcode?"
) ? void (0) : __assert_fail ("Opc && \"Expected an opcode?\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 5627, __extension__ __PRETTY_FUNCTION__));
assert(NumVecs > 1 && NumVecs < 5 && "Only support 2, 3, or 4 vectors")(static_cast <bool> (NumVecs > 1 && NumVecs <
&& "Only support 2, 3, or 4 vectors") ? void (0) :
 __assert_fail ("NumVecs > 1 && NumVecs < 5 && \"Only support 2, 3, or 4 vectors\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 5628, __extension__ __PRETTY_FUNCTION__));
auto &MRI = *MIB.getMRI();
LLT Ty = MRI.getType(I.getOperand(0).getReg());
unsigned Size = Ty.getSizeInBits();
assert((Size == 64 || Size == 128) &&(static_cast <bool> ((Size == 64 || Size == 128) &&
 "Destination must be 64 bits or 128 bits?") ? void (0) : __assert_fail
 ("(Size == 64 || Size == 128) && \"Destination must be 64 bits or 128 bits?\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 5633, __extension__ __PRETTY_FUNCTION__))
       "Destination must be 64 bits or 128 bits?")(static_cast <bool> ((Size == 64 || Size == 128) &&
 "Destination must be 64 bits or 128 bits?") ? void (0) : __assert_fail
 ("(Size == 64 || Size == 128) && \"Destination must be 64 bits or 128 bits?\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 5633, __extension__ __PRETTY_FUNCTION__));
unsigned SubReg = Size == 64 ? AArch64::dsub0 : AArch64::qsub0;
auto Ptr = I.getOperand(I.getNumOperands() - 1).getReg();
assert(MRI.getType(Ptr).isPointer() && "Expected a pointer type?")(static_cast <bool> (MRI.getType(Ptr).isPointer() &&
 "Expected a pointer type?") ? void (0) : __assert_fail ("MRI.getType(Ptr).isPointer() && \"Expected a pointer type?\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 5636, __extension__ __PRETTY_FUNCTION__));
auto Load = MIB.buildInstr(Opc, {Ty}, {Ptr});
Load.cloneMemRefs(I);
constrainSelectedInstRegOperands(*Load, TII, TRI, RBI);
Register SelectedLoadDst = Load->getOperand(0).getReg();
for (unsigned Idx = 0; Idx < NumVecs; ++Idx) {
  auto Vec = MIB.buildInstr(TargetOpcode::COPY, {I.getOperand(Idx)}, {})
                 .addReg(SelectedLoadDst, 0, SubReg + Idx);
  // Emit the subreg copies and immediately select them.
  // FIXME: We should refactor our copy code into an emitCopy helper and
  // clean up uses of this pattern elsewhere in the selector.
  selectCopy(*Vec, TII, MRI, TRI, RBI);
}
return true;
5650}

5652bool AArch64InstructionSelector::selectIntrinsicWithSideEffects(
  MachineInstr &I, MachineRegisterInfo &MRI) {
// Find the intrinsic ID.
unsigned IntrinID = I.getIntrinsicID();

const LLT S8 = LLT::scalar(8);
const LLT S16 = LLT::scalar(16);
const LLT S32 = LLT::scalar(32);
const LLT S64 = LLT::scalar(64);
const LLT P0 = LLT::pointer(0, 64);
// Select the instruction.
switch (IntrinID) {
default:
  return false;
case Intrinsic::aarch64_ldxp:
case Intrinsic::aarch64_ldaxp: {
  auto NewI = MIB.buildInstr(
      IntrinID == Intrinsic::aarch64_ldxp ? AArch64::LDXPX : AArch64::LDAXPX,
      {I.getOperand(0).getReg(), I.getOperand(1).getReg()},
      {I.getOperand(3)});
  NewI.cloneMemRefs(I);
  constrainSelectedInstRegOperands(*NewI, TII, TRI, RBI);
  break;
}
case Intrinsic::trap:
  MIB.buildInstr(AArch64::BRK, {}, {}).addImm(1);
  break;
case Intrinsic::debugtrap:
  MIB.buildInstr(AArch64::BRK, {}, {}).addImm(0xF000);
  break;
case Intrinsic::ubsantrap:
  MIB.buildInstr(AArch64::BRK, {}, {})
      .addImm(I.getOperand(1).getImm() | ('U' << 8));
  break;
case Intrinsic::aarch64_neon_ld2: {
  LLT Ty = MRI.getType(I.getOperand(0).getReg());
  unsigned Opc = 0;
  if (Ty == LLT::fixed_vector(8, S8))
    Opc = AArch64::LD2Twov8b;
  else if (Ty == LLT::fixed_vector(16, S8))
    Opc = AArch64::LD2Twov16b;
  else if (Ty == LLT::fixed_vector(4, S16))
    Opc = AArch64::LD2Twov4h;
  else if (Ty == LLT::fixed_vector(8, S16))
    Opc = AArch64::LD2Twov8h;
  else if (Ty == LLT::fixed_vector(2, S32))
    Opc = AArch64::LD2Twov2s;
  else if (Ty == LLT::fixed_vector(4, S32))
    Opc = AArch64::LD2Twov4s;
  else if (Ty == LLT::fixed_vector(2, S64) || Ty == LLT::fixed_vector(2, P0))
    Opc = AArch64::LD2Twov2d;
  else if (Ty == S64 || Ty == P0)
    Opc = AArch64::LD1Twov1d;
  else
    llvm_unreachable("Unexpected type for ld2!")::llvm::llvm_unreachable_internal("Unexpected type for ld2!",
 "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 5706);
  selectVectorLoadIntrinsic(Opc, 2, I);
  break;
}
case Intrinsic::aarch64_neon_ld4: {
  LLT Ty = MRI.getType(I.getOperand(0).getReg());
  unsigned Opc = 0;
  if (Ty == LLT::fixed_vector(8, S8))
    Opc = AArch64::LD4Fourv8b;
  else if (Ty == LLT::fixed_vector(16, S8))
    Opc = AArch64::LD4Fourv16b;
  else if (Ty == LLT::fixed_vector(4, S16))
    Opc = AArch64::LD4Fourv4h;
  else if (Ty == LLT::fixed_vector(8, S16))
    Opc = AArch64::LD4Fourv8h;
  else if (Ty == LLT::fixed_vector(2, S32))
    Opc = AArch64::LD4Fourv2s;
  else if (Ty == LLT::fixed_vector(4, S32))
    Opc = AArch64::LD4Fourv4s;
  else if (Ty == LLT::fixed_vector(2, S64) || Ty == LLT::fixed_vector(2, P0))
    Opc = AArch64::LD4Fourv2d;
  else if (Ty == S64 || Ty == P0)
    Opc = AArch64::LD1Fourv1d;
  else
    llvm_unreachable("Unexpected type for ld4!")::llvm::llvm_unreachable_internal("Unexpected type for ld4!",
 "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 5730);
  selectVectorLoadIntrinsic(Opc, 4, I);
  break;
}
case Intrinsic::aarch64_neon_st2: {
  Register Src1 = I.getOperand(1).getReg();
  Register Src2 = I.getOperand(2).getReg();
  Register Ptr = I.getOperand(3).getReg();
  LLT Ty = MRI.getType(Src1);
  unsigned Opc;
  if (Ty == LLT::fixed_vector(8, S8))
    Opc = AArch64::ST2Twov8b;
  else if (Ty == LLT::fixed_vector(16, S8))
    Opc = AArch64::ST2Twov16b;
  else if (Ty == LLT::fixed_vector(4, S16))
    Opc = AArch64::ST2Twov4h;
  else if (Ty == LLT::fixed_vector(8, S16))
    Opc = AArch64::ST2Twov8h;
  else if (Ty == LLT::fixed_vector(2, S32))
    Opc = AArch64::ST2Twov2s;
  else if (Ty == LLT::fixed_vector(4, S32))
    Opc = AArch64::ST2Twov4s;
  else if (Ty == LLT::fixed_vector(2, S64) || Ty == LLT::fixed_vector(2, P0))
    Opc = AArch64::ST2Twov2d;
  else if (Ty == S64 || Ty == P0)
    Opc = AArch64::ST1Twov1d;
  else
    llvm_unreachable("Unexpected type for st2!")::llvm::llvm_unreachable_internal("Unexpected type for st2!",
 "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 5757);
  SmallVector<Register, 2> Regs = {Src1, Src2};
  Register Tuple = Ty.getSizeInBits() == 128 ? createQTuple(Regs, MIB)
                                             : createDTuple(Regs, MIB);
  auto Store = MIB.buildInstr(Opc, {}, {Tuple, Ptr});
  Store.cloneMemRefs(I);
  constrainSelectedInstRegOperands(*Store, TII, TRI, RBI);
  break;
}
case Intrinsic::aarch64_mops_memset_tag: {
  // Transform
  //    %dst:gpr(p0) = \
  //      G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.aarch64.mops.memset.tag),
  //      \ %dst:gpr(p0), %val:gpr(s64), %n:gpr(s64)
  // where %dst is updated, into
  //    %Rd:GPR64common, %Rn:GPR64) = \
  //      MOPSMemorySetTaggingPseudo \
  //      %Rd:GPR64common, %Rn:GPR64, %Rm:GPR64
  // where Rd and Rn are tied.
  // It is expected that %val has been extended to s64 in legalization.
  // Note that the order of the size/value operands are swapped.

  Register DstDef = I.getOperand(0).getReg();
  // I.getOperand(1) is the intrinsic function
  Register DstUse = I.getOperand(2).getReg();
  Register ValUse = I.getOperand(3).getReg();
  Register SizeUse = I.getOperand(4).getReg();

  // MOPSMemorySetTaggingPseudo has two defs; the intrinsic call has only one.
  // Therefore an additional virtual register is requried for the updated size
  // operand. This value is not accessible via the semantics of the intrinsic.
  Register SizeDef = MRI.createGenericVirtualRegister(LLT::scalar(64));

  auto Memset = MIB.buildInstr(AArch64::MOPSMemorySetTaggingPseudo,
                               {DstDef, SizeDef}, {DstUse, SizeUse, ValUse});
  Memset.cloneMemRefs(I);
  constrainSelectedInstRegOperands(*Memset, TII, TRI, RBI);
  break;
}
}

I.eraseFromParent();
return true;
5800}

5802bool AArch64InstructionSelector::selectIntrinsic(MachineInstr &I,
                                               MachineRegisterInfo &MRI) {
unsigned IntrinID = I.getIntrinsicID();

switch (IntrinID) {
default:
  break;
case Intrinsic::aarch64_crypto_sha1h: {
  Register DstReg = I.getOperand(0).getReg();
  Register SrcReg = I.getOperand(2).getReg();

  // FIXME: Should this be an assert?
  if (MRI.getType(DstReg).getSizeInBits() != 32 ||
      MRI.getType(SrcReg).getSizeInBits() != 32)
    return false;

  // The operation has to happen on FPRs. Set up some new FPR registers for
  // the source and destination if they are on GPRs.
  if (RBI.getRegBank(SrcReg, MRI, TRI)->getID() != AArch64::FPRRegBankID) {
    SrcReg = MRI.createVirtualRegister(&AArch64::FPR32RegClass);
    MIB.buildCopy({SrcReg}, {I.getOperand(2)});

    // Make sure the copy ends up getting constrained properly.
    RBI.constrainGenericRegister(I.getOperand(2).getReg(),
                                 AArch64::GPR32RegClass, MRI);
  }

  if (RBI.getRegBank(DstReg, MRI, TRI)->getID() != AArch64::FPRRegBankID)
    DstReg = MRI.createVirtualRegister(&AArch64::FPR32RegClass);

  // Actually insert the instruction.
  auto SHA1Inst = MIB.buildInstr(AArch64::SHA1Hrr, {DstReg}, {SrcReg});
  constrainSelectedInstRegOperands(*SHA1Inst, TII, TRI, RBI);

  // Did we create a new register for the destination?
  if (DstReg != I.getOperand(0).getReg()) {
    // Yep. Copy the result of the instruction back into the original
    // destination.
    MIB.buildCopy({I.getOperand(0)}, {DstReg});
    RBI.constrainGenericRegister(I.getOperand(0).getReg(),
                                 AArch64::GPR32RegClass, MRI);
  }

  I.eraseFromParent();
  return true;
}
case Intrinsic::ptrauth_sign: {
  Register DstReg = I.getOperand(0).getReg();
  Register ValReg = I.getOperand(2).getReg();
  uint64_t Key = I.getOperand(3).getImm();
  Register DiscReg = I.getOperand(4).getReg();
  auto DiscVal = getIConstantVRegVal(DiscReg, MRI);
  bool IsDiscZero = DiscVal && DiscVal->isZero();

  if (Key > AArch64PACKey::LAST)
    return false;

  unsigned Opcodes[][4] = {
      {AArch64::PACIA, AArch64::PACIB, AArch64::PACDA, AArch64::PACDB},
      {AArch64::PACIZA, AArch64::PACIZB, AArch64::PACDZA, AArch64::PACDZB}};
  unsigned Opcode = Opcodes[IsDiscZero][Key];

  auto PAC = MIB.buildInstr(Opcode, {DstReg}, {ValReg});

  if (!IsDiscZero) {
    PAC.addUse(DiscReg);
    RBI.constrainGenericRegister(DiscReg, AArch64::GPR64spRegClass, MRI);
  }

  RBI.constrainGenericRegister(DstReg, AArch64::GPR64RegClass, MRI);
  I.eraseFromParent();
  return true;
}
case Intrinsic::ptrauth_strip: {
  Register DstReg = I.getOperand(0).getReg();
  Register ValReg = I.getOperand(2).getReg();
  uint64_t Key = I.getOperand(3).getImm();

  if (Key > AArch64PACKey::LAST)
    return false;
  unsigned Opcode = getXPACOpcodeForKey((AArch64PACKey::ID)Key);

  MIB.buildInstr(Opcode, {DstReg}, {ValReg});

  RBI.constrainGenericRegister(DstReg, AArch64::GPR64RegClass, MRI);
  RBI.constrainGenericRegister(ValReg, AArch64::GPR64RegClass, MRI);
  I.eraseFromParent();
  return true;
}
case Intrinsic::frameaddress:
case Intrinsic::returnaddress: {
  MachineFunction &MF = *I.getParent()->getParent();
  MachineFrameInfo &MFI = MF.getFrameInfo();

  unsigned Depth = I.getOperand(2).getImm();
  Register DstReg = I.getOperand(0).getReg();
  RBI.constrainGenericRegister(DstReg, AArch64::GPR64RegClass, MRI);

  if (Depth == 0 && IntrinID == Intrinsic::returnaddress) {
    if (!MFReturnAddr) {
      // Insert the copy from LR/X30 into the entry block, before it can be
      // clobbered by anything.
      MFI.setReturnAddressIsTaken(true);
      MFReturnAddr = getFunctionLiveInPhysReg(
          MF, TII, AArch64::LR, AArch64::GPR64RegClass, I.getDebugLoc());
    }

    if (STI.hasPAuth()) {
      MIB.buildInstr(AArch64::XPACI, {DstReg}, {MFReturnAddr});
    } else {
      MIB.buildCopy({Register(AArch64::LR)}, {MFReturnAddr});
      MIB.buildInstr(AArch64::XPACLRI);
      MIB.buildCopy({DstReg}, {Register(AArch64::LR)});
    }

    I.eraseFromParent();
    return true;
  }

  MFI.setFrameAddressIsTaken(true);
  Register FrameAddr(AArch64::FP);
  while (Depth--) {
    Register NextFrame = MRI.createVirtualRegister(&AArch64::GPR64spRegClass);
    auto Ldr =
        MIB.buildInstr(AArch64::LDRXui, {NextFrame}, {FrameAddr}).addImm(0);
    constrainSelectedInstRegOperands(*Ldr, TII, TRI, RBI);
    FrameAddr = NextFrame;
  }

  if (IntrinID == Intrinsic::frameaddress)
    MIB.buildCopy({DstReg}, {FrameAddr});
  else {
    MFI.setReturnAddressIsTaken(true);

    if (STI.hasPAuth()) {
      Register TmpReg = MRI.createVirtualRegister(&AArch64::GPR64RegClass);
      MIB.buildInstr(AArch64::LDRXui, {TmpReg}, {FrameAddr}).addImm(1);
      MIB.buildInstr(AArch64::XPACI, {DstReg}, {TmpReg});
    } else {
      MIB.buildInstr(AArch64::LDRXui, {Register(AArch64::LR)}, {FrameAddr})
          .addImm(1);
      MIB.buildInstr(AArch64::XPACLRI);
      MIB.buildCopy({DstReg}, {Register(AArch64::LR)});
    }
  }

  I.eraseFromParent();
  return true;
}
case Intrinsic::swift_async_context_addr:
  auto Sub = MIB.buildInstr(AArch64::SUBXri, {I.getOperand(0).getReg()},
                            {Register(AArch64::FP)})
                 .addImm(8)
                 .addImm(0);
  constrainSelectedInstRegOperands(*Sub, TII, TRI, RBI);

  MF->getFrameInfo().setFrameAddressIsTaken(true);
  MF->getInfo<AArch64FunctionInfo>()->setHasSwiftAsyncContext(true);
  I.eraseFromParent();
  return true;
}
return false;
5964}

5966InstructionSelector::ComplexRendererFns
5967AArch64InstructionSelector::selectShiftA_32(const MachineOperand &Root) const {
auto MaybeImmed = getImmedFromMO(Root);
if (MaybeImmed == std::nullopt || *MaybeImmed > 31)
  return std::nullopt;
uint64_t Enc = (32 - *MaybeImmed) & 0x1f;
return {{[=](MachineInstrBuilder &MIB) { MIB.addImm(Enc); }}};
5973}

5975InstructionSelector::ComplexRendererFns
5976AArch64InstructionSelector::selectShiftB_32(const MachineOperand &Root) const {
auto MaybeImmed = getImmedFromMO(Root);
if (MaybeImmed == std::nullopt || *MaybeImmed > 31)
  return std::nullopt;
uint64_t Enc = 31 - *MaybeImmed;
return {{[=](MachineInstrBuilder &MIB) { MIB.addImm(Enc); }}};
5982}

5984InstructionSelector::ComplexRendererFns
5985AArch64InstructionSelector::selectShiftA_64(const MachineOperand &Root) const {
auto MaybeImmed = getImmedFromMO(Root);
if (MaybeImmed == std::nullopt || *MaybeImmed > 63)
  return std::nullopt;
uint64_t Enc = (64 - *MaybeImmed) & 0x3f;
return {{[=](MachineInstrBuilder &MIB) { MIB.addImm(Enc); }}};
5991}

5993InstructionSelector::ComplexRendererFns
5994AArch64InstructionSelector::selectShiftB_64(const MachineOperand &Root) const {
auto MaybeImmed = getImmedFromMO(Root);
if (MaybeImmed == std::nullopt || *MaybeImmed > 63)
  return std::nullopt;
uint64_t Enc = 63 - *MaybeImmed;
return {{[=](MachineInstrBuilder &MIB) { MIB.addImm(Enc); }}};
6000}

6002/// Helper to select an immediate value that can be represented as a 12-bit
6003/// value shifted left by either 0 or 12. If it is possible to do so, return
6004/// the immediate and shift value. If not, return std::nullopt.
6005///
6006/// Used by selectArithImmed and selectNegArithImmed.
6007InstructionSelector::ComplexRendererFns
6008AArch64InstructionSelector::select12BitValueWithLeftShift(
  uint64_t Immed) const {
unsigned ShiftAmt;
if (Immed >> 12 == 0) {
  ShiftAmt = 0;
} else if ((Immed & 0xfff) == 0 && Immed >> 24 == 0) {
  ShiftAmt = 12;
  Immed = Immed >> 12;
} else
  return std::nullopt;

unsigned ShVal = AArch64_AM::getShifterImm(AArch64_AM::LSL, ShiftAmt);
return {{
    [=](MachineInstrBuilder &MIB) { MIB.addImm(Immed); },
    [=](MachineInstrBuilder &MIB) { MIB.addImm(ShVal); },
}};
6024}

6026/// SelectArithImmed - Select an immediate value that can be represented as
6027/// a 12-bit value shifted left by either 0 or 12.  If so, return true with
6028/// Val set to the 12-bit value and Shift set to the shifter operand.
6029InstructionSelector::ComplexRendererFns
6030AArch64InstructionSelector::selectArithImmed(MachineOperand &Root) const {
// This function is called from the addsub_shifted_imm ComplexPattern,
// which lists [imm] as the list of opcode it's interested in, however
// we still need to check whether the operand is actually an immediate
// here because the ComplexPattern opcode list is only used in
// root-level opcode matching.
auto MaybeImmed = getImmedFromMO(Root);
if (MaybeImmed == std::nullopt)
  return std::nullopt;
return select12BitValueWithLeftShift(*MaybeImmed);
6040}

6042/// SelectNegArithImmed - As above, but negates the value before trying to
6043/// select it.
6044InstructionSelector::ComplexRendererFns
6045AArch64InstructionSelector::selectNegArithImmed(MachineOperand &Root) const {
// We need a register here, because we need to know if we have a 64 or 32
// bit immediate.
if (!Root.isReg())
  return std::nullopt;
auto MaybeImmed = getImmedFromMO(Root);
if (MaybeImmed == std::nullopt)
  return std::nullopt;
uint64_t Immed = *MaybeImmed;

// This negation is almost always valid, but "cmp wN, #0" and "cmn wN, #0"
// have the opposite effect on the C flag, so this pattern mustn't match under
// those circumstances.
if (Immed == 0)
  return std::nullopt;

// Check if we're dealing with a 32-bit type on the root or a 64-bit type on
// the root.
MachineRegisterInfo &MRI = Root.getParent()->getMF()->getRegInfo();
if (MRI.getType(Root.getReg()).getSizeInBits() == 32)
  Immed = ~((uint32_t)Immed) + 1;
else
  Immed = ~Immed + 1ULL;

if (Immed & 0xFFFFFFFFFF000000ULL)
  return std::nullopt;

Immed &= 0xFFFFFFULL;
return select12BitValueWithLeftShift(Immed);
6074}

6076/// Return true if it is worth folding MI into an extended register. That is,
6077/// if it's safe to pull it into the addressing mode of a load or store as a
6078/// shift.
6079bool AArch64InstructionSelector::isWorthFoldingIntoExtendedReg(
  MachineInstr &MI, const MachineRegisterInfo &MRI) const {
// Always fold if there is one use, or if we're optimizing for size.
Register DefReg = MI.getOperand(0).getReg();
if (MRI.hasOneNonDBGUse(DefReg) ||
    MI.getParent()->getParent()->getFunction().hasOptSize())
  return true;

// It's better to avoid folding and recomputing shifts when we don't have a
// fastpath.
if (!STI.hasLSLFast())
  return false;

// We have a fastpath, so folding a shift in and potentially computing it
// many times may be beneficial. Check if this is only used in memory ops.
// If it is, then we should fold.
return all_of(MRI.use_nodbg_instructions(DefReg),
              [](MachineInstr &Use) { return Use.mayLoadOrStore(); });
6097}

6099static bool isSignExtendShiftType(AArch64_AM::ShiftExtendType Type) {
switch (Type) {
case AArch64_AM::SXTB:
case AArch64_AM::SXTH:
case AArch64_AM::SXTW:
  return true;
default:
  return false;
}
6108}

6110InstructionSelector::ComplexRendererFns
6111AArch64InstructionSelector::selectExtendedSHL(
  MachineOperand &Root, MachineOperand &Base, MachineOperand &Offset,
  unsigned SizeInBytes, bool WantsExt) const {
assert(Base.isReg() && "Expected base to be a register operand")(static_cast <bool> (Base.isReg() && "Expected base to be a register operand"
) ? void (0) : __assert_fail ("Base.isReg() && \"Expected base to be a register operand\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 6114, __extension__ __PRETTY_FUNCTION__));
assert(Offset.isReg() && "Expected offset to be a register operand")(static_cast <bool> (Offset.isReg() && "Expected offset to be a register operand"
) ? void (0) : __assert_fail ("Offset.isReg() && \"Expected offset to be a register operand\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 6115, __extension__ __PRETTY_FUNCTION__));

MachineRegisterInfo &MRI = Root.getParent()->getMF()->getRegInfo();
MachineInstr *OffsetInst = MRI.getVRegDef(Offset.getReg());

unsigned OffsetOpc = OffsetInst->getOpcode();
bool LookedThroughZExt = false;
if (OffsetOpc != TargetOpcode::G_SHL && OffsetOpc != TargetOpcode::G_MUL) {
  // Try to look through a ZEXT.
  if (OffsetOpc != TargetOpcode::G_ZEXT || !WantsExt)
    return std::nullopt;

  OffsetInst = MRI.getVRegDef(OffsetInst->getOperand(1).getReg());
  OffsetOpc = OffsetInst->getOpcode();
  LookedThroughZExt = true;

  if (OffsetOpc != TargetOpcode::G_SHL && OffsetOpc != TargetOpcode::G_MUL)
    return std::nullopt;
}
// Make sure that the memory op is a valid size.
int64_t LegalShiftVal = Log2_32(SizeInBytes);
if (LegalShiftVal == 0)
  return std::nullopt;
if (!isWorthFoldingIntoExtendedReg(*OffsetInst, MRI))
  return std::nullopt;

// Now, try to find the specific G_CONSTANT. Start by assuming that the
// register we will offset is the LHS, and the register containing the
// constant is the RHS.
Register OffsetReg = OffsetInst->getOperand(1).getReg();
Register ConstantReg = OffsetInst->getOperand(2).getReg();
auto ValAndVReg = getIConstantVRegValWithLookThrough(ConstantReg, MRI);
if (!ValAndVReg) {
  // We didn't get a constant on the RHS. If the opcode is a shift, then
  // we're done.
  if (OffsetOpc == TargetOpcode::G_SHL)
    return std::nullopt;

  // If we have a G_MUL, we can use either register. Try looking at the RHS.
  std::swap(OffsetReg, ConstantReg);
  ValAndVReg = getIConstantVRegValWithLookThrough(ConstantReg, MRI);
  if (!ValAndVReg)
    return std::nullopt;
}

// The value must fit into 3 bits, and must be positive. Make sure that is
// true.
int64_t ImmVal = ValAndVReg->Value.getSExtValue();

// Since we're going to pull this into a shift, the constant value must be
// a power of 2. If we got a multiply, then we need to check this.
if (OffsetOpc == TargetOpcode::G_MUL) {
  if (!llvm::has_single_bit<uint32_t>(ImmVal))
    return std::nullopt;

  // Got a power of 2. So, the amount we'll shift is the log base-2 of that.
  ImmVal = Log2_32(ImmVal);
}

if ((ImmVal & 0x7) != ImmVal)
  return std::nullopt;

// We are only allowed to shift by LegalShiftVal. This shift value is built
// into the instruction, so we can't just use whatever we want.
if (ImmVal != LegalShiftVal)
  return std::nullopt;

unsigned SignExtend = 0;
if (WantsExt) {
  // Check if the offset is defined by an extend, unless we looked through a
  // G_ZEXT earlier.
  if (!LookedThroughZExt) {
    MachineInstr *ExtInst = getDefIgnoringCopies(OffsetReg, MRI);
    auto Ext = getExtendTypeForInst(*ExtInst, MRI, true);
    if (Ext == AArch64_AM::InvalidShiftExtend)
      return std::nullopt;

    SignExtend = isSignExtendShiftType(Ext) ? 1 : 0;
    // We only support SXTW for signed extension here.
    if (SignExtend && Ext != AArch64_AM::SXTW)
      return std::nullopt;
    OffsetReg = ExtInst->getOperand(1).getReg();
  }

  // Need a 32-bit wide register here.
  MachineIRBuilder MIB(*MRI.getVRegDef(Root.getReg()));
  OffsetReg = moveScalarRegClass(OffsetReg, AArch64::GPR32RegClass, MIB);
}

// We can use the LHS of the GEP as the base, and the LHS of the shift as an
// offset. Signify that we are shifting by setting the shift flag to 1.
return {{[=](MachineInstrBuilder &MIB) { MIB.addUse(Base.getReg()); },
         [=](MachineInstrBuilder &MIB) { MIB.addUse(OffsetReg); },
         [=](MachineInstrBuilder &MIB) {
           // Need to add both immediates here to make sure that they are both
           // added to the instruction.
           MIB.addImm(SignExtend);
           MIB.addImm(1);
         }}};
6214}

6216/// This is used for computing addresses like this:
6217///
6218/// ldr x1, [x2, x3, lsl #3]
6219///
6220/// Where x2 is the base register, and x3 is an offset register. The shift-left
6221/// is a constant value specific to this load instruction. That is, we'll never
6222/// see anything other than a 3 here (which corresponds to the size of the
6223/// element being loaded.)
6224InstructionSelector::ComplexRendererFns
6225AArch64InstructionSelector::selectAddrModeShiftedExtendXReg(
  MachineOperand &Root, unsigned SizeInBytes) const {
if (!Root.isReg())
  return std::nullopt;
MachineRegisterInfo &MRI = Root.getParent()->getMF()->getRegInfo();

// We want to find something like this:
//
// val = G_CONSTANT LegalShiftVal
// shift = G_SHL off_reg val
// ptr = G_PTR_ADD base_reg shift
// x = G_LOAD ptr
//
// And fold it into this addressing mode:
//
// ldr x, [base_reg, off_reg, lsl #LegalShiftVal]

// Check if we can find the G_PTR_ADD.
MachineInstr *PtrAdd =
    getOpcodeDef(TargetOpcode::G_PTR_ADD, Root.getReg(), MRI);
if (!PtrAdd || !isWorthFoldingIntoExtendedReg(*PtrAdd, MRI))
  return std::nullopt;

// Now, try to match an opcode which will match our specific offset.
// We want a G_SHL or a G_MUL.
MachineInstr *OffsetInst =
    getDefIgnoringCopies(PtrAdd->getOperand(2).getReg(), MRI);
return selectExtendedSHL(Root, PtrAdd->getOperand(1),
                         OffsetInst->getOperand(0), SizeInBytes,
                         /*WantsExt=*/false);
6255}

6257/// This is used for computing addresses like this:
6258///
6259/// ldr x1, [x2, x3]
6260///
6261/// Where x2 is the base register, and x3 is an offset register.
6262///
6263/// When possible (or profitable) to fold a G_PTR_ADD into the address
6264/// calculation, this will do so. Otherwise, it will return std::nullopt.
6265InstructionSelector::ComplexRendererFns
6266AArch64InstructionSelector::selectAddrModeRegisterOffset(
  MachineOperand &Root) const {
MachineRegisterInfo &MRI = Root.getParent()->getMF()->getRegInfo();

// We need a GEP.
MachineInstr *Gep = MRI.getVRegDef(Root.getReg());
if (Gep->getOpcode() != TargetOpcode::G_PTR_ADD)
  return std::nullopt;

// If this is used more than once, let's not bother folding.
// TODO: Check if they are memory ops. If they are, then we can still fold
// without having to recompute anything.
if (!MRI.hasOneNonDBGUse(Gep->getOperand(0).getReg()))
  return std::nullopt;

// Base is the GEP's LHS, offset is its RHS.
return {{[=](MachineInstrBuilder &MIB) {
           MIB.addUse(Gep->getOperand(1).getReg());
         },
         [=](MachineInstrBuilder &MIB) {
           MIB.addUse(Gep->getOperand(2).getReg());
         },
         [=](MachineInstrBuilder &MIB) {
           // Need to add both immediates here to make sure that they are both
           // added to the instruction.
           MIB.addImm(0);
           MIB.addImm(0);
         }}};
6294}

6296/// This is intended to be equivalent to selectAddrModeXRO in
6297/// AArch64ISelDAGtoDAG. It's used for selecting X register offset loads.
6298InstructionSelector::ComplexRendererFns
6299AArch64InstructionSelector::selectAddrModeXRO(MachineOperand &Root,
                                            unsigned SizeInBytes) const {
MachineRegisterInfo &MRI = Root.getParent()->getMF()->getRegInfo();
if (!Root.isReg())
  return std::nullopt;
MachineInstr *PtrAdd =
    getOpcodeDef(TargetOpcode::G_PTR_ADD, Root.getReg(), MRI);
if (!PtrAdd)
  return std::nullopt;

// Check for an immediates which cannot be encoded in the [base + imm]
// addressing mode, and can't be encoded in an add/sub. If this happens, we'll
// end up with code like:
//
// mov x0, wide
// add x1 base, x0
// ldr x2, [x1, x0]
//
// In this situation, we can use the [base, xreg] addressing mode to save an
// add/sub:
//
// mov x0, wide
// ldr x2, [base, x0]
auto ValAndVReg =
    getIConstantVRegValWithLookThrough(PtrAdd->getOperand(2).getReg(), MRI);
if (ValAndVReg) {
  unsigned Scale = Log2_32(SizeInBytes);
  int64_t ImmOff = ValAndVReg->Value.getSExtValue();

  // Skip immediates that can be selected in the load/store addresing
  // mode.
  if (ImmOff % SizeInBytes == 0 && ImmOff >= 0 &&
      ImmOff < (0x1000 << Scale))
    return std::nullopt;

  // Helper lambda to decide whether or not it is preferable to emit an add.
  auto isPreferredADD = [](int64_t ImmOff) {
    // Constants in [0x0, 0xfff] can be encoded in an add.
    if ((ImmOff & 0xfffffffffffff000LL) == 0x0LL)
      return true;

    // Can it be encoded in an add lsl #12?
    if ((ImmOff & 0xffffffffff000fffLL) != 0x0LL)
      return false;

    // It can be encoded in an add lsl #12, but we may not want to. If it is
    // possible to select this as a single movz, then prefer that. A single
    // movz is faster than an add with a shift.
    return (ImmOff & 0xffffffffff00ffffLL) != 0x0LL &&
           (ImmOff & 0xffffffffffff0fffLL) != 0x0LL;
  };

  // If the immediate can be encoded in a single add/sub, then bail out.
  if (isPreferredADD(ImmOff) || isPreferredADD(-ImmOff))
    return std::nullopt;
}

// Try to fold shifts into the addressing mode.
auto AddrModeFns = selectAddrModeShiftedExtendXReg(Root, SizeInBytes);
if (AddrModeFns)
  return AddrModeFns;

// If that doesn't work, see if it's possible to fold in registers from
// a GEP.
return selectAddrModeRegisterOffset(Root);
6364}

6366/// This is used for computing addresses like this:
6367///
6368/// ldr x0, [xBase, wOffset, sxtw #LegalShiftVal]
6369///
6370/// Where we have a 64-bit base register, a 32-bit offset register, and an
6371/// extend (which may or may not be signed).
6372InstructionSelector::ComplexRendererFns
6373AArch64InstructionSelector::selectAddrModeWRO(MachineOperand &Root,
                                            unsigned SizeInBytes) const {
MachineRegisterInfo &MRI = Root.getParent()->getMF()->getRegInfo();

MachineInstr *PtrAdd =
    getOpcodeDef(TargetOpcode::G_PTR_ADD, Root.getReg(), MRI);
if (!PtrAdd || !isWorthFoldingIntoExtendedReg(*PtrAdd, MRI))
  return std::nullopt;

MachineOperand &LHS = PtrAdd->getOperand(1);
MachineOperand &RHS = PtrAdd->getOperand(2);
MachineInstr *OffsetInst = getDefIgnoringCopies(RHS.getReg(), MRI);

// The first case is the same as selectAddrModeXRO, except we need an extend.
// In this case, we try to find a shift and extend, and fold them into the
// addressing mode.
//
// E.g.
//
// off_reg = G_Z/S/ANYEXT ext_reg
// val = G_CONSTANT LegalShiftVal
// shift = G_SHL off_reg val
// ptr = G_PTR_ADD base_reg shift
// x = G_LOAD ptr
//
// In this case we can get a load like this:
//
// ldr x0, [base_reg, ext_reg, sxtw #LegalShiftVal]
auto ExtendedShl = selectExtendedSHL(Root, LHS, OffsetInst->getOperand(0),
                                     SizeInBytes, /*WantsExt=*/true);
if (ExtendedShl)
  return ExtendedShl;

// There was no shift. We can try and fold a G_Z/S/ANYEXT in alone though.
//
// e.g.
// ldr something, [base_reg, ext_reg, sxtw]
if (!isWorthFoldingIntoExtendedReg(*OffsetInst, MRI))
  return std::nullopt;

// Check if this is an extend. We'll get an extend type if it is.
AArch64_AM::ShiftExtendType Ext =
    getExtendTypeForInst(*OffsetInst, MRI, /*IsLoadStore=*/true);
if (Ext == AArch64_AM::InvalidShiftExtend)
  return std::nullopt;

// Need a 32-bit wide register.
MachineIRBuilder MIB(*PtrAdd);
Register ExtReg = moveScalarRegClass(OffsetInst->getOperand(1).getReg(),
                                     AArch64::GPR32RegClass, MIB);
unsigned SignExtend = Ext == AArch64_AM::SXTW;

// Base is LHS, offset is ExtReg.
return {{[=](MachineInstrBuilder &MIB) { MIB.addUse(LHS.getReg()); },
         [=](MachineInstrBuilder &MIB) { MIB.addUse(ExtReg); },
         [=](MachineInstrBuilder &MIB) {
           MIB.addImm(SignExtend);
           MIB.addImm(0);
         }}};
6432}

6434/// Select a "register plus unscaled signed 9-bit immediate" address.  This
6435/// should only match when there is an offset that is not valid for a scaled
6436/// immediate addressing mode.  The "Size" argument is the size in bytes of the
6437/// memory reference, which is needed here to know what is valid for a scaled
6438/// immediate.
6439InstructionSelector::ComplexRendererFns
6440AArch64InstructionSelector::selectAddrModeUnscaled(MachineOperand &Root,
                                                 unsigned Size) const {
MachineRegisterInfo &MRI =
    Root.getParent()->getParent()->getParent()->getRegInfo();

if (!Root.isReg())
12
←
Taking false branch→
  return std::nullopt;

if (!isBaseWithConstantOffset(Root, MRI))
13
←
Assuming the condition is false→
14
←
Taking false branch→
  return std::nullopt;

MachineInstr *RootDef = MRI.getVRegDef(Root.getReg());

MachineOperand &OffImm = RootDef->getOperand(2);
if (!OffImm.isReg())
15
←
Taking false branch→
  return std::nullopt;
MachineInstr *RHS = MRI.getVRegDef(OffImm.getReg());
if (RHS->getOpcode() != TargetOpcode::G_CONSTANT)
16
←
Assuming the condition is false→
17
←
Taking false branch→
  return std::nullopt;
int64_t RHSC;
MachineOperand &RHSOp1 = RHS->getOperand(1);
if (!RHSOp1.isCImm() || RHSOp1.getCImm()->getBitWidth() > 64)
18
←
Assuming the condition is false→
19
←
Taking false branch→
  return std::nullopt;
RHSC = RHSOp1.getCImm()->getSExtValue();

// If the offset is valid as a scaled immediate, don't match here.
if ((RHSC & (Size - 1)) == 0 && RHSC >= 0 && RHSC < (0x1000 << Log2_32(Size)))
20
←
Assuming the condition is true→
21
←
Assuming 'RHSC' is >= 0→
22
←
Calling 'Log2_32'→
24
←
Returning from 'Log2_32'→
25
←
The result of the left shift is undefined due to shifting by '4294967295', which is greater or equal to the width of type 'int'
  return std::nullopt;
if (RHSC >= -256 && RHSC < 256) {
  MachineOperand &Base = RootDef->getOperand(1);
  return {{
      [=](MachineInstrBuilder &MIB) { MIB.add(Base); },
      [=](MachineInstrBuilder &MIB) { MIB.addImm(RHSC); },
  }};
}
return std::nullopt;
6476}

6478InstructionSelector::ComplexRendererFns
6479AArch64InstructionSelector::tryFoldAddLowIntoImm(MachineInstr &RootDef,
                                               unsigned Size,
                                               MachineRegisterInfo &MRI) const {
if (RootDef.getOpcode() != AArch64::G_ADD_LOW)
  return std::nullopt;
MachineInstr &Adrp = *MRI.getVRegDef(RootDef.getOperand(1).getReg());
if (Adrp.getOpcode() != AArch64::ADRP)
  return std::nullopt;

// TODO: add heuristics like isWorthFoldingADDlow() from SelectionDAG.
auto Offset = Adrp.getOperand(1).getOffset();
if (Offset % Size != 0)
  return std::nullopt;

auto GV = Adrp.getOperand(1).getGlobal();
if (GV->isThreadLocal())
  return std::nullopt;

auto &MF = *RootDef.getParent()->getParent();
if (GV->getPointerAlignment(MF.getDataLayout()) < Size)
  return std::nullopt;

unsigned OpFlags = STI.ClassifyGlobalReference(GV, MF.getTarget());
MachineIRBuilder MIRBuilder(RootDef);
Register AdrpReg = Adrp.getOperand(0).getReg();
return {{[=](MachineInstrBuilder &MIB) { MIB.addUse(AdrpReg); },
         [=](MachineInstrBuilder &MIB) {
           MIB.addGlobalAddress(GV, Offset,
                                OpFlags | AArch64II::MO_PAGEOFF |
                                    AArch64II::MO_NC);
         }}};
6510}

6512/// Select a "register plus scaled unsigned 12-bit immediate" address.  The
6513/// "Size" argument is the size in bytes of the memory reference, which
6514/// determines the scale.
6515InstructionSelector::ComplexRendererFns
6516AArch64InstructionSelector::selectAddrModeIndexed(MachineOperand &Root,
                                                unsigned Size) const {
MachineFunction &MF = *Root.getParent()->getParent()->getParent();
MachineRegisterInfo &MRI = MF.getRegInfo();

if (!Root.isReg())
4
←
Taking false branch→
  return std::nullopt;

MachineInstr *RootDef = MRI.getVRegDef(Root.getReg());
if (RootDef->getOpcode() == TargetOpcode::G_FRAME_INDEX) {
5
←
Assuming the condition is false→
6
←
Taking false branch→
  return {{
      [=](MachineInstrBuilder &MIB) { MIB.add(RootDef->getOperand(1)); },
      [=](MachineInstrBuilder &MIB) { MIB.addImm(0); },
  }};
}

CodeModel::Model CM = MF.getTarget().getCodeModel();
// Check if we can fold in the ADD of small code model ADRP + ADD address.
if (CM == CodeModel::Small) {
7
←
Assuming 'CM' is not equal to Small→
8
←
Taking false branch→
  auto OpFns = tryFoldAddLowIntoImm(*RootDef, Size, MRI);
  if (OpFns)
    return OpFns;
}

if (isBaseWithConstantOffset(Root, MRI)) {
9
←
Assuming the condition is false→
10
←
Taking false branch→
  MachineOperand &LHS = RootDef->getOperand(1);
  MachineOperand &RHS = RootDef->getOperand(2);
  MachineInstr *LHSDef = MRI.getVRegDef(LHS.getReg());
  MachineInstr *RHSDef = MRI.getVRegDef(RHS.getReg());

  int64_t RHSC = (int64_t)RHSDef->getOperand(1).getCImm()->getZExtValue();
  unsigned Scale = Log2_32(Size);
  if ((RHSC & (Size - 1)) == 0 && RHSC >= 0 && RHSC < (0x1000 << Scale)) {
    if (LHSDef->getOpcode() == TargetOpcode::G_FRAME_INDEX)
      return {{
          [=](MachineInstrBuilder &MIB) { MIB.add(LHSDef->getOperand(1)); },
          [=](MachineInstrBuilder &MIB) { MIB.addImm(RHSC >> Scale); },
      }};

    return {{
        [=](MachineInstrBuilder &MIB) { MIB.add(LHS); },
        [=](MachineInstrBuilder &MIB) { MIB.addImm(RHSC >> Scale); },
    }};
  }
}

// Before falling back to our general case, check if the unscaled
// instructions can handle this. If so, that's preferable.
if (selectAddrModeUnscaled(Root, Size))
11
←
Calling 'AArch64InstructionSelector::selectAddrModeUnscaled'→
  return std::nullopt;

return {{
    [=](MachineInstrBuilder &MIB) { MIB.add(Root); },
    [=](MachineInstrBuilder &MIB) { MIB.addImm(0); },
}};
6571}

6573/// Given a shift instruction, return the correct shift type for that
6574/// instruction.
6575static AArch64_AM::ShiftExtendType getShiftTypeForInst(MachineInstr &MI) {
switch (MI.getOpcode()) {
default:
  return AArch64_AM::InvalidShiftExtend;
case TargetOpcode::G_SHL:
  return AArch64_AM::LSL;
case TargetOpcode::G_LSHR:
  return AArch64_AM::LSR;
case TargetOpcode::G_ASHR:
  return AArch64_AM::ASR;
case TargetOpcode::G_ROTR:
  return AArch64_AM::ROR;
}
6588}

6590/// Select a "shifted register" operand. If the value is not shifted, set the
6591/// shift operand to a default value of "lsl 0".
6592InstructionSelector::ComplexRendererFns
6593AArch64InstructionSelector::selectShiftedRegister(MachineOperand &Root,
                                                bool AllowROR) const {
if (!Root.isReg())
  return std::nullopt;
MachineRegisterInfo &MRI =
    Root.getParent()->getParent()->getParent()->getRegInfo();

// Check if the operand is defined by an instruction which corresponds to
// a ShiftExtendType. E.g. a G_SHL, G_LSHR, etc.
MachineInstr *ShiftInst = MRI.getVRegDef(Root.getReg());
AArch64_AM::ShiftExtendType ShType = getShiftTypeForInst(*ShiftInst);
if (ShType == AArch64_AM::InvalidShiftExtend)
  return std::nullopt;
if (ShType == AArch64_AM::ROR && !AllowROR)
  return std::nullopt;
if (!isWorthFoldingIntoExtendedReg(*ShiftInst, MRI))
  return std::nullopt;

// Need an immediate on the RHS.
MachineOperand &ShiftRHS = ShiftInst->getOperand(2);
auto Immed = getImmedFromMO(ShiftRHS);
if (!Immed)
  return std::nullopt;

// We have something that we can fold. Fold in the shift's LHS and RHS into
// the instruction.
MachineOperand &ShiftLHS = ShiftInst->getOperand(1);
Register ShiftReg = ShiftLHS.getReg();

unsigned NumBits = MRI.getType(ShiftReg).getSizeInBits();
unsigned Val = *Immed & (NumBits - 1);
unsigned ShiftVal = AArch64_AM::getShifterImm(ShType, Val);

return {{[=](MachineInstrBuilder &MIB) { MIB.addUse(ShiftReg); },
         [=](MachineInstrBuilder &MIB) { MIB.addImm(ShiftVal); }}};
6628}

6630AArch64_AM::ShiftExtendType AArch64InstructionSelector::getExtendTypeForInst(
  MachineInstr &MI, MachineRegisterInfo &MRI, bool IsLoadStore) const {
unsigned Opc = MI.getOpcode();

// Handle explicit extend instructions first.
if (Opc == TargetOpcode::G_SEXT || Opc == TargetOpcode::G_SEXT_INREG) {
  unsigned Size;
  if (Opc == TargetOpcode::G_SEXT)
    Size = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
  else
    Size = MI.getOperand(2).getImm();
  assert(Size != 64 && "Extend from 64 bits?")(static_cast <bool> (Size != 64 && "Extend from 64 bits?"
) ? void (0) : __assert_fail ("Size != 64 && \"Extend from 64 bits?\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 6641, __extension__ __PRETTY_FUNCTION__));
  switch (Size) {
  case 8:
    return IsLoadStore ? AArch64_AM::InvalidShiftExtend : AArch64_AM::SXTB;
  case 16:
    return IsLoadStore ? AArch64_AM::InvalidShiftExtend : AArch64_AM::SXTH;
  case 32:
    return AArch64_AM::SXTW;
  default:
    return AArch64_AM::InvalidShiftExtend;
  }
}

if (Opc == TargetOpcode::G_ZEXT || Opc == TargetOpcode::G_ANYEXT) {
  unsigned Size = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
  assert(Size != 64 && "Extend from 64 bits?")(static_cast <bool> (Size != 64 && "Extend from 64 bits?"
) ? void (0) : __assert_fail ("Size != 64 && \"Extend from 64 bits?\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 6656, __extension__ __PRETTY_FUNCTION__));
  switch (Size) {
  case 8:
    return IsLoadStore ? AArch64_AM::InvalidShiftExtend : AArch64_AM::UXTB;
  case 16:
    return IsLoadStore ? AArch64_AM::InvalidShiftExtend : AArch64_AM::UXTH;
  case 32:
    return AArch64_AM::UXTW;
  default:
    return AArch64_AM::InvalidShiftExtend;
  }
}

// Don't have an explicit extend. Try to handle a G_AND with a constant mask
// on the RHS.
if (Opc != TargetOpcode::G_AND)
  return AArch64_AM::InvalidShiftExtend;

std::optional<uint64_t> MaybeAndMask = getImmedFromMO(MI.getOperand(2));
if (!MaybeAndMask)
  return AArch64_AM::InvalidShiftExtend;
uint64_t AndMask = *MaybeAndMask;
switch (AndMask) {
default:
  return AArch64_AM::InvalidShiftExtend;
case 0xFF:
  return !IsLoadStore ? AArch64_AM::UXTB : AArch64_AM::InvalidShiftExtend;
case 0xFFFF:
  return !IsLoadStore ? AArch64_AM::UXTH : AArch64_AM::InvalidShiftExtend;
case 0xFFFFFFFF:
  return AArch64_AM::UXTW;
}
6688}

6690Register AArch64InstructionSelector::moveScalarRegClass(
  Register Reg, const TargetRegisterClass &RC, MachineIRBuilder &MIB) const {
MachineRegisterInfo &MRI = *MIB.getMRI();
auto Ty = MRI.getType(Reg);
assert(!Ty.isVector() && "Expected scalars only!")(static_cast <bool> (!Ty.isVector() && "Expected scalars only!"
) ? void (0) : __assert_fail ("!Ty.isVector() && \"Expected scalars only!\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 6694, __extension__ __PRETTY_FUNCTION__));
if (Ty.getSizeInBits() == TRI.getRegSizeInBits(RC))
  return Reg;

// Create a copy and immediately select it.
// FIXME: We should have an emitCopy function?
auto Copy = MIB.buildCopy({&RC}, {Reg});
selectCopy(*Copy, TII, MRI, TRI, RBI);
return Copy.getReg(0);
6703}

6705/// Select an "extended register" operand. This operand folds in an extend
6706/// followed by an optional left shift.
6707InstructionSelector::ComplexRendererFns
6708AArch64InstructionSelector::selectArithExtendedRegister(
  MachineOperand &Root) const {
if (!Root.isReg())
  return std::nullopt;
MachineRegisterInfo &MRI =
    Root.getParent()->getParent()->getParent()->getRegInfo();

uint64_t ShiftVal = 0;
Register ExtReg;
AArch64_AM::ShiftExtendType Ext;
MachineInstr *RootDef = getDefIgnoringCopies(Root.getReg(), MRI);
if (!RootDef)
  return std::nullopt;

if (!isWorthFoldingIntoExtendedReg(*RootDef, MRI))
  return std::nullopt;

// Check if we can fold a shift and an extend.
if (RootDef->getOpcode() == TargetOpcode::G_SHL) {
  // Look for a constant on the RHS of the shift.
  MachineOperand &RHS = RootDef->getOperand(2);
  std::optional<uint64_t> MaybeShiftVal = getImmedFromMO(RHS);
  if (!MaybeShiftVal)
    return std::nullopt;
  ShiftVal = *MaybeShiftVal;
  if (ShiftVal > 4)
    return std::nullopt;
  // Look for a valid extend instruction on the LHS of the shift.
  MachineOperand &LHS = RootDef->getOperand(1);
  MachineInstr *ExtDef = getDefIgnoringCopies(LHS.getReg(), MRI);
  if (!ExtDef)
    return std::nullopt;
  Ext = getExtendTypeForInst(*ExtDef, MRI);
  if (Ext == AArch64_AM::InvalidShiftExtend)
    return std::nullopt;
  ExtReg = ExtDef->getOperand(1).getReg();
} else {
  // Didn't get a shift. Try just folding an extend.
  Ext = getExtendTypeForInst(*RootDef, MRI);
  if (Ext == AArch64_AM::InvalidShiftExtend)
    return std::nullopt;
  ExtReg = RootDef->getOperand(1).getReg();

  // If we have a 32 bit instruction which zeroes out the high half of a
  // register, we get an implicit zero extend for free. Check if we have one.
  // FIXME: We actually emit the extend right now even though we don't have
  // to.
  if (Ext == AArch64_AM::UXTW && MRI.getType(ExtReg).getSizeInBits() == 32) {
    MachineInstr *ExtInst = MRI.getVRegDef(ExtReg);
    if (isDef32(*ExtInst))
      return std::nullopt;
  }
}

// We require a GPR32 here. Narrow the ExtReg if needed using a subregister
// copy.
MachineIRBuilder MIB(*RootDef);
ExtReg = moveScalarRegClass(ExtReg, AArch64::GPR32RegClass, MIB);

return {{[=](MachineInstrBuilder &MIB) { MIB.addUse(ExtReg); },
         [=](MachineInstrBuilder &MIB) {
           MIB.addImm(getArithExtendImm(Ext, ShiftVal));
         }}};
6771}

6773void AArch64InstructionSelector::renderTruncImm(MachineInstrBuilder &MIB,
                                              const MachineInstr &MI,
                                              int OpIdx) const {
const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&(static_cast <bool> (MI.getOpcode() == TargetOpcode::G_CONSTANT
 && OpIdx == -1 && "Expected G_CONSTANT") ? void
 (0) : __assert_fail ("MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 && \"Expected G_CONSTANT\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 6778, __extension__ __PRETTY_FUNCTION__))
       "Expected G_CONSTANT")(static_cast <bool> (MI.getOpcode() == TargetOpcode::G_CONSTANT
 && OpIdx == -1 && "Expected G_CONSTANT") ? void
 (0) : __assert_fail ("MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 && \"Expected G_CONSTANT\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 6778, __extension__ __PRETTY_FUNCTION__));
std::optional<int64_t> CstVal =
    getIConstantVRegSExtVal(MI.getOperand(0).getReg(), MRI);
assert(CstVal && "Expected constant value")(static_cast <bool> (CstVal && "Expected constant value"
) ? void (0) : __assert_fail ("CstVal && \"Expected constant value\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 6781, __extension__ __PRETTY_FUNCTION__));
MIB.addImm(*CstVal);
6783}

6785void AArch64InstructionSelector::renderLogicalImm32(
MachineInstrBuilder &MIB, const MachineInstr &I, int OpIdx) const {
assert(I.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&(static_cast <bool> (I.getOpcode() == TargetOpcode::G_CONSTANT
 && OpIdx == -1 && "Expected G_CONSTANT") ? void
 (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 && \"Expected G_CONSTANT\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 6788, __extension__ __PRETTY_FUNCTION__))
       "Expected G_CONSTANT")(static_cast <bool> (I.getOpcode() == TargetOpcode::G_CONSTANT
 && OpIdx == -1 && "Expected G_CONSTANT") ? void
 (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 && \"Expected G_CONSTANT\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 6788, __extension__ __PRETTY_FUNCTION__));
uint64_t CstVal = I.getOperand(1).getCImm()->getZExtValue();
uint64_t Enc = AArch64_AM::encodeLogicalImmediate(CstVal, 32);
MIB.addImm(Enc);
6792}

6794void AArch64InstructionSelector::renderLogicalImm64(
MachineInstrBuilder &MIB, const MachineInstr &I, int OpIdx) const {
assert(I.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&(static_cast <bool> (I.getOpcode() == TargetOpcode::G_CONSTANT
 && OpIdx == -1 && "Expected G_CONSTANT") ? void
 (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 && \"Expected G_CONSTANT\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 6797, __extension__ __PRETTY_FUNCTION__))
       "Expected G_CONSTANT")(static_cast <bool> (I.getOpcode() == TargetOpcode::G_CONSTANT
 && OpIdx == -1 && "Expected G_CONSTANT") ? void
 (0) : __assert_fail ("I.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 && \"Expected G_CONSTANT\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 6797, __extension__ __PRETTY_FUNCTION__));
uint64_t CstVal = I.getOperand(1).getCImm()->getZExtValue();
uint64_t Enc = AArch64_AM::encodeLogicalImmediate(CstVal, 64);
MIB.addImm(Enc);
6801}

6803void AArch64InstructionSelector::renderFPImm16(MachineInstrBuilder &MIB,
                                             const MachineInstr &MI,
                                             int OpIdx) const {
assert(MI.getOpcode() == TargetOpcode::G_FCONSTANT && OpIdx == -1 &&(static_cast <bool> (MI.getOpcode() == TargetOpcode::G_FCONSTANT
 && OpIdx == -1 && "Expected G_FCONSTANT") ? void
 (0) : __assert_fail ("MI.getOpcode() == TargetOpcode::G_FCONSTANT && OpIdx == -1 && \"Expected G_FCONSTANT\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 6807, __extension__ __PRETTY_FUNCTION__))
       "Expected G_FCONSTANT")(static_cast <bool> (MI.getOpcode() == TargetOpcode::G_FCONSTANT
 && OpIdx == -1 && "Expected G_FCONSTANT") ? void
 (0) : __assert_fail ("MI.getOpcode() == TargetOpcode::G_FCONSTANT && OpIdx == -1 && \"Expected G_FCONSTANT\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 6807, __extension__ __PRETTY_FUNCTION__));
MIB.addImm(
    AArch64_AM::getFP16Imm(MI.getOperand(1).getFPImm()->getValueAPF()));
6810}

6812void AArch64InstructionSelector::renderFPImm32(MachineInstrBuilder &MIB,
                                             const MachineInstr &MI,
                                             int OpIdx) const {
assert(MI.getOpcode() == TargetOpcode::G_FCONSTANT && OpIdx == -1 &&(static_cast <bool> (MI.getOpcode() == TargetOpcode::G_FCONSTANT
 && OpIdx == -1 && "Expected G_FCONSTANT") ? void
 (0) : __assert_fail ("MI.getOpcode() == TargetOpcode::G_FCONSTANT && OpIdx == -1 && \"Expected G_FCONSTANT\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 6816, __extension__ __PRETTY_FUNCTION__))
       "Expected G_FCONSTANT")(static_cast <bool> (MI.getOpcode() == TargetOpcode::G_FCONSTANT
 && OpIdx == -1 && "Expected G_FCONSTANT") ? void
 (0) : __assert_fail ("MI.getOpcode() == TargetOpcode::G_FCONSTANT && OpIdx == -1 && \"Expected G_FCONSTANT\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 6816, __extension__ __PRETTY_FUNCTION__));
MIB.addImm(
    AArch64_AM::getFP32Imm(MI.getOperand(1).getFPImm()->getValueAPF()));
6819}

6821void AArch64InstructionSelector::renderFPImm64(MachineInstrBuilder &MIB,
                                             const MachineInstr &MI,
                                             int OpIdx) const {
assert(MI.getOpcode() == TargetOpcode::G_FCONSTANT && OpIdx == -1 &&(static_cast <bool> (MI.getOpcode() == TargetOpcode::G_FCONSTANT
 && OpIdx == -1 && "Expected G_FCONSTANT") ? void
 (0) : __assert_fail ("MI.getOpcode() == TargetOpcode::G_FCONSTANT && OpIdx == -1 && \"Expected G_FCONSTANT\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 6825, __extension__ __PRETTY_FUNCTION__))
       "Expected G_FCONSTANT")(static_cast <bool> (MI.getOpcode() == TargetOpcode::G_FCONSTANT
 && OpIdx == -1 && "Expected G_FCONSTANT") ? void
 (0) : __assert_fail ("MI.getOpcode() == TargetOpcode::G_FCONSTANT && OpIdx == -1 && \"Expected G_FCONSTANT\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 6825, __extension__ __PRETTY_FUNCTION__));
MIB.addImm(
    AArch64_AM::getFP64Imm(MI.getOperand(1).getFPImm()->getValueAPF()));
6828}

6830void AArch64InstructionSelector::renderFPImm32SIMDModImmType4(
  MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
assert(MI.getOpcode() == TargetOpcode::G_FCONSTANT && OpIdx == -1 &&(static_cast <bool> (MI.getOpcode() == TargetOpcode::G_FCONSTANT
 && OpIdx == -1 && "Expected G_FCONSTANT") ? void
 (0) : __assert_fail ("MI.getOpcode() == TargetOpcode::G_FCONSTANT && OpIdx == -1 && \"Expected G_FCONSTANT\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 6833, __extension__ __PRETTY_FUNCTION__))
       "Expected G_FCONSTANT")(static_cast <bool> (MI.getOpcode() == TargetOpcode::G_FCONSTANT
 && OpIdx == -1 && "Expected G_FCONSTANT") ? void
 (0) : __assert_fail ("MI.getOpcode() == TargetOpcode::G_FCONSTANT && OpIdx == -1 && \"Expected G_FCONSTANT\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 6833, __extension__ __PRETTY_FUNCTION__));
MIB.addImm(AArch64_AM::encodeAdvSIMDModImmType4(MI.getOperand(1)
                                                    .getFPImm()
                                                    ->getValueAPF()
                                                    .bitcastToAPInt()
                                                    .getZExtValue()));
6839}

6841bool AArch64InstructionSelector::isLoadStoreOfNumBytes(
  const MachineInstr &MI, unsigned NumBytes) const {
if (!MI.mayLoadOrStore())
  return false;
assert(MI.hasOneMemOperand() &&(static_cast <bool> (MI.hasOneMemOperand() && "Expected load/store to have only one mem op!"
) ? void (0) : __assert_fail ("MI.hasOneMemOperand() && \"Expected load/store to have only one mem op!\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 6846, __extension__ __PRETTY_FUNCTION__))
       "Expected load/store to have only one mem op!")(static_cast <bool> (MI.hasOneMemOperand() && "Expected load/store to have only one mem op!"
) ? void (0) : __assert_fail ("MI.hasOneMemOperand() && \"Expected load/store to have only one mem op!\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 6846, __extension__ __PRETTY_FUNCTION__));
return (*MI.memoperands_begin())->getSize() == NumBytes;
6848}

6850bool AArch64InstructionSelector::isDef32(const MachineInstr &MI) const {
const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
if (MRI.getType(MI.getOperand(0).getReg()).getSizeInBits() != 32)
  return false;

// Only return true if we know the operation will zero-out the high half of
// the 64-bit register. Truncates can be subregister copies, which don't
// zero out the high bits. Copies and other copy-like instructions can be
// fed by truncates, or could be lowered as subregister copies.
switch (MI.getOpcode()) {
default:
  return true;
case TargetOpcode::COPY:
case TargetOpcode::G_BITCAST:
case TargetOpcode::G_TRUNC:
case TargetOpcode::G_PHI:
  return false;
}
6868}


6871// Perform fixups on the given PHI instruction's operands to force them all
6872// to be the same as the destination regbank.
6873static void fixupPHIOpBanks(MachineInstr &MI, MachineRegisterInfo &MRI,
                          const AArch64RegisterBankInfo &RBI) {
assert(MI.getOpcode() == TargetOpcode::G_PHI && "Expected a G_PHI")(static_cast <bool> (MI.getOpcode() == TargetOpcode::G_PHI
 && "Expected a G_PHI") ? void (0) : __assert_fail ("MI.getOpcode() == TargetOpcode::G_PHI && \"Expected a G_PHI\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 6875, __extension__ __PRETTY_FUNCTION__));
Register DstReg = MI.getOperand(0).getReg();
const RegisterBank *DstRB = MRI.getRegBankOrNull(DstReg);
assert(DstRB && "Expected PHI dst to have regbank assigned")(static_cast <bool> (DstRB && "Expected PHI dst to have regbank assigned"
) ? void (0) : __assert_fail ("DstRB && \"Expected PHI dst to have regbank assigned\""
, "llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp"
, 6878, __extension__ __PRETTY_FUNCTION__));
MachineIRBuilder MIB(MI);

// Go through each operand and ensure it has the same regbank.
for (MachineOperand &MO : llvm::drop_begin(MI.operands())) {
  if (!MO.isReg())
    continue;
  Register OpReg = MO.getReg();
  const RegisterBank *RB = MRI.getRegBankOrNull(OpReg);
  if (RB != DstRB) {
    // Insert a cross-bank copy.
    auto *OpDef = MRI.getVRegDef(OpReg);
    const LLT &Ty = MRI.getType(OpReg);
    MachineBasicBlock &OpDefBB = *OpDef->getParent();

    // Any instruction we insert must appear after all PHIs in the block
    // for the block to be valid MIR.
    MachineBasicBlock::iterator InsertPt = std::next(OpDef->getIterator());
    if (InsertPt != OpDefBB.end() && InsertPt->isPHI())
      InsertPt = OpDefBB.getFirstNonPHI();
    MIB.setInsertPt(*OpDef->getParent(), InsertPt);
    auto Copy = MIB.buildCopy(Ty, OpReg);
    MRI.setRegBank(Copy.getReg(0), *DstRB);
    MO.setReg(Copy.getReg(0));
  }
}
6904}

6906void AArch64InstructionSelector::processPHIs(MachineFunction &MF) {
// We're looking for PHIs, build a list so we don't invalidate iterators.
MachineRegisterInfo &MRI = MF.getRegInfo();
SmallVector<MachineInstr *, 32> Phis;
for (auto &BB : MF) {
  for (auto &MI : BB) {
    if (MI.getOpcode() == TargetOpcode::G_PHI)
      Phis.emplace_back(&MI);
  }
}

for (auto *MI : Phis) {
  // We need to do some work here if the operand types are < 16 bit and they
  // are split across fpr/gpr banks. Since all types <32b on gpr
  // end up being assigned gpr32 regclasses, we can end up with PHIs here
  // which try to select between a gpr32 and an fpr16. Ideally RBS shouldn't
  // be selecting heterogenous regbanks for operands if possible, but we
  // still need to be able to deal with it here.
  //
  // To fix this, if we have a gpr-bank operand < 32b in size and at least
  // one other operand is on the fpr bank, then we add cross-bank copies
  // to homogenize the operand banks. For simplicity the bank that we choose
  // to settle on is whatever bank the def operand has. For example:
  //
  // %endbb:
  //   %dst:gpr(s16) = G_PHI %in1:gpr(s16), %bb1, %in2:fpr(s16), %bb2
  //  =>
  // %bb2:
  //   ...
  //   %in2_copy:gpr(s16) = COPY %in2:fpr(s16)
  //   ...
  // %endbb:
  //   %dst:gpr(s16) = G_PHI %in1:gpr(s16), %bb1, %in2_copy:gpr(s16), %bb2
  bool HasGPROp = false, HasFPROp = false;
  for (const MachineOperand &MO : llvm::drop_begin(MI->operands())) {
    if (!MO.isReg())
      continue;
    const LLT &Ty = MRI.getType(MO.getReg());
    if (!Ty.isValid() || !Ty.isScalar())
      break;
    if (Ty.getSizeInBits() >= 32)
      break;
    const RegisterBank *RB = MRI.getRegBankOrNull(MO.getReg());
    // If for some reason we don't have a regbank yet. Don't try anything.
    if (!RB)
      break;

    if (RB->getID() == AArch64::GPRRegBankID)
      HasGPROp = true;
    else
      HasFPROp = true;
  }
  // We have heterogenous regbanks, need to fixup.
  if (HasGPROp && HasFPROp)
    fixupPHIOpBanks(*MI, MRI, RBI);
}
6962}

6964namespace llvm {
6965InstructionSelector *
6966createAArch64InstructionSelector(const AArch64TargetMachine &TM,
                               AArch64Subtarget &Subtarget,
                               AArch64RegisterBankInfo &RBI) {
return new AArch64InstructionSelector(TM, Subtarget, RBI);
6970}
6971}

←

/build/source/llvm/include/llvm/Support/MathExtras.h

1//===-- llvm/Support/MathExtras.h - Useful math functions -------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains some functions that are useful for math stuff.
10//
11//===----------------------------------------------------------------------===//
12 
13#ifndef LLVM_SUPPORT_MATHEXTRAS_H
14#define LLVM_SUPPORT_MATHEXTRAS_H
15 
16#include "llvm/ADT/bit.h"
17#include "llvm/Support/Compiler.h"
18#include <cassert>
19#include <climits>
20#include <cstdint>
21#include <cstring>
22#include <limits>
23#include <type_traits>
24 
25namespace llvm {
26 
27/// Mathematical constants.
28namespace numbers {
29// TODO: Track C++20 std::numbers.
30// TODO: Favor using the hexadecimal FP constants (requires C++17).
31constexpr double e          = 2.7182818284590452354, // (0x1.5bf0a8b145749P+1) https://oeis.org/A001113
32                 egamma     = .57721566490153286061, // (0x1.2788cfc6fb619P-1) https://oeis.org/A001620
33                 ln2        = .69314718055994530942, // (0x1.62e42fefa39efP-1) https://oeis.org/A002162
34                 ln10       = 2.3025850929940456840, // (0x1.24bb1bbb55516P+1) https://oeis.org/A002392
35                 log2e      = 1.4426950408889634074, // (0x1.71547652b82feP+0)
36                 log10e     = .43429448190325182765, // (0x1.bcb7b1526e50eP-2)
37                 pi         = 3.1415926535897932385, // (0x1.921fb54442d18P+1) https://oeis.org/A000796
38                 inv_pi     = .31830988618379067154, // (0x1.45f306bc9c883P-2) https://oeis.org/A049541
39                 sqrtpi     = 1.7724538509055160273, // (0x1.c5bf891b4ef6bP+0) https://oeis.org/A002161
40                 inv_sqrtpi = .56418958354775628695, // (0x1.20dd750429b6dP-1) https://oeis.org/A087197
41                 sqrt2      = 1.4142135623730950488, // (0x1.6a09e667f3bcdP+0) https://oeis.org/A00219
42                 inv_sqrt2  = .70710678118654752440, // (0x1.6a09e667f3bcdP-1)
43                 sqrt3      = 1.7320508075688772935, // (0x1.bb67ae8584caaP+0) https://oeis.org/A002194
44                 inv_sqrt3  = .57735026918962576451, // (0x1.279a74590331cP-1)
45                 phi        = 1.6180339887498948482; // (0x1.9e3779b97f4a8P+0) https://oeis.org/A001622
46constexpr float ef          = 2.71828183F, // (0x1.5bf0a8P+1) https://oeis.org/A001113
47                egammaf     = .577215665F, // (0x1.2788d0P-1) https://oeis.org/A001620
48                ln2f        = .693147181F, // (0x1.62e430P-1) https://oeis.org/A002162
49                ln10f       = 2.30258509F, // (0x1.26bb1cP+1) https://oeis.org/A002392
50                log2ef      = 1.44269504F, // (0x1.715476P+0)
51                log10ef     = .434294482F, // (0x1.bcb7b2P-2)
52                pif         = 3.14159265F, // (0x1.921fb6P+1) https://oeis.org/A000796
53                inv_pif     = .318309886F, // (0x1.45f306P-2) https://oeis.org/A049541
54                sqrtpif     = 1.77245385F, // (0x1.c5bf8aP+0) https://oeis.org/A002161
55                inv_sqrtpif = .564189584F, // (0x1.20dd76P-1) https://oeis.org/A087197
56                sqrt2f      = 1.41421356F, // (0x1.6a09e6P+0) https://oeis.org/A002193
57                inv_sqrt2f  = .707106781F, // (0x1.6a09e6P-1)
58                sqrt3f      = 1.73205081F, // (0x1.bb67aeP+0) https://oeis.org/A002194
59                inv_sqrt3f  = .577350269F, // (0x1.279a74P-1)
60                phif        = 1.61803399F; // (0x1.9e377aP+0) https://oeis.org/A001622
61} // namespace numbers
62 
63/// Count number of 0's from the least significant bit to the most
64///   stopping at the first 1.
65///
66/// Only unsigned integral types are allowed.
67///
68/// Returns std::numeric_limits<T>::digits on an input of 0.
69template <typename T>
70LLVM_DEPRECATED("Use llvm::countr_zero instead.", "llvm::countr_zero")__attribute__((deprecated("Use llvm::countr_zero instead.", "llvm::countr_zero"
)))
71unsigned countTrailingZeros(T Val) {
72  static_assert(std::is_unsigned_v<T>,
73                "Only unsigned integral types are allowed.");
74  return llvm::countr_zero(Val);
75}
76 
77/// Count number of 0's from the most significant bit to the least
78///   stopping at the first 1.
79///
80/// Only unsigned integral types are allowed.
81///
82/// Returns std::numeric_limits<T>::digits on an input of 0.
83template <typename T>
84LLVM_DEPRECATED("Use llvm::countl_zero instead.", "llvm::countl_zero")__attribute__((deprecated("Use llvm::countl_zero instead.", "llvm::countl_zero"
)))
85unsigned countLeadingZeros(T Val) {
86  static_assert(std::is_unsigned_v<T>,
87                "Only unsigned integral types are allowed.");
88  return llvm::countl_zero(Val);
89}
90 
91/// Create a bitmask with the N right-most bits set to 1, and all other
92/// bits set to 0.  Only unsigned types are allowed.
93template <typename T> T maskTrailingOnes(unsigned N) {
94  static_assert(std::is_unsigned_v<T>, "Invalid type!");
95  const unsigned Bits = CHAR_BIT8 * sizeof(T);
96  assert(N <= Bits && "Invalid bit index")(static_cast <bool> (N <= Bits && "Invalid bit index"
) ? void (0) : __assert_fail ("N <= Bits && \"Invalid bit index\""
, "llvm/include/llvm/Support/MathExtras.h", 96, __extension__
 __PRETTY_FUNCTION__));
97  return N == 0 ? 0 : (T(-1) >> (Bits - N));
98}
99 
100/// Create a bitmask with the N left-most bits set to 1, and all other
101/// bits set to 0.  Only unsigned types are allowed.
102template <typename T> T maskLeadingOnes(unsigned N) {
103  return ~maskTrailingOnes<T>(CHAR_BIT8 * sizeof(T) - N);
104}
105 
106/// Create a bitmask with the N right-most bits set to 0, and all other
107/// bits set to 1.  Only unsigned types are allowed.
108template <typename T> T maskTrailingZeros(unsigned N) {
109  return maskLeadingOnes<T>(CHAR_BIT8 * sizeof(T) - N);
110}
111 
112/// Create a bitmask with the N left-most bits set to 0, and all other
113/// bits set to 1.  Only unsigned types are allowed.
114template <typename T> T maskLeadingZeros(unsigned N) {
115  return maskTrailingOnes<T>(CHAR_BIT8 * sizeof(T) - N);
116}
117 
118/// Macro compressed bit reversal table for 256 bits.
119///
120/// http://graphics.stanford.edu/~seander/bithacks.html#BitReverseTable
121static const unsigned char BitReverseTable256[256] = {
122#define R2(n) n, n + 2 * 64, n + 1 * 64, n + 3 * 64
123#define R4(n) R2(n), R2(n + 2 * 16), R2(n + 1 * 16), R2(n + 3 * 16)
124#define R6(n) R4(n), R4(n + 2 * 4), R4(n + 1 * 4), R4(n + 3 * 4)
125  R6(0), R6(2), R6(1), R6(3)
126#undef R2
127#undef R4
128#undef R6
129};
130 
131/// Reverse the bits in \p Val.
132template <typename T> T reverseBits(T Val) {
133#if __has_builtin(__builtin_bitreverse8)1
134  if constexpr (std::is_same_v<T, uint8_t>)
135    return __builtin_bitreverse8(Val);
136#endif
137#if __has_builtin(__builtin_bitreverse16)1
138  if constexpr (std::is_same_v<T, uint16_t>)
139    return __builtin_bitreverse16(Val);
140#endif
141#if __has_builtin(__builtin_bitreverse32)1
142  if constexpr (std::is_same_v<T, uint32_t>)
143    return __builtin_bitreverse32(Val);
144#endif
145#if __has_builtin(__builtin_bitreverse64)1
146  if constexpr (std::is_same_v<T, uint64_t>)
147    return __builtin_bitreverse64(Val);
148#endif
149 
150  unsigned char in[sizeof(Val)];
151  unsigned char out[sizeof(Val)];
152  std::memcpy(in, &Val, sizeof(Val));
153  for (unsigned i = 0; i < sizeof(Val); ++i)
154    out[(sizeof(Val) - i) - 1] = BitReverseTable256[in[i]];
155  std::memcpy(&Val, out, sizeof(Val));
156  return Val;
157}
158 
159// NOTE: The following support functions use the _32/_64 extensions instead of
160// type overloading so that signed and unsigned integers can be used without
161// ambiguity.
162 
163/// Return the high 32 bits of a 64 bit value.
164constexpr inline uint32_t Hi_32(uint64_t Value) {
165  return static_cast<uint32_t>(Value >> 32);
166}
167 
168/// Return the low 32 bits of a 64 bit value.
169constexpr inline uint32_t Lo_32(uint64_t Value) {
170  return static_cast<uint32_t>(Value);
171}
172 
173/// Make a 64-bit integer from a high / low pair of 32-bit integers.
174constexpr inline uint64_t Make_64(uint32_t High, uint32_t Low) {
175  return ((uint64_t)High << 32) | (uint64_t)Low;
176}
177 
178/// Checks if an integer fits into the given bit width.
179template <unsigned N> constexpr inline bool isInt(int64_t x) {
180  if constexpr (N == 8)
181    return static_cast<int8_t>(x) == x;
182  if constexpr (N == 16)
183    return static_cast<int16_t>(x) == x;
184  if constexpr (N == 32)
185    return static_cast<int32_t>(x) == x;
186  if constexpr (N < 64)
187    return -(INT64_C(1)1L << (N - 1)) <= x && x < (INT64_C(1)1L << (N - 1));
188  (void)x; // MSVC v19.25 warns that x is unused.
189  return true;
190}
191 
192/// Checks if a signed integer is an N bit number shifted left by S.
193template <unsigned N, unsigned S>
194constexpr inline bool isShiftedInt(int64_t x) {
195  static_assert(
196      N > 0, "isShiftedInt<0> doesn't make sense (refers to a 0-bit number.");
197  static_assert(N + S <= 64, "isShiftedInt<N, S> with N + S > 64 is too wide.");
198  return isInt<N + S>(x) && (x % (UINT64_C(1)1UL << S) == 0);
199}
200 
201/// Checks if an unsigned integer fits into the given bit width.
202template <unsigned N> constexpr inline bool isUInt(uint64_t x) {
203  static_assert(N > 0, "isUInt<0> doesn't make sense");
204  if constexpr (N == 8)
205    return static_cast<uint8_t>(x) == x;
206  if constexpr (N == 16)
207    return static_cast<uint16_t>(x) == x;
208  if constexpr (N == 32)
209    return static_cast<uint32_t>(x) == x;
210  if constexpr (N < 64)
211    return x < (UINT64_C(1)1UL << (N));
212  (void)x; // MSVC v19.25 warns that x is unused.
213  return true;
214}
215 
216/// Checks if a unsigned integer is an N bit number shifted left by S.
217template <unsigned N, unsigned S>
218constexpr inline bool isShiftedUInt(uint64_t x) {
219  static_assert(
220      N > 0, "isShiftedUInt<0> doesn't make sense (refers to a 0-bit number)");
221  static_assert(N + S <= 64,
222                "isShiftedUInt<N, S> with N + S > 64 is too wide.");
223  // Per the two static_asserts above, S must be strictly less than 64.  So
224  // 1 << S is not undefined behavior.
225  return isUInt<N + S>(x) && (x % (UINT64_C(1)1UL << S) == 0);
226}
227 
228/// Gets the maximum value for a N-bit unsigned integer.
229inline uint64_t maxUIntN(uint64_t N) {
230  assert(N > 0 && N <= 64 && "integer width out of range")(static_cast <bool> (N > 0 && N <= 64 &&
 "integer width out of range") ? void (0) : __assert_fail ("N > 0 && N <= 64 && \"integer width out of range\""
, "llvm/include/llvm/Support/MathExtras.h", 230, __extension__
 __PRETTY_FUNCTION__));
231 
232  // uint64_t(1) << 64 is undefined behavior, so we can't do
233  //   (uint64_t(1) << N) - 1
234  // without checking first that N != 64.  But this works and doesn't have a
235  // branch.
236  return UINT64_MAX(18446744073709551615UL) >> (64 - N);
237}
238 
239/// Gets the minimum value for a N-bit signed integer.
240inline int64_t minIntN(int64_t N) {
241  assert(N > 0 && N <= 64 && "integer width out of range")(static_cast <bool> (N > 0 && N <= 64 &&
 "integer width out of range") ? void (0) : __assert_fail ("N > 0 && N <= 64 && \"integer width out of range\""
, "llvm/include/llvm/Support/MathExtras.h", 241, __extension__
 __PRETTY_FUNCTION__));
242 
243  return UINT64_C(1)1UL + ~(UINT64_C(1)1UL << (N - 1));
244}
245 
246/// Gets the maximum value for a N-bit signed integer.
247inline int64_t maxIntN(int64_t N) {
248  assert(N > 0 && N <= 64 && "integer width out of range")(static_cast <bool> (N > 0 && N <= 64 &&
 "integer width out of range") ? void (0) : __assert_fail ("N > 0 && N <= 64 && \"integer width out of range\""
, "llvm/include/llvm/Support/MathExtras.h", 248, __extension__
 __PRETTY_FUNCTION__));
249 
250  // This relies on two's complement wraparound when N == 64, so we convert to
251  // int64_t only at the very end to avoid UB.
252  return (UINT64_C(1)1UL << (N - 1)) - 1;
253}
254 
255/// Checks if an unsigned integer fits into the given (dynamic) bit width.
256inline bool isUIntN(unsigned N, uint64_t x) {
257  return N >= 64 || x <= maxUIntN(N);
258}
259 
260/// Checks if an signed integer fits into the given (dynamic) bit width.
261inline bool isIntN(unsigned N, int64_t x) {
262  return N >= 64 || (minIntN(N) <= x && x <= maxIntN(N));
263}
264 
265/// Return true if the argument is a non-empty sequence of ones starting at the
266/// least significant bit with the remainder zero (32 bit version).
267/// Ex. isMask_32(0x0000FFFFU) == true.
268constexpr inline bool isMask_32(uint32_t Value) {
269  return Value && ((Value + 1) & Value) == 0;
270}
271 
272/// Return true if the argument is a non-empty sequence of ones starting at the
273/// least significant bit with the remainder zero (64 bit version).
274constexpr inline bool isMask_64(uint64_t Value) {
275  return Value && ((Value + 1) & Value) == 0;
276}
277 
278/// Return true if the argument contains a non-empty sequence of ones with the
279/// remainder zero (32 bit version.) Ex. isShiftedMask_32(0x0000FF00U) == true.
280constexpr inline bool isShiftedMask_32(uint32_t Value) {
281  return Value && isMask_32((Value - 1) | Value);
282}
283 
284/// Return true if the argument contains a non-empty sequence of ones with the
285/// remainder zero (64 bit version.)
286constexpr inline bool isShiftedMask_64(uint64_t Value) {
287  return Value && isMask_64((Value - 1) | Value);
288}
289 
290/// Return true if the argument is a power of two > 0.
291/// Ex. isPowerOf2_32(0x00100000U) == true (32 bit edition.)
292constexpr inline bool isPowerOf2_32(uint32_t Value) {
293  return llvm::has_single_bit(Value);
294}
295 
296/// Return true if the argument is a power of two > 0 (64 bit edition.)
297constexpr inline bool isPowerOf2_64(uint64_t Value) {
298  return llvm::has_single_bit(Value);
299}
300 
301/// Count the number of ones from the most significant bit to the first
302/// zero bit.
303///
304/// Ex. countLeadingOnes(0xFF0FFF00) == 8.
305/// Only unsigned integral types are allowed.
306///
307/// Returns std::numeric_limits<T>::digits on an input of all ones.
308template <typename T>
309LLVM_DEPRECATED("Use llvm::countl_one instead.", "llvm::countl_one")__attribute__((deprecated("Use llvm::countl_one instead.", "llvm::countl_one"
)))
310unsigned countLeadingOnes(T Value) {
311  static_assert(std::is_unsigned_v<T>,
312                "Only unsigned integral types are allowed.");
313  return llvm::countl_one<T>(Value);
314}
315 
316/// Count the number of ones from the least significant bit to the first
317/// zero bit.
318///
319/// Ex. countTrailingOnes(0x00FF00FF) == 8.
320/// Only unsigned integral types are allowed.
321///
322/// Returns std::numeric_limits<T>::digits on an input of all ones.
323template <typename T>
324LLVM_DEPRECATED("Use llvm::countr_one instead.", "llvm::countr_one")__attribute__((deprecated("Use llvm::countr_one instead.", "llvm::countr_one"
)))
325unsigned countTrailingOnes(T Value) {
326  static_assert(std::is_unsigned_v<T>,
327                "Only unsigned integral types are allowed.");
328  return llvm::countr_one<T>(Value);
329}
330 
331/// Count the number of set bits in a value.
332/// Ex. countPopulation(0xF000F000) = 8
333/// Returns 0 if the word is zero.
334template <typename T>
335LLVM_DEPRECATED("Use llvm::popcount instead.", "llvm::popcount")__attribute__((deprecated("Use llvm::popcount instead.", "llvm::popcount"
)))
336inline unsigned countPopulation(T Value) {
337  static_assert(std::is_unsigned_v<T>,
338                "Only unsigned integral types are allowed.");
339  return (unsigned)llvm::popcount(Value);
340}
341 
342/// Return true if the argument contains a non-empty sequence of ones with the
343/// remainder zero (32 bit version.) Ex. isShiftedMask_32(0x0000FF00U) == true.
344/// If true, \p MaskIdx will specify the index of the lowest set bit and \p
345/// MaskLen is updated to specify the length of the mask, else neither are
346/// updated.
347inline bool isShiftedMask_32(uint32_t Value, unsigned &MaskIdx,
348                             unsigned &MaskLen) {
349  if (!isShiftedMask_32(Value))
350    return false;
351  MaskIdx = llvm::countr_zero(Value);
352  MaskLen = llvm::popcount(Value);
353  return true;
354}
355 
356/// Return true if the argument contains a non-empty sequence of ones with the
357/// remainder zero (64 bit version.) If true, \p MaskIdx will specify the index
358/// of the lowest set bit and \p MaskLen is updated to specify the length of the
359/// mask, else neither are updated.
360inline bool isShiftedMask_64(uint64_t Value, unsigned &MaskIdx,
361                             unsigned &MaskLen) {
362  if (!isShiftedMask_64(Value))
363    return false;
364  MaskIdx = llvm::countr_zero(Value);
365  MaskLen = llvm::popcount(Value);
366  return true;
367}
368 
369/// Compile time Log2.
370/// Valid only for positive powers of two.
371template <size_t kValue> constexpr inline size_t CTLog2() {
372  static_assert(kValue > 0 && llvm::isPowerOf2_64(kValue),
373                "Value is not a valid power of 2");
374  return 1 + CTLog2<kValue / 2>();
375}
376 
377template <> constexpr inline size_t CTLog2<1>() { return 0; }
378 
379/// Return the floor log base 2 of the specified value, -1 if the value is zero.
380/// (32 bit edition.)
381/// Ex. Log2_32(32) == 5, Log2_32(1) == 0, Log2_32(0) == -1, Log2_32(6) == 2
382inline unsigned Log2_32(uint32_t Value) {
383  return 31 - llvm::countl_zero(Value);
23
←
Returning the value 4294967295→
384}
385 
386/// Return the floor log base 2 of the specified value, -1 if the value is zero.
387/// (64 bit edition.)
388inline unsigned Log2_64(uint64_t Value) {
389  return 63 - llvm::countl_zero(Value);
390}
391 
392/// Return the ceil log base 2 of the specified value, 32 if the value is zero.
393/// (32 bit edition).
394/// Ex. Log2_32_Ceil(32) == 5, Log2_32_Ceil(1) == 0, Log2_32_Ceil(6) == 3
395inline unsigned Log2_32_Ceil(uint32_t Value) {
396  return 32 - llvm::countl_zero(Value - 1);
397}
398 
399/// Return the ceil log base 2 of the specified value, 64 if the value is zero.
400/// (64 bit edition.)
401inline unsigned Log2_64_Ceil(uint64_t Value) {
402  return 64 - llvm::countl_zero(Value - 1);
403}
404 
405/// This function takes a 64-bit integer and returns the bit equivalent double.
406LLVM_DEPRECATED("use llvm::bit_cast instead", "llvm::bit_cast<double>")__attribute__((deprecated("use llvm::bit_cast instead", "llvm::bit_cast<double>"
)))
407inline double BitsToDouble(uint64_t Bits) {
408  static_assert(sizeof(uint64_t) == sizeof(double), "Unexpected type sizes");
409  return llvm::bit_cast<double>(Bits);
410}
411 
412/// This function takes a 32-bit integer and returns the bit equivalent float.
413LLVM_DEPRECATED("use llvm::bit_cast instead", "llvm::bit_cast<float>")__attribute__((deprecated("use llvm::bit_cast instead", "llvm::bit_cast<float>"
)))
414inline float BitsToFloat(uint32_t Bits) {
415  static_assert(sizeof(uint32_t) == sizeof(float), "Unexpected type sizes");
416  return llvm::bit_cast<float>(Bits);
417}
418 
419/// This function takes a double and returns the bit equivalent 64-bit integer.
420/// Note that copying doubles around changes the bits of NaNs on some hosts,
421/// notably x86, so this routine cannot be used if these bits are needed.
422LLVM_DEPRECATED("use llvm::bit_cast instead", "llvm::bit_cast<uint64_t>")__attribute__((deprecated("use llvm::bit_cast instead", "llvm::bit_cast<uint64_t>"
)))
423inline uint64_t DoubleToBits(double Double) {
424  static_assert(sizeof(uint64_t) == sizeof(double), "Unexpected type sizes");
425  return llvm::bit_cast<uint64_t>(Double);
426}
427 
428/// This function takes a float and returns the bit equivalent 32-bit integer.
429/// Note that copying floats around changes the bits of NaNs on some hosts,
430/// notably x86, so this routine cannot be used if these bits are needed.
431LLVM_DEPRECATED("use llvm::bit_cast instead", "llvm::bit_cast<uint32_t>")__attribute__((deprecated("use llvm::bit_cast instead", "llvm::bit_cast<uint32_t>"
)))
432inline uint32_t FloatToBits(float Float) {
433  static_assert(sizeof(uint32_t) == sizeof(float), "Unexpected type sizes");
434  return llvm::bit_cast<uint32_t>(Float);
435}
436 
437/// A and B are either alignments or offsets. Return the minimum alignment that
438/// may be assumed after adding the two together.
439constexpr inline uint64_t MinAlign(uint64_t A, uint64_t B) {
440  // The largest power of 2 that divides both A and B.
441  //
442  // Replace "-Value" by "1+~Value" in the following commented code to avoid
443  // MSVC warning C4146
444  //    return (A | B) & -(A | B);
445  return (A | B) & (1 + ~(A | B));
446}
447 
448/// Returns the next power of two (in 64-bits) that is strictly greater than A.
449/// Returns zero on overflow.
450constexpr inline uint64_t NextPowerOf2(uint64_t A) {
451  A |= (A >> 1);
452  A |= (A >> 2);
453  A |= (A >> 4);
454  A |= (A >> 8);
455  A |= (A >> 16);
456  A |= (A >> 32);
457  return A + 1;
458}
459 
460/// Returns the power of two which is less than or equal to the given value.
461/// Essentially, it is a floor operation across the domain of powers of two.
462LLVM_DEPRECATED("use llvm::bit_floor instead", "llvm::bit_floor")__attribute__((deprecated("use llvm::bit_floor instead", "llvm::bit_floor"
)))
463inline uint64_t PowerOf2Floor(uint64_t A) {
464  return llvm::bit_floor(A);
465}
466 
467/// Returns the power of two which is greater than or equal to the given value.
468/// Essentially, it is a ceil operation across the domain of powers of two.
469inline uint64_t PowerOf2Ceil(uint64_t A) {
470  if (!A)
471    return 0;
472  return NextPowerOf2(A - 1);
473}
474 
475/// Returns the next integer (mod 2**64) that is greater than or equal to
476/// \p Value and is a multiple of \p Align. \p Align must be non-zero.
477///
478/// Examples:
479/// \code
480///   alignTo(5, 8) = 8
481///   alignTo(17, 8) = 24
482///   alignTo(~0LL, 8) = 0
483///   alignTo(321, 255) = 510
484/// \endcode
485inline uint64_t alignTo(uint64_t Value, uint64_t Align) {
486  assert(Align != 0u && "Align can't be 0.")(static_cast <bool> (Align != 0u && "Align can't be 0."
) ? void (0) : __assert_fail ("Align != 0u && \"Align can't be 0.\""
, "llvm/include/llvm/Support/MathExtras.h", 486, __extension__
 __PRETTY_FUNCTION__));
487  return (Value + Align - 1) / Align * Align;
488}
489 
490inline uint64_t alignToPowerOf2(uint64_t Value, uint64_t Align) {
491  assert(Align != 0 && (Align & (Align - 1)) == 0 &&(static_cast <bool> (Align != 0 && (Align &
 (Align - 1)) == 0 && "Align must be a power of 2") ?
 void (0) : __assert_fail ("Align != 0 && (Align & (Align - 1)) == 0 && \"Align must be a power of 2\""
, "llvm/include/llvm/Support/MathExtras.h", 492, __extension__
 __PRETTY_FUNCTION__))
492         "Align must be a power of 2")(static_cast <bool> (Align != 0 && (Align &
 (Align - 1)) == 0 && "Align must be a power of 2") ?
 void (0) : __assert_fail ("Align != 0 && (Align & (Align - 1)) == 0 && \"Align must be a power of 2\""
, "llvm/include/llvm/Support/MathExtras.h", 492, __extension__
 __PRETTY_FUNCTION__));
493  return (Value + Align - 1) & -Align;
494}
495 
496/// If non-zero \p Skew is specified, the return value will be a minimal integer
497/// that is greater than or equal to \p Size and equal to \p A * N + \p Skew for
498/// some integer N. If \p Skew is larger than \p A, its value is adjusted to '\p
499/// Skew mod \p A'. \p Align must be non-zero.
500///
501/// Examples:
502/// \code
503///   alignTo(5, 8, 7) = 7
504///   alignTo(17, 8, 1) = 17
505///   alignTo(~0LL, 8, 3) = 3
506///   alignTo(321, 255, 42) = 552
507/// \endcode
508inline uint64_t alignTo(uint64_t Value, uint64_t Align, uint64_t Skew) {
509  assert(Align != 0u && "Align can't be 0.")(static_cast <bool> (Align != 0u && "Align can't be 0."
) ? void (0) : __assert_fail ("Align != 0u && \"Align can't be 0.\""
, "llvm/include/llvm/Support/MathExtras.h", 509, __extension__
 __PRETTY_FUNCTION__));
510  Skew %= Align;
511  return alignTo(Value - Skew, Align) + Skew;
512}
513 
514/// Returns the next integer (mod 2**64) that is greater than or equal to
515/// \p Value and is a multiple of \c Align. \c Align must be non-zero.
516template <uint64_t Align> constexpr inline uint64_t alignTo(uint64_t Value) {
517  static_assert(Align != 0u, "Align must be non-zero");
518  return (Value + Align - 1) / Align * Align;
519}
520 
521/// Returns the integer ceil(Numerator / Denominator).
522inline uint64_t divideCeil(uint64_t Numerator, uint64_t Denominator) {
523  return alignTo(Numerator, Denominator) / Denominator;
524}
525 
526/// Returns the integer nearest(Numerator / Denominator).
527inline uint64_t divideNearest(uint64_t Numerator, uint64_t Denominator) {
528  return (Numerator + (Denominator / 2)) / Denominator;
529}
530 
531/// Returns the largest uint64_t less than or equal to \p Value and is
532/// \p Skew mod \p Align. \p Align must be non-zero
533inline uint64_t alignDown(uint64_t Value, uint64_t Align, uint64_t Skew = 0) {
534  assert(Align != 0u && "Align can't be 0.")(static_cast <bool> (Align != 0u && "Align can't be 0."
) ? void (0) : __assert_fail ("Align != 0u && \"Align can't be 0.\""
, "llvm/include/llvm/Support/MathExtras.h", 534, __extension__
 __PRETTY_FUNCTION__));
535  Skew %= Align;
536  return (Value - Skew) / Align * Align + Skew;
537}
538 
539/// Sign-extend the number in the bottom B bits of X to a 32-bit integer.
540/// Requires 0 < B <= 32.
541template <unsigned B> constexpr inline int32_t SignExtend32(uint32_t X) {
542  static_assert(B > 0, "Bit width can't be 0.");
543  static_assert(B <= 32, "Bit width out of range.");
544  return int32_t(X << (32 - B)) >> (32 - B);
545}
546 
547/// Sign-extend the number in the bottom B bits of X to a 32-bit integer.
548/// Requires 0 < B <= 32.
549inline int32_t SignExtend32(uint32_t X, unsigned B) {
550  assert(B > 0 && "Bit width can't be 0.")(static_cast <bool> (B > 0 && "Bit width can't be 0."
) ? void (0) : __assert_fail ("B > 0 && \"Bit width can't be 0.\""
, "llvm/include/llvm/Support/MathExtras.h", 550, __extension__
 __PRETTY_FUNCTION__));
551  assert(B <= 32 && "Bit width out of range.")(static_cast <bool> (B <= 32 && "Bit width out of range."
) ? void (0) : __assert_fail ("B <= 32 && \"Bit width out of range.\""
, "llvm/include/llvm/Support/MathExtras.h", 551, __extension__
 __PRETTY_FUNCTION__));
552  return int32_t(X << (32 - B)) >> (32 - B);
553}
554 
555/// Sign-extend the number in the bottom B bits of X to a 64-bit integer.
556/// Requires 0 < B <= 64.
557template <unsigned B> constexpr inline int64_t SignExtend64(uint64_t x) {
558  static_assert(B > 0, "Bit width can't be 0.");
559  static_assert(B <= 64, "Bit width out of range.");
560  return int64_t(x << (64 - B)) >> (64 - B);
561}
562 
563/// Sign-extend the number in the bottom B bits of X to a 64-bit integer.
564/// Requires 0 < B <= 64.
565inline int64_t SignExtend64(uint64_t X, unsigned B) {
566  assert(B > 0 && "Bit width can't be 0.")(static_cast <bool> (B > 0 && "Bit width can't be 0."
) ? void (0) : __assert_fail ("B > 0 && \"Bit width can't be 0.\""
, "llvm/include/llvm/Support/MathExtras.h", 566, __extension__
 __PRETTY_FUNCTION__));
567  assert(B <= 64 && "Bit width out of range.")(static_cast <bool> (B <= 64 && "Bit width out of range."
) ? void (0) : __assert_fail ("B <= 64 && \"Bit width out of range.\""
, "llvm/include/llvm/Support/MathExtras.h", 567, __extension__
 __PRETTY_FUNCTION__));
568  return int64_t(X << (64 - B)) >> (64 - B);
569}
570 
571/// Subtract two unsigned integers, X and Y, of type T and return the absolute
572/// value of the result.
573template <typename T>
574std::enable_if_t<std::is_unsigned_v<T>, T> AbsoluteDifference(T X, T Y) {
575  return X > Y ? (X - Y) : (Y - X);
576}
577 
578/// Add two unsigned integers, X and Y, of type T.  Clamp the result to the
579/// maximum representable value of T on overflow.  ResultOverflowed indicates if
580/// the result is larger than the maximum representable value of type T.
581template <typename T>
582std::enable_if_t<std::is_unsigned_v<T>, T>
583SaturatingAdd(T X, T Y, bool *ResultOverflowed = nullptr) {
584  bool Dummy;
585  bool &Overflowed = ResultOverflowed ? *ResultOverflowed : Dummy;
586  // Hacker's Delight, p. 29
587  T Z = X + Y;
588  Overflowed = (Z < X || Z < Y);
589  if (Overflowed)
590    return std::numeric_limits<T>::max();
591  else
592    return Z;
593}
594 
595/// Add multiple unsigned integers of type T.  Clamp the result to the
596/// maximum representable value of T on overflow.
597template <class T, class... Ts>
598std::enable_if_t<std::is_unsigned_v<T>, T> SaturatingAdd(T X, T Y, T Z,
599                                                         Ts... Args) {
600  bool Overflowed = false;
601  T XY = SaturatingAdd(X, Y, &Overflowed);
602  if (Overflowed)
603    return SaturatingAdd(std::numeric_limits<T>::max(), T(1), Args...);
604  return SaturatingAdd(XY, Z, Args...);
605}
606 
607/// Multiply two unsigned integers, X and Y, of type T.  Clamp the result to the
608/// maximum representable value of T on overflow.  ResultOverflowed indicates if
609/// the result is larger than the maximum representable value of type T.
610template <typename T>
611std::enable_if_t<std::is_unsigned_v<T>, T>
612SaturatingMultiply(T X, T Y, bool *ResultOverflowed = nullptr) {
613  bool Dummy;
614  bool &Overflowed = ResultOverflowed ? *ResultOverflowed : Dummy;
615 
616  // Hacker's Delight, p. 30 has a different algorithm, but we don't use that
617  // because it fails for uint16_t (where multiplication can have undefined
618  // behavior due to promotion to int), and requires a division in addition
619  // to the multiplication.
620 
621  Overflowed = false;
622 
623  // Log2(Z) would be either Log2Z or Log2Z + 1.
624  // Special case: if X or Y is 0, Log2_64 gives -1, and Log2Z
625  // will necessarily be less than Log2Max as desired.
626  int Log2Z = Log2_64(X) + Log2_64(Y);
627  const T Max = std::numeric_limits<T>::max();
628  int Log2Max = Log2_64(Max);
629  if (Log2Z < Log2Max) {
630    return X * Y;
631  }
632  if (Log2Z > Log2Max) {
633    Overflowed = true;
634    return Max;
635  }
636 
637  // We're going to use the top bit, and maybe overflow one
638  // bit past it. Multiply all but the bottom bit then add
639  // that on at the end.
640  T Z = (X >> 1) * Y;
641  if (Z & ~(Max >> 1)) {
642    Overflowed = true;
643    return Max;
644  }
645  Z <<= 1;
646  if (X & 1)
647    return SaturatingAdd(Z, Y, ResultOverflowed);
648 
649  return Z;
650}
651 
652/// Multiply two unsigned integers, X and Y, and add the unsigned integer, A to
653/// the product. Clamp the result to the maximum representable value of T on
654/// overflow. ResultOverflowed indicates if the result is larger than the
655/// maximum representable value of type T.
656template <typename T>
657std::enable_if_t<std::is_unsigned_v<T>, T>
658SaturatingMultiplyAdd(T X, T Y, T A, bool *ResultOverflowed = nullptr) {
659  bool Dummy;
660  bool &Overflowed = ResultOverflowed ? *ResultOverflowed : Dummy;
661 
662  T Product = SaturatingMultiply(X, Y, &Overflowed);
663  if (Overflowed)
664    return Product;
665 
666  return SaturatingAdd(A, Product, &Overflowed);
667}
668 
669/// Use this rather than HUGE_VALF; the latter causes warnings on MSVC.
670extern const float huge_valf;
671 
672 
673/// Add two signed integers, computing the two's complement truncated result,
674/// returning true if overflow occurred.
675template <typename T>
676std::enable_if_t<std::is_signed_v<T>, T> AddOverflow(T X, T Y, T &Result) {
677#if __has_builtin(__builtin_add_overflow)1
678  return __builtin_add_overflow(X, Y, &Result);
679#else
680  // Perform the unsigned addition.
681  using U = std::make_unsigned_t<T>;
682  const U UX = static_cast<U>(X);
683  const U UY = static_cast<U>(Y);
684  const U UResult = UX + UY;
685 
686  // Convert to signed.
687  Result = static_cast<T>(UResult);
688 
689  // Adding two positive numbers should result in a positive number.
690  if (X > 0 && Y > 0)
691    return Result <= 0;
692  // Adding two negatives should result in a negative number.
693  if (X < 0 && Y < 0)
694    return Result >= 0;
695  return false;
696#endif
697}
698 
699/// Subtract two signed integers, computing the two's complement truncated
700/// result, returning true if an overflow ocurred.
701template <typename T>
702std::enable_if_t<std::is_signed_v<T>, T> SubOverflow(T X, T Y, T &Result) {
703#if __has_builtin(__builtin_sub_overflow)1
704  return __builtin_sub_overflow(X, Y, &Result);
705#else
706  // Perform the unsigned addition.
707  using U = std::make_unsigned_t<T>;
708  const U UX = static_cast<U>(X);
709  const U UY = static_cast<U>(Y);
710  const U UResult = UX - UY;
711 
712  // Convert to signed.
713  Result = static_cast<T>(UResult);
714 
715  // Subtracting a positive number from a negative results in a negative number.
716  if (X <= 0 && Y > 0)
717    return Result >= 0;
718  // Subtracting a negative number from a positive results in a positive number.
719  if (X >= 0 && Y < 0)
720    return Result <= 0;
721  return false;
722#endif
723}
724 
725/// Multiply two signed integers, computing the two's complement truncated
726/// result, returning true if an overflow ocurred.
727template <typename T>
728std::enable_if_t<std::is_signed_v<T>, T> MulOverflow(T X, T Y, T &Result) {
729  // Perform the unsigned multiplication on absolute values.
730  using U = std::make_unsigned_t<T>;
731  const U UX = X < 0 ? (0 - static_cast<U>(X)) : static_cast<U>(X);
732  const U UY = Y < 0 ? (0 - static_cast<U>(Y)) : static_cast<U>(Y);
733  const U UResult = UX * UY;
734 
735  // Convert to signed.
736  const bool IsNegative = (X < 0) ^ (Y < 0);
737  Result = IsNegative ? (0 - UResult) : UResult;
738 
739  // If any of the args was 0, result is 0 and no overflow occurs.
740  if (UX == 0 || UY == 0)
741    return false;
742 
743  // UX and UY are in [1, 2^n], where n is the number of digits.
744  // Check how the max allowed absolute value (2^n for negative, 2^(n-1) for
745  // positive) divided by an argument compares to the other.
746  if (IsNegative)
747    return UX > (static_cast<U>(std::numeric_limits<T>::max()) + U(1)) / UY;
748  else
749    return UX > (static_cast<U>(std::numeric_limits<T>::max())) / UY;
750}
751 
752} // End llvm namespace
753 
754#endif