AArch64AsmParser.cpp

Go to the documentation of this file.

//==- AArch64AsmParser.cpp - Parse AArch64 assembly to MCInst instructions -==//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
 
#include "AArch64InstrInfo.h"
#include "MCTargetDesc/AArch64AddressingModes.h"
#include "MCTargetDesc/AArch64InstPrinter.h"
#include "MCTargetDesc/AArch64MCExpr.h"
#include "MCTargetDesc/AArch64MCTargetDesc.h"
#include "MCTargetDesc/AArch64TargetStreamer.h"
#include "TargetInfo/AArch64TargetInfo.h"
#include "Utils/AArch64BaseInfo.h"
#include "llvm/ADT/APFloat.h"
#include "llvm/ADT/APInt.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SmallSet.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/StringExtras.h"
#include "llvm/ADT/StringMap.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/ADT/StringSwitch.h"
#include "llvm/ADT/Twine.h"
#include "llvm/MC/MCContext.h"
#include "llvm/MC/MCExpr.h"
#include "llvm/MC/MCInst.h"
#include "llvm/MC/MCLinkerOptimizationHint.h"
#include "llvm/MC/MCObjectFileInfo.h"
#include "llvm/MC/MCParser/MCAsmLexer.h"
#include "llvm/MC/MCParser/MCAsmParser.h"
#include "llvm/MC/MCParser/MCAsmParserExtension.h"
#include "llvm/MC/MCParser/MCParsedAsmOperand.h"
#include "llvm/MC/MCParser/MCTargetAsmParser.h"
#include "llvm/MC/MCRegisterInfo.h"
#include "llvm/MC/MCStreamer.h"
#include "llvm/MC/MCSubtargetInfo.h"
#include "llvm/MC/MCSymbol.h"
#include "llvm/MC/MCTargetOptions.h"
#include "llvm/MC/MCValue.h"
#include "llvm/MC/TargetRegistry.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/Compiler.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/MathExtras.h"
#include "llvm/Support/SMLoc.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/TargetParser/AArch64TargetParser.h"
#include "llvm/TargetParser/SubtargetFeature.h"
#include <cassert>
#include <cctype>
#include <cstdint>
#include <cstdio>
#include <optional>
#include <string>
#include <tuple>
#include <utility>
#include <vector>
 
using namespace llvm;
 
namespace {
 
enum class RegKind {
  Scalar,
  NeonVector,
  SVEDataVector,
  SVEPredicateAsCounter,
  SVEPredicateVector,
  Matrix,
  LookupTable
};
 
enum class MatrixKind { Array, Tile, Row, Col };
 
enum RegConstraintEqualityTy {
  EqualsReg,
  EqualsSuperReg,
  EqualsSubReg
};
 
class AArch64AsmParser : public MCTargetAsmParser {
private:
  StringRef Mnemonic; ///< Instruction mnemonic.
 
  // Map of register aliases registers via the .req directive.
  StringMap<std::pair<RegKind, unsigned>> RegisterReqs;
 
  class PrefixInfo {
  public:
    static PrefixInfo CreateFromInst(const MCInst &Inst, uint64_t TSFlags) {
      PrefixInfo Prefix;
      switch (Inst.getOpcode()) {
      case AArch64::MOVPRFX_ZZ:
        Prefix.Active = true;
        Prefix.Dst = Inst.getOperand(0).getReg();
        break;
      case AArch64::MOVPRFX_ZPmZ_B:
      case AArch64::MOVPRFX_ZPmZ_H:
      case AArch64::MOVPRFX_ZPmZ_S:
      case AArch64::MOVPRFX_ZPmZ_D:
        Prefix.Active = true;
        Prefix.Predicated = true;
        Prefix.ElementSize = TSFlags & AArch64::ElementSizeMask;
        assert(Prefix.ElementSize != AArch64::ElementSizeNone &&
               "No destructive element size set for movprfx");
        Prefix.Dst = Inst.getOperand(0).getReg();
        Prefix.Pg = Inst.getOperand(2).getReg();
        break;
      case AArch64::MOVPRFX_ZPzZ_B:
      case AArch64::MOVPRFX_ZPzZ_H:
      case AArch64::MOVPRFX_ZPzZ_S:
      case AArch64::MOVPRFX_ZPzZ_D:
        Prefix.Active = true;
        Prefix.Predicated = true;
        Prefix.ElementSize = TSFlags & AArch64::ElementSizeMask;
        assert(Prefix.ElementSize != AArch64::ElementSizeNone &&
               "No destructive element size set for movprfx");
        Prefix.Dst = Inst.getOperand(0).getReg();
        Prefix.Pg = Inst.getOperand(1).getReg();
        break;
      default:
        break;
      }
 
      return Prefix;
    }
 
    PrefixInfo() = default;
    bool isActive() const { return Active; }
    bool isPredicated() const { return Predicated; }
    unsigned getElementSize() const {
      assert(Predicated);
      return ElementSize;
    }
    MCRegister getDstReg() const { return Dst; }
    MCRegister getPgReg() const {
      assert(Predicated);
      return Pg;
    }
 
  private:
    bool Active = false;
    bool Predicated = false;
    unsigned ElementSize;
    MCRegister Dst;
    MCRegister Pg;
  } NextPrefix;
 
  AArch64TargetStreamer &getTargetStreamer() {
    MCTargetStreamer &TS = *getParser().getStreamer().getTargetStreamer();
    return static_cast<AArch64TargetStreamer &>(TS);
  }
 
  SMLoc getLoc() const { return getParser().getTok().getLoc(); }
 
  bool parseSysAlias(StringRef Name, SMLoc NameLoc, OperandVector &Operands);
  bool parseSyspAlias(StringRef Name, SMLoc NameLoc, OperandVector &Operands);
  void createSysAlias(uint16_t Encoding, OperandVector &Operands, SMLoc S);
  AArch64CC::CondCode parseCondCodeString(StringRef Cond,
                                          std::string &Suggestion);
  bool parseCondCode(OperandVector &Operands, bool invertCondCode);
  unsigned matchRegisterNameAlias(StringRef Name, RegKind Kind);
  bool parseRegister(OperandVector &Operands);
  bool parseSymbolicImmVal(const MCExpr *&ImmVal);
  bool parseNeonVectorList(OperandVector &Operands);
  bool parseOptionalMulOperand(OperandVector &Operands);
  bool parseOptionalVGOperand(OperandVector &Operands, StringRef &VecGroup);
  bool parseKeywordOperand(OperandVector &Operands);
  bool parseOperand(OperandVector &Operands, bool isCondCode,
                    bool invertCondCode);
  bool parseImmExpr(int64_t &Out);
  bool parseComma();
  bool parseRegisterInRange(unsigned &Out, unsigned Base, unsigned First,
                            unsigned Last);
 
  bool showMatchError(SMLoc Loc, unsigned ErrCode, uint64_t ErrorInfo,
                      OperandVector &Operands);
 
  bool parseAuthExpr(const MCExpr *&Res, SMLoc &EndLoc);
 
  bool parseDirectiveArch(SMLoc L);
  bool parseDirectiveArchExtension(SMLoc L);
  bool parseDirectiveCPU(SMLoc L);
  bool parseDirectiveInst(SMLoc L);
 
  bool parseDirectiveTLSDescCall(SMLoc L);
 
  bool parseDirectiveLOH(StringRef LOH, SMLoc L);
  bool parseDirectiveLtorg(SMLoc L);
 
  bool parseDirectiveReq(StringRef Name, SMLoc L);
  bool parseDirectiveUnreq(SMLoc L);
  bool parseDirectiveCFINegateRAState();
  bool parseDirectiveCFINegateRAStateWithPC();
  bool parseDirectiveCFIBKeyFrame();
  bool parseDirectiveCFIMTETaggedFrame();
 
  bool parseDirectiveVariantPCS(SMLoc L);
 
  bool parseDirectiveSEHAllocStack(SMLoc L);
  bool parseDirectiveSEHPrologEnd(SMLoc L);
  bool parseDirectiveSEHSaveR19R20X(SMLoc L);
  bool parseDirectiveSEHSaveFPLR(SMLoc L);
  bool parseDirectiveSEHSaveFPLRX(SMLoc L);
  bool parseDirectiveSEHSaveReg(SMLoc L);
  bool parseDirectiveSEHSaveRegX(SMLoc L);
  bool parseDirectiveSEHSaveRegP(SMLoc L);
  bool parseDirectiveSEHSaveRegPX(SMLoc L);
  bool parseDirectiveSEHSaveLRPair(SMLoc L);
  bool parseDirectiveSEHSaveFReg(SMLoc L);
  bool parseDirectiveSEHSaveFRegX(SMLoc L);
  bool parseDirectiveSEHSaveFRegP(SMLoc L);
  bool parseDirectiveSEHSaveFRegPX(SMLoc L);
  bool parseDirectiveSEHSetFP(SMLoc L);
  bool parseDirectiveSEHAddFP(SMLoc L);
  bool parseDirectiveSEHNop(SMLoc L);
  bool parseDirectiveSEHSaveNext(SMLoc L);
  bool parseDirectiveSEHEpilogStart(SMLoc L);
  bool parseDirectiveSEHEpilogEnd(SMLoc L);
  bool parseDirectiveSEHTrapFrame(SMLoc L);
  bool parseDirectiveSEHMachineFrame(SMLoc L);
  bool parseDirectiveSEHContext(SMLoc L);
  bool parseDirectiveSEHECContext(SMLoc L);
  bool parseDirectiveSEHClearUnwoundToCall(SMLoc L);
  bool parseDirectiveSEHPACSignLR(SMLoc L);
  bool parseDirectiveSEHSaveAnyReg(SMLoc L, bool Paired, bool Writeback);
 
  bool validateInstruction(MCInst &Inst, SMLoc &IDLoc,
                           SmallVectorImpl<SMLoc> &Loc);
  unsigned getNumRegsForRegKind(RegKind K);
  bool matchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
                               OperandVector &Operands, MCStreamer &Out,
                               uint64_t &ErrorInfo,
                               bool MatchingInlineAsm) override;
  /// @name Auto-generated Match Functions
  /// {
 
#define GET_ASSEMBLER_HEADER
#include "AArch64GenAsmMatcher.inc"
 
  /// }
 
  ParseStatus tryParseScalarRegister(MCRegister &Reg);
  ParseStatus tryParseVectorRegister(MCRegister &Reg, StringRef &Kind,
                                     RegKind MatchKind);
  ParseStatus tryParseMatrixRegister(OperandVector &Operands);
  ParseStatus tryParseSVCR(OperandVector &Operands);
  ParseStatus tryParseOptionalShiftExtend(OperandVector &Operands);
  ParseStatus tryParseBarrierOperand(OperandVector &Operands);
  ParseStatus tryParseBarriernXSOperand(OperandVector &Operands);
  ParseStatus tryParseSysReg(OperandVector &Operands);
  ParseStatus tryParseSysCROperand(OperandVector &Operands);
  template <bool IsSVEPrefetch = false>
  ParseStatus tryParsePrefetch(OperandVector &Operands);
  ParseStatus tryParseRPRFMOperand(OperandVector &Operands);
  ParseStatus tryParsePSBHint(OperandVector &Operands);
  ParseStatus tryParseBTIHint(OperandVector &Operands);
  ParseStatus tryParseAdrpLabel(OperandVector &Operands);
  ParseStatus tryParseAdrLabel(OperandVector &Operands);
  template <bool AddFPZeroAsLiteral>
  ParseStatus tryParseFPImm(OperandVector &Operands);
  ParseStatus tryParseImmWithOptionalShift(OperandVector &Operands);
  ParseStatus tryParseGPR64sp0Operand(OperandVector &Operands);
  bool tryParseNeonVectorRegister(OperandVector &Operands);
  ParseStatus tryParseVectorIndex(OperandVector &Operands);
  ParseStatus tryParseGPRSeqPair(OperandVector &Operands);
  ParseStatus tryParseSyspXzrPair(OperandVector &Operands);
  template <bool ParseShiftExtend,
            RegConstraintEqualityTy EqTy = RegConstraintEqualityTy::EqualsReg>
  ParseStatus tryParseGPROperand(OperandVector &Operands);
  ParseStatus tryParseZTOperand(OperandVector &Operands);
  template <bool ParseShiftExtend, bool ParseSuffix>
  ParseStatus tryParseSVEDataVector(OperandVector &Operands);
  template <RegKind RK>
  ParseStatus tryParseSVEPredicateVector(OperandVector &Operands);
  ParseStatus
  tryParseSVEPredicateOrPredicateAsCounterVector(OperandVector &Operands);
  template <RegKind VectorKind>
  ParseStatus tryParseVectorList(OperandVector &Operands,
                                 bool ExpectMatch = false);
  ParseStatus tryParseMatrixTileList(OperandVector &Operands);
  ParseStatus tryParseSVEPattern(OperandVector &Operands);
  ParseStatus tryParseSVEVecLenSpecifier(OperandVector &Operands);
  ParseStatus tryParseGPR64x8(OperandVector &Operands);
  ParseStatus tryParseImmRange(OperandVector &Operands);
  template <int> ParseStatus tryParseAdjImm0_63(OperandVector &Operands);
  ParseStatus tryParsePHintInstOperand(OperandVector &Operands);
 
public:
  enum AArch64MatchResultTy {
    Match_InvalidSuffix = FIRST_TARGET_MATCH_RESULT_TY,
#define GET_OPERAND_DIAGNOSTIC_TYPES
#include "AArch64GenAsmMatcher.inc"
  };
  bool IsILP32;
  bool IsWindowsArm64EC;
 
  AArch64AsmParser(const MCSubtargetInfo &STI, MCAsmParser &Parser,
                   const MCInstrInfo &MII, const MCTargetOptions &Options)
    : MCTargetAsmParser(Options, STI, MII) {
    IsILP32 = STI.getTargetTriple().getEnvironment() == Triple::GNUILP32;
    IsWindowsArm64EC = STI.getTargetTriple().isWindowsArm64EC();
    MCAsmParserExtension::Initialize(Parser);
    MCStreamer &S = getParser().getStreamer();
    if (S.getTargetStreamer() == nullptr)
      new AArch64TargetStreamer(S);
 
    // Alias .hword/.word/.[dx]word to the target-independent
    // .2byte/.4byte/.8byte directives as they have the same form and
    // semantics:
    ///  ::= (.hword | .word | .dword | .xword ) [ expression (, expression)* ]
    Parser.addAliasForDirective(".hword", ".2byte");
    Parser.addAliasForDirective(".word", ".4byte");
    Parser.addAliasForDirective(".dword", ".8byte");
    Parser.addAliasForDirective(".xword", ".8byte");
 
    // Initialize the set of available features.
    setAvailableFeatures(ComputeAvailableFeatures(getSTI().getFeatureBits()));
  }
 
  bool areEqualRegs(const MCParsedAsmOperand &Op1,
                    const MCParsedAsmOperand &Op2) const override;
  bool parseInstruction(ParseInstructionInfo &Info, StringRef Name,
                        SMLoc NameLoc, OperandVector &Operands) override;
  bool parseRegister(MCRegister &Reg, SMLoc &StartLoc, SMLoc &EndLoc) override;
  ParseStatus tryParseRegister(MCRegister &Reg, SMLoc &StartLoc,
                               SMLoc &EndLoc) override;
  bool ParseDirective(AsmToken DirectiveID) override;
  unsigned validateTargetOperandClass(MCParsedAsmOperand &Op,
                                      unsigned Kind) override;
 
  bool parsePrimaryExpr(const MCExpr *&Res, SMLoc &EndLoc) override;
 
  static bool classifySymbolRef(const MCExpr *Expr,
                                AArch64MCExpr::VariantKind &ELFRefKind,
                                MCSymbolRefExpr::VariantKind &DarwinRefKind,
                                int64_t &Addend);
};
 
/// AArch64Operand - Instances of this class represent a parsed AArch64 machine
/// instruction.
class AArch64Operand : public MCParsedAsmOperand {
private:
  enum KindTy {
    k_Immediate,
    k_ShiftedImm,
    k_ImmRange,
    k_CondCode,
    k_Register,
    k_MatrixRegister,
    k_MatrixTileList,
    k_SVCR,
    k_VectorList,
    k_VectorIndex,
    k_Token,
    k_SysReg,
    k_SysCR,
    k_Prefetch,
    k_ShiftExtend,
    k_FPImm,
    k_Barrier,
    k_PSBHint,
    k_PHint,
    k_BTIHint,
  } Kind;
 
  SMLoc StartLoc, EndLoc;
 
  struct TokOp {
    const char *Data;
    unsigned Length;
    bool IsSuffix; // Is the operand actually a suffix on the mnemonic.
  };
 
  // Separate shift/extend operand.
  struct ShiftExtendOp {
    AArch64_AM::ShiftExtendType Type;
    unsigned Amount;
    bool HasExplicitAmount;
  };
 
  struct RegOp {
    unsigned RegNum;
    RegKind Kind;
    int ElementWidth;
 
    // The register may be allowed as a different register class,
    // e.g. for GPR64as32 or GPR32as64.
    RegConstraintEqualityTy EqualityTy;
 
    // In some cases the shift/extend needs to be explicitly parsed together
    // with the register, rather than as a separate operand. This is needed
    // for addressing modes where the instruction as a whole dictates the
    // scaling/extend, rather than specific bits in the instruction.
    // By parsing them as a single operand, we avoid the need to pass an
    // extra operand in all CodeGen patterns (because all operands need to
    // have an associated value), and we avoid the need to update TableGen to
    // accept operands that have no associated bits in the instruction.
    //
    // An added benefit of parsing them together is that the assembler
    // can give a sensible diagnostic if the scaling is not correct.
    //
    // The default is 'lsl #0' (HasExplicitAmount = false) if no
    // ShiftExtend is specified.
    ShiftExtendOp ShiftExtend;
  };
 
  struct MatrixRegOp {
    unsigned RegNum;
    unsigned ElementWidth;
    MatrixKind Kind;
  };
 
  struct MatrixTileListOp {
    unsigned RegMask = 0;
  };
 
  struct VectorListOp {
    unsigned RegNum;
    unsigned Count;
    unsigned Stride;
    unsigned NumElements;
    unsigned ElementWidth;
    RegKind  RegisterKind;
  };
 
  struct VectorIndexOp {
    int Val;
  };
 
  struct ImmOp {
    const MCExpr *Val;
  };
 
  struct ShiftedImmOp {
    const MCExpr *Val;
    unsigned ShiftAmount;
  };
 
  struct ImmRangeOp {
    unsigned First;
    unsigned Last;
  };
 
  struct CondCodeOp {
    AArch64CC::CondCode Code;
  };
 
  struct FPImmOp {
    uint64_t Val; // APFloat value bitcasted to uint64_t.
    bool IsExact; // describes whether parsed value was exact.
  };
 
  struct BarrierOp {
    const char *Data;
    unsigned Length;
    unsigned Val; // Not the enum since not all values have names.
    bool HasnXSModifier;
  };
 
  struct SysRegOp {
    const char *Data;
    unsigned Length;
    uint32_t MRSReg;
    uint32_t MSRReg;
    uint32_t PStateField;
  };
 
  struct SysCRImmOp {
    unsigned Val;
  };
 
  struct PrefetchOp {
    const char *Data;
    unsigned Length;
    unsigned Val;
  };
 
  struct PSBHintOp {
    const char *Data;
    unsigned Length;
    unsigned Val;
  };
  struct PHintOp {
    const char *Data;
    unsigned Length;
    unsigned Val;
  };
  struct BTIHintOp {
    const char *Data;
    unsigned Length;
    unsigned Val;
  };
 
  struct SVCROp {
    const char *Data;
    unsigned Length;
    unsigned PStateField;
  };
 
  union {
    struct TokOp Tok;
    struct RegOp Reg;
    struct MatrixRegOp MatrixReg;
    struct MatrixTileListOp MatrixTileList;
    struct VectorListOp VectorList;
    struct VectorIndexOp VectorIndex;
    struct ImmOp Imm;
    struct ShiftedImmOp ShiftedImm;
    struct ImmRangeOp ImmRange;
    struct CondCodeOp CondCode;
    struct FPImmOp FPImm;
    struct BarrierOp Barrier;
    struct SysRegOp SysReg;
    struct SysCRImmOp SysCRImm;
    struct PrefetchOp Prefetch;
    struct PSBHintOp PSBHint;
    struct PHintOp PHint;
    struct BTIHintOp BTIHint;
    struct ShiftExtendOp ShiftExtend;
    struct SVCROp SVCR;
  };
 
  // Keep the MCContext around as the MCExprs may need manipulated during
  // the add<>Operands() calls.
  MCContext &Ctx;
 
public:
  AArch64Operand(KindTy K, MCContext &Ctx) : Kind(K), Ctx(Ctx) {}
 
  AArch64Operand(const AArch64Operand &o) : MCParsedAsmOperand(), Ctx(o.Ctx) {
    Kind = o.Kind;
    StartLoc = o.StartLoc;
    EndLoc = o.EndLoc;
    switch (Kind) {
    case k_Token:
      Tok = o.Tok;
      break;
    case k_Immediate:
      Imm = o.Imm;
      break;
    case k_ShiftedImm:
      ShiftedImm = o.ShiftedImm;
      break;
    case k_ImmRange:
      ImmRange = o.ImmRange;
      break;
    case k_CondCode:
      CondCode = o.CondCode;
      break;
    case k_FPImm:
      FPImm = o.FPImm;
      break;
    case k_Barrier:
      Barrier = o.Barrier;
      break;
    case k_Register:
      Reg = o.Reg;
      break;
    case k_MatrixRegister:
      MatrixReg = o.MatrixReg;
      break;
    case k_MatrixTileList:
      MatrixTileList = o.MatrixTileList;
      break;
    case k_VectorList:
      VectorList = o.VectorList;
      break;
    case k_VectorIndex:
      VectorIndex = o.VectorIndex;
      break;
    case k_SysReg:
      SysReg = o.SysReg;
      break;
    case k_SysCR:
      SysCRImm = o.SysCRImm;
      break;
    case k_Prefetch:
      Prefetch = o.Prefetch;
      break;
    case k_PSBHint:
      PSBHint = o.PSBHint;
      break;
    case k_PHint:
      PHint = o.PHint;
      break;
    case k_BTIHint:
      BTIHint = o.BTIHint;
      break;
    case k_ShiftExtend:
      ShiftExtend = o.ShiftExtend;
      break;
    case k_SVCR:
      SVCR = o.SVCR;
      break;
    }
  }
 
  /// getStartLoc - Get the location of the first token of this operand.
  SMLoc getStartLoc() const override { return StartLoc; }
  /// getEndLoc - Get the location of the last token of this operand.
  SMLoc getEndLoc() const override { return EndLoc; }
 
  StringRef getToken() const {
    assert(Kind == k_Token && "Invalid access!");
    return StringRef(Tok.Data, Tok.Length);
  }
 
  bool isTokenSuffix() const {
    assert(Kind == k_Token && "Invalid access!");
    return Tok.IsSuffix;
  }
 
  const MCExpr *getImm() const {
    assert(Kind == k_Immediate && "Invalid access!");
    return Imm.Val;
  }
 
  const MCExpr *getShiftedImmVal() const {
    assert(Kind == k_ShiftedImm && "Invalid access!");
    return ShiftedImm.Val;
  }
 
  unsigned getShiftedImmShift() const {
    assert(Kind == k_ShiftedImm && "Invalid access!");
    return ShiftedImm.ShiftAmount;
  }
 
  unsigned getFirstImmVal() const {
    assert(Kind == k_ImmRange && "Invalid access!");
    return ImmRange.First;
  }
 
  unsigned getLastImmVal() const {
    assert(Kind == k_ImmRange && "Invalid access!");
    return ImmRange.Last;
  }
 
  AArch64CC::CondCode getCondCode() const {
    assert(Kind == k_CondCode && "Invalid access!");
    return CondCode.Code;
  }
 
  APFloat getFPImm() const {
    assert (Kind == k_FPImm && "Invalid access!");
    return APFloat(APFloat::IEEEdouble(), APInt(64, FPImm.Val, true));
  }
 
  bool getFPImmIsExact() const {
    assert (Kind == k_FPImm && "Invalid access!");
    return FPImm.IsExact;
  }
 
  unsigned getBarrier() const {
    assert(Kind == k_Barrier && "Invalid access!");
    return Barrier.Val;
  }
 
  StringRef getBarrierName() const {
    assert(Kind == k_Barrier && "Invalid access!");
    return StringRef(Barrier.Data, Barrier.Length);
  }
 
  bool getBarriernXSModifier() const {
    assert(Kind == k_Barrier && "Invalid access!");
    return Barrier.HasnXSModifier;
  }
 
  MCRegister getReg() const override {
    assert(Kind == k_Register && "Invalid access!");
    return Reg.RegNum;
  }
 
  unsigned getMatrixReg() const {
    assert(Kind == k_MatrixRegister && "Invalid access!");
    return MatrixReg.RegNum;
  }
 
  unsigned getMatrixElementWidth() const {
    assert(Kind == k_MatrixRegister && "Invalid access!");
    return MatrixReg.ElementWidth;
  }
 
  MatrixKind getMatrixKind() const {
    assert(Kind == k_MatrixRegister && "Invalid access!");
    return MatrixReg.Kind;
  }
 
  unsigned getMatrixTileListRegMask() const {
    assert(isMatrixTileList() && "Invalid access!");
    return MatrixTileList.RegMask;
  }
 
  RegConstraintEqualityTy getRegEqualityTy() const {
    assert(Kind == k_Register && "Invalid access!");
    return Reg.EqualityTy;
  }
 
  unsigned getVectorListStart() const {
    assert(Kind == k_VectorList && "Invalid access!");
    return VectorList.RegNum;
  }
 
  unsigned getVectorListCount() const {
    assert(Kind == k_VectorList && "Invalid access!");
    return VectorList.Count;
  }
 
  unsigned getVectorListStride() const {
    assert(Kind == k_VectorList && "Invalid access!");
    return VectorList.Stride;
  }
 
  int getVectorIndex() const {
    assert(Kind == k_VectorIndex && "Invalid access!");
    return VectorIndex.Val;
  }
 
  StringRef getSysReg() const {
    assert(Kind == k_SysReg && "Invalid access!");
    return StringRef(SysReg.Data, SysReg.Length);
  }
 
  unsigned getSysCR() const {
    assert(Kind == k_SysCR && "Invalid access!");
    return SysCRImm.Val;
  }
 
  unsigned getPrefetch() const {
    assert(Kind == k_Prefetch && "Invalid access!");
    return Prefetch.Val;
  }
 
  unsigned getPSBHint() const {
    assert(Kind == k_PSBHint && "Invalid access!");
    return PSBHint.Val;
  }
 
  unsigned getPHint() const {
    assert(Kind == k_PHint && "Invalid access!");
    return PHint.Val;
  }
 
  StringRef getPSBHintName() const {
    assert(Kind == k_PSBHint && "Invalid access!");
    return StringRef(PSBHint.Data, PSBHint.Length);
  }
 
  StringRef getPHintName() const {
    assert(Kind == k_PHint && "Invalid access!");
    return StringRef(PHint.Data, PHint.Length);
  }
 
  unsigned getBTIHint() const {
    assert(Kind == k_BTIHint && "Invalid access!");
    return BTIHint.Val;
  }
 
  StringRef getBTIHintName() const {
    assert(Kind == k_BTIHint && "Invalid access!");
    return StringRef(BTIHint.Data, BTIHint.Length);
  }
 
  StringRef getSVCR() const {
    assert(Kind == k_SVCR && "Invalid access!");
    return StringRef(SVCR.Data, SVCR.Length);
  }
 
  StringRef getPrefetchName() const {
    assert(Kind == k_Prefetch && "Invalid access!");
    return StringRef(Prefetch.Data, Prefetch.Length);
  }
 
  AArch64_AM::ShiftExtendType getShiftExtendType() const {
    if (Kind == k_ShiftExtend)
      return ShiftExtend.Type;
    if (Kind == k_Register)
      return Reg.ShiftExtend.Type;
    llvm_unreachable("Invalid access!");
  }
 
  unsigned getShiftExtendAmount() const {
    if (Kind == k_ShiftExtend)
      return ShiftExtend.Amount;
    if (Kind == k_Register)
      return Reg.ShiftExtend.Amount;
    llvm_unreachable("Invalid access!");
  }
 
  bool hasShiftExtendAmount() const {
    if (Kind == k_ShiftExtend)
      return ShiftExtend.HasExplicitAmount;
    if (Kind == k_Register)
      return Reg.ShiftExtend.HasExplicitAmount;
    llvm_unreachable("Invalid access!");
  }
 
  bool isImm() const override { return Kind == k_Immediate; }
  bool isMem() const override { return false; }
 
  bool isUImm6() const {
    if (!isImm())
      return false;
    const MCConstantExpr *MCE = dyn_cast<MCConstantExpr>(getImm());
    if (!MCE)
      return false;
    int64_t Val = MCE->getValue();
    return (Val >= 0 && Val < 64);
  }
 
  template <int Width> bool isSImm() const { return isSImmScaled<Width, 1>(); }
 
  template <int Bits, int Scale> DiagnosticPredicate isSImmScaled() const {
    return isImmScaled<Bits, Scale>(true);
  }
 
  template <int Bits, int Scale, int Offset = 0, bool IsRange = false>
  DiagnosticPredicate isUImmScaled() const {
    if (IsRange && isImmRange() &&
        (getLastImmVal() != getFirstImmVal() + Offset))
      return DiagnosticPredicateTy::NoMatch;
 
    return isImmScaled<Bits, Scale, IsRange>(false);
  }
 
  template <int Bits, int Scale, bool IsRange = false>
  DiagnosticPredicate isImmScaled(bool Signed) const {
    if ((!isImm() && !isImmRange()) || (isImm() && IsRange) ||
        (isImmRange() && !IsRange))
      return DiagnosticPredicateTy::NoMatch;
 
    int64_t Val;
    if (isImmRange())
      Val = getFirstImmVal();
    else {
      const MCConstantExpr *MCE = dyn_cast<MCConstantExpr>(getImm());
      if (!MCE)
        return DiagnosticPredicateTy::NoMatch;
      Val = MCE->getValue();
    }
 
    int64_t MinVal, MaxVal;
    if (Signed) {
      int64_t Shift = Bits - 1;
      MinVal = (int64_t(1) << Shift) * -Scale;
      MaxVal = ((int64_t(1) << Shift) - 1) * Scale;
    } else {
      MinVal = 0;
      MaxVal = ((int64_t(1) << Bits) - 1) * Scale;
    }
 
    if (Val >= MinVal && Val <= MaxVal && (Val % Scale) == 0)
      return DiagnosticPredicateTy::Match;
 
    return DiagnosticPredicateTy::NearMatch;
  }
 
  DiagnosticPredicate isSVEPattern() const {
    if (!isImm())
      return DiagnosticPredicateTy::NoMatch;
    auto *MCE = dyn_cast<MCConstantExpr>(getImm());
    if (!MCE)
      return DiagnosticPredicateTy::NoMatch;
    int64_t Val = MCE->getValue();
    if (Val >= 0 && Val < 32)
      return DiagnosticPredicateTy::Match;
    return DiagnosticPredicateTy::NearMatch;
  }
 
  DiagnosticPredicate isSVEVecLenSpecifier() const {
    if (!isImm())
      return DiagnosticPredicateTy::NoMatch;
    auto *MCE = dyn_cast<MCConstantExpr>(getImm());
    if (!MCE)
      return DiagnosticPredicateTy::NoMatch;
    int64_t Val = MCE->getValue();
    if (Val >= 0 && Val <= 1)
      return DiagnosticPredicateTy::Match;
    return DiagnosticPredicateTy::NearMatch;
  }
 
  bool isSymbolicUImm12Offset(const MCExpr *Expr) const {
    AArch64MCExpr::VariantKind ELFRefKind;
    MCSymbolRefExpr::VariantKind DarwinRefKind;
    int64_t Addend;
    if (!AArch64AsmParser::classifySymbolRef(Expr, ELFRefKind, DarwinRefKind,
                                           Addend)) {
      // If we don't understand the expression, assume the best and
      // let the fixup and relocation code deal with it.
      return true;
    }
 
    if (DarwinRefKind == MCSymbolRefExpr::VK_PAGEOFF ||
        ELFRefKind == AArch64MCExpr::VK_LO12 ||
        ELFRefKind == AArch64MCExpr::VK_GOT_LO12 ||
        ELFRefKind == AArch64MCExpr::VK_GOT_AUTH_LO12 ||
        ELFRefKind == AArch64MCExpr::VK_DTPREL_LO12 ||
        ELFRefKind == AArch64MCExpr::VK_DTPREL_LO12_NC ||
        ELFRefKind == AArch64MCExpr::VK_TPREL_LO12 ||
        ELFRefKind == AArch64MCExpr::VK_TPREL_LO12_NC ||
        ELFRefKind == AArch64MCExpr::VK_GOTTPREL_LO12_NC ||
        ELFRefKind == AArch64MCExpr::VK_TLSDESC_LO12 ||
        ELFRefKind == AArch64MCExpr::VK_TLSDESC_AUTH_LO12 ||
        ELFRefKind == AArch64MCExpr::VK_SECREL_LO12 ||
        ELFRefKind == AArch64MCExpr::VK_SECREL_HI12 ||
        ELFRefKind == AArch64MCExpr::VK_GOT_PAGE_LO15) {
      // Note that we don't range-check the addend. It's adjusted modulo page
      // size when converted, so there is no "out of range" condition when using
      // @pageoff.
      return true;
    } else if (DarwinRefKind == MCSymbolRefExpr::VK_GOTPAGEOFF ||
               DarwinRefKind == MCSymbolRefExpr::VK_TLVPPAGEOFF) {
      // @gotpageoff/@tlvppageoff can only be used directly, not with an addend.
      return Addend == 0;
    }
 
    return false;
  }
 
  template <int Scale> bool isUImm12Offset() const {
    if (!isImm())
      return false;
 
    const MCConstantExpr *MCE = dyn_cast<MCConstantExpr>(getImm());
    if (!MCE)
      return isSymbolicUImm12Offset(getImm());
 
    int64_t Val = MCE->getValue();
    return (Val % Scale) == 0 && Val >= 0 && (Val / Scale) < 0x1000;
  }
 
  template <int N, int M>
  bool isImmInRange() const {
    if (!isImm())
      return false;
    const MCConstantExpr *MCE = dyn_cast<MCConstantExpr>(getImm());
    if (!MCE)
      return false;
    int64_t Val = MCE->getValue();
    return (Val >= N && Val <= M);
  }
 
  // NOTE: Also used for isLogicalImmNot as anything that can be represented as
  // a logical immediate can always be represented when inverted.
  template <typename T>
  bool isLogicalImm() const {
    if (!isImm())
      return false;
    const MCConstantExpr *MCE = dyn_cast<MCConstantExpr>(getImm());
    if (!MCE)
      return false;
 
    int64_t Val = MCE->getValue();
    // Avoid left shift by 64 directly.
    uint64_t Upper = UINT64_C(-1) << (sizeof(T) * 4) << (sizeof(T) * 4);
    // Allow all-0 or all-1 in top bits to permit bitwise NOT.
    if ((Val & Upper) && (Val & Upper) != Upper)
      return false;
 
    return AArch64_AM::isLogicalImmediate(Val & ~Upper, sizeof(T) * 8);
  }
 
  bool isShiftedImm() const { return Kind == k_ShiftedImm; }
 
  bool isImmRange() const { return Kind == k_ImmRange; }
 
  /// Returns the immediate value as a pair of (imm, shift) if the immediate is
  /// a shifted immediate by value 'Shift' or '0', or if it is an unshifted
  /// immediate that can be shifted by 'Shift'.
  template <unsigned Width>
  std::optional<std::pair<int64_t, unsigned>> getShiftedVal() const {
    if (isShiftedImm() && Width == getShiftedImmShift())
      if (auto *CE = dyn_cast<MCConstantExpr>(getShiftedImmVal()))
        return std::make_pair(CE->getValue(), Width);
 
    if (isImm())
      if (auto *CE = dyn_cast<MCConstantExpr>(getImm())) {
        int64_t Val = CE->getValue();
        if ((Val != 0) && (uint64_t(Val >> Width) << Width) == uint64_t(Val))
          return std::make_pair(Val >> Width, Width);
        else
          return std::make_pair(Val, 0u);
      }
 
    return {};
  }
 
  bool isAddSubImm() const {
    if (!isShiftedImm() && !isImm())
      return false;
 
    const MCExpr *Expr;
 
    // An ADD/SUB shifter is either 'lsl #0' or 'lsl #12'.
    if (isShiftedImm()) {
      unsigned Shift = ShiftedImm.ShiftAmount;
      Expr = ShiftedImm.Val;
      if (Shift != 0 && Shift != 12)
        return false;
    } else {
      Expr = getImm();
    }
 
    AArch64MCExpr::VariantKind ELFRefKind;
    MCSymbolRefExpr::VariantKind DarwinRefKind;
    int64_t Addend;
    if (AArch64AsmParser::classifySymbolRef(Expr, ELFRefKind,
                                          DarwinRefKind, Addend)) {
      return DarwinRefKind == MCSymbolRefExpr::VK_PAGEOFF ||
             DarwinRefKind == MCSymbolRefExpr::VK_TLVPPAGEOFF ||
             (DarwinRefKind == MCSymbolRefExpr::VK_GOTPAGEOFF && Addend == 0) ||
             ELFRefKind == AArch64MCExpr::VK_LO12 ||
             ELFRefKind == AArch64MCExpr::VK_GOT_AUTH_LO12 ||
             ELFRefKind == AArch64MCExpr::VK_DTPREL_HI12 ||
             ELFRefKind == AArch64MCExpr::VK_DTPREL_LO12 ||
             ELFRefKind == AArch64MCExpr::VK_DTPREL_LO12_NC ||
             ELFRefKind == AArch64MCExpr::VK_TPREL_HI12 ||
             ELFRefKind == AArch64MCExpr::VK_TPREL_LO12 ||
             ELFRefKind == AArch64MCExpr::VK_TPREL_LO12_NC ||
             ELFRefKind == AArch64MCExpr::VK_TLSDESC_LO12 ||
             ELFRefKind == AArch64MCExpr::VK_TLSDESC_AUTH_LO12 ||
             ELFRefKind == AArch64MCExpr::VK_SECREL_HI12 ||
             ELFRefKind == AArch64MCExpr::VK_SECREL_LO12;
    }
 
    // If it's a constant, it should be a real immediate in range.
    if (auto ShiftedVal = getShiftedVal<12>())
      return ShiftedVal->first >= 0 && ShiftedVal->first <= 0xfff;
 
    // If it's an expression, we hope for the best and let the fixup/relocation
    // code deal with it.
    return true;
  }
 
  bool isAddSubImmNeg() const {
    if (!isShiftedImm() && !isImm())
      return false;
 
    // Otherwise it should be a real negative immediate in range.
    if (auto ShiftedVal = getShiftedVal<12>())
      return ShiftedVal->first < 0 && -ShiftedVal->first <= 0xfff;
 
    return false;
  }
 
  // Signed value in the range -128 to +127. For element widths of
  // 16 bits or higher it may also be a signed multiple of 256 in the
  // range -32768 to +32512.
  // For element-width of 8 bits a range of -128 to 255 is accepted,
  // since a copy of a byte can be either signed/unsigned.
  template <typename T>
  DiagnosticPredicate isSVECpyImm() const {
    if (!isShiftedImm() && (!isImm() || !isa<MCConstantExpr>(getImm())))
      return DiagnosticPredicateTy::NoMatch;
 
    bool IsByte = std::is_same<int8_t, std::make_signed_t<T>>::value ||
                  std::is_same<int8_t, T>::value;
    if (auto ShiftedImm = getShiftedVal<8>())
      if (!(IsByte && ShiftedImm->second) &&
          AArch64_AM::isSVECpyImm<T>(uint64_t(ShiftedImm->first)
                                     << ShiftedImm->second))
        return DiagnosticPredicateTy::Match;
 
    return DiagnosticPredicateTy::NearMatch;
  }
 
  // Unsigned value in the range 0 to 255. For element widths of
  // 16 bits or higher it may also be a signed multiple of 256 in the
  // range 0 to 65280.
  template <typename T> DiagnosticPredicate isSVEAddSubImm() const {
    if (!isShiftedImm() && (!isImm() || !isa<MCConstantExpr>(getImm())))
      return DiagnosticPredicateTy::NoMatch;
 
    bool IsByte = std::is_same<int8_t, std::make_signed_t<T>>::value ||
                  std::is_same<int8_t, T>::value;
    if (auto ShiftedImm = getShiftedVal<8>())
      if (!(IsByte && ShiftedImm->second) &&
          AArch64_AM::isSVEAddSubImm<T>(ShiftedImm->first
                                        << ShiftedImm->second))
        return DiagnosticPredicateTy::Match;
 
    return DiagnosticPredicateTy::NearMatch;
  }
 
  template <typename T> DiagnosticPredicate isSVEPreferredLogicalImm() const {
    if (isLogicalImm<T>() && !isSVECpyImm<T>())
      return DiagnosticPredicateTy::Match;
    return DiagnosticPredicateTy::NoMatch;
  }
 
  bool isCondCode() const { return Kind == k_CondCode; }
 
  bool isSIMDImmType10() const {
    if (!isImm())
      return false;
    const MCConstantExpr *MCE = dyn_cast<MCConstantExpr>(getImm());
    if (!MCE)
      return false;
    return AArch64_AM::isAdvSIMDModImmType10(MCE->getValue());
  }
 
  template<int N>
  bool isBranchTarget() const {
    if (!isImm())
      return false;
    const MCConstantExpr *MCE = dyn_cast<MCConstantExpr>(getImm());
    if (!MCE)
      return true;
    int64_t Val = MCE->getValue();
    if (Val & 0x3)
      return false;
    assert(N > 0 && "Branch target immediate cannot be 0 bits!");
    return (Val >= -((1<<(N-1)) << 2) && Val <= (((1<<(N-1))-1) << 2));
  }
 
  bool
  isMovWSymbol(ArrayRef<AArch64MCExpr::VariantKind> AllowedModifiers) const {
    if (!isImm())
      return false;
 
    AArch64MCExpr::VariantKind ELFRefKind;
    MCSymbolRefExpr::VariantKind DarwinRefKind;
    int64_t Addend;
    if (!AArch64AsmParser::classifySymbolRef(getImm(), ELFRefKind,
                                             DarwinRefKind, Addend)) {
      return false;
    }
    if (DarwinRefKind != MCSymbolRefExpr::VK_None)
      return false;
 
    return llvm::is_contained(AllowedModifiers, ELFRefKind);
  }
 
  bool isMovWSymbolG3() const {
    return isMovWSymbol({AArch64MCExpr::VK_ABS_G3, AArch64MCExpr::VK_PREL_G3});
  }
 
  bool isMovWSymbolG2() const {
    return isMovWSymbol(
        {AArch64MCExpr::VK_ABS_G2, AArch64MCExpr::VK_ABS_G2_S,
         AArch64MCExpr::VK_ABS_G2_NC, AArch64MCExpr::VK_PREL_G2,
         AArch64MCExpr::VK_PREL_G2_NC, AArch64MCExpr::VK_TPREL_G2,
         AArch64MCExpr::VK_DTPREL_G2});
  }
 
  bool isMovWSymbolG1() const {
    return isMovWSymbol(
        {AArch64MCExpr::VK_ABS_G1, AArch64MCExpr::VK_ABS_G1_S,
         AArch64MCExpr::VK_ABS_G1_NC, AArch64MCExpr::VK_PREL_G1,
         AArch64MCExpr::VK_PREL_G1_NC, AArch64MCExpr::VK_GOTTPREL_G1,
         AArch64MCExpr::VK_TPREL_G1, AArch64MCExpr::VK_TPREL_G1_NC,
         AArch64MCExpr::VK_DTPREL_G1, AArch64MCExpr::VK_DTPREL_G1_NC});
  }
 
  bool isMovWSymbolG0() const {
    return isMovWSymbol(
        {AArch64MCExpr::VK_ABS_G0, AArch64MCExpr::VK_ABS_G0_S,
         AArch64MCExpr::VK_ABS_G0_NC, AArch64MCExpr::VK_PREL_G0,
         AArch64MCExpr::VK_PREL_G0_NC, AArch64MCExpr::VK_GOTTPREL_G0_NC,
         AArch64MCExpr::VK_TPREL_G0, AArch64MCExpr::VK_TPREL_G0_NC,
         AArch64MCExpr::VK_DTPREL_G0, AArch64MCExpr::VK_DTPREL_G0_NC});
  }
 
  template<int RegWidth, int Shift>
  bool isMOVZMovAlias() const {
    if (!isImm()) return false;
 
    const MCExpr *E = getImm();
    if (const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(E)) {
      uint64_t Value = CE->getValue();
 
      return AArch64_AM::isMOVZMovAlias(Value, Shift, RegWidth);
    }
    // Only supports the case of Shift being 0 if an expression is used as an
    // operand
    return !Shift && E;
  }
 
  template<int RegWidth, int Shift>
  bool isMOVNMovAlias() const {
    if (!isImm()) return false;
 
    const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
    if (!CE) return false;
    uint64_t Value = CE->getValue();
 
    return AArch64_AM::isMOVNMovAlias(Value, Shift, RegWidth);
  }
 
  bool isFPImm() const {
    return Kind == k_FPImm &&
           AArch64_AM::getFP64Imm(getFPImm().bitcastToAPInt()) != -1;
  }
 
  bool isBarrier() const {
    return Kind == k_Barrier && !getBarriernXSModifier();
  }
  bool isBarriernXS() const {
    return Kind == k_Barrier && getBarriernXSModifier();
  }
  bool isSysReg() const { return Kind == k_SysReg; }
 
  bool isMRSSystemRegister() const {
    if (!isSysReg()) return false;
 
    return SysReg.MRSReg != -1U;
  }
 
  bool isMSRSystemRegister() const {
    if (!isSysReg()) return false;
    return SysReg.MSRReg != -1U;
  }
 
  bool isSystemPStateFieldWithImm0_1() const {
    if (!isSysReg()) return false;
    return AArch64PState::lookupPStateImm0_1ByEncoding(SysReg.PStateField);
  }
 
  bool isSystemPStateFieldWithImm0_15() const {
    if (!isSysReg())
      return false;
    return AArch64PState::lookupPStateImm0_15ByEncoding(SysReg.PStateField);
  }
 
  bool isSVCR() const {
    if (Kind != k_SVCR)
      return false;
    return SVCR.PStateField != -1U;
  }
 
  bool isReg() const override {
    return Kind == k_Register;
  }
 
  bool isVectorList() const { return Kind == k_VectorList; }
 
  bool isScalarReg() const {
    return Kind == k_Register && Reg.Kind == RegKind::Scalar;
  }
 
  bool isNeonVectorReg() const {
    return Kind == k_Register && Reg.Kind == RegKind::NeonVector;
  }
 
  bool isNeonVectorRegLo() const {
    return Kind == k_Register && Reg.Kind == RegKind::NeonVector &&
           (AArch64MCRegisterClasses[AArch64::FPR128_loRegClassID].contains(
                Reg.RegNum) ||
            AArch64MCRegisterClasses[AArch64::FPR64_loRegClassID].contains(
                Reg.RegNum));
  }
 
  bool isNeonVectorReg0to7() const {
    return Kind == k_Register && Reg.Kind == RegKind::NeonVector &&
           (AArch64MCRegisterClasses[AArch64::FPR128_0to7RegClassID].contains(
               Reg.RegNum));
  }
 
  bool isMatrix() const { return Kind == k_MatrixRegister; }
  bool isMatrixTileList() const { return Kind == k_MatrixTileList; }
 
  template <unsigned Class> bool isSVEPredicateAsCounterReg() const {
    RegKind RK;
    switch (Class) {
    case AArch64::PPRRegClassID:
    case AArch64::PPR_3bRegClassID:
    case AArch64::PPR_p8to15RegClassID:
    case AArch64::PNRRegClassID:
    case AArch64::PNR_p8to15RegClassID:
    case AArch64::PPRorPNRRegClassID:
      RK = RegKind::SVEPredicateAsCounter;
      break;
    default:
      llvm_unreachable("Unsupport register class");
    }
 
    return (Kind == k_Register && Reg.Kind == RK) &&
           AArch64MCRegisterClasses[Class].contains(getReg());
  }
 
  template <unsigned Class> bool isSVEVectorReg() const {
    RegKind RK;
    switch (Class) {
    case AArch64::ZPRRegClassID:
    case AArch64::ZPR_3bRegClassID:
    case AArch64::ZPR_4bRegClassID:
    case AArch64::ZPRMul2_LoRegClassID:
    case AArch64::ZPRMul2_HiRegClassID:
    case AArch64::ZPR_KRegClassID:
      RK = RegKind::SVEDataVector;
      break;
    case AArch64::PPRRegClassID:
    case AArch64::PPR_3bRegClassID:
    case AArch64::PPR_p8to15RegClassID:
    case AArch64::PNRRegClassID:
    case AArch64::PNR_p8to15RegClassID:
    case AArch64::PPRorPNRRegClassID:
      RK = RegKind::SVEPredicateVector;
      break;
    default:
      llvm_unreachable("Unsupport register class");
    }
 
    return (Kind == k_Register && Reg.Kind == RK) &&
           AArch64MCRegisterClasses[Class].contains(getReg());
  }
 
  template <unsigned Class> bool isFPRasZPR() const {
    return Kind == k_Register && Reg.Kind == RegKind::Scalar &&
           AArch64MCRegisterClasses[Class].contains(getReg());
  }
 
  template <int ElementWidth, unsigned Class>
  DiagnosticPredicate isSVEPredicateVectorRegOfWidth() const {
    if (Kind != k_Register || Reg.Kind != RegKind::SVEPredicateVector)
      return DiagnosticPredicateTy::NoMatch;
 
    if (isSVEVectorReg<Class>() && (Reg.ElementWidth == ElementWidth))
      return DiagnosticPredicateTy::Match;
 
    return DiagnosticPredicateTy::NearMatch;
  }
 
  template <int ElementWidth, unsigned Class>
  DiagnosticPredicate isSVEPredicateOrPredicateAsCounterRegOfWidth() const {
    if (Kind != k_Register || (Reg.Kind != RegKind::SVEPredicateAsCounter &&
                               Reg.Kind != RegKind::SVEPredicateVector))
      return DiagnosticPredicateTy::NoMatch;
 
    if ((isSVEPredicateAsCounterReg<Class>() ||
         isSVEPredicateVectorRegOfWidth<ElementWidth, Class>()) &&
        Reg.ElementWidth == ElementWidth)
      return DiagnosticPredicateTy::Match;
 
    return DiagnosticPredicateTy::NearMatch;
  }
 
  template <int ElementWidth, unsigned Class>
  DiagnosticPredicate isSVEPredicateAsCounterRegOfWidth() const {
    if (Kind != k_Register || Reg.Kind != RegKind::SVEPredicateAsCounter)
      return DiagnosticPredicateTy::NoMatch;
 
    if (isSVEPredicateAsCounterReg<Class>() && (Reg.ElementWidth == ElementWidth))
      return DiagnosticPredicateTy::Match;
 
    return DiagnosticPredicateTy::NearMatch;
  }
 
  template <int ElementWidth, unsigned Class>
  DiagnosticPredicate isSVEDataVectorRegOfWidth() const {
    if (Kind != k_Register || Reg.Kind != RegKind::SVEDataVector)
      return DiagnosticPredicateTy::NoMatch;
 
    if (isSVEVectorReg<Class>() && Reg.ElementWidth == ElementWidth)
      return DiagnosticPredicateTy::Match;
 
    return DiagnosticPredicateTy::NearMatch;
  }
 
  template <int ElementWidth, unsigned Class,
            AArch64_AM::ShiftExtendType ShiftExtendTy, int ShiftWidth,
            bool ShiftWidthAlwaysSame>
  DiagnosticPredicate isSVEDataVectorRegWithShiftExtend() const {
    auto VectorMatch = isSVEDataVectorRegOfWidth<ElementWidth, Class>();
    if (!VectorMatch.isMatch())
      return DiagnosticPredicateTy::NoMatch;
 
    // Give a more specific diagnostic when the user has explicitly typed in
    // a shift-amount that does not match what is expected, but for which
    // there is also an unscaled addressing mode (e.g. sxtw/uxtw).
    bool MatchShift = getShiftExtendAmount() == Log2_32(ShiftWidth / 8);
    if (!MatchShift && (ShiftExtendTy == AArch64_AM::UXTW ||
                        ShiftExtendTy == AArch64_AM::SXTW) &&
        !ShiftWidthAlwaysSame && hasShiftExtendAmount() && ShiftWidth == 8)
      return DiagnosticPredicateTy::NoMatch;
 
    if (MatchShift && ShiftExtendTy == getShiftExtendType())
      return DiagnosticPredicateTy::Match;
 
    return DiagnosticPredicateTy::NearMatch;
  }
 
  bool isGPR32as64() const {
    return Kind == k_Register && Reg.Kind == RegKind::Scalar &&
      AArch64MCRegisterClasses[AArch64::GPR64RegClassID].contains(Reg.RegNum);
  }
 
  bool isGPR64as32() const {
    return Kind == k_Register && Reg.Kind == RegKind::Scalar &&
      AArch64MCRegisterClasses[AArch64::GPR32RegClassID].contains(Reg.RegNum);
  }
 
  bool isGPR64x8() const {
    return Kind == k_Register && Reg.Kind == RegKind::Scalar &&
           AArch64MCRegisterClasses[AArch64::GPR64x8ClassRegClassID].contains(
               Reg.RegNum);
  }
 
  bool isWSeqPair() const {
    return Kind == k_Register && Reg.Kind == RegKind::Scalar &&
           AArch64MCRegisterClasses[AArch64::WSeqPairsClassRegClassID].contains(
               Reg.RegNum);
  }
 
  bool isXSeqPair() const {
    return Kind == k_Register && Reg.Kind == RegKind::Scalar &&
           AArch64MCRegisterClasses[AArch64::XSeqPairsClassRegClassID].contains(
               Reg.RegNum);
  }
 
  bool isSyspXzrPair() const {
    return isGPR64<AArch64::GPR64RegClassID>() && Reg.RegNum == AArch64::XZR;
  }
 
  template<int64_t Angle, int64_t Remainder>
  DiagnosticPredicate isComplexRotation() const {
    if (!isImm()) return DiagnosticPredicateTy::NoMatch;
 
    const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
    if (!CE) return DiagnosticPredicateTy::NoMatch;
    uint64_t Value = CE->getValue();
 
    if (Value % Angle == Remainder && Value <= 270)
      return DiagnosticPredicateTy::Match;
    return DiagnosticPredicateTy::NearMatch;
  }
 
  template <unsigned RegClassID> bool isGPR64() const {
    return Kind == k_Register && Reg.Kind == RegKind::Scalar &&
           AArch64MCRegisterClasses[RegClassID].contains(getReg());
  }
 
  template <unsigned RegClassID, int ExtWidth>
  DiagnosticPredicate isGPR64WithShiftExtend() const {
    if (Kind != k_Register || Reg.Kind != RegKind::Scalar)
      return DiagnosticPredicateTy::NoMatch;
 
    if (isGPR64<RegClassID>() && getShiftExtendType() == AArch64_AM::LSL &&
        getShiftExtendAmount() == Log2_32(ExtWidth / 8))
      return DiagnosticPredicateTy::Match;
    return DiagnosticPredicateTy::NearMatch;
  }
 
  /// Is this a vector list with the type implicit (presumably attached to the
  /// instruction itself)?
  template <RegKind VectorKind, unsigned NumRegs, bool IsConsecutive = false>
  bool isImplicitlyTypedVectorList() const {
    return Kind == k_VectorList && VectorList.Count == NumRegs &&
           VectorList.NumElements == 0 &&
           VectorList.RegisterKind == VectorKind &&
           (!IsConsecutive || (VectorList.Stride == 1));
  }
 
  template <RegKind VectorKind, unsigned NumRegs, unsigned NumElements,
            unsigned ElementWidth, unsigned Stride = 1>
  bool isTypedVectorList() const {
    if (Kind != k_VectorList)
      return false;
    if (VectorList.Count != NumRegs)
      return false;
    if (VectorList.RegisterKind != VectorKind)
      return false;
    if (VectorList.ElementWidth != ElementWidth)
      return false;
    if (VectorList.Stride != Stride)
      return false;
    return VectorList.NumElements == NumElements;
  }
 
  template <RegKind VectorKind, unsigned NumRegs, unsigned NumElements,
            unsigned ElementWidth, unsigned RegClass>
  DiagnosticPredicate isTypedVectorListMultiple() const {
    bool Res =
        isTypedVectorList<VectorKind, NumRegs, NumElements, ElementWidth>();
    if (!Res)
      return DiagnosticPredicateTy::NoMatch;
    if (!AArch64MCRegisterClasses[RegClass].contains(VectorList.RegNum))
      return DiagnosticPredicateTy::NearMatch;
    return DiagnosticPredicateTy::Match;
  }
 
  template <RegKind VectorKind, unsigned NumRegs, unsigned Stride,
            unsigned ElementWidth>
  DiagnosticPredicate isTypedVectorListStrided() const {
    bool Res = isTypedVectorList<VectorKind, NumRegs, /*NumElements*/ 0,
                                 ElementWidth, Stride>();
    if (!Res)
      return DiagnosticPredicateTy::NoMatch;
    if ((VectorList.RegNum < (AArch64::Z0 + Stride)) ||
        ((VectorList.RegNum >= AArch64::Z16) &&
         (VectorList.RegNum < (AArch64::Z16 + Stride))))
      return DiagnosticPredicateTy::Match;
    return DiagnosticPredicateTy::NoMatch;
  }
 
  template <int Min, int Max>
  DiagnosticPredicate isVectorIndex() const {
    if (Kind != k_VectorIndex)
      return DiagnosticPredicateTy::NoMatch;
    if (VectorIndex.Val >= Min && VectorIndex.Val <= Max)
      return DiagnosticPredicateTy::Match;
    return DiagnosticPredicateTy::NearMatch;
  }
 
  bool isToken() const override { return Kind == k_Token; }
 
  bool isTokenEqual(StringRef Str) const {
    return Kind == k_Token && getToken() == Str;
  }
  bool isSysCR() const { return Kind == k_SysCR; }
  bool isPrefetch() const { return Kind == k_Prefetch; }
  bool isPSBHint() const { return Kind == k_PSBHint; }
  bool isPHint() const { return Kind == k_PHint; }
  bool isBTIHint() const { return Kind == k_BTIHint; }
  bool isShiftExtend() const { return Kind == k_ShiftExtend; }
  bool isShifter() const {
    if (!isShiftExtend())
      return false;
 
    AArch64_AM::ShiftExtendType ST = getShiftExtendType();
    return (ST == AArch64_AM::LSL || ST == AArch64_AM::LSR ||
            ST == AArch64_AM::ASR || ST == AArch64_AM::ROR ||
            ST == AArch64_AM::MSL);
  }
 
  template <unsigned ImmEnum> DiagnosticPredicate isExactFPImm() const {
    if (Kind != k_FPImm)
      return DiagnosticPredicateTy::NoMatch;
 
    if (getFPImmIsExact()) {
      // Lookup the immediate from table of supported immediates.
      auto *Desc = AArch64ExactFPImm::lookupExactFPImmByEnum(ImmEnum);
      assert(Desc && "Unknown enum value");
 
      // Calculate its FP value.
      APFloat RealVal(APFloat::IEEEdouble());
      auto StatusOrErr =
          RealVal.convertFromString(Desc->Repr, APFloat::rmTowardZero);
      if (errorToBool(StatusOrErr.takeError()) || *StatusOrErr != APFloat::opOK)
        llvm_unreachable("FP immediate is not exact");
 
      if (getFPImm().bitwiseIsEqual(RealVal))
        return DiagnosticPredicateTy::Match;
    }
 
    return DiagnosticPredicateTy::NearMatch;
  }
 
  template <unsigned ImmA, unsigned ImmB>
  DiagnosticPredicate isExactFPImm() const {
    DiagnosticPredicate Res = DiagnosticPredicateTy::NoMatch;
    if ((Res = isExactFPImm<ImmA>()))
      return DiagnosticPredicateTy::Match;
    if ((Res = isExactFPImm<ImmB>()))
      return DiagnosticPredicateTy::Match;
    return Res;
  }
 
  bool isExtend() const {
    if (!isShiftExtend())
      return false;
 
    AArch64_AM::ShiftExtendType ET = getShiftExtendType();
    return (ET == AArch64_AM::UXTB || ET == AArch64_AM::SXTB ||
            ET == AArch64_AM::UXTH || ET == AArch64_AM::SXTH ||
            ET == AArch64_AM::UXTW || ET == AArch64_AM::SXTW ||
            ET == AArch64_AM::UXTX || ET == AArch64_AM::SXTX ||
            ET == AArch64_AM::LSL) &&
           getShiftExtendAmount() <= 4;
  }
 
  bool isExtend64() const {
    if (!isExtend())
      return false;
    // Make sure the extend expects a 32-bit source register.
    AArch64_AM::ShiftExtendType ET = getShiftExtendType();
    return ET == AArch64_AM::UXTB || ET == AArch64_AM::SXTB ||
           ET == AArch64_AM::UXTH || ET == AArch64_AM::SXTH ||
           ET == AArch64_AM::UXTW || ET == AArch64_AM::SXTW;
  }
 
  bool isExtendLSL64() const {
    if (!isExtend())
      return false;
    AArch64_AM::ShiftExtendType ET = getShiftExtendType();
    return (ET == AArch64_AM::UXTX || ET == AArch64_AM::SXTX ||
            ET == AArch64_AM::LSL) &&
           getShiftExtendAmount() <= 4;
  }
 
  bool isLSLImm3Shift() const {
    if (!isShiftExtend())
      return false;
    AArch64_AM::ShiftExtendType ET = getShiftExtendType();
    return ET == AArch64_AM::LSL && getShiftExtendAmount() <= 7;
  }
 
  template<int Width> bool isMemXExtend() const {
    if (!isExtend())
      return false;
    AArch64_AM::ShiftExtendType ET = getShiftExtendType();
    return (ET == AArch64_AM::LSL || ET == AArch64_AM::SXTX) &&
           (getShiftExtendAmount() == Log2_32(Width / 8) ||
            getShiftExtendAmount() == 0);
  }
 
  template<int Width> bool isMemWExtend() const {
    if (!isExtend())
      return false;
    AArch64_AM::ShiftExtendType ET = getShiftExtendType();
    return (ET == AArch64_AM::UXTW || ET == AArch64_AM::SXTW) &&
           (getShiftExtendAmount() == Log2_32(Width / 8) ||
            getShiftExtendAmount() == 0);
  }
 
  template <unsigned width>
  bool isArithmeticShifter() const {
    if (!isShifter())
      return false;
 
    // An arithmetic shifter is LSL, LSR, or ASR.
    AArch64_AM::ShiftExtendType ST = getShiftExtendType();
    return (ST == AArch64_AM::LSL || ST == AArch64_AM::LSR ||
            ST == AArch64_AM::ASR) && getShiftExtendAmount() < width;
  }
 
  template <unsigned width>
  bool isLogicalShifter() const {
    if (!isShifter())
      return false;
 
    // A logical shifter is LSL, LSR, ASR or ROR.
    AArch64_AM::ShiftExtendType ST = getShiftExtendType();
    return (ST == AArch64_AM::LSL || ST == AArch64_AM::LSR ||
            ST == AArch64_AM::ASR || ST == AArch64_AM::ROR) &&
           getShiftExtendAmount() < width;
  }
 
  bool isMovImm32Shifter() const {
    if (!isShifter())
      return false;
 
    // A MOVi shifter is LSL of 0, 16, 32, or 48.
    AArch64_AM::ShiftExtendType ST = getShiftExtendType();
    if (ST != AArch64_AM::LSL)
      return false;
    uint64_t Val = getShiftExtendAmount();
    return (Val == 0 || Val == 16);
  }
 
  bool isMovImm64Shifter() const {
    if (!isShifter())
      return false;
 
    // A MOVi shifter is LSL of 0 or 16.
    AArch64_AM::ShiftExtendType ST = getShiftExtendType();
    if (ST != AArch64_AM::LSL)
      return false;
    uint64_t Val = getShiftExtendAmount();
    return (Val == 0 || Val == 16 || Val == 32 || Val == 48);
  }
 
  bool isLogicalVecShifter() const {
    if (!isShifter())
      return false;
 
    // A logical vector shifter is a left shift by 0, 8, 16, or 24.
    unsigned Shift = getShiftExtendAmount();
    return getShiftExtendType() == AArch64_AM::LSL &&
           (Shift == 0 || Shift == 8 || Shift == 16 || Shift == 24);
  }
 
  bool isLogicalVecHalfWordShifter() const {
    if (!isLogicalVecShifter())
      return false;
 
    // A logical vector shifter is a left shift by 0 or 8.
    unsigned Shift = getShiftExtendAmount();
    return getShiftExtendType() == AArch64_AM::LSL &&
           (Shift == 0 || Shift == 8);
  }
 
  bool isMoveVecShifter() const {
    if (!isShiftExtend())
      return false;
 
    // A logical vector shifter is a left shift by 8 or 16.
    unsigned Shift = getShiftExtendAmount();
    return getShiftExtendType() == AArch64_AM::MSL &&
           (Shift == 8 || Shift == 16);
  }
 
  // Fallback unscaled operands are for aliases of LDR/STR that fall back
  // to LDUR/STUR when the offset is not legal for the former but is for
  // the latter. As such, in addition to checking for being a legal unscaled
  // address, also check that it is not a legal scaled address. This avoids
  // ambiguity in the matcher.
  template<int Width>
  bool isSImm9OffsetFB() const {
    return isSImm<9>() && !isUImm12Offset<Width / 8>();
  }
 
  bool isAdrpLabel() const {
    // Validation was handled during parsing, so we just verify that
    // something didn't go haywire.
    if (!isImm())
        return false;
 
    if (const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(Imm.Val)) {
      int64_t Val = CE->getValue();
      int64_t Min = - (4096 * (1LL << (21 - 1)));
      int64_t Max = 4096 * ((1LL << (21 - 1)) - 1);
      return (Val % 4096) == 0 && Val >= Min && Val <= Max;
    }
 
    return true;
  }
 
  bool isAdrLabel() const {
    // Validation was handled during parsing, so we just verify that
    // something didn't go haywire.
    if (!isImm())
        return false;
 
    if (const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(Imm.Val)) {
      int64_t Val = CE->getValue();
      int64_t Min = - (1LL << (21 - 1));
      int64_t Max = ((1LL << (21 - 1)) - 1);
      return Val >= Min && Val <= Max;
    }
 
    return true;
  }
 
  template <MatrixKind Kind, unsigned EltSize, unsigned RegClass>
  DiagnosticPredicate isMatrixRegOperand() const {
    if (!isMatrix())
      return DiagnosticPredicateTy::NoMatch;
    if (getMatrixKind() != Kind ||
        !AArch64MCRegisterClasses[RegClass].contains(getMatrixReg()) ||
        EltSize != getMatrixElementWidth())
      return DiagnosticPredicateTy::NearMatch;
    return DiagnosticPredicateTy::Match;
  }
 
  bool isPAuthPCRelLabel16Operand() const {
    // PAuth PCRel16 operands are similar to regular branch targets, but only
    // negative values are allowed for concrete immediates as signing instr
    // should be in a lower address.
    if (!isImm())
      return false;
    const MCConstantExpr *MCE = dyn_cast<MCConstantExpr>(getImm());
    if (!MCE)
      return true;
    int64_t Val = MCE->getValue();
    if (Val & 0b11)
      return false;
    return (Val <= 0) && (Val > -(1 << 18));
  }
 
  void addExpr(MCInst &Inst, const MCExpr *Expr) const {
    // Add as immediates when possible.  Null MCExpr = 0.
    if (!Expr)
      Inst.addOperand(MCOperand::createImm(0));
    else if (const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(Expr))
      Inst.addOperand(MCOperand::createImm(CE->getValue()));
    else
      Inst.addOperand(MCOperand::createExpr(Expr));
  }
 
  void addRegOperands(MCInst &Inst, unsigned N) const {
    assert(N == 1 && "Invalid number of operands!");
    Inst.addOperand(MCOperand::createReg(getReg()));
  }
 
  void addMatrixOperands(MCInst &Inst, unsigned N) const {
    assert(N == 1 && "Invalid number of operands!");
    Inst.addOperand(MCOperand::createReg(getMatrixReg()));
  }
 
  void addGPR32as64Operands(MCInst &Inst, unsigned N) const {
    assert(N == 1 && "Invalid number of operands!");
    assert(
        AArch64MCRegisterClasses[AArch64::GPR64RegClassID].contains(getReg()));
 
    const MCRegisterInfo *RI = Ctx.getRegisterInfo();
    uint32_t Reg = RI->getRegClass(AArch64::GPR32RegClassID).getRegister(
        RI->getEncodingValue(getReg()));
 
    Inst.addOperand(MCOperand::createReg(Reg));
  }
 
  void addGPR64as32Operands(MCInst &Inst, unsigned N) const {
    assert(N == 1 && "Invalid number of operands!");
    assert(
        AArch64MCRegisterClasses[AArch64::GPR32RegClassID].contains(getReg()));
 
    const MCRegisterInfo *RI = Ctx.getRegisterInfo();
    uint32_t Reg = RI->getRegClass(AArch64::GPR64RegClassID).getRegister(
        RI->getEncodingValue(getReg()));
 
    Inst.addOperand(MCOperand::createReg(Reg));
  }
 
  template <int Width>
  void addFPRasZPRRegOperands(MCInst &Inst, unsigned N) const {
    unsigned Base;
    switch (Width) {
    case 8:   Base = AArch64::B0; break;
    case 16:  Base = AArch64::H0; break;
    case 32:  Base = AArch64::S0; break;
    case 64:  Base = AArch64::D0; break;
    case 128: Base = AArch64::Q0; break;
    default:
      llvm_unreachable("Unsupported width");
    }
    Inst.addOperand(MCOperand::createReg(AArch64::Z0 + getReg() - Base));
  }
 
  void addPPRorPNRRegOperands(MCInst &Inst, unsigned N) const {
    assert(N == 1 && "Invalid number of operands!");
    unsigned Reg = getReg();
    // Normalise to PPR
    if (Reg >= AArch64::PN0 && Reg <= AArch64::PN15)
      Reg = Reg - AArch64::PN0 + AArch64::P0;
    Inst.addOperand(MCOperand::createReg(Reg));
  }
 
  void addPNRasPPRRegOperands(MCInst &Inst, unsigned N) const {
    assert(N == 1 && "Invalid number of operands!");
    Inst.addOperand(
        MCOperand::createReg((getReg() - AArch64::PN0) + AArch64::P0));
  }
 
  void addVectorReg64Operands(MCInst &Inst, unsigned N) const {
    assert(N == 1 && "Invalid number of operands!");
    assert(
        AArch64MCRegisterClasses[AArch64::FPR128RegClassID].contains(getReg()));
    Inst.addOperand(MCOperand::createReg(AArch64::D0 + getReg() - AArch64::Q0));
  }
 
  void addVectorReg128Operands(MCInst &Inst, unsigned N) const {
    assert(N == 1 && "Invalid number of operands!");
    assert(
        AArch64MCRegisterClasses[AArch64::FPR128RegClassID].contains(getReg()));
    Inst.addOperand(MCOperand::createReg(getReg()));
  }
 
  void addVectorRegLoOperands(MCInst &Inst, unsigned N) const {
    assert(N == 1 && "Invalid number of operands!");
    Inst.addOperand(MCOperand::createReg(getReg()));
  }
 
  void addVectorReg0to7Operands(MCInst &Inst, unsigned N) const {
    assert(N == 1 && "Invalid number of operands!");
    Inst.addOperand(MCOperand::createReg(getReg()));
  }
 
  enum VecListIndexType {
    VecListIdx_DReg = 0,
    VecListIdx_QReg = 1,
    VecListIdx_ZReg = 2,
    VecListIdx_PReg = 3,
  };
 
  template <VecListIndexType RegTy, unsigned NumRegs,
            bool IsConsecutive = false>
  void addVectorListOperands(MCInst &Inst, unsigned N) const {
    assert(N == 1 && "Invalid number of operands!");
    assert((!IsConsecutive || (getVectorListStride() == 1)) &&
           "Expected consecutive registers");
    static const unsigned FirstRegs[][5] = {
      /* DReg */ { AArch64::Q0,
                   AArch64::D0,       AArch64::D0_D1,
                   AArch64::D0_D1_D2, AArch64::D0_D1_D2_D3 },
      /* QReg */ { AArch64::Q0,
                   AArch64::Q0,       AArch64::Q0_Q1,
                   AArch64::Q0_Q1_Q2, AArch64::Q0_Q1_Q2_Q3 },
      /* ZReg */ { AArch64::Z0,
                   AArch64::Z0,       AArch64::Z0_Z1,
                   AArch64::Z0_Z1_Z2, AArch64::Z0_Z1_Z2_Z3 },
      /* PReg */ { AArch64::P0,
                   AArch64::P0,       AArch64::P0_P1 }
    };
 
    assert((RegTy != VecListIdx_ZReg || NumRegs <= 4) &&
           " NumRegs must be <= 4 for ZRegs");
 
    assert((RegTy != VecListIdx_PReg || NumRegs <= 2) &&
           " NumRegs must be <= 2 for PRegs");
 
    unsigned FirstReg = FirstRegs[(unsigned)RegTy][NumRegs];
    Inst.addOperand(MCOperand::createReg(FirstReg + getVectorListStart() -
                                         FirstRegs[(unsigned)RegTy][0]));
  }
 
  template <unsigned NumRegs>
  void addStridedVectorListOperands(MCInst &Inst, unsigned N) const {
    assert(N == 1 && "Invalid number of operands!");
    assert((NumRegs == 2 || NumRegs == 4) && " NumRegs must be 2 or 4");
 
    switch (NumRegs) {
    case 2:
      if (getVectorListStart() < AArch64::Z16) {
        assert((getVectorListStart() < AArch64::Z8) &&
               (getVectorListStart() >= AArch64::Z0) && "Invalid Register");
        Inst.addOperand(MCOperand::createReg(
            AArch64::Z0_Z8 + getVectorListStart() - AArch64::Z0));
      } else {
        assert((getVectorListStart() < AArch64::Z24) &&
               (getVectorListStart() >= AArch64::Z16) && "Invalid Register");
        Inst.addOperand(MCOperand::createReg(
            AArch64::Z16_Z24 + getVectorListStart() - AArch64::Z16));
      }
      break;
    case 4:
      if (getVectorListStart() < AArch64::Z16) {
        assert((getVectorListStart() < AArch64::Z4) &&
               (getVectorListStart() >= AArch64::Z0) && "Invalid Register");
        Inst.addOperand(MCOperand::createReg(
            AArch64::Z0_Z4_Z8_Z12 + getVectorListStart() - AArch64::Z0));
      } else {
        assert((getVectorListStart() < AArch64::Z20) &&
               (getVectorListStart() >= AArch64::Z16) && "Invalid Register");
        Inst.addOperand(MCOperand::createReg(
            AArch64::Z16_Z20_Z24_Z28 + getVectorListStart() - AArch64::Z16));
      }
      break;
    default:
      llvm_unreachable("Unsupported number of registers for strided vec list");
    }
  }
 
  void addMatrixTileListOperands(MCInst &Inst, unsigned N) const {
    assert(N == 1 && "Invalid number of operands!");
    unsigned RegMask = getMatrixTileListRegMask();
    assert(RegMask <= 0xFF && "Invalid mask!");
    Inst.addOperand(MCOperand::createImm(RegMask));
  }
 
  void addVectorIndexOperands(MCInst &Inst, unsigned N) const {
    assert(N == 1 && "Invalid number of operands!");
    Inst.addOperand(MCOperand::createImm(getVectorIndex()));
  }
 
  template <unsigned ImmIs0, unsigned ImmIs1>
  void addExactFPImmOperands(MCInst &Inst, unsigned N) const {
    assert(N == 1 && "Invalid number of operands!");
    assert(bool(isExactFPImm<ImmIs0, ImmIs1>()) && "Invalid operand");
    Inst.addOperand(MCOperand::createImm(bool(isExactFPImm<ImmIs1>())));
  }
 
  void addImmOperands(MCInst &Inst, unsigned N) const {
    assert(N == 1 && "Invalid number of operands!");
    // If this is a pageoff symrefexpr with an addend, adjust the addend
    // to be only the page-offset portion. Otherwise, just add the expr
    // as-is.
    addExpr(Inst, getImm());
  }
 
  template <int Shift>
  void addImmWithOptionalShiftOperands(MCInst &Inst, unsigned N) const {
    assert(N == 2 && "Invalid number of operands!");
    if (auto ShiftedVal = getShiftedVal<Shift>()) {
      Inst.addOperand(MCOperand::createImm(ShiftedVal->first));
      Inst.addOperand(MCOperand::createImm(ShiftedVal->second));
    } else if (isShiftedImm()) {
      addExpr(Inst, getShiftedImmVal());
      Inst.addOperand(MCOperand::createImm(getShiftedImmShift()));
    } else {
      addExpr(Inst, getImm());
      Inst.addOperand(MCOperand::createImm(0));
    }
  }
 
  template <int Shift>
  void addImmNegWithOptionalShiftOperands(MCInst &Inst, unsigned N) const {
    assert(N == 2 && "Invalid number of operands!");
    if (auto ShiftedVal = getShiftedVal<Shift>()) {
      Inst.addOperand(MCOperand::createImm(-ShiftedVal->first));
      Inst.addOperand(MCOperand::createImm(ShiftedVal->second));
    } else
      llvm_unreachable("Not a shifted negative immediate");
  }
 
  void addCondCodeOperands(MCInst &Inst, unsigned N) const {
    assert(N == 1 && "Invalid number of operands!");
    Inst.addOperand(MCOperand::createImm(getCondCode()));
  }
 
  void addAdrpLabelOperands(MCInst &Inst, unsigned N) const {
    assert(N == 1 && "Invalid number of operands!");
    const MCConstantExpr *MCE = dyn_cast<MCConstantExpr>(getImm());
    if (!MCE)
      addExpr(Inst, getImm());
    else
      Inst.addOperand(MCOperand::createImm(MCE->getValue() >> 12));
  }
 
  void addAdrLabelOperands(MCInst &Inst, unsigned N) const {
    addImmOperands(Inst, N);
  }
 
  template<int Scale>
  void addUImm12OffsetOperands(MCInst &Inst, unsigned N) const {
    assert(N == 1 && "Invalid number of operands!");
    const MCConstantExpr *MCE = dyn_cast<MCConstantExpr>(getImm());
 
    if (!MCE) {
      Inst.addOperand(MCOperand::createExpr(getImm()));
      return;
    }
    Inst.addOperand(MCOperand::createImm(MCE->getValue() / Scale));
  }
 
  void addUImm6Operands(MCInst &Inst, unsigned N) const {
    assert(N == 1 && "Invalid number of operands!");
    const MCConstantExpr *MCE = cast<MCConstantExpr>(getImm());
    Inst.addOperand(MCOperand::createImm(MCE->getValue()));
  }
 
  template <int Scale>
  void addImmScaledOperands(MCInst &Inst, unsigned N) const {
    assert(N == 1 && "Invalid number of operands!");
    const MCConstantExpr *MCE = cast<MCConstantExpr>(getImm());
    Inst.addOperand(MCOperand::createImm(MCE->getValue() / Scale));
  }
 
  template <int Scale>
  void addImmScaledRangeOperands(MCInst &Inst, unsigned N) const {
    assert(N == 1 && "Invalid number of operands!");
    Inst.addOperand(MCOperand::createImm(getFirstImmVal() / Scale));
  }
 
  template <typename T>
  void addLogicalImmOperands(MCInst &Inst, unsigned N) const {
    assert(N == 1 && "Invalid number of operands!");
    const MCConstantExpr *MCE = cast<MCConstantExpr>(getImm());
    std::make_unsigned_t<T> Val = MCE->getValue();
    uint64_t encoding = AArch64_AM::encodeLogicalImmediate(Val, sizeof(T) * 8);
    Inst.addOperand(MCOperand::createImm(encoding));
  }
 
  template <typename T>
  void addLogicalImmNotOperands(MCInst &Inst, unsigned N) const {
    assert(N == 1 && "Invalid number of operands!");
    const MCConstantExpr *MCE = cast<MCConstantExpr>(getImm());
    std::make_unsigned_t<T> Val = ~MCE->getValue();
    uint64_t encoding = AArch64_AM::encodeLogicalImmediate(Val, sizeof(T) * 8);
    Inst.addOperand(MCOperand::createImm(encoding));
  }
 
  void addSIMDImmType10Operands(MCInst &Inst, unsigned N) const {
    assert(N == 1 && "Invalid number of operands!");
    const MCConstantExpr *MCE = cast<MCConstantExpr>(getImm());
    uint64_t encoding = AArch64_AM::encodeAdvSIMDModImmType10(MCE->getValue());
    Inst.addOperand(MCOperand::createImm(encoding));
  }
 
  void addBranchTarget26Operands(MCInst &Inst, unsigned N) const {
    // Branch operands don't encode the low bits, so shift them off
    // here. If it's a label, however, just put it on directly as there's
    // not enough information now to do anything.
    assert(N == 1 && "Invalid number of operands!");
    const MCConstantExpr *MCE = dyn_cast<MCConstantExpr>(getImm());
    if (!MCE) {
      addExpr(Inst, getImm());
      return;
    }
    assert(MCE && "Invalid constant immediate operand!");
    Inst.addOperand(MCOperand::createImm(MCE->getValue() >> 2));
  }
 
  void addPAuthPCRelLabel16Operands(MCInst &Inst, unsigned N) const {
    // PC-relative operands don't encode the low bits, so shift them off
    // here. If it's a label, however, just put it on directly as there's
    // not enough information now to do anything.
    assert(N == 1 && "Invalid number of operands!");
    const MCConstantExpr *MCE = dyn_cast<MCConstantExpr>(getImm());
    if (!MCE) {
      addExpr(Inst, getImm());
      return;
    }
    Inst.addOperand(MCOperand::createImm(MCE->getValue() >> 2));
  }
 
  void addPCRelLabel19Operands(MCInst &Inst, unsigned N) const {
    // Branch operands don't encode the low bits, so shift them off
    // here. If it's a label, however, just put it on directly as there's
    // not enough information now to do anything.
    assert(N == 1 && "Invalid number of operands!");
    const MCConstantExpr *MCE = dyn_cast<MCConstantExpr>(getImm());
    if (!MCE) {
      addExpr(Inst, getImm());
      return;
    }
    assert(MCE && "Invalid constant immediate operand!");
    Inst.addOperand(MCOperand::createImm(MCE->getValue() >> 2));
  }
 
  void addPCRelLabel9Operands(MCInst &Inst, unsigned N) const {
    // Branch operands don't encode the low bits, so shift them off
    // here. If it's a label, however, just put it on directly as there's
    // not enough information now to do anything.
    assert(N == 1 && "Invalid number of operands!");
    const MCConstantExpr *MCE = dyn_cast<MCConstantExpr>(getImm());
    if (!MCE) {
      addExpr(Inst, getImm());
      return;
    }
    assert(MCE && "Invalid constant immediate operand!");
    Inst.addOperand(MCOperand::createImm(MCE->getValue() >> 2));
  }
 
  void addBranchTarget14Operands(MCInst &Inst, unsigned N) const {
    // Branch operands don't encode the low bits, so shift them off
    // here. If it's a label, however, just put it on directly as there's
    // not enough information now to do anything.
    assert(N == 1 && "Invalid number of operands!");
    const MCConstantExpr *MCE = dyn_cast<MCConstantExpr>(getImm());
    if (!MCE) {
      addExpr(Inst, getImm());
      return;
    }
    assert(MCE && "Invalid constant immediate operand!");
    Inst.addOperand(MCOperand::createImm(MCE->getValue() >> 2));
  }
 
  void addFPImmOperands(MCInst &Inst, unsigned N) const {
    assert(N == 1 && "Invalid number of operands!");
    Inst.addOperand(MCOperand::createImm(
        AArch64_AM::getFP64Imm(getFPImm().bitcastToAPInt())));
  }
 
  void addBarrierOperands(MCInst &Inst, unsigned N) const {
    assert(N == 1 && "Invalid number of operands!");
    Inst.addOperand(MCOperand::createImm(getBarrier()));
  }
 
  void addBarriernXSOperands(MCInst &Inst, unsigned N) const {
    assert(N == 1 && "Invalid number of operands!");
    Inst.addOperand(MCOperand::createImm(getBarrier()));
  }
 
  void addMRSSystemRegisterOperands(MCInst &Inst, unsigned N) const {
    assert(N == 1 && "Invalid number of operands!");
 
    Inst.addOperand(MCOperand::createImm(SysReg.MRSReg));
  }
 
  void addMSRSystemRegisterOperands(MCInst &Inst, unsigned N) const {
    assert(N == 1 && "Invalid number of operands!");
 
    Inst.addOperand(MCOperand::createImm(SysReg.MSRReg));
  }
 
  void addSystemPStateFieldWithImm0_1Operands(MCInst &Inst, unsigned N) const {
    assert(N == 1 && "Invalid number of operands!");
 
    Inst.addOperand(MCOperand::createImm(SysReg.PStateField));
  }
 
  void addSVCROperands(MCInst &Inst, unsigned N) const {
    assert(N == 1 && "Invalid number of operands!");
 
    Inst.addOperand(MCOperand::createImm(SVCR.PStateField));
  }
 
  void addSystemPStateFieldWithImm0_15Operands(MCInst &Inst, unsigned N) const {
    assert(N == 1 && "Invalid number of operands!");
 
    Inst.addOperand(MCOperand::createImm(SysReg.PStateField));
  }
 
  void addSysCROperands(MCInst &Inst, unsigned N) const {
    assert(N == 1 && "Invalid number of operands!");
    Inst.addOperand(MCOperand::createImm(getSysCR()));
  }
 
  void addPrefetchOperands(MCInst &Inst, unsigned N) const {
    assert(N == 1 && "Invalid number of operands!");
    Inst.addOperand(MCOperand::createImm(getPrefetch()));
  }
 
  void addPSBHintOperands(MCInst &Inst, unsigned N) const {
    assert(N == 1 && "Invalid number of operands!");
    Inst.addOperand(MCOperand::createImm(getPSBHint()));
  }
 
  void addPHintOperands(MCInst &Inst, unsigned N) const {
    assert(N == 1 && "Invalid number of operands!");
    Inst.addOperand(MCOperand::createImm(getPHint()));
  }
 
  void addBTIHintOperands(MCInst &Inst, unsigned N) const {
    assert(N == 1 && "Invalid number of operands!");
    Inst.addOperand(MCOperand::createImm(getBTIHint()));
  }
 
  void addShifterOperands(MCInst &Inst, unsigned N) const {
    assert(N == 1 && "Invalid number of operands!");
    unsigned Imm =
        AArch64_AM::getShifterImm(getShiftExtendType(), getShiftExtendAmount());
    Inst.addOperand(MCOperand::createImm(Imm));
  }
 
  void addLSLImm3ShifterOperands(MCInst &Inst, unsigned N) const {
    assert(N == 1 && "Invalid number of operands!");
    unsigned Imm = getShiftExtendAmount();
    Inst.addOperand(MCOperand::createImm(Imm));
  }
 
  void addSyspXzrPairOperand(MCInst &Inst, unsigned N) const {
    assert(N == 1 && "Invalid number of operands!");
 
    if (!isScalarReg())
      return;
 
    const MCRegisterInfo *RI = Ctx.getRegisterInfo();
    uint32_t Reg = RI->getRegClass(AArch64::GPR64RegClassID)
                       .getRegister(RI->getEncodingValue(getReg()));
    if (Reg != AArch64::XZR)
      llvm_unreachable("wrong register");
 
    Inst.addOperand(MCOperand::createReg(AArch64::XZR));
  }
 
  void addExtendOperands(MCInst &Inst, unsigned N) const {
    assert(N == 1 && "Invalid number of operands!");
    AArch64_AM::ShiftExtendType ET = getShiftExtendType();
    if (ET == AArch64_AM::LSL) ET = AArch64_AM::UXTW;
    unsigned Imm = AArch64_AM::getArithExtendImm(ET, getShiftExtendAmount());
    Inst.addOperand(MCOperand::createImm(Imm));
  }
 
  void addExtend64Operands(MCInst &Inst, unsigned N) const {
    assert(N == 1 && "Invalid number of operands!");
    AArch64_AM::ShiftExtendType ET = getShiftExtendType();
    if (ET == AArch64_AM::LSL) ET = AArch64_AM::UXTX;
    unsigned Imm = AArch64_AM::getArithExtendImm(ET, getShiftExtendAmount());
    Inst.addOperand(MCOperand::createImm(Imm));
  }
 
  void addMemExtendOperands(MCInst &Inst, unsigned N) const {
    assert(N == 2 && "Invalid number of operands!");
    AArch64_AM::ShiftExtendType ET = getShiftExtendType();
    bool IsSigned = ET == AArch64_AM::SXTW || ET == AArch64_AM::SXTX;
    Inst.addOperand(MCOperand::createImm(IsSigned));
    Inst.addOperand(MCOperand::createImm(getShiftExtendAmount() != 0));
  }
 
  // For 8-bit load/store instructions with a register offset, both the
  // "DoShift" and "NoShift" variants have a shift of 0. Because of this,
  // they're disambiguated by whether the shift was explicit or implicit rather
  // than its size.
  void addMemExtend8Operands(MCInst &Inst, unsigned N) const {
    assert(N == 2 && "Invalid number of operands!");
    AArch64_AM::ShiftExtendType ET = getShiftExtendType();
    bool IsSigned = ET == AArch64_AM::SXTW || ET == AArch64_AM::SXTX;
    Inst.addOperand(MCOperand::createImm(IsSigned));
    Inst.addOperand(MCOperand::createImm(hasShiftExtendAmount()));
  }
 
  template<int Shift>
  void addMOVZMovAliasOperands(MCInst &Inst, unsigned N) const {
    assert(N == 1 && "Invalid number of operands!");
 
    const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
    if (CE) {
      uint64_t Value = CE->getValue();
      Inst.addOperand(MCOperand::createImm((Value >> Shift) & 0xffff));
    } else {
      addExpr(Inst, getImm());
    }
  }
 
  template<int Shift>
  void addMOVNMovAliasOperands(MCInst &Inst, unsigned N) const {
    assert(N == 1 && "Invalid number of operands!");
 
    const MCConstantExpr *CE = cast<MCConstantExpr>(getImm());
    uint64_t Value = CE->getValue();
    Inst.addOperand(MCOperand::createImm((~Value >> Shift) & 0xffff));
  }
 
  void addComplexRotationEvenOperands(MCInst &Inst, unsigned N) const {
    assert(N == 1 && "Invalid number of operands!");
    const MCConstantExpr *MCE = cast<MCConstantExpr>(getImm());
    Inst.addOperand(MCOperand::createImm(MCE->getValue() / 90));
  }
 
  void addComplexRotationOddOperands(MCInst &Inst, unsigned N) const {
    assert(N == 1 && "Invalid number of operands!");
    const MCConstantExpr *MCE = cast<MCConstantExpr>(getImm());
    Inst.addOperand(MCOperand::createImm((MCE->getValue() - 90) / 180));
  }
 
  void print(raw_ostream &OS) const override;
 
  static std::unique_ptr<AArch64Operand>
  CreateToken(StringRef Str, SMLoc S, MCContext &Ctx, bool IsSuffix = false) {
    auto Op = std::make_unique<AArch64Operand>(k_Token, Ctx);
    Op->Tok.Data = Str.data();
    Op->Tok.Length = Str.size();
    Op->Tok.IsSuffix = IsSuffix;
    Op->StartLoc = S;
    Op->EndLoc = S;
    return Op;
  }
 
  static std::unique_ptr<AArch64Operand>
  CreateReg(unsigned RegNum, RegKind Kind, SMLoc S, SMLoc E, MCContext &Ctx,
            RegConstraintEqualityTy EqTy = RegConstraintEqualityTy::EqualsReg,
            AArch64_AM::ShiftExtendType ExtTy = AArch64_AM::LSL,
            unsigned ShiftAmount = 0,
            unsigned HasExplicitAmount = false) {
    auto Op = std::make_unique<AArch64Operand>(k_Register, Ctx);
    Op->Reg.RegNum = RegNum;
    Op->Reg.Kind = Kind;
    Op->Reg.ElementWidth = 0;
    Op->Reg.EqualityTy = EqTy;
    Op->Reg.ShiftExtend.Type = ExtTy;
    Op->Reg.ShiftExtend.Amount = ShiftAmount;
    Op->Reg.ShiftExtend.HasExplicitAmount = HasExplicitAmount;
    Op->StartLoc = S;
    Op->EndLoc = E;
    return Op;
  }
 
  static std::unique_ptr<AArch64Operand>
  CreateVectorReg(unsigned RegNum, RegKind Kind, unsigned ElementWidth,
                  SMLoc S, SMLoc E, MCContext &Ctx,
                  AArch64_AM::ShiftExtendType ExtTy = AArch64_AM::LSL,
                  unsigned ShiftAmount = 0,
                  unsigned HasExplicitAmount = false) {
    assert((Kind == RegKind::NeonVector || Kind == RegKind::SVEDataVector ||
            Kind == RegKind::SVEPredicateVector ||
            Kind == RegKind::SVEPredicateAsCounter) &&
           "Invalid vector kind");
    auto Op = CreateReg(RegNum, Kind, S, E, Ctx, EqualsReg, ExtTy, ShiftAmount,
                        HasExplicitAmount);
    Op->Reg.ElementWidth = ElementWidth;
    return Op;
  }
 
  static std::unique_ptr<AArch64Operand>
  CreateVectorList(unsigned RegNum, unsigned Count, unsigned Stride,
                   unsigned NumElements, unsigned ElementWidth,
                   RegKind RegisterKind, SMLoc S, SMLoc E, MCContext &Ctx) {
    auto Op = std::make_unique<AArch64Operand>(k_VectorList, Ctx);
    Op->VectorList.RegNum = RegNum;
    Op->VectorList.Count = Count;
    Op->VectorList.Stride = Stride;
    Op->VectorList.NumElements = NumElements;
    Op->VectorList.ElementWidth = ElementWidth;
    Op->VectorList.RegisterKind = RegisterKind;
    Op->StartLoc = S;
    Op->EndLoc = E;
    return Op;
  }
 
  static std::unique_ptr<AArch64Operand>
  CreateVectorIndex(int Idx, SMLoc S, SMLoc E, MCContext &Ctx) {
    auto Op = std::make_unique<AArch64Operand>(k_VectorIndex, Ctx);
    Op->VectorIndex.Val = Idx;
    Op->StartLoc = S;
    Op->EndLoc = E;
    return Op;
  }
 
  static std::unique_ptr<AArch64Operand>
  CreateMatrixTileList(unsigned RegMask, SMLoc S, SMLoc E, MCContext &Ctx) {
    auto Op = std::make_unique<AArch64Operand>(k_MatrixTileList, Ctx);
    Op->MatrixTileList.RegMask = RegMask;
    Op->StartLoc = S;
    Op->EndLoc = E;
    return Op;
  }
 
  static void ComputeRegsForAlias(unsigned Reg, SmallSet<unsigned, 8> &OutRegs,
                                  const unsigned ElementWidth) {
    static std::map<std::pair<unsigned, unsigned>, std::vector<unsigned>>
        RegMap = {
            {{0, AArch64::ZAB0},
             {AArch64::ZAD0, AArch64::ZAD1, AArch64::ZAD2, AArch64::ZAD3,
              AArch64::ZAD4, AArch64::ZAD5, AArch64::ZAD6, AArch64::ZAD7}},
            {{8, AArch64::ZAB0},
             {AArch64::ZAD0, AArch64::ZAD1, AArch64::ZAD2, AArch64::ZAD3,
              AArch64::ZAD4, AArch64::ZAD5, AArch64::ZAD6, AArch64::ZAD7}},
            {{16, AArch64::ZAH0},
             {AArch64::ZAD0, AArch64::ZAD2, AArch64::ZAD4, AArch64::ZAD6}},
            {{16, AArch64::ZAH1},
             {AArch64::ZAD1, AArch64::ZAD3, AArch64::ZAD5, AArch64::ZAD7}},
            {{32, AArch64::ZAS0}, {AArch64::ZAD0, AArch64::ZAD4}},
            {{32, AArch64::ZAS1}, {AArch64::ZAD1, AArch64::ZAD5}},
            {{32, AArch64::ZAS2}, {AArch64::ZAD2, AArch64::ZAD6}},
            {{32, AArch64::ZAS3}, {AArch64::ZAD3, AArch64::ZAD7}},
        };
 
    if (ElementWidth == 64)
      OutRegs.insert(Reg);
    else {
      std::vector<unsigned> Regs = RegMap[std::make_pair(ElementWidth, Reg)];
      assert(!Regs.empty() && "Invalid tile or element width!");
      for (auto OutReg : Regs)
        OutRegs.insert(OutReg);
    }
  }
 
  static std::unique_ptr<AArch64Operand> CreateImm(const MCExpr *Val, SMLoc S,
                                                   SMLoc E, MCContext &Ctx) {
    auto Op = std::make_unique<AArch64Operand>(k_Immediate, Ctx);
    Op->Imm.Val = Val;
    Op->StartLoc = S;
    Op->EndLoc = E;
    return Op;
  }
 
  static std::unique_ptr<AArch64Operand> CreateShiftedImm(const MCExpr *Val,
                                                          unsigned ShiftAmount,
                                                          SMLoc S, SMLoc E,
                                                          MCContext &Ctx) {
    auto Op = std::make_unique<AArch64Operand>(k_ShiftedImm, Ctx);
    Op->ShiftedImm .Val = Val;
    Op->ShiftedImm.ShiftAmount = ShiftAmount;
    Op->StartLoc = S;
    Op->EndLoc = E;
    return Op;
  }
 
  static std::unique_ptr<AArch64Operand> CreateImmRange(unsigned First,
                                                        unsigned Last, SMLoc S,
                                                        SMLoc E,
                                                        MCContext &Ctx) {
    auto Op = std::make_unique<AArch64Operand>(k_ImmRange, Ctx);
    Op->ImmRange.First = First;
    Op->ImmRange.Last = Last;
    Op->EndLoc = E;
    return Op;
  }
 
  static std::unique_ptr<AArch64Operand>
  CreateCondCode(AArch64CC::CondCode Code, SMLoc S, SMLoc E, MCContext &Ctx) {
    auto Op = std::make_unique<AArch64Operand>(k_CondCode, Ctx);
    Op->CondCode.Code = Code;
    Op->StartLoc = S;
    Op->EndLoc = E;
    return Op;
  }
 
  static std::unique_ptr<AArch64Operand>
  CreateFPImm(APFloat Val, bool IsExact, SMLoc S, MCContext &Ctx) {
    auto Op = std::make_unique<AArch64Operand>(k_FPImm, Ctx);
    Op->FPImm.Val = Val.bitcastToAPInt().getSExtValue();
    Op->FPImm.IsExact = IsExact;
    Op->StartLoc = S;
    Op->EndLoc = S;
    return Op;
  }
 
  static std::unique_ptr<AArch64Operand> CreateBarrier(unsigned Val,
                                                       StringRef Str,
                                                       SMLoc S,
                                                       MCContext &Ctx,
                                                       bool HasnXSModifier) {
    auto Op = std::make_unique<AArch64Operand>(k_Barrier, Ctx);
    Op->Barrier.Val = Val;
    Op->Barrier.Data = Str.data();
    Op->Barrier.Length = Str.size();
    Op->Barrier.HasnXSModifier = HasnXSModifier;
    Op->StartLoc = S;
    Op->EndLoc = S;
    return Op;
  }
 
  static std::unique_ptr<AArch64Operand> CreateSysReg(StringRef Str, SMLoc S,
                                                      uint32_t MRSReg,
                                                      uint32_t MSRReg,
                                                      uint32_t PStateField,
                                                      MCContext &Ctx) {
    auto Op = std::make_unique<AArch64Operand>(k_SysReg, Ctx);
    Op->SysReg.Data = Str.data();
    Op->SysReg.Length = Str.size();
    Op->SysReg.MRSReg = MRSReg;
    Op->SysReg.MSRReg = MSRReg;
    Op->SysReg.PStateField = PStateField;
    Op->StartLoc = S;
    Op->EndLoc = S;
    return Op;
  }
 
  static std::unique_ptr<AArch64Operand>
  CreatePHintInst(unsigned Val, StringRef Str, SMLoc S, MCContext &Ctx) {
    auto Op = std::make_unique<AArch64Operand>(k_PHint, Ctx);
    Op->PHint.Val = Val;
    Op->PHint.Data = Str.data();
    Op->PHint.Length = Str.size();
    Op->StartLoc = S;
    Op->EndLoc = S;
    return Op;
  }
 
  static std::unique_ptr<AArch64Operand> CreateSysCR(unsigned Val, SMLoc S,
                                                     SMLoc E, MCContext &Ctx) {
    auto Op = std::make_unique<AArch64Operand>(k_SysCR, Ctx);
    Op->SysCRImm.Val = Val;
    Op->StartLoc = S;
    Op->EndLoc = E;
    return Op;
  }
 
  static std::unique_ptr<AArch64Operand> CreatePrefetch(unsigned Val,
                                                        StringRef Str,
                                                        SMLoc S,
                                                        MCContext &Ctx) {
    auto Op = std::make_unique<AArch64Operand>(k_Prefetch, Ctx);
    Op->Prefetch.Val = Val;
    Op->Barrier.Data = Str.data();
    Op->Barrier.Length = Str.size();
    Op->StartLoc = S;
    Op->EndLoc = S;
    return Op;
  }
 
  static std::unique_ptr<AArch64Operand> CreatePSBHint(unsigned Val,
                                                       StringRef Str,
                                                       SMLoc S,
                                                       MCContext &Ctx) {
    auto Op = std::make_unique<AArch64Operand>(k_PSBHint, Ctx);
    Op->PSBHint.Val = Val;
    Op->PSBHint.Data = Str.data();
    Op->PSBHint.Length = Str.size();
    Op->StartLoc = S;
    Op->EndLoc = S;
    return Op;
  }
 
  static std::unique_ptr<AArch64Operand> CreateBTIHint(unsigned Val,
                                                       StringRef Str,
                                                       SMLoc S,
                                                       MCContext &Ctx) {
    auto Op = std::make_unique<AArch64Operand>(k_BTIHint, Ctx);
    Op->BTIHint.Val = Val | 32;
    Op->BTIHint.Data = Str.data();
    Op->BTIHint.Length = Str.size();
    Op->StartLoc = S;
    Op->EndLoc = S;
    return Op;
  }
 
  static std::unique_ptr<AArch64Operand>
  CreateMatrixRegister(unsigned RegNum, unsigned ElementWidth, MatrixKind Kind,
                       SMLoc S, SMLoc E, MCContext &Ctx) {
    auto Op = std::make_unique<AArch64Operand>(k_MatrixRegister, Ctx);
    Op->MatrixReg.RegNum = RegNum;
    Op->MatrixReg.ElementWidth = ElementWidth;
    Op->MatrixReg.Kind = Kind;
    Op->StartLoc = S;
    Op->EndLoc = E;
    return Op;
  }
 
  static std::unique_ptr<AArch64Operand>
  CreateSVCR(uint32_t PStateField, StringRef Str, SMLoc S, MCContext &Ctx) {
    auto Op = std::make_unique<AArch64Operand>(k_SVCR, Ctx);
    Op->SVCR.PStateField = PStateField;
    Op->SVCR.Data = Str.data();
    Op->SVCR.Length = Str.size();
    Op->StartLoc = S;
    Op->EndLoc = S;
    return Op;
  }
 
  static std::unique_ptr<AArch64Operand>
  CreateShiftExtend(AArch64_AM::ShiftExtendType ShOp, unsigned Val,
                    bool HasExplicitAmount, SMLoc S, SMLoc E, MCContext &Ctx) {
    auto Op = std::make_unique<AArch64Operand>(k_ShiftExtend, Ctx);
    Op->ShiftExtend.Type = ShOp;
    Op->ShiftExtend.Amount = Val;
    Op->ShiftExtend.HasExplicitAmount = HasExplicitAmount;
    Op->StartLoc = S;
    Op->EndLoc = E;
    return Op;
  }
};
 
} // end anonymous namespace.
 
void AArch64Operand::print(raw_ostream &OS) const {
  switch (Kind) {
  case k_FPImm:
    OS << "<fpimm " << getFPImm().bitcastToAPInt().getZExtValue();
    if (!getFPImmIsExact())
      OS << " (inexact)";
    OS << ">";
    break;
  case k_Barrier: {
    StringRef Name = getBarrierName();
    if (!Name.empty())
      OS << "<barrier " << Name << ">";
    else
      OS << "<barrier invalid #" << getBarrier() << ">";
    break;
  }
  case k_Immediate:
    OS << *getImm();
    break;
  case k_ShiftedImm: {
    unsigned Shift = getShiftedImmShift();
    OS << "<shiftedimm ";
    OS << *getShiftedImmVal();
    OS << ", lsl #" << AArch64_AM::getShiftValue(Shift) << ">";
    break;
  }
  case k_ImmRange: {
    OS << "<immrange ";
    OS << getFirstImmVal();
    OS << ":" << getLastImmVal() << ">";
    break;
  }
  case k_CondCode:
    OS << "<condcode " << getCondCode() << ">";
    break;
  case k_VectorList: {
    OS << "<vectorlist ";
    unsigned Reg = getVectorListStart();
    for (unsigned i = 0, e = getVectorListCount(); i != e; ++i)
      OS << Reg + i * getVectorListStride() << " ";
    OS << ">";
    break;
  }
  case k_VectorIndex:
    OS << "<vectorindex " << getVectorIndex() << ">";
    break;
  case k_SysReg:
    OS << "<sysreg: " << getSysReg() << '>';
    break;
  case k_Token:
    OS << "'" << getToken() << "'";
    break;
  case k_SysCR:
    OS << "c" << getSysCR();
    break;
  case k_Prefetch: {
    StringRef Name = getPrefetchName();
    if (!Name.empty())
      OS << "<prfop " << Name << ">";
    else
      OS << "<prfop invalid #" << getPrefetch() << ">";
    break;
  }
  case k_PSBHint:
    OS << getPSBHintName();
    break;
  case k_PHint:
    OS << getPHintName();
    break;
  case k_BTIHint:
    OS << getBTIHintName();
    break;
  case k_MatrixRegister:
    OS << "<matrix " << getMatrixReg() << ">";
    break;
  case k_MatrixTileList: {
    OS << "<matrixlist ";
    unsigned RegMask = getMatrixTileListRegMask();
    unsigned MaxBits = 8;
    for (unsigned I = MaxBits; I > 0; --I)
      OS << ((RegMask & (1 << (I - 1))) >> (I - 1));
    OS << '>';
    break;
  }
  case k_SVCR: {
    OS << getSVCR();
    break;
  }
  case k_Register:
    OS << "<register " << getReg() << ">";
    if (!getShiftExtendAmount() && !hasShiftExtendAmount())
      break;
    [[fallthrough]];
  case k_ShiftExtend:
    OS << "<" << AArch64_AM::getShiftExtendName(getShiftExtendType()) << " #"
       << getShiftExtendAmount();
    if (!hasShiftExtendAmount())
      OS << "<imp>";
    OS << '>';
    break;
  }
}
 
/// @name Auto-generated Match Functions
/// {
 
static MCRegister MatchRegisterName(StringRef Name);
 
/// }
 
static unsigned MatchNeonVectorRegName(StringRef Name) {
  return StringSwitch<unsigned>(Name.lower())
      .Case("v0", AArch64::Q0)
      .Case("v1", AArch64::Q1)
      .Case("v2", AArch64::Q2)
      .Case("v3", AArch64::Q3)
      .Case("v4", AArch64::Q4)
      .Case("v5", AArch64::Q5)
      .Case("v6", AArch64::Q6)
      .Case("v7", AArch64::Q7)
      .Case("v8", AArch64::Q8)
      .Case("v9", AArch64::Q9)
      .Case("v10", AArch64::Q10)
      .Case("v11", AArch64::Q11)
      .Case("v12", AArch64::Q12)
      .Case("v13", AArch64::Q13)
      .Case("v14", AArch64::Q14)
      .Case("v15", AArch64::Q15)
      .Case("v16", AArch64::Q16)
      .Case("v17", AArch64::Q17)
      .Case("v18", AArch64::Q18)
      .Case("v19", AArch64::Q19)
      .Case("v20", AArch64::Q20)
      .Case("v21", AArch64::Q21)
      .Case("v22", AArch64::Q22)
      .Case("v23", AArch64::Q23)
      .Case("v24", AArch64::Q24)
      .Case("v25", AArch64::Q25)
      .Case("v26", AArch64::Q26)
      .Case("v27", AArch64::Q27)
      .Case("v28", AArch64::Q28)
      .Case("v29", AArch64::Q29)
      .Case("v30", AArch64::Q30)
      .Case("v31", AArch64::Q31)
      .Default(0);
}
 
/// Returns an optional pair of (#elements, element-width) if Suffix
/// is a valid vector kind. Where the number of elements in a vector
/// or the vector width is implicit or explicitly unknown (but still a
/// valid suffix kind), 0 is used.
static std::optional<std::pair<int, int>> parseVectorKind(StringRef Suffix,
                                                          RegKind VectorKind) {
  std::pair<int, int> Res = {-1, -1};
 
  switch (VectorKind) {
  case RegKind::NeonVector:
    Res = StringSwitch<std::pair<int, int>>(Suffix.lower())
              .Case("", {0, 0})
              .Case(".1d", {1, 64})
              .Case(".1q", {1, 128})
              // '.2h' needed for fp16 scalar pairwise reductions
              .Case(".2h", {2, 16})
              .Case(".2b", {2, 8})
              .Case(".2s", {2, 32})
              .Case(".2d", {2, 64})
              // '.4b' is another special case for the ARMv8.2a dot product
              // operand
              .Case(".4b", {4, 8})
              .Case(".4h", {4, 16})
              .Case(".4s", {4, 32})
              .Case(".8b", {8, 8})
              .Case(".8h", {8, 16})
              .Case(".16b", {16, 8})
              // Accept the width neutral ones, too, for verbose syntax. If
              // those aren't used in the right places, the token operand won't
              // match so all will work out.
              .Case(".b", {0, 8})
              .Case(".h", {0, 16})
              .Case(".s", {0, 32})
              .Case(".d", {0, 64})
              .Default({-1, -1});
    break;
  case RegKind::SVEPredicateAsCounter:
  case RegKind::SVEPredicateVector:
  case RegKind::SVEDataVector:
  case RegKind::Matrix:
    Res = StringSwitch<std::pair<int, int>>(Suffix.lower())
              .Case("", {0, 0})
              .Case(".b", {0, 8})
              .Case(".h", {0, 16})
              .Case(".s", {0, 32})
              .Case(".d", {0, 64})
              .Case(".q", {0, 128})
              .Default({-1, -1});
    break;
  default:
    llvm_unreachable("Unsupported RegKind");
  }
 
  if (Res == std::make_pair(-1, -1))
    return std::nullopt;
 
  return std::optional<std::pair<int, int>>(Res);
}
 
static bool isValidVectorKind(StringRef Suffix, RegKind VectorKind) {
  return parseVectorKind(Suffix, VectorKind).has_value();
}
 
static unsigned matchSVEDataVectorRegName(StringRef Name) {
  return StringSwitch<unsigned>(Name.lower())
      .Case("z0", AArch64::Z0)
      .Case("z1", AArch64::Z1)
      .Case("z2", AArch64::Z2)
      .Case("z3", AArch64::Z3)
      .Case("z4", AArch64::Z4)
      .Case("z5", AArch64::Z5)
      .Case("z6", AArch64::Z6)
      .Case("z7", AArch64::Z7)
      .Case("z8", AArch64::Z8)
      .Case("z9", AArch64::Z9)
      .Case("z10", AArch64::Z10)
      .Case("z11", AArch64::Z11)
      .Case("z12", AArch64::Z12)
      .Case("z13", AArch64::Z13)
      .Case("z14", AArch64::Z14)
      .Case("z15", AArch64::Z15)
      .Case("z16", AArch64::Z16)
      .Case("z17", AArch64::Z17)
      .Case("z18", AArch64::Z18)
      .Case("z19", AArch64::Z19)
      .Case("z20", AArch64::Z20)
      .Case("z21", AArch64::Z21)
      .Case("z22", AArch64::Z22)
      .Case("z23", AArch64::Z23)
      .Case("z24", AArch64::Z24)
      .Case("z25", AArch64::Z25)
      .Case("z26", AArch64::Z26)
      .Case("z27", AArch64::Z27)
      .Case("z28", AArch64::Z28)
      .Case("z29", AArch64::Z29)
      .Case("z30", AArch64::Z30)
      .Case("z31", AArch64::Z31)
      .Default(0);
}
 
static unsigned matchSVEPredicateVectorRegName(StringRef Name) {
  return StringSwitch<unsigned>(Name.lower())
      .Case("p0", AArch64::P0)
      .Case("p1", AArch64::P1)
      .Case("p2", AArch64::P2)
      .Case("p3", AArch64::P3)
      .Case("p4", AArch64::P4)
      .Case("p5", AArch64::P5)
      .Case("p6", AArch64::P6)
      .Case("p7", AArch64::P7)
      .Case("p8", AArch64::P8)
      .Case("p9", AArch64::P9)
      .Case("p10", AArch64::P10)
      .Case("p11", AArch64::P11)
      .Case("p12", AArch64::P12)
      .Case("p13", AArch64::P13)
      .Case("p14", AArch64::P14)
      .Case("p15", AArch64::P15)
      .Default(0);
}
 
static unsigned matchSVEPredicateAsCounterRegName(StringRef Name) {
  return StringSwitch<unsigned>(Name.lower())
      .Case("pn0", AArch64::PN0)
      .Case("pn1", AArch64::PN1)
      .Case("pn2", AArch64::PN2)
      .Case("pn3", AArch64::PN3)
      .Case("pn4", AArch64::PN4)
      .Case("pn5", AArch64::PN5)
      .Case("pn6", AArch64::PN6)
      .Case("pn7", AArch64::PN7)
      .Case("pn8", AArch64::PN8)
      .Case("pn9", AArch64::PN9)
      .Case("pn10", AArch64::PN10)
      .Case("pn11", AArch64::PN11)
      .Case("pn12", AArch64::PN12)
      .Case("pn13", AArch64::PN13)
      .Case("pn14", AArch64::PN14)
      .Case("pn15", AArch64::PN15)
      .Default(0);
}
 
static unsigned matchMatrixTileListRegName(StringRef Name) {
  return StringSwitch<unsigned>(Name.lower())
      .Case("za0.d", AArch64::ZAD0)
      .Case("za1.d", AArch64::ZAD1)
      .Case("za2.d", AArch64::ZAD2)
      .Case("za3.d", AArch64::ZAD3)
      .Case("za4.d", AArch64::ZAD4)
      .Case("za5.d", AArch64::ZAD5)
      .Case("za6.d", AArch64::ZAD6)
      .Case("za7.d", AArch64::ZAD7)
      .Case("za0.s", AArch64::ZAS0)
      .Case("za1.s", AArch64::ZAS1)
      .Case("za2.s", AArch64::ZAS2)
      .Case("za3.s", AArch64::ZAS3)
      .Case("za0.h", AArch64::ZAH0)
      .Case("za1.h", AArch64::ZAH1)
      .Case("za0.b", AArch64::ZAB0)
      .Default(0);
}
 
static unsigned matchMatrixRegName(StringRef Name) {
  return StringSwitch<unsigned>(Name.lower())
      .Case("za", AArch64::ZA)
      .Case("za0.q", AArch64::ZAQ0)
      .Case("za1.q", AArch64::ZAQ1)
      .Case("za2.q", AArch64::ZAQ2)
      .Case("za3.q", AArch64::ZAQ3)
      .Case("za4.q", AArch64::ZAQ4)
      .Case("za5.q", AArch64::ZAQ5)
      .Case("za6.q", AArch64::ZAQ6)
      .Case("za7.q", AArch64::ZAQ7)
      .Case("za8.q", AArch64::ZAQ8)
      .Case("za9.q", AArch64::ZAQ9)
      .Case("za10.q", AArch64::ZAQ10)
      .Case("za11.q", AArch64::ZAQ11)
      .Case("za12.q", AArch64::ZAQ12)
      .Case("za13.q", AArch64::ZAQ13)
      .Case("za14.q", AArch64::ZAQ14)
      .Case("za15.q", AArch64::ZAQ15)
      .Case("za0.d", AArch64::ZAD0)
      .Case("za1.d", AArch64::ZAD1)
      .Case("za2.d", AArch64::ZAD2)
      .Case("za3.d", AArch64::ZAD3)
      .Case("za4.d", AArch64::ZAD4)
      .Case("za5.d", AArch64::ZAD5)
      .Case("za6.d", AArch64::ZAD6)
      .Case("za7.d", AArch64::ZAD7)
      .Case("za0.s", AArch64::ZAS0)
      .Case("za1.s", AArch64::ZAS1)
      .Case("za2.s", AArch64::ZAS2)
      .Case("za3.s", AArch64::ZAS3)
      .Case("za0.h", AArch64::ZAH0)
      .Case("za1.h", AArch64::ZAH1)
      .Case("za0.b", AArch64::ZAB0)
      .Case("za0h.q", AArch64::ZAQ0)
      .Case("za1h.q", AArch64::ZAQ1)
      .Case("za2h.q", AArch64::ZAQ2)
      .Case("za3h.q", AArch64::ZAQ3)
      .Case("za4h.q", AArch64::ZAQ4)
      .Case("za5h.q", AArch64::ZAQ5)
      .Case("za6h.q", AArch64::ZAQ6)
      .Case("za7h.q", AArch64::ZAQ7)
      .Case("za8h.q", AArch64::ZAQ8)
      .Case("za9h.q", AArch64::ZAQ9)
      .Case("za10h.q", AArch64::ZAQ10)
      .Case("za11h.q", AArch64::ZAQ11)
      .Case("za12h.q", AArch64::ZAQ12)
      .Case("za13h.q", AArch64::ZAQ13)
      .Case("za14h.q", AArch64::ZAQ14)
      .Case("za15h.q", AArch64::ZAQ15)
      .Case("za0h.d", AArch64::ZAD0)
      .Case("za1h.d", AArch64::ZAD1)
      .Case("za2h.d", AArch64::ZAD2)
      .Case("za3h.d", AArch64::ZAD3)
      .Case("za4h.d", AArch64::ZAD4)
      .Case("za5h.d", AArch64::ZAD5)
      .Case("za6h.d", AArch64::ZAD6)
      .Case("za7h.d", AArch64::ZAD7)
      .Case("za0h.s", AArch64::ZAS0)
      .Case("za1h.s", AArch64::ZAS1)
      .Case("za2h.s", AArch64::ZAS2)
      .Case("za3h.s", AArch64::ZAS3)
      .Case("za0h.h", AArch64::ZAH0)
      .Case("za1h.h", AArch64::ZAH1)
      .Case("za0h.b", AArch64::ZAB0)
      .Case("za0v.q", AArch64::ZAQ0)
      .Case("za1v.q", AArch64::ZAQ1)
      .Case("za2v.q", AArch64::ZAQ2)
      .Case("za3v.q", AArch64::ZAQ3)
      .Case("za4v.q", AArch64::ZAQ4)
      .Case("za5v.q", AArch64::ZAQ5)
      .Case("za6v.q", AArch64::ZAQ6)
      .Case("za7v.q", AArch64::ZAQ7)
      .Case("za8v.q", AArch64::ZAQ8)
      .Case("za9v.q", AArch64::ZAQ9)
      .Case("za10v.q", AArch64::ZAQ10)
      .Case("za11v.q", AArch64::ZAQ11)
      .Case("za12v.q", AArch64::ZAQ12)
      .Case("za13v.q", AArch64::ZAQ13)
      .Case("za14v.q", AArch64::ZAQ14)
      .Case("za15v.q", AArch64::ZAQ15)
      .Case("za0v.d", AArch64::ZAD0)
      .Case("za1v.d", AArch64::ZAD1)
      .Case("za2v.d", AArch64::ZAD2)
      .Case("za3v.d", AArch64::ZAD3)
      .Case("za4v.d", AArch64::ZAD4)
      .Case("za5v.d", AArch64::ZAD5)
      .Case("za6v.d", AArch64::ZAD6)
      .Case("za7v.d", AArch64::ZAD7)
      .Case("za0v.s", AArch64::ZAS0)
      .Case("za1v.s", AArch64::ZAS1)
      .Case("za2v.s", AArch64::ZAS2)
      .Case("za3v.s", AArch64::ZAS3)
      .Case("za0v.h", AArch64::ZAH0)
      .Case("za1v.h", AArch64::ZAH1)
      .Case("za0v.b", AArch64::ZAB0)
      .Default(0);
}
 
bool AArch64AsmParser::parseRegister(MCRegister &Reg, SMLoc &StartLoc,
                                     SMLoc &EndLoc) {
  return !tryParseRegister(Reg, StartLoc, EndLoc).isSuccess();
}
 
ParseStatus AArch64AsmParser::tryParseRegister(MCRegister &Reg, SMLoc &StartLoc,
                                               SMLoc &EndLoc) {
  StartLoc = getLoc();
  ParseStatus Res = tryParseScalarRegister(Reg);
  EndLoc = SMLoc::getFromPointer(getLoc().getPointer() - 1);
  return Res;
}
 
// Matches a register name or register alias previously defined by '.req'
unsigned AArch64AsmParser::matchRegisterNameAlias(StringRef Name,
                                                  RegKind Kind) {
  unsigned RegNum = 0;
  if ((RegNum = matchSVEDataVectorRegName(Name)))
    return Kind == RegKind::SVEDataVector ? RegNum : 0;
 
  if ((RegNum = matchSVEPredicateVectorRegName(Name)))
    return Kind == RegKind::SVEPredicateVector ? RegNum : 0;
 
  if ((RegNum = matchSVEPredicateAsCounterRegName(Name)))
    return Kind == RegKind::SVEPredicateAsCounter ? RegNum : 0;
 
  if ((RegNum = MatchNeonVectorRegName(Name)))
    return Kind == RegKind::NeonVector ? RegNum : 0;
 
  if ((RegNum = matchMatrixRegName(Name)))
    return Kind == RegKind::Matrix ? RegNum : 0;
 
 if (Name.equals_insensitive("zt0"))
    return Kind == RegKind::LookupTable ? unsigned(AArch64::ZT0) : 0;
 
  // The parsed register must be of RegKind Scalar
  if ((RegNum = MatchRegisterName(Name)))
    return (Kind == RegKind::Scalar) ? RegNum : 0;
 
  if (!RegNum) {
    // Handle a few common aliases of registers.
    if (auto RegNum = StringSwitch<unsigned>(Name.lower())
                    .Case("fp", AArch64::FP)
                    .Case("lr",  AArch64::LR)
                    .Case("x31", AArch64::XZR)
                    .Case("w31", AArch64::WZR)
                    .Default(0))
      return Kind == RegKind::Scalar ? RegNum : 0;
 
    // Check for aliases registered via .req. Canonicalize to lower case.
    // That's more consistent since register names are case insensitive, and
    // it's how the original entry was passed in from MC/MCParser/AsmParser.
    auto Entry = RegisterReqs.find(Name.lower());
    if (Entry == RegisterReqs.end())
      return 0;
 
    // set RegNum if the match is the right kind of register
    if (Kind == Entry->getValue().first)
      RegNum = Entry->getValue().second;
  }
  return RegNum;
}
 
unsigned AArch64AsmParser::getNumRegsForRegKind(RegKind K) {
  switch (K) {
  case RegKind::Scalar:
  case RegKind::NeonVector:
  case RegKind::SVEDataVector:
    return 32;
  case RegKind::Matrix:
  case RegKind::SVEPredicateVector:
  case RegKind::SVEPredicateAsCounter:
    return 16;
  case RegKind::LookupTable:
    return 1;
  }
  llvm_unreachable("Unsupported RegKind");
}
 
/// tryParseScalarRegister - Try to parse a register name. The token must be an
/// Identifier when called, and if it is a register name the token is eaten and
/// the register is added to the operand list.
ParseStatus AArch64AsmParser::tryParseScalarRegister(MCRegister &RegNum) {
  const AsmToken &Tok = getTok();
  if (Tok.isNot(AsmToken::Identifier))
    return ParseStatus::NoMatch;
 
  std::string lowerCase = Tok.getString().lower();
  unsigned Reg = matchRegisterNameAlias(lowerCase, RegKind::Scalar);
  if (Reg == 0)
    return ParseStatus::NoMatch;
 
  RegNum = Reg;
  Lex(); // Eat identifier token.
  return ParseStatus::Success;
}
 
/// tryParseSysCROperand - Try to parse a system instruction CR operand name.
ParseStatus AArch64AsmParser::tryParseSysCROperand(OperandVector &Operands) {
  SMLoc S = getLoc();
 
  if (getTok().isNot(AsmToken::Identifier))
    return Error(S, "Expected cN operand where 0 <= N <= 15");
 
  StringRef Tok = getTok().getIdentifier();
  if (Tok[0] != 'c' && Tok[0] != 'C')
    return Error(S, "Expected cN operand where 0 <= N <= 15");
 
  uint32_t CRNum;
  bool BadNum = Tok.drop_front().getAsInteger(10, CRNum);
  if (BadNum || CRNum > 15)
    return Error(S, "Expected cN operand where 0 <= N <= 15");
 
  Lex(); // Eat identifier token.
  Operands.push_back(
      AArch64Operand::CreateSysCR(CRNum, S, getLoc(), getContext()));
  return ParseStatus::Success;
}
 
// Either an identifier for named values or a 6-bit immediate.
ParseStatus AArch64AsmParser::tryParseRPRFMOperand(OperandVector &Operands) {
  SMLoc S = getLoc();
  const AsmToken &Tok = getTok();
 
  unsigned MaxVal = 63;
 
  // Immediate case, with optional leading hash:
  if (parseOptionalToken(AsmToken::Hash) ||
      Tok.is(AsmToken::Integer)) {
    const MCExpr *ImmVal;
    if (getParser().parseExpression(ImmVal))
      return ParseStatus::Failure;
 
    const MCConstantExpr *MCE = dyn_cast<MCConstantExpr>(ImmVal);
    if (!MCE)
      return TokError("immediate value expected for prefetch operand");
    unsigned prfop = MCE->getValue();
    if (prfop > MaxVal)
      return TokError("prefetch operand out of range, [0," + utostr(MaxVal) +
                      "] expected");
 
    auto RPRFM = AArch64RPRFM::lookupRPRFMByEncoding(MCE->getValue());
    Operands.push_back(AArch64Operand::CreatePrefetch(
        prfop, RPRFM ? RPRFM->Name : "", S, getContext()));
    return ParseStatus::Success;
  }
 
  if (Tok.isNot(AsmToken::Identifier))
    return TokError("prefetch hint expected");
 
  auto RPRFM = AArch64RPRFM::lookupRPRFMByName(Tok.getString());
  if (!RPRFM)
    return TokError("prefetch hint expected");
 
  Operands.push_back(AArch64Operand::CreatePrefetch(
      RPRFM->Encoding, Tok.getString(), S, getContext()));
  Lex(); // Eat identifier token.
  return ParseStatus::Success;
}
 
/// tryParsePrefetch - Try to parse a prefetch operand.
template <bool IsSVEPrefetch>
ParseStatus AArch64AsmParser::tryParsePrefetch(OperandVector &Operands) {
  SMLoc S = getLoc();
  const AsmToken &Tok = getTok();
 
  auto LookupByName = [](StringRef N) {
    if (IsSVEPrefetch) {
      if (auto Res = AArch64SVEPRFM::lookupSVEPRFMByName(N))
        return std::optional<unsigned>(Res->Encoding);
    } else if (auto Res = AArch64PRFM::lookupPRFMByName(N))
      return std::optional<unsigned>(Res->Encoding);
    return std::optional<unsigned>();
  };
 
  auto LookupByEncoding = [](unsigned E) {
    if (IsSVEPrefetch) {
      if (auto Res = AArch64SVEPRFM::lookupSVEPRFMByEncoding(E))
        return std::optional<StringRef>(Res->Name);
    } else if (auto Res = AArch64PRFM::lookupPRFMByEncoding(E))
      return std::optional<StringRef>(Res->Name);
    return std::optional<StringRef>();
  };
  unsigned MaxVal = IsSVEPrefetch ? 15 : 31;
 
  // Either an identifier for named values or a 5-bit immediate.
  // Eat optional hash.
  if (parseOptionalToken(AsmToken::Hash) ||
      Tok.is(AsmToken::Integer)) {
    const MCExpr *ImmVal;
    if (getParser().parseExpression(ImmVal))
      return ParseStatus::Failure;
 
    const MCConstantExpr *MCE = dyn_cast<MCConstantExpr>(ImmVal);
    if (!MCE)
      return TokError("immediate value expected for prefetch operand");
    unsigned prfop = MCE->getValue();
    if (prfop > MaxVal)
      return TokError("prefetch operand out of range, [0," + utostr(MaxVal) +
                      "] expected");
 
    auto PRFM = LookupByEncoding(MCE->getValue());
    Operands.push_back(AArch64Operand::CreatePrefetch(prfop, PRFM.value_or(""),
                                                      S, getContext()));
    return ParseStatus::Success;
  }
 
  if (Tok.isNot(AsmToken::Identifier))
    return TokError("prefetch hint expected");
 
  auto PRFM = LookupByName(Tok.getString());
  if (!PRFM)
    return TokError("prefetch hint expected");
 
  Operands.push_back(AArch64Operand::CreatePrefetch(
      *PRFM, Tok.getString(), S, getContext()));
  Lex(); // Eat identifier token.
  return ParseStatus::Success;
}
 
/// tryParsePSBHint - Try to parse a PSB operand, mapped to Hint command
ParseStatus AArch64AsmParser::tryParsePSBHint(OperandVector &Operands) {
  SMLoc S = getLoc();
  const AsmToken &Tok = getTok();
  if (Tok.isNot(AsmToken::Identifier))
    return TokError("invalid operand for instruction");
 
  auto PSB = AArch64PSBHint::lookupPSBByName(Tok.getString());
  if (!PSB)
    return TokError("invalid operand for instruction");
 
  Operands.push_back(AArch64Operand::CreatePSBHint(
      PSB->Encoding, Tok.getString(), S, getContext()));
  Lex(); // Eat identifier token.
  return ParseStatus::Success;
}
 
ParseStatus AArch64AsmParser::tryParseSyspXzrPair(OperandVector &Operands) {
  SMLoc StartLoc = getLoc();
 
  MCRegister RegNum;
 
  // The case where xzr, xzr is not present is handled by an InstAlias.
 
  auto RegTok = getTok(); // in case we need to backtrack
  if (!tryParseScalarRegister(RegNum).isSuccess())
    return ParseStatus::NoMatch;
 
  if (RegNum != AArch64::XZR) {
    getLexer().UnLex(RegTok);
    return ParseStatus::NoMatch;
  }
 
  if (parseComma())
    return ParseStatus::Failure;
 
  if (!tryParseScalarRegister(RegNum).isSuccess())
    return TokError("expected register operand");
 
  if (RegNum != AArch64::XZR)
    return TokError("xzr must be followed by xzr");
 
  // We need to push something, since we claim this is an operand in .td.
  // See also AArch64AsmParser::parseKeywordOperand.
  Operands.push_back(AArch64Operand::CreateReg(
      RegNum, RegKind::Scalar, StartLoc, getLoc(), getContext()));
 
  return ParseStatus::Success;
}
 
/// tryParseBTIHint - Try to parse a BTI operand, mapped to Hint command
ParseStatus AArch64AsmParser::tryParseBTIHint(OperandVector &Operands) {
  SMLoc S = getLoc();
  const AsmToken &Tok = getTok();
  if (Tok.isNot(AsmToken::Identifier))
    return TokError("invalid operand for instruction");
 
  auto BTI = AArch64BTIHint::lookupBTIByName(Tok.getString());
  if (!BTI)
    return TokError("invalid operand for instruction");
 
  Operands.push_back(AArch64Operand::CreateBTIHint(
      BTI->Encoding, Tok.getString(), S, getContext()));
  Lex(); // Eat identifier token.
  return ParseStatus::Success;
}
 
/// tryParseAdrpLabel - Parse and validate a source label for the ADRP
/// instruction.
ParseStatus AArch64AsmParser::tryParseAdrpLabel(OperandVector &Operands) {
  SMLoc S = getLoc();
  const MCExpr *Expr = nullptr;
 
  if (getTok().is(AsmToken::Hash)) {
    Lex(); // Eat hash token.
  }
 
  if (parseSymbolicImmVal(Expr))
    return ParseStatus::Failure;
 
  AArch64MCExpr::VariantKind ELFRefKind;
  MCSymbolRefExpr::VariantKind DarwinRefKind;
  int64_t Addend;
  if (classifySymbolRef(Expr, ELFRefKind, DarwinRefKind, Addend)) {
    if (DarwinRefKind == MCSymbolRefExpr::VK_None &&
        ELFRefKind == AArch64MCExpr::VK_INVALID) {
      // No modifier was specified at all; this is the syntax for an ELF basic
      // ADRP relocation (unfortunately).
      Expr =
          AArch64MCExpr::create(Expr, AArch64MCExpr::VK_ABS_PAGE, getContext());
    } else if ((DarwinRefKind == MCSymbolRefExpr::VK_GOTPAGE ||
                DarwinRefKind == MCSymbolRefExpr::VK_TLVPPAGE) &&
               Addend != 0) {
      return Error(S, "gotpage label reference not allowed an addend");
    } else if (DarwinRefKind != MCSymbolRefExpr::VK_PAGE &&
               DarwinRefKind != MCSymbolRefExpr::VK_GOTPAGE &&
               DarwinRefKind != MCSymbolRefExpr::VK_TLVPPAGE &&
               ELFRefKind != AArch64MCExpr::VK_ABS_PAGE_NC &&
               ELFRefKind != AArch64MCExpr::VK_GOT_PAGE &&
               ELFRefKind != AArch64MCExpr::VK_GOT_AUTH_PAGE &&
               ELFRefKind != AArch64MCExpr::VK_GOT_PAGE_LO15 &&
               ELFRefKind != AArch64MCExpr::VK_GOTTPREL_PAGE &&
               ELFRefKind != AArch64MCExpr::VK_TLSDESC_PAGE &&
               ELFRefKind != AArch64MCExpr::VK_TLSDESC_AUTH_PAGE) {
      // The operand must be an @page or @gotpage qualified symbolref.
      return Error(S, "page or gotpage label reference expected");
    }
  }
 
  // We have either a label reference possibly with addend or an immediate. The
  // addend is a raw value here. The linker will adjust it to only reference the
  // page.
  SMLoc E = SMLoc::getFromPointer(getLoc().getPointer() - 1);
  Operands.push_back(AArch64Operand::CreateImm(Expr, S, E, getContext()));
 
  return ParseStatus::Success;
}
 
/// tryParseAdrLabel - Parse and validate a source label for the ADR
/// instruction.
ParseStatus AArch64AsmParser::tryParseAdrLabel(OperandVector &Operands) {
  SMLoc S = getLoc();
  const MCExpr *Expr = nullptr;
 
  // Leave anything with a bracket to the default for SVE
  if (getTok().is(AsmToken::LBrac))
    return ParseStatus::NoMatch;
 
  if (getTok().is(AsmToken::Hash))
    Lex(); // Eat hash token.
 
  if (parseSymbolicImmVal(Expr))
    return ParseStatus::Failure;
 
  AArch64MCExpr::VariantKind ELFRefKind;
  MCSymbolRefExpr::VariantKind DarwinRefKind;
  int64_t Addend;
  if (classifySymbolRef(Expr, ELFRefKind, DarwinRefKind, Addend)) {
    if (DarwinRefKind == MCSymbolRefExpr::VK_None &&
        ELFRefKind == AArch64MCExpr::VK_INVALID) {
      // No modifier was specified at all; this is the syntax for an ELF basic
      // ADR relocation (unfortunately).
      Expr = AArch64MCExpr::create(Expr, AArch64MCExpr::VK_ABS, getContext());
    } else if (ELFRefKind != AArch64MCExpr::VK_GOT_AUTH_PAGE) {
      // For tiny code model, we use :got_auth: operator to fill 21-bit imm of
      // adr. It's not actually GOT entry page address but the GOT address
      // itself - we just share the same variant kind with :got_auth: operator
      // applied for adrp.
      // TODO: can we somehow get current TargetMachine object to call
      // getCodeModel() on it to ensure we are using tiny code model?
      return Error(S, "unexpected adr label");
    }
  }
 
  SMLoc E = SMLoc::getFromPointer(getLoc().getPointer() - 1);
  Operands.push_back(AArch64Operand::CreateImm(Expr, S, E, getContext()));
  return ParseStatus::Success;
}
 
/// tryParseFPImm - A floating point immediate expression operand.
template <bool AddFPZeroAsLiteral>
ParseStatus AArch64AsmParser::tryParseFPImm(OperandVector &Operands) {
  SMLoc S = getLoc();
 
  bool Hash = parseOptionalToken(AsmToken::Hash);
 
  // Handle negation, as that still comes through as a separate token.
  bool isNegative = parseOptionalToken(AsmToken::Minus);
 
  const AsmToken &Tok = getTok();
  if (!Tok.is(AsmToken::Real) && !Tok.is(AsmToken::Integer)) {
    if (!Hash)
      return ParseStatus::NoMatch;
    return TokError("invalid floating point immediate");
  }
 
  // Parse hexadecimal representation.
  if (Tok.is(AsmToken::Integer) && Tok.getString().starts_with("0x")) {
    if (Tok.getIntVal() > 255 || isNegative)
      return TokError("encoded floating point value out of range");
 
    APFloat F((double)AArch64_AM::getFPImmFloat(Tok.getIntVal()));
    Operands.push_back(
        AArch64Operand::CreateFPImm(F, true, S, getContext()));
  } else {
    // Parse FP representation.
    APFloat RealVal(APFloat::IEEEdouble());
    auto StatusOrErr =
        RealVal.convertFromString(Tok.getString(), APFloat::rmTowardZero);
    if (errorToBool(StatusOrErr.takeError()))
      return TokError("invalid floating point representation");
 
    if (isNegative)
      RealVal.changeSign();
 
    if (AddFPZeroAsLiteral && RealVal.isPosZero()) {
      Operands.push_back(AArch64Operand::CreateToken("#0", S, getContext()));
      Operands.push_back(AArch64Operand::CreateToken(".0", S, getContext()));
    } else
      Operands.push_back(AArch64Operand::CreateFPImm(
          RealVal, *StatusOrErr == APFloat::opOK, S, getContext()));
  }
 
  Lex(); // Eat the token.
 
  return ParseStatus::Success;
}
 
/// tryParseImmWithOptionalShift - Parse immediate operand, optionally with
/// a shift suffix, for example '#1, lsl #12'.
ParseStatus
AArch64AsmParser::tryParseImmWithOptionalShift(OperandVector &Operands) {
  SMLoc S = getLoc();
 
  if (getTok().is(AsmToken::Hash))
    Lex(); // Eat '#'
  else if (getTok().isNot(AsmToken::Integer))
    // Operand should start from # or should be integer, emit error otherwise.
    return ParseStatus::NoMatch;
 
  if (getTok().is(AsmToken::Integer) &&
      getLexer().peekTok().is(AsmToken::Colon))
    return tryParseImmRange(Operands);
 
  const MCExpr *Imm = nullptr;
  if (parseSymbolicImmVal(Imm))
    return ParseStatus::Failure;
  else if (getTok().isNot(AsmToken::Comma)) {
    Operands.push_back(
        AArch64Operand::CreateImm(Imm, S, getLoc(), getContext()));
    return ParseStatus::Success;
  }
 
  // Eat ','
  Lex();
  StringRef VecGroup;
  if (!parseOptionalVGOperand(Operands, VecGroup)) {
    Operands.push_back(
        AArch64Operand::CreateImm(Imm, S, getLoc(), getContext()));
    Operands.push_back(
        AArch64Operand::CreateToken(VecGroup, getLoc(), getContext()));
    return ParseStatus::Success;
  }
 
  // The optional operand must be "lsl #N" where N is non-negative.
  if (!getTok().is(AsmToken::Identifier) ||
      !getTok().getIdentifier().equals_insensitive("lsl"))
    return Error(getLoc(), "only 'lsl #+N' valid after immediate");
 
  // Eat 'lsl'
  Lex();
 
  parseOptionalToken(AsmToken::Hash);
 
  if (getTok().isNot(AsmToken::Integer))
    return Error(getLoc(), "only 'lsl #+N' valid after immediate");
 
  int64_t ShiftAmount = getTok().getIntVal();
 
  if (ShiftAmount < 0)
    return Error(getLoc(), "positive shift amount required");
  Lex(); // Eat the number
 
  // Just in case the optional lsl #0 is used for immediates other than zero.
  if (ShiftAmount == 0 && Imm != nullptr) {
    Operands.push_back(
        AArch64Operand::CreateImm(Imm, S, getLoc(), getContext()));
    return ParseStatus::Success;
  }
 
  Operands.push_back(AArch64Operand::CreateShiftedImm(Imm, ShiftAmount, S,
                                                      getLoc(), getContext()));
  return ParseStatus::Success;
}
 
/// parseCondCodeString - Parse a Condition Code string, optionally returning a
/// suggestion to help common typos.
AArch64CC::CondCode
AArch64AsmParser::parseCondCodeString(StringRef Cond, std::string &Suggestion) {
  AArch64CC::CondCode CC = StringSwitch<AArch64CC::CondCode>(Cond.lower())
                    .Case("eq", AArch64CC::EQ)
                    .Case("ne", AArch64CC::NE)
                    .Case("cs", AArch64CC::HS)
                    .Case("hs", AArch64CC::HS)
                    .Case("cc", AArch64CC::LO)
                    .Case("lo", AArch64CC::LO)
                    .Case("mi", AArch64CC::MI)
                    .Case("pl", AArch64CC::PL)
                    .Case("vs", AArch64CC::VS)
                    .Case("vc", AArch64CC::VC)
                    .Case("hi", AArch64CC::HI)
                    .Case("ls", AArch64CC::LS)
                    .Case("ge", AArch64CC::GE)
                    .Case("lt", AArch64CC::LT)
                    .Case("gt", AArch64CC::GT)
                    .Case("le", AArch64CC::LE)
                    .Case("al", AArch64CC::AL)
                    .Case("nv", AArch64CC::NV)
                    .Default(AArch64CC::Invalid);
 
  if (CC == AArch64CC::Invalid && getSTI().hasFeature(AArch64::FeatureSVE)) {
    CC = StringSwitch<AArch64CC::CondCode>(Cond.lower())
                    .Case("none",  AArch64CC::EQ)
                    .Case("any",   AArch64CC::NE)
                    .Case("nlast", AArch64CC::HS)
                    .Case("last",  AArch64CC::LO)
                    .Case("first", AArch64CC::MI)
                    .Case("nfrst", AArch64CC::PL)
                    .Case("pmore", AArch64CC::HI)
                    .Case("plast", AArch64CC::LS)
                    .Case("tcont", AArch64CC::GE)
                    .Case("tstop", AArch64CC::LT)
                    .Default(AArch64CC::Invalid);
 
    if (CC == AArch64CC::Invalid && Cond.lower() == "nfirst")
      Suggestion = "nfrst";
  }
  return CC;
}
 
/// parseCondCode - Parse a Condition Code operand.
bool AArch64AsmParser::parseCondCode(OperandVector &Operands,
                                     bool invertCondCode) {
  SMLoc S = getLoc();
  const AsmToken &Tok = getTok();
  assert(Tok.is(AsmToken::Identifier) && "Token is not an Identifier");
 
  StringRef Cond = Tok.getString();
  std::string Suggestion;
  AArch64CC::CondCode CC = parseCondCodeString(Cond, Suggestion);
  if (CC == AArch64CC::Invalid) {
    std::string Msg = "invalid condition code";
    if (!Suggestion.empty())
      Msg += ", did you mean " + Suggestion + "?";
    return TokError(Msg);
  }
  Lex(); // Eat identifier token.
 
  if (invertCondCode) {
    if (CC == AArch64CC::AL || CC == AArch64CC::NV)
      return TokError("condition codes AL and NV are invalid for this instruction");
    CC = AArch64CC::getInvertedCondCode(AArch64CC::CondCode(CC));
  }
 
  Operands.push_back(
      AArch64Operand::CreateCondCode(CC, S, getLoc(), getContext()));
  return false;
}
 
ParseStatus AArch64AsmParser::tryParseSVCR(OperandVector &Operands) {
  const AsmToken &Tok = getTok();
  SMLoc S = getLoc();
 
  if (Tok.isNot(AsmToken::Identifier))
    return TokError("invalid operand for instruction");
 
  unsigned PStateImm = -1;
  const auto *SVCR = AArch64SVCR::lookupSVCRByName(Tok.getString());
  if (!SVCR)
    return ParseStatus::NoMatch;
  if (SVCR->haveFeatures(getSTI().getFeatureBits()))
    PStateImm = SVCR->Encoding;
 
  Operands.push_back(
      AArch64Operand::CreateSVCR(PStateImm, Tok.getString(), S, getContext()));
  Lex(); // Eat identifier token.
  return ParseStatus::Success;
}
 
ParseStatus AArch64AsmParser::tryParseMatrixRegister(OperandVector &Operands) {
  const AsmToken &Tok = getTok();
  SMLoc S = getLoc();
 
  StringRef Name = Tok.getString();
 
  if (Name.equals_insensitive("za") || Name.starts_with_insensitive("za.")) {
    Lex(); // eat "za[.(b|h|s|d)]"
    unsigned ElementWidth = 0;
    auto DotPosition = Name.find('.');
    if (DotPosition != StringRef::npos) {
      const auto &KindRes =
          parseVectorKind(Name.drop_front(DotPosition), RegKind::Matrix);
      if (!KindRes)
        return TokError(
            "Expected the register to be followed by element width suffix");
      ElementWidth = KindRes->second;
    }
    Operands.push_back(AArch64Operand::CreateMatrixRegister(
        AArch64::ZA, ElementWidth, MatrixKind::Array, S, getLoc(),
        getContext()));
    if (getLexer().is(AsmToken::LBrac)) {
      // There's no comma after matrix operand, so we can parse the next operand
      // immediately.
      if (parseOperand(Operands, false, false))
        return ParseStatus::NoMatch;
    }
    return ParseStatus::Success;
  }
 
  // Try to parse matrix register.
  unsigned Reg = matchRegisterNameAlias(Name, RegKind::Matrix);
  if (!Reg)
    return ParseStatus::NoMatch;
 
  size_t DotPosition = Name.find('.');
  assert(DotPosition != StringRef::npos && "Unexpected register");
 
  StringRef Head = Name.take_front(DotPosition);
  StringRef Tail = Name.drop_front(DotPosition);
  StringRef RowOrColumn = Head.take_back();
 
  MatrixKind Kind = StringSwitch<MatrixKind>(RowOrColumn.lower())
                        .Case("h", MatrixKind::Row)
                        .Case("v", MatrixKind::Col)
                        .Default(MatrixKind::Tile);
 
  // Next up, parsing the suffix
  const auto &KindRes = parseVectorKind(Tail, RegKind::Matrix);
  if (!KindRes)
    return TokError(
        "Expected the register to be followed by element width suffix");
  unsigned ElementWidth = KindRes->second;
 
  Lex();
 
  Operands.push_back(AArch64Operand::CreateMatrixRegister(
      Reg, ElementWidth, Kind, S, getLoc(), getContext()));
 
  if (getLexer().is(AsmToken::LBrac)) {
    // There's no comma after matrix operand, so we can parse the next operand
    // immediately.
    if (parseOperand(Operands, false, false))
      return ParseStatus::NoMatch;
  }
  return ParseStatus::Success;
}
 
/// tryParseOptionalShift - Some operands take an optional shift argument. Parse
/// them if present.
ParseStatus
AArch64AsmParser::tryParseOptionalShiftExtend(OperandVector &Operands) {
  const AsmToken &Tok = getTok();
  std::string LowerID = Tok.getString().lower();
  AArch64_AM::ShiftExtendType ShOp =
      StringSwitch<AArch64_AM::ShiftExtendType>(LowerID)
          .Case("lsl", AArch64_AM::LSL)
          .Case("lsr", AArch64_AM::LSR)
          .Case("asr", AArch64_AM::ASR)
          .Case("ror", AArch64_AM::ROR)
          .Case("msl", AArch64_AM::MSL)
          .Case("uxtb", AArch64_AM::UXTB)
          .Case("uxth", AArch64_AM::UXTH)
          .Case("uxtw", AArch64_AM::UXTW)
          .Case("uxtx", AArch64_AM::UXTX)
          .Case("sxtb", AArch64_AM::SXTB)
          .Case("sxth", AArch64_AM::SXTH)
          .Case("sxtw", AArch64_AM::SXTW)
          .Case("sxtx", AArch64_AM::SXTX)
          .Default(AArch64_AM::InvalidShiftExtend);
 
  if (ShOp == AArch64_AM::InvalidShiftExtend)
    return ParseStatus::NoMatch;
 
  SMLoc S = Tok.getLoc();
  Lex();
 
  bool Hash = parseOptionalToken(AsmToken::Hash);
 
  if (!Hash && getLexer().isNot(AsmToken::Integer)) {
    if (ShOp == AArch64_AM::LSL || ShOp == AArch64_AM::LSR ||
        ShOp == AArch64_AM::ASR || ShOp == AArch64_AM::ROR ||
        ShOp == AArch64_AM::MSL) {
      // We expect a number here.
      return TokError("expected #imm after shift specifier");
    }
 
    // "extend" type operations don't need an immediate, #0 is implicit.
    SMLoc E = SMLoc::getFromPointer(getLoc().getPointer() - 1);
    Operands.push_back(
        AArch64Operand::CreateShiftExtend(ShOp, 0, false, S, E, getContext()));
    return ParseStatus::Success;
  }
 
  // Make sure we do actually have a number, identifier or a parenthesized
  // expression.
  SMLoc E = getLoc();
  if (!getTok().is(AsmToken::Integer) && !getTok().is(AsmToken::LParen) &&
      !getTok().is(AsmToken::Identifier))
    return Error(E, "expected integer shift amount");
 
  const MCExpr *ImmVal;
  if (getParser().parseExpression(ImmVal))
    return ParseStatus::Failure;
 
  const MCConstantExpr *MCE = dyn_cast<MCConstantExpr>(ImmVal);
  if (!MCE)
    return Error(E, "expected constant '#imm' after shift specifier");
 
  E = SMLoc::getFromPointer(getLoc().getPointer() - 1);
  Operands.push_back(AArch64Operand::CreateShiftExtend(
      ShOp, MCE->getValue(), true, S, E, getContext()));
  return ParseStatus::Success;
}
 
static const struct Extension {
  const char *Name;
  const FeatureBitset Features;
} ExtensionMap[] = {
    {"crc", {AArch64::FeatureCRC}},
    {"sm4", {AArch64::FeatureSM4}},
    {"sha3", {AArch64::FeatureSHA3}},
    {"sha2", {AArch64::FeatureSHA2}},
    {"aes", {AArch64::FeatureAES}},
    {"crypto", {AArch64::FeatureCrypto}},
    {"fp", {AArch64::FeatureFPARMv8}},
    {"simd", {AArch64::FeatureNEON}},
    {"ras", {AArch64::FeatureRAS}},
    {"rasv2", {AArch64::FeatureRASv2}},
    {"lse", {AArch64::FeatureLSE}},
    {"predres", {AArch64::FeaturePredRes}},
    {"predres2", {AArch64::FeatureSPECRES2}},
    {"ccdp", {AArch64::FeatureCacheDeepPersist}},
    {"mte", {AArch64::FeatureMTE}},
    {"memtag", {AArch64::FeatureMTE}},
    {"tlb-rmi", {AArch64::FeatureTLB_RMI}},
    {"pan", {AArch64::FeaturePAN}},
    {"pan-rwv", {AArch64::FeaturePAN_RWV}},
    {"ccpp", {AArch64::FeatureCCPP}},
    {"rcpc", {AArch64::FeatureRCPC}},
    {"rng", {AArch64::FeatureRandGen}},
    {"sve", {AArch64::FeatureSVE}},
    {"sve-b16b16", {AArch64::FeatureSVEB16B16}},
    {"sve2", {AArch64::FeatureSVE2}},
    {"sve-aes", {AArch64::FeatureSVEAES}},
    {"sve2-aes", {AArch64::FeatureAliasSVE2AES, AArch64::FeatureSVEAES}},
    {"sve2-sm4", {AArch64::FeatureSVE2SM4}},
    {"sve2-sha3", {AArch64::FeatureSVE2SHA3}},
    {"sve2-bitperm", {AArch64::FeatureSVE2BitPerm}},
    {"sve2p1", {AArch64::FeatureSVE2p1}},
    {"ls64", {AArch64::FeatureLS64}},
    {"xs", {AArch64::FeatureXS}},
    {"pauth", {AArch64::FeaturePAuth}},
    {"flagm", {AArch64::FeatureFlagM}},
    {"rme", {AArch64::FeatureRME}},
    {"sme", {AArch64::FeatureSME}},
    {"sme-f64f64", {AArch64::FeatureSMEF64F64}},
    {"sme-f16f16", {AArch64::FeatureSMEF16F16}},
    {"sme-i16i64", {AArch64::FeatureSMEI16I64}},
    {"sme2", {AArch64::FeatureSME2}},
    {"sme2p1", {AArch64::FeatureSME2p1}},
    {"sme-b16b16", {AArch64::FeatureSMEB16B16}},
    {"hbc", {AArch64::FeatureHBC}},
    {"mops", {AArch64::FeatureMOPS}},
    {"mec", {AArch64::FeatureMEC}},
    {"the", {AArch64::FeatureTHE}},
    {"d128", {AArch64::FeatureD128}},
    {"lse128", {AArch64::FeatureLSE128}},
    {"ite", {AArch64::FeatureITE}},
    {"cssc", {AArch64::FeatureCSSC}},
    {"rcpc3", {AArch64::FeatureRCPC3}},
    {"gcs", {AArch64::FeatureGCS}},
    {"bf16", {AArch64::FeatureBF16}},
    {"compnum", {AArch64::FeatureComplxNum}},
    {"dotprod", {AArch64::FeatureDotProd}},
    {"f32mm", {AArch64::FeatureMatMulFP32}},
    {"f64mm", {AArch64::FeatureMatMulFP64}},
    {"fp16", {AArch64::FeatureFullFP16}},
    {"fp16fml", {AArch64::FeatureFP16FML}},
    {"i8mm", {AArch64::FeatureMatMulInt8}},
    {"lor", {AArch64::FeatureLOR}},
    {"profile", {AArch64::FeatureSPE}},
    // "rdma" is the name documented by binutils for the feature, but
    // binutils also accepts incomplete prefixes of features, so "rdm"
    // works too. Support both spellings here.
    {"rdm", {AArch64::FeatureRDM}},
    {"rdma", {AArch64::FeatureRDM}},
    {"sb", {AArch64::FeatureSB}},
    {"ssbs", {AArch64::FeatureSSBS}},
    {"tme", {AArch64::FeatureTME}},
    {"fp8", {AArch64::FeatureFP8}},
    {"faminmax", {AArch64::FeatureFAMINMAX}},
    {"fp8fma", {AArch64::FeatureFP8FMA}},
    {"ssve-fp8fma", {AArch64::FeatureSSVE_FP8FMA}},
    {"fp8dot2", {AArch64::FeatureFP8DOT2}},
    {"ssve-fp8dot2", {AArch64::FeatureSSVE_FP8DOT2}},
    {"fp8dot4", {AArch64::FeatureFP8DOT4}},
    {"ssve-fp8dot4", {AArch64::FeatureSSVE_FP8DOT4}},
    {"lut", {AArch64::FeatureLUT}},
    {"sme-lutv2", {AArch64::FeatureSME_LUTv2}},
    {"sme-f8f16", {AArch64::FeatureSMEF8F16}},
    {"sme-f8f32", {AArch64::FeatureSMEF8F32}},
    {"sme-fa64", {AArch64::FeatureSMEFA64}},
    {"cpa", {AArch64::FeatureCPA}},
    {"tlbiw", {AArch64::FeatureTLBIW}},
    {"pops", {AArch64::FeaturePoPS}},
    {"cmpbr", {AArch64::FeatureCMPBR}},
    {"f8f32mm", {AArch64::FeatureF8F32MM}},
    {"f8f16mm", {AArch64::FeatureF8F16MM}},
    {"fprcvt", {AArch64::FeatureFPRCVT}},
    {"lsfe", {AArch64::FeatureLSFE}},
    {"sme2p2", {AArch64::FeatureSME2p2}},
    {"ssve-aes", {AArch64::FeatureSSVE_AES}},
    {"sve2p2", {AArch64::FeatureSVE2p2}},
    {"sve-aes2", {AArch64::FeatureSVEAES2}},
    {"sve-bfscale", {AArch64::FeatureSVEBFSCALE}},
    {"sve-f16f32mm", {AArch64::FeatureSVE_F16F32MM}},
    {"lsui", {AArch64::FeatureLSUI}},
    {"occmo", {AArch64::FeatureOCCMO}},
    {"pcdphint", {AArch64::FeaturePCDPHINT}},
};
 
static void setRequiredFeatureString(FeatureBitset FBS, std::string &Str) {
  if (FBS[AArch64::HasV8_0aOps])
    Str += "ARMv8a";
  if (FBS[AArch64::HasV8_1aOps])
    Str += "ARMv8.1a";
  else if (FBS[AArch64::HasV8_2aOps])
    Str += "ARMv8.2a";
  else if (FBS[AArch64::HasV8_3aOps])
    Str += "ARMv8.3a";
  else if (FBS[AArch64::HasV8_4aOps])
    Str += "ARMv8.4a";
  else if (FBS[AArch64::HasV8_5aOps])
    Str += "ARMv8.5a";
  else if (FBS[AArch64::HasV8_6aOps])
    Str += "ARMv8.6a";
  else if (FBS[AArch64::HasV8_7aOps])
    Str += "ARMv8.7a";
  else if (FBS[AArch64::HasV8_8aOps])
    Str += "ARMv8.8a";
  else if (FBS[AArch64::HasV8_9aOps])
    Str += "ARMv8.9a";
  else if (FBS[AArch64::HasV9_0aOps])
    Str += "ARMv9-a";
  else if (FBS[AArch64::HasV9_1aOps])
    Str += "ARMv9.1a";
  else if (FBS[AArch64::HasV9_2aOps])
    Str += "ARMv9.2a";
  else if (FBS[AArch64::HasV9_3aOps])
    Str += "ARMv9.3a";
  else if (FBS[AArch64::HasV9_4aOps])
    Str += "ARMv9.4a";
  else if (FBS[AArch64::HasV9_5aOps])
    Str += "ARMv9.5a";
  else if (FBS[AArch64::HasV9_6aOps])
    Str += "ARMv9.6a";
  else if (FBS[AArch64::HasV8_0rOps])
    Str += "ARMv8r";
  else {
    SmallVector<std::string, 2> ExtMatches;
    for (const auto& Ext : ExtensionMap) {
      // Use & in case multiple features are enabled
      if ((FBS & Ext.Features) != FeatureBitset())
        ExtMatches.push_back(Ext.Name);
    }
    Str += !ExtMatches.empty() ? llvm::join(ExtMatches, ", ") : "(unknown)";
  }
}
 
void AArch64AsmParser::createSysAlias(uint16_t Encoding, OperandVector &Operands,
                                      SMLoc S) {
  const uint16_t Op2 = Encoding & 7;
  const uint16_t Cm = (Encoding & 0x78) >> 3;
  const uint16_t Cn = (Encoding & 0x780) >> 7;
  const uint16_t Op1 = (Encoding & 0x3800) >> 11;
 
  const MCExpr *Expr = MCConstantExpr::create(Op1, getContext());
 
  Operands.push_back(
      AArch64Operand::CreateImm(Expr, S, getLoc(), getContext()));
  Operands.push_back(
      AArch64Operand::CreateSysCR(Cn, S, getLoc(), getContext()));
  Operands.push_back(
      AArch64Operand::CreateSysCR(Cm, S, getLoc(), getContext()));
  Expr = MCConstantExpr::create(Op2, getContext());
  Operands.push_back(
      AArch64Operand::CreateImm(Expr, S, getLoc(), getContext()));
}
 
/// parseSysAlias - The IC, DC, AT, and TLBI instructions are simple aliases for
/// the SYS instruction. Parse them specially so that we create a SYS MCInst.
bool AArch64AsmParser::parseSysAlias(StringRef Name, SMLoc NameLoc,
                                   OperandVector &Operands) {
  if (Name.contains('.'))
    return TokError("invalid operand");
 
  Mnemonic = Name;
  Operands.push_back(AArch64Operand::CreateToken("sys", NameLoc, getContext()));
 
  const AsmToken &Tok = getTok();
  StringRef Op = Tok.getString();
  SMLoc S = Tok.getLoc();
 
  if (Mnemonic == "ic") {
    const AArch64IC::IC *IC = AArch64IC::lookupICByName(Op);
    if (!IC)
      return TokError("invalid operand for IC instruction");
    else if (!IC->haveFeatures(getSTI().getFeatureBits())) {
      std::string Str("IC " + std::string(IC->Name) + " requires: ");
      setRequiredFeatureString(IC->getRequiredFeatures(), Str);
      return TokError(Str);
    }
    createSysAlias(IC->Encoding, Operands, S);
  } else if (Mnemonic == "dc") {
    const AArch64DC::DC *DC = AArch64DC::lookupDCByName(Op);
    if (!DC)
      return TokError("invalid operand for DC instruction");
    else if (!DC->haveFeatures(getSTI().getFeatureBits())) {
      std::string Str("DC " + std::string(DC->Name) + " requires: ");
      setRequiredFeatureString(DC->getRequiredFeatures(), Str);
      return TokError(Str);
    }
    createSysAlias(DC->Encoding, Operands, S);
  } else if (Mnemonic == "at") {
    const AArch64AT::AT *AT = AArch64AT::lookupATByName(Op);
    if (!AT)
      return TokError("invalid operand for AT instruction");
    else if (!AT->haveFeatures(getSTI().getFeatureBits())) {
      std::string Str("AT " + std::string(AT->Name) + " requires: ");
      setRequiredFeatureString(AT->getRequiredFeatures(), Str);
      return TokError(Str);
    }
    createSysAlias(AT->Encoding, Operands, S);
  } else if (Mnemonic == "tlbi") {
    const AArch64TLBI::TLBI *TLBI = AArch64TLBI::lookupTLBIByName(Op);
    if (!TLBI)
      return TokError("invalid operand for TLBI instruction");
    else if (!TLBI->haveFeatures(getSTI().getFeatureBits())) {
      std::string Str("TLBI " + std::string(TLBI->Name) + " requires: ");
      setRequiredFeatureString(TLBI->getRequiredFeatures(), Str);
      return TokError(Str);
    }
    createSysAlias(TLBI->Encoding, Operands, S);
  } else if (Mnemonic == "cfp" || Mnemonic == "dvp" || Mnemonic == "cpp" || Mnemonic == "cosp") {
 
    if (Op.lower() != "rctx")
      return TokError("invalid operand for prediction restriction instruction");
 
    bool hasAll = getSTI().hasFeature(AArch64::FeatureAll);
    bool hasPredres = hasAll || getSTI().hasFeature(AArch64::FeaturePredRes);
    bool hasSpecres2 = hasAll || getSTI().hasFeature(AArch64::FeatureSPECRES2);
 
    if (Mnemonic == "cosp" && !hasSpecres2)
      return TokError("COSP requires: predres2");
    if (!hasPredres)
      return TokError(Mnemonic.upper() + "RCTX requires: predres");
 
    uint16_t PRCTX_Op2 = Mnemonic == "cfp"    ? 0b100
                         : Mnemonic == "dvp"  ? 0b101
                         : Mnemonic == "cosp" ? 0b110
                         : Mnemonic == "cpp"  ? 0b111
                                              : 0;
    assert(PRCTX_Op2 &&
           "Invalid mnemonic for prediction restriction instruction");
    const auto SYS_3_7_3 = 0b01101110011; // op=3, CRn=7, CRm=3
    const auto Encoding = SYS_3_7_3 << 3 | PRCTX_Op2;
 
    createSysAlias(Encoding, Operands, S);
  }
 
  Lex(); // Eat operand.
 
  bool ExpectRegister = !Op.contains_insensitive("all");
  bool HasRegister = false;
 
  // Check for the optional register operand.
  if (parseOptionalToken(AsmToken::Comma)) {
    if (Tok.isNot(AsmToken::Identifier) || parseRegister(Operands))
      return TokError("expected register operand");
    HasRegister = true;
  }
 
  if (ExpectRegister && !HasRegister)
    return TokError("specified " + Mnemonic + " op requires a register");
  else if (!ExpectRegister && HasRegister)
    return TokError("specified " + Mnemonic + " op does not use a register");
 
  if (parseToken(AsmToken::EndOfStatement, "unexpected token in argument list"))
    return true;
 
  return false;
}
 
/// parseSyspAlias - The TLBIP instructions are simple aliases for
/// the SYSP instruction. Parse them specially so that we create a SYSP MCInst.
bool AArch64AsmParser::parseSyspAlias(StringRef Name, SMLoc NameLoc,
                                      OperandVector &Operands) {
  if (Name.contains('.'))
    return TokError("invalid operand");
 
  Mnemonic = Name;
  Operands.push_back(
      AArch64Operand::CreateToken("sysp", NameLoc, getContext()));
 
  const AsmToken &Tok = getTok();
  StringRef Op = Tok.getString();
  SMLoc S = Tok.getLoc();
 
  if (Mnemonic == "tlbip") {
    bool HasnXSQualifier = Op.ends_with_insensitive("nXS");
    if (HasnXSQualifier) {
      Op = Op.drop_back(3);
    }
    const AArch64TLBI::TLBI *TLBIorig = AArch64TLBI::lookupTLBIByName(Op);
    if (!TLBIorig)
      return TokError("invalid operand for TLBIP instruction");
    const AArch64TLBI::TLBI TLBI(
        TLBIorig->Name, TLBIorig->Encoding | (HasnXSQualifier ? (1 << 7) : 0),
        TLBIorig->NeedsReg,
        HasnXSQualifier
            ? TLBIorig->FeaturesRequired | FeatureBitset({AArch64::FeatureXS})
            : TLBIorig->FeaturesRequired);
    if (!TLBI.haveFeatures(getSTI().getFeatureBits())) {
      std::string Name =
          std::string(TLBI.Name) + (HasnXSQualifier ? "nXS" : "");
      std::string Str("TLBIP " + Name + " requires: ");
      setRequiredFeatureString(TLBI.getRequiredFeatures(), Str);
      return TokError(Str);
    }
    createSysAlias(TLBI.Encoding, Operands, S);
  }
 
  Lex(); // Eat operand.
 
  if (parseComma())
    return true;
 
  if (Tok.isNot(AsmToken::Identifier))
    return TokError("expected register identifier");
  auto Result = tryParseSyspXzrPair(Operands);
  if (Result.isNoMatch())
    Result = tryParseGPRSeqPair(Operands);
  if (!Result.isSuccess())
    return TokError("specified " + Mnemonic +
                    " op requires a pair of registers");
 
  if (parseToken(AsmToken::EndOfStatement, "unexpected token in argument list"))
    return true;
 
  return false;
}
 
ParseStatus AArch64AsmParser::tryParseBarrierOperand(OperandVector &Operands) {
  MCAsmParser &Parser = getParser();
  const AsmToken &Tok = getTok();
 
  if (Mnemonic == "tsb" && Tok.isNot(AsmToken::Identifier))
    return TokError("'csync' operand expected");
  if (parseOptionalToken(AsmToken::Hash) || Tok.is(AsmToken::Integer)) {
    // Immediate operand.
    const MCExpr *ImmVal;
    SMLoc ExprLoc = getLoc();
    AsmToken IntTok = Tok;
    if (getParser().parseExpression(ImmVal))
      return ParseStatus::Failure;
    const MCConstantExpr *MCE = dyn_cast<MCConstantExpr>(ImmVal);
    if (!MCE)
      return Error(ExprLoc, "immediate value expected for barrier operand");
    int64_t Value = MCE->getValue();
    if (Mnemonic == "dsb" && Value > 15) {
      // This case is a no match here, but it might be matched by the nXS
      // variant. Deliberately not unlex the optional '#' as it is not necessary
      // to characterize an integer immediate.
      Parser.getLexer().UnLex(IntTok);
      return ParseStatus::NoMatch;
    }
    if (Value < 0 || Value > 15)
      return Error(ExprLoc, "barrier operand out of range");
    auto DB = AArch64DB::lookupDBByEncoding(Value);
    Operands.push_back(AArch64Operand::CreateBarrier(Value, DB ? DB->Name : "",
                                                     ExprLoc, getContext(),
                                                     false /*hasnXSModifier*/));
    return ParseStatus::Success;
  }
 
  if (Tok.isNot(AsmToken::Identifier))
    return TokError("invalid operand for instruction");
 
  StringRef Operand = Tok.getString();
  auto TSB = AArch64TSB::lookupTSBByName(Operand);
  auto DB = AArch64DB::lookupDBByName(Operand);
  // The only valid named option for ISB is 'sy'
  if (Mnemonic == "isb" && (!DB || DB->Encoding != AArch64DB::sy))
    return TokError("'sy' or #imm operand expected");
  // The only valid named option for TSB is 'csync'
  if (Mnemonic == "tsb" && (!TSB || TSB->Encoding != AArch64TSB::csync))
    return TokError("'csync' operand expected");
  if (!DB && !TSB) {
    if (Mnemonic == "dsb") {
      // This case is a no match here, but it might be matched by the nXS
      // variant.
      return ParseStatus::NoMatch;
    }
    return TokError("invalid barrier option name");
  }
 
  Operands.push_back(AArch64Operand::CreateBarrier(
      DB ? DB->Encoding : TSB->Encoding, Tok.getString(), getLoc(),
      getContext(), false /*hasnXSModifier*/));
  Lex(); // Consume the option
 
  return ParseStatus::Success;
}
 
ParseStatus
AArch64AsmParser::tryParseBarriernXSOperand(OperandVector &Operands) {
  const AsmToken &Tok = getTok();
 
  assert(Mnemonic == "dsb" && "Instruction does not accept nXS operands");
  if (Mnemonic != "dsb")
    return ParseStatus::Failure;
 
  if (parseOptionalToken(AsmToken::Hash) || Tok.is(AsmToken::Integer)) {
    // Immediate operand.
    const MCExpr *ImmVal;
    SMLoc ExprLoc = getLoc();
    if (getParser().parseExpression(ImmVal))
      return ParseStatus::Failure;
    const MCConstantExpr *MCE = dyn_cast<MCConstantExpr>(ImmVal);
    if (!MCE)
      return Error(ExprLoc, "immediate value expected for barrier operand");
    int64_t Value = MCE->getValue();
    // v8.7-A DSB in the nXS variant accepts only the following immediate
    // values: 16, 20, 24, 28.
    if (Value != 16 && Value != 20 && Value != 24 && Value != 28)
      return Error(ExprLoc, "barrier operand out of range");
    auto DB = AArch64DBnXS::lookupDBnXSByImmValue(Value);
    Operands.push_back(AArch64Operand::CreateBarrier(DB->Encoding, DB->Name,
                                                     ExprLoc, getContext(),
                                                     true /*hasnXSModifier*/));
    return ParseStatus::Success;
  }
 
  if (Tok.isNot(AsmToken::Identifier))
    return TokError("invalid operand for instruction");
 
  StringRef Operand = Tok.getString();
  auto DB = AArch64DBnXS::lookupDBnXSByName(Operand);
 
  if (!DB)
    return TokError("invalid barrier option name");
 
  Operands.push_back(
      AArch64Operand::CreateBarrier(DB->Encoding, Tok.getString(), getLoc(),
                                    getContext(), true /*hasnXSModifier*/));
  Lex(); // Consume the option
 
  return ParseStatus::Success;
}
 
ParseStatus AArch64AsmParser::tryParseSysReg(OperandVector &Operands) {
  const AsmToken &Tok = getTok();
 
  if (Tok.isNot(AsmToken::Identifier))
    return ParseStatus::NoMatch;
 
  if (AArch64SVCR::lookupSVCRByName(Tok.getString()))
    return ParseStatus::NoMatch;
 
  int MRSReg, MSRReg;
  auto SysReg = AArch64SysReg::lookupSysRegByName(Tok.getString());
  if (SysReg && SysReg->haveFeatures(getSTI().getFeatureBits())) {
    MRSReg = SysReg->Readable ? SysReg->Encoding : -1;
    MSRReg = SysReg->Writeable ? SysReg->Encoding : -1;
  } else
    MRSReg = MSRReg = AArch64SysReg::parseGenericRegister(Tok.getString());
 
  unsigned PStateImm = -1;
  auto PState15 = AArch64PState::lookupPStateImm0_15ByName(Tok.getString());
  if (PState15 && PState15->haveFeatures(getSTI().getFeatureBits()))
    PStateImm = PState15->Encoding;
  if (!PState15) {
    auto PState1 = AArch64PState::lookupPStateImm0_1ByName(Tok.getString());
    if (PState1 && PState1->haveFeatures(getSTI().getFeatureBits()))
      PStateImm = PState1->Encoding;
  }
 
  Operands.push_back(
      AArch64Operand::CreateSysReg(Tok.getString(), getLoc(), MRSReg, MSRReg,
                                   PStateImm, getContext()));
  Lex(); // Eat identifier
 
  return ParseStatus::Success;
}
 
ParseStatus
AArch64AsmParser::tryParsePHintInstOperand(OperandVector &Operands) {
  SMLoc S = getLoc();
  const AsmToken &Tok = getTok();
  if (Tok.isNot(AsmToken::Identifier))
    return TokError("invalid operand for instruction");
 
  auto PH = AArch64PHint::lookupPHintByName(Tok.getString());
  if (!PH)
    return TokError("invalid operand for instruction");
 
  Operands.push_back(AArch64Operand::CreatePHintInst(
      PH->Encoding, Tok.getString(), S, getContext()));
  Lex(); // Eat identifier token.
  return ParseStatus::Success;
}
 
/// tryParseNeonVectorRegister - Parse a vector register operand.
bool AArch64AsmParser::tryParseNeonVectorRegister(OperandVector &Operands) {
  if (getTok().isNot(AsmToken::Identifier))
    return true;
 
  SMLoc S = getLoc();
  // Check for a vector register specifier first.
  StringRef Kind;
  MCRegister Reg;
  ParseStatus Res = tryParseVectorRegister(Reg, Kind, RegKind::NeonVector);
  if (!Res.isSuccess())
    return true;
 
  const auto &KindRes = parseVectorKind(Kind, RegKind::NeonVector);
  if (!KindRes)
    return true;
 
  unsigned ElementWidth = KindRes->second;
  Operands.push_back(
      AArch64Operand::CreateVectorReg(Reg, RegKind::NeonVector, ElementWidth,
                                      S, getLoc(), getContext()));
 
  // If there was an explicit qualifier, that goes on as a literal text
  // operand.
  if (!Kind.empty())
    Operands.push_back(AArch64Operand::CreateToken(Kind, S, getContext()));
 
  return tryParseVectorIndex(Operands).isFailure();
}
 
ParseStatus AArch64AsmParser::tryParseVectorIndex(OperandVector &Operands) {
  SMLoc SIdx = getLoc();
  if (parseOptionalToken(AsmToken::LBrac)) {
    const MCExpr *ImmVal;
    if (getParser().parseExpression(ImmVal))
      return ParseStatus::NoMatch;
    const MCConstantExpr *MCE = dyn_cast<MCConstantExpr>(ImmVal);
    if (!MCE)
      return TokError("immediate value expected for vector index");
 
    SMLoc E = getLoc();
 
    if (parseToken(AsmToken::RBrac, "']' expected"))
      return ParseStatus::Failure;
 
    Operands.push_back(AArch64Operand::CreateVectorIndex(MCE->getValue(), SIdx,
                                                         E, getContext()));
    return ParseStatus::Success;
  }
 
  return ParseStatus::NoMatch;
}
 
// tryParseVectorRegister - Try to parse a vector register name with
// optional kind specifier. If it is a register specifier, eat the token
// and return it.
ParseStatus AArch64AsmParser::tryParseVectorRegister(MCRegister &Reg,
                                                     StringRef &Kind,
                                                     RegKind MatchKind) {
  const AsmToken &Tok = getTok();
 
  if (Tok.isNot(AsmToken::Identifier))
    return ParseStatus::NoMatch;
 
  StringRef Name = Tok.getString();
  // If there is a kind specifier, it's separated from the register name by
  // a '.'.
  size_t Start = 0, Next = Name.find('.');
  StringRef Head = Name.slice(Start, Next);
  unsigned RegNum = matchRegisterNameAlias(Head, MatchKind);
 
  if (RegNum) {
    if (Next != StringRef::npos) {
      Kind = Name.substr(Next);
      if (!isValidVectorKind(Kind, MatchKind))
        return TokError("invalid vector kind qualifier");
    }
    Lex(); // Eat the register token.
 
    Reg = RegNum;
    return ParseStatus::Success;
  }
 
  return ParseStatus::NoMatch;
}
 
ParseStatus AArch64AsmParser::tryParseSVEPredicateOrPredicateAsCounterVector(
    OperandVector &Operands) {
  ParseStatus Status =
      tryParseSVEPredicateVector<RegKind::SVEPredicateAsCounter>(Operands);
  if (!Status.isSuccess())
    Status = tryParseSVEPredicateVector<RegKind::SVEPredicateVector>(Operands);
  return Status;
}
 
/// tryParseSVEPredicateVector - Parse a SVE predicate register operand.
template <RegKind RK>
ParseStatus
AArch64AsmParser::tryParseSVEPredicateVector(OperandVector &Operands) {
  // Check for a SVE predicate register specifier first.
  const SMLoc S = getLoc();
  StringRef Kind;
  MCRegister RegNum;
  auto Res = tryParseVectorRegister(RegNum, Kind, RK);
  if (!Res.isSuccess())
    return Res;
 
  const auto &KindRes = parseVectorKind(Kind, RK);
  if (!KindRes)
    return ParseStatus::NoMatch;
 
  unsigned ElementWidth = KindRes->second;
  Operands.push_back(AArch64Operand::CreateVectorReg(
      RegNum, RK, ElementWidth, S,
      getLoc(), getContext()));
 
  if (getLexer().is(AsmToken::LBrac)) {
    if (RK == RegKind::SVEPredicateAsCounter) {
      ParseStatus ResIndex = tryParseVectorIndex(Operands);
      if (ResIndex.isSuccess())
        return ParseStatus::Success;
    } else {
      // Indexed predicate, there's no comma so try parse the next operand
      // immediately.
      if (parseOperand(Operands, false, false))
        return ParseStatus::NoMatch;
    }
  }
 
  // Not all predicates are followed by a '/m' or '/z'.
  if (getTok().isNot(AsmToken::Slash))
    return ParseStatus::Success;
 
  // But when they do they shouldn't have an element type suffix.
  if (!Kind.empty())
    return Error(S, "not expecting size suffix");
 
  // Add a literal slash as operand
  Operands.push_back(AArch64Operand::CreateToken("/", getLoc(), getContext()));
 
  Lex(); // Eat the slash.
 
  // Zeroing or merging?
  auto Pred = getTok().getString().lower();
  if (RK == RegKind::SVEPredicateAsCounter && Pred != "z")
    return Error(getLoc(), "expecting 'z' predication");
 
  if (RK == RegKind::SVEPredicateVector && Pred != "z" && Pred != "m")
    return Error(getLoc(), "expecting 'm' or 'z' predication");
 
  // Add zero/merge token.
  const char *ZM = Pred == "z" ? "z" : "m";
  Operands.push_back(AArch64Operand::CreateToken(ZM, getLoc(), getContext()));
 
  Lex(); // Eat zero/merge token.
  return ParseStatus::Success;
}
 
/// parseRegister - Parse a register operand.
bool AArch64AsmParser::parseRegister(OperandVector &Operands) {
  // Try for a Neon vector register.
  if (!tryParseNeonVectorRegister(Operands))
    return false;
 
  if (tryParseZTOperand(Operands).isSuccess())
    return false;
 
  // Otherwise try for a scalar register.
  if (tryParseGPROperand<false>(Operands).isSuccess())
    return false;
 
  return true;
}
 
bool AArch64AsmParser::parseSymbolicImmVal(const MCExpr *&ImmVal) {
  bool HasELFModifier = false;
  AArch64MCExpr::VariantKind RefKind;
 
  if (parseOptionalToken(AsmToken::Colon)) {
    HasELFModifier = true;
 
    if (getTok().isNot(AsmToken::Identifier))
      return TokError("expect relocation specifier in operand after ':'");
 
    std::string LowerCase = getTok().getIdentifier().lower();
    RefKind =
        StringSwitch<AArch64MCExpr::VariantKind>(LowerCase)
            .Case("lo12", AArch64MCExpr::VK_LO12)
            .Case("abs_g3", AArch64MCExpr::VK_ABS_G3)
            .Case("abs_g2", AArch64MCExpr::VK_ABS_G2)
            .Case("abs_g2_s", AArch64MCExpr::VK_ABS_G2_S)
            .Case("abs_g2_nc", AArch64MCExpr::VK_ABS_G2_NC)
            .Case("abs_g1", AArch64MCExpr::VK_ABS_G1)
            .Case("abs_g1_s", AArch64MCExpr::VK_ABS_G1_S)
            .Case("abs_g1_nc", AArch64MCExpr::VK_ABS_G1_NC)
            .Case("abs_g0", AArch64MCExpr::VK_ABS_G0)
            .Case("abs_g0_s", AArch64MCExpr::VK_ABS_G0_S)
            .Case("abs_g0_nc", AArch64MCExpr::VK_ABS_G0_NC)
            .Case("prel_g3", AArch64MCExpr::VK_PREL_G3)
            .Case("prel_g2", AArch64MCExpr::VK_PREL_G2)
            .Case("prel_g2_nc", AArch64MCExpr::VK_PREL_G2_NC)
            .Case("prel_g1", AArch64MCExpr::VK_PREL_G1)
            .Case("prel_g1_nc", AArch64MCExpr::VK_PREL_G1_NC)
            .Case("prel_g0", AArch64MCExpr::VK_PREL_G0)
            .Case("prel_g0_nc", AArch64MCExpr::VK_PREL_G0_NC)
            .Case("dtprel_g2", AArch64MCExpr::VK_DTPREL_G2)
            .Case("dtprel_g1", AArch64MCExpr::VK_DTPREL_G1)
            .Case("dtprel_g1_nc", AArch64MCExpr::VK_DTPREL_G1_NC)
            .Case("dtprel_g0", AArch64MCExpr::VK_DTPREL_G0)
            .Case("dtprel_g0_nc", AArch64MCExpr::VK_DTPREL_G0_NC)
            .Case("dtprel_hi12", AArch64MCExpr::VK_DTPREL_HI12)
            .Case("dtprel_lo12", AArch64MCExpr::VK_DTPREL_LO12)
            .Case("dtprel_lo12_nc", AArch64MCExpr::VK_DTPREL_LO12_NC)
            .Case("pg_hi21_nc", AArch64MCExpr::VK_ABS_PAGE_NC)
            .Case("tprel_g2", AArch64MCExpr::VK_TPREL_G2)
            .Case("tprel_g1", AArch64MCExpr::VK_TPREL_G1)
            .Case("tprel_g1_nc", AArch64MCExpr::VK_TPREL_G1_NC)
            .Case("tprel_g0", AArch64MCExpr::VK_TPREL_G0)
            .Case("tprel_g0_nc", AArch64MCExpr::VK_TPREL_G0_NC)
            .Case("tprel_hi12", AArch64MCExpr::VK_TPREL_HI12)
            .Case("tprel_lo12", AArch64MCExpr::VK_TPREL_LO12)
            .Case("tprel_lo12_nc", AArch64MCExpr::VK_TPREL_LO12_NC)
            .Case("tlsdesc_lo12", AArch64MCExpr::VK_TLSDESC_LO12)
            .Case("tlsdesc_auth_lo12", AArch64MCExpr::VK_TLSDESC_AUTH_LO12)
            .Case("got", AArch64MCExpr::VK_GOT_PAGE)
            .Case("gotpage_lo15", AArch64MCExpr::VK_GOT_PAGE_LO15)
            .Case("got_lo12", AArch64MCExpr::VK_GOT_LO12)
            .Case("got_auth", AArch64MCExpr::VK_GOT_AUTH_PAGE)
            .Case("got_auth_lo12", AArch64MCExpr::VK_GOT_AUTH_LO12)
            .Case("gottprel", AArch64MCExpr::VK_GOTTPREL_PAGE)
            .Case("gottprel_lo12", AArch64MCExpr::VK_GOTTPREL_LO12_NC)
            .Case("gottprel_g1", AArch64MCExpr::VK_GOTTPREL_G1)
            .Case("gottprel_g0_nc", AArch64MCExpr::VK_GOTTPREL_G0_NC)
            .Case("tlsdesc", AArch64MCExpr::VK_TLSDESC_PAGE)
            .Case("tlsdesc_auth", AArch64MCExpr::VK_TLSDESC_AUTH_PAGE)
            .Case("secrel_lo12", AArch64MCExpr::VK_SECREL_LO12)
            .Case("secrel_hi12", AArch64MCExpr::VK_SECREL_HI12)
            .Default(AArch64MCExpr::VK_INVALID);
 
    if (RefKind == AArch64MCExpr::VK_INVALID)
      return TokError("expect relocation specifier in operand after ':'");
 
    Lex(); // Eat identifier
 
    if (parseToken(AsmToken::Colon, "expect ':' after relocation specifier"))
      return true;
  }
 
  if (getParser().parseExpression(ImmVal))
    return true;
 
  if (HasELFModifier)
    ImmVal = AArch64MCExpr::create(ImmVal, RefKind, getContext());
 
  return false;
}
 
ParseStatus AArch64AsmParser::tryParseMatrixTileList(OperandVector &Operands) {
  if (getTok().isNot(AsmToken::LCurly))
    return ParseStatus::NoMatch;
 
  auto ParseMatrixTile = [this](unsigned &Reg,
                                unsigned &ElementWidth) -> ParseStatus {
    StringRef Name = getTok().getString();
    size_t DotPosition = Name.find('.');
    if (DotPosition == StringRef::npos)
      return ParseStatus::NoMatch;
 
    unsigned RegNum = matchMatrixTileListRegName(Name);
    if (!RegNum)
      return ParseStatus::NoMatch;
 
    StringRef Tail = Name.drop_front(DotPosition);
    const std::optional<std::pair<int, int>> &KindRes =
        parseVectorKind(Tail, RegKind::Matrix);
    if (!KindRes)
      return TokError(
          "Expected the register to be followed by element width suffix");
    ElementWidth = KindRes->second;
    Reg = RegNum;
    Lex(); // Eat the register.
    return ParseStatus::Success;
  };
 
  SMLoc S = getLoc();
  auto LCurly = getTok();
  Lex(); // Eat left bracket token.
 
  // Empty matrix list
  if (parseOptionalToken(AsmToken::RCurly)) {
    Operands.push_back(AArch64Operand::CreateMatrixTileList(
        /*RegMask=*/0, S, getLoc(), getContext()));
    return ParseStatus::Success;
  }
 
  // Try parse {za} alias early
  if (getTok().getString().equals_insensitive("za")) {
    Lex(); // Eat 'za'
 
    if (parseToken(AsmToken::RCurly, "'}' expected"))
      return ParseStatus::Failure;
 
    Operands.push_back(AArch64Operand::CreateMatrixTileList(
        /*RegMask=*/0xFF, S, getLoc(), getContext()));
    return ParseStatus::Success;
  }
 
  SMLoc TileLoc = getLoc();
 
  unsigned FirstReg, ElementWidth;
  auto ParseRes = ParseMatrixTile(FirstReg, ElementWidth);
  if (!ParseRes.isSuccess()) {
    getLexer().UnLex(LCurly);
    return ParseRes;
  }
 
  const MCRegisterInfo *RI = getContext().getRegisterInfo();
 
  unsigned PrevReg = FirstReg;
 
  SmallSet<unsigned, 8> DRegs;
  AArch64Operand::ComputeRegsForAlias(FirstReg, DRegs, ElementWidth);
 
  SmallSet<unsigned, 8> SeenRegs;
  SeenRegs.insert(FirstReg);
 
  while (parseOptionalToken(AsmToken::Comma)) {
    TileLoc = getLoc();
    unsigned Reg, NextElementWidth;
    ParseRes = ParseMatrixTile(Reg, NextElementWidth);
    if (!ParseRes.isSuccess())
      return ParseRes;
 
    // Element size must match on all regs in the list.
    if (ElementWidth != NextElementWidth)
      return Error(TileLoc, "mismatched register size suffix");
 
    if (RI->getEncodingValue(Reg) <= (RI->getEncodingValue(PrevReg)))
      Warning(TileLoc, "tile list not in ascending order");
 
    if (SeenRegs.contains(Reg))
      Warning(TileLoc, "duplicate tile in list");
    else {
      SeenRegs.insert(Reg);
      AArch64Operand::ComputeRegsForAlias(Reg, DRegs, ElementWidth);
    }
 
    PrevReg = Reg;
  }
 
  if (parseToken(AsmToken::RCurly, "'}' expected"))
    return ParseStatus::Failure;
 
  unsigned RegMask = 0;
  for (auto Reg : DRegs)
    RegMask |= 0x1 << (RI->getEncodingValue(Reg) -
                       RI->getEncodingValue(AArch64::ZAD0));
  Operands.push_back(
      AArch64Operand::CreateMatrixTileList(RegMask, S, getLoc(), getContext()));
 
  return ParseStatus::Success;
}
 
template <RegKind VectorKind>
ParseStatus AArch64AsmParser::tryParseVectorList(OperandVector &Operands,
                                                 bool ExpectMatch) {
  MCAsmParser &Parser = getParser();
  if (!getTok().is(AsmToken::LCurly))
    return ParseStatus::NoMatch;
 
  // Wrapper around parse function
  auto ParseVector = [this](MCRegister &Reg, StringRef &Kind, SMLoc Loc,
                            bool NoMatchIsError) -> ParseStatus {
    auto RegTok = getTok();
    auto ParseRes = tryParseVectorRegister(Reg, Kind, VectorKind);
    if (ParseRes.isSuccess()) {
      if (parseVectorKind(Kind, VectorKind))
        return ParseRes;
      llvm_unreachable("Expected a valid vector kind");
    }
 
    if (RegTok.is(AsmToken::Identifier) && ParseRes.isNoMatch() &&
        RegTok.getString().equals_insensitive("zt0"))
      return ParseStatus::NoMatch;
 
    if (RegTok.isNot(AsmToken::Identifier) || ParseRes.isFailure() ||
        (ParseRes.isNoMatch() && NoMatchIsError &&
         !RegTok.getString().starts_with_insensitive("za")))
      return Error(Loc, "vector register expected");
 
    return ParseStatus::NoMatch;
  };
 
  int NumRegs = getNumRegsForRegKind(VectorKind);
  SMLoc S = getLoc();
  auto LCurly = getTok();
  Lex(); // Eat left bracket token.
 
  StringRef Kind;
  MCRegister FirstReg;
  auto ParseRes = ParseVector(FirstReg, Kind, getLoc(), ExpectMatch);
 
  // Put back the original left bracket if there was no match, so that
  // different types of list-operands can be matched (e.g. SVE, Neon).
  if (ParseRes.isNoMatch())
    Parser.getLexer().UnLex(LCurly);
 
  if (!ParseRes.isSuccess())
    return ParseRes;
 
  int64_t PrevReg = FirstReg;
  unsigned Count = 1;
 
  int Stride = 1;
  if (parseOptionalToken(AsmToken::Minus)) {
    SMLoc Loc = getLoc();
    StringRef NextKind;
 
    MCRegister Reg;
    ParseRes = ParseVector(Reg, NextKind, getLoc(), true);
    if (!ParseRes.isSuccess())
      return ParseRes;
 
    // Any Kind suffices must match on all regs in the list.
    if (Kind != NextKind)
      return Error(Loc, "mismatched register size suffix");
 
    unsigned Space =
        (PrevReg < Reg) ? (Reg - PrevReg) : (Reg + NumRegs - PrevReg);
 
    if (Space == 0 || Space > 3)
      return Error(Loc, "invalid number of vectors");
 
    Count += Space;
  }
  else {
    bool HasCalculatedStride = false;
    while (parseOptionalToken(AsmToken::Comma)) {
      SMLoc Loc = getLoc();
      StringRef NextKind;
      MCRegister Reg;
      ParseRes = ParseVector(Reg, NextKind, getLoc(), true);
      if (!ParseRes.isSuccess())
        return ParseRes;
 
      // Any Kind suffices must match on all regs in the list.
      if (Kind != NextKind)
        return Error(Loc, "mismatched register size suffix");
 
      unsigned RegVal = getContext().getRegisterInfo()->getEncodingValue(Reg);
      unsigned PrevRegVal =
          getContext().getRegisterInfo()->getEncodingValue(PrevReg);
      if (!HasCalculatedStride) {
        Stride = (PrevRegVal < RegVal) ? (RegVal - PrevRegVal)
                                       : (RegVal + NumRegs - PrevRegVal);
        HasCalculatedStride = true;
      }
 
      // Register must be incremental (with a wraparound at last register).
      if (Stride == 0 || RegVal != ((PrevRegVal + Stride) % NumRegs))
        return Error(Loc, "registers must have the same sequential stride");
 
      PrevReg = Reg;
      ++Count;
    }
  }
 
  if (parseToken(AsmToken::RCurly, "'}' expected"))
    return ParseStatus::Failure;
 
  if (Count > 4)
    return Error(S, "invalid number of vectors");
 
  unsigned NumElements = 0;
  unsigned ElementWidth = 0;
  if (!Kind.empty()) {
    if (const auto &VK = parseVectorKind(Kind, VectorKind))
      std::tie(NumElements, ElementWidth) = *VK;
  }
 
  Operands.push_back(AArch64Operand::CreateVectorList(
      FirstReg, Count, Stride, NumElements, ElementWidth, VectorKind, S,
      getLoc(), getContext()));
 
  return ParseStatus::Success;
}
 
/// parseNeonVectorList - Parse a vector list operand for AdvSIMD instructions.
bool AArch64AsmParser::parseNeonVectorList(OperandVector &Operands) {
  auto ParseRes = tryParseVectorList<RegKind::NeonVector>(Operands, true);
  if (!ParseRes.isSuccess())
    return true;
 
  return tryParseVectorIndex(Operands).isFailure();
}
 
ParseStatus AArch64AsmParser::tryParseGPR64sp0Operand(OperandVector &Operands) {
  SMLoc StartLoc = getLoc();
 
  MCRegister RegNum;
  ParseStatus Res = tryParseScalarRegister(RegNum);
  if (!Res.isSuccess())
    return Res;
 
  if (!parseOptionalToken(AsmToken::Comma)) {
    Operands.push_back(AArch64Operand::CreateReg(
        RegNum, RegKind::Scalar, StartLoc, getLoc(), getContext()));
    return ParseStatus::Success;
  }
 
  parseOptionalToken(AsmToken::Hash);
 
  if (getTok().isNot(AsmToken::Integer))
    return Error(getLoc(), "index must be absent or #0");
 
  const MCExpr *ImmVal;
  if (getParser().parseExpression(ImmVal) || !isa<MCConstantExpr>(ImmVal) ||
      cast<MCConstantExpr>(ImmVal)->getValue() != 0)
    return Error(getLoc(), "index must be absent or #0");
 
  Operands.push_back(AArch64Operand::CreateReg(
      RegNum, RegKind::Scalar, StartLoc, getLoc(), getContext()));
  return ParseStatus::Success;
}
 
ParseStatus AArch64AsmParser::tryParseZTOperand(OperandVector &Operands) {
  SMLoc StartLoc = getLoc();
  const AsmToken &Tok = getTok();
  std::string Name = Tok.getString().lower();
 
  unsigned RegNum = matchRegisterNameAlias(Name, RegKind::LookupTable);
 
  if (RegNum == 0)
    return ParseStatus::NoMatch;
 
  Operands.push_back(AArch64Operand::CreateReg(
      RegNum, RegKind::LookupTable, StartLoc, getLoc(), getContext()));
  Lex(); // Eat register.
 
  // Check if register is followed by an index
  if (parseOptionalToken(AsmToken::LBrac)) {
    Operands.push_back(
        AArch64Operand::CreateToken("[", getLoc(), getContext()));
    const MCExpr *ImmVal;
    if (getParser().parseExpression(ImmVal))
      return ParseStatus::NoMatch;
    const MCConstantExpr *MCE = dyn_cast<MCConstantExpr>(ImmVal);
    if (!MCE)
      return TokError("immediate value expected for vector index");
    Operands.push_back(AArch64Operand::CreateImm(
        MCConstantExpr::create(MCE->getValue(), getContext()), StartLoc,
        getLoc(), getContext()));
    if (parseOptionalToken(AsmToken::Comma))
      if (parseOptionalMulOperand(Operands))
        return ParseStatus::Failure;
    if (parseToken(AsmToken::RBrac, "']' expected"))
      return ParseStatus::Failure;
    Operands.push_back(
        AArch64Operand::CreateToken("]", getLoc(), getContext()));
  }
  return ParseStatus::Success;
}
 
template <bool ParseShiftExtend, RegConstraintEqualityTy EqTy>
ParseStatus AArch64AsmParser::tryParseGPROperand(OperandVector &Operands) {
  SMLoc StartLoc = getLoc();
 
  MCRegister RegNum;
  ParseStatus Res = tryParseScalarRegister(RegNum);
  if (!Res.isSuccess())
    return Res;
 
  // No shift/extend is the default.
  if (!ParseShiftExtend || getTok().isNot(AsmToken::Comma)) {
    Operands.push_back(AArch64Operand::CreateReg(
        RegNum, RegKind::Scalar, StartLoc, getLoc(), getContext(), EqTy));
    return ParseStatus::Success;
  }
 
  // Eat the comma
  Lex();
 
  // Match the shift
  SmallVector<std::unique_ptr<MCParsedAsmOperand>, 1> ExtOpnd;
  Res = tryParseOptionalShiftExtend(ExtOpnd);
  if (!Res.isSuccess())
    return Res;
 
  auto Ext = static_cast<AArch64Operand*>(ExtOpnd.back().get());
  Operands.push_back(AArch64Operand::CreateReg(
      RegNum, RegKind::Scalar, StartLoc, Ext->getEndLoc(), getContext(), EqTy,
      Ext->getShiftExtendType(), Ext->getShiftExtendAmount(),
      Ext->hasShiftExtendAmount()));
 
  return ParseStatus::Success;
}
 
bool AArch64AsmParser::parseOptionalMulOperand(OperandVector &Operands) {
  MCAsmParser &Parser = getParser();
 
  // Some SVE instructions have a decoration after the immediate, i.e.
  // "mul vl". We parse them here and add tokens, which must be present in the
  // asm string in the tablegen instruction.
  bool NextIsVL =
      Parser.getLexer().peekTok().getString().equals_insensitive("vl");
  bool NextIsHash = Parser.getLexer().peekTok().is(AsmToken::Hash);
  if (!getTok().getString().equals_insensitive("mul") ||
      !(NextIsVL || NextIsHash))
    return true;
 
  Operands.push_back(
      AArch64Operand::CreateToken("mul", getLoc(), getContext()));
  Lex(); // Eat the "mul"
 
  if (NextIsVL) {
    Operands.push_back(
        AArch64Operand::CreateToken("vl", getLoc(), getContext()));
    Lex(); // Eat the "vl"
    return false;
  }
 
  if (NextIsHash) {
    Lex(); // Eat the #
    SMLoc S = getLoc();
 
    // Parse immediate operand.
    const MCExpr *ImmVal;
    if (!Parser.parseExpression(ImmVal))
      if (const MCConstantExpr *MCE = dyn_cast<MCConstantExpr>(ImmVal)) {
        Operands.push_back(AArch64Operand::CreateImm(
            MCConstantExpr::create(MCE->getValue(), getContext()), S, getLoc(),
            getContext()));
        return false;
      }
  }
 
  return Error(getLoc(), "expected 'vl' or '#<imm>'");
}
 
bool AArch64AsmParser::parseOptionalVGOperand(OperandVector &Operands,
                                              StringRef &VecGroup) {
  MCAsmParser &Parser = getParser();
  auto Tok = Parser.getTok();
  if (Tok.isNot(AsmToken::Identifier))
    return true;
 
  StringRef VG = StringSwitch<StringRef>(Tok.getString().lower())
                     .Case("vgx2", "vgx2")
                     .Case("vgx4", "vgx4")
                     .Default("");
 
  if (VG.empty())
    return true;
 
  VecGroup = VG;
  Parser.Lex(); // Eat vgx[2|4]
  return false;
}
 
bool AArch64AsmParser::parseKeywordOperand(OperandVector &Operands) {
  auto Tok = getTok();
  if (Tok.isNot(AsmToken::Identifier))
    return true;
 
  auto Keyword = Tok.getString();
  Keyword = StringSwitch<StringRef>(Keyword.lower())
                .Case("sm", "sm")
                .Case("za", "za")
                .Default(Keyword);
  Operands.push_back(
      AArch64Operand::CreateToken(Keyword, Tok.getLoc(), getContext()));